| 1 | // codewhale-cu tool schemas — single source of truth for tools/list. |
| 2 | // Every action/observation tool accepts an optional `computer` id; supplying it |
| 3 | // switches the active computer first (switch-by-use is the default model). |
| 4 | const computerParam = { |
| 5 | type: "string", |
| 6 | description: "Computer id to act on. Defaults to the active computer. Providing a different registered id switches to it first (sticky).", |
| 7 | }; |
| 8 | |
| 9 | const strategyParam = { |
| 10 | enum: ["auto", "a11y", "event", "app"], |
| 11 | description: "macOS auto (default): element targets press that exact revalidated element and fail closed, with no coordinate fallback; coordinate targets hit-test the point for an accessibility press, including focus of a field that is not AXPressable. a11y: require an accessibility press or focus and fail closed otherwise. app: if accessibility cannot act, post a pointer event only when the point is inside the bound app's window, then restore the cursor — never a global desktop click. event: force the guarded raw pointer event (shared-desktop / activate:true). Other platforms use raw events. action_sent confirms dispatch, not the effect; observe again before deciding another action.", |
| 12 | }; |
| 13 | |
| 14 | const elementTargetSchema = { |
| 15 | type: "object", |
| 16 | description: "Element target: the flat index from the latest get_app_state on this computer. state_id is optional — supply it only to pin a specific earlier observation.", |
| 17 | required: ["type", "index"], |
| 18 | properties: { |
| 19 | type: { const: "element" }, |
| 20 | state_id: { type: "string" }, |
| 21 | index: { type: "integer", minimum: 0 }, |
| 22 | }, |
| 23 | additionalProperties: false, |
| 24 | }; |
| 25 | |
| 26 | const targetSchema = { |
| 27 | oneOf: [ |
| 28 | elementTargetSchema, |
| 29 | { |
| 30 | type: "object", |
| 31 | description: "Pixel coordinates in the latest returned raster (screenshot or zoom) for this computer.", |
| 32 | required: ["type", "x", "y"], |
| 33 | properties: { |
| 34 | type: { const: "coordinate" }, |
| 35 | x: { type: "integer" }, |
| 36 | y: { type: "integer" }, |
| 37 | space: { enum: ["raster", "screen"], description: "raster (default): pixels in the latest screenshot/OCR/zoom. screen: absolute screen points; do not convert them yourself." }, |
| 38 | }, |
| 39 | additionalProperties: false, |
| 40 | }, |
| 41 | ], |
| 42 | }; |
| 43 | |
| 44 | export const TOOLS = [ |
| 45 | { name: "preview", description: "macOS: show or hide the nonactivating app preview with the drawn agent cursor. On by default while an app is bound — each action updates the captured window and cursor without moving the real pointer. Set enabled:false to mute it for the session.", inputSchema: { type: "object", properties: { enabled: { type: "boolean" }, computer: computerParam }, additionalProperties: false } }, |
| 46 | // ---- computers (switching is a default) ---- |
| 47 | { |
| 48 | name: "computer", description: "The computer registry. action list | switch | register | spawn | remove. switch/register/spawn/remove take `id`; register also takes transport (local|ssh|hdc) plus host/port/user/target/installAgent; spawn takes transport (docker) plus optional image/label and creates a task-owned disposable desktop that remove or session end destroys. Prefer a spawned computer for work that does not need the user's own session. Every other tool also accepts `computer` to switch stickily on use.", |
| 49 | inputSchema: { type: "object", required: ["action"], properties: { action: { enum: ["list", "switch", "register", "spawn", "remove"] }, id: { type: "string", description: "Short id for the registered computer (letters, digits, dot, dash)" }, transport: { enum: ["local", "ssh", "hdc", "docker"] }, label: { type: "string" }, image: { type: "string", description: "spawn/docker: image to run (default the plugin's Linux desktop image)" }, host: { type: "string", description: "ssh: hostname" }, port: { type: "integer", description: "ssh: port (default 22)" }, user: { type: "string", description: "ssh: user" }, target: { type: "string", description: "hdc: target key (omit for the only connected device)" }, installAgent: { type: "boolean", description: "ssh: push the remote agent before first use (default true)" } }, additionalProperties: false }, |
| 50 | }, |
| 51 | { |
| 52 | name: "computer_list", |
| 53 | description: "List registered computers (local, ssh, docker, harmony/hdc) and which one is active. Every other tool acts on the active computer unless given `computer`.", |
| 54 | inputSchema: { type: "object", properties: {}, additionalProperties: false }, |
| 55 | }, |
| 56 | { |
| 57 | name: "computer_switch", |
| 58 | description: "Switch the active computer. Subsequent tools act on it by default.", |
| 59 | inputSchema: { type: "object", required: ["computer"], properties: { computer: { type: "string", description: "Registered computer id (see computer_list)" } }, additionalProperties: false }, |
| 60 | }, |
| 61 | { |
| 62 | name: "computer_register", |
| 63 | description: "Register or update a computer. transport=local (this machine), ssh (runs the bundled remote agent over ssh; agent is pushed automatically), hdc (HarmonyOS device via hdc).", |
| 64 | inputSchema: { |
| 65 | type: "object", |
| 66 | required: ["computer", "transport"], |
| 67 | properties: { |
| 68 | computer: { type: "string", description: "Short id for the computer (letters, digits, dot, dash)" }, |
| 69 | transport: { enum: ["local", "ssh", "hdc"] }, |
| 70 | label: { type: "string" }, |
| 71 | host: { type: "string", description: "ssh: hostname" }, |
| 72 | port: { type: "integer", description: "ssh: port (default 22)" }, |
| 73 | user: { type: "string", description: "ssh: user" }, |
| 74 | target: { type: "string", description: "hdc: target key (omit for the only connected device)" }, |
| 75 | installAgent: { type: "boolean", description: "ssh: push the remote agent before first use (default true)" }, |
| 76 | }, |
| 77 | additionalProperties: false, |
| 78 | }, |
| 79 | }, |
| 80 | { |
| 81 | name: "computer_spawn", |
| 82 | description: "Spawn a task-owned disposable computer. transport=docker provisions an isolated Linux desktop container registered under `computer`; every other tool works on it unchanged. The spawned computer is destroyed by computer_remove or when the session ends. Prefer it over local when the task does not need the user's own session.", |
| 83 | inputSchema: { |
| 84 | type: "object", |
| 85 | required: ["computer", "transport"], |
| 86 | properties: { |
| 87 | computer: { type: "string", description: "Short id for the spawned computer (letters, digits, dot, dash)" }, |
| 88 | transport: { enum: ["docker"] }, |
| 89 | image: { type: "string", description: "docker image (default the plugin's Linux desktop image)" }, |
| 90 | label: { type: "string" }, |
| 91 | }, |
| 92 | additionalProperties: false, |
| 93 | }, |
| 94 | }, |
| 95 | { |
| 96 | name: "computer_remove", |
| 97 | description: "Remove a registered computer. 'local' cannot be removed.", |
| 98 | inputSchema: { type: "object", required: ["computer"], properties: { computer: { type: "string" } }, additionalProperties: false }, |
| 99 | }, |
| 100 | { |
| 101 | name: "consent", |
| 102 | description: "Per-app consent on the local computer. Any call that targets an app — open_application, an app_ref, an element, or an action on the bound app — refuses consent_required until the user decides; record their answer here. action status | allow | deny | revoke. app is a name or bundle id (or pid:/number for a pid); scope 'foreground' is the separate darwin decision for taking the shared pointer (open_application activate:true). Decisions apply to this session; remember:true persists them.", |
| 103 | inputSchema: { |
| 104 | type: "object", |
| 105 | required: ["action"], |
| 106 | properties: { |
| 107 | action: { enum: ["status", "allow", "deny", "revoke"] }, |
| 108 | app: { type: "string", description: "App identity: name ('Safari'), bundle id ('com.apple.Safari'), or pid ('pid:1234')" }, |
| 109 | name: { type: "string" }, bundle_id: { type: "string" }, pid: { type: "integer" }, |
| 110 | scope: { enum: ["app", "foreground"], description: "app (default): consent to use one application. foreground: consent to take the shared pointer/focus (darwin activate:true)" }, |
| 111 | remember: { type: "boolean", description: "Persist the decision across sessions (default: this session only)" }, |
| 112 | computer: computerParam, |
| 113 | }, |
| 114 | additionalProperties: false, |
| 115 | }, |
| 116 | }, |
| 117 | { |
| 118 | name: "consent_status", |
| 119 | description: "List recorded app-consent decisions for a computer (persisted and this session's) plus the foreground decision.", |
| 120 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 121 | }, |
| 122 | { |
| 123 | name: "consent_allow", |
| 124 | description: "Record an allow decision: app (name/bundle_id/pid/app string) or scope:'foreground'. remember:true persists it.", |
| 125 | inputSchema: { type: "object", properties: { computer: computerParam, app: { type: "string" }, name: { type: "string" }, bundle_id: { type: "string" }, pid: { type: "integer" }, scope: { enum: ["app", "foreground"] }, remember: { type: "boolean" } }, additionalProperties: false }, |
| 126 | }, |
| 127 | { |
| 128 | name: "consent_deny", |
| 129 | description: "Record a deny decision: app (name/bundle_id/pid/app string) or scope:'foreground'. remember:true persists it.", |
| 130 | inputSchema: { type: "object", properties: { computer: computerParam, app: { type: "string" }, name: { type: "string" }, bundle_id: { type: "string" }, pid: { type: "integer" }, scope: { enum: ["app", "foreground"] }, remember: { type: "boolean" } }, additionalProperties: false }, |
| 131 | }, |
| 132 | { |
| 133 | name: "consent_revoke", |
| 134 | description: "Remove recorded decisions for an app or scope:'foreground' (session and persisted).", |
| 135 | inputSchema: { type: "object", properties: { computer: computerParam, app: { type: "string" }, name: { type: "string" }, bundle_id: { type: "string" }, pid: { type: "integer" }, scope: { enum: ["app", "foreground"] } }, additionalProperties: false }, |
| 136 | }, |
| 137 | // ---- observe & resolve ---- |
| 138 | { |
| 139 | name: "request_access", |
| 140 | description: "Probe permissions and capabilities of a computer (accessibility, screen capture, recording, missing tools). Call once when readiness is unknown or a permission failure is explicitly named.", |
| 141 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 142 | }, |
| 143 | { |
| 144 | name: "list_displays", |
| 145 | description: "List displays/panels with geometry and pixel scale.", |
| 146 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 147 | }, |
| 148 | { |
| 149 | name: "switch_display", |
| 150 | description: "Set which display subsequent screenshots/recordings capture on this computer.", |
| 151 | inputSchema: { type: "object", required: ["index"], properties: { index: { type: "integer", minimum: 1 }, computer: computerParam }, additionalProperties: false }, |
| 152 | }, |
| 153 | { |
| 154 | name: "list_apps", |
| 155 | description: "List running applications (name, pid, bundle id, frontmost). Defaults to regular user-facing apps; pass all:true to include background agents and helpers (menu-bar extras, XPC services, CLI processes); pass installed:true for the installed catalog of openable apps (running or not, with a running flag) — that scan takes a moment.", |
| 156 | inputSchema: { type: "object", properties: { all: { type: "boolean", description: "Include accessory/background processes, not just regular apps. Use when looking for a menu-bar or helper process; keep the default for picking an app to control." }, installed: { type: "boolean", description: "List installed apps (openable, running or not) from the standard Applications folders instead of running processes." }, computer: computerParam }, additionalProperties: false }, |
| 157 | }, |
| 158 | { |
| 159 | name: "list_windows", |
| 160 | description: "List application windows. On macOS, app_ref selects the app; omission follows the app selected by open_application, or the frontmost app before a selection. Other platforms list all windows and reject app_ref selectors.", |
| 161 | inputSchema: { |
| 162 | type: "object", |
| 163 | properties: { |
| 164 | app_ref: { |
| 165 | type: "object", |
| 166 | properties: { |
| 167 | pid: { type: "integer" }, name: { type: "string" }, bundle_id: { type: "string" }, |
| 168 | }, |
| 169 | additionalProperties: false, |
| 170 | }, |
| 171 | computer: computerParam, |
| 172 | }, |
| 173 | additionalProperties: false, |
| 174 | }, |
| 175 | }, |
| 176 | { |
| 177 | name: "wait_for", |
| 178 | description: "Poll this computer's accessibility state until elements matching query/role appear (state:\"present\", default) or until none remain (state:\"absent\"). Returns the matched elements bound to a fresh state_id, ready to target. Prefer this over a get_app_state/wait loop after actions that load, animate or dismiss UI.", |
| 179 | inputSchema: { |
| 180 | type: "object", |
| 181 | properties: { |
| 182 | query: { type: "string", description: "Case-insensitive substring over label, value and role. At least one of query/role is required." }, |
| 183 | role: { type: "string", description: "Exact accessibility role, e.g. AXButton, AXTextField." }, |
| 184 | state: { enum: ["present", "absent"], default: "present", description: "present: wait until a match exists. absent: wait until no match remains (dialogs dismissed, loading finished)." }, |
| 185 | timeout: { type: "number", minimum: 0.5, maximum: 60, description: "Seconds to poll before giving up; default 10." }, |
| 186 | interval: { type: "integer", minimum: 100, maximum: 5000, description: "Milliseconds between observations; default 400." }, |
| 187 | limit: { type: "integer", minimum: 1, maximum: 100, description: "Max matched elements to return; default 20." }, |
| 188 | app_ref: { type: "object", properties: { pid: { type: "integer" }, name: { type: "string" }, bundle_id: { type: "string" } }, additionalProperties: false, description: "Same selector rules as get_app_state; omission follows the app selected by open_application." }, |
| 189 | window_id: { type: "integer", description: "macOS only: zero-based window index within the app." }, |
| 190 | computer: computerParam, |
| 191 | }, |
| 192 | additionalProperties: false, |
| 193 | }, |
| 194 | }, |
| 195 | { |
| 196 | name: "get_app_state", |
| 197 | description: "Read an application's text, controls, actions and layout without requiring vision. The default summary keeps app content and top-level menus; full adds nested menus and tree structure. Act on observed elements with {type:'element', index} and refresh after UI changes. Missing labels or values are unknown, not an invitation to guess; request a screenshot only when useful.", |
| 198 | inputSchema: { |
| 199 | type: "object", |
| 200 | properties: { |
| 201 | app_ref: { type: "object", properties: { pid: { type: "integer" }, name: { type: "string" }, bundle_id: { type: "string" } }, additionalProperties: false, description: "macOS accepts PID, name and bundle identity. Linux accepts only a unique exact AT-SPI app name. Windows accepts only a unique exact window title in name (from list_windows.title). HarmonyOS rejects explicit app selectors." }, |
| 202 | window_id: { type: "integer", description: "macOS only: zero-based window index within the app. Other platforms reject this selector." }, |
| 203 | detail: { enum: ["summary", "compact", "full"], default: "summary", description: "Summary is the concise default (controls, values, actions, layout). compact is smaller: same indices, shorter labels, no nested menus. full includes nested menus and tree paths." }, |
| 204 | query: { type: "string", description: "Case-insensitive substring over label, value and role. Use this instead of downloading the whole tree." }, |
| 205 | role: { type: "string", description: "Exact accessibility role filter, e.g. AXButton, AXTextField." }, |
| 206 | limit: { type: "integer", minimum: 1, maximum: 200, description: "Max elements to return after filtering. Prefer this over a second unfiltered dump." }, |
| 207 | offset: { type: "integer", minimum: 0, description: "Skip this many matching elements (pagination)." }, |
| 208 | include_ocr: { type: "boolean", default: false, description: "On macOS, also recognize visible text locally from the selected app window. Requires Screen Recording permission. Returns text, confidence and raster coordinate targets for UI that accessibility cannot read; no vision model is required. Do not combine with compact unless you need the blocks." }, |
| 209 | ocr_region: { type: "array", items: { type: "number" }, minItems: 4, maxItems: 4, description: "[x, y, w, h] in screen points. When include_ocr is true, recognize only this rect instead of the whole window." }, |
| 210 | computer: computerParam, |
| 211 | }, |
| 212 | additionalProperties: false, |
| 213 | }, |
| 214 | }, |
| 215 | { |
| 216 | name: "screenshot", |
| 217 | description: "Capture the screen (all or one display, optional region) as PNG/JPEG. The receipt carries raster geometry; later coordinate targets refer to this raster.", |
| 218 | inputSchema: { |
| 219 | type: "object", |
| 220 | properties: { |
| 221 | app_ref: { type: "object", properties: { name: { type: "string" }, bundle_id: { type: "string" }, pid: { type: "integer" } }, description: "macOS: capture this app window even when it is in the background." }, |
| 222 | display: { type: ["integer", "string"], description: "Display index or 'all'" }, |
| 223 | region: { type: "array", items: { type: "number" }, minItems: 4, maxItems: 4, description: "[x, y, w, h] in screen points" }, |
| 224 | path: { type: "string", description: "Optional output path (absolute). Defaults into the recordings directory." }, |
| 225 | computer: computerParam, |
| 226 | }, |
| 227 | additionalProperties: false, |
| 228 | }, |
| 229 | }, |
| 230 | { |
| 231 | name: "zoom", |
| 232 | description: "Close-up crop of the latest screenshot. Choose points from the returned child raster only.", |
| 233 | inputSchema: { |
| 234 | type: "object", |
| 235 | required: ["region"], |
| 236 | properties: { |
| 237 | region: { type: "array", items: { type: "number" }, minItems: 4, maxItems: 4, description: "[x, y, w, h] in last-raster pixels" }, |
| 238 | path: { type: "string" }, |
| 239 | computer: computerParam, |
| 240 | }, |
| 241 | additionalProperties: false, |
| 242 | }, |
| 243 | }, |
| 244 | { |
| 245 | name: "cursor_position", |
| 246 | description: "Read the current pointer position in screen points.", |
| 247 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 248 | }, |
| 249 | { |
| 250 | name: "list_sessions", |
| 251 | description: "List the live computer sessions on this machine: bound target, delivery mode, current action, idle age, and whether any session currently holds a pointer. Read-only and content-free (no task text is ever recorded). Use it to see who else — another model or agent — is driving the computer before you act.", |
| 252 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 253 | }, |
| 254 | { |
| 255 | name: "kill_app", |
| 256 | description: "Quit a running application by exact name, bundle_id or pid. Refuses when several running applications match (pass pid) and never terminates the Computer Use helper itself. force:true force-quits an unresponsive app — unsaved work is discarded.", |
| 257 | inputSchema: { type: "object", properties: { name: { type: "string" }, bundle_id: { type: "string" }, pid: { type: "integer" }, force: { type: "boolean", description: "force-quit when the graceful quit does not complete" }, computer: computerParam }, additionalProperties: false }, |
| 258 | }, |
| 259 | { |
| 260 | name: "browser", |
| 261 | description: "Drive a Chromium-family browser over the DevTools protocol — exact element addressing instead of pixel clicking, in a self-owned profile (the user's own browser is never touched). Actions: start {url?} | status | navigate {url} | click {selector | point} | type {text, selector?, enter?} | screenshot {full?} | stop. Elements are CSS selectors; coordinates are page-viewport pixels from screenshot (never screen points). One tab per session; the last session out closes the browser.", |
| 262 | inputSchema: { |
| 263 | type: "object", required: ["action"], |
| 264 | properties: { |
| 265 | action: { enum: ["start", "status", "navigate", "click", "type", "screenshot", "stop"] }, |
| 266 | url: { type: "string", description: "http(s):// or about:blank (start, navigate)" }, |
| 267 | selector: { type: "string", description: "CSS selector (click, or type focus)" }, |
| 268 | point: { type: "object", properties: { x: { type: "number" }, y: { type: "number" } }, required: ["x", "y"], additionalProperties: false, description: "page-viewport pixels — the browser screenshot space, never screen points" }, |
| 269 | text: { type: "string", description: "text to insert (type)" }, |
| 270 | enter: { type: "boolean", description: "press Enter after typing" }, |
| 271 | full: { type: "boolean", description: "capture the full page, not just the viewport" }, |
| 272 | computer: computerParam, |
| 273 | }, |
| 274 | additionalProperties: false, |
| 275 | }, |
| 276 | }, |
| 277 | { |
| 278 | name: "browser_start", |
| 279 | description: "Launch or reuse the self-owned Chromium profile and open this session's tab. The user's own browser is never touched.", |
| 280 | inputSchema: { type: "object", properties: { url: { type: "string", description: "optional http(s) URL to open" }, computer: computerParam }, additionalProperties: false }, |
| 281 | }, |
| 282 | { |
| 283 | name: "browser_status", |
| 284 | description: "Read the browser session: running, tabs, and the active tab's url/title.", |
| 285 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 286 | }, |
| 287 | { |
| 288 | name: "browser_navigate", |
| 289 | description: "Navigate this session's tab to an http(s) or about:blank URL and wait for load.", |
| 290 | inputSchema: { type: "object", required: ["url"], properties: { url: { type: "string" }, computer: computerParam }, additionalProperties: false }, |
| 291 | }, |
| 292 | { |
| 293 | name: "browser_click", |
| 294 | description: "Click in the page: a CSS selector's box center, or a page-viewport point.", |
| 295 | inputSchema: { type: "object", properties: { selector: { type: "string" }, point: { type: "object", properties: { x: { type: "number" }, y: { type: "number" } }, required: ["x", "y"], additionalProperties: false }, computer: computerParam }, additionalProperties: false }, |
| 296 | }, |
| 297 | { |
| 298 | name: "browser_type", |
| 299 | description: "Insert text into the page (optionally focusing a CSS selector first); enter:true presses Enter.", |
| 300 | inputSchema: { type: "object", required: ["text"], properties: { text: { type: "string" }, selector: { type: "string" }, enter: { type: "boolean" }, computer: computerParam }, additionalProperties: false }, |
| 301 | }, |
| 302 | { |
| 303 | name: "browser_screenshot", |
| 304 | description: "Capture the page (viewport, or the full page with full:true) as a PNG in the recordings dir.", |
| 305 | inputSchema: { type: "object", properties: { full: { type: "boolean" }, computer: computerParam }, additionalProperties: false }, |
| 306 | }, |
| 307 | { |
| 308 | name: "browser_stop", |
| 309 | description: "Close this session's tab; the shared browser closes when no tabs remain.", |
| 310 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 311 | }, |
| 312 | { |
| 313 | name: "trajectory", |
| 314 | description: "Record this session's tool calls to a local JSONL and replay them later. Actions: start | stop | status (file, turns, recent files) | replay {id?, dry_run?} — replay re-enters the normal tool pipeline, so permissions, grants and the kill switch still apply, and it stops at the first refusal. Off unless started; arguments are stored verbatim (typed text included) so replay is faithful; files stay in the recordings dir on this machine.", |
| 315 | inputSchema: { type: "object", required: ["action"], properties: { action: { enum: ["start", "stop", "status", "replay"] }, id: { type: "string", description: "traj-*.jsonl name from status; defaults to the most recent" }, dry_run: { type: "boolean", description: "list what replay would do without executing anything" }, computer: computerParam }, additionalProperties: false }, |
| 316 | }, |
| 317 | { |
| 318 | name: "trajectory_start", |
| 319 | description: "Start recording this session's tool calls to a local JSONL.", |
| 320 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 321 | }, |
| 322 | { |
| 323 | name: "trajectory_stop", |
| 324 | description: "Stop recording and report the file and turn count.", |
| 325 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 326 | }, |
| 327 | { |
| 328 | name: "trajectory_status", |
| 329 | description: "Report whether a trajectory is recording, the file, and recent trajectories.", |
| 330 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 331 | }, |
| 332 | { |
| 333 | name: "trajectory_replay", |
| 334 | description: "Replay a recorded trajectory through the normal tool pipeline, stopping at the first refusal.", |
| 335 | inputSchema: { type: "object", properties: { id: { type: "string" }, dry_run: { type: "boolean" }, computer: computerParam }, additionalProperties: false }, |
| 336 | }, |
| 337 | { |
| 338 | name: "set_window_frame", |
| 339 | description: "Move or resize one window by exact geometry and read the result back. frame is in screen points, the same space list_windows reports: {x,y,w,h}. window_id is the zero-based window index from list_windows. Some windows refuse (fullscreen, tiled); the receipt carries the app's own before/after readback and `verified`.", |
| 340 | inputSchema: { type: "object", required: ["window_id", "frame"], properties: { app_ref: { type: "object", properties: { pid: { type: "integer" }, name: { type: "string" }, bundle_id: { type: "string" } }, additionalProperties: false, description: "defaults to the bound app" }, window_id: { type: "integer", minimum: 0, description: "zero-based window index from list_windows" }, frame: { type: "object", properties: { x: { type: "number" }, y: { type: "number" }, w: { type: "number" }, h: { type: "number" } }, required: ["x", "y", "w", "h"], additionalProperties: false }, computer: computerParam }, additionalProperties: false }, |
| 341 | }, |
| 342 | { |
| 343 | name: "open_application", |
| 344 | description: "Launch or activate an application. Copy user-provided names character-for-character; never translate, normalize, or strip suffixes. On macOS prefer bundle_id when known.", |
| 345 | inputSchema: { |
| 346 | type: "object", |
| 347 | properties: { |
| 348 | name: { type: "string" }, bundle_id: { type: "string" }, url: { type: "string" }, |
| 349 | pid: { type: "integer", description: "Bind to this exact process. Use when two processes share a bundle id (list_apps shows both); it takes precedence over name and bundle_id and never launches anything." }, |
| 350 | activate: { type: "boolean", description: "Bring to foreground; defaults to false — background is the default on every platform. On macOS false keeps process-bound keyboard/accessibility control and refuses shared pointer gestures; on Windows it launches the app minimized; on Linux it restores the previously focused window after launch. True selects shared-desktop control and requires the separate foreground consent; use only when the user has authorized exclusive desktop use. Neither mode is an isolated computer." }, |
| 351 | computer: computerParam, |
| 352 | }, |
| 353 | additionalProperties: false, |
| 354 | }, |
| 355 | }, |
| 356 | // ---- pointer ---- |
| 357 | { |
| 358 | name: "click", description: "Click a target: `button` left/right/middle (left default) and `clicks` 1..3 (left only). Element targets press that exact accessibility element; coordinate targets need a fresh raster. The per-action names (left_click, double_click, right_click, middle_click, triple_click) stay callable as aliases.", |
| 359 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, button: { enum: ["left", "right", "middle"], default: "left" }, clicks: { type: "integer", minimum: 1, maximum: 3, default: 1 }, strategy: strategyParam, computer: computerParam }, additionalProperties: false }, |
| 360 | }, |
| 361 | { |
| 362 | name: "pointer", description: "Raw pointer primitives: action \"move\" (hover without clicking), \"down\" (press and hold), \"up\" (release; target optional — releases at the last point). Background mode refuses these (shared pointer); they exist for explicit shared-desktop work.", |
| 363 | inputSchema: { type: "object", required: ["action"], properties: { action: { enum: ["move", "down", "up"] }, target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 364 | }, |
| 365 | { |
| 366 | name: "left_click", description: "Left-click a coordinate (pixels in the latest raster) or perform an element's press action. macOS background mode uses accessibility and refuses fallbacks that require keyboard focus.", |
| 367 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, strategy: strategyParam, computer: computerParam }, additionalProperties: false }, |
| 368 | }, |
| 369 | { |
| 370 | name: "double_click", description: "Double-click a target.", |
| 371 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 372 | }, |
| 373 | { |
| 374 | name: "triple_click", description: "Triple-click a target (e.g. select a paragraph).", |
| 375 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 376 | }, |
| 377 | { |
| 378 | name: "right_click", description: "Right-click a target (context menu).", |
| 379 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 380 | }, |
| 381 | { |
| 382 | name: "middle_click", description: "Middle-click a target.", |
| 383 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 384 | }, |
| 385 | { |
| 386 | name: "mouse_move", description: "Move the pointer without clicking (hover).", |
| 387 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 388 | }, |
| 389 | { |
| 390 | name: "left_click_drag", description: "Press at from_target, move in steps, release at to. macOS requires explicit foreground control; background mode refuses because window-routed events still take keyboard focus.", |
| 391 | inputSchema: { type: "object", required: ["from_target", "to"], properties: { from_target: targetSchema, to: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 392 | }, |
| 393 | { |
| 394 | name: "left_mouse_down", description: "Press and hold the left button at a target. Release with left_mouse_up.", |
| 395 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 396 | }, |
| 397 | { |
| 398 | name: "left_mouse_up", description: "Release the left button pressed by left_mouse_down. An optional target releases at that point instead of where the button went down.", |
| 399 | inputSchema: { type: "object", properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 400 | }, |
| 401 | { |
| 402 | name: "scroll", description: "Scroll up/down/left/right at a target. macOS background mode uses accessibility scrollbars; amount counts native increments or 5% normalized steps, named in the receipt. It refuses wheel-event fallbacks that take focus. Other raw routes use lines/notches. Prefer an observed scroll-area element.", |
| 403 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, direction: { enum: ["up", "down", "left", "right"] }, amount: { type: "integer", minimum: 1, maximum: 100 }, computer: computerParam }, additionalProperties: false }, |
| 404 | }, |
| 405 | // ---- text & keyboard ---- |
| 406 | { |
| 407 | name: "type", description: "Type unicode text into the focused control. Newlines in `text` are Return/Enter key presses, not literal characters — never put \\n in a composer by hoping it will send. Focus the field first (click, focus, or set_value), or pass an element `target` to focus it in the same call. On macOS the receipt carries `verified:true` only when the focused control's value actually reflects the typed text; on `verified:false` the text may have gone nowhere — observe again before relying on it.", |
| 408 | inputSchema: { type: "object", required: ["text"], properties: { text: { type: "string" }, press_enter: { type: "boolean", description: "After typing, press Return/Enter once. Prefer this to putting a newline in `text` when you want to send." }, target: { ...elementTargetSchema, description: "Element target from get_app_state; it is accessibility-focused first, then the text is typed. Element targets only." }, computer: computerParam }, additionalProperties: false }, |
| 409 | }, |
| 410 | { |
| 411 | name: "key", description: "Press a named key or chord. Examples: return, enter, backspace, tab, escape, cmd+c (macOS), ctrl+c (Linux/Windows). This is the key-press tool; type() cannot send modifiers or Return by itself except via newlines/press_enter. macOS background mode refuses modified or window-targeted keys that need keyboard focus; prefer invoke_menu. Repeat with `repeat`. Pass an element `target` to accessibility-focus it first. `duration` holds the key instead of tapping (hold_key semantics) and cannot be combined with repeat or target.", |
| 412 | inputSchema: { type: "object", required: ["text"], properties: { text: { type: "string" }, repeat: { type: "integer", minimum: 1, maximum: 100 }, duration: { type: "number", minimum: 0.05, maximum: 30, description: "Hold the key for this many seconds instead of tapping." }, target: { ...elementTargetSchema, description: "Element target from get_app_state; it is accessibility-focused first, then the key is sent. Element targets only." }, computer: computerParam }, additionalProperties: false }, |
| 413 | }, |
| 414 | { |
| 415 | name: "hold_key", description: "Hold a key for `duration` seconds (0.05..30).", |
| 416 | inputSchema: { type: "object", required: ["text", "duration"], properties: { text: { type: "string" }, duration: { type: "number", minimum: 0.05, maximum: 30 }, computer: computerParam }, additionalProperties: false }, |
| 417 | }, |
| 418 | { |
| 419 | name: "set_value", description: "Set an editable element's value with readback verification. On macOS native controls use AXValue; web-area replacement requires foreground control and refuses in background mode. Prefer browser control for web fields. Element targets only.", |
| 420 | inputSchema: { type: "object", required: ["target", "value"], properties: { target: elementTargetSchema, value: { type: "string" }, computer: computerParam }, additionalProperties: false }, |
| 421 | }, |
| 422 | { |
| 423 | name: "focus", description: "Focus an observed element through the accessibility layer (background-safe). Prefer this before type() on composers that ignore AXPress.", |
| 424 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 425 | }, |
| 426 | { |
| 427 | name: "get_value", description: "Read the live accessibility value of an observed element (text fields, sliders). Prefer this over dumping the whole tree.", |
| 428 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 429 | }, |
| 430 | { |
| 431 | name: "find_elements", description: "Search the latest get_app_state (or take a fresh one) for elements matching query/role without returning the full dump.", |
| 432 | inputSchema: { |
| 433 | type: "object", |
| 434 | properties: { |
| 435 | query: { type: "string" }, |
| 436 | role: { type: "string" }, |
| 437 | state_id: { type: "string", description: "Reuse a previous observation; omit to observe now." }, |
| 438 | limit: { type: "integer", minimum: 1, maximum: 100 }, |
| 439 | app_ref: { type: "object", properties: { pid: { type: "integer" }, name: { type: "string" }, bundle_id: { type: "string" } }, additionalProperties: false }, |
| 440 | computer: computerParam, |
| 441 | }, |
| 442 | additionalProperties: false, |
| 443 | }, |
| 444 | }, |
| 445 | { |
| 446 | name: "run_actions", description: "Run up to 8 computer-use tools in order on this computer. Stops on the first failure. Each step is {tool, arguments}. Use for click→type→key(return)→get_value without extra round trips.", |
| 447 | inputSchema: { |
| 448 | type: "object", |
| 449 | required: ["steps"], |
| 450 | properties: { |
| 451 | steps: { |
| 452 | type: "array", |
| 453 | minItems: 1, |
| 454 | maxItems: 8, |
| 455 | items: { |
| 456 | type: "object", |
| 457 | required: ["tool"], |
| 458 | properties: { |
| 459 | tool: { type: "string" }, |
| 460 | arguments: { type: "object" }, |
| 461 | }, |
| 462 | additionalProperties: false, |
| 463 | }, |
| 464 | }, |
| 465 | computer: computerParam, |
| 466 | }, |
| 467 | additionalProperties: false, |
| 468 | }, |
| 469 | }, |
| 470 | { |
| 471 | name: "select_text", description: "Select a text range [start, length] in an element, or place the caret when the range is omitted. Element targets only.", |
| 472 | inputSchema: { type: "object", required: ["target"], properties: { target: elementTargetSchema, text_range: { type: "array", items: { type: "integer" }, minItems: 2, maxItems: 2 }, computer: computerParam }, additionalProperties: false }, |
| 473 | }, |
| 474 | { |
| 475 | name: "perform_action", description: "Invoke a named accessibility action on an element (e.g. AXPress on macOS, Invoke on Windows/UIA, click on harmony). Only actions the element advertises. Element targets only.", |
| 476 | inputSchema: { type: "object", required: ["target", "action"], properties: { target: elementTargetSchema, action: { type: "string" }, computer: computerParam }, additionalProperties: false }, |
| 477 | }, |
| 478 | { |
| 479 | name: "invoke_menu", description: "macOS: invoke an application menu item by title path (e.g. [\"File\",\"New\"]). Runs through accessibility with no focus lease and no key events — prefer this over cmd-key chords for app commands (New, Save, Quit and menu-only actions). App-level commands work without a key window; window-targeted items (Close) can validate against the app's key window and may no-op in the background — prefer the window's close-button element for those. Acts on the app bound with open_application. Verify the effect (list_windows / get_app_state) before reporting success.", |
| 480 | inputSchema: { |
| 481 | type: "object", required: ["path"], |
| 482 | properties: { |
| 483 | path: { type: "array", minItems: 1, maxItems: 3, items: { type: "string", minLength: 1 }, description: "Menu titles from the menu bar inward, e.g. [\"File\",\"Close Window\"]. Exact titles as shown, including an ellipsis when the app shows one. Application menus (the second menu bar group named after the app) work too." }, |
| 484 | computer: computerParam, |
| 485 | }, |
| 486 | additionalProperties: false, |
| 487 | }, |
| 488 | }, |
| 489 | // ---- clipboard / runtime ---- |
| 490 | { |
| 491 | name: "clipboard", description: "Read or write the system clipboard as UTF-8 text: action \"read\" or \"write\" (write requires text). This is the user's real clipboard — restore it when a round-trip is needed.", |
| 492 | inputSchema: { type: "object", required: ["action"], properties: { action: { enum: ["read", "write"] }, text: { type: "string" }, computer: computerParam }, additionalProperties: false }, |
| 493 | }, |
| 494 | { |
| 495 | name: "read_clipboard", description: "Read the system clipboard as UTF-8 text.", |
| 496 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 497 | }, |
| 498 | { |
| 499 | name: "write_clipboard", description: "Write UTF-8 text to the system clipboard.", |
| 500 | inputSchema: { type: "object", required: ["text"], properties: { text: { type: "string" }, computer: computerParam }, additionalProperties: false }, |
| 501 | }, |
| 502 | // ---- recording ---- |
| 503 | { |
| 504 | name: "recording", description: "Screen recordings: action start | stop | status | list. `start` accepts display/fps/region/app_ref/window_id/durationSec/intervalMs; stop/status take the recording `id`; list reports what exists. Darwin records through ScreenCaptureKit; other platforms state their own limits in the receipt.", |
| 505 | inputSchema: { type: "object", required: ["action"], properties: { action: { enum: ["start", "stop", "status", "list"] }, id: { type: "string", description: "Recording id for stop/status" }, display: { type: "integer" }, fps: { type: "number" }, region: { type: "array", items: { type: "number" }, minItems: 4, maxItems: 4 }, app_ref: { type: "object", properties: { pid: { type: "integer" }, name: { type: "string" }, bundle_id: { type: "string" } }, additionalProperties: false }, window_id: { type: "integer" }, durationSec: { type: "number" }, intervalMs: { type: "integer" }, computer: computerParam }, additionalProperties: false }, |
| 506 | }, |
| 507 | { |
| 508 | name: "recording_start", |
| 509 | description: "Start screen recording on a computer (mp4/mov). Darwin: ScreenCaptureKit via the native helper (timed or until recording_stop; honors region, no recorder overlay, stops on session exit). Pass app_ref to record only the selected app's window rect — captured at start and not tracked across moves. Linux and Windows: unavailable pending session-owned recorder cleanup; use screenshots. HarmonyOS: snapshot-series muxed with ffmpeg.", |
| 510 | inputSchema: { |
| 511 | type: "object", |
| 512 | properties: { |
| 513 | display: { type: ["integer", "string"] }, |
| 514 | fps: { type: "integer", minimum: 1, maximum: 60, description: "Linux/Windows/harmony-series only" }, |
| 515 | region: { type: "array", items: { type: "number" }, minItems: 4, maxItems: 4, description: "[x, y, w, h] in screen points" }, |
| 516 | app_ref: { type: "object", properties: { pid: { type: "integer" }, name: { type: "string" }, bundle_id: { type: "string" } }, additionalProperties: false, description: "macOS only: record the rect this app's window occupies at start. Omission follows the app selected by open_application." }, |
| 517 | window_id: { type: "integer", description: "macOS only: zero-based window index within the app; requires or implies app_ref." }, |
| 518 | durationSec: { type: "number", minimum: 1, maximum: 7200, description: "macOS only: auto-stop after N seconds" }, |
| 519 | intervalMs: { type: "integer", minimum: 150, maximum: 5000, description: "harmony snapshot-series frame interval" }, |
| 520 | computer: computerParam, |
| 521 | }, |
| 522 | additionalProperties: false, |
| 523 | }, |
| 524 | }, |
| 525 | { |
| 526 | name: "recording_stop", |
| 527 | description: "Stop a running recording and finalize the file.", |
| 528 | inputSchema: { type: "object", required: ["id"], properties: { id: { type: "string" }, computer: computerParam }, additionalProperties: false }, |
| 529 | }, |
| 530 | { |
| 531 | name: "recording_status", |
| 532 | description: "Status of one recording (running, bytes so far).", |
| 533 | inputSchema: { type: "object", required: ["id"], properties: { id: { type: "string" }, computer: computerParam }, additionalProperties: false }, |
| 534 | }, |
| 535 | { |
| 536 | name: "recording_list", |
| 537 | description: "List recordings and screenshots saved on a computer.", |
| 538 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 539 | }, |
| 540 | // ---- programmatic interface ---- |
| 541 | { |
| 542 | name: "app_script", |
| 543 | description: "macOS, local computer only: run an AppleScript or JXA (JavaScript for Automation) script through osascript — the programmatic interface inside apps that have a scripting dictionary (Finder, Mail, Safari, Calendar, Notes, Reminders, Music, System Events and most native apps). Prefer this over clicking when the app exposes one: deterministic, returns values, needs no Accessibility grant and never touches the pointer. The receipt carries stdout as `result`; a non-zero exit fails `script_error` with stderr, a user-declined consent fails `automation_denied` (the fix is System Settings → Privacy & Security → Automation, not a retry). Refused on ssh/hdc computers (`unsupported_on_transport`) — the remote channel stays computer-use only, never a shell.", |
| 544 | inputSchema: { |
| 545 | type: "object", required: ["script"], |
| 546 | properties: { |
| 547 | script: { type: "string", minLength: 1, description: "Script source. For app arguments use `on run argv` in JXA or read them inside the script; keep scripts single-purpose." }, |
| 548 | language: { enum: ["applescript", "javascript"], description: "applescript (default) or javascript for JXA" }, |
| 549 | timeout: { type: "number", minimum: 1, maximum: 120, description: "Seconds before the script is killed; default 30." }, |
| 550 | computer: computerParam, |
| 551 | }, |
| 552 | additionalProperties: false, |
| 553 | }, |
| 554 | }, |
| 555 | // ---- kill switch ---- |
| 556 | { |
| 557 | name: "stop_computer_control", |
| 558 | description: "Kill switch: refuse all further computer-use actions for the rest of the session. Read-only probes stay available.", |
| 559 | inputSchema: { type: "object", properties: { reason: { type: "string" } }, additionalProperties: false }, |
| 560 | }, |
| 561 | { |
| 562 | name: "wait", |
| 563 | description: "Pause before the next observation (0..30s). Use after actions that animate or load.", |
| 564 | inputSchema: { type: "object", properties: { seconds: { type: "number", minimum: 0, maximum: 30 } }, additionalProperties: false }, |
| 565 | }, |
| 566 | ]; |
| 567 | |
| 568 | export const TOOL_NAMES = new Set(TOOLS.map((t) => t.name)); |
| 569 | |
| 570 | /** Required argument names per tool, straight from each inputSchema. */ |
| 571 | export const REQUIRED_ARGS = new Map(TOOLS.map((t) => [t.name, t.inputSchema.required ?? []])); |
| 572 | |
| 573 | /** Tools whose target must be an observed element — a coordinate reaches the |
| 574 | * backend unresolvable and fails opaquely, so refuse it at the boundary. */ |
| 575 | export const ELEMENT_ONLY_TARGET = new Set(["set_value", "select_text", "perform_action"]); |
| 576 | |
| 577 | /** Tools that never touch a computer (available even after kill switch). */ |
| 578 | export const READ_ONLY_TOOLS = new Set([ |
| 579 | "computer_list", "stop_computer_control", "wait", "request_access", "recording_list", "recording_status", |
| 580 | "find_elements", "get_value", "list_sessions", "browser_status", "trajectory_status", "trajectory_start", "trajectory_stop", |
| 581 | "consent_status", |
| 582 | ]); |
| 583 | |
| 584 | /** Tools dispatchable to a remote agent over ssh (allow-list must match agent.mjs). */ |
| 585 | export const REMOTE_TOOLS = new Set([ |
| 586 | "preview", "probe", "list_displays", "switch_display", "list_apps", "list_sessions", "list_windows", |
| 587 | "open_application", "kill_app", "set_window_frame", "get_app_state", "resolve_element", "screenshot", "zoom", |
| 588 | "browser_start", "browser_status", "browser_navigate", "browser_click", "browser_type", "browser_screenshot", "browser_stop", |
| 589 | "left_click", "double_click", "triple_click", "right_click", "middle_click", |
| 590 | "mouse_move", "left_click_drag", "left_mouse_down", "left_mouse_up", "scroll", |
| 591 | "type", "key", "hold_key", "set_value", "focus", "get_value", "select_text", "perform_action", "invoke_menu", |
| 592 | "read_clipboard", "write_clipboard", "cursor_position", |
| 593 | "recordingStart", "recordingStop", "recordingStatus", "recordingList", |
| 594 | "app_script", |
| 595 | ]); |
| 596 | |
| 597 | /** Map public tool name -> backend method name. */ |
| 598 | export const BACKEND_METHOD = Object.fromEntries( |
| 599 | TOOLS.filter((t) => !["computer_list", "computer_switch", "computer_register", "computer_spawn", "computer_remove", "consent_status", "consent_allow", "consent_deny", "consent_revoke", "stop_computer_control", "wait", "wait_for", "find_elements", "run_actions", "trajectory_start", "trajectory_stop", "trajectory_status", "trajectory_replay"].includes(t.name)) |
| 600 | .map((t) => [t.name, { |
| 601 | request_access: "probe", |
| 602 | recording_start: "recordingStart", |
| 603 | recording_stop: "recordingStop", |
| 604 | recording_status: "recordingStatus", |
| 605 | recording_list: "recordingList", |
| 606 | }[t.name] ?? t.name]), |
| 607 | ); |
| 608 | |
| 609 | /** |
| 610 | * Wire-name expansion for merged tools, used by capability grants: naming a |
| 611 | * merged tool admits every action it can dispatch to. |
| 612 | */ |
| 613 | export const MERGED_EXPANSION = { |
| 614 | click: ["left_click", "double_click", "triple_click", "right_click", "middle_click"], |
| 615 | pointer: ["mouse_move", "left_mouse_down", "left_mouse_up"], |
| 616 | clipboard: ["read_clipboard", "write_clipboard"], |
| 617 | recording: ["recording_start", "recording_stop", "recording_status", "recording_list"], |
| 618 | computer: ["computer_list", "computer_switch", "computer_register", "computer_spawn", "computer_remove"], |
| 619 | consent: ["consent_status", "consent_allow", "consent_deny", "consent_revoke"], |
| 620 | key: ["key", "hold_key"], |
| 621 | browser: ["browser_start", "browser_status", "browser_navigate", "browser_click", "browser_type", "browser_screenshot", "browser_stop"], |
| 622 | trajectory: ["trajectory_start", "trajectory_stop", "trajectory_status", "trajectory_replay"], |
| 623 | }; |
| 624 | |
| 625 | /** |
| 626 | * Parse CODEWHALE_CU_GRANT — "read-only", or a comma list of tool names — |
| 627 | * into a wire-name set. The grant is fixed when the server starts (there is |
| 628 | * no tool that can widen it) and it is enforced twice: here, so the model |
| 629 | * never sees or reaches an ungranted tool, and at the app daemon, so a |
| 630 | * narrowed server cannot smuggle one through. Returns null when unset. |
| 631 | */ |
| 632 | export function parseGrant(value) { |
| 633 | if (value == null || (typeof value === "string" && !value.trim())) return null; |
| 634 | const out = new Set(); |
| 635 | for (const raw of String(value).split(",")) { |
| 636 | const name = raw.trim(); |
| 637 | if (!name) continue; |
| 638 | if (name === "read-only") { for (const tool of OBSERVATION_TOOLS) out.add(tool); continue; } |
| 639 | if (MERGED_EXPANSION[name]) { for (const tool of MERGED_EXPANSION[name]) out.add(tool); continue; } |
| 640 | out.add(name); |
| 641 | } |
| 642 | return out.size ? out : null; |
| 643 | } |
| 644 | |
| 645 | // ---------- MCP tool annotations ---------- |
| 646 | // Host-facing hints for approval and sandbox policy (MCP spec `annotations`). |
| 647 | // Hints describe the tool's design; they are not runtime gates. Observation |
| 648 | // tools read local state; `openWorld` is true when a tool acts on applications |
| 649 | // or computers outside this process; `destructive` marks tools that change what |
| 650 | // the user sees or holds (input, clipboard, registrations). |
| 651 | const READ_ONLY_ANNOTATION = { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }; |
| 652 | const INPUT_ANNOTATION = { readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: true }; |
| 653 | const TOOL_ANNOTATIONS = { |
| 654 | // Observation — reads only. |
| 655 | request_access: READ_ONLY_ANNOTATION, computer_list: READ_ONLY_ANNOTATION, list_displays: READ_ONLY_ANNOTATION, |
| 656 | list_apps: READ_ONLY_ANNOTATION, list_windows: READ_ONLY_ANNOTATION, wait_for: READ_ONLY_ANNOTATION, |
| 657 | list_sessions: READ_ONLY_ANNOTATION, |
| 658 | kill_app: { readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: true }, |
| 659 | set_window_frame: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true }, |
| 660 | get_app_state: READ_ONLY_ANNOTATION, find_elements: READ_ONLY_ANNOTATION, get_value: READ_ONLY_ANNOTATION, |
| 661 | screenshot: READ_ONLY_ANNOTATION, zoom: READ_ONLY_ANNOTATION, cursor_position: READ_ONLY_ANNOTATION, |
| 662 | read_clipboard: READ_ONLY_ANNOTATION, recording_list: READ_ONLY_ANNOTATION, recording_status: READ_ONLY_ANNOTATION, |
| 663 | wait: READ_ONLY_ANNOTATION, |
| 664 | // Session controls — local state, not the user's apps. |
| 665 | preview: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 666 | stop_computer_control: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 667 | switch_display: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 668 | recording_start: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }, |
| 669 | recording_stop: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 670 | // Computer registry — touches other machines. |
| 671 | computer_switch: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: true }, |
| 672 | computer_register: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: true }, |
| 673 | computer_spawn: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 674 | computer_remove: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true }, |
| 675 | // Consent — the user's own decision record, not an action on apps. |
| 676 | consent: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 677 | consent_status: READ_ONLY_ANNOTATION, |
| 678 | consent_allow: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 679 | consent_deny: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 680 | consent_revoke: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 681 | open_application: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: true }, |
| 682 | // Input — changes what the user sees. |
| 683 | left_click: INPUT_ANNOTATION, double_click: INPUT_ANNOTATION, triple_click: INPUT_ANNOTATION, |
| 684 | right_click: INPUT_ANNOTATION, middle_click: INPUT_ANNOTATION, left_click_drag: INPUT_ANNOTATION, |
| 685 | left_mouse_down: INPUT_ANNOTATION, left_mouse_up: INPUT_ANNOTATION, |
| 686 | type: INPUT_ANNOTATION, key: INPUT_ANNOTATION, hold_key: INPUT_ANNOTATION, invoke_menu: INPUT_ANNOTATION, |
| 687 | perform_action: INPUT_ANNOTATION, run_actions: INPUT_ANNOTATION, |
| 688 | // Scripting — acts on apps through their own dictionaries, not through input. |
| 689 | app_script: INPUT_ANNOTATION, |
| 690 | mouse_move: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true }, |
| 691 | scroll: { readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: true }, |
| 692 | set_value: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true }, |
| 693 | focus: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true }, |
| 694 | select_text: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true }, |
| 695 | write_clipboard: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false }, |
| 696 | // Merged surface (aliases keep the wire names above callable). |
| 697 | click: INPUT_ANNOTATION, |
| 698 | pointer: INPUT_ANNOTATION, |
| 699 | browser: INPUT_ANNOTATION, |
| 700 | trajectory: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }, |
| 701 | trajectory_status: READ_ONLY_ANNOTATION, |
| 702 | trajectory_start: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 703 | trajectory_stop: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 704 | trajectory_replay: { readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: true }, |
| 705 | browser_status: READ_ONLY_ANNOTATION, |
| 706 | browser_screenshot: READ_ONLY_ANNOTATION, |
| 707 | browser_start: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: true }, |
| 708 | browser_stop: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: true }, |
| 709 | browser_navigate: INPUT_ANNOTATION, |
| 710 | browser_click: INPUT_ANNOTATION, |
| 711 | browser_type: INPUT_ANNOTATION, |
| 712 | clipboard: { readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: false }, |
| 713 | recording: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }, |
| 714 | computer: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true }, |
| 715 | }; |
| 716 | for (const tool of TOOLS) { |
| 717 | tool.annotations = TOOL_ANNOTATIONS[tool.name] ?? INPUT_ANNOTATION; |
| 718 | } |
| 719 | |
| 720 | /** |
| 721 | * Tools that only observe, straight from their annotations. This is the |
| 722 | * "read-only" capability grant — distinct from READ_ONLY_TOOLS (the smaller |
| 723 | * post-kill-switch set that also drives the safety valve). |
| 724 | */ |
| 725 | export const OBSERVATION_TOOLS = new Set(TOOLS.filter((t) => t.annotations.readOnlyHint === true).map((t) => t.name)); |
| 726 | |
| 727 | /** |
| 728 | * Merged-away names. They stay callable as aliases (receipts, pinned hosts and |
| 729 | * existing tests keep working) but never appear in tools/list — the advertised |
| 730 | * surface is what costs every session context. |
| 731 | */ |
| 732 | const HIDDEN_FROM_LIST = new Set([ |
| 733 | "left_click", "double_click", "triple_click", "right_click", "middle_click", |
| 734 | "mouse_move", "left_mouse_down", "left_mouse_up", |
| 735 | "read_clipboard", "write_clipboard", |
| 736 | "recording_start", "recording_stop", "recording_status", "recording_list", |
| 737 | "computer_list", "computer_switch", "computer_register", "computer_spawn", "computer_remove", |
| 738 | "consent_status", "consent_allow", "consent_deny", "consent_revoke", |
| 739 | "hold_key", |
| 740 | "browser_start", "browser_status", "browser_navigate", "browser_click", "browser_type", "browser_screenshot", "browser_stop", |
| 741 | "trajectory_start", "trajectory_stop", "trajectory_status", "trajectory_replay", |
| 742 | ]); |
| 743 | for (const tool of TOOLS) { |
| 744 | if (HIDDEN_FROM_LIST.has(tool.name)) tool.hidden = true; |
| 745 | } |
| 746 | |
| 747 | /** |
| 748 | * Expand a merged, advertised tool into the wire tool it dispatches to. |
| 749 | * Runs before every gate in the dispatcher (required args, kill switch, |
| 750 | * routing), so a merged call can never bypass one; validation that the wire |
| 751 | * schema cannot express (which action, what each action needs) lives here and |
| 752 | * fails as bad_args with the requested name. Unknown names pass through |
| 753 | * unchanged — the alias surface is the rest of TOOLS. |
| 754 | */ |
| 755 | export function resolveTool(name, args = {}) { |
| 756 | const bad = (message) => Object.assign(new Error(message), { code: "bad_args" }); |
| 757 | switch (name) { |
| 758 | case "click": { |
| 759 | const button = args.button ?? "left"; |
| 760 | const clicks = args.clicks ?? 1; |
| 761 | const rest = { ...args }; |
| 762 | delete rest.button; |
| 763 | delete rest.clicks; |
| 764 | const wire = button === "left" && clicks === 1 ? "left_click" |
| 765 | : button === "left" && clicks === 2 ? "double_click" |
| 766 | : button === "left" && clicks === 3 ? "triple_click" |
| 767 | : button === "right" && clicks === 1 ? "right_click" |
| 768 | : button === "middle" && clicks === 1 ? "middle_click" |
| 769 | : null; |
| 770 | if (!wire) throw bad(`click supports left with 1-3 clicks, right x1 or middle x1 (got ${JSON.stringify(button)} x${clicks})`); |
| 771 | if (button !== "left") delete rest.strategy; // strategy is an a11y-left-click concept |
| 772 | return { name: wire, args: rest }; |
| 773 | } |
| 774 | case "pointer": { |
| 775 | const rest = { ...args }; |
| 776 | delete rest.action; |
| 777 | const wire = { move: "mouse_move", down: "left_mouse_down", up: "left_mouse_up" }[args.action]; |
| 778 | if (!wire) throw bad(`pointer action must be "move", "down" or "up" (got ${JSON.stringify(args.action)})`); |
| 779 | return { name: wire, args: rest }; |
| 780 | } |
| 781 | case "clipboard": { |
| 782 | const rest = { ...args }; |
| 783 | delete rest.action; |
| 784 | if (args.action === "read") return { name: "read_clipboard", args: { computer: rest.computer } }; |
| 785 | if (args.action === "write") { |
| 786 | if (typeof rest.text !== "string") throw bad("clipboard action \"write\" requires text"); |
| 787 | return { name: "write_clipboard", args: { text: rest.text, computer: rest.computer } }; |
| 788 | } |
| 789 | throw bad(`clipboard action must be "read" or "write" (got ${JSON.stringify(args.action)})`); |
| 790 | } |
| 791 | case "recording": { |
| 792 | const rest = { ...args }; |
| 793 | delete rest.action; |
| 794 | const wire = { start: "recording_start", stop: "recording_stop", status: "recording_status", list: "recording_list" }[args.action]; |
| 795 | if (!wire) throw bad(`recording action must be start, stop, status or list (got ${JSON.stringify(args.action)})`); |
| 796 | if ((args.action === "stop" || args.action === "status") && rest.id == null) throw bad(`recording action "${args.action}" requires id`); |
| 797 | return { name: wire, args: rest }; |
| 798 | } |
| 799 | case "computer": { |
| 800 | const rest = { ...args }; |
| 801 | delete rest.action; |
| 802 | const wire = { list: "computer_list", switch: "computer_switch", register: "computer_register", spawn: "computer_spawn", remove: "computer_remove" }[args.action]; |
| 803 | if (!wire) throw bad(`computer action must be list, switch, register, spawn or remove (got ${JSON.stringify(args.action)})`); |
| 804 | if (args.action === "list") return { name: wire, args: {} }; |
| 805 | if (rest.id == null) throw bad(`computer action "${args.action}" requires id`); |
| 806 | const id = rest.id; |
| 807 | delete rest.id; |
| 808 | return { name: wire, args: { ...rest, computer: id } }; |
| 809 | } |
| 810 | case "consent": { |
| 811 | const rest = { ...args }; |
| 812 | delete rest.action; |
| 813 | const wire = { status: "consent_status", allow: "consent_allow", deny: "consent_deny", revoke: "consent_revoke" }[args.action]; |
| 814 | if (!wire) throw bad(`consent action must be status, allow, deny or revoke (got ${JSON.stringify(args.action)})`); |
| 815 | if (args.action === "status") return { name: wire, args: { computer: rest.computer } }; |
| 816 | const foreground = rest.scope === "foreground"; |
| 817 | if (!foreground && rest.app == null && rest.name == null && rest.bundle_id == null && rest.pid == null) { |
| 818 | throw bad(`consent action "${args.action}" needs an app (name, bundle_id, pid or app string) — or scope:"foreground" for the shared-pointer decision`); |
| 819 | } |
| 820 | return { name: wire, args: rest }; |
| 821 | } |
| 822 | case "key": { |
| 823 | if (args.duration == null) return { name, args }; |
| 824 | const { duration, repeat, target, ...rest } = args; |
| 825 | if (repeat != null || target != null) throw bad("key with duration holds the key — repeat and target cannot be combined with it"); |
| 826 | if (!Number.isFinite(duration) || duration < 0.05 || duration > 30) throw bad("duration must be 0.05..30 seconds"); |
| 827 | return { name: "hold_key", args: { ...rest, duration } }; |
| 828 | } |
| 829 | case "browser": { |
| 830 | const rest = { ...args }; |
| 831 | delete rest.action; |
| 832 | switch (args.action) { |
| 833 | case "start": |
| 834 | if (rest.url != null && typeof rest.url !== "string") throw bad("browser start url must be a string"); |
| 835 | return { name: "browser_start", args: rest }; |
| 836 | case "status": return { name: "browser_status", args: rest }; |
| 837 | case "navigate": |
| 838 | if (typeof rest.url !== "string" || !rest.url.trim()) throw bad('browser action "navigate" requires url'); |
| 839 | return { name: "browser_navigate", args: rest }; |
| 840 | case "click": { |
| 841 | const hasSelector = typeof rest.selector === "string" && rest.selector.trim(); |
| 842 | const hasPoint = rest.point != null && Number.isFinite(rest.point?.x) && Number.isFinite(rest.point?.y); |
| 843 | if (hasSelector && hasPoint) throw bad('browser action "click" takes selector or point, not both — pick one target'); |
| 844 | if (!hasSelector && !hasPoint) throw bad('browser action "click" needs selector (CSS) or point {x,y}'); |
| 845 | if (!hasSelector) delete rest.selector; |
| 846 | if (!hasPoint) delete rest.point; |
| 847 | return { name: "browser_click", args: rest }; |
| 848 | } |
| 849 | case "type": |
| 850 | if (typeof rest.text !== "string" || !rest.text.length) throw bad('browser action "type" requires text'); |
| 851 | return { name: "browser_type", args: rest }; |
| 852 | case "screenshot": return { name: "browser_screenshot", args: rest }; |
| 853 | case "stop": return { name: "browser_stop", args: rest }; |
| 854 | default: |
| 855 | throw bad(`browser action must be start, status, navigate, click, type, screenshot or stop (got ${JSON.stringify(args.action)})`); |
| 856 | } |
| 857 | } |
| 858 | case "trajectory": { |
| 859 | const rest = { ...args }; |
| 860 | delete rest.action; |
| 861 | switch (args.action) { |
| 862 | case "start": return { name: "trajectory_start", args: rest }; |
| 863 | case "stop": return { name: "trajectory_stop", args: rest }; |
| 864 | case "status": return { name: "trajectory_status", args: rest }; |
| 865 | case "replay": return { name: "trajectory_replay", args: rest }; |
| 866 | default: |
| 867 | throw bad(`trajectory action must be start, stop, status or replay (got ${JSON.stringify(args.action)})`); |
| 868 | } |
| 869 | } |
| 870 | default: |
| 871 | return { name, args }; |
| 872 | } |
| 873 | } |
| 874 |