| 1 | // Windows backend. Every operation is one PowerShell invocation whose script |
| 2 | // travels as a base64 -EncodedCommand, so tool arguments never become shell |
| 3 | // syntax. Screenshots + UIA accessibility come from .NET; raw pointer and |
| 4 | // keyboard events come from user32 P/Invoke (SendInput/mouse_event). |
| 5 | // Recording stays unavailable until its native process has session-owned cleanup. |
| 6 | import fs from "node:fs"; |
| 7 | import os from "node:os"; |
| 8 | import path from "node:path"; |
| 9 | import crypto from "node:crypto"; |
| 10 | import { run, ExecError, tryJson, withSignal, throwIfAborted } from "../exec.mjs"; |
| 11 | import { createBrowser } from "../browser-cdp.mjs"; |
| 12 | |
| 13 | const USER32 = ` |
| 14 | using System; |
| 15 | using System.Runtime.InteropServices; |
| 16 | public static class User32 { |
| 17 | [DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); |
| 18 | [DllImport("user32.dll")] public static extern void mouse_event(uint dwFlags, int dx, int dy, uint dwData, UIntPtr dwExtraInfo); |
| 19 | [DllImport("user32.dll", SetLastError = true)] private static extern uint SendInput(uint nInputs, INPUT[] pInputs, int cbSize); |
| 20 | [DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT lpPoint); |
| 21 | [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); |
| 22 | [StructLayout(LayoutKind.Sequential)] public struct POINT { public int X; public int Y; } |
| 23 | [StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left; public int Top; public int Right; public int Bottom; } |
| 24 | [StructLayout(LayoutKind.Sequential)] public struct MOUSEINPUT { public int dx; public int dy; public uint mouseData; public uint dwFlags; public uint time; public IntPtr dwExtraInfo; } |
| 25 | [StructLayout(LayoutKind.Sequential)] public struct KEYBDINPUT { public ushort wVk; public ushort wScan; public uint dwFlags; public uint time; public IntPtr dwExtraInfo; } |
| 26 | [StructLayout(LayoutKind.Explicit)] public struct INPUTUNION { [FieldOffset(0)] public MOUSEINPUT mi; [FieldOffset(0)] public KEYBDINPUT ki; } |
| 27 | [StructLayout(LayoutKind.Sequential)] public struct INPUT { public uint type; public INPUTUNION u; } |
| 28 | public const uint MOVED = 0x0001, LEFTDOWN = 0x0002, LEFTUP = 0x0004, RIGHTDOWN = 0x0008, RIGHTUP = 0x0010, |
| 29 | MIDDLEDOWN = 0x0020, MIDDLEUP = 0x0040, WHEEL = 0x0800, HWHEEL = 0x1000, ABSOLUTE = 0x8000, VIRTUALKEY = 0x4000, KEYUP = 0x0002, UNICODE = 0x0004; |
| 30 | public static INPUT MouseInput(uint flags, int x, int y, uint data) { |
| 31 | return new INPUT { type = 0, u = new INPUTUNION { mi = new MOUSEINPUT { dx = x, dy = y, mouseData = data, dwFlags = flags, time = 0, dwExtraInfo = IntPtr.Zero } } }; |
| 32 | } |
| 33 | public static INPUT KeyInput(ushort vk, ushort scan, uint flags) { |
| 34 | return new INPUT { type = 1, u = new INPUTUNION { ki = new KEYBDINPUT { wVk = vk, wScan = scan, dwFlags = flags, time = 0, dwExtraInfo = IntPtr.Zero } } }; |
| 35 | } |
| 36 | public sealed class InputDeliveryException : InvalidOperationException { |
| 37 | public readonly uint Inserted; |
| 38 | public InputDeliveryException(uint inserted, int expected, int error) |
| 39 | : base(String.Format("SendInput inserted {0} of {1} events (Win32 error {2})", inserted, expected, error)) { Inserted = inserted; } |
| 40 | } |
| 41 | public static void SendChecked(INPUT[] inputs) { |
| 42 | uint sent = SendInput((uint)inputs.Length, inputs, Marshal.SizeOf(typeof(INPUT))); |
| 43 | if (sent != (uint)inputs.Length) { |
| 44 | int error = Marshal.GetLastWin32Error(); |
| 45 | throw new InputDeliveryException(sent, inputs.Length, error); |
| 46 | } |
| 47 | } |
| 48 | public static void SendKey(ushort vk, uint flags) { |
| 49 | SendChecked(new INPUT[] { KeyInput(vk, 0, flags) }); |
| 50 | } |
| 51 | public static void SendString(string text) { |
| 52 | // UTF-16 code units, including both halves of surrogate pairs. UNICODE |
| 53 | // requires wVk=0 and may only be combined with KEYUP, never SCANCODE. |
| 54 | foreach (char c in text) { |
| 55 | INPUT up = KeyInput(0, c, UNICODE | KEYUP); |
| 56 | try { SendChecked(new INPUT[] { KeyInput(0, c, UNICODE), up }); } |
| 57 | catch (InputDeliveryException failure) { |
| 58 | // A partial pair may leave only our down event inserted. Release it |
| 59 | // once, never replay text, and still report the original failure. |
| 60 | if (failure.Inserted == 1) { |
| 61 | try { SendChecked(new INPUT[] { up }); } |
| 62 | catch (Exception cleanup) { |
| 63 | throw new AggregateException("Unicode input failed: " + failure.Message + "; key-up recovery failed: " + cleanup.Message, failure, cleanup); |
| 64 | } |
| 65 | } |
| 66 | throw; |
| 67 | } |
| 68 | } |
| 69 | } |
| 70 | }`; |
| 71 | |
| 72 | function recordingsDir() { |
| 73 | return process.env.CODEWHALE_CU_RECORDINGS_DIR || path.join(os.homedir(), ".codewhale-cu", "recordings"); |
| 74 | } |
| 75 | |
| 76 | // Windows virtual-key codes for named keys. |
| 77 | const VK = { |
| 78 | return: 0x0d, enter: 0x0d, tab: 0x09, escape: 0x1b, esc: 0x1b, space: 0x20, |
| 79 | backspace: 0x08, delete: 0x2e, home: 0x24, end: 0x23, pageup: 0x21, pagedown: 0x22, |
| 80 | left: 0x25, up: 0x26, right: 0x27, down: 0x28, capslock: 0x14, insert: 0x2d, |
| 81 | f1: 0x70, f2: 0x71, f3: 0x72, f4: 0x73, f5: 0x74, f6: 0x75, f7: 0x76, f8: 0x77, |
| 82 | f9: 0x78, f10: 0x79, f11: 0x7a, f12: 0x7b, |
| 83 | }; |
| 84 | const MODVK = { ctrl: 0x11, control: 0x11, alt: 0x12, shift: 0x10, win: 0x5b, meta: 0x5b, cmd: 0x5b }; |
| 85 | |
| 86 | // Every action runs in a fresh powershell.exe process, so a bootstrap process |
| 87 | // can never register the User32 type for later spawns. Each User32-backed |
| 88 | // invocation therefore carries its own type definition via this prelude. |
| 89 | // Compilation, conversion and native-call exceptions must stop before success. |
| 90 | const USER32_PRELUDE = `$ErrorActionPreference = 'Stop';\nAdd-Type -TypeDefinition @'\n${USER32}\n'@ -ErrorAction Stop;`; |
| 91 | |
| 92 | // Initialize UIA proxies from a typed CLR frame. The legacy proxy loader |
| 93 | // walks ReflectedType on its call stack; PowerShell dynamic frames can be null |
| 94 | // and leave standard controls exposed as plain panes without action patterns. |
| 95 | const UIA_PRELUDE = `Add-Type -AssemblyName UIAutomationClient; |
| 96 | Add-Type -AssemblyName UIAutomationTypes; |
| 97 | Add-Type -ReferencedAssemblies ([System.Windows.Automation.AutomationElement].Assembly.Location) -TypeDefinition @' |
| 98 | using System.Windows.Automation; |
| 99 | public static class CUAutomationProviders { |
| 100 | public static void Register() { |
| 101 | var assembly = typeof(AutomationElement).Assembly.GetName(); |
| 102 | assembly.Name = "UIAutomationClientsideProviders"; |
| 103 | // Legacy .NET clears its one-time default-proxy flag before walking a |
| 104 | // dynamic PowerShell stack, which can throw NullReferenceException. |
| 105 | // Retry only registration (no UI action); a second failure propagates. |
| 106 | try { ClientSettings.RegisterClientSideProviderAssembly(assembly); } |
| 107 | catch (System.NullReferenceException) { ClientSettings.RegisterClientSideProviderAssembly(assembly); } |
| 108 | } |
| 109 | } |
| 110 | '@; |
| 111 | [CUAutomationProviders]::Register();`; |
| 112 | |
| 113 | /** Coordinate clicks on this backend are always raw pointer events; strategy="a11y" must fail closed rather than silently degrade. */ |
| 114 | function assertEventStrategy(strategy) { |
| 115 | if (strategy != null && strategy !== "auto" && strategy !== "event") { |
| 116 | throw new ExecError(`strategy "${strategy}" is macOS-only; this backend dispatches coordinate clicks as raw pointer events — use an element target for a semantic action`); |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | function unsupportedSelector(message) { |
| 121 | return Object.assign(new ExecError(message), { code: "unsupported_selector" }); |
| 122 | } |
| 123 | |
| 124 | // A cached child path alone is unsafe: sibling/window order can change. |
| 125 | // Runtime IDs bind both the root and leaf to the observation that supplied them. |
| 126 | function elementScript(target) { |
| 127 | const ids = [target?.runtime_id, target?.window_runtime_id]; |
| 128 | if (ids.some(id => !Array.isArray(id) || !id.length || !id.every(Number.isInteger)) |
| 129 | || !Array.isArray(target?.path) || target.path[0] !== 0 || !target.path.every(i => Number.isInteger(i) && i >= 0) |
| 130 | || (target.windowIndex != null && target.windowIndex !== 0) || Object.hasOwn(target, "window_id")) { |
| 131 | throw unsupportedSelector("Windows semantic actions require a fresh observed element with window and element runtime identities"); |
| 132 | } |
| 133 | const encoded = Buffer.from(JSON.stringify(target), "utf16le").toString("base64"); |
| 134 | return `${UIA_PRELUDE} |
| 135 | $target = [Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${encoded}')) | ConvertFrom-Json; |
| 136 | $root = [System.Windows.Automation.AutomationElement]::RootElement; |
| 137 | $windows = @($root.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition)); |
| 138 | $matches = @($windows | Where-Object { ($_.GetRuntimeId() -join ',') -ceq ($target.window_runtime_id -join ',') }); |
| 139 | if ($matches.Count -ne 1) { throw 'element_stale: observed window no longer exists' } |
| 140 | $cur = $matches[0]; |
| 141 | if ($target.app_ref.name -and $cur.Current.Name -cne $target.app_ref.name) { throw 'element_stale: window identity changed' } |
| 142 | if ($target.app_ref.pid -and $cur.Current.ProcessId -ne $target.app_ref.pid) { throw 'element_stale: window process changed' } |
| 143 | for ($step = 1; $step -lt $target.path.Count; $step++) { |
| 144 | $kids = $cur.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition); |
| 145 | $i = $target.path[$step]; |
| 146 | if ($i -ge $kids.Count) { throw 'element_stale: observed path changed' } |
| 147 | $cur = $kids[$i]; |
| 148 | } |
| 149 | if (($cur.GetRuntimeId() -join ',') -cne ($target.runtime_id -join ',')) { throw 'element_stale: observed element was replaced' } |
| 150 | if ($target.role -and ($cur.Current.ControlType.ProgrammaticName -replace '^ControlType\\.', '') -cne $target.role) { throw 'element_stale: role changed' } |
| 151 | if ($null -ne $target.label -and $cur.Current.Name -cne $target.label) { throw 'element_stale: label changed' } |
| 152 | if (-not $cur.Current.IsEnabled) { throw 'element_disabled: observed element is disabled' } |
| 153 | `; |
| 154 | } |
| 155 | |
| 156 | export function create(opts = {}) { |
| 157 | // Allow tests (and other embedders) to inject a runner so no real |
| 158 | // powershell.exe is spawned. Production uses the imported runner. |
| 159 | const injectedRun = opts.exec && typeof opts.exec.run === "function" ? opts.exec.run : null; |
| 160 | const runner = injectedRun ?? run; |
| 161 | const browser = createBrowser({ platform: "win32" }); |
| 162 | |
| 163 | function requireInputOwner() { |
| 164 | if (opts.exec?.persistentInputOwner !== true) throw Object.assign(new ExecError( |
| 165 | "This held-input gesture requires a connected Codewhale Computer Use desktop helper so a disconnected client cannot leave keys or buttons pressed. Start the helper and reconnect before retrying." |
| 166 | ), { code: "input_owner_required" }); |
| 167 | } |
| 168 | |
| 169 | async function ps(script, o = {}) { |
| 170 | throwIfAborted(); |
| 171 | const encoded = Buffer.from(`$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';\n[Console]::OutputEncoding = New-Object System.Text.UTF8Encoding($false);\n${script}`, "utf16le").toString("base64"); |
| 172 | return runner("powershell.exe", ["-NoProfile", "-NonInteractive", "-EncodedCommand", encoded], { |
| 173 | timeoutMs: o.timeoutMs ?? 25_000, |
| 174 | maxBuffer: 32 * 1024 * 1024, |
| 175 | }); |
| 176 | } |
| 177 | |
| 178 | /** ps() but truthful: timeout, nonzero exit, and spawn failure all throw. */ |
| 179 | async function psOk(script, o = {}) { |
| 180 | const r = await ps(script, o); |
| 181 | if (r.aborted) throw Object.assign(new ExecError("computer request cancelled", r), { code: "cancelled" }); |
| 182 | if (r.timedOut) throw new ExecError(`powershell timed out after ${o.timeoutMs ?? 25_000}ms`, r); |
| 183 | if (r.code !== 0) { |
| 184 | const raw = (r.stderr || r.stdout).trim(); |
| 185 | // EncodedCommand serializes errors as CLIXML; surface the error strings, |
| 186 | // not a truncated XML/progress header that conceals the actual failure. |
| 187 | const messages = [...raw.matchAll(/<S S="Error">([\s\S]*?)<\/S>/g)].map(m => m[1] |
| 188 | .replace(/_x([0-9A-Fa-f]{4})_/g, (_, hex) => String.fromCharCode(parseInt(hex, 16))) |
| 189 | .replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, "&")); |
| 190 | throw new ExecError(`powershell.exe exited ${r.code}: ${(messages.join("") || raw).slice(0, 1600)}`, r); |
| 191 | } |
| 192 | return r; |
| 193 | } |
| 194 | |
| 195 | async function psJson(script, o = {}) { |
| 196 | const r = await psOk(script, o); |
| 197 | const out = r.stdout.trim(); |
| 198 | const j = tryJson(out, null); |
| 199 | if (!j) throw new ExecError(`powershell did not return JSON: ${(r.stderr || out).trim().slice(0, 300)}`, r); |
| 200 | return j; |
| 201 | } |
| 202 | |
| 203 | let lastRaster = null; |
| 204 | let activeDisplay = null; |
| 205 | const heldButtons = new Set(); |
| 206 | const heldKeys = new Set(); |
| 207 | |
| 208 | async function releaseInput({ buttons = [...heldButtons], keys = [...heldKeys] } = {}) { |
| 209 | buttons = buttons.filter((button) => heldButtons.has(button)); |
| 210 | keys = keys.filter((key) => heldKeys.has(key)); |
| 211 | if (!buttons.length && !keys.length) return; |
| 212 | const releases = [ |
| 213 | ...buttons.map((button) => `[User32]::mouse_event([User32]::${button}UP, 0, 0, 0, [UIntPtr]::Zero);`), |
| 214 | ...keys.reverse().map((vk) => `[User32]::SendKey(${vk}, 2);`), |
| 215 | ]; |
| 216 | await withSignal(null, () => withUser32(releases.join("\n"), { timeoutMs: 2_000 })); |
| 217 | for (const button of buttons) heldButtons.delete(button); |
| 218 | for (const key of keys) heldKeys.delete(key); |
| 219 | } |
| 220 | |
| 221 | async function withKeys(keys, action) { |
| 222 | throwIfAborted(); |
| 223 | for (const code of keys) heldKeys.add(code); |
| 224 | try { |
| 225 | const result = await action(); |
| 226 | for (const code of keys) heldKeys.delete(code); |
| 227 | return result; |
| 228 | } catch (failure) { |
| 229 | try { await releaseInput({ buttons: [], keys }); } |
| 230 | catch (cleanup) { |
| 231 | throw Object.assign(new ExecError(`Keyboard input failed: ${failure.message}; release failed: ${cleanup.message}`, failure.result), { |
| 232 | code: failure.code, cause: failure, cleanupError: cleanup, |
| 233 | }); |
| 234 | } |
| 235 | throw failure; |
| 236 | } |
| 237 | } |
| 238 | |
| 239 | function keyChord(text) { |
| 240 | const parts = String(text).split("+").map((part) => part.trim().toLowerCase()); |
| 241 | const key = parts.pop(); |
| 242 | const mods = parts.map((part) => MODVK[part]); |
| 243 | const vk = VK[key] ?? MODVK[key] ?? (/^[a-z0-9]$/.test(key) ? key.toUpperCase().charCodeAt(0) : null); |
| 244 | if (vk == null || mods.some((mod) => mod == null)) throw new ExecError(`unknown key combination "${text}"`); |
| 245 | return { key, vk, mods: [...new Set(mods)] }; |
| 246 | } |
| 247 | |
| 248 | /** Self-contained User32 invocation: prelude + script, fails truthfully. */ |
| 249 | async function withUser32(script, opts) { |
| 250 | return psOk(`${USER32_PRELUDE}\n${script}`, opts); |
| 251 | } |
| 252 | |
| 253 | return { |
| 254 | platform: "win32", |
| 255 | releaseInput, |
| 256 | browser_start: browser.start, |
| 257 | browser_status: browser.status, |
| 258 | browser_navigate: browser.navigate, |
| 259 | browser_click: browser.click, |
| 260 | browser_type: browser.type, |
| 261 | browser_screenshot: browser.screenshot, |
| 262 | browser_stop: browser.stop, |
| 263 | closeSession: async () => { await browser.close().catch(() => {}); }, |
| 264 | probe: async () => { |
| 265 | const psOk = await ps("Write-Output 'ok'").then((r) => r.code === 0).catch(() => false); |
| 266 | return { |
| 267 | platform: "win32", |
| 268 | powershell: psOk, |
| 269 | capabilities: { screenshot: psOk, accessibility_tree: psOk, clipboard: psOk, recording: false, raw_input: psOk, held_input: psOk && opts.exec?.persistentInputOwner === true }, |
| 270 | note: "Recording is unavailable until session-owned cleanup is implemented; use screenshots. UIA accessibility works without extra installs. Held keys, held buttons and drag require a connected Computer Use desktop helper.", |
| 271 | }; |
| 272 | }, |
| 273 | list_displays: async () => { |
| 274 | const d = await psJson(`Add-Type -AssemblyName System.Windows.Forms; |
| 275 | $arr = @([System.Windows.Forms.Screen]::AllScreens | ForEach-Object { [pscustomobject]@{ name = $_.DeviceName; primary = $_.Primary; x = $_.Bounds.X; y = $_.Bounds.Y; w = $_.Bounds.Width; h = $_.Bounds.Height } }); |
| 276 | @{ displays = $arr } | ConvertTo-Json -Depth 4 -Compress;`, { timeoutMs: 15_000 }); |
| 277 | return d.displays.map((x, i) => ({ index: i + 1, name: x.name, points: { x: x.x, y: x.y, w: x.w, h: x.h }, pixels: { w: x.w, h: x.h }, scale: 1, main: !!x.primary })); |
| 278 | }, |
| 279 | async switch_display({ index = 1 }) { |
| 280 | const displays = await this.list_displays(); |
| 281 | if (!displays.some(d => d.index === index)) throw new ExecError("display index is out of range"); |
| 282 | activeDisplay = index; |
| 283 | return { activeDisplay }; |
| 284 | }, |
| 285 | list_apps: async () => { |
| 286 | const j = await psJson(`Add-Type -AssemblyName System.Windows.Forms; |
| 287 | $out = Get-Process | Where-Object { $_.MainWindowTitle } | ForEach-Object { [pscustomobject]@{ name = $_.ProcessName; pid2 = $_.Id; title = $_.MainWindowTitle } } | ConvertTo-Json -Compress; |
| 288 | if (-not $out) { $out = '[]' } |
| 289 | Write-Output ('{"apps": ' + $out + '}');`); |
| 290 | return { apps: (Array.isArray(j.apps) ? j.apps : [j.apps]).map((a) => ({ name: a.name, pid: a.pid2, title: a.title })) }; |
| 291 | }, |
| 292 | list_windows: async (args = {}) => { |
| 293 | if (Object.hasOwn(args, "app_ref") || Object.hasOwn(args, "window_id")) throw unsupportedSelector("Windows list_windows does not support app_ref or window_id; omit them to list all windows"); |
| 294 | const j = await psJson(`$ErrorActionPreference = 'Stop'; |
| 295 | Add-Type -AssemblyName System.Windows.Forms; |
| 296 | Add-Type -TypeDefinition @' |
| 297 | using System; |
| 298 | using System.Text; |
| 299 | using System.Collections.Generic; |
| 300 | using System.Runtime.InteropServices; |
| 301 | public static class WinEnum { |
| 302 | [DllImport("user32.dll")] static extern bool EnumWindows(EnumWindowsProc lpEnumFunc, IntPtr lParam); |
| 303 | [DllImport("user32.dll")] static extern bool IsWindowVisible(IntPtr hWnd); |
| 304 | [DllImport("user32.dll")] static extern int GetWindowText(IntPtr hWnd, StringBuilder text, int count); |
| 305 | [DllImport("user32.dll")] static extern int GetWindowTextLength(IntPtr hWnd); |
| 306 | [DllImport("user32.dll")] static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint pid); |
| 307 | [DllImport("user32.dll")] static extern bool GetWindowRect(IntPtr hWnd, out RECT rect); |
| 308 | [StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left; public int Top; public int Right; public int Bottom; } |
| 309 | delegate bool EnumWindowsProc(IntPtr hWnd, IntPtr lParam); |
| 310 | public static List<string> List() { |
| 311 | var result = new List<string>(); |
| 312 | EnumWindows((h, l) => { |
| 313 | if (!IsWindowVisible(h)) return true; |
| 314 | int len = GetWindowTextLength(h); |
| 315 | var sb = new StringBuilder(len + 1); |
| 316 | GetWindowText(h, sb, sb.Capacity); |
| 317 | uint pid; GetWindowThreadProcessId(h, out pid); |
| 318 | RECT r; GetWindowRect(h, out r); |
| 319 | if (sb.Length > 0 && r.Right > r.Left) |
| 320 | result.Add(pid + "|" + r.Left + "," + r.Top + "," + (r.Right - r.Left) + "," + (r.Bottom - r.Top) + "|" + sb.ToString()); |
| 321 | return true; |
| 322 | }, IntPtr.Zero); |
| 323 | return result; |
| 324 | } |
| 325 | } |
| 326 | '@; |
| 327 | $json = [WinEnum]::List() | ForEach-Object { $p = $_.Split('|', 2); $parts = $p[1].Split('|', 2); [pscustomobject]@{ pid2 = [int]$p[0]; geom = $parts[0]; title = $parts[1] } } | ConvertTo-Json -Compress; |
| 328 | if (-not $json) { $json = '[]' } |
| 329 | Write-Output ('{"windows": ' + $json + '}');`, { timeoutMs: 25_000 }); |
| 330 | return { |
| 331 | windows: (Array.isArray(j.windows) ? j.windows : [j.windows]).map((w) => { |
| 332 | const g = String(w.geom).split(",").map(Number); |
| 333 | return { pid: w.pid2, title: w.title, position: { x: g[0], y: g[1] }, size: { w: g[2], h: g[3] } }; |
| 334 | }), |
| 335 | }; |
| 336 | }, |
| 337 | open_application: async ({ name, bundle_id: bid, url: urlArg, activate } = {}) => { |
| 338 | const target = name ?? bid; |
| 339 | if (typeof target !== "string" || !/^[A-Za-z0-9][A-Za-z0-9 .:_-]*$/.test(target)) throw new ExecError("open_application needs a plain app or executable name"); |
| 340 | let argumentsScript = ""; |
| 341 | if (urlArg != null) { |
| 342 | if (typeof urlArg !== "string" || !URL.canParse(urlArg) || /[\0\r\n]/.test(urlArg)) throw new ExecError("open_application url must be an absolute URL"); |
| 343 | // Start-Process joins ArgumentList into a Windows command line. Quote |
| 344 | // one argument there, and transport that string as data into PowerShell. |
| 345 | const quoted = '"' + urlArg.replace(/(\\*)"/g, '$1$1\\"').replace(/(\\+)$/g, '$1$1') + '"'; |
| 346 | const encoded = Buffer.from(quoted, "utf16le").toString("base64"); |
| 347 | argumentsScript = `$launchArg = [Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${encoded}')); `; |
| 348 | } |
| 349 | // activate defaults to background on every platform: a minimized |
| 350 | // launch leaves the user's foreground window alone. Windows input is |
| 351 | // still shared-surface — this only controls the launch, not input. |
| 352 | const windowStyle = activate === true ? "" : " -WindowStyle Minimized"; |
| 353 | const r = await psOk(`${argumentsScript}Start-Process -FilePath "${target}"${windowStyle}${urlArg != null ? " -ArgumentList $launchArg" : ""}; Write-Output '{"launched": true}'`, { timeoutMs: 20_000 }); |
| 354 | if (r.code !== 0) throw new ExecError(`Start-Process failed: ${r.stderr.trim().slice(0, 200)}`, r); |
| 355 | return { launched: true, name: target, url: urlArg ?? null, activate: activate === true }; |
| 356 | }, |
| 357 | get_app_state: async (args = {}) => { |
| 358 | if (Object.hasOwn(args, "window_id")) throw unsupportedSelector("Windows get_app_state does not support window_id"); |
| 359 | const { app_ref, detail } = args; |
| 360 | if (Object.hasOwn(args, "app_ref") && (!app_ref || typeof app_ref !== "object" || Array.isArray(app_ref) |
| 361 | || Object.keys(app_ref).length !== 1 || !Object.hasOwn(app_ref, "name") || typeof app_ref.name !== "string" || !app_ref.name.trim())) { |
| 362 | throw unsupportedSelector("Windows get_app_state supports only app_ref: { name: exact window title }; PID, bundle_id and other references are unsupported"); |
| 363 | } |
| 364 | const filter = Buffer.from(app_ref?.name ?? "", "utf16le").toString("base64"); |
| 365 | const maxEls = detail === "full" ? 800 : 400; |
| 366 | const j = await psJson(`${UIA_PRELUDE} |
| 367 | $max = ${maxEls}; |
| 368 | $filter = [Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${filter}')); |
| 369 | $root = [System.Windows.Automation.AutomationElement]::RootElement; |
| 370 | $els = New-Object System.Collections.ArrayList; |
| 371 | $found = $false; $truncated = $false; $appName = $null; |
| 372 | $targets = @($root.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition)); |
| 373 | if ($filter) { |
| 374 | $targets = @($targets | Where-Object { [string]::Equals($_.Current.Name, $filter, [StringComparison]::OrdinalIgnoreCase) }); |
| 375 | if ($targets.Count -gt 1) { throw 'More than one application window has this exact name' } |
| 376 | } |
| 377 | foreach ($t in $targets) { |
| 378 | $nm = $t.Current.Name; |
| 379 | $found = $true; $appName = $nm; |
| 380 | $stack = New-Object System.Collections.Stack; |
| 381 | $windowId = @($t.GetRuntimeId()); |
| 382 | $stack.Push(@($t, @(0))); |
| 383 | while ($stack.Count -gt 0) { |
| 384 | $entry = $stack.Pop(); $cur = $entry[0]; $path = $entry[1]; |
| 385 | if ($els.Count -ge $max) { $truncated = $true; break } |
| 386 | $rect = $cur.Current.BoundingRectangle; |
| 387 | $acts = @(); |
| 388 | try { $acts = @($cur.GetSupportedPatterns() | ForEach-Object { $_.ProgrammaticName -replace 'PatternIdentifiers\\.Pattern$','' -replace 'Pattern$','' }) } catch {} |
| 389 | $value = ''; $vp = $null; |
| 390 | if (-not $cur.Current.IsPassword -and $cur.TryGetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern, [ref]$vp)) { $value = [string]$vp.Current.Value } |
| 391 | [void]$els.Add([pscustomobject]@{ index = $els.Count; path = @($path); runtime_id = @($cur.GetRuntimeId()); window_runtime_id = $windowId; role = [string]$cur.Current.ControlType.ProgrammaticName; label = [string]$cur.Current.Name; value = $value.Substring(0, [Math]::Min(120, $value.Length)); enabled = $cur.Current.IsEnabled; |
| 392 | x = [int]$rect.X; y = [int]$rect.Y; w = [int]$rect.Width; h = [int]$rect.Height; actions = $acts }); |
| 393 | $kids = $cur.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition); |
| 394 | for ($i = $kids.Count - 1; $i -ge 0; $i--) { $stack.Push(@($kids[$i], ($path + $i))) } |
| 395 | } |
| 396 | break; |
| 397 | } |
| 398 | $result = [pscustomobject]@{ found = $found; name = $appName; truncated = $truncated; elements = @($els | ForEach-Object { [pscustomobject]@{ index = $_.index; path = @($_.path); runtime_id = $_.runtime_id; window_runtime_id = $_.window_runtime_id; role = ($_.role -replace 'ControlType.',''); label = $_.label; value = $_.value; enabled = $_.enabled; position = [pscustomobject]@{ x = $_.x; y = $_.y }; size = [pscustomobject]@{ w = $_.w; h = $_.h }; actions = $_.actions } }) }; |
| 399 | Write-Output ($result | ConvertTo-Json -Depth 6 -Compress);`, { timeoutMs: 60_000 }); |
| 400 | if (!j.found) throw new ExecError("application window not found in UIA tree — pass app_ref.name as the exact window title from list_windows or list_apps.title"); |
| 401 | return j; |
| 402 | }, |
| 403 | screenshot: async (args = {}) => { |
| 404 | if (Object.hasOwn(args, "app_ref") || Object.hasOwn(args, "window_id")) throw unsupportedSelector("Windows screenshot does not support app_ref or window_id; omit them for a desktop screenshot"); |
| 405 | const { display = activeDisplay, region, path: outPath } = args; |
| 406 | if (display != null && (!Number.isInteger(display) || display < 1)) throw new ExecError("display index must be a positive integer"); |
| 407 | if (region != null && (!Array.isArray(region) || region.length !== 4 || !region.every(Number.isInteger) || region[2] <= 0 || region[3] <= 0)) throw new ExecError("region must be integer [x,y,width,height] with positive size"); |
| 408 | const dir = recordingsDir(); |
| 409 | fs.mkdirSync(dir, { recursive: true }); |
| 410 | const file = path.resolve(outPath || path.join(dir, `shot-${crypto.randomBytes(6).toString("hex")}.png`)); |
| 411 | const meta = await psJson(`Add-Type -AssemblyName System.Windows.Forms; Add-Type -AssemblyName System.Drawing; |
| 412 | $bounds = [System.Windows.Forms.SystemInformation]::VirtualScreen; |
| 413 | ${display == null ? "" : `$screens = [System.Windows.Forms.Screen]::AllScreens; if (${display} -gt $screens.Count) { throw 'display index is out of range' }; $bounds = $screens[${display - 1}].Bounds;`} |
| 414 | ${region == null ? "" : `$crop = New-Object System.Drawing.Rectangle(${region.join(",")}); if (-not $bounds.Contains($crop)) { throw 'region is outside capture bounds' }; $bounds = $crop;`} |
| 415 | $bmp = New-Object System.Drawing.Bitmap($bounds.Width, $bounds.Height); |
| 416 | try { |
| 417 | $g = [System.Drawing.Graphics]::FromImage($bmp); |
| 418 | try { $g.CopyFromScreen($bounds.X, $bounds.Y, 0, 0, $bounds.Size); } finally { $g.Dispose(); } |
| 419 | $bmp.Save('${file.replace(/'/g, "''")}', [System.Drawing.Imaging.ImageFormat]::Png); |
| 420 | } finally { $bmp.Dispose(); } |
| 421 | @{ x = $bounds.X; y = $bounds.Y; w = $bounds.Width; h = $bounds.Height } | ConvertTo-Json -Compress;`, { timeoutMs: 30_000 }); |
| 422 | if (!fs.existsSync(file) || ![meta.x, meta.y, meta.w, meta.h].every(Number.isFinite) || meta.w <= 0 || meta.h <= 0) throw new ExecError("screenshot did not return a valid raster"); |
| 423 | lastRaster = { file, bytes: fs.statSync(file).size, points: { x: meta.x, y: meta.y, w: meta.w, h: meta.h }, pixels: { w: meta.w, h: meta.h }, scale: 1, capturedAt: new Date().toISOString() }; |
| 424 | return { ...lastRaster }; |
| 425 | }, |
| 426 | zoom: async ({ source, region, path: outPath }) => { |
| 427 | const src = source ?? lastRaster?.file; |
| 428 | if (!src) throw new ExecError("no screenshot taken yet on this computer — call screenshot first"); |
| 429 | const out = outPath || path.join(recordingsDir(), `zoom-${crypto.randomBytes(4).toString("hex")}.png`); |
| 430 | const script = `Add-Type -AssemblyName System.Drawing; |
| 431 | $img = [System.Drawing.Image]::FromFile('${src.replace(/'/g, "''")}'); |
| 432 | $rect = New-Object System.Drawing.Rectangle(${Math.round(region[0])}, ${Math.round(region[1])}, ${Math.round(region[2])}, ${Math.round(region[3])}); |
| 433 | $bmp = New-Object System.Drawing.Bitmap($rect.Width, $rect.Height); |
| 434 | $g = [System.Drawing.Graphics]::FromImage($bmp); |
| 435 | $g.DrawImage($img, (New-Object System.Drawing.Rectangle(0, 0, $rect.Width, $rect.Height)), $rect, [System.Drawing.GraphicsUnit]::Pixel); |
| 436 | $g.Dispose(); |
| 437 | $bmp.Save('${out.replace(/'/g, "''")}', [System.Drawing.Imaging.ImageFormat]::Png); |
| 438 | $bmp.Dispose(); $img.Dispose(); |
| 439 | Write-Output '{"ok": true}';`; |
| 440 | const r = await psOk(script, { timeoutMs: 20_000 }); |
| 441 | if (r.code !== 0 || !fs.existsSync(out)) throw new ExecError(`zoom failed: ${(r.stderr || "").slice(0, 250)}`, r); |
| 442 | return { file: out, bytes: fs.statSync(out).size, region, source: src }; |
| 443 | }, |
| 444 | left_click: ({ target, strategy }) => { assertEventStrategy(strategy); return clickAt(0, target.x, target.y, 1); }, |
| 445 | double_click: ({ target }) => clickAt(0, target.x, target.y, 2), |
| 446 | triple_click: ({ target }) => clickAt(0, target.x, target.y, 3), |
| 447 | right_click: ({ target }) => clickAt(1, target.x, target.y, 1), |
| 448 | middle_click: ({ target }) => clickAt(2, target.x, target.y, 1), |
| 449 | mouse_move: async ({ target }) => { |
| 450 | await withUser32(`[User32]::SetCursorPos(${Math.round(target.x)}, ${Math.round(target.y)}) | Out-Null; Write-Output '{"ok": true}'`); |
| 451 | return { action_sent: true, at: { x: target.x, y: target.y } }; |
| 452 | }, |
| 453 | left_click_drag: async ({ from_target: from, to }) => { |
| 454 | requireInputOwner(); |
| 455 | throwIfAborted(); |
| 456 | heldButtons.add("LEFT"); |
| 457 | try { |
| 458 | await withUser32(`[User32]::SetCursorPos(${Math.round(from.x)}, ${Math.round(from.y)}) | Out-Null; |
| 459 | Start-Sleep -Milliseconds 80; |
| 460 | [User32]::mouse_event([User32]::LEFTDOWN, 0, 0, 0, [UIntPtr]::Zero); |
| 461 | Start-Sleep -Milliseconds 80; |
| 462 | [User32]::SetCursorPos(${Math.round(to.x)}, ${Math.round(to.y)}) | Out-Null; |
| 463 | Start-Sleep -Milliseconds 80; |
| 464 | [User32]::mouse_event([User32]::LEFTUP, 0, 0, 0, [UIntPtr]::Zero); |
| 465 | Write-Output '{"ok": true}';`, { timeoutMs: 20_000 }); |
| 466 | heldButtons.delete("LEFT"); |
| 467 | return { action_sent: true, from, to }; |
| 468 | } finally { await releaseInput({ buttons: ["LEFT"], keys: [] }); } |
| 469 | }, |
| 470 | left_mouse_down: async ({ target }) => { |
| 471 | requireInputOwner(); |
| 472 | // Ternary must select ONLY the optional move prefix; the LEFTDOWN press |
| 473 | // always runs, so a targeted press both moves and presses. |
| 474 | const move = target ? `[User32]::SetCursorPos(${Math.round(target.x)}, ${Math.round(target.y)}) | Out-Null;\n` : ""; |
| 475 | throwIfAborted(); |
| 476 | heldButtons.add("LEFT"); |
| 477 | try { await withUser32(`${move}[User32]::mouse_event([User32]::LEFTDOWN, 0, 0, 0, [UIntPtr]::Zero); Write-Output '{"ok": true}'`); } |
| 478 | catch (err) { await releaseInput({ buttons: ["LEFT"], keys: [] }); throw err; } |
| 479 | return { action_sent: true }; |
| 480 | }, |
| 481 | left_mouse_up: async () => { |
| 482 | if (!heldButtons.has("LEFT")) throw Object.assign(new ExecError("no agent pointer press to release"), { code: "input_not_held" }); |
| 483 | await releaseInput({ buttons: ["LEFT"], keys: [] }); |
| 484 | return { action_sent: true }; |
| 485 | }, |
| 486 | scroll: async ({ target, direction = "down", amount = 3 }) => { |
| 487 | const notches = Math.max(1, Math.min(30, amount)); |
| 488 | // Preserve signed wheel deltas as DWORD bits before PowerShell converts |
| 489 | // the argument: its signed 0xFFFFFFFF literal cannot mask negatives. |
| 490 | const data = (direction === "down" ? -1 : direction === "up" ? 1 : 0) * notches * 120; |
| 491 | const hdata = (direction === "right" ? 1 : direction === "left" ? -1 : 0) * notches * 120; |
| 492 | await withUser32(`[User32]::SetCursorPos(${Math.round(target.x)}, ${Math.round(target.y)}) | Out-Null; |
| 493 | Start-Sleep -Milliseconds 60; |
| 494 | [User32]::mouse_event([User32]::WHEEL, 0, 0, ${data >>> 0}, [UIntPtr]::Zero); |
| 495 | [User32]::mouse_event([User32]::HWHEEL, 0, 0, ${hdata >>> 0}, [UIntPtr]::Zero); |
| 496 | Write-Output '{"ok": true}';`); |
| 497 | return { action_sent: true, direction, amount }; |
| 498 | }, |
| 499 | type: async ({ text }) => { |
| 500 | if (!text) return { action_sent: false, note: "empty text" }; |
| 501 | const b64 = Buffer.from(String(text), "utf16le").toString("base64"); |
| 502 | const script = `$text = [System.Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${b64}')); |
| 503 | [User32]::SendString($text); |
| 504 | Write-Output ('{"ok": true, "chars": ' + $text.Length + '}');`; |
| 505 | await withUser32(script, { timeoutMs: Math.max(20_000, text.length * 60) }); |
| 506 | return { action_sent: true, chars: text.length, strategy: "unicode-sendinput" }; |
| 507 | }, |
| 508 | key: async ({ text, repeat = 1 }) => { |
| 509 | const { key, vk, mods } = keyChord(text); |
| 510 | return withKeys([...mods, vk], async () => { |
| 511 | const n = Math.max(1, Math.min(100, repeat)); |
| 512 | await withUser32(`$seq = @(); |
| 513 | ${mods.map((m) => `$seq += [User32]::KeyInput(${m}, 0, 0);`).join("\n")} |
| 514 | for ($i = 0; $i -lt ${n}; $i++) { $seq += [User32]::KeyInput(${vk}, 0, 0); $seq += [User32]::KeyInput(${vk}, 0, 2); } |
| 515 | ${[...mods].reverse().map((m) => `$seq += [User32]::KeyInput(${m}, 0, 2);`).join("\n")} |
| 516 | [User32]::SendChecked([User32+INPUT[]]$seq); |
| 517 | Write-Output '{"ok": true}';`); |
| 518 | return { action_sent: true, key, repeat: n }; |
| 519 | }); |
| 520 | }, |
| 521 | hold_key: async ({ text, duration }) => { |
| 522 | requireInputOwner(); |
| 523 | const { key, vk, mods } = keyChord(text); |
| 524 | const d = Math.max(0.05, Math.min(30, Number(duration) || 1)); |
| 525 | const keys = [...mods, vk]; |
| 526 | const event = (code, flags) => `[User32]::SendKey(${code}, ${flags});`; |
| 527 | return withKeys(keys, async () => { |
| 528 | await withUser32(`${keys.map((code) => event(code, 0)).join("\n")} |
| 529 | Start-Sleep -Milliseconds ${Math.round(d * 1000)}; |
| 530 | ${[...keys].reverse().map((code) => event(code, 2)).join("\n")} |
| 531 | Write-Output '{"ok": true}';`, { timeoutMs: Math.max(10_000, d * 1000 + 8000) }); |
| 532 | return { action_sent: true, key, heldSec: d }; |
| 533 | }); |
| 534 | }, |
| 535 | set_value: async ({ target, value }) => { |
| 536 | const resolve = elementScript(target); |
| 537 | const b64 = Buffer.from(String(value ?? ""), "utf16le").toString("base64"); |
| 538 | const j = await psJson(`${resolve} |
| 539 | $val = [Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${b64}')); |
| 540 | $vp = $cur.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern); |
| 541 | if ($vp.Current.IsReadOnly) { throw 'element_read_only' } |
| 542 | $vp.SetValue($val); |
| 543 | @{ ok = $true; verified = ($vp.Current.Value -ceq $val) } | ConvertTo-Json -Compress;`, { timeoutMs: 45_000 }); |
| 544 | if (!j.ok) throw new ExecError("set_value failed"); |
| 545 | return { action_sent: true, strategy: "a11y", verified: j.verified === true }; |
| 546 | }, |
| 547 | select_text: async () => { throw new ExecError("select_text is not implemented on the win32 backend yet — fail-closed"); }, |
| 548 | perform_action: async ({ target, action = "Invoke" }) => { |
| 549 | const resolve = elementScript(target); |
| 550 | const actions = { invoke: ["Invoke", "Invoke"], click: ["Invoke", "Invoke"], toggle: ["Toggle", "Toggle"], expand: ["ExpandCollapse", "Expand"], expandcollapse: ["ExpandCollapse", "Expand"], collapse: ["ExpandCollapse", "Collapse"], select: ["SelectionItem", "Select"], selectionitem: ["SelectionItem", "Select"] }; |
| 551 | const chosen = actions[String(action).toLowerCase()]; |
| 552 | if (!chosen) throw new ExecError("unsupported UIA action"); |
| 553 | await psOk(`${resolve} |
| 554 | $pattern = $cur.GetCurrentPattern([System.Windows.Automation.${chosen[0]}Pattern]::Pattern); |
| 555 | $pattern.${chosen[1]}();`, { timeoutMs: 45_000 }); |
| 556 | return { action_sent: true, strategy: "a11y", action }; |
| 557 | }, |
| 558 | read_clipboard: async () => { |
| 559 | const j = await psJson(`$t = Get-Clipboard -Raw -ErrorAction SilentlyContinue; |
| 560 | @{ text = [string]$t } | ConvertTo-Json -Compress;`, { timeoutMs: 10_000 }); |
| 561 | return { text: j.text ?? "", encoding: "utf8" }; |
| 562 | }, |
| 563 | write_clipboard: async ({ text }) => { |
| 564 | const b64 = Buffer.from(String(text ?? ""), "utf16le").toString("base64"); |
| 565 | await psOk(`Set-Clipboard -Value ([System.Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${b64}'))); |
| 566 | Write-Output '{"ok": true}';`, { timeoutMs: 10_000 }); |
| 567 | return { written: String(text ?? "").length }; |
| 568 | }, |
| 569 | cursor_position: async () => { |
| 570 | const j = await psJson(`${USER32_PRELUDE} |
| 571 | $p = New-Object User32+POINT; |
| 572 | [void][User32]::GetCursorPos([ref]$p); |
| 573 | Write-Output ('{"x": ' + $p.X + ', "y": ' + $p.Y + '}');`); |
| 574 | return { x: j.x, y: j.y }; |
| 575 | }, |
| 576 | recordingStart: async () => { |
| 577 | throw Object.assign(new ExecError("Recording is unavailable on this platform until the recorder has session-owned cleanup. Use screenshots instead."), { code: "owned_recording_unavailable" }); |
| 578 | }, |
| 579 | recordingStop: async ({ id }) => { throw new ExecError(`unknown recording "${id}"`); }, |
| 580 | recordingStatus: ({ id }) => ({ id, running: false }), |
| 581 | recordingList: async () => { |
| 582 | const dir = recordingsDir(); |
| 583 | const out = fs.existsSync(dir) |
| 584 | ? fs.readdirSync(dir).filter((f) => /\.mp4$/i.test(f)).map((f) => { |
| 585 | const st = fs.statSync(path.join(dir, f)); |
| 586 | return { file: path.join(dir, f), bytes: st.size, modifiedAt: st.mtime.toISOString() }; |
| 587 | }).sort((a, b) => b.modifiedAt.localeCompare(a.modifiedAt)).slice(0, 50) |
| 588 | : []; |
| 589 | return { dir, recordings: out, running: [] }; |
| 590 | }, |
| 591 | }; |
| 592 | |
| 593 | async function clickAt(button, x, y, clicks) { |
| 594 | if (!Number.isFinite(Number(x)) || !Number.isFinite(Number(y))) throw new ExecError("coordinates must be finite numbers"); |
| 595 | const flags = button === 1 ? "RIGHTDOWN, RIGHTUP" : button === 2 ? "MIDDLEDOWN, MIDDLEUP" : "LEFTDOWN, LEFTUP"; |
| 596 | const seq = []; |
| 597 | for (let i = 0; i < clicks; i++) seq.push(`[User32]::mouse_event([User32]::${flags.split(",")[0].trim()}, 0, 0, 0, [UIntPtr]::Zero); Start-Sleep -Milliseconds 40; [User32]::mouse_event([User32]::${flags.split(",")[1].trim()}, 0, 0, 0, [UIntPtr]::Zero); Start-Sleep -Milliseconds 60;`); |
| 598 | const held = button === 1 ? "RIGHT" : button === 2 ? "MIDDLE" : "LEFT"; |
| 599 | throwIfAborted(); |
| 600 | heldButtons.add(held); |
| 601 | try { |
| 602 | await withUser32(`[User32]::SetCursorPos(${Math.round(x)}, ${Math.round(y)}) | Out-Null; |
| 603 | Start-Sleep -Milliseconds 60; |
| 604 | ${seq.join("\n")} |
| 605 | Write-Output '{"ok": true}';`, { timeoutMs: 20_000 }); |
| 606 | heldButtons.delete(held); |
| 607 | return { action_sent: true, at: { x: Number(x), y: Number(y) }, button, clicks }; |
| 608 | } finally { await releaseInput({ buttons: [held], keys: [] }); } |
| 609 | } |
| 610 | } |
| 611 | |
| 612 | export default { create }; |
| 613 |