返回 CodeWhale
linux.mjs
根目录 / crates / tui / plugins / computer-use / src / backends / linux.mjs
1 // Linux backend — X11 first (xdotool/wmctrl/scrot/xclip), Wayland where the
2 // right tools exist (grim/wtype/ydotool/wf-recorder/wl-clipboard). The
3 // accessibility tree comes from AT-SPI via python3+pyatspi when installed.
4 // Everything probes at call time and fails closed with the missing tool named.
5 import fs from "node:fs";
6 import os from "node:os";
7 import path from "node:path";
8 import crypto from "node:crypto";
9 import { spawn } from "node:child_process";
10 import { run as nativeRun, runOk, ExecError, tryJson, have as nativeHave, withSignal, throwIfAborted, wait } from "../exec.mjs";
11 import { pngSize } from "../png-size.mjs";
12 import { createBrowser } from "../browser-cdp.mjs";
13
14 const XKEYS = {
15 return: "Return", enter: "Return", tab: "Tab", escape: "Escape", esc: "Escape",
16 space: "space", backspace: "BackSpace", delete: "Delete", home: "Home", end: "End",
17 pageup: "Page_Up", pagedown: "Page_Down", left: "Left", right: "Right", up: "Up",
18 down: "Down", capslock: "Caps_Lock", menu: "Menu", print: "Print",
19 };
20
21 function spawnDetached(cmd, args, stdinText = "", quiet = true) {
22 const child = spawn(cmd, args, {
23 stdio: stdinText ? ["pipe", "ignore", quiet ? "ignore" : "pipe"] : ["ignore", "ignore", quiet ? "ignore" : "pipe"],
24 detached: true,
25 });
26 if (stdinText) child.stdin.end(stdinText);
27 child.unref();
28 return child;
29 }
30
31 /** Coordinate clicks on this backend are always raw pointer events; strategy="a11y" must fail closed rather than silently degrade. */
32 function assertEventStrategy(strategy) {
33 if (strategy != null && strategy !== "auto" && strategy !== "event") {
34 throw new ExecError(`strategy "${strategy}" is macOS-only; this backend dispatches coordinate clicks as raw pointer events — use an element target for a semantic action`);
35 }
36 }
37
38 function appName(ref) {
39 if (ref === undefined) return "";
40 if (!ref || typeof ref !== "object" || Array.isArray(ref) || Object.keys(ref).length !== 1 || typeof ref.name !== "string" || !ref.name.trim()) {
41 throw Object.assign(new ExecError("Linux accessibility targeting supports only a nonblank app_ref.name; PID and bundle selectors are unavailable"), { code: "unsupported_selector" });
42 }
43 return ref.name;
44 }
45
46 function rejectWindowSelectors(args) {
47 if (Object.hasOwn(args, "app_ref") || Object.hasOwn(args, "window_id")) throw Object.assign(new ExecError("Linux window listing and screenshots do not support app_ref or window_id selectors"), { code: "unsupported_selector" });
48 }
49
50 function assertAppRootWindow(index, windowId) {
51 if (windowId !== undefined || (index !== undefined && index !== 0)) throw Object.assign(new ExecError("Linux accessibility paths start at the app root; window selectors are unavailable"), { code: "unsupported_selector" });
52 }
53
54 function outputPath(file) {
55 if (typeof file !== "string" || !path.isAbsolute(file) || file.includes("\0")) throw new ExecError("output path must be an absolute filename");
56 return file;
57 }
58
59 export function create({ exec } = {}) {
60 const run = exec?.run ?? nativeRun;
61 const have = exec?.have ?? nativeHave;
62 const browser = createBrowser({ platform: "linux" });
63 function requireInputOwner() {
64 if (exec?.persistentInputOwner !== true) throw Object.assign(new ExecError(
65 "This held-input gesture requires a connected Codewhale Computer Use desktop helper so a disconnected client cannot leave keys or buttons pressed. Start the helper and reconnect before retrying."
66 ), { code: "input_owner_required" });
67 }
68
69 const tools = {};
70 let session = null; // "x11" | "wayland"
71 let probed = false;
72 let lastRaster = null;
73 let mouseHeld = false;
74 const heldKeys = new Set();
75
76 async function releaseMouse() {
77 if (!mouseHeld) return;
78 await withSignal(null, () => session === "x11" ? xdotool(["mouseup", "1"], { timeoutMs: 2_000 }) : ydotool(["click", "0x80"], { timeoutMs: 2_000 }));
79 mouseHeld = false;
80 }
81
82 async function releaseKey(key) {
83 if (!heldKeys.has(key)) return;
84 await withSignal(null, () => xdotool(["keyup", key], { timeoutMs: 2_000 }));
85 heldKeys.delete(key);
86 }
87
88 async function releaseInput() {
89 await releaseMouse();
90 for (const key of heldKeys) await releaseKey(key);
91 }
92
93 async function probeSession() {
94 if (probed) return session;
95 throwIfAborted();
96 const wayland = !!(process.env.WAYLAND_DISPLAY || process.env.XDG_SESSION_TYPE === "wayland");
97 const x11 = !!(process.env.DISPLAY || process.env.XDG_SESSION_TYPE === "x11");
98 session = wayland && !x11 ? "wayland" : x11 ? "x11" : null;
99 if (session === null) {
100 const e = new ExecError("no X11 ($DISPLAY) or Wayland ($WAYLAND_DISPLAY) session visible to this process — set DISPLAY or run inside the desktop session");
101 e.code = "no_session";
102 throw e;
103 }
104 for (const t of ["xdotool", "wmctrl", "scrot", "import", "grim", "slurp", "wtype", "ydotool", "wf-recorder", "ffmpeg", "xclip", "xsel", "wl-copy", "wl-paste", "python3", "xrandr", "swaymsg", "hyprctl"]) {
105 tools[t] = await have(t);
106 }
107 tools.pyatspi = tools.python3 && (await run("python3", ["-c", "import pyatspi"], { timeoutMs: 10_000 })).code === 0;
108 throwIfAborted();
109 probed = true;
110 return session;
111 }
112
113 function need(tool, purpose) {
114 if (!tools[tool]) throw new ExecError(`linux backend needs "${tool}" for ${purpose} — install it and retry`);
115 }
116
117 async function shotTool() {
118 if (session === "wayland") { need("grim", "screenshots on Wayland"); return { cmd: "grim", base: [] }; }
119 if (session === "x11") {
120 if (tools.scrot) return { cmd: "scrot", base: ["-z"] };
121 need("import", "screenshots on X11 (imagemagick)");
122 return { cmd: "import", base: ["-window", "root"] };
123 }
124 throw new ExecError("no X11 ($DISPLAY) or Wayland ($WAYLAND_DISPLAY) session visible to this process");
125 }
126
127 /** Capture a PNG to `file`, optionally cropped to region [x,y,w,h] points. */
128 async function takeShot(file, region) {
129 outputPath(file);
130 const { cmd, base } = await shotTool();
131 let args = [...base];
132 if (cmd === "grim") {
133 if (region) args.push("-g", `${Math.round(region[0])},${Math.round(region[1])} ${Math.round(region[2])}x${Math.round(region[3])}`);
134 args.push(file);
135 } else if (cmd === "scrot") {
136 if (region) args.push("-a", `${Math.round(region[0])},${Math.round(region[1])},${Math.round(region[2])},${Math.round(region[3])}`);
137 args.push(file);
138 } else {
139 if (region) args.push("-crop", `${Math.round(region[2])}x${Math.round(region[3])}+${Math.round(region[0])}+${Math.round(region[1])}`);
140 args.push(file);
141 }
142 const r = await run(cmd, args, { timeoutMs: 10_000 });
143 if (r.code !== 0) throw new ExecError(`${cmd} exited ${r.code}: ${r.stderr.trim().slice(0, 300)}`, r);
144 }
145
146 function recordingsDir() {
147 return path.resolve(process.env.CODEWHALE_CU_RECORDINGS_DIR || path.join(os.homedir(), ".codewhale-cu", "recordings"));
148 }
149
150 async function xdotool(args, opts = {}) {
151 need("xdotool", "input on X11");
152 throwIfAborted();
153 const r = await run("xdotool", args, opts);
154 throwIfAborted();
155 if (r.code !== 0) throw new ExecError(`xdotool ${args[0]} exited ${r.code}: ${r.stderr.trim().slice(0, 200)}`, r);
156 return r.stdout.trim();
157 }
158
159 async function ydotool(args, opts = {}) {
160 need("ydotool", "input on Wayland (ydotool needs its daemon running: sudo ydotoold)");
161 throwIfAborted();
162 const r = await run("ydotool", args, opts);
163 throwIfAborted();
164 if (r.code !== 0) throw new ExecError(`ydotool exited ${r.code}: ${r.stderr.trim().slice(0, 200)}`, r);
165 return r.stdout.trim();
166 }
167
168 function xdotoolKey(text) {
169 return String(text).split("+").map((p) => {
170 const k = p.trim().toLowerCase();
171 if (XKEYS[k]) return XKEYS[k];
172 if (/^f\d{1,2}$/.test(k)) return k.toUpperCase();
173 return p.trim(); // pass through names already in xdotool form
174 }).join("+");
175 }
176
177 async function waylandKey(text, { repeat = 1, holdMs = 0 } = {}) {
178 need("wtype", "key presses on Wayland");
179 const parts = String(text).split("+").map((part) => part.trim().toLowerCase());
180 const aliases = { control: "ctrl", meta: "logo", cmd: "logo", super: "logo" };
181 const modifiers = new Set(["ctrl", "alt", "shift", "logo", "win", "altgr", "capslock"]);
182 const rawKey = parts.pop();
183 const key = xdotoolKey(rawKey);
184 const mods = parts.map((part) => aliases[part] ?? part);
185 if (!key || mods.some((mod) => !modifiers.has(mod))) throw new ExecError(`unknown key combination "${text}"`);
186 const modKey = aliases[rawKey] ?? rawKey;
187 const onlyModifier = modifiers.has(modKey);
188 const args = mods.flatMap((mod) => ["-M", mod]);
189 for (let i = 0; i < repeat; i++) {
190 args.push(onlyModifier ? "-M" : "-P", onlyModifier ? modKey : key);
191 if (holdMs) args.push("-s", String(holdMs));
192 args.push(onlyModifier ? "-m" : "-p", onlyModifier ? modKey : key);
193 }
194 args.push(...mods.reverse().flatMap((mod) => ["-m", mod]));
195 // wtype owns a temporary Wayland keyboard; the compositor releases its
196 // keys on process exit, including cancellation. Keep the complete gesture
197 // in one process (https://github.com/atx/wtype#usage).
198 throwIfAborted();
199 const result = await run("wtype", args, { timeoutMs: Math.max(10_000, holdMs + 8_000) });
200 throwIfAborted();
201 if (result.code !== 0) throw new ExecError(`wtype exited ${result.code}: ${result.stderr.trim().slice(0, 200)}`, result);
202 }
203
204 function assertNum(v, name) {
205 const n = Number(v);
206 if (!Number.isFinite(n)) throw new ExecError(`${name} must be a finite number`);
207 return n;
208 }
209
210 // ---------- AT-SPI tree ----------
211 const PYATSPI_APP = `def resolve_app(desktop, app_name):
212 apps = [desktop.getChildAtIndex(i) for i in range(desktop.childCount)]
213 if app_name:
214 matches = [app for app in apps if app and app_name.casefold() == (app.name or "").casefold()]
215 return matches[0] if len(matches) == 1 else None
216 return next((app for app in apps if app and app.childCount), None)
217 `;
218 const PYATSPI_WALK = `import json, sys, pyatspi
219 ${PYATSPI_APP}
220 app_name = sys.argv[1] if len(sys.argv) > 1 else None
221 depth_max = int(sys.argv[2]) if len(sys.argv) > 2 else 8
222 max_el = int(sys.argv[3]) if len(sys.argv) > 3 else 400
223 desktop = pyatspi.Registry.getDesktop(0)
224 root = resolve_app(desktop, app_name)
225 if root is None:
226 print(json.dumps({"found": False}))
227 sys.exit(0)
228 els = []
229 truncated = False
230 def info(e, path):
231 ext = None
232 try: ext = e.queryComponent().getExtents(pyatspi.DESKTOP_COORDS)
233 except Exception: pass
234 txt = None
235 try:
236 q = e.queryText()
237 n = min(120, q.characterCount)
238 if n > 0: txt = q.getText(0, n)
239 except Exception: pass
240 acts = []
241 try:
242 a = e.queryAction()
243 acts = [a.getName(i) for i in range(a.nActions)]
244 except Exception: pass
245 els.append({"index": len(els), "path": path, "role": e.getRoleName() if e.getRoleName() else None,
246 "label": e.name or None, "value": txt,
247 "position": {"x": ext.x, "y": ext.y} if ext else None,
248 "size": {"w": ext.width, "h": ext.height} if ext else None,
249 "actions": acts})
250 def walk(e, path, d):
251 global truncated
252 if len(els) >= max_el or d > depth_max:
253 truncated = True
254 return
255 try: info(e, path)
256 except Exception: return
257 for i in range(e.childCount):
258 try: c = e.getChildAtIndex(i)
259 except Exception: continue
260 if c: walk(c, path + [i], d + 1)
261 walk(root, [], 0)
262 print(json.dumps({"found": True, "name": root.name, "elements": els, "truncated": truncated}))`;
263
264 async function atspiResolve(target, pythonBody, extraArg = null) {
265 const name = appName(target.app_ref);
266 assertAppRootWindow(target.windowIndex, target.window_id);
267 await probeSession();
268 need("python3", "semantic element actions (AT-SPI)");
269 const script = `import json, sys, pyatspi
270 ${PYATSPI_APP}
271 desktop = pyatspi.Registry.getDesktop(0)
272 app_name = sys.argv[1] if len(sys.argv) > 1 and sys.argv[1] else None
273 target_path = json.loads(sys.argv[2])
274 extra = sys.argv[3] if len(sys.argv) > 3 else None
275 node = resolve_app(desktop, app_name)
276 if node is None:
277 print(json.dumps({"ok": False, "code": "app_not_found"}))
278 sys.exit(0)
279 found = None
280 stack = [(node, [])]
281 while stack:
282 n, p = stack.pop(0)
283 if p == target_path:
284 found = n
285 break
286 if len(p) > 12: continue
287 try:
288 for i in range(n.childCount):
289 c = n.getChildAtIndex(i)
290 if c: stack.append((c, p + [i]))
291 except Exception: pass
292 if found is None:
293 print(json.dumps({"ok": False, "code": "element_stale"}))
294 sys.exit(0)
295 try:
296 ${pythonBody}
297 except Exception as e:
298 print(json.dumps({"ok": False, "code": str(e)}))`;
299 const argv = ["-c", script, name, JSON.stringify(target.path ?? [])];
300 if (extraArg != null) argv.push(String(extraArg));
301 const r = await run("python3", argv, { timeoutMs: 30_000 });
302 const out = tryJson((r.stdout.trim().split("\n").pop() ?? ""), null);
303 if (!out) throw new ExecError(`AT-SPI action failed: ${(r.stderr || r.stdout).slice(0, 250)}`, r);
304 return out;
305 }
306
307 // ---------- input helpers ----------
308 function clickButton(button, clicks) {
309 if (session === "x11") {
310 const args = ["click"];
311 if (clicks > 1) args.push("--repeat", String(clicks), "--delay", "80");
312 args.push(String(button));
313 return xdotool(args);
314 }
315 // ydotool click mask: down|up|count nibble (0xC0 = left click once, +1 per extra click;
316 // 0x04 bit selects right button, 0x02 middle).
317 const count = Math.max(1, Math.min(3, clicks));
318 const code = button === 3 ? 0xc0 + count + 0x04 : button === 2 ? 0xc0 + count + 0x02 : 0xc0 + count - 1;
319 return ydotool(["click", "0x" + code.toString(16)]);
320 }
321
322 return {
323 platform: "linux",
324 releaseInput,
325 browser_start: browser.start,
326 browser_status: browser.status,
327 browser_navigate: browser.navigate,
328 browser_click: browser.click,
329 browser_type: browser.type,
330 browser_screenshot: browser.screenshot,
331 browser_stop: browser.stop,
332 closeSession: async () => { await browser.close().catch(() => {}); },
333 probe: async () => {
334 const s = await probeSession();
335 const caps = {
336 screenshot: !!((session === "wayland" && tools.grim) || (session === "x11" && (tools.scrot || tools.import))),
337 clipboard: !!(tools.xclip || tools.xsel || (tools["wl-copy"] && tools["wl-paste"])),
338 recording: false,
339 accessibility_tree: tools.pyatspi,
340 held_input: exec?.persistentInputOwner === true && !!(session === "x11" ? tools.xdotool : tools.wtype && tools.ydotool),
341 };
342 const missing = [];
343 if (exec?.persistentInputOwner !== true) missing.push("connected Computer Use desktop helper (held keys, held buttons and drag)");
344 if (session === "x11" && !tools.xdotool) missing.push("xdotool (input)");
345 if (session === "wayland" && !tools.ydotool) missing.push("ydotool+ydotoold (mouse input)");
346 if (session === "wayland" && !tools.grim) missing.push("grim (screenshots)");
347 if (session === "x11" && !tools.scrot && !tools.import) missing.push("scrot or imagemagick (screenshots)");
348 missing.push("session-owned recording (unavailable in this version; use screenshots)");
349 if (!tools.pyatspi) missing.push("python3-pyatspi (accessibility tree)");
350 // Real permission probes, not just `have()`: each check is bounded to 10s.
351 const permissions = { input: "failed", screen_capture: "failed", accessibility: "unavailable" };
352 if (session === "x11" && tools.xdotool) {
353 const r = await run("xdotool", ["getdisplaygeometry"], { timeoutMs: 10_000 });
354 permissions.input = r.code === 0 ? "ok" : "failed";
355 } else if (session === "wayland" && tools.ydotool) {
356 permissions.input = "unavailable"; // ydotool can't be probed without moving the pointer
357 }
358 try {
359 const dir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-probe-"));
360 try {
361 await takeShot(path.join(dir, "probe.png"), [0, 0, 2, 2]);
362 permissions.screen_capture = "ok";
363 } finally {
364 try { fs.rmSync(dir, { recursive: true, force: true }); } catch {}
365 }
366 } catch {}
367 if (tools.pyatspi) {
368 const r = await run("python3", ["-c", "import pyatspi; pyatspi.Registry.getDesktop(0).childCount"], { timeoutMs: 10_000 });
369 permissions.accessibility = r.code === 0 ? "ok" : "failed";
370 }
371 if (permissions.screen_capture === "failed" || permissions.input === "failed") {
372 const bad = [];
373 if (permissions.input === "failed") bad.push(`input (${session === "wayland" ? "ydotool" : "xdotool getdisplaygeometry"})`);
374 if (permissions.screen_capture === "failed") bad.push(`screen_capture (${session === "wayland" ? "grim" : "scrot/import"} probe shot)`);
375 const e = new ExecError(`permission checks failed: ${bad.join("; ")} — permissions ${JSON.stringify(permissions)}`);
376 e.code = "permissions_denied";
377 throw e;
378 }
379 return { platform: "linux", session: s, capabilities: caps, permissions, missing, note: "Every capability probes at call time and fails closed naming the missing tool." };
380 },
381 list_displays: async () => {
382 await probeSession();
383 if (session === "x11" && tools.xrandr) {
384 const r = await runOk("xrandr", ["--query"], { timeoutMs: 15_000 });
385 const displays = [];
386 let i = 1;
387 for (const m of r.stdout.matchAll(/^(\S+) connected (?:primary )?(\d+)x(\d+)\+(\d+)\+(\d+)/gm)) {
388 displays.push({ index: i++, name: m[1], points: { x: Number(m[4]), y: Number(m[5]), w: Number(m[2]), h: Number(m[3]) }, pixels: { w: Number(m[2]), h: Number(m[3]) }, scale: 1, main: /primary/.test(m[0]) || i === 1 });
389 }
390 if (displays.length) return displays;
391 }
392 if (session === "wayland" && tools.swaymsg) {
393 const r = await run("swaymsg", ["-t", "get_outputs", "-r"], { timeoutMs: 15_000 });
394 const outs = tryJson(r.stdout, []);
395 if (Array.isArray(outs) && outs.length) {
396 return outs.map((o, i) => ({ index: i + 1, name: o.name, points: { x: o.rect?.x, y: o.rect?.y, w: o.rect?.width, h: o.rect?.height }, pixels: { w: o.current_mode?.width, h: o.current_mode?.height }, scale: o.scale ?? 1, main: i === 0 }));
397 }
398 }
399 if (session === "wayland" && tools.hyprctl) {
400 const r = await run("hyprctl", ["-j", "monitors"], { timeoutMs: 15_000 });
401 const ms = tryJson(r.stdout, []);
402 if (Array.isArray(ms) && ms.length) {
403 return ms.map((o, i) => ({ index: i + 1, name: o.name, points: { x: o.x, y: o.y, w: o.width, h: o.height }, pixels: { w: o.width, h: o.height }, scale: o.scale ?? 1, main: !!o.main || i === 0 }));
404 }
405 }
406 throw new ExecError("display enumeration needs xrandr (X11) or swaymsg/hyprctl (Wayland) — install one and retry");
407 },
408 switch_display: async ({ index }) => ({ activeDisplay: index ?? 1, note: "linux screenshots grab the compositor's virtual screen; per-display selection applies only where the shot tool supports it" }),
409 list_apps: async () => {
410 await probeSession();
411 if (session === "x11" && tools.wmctrl) {
412 const r = await runOk("wmctrl", ["-lx"], { timeoutMs: 15_000 });
413 const seen = new Map();
414 for (const line of r.stdout.split("\n")) {
415 const parts = line.split(/\s+/);
416 const wmClass = parts[2];
417 if (wmClass) seen.set(wmClass, { name: wmClass.split(".")[0], wm_class: wmClass });
418 }
419 return { apps: [...seen.values()] };
420 }
421 if (session === "wayland" && (tools.swaymsg || tools.hyprctl)) {
422 const w = await this.list_windows();
423 const seen = new Map();
424 for (const win of w.windows) {
425 const cls = win.wm_class || win.app_id;
426 if (cls) seen.set(cls, { name: cls, wm_class: cls });
427 }
428 return { apps: [...seen.values()] };
429 }
430 throw new ExecError("list_apps needs wmctrl (X11) or swaymsg/hyprctl (Wayland)");
431 },
432 list_windows: async (args = {}) => {
433 rejectWindowSelectors(args);
434 await probeSession();
435 if (session === "x11" && tools.wmctrl) {
436 const r = await runOk("wmctrl", ["-lGx"], { timeoutMs: 15_000 });
437 const windows = [];
438 for (const line of r.stdout.split("\n")) {
439 const m = /^(\S+)\s+(-?\d+)\s+(-?\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(.*)$/.exec(line);
440 if (m) windows.push({ id: m[1], desktop: m[2], position: { x: Number(m[3]), y: Number(m[4]) }, size: { w: Number(m[5]), h: Number(m[6]) }, wm_class: m[7], title: m[8] });
441 }
442 return { windows };
443 }
444 if (session === "wayland" && tools.swaymsg) {
445 const r = await run("swaymsg", ["-t", "get_tree", "-r"], { timeoutMs: 15_000 });
446 const windows = [];
447 const walk = (n) => {
448 if (n.type === "con" && n.name) windows.push({ id: String(n.id), title: n.name, wm_class: n.app_id ?? null, position: { x: n.rect?.x, y: n.rect?.y }, size: { w: n.rect?.width, h: n.rect?.height }, focused: !!n.focused });
449 (n.nodes ?? []).forEach(walk);
450 (n.floating_nodes ?? []).forEach(walk);
451 };
452 walk(tryJson(r.stdout, {}));
453 return { windows };
454 }
455 if (session === "wayland" && tools.hyprctl) {
456 const r = await run("hyprctl", ["-j", "clients"], { timeoutMs: 15_000 });
457 const clients = tryJson(r.stdout, []);
458 return { windows: clients.map((c) => ({ id: String(c.address), title: c.title, wm_class: c.class, position: { x: c.at?.[0], y: c.at?.[1] }, size: { w: c.size?.[0], h: c.size?.[1] }, focused: !!c.focused })) };
459 }
460 throw new ExecError("list_windows needs wmctrl (X11), swaymsg (sway) or hyprctl (hyprland)");
461 },
462 open_application: async ({ name, bundle_id: bid, url: urlArg, activate } = {}) => {
463 const target = name ?? bid;
464 if (!target || !/^[A-Za-z0-9][A-Za-z0-9 ._-]*$/.test(target)) throw new ExecError("open_application needs a plain executable/desktop name");
465 // activate defaults to background: on X11 a new window grabs focus, so
466 // remember the active window and hand focus back after the launch.
467 let prevWindow = null;
468 if (activate !== true) {
469 try {
470 await probeSession();
471 if (session === "x11" && tools.xdotool) {
472 const active = await run("xdotool", ["getactivewindow"], { timeoutMs: 3_000 });
473 if (active.code === 0 && /^\d+$/.test(active.stdout.trim())) prevWindow = active.stdout.trim();
474 }
475 } catch { /* no session/tools — the launch itself is still fine */ }
476 }
477 spawnDetached(target, urlArg ? [urlArg] : [], "", true);
478 await new Promise((r) => setTimeout(r, 500));
479 let focusRestored = false;
480 if (prevWindow) {
481 try {
482 focusRestored = (await run("xdotool", ["windowactivate", prevWindow], { timeoutMs: 3_000 })).code === 0;
483 } catch { /* best-effort */ }
484 }
485 return { launched: true, name: target, url: urlArg ?? null, activate: activate === true, ...(activate === true ? {} : { focus_restored: focusRestored }) };
486 },
487 get_app_state: async ({ app_ref, window_id } = {}) => {
488 const name = appName(app_ref);
489 if (window_id !== undefined) throw Object.assign(new ExecError("Linux app-state window_id selection is unavailable"), { code: "unsupported_selector" });
490 const t = await run("python3", ["-c", PYATSPI_WALK, name, "10", "500"], { timeoutMs: 45_000 }).then((r) =>
491 tryJson((r.stdout.trim().split("\n").pop() ?? ""), null));
492 if (!t) throw new ExecError("AT-SPI walk failed — is python3-pyatspi installed and the desktop running an accessibility bus (AT_SPI_BUS)?");
493 if (!t.found) throw new ExecError("application not found or name is ambiguous in the AT-SPI tree — use a unique exact app_ref.name");
494 return t;
495 },
496 screenshot: async (args = {}) => {
497 rejectWindowSelectors(args);
498 const { display, region, path: outPath } = args;
499 if (outPath != null) outputPath(outPath);
500 await probeSession();
501 const dir = recordingsDir();
502 fs.mkdirSync(dir, { recursive: true });
503 const file = outPath || path.join(dir, `shot-${new Date().toISOString().replace(/[:.]/g, "-")}-${crypto.randomBytes(3).toString("hex")}.png`);
504 await takeShot(file, region);
505 const dims = pngSize(file);
506 lastRaster = {
507 file,
508 bytes: fs.statSync(file).size,
509 // Region rasters describe the region; full shots get geometry from the
510 // PNG itself (Linux shots are always scale 1: points == pixels).
511 points: region ? { x: region[0], y: region[1], w: region[2], h: region[3] } : dims ? { x: 0, y: 0, w: dims.w, h: dims.h } : null,
512 pixels: dims ?? (region ? { w: Math.round(region[2]), h: Math.round(region[3]) } : null),
513 scale: 1,
514 capturedAt: new Date().toISOString(),
515 };
516 return { ...lastRaster };
517 },
518 resolve_element: async ({ app_ref, windowIndex, window_id, path: pathArr } = {}) => {
519 const name = appName(app_ref);
520 assertAppRootWindow(windowIndex, window_id);
521 await probeSession();
522 need("python3", "element resolution (AT-SPI)");
523 const script = `import json, sys, pyatspi
524 ${PYATSPI_APP}
525 desktop = pyatspi.Registry.getDesktop(0)
526 app_name = sys.argv[1] if len(sys.argv) > 1 and sys.argv[1] else None
527 target_path = json.loads(sys.argv[2])
528 root = resolve_app(desktop, app_name)
529 if root is None:
530 print(json.dumps({"found": False, "element": None, "reason": "app_not_found"}))
531 sys.exit(0)
532 node = root
533 ok = True
534 for k in target_path:
535 found = None
536 try:
537 if k < node.childCount:
538 found = node.getChildAtIndex(k)
539 except Exception:
540 found = None
541 if found is None:
542 ok = False
543 break
544 node = found
545 if not ok:
546 print(json.dumps({"found": True, "element": None, "reason": "element_stale"}))
547 sys.exit(0)
548 ext = None
549 try: ext = node.queryComponent().getExtents(pyatspi.DESKTOP_COORDS)
550 except Exception: pass
551 print(json.dumps({"found": True, "reason": None, "element": {
552 "role": node.getRoleName() or None, "label": node.name or None,
553 "position": {"x": ext.x, "y": ext.y} if ext else None,
554 "size": {"w": ext.width, "h": ext.height} if ext else None}}))`;
555 const r = await run("python3", ["-c", script, name, JSON.stringify(pathArr ?? [])], { timeoutMs: 30_000 });
556 const out = tryJson(r.stdout.trim().split("\n").pop() ?? "", null);
557 if (!out) throw new ExecError(`AT-SPI resolve failed: ${(r.stderr || r.stdout).slice(0, 250)}`, r);
558 return out;
559 },
560 zoom: async ({ source, region, path: outPath }) => {
561 need("ffmpeg", "zoom/crop");
562 const src = source ?? lastRaster?.file;
563 if (!src) throw new ExecError("no screenshot taken yet on this computer — call screenshot first");
564 const out = outputPath(outPath ?? path.join(recordingsDir(), `zoom-${crypto.randomBytes(4).toString("hex")}.png`));
565 await runOk("ffmpeg", ["-y", "-loglevel", "error", "-i", src, "-vf", `crop=${Math.round(region[2])}:${Math.round(region[3])}:${Math.round(region[0])}:${Math.round(region[1])}`, out], { timeoutMs: 20_000 });
566 return { file: out, bytes: fs.statSync(out).size, region, source: src };
567 },
568 left_click: ({ target, strategy }) => { assertNum(target.x, "x"); assertNum(target.y, "y"); assertEventStrategy(strategy); return inputChain(target.x, target.y, () => clickButton(1, 1)); },
569 double_click: ({ target }) => inputChain(target.x, target.y, () => clickButton(1, 2)),
570 triple_click: ({ target }) => inputChain(target.x, target.y, () => clickButton(1, 3)),
571 right_click: ({ target }) => inputChain(target.x, target.y, () => clickButton(3, 1)),
572 middle_click: ({ target }) => inputChain(target.x, target.y, () => clickButton(2, 1)),
573 mouse_move: ({ target }) => inputMove(target.x, target.y),
574 left_click_drag: async ({ from_target: from, to }) => {
575 requireInputOwner();
576 await inputMove(from.x, from.y);
577 throwIfAborted();
578 mouseHeld = true;
579 try {
580 if (session === "x11") await xdotool(["mousedown", "1"]);
581 else await ydotool(["click", "0x40"]);
582 for (let i = 1; i <= 10; i++) {
583 await wait(20);
584 await inputMove(from.x + ((to.x - from.x) * i) / 10, from.y + ((to.y - from.y) * i) / 10);
585 }
586 } finally { await releaseMouse(); }
587 return { action_sent: true, from, to };
588 },
589 left_mouse_down: async ({ target } = {}) => {
590 requireInputOwner();
591 await probeSession();
592 if (target) await inputMove(target.x, target.y);
593 throwIfAborted();
594 mouseHeld = true;
595 try {
596 if (session === "x11") await xdotool(["mousedown", "1"]);
597 else await ydotool(["click", "0x40"]);
598 } catch (err) { await releaseMouse(); throw err; }
599 return { action_sent: true };
600 },
601 left_mouse_up: async () => {
602 if (!mouseHeld) throw Object.assign(new ExecError("no agent pointer press to release"), { code: "input_not_held" });
603 await releaseMouse();
604 return { action_sent: true };
605 },
606 scroll: async ({ target, direction = "down", amount = 3 }) => {
607 await inputMove(target.x, target.y);
608 if (session === "x11") {
609 const buttons = { down: 5, up: 4, right: 7, left: 6 };
610 await xdotool(["click", "--repeat", String(Math.max(1, Math.min(30, amount))), "--delay", "60", String(buttons[direction] ?? 5)]);
611 return { action_sent: true, direction, amount };
612 }
613 // Wayland: synthesize wheel via ydotool is not wired in this build — honest refusal.
614 throw new ExecError('scroll on Wayland is not available in this build; use swipe-style drags or run an X11/XWayland window. (Roadmap: ydotool wheel events.)');
615 },
616 type: async ({ text }) => {
617 if (!text) return { action_sent: false, note: "empty text" };
618 await probeSession();
619 if (session === "x11") {
620 // xdotool `type` remaps a spare keycode for characters absent from the
621 // current keymap. Two failure modes follow: a cased letter produces a
622 // single-symbol key whose XKB level 0 is the lowercase form (Ü → ü),
623 // and consecutive remaps inside one `type` call race the X server's
624 // keymap-change propagation, so non-ASCII chars intermittently drop or
625 // arrive mangled (héllo → hllo, 日本 → 本). Route every non-ASCII char
626 // through `key U<hex>` — one synchronous remap+press+restore per char —
627 // adding Shift only when the char is cased-uppercase, and batch ASCII
628 // runs through `type` as before.
629 let runText = "";
630 const chunks = [];
631 for (const ch of String(text)) {
632 if (ch.codePointAt(0) > 127) {
633 if (runText) { chunks.push(runText); runText = ""; }
634 chunks.push(ch);
635 } else runText += ch;
636 }
637 if (runText) chunks.push(runText);
638 for (const chunk of chunks) {
639 // Supplementary-plane chars are one code point but length 2; test
640 // the code point, not the string length.
641 if (chunk.codePointAt(0) > 127) {
642 const hex = chunk.codePointAt(0).toString(16).toUpperCase().padStart(4, "0");
643 const shift = chunk !== chunk.toLowerCase() ? "shift+" : "";
644 await xdotool(["key", `${shift}U${hex}`]);
645 // Each temp remap restores the keymap as soon as the event is
646 // queued; a lagging app can then read the press against the
647 // restored map and drop it. A short settle narrows that window.
648 // Under heavy host saturation XTEST drops remain possible — that
649 // residual is documented in the suite's known_limitations.
650 await new Promise((r) => setTimeout(r, 30));
651 } else {
652 await xdotool(["type", "--delay", "12", "--", chunk]);
653 }
654 }
655 return { action_sent: true, chars: text.length };
656 }
657 need("wtype", "typing on Wayland");
658 const r = await run("wtype", ["--", String(text)], { timeoutMs: 15_000 });
659 if (r.code !== 0) throw new ExecError(`wtype failed: ${r.stderr.slice(0, 200)}`, r);
660 return { action_sent: true, chars: text.length };
661 },
662 key: async ({ text, repeat = 1 }) => {
663 await probeSession();
664 const k = xdotoolKey(text);
665 const n = Math.max(1, Math.min(100, Number(repeat) || 1));
666 if (session === "x11") {
667 throwIfAborted();
668 heldKeys.add(k);
669 try {
670 await xdotool(["key", "--repeat", String(n), "--delay", "60", k]);
671 heldKeys.delete(k);
672 } finally { await releaseKey(k); }
673 } else await waylandKey(text, { repeat: n });
674 return { action_sent: true, key: k };
675 },
676 hold_key: async ({ text, duration }) => {
677 requireInputOwner();
678 await probeSession();
679 const k = xdotoolKey(text);
680 const d = Math.max(0.05, Math.min(30, Number(duration) || 1));
681 if (session === "x11") {
682 throwIfAborted();
683 heldKeys.add(k);
684 try {
685 await xdotool(["keydown", k]);
686 await wait(d * 1000);
687 } finally { await releaseKey(k); }
688 } else await waylandKey(text, { holdMs: Math.round(d * 1000) });
689 return { action_sent: true, key: k, heldSec: d };
690 },
691 set_value: async ({ target, value }) => {
692 // Select a supported interface before sending input. A refused write or
693 // failed readback must never trigger a second, ambiguously applied edit.
694 const out = await atspiResolve(target, ` state = found.getState()
695 if not state.contains(pyatspi.STATE_ENABLED):
696 raise RuntimeError("element_disabled")
697 try:
698 editor = found.queryEditableText()
699 except NotImplementedError:
700 editor = None
701 if editor is not None:
702 if not state.contains(pyatspi.STATE_EDITABLE):
703 raise RuntimeError("element_read_only")
704 if not editor.setTextContents(extra):
705 raise RuntimeError("value_rejected")
706 text = found.queryText()
707 after = text.getText(0, text.characterCount)
708 if after != extra:
709 raise RuntimeError("value_verification_failed")
710 else:
711 import math
712 desired = float(extra)
713 if not math.isfinite(desired):
714 raise RuntimeError("invalid_value")
715 numeric = found.queryValue()
716 numeric.currentValue = desired
717 after = numeric.currentValue
718 if after != desired:
719 raise RuntimeError("value_verification_failed")
720 print(json.dumps({"ok": True, "after": after}))`, String(value));
721 if (!out.ok) throw new ExecError(`set_value failed: ${out.code}`);
722 return { action_sent: true, strategy: "a11y", verified: true, after: out.after };
723 },
724 select_text: async () => { throw new ExecError("select_text is not implemented on the linux backend — fail-closed"); },
725 perform_action: async ({ target, action }) => {
726 const body = ` a = found.queryAction()
727 names = [a.getName(i) for i in range(a.nActions)]
728 want = (extra or "click").lower()
729 match = next((n for n in names if n.lower() == want), None)
730 if match is None and want == "click":
731 match = next((n for n in names if n.lower() in ("click", "press", "activate")), None)
732 if match is None:
733 print(json.dumps({"ok": False, "code": "action_not_found: " + ",".join(names)}))
734 else:
735 a.doAction(names.index(match))
736 print(json.dumps({"ok": True, "sent": True}))`;
737 const out = await atspiResolve(target, body, String(action));
738 if (!out.ok) throw new ExecError(`perform_action failed: ${out.code}`);
739 return { action_sent: true, strategy: "a11y", action };
740 },
741 read_clipboard: async () => {
742 await probeSession();
743 const cmd = session === "x11"
744 ? (tools.xclip ? ["xclip", "-selection", "clipboard", "-o"] : ["xsel", "--clipboard", "--output"])
745 : ["wl-paste"];
746 need(cmd[0], "clipboard read");
747 const r = await run(cmd[0], cmd.slice(1), { timeoutMs: 10_000 });
748 if (r.code !== 0) throw new ExecError("clipboard read failed", r);
749 return { text: r.stdout, encoding: "utf8" };
750 },
751 write_clipboard: async ({ text }) => {
752 await probeSession();
753 const cmd = session === "x11"
754 ? (tools.xclip ? ["xclip", "-selection", "clipboard"] : ["xsel", "--clipboard", "--input"])
755 : ["wl-copy"];
756 need(cmd[0], "clipboard write");
757 spawnDetached(cmd[0], cmd.slice(1), String(text ?? ""), true);
758 return { written: String(text ?? "").length };
759 },
760 cursor_position: async () => {
761 await probeSession();
762 if (session === "x11") {
763 const out = await xdotool(["getmouselocation"]);
764 const m = /x:(-?\d+)\s+y:(-?\d+)/.exec(out);
765 if (!m) throw new ExecError(`could not parse xdotool getmouselocation output: ${out}`);
766 return { x: Number(m[1]), y: Number(m[2]) };
767 }
768 throw new ExecError("cursor position needs an X11 session in this build");
769 },
770 recordingStart: async () => {
771 throw Object.assign(new ExecError("Recording is unavailable on this platform until the recorder has session-owned cleanup. Use screenshots instead."), { code: "owned_recording_unavailable" });
772 },
773 recordingStop: async ({ id }) => { throw new ExecError(`unknown recording "${id}"`); },
774 recordingStatus: ({ id }) => ({ id, running: false }),
775 recordingList: async () => {
776 const dir = recordingsDir();
777 const out = fs.existsSync(dir)
778 ? fs.readdirSync(dir).filter((f) => /\.(mp4|mkv|png)$/i.test(f)).map((f) => {
779 const st = fs.statSync(path.join(dir, f));
780 return { file: path.join(dir, f), bytes: st.size, modifiedAt: st.mtime.toISOString() };
781 }).sort((a, b) => b.modifiedAt.localeCompare(a.modifiedAt)).slice(0, 50)
782 : [];
783 return { dir, recordings: out, running: [] };
784 },
785 };
786
787 async function inputChain(x, y, act) {
788 await inputMove(x, y);
789 await act();
790 return { action_sent: true, at: { x: Number(x), y: Number(y) } };
791 }
792
793 async function inputMove(x, y) {
794 await probeSession();
795 const nx = Math.round(assertNum(x, "x"));
796 const ny = Math.round(assertNum(y, "y"));
797 if (session === "x11") await xdotool(["mousemove", "--sync", String(nx), String(ny)]);
798 else await ydotool(["moveto", String(nx), String(ny)]);
799 }
800 }
801
802 export default { create };
803
803 lines Plain Text