返回 CodeWhale
browser-cdp.mjs
根目录 / crates / tui / plugins / computer-use / src / browser-cdp.mjs
1 // Browser control over the Chrome DevTools Protocol (CDP).
2 //
3 // A self-owned Chromium-family browser: launch (or reuse) an instance with its
4 // own --user-data-dir under the state dir and a loopback-only debugging port,
5 // then speak CDP over the browser WebSocket. The person's own browser profile
6 // is never attached to, never typed into, and never closed. No screen
7 // coordinates and no accessibility are involved: page elements are addressed
8 // by CSS selector through the DOM domain, and coordinate clicks are page
9 // viewport pixels — a different space from screen points, named differently so
10 // the two can never be confused.
11 //
12 // One tab per computer session; the last session out closes the shared
13 // browser. Node needs a global WebSocket (22+, or 21 with the default-on
14 // flag); older runtimes refuse with `unsupported_runtime` instead of
15 // half-working.
16 import fs from "node:fs";
17 import os from "node:os";
18 import path from "node:path";
19 import crypto from "node:crypto";
20 import { spawn } from "node:child_process";
21 import { ExecError, currentSignal } from "./exec.mjs";
22 import { stateDir } from "./registry.mjs";
23
24 const APPLICATIONS = ["Google Chrome", "Chromium", "Brave Browser", "Microsoft Edge"];
25 const LINUX_BINARIES = ["google-chrome", "chromium", "chromium-browser", "brave-browser", "microsoft-edge"];
26
27 const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
28 const badArgs = (message) => Object.assign(new ExecError(message), { code: "bad_args" });
29
30 function defaultRecordingsDir() {
31 return process.env.CODEWHALE_CU_RECORDINGS_DIR || path.join(stateDir(), "recordings");
32 }
33
34 /** Only http(s) and about:blank can be navigated to; everything else is refused. */
35 export function checkBrowserUrl(url) {
36 if (typeof url !== "string" || !url.trim()) throw badArgs("browser navigate/start need a url (http:// or https:// or about:blank)");
37 const trimmed = url.trim();
38 if (/^about:blank$/i.test(trimmed)) return trimmed;
39 let parsed;
40 try { parsed = new URL(trimmed); } catch { throw badArgs(`"${trimmed}" is not a URL`); }
41 if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
42 throw badArgs(`only http(s):// and about:blank URLs can be opened (got "${parsed.protocol}//")`);
43 }
44 return parsed.href;
45 }
46
47 /** Locate a Chromium-family browser app or binary; CODEWHALE_CU_BROWSER_APP overrides. */
48 export function findBrowserApp(platform = process.platform, env = process.env, exists = fs.existsSync) {
49 if (env.CODEWHALE_CU_BROWSER_APP) return env.CODEWHALE_CU_BROWSER_APP;
50 if (platform === "darwin") {
51 for (const name of APPLICATIONS) {
52 for (const root of ["/Applications", path.join(os.homedir(), "Applications")]) {
53 const candidate = path.posix.join(root, `${name}.app`);
54 if (exists(candidate)) return candidate;
55 }
56 }
57 return null;
58 }
59 if (platform === "win32") {
60 const installs = [
61 ["Google", "Chrome", "Application", "chrome.exe"],
62 ["Chromium", "Application", "chrome.exe"],
63 ["BraveSoftware", "Brave-Browser", "Application", "brave.exe"],
64 ["Microsoft", "Edge", "Application", "msedge.exe"],
65 ];
66 for (const relative of installs) {
67 for (const root of [env.ProgramFiles, env["ProgramFiles(x86)"], env.LOCALAPPDATA]) {
68 if (!root) continue;
69 const candidate = path.win32.join(root, ...relative);
70 if (exists(candidate)) return candidate;
71 }
72 }
73 return null;
74 }
75 for (const bin of LINUX_BINARIES) {
76 for (const dir of (env.PATH ?? "").split(":")) {
77 if (dir && exists(path.join(dir, bin))) return path.join(dir, bin);
78 }
79 }
80 return null;
81 }
82
83 /** Launch detached so the browser is its own process, never a child we must reap. */
84 function defaultLaunch({ app, profileDir, url, platform = process.platform }) {
85 const flags = ["--remote-debugging-port=0", `--user-data-dir=${profileDir}`, "--no-first-run", "--no-default-browser-check"];
86 const target = url || "about:blank";
87 let cmd, args;
88 // -n (new instance) matters: without it, `open --args` is ignored whenever
89 // the person already has Chrome running — the args never reach a new process.
90 if (platform === "darwin") { cmd = "open"; args = ["-g", "-n", "-a", app, "--args", ...flags, target]; }
91 else { cmd = app; args = [...flags, target]; }
92 const child = spawn(cmd, args, { detached: true, stdio: "ignore" });
93 child.on("error", () => {});
94 child.unref();
95 }
96
97 /**
98 * Open the CDP WebSocket. Injectable: tests supply a fake ws-like object so
99 * the command sequence is verifiable without a browser.
100 */
101 function defaultConnect(url, { timeoutMs = 8_000 } = {}) {
102 return new Promise((resolve, reject) => {
103 if (typeof WebSocket === "undefined") {
104 reject(Object.assign(new ExecError("browser actions need a Node runtime with a global WebSocket (22+); this runtime does not have one"), { code: "unsupported_runtime" }));
105 return;
106 }
107 let ws;
108 try { ws = new WebSocket(url); } catch (error) {
109 reject(Object.assign(new ExecError(`cannot open a CDP socket at ${url}: ${error.message}`), { code: "browser_unavailable" }));
110 return;
111 }
112 const timer = setTimeout(() => {
113 try { ws.close(); } catch {}
114 reject(Object.assign(new ExecError(`the CDP socket at ${url} did not open within ${timeoutMs}ms`), { code: "browser_unavailable" }));
115 }, timeoutMs);
116 ws.addEventListener("open", () => { clearTimeout(timer); resolve(ws); }, { once: true });
117 ws.addEventListener("error", () => {
118 clearTimeout(timer);
119 reject(Object.assign(new ExecError(`the CDP socket at ${url} refused the connection`), { code: "browser_unavailable" }));
120 }, { once: true });
121 });
122 }
123
124 /** id-matched JSON-RPC over the WebSocket, plus CDP event fan-out. */
125 function makeChannel(ws) {
126 let nextId = 0;
127 const pending = new Map();
128 const listeners = new Map();
129 let closed = false;
130 const failAll = (reason) => {
131 closed = true;
132 for (const [, entry] of pending) entry.reject(Object.assign(new ExecError(reason), { code: "browser_not_running" }));
133 pending.clear();
134 };
135 ws.addEventListener("message", (event) => {
136 let msg;
137 try { msg = JSON.parse(typeof event.data === "string" ? event.data : String(event.data)); } catch { return; }
138 if (msg.id != null && pending.has(msg.id)) {
139 const entry = pending.get(msg.id);
140 pending.delete(msg.id);
141 if (msg.error) entry.reject(Object.assign(new ExecError(`CDP ${msg.method ?? ""} failed: ${msg.error.message}`), { code: "cdp_error", cdp: msg.error }));
142 else entry.resolve(msg.result ?? {});
143 return;
144 }
145 if (msg.method) for (const fn of listeners.get(msg.method) ?? []) fn(msg);
146 });
147 ws.addEventListener("close", () => failAll("the browser closed the CDP connection"));
148 ws.addEventListener("error", () => {});
149 return {
150 get alive() { return !closed; },
151 send(method, params = {}, sessionId) {
152 if (closed) return Promise.reject(Object.assign(new ExecError("the browser is no longer reachable (CDP connection closed)"), { code: "browser_not_running" }));
153 return new Promise((resolve, reject) => {
154 const id = ++nextId;
155 pending.set(id, { resolve, reject });
156 ws.send(JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) }));
157 });
158 },
159 waitFor(method, { sessionId, timeoutMs = 15_000 } = {}) {
160 return new Promise((resolve, reject) => {
161 const signal = currentSignal();
162 const done = () => {
163 clearTimeout(timer);
164 signal?.removeEventListener("abort", onAbort);
165 const list = listeners.get(method) ?? [];
166 listeners.set(method, list.filter((fn) => fn !== handler));
167 };
168 const onAbort = () => { done(); reject(Object.assign(new ExecError("computer request cancelled"), { code: "cancelled" })); };
169 const handler = (msg) => {
170 if (sessionId && msg.sessionId !== sessionId) return;
171 done();
172 resolve(msg.params ?? {});
173 };
174 const timer = setTimeout(() => {
175 done();
176 reject(Object.assign(new ExecError(`timed out waiting for ${method}`), { code: "timeout" }));
177 }, timeoutMs);
178 if (signal?.aborted) { onAbort(); return; }
179 signal?.addEventListener("abort", onAbort, { once: true });
180 listeners.set(method, [...(listeners.get(method) ?? []), handler]);
181 });
182 },
183 close() { closed = true; try { ws.close(); } catch {} },
184 };
185 }
186
187 /**
188 * The browser bridge. One instance per backend (per computer session).
189 * Exposes browser_start / browser_status / browser_navigate / browser_click /
190 * browser_type / browser_screenshot / browser_stop plus close() for the
191 * session teardown hook.
192 */
193 export function createBrowser({
194 connect = defaultConnect,
195 launch = defaultLaunch,
196 findApp = findBrowserApp,
197 recordingsDir = defaultRecordingsDir,
198 platform = process.platform,
199 } = {}) {
200 const state = { channel: null, port: null, profileDir: null, app: null, targetId: null, sessionId: null, pageEnabled: false, domEnabled: false };
201
202 const profileDir = () => path.join(stateDir(), "browser", "profile");
203 const loadTimeout = () => Number(process.env.CODEWHALE_CU_BROWSER_LOAD_TIMEOUT_MS) || 15_000;
204 const requireRunning = () => {
205 if (!state.channel) throw Object.assign(new ExecError("no browser for this session yet — run browser {action:\"start\"} first"), { code: "browser_not_running" });
206 };
207 async function targetInfo(targetId) {
208 const { targetInfo: info } = await state.channel.send("Target.getTargetInfo", { targetId });
209 return { url: info?.url ?? "", title: info?.title ?? "" };
210 }
211 async function listTabs() {
212 const { targetInfos } = await state.channel.send("Target.getTargets", {});
213 return (targetInfos ?? []).filter((t) => t.type === "page");
214 }
215 async function ensureDom() {
216 if (!state.domEnabled) { await state.channel.send("DOM.enable", {}, state.sessionId); state.domEnabled = true; }
217 }
218 async function resolveNode(selector) {
219 if (typeof selector !== "string" || !selector.trim()) throw badArgs("selector must be a non-empty CSS selector");
220 await ensureDom();
221 const { root } = await state.channel.send("DOM.getDocument", { depth: 0 }, state.sessionId);
222 const { nodeId } = await state.channel.send("DOM.querySelector", { nodeId: root.nodeId, selector }, state.sessionId);
223 if (!nodeId) throw Object.assign(new ExecError(`no element on this page matches ${JSON.stringify(selector)}`), { code: "selector_not_found" });
224 return nodeId;
225 }
226 async function viewport() {
227 const metrics = await state.channel.send("Page.getLayoutMetrics", {}, state.sessionId);
228 const vp = metrics.cssLayoutViewport ?? {};
229 return { w: vp.clientWidth ?? 0, h: vp.clientHeight ?? 0, scale: metrics.cssVisualViewport?.scale ?? 1, metrics };
230 }
231 async function mousePoint(x, y) {
232 for (const [type, extra] of [["mouseMoved", {}], ["mousePressed", { button: "left", clickCount: 1, buttons: 1 }], ["mouseReleased", { button: "left", clickCount: 1, buttons: 0 }]]) {
233 await state.channel.send("Input.dispatchMouseEvent", { type, x, y, ...extra }, state.sessionId);
234 }
235 }
236 async function waitLoad(timeoutMs) {
237 await state.channel.send("Page.enable", {}, state.sessionId).catch(() => {});
238 state.pageEnabled = true;
239 return state.channel.waitFor("Page.loadEventFired", { sessionId: state.sessionId, timeoutMs });
240 }
241 async function bindTab(url, reused) {
242 // A fresh launch already came with one about:blank tab — adopt it instead
243 // of stacking a second tab that nothing will ever close. A single
244 // leftover blank tab (a dead session's) is adopted too; visible tabs are
245 // never stolen (an instance with real tabs gets a new tab of this
246 // session's own).
247 const tabs = await listTabs();
248 let targetId = null;
249 if (tabs.length === 1 && (!reused || tabs[0].url === "about:blank")) targetId = tabs[0].targetId;
250 if (!targetId) ({ targetId } = await state.channel.send("Target.createTarget", { url: "about:blank" }));
251 const { sessionId } = await state.channel.send("Target.attachToTarget", { targetId, flatten: true });
252 state.targetId = targetId;
253 state.sessionId = sessionId;
254 state.pageEnabled = false;
255 state.domEnabled = false;
256 let verified = true;
257 if (url && url !== "about:blank") {
258 const load = waitLoad(loadTimeout());
259 await state.channel.send("Page.navigate", { url }, sessionId);
260 verified = await load.then(() => true).catch(() => false);
261 }
262 return { verified };
263 }
264
265 const api = {
266 async start({ url } = {}) {
267 const target = url ? checkBrowserUrl(url) : "about:blank";
268 if (state.channel) {
269 if (url) await this.navigate({ url: target });
270 return { ...(await this.status()), already_running: true };
271 }
272 if (typeof WebSocket === "undefined") throw Object.assign(new ExecError("browser actions need a Node runtime with a global WebSocket (22+); this runtime does not have one"), { code: "unsupported_runtime" });
273 const app = findApp();
274 if (!app) throw Object.assign(new ExecError(`no Chromium-family browser found (looked for ${APPLICATIONS.join(", ")}); set CODEWHALE_CU_BROWSER_APP to the app path`), { code: "browser_not_installed" });
275 const dir = profileDir();
276 fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
277
278 // Reuse a live instance for this profile (a previous session may not have
279 // stopped it); a stale port file must never shadow a fresh launch. The
280 // port file carries the browser WebSocket path, so no HTTP probe is
281 // needed — a socket that opens is an instance that is alive.
282 const portFile = path.join(dir, "DevToolsActivePort");
283 const readPortFile = () => {
284 try {
285 const [port, wsPath] = fs.readFileSync(portFile, "utf8").split("\n");
286 return /^\d+$/.test((port ?? "").trim()) && (wsPath ?? "").trim()
287 ? { port: Number(port.trim()), wsPath: wsPath.trim() }
288 : null;
289 } catch { return null; }
290 };
291 const attempt = async (file) => {
292 if (!file) return null;
293 try { return await connect(`ws://127.0.0.1:${file.port}${file.wsPath}`); } catch { return null; }
294 };
295 let ws = null, reused = false;
296 const existing = readPortFile();
297 if (existing) {
298 ws = await attempt(existing);
299 if (ws) { reused = true; state.port = existing.port; }
300 else { try { fs.rmSync(portFile, { force: true }); } catch {} }
301 }
302 if (!ws) {
303 await launch({ app, profileDir: dir, url: "about:blank", platform });
304 const deadline = Date.now() + 20_000;
305 for (;;) {
306 if (currentSignal()?.aborted) throw Object.assign(new ExecError("computer request cancelled"), { code: "cancelled" });
307 const file = readPortFile();
308 if (file) {
309 ws = await attempt(file);
310 if (ws) { state.port = file.port; break; }
311 }
312 if (Date.now() > deadline) throw Object.assign(new ExecError(`the browser started but its debugging endpoint never came up (profile ${dir}); is it running with a usable profile?`), { code: "browser_unavailable" });
313 await sleep(300);
314 }
315 }
316 state.channel = makeChannel(ws);
317 state.profileDir = dir;
318 state.app = app;
319 const { verified } = await bindTab(target, reused);
320 const info = await targetInfo(state.targetId);
321 return {
322 running: true, launched: !reused, reused, browser: app, profile: dir,
323 tab: { id: state.targetId, url: info.url, title: info.title }, verified,
324 note: "self-owned profile under the state dir; the user's own browser was not touched",
325 };
326 },
327
328 async status() {
329 if (!state.channel) return { running: false, browser: state.app, profile: state.profileDir ?? profileDir(), note: "no browser session for this computer session yet — browser {action:\"start\"} launches a self-owned instance" };
330 try {
331 const tabs = await listTabs();
332 return {
333 running: true, browser: state.app, port: state.port, profile: state.profileDir,
334 tabs: tabs.map((t) => ({ id: t.targetId, title: t.title, url: t.url })),
335 activeTab: tabs.some((t) => t.targetId === state.targetId) ? (({ url, title }) => ({ id: state.targetId, url, title }))(await targetInfo(state.targetId)) : null,
336 };
337 } catch (error) {
338 state.channel?.close();
339 state.channel = null; state.targetId = null; state.sessionId = null;
340 return { running: false, note: `the browser went away: ${error.message}` };
341 }
342 },
343
344 async navigate({ url } = {}) {
345 requireRunning();
346 const target = checkBrowserUrl(url);
347 const loadPromise = target === "about:blank" ? null : waitLoad(loadTimeout());
348 await state.channel.send("Page.navigate", { url: target }, state.sessionId);
349 let verified = true;
350 if (loadPromise) {
351 try { await loadPromise; } catch (error) { verified = false; void error; }
352 }
353 const info = await targetInfo(state.targetId).catch(() => ({ url: target, title: "" }));
354 return {
355 action_sent: true, url: info.url || target, title: info.title, verified,
356 ...(verified ? {} : { note: "the page did not report load completion (slow page or same-document navigation) — observe before relying on it" }),
357 };
358 },
359
360 async click({ selector, point } = {}) {
361 requireRunning();
362 let where;
363 if (typeof selector === "string" && selector.trim()) {
364 const nodeId = await resolveNode(selector);
365 await state.channel.send("DOM.scrollIntoViewIfNeeded", { nodeId }, state.sessionId);
366 const { model } = await state.channel.send("DOM.getBoxModel", { nodeId }, state.sessionId);
367 const quad = model.content;
368 where = { x: Math.round((quad[0] + quad[2] + quad[4] + quad[6]) / 4), y: Math.round((quad[1] + quad[3] + quad[5] + quad[7]) / 4), selector };
369 } else if (point && Number.isFinite(point.x) && Number.isFinite(point.y)) {
370 const vp = await viewport();
371 if (point.x < 0 || point.y < 0 || (vp.w && point.x >= vp.w) || (vp.h && point.y >= vp.h)) {
372 throw Object.assign(new ExecError(`(${point.x}, ${point.y}) is outside the page viewport (${vp.w}x${vp.h} CSS px) — browser {action:\"screenshot\"} shows this space`), { code: "bad_target" });
373 }
374 where = { x: point.x, y: point.y };
375 } else {
376 throw badArgs("browser click needs selector (CSS) or point {x,y} (page viewport pixels)");
377 }
378 await mousePoint(where.x, where.y);
379 return {
380 action_sent: true, ...(where.selector ? { selector: where.selector } : {}), point: { x: where.x, y: where.y },
381 pointer_moved: false, verified: false, verification_required: "screenshot or status",
382 note: "clicked in the page viewport; the user's pointer never moved",
383 };
384 },
385
386 async type({ text, selector, enter } = {}) {
387 requireRunning();
388 if (typeof text !== "string" || !text.length) throw badArgs("browser type needs text");
389 let focused = null;
390 if (selector != null) {
391 const nodeId = await resolveNode(selector);
392 await state.channel.send("DOM.focus", { nodeId }, state.sessionId);
393 focused = selector;
394 }
395 await state.channel.send("Input.insertText", { text }, state.sessionId);
396 if (enter) {
397 for (const type of ["keyDown", "keyUp"]) {
398 await state.channel.send("Input.dispatchKeyEvent", { type, key: "Enter", code: "Enter", windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13, ...(type === "keyDown" ? { text: "\r" } : {}) }, state.sessionId);
399 }
400 }
401 return {
402 action_sent: true, chars: text.length, ...(focused ? { selector: focused } : {}), ...(enter ? { entered: true } : {}),
403 verified: false, verification_required: "screenshot or status",
404 note: focused ? "text was inserted into the selector's element" : "text was inserted at the page's current focus",
405 };
406 },
407
408 async screenshot({ full } = {}) {
409 requireRunning();
410 const vp = await viewport();
411 const shot = await state.channel.send("Page.captureScreenshot", { format: "png", ...(full ? { captureBeyondViewport: true } : {}) }, state.sessionId);
412 const dir = path.join(recordingsDir(), "captures");
413 fs.mkdirSync(dir, { recursive: true });
414 const file = path.join(dir, `browser-${crypto.randomBytes(4).toString("hex")}.png`);
415 fs.writeFileSync(file, Buffer.from(shot.data, "base64"));
416 return {
417 file, bytes: fs.statSync(file).size, format: "png", space: "page-viewport",
418 viewport: { w: vp.w, h: vp.h }, scale: vp.scale,
419 note: "page pixels, not screen pixels — the same space browser click point targets use",
420 };
421 },
422
423 async stop() {
424 if (!state.channel) return { running: false, note: "no browser session for this computer session" };
425 try { await state.channel.send("Target.closeTarget", { targetId: state.targetId }); } catch { /* the tab may already be gone */ }
426 let remaining = null;
427 try { remaining = (await listTabs()).length; } catch { remaining = null; }
428 let browserClosed = false;
429 if (remaining === 0) {
430 try { await state.channel.send("Browser.close"); browserClosed = true; } catch {}
431 await sleep(200);
432 }
433 state.channel.close();
434 state.channel = null; state.targetId = null; state.sessionId = null; state.pageEnabled = false; state.domEnabled = false;
435 return { running: false, closed: true, browser_closed: browserClosed,
436 ...(remaining ? { note: `${remaining} tab(s) from other sessions remain open; the shared browser stays up` } : {}) };
437 },
438
439 /** Session teardown: close this session's tab; last one out closes the browser. */
440 async close() {
441 if (!state.channel) return;
442 try { await this.stop(); } catch { state.channel?.close(); state.channel = null; }
443 },
444 };
445 // Bound once so backends can hand the methods out individually without
446 // losing `this` (start/close re-enter the api by name).
447 for (const key of Object.keys(api)) api[key] = api[key].bind(api);
448 return api;
449 }
450
450 lines Plain Text