feat(browser): add chrome MCP existing-session support
This commit is contained in:
Peter Steinberger
68
src/browser/chrome-mcp.snapshot.test.ts
Normal file
68
src/browser/chrome-mcp.snapshot.test.ts
Normal file
@@ -0,0 +1,68 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildAiSnapshotFromChromeMcpSnapshot,
|
||||
flattenChromeMcpSnapshotToAriaNodes,
|
||||
} from "./chrome-mcp.snapshot.js";
|
||||
|
||||
const snapshot = {
|
||||
id: "root",
|
||||
role: "document",
|
||||
name: "Example",
|
||||
children: [
|
||||
{
|
||||
id: "btn-1",
|
||||
role: "button",
|
||||
name: "Continue",
|
||||
},
|
||||
{
|
||||
id: "txt-1",
|
||||
role: "textbox",
|
||||
name: "Email",
|
||||
value: "peter@example.com",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
describe("chrome MCP snapshot conversion", () => {
|
||||
it("flattens structured snapshots into aria-style nodes", () => {
|
||||
const nodes = flattenChromeMcpSnapshotToAriaNodes(snapshot, 10);
|
||||
expect(nodes).toEqual([
|
||||
{
|
||||
ref: "root",
|
||||
role: "document",
|
||||
name: "Example",
|
||||
value: undefined,
|
||||
description: undefined,
|
||||
depth: 0,
|
||||
},
|
||||
{
|
||||
ref: "btn-1",
|
||||
role: "button",
|
||||
name: "Continue",
|
||||
value: undefined,
|
||||
description: undefined,
|
||||
depth: 1,
|
||||
},
|
||||
{
|
||||
ref: "txt-1",
|
||||
role: "textbox",
|
||||
name: "Email",
|
||||
value: "peter@example.com",
|
||||
description: undefined,
|
||||
depth: 1,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("builds AI snapshots that preserve Chrome MCP uids as refs", () => {
|
||||
const result = buildAiSnapshotFromChromeMcpSnapshot({ root: snapshot });
|
||||
|
||||
expect(result.snapshot).toContain('- button "Continue" [ref=btn-1]');
|
||||
expect(result.snapshot).toContain('- textbox "Email" [ref=txt-1] value="peter@example.com"');
|
||||
expect(result.refs).toEqual({
|
||||
"btn-1": { role: "button", name: "Continue" },
|
||||
"txt-1": { role: "textbox", name: "Email" },
|
||||
});
|
||||
expect(result.stats.refs).toBe(2);
|
||||
});
|
||||
});
|
||||
246
src/browser/chrome-mcp.snapshot.ts
Normal file
246
src/browser/chrome-mcp.snapshot.ts
Normal file
@@ -0,0 +1,246 @@
|
||||
import type { SnapshotAriaNode } from "./client.js";
|
||||
import {
|
||||
getRoleSnapshotStats,
|
||||
type RoleRefMap,
|
||||
type RoleSnapshotOptions,
|
||||
} from "./pw-role-snapshot.js";
|
||||
|
||||
export type ChromeMcpSnapshotNode = {
|
||||
id?: string;
|
||||
role?: string;
|
||||
name?: string;
|
||||
value?: string | number | boolean;
|
||||
description?: string;
|
||||
children?: ChromeMcpSnapshotNode[];
|
||||
};
|
||||
|
||||
const INTERACTIVE_ROLES = new Set([
|
||||
"button",
|
||||
"checkbox",
|
||||
"combobox",
|
||||
"link",
|
||||
"listbox",
|
||||
"menuitem",
|
||||
"menuitemcheckbox",
|
||||
"menuitemradio",
|
||||
"option",
|
||||
"radio",
|
||||
"searchbox",
|
||||
"slider",
|
||||
"spinbutton",
|
||||
"switch",
|
||||
"tab",
|
||||
"textbox",
|
||||
"treeitem",
|
||||
]);
|
||||
|
||||
const CONTENT_ROLES = new Set([
|
||||
"article",
|
||||
"cell",
|
||||
"columnheader",
|
||||
"gridcell",
|
||||
"heading",
|
||||
"listitem",
|
||||
"main",
|
||||
"navigation",
|
||||
"region",
|
||||
"rowheader",
|
||||
]);
|
||||
|
||||
const STRUCTURAL_ROLES = new Set([
|
||||
"application",
|
||||
"directory",
|
||||
"document",
|
||||
"generic",
|
||||
"group",
|
||||
"ignored",
|
||||
"list",
|
||||
"menu",
|
||||
"menubar",
|
||||
"none",
|
||||
"presentation",
|
||||
"row",
|
||||
"rowgroup",
|
||||
"tablist",
|
||||
"table",
|
||||
"toolbar",
|
||||
"tree",
|
||||
"treegrid",
|
||||
]);
|
||||
|
||||
function normalizeRole(node: ChromeMcpSnapshotNode): string {
|
||||
const role = typeof node.role === "string" ? node.role.trim().toLowerCase() : "";
|
||||
return role || "generic";
|
||||
}
|
||||
|
||||
function normalizeString(value: unknown): string | undefined {
|
||||
if (typeof value === "string") {
|
||||
const trimmed = value.trim();
|
||||
return trimmed || undefined;
|
||||
}
|
||||
if (typeof value === "number" || typeof value === "boolean") {
|
||||
return String(value);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function escapeQuoted(value: string): string {
|
||||
return value.replaceAll("\\", "\\\\").replaceAll('"', '\\"');
|
||||
}
|
||||
|
||||
function shouldIncludeNode(params: {
|
||||
role: string;
|
||||
name?: string;
|
||||
options?: RoleSnapshotOptions;
|
||||
}): boolean {
|
||||
if (params.options?.interactive && !INTERACTIVE_ROLES.has(params.role)) {
|
||||
return false;
|
||||
}
|
||||
if (params.options?.compact && STRUCTURAL_ROLES.has(params.role) && !params.name) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function shouldCreateRef(role: string, name?: string): boolean {
|
||||
return INTERACTIVE_ROLES.has(role) || (CONTENT_ROLES.has(role) && Boolean(name));
|
||||
}
|
||||
|
||||
type DuplicateTracker = {
|
||||
counts: Map<string, number>;
|
||||
keysByRef: Map<string, string>;
|
||||
duplicates: Set<string>;
|
||||
};
|
||||
|
||||
function createDuplicateTracker(): DuplicateTracker {
|
||||
return {
|
||||
counts: new Map(),
|
||||
keysByRef: new Map(),
|
||||
duplicates: new Set(),
|
||||
};
|
||||
}
|
||||
|
||||
function registerRef(
|
||||
tracker: DuplicateTracker,
|
||||
ref: string,
|
||||
role: string,
|
||||
name?: string,
|
||||
): number | undefined {
|
||||
const key = `${role}:${name ?? ""}`;
|
||||
const count = tracker.counts.get(key) ?? 0;
|
||||
tracker.counts.set(key, count + 1);
|
||||
tracker.keysByRef.set(ref, key);
|
||||
if (count > 0) {
|
||||
tracker.duplicates.add(key);
|
||||
return count;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function flattenChromeMcpSnapshotToAriaNodes(
|
||||
root: ChromeMcpSnapshotNode,
|
||||
limit = 500,
|
||||
): SnapshotAriaNode[] {
|
||||
const boundedLimit = Math.max(1, Math.min(2000, Math.floor(limit)));
|
||||
const out: SnapshotAriaNode[] = [];
|
||||
|
||||
const visit = (node: ChromeMcpSnapshotNode, depth: number) => {
|
||||
if (out.length >= boundedLimit) {
|
||||
return;
|
||||
}
|
||||
const ref = normalizeString(node.id);
|
||||
if (ref) {
|
||||
out.push({
|
||||
ref,
|
||||
role: normalizeRole(node),
|
||||
name: normalizeString(node.name) ?? "",
|
||||
value: normalizeString(node.value),
|
||||
description: normalizeString(node.description),
|
||||
depth,
|
||||
});
|
||||
}
|
||||
for (const child of node.children ?? []) {
|
||||
visit(child, depth + 1);
|
||||
if (out.length >= boundedLimit) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
visit(root, 0);
|
||||
return out;
|
||||
}
|
||||
|
||||
export function buildAiSnapshotFromChromeMcpSnapshot(params: {
|
||||
root: ChromeMcpSnapshotNode;
|
||||
options?: RoleSnapshotOptions;
|
||||
maxChars?: number;
|
||||
}): {
|
||||
snapshot: string;
|
||||
truncated?: boolean;
|
||||
refs: RoleRefMap;
|
||||
stats: { lines: number; chars: number; refs: number; interactive: number };
|
||||
} {
|
||||
const refs: RoleRefMap = {};
|
||||
const tracker = createDuplicateTracker();
|
||||
const lines: string[] = [];
|
||||
|
||||
const visit = (node: ChromeMcpSnapshotNode, depth: number) => {
|
||||
const role = normalizeRole(node);
|
||||
const name = normalizeString(node.name);
|
||||
const value = normalizeString(node.value);
|
||||
const description = normalizeString(node.description);
|
||||
const maxDepth = params.options?.maxDepth;
|
||||
if (maxDepth !== undefined && depth > maxDepth) {
|
||||
return;
|
||||
}
|
||||
|
||||
const includeNode = shouldIncludeNode({ role, name, options: params.options });
|
||||
if (includeNode) {
|
||||
let line = `${" ".repeat(depth)}- ${role}`;
|
||||
if (name) {
|
||||
line += ` "${escapeQuoted(name)}"`;
|
||||
}
|
||||
const ref = normalizeString(node.id);
|
||||
if (ref && shouldCreateRef(role, name)) {
|
||||
const nth = registerRef(tracker, ref, role, name);
|
||||
refs[ref] = nth === undefined ? { role, name } : { role, name, nth };
|
||||
line += ` [ref=${ref}]`;
|
||||
}
|
||||
if (value) {
|
||||
line += ` value="${escapeQuoted(value)}"`;
|
||||
}
|
||||
if (description) {
|
||||
line += ` description="${escapeQuoted(description)}"`;
|
||||
}
|
||||
lines.push(line);
|
||||
}
|
||||
|
||||
for (const child of node.children ?? []) {
|
||||
visit(child, depth + 1);
|
||||
}
|
||||
};
|
||||
|
||||
visit(params.root, 0);
|
||||
|
||||
for (const [ref, data] of Object.entries(refs)) {
|
||||
const key = tracker.keysByRef.get(ref);
|
||||
if (key && !tracker.duplicates.has(key)) {
|
||||
delete data.nth;
|
||||
}
|
||||
}
|
||||
|
||||
let snapshot = lines.join("\n");
|
||||
let truncated = false;
|
||||
const maxChars =
|
||||
typeof params.maxChars === "number" && Number.isFinite(params.maxChars) && params.maxChars > 0
|
||||
? Math.floor(params.maxChars)
|
||||
: undefined;
|
||||
if (maxChars && snapshot.length > maxChars) {
|
||||
snapshot = `${snapshot.slice(0, maxChars)}\n\n[...TRUNCATED - page too large]`;
|
||||
truncated = true;
|
||||
}
|
||||
|
||||
const stats = getRoleSnapshotStats(snapshot, refs);
|
||||
return truncated ? { snapshot, truncated, refs, stats } : { snapshot, refs, stats };
|
||||
}
|
||||
100
src/browser/chrome-mcp.test.ts
Normal file
100
src/browser/chrome-mcp.test.ts
Normal file
@@ -0,0 +1,100 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
listChromeMcpTabs,
|
||||
openChromeMcpTab,
|
||||
resetChromeMcpSessionsForTest,
|
||||
setChromeMcpSessionFactoryForTest,
|
||||
} from "./chrome-mcp.js";
|
||||
|
||||
type ToolCall = {
|
||||
name: string;
|
||||
arguments?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
function createFakeSession() {
|
||||
const callTool = vi.fn(async ({ name }: ToolCall) => {
|
||||
if (name === "list_pages") {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: [
|
||||
"## Pages",
|
||||
"1: https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session [selected]",
|
||||
"2: https://github.com/openclaw/openclaw/pull/45318",
|
||||
].join("\n"),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (name === "new_page") {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: [
|
||||
"## Pages",
|
||||
"1: https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session",
|
||||
"2: https://github.com/openclaw/openclaw/pull/45318",
|
||||
"3: https://example.com/ [selected]",
|
||||
].join("\n"),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected tool ${name}`);
|
||||
});
|
||||
|
||||
return {
|
||||
client: {
|
||||
callTool,
|
||||
listTools: vi.fn().mockResolvedValue({ tools: [{ name: "list_pages" }] }),
|
||||
close: vi.fn().mockResolvedValue(undefined),
|
||||
connect: vi.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
transport: {
|
||||
pid: 123,
|
||||
},
|
||||
ready: Promise.resolve(),
|
||||
};
|
||||
}
|
||||
|
||||
describe("chrome MCP page parsing", () => {
|
||||
beforeEach(async () => {
|
||||
await resetChromeMcpSessionsForTest();
|
||||
});
|
||||
|
||||
it("parses list_pages text responses when structuredContent is missing", async () => {
|
||||
setChromeMcpSessionFactoryForTest(async () => createFakeSession());
|
||||
|
||||
const tabs = await listChromeMcpTabs("chrome-live");
|
||||
|
||||
expect(tabs).toEqual([
|
||||
{
|
||||
targetId: "1",
|
||||
title: "",
|
||||
url: "https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session",
|
||||
type: "page",
|
||||
},
|
||||
{
|
||||
targetId: "2",
|
||||
title: "",
|
||||
url: "https://github.com/openclaw/openclaw/pull/45318",
|
||||
type: "page",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("parses new_page text responses and returns the created tab", async () => {
|
||||
setChromeMcpSessionFactoryForTest(async () => createFakeSession());
|
||||
|
||||
const tab = await openChromeMcpTab("chrome-live", "https://example.com/");
|
||||
|
||||
expect(tab).toEqual({
|
||||
targetId: "3",
|
||||
title: "",
|
||||
url: "https://example.com/",
|
||||
type: "page",
|
||||
});
|
||||
});
|
||||
});
|
||||
488
src/browser/chrome-mcp.ts
Normal file
488
src/browser/chrome-mcp.ts
Normal file
@@ -0,0 +1,488 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
||||
import type { ChromeMcpSnapshotNode } from "./chrome-mcp.snapshot.js";
|
||||
import type { BrowserTab } from "./client.js";
|
||||
import { BrowserProfileUnavailableError, BrowserTabNotFoundError } from "./errors.js";
|
||||
|
||||
type ChromeMcpStructuredPage = {
|
||||
id: number;
|
||||
url?: string;
|
||||
selected?: boolean;
|
||||
};
|
||||
|
||||
type ChromeMcpToolResult = {
|
||||
structuredContent?: Record<string, unknown>;
|
||||
content?: Array<Record<string, unknown>>;
|
||||
isError?: boolean;
|
||||
};
|
||||
|
||||
type ChromeMcpSession = {
|
||||
client: Client;
|
||||
transport: StdioClientTransport;
|
||||
ready: Promise<void>;
|
||||
};
|
||||
|
||||
type ChromeMcpSessionFactory = (profileName: string) => Promise<ChromeMcpSession>;
|
||||
|
||||
const DEFAULT_CHROME_MCP_COMMAND = "npx";
|
||||
const DEFAULT_CHROME_MCP_ARGS = [
|
||||
"-y",
|
||||
"chrome-devtools-mcp@latest",
|
||||
"--autoConnect",
|
||||
"--experimental-page-id-routing",
|
||||
];
|
||||
|
||||
const sessions = new Map<string, ChromeMcpSession>();
|
||||
let sessionFactory: ChromeMcpSessionFactory | null = null;
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
return value && typeof value === "object" && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: null;
|
||||
}
|
||||
|
||||
function asPages(value: unknown): ChromeMcpStructuredPage[] {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
const out: ChromeMcpStructuredPage[] = [];
|
||||
for (const entry of value) {
|
||||
const record = asRecord(entry);
|
||||
if (!record || typeof record.id !== "number") {
|
||||
continue;
|
||||
}
|
||||
out.push({
|
||||
id: record.id,
|
||||
url: typeof record.url === "string" ? record.url : undefined,
|
||||
selected: record.selected === true,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function parsePageId(targetId: string): number {
|
||||
const parsed = Number.parseInt(targetId.trim(), 10);
|
||||
if (!Number.isFinite(parsed)) {
|
||||
throw new BrowserTabNotFoundError();
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function toBrowserTabs(pages: ChromeMcpStructuredPage[]): BrowserTab[] {
|
||||
return pages.map((page) => ({
|
||||
targetId: String(page.id),
|
||||
title: "",
|
||||
url: page.url ?? "",
|
||||
type: "page",
|
||||
}));
|
||||
}
|
||||
|
||||
function extractStructuredContent(result: ChromeMcpToolResult): Record<string, unknown> {
|
||||
return asRecord(result.structuredContent) ?? {};
|
||||
}
|
||||
|
||||
function extractTextContent(result: ChromeMcpToolResult): string[] {
|
||||
const content = Array.isArray(result.content) ? result.content : [];
|
||||
return content
|
||||
.map((entry) => {
|
||||
const record = asRecord(entry);
|
||||
return record && typeof record.text === "string" ? record.text : "";
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function extractTextPages(result: ChromeMcpToolResult): ChromeMcpStructuredPage[] {
|
||||
const pages: ChromeMcpStructuredPage[] = [];
|
||||
for (const block of extractTextContent(result)) {
|
||||
for (const line of block.split(/\r?\n/)) {
|
||||
const match = line.match(/^\s*(\d+):\s+(.+?)(?:\s+\[(selected)\])?\s*$/i);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
pages.push({
|
||||
id: Number.parseInt(match[1] ?? "", 10),
|
||||
url: match[2]?.trim() || undefined,
|
||||
selected: Boolean(match[3]),
|
||||
});
|
||||
}
|
||||
}
|
||||
return pages;
|
||||
}
|
||||
|
||||
function extractStructuredPages(result: ChromeMcpToolResult): ChromeMcpStructuredPage[] {
|
||||
const structured = asPages(extractStructuredContent(result).pages);
|
||||
return structured.length > 0 ? structured : extractTextPages(result);
|
||||
}
|
||||
|
||||
function extractSnapshot(result: ChromeMcpToolResult): ChromeMcpSnapshotNode {
|
||||
const structured = extractStructuredContent(result);
|
||||
const snapshot = asRecord(structured.snapshot);
|
||||
if (!snapshot) {
|
||||
throw new Error("Chrome MCP snapshot response was missing structured snapshot data.");
|
||||
}
|
||||
return snapshot as unknown as ChromeMcpSnapshotNode;
|
||||
}
|
||||
|
||||
function extractJsonBlock(text: string): unknown {
|
||||
const match = text.match(/```json\s*([\s\S]*?)\s*```/i);
|
||||
const raw = match?.[1]?.trim() || text.trim();
|
||||
return raw ? JSON.parse(raw) : null;
|
||||
}
|
||||
|
||||
async function createRealSession(profileName: string): Promise<ChromeMcpSession> {
|
||||
const transport = new StdioClientTransport({
|
||||
command: DEFAULT_CHROME_MCP_COMMAND,
|
||||
args: DEFAULT_CHROME_MCP_ARGS,
|
||||
stderr: "pipe",
|
||||
});
|
||||
const client = new Client(
|
||||
{
|
||||
name: "openclaw-browser",
|
||||
version: "0.0.0",
|
||||
},
|
||||
{},
|
||||
);
|
||||
|
||||
const ready = (async () => {
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const tools = await client.listTools();
|
||||
if (!tools.tools.some((tool) => tool.name === "list_pages")) {
|
||||
throw new Error("Chrome MCP server did not expose the expected navigation tools.");
|
||||
}
|
||||
} catch (err) {
|
||||
await client.close().catch(() => {});
|
||||
throw new BrowserProfileUnavailableError(
|
||||
`Chrome MCP existing-session attach failed for profile "${profileName}". ` +
|
||||
`Make sure Chrome is running, enable chrome://inspect/#remote-debugging, and approve the connection. ` +
|
||||
`Details: ${String(err)}`,
|
||||
);
|
||||
}
|
||||
})();
|
||||
|
||||
return {
|
||||
client,
|
||||
transport,
|
||||
ready,
|
||||
};
|
||||
}
|
||||
|
||||
async function getSession(profileName: string): Promise<ChromeMcpSession> {
|
||||
let session = sessions.get(profileName);
|
||||
if (session && session.transport.pid === null) {
|
||||
sessions.delete(profileName);
|
||||
session = undefined;
|
||||
}
|
||||
if (!session) {
|
||||
session = await (sessionFactory ?? createRealSession)(profileName);
|
||||
sessions.set(profileName, session);
|
||||
}
|
||||
try {
|
||||
await session.ready;
|
||||
return session;
|
||||
} catch (err) {
|
||||
const current = sessions.get(profileName);
|
||||
if (current?.transport === session.transport) {
|
||||
sessions.delete(profileName);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function callTool(
|
||||
profileName: string,
|
||||
name: string,
|
||||
args: Record<string, unknown> = {},
|
||||
): Promise<ChromeMcpToolResult> {
|
||||
const session = await getSession(profileName);
|
||||
try {
|
||||
return (await session.client.callTool({
|
||||
name,
|
||||
arguments: args,
|
||||
})) as ChromeMcpToolResult;
|
||||
} catch (err) {
|
||||
sessions.delete(profileName);
|
||||
await session.client.close().catch(() => {});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function withTempFile<T>(fn: (filePath: string) => Promise<T>): Promise<T> {
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-chrome-mcp-"));
|
||||
const filePath = path.join(dir, randomUUID());
|
||||
try {
|
||||
return await fn(filePath);
|
||||
} finally {
|
||||
await fs.rm(dir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
async function findPageById(profileName: string, pageId: number): Promise<ChromeMcpStructuredPage> {
|
||||
const pages = await listChromeMcpPages(profileName);
|
||||
const page = pages.find((entry) => entry.id === pageId);
|
||||
if (!page) {
|
||||
throw new BrowserTabNotFoundError();
|
||||
}
|
||||
return page;
|
||||
}
|
||||
|
||||
export async function ensureChromeMcpAvailable(profileName: string): Promise<void> {
|
||||
await getSession(profileName);
|
||||
}
|
||||
|
||||
export function getChromeMcpPid(profileName: string): number | null {
|
||||
return sessions.get(profileName)?.transport.pid ?? null;
|
||||
}
|
||||
|
||||
export async function closeChromeMcpSession(profileName: string): Promise<boolean> {
|
||||
const session = sessions.get(profileName);
|
||||
if (!session) {
|
||||
return false;
|
||||
}
|
||||
sessions.delete(profileName);
|
||||
await session.client.close().catch(() => {});
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function stopAllChromeMcpSessions(): Promise<void> {
|
||||
const names = [...sessions.keys()];
|
||||
for (const name of names) {
|
||||
await closeChromeMcpSession(name).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
export async function listChromeMcpPages(profileName: string): Promise<ChromeMcpStructuredPage[]> {
|
||||
const result = await callTool(profileName, "list_pages");
|
||||
return extractStructuredPages(result);
|
||||
}
|
||||
|
||||
export async function listChromeMcpTabs(profileName: string): Promise<BrowserTab[]> {
|
||||
return toBrowserTabs(await listChromeMcpPages(profileName));
|
||||
}
|
||||
|
||||
export async function openChromeMcpTab(profileName: string, url: string): Promise<BrowserTab> {
|
||||
const result = await callTool(profileName, "new_page", { url });
|
||||
const pages = extractStructuredPages(result);
|
||||
const chosen = pages.find((page) => page.selected) ?? pages.at(-1);
|
||||
if (!chosen) {
|
||||
throw new Error("Chrome MCP did not return the created page.");
|
||||
}
|
||||
return {
|
||||
targetId: String(chosen.id),
|
||||
title: "",
|
||||
url: chosen.url ?? url,
|
||||
type: "page",
|
||||
};
|
||||
}
|
||||
|
||||
export async function focusChromeMcpTab(profileName: string, targetId: string): Promise<void> {
|
||||
await callTool(profileName, "select_page", {
|
||||
pageId: parsePageId(targetId),
|
||||
bringToFront: true,
|
||||
});
|
||||
}
|
||||
|
||||
export async function closeChromeMcpTab(profileName: string, targetId: string): Promise<void> {
|
||||
await callTool(profileName, "close_page", { pageId: parsePageId(targetId) });
|
||||
}
|
||||
|
||||
export async function navigateChromeMcpPage(params: {
|
||||
profileName: string;
|
||||
targetId: string;
|
||||
url: string;
|
||||
timeoutMs?: number;
|
||||
}): Promise<{ url: string }> {
|
||||
await callTool(params.profileName, "navigate_page", {
|
||||
pageId: parsePageId(params.targetId),
|
||||
type: "url",
|
||||
url: params.url,
|
||||
...(typeof params.timeoutMs === "number" ? { timeout: params.timeoutMs } : {}),
|
||||
});
|
||||
const page = await findPageById(params.profileName, parsePageId(params.targetId));
|
||||
return { url: page.url ?? params.url };
|
||||
}
|
||||
|
||||
export async function takeChromeMcpSnapshot(params: {
|
||||
profileName: string;
|
||||
targetId: string;
|
||||
}): Promise<ChromeMcpSnapshotNode> {
|
||||
const result = await callTool(params.profileName, "take_snapshot", {
|
||||
pageId: parsePageId(params.targetId),
|
||||
});
|
||||
return extractSnapshot(result);
|
||||
}
|
||||
|
||||
export async function takeChromeMcpScreenshot(params: {
|
||||
profileName: string;
|
||||
targetId: string;
|
||||
uid?: string;
|
||||
fullPage?: boolean;
|
||||
format?: "png" | "jpeg";
|
||||
}): Promise<Buffer> {
|
||||
return await withTempFile(async (filePath) => {
|
||||
await callTool(params.profileName, "take_screenshot", {
|
||||
pageId: parsePageId(params.targetId),
|
||||
filePath,
|
||||
format: params.format ?? "png",
|
||||
...(params.uid ? { uid: params.uid } : {}),
|
||||
...(params.fullPage ? { fullPage: true } : {}),
|
||||
});
|
||||
return await fs.readFile(filePath);
|
||||
});
|
||||
}
|
||||
|
||||
export async function clickChromeMcpElement(params: {
|
||||
profileName: string;
|
||||
targetId: string;
|
||||
uid: string;
|
||||
doubleClick?: boolean;
|
||||
}): Promise<void> {
|
||||
await callTool(params.profileName, "click", {
|
||||
pageId: parsePageId(params.targetId),
|
||||
uid: params.uid,
|
||||
...(params.doubleClick ? { dblClick: true } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
export async function fillChromeMcpElement(params: {
|
||||
profileName: string;
|
||||
targetId: string;
|
||||
uid: string;
|
||||
value: string;
|
||||
}): Promise<void> {
|
||||
await callTool(params.profileName, "fill", {
|
||||
pageId: parsePageId(params.targetId),
|
||||
uid: params.uid,
|
||||
value: params.value,
|
||||
});
|
||||
}
|
||||
|
||||
export async function fillChromeMcpForm(params: {
|
||||
profileName: string;
|
||||
targetId: string;
|
||||
elements: Array<{ uid: string; value: string }>;
|
||||
}): Promise<void> {
|
||||
await callTool(params.profileName, "fill_form", {
|
||||
pageId: parsePageId(params.targetId),
|
||||
elements: params.elements,
|
||||
});
|
||||
}
|
||||
|
||||
export async function hoverChromeMcpElement(params: {
|
||||
profileName: string;
|
||||
targetId: string;
|
||||
uid: string;
|
||||
}): Promise<void> {
|
||||
await callTool(params.profileName, "hover", {
|
||||
pageId: parsePageId(params.targetId),
|
||||
uid: params.uid,
|
||||
});
|
||||
}
|
||||
|
||||
export async function dragChromeMcpElement(params: {
|
||||
profileName: string;
|
||||
targetId: string;
|
||||
fromUid: string;
|
||||
toUid: string;
|
||||
}): Promise<void> {
|
||||
await callTool(params.profileName, "drag", {
|
||||
pageId: parsePageId(params.targetId),
|
||||
from_uid: params.fromUid,
|
||||
to_uid: params.toUid,
|
||||
});
|
||||
}
|
||||
|
||||
export async function uploadChromeMcpFile(params: {
|
||||
profileName: string;
|
||||
targetId: string;
|
||||
uid: string;
|
||||
filePath: string;
|
||||
}): Promise<void> {
|
||||
await callTool(params.profileName, "upload_file", {
|
||||
pageId: parsePageId(params.targetId),
|
||||
uid: params.uid,
|
||||
filePath: params.filePath,
|
||||
});
|
||||
}
|
||||
|
||||
export async function pressChromeMcpKey(params: {
|
||||
profileName: string;
|
||||
targetId: string;
|
||||
key: string;
|
||||
}): Promise<void> {
|
||||
await callTool(params.profileName, "press_key", {
|
||||
pageId: parsePageId(params.targetId),
|
||||
key: params.key,
|
||||
});
|
||||
}
|
||||
|
||||
export async function resizeChromeMcpPage(params: {
|
||||
profileName: string;
|
||||
targetId: string;
|
||||
width: number;
|
||||
height: number;
|
||||
}): Promise<void> {
|
||||
await callTool(params.profileName, "resize_page", {
|
||||
pageId: parsePageId(params.targetId),
|
||||
width: params.width,
|
||||
height: params.height,
|
||||
});
|
||||
}
|
||||
|
||||
export async function handleChromeMcpDialog(params: {
|
||||
profileName: string;
|
||||
targetId: string;
|
||||
action: "accept" | "dismiss";
|
||||
promptText?: string;
|
||||
}): Promise<void> {
|
||||
await callTool(params.profileName, "handle_dialog", {
|
||||
pageId: parsePageId(params.targetId),
|
||||
action: params.action,
|
||||
...(params.promptText ? { promptText: params.promptText } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
export async function evaluateChromeMcpScript(params: {
|
||||
profileName: string;
|
||||
targetId: string;
|
||||
fn: string;
|
||||
args?: string[];
|
||||
}): Promise<unknown> {
|
||||
const result = await callTool(params.profileName, "evaluate_script", {
|
||||
pageId: parsePageId(params.targetId),
|
||||
function: params.fn,
|
||||
...(params.args?.length ? { args: params.args } : {}),
|
||||
});
|
||||
const message = extractStructuredContent(result).message;
|
||||
const text = typeof message === "string" ? message : "";
|
||||
if (!text.trim()) {
|
||||
return null;
|
||||
}
|
||||
return extractJsonBlock(text);
|
||||
}
|
||||
|
||||
export async function waitForChromeMcpText(params: {
|
||||
profileName: string;
|
||||
targetId: string;
|
||||
text: string[];
|
||||
timeoutMs?: number;
|
||||
}): Promise<void> {
|
||||
await callTool(params.profileName, "wait_for", {
|
||||
pageId: parsePageId(params.targetId),
|
||||
text: params.text,
|
||||
...(typeof params.timeoutMs === "number" ? { timeout: params.timeoutMs } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
export function setChromeMcpSessionFactoryForTest(factory: ChromeMcpSessionFactory | null): void {
|
||||
sessionFactory = factory;
|
||||
}
|
||||
|
||||
export async function resetChromeMcpSessionsForTest(): Promise<void> {
|
||||
sessionFactory = null;
|
||||
await stopAllChromeMcpSessions();
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { fetchBrowserJson } from "./client-fetch.js";
|
||||
export type BrowserStatus = {
|
||||
enabled: boolean;
|
||||
profile?: string;
|
||||
driver?: "openclaw" | "extension" | "existing-session";
|
||||
running: boolean;
|
||||
cdpReady?: boolean;
|
||||
cdpHttp?: boolean;
|
||||
@@ -26,6 +27,7 @@ export type ProfileStatus = {
|
||||
cdpPort: number;
|
||||
cdpUrl: string;
|
||||
color: string;
|
||||
driver: "openclaw" | "extension" | "existing-session";
|
||||
running: boolean;
|
||||
tabCount: number;
|
||||
isDefault: boolean;
|
||||
@@ -165,7 +167,7 @@ export async function browserCreateProfile(
|
||||
name: string;
|
||||
color?: string;
|
||||
cdpUrl?: string;
|
||||
driver?: "openclaw" | "extension";
|
||||
driver?: "openclaw" | "extension" | "existing-session";
|
||||
},
|
||||
): Promise<BrowserCreateProfileResult> {
|
||||
return await fetchBrowserJson<BrowserCreateProfileResult>(
|
||||
|
||||
@@ -46,7 +46,7 @@ export type ResolvedBrowserProfile = {
|
||||
cdpHost: string;
|
||||
cdpIsLoopback: boolean;
|
||||
color: string;
|
||||
driver: "openclaw" | "extension";
|
||||
driver: "openclaw" | "extension" | "existing-session";
|
||||
attachOnly: boolean;
|
||||
};
|
||||
|
||||
@@ -335,7 +335,12 @@ export function resolveProfile(
|
||||
let cdpHost = resolved.cdpHost;
|
||||
let cdpPort = profile.cdpPort ?? 0;
|
||||
let cdpUrl = "";
|
||||
const driver = profile.driver === "extension" ? "extension" : "openclaw";
|
||||
const driver =
|
||||
profile.driver === "extension"
|
||||
? "extension"
|
||||
: profile.driver === "existing-session"
|
||||
? "existing-session"
|
||||
: "openclaw";
|
||||
|
||||
if (rawProfileUrl) {
|
||||
const parsed = parseHttpUrl(rawProfileUrl, `browser.profiles.${profileName}.cdpUrl`);
|
||||
@@ -356,7 +361,7 @@ export function resolveProfile(
|
||||
cdpIsLoopback: isLoopbackHost(cdpHost),
|
||||
color: profile.color,
|
||||
driver,
|
||||
attachOnly: profile.attachOnly ?? resolved.attachOnly,
|
||||
attachOnly: driver === "existing-session" ? true : (profile.attachOnly ?? resolved.attachOnly),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
import type { ResolvedBrowserProfile } from "./config.js";
|
||||
|
||||
export type BrowserProfileMode = "local-managed" | "local-extension-relay" | "remote-cdp";
|
||||
export type BrowserProfileMode =
|
||||
| "local-managed"
|
||||
| "local-extension-relay"
|
||||
| "local-existing-session"
|
||||
| "remote-cdp";
|
||||
|
||||
export type BrowserProfileCapabilities = {
|
||||
mode: BrowserProfileMode;
|
||||
@@ -31,6 +35,20 @@ export function getBrowserProfileCapabilities(
|
||||
};
|
||||
}
|
||||
|
||||
if (profile.driver === "existing-session") {
|
||||
return {
|
||||
mode: "local-existing-session",
|
||||
isRemote: false,
|
||||
requiresRelay: false,
|
||||
requiresAttachedTab: false,
|
||||
usesPersistentPlaywright: false,
|
||||
supportsPerTabWs: false,
|
||||
supportsJsonTabEndpoints: false,
|
||||
supportsReset: false,
|
||||
supportsManagedTabLimit: false,
|
||||
};
|
||||
}
|
||||
|
||||
if (!profile.cdpIsLoopback) {
|
||||
return {
|
||||
mode: "remote-cdp",
|
||||
@@ -75,6 +93,9 @@ export function resolveDefaultSnapshotFormat(params: {
|
||||
if (capabilities.mode === "local-extension-relay") {
|
||||
return "aria";
|
||||
}
|
||||
if (capabilities.mode === "local-existing-session") {
|
||||
return "ai";
|
||||
}
|
||||
|
||||
return params.hasPlaywright ? "ai" : "aria";
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { resolveBrowserConfig } from "./config.js";
|
||||
import { createBrowserProfilesService } from "./profiles-service.js";
|
||||
import type { BrowserRouteContext, BrowserServerState } from "./server-context.js";
|
||||
@@ -57,6 +57,10 @@ async function createWorkProfileWithConfig(params: {
|
||||
}
|
||||
|
||||
describe("BrowserProfilesService", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("allocates next local port for new profiles", async () => {
|
||||
const { result, state } = await createWorkProfileWithConfig({
|
||||
resolved: resolveBrowserConfig({}),
|
||||
@@ -163,6 +167,56 @@ describe("BrowserProfilesService", () => {
|
||||
).rejects.toThrow(/requires an explicit loopback cdpUrl/i);
|
||||
});
|
||||
|
||||
it("creates existing-session profiles as attach-only local entries", async () => {
|
||||
const resolved = resolveBrowserConfig({});
|
||||
const { ctx, state } = createCtx(resolved);
|
||||
vi.mocked(loadConfig).mockReturnValue({ browser: { profiles: {} } });
|
||||
|
||||
const service = createBrowserProfilesService(ctx);
|
||||
const result = await service.createProfile({
|
||||
name: "chrome-live",
|
||||
driver: "existing-session",
|
||||
});
|
||||
|
||||
expect(result.cdpPort).toBe(18801);
|
||||
expect(result.isRemote).toBe(false);
|
||||
expect(state.resolved.profiles["chrome-live"]).toEqual({
|
||||
cdpPort: 18801,
|
||||
driver: "existing-session",
|
||||
attachOnly: true,
|
||||
color: expect.any(String),
|
||||
});
|
||||
expect(writeConfigFile).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
browser: expect.objectContaining({
|
||||
profiles: expect.objectContaining({
|
||||
"chrome-live": expect.objectContaining({
|
||||
cdpPort: 18801,
|
||||
driver: "existing-session",
|
||||
attachOnly: true,
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects driver=existing-session when cdpUrl is provided", async () => {
|
||||
const resolved = resolveBrowserConfig({});
|
||||
const { ctx } = createCtx(resolved);
|
||||
vi.mocked(loadConfig).mockReturnValue({ browser: { profiles: {} } });
|
||||
|
||||
const service = createBrowserProfilesService(ctx);
|
||||
|
||||
await expect(
|
||||
service.createProfile({
|
||||
name: "chrome-live",
|
||||
driver: "existing-session",
|
||||
cdpUrl: "http://127.0.0.1:9222",
|
||||
}),
|
||||
).rejects.toThrow(/does not accept cdpUrl/i);
|
||||
});
|
||||
|
||||
it("deletes remote profiles without stopping or removing local data", async () => {
|
||||
const resolved = resolveBrowserConfig({
|
||||
profiles: {
|
||||
@@ -218,4 +272,40 @@ describe("BrowserProfilesService", () => {
|
||||
expect(result.deleted).toBe(true);
|
||||
expect(movePathToTrash).toHaveBeenCalledWith(path.dirname(userDataDir));
|
||||
});
|
||||
|
||||
it("deletes existing-session profiles without touching local browser data", async () => {
|
||||
const resolved = resolveBrowserConfig({
|
||||
profiles: {
|
||||
"chrome-live": {
|
||||
cdpPort: 18801,
|
||||
color: "#0066CC",
|
||||
driver: "existing-session",
|
||||
attachOnly: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
const { ctx } = createCtx(resolved);
|
||||
|
||||
vi.mocked(loadConfig).mockReturnValue({
|
||||
browser: {
|
||||
defaultProfile: "openclaw",
|
||||
profiles: {
|
||||
openclaw: { cdpPort: 18800, color: "#FF4500" },
|
||||
"chrome-live": {
|
||||
cdpPort: 18801,
|
||||
color: "#0066CC",
|
||||
driver: "existing-session",
|
||||
attachOnly: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const service = createBrowserProfilesService(ctx);
|
||||
const result = await service.deleteProfile("chrome-live");
|
||||
|
||||
expect(result.deleted).toBe(false);
|
||||
expect(ctx.forProfile).not.toHaveBeenCalled();
|
||||
expect(movePathToTrash).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -27,7 +27,7 @@ export type CreateProfileParams = {
|
||||
name: string;
|
||||
color?: string;
|
||||
cdpUrl?: string;
|
||||
driver?: "openclaw" | "extension";
|
||||
driver?: "openclaw" | "extension" | "existing-session";
|
||||
};
|
||||
|
||||
export type CreateProfileResult = {
|
||||
@@ -79,7 +79,12 @@ export function createBrowserProfilesService(ctx: BrowserRouteContext) {
|
||||
const createProfile = async (params: CreateProfileParams): Promise<CreateProfileResult> => {
|
||||
const name = params.name.trim();
|
||||
const rawCdpUrl = params.cdpUrl?.trim() || undefined;
|
||||
const driver = params.driver === "extension" ? "extension" : undefined;
|
||||
const driver =
|
||||
params.driver === "extension"
|
||||
? "extension"
|
||||
: params.driver === "existing-session"
|
||||
? "existing-session"
|
||||
: undefined;
|
||||
|
||||
if (!isValidProfileName(name)) {
|
||||
throw new BrowserValidationError(
|
||||
@@ -118,6 +123,11 @@ export function createBrowserProfilesService(ctx: BrowserRouteContext) {
|
||||
);
|
||||
}
|
||||
}
|
||||
if (driver === "existing-session") {
|
||||
throw new BrowserValidationError(
|
||||
"driver=existing-session does not accept cdpUrl; it attaches via the Chrome MCP auto-connect flow",
|
||||
);
|
||||
}
|
||||
profileConfig = {
|
||||
cdpUrl: parsed.normalized,
|
||||
...(driver ? { driver } : {}),
|
||||
@@ -136,6 +146,7 @@ export function createBrowserProfilesService(ctx: BrowserRouteContext) {
|
||||
profileConfig = {
|
||||
cdpPort,
|
||||
...(driver ? { driver } : {}),
|
||||
...(driver === "existing-session" ? { attachOnly: true } : {}),
|
||||
color: profileColor,
|
||||
};
|
||||
}
|
||||
@@ -195,7 +206,7 @@ export function createBrowserProfilesService(ctx: BrowserRouteContext) {
|
||||
const state = ctx.state();
|
||||
const resolved = resolveProfile(state.resolved, name);
|
||||
|
||||
if (resolved?.cdpIsLoopback) {
|
||||
if (resolved?.cdpIsLoopback && resolved.driver === "openclaw") {
|
||||
try {
|
||||
await ctx.forProfile(name).stopRunningBrowser();
|
||||
} catch {
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import type { BrowserRouteContext } from "../server-context.js";
|
||||
import { readBody, resolveTargetIdFromBody, withPlaywrightRouteContext } from "./agent.shared.js";
|
||||
import {
|
||||
readBody,
|
||||
requirePwAi,
|
||||
resolveTargetIdFromBody,
|
||||
withRouteTabContext,
|
||||
} from "./agent.shared.js";
|
||||
import { ensureOutputRootDir, resolveWritableOutputPathOrRespond } from "./output-paths.js";
|
||||
import { DEFAULT_DOWNLOAD_DIR } from "./path-output.js";
|
||||
import type { BrowserRouteRegistrar } from "./types.js";
|
||||
@@ -23,13 +28,23 @@ export function registerBrowserAgentActDownloadRoutes(
|
||||
const out = toStringOrEmpty(body.path) || "";
|
||||
const timeoutMs = toNumber(body.timeoutMs);
|
||||
|
||||
await withPlaywrightRouteContext({
|
||||
await withRouteTabContext({
|
||||
req,
|
||||
res,
|
||||
ctx,
|
||||
targetId,
|
||||
feature: "wait for download",
|
||||
run: async ({ cdpUrl, tab, pw }) => {
|
||||
run: async ({ profileCtx, cdpUrl, tab }) => {
|
||||
if (profileCtx.profile.driver === "existing-session") {
|
||||
return jsonError(
|
||||
res,
|
||||
501,
|
||||
"download waiting is not supported for existing-session profiles yet.",
|
||||
);
|
||||
}
|
||||
const pw = await requirePwAi(res, "wait for download");
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
await ensureOutputRootDir(DEFAULT_DOWNLOAD_DIR);
|
||||
let downloadPath: string | undefined;
|
||||
if (out.trim()) {
|
||||
@@ -67,13 +82,23 @@ export function registerBrowserAgentActDownloadRoutes(
|
||||
return jsonError(res, 400, "path is required");
|
||||
}
|
||||
|
||||
await withPlaywrightRouteContext({
|
||||
await withRouteTabContext({
|
||||
req,
|
||||
res,
|
||||
ctx,
|
||||
targetId,
|
||||
feature: "download",
|
||||
run: async ({ cdpUrl, tab, pw }) => {
|
||||
run: async ({ profileCtx, cdpUrl, tab }) => {
|
||||
if (profileCtx.profile.driver === "existing-session") {
|
||||
return jsonError(
|
||||
res,
|
||||
501,
|
||||
"downloads are not supported for existing-session profiles yet.",
|
||||
);
|
||||
}
|
||||
const pw = await requirePwAi(res, "download");
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
await ensureOutputRootDir(DEFAULT_DOWNLOAD_DIR);
|
||||
const downloadPath = await resolveWritableOutputPathOrRespond({
|
||||
res,
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
import { evaluateChromeMcpScript, uploadChromeMcpFile } from "../chrome-mcp.js";
|
||||
import type { BrowserRouteContext } from "../server-context.js";
|
||||
import { readBody, resolveTargetIdFromBody, withPlaywrightRouteContext } from "./agent.shared.js";
|
||||
import {
|
||||
readBody,
|
||||
requirePwAi,
|
||||
resolveTargetIdFromBody,
|
||||
withRouteTabContext,
|
||||
} from "./agent.shared.js";
|
||||
import { DEFAULT_UPLOAD_DIR, resolveExistingPathsWithinRoot } from "./path-output.js";
|
||||
import type { BrowserRouteRegistrar } from "./types.js";
|
||||
import { jsonError, toBoolean, toNumber, toStringArray, toStringOrEmpty } from "./utils.js";
|
||||
@@ -20,13 +26,12 @@ export function registerBrowserAgentActHookRoutes(
|
||||
return jsonError(res, 400, "paths are required");
|
||||
}
|
||||
|
||||
await withPlaywrightRouteContext({
|
||||
await withRouteTabContext({
|
||||
req,
|
||||
res,
|
||||
ctx,
|
||||
targetId,
|
||||
feature: "file chooser hook",
|
||||
run: async ({ cdpUrl, tab, pw }) => {
|
||||
run: async ({ profileCtx, cdpUrl, tab }) => {
|
||||
const uploadPathsResult = await resolveExistingPathsWithinRoot({
|
||||
rootDir: DEFAULT_UPLOAD_DIR,
|
||||
requestedPaths: paths,
|
||||
@@ -38,6 +43,39 @@ export function registerBrowserAgentActHookRoutes(
|
||||
}
|
||||
const resolvedPaths = uploadPathsResult.paths;
|
||||
|
||||
if (profileCtx.profile.driver === "existing-session") {
|
||||
if (element) {
|
||||
return jsonError(
|
||||
res,
|
||||
501,
|
||||
"existing-session file uploads do not support element selectors; use ref/inputRef.",
|
||||
);
|
||||
}
|
||||
if (resolvedPaths.length !== 1) {
|
||||
return jsonError(
|
||||
res,
|
||||
501,
|
||||
"existing-session file uploads currently support one file at a time.",
|
||||
);
|
||||
}
|
||||
const uid = inputRef || ref;
|
||||
if (!uid) {
|
||||
return jsonError(res, 501, "existing-session file uploads require ref or inputRef.");
|
||||
}
|
||||
await uploadChromeMcpFile({
|
||||
profileName: profileCtx.profile.name,
|
||||
targetId: tab.targetId,
|
||||
uid,
|
||||
filePath: resolvedPaths[0] ?? "",
|
||||
});
|
||||
return res.json({ ok: true });
|
||||
}
|
||||
|
||||
const pw = await requirePwAi(res, "file chooser hook");
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (inputRef || element) {
|
||||
if (ref) {
|
||||
return jsonError(res, 400, "ref cannot be combined with inputRef/element");
|
||||
@@ -79,13 +117,69 @@ export function registerBrowserAgentActHookRoutes(
|
||||
return jsonError(res, 400, "accept is required");
|
||||
}
|
||||
|
||||
await withPlaywrightRouteContext({
|
||||
await withRouteTabContext({
|
||||
req,
|
||||
res,
|
||||
ctx,
|
||||
targetId,
|
||||
feature: "dialog hook",
|
||||
run: async ({ cdpUrl, tab, pw }) => {
|
||||
run: async ({ profileCtx, cdpUrl, tab }) => {
|
||||
if (profileCtx.profile.driver === "existing-session") {
|
||||
if (timeoutMs) {
|
||||
return jsonError(
|
||||
res,
|
||||
501,
|
||||
"existing-session dialog handling does not support timeoutMs.",
|
||||
);
|
||||
}
|
||||
await evaluateChromeMcpScript({
|
||||
profileName: profileCtx.profile.name,
|
||||
targetId: tab.targetId,
|
||||
fn: `() => {
|
||||
const state = (window.__openclawDialogHook ??= {});
|
||||
if (!state.originals) {
|
||||
state.originals = {
|
||||
alert: window.alert.bind(window),
|
||||
confirm: window.confirm.bind(window),
|
||||
prompt: window.prompt.bind(window),
|
||||
};
|
||||
}
|
||||
const originals = state.originals;
|
||||
const restore = () => {
|
||||
window.alert = originals.alert;
|
||||
window.confirm = originals.confirm;
|
||||
window.prompt = originals.prompt;
|
||||
delete window.__openclawDialogHook;
|
||||
};
|
||||
window.alert = (...args) => {
|
||||
try {
|
||||
return undefined;
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
};
|
||||
window.confirm = (...args) => {
|
||||
try {
|
||||
return ${accept ? "true" : "false"};
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
};
|
||||
window.prompt = (...args) => {
|
||||
try {
|
||||
return ${accept ? JSON.stringify(promptText ?? "") : "null"};
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
};
|
||||
return true;
|
||||
}`,
|
||||
});
|
||||
return res.json({ ok: true });
|
||||
}
|
||||
const pw = await requirePwAi(res, "dialog hook");
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
await pw.armDialogViaPlaywright({
|
||||
cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
|
||||
@@ -1,3 +1,14 @@
|
||||
import {
|
||||
clickChromeMcpElement,
|
||||
closeChromeMcpTab,
|
||||
dragChromeMcpElement,
|
||||
evaluateChromeMcpScript,
|
||||
fillChromeMcpElement,
|
||||
fillChromeMcpForm,
|
||||
hoverChromeMcpElement,
|
||||
pressChromeMcpKey,
|
||||
resizeChromeMcpPage,
|
||||
} from "../chrome-mcp.js";
|
||||
import type { BrowserFormField } from "../client-actions-core.js";
|
||||
import { normalizeBrowserFormField } from "../form-fields.js";
|
||||
import type { BrowserRouteContext } from "../server-context.js";
|
||||
@@ -11,13 +22,88 @@ import {
|
||||
} from "./agent.act.shared.js";
|
||||
import {
|
||||
readBody,
|
||||
requirePwAi,
|
||||
resolveTargetIdFromBody,
|
||||
withPlaywrightRouteContext,
|
||||
withRouteTabContext,
|
||||
SELECTOR_UNSUPPORTED_MESSAGE,
|
||||
} from "./agent.shared.js";
|
||||
import type { BrowserRouteRegistrar } from "./types.js";
|
||||
import { jsonError, toBoolean, toNumber, toStringArray, toStringOrEmpty } from "./utils.js";
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function buildExistingSessionWaitPredicate(params: {
|
||||
text?: string;
|
||||
textGone?: string;
|
||||
selector?: string;
|
||||
url?: string;
|
||||
loadState?: "load" | "domcontentloaded" | "networkidle";
|
||||
fn?: string;
|
||||
}): string | null {
|
||||
const checks: string[] = [];
|
||||
if (params.text) {
|
||||
checks.push(`Boolean(document.body?.innerText?.includes(${JSON.stringify(params.text)}))`);
|
||||
}
|
||||
if (params.textGone) {
|
||||
checks.push(`!document.body?.innerText?.includes(${JSON.stringify(params.textGone)})`);
|
||||
}
|
||||
if (params.selector) {
|
||||
checks.push(`Boolean(document.querySelector(${JSON.stringify(params.selector)}))`);
|
||||
}
|
||||
if (params.url) {
|
||||
checks.push(`window.location.href === ${JSON.stringify(params.url)}`);
|
||||
}
|
||||
if (params.loadState === "domcontentloaded") {
|
||||
checks.push(`document.readyState === "interactive" || document.readyState === "complete"`);
|
||||
} else if (params.loadState === "load" || params.loadState === "networkidle") {
|
||||
checks.push(`document.readyState === "complete"`);
|
||||
}
|
||||
if (params.fn) {
|
||||
checks.push(`Boolean(await (${params.fn})())`);
|
||||
}
|
||||
if (checks.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return checks.length === 1 ? checks[0] : checks.map((check) => `(${check})`).join(" && ");
|
||||
}
|
||||
|
||||
async function waitForExistingSessionCondition(params: {
|
||||
profileName: string;
|
||||
targetId: string;
|
||||
timeMs?: number;
|
||||
text?: string;
|
||||
textGone?: string;
|
||||
selector?: string;
|
||||
url?: string;
|
||||
loadState?: "load" | "domcontentloaded" | "networkidle";
|
||||
fn?: string;
|
||||
timeoutMs?: number;
|
||||
}): Promise<void> {
|
||||
if (params.timeMs && params.timeMs > 0) {
|
||||
await sleep(params.timeMs);
|
||||
}
|
||||
const predicate = buildExistingSessionWaitPredicate(params);
|
||||
if (!predicate) {
|
||||
return;
|
||||
}
|
||||
const timeoutMs = Math.max(250, params.timeoutMs ?? 10_000);
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
const ready = await evaluateChromeMcpScript({
|
||||
profileName: params.profileName,
|
||||
targetId: params.targetId,
|
||||
fn: `async () => ${predicate}`,
|
||||
});
|
||||
if (ready) {
|
||||
return;
|
||||
}
|
||||
await sleep(250);
|
||||
}
|
||||
throw new Error("Timed out waiting for condition");
|
||||
}
|
||||
|
||||
export function registerBrowserAgentActRoutes(
|
||||
app: BrowserRouteRegistrar,
|
||||
ctx: BrowserRouteContext,
|
||||
@@ -34,14 +120,15 @@ export function registerBrowserAgentActRoutes(
|
||||
return jsonError(res, 400, SELECTOR_UNSUPPORTED_MESSAGE);
|
||||
}
|
||||
|
||||
await withPlaywrightRouteContext({
|
||||
await withRouteTabContext({
|
||||
req,
|
||||
res,
|
||||
ctx,
|
||||
targetId,
|
||||
feature: `act:${kind}`,
|
||||
run: async ({ cdpUrl, tab, pw }) => {
|
||||
run: async ({ profileCtx, cdpUrl, tab }) => {
|
||||
const evaluateEnabled = ctx.state().resolved.evaluateEnabled;
|
||||
const isExistingSession = profileCtx.profile.driver === "existing-session";
|
||||
const profileName = profileCtx.profile.name;
|
||||
|
||||
switch (kind) {
|
||||
case "click": {
|
||||
@@ -63,6 +150,26 @@ export function registerBrowserAgentActRoutes(
|
||||
return jsonError(res, 400, parsedModifiers.error);
|
||||
}
|
||||
const modifiers = parsedModifiers.modifiers;
|
||||
if (isExistingSession) {
|
||||
if ((button && button !== "left") || (modifiers && modifiers.length > 0)) {
|
||||
return jsonError(
|
||||
res,
|
||||
501,
|
||||
"existing-session click currently supports left-click only (no button overrides/modifiers).",
|
||||
);
|
||||
}
|
||||
await clickChromeMcpElement({
|
||||
profileName,
|
||||
targetId: tab.targetId,
|
||||
uid: ref,
|
||||
doubleClick,
|
||||
});
|
||||
return res.json({ ok: true, targetId: tab.targetId, url: tab.url });
|
||||
}
|
||||
const pw = await requirePwAi(res, `act:${kind}`);
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
const clickRequest: Parameters<typeof pw.clickViaPlaywright>[0] = {
|
||||
cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
@@ -93,6 +200,33 @@ export function registerBrowserAgentActRoutes(
|
||||
const submit = toBoolean(body.submit) ?? false;
|
||||
const slowly = toBoolean(body.slowly) ?? false;
|
||||
const timeoutMs = toNumber(body.timeoutMs);
|
||||
if (isExistingSession) {
|
||||
if (slowly) {
|
||||
return jsonError(
|
||||
res,
|
||||
501,
|
||||
"existing-session type does not support slowly=true; use fill/press instead.",
|
||||
);
|
||||
}
|
||||
await fillChromeMcpElement({
|
||||
profileName,
|
||||
targetId: tab.targetId,
|
||||
uid: ref,
|
||||
value: text,
|
||||
});
|
||||
if (submit) {
|
||||
await pressChromeMcpKey({
|
||||
profileName,
|
||||
targetId: tab.targetId,
|
||||
key: "Enter",
|
||||
});
|
||||
}
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
const pw = await requirePwAi(res, `act:${kind}`);
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
const typeRequest: Parameters<typeof pw.typeViaPlaywright>[0] = {
|
||||
cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
@@ -113,6 +247,17 @@ export function registerBrowserAgentActRoutes(
|
||||
return jsonError(res, 400, "key is required");
|
||||
}
|
||||
const delayMs = toNumber(body.delayMs);
|
||||
if (isExistingSession) {
|
||||
if (delayMs) {
|
||||
return jsonError(res, 501, "existing-session press does not support delayMs.");
|
||||
}
|
||||
await pressChromeMcpKey({ profileName, targetId: tab.targetId, key });
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
const pw = await requirePwAi(res, `act:${kind}`);
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
await pw.pressKeyViaPlaywright({
|
||||
cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
@@ -127,6 +272,21 @@ export function registerBrowserAgentActRoutes(
|
||||
return jsonError(res, 400, "ref is required");
|
||||
}
|
||||
const timeoutMs = toNumber(body.timeoutMs);
|
||||
if (isExistingSession) {
|
||||
if (timeoutMs) {
|
||||
return jsonError(
|
||||
res,
|
||||
501,
|
||||
"existing-session hover does not support timeoutMs overrides.",
|
||||
);
|
||||
}
|
||||
await hoverChromeMcpElement({ profileName, targetId: tab.targetId, uid: ref });
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
const pw = await requirePwAi(res, `act:${kind}`);
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
await pw.hoverViaPlaywright({
|
||||
cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
@@ -141,6 +301,26 @@ export function registerBrowserAgentActRoutes(
|
||||
return jsonError(res, 400, "ref is required");
|
||||
}
|
||||
const timeoutMs = toNumber(body.timeoutMs);
|
||||
if (isExistingSession) {
|
||||
if (timeoutMs) {
|
||||
return jsonError(
|
||||
res,
|
||||
501,
|
||||
"existing-session scrollIntoView does not support timeoutMs overrides.",
|
||||
);
|
||||
}
|
||||
await evaluateChromeMcpScript({
|
||||
profileName,
|
||||
targetId: tab.targetId,
|
||||
fn: `(el) => { el.scrollIntoView({ block: "center", inline: "center" }); return true; }`,
|
||||
args: [ref],
|
||||
});
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
const pw = await requirePwAi(res, `act:${kind}`);
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
const scrollRequest: Parameters<typeof pw.scrollIntoViewViaPlaywright>[0] = {
|
||||
cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
@@ -159,6 +339,26 @@ export function registerBrowserAgentActRoutes(
|
||||
return jsonError(res, 400, "startRef and endRef are required");
|
||||
}
|
||||
const timeoutMs = toNumber(body.timeoutMs);
|
||||
if (isExistingSession) {
|
||||
if (timeoutMs) {
|
||||
return jsonError(
|
||||
res,
|
||||
501,
|
||||
"existing-session drag does not support timeoutMs overrides.",
|
||||
);
|
||||
}
|
||||
await dragChromeMcpElement({
|
||||
profileName,
|
||||
targetId: tab.targetId,
|
||||
fromUid: startRef,
|
||||
toUid: endRef,
|
||||
});
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
const pw = await requirePwAi(res, `act:${kind}`);
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
await pw.dragViaPlaywright({
|
||||
cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
@@ -175,6 +375,33 @@ export function registerBrowserAgentActRoutes(
|
||||
return jsonError(res, 400, "ref and values are required");
|
||||
}
|
||||
const timeoutMs = toNumber(body.timeoutMs);
|
||||
if (isExistingSession) {
|
||||
if (values.length !== 1) {
|
||||
return jsonError(
|
||||
res,
|
||||
501,
|
||||
"existing-session select currently supports a single value only.",
|
||||
);
|
||||
}
|
||||
if (timeoutMs) {
|
||||
return jsonError(
|
||||
res,
|
||||
501,
|
||||
"existing-session select does not support timeoutMs overrides.",
|
||||
);
|
||||
}
|
||||
await fillChromeMcpElement({
|
||||
profileName,
|
||||
targetId: tab.targetId,
|
||||
uid: ref,
|
||||
value: values[0] ?? "",
|
||||
});
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
const pw = await requirePwAi(res, `act:${kind}`);
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
await pw.selectOptionViaPlaywright({
|
||||
cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
@@ -198,6 +425,28 @@ export function registerBrowserAgentActRoutes(
|
||||
return jsonError(res, 400, "fields are required");
|
||||
}
|
||||
const timeoutMs = toNumber(body.timeoutMs);
|
||||
if (isExistingSession) {
|
||||
if (timeoutMs) {
|
||||
return jsonError(
|
||||
res,
|
||||
501,
|
||||
"existing-session fill does not support timeoutMs overrides.",
|
||||
);
|
||||
}
|
||||
await fillChromeMcpForm({
|
||||
profileName,
|
||||
targetId: tab.targetId,
|
||||
elements: fields.map((field) => ({
|
||||
uid: field.ref,
|
||||
value: String(field.value ?? ""),
|
||||
})),
|
||||
});
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
const pw = await requirePwAi(res, `act:${kind}`);
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
await pw.fillFormViaPlaywright({
|
||||
cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
@@ -212,6 +461,19 @@ export function registerBrowserAgentActRoutes(
|
||||
if (!width || !height) {
|
||||
return jsonError(res, 400, "width and height are required");
|
||||
}
|
||||
if (isExistingSession) {
|
||||
await resizeChromeMcpPage({
|
||||
profileName,
|
||||
targetId: tab.targetId,
|
||||
width,
|
||||
height,
|
||||
});
|
||||
return res.json({ ok: true, targetId: tab.targetId, url: tab.url });
|
||||
}
|
||||
const pw = await requirePwAi(res, `act:${kind}`);
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
await pw.resizeViewportViaPlaywright({
|
||||
cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
@@ -260,6 +522,25 @@ export function registerBrowserAgentActRoutes(
|
||||
"wait requires at least one of: timeMs, text, textGone, selector, url, loadState, fn",
|
||||
);
|
||||
}
|
||||
if (isExistingSession) {
|
||||
await waitForExistingSessionCondition({
|
||||
profileName,
|
||||
targetId: tab.targetId,
|
||||
timeMs,
|
||||
text,
|
||||
textGone,
|
||||
selector,
|
||||
url,
|
||||
loadState,
|
||||
fn,
|
||||
timeoutMs,
|
||||
});
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
const pw = await requirePwAi(res, `act:${kind}`);
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
await pw.waitForViaPlaywright({
|
||||
cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
@@ -291,6 +572,31 @@ export function registerBrowserAgentActRoutes(
|
||||
}
|
||||
const ref = toStringOrEmpty(body.ref) || undefined;
|
||||
const evalTimeoutMs = toNumber(body.timeoutMs);
|
||||
if (isExistingSession) {
|
||||
if (evalTimeoutMs !== undefined) {
|
||||
return jsonError(
|
||||
res,
|
||||
501,
|
||||
"existing-session evaluate does not support timeoutMs overrides.",
|
||||
);
|
||||
}
|
||||
const result = await evaluateChromeMcpScript({
|
||||
profileName,
|
||||
targetId: tab.targetId,
|
||||
fn,
|
||||
args: ref ? [ref] : undefined,
|
||||
});
|
||||
return res.json({
|
||||
ok: true,
|
||||
targetId: tab.targetId,
|
||||
url: tab.url,
|
||||
result,
|
||||
});
|
||||
}
|
||||
const pw = await requirePwAi(res, `act:${kind}`);
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
const evalRequest: Parameters<typeof pw.evaluateViaPlaywright>[0] = {
|
||||
cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
@@ -310,6 +616,14 @@ export function registerBrowserAgentActRoutes(
|
||||
});
|
||||
}
|
||||
case "close": {
|
||||
if (isExistingSession) {
|
||||
await closeChromeMcpTab(profileName, tab.targetId);
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
const pw = await requirePwAi(res, `act:${kind}`);
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
await pw.closePageViaPlaywright({ cdpUrl, targetId: tab.targetId });
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
@@ -334,13 +648,23 @@ export function registerBrowserAgentActRoutes(
|
||||
return jsonError(res, 400, "url is required");
|
||||
}
|
||||
|
||||
await withPlaywrightRouteContext({
|
||||
await withRouteTabContext({
|
||||
req,
|
||||
res,
|
||||
ctx,
|
||||
targetId,
|
||||
feature: "response body",
|
||||
run: async ({ cdpUrl, tab, pw }) => {
|
||||
run: async ({ profileCtx, cdpUrl, tab }) => {
|
||||
if (profileCtx.profile.driver === "existing-session") {
|
||||
return jsonError(
|
||||
res,
|
||||
501,
|
||||
"response body is not supported for existing-session profiles yet.",
|
||||
);
|
||||
}
|
||||
const pw = await requirePwAi(res, "response body");
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
const result = await pw.responseBodyViaPlaywright({
|
||||
cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
@@ -361,13 +685,39 @@ export function registerBrowserAgentActRoutes(
|
||||
return jsonError(res, 400, "ref is required");
|
||||
}
|
||||
|
||||
await withPlaywrightRouteContext({
|
||||
await withRouteTabContext({
|
||||
req,
|
||||
res,
|
||||
ctx,
|
||||
targetId,
|
||||
feature: "highlight",
|
||||
run: async ({ cdpUrl, tab, pw }) => {
|
||||
run: async ({ profileCtx, cdpUrl, tab }) => {
|
||||
if (profileCtx.profile.driver === "existing-session") {
|
||||
await evaluateChromeMcpScript({
|
||||
profileName: profileCtx.profile.name,
|
||||
targetId: tab.targetId,
|
||||
args: [ref],
|
||||
fn: `(el) => {
|
||||
if (!(el instanceof Element)) {
|
||||
return false;
|
||||
}
|
||||
el.scrollIntoView({ block: "center", inline: "center" });
|
||||
const previousOutline = el.style.outline;
|
||||
const previousOffset = el.style.outlineOffset;
|
||||
el.style.outline = "3px solid #FF4500";
|
||||
el.style.outlineOffset = "2px";
|
||||
setTimeout(() => {
|
||||
el.style.outline = previousOutline;
|
||||
el.style.outlineOffset = previousOffset;
|
||||
}, 2000);
|
||||
return true;
|
||||
}`,
|
||||
});
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
const pw = await requirePwAi(res, "highlight");
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
await pw.highlightViaPlaywright({
|
||||
cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
|
||||
@@ -1,6 +1,20 @@
|
||||
import path from "node:path";
|
||||
import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js";
|
||||
import { captureScreenshot, snapshotAria } from "../cdp.js";
|
||||
import {
|
||||
evaluateChromeMcpScript,
|
||||
navigateChromeMcpPage,
|
||||
takeChromeMcpScreenshot,
|
||||
takeChromeMcpSnapshot,
|
||||
} from "../chrome-mcp.js";
|
||||
import {
|
||||
buildAiSnapshotFromChromeMcpSnapshot,
|
||||
flattenChromeMcpSnapshotToAriaNodes,
|
||||
} from "../chrome-mcp.snapshot.js";
|
||||
import {
|
||||
assertBrowserNavigationAllowed,
|
||||
assertBrowserNavigationResultAllowed,
|
||||
} from "../navigation-guard.js";
|
||||
import { withBrowserNavigationPolicy } from "../navigation-guard.js";
|
||||
import {
|
||||
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
@@ -25,6 +39,89 @@ import {
|
||||
import type { BrowserResponse, BrowserRouteRegistrar } from "./types.js";
|
||||
import { jsonError, toBoolean, toStringOrEmpty } from "./utils.js";
|
||||
|
||||
const CHROME_MCP_OVERLAY_ATTR = "data-openclaw-mcp-overlay";
|
||||
|
||||
async function clearChromeMcpOverlay(params: {
|
||||
profileName: string;
|
||||
targetId: string;
|
||||
}): Promise<void> {
|
||||
await evaluateChromeMcpScript({
|
||||
profileName: params.profileName,
|
||||
targetId: params.targetId,
|
||||
fn: `() => {
|
||||
document.querySelectorAll("[${CHROME_MCP_OVERLAY_ATTR}]").forEach((node) => node.remove());
|
||||
return true;
|
||||
}`,
|
||||
}).catch(() => {});
|
||||
}
|
||||
|
||||
async function renderChromeMcpLabels(params: {
|
||||
profileName: string;
|
||||
targetId: string;
|
||||
refs: string[];
|
||||
}): Promise<{ labels: number; skipped: number }> {
|
||||
const refList = JSON.stringify(params.refs);
|
||||
const result = await evaluateChromeMcpScript({
|
||||
profileName: params.profileName,
|
||||
targetId: params.targetId,
|
||||
args: params.refs,
|
||||
fn: `(...elements) => {
|
||||
const refs = ${refList};
|
||||
document.querySelectorAll("[${CHROME_MCP_OVERLAY_ATTR}]").forEach((node) => node.remove());
|
||||
const root = document.createElement("div");
|
||||
root.setAttribute("${CHROME_MCP_OVERLAY_ATTR}", "labels");
|
||||
root.style.position = "fixed";
|
||||
root.style.inset = "0";
|
||||
root.style.pointerEvents = "none";
|
||||
root.style.zIndex = "2147483647";
|
||||
let labels = 0;
|
||||
let skipped = 0;
|
||||
elements.forEach((el, index) => {
|
||||
if (!(el instanceof Element)) {
|
||||
skipped += 1;
|
||||
return;
|
||||
}
|
||||
const rect = el.getBoundingClientRect();
|
||||
if (rect.width <= 0 && rect.height <= 0) {
|
||||
skipped += 1;
|
||||
return;
|
||||
}
|
||||
labels += 1;
|
||||
const badge = document.createElement("div");
|
||||
badge.setAttribute("${CHROME_MCP_OVERLAY_ATTR}", "label");
|
||||
badge.textContent = refs[index] || String(labels);
|
||||
badge.style.position = "fixed";
|
||||
badge.style.left = \`\${Math.max(0, rect.left)}px\`;
|
||||
badge.style.top = \`\${Math.max(0, rect.top)}px\`;
|
||||
badge.style.transform = "translateY(-100%)";
|
||||
badge.style.padding = "2px 6px";
|
||||
badge.style.borderRadius = "999px";
|
||||
badge.style.background = "#FF4500";
|
||||
badge.style.color = "#fff";
|
||||
badge.style.font = "600 12px ui-monospace, SFMono-Regular, Menlo, monospace";
|
||||
badge.style.boxShadow = "0 2px 6px rgba(0,0,0,0.35)";
|
||||
badge.style.whiteSpace = "nowrap";
|
||||
root.appendChild(badge);
|
||||
});
|
||||
document.documentElement.appendChild(root);
|
||||
return { labels, skipped };
|
||||
}`,
|
||||
});
|
||||
const labels =
|
||||
result &&
|
||||
typeof result === "object" &&
|
||||
typeof (result as { labels?: unknown }).labels === "number"
|
||||
? (result as { labels: number }).labels
|
||||
: 0;
|
||||
const skipped =
|
||||
result &&
|
||||
typeof result === "object" &&
|
||||
typeof (result as { skipped?: unknown }).skipped === "number"
|
||||
? (result as { skipped: number }).skipped
|
||||
: 0;
|
||||
return { labels, skipped };
|
||||
}
|
||||
|
||||
async function saveBrowserMediaResponse(params: {
|
||||
res: BrowserResponse;
|
||||
buffer: Buffer;
|
||||
@@ -96,13 +193,27 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||
if (!url) {
|
||||
return jsonError(res, 400, "url is required");
|
||||
}
|
||||
await withPlaywrightRouteContext({
|
||||
await withRouteTabContext({
|
||||
req,
|
||||
res,
|
||||
ctx,
|
||||
targetId,
|
||||
feature: "navigate",
|
||||
run: async ({ cdpUrl, tab, pw, profileCtx }) => {
|
||||
run: async ({ profileCtx, tab, cdpUrl }) => {
|
||||
if (profileCtx.profile.driver === "existing-session") {
|
||||
const ssrfPolicyOpts = withBrowserNavigationPolicy(ctx.state().resolved.ssrfPolicy);
|
||||
await assertBrowserNavigationAllowed({ url, ...ssrfPolicyOpts });
|
||||
const result = await navigateChromeMcpPage({
|
||||
profileName: profileCtx.profile.name,
|
||||
targetId: tab.targetId,
|
||||
url,
|
||||
});
|
||||
await assertBrowserNavigationResultAllowed({ url: result.url, ...ssrfPolicyOpts });
|
||||
return res.json({ ok: true, targetId: tab.targetId, ...result });
|
||||
}
|
||||
const pw = await requirePwAi(res, "navigate");
|
||||
if (!pw) {
|
||||
return;
|
||||
}
|
||||
const result = await pw.navigateViaPlaywright({
|
||||
cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
@@ -122,6 +233,17 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||
app.post("/pdf", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = toStringOrEmpty(body.targetId) || undefined;
|
||||
const profileCtx = resolveProfileContext(req, res, ctx);
|
||||
if (!profileCtx) {
|
||||
return;
|
||||
}
|
||||
if (profileCtx.profile.driver === "existing-session") {
|
||||
return jsonError(
|
||||
res,
|
||||
501,
|
||||
"pdf is not supported for existing-session profiles yet; use screenshot/snapshot instead.",
|
||||
);
|
||||
}
|
||||
await withPlaywrightRouteContext({
|
||||
req,
|
||||
res,
|
||||
@@ -163,6 +285,36 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||
ctx,
|
||||
targetId,
|
||||
run: async ({ profileCtx, tab, cdpUrl }) => {
|
||||
if (profileCtx.profile.driver === "existing-session") {
|
||||
if (element) {
|
||||
return jsonError(
|
||||
res,
|
||||
400,
|
||||
"element screenshots are not supported for existing-session profiles; use ref from snapshot.",
|
||||
);
|
||||
}
|
||||
const buffer = await takeChromeMcpScreenshot({
|
||||
profileName: profileCtx.profile.name,
|
||||
targetId: tab.targetId,
|
||||
uid: ref,
|
||||
fullPage,
|
||||
format: type,
|
||||
});
|
||||
const normalized = await normalizeBrowserScreenshot(buffer, {
|
||||
maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
|
||||
maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
});
|
||||
await saveBrowserMediaResponse({
|
||||
res,
|
||||
buffer: normalized.buffer,
|
||||
contentType: normalized.contentType ?? `image/${type}`,
|
||||
maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
targetId: tab.targetId,
|
||||
url: tab.url,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
let buffer: Buffer;
|
||||
const shouldUsePlaywright = shouldUsePlaywrightForScreenshot({
|
||||
profile: profileCtx.profile,
|
||||
@@ -227,6 +379,90 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||
if ((plan.labels || plan.mode === "efficient") && plan.format === "aria") {
|
||||
return jsonError(res, 400, "labels/mode=efficient require format=ai");
|
||||
}
|
||||
if (profileCtx.profile.driver === "existing-session") {
|
||||
if (plan.labels) {
|
||||
return jsonError(res, 501, "labels are not supported for existing-session profiles yet.");
|
||||
}
|
||||
if (plan.selectorValue || plan.frameSelectorValue) {
|
||||
return jsonError(
|
||||
res,
|
||||
400,
|
||||
"selector/frame snapshots are not supported for existing-session profiles; snapshot the whole page and use refs.",
|
||||
);
|
||||
}
|
||||
const snapshot = await takeChromeMcpSnapshot({
|
||||
profileName: profileCtx.profile.name,
|
||||
targetId: tab.targetId,
|
||||
});
|
||||
if (plan.format === "aria") {
|
||||
return res.json({
|
||||
ok: true,
|
||||
format: "aria",
|
||||
targetId: tab.targetId,
|
||||
url: tab.url,
|
||||
nodes: flattenChromeMcpSnapshotToAriaNodes(snapshot, plan.limit),
|
||||
});
|
||||
}
|
||||
const built = buildAiSnapshotFromChromeMcpSnapshot({
|
||||
root: snapshot,
|
||||
options: {
|
||||
interactive: plan.interactive ?? undefined,
|
||||
compact: plan.compact ?? undefined,
|
||||
maxDepth: plan.depth ?? undefined,
|
||||
},
|
||||
maxChars: plan.resolvedMaxChars,
|
||||
});
|
||||
if (plan.labels) {
|
||||
const refs = Object.keys(built.refs);
|
||||
const labelResult = await renderChromeMcpLabels({
|
||||
profileName: profileCtx.profile.name,
|
||||
targetId: tab.targetId,
|
||||
refs,
|
||||
});
|
||||
try {
|
||||
const labeled = await takeChromeMcpScreenshot({
|
||||
profileName: profileCtx.profile.name,
|
||||
targetId: tab.targetId,
|
||||
format: "png",
|
||||
});
|
||||
const normalized = await normalizeBrowserScreenshot(labeled, {
|
||||
maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
|
||||
maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
});
|
||||
await ensureMediaDir();
|
||||
const saved = await saveMediaBuffer(
|
||||
normalized.buffer,
|
||||
normalized.contentType ?? "image/png",
|
||||
"browser",
|
||||
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
);
|
||||
return res.json({
|
||||
ok: true,
|
||||
format: "ai",
|
||||
targetId: tab.targetId,
|
||||
url: tab.url,
|
||||
labels: true,
|
||||
labelsCount: labelResult.labels,
|
||||
labelsSkipped: labelResult.skipped,
|
||||
imagePath: path.resolve(saved.path),
|
||||
imageType: normalized.contentType?.includes("jpeg") ? "jpeg" : "png",
|
||||
...built,
|
||||
});
|
||||
} finally {
|
||||
await clearChromeMcpOverlay({
|
||||
profileName: profileCtx.profile.name,
|
||||
targetId: tab.targetId,
|
||||
});
|
||||
}
|
||||
}
|
||||
return res.json({
|
||||
ok: true,
|
||||
format: "ai",
|
||||
targetId: tab.targetId,
|
||||
url: tab.url,
|
||||
...built,
|
||||
});
|
||||
}
|
||||
if (plan.format === "ai") {
|
||||
const pw = await requirePwAi(res, "ai snapshot");
|
||||
if (!pw) {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { getChromeMcpPid } from "../chrome-mcp.js";
|
||||
import { resolveBrowserExecutableForPlatform } from "../chrome.executables.js";
|
||||
import { toBrowserErrorResponse } from "../errors.js";
|
||||
import { createBrowserProfilesService } from "../profiles-service.js";
|
||||
@@ -76,10 +77,14 @@ export function registerBrowserBasicRoutes(app: BrowserRouteRegistrar, ctx: Brow
|
||||
res.json({
|
||||
enabled: current.resolved.enabled,
|
||||
profile: profileCtx.profile.name,
|
||||
driver: profileCtx.profile.driver,
|
||||
running: cdpReady,
|
||||
cdpReady,
|
||||
cdpHttp,
|
||||
pid: profileState?.running?.pid ?? null,
|
||||
pid:
|
||||
profileCtx.profile.driver === "existing-session"
|
||||
? getChromeMcpPid(profileCtx.profile.name)
|
||||
: (profileState?.running?.pid ?? null),
|
||||
cdpPort: profileCtx.profile.cdpPort,
|
||||
cdpUrl: profileCtx.profile.cdpUrl,
|
||||
chosenBrowser: profileState?.running?.exe.kind ?? null,
|
||||
@@ -146,6 +151,7 @@ export function registerBrowserBasicRoutes(app: BrowserRouteRegistrar, ctx: Brow
|
||||
const driver = toStringOrEmpty((req.body as { driver?: unknown })?.driver) as
|
||||
| "openclaw"
|
||||
| "extension"
|
||||
| "existing-session"
|
||||
| "";
|
||||
|
||||
if (!name) {
|
||||
@@ -158,7 +164,12 @@ export function registerBrowserBasicRoutes(app: BrowserRouteRegistrar, ctx: Brow
|
||||
name,
|
||||
color: color || undefined,
|
||||
cdpUrl: cdpUrl || undefined,
|
||||
driver: driver === "extension" ? "extension" : undefined,
|
||||
driver:
|
||||
driver === "extension"
|
||||
? "extension"
|
||||
: driver === "existing-session"
|
||||
? "existing-session"
|
||||
: undefined,
|
||||
});
|
||||
res.json(result);
|
||||
} catch (err) {
|
||||
|
||||
@@ -3,6 +3,11 @@ import {
|
||||
PROFILE_POST_RESTART_WS_TIMEOUT_MS,
|
||||
resolveCdpReachabilityTimeouts,
|
||||
} from "./cdp-timeouts.js";
|
||||
import {
|
||||
closeChromeMcpSession,
|
||||
ensureChromeMcpAvailable,
|
||||
listChromeMcpTabs,
|
||||
} from "./chrome-mcp.js";
|
||||
import {
|
||||
isChromeCdpReady,
|
||||
isChromeReachable,
|
||||
@@ -60,11 +65,19 @@ export function createProfileAvailability({
|
||||
});
|
||||
|
||||
const isReachable = async (timeoutMs?: number) => {
|
||||
if (profile.driver === "existing-session") {
|
||||
await ensureChromeMcpAvailable(profile.name);
|
||||
await listChromeMcpTabs(profile.name);
|
||||
return true;
|
||||
}
|
||||
const { httpTimeoutMs, wsTimeoutMs } = resolveTimeouts(timeoutMs);
|
||||
return await isChromeCdpReady(profile.cdpUrl, httpTimeoutMs, wsTimeoutMs);
|
||||
};
|
||||
|
||||
const isHttpReachable = async (timeoutMs?: number) => {
|
||||
if (profile.driver === "existing-session") {
|
||||
return await isReachable(timeoutMs);
|
||||
}
|
||||
const { httpTimeoutMs } = resolveTimeouts(timeoutMs);
|
||||
return await isChromeReachable(profile.cdpUrl, httpTimeoutMs);
|
||||
};
|
||||
@@ -109,6 +122,9 @@ export function createProfileAvailability({
|
||||
if (previousProfile.driver === "extension") {
|
||||
await stopChromeExtensionRelayServer({ cdpUrl: previousProfile.cdpUrl }).catch(() => false);
|
||||
}
|
||||
if (previousProfile.driver === "existing-session") {
|
||||
await closeChromeMcpSession(previousProfile.name).catch(() => false);
|
||||
}
|
||||
await closePlaywrightBrowserConnectionForProfile(previousProfile.cdpUrl);
|
||||
if (previousProfile.cdpUrl !== profile.cdpUrl) {
|
||||
await closePlaywrightBrowserConnectionForProfile(profile.cdpUrl);
|
||||
@@ -138,6 +154,10 @@ export function createProfileAvailability({
|
||||
|
||||
const ensureBrowserAvailable = async (): Promise<void> => {
|
||||
await reconcileProfileRuntime();
|
||||
if (profile.driver === "existing-session") {
|
||||
await ensureChromeMcpAvailable(profile.name);
|
||||
return;
|
||||
}
|
||||
const current = state();
|
||||
const remoteCdp = capabilities.isRemote;
|
||||
const attachOnly = profile.attachOnly;
|
||||
@@ -238,6 +258,10 @@ export function createProfileAvailability({
|
||||
|
||||
const stopRunningBrowser = async (): Promise<{ stopped: boolean }> => {
|
||||
await reconcileProfileRuntime();
|
||||
if (profile.driver === "existing-session") {
|
||||
const stopped = await closeChromeMcpSession(profile.name);
|
||||
return { stopped };
|
||||
}
|
||||
if (capabilities.requiresRelay) {
|
||||
const stopped = await stopChromeExtensionRelayServer({
|
||||
cdpUrl: profile.cdpUrl,
|
||||
|
||||
102
src/browser/server-context.existing-session.test.ts
Normal file
102
src/browser/server-context.existing-session.test.ts
Normal file
@@ -0,0 +1,102 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { createBrowserRouteContext } from "./server-context.js";
|
||||
import type { BrowserServerState } from "./server-context.js";
|
||||
|
||||
vi.mock("./chrome-mcp.js", () => ({
|
||||
closeChromeMcpSession: vi.fn(async () => true),
|
||||
ensureChromeMcpAvailable: vi.fn(async () => {}),
|
||||
focusChromeMcpTab: vi.fn(async () => {}),
|
||||
listChromeMcpTabs: vi.fn(async () => [
|
||||
{ targetId: "7", title: "", url: "https://example.com", type: "page" },
|
||||
]),
|
||||
openChromeMcpTab: vi.fn(async () => ({
|
||||
targetId: "8",
|
||||
title: "",
|
||||
url: "https://openclaw.ai",
|
||||
type: "page",
|
||||
})),
|
||||
closeChromeMcpTab: vi.fn(async () => {}),
|
||||
getChromeMcpPid: vi.fn(() => 4321),
|
||||
}));
|
||||
|
||||
import * as chromeMcp from "./chrome-mcp.js";
|
||||
|
||||
function makeState(): BrowserServerState {
|
||||
return {
|
||||
server: null,
|
||||
port: 0,
|
||||
resolved: {
|
||||
enabled: true,
|
||||
evaluateEnabled: true,
|
||||
controlPort: 18791,
|
||||
cdpPortRangeStart: 18800,
|
||||
cdpPortRangeEnd: 18899,
|
||||
cdpProtocol: "http",
|
||||
cdpHost: "127.0.0.1",
|
||||
cdpIsLoopback: true,
|
||||
remoteCdpTimeoutMs: 1500,
|
||||
remoteCdpHandshakeTimeoutMs: 3000,
|
||||
color: "#FF4500",
|
||||
headless: false,
|
||||
noSandbox: false,
|
||||
attachOnly: false,
|
||||
defaultProfile: "chrome-live",
|
||||
profiles: {
|
||||
"chrome-live": {
|
||||
cdpPort: 18801,
|
||||
color: "#0066CC",
|
||||
driver: "existing-session",
|
||||
attachOnly: true,
|
||||
},
|
||||
},
|
||||
extraArgs: [],
|
||||
ssrfPolicy: { dangerouslyAllowPrivateNetwork: true },
|
||||
},
|
||||
profiles: new Map(),
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("browser server-context existing-session profile", () => {
|
||||
it("routes tab operations through the Chrome MCP backend", async () => {
|
||||
const state = makeState();
|
||||
const ctx = createBrowserRouteContext({ getState: () => state });
|
||||
const live = ctx.forProfile("chrome-live");
|
||||
|
||||
vi.mocked(chromeMcp.listChromeMcpTabs)
|
||||
.mockResolvedValueOnce([
|
||||
{ targetId: "7", title: "", url: "https://example.com", type: "page" },
|
||||
])
|
||||
.mockResolvedValueOnce([
|
||||
{ targetId: "8", title: "", url: "https://openclaw.ai", type: "page" },
|
||||
])
|
||||
.mockResolvedValueOnce([
|
||||
{ targetId: "8", title: "", url: "https://openclaw.ai", type: "page" },
|
||||
])
|
||||
.mockResolvedValueOnce([
|
||||
{ targetId: "7", title: "", url: "https://example.com", type: "page" },
|
||||
]);
|
||||
|
||||
await live.ensureBrowserAvailable();
|
||||
const tabs = await live.listTabs();
|
||||
expect(tabs.map((tab) => tab.targetId)).toEqual(["7"]);
|
||||
|
||||
const opened = await live.openTab("https://openclaw.ai");
|
||||
expect(opened.targetId).toBe("8");
|
||||
|
||||
const selected = await live.ensureTabAvailable();
|
||||
expect(selected.targetId).toBe("8");
|
||||
|
||||
await live.focusTab("7");
|
||||
await live.stopRunningBrowser();
|
||||
|
||||
expect(chromeMcp.ensureChromeMcpAvailable).toHaveBeenCalledWith("chrome-live");
|
||||
expect(chromeMcp.listChromeMcpTabs).toHaveBeenCalledWith("chrome-live");
|
||||
expect(chromeMcp.openChromeMcpTab).toHaveBeenCalledWith("chrome-live", "https://openclaw.ai");
|
||||
expect(chromeMcp.focusChromeMcpTab).toHaveBeenCalledWith("chrome-live", "7");
|
||||
expect(chromeMcp.closeChromeMcpSession).toHaveBeenCalledWith("chrome-live");
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { fetchOk, normalizeCdpHttpBaseForJsonEndpoints } from "./cdp.helpers.js";
|
||||
import { appendCdpPath } from "./cdp.js";
|
||||
import { closeChromeMcpTab, focusChromeMcpTab } from "./chrome-mcp.js";
|
||||
import type { ResolvedBrowserProfile } from "./config.js";
|
||||
import { BrowserTabNotFoundError, BrowserTargetAmbiguousError } from "./errors.js";
|
||||
import { getBrowserProfileCapabilities } from "./profile-capabilities.js";
|
||||
@@ -111,6 +112,13 @@ export function createProfileSelectionOps({
|
||||
const focusTab = async (targetId: string): Promise<void> => {
|
||||
const resolvedTargetId = await resolveTargetIdOrThrow(targetId);
|
||||
|
||||
if (profile.driver === "existing-session") {
|
||||
await focusChromeMcpTab(profile.name, resolvedTargetId);
|
||||
const profileState = getProfileState();
|
||||
profileState.lastTargetId = resolvedTargetId;
|
||||
return;
|
||||
}
|
||||
|
||||
if (capabilities.usesPersistentPlaywright) {
|
||||
const mod = await getPwAiModule({ mode: "strict" });
|
||||
const focusPageByTargetIdViaPlaywright = (mod as Partial<PwAiModule> | null)
|
||||
@@ -134,6 +142,11 @@ export function createProfileSelectionOps({
|
||||
const closeTab = async (targetId: string): Promise<void> => {
|
||||
const resolvedTargetId = await resolveTargetIdOrThrow(targetId);
|
||||
|
||||
if (profile.driver === "existing-session") {
|
||||
await closeChromeMcpTab(profile.name, resolvedTargetId);
|
||||
return;
|
||||
}
|
||||
|
||||
// For remote profiles, use Playwright's persistent connection to close tabs
|
||||
if (capabilities.usesPersistentPlaywright) {
|
||||
const mod = await getPwAiModule({ mode: "strict" });
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { CDP_JSON_NEW_TIMEOUT_MS } from "./cdp-timeouts.js";
|
||||
import { fetchJson, fetchOk, normalizeCdpHttpBaseForJsonEndpoints } from "./cdp.helpers.js";
|
||||
import { appendCdpPath, createTargetViaCdp, normalizeCdpWsUrl } from "./cdp.js";
|
||||
import { listChromeMcpTabs, openChromeMcpTab } from "./chrome-mcp.js";
|
||||
import type { ResolvedBrowserProfile } from "./config.js";
|
||||
import {
|
||||
assertBrowserNavigationAllowed,
|
||||
@@ -65,6 +66,10 @@ export function createProfileTabOps({
|
||||
const capabilities = getBrowserProfileCapabilities(profile);
|
||||
|
||||
const listTabs = async (): Promise<BrowserTab[]> => {
|
||||
if (profile.driver === "existing-session") {
|
||||
return await listChromeMcpTabs(profile.name);
|
||||
}
|
||||
|
||||
if (capabilities.usesPersistentPlaywright) {
|
||||
const mod = await getPwAiModule({ mode: "strict" });
|
||||
const listPagesViaPlaywright = (mod as Partial<PwAiModule> | null)?.listPagesViaPlaywright;
|
||||
@@ -134,6 +139,15 @@ export function createProfileTabOps({
|
||||
const openTab = async (url: string): Promise<BrowserTab> => {
|
||||
const ssrfPolicyOpts = withBrowserNavigationPolicy(state().resolved.ssrfPolicy);
|
||||
|
||||
if (profile.driver === "existing-session") {
|
||||
await assertBrowserNavigationAllowed({ url, ...ssrfPolicyOpts });
|
||||
const page = await openChromeMcpTab(profile.name, url);
|
||||
const profileState = getProfileState();
|
||||
profileState.lastTargetId = page.targetId;
|
||||
await assertBrowserNavigationResultAllowed({ url: page.url, ...ssrfPolicyOpts });
|
||||
return page;
|
||||
}
|
||||
|
||||
if (capabilities.usesPersistentPlaywright) {
|
||||
const mod = await getPwAiModule({ mode: "strict" });
|
||||
const createPageViaPlaywright = (mod as Partial<PwAiModule> | null)?.createPageViaPlaywright;
|
||||
|
||||
@@ -162,12 +162,22 @@ export function createBrowserRouteContext(opts: ContextOptions): BrowserRouteCon
|
||||
|
||||
let tabCount = 0;
|
||||
let running = false;
|
||||
const profileCtx = createProfileContext(opts, profile);
|
||||
|
||||
if (profileState?.running) {
|
||||
if (profile.driver === "existing-session") {
|
||||
try {
|
||||
running = await profileCtx.isReachable(300);
|
||||
if (running) {
|
||||
const tabs = await profileCtx.listTabs();
|
||||
tabCount = tabs.filter((t) => t.type === "page").length;
|
||||
}
|
||||
} catch {
|
||||
// Chrome MCP not available
|
||||
}
|
||||
} else if (profileState?.running) {
|
||||
running = true;
|
||||
try {
|
||||
const ctx = createProfileContext(opts, profile);
|
||||
const tabs = await ctx.listTabs();
|
||||
const tabs = await profileCtx.listTabs();
|
||||
tabCount = tabs.filter((t) => t.type === "page").length;
|
||||
} catch {
|
||||
// Browser might not be responsive
|
||||
@@ -178,8 +188,7 @@ export function createBrowserRouteContext(opts: ContextOptions): BrowserRouteCon
|
||||
const reachable = await isChromeReachable(profile.cdpUrl, 200);
|
||||
if (reachable) {
|
||||
running = true;
|
||||
const ctx = createProfileContext(opts, profile);
|
||||
const tabs = await ctx.listTabs().catch(() => []);
|
||||
const tabs = await profileCtx.listTabs().catch(() => []);
|
||||
tabCount = tabs.filter((t) => t.type === "page").length;
|
||||
}
|
||||
} catch {
|
||||
@@ -192,6 +201,7 @@ export function createBrowserRouteContext(opts: ContextOptions): BrowserRouteCon
|
||||
cdpPort: profile.cdpPort,
|
||||
cdpUrl: profile.cdpUrl,
|
||||
color: profile.color,
|
||||
driver: profile.driver,
|
||||
running,
|
||||
tabCount,
|
||||
isDefault: name === current.resolved.defaultProfile,
|
||||
|
||||
@@ -56,6 +56,7 @@ export type ProfileStatus = {
|
||||
cdpPort: number;
|
||||
cdpUrl: string;
|
||||
color: string;
|
||||
driver: ResolvedBrowserProfile["driver"];
|
||||
running: boolean;
|
||||
tabCount: number;
|
||||
isDefault: boolean;
|
||||
|
||||
@@ -407,7 +407,8 @@ export function registerBrowserManageCommands(
|
||||
const def = p.isDefault ? " [default]" : "";
|
||||
const loc = p.isRemote ? `cdpUrl: ${p.cdpUrl}` : `port: ${p.cdpPort}`;
|
||||
const remote = p.isRemote ? " [remote]" : "";
|
||||
return `${p.name}: ${status}${tabs}${def}${remote}\n ${loc}, color: ${p.color}`;
|
||||
const driver = p.driver !== "openclaw" ? ` [${p.driver}]` : "";
|
||||
return `${p.name}: ${status}${tabs}${def}${remote}${driver}\n ${loc}, color: ${p.color}`;
|
||||
})
|
||||
.join("\n"),
|
||||
);
|
||||
@@ -420,7 +421,10 @@ export function registerBrowserManageCommands(
|
||||
.requiredOption("--name <name>", "Profile name (lowercase, numbers, hyphens)")
|
||||
.option("--color <hex>", "Profile color (hex format, e.g. #0066CC)")
|
||||
.option("--cdp-url <url>", "CDP URL for remote Chrome (http/https)")
|
||||
.option("--driver <driver>", "Profile driver (openclaw|extension). Default: openclaw")
|
||||
.option(
|
||||
"--driver <driver>",
|
||||
"Profile driver (openclaw|extension|existing-session). Default: openclaw",
|
||||
)
|
||||
.action(
|
||||
async (opts: { name: string; color?: string; cdpUrl?: string; driver?: string }, cmd) => {
|
||||
const parent = parentOpts(cmd);
|
||||
@@ -434,7 +438,12 @@ export function registerBrowserManageCommands(
|
||||
name: opts.name,
|
||||
color: opts.color,
|
||||
cdpUrl: opts.cdpUrl,
|
||||
driver: opts.driver === "extension" ? "extension" : undefined,
|
||||
driver:
|
||||
opts.driver === "extension"
|
||||
? "extension"
|
||||
: opts.driver === "existing-session"
|
||||
? "existing-session"
|
||||
: undefined,
|
||||
},
|
||||
},
|
||||
{ timeoutMs: 10_000 },
|
||||
@@ -446,7 +455,11 @@ export function registerBrowserManageCommands(
|
||||
defaultRuntime.log(
|
||||
info(
|
||||
`🦞 Created profile "${result.profile}"\n${loc}\n color: ${result.color}${
|
||||
opts.driver === "extension" ? "\n driver: extension" : ""
|
||||
opts.driver === "extension"
|
||||
? "\n driver: extension"
|
||||
: opts.driver === "existing-session"
|
||||
? "\n driver: existing-session"
|
||||
: ""
|
||||
}`,
|
||||
),
|
||||
);
|
||||
|
||||
@@ -4,7 +4,7 @@ export type BrowserProfileConfig = {
|
||||
/** CDP URL for this profile (use for remote Chrome). */
|
||||
cdpUrl?: string;
|
||||
/** Profile driver (default: openclaw). */
|
||||
driver?: "openclaw" | "clawd" | "extension";
|
||||
driver?: "openclaw" | "clawd" | "extension" | "existing-session";
|
||||
/** If true, never launch a browser for this profile; only attach. Falls back to browser.attachOnly. */
|
||||
attachOnly?: boolean;
|
||||
/** Profile color (hex). Auto-assigned at creation. */
|
||||
|
||||
@@ -360,7 +360,12 @@ export const OpenClawSchema = z
|
||||
cdpPort: z.number().int().min(1).max(65535).optional(),
|
||||
cdpUrl: z.string().optional(),
|
||||
driver: z
|
||||
.union([z.literal("openclaw"), z.literal("clawd"), z.literal("extension")])
|
||||
.union([
|
||||
z.literal("openclaw"),
|
||||
z.literal("clawd"),
|
||||
z.literal("extension"),
|
||||
z.literal("existing-session"),
|
||||
])
|
||||
.optional(),
|
||||
attachOnly: z.boolean().optional(),
|
||||
color: HexColorSchema,
|
||||
|
||||
Reference in New Issue
Block a user