Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

feat(server): add local sandbox browser runtime (#32)

Leofc45f9828d88
diff --git a/apps/server/package.json b/apps/server/package.json
index e0e240169..052b5ff60 100644
--- a/apps/server/package.json
+++ b/apps/server/package.json
@@ -40,7 +40,8 @@
"msgpackr-extract": "3.0.4",
"node-pty": "^1.1.0",
"pngjs": "7.0.0",
- "yaml": "catalog:"
+ "yaml": "catalog:",
+ "zod": "^4.4.3"
},
"devDependencies": {
"@effect/vitest": "catalog:",
diff --git a/apps/server/src/provider/botBrowser.integration.test.ts b/apps/server/src/provider/botBrowser.integration.test.ts
new file mode 100644
index 000000000..68b8faac8
--- /dev/null
+++ b/apps/server/src/provider/botBrowser.integration.test.ts
@@ -0,0 +1,73 @@
+// @effect-diagnostics nodeBuiltinImport:off
+import * as NodeHttp from "node:http";
+import * as NodeOS from "node:os";
+import * as NodePath from "node:path";
+
+import { LocalFilesystem, LocalSandbox, Workspace } from "@mastra/core/workspace";
+import { describe, expect, it } from "vite-plus/test";
+
+import { createBotBrowser } from "./botBrowser.ts";
+
+async function executeTool(
+ tool: unknown,
+ input: Readonly<Record<string, unknown>>,
+): Promise<unknown> {
+ const execute = (tool as { execute?: (input: Readonly<Record<string, unknown>>) => unknown })
+ .execute;
+ if (!execute) throw new Error("expected executable tool");
+ return execute(input);
+}
+
+describe.runIf(process.env.T3_BOT_BROWSER_INTEGRATION === "1")(
+ "sandbox bot browser integration",
+ () => {
+ it("clicks a target on the bottom edge and keeps the same browser task", async () => {
+ const server = NodeHttp.createServer((request, response) => {
+ response.setHeader("content-type", "text/html; charset=utf-8");
+ if (request.url === "/clicked") {
+ response.end("<!doctype html><title>Clicked</title><main>Bottom edge clicked</main>");
+ return;
+ }
+ response.end(`<!doctype html>
+ <title>Bottom edge target</title>
+ <style>html,body{margin:0;height:100%}main{height:100vh;position:relative}button{position:absolute;left:0;bottom:0;height:40px}</style>
+ <main><button id="bottom-target" onclick="location.href='/clicked'">Click bottom edge</button></main>`);
+ });
+ await new Promise<void>((resolve, reject) => {
+ server.once("error", reject);
+ server.listen(0, "127.0.0.1", resolve);
+ });
+ const address = server.address();
+ if (!address || typeof address === "string") throw new Error("test server has no port");
+
+ const workspace = new Workspace({
+ filesystem: new LocalFilesystem({ basePath: process.cwd() }),
+ sandbox: new LocalSandbox({ workingDirectory: process.cwd() }),
+ });
+ await workspace.init();
+ const browser = createBotBrowser({
+ threadId: "bottom-edge-integration",
+ workspace,
+ cacheDir: NodePath.join(NodeOS.tmpdir(), "akeru-lightpanda-cache"),
+ });
+
+ try {
+ await executeTool(browser.tools.browser_navigate, {
+ url: `http://127.0.0.1:${address.port}/`,
+ });
+ await executeTool(browser.tools.browser_click, { selector: "#bottom-target" });
+ await browser.reconnect();
+ const snapshot = await executeTool(browser.tools.browser_snapshot, {});
+ expect(snapshot).toMatchObject({
+ snapshot: expect.stringContaining("Bottom edge clicked"),
+ });
+ } finally {
+ await browser.close();
+ await workspace.destroy();
+ await new Promise<void>((resolve, reject) =>
+ server.close((error) => (error ? reject(error) : resolve())),
+ );
+ }
+ }, 360_000);
+ },
+);
diff --git a/apps/server/src/provider/botBrowser.test.ts b/apps/server/src/provider/botBrowser.test.ts
new file mode 100644
index 000000000..148179997
--- /dev/null
+++ b/apps/server/src/provider/botBrowser.test.ts
@@ -0,0 +1,133 @@
+import { LocalFilesystem, LocalSandbox, Workspace } from "@mastra/core/workspace";
+import { describe, expect, it, vi } from "vite-plus/test";
+
+import {
+ createBotBrowser,
+ createBotBrowserTools,
+ lightpandaMcpCommand,
+ type BotBrowserRpc,
+} from "./botBrowser.ts";
+
+function rpc() {
+ const call = vi.fn(async (name: string) =>
+ name === "tree" ? "semantic tree" : `${name} complete`,
+ );
+ const attachment = vi.fn(async () => ({
+ browserUrl: "http://127.0.0.1:9222",
+ mcpSessionId: "browser-session-1",
+ requestHeaders: {},
+ localRequestHeaders: {},
+ availableToHostedPlugins: false,
+ }));
+ const close = vi.fn(async () => undefined);
+ const reconnect = vi.fn(async () => undefined);
+ return { call, attachment, reconnect, close } satisfies BotBrowserRpc;
+}
+
+async function executeTool(
+ tool: unknown,
+ input: Readonly<Record<string, unknown>>,
+): Promise<unknown> {
+ const execute = (tool as { execute?: (input: Readonly<Record<string, unknown>>) => unknown })
+ .execute;
+ if (!execute) throw new Error("expected executable tool");
+ return execute(input);
+}
+
+describe("sandbox bot browser", () => {
+ it("exposes only the Akeru navigate, snapshot, click, and type tools", () => {
+ expect(Object.keys(createBotBrowserTools(rpc()))).toEqual([
+ "browser_navigate",
+ "browser_snapshot",
+ "browser_click",
+ "browser_type",
+ ]);
+ });
+
+ it("binds the HTTP MCP browser to loopback without stdin or CDP", () => {
+ expect(lightpandaMcpCommand("/tmp/lightpanda", 9223, "127.0.0.1")).toBe(
+ "'/tmp/lightpanda' mcp --host '127.0.0.1' --port 9223",
+ );
+ });
+
+ it("routes all four tools through one browser RPC session", async () => {
+ const browserRpc = rpc();
+ const tools = createBotBrowserTools(browserRpc);
+
+ await executeTool(tools.browser_navigate, { url: "https://example.com" });
+ await executeTool(tools.browser_snapshot, {});
+ await executeTool(tools.browser_click, { backendNodeId: 42 });
+ await executeTool(tools.browser_type, { selector: "#email", text: "[email protected]" });
+
+ expect(browserRpc.call.mock.calls).toEqual([
+ ["goto", { url: "https://example.com" }],
+ ["tree", {}],
+ ["click", { backendNodeId: 42 }],
+ ["fill", { selector: "#email", value: "[email protected]" }],
+ ]);
+ });
+
+ it("redacts sensitive text from semantic browser snapshots", async () => {
+ const browserRpc = rpc();
+ browserRpc.call.mockResolvedValueOnce(
+ "Email [email protected] token=secret-value-1234 path=/root/.ssh/id_rsa",
+ );
+ const result = await executeTool(createBotBrowserTools(browserRpc).browser_snapshot, {});
+
+ expect(result).toEqual({
+ snapshot: "Email [REDACTED] [REDACTED]",
+ truncated: false,
+ });
+ });
+
+ it("redacts sensitive text from browser action results", async () => {
+ const browserRpc = rpc();
+ browserRpc.call.mockResolvedValue(
+ "Email [email protected] token=secret-value-1234 path=/root/.ssh/id_rsa",
+ );
+ const tools = createBotBrowserTools(browserRpc);
+
+ for (const [tool, input] of [
+ [tools.browser_navigate, { url: "https://example.com" }],
+ [tools.browser_click, { selector: "button" }],
+ [tools.browser_type, { selector: "#email", text: "hello" }],
+ ] as const) {
+ await expect(executeTool(tool, input)).resolves.toEqual({
+ result: "Email [REDACTED] [REDACTED]",
+ });
+ }
+ });
+
+ it("uses the same browser session for tools, MCP attachment, and cleanup", async () => {
+ const browserRpc = rpc();
+ const workspace = new Workspace({
+ filesystem: new LocalFilesystem({ basePath: process.cwd() }),
+ sandbox: new LocalSandbox({ workingDirectory: process.cwd() }),
+ });
+ const makeRpc = vi.fn(() => browserRpc);
+ const browser = createBotBrowser({
+ threadId: "thread-1",
+ workspace,
+ cacheDir: "/tmp/akeru-browser-test",
+ makeRpc,
+ });
+
+ await executeTool(browser.tools.browser_click, { selector: "button" });
+ await expect(browser.attachment()).resolves.toEqual({
+ browserUrl: "http://127.0.0.1:9222",
+ mcpSessionId: "browser-session-1",
+ requestHeaders: {},
+ localRequestHeaders: {},
+ availableToHostedPlugins: false,
+ });
+ await browser.close();
+
+ expect(makeRpc).toHaveBeenCalledOnce();
+ expect(browserRpc.call).toHaveBeenCalledWith("click", { selector: "button" });
+ expect(browserRpc.attachment).toHaveBeenCalledOnce();
+ await browser.reconnect().catch(() => undefined);
+ expect(browserRpc.reconnect).toHaveBeenCalledOnce();
+ expect(browserRpc.close).toHaveBeenCalledOnce();
+ await workspace.destroy();
+ });
+});
diff --git a/apps/server/src/provider/botBrowser.ts b/apps/server/src/provider/botBrowser.ts
new file mode 100644
index 000000000..40df4cb43
--- /dev/null
+++ b/apps/server/src/provider/botBrowser.ts
@@ -0,0 +1,506 @@
+// @effect-diagnostics nodeBuiltinImport:off
+import * as NodeNet from "node:net";
+import * as NodePath from "node:path";
+import * as NodeTimersPromises from "node:timers/promises";
+
+import type { ToolsInput } from "@mastra/core/agent";
+import { createTool } from "@mastra/core/tools";
+import type { ProcessHandle, Workspace, WorkspaceSandbox } from "@mastra/core/workspace";
+import { z } from "zod";
+
+import { redactSensitiveText } from "../mcp/SensitiveDataRedaction.ts";
+
+const LIGHTPANDA_VERSION = "0.3.7";
+const MAX_SNAPSHOT_LENGTH = 50 * 1_024;
+const MCP_PROTOCOL_VERSION = "2024-11-05";
+const BROWSER_REQUEST_TIMEOUT_MS = 30_000;
+
+interface LightpandaRelease {
+ readonly url: string;
+ readonly sha256: string;
+}
+
+const LIGHTPANDA_RELEASES = {
+ "darwin-arm64": {
+ url: `https://github.com/lightpanda-io/browser/releases/download/${LIGHTPANDA_VERSION}/lightpanda-aarch64-macos`,
+ sha256: "ae99542d81af23087296ec037abb0d57a57002502f5ff4c1b0b05dfa484b79b8",
+ },
+ "darwin-x64": {
+ url: `https://github.com/lightpanda-io/browser/releases/download/${LIGHTPANDA_VERSION}/lightpanda-x86_64-macos`,
+ sha256: "5e118b6e91c2cccb1ce7f0d34fc39dab262b947e4dea29a90b1a75b9399d7862",
+ },
+ "linux-arm64": {
+ url: `https://github.com/lightpanda-io/browser/releases/download/${LIGHTPANDA_VERSION}/lightpanda-aarch64-linux`,
+ sha256: "4c0ecb28b4fcfb6d5bce82ec86e15fc6cde89cea168cf3840494f0ee26755852",
+ },
+ "linux-x64": {
+ url: `https://github.com/lightpanda-io/browser/releases/download/${LIGHTPANDA_VERSION}/lightpanda-x86_64-linux`,
+ sha256: "895339b02205171a181dde743ae0068bb4564884076feac8482baca9c212aa5a",
+ },
+} as const satisfies Readonly<Record<string, LightpandaRelease>>;
+
+type LightpandaPlatform = keyof typeof LIGHTPANDA_RELEASES;
+
+export interface BotBrowserAttachment {
+ readonly browserUrl: string;
+ readonly mcpSessionId: string;
+ readonly requestHeaders: Readonly<Record<string, string>>;
+ readonly localRequestHeaders: Readonly<Record<string, string>>;
+ readonly availableToHostedPlugins: boolean;
+}
+
+export interface BotBrowser {
+ readonly tools: ToolsInput;
+ readonly attachment: () => Promise<BotBrowserAttachment | undefined>;
+ readonly reconnect: () => Promise<void>;
+ readonly close: () => Promise<void>;
+}
+
+export interface BotBrowserRpc {
+ readonly call: (name: string, arguments_: Readonly<Record<string, unknown>>) => Promise<string>;
+ readonly attachment: () => Promise<BotBrowserAttachment | undefined>;
+ readonly reconnect: () => Promise<void>;
+ readonly close: () => Promise<void>;
+}
+
+export interface CreateBotBrowserInput {
+ readonly threadId: string;
+ readonly workspace: Workspace;
+ readonly cacheDir: string;
+ readonly makeRpc?: (input: BotBrowserProcessInput) => BotBrowserRpc;
+}
+
+export interface BotBrowserProcessInput {
+ readonly threadId: string;
+ readonly workspace: Workspace;
+ readonly cacheDir: string;
+}
+
+interface JsonRpcResponse {
+ readonly id?: number;
+ readonly result?: unknown;
+ readonly error?: { readonly code?: number; readonly message?: string };
+}
+
+export interface BrowserHttpResponse {
+ readonly status: number;
+ readonly body: string;
+ readonly sessionId?: string;
+}
+
+export interface BrowserRequest {
+ readonly method: "POST" | "DELETE";
+ readonly body?: string;
+ readonly sessionId?: string;
+}
+
+type BrowserRequestTransport = (request: BrowserRequest) => Promise<BrowserHttpResponse>;
+
+function platformFromUname(system: string, machine: string): LightpandaPlatform {
+ const os = system.trim().toLowerCase();
+ const arch = machine.trim().toLowerCase();
+ const normalizedOs = os === "darwin" ? "darwin" : os === "linux" ? "linux" : null;
+ const normalizedArch =
+ arch === "arm64" || arch === "aarch64"
+ ? "arm64"
+ : arch === "x86_64" || arch === "amd64"
+ ? "x64"
+ : null;
+ const key = normalizedOs && normalizedArch ? `${normalizedOs}-${normalizedArch}` : null;
+ if (!key || !(key in LIGHTPANDA_RELEASES)) {
+ throw new Error(`The sandbox browser does not support ${system.trim()} ${machine.trim()}.`);
+ }
+ return key as LightpandaPlatform;
+}
+
+async function execute(
+ sandbox: WorkspaceSandbox,
+ command: string,
+ args: string[],
+ timeout = 30_000,
+): Promise<string> {
+ if (!sandbox.executeCommand) {
+ throw new Error(`Sandbox '${sandbox.provider}' cannot run browser commands.`);
+ }
+ const result = await sandbox.executeCommand(command, args, { timeout });
+ if (!result.success) {
+ const detail = result.stderr.trim() || result.stdout.trim() || `exit code ${result.exitCode}`;
+ throw new Error(`Sandbox browser command '${command}' failed: ${detail}`);
+ }
+ return result.stdout;
+}
+
+async function isExecutable(sandbox: WorkspaceSandbox, path: string): Promise<boolean> {
+ if (!sandbox.executeCommand) return false;
+ const result = await sandbox.executeCommand("test", ["-x", path], { timeout: 5_000 });
+ return result.success;
+}
+
+const lightpandaInstalls = new Map<string, Promise<string>>();
+
+async function installLightpanda(sandbox: WorkspaceSandbox, cacheDir: string): Promise<string> {
+ const system = await execute(sandbox, "uname", ["-s"]);
+ const machine = await execute(sandbox, "uname", ["-m"]);
+ const platform = platformFromUname(system, machine);
+ const release = LIGHTPANDA_RELEASES[platform];
+ const root = sandbox.provider === "local" ? cacheDir : `/tmp/akeru-browser-${LIGHTPANDA_VERSION}`;
+ const binaryPath = NodePath.posix.join(root, "lightpanda");
+ const installKey = sandbox.provider === "local" ? binaryPath : `${sandbox.id}:${binaryPath}`;
+ const activeInstall = lightpandaInstalls.get(installKey);
+ if (activeInstall) return activeInstall;
+
+ const install = (async () => {
+ if (await isExecutable(sandbox, binaryPath)) return binaryPath;
+ const temporaryPath = `${binaryPath}.download`;
+ await execute(sandbox, "mkdir", ["-p", root]);
+ await execute(
+ sandbox,
+ "curl",
+ ["-fsSL", "--retry", "2", release.url, "-o", temporaryPath],
+ 300_000,
+ );
+ const hashCommand = platform.startsWith("darwin-") ? "shasum" : "sha256sum";
+ const hashArgs = platform.startsWith("darwin-")
+ ? ["-a", "256", temporaryPath]
+ : [temporaryPath];
+ const actualHash = (await execute(sandbox, hashCommand, hashArgs)).trim().split(/\s+/)[0];
+ if (actualHash !== release.sha256) {
+ await sandbox.executeCommand?.("rm", ["-f", temporaryPath], { timeout: 5_000 });
+ throw new Error(`Sandbox browser download failed integrity verification for ${platform}.`);
+ }
+ await execute(sandbox, "chmod", ["700", temporaryPath]);
+ await execute(sandbox, "mv", [temporaryPath, binaryPath]);
+ return binaryPath;
+ })();
+ lightpandaInstalls.set(installKey, install);
+ try {
+ return await install;
+ } finally {
+ lightpandaInstalls.delete(installKey);
+ }
+}
+
+function shellQuote(value: string): string {
+ return `'${value.replaceAll("'", `'\\''`)}'`;
+}
+
+export function lightpandaMcpCommand(binaryPath: string, port: number, host = "0.0.0.0"): string {
+ return `${shellQuote(binaryPath)} mcp --host ${shellQuote(host)} --port ${port}`;
+}
+
+function availableLocalPort(): Promise<number> {
+ return new Promise((resolve, reject) => {
+ const server = NodeNet.createServer();
+ server.unref();
+ server.once("error", reject);
+ server.listen(0, "127.0.0.1", () => {
+ const address = server.address();
+ if (!address || typeof address === "string") {
+ server.close();
+ reject(new Error("Could not reserve a local sandbox browser port."));
+ return;
+ }
+ const port = address.port;
+ server.close((cause) => (cause ? reject(cause) : resolve(port)));
+ });
+ });
+}
+
+function browserRequestTransport(url: string): BrowserRequestTransport {
+ return async (request) => {
+ // This runtime is owned by Mastra's promise-based workspace API, not an Effect layer.
+ // @effect-diagnostics-next-line globalFetch:off
+ const response = await fetch(url, {
+ method: request.method,
+ headers: {
+ accept: "application/json, text/event-stream",
+ ...(request.method === "POST" ? { "content-type": "application/json" } : {}),
+ ...(request.sessionId ? { "mcp-session-id": request.sessionId } : {}),
+ },
+ ...(request.body ? { body: request.body } : {}),
+ signal: AbortSignal.timeout(BROWSER_REQUEST_TIMEOUT_MS),
+ });
+ const sessionId = response.headers.get("mcp-session-id");
+ return {
+ status: response.status,
+ body: await response.text(),
+ ...(sessionId ? { sessionId } : {}),
+ };
+ };
+}
+
+function rpcResultText(value: unknown): string {
+ if (typeof value !== "object" || value === null || !("content" in value)) {
+ return JSON.stringify(value);
+ }
+ const content = value.content;
+ if (!Array.isArray(content)) return JSON.stringify(value);
+ return content
+ .flatMap((item) => {
+ if (typeof item !== "object" || item === null || !("text" in item)) return [];
+ return typeof item.text === "string" ? [item.text] : [];
+ })
+ .join("\n");
+}
+
+class LightpandaRpc implements BotBrowserRpc {
+ private readonly input: BotBrowserProcessInput;
+ private nextId = 1;
+ private processes: ProcessHandle[] = [];
+ private startPromise: Promise<void> | undefined;
+ private requestTransport: BrowserRequestTransport | undefined;
+ private attachmentValue: BotBrowserAttachment | undefined;
+ private sessionId: string | undefined;
+ private resumeUrl: string | undefined;
+ private closed = false;
+
+ constructor(input: BotBrowserProcessInput) {
+ this.input = input;
+ }
+
+ async call(name: string, arguments_: Readonly<Record<string, unknown>>): Promise<string> {
+ const result = await this.request("tools/call", { name, arguments: arguments_ });
+ if (name === "goto" || name === "click" || name === "fill") {
+ await this.rememberCurrentUrl().catch(() => undefined);
+ }
+ return rpcResultText(result);
+ }
+
+ async attachment(): Promise<BotBrowserAttachment | undefined> {
+ await this.ensureStarted();
+ return this.attachmentValue;
+ }
+
+ async reconnect(): Promise<void> {
+ if (this.closed) throw new Error(`Sandbox browser for '${this.input.threadId}' is closed.`);
+ await this.stopProcesses();
+ }
+
+ async close(): Promise<void> {
+ if (this.closed) return;
+ this.closed = true;
+ await this.stopProcesses();
+ }
+
+ private async ensureStarted(): Promise<void> {
+ if (this.closed) throw new Error(`Sandbox browser for '${this.input.threadId}' is closed.`);
+ this.startPromise ??= this.start().catch((cause: unknown) => {
+ this.startPromise = undefined;
+ throw cause;
+ });
+ return this.startPromise;
+ }
+
+ private async start(): Promise<void> {
+ const sandbox = this.input.workspace.sandbox;
+ if (!sandbox?.processes) {
+ throw new Error(`Workspace '${this.input.workspace.id}' cannot host a sandbox browser.`);
+ }
+ if (sandbox.provider !== "local") {
+ throw new Error(`Sandbox '${sandbox.provider}' has no Akeru browser adapter.`);
+ }
+ const binaryPath = await installLightpanda(sandbox, this.input.cacheDir);
+ if (this.closed) throw new Error(`Sandbox browser for '${this.input.threadId}' is closed.`);
+ const browserPort = await availableLocalPort();
+ const browserHandle = await sandbox.processes.spawn(
+ lightpandaMcpCommand(binaryPath, browserPort, "127.0.0.1"),
+ { maxRetainedBytes: 64 * 1_024 },
+ );
+ const processes = [browserHandle];
+ if (this.closed) {
+ await Promise.all(processes.map((process) => process.kill().catch(() => false)));
+ throw new Error(`Sandbox browser for '${this.input.threadId}' is closed.`);
+ }
+ this.processes = processes;
+ for (const process of processes) {
+ void process.wait().then(() => this.processStopped(process));
+ }
+
+ try {
+ const browserUrl = `http://127.0.0.1:${browserPort}`;
+ this.requestTransport = browserRequestTransport(browserUrl);
+ await this.initialize();
+ await this.restoreCurrentUrl();
+ this.attachmentValue = {
+ browserUrl,
+ mcpSessionId: this.sessionId!,
+ requestHeaders: {},
+ localRequestHeaders: {},
+ availableToHostedPlugins: false,
+ };
+ } catch (cause) {
+ this.processes = [];
+ this.requestTransport = undefined;
+ this.attachmentValue = undefined;
+ this.sessionId = undefined;
+ await Promise.all(processes.map((process) => process.kill().catch(() => false)));
+ throw cause;
+ }
+ }
+
+ private processStopped(process: ProcessHandle): void {
+ if (this.closed || !this.processes.includes(process)) return;
+ const remaining = this.processes.filter((candidate) => candidate !== process);
+ this.processes = [];
+ this.requestTransport = undefined;
+ this.attachmentValue = undefined;
+ this.sessionId = undefined;
+ this.startPromise = undefined;
+ void Promise.all(remaining.map((candidate) => candidate.kill().catch(() => false)));
+ }
+
+ private async stopProcesses(): Promise<void> {
+ const transport = this.requestTransport;
+ const sessionId = this.sessionId;
+ const processes = this.processes;
+ this.processes = [];
+ this.requestTransport = undefined;
+ this.attachmentValue = undefined;
+ this.sessionId = undefined;
+ this.startPromise = undefined;
+ if (transport && sessionId) {
+ await transport({ method: "DELETE", sessionId }).catch(() => undefined);
+ }
+ await Promise.all(processes.map((process) => process.kill().catch(() => false)));
+ }
+
+ private async rememberCurrentUrl(): Promise<void> {
+ const result = await this.send("tools/call", { name: "session_list", arguments: {} });
+ const text = rpcResultText(result.result);
+ const sessions = JSON.parse(text) as ReadonlyArray<{ readonly url?: unknown }>;
+ const current = sessions.find((session) => typeof session.url === "string");
+ if (typeof current?.url === "string" && current.url.length > 0) this.resumeUrl = current.url;
+ }
+
+ private async restoreCurrentUrl(): Promise<void> {
+ if (!this.resumeUrl) return;
+ await this.send("tools/call", { name: "goto", arguments: { url: this.resumeUrl } });
+ }
+
+ private async initialize(): Promise<void> {
+ let lastCause: unknown;
+ for (let attempt = 0; attempt < 30; attempt++) {
+ try {
+ const response = await this.send("initialize", {
+ protocolVersion: MCP_PROTOCOL_VERSION,
+ capabilities: {},
+ clientInfo: { name: "akeru", version: "1.0.0" },
+ });
+ if (!response.sessionId) throw new Error("Sandbox browser did not create an MCP session.");
+ this.sessionId = response.sessionId;
+ await this.sendNotification("notifications/initialized");
+ return;
+ } catch (cause) {
+ lastCause = cause;
+ await NodeTimersPromises.setTimeout(100);
+ }
+ }
+ throw lastCause instanceof Error
+ ? lastCause
+ : new Error("Sandbox browser did not become ready.");
+ }
+
+ private async request(method: string, params: unknown): Promise<unknown> {
+ await this.ensureStarted();
+ return (await this.send(method, params)).result;
+ }
+
+ private async send(
+ method: string,
+ params: unknown,
+ ): Promise<{ readonly result: unknown; readonly sessionId?: string }> {
+ const transport = this.requestTransport;
+ if (!transport) throw new Error("Sandbox browser HTTP transport is not ready.");
+ const id = this.nextId++;
+ const response = await transport({
+ method: "POST",
+ body: JSON.stringify({ jsonrpc: "2.0", id, method, params }),
+ ...(this.sessionId ? { sessionId: this.sessionId } : {}),
+ });
+ if (response.status < 200 || response.status >= 300) {
+ throw new Error(`Sandbox browser HTTP request failed with status ${response.status}.`);
+ }
+ const parsed = JSON.parse(response.body) as JsonRpcResponse;
+ if (parsed.error) {
+ throw new Error(parsed.error.message ?? `Browser RPC ${parsed.error.code ?? "failed"}.`);
+ }
+ return {
+ result: parsed.result,
+ ...(response.sessionId ? { sessionId: response.sessionId } : {}),
+ };
+ }
+
+ private async sendNotification(method: string): Promise<void> {
+ const transport = this.requestTransport;
+ if (!transport) throw new Error("Sandbox browser HTTP transport is not ready.");
+ const response = await transport({
+ method: "POST",
+ body: JSON.stringify({ jsonrpc: "2.0", method }),
+ ...(this.sessionId ? { sessionId: this.sessionId } : {}),
+ });
+ if (response.status < 200 || response.status >= 300) {
+ throw new Error(`Sandbox browser notification failed with status ${response.status}.`);
+ }
+ }
+}
+
+const browserTargetSchema = z
+ .object({
+ selector: z.string().trim().min(1).optional(),
+ backendNodeId: z.number().int().positive().optional(),
+ })
+ .refine((input) => input.selector !== undefined || input.backendNodeId !== undefined, {
+ message: "Provide selector or backendNodeId.",
+ });
+
+export function createBotBrowserTools(rpc: BotBrowserRpc): ToolsInput {
+ const call = async (name: string, input: Readonly<Record<string, unknown>>) =>
+ redactSensitiveText(await rpc.call(name, input)).value;
+
+ return {
+ browser_navigate: createTool({
+ id: "browser_navigate",
+ description: "Navigate the browser inside the active sandbox to one URL.",
+ inputSchema: z.object({ url: z.url() }),
+ execute: async ({ url }) => ({ result: await call("goto", { url }) }),
+ }),
+ browser_snapshot: createTool({
+ id: "browser_snapshot",
+ description:
+ "Read the current sandbox browser page as a semantic tree with selectors and backend node ids.",
+ inputSchema: z.object({}),
+ execute: async () => {
+ const snapshot = await call("tree", {});
+ const redacted = redactSensitiveText(snapshot).value;
+ return {
+ snapshot: redacted.slice(0, MAX_SNAPSHOT_LENGTH),
+ truncated: redacted.length > MAX_SNAPSHOT_LENGTH,
+ };
+ },
+ }),
+ browser_click: createTool({
+ id: "browser_click",
+ description: "Click one element in the current sandbox browser page.",
+ inputSchema: browserTargetSchema,
+ execute: async (input) => ({ result: await call("click", input) }),
+ }),
+ browser_type: createTool({
+ id: "browser_type",
+ description: "Replace the text in one field in the current sandbox browser page.",
+ inputSchema: browserTargetSchema.extend({ text: z.string() }),
+ execute: async ({ text, ...target }) => ({
+ result: await call("fill", { ...target, value: text }),
+ }),
+ }),
+ };
+}
+
+export function createBotBrowser(input: CreateBotBrowserInput): BotBrowser {
+ const rpc = input.makeRpc?.(input) ?? new LightpandaRpc(input);
+ return {
+ tools: createBotBrowserTools(rpc),
+ attachment: () => rpc.attachment(),
+ reconnect: () => rpc.reconnect(),
+ close: () => rpc.close(),
+ };
+}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 91044b76f..cfc849a78 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -514,6 +514,9 @@ importers:
yaml:
specifier: ^2.9.0
version: 2.9.0
+ zod:
+ specifier: ^4.4.3
+ version: 4.4.3
devDependencies:
'@effect/vitest':
specifier: 4.0.0-beta.103