Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

8.2 KB
Raw Blame
import type { PluginConfig, PluginRuntimeContext } from "../types.js";
import { pinchtabFetch, textResult, imageResult, resourceResult } from "../client.js";
import { checkNavigationPolicy, enforcePolicyOrReturn } from "../policy.js";
import { ensureServerRunning, waitForInstanceReady, getEnhancedHealth, resolveEffectiveConfig, resolveProfile, getLastTabId, rememberRuntimeContext, setLastTabId } from "../session.js";

export const browserToolSchema = {
  type: "object",
  properties: {
    action: {
      type: "string",
      enum: ["navigate", "snapshot", "screenshot", "click", "type", "fill", "press", "hover", "scroll", "select", "tabs", "pdf", "status"],
      description: "Browser action",
    },
    url: { type: "string", description: "URL to navigate to" },
    profile: { type: "string", description: "Profile: 'openclaw' (isolated) or 'user' (attached session)" },
    ref: { type: "string", description: "Element ref from snapshot (e.g. e5)" },
    text: { type: "string", description: "Text to type or fill" },
    key: { type: "string", description: "Key to press (Enter, Tab, Escape)" },
    value: { type: "string", description: "Value for select" },
    selector: { type: "string", description: "CSS selector for snapshot scope" },
    format: { type: "string", enum: ["compact", "json", "text"], description: "Output format" },
    maxTokens: { type: "number", description: "Truncate output to ~N tokens" },
    quality: { type: "number", description: "Screenshot JPEG quality 1-100" },
    tabId: { type: "string", description: "Target tab ID" },
    tabAction: { type: "string", enum: ["list", "new", "close"], description: "Tab operation (for action=tabs)" },
    newTab: { type: "boolean", description: "Open URL in new tab" },
    landscape: { type: "boolean", description: "PDF landscape orientation" },
    scale: { type: "number", description: "PDF print scale" },
  },
  required: ["action"],
};

export const browserToolDescription = `OpenClaw-compatible browser control (backed by PinchTab).

Actions:
- navigate: go to URL (url, profile?, newTab?)
- snapshot: accessibility tree for interactions (selector?, format?, maxTokens?)
- screenshot: capture page image (quality?, format?)
- click/type/fill/press/hover/scroll/select: element actions (ref, text?, key?, value?)
- tabs: list/new/close tabs (tabAction?, url?, tabId?)
- pdf: export page as PDF (landscape?, scale?)
- status: check browser/server health

Profiles: "openclaw" (default isolated), "user" (attach to existing session)`;

export async function executeBrowserAction(rawCfg: PluginConfig, rawParams: any, context?: PluginRuntimeContext): Promise<any> {
  rememberRuntimeContext(context);
  const cfg = await resolveEffectiveConfig(rawCfg);
  const params = { ...rawParams };
  const { action, profile } = params;

  // Resolve profile to instance
  const profileConfig = resolveProfile(cfg, profile, context);

  // Auto-start server if needed
  if (action !== "status") {
    const serverCheck = await ensureServerRunning(cfg, context);
    if (!serverCheck.ok) {
      return textResult({ error: serverCheck.error });
    }
    const readyCheck = await waitForInstanceReady(cfg, profileConfig.instanceId, context);
    if (!readyCheck.ok) {
      return textResult({ error: readyCheck.error });
    }
  }

  // Session tab persistence
  if (cfg.persistSessionTabs !== false && !params.tabId && getLastTabId(context)) {
    params.tabId = getLastTabId(context);
  }

  // --- navigate ---
  if (action === "navigate") {
    const navPolicy = enforcePolicyOrReturn(checkNavigationPolicy(cfg, params.url));
    if (navPolicy) return navPolicy;

    let result;
    if (params.newTab) {
      if (profileConfig.instanceId) {
        result = await pinchtabFetch(cfg, `/instances/${encodeURIComponent(profileConfig.instanceId)}/tabs/open`, { body: { url: params.url } }, context);
      } else {
        result = await pinchtabFetch(cfg, "/tab", { body: { action: "new", url: params.url } }, context);
      }
    } else if (params.tabId) {
      result = await pinchtabFetch(cfg, `/tabs/${encodeURIComponent(params.tabId)}/navigate`, { body: { url: params.url } }, context);
    } else if (profileConfig.instanceId) {
      result = await pinchtabFetch(cfg, `/instances/${encodeURIComponent(profileConfig.instanceId)}/tabs/open`, { body: { url: params.url } }, context);
    } else {
      result = await pinchtabFetch(cfg, "/navigate", { body: { url: params.url } }, context);
    }
    if (result?.tabId) setLastTabId(result.tabId, context);
    return textResult(result);
  }

  // --- snapshot ---
  if (action === "snapshot") {
    const query = new URLSearchParams();
    if (params.tabId) query.set("tabId", params.tabId);
    query.set("filter", "interactive");
    query.set("format", params.format || cfg.defaultSnapshotFormat || "compact");
    if (params.selector) query.set("selector", params.selector);
    if (params.maxTokens) query.set("maxTokens", String(params.maxTokens));
    return textResult(await pinchtabFetch(cfg, `/snapshot?${query.toString()}`, {}, context));
  }

  // --- screenshot ---
  if (action === "screenshot") {
    const query = new URLSearchParams();
    if (params.tabId) query.set("tabId", params.tabId);
    const fmt = params.format === "png" ? "png" : (cfg.screenshotFormat || "jpeg");
    if (fmt === "png") query.set("format", "png");
    query.set("quality", String(params.quality || cfg.screenshotQuality || 80));
    try {
      const res = await pinchtabFetch(cfg, `/screenshot?${query.toString()}`, { rawResponse: true }, context);
      if (res instanceof Response) {
        if (!res.ok) return textResult({ error: `Screenshot failed: ${res.status}` });
        const buf = await res.arrayBuffer();
        const b64 = Buffer.from(buf).toString("base64");
        return imageResult(b64, fmt === "png" ? "image/png" : "image/jpeg");
      }
      return textResult(res);
    } catch (err: any) {
      return textResult({ error: `Screenshot failed: ${err?.message}` });
    }
  }

  // --- element actions ---
  if (["click", "type", "fill", "press", "hover", "scroll", "select"].includes(action)) {
    const body: any = { kind: action };
    if (params.ref) body.ref = params.ref;
    if (params.text) body.text = params.text;
    if (params.key) body.key = params.key;
    if (params.value) body.value = params.value;
    if (params.tabId) body.tabId = params.tabId;
    return textResult(await pinchtabFetch(cfg, "/action", { body }, context));
  }

  // --- tabs ---
  if (action === "tabs") {
    const tabAction = params.tabAction || "list";
    if (tabAction === "list") {
      return textResult(await pinchtabFetch(cfg, "/tabs", {}, context));
    }
    if (tabAction === "close") {
      const body: any = {};
      if (params.tabId) body.tabId = params.tabId;
      const result = await pinchtabFetch(cfg, "/close", { body }, context);
      if (params.tabId && params.tabId === getLastTabId(context)) setLastTabId(undefined, context);
      return textResult(result);
    }
    const body: any = { action: tabAction };
    if (params.url) body.url = params.url;
    if (params.tabId) body.tabId = params.tabId;
    const result = await pinchtabFetch(cfg, "/tab", { body }, context);
    if (tabAction === "new" && result?.tabId) setLastTabId(result.tabId, context);
    return textResult(result);
  }

  // --- pdf ---
  if (action === "pdf") {
    const query = new URLSearchParams();
    query.set("raw", "true");
    if (params.tabId) query.set("tabId", params.tabId);
    if (params.landscape) query.set("landscape", "true");
    if (params.scale) query.set("scale", String(params.scale));
    try {
      const res = await pinchtabFetch(cfg, `/pdf?${query.toString()}`, { rawResponse: true }, context);
      if (res instanceof Response) {
        if (!res.ok) return textResult({ error: `PDF failed: ${res.status}` });
        const buf = await res.arrayBuffer();
        const b64 = Buffer.from(buf).toString("base64");
        return resourceResult("pdf://export", "application/pdf", b64);
      }
      return textResult(res);
    } catch (err: any) {
      return textResult({ error: `PDF failed: ${err?.message}` });
    }
  }

  // --- status ---
  if (action === "status") {
    return textResult(await getEnhancedHealth(cfg, context));
  }

  return textResult({ error: `Unknown browser action: ${action}` });
}