Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

23.3 KB
Raw Blame
import type { PluginConfig, PluginRuntimeContext } from "../types.js";
import { pinchtabFetch, textResult, imageResult, resourceResult, normalizeActionParams, looksLikeStaleRef } from "../client.js";
import { checkNavigationPolicy, checkEvaluatePolicy, checkDownloadPolicy, checkUploadPolicy, checkNetworkInterceptPolicy, enforcePolicyOrReturn } from "../policy.js";
import { ensureReady, getEnhancedHealth, getLastTabId, rememberRuntimeContext, resolveEffectiveConfig, setLastTabId } from "../session.js";

export const pinchtabToolSchema = {
  type: "object",
  properties: {
    action: {
      type: "string",
      enum: [
        "navigate", "snapshot", "click", "type", "press", "fill", "hover",
        "mouse-move", "mouse-down", "mouse-up", "mouse-wheel", "scroll",
        "select", "focus", "text", "wait", "handoff", "tabs", "screenshot",
        "evaluate", "pdf", "download", "upload", "network", "health",
      ],
      description: "Action to perform",
    },
    url: { type: "string", description: "URL for navigate or new tab" },
    ref: { type: "string", description: "Element ref from snapshot (e.g. e5)" },
    text: { type: "string", description: "Text to type or fill" },
    key: { type: "string", description: "Key to press (e.g. Enter, Tab, Escape)" },
    expression: { type: "string", description: "JavaScript expression for evaluate" },
    selector: { type: "string", description: "CSS selector for snapshot scope or action target" },
    filter: { type: "string", enum: ["interactive", "all"], description: "Snapshot filter" },
    format: { type: "string", enum: ["json", "compact", "text", "yaml"], description: "Snapshot format" },
    maxTokens: { type: "number", description: "Truncate snapshot to ~N tokens" },
    depth: { type: "number", description: "Max snapshot tree depth" },
    diff: { type: "boolean", description: "Snapshot diff: only changes since last" },
    value: { type: "string", description: "Value for select dropdown" },
    scrollY: { type: "number", description: "Pixels to scroll vertically" },
    x: { type: "number", description: "Mouse X coordinate" },
    y: { type: "number", description: "Mouse Y coordinate" },
    button: { type: "string", enum: ["left", "right", "middle"], description: "Mouse button" },
    deltaX: { type: "number", description: "Mouse wheel horizontal delta" },
    deltaY: { type: "number", description: "Mouse wheel vertical delta" },
    waitNav: { type: "boolean", description: "Wait for navigation after action" },
    tabId: { type: "string", description: "Target tab ID" },
    tabAction: { type: "string", enum: ["list", "new", "close"], description: "Tab sub-action" },
    newTab: { type: "boolean", description: "Open URL in new tab" },
    blockImages: { type: "boolean", description: "Block image loading" },
    timeout: { type: "number", description: "Navigation timeout in seconds" },
    quality: { type: "number", description: "JPEG quality 1-100" },
    screenshotFormat: { type: "string", enum: ["jpeg", "png"], description: "Screenshot format" },
    mode: { type: "string", enum: ["readability", "raw"], description: "Text extraction mode" },
    ms: { type: "number", description: "Wait milliseconds" },
    state: { type: "string", enum: ["visible", "hidden", "attached", "detached"], description: "Wait state" },
    load: { type: "string", enum: ["load", "domcontentloaded", "networkidle"], description: "Document load state" },
    fn: { type: "string", description: "JavaScript predicate for wait" },
    humanReason: { type: "string", description: "Reason for manual handoff" },
    humanPrompt: { type: "string", description: "Instruction for human handoff" },
    landscape: { type: "boolean", description: "PDF landscape orientation" },
    scale: { type: "number", description: "PDF print scale" },
    downloadUrl: { type: "string", description: "URL to download file from" },
    savePath: { type: "string", description: "Path to save downloaded file" },
    files: { type: "array", items: { type: "string" }, description: "Base64-encoded files for upload" },
    paths: { type: "array", items: { type: "string" }, description: "File paths for upload" },
    networkAction: { type: "string", enum: ["list", "get", "export", "clear", "route", "unroute"], description: "Network sub-action" },
    requestId: { type: "string", description: "Network request ID for get action" },
    method: { type: "string", description: "Filter network by HTTP method (also: limits a route rule to one method; fulfill rules without method skip OPTIONS preflights to avoid breaking CORS)" },
    status: { type: "string", description: "Filter network by status (e.g. 4xx, 5xx, 200)" },
    resourceType: { type: "string", description: "Filter network by resource type, or limit a route to one type (xhr, fetch, document, script, image, etc)" },
    limit: { type: "number", description: "Limit network results" },
    exportFormat: { type: "string", enum: ["har", "json"], description: "Network export format" },
    pattern: { type: "string", description: "URL pattern for network route/unroute (substring or glob)" },
    routeAction: { type: "string", enum: ["continue", "abort", "fulfill"], description: "Network route behavior. Default 'continue' (pass-through)." },
    responseBody: { type: "string", description: "Response body for routeAction=fulfill (sent verbatim; not auto-encoded)" },
    contentType: { type: "string", description: "Response Content-Type for fulfill (default application/json)" },
    responseStatus: { type: "number", description: "Response status code for fulfill (default 200)" },
  },
  required: ["action"],
};

export const pinchtabToolDescription = `Browser control via PinchTab. Actions:
- navigate: go to URL (url, tabId?, newTab?, blockImages?, timeout?)
- snapshot: accessibility tree (filter?, format?, selector?, maxTokens?, depth?, diff?, tabId?)
- click/type/press/fill/hover/scroll/select/focus: act on element (ref, text?, key?, value?, scrollY?, waitNav?, tabId?)
- mouse-move/mouse-down/mouse-up/mouse-wheel: low-level mouse controls (ref|selector|x+y, button?, deltaX?, deltaY?, tabId?)
- text: extract readable text (mode?, tabId?)
- wait: pause until condition (selector|text|url|load|fn|ms, tabId?, timeout?, state?)
- handoff: request human intervention mid-flow (captcha/login/2FA/credentials)
- tabs: list/new/close tabs (tabAction?, url?, tabId?)
- screenshot: JPEG screenshot (quality?, tabId?)
- evaluate: run JS (expression, tabId?)
- pdf: export page as PDF (landscape?, scale?, tabId?)
- download: download file from URL (downloadUrl, savePath?, tabId?)
- upload: upload files to file input (selector, files[]|paths[], tabId?)
- network: capture/inspect/intercept network requests (networkAction: list|get|export|clear|route|unroute). For route: pattern, routeAction (continue|abort|fulfill), responseBody?, contentType?, responseStatus?, resourceType?. For unroute: pattern? (omit to clear all). Fulfill is BLOCKED on hosts in security.allowedDomains (sensitive surfaces) and allowed elsewhere; rules on allowlisted hosts fall through to real fetch.
- health: check connectivity

Token strategy: use "text" for reading (~800 tokens), "snapshot" with filter=interactive&format=compact for interactions (~3,600), diff=true on subsequent snapshots.`;

type ActionHandler = (cfg: PluginConfig, normalized: any, context?: PluginRuntimeContext) => Promise<any>;

const elementActions = [
  "click", "type", "press", "fill", "hover", "mouse-move", "mouse-down",
  "mouse-up", "mouse-wheel", "scroll", "select", "focus",
];

async function handleNavigate(cfg: PluginConfig, normalized: any, context?: PluginRuntimeContext): Promise<any> {
  const navPolicy = enforcePolicyOrReturn(checkNavigationPolicy(cfg, normalized.url));
  if (navPolicy) return navPolicy;

  let result;
  if (normalized.newTab) {
    const body: any = { action: "new", url: normalized.url };
    if (normalized.blockImages) body.blockImages = true;
    if (normalized.timeout) body.timeout = normalized.timeout;
    result = await pinchtabFetch(cfg, "/tab", { body }, context);
  } else if (normalized.tabId) {
    const body: any = { url: normalized.url };
    if (normalized.blockImages) body.blockImages = true;
    if (normalized.timeout) body.timeout = normalized.timeout;
    result = await pinchtabFetch(cfg, `/tabs/${encodeURIComponent(normalized.tabId)}/navigate`, { body }, context);
  } else {
    const body: any = { url: normalized.url };
    if (normalized.blockImages) body.blockImages = true;
    if (normalized.timeout) body.timeout = normalized.timeout;
    result = await pinchtabFetch(cfg, "/navigate", { body }, context);
  }
  if (result?.tabId) setLastTabId(result.tabId, context);
  return textResult(result);
}

async function handleSnapshot(cfg: PluginConfig, normalized: any, context?: PluginRuntimeContext): Promise<any> {
  const query = new URLSearchParams();
  if (normalized.tabId) query.set("tabId", normalized.tabId);
  const filter = normalized.filter ?? cfg.defaultSnapshotFilter ?? "interactive";
  const format = normalized.format ?? cfg.defaultSnapshotFormat ?? "compact";
  query.set("filter", filter);
  query.set("format", format);
  if (normalized.selector) query.set("selector", normalized.selector);
  if (normalized.maxTokens) query.set("maxTokens", String(normalized.maxTokens));
  if (normalized.depth) query.set("depth", String(normalized.depth));
  if (normalized.diff) query.set("diff", "true");
  return textResult(await pinchtabFetch(cfg, `/snapshot?${query.toString()}`, {}, context));
}

async function handleElementAction(cfg: PluginConfig, normalized: any, context?: PluginRuntimeContext): Promise<any> {
  const body: any = { kind: normalized.action };
  for (const k of ["ref", "text", "key", "selector", "value", "scrollY", "x", "y", "button", "deltaX", "deltaY", "tabId", "waitNav"]) {
    if (normalized[k] !== undefined) body[k] = normalized[k];
  }

  let result = await pinchtabFetch(cfg, "/action", { body }, context);

  // Stale ref recovery
  if (body.ref && looksLikeStaleRef(result)) {
    const q = new URLSearchParams();
    q.set("filter", "interactive");
    q.set("format", "compact");
    if (body.tabId) q.set("tabId", body.tabId);
    await pinchtabFetch(cfg, `/snapshot?${q.toString()}`, {}, context);
    const retried = await pinchtabFetch(cfg, "/action", { body }, context);
    result = retried?.error
      ? { ...retried, warning: "Action retried once after snapshot refresh but still failed." }
      : { ...retried, warning: "Action succeeded after automatic snapshot refresh (stale ref recovery)." };
  }

  // Fill fallback to type
  if (normalized.action === "fill" && result?.error && body.ref && (typeof body.text === "string" || typeof body.value === "string")) {
    const typeBody: any = { ...body, kind: "type" };
    if (typeof typeBody.text !== "string" && typeof typeBody.value === "string") {
      typeBody.text = typeBody.value;
    }
    delete typeBody.value;
    const typed = await pinchtabFetch(cfg, "/action", { body: typeBody }, context);
    if (!typed?.error) {
      result = { ...typed, warning: "Fill failed; type fallback succeeded." };
    }
  }

  return textResult(result);
}

async function handleText(cfg: PluginConfig, normalized: any, context?: PluginRuntimeContext): Promise<any> {
  const query = new URLSearchParams();
  if (normalized.tabId) query.set("tabId", normalized.tabId);
  if (normalized.mode) query.set("mode", normalized.mode);
  return textResult(await pinchtabFetch(cfg, `/text?${query.toString()}`, {}, context));
}

// Wait CONDITION fields decide whether a wait should run; the full body also
// carries tab/timeout/state config. Keeping the body list a superset of the
// condition list ensures a new wait condition propagates to both.
const WAIT_CONDITION_FIELDS = ["selector", "text", "url", "load", "fn", "ms"] as const;
const WAIT_BODY_FIELDS = [...WAIT_CONDITION_FIELDS, "tabId", "timeout", "state"] as const;

function buildWaitBody(normalized: any): any {
  const body: any = {};
  for (const k of WAIT_BODY_FIELDS) {
    if (normalized[k] !== undefined) body[k] = normalized[k];
  }
  return body;
}

function hasWaitCondition(normalized: any): boolean {
  return WAIT_CONDITION_FIELDS.some((k) => normalized[k] !== undefined);
}

async function handleWait(cfg: PluginConfig, normalized: any, context?: PluginRuntimeContext): Promise<any> {
  return textResult(await pinchtabFetch(cfg, "/wait", { body: buildWaitBody(normalized) }, context));
}

async function handleHandoff(cfg: PluginConfig, normalized: any, context?: PluginRuntimeContext): Promise<any> {
  const handoffMeta = {
    status: "human_handoff_required",
    reason: normalized.humanReason || "Manual intervention required (captcha/login/2FA/credential entry).",
    instructions: normalized.humanPrompt || "Please complete the step in the headed browser, then resume automation.",
  };

  if (!hasWaitCondition(normalized)) {
    return textResult({ ...handoffMeta, resumed: false, next: "Call with a wait condition or use action='wait' to resume." });
  }

  const waitResult = await pinchtabFetch(cfg, "/wait", { body: buildWaitBody(normalized) }, context);
  return textResult({ ...handoffMeta, resumed: !waitResult?.error, waitResult });
}

async function handleTabs(cfg: PluginConfig, normalized: any, context?: PluginRuntimeContext): Promise<any> {
  const tabAction = normalized.tabAction || "list";
  if (tabAction === "list") {
    const listed = await pinchtabFetch(cfg, "/tabs", {}, context);
    const tabs = Array.isArray(listed?.tabs) ? listed.tabs : Array.isArray(listed) ? listed : [];
    if (tabs.length > 0) return textResult(listed);

    const instances = await pinchtabFetch(cfg, "/instances", {}, context);
    const list = Array.isArray(instances?.value) ? instances.value : Array.isArray(instances) ? instances : [];
    const running = list.find((i: any) => i?.status === "running" && i?.id);
    if (!running) {
      return textResult({ ...listed, warning: "No tabs returned and no running instance found." });
    }
    const instanceTabs = await pinchtabFetch(cfg, `/instances/${encodeURIComponent(running.id)}/tabs`, {}, context);
    return textResult({ source: "instance-fallback", instanceId: running.id, tabs: instanceTabs?.tabs ?? instanceTabs });
  }
  if (tabAction === "close") {
    const body: any = {};
    if (normalized.tabId) body.tabId = normalized.tabId;
    const result = await pinchtabFetch(cfg, "/close", { body }, context);
    if (normalized.tabId && normalized.tabId === getLastTabId(context)) setLastTabId(undefined, context);
    return textResult(result);
  }
  const body: any = { action: tabAction };
  if (normalized.url) body.url = normalized.url;
  if (normalized.tabId) body.tabId = normalized.tabId;
  const result = await pinchtabFetch(cfg, "/tab", { body }, context);
  if (tabAction === "new" && result?.tabId) setLastTabId(result.tabId, context);
  return textResult(result);
}

// fetchBinaryResult performs a raw-response GET and adapts it to a tool result:
// non-2xx → an {error} textResult, a thrown error → an {error} textResult, a
// non-Response (e.g. already-JSON) body → textResult, and a 2xx binary body →
// toResult(base64, response). The response is passed so callers can derive a
// MIME type from its headers.
async function fetchBinaryResult(
  cfg: PluginConfig,
  path: string,
  label: string,
  toResult: (b64: string, res: Response) => any,
  context?: PluginRuntimeContext,
): Promise<any> {
  try {
    const res = await pinchtabFetch(cfg, path, { rawResponse: true }, context);
    if (res instanceof Response) {
      if (!res.ok) return textResult({ error: `${label} failed: ${res.status} ${await res.text()}` });
      const buf = await res.arrayBuffer();
      const b64 = Buffer.from(buf).toString("base64");
      return toResult(b64, res);
    }
    return textResult(res);
  } catch (err: any) {
    return textResult({ error: `${label} failed: ${err?.message}` });
  }
}

async function handleScreenshot(cfg: PluginConfig, normalized: any, context?: PluginRuntimeContext): Promise<any> {
  const query = new URLSearchParams();
  if (normalized.tabId) query.set("tabId", normalized.tabId);
  const format = normalized.screenshotFormat ?? cfg.screenshotFormat ?? "jpeg";
  const quality = normalized.quality ?? cfg.screenshotQuality ?? 80;
  if (format === "png") query.set("format", "png");
  query.set("quality", String(quality));
  return fetchBinaryResult(
    cfg,
    `/screenshot?${query.toString()}`,
    "Screenshot",
    (b64) => imageResult(b64, format === "png" ? "image/png" : "image/jpeg"),
    context,
  );
}

async function handleEvaluate(cfg: PluginConfig, normalized: any, context?: PluginRuntimeContext): Promise<any> {
  const policyResult = enforcePolicyOrReturn(checkEvaluatePolicy(cfg));
  if (policyResult) return policyResult;

  const body: any = { expression: normalized.expression };
  if (normalized.tabId) body.tabId = normalized.tabId;
  return textResult(await pinchtabFetch(cfg, "/evaluate", { body }, context));
}

async function handlePdf(cfg: PluginConfig, normalized: any, context?: PluginRuntimeContext): Promise<any> {
  const query = new URLSearchParams();
  query.set("raw", "true");
  if (normalized.tabId) query.set("tabId", normalized.tabId);
  if (normalized.landscape) query.set("landscape", "true");
  if (normalized.scale) query.set("scale", String(normalized.scale));
  return fetchBinaryResult(
    cfg,
    `/pdf?${query.toString()}`,
    "PDF",
    (b64) => resourceResult("pdf://export", "application/pdf", b64),
    context,
  );
}

async function handleNetwork(cfg: PluginConfig, normalized: any, context?: PluginRuntimeContext): Promise<any> {
  const networkAction = normalized.networkAction || "list";

  if (networkAction === "clear") {
    const query = new URLSearchParams();
    if (normalized.tabId) query.set("tabId", normalized.tabId);
    return textResult(await pinchtabFetch(cfg, `/network/clear?${query.toString()}`, { method: "POST" }, context));
  }

  if (networkAction === "get" && normalized.requestId) {
    return textResult(await pinchtabFetch(cfg, `/network/${encodeURIComponent(normalized.requestId)}`, {}, context));
  }

  if (networkAction === "export") {
    const query = new URLSearchParams();
    if (normalized.tabId) query.set("tabId", normalized.tabId);
    if (normalized.exportFormat === "json") query.set("format", "json");
    const result = await pinchtabFetch(cfg, `/network/export?${query.toString()}`, {}, context);
    if (normalized.exportFormat === "har" || !normalized.exportFormat) {
      return resourceResult("har://export", "application/json", Buffer.from(JSON.stringify(result)).toString("base64"));
    }
    return textResult(result);
  }

  if (networkAction === "route") {
    const policyResult = enforcePolicyOrReturn(checkNetworkInterceptPolicy(cfg));
    if (policyResult) return policyResult;
    if (!normalized.tabId) return textResult({ error: "route requires tabId" });
    if (!normalized.pattern) return textResult({ error: "route requires pattern" });
    const payload: Record<string, any> = {
      pattern: normalized.pattern,
      action: normalized.routeAction || "continue",
    };
    if (normalized.responseBody !== undefined) payload.body = normalized.responseBody;
    if (normalized.contentType) payload.contentType = normalized.contentType;
    if (typeof normalized.responseStatus === "number") payload.status = normalized.responseStatus;
    if (normalized.resourceType) payload.resourceType = normalized.resourceType;
    if (normalized.method) payload.method = normalized.method;
    return textResult(await pinchtabFetch(cfg, `/tabs/${encodeURIComponent(normalized.tabId)}/network/route`, {
      method: "POST",
      body: payload,
    }, context));
  }

  if (networkAction === "unroute") {
    const policyResult = enforcePolicyOrReturn(checkNetworkInterceptPolicy(cfg));
    if (policyResult) return policyResult;
    if (!normalized.tabId) return textResult({ error: "unroute requires tabId" });
    const query = new URLSearchParams();
    if (normalized.pattern) query.set("pattern", normalized.pattern);
    const qs = query.toString();
    const path = `/tabs/${encodeURIComponent(normalized.tabId)}/network/route${qs ? `?${qs}` : ""}`;
    return textResult(await pinchtabFetch(cfg, path, { method: "DELETE" }, context));
  }

  // list
  const query = new URLSearchParams();
  if (normalized.tabId) query.set("tabId", normalized.tabId);
  if (normalized.method) query.set("method", normalized.method);
  if (normalized.status) query.set("status", normalized.status);
  if (normalized.resourceType) query.set("type", normalized.resourceType);
  if (normalized.limit) query.set("limit", String(normalized.limit));
  return textResult(await pinchtabFetch(cfg, `/network?${query.toString()}`, {}, context));
}

async function handleDownload(cfg: PluginConfig, normalized: any, context?: PluginRuntimeContext): Promise<any> {
  const policyResult = enforcePolicyOrReturn(checkDownloadPolicy(cfg));
  if (policyResult) return policyResult;

  const query = new URLSearchParams();
  if (normalized.downloadUrl) query.set("url", normalized.downloadUrl);
  if (normalized.tabId) query.set("tabId", normalized.tabId);
  if (normalized.savePath) {
    query.set("output", "file");
    query.set("path", normalized.savePath);
    return textResult(await pinchtabFetch(cfg, `/download?${query.toString()}`, {}, context));
  }
  query.set("raw", "true");
  return fetchBinaryResult(
    cfg,
    `/download?${query.toString()}`,
    "download",
    (b64, res) => resourceResult("download://file", res.headers.get("content-type") || "application/octet-stream", b64),
    context,
  );
}

async function handleUpload(cfg: PluginConfig, normalized: any, context?: PluginRuntimeContext): Promise<any> {
  const policyResult = enforcePolicyOrReturn(checkUploadPolicy(cfg));
  if (policyResult) return policyResult;

  const body: any = {};
  if (normalized.tabId) body.tabId = normalized.tabId;
  if (normalized.selector) body.selector = normalized.selector;
  if (normalized.files) body.files = normalized.files;
  if (normalized.paths) body.paths = normalized.paths;
  return textResult(await pinchtabFetch(cfg, "/upload", { body }, context));
}

async function handleHealth(cfg: PluginConfig, _normalized: any, context?: PluginRuntimeContext): Promise<any> {
  return textResult(await getEnhancedHealth(cfg, context));
}

const actionHandlers: Record<string, ActionHandler> = {
  navigate: handleNavigate,
  snapshot: handleSnapshot,
  ...Object.fromEntries(elementActions.map((a) => [a, handleElementAction])),
  text: handleText,
  wait: handleWait,
  handoff: handleHandoff,
  tabs: handleTabs,
  screenshot: handleScreenshot,
  evaluate: handleEvaluate,
  pdf: handlePdf,
  network: handleNetwork,
  download: handleDownload,
  upload: handleUpload,
  health: handleHealth,
};

export async function executePinchtabAction(rawCfg: PluginConfig, params: any, context?: PluginRuntimeContext): Promise<any> {
  rememberRuntimeContext(context);
  const cfg = await resolveEffectiveConfig(rawCfg);
  const normalized = normalizeActionParams(params);
  const { action } = normalized;

  // Auto-start / readiness — cached so routine actions skip the per-call probe.
  if (action !== "health") {
    const ready = await ensureReady(cfg, context);
    if (!ready.ok) {
      return textResult({ error: ready.error });
    }
  }

  // Session tab persistence
  if (cfg.persistSessionTabs !== false && !normalized.tabId && getLastTabId(context)) {
    normalized.tabId = getLastTabId(context);
  }

  const handler = actionHandlers[action];
  if (!handler) {
    return textResult({ error: `Unknown action: ${action}` });
  }
  return handler(cfg, normalized, context);
}