Skip to content

An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.

19.6 KB
Raw Blame
import { TOOL_GROUPS } from '@/lib/actions/groups';
import type { ToolDescriptor } from '@/lib/actions/manifest';
import { failure, type ActionResult } from '@/lib/actions/protocol';
import { FOCUS_SHOT_ACTION, RESERVED_ACTIONS, SAVE_SITE_MAP_ACTION } from '@/lib/actions/reserved';
import { actionNameFor, assertToolNamesRoundTrip, OPEN_SKILL_TOOL, STATUS_TOOL, toolNameFor, USE_TOOL } from '@/lib/actions/tool-names';
import { loadSkills } from './agent/skills';
import type { Bridge, Described } from './control';
import { IMAGE_NOTE, fence, sealSecrets, shouldFence, type Policy } from './guardrails';
import { GROUP_INDEX, groupGuide, grouped, groupNamed, groupsOf, pageActionNamed } from './tool-groups';

const EXTRACT_ACTION = 'page.extractText';

const RENDERERS: Record<string, (result: ActionResult, fenceWith?: string) => ToolReply> = {
  'page.screenshot': renderScreenshot,
  'page.pickElement': renderPick,
  'page.solveCaptcha': renderCaptcha,
  [FOCUS_SHOT_ACTION]: renderFocusShot,
};

const OPEN_SKILL = {
  name: OPEN_SKILL_TOOL,
  description: `Open the guide to one of these less common capabilities: how to go about it, then its tools with their input schemas.\n${GROUP_INDEX}`,
  inputSchema: {
    type: 'object' as const,
    properties: { name: { type: 'string', enum: Object.keys(TOOL_GROUPS), description: 'The capability whose guide to open.' } },
    required: ['name'],
    additionalProperties: false,
  },
};

const USE = {
  name: USE_TOOL,
  description:
    `Run a browser action that is not in your tool list. A guide from ${OPEN_SKILL_TOOL} gives each one with its input schema, ` +
    'and a skill or a result may name one, as page.solveCaptcha or page_dragElement: call it here, with that name as action and ' +
    'its parameters as input. It runs and answers exactly as a tool of its own would.',
  inputSchema: {
    type: 'object' as const,
    properties: {
      action: { type: 'string', description: 'The action to run, e.g. page_dragElement.' },
      input: { type: 'object', description: 'Its parameters, as its input schema lays them out. Leave it out when there are none.' },
    },
    required: ['action'],
  },
};

const FOCUS_SHOT_TOOL = {
  name: toolNameFor(FOCUS_SHOT_ACTION),
  description:
    'Show the screenshot of the element the user pointed at with A-Eye, taken at the instant they picked it. ' +
    'Only answers when the current instruction arrived with a pick attached.',
  inputSchema: { type: 'object' as const, properties: {}, additionalProperties: false },
};

const SAVE_SITE_MAP_TOOL = {
  name: toolNameFor(SAVE_SITE_MAP_ACTION),
  description:
    'Write up a finished site map. Call this exactly once, at the end of a mapping run. The map is staged for the user to review before it takes effect — it does not apply immediately.',
  inputSchema: {
    type: 'object' as const,
    additionalProperties: false,
    required: ['report'],
    properties: {
      report: {
        type: 'object',
        additionalProperties: false,
        required: ['summary', 'pages'],
        properties: {
          summary: { type: 'string', description: 'What this site is, in two or three sentences.' },
          landmarks: {
            type: 'array',
            description: 'Durable parts of the interface: the primary nav, a search box, a cookie wall.',
            items: {
              type: 'object',
              additionalProperties: false,
              required: ['name'],
              properties: {
                name: { type: 'string', description: 'What it is called, as a person would say it.' },
                selector: { type: 'string', description: 'A CSS selector that finds it.' },
                note: { type: 'string', description: 'One line on how it behaves.' },
              },
            },
          },
          pages: {
            type: 'array',
            description: 'Each page visited, once.',
            items: {
              type: 'object',
              additionalProperties: false,
              required: ['path', 'title', 'purpose'],
              properties: {
                path: { type: 'string', description: 'The path on the mapped site, e.g. /pricing.' },
                title: { type: 'string', description: 'The page title.' },
                purpose: { type: 'string', description: 'What the page is for, in one short phrase.' },
                reachedBy: { type: 'string', description: 'How you got there, e.g. "Pricing" in the top nav.' },
                screenshot: { type: 'string', description: 'Filename of a screenshot you took of this page.' },
                notes: { type: 'string', description: 'Anything else worth recording. Kept out of the prompt.' },
              },
            },
          },
          links: {
            type: 'array',
            description: 'How the pages connect: one entry per link you followed or saw.',
            items: {
              type: 'object',
              additionalProperties: false,
              required: ['from', 'to'],
              properties: {
                from: { type: 'string', description: 'Path the link is on.' },
                to: { type: 'string', description: 'Path it leads to.' },
              },
            },
          },
          quirks: {
            type: 'array',
            description: 'Things that would trip up someone driving this site. Observations, never advice.',
            items: { type: 'string' },
          },
        },
      },
    },
  },
};

const RESERVED_TOOLS = [
  { action: SAVE_SITE_MAP_ACTION, descriptor: SAVE_SITE_MAP_TOOL },
  { action: FOCUS_SHOT_ACTION, descriptor: FOCUS_SHOT_TOOL },
];

export interface ListedTool {
  name: string;
  description: string;
  inputSchema: { type: 'object' } & Record<string, unknown>;
}

export interface ListedGroup {
  name: string;
  description: string;
  tools: ListedTool[];
}

export interface ToolList {
  tools: ListedTool[];
  /** Offered later in the conversation, perhaps — see `Described.withheld`. */
  withheld: ListedTool[];
  /** A namespaced run's grouped tools, for its CLI to load when its own search asks for them. */
  groups: ListedGroup[];
}

export type ToolContent = { type: 'text'; text: string } | { type: 'image'; data: string; mimeType: string };

export interface ToolReply {
  content: ToolContent[];
  isError?: boolean;
}

/**
 * The browser's tools as an agent sees them — listed, called, and their results sealed, fenced and
 * turned into pictures where they hold one. The MCP server serves these to any client; a CLI that
 * takes its tools some other way is handed the same host, so a result reads the same either way.
 */
export interface ToolHost {
  list(): Promise<ToolList>;
  /** `signal` aborts when the client stops waiting, so an approval the call waits on is withdrawn. */
  call(name: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<ToolReply>;
}

interface HostOptions {
  agentRun?: boolean;
  /** The CLI takes deferred namespaces (Codex's app-server). Without them, a namespaced run's grouped tools are listed. */
  namespaces?: boolean;
  /** The most text, in UTF-8 bytes, the CLI on the other end passes to its model; unset for no limit. */
  resultBytes?: number;
  policy: Policy;
  tag: string;
}

export function toolHost(bridge: Bridge, { agentRun = false, namespaces = false, resultBytes, policy, tag }: HostOptions): ToolHost {
  const invoke = (action: string, args: Record<string, unknown>, signal?: AbortSignal) =>
    bridge.invoke(action, resultBytes && action === EXTRACT_ACTION ? fitRead(args, resultBytes) : args, signal);

  const present = (action: string, result: ActionResult) =>
    (RENDERERS[action] ?? render)(result, shouldFence(action, policy) ? tag : undefined);

  const offer = async (): Promise<Described> => {
    const described = await bridge.describe();
    return described.delivery === 'namespaced' && !namespaces ? { ...described, onDemand: [], delivery: 'listed' } : described;
  };

  const openSkill = async ({ name }: Record<string, unknown>): Promise<ToolReply> => {
    const group = typeof name === 'string' ? groupNamed(name) : undefined;
    if (!group) return render(failure('UNKNOWN_SKILL', `There is no guide named "${String(name ?? '')}". The guides are:\n${GROUP_INDEX}`));
    const guide = groupGuide(group, await offer(), loadSkills());
    return guide.ok ? { content: [{ type: 'text', text: guide.data }] } : render(guide);
  };

  const withSchema = async (action: string, result: ActionResult): Promise<ActionResult> => {
    if (result.ok || result.error.code !== 'INVALID_INPUT') return result;
    const schema = (await bridge.describe()).tools.find((tool) => tool.name === action)?.inputSchema;
    return schema ? failure('INVALID_INPUT', `${result.error.message}\n${toolNameFor(action)} takes: ${JSON.stringify(schema)}`) : result;
  };

  const use = async ({ action: named, input, ...flat }: Record<string, unknown>, signal?: AbortSignal): Promise<ToolReply> => {
    const action = typeof named === 'string' ? pageActionNamed(named) : undefined;
    if (!action) {
      return render(
        failure(
          'UNKNOWN_ACTION',
          `There is no browser action named "${String(named ?? '')}". Open the guide that covers what you need with ${OPEN_SKILL_TOOL}:\n${GROUP_INDEX}`,
        ),
      );
    }
    const args = inputFrom(input ?? flat);
    if (!args) return render(failure('INVALID_INPUT', `input must be an object, or a JSON string of one, holding ${toolNameFor(action)}'s parameters.`));
    return present(action, await withSchema(action, await invoke(action, args, signal)));
  };

  const answer = async (name: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<ToolReply> => {
    if (name === STATUS_TOOL) return render(await status(bridge));
    if (name === OPEN_SKILL_TOOL) return openSkill(args);
    if (name === USE_TOOL) return use(args, signal);
    const action = actionNameFor(name);
    return present(action, await invoke(action, args, signal));
  };

  return {
    async list() {
      const {
        tools: actions,
        reserved = agentRun ? RESERVED_TOOLS.map((tool) => tool.action) : [],
        withheld = [],
        onDemand = [],
        delivery,
      } = await offer();
      assertToolNamesRoundTrip([...actions.map((action) => action.name), ...RESERVED_ACTIONS]);
      const namespaced = delivery === 'namespaced';
      const inItsGroup = (tool: ToolDescriptor) => namespaced && grouped(tool.name);
      return {
        tools: [
          ...actions.filter((action) => !onDemand.includes(action.name)).map(listed),
          {
            name: STATUS_TOOL,
            description:
              'Report whether the Browsentic browser extension is connected, its version, and the active tab. Use this first if a page tool fails.',
            inputSchema: { type: 'object' as const, properties: {}, additionalProperties: false },
          },
          OPEN_SKILL,
          ...(onDemand.length && !namespaced ? [USE] : []),
          ...RESERVED_TOOLS.filter((tool) => reserved.includes(tool.action)).map((tool) => tool.descriptor),
        ],
        withheld: withheld.filter((tool) => !inItsGroup(tool)).map(listed),
        groups: namespaced
          ? groupsOf([...actions.filter((action) => onDemand.includes(action.name)), ...withheld.filter(inItsGroup)]).map((group) => ({
              ...group,
              tools: group.tools.map(listed),
            }))
          : [],
      };
    },

    async call(name, args, signal) {
      const reply = await answer(name, args, signal);
      return resultBytes ? withinLimit(reply, resultBytes) : reply;
    },
  };
}

/**
 * A read sized so its group fits the limit whatever the text: four bytes a character covers any
 * script and the JSON it is escaped into. The rest comes in the next group, by its cursor.
 */
function fitRead(args: Record<string, unknown>, resultBytes: number): Record<string, unknown> {
  const most = Math.floor(resultBytes / 4);
  const asked = typeof args.maxLength === 'number' ? args.maxLength : Infinity;
  return { ...args, maxLength: Math.min(asked, most) };
}

/**
 * The reply, or a refusal when its text is longer than the CLI passes on. Counted as Cursor counts
 * it — its text blocks joined by a blank line, pictures aside — and refused whole rather than cut,
 * because a cut body is not JSON any more and loses the fence's closing line.
 */
function withinLimit(reply: ToolReply, resultBytes: number): ToolReply {
  const text = reply.content.flatMap((block) => (block.type === 'text' ? [block.text] : [])).join('\n\n');
  const bytes = Buffer.byteLength(text, 'utf8');
  if (bytes <= resultBytes) return reply;
  return {
    isError: true,
    content: [
      {
        type: 'text',
        text:
          `RESULT_TOO_LARGE: The call went through, but its result came to ${bytes.toLocaleString('en')} bytes, and the agent CLI ` +
          `running you keeps anything over ${resultBytes.toLocaleString('en')} from you, so it was not sent. Do not repeat an action ` +
          'for its result; ask for less at a time instead — page_extractText with a smaller maxLength and the cursor it hands back, ' +
          'page_getPageInfo with a smaller maxPerKind, or a target that narrows the read to one element.',
      },
    ],
  };
}

/** An action's input as a model sends it: an object, or that object as a JSON string, which some CLIs pass instead. */
function inputFrom(input: unknown): Record<string, unknown> | undefined {
  const value = typeof input === 'string' ? parseJson(input) : input;
  return value && typeof value === 'object' && !Array.isArray(value) ? (value as Record<string, unknown>) : undefined;
}

function parseJson(text: string): unknown {
  if (!text.trim()) return {};
  try {
    return JSON.parse(text);
  } catch {
    return undefined;
  }
}

const listed = (action: ToolDescriptor): ListedTool => ({
  name: toolNameFor(action.name),
  description: action.description,
  inputSchema: action.inputSchema as ListedTool['inputSchema'],
});

interface PageInfo {
  document: { url: string; title: string };
}

async function status(bridge: Bridge): Promise<ActionResult> {
  const base = await bridge.status();
  if (!base.connected) {
    return { ok: true, data: { ...base, activeTab: null, hint: 'Open your browser with the Browsentic extension loaded.' } };
  }
  const monitors = await activeMonitors(bridge);
  const page = await bridge.invoke('page.getPageInfo', { maxPerKind: 1 });
  const hints = [
    base.manifestInSync
      ? ''
      : 'The extension and Browsentic Bridge are different versions, which is normal while one of them waits on an update. Your tool list came from the extension, which is what runs the tools, so it is the right one. Use the tools you were given and do not improvise around one you think should exist.',
    page.ok ? '' : `Cannot read the active tab (${page.error.code}). Use page_navigate to open an http(s) page first.`,
  ].filter(Boolean);

  return {
    ok: true,
    data: {
      ...base,
      activeTab: page.ok ? (page.data as PageInfo).document : null,
      ...monitors,
      ...(hints.length ? { hint: hints.join(' ') } : {}),
    },
  };
}

interface MonitorSummary {
  monitorId: string;
  label?: string;
  host: string;
  phase: string;
  percent?: number;
}

async function activeMonitors(bridge: Bridge): Promise<{ monitors: MonitorSummary[] } | undefined> {
  const result = await bridge.invoke('page.monitorStatus', {});
  if (!result.ok) return undefined;
  const monitors = (result.data as { monitors?: MonitorSummary[] } | null)?.monitors;
  if (!monitors?.length) return undefined;
  return { monitors: monitors.map(({ monitorId, label, host, phase, percent }) => ({ monitorId, label, host, phase, percent })) };
}

function renderScreenshot(result: ActionResult): ToolReply {
  if (!result.ok) return render(result);
  const data = result.data as {
    dataUrl?: string;
    format?: string;
    width?: number;
    height?: number;
    savedTo?: string;
    saveError?: string;
    truncated?: boolean;
  };
  if (typeof data.dataUrl !== 'string') return render(result);

  const [mimeType, base64] = splitDataUrl(data.dataUrl);
  const notes = [
    IMAGE_NOTE,
    `Captured ${data.width}×${data.height} ${data.format ?? 'image'}.`,
    data.truncated ? 'The page was taller than the capture limit, so the bottom is cut off.' : '',
    data.savedTo
      ? `Saved to ${data.savedTo}. Show this screenshot to the user: read that path so the image renders, and include the path in your reply.`
      : '',
    data.saveError ? `Requested save failed: ${data.saveError}.` : '',
  ]
    .filter(Boolean)
    .join(' ');

  return {
    content: [
      { type: 'image', data: base64, mimeType },
      { type: 'text', text: sealSecrets(notes) },
    ],
  };
}

function renderPick(result: ActionResult, fenceWith?: string): ToolReply {
  if (!result.ok) return render(result, fenceWith);
  const { shot, ...rest } = result.data as { shot?: { dataUrl?: string; width?: number; height?: number } } & Record<
    string,
    unknown
  >;
  if (typeof shot?.dataUrl !== 'string') return render(result, fenceWith);

  const [mimeType, base64] = splitDataUrl(shot.dataUrl);
  const rendered = render({ ok: true, data: rest }, fenceWith);
  return {
    content: [
      ...rendered.content,
      { type: 'image', data: base64, mimeType },
      { type: 'text', text: `${IMAGE_NOTE} This is the picked element photographed at the instant the user clicked it.` },
    ],
  };
}

/** An open challenge comes back as a picture beside its description, so the caller can answer it by looking. */
function renderCaptcha(result: ActionResult, fenceWith?: string): ToolReply {
  if (!result.ok) return render(result, fenceWith);
  const data = result.data as { challenge?: { image?: unknown } & Record<string, unknown> } & Record<string, unknown>;
  const image = data.challenge?.image;
  if (typeof image !== 'string') return render(result, fenceWith);

  const { image: _image, ...challenge } = data.challenge!;
  const [mimeType, base64] = splitDataUrl(image);
  const rendered = render({ ok: true, data: { ...data, challenge } }, fenceWith);
  return {
    content: [
      ...rendered.content,
      { type: 'image', data: base64, mimeType },
      {
        type: 'text',
        text: `${IMAGE_NOTE} This is the open captcha challenge, ${challenge.imageWidth}×${challenge.imageHeight} pixels${
          challenge.kind === 'tiles' ? ', each tile numbered in its corner' : ''
        }.`,
      },
    ],
  };
}

function renderFocusShot(result: ActionResult): ToolReply {
  if (!result.ok) return render(result);
  const dataUrl = (result.data as { dataUrl?: unknown } | null)?.dataUrl;
  if (typeof dataUrl !== 'string') return render(result);

  const [mimeType, base64] = splitDataUrl(dataUrl);
  return {
    content: [
      { type: 'image', data: base64, mimeType },
      { type: 'text', text: `${IMAGE_NOTE} This is the element the user pointed at with A-Eye, as it stood when they picked it.` },
    ],
  };
}

function splitDataUrl(dataUrl: string): [mimeType: string, base64: string] {
  const match = /^data:([^;,]+);base64,(.*)$/s.exec(dataUrl);
  return match ? [match[1], match[2]] : ['image/png', ''];
}

/**
 * `fenceWith` marks the payload as untrusted page data. Failures are left bare: they
 * are daemon-authored and the run preamble teaches the agent to read `CODE: message`.
 *
 * The seal runs on every body, fenced or not. The extension has normally sealed already
 * and this pass leaves its handles alone; what it catches is a result that reached the
 * daemon another way.
 */
function render(result: ActionResult, fenceWith?: string): ToolReply {
  if (result.ok) {
    const body = sealSecrets(JSON.stringify(result.data));
    return { content: [{ type: 'text', text: fenceWith ? fence(body, fenceWith) : body }] };
  }
  return { isError: true, content: [{ type: 'text', text: sealSecrets(`${result.error.code}: ${result.error.message}`) }] };
}