Skip to content

An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.

refactor(daemon): serve the browser tools from one host the MCP server wraps, so a CLI that takes them another way gets the same results, and tell a run's caller which tools it is withholding for now

Shahriare8e6ac893170
diff --git a/src/daemon/control.ts b/src/daemon/control.ts
index 38f4824..307975a 100644
--- a/src/daemon/control.ts
+++ b/src/daemon/control.ts
@@ -43,7 +43,7 @@ export type ControlRequest =
| { id: string; op: 'agent'; set?: AgentKind; grant?: AgentKind; models?: AgentKind };
export type ControlMessage =
- | { id: string; op: 'describe'; tools: ToolDescriptor[]; reserved?: string[] }
+ | { id: string; op: 'describe'; tools: ToolDescriptor[]; reserved?: string[]; withheld?: ToolDescriptor[] }
| { id: string; op: 'status'; status: BridgeStatus }
| { id: string; op: 'invoke'; result: ActionResult }
| { id: string; op: 'working' }
@@ -57,6 +57,12 @@ export interface Described {
tools: ToolDescriptor[];
/** Reserved actions this caller may be offered. Absent when the daemon predates the field. */
reserved?: string[];
+ /**
+ * Tools a run is not offered right now but may be later in its conversation — the page-code tools,
+ * until the user turns Live tool on. A CLI whose tool list is fixed when its session begins lists
+ * them out of sight up front; a call to one while it is withheld is refused.
+ */
+ withheld?: ToolDescriptor[];
}
export interface Bridge {
diff --git a/src/daemon/daemon.ts b/src/daemon/daemon.ts
index b5b28b0..e7d1529 100644
--- a/src/daemon/daemon.ts
+++ b/src/daemon/daemon.ts
@@ -679,7 +679,12 @@ export async function startDaemon({ version, idleExit = true }: DaemonOptions):
const running = runId ? [...agents].find(([, agent]) => agent.owns(runId)) : undefined;
const offer = running && runId ? running[1].offerFor(runId) : null;
if (offer && running) {
- return { tools: running[0].tools.filter(({ name }) => !offer.withheld.includes(name)), reserved: offer.reserved };
+ const withheld = (name: string) => offer.withheld.includes(name);
+ return {
+ tools: running[0].tools.filter(({ name }) => !withheld(name)),
+ reserved: offer.reserved,
+ withheld: running[0].tools.filter(({ name }) => withheld(name)),
+ };
}
const tools = routeFor(binding)?.tools ?? bundled;
diff --git a/src/daemon/remote-bridge.ts b/src/daemon/remote-bridge.ts
index f55403f..4bca252 100644
--- a/src/daemon/remote-bridge.ts
+++ b/src/daemon/remote-bridge.ts
@@ -34,7 +34,7 @@ export class RemoteBridge implements Bridge {
async describe(): Promise<Described> {
const reply = await this.request({ id: randomUUID(), op: 'describe', runId: this.runId });
- return reply && 'tools' in reply ? { tools: reply.tools, reserved: reply.reserved } : { tools: [] };
+ return reply && 'tools' in reply ? { tools: reply.tools, reserved: reply.reserved, withheld: reply.withheld } : { tools: [] };
}
async invoke(action: string, input?: unknown): Promise<ActionResult> {
diff --git a/src/daemon/server.ts b/src/daemon/server.ts
index cbc9927..99ab22f 100644
--- a/src/daemon/server.ts
+++ b/src/daemon/server.ts
@@ -6,24 +6,11 @@ import {
ReadResourceRequestSchema,
} from '@modelcontextprotocol/sdk/types.js';
import type { ActionResult } from '@/lib/actions/protocol';
-import { FOCUS_SHOT_ACTION, RESERVED_ACTIONS, SAVE_SITE_MAP_ACTION } from '@/lib/actions/reserved';
-import { actionNameFor, assertToolNamesRoundTrip, STATUS_TOOL, toolNameFor } from '@/lib/actions/tool-names';
import { readAgentConfig } from './agent/config';
import type { Bridge } from './control';
-import { IMAGE_NOTE, fence, fenceTag, policyFrom, sealSecrets, shouldFence } from './guardrails';
+import { fence, fenceTag, policyFrom, sealSecrets } from './guardrails';
import { log } from './log';
-
-const SCREENSHOT_TOOL = 'page_screenshot';
-const PICK_TOOL = 'page_pickElement';
-const CAPTCHA_TOOL = 'page_solveCaptcha';
-
-const FOCUS_SHOT_TOOL = {
- name: toolNameFor(FOCUS_SHOT_ACTION),
- description:
- 'Show the screenshot of the element the user pointed at with A-Eye, taken at the instant they picked it. ' +
- 'Only answers when the current instruction arrived with a pick attached.',
- inputSchema: { type: 'object' as const, properties: {}, additionalProperties: false },
-};
+import { toolHost } from './tool-host';
const RESOURCES = [
{
@@ -46,81 +33,6 @@ const RESOURCES = [
},
] as const;
-const SAVE_SITE_MAP_TOOL = {
- name: toolNameFor(SAVE_SITE_MAP_ACTION),
- description:
- 'Write up a finished site map. Call this exactly once, at the end of a mapping run. The map is staged for the user to review before it takes effect — it does not apply immediately.',
- inputSchema: {
- type: 'object' as const,
- additionalProperties: false,
- required: ['report'],
- properties: {
- report: {
- type: 'object',
- additionalProperties: false,
- required: ['summary', 'pages'],
- properties: {
- summary: { type: 'string', description: 'What this site is, in two or three sentences.' },
- landmarks: {
- type: 'array',
- description: 'Durable parts of the interface: the primary nav, a search box, a cookie wall.',
- items: {
- type: 'object',
- additionalProperties: false,
- required: ['name'],
- properties: {
- name: { type: 'string', description: 'What it is called, as a person would say it.' },
- selector: { type: 'string', description: 'A CSS selector that finds it.' },
- note: { type: 'string', description: 'One line on how it behaves.' },
- },
- },
- },
- pages: {
- type: 'array',
- description: 'Each page visited, once.',
- items: {
- type: 'object',
- additionalProperties: false,
- required: ['path', 'title', 'purpose'],
- properties: {
- path: { type: 'string', description: 'The path on the mapped site, e.g. /pricing.' },
- title: { type: 'string', description: 'The page title.' },
- purpose: { type: 'string', description: 'What the page is for, in one short phrase.' },
- reachedBy: { type: 'string', description: 'How you got there, e.g. "Pricing" in the top nav.' },
- screenshot: { type: 'string', description: 'Filename of a screenshot you took of this page.' },
- notes: { type: 'string', description: 'Anything else worth recording. Kept out of the prompt.' },
- },
- },
- },
- links: {
- type: 'array',
- description: 'How the pages connect: one entry per link you followed or saw.',
- items: {
- type: 'object',
- additionalProperties: false,
- required: ['from', 'to'],
- properties: {
- from: { type: 'string', description: 'Path the link is on.' },
- to: { type: 'string', description: 'Path it leads to.' },
- },
- },
- },
- quirks: {
- type: 'array',
- description: 'Things that would trip up someone driving this site. Observations, never advice.',
- items: { type: 'string' },
- },
- },
- },
- },
- },
-};
-
-const RESERVED_TOOLS = [
- { action: SAVE_SITE_MAP_ACTION, descriptor: SAVE_SITE_MAP_TOOL },
- { action: FOCUS_SHOT_ACTION, descriptor: FOCUS_SHOT_TOOL },
-];
-
export function createMcpServer(bridge: Bridge, version: string, opts: { agentRun?: boolean } = {}): Server {
// Page text is marked as data on the way out, for every client — the system prompt
// that says so only reaches Browsentic's own runs. The tag is per-process so a page
@@ -146,38 +58,12 @@ export function createMcpServer(bridge: Bridge, version: string, opts: { agentRu
},
);
- server.setRequestHandler(ListToolsRequestSchema, async () => {
- const { tools: actions, reserved = opts.agentRun ? RESERVED_TOOLS.map((tool) => tool.action) : [] } =
- await bridge.describe();
- assertToolNamesRoundTrip([...actions.map((action) => action.name), ...RESERVED_ACTIONS]);
- return {
- tools: [
- ...actions.map((action) => ({
- name: toolNameFor(action.name),
- description: action.description,
- inputSchema: action.inputSchema as { type: 'object' },
- })),
- {
- name: STATUS_TOOL,
- description:
- 'Report whether the Browsentic browser extension is connected, its version, and the active tab. Use this first if a page tool fails.',
- inputSchema: { type: 'object' as const, properties: {}, additionalProperties: false },
- },
- ...RESERVED_TOOLS.filter((tool) => reserved.includes(tool.action)).map((tool) => tool.descriptor),
- ],
- };
- });
+ const tools = toolHost(bridge, { agentRun: opts.agentRun, policy, tag });
- server.setRequestHandler(CallToolRequestSchema, async ({ params }) => {
- if (params.name === STATUS_TOOL) return render(await status(bridge));
- const action = actionNameFor(params.name);
- const result = await bridge.invoke(action, params.arguments ?? {});
- if (params.name === SCREENSHOT_TOOL) return renderScreenshot(result);
- if (params.name === FOCUS_SHOT_TOOL.name) return renderFocusShot(result);
- if (params.name === PICK_TOOL) return renderPick(result, shouldFence(action, policy) ? tag : undefined);
- if (params.name === CAPTCHA_TOOL) return renderCaptcha(result, shouldFence(action, policy) ? tag : undefined);
- return render(result, shouldFence(action, policy) ? tag : undefined);
- });
+ // A client listed again sees whatever is withheld once it is offered, so there is nothing to list ahead.
+ server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: (await tools.list()).tools }));
+
+ server.setRequestHandler(CallToolRequestSchema, async ({ params }) => ({ ...(await tools.call(params.name, params.arguments ?? {})) }));
server.setRequestHandler(ListResourcesRequestSchema, async () => ({ resources: [...RESOURCES] }));
@@ -217,171 +103,6 @@ interface PageInfo {
layout: { diagram: string };
}
-async function status(bridge: Bridge) {
- const base = await bridge.status();
- if (!base.connected) {
- return {
- ok: true as const,
- data: { ...base, activeTab: null, hint: 'Open your browser with the Browsentic extension loaded.' },
- };
- }
- const monitors = await activeMonitors(bridge);
- const page = await bridge.invoke('page.getPageInfo', { maxPerKind: 1 });
- const hints = [
- base.manifestInSync
- ? ''
- : 'The extension is running an older build than the daemon, so your tool list came from the extension and is stale — capabilities that exist in this repository may be missing entirely. You cannot fix this yourself: tell the user to run `yarn build && yarn daemon:build` and then press Reload on Browsentic at chrome://extensions. Do not improvise around a tool you think should exist.',
- page.ok ? '' : `Cannot read the active tab (${page.error.code}). Use page_navigate to open an http(s) page first.`,
- ].filter(Boolean);
-
- return {
- ok: true as const,
- data: {
- ...base,
- activeTab: page.ok ? (page.data as PageInfo).document : null,
- ...monitors,
- ...(hints.length ? { hint: hints.join(' ') } : {}),
- },
- };
-}
-
-interface MonitorSummary {
- monitorId: string;
- label?: string;
- host: string;
- phase: string;
- percent?: number;
-}
-
-async function activeMonitors(bridge: Bridge): Promise<{ monitors: MonitorSummary[] } | undefined> {
- const result = await bridge.invoke('page.monitorStatus', {});
- if (!result.ok) return undefined;
- const monitors = (result.data as { monitors?: MonitorSummary[] } | null)?.monitors;
- if (!monitors?.length) return undefined;
- return { monitors: monitors.map(({ monitorId, label, host, phase, percent }) => ({ monitorId, label, host, phase, percent })) };
-}
-
-function renderScreenshot(result: ActionResult) {
- if (!result.ok) return render(result);
- const data = result.data as {
- dataUrl?: string;
- format?: string;
- width?: number;
- height?: number;
- savedTo?: string;
- saveError?: string;
- truncated?: boolean;
- };
- if (typeof data.dataUrl !== 'string') return render(result);
-
- const [mimeType, base64] = splitDataUrl(data.dataUrl);
- const notes = [
- IMAGE_NOTE,
- `Captured ${data.width}×${data.height} ${data.format ?? 'image'}.`,
- data.truncated ? 'The page was taller than the capture limit, so the bottom is cut off.' : '',
- data.savedTo
- ? `Saved to ${data.savedTo}. Show this screenshot to the user: read that path so the image renders, and include the path in your reply.`
- : '',
- data.saveError ? `Requested save failed: ${data.saveError}.` : '',
- ]
- .filter(Boolean)
- .join(' ');
-
- return {
- content: [
- { type: 'image' as const, data: base64, mimeType },
- { type: 'text' as const, text: sealSecrets(notes) },
- ],
- };
-}
-
-function renderPick(result: ActionResult, fenceWith?: string) {
- if (!result.ok) return render(result, fenceWith);
- const { shot, ...rest } = result.data as { shot?: { dataUrl?: string; width?: number; height?: number } } & Record<
- string,
- unknown
- >;
- if (typeof shot?.dataUrl !== 'string') return render(result, fenceWith);
-
- const [mimeType, base64] = splitDataUrl(shot.dataUrl);
- const rendered = render({ ok: true, data: rest }, fenceWith);
- return {
- content: [
- ...rendered.content,
- { type: 'image' as const, data: base64, mimeType },
- {
- type: 'text' as const,
- text: `${IMAGE_NOTE} This is the picked element photographed at the instant the user clicked it.`,
- },
- ],
- };
-}
-
-/** An open challenge comes back as a picture beside its description, so the caller can answer it by looking. */
-function renderCaptcha(result: ActionResult, fenceWith?: string) {
- if (!result.ok) return render(result, fenceWith);
- const data = result.data as { challenge?: { image?: unknown } & Record<string, unknown> } & Record<string, unknown>;
- const image = data.challenge?.image;
- if (typeof image !== 'string') return render(result, fenceWith);
-
- const { image: _image, ...challenge } = data.challenge!;
- const [mimeType, base64] = splitDataUrl(image);
- const rendered = render({ ok: true, data: { ...data, challenge } }, fenceWith);
- return {
- content: [
- ...rendered.content,
- { type: 'image' as const, data: base64, mimeType },
- {
- type: 'text' as const,
- text: `${IMAGE_NOTE} This is the open captcha challenge, ${challenge.imageWidth}×${challenge.imageHeight} pixels${
- challenge.kind === 'tiles' ? ', each tile numbered in its corner' : ''
- }.`,
- },
- ],
- };
-}
-
-function renderFocusShot(result: ActionResult) {
- if (!result.ok) return render(result);
- const dataUrl = (result.data as { dataUrl?: unknown } | null)?.dataUrl;
- if (typeof dataUrl !== 'string') return render(result);
-
- const [mimeType, base64] = splitDataUrl(dataUrl);
- return {
- content: [
- { type: 'image' as const, data: base64, mimeType },
- {
- type: 'text' as const,
- text: `${IMAGE_NOTE} This is the element the user pointed at with A-Eye, as it stood when they picked it.`,
- },
- ],
- };
-}
-
-function splitDataUrl(dataUrl: string): [mimeType: string, base64: string] {
- const match = /^data:([^;,]+);base64,(.*)$/s.exec(dataUrl);
- return match ? [match[1], match[2]] : ['image/png', ''];
-}
-
-/**
- * `fenceWith` marks the payload as untrusted page data. Failures are left bare: they
- * are daemon-authored and the run preamble teaches the agent to read `CODE: message`.
- *
- * The seal runs on every body, fenced or not. The extension has normally sealed already
- * and this pass leaves its handles alone; what it catches is a result that reached the
- * daemon another way.
- */
-function render(result: ActionResult, fenceWith?: string) {
- if (result.ok) {
- const body = sealSecrets(JSON.stringify(result.data));
- return { content: [{ type: 'text' as const, text: fenceWith ? fence(body, fenceWith) : body }] };
- }
- return {
- isError: true,
- content: [{ type: 'text' as const, text: sealSecrets(`${result.error.code}: ${result.error.message}`) }],
- };
-}
-
function unwrap(result: ActionResult, project: (data: unknown) => string): string {
if (!result.ok) throw new Error(`${result.error.code}: ${result.error.message}`);
return project(result.data);
diff --git a/src/daemon/tool-host.ts b/src/daemon/tool-host.ts
new file mode 100644
index 0000000..cd1f5f6
--- /dev/null
+++ b/src/daemon/tool-host.ts
@@ -0,0 +1,319 @@
+import type { ToolDescriptor } from '@/lib/actions/manifest';
+import type { ActionResult } from '@/lib/actions/protocol';
+import { FOCUS_SHOT_ACTION, RESERVED_ACTIONS, SAVE_SITE_MAP_ACTION } from '@/lib/actions/reserved';
+import { actionNameFor, assertToolNamesRoundTrip, STATUS_TOOL, toolNameFor } from '@/lib/actions/tool-names';
+import type { Bridge } from './control';
+import { IMAGE_NOTE, fence, sealSecrets, shouldFence, type Policy } from './guardrails';
+
+const SCREENSHOT_TOOL = 'page_screenshot';
+const PICK_TOOL = 'page_pickElement';
+const CAPTCHA_TOOL = 'page_solveCaptcha';
+
+const FOCUS_SHOT_TOOL = {
+ name: toolNameFor(FOCUS_SHOT_ACTION),
+ description:
+ 'Show the screenshot of the element the user pointed at with A-Eye, taken at the instant they picked it. ' +
+ 'Only answers when the current instruction arrived with a pick attached.',
+ inputSchema: { type: 'object' as const, properties: {}, additionalProperties: false },
+};
+
+const SAVE_SITE_MAP_TOOL = {
+ name: toolNameFor(SAVE_SITE_MAP_ACTION),
+ description:
+ 'Write up a finished site map. Call this exactly once, at the end of a mapping run. The map is staged for the user to review before it takes effect — it does not apply immediately.',
+ inputSchema: {
+ type: 'object' as const,
+ additionalProperties: false,
+ required: ['report'],
+ properties: {
+ report: {
+ type: 'object',
+ additionalProperties: false,
+ required: ['summary', 'pages'],
+ properties: {
+ summary: { type: 'string', description: 'What this site is, in two or three sentences.' },
+ landmarks: {
+ type: 'array',
+ description: 'Durable parts of the interface: the primary nav, a search box, a cookie wall.',
+ items: {
+ type: 'object',
+ additionalProperties: false,
+ required: ['name'],
+ properties: {
+ name: { type: 'string', description: 'What it is called, as a person would say it.' },
+ selector: { type: 'string', description: 'A CSS selector that finds it.' },
+ note: { type: 'string', description: 'One line on how it behaves.' },
+ },
+ },
+ },
+ pages: {
+ type: 'array',
+ description: 'Each page visited, once.',
+ items: {
+ type: 'object',
+ additionalProperties: false,
+ required: ['path', 'title', 'purpose'],
+ properties: {
+ path: { type: 'string', description: 'The path on the mapped site, e.g. /pricing.' },
+ title: { type: 'string', description: 'The page title.' },
+ purpose: { type: 'string', description: 'What the page is for, in one short phrase.' },
+ reachedBy: { type: 'string', description: 'How you got there, e.g. "Pricing" in the top nav.' },
+ screenshot: { type: 'string', description: 'Filename of a screenshot you took of this page.' },
+ notes: { type: 'string', description: 'Anything else worth recording. Kept out of the prompt.' },
+ },
+ },
+ },
+ links: {
+ type: 'array',
+ description: 'How the pages connect: one entry per link you followed or saw.',
+ items: {
+ type: 'object',
+ additionalProperties: false,
+ required: ['from', 'to'],
+ properties: {
+ from: { type: 'string', description: 'Path the link is on.' },
+ to: { type: 'string', description: 'Path it leads to.' },
+ },
+ },
+ },
+ quirks: {
+ type: 'array',
+ description: 'Things that would trip up someone driving this site. Observations, never advice.',
+ items: { type: 'string' },
+ },
+ },
+ },
+ },
+ },
+};
+
+const RESERVED_TOOLS = [
+ { action: SAVE_SITE_MAP_ACTION, descriptor: SAVE_SITE_MAP_TOOL },
+ { action: FOCUS_SHOT_ACTION, descriptor: FOCUS_SHOT_TOOL },
+];
+
+export interface ListedTool {
+ name: string;
+ description: string;
+ inputSchema: { type: 'object' } & Record<string, unknown>;
+}
+
+export interface ToolList {
+ tools: ListedTool[];
+ /** Offered later in the conversation, perhaps — see `Described.withheld`. */
+ withheld: ListedTool[];
+}
+
+export type ToolContent = { type: 'text'; text: string } | { type: 'image'; data: string; mimeType: string };
+
+export interface ToolReply {
+ content: ToolContent[];
+ isError?: boolean;
+}
+
+/**
+ * The browser's tools as an agent sees them — listed, called, and their results sealed, fenced and
+ * turned into pictures where they hold one. The MCP server serves these to any client; a CLI that
+ * takes its tools some other way is handed the same host, so a result reads the same either way.
+ */
+export interface ToolHost {
+ list(): Promise<ToolList>;
+ call(name: string, args: Record<string, unknown>): Promise<ToolReply>;
+}
+
+export function toolHost(bridge: Bridge, { agentRun = false, policy, tag }: { agentRun?: boolean; policy: Policy; tag: string }): ToolHost {
+ return {
+ async list() {
+ const { tools: actions, reserved = agentRun ? RESERVED_TOOLS.map((tool) => tool.action) : [], withheld = [] } = await bridge.describe();
+ assertToolNamesRoundTrip([...actions.map((action) => action.name), ...RESERVED_ACTIONS]);
+ return {
+ tools: [
+ ...actions.map(listed),
+ {
+ name: STATUS_TOOL,
+ description:
+ 'Report whether the Browsentic browser extension is connected, its version, and the active tab. Use this first if a page tool fails.',
+ inputSchema: { type: 'object' as const, properties: {}, additionalProperties: false },
+ },
+ ...RESERVED_TOOLS.filter((tool) => reserved.includes(tool.action)).map((tool) => tool.descriptor),
+ ],
+ withheld: withheld.map(listed),
+ };
+ },
+
+ async call(name, args) {
+ if (name === STATUS_TOOL) return render(await status(bridge));
+ const action = actionNameFor(name);
+ const result = await bridge.invoke(action, args);
+ const fenceWith = shouldFence(action, policy) ? tag : undefined;
+ if (name === SCREENSHOT_TOOL) return renderScreenshot(result);
+ if (name === FOCUS_SHOT_TOOL.name) return renderFocusShot(result);
+ if (name === PICK_TOOL) return renderPick(result, fenceWith);
+ if (name === CAPTCHA_TOOL) return renderCaptcha(result, fenceWith);
+ return render(result, fenceWith);
+ },
+ };
+}
+
+const listed = (action: ToolDescriptor): ListedTool => ({
+ name: toolNameFor(action.name),
+ description: action.description,
+ inputSchema: action.inputSchema as ListedTool['inputSchema'],
+});
+
+interface PageInfo {
+ document: { url: string; title: string };
+}
+
+async function status(bridge: Bridge): Promise<ActionResult> {
+ const base = await bridge.status();
+ if (!base.connected) {
+ return { ok: true, data: { ...base, activeTab: null, hint: 'Open your browser with the Browsentic extension loaded.' } };
+ }
+ const monitors = await activeMonitors(bridge);
+ const page = await bridge.invoke('page.getPageInfo', { maxPerKind: 1 });
+ const hints = [
+ base.manifestInSync
+ ? ''
+ : 'The extension is running an older build than the daemon, so your tool list came from the extension and is stale — capabilities that exist in this repository may be missing entirely. You cannot fix this yourself: tell the user to run `yarn build && yarn daemon:build` and then press Reload on Browsentic at chrome://extensions. Do not improvise around a tool you think should exist.',
+ page.ok ? '' : `Cannot read the active tab (${page.error.code}). Use page_navigate to open an http(s) page first.`,
+ ].filter(Boolean);
+
+ return {
+ ok: true,
+ data: {
+ ...base,
+ activeTab: page.ok ? (page.data as PageInfo).document : null,
+ ...monitors,
+ ...(hints.length ? { hint: hints.join(' ') } : {}),
+ },
+ };
+}
+
+interface MonitorSummary {
+ monitorId: string;
+ label?: string;
+ host: string;
+ phase: string;
+ percent?: number;
+}
+
+async function activeMonitors(bridge: Bridge): Promise<{ monitors: MonitorSummary[] } | undefined> {
+ const result = await bridge.invoke('page.monitorStatus', {});
+ if (!result.ok) return undefined;
+ const monitors = (result.data as { monitors?: MonitorSummary[] } | null)?.monitors;
+ if (!monitors?.length) return undefined;
+ return { monitors: monitors.map(({ monitorId, label, host, phase, percent }) => ({ monitorId, label, host, phase, percent })) };
+}
+
+function renderScreenshot(result: ActionResult): ToolReply {
+ if (!result.ok) return render(result);
+ const data = result.data as {
+ dataUrl?: string;
+ format?: string;
+ width?: number;
+ height?: number;
+ savedTo?: string;
+ saveError?: string;
+ truncated?: boolean;
+ };
+ if (typeof data.dataUrl !== 'string') return render(result);
+
+ const [mimeType, base64] = splitDataUrl(data.dataUrl);
+ const notes = [
+ IMAGE_NOTE,
+ `Captured ${data.width}×${data.height} ${data.format ?? 'image'}.`,
+ data.truncated ? 'The page was taller than the capture limit, so the bottom is cut off.' : '',
+ data.savedTo
+ ? `Saved to ${data.savedTo}. Show this screenshot to the user: read that path so the image renders, and include the path in your reply.`
+ : '',
+ data.saveError ? `Requested save failed: ${data.saveError}.` : '',
+ ]
+ .filter(Boolean)
+ .join(' ');
+
+ return {
+ content: [
+ { type: 'image', data: base64, mimeType },
+ { type: 'text', text: sealSecrets(notes) },
+ ],
+ };
+}
+
+function renderPick(result: ActionResult, fenceWith?: string): ToolReply {
+ if (!result.ok) return render(result, fenceWith);
+ const { shot, ...rest } = result.data as { shot?: { dataUrl?: string; width?: number; height?: number } } & Record<
+ string,
+ unknown
+ >;
+ if (typeof shot?.dataUrl !== 'string') return render(result, fenceWith);
+
+ const [mimeType, base64] = splitDataUrl(shot.dataUrl);
+ const rendered = render({ ok: true, data: rest }, fenceWith);
+ return {
+ content: [
+ ...rendered.content,
+ { type: 'image', data: base64, mimeType },
+ { type: 'text', text: `${IMAGE_NOTE} This is the picked element photographed at the instant the user clicked it.` },
+ ],
+ };
+}
+
+/** An open challenge comes back as a picture beside its description, so the caller can answer it by looking. */
+function renderCaptcha(result: ActionResult, fenceWith?: string): ToolReply {
+ if (!result.ok) return render(result, fenceWith);
+ const data = result.data as { challenge?: { image?: unknown } & Record<string, unknown> } & Record<string, unknown>;
+ const image = data.challenge?.image;
+ if (typeof image !== 'string') return render(result, fenceWith);
+
+ const { image: _image, ...challenge } = data.challenge!;
+ const [mimeType, base64] = splitDataUrl(image);
+ const rendered = render({ ok: true, data: { ...data, challenge } }, fenceWith);
+ return {
+ content: [
+ ...rendered.content,
+ { type: 'image', data: base64, mimeType },
+ {
+ type: 'text',
+ text: `${IMAGE_NOTE} This is the open captcha challenge, ${challenge.imageWidth}×${challenge.imageHeight} pixels${
+ challenge.kind === 'tiles' ? ', each tile numbered in its corner' : ''
+ }.`,
+ },
+ ],
+ };
+}
+
+function renderFocusShot(result: ActionResult): ToolReply {
+ if (!result.ok) return render(result);
+ const dataUrl = (result.data as { dataUrl?: unknown } | null)?.dataUrl;
+ if (typeof dataUrl !== 'string') return render(result);
+
+ const [mimeType, base64] = splitDataUrl(dataUrl);
+ return {
+ content: [
+ { type: 'image', data: base64, mimeType },
+ { type: 'text', text: `${IMAGE_NOTE} This is the element the user pointed at with A-Eye, as it stood when they picked it.` },
+ ],
+ };
+}
+
+function splitDataUrl(dataUrl: string): [mimeType: string, base64: string] {
+ const match = /^data:([^;,]+);base64,(.*)$/s.exec(dataUrl);
+ return match ? [match[1], match[2]] : ['image/png', ''];
+}
+
+/**
+ * `fenceWith` marks the payload as untrusted page data. Failures are left bare: they
+ * are daemon-authored and the run preamble teaches the agent to read `CODE: message`.
+ *
+ * The seal runs on every body, fenced or not. The extension has normally sealed already
+ * and this pass leaves its handles alone; what it catches is a result that reached the
+ * daemon another way.
+ */
+function render(result: ActionResult, fenceWith?: string): ToolReply {
+ if (result.ok) {
+ const body = sealSecrets(JSON.stringify(result.data));
+ return { content: [{ type: 'text', text: fenceWith ? fence(body, fenceWith) : body }] };
+ }
+ return { isError: true, content: [{ type: 'text', text: sealSecrets(`${result.error.code}: ${result.error.message}`) }] };
+}