Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
/** Runs the model's browser actions (core/inference/browser_agent.py) in the agent's tab and posts
* each result back. Every action goes through the gates in gate.ts first. */
import { authFetch } from "@/features/auth";
import { openChatItemById, useChatNavigationStore, visibleChatItems } from "@/features/chat";
import { getLocale, translate } from "@/i18n";
import { toast } from "@/lib/toast";
import { isStudioUrl, isTauri } from "@/lib/api-base";
import { unwrapRedirect } from "../address";
import { captureHiddenPage, screenshotPage } from "../capture";
import { useBrowserHistoryStore } from "../history-store";
import { hasNativeView, keepInBackground, nativeAction, releaseBackground, whenViewOpen } from "../native-view";
import { isLoopbackUrl } from "../address";
import { callNative, localOriginGranted, useNativeBrowser } from "../native-support";
import { requestFrameAgent, sendFrameCommand } from "../page-frame";
import { browserPanelAvailable } from "../panel-availability";
import { type BrowserEntry, type BrowserTab, browserFile, chatKeyOfThread, currentEntry, onChatAdopted, useBrowserStore } from "../store";
import {
type AgentStep,
type Answer,
adoptAgentChat,
agentChat,
askUser,
liveChat,
onBrowserAgentRunEnd,
patchAgentChat,
setActiveAction,
useBrowserAgentStore,
} from "./agent-store";
import { type Gate, type Question, type SitePolicy, gateElement, gateNavigation, hostOfUrl, navigationBlocks } from "./gate";
import { type SiteRule, ruleFor } from "./permissions";
import { type AgentAnswer, type ElementInfo, pageAgentSource } from "./page-agent";
import { type FileTabKind, MAX_EDIT_BYTES, applyFileEdit, numberedLines } from "./file-tab";
import { waitForPage } from "./page-wait";
export type BrowserAction = {
id: string;
sessionId: string;
op: string;
url?: string;
ref?: string;
text?: string;
submit?: boolean;
value?: string;
direction?: "up" | "down";
key?: string;
x?: number;
y?: number;
toRef?: string;
toX?: number;
toY?: number;
script?: string;
fullPage?: boolean;
query?: string;
limit?: number;
tab?: number;
startLine?: number;
oldText?: string;
newText?: string;
replaceAll?: boolean;
content?: string;
/** How long the server waits for this answer. */
timeoutS?: number;
policy: SitePolicy;
};
/** `scope`: the chat the run belongs to; sites approved in one chat aren't approved in another.
* `chat`: its browser chat when that isn't the thread's own (a compare pane's pair).
* `temporary`: the run's chat is temporary, so its pages stay out of history whatever chat is shown. */
export type ActionContext = {
runKey: string;
scope?: string;
chat?: string;
temporary?: boolean;
userText: string;
fullAccess?: boolean;
signal?: AbortSignal;
};
type Run = ActionContext & { session: Session; inFile?: boolean };
type Result = { ok: boolean; error?: string; url?: string; title?: string; file?: string; text?: string; note?: string; image?: string };
const LOAD_TIMEOUT_MS = 30_000;
const SETTLE_QUIET_MS = 350;
const SETTLE_MAX_MS = 3_000;
const FILL_QUIET_MAX_MS = 1_000;
// A loaded page that opened no request has drawn what it will; content still on its way arrives by a request.
const FILL_IDLE_QUIET_MS = 120;
const PAGE_READY_MS = 4_000;
const FILE_FRAME_TRY_MS = 1_500;
const READ_OPS = new Set(["snapshot", "find", "status", "describe", "locate"]);
const POST_TIMEOUT_MS = 15_000;
const EXPIRED = "Not done: the action ran out of time, likely waiting for the user to answer in the panel. Ask them before trying again.";
export const STOPPED = "The user stopped the browser agent. Do not use browser tools again unless they ask.";
const str = (value: unknown, max: number): string | undefined =>
typeof value === "string" && value.length <= max ? value : undefined;
const num = (value: unknown): number | undefined =>
typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined;
const whole = (value: unknown): number | undefined =>
typeof value === "number" && Number.isInteger(value) && value > 0 ? value : undefined;
// browser_agent._MAX_FILE_EDIT_CHARS
const MAX_EDIT_CHARS = 500_000;
const strings = (value: unknown): string[] =>
Array.isArray(value) ? value.filter((item): item is string => typeof item === "string").slice(0, 100) : [];
/** The `browser_action` of a tool_output event, checked: it reaches here off the chat stream. */
export function parseBrowserAction(value: unknown): BrowserAction | null {
if (!value || typeof value !== "object") return null;
const raw = value as Record<string, unknown>;
const id = str(raw.id, 64);
const op = str(raw.op, 32);
if (!id || !op) return null;
return {
id,
sessionId: str(raw.session_id, 256) ?? "",
op,
url: str(raw.url, 8192),
ref: str(raw.ref, 16),
text: str(raw.text, 8192),
submit: raw.submit === true,
value: str(raw.value, 512),
direction: raw.direction === "up" ? "up" : "down",
key: str(raw.key, 32),
x: num(raw.x),
y: num(raw.y),
toRef: str(raw.to_ref, 16),
toX: num(raw.to_x),
toY: num(raw.to_y),
script: str(raw.script, 16_000),
fullPage: raw.full_page === true,
query: str(raw.query, 256),
limit: typeof raw.limit === "number" && Number.isInteger(raw.limit) ? raw.limit : undefined,
tab: whole(raw.tab),
startLine: whole(raw.start_line),
oldText: str(raw.old_text, MAX_EDIT_CHARS),
newText: str(raw.new_text, MAX_EDIT_CHARS),
replaceAll: raw.replace_all === true,
content: str(raw.content, MAX_EDIT_CHARS),
timeoutS: typeof raw.timeout_s === "number" && raw.timeout_s > 0 ? Math.min(raw.timeout_s, 3600) : undefined,
policy: { allowedDomains: strings(raw.allowed_domains), blockedDomains: strings(raw.blocked_domains) },
};
}
type Session = {
chat: string;
// Sites the user approved or named, where the model may open addresses it made up; every site the
// agent has been on (a page's link reaches its site, but the page chose it, not the user); and the
// links on the page the agent last read.
knownHosts: Set<string>;
reachedHosts: Set<string>;
opaqueHosts: Set<string>;
pageLinks: Set<string>;
// Where the agent's own moves landed: a native view's Back or Forward doesn't say where it goes,
// so this is how it is checked before it loads.
trail: string[];
at: number;
// A chat never saw another's screenshots: its x and y are CSS pixels until it takes one.
pointScale: Map<string, ShotView & { scaleX: number; scaleY: number }>;
queue: Promise<void>;
alwaysAllowed: Set<string>;
};
const sessions = new Map<string, Session>();
const MAX_SESSIONS = 32;
function sessionFor(chat: string): Session {
let session = sessions.get(chat);
if (!session) {
session = {
chat,
knownHosts: new Set(),
reachedHosts: new Set(),
opaqueHosts: new Set(),
pageLinks: new Set(),
trail: [],
at: -1,
pointScale: new Map(),
queue: Promise.resolve(),
alwaysAllowed: new Set(),
};
sessions.set(chat, session);
for (const old of sessions.keys()) {
if (sessions.size <= MAX_SESSIONS) break;
if (old !== chat && !agentChat(old).runKey) sessions.delete(old);
}
}
return session;
}
function chatOf(context: ActionContext): string {
if (context.chat) return liveChat(context.chat);
const scope = context.scope;
return liveChat(scope && scope !== "__default" ? chatKeyOfThread(scope) : useBrowserStore.getState().chat);
}
onChatAdopted((from, to) => {
const session = sessions.get(from);
sessions.delete(from);
if (session && !sessions.has(to)) sessions.set(to, Object.assign(session, { chat: to }));
adoptAgentChat(from, to);
});
// A chat's page stays open in the background only while its run lasts.
onBrowserAgentRunEnd((chat) => {
const tabId = agentChat(chat).tabId;
if (tabId) releaseBackground(tabId);
});
function record(session: Session, op: string, url: string): void {
const { trail, at } = session;
if (op === "back" && trail[at - 1] === url) session.at -= 1;
else if (op === "forward" && trail[at + 1] === url) session.at += 1;
else if (op !== "reload" || trail[at] !== url) {
session.trail = [...trail.slice(0, at + 1), url];
session.at = session.trail.length - 1;
}
}
function admit(url: string, gate: Gate, context: Run): void {
if (context.fullAccess === true) return;
const host = hostOfUrl(url);
if (!host) return;
context.session.reachedHosts.add(host);
if (!(gate.kind === "allow" && gate.linked)) context.session.knownHosts.add(host);
}
const TEXT_ROLES = new Set(["textbox", "searchbox", "password", "combobox", "spinbutton"]);
// A replayed frame must not click twice; ids are unguessable, so only recent ones are kept.
const seen = new Set<string>();
const MAX_SEEN = 4_000;
export function runBrowserAction(action: BrowserAction, actionContext: ActionContext): Promise<void> {
const session = sessionFor(chatOf(actionContext));
const context: Run = { ...actionContext, session };
const { chat } = session;
if (seen.has(action.id)) return session.queue;
seen.add(action.id);
for (const old of seen) {
if (seen.size <= MAX_SEEN) break;
seen.delete(old);
}
// The server's wait starts when it sends the action, not when an earlier one lets this one run.
const expires = action.timeoutS ? Date.now() + (action.timeoutS - 2) * 1000 : null;
const next = session.queue.then(async () => {
// Ends with the chat run, with Stop, or when the server stops waiting: a question still open
// then must not be answerable, or a late Allow would act for nobody.
const controller = new AbortController();
const abort = () => controller.abort();
context.signal?.addEventListener("abort", abort, { once: true });
if (context.signal?.aborted) abort();
// Out of time is not "the user declined" or "the tab closed", which an abort reads as below.
let expired = false;
const expire = () => {
expired = true;
abort();
};
if (expires !== null && expires <= Date.now()) expire();
const deadline = expires !== null ? setTimeout(expire, Math.max(0, expires - Date.now())) : null;
setActiveAction(chat, controller);
let result: Result;
try {
result = await perform(action, { ...context, signal: controller.signal });
} catch (error) {
result = { ok: false, error: error instanceof Error ? error.message : String(error) };
} finally {
if (deadline) clearTimeout(deadline);
context.signal?.removeEventListener("abort", abort);
setActiveAction(chat, null);
patchAgentChat(chat, { working: false, step: null });
}
if (expired && !result.ok) result = { ok: false, error: EXPIRED };
if (agentChat(chat).stoppedRun === context.runKey) result = { ok: false, error: STOPPED };
await postResult(action, result);
});
session.queue = next.catch(() => undefined);
return next;
}
/** Retried: a lost answer leaves the tool waiting for its whole timeout. */
async function postResult(action: BrowserAction, result: Result): Promise<void> {
for (let attempt = 0; attempt < 4; attempt++) {
try {
const response = await authFetch("/api/inference/browser-action-result", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ action_id: action.id, session_id: action.sessionId, result }),
// A request that never answers would hold every later action behind it.
signal: AbortSignal.timeout(POST_TIMEOUT_MS),
});
// 404: the server already gave up on this action; nothing to deliver.
if (response.ok || (response.status >= 400 && response.status < 500)) return;
} catch {
// Network or auth refresh: try again.
}
await sleep(500 * 2 ** attempt);
}
}
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
const findTab = (tabId: string) => useBrowserStore.getState().tabs.find((candidate) => candidate.id === tabId);
function agentTab(context: Run): BrowserTab | null {
return findTab(agentChat(context.session.chat).tabId ?? "") ?? null;
}
/** The run's chat is on screen with the panel beside it, so the user can watch and answer there. */
function chatOnScreen(context: Run): boolean {
return context.session.chat === useBrowserStore.getState().chat && browserPanelAvailable();
}
function tabUrl(tab: BrowserTab): string {
const entry = currentEntry(tab);
return entry.kind === "web" ? (tab.displayUrl ?? entry.url) : "";
}
/** The agent's tab in front when its chat is on screen, so the user sees what it does; else its
* page works on in the background, with the user's own tabs left where they were. */
function showTab(tab: BrowserTab, context: Run): void {
// Kept from the start: the user can switch chats mid-run, and closing the panel then must leave it.
if (useNativeBrowser.getState().enabled && currentEntry(tab).kind === "web") keepInBackground(tab.id);
// Off screen it shows when the user goes back to that chat; their tabs here stay put. A chat on
// screen without room for the panel isn't given one.
const store = useBrowserStore.getState();
if (context.session.chat !== store.chat || browserPanelAvailable()) store.revealIn(context.session.chat, tab.id);
}
async function waitForLoad(tabId: string, signal?: AbortSignal): Promise<BrowserTab | null> {
const started = Date.now();
// A navigation marks the tab loading at once; a native one may report a beat later.
await sleep(150);
while (Date.now() - started < LOAD_TIMEOUT_MS) {
if (signal?.aborted) return null;
const tab = findTab(tabId);
if (!tab) return null;
if (!tab.loading) return tab;
await sleep(100);
}
return findTab(tabId) ?? null;
}
// The page agent's own error messages (page-agent.ts). Errors reach the model outside the
// untrusted-content fence, and a page can answer in the agent's place, so nothing else passes.
const AGENT_ERRORS = [
/^r\d{1,9} is covered by an overlay; snapshot to find its close button, or scroll first\.$/,
/^r\d{1,9} is not a text field\.$/,
/^r\d{1,9} is disabled or read-only\.$/,
/^r\d{1,9} takes a value like (2026-03-14|09:30|2026-03-14T09:30|2026-03|2026-W11|a number such as 42)\.$/,
/^r\d{1,9} is a radio button; select another option in its group instead\.$/,
/^r\d{1,9} is a (checkbox|radio|switch); value must be "true", "false" or its own label\. Pick the ref of the option to choose\.$/,
/^that key does nothing in this field; click or type instead\.$/,
/^r\d{1,9} has no such option; snapshot it to see its options\.$/,
/^r\d{1,9} is not a dropdown or checkbox; click it, then click the option\.$/,
/^r\d{1,9} is no longer on the page; call browser_snapshot for fresh refs\.$/,
/^this page has too many elements to number; use browser_find\.$/,
/^the page's own script failed during that action; take a fresh snapshot\.$/,
/^that point is outside the visible page; take a screenshot to see what is there\.$/,
/^that point is inside an embedded frame, which the browser tools can't reach\.$/,
/^the script's page went away before it finished; take a snapshot\.$/,
/^the page changed under that point; take a screenshot to see it again\.$/,
/^the page changed while the user was asked, so the script did not run; take a snapshot\.$/,
/^unknown command$/,
];
function agentError(value: unknown): string | undefined {
if (value === undefined) return undefined;
const text = typeof value === "string" ? value : "";
return AGENT_ERRORS.some((pattern) => pattern.test(text)) ? text : "the page gave an answer the panel doesn't recognise.";
}
function checkedAnswer(value: unknown): AgentAnswer | null {
if (!value || typeof value !== "object") return null;
const raw = value as Record<string, unknown>;
if (typeof raw.ok !== "boolean") return null;
const info = raw.info && typeof raw.info === "object" ? (raw.info as Record<string, unknown>) : null;
return {
ok: raw.ok,
error: agentError(raw.error),
text: typeof raw.text === "string" ? raw.text.slice(0, 200_000) : undefined,
links: Array.isArray(raw.links)
? raw.links.filter((link): link is string => typeof link === "string" && link.length <= 8192).slice(0, 400)
: [],
note: str(raw.note, 1000),
navigate: str(raw.navigate, 8192),
quietMs: typeof raw.quietMs === "number" ? raw.quietMs : undefined,
requests: typeof raw.requests === "number" ? raw.requests : undefined,
x: num(raw.x),
y: num(raw.y),
width: num(raw.width),
height: num(raw.height),
pending: raw.pending === true,
hit: str(raw.hit, 16),
sig: str(raw.sig, 20_000),
doc: str(raw.doc, 16),
scrollX: num(raw.scrollX),
scrollY: num(raw.scrollY),
scrolls: num(raw.scrolls),
title: typeof raw.title === "string" ? raw.title.slice(0, 300) : undefined,
info: info
? {
role: String(info.role ?? "").slice(0, 40),
name: String(info.name ?? "").slice(0, 200),
tag: String(info.tag ?? "").slice(0, 40),
type: String(info.type ?? "").slice(0, 40),
href: String(info.href ?? "").slice(0, 8192),
autocomplete: String(info.autocomplete ?? "").slice(0, 200),
formMethod: String(info.formMethod ?? "").slice(0, 10),
formAction: String(info.formAction ?? "").slice(0, 8192),
formHasPassword: info.formHasPassword === true,
formPasswordFilled: info.formPasswordFilled === true,
submits: info.submits === true,
...(typeof info.option === "string" ? { option: info.option.slice(0, 200) } : {}),
...(typeof info.submitter === "string" ? { submitter: info.submitter.slice(0, 200) } : {}),
}
: undefined,
};
}
// Each newly installed page numbers its refs in its own range (page-agent.ts `base`, `span`),
// within the server's 9 digits: r1, r100001, r200001...
let nextBase = 1;
const REF_SPAN = 100_000;
const LAST_BASE = 9_000 * REF_SPAN;
/** One command in the tab's page, installing the agent there first if this page lacks it. */
let accentRaw = "";
let accentColor = "";
/** The theme's accent (toggles, badges; a custom Accent included) as rgb(), for the cursor's glow. */
function cursorAccent(): string {
if (typeof document === "undefined") return "";
const raw = getComputedStyle(document.documentElement).getPropertyValue("--control-accent").trim();
if (raw !== accentRaw) {
accentRaw = raw;
accentColor = "";
// Painted, not read back from fillStyle, which keeps oklch() and color() as written.
const canvas = document.createElement("canvas");
canvas.width = 1;
canvas.height = 1;
const context = canvas.getContext("2d", { willReadFrequently: true });
if (context && raw) {
context.fillStyle = raw;
context.fillRect(0, 0, 1, 1);
const [r, g, b, a] = context.getImageData(0, 0, 1, 1).data;
if (a > 0) accentColor = `rgb(${r}, ${g}, ${b})`;
}
}
return accentColor;
}
/** Settings > Appearance > Reduce motion as `prefersReducedMotion` resolves it, read off the classes
* it sets on <html>: importing the settings store here would cycle through features/settings. */
function reducedMotion(): boolean {
if (typeof document === "undefined") return false;
const html = document.documentElement.classList;
if (html.contains("force-reduced-motion")) return true;
if (html.contains("force-motion")) return false;
return window.matchMedia?.("(prefers-reduced-motion: reduce)").matches === true;
}
async function pageCall(tab: BrowserTab, request: Record<string, unknown>): Promise<AgentAnswer> {
const cursor = useBrowserAgentStore.getState().showCursor;
const command = {
...request,
cursor,
...(cursor ? { cursorColor: cursorAccent(), cursorStill: reducedMotion() } : {}),
};
// File previews are frames, never native views.
const native = useNativeBrowser.getState().enabled && currentEntry(tab).kind === "web";
const started = Date.now();
const fresh = () => {
const base = nextBase;
nextBase = nextBase > LAST_BASE ? 1 : nextBase + REF_SPAN;
return { ...command, base, span: REF_SPAN };
};
while (true) {
let reply: { installed: boolean; result: unknown } | null = null;
if (native) {
if (hasNativeView(tab.id)) {
const call = (op: Record<string, unknown>, code?: string) =>
callNative<{ installed: boolean; result: unknown }>("browser_view_agent", {
tabId: tab.id,
command: { op, ...(code ? { code } : {}) },
}).catch(() => null);
reply = await call(command);
if (reply && !reply.installed) reply = await call(fresh(), pageAgentSource());
}
} else {
// Short tries for reads only: they are safe to repeat.
const wait = currentEntry(tab).kind === "file" && READ_OPS.has(String(request.op)) ? FILE_FRAME_TRY_MS : undefined;
reply = await requestFrameAgent(tab.id, command, undefined, wait);
if (reply && !reply.installed) reply = await requestFrameAgent(tab.id, fresh(), pageAgentSource(), wait);
}
const answer = reply?.installed ? checkedAnswer(reply.result) : null;
if (answer) return answer;
// The page may still be mounting after a navigation.
if (Date.now() - started > PAGE_READY_MS) {
throw new Error("the page is not answering; it may still be loading, or it is not a web page. Try browser_snapshot again.");
}
await sleep(250);
}
}
/** Requests started since `since` and a quiet DOM; `left` says the tab is on another page. */
function waitOnPage(
tabId: string,
left: (tab: BrowserTab) => boolean,
since: number,
changingMaxMs: number,
signal?: AbortSignal,
idleQuietMs?: number,
) {
return waitForPage({
maxMs: SETTLE_MAX_MS, changingMaxMs, quietMs: SETTLE_QUIET_MS, idleQuietMs, now: Date.now, sleep,
status: () => {
const tab = findTab(tabId);
return tab ? pageCall(tab, { op: "status", since }).catch(() => null) : Promise.resolve(null);
},
moved: () => {
const tab = findTab(tabId);
return !tab || tab.loading || left(tab);
},
aborted: () => Boolean(signal?.aborted),
});
}
/** Until the page settles after an action sent at `since`, following any navigation it started. */
async function settle(tabId: string, urlBefore: string, since: number, signal?: AbortSignal): Promise<BrowserTab | null> {
await sleep(120);
const state = await waitOnPage(tabId, (tab) => tabUrl(tab) !== urlBefore, since, SETTLE_MAX_MS, signal);
if (state === "moved" && findTab(tabId)) return waitForLoad(tabId, signal);
return findTab(tabId) ?? null;
}
/** Until a loaded page fills itself in (lists fetched after load); true if requests were still open.
* A DOM that never rests gets only FILL_QUIET_MAX_MS. A router rewriting the address is still this page. */
async function untilFilled(tab: BrowserTab, since: number, signal?: AbortSignal): Promise<boolean> {
const entry = currentEntry(tab);
return (await waitOnPage(tab.id, (now) => currentEntry(now) !== entry, since, FILL_QUIET_MAX_MS, signal, FILL_IDLE_QUIET_MS)) === "loading";
}
async function pass(gate: Gate, context: Run, remember = true): Promise<string | null> {
if (gate.kind === "allow") return null;
if (gate.kind === "block") return `Not done: ${gate.reason}.`;
const { ask, host, name, detail } = gate;
const allowed = await askInPanel({ ask, host, name, detail }, context, remember);
// Not gate.prompt: it quotes the page's label, and errors reach the model outside the fence.
return allowed ? null : `The user declined ${DECLINED[ask]} on ${host || "this page"}. Do something else, or ask them.`;
}
const DECLINED: Record<Question["ask"], string> = {
open: "opening a page",
signIn: "signing in",
submit: "sending a form",
click: "that click",
history: "sharing their browsing history",
script: "running that script",
pointer: "clicking where the page doesn't say what is there",
};
/** A question waits in a chat that isn't on screen: say so, with a way there. One toast per chat. */
function notifyOffScreenQuestion(chat: string): void {
// A single chat's key is `single:<thread>`, a compare's `compare:<pair>`; the sidebar row has that id.
const [type, id] = [chat.slice(0, chat.indexOf(":")), chat.slice(chat.indexOf(":") + 1)];
const item =
visibleChatItems(useChatNavigationStore.getState()).find((candidate) => candidate.id === id && candidate.type === type) ?? null;
toast(translate("browser.agent.waiting", { chat: item?.title || translate("browser.agent.otherChat", undefined, getLocale()) }, getLocale()), {
id: `browser-agent-question:${chat}`,
action: item ? { label: translate("browser.agent.show", undefined, getLocale()), onClick: () => openChatItemById(item) } : undefined,
});
}
/** Questions show in the chat's agent bar: the panel opens for one in the chat on screen, and one
* from a chat off screen waits there, with a toast to go answer it. A script is asked every time. */
async function askInPanel(question: Question, context: Run, remember = true): Promise<Answer> {
// As askUser does: a stopped action isn't answered, even by Always allow.
if (context.signal?.aborted) return false;
// A click is asked about by its label, so Always allow covers that button, not every risky one.
const key = [question.ask, question.host, question.ask === "click" ? question.name : ""].join("\n");
// A "__default" first turn resolves to whichever chat is on screen, which may not be its own.
const keep = remember && question.ask !== "script" && Boolean(context.chat || (context.scope && context.scope !== "__default"));
const granted = context.session.alwaysAllowed;
if (keep && granted.has(key)) return "always";
const tabId = agentChat(context.session.chat).tabId;
const store = useBrowserStore.getState();
if (tabId) store.revealIn(context.session.chat, tabId);
// Even when already open: a project's overlay or Research can be hiding it, and only a new open shows it.
if (chatOnScreen(context)) store.openPanel();
else notifyOffScreenQuestion(context.session.chat);
const answer = await askUser(context.session.chat, question, context.signal, keep);
if (answer === "always" && keep) granted.add(key);
return answer;
}
async function observe(tab: BrowserTab, context: Run, note?: string, op: Record<string, unknown> = { op: "snapshot" }): Promise<Result> {
const entry = currentEntry(tab);
if (entry.kind === "file") return observeFile(tab, entry, context, note, op);
if (entry.kind !== "web") return { ok: false, error: "No web page is open. Use browser_navigate first." };
if (tab.pageError || tab.nativeError) {
return {
ok: false,
// The host only: an error is outside the untrusted-content fence, and a path is the page's words.
error: `${hostOfUrl(tabUrl(tab)) || "The page"} could not be loaded (the site refused, needs a check the panel can't pass, or is unreachable).`,
};
}
const answer = await pageCall(tab, op);
if (!answer.ok) return { ok: false, error: answer.error ?? "the page could not be read." };
const fresh = findTab(tab.id) ?? tab;
// A page that moved during the read is one nobody checked: its words and links stay out.
if (fresh.loading || tabUrl(fresh) !== tabUrl(tab)) {
return { ok: false, error: "the page changed while it was being read; take a snapshot again." };
}
if (op.op === "snapshot" || op.op === "find") context.session.pageLinks = new Set(context.fullAccess === true ? [] : linkedPages(answer.links ?? []));
return {
ok: true,
url: tabUrl(fresh),
title: fresh.customTitle ?? fresh.title,
text: answer.text,
note: [note, answer.note].filter(Boolean).join(" ") || undefined,
};
}
// By its entry, not the page shown: a granted page can show a data: or about:blank document.
function onGrantedOrigin(tab: BrowserTab): boolean {
const entry = currentEntry(tab);
return entry.kind === "web" && localOriginGranted(tab.id, entry.url);
}
function grantedLocalTab(url: string, chat: string): BrowserTab | undefined {
return useBrowserStore.getState().tabs.find((tab) => tab.chat === chat && localOriginGranted(tab.id, url));
}
function moveRun(context: Run, tabId: string): void {
const previous = agentChat(context.session.chat).tabId;
if (previous && previous !== tabId) releaseBackground(previous);
patchAgentChat(context.session.chat, { tabId });
context.session.trail = [];
context.session.at = -1;
}
async function open(url: string, context: Run): Promise<BrowserTab | null> {
const store = useBrowserStore.getState();
const agent = agentTab(context);
const existing = isLoopbackUrl(url)
? agent && localOriginGranted(agent.id, url) ? agent : grantedLocalTab(url, context.session.chat)
: agent && !onGrantedOrigin(agent) && currentEntry(agent).kind !== "file" ? agent : undefined;
if (existing) {
if (agentChat(context.session.chat).tabId !== existing.id) moveRun(context, existing.id);
store.navigate(existing.id, { url, temporary: context.temporary }, { replace: false });
showTab(existing, context);
return waitForLoad(existing.id, context.signal);
}
// In its own chat's tabs: a chat off screen gets a tab there, and the panel on screen stays put.
const tabId = store.openTabIn(context.session.chat, url, { temporary: context.temporary });
if (!tabId) return null;
// Off screen the view opens a beat later: loading until it reports in, so the wait below doesn't read the blank view.
if (!chatOnScreen(context)) store.updateTab(tabId, { loading: true });
moveRun(context, tabId);
const tab = findTab(tabId);
if (tab) showTab(tab, context);
return waitForLoad(tabId, context.signal);
}
function stepOf(action: BrowserAction): AgentStep {
switch (action.op) {
case "navigate":
return { kind: "open", detail: hostOfUrl(action.url ?? "") };
case "back":
case "forward":
case "reload":
return { kind: action.op, detail: "" };
case "find":
return { kind: "find", detail: action.query ?? "" };
case "click":
case "type":
case "select":
return { kind: action.op, detail: action.ref ?? "" };
case "press":
return { kind: "press", detail: action.key ?? "" };
case "scroll":
case "screenshot":
case "evaluate":
case "history":
case "move":
case "drag":
case "tabs":
return { kind: action.op, detail: "" };
case "read_file":
return { kind: "readFile", detail: "" };
case "edit_file":
return { kind: "editFile", detail: "" };
default:
return { kind: "read", detail: "" };
}
}
function nameStep(chat: string, info: ElementInfo): void {
const current = agentChat(chat).step;
if (!current) return;
patchAgentChat(chat, { step: { ...current, detail: info.name || info.role || current.detail } });
}
function permissions(context: Pick<Run, "fullAccess" | "inFile">) {
const stored = useBrowserAgentStore.getState().permissions;
// Preview forms can't send (form-action 'none'); links are still gated.
const saved = context.inFile
? { ...stored, defaults: { ...stored.defaults, actions: "allow" as const, unlabelled: "allow" as const } }
: stored;
if (!context.fullAccess) return saved;
const allow = (rule: SiteRule): SiteRule => ({
browsing: rule.browsing === "block" ? "block" : "allow",
actions: "allow",
downloads: rule.downloads === "block" ? "block" : "allow",
unlabelled: "allow",
});
return { defaults: allow(saved.defaults), sites: Object.fromEntries(Object.entries(saved.sites).map(([site, rule]) => [site, allow(rule)])) };
}
function navigationContext(action: BrowserAction, context: Run) {
const tab = agentTab(context);
const currentUrl = tab ? tabUrl(tab) : "";
const { reachedHosts, knownHosts, pageLinks } = context.session;
return {
policy: action.policy,
permissions: permissions(context),
currentUrl: reachedHosts.has(hostOfUrl(currentUrl)) ? currentUrl : "",
knownHosts,
pageLinks,
userText: context.userText,
};
}
/** Where a search engine's redirect link lands: the tab opens that page, so that page is the one gated. */
/** Every URL a redirect link passes through, the link first and its landing page last. */
function redirectChain(url: string): string[] {
const chain = [url];
// Each hop's target is a strictly shorter part of the link, so this ends.
for (let next = unwrapRedirect(url); next !== chain[chain.length - 1]; next = unwrapRedirect(next)) chain.push(next);
return chain;
}
/** A page's links, with each search-result redirect's landing page too: the tab opens that page,
* and the link to it is on the page, so following it is as linked as any other link there. */
function linkedPages(links: string[]): string[] {
return links.flatMap((link) =>
redirectChain(link).flatMap((url) => {
try {
return [new URL(url).href];
} catch {
return [];
}
}),
);
}
function redirectEnd(url: string): string {
const chain = redirectChain(url);
return chain[chain.length - 1];
}
async function perform(action: BrowserAction, context: Run): Promise<Result> {
const { signal, session } = context;
const { chat, reachedHosts } = session;
if (agentChat(chat).stoppedRun === context.runKey) return { ok: false, error: STOPPED };
if (signal?.aborted) return { ok: false, error: "the action expired before it could run." };
// Off screen, only a native page can work on: a framed one runs in the panel.
if (!PANEL_FREE.has(action.op) && !chatOnScreen(context) && !useNativeBrowser.getState().enabled) {
return {
ok: false,
error:
chat === useBrowserStore.getState().chat
? "the browser panel can't be shown in this window; ask the user to widen it or open the chat."
: "this chat's browser only runs while the chat is open here; ask the user to switch back to it.",
};
}
const owner = agentChat(chat).runKey;
if (owner && owner !== context.runKey) {
return { ok: false, error: "Another reply in this chat is using the browser right now. Wait for it to finish, or tell the user." };
}
const front = owner ? null : frontTab(chat);
if (front && currentEntry(front).kind === "file" && agentChat(chat).tabId !== front.id) moveRun(context, front.id);
patchAgentChat(chat, { working: true, runKey: context.runKey, step: stepOf(action) });
if (action.op === "navigate") return navigateTo(action.url ?? "", action, context);
if (action.op === "history") return readHistory(action, context);
if (action.op === "evaluate" && useBrowserAgentStore.getState().scriptAccess === "never") {
return { ok: false, error: SCRIPTS_OFF };
}
if (action.op === "tabs") return listTabs(action, context);
if (action.op === "read_file" || action.op === "edit_file") {
const target = fileTab(context);
if (!target) return { ok: false, error: NO_FILE };
useBrowserStore.getState().revealIn(chat, target.id);
return action.op === "read_file" ? readFile(target, action.startLine ?? 1) : editFile(target, action, signal);
}
const shown = agentTab(context) ?? fileTab(context);
if (!shown) {
return { ok: false, error: "No page is open yet. Call browser_navigate first, or browser_tabs to work in a tab already open." };
}
showTab(shown, context);
// A kept page the cap closed reopens a beat later; its history flags are only good once it has.
if (useNativeBrowser.getState().enabled && currentEntry(shown).kind === "web" && !hasNativeView(shown.id)) {
await whenViewOpen(shown.id);
}
const tab = findTab(shown.id);
if (!tab) return { ok: false, error: "the browser tab closed." };
const before = tabUrl(tab);
if (action.op === "back" || action.op === "forward" || action.op === "reload") {
const native = useNativeBrowser.getState().enabled && currentEntry(tab).kind === "web";
const store = useBrowserStore.getState();
const nativeMove = native && action.op !== "reload" && Boolean(tab.nativeHistory?.[action.op]);
const step = action.op === "back" ? -1 : action.op === "forward" ? 1 : 0;
const entry = tab.history[tab.index + step];
const destination =
action.op === "reload"
? before
: nativeMove
? (session.trail[session.at] === before && session.trail[session.at + step]) || ""
: entry?.kind === "web"
? entry.url
: "";
// A router's pushState / hash history lives in the page and comes first, as the toolbar's Back does.
if (action.op !== "reload" && !nativeMove && tab.pageHistory?.[action.op]) {
const since = Date.now();
if (sendFrameCommand(tab.id, { command: "go", delta: action.op === "back" ? -1 : 1 })) {
const after = await settle(tab.id, before, since, signal);
if (!after) return { ok: false, error: "the browser tab closed." };
return arrive(after, before, action, context, since);
}
}
if (action.op !== "reload" && !nativeMove && !entry) {
const ready = tab.loading ? await waitForLoad(tab.id, signal) : tab;
if (!ready) return { ok: false, error: "the browser tab closed." };
const refused = await stillAllowed(ready, action, context);
if (refused) return { ok: false, error: refused };
return observe(ready, context, `There is no ${action.op === "back" ? "previous" : "next"} page in this tab's history; this is still the same page.`);
}
if (destination) {
const blocked = navigationBlocks(destination, action.policy, permissions(context));
const gate: Gate = blocked
? { kind: "block", reason: blocked }
: hostOfUrl(destination) === hostOfUrl(before) && reachedHosts.has(hostOfUrl(before))
? { kind: "allow" }
: gateNavigation(destination, navigationContext(action, context));
const refused = await pass(gate, context);
if (refused) return { ok: false, error: refused };
admit(destination, gate, context);
} else if (nativeMove) {
// History from before this chat, or from the user browsing in the tab: where it leads is unknown.
const where = action.op === "back" ? "the previous page" : "the next page";
// Not kept by Always allow: `where` names no site, so it would answer for wherever a later move leads.
const refused = await pass({ kind: "ask", ask: "open", host: where, name: "", detail: "", prompt: `Open ${where}?` }, context, false);
if (refused) return { ok: false, error: refused };
}
const since = Date.now();
// A native move starts when the view reports it; loading from now, so the wait below sees it.
if ((nativeMove || (native && action.op === "reload")) && hasNativeView(tab.id)) store.updateTab(tab.id, { loading: true });
if (nativeMove) nativeAction(tab.id, action.op as "back" | "forward");
else if (native && action.op === "reload") nativeAction(tab.id, "reload");
else if (action.op === "back") store.goBack(tab.id);
else if (action.op === "forward") store.goForward(tab.id);
else store.reload(tab.id);
const loaded = await waitForLoad(tab.id, signal);
return loaded ? arrive(loaded, before, action, context, since) : { ok: false, error: "the browser tab closed." };
}
const ready = tab.loading ? await waitForLoad(tab.id, signal) : tab;
if (!ready) return { ok: false, error: "the browser tab closed." };
const here = tabUrl(ready);
const moved = await stillAllowed(ready, action, context);
if (moved) return { ok: false, error: moved };
const file = currentEntry(ready);
if (file.kind === "file") {
const preview = (await kindOf(file)) === "preview";
if (!preview && action.op !== "snapshot" && action.op !== "find" && action.op !== "screenshot") return { ok: false, error: NO_PAGE_IN_FILE };
if (action.op === "evaluate") return { ok: false, error: "scripts don't run in a file's preview; change the file with browser_edit_file instead." };
if (preview && action.op !== "snapshot" && action.op !== "find") {
if (!chatOnScreen(context)) return { ok: false, error: OFF_SCREEN };
context.inFile = true;
showPreview(ready);
const warm = await pageCall(ready, { op: "status" }).catch((error: Error) => ({ ok: false, error: error.message }));
if (!warm.ok) return { ok: false, error: warm.error ?? NO_PAGE_IN_FILE };
}
}
switch (action.op) {
case "snapshot":
return observe(ready, context, undefined, { op: "snapshot", fullPage: action.fullPage, ...(action.ref ? { ref: action.ref } : {}) });
case "find":
return observe(ready, context, undefined, { op: "find", query: action.query ?? "" });
case "scroll": {
const since = Date.now();
const moved = await pageCall(ready, { op: "scroll", direction: action.direction, ...(action.ref ? { ref: action.ref } : {}) });
if (!moved.ok) return { ok: false, error: moved.error };
const after = await settle(ready.id, here, since, signal);
if (!after) return { ok: false, error: "the browser tab closed." };
// A page can navigate on scroll: the new address is checked like any other arrival.
const now = tabUrl(after);
if (now && now !== here) return arrive(after, here, action, context, since, moved.note);
return observe(after, context, moved.note);
}
case "screenshot":
return screenshot(ready, context);
case "move":
return moveTo(ready, action, context, here);
case "drag":
return dragBetween(ready, action, context, here);
case "evaluate":
return runScript(ready, action, context, here);
}
// Acting on an element: what it is decides whether the user is asked first.
const ref = action.ref ?? "";
const point = action.op === "click" ? await pagePoint(ready, session, action.x, action.y) : null;
if (typeof point === "string") return { ok: false, error: point };
const described = await pageCall(ready, {
op: "describe",
ref,
...point,
...(action.op === "select" ? { value: action.value ?? "" } : {}),
});
if (!described.ok || !described.info) {
return { ok: false, error: described.error ?? (point ? "nothing is at that point." : `${ref} could not be found.`) };
}
const info: ElementInfo = described.info;
const gated = gateable(info, context);
if (action.op !== "press") nameStep(chat, action.op === "select" && info.option ? { ...info, name: info.option } : info);
const gates: Gate[] = [];
if (action.op === "click" || action.op === "type") gates.push(gateElement(action.op, gated, here, action.policy, permissions(context)));
const opaque = point ? opaqueGate(info, here, context) : null;
if (opaque) gates.push(opaque);
if (action.op === "select") {
// The option the page will select: its value can differ from the label the user sees.
const chosen = info.role === "combobox" ? { ...gated, name: info.option || action.value || "" } : gated;
// A card-expiry or other secret dropdown is the user's to fill, as its text field would be.
if (info.tag === "select" || info.role === "combobox") gates.push(gateElement("type", gated, here, action.policy, permissions(context)));
gates.push(gateElement("select", chosen, here, action.policy, permissions(context)));
}
// Enter or Space on a focused button or link is a click on it, so it asks what a click would.
const activates = action.op === "press" && (action.key === "Enter" || action.key === "Space");
// The frame would follow a pressed link unchecked.
if (context.inFile && activates && info.role === "link") return { ok: false, error: "click the link instead of pressing a key on it." };
if (activates && !TEXT_ROLES.has(info.role)) gates.push(gateElement("click", gated, here, action.policy, permissions(context)));
// Deleting is editing: a password or card field the agent can't type into it can't empty either.
if (action.op === "press" && (action.key === "Backspace" || action.key === "Delete")) {
gates.push(gateElement("type", gated, here, action.policy, permissions(context)));
// Outside a text field it is an app's delete shortcut, asked like a Delete button.
if (!TEXT_ROLES.has(info.role)) {
gates.push(gateElement("click", { ...gated, name: `delete key on ${info.name || info.role}` }, here, action.policy, permissions(context)));
}
}
// Enter in a field sends its form, so it is held to what a submit button would be.
const submits =
(action.op === "type" && action.submit) ||
(action.op === "press" && action.key === "Enter" && TEXT_ROLES.has(info.role)) ||
((action.op === "click" || activates) && info.submits);
// A link the page follows itself (a native view's click, Enter on a link) is gated before it
// loads; the proxied frame's clicks come back as `navigate` and are gated below.
const followsLink =
!context.inFile &&
/^https?:/i.test(info.href) &&
info.href.split("#")[0] !== here.split("#")[0] &&
((action.op === "click" && useNativeBrowser.getState().enabled) || (activates && info.role === "link"));
// A search engine's redirect link is gated at the page it lands on, as a navigate would be.
const linkEnd = followsLink ? redirectEnd(info.href) : "";
const linkGate = followsLink ? gateNavigation(linkEnd, navigationContext(action, context)) : null;
if (linkGate) gates.push(linkGate);
// The page itself still loads the wrapper first, so the wrapper keeps its own checks too.
const wrappers = followsLink ? redirectChain(info.href).slice(0, -1) : [];
const wrapperGates = wrappers.map((url) => gateNavigation(url, navigationContext(action, context)));
gates.push(...wrapperGates);
// A form sending to another site carries what was typed there: a new site asks, as a navigation would.
const formAction = submits && gated.formAction ? gated.formAction : "";
const formGate =
formAction && hostOfUrl(formAction) !== hostOfUrl(here) ? gateNavigation(formAction, navigationContext(action, context)) : null;
if (formAction) {
// A submit button's own click gate already asks what a submit would.
if (!info.submits) gates.push(gateElement("submit", info, here, action.policy, permissions(context)));
if (formGate) gates.push(formGate);
}
for (const gate of gates) {
const refused = await pass(gate, context);
if (refused) return { ok: false, error: refused };
}
if (signal?.aborted) return { ok: false, error: STOPPED };
if (opaque) session.opaqueHosts.add(opaque.host);
// A point's control can be relabelled or rewired around the same leaf while the user decides.
if (point && gates.some((gate) => gate.kind === "ask")) {
if (!(await unchanged(ready, point, described.hit ?? "", JSON.stringify(info)))) {
return { ok: false, error: CHANGED_UNDER_POINT };
}
if (signal?.aborted) return { ok: false, error: STOPPED };
}
if (formGate) admit(formAction, formGate, context);
if (linkGate) admit(linkEnd, linkGate, context);
wrappers.forEach((url, index) => admit(url, wrapperGates[index], context));
// Before the action goes out, after any question: a request its handler starts at once is the action's too.
const since = Date.now();
let note: string | undefined;
if (action.op === "click") {
const clicked = await pageCall(ready, {
op: "click",
ref,
...(point ? { ...point, expect: described.hit ?? "", sig: described.sig ?? "" } : {}),
directLinks: context.inFile === true || !useNativeBrowser.getState().enabled,
});
if (!clicked.ok) return { ok: false, error: clicked.error };
if (clicked.navigate && context.inFile) return openFromFile(clicked.navigate, action, context);
if (clicked.navigate) {
const pressedAt = Date.now();
// A link: opened in this tab, under the same rules as any other address. Its href is read
// now, so it may not be one the last snapshot listed.
const end = redirectEnd(clicked.navigate);
const gate = gateNavigation(end, navigationContext(action, context));
const refused = await pass(gate, context);
if (refused) return { ok: false, error: refused };
admit(end, gate, context);
// The page goes the moment the tab navigates, taking the cursor with it: let its press show first.
if (useBrowserAgentStore.getState().showCursor) {
await sleep(Math.max(0, LINK_PRESS_MS - (Date.now() - pressedAt)));
if (signal?.aborted) return { ok: false, error: STOPPED };
}
const sent = Date.now();
// Straight to the landing page that was gated: the store unwraps only one redirect hop.
useBrowserStore.getState().navigate(ready.id, { url: end, from: here, temporary: context.temporary }, { replace: false });
const loaded = await waitForLoad(ready.id, signal);
return loaded ? arrive(loaded, end, action, context, sent) : { ok: false, error: "the browser tab closed." };
}
note = clicked.note;
} else if (action.op === "type") {
const typed = await pageCall(ready, { op: "type", ref, text: action.text ?? "", submit: action.submit === true });
if (!typed.ok) return { ok: false, error: typed.error };
note = typed.note;
} else if (action.op === "select") {
const chosen = await pageCall(ready, { op: "select", ref, value: action.value ?? "" });
if (!chosen.ok) return { ok: false, error: chosen.error };
note = chosen.note;
} else if (action.op === "press") {
const pressed = await pageCall(ready, { op: "press", key: action.key ?? "" });
if (!pressed.ok) return { ok: false, error: pressed.error };
note = pressed.note;
} else {
return { ok: false, error: `unknown browser action ${action.op}.` };
}
const after = await settle(ready.id, here, since, signal);
if (!after) return { ok: false, error: "the browser tab closed." };
const now = tabUrl(after);
if (now && now !== here) return arrive(after, here, action, context, since, note);
return observe(after, context, note);
}
async function navigateTo(address: string, action: BrowserAction, context: Run): Promise<Result> {
const url = redirectEnd(address);
if (isLoopbackUrl(url) && !grantedLocalTab(url, context.session.chat)) {
const { host } = new URL(url);
return {
ok: false,
error: useNativeBrowser.getState().enabled
? `${host} is on the user's computer, and the browser panel opens a local address only in the tab where the user types it in this chat's address bar. Ask the user to open it there first.`
: `${host} is a local address, and the browser only opens public sites.`,
};
}
const gate = gateNavigation(url, navigationContext(action, context));
const refused = await pass(gate, context);
if (refused) return { ok: false, error: refused };
admit(url, gate, context);
const since = Date.now();
const tab = await open(url, context);
if (!tab) return { ok: false, error: "the browser tab closed." };
return arrive(tab, url, action, context, since);
}
const PANEL_FREE = new Set(["tabs", "read_file", "edit_file"]);
const NO_FILE = "No file is open in the browser panel. browser_tabs lists the tabs.";
const NO_PAGE_IN_FILE =
"This tab shows a file with no page to act on. browser_read_file reads it and browser_edit_file changes it.";
const OFF_SCREEN = "this chat's browser only runs while the chat is open here; ask the user to switch back to it.";
type FileEntry = Extract<BrowserEntry, { kind: "file" }>;
let readerModule: Promise<typeof import("./file-reader")> | null = null;
const fileReader = () => {
readerModule ??= import("./file-reader");
return readerModule;
};
async function kindOf(entry: FileEntry): Promise<FileTabKind> {
return (await fileReader()).fileTabKind(entry);
}
const KIND_LABELS: Record<FileTabKind, string> = {
preview: "HTML or React, with a preview",
text: "text",
document: "document",
image: "image",
media: "audio or video",
other: "file",
};
async function fileHint(entry: FileEntry): Promise<string> {
const reader = await fileReader();
const kind = reader.fileTabKind(entry);
const hints: string[] = [];
if (kind === "preview") hints.push("Its preview works as a page: browser_snapshot, browser_click and browser_type act in it.");
if (kind === "image") hints.push(isTauri ? "browser_screenshot shows it." : "Looking at an image needs the desktop app.");
if (reader.editable(entry, kind)) hints.push("browser_read_file reads its source and browser_edit_file changes it.");
else if (kind === "document") hints.push("browser_read_file reads its text; it can't be edited here.");
else if (kind !== "image") hints.push("It has no text to read.");
return hints.join(" ");
}
function showPreview(tab: BrowserTab): void {
useBrowserStore.getState().setFileView(tab.id, { mode: "preview" });
}
function frontTab(chat: string): BrowserTab | null {
const store = useBrowserStore.getState();
const id = chat === store.chat ? store.activeTabId : store.chats[chat]?.activeTabId;
return store.tabs.find((tab) => tab.id === id && tab.chat === chat) ?? null;
}
function fileTab(context: Run): BrowserTab | null {
const agent = agentTab(context);
if (agent && currentEntry(agent).kind === "file") return agent;
const front = frontTab(context.session.chat);
if (!front || currentEntry(front).kind !== "file") return null;
moveRun(context, front.id);
return front;
}
/** Web tabs by site alone: a title is browsing history. */
async function listTabs(action: BrowserAction, context: Run): Promise<Result> {
const { chat } = context.session;
const tabs = useBrowserStore.getState().tabs.filter((tab) => tab.chat === chat);
const where = (tab: BrowserTab) => `a web page on ${hostOfUrl(tabUrl(tab)) || "this computer"}`;
if (action.tab !== undefined) {
const tab = tabs[action.tab - 1];
if (!tab) return { ok: false, error: `There is no tab ${action.tab}; browser_tabs lists them.` };
const entry = currentEntry(tab);
if (entry.kind !== "web" && entry.kind !== "file") return { ok: false, error: "That tab has no page or file in it." };
if (agentChat(chat).tabId !== tab.id) moveRun(context, tab.id);
showTab(tab, context);
if (entry.kind === "web") return { ok: true, note: `Switched to tab ${action.tab}, ${where(tab)}. browser_snapshot reads it.` };
return { ok: true, file: entry.name, note: `Switched to tab ${action.tab}. ${await fileHint(entry)}` };
}
const agentId = agentChat(chat).tabId;
const frontId = frontTab(chat)?.id;
const rows = await Promise.all(
tabs.map(async (tab, index) => {
const entry = currentEntry(tab);
const what =
entry.kind === "file"
? `File: ${entry.name} (${KIND_LABELS[await kindOf(entry)]})`
: entry.kind === "web"
? `Web page on ${hostOfUrl(tabUrl(tab)) || "this computer"}`
: entry.kind === "internal"
? `The browser's ${entry.page} page`
: "New tab";
const marks = [tab.id === agentId ? "the agent's tab" : "", tab.id === frontId ? "in front" : ""].filter(Boolean);
return `${index + 1}. ${what}${marks.length ? ` (${marks.join(", ")})` : ""}`;
}),
);
return { ok: true, text: rows.join("\n") || "No tabs are open.", note: rows.length ? "browser_tabs with tab switches to one." : undefined };
}
async function fileContent(tab: BrowserTab) {
const entry = currentEntry(tab);
const blob = entry.kind === "file" ? browserFile(entry.fileId) : undefined;
if (entry.kind !== "file" || !blob) return null;
const reader = await fileReader();
const kind = reader.fileTabKind(entry);
return { entry, blob, reader, kind, editable: reader.editable(entry, kind) };
}
async function readFile(tab: BrowserTab, start: number, note?: string): Promise<Result> {
const file = await fileContent(tab);
if (!file) return { ok: false, error: NO_FILE };
const { entry, blob, reader, kind } = file;
if (!file.editable && kind !== "document") return { ok: true, file: entry.name, note: [note, await fileHint(entry)].filter(Boolean).join(" ") };
let read: { text: string; cut: boolean };
try {
read = await reader.fileText(entry, blob, kind);
} catch {
return { ok: false, error: "the file's text could not be read." };
}
const lines = numberedLines(read.text, start);
const cut = read.cut ? "Only the start of this large file was read." : "";
return { ok: true, file: entry.name, text: lines.text, note: [note, lines.note, cut].filter(Boolean).join(" ") };
}
const MAX_FOUND_LINES = 40;
async function findInFile(tab: BrowserTab, query: string, note: string): Promise<Result> {
const file = await fileContent(tab);
if (!file) return { ok: false, error: NO_FILE };
if (!file.editable && file.kind !== "document") return readFile(tab, 1, note);
let text: string;
try {
({ text } = await file.reader.fileText(file.entry, file.blob, file.kind));
} catch {
return { ok: false, error: "the file's text could not be read." };
}
const wanted = query.toLowerCase();
const found: string[] = [];
const lines = text.split(/\r?\n/);
for (let index = 0; index < lines.length && found.length < MAX_FOUND_LINES; index++) {
const line = lines[index] ?? "";
if (line.toLowerCase().includes(wanted)) found.push(`${index + 1}\t${line.length > 500 ? `${line.slice(0, 500)} …` : line}`);
}
const summary = found.length ? `${found.length === MAX_FOUND_LINES ? "The first " : ""}${found.length} matching lines.` : `No line holds "${query}".`;
return { ok: true, file: file.entry.name, text: found.join("\n"), note: [note, summary].filter(Boolean).join(" ") };
}
async function editFile(tab: BrowserTab, action: BrowserAction, signal?: AbortSignal): Promise<Result> {
const file = await fileContent(tab);
if (!file) return { ok: false, error: NO_FILE };
const { entry, blob } = file;
if (!file.editable) {
return { ok: false, error: "only HTML, React, Markdown, code, text, SVG and CSV files can be edited here, not PDFs, documents, images or media." };
}
if (blob.size > MAX_EDIT_BYTES) return { ok: false, error: "this file is too large to edit here." };
const source = await blob.text();
const edited = applyFileEdit(source, { oldText: action.oldText, newText: action.newText, replaceAll: action.replaceAll, content: action.content });
if ("error" in edited) return { ok: false, error: edited.error };
if (edited.text === source) return { ok: true, file: entry.name, note: "The file already reads that way; nothing changed." };
if (signal?.aborted) return { ok: false, error: STOPPED };
const now = findTab(tab.id);
if (!now || currentEntry(now) !== entry) return { ok: false, error: "the tab changed while the file was being edited; read it again." };
useBrowserStore.getState().editFile(tab.id, new Blob([edited.text], { type: blob.type || entry.contentType }));
const changed = (action.newText ?? action.content ?? "").split("\n").length;
const shown = numberedLines(edited.text, Math.max(1, edited.line - 2)).text.split("\n").slice(0, Math.min(40, changed + 4));
return {
ok: true,
file: entry.name,
text: shown.join("\n"),
note: `Edited from line ${edited.line}. The panel shows the new version, and its Back button the one before.`,
};
}
async function observeFile(tab: BrowserTab, entry: FileEntry, context: Run, note: string | undefined, op: Record<string, unknown>): Promise<Result> {
if ((await kindOf(entry)) !== "preview") {
const about = [note, "This tab shows a file, not a page."].filter(Boolean).join(" ");
return op.op === "find" ? findInFile(tab, String(op.query ?? ""), about) : readFile(tab, 1, about);
}
if (!chatOnScreen(context)) return { ok: false, error: OFF_SCREEN };
showPreview(tab);
const answer = await pageCall(tab, op);
if (!answer.ok) return { ok: false, error: answer.error ?? "the page could not be read." };
const fresh = findTab(tab.id);
if (!fresh || fresh.loading || currentEntry(fresh) !== entry) {
return { ok: false, error: "the page changed while it was being read; take a snapshot again." };
}
// The agent can write this page, so its links are not the user's.
if (op.op === "snapshot" || op.op === "find") context.session.pageLinks = new Set();
const source = op.op === "snapshot" ? "This is a file's preview; browser_read_file and browser_edit_file work on its source." : "";
return { ok: true, file: entry.name, text: answer.text, note: [note, answer.note, source].filter(Boolean).join(" ") || undefined };
}
function openFromFile(url: string, action: BrowserAction, context: Run): Promise<Result> {
if (isStudioUrl(url)) {
return Promise.resolve({
ok: false,
error: "that link points at a file beside this one, which the panel doesn't have; link to a full web address, or edit the page.",
});
}
context.inFile = false;
return navigateTo(url, action, context);
}
/** `since`: when the action went out, so the wait covers the requests it started. */
async function afterward(
tab: BrowserTab,
before: string,
action: BrowserAction,
context: Run,
since: number,
note?: string,
): Promise<Result> {
const after = await settle(tab.id, before, since, context.signal);
if (!after) return { ok: false, error: "the browser tab closed." };
const now = tabUrl(after);
if (now && now !== before) return arrive(after, before, action, context, since, note);
return observe(after, context, note);
}
// CSS px per screenshot px, per tab; only valid for the page and viewport it was taken of.
type ShotView = {
url: string;
width?: number;
height?: number;
scrollX?: number;
scrollY?: number;
scrolls?: number;
doc?: string;
};
const MAX_SHOT_EDGE = 1024;
const LINK_PRESS_MS = 350;
async function pagePoint(tab: BrowserTab, session: Session, x?: number, y?: number): Promise<{ x: number; y: number } | string | null> {
if (x === undefined || y === undefined) return null;
const shot = session.pointScale.get(tab.id);
if (!shot) return { x, y };
if (tabUrl(tab) !== shot.url) return STALE_SHOT;
const status = await pageCall(tab, { op: "status" });
const keys = ["width", "height", "scrollX", "scrollY", "scrolls", "doc"] as const;
if (keys.some((key) => status[key] !== shot[key])) return STALE_SHOT;
return { x: x * shot.scaleX, y: y * shot.scaleY };
}
const STALE_SHOT = "the page changed since the screenshot those x and y come from; take a new one.";
async function onScreen(
tab: BrowserTab,
session: Session,
ref: string | undefined,
x?: number,
y?: number,
scroll = true,
): Promise<{ x: number; y: number } | string> {
const point = await pagePoint(tab, session, x, y);
if (point) return point;
const found = await pageCall(tab, { op: "locate", ref: ref ?? "", scroll });
if (!found.ok) return found.error ?? `${ref} could not be found.`;
if (found.x === undefined || found.y === undefined) return `${ref} is not on screen; scroll to it first.`;
return { x: found.x, y: found.y };
}
async function moveTo(tab: BrowserTab, action: BrowserAction, context: Run, before: string): Promise<Result> {
const at = await onScreen(tab, context.session, action.ref, action.x, action.y);
if (typeof at === "string") return { ok: false, error: at };
const since = Date.now();
const moved = await pageCall(tab, { op: "pointer", phase: "move", ...at });
if (!moved.ok) return { ok: false, error: moved.error };
return afterward(tab, before, action, context, since, moved.note);
}
const DRAG_STEP_PX = 20;
const DRAG_FRAME_MS = 16;
/** browser_drag: both ends are asked about as clicks, since a drop can delete, send or buy too. */
async function dragBetween(tab: BrowserTab, action: BrowserAction, context: Run, before: string): Promise<Result> {
const { signal } = context;
const scroll = Boolean(action.ref && action.toRef);
const start = await onScreen(tab, context.session, action.ref, action.x, action.y, scroll);
if (typeof start === "string") return { ok: false, error: start };
const end = await onScreen(tab, context.session, action.toRef, action.toX, action.toY, scroll);
if (typeof end === "string") return { ok: false, error: end };
const from = scroll ? await onScreen(tab, context.session, action.ref, undefined, undefined, false) : start;
if (typeof from === "string") {
return { ok: false, error: "the start and end don't fit on screen together; drag by x and y, or in shorter steps." };
}
const hits: string[] = [];
const sigs: string[] = [];
const infos: string[] = [];
const docs: string[] = [];
let asked = false;
for (const point of [from, end]) {
const described = await pageCall(tab, { op: "describe", ref: "", ...point });
if (!described.ok || !described.info) return { ok: false, error: described.error ?? "nothing is at that point." };
for (const gate of [gateElement("click", gateable(described.info, context), before, action.policy, permissions(context)), opaqueGate(described.info, before, context)]) {
if (!gate) continue;
asked ||= gate.kind === "ask";
const refused = await pass(gate, context);
if (refused) return { ok: false, error: refused };
if (gate.kind === "ask" && gate.ask === "pointer") context.session.opaqueHosts.add(gate.host);
}
hits.push(described.hit ?? "");
sigs.push(described.sig ?? "");
infos.push(JSON.stringify(described.info));
docs.push(described.doc ?? "");
}
if (signal?.aborted) return { ok: false, error: STOPPED };
if (asked && !(await unchanged(tab, from, hits[0], infos[0]))) return { ok: false, error: CHANGED_UNDER_POINT };
// A handler can navigate mid-drag: the rest must not act on a page nobody gated.
const doc = docs[0];
const sameTab = () => {
const now = useBrowserStore.getState().tabs.find((candidate) => candidate.id === tab.id);
return Boolean(now && !now.loading && tabUrl(now) === before);
};
const since = Date.now();
const down = await pageCall(tab, { op: "pointer", phase: "down", ...from, expect: hits[0], sig: sigs[0] });
if (!down.ok) return { ok: false, error: down.error };
// Paced like a hand: pages often take one move per frame, or wait for a distance before a drag starts.
const steps = Math.min(25, Math.max(5, Math.round(Math.hypot(end.x - from.x, end.y - from.y) / DRAG_STEP_PX)));
let at = from;
let sent = performance.now();
for (let step = 1; step <= steps && !signal?.aborted; step++) {
await sleep(Math.max(0, DRAG_FRAME_MS - (performance.now() - sent)));
if (!sameTab()) break;
sent = performance.now();
// The last step lands on `end` exactly: interpolated floats can miss it by an ulp.
const next = step === steps ? end : { x: from.x + ((end.x - from.x) * step) / steps, y: from.y + ((end.y - from.y) * step) / steps };
const moved = await pageCall(tab, { op: "pointer", phase: "move", ...next }).catch(() => null);
if (!moved?.ok) break;
at = next;
}
const cancel = async (error: string): Promise<Result> => {
await pageCall(tab, { op: "pointer", phase: "cancel", ...from }).catch(() => null);
return { ok: false, error };
};
// A new document gets nothing; the same one (a hash or pushState move) has its drag cancelled.
const left = async (current?: string): Promise<Result> => {
const error = "the page changed during the drag, so it was not finished; take a snapshot.";
const now = current ?? (await pageCall(tab, { op: "status" }).catch(() => null))?.doc;
return doc && now === doc ? cancel(error) : { ok: false, error };
};
if (signal?.aborted) return cancel(STOPPED);
if (!doc || !sameTab()) return left();
// Drop zones can appear or relabel once a drag starts: re-ask unless the end is unchanged. Its doc
// says whether this is still the document the drag began on.
const target = await pageCall(tab, { op: "describe", ref: "", ...end }).catch(() => null);
if (target?.doc !== doc) return left(target?.doc);
if (at.x !== end.x || at.y !== end.y) return cancel("the drag stopped short; take a snapshot to see where things are.");
if (!target?.ok || !target.info) return cancel(target?.error ?? "nothing is at the drag's end.");
if (target.hit !== hits[1] || JSON.stringify(target.info) !== infos[1]) {
for (const gate of [gateElement("click", gateable(target.info, context), before, action.policy, permissions(context)), opaqueGate(target.info, before, context)]) {
if (!gate) continue;
const refused = await pass(gate, context);
if (refused) return cancel(refused);
if (gate.kind === "ask" && gate.ask === "pointer") context.session.opaqueHosts.add(gate.host);
}
const again = await pageCall(tab, { op: "describe", ref: "", ...end }).catch(() => null);
if (!sameTab() || again?.doc !== doc) return left(again?.doc);
if (!again?.ok || again.hit !== target.hit || JSON.stringify(again.info) !== JSON.stringify(target.info)) return cancel(CHANGED_UNDER_POINT);
}
if (signal?.aborted) return cancel(STOPPED);
const up = await pageCall(tab, { op: "pointer", phase: "up", ...end, expect: target.hit ?? "", sig: target.sig ?? "" });
if (!up.ok) return cancel(up.error ?? CHANGED_UNDER_POINT);
return afterward(tab, before, action, context, since, up.note);
}
function gateable(info: ElementInfo, context: Run): ElementInfo {
return context.inFile ? { ...info, formAction: "" } : info;
}
/** An unlabelled point (a canvas) may be a Buy button nothing can read: ask once per site per chat. */
function opaqueGate(info: ElementInfo, pageUrl: string, context: Run): (Gate & { kind: "ask" }) | null {
if (info.role !== "element" || info.name.trim() || info.href) return null;
const host = hostOfUrl(pageUrl);
const rule = ruleFor(pageUrl, permissions(context)).rule;
if (!host || context.session.opaqueHosts.has(host) || rule.actions === "allow" || rule.unlabelled === "allow") return null;
return { kind: "ask", ask: "pointer", host, name: "", detail: info.tag, prompt: `Click unlabelled parts of ${host}?` };
}
const CHANGED_UNDER_POINT = "the page changed under that point; take a screenshot to see it again.";
async function unchanged(tab: BrowserTab, point: { x: number; y: number }, hit: string, info: string): Promise<boolean> {
const again = await pageCall(tab, { op: "describe", ref: "", ...point }).catch(() => null);
return Boolean(again?.ok && again.hit === hit && JSON.stringify(again.info) === info);
}
const SCRIPT_WAIT_MS = 15_000;
const SCRIPTS_OFF = "The user doesn't let the agent run scripts on pages.";
/** A script bypasses every rule above, so it is shown verbatim and asked every time, whatever the site permissions. */
async function runScript(tab: BrowserTab, action: BrowserAction, context: Run, before: string): Promise<Result> {
const { signal } = context;
const script = action.script ?? "";
if (!script.trim()) return { ok: false, error: "the script is empty." };
// The document the user sees as they approve: a reload or a page at the same address is another one.
const doc = (await pageCall(tab, { op: "status" })).doc;
if (!doc) return { ok: false, error: "the page is not answering; take a snapshot." };
const refused = await pass({ kind: "ask", ask: "script", host: hostOfUrl(before), name: "", detail: script, prompt: "" }, context);
if (refused) return { ok: false, error: refused };
if (signal?.aborted) return { ok: false, error: STOPPED };
// Settings can change while the question is open.
if (useBrowserAgentStore.getState().scriptAccess === "never") return { ok: false, error: SCRIPTS_OFF };
const current = useBrowserStore.getState().tabs.find((candidate) => candidate.id === tab.id);
if (!current || current.loading || tabUrl(current) !== before) {
return { ok: false, error: "the page changed while the user was asked, so the script did not run; take a snapshot." };
}
const since = Date.now();
const id = `s${Date.now().toString(36)}${Math.random().toString(36).slice(2, 10)}`;
let started: AgentAnswer | null = null;
if (useNativeBrowser.getState().enabled) {
const reply = hasNativeView(tab.id)
? await callNative<{ installed: boolean; result: unknown }>("browser_view_eval", { tabId: tab.id, id, doc, code: script }).catch(
() => null,
)
: null;
if (reply && !reply.installed) {
return { ok: false, error: "the page changed while the user was asked, so the script did not run; take a snapshot." };
}
started = reply ? checkedAnswer(reply.result) : null;
} else {
started = await pageCall(tab, { op: "evaluate", id, doc, script });
}
if (!started) return { ok: false, error: "the script did not run: the page changed while the user was asked, or it has a syntax error." };
if (!started.ok) return { ok: false, error: started.error };
let outcome = `Still running after ${SCRIPT_WAIT_MS / 1000} s; its result is lost.`;
const until = Date.now() + SCRIPT_WAIT_MS;
while (Date.now() < until) {
if (signal?.aborted) return { ok: false, error: STOPPED };
const now = useBrowserStore.getState().tabs.find((candidate) => candidate.id === tab.id);
if (!now || now.loading || tabUrl(now) !== before) {
outcome = "The page moved on before the script finished.";
break;
}
const answer = await pageCall(tab, { op: "evaluated", id });
if (!answer.ok) return { ok: false, error: answer.error };
if (!answer.pending) {
outcome = answer.text ?? "";
break;
}
await sleep(100);
}
const after = await settle(tab.id, before, since, signal);
if (!after) return { ok: false, error: "the browser tab closed." };
const now = tabUrl(after);
if (now && now !== before) return arrive(after, before, action, context, since, outcome.slice(0, 900));
return { ok: true, url: now, title: after.customTitle ?? after.title, text: outcome };
}
/** Where a navigation ended up, checked before the model reads it: a redirect or a script can land
* on a blocked site, or on a new one the user never approved. */
async function arrive(
tab: BrowserTab,
requested: string,
action: BrowserAction,
context: Run,
since: number,
note?: string,
): Promise<Result> {
const url = tabUrl(tab);
const host = hostOfUrl(url);
if (url) {
const blocked = navigationBlocks(url, action.policy, permissions(context));
if (blocked) return { ok: false, error: `The page went to ${host}, but ${blocked}. Go back.` };
if (host && host !== hostOfUrl(requested) && !context.session.reachedHosts.has(host)) {
const gate = gateNavigation(url, { ...navigationContext(action, context), currentUrl: requested });
const refused = await pass(gate, context);
if (refused) return { ok: false, error: `The page went to ${host}. ${refused}` };
admit(url, gate, context);
}
if (host && context.fullAccess !== true) context.session.reachedHosts.add(host);
record(context.session, action.op, url);
}
return observeLoaded(tab, context, since, note);
}
/** The page the tab shows now, before the model reads or acts on it: the page may have moved on
* its own, or the user declined where a redirect landed and it is still loaded. */
async function stillAllowed(tab: BrowserTab, action: BrowserAction, context: Run): Promise<string | null> {
const url = tabUrl(tab);
const host = hostOfUrl(url);
if (!host) return null;
const blocked = navigationBlocks(url, action.policy, permissions(context));
if (blocked) return `The page is on ${host}, but ${blocked}. Use browser_navigate to go elsewhere.`;
if (context.session.reachedHosts.has(host)) return null;
const gate = gateNavigation(url, { ...navigationContext(action, context), currentUrl: "" });
const refused = await pass(gate, context);
if (refused) return `The page is on ${host}. ${refused}`;
admit(url, gate, context);
return null;
}
async function observeLoaded(tab: BrowserTab, context: Run, since: number, note?: string): Promise<Result> {
const entry = currentEntry(tab);
if (entry.kind === "web" && tab.documentType && !/html/i.test(tab.documentType)) {
return {
ok: true,
url: tabUrl(tab),
title: tab.title,
note: `${note ? `${note} ` : ""}This address is a file (${tab.documentType}), shown to the user; the browser tools can't read it. web_search with this url can.`,
};
}
if (entry.kind === "web" && !tab.pageError && !tab.nativeError && (await untilFilled(tab, since, context.signal))) {
note = [note, "The page still had requests open when read; snapshot again if something is missing."].filter(Boolean).join(" ");
}
// Read at the address its router (same site, shell-checked) moved to while it filled in.
const fresh = findTab(tab.id);
return observe(fresh && !fresh.loading && currentEntry(fresh) === entry ? fresh : tab, context, note);
}
async function screenshot(tab: BrowserTab, context: Run): Promise<Result> {
const entry = currentEntry(tab);
const kind = entry.kind === "file" ? await kindOf(entry) : null;
if (entry.kind === "file" && kind === "image") return imageFile(entry);
if (entry.kind === "file" && !chatOnScreen(context)) return { ok: false, error: OFF_SCREEN };
if (!isTauri) return { ok: false, error: "screenshots of the browser panel need the desktop app; use browser_snapshot." };
// A page at work off screen is captured on its own; one on screen as the panel shows it.
const hidden = !chatOnScreen(context) && hasNativeView(tab.id);
const pages = document.querySelectorAll<HTMLElement>("[data-browser-page]");
const element = pages[pages.length - 1];
if (!hidden && !element) return { ok: false, error: "the browser panel is not on screen." };
// What the user typed into card, code and password fields stays out of the picture, as out of the text.
const page = entry.kind === "web" || kind === "preview";
const masked: AgentAnswer | null = page ? await pageCall(tab, { op: "mask", on: true }).catch(() => null) : { ok: true };
if (!masked?.ok) return { ok: false, error: "the page could not be prepared for a screenshot; use browser_snapshot." };
let blob: Blob | null;
try {
await sleep(100);
blob = hidden || !element ? await captureHiddenPage(tab) : await screenshotPage(tab, element);
} finally {
if (page) await pageCall(tab, { op: "mask", on: false }).catch(() => null);
}
if (!blob) return { ok: false, error: "the screenshot failed." };
// A page that moved during the capture is one nobody checked or masked: no picture of it.
const now = findTab(tab.id);
if (!now || now.loading || tabUrl(now) !== tabUrl(tab) || currentEntry(now) !== entry) {
return { ok: false, error: "the page changed while the screenshot was taken; take a snapshot first." };
}
blob = await toPointScale(tab, context.session, blob, masked);
const image = await dataUrl(blob);
return { ok: true, url: tabUrl(tab), title: tab.title, ...(entry.kind === "file" ? { file: entry.name } : {}), image };
}
function dataUrl(blob: Blob): Promise<string> {
return new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(String(reader.result));
reader.onerror = () => reject(reader.error);
reader.readAsDataURL(blob);
});
}
// PNG: the server reads no SVG or AVIF.
async function imageFile(entry: FileEntry): Promise<Result> {
const blob = browserFile(entry.fileId);
const url = blob ? URL.createObjectURL(blob) : null;
if (!url) return { ok: false, error: NO_FILE };
try {
const image = new Image();
image.src = url;
await image.decode();
const width = image.naturalWidth || 300;
const height = image.naturalHeight || 150;
const fit = Math.min(1, MAX_SHOT_EDGE / Math.max(width, height));
const canvas = document.createElement("canvas");
canvas.width = Math.max(1, Math.round(width * fit));
canvas.height = Math.max(1, Math.round(height * fit));
canvas.getContext("2d")?.drawImage(image, 0, 0, canvas.width, canvas.height);
const png = await new Promise<Blob | null>((resolve) => canvas.toBlob(resolve, "image/png"));
return png ? { ok: true, file: entry.name, image: await dataUrl(png) } : { ok: false, error: "the image could not be read." };
} catch {
return { ok: false, error: "the image could not be read." };
} finally {
URL.revokeObjectURL(url);
}
}
/** At CSS size within mcp_images.MAX_IMAGE_EDGE, so x and y read off it map back exactly. */
async function toPointScale(tab: BrowserTab, session: Session, blob: Blob, size: AgentAnswer): Promise<Blob> {
let bitmap: ImageBitmap;
try {
bitmap = await createImageBitmap(blob);
} catch {
return blob;
}
try {
const cssWidth = size.width || bitmap.width;
const cssHeight = size.height || bitmap.height;
const fit = Math.min(1, MAX_SHOT_EDGE / Math.max(cssWidth, cssHeight));
const width = Math.max(1, Math.round(cssWidth * fit));
const height = Math.max(1, Math.round(cssHeight * fit));
const at: ShotView = {
url: tabUrl(tab),
width: size.width,
height: size.height,
scrollX: size.scrollX,
scrollY: size.scrollY,
scrolls: size.scrolls,
doc: size.doc,
};
session.pointScale.set(tab.id, { ...at, scaleX: cssWidth / bitmap.width, scaleY: cssHeight / bitmap.height });
if (width === bitmap.width && height === bitmap.height) return blob;
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
canvas.getContext("2d")?.drawImage(bitmap, 0, 0, width, height);
const scaled = await new Promise<Blob | null>((resolve) => canvas.toBlob(resolve, "image/png"));
// Width and height round apart, so each axis keeps its own scale.
if (scaled) session.pointScale.set(tab.id, { ...at, scaleX: cssWidth / width, scaleY: cssHeight / height });
return scaled ?? blob;
} finally {
bitmap.close();
}
}
const HISTORY_LIMIT = 50;
/** The browser_history tool: the user's visits, by their Settings choice (never, ask, allow). */
async function readHistory(action: BrowserAction, context: Run): Promise<Result> {
const saved = useBrowserAgentStore.getState().historyAccess;
const access = context.fullAccess && saved === "ask" ? "allow" : saved;
if (access === "never") return { ok: false, error: "The user doesn't let the agent read their browsing history." };
if (access === "ask") {
const query = action.query ?? "";
const allowed = await askInPanel({ ask: "history", host: "", name: query, detail: query }, context);
if (!allowed) return { ok: false, error: "The user declined to share their browsing history. Do something else, or ask them." };
}
const terms = (action.query ?? "").toLowerCase().split(/\s+/).filter(Boolean);
const limit = Math.min(Math.max(action.limit ?? 20, 1), HISTORY_LIMIT);
const visits = useBrowserHistoryStore
.getState()
.history.filter((visit) => {
const haystack = `${visit.title} ${visit.url}`.toLowerCase();
return terms.every((term) => haystack.includes(term));
})
.slice(0, limit);
const when = (at: number) => {
const date = new Date(at);
return Number.isNaN(date.getTime()) ? "(unknown time)" : date.toISOString().slice(0, 16).replace("T", " ");
};
const lines = visits.map((visit) => `${when(visit.visitedAt)} ${visit.title || "(untitled)"} ${visit.url}`);
return { ok: true, text: lines.length ? lines.join("\n") : "No pages in the browsing history match." };
}