Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: let browser use read, click and edit files in the browser panel (#16644)

Michael Han71c643e02a54
diff --git a/studio/backend/core/inference/browser_agent.py b/studio/backend/core/inference/browser_agent.py
index d05dbf99ca4..9727880087c 100644
--- a/studio/backend/core/inference/browser_agent.py
+++ b/studio/backend/core/inference/browser_agent.py
@@ -1,9 +1,10 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Lets the model drive the chat tab browser panel (user can watch, stop, take over; tab gates risky
-actions). Enforces argument shapes, http(s) only, public or this-machine hosts off Studio ports,
-website policy before and after navigation, and fenced untrusted page text."""
+"""Lets the model drive the chat tab browser panel, its web pages and the files it shows (user can
+watch, stop, take over; tab gates risky actions). Enforces argument shapes, http(s) only, public or
+this-machine hosts off Studio ports, website policy before and after navigation, and fenced
+untrusted page text."""
from __future__ import annotations
@@ -288,6 +289,81 @@ BROWSER_HISTORY_TOOL = {
},
}
+BROWSER_TABS_TOOL = {
+ "type": "function",
+ "function": {
+ "name": "browser_tabs",
+ "description": (
+ "List the tabs open in the browser panel, web pages and files alike (an HTML page or "
+ "React component made in this chat, a PDF, an image, code, text), or switch to one by "
+ "its number. The other browser tools then work in that tab: on an HTML or React file "
+ "they read and click its preview as on any page, and browser_read_file and "
+ "browser_edit_file read and change the file itself."
+ ),
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "tab": {
+ "type": "integer",
+ "description": "The number of the tab to switch to; leave out to only list.",
+ }
+ },
+ },
+ },
+}
+
+BROWSER_READ_FILE_TOOL = {
+ "type": "function",
+ "function": {
+ "name": "browser_read_file",
+ "description": (
+ "Read the file shown in the browser panel: the source of an HTML, React, Markdown, "
+ "code or text file, or the text of a PDF or document, with line numbers (not part "
+ "of the file). Works in the agent's tab, or the file tab in front if the agent's tab "
+ "is a web page."
+ ),
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "start_line": {
+ "type": "integer",
+ "description": "First line to read, from 1; for files too long for one read.",
+ }
+ },
+ },
+ },
+}
+
+BROWSER_EDIT_FILE_TOOL = {
+ "type": "function",
+ "function": {
+ "name": "browser_edit_file",
+ "description": (
+ "Change the HTML, React, Markdown, code or text file shown in the browser panel: "
+ "replace old_text, copied exactly from browser_read_file without the line numbers, "
+ "with new_text, or give content to rewrite the whole file. The preview reloads with "
+ "the change, and Back in the panel shows the version before it. PDFs, documents and "
+ "images can't be edited here. For files in the panel, not edit_file, which works on "
+ "the code sandbox's files."
+ ),
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "old_text": {
+ "type": "string",
+ "description": "Text to replace; it must appear once, unless replace_all.",
+ },
+ "new_text": {"type": "string"},
+ "replace_all": {"type": "boolean"},
+ "content": {
+ "type": "string",
+ "description": "The whole new file, instead of old_text and new_text.",
+ },
+ },
+ },
+ },
+}
+
BROWSER_TOOLS = [
BROWSER_NAVIGATE_TOOL,
BROWSER_SNAPSHOT_TOOL,
@@ -302,6 +378,9 @@ BROWSER_TOOLS = [
BROWSER_EVALUATE_TOOL,
BROWSER_SCREENSHOT_TOOL,
BROWSER_HISTORY_TOOL,
+ BROWSER_TABS_TOOL,
+ BROWSER_READ_FILE_TOOL,
+ BROWSER_EDIT_FILE_TOOL,
]
BROWSER_TOOL_NAMES = frozenset(tool["function"]["name"] for tool in BROWSER_TOOLS)
@@ -314,6 +393,8 @@ _MIN_ACTION_TIMEOUT_S = 30.0
_MAX_URL_CHARS = 4096
_MAX_TEXT_CHARS = 8000
_MAX_SCRIPT_CHARS = 8000
+_MAX_FILE_EDIT_CHARS = 500_000
+_MAX_TAB = 1000
_HIDDEN_TAIL_RE = re.compile(r"(?:[ \t]*\r?\n){4,}|[ \t]{200,}")
# Past any real screen: a point there is a mistake, not a far corner.
_MAX_POINT = 100_000
@@ -589,9 +670,63 @@ def _validated_request(name: str, arguments: dict, website_policy: Optional[dict
"query": str(arguments.get("query") or "").strip()[:200],
"limit": max(1, min(limit, 50)),
}
+ if name == "browser_tabs":
+ tab = _whole_number(arguments, "tab")
+ if tab is not None and not 1 <= tab <= _MAX_TAB:
+ raise ValueError("tab must be the number of a tab in the list.")
+ return {"op": "tabs", **({"tab": tab} if tab is not None else {})}
+ if name == "browser_read_file":
+ start = _whole_number(arguments, "start_line") or 1
+ if start < 1:
+ raise ValueError("start_line counts from 1.")
+ return {"op": "read_file", "start_line": start}
+ if name == "browser_edit_file":
+ return {"op": "edit_file", **_file_edit(arguments)}
raise ValueError(f"unknown browser tool {name}.")
+def _whole_number(arguments: dict, key: str) -> Optional[int]:
+ raw = arguments.get(key)
+ if raw is None or raw == "":
+ return None
+ try:
+ value = float(raw)
+ except (TypeError, ValueError, OverflowError):
+ raise ValueError(f"{key} must be a whole number.")
+ if isinstance(raw, bool) or not value.is_integer():
+ raise ValueError(f"{key} must be a whole number.")
+ return int(value)
+
+
+def _file_edit(arguments: dict) -> dict:
+ def text(key: str) -> Optional[str]:
+ value = arguments.get(key)
+ if value is None:
+ return None
+ if not isinstance(value, str):
+ raise ValueError(f"{key} must be a string.")
+ if len(value) > _MAX_FILE_EDIT_CHARS:
+ raise ValueError(f"{key} is longer than {_MAX_FILE_EDIT_CHARS} characters.")
+ return value
+
+ content = text("content")
+ old = text("old_text")
+ new = text("new_text")
+ if content is not None:
+ if old is not None or new is not None:
+ raise ValueError("give either content, or old_text and new_text, not both.")
+ return {"content": content}
+ if not old:
+ raise ValueError(
+ "old_text is empty. Give the text to replace, or content for the whole file."
+ )
+ if new is None:
+ raise ValueError("new_text is required with old_text; an empty string deletes old_text.")
+ if old == new:
+ raise ValueError("old_text and new_text are the same, so nothing would change.")
+ return {"old_text": old, "new_text": new, "replace_all": arguments.get("replace_all") is True}
+
+
def _clip(value: Any, limit: int) -> str:
return value[:limit] if isinstance(value, str) else ""
@@ -634,18 +769,23 @@ def format_result(name: str, result: dict, website_policy: Optional[dict]) -> st
"disallows, so its content is withheld. Navigate somewhere else."
)
title = _one_line(_clip(result.get("title"), 300))
+ file = _one_line(_clip(result.get("file"), 300))
note = _one_line(_clip(result.get("note"), 1000))
text = _clip(result.get("text"), 400_000)
if len(text) > _MAX_MODEL_CHARS:
- text = (
- text[:_MAX_MODEL_CHARS]
- + "\n... (cut here: use browser_find, browser_scroll or a snapshot ref to see more)"
+ more = (
+ "browser_read_file with a later start_line"
+ if name == "browser_read_file"
+ else "browser_find, browser_scroll or a snapshot ref"
)
+ text = text[:_MAX_MODEL_CHARS] + f"\n... (cut here: use {more} to see more)"
# The title and note come from the page too (a selected option's words), so everything but
# this tool's own framing goes inside the fence.
page = []
if url:
page.append(f"Page: {title + ' ' if title else ''}<{url}>")
+ elif file:
+ page.append(f"File: {file}")
if note:
page.append(note)
if text:
diff --git a/studio/backend/core/inference/tool_loop_controller.py b/studio/backend/core/inference/tool_loop_controller.py
index 97a04455595..700837fbf42 100644
--- a/studio/backend/core/inference/tool_loop_controller.py
+++ b/studio/backend/core/inference/tool_loop_controller.py
@@ -884,6 +884,12 @@ def _browser_status(tool_name: str, arguments: Mapping[str, Any]) -> str:
return "Moving the mouse"
if tool_name == "browser_drag":
return "Dragging"
+ if tool_name == "browser_tabs":
+ return "Switching tabs" if text("tab") else "Listing browser tabs"
+ if tool_name == "browser_read_file":
+ return "Reading the file"
+ if tool_name == "browser_edit_file":
+ return "Editing the file"
return f"Calling: {tool_name}"
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 21231dbb389..cf1b9dfaebd 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -3942,6 +3942,35 @@ _ARTIFACT_PREVIEW_FRAME_HTML = """<!doctype html>
const annotationFactory = (post) => {
/* ANNOTATE_JS */
};
+ // No eval under this CSP: the page agent arrives as an inline script, on a one-time key.
+ const buildAgent = (() => {
+ let apply, create, root, setText, append, detach, random, join, Words;
+ try {
+ apply = Reflect.apply;
+ create = Document.prototype.createElement;
+ root = Object.getOwnPropertyDescriptor(Document.prototype, "documentElement").get;
+ setText = Object.getOwnPropertyDescriptor(HTMLScriptElement.prototype, "text").set;
+ append = Node.prototype.appendChild;
+ detach = Element.prototype.remove;
+ random = Crypto.prototype.getRandomValues;
+ join = Array.prototype.join;
+ Words = Uint32Array;
+ } catch {
+ return () => null;
+ }
+ return (code) => {
+ const parent = apply(root, document, []);
+ if (!parent) return null;
+ const key = "unslothAgent" + apply(join, apply(random, crypto, [new Words(4)]), ["_"]);
+ const script = apply(create, document, ["script"]);
+ apply(setText, script, ["window." + key + " = " + code]);
+ apply(append, parent, [script]);
+ apply(detach, script, []);
+ const agent = window[key];
+ delete window[key];
+ return agent && typeof agent.run === "function" ? agent : null;
+ };
+ })();
// Named for canvasStack() in the frontend, which trims from this frame down.
const unslothRenderArtifact = (html) => {
installStorageFallbacks();
@@ -3955,6 +3984,7 @@ _ARTIFACT_PREVIEW_FRAME_HTML = """<!doctype html>
document.close();
// Instantiate only on a parent command; the factory predates the page's meta CSP.
let annotation = null;
+ let agent = null;
const postAnnotation = (message) => parent.postMessage({ source: "unsloth-browser", ...message }, "*");
window.addEventListener("message", (event) => {
if (event.source !== parent) return;
@@ -3967,6 +3997,10 @@ _ARTIFACT_PREVIEW_FRAME_HTML = """<!doctype html>
else annotation?.stop();
} else if (data.command === "annotateForget") annotation?.forget(Number(data.id));
else if (data.command === "annotateNumbers") annotation?.number(data.numbers);
+ else if (data.command === "agent" && typeof data.id === "string") {
+ if (!agent && typeof data.code === "string" && data.code.length <= 262144) agent = buildAgent(data.code);
+ postAnnotation({ type: "agent", id: data.id, installed: Boolean(agent), result: agent ? agent.run(data.op || {}) : null });
+ }
});
postAnnotation({ type: "annotate", event: "ready" });
// document.open() drops listeners bound before it, so rebind here.
diff --git a/studio/backend/tests/test_browser_agent.py b/studio/backend/tests/test_browser_agent.py
index c91d18b4d7e..ad250b265ac 100644
--- a/studio/backend/tests/test_browser_agent.py
+++ b/studio/backend/tests/test_browser_agent.py
@@ -564,3 +564,76 @@ def test_redirect_target_honours_the_website_policy():
"op": "navigate",
"url": "https://www.google.com/url?q=https://unsloth.ai/",
}
+
+
+def test_file_tools_take_bounded_arguments():
+ assert _request("browser_tabs", {}) == {"op": "tabs"}
+ assert _request("browser_tabs", {"tab": 2}) == {"op": "tabs", "tab": 2}
+ assert _request("browser_tabs", {"tab": "3"}) == {"op": "tabs", "tab": 3}
+ for bad in (0, -1, 1.5, True, "two", 5000):
+ with pytest.raises(ValueError):
+ _request("browser_tabs", {"tab": bad})
+ assert _request("browser_read_file", {}) == {"op": "read_file", "start_line": 1}
+ assert _request("browser_read_file", {"start_line": 40}) == {
+ "op": "read_file",
+ "start_line": 40,
+ }
+ with pytest.raises(ValueError, match = "from 1"):
+ _request("browser_read_file", {"start_line": -3})
+
+ assert _request("browser_edit_file", {"old_text": "<h1>", "new_text": "<h2>"}) == {
+ "op": "edit_file",
+ "old_text": "<h1>",
+ "new_text": "<h2>",
+ "replace_all": False,
+ }
+ assert _request("browser_edit_file", {"old_text": "x", "new_text": ""})["new_text"] == ""
+ assert _request("browser_edit_file", {"content": ""}) == {"op": "edit_file", "content": ""}
+ for arguments, message in (
+ ({}, "old_text is empty"),
+ ({"old_text": "a"}, "new_text is required"),
+ ({"old_text": "a", "new_text": "a"}, "the same"),
+ ({"content": "x", "old_text": "a", "new_text": "b"}, "not both"),
+ ({"content": 5}, "must be a string"),
+ ({"content": "x" * 500_001}, "longer than"),
+ ):
+ with pytest.raises(ValueError, match = message):
+ _request("browser_edit_file", arguments)
+
+
+def test_file_results_name_the_file_inside_the_fence():
+ out = browser_agent.format_result(
+ "browser_read_file",
+ {"ok": True, "file": "page.html", "text": "1\t<h1>Hi</h1>", "note": "Lines 1-1 of 1."},
+ None,
+ )
+ assert out.index("File: page.html") > out.index("<untrusted_web_content>")
+ assert "1\t<h1>Hi</h1>" in out
+ long = browser_agent.format_result(
+ "browser_read_file", {"ok": True, "file": "a.txt", "text": "x" * 50_000}, None
+ )
+ assert "browser_read_file with a later start_line" in long
+
+
+def test_file_tools_have_status_badges_and_are_browser_tools():
+ from core.inference.tool_loop_controller import status_for_tool
+
+ for name in ("browser_tabs", "browser_read_file", "browser_edit_file"):
+ assert browser_agent.is_browser_tool(name)
+ assert status_for_tool("browser_tabs", {}) == "Listing browser tabs"
+ assert status_for_tool("browser_tabs", {"tab": 2}) == "Switching tabs"
+ assert status_for_tool("browser_read_file", {}) == "Reading the file"
+ assert (
+ status_for_tool("browser_edit_file", {"old_text": "a", "new_text": "b"})
+ == "Editing the file"
+ )
+
+
+def test_the_artifact_frame_installs_the_agent_without_eval():
+ from routes.inference import _ARTIFACT_PREVIEW_FRAME_STRICT_CSP, _artifact_preview_frame_html
+
+ html = _artifact_preview_frame_html()
+ assert 'data.command === "agent"' in html
+ assert "buildAgent(data.code)" in html
+ assert "unsafe-eval" not in _ARTIFACT_PREVIEW_FRAME_STRICT_CSP
+ assert "new Function" not in html and "eval(" not in html
diff --git a/studio/frontend/src/features/browser/agent/agent-store.ts b/studio/frontend/src/features/browser/agent/agent-store.ts
index cd1e81cda6e..fbcd73e4029 100644
--- a/studio/frontend/src/features/browser/agent/agent-store.ts
+++ b/studio/frontend/src/features/browser/agent/agent-store.ts
@@ -40,7 +40,10 @@ export type AgentStep = {
| "evaluate"
| "history"
| "move"
- | "drag";
+ | "drag"
+ | "tabs"
+ | "readFile"
+ | "editFile";
detail: string;
};
diff --git a/studio/frontend/src/features/browser/agent/executor.ts b/studio/frontend/src/features/browser/agent/executor.ts
index 289c5e40c58..c312b0d597e 100644
--- a/studio/frontend/src/features/browser/agent/executor.ts
+++ b/studio/frontend/src/features/browser/agent/executor.ts
@@ -8,7 +8,7 @@ import { authFetch } from "@/features/auth";
import { openChatItemById, useChatNavigationStore, visibleChatItems } from "@/features/chat";
import { getLocale, translate } from "@/i18n";
import { toast } from "@/lib/toast";
-import { isTauri } from "@/lib/api-base";
+import { isStudioUrl, isTauri } from "@/lib/api-base";
import { unwrapRedirect } from "../address";
import { captureHiddenPage, screenshotPage } from "../capture";
import { useBrowserHistoryStore } from "../history-store";
@@ -17,7 +17,7 @@ import { isLoopbackUrl } from "../address";
import { callNative, localOriginGranted, useNativeBrowser } from "../native-support";
import { requestFrameAgent, sendFrameCommand } from "../page-frame";
import { browserPanelAvailable } from "../panel-availability";
-import { type BrowserTab, chatKeyOfThread, currentEntry, onChatAdopted, useBrowserStore } from "../store";
+import { type BrowserEntry, type BrowserTab, browserFile, chatKeyOfThread, currentEntry, onChatAdopted, useBrowserStore } from "../store";
import {
type AgentStep,
type Answer,
@@ -33,6 +33,7 @@ import {
import { type Gate, type Question, type SitePolicy, gateElement, gateNavigation, hostOfUrl, navigationBlocks } from "./gate";
import { type SiteRule, ruleFor } from "./permissions";
import { type AgentAnswer, type ElementInfo, pageAgentSource } from "./page-agent";
+import { type FileTabKind, MAX_EDIT_BYTES, applyFileEdit, numberedLines } from "./file-tab";
import { waitForPage } from "./page-wait";
export type BrowserAction = {
@@ -55,6 +56,12 @@ export type BrowserAction = {
fullPage?: boolean;
query?: string;
limit?: number;
+ tab?: number;
+ startLine?: number;
+ oldText?: string;
+ newText?: string;
+ replaceAll?: boolean;
+ content?: string;
/** How long the server waits for this answer. */
timeoutS?: number;
policy: SitePolicy;
@@ -73,9 +80,9 @@ export type ActionContext = {
signal?: AbortSignal;
};
-type Run = ActionContext & { session: Session };
+type Run = ActionContext & { session: Session; inFile?: boolean };
-type Result = { ok: boolean; error?: string; url?: string; title?: string; text?: string; note?: string; image?: string };
+type Result = { ok: boolean; error?: string; url?: string; title?: string; file?: string; text?: string; note?: string; image?: string };
const LOAD_TIMEOUT_MS = 30_000;
const SETTLE_QUIET_MS = 350;
@@ -84,6 +91,8 @@ const FILL_QUIET_MAX_MS = 1_000;
// A loaded page that opened no request has drawn what it will; content still on its way arrives by a request.
const FILL_IDLE_QUIET_MS = 120;
const PAGE_READY_MS = 4_000;
+const FILE_FRAME_TRY_MS = 1_500;
+const READ_OPS = new Set(["snapshot", "find", "status", "describe", "locate"]);
const POST_TIMEOUT_MS = 15_000;
const EXPIRED = "Not done: the action ran out of time, likely waiting for the user to answer in the panel. Ask them before trying again.";
export const STOPPED = "The user stopped the browser agent. Do not use browser tools again unless they ask.";
@@ -92,6 +101,10 @@ const str = (value: unknown, max: number): string | undefined =>
typeof value === "string" && value.length <= max ? value : undefined;
const num = (value: unknown): number | undefined =>
typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined;
+const whole = (value: unknown): number | undefined =>
+ typeof value === "number" && Number.isInteger(value) && value > 0 ? value : undefined;
+// browser_agent._MAX_FILE_EDIT_CHARS
+const MAX_EDIT_CHARS = 500_000;
const strings = (value: unknown): string[] =>
Array.isArray(value) ? value.filter((item): item is string => typeof item === "string").slice(0, 100) : [];
@@ -122,6 +135,12 @@ export function parseBrowserAction(value: unknown): BrowserAction | null {
fullPage: raw.full_page === true,
query: str(raw.query, 256),
limit: typeof raw.limit === "number" && Number.isInteger(raw.limit) ? raw.limit : undefined,
+ tab: whole(raw.tab),
+ startLine: whole(raw.start_line),
+ oldText: str(raw.old_text, MAX_EDIT_CHARS),
+ newText: str(raw.new_text, MAX_EDIT_CHARS),
+ replaceAll: raw.replace_all === true,
+ content: str(raw.content, MAX_EDIT_CHARS),
timeoutS: typeof raw.timeout_s === "number" && raw.timeout_s > 0 ? Math.min(raw.timeout_s, 3600) : undefined,
policy: { allowedDomains: strings(raw.allowed_domains), blockedDomains: strings(raw.blocked_domains) },
};
@@ -305,7 +324,7 @@ function tabUrl(tab: BrowserTab): string {
* page works on in the background, with the user's own tabs left where they were. */
function showTab(tab: BrowserTab, context: Run): void {
// Kept from the start: the user can switch chats mid-run, and closing the panel then must leave it.
- if (useNativeBrowser.getState().enabled) keepInBackground(tab.id);
+ if (useNativeBrowser.getState().enabled && currentEntry(tab).kind === "web") keepInBackground(tab.id);
// Off screen it shows when the user goes back to that chat; their tabs here stay put. A chat on
// screen without room for the panel isn't given one.
const store = useBrowserStore.getState();
@@ -452,7 +471,8 @@ async function pageCall(tab: BrowserTab, request: Record<string, unknown>): Prom
cursor,
...(cursor ? { cursorColor: cursorAccent(), cursorStill: reducedMotion() } : {}),
};
- const native = useNativeBrowser.getState().enabled;
+ // File previews are frames, never native views.
+ const native = useNativeBrowser.getState().enabled && currentEntry(tab).kind === "web";
const started = Date.now();
const fresh = () => {
const base = nextBase;
@@ -472,8 +492,10 @@ async function pageCall(tab: BrowserTab, request: Record<string, unknown>): Prom
if (reply && !reply.installed) reply = await call(fresh(), pageAgentSource());
}
} else {
- reply = await requestFrameAgent(tab.id, command);
- if (reply && !reply.installed) reply = await requestFrameAgent(tab.id, fresh(), pageAgentSource());
+ // Short tries for reads only: they are safe to repeat.
+ const wait = currentEntry(tab).kind === "file" && READ_OPS.has(String(request.op)) ? FILE_FRAME_TRY_MS : undefined;
+ reply = await requestFrameAgent(tab.id, command, undefined, wait);
+ if (reply && !reply.installed) reply = await requestFrameAgent(tab.id, fresh(), pageAgentSource(), wait);
}
const answer = reply?.installed ? checkedAnswer(reply.result) : null;
if (answer) return answer;
@@ -578,6 +600,7 @@ async function askInPanel(question: Question, context: Run, remember = true): Pr
async function observe(tab: BrowserTab, context: Run, note?: string, op: Record<string, unknown> = { op: "snapshot" }): Promise<Result> {
const entry = currentEntry(tab);
+ if (entry.kind === "file") return observeFile(tab, entry, context, note, op);
if (entry.kind !== "web") return { ok: false, error: "No web page is open. Use browser_navigate first." };
if (tab.pageError || tab.nativeError) {
return {
@@ -626,7 +649,7 @@ async function open(url: string, context: Run): Promise<BrowserTab | null> {
const agent = agentTab(context);
const existing = isLoopbackUrl(url)
? agent && localOriginGranted(agent.id, url) ? agent : grantedLocalTab(url, context.session.chat)
- : agent && !onGrantedOrigin(agent) ? agent : undefined;
+ : agent && !onGrantedOrigin(agent) && currentEntry(agent).kind !== "file" ? agent : undefined;
if (existing) {
if (agentChat(context.session.chat).tabId !== existing.id) moveRun(context, existing.id);
store.navigate(existing.id, { url, temporary: context.temporary }, { replace: false });
@@ -666,7 +689,12 @@ function stepOf(action: BrowserAction): AgentStep {
case "history":
case "move":
case "drag":
+ case "tabs":
return { kind: action.op, detail: "" };
+ case "read_file":
+ return { kind: "readFile", detail: "" };
+ case "edit_file":
+ return { kind: "editFile", detail: "" };
default:
return { kind: "read", detail: "" };
}
@@ -678,8 +706,12 @@ function nameStep(chat: string, info: ElementInfo): void {
patchAgentChat(chat, { step: { ...current, detail: info.name || info.role || current.detail } });
}
-function permissions(context: Pick<ActionContext, "fullAccess">) {
- const saved = useBrowserAgentStore.getState().permissions;
+function permissions(context: Pick<Run, "fullAccess" | "inFile">) {
+ const stored = useBrowserAgentStore.getState().permissions;
+ // Preview forms can't send (form-action 'none'); links are still gated.
+ const saved = context.inFile
+ ? { ...stored, defaults: { ...stored.defaults, actions: "allow" as const, unlabelled: "allow" as const } }
+ : stored;
if (!context.fullAccess) return saved;
const allow = (rule: SiteRule): SiteRule => ({
browsing: rule.browsing === "block" ? "block" : "allow",
@@ -738,7 +770,7 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {
if (agentChat(chat).stoppedRun === context.runKey) return { ok: false, error: STOPPED };
if (signal?.aborted) return { ok: false, error: "the action expired before it could run." };
// Off screen, only a native page can work on: a framed one runs in the panel.
- if (!chatOnScreen(context) && !useNativeBrowser.getState().enabled) {
+ if (!PANEL_FREE.has(action.op) && !chatOnScreen(context) && !useNativeBrowser.getState().enabled) {
return {
ok: false,
error:
@@ -751,36 +783,29 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {
if (owner && owner !== context.runKey) {
return { ok: false, error: "Another reply in this chat is using the browser right now. Wait for it to finish, or tell the user." };
}
+ const front = owner ? null : frontTab(chat);
+ if (front && currentEntry(front).kind === "file" && agentChat(chat).tabId !== front.id) moveRun(context, front.id);
patchAgentChat(chat, { working: true, runKey: context.runKey, step: stepOf(action) });
- if (action.op === "navigate") {
- const url = redirectEnd(action.url ?? "");
- if (isLoopbackUrl(url) && !grantedLocalTab(url, chat)) {
- const { host } = new URL(url);
- return {
- ok: false,
- error: useNativeBrowser.getState().enabled
- ? `${host} is on the user's computer, and the browser panel opens a local address only in the tab where the user types it in this chat's address bar. Ask the user to open it there first.`
- : `${host} is a local address, and the browser only opens public sites.`,
- };
- }
- const gate = gateNavigation(url, navigationContext(action, context));
- const refused = await pass(gate, context);
- if (refused) return { ok: false, error: refused };
- admit(url, gate, context);
- const since = Date.now();
- const tab = await open(url, context);
- if (!tab) return { ok: false, error: "the browser tab closed." };
- return arrive(tab, url, action, context, since);
- }
+ if (action.op === "navigate") return navigateTo(action.url ?? "", action, context);
if (action.op === "history") return readHistory(action, context);
if (action.op === "evaluate" && useBrowserAgentStore.getState().scriptAccess === "never") {
return { ok: false, error: SCRIPTS_OFF };
}
- const shown = agentTab(context);
- if (!shown) return { ok: false, error: "No page is open yet. Call browser_navigate first." };
+ if (action.op === "tabs") return listTabs(action, context);
+ if (action.op === "read_file" || action.op === "edit_file") {
+ const target = fileTab(context);
+ if (!target) return { ok: false, error: NO_FILE };
+ useBrowserStore.getState().revealIn(chat, target.id);
+ return action.op === "read_file" ? readFile(target, action.startLine ?? 1) : editFile(target, action, signal);
+ }
+
+ const shown = agentTab(context) ?? fileTab(context);
+ if (!shown) {
+ return { ok: false, error: "No page is open yet. Call browser_navigate first, or browser_tabs to work in a tab already open." };
+ }
showTab(shown, context);
// A kept page the cap closed reopens a beat later; its history flags are only good once it has.
if (useNativeBrowser.getState().enabled && currentEntry(shown).kind === "web" && !hasNativeView(shown.id)) {
@@ -791,7 +816,7 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {
const before = tabUrl(tab);
if (action.op === "back" || action.op === "forward" || action.op === "reload") {
- const native = useNativeBrowser.getState().enabled;
+ const native = useNativeBrowser.getState().enabled && currentEntry(tab).kind === "web";
const store = useBrowserStore.getState();
const nativeMove = native && action.op !== "reload" && Boolean(tab.nativeHistory?.[action.op]);
const step = action.op === "back" ? -1 : action.op === "forward" ? 1 : 0;
@@ -854,6 +879,19 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {
const here = tabUrl(ready);
const moved = await stillAllowed(ready, action, context);
if (moved) return { ok: false, error: moved };
+ const file = currentEntry(ready);
+ if (file.kind === "file") {
+ const preview = (await kindOf(file)) === "preview";
+ if (!preview && action.op !== "snapshot" && action.op !== "find" && action.op !== "screenshot") return { ok: false, error: NO_PAGE_IN_FILE };
+ if (action.op === "evaluate") return { ok: false, error: "scripts don't run in a file's preview; change the file with browser_edit_file instead." };
+ if (preview && action.op !== "snapshot" && action.op !== "find") {
+ if (!chatOnScreen(context)) return { ok: false, error: OFF_SCREEN };
+ context.inFile = true;
+ showPreview(ready);
+ const warm = await pageCall(ready, { op: "status" }).catch((error: Error) => ({ ok: false, error: error.message }));
+ if (!warm.ok) return { ok: false, error: warm.error ?? NO_PAGE_IN_FILE };
+ }
+ }
switch (action.op) {
case "snapshot":
@@ -895,27 +933,30 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {
return { ok: false, error: described.error ?? (point ? "nothing is at that point." : `${ref} could not be found.`) };
}
const info: ElementInfo = described.info;
+ const gated = gateable(info, context);
if (action.op !== "press") nameStep(chat, action.op === "select" && info.option ? { ...info, name: info.option } : info);
const gates: Gate[] = [];
- if (action.op === "click" || action.op === "type") gates.push(gateElement(action.op, info, here, action.policy, permissions(context)));
+ if (action.op === "click" || action.op === "type") gates.push(gateElement(action.op, gated, here, action.policy, permissions(context)));
const opaque = point ? opaqueGate(info, here, context) : null;
if (opaque) gates.push(opaque);
if (action.op === "select") {
// The option the page will select: its value can differ from the label the user sees.
- const chosen = info.role === "combobox" ? { ...info, name: info.option || action.value || "" } : info;
+ const chosen = info.role === "combobox" ? { ...gated, name: info.option || action.value || "" } : gated;
// A card-expiry or other secret dropdown is the user's to fill, as its text field would be.
- if (info.tag === "select" || info.role === "combobox") gates.push(gateElement("type", info, here, action.policy, permissions(context)));
+ if (info.tag === "select" || info.role === "combobox") gates.push(gateElement("type", gated, here, action.policy, permissions(context)));
gates.push(gateElement("select", chosen, here, action.policy, permissions(context)));
}
// Enter or Space on a focused button or link is a click on it, so it asks what a click would.
const activates = action.op === "press" && (action.key === "Enter" || action.key === "Space");
- if (activates && !TEXT_ROLES.has(info.role)) gates.push(gateElement("click", info, here, action.policy, permissions(context)));
+ // The frame would follow a pressed link unchecked.
+ if (context.inFile && activates && info.role === "link") return { ok: false, error: "click the link instead of pressing a key on it." };
+ if (activates && !TEXT_ROLES.has(info.role)) gates.push(gateElement("click", gated, here, action.policy, permissions(context)));
// Deleting is editing: a password or card field the agent can't type into it can't empty either.
if (action.op === "press" && (action.key === "Backspace" || action.key === "Delete")) {
- gates.push(gateElement("type", info, here, action.policy, permissions(context)));
+ gates.push(gateElement("type", gated, here, action.policy, permissions(context)));
// Outside a text field it is an app's delete shortcut, asked like a Delete button.
if (!TEXT_ROLES.has(info.role)) {
- gates.push(gateElement("click", { ...info, name: `delete key on ${info.name || info.role}` }, here, action.policy, permissions(context)));
+ gates.push(gateElement("click", { ...gated, name: `delete key on ${info.name || info.role}` }, here, action.policy, permissions(context)));
}
}
// Enter in a field sends its form, so it is held to what a submit button would be.
@@ -926,6 +967,7 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {
// A link the page follows itself (a native view's click, Enter on a link) is gated before it
// loads; the proxied frame's clicks come back as `navigate` and are gated below.
const followsLink =
+ !context.inFile &&
/^https?:/i.test(info.href) &&
info.href.split("#")[0] !== here.split("#")[0] &&
((action.op === "click" && useNativeBrowser.getState().enabled) || (activates && info.role === "link"));
@@ -938,7 +980,7 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {
const wrapperGates = wrappers.map((url) => gateNavigation(url, navigationContext(action, context)));
gates.push(...wrapperGates);
// A form sending to another site carries what was typed there: a new site asks, as a navigation would.
- const formAction = submits && info.formAction ? info.formAction : "";
+ const formAction = submits && gated.formAction ? gated.formAction : "";
const formGate =
formAction && hostOfUrl(formAction) !== hostOfUrl(here) ? gateNavigation(formAction, navigationContext(action, context)) : null;
if (formAction) {
@@ -971,9 +1013,10 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {
op: "click",
ref,
...(point ? { ...point, expect: described.hit ?? "", sig: described.sig ?? "" } : {}),
- directLinks: !useNativeBrowser.getState().enabled,
+ directLinks: context.inFile === true || !useNativeBrowser.getState().enabled,
});
if (!clicked.ok) return { ok: false, error: clicked.error };
+ if (clicked.navigate && context.inFile) return openFromFile(clicked.navigate, action, context);
if (clicked.navigate) {
const pressedAt = Date.now();
// A link: opened in this tab, under the same rules as any other address. Its href is read
@@ -1017,6 +1060,224 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {
return observe(after, context, note);
}
+async function navigateTo(address: string, action: BrowserAction, context: Run): Promise<Result> {
+ const url = redirectEnd(address);
+ if (isLoopbackUrl(url) && !grantedLocalTab(url, context.session.chat)) {
+ const { host } = new URL(url);
+ return {
+ ok: false,
+ error: useNativeBrowser.getState().enabled
+ ? `${host} is on the user's computer, and the browser panel opens a local address only in the tab where the user types it in this chat's address bar. Ask the user to open it there first.`
+ : `${host} is a local address, and the browser only opens public sites.`,
+ };
+ }
+ const gate = gateNavigation(url, navigationContext(action, context));
+ const refused = await pass(gate, context);
+ if (refused) return { ok: false, error: refused };
+ admit(url, gate, context);
+ const since = Date.now();
+ const tab = await open(url, context);
+ if (!tab) return { ok: false, error: "the browser tab closed." };
+ return arrive(tab, url, action, context, since);
+}
+
+const PANEL_FREE = new Set(["tabs", "read_file", "edit_file"]);
+const NO_FILE = "No file is open in the browser panel. browser_tabs lists the tabs.";
+const NO_PAGE_IN_FILE =
+ "This tab shows a file with no page to act on. browser_read_file reads it and browser_edit_file changes it.";
+const OFF_SCREEN = "this chat's browser only runs while the chat is open here; ask the user to switch back to it.";
+
+type FileEntry = Extract<BrowserEntry, { kind: "file" }>;
+
+let readerModule: Promise<typeof import("./file-reader")> | null = null;
+const fileReader = () => {
+ readerModule ??= import("./file-reader");
+ return readerModule;
+};
+
+async function kindOf(entry: FileEntry): Promise<FileTabKind> {
+ return (await fileReader()).fileTabKind(entry);
+}
+
+const KIND_LABELS: Record<FileTabKind, string> = {
+ preview: "HTML or React, with a preview",
+ text: "text",
+ document: "document",
+ image: "image",
+ media: "audio or video",
+ other: "file",
+};
+
+async function fileHint(entry: FileEntry): Promise<string> {
+ const reader = await fileReader();
+ const kind = reader.fileTabKind(entry);
+ const hints: string[] = [];
+ if (kind === "preview") hints.push("Its preview works as a page: browser_snapshot, browser_click and browser_type act in it.");
+ if (kind === "image") hints.push(isTauri ? "browser_screenshot shows it." : "Looking at an image needs the desktop app.");
+ if (reader.editable(entry, kind)) hints.push("browser_read_file reads its source and browser_edit_file changes it.");
+ else if (kind === "document") hints.push("browser_read_file reads its text; it can't be edited here.");
+ else if (kind !== "image") hints.push("It has no text to read.");
+ return hints.join(" ");
+}
+
+function showPreview(tab: BrowserTab): void {
+ useBrowserStore.getState().setFileView(tab.id, { mode: "preview" });
+}
+
+function frontTab(chat: string): BrowserTab | null {
+ const store = useBrowserStore.getState();
+ const id = chat === store.chat ? store.activeTabId : store.chats[chat]?.activeTabId;
+ return store.tabs.find((tab) => tab.id === id && tab.chat === chat) ?? null;
+}
+
+function fileTab(context: Run): BrowserTab | null {
+ const agent = agentTab(context);
+ if (agent && currentEntry(agent).kind === "file") return agent;
+ const front = frontTab(context.session.chat);
+ if (!front || currentEntry(front).kind !== "file") return null;
+ moveRun(context, front.id);
+ return front;
+}
+
+/** Web tabs by site alone: a title is browsing history. */
+async function listTabs(action: BrowserAction, context: Run): Promise<Result> {
+ const { chat } = context.session;
+ const tabs = useBrowserStore.getState().tabs.filter((tab) => tab.chat === chat);
+ const where = (tab: BrowserTab) => `a web page on ${hostOfUrl(tabUrl(tab)) || "this computer"}`;
+ if (action.tab !== undefined) {
+ const tab = tabs[action.tab - 1];
+ if (!tab) return { ok: false, error: `There is no tab ${action.tab}; browser_tabs lists them.` };
+ const entry = currentEntry(tab);
+ if (entry.kind !== "web" && entry.kind !== "file") return { ok: false, error: "That tab has no page or file in it." };
+ if (agentChat(chat).tabId !== tab.id) moveRun(context, tab.id);
+ showTab(tab, context);
+ if (entry.kind === "web") return { ok: true, note: `Switched to tab ${action.tab}, ${where(tab)}. browser_snapshot reads it.` };
+ return { ok: true, file: entry.name, note: `Switched to tab ${action.tab}. ${await fileHint(entry)}` };
+ }
+ const agentId = agentChat(chat).tabId;
+ const frontId = frontTab(chat)?.id;
+ const rows = await Promise.all(
+ tabs.map(async (tab, index) => {
+ const entry = currentEntry(tab);
+ const what =
+ entry.kind === "file"
+ ? `File: ${entry.name} (${KIND_LABELS[await kindOf(entry)]})`
+ : entry.kind === "web"
+ ? `Web page on ${hostOfUrl(tabUrl(tab)) || "this computer"}`
+ : entry.kind === "internal"
+ ? `The browser's ${entry.page} page`
+ : "New tab";
+ const marks = [tab.id === agentId ? "the agent's tab" : "", tab.id === frontId ? "in front" : ""].filter(Boolean);
+ return `${index + 1}. ${what}${marks.length ? ` (${marks.join(", ")})` : ""}`;
+ }),
+ );
+ return { ok: true, text: rows.join("\n") || "No tabs are open.", note: rows.length ? "browser_tabs with tab switches to one." : undefined };
+}
+
+async function fileContent(tab: BrowserTab) {
+ const entry = currentEntry(tab);
+ const blob = entry.kind === "file" ? browserFile(entry.fileId) : undefined;
+ if (entry.kind !== "file" || !blob) return null;
+ const reader = await fileReader();
+ const kind = reader.fileTabKind(entry);
+ return { entry, blob, reader, kind, editable: reader.editable(entry, kind) };
+}
+
+async function readFile(tab: BrowserTab, start: number, note?: string): Promise<Result> {
+ const file = await fileContent(tab);
+ if (!file) return { ok: false, error: NO_FILE };
+ const { entry, blob, reader, kind } = file;
+ if (!file.editable && kind !== "document") return { ok: true, file: entry.name, note: [note, await fileHint(entry)].filter(Boolean).join(" ") };
+ let read: { text: string; cut: boolean };
+ try {
+ read = await reader.fileText(entry, blob, kind);
+ } catch {
+ return { ok: false, error: "the file's text could not be read." };
+ }
+ const lines = numberedLines(read.text, start);
+ const cut = read.cut ? "Only the start of this large file was read." : "";
+ return { ok: true, file: entry.name, text: lines.text, note: [note, lines.note, cut].filter(Boolean).join(" ") };
+}
+
+const MAX_FOUND_LINES = 40;
+
+async function findInFile(tab: BrowserTab, query: string, note: string): Promise<Result> {
+ const file = await fileContent(tab);
+ if (!file) return { ok: false, error: NO_FILE };
+ if (!file.editable && file.kind !== "document") return readFile(tab, 1, note);
+ let text: string;
+ try {
+ ({ text } = await file.reader.fileText(file.entry, file.blob, file.kind));
+ } catch {
+ return { ok: false, error: "the file's text could not be read." };
+ }
+ const wanted = query.toLowerCase();
+ const found: string[] = [];
+ const lines = text.split(/\r?\n/);
+ for (let index = 0; index < lines.length && found.length < MAX_FOUND_LINES; index++) {
+ const line = lines[index] ?? "";
+ if (line.toLowerCase().includes(wanted)) found.push(`${index + 1}\t${line.length > 500 ? `${line.slice(0, 500)} …` : line}`);
+ }
+ const summary = found.length ? `${found.length === MAX_FOUND_LINES ? "The first " : ""}${found.length} matching lines.` : `No line holds "${query}".`;
+ return { ok: true, file: file.entry.name, text: found.join("\n"), note: [note, summary].filter(Boolean).join(" ") };
+}
+
+async function editFile(tab: BrowserTab, action: BrowserAction, signal?: AbortSignal): Promise<Result> {
+ const file = await fileContent(tab);
+ if (!file) return { ok: false, error: NO_FILE };
+ const { entry, blob } = file;
+ if (!file.editable) {
+ return { ok: false, error: "only HTML, React, Markdown, code, text, SVG and CSV files can be edited here, not PDFs, documents, images or media." };
+ }
+ if (blob.size > MAX_EDIT_BYTES) return { ok: false, error: "this file is too large to edit here." };
+ const source = await blob.text();
+ const edited = applyFileEdit(source, { oldText: action.oldText, newText: action.newText, replaceAll: action.replaceAll, content: action.content });
+ if ("error" in edited) return { ok: false, error: edited.error };
+ if (edited.text === source) return { ok: true, file: entry.name, note: "The file already reads that way; nothing changed." };
+ if (signal?.aborted) return { ok: false, error: STOPPED };
+ const now = findTab(tab.id);
+ if (!now || currentEntry(now) !== entry) return { ok: false, error: "the tab changed while the file was being edited; read it again." };
+ useBrowserStore.getState().editFile(tab.id, new Blob([edited.text], { type: blob.type || entry.contentType }));
+ const changed = (action.newText ?? action.content ?? "").split("\n").length;
+ const shown = numberedLines(edited.text, Math.max(1, edited.line - 2)).text.split("\n").slice(0, Math.min(40, changed + 4));
+ return {
+ ok: true,
+ file: entry.name,
+ text: shown.join("\n"),
+ note: `Edited from line ${edited.line}. The panel shows the new version, and its Back button the one before.`,
+ };
+}
+
+async function observeFile(tab: BrowserTab, entry: FileEntry, context: Run, note: string | undefined, op: Record<string, unknown>): Promise<Result> {
+ if ((await kindOf(entry)) !== "preview") {
+ const about = [note, "This tab shows a file, not a page."].filter(Boolean).join(" ");
+ return op.op === "find" ? findInFile(tab, String(op.query ?? ""), about) : readFile(tab, 1, about);
+ }
+ if (!chatOnScreen(context)) return { ok: false, error: OFF_SCREEN };
+ showPreview(tab);
+ const answer = await pageCall(tab, op);
+ if (!answer.ok) return { ok: false, error: answer.error ?? "the page could not be read." };
+ const fresh = findTab(tab.id);
+ if (!fresh || fresh.loading || currentEntry(fresh) !== entry) {
+ return { ok: false, error: "the page changed while it was being read; take a snapshot again." };
+ }
+ // The agent can write this page, so its links are not the user's.
+ if (op.op === "snapshot" || op.op === "find") context.session.pageLinks = new Set();
+ const source = op.op === "snapshot" ? "This is a file's preview; browser_read_file and browser_edit_file work on its source." : "";
+ return { ok: true, file: entry.name, text: answer.text, note: [note, answer.note, source].filter(Boolean).join(" ") || undefined };
+}
+
+function openFromFile(url: string, action: BrowserAction, context: Run): Promise<Result> {
+ if (isStudioUrl(url)) {
+ return Promise.resolve({
+ ok: false,
+ error: "that link points at a file beside this one, which the panel doesn't have; link to a full web address, or edit the page.",
+ });
+ }
+ context.inFile = false;
+ return navigateTo(url, action, context);
+}
+
/** `since`: when the action went out, so the wait covers the requests it started. */
async function afterward(
tab: BrowserTab,
@@ -1107,7 +1368,7 @@ async function dragBetween(tab: BrowserTab, action: BrowserAction, context: Run,
for (const point of [from, end]) {
const described = await pageCall(tab, { op: "describe", ref: "", ...point });
if (!described.ok || !described.info) return { ok: false, error: described.error ?? "nothing is at that point." };
- for (const gate of [gateElement("click", described.info, before, action.policy, permissions(context)), opaqueGate(described.info, before, context)]) {
+ for (const gate of [gateElement("click", gateable(described.info, context), before, action.policy, permissions(context)), opaqueGate(described.info, before, context)]) {
if (!gate) continue;
asked ||= gate.kind === "ask";
const refused = await pass(gate, context);
@@ -1163,7 +1424,7 @@ async function dragBetween(tab: BrowserTab, action: BrowserAction, context: Run,
if (at.x !== end.x || at.y !== end.y) return cancel("the drag stopped short; take a snapshot to see where things are.");
if (!target?.ok || !target.info) return cancel(target?.error ?? "nothing is at the drag's end.");
if (target.hit !== hits[1] || JSON.stringify(target.info) !== infos[1]) {
- for (const gate of [gateElement("click", target.info, before, action.policy, permissions(context)), opaqueGate(target.info, before, context)]) {
+ for (const gate of [gateElement("click", gateable(target.info, context), before, action.policy, permissions(context)), opaqueGate(target.info, before, context)]) {
if (!gate) continue;
const refused = await pass(gate, context);
if (refused) return cancel(refused);
@@ -1179,6 +1440,10 @@ async function dragBetween(tab: BrowserTab, action: BrowserAction, context: Run,
return afterward(tab, before, action, context, since, up.note);
}
+function gateable(info: ElementInfo, context: Run): ElementInfo {
+ return context.inFile ? { ...info, formAction: "" } : info;
+}
+
/** An unlabelled point (a canvas) may be a Buy button nothing can read: ask once per site per chat. */
function opaqueGate(info: ElementInfo, pageUrl: string, context: Run): (Gate & { kind: "ask" }) | null {
if (info.role !== "element" || info.name.trim() || info.href) return null;
@@ -1319,6 +1584,10 @@ async function observeLoaded(tab: BrowserTab, context: Run, since: number, note?
}
async function screenshot(tab: BrowserTab, context: Run): Promise<Result> {
+ const entry = currentEntry(tab);
+ const kind = entry.kind === "file" ? await kindOf(entry) : null;
+ if (entry.kind === "file" && kind === "image") return imageFile(entry);
+ if (entry.kind === "file" && !chatOnScreen(context)) return { ok: false, error: OFF_SCREEN };
if (!isTauri) return { ok: false, error: "screenshots of the browser panel need the desktop app; use browser_snapshot." };
// A page at work off screen is captured on its own; one on screen as the panel shows it.
const hidden = !chatOnScreen(context) && hasNativeView(tab.id);
@@ -1326,29 +1595,59 @@ async function screenshot(tab: BrowserTab, context: Run): Promise<Result> {
const element = pages[pages.length - 1];
if (!hidden && !element) return { ok: false, error: "the browser panel is not on screen." };
// What the user typed into card, code and password fields stays out of the picture, as out of the text.
- const masked = await pageCall(tab, { op: "mask", on: true }).catch(() => null);
+ const page = entry.kind === "web" || kind === "preview";
+ const masked: AgentAnswer | null = page ? await pageCall(tab, { op: "mask", on: true }).catch(() => null) : { ok: true };
if (!masked?.ok) return { ok: false, error: "the page could not be prepared for a screenshot; use browser_snapshot." };
let blob: Blob | null;
try {
await sleep(100);
blob = hidden || !element ? await captureHiddenPage(tab) : await screenshotPage(tab, element);
} finally {
- await pageCall(tab, { op: "mask", on: false }).catch(() => null);
+ if (page) await pageCall(tab, { op: "mask", on: false }).catch(() => null);
}
if (!blob) return { ok: false, error: "the screenshot failed." };
// A page that moved during the capture is one nobody checked or masked: no picture of it.
const now = findTab(tab.id);
- if (!now || now.loading || tabUrl(now) !== tabUrl(tab)) {
+ if (!now || now.loading || tabUrl(now) !== tabUrl(tab) || currentEntry(now) !== entry) {
return { ok: false, error: "the page changed while the screenshot was taken; take a snapshot first." };
}
blob = await toPointScale(tab, context.session, blob, masked);
- const image = await new Promise<string>((resolve, reject) => {
+ const image = await dataUrl(blob);
+ return { ok: true, url: tabUrl(tab), title: tab.title, ...(entry.kind === "file" ? { file: entry.name } : {}), image };
+}
+
+function dataUrl(blob: Blob): Promise<string> {
+ return new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(String(reader.result));
reader.onerror = () => reject(reader.error);
reader.readAsDataURL(blob);
});
- return { ok: true, url: tabUrl(tab), title: tab.title, image };
+}
+
+// PNG: the server reads no SVG or AVIF.
+async function imageFile(entry: FileEntry): Promise<Result> {
+ const blob = browserFile(entry.fileId);
+ const url = blob ? URL.createObjectURL(blob) : null;
+ if (!url) return { ok: false, error: NO_FILE };
+ try {
+ const image = new Image();
+ image.src = url;
+ await image.decode();
+ const width = image.naturalWidth || 300;
+ const height = image.naturalHeight || 150;
+ const fit = Math.min(1, MAX_SHOT_EDGE / Math.max(width, height));
+ const canvas = document.createElement("canvas");
+ canvas.width = Math.max(1, Math.round(width * fit));
+ canvas.height = Math.max(1, Math.round(height * fit));
+ canvas.getContext("2d")?.drawImage(image, 0, 0, canvas.width, canvas.height);
+ const png = await new Promise<Blob | null>((resolve) => canvas.toBlob(resolve, "image/png"));
+ return png ? { ok: true, file: entry.name, image: await dataUrl(png) } : { ok: false, error: "the image could not be read." };
+ } catch {
+ return { ok: false, error: "the image could not be read." };
+ } finally {
+ URL.revokeObjectURL(url);
+ }
}
/** At CSS size within mcp_images.MAX_IMAGE_EDGE, so x and y read off it map back exactly. */
diff --git a/studio/frontend/src/features/browser/agent/file-reader.ts b/studio/frontend/src/features/browser/agent/file-reader.ts
new file mode 100644
index 00000000000..34ad6f356d5
--- /dev/null
+++ b/studio/frontend/src/features/browser/agent/file-reader.ts
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { documentKind } from "@/components/file-viewer";
+import { mediaKind, textFileKind } from "../file-kind";
+import type { BrowserEntry } from "../store";
+import { type FileTabKind, MAX_EDIT_BYTES } from "./file-tab";
+
+type FileEntry = Extract<BrowserEntry, { kind: "file" }>;
+
+const TEXT_UNDERNEATH = /\.(svg|csv|tsv)$/i;
+const TEXT_UNDERNEATH_TYPE = /^(image\/svg\+xml|text\/csv|text\/tab-separated-values)\b/i;
+
+export function fileTabKind(entry: FileEntry): FileTabKind {
+ const { name, contentType, plainText } = entry;
+ const text = textFileKind(name, contentType, plainText);
+ if (text === "html" || text === "react") return "preview";
+ if (text) return "text";
+ const media = mediaKind(name, contentType);
+ if (media === "image") return "image";
+ if (media) return "media";
+ return documentKind(name, contentType) ? "document" : "other";
+}
+
+export function editable(entry: FileEntry, kind: FileTabKind): boolean {
+ return kind === "preview" || kind === "text" || TEXT_UNDERNEATH.test(entry.name) || TEXT_UNDERNEATH_TYPE.test(entry.contentType);
+}
+
+export async function fileText(entry: FileEntry, blob: Blob, kind: FileTabKind): Promise<{ text: string; cut: boolean }> {
+ if (editable(entry, kind)) {
+ const cut = blob.size > MAX_EDIT_BYTES;
+ return { text: await (cut ? blob.slice(0, MAX_EDIT_BYTES) : blob).text(), cut };
+ }
+ const { readAttachmentText } = await import("@/features/chat/attachment-content");
+ const file = new File([blob], entry.name, { type: entry.contentType });
+ const { text, truncated } = await readAttachmentText(file, entry.name, entry.contentType || undefined);
+ return { text, cut: truncated };
+}
diff --git a/studio/frontend/src/features/browser/agent/file-tab.ts b/studio/frontend/src/features/browser/agent/file-tab.ts
new file mode 100644
index 00000000000..c96085b6d15
--- /dev/null
+++ b/studio/frontend/src/features/browser/agent/file-tab.ts
@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// No imports: executor.ts loads this without the file viewers.
+
+export type FileTabKind = "preview" | "text" | "document" | "image" | "media" | "other";
+
+export type FileEdit = { oldText?: string; newText?: string; replaceAll?: boolean; content?: string };
+
+export const MAX_EDIT_BYTES = 5 * 1024 * 1024;
+
+const READ_CHARS = 20_000;
+const LINE_CHARS = 2_000;
+
+export function numberedLines(text: string, start: number): { text: string; note: string } {
+ const lines = text.split(/\r?\n/);
+ if (lines.length > 1 && lines[lines.length - 1] === "") lines.pop();
+ const total = lines.length;
+ if (start > total) return { text: "", note: `The file has ${total} line${total === 1 ? "" : "s"}.` };
+ const out: string[] = [];
+ let used = 0;
+ let end = start - 1;
+ for (let index = start - 1; index < total; index++) {
+ const raw = lines[index] ?? "";
+ const line = raw.length > LINE_CHARS ? `${raw.slice(0, LINE_CHARS)} … (line cut; ${raw.length} characters)` : raw;
+ const row = `${index + 1}\t${line}`;
+ if (out.length && used + row.length > READ_CHARS) break;
+ out.push(row);
+ used += row.length + 1;
+ end = index + 1;
+ }
+ const rest = end < total ? ` browser_read_file with start_line ${end + 1} reads on.` : "";
+ return { text: out.join("\n"), note: `Lines ${start}-${end} of ${total}.${rest}` };
+}
+
+const count = (source: string, part: string) => source.split(part).length - 1;
+
+/** Errors are this tool's own words: they reach the model outside the untrusted-content fence. */
+export function applyFileEdit(source: string, edit: FileEdit): { text: string; line: number } | { error: string } {
+ if (edit.content !== undefined) return { text: edit.content, line: 1 };
+ let old = edit.oldText ?? "";
+ let next = edit.newText ?? "";
+ if (!old) return { error: "old_text is empty." };
+ if (!source.includes(old) && source.includes("\r\n") && old.includes("\n")) {
+ old = old.replace(/\r?\n/g, "\r\n");
+ next = next.replace(/\r?\n/g, "\r\n");
+ }
+ const found = count(source, old);
+ if (found === 0) {
+ return { error: "old_text is not in the file. Read it again with browser_read_file and copy the text exactly, without the line numbers." };
+ }
+ if (found > 1 && !edit.replaceAll) {
+ return { error: `old_text appears ${found} times. Add the text around it so it matches once, or set replace_all.` };
+ }
+ const line = count(source.slice(0, source.indexOf(old)), "\n") + 1;
+ return { text: edit.replaceAll ? source.split(old).join(next) : source.replace(old, () => next), line };
+}
diff --git a/studio/frontend/src/features/browser/agent/page-agent.ts b/studio/frontend/src/features/browser/agent/page-agent.ts
index 947de523f23..6818c060689 100644
--- a/studio/frontend/src/features/browser/agent/page-agent.ts
+++ b/studio/frontend/src/features/browser/agent/page-agent.ts
@@ -376,7 +376,8 @@ export function installPageAgent(): PageAgent {
};
const flush = () => {
const text = clean(buffer, 400);
- if (text && bufferShown && text.length > 1) push(text);
+ // A lone symbol is an icon or a bullet; a lone digit or letter is a count or a grade.
+ if (text && bufferShown && (text.length > 1 || /[\p{L}\p{N}]/u.test(text))) push(text);
buffer = "";
bufferShown = false;
};
@@ -821,7 +822,8 @@ export function installPageAgent(): PageAgent {
const href = anchor?.href ?? "";
if (!anchor || !/^https?:/i.test(href)) return null;
const target = new URL(href);
- const hashOnly = target.hash && target.href.split("#")[0] === pageUrl().href.split("#")[0];
+ // href="#" has an empty hash, yet stays on the page like any other fragment.
+ const hashOnly = href.includes("#") && target.href.split("#")[0] === pageUrl().href.split("#")[0];
return !hashOnly && (directLinks || anchor.target === "_blank") ? href : null;
};
diff --git a/studio/frontend/src/features/browser/agent/tools.ts b/studio/frontend/src/features/browser/agent/tools.ts
index dc8a0ff24dc..c81082ef5d0 100644
--- a/studio/frontend/src/features/browser/agent/tools.ts
+++ b/studio/frontend/src/features/browser/agent/tools.ts
@@ -18,6 +18,9 @@ export const BROWSER_AGENT_TOOLS = [
"browser_select",
"browser_scroll",
"browser_press_key",
+ "browser_tabs",
+ "browser_read_file",
+ "browser_edit_file",
] as const;
/** Captured from the panel without a prompt only in the desktop app; the server keeps it for vision models. */
diff --git a/studio/frontend/src/features/browser/page-frame.tsx b/studio/frontend/src/features/browser/page-frame.tsx
index f908d83a8c0..6cd3cf7947f 100644
--- a/studio/frontend/src/features/browser/page-frame.tsx
+++ b/studio/frontend/src/features/browser/page-frame.tsx
@@ -89,7 +89,7 @@ export function onFrameAnnotate(tabId: string, listener: (event: AnnotateEvent)
};
}
-/** Register an artifact's opaque-origin frame for annotation commands only. */
+/** Register an artifact's opaque-origin frame for annotation and browser agent commands only. */
export function registerArtifactFrame(tabId: string, frame: HTMLIFrameElement): () => void {
frames.set(tabId, frame);
for (const notify of frameListeners) notify();
@@ -97,6 +97,7 @@ export function registerArtifactFrame(tabId: string, frame: HTMLIFrameElement):
if (event.source !== frame.contentWindow || event.origin !== "null") return;
const message = parseFrameMessage(event.data);
if (message?.type === "annotate") annotateListeners.get(tabId)?.(message.event);
+ else if (message?.type === "agent") agentRequests.get(message.id)?.({ installed: message.installed, result: message.result });
};
window.addEventListener("message", listener);
return () => {
diff --git a/studio/frontend/src/features/browser/store.ts b/studio/frontend/src/features/browser/store.ts
index 66aa12f84d0..0bc6a0eb0e0 100644
--- a/studio/frontend/src/features/browser/store.ts
+++ b/studio/frontend/src/features/browser/store.ts
@@ -366,6 +366,7 @@ type BrowserState = {
options?: { newTab?: boolean; background?: boolean; method?: "GET" | "POST"; body?: string; from?: string },
) => void;
openFile: (input: OpenFileInput) => void;
+ editFile: (tabId: string, blob: Blob) => void;
navigate: (
tabId: string,
request: { url: string; method?: "GET" | "POST"; body?: string; from?: string; temporary?: boolean },
@@ -730,6 +731,16 @@ export const useBrowserStore = create<BrowserState>((set, get) => {
openTab(tab);
if (page) get().setFileView(tab.id, { pageTarget: { page } });
},
+ editFile: (tabId, blob) => {
+ const tab = get().tabs.find((candidate) => candidate.id === tabId);
+ const shown = tab && currentEntry(tab);
+ if (shown?.kind !== "file") return;
+ const fileId = newId("file");
+ files.set(fileId, blob);
+ // No openKey: a reopen from chat would replace the edit in place.
+ const { openKey: _, ...rest } = shown;
+ set((state) => ({ tabs: patchTab(state.tabs, tabId, (current) => pushEntry(current, { ...rest, fileId })) }));
+ },
navigate: (tabId, request, options) => {
if (!isWeb(request.url)) return;
set((state) => ({
diff --git a/studio/frontend/src/features/chat/api/durable-gate.ts b/studio/frontend/src/features/chat/api/durable-gate.ts
index d266941f583..47592b17e51 100644
--- a/studio/frontend/src/features/chat/api/durable-gate.ts
+++ b/studio/frontend/src/features/chat/api/durable-gate.ts
@@ -25,6 +25,9 @@ export const BROWSER_EXECUTED_TOOLS: ReadonlySet<string> = new Set<string>([
"browser_evaluate",
"browser_screenshot",
"browser_history",
+ "browser_tabs",
+ "browser_read_file",
+ "browser_edit_file",
]);
/**
diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts
index 0dd1356b54f..e5a70521c35 100644
--- a/studio/frontend/src/i18n/locales/en.ts
+++ b/studio/frontend/src/i18n/locales/en.ts
@@ -195,6 +195,9 @@ export const en = {
history: "Reading your browsing history",
move: "Moving the mouse",
drag: "Dragging",
+ tabs: "Looking at the tabs",
+ readFile: "Reading the file",
+ editFile: "Editing the file",
},
},
title: "Browser",
diff --git a/studio/frontend/tests/artifact-frame-agent.test.ts b/studio/frontend/tests/artifact-frame-agent.test.ts
new file mode 100644
index 00000000000..270c9de38f7
--- /dev/null
+++ b/studio/frontend/tests/artifact-frame-agent.test.ts
@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import test from "node:test";
+import vm from "node:vm";
+
+const backend = readFileSync(new URL("../../backend/routes/inference.py", import.meta.url), "utf8");
+const shell = backend.match(/_ARTIFACT_PREVIEW_FRAME_HTML = """([\s\S]*?)"""/)![1]!.replace(/\\(["\\])/g, "$1");
+const script = shell.match(/<script>([\s\S]*?)<\/script>/)![1]!;
+
+function preview() {
+ const listeners = new Map<string, Array<(event: unknown) => void>>();
+ const reports: Array<Record<string, unknown>> = [];
+ const parent = { postMessage: (value: Record<string, unknown>) => reports.push(value) };
+ let context: vm.Context;
+ class Node {
+ appendChild(child: { source: string }) {
+ vm.runInContext(child.source, context);
+ return child;
+ }
+ }
+ class Element extends Node {
+ remove() {}
+ }
+ class HTMLScriptElement extends Element {
+ source = "";
+ set text(value: string) {
+ this.source = value;
+ }
+ }
+ class Document extends Node {
+ get documentElement() {
+ return new Element();
+ }
+ createElement() {
+ return new HTMLScriptElement();
+ }
+ open() {
+ listeners.clear();
+ }
+ write() {}
+ close() {}
+ addEventListener() {}
+ }
+ class Crypto {
+ getRandomValues(words: Uint32Array) {
+ return words.fill(7);
+ }
+ }
+ context = vm.createContext(
+ {
+ parent, URLSearchParams, location: { search: "?v=test" }, Document, Node, Element, HTMLScriptElement, Crypto,
+ crypto: new Crypto(), document: new Document(),
+ console: Object.fromEntries(["error", "warn", "info", "log", "debug"].map((level) => [level, () => {}])),
+ addEventListener: (type: string, listener: (event: unknown) => void) => {
+ listeners.set(type, [...(listeners.get(type) ?? []), listener]);
+ },
+ },
+ { codeGeneration: { strings: false } },
+ );
+ context.window = context;
+ vm.runInContext(script.replace("/* ANNOTATE_JS */", "return {};"), context);
+ const message = (data: unknown, source: unknown = parent) => {
+ for (const listener of [...(listeners.get("message") ?? [])]) listener({ source, data });
+ };
+ message({ type: "unsloth:artifact-html", html: "<h1>Coffee</h1>" });
+ return { context, message, reports };
+}
+
+const plain = (value: unknown) => JSON.parse(JSON.stringify(value));
+const agentCode = "(() => ({ run: (op) => ({ ok: true, text: `ran ${op.op}` }) }))();";
+const agent = (id: string, fields = {}) => ({ type: "unsloth:browser-command", command: "agent", id, op: { op: "snapshot" }, ...fields });
+
+test("the canvas installs the browser agent from an inline script and answers its commands", () => {
+ const frame = preview();
+ frame.message(agent("a1"));
+ assert.deepEqual(plain(frame.reports.at(-1)), { source: "unsloth-browser", type: "agent", id: "a1", installed: false, result: null });
+ frame.message(agent("a2", { code: agentCode }));
+ assert.deepEqual(plain(frame.reports.at(-1)), { source: "unsloth-browser", type: "agent", id: "a2", installed: true, result: { ok: true, text: "ran snapshot" } });
+ frame.message(agent("a3"));
+ assert.equal(frame.reports.at(-1)?.installed, true);
+ assert.deepEqual(Object.keys(frame.context).filter((key) => key.startsWith("unslothAgent")), []);
+});
+
+test("only the embedder can install the agent", () => {
+ const frame = preview();
+ frame.message(agent("a1", { code: agentCode }), frame.context);
+ assert.equal(frame.reports.some((report) => report.type === "agent"), false);
+});
diff --git a/studio/frontend/tests/browser-agent-files.test.ts b/studio/frontend/tests/browser-agent-files.test.ts
new file mode 100644
index 00000000000..87fb9db91bb
--- /dev/null
+++ b/studio/frontend/tests/browser-agent-files.test.ts
@@ -0,0 +1,238 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import { register } from "node:module";
+import { test } from "node:test";
+
+register("./helpers/browser-executor-resolver.mjs", import.meta.url);
+const { runBrowserAction } = await import("../src/features/browser/agent/executor.ts");
+const { agentChat, answerQuestion, endBrowserAgentRun, stopBrowserAgent } = await import("../src/features/browser/agent/agent-store.ts");
+const { applyFileEdit, numberedLines } = await import("../src/features/browser/agent/file-tab.ts");
+const { browserFile, currentEntry, useBrowserStore } = await import("../src/features/browser/store.ts");
+
+const globals = globalThis as Record<string, unknown>;
+const policy = { allowedDomains: [], blockedDomains: [] };
+const RUN = "r";
+type Result = { ok: boolean; error?: string; text?: string; note?: string; file?: string };
+
+let next = 0;
+async function act(op: string, fields: Record<string, unknown> = {}, runKey = RUN): Promise<Result> {
+ const posted: { result: Result }[] = [];
+ globals.__authFetch = async (_url: string, init: { body: string }) => {
+ posted.push(JSON.parse(init.body));
+ return new Response(null, { status: 200 });
+ };
+ await runBrowserAction({ id: `f${next++}`, sessionId: "s", op, direction: "down", policy, ...fields }, { runKey, userText: "" });
+ assert.equal(posted.length, 1);
+ return posted[0].result;
+}
+
+function shownText(): Promise<string> {
+ const store = useBrowserStore.getState();
+ const tab = store.tabs.find((candidate) => candidate.id === store.activeTabId);
+ const entry = tab && currentEntry(tab);
+ assert.equal(entry?.kind, "file");
+ return browserFile(entry.kind === "file" ? entry.fileId : "")?.text() ?? Promise.resolve("");
+}
+
+test("numbered lines page through a file and say where to go on", () => {
+ const text = Array.from({ length: 3000 }, (_, i) => `line ${i + 1}`).join("\n");
+ const first = numberedLines(text, 1);
+ assert.match(first.text, /^1\tline 1\n2\tline 2/);
+ const end = Number(/Lines 1-(\d+) of 3000/.exec(first.note)?.[1]);
+ assert.ok(end > 100 && end < 3000);
+ assert.match(first.note, new RegExp(`start_line ${end + 1}`));
+ assert.equal(numberedLines(text, 2999).note, "Lines 2999-3000 of 3000.");
+ assert.equal(numberedLines("a\n", 5).note, "The file has 1 line.");
+});
+
+test("an edit replaces text found once, refuses ambiguous or missing text, and keeps CRLF", () => {
+ assert.deepEqual(applyFileEdit("a\nb\nc", { oldText: "b", newText: "B" }), { text: "a\nB\nc", line: 2 });
+ assert.match((applyFileEdit("x x", { oldText: "x", newText: "y" }) as { error: string }).error, /appears 2 times/);
+ assert.deepEqual(applyFileEdit("x x", { oldText: "x", newText: "y", replaceAll: true }), { text: "y y", line: 1 });
+ assert.match((applyFileEdit("abc", { oldText: "z", newText: "y" }) as { error: string }).error, /not in the file/);
+ assert.deepEqual(applyFileEdit("a\r\nb\r\nc", { oldText: "a\nb", newText: "a\nB" }), { text: "a\r\nB\r\nc", line: 1 });
+ assert.deepEqual(applyFileEdit("price", { oldText: "price", newText: "$&5" }), { text: "$&5", line: 1 });
+ assert.deepEqual(applyFileEdit("old", { content: "new" }), { text: "new", line: 1 });
+});
+
+test("the agent reads and edits the HTML file in front, and Back shows the version before", async () => {
+ endBrowserAgentRun(RUN);
+ const html = "<!doctype html>\n<h1>Hello</h1>\n<button>Go</button>\n";
+ useBrowserStore.getState().openFile({ blob: new Blob([html], { type: "text/html" }), name: "page.html", contentType: "text/html", key: "html:a1" });
+ const read = await act("read_file", { startLine: 1 });
+ assert.equal(read.ok, true);
+ assert.equal(read.file, "page.html");
+ assert.match(read.text ?? "", /^1\t<!doctype html>\n2\t<h1>Hello<\/h1>/);
+
+ const edited = await act("edit_file", { oldText: "<h1>Hello</h1>", newText: "<h1>Hi there</h1>" });
+ assert.equal(edited.ok, true, edited.error);
+ assert.match(edited.text ?? "", /2\t<h1>Hi there<\/h1>/);
+ assert.equal(await shownText(), html.replace("Hello", "Hi there"));
+
+ const missing = await act("edit_file", { oldText: "<h1>Hello</h1>", newText: "x" });
+ assert.equal(missing.ok, false);
+ assert.match(missing.error ?? "", /not in the file/);
+
+ const store = useBrowserStore.getState();
+ store.goBack(store.activeTabId ?? "");
+ assert.equal(await shownText(), html);
+});
+
+test("a file's preview is read through its frame, and its links are not taken as the user's", async () => {
+ endBrowserAgentRun(RUN);
+ useBrowserStore.getState().openFile({ blob: new Blob(["<a href='https://example.com/'>x</a>"], { type: "text/html" }), name: "links.html", contentType: "text/html", key: "html:a2" });
+ globals.__frameAgent = async (_tabId: string, op: Record<string, unknown>) => {
+ if (op.op === "status") return { installed: true, result: { ok: true, quietMs: 1000 } };
+ return { installed: true, result: { ok: true, text: "link r1 x", links: ["https://example.com/"] } };
+ };
+ const snapshot = await act("snapshot");
+ assert.equal(snapshot.ok, true, snapshot.error);
+ assert.equal(snapshot.file, "links.html");
+ assert.equal(snapshot.text, "link r1 x");
+ assert.match(snapshot.note ?? "", /browser_edit_file/);
+
+ globals.__native = true;
+ try {
+ const desktop = await act("snapshot");
+ assert.equal(desktop.text, "link r1 x", desktop.error);
+ } finally {
+ globals.__native = false;
+ }
+
+ let navigate = "https://example.com/";
+ globals.__frameAgent = async (_tabId: string, op: Record<string, unknown>) => {
+ if (op.op === "status") return { installed: true, result: { ok: true, quietMs: 1000 } };
+ if (op.op === "describe") return { installed: true, result: { ok: true, info: { role: "link", name: "x", tag: "a", href: navigate } } };
+ return { installed: true, result: { ok: true, navigate } };
+ };
+ const pending = act("click", { ref: "r1" });
+ let question = agentChat("new").question;
+ for (let i = 0; !question && i < 200; i++) {
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ question = agentChat("new").question;
+ }
+ assert.equal(question?.ask, "open");
+ assert.equal(question?.host, "example.com");
+ answerQuestion(question.id, false);
+ assert.match((await pending).error ?? "", /declined opening a page/);
+
+ navigate = "http://localhost/api/inference/other.html";
+ const beside = await act("click", { ref: "r1" });
+ assert.match(beside.error ?? "", /beside this one/);
+});
+
+test("a preview that misses the first call, as one just switched on from Source does, still answers", async () => {
+ endBrowserAgentRun(RUN);
+ let calls = 0;
+ globals.__frameAgent = async (_tabId: string, op: Record<string, unknown>, _code: string, timeoutMs: number) => {
+ if (calls++ === 0) return new Promise((resolve) => setTimeout(() => resolve(null), timeoutMs));
+ if (op.op === "status") return { installed: true, result: { ok: true, quietMs: 1000 } };
+ return { installed: true, result: { ok: true, text: "ready", links: [] } };
+ };
+ const snapshot = await act("snapshot");
+ assert.equal(snapshot.text, "ready", snapshot.error);
+});
+
+test("a click that takes the preview a while is sent once, not again after a short wait", async () => {
+ endBrowserAgentRun(RUN);
+ let clicks = 0;
+ globals.__frameAgent = (_tabId: string, op: Record<string, unknown>, _code: string, timeoutMs: number) => {
+ const answer = (value: unknown, after = 0) =>
+ Promise.race([new Promise((resolve) => setTimeout(() => resolve(value), after)), new Promise((resolve) => setTimeout(() => resolve(null), timeoutMs))]);
+ if (op.op === "click") {
+ clicks++;
+ return answer({ installed: true, result: { ok: true } }, 2000);
+ }
+ if (op.op === "describe") return answer({ installed: true, result: { ok: true, info: { role: "button", name: "Go", tag: "button" } } });
+ if (op.op === "status") return answer({ installed: true, result: { ok: true, quietMs: 1000 } });
+ return answer({ installed: true, result: { ok: true, text: "page", links: [] } });
+ };
+ const click = await act("click", { ref: "r1" });
+ assert.equal(click.ok, true, click.error);
+ assert.equal(clicks, 1);
+});
+
+test("a screenshot of a file in a chat that is off screen is refused, not taken of the panel shown", async () => {
+ endBrowserAgentRun(RUN);
+ useBrowserStore.getState().openFile({ blob: new Blob(["notes"], { type: "text/plain" }), name: "shot.txt", contentType: "text/plain", key: "f:shot" });
+ globals.__native = true;
+ globals.__panel = false;
+ try {
+ const shot = await act("screenshot");
+ assert.equal(shot.ok, false);
+ assert.match(shot.error ?? "", /only runs while the chat is open/);
+ } finally {
+ globals.__native = false;
+ globals.__panel = true;
+ }
+});
+
+test("reloading a file's preview in the desktop app reloads its frame", async () => {
+ endBrowserAgentRun(RUN);
+ useBrowserStore.getState().openFile({ blob: new Blob(["<p>x</p>"], { type: "text/html" }), name: "reload.html", contentType: "text/html", key: "html:reload" });
+ globals.__frameAgent = async (_tabId: string, op: Record<string, unknown>) =>
+ op.op === "status" ? { installed: true, result: { ok: true, quietMs: 1000 } } : { installed: true, result: { ok: true, text: "x", links: [] } };
+ const store = useBrowserStore.getState();
+ const before = store.tabs.find((tab) => tab.id === store.activeTabId)?.reloadKey;
+ globals.__native = true;
+ try {
+ const reloaded = await act("reload");
+ assert.equal(reloaded.ok, true, reloaded.error);
+ } finally {
+ globals.__native = false;
+ }
+ const after = useBrowserStore.getState().tabs.find((tab) => tab.id === store.activeTabId)?.reloadKey;
+ assert.equal(after, (before ?? 0) + 1);
+});
+
+test("an edit stopped while the file is read leaves the file as it was", async () => {
+ endBrowserAgentRun(RUN);
+ const source = "<h1>Keep</h1>";
+ let release = () => {};
+ const released = new Promise<void>((resolve) => { release = resolve; });
+ const slow = Object.assign(new Blob([source], { type: "text/html" }), { text: () => released.then(() => source) });
+ useBrowserStore.getState().openFile({ blob: slow, name: "stop.html", contentType: "text/html", key: "html:stop" });
+ const pending = act("edit_file", { oldText: "Keep", newText: "Changed" }, "stopped-run");
+ for (let i = 0; i < 100 && agentChat("new").runKey !== "stopped-run"; i++) await new Promise((resolve) => setTimeout(resolve, 5));
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ stopBrowserAgent("new");
+ release();
+ const result = await pending;
+ assert.equal(result.ok, false);
+ assert.equal(await shownText(), source);
+ endBrowserAgentRun("stopped-run");
+});
+
+test("a PDF is read as text but not edited, and a text file has nothing to click", async () => {
+ endBrowserAgentRun(RUN);
+ useBrowserStore.getState().openFile({ blob: new Blob(["report body"], { type: "application/pdf" }), name: "report.pdf", contentType: "application/pdf", key: "f:pdf" });
+ const read = await act("read_file");
+ assert.equal(read.text, "1\treport body");
+ const edit = await act("edit_file", { content: "x" });
+ assert.equal(edit.ok, false);
+ assert.match(edit.error ?? "", /not PDFs/);
+
+ useBrowserStore.getState().openFile({ blob: new Blob(["alpha\nbeta\n"], { type: "text/plain" }), name: "notes.txt", contentType: "text/plain", key: "f:txt" });
+ await act("tabs", { tab: useBrowserStore.getState().tabs.filter((tab) => tab.chat === useBrowserStore.getState().chat).length });
+ const found = await act("find", { query: "BETA" });
+ assert.equal(found.text, "2\tbeta");
+ const click = await act("click", { ref: "r1" });
+ assert.equal(click.ok, false);
+ assert.match(click.error ?? "", /no page to act on/);
+});
+
+test("browser_tabs lists files by name and web pages by site, and switches by number", async () => {
+ endBrowserAgentRun(RUN);
+ const listed = await act("tabs");
+ assert.equal(listed.ok, true);
+ assert.match(listed.text ?? "", /File: page\.html \(HTML or React, with a preview\)/);
+ assert.match(listed.text ?? "", /File: report\.pdf \(document\)/);
+ assert.match(listed.text ?? "", /\(the agent's tab, in front\)/);
+ const missing = await act("tabs", { tab: 99 });
+ assert.equal(missing.ok, false);
+ const switched = await act("tabs", { tab: 2 });
+ assert.equal(switched.ok, true);
+ assert.equal(switched.file, "links.html");
+});
diff --git a/studio/frontend/tests/helpers/browser-executor-resolver.mjs b/studio/frontend/tests/helpers/browser-executor-resolver.mjs
index 2d0941aec1c..85c8b1018c4 100644
--- a/studio/frontend/tests/helpers/browser-executor-resolver.mjs
+++ b/studio/frontend/tests/helpers/browser-executor-resolver.mjs
@@ -9,7 +9,7 @@ const stub = (source) => ({ url: `data:text/javascript,${encodeURIComponent(sour
const STUBS = {
"../page-frame":
- "export const requestFrameAgent = (tabId, op, code) => globalThis.__frameAgent(tabId, op, code);" +
+ "export const requestFrameAgent = (tabId, op, code, timeoutMs = 5000) => globalThis.__frameAgent(tabId, op, code, timeoutMs);" +
" export const sendFrameCommand = (tabId, command) => globalThis.__frameCommand?.(tabId, command) ?? false;",
"../capture": "export const screenshotPage = async () => null; export const captureHiddenPage = async () => null;",
// Native pages only when a test sets globalThis.__native; it answers their calls via __callNative.
@@ -24,6 +24,10 @@ const STUBS = {
" export const localOriginGranted = (tabId, url) => globalThis.__granted?.get(tabId) === new URL(url).origin;",
"../panel-availability": "export const browserPanelAvailable = () => globalThis.__panel !== false;",
"../history-store": "export const useBrowserHistoryStore = { getState: () => ({ recordVisit() {} }) };",
+ "./file-reader":
+ "export const fileTabKind = ({ name }) => /\\.html?$/.test(name) ? 'preview' : /\\.pdf$/.test(name) ? 'document' : /\\.png$/.test(name) ? 'image' : 'text';" +
+ " export const editable = (entry, kind) => kind === 'preview' || kind === 'text';" +
+ " export const fileText = async (entry, blob) => ({ text: await blob.text(), cut: false });",
};
export function resolve(specifier, context, next) {