Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: let browser use read, click and edit files in the browser panel (#16644)
Michael Han71c643e02a54
diff --git a/studio/backend/core/inference/browser_agent.py b/studio/backend/core/inference/browser_agent.pyindex d05dbf99ca4..9727880087c 100644--- a/studio/backend/core/inference/browser_agent.py+++ b/studio/backend/core/inference/browser_agent.py@@ -1,9 +1,10 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""Lets the model drive the chat tab browser panel (user can watch, stop, take over; tab gates risky-actions). Enforces argument shapes, http(s) only, public or this-machine hosts off Studio ports,-website policy before and after navigation, and fenced untrusted page text."""+"""Lets the model drive the chat tab browser panel, its web pages and the files it shows (user can+watch, stop, take over; tab gates risky actions). Enforces argument shapes, http(s) only, public or+this-machine hosts off Studio ports, website policy before and after navigation, and fenced+untrusted page text."""from __future__ import annotations@@ -288,6 +289,81 @@ BROWSER_HISTORY_TOOL = {},}+BROWSER_TABS_TOOL = {+ "type": "function",+ "function": {+ "name": "browser_tabs",+ "description": (+ "List the tabs open in the browser panel, web pages and files alike (an HTML page or "+ "React component made in this chat, a PDF, an image, code, text), or switch to one by "+ "its number. The other browser tools then work in that tab: on an HTML or React file "+ "they read and click its preview as on any page, and browser_read_file and "+ "browser_edit_file read and change the file itself."+ ),+ "parameters": {+ "type": "object",+ "properties": {+ "tab": {+ "type": "integer",+ "description": "The number of the tab to switch to; leave out to only list.",+ }+ },+ },+ },+}++BROWSER_READ_FILE_TOOL = {+ "type": "function",+ "function": {+ "name": "browser_read_file",+ "description": (+ "Read the file shown in the browser panel: the source of an HTML, React, Markdown, "+ "code or text file, or the text of a PDF or document, with line numbers (not part "+ "of the file). Works in the agent's tab, or the file tab in front if the agent's tab "+ "is a web page."+ ),+ "parameters": {+ "type": "object",+ "properties": {+ "start_line": {+ "type": "integer",+ "description": "First line to read, from 1; for files too long for one read.",+ }+ },+ },+ },+}++BROWSER_EDIT_FILE_TOOL = {+ "type": "function",+ "function": {+ "name": "browser_edit_file",+ "description": (+ "Change the HTML, React, Markdown, code or text file shown in the browser panel: "+ "replace old_text, copied exactly from browser_read_file without the line numbers, "+ "with new_text, or give content to rewrite the whole file. The preview reloads with "+ "the change, and Back in the panel shows the version before it. PDFs, documents and "+ "images can't be edited here. For files in the panel, not edit_file, which works on "+ "the code sandbox's files."+ ),+ "parameters": {+ "type": "object",+ "properties": {+ "old_text": {+ "type": "string",+ "description": "Text to replace; it must appear once, unless replace_all.",+ },+ "new_text": {"type": "string"},+ "replace_all": {"type": "boolean"},+ "content": {+ "type": "string",+ "description": "The whole new file, instead of old_text and new_text.",+ },+ },+ },+ },+}+BROWSER_TOOLS = [BROWSER_NAVIGATE_TOOL,BROWSER_SNAPSHOT_TOOL,@@ -302,6 +378,9 @@ BROWSER_TOOLS = [BROWSER_EVALUATE_TOOL,BROWSER_SCREENSHOT_TOOL,BROWSER_HISTORY_TOOL,+ BROWSER_TABS_TOOL,+ BROWSER_READ_FILE_TOOL,+ BROWSER_EDIT_FILE_TOOL,]BROWSER_TOOL_NAMES = frozenset(tool["function"]["name"] for tool in BROWSER_TOOLS)@@ -314,6 +393,8 @@ _MIN_ACTION_TIMEOUT_S = 30.0_MAX_URL_CHARS = 4096_MAX_TEXT_CHARS = 8000_MAX_SCRIPT_CHARS = 8000+_MAX_FILE_EDIT_CHARS = 500_000+_MAX_TAB = 1000_HIDDEN_TAIL_RE = re.compile(r"(?:[ \t]*\r?\n){4,}|[ \t]{200,}")# Past any real screen: a point there is a mistake, not a far corner._MAX_POINT = 100_000@@ -589,9 +670,63 @@ def _validated_request(name: str, arguments: dict, website_policy: Optional[dict"query": str(arguments.get("query") or "").strip()[:200],"limit": max(1, min(limit, 50)),}+ if name == "browser_tabs":+ tab = _whole_number(arguments, "tab")+ if tab is not None and not 1 <= tab <= _MAX_TAB:+ raise ValueError("tab must be the number of a tab in the list.")+ return {"op": "tabs", **({"tab": tab} if tab is not None else {})}+ if name == "browser_read_file":+ start = _whole_number(arguments, "start_line") or 1+ if start < 1:+ raise ValueError("start_line counts from 1.")+ return {"op": "read_file", "start_line": start}+ if name == "browser_edit_file":+ return {"op": "edit_file", **_file_edit(arguments)}raise ValueError(f"unknown browser tool {name}.")+def _whole_number(arguments: dict, key: str) -> Optional[int]:+ raw = arguments.get(key)+ if raw is None or raw == "":+ return None+ try:+ value = float(raw)+ except (TypeError, ValueError, OverflowError):+ raise ValueError(f"{key} must be a whole number.")+ if isinstance(raw, bool) or not value.is_integer():+ raise ValueError(f"{key} must be a whole number.")+ return int(value)+++def _file_edit(arguments: dict) -> dict:+ def text(key: str) -> Optional[str]:+ value = arguments.get(key)+ if value is None:+ return None+ if not isinstance(value, str):+ raise ValueError(f"{key} must be a string.")+ if len(value) > _MAX_FILE_EDIT_CHARS:+ raise ValueError(f"{key} is longer than {_MAX_FILE_EDIT_CHARS} characters.")+ return value++ content = text("content")+ old = text("old_text")+ new = text("new_text")+ if content is not None:+ if old is not None or new is not None:+ raise ValueError("give either content, or old_text and new_text, not both.")+ return {"content": content}+ if not old:+ raise ValueError(+ "old_text is empty. Give the text to replace, or content for the whole file."+ )+ if new is None:+ raise ValueError("new_text is required with old_text; an empty string deletes old_text.")+ if old == new:+ raise ValueError("old_text and new_text are the same, so nothing would change.")+ return {"old_text": old, "new_text": new, "replace_all": arguments.get("replace_all") is True}++def _clip(value: Any, limit: int) -> str:return value[:limit] if isinstance(value, str) else ""@@ -634,18 +769,23 @@ def format_result(name: str, result: dict, website_policy: Optional[dict]) -> st"disallows, so its content is withheld. Navigate somewhere else.")title = _one_line(_clip(result.get("title"), 300))+ file = _one_line(_clip(result.get("file"), 300))note = _one_line(_clip(result.get("note"), 1000))text = _clip(result.get("text"), 400_000)if len(text) > _MAX_MODEL_CHARS:- text = (- text[:_MAX_MODEL_CHARS]- + "\n... (cut here: use browser_find, browser_scroll or a snapshot ref to see more)"+ more = (+ "browser_read_file with a later start_line"+ if name == "browser_read_file"+ else "browser_find, browser_scroll or a snapshot ref")+ text = text[:_MAX_MODEL_CHARS] + f"\n... (cut here: use {more} to see more)"# The title and note come from the page too (a selected option's words), so everything but# this tool's own framing goes inside the fence.page = []if url:page.append(f"Page: {title + ' ' if title else ''}<{url}>")+ elif file:+ page.append(f"File: {file}")if note:page.append(note)if text:diff --git a/studio/backend/core/inference/tool_loop_controller.py b/studio/backend/core/inference/tool_loop_controller.pyindex 97a04455595..700837fbf42 100644--- a/studio/backend/core/inference/tool_loop_controller.py+++ b/studio/backend/core/inference/tool_loop_controller.py@@ -884,6 +884,12 @@ def _browser_status(tool_name: str, arguments: Mapping[str, Any]) -> str:return "Moving the mouse"if tool_name == "browser_drag":return "Dragging"+ if tool_name == "browser_tabs":+ return "Switching tabs" if text("tab") else "Listing browser tabs"+ if tool_name == "browser_read_file":+ return "Reading the file"+ if tool_name == "browser_edit_file":+ return "Editing the file"return f"Calling: {tool_name}"diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 21231dbb389..cf1b9dfaebd 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -3942,6 +3942,35 @@ _ARTIFACT_PREVIEW_FRAME_HTML = """<!doctype html>const annotationFactory = (post) => {/* ANNOTATE_JS */};+ // No eval under this CSP: the page agent arrives as an inline script, on a one-time key.+ const buildAgent = (() => {+ let apply, create, root, setText, append, detach, random, join, Words;+ try {+ apply = Reflect.apply;+ create = Document.prototype.createElement;+ root = Object.getOwnPropertyDescriptor(Document.prototype, "documentElement").get;+ setText = Object.getOwnPropertyDescriptor(HTMLScriptElement.prototype, "text").set;+ append = Node.prototype.appendChild;+ detach = Element.prototype.remove;+ random = Crypto.prototype.getRandomValues;+ join = Array.prototype.join;+ Words = Uint32Array;+ } catch {+ return () => null;+ }+ return (code) => {+ const parent = apply(root, document, []);+ if (!parent) return null;+ const key = "unslothAgent" + apply(join, apply(random, crypto, [new Words(4)]), ["_"]);+ const script = apply(create, document, ["script"]);+ apply(setText, script, ["window." + key + " = " + code]);+ apply(append, parent, [script]);+ apply(detach, script, []);+ const agent = window[key];+ delete window[key];+ return agent && typeof agent.run === "function" ? agent : null;+ };+ })();// Named for canvasStack() in the frontend, which trims from this frame down.const unslothRenderArtifact = (html) => {installStorageFallbacks();@@ -3955,6 +3984,7 @@ _ARTIFACT_PREVIEW_FRAME_HTML = """<!doctype html>document.close();// Instantiate only on a parent command; the factory predates the page's meta CSP.let annotation = null;+ let agent = null;const postAnnotation = (message) => parent.postMessage({ source: "unsloth-browser", ...message }, "*");window.addEventListener("message", (event) => {if (event.source !== parent) return;@@ -3967,6 +3997,10 @@ _ARTIFACT_PREVIEW_FRAME_HTML = """<!doctype html>else annotation?.stop();} else if (data.command === "annotateForget") annotation?.forget(Number(data.id));else if (data.command === "annotateNumbers") annotation?.number(data.numbers);+ else if (data.command === "agent" && typeof data.id === "string") {+ if (!agent && typeof data.code === "string" && data.code.length <= 262144) agent = buildAgent(data.code);+ postAnnotation({ type: "agent", id: data.id, installed: Boolean(agent), result: agent ? agent.run(data.op || {}) : null });+ }});postAnnotation({ type: "annotate", event: "ready" });// document.open() drops listeners bound before it, so rebind here.diff --git a/studio/backend/tests/test_browser_agent.py b/studio/backend/tests/test_browser_agent.pyindex c91d18b4d7e..ad250b265ac 100644--- a/studio/backend/tests/test_browser_agent.py+++ b/studio/backend/tests/test_browser_agent.py@@ -564,3 +564,76 @@ def test_redirect_target_honours_the_website_policy():"op": "navigate","url": "https://www.google.com/url?q=https://unsloth.ai/",}+++def test_file_tools_take_bounded_arguments():+ assert _request("browser_tabs", {}) == {"op": "tabs"}+ assert _request("browser_tabs", {"tab": 2}) == {"op": "tabs", "tab": 2}+ assert _request("browser_tabs", {"tab": "3"}) == {"op": "tabs", "tab": 3}+ for bad in (0, -1, 1.5, True, "two", 5000):+ with pytest.raises(ValueError):+ _request("browser_tabs", {"tab": bad})+ assert _request("browser_read_file", {}) == {"op": "read_file", "start_line": 1}+ assert _request("browser_read_file", {"start_line": 40}) == {+ "op": "read_file",+ "start_line": 40,+ }+ with pytest.raises(ValueError, match = "from 1"):+ _request("browser_read_file", {"start_line": -3})++ assert _request("browser_edit_file", {"old_text": "<h1>", "new_text": "<h2>"}) == {+ "op": "edit_file",+ "old_text": "<h1>",+ "new_text": "<h2>",+ "replace_all": False,+ }+ assert _request("browser_edit_file", {"old_text": "x", "new_text": ""})["new_text"] == ""+ assert _request("browser_edit_file", {"content": ""}) == {"op": "edit_file", "content": ""}+ for arguments, message in (+ ({}, "old_text is empty"),+ ({"old_text": "a"}, "new_text is required"),+ ({"old_text": "a", "new_text": "a"}, "the same"),+ ({"content": "x", "old_text": "a", "new_text": "b"}, "not both"),+ ({"content": 5}, "must be a string"),+ ({"content": "x" * 500_001}, "longer than"),+ ):+ with pytest.raises(ValueError, match = message):+ _request("browser_edit_file", arguments)+++def test_file_results_name_the_file_inside_the_fence():+ out = browser_agent.format_result(+ "browser_read_file",+ {"ok": True, "file": "page.html", "text": "1\t<h1>Hi</h1>", "note": "Lines 1-1 of 1."},+ None,+ )+ assert out.index("File: page.html") > out.index("<untrusted_web_content>")+ assert "1\t<h1>Hi</h1>" in out+ long = browser_agent.format_result(+ "browser_read_file", {"ok": True, "file": "a.txt", "text": "x" * 50_000}, None+ )+ assert "browser_read_file with a later start_line" in long+++def test_file_tools_have_status_badges_and_are_browser_tools():+ from core.inference.tool_loop_controller import status_for_tool++ for name in ("browser_tabs", "browser_read_file", "browser_edit_file"):+ assert browser_agent.is_browser_tool(name)+ assert status_for_tool("browser_tabs", {}) == "Listing browser tabs"+ assert status_for_tool("browser_tabs", {"tab": 2}) == "Switching tabs"+ assert status_for_tool("browser_read_file", {}) == "Reading the file"+ assert (+ status_for_tool("browser_edit_file", {"old_text": "a", "new_text": "b"})+ == "Editing the file"+ )+++def test_the_artifact_frame_installs_the_agent_without_eval():+ from routes.inference import _ARTIFACT_PREVIEW_FRAME_STRICT_CSP, _artifact_preview_frame_html++ html = _artifact_preview_frame_html()+ assert 'data.command === "agent"' in html+ assert "buildAgent(data.code)" in html+ assert "unsafe-eval" not in _ARTIFACT_PREVIEW_FRAME_STRICT_CSP+ assert "new Function" not in html and "eval(" not in htmldiff --git a/studio/frontend/src/features/browser/agent/agent-store.ts b/studio/frontend/src/features/browser/agent/agent-store.tsindex cd1e81cda6e..fbcd73e4029 100644--- a/studio/frontend/src/features/browser/agent/agent-store.ts+++ b/studio/frontend/src/features/browser/agent/agent-store.ts@@ -40,7 +40,10 @@ export type AgentStep = {| "evaluate"| "history"| "move"- | "drag";+ | "drag"+ | "tabs"+ | "readFile"+ | "editFile";detail: string;};diff --git a/studio/frontend/src/features/browser/agent/executor.ts b/studio/frontend/src/features/browser/agent/executor.tsindex 289c5e40c58..c312b0d597e 100644--- a/studio/frontend/src/features/browser/agent/executor.ts+++ b/studio/frontend/src/features/browser/agent/executor.ts@@ -8,7 +8,7 @@ import { authFetch } from "@/features/auth";import { openChatItemById, useChatNavigationStore, visibleChatItems } from "@/features/chat";import { getLocale, translate } from "@/i18n";import { toast } from "@/lib/toast";-import { isTauri } from "@/lib/api-base";+import { isStudioUrl, isTauri } from "@/lib/api-base";import { unwrapRedirect } from "../address";import { captureHiddenPage, screenshotPage } from "../capture";import { useBrowserHistoryStore } from "../history-store";@@ -17,7 +17,7 @@ import { isLoopbackUrl } from "../address";import { callNative, localOriginGranted, useNativeBrowser } from "../native-support";import { requestFrameAgent, sendFrameCommand } from "../page-frame";import { browserPanelAvailable } from "../panel-availability";-import { type BrowserTab, chatKeyOfThread, currentEntry, onChatAdopted, useBrowserStore } from "../store";+import { type BrowserEntry, type BrowserTab, browserFile, chatKeyOfThread, currentEntry, onChatAdopted, useBrowserStore } from "../store";import {type AgentStep,type Answer,@@ -33,6 +33,7 @@ import {import { type Gate, type Question, type SitePolicy, gateElement, gateNavigation, hostOfUrl, navigationBlocks } from "./gate";import { type SiteRule, ruleFor } from "./permissions";import { type AgentAnswer, type ElementInfo, pageAgentSource } from "./page-agent";+import { type FileTabKind, MAX_EDIT_BYTES, applyFileEdit, numberedLines } from "./file-tab";import { waitForPage } from "./page-wait";export type BrowserAction = {@@ -55,6 +56,12 @@ export type BrowserAction = {fullPage?: boolean;query?: string;limit?: number;+ tab?: number;+ startLine?: number;+ oldText?: string;+ newText?: string;+ replaceAll?: boolean;+ content?: string;/** How long the server waits for this answer. */timeoutS?: number;policy: SitePolicy;@@ -73,9 +80,9 @@ export type ActionContext = {signal?: AbortSignal;};-type Run = ActionContext & { session: Session };+type Run = ActionContext & { session: Session; inFile?: boolean };-type Result = { ok: boolean; error?: string; url?: string; title?: string; text?: string; note?: string; image?: string };+type Result = { ok: boolean; error?: string; url?: string; title?: string; file?: string; text?: string; note?: string; image?: string };const LOAD_TIMEOUT_MS = 30_000;const SETTLE_QUIET_MS = 350;@@ -84,6 +91,8 @@ const FILL_QUIET_MAX_MS = 1_000;// A loaded page that opened no request has drawn what it will; content still on its way arrives by a request.const FILL_IDLE_QUIET_MS = 120;const PAGE_READY_MS = 4_000;+const FILE_FRAME_TRY_MS = 1_500;+const READ_OPS = new Set(["snapshot", "find", "status", "describe", "locate"]);const POST_TIMEOUT_MS = 15_000;const EXPIRED = "Not done: the action ran out of time, likely waiting for the user to answer in the panel. Ask them before trying again.";export const STOPPED = "The user stopped the browser agent. Do not use browser tools again unless they ask.";@@ -92,6 +101,10 @@ const str = (value: unknown, max: number): string | undefined =>typeof value === "string" && value.length <= max ? value : undefined;const num = (value: unknown): number | undefined =>typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined;+const whole = (value: unknown): number | undefined =>+ typeof value === "number" && Number.isInteger(value) && value > 0 ? value : undefined;+// browser_agent._MAX_FILE_EDIT_CHARS+const MAX_EDIT_CHARS = 500_000;const strings = (value: unknown): string[] =>Array.isArray(value) ? value.filter((item): item is string => typeof item === "string").slice(0, 100) : [];@@ -122,6 +135,12 @@ export function parseBrowserAction(value: unknown): BrowserAction | null {fullPage: raw.full_page === true,query: str(raw.query, 256),limit: typeof raw.limit === "number" && Number.isInteger(raw.limit) ? raw.limit : undefined,+ tab: whole(raw.tab),+ startLine: whole(raw.start_line),+ oldText: str(raw.old_text, MAX_EDIT_CHARS),+ newText: str(raw.new_text, MAX_EDIT_CHARS),+ replaceAll: raw.replace_all === true,+ content: str(raw.content, MAX_EDIT_CHARS),timeoutS: typeof raw.timeout_s === "number" && raw.timeout_s > 0 ? Math.min(raw.timeout_s, 3600) : undefined,policy: { allowedDomains: strings(raw.allowed_domains), blockedDomains: strings(raw.blocked_domains) },};@@ -305,7 +324,7 @@ function tabUrl(tab: BrowserTab): string {* page works on in the background, with the user's own tabs left where they were. */function showTab(tab: BrowserTab, context: Run): void {// Kept from the start: the user can switch chats mid-run, and closing the panel then must leave it.- if (useNativeBrowser.getState().enabled) keepInBackground(tab.id);+ if (useNativeBrowser.getState().enabled && currentEntry(tab).kind === "web") keepInBackground(tab.id);// Off screen it shows when the user goes back to that chat; their tabs here stay put. A chat on// screen without room for the panel isn't given one.const store = useBrowserStore.getState();@@ -452,7 +471,8 @@ async function pageCall(tab: BrowserTab, request: Record<string, unknown>): Promcursor,...(cursor ? { cursorColor: cursorAccent(), cursorStill: reducedMotion() } : {}),};- const native = useNativeBrowser.getState().enabled;+ // File previews are frames, never native views.+ const native = useNativeBrowser.getState().enabled && currentEntry(tab).kind === "web";const started = Date.now();const fresh = () => {const base = nextBase;@@ -472,8 +492,10 @@ async function pageCall(tab: BrowserTab, request: Record<string, unknown>): Promif (reply && !reply.installed) reply = await call(fresh(), pageAgentSource());}} else {- reply = await requestFrameAgent(tab.id, command);- if (reply && !reply.installed) reply = await requestFrameAgent(tab.id, fresh(), pageAgentSource());+ // Short tries for reads only: they are safe to repeat.+ const wait = currentEntry(tab).kind === "file" && READ_OPS.has(String(request.op)) ? FILE_FRAME_TRY_MS : undefined;+ reply = await requestFrameAgent(tab.id, command, undefined, wait);+ if (reply && !reply.installed) reply = await requestFrameAgent(tab.id, fresh(), pageAgentSource(), wait);}const answer = reply?.installed ? checkedAnswer(reply.result) : null;if (answer) return answer;@@ -578,6 +600,7 @@ async function askInPanel(question: Question, context: Run, remember = true): Prasync function observe(tab: BrowserTab, context: Run, note?: string, op: Record<string, unknown> = { op: "snapshot" }): Promise<Result> {const entry = currentEntry(tab);+ if (entry.kind === "file") return observeFile(tab, entry, context, note, op);if (entry.kind !== "web") return { ok: false, error: "No web page is open. Use browser_navigate first." };if (tab.pageError || tab.nativeError) {return {@@ -626,7 +649,7 @@ async function open(url: string, context: Run): Promise<BrowserTab | null> {const agent = agentTab(context);const existing = isLoopbackUrl(url)? agent && localOriginGranted(agent.id, url) ? agent : grantedLocalTab(url, context.session.chat)- : agent && !onGrantedOrigin(agent) ? agent : undefined;+ : agent && !onGrantedOrigin(agent) && currentEntry(agent).kind !== "file" ? agent : undefined;if (existing) {if (agentChat(context.session.chat).tabId !== existing.id) moveRun(context, existing.id);store.navigate(existing.id, { url, temporary: context.temporary }, { replace: false });@@ -666,7 +689,12 @@ function stepOf(action: BrowserAction): AgentStep {case "history":case "move":case "drag":+ case "tabs":return { kind: action.op, detail: "" };+ case "read_file":+ return { kind: "readFile", detail: "" };+ case "edit_file":+ return { kind: "editFile", detail: "" };default:return { kind: "read", detail: "" };}@@ -678,8 +706,12 @@ function nameStep(chat: string, info: ElementInfo): void {patchAgentChat(chat, { step: { ...current, detail: info.name || info.role || current.detail } });}-function permissions(context: Pick<ActionContext, "fullAccess">) {- const saved = useBrowserAgentStore.getState().permissions;+function permissions(context: Pick<Run, "fullAccess" | "inFile">) {+ const stored = useBrowserAgentStore.getState().permissions;+ // Preview forms can't send (form-action 'none'); links are still gated.+ const saved = context.inFile+ ? { ...stored, defaults: { ...stored.defaults, actions: "allow" as const, unlabelled: "allow" as const } }+ : stored;if (!context.fullAccess) return saved;const allow = (rule: SiteRule): SiteRule => ({browsing: rule.browsing === "block" ? "block" : "allow",@@ -738,7 +770,7 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {if (agentChat(chat).stoppedRun === context.runKey) return { ok: false, error: STOPPED };if (signal?.aborted) return { ok: false, error: "the action expired before it could run." };// Off screen, only a native page can work on: a framed one runs in the panel.- if (!chatOnScreen(context) && !useNativeBrowser.getState().enabled) {+ if (!PANEL_FREE.has(action.op) && !chatOnScreen(context) && !useNativeBrowser.getState().enabled) {return {ok: false,error:@@ -751,36 +783,29 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {if (owner && owner !== context.runKey) {return { ok: false, error: "Another reply in this chat is using the browser right now. Wait for it to finish, or tell the user." };}+ const front = owner ? null : frontTab(chat);+ if (front && currentEntry(front).kind === "file" && agentChat(chat).tabId !== front.id) moveRun(context, front.id);patchAgentChat(chat, { working: true, runKey: context.runKey, step: stepOf(action) });- if (action.op === "navigate") {- const url = redirectEnd(action.url ?? "");- if (isLoopbackUrl(url) && !grantedLocalTab(url, chat)) {- const { host } = new URL(url);- return {- ok: false,- error: useNativeBrowser.getState().enabled- ? `${host} is on the user's computer, and the browser panel opens a local address only in the tab where the user types it in this chat's address bar. Ask the user to open it there first.`- : `${host} is a local address, and the browser only opens public sites.`,- };- }- const gate = gateNavigation(url, navigationContext(action, context));- const refused = await pass(gate, context);- if (refused) return { ok: false, error: refused };- admit(url, gate, context);- const since = Date.now();- const tab = await open(url, context);- if (!tab) return { ok: false, error: "the browser tab closed." };- return arrive(tab, url, action, context, since);- }+ if (action.op === "navigate") return navigateTo(action.url ?? "", action, context);if (action.op === "history") return readHistory(action, context);if (action.op === "evaluate" && useBrowserAgentStore.getState().scriptAccess === "never") {return { ok: false, error: SCRIPTS_OFF };}- const shown = agentTab(context);- if (!shown) return { ok: false, error: "No page is open yet. Call browser_navigate first." };+ if (action.op === "tabs") return listTabs(action, context);+ if (action.op === "read_file" || action.op === "edit_file") {+ const target = fileTab(context);+ if (!target) return { ok: false, error: NO_FILE };+ useBrowserStore.getState().revealIn(chat, target.id);+ return action.op === "read_file" ? readFile(target, action.startLine ?? 1) : editFile(target, action, signal);+ }++ const shown = agentTab(context) ?? fileTab(context);+ if (!shown) {+ return { ok: false, error: "No page is open yet. Call browser_navigate first, or browser_tabs to work in a tab already open." };+ }showTab(shown, context);// A kept page the cap closed reopens a beat later; its history flags are only good once it has.if (useNativeBrowser.getState().enabled && currentEntry(shown).kind === "web" && !hasNativeView(shown.id)) {@@ -791,7 +816,7 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {const before = tabUrl(tab);if (action.op === "back" || action.op === "forward" || action.op === "reload") {- const native = useNativeBrowser.getState().enabled;+ const native = useNativeBrowser.getState().enabled && currentEntry(tab).kind === "web";const store = useBrowserStore.getState();const nativeMove = native && action.op !== "reload" && Boolean(tab.nativeHistory?.[action.op]);const step = action.op === "back" ? -1 : action.op === "forward" ? 1 : 0;@@ -854,6 +879,19 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {const here = tabUrl(ready);const moved = await stillAllowed(ready, action, context);if (moved) return { ok: false, error: moved };+ const file = currentEntry(ready);+ if (file.kind === "file") {+ const preview = (await kindOf(file)) === "preview";+ if (!preview && action.op !== "snapshot" && action.op !== "find" && action.op !== "screenshot") return { ok: false, error: NO_PAGE_IN_FILE };+ if (action.op === "evaluate") return { ok: false, error: "scripts don't run in a file's preview; change the file with browser_edit_file instead." };+ if (preview && action.op !== "snapshot" && action.op !== "find") {+ if (!chatOnScreen(context)) return { ok: false, error: OFF_SCREEN };+ context.inFile = true;+ showPreview(ready);+ const warm = await pageCall(ready, { op: "status" }).catch((error: Error) => ({ ok: false, error: error.message }));+ if (!warm.ok) return { ok: false, error: warm.error ?? NO_PAGE_IN_FILE };+ }+ }switch (action.op) {case "snapshot":@@ -895,27 +933,30 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {return { ok: false, error: described.error ?? (point ? "nothing is at that point." : `${ref} could not be found.`) };}const info: ElementInfo = described.info;+ const gated = gateable(info, context);if (action.op !== "press") nameStep(chat, action.op === "select" && info.option ? { ...info, name: info.option } : info);const gates: Gate[] = [];- if (action.op === "click" || action.op === "type") gates.push(gateElement(action.op, info, here, action.policy, permissions(context)));+ if (action.op === "click" || action.op === "type") gates.push(gateElement(action.op, gated, here, action.policy, permissions(context)));const opaque = point ? opaqueGate(info, here, context) : null;if (opaque) gates.push(opaque);if (action.op === "select") {// The option the page will select: its value can differ from the label the user sees.- const chosen = info.role === "combobox" ? { ...info, name: info.option || action.value || "" } : info;+ const chosen = info.role === "combobox" ? { ...gated, name: info.option || action.value || "" } : gated;// A card-expiry or other secret dropdown is the user's to fill, as its text field would be.- if (info.tag === "select" || info.role === "combobox") gates.push(gateElement("type", info, here, action.policy, permissions(context)));+ if (info.tag === "select" || info.role === "combobox") gates.push(gateElement("type", gated, here, action.policy, permissions(context)));gates.push(gateElement("select", chosen, here, action.policy, permissions(context)));}// Enter or Space on a focused button or link is a click on it, so it asks what a click would.const activates = action.op === "press" && (action.key === "Enter" || action.key === "Space");- if (activates && !TEXT_ROLES.has(info.role)) gates.push(gateElement("click", info, here, action.policy, permissions(context)));+ // The frame would follow a pressed link unchecked.+ if (context.inFile && activates && info.role === "link") return { ok: false, error: "click the link instead of pressing a key on it." };+ if (activates && !TEXT_ROLES.has(info.role)) gates.push(gateElement("click", gated, here, action.policy, permissions(context)));// Deleting is editing: a password or card field the agent can't type into it can't empty either.if (action.op === "press" && (action.key === "Backspace" || action.key === "Delete")) {- gates.push(gateElement("type", info, here, action.policy, permissions(context)));+ gates.push(gateElement("type", gated, here, action.policy, permissions(context)));// Outside a text field it is an app's delete shortcut, asked like a Delete button.if (!TEXT_ROLES.has(info.role)) {- gates.push(gateElement("click", { ...info, name: `delete key on ${info.name || info.role}` }, here, action.policy, permissions(context)));+ gates.push(gateElement("click", { ...gated, name: `delete key on ${info.name || info.role}` }, here, action.policy, permissions(context)));}}// Enter in a field sends its form, so it is held to what a submit button would be.@@ -926,6 +967,7 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {// A link the page follows itself (a native view's click, Enter on a link) is gated before it// loads; the proxied frame's clicks come back as `navigate` and are gated below.const followsLink =+ !context.inFile &&/^https?:/i.test(info.href) &&info.href.split("#")[0] !== here.split("#")[0] &&((action.op === "click" && useNativeBrowser.getState().enabled) || (activates && info.role === "link"));@@ -938,7 +980,7 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {const wrapperGates = wrappers.map((url) => gateNavigation(url, navigationContext(action, context)));gates.push(...wrapperGates);// A form sending to another site carries what was typed there: a new site asks, as a navigation would.- const formAction = submits && info.formAction ? info.formAction : "";+ const formAction = submits && gated.formAction ? gated.formAction : "";const formGate =formAction && hostOfUrl(formAction) !== hostOfUrl(here) ? gateNavigation(formAction, navigationContext(action, context)) : null;if (formAction) {@@ -971,9 +1013,10 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {op: "click",ref,...(point ? { ...point, expect: described.hit ?? "", sig: described.sig ?? "" } : {}),- directLinks: !useNativeBrowser.getState().enabled,+ directLinks: context.inFile === true || !useNativeBrowser.getState().enabled,});if (!clicked.ok) return { ok: false, error: clicked.error };+ if (clicked.navigate && context.inFile) return openFromFile(clicked.navigate, action, context);if (clicked.navigate) {const pressedAt = Date.now();// A link: opened in this tab, under the same rules as any other address. Its href is read@@ -1017,6 +1060,224 @@ async function perform(action: BrowserAction, context: Run): Promise<Result> {return observe(after, context, note);}+async function navigateTo(address: string, action: BrowserAction, context: Run): Promise<Result> {+ const url = redirectEnd(address);+ if (isLoopbackUrl(url) && !grantedLocalTab(url, context.session.chat)) {+ const { host } = new URL(url);+ return {+ ok: false,+ error: useNativeBrowser.getState().enabled+ ? `${host} is on the user's computer, and the browser panel opens a local address only in the tab where the user types it in this chat's address bar. Ask the user to open it there first.`+ : `${host} is a local address, and the browser only opens public sites.`,+ };+ }+ const gate = gateNavigation(url, navigationContext(action, context));+ const refused = await pass(gate, context);+ if (refused) return { ok: false, error: refused };+ admit(url, gate, context);+ const since = Date.now();+ const tab = await open(url, context);+ if (!tab) return { ok: false, error: "the browser tab closed." };+ return arrive(tab, url, action, context, since);+}++const PANEL_FREE = new Set(["tabs", "read_file", "edit_file"]);+const NO_FILE = "No file is open in the browser panel. browser_tabs lists the tabs.";+const NO_PAGE_IN_FILE =+ "This tab shows a file with no page to act on. browser_read_file reads it and browser_edit_file changes it.";+const OFF_SCREEN = "this chat's browser only runs while the chat is open here; ask the user to switch back to it.";++type FileEntry = Extract<BrowserEntry, { kind: "file" }>;++let readerModule: Promise<typeof import("./file-reader")> | null = null;+const fileReader = () => {+ readerModule ??= import("./file-reader");+ return readerModule;+};++async function kindOf(entry: FileEntry): Promise<FileTabKind> {+ return (await fileReader()).fileTabKind(entry);+}++const KIND_LABELS: Record<FileTabKind, string> = {+ preview: "HTML or React, with a preview",+ text: "text",+ document: "document",+ image: "image",+ media: "audio or video",+ other: "file",+};++async function fileHint(entry: FileEntry): Promise<string> {+ const reader = await fileReader();+ const kind = reader.fileTabKind(entry);+ const hints: string[] = [];+ if (kind === "preview") hints.push("Its preview works as a page: browser_snapshot, browser_click and browser_type act in it.");+ if (kind === "image") hints.push(isTauri ? "browser_screenshot shows it." : "Looking at an image needs the desktop app.");+ if (reader.editable(entry, kind)) hints.push("browser_read_file reads its source and browser_edit_file changes it.");+ else if (kind === "document") hints.push("browser_read_file reads its text; it can't be edited here.");+ else if (kind !== "image") hints.push("It has no text to read.");+ return hints.join(" ");+}++function showPreview(tab: BrowserTab): void {+ useBrowserStore.getState().setFileView(tab.id, { mode: "preview" });+}++function frontTab(chat: string): BrowserTab | null {+ const store = useBrowserStore.getState();+ const id = chat === store.chat ? store.activeTabId : store.chats[chat]?.activeTabId;+ return store.tabs.find((tab) => tab.id === id && tab.chat === chat) ?? null;+}++function fileTab(context: Run): BrowserTab | null {+ const agent = agentTab(context);+ if (agent && currentEntry(agent).kind === "file") return agent;+ const front = frontTab(context.session.chat);+ if (!front || currentEntry(front).kind !== "file") return null;+ moveRun(context, front.id);+ return front;+}++/** Web tabs by site alone: a title is browsing history. */+async function listTabs(action: BrowserAction, context: Run): Promise<Result> {+ const { chat } = context.session;+ const tabs = useBrowserStore.getState().tabs.filter((tab) => tab.chat === chat);+ const where = (tab: BrowserTab) => `a web page on ${hostOfUrl(tabUrl(tab)) || "this computer"}`;+ if (action.tab !== undefined) {+ const tab = tabs[action.tab - 1];+ if (!tab) return { ok: false, error: `There is no tab ${action.tab}; browser_tabs lists them.` };+ const entry = currentEntry(tab);+ if (entry.kind !== "web" && entry.kind !== "file") return { ok: false, error: "That tab has no page or file in it." };+ if (agentChat(chat).tabId !== tab.id) moveRun(context, tab.id);+ showTab(tab, context);+ if (entry.kind === "web") return { ok: true, note: `Switched to tab ${action.tab}, ${where(tab)}. browser_snapshot reads it.` };+ return { ok: true, file: entry.name, note: `Switched to tab ${action.tab}. ${await fileHint(entry)}` };+ }+ const agentId = agentChat(chat).tabId;+ const frontId = frontTab(chat)?.id;+ const rows = await Promise.all(+ tabs.map(async (tab, index) => {+ const entry = currentEntry(tab);+ const what =+ entry.kind === "file"+ ? `File: ${entry.name} (${KIND_LABELS[await kindOf(entry)]})`+ : entry.kind === "web"+ ? `Web page on ${hostOfUrl(tabUrl(tab)) || "this computer"}`+ : entry.kind === "internal"+ ? `The browser's ${entry.page} page`+ : "New tab";+ const marks = [tab.id === agentId ? "the agent's tab" : "", tab.id === frontId ? "in front" : ""].filter(Boolean);+ return `${index + 1}. ${what}${marks.length ? ` (${marks.join(", ")})` : ""}`;+ }),+ );+ return { ok: true, text: rows.join("\n") || "No tabs are open.", note: rows.length ? "browser_tabs with tab switches to one." : undefined };+}++async function fileContent(tab: BrowserTab) {+ const entry = currentEntry(tab);+ const blob = entry.kind === "file" ? browserFile(entry.fileId) : undefined;+ if (entry.kind !== "file" || !blob) return null;+ const reader = await fileReader();+ const kind = reader.fileTabKind(entry);+ return { entry, blob, reader, kind, editable: reader.editable(entry, kind) };+}++async function readFile(tab: BrowserTab, start: number, note?: string): Promise<Result> {+ const file = await fileContent(tab);+ if (!file) return { ok: false, error: NO_FILE };+ const { entry, blob, reader, kind } = file;+ if (!file.editable && kind !== "document") return { ok: true, file: entry.name, note: [note, await fileHint(entry)].filter(Boolean).join(" ") };+ let read: { text: string; cut: boolean };+ try {+ read = await reader.fileText(entry, blob, kind);+ } catch {+ return { ok: false, error: "the file's text could not be read." };+ }+ const lines = numberedLines(read.text, start);+ const cut = read.cut ? "Only the start of this large file was read." : "";+ return { ok: true, file: entry.name, text: lines.text, note: [note, lines.note, cut].filter(Boolean).join(" ") };+}++const MAX_FOUND_LINES = 40;++async function findInFile(tab: BrowserTab, query: string, note: string): Promise<Result> {+ const file = await fileContent(tab);+ if (!file) return { ok: false, error: NO_FILE };+ if (!file.editable && file.kind !== "document") return readFile(tab, 1, note);+ let text: string;+ try {+ ({ text } = await file.reader.fileText(file.entry, file.blob, file.kind));+ } catch {+ return { ok: false, error: "the file's text could not be read." };+ }+ const wanted = query.toLowerCase();+ const found: string[] = [];+ const lines = text.split(/\r?\n/);+ for (let index = 0; index < lines.length && found.length < MAX_FOUND_LINES; index++) {+ const line = lines[index] ?? "";+ if (line.toLowerCase().includes(wanted)) found.push(`${index + 1}\t${line.length > 500 ? `${line.slice(0, 500)} …` : line}`);+ }+ const summary = found.length ? `${found.length === MAX_FOUND_LINES ? "The first " : ""}${found.length} matching lines.` : `No line holds "${query}".`;+ return { ok: true, file: file.entry.name, text: found.join("\n"), note: [note, summary].filter(Boolean).join(" ") };+}++async function editFile(tab: BrowserTab, action: BrowserAction, signal?: AbortSignal): Promise<Result> {+ const file = await fileContent(tab);+ if (!file) return { ok: false, error: NO_FILE };+ const { entry, blob } = file;+ if (!file.editable) {+ return { ok: false, error: "only HTML, React, Markdown, code, text, SVG and CSV files can be edited here, not PDFs, documents, images or media." };+ }+ if (blob.size > MAX_EDIT_BYTES) return { ok: false, error: "this file is too large to edit here." };+ const source = await blob.text();+ const edited = applyFileEdit(source, { oldText: action.oldText, newText: action.newText, replaceAll: action.replaceAll, content: action.content });+ if ("error" in edited) return { ok: false, error: edited.error };+ if (edited.text === source) return { ok: true, file: entry.name, note: "The file already reads that way; nothing changed." };+ if (signal?.aborted) return { ok: false, error: STOPPED };+ const now = findTab(tab.id);+ if (!now || currentEntry(now) !== entry) return { ok: false, error: "the tab changed while the file was being edited; read it again." };+ useBrowserStore.getState().editFile(tab.id, new Blob([edited.text], { type: blob.type || entry.contentType }));+ const changed = (action.newText ?? action.content ?? "").split("\n").length;+ const shown = numberedLines(edited.text, Math.max(1, edited.line - 2)).text.split("\n").slice(0, Math.min(40, changed + 4));+ return {+ ok: true,+ file: entry.name,+ text: shown.join("\n"),+ note: `Edited from line ${edited.line}. The panel shows the new version, and its Back button the one before.`,+ };+}++async function observeFile(tab: BrowserTab, entry: FileEntry, context: Run, note: string | undefined, op: Record<string, unknown>): Promise<Result> {+ if ((await kindOf(entry)) !== "preview") {+ const about = [note, "This tab shows a file, not a page."].filter(Boolean).join(" ");+ return op.op === "find" ? findInFile(tab, String(op.query ?? ""), about) : readFile(tab, 1, about);+ }+ if (!chatOnScreen(context)) return { ok: false, error: OFF_SCREEN };+ showPreview(tab);+ const answer = await pageCall(tab, op);+ if (!answer.ok) return { ok: false, error: answer.error ?? "the page could not be read." };+ const fresh = findTab(tab.id);+ if (!fresh || fresh.loading || currentEntry(fresh) !== entry) {+ return { ok: false, error: "the page changed while it was being read; take a snapshot again." };+ }+ // The agent can write this page, so its links are not the user's.+ if (op.op === "snapshot" || op.op === "find") context.session.pageLinks = new Set();+ const source = op.op === "snapshot" ? "This is a file's preview; browser_read_file and browser_edit_file work on its source." : "";+ return { ok: true, file: entry.name, text: answer.text, note: [note, answer.note, source].filter(Boolean).join(" ") || undefined };+}++function openFromFile(url: string, action: BrowserAction, context: Run): Promise<Result> {+ if (isStudioUrl(url)) {+ return Promise.resolve({+ ok: false,+ error: "that link points at a file beside this one, which the panel doesn't have; link to a full web address, or edit the page.",+ });+ }+ context.inFile = false;+ return navigateTo(url, action, context);+}+/** `since`: when the action went out, so the wait covers the requests it started. */async function afterward(tab: BrowserTab,@@ -1107,7 +1368,7 @@ async function dragBetween(tab: BrowserTab, action: BrowserAction, context: Run,for (const point of [from, end]) {const described = await pageCall(tab, { op: "describe", ref: "", ...point });if (!described.ok || !described.info) return { ok: false, error: described.error ?? "nothing is at that point." };- for (const gate of [gateElement("click", described.info, before, action.policy, permissions(context)), opaqueGate(described.info, before, context)]) {+ for (const gate of [gateElement("click", gateable(described.info, context), before, action.policy, permissions(context)), opaqueGate(described.info, before, context)]) {if (!gate) continue;asked ||= gate.kind === "ask";const refused = await pass(gate, context);@@ -1163,7 +1424,7 @@ async function dragBetween(tab: BrowserTab, action: BrowserAction, context: Run,if (at.x !== end.x || at.y !== end.y) return cancel("the drag stopped short; take a snapshot to see where things are.");if (!target?.ok || !target.info) return cancel(target?.error ?? "nothing is at the drag's end.");if (target.hit !== hits[1] || JSON.stringify(target.info) !== infos[1]) {- for (const gate of [gateElement("click", target.info, before, action.policy, permissions(context)), opaqueGate(target.info, before, context)]) {+ for (const gate of [gateElement("click", gateable(target.info, context), before, action.policy, permissions(context)), opaqueGate(target.info, before, context)]) {if (!gate) continue;const refused = await pass(gate, context);if (refused) return cancel(refused);@@ -1179,6 +1440,10 @@ async function dragBetween(tab: BrowserTab, action: BrowserAction, context: Run,return afterward(tab, before, action, context, since, up.note);}+function gateable(info: ElementInfo, context: Run): ElementInfo {+ return context.inFile ? { ...info, formAction: "" } : info;+}+/** An unlabelled point (a canvas) may be a Buy button nothing can read: ask once per site per chat. */function opaqueGate(info: ElementInfo, pageUrl: string, context: Run): (Gate & { kind: "ask" }) | null {if (info.role !== "element" || info.name.trim() || info.href) return null;@@ -1319,6 +1584,10 @@ async function observeLoaded(tab: BrowserTab, context: Run, since: number, note?}async function screenshot(tab: BrowserTab, context: Run): Promise<Result> {+ const entry = currentEntry(tab);+ const kind = entry.kind === "file" ? await kindOf(entry) : null;+ if (entry.kind === "file" && kind === "image") return imageFile(entry);+ if (entry.kind === "file" && !chatOnScreen(context)) return { ok: false, error: OFF_SCREEN };if (!isTauri) return { ok: false, error: "screenshots of the browser panel need the desktop app; use browser_snapshot." };// A page at work off screen is captured on its own; one on screen as the panel shows it.const hidden = !chatOnScreen(context) && hasNativeView(tab.id);@@ -1326,29 +1595,59 @@ async function screenshot(tab: BrowserTab, context: Run): Promise<Result> {const element = pages[pages.length - 1];if (!hidden && !element) return { ok: false, error: "the browser panel is not on screen." };// What the user typed into card, code and password fields stays out of the picture, as out of the text.- const masked = await pageCall(tab, { op: "mask", on: true }).catch(() => null);+ const page = entry.kind === "web" || kind === "preview";+ const masked: AgentAnswer | null = page ? await pageCall(tab, { op: "mask", on: true }).catch(() => null) : { ok: true };if (!masked?.ok) return { ok: false, error: "the page could not be prepared for a screenshot; use browser_snapshot." };let blob: Blob | null;try {await sleep(100);blob = hidden || !element ? await captureHiddenPage(tab) : await screenshotPage(tab, element);} finally {- await pageCall(tab, { op: "mask", on: false }).catch(() => null);+ if (page) await pageCall(tab, { op: "mask", on: false }).catch(() => null);}if (!blob) return { ok: false, error: "the screenshot failed." };// A page that moved during the capture is one nobody checked or masked: no picture of it.const now = findTab(tab.id);- if (!now || now.loading || tabUrl(now) !== tabUrl(tab)) {+ if (!now || now.loading || tabUrl(now) !== tabUrl(tab) || currentEntry(now) !== entry) {return { ok: false, error: "the page changed while the screenshot was taken; take a snapshot first." };}blob = await toPointScale(tab, context.session, blob, masked);- const image = await new Promise<string>((resolve, reject) => {+ const image = await dataUrl(blob);+ return { ok: true, url: tabUrl(tab), title: tab.title, ...(entry.kind === "file" ? { file: entry.name } : {}), image };+}++function dataUrl(blob: Blob): Promise<string> {+ return new Promise<string>((resolve, reject) => {const reader = new FileReader();reader.onload = () => resolve(String(reader.result));reader.onerror = () => reject(reader.error);reader.readAsDataURL(blob);});- return { ok: true, url: tabUrl(tab), title: tab.title, image };+}++// PNG: the server reads no SVG or AVIF.+async function imageFile(entry: FileEntry): Promise<Result> {+ const blob = browserFile(entry.fileId);+ const url = blob ? URL.createObjectURL(blob) : null;+ if (!url) return { ok: false, error: NO_FILE };+ try {+ const image = new Image();+ image.src = url;+ await image.decode();+ const width = image.naturalWidth || 300;+ const height = image.naturalHeight || 150;+ const fit = Math.min(1, MAX_SHOT_EDGE / Math.max(width, height));+ const canvas = document.createElement("canvas");+ canvas.width = Math.max(1, Math.round(width * fit));+ canvas.height = Math.max(1, Math.round(height * fit));+ canvas.getContext("2d")?.drawImage(image, 0, 0, canvas.width, canvas.height);+ const png = await new Promise<Blob | null>((resolve) => canvas.toBlob(resolve, "image/png"));+ return png ? { ok: true, file: entry.name, image: await dataUrl(png) } : { ok: false, error: "the image could not be read." };+ } catch {+ return { ok: false, error: "the image could not be read." };+ } finally {+ URL.revokeObjectURL(url);+ }}/** At CSS size within mcp_images.MAX_IMAGE_EDGE, so x and y read off it map back exactly. */diff --git a/studio/frontend/src/features/browser/agent/file-reader.ts b/studio/frontend/src/features/browser/agent/file-reader.tsnew file mode 100644index 00000000000..34ad6f356d5--- /dev/null+++ b/studio/frontend/src/features/browser/agent/file-reader.ts@@ -0,0 +1,38 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import { documentKind } from "@/components/file-viewer";+import { mediaKind, textFileKind } from "../file-kind";+import type { BrowserEntry } from "../store";+import { type FileTabKind, MAX_EDIT_BYTES } from "./file-tab";++type FileEntry = Extract<BrowserEntry, { kind: "file" }>;++const TEXT_UNDERNEATH = /\.(svg|csv|tsv)$/i;+const TEXT_UNDERNEATH_TYPE = /^(image\/svg\+xml|text\/csv|text\/tab-separated-values)\b/i;++export function fileTabKind(entry: FileEntry): FileTabKind {+ const { name, contentType, plainText } = entry;+ const text = textFileKind(name, contentType, plainText);+ if (text === "html" || text === "react") return "preview";+ if (text) return "text";+ const media = mediaKind(name, contentType);+ if (media === "image") return "image";+ if (media) return "media";+ return documentKind(name, contentType) ? "document" : "other";+}++export function editable(entry: FileEntry, kind: FileTabKind): boolean {+ return kind === "preview" || kind === "text" || TEXT_UNDERNEATH.test(entry.name) || TEXT_UNDERNEATH_TYPE.test(entry.contentType);+}++export async function fileText(entry: FileEntry, blob: Blob, kind: FileTabKind): Promise<{ text: string; cut: boolean }> {+ if (editable(entry, kind)) {+ const cut = blob.size > MAX_EDIT_BYTES;+ return { text: await (cut ? blob.slice(0, MAX_EDIT_BYTES) : blob).text(), cut };+ }+ const { readAttachmentText } = await import("@/features/chat/attachment-content");+ const file = new File([blob], entry.name, { type: entry.contentType });+ const { text, truncated } = await readAttachmentText(file, entry.name, entry.contentType || undefined);+ return { text, cut: truncated };+}diff --git a/studio/frontend/src/features/browser/agent/file-tab.ts b/studio/frontend/src/features/browser/agent/file-tab.tsnew file mode 100644index 00000000000..c96085b6d15--- /dev/null+++ b/studio/frontend/src/features/browser/agent/file-tab.ts@@ -0,0 +1,57 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++// No imports: executor.ts loads this without the file viewers.++export type FileTabKind = "preview" | "text" | "document" | "image" | "media" | "other";++export type FileEdit = { oldText?: string; newText?: string; replaceAll?: boolean; content?: string };++export const MAX_EDIT_BYTES = 5 * 1024 * 1024;++const READ_CHARS = 20_000;+const LINE_CHARS = 2_000;++export function numberedLines(text: string, start: number): { text: string; note: string } {+ const lines = text.split(/\r?\n/);+ if (lines.length > 1 && lines[lines.length - 1] === "") lines.pop();+ const total = lines.length;+ if (start > total) return { text: "", note: `The file has ${total} line${total === 1 ? "" : "s"}.` };+ const out: string[] = [];+ let used = 0;+ let end = start - 1;+ for (let index = start - 1; index < total; index++) {+ const raw = lines[index] ?? "";+ const line = raw.length > LINE_CHARS ? `${raw.slice(0, LINE_CHARS)} … (line cut; ${raw.length} characters)` : raw;+ const row = `${index + 1}\t${line}`;+ if (out.length && used + row.length > READ_CHARS) break;+ out.push(row);+ used += row.length + 1;+ end = index + 1;+ }+ const rest = end < total ? ` browser_read_file with start_line ${end + 1} reads on.` : "";+ return { text: out.join("\n"), note: `Lines ${start}-${end} of ${total}.${rest}` };+}++const count = (source: string, part: string) => source.split(part).length - 1;++/** Errors are this tool's own words: they reach the model outside the untrusted-content fence. */+export function applyFileEdit(source: string, edit: FileEdit): { text: string; line: number } | { error: string } {+ if (edit.content !== undefined) return { text: edit.content, line: 1 };+ let old = edit.oldText ?? "";+ let next = edit.newText ?? "";+ if (!old) return { error: "old_text is empty." };+ if (!source.includes(old) && source.includes("\r\n") && old.includes("\n")) {+ old = old.replace(/\r?\n/g, "\r\n");+ next = next.replace(/\r?\n/g, "\r\n");+ }+ const found = count(source, old);+ if (found === 0) {+ return { error: "old_text is not in the file. Read it again with browser_read_file and copy the text exactly, without the line numbers." };+ }+ if (found > 1 && !edit.replaceAll) {+ return { error: `old_text appears ${found} times. Add the text around it so it matches once, or set replace_all.` };+ }+ const line = count(source.slice(0, source.indexOf(old)), "\n") + 1;+ return { text: edit.replaceAll ? source.split(old).join(next) : source.replace(old, () => next), line };+}diff --git a/studio/frontend/src/features/browser/agent/page-agent.ts b/studio/frontend/src/features/browser/agent/page-agent.tsindex 947de523f23..6818c060689 100644--- a/studio/frontend/src/features/browser/agent/page-agent.ts+++ b/studio/frontend/src/features/browser/agent/page-agent.ts@@ -376,7 +376,8 @@ export function installPageAgent(): PageAgent {};const flush = () => {const text = clean(buffer, 400);- if (text && bufferShown && text.length > 1) push(text);+ // A lone symbol is an icon or a bullet; a lone digit or letter is a count or a grade.+ if (text && bufferShown && (text.length > 1 || /[\p{L}\p{N}]/u.test(text))) push(text);buffer = "";bufferShown = false;};@@ -821,7 +822,8 @@ export function installPageAgent(): PageAgent {const href = anchor?.href ?? "";if (!anchor || !/^https?:/i.test(href)) return null;const target = new URL(href);- const hashOnly = target.hash && target.href.split("#")[0] === pageUrl().href.split("#")[0];+ // href="#" has an empty hash, yet stays on the page like any other fragment.+ const hashOnly = href.includes("#") && target.href.split("#")[0] === pageUrl().href.split("#")[0];return !hashOnly && (directLinks || anchor.target === "_blank") ? href : null;};diff --git a/studio/frontend/src/features/browser/agent/tools.ts b/studio/frontend/src/features/browser/agent/tools.tsindex dc8a0ff24dc..c81082ef5d0 100644--- a/studio/frontend/src/features/browser/agent/tools.ts+++ b/studio/frontend/src/features/browser/agent/tools.ts@@ -18,6 +18,9 @@ export const BROWSER_AGENT_TOOLS = ["browser_select","browser_scroll","browser_press_key",+ "browser_tabs",+ "browser_read_file",+ "browser_edit_file",] as const;/** Captured from the panel without a prompt only in the desktop app; the server keeps it for vision models. */diff --git a/studio/frontend/src/features/browser/page-frame.tsx b/studio/frontend/src/features/browser/page-frame.tsxindex f908d83a8c0..6cd3cf7947f 100644--- a/studio/frontend/src/features/browser/page-frame.tsx+++ b/studio/frontend/src/features/browser/page-frame.tsx@@ -89,7 +89,7 @@ export function onFrameAnnotate(tabId: string, listener: (event: AnnotateEvent)};}-/** Register an artifact's opaque-origin frame for annotation commands only. */+/** Register an artifact's opaque-origin frame for annotation and browser agent commands only. */export function registerArtifactFrame(tabId: string, frame: HTMLIFrameElement): () => void {frames.set(tabId, frame);for (const notify of frameListeners) notify();@@ -97,6 +97,7 @@ export function registerArtifactFrame(tabId: string, frame: HTMLIFrameElement):if (event.source !== frame.contentWindow || event.origin !== "null") return;const message = parseFrameMessage(event.data);if (message?.type === "annotate") annotateListeners.get(tabId)?.(message.event);+ else if (message?.type === "agent") agentRequests.get(message.id)?.({ installed: message.installed, result: message.result });};window.addEventListener("message", listener);return () => {diff --git a/studio/frontend/src/features/browser/store.ts b/studio/frontend/src/features/browser/store.tsindex 66aa12f84d0..0bc6a0eb0e0 100644--- a/studio/frontend/src/features/browser/store.ts+++ b/studio/frontend/src/features/browser/store.ts@@ -366,6 +366,7 @@ type BrowserState = {options?: { newTab?: boolean; background?: boolean; method?: "GET" | "POST"; body?: string; from?: string },) => void;openFile: (input: OpenFileInput) => void;+ editFile: (tabId: string, blob: Blob) => void;navigate: (tabId: string,request: { url: string; method?: "GET" | "POST"; body?: string; from?: string; temporary?: boolean },@@ -730,6 +731,16 @@ export const useBrowserStore = create<BrowserState>((set, get) => {openTab(tab);if (page) get().setFileView(tab.id, { pageTarget: { page } });},+ editFile: (tabId, blob) => {+ const tab = get().tabs.find((candidate) => candidate.id === tabId);+ const shown = tab && currentEntry(tab);+ if (shown?.kind !== "file") return;+ const fileId = newId("file");+ files.set(fileId, blob);+ // No openKey: a reopen from chat would replace the edit in place.+ const { openKey: _, ...rest } = shown;+ set((state) => ({ tabs: patchTab(state.tabs, tabId, (current) => pushEntry(current, { ...rest, fileId })) }));+ },navigate: (tabId, request, options) => {if (!isWeb(request.url)) return;set((state) => ({diff --git a/studio/frontend/src/features/chat/api/durable-gate.ts b/studio/frontend/src/features/chat/api/durable-gate.tsindex d266941f583..47592b17e51 100644--- a/studio/frontend/src/features/chat/api/durable-gate.ts+++ b/studio/frontend/src/features/chat/api/durable-gate.ts@@ -25,6 +25,9 @@ export const BROWSER_EXECUTED_TOOLS: ReadonlySet<string> = new Set<string>(["browser_evaluate","browser_screenshot","browser_history",+ "browser_tabs",+ "browser_read_file",+ "browser_edit_file",]);/**diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.tsindex 0dd1356b54f..e5a70521c35 100644--- a/studio/frontend/src/i18n/locales/en.ts+++ b/studio/frontend/src/i18n/locales/en.ts@@ -195,6 +195,9 @@ export const en = {history: "Reading your browsing history",move: "Moving the mouse",drag: "Dragging",+ tabs: "Looking at the tabs",+ readFile: "Reading the file",+ editFile: "Editing the file",},},title: "Browser",diff --git a/studio/frontend/tests/artifact-frame-agent.test.ts b/studio/frontend/tests/artifact-frame-agent.test.tsnew file mode 100644index 00000000000..270c9de38f7--- /dev/null+++ b/studio/frontend/tests/artifact-frame-agent.test.ts@@ -0,0 +1,91 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import { readFileSync } from "node:fs";+import test from "node:test";+import vm from "node:vm";++const backend = readFileSync(new URL("../../backend/routes/inference.py", import.meta.url), "utf8");+const shell = backend.match(/_ARTIFACT_PREVIEW_FRAME_HTML = """([\s\S]*?)"""/)![1]!.replace(/\\(["\\])/g, "$1");+const script = shell.match(/<script>([\s\S]*?)<\/script>/)![1]!;++function preview() {+ const listeners = new Map<string, Array<(event: unknown) => void>>();+ const reports: Array<Record<string, unknown>> = [];+ const parent = { postMessage: (value: Record<string, unknown>) => reports.push(value) };+ let context: vm.Context;+ class Node {+ appendChild(child: { source: string }) {+ vm.runInContext(child.source, context);+ return child;+ }+ }+ class Element extends Node {+ remove() {}+ }+ class HTMLScriptElement extends Element {+ source = "";+ set text(value: string) {+ this.source = value;+ }+ }+ class Document extends Node {+ get documentElement() {+ return new Element();+ }+ createElement() {+ return new HTMLScriptElement();+ }+ open() {+ listeners.clear();+ }+ write() {}+ close() {}+ addEventListener() {}+ }+ class Crypto {+ getRandomValues(words: Uint32Array) {+ return words.fill(7);+ }+ }+ context = vm.createContext(+ {+ parent, URLSearchParams, location: { search: "?v=test" }, Document, Node, Element, HTMLScriptElement, Crypto,+ crypto: new Crypto(), document: new Document(),+ console: Object.fromEntries(["error", "warn", "info", "log", "debug"].map((level) => [level, () => {}])),+ addEventListener: (type: string, listener: (event: unknown) => void) => {+ listeners.set(type, [...(listeners.get(type) ?? []), listener]);+ },+ },+ { codeGeneration: { strings: false } },+ );+ context.window = context;+ vm.runInContext(script.replace("/* ANNOTATE_JS */", "return {};"), context);+ const message = (data: unknown, source: unknown = parent) => {+ for (const listener of [...(listeners.get("message") ?? [])]) listener({ source, data });+ };+ message({ type: "unsloth:artifact-html", html: "<h1>Coffee</h1>" });+ return { context, message, reports };+}++const plain = (value: unknown) => JSON.parse(JSON.stringify(value));+const agentCode = "(() => ({ run: (op) => ({ ok: true, text: `ran ${op.op}` }) }))();";+const agent = (id: string, fields = {}) => ({ type: "unsloth:browser-command", command: "agent", id, op: { op: "snapshot" }, ...fields });++test("the canvas installs the browser agent from an inline script and answers its commands", () => {+ const frame = preview();+ frame.message(agent("a1"));+ assert.deepEqual(plain(frame.reports.at(-1)), { source: "unsloth-browser", type: "agent", id: "a1", installed: false, result: null });+ frame.message(agent("a2", { code: agentCode }));+ assert.deepEqual(plain(frame.reports.at(-1)), { source: "unsloth-browser", type: "agent", id: "a2", installed: true, result: { ok: true, text: "ran snapshot" } });+ frame.message(agent("a3"));+ assert.equal(frame.reports.at(-1)?.installed, true);+ assert.deepEqual(Object.keys(frame.context).filter((key) => key.startsWith("unslothAgent")), []);+});++test("only the embedder can install the agent", () => {+ const frame = preview();+ frame.message(agent("a1", { code: agentCode }), frame.context);+ assert.equal(frame.reports.some((report) => report.type === "agent"), false);+});diff --git a/studio/frontend/tests/browser-agent-files.test.ts b/studio/frontend/tests/browser-agent-files.test.tsnew file mode 100644index 00000000000..87fb9db91bb--- /dev/null+++ b/studio/frontend/tests/browser-agent-files.test.ts@@ -0,0 +1,238 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import { register } from "node:module";+import { test } from "node:test";++register("./helpers/browser-executor-resolver.mjs", import.meta.url);+const { runBrowserAction } = await import("../src/features/browser/agent/executor.ts");+const { agentChat, answerQuestion, endBrowserAgentRun, stopBrowserAgent } = await import("../src/features/browser/agent/agent-store.ts");+const { applyFileEdit, numberedLines } = await import("../src/features/browser/agent/file-tab.ts");+const { browserFile, currentEntry, useBrowserStore } = await import("../src/features/browser/store.ts");++const globals = globalThis as Record<string, unknown>;+const policy = { allowedDomains: [], blockedDomains: [] };+const RUN = "r";+type Result = { ok: boolean; error?: string; text?: string; note?: string; file?: string };++let next = 0;+async function act(op: string, fields: Record<string, unknown> = {}, runKey = RUN): Promise<Result> {+ const posted: { result: Result }[] = [];+ globals.__authFetch = async (_url: string, init: { body: string }) => {+ posted.push(JSON.parse(init.body));+ return new Response(null, { status: 200 });+ };+ await runBrowserAction({ id: `f${next++}`, sessionId: "s", op, direction: "down", policy, ...fields }, { runKey, userText: "" });+ assert.equal(posted.length, 1);+ return posted[0].result;+}++function shownText(): Promise<string> {+ const store = useBrowserStore.getState();+ const tab = store.tabs.find((candidate) => candidate.id === store.activeTabId);+ const entry = tab && currentEntry(tab);+ assert.equal(entry?.kind, "file");+ return browserFile(entry.kind === "file" ? entry.fileId : "")?.text() ?? Promise.resolve("");+}++test("numbered lines page through a file and say where to go on", () => {+ const text = Array.from({ length: 3000 }, (_, i) => `line ${i + 1}`).join("\n");+ const first = numberedLines(text, 1);+ assert.match(first.text, /^1\tline 1\n2\tline 2/);+ const end = Number(/Lines 1-(\d+) of 3000/.exec(first.note)?.[1]);+ assert.ok(end > 100 && end < 3000);+ assert.match(first.note, new RegExp(`start_line ${end + 1}`));+ assert.equal(numberedLines(text, 2999).note, "Lines 2999-3000 of 3000.");+ assert.equal(numberedLines("a\n", 5).note, "The file has 1 line.");+});++test("an edit replaces text found once, refuses ambiguous or missing text, and keeps CRLF", () => {+ assert.deepEqual(applyFileEdit("a\nb\nc", { oldText: "b", newText: "B" }), { text: "a\nB\nc", line: 2 });+ assert.match((applyFileEdit("x x", { oldText: "x", newText: "y" }) as { error: string }).error, /appears 2 times/);+ assert.deepEqual(applyFileEdit("x x", { oldText: "x", newText: "y", replaceAll: true }), { text: "y y", line: 1 });+ assert.match((applyFileEdit("abc", { oldText: "z", newText: "y" }) as { error: string }).error, /not in the file/);+ assert.deepEqual(applyFileEdit("a\r\nb\r\nc", { oldText: "a\nb", newText: "a\nB" }), { text: "a\r\nB\r\nc", line: 1 });+ assert.deepEqual(applyFileEdit("price", { oldText: "price", newText: "$&5" }), { text: "$&5", line: 1 });+ assert.deepEqual(applyFileEdit("old", { content: "new" }), { text: "new", line: 1 });+});++test("the agent reads and edits the HTML file in front, and Back shows the version before", async () => {+ endBrowserAgentRun(RUN);+ const html = "<!doctype html>\n<h1>Hello</h1>\n<button>Go</button>\n";+ useBrowserStore.getState().openFile({ blob: new Blob([html], { type: "text/html" }), name: "page.html", contentType: "text/html", key: "html:a1" });+ const read = await act("read_file", { startLine: 1 });+ assert.equal(read.ok, true);+ assert.equal(read.file, "page.html");+ assert.match(read.text ?? "", /^1\t<!doctype html>\n2\t<h1>Hello<\/h1>/);++ const edited = await act("edit_file", { oldText: "<h1>Hello</h1>", newText: "<h1>Hi there</h1>" });+ assert.equal(edited.ok, true, edited.error);+ assert.match(edited.text ?? "", /2\t<h1>Hi there<\/h1>/);+ assert.equal(await shownText(), html.replace("Hello", "Hi there"));++ const missing = await act("edit_file", { oldText: "<h1>Hello</h1>", newText: "x" });+ assert.equal(missing.ok, false);+ assert.match(missing.error ?? "", /not in the file/);++ const store = useBrowserStore.getState();+ store.goBack(store.activeTabId ?? "");+ assert.equal(await shownText(), html);+});++test("a file's preview is read through its frame, and its links are not taken as the user's", async () => {+ endBrowserAgentRun(RUN);+ useBrowserStore.getState().openFile({ blob: new Blob(["<a href='https://example.com/'>x</a>"], { type: "text/html" }), name: "links.html", contentType: "text/html", key: "html:a2" });+ globals.__frameAgent = async (_tabId: string, op: Record<string, unknown>) => {+ if (op.op === "status") return { installed: true, result: { ok: true, quietMs: 1000 } };+ return { installed: true, result: { ok: true, text: "link r1 x", links: ["https://example.com/"] } };+ };+ const snapshot = await act("snapshot");+ assert.equal(snapshot.ok, true, snapshot.error);+ assert.equal(snapshot.file, "links.html");+ assert.equal(snapshot.text, "link r1 x");+ assert.match(snapshot.note ?? "", /browser_edit_file/);++ globals.__native = true;+ try {+ const desktop = await act("snapshot");+ assert.equal(desktop.text, "link r1 x", desktop.error);+ } finally {+ globals.__native = false;+ }++ let navigate = "https://example.com/";+ globals.__frameAgent = async (_tabId: string, op: Record<string, unknown>) => {+ if (op.op === "status") return { installed: true, result: { ok: true, quietMs: 1000 } };+ if (op.op === "describe") return { installed: true, result: { ok: true, info: { role: "link", name: "x", tag: "a", href: navigate } } };+ return { installed: true, result: { ok: true, navigate } };+ };+ const pending = act("click", { ref: "r1" });+ let question = agentChat("new").question;+ for (let i = 0; !question && i < 200; i++) {+ await new Promise((resolve) => setTimeout(resolve, 10));+ question = agentChat("new").question;+ }+ assert.equal(question?.ask, "open");+ assert.equal(question?.host, "example.com");+ answerQuestion(question.id, false);+ assert.match((await pending).error ?? "", /declined opening a page/);++ navigate = "http://localhost/api/inference/other.html";+ const beside = await act("click", { ref: "r1" });+ assert.match(beside.error ?? "", /beside this one/);+});++test("a preview that misses the first call, as one just switched on from Source does, still answers", async () => {+ endBrowserAgentRun(RUN);+ let calls = 0;+ globals.__frameAgent = async (_tabId: string, op: Record<string, unknown>, _code: string, timeoutMs: number) => {+ if (calls++ === 0) return new Promise((resolve) => setTimeout(() => resolve(null), timeoutMs));+ if (op.op === "status") return { installed: true, result: { ok: true, quietMs: 1000 } };+ return { installed: true, result: { ok: true, text: "ready", links: [] } };+ };+ const snapshot = await act("snapshot");+ assert.equal(snapshot.text, "ready", snapshot.error);+});++test("a click that takes the preview a while is sent once, not again after a short wait", async () => {+ endBrowserAgentRun(RUN);+ let clicks = 0;+ globals.__frameAgent = (_tabId: string, op: Record<string, unknown>, _code: string, timeoutMs: number) => {+ const answer = (value: unknown, after = 0) =>+ Promise.race([new Promise((resolve) => setTimeout(() => resolve(value), after)), new Promise((resolve) => setTimeout(() => resolve(null), timeoutMs))]);+ if (op.op === "click") {+ clicks++;+ return answer({ installed: true, result: { ok: true } }, 2000);+ }+ if (op.op === "describe") return answer({ installed: true, result: { ok: true, info: { role: "button", name: "Go", tag: "button" } } });+ if (op.op === "status") return answer({ installed: true, result: { ok: true, quietMs: 1000 } });+ return answer({ installed: true, result: { ok: true, text: "page", links: [] } });+ };+ const click = await act("click", { ref: "r1" });+ assert.equal(click.ok, true, click.error);+ assert.equal(clicks, 1);+});++test("a screenshot of a file in a chat that is off screen is refused, not taken of the panel shown", async () => {+ endBrowserAgentRun(RUN);+ useBrowserStore.getState().openFile({ blob: new Blob(["notes"], { type: "text/plain" }), name: "shot.txt", contentType: "text/plain", key: "f:shot" });+ globals.__native = true;+ globals.__panel = false;+ try {+ const shot = await act("screenshot");+ assert.equal(shot.ok, false);+ assert.match(shot.error ?? "", /only runs while the chat is open/);+ } finally {+ globals.__native = false;+ globals.__panel = true;+ }+});++test("reloading a file's preview in the desktop app reloads its frame", async () => {+ endBrowserAgentRun(RUN);+ useBrowserStore.getState().openFile({ blob: new Blob(["<p>x</p>"], { type: "text/html" }), name: "reload.html", contentType: "text/html", key: "html:reload" });+ globals.__frameAgent = async (_tabId: string, op: Record<string, unknown>) =>+ op.op === "status" ? { installed: true, result: { ok: true, quietMs: 1000 } } : { installed: true, result: { ok: true, text: "x", links: [] } };+ const store = useBrowserStore.getState();+ const before = store.tabs.find((tab) => tab.id === store.activeTabId)?.reloadKey;+ globals.__native = true;+ try {+ const reloaded = await act("reload");+ assert.equal(reloaded.ok, true, reloaded.error);+ } finally {+ globals.__native = false;+ }+ const after = useBrowserStore.getState().tabs.find((tab) => tab.id === store.activeTabId)?.reloadKey;+ assert.equal(after, (before ?? 0) + 1);+});++test("an edit stopped while the file is read leaves the file as it was", async () => {+ endBrowserAgentRun(RUN);+ const source = "<h1>Keep</h1>";+ let release = () => {};+ const released = new Promise<void>((resolve) => { release = resolve; });+ const slow = Object.assign(new Blob([source], { type: "text/html" }), { text: () => released.then(() => source) });+ useBrowserStore.getState().openFile({ blob: slow, name: "stop.html", contentType: "text/html", key: "html:stop" });+ const pending = act("edit_file", { oldText: "Keep", newText: "Changed" }, "stopped-run");+ for (let i = 0; i < 100 && agentChat("new").runKey !== "stopped-run"; i++) await new Promise((resolve) => setTimeout(resolve, 5));+ await new Promise((resolve) => setTimeout(resolve, 20));+ stopBrowserAgent("new");+ release();+ const result = await pending;+ assert.equal(result.ok, false);+ assert.equal(await shownText(), source);+ endBrowserAgentRun("stopped-run");+});++test("a PDF is read as text but not edited, and a text file has nothing to click", async () => {+ endBrowserAgentRun(RUN);+ useBrowserStore.getState().openFile({ blob: new Blob(["report body"], { type: "application/pdf" }), name: "report.pdf", contentType: "application/pdf", key: "f:pdf" });+ const read = await act("read_file");+ assert.equal(read.text, "1\treport body");+ const edit = await act("edit_file", { content: "x" });+ assert.equal(edit.ok, false);+ assert.match(edit.error ?? "", /not PDFs/);++ useBrowserStore.getState().openFile({ blob: new Blob(["alpha\nbeta\n"], { type: "text/plain" }), name: "notes.txt", contentType: "text/plain", key: "f:txt" });+ await act("tabs", { tab: useBrowserStore.getState().tabs.filter((tab) => tab.chat === useBrowserStore.getState().chat).length });+ const found = await act("find", { query: "BETA" });+ assert.equal(found.text, "2\tbeta");+ const click = await act("click", { ref: "r1" });+ assert.equal(click.ok, false);+ assert.match(click.error ?? "", /no page to act on/);+});++test("browser_tabs lists files by name and web pages by site, and switches by number", async () => {+ endBrowserAgentRun(RUN);+ const listed = await act("tabs");+ assert.equal(listed.ok, true);+ assert.match(listed.text ?? "", /File: page\.html \(HTML or React, with a preview\)/);+ assert.match(listed.text ?? "", /File: report\.pdf \(document\)/);+ assert.match(listed.text ?? "", /\(the agent's tab, in front\)/);+ const missing = await act("tabs", { tab: 99 });+ assert.equal(missing.ok, false);+ const switched = await act("tabs", { tab: 2 });+ assert.equal(switched.ok, true);+ assert.equal(switched.file, "links.html");+});diff --git a/studio/frontend/tests/helpers/browser-executor-resolver.mjs b/studio/frontend/tests/helpers/browser-executor-resolver.mjsindex 2d0941aec1c..85c8b1018c4 100644--- a/studio/frontend/tests/helpers/browser-executor-resolver.mjs+++ b/studio/frontend/tests/helpers/browser-executor-resolver.mjs@@ -9,7 +9,7 @@ const stub = (source) => ({ url: `data:text/javascript,${encodeURIComponent(sourconst STUBS = {"../page-frame":- "export const requestFrameAgent = (tabId, op, code) => globalThis.__frameAgent(tabId, op, code);" ++ "export const requestFrameAgent = (tabId, op, code, timeoutMs = 5000) => globalThis.__frameAgent(tabId, op, code, timeoutMs);" +" export const sendFrameCommand = (tabId, command) => globalThis.__frameCommand?.(tabId, command) ?? false;","../capture": "export const screenshotPage = async () => null; export const captureHiddenPage = async () => null;",// Native pages only when a test sets globalThis.__native; it answers their calls via __callNative.@@ -24,6 +24,10 @@ const STUBS = {" export const localOriginGranted = (tabId, url) => globalThis.__granted?.get(tabId) === new URL(url).origin;","../panel-availability": "export const browserPanelAvailable = () => globalThis.__panel !== false;","../history-store": "export const useBrowserHistoryStore = { getState: () => ({ recordVisit() {} }) };",+ "./file-reader":+ "export const fileTabKind = ({ name }) => /\\.html?$/.test(name) ? 'preview' : /\\.pdf$/.test(name) ? 'document' : /\\.png$/.test(name) ? 'image' : 'text';" ++ " export const editable = (entry, kind) => kind === 'preview' || kind === 'text';" ++ " export const fileText = async (entry, blob) => ({ text: await blob.text(), cut: false });",};export function resolve(specifier, context, next) {