Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio browser: annotate code on demand, no idle desktop polling, smaller page cache on low-memory machines (#12734)
Daniel Hanc47671dd6c07
diff --git a/studio/backend/routes/browser.py b/studio/backend/routes/browser.pyindex 05cc1bbf9fc..47c42ef4913 100644--- a/studio/backend/routes/browser.py+++ b/studio/backend/routes/browser.py@@ -104,6 +104,8 @@ _FRAME_HTML = r"""<!doctype html><body><script>const boot = (cfg) => {+ // Captured before the page's scripts run, so they can't swap it.+ const compile = Function;// about:srcdoc has an opaque origin, and data: and blob: URLs share it.const shellOrigin = location.origin === "null" ? null : location.origin;let pageUrl = cfg.url || location.href;@@ -425,10 +427,234 @@ _FRAME_HTML = r"""<!doctype html>};})();if (cfg.muted) setMuted(true);- // Annotate: the panel marks parts of the page to ask about. While it is on, pointer input- // is the panel's: the page only reports the block under the pointer, what a click or drag- // marks, and where the marks sit as it scrolls.- const annotation = (() => {+ // Annotate: the panel marks parts of the page to ask about (_ANNOTATE_JS). Its code comes from+ // the panel when annotate mode turns on, so pages never annotated don't parse it. Until then+ // only the latest on/off and numbering are kept, to apply once it is installed.+ let annotation = null, wanted = null, numbers = null;+ const install = (code) => {+ if (annotation || typeof code !== "string" || code.length > 262144) return;+ try { annotation = compile("post", code)(post); } catch { return; }+ if (wanted?.on) annotation.start(wanted.color);+ if (numbers) annotation.number(numbers);+ wanted = numbers = null;+ };+ // The page as it is now, for printing: scripts' DOM, typed values and canvases included.+ // Printed from a copy in a separate frame, as this sandbox can't open the print dialog.+ const snapshot = () => {+ try {+ const source = document.documentElement;+ const copy = source.cloneNode(true);+ const twins = (selector) => [source.querySelectorAll(selector), copy.querySelectorAll(selector)];+ const [fields, fieldCopies] = twins("input, textarea, select");+ fields.forEach((field, index) => {+ const twin = fieldCopies[index];+ if (!twin) return;+ if (field.tagName === "TEXTAREA") twin.textContent = field.value;+ else if (field.tagName === "SELECT") [...twin.options].forEach((option, at) => option.toggleAttribute("selected", Boolean(field.options[at]?.selected)));+ else if (field.type === "checkbox" || field.type === "radio") twin.toggleAttribute("checked", field.checked);+ else if (field.type !== "password" && field.type !== "file") twin.setAttribute("value", field.value);+ });+ const [canvases, canvasCopies] = twins("canvas");+ canvases.forEach((canvas, index) => {+ const twin = canvasCopies[index];+ if (!twin) return;+ try {+ const image = document.createElement("img");+ for (const name of ["class", "style", "width", "height"]) if (twin.hasAttribute(name)) image.setAttribute(name, twin.getAttribute(name));+ image.src = canvas.toDataURL();+ twin.replaceWith(image);+ } catch {}+ });+ // Styles added through the CSSOM (CSS-in-JS) aren't in the markup.+ const [styles, styleCopies] = twins("style");+ styles.forEach((style, index) => {+ try {+ const rules = style.sheet ? [...style.sheet.cssRules].map((rule) => rule.cssText).join("\n") : null;+ if (rules && styleCopies[index]) styleCopies[index].textContent = rules;+ } catch {}+ });+ try {+ const adopted = (document.adoptedStyleSheets || []).flatMap((sheet) => [...sheet.cssRules].map((rule) => rule.cssText));+ if (adopted.length) {+ const extra = document.createElement("style");+ extra.textContent = adopted.join("\n");+ (copy.querySelector("head") || copy).appendChild(extra);+ }+ } catch {}+ for (const node of copy.querySelectorAll("script, [data-unsloth-annotate]")) node.remove();+ const html = "<!doctype html>" + copy.outerHTML;+ return html.length <= 8 * 1024 * 1024 ? html : null;+ } catch {+ return null;+ }+ };++ // Find in page for Studio's find bar: every visible match of the query, case-insensitive,+ // painted with the same two highlights as the chat's (all matches, then the active one) and+ // counted back to the bar. Highlights sit over the text, so the page itself is not changed.+ const finder = (() => {+ const MAX = 1000;+ const SKIP = new Set(["SCRIPT", "STYLE", "NOSCRIPT", "TEMPLATE", "SELECT", "OPTION", "TEXTAREA"]);+ const painted = typeof Highlight === "function" && typeof CSS !== "undefined" && CSS.highlights;+ let ranges = [];+ let active = -1;+ let styled = false;+ const style = () => {+ if (styled || !painted) return;+ styled = true;+ const sheet = document.createElement("style");+ sheet.textContent = "::highlight(unsloth-find){background-color:#ffd84d;color:#1a1a1a}::highlight(unsloth-find-active){background-color:#ff8c1a;color:#1a1a1a}";+ (document.head || document.documentElement).appendChild(sheet);+ };+ const shown = (element) => {+ if (typeof element.checkVisibility === "function") return element.checkVisibility({ visibilityProperty: true });+ return element.getClientRects().length > 0;+ };+ const collect = (query) => {+ const needle = query.toLowerCase();+ const found = [];+ if (!document.body) return found;+ const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, {+ acceptNode: (node) => {+ const element = node.parentElement;+ if (!element || SKIP.has(element.tagName) || element.closest("[data-unsloth-annotate]")) return NodeFilter.FILTER_REJECT;+ return NodeFilter.FILTER_ACCEPT;+ },+ });+ for (let node = walker.nextNode(); node && found.length < MAX; node = walker.nextNode()) {+ const text = node.nodeValue || "";+ const lower = text.toLowerCase();+ // Lowercasing can change a few characters' lengths; those nodes would map wrongly.+ if (lower.length !== text.length) continue;+ let at = lower.indexOf(needle);+ if (at < 0 || !shown(node.parentElement)) continue;+ while (at >= 0 && found.length < MAX) {+ const range = document.createRange();+ range.setStart(node, at);+ range.setEnd(node, at + needle.length);+ found.push(range);+ at = lower.indexOf(needle, at + needle.length);+ }+ }+ return found;+ };+ const paint = () => {+ if (!painted) return;+ style();+ if (ranges.length) CSS.highlights.set("unsloth-find", new Highlight(...ranges));+ else CSS.highlights.delete("unsloth-find");+ if (ranges[active]) CSS.highlights.set("unsloth-find-active", new Highlight(ranges[active]));+ else CSS.highlights.delete("unsloth-find-active");+ };+ const reveal = () => {+ const range = ranges[active];+ if (!range) return;+ const rect = range.getBoundingClientRect();+ if (rect.top < 48 || rect.bottom > innerHeight - 24 || rect.left < 0 || rect.right > innerWidth) {+ range.startContainer.parentElement?.scrollIntoView({ block: "center", inline: "nearest" });+ }+ // Without highlights, the selection marks the match instead.+ if (!painted) {+ const selection = getSelection();+ selection?.removeAllRanges();+ selection?.addRange(range);+ }+ };+ const report = () => post({ type: "findResult", count: ranges.length, active });+ return {+ search: (query) => {+ ranges = query ? collect(query) : [];+ // From where the reader is: the first match at or below the top of the view.+ const below = ranges.findIndex((range) => range.getBoundingClientRect().bottom >= 0);+ active = ranges.length ? Math.max(below, 0) : -1;+ paint();+ reveal();+ report();+ },+ step: (delta) => {+ if (ranges.length) {+ active = (active + delta + ranges.length) % ranges.length;+ paint();+ reveal();+ }+ report();+ },+ };+ })();+ // Commands from the panel, relayed by the shell (the only parent this page has).+ window.addEventListener("message", (event) => {+ if (event.source !== parent) return;+ const data = event.data;+ if (!data || data.type !== "unsloth:browser-command") return;+ if (data.command === "annotateInstall") install(data.code);+ else if (data.command === "annotate") {+ if (!annotation) wanted = { on: data.on === true, color: data.color };+ else if (data.on === true) annotation.start(data.color);+ else annotation.stop();+ }+ else if (data.command === "annotateForget") annotation?.forget(Number(data.id));+ else if (data.command === "annotateNumbers") annotation ? annotation.number(data.numbers) : (numbers = data.numbers);+ else if (data.command === "zoom") applyZoom(data.value);+ else if (data.command === "mute") setMuted(data.on === true);+ else if (data.command === "find" && typeof data.query === "string" && data.query.length <= 1000) finder.search(data.query);+ else if (data.command === "findStep") finder.step(data.delta === -1 ? -1 : 1);+ else if (data.command === "snapshot") post({ type: "snapshot", html: snapshot() });+ });+ // The panel may have asked for annotate before this page could hear it.+ post({ type: "annotate", event: "ready" });+ };+ const escapeAttr = (value) => value.replace(/&/g, "&").replace(/"/g, """).replace(/</g, "<");+ const inject = (html, tags) => {+ const at = (match) => match.index + match[0].length;+ const head = /<head\b[^<>]*>/i.exec(html);+ if (head) return html.slice(0, at(head)) + tags + html.slice(at(head));+ const root = /<html\b[^<>]*>/i.exec(html) || /^\s*<!doctype[^>]*>/i.exec(html);+ if (root) return html.slice(0, at(root)) + "<head>" + tags + "</head>" + html.slice(at(root));+ return "<head>" + tags + "</head>" + html;+ };+ let page = null;+ window.addEventListener("message", (event) => {+ // Relay the page's messages; the panel only trusts this window.+ if (page && event.source === page.contentWindow) {+ if (event.data && event.data.source === "unsloth-browser") parent.postMessage(event.data, "*");+ return;+ }+ if (event.source !== parent) return;+ const data = event.data;+ if (page) {+ if (data && data.type === "unsloth:browser-command") page.contentWindow.postMessage(data, "*");+ return;+ }+ // Only the parent may drive the shell, once.+ if (!data || data.type !== "unsloth:browser-html" || typeof data.html !== "string") return;+ const cfg = { url: data.url || null, refresh: data.refresh || null, zoom: Number(data.zoom) || 1, muted: data.muted === true };+ const base = data.base ? `<base href="${escapeAttr(data.base)}">` : "";+ const script = `<script>(${boot.toString()})(${JSON.stringify(cfg).replace(/</g, "\\u003c")});<\/script>`;+ page = document.createElement("iframe");+ page.setAttribute("sandbox", "allow-scripts allow-forms");+ page.srcdoc = inject(data.html, base + script);+ document.body.appendChild(page);+ // A script navigation the page's hooks missed (no Navigation API) hits the lock below; the+ // report only names the origin, so the panel can say what happened but not follow it.+ let blocked = false;+ document.addEventListener("securitypolicyviolation", (event) => {+ if (blocked || event.effectiveDirective !== "frame-src") return;+ blocked = true;+ parent.postMessage({ source: "unsloth-browser", type: "scriptNavigation" }, "*");+ });+ const lock = document.createElement("meta");+ lock.httpEquiv = "Content-Security-Policy";+ lock.content = "frame-src 'none'";+ document.head.appendChild(lock);+ });+ </script>+ </body>+</html>"""++# Annotate, the body of a function of `post` that the page shell's `install` runs on the panel's+# request: while it is on, pointer input is the panel's, and the page only reports the block under+# the pointer, what a click or drag marks, and where the marks sit as it scrolls.+_ANNOTATE_JS = r"""// Clicks mark these whole; anything else marks the nearest element that holds text itself.const BLOCK = "p, li, h1, h2, h3, h4, h5, h6, pre, blockquote, td, th, dt, dd, figcaption, caption, img, picture, video, svg, button, label, a[href], input, textarea, select";// A list item marks its own line, not the lists nested under it.@@ -748,214 +974,7 @@ _FRAME_HTML = r"""<!doctype html>marks.clear();};return { start, stop, forget, number };- })();- // The page as it is now, for printing: scripts' DOM, typed values and canvases included.- // Printed from a copy in a separate frame, as this sandbox can't open the print dialog.- const snapshot = () => {- try {- const source = document.documentElement;- const copy = source.cloneNode(true);- const twins = (selector) => [source.querySelectorAll(selector), copy.querySelectorAll(selector)];- const [fields, fieldCopies] = twins("input, textarea, select");- fields.forEach((field, index) => {- const twin = fieldCopies[index];- if (!twin) return;- if (field.tagName === "TEXTAREA") twin.textContent = field.value;- else if (field.tagName === "SELECT") [...twin.options].forEach((option, at) => option.toggleAttribute("selected", Boolean(field.options[at]?.selected)));- else if (field.type === "checkbox" || field.type === "radio") twin.toggleAttribute("checked", field.checked);- else if (field.type !== "password" && field.type !== "file") twin.setAttribute("value", field.value);- });- const [canvases, canvasCopies] = twins("canvas");- canvases.forEach((canvas, index) => {- const twin = canvasCopies[index];- if (!twin) return;- try {- const image = document.createElement("img");- for (const name of ["class", "style", "width", "height"]) if (twin.hasAttribute(name)) image.setAttribute(name, twin.getAttribute(name));- image.src = canvas.toDataURL();- twin.replaceWith(image);- } catch {}- });- // Styles added through the CSSOM (CSS-in-JS) aren't in the markup.- const [styles, styleCopies] = twins("style");- styles.forEach((style, index) => {- try {- const rules = style.sheet ? [...style.sheet.cssRules].map((rule) => rule.cssText).join("\n") : null;- if (rules && styleCopies[index]) styleCopies[index].textContent = rules;- } catch {}- });- try {- const adopted = (document.adoptedStyleSheets || []).flatMap((sheet) => [...sheet.cssRules].map((rule) => rule.cssText));- if (adopted.length) {- const extra = document.createElement("style");- extra.textContent = adopted.join("\n");- (copy.querySelector("head") || copy).appendChild(extra);- }- } catch {}- for (const node of copy.querySelectorAll("script, [data-unsloth-annotate]")) node.remove();- const html = "<!doctype html>" + copy.outerHTML;- return html.length <= 8 * 1024 * 1024 ? html : null;- } catch {- return null;- }- };-- // Find in page for Studio's find bar: every visible match of the query, case-insensitive,- // painted with the same two highlights as the chat's (all matches, then the active one) and- // counted back to the bar. Highlights sit over the text, so the page itself is not changed.- const finder = (() => {- const MAX = 1000;- const SKIP = new Set(["SCRIPT", "STYLE", "NOSCRIPT", "TEMPLATE", "SELECT", "OPTION", "TEXTAREA"]);- const painted = typeof Highlight === "function" && typeof CSS !== "undefined" && CSS.highlights;- let ranges = [];- let active = -1;- let styled = false;- const style = () => {- if (styled || !painted) return;- styled = true;- const sheet = document.createElement("style");- sheet.textContent = "::highlight(unsloth-find){background-color:#ffd84d;color:#1a1a1a}::highlight(unsloth-find-active){background-color:#ff8c1a;color:#1a1a1a}";- (document.head || document.documentElement).appendChild(sheet);- };- const shown = (element) => {- if (typeof element.checkVisibility === "function") return element.checkVisibility({ visibilityProperty: true });- return element.getClientRects().length > 0;- };- const collect = (query) => {- const needle = query.toLowerCase();- const found = [];- if (!document.body) return found;- const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, {- acceptNode: (node) => {- const element = node.parentElement;- if (!element || SKIP.has(element.tagName) || element.closest("[data-unsloth-annotate]")) return NodeFilter.FILTER_REJECT;- return NodeFilter.FILTER_ACCEPT;- },- });- for (let node = walker.nextNode(); node && found.length < MAX; node = walker.nextNode()) {- const text = node.nodeValue || "";- const lower = text.toLowerCase();- // Lowercasing can change a few characters' lengths; those nodes would map wrongly.- if (lower.length !== text.length) continue;- let at = lower.indexOf(needle);- if (at < 0 || !shown(node.parentElement)) continue;- while (at >= 0 && found.length < MAX) {- const range = document.createRange();- range.setStart(node, at);- range.setEnd(node, at + needle.length);- found.push(range);- at = lower.indexOf(needle, at + needle.length);- }- }- return found;- };- const paint = () => {- if (!painted) return;- style();- if (ranges.length) CSS.highlights.set("unsloth-find", new Highlight(...ranges));- else CSS.highlights.delete("unsloth-find");- if (ranges[active]) CSS.highlights.set("unsloth-find-active", new Highlight(ranges[active]));- else CSS.highlights.delete("unsloth-find-active");- };- const reveal = () => {- const range = ranges[active];- if (!range) return;- const rect = range.getBoundingClientRect();- if (rect.top < 48 || rect.bottom > innerHeight - 24 || rect.left < 0 || rect.right > innerWidth) {- range.startContainer.parentElement?.scrollIntoView({ block: "center", inline: "nearest" });- }- // Without highlights, the selection marks the match instead.- if (!painted) {- const selection = getSelection();- selection?.removeAllRanges();- selection?.addRange(range);- }- };- const report = () => post({ type: "findResult", count: ranges.length, active });- return {- search: (query) => {- ranges = query ? collect(query) : [];- // From where the reader is: the first match at or below the top of the view.- const below = ranges.findIndex((range) => range.getBoundingClientRect().bottom >= 0);- active = ranges.length ? Math.max(below, 0) : -1;- paint();- reveal();- report();- },- step: (delta) => {- if (ranges.length) {- active = (active + delta + ranges.length) % ranges.length;- paint();- reveal();- }- report();- },- };- })();- // Commands from the panel, relayed by the shell (the only parent this page has).- window.addEventListener("message", (event) => {- if (event.source !== parent) return;- const data = event.data;- if (!data || data.type !== "unsloth:browser-command") return;- if (data.command === "annotate") data.on === true ? annotation.start(data.color) : annotation.stop();- else if (data.command === "annotateForget") annotation.forget(Number(data.id));- else if (data.command === "annotateNumbers") annotation.number(data.numbers);- else if (data.command === "zoom") applyZoom(data.value);- else if (data.command === "mute") setMuted(data.on === true);- else if (data.command === "find" && typeof data.query === "string" && data.query.length <= 1000) finder.search(data.query);- else if (data.command === "findStep") finder.step(data.delta === -1 ? -1 : 1);- else if (data.command === "snapshot") post({ type: "snapshot", html: snapshot() });- });- // The panel may have asked for annotate before this page could hear it.- post({ type: "annotate", event: "ready" });- };- const escapeAttr = (value) => value.replace(/&/g, "&").replace(/"/g, """).replace(/</g, "<");- const inject = (html, tags) => {- const at = (match) => match.index + match[0].length;- const head = /<head\b[^<>]*>/i.exec(html);- if (head) return html.slice(0, at(head)) + tags + html.slice(at(head));- const root = /<html\b[^<>]*>/i.exec(html) || /^\s*<!doctype[^>]*>/i.exec(html);- if (root) return html.slice(0, at(root)) + "<head>" + tags + "</head>" + html.slice(at(root));- return "<head>" + tags + "</head>" + html;- };- let page = null;- window.addEventListener("message", (event) => {- // Relay the page's messages; the panel only trusts this window.- if (page && event.source === page.contentWindow) {- if (event.data && event.data.source === "unsloth-browser") parent.postMessage(event.data, "*");- return;- }- if (event.source !== parent) return;- const data = event.data;- if (page) {- if (data && data.type === "unsloth:browser-command") page.contentWindow.postMessage(data, "*");- return;- }- // Only the parent may drive the shell, once.- if (!data || data.type !== "unsloth:browser-html" || typeof data.html !== "string") return;- const cfg = { url: data.url || null, refresh: data.refresh || null, zoom: Number(data.zoom) || 1, muted: data.muted === true };- const base = data.base ? `<base href="${escapeAttr(data.base)}">` : "";- const script = `<script>(${boot.toString()})(${JSON.stringify(cfg).replace(/</g, "\\u003c")});<\/script>`;- page = document.createElement("iframe");- page.setAttribute("sandbox", "allow-scripts allow-forms");- page.srcdoc = inject(data.html, base + script);- document.body.appendChild(page);- // A script navigation the page's hooks missed (no Navigation API) hits the lock below; the- // report only names the origin, so the panel can say what happened but not follow it.- let blocked = false;- document.addEventListener("securitypolicyviolation", (event) => {- if (blocked || event.effectiveDirective !== "frame-src") return;- blocked = true;- parent.postMessage({ source: "unsloth-browser", type: "scriptNavigation" }, "*");- });- const lock = document.createElement("meta");- lock.httpEquiv = "Content-Security-Policy";- lock.content = "frame-src 'none'";- document.head.appendChild(lock);- });- </script>- </body>-</html>"""+"""# Print shell: the panel posts it a copy of the page (`snapshot` above), which it shows without@@ -1256,3 +1275,17 @@ async def browser_frame():"X-Content-Type-Options": "nosniff",},)++[email protected]("/annotate.js", include_in_schema = False)+async def browser_annotate_script():+ """The page shell's annotate code, sent into a page when annotate mode turns on; static like ``/frame``."""+ return Response(+ content = _ANNOTATE_JS,+ media_type = "text/javascript; charset=utf-8",+ headers = {+ "Cache-Control": "no-cache",+ "Referrer-Policy": "no-referrer",+ "X-Content-Type-Options": "nosniff",+ },+ )diff --git a/studio/backend/tests/test_browser_routes.py b/studio/backend/tests/test_browser_routes.pyindex f60c39d2df2..35333d5eb06 100644--- a/studio/backend/tests/test_browser_routes.py+++ b/studio/backend/tests/test_browser_routes.py@@ -73,7 +73,6 @@ def test_the_shell_guards():'post({ type: "upload" }); return;','event.effectiveDirective !== "frame-src"',"if (raw.length > 1) followHash(raw);",- 'const value = node.type === "password" ? "" : node.value;',"if (result === false) cancelled = true;",):assert guard in shell, guard@@ -88,6 +87,21 @@ def test_the_shell_guards():assert "requestSubmit = " not in shell+def test_annotate_code_is_sent_only_when_annotating():+ shell, code = browser_mod._FRAME_HTML, browser_mod._ANNOTATE_JS+ # Not in the shell (so not in every page), installed once and only from the shell's own message.+ assert "const BLOCK =" in code and "const BLOCK =" not in shell+ assert 'const value = node.type === "password" ? "" : node.value;' in code+ assert code.rstrip().endswith("return { start, stop, forget, number };")+ assert 'if (data.command === "annotateInstall") install(data.code);' in shell+ assert 'if (annotation || typeof code !== "string"' in shell+ assert shell.index("const compile = Function;") < shell.index("const install = ")+ response = asyncio.run(browser_mod.browser_annotate_script())+ assert response.body.decode() == code+ assert response.media_type.startswith("text/javascript")+ assert response.headers["x-content-type-options"] == "nosniff"++def test_prepare_page_strips_what_would_escape_the_sandbox():page = ('<html><head><base href="/docs/">'diff --git a/studio/frontend/src/features/browser/annotate-layer.tsx b/studio/frontend/src/features/browser/annotate-layer.tsxindex 13271d2655a..d123bb8f094 100644--- a/studio/frontend/src/features/browser/annotate-layer.tsx+++ b/studio/frontend/src/features/browser/annotate-layer.tsx@@ -21,6 +21,7 @@ import {useRef,useState,} from "react";+import { annotateScript } from "./api";import { type AnnotateEvent, type AnnotateRect, MAX_MARKS } from "./frame-message";import { frameRect, onFrameAnnotate, sendFrameCommand } from "./page-frame";import { useBrowserStore } from "./store";@@ -769,6 +770,8 @@ export function WebAnnotateLayer({const t = useT();const layerRef = useRef<HTMLDivElement | null>(null);const inputRef = useRef<HTMLInputElement | null>(null);+ // The tab this layer annotates while mounted; a code fetch finishing after it left sends nothing.+ const liveTab = useRef<string | null>(null);const [rects, setRects] = useState<ReadonlyMap<number, AnnotateRect | null>>(new Map());const [items, setItems] = useState<WebMark[]>([]);const [pending, setPending] = useState<WebPending | null>(null);@@ -779,8 +782,16 @@ export function WebAnnotateLayer({const exit = () => setAnnotating(null);const forget = (id: number) => sendFrameCommand(tabId, { command: "annotateForget", id });+ // The page gets the annotate code only now (each document once; repeats are ignored there).const start = () =>- sendFrameCommand(tabId, { command: "annotate", on: true, color: accentColor() });+ void annotateScript().then(+ (code) => {+ if (liveTab.current !== tabId) return;+ sendFrameCommand(tabId, { command: "annotateInstall", code });+ sendFrameCommand(tabId, { command: "annotate", on: true, color: accentColor() });+ },+ () => liveTab.current === tabId && exit(),+ );const committed = (): WebMark[] => {if (!pending) return items;@@ -868,12 +879,16 @@ export function WebAnnotateLayer({};const handleRef = useRef(handle);handleRef.current = handle;+ const startRef = useRef(start);+ startRef.current = start;useEffect(() => {const stop = onFrameAnnotate(tabId, (event) => handleRef.current(event));+ liveTab.current = tabId;// Now, for a page already loaded; a page still loading asks with "ready".- sendFrameCommand(tabId, { command: "annotate", on: true, color: accentColor() });+ startRef.current();return () => {+ liveTab.current = null;stop();sendFrameCommand(tabId, { command: "annotate", on: false });};diff --git a/studio/frontend/src/features/browser/api.ts b/studio/frontend/src/features/browser/api.tsindex c2122346624..e99c6d203fa 100644--- a/studio/frontend/src/features/browser/api.ts+++ b/studio/frontend/src/features/browser/api.ts@@ -22,6 +22,22 @@ export type BrowserRequest = {maxBytes?: number;};+let annotateCode: Promise<string> | null = null;++/** The page's annotate code (`_ANNOTATE_JS`), fetched on first use and kept; a failure tries again next time. */+export function annotateScript(): Promise<string> {+ annotateCode ??= authFetch("/api/browser/annotate.js")+ .then((response) => {+ if (!response.ok) throw new Error(`annotate code: ${response.status}`);+ return response.text();+ })+ .catch((error: unknown) => {+ annotateCode = null;+ throw error;+ });+ return annotateCode;+}+export class BrowserFetchError extends Error {readonly botCheck: boolean;diff --git a/studio/frontend/src/features/browser/page-cache.ts b/studio/frontend/src/features/browser/page-cache.tsindex e97b0aecf86..11eaea6d25e 100644--- a/studio/frontend/src/features/browser/page-cache.ts+++ b/studio/frontend/src/features/browser/page-cache.ts@@ -3,10 +3,29 @@import type { BrowserPage } from "./api";+const MB = 1024 * 1024;const MAX_PAGES = 12;// Total and per-page budget: a fetched page can be 50 MB.-const MAX_TOTAL_BYTES = 32 * 1024 * 1024;-const MAX_PAGE_BYTES = 8 * 1024 * 1024;+const MAX_TOTAL_BYTES = 32 * MB;+const MAX_PAGE_BYTES = 8 * MB;++/** Page and byte limits for a machine reporting `deviceMemory` GB; unknown keeps the defaults. */+export function cacheLimits(deviceMemory: unknown): { maxPages: number; maxTotalBytes: number } {+ if (typeof deviceMemory === "number" && Number.isFinite(deviceMemory) && deviceMemory > 0) {+ if (deviceMemory <= 2) return { maxPages: 4, maxTotalBytes: 8 * MB };+ if (deviceMemory <= 4) return { maxPages: 6, maxTotalBytes: 16 * MB };+ }+ return { maxPages: MAX_PAGES, maxTotalBytes: MAX_TOTAL_BYTES };+}++/** `navigator.deviceMemory`, which only Chromium reports (Chrome, Edge, WebView2). */+export function reportedDeviceMemory(): unknown {+ try {+ return typeof navigator === "undefined" ? undefined : (navigator as { deviceMemory?: unknown }).deviceMemory;+ } catch {+ return undefined;+ }+}/** Approximate memory held by a page. JS strings are UTF-16. */export function pageBytes(page: BrowserPage): number {diff --git a/studio/frontend/src/features/browser/page-frame.tsx b/studio/frontend/src/features/browser/page-frame.tsxindex dd8eeb0addd..66d7a6ee680 100644--- a/studio/frontend/src/features/browser/page-frame.tsx+++ b/studio/frontend/src/features/browser/page-frame.tsx@@ -17,6 +17,7 @@ export type FrameCommand =| { command: "snapshot" }| { command: "zoom"; value: number }| { command: "mute"; on: boolean }+ | { command: "annotateInstall"; code: string }| { command: "annotate"; on: boolean; color?: string }| { command: "annotateForget"; id: number }| { command: "annotateNumbers"; numbers: Array<[number, number]> };diff --git a/studio/frontend/src/features/browser/store.ts b/studio/frontend/src/features/browser/store.tsindex 442d31df8f7..343491272af 100644--- a/studio/frontend/src/features/browser/store.ts+++ b/studio/frontend/src/features/browser/store.ts@@ -5,7 +5,7 @@ import { type DocumentAnnotations, useChatArtifactsStore } from "@/features/chatimport { create } from "zustand";import { unwrapRedirect } from "./address";import type { BrowserPage } from "./api";-import { PageCache } from "./page-cache";+import { PageCache, cacheLimits, reportedDeviceMemory } from "./page-cache";export type BrowserEntry =| { kind: "newtab" }@@ -110,7 +110,9 @@ export function setNativeWebHistory(native: boolean): void {const MAX_HISTORY = 50;// Loaded pages by history entry, so back and forward skip the fetch. Reload drops only its own entry.-const pageCache = new PageCache<BrowserEntry>();+// Fewer on a low-memory machine.+const cacheLimit = cacheLimits(reportedDeviceMemory());+const pageCache = new PageCache<BrowserEntry>(cacheLimit.maxPages, cacheLimit.maxTotalBytes);const entryIds = new WeakMap<BrowserEntry, number>();let nextEntryId = 0;diff --git a/studio/frontend/tests/browser-page-cache.test.ts b/studio/frontend/tests/browser-page-cache.test.tsindex 474a3b87cac..e16597d4a91 100644--- a/studio/frontend/tests/browser-page-cache.test.ts+++ b/studio/frontend/tests/browser-page-cache.test.ts@@ -4,7 +4,7 @@import assert from "node:assert/strict";import { test } from "node:test";import type { BrowserPage } from "../src/features/browser/api.ts";-import { PageCache } from "../src/features/browser/page-cache.ts";+import { PageCache, cacheLimits, reportedDeviceMemory } from "../src/features/browser/page-cache.ts";const MB = 1024 * 1024;const html = (bytes: number): BrowserPage => ({@@ -50,3 +50,31 @@ test("the least recently used page goes first", () => {assert.ok(cache.get(a));assert.equal(cache.get(b), undefined);});++test("low-memory machines keep fewer pages; an unknown or bogus reading keeps the defaults", () => {+ const today = { maxPages: 12, maxTotalBytes: 32 * MB };+ assert.deepEqual(cacheLimits(0.5), { maxPages: 4, maxTotalBytes: 8 * MB });+ assert.deepEqual(cacheLimits(2), { maxPages: 4, maxTotalBytes: 8 * MB });+ assert.deepEqual(cacheLimits(3), { maxPages: 6, maxTotalBytes: 16 * MB });+ assert.deepEqual(cacheLimits(4), { maxPages: 6, maxTotalBytes: 16 * MB });+ assert.deepEqual(cacheLimits(8), today);+ for (const value of [undefined, null, "2", Number.NaN, 0, -2, Number.POSITIVE_INFINITY]) {+ assert.deepEqual(cacheLimits(value), today);+ }+});++test("the memory reading survives a missing or throwing navigator", () => {+ const original = Object.getOwnPropertyDescriptor(globalThis, "navigator");+ const define = (value: PropertyDescriptor) => Object.defineProperty(globalThis, "navigator", { configurable: true, ...value });+ try {+ define({ value: { deviceMemory: 4 } });+ assert.equal(reportedDeviceMemory(), 4);+ define({ value: undefined });+ assert.equal(reportedDeviceMemory(), undefined);+ define({ get: () => ({ get deviceMemory() { throw new Error("blocked"); } }) });+ assert.equal(reportedDeviceMemory(), undefined);+ } finally {+ if (original) Object.defineProperty(globalThis, "navigator", original);+ else delete (globalThis as { navigator?: unknown }).navigator;+ }+});diff --git a/studio/frontend/tests/browser-store-reopen.test.ts b/studio/frontend/tests/browser-store-reopen.test.tsindex 9b32f63c76d..adc082bde50 100644--- a/studio/frontend/tests/browser-store-reopen.test.ts+++ b/studio/frontend/tests/browser-store-reopen.test.ts@@ -146,3 +146,22 @@ test("a duplicated form result asks before posting again; a duplicated page justassert.equal(sentPosts.has(postCopy), true);assert.equal(sentPosts.has(currentEntry(pageTab)), false);});++test("a form result pushed out of the cache still asks before posting again", () => {+ const store = useBrowserStore.getState();+ store.openUrl("https://example.com/pay", { method: "POST", body: "card=1" });+ const posted = useBrowserStore.getState().tabs.at(-1);+ assert.ok(posted);+ const entry = currentEntry(posted);+ sentPosts.add(entry);+ cachePage(entry, { kind: "html", url: "https://example.com/pay", base: "https://example.com/pay", refresh: null, html: "paid" });+ for (let i = 0; i < 12; i++) {+ store.openUrl(`https://example.com/${i}`, { newTab: true });+ const tab = useBrowserStore.getState().tabs.at(-1);+ assert.ok(tab);+ cachePage(currentEntry(tab), { kind: "html", url: `https://example.com/${i}`, base: `https://example.com/${i}`, refresh: null, html: "x" });+ }+ // What the page view checks: no cached copy, already sent, so it asks rather than resending.+ assert.equal(cachedPage(entry), undefined);+ assert.equal(sentPosts.has(entry), true);+});diff --git a/studio/src-tauri/Cargo.toml b/studio/src-tauri/Cargo.tomlindex e3619924ad4..663f96476b0 100644--- a/studio/src-tauri/Cargo.toml+++ b/studio/src-tauri/Cargo.toml@@ -53,6 +53,10 @@ time = { version = "0.3", features = ["formatting"] }libc = "0.2"signal-hook = "0.4"+[dev-dependencies]+# A paused clock for the address poll's tests (browser_webview.rs).+tokio = { version = "1", features = ["test-util"] }+[target.'cfg(target_os = "macos")'.dependencies]objc2 = { version = "0.6", features = ["exception"] }diff --git a/studio/src-tauri/src/browser_webview.rs b/studio/src-tauri/src/browser_webview.rsindex c4a503288d1..b623932a3bc 100644--- a/studio/src-tauri/src/browser_webview.rs+++ b/studio/src-tauri/src/browser_webview.rs@@ -18,11 +18,13 @@ use tauri::{AppHandle, Emitter, LogicalPosition, LogicalSize, Manager, Rect, Runtime, State, Url, Webview,WebviewBuilder, WebviewUrl,};+use tokio::sync::watch;const LABEL_PREFIX: &str = "unsloth-browser-";const EVENT: &str = "unsloth-browser";/// The app's own webview, the only caller these commands answer.const MAIN_WEBVIEW: &str = "main";+const MAIN_WINDOW: &str = "main";const URL_POLL: Duration = Duration::from_millis(800);/// macOS 14+ data store for pages (fixed, so it persists).#[cfg(target_os = "macos")]@@ -101,9 +103,45 @@ const STATE_SCRIPT: &str = r#"(() => {} catch { return ""; }})()"#;-#[derive(Default)]pub struct BrowserViews {inner: Mutex<ViewsState>,+ /// Whether the address poll has anything to watch; it sleeps on this rather than a timer.+ gate: watch::Sender<PollGate>,+}++impl Default for BrowserViews {+ fn default() -> Self {+ Self {+ inner: Mutex::default(),+ gate: watch::channel(PollGate::default()).0,+ }+ }+}++/// A view is shown, and the window it sits in is in use (focused, visible, not minimised).+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+struct PollGate {+ shown: bool,+ window_active: bool,+ /// Bumped when the shown tab changes, so a switch between two shown tabs reads at once too.+ switches: u64,+}++impl Default for PollGate {+ fn default() -> Self {+ // An unknown window state polls, as it always did.+ Self {+ shown: false,+ window_active: true,+ switches: 0,+ }+ }+}++impl PollGate {+ fn polls(self) -> bool {+ self.shown && self.window_active+ }}#[derive(Default)]@@ -122,6 +160,35 @@ pub fn new_browser_views() -> BrowserViews {BrowserViews::default()}+fn set_shown(views: &BrowserViews, inner: &mut ViewsState, shown: Option<String>) {+ if inner.shown == shown {+ return;+ }+ let visible = shown.is_some();+ inner.shown = shown;+ views.gate.send_modify(|gate| {+ gate.shown = visible;+ gate.switches = gate.switches.wrapping_add(1);+ });+}++/// Re-read the main window's state after its focus, size or visibility changed.+pub fn window_changed<R: Runtime>(app: &AppHandle<R>, focused: Option<bool>) {+ let (Some(views), Some(window)) =+ (app.try_state::<BrowserViews>(), app.get_window(MAIN_WINDOW))+ else {+ return;+ };+ // Windows reports keyboard focus moving into a child webview as the window losing focus, so+ // there only a minimised or hidden window pauses the poll; macOS and GTK report activation.+ let focused = cfg!(windows) || focused == Some(true) || window.is_focused().unwrap_or(true);+ let active =+ focused && window.is_visible().unwrap_or(true) && !window.is_minimized().unwrap_or(false);+ views+ .gate+ .send_if_modified(|gate| std::mem::replace(&mut gate.window_active, active) != active);+}+#[derive(Clone, Serialize)]#[serde(rename_all = "camelCase",@@ -709,7 +776,8 @@ async fn page_url<R: Runtime>(webview: &Webview<R>) -> Option<String> {webview.url().ok().map(|url| url.to_string())}-/// Poll the shown tab's address, which pushState changes without a load. Started once.+/// Poll the shown tab's address, which pushState changes without a load. Started once; it only+/// runs while a view is shown in a window in use.fn start_url_poll<R: Runtime>(app: &AppHandle<R>) {let state = app.state::<BrowserViews>();{@@ -719,31 +787,62 @@ fn start_url_poll<R: Runtime>(app: &AppHandle<R>) {}inner.polling = true;}+ window_changed(app, None);+ let gate = state.gate.subscribe();let app = app.clone();- tauri::async_runtime::spawn(async move {- loop {- tokio::time::sleep(URL_POLL).await;- let state = app.state::<BrowserViews>();- let Some(tab_id) = state.inner.lock().unwrap().shown.clone() else {- continue;- };- let Ok(webview) = view(&app, &tab_id) else {- continue;- };- let Some(url) = page_url(&webview).await.filter(|url| reportable(url)) else {- continue;- };- let changed = {- let mut inner = state.inner.lock().unwrap();- let last = inner.urls.insert(tab_id.clone(), url.clone());- last.as_deref() != Some(url.as_str())- };- if changed {- emit(&app, BrowserEvent::Url { tab_id, url });- refresh_history(&webview);+ tauri::async_runtime::spawn(run_url_poll(gate, move || poll_url(app.clone())));+}++/// Read now and every `URL_POLL` while the gate is open; while it is shut, wait on it with no+/// timer armed. A change while open reads at once (a tab switch or the window coming back).+async fn run_url_poll<F, Fut>(mut gate: watch::Receiver<PollGate>, mut poll: F)+where+ F: FnMut() -> Fut,+ Fut: std::future::Future<Output = ()>,+{+ loop {+ if !gate.borrow_and_update().polls() {+ if gate.changed().await.is_err() {+ return;}+ continue;}- });+ poll().await;+ tokio::select! {+ _ = tokio::time::sleep(URL_POLL) => {}+ changed = gate.changed() => {+ if changed.is_err() {+ return;+ }+ }+ }+ }+}++async fn poll_url<R: Runtime>(app: AppHandle<R>) {+ let state = app.state::<BrowserViews>();+ let Some(tab_id) = state.inner.lock().unwrap().shown.clone() else {+ return;+ };+ let Ok(webview) = view(&app, &tab_id) else {+ return;+ };+ let Some(url) = page_url(&webview).await.filter(|url| reportable(url)) else {+ return;+ };+ let changed = {+ let mut inner = state.inner.lock().unwrap();+ // Switched or closed while the address was read: it belongs to a view no longer shown.+ if inner.shown.as_deref() != Some(tab_id.as_str()) {+ return;+ }+ let last = inner.urls.insert(tab_id.clone(), url.clone());+ last.as_deref() != Some(url.as_str())+ };+ if changed {+ emit(&app, BrowserEvent::Url { tab_id, url });+ refresh_history(&webview);+ }}fn create_view<R: Runtime>(@@ -1176,7 +1275,7 @@ pub async fn browser_view_show<R: Runtime>(let _ = view.hide();}}- state.inner.lock().unwrap().shown = shown;+ set_shown(&state, &mut state.inner.lock().unwrap(), shown);Ok(())}@@ -1324,7 +1423,7 @@ pub fn browser_view_close<R: Runtime>(inner.download_starts.remove(&tab_id);// `muted` stays: a pruned view reopens muted; unmuting is what forgets it.if inner.shown.as_deref() == Some(tab_id.as_str()) {- inner.shown = None;+ set_shown(&state, &mut inner, None);}}if let Ok(page) = view(webview.app_handle(), &tab_id) {@@ -1347,7 +1446,7 @@ pub async fn browser_view_clear_data<R: Runtime>({let mut inner = state.inner.lock().unwrap();inner.urls.clear();- inner.shown = None;+ set_shown(&state, &mut inner, None);}for page in browser_views(&app) {let _ = page.close();@@ -1470,6 +1569,118 @@ fn platform_clear(platform: tauri::webview::PlatformWebview, finish: ClearFinishmod tests {use super::*;+ mod url_poll {+ use super::*;+ use std::sync::atomic::{AtomicUsize, Ordering};+ use std::sync::Arc;++ const TEN_MINUTES: Duration = Duration::from_secs(600);++ /// Runs the poll for `span` from `start`, returning the reads it made.+ async fn reads(+ start: PollGate,+ span: Duration,+ ) -> (watch::Sender<PollGate>, Arc<AtomicUsize>) {+ let (gate, receiver) = watch::channel(start);+ let count = Arc::new(AtomicUsize::new(0));+ let counter = count.clone();+ tokio::spawn(run_url_poll(receiver, move || {+ counter.fetch_add(1, Ordering::SeqCst);+ async {}+ }));+ tokio::time::sleep(span).await;+ (gate, count)+ }++ fn gate(shown: bool, window_active: bool) -> PollGate {+ PollGate {+ shown,+ window_active,+ switches: 0,+ }+ }++ #[tokio::test(start_paused = true)]+ async fn polls_every_800_ms_only_while_a_view_shows_in_a_window_in_use() {+ // One at once, then 600 s / 0.8 s = 750 (just past the last one, so no tie with it).+ let span = TEN_MINUTES + Duration::from_millis(1);+ assert_eq!(+ reads(gate(true, true), span).await.1.load(Ordering::SeqCst),+ 751+ );+ for idle in [gate(true, false), gate(false, true), gate(false, false)] {+ assert_eq!(+ reads(idle, TEN_MINUTES).await.1.load(Ordering::SeqCst),+ 0,+ "{idle:?}"+ );+ }+ }++ #[tokio::test(start_paused = true)]+ async fn coming_back_reads_at_once_then_resumes_the_interval() {+ let (gate_tx, count) = reads(gate(true, false), TEN_MINUTES).await;+ gate_tx.send_modify(|gate| gate.window_active = true);+ tokio::time::sleep(Duration::from_millis(1)).await;+ assert_eq!(count.load(Ordering::SeqCst), 1);+ tokio::time::sleep(URL_POLL * 10).await;+ assert_eq!(count.load(Ordering::SeqCst), 11);+ gate_tx.send_modify(|gate| gate.window_active = false);+ tokio::time::sleep(TEN_MINUTES).await;+ assert_eq!(count.load(Ordering::SeqCst), 11);+ }++ #[tokio::test(start_paused = true)]+ async fn rapid_flips_read_once_per_opening_and_dropping_the_gate_ends_the_poll() {+ let (gate_tx, count) = reads(gate(false, true), Duration::from_millis(1)).await;+ for _ in 0..5 {+ gate_tx.send_modify(|gate| gate.shown = true);+ tokio::time::sleep(Duration::from_millis(1)).await;+ gate_tx.send_modify(|gate| gate.shown = false);+ tokio::time::sleep(Duration::from_millis(1)).await;+ }+ assert_eq!(count.load(Ordering::SeqCst), 5);+ gate_tx.send_modify(|gate| gate.shown = true);+ drop(gate_tx);+ tokio::time::sleep(TEN_MINUTES).await;+ // The read made as it opened, then nothing: the poll ended with its gate.+ assert_eq!(count.load(Ordering::SeqCst), 6);+ }++ #[test]+ fn shown_is_tracked_with_the_gate() {+ let views = BrowserViews::default();+ let mut inner = ViewsState::default();+ set_shown(&views, &mut inner, Some("a".into()));+ assert!(views.gate.borrow().polls());+ set_shown(&views, &mut inner, None);+ assert!(!views.gate.borrow().polls());+ assert_eq!(inner.shown, None);+ }++ #[tokio::test(start_paused = true)]+ async fn switching_between_shown_tabs_reads_at_once() {+ let views = BrowserViews::default();+ let mut inner = ViewsState::default();+ set_shown(&views, &mut inner, Some("a".into()));+ let count = Arc::new(AtomicUsize::new(0));+ let counter = count.clone();+ tokio::spawn(run_url_poll(views.gate.subscribe(), move || {+ counter.fetch_add(1, Ordering::SeqCst);+ async {}+ }));+ tokio::time::sleep(Duration::from_millis(1)).await;+ assert_eq!(count.load(Ordering::SeqCst), 1);+ set_shown(&views, &mut inner, Some("b".into()));+ tokio::time::sleep(Duration::from_millis(1)).await;+ assert_eq!(count.load(Ordering::SeqCst), 2);+ // Showing the same tab again is not a switch.+ set_shown(&views, &mut inner, Some("b".into()));+ tokio::time::sleep(Duration::from_millis(1)).await;+ assert_eq!(count.load(Ordering::SeqCst), 2);+ }+ }+fn allowed(url: &str) -> bool {navigation_allowed(&Url::parse(url).unwrap())}diff --git a/studio/src-tauri/src/main.rs b/studio/src-tauri/src/main.rsindex a12fcea380e..aa022cd0e71 100644--- a/studio/src-tauri/src/main.rs+++ b/studio/src-tauri/src/main.rs@@ -1337,6 +1337,7 @@ fn show_main_window(app: &tauri::AppHandle) {let _ = window.unminimize();let _ = window.set_focus();}+ browser_webview::window_changed(app, None);}/// Coordinates the one visible quit confirmation and, on macOS, one AppKit termination request@@ -2325,6 +2326,17 @@ fn main() {Ok(())}).on_window_event(|window, event| {+ if window.label() == "main" {+ match event {+ tauri::WindowEvent::Focused(focused) => {+ browser_webview::window_changed(window.app_handle(), Some(*focused))+ }+ tauri::WindowEvent::Resized(_) => {+ browser_webview::window_changed(window.app_handle(), None)+ }+ _ => {}+ }+ }// Record real drops here, in Rust, so the renderer can only register paths the// OS actually handed to the native intake commands.if let tauri::WindowEvent::DragDrop(tauri::DragDropEvent::Drop { paths, .. }) = event {@@ -2340,6 +2352,7 @@ fn main() {MainWindowCloseAction::Hide => {// The tray's Open action and a second app launch restore the window.let _ = window.hide();+ browser_webview::window_changed(window.app_handle(), None);}MainWindowCloseAction::Quit => request_quit(window.app_handle()),}