Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: faster browser agent page reads after a load, and leaner drags (#13602)
Daniel Handd5ec4877ee0
diff --git a/studio/frontend/src/features/browser/agent/executor.ts b/studio/frontend/src/features/browser/agent/executor.tsindex c45c8fe0d34..51aab1a4d6c 100644--- a/studio/frontend/src/features/browser/agent/executor.ts+++ b/studio/frontend/src/features/browser/agent/executor.ts@@ -53,6 +53,8 @@ const LOAD_TIMEOUT_MS = 30_000;const SETTLE_QUIET_MS = 350;const SETTLE_MAX_MS = 3_000;const FILL_QUIET_MAX_MS = 1_000;+// A loaded page that opened no request has drawn what it will; content still on its way arrives by a request.+const FILL_IDLE_QUIET_MS = 120;const PAGE_READY_MS = 4_000;const POST_TIMEOUT_MS = 15_000;const EXPIRED = "Not done: the action ran out of time, likely waiting for the user to answer in the panel. Ask them before trying again.";@@ -358,9 +360,16 @@ async function pageCall(tab: BrowserTab, command: Record<string, unknown>): Prom}/** Requests started since `since` and a quiet DOM; `left` says the tab is on another page. */-function waitOnPage(tabId: string, left: (tab: BrowserTab) => boolean, since: number, changingMaxMs: number, signal?: AbortSignal) {+function waitOnPage(+ tabId: string,+ left: (tab: BrowserTab) => boolean,+ since: number,+ changingMaxMs: number,+ signal?: AbortSignal,+ idleQuietMs?: number,+) {return waitForPage({- maxMs: SETTLE_MAX_MS, changingMaxMs, quietMs: SETTLE_QUIET_MS, now: Date.now, sleep,+ maxMs: SETTLE_MAX_MS, changingMaxMs, quietMs: SETTLE_QUIET_MS, idleQuietMs, now: Date.now, sleep,status: () => {const tab = findTab(tabId);return tab ? pageCall(tab, { op: "status", since }).catch(() => null) : Promise.resolve(null);@@ -385,7 +394,7 @@ async function settle(tabId: string, urlBefore: string, since: number, signal?:* A DOM that never rests gets only FILL_QUIET_MAX_MS. A router rewriting the address is still this page. */async function untilFilled(tab: BrowserTab, since: number, signal?: AbortSignal): Promise<boolean> {const entry = currentEntry(tab);- return (await waitOnPage(tab.id, (now) => currentEntry(now) !== entry, since, FILL_QUIET_MAX_MS, signal)) === "loading";+ return (await waitOnPage(tab.id, (now) => currentEntry(now) !== entry, since, FILL_QUIET_MAX_MS, signal, FILL_IDLE_QUIET_MS)) === "loading";}async function pass(gate: Gate, signal?: AbortSignal): Promise<string | null> {@@ -786,6 +795,7 @@ async function dragBetween(tab: BrowserTab, action: BrowserAction, context: Acticonst hits: string[] = [];const sigs: string[] = [];const infos: string[] = [];+ const docs: string[] = [];let asked = false;for (const point of [from, end]) {const described = await pageCall(tab, { op: "describe", ref: "", ...point });@@ -800,16 +810,16 @@ async function dragBetween(tab: BrowserTab, action: BrowserAction, context: Actihits.push(described.hit ?? "");sigs.push(described.sig ?? "");infos.push(JSON.stringify(described.info));+ docs.push(described.doc ?? "");}if (signal?.aborted) return { ok: false, error: STOPPED };if (asked && !(await unchanged(tab, from, hits[0], infos[0]))) return { ok: false, error: CHANGED_UNDER_POINT };// A handler can navigate mid-drag: the rest must not act on a page nobody gated.- const doc = (await pageCall(tab, { op: "status" })).doc;+ const doc = docs[0];const sameTab = () => {const now = useBrowserStore.getState().tabs.find((candidate) => candidate.id === tab.id);return Boolean(now && !now.loading && tabUrl(now) === before);};- const samePage = async () => sameTab() && (await pageCall(tab, { op: "status" }).catch(() => null))?.doc === doc;const since = Date.now();const down = await pageCall(tab, { op: "pointer", phase: "down", ...from, expect: hits[0], sig: sigs[0] });if (!down.ok) return { ok: false, error: down.error };@@ -832,16 +842,18 @@ async function dragBetween(tab: BrowserTab, action: BrowserAction, context: Actireturn { ok: false, error };};// A new document gets nothing; the same one (a hash or pushState move) has its drag cancelled.- const left = async (): Promise<Result> => {+ const left = async (current?: string): Promise<Result> => {const error = "the page changed during the drag, so it was not finished; take a snapshot.";- const status = await pageCall(tab, { op: "status" }).catch(() => null);- return doc && status?.doc === doc ? cancel(error) : { ok: false, error };+ const now = current ?? (await pageCall(tab, { op: "status" }).catch(() => null))?.doc;+ return doc && now === doc ? cancel(error) : { ok: false, error };};if (signal?.aborted) return cancel(STOPPED);- if (!doc || !(await samePage())) return left();- if (at.x !== end.x || at.y !== end.y) return cancel("the drag stopped short; take a snapshot to see where things are.");- // Drop zones can appear or relabel once a drag starts: re-ask unless the end is unchanged.+ if (!doc || !sameTab()) return left();+ // Drop zones can appear or relabel once a drag starts: re-ask unless the end is unchanged. Its doc+ // says whether this is still the document the drag began on.const target = await pageCall(tab, { op: "describe", ref: "", ...end }).catch(() => null);+ if (target?.doc !== doc) return left(target?.doc);+ if (at.x !== end.x || at.y !== end.y) return cancel("the drag stopped short; take a snapshot to see where things are.");if (!target?.ok || !target.info) return cancel(target?.error ?? "nothing is at the drag's end.");if (target.hit !== hits[1] || JSON.stringify(target.info) !== infos[1]) {for (const gate of [gateElement("click", target.info, before, action.policy, permissions()), opaqueGate(target.info, before)]) {@@ -850,8 +862,9 @@ async function dragBetween(tab: BrowserTab, action: BrowserAction, context: Actiif (refused) return cancel(refused);if (gate.kind === "ask" && gate.ask === "pointer") opaqueHosts.add(gate.host);}- if (!(await unchanged(tab, end, target.hit ?? "", JSON.stringify(target.info)))) return cancel(CHANGED_UNDER_POINT);- if (!(await samePage())) return left();+ const again = await pageCall(tab, { op: "describe", ref: "", ...end }).catch(() => null);+ if (!sameTab() || again?.doc !== doc) return left(again?.doc);+ if (!again?.ok || again.hit !== target.hit || JSON.stringify(again.info) !== JSON.stringify(target.info)) return cancel(CHANGED_UNDER_POINT);}if (signal?.aborted) return cancel(STOPPED);const up = await pageCall(tab, { op: "pointer", phase: "up", ...end, expect: target.hit ?? "", sig: target.sig ?? "" });diff --git a/studio/frontend/src/features/browser/agent/page-agent.ts b/studio/frontend/src/features/browser/agent/page-agent.tsindex ad781bf694c..0448c24cb50 100644--- a/studio/frontend/src/features/browser/agent/page-agent.ts+++ b/studio/frontend/src/features/browser/agent/page-agent.ts@@ -592,7 +592,7 @@ export function installPageAgent(): PageAgent {: {}),...(!submits && submitter ? { submitter: clean(nameOf(submitter), 200) } : {}),};- return point ? { ok: true, info, hit: refOf(found), sig: JSON.stringify(info) } : { ok: true, info };+ return point ? { ok: true, info, hit: refOf(found), sig: JSON.stringify(info), doc: refOf(document.documentElement) } : { ok: true, info };};const stillAt = (at: Point, expect: unknown, sig?: unknown): Element => {@@ -1120,7 +1120,8 @@ export function installPageAgent(): PageAgent {return {ok: true,quietMs: Math.round(performance.now() - lastMutation),- requests: typeof pending === "function" ? Number(pending(since)) || 0 : 0,+ // Absent where nothing counts them (native views), so a wait can't take that for "none open".+ ...(typeof pending === "function" ? { requests: Number(pending(since)) || 0 } : {}),title: document.title,...view(),};diff --git a/studio/frontend/src/features/browser/agent/page-wait.ts b/studio/frontend/src/features/browser/agent/page-wait.tsindex 8eeecb33f92..43baeb77c04 100644--- a/studio/frontend/src/features/browser/agent/page-wait.ts+++ b/studio/frontend/src/features/browser/agent/page-wait.ts@@ -10,6 +10,8 @@ export type PageWait = {/** A DOM that keeps changing with no request open is waited on only this long. */changingMaxMs: number;quietMs: number;+ /** Enough quiet when the page counts its requests and none was open during the wait. */+ idleQuietMs?: number;now: () => number;sleep: (ms: number) => Promise<unknown>;/** Open requests started since the action, and how long the DOM has been quiet. */@@ -26,6 +28,7 @@ export async function waitForPage(wait: PageWait): Promise<"settled" | "loading"// After a request closes, quiet and the changing-DOM allowance count from then: its DOM work comes after it.let free: number | null = null;let opened = false;+ let counted = true;while (!wait.aborted?.()) {if (wait.moved()) return "moved";const status = await wait.status();@@ -34,9 +37,11 @@ export async function waitForPage(wait: PageWait): Promise<"settled" | "loading"const now = wait.now();const waited = now - started;if (loading) opened = true;+ if (status.requests === undefined) counted = false;free = loading ? null : (free ?? (opened ? now : started));const quiet = opened && free !== null ? Math.min(status.quietMs ?? 0, now - free) : (status.quietMs ?? 0);- if (free !== null && (quiet >= wait.quietMs || now - free >= wait.changingMaxMs)) return "settled";+ const needed = opened || !counted ? wait.quietMs : Math.min(wait.quietMs, wait.idleQuietMs ?? wait.quietMs);+ if (free !== null && (quiet >= needed || now - free >= wait.changingMaxMs)) return "settled";if (waited >= wait.maxMs) return loading ? "loading" : "settled";await wait.sleep(POLL_MS);}diff --git a/studio/frontend/tests/browser-agent-drag.test.ts b/studio/frontend/tests/browser-agent-drag.test.tsindex 08f9efedb76..01d8c94e54d 100644--- a/studio/frontend/tests/browser-agent-drag.test.ts+++ b/studio/frontend/tests/browser-agent-drag.test.ts@@ -25,7 +25,7 @@ test("a drag whose press navigates the tab is not released on the new page", asyglobals.__frameAgent = async (_tabId: string, op: Record<string, unknown>) => {sent.push(`${op.op}${op.phase ? `:${op.phase}` : ""}`);if (op.op === "status") return { installed: true, result: { ok: true, quietMs: 1000, doc } };- if (op.op === "describe") return { installed: true, result: { ok: true, info, hit: "r5", sig: "s" } };+ if (op.op === "describe") return { installed: true, result: { ok: true, info, hit: "r5", sig: "s", doc } };if (op.op === "pointer" && op.phase === "down") {// The page's own mousedown handler loads another document at the same address.doc = "r9001";@@ -53,7 +53,7 @@ test("a drag between fractional points reaches its end and is released", async (globals.__frameAgent = async (_tabId: string, op: Record<string, unknown>) => {sent.push(op);if (op.op === "status") return { installed: true, result: { ok: true, quietMs: 1000, doc: "r1" } };- if (op.op === "describe") return { installed: true, result: { ok: true, info, hit: "r5", sig: "s" } };+ if (op.op === "describe") return { installed: true, result: { ok: true, info, hit: "r5", sig: "s", doc: "r1" } };return { installed: true, result: { ok: true, text: "page", links: [] } };};// On the tab the test above opened.@@ -63,6 +63,8 @@ test("a drag between fractional points reaches its end and is released", async (assert.equal(posted.at(-1)?.result.error, undefined);const up = sent.find((op) => op.op === "pointer" && op.phase === "up");assert.deepEqual(up && [up.x, up.y], [1.9, 10]);+ // The document check rides on the describes the drag makes anyway: no status round trips before the drop.+ assert.equal(sent.slice(0, sent.indexOf(up!)).filter((op) => op.op === "status").length, 0);});/** Drags on the tab the first test opened, with `onOp` seeing each page command; returns what was sent and the result. */@@ -77,7 +79,7 @@ async function drag(id: string, runKey: string, onOp: (op: Record<string, unknowsent.push(`${op.op}${op.phase ? `:${op.phase}` : ""}`);onOp(op, sent);if (op.op === "status") return { installed: true, result: { ok: true, quietMs: 1000, doc: "r1" } };- if (op.op === "describe") return { installed: true, result: { ok: true, info, hit: "r5", sig: "s" } };+ if (op.op === "describe") return { installed: true, result: { ok: true, info, hit: "r5", sig: "s", doc: "r1" } };return { installed: true, result: { ok: true, text: "page", links: [] } };};const tabId = useBrowserStore.getState().tabs[0].id;@@ -103,7 +105,7 @@ test("a drag whose page moves its address without a new document is cancelled, ntest("Stop during the checks before the drop cancels the drag instead of releasing it", async () => {const { sent, result } = await drag("p2", "r", (op, sent) => {- if (op.op === "status" && sent.includes("pointer:move")) stopBrowserAgent();+ if (op.op === "describe" && sent.includes("pointer:move")) stopBrowserAgent();});assert.equal(result?.ok, false);assert.ok(sent.includes("pointer:cancel"), sent.join(", "));diff --git a/studio/frontend/tests/browser-agent-page-wait.test.ts b/studio/frontend/tests/browser-agent-page-wait.test.tsindex c0c9784051d..c39070c5e2f 100644--- a/studio/frontend/tests/browser-agent-page-wait.test.ts+++ b/studio/frontend/tests/browser-agent-page-wait.test.ts@@ -86,3 +86,27 @@ test("a DOM quiet while a request was open is not read as settled the moment itassert.equal(await waitForPage(p.wait(0, 3000, 1000)), "settled");assert.ok(p.now() >= 1000 + QUIET, `read at ${p.now()}, before the list was drawn`);});++test("a loaded page that opened no request is read after the short idle quiet", async () => {+ const p = page([], [0]);+ assert.equal(await waitForPage({ ...p.wait(0, 3000, 1000), idleQuietMs: 120 }), "settled");+ assert.ok(p.now() >= 120 && p.now() < QUIET, `read at ${p.now()}`);+});++test("a page whose requests nothing counts (a native view) still gets the full quiet", async () => {+ const p = page([], [0]);+ const wait = p.wait(0, 3000, 1000);+ const counted = wait.status;+ wait.status = async () => {+ const status = await counted();+ return status && { quietMs: status.quietMs };+ };+ assert.equal(await waitForPage({ ...wait, idleQuietMs: 120 }), "settled");+ assert.ok(p.now() >= QUIET, `read at ${p.now()}`);+});++test("a page that had a request open gets the full quiet after it closes, short idle or not", async () => {+ const p = page([{ start: 0, end: 300 }], [0, 300]);+ assert.equal(await waitForPage({ ...p.wait(0, 3000, 1000), idleQuietMs: 120 }), "settled");+ assert.ok(p.now() >= 300 + QUIET, `read at ${p.now()}`);+});