Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: faster browser agent page reads after a load, and leaner drags (#13602)

Daniel Handd5ec4877ee0
diff --git a/studio/frontend/src/features/browser/agent/executor.ts b/studio/frontend/src/features/browser/agent/executor.ts
index c45c8fe0d34..51aab1a4d6c 100644
--- a/studio/frontend/src/features/browser/agent/executor.ts
+++ b/studio/frontend/src/features/browser/agent/executor.ts
@@ -53,6 +53,8 @@ const LOAD_TIMEOUT_MS = 30_000;
const SETTLE_QUIET_MS = 350;
const SETTLE_MAX_MS = 3_000;
const FILL_QUIET_MAX_MS = 1_000;
+// A loaded page that opened no request has drawn what it will; content still on its way arrives by a request.
+const FILL_IDLE_QUIET_MS = 120;
const PAGE_READY_MS = 4_000;
const POST_TIMEOUT_MS = 15_000;
const EXPIRED = "Not done: the action ran out of time, likely waiting for the user to answer in the panel. Ask them before trying again.";
@@ -358,9 +360,16 @@ async function pageCall(tab: BrowserTab, command: Record<string, unknown>): Prom
}
/** Requests started since `since` and a quiet DOM; `left` says the tab is on another page. */
-function waitOnPage(tabId: string, left: (tab: BrowserTab) => boolean, since: number, changingMaxMs: number, signal?: AbortSignal) {
+function waitOnPage(
+ tabId: string,
+ left: (tab: BrowserTab) => boolean,
+ since: number,
+ changingMaxMs: number,
+ signal?: AbortSignal,
+ idleQuietMs?: number,
+) {
return waitForPage({
- maxMs: SETTLE_MAX_MS, changingMaxMs, quietMs: SETTLE_QUIET_MS, now: Date.now, sleep,
+ maxMs: SETTLE_MAX_MS, changingMaxMs, quietMs: SETTLE_QUIET_MS, idleQuietMs, now: Date.now, sleep,
status: () => {
const tab = findTab(tabId);
return tab ? pageCall(tab, { op: "status", since }).catch(() => null) : Promise.resolve(null);
@@ -385,7 +394,7 @@ async function settle(tabId: string, urlBefore: string, since: number, signal?:
* A DOM that never rests gets only FILL_QUIET_MAX_MS. A router rewriting the address is still this page. */
async function untilFilled(tab: BrowserTab, since: number, signal?: AbortSignal): Promise<boolean> {
const entry = currentEntry(tab);
- return (await waitOnPage(tab.id, (now) => currentEntry(now) !== entry, since, FILL_QUIET_MAX_MS, signal)) === "loading";
+ return (await waitOnPage(tab.id, (now) => currentEntry(now) !== entry, since, FILL_QUIET_MAX_MS, signal, FILL_IDLE_QUIET_MS)) === "loading";
}
async function pass(gate: Gate, signal?: AbortSignal): Promise<string | null> {
@@ -786,6 +795,7 @@ async function dragBetween(tab: BrowserTab, action: BrowserAction, context: Acti
const hits: string[] = [];
const sigs: string[] = [];
const infos: string[] = [];
+ const docs: string[] = [];
let asked = false;
for (const point of [from, end]) {
const described = await pageCall(tab, { op: "describe", ref: "", ...point });
@@ -800,16 +810,16 @@ async function dragBetween(tab: BrowserTab, action: BrowserAction, context: Acti
hits.push(described.hit ?? "");
sigs.push(described.sig ?? "");
infos.push(JSON.stringify(described.info));
+ docs.push(described.doc ?? "");
}
if (signal?.aborted) return { ok: false, error: STOPPED };
if (asked && !(await unchanged(tab, from, hits[0], infos[0]))) return { ok: false, error: CHANGED_UNDER_POINT };
// A handler can navigate mid-drag: the rest must not act on a page nobody gated.
- const doc = (await pageCall(tab, { op: "status" })).doc;
+ const doc = docs[0];
const sameTab = () => {
const now = useBrowserStore.getState().tabs.find((candidate) => candidate.id === tab.id);
return Boolean(now && !now.loading && tabUrl(now) === before);
};
- const samePage = async () => sameTab() && (await pageCall(tab, { op: "status" }).catch(() => null))?.doc === doc;
const since = Date.now();
const down = await pageCall(tab, { op: "pointer", phase: "down", ...from, expect: hits[0], sig: sigs[0] });
if (!down.ok) return { ok: false, error: down.error };
@@ -832,16 +842,18 @@ async function dragBetween(tab: BrowserTab, action: BrowserAction, context: Acti
return { ok: false, error };
};
// A new document gets nothing; the same one (a hash or pushState move) has its drag cancelled.
- const left = async (): Promise<Result> => {
+ const left = async (current?: string): Promise<Result> => {
const error = "the page changed during the drag, so it was not finished; take a snapshot.";
- const status = await pageCall(tab, { op: "status" }).catch(() => null);
- return doc && status?.doc === doc ? cancel(error) : { ok: false, error };
+ const now = current ?? (await pageCall(tab, { op: "status" }).catch(() => null))?.doc;
+ return doc && now === doc ? cancel(error) : { ok: false, error };
};
if (signal?.aborted) return cancel(STOPPED);
- if (!doc || !(await samePage())) return left();
- if (at.x !== end.x || at.y !== end.y) return cancel("the drag stopped short; take a snapshot to see where things are.");
- // Drop zones can appear or relabel once a drag starts: re-ask unless the end is unchanged.
+ if (!doc || !sameTab()) return left();
+ // Drop zones can appear or relabel once a drag starts: re-ask unless the end is unchanged. Its doc
+ // says whether this is still the document the drag began on.
const target = await pageCall(tab, { op: "describe", ref: "", ...end }).catch(() => null);
+ if (target?.doc !== doc) return left(target?.doc);
+ if (at.x !== end.x || at.y !== end.y) return cancel("the drag stopped short; take a snapshot to see where things are.");
if (!target?.ok || !target.info) return cancel(target?.error ?? "nothing is at the drag's end.");
if (target.hit !== hits[1] || JSON.stringify(target.info) !== infos[1]) {
for (const gate of [gateElement("click", target.info, before, action.policy, permissions()), opaqueGate(target.info, before)]) {
@@ -850,8 +862,9 @@ async function dragBetween(tab: BrowserTab, action: BrowserAction, context: Acti
if (refused) return cancel(refused);
if (gate.kind === "ask" && gate.ask === "pointer") opaqueHosts.add(gate.host);
}
- if (!(await unchanged(tab, end, target.hit ?? "", JSON.stringify(target.info)))) return cancel(CHANGED_UNDER_POINT);
- if (!(await samePage())) return left();
+ const again = await pageCall(tab, { op: "describe", ref: "", ...end }).catch(() => null);
+ if (!sameTab() || again?.doc !== doc) return left(again?.doc);
+ if (!again?.ok || again.hit !== target.hit || JSON.stringify(again.info) !== JSON.stringify(target.info)) return cancel(CHANGED_UNDER_POINT);
}
if (signal?.aborted) return cancel(STOPPED);
const up = await pageCall(tab, { op: "pointer", phase: "up", ...end, expect: target.hit ?? "", sig: target.sig ?? "" });
diff --git a/studio/frontend/src/features/browser/agent/page-agent.ts b/studio/frontend/src/features/browser/agent/page-agent.ts
index ad781bf694c..0448c24cb50 100644
--- a/studio/frontend/src/features/browser/agent/page-agent.ts
+++ b/studio/frontend/src/features/browser/agent/page-agent.ts
@@ -592,7 +592,7 @@ export function installPageAgent(): PageAgent {
: {}),
...(!submits && submitter ? { submitter: clean(nameOf(submitter), 200) } : {}),
};
- return point ? { ok: true, info, hit: refOf(found), sig: JSON.stringify(info) } : { ok: true, info };
+ return point ? { ok: true, info, hit: refOf(found), sig: JSON.stringify(info), doc: refOf(document.documentElement) } : { ok: true, info };
};
const stillAt = (at: Point, expect: unknown, sig?: unknown): Element => {
@@ -1120,7 +1120,8 @@ export function installPageAgent(): PageAgent {
return {
ok: true,
quietMs: Math.round(performance.now() - lastMutation),
- requests: typeof pending === "function" ? Number(pending(since)) || 0 : 0,
+ // Absent where nothing counts them (native views), so a wait can't take that for "none open".
+ ...(typeof pending === "function" ? { requests: Number(pending(since)) || 0 } : {}),
title: document.title,
...view(),
};
diff --git a/studio/frontend/src/features/browser/agent/page-wait.ts b/studio/frontend/src/features/browser/agent/page-wait.ts
index 8eeecb33f92..43baeb77c04 100644
--- a/studio/frontend/src/features/browser/agent/page-wait.ts
+++ b/studio/frontend/src/features/browser/agent/page-wait.ts
@@ -10,6 +10,8 @@ export type PageWait = {
/** A DOM that keeps changing with no request open is waited on only this long. */
changingMaxMs: number;
quietMs: number;
+ /** Enough quiet when the page counts its requests and none was open during the wait. */
+ idleQuietMs?: number;
now: () => number;
sleep: (ms: number) => Promise<unknown>;
/** Open requests started since the action, and how long the DOM has been quiet. */
@@ -26,6 +28,7 @@ export async function waitForPage(wait: PageWait): Promise<"settled" | "loading"
// After a request closes, quiet and the changing-DOM allowance count from then: its DOM work comes after it.
let free: number | null = null;
let opened = false;
+ let counted = true;
while (!wait.aborted?.()) {
if (wait.moved()) return "moved";
const status = await wait.status();
@@ -34,9 +37,11 @@ export async function waitForPage(wait: PageWait): Promise<"settled" | "loading"
const now = wait.now();
const waited = now - started;
if (loading) opened = true;
+ if (status.requests === undefined) counted = false;
free = loading ? null : (free ?? (opened ? now : started));
const quiet = opened && free !== null ? Math.min(status.quietMs ?? 0, now - free) : (status.quietMs ?? 0);
- if (free !== null && (quiet >= wait.quietMs || now - free >= wait.changingMaxMs)) return "settled";
+ const needed = opened || !counted ? wait.quietMs : Math.min(wait.quietMs, wait.idleQuietMs ?? wait.quietMs);
+ if (free !== null && (quiet >= needed || now - free >= wait.changingMaxMs)) return "settled";
if (waited >= wait.maxMs) return loading ? "loading" : "settled";
await wait.sleep(POLL_MS);
}
diff --git a/studio/frontend/tests/browser-agent-drag.test.ts b/studio/frontend/tests/browser-agent-drag.test.ts
index 08f9efedb76..01d8c94e54d 100644
--- a/studio/frontend/tests/browser-agent-drag.test.ts
+++ b/studio/frontend/tests/browser-agent-drag.test.ts
@@ -25,7 +25,7 @@ test("a drag whose press navigates the tab is not released on the new page", asy
globals.__frameAgent = async (_tabId: string, op: Record<string, unknown>) => {
sent.push(`${op.op}${op.phase ? `:${op.phase}` : ""}`);
if (op.op === "status") return { installed: true, result: { ok: true, quietMs: 1000, doc } };
- if (op.op === "describe") return { installed: true, result: { ok: true, info, hit: "r5", sig: "s" } };
+ if (op.op === "describe") return { installed: true, result: { ok: true, info, hit: "r5", sig: "s", doc } };
if (op.op === "pointer" && op.phase === "down") {
// The page's own mousedown handler loads another document at the same address.
doc = "r9001";
@@ -53,7 +53,7 @@ test("a drag between fractional points reaches its end and is released", async (
globals.__frameAgent = async (_tabId: string, op: Record<string, unknown>) => {
sent.push(op);
if (op.op === "status") return { installed: true, result: { ok: true, quietMs: 1000, doc: "r1" } };
- if (op.op === "describe") return { installed: true, result: { ok: true, info, hit: "r5", sig: "s" } };
+ if (op.op === "describe") return { installed: true, result: { ok: true, info, hit: "r5", sig: "s", doc: "r1" } };
return { installed: true, result: { ok: true, text: "page", links: [] } };
};
// On the tab the test above opened.
@@ -63,6 +63,8 @@ test("a drag between fractional points reaches its end and is released", async (
assert.equal(posted.at(-1)?.result.error, undefined);
const up = sent.find((op) => op.op === "pointer" && op.phase === "up");
assert.deepEqual(up && [up.x, up.y], [1.9, 10]);
+ // The document check rides on the describes the drag makes anyway: no status round trips before the drop.
+ assert.equal(sent.slice(0, sent.indexOf(up!)).filter((op) => op.op === "status").length, 0);
});
/** Drags on the tab the first test opened, with `onOp` seeing each page command; returns what was sent and the result. */
@@ -77,7 +79,7 @@ async function drag(id: string, runKey: string, onOp: (op: Record<string, unknow
sent.push(`${op.op}${op.phase ? `:${op.phase}` : ""}`);
onOp(op, sent);
if (op.op === "status") return { installed: true, result: { ok: true, quietMs: 1000, doc: "r1" } };
- if (op.op === "describe") return { installed: true, result: { ok: true, info, hit: "r5", sig: "s" } };
+ if (op.op === "describe") return { installed: true, result: { ok: true, info, hit: "r5", sig: "s", doc: "r1" } };
return { installed: true, result: { ok: true, text: "page", links: [] } };
};
const tabId = useBrowserStore.getState().tabs[0].id;
@@ -103,7 +105,7 @@ test("a drag whose page moves its address without a new document is cancelled, n
test("Stop during the checks before the drop cancels the drag instead of releasing it", async () => {
const { sent, result } = await drag("p2", "r", (op, sent) => {
- if (op.op === "status" && sent.includes("pointer:move")) stopBrowserAgent();
+ if (op.op === "describe" && sent.includes("pointer:move")) stopBrowserAgent();
});
assert.equal(result?.ok, false);
assert.ok(sent.includes("pointer:cancel"), sent.join(", "));
diff --git a/studio/frontend/tests/browser-agent-page-wait.test.ts b/studio/frontend/tests/browser-agent-page-wait.test.ts
index c0c9784051d..c39070c5e2f 100644
--- a/studio/frontend/tests/browser-agent-page-wait.test.ts
+++ b/studio/frontend/tests/browser-agent-page-wait.test.ts
@@ -86,3 +86,27 @@ test("a DOM quiet while a request was open is not read as settled the moment it
assert.equal(await waitForPage(p.wait(0, 3000, 1000)), "settled");
assert.ok(p.now() >= 1000 + QUIET, `read at ${p.now()}, before the list was drawn`);
});
+
+test("a loaded page that opened no request is read after the short idle quiet", async () => {
+ const p = page([], [0]);
+ assert.equal(await waitForPage({ ...p.wait(0, 3000, 1000), idleQuietMs: 120 }), "settled");
+ assert.ok(p.now() >= 120 && p.now() < QUIET, `read at ${p.now()}`);
+});
+
+test("a page whose requests nothing counts (a native view) still gets the full quiet", async () => {
+ const p = page([], [0]);
+ const wait = p.wait(0, 3000, 1000);
+ const counted = wait.status;
+ wait.status = async () => {
+ const status = await counted();
+ return status && { quietMs: status.quietMs };
+ };
+ assert.equal(await waitForPage({ ...wait, idleQuietMs: 120 }), "settled");
+ assert.ok(p.now() >= QUIET, `read at ${p.now()}`);
+});
+
+test("a page that had a request open gets the full quiet after it closes, short idle or not", async () => {
+ const p = page([{ start: 0, end: 300 }], [0, 300]);
+ assert.equal(await waitForPage({ ...p.wait(0, 3000, 1000), idleQuietMs: 120 }), "settled");
+ assert.ok(p.now() >= 300 + QUIET, `read at ${p.now()}`);
+});