Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep a dragged annotate area the size it was drawn (#13070)

Michael Hanb0f01077e117
diff --git a/studio/backend/routes/browser.py b/studio/backend/routes/browser.py
index 5194267fe53..fef649d56c9 100644
--- a/studio/backend/routes/browser.py
+++ b/studio/backend/routes/browser.py
@@ -655,7 +655,11 @@ _FRAME_HTML = r"""<!doctype html>
}
else if (data.command === "annotateForget") annotation?.forget(Number(data.id));
else if (data.command === "annotateNumbers") annotation ? annotation.number(data.numbers) : (numbers = data.numbers);
- else if (data.command === "zoom") applyZoom(data.value);
+ else if (data.command === "zoom") {
+ applyZoom(data.value);
+ // A zoom fires no scroll or resize, so marks are redrawn here.
+ annotation?.redraw();
+ }
else if (data.command === "mute") setMuted(data.on === true);
else if (data.command === "find" && typeof data.query === "string" && data.query.length <= 1000) finder.search(data.query);
else if (data.command === "findStep") finder.step(data.delta === -1 ? -1 : 1);
@@ -795,13 +799,21 @@ _ANNOTATE_JS = r"""
}
return document.documentElement;
};
- // Where a mark is now: around what it marks, or its blank area, moved with its anchor.
+ // Where a mark is now: a drag as drawn, a click around its target, a blank area by its anchor.
const markBox = (mark) => {
- if (mark.ranges) return boxOf(mark.ranges);
- const { element, dx, dy, width, height } = mark.anchor;
+ if (mark.ranges) {
+ const box = boxOf(mark.ranges);
+ if (!box || !mark.frame) return box;
+ // Scale by the zoom change since the drag, as the content did.
+ const { dx, dy, width, height, zoom } = mark.frame;
+ const k = zoomOf() / zoom;
+ return { left: box.left + PAD + dx * k, top: box.top + PAD + dy * k, width: width * k, height: height * k };
+ }
+ const { element, dx, dy, width, height, zoom } = mark.anchor;
if (!element.isConnected) return null;
const rect = element.getBoundingClientRect();
- return { left: rect.left + dx, top: rect.top + dy, width, height };
+ const k = zoomOf() / zoom;
+ return { left: rect.left + dx * k, top: rect.top + dy * k, width: width * k, height: height * k };
};
const boxOf = (ranges) => {
let left = Infinity, top = Infinity, right = -Infinity, bottom = -Infinity;
@@ -902,18 +914,23 @@ _ANNOTATE_JS = r"""
marks.set(id, mark);
send("mark", { id, rect: mark.at, ...details });
};
- const addMark = (ranges) => {
+ const addMark = (ranges, area = null) => {
if (!ranges || !ranges.length) return;
const quote = quoteOf(ranges);
const picture = pictureOf(ranges);
if (!quote && picture === null) return;
- createMark({ ranges }, { quote, image: picture !== null, alt: picture || "", area: false });
+ // Keep a drag's box as drawn, not shrunk to its text.
+ const content = area && boxOf(ranges);
+ const frame = content
+ ? { dx: area.left - content.left - PAD, dy: area.top - content.top - PAD, width: area.width, height: area.height, zoom: zoomOf() }
+ : null;
+ createMark({ ranges, frame }, { quote, image: picture !== null, alt: picture || "", area: false });
};
// An area with nothing in it marks that part of the page itself.
const addArea = (area) => {
const element = anchorOf(area);
const rect = element.getBoundingClientRect();
- const anchor = { element, dx: area.left - rect.left, dy: area.top - rect.top, width: area.width, height: area.height };
+ const anchor = { element, dx: area.left - rect.left, dy: area.top - rect.top, width: area.width, height: area.height, zoom: zoomOf() };
createMark({ anchor }, { quote: "", image: false, alt: "", area: true });
};
// The panel numbers the marks, matching the order they take in the chat.
@@ -983,7 +1000,7 @@ _ANNOTATE_JS = r"""
send("up");
if (box) {
const ranges = blocksIn(new DOMRect(box.left, box.top, box.width, box.height));
- ranges.length ? addMark(ranges) : addArea(box);
+ ranges.length ? addMark(ranges, box) : addArea(box);
}
else if (pin !== null) send("open", { id: pin });
else addMark(blockAt(document.elementFromPoint(event.clientX, event.clientY) || event.target));
@@ -1034,7 +1051,7 @@ _ANNOTATE_JS = r"""
pending = press = hover = lastMove = overlay = root = null;
marks.clear();
};
- return { start, stop, forget, number };
+ return { start, stop, forget, number, redraw };
"""
diff --git a/studio/backend/tests/test_browser_routes.py b/studio/backend/tests/test_browser_routes.py
index b670a23fc73..794f4057d93 100644
--- a/studio/backend/tests/test_browser_routes.py
+++ b/studio/backend/tests/test_browser_routes.py
@@ -92,7 +92,7 @@ def test_annotate_code_is_sent_only_when_annotating():
# Not in the shell (so not in every page), installed once and only from the shell's own message.
assert "const BLOCK =" in code and "const BLOCK =" not in shell
assert 'const value = node.type === "password" ? "" : node.value;' in code
- assert code.rstrip().endswith("return { start, stop, forget, number };")
+ assert code.rstrip().endswith("return { start, stop, forget, number, redraw };")
assert 'if (data.command === "annotateInstall") install(data.code);' in shell
assert 'if (annotation || typeof code !== "string"' in shell
assert shell.index("const compile = Function;") < shell.index("const install = ")
diff --git a/studio/frontend/src/features/browser/annotate-layer.tsx b/studio/frontend/src/features/browser/annotate-layer.tsx
index 3e535b0a18e..a502dd948a2 100644
--- a/studio/frontend/src/features/browser/annotate-layer.tsx
+++ b/studio/frontend/src/features/browser/annotate-layer.tsx
@@ -55,9 +55,12 @@ type Annotation = {
id: number;
ranges: Range[];
quote: string;
+ frame?: Frame;
request: string;
};
-type Pending = { id: number | null; ranges: Range[]; quote: string };
+/** A drag's box, offset from its content so it scrolls with it, at the zoom it was drawn at. */
+type Frame = { left: number; top: number; width: number; height: number; zoom: number };
+type Pending = { id: number | null; ranges: Range[]; quote: string; frame?: Frame };
type Box = { left: number; top: number; width: number; height: number };
function hasOwnText(element: Element): boolean {
@@ -197,6 +200,50 @@ function boxOf(ranges: Range[], origin: DOMRect): Box | null {
};
}
+/** Effective CSS zoom of an element. */
+function cssZoomOf(element: Element | null): number {
+ if (!element) return 1;
+ // Typed as always present, but older engines lack it.
+ const reported: unknown = element.currentCSSZoom;
+ if (typeof reported === "number") return reported;
+ // Older engines: multiply each ancestor's own zoom.
+ let zoom = 1;
+ for (let el: Element | null = element; el; el = el.parentElement) {
+ zoom *= Number.parseFloat(getComputedStyle(el).zoom) || 1;
+ }
+ return zoom;
+}
+
+/** Effective CSS zoom at the content. */
+function zoomAt(ranges: Range[]): number {
+ const range = ranges[0];
+ if (!range) return 1;
+ const node = range.startContainer;
+ // A selected node (an image) can carry its own zoom; its range starts at the parent.
+ const selected = node.childNodes[range.startOffset];
+ const element =
+ selected instanceof Element && range.endContainer === node && range.endOffset === range.startOffset + 1
+ ? selected
+ : node instanceof Element
+ ? node
+ : node.parentElement;
+ return cssZoomOf(element);
+}
+
+/** A drag's box if any, else the box around the content. */
+function markBoxOf(ranges: Range[], frame: Frame | undefined, origin: DOMRect): Box | null {
+ const content = boxOf(ranges, origin);
+ if (!content || !frame) return content;
+ // Scale by the zoom change since the drag, as the content did.
+ const k = zoomAt(ranges) / frame.zoom;
+ return {
+ left: content.left + PAD + frame.left * k,
+ top: content.top + PAD + frame.top * k,
+ width: frame.width * k,
+ height: frame.height * k,
+ };
+}
+
const sameRanges = (a: Range[] | null, b: Range[] | null) =>
a !== null &&
b !== null &&
@@ -237,7 +284,10 @@ export function AnnotateLayer({
const request = draft.trim();
if (pending.id === null) {
if (!request) return items;
- return [...items, { id: nextId.current++, ranges: pending.ranges, quote: pending.quote, request }];
+ return [
+ ...items,
+ { id: nextId.current++, ranges: pending.ranges, quote: pending.quote, frame: pending.frame, request },
+ ];
}
return request
? items.map((item) => (item.id === pending.id ? { ...item, request } : item))
@@ -316,7 +366,7 @@ export function AnnotateLayer({
Math.abs(event.clientY - press.y),
);
};
- const mark = (ranges: Range[] | null) => {
+ const mark = (ranges: Range[] | null, area?: DOMRect) => {
if (!ranges || ranges.length === 0) return;
const image = ranges.some((range) =>
range.cloneContents().querySelector("img"),
@@ -324,7 +374,18 @@ export function AnnotateLayer({
const quote =
quoteOf(ranges) || (image ? t("browser.annotate.imageQuote") : "");
if (!quote) return;
- setPending({ id: null, ranges, quote });
+ // Keep a drag's box as drawn, not shrunk to its text.
+ const content = area ? boxOf(ranges, new DOMRect()) : null;
+ const frame = area && content
+ ? {
+ left: area.left - content.left - PAD,
+ top: area.top - content.top - PAD,
+ width: area.width,
+ height: area.height,
+ zoom: zoomAt(ranges),
+ }
+ : undefined;
+ setPending({ id: null, ranges, quote, frame });
setDraft("");
};
const onDown = (event: PointerEvent) => {
@@ -384,7 +445,7 @@ export function AnnotateLayer({
press = null;
setArea(null);
saveRef.current();
- if (dragging && rect) mark(blocksIn(page, rect));
+ if (dragging && rect) mark(blocksIn(page, rect), rect);
else if (event.target instanceof Element && !ownUi(event.target))
mark(blockAt(event.target, page));
};
@@ -436,7 +497,7 @@ export function AnnotateLayer({
const hoverBox = hover ? boxOf(hover, origin) : null;
if (hoverBox) lastHoverBox.current = hoverBox;
const shownHover = hoverBox ?? lastHoverBox.current;
- const pendingBox = pending ? boxOf(pending.ranges, origin) : null;
+ const pendingBox = pending ? markBoxOf(pending.ranges, pending.frame, origin) : null;
const count = items.length;
// A first comment still being typed can go too: Send commits it.
const canSend = count > 0 || (pending?.id === null && draft.trim() !== "");
@@ -463,7 +524,7 @@ export function AnnotateLayer({
/>
) : null}
{items.map((item, index) => {
- const box = item.id === pending?.id ? null : boxOf(item.ranges, origin);
+ const box = item.id === pending?.id ? null : markBoxOf(item.ranges, item.frame, origin);
return box ? (
<Mark
key={item.id}
@@ -475,6 +536,7 @@ export function AnnotateLayer({
id: item.id,
ranges: item.ranges,
quote: item.quote,
+ frame: item.frame,
});
setDraft(item.request);
}}