Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep annotate marks on PDF pages and slides that remount (#13119)

Michael Hane7cf10f05a5b
diff --git a/studio/frontend/src/features/browser/annotate-layer.tsx b/studio/frontend/src/features/browser/annotate-layer.tsx
index 600e27243f5..11b0d640d4a 100644
--- a/studio/frontend/src/features/browser/annotate-layer.tsx
+++ b/studio/frontend/src/features/browser/annotate-layer.tsx
@@ -54,13 +54,16 @@ async function annotationScreenshot(page: HTMLElement | null): Promise<File[]> {
type Annotation = {
id: number;
ranges: Range[];
+ places?: (Place | null)[];
quote: string;
frame?: Frame;
request: string;
};
-/** A drag's box, offset from its content so it scrolls with it, at the zoom it was drawn at. */
-type Frame = { left: number; top: number; width: number; height: number; zoom: number };
-type Pending = { id: number | null; ranges: Range[]; quote: string; frame?: Frame };
+/** A drag's box, offset from its content so it scrolls with it, at the zoom and page width it was drawn at. */
+type Frame = { left: number; top: number; width: number; height: number; zoom: number; item?: number };
+type Pending = { id: number | null; ranges: Range[]; places?: (Place | null)[]; quote: string; frame?: Frame };
+/** Where a range sits in a virtualized page: its text offsets, or the path to the element it selects. */
+type Place = { item: string; start: number; end: number; text: string } | { item: string; path: number[]; tag: string };
type Box = { left: number; top: number; width: number; height: number };
function hasOwnText(element: Element): boolean {
@@ -230,12 +233,157 @@ function zoomAt(ranges: Range[]): number {
return cssZoomOf(element);
}
+// PDF pages and slides: virtualized, and remounted at a new width on zoom.
+const ITEM = "[data-index]";
+
+const itemOf = (node: Node): HTMLElement | null =>
+ (node instanceof Element ? node : node.parentElement)?.closest<HTMLElement>(ITEM) ?? null;
+
+/** A page's text nodes and where each starts, built once and shared by all its ranges. */
+type TextIndex = { nodes: Text[]; starts: number[]; at: Map<Node, number>; length: number };
+
+function textIndex(item: Element): TextIndex {
+ const nodes: Text[] = [];
+ const starts: number[] = [];
+ const at = new Map<Node, number>();
+ let length = 0;
+ const walker = document.createTreeWalker(item, NodeFilter.SHOW_TEXT);
+ for (let node = walker.nextNode(); node; node = walker.nextNode()) {
+ nodes.push(node as Text);
+ starts.push(length);
+ at.set(node, length);
+ length += node.textContent?.length ?? 0;
+ }
+ return { nodes, starts, at, length };
+}
+
+/** Index of the last item `test` holds for, when it holds for a prefix; else -1. */
+function lastWhere<T>(items: readonly T[], test: (item: T) => boolean): number {
+ let low = 0;
+ let high = items.length;
+ while (low < high) {
+ const middle = (low + high) >> 1;
+ if (test(items[middle] as T)) low = middle + 1;
+ else high = middle;
+ }
+ return low - 1;
+}
+
+/** Characters of the page before a boundary point. */
+function offsetIn(index: TextIndex, node: Node, offset: number): number {
+ const start = index.at.get(node);
+ if (start !== undefined) return start + offset;
+ // A point beside a text node, as around a span's own text.
+ const next = node.childNodes[offset];
+ const after = next && index.at.get(next);
+ if (after !== undefined) return after;
+ const prior = node.childNodes[offset - 1];
+ const before = prior && index.at.get(prior);
+ if (before !== undefined) return before + (prior.textContent?.length ?? 0);
+ // Elsewhere between nodes: count the text nodes that start before it.
+ const point = document.createRange();
+ point.setStart(node, offset);
+ const last = lastWhere(index.nodes, (text) => point.comparePoint(text, 0) < 0);
+ const text = index.nodes[last];
+ return text ? (index.starts[last] ?? 0) + text.length : 0;
+}
+
+/** The text node and offset `offset` characters in; a start between nodes takes the next. */
+function pointIn(index: TextIndex, offset: number, start: boolean): [Node, number] | null {
+ const found = lastWhere(index.starts, (at) => (start ? at <= offset : at < offset));
+ const node = index.nodes[Math.max(found, 0)];
+ const local = offset - (index.starts[Math.max(found, 0)] ?? 0);
+ return node && local >= 0 && local <= node.length ? [node, local] : null;
+}
+
+/** The element a range selects whole, as an image's does. */
+function selectedElement(range: Range): Element | null {
+ const node = range.startContainer.childNodes[range.startOffset];
+ return node instanceof Element && range.endContainer === range.startContainer && range.endOffset === range.startOffset + 1
+ ? node
+ : null;
+}
+
+function placesOf(ranges: Range[]): (Place | null)[] {
+ const indexes = new Map<Element, TextIndex>();
+ return ranges.map((range) => {
+ const item = itemOf(range.startContainer);
+ if (!item || itemOf(range.endContainer) !== item) return null;
+ const text = range.toString();
+ const element = selectedElement(range);
+ if (!text.trim()) {
+ if (!element) return null;
+ const path: number[] = [];
+ for (let node: Element = element; node !== item; node = node.parentElement!) {
+ path.unshift([...node.parentElement!.children].indexOf(node));
+ }
+ return { item: item.dataset.index ?? "", path, tag: element.tagName };
+ }
+ let index = indexes.get(item);
+ if (!index) indexes.set(item, (index = textIndex(item)));
+ const start = offsetIn(index, range.startContainer, range.startOffset);
+ const end = offsetIn(index, range.endContainer, range.endOffset);
+ return { item: item.dataset.index ?? "", start, end, text };
+ });
+}
+
+/** The place's content in its remounted page, if that page shows the same thing. */
+function rangeAt(place: Place, page: HTMLElement, indexes: Map<Element, TextIndex>): Range | null {
+ // Hidden tabs keep their documents, with the same page numbers: only a shown page counts.
+ const item = [...page.querySelectorAll(`${ITEM}[data-index="${CSS.escape(place.item)}"]`)].find(
+ (candidate) => candidate.getClientRects().length > 0,
+ );
+ if (!item) return null;
+ const range = document.createRange();
+ if ("path" in place) {
+ let element: Element | undefined = item;
+ for (const step of place.path) element = element?.children[step];
+ if (element?.tagName !== place.tag) return null;
+ range.selectNode(element);
+ return range;
+ }
+ let index = indexes.get(item);
+ if (!index) indexes.set(item, (index = textIndex(item)));
+ const start = pointIn(index, place.start, true);
+ const end = pointIn(index, place.end, false);
+ if (!start || !end) return null;
+ range.setStart(...start);
+ range.setEnd(...end);
+ return range.toString() === place.text ? range : null;
+}
+
+const rebound = new WeakMap<Range, Range>();
+
+/** Whether a range still holds what was marked; removed nodes collapse it onto what is left. */
+function holds(range: Range, place: Place): boolean {
+ if (!("path" in place)) return range.toString() === place.text;
+ return selectedElement(range)?.tagName === place.tag && range.startContainer.isConnected;
+}
+
+/** The ranges, found again where their page remounted; `indexes` is shared by every mark in a render. */
+function liveRanges(
+ mark: { ranges: Range[]; places?: (Place | null)[] },
+ page: HTMLElement,
+ indexes: Map<Element, TextIndex>,
+): Range[] {
+ return mark.ranges.map((range, index) => {
+ const place = mark.places?.[index];
+ if (!place || holds(range, place)) return range;
+ const known = rebound.get(range);
+ if (known && holds(known, place)) return known;
+ const found = rangeAt(place, page, indexes);
+ if (found) rebound.set(range, found);
+ return found ?? range;
+ });
+}
+
/** A drag's box if any, else the box around the content. */
function markBoxOf(ranges: Range[], frame: Frame | undefined, origin: DOMRect): Box | null {
const content = boxOf(ranges, origin);
if (!content || !frame) return content;
- // Scale by the zoom change since the drag, as the content did.
- const k = zoomAt(ranges) / frame.zoom;
+ // Scale by the zoom and page width change since the drag, as the content did.
+ const item = ranges[0] && frame.item ? itemOf(ranges[0].startContainer) : null;
+ const k = (zoomAt(ranges) / frame.zoom) * (item && frame.item ? item.offsetWidth / frame.item : 1);
return {
left: content.left + PAD + frame.left * k,
top: content.top + PAD + frame.top * k,
@@ -287,7 +435,14 @@ export function AnnotateLayer({
if (!request) return items;
return [
...items,
- { id: nextId.current++, ranges: pending.ranges, quote: pending.quote, frame: pending.frame, request },
+ {
+ id: nextId.current++,
+ ranges: pending.ranges,
+ places: pending.places,
+ quote: pending.quote,
+ frame: pending.frame,
+ request,
+ },
];
}
return request
@@ -336,12 +491,18 @@ export function AnnotateLayer({
page.addEventListener("transitionend", zoomed, true);
const resize = new ResizeObserver(redraw);
resize.observe(page);
+ // A remounted page renders its text later; marks are found again then.
+ const added = new MutationObserver((records) => {
+ if (records.some((record) => record.addedNodes.length && !layerRef.current?.contains(record.target))) redraw();
+ });
+ added.observe(page, { childList: true, subtree: true });
return () => {
cancelAnimationFrame(frame);
window.clearTimeout(settle);
page.removeEventListener("scroll", redraw, { capture: true });
page.removeEventListener("transitionend", zoomed, true);
resize.disconnect();
+ added.disconnect();
};
}, [page]);
@@ -394,9 +555,10 @@ export function AnnotateLayer({
width: area.width,
height: area.height,
zoom: zoomAt(ranges),
+ item: itemOf(ranges[0]!.startContainer)?.offsetWidth,
}
: undefined;
- setPending({ id: null, ranges, quote, frame });
+ setPending({ id: null, ranges, places: placesOf(ranges), quote, frame });
setDraft("");
};
const onDown = (event: PointerEvent) => {
@@ -508,7 +670,8 @@ export function AnnotateLayer({
const hoverBox = hover ? boxOf(hover, origin) : null;
if (hoverBox) lastHoverBox.current = hoverBox;
const shownHover = hoverBox ?? lastHoverBox.current;
- const pendingBox = pending ? markBoxOf(pending.ranges, pending.frame, origin) : null;
+ const indexes = new Map<Element, TextIndex>();
+ const pendingBox = pending ? markBoxOf(liveRanges(pending, page, indexes), pending.frame, origin) : null;
const count = items.length;
// A first comment still being typed can go too: Send commits it.
const canSend = count > 0 || (pending?.id === null && draft.trim() !== "");
@@ -535,7 +698,7 @@ export function AnnotateLayer({
/>
) : null}
{items.map((item, index) => {
- const box = item.id === pending?.id ? null : markBoxOf(item.ranges, item.frame, origin);
+ const box = item.id === pending?.id ? null : markBoxOf(liveRanges(item, page, indexes), item.frame, origin);
return box ? (
<Mark
key={item.id}
@@ -546,6 +709,7 @@ export function AnnotateLayer({
setPending({
id: item.id,
ranges: item.ranges,
+ places: item.places,
quote: item.quote,
frame: item.frame,
});