Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import {
StudioDictationAdapter,
isStudioDictationAvailable,
notifyStudioDictationUnavailable,
} from "@/features/chat";
import { useT } from "@/i18n";
import { DragDropVerticalIcon } from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
import { MicIcon } from "@/lib/mic-icon";
import { Tick02Icon } from "@/lib/tick-icon";
import { toast } from "@/lib/toast";
import { cn } from "@/lib/utils";
import {
type PointerEvent as ReactPointerEvent,
type RefObject,
useEffect,
useLayoutEffect,
useRef,
useState,
} from "react";
import { flushSync } from "react-dom";
import { annotateScript } from "./api";
import { canScreenshot } from "./screenshot-support";
import { screenshotPage } from "./capture";
import { type AnnotateEvent, type AnnotateRect, MAX_MARKS } from "./frame-message";
import { startNativeAnnotate } from "./native-annotate";
import { focusPanel, nativeViewBounds } from "./native-view";
import { type FrameCommand, frameRect, frameScale, onFrameAnnotate, sendFrameCommand } from "./page-frame";
import { useBrowserPrefsStore } from "./prefs-store";
import { useBrowserStore } from "./store";
// The blocks a click marks whole. Anything else marks the nearest element that holds text itself.
const BLOCK =
"p, li, h1, h2, h3, h4, h5, h6, pre, blockquote, td, th, dt, dd, figcaption, caption, img";
const NESTED_BLOCK = "ul, ol, table, pre, blockquote, div";
const MAX_QUOTE_CHARS = 280;
const PAD = 5;
const DRAG_THRESHOLD = 4;
const BUBBLE = 28;
/** A screenshot of the marked page when Settings asks for one; failures just skip it. */
async function annotationScreenshot(page: HTMLElement | null): Promise<File[]> {
if (useBrowserPrefsStore.getState().annotationScreenshots !== "always" || !page || !canScreenshot()) return [];
const { tabs, annotateTabId } = useBrowserStore.getState();
const tab = tabs.find((candidate) => candidate.id === annotateTabId);
if (!tab) return [];
const blob = await screenshotPage(tab, page).catch(() => null);
return blob ? [new File([blob], "Annotated page.png", { type: "image/png" })] : [];
}
type Annotation = {
id: number;
ranges: Range[];
places?: (Place | null)[];
quote: string;
frame?: Frame;
request: string;
};
/** A drag's box, offset from its content so it scrolls with it, at the zoom and page width it was drawn at. */
type Frame = { left: number; top: number; width: number; height: number; zoom: number; item?: number };
type Pending = { id: number | null; ranges: Range[]; places?: (Place | null)[]; quote: string; frame?: Frame };
/** Where a range sits in a virtualized page: its text offsets, or the path to the element it selects. */
type Place = { item: string; start: number; end: number; text: string } | { item: string; path: number[]; tag: string };
type Box = { left: number; top: number; width: number; height: number };
function hasOwnText(element: Element): boolean {
return [...element.childNodes].some(
(node) => node.nodeType === Node.TEXT_NODE && node.textContent?.trim(),
);
}
function pdfLine(span: Element): Range[] {
const layer = span.closest(".textLayer");
if (!layer) return [];
const target = span.getBoundingClientRect();
const middle = target.top + target.height / 2;
return [...layer.querySelectorAll("span")]
.filter((candidate) => {
if (candidate.querySelector("span") || !candidate.textContent?.trim())
return false;
const rect = candidate.getBoundingClientRect();
return (
Math.abs(rect.top + rect.height / 2 - middle) <
Math.max(target.height, rect.height) * 0.6
);
})
.map((candidate) => {
const range = document.createRange();
range.selectNodeContents(candidate);
return range;
});
}
function blockRange(block: Element): Range {
const range = document.createRange();
if (block.tagName === "IMG") {
range.selectNode(block);
return range;
}
range.selectNodeContents(block);
if (block.tagName === "LI") {
const nested = [...block.children].find((child) =>
child.matches(NESTED_BLOCK),
);
if (nested) range.setEndBefore(nested);
}
return range;
}
function blockAt(target: Element, root: Element): Range[] | null {
if (target.closest(".textLayer")) {
const line = pdfLine(target);
return line.length > 0 ? line : null;
}
let block = target.closest(BLOCK);
if (block && !root.contains(block)) block = null;
if (!block) {
let element: Element | null = target;
while (element && element !== root && !hasOwnText(element))
element = element.parentElement;
if (!element || element === root) return null;
block = element;
}
return [blockRange(block)];
}
const overlaps = (a: DOMRect, b: DOMRect) =>
a.width > 0 &&
a.height > 0 &&
a.left < b.right &&
a.right > b.left &&
a.top < b.bottom &&
a.bottom > b.top;
function blocksIn(root: Element, area: DOMRect): Range[] {
const candidates = [
...root.querySelectorAll(BLOCK),
...root.querySelectorAll(".textLayer span"),
].filter(
(element) =>
!element.closest("[data-annotate-ui]") &&
(element.closest(".textLayer")
? !element.querySelector("span") && element.textContent?.trim()
: true) &&
overlaps(element.getBoundingClientRect(), area),
);
const found = candidates.flatMap((element) => {
const range = blockRange(element);
return [...range.getClientRects()].some((rect) => overlaps(rect, area))
? [{ element, range }]
: [];
});
// Drop blocks containing another hit (list items already stop before nested lists).
return found
.filter(
({ element, range }) =>
!found.some(
(other) =>
other.element !== element &&
element.contains(other.element) &&
range.intersectsNode(other.element),
),
)
.map(({ range }) => range);
}
function quoteOf(ranges: Range[]): string {
const text = ranges
.map((range) => range.toString())
.join(" ")
.replace(/\s+/g, " ")
.trim();
return text.length > MAX_QUOTE_CHARS
? `${text.slice(0, MAX_QUOTE_CHARS - 1)}…`
: text;
}
function boxOf(ranges: Range[], origin: DOMRect): Box | null {
let left = Number.POSITIVE_INFINITY;
let top = Number.POSITIVE_INFINITY;
let right = Number.NEGATIVE_INFINITY;
let bottom = Number.NEGATIVE_INFINITY;
for (const range of ranges) {
// A range whose nodes left the page (a virtualized page scrolled away) measures empty.
for (const rect of range.getClientRects()) {
if (rect.width === 0 && rect.height === 0) continue;
left = Math.min(left, rect.left);
top = Math.min(top, rect.top);
right = Math.max(right, rect.right);
bottom = Math.max(bottom, rect.bottom);
}
}
if (!Number.isFinite(left)) return null;
return {
left: left - origin.left - PAD,
top: top - origin.top - PAD,
width: right - left + PAD * 2,
height: bottom - top + PAD * 2,
};
}
/** Effective CSS zoom of an element. */
function cssZoomOf(element: Element | null): number {
if (!element) return 1;
// Typed as always present, but older engines lack it.
const reported: unknown = element.currentCSSZoom;
if (typeof reported === "number") return reported;
// Older engines: multiply each ancestor's own zoom.
let zoom = 1;
for (let el: Element | null = element; el; el = el.parentElement) {
zoom *= Number.parseFloat(getComputedStyle(el).zoom) || 1;
}
return zoom;
}
/** Effective CSS zoom at the content. */
function zoomAt(ranges: Range[]): number {
const range = ranges[0];
if (!range) return 1;
const node = range.startContainer;
// A selected node (an image) can carry its own zoom; its range starts at the parent.
const selected = node.childNodes[range.startOffset];
const element =
selected instanceof Element && range.endContainer === node && range.endOffset === range.startOffset + 1
? selected
: node instanceof Element
? node
: node.parentElement;
return cssZoomOf(element);
}
// PDF pages and slides: virtualized, and remounted at a new width on zoom.
const ITEM = "[data-index]";
const itemOf = (node: Node): HTMLElement | null =>
(node instanceof Element ? node : node.parentElement)?.closest<HTMLElement>(ITEM) ?? null;
/** A page's text nodes and where each starts, built once and shared by all its ranges. */
type TextIndex = { nodes: Text[]; starts: number[]; at: Map<Node, number>; length: number };
function textIndex(item: Element): TextIndex {
const nodes: Text[] = [];
const starts: number[] = [];
const at = new Map<Node, number>();
let length = 0;
const walker = document.createTreeWalker(item, NodeFilter.SHOW_TEXT);
for (let node = walker.nextNode(); node; node = walker.nextNode()) {
nodes.push(node as Text);
starts.push(length);
at.set(node, length);
length += node.textContent?.length ?? 0;
}
return { nodes, starts, at, length };
}
/** Index of the last item `test` holds for, when it holds for a prefix; else -1. */
function lastWhere<T>(items: readonly T[], test: (item: T) => boolean): number {
let low = 0;
let high = items.length;
while (low < high) {
const middle = (low + high) >> 1;
if (test(items[middle] as T)) low = middle + 1;
else high = middle;
}
return low - 1;
}
/** Characters of the page before a boundary point. */
function offsetIn(index: TextIndex, node: Node, offset: number): number {
const start = index.at.get(node);
if (start !== undefined) return start + offset;
// A point beside a text node, as around a span's own text.
const next = node.childNodes[offset];
const after = next && index.at.get(next);
if (after !== undefined) return after;
const prior = node.childNodes[offset - 1];
const before = prior && index.at.get(prior);
if (before !== undefined) return before + (prior.textContent?.length ?? 0);
// Elsewhere between nodes: count the text nodes that start before it.
const point = document.createRange();
point.setStart(node, offset);
const last = lastWhere(index.nodes, (text) => point.comparePoint(text, 0) < 0);
const text = index.nodes[last];
return text ? (index.starts[last] ?? 0) + text.length : 0;
}
/** The text node and offset `offset` characters in; a start between nodes takes the next. */
function pointIn(index: TextIndex, offset: number, start: boolean): [Node, number] | null {
const found = lastWhere(index.starts, (at) => (start ? at <= offset : at < offset));
const node = index.nodes[Math.max(found, 0)];
const local = offset - (index.starts[Math.max(found, 0)] ?? 0);
return node && local >= 0 && local <= node.length ? [node, local] : null;
}
/** The element a range selects whole, as an image's does. */
function selectedElement(range: Range): Element | null {
const node = range.startContainer.childNodes[range.startOffset];
return node instanceof Element && range.endContainer === range.startContainer && range.endOffset === range.startOffset + 1
? node
: null;
}
function placesOf(ranges: Range[]): (Place | null)[] {
const indexes = new Map<Element, TextIndex>();
return ranges.map((range) => {
const item = itemOf(range.startContainer);
if (!item || itemOf(range.endContainer) !== item) return null;
const text = range.toString();
const element = selectedElement(range);
if (!text.trim()) {
if (!element) return null;
const path: number[] = [];
for (let node: Element = element; node !== item; node = node.parentElement!) {
path.unshift([...node.parentElement!.children].indexOf(node));
}
return { item: item.dataset.index ?? "", path, tag: element.tagName };
}
let index = indexes.get(item);
if (!index) indexes.set(item, (index = textIndex(item)));
const start = offsetIn(index, range.startContainer, range.startOffset);
const end = offsetIn(index, range.endContainer, range.endOffset);
return { item: item.dataset.index ?? "", start, end, text };
});
}
/** The place's content in its remounted page, if that page shows the same thing. */
function rangeAt(place: Place, page: HTMLElement, indexes: Map<Element, TextIndex>): Range | null {
// Hidden tabs keep their documents, with the same page numbers: only a shown page counts.
const item = [...page.querySelectorAll(`${ITEM}[data-index="${CSS.escape(place.item)}"]`)].find(
(candidate) => candidate.getClientRects().length > 0,
);
if (!item) return null;
const range = document.createRange();
if ("path" in place) {
let element: Element | undefined = item;
for (const step of place.path) element = element?.children[step];
if (element?.tagName !== place.tag) return null;
range.selectNode(element);
return range;
}
let index = indexes.get(item);
if (!index) indexes.set(item, (index = textIndex(item)));
const start = pointIn(index, place.start, true);
const end = pointIn(index, place.end, false);
if (!start || !end) return null;
range.setStart(...start);
range.setEnd(...end);
return range.toString() === place.text ? range : null;
}
const rebound = new WeakMap<Range, Range>();
/** Whether a range still holds what was marked; removed nodes collapse it onto what is left. */
function holds(range: Range, place: Place): boolean {
if (!("path" in place)) return range.toString() === place.text;
return selectedElement(range)?.tagName === place.tag && range.startContainer.isConnected;
}
/** The ranges, found again where their page remounted; `indexes` is shared by every mark in a render. */
function liveRanges(
mark: { ranges: Range[]; places?: (Place | null)[] },
page: HTMLElement,
indexes: Map<Element, TextIndex>,
): Range[] {
return mark.ranges.map((range, index) => {
const place = mark.places?.[index];
if (!place || holds(range, place)) return range;
const known = rebound.get(range);
if (known && holds(known, place)) return known;
const found = rangeAt(place, page, indexes);
if (found) rebound.set(range, found);
return found ?? range;
});
}
/** A drag's box if any, else the box around the content. */
function markBoxOf(ranges: Range[], frame: Frame | undefined, origin: DOMRect): Box | null {
const content = boxOf(ranges, origin);
if (!content || !frame) return content;
// Scale by the zoom and page width change since the drag, as the content did.
const item = ranges[0] && frame.item ? itemOf(ranges[0].startContainer) : null;
const k = (zoomAt(ranges) / frame.zoom) * (item && frame.item ? item.offsetWidth / frame.item : 1);
return {
left: content.left + PAD + frame.left * k,
top: content.top + PAD + frame.top * k,
width: frame.width * k,
height: frame.height * k,
};
}
const sameRanges = (a: Range[] | null, b: Range[] | null) =>
a !== null &&
b !== null &&
a.length === b.length &&
a.every(
(range, index) =>
range.startContainer === b[index]?.startContainer &&
range.startOffset === b[index]?.startOffset &&
range.endContainer === b[index]?.endContainer &&
range.endOffset === b[index]?.endOffset,
);
/** Request edits on a file or a Studio page: click marks a block, drag marks an area; Send posts all as one message. */
export function AnnotateLayer({
page,
fileName,
url,
zoom = 1,
}: { page: HTMLElement; fileName: string; url?: string; zoom?: number }) {
const t = useT();
const layerRef = useRef<HTMLDivElement | null>(null);
const cursorRef = useRef<HTMLDivElement | null>(null);
const inputRef = useRef<HTMLInputElement | null>(null);
const lastHoverBox = useRef<Box | null>(null);
const [area, setArea] = useState<Box | null>(null);
const nextId = useRef(1);
const [hover, setHover] = useState<Range[] | null>(null);
const [items, setItems] = useState<Annotation[]>([]);
const [pending, setPending] = useState<Pending | null>(null);
const [draft, setDraft] = useState("");
const [sending, setSending] = useState(false);
const [, setFrame] = useState(0);
const { setAnnotating, sendAnnotations } = useBrowserStore.getState();
const exit = () => setAnnotating(null);
const committed = (): Annotation[] => {
if (!pending) return items;
const request = draft.trim();
if (pending.id === null) {
if (!request) return items;
return [
...items,
{
id: nextId.current++,
ranges: pending.ranges,
places: pending.places,
quote: pending.quote,
frame: pending.frame,
request,
},
];
}
return request
? items.map((item) => (item.id === pending.id ? { ...item, request } : item))
: items.filter((item) => item.id !== pending.id);
};
const save = () => {
if (!pending) return;
setItems(committed());
setPending(null);
setDraft("");
};
const saveRef = useRef(save);
saveRef.current = save;
const send = async () => {
const outgoing = committed();
if (outgoing.length === 0 || !sendAnnotations || sending) return;
// One at a time: a second click while staging would add the annotations twice.
setSending(true);
const files = await annotationScreenshot(page);
const sent = await sendAnnotations(
{ file: fileName, url, items: outgoing.map(({ quote, request }) => ({ quote, request })) },
files,
).finally(() => setSending(false));
if (sent) exit();
};
useEffect(() => {
let frame = 0;
let settle = 0;
const redraw = () => {
layerRef.current?.setAttribute("data-scrolling", "");
window.clearTimeout(settle);
settle = window.setTimeout(
() => layerRef.current?.removeAttribute("data-scrolling"),
150,
);
cancelAnimationFrame(frame);
frame = requestAnimationFrame(() => setFrame((value) => value + 1));
};
// A zoom that transitions settles when it ends; an instant one is redrawn by the [zoom] effect below.
const zoomed = (event: TransitionEvent) => event.propertyName === "zoom" && redraw();
page.addEventListener("scroll", redraw, { capture: true, passive: true });
page.addEventListener("transitionend", zoomed, true);
const resize = new ResizeObserver(redraw);
resize.observe(page);
// A remounted page renders its text later; marks are found again then.
const added = new MutationObserver((records) => {
if (records.some((record) => record.addedNodes.length && !layerRef.current?.contains(record.target))) redraw();
});
added.observe(page, { childList: true, subtree: true });
return () => {
cancelAnimationFrame(frame);
window.clearTimeout(settle);
page.removeEventListener("scroll", redraw, { capture: true });
page.removeEventListener("transitionend", zoomed, true);
resize.disconnect();
added.disconnect();
};
}, [page]);
// A zoom resizes the content after this render measured it, and fires no scroll or resize.
useEffect(() => {
const frame = requestAnimationFrame(() => setFrame((value) => value + 1));
return () => cancelAnimationFrame(frame);
}, [zoom]);
// Capture phase, so the page's own links and click handlers never see a marking press.
useEffect(() => {
const ownUi = (target: EventTarget | null) =>
target instanceof Element && target.closest("[data-annotate-ui]");
let press: { x: number; y: number; dragging: boolean } | null = null;
const moveBubble = (event: PointerEvent | null) => {
const bubble = cursorRef.current;
const layer = layerRef.current;
if (!bubble || !layer) return;
if (!event || ownUi(event.target)) {
bubble.dataset.shown = "false";
return;
}
const origin = layer.getBoundingClientRect();
bubble.style.transform = `translate(${event.clientX - origin.left}px, ${event.clientY - origin.top - BUBBLE}px)`;
bubble.dataset.shown = "true";
};
const areaFrom = (event: PointerEvent): DOMRect | null => {
if (!press) return null;
return new DOMRect(
Math.min(press.x, event.clientX),
Math.min(press.y, event.clientY),
Math.abs(event.clientX - press.x),
Math.abs(event.clientY - press.y),
);
};
const mark = (ranges: Range[] | null, area?: DOMRect) => {
if (!ranges || ranges.length === 0) return;
const image = ranges.some((range) =>
range.cloneContents().querySelector("img"),
);
const quote =
quoteOf(ranges) || (image ? t("browser.annotate.imageQuote") : "");
if (!quote) return;
// Keep a drag's box as drawn, not shrunk to its text.
const content = area ? boxOf(ranges, new DOMRect()) : null;
const frame = area && content
? {
left: area.left - content.left - PAD,
top: area.top - content.top - PAD,
width: area.width,
height: area.height,
zoom: zoomAt(ranges),
item: itemOf(ranges[0]!.startContainer)?.offsetWidth,
}
: undefined;
setPending({ id: null, ranges, places: placesOf(ranges), quote, frame });
setDraft("");
};
const onDown = (event: PointerEvent) => {
if (event.button !== 0 || ownUi(event.target)) return;
event.preventDefault();
press = { x: event.clientX, y: event.clientY, dragging: false };
};
const handleMove = (event: PointerEvent) => {
moveBubble(event);
if (press) {
if (
!press.dragging &&
Math.hypot(event.clientX - press.x, event.clientY - press.y) <
DRAG_THRESHOLD
)
return;
press.dragging = true;
const rect = areaFrom(event);
const origin = layerRef.current?.getBoundingClientRect();
if (rect && origin) {
setHover(null);
setArea({
left: rect.left - origin.left,
top: rect.top - origin.top,
width: rect.width,
height: rect.height,
});
}
return;
}
if (ownUi(event.target) || !(event.target instanceof Element)) return;
const next = blockAt(event.target, page);
setHover((current) => (sameRanges(current, next) ? current : next));
};
let moveFrame = 0;
let lastMove: PointerEvent | null = null;
const onMove = (event: PointerEvent) => {
lastMove = event;
if (moveFrame) return;
moveFrame = requestAnimationFrame(() => {
moveFrame = 0;
const latest = lastMove;
lastMove = null;
if (latest) handleMove(latest);
});
};
const cancelMove = () => {
cancelAnimationFrame(moveFrame);
moveFrame = 0;
lastMove = null;
};
const onUp = (event: PointerEvent) => {
cancelMove();
if (!press) return;
const { dragging } = press;
const rect = areaFrom(event);
press = null;
setArea(null);
saveRef.current();
if (dragging && rect) mark(blocksIn(page, rect), rect);
else if (event.target instanceof Element && !ownUi(event.target))
mark(blockAt(event.target, page));
};
const onLeave = () => {
cancelMove();
setHover(null);
moveBubble(null);
};
const onClick = (event: MouseEvent) => {
if (ownUi(event.target)) return;
event.preventDefault();
event.stopPropagation();
};
page.classList.add("browser-annotating");
page.addEventListener("pointerdown", onDown, true);
page.addEventListener("pointermove", onMove, true);
window.addEventListener("pointerup", onUp, true);
page.addEventListener("pointerleave", onLeave);
page.addEventListener("click", onClick, true);
return () => {
cancelMove();
page.classList.remove("browser-annotating");
page.removeEventListener("pointerdown", onDown, true);
page.removeEventListener("pointermove", onMove, true);
window.removeEventListener("pointerup", onUp, true);
page.removeEventListener("pointerleave", onLeave);
page.removeEventListener("click", onClick, true);
};
}, [page, t]);
useEffect(() => {
const onKeyDown = (event: KeyboardEvent) => {
if (event.key !== "Escape") return;
event.preventDefault();
if (pending) {
setPending(null);
setDraft("");
} else exit();
};
window.addEventListener("keydown", onKeyDown);
return () => window.removeEventListener("keydown", onKeyDown);
});
useLayoutEffect(() => {
if (pending) inputRef.current?.focus();
}, [pending]);
const origin = layerRef.current?.getBoundingClientRect() ?? new DOMRect();
const hoverBox = hover ? boxOf(hover, origin) : null;
if (hoverBox) lastHoverBox.current = hoverBox;
const shownHover = hoverBox ?? lastHoverBox.current;
const indexes = new Map<Element, TextIndex>();
const pendingBox = pending ? markBoxOf(liveRanges(pending, page, indexes), pending.frame, origin) : null;
const count = items.length;
// A first comment still being typed can go too: Send commits it.
const canSend = count > 0 || (pending?.id === null && draft.trim() !== "");
const shown = count + (pending?.id === null && draft.trim() !== "" ? 1 : 0);
return (
<div
ref={layerRef}
className="group/annotate pointer-events-none absolute inset-0 z-10 overflow-hidden"
>
{shownHover ? (
<div
className={cn(
"absolute rounded-md border-[1.5px] border-wash-70 transition-[left,top,width,height,opacity] duration-150 ease-out group-data-[scrolling]/annotate:transition-none motion-reduce:transition-none",
hoverBox && !area ? "opacity-100" : "opacity-0",
)}
style={shownHover}
/>
) : null}
{area ? (
<div
className="absolute rounded-md border-[1.5px] border-dashed border-wash-70 bg-wash-5"
style={area}
/>
) : null}
{items.map((item, index) => {
const box = item.id === pending?.id ? null : markBoxOf(liveRanges(item, page, indexes), item.frame, origin);
return box ? (
<Mark
key={item.id}
box={box}
number={index + 1}
onOpen={() => {
saveRef.current();
setPending({
id: item.id,
ranges: item.ranges,
places: item.places,
quote: item.quote,
frame: item.frame,
});
setDraft(item.request);
}}
label={item.request}
/>
) : null;
})}
{pending && pendingBox ? (
<>
<Mark box={pendingBox} label={draft} number={markNumber(items, pending.id)} />
<CommentForm
inputRef={inputRef}
box={pendingBox}
layer={origin}
draft={draft}
onDraft={setDraft}
onSave={save}
/>
</>
) : null}
{shown > 0 ? (
<AnnotateBar
count={shown}
canSend={canSend}
sendDisabled={!sendAnnotations || sending}
onSend={send}
onExit={exit}
/>
) : null}
<div ref={cursorRef} aria-hidden={true} data-shown="false" className={CURSOR} />
</div>
);
}
function Mark({
box,
label,
number,
onOpen,
}: { box: Box; label: string; number: number; onOpen?: () => void }) {
return (
<>
<div
className="absolute animate-in rounded-md border-[1.5px] border-dashed border-wash-60 bg-wash-5 fade-in-0 duration-150"
style={box}
/>
<button
type="button"
data-annotate-ui=""
aria-label={label}
title={label}
onClick={onOpen}
disabled={!onOpen}
className="pointer-events-auto absolute flex size-8 animate-in cursor-pointer items-center justify-center rounded-full rounded-bl-[4px] bg-primary text-ui-13 font-semibold tabular-nums text-primary-foreground shadow-md ring-2 ring-background transition-transform fade-in-0 zoom-in-50 duration-150 hover:scale-110 disabled:cursor-default"
style={{ left: box.left + box.width - 14, top: box.top - 18 }}
>
{number}
</button>
</>
);
}
const CURSOR =
"absolute top-0 left-0 size-7 rounded-full rounded-bl-[4px] bg-primary shadow-md ring-2 ring-background transition-[opacity,scale] duration-100 data-[shown=false]:scale-50 data-[shown=false]:opacity-0 motion-reduce:transition-none";
function markNumber<Item extends { id: number }>(items: Item[], id: number | null): number {
const index = id === null ? -1 : items.findIndex((item) => item.id === id);
return index === -1 ? items.length + 1 : index + 1;
}
// Deep shadow and faint rim so it stands out on dark pages.
const SURFACE =
"border border-border bg-background text-foreground shadow-[0_8px_28px_-6px_rgba(0,0,0,0.18)] dark:border-white/10 dark:bg-neutral-800 dark:text-white dark:shadow-[0_12px_40px_-4px_rgba(0,0,0,0.85),0_4px_12px_-2px_rgba(0,0,0,0.6)]";
function useCommentDictation(draft: string, onDraft: (value: string) => void) {
const t = useT();
const session = useRef<ReturnType<StudioDictationAdapter["listen"]> | null>(null);
const [listening, setListening] = useState(false);
useEffect(() => () => session.current?.cancel(), []);
const start = () => {
if (!isStudioDictationAvailable()) {
notifyStudioDictationUnavailable();
return;
}
let base = draft;
const joined = (text: string) =>
base && text && !base.endsWith(" ") ? `${base} ${text}` : base + text;
try {
const next = new StudioDictationAdapter({ chatId: null }).listen();
session.current = next;
setListening(true);
next.onSpeech((result) => {
if (session.current !== next) return;
const text = joined(result.transcript);
if (result.isFinal !== false) base = text;
onDraft(text);
});
const end = () => {
if (session.current !== next) return;
session.current = null;
setListening(false);
};
next.onSpeechEnd(end);
next.onEnd?.(end);
} catch (error) {
session.current = null;
setListening(false);
toast.error(t("browser.annotate.dictateFailed"), {
description: error instanceof Error ? error.message : undefined,
});
}
};
const stop = () => void session.current?.stop();
return { listening, toggle: () => (listening ? stop() : start()) };
}
function CommentForm({
inputRef,
box,
layer,
draft,
onDraft,
onSave,
placeholder,
}: {
placeholder?: string;
inputRef: RefObject<HTMLInputElement | null>;
box: Box;
layer: { width: number; height: number };
draft: string;
onDraft: (value: string) => void;
onSave: () => void;
}) {
const t = useT();
const dictation = useCommentDictation(draft, onDraft);
const width = Math.min(448, Math.max(240, layer.width - 24));
const top =
box.top + box.height + 8 + 48 > layer.height
? Math.max(8, box.top - 56)
: box.top + box.height + 8;
const left = Math.min(Math.max(12, box.left + box.width - width), layer.width - width - 12);
const micLabel = t(dictation.listening ? "browser.annotate.stopDictating" : "browser.annotate.dictate");
// The tick replaces the microphone unless it is listening, so dictation can always be stopped.
const canSave = Boolean(draft.trim()) && !dictation.listening;
return (
<form
data-annotate-ui=""
data-annotate-chrome=""
data-native-cover=""
onSubmit={(event) => {
event.preventDefault();
onSave();
}}
className={cn(
"pointer-events-auto absolute flex h-12 animate-in items-center gap-2 rounded-full pr-1.5 pl-5 fade-in-0 zoom-in-95 duration-150",
SURFACE,
)}
style={{ top, left, width }}
>
<input
ref={inputRef}
value={draft}
onChange={(event) => onDraft(event.target.value)}
placeholder={placeholder ?? t("browser.annotate.placeholder")}
aria-label={placeholder ?? t("browser.annotate.placeholder")}
className="min-w-0 flex-1 bg-transparent text-ui-15 outline-none placeholder:text-muted-foreground dark:placeholder:text-neutral-400"
/>
{canSave ? (
<button
type="submit"
aria-label={t("browser.annotate.save")}
title={t("browser.annotate.save")}
className="flex size-9 shrink-0 animate-in cursor-pointer items-center justify-center rounded-full bg-action text-background transition-opacity fade-in-0 zoom-in-75 duration-150 hover:opacity-90"
>
<HugeiconsIcon icon={Tick02Icon} strokeWidth={2} className="size-4.5" />
</button>
) : (
<button
type="button"
aria-label={micLabel}
title={micLabel}
aria-pressed={dictation.listening}
onClick={() => {
dictation.toggle();
inputRef.current?.focus();
}}
className={cn(
"flex size-9 shrink-0 cursor-pointer items-center justify-center rounded-full text-muted-foreground transition-colors hover:bg-muted hover:text-foreground dark:text-neutral-300 dark:hover:bg-neutral-700 dark:hover:text-white",
dictation.listening &&
"animate-pulse bg-primary text-primary-foreground hover:bg-primary hover:text-primary-foreground dark:bg-primary dark:text-primary-foreground dark:hover:bg-primary dark:hover:text-primary-foreground",
)}
>
<MicIcon className="size-4.5" />
</button>
)}
</form>
);
}
function AnnotateBar({
count,
canSend,
sendDisabled,
onSend,
onExit,
besidePage = false,
}: {
count: number;
canSend: boolean;
sendDisabled: boolean;
onSend: () => void;
onExit: () => void;
/** Over a native view: the page ends above the bar, so it only slides sideways. */
besidePage?: boolean;
}) {
const t = useT();
const [offset, setOffset] = useState({ x: 0, y: 0 });
const barRef = useRef<HTMLDivElement | null>(null);
const drag = useRef<{ x: number; y: number; id: number } | null>(null);
// Captured, so the drag keeps going over a web page's frame.
const startDrag = (event: ReactPointerEvent<HTMLButtonElement>) => {
event.preventDefault();
event.currentTarget.setPointerCapture(event.pointerId);
drag.current = { x: event.clientX - offset.x, y: event.clientY - offset.y, id: event.pointerId };
};
const moveDrag = (event: ReactPointerEvent<HTMLButtonElement>) => {
const start = drag.current;
const bar = barRef.current;
const layer = bar?.parentElement;
if (!start || start.id !== event.pointerId || !bar || !layer) return;
const bounds = layer.getBoundingClientRect();
const width = bar.offsetWidth;
const height = bar.offsetHeight;
const maxX = Math.max(0, (bounds.width - width) / 2 - 8);
const minY = -(bounds.height - height - 20 - 8);
setOffset({
x: Math.min(maxX, Math.max(-maxX, event.clientX - start.x)),
y: besidePage ? 0 : Math.min(12, Math.max(minY, event.clientY - start.y)),
});
};
const endDrag = (event: ReactPointerEvent<HTMLButtonElement>) => {
if (drag.current?.id === event.pointerId) drag.current = null;
};
return (
<div
ref={barRef}
data-annotate-ui=""
data-annotate-chrome=""
data-native-inset={besidePage ? "" : undefined}
className={cn(
"pointer-events-auto absolute bottom-5 left-1/2 flex h-12 items-center gap-1 rounded-full pr-1.5 pl-2.5 text-ui-15",
SURFACE,
)}
style={{
transform: `translate(calc(-50% + ${offset.x}px), ${offset.y}px)`,
}}
>
<button
type="button"
aria-label={t("browser.annotate.move")}
onPointerDown={startDrag}
onPointerMove={moveDrag}
onPointerUp={endDrag}
onPointerCancel={endDrag}
className="flex h-9 w-7 cursor-grab touch-none items-center justify-center text-muted-foreground active:cursor-grabbing dark:text-neutral-400"
>
<HugeiconsIcon icon={DragDropVerticalIcon} strokeWidth={2} className="size-4.5" />
</button>
<span className="whitespace-nowrap px-2">
{t(count === 1 ? "browser.annotate.countOne" : "browser.annotate.countMany", { count })}
</span>
{canSend ? <span aria-hidden={true} className="mx-1 h-5 w-px bg-border dark:bg-neutral-600" /> : null}
<button
type="button"
onClick={onExit}
className="h-9 cursor-pointer whitespace-nowrap rounded-full px-3 transition-colors hover:bg-muted dark:hover:bg-neutral-700"
>
{t("browser.annotate.cancel")}
</button>
{canSend ? (
<button
type="button"
onClick={onSend}
disabled={sendDisabled}
className="cursor-pointer whitespace-nowrap rounded-full bg-action px-3 py-2 font-medium text-background transition-opacity hover:opacity-90 disabled:cursor-default disabled:opacity-50"
>
{t("browser.annotate.send")}
</button>
) : null}
</div>
);
}
type WebMark = { id: number; quote: string; request: string };
type WebPending = { id: number; quote: string; saved: boolean };
function accentColor(): string {
const probe = document.createElement("span");
probe.style.color = "var(--primary)";
document.body.append(probe);
const value = getComputedStyle(probe).color;
probe.remove();
// Computed colours can come back as oklch(); a canvas reads any of them out as RGB.
const context = document.createElement("canvas").getContext("2d");
if (!context) return value;
context.fillStyle = value;
context.fillRect(0, 0, 1, 1);
const [r, g, b] = context.getImageData(0, 0, 1, 1).data;
return `rgb(${r}, ${g}, ${b})`;
}
/** Ask about a web page: the page's own script tracks the pointer and draws marks (`annotation` in routes/browser.py); this layer holds the comments and the bar.
* Framed pages are driven through their shell, native views through `startNativeAnnotate`. */
export function WebAnnotateLayer({
tabId,
title,
url,
page,
native = false,
}: { tabId: string; title: string; url?: string; page: HTMLElement | null; native?: boolean }) {
const t = useT();
const layerRef = useRef<HTMLDivElement | null>(null);
const inputRef = useRef<HTMLInputElement | null>(null);
// The tab this layer annotates while mounted; a code fetch finishing after it left sends nothing.
const liveTab = useRef<string | null>(null);
const [rects, setRects] = useState<ReadonlyMap<number, AnnotateRect | null>>(new Map());
const [items, setItems] = useState<WebMark[]>([]);
const [pending, setPending] = useState<WebPending | null>(null);
const [draft, setDraft] = useState("");
const [sending, setSending] = useState(false);
const [, setLayout] = useState(0);
const zoom = useBrowserStore((state) => state.tabs.find((tab) => tab.id === tabId)?.zoom ?? 1);
const { setAnnotating, sendAnnotations } = useBrowserStore.getState();
const nativeSend = useRef<((command: FrameCommand) => void) | null>(null);
const exit = () => setAnnotating(null);
const command = (next: FrameCommand) =>
native ? nativeSend.current?.(next) : sendFrameCommand(tabId, next);
const forget = (id: number) => command({ command: "annotateForget", id });
// The page gets the annotate code only now (each document once; repeats are ignored there).
const start = () =>
void annotateScript().then(
(code) => {
if (liveTab.current !== tabId) return;
command({ command: "annotateInstall", code });
command({ command: "annotate", on: true, color: accentColor() });
},
() => liveTab.current === tabId && exit(),
);
const committed = (): WebMark[] => {
if (!pending) return items;
const request = draft.trim();
if (!pending.saved) {
if (!request) return items;
return [...items, { id: pending.id, quote: pending.quote, request }];
}
return request
? items.map((item) => (item.id === pending.id ? { ...item, request } : item))
: items.filter((item) => item.id !== pending.id);
};
const save = () => {
if (!pending) return;
const next = committed();
if (!next.some((item) => item.id === pending.id)) forget(pending.id);
setItems(next);
setPending(null);
setDraft("");
};
const discard = () => {
if (pending && !pending.saved) forget(pending.id);
setPending(null);
setDraft("");
};
const send = async () => {
const outgoing = committed();
if (outgoing.length === 0 || !sendAnnotations || sending) return;
setSending(true);
// Keep the open comment and close its form, so a native view shows again for the screenshot.
setItems(outgoing);
setPending(null);
setDraft("");
const files = await annotationScreenshot(page);
const sent = await sendAnnotations(
{ file: title || url || "", url, items: outgoing.map(({ quote, request }) => ({ quote, request })) },
files,
).finally(() => setSending(false));
if (sent) exit();
};
// A press in a native view keeps key focus there: move it back to the comment.
const takeKeys = () => {
if (native) void focusPanel(tabId).then(() => inputRef.current?.focus());
};
// Read on each report, so one always sees this render's state.
const handle = (event: AnnotateEvent) => {
switch (event.kind) {
case "ready":
// New document (a native view navigated in place): drop the old marks.
setItems([]);
setPending(null);
setDraft("");
setRects(new Map());
start();
break;
case "up":
save();
break;
case "escape":
if (pending) discard();
else exit();
break;
case "open": {
const item = items.find((entry) => entry.id === event.id);
if (!item) break;
setPending({ id: item.id, quote: item.quote, saved: true });
setDraft(item.request);
takeKeys();
break;
}
case "mark": {
const quote =
event.quote ||
(event.image
? event.alt || t("browser.annotate.imageQuote")
: event.area
? t("browser.annotate.areaQuote")
: "");
// The page can post marks itself: cap both counts, as it can also empty its rects report.
const full = !rects.has(event.id) && (rects.size >= MAX_MARKS || items.length >= MAX_MARKS);
if (!quote || full) {
forget(event.id);
break;
}
setRects((current) => new Map(current).set(event.id, event.rect));
setPending({ id: event.id, quote, saved: false });
setDraft("");
takeKeys();
break;
}
case "rects":
setRects(new Map(event.rects));
break;
}
};
const handleRef = useRef(handle);
handleRef.current = handle;
const startRef = useRef(start);
startRef.current = start;
useEffect(() => {
const listener = (event: AnnotateEvent) => handleRef.current(event);
let stop: () => void;
if (native) {
// One poll can carry several reports: render between them so each sees the last one's state.
const channel = startNativeAnnotate(tabId, (event) => flushSync(() => listener(event)));
nativeSend.current = channel.send;
stop = () => {
nativeSend.current = null;
channel.stop();
};
} else {
const unlisten = onFrameAnnotate(tabId, listener);
stop = () => {
unlisten();
sendFrameCommand(tabId, { command: "annotate", on: false });
};
}
liveTab.current = tabId;
// Now, for a page already loaded; a page still loading asks with "ready".
startRef.current();
return () => {
liveTab.current = null;
stop();
};
}, [tabId, native]);
useEffect(() => {
const layer = layerRef.current;
if (!layer) return;
const observer = new ResizeObserver(() => setLayout((value) => value + 1));
observer.observe(layer);
return () => observer.disconnect();
}, []);
useEffect(() => {
const onKeyDown = (event: KeyboardEvent) => {
if (event.key !== "Escape") return;
event.preventDefault();
if (pending) discard();
else exit();
};
window.addEventListener("keydown", onKeyDown);
return () => window.removeEventListener("keydown", onKeyDown);
});
useLayoutEffect(() => {
if (pending) inputRef.current?.focus();
}, [pending]);
useEffect(() => {
const numbers: Array<[number, number]> = items.map((item, index) => [item.id, index + 1]);
if (pending && !pending.saved) numbers.push([pending.id, items.length + 1]);
command({ command: "annotateNumbers", numbers });
}, [tabId, items, pending]);
const origin = layerRef.current?.getBoundingClientRect() ?? new DOMRect();
// Native views report CSS pixels; scale by their zoom.
const bounds = native ? nativeViewBounds(tabId) : null;
const frame = bounds ? { left: bounds.x, top: bounds.y } : (frameRect(tabId) ?? origin);
const scale = bounds ? zoom : frameScale(tabId);
const rect = pending ? rects.get(pending.id) : null;
const pendingBox: Box | null = rect
? {
left: rect.left * scale + frame.left - origin.left,
top: rect.top * scale + frame.top - origin.top,
width: rect.width * scale,
height: rect.height * scale,
}
: null;
const count = items.length;
const canSend = count > 0 || (pending?.saved === false && draft.trim() !== "");
const shown = count + (pending?.saved === false && draft.trim() !== "" ? 1 : 0);
return (
<div ref={layerRef} className="pointer-events-none absolute inset-0 z-10 overflow-hidden">
{native && pending ? (
// The page is a snapshot while commenting; a press on it saves, as on a live page.
<div
data-annotate-ui=""
data-native-cover=""
aria-hidden={true}
className="pointer-events-auto absolute inset-0"
onPointerDown={save}
/>
) : null}
{pending && pendingBox ? (
<CommentForm
key={pending.id}
inputRef={inputRef}
box={pendingBox}
layer={origin}
draft={draft}
onDraft={setDraft}
onSave={save}
placeholder={t("browser.annotate.pagePlaceholder")}
/>
) : null}
{shown > 0 ? (
<AnnotateBar
count={shown}
canSend={canSend}
sendDisabled={!sendAnnotations || sending}
onSend={send}
onExit={exit}
besidePage={native}
/>
) : null}
</div>
);
}