Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

feat: add persistent clickable annotate overlay for copying element references

Luigi Agosti047ed665f14d
diff --git a/cmd/pinchtab/cmd_cli_browser_inspect.go b/cmd/pinchtab/cmd_cli_browser_inspect.go
index d5710610..a307c2e3 100644
--- a/cmd/pinchtab/cmd_cli_browser_inspect.go
+++ b/cmd/pinchtab/cmd_cli_browser_inspect.go
@@ -42,6 +42,16 @@ var screenshotCmd = &cobra.Command{
},
}
+var annotateCmd = &cobra.Command{
+ Use: "annotate",
+ Short: "Inject a persistent, clickable annotation overlay on the live page (--clear to remove)",
+ Run: func(cmd *cobra.Command, args []string) {
+ runCLI(func(rt cliRuntime) {
+ browseractions.Annotate(rt.client, rt.base, rt.token, cmd)
+ })
+ },
+}
+
var captureCmd = &cobra.Command{
Use: "capture",
Aliases: []string{"cap"},
diff --git a/cmd/pinchtab/cmd_cli_register.go b/cmd/pinchtab/cmd_cli_register.go
index 7e13ea92..40d530f1 100644
--- a/cmd/pinchtab/cmd_cli_register.go
+++ b/cmd/pinchtab/cmd_cli_register.go
@@ -19,7 +19,7 @@ func init() {
func browserRootCommands() []*cobra.Command {
return []*cobra.Command{
quickCmd, navCmd, backCmd, forwardCmd, reloadCmd, snapCmd, frameCmd, clickCmd,
- dblclickCmd, dragCmd, typeCmd, screenshotCmd, captureCmd, tabsCmd, pressCmd, fillCmd,
+ dblclickCmd, dragCmd, typeCmd, screenshotCmd, annotateCmd, captureCmd, tabsCmd, pressCmd, fillCmd,
hoverCmd, mouseCmd, focusCmd, scrollCmd, evalCmd, pdfCmd, textCmd, titleCmd, urlCmd,
htmlCmd, stylesCmd, valueCmd, attrCmd, countCmd, boxCmd, visibleCmd, enabledCmd, checkedCmd,
downloadCmd, uploadCmd, findCmd, selectCmd, checkCmd, uncheckCmd, networkCmd, waitCmd,
@@ -119,6 +119,10 @@ func configureBrowserFlags() {
screenshotCmd.Flags().Bool("css-1x", false, "deprecated: use --scale")
_ = screenshotCmd.Flags().MarkDeprecated("css-1x", "css-1x was removed; use --scale to rescale output")
+ annotateCmd.Flags().StringP("selector", "s", "", "Scope the overlay to elements within this selector (ref/CSS/XPath/text)")
+ annotateCmd.Flags().String("tab", "", "Tab ID")
+ annotateCmd.Flags().Bool("clear", false, "Remove the persistent annotation overlay instead of injecting it")
+
captureCmd.Flags().StringP("output", "o", "", "Save the captured image to this local file path (default: capture-<ts>.jpg)")
captureCmd.Flags().StringP("selector", "s", "", "Scope: clips screenshot and filters snapshot subtree to the same element")
captureCmd.Flags().String("filter", "", "Snapshot filter: 'interactive' or 'all' (default: interactive)")
diff --git a/docs/commands.md b/docs/commands.md
index 35215907..4cc9f36d 100644
--- a/docs/commands.md
+++ b/docs/commands.md
@@ -213,6 +213,10 @@ pinchtab screenshot -q <0-100> # JPEG quality
pinchtab screenshot -s <selector> # Capture a specific element by selector
pinchtab screenshot --scale 0.5 # Half-size output (quarter the pixels)
pinchtab screenshot --beyond-viewport # Capture the full scrollable document (ignored with -s)
+pinchtab screenshot --annotate # Bake numbered ref boxes into the image (for vision models)
+pinchtab annotate # Inject a persistent, clickable overlay on the LIVE page
+pinchtab annotate -s <selector> # Scope the overlay to elements within a selector
+pinchtab annotate --clear # Remove the persistent overlay
pinchtab capture # Paired screenshot + accessibility snapshot from the same DOM epoch
pinchtab capture -o <path> # Save the paired image to a chosen path
pinchtab capture --beyond-viewport # Capture the full document; bounds in page coords
diff --git a/docs/endpoints.md b/docs/endpoints.md
index a03e63e4..997ef127 100644
--- a/docs/endpoints.md
+++ b/docs/endpoints.md
@@ -241,6 +241,8 @@ Find body fields:
```text
GET /screenshot
GET /tabs/{id}/screenshot
+GET /annotate
+GET /tabs/{id}/annotate
GET /capture
GET /tabs/{id}/capture
GET /pdf
@@ -267,6 +269,20 @@ Screenshot query parameters:
- `scale=<float>` — rescale the output bitmap (e.g. `0.5` = half size,
`0.25` = quarter). Default `1`.
+`/annotate` injects a persistent, clickable annotation overlay onto the live
+page — one labelled box per interactive element — and leaves it there (the
+`screenshot?annotate=true` overlay is transient and baked into the image
+instead). Intended for headed browsers: clicking a label copies a reference
+block (page, ref, role, accessible name, CSS selector, XPath) to the clipboard.
+`?clear=true` removes it; `?selector=` scopes it. See
+[Fix your website faster with an LLM](guides/annotate-for-llm-fixes.md).
+
+Annotate query parameters:
+
+- `tabId`
+- `selector` — scope the overlay to elements within this selector
+- `clear=true` — remove the overlay instead of injecting it
+
`/capture` returns a screenshot and an accessibility snapshot from the same
DOM epoch in a single call. It is the vision-grounded alternative to issuing
`/screenshot` and `/snapshot` back-to-back — the two unpaired calls share no
diff --git a/docs/guides/annotate-for-llm-fixes.md b/docs/guides/annotate-for-llm-fixes.md
new file mode 100644
index 00000000..705b88e9
--- /dev/null
+++ b/docs/guides/annotate-for-llm-fixes.md
@@ -0,0 +1,102 @@
+# Fix your website faster with an LLM
+
+When you ask an LLM to "fix this button," it has to *guess* which element you mean from a vague description. That round-trip — "which button?", "the blue one, top right", "I don't see it" — is where most of the time goes.
+
+PinchTab's **annotate** overlay removes the guessing. It draws a labelled box on every interactive element on the live page, and each label is a one-click "copy a precise reference for this element" button. You click the thing you want changed, paste the reference into your LLM chat, and the model knows *exactly* which element you mean — page, ref, role, accessible name, CSS selector, and XPath, with no ambiguity.
+
+This guide shows the full loop.
+
+## The loop
+
+1. **Annotate** the page — draw a labelled box on every interactive element.
+2. **Click** the label on the element you want to change.
+3. **Paste** the copied reference into your LLM chat.
+4. Let the LLM locate and fix the element from an unambiguous selector.
+
+## 1. Open the page in a headed browser
+
+Annotation is a *human-facing* overlay, so run a headed instance you can actually look at. See [Headed mode](./headed-mode.md) for details.
+
+```bash
+# start a visible browser and open your site
+pinchtab instance start --browser cloak --mode headed
+pinchtab --server http://127.0.0.1:9870 nav https://your-site.com
+```
+
+## 2. Annotate
+
+```bash
+pinchtab --server http://127.0.0.1:9870 annotate
+```
+
+Every interactive element gets a pink box labelled with its ref (`e0`, `e1`, …). The command also prints a legend so you can correlate the on-screen labels with roles and text:
+
+```
+Annotated 93 elements — click a label in the browser to copy its reference
+e0 switch "Switch between human and agent mode"
+e4 link "View on GitHub"
+e5 link "Docs"
+…
+```
+
+![Annotated page with a labelled box on every interactive element](../media/annotate-overlay.png)
+
+The overlay is real DOM on the live page — it stays there and scrolls with the content, so you can browse around and pick your target. (It's a one-shot injection: after navigating or a big layout change, run `annotate` again to refresh.)
+
+## 3. Click the label to copy a reference
+
+Click the **label** (the little pink tag, not the box) on the element you want to change. It flashes green to confirm the copy:
+
+![The e5 "Docs" label flipped to a green checkmark after clicking, confirming the reference was copied](../media/annotate-copied.png)
+
+Your clipboard now holds a complete, unambiguous reference:
+
+```
+Page: PinchTab — Browser Control for AI Agents (https://pinchtab.com/)
+Element: e5 — link "Docs"
+CSS: body > main > section:nth-of-type(1) > div:nth-of-type(3) > div:nth-of-type(4) > a:nth-of-type(2)
+XPath: /html[1]/body[1]/main[1]/section[1]/div[3]/div[4]/a[2]
+```
+
+Every field an LLM needs to locate the element in your source is there:
+
+- **Page** — which page/route the element is on.
+- **Element** — the ref plus its role and accessible name ("link, Docs").
+- **CSS** — a unique CSS selector path.
+- **XPath** — an absolute XPath as a fallback.
+
+## 4. Paste into your LLM chat
+
+Now just paste and describe the change:
+
+> Fix this — the link should open in a new tab:
+> ```
+> Page: PinchTab — Browser Control for AI Agents (https://pinchtab.com/)
+> Element: e5 — link "Docs"
+> CSS: body > main > section:nth-of-type(1) > div:nth-of-type(3) > div:nth-of-type(4) > a:nth-of-type(2)
+> XPath: /html[1]/body[1]/main[1]/section[1]/div[3]/div[4]/a[2]
+> ```
+
+The LLM has the accessible name to grep your source ("Docs"), the CSS/XPath to disambiguate which of several similar elements you mean, and the page URL for context. No "which button?" round-trip.
+
+## Clean up
+
+Remove the overlay when you're done:
+
+```bash
+pinchtab --server http://127.0.0.1:9870 annotate --clear
+```
+
+## Notes
+
+- **No `eval` permission required.** The overlay is injected through PinchTab's internal engine, so it works with `security.allowEvaluate: false`.
+- **Copying uses the page's clipboard** (`navigator.clipboard`) on your real click, so it needs a focused, secure-context (HTTPS) page — exactly the case when you're looking at a headed window.
+- **Scope it** with `--selector` to annotate only part of a busy page:
+ ```bash
+ pinchtab --server http://127.0.0.1:9870 annotate --selector "#pricing"
+ ```
+- The refs (`e5`, …) are the same refs the rest of PinchTab uses, so you can also act on them directly: `pinchtab --server http://127.0.0.1:9870 click e5`.
+
+## How it compares to annotated screenshots
+
+`pinchtab screenshot --annotate` bakes the boxes into an *image* and then removes the overlay — great for feeding a screenshot to a vision model. `pinchtab annotate` is the opposite: it *leaves* the boxes on the live page and makes them clickable, for a human driving the fix loop. Use the screenshot form for agents, the `annotate` form for yourself.
diff --git a/docs/index.json b/docs/index.json
index a682e9ba..faf224e8 100644
--- a/docs/index.json
+++ b/docs/index.json
@@ -23,6 +23,7 @@
"guides/data-storage.md",
"guides/docker.md",
"guides/headed-mode.md",
+ "guides/annotate-for-llm-fixes.md",
"guides/raspberry-pi.md"
],
"architecture": [
diff --git a/docs/media/annotate-copied.png b/docs/media/annotate-copied.png
new file mode 100644
index 00000000..4c230325
Binary files /dev/null and b/docs/media/annotate-copied.png differ
diff --git a/docs/media/annotate-overlay.png b/docs/media/annotate-overlay.png
new file mode 100644
index 00000000..0b98a552
Binary files /dev/null and b/docs/media/annotate-overlay.png differ
diff --git a/internal/cdptk/annotate_interactive.go b/internal/cdptk/annotate_interactive.go
new file mode 100644
index 00000000..26f57e5c
--- /dev/null
+++ b/internal/cdptk/annotate_interactive.go
@@ -0,0 +1,213 @@
+package cdptk
+
+import (
+ "context"
+ "encoding/json"
+ "strings"
+
+ "github.com/chromedp/chromedp"
+)
+
+// InteractiveOverlayRootID is the stable DOM id for the persistent, clickable
+// annotation layer. It is intentionally distinct from OverlayRootID so the
+// transient screenshot overlay and the persistent interactive overlay never
+// clobber each other: taking an annotated screenshot must not wipe a
+// human-facing overlay a user injected with `annotate`.
+const InteractiveOverlayRootID = "__pinchtab_annotations_interactive__"
+
+// interactiveOverlayScript builds the persistent overlay. Unlike
+// InjectAnnotationOverlay (which is torn down right after a screenshot), this
+// overlay stays on the page and each label is clickable: clicking copies a
+// reference block for that element to the clipboard so a human can paste it
+// into an LLM ("fix this: <ref>, <css>, ...").
+//
+// {{DATA}} is the JSON item array; {{ROOT}} is the overlay root id. The body is
+// kept free of Go fmt verbs so we can substitute via a plain replacer.
+const interactiveOverlayScript = `(function(items, rootId) {
+ var prev = document.getElementById(rootId);
+ if (prev) prev.remove();
+ var sx = window.scrollX || window.pageXOffset || 0;
+ var sy = window.scrollY || window.pageYOffset || 0;
+ window.__ptLastCopied = null;
+
+ function cssPath(el) {
+ if (!el || el.nodeType !== 1) return '';
+ var parts = [];
+ while (el && el.nodeType === 1 && el !== document.documentElement) {
+ var sel = el.nodeName.toLowerCase();
+ if (el.id) { parts.unshift('#' + CSS.escape(el.id)); break; }
+ var p = el.parentElement;
+ if (p) {
+ var same = Array.prototype.filter.call(p.children, function(c) { return c.nodeName === el.nodeName; });
+ if (same.length > 1) sel += ':nth-of-type(' + (same.indexOf(el) + 1) + ')';
+ }
+ parts.unshift(sel);
+ el = el.parentElement;
+ }
+ return parts.join(' > ');
+ }
+ function xPath(el) {
+ var parts = [];
+ while (el && el.nodeType === 1) {
+ var i = 1, s = el.previousElementSibling;
+ while (s) { if (s.nodeName === el.nodeName) i++; s = s.previousElementSibling; }
+ parts.unshift(el.nodeName.toLowerCase() + '[' + i + ']');
+ el = el.parentElement;
+ }
+ return '/' + parts.join('/');
+ }
+ // resolveEl maps an annotation box back to the live DOM element. Boxes carry
+ // pointer-events:none so elementFromPoint at the box centre passes through to
+ // the underlying element; we then walk up to the ancestor whose rect best
+ // matches the annotation box, since the point may land on a child node.
+ function resolveEl(it) {
+ var cx = it.x + sx - (window.scrollX || window.pageXOffset || 0) + it.w / 2;
+ var cy = it.y + sy - (window.scrollY || window.pageYOffset || 0) + it.h / 2;
+ var el = document.elementFromPoint(cx, cy);
+ if (!el) return null;
+ var best = el, bestDiff = Infinity, node = el;
+ for (var k = 0; k < 6 && node; k++) {
+ var r = node.getBoundingClientRect();
+ var diff = Math.abs(r.width - it.w) + Math.abs(r.height - it.h);
+ if (diff < bestDiff) { bestDiff = diff; best = node; }
+ node = node.parentElement;
+ }
+ return best;
+ }
+ function fallbackCopy(t) {
+ var ta = document.createElement('textarea');
+ ta.value = t; ta.style.position = 'fixed'; ta.style.opacity = '0';
+ document.body.appendChild(ta); ta.focus(); ta.select();
+ try { document.execCommand('copy'); } catch (e) {}
+ ta.remove();
+ return Promise.resolve();
+ }
+ function copyText(t) {
+ window.__ptLastCopied = t;
+ if (navigator.clipboard && navigator.clipboard.writeText) {
+ return navigator.clipboard.writeText(t).catch(function() { return fallbackCopy(t); });
+ }
+ return fallbackCopy(t);
+ }
+ function reference(it) {
+ var el = resolveEl(it);
+ var css = el ? cssPath(el) : '(unresolved)';
+ var xp = el ? xPath(el) : '';
+ var head = it.ref;
+ if (it.role) head += ' — ' + it.role;
+ if (it.name) head += ' "' + it.name + '"';
+ return 'Page: ' + document.title + ' (' + location.href + ')\n' +
+ 'Element: ' + head + '\n' +
+ 'CSS: ' + css + '\n' +
+ 'XPath: ' + xp;
+ }
+
+ var root = document.createElement('div');
+ root.id = rootId;
+ root.style.position = 'absolute';
+ root.style.top = '0';
+ root.style.left = '0';
+ root.style.width = '0';
+ root.style.height = '0';
+ root.style.pointerEvents = 'none';
+ root.style.zIndex = '2147483646';
+
+ items.forEach(function(it) {
+ var box = document.createElement('div');
+ box.style.position = 'absolute';
+ box.style.left = (it.x + sx) + 'px';
+ box.style.top = (it.y + sy) + 'px';
+ box.style.width = it.w + 'px';
+ box.style.height = it.h + 'px';
+ box.style.boxSizing = 'border-box';
+ box.style.border = '2px solid rgba(255, 51, 102, 0.95)';
+ box.style.borderRadius = '2px';
+ box.style.pointerEvents = 'none';
+
+ var label = document.createElement('div');
+ label.textContent = it.ref;
+ label.title = 'Click to copy a reference for ' + it.ref;
+ label.style.position = 'absolute';
+ label.style.left = (it.x + sx) + 'px';
+ label.style.top = Math.max(0, it.y + sy - 16) + 'px';
+ label.style.padding = '0 5px';
+ label.style.background = 'rgba(255, 51, 102, 0.95)';
+ label.style.color = '#fff';
+ label.style.font = '600 12px/16px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace';
+ label.style.borderRadius = '2px 2px 0 0';
+ label.style.whiteSpace = 'nowrap';
+ label.style.userSelect = 'none';
+ label.style.cursor = 'pointer';
+ label.style.pointerEvents = 'auto';
+ label.addEventListener('click', function(ev) {
+ ev.preventDefault();
+ ev.stopPropagation();
+ copyText(reference(it)).then(function() {
+ var orig = label.textContent;
+ label.textContent = '✓ ' + it.ref;
+ label.style.background = 'rgba(40, 200, 90, 0.95)';
+ setTimeout(function() {
+ label.textContent = orig;
+ label.style.background = 'rgba(255, 51, 102, 0.95)';
+ }, 1300);
+ });
+ });
+
+ root.appendChild(box);
+ root.appendChild(label);
+ });
+ document.documentElement.appendChild(root);
+ return items.length;
+})({{DATA}}, "{{ROOT}}")`
+
+// InjectInteractiveOverlay paints a persistent overlay whose labels are
+// clickable. Coordinates are viewport-relative (as produced by
+// AnnotationRectForNode); the page adds scrollX/scrollY so boxes track the
+// document while scrolling. The overlay is NOT auto-removed — callers remove it
+// via RemoveInteractiveOverlay.
+func InjectInteractiveOverlay(ctx context.Context, items []AnnotationItem) error {
+ if len(items) == 0 {
+ return nil
+ }
+ type overlayItem struct {
+ Ref string `json:"ref"`
+ Role string `json:"role,omitempty"`
+ Name string `json:"name,omitempty"`
+ X float64 `json:"x"`
+ Y float64 `json:"y"`
+ W float64 `json:"w"`
+ H float64 `json:"h"`
+ }
+ payload := make([]overlayItem, len(items))
+ for i, it := range items {
+ payload[i] = overlayItem{
+ Ref: it.Ref,
+ Role: it.Role,
+ Name: it.Name,
+ X: it.Box.X,
+ Y: it.Box.Y,
+ W: it.Box.W,
+ H: it.Box.H,
+ }
+ }
+ encoded, err := json.Marshal(payload)
+ if err != nil {
+ return err
+ }
+ script := strings.NewReplacer(
+ "{{DATA}}", string(encoded),
+ "{{ROOT}}", InteractiveOverlayRootID,
+ ).Replace(interactiveOverlayScript)
+ return chromedp.Run(ctx, chromedp.Evaluate(script, nil))
+}
+
+// RemoveInteractiveOverlay removes the persistent overlay if present. Safe to
+// call when no overlay exists.
+func RemoveInteractiveOverlay(ctx context.Context) error {
+ script := `(function(rootId) {
+ var el = document.getElementById(rootId);
+ if (el) el.remove();
+ return true;
+ })("` + InteractiveOverlayRootID + `")`
+ return chromedp.Run(ctx, chromedp.Evaluate(script, nil))
+}
diff --git a/internal/cdptk/annotate_interactive_test.go b/internal/cdptk/annotate_interactive_test.go
new file mode 100644
index 00000000..f9ff1c50
--- /dev/null
+++ b/internal/cdptk/annotate_interactive_test.go
@@ -0,0 +1,36 @@
+package cdptk
+
+import (
+ "strings"
+ "testing"
+)
+
+// The persistent interactive overlay must use a DOM id distinct from the
+// transient screenshot overlay, otherwise taking an annotated screenshot would
+// remove a human-facing overlay a user injected with `annotate` (and vice
+// versa).
+func TestInteractiveOverlayRootID_DistinctFromScreenshotOverlay(t *testing.T) {
+ if InteractiveOverlayRootID == OverlayRootID {
+ t.Fatalf("interactive and screenshot overlays share id %q; they must be distinct", OverlayRootID)
+ }
+}
+
+// Guard the interactive overlay's behavioural contract: it must be clickable,
+// copy a reference to the clipboard, and compute a CSS/XPath path for the
+// clicked element. These are the properties the LLM-fix workflow depends on.
+func TestInteractiveOverlayScript_Contract(t *testing.T) {
+ for _, needle := range []string{
+ "{{DATA}}", // item payload placeholder
+ "{{ROOT}}", // root id placeholder
+ "addEventListener('click'", // labels are clickable
+ "clipboard", // copies to the clipboard
+ "__ptLastCopied", // stages copied text for verification
+ "function cssPath", // computes a CSS selector
+ "function xPath", // computes an XPath
+ "pointerEvents = 'auto'", // labels receive pointer events
+ } {
+ if !strings.Contains(interactiveOverlayScript, needle) {
+ t.Errorf("interactiveOverlayScript missing %q", needle)
+ }
+ }
+}
diff --git a/internal/cli/actions/actions_annotate.go b/internal/cli/actions/actions_annotate.go
new file mode 100644
index 00000000..f64f688c
--- /dev/null
+++ b/internal/cli/actions/actions_annotate.go
@@ -0,0 +1,65 @@
+package actions
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+
+ "github.com/pinchtab/pinchtab/internal/cli"
+ "github.com/pinchtab/pinchtab/internal/cli/apiclient"
+ "github.com/spf13/cobra"
+)
+
+// Annotate injects (or clears) the persistent, clickable annotation overlay.
+// On inject it prints a legend correlating each visible ref label with its role
+// and accessible name, mirroring `screenshot --annotate`.
+func Annotate(client *http.Client, base, token string, cmd *cobra.Command) {
+ params := url.Values{}
+ if v, _ := cmd.Flags().GetString("selector"); v != "" {
+ params.Set("selector", v)
+ }
+ if v, _ := cmd.Flags().GetString("tab"); v != "" {
+ params.Set("tabId", v)
+ }
+
+ if clear, _ := cmd.Flags().GetBool("clear"); clear {
+ params.Set("clear", "true")
+ if apiclient.DoGetRaw(client, base, token, "/annotate", params) == nil {
+ return
+ }
+ fmt.Println(cli.StyleStdout(cli.SuccessStyle, "Overlay cleared"))
+ return
+ }
+
+ raw := apiclient.DoGetRaw(client, base, token, "/annotate", params)
+ if raw == nil {
+ return
+ }
+ var envelope struct {
+ Annotated int `json:"annotated"`
+ Annotations []struct {
+ Ref string `json:"ref"`
+ Role string `json:"role"`
+ Name string `json:"name"`
+ Tag string `json:"tag"`
+ } `json:"annotations"`
+ }
+ if err := json.Unmarshal(raw, &envelope); err != nil {
+ cli.Fatal("Decode failed: %v", err)
+ }
+ fmt.Println(cli.StyleStdout(cli.SuccessStyle,
+ fmt.Sprintf("Annotated %d elements — click a label in the browser to copy its reference", envelope.Annotated)))
+ for _, a := range envelope.Annotations {
+ line := a.Ref
+ if a.Role != "" {
+ line += " " + a.Role
+ }
+ if a.Name != "" {
+ line += " \"" + a.Name + "\""
+ } else if a.Tag != "" {
+ line += " <" + a.Tag + ">"
+ }
+ fmt.Println(line)
+ }
+}
diff --git a/internal/handlers/annotate.go b/internal/handlers/annotate.go
new file mode 100644
index 00000000..473ae844
--- /dev/null
+++ b/internal/handlers/annotate.go
@@ -0,0 +1,67 @@
+package handlers
+
+import (
+ "fmt"
+ "net/http"
+
+ "github.com/pinchtab/pinchtab/internal/cdptk"
+ "github.com/pinchtab/pinchtab/internal/httpx"
+)
+
+// @Endpoint GET /annotate
+//
+// HandleAnnotate injects a persistent, clickable annotation overlay onto the
+// current page: one pink box per interactive element, each labelled with its
+// ref. Unlike `screenshot?annotate=true` (which bakes boxes into an image and
+// removes them), this overlay stays on the live page — intended for headed
+// browsers where a human clicks a label to copy a reference block for that
+// element. `?clear=true` removes the overlay instead.
+func (h *Handlers) HandleAnnotate(w http.ResponseWriter, r *http.Request) {
+ if !h.ensureBrowserOrRespond(w, h.Config) {
+ return
+ }
+
+ q := r.URL.Query()
+ tabID := q.Get("tabId")
+ selector := q.Get("selector")
+ clear := q.Get("clear") == "true" || q.Get("clear") == "1"
+
+ resolvedTabID, tCtx, cancel, ok := h.resolveBinaryReadContext(w, r, tabID, h.Config.ActionTimeout)
+ if !ok {
+ return
+ }
+ defer cancel()
+
+ if clear {
+ if err := cdptk.RemoveInteractiveOverlay(tCtx); err != nil {
+ httpx.Error(w, 500, fmt.Errorf("annotate clear: %w", err))
+ return
+ }
+ httpx.JSON(w, 200, map[string]any{"cleared": true})
+ return
+ }
+
+ // Reuse the screenshot annotation collector: it records refs in the tab's
+ // ref cache (so a follow-up `click e5` resolves the same node) and returns
+ // viewport-relative rects. We keep every candidate — not just the visible
+ // viewport — so scrolling the page reveals boxes across the whole document.
+ items, _, err := h.collectScreenshotAnnotations(tCtx, resolvedTabID, selector)
+ if err != nil {
+ httpx.Error(w, 500, fmt.Errorf("annotate: %w", err))
+ return
+ }
+ if err := cdptk.InjectInteractiveOverlay(tCtx, items); err != nil {
+ httpx.Error(w, 500, fmt.Errorf("annotate inject: %w", err))
+ return
+ }
+
+ httpx.JSON(w, 200, map[string]any{
+ "annotated": len(items),
+ "annotations": items,
+ })
+}
+
+// @Endpoint GET /tabs/{id}/annotate
+func (h *Handlers) HandleTabAnnotate(w http.ResponseWriter, r *http.Request) {
+ h.withPathTabID(w, r, h.HandleAnnotate)
+}
diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go
index 6c89ff81..67e3802f 100644
--- a/internal/handlers/handlers.go
+++ b/internal/handlers/handlers.go
@@ -288,6 +288,7 @@ func (h *Handlers) bridgeBindings() []routeBinding {
{pattern: "GET /frame", root: h.HandleFrame, tab: h.HandleTabFrame},
{pattern: "POST /frame", root: h.HandleFrame, tab: h.HandleTabFrame},
{pattern: "GET /screenshot", root: h.HandleScreenshot, tab: h.HandleTabScreenshot},
+ {pattern: "GET /annotate", root: h.HandleAnnotate, tab: h.HandleTabAnnotate},
{pattern: "GET /capture", root: h.HandleCapture, tab: h.HandleTabCapture},
{pattern: "GET /text", root: h.HandleText, tab: h.HandleTabText},
{pattern: "GET /title", root: h.HandleTitle, tab: h.HandleTabTitle},
diff --git a/internal/routes/routes.go b/internal/routes/routes.go
index 66defb49..5ef98468 100644
--- a/internal/routes/routes.go
+++ b/internal/routes/routes.go
@@ -85,6 +85,7 @@ var coreEndpoints = []Endpoint{
{"GET", "/frame", "Get current frame scope", CapNone, true},
{"POST", "/frame", "Set current frame scope", CapNone, true},
{"GET", "/screenshot", "Page screenshot", CapNone, true},
+ {"GET", "/annotate", "Inject or clear the persistent clickable annotation overlay", CapNone, true},
{"GET", "/capture", "Paired screenshot + accessibility snapshot from the same DOM epoch", CapNone, true},
{"GET", "/text", "Extract page text", CapNone, true},
{"GET", "/title", "Read page title", CapNone, true},
diff --git a/internal/server/helpers.go b/internal/server/helpers.go
index b57e53c2..f5e5a05f 100644
--- a/internal/server/helpers.go
+++ b/internal/server/helpers.go
@@ -61,7 +61,7 @@ func CheckPinchTabRunning(port, token string) bool {
// is deliberately narrow (not the full shorthand surface); every entry must be a
// real catalog route, enforced by TestDefaultProxyShorthandsAreCatalogRoutes.
var DefaultProxyShorthands = []string{
- "GET /snapshot", "GET /screenshot", "GET /text",
+ "GET /snapshot", "GET /screenshot", "GET /annotate", "GET /text",
"POST /navigate", "POST /action", "POST /actions", "POST /evaluate",
"POST /tab", "POST /lock", "POST /unlock",
"GET /cookies", "POST /cookies", "DELETE /cookies",
diff --git a/tests/e2e/scenarios/api/annotate-basic.sh b/tests/e2e/scenarios/api/annotate-basic.sh
new file mode 100755
index 00000000..5fb6b328
--- /dev/null
+++ b/tests/e2e/scenarios/api/annotate-basic.sh
@@ -0,0 +1,115 @@
+#!/bin/bash
+# annotate-basic.sh — persistent, clickable annotation overlay.
+#
+# Covers the public contract of GET /annotate (and ?clear=true):
+# - JSON envelope shape (annotated count + annotations array)
+# - the persistent __pinchtab_annotations_interactive__ overlay is injected
+# and, unlike the screenshot overlay, STAYS on the page
+# - clicking a label copies a reference block (page + ref + CSS + XPath) —
+# verified by driving the label's click handler and reading what it staged
+# - returned ref is usable for follow-up actions in the same tab
+# - ?clear=true removes the overlay
+
+GROUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${GROUP_DIR}/../../helpers/api.sh"
+
+OVERLAY_ID='__pinchtab_annotations_interactive__'
+
+# ─────────────────────────────────────────────────────────────────
+start_test "annotate: JSON envelope shape"
+
+pt_post /navigate -d "{\"url\":\"${FIXTURES_URL}/form.html\"}"
+
+pt_get "/annotate"
+assert_ok "annotate inject"
+
+assert_result_exists '.annotated' 'envelope.annotated present'
+assert_result_exists '.annotations' 'envelope.annotations present'
+assert_json_length_gte "$RESULT" '.annotations' 1 'has at least one annotation'
+assert_result_jq \
+ '.annotations[0].ref | test("^[a-z]+[0-9]+$")' \
+ 'annotations[0].ref looks like e<digits>' \
+ 'annotations[0].ref is malformed'
+
+FIRST_REF=$(echo "$RESULT" | jq -r '.annotations[0].ref')
+
+end_test
+
+# ─────────────────────────────────────────────────────────────────
+start_test "annotate: overlay persists on the live page"
+
+# Opposite of the screenshot overlay: this one must remain after the call so a
+# human can look at it and click a label.
+pt_post /evaluate -d "{\"expression\":\"document.getElementById('${OVERLAY_ID}') !== null\"}"
+assert_ok "evaluate overlay-present probe"
+assert_result_eq '.result' 'true' 'persistent overlay node present after inject'
+
+end_test
+
+# ─────────────────────────────────────────────────────────────────
+start_test "annotate: clicking a label copies a reference block"
+
+# Drive the label's real click handler (same code path as a human click) and
+# read window.__ptLastCopied, which the handler stages synchronously before the
+# async clipboard write. This is the "click the item and get the data" contract.
+CLICK_EXPR="(function(){var r=document.getElementById('${OVERLAY_ID}');if(!r)return 'NO_OVERLAY';var t=null;r.querySelectorAll('div').forEach(function(l){if(l.textContent==='${FIRST_REF}')t=l;});if(!t)return 'NO_LABEL';t.dispatchEvent(new MouseEvent('click',{bubbles:true}));return window.__ptLastCopied||'NO_COPY';})()"
+CLICK_PAYLOAD=$(jq -nc --arg e "$CLICK_EXPR" '{expression:$e}')
+
+pt_post /evaluate -d "$CLICK_PAYLOAD"
+assert_ok "evaluate label click"
+
+# The copied block must carry every field an LLM needs to locate the element.
+assert_result_jq \
+ ".result | contains(\"Element: ${FIRST_REF}\")" \
+ 'copied block names the clicked ref' \
+ 'copied block missing the ref'
+assert_result_jq \
+ '.result | contains("Page: ")' \
+ 'copied block includes the page title/url' \
+ 'copied block missing page context'
+assert_result_jq \
+ '.result | contains("CSS: ")' \
+ 'copied block includes a CSS selector' \
+ 'copied block missing CSS selector'
+assert_result_jq \
+ '.result | contains("XPath: ")' \
+ 'copied block includes an XPath' \
+ 'copied block missing XPath'
+
+end_test
+
+# ─────────────────────────────────────────────────────────────────
+start_test "annotate: returned ref is clickable"
+
+# annotate seeds the tab ref cache, so a follow-up action resolves the ref
+# without a separate snapshot.
+if [ -n "$FIRST_REF" ] && [ "$FIRST_REF" != "null" ]; then
+ pt_post /action -d "{\"kind\":\"click\",\"ref\":\"${FIRST_REF}\"}"
+ if echo "$RESULT" | grep -qi "ref.*not.*found\|unknown.*ref"; then
+ fail_assert "ref ${FIRST_REF} not recognised after annotate"
+ else
+ pass_assert "ref ${FIRST_REF} resolved by click handler"
+ fi
+else
+ fail_assert "no ref returned to click"
+fi
+
+end_test
+
+# ─────────────────────────────────────────────────────────────────
+start_test "annotate: clear removes the overlay"
+
+# Re-inject first (the click test may have navigated the ref's target away).
+pt_post /navigate -d "{\"url\":\"${FIXTURES_URL}/form.html\"}"
+pt_get "/annotate"
+assert_ok "annotate re-inject before clear"
+
+pt_get "/annotate?clear=true"
+assert_ok "annotate clear"
+assert_result_eq '.cleared' 'true' 'clear returns cleared=true'
+
+pt_post /evaluate -d "{\"expression\":\"document.getElementById('${OVERLAY_ID}') === null\"}"
+assert_ok "evaluate overlay-removed probe"
+assert_result_eq '.result' 'true' 'overlay node removed after clear'
+
+end_test