High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
feat: add persistent clickable annotate overlay for copying element references
Luigi Agosti047ed665f14d
diff --git a/cmd/pinchtab/cmd_cli_browser_inspect.go b/cmd/pinchtab/cmd_cli_browser_inspect.goindex d5710610..a307c2e3 100644--- a/cmd/pinchtab/cmd_cli_browser_inspect.go+++ b/cmd/pinchtab/cmd_cli_browser_inspect.go@@ -42,6 +42,16 @@ var screenshotCmd = &cobra.Command{},}+var annotateCmd = &cobra.Command{+ Use: "annotate",+ Short: "Inject a persistent, clickable annotation overlay on the live page (--clear to remove)",+ Run: func(cmd *cobra.Command, args []string) {+ runCLI(func(rt cliRuntime) {+ browseractions.Annotate(rt.client, rt.base, rt.token, cmd)+ })+ },+}+var captureCmd = &cobra.Command{Use: "capture",Aliases: []string{"cap"},diff --git a/cmd/pinchtab/cmd_cli_register.go b/cmd/pinchtab/cmd_cli_register.goindex 7e13ea92..40d530f1 100644--- a/cmd/pinchtab/cmd_cli_register.go+++ b/cmd/pinchtab/cmd_cli_register.go@@ -19,7 +19,7 @@ func init() {func browserRootCommands() []*cobra.Command {return []*cobra.Command{quickCmd, navCmd, backCmd, forwardCmd, reloadCmd, snapCmd, frameCmd, clickCmd,- dblclickCmd, dragCmd, typeCmd, screenshotCmd, captureCmd, tabsCmd, pressCmd, fillCmd,+ dblclickCmd, dragCmd, typeCmd, screenshotCmd, annotateCmd, captureCmd, tabsCmd, pressCmd, fillCmd,hoverCmd, mouseCmd, focusCmd, scrollCmd, evalCmd, pdfCmd, textCmd, titleCmd, urlCmd,htmlCmd, stylesCmd, valueCmd, attrCmd, countCmd, boxCmd, visibleCmd, enabledCmd, checkedCmd,downloadCmd, uploadCmd, findCmd, selectCmd, checkCmd, uncheckCmd, networkCmd, waitCmd,@@ -119,6 +119,10 @@ func configureBrowserFlags() {screenshotCmd.Flags().Bool("css-1x", false, "deprecated: use --scale")_ = screenshotCmd.Flags().MarkDeprecated("css-1x", "css-1x was removed; use --scale to rescale output")+ annotateCmd.Flags().StringP("selector", "s", "", "Scope the overlay to elements within this selector (ref/CSS/XPath/text)")+ annotateCmd.Flags().String("tab", "", "Tab ID")+ annotateCmd.Flags().Bool("clear", false, "Remove the persistent annotation overlay instead of injecting it")+captureCmd.Flags().StringP("output", "o", "", "Save the captured image to this local file path (default: capture-<ts>.jpg)")captureCmd.Flags().StringP("selector", "s", "", "Scope: clips screenshot and filters snapshot subtree to the same element")captureCmd.Flags().String("filter", "", "Snapshot filter: 'interactive' or 'all' (default: interactive)")diff --git a/docs/commands.md b/docs/commands.mdindex 35215907..4cc9f36d 100644--- a/docs/commands.md+++ b/docs/commands.md@@ -213,6 +213,10 @@ pinchtab screenshot -q <0-100> # JPEG qualitypinchtab screenshot -s <selector> # Capture a specific element by selectorpinchtab screenshot --scale 0.5 # Half-size output (quarter the pixels)pinchtab screenshot --beyond-viewport # Capture the full scrollable document (ignored with -s)+pinchtab screenshot --annotate # Bake numbered ref boxes into the image (for vision models)+pinchtab annotate # Inject a persistent, clickable overlay on the LIVE page+pinchtab annotate -s <selector> # Scope the overlay to elements within a selector+pinchtab annotate --clear # Remove the persistent overlaypinchtab capture # Paired screenshot + accessibility snapshot from the same DOM epochpinchtab capture -o <path> # Save the paired image to a chosen pathpinchtab capture --beyond-viewport # Capture the full document; bounds in page coordsdiff --git a/docs/endpoints.md b/docs/endpoints.mdindex a03e63e4..997ef127 100644--- a/docs/endpoints.md+++ b/docs/endpoints.md@@ -241,6 +241,8 @@ Find body fields:```textGET /screenshotGET /tabs/{id}/screenshot+GET /annotate+GET /tabs/{id}/annotateGET /captureGET /tabs/{id}/captureGET /pdf@@ -267,6 +269,20 @@ Screenshot query parameters:- `scale=<float>` — rescale the output bitmap (e.g. `0.5` = half size,`0.25` = quarter). Default `1`.+`/annotate` injects a persistent, clickable annotation overlay onto the live+page — one labelled box per interactive element — and leaves it there (the+`screenshot?annotate=true` overlay is transient and baked into the image+instead). Intended for headed browsers: clicking a label copies a reference+block (page, ref, role, accessible name, CSS selector, XPath) to the clipboard.+`?clear=true` removes it; `?selector=` scopes it. See+[Fix your website faster with an LLM](guides/annotate-for-llm-fixes.md).++Annotate query parameters:++- `tabId`+- `selector` — scope the overlay to elements within this selector+- `clear=true` — remove the overlay instead of injecting it+`/capture` returns a screenshot and an accessibility snapshot from the sameDOM epoch in a single call. It is the vision-grounded alternative to issuing`/screenshot` and `/snapshot` back-to-back — the two unpaired calls share nodiff --git a/docs/guides/annotate-for-llm-fixes.md b/docs/guides/annotate-for-llm-fixes.mdnew file mode 100644index 00000000..705b88e9--- /dev/null+++ b/docs/guides/annotate-for-llm-fixes.md@@ -0,0 +1,102 @@+# Fix your website faster with an LLM++When you ask an LLM to "fix this button," it has to *guess* which element you mean from a vague description. That round-trip — "which button?", "the blue one, top right", "I don't see it" — is where most of the time goes.++PinchTab's **annotate** overlay removes the guessing. It draws a labelled box on every interactive element on the live page, and each label is a one-click "copy a precise reference for this element" button. You click the thing you want changed, paste the reference into your LLM chat, and the model knows *exactly* which element you mean — page, ref, role, accessible name, CSS selector, and XPath, with no ambiguity.++This guide shows the full loop.++## The loop++1. **Annotate** the page — draw a labelled box on every interactive element.+2. **Click** the label on the element you want to change.+3. **Paste** the copied reference into your LLM chat.+4. Let the LLM locate and fix the element from an unambiguous selector.++## 1. Open the page in a headed browser++Annotation is a *human-facing* overlay, so run a headed instance you can actually look at. See [Headed mode](./headed-mode.md) for details.++```bash+# start a visible browser and open your site+pinchtab instance start --browser cloak --mode headed+pinchtab --server http://127.0.0.1:9870 nav https://your-site.com+```++## 2. Annotate++```bash+pinchtab --server http://127.0.0.1:9870 annotate+```++Every interactive element gets a pink box labelled with its ref (`e0`, `e1`, …). The command also prints a legend so you can correlate the on-screen labels with roles and text:++```+Annotated 93 elements — click a label in the browser to copy its reference+e0 switch "Switch between human and agent mode"+e4 link "View on GitHub"+e5 link "Docs"+…+```++++The overlay is real DOM on the live page — it stays there and scrolls with the content, so you can browse around and pick your target. (It's a one-shot injection: after navigating or a big layout change, run `annotate` again to refresh.)++## 3. Click the label to copy a reference++Click the **label** (the little pink tag, not the box) on the element you want to change. It flashes green to confirm the copy:++++Your clipboard now holds a complete, unambiguous reference:++```+Page: PinchTab — Browser Control for AI Agents (https://pinchtab.com/)+Element: e5 — link "Docs"+CSS: body > main > section:nth-of-type(1) > div:nth-of-type(3) > div:nth-of-type(4) > a:nth-of-type(2)+XPath: /html[1]/body[1]/main[1]/section[1]/div[3]/div[4]/a[2]+```++Every field an LLM needs to locate the element in your source is there:++- **Page** — which page/route the element is on.+- **Element** — the ref plus its role and accessible name ("link, Docs").+- **CSS** — a unique CSS selector path.+- **XPath** — an absolute XPath as a fallback.++## 4. Paste into your LLM chat++Now just paste and describe the change:++> Fix this — the link should open in a new tab:+> ```+> Page: PinchTab — Browser Control for AI Agents (https://pinchtab.com/)+> Element: e5 — link "Docs"+> CSS: body > main > section:nth-of-type(1) > div:nth-of-type(3) > div:nth-of-type(4) > a:nth-of-type(2)+> XPath: /html[1]/body[1]/main[1]/section[1]/div[3]/div[4]/a[2]+> ```++The LLM has the accessible name to grep your source ("Docs"), the CSS/XPath to disambiguate which of several similar elements you mean, and the page URL for context. No "which button?" round-trip.++## Clean up++Remove the overlay when you're done:++```bash+pinchtab --server http://127.0.0.1:9870 annotate --clear+```++## Notes++- **No `eval` permission required.** The overlay is injected through PinchTab's internal engine, so it works with `security.allowEvaluate: false`.+- **Copying uses the page's clipboard** (`navigator.clipboard`) on your real click, so it needs a focused, secure-context (HTTPS) page — exactly the case when you're looking at a headed window.+- **Scope it** with `--selector` to annotate only part of a busy page:+ ```bash+ pinchtab --server http://127.0.0.1:9870 annotate --selector "#pricing"+ ```+- The refs (`e5`, …) are the same refs the rest of PinchTab uses, so you can also act on them directly: `pinchtab --server http://127.0.0.1:9870 click e5`.++## How it compares to annotated screenshots++`pinchtab screenshot --annotate` bakes the boxes into an *image* and then removes the overlay — great for feeding a screenshot to a vision model. `pinchtab annotate` is the opposite: it *leaves* the boxes on the live page and makes them clickable, for a human driving the fix loop. Use the screenshot form for agents, the `annotate` form for yourself.diff --git a/docs/index.json b/docs/index.jsonindex a682e9ba..faf224e8 100644--- a/docs/index.json+++ b/docs/index.json@@ -23,6 +23,7 @@"guides/data-storage.md","guides/docker.md","guides/headed-mode.md",+ "guides/annotate-for-llm-fixes.md","guides/raspberry-pi.md"],"architecture": [diff --git a/docs/media/annotate-copied.png b/docs/media/annotate-copied.pngnew file mode 100644index 00000000..4c230325Binary files /dev/null and b/docs/media/annotate-copied.png differdiff --git a/docs/media/annotate-overlay.png b/docs/media/annotate-overlay.pngnew file mode 100644index 00000000..0b98a552Binary files /dev/null and b/docs/media/annotate-overlay.png differdiff --git a/internal/cdptk/annotate_interactive.go b/internal/cdptk/annotate_interactive.gonew file mode 100644index 00000000..26f57e5c--- /dev/null+++ b/internal/cdptk/annotate_interactive.go@@ -0,0 +1,213 @@+package cdptk++import (+ "context"+ "encoding/json"+ "strings"++ "github.com/chromedp/chromedp"+)++// InteractiveOverlayRootID is the stable DOM id for the persistent, clickable+// annotation layer. It is intentionally distinct from OverlayRootID so the+// transient screenshot overlay and the persistent interactive overlay never+// clobber each other: taking an annotated screenshot must not wipe a+// human-facing overlay a user injected with `annotate`.+const InteractiveOverlayRootID = "__pinchtab_annotations_interactive__"++// interactiveOverlayScript builds the persistent overlay. Unlike+// InjectAnnotationOverlay (which is torn down right after a screenshot), this+// overlay stays on the page and each label is clickable: clicking copies a+// reference block for that element to the clipboard so a human can paste it+// into an LLM ("fix this: <ref>, <css>, ...").+//+// {{DATA}} is the JSON item array; {{ROOT}} is the overlay root id. The body is+// kept free of Go fmt verbs so we can substitute via a plain replacer.+const interactiveOverlayScript = `(function(items, rootId) {+ var prev = document.getElementById(rootId);+ if (prev) prev.remove();+ var sx = window.scrollX || window.pageXOffset || 0;+ var sy = window.scrollY || window.pageYOffset || 0;+ window.__ptLastCopied = null;++ function cssPath(el) {+ if (!el || el.nodeType !== 1) return '';+ var parts = [];+ while (el && el.nodeType === 1 && el !== document.documentElement) {+ var sel = el.nodeName.toLowerCase();+ if (el.id) { parts.unshift('#' + CSS.escape(el.id)); break; }+ var p = el.parentElement;+ if (p) {+ var same = Array.prototype.filter.call(p.children, function(c) { return c.nodeName === el.nodeName; });+ if (same.length > 1) sel += ':nth-of-type(' + (same.indexOf(el) + 1) + ')';+ }+ parts.unshift(sel);+ el = el.parentElement;+ }+ return parts.join(' > ');+ }+ function xPath(el) {+ var parts = [];+ while (el && el.nodeType === 1) {+ var i = 1, s = el.previousElementSibling;+ while (s) { if (s.nodeName === el.nodeName) i++; s = s.previousElementSibling; }+ parts.unshift(el.nodeName.toLowerCase() + '[' + i + ']');+ el = el.parentElement;+ }+ return '/' + parts.join('/');+ }+ // resolveEl maps an annotation box back to the live DOM element. Boxes carry+ // pointer-events:none so elementFromPoint at the box centre passes through to+ // the underlying element; we then walk up to the ancestor whose rect best+ // matches the annotation box, since the point may land on a child node.+ function resolveEl(it) {+ var cx = it.x + sx - (window.scrollX || window.pageXOffset || 0) + it.w / 2;+ var cy = it.y + sy - (window.scrollY || window.pageYOffset || 0) + it.h / 2;+ var el = document.elementFromPoint(cx, cy);+ if (!el) return null;+ var best = el, bestDiff = Infinity, node = el;+ for (var k = 0; k < 6 && node; k++) {+ var r = node.getBoundingClientRect();+ var diff = Math.abs(r.width - it.w) + Math.abs(r.height - it.h);+ if (diff < bestDiff) { bestDiff = diff; best = node; }+ node = node.parentElement;+ }+ return best;+ }+ function fallbackCopy(t) {+ var ta = document.createElement('textarea');+ ta.value = t; ta.style.position = 'fixed'; ta.style.opacity = '0';+ document.body.appendChild(ta); ta.focus(); ta.select();+ try { document.execCommand('copy'); } catch (e) {}+ ta.remove();+ return Promise.resolve();+ }+ function copyText(t) {+ window.__ptLastCopied = t;+ if (navigator.clipboard && navigator.clipboard.writeText) {+ return navigator.clipboard.writeText(t).catch(function() { return fallbackCopy(t); });+ }+ return fallbackCopy(t);+ }+ function reference(it) {+ var el = resolveEl(it);+ var css = el ? cssPath(el) : '(unresolved)';+ var xp = el ? xPath(el) : '';+ var head = it.ref;+ if (it.role) head += ' — ' + it.role;+ if (it.name) head += ' "' + it.name + '"';+ return 'Page: ' + document.title + ' (' + location.href + ')\n' ++ 'Element: ' + head + '\n' ++ 'CSS: ' + css + '\n' ++ 'XPath: ' + xp;+ }++ var root = document.createElement('div');+ root.id = rootId;+ root.style.position = 'absolute';+ root.style.top = '0';+ root.style.left = '0';+ root.style.width = '0';+ root.style.height = '0';+ root.style.pointerEvents = 'none';+ root.style.zIndex = '2147483646';++ items.forEach(function(it) {+ var box = document.createElement('div');+ box.style.position = 'absolute';+ box.style.left = (it.x + sx) + 'px';+ box.style.top = (it.y + sy) + 'px';+ box.style.width = it.w + 'px';+ box.style.height = it.h + 'px';+ box.style.boxSizing = 'border-box';+ box.style.border = '2px solid rgba(255, 51, 102, 0.95)';+ box.style.borderRadius = '2px';+ box.style.pointerEvents = 'none';++ var label = document.createElement('div');+ label.textContent = it.ref;+ label.title = 'Click to copy a reference for ' + it.ref;+ label.style.position = 'absolute';+ label.style.left = (it.x + sx) + 'px';+ label.style.top = Math.max(0, it.y + sy - 16) + 'px';+ label.style.padding = '0 5px';+ label.style.background = 'rgba(255, 51, 102, 0.95)';+ label.style.color = '#fff';+ label.style.font = '600 12px/16px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace';+ label.style.borderRadius = '2px 2px 0 0';+ label.style.whiteSpace = 'nowrap';+ label.style.userSelect = 'none';+ label.style.cursor = 'pointer';+ label.style.pointerEvents = 'auto';+ label.addEventListener('click', function(ev) {+ ev.preventDefault();+ ev.stopPropagation();+ copyText(reference(it)).then(function() {+ var orig = label.textContent;+ label.textContent = '✓ ' + it.ref;+ label.style.background = 'rgba(40, 200, 90, 0.95)';+ setTimeout(function() {+ label.textContent = orig;+ label.style.background = 'rgba(255, 51, 102, 0.95)';+ }, 1300);+ });+ });++ root.appendChild(box);+ root.appendChild(label);+ });+ document.documentElement.appendChild(root);+ return items.length;+})({{DATA}}, "{{ROOT}}")`++// InjectInteractiveOverlay paints a persistent overlay whose labels are+// clickable. Coordinates are viewport-relative (as produced by+// AnnotationRectForNode); the page adds scrollX/scrollY so boxes track the+// document while scrolling. The overlay is NOT auto-removed — callers remove it+// via RemoveInteractiveOverlay.+func InjectInteractiveOverlay(ctx context.Context, items []AnnotationItem) error {+ if len(items) == 0 {+ return nil+ }+ type overlayItem struct {+ Ref string `json:"ref"`+ Role string `json:"role,omitempty"`+ Name string `json:"name,omitempty"`+ X float64 `json:"x"`+ Y float64 `json:"y"`+ W float64 `json:"w"`+ H float64 `json:"h"`+ }+ payload := make([]overlayItem, len(items))+ for i, it := range items {+ payload[i] = overlayItem{+ Ref: it.Ref,+ Role: it.Role,+ Name: it.Name,+ X: it.Box.X,+ Y: it.Box.Y,+ W: it.Box.W,+ H: it.Box.H,+ }+ }+ encoded, err := json.Marshal(payload)+ if err != nil {+ return err+ }+ script := strings.NewReplacer(+ "{{DATA}}", string(encoded),+ "{{ROOT}}", InteractiveOverlayRootID,+ ).Replace(interactiveOverlayScript)+ return chromedp.Run(ctx, chromedp.Evaluate(script, nil))+}++// RemoveInteractiveOverlay removes the persistent overlay if present. Safe to+// call when no overlay exists.+func RemoveInteractiveOverlay(ctx context.Context) error {+ script := `(function(rootId) {+ var el = document.getElementById(rootId);+ if (el) el.remove();+ return true;+ })("` + InteractiveOverlayRootID + `")`+ return chromedp.Run(ctx, chromedp.Evaluate(script, nil))+}diff --git a/internal/cdptk/annotate_interactive_test.go b/internal/cdptk/annotate_interactive_test.gonew file mode 100644index 00000000..f9ff1c50--- /dev/null+++ b/internal/cdptk/annotate_interactive_test.go@@ -0,0 +1,36 @@+package cdptk++import (+ "strings"+ "testing"+)++// The persistent interactive overlay must use a DOM id distinct from the+// transient screenshot overlay, otherwise taking an annotated screenshot would+// remove a human-facing overlay a user injected with `annotate` (and vice+// versa).+func TestInteractiveOverlayRootID_DistinctFromScreenshotOverlay(t *testing.T) {+ if InteractiveOverlayRootID == OverlayRootID {+ t.Fatalf("interactive and screenshot overlays share id %q; they must be distinct", OverlayRootID)+ }+}++// Guard the interactive overlay's behavioural contract: it must be clickable,+// copy a reference to the clipboard, and compute a CSS/XPath path for the+// clicked element. These are the properties the LLM-fix workflow depends on.+func TestInteractiveOverlayScript_Contract(t *testing.T) {+ for _, needle := range []string{+ "{{DATA}}", // item payload placeholder+ "{{ROOT}}", // root id placeholder+ "addEventListener('click'", // labels are clickable+ "clipboard", // copies to the clipboard+ "__ptLastCopied", // stages copied text for verification+ "function cssPath", // computes a CSS selector+ "function xPath", // computes an XPath+ "pointerEvents = 'auto'", // labels receive pointer events+ } {+ if !strings.Contains(interactiveOverlayScript, needle) {+ t.Errorf("interactiveOverlayScript missing %q", needle)+ }+ }+}diff --git a/internal/cli/actions/actions_annotate.go b/internal/cli/actions/actions_annotate.gonew file mode 100644index 00000000..f64f688c--- /dev/null+++ b/internal/cli/actions/actions_annotate.go@@ -0,0 +1,65 @@+package actions++import (+ "encoding/json"+ "fmt"+ "net/http"+ "net/url"++ "github.com/pinchtab/pinchtab/internal/cli"+ "github.com/pinchtab/pinchtab/internal/cli/apiclient"+ "github.com/spf13/cobra"+)++// Annotate injects (or clears) the persistent, clickable annotation overlay.+// On inject it prints a legend correlating each visible ref label with its role+// and accessible name, mirroring `screenshot --annotate`.+func Annotate(client *http.Client, base, token string, cmd *cobra.Command) {+ params := url.Values{}+ if v, _ := cmd.Flags().GetString("selector"); v != "" {+ params.Set("selector", v)+ }+ if v, _ := cmd.Flags().GetString("tab"); v != "" {+ params.Set("tabId", v)+ }++ if clear, _ := cmd.Flags().GetBool("clear"); clear {+ params.Set("clear", "true")+ if apiclient.DoGetRaw(client, base, token, "/annotate", params) == nil {+ return+ }+ fmt.Println(cli.StyleStdout(cli.SuccessStyle, "Overlay cleared"))+ return+ }++ raw := apiclient.DoGetRaw(client, base, token, "/annotate", params)+ if raw == nil {+ return+ }+ var envelope struct {+ Annotated int `json:"annotated"`+ Annotations []struct {+ Ref string `json:"ref"`+ Role string `json:"role"`+ Name string `json:"name"`+ Tag string `json:"tag"`+ } `json:"annotations"`+ }+ if err := json.Unmarshal(raw, &envelope); err != nil {+ cli.Fatal("Decode failed: %v", err)+ }+ fmt.Println(cli.StyleStdout(cli.SuccessStyle,+ fmt.Sprintf("Annotated %d elements — click a label in the browser to copy its reference", envelope.Annotated)))+ for _, a := range envelope.Annotations {+ line := a.Ref+ if a.Role != "" {+ line += " " + a.Role+ }+ if a.Name != "" {+ line += " \"" + a.Name + "\""+ } else if a.Tag != "" {+ line += " <" + a.Tag + ">"+ }+ fmt.Println(line)+ }+}diff --git a/internal/handlers/annotate.go b/internal/handlers/annotate.gonew file mode 100644index 00000000..473ae844--- /dev/null+++ b/internal/handlers/annotate.go@@ -0,0 +1,67 @@+package handlers++import (+ "fmt"+ "net/http"++ "github.com/pinchtab/pinchtab/internal/cdptk"+ "github.com/pinchtab/pinchtab/internal/httpx"+)++// @Endpoint GET /annotate+//+// HandleAnnotate injects a persistent, clickable annotation overlay onto the+// current page: one pink box per interactive element, each labelled with its+// ref. Unlike `screenshot?annotate=true` (which bakes boxes into an image and+// removes them), this overlay stays on the live page — intended for headed+// browsers where a human clicks a label to copy a reference block for that+// element. `?clear=true` removes the overlay instead.+func (h *Handlers) HandleAnnotate(w http.ResponseWriter, r *http.Request) {+ if !h.ensureBrowserOrRespond(w, h.Config) {+ return+ }++ q := r.URL.Query()+ tabID := q.Get("tabId")+ selector := q.Get("selector")+ clear := q.Get("clear") == "true" || q.Get("clear") == "1"++ resolvedTabID, tCtx, cancel, ok := h.resolveBinaryReadContext(w, r, tabID, h.Config.ActionTimeout)+ if !ok {+ return+ }+ defer cancel()++ if clear {+ if err := cdptk.RemoveInteractiveOverlay(tCtx); err != nil {+ httpx.Error(w, 500, fmt.Errorf("annotate clear: %w", err))+ return+ }+ httpx.JSON(w, 200, map[string]any{"cleared": true})+ return+ }++ // Reuse the screenshot annotation collector: it records refs in the tab's+ // ref cache (so a follow-up `click e5` resolves the same node) and returns+ // viewport-relative rects. We keep every candidate — not just the visible+ // viewport — so scrolling the page reveals boxes across the whole document.+ items, _, err := h.collectScreenshotAnnotations(tCtx, resolvedTabID, selector)+ if err != nil {+ httpx.Error(w, 500, fmt.Errorf("annotate: %w", err))+ return+ }+ if err := cdptk.InjectInteractiveOverlay(tCtx, items); err != nil {+ httpx.Error(w, 500, fmt.Errorf("annotate inject: %w", err))+ return+ }++ httpx.JSON(w, 200, map[string]any{+ "annotated": len(items),+ "annotations": items,+ })+}++// @Endpoint GET /tabs/{id}/annotate+func (h *Handlers) HandleTabAnnotate(w http.ResponseWriter, r *http.Request) {+ h.withPathTabID(w, r, h.HandleAnnotate)+}diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.goindex 6c89ff81..67e3802f 100644--- a/internal/handlers/handlers.go+++ b/internal/handlers/handlers.go@@ -288,6 +288,7 @@ func (h *Handlers) bridgeBindings() []routeBinding {{pattern: "GET /frame", root: h.HandleFrame, tab: h.HandleTabFrame},{pattern: "POST /frame", root: h.HandleFrame, tab: h.HandleTabFrame},{pattern: "GET /screenshot", root: h.HandleScreenshot, tab: h.HandleTabScreenshot},+ {pattern: "GET /annotate", root: h.HandleAnnotate, tab: h.HandleTabAnnotate},{pattern: "GET /capture", root: h.HandleCapture, tab: h.HandleTabCapture},{pattern: "GET /text", root: h.HandleText, tab: h.HandleTabText},{pattern: "GET /title", root: h.HandleTitle, tab: h.HandleTabTitle},diff --git a/internal/routes/routes.go b/internal/routes/routes.goindex 66defb49..5ef98468 100644--- a/internal/routes/routes.go+++ b/internal/routes/routes.go@@ -85,6 +85,7 @@ var coreEndpoints = []Endpoint{{"GET", "/frame", "Get current frame scope", CapNone, true},{"POST", "/frame", "Set current frame scope", CapNone, true},{"GET", "/screenshot", "Page screenshot", CapNone, true},+ {"GET", "/annotate", "Inject or clear the persistent clickable annotation overlay", CapNone, true},{"GET", "/capture", "Paired screenshot + accessibility snapshot from the same DOM epoch", CapNone, true},{"GET", "/text", "Extract page text", CapNone, true},{"GET", "/title", "Read page title", CapNone, true},diff --git a/internal/server/helpers.go b/internal/server/helpers.goindex b57e53c2..f5e5a05f 100644--- a/internal/server/helpers.go+++ b/internal/server/helpers.go@@ -61,7 +61,7 @@ func CheckPinchTabRunning(port, token string) bool {// is deliberately narrow (not the full shorthand surface); every entry must be a// real catalog route, enforced by TestDefaultProxyShorthandsAreCatalogRoutes.var DefaultProxyShorthands = []string{- "GET /snapshot", "GET /screenshot", "GET /text",+ "GET /snapshot", "GET /screenshot", "GET /annotate", "GET /text","POST /navigate", "POST /action", "POST /actions", "POST /evaluate","POST /tab", "POST /lock", "POST /unlock","GET /cookies", "POST /cookies", "DELETE /cookies",diff --git a/tests/e2e/scenarios/api/annotate-basic.sh b/tests/e2e/scenarios/api/annotate-basic.shnew file mode 100755index 00000000..5fb6b328--- /dev/null+++ b/tests/e2e/scenarios/api/annotate-basic.sh@@ -0,0 +1,115 @@+#!/bin/bash+# annotate-basic.sh — persistent, clickable annotation overlay.+#+# Covers the public contract of GET /annotate (and ?clear=true):+# - JSON envelope shape (annotated count + annotations array)+# - the persistent __pinchtab_annotations_interactive__ overlay is injected+# and, unlike the screenshot overlay, STAYS on the page+# - clicking a label copies a reference block (page + ref + CSS + XPath) —+# verified by driving the label's click handler and reading what it staged+# - returned ref is usable for follow-up actions in the same tab+# - ?clear=true removes the overlay++GROUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+source "${GROUP_DIR}/../../helpers/api.sh"++OVERLAY_ID='__pinchtab_annotations_interactive__'++# ─────────────────────────────────────────────────────────────────+start_test "annotate: JSON envelope shape"++pt_post /navigate -d "{\"url\":\"${FIXTURES_URL}/form.html\"}"++pt_get "/annotate"+assert_ok "annotate inject"++assert_result_exists '.annotated' 'envelope.annotated present'+assert_result_exists '.annotations' 'envelope.annotations present'+assert_json_length_gte "$RESULT" '.annotations' 1 'has at least one annotation'+assert_result_jq \+ '.annotations[0].ref | test("^[a-z]+[0-9]+$")' \+ 'annotations[0].ref looks like e<digits>' \+ 'annotations[0].ref is malformed'++FIRST_REF=$(echo "$RESULT" | jq -r '.annotations[0].ref')++end_test++# ─────────────────────────────────────────────────────────────────+start_test "annotate: overlay persists on the live page"++# Opposite of the screenshot overlay: this one must remain after the call so a+# human can look at it and click a label.+pt_post /evaluate -d "{\"expression\":\"document.getElementById('${OVERLAY_ID}') !== null\"}"+assert_ok "evaluate overlay-present probe"+assert_result_eq '.result' 'true' 'persistent overlay node present after inject'++end_test++# ─────────────────────────────────────────────────────────────────+start_test "annotate: clicking a label copies a reference block"++# Drive the label's real click handler (same code path as a human click) and+# read window.__ptLastCopied, which the handler stages synchronously before the+# async clipboard write. This is the "click the item and get the data" contract.+CLICK_EXPR="(function(){var r=document.getElementById('${OVERLAY_ID}');if(!r)return 'NO_OVERLAY';var t=null;r.querySelectorAll('div').forEach(function(l){if(l.textContent==='${FIRST_REF}')t=l;});if(!t)return 'NO_LABEL';t.dispatchEvent(new MouseEvent('click',{bubbles:true}));return window.__ptLastCopied||'NO_COPY';})()"+CLICK_PAYLOAD=$(jq -nc --arg e "$CLICK_EXPR" '{expression:$e}')++pt_post /evaluate -d "$CLICK_PAYLOAD"+assert_ok "evaluate label click"++# The copied block must carry every field an LLM needs to locate the element.+assert_result_jq \+ ".result | contains(\"Element: ${FIRST_REF}\")" \+ 'copied block names the clicked ref' \+ 'copied block missing the ref'+assert_result_jq \+ '.result | contains("Page: ")' \+ 'copied block includes the page title/url' \+ 'copied block missing page context'+assert_result_jq \+ '.result | contains("CSS: ")' \+ 'copied block includes a CSS selector' \+ 'copied block missing CSS selector'+assert_result_jq \+ '.result | contains("XPath: ")' \+ 'copied block includes an XPath' \+ 'copied block missing XPath'++end_test++# ─────────────────────────────────────────────────────────────────+start_test "annotate: returned ref is clickable"++# annotate seeds the tab ref cache, so a follow-up action resolves the ref+# without a separate snapshot.+if [ -n "$FIRST_REF" ] && [ "$FIRST_REF" != "null" ]; then+ pt_post /action -d "{\"kind\":\"click\",\"ref\":\"${FIRST_REF}\"}"+ if echo "$RESULT" | grep -qi "ref.*not.*found\|unknown.*ref"; then+ fail_assert "ref ${FIRST_REF} not recognised after annotate"+ else+ pass_assert "ref ${FIRST_REF} resolved by click handler"+ fi+else+ fail_assert "no ref returned to click"+fi++end_test++# ─────────────────────────────────────────────────────────────────+start_test "annotate: clear removes the overlay"++# Re-inject first (the click test may have navigated the ref's target away).+pt_post /navigate -d "{\"url\":\"${FIXTURES_URL}/form.html\"}"+pt_get "/annotate"+assert_ok "annotate re-inject before clear"++pt_get "/annotate?clear=true"+assert_ok "annotate clear"+assert_result_eq '.cleared' 'true' 'clear returns cleared=true'++pt_post /evaluate -d "{\"expression\":\"document.getElementById('${OVERLAY_ID}') === null\"}"+assert_ok "evaluate overlay-removed probe"+assert_result_eq '.result' 'true' 'overlay node removed after clear'++end_test