Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix: capture bounds are off by the scroll offset because AnnotateBounds transforms getBoxModel quads the wrong way (PIN-034)

Luigi Agostiec01d09722c4
diff --git a/internal/bridge/capture_clip_bounds_test.go b/internal/bridge/capture_clip_bounds_test.go
new file mode 100644
index 00000000..7d90e5d6
--- /dev/null
+++ b/internal/bridge/capture_clip_bounds_test.go
@@ -0,0 +1,184 @@
+package bridge
+
+import (
+ "context"
+ "encoding/base64"
+ "testing"
+ "time"
+
+ "github.com/chromedp/chromedp"
+ "github.com/pinchtab/pinchtab/internal/testbrowser"
+)
+
+// The target sits far enough down and across that a scroll is required to reach
+// it, so the clip origin, the scroll offset and the viewport rect are three
+// different numbers. At scroll 0 they collapse into one and the clip path's
+// mixed-space subtraction is invisible.
+const clipScrollFixtureHTML = `<body style="margin:0;width:2400px;height:2400px">
+<div style="position:absolute;left:900px;top:1100px;width:120px;height:40px">
+<button id="target">target</button>
+</div>
+</body>`
+
+func newClipScrollFixture(t *testing.T) (context.Context, int64) {
+ t.Helper()
+ chromePath := testbrowser.Path(t)
+
+ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(
+ chromedp.DefaultExecAllocatorOptions[:],
+ chromedp.ExecPath(chromePath),
+ chromedp.UserDataDir(t.TempDir()),
+ chromedp.Flag("headless", true),
+ chromedp.Flag("no-sandbox", true),
+ )...)
+ ctx, cancelBrowser := chromedp.NewContext(alloc)
+ ctx, cancelTimeout := context.WithTimeout(ctx, 60*time.Second)
+ t.Cleanup(func() {
+ cancelTimeout()
+ cancelBrowser()
+ cancelAlloc()
+ })
+
+ dataURL := "data:text/html;base64," + base64.StdEncoding.EncodeToString([]byte(clipScrollFixtureHTML))
+ var scrolled bool
+ if err := chromedp.Run(ctx,
+ chromedp.Navigate(dataURL),
+ chromedp.WaitVisible("#target", chromedp.ByID),
+ chromedp.Evaluate(`(() => { window.scrollTo(300, 400); return window.scrollY === 400; })()`, &scrolled),
+ ); err != nil {
+ t.Fatal(err)
+ }
+ if !scrolled {
+ t.Fatal("fixture did not scroll")
+ }
+
+ rawNodes, err := FetchAXTree(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ nodes, _ := BuildSnapshot(rawNodes, "", -1)
+ for _, n := range nodes {
+ if n.Role == "button" && n.Name == "target" && n.NodeID != 0 {
+ return ctx, n.NodeID
+ }
+ }
+ t.Fatalf("no button named \"target\" in the snapshot (%d nodes)", len(nodes))
+ return nil, 0
+}
+
+func boundsForNode(t *testing.T, res *PairedResult, nodeID int64) BoundingBox {
+ t.Helper()
+ for _, n := range res.Nodes {
+ if n.NodeID == nodeID && n.BoundingBox != nil {
+ return *n.BoundingBox
+ }
+ }
+ t.Fatalf("capture produced no bounding box for node %d (%d nodes)", nodeID, len(res.Nodes))
+ return BoundingBox{}
+}
+
+func clientRectOfTarget(t *testing.T, ctx context.Context) (x, y float64) {
+ t.Helper()
+ var rect struct {
+ X float64 `json:"x"`
+ Y float64 `json:"y"`
+ }
+ if err := chromedp.Run(ctx, chromedp.Evaluate(
+ `(() => { const r = document.getElementById('target').getBoundingClientRect(); return {x: r.x, y: r.y}; })()`,
+ &rect)); err != nil {
+ t.Fatal(err)
+ }
+ return rect.X, rect.Y
+}
+
+// The capture path measures the content box and getBoundingClientRect the border
+// box, so they differ by the button's border and padding — single digits, not
+// the hundreds of pixels a mixed-up coordinate space introduces.
+const captureBoxSlack = 12.0
+
+func assertNearRect(t *testing.T, label string, got, want float64) {
+ t.Helper()
+ if diff := got - want; diff > captureBoxSlack || diff < -captureBoxSlack {
+ t.Errorf("%s = %.2f, want %.2f (off by %.2f)", label, got, want, diff)
+ }
+}
+
+// All three spaces PairedCapture can report, on one scrolled page, each checked
+// against the browser's own rect. Each space must mean what CoordinateSpace
+// calls it: viewport is the raw rect, document is the rect plus the scroll, and
+// clip is measured from the clip origin.
+func TestPairedCaptureBoundsInEveryCoordinateSpaceWhenScrolled(t *testing.T) {
+ t.Run("viewport", func(t *testing.T) {
+ ctx, nodeID := newClipScrollFixture(t)
+
+ res, err := PairedCapture(ctx, CaptureOpts{WithBounds: true, MaxDepth: -1, Image: ScreenshotOpts{Format: ScreenshotFormatPng}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if res.CoordinateSpace != "viewport" {
+ t.Fatalf("CoordinateSpace = %q, want %q", res.CoordinateSpace, "viewport")
+ }
+ if res.Viewport.ScrollY == 0 || res.Viewport.ScrollX == 0 {
+ t.Fatalf("capture did not see a scrolled page: %+v", res.Viewport)
+ }
+
+ box := boundsForNode(t, res, nodeID)
+ rectX, rectY := clientRectOfTarget(t, ctx)
+ assertNearRect(t, "viewport x", box.X, rectX)
+ assertNearRect(t, "viewport y", box.Y, rectY)
+ })
+
+ t.Run("document", func(t *testing.T) {
+ ctx, nodeID := newClipScrollFixture(t)
+
+ res, err := PairedCapture(ctx, CaptureOpts{
+ WithBounds: true,
+ MaxDepth: -1,
+ Image: ScreenshotOpts{Format: ScreenshotFormatPng, BeyondViewport: true},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if res.CoordinateSpace != "document" {
+ t.Fatalf("CoordinateSpace = %q, want %q", res.CoordinateSpace, "document")
+ }
+
+ box := boundsForNode(t, res, nodeID)
+ rectX, rectY := clientRectOfTarget(t, ctx)
+ assertNearRect(t, "document x", box.X, rectX+res.Viewport.ScrollX)
+ assertNearRect(t, "document y", box.Y, rectY+res.Viewport.ScrollY)
+ })
+
+ // The one place two independently-derived spaces meet: the clip origin comes
+ // from ScreenshotClipForNode's JS, which adds window.scrollX/scrollY, so it is
+ // document-relative and the bounds it is subtracted from must be too. The
+ // clip is the target's own border box, so the target must land at the clip's
+ // origin — anything else is the scroll leaking in.
+ t.Run("clip", func(t *testing.T) {
+ ctx, nodeID := newClipScrollFixture(t)
+
+ clip, err := ScreenshotClipForNode(ctx, nodeID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if clip.X < 100 || clip.Y < 100 {
+ t.Fatalf("clip origin %+v is too close to the document origin for this test to distinguish spaces", clip)
+ }
+
+ res, err := PairedCapture(ctx, CaptureOpts{
+ WithBounds: true,
+ MaxDepth: -1,
+ Image: ScreenshotOpts{Format: ScreenshotFormatPng, Clip: clip},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if res.CoordinateSpace != "clip" {
+ t.Fatalf("CoordinateSpace = %q, want %q", res.CoordinateSpace, "clip")
+ }
+
+ box := boundsForNode(t, res, nodeID)
+ assertNearRect(t, "clip x", box.X, 0)
+ assertNearRect(t, "clip y", box.Y, 0)
+ })
+}
diff --git a/internal/bridge/observe/bounds.go b/internal/bridge/observe/bounds.go
index ccd780e5..7fbca5c2 100644
--- a/internal/bridge/observe/bounds.go
+++ b/internal/bridge/observe/bounds.go
@@ -65,10 +65,10 @@ func FetchLayout(ctx context.Context) (ViewportInfo, error) {
// exactly — which is what makes it correct for nodes inside iframes, where a
// rect taken in the element's own context is frame-relative.
//
-// pageCoords=true leaves boxes in that space for beyondViewport/clip captures.
-// pageCoords=false subtracts the current scroll offset; on a scrolled page that
-// shifts an already-viewport-relative box by the scroll, which is its own
-// defect and is deliberately left alone here.
+// pageCoords=false leaves boxes in that viewport space. pageCoords=true adds
+// the scroll offset to reach document space, which is what beyondViewport and
+// clip captures report and what projectBoundsToClip subtracts a clip origin
+// from — that origin is document-relative, so both sides must be.
//
// Visibility heuristic: a node is Visible if its rect has non-zero area and
// intersects the viewport. The check is intentionally cheap — strict
@@ -83,9 +83,9 @@ func AnnotateBounds(ctx context.Context, nodes []A11yNode, pageCoords bool, vp V
continue
}
visible := isVisible(box, true, vp)
- if !pageCoords {
- box.X -= vp.ScrollX
- box.Y -= vp.ScrollY
+ if pageCoords {
+ box.X += vp.ScrollX
+ box.Y += vp.ScrollY
}
nodes[i].BoundingBox = &box
nodes[i].Visible = visible
diff --git a/internal/bridge/observe/bounds_test.go b/internal/bridge/observe/bounds_test.go
index 3b02c1ed..6828164a 100644
--- a/internal/bridge/observe/bounds_test.go
+++ b/internal/bridge/observe/bounds_test.go
@@ -1,6 +1,15 @@
package observe
-import "testing"
+import (
+ "context"
+ "encoding/base64"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/chromedp/chromedp"
+ "github.com/pinchtab/pinchtab/internal/testbrowser"
+)
func TestIsVisibleWithDocumentCoordinates(t *testing.T) {
vp := ViewportInfo{Width: 800, Height: 600, ScrollX: 0, ScrollY: 1000}
@@ -12,3 +21,184 @@ func TestIsVisibleWithDocumentCoordinates(t *testing.T) {
t.Fatal("box above scrolled viewport should not be visible")
}
}
+
+// scrollFixtureHTML is tall enough to scroll in both axes, with a target that
+// sits below the fold so a scroll is required to bring it into view. Only a
+// scrolled page can tell the three coordinate spaces apart — at scroll 0 every
+// transform is the identity, which is why this defect went unnoticed.
+const scrollFixtureHTML = `<body style="margin:0;width:3000px;height:3000px">
+<div style="position:absolute;left:900px;top:1200px;width:120px;height:40px">
+<button id="target">target</button>
+</div>
+</body>`
+
+type scrollFixture struct {
+ ctx context.Context
+ nodeID int64
+}
+
+func newScrollFixture(t *testing.T, scrollX, scrollY int) scrollFixture {
+ t.Helper()
+ chromePath := testbrowser.Path(t)
+
+ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(
+ chromedp.DefaultExecAllocatorOptions[:],
+ chromedp.ExecPath(chromePath),
+ chromedp.UserDataDir(t.TempDir()),
+ chromedp.Flag("headless", true),
+ chromedp.Flag("no-sandbox", true),
+ )...)
+ ctx, cancelBrowser := chromedp.NewContext(alloc)
+ ctx, cancelTimeout := context.WithTimeout(ctx, 30*time.Second)
+ t.Cleanup(func() {
+ cancelTimeout()
+ cancelBrowser()
+ cancelAlloc()
+ })
+
+ dataURL := "data:text/html;base64," + base64.StdEncoding.EncodeToString([]byte(scrollFixtureHTML))
+ if err := chromedp.Run(ctx,
+ chromedp.Navigate(dataURL),
+ chromedp.WaitVisible("#target", chromedp.ByID),
+ ); err != nil {
+ t.Fatal(err)
+ }
+ if err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
+ var done bool
+ return chromedp.Evaluate(scrollScript(scrollX, scrollY), &done).Do(ctx)
+ })); err != nil {
+ t.Fatal(err)
+ }
+
+ return scrollFixture{ctx: ctx, nodeID: targetNodeID(t, ctx)}
+}
+
+func scrollScript(x, y int) string {
+ return fmt.Sprintf(`(() => { window.scrollTo(%d, %d); return window.scrollY === %d; })()`, x, y, y)
+}
+
+func targetNodeID(t *testing.T, ctx context.Context) int64 {
+ t.Helper()
+ rawNodes, err := FetchAXTree(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ nodes, _ := BuildSnapshot(rawNodes, "", -1)
+ for _, n := range nodes {
+ if n.Role == "button" && n.Name == "target" && n.NodeID != 0 {
+ return n.NodeID
+ }
+ }
+ t.Fatalf("no button named %q in the snapshot (%d nodes)", "target", len(nodes))
+ return 0
+}
+
+// clientRect is the browser's own answer for the target, in viewport space.
+func (f scrollFixture) clientRect(t *testing.T) BoundingBox {
+ t.Helper()
+ var rect struct {
+ X float64 `json:"x"`
+ Y float64 `json:"y"`
+ W float64 `json:"width"`
+ H float64 `json:"height"`
+ }
+ if err := chromedp.Run(f.ctx, chromedp.Evaluate(
+ `(() => { const r = document.getElementById('target').getBoundingClientRect(); return {x: r.x, y: r.y, width: r.width, height: r.height}; })()`,
+ &rect)); err != nil {
+ t.Fatal(err)
+ }
+ return BoundingBox{X: rect.X, Y: rect.Y, W: rect.W, H: rect.H}
+}
+
+func (f scrollFixture) annotate(t *testing.T, pageCoords bool) (BoundingBox, bool, ViewportInfo) {
+ t.Helper()
+ vp, err := FetchLayout(f.ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ nodes := []A11yNode{{NodeID: f.nodeID}}
+ if err := AnnotateBounds(f.ctx, nodes, pageCoords, vp); err != nil {
+ t.Fatal(err)
+ }
+ if nodes[0].BoundingBox == nil {
+ t.Fatal("AnnotateBounds produced no bounding box for the target")
+ }
+ return *nodes[0].BoundingBox, nodes[0].Visible, vp
+}
+
+// getBoxModel reports the content box and getBoundingClientRect the border box,
+// so the two differ by the button's border and padding — single digits, nothing
+// like the scroll offset a wrong transform introduces.
+const boxModelSlack = 12.0
+
+func assertNear(t *testing.T, label string, got, want float64) {
+ t.Helper()
+ if diff := got - want; diff > boxModelSlack || diff < -boxModelSlack {
+ t.Errorf("%s = %.2f, want %.2f (off by %.2f)", label, got, want, diff)
+ }
+}
+
+// The default /capture path. The raw quad is already viewport-relative, so
+// pageCoords=false must hand it back untouched — subtracting the scroll moved
+// every annotated bound on every scrolled page a full scrollY away.
+func TestAnnotateBoundsViewportSpaceMatchesClientRectWhenScrolled(t *testing.T) {
+ f := newScrollFixture(t, 300, 400)
+
+ box, _, vp := f.annotate(t, false)
+ if vp.ScrollY == 0 || vp.ScrollX == 0 {
+ t.Fatalf("fixture did not scroll: %+v", vp)
+ }
+ rect := f.clientRect(t)
+
+ assertNear(t, "viewport x", box.X, rect.X)
+ assertNear(t, "viewport y", box.Y, rect.Y)
+}
+
+// The "document" label has to be true: beyondViewport and clip captures report
+// this space, and a caller doing its own document arithmetic from the label was
+// silently off by the scroll.
+func TestAnnotateBoundsDocumentSpaceAddsScrollWhenScrolled(t *testing.T) {
+ f := newScrollFixture(t, 300, 400)
+
+ box, _, vp := f.annotate(t, true)
+ rect := f.clientRect(t)
+
+ assertNear(t, "document x", box.X, rect.X+vp.ScrollX)
+ assertNear(t, "document y", box.Y, rect.Y+vp.ScrollY)
+
+ if box.Y-rect.Y < vp.ScrollY/2 {
+ t.Errorf("document y (%.2f) is not a full scroll (%.2f) past the viewport y (%.2f)", box.Y, vp.ScrollY, rect.Y)
+ }
+}
+
+// At scroll 0 both spaces coincide, which is why this was invisible until a
+// fixture scrolled. Pins that the fix moved nothing for unscrolled pages.
+func TestAnnotateBoundsSpacesAgreeWhenUnscrolled(t *testing.T) {
+ f := newScrollFixture(t, 0, 0)
+
+ viewportBox, viewportVisible, vp := f.annotate(t, false)
+ documentBox, documentVisible, _ := f.annotate(t, true)
+
+ if vp.ScrollX != 0 || vp.ScrollY != 0 {
+ t.Fatalf("fixture unexpectedly scrolled: %+v", vp)
+ }
+ if viewportBox != documentBox {
+ t.Errorf("unscrolled page: viewport %+v and document %+v must be identical", viewportBox, documentBox)
+ }
+ if viewportVisible != documentVisible {
+ t.Errorf("unscrolled page: Visible differs between spaces (%v vs %v)", viewportVisible, documentVisible)
+ }
+}
+
+// Visible is a viewport-intersection test taken before the transform, so it must
+// not move when the caller asks for document space.
+func TestAnnotateBoundsVisibleIsIndependentOfCoordinateSpace(t *testing.T) {
+ f := newScrollFixture(t, 300, 400)
+
+ _, viewportVisible, _ := f.annotate(t, false)
+ _, documentVisible, _ := f.annotate(t, true)
+
+ if viewportVisible != documentVisible {
+ t.Errorf("Visible = %v in viewport space and %v in document space; it is computed from the same viewport-relative box either way", viewportVisible, documentVisible)
+ }
+}