High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix: capture bounds are off by the scroll offset because AnnotateBounds transforms getBoxModel quads the wrong way (PIN-034)
Luigi Agostiec01d09722c4
diff --git a/internal/bridge/capture_clip_bounds_test.go b/internal/bridge/capture_clip_bounds_test.gonew file mode 100644index 00000000..7d90e5d6--- /dev/null+++ b/internal/bridge/capture_clip_bounds_test.go@@ -0,0 +1,184 @@+package bridge++import (+ "context"+ "encoding/base64"+ "testing"+ "time"++ "github.com/chromedp/chromedp"+ "github.com/pinchtab/pinchtab/internal/testbrowser"+)++// The target sits far enough down and across that a scroll is required to reach+// it, so the clip origin, the scroll offset and the viewport rect are three+// different numbers. At scroll 0 they collapse into one and the clip path's+// mixed-space subtraction is invisible.+const clipScrollFixtureHTML = `<body style="margin:0;width:2400px;height:2400px">+<div style="position:absolute;left:900px;top:1100px;width:120px;height:40px">+<button id="target">target</button>+</div>+</body>`++func newClipScrollFixture(t *testing.T) (context.Context, int64) {+ t.Helper()+ chromePath := testbrowser.Path(t)++ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(+ chromedp.DefaultExecAllocatorOptions[:],+ chromedp.ExecPath(chromePath),+ chromedp.UserDataDir(t.TempDir()),+ chromedp.Flag("headless", true),+ chromedp.Flag("no-sandbox", true),+ )...)+ ctx, cancelBrowser := chromedp.NewContext(alloc)+ ctx, cancelTimeout := context.WithTimeout(ctx, 60*time.Second)+ t.Cleanup(func() {+ cancelTimeout()+ cancelBrowser()+ cancelAlloc()+ })++ dataURL := "data:text/html;base64," + base64.StdEncoding.EncodeToString([]byte(clipScrollFixtureHTML))+ var scrolled bool+ if err := chromedp.Run(ctx,+ chromedp.Navigate(dataURL),+ chromedp.WaitVisible("#target", chromedp.ByID),+ chromedp.Evaluate(`(() => { window.scrollTo(300, 400); return window.scrollY === 400; })()`, &scrolled),+ ); err != nil {+ t.Fatal(err)+ }+ if !scrolled {+ t.Fatal("fixture did not scroll")+ }++ rawNodes, err := FetchAXTree(ctx)+ if err != nil {+ t.Fatal(err)+ }+ nodes, _ := BuildSnapshot(rawNodes, "", -1)+ for _, n := range nodes {+ if n.Role == "button" && n.Name == "target" && n.NodeID != 0 {+ return ctx, n.NodeID+ }+ }+ t.Fatalf("no button named \"target\" in the snapshot (%d nodes)", len(nodes))+ return nil, 0+}++func boundsForNode(t *testing.T, res *PairedResult, nodeID int64) BoundingBox {+ t.Helper()+ for _, n := range res.Nodes {+ if n.NodeID == nodeID && n.BoundingBox != nil {+ return *n.BoundingBox+ }+ }+ t.Fatalf("capture produced no bounding box for node %d (%d nodes)", nodeID, len(res.Nodes))+ return BoundingBox{}+}++func clientRectOfTarget(t *testing.T, ctx context.Context) (x, y float64) {+ t.Helper()+ var rect struct {+ X float64 `json:"x"`+ Y float64 `json:"y"`+ }+ if err := chromedp.Run(ctx, chromedp.Evaluate(+ `(() => { const r = document.getElementById('target').getBoundingClientRect(); return {x: r.x, y: r.y}; })()`,+ &rect)); err != nil {+ t.Fatal(err)+ }+ return rect.X, rect.Y+}++// The capture path measures the content box and getBoundingClientRect the border+// box, so they differ by the button's border and padding — single digits, not+// the hundreds of pixels a mixed-up coordinate space introduces.+const captureBoxSlack = 12.0++func assertNearRect(t *testing.T, label string, got, want float64) {+ t.Helper()+ if diff := got - want; diff > captureBoxSlack || diff < -captureBoxSlack {+ t.Errorf("%s = %.2f, want %.2f (off by %.2f)", label, got, want, diff)+ }+}++// All three spaces PairedCapture can report, on one scrolled page, each checked+// against the browser's own rect. Each space must mean what CoordinateSpace+// calls it: viewport is the raw rect, document is the rect plus the scroll, and+// clip is measured from the clip origin.+func TestPairedCaptureBoundsInEveryCoordinateSpaceWhenScrolled(t *testing.T) {+ t.Run("viewport", func(t *testing.T) {+ ctx, nodeID := newClipScrollFixture(t)++ res, err := PairedCapture(ctx, CaptureOpts{WithBounds: true, MaxDepth: -1, Image: ScreenshotOpts{Format: ScreenshotFormatPng}})+ if err != nil {+ t.Fatal(err)+ }+ if res.CoordinateSpace != "viewport" {+ t.Fatalf("CoordinateSpace = %q, want %q", res.CoordinateSpace, "viewport")+ }+ if res.Viewport.ScrollY == 0 || res.Viewport.ScrollX == 0 {+ t.Fatalf("capture did not see a scrolled page: %+v", res.Viewport)+ }++ box := boundsForNode(t, res, nodeID)+ rectX, rectY := clientRectOfTarget(t, ctx)+ assertNearRect(t, "viewport x", box.X, rectX)+ assertNearRect(t, "viewport y", box.Y, rectY)+ })++ t.Run("document", func(t *testing.T) {+ ctx, nodeID := newClipScrollFixture(t)++ res, err := PairedCapture(ctx, CaptureOpts{+ WithBounds: true,+ MaxDepth: -1,+ Image: ScreenshotOpts{Format: ScreenshotFormatPng, BeyondViewport: true},+ })+ if err != nil {+ t.Fatal(err)+ }+ if res.CoordinateSpace != "document" {+ t.Fatalf("CoordinateSpace = %q, want %q", res.CoordinateSpace, "document")+ }++ box := boundsForNode(t, res, nodeID)+ rectX, rectY := clientRectOfTarget(t, ctx)+ assertNearRect(t, "document x", box.X, rectX+res.Viewport.ScrollX)+ assertNearRect(t, "document y", box.Y, rectY+res.Viewport.ScrollY)+ })++ // The one place two independently-derived spaces meet: the clip origin comes+ // from ScreenshotClipForNode's JS, which adds window.scrollX/scrollY, so it is+ // document-relative and the bounds it is subtracted from must be too. The+ // clip is the target's own border box, so the target must land at the clip's+ // origin — anything else is the scroll leaking in.+ t.Run("clip", func(t *testing.T) {+ ctx, nodeID := newClipScrollFixture(t)++ clip, err := ScreenshotClipForNode(ctx, nodeID)+ if err != nil {+ t.Fatal(err)+ }+ if clip.X < 100 || clip.Y < 100 {+ t.Fatalf("clip origin %+v is too close to the document origin for this test to distinguish spaces", clip)+ }++ res, err := PairedCapture(ctx, CaptureOpts{+ WithBounds: true,+ MaxDepth: -1,+ Image: ScreenshotOpts{Format: ScreenshotFormatPng, Clip: clip},+ })+ if err != nil {+ t.Fatal(err)+ }+ if res.CoordinateSpace != "clip" {+ t.Fatalf("CoordinateSpace = %q, want %q", res.CoordinateSpace, "clip")+ }++ box := boundsForNode(t, res, nodeID)+ assertNearRect(t, "clip x", box.X, 0)+ assertNearRect(t, "clip y", box.Y, 0)+ })+}diff --git a/internal/bridge/observe/bounds.go b/internal/bridge/observe/bounds.goindex ccd780e5..7fbca5c2 100644--- a/internal/bridge/observe/bounds.go+++ b/internal/bridge/observe/bounds.go@@ -65,10 +65,10 @@ func FetchLayout(ctx context.Context) (ViewportInfo, error) {// exactly — which is what makes it correct for nodes inside iframes, where a// rect taken in the element's own context is frame-relative.//-// pageCoords=true leaves boxes in that space for beyondViewport/clip captures.-// pageCoords=false subtracts the current scroll offset; on a scrolled page that-// shifts an already-viewport-relative box by the scroll, which is its own-// defect and is deliberately left alone here.+// pageCoords=false leaves boxes in that viewport space. pageCoords=true adds+// the scroll offset to reach document space, which is what beyondViewport and+// clip captures report and what projectBoundsToClip subtracts a clip origin+// from — that origin is document-relative, so both sides must be.//// Visibility heuristic: a node is Visible if its rect has non-zero area and// intersects the viewport. The check is intentionally cheap — strict@@ -83,9 +83,9 @@ func AnnotateBounds(ctx context.Context, nodes []A11yNode, pageCoords bool, vp Vcontinue}visible := isVisible(box, true, vp)- if !pageCoords {- box.X -= vp.ScrollX- box.Y -= vp.ScrollY+ if pageCoords {+ box.X += vp.ScrollX+ box.Y += vp.ScrollY}nodes[i].BoundingBox = &boxnodes[i].Visible = visiblediff --git a/internal/bridge/observe/bounds_test.go b/internal/bridge/observe/bounds_test.goindex 3b02c1ed..6828164a 100644--- a/internal/bridge/observe/bounds_test.go+++ b/internal/bridge/observe/bounds_test.go@@ -1,6 +1,15 @@package observe-import "testing"+import (+ "context"+ "encoding/base64"+ "fmt"+ "testing"+ "time"++ "github.com/chromedp/chromedp"+ "github.com/pinchtab/pinchtab/internal/testbrowser"+)func TestIsVisibleWithDocumentCoordinates(t *testing.T) {vp := ViewportInfo{Width: 800, Height: 600, ScrollX: 0, ScrollY: 1000}@@ -12,3 +21,184 @@ func TestIsVisibleWithDocumentCoordinates(t *testing.T) {t.Fatal("box above scrolled viewport should not be visible")}}++// scrollFixtureHTML is tall enough to scroll in both axes, with a target that+// sits below the fold so a scroll is required to bring it into view. Only a+// scrolled page can tell the three coordinate spaces apart — at scroll 0 every+// transform is the identity, which is why this defect went unnoticed.+const scrollFixtureHTML = `<body style="margin:0;width:3000px;height:3000px">+<div style="position:absolute;left:900px;top:1200px;width:120px;height:40px">+<button id="target">target</button>+</div>+</body>`++type scrollFixture struct {+ ctx context.Context+ nodeID int64+}++func newScrollFixture(t *testing.T, scrollX, scrollY int) scrollFixture {+ t.Helper()+ chromePath := testbrowser.Path(t)++ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(+ chromedp.DefaultExecAllocatorOptions[:],+ chromedp.ExecPath(chromePath),+ chromedp.UserDataDir(t.TempDir()),+ chromedp.Flag("headless", true),+ chromedp.Flag("no-sandbox", true),+ )...)+ ctx, cancelBrowser := chromedp.NewContext(alloc)+ ctx, cancelTimeout := context.WithTimeout(ctx, 30*time.Second)+ t.Cleanup(func() {+ cancelTimeout()+ cancelBrowser()+ cancelAlloc()+ })++ dataURL := "data:text/html;base64," + base64.StdEncoding.EncodeToString([]byte(scrollFixtureHTML))+ if err := chromedp.Run(ctx,+ chromedp.Navigate(dataURL),+ chromedp.WaitVisible("#target", chromedp.ByID),+ ); err != nil {+ t.Fatal(err)+ }+ if err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {+ var done bool+ return chromedp.Evaluate(scrollScript(scrollX, scrollY), &done).Do(ctx)+ })); err != nil {+ t.Fatal(err)+ }++ return scrollFixture{ctx: ctx, nodeID: targetNodeID(t, ctx)}+}++func scrollScript(x, y int) string {+ return fmt.Sprintf(`(() => { window.scrollTo(%d, %d); return window.scrollY === %d; })()`, x, y, y)+}++func targetNodeID(t *testing.T, ctx context.Context) int64 {+ t.Helper()+ rawNodes, err := FetchAXTree(ctx)+ if err != nil {+ t.Fatal(err)+ }+ nodes, _ := BuildSnapshot(rawNodes, "", -1)+ for _, n := range nodes {+ if n.Role == "button" && n.Name == "target" && n.NodeID != 0 {+ return n.NodeID+ }+ }+ t.Fatalf("no button named %q in the snapshot (%d nodes)", "target", len(nodes))+ return 0+}++// clientRect is the browser's own answer for the target, in viewport space.+func (f scrollFixture) clientRect(t *testing.T) BoundingBox {+ t.Helper()+ var rect struct {+ X float64 `json:"x"`+ Y float64 `json:"y"`+ W float64 `json:"width"`+ H float64 `json:"height"`+ }+ if err := chromedp.Run(f.ctx, chromedp.Evaluate(+ `(() => { const r = document.getElementById('target').getBoundingClientRect(); return {x: r.x, y: r.y, width: r.width, height: r.height}; })()`,+ &rect)); err != nil {+ t.Fatal(err)+ }+ return BoundingBox{X: rect.X, Y: rect.Y, W: rect.W, H: rect.H}+}++func (f scrollFixture) annotate(t *testing.T, pageCoords bool) (BoundingBox, bool, ViewportInfo) {+ t.Helper()+ vp, err := FetchLayout(f.ctx)+ if err != nil {+ t.Fatal(err)+ }+ nodes := []A11yNode{{NodeID: f.nodeID}}+ if err := AnnotateBounds(f.ctx, nodes, pageCoords, vp); err != nil {+ t.Fatal(err)+ }+ if nodes[0].BoundingBox == nil {+ t.Fatal("AnnotateBounds produced no bounding box for the target")+ }+ return *nodes[0].BoundingBox, nodes[0].Visible, vp+}++// getBoxModel reports the content box and getBoundingClientRect the border box,+// so the two differ by the button's border and padding — single digits, nothing+// like the scroll offset a wrong transform introduces.+const boxModelSlack = 12.0++func assertNear(t *testing.T, label string, got, want float64) {+ t.Helper()+ if diff := got - want; diff > boxModelSlack || diff < -boxModelSlack {+ t.Errorf("%s = %.2f, want %.2f (off by %.2f)", label, got, want, diff)+ }+}++// The default /capture path. The raw quad is already viewport-relative, so+// pageCoords=false must hand it back untouched — subtracting the scroll moved+// every annotated bound on every scrolled page a full scrollY away.+func TestAnnotateBoundsViewportSpaceMatchesClientRectWhenScrolled(t *testing.T) {+ f := newScrollFixture(t, 300, 400)++ box, _, vp := f.annotate(t, false)+ if vp.ScrollY == 0 || vp.ScrollX == 0 {+ t.Fatalf("fixture did not scroll: %+v", vp)+ }+ rect := f.clientRect(t)++ assertNear(t, "viewport x", box.X, rect.X)+ assertNear(t, "viewport y", box.Y, rect.Y)+}++// The "document" label has to be true: beyondViewport and clip captures report+// this space, and a caller doing its own document arithmetic from the label was+// silently off by the scroll.+func TestAnnotateBoundsDocumentSpaceAddsScrollWhenScrolled(t *testing.T) {+ f := newScrollFixture(t, 300, 400)++ box, _, vp := f.annotate(t, true)+ rect := f.clientRect(t)++ assertNear(t, "document x", box.X, rect.X+vp.ScrollX)+ assertNear(t, "document y", box.Y, rect.Y+vp.ScrollY)++ if box.Y-rect.Y < vp.ScrollY/2 {+ t.Errorf("document y (%.2f) is not a full scroll (%.2f) past the viewport y (%.2f)", box.Y, vp.ScrollY, rect.Y)+ }+}++// At scroll 0 both spaces coincide, which is why this was invisible until a+// fixture scrolled. Pins that the fix moved nothing for unscrolled pages.+func TestAnnotateBoundsSpacesAgreeWhenUnscrolled(t *testing.T) {+ f := newScrollFixture(t, 0, 0)++ viewportBox, viewportVisible, vp := f.annotate(t, false)+ documentBox, documentVisible, _ := f.annotate(t, true)++ if vp.ScrollX != 0 || vp.ScrollY != 0 {+ t.Fatalf("fixture unexpectedly scrolled: %+v", vp)+ }+ if viewportBox != documentBox {+ t.Errorf("unscrolled page: viewport %+v and document %+v must be identical", viewportBox, documentBox)+ }+ if viewportVisible != documentVisible {+ t.Errorf("unscrolled page: Visible differs between spaces (%v vs %v)", viewportVisible, documentVisible)+ }+}++// Visible is a viewport-intersection test taken before the transform, so it must+// not move when the caller asks for document space.+func TestAnnotateBoundsVisibleIsIndependentOfCoordinateSpace(t *testing.T) {+ f := newScrollFixture(t, 300, 400)++ _, viewportVisible, _ := f.annotate(t, false)+ _, documentVisible, _ := f.annotate(t, true)++ if viewportVisible != documentVisible {+ t.Errorf("Visible = %v in viewport space and %v in document space; it is computed from the same viewport-relative box either way", viewportVisible, documentVisible)+ }+}