High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix: /box returns frame-relative coordinates for cross-frame refs PIN-032
Luigi Agosti95d4b0137d52
diff --git a/internal/bridge/cdpops/geometry.go b/internal/bridge/cdpops/geometry.goindex 63086e8c..f0db509c 100644--- a/internal/bridge/cdpops/geometry.go+++ b/internal/bridge/cdpops/geometry.go@@ -97,73 +97,15 @@ func GetElementCenterJS(ctx context.Context, backendNodeID int64) (float64, floareturn callRes.Result.Value.X, callRes.Result.Value.Y, nil}-// ScrollIntoViewAndGetBox scrolls the element into view and returns its box.-func ScrollIntoViewAndGetBox(ctx context.Context, nodeID int64) (map[string]any, error) {+// ScrollIntoViewIfNeeded scrolls the element into view. Measuring the resulting+// box is deliberately not done here: getBoundingClientRect in the element's own+// context is frame-relative, so the caller measures through the shared+// document-relative box-model path instead.+func ScrollIntoViewIfNeeded(ctx context.Context, nodeID int64) error {if err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {return chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.scrollIntoViewIfNeeded", map[string]any{"backendNodeId": nodeID}, nil)})); err != nil {- return nil, fmt.Errorf("scrollIntoViewIfNeeded: %w", err)+ return fmt.Errorf("scrollIntoViewIfNeeded: %w", err)}-- var resolveResult json.RawMessage- if err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {- return chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.resolveNode", map[string]any{- "backendNodeId": nodeID,- }, &resolveResult)- })); err != nil {- return nil, fmt.Errorf("DOM.resolveNode: %w", err)- }-- var resolved struct {- Object struct {- ObjectID string `json:"objectId"`- } `json:"object"`- }- if err := json.Unmarshal(resolveResult, &resolved); err != nil {- return nil, err- }- if resolved.Object.ObjectID == "" {- return nil, fmt.Errorf("element not found in DOM (backendNodeId=%d)", nodeID)- }-- const boxFn = `function() {- var r = this.getBoundingClientRect();- return { x: Math.round(r.x), y: Math.round(r.y), width: Math.round(r.width), height: Math.round(r.height) };- }`-- var callResult json.RawMessage- if err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {- return chromedp.FromContext(ctx).Target.Execute(ctx, "Runtime.callFunctionOn", map[string]any{- "functionDeclaration": boxFn,- "objectId": resolved.Object.ObjectID,- "returnByValue": true,- }, &callResult)- })); err != nil {- return nil, fmt.Errorf("getBoundingClientRect: %w", err)- }-- var callRes struct {- Result struct {- Value struct {- X float64 `json:"x"`- Y float64 `json:"y"`- Width float64 `json:"width"`- Height float64 `json:"height"`- } `json:"value"`- } `json:"result"`- }- if err := json.Unmarshal(callResult, &callRes); err != nil {- return nil, err- }-- box := callRes.Result.Value- return map[string]any{- "scrolled": true,- "box": map[string]any{- "x": box.X,- "y": box.Y,- "width": box.Width,- "height": box.Height,- },- }, nil+ return nil}diff --git a/internal/bridge/cdpops_facade.go b/internal/bridge/cdpops_facade.goindex ba6e4973..22c43cac 100644--- a/internal/bridge/cdpops_facade.go+++ b/internal/bridge/cdpops_facade.go@@ -2,6 +2,8 @@ package bridgeimport ("context"+ "fmt"+ "math""time"bridgecdpops "github.com/pinchtab/pinchtab/internal/bridge/cdpops"@@ -47,8 +49,28 @@ func getElementCenterJS(ctx context.Context, backendNodeID int64) (float64, floareturn bridgecdpops.GetElementCenterJS(ctx, backendNodeID)}+// ScrollIntoViewAndGetBox scrolls the element into view and reports the box it+// ended up at, in the same top-level viewport space /box and /capture report.+// It shared /box's defect until now: measuring with getBoundingClientRect in the+// element's own context returns frame-relative coordinates for anything inside+// an iframe.func ScrollIntoViewAndGetBox(ctx context.Context, nodeID int64) (map[string]any, error) {- return bridgecdpops.ScrollIntoViewAndGetBox(ctx, nodeID)+ if err := bridgecdpops.ScrollIntoViewIfNeeded(ctx, nodeID); err != nil {+ return nil, err+ }+ box, ok := ElementBorderBox(ctx, nodeID)+ if !ok {+ return nil, fmt.Errorf("element has no box model (backendNodeId=%d)", nodeID)+ }+ return map[string]any{+ "scrolled": true,+ "box": map[string]any{+ "x": math.Round(box.X),+ "y": math.Round(box.Y),+ "width": math.Round(box.W),+ "height": math.Round(box.H),+ },+ }, nil}func PointerPointForNode(ctx context.Context, nodeID int64, requireTopMost bool) (float64, float64, error) {diff --git a/internal/bridge/observe/bounds.go b/internal/bridge/observe/bounds.goindex 7839b53e..ccd780e5 100644--- a/internal/bridge/observe/bounds.go+++ b/internal/bridge/observe/bounds.go@@ -60,10 +60,15 @@ func FetchLayout(ctx context.Context) (ViewportInfo, error) {// Each node costs one DOM.getBoxModel round trip; for the typical// FilterInteractive snapshot of <50 nodes the total budget is ~250ms.//-// DOM.getBoxModel returns document-relative CSS coordinates. pageCoords=true-// leaves boxes in that space for beyondViewport/clip captures. pageCoords=false-// projects boxes into viewport coordinates by subtracting the current scroll-// offset, matching the default viewport-only screenshot.+// DOM.getBoxModel returns CSS coordinates relative to the MAIN FRAME'S VIEWPORT+// — measured against getBoundingClientRect on a scrolled page, the two agree+// exactly — which is what makes it correct for nodes inside iframes, where a+// rect taken in the element's own context is frame-relative.+//+// pageCoords=true leaves boxes in that space for beyondViewport/clip captures.+// pageCoords=false subtracts the current scroll offset; on a scrolled page that+// shifts an already-viewport-relative box by the scroll, which is its own+// defect and is deliberately left alone here.//// Visibility heuristic: a node is Visible if its rect has non-zero area and// intersects the viewport. The check is intentionally cheap — strict@@ -89,6 +94,26 @@ func AnnotateBounds(ctx context.Context, nodes []A11yNode, pageCoords bool, vp V}func getBoxAABB(ctx context.Context, backendNodeID int64) (BoundingBox, bool) {+ return boxModelAABB(ctx, backendNodeID, boxQuadContent)+}++// ElementBorderBox is the border-box rectangle of a node in document-relative+// CSS coordinates, the same measurement the capture path uses. It is the+// cross-frame-correct alternative to evaluating getBoundingClientRect in the+// element's own context, which is relative to the document the element lives in+// and therefore frame-relative for anything inside an iframe.+func ElementBorderBox(ctx context.Context, backendNodeID int64) (BoundingBox, bool) {+ return boxModelAABB(ctx, backendNodeID, boxQuadBorder)+}++type boxQuad string++const (+ boxQuadContent boxQuad = "content"+ boxQuadBorder boxQuad = "border"+)++func boxModelAABB(ctx context.Context, backendNodeID int64, quad boxQuad) (BoundingBox, bool) {var result json.RawMessageerr := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {return chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.getBoxModel", map[string]any{@@ -101,12 +126,16 @@ func getBoxAABB(ctx context.Context, backendNodeID int64) (BoundingBox, bool) {var box struct {Model struct {Content []float64 `json:"content"`+ Border []float64 `json:"border"`} `json:"model"`}if err := json.Unmarshal(result, &box); err != nil {return BoundingBox{}, false}q := box.Model.Content+ if quad == boxQuadBorder {+ q = box.Model.Border+ }if len(q) < 8 {return BoundingBox{}, false}diff --git a/internal/bridge/observe_facade.go b/internal/bridge/observe_facade.goindex 2e34f845..40a494de 100644--- a/internal/bridge/observe_facade.go+++ b/internal/bridge/observe_facade.go@@ -72,6 +72,10 @@ func AnnotateBounds(ctx context.Context, nodes []A11yNode, pageCoords bool, vp Vreturn bridgeobserve.AnnotateBounds(ctx, nodes, pageCoords, vp)}+func ElementBorderBox(ctx context.Context, backendNodeID int64) (BoundingBox, bool) {+ return bridgeobserve.ElementBorderBox(ctx, backendNodeID)+}+func FetchAXTree(ctx context.Context) ([]RawAXNode, error) {return bridgeobserve.FetchAXTree(ctx)}diff --git a/internal/handlers/box.go b/internal/handlers/box.goindex e35cd43e..75b27f2b 100644--- a/internal/handlers/box.go+++ b/internal/handlers/box.go@@ -2,7 +2,10 @@ package handlersimport ("context"+ "fmt""net/http"++ "github.com/pinchtab/pinchtab/internal/bridge")type boundingBox struct {@@ -42,19 +45,35 @@ func (h *Handlers) HandleTabGetBox(w http.ResponseWriter, r *http.Request) {h.withPathTabID(w, r, h.HandleGetBox)}-// getElementBox resolves a unified selector to a DOM node and returns its bounding client rect.+// getElementBox resolves a unified selector to a DOM node and returns its+// border box in top-level viewport coordinates.+//+// It measures through the same DOM.getBoxModel path the capture path uses, which+// reports every node in the top-level frame's coordinate space regardless of+// which frame the node lives in (measured: identical to a top-level+// getBoundingClientRect, scrolled or not). Evaluating getBoundingClientRect in+// the element's own context — the obvious alternative — is relative to the+// document the element lives in, so an element inside an iframe came back+// frame-relative and its rectangle landed outside the iframe containing it.func (h *Handlers) getElementBox(ctx context.Context, tabID, sel string) (*boundingBox, error) {nodeID, err := h.resolveElementNodeID(ctx, tabID, sel)if err != nil {return nil, err}- var result boundingBox- err = h.Bridge.CallFunctionOnNode(ctx, nodeID,- `function() { var r = this.getBoundingClientRect(); return {x: r.x, y: r.y, width: r.width, height: r.height, top: r.top, right: r.right, bottom: r.bottom, left: r.left}; }`,- nil, &result)- if err != nil {- return nil, err+ box, ok := bridge.ElementBorderBox(ctx, nodeID)+ if !ok {+ return nil, fmt.Errorf("element has no box model: %s", sel)}- return &result, nil++ return &boundingBox{+ X: box.X,+ Y: box.Y,+ Width: box.W,+ Height: box.H,+ Top: box.Y,+ Right: box.X + box.W,+ Bottom: box.Y + box.H,+ Left: box.X,+ }, nil}diff --git a/internal/handlers/box_frame_test.go b/internal/handlers/box_frame_test.gonew file mode 100644index 00000000..255f1f97--- /dev/null+++ b/internal/handlers/box_frame_test.go@@ -0,0 +1,281 @@+package handlers++import (+ "context"+ "encoding/base64"+ "testing"+ "time"++ "github.com/chromedp/chromedp"+ "github.com/pinchtab/pinchtab/internal/bridge"+ "github.com/pinchtab/pinchtab/internal/config"+ "github.com/pinchtab/pinchtab/internal/testbrowser"+)++// frameFixtureHTML mirrors the reported fixture: two top-level buttons and an+// iframe holding a third. The defect is invisible on a single-document page —+// only a node inside the frame can be measured in the wrong origin.+const frameFixtureHTML = `<h1>Dialog fixture</h1>+<button id="a">Alert</button>+<button id="c">Confirm</button>+<iframe id="f" srcdoc="<p>inside frame</p><button id='ib'>frame button</button>" width="300" height="100"></iframe>`++type frameFixture struct {+ handlers *Handlers+ ctx context.Context+ nodes []bridge.A11yNode+}++func (f frameFixture) nodeIDByName(t *testing.T, role, name string) int64 {+ t.Helper()+ for _, node := range f.nodes {+ if node.Role == role && node.Name == name && node.NodeID != 0 {+ return node.NodeID+ }+ }+ t.Fatalf("no %s node named %q in the snapshot (%d nodes)", role, name, len(f.nodes))+ return 0+}++func (f frameFixture) iframeNodeID(t *testing.T) int64 {+ t.Helper()+ for _, node := range f.nodes {+ if node.Tag == "iframe" && node.NodeID != 0 {+ return node.NodeID+ }+ }+ for _, node := range f.nodes {+ if node.Role == "Iframe" && node.NodeID != 0 {+ return node.NodeID+ }+ }+ t.Fatalf("no iframe node in the snapshot (%d nodes)", len(f.nodes))+ return 0+}++func newFrameFixture(t *testing.T) frameFixture {+ t.Helper()+ chromePath := testbrowser.Path(t)++ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(+ chromedp.DefaultExecAllocatorOptions[:],+ chromedp.ExecPath(chromePath),+ chromedp.UserDataDir(t.TempDir()),+ chromedp.Flag("headless", true),+ chromedp.Flag("no-sandbox", true),+ )...)+ ctx, cancelBrowser := chromedp.NewContext(alloc)+ ctx, cancelTimeout := context.WithTimeout(ctx, 30*time.Second)+ t.Cleanup(func() {+ cancelTimeout()+ cancelBrowser()+ cancelAlloc()+ })++ dataURL := "data:text/html;base64," + base64.StdEncoding.EncodeToString([]byte(frameFixtureHTML))+ if err := chromedp.Run(ctx, chromedp.Navigate(dataURL), chromedp.WaitVisible("#f", chromedp.ByID)); err != nil {+ t.Fatal(err)+ }++ cfg := &config.RuntimeConfig{ActionTimeout: 5 * time.Second, DefaultBrowser: config.BrowserChrome, StateDir: t.TempDir()}+ b := bridge.New(context.Background(), ctx, cfg)+ b.RegisterTab("tab-frame", ctx)++ rawNodes, err := bridge.FetchAXTree(ctx)+ if err != nil {+ t.Fatal(err)+ }+ nodes, _ := bridge.BuildSnapshot(rawNodes, "", -1)+ if err := bridge.EnrichA11yNodesWithDOMMetadata(ctx, nodes); err != nil {+ t.Fatal(err)+ }++ return frameFixture{handlers: New(b, cfg, nil, nil, nil), ctx: ctx, nodes: nodes}+}++func boxForNode(t *testing.T, f frameFixture, nodeID int64) boundingBox {+ t.Helper()+ f.handlers.Bridge.SetRefCache("tab-frame", &bridge.RefCache{Targets: map[string]bridge.RefTarget{+ "e0": {BackendNodeID: nodeID},+ }})+ box, err := f.handlers.getElementBox(f.ctx, "tab-frame", "ref:e0")+ if err != nil {+ t.Fatalf("getElementBox: %v", err)+ }+ return *box+}++func boxContains(outer, inner boundingBox) bool {+ return inner.Left >= outer.Left && inner.Top >= outer.Top &&+ inner.Right <= outer.Right && inner.Bottom <= outer.Bottom+}++// The reported defect: an element inside an iframe was measured in the frame's+// own origin, so its rectangle landed entirely outside the iframe that contains+// it — geometrically impossible in page space.+func TestBoxForInFrameRefIsInsideItsIframe(t *testing.T) {+ f := newFrameFixture(t)++ iframeBox := boxForNode(t, f, f.iframeNodeID(t))+ buttonBox := boxForNode(t, f, f.nodeIDByName(t, "button", "frame button"))++ if buttonBox.Width <= 0 || buttonBox.Height <= 0 {+ t.Fatalf("in-frame button box has no area: %+v", buttonBox)+ }+ if !boxContains(iframeBox, buttonBox) {+ t.Fatalf("in-frame button %+v is not inside its iframe %+v — the frame offset was not applied", buttonBox, iframeBox)+ }+}++// /box and the capture path must report the same coordinate space. The capture+// path measures the content box and /box the border box, so allow a small+// padding/border difference but nothing like the frame-sized offset the defect+// produced.+func TestBoxAgreesWithCaptureCoordinateSpaceForInFrameRef(t *testing.T) {+ f := newFrameFixture(t)+ nodeID := f.nodeIDByName(t, "button", "frame button")++ boxed := boxForNode(t, f, nodeID)++ vp, err := bridge.FetchLayout(f.ctx)+ if err != nil {+ t.Fatal(err)+ }+ captureNodes := []bridge.A11yNode{{NodeID: nodeID}}+ if err := bridge.AnnotateBounds(f.ctx, captureNodes, false, vp); err != nil {+ t.Fatal(err)+ }+ captured := captureNodes[0].BoundingBox+ if captured == nil {+ t.Fatal("capture path produced no bounding box for the in-frame node")+ }++ const boxModelSlack = 12.0+ if diff := boxed.X - captured.X; diff > boxModelSlack || diff < -boxModelSlack {+ t.Errorf("x differs by %.2f (box %.2f vs capture %.2f) — more than a border/padding difference", diff, boxed.X, captured.X)+ }+ if diff := boxed.Y - captured.Y; diff > boxModelSlack || diff < -boxModelSlack {+ t.Errorf("y differs by %.2f (box %.2f vs capture %.2f) — more than a border/padding difference", diff, boxed.Y, captured.Y)+ }+}++// Top-level refs keep the coordinates they already had: same origin, and the+// border box getBoundingClientRect reported.+func TestBoxForTopLevelRefMatchesGetBoundingClientRect(t *testing.T) {+ f := newFrameFixture(t)++ boxed := boxForNode(t, f, f.nodeIDByName(t, "button", "Alert"))++ var rect struct {+ X float64 `json:"x"`+ Y float64 `json:"y"`+ Width float64 `json:"width"`+ Height float64 `json:"height"`+ }+ if err := chromedp.Run(f.ctx, chromedp.Evaluate(+ `(() => { const r = document.getElementById('a').getBoundingClientRect(); return {x: r.x, y: r.y, width: r.width, height: r.height}; })()`,+ &rect)); err != nil {+ t.Fatal(err)+ }++ const tolerance = 0.5+ for _, cmp := range []struct {+ name string+ got, want float64+ }{+ {"x", boxed.X, rect.X},+ {"y", boxed.Y, rect.Y},+ {"width", boxed.Width, rect.Width},+ {"height", boxed.Height, rect.Height},+ } {+ if diff := cmp.got - cmp.want; diff > tolerance || diff < -tolerance {+ t.Errorf("%s = %.2f, want %.2f (getBoundingClientRect)", cmp.name, cmp.got, cmp.want)+ }+ }+}++// /visible is the sibling that evaluates the same raw rect in the element's own+// context. Frame-local geometry is the right answer for a visibility question —+// an element can be visible within its frame — so this pins the behaviour that+// its description documents rather than changing it.+func TestVisibleIsFrameLocalForInFrameRef(t *testing.T) {+ f := newFrameFixture(t)+ nodeID := f.nodeIDByName(t, "button", "frame button")++ f.handlers.Bridge.SetRefCache("tab-frame", &bridge.RefCache{Targets: map[string]bridge.RefTarget{+ "e0": {BackendNodeID: nodeID},+ }})+ visible, err := f.handlers.getElementVisible(f.ctx, "tab-frame", "ref:e0")+ if err != nil {+ t.Fatalf("getElementVisible: %v", err)+ }+ if !visible {+ t.Fatal("an in-frame button that is on screen reported not visible")+ }+}++// The reported space is the top-level viewport, so a scrolled page must not+// shift the box by the scroll offset: document-relative measurement has to be+// projected back into viewport space.+func TestBoxForTopLevelRefTracksScroll(t *testing.T) {+ f := newFrameFixture(t)++ if err := chromedp.Run(f.ctx, chromedp.Evaluate(+ `(() => { document.body.style.height = '3000px'; window.scrollTo(0, 400); return window.scrollY; })()`,+ nil)); err != nil {+ t.Fatal(err)+ }++ boxed := boxForNode(t, f, f.nodeIDByName(t, "button", "Alert"))++ var rect struct {+ X float64 `json:"x"`+ Y float64 `json:"y"`+ }+ if err := chromedp.Run(f.ctx, chromedp.Evaluate(+ `(() => { const r = document.getElementById('a').getBoundingClientRect(); return {x: r.x, y: r.y}; })()`,+ &rect)); err != nil {+ t.Fatal(err)+ }++ const tolerance = 0.5+ if diff := boxed.Y - rect.Y; diff > tolerance || diff < -tolerance {+ t.Fatalf("y = %.2f on a scrolled page, want %.2f (viewport space)", boxed.Y, rect.Y)+ }+}++// scrollintoview shared the same defect: it measured the scrolled-to element+// with the same raw rect, so an in-frame ref reported a frame-relative box.+func TestScrollIntoViewBoxForInFrameRefIsInsideItsIframe(t *testing.T) {+ f := newFrameFixture(t)++ iframeBox := boxForNode(t, f, f.iframeNodeID(t))++ result, err := bridge.ScrollIntoViewAndGetBox(f.ctx, f.nodeIDByName(t, "button", "frame button"))+ if err != nil {+ t.Fatalf("ScrollIntoViewAndGetBox: %v", err)+ }+ raw, ok := result["box"].(map[string]any)+ if !ok {+ t.Fatalf("result has no box: %+v", result)+ }+ scrolled := boundingBox{+ X: raw["x"].(float64),+ Y: raw["y"].(float64),+ Width: raw["width"].(float64),+ Height: raw["height"].(float64),+ }+ scrolled.Left, scrolled.Top = scrolled.X, scrolled.Y+ scrolled.Right, scrolled.Bottom = scrolled.X+scrolled.Width, scrolled.Y+scrolled.Height++ if scrolled.Width <= 0 || scrolled.Height <= 0 {+ t.Fatalf("scrollintoview box has no area: %+v", scrolled)+ }+ // Rounded to whole pixels by the action contract, so allow a pixel of slack+ // on each edge rather than demanding strict containment.+ const roundingSlack = 1.0+ if scrolled.Left < iframeBox.Left-roundingSlack || scrolled.Top < iframeBox.Top-roundingSlack ||+ scrolled.Right > iframeBox.Right+roundingSlack || scrolled.Bottom > iframeBox.Bottom+roundingSlack {+ t.Fatalf("scrollintoview box %+v is not inside its iframe %+v — the frame offset was not applied", scrolled, iframeBox)+ }+}diff --git a/internal/routes/routes.go b/internal/routes/routes.goindex b34d7685..cbb5a4ef 100644--- a/internal/routes/routes.go+++ b/internal/routes/routes.go@@ -95,8 +95,8 @@ var coreEndpoints = []Endpoint{{"GET", "/value", "Read form element value by ref", CapNone, true},{"GET", "/attr", "Read element HTML attribute by ref", CapNone, true},{"GET", "/count", "Count elements matching selector", CapNone, true},- {"GET", "/box", "Get element bounding box by ref", CapNone, true},- {"GET", "/visible", "Check if element is visible by ref", CapNone, true},+ {"GET", "/box", "Get element border box by ref, in top-level viewport coordinates (frame offsets applied, same space as /capture)", CapNone, true},+ {"GET", "/visible", "Check if element is visible by ref (geometry is frame-local: an element inside a scrolled-away iframe still reports visible within its own frame)", CapNone, true},{"GET", "/enabled", "Check if element is enabled by ref", CapNone, true},{"GET", "/checked", "Check if element is checked by ref", CapNone, true},{"GET", "/pdf", "Export as PDF (GET)", CapNone, true},