High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix: cdptk annotation rects resolve in the main world, and the frame walk loses offsets from the isolated handle (PIN-062)
Luigi Agosti30e87a7b4f9d
diff --git a/internal/bridge/isolated_world_test.go b/internal/bridge/isolated_world_test.goindex 157cac21..b3515927 100644--- a/internal/bridge/isolated_world_test.go+++ b/internal/bridge/isolated_world_test.go@@ -181,7 +181,6 @@ var mainWorldResolvers = map[string]string{"internal/bridge/cdpops/pointer.go": "pointer actions act on an already-chosen node","internal/bridge/cdpops/validation.go": "post-action validation reads an already-chosen node","internal/bridge/cdpops/geometry.go": "click-point geometry for an already-chosen node",- "internal/cdptk/annotate.go": "annotation reads already-chosen nodes",}func TestIsolatedWorldBoundaryCensus(t *testing.T) {diff --git a/internal/cdptk/annotate.go b/internal/cdptk/annotate.goindex 62d5048f..6fde3167 100644--- a/internal/cdptk/annotate.go+++ b/internal/cdptk/annotate.go@@ -23,24 +23,12 @@ func ScrollNodeIntoView(ctx context.Context, nodeID int64) error {// backend node id. Document scroll is intentionally NOT added; the overlay// injector adds scrollX/scrollY when placing absolute-positioned boxes.func AnnotationRectForNode(ctx context.Context, nodeID int64) (*AnnotationRect, error) {- var resolveResult json.RawMessage- if err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {- return chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.resolveNode", map[string]any{- "backendNodeId": nodeID,- }, &resolveResult)- })); err != nil {- return nil, err- }- var resolved struct {- Object struct {- ObjectID string `json:"objectId"`- } `json:"object"`- }- if err := json.Unmarshal(resolveResult, &resolved); err != nil {- return nil, err- }- if resolved.Object.ObjectID == "" {- return nil, fmt.Errorf("element not found (backendNodeId=%d)", nodeID)+ // Isolated world: boxFn reads getBoundingClientRect on this handle and on+ // every ancestor frame element, so a main-world handle would let page script+ // place the annotation boxes a vision model is then told to read.+ objectID, err := IsolatedNodeObjectID(ctx, nodeID)+ if err != nil {+ return nil, fmt.Errorf("resolve node %d: %w", nodeID, err)}// boxFn returns top-level viewport-relative rects. Walks frame ancestors,@@ -48,13 +36,18 @@ func AnnotationRectForNode(ctx context.Context, nodeID int64) (*AnnotationRect,// viewport). If we hit a null frameElement before reaching the top window// — typical cross-origin barrier — we flag the rect unprojectable rather// than emitting partial frame-local coordinates.+ // The frame walk starts from the NODE's own view, not the ambient `window`:+ // the handle runs in the world it was resolved into, which is the top frame's,+ // and a bare `window` there has an empty frameElement chain — every frame+ // offset would be silently dropped instead of flagged.const boxFn = `function() {+ const view = (this.ownerDocument && this.ownerDocument.defaultView) || window;const rect = this.getBoundingClientRect();let x = rect.x;let y = rect.y;let frameError = false;try {- let cur = window;+ let cur = view;while (cur && cur.parent && cur !== cur.parent) {const frameEl = cur.frameElement;if (!frameEl) {@@ -79,7 +72,7 @@ func AnnotationRectForNode(ctx context.Context, nodeID int64) (*AnnotationRect,if err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {return chromedp.FromContext(ctx).Target.Execute(ctx, "Runtime.callFunctionOn", map[string]any{"functionDeclaration": boxFn,- "objectId": resolved.Object.ObjectID,+ "objectId": objectID,"returnByValue": true,}, &callResult)})); err != nil {diff --git a/internal/cdptk/screenshot.go b/internal/cdptk/screenshot.goindex c8c47cc7..43815ba6 100644--- a/internal/cdptk/screenshot.go+++ b/internal/cdptk/screenshot.go@@ -28,12 +28,16 @@ func ClipForNode(ctx context.Context, nodeID int64, css1x bool) (*ScreenshotClip// Translate the element box into top-level page coordinates. captureScreenshot// clip coordinates are page-relative, so viewport-relative rects need scroll// offsets from the current document and each ancestor frame.+ // The frame walk starts from the NODE's own view for the same reason as the+ // bridge clip builder: the isolated handle runs in the top frame's world, so a+ // bare `window` there loses every frame offset.const boxFn = `function() {+ const view = (this.ownerDocument && this.ownerDocument.defaultView) || window;const rect = this.getBoundingClientRect();- let x = rect.left + (window.scrollX || window.pageXOffset || 0);- let y = rect.top + (window.scrollY || window.pageYOffset || 0);+ let x = rect.left + (view.scrollX || view.pageXOffset || 0);+ let y = rect.top + (view.scrollY || view.pageYOffset || 0);try {- let current = window;+ let current = view;while (current && current.parent && current !== current.parent) {const frameEl = current.frameElement;if (!frameEl) {@@ -54,7 +58,7 @@ func ClipForNode(ctx context.Context, nodeID int64, css1x bool) (*ScreenshotClipy,width: rect.width,height: rect.height,- dpr: window.devicePixelRatio || 1+ dpr: view.devicePixelRatio || 1};}`diff --git a/internal/cdptk/screenshot_test.go b/internal/cdptk/screenshot_test.gonew file mode 100644index 00000000..289f7f17--- /dev/null+++ b/internal/cdptk/screenshot_test.go@@ -0,0 +1,144 @@+package cdptk_test++import (+ "context"+ "encoding/base64"+ "testing"+ "time"++ "github.com/chromedp/chromedp"+ "github.com/pinchtab/pinchtab/internal/cdptk"+ "github.com/pinchtab/pinchtab/internal/testbrowser"+)++// An external test package: internal/testbrowser reaches cdptk through+// internal/browsers/chrome, so an in-package test importing it would cycle.++// The page redefines the method both clip builders read. In the main world it+// answers them; an isolated-world handle never sees it.+const hijackFixtureHTML = `<body style="margin:0">+<div id="target" style="position:absolute;left:40px;top:60px;width:120px;height:60px;background:#000"></div>+<script>+(() => {+ Element.prototype.getBoundingClientRect = function () {+ return {x: 999, y: 999, left: 999, top: 999, right: 1099, bottom: 1049, width: 100, height: 50};+ };+})();+</script>+</body>`++func newHijackFixture(t *testing.T) context.Context {+ t.Helper()+ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(+ chromedp.DefaultExecAllocatorOptions[:],+ chromedp.ExecPath(testbrowser.Path(t)),+ chromedp.UserDataDir(t.TempDir()),+ chromedp.Flag("headless", true),+ chromedp.Flag("no-sandbox", true),+ )...)+ ctx, cancelBrowser := chromedp.NewContext(alloc)+ ctx, cancelTimeout := context.WithTimeout(ctx, 30*time.Second)+ t.Cleanup(func() {+ cancelTimeout()+ cancelBrowser()+ cancelAlloc()+ })++ dataURL := "data:text/html;base64," + base64.StdEncoding.EncodeToString([]byte(hijackFixtureHTML))+ if err := chromedp.Run(ctx,+ chromedp.Navigate(dataURL),+ chromedp.WaitVisible("#target", chromedp.ByID),+ ); err != nil {+ t.Fatal(err)+ }+ return ctx+}++func targetNodeID(t *testing.T, ctx context.Context) int64 {+ t.Helper()+ var nodeID int64+ if err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {+ var doc struct {+ Root struct {+ NodeID int64 `json:"nodeId"`+ } `json:"root"`+ }+ if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.getDocument", map[string]any{"depth": -1}, &doc); err != nil {+ return err+ }+ var found struct {+ NodeID int64 `json:"nodeId"`+ }+ if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.querySelector", map[string]any{+ "nodeId": doc.Root.NodeID,+ "selector": "#target",+ }, &found); err != nil {+ return err+ }+ var described struct {+ Node struct {+ BackendNodeID int64 `json:"backendNodeId"`+ } `json:"node"`+ }+ if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.describeNode", map[string]any{+ "nodeId": found.NodeID,+ }, &described); err != nil {+ return err+ }+ nodeID = described.Node.BackendNodeID+ return nil+ })); err != nil {+ t.Fatal(err)+ }+ if nodeID == 0 {+ t.Fatal("fixture element #target has no backend node id")+ }+ return nodeID+}++// The cdptk analogue of TestScreenshotClipResistsMainWorldSubstitution: the clip+// origin must come from the element, not from the rectangle page script forges.+func TestClipForNodeResistsMainWorldSubstitution(t *testing.T) {+ ctx := newHijackFixture(t)+ nodeID := targetNodeID(t, ctx)++ clip, err := cdptk.ClipForNode(ctx, nodeID, true)+ if err != nil {+ t.Fatalf("ClipForNode: %v", err)+ }++ // The forged rect is the one value the page can produce; anything else means+ // the read happened somewhere the page could not reach.+ if clip.X == 999 || clip.Y == 999 || (clip.Width == 100 && clip.Height == 50) {+ t.Errorf("clip %+v matches the rectangle page script forged; the origin was read in the main world", clip)+ }+ if clip.Width != 120 || clip.Height != 60 {+ t.Errorf("clip %+v does not match the element's real 120x60 box", clip)+ }+}++// The same boundary on the annotation rect: its frame walk reads+// getBoundingClientRect too, and its output places the overlay boxes a vision+// model is told to read.+func TestAnnotationRectForNodeResistsMainWorldSubstitution(t *testing.T) {+ ctx := newHijackFixture(t)+ nodeID := targetNodeID(t, ctx)++ rect, err := cdptk.AnnotationRectForNode(ctx, nodeID)+ if err != nil {+ t.Fatalf("AnnotationRectForNode: %v", err)+ }+ if rect == nil {+ t.Fatal("AnnotationRectForNode returned no rect for a top-frame element")+ }++ if rect.X == 999 || rect.Y == 999 || (rect.W == 100 && rect.H == 50) {+ t.Errorf("rect %+v matches the rectangle page script forged; it was read in the main world", rect)+ }+ if rect.W != 120 || rect.H != 60 {+ t.Errorf("rect %+v does not match the element's real 120x60 box", rect)+ }+ if rect.X != 40 || rect.Y != 60 {+ t.Errorf("rect %+v is not the element's viewport position (40,60)", rect)+ }+}