Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix: cdptk annotation rects resolve in the main world, and the frame walk loses offsets from the isolated handle (PIN-062)

Luigi Agosti30e87a7b4f9d
diff --git a/internal/bridge/isolated_world_test.go b/internal/bridge/isolated_world_test.go
index 157cac21..b3515927 100644
--- a/internal/bridge/isolated_world_test.go
+++ b/internal/bridge/isolated_world_test.go
@@ -181,7 +181,6 @@ var mainWorldResolvers = map[string]string{
"internal/bridge/cdpops/pointer.go": "pointer actions act on an already-chosen node",
"internal/bridge/cdpops/validation.go": "post-action validation reads an already-chosen node",
"internal/bridge/cdpops/geometry.go": "click-point geometry for an already-chosen node",
- "internal/cdptk/annotate.go": "annotation reads already-chosen nodes",
}
func TestIsolatedWorldBoundaryCensus(t *testing.T) {
diff --git a/internal/cdptk/annotate.go b/internal/cdptk/annotate.go
index 62d5048f..6fde3167 100644
--- a/internal/cdptk/annotate.go
+++ b/internal/cdptk/annotate.go
@@ -23,24 +23,12 @@ func ScrollNodeIntoView(ctx context.Context, nodeID int64) error {
// backend node id. Document scroll is intentionally NOT added; the overlay
// injector adds scrollX/scrollY when placing absolute-positioned boxes.
func AnnotationRectForNode(ctx context.Context, nodeID int64) (*AnnotationRect, error) {
- var resolveResult json.RawMessage
- if err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
- return chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.resolveNode", map[string]any{
- "backendNodeId": nodeID,
- }, &resolveResult)
- })); err != nil {
- return nil, err
- }
- var resolved struct {
- Object struct {
- ObjectID string `json:"objectId"`
- } `json:"object"`
- }
- if err := json.Unmarshal(resolveResult, &resolved); err != nil {
- return nil, err
- }
- if resolved.Object.ObjectID == "" {
- return nil, fmt.Errorf("element not found (backendNodeId=%d)", nodeID)
+ // Isolated world: boxFn reads getBoundingClientRect on this handle and on
+ // every ancestor frame element, so a main-world handle would let page script
+ // place the annotation boxes a vision model is then told to read.
+ objectID, err := IsolatedNodeObjectID(ctx, nodeID)
+ if err != nil {
+ return nil, fmt.Errorf("resolve node %d: %w", nodeID, err)
}
// boxFn returns top-level viewport-relative rects. Walks frame ancestors,
@@ -48,13 +36,18 @@ func AnnotationRectForNode(ctx context.Context, nodeID int64) (*AnnotationRect,
// viewport). If we hit a null frameElement before reaching the top window
// — typical cross-origin barrier — we flag the rect unprojectable rather
// than emitting partial frame-local coordinates.
+ // The frame walk starts from the NODE's own view, not the ambient `window`:
+ // the handle runs in the world it was resolved into, which is the top frame's,
+ // and a bare `window` there has an empty frameElement chain — every frame
+ // offset would be silently dropped instead of flagged.
const boxFn = `function() {
+ const view = (this.ownerDocument && this.ownerDocument.defaultView) || window;
const rect = this.getBoundingClientRect();
let x = rect.x;
let y = rect.y;
let frameError = false;
try {
- let cur = window;
+ let cur = view;
while (cur && cur.parent && cur !== cur.parent) {
const frameEl = cur.frameElement;
if (!frameEl) {
@@ -79,7 +72,7 @@ func AnnotationRectForNode(ctx context.Context, nodeID int64) (*AnnotationRect,
if err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
return chromedp.FromContext(ctx).Target.Execute(ctx, "Runtime.callFunctionOn", map[string]any{
"functionDeclaration": boxFn,
- "objectId": resolved.Object.ObjectID,
+ "objectId": objectID,
"returnByValue": true,
}, &callResult)
})); err != nil {
diff --git a/internal/cdptk/screenshot.go b/internal/cdptk/screenshot.go
index c8c47cc7..43815ba6 100644
--- a/internal/cdptk/screenshot.go
+++ b/internal/cdptk/screenshot.go
@@ -28,12 +28,16 @@ func ClipForNode(ctx context.Context, nodeID int64, css1x bool) (*ScreenshotClip
// Translate the element box into top-level page coordinates. captureScreenshot
// clip coordinates are page-relative, so viewport-relative rects need scroll
// offsets from the current document and each ancestor frame.
+ // The frame walk starts from the NODE's own view for the same reason as the
+ // bridge clip builder: the isolated handle runs in the top frame's world, so a
+ // bare `window` there loses every frame offset.
const boxFn = `function() {
+ const view = (this.ownerDocument && this.ownerDocument.defaultView) || window;
const rect = this.getBoundingClientRect();
- let x = rect.left + (window.scrollX || window.pageXOffset || 0);
- let y = rect.top + (window.scrollY || window.pageYOffset || 0);
+ let x = rect.left + (view.scrollX || view.pageXOffset || 0);
+ let y = rect.top + (view.scrollY || view.pageYOffset || 0);
try {
- let current = window;
+ let current = view;
while (current && current.parent && current !== current.parent) {
const frameEl = current.frameElement;
if (!frameEl) {
@@ -54,7 +58,7 @@ func ClipForNode(ctx context.Context, nodeID int64, css1x bool) (*ScreenshotClip
y,
width: rect.width,
height: rect.height,
- dpr: window.devicePixelRatio || 1
+ dpr: view.devicePixelRatio || 1
};
}`
diff --git a/internal/cdptk/screenshot_test.go b/internal/cdptk/screenshot_test.go
new file mode 100644
index 00000000..289f7f17
--- /dev/null
+++ b/internal/cdptk/screenshot_test.go
@@ -0,0 +1,144 @@
+package cdptk_test
+
+import (
+ "context"
+ "encoding/base64"
+ "testing"
+ "time"
+
+ "github.com/chromedp/chromedp"
+ "github.com/pinchtab/pinchtab/internal/cdptk"
+ "github.com/pinchtab/pinchtab/internal/testbrowser"
+)
+
+// An external test package: internal/testbrowser reaches cdptk through
+// internal/browsers/chrome, so an in-package test importing it would cycle.
+
+// The page redefines the method both clip builders read. In the main world it
+// answers them; an isolated-world handle never sees it.
+const hijackFixtureHTML = `<body style="margin:0">
+<div id="target" style="position:absolute;left:40px;top:60px;width:120px;height:60px;background:#000"></div>
+<script>
+(() => {
+ Element.prototype.getBoundingClientRect = function () {
+ return {x: 999, y: 999, left: 999, top: 999, right: 1099, bottom: 1049, width: 100, height: 50};
+ };
+})();
+</script>
+</body>`
+
+func newHijackFixture(t *testing.T) context.Context {
+ t.Helper()
+ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(
+ chromedp.DefaultExecAllocatorOptions[:],
+ chromedp.ExecPath(testbrowser.Path(t)),
+ chromedp.UserDataDir(t.TempDir()),
+ chromedp.Flag("headless", true),
+ chromedp.Flag("no-sandbox", true),
+ )...)
+ ctx, cancelBrowser := chromedp.NewContext(alloc)
+ ctx, cancelTimeout := context.WithTimeout(ctx, 30*time.Second)
+ t.Cleanup(func() {
+ cancelTimeout()
+ cancelBrowser()
+ cancelAlloc()
+ })
+
+ dataURL := "data:text/html;base64," + base64.StdEncoding.EncodeToString([]byte(hijackFixtureHTML))
+ if err := chromedp.Run(ctx,
+ chromedp.Navigate(dataURL),
+ chromedp.WaitVisible("#target", chromedp.ByID),
+ ); err != nil {
+ t.Fatal(err)
+ }
+ return ctx
+}
+
+func targetNodeID(t *testing.T, ctx context.Context) int64 {
+ t.Helper()
+ var nodeID int64
+ if err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
+ var doc struct {
+ Root struct {
+ NodeID int64 `json:"nodeId"`
+ } `json:"root"`
+ }
+ if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.getDocument", map[string]any{"depth": -1}, &doc); err != nil {
+ return err
+ }
+ var found struct {
+ NodeID int64 `json:"nodeId"`
+ }
+ if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.querySelector", map[string]any{
+ "nodeId": doc.Root.NodeID,
+ "selector": "#target",
+ }, &found); err != nil {
+ return err
+ }
+ var described struct {
+ Node struct {
+ BackendNodeID int64 `json:"backendNodeId"`
+ } `json:"node"`
+ }
+ if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.describeNode", map[string]any{
+ "nodeId": found.NodeID,
+ }, &described); err != nil {
+ return err
+ }
+ nodeID = described.Node.BackendNodeID
+ return nil
+ })); err != nil {
+ t.Fatal(err)
+ }
+ if nodeID == 0 {
+ t.Fatal("fixture element #target has no backend node id")
+ }
+ return nodeID
+}
+
+// The cdptk analogue of TestScreenshotClipResistsMainWorldSubstitution: the clip
+// origin must come from the element, not from the rectangle page script forges.
+func TestClipForNodeResistsMainWorldSubstitution(t *testing.T) {
+ ctx := newHijackFixture(t)
+ nodeID := targetNodeID(t, ctx)
+
+ clip, err := cdptk.ClipForNode(ctx, nodeID, true)
+ if err != nil {
+ t.Fatalf("ClipForNode: %v", err)
+ }
+
+ // The forged rect is the one value the page can produce; anything else means
+ // the read happened somewhere the page could not reach.
+ if clip.X == 999 || clip.Y == 999 || (clip.Width == 100 && clip.Height == 50) {
+ t.Errorf("clip %+v matches the rectangle page script forged; the origin was read in the main world", clip)
+ }
+ if clip.Width != 120 || clip.Height != 60 {
+ t.Errorf("clip %+v does not match the element's real 120x60 box", clip)
+ }
+}
+
+// The same boundary on the annotation rect: its frame walk reads
+// getBoundingClientRect too, and its output places the overlay boxes a vision
+// model is told to read.
+func TestAnnotationRectForNodeResistsMainWorldSubstitution(t *testing.T) {
+ ctx := newHijackFixture(t)
+ nodeID := targetNodeID(t, ctx)
+
+ rect, err := cdptk.AnnotationRectForNode(ctx, nodeID)
+ if err != nil {
+ t.Fatalf("AnnotationRectForNode: %v", err)
+ }
+ if rect == nil {
+ t.Fatal("AnnotationRectForNode returned no rect for a top-frame element")
+ }
+
+ if rect.X == 999 || rect.Y == 999 || (rect.W == 100 && rect.H == 50) {
+ t.Errorf("rect %+v matches the rectangle page script forged; it was read in the main world", rect)
+ }
+ if rect.W != 120 || rect.H != 60 {
+ t.Errorf("rect %+v does not match the element's real 120x60 box", rect)
+ }
+ if rect.X != 40 || rect.Y != 60 {
+ t.Errorf("rect %+v is not the element's viewport position (40,60)", rect)
+ }
+}