Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix(capture): composite the default capture into the viewport it reports, so boundingBox scaled by devicePixelRatio lands on the image (PIN-302)

Luigi Agostieffbe1139589
diff --git a/docs/endpoints.md b/docs/endpoints.md
index 7c289892..6ea8d6bc 100644
--- a/docs/endpoints.md
+++ b/docs/endpoints.md
@@ -455,6 +455,16 @@ to hold: the coordinate ORIGIN, which `image.coordinateSpace` names
(`viewport` or `document`), and the box-model EDGE, which is always the
border box.
+Stated as the guarantee: **scaling a `boundingBox` by `image.devicePixelRatio`,
+from the origin `image.coordinateSpace` names, lands on that image's own
+pixels — in every mode, with or without an emulated viewport.** The image
+measures exactly the reported space times the reported ratio, so a client
+never branches on configuration and never has to detect whether a linear
+mapping happens to exist. The default (viewport) capture composites the page
+to keep that promise, which costs it the faster read-the-view path
+`/screenshot` still takes: on an idle headed browser a `/capture` can
+therefore block until its deadline where a `/screenshot` returns at once.
+
The response carries an `epoch.domEpoch` token cached on the tab's ref-cache.
Future client work can pass `expectedEpoch` to action endpoints to detect
stale refs at the use site; in P1 it is informational. `pairing.navigated`
diff --git a/docs/reference/capture.md b/docs/reference/capture.md
index 026b00df..c65e2dc8 100644
--- a/docs/reference/capture.md
+++ b/docs/reference/capture.md
@@ -107,6 +107,15 @@ The coordinate space depends on `selector` and `beyondViewport`:
(`box.x` and `box.y` include scroll offset). The image is the full
document.
+In all three, the image measures exactly the reported space times
+`image.devicePixelRatio`: scaling a `boundingBox` by that ratio, from the
+origin the space names, lands on the image's pixels. This holds under
+`pinchtab set viewport` as well, so an overlay never needs a mode-specific
+factor. Keeping it costs the default capture the faster read-the-view path —
+that path returns the real window surface, whose scale factor is the screen's
+rather than the page's and which viewport emulation does not touch — so an
+idle headed browser can make a `/capture` slower than a `/screenshot`.
+
`visible` is true when the box has positive area and intersects the
viewport — a cheap heuristic, not a strict occlusion check. A node
scrolled past, in either direction, is measured and reports
diff --git a/internal/bridge/capture.go b/internal/bridge/capture.go
index 0b5e3c57..f659a082 100644
--- a/internal/bridge/capture.go
+++ b/internal/bridge/capture.go
@@ -8,6 +8,8 @@ import (
"github.com/chromedp/cdproto/page"
"github.com/chromedp/chromedp"
+
+ "github.com/pinchtab/pinchtab/internal/cdptk"
)
type CaptureOpts struct {
@@ -112,7 +114,7 @@ func PairedCapture(ctx context.Context, opts CaptureOpts) (*PairedResult, error)
// Image first. Order matters only when BeyondViewport is true (P3 concern);
// at viewport scale either order is equivalent.
- imgBytes, err := CaptureScreenshot(ctx, opts.Image)
+ imgBytes, err := CaptureScreenshot(ctx, captureImageOpts(opts.Image, res.Viewport))
if err != nil {
return nil, err
}
@@ -171,6 +173,23 @@ func PairedCapture(ctx context.Context, opts CaptureOpts) (*PairedResult, error)
return res, nil
}
+// captureImageOpts settles which region the capture renders. A caller's own clip and a
+// beyond-viewport capture already composite, so they are left exactly as asked; only the
+// default — no clip, viewport space — is redirected onto cdptk.ViewportClip, which is
+// what makes the returned image match the viewport and devicePixelRatio this response
+// reports. The clip is added to the SCREENSHOT options alone: the response metadata is
+// derived from what the caller asked for, so a synthesized clip must not turn the
+// reported coordinateSpace into "clip".
+func captureImageOpts(image ScreenshotOpts, vp ViewportInfo) ScreenshotOpts {
+ if image.Clip != nil || image.BeyondViewport {
+ return image
+ }
+ if clip := cdptk.ViewportClip(vp.ScrollX, vp.ScrollY, vp.Width, vp.Height); clip != nil {
+ image.Clip = clip
+ }
+ return image
+}
+
func imageFormatString(f page.CaptureScreenshotFormat) string {
return string(f)
}
diff --git a/internal/bridge/capture_image_opts_test.go b/internal/bridge/capture_image_opts_test.go
new file mode 100644
index 00000000..120f6933
--- /dev/null
+++ b/internal/bridge/capture_image_opts_test.go
@@ -0,0 +1,50 @@
+package bridge
+
+import (
+ "testing"
+
+ "github.com/chromedp/cdproto/page"
+)
+
+// The browserless twin of TestCaptureImageMatchesTheSpaceItReports, which needs a
+// browser CI does not have. It cannot see pixels, so it pins the decision that
+// produces them: which region each mode asks CDP to render.
+func TestCaptureImageOptsRedirectsOnlyTheDefaultMode(t *testing.T) {
+ vp := ViewportInfo{Width: 800, Height: 600, ScrollX: 30, ScrollY: 40, DevicePixelRatio: 2}
+ callerClip := &page.Viewport{X: 1, Y: 2, Width: 3, Height: 4, Scale: 1}
+
+ t.Run("default capture renders the viewport it reports", func(t *testing.T) {
+ got := captureImageOpts(ScreenshotOpts{Format: ScreenshotFormatPng}, vp)
+ if got.Clip == nil {
+ t.Fatal("the default capture still reads the window surface, which no reported devicePixelRatio describes")
+ }
+ want := page.Viewport{X: 30, Y: 40, Width: 800, Height: 600, Scale: 1}
+ if *got.Clip != want {
+ t.Errorf("clip = %+v, want %+v — page coordinates, and scale 1 because CDP applies the page ratio itself", *got.Clip, want)
+ }
+ })
+
+ t.Run("a caller's own clip is untouched", func(t *testing.T) {
+ got := captureImageOpts(ScreenshotOpts{Format: ScreenshotFormatPng, Clip: callerClip}, vp)
+ if got.Clip != callerClip {
+ t.Errorf("clip = %+v, want the caller's %+v", got.Clip, callerClip)
+ }
+ })
+
+ t.Run("beyondViewport keeps its nil clip", func(t *testing.T) {
+ got := captureImageOpts(ScreenshotOpts{Format: ScreenshotFormatPng, BeyondViewport: true}, vp)
+ if got.Clip != nil {
+ t.Errorf("clip = %+v, want nil; a clip and captureBeyondViewport are two different regions and CDP honours only one", got.Clip)
+ }
+ })
+
+ // A layout read can fail, and a zero viewport would ask CDP for an empty region.
+ // Falling back to the old behaviour returns a picture that does not match the
+ // metadata; returning nothing returns no picture at all.
+ t.Run("an unreadable viewport falls back rather than clipping to nothing", func(t *testing.T) {
+ got := captureImageOpts(ScreenshotOpts{Format: ScreenshotFormatPng}, ViewportInfo{})
+ if got.Clip != nil {
+ t.Errorf("clip = %+v, want nil when the layout read gave no viewport", got.Clip)
+ }
+ })
+}
diff --git a/internal/bridge/capture_image_space_test.go b/internal/bridge/capture_image_space_test.go
new file mode 100644
index 00000000..95a71643
--- /dev/null
+++ b/internal/bridge/capture_image_space_test.go
@@ -0,0 +1,181 @@
+package bridge
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "image"
+ _ "image/png"
+ "testing"
+ "time"
+
+ "github.com/chromedp/cdproto/emulation"
+ "github.com/chromedp/chromedp"
+ "github.com/pinchtab/pinchtab/internal/testbrowser"
+)
+
+const imageSpaceFixtureHTML = `<body style="margin:0;width:1200px;height:2400px">
+<button id="target" style="position:absolute;left:40px;top:60px;width:120px;height:40px">target</button>
+</body>`
+
+func newImageSpaceFixture(t *testing.T) (context.Context, int64) {
+ t.Helper()
+
+ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(
+ chromedp.DefaultExecAllocatorOptions[:],
+ chromedp.ExecPath(testbrowser.Path(t)),
+ chromedp.UserDataDir(testbrowser.ProfileDir(t)),
+ chromedp.Flag("headless", true),
+ chromedp.Flag("no-sandbox", true),
+ )...)
+ ctx, cancelBrowser := chromedp.NewContext(alloc)
+ ctx, cancelTimeout := context.WithTimeout(ctx, 90*time.Second)
+ t.Cleanup(func() {
+ cancelTimeout()
+ cancelBrowser()
+ cancelAlloc()
+ })
+
+ dataURL := "data:text/html;base64," + base64.StdEncoding.EncodeToString([]byte(imageSpaceFixtureHTML))
+ if err := chromedp.Run(ctx,
+ chromedp.Navigate(dataURL),
+ chromedp.WaitVisible("#target", chromedp.ByID),
+ ); err != nil {
+ t.Fatal(err)
+ }
+
+ rawNodes, err := FetchAXTree(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ nodes, _ := BuildSnapshot(rawNodes, "", -1)
+ for _, n := range nodes {
+ if n.Role == "button" && n.Name == "target" && n.NodeID != 0 {
+ return ctx, n.NodeID
+ }
+ }
+ t.Fatalf("no button named \"target\" in the snapshot (%d nodes)", len(nodes))
+ return nil, 0
+}
+
+func imageDimensions(t *testing.T, buf []byte) (float64, float64) {
+ t.Helper()
+
+ cfg, _, err := image.DecodeConfig(bytes.NewReader(buf))
+ if err != nil {
+ t.Fatalf("capture did not return a decodable image: %v", err)
+ }
+ return float64(cfg.Width), float64(cfg.Height)
+}
+
+func documentSize(t *testing.T, ctx context.Context) (float64, float64) {
+ t.Helper()
+
+ var size struct {
+ W float64 `json:"w"`
+ H float64 `json:"h"`
+ }
+ if err := chromedp.Run(ctx, chromedp.Evaluate(
+ `(() => ({w: document.documentElement.scrollWidth, h: document.documentElement.scrollHeight}))()`,
+ &size)); err != nil {
+ t.Fatal(err)
+ }
+ return size.W, size.H
+}
+
+// The contract /capture publishes: scaling a node's boundingBox by the reported
+// devicePixelRatio, from the origin the reported coordinateSpace names, lands on the
+// image's own pixels. That is checkable as a dimension identity — the image measures
+// exactly the reported space times the reported ratio — and it has to hold in every
+// mode, because a client is promised it never has to branch.
+//
+// Three modes and two emulation states, because the defect was invisible in a
+// plain-only test: the two clipped modes were already correct, and the default one
+// happened to match whenever the page's device pixel ratio equalled the screen's.
+func TestCaptureImageMatchesTheSpaceItReports(t *testing.T) {
+ for _, emulated := range []struct {
+ name string
+ width, height int64
+ deviceScaleFactor float64
+ }{
+ {name: "no emulation"},
+ {name: "emulated 400x300 at dpr 1", width: 400, height: 300, deviceScaleFactor: 1},
+ {name: "emulated 400x300 at dpr 2", width: 400, height: 300, deviceScaleFactor: 2},
+ } {
+ t.Run(emulated.name, func(t *testing.T) {
+ ctx, nodeID := newImageSpaceFixture(t)
+ if emulated.width > 0 {
+ if err := chromedp.Run(ctx, emulation.SetDeviceMetricsOverride(
+ emulated.width, emulated.height, emulated.deviceScaleFactor, false)); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ t.Run("viewport", func(t *testing.T) {
+ res, err := PairedCapture(ctx, CaptureOpts{MaxDepth: -1, Image: ScreenshotOpts{Format: ScreenshotFormatPng}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if res.CoordinateSpace != "viewport" {
+ t.Fatalf("CoordinateSpace = %q, want viewport", res.CoordinateSpace)
+ }
+ assertImageIs(t, res, res.Viewport.Width, res.Viewport.Height)
+ })
+
+ t.Run("beyondViewport", func(t *testing.T) {
+ res, err := PairedCapture(ctx, CaptureOpts{MaxDepth: -1, Image: ScreenshotOpts{
+ Format: ScreenshotFormatPng, BeyondViewport: true,
+ }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if res.CoordinateSpace != "document" {
+ t.Fatalf("CoordinateSpace = %q, want document", res.CoordinateSpace)
+ }
+ docWidth, docHeight := documentSize(t, ctx)
+ assertImageIs(t, res, docWidth, docHeight)
+ })
+
+ t.Run("selector", func(t *testing.T) {
+ clip, err := ScreenshotClipForNode(ctx, nodeID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ res, err := PairedCapture(ctx, CaptureOpts{MaxDepth: -1, Image: ScreenshotOpts{
+ Format: ScreenshotFormatPng,
+ Clip: clip,
+ }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if res.CoordinateSpace != "clip" {
+ t.Fatalf("CoordinateSpace = %q, want clip", res.CoordinateSpace)
+ }
+ assertImageIs(t, res, clip.Width, clip.Height)
+ })
+ })
+ }
+}
+
+// assertImageIs is the contract as one identity: the image is the reported space at the
+// reported ratio. Rounding is a single pixel per axis — CDP rounds the composited size —
+// and anything larger is a client's overlay landing somewhere else.
+func assertImageIs(t *testing.T, res *PairedResult, spaceWidth, spaceHeight float64) {
+ t.Helper()
+
+ dpr := res.Viewport.DevicePixelRatio
+ if dpr <= 0 {
+ t.Fatalf("capture reported devicePixelRatio %v; a client scaling by it would map every box onto nothing", dpr)
+ }
+ gotWidth, gotHeight := imageDimensions(t, res.ImageBytes)
+ wantWidth, wantHeight := spaceWidth*dpr, spaceHeight*dpr
+
+ if diff := gotWidth - wantWidth; diff > 1 || diff < -1 {
+ t.Errorf("image is %.0f px wide; the response reports %s %.0f at devicePixelRatio %v, so a boundingBox scaled by it lands %.2fx off",
+ gotWidth, res.CoordinateSpace, spaceWidth, dpr, gotWidth/wantWidth)
+ }
+ if diff := gotHeight - wantHeight; diff > 1 || diff < -1 {
+ t.Errorf("image is %.0f px tall; the response reports %s %.0f at devicePixelRatio %v, so a boundingBox scaled by it lands %.2fx off",
+ gotHeight, res.CoordinateSpace, spaceHeight, dpr, gotHeight/wantHeight)
+ }
+}
diff --git a/internal/cdptk/screenshot.go b/internal/cdptk/screenshot.go
index 0f52f0ff..7a275fca 100644
--- a/internal/cdptk/screenshot.go
+++ b/internal/cdptk/screenshot.go
@@ -149,6 +149,37 @@ func captureFromSurface(goos string, beyondViewport bool, clip *page.Viewport) b
return goos == "windows" || beyondViewport || clip != nil
}
+// ViewportClip is the clip that makes a DEFAULT capture come back in the space its
+// own metadata reports. It is the answer to a contract, not an optimisation: /capture
+// publishes coordinateSpace, viewport and devicePixelRatio so a client can scale a
+// node's boundingBox onto image pixels without guessing, and the read-the-view fast
+// path cannot keep that promise. That path returns the real window surface, which
+// Emulation.setDeviceMetricsOverride does not touch and whose scale factor is the
+// screen's, not the page's — measured in headless Chrome on a HiDPI screen, a page
+// reporting 756x413 at devicePixelRatio 1 came back 1512x826, and the same page
+// emulated to 400x300 came back 800x600. Both are a caller's overlay landing on the
+// wrong pixels, and no reported dpr rescues the emulated case, where the surface can
+// carry a different aspect ratio entirely.
+//
+// A clip covering the viewport composites the page and lets CDP apply the page's own
+// device pixel ratio, which is what the beyond-viewport and selector paths already do
+// and why those two were correct while the default one was not. Scale stays 1: CDP
+// multiplies a clipped capture by the page dpr itself, so a scale of dpr would square
+// it. Coordinates are page-relative, hence the scroll offsets.
+//
+// The cost is stated rather than hidden: a clipped capture takes fromSurface=true, so
+// the default /capture gives up the read-the-view fast path this file documents and an
+// idle headed browser can stall it until the action deadline. /screenshot and the
+// screencast keep the fast path — they publish no coordinate contract to break — so the
+// latency trade is paid only by the endpoint whose whole purpose is mapping refs onto
+// pixels.
+func ViewportClip(scrollX, scrollY, width, height float64) *page.Viewport {
+ if width <= 0 || height <= 0 {
+ return nil
+ }
+ return &page.Viewport{X: scrollX, Y: scrollY, Width: width, Height: height, Scale: 1}
+}
+
// captureRefusal is the CDP error a renderer returns when it will not serve the
// read-the-view fast path. It carries no code the client can match on, so the message is
// the only discriminator available.
diff --git a/internal/cdptk/screenshot_test.go b/internal/cdptk/screenshot_test.go
index 791581c6..2b4d4b9d 100644
--- a/internal/cdptk/screenshot_test.go
+++ b/internal/cdptk/screenshot_test.go
@@ -521,3 +521,25 @@ func TestTheClipToViewportCensusFailsOnSourceItCannotParse(t *testing.T) {
t.Error("unparseable source reported no conversion instead of an error; a file the census cannot read must fail it, not pass silently")
}
}
+
+func TestViewportClipIsPageRelativeAtNativeScale(t *testing.T) {
+ clip := cdptk.ViewportClip(30, 40, 800, 600)
+ if clip == nil {
+ t.Fatal("ViewportClip returned nil for a real viewport")
+ }
+ want := page.Viewport{X: 30, Y: 40, Width: 800, Height: 600, Scale: 1}
+ if *clip != want {
+ t.Fatalf("ViewportClip = %+v, want %+v", *clip, want)
+ }
+ if !cdptk.CaptureFromSurface(false, clip) {
+ t.Error("the viewport clip does not take the surface path; CDP discards a clip on the read-the-view path, so the capture would silently come back unclipped")
+ }
+}
+
+func TestViewportClipRefusesAnEmptyViewport(t *testing.T) {
+ for _, tc := range []struct{ w, h float64 }{{0, 600}, {800, 0}, {-1, -1}} {
+ if clip := cdptk.ViewportClip(0, 0, tc.w, tc.h); clip != nil {
+ t.Errorf("ViewportClip(%v, %v) = %+v, want nil; an empty clip asks CDP for no pixels at all", tc.w, tc.h, clip)
+ }
+ }
+}