High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix(capture): composite the default capture into the viewport it reports, so boundingBox scaled by devicePixelRatio lands on the image (PIN-302)
Luigi Agostieffbe1139589
diff --git a/docs/endpoints.md b/docs/endpoints.mdindex 7c289892..6ea8d6bc 100644--- a/docs/endpoints.md+++ b/docs/endpoints.md@@ -455,6 +455,16 @@ to hold: the coordinate ORIGIN, which `image.coordinateSpace` names(`viewport` or `document`), and the box-model EDGE, which is always theborder box.+Stated as the guarantee: **scaling a `boundingBox` by `image.devicePixelRatio`,+from the origin `image.coordinateSpace` names, lands on that image's own+pixels — in every mode, with or without an emulated viewport.** The image+measures exactly the reported space times the reported ratio, so a client+never branches on configuration and never has to detect whether a linear+mapping happens to exist. The default (viewport) capture composites the page+to keep that promise, which costs it the faster read-the-view path+`/screenshot` still takes: on an idle headed browser a `/capture` can+therefore block until its deadline where a `/screenshot` returns at once.+The response carries an `epoch.domEpoch` token cached on the tab's ref-cache.Future client work can pass `expectedEpoch` to action endpoints to detectstale refs at the use site; in P1 it is informational. `pairing.navigated`diff --git a/docs/reference/capture.md b/docs/reference/capture.mdindex 026b00df..c65e2dc8 100644--- a/docs/reference/capture.md+++ b/docs/reference/capture.md@@ -107,6 +107,15 @@ The coordinate space depends on `selector` and `beyondViewport`:(`box.x` and `box.y` include scroll offset). The image is the fulldocument.+In all three, the image measures exactly the reported space times+`image.devicePixelRatio`: scaling a `boundingBox` by that ratio, from the+origin the space names, lands on the image's pixels. This holds under+`pinchtab set viewport` as well, so an overlay never needs a mode-specific+factor. Keeping it costs the default capture the faster read-the-view path —+that path returns the real window surface, whose scale factor is the screen's+rather than the page's and which viewport emulation does not touch — so an+idle headed browser can make a `/capture` slower than a `/screenshot`.+`visible` is true when the box has positive area and intersects theviewport — a cheap heuristic, not a strict occlusion check. A nodescrolled past, in either direction, is measured and reportsdiff --git a/internal/bridge/capture.go b/internal/bridge/capture.goindex 0b5e3c57..f659a082 100644--- a/internal/bridge/capture.go+++ b/internal/bridge/capture.go@@ -8,6 +8,8 @@ import ("github.com/chromedp/cdproto/page""github.com/chromedp/chromedp"++ "github.com/pinchtab/pinchtab/internal/cdptk")type CaptureOpts struct {@@ -112,7 +114,7 @@ func PairedCapture(ctx context.Context, opts CaptureOpts) (*PairedResult, error)// Image first. Order matters only when BeyondViewport is true (P3 concern);// at viewport scale either order is equivalent.- imgBytes, err := CaptureScreenshot(ctx, opts.Image)+ imgBytes, err := CaptureScreenshot(ctx, captureImageOpts(opts.Image, res.Viewport))if err != nil {return nil, err}@@ -171,6 +173,23 @@ func PairedCapture(ctx context.Context, opts CaptureOpts) (*PairedResult, error)return res, nil}+// captureImageOpts settles which region the capture renders. A caller's own clip and a+// beyond-viewport capture already composite, so they are left exactly as asked; only the+// default — no clip, viewport space — is redirected onto cdptk.ViewportClip, which is+// what makes the returned image match the viewport and devicePixelRatio this response+// reports. The clip is added to the SCREENSHOT options alone: the response metadata is+// derived from what the caller asked for, so a synthesized clip must not turn the+// reported coordinateSpace into "clip".+func captureImageOpts(image ScreenshotOpts, vp ViewportInfo) ScreenshotOpts {+ if image.Clip != nil || image.BeyondViewport {+ return image+ }+ if clip := cdptk.ViewportClip(vp.ScrollX, vp.ScrollY, vp.Width, vp.Height); clip != nil {+ image.Clip = clip+ }+ return image+}+func imageFormatString(f page.CaptureScreenshotFormat) string {return string(f)}diff --git a/internal/bridge/capture_image_opts_test.go b/internal/bridge/capture_image_opts_test.gonew file mode 100644index 00000000..120f6933--- /dev/null+++ b/internal/bridge/capture_image_opts_test.go@@ -0,0 +1,50 @@+package bridge++import (+ "testing"++ "github.com/chromedp/cdproto/page"+)++// The browserless twin of TestCaptureImageMatchesTheSpaceItReports, which needs a+// browser CI does not have. It cannot see pixels, so it pins the decision that+// produces them: which region each mode asks CDP to render.+func TestCaptureImageOptsRedirectsOnlyTheDefaultMode(t *testing.T) {+ vp := ViewportInfo{Width: 800, Height: 600, ScrollX: 30, ScrollY: 40, DevicePixelRatio: 2}+ callerClip := &page.Viewport{X: 1, Y: 2, Width: 3, Height: 4, Scale: 1}++ t.Run("default capture renders the viewport it reports", func(t *testing.T) {+ got := captureImageOpts(ScreenshotOpts{Format: ScreenshotFormatPng}, vp)+ if got.Clip == nil {+ t.Fatal("the default capture still reads the window surface, which no reported devicePixelRatio describes")+ }+ want := page.Viewport{X: 30, Y: 40, Width: 800, Height: 600, Scale: 1}+ if *got.Clip != want {+ t.Errorf("clip = %+v, want %+v — page coordinates, and scale 1 because CDP applies the page ratio itself", *got.Clip, want)+ }+ })++ t.Run("a caller's own clip is untouched", func(t *testing.T) {+ got := captureImageOpts(ScreenshotOpts{Format: ScreenshotFormatPng, Clip: callerClip}, vp)+ if got.Clip != callerClip {+ t.Errorf("clip = %+v, want the caller's %+v", got.Clip, callerClip)+ }+ })++ t.Run("beyondViewport keeps its nil clip", func(t *testing.T) {+ got := captureImageOpts(ScreenshotOpts{Format: ScreenshotFormatPng, BeyondViewport: true}, vp)+ if got.Clip != nil {+ t.Errorf("clip = %+v, want nil; a clip and captureBeyondViewport are two different regions and CDP honours only one", got.Clip)+ }+ })++ // A layout read can fail, and a zero viewport would ask CDP for an empty region.+ // Falling back to the old behaviour returns a picture that does not match the+ // metadata; returning nothing returns no picture at all.+ t.Run("an unreadable viewport falls back rather than clipping to nothing", func(t *testing.T) {+ got := captureImageOpts(ScreenshotOpts{Format: ScreenshotFormatPng}, ViewportInfo{})+ if got.Clip != nil {+ t.Errorf("clip = %+v, want nil when the layout read gave no viewport", got.Clip)+ }+ })+}diff --git a/internal/bridge/capture_image_space_test.go b/internal/bridge/capture_image_space_test.gonew file mode 100644index 00000000..95a71643--- /dev/null+++ b/internal/bridge/capture_image_space_test.go@@ -0,0 +1,181 @@+package bridge++import (+ "bytes"+ "context"+ "encoding/base64"+ "image"+ _ "image/png"+ "testing"+ "time"++ "github.com/chromedp/cdproto/emulation"+ "github.com/chromedp/chromedp"+ "github.com/pinchtab/pinchtab/internal/testbrowser"+)++const imageSpaceFixtureHTML = `<body style="margin:0;width:1200px;height:2400px">+<button id="target" style="position:absolute;left:40px;top:60px;width:120px;height:40px">target</button>+</body>`++func newImageSpaceFixture(t *testing.T) (context.Context, int64) {+ t.Helper()++ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(+ chromedp.DefaultExecAllocatorOptions[:],+ chromedp.ExecPath(testbrowser.Path(t)),+ chromedp.UserDataDir(testbrowser.ProfileDir(t)),+ chromedp.Flag("headless", true),+ chromedp.Flag("no-sandbox", true),+ )...)+ ctx, cancelBrowser := chromedp.NewContext(alloc)+ ctx, cancelTimeout := context.WithTimeout(ctx, 90*time.Second)+ t.Cleanup(func() {+ cancelTimeout()+ cancelBrowser()+ cancelAlloc()+ })++ dataURL := "data:text/html;base64," + base64.StdEncoding.EncodeToString([]byte(imageSpaceFixtureHTML))+ if err := chromedp.Run(ctx,+ chromedp.Navigate(dataURL),+ chromedp.WaitVisible("#target", chromedp.ByID),+ ); err != nil {+ t.Fatal(err)+ }++ rawNodes, err := FetchAXTree(ctx)+ if err != nil {+ t.Fatal(err)+ }+ nodes, _ := BuildSnapshot(rawNodes, "", -1)+ for _, n := range nodes {+ if n.Role == "button" && n.Name == "target" && n.NodeID != 0 {+ return ctx, n.NodeID+ }+ }+ t.Fatalf("no button named \"target\" in the snapshot (%d nodes)", len(nodes))+ return nil, 0+}++func imageDimensions(t *testing.T, buf []byte) (float64, float64) {+ t.Helper()++ cfg, _, err := image.DecodeConfig(bytes.NewReader(buf))+ if err != nil {+ t.Fatalf("capture did not return a decodable image: %v", err)+ }+ return float64(cfg.Width), float64(cfg.Height)+}++func documentSize(t *testing.T, ctx context.Context) (float64, float64) {+ t.Helper()++ var size struct {+ W float64 `json:"w"`+ H float64 `json:"h"`+ }+ if err := chromedp.Run(ctx, chromedp.Evaluate(+ `(() => ({w: document.documentElement.scrollWidth, h: document.documentElement.scrollHeight}))()`,+ &size)); err != nil {+ t.Fatal(err)+ }+ return size.W, size.H+}++// The contract /capture publishes: scaling a node's boundingBox by the reported+// devicePixelRatio, from the origin the reported coordinateSpace names, lands on the+// image's own pixels. That is checkable as a dimension identity — the image measures+// exactly the reported space times the reported ratio — and it has to hold in every+// mode, because a client is promised it never has to branch.+//+// Three modes and two emulation states, because the defect was invisible in a+// plain-only test: the two clipped modes were already correct, and the default one+// happened to match whenever the page's device pixel ratio equalled the screen's.+func TestCaptureImageMatchesTheSpaceItReports(t *testing.T) {+ for _, emulated := range []struct {+ name string+ width, height int64+ deviceScaleFactor float64+ }{+ {name: "no emulation"},+ {name: "emulated 400x300 at dpr 1", width: 400, height: 300, deviceScaleFactor: 1},+ {name: "emulated 400x300 at dpr 2", width: 400, height: 300, deviceScaleFactor: 2},+ } {+ t.Run(emulated.name, func(t *testing.T) {+ ctx, nodeID := newImageSpaceFixture(t)+ if emulated.width > 0 {+ if err := chromedp.Run(ctx, emulation.SetDeviceMetricsOverride(+ emulated.width, emulated.height, emulated.deviceScaleFactor, false)); err != nil {+ t.Fatal(err)+ }+ }++ t.Run("viewport", func(t *testing.T) {+ res, err := PairedCapture(ctx, CaptureOpts{MaxDepth: -1, Image: ScreenshotOpts{Format: ScreenshotFormatPng}})+ if err != nil {+ t.Fatal(err)+ }+ if res.CoordinateSpace != "viewport" {+ t.Fatalf("CoordinateSpace = %q, want viewport", res.CoordinateSpace)+ }+ assertImageIs(t, res, res.Viewport.Width, res.Viewport.Height)+ })++ t.Run("beyondViewport", func(t *testing.T) {+ res, err := PairedCapture(ctx, CaptureOpts{MaxDepth: -1, Image: ScreenshotOpts{+ Format: ScreenshotFormatPng, BeyondViewport: true,+ }})+ if err != nil {+ t.Fatal(err)+ }+ if res.CoordinateSpace != "document" {+ t.Fatalf("CoordinateSpace = %q, want document", res.CoordinateSpace)+ }+ docWidth, docHeight := documentSize(t, ctx)+ assertImageIs(t, res, docWidth, docHeight)+ })++ t.Run("selector", func(t *testing.T) {+ clip, err := ScreenshotClipForNode(ctx, nodeID)+ if err != nil {+ t.Fatal(err)+ }+ res, err := PairedCapture(ctx, CaptureOpts{MaxDepth: -1, Image: ScreenshotOpts{+ Format: ScreenshotFormatPng,+ Clip: clip,+ }})+ if err != nil {+ t.Fatal(err)+ }+ if res.CoordinateSpace != "clip" {+ t.Fatalf("CoordinateSpace = %q, want clip", res.CoordinateSpace)+ }+ assertImageIs(t, res, clip.Width, clip.Height)+ })+ })+ }+}++// assertImageIs is the contract as one identity: the image is the reported space at the+// reported ratio. Rounding is a single pixel per axis — CDP rounds the composited size —+// and anything larger is a client's overlay landing somewhere else.+func assertImageIs(t *testing.T, res *PairedResult, spaceWidth, spaceHeight float64) {+ t.Helper()++ dpr := res.Viewport.DevicePixelRatio+ if dpr <= 0 {+ t.Fatalf("capture reported devicePixelRatio %v; a client scaling by it would map every box onto nothing", dpr)+ }+ gotWidth, gotHeight := imageDimensions(t, res.ImageBytes)+ wantWidth, wantHeight := spaceWidth*dpr, spaceHeight*dpr++ if diff := gotWidth - wantWidth; diff > 1 || diff < -1 {+ t.Errorf("image is %.0f px wide; the response reports %s %.0f at devicePixelRatio %v, so a boundingBox scaled by it lands %.2fx off",+ gotWidth, res.CoordinateSpace, spaceWidth, dpr, gotWidth/wantWidth)+ }+ if diff := gotHeight - wantHeight; diff > 1 || diff < -1 {+ t.Errorf("image is %.0f px tall; the response reports %s %.0f at devicePixelRatio %v, so a boundingBox scaled by it lands %.2fx off",+ gotHeight, res.CoordinateSpace, spaceHeight, dpr, gotHeight/wantHeight)+ }+}diff --git a/internal/cdptk/screenshot.go b/internal/cdptk/screenshot.goindex 0f52f0ff..7a275fca 100644--- a/internal/cdptk/screenshot.go+++ b/internal/cdptk/screenshot.go@@ -149,6 +149,37 @@ func captureFromSurface(goos string, beyondViewport bool, clip *page.Viewport) breturn goos == "windows" || beyondViewport || clip != nil}+// ViewportClip is the clip that makes a DEFAULT capture come back in the space its+// own metadata reports. It is the answer to a contract, not an optimisation: /capture+// publishes coordinateSpace, viewport and devicePixelRatio so a client can scale a+// node's boundingBox onto image pixels without guessing, and the read-the-view fast+// path cannot keep that promise. That path returns the real window surface, which+// Emulation.setDeviceMetricsOverride does not touch and whose scale factor is the+// screen's, not the page's — measured in headless Chrome on a HiDPI screen, a page+// reporting 756x413 at devicePixelRatio 1 came back 1512x826, and the same page+// emulated to 400x300 came back 800x600. Both are a caller's overlay landing on the+// wrong pixels, and no reported dpr rescues the emulated case, where the surface can+// carry a different aspect ratio entirely.+//+// A clip covering the viewport composites the page and lets CDP apply the page's own+// device pixel ratio, which is what the beyond-viewport and selector paths already do+// and why those two were correct while the default one was not. Scale stays 1: CDP+// multiplies a clipped capture by the page dpr itself, so a scale of dpr would square+// it. Coordinates are page-relative, hence the scroll offsets.+//+// The cost is stated rather than hidden: a clipped capture takes fromSurface=true, so+// the default /capture gives up the read-the-view fast path this file documents and an+// idle headed browser can stall it until the action deadline. /screenshot and the+// screencast keep the fast path — they publish no coordinate contract to break — so the+// latency trade is paid only by the endpoint whose whole purpose is mapping refs onto+// pixels.+func ViewportClip(scrollX, scrollY, width, height float64) *page.Viewport {+ if width <= 0 || height <= 0 {+ return nil+ }+ return &page.Viewport{X: scrollX, Y: scrollY, Width: width, Height: height, Scale: 1}+}+// captureRefusal is the CDP error a renderer returns when it will not serve the// read-the-view fast path. It carries no code the client can match on, so the message is// the only discriminator available.diff --git a/internal/cdptk/screenshot_test.go b/internal/cdptk/screenshot_test.goindex 791581c6..2b4d4b9d 100644--- a/internal/cdptk/screenshot_test.go+++ b/internal/cdptk/screenshot_test.go@@ -521,3 +521,25 @@ func TestTheClipToViewportCensusFailsOnSourceItCannotParse(t *testing.T) {t.Error("unparseable source reported no conversion instead of an error; a file the census cannot read must fail it, not pass silently")}}++func TestViewportClipIsPageRelativeAtNativeScale(t *testing.T) {+ clip := cdptk.ViewportClip(30, 40, 800, 600)+ if clip == nil {+ t.Fatal("ViewportClip returned nil for a real viewport")+ }+ want := page.Viewport{X: 30, Y: 40, Width: 800, Height: 600, Scale: 1}+ if *clip != want {+ t.Fatalf("ViewportClip = %+v, want %+v", *clip, want)+ }+ if !cdptk.CaptureFromSurface(false, clip) {+ t.Error("the viewport clip does not take the surface path; CDP discards a clip on the read-the-view path, so the capture would silently come back unclipped")+ }+}++func TestViewportClipRefusesAnEmptyViewport(t *testing.T) {+ for _, tc := range []struct{ w, h float64 }{{0, 600}, {800, 0}, {-1, -1}} {+ if clip := cdptk.ViewportClip(0, 0, tc.w, tc.h); clip != nil {+ t.Errorf("ViewportClip(%v, %v) = %+v, want nil; an empty clip asks CDP for no pixels at all", tc.w, tc.h, clip)+ }+ }+}