Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix: screenshot selector is accepted and silently ignored, always returning the full viewport (PIN-033)

Luigi Agostid396b3baabd6
diff --git a/internal/bridge/screenshot.go b/internal/bridge/screenshot.go
index fc6f327f..98bcdd50 100644
--- a/internal/bridge/screenshot.go
+++ b/internal/bridge/screenshot.go
@@ -142,16 +142,16 @@ func scaledScreenshotClip(opts ScreenshotOpts, viewportWidth, viewportHeight, do
// waiting for a fresh compositor surface frame. On idle pages in headed browsers
// (e.g. Cloak) the surface stops swapping frames, so the default fromSurface=true
// blocks until the action deadline (~30s); reading from the view avoids that for
-// plain captures. But capture-beyond-viewport and any render-time rescale
-// (clip.Scale != 1) both need the page recomposited at a new size, which only
-// happens with fromSurface=true — forcing it off there silently drops the
-// scale/beyond-viewport effect. So keep it off only for the plain/native-scale
-// path.
+// plain captures.
+//
+// Anything that changes the captured region needs the page recomposited, which
+// only happens with fromSurface=true: capture-beyond-viewport, a render-time
+// rescale, and the clip itself. Reading the view ignores the clip outright and
+// returns the whole viewport, so every non-nil clip takes the surface path —
+// keeping it off for a native-scale clip is what made a selector capture come
+// back byte-identical to an unclipped one.
func captureFromSurface(beyondViewport bool, clip *page.Viewport) bool {
- if beyondViewport {
- return true
- }
- return clip != nil && clip.Scale != 0 && clip.Scale != 1
+ return beyondViewport || clip != nil
}
// captureScreenshotWithoutActivation wakes a background renderer before
diff --git a/internal/bridge/screenshot_clip_test.go b/internal/bridge/screenshot_clip_test.go
new file mode 100644
index 00000000..a74a1144
--- /dev/null
+++ b/internal/bridge/screenshot_clip_test.go
@@ -0,0 +1,194 @@
+package bridge
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "image"
+ _ "image/png"
+ "testing"
+ "time"
+
+ "github.com/chromedp/cdproto/page"
+ "github.com/chromedp/chromedp"
+ "github.com/pinchtab/pinchtab/internal/testbrowser"
+)
+
+// A single element of known size at a known offset, materially smaller than any
+// viewport, so the expected clipped dimensions are exact rather than a ratio.
+const clipFixtureHTML = `<body style="margin:0;background:#fff">
+<div id="small" style="position:absolute;left:40px;top:60px;width:120px;height:60px;background:#c00"></div>
+<div id="empty" style="position:absolute;left:0;top:0;width:0;height:0"></div>
+<iframe id="f" style="position:absolute;left:400px;top:300px;border:0" width="300" height="200"
+ srcdoc="<body style='margin:0'><div id='inner' style='position:absolute;left:10px;top:20px;width:80px;height:40px;background:#00c'></div></body>"></iframe>
+</body>`
+
+const clipFixtureSmallWidth, clipFixtureSmallHeight = 120, 60
+
+func newClipFixture(t *testing.T) context.Context {
+ t.Helper()
+ chromePath := testbrowser.Path(t)
+
+ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(
+ chromedp.DefaultExecAllocatorOptions[:],
+ chromedp.ExecPath(chromePath),
+ chromedp.UserDataDir(t.TempDir()),
+ chromedp.Flag("headless", true),
+ chromedp.Flag("no-sandbox", true),
+ )...)
+ ctx, cancelBrowser := chromedp.NewContext(alloc)
+ ctx, cancelTimeout := context.WithTimeout(ctx, 60*time.Second)
+ t.Cleanup(func() {
+ cancelTimeout()
+ cancelBrowser()
+ cancelAlloc()
+ })
+
+ dataURL := "data:text/html;base64," + base64.StdEncoding.EncodeToString([]byte(clipFixtureHTML))
+ if err := chromedp.Run(ctx,
+ chromedp.Navigate(dataURL),
+ chromedp.WaitVisible("#small", chromedp.ByID),
+ ); err != nil {
+ t.Fatal(err)
+ }
+ return ctx
+}
+
+func imageSize(t *testing.T, buf []byte) (int, int) {
+ t.Helper()
+ cfg, _, err := image.DecodeConfig(bytes.NewReader(buf))
+ if err != nil {
+ t.Fatalf("decode screenshot: %v", err)
+ }
+ return cfg.Width, cfg.Height
+}
+
+func nodeIDForSelector(t *testing.T, ctx context.Context, css string) int64 {
+ t.Helper()
+ nodeID, err := ResolveCSSToNodeID(ctx, css)
+ if err != nil {
+ t.Fatalf("resolve %q: %v", css, err)
+ }
+ return nodeID
+}
+
+// A CSS selector does not cross an iframe boundary, so the in-frame element has
+// to be resolved in the child frame's own execution context.
+func inFrameNodeID(t *testing.T, ctx context.Context, css string) int64 {
+ t.Helper()
+ tree, err := FetchFrameTree(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, child := range tree.ChildFrames {
+ if nodeID, err := ResolveCSSToNodeIDInFrame(ctx, child.Frame.ID, css); err == nil {
+ return nodeID
+ }
+ }
+ t.Fatalf("no child frame resolves %q (%d children)", css, len(tree.ChildFrames))
+ return 0
+}
+
+func capturePNG(t *testing.T, ctx context.Context, clip *page.Viewport) []byte {
+ t.Helper()
+ buf, err := CaptureScreenshot(ctx, ScreenshotOpts{Format: ScreenshotFormatPng, Clip: clip})
+ if err != nil {
+ t.Fatalf("capture: %v", err)
+ }
+ return buf
+}
+
+// The reported defect: a clip was computed correctly and handed to CDP, and the
+// image came back the full viewport anyway — byte-identical to a capture that
+// asked for no clip at all.
+func TestCaptureScreenshotHonoursClipDimensions(t *testing.T) {
+ ctx := newClipFixture(t)
+
+ clip, err := ScreenshotClipForNode(ctx, nodeIDForSelector(t, ctx, "#small"))
+ if err != nil {
+ t.Fatalf("clip for #small: %v", err)
+ }
+ if clip.Width != clipFixtureSmallWidth || clip.Height != clipFixtureSmallHeight {
+ t.Fatalf("clip = %+v, want %dx%d — the clip itself is wrong, before any capture",
+ clip, clipFixtureSmallWidth, clipFixtureSmallHeight)
+ }
+
+ clipped := capturePNG(t, ctx, clip)
+ w, h := imageSize(t, clipped)
+ if w != clipFixtureSmallWidth || h != clipFixtureSmallHeight {
+ t.Errorf("clipped capture is %dx%d, want %dx%d", w, h, clipFixtureSmallWidth, clipFixtureSmallHeight)
+ }
+
+ full := capturePNG(t, ctx, nil)
+ fw, fh := imageSize(t, full)
+ if fw == w && fh == h {
+ t.Fatalf("clipped and unclipped captures are both %dx%d — the fixture cannot tell them apart", w, h)
+ }
+ if bytes.Equal(clipped, full) {
+ t.Error("clipped and unclipped captures are byte-identical: the clip was accepted and ignored")
+ }
+}
+
+// ScreenshotClipForNode walks frameElement ancestors adding each frame's offset,
+// so an in-frame element must clip the region it occupies in the PAGE. Measured
+// inside its own frame the element is at (10,20); in the page it is at (410,320).
+func TestScreenshotClipForNodeAppliesFrameOffset(t *testing.T) {
+ ctx := newClipFixture(t)
+
+ clip, err := ScreenshotClipForNode(ctx, inFrameNodeID(t, ctx, "#inner"))
+ if err != nil {
+ t.Fatalf("clip for in-frame element: %v", err)
+ }
+
+ const wantX, wantY, wantW, wantH = 410, 320, 80, 40
+ if clip.X != wantX || clip.Y != wantY {
+ t.Errorf("in-frame clip origin = (%.0f,%.0f), want (%d,%d) — frame offset not applied (frame-local origin is (10,20))",
+ clip.X, clip.Y, wantX, wantY)
+ }
+ if clip.Width != wantW || clip.Height != wantH {
+ t.Errorf("in-frame clip size = %.0fx%.0f, want %dx%d", clip.Width, clip.Height, wantW, wantH)
+ }
+
+ buf := capturePNG(t, ctx, clip)
+ if w, h := imageSize(t, buf); w != wantW || h != wantH {
+ t.Errorf("in-frame clipped capture is %dx%d, want %dx%d", w, h, wantW, wantH)
+ }
+}
+
+// An element with no box cannot be clipped; that must be an error rather than a
+// silent full-viewport capture.
+func TestScreenshotClipForNodeRejectsEmptyElement(t *testing.T) {
+ ctx := newClipFixture(t)
+
+ if clip, err := ScreenshotClipForNode(ctx, nodeIDForSelector(t, ctx, "#empty")); err == nil {
+ t.Fatalf("zero-size element produced clip %+v, want an error", clip)
+ }
+}
+
+// /capture shares this engine and was strictly worse than /screenshot: it
+// reported coordinateSpace "clip" and a clip rect while returning the full
+// viewport, so a vision model was told to map refs into a 120x60 space while
+// looking at a whole page. The image must match the clip the response advertises.
+func TestPairedCaptureImageMatchesReportedClip(t *testing.T) {
+ ctx := newClipFixture(t)
+
+ clip, err := ScreenshotClipForNode(ctx, nodeIDForSelector(t, ctx, "#small"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ res, err := PairedCapture(ctx, CaptureOpts{
+ MaxDepth: -1,
+ Image: ScreenshotOpts{Format: ScreenshotFormatPng, Clip: clip},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if res.CoordinateSpace != "clip" || res.Clip == nil {
+ t.Fatalf("CoordinateSpace = %q, Clip = %+v; want a clip capture", res.CoordinateSpace, res.Clip)
+ }
+ w, h := imageSize(t, res.ImageBytes)
+ if float64(w) != res.Clip.Width || float64(h) != res.Clip.Height {
+ t.Errorf("image is %dx%d but the response advertises a %.0fx%.0f clip", w, h, res.Clip.Width, res.Clip.Height)
+ }
+}
diff --git a/internal/bridge/screenshot_test.go b/internal/bridge/screenshot_test.go
index 79feeb1b..d2b8e01c 100644
--- a/internal/bridge/screenshot_test.go
+++ b/internal/bridge/screenshot_test.go
@@ -112,8 +112,8 @@ func TestCaptureFromSurface(t *testing.T) {
}{
{name: "plain viewport capture", want: false},
{name: "beyond viewport forces surface", beyondViewport: true, want: true},
- {name: "native-scale clip stays off", clip: &page.Viewport{Scale: 1}, want: false},
- {name: "zero-scale clip treated as native", clip: &page.Viewport{Scale: 0}, want: false},
+ {name: "native-scale clip forces surface", clip: &page.Viewport{Scale: 1}, want: true},
+ {name: "zero-scale clip forces surface", clip: &page.Viewport{Scale: 0}, want: true},
{name: "downscaled clip forces surface", clip: &page.Viewport{Scale: 0.25}, want: true},
{name: "upscaled clip forces surface", clip: &page.Viewport{Scale: 2}, want: true},
}
diff --git a/internal/handlers/screenshot_selector_test.go b/internal/handlers/screenshot_selector_test.go
new file mode 100644
index 00000000..c552aa88
--- /dev/null
+++ b/internal/handlers/screenshot_selector_test.go
@@ -0,0 +1,177 @@
+package handlers
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "image"
+ _ "image/png"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/chromedp/chromedp"
+ "github.com/pinchtab/pinchtab/internal/bridge"
+ "github.com/pinchtab/pinchtab/internal/config"
+ "github.com/pinchtab/pinchtab/internal/testbrowser"
+)
+
+// One element of a known size, far smaller than any viewport, so the expected
+// clipped dimensions are exact rather than a ratio.
+const screenshotFixtureHTML = `<body style="margin:0;background:#fff">
+<div id="small" style="position:absolute;left:40px;top:60px;width:120px;height:60px;background:#c00">small</div>
+<div id="empty" style="position:absolute;left:0;top:0;width:0;height:0"></div>
+</body>`
+
+const (
+ screenshotSmallWidth = 120
+ screenshotSmallHeight = 60
+ screenshotTabID = "tab-shot"
+)
+
+type screenshotFixture struct {
+ handlers *Handlers
+ ctx context.Context
+}
+
+func newScreenshotFixture(t *testing.T) screenshotFixture {
+ t.Helper()
+ chromePath := testbrowser.Path(t)
+
+ dataURL := "data:text/html;base64," + base64.StdEncoding.EncodeToString([]byte(screenshotFixtureHTML))
+ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(
+ chromedp.DefaultExecAllocatorOptions[:],
+ chromedp.ExecPath(chromePath),
+ chromedp.UserDataDir(t.TempDir()),
+ chromedp.Flag("headless", true),
+ chromedp.Flag("no-sandbox", true),
+ )...)
+ ctx, cancelBrowser := chromedp.NewContext(alloc)
+ ctx, cancelTimeout := context.WithTimeout(ctx, 60*time.Second)
+ t.Cleanup(func() {
+ cancelTimeout()
+ cancelBrowser()
+ cancelAlloc()
+ })
+
+ if err := chromedp.Run(ctx,
+ chromedp.Navigate(dataURL),
+ chromedp.WaitVisible("#small", chromedp.ByID),
+ ); err != nil {
+ t.Fatal(err)
+ }
+
+ cfg := &config.RuntimeConfig{ActionTimeout: 10 * time.Second, DefaultBrowser: config.BrowserChrome, StateDir: t.TempDir()}
+ b := bridge.New(context.Background(), ctx, cfg)
+ b.RegisterTab(screenshotTabID, ctx)
+
+ return screenshotFixture{handlers: New(b, cfg, nil, nil, nil), ctx: ctx}
+}
+
+// shoot drives the real endpoint and returns the decoded PNG plus its status.
+func (f screenshotFixture) shoot(t *testing.T, query string) (int, []byte) {
+ t.Helper()
+ rec := httptest.NewRecorder()
+ f.handlers.HandleScreenshot(rec, httptest.NewRequest(http.MethodGet, "/screenshot?format=png&tabId="+screenshotTabID+query, nil))
+ if rec.Code != http.StatusOK {
+ return rec.Code, rec.Body.Bytes()
+ }
+ var resp struct {
+ Base64 string `json:"base64"`
+ }
+ if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
+ t.Fatalf("decode response: %v (body %s)", err, rec.Body.String())
+ }
+ buf, err := base64.StdEncoding.DecodeString(resp.Base64)
+ if err != nil {
+ t.Fatalf("decode base64 image: %v", err)
+ }
+ return rec.Code, buf
+}
+
+func pngSize(t *testing.T, buf []byte) (int, int) {
+ t.Helper()
+ cfg, _, err := image.DecodeConfig(bytes.NewReader(buf))
+ if err != nil {
+ t.Fatalf("decode png: %v", err)
+ }
+ return cfg.Width, cfg.Height
+}
+
+// The reported defect, at the endpoint: a selector was accepted, resolved, and
+// silently ignored — every capture came back the full viewport and byte-identical
+// to one that asked for no selector at all.
+func TestScreenshotSelectorClipsToElement(t *testing.T) {
+ f := newScreenshotFixture(t)
+
+ statusFull, full := f.shoot(t, "")
+ if statusFull != http.StatusOK {
+ t.Fatalf("no-selector capture: status %d, body %s", statusFull, full)
+ }
+ fullW, fullH := pngSize(t, full)
+ if fullW == screenshotSmallWidth && fullH == screenshotSmallHeight {
+ t.Fatalf("viewport is %dx%d, the same as the target element — the fixture cannot tell them apart", fullW, fullH)
+ }
+
+ // A CSS selector and a ref for the same element must give the same answer:
+ // the ref path resolves through the snapshot cache and the CSS path does not.
+ nodeID, err := bridge.ResolveCSSToNodeID(f.ctx, "#small")
+ if err != nil {
+ t.Fatal(err)
+ }
+ f.handlers.Bridge.SetRefCache(screenshotTabID, &bridge.RefCache{Targets: map[string]bridge.RefTarget{
+ "e0": {BackendNodeID: nodeID},
+ }})
+
+ for _, sel := range []string{"%23small", "css:%23small", "e0", "ref:e0"} {
+ status, buf := f.shoot(t, "&selector="+sel)
+ if status != http.StatusOK {
+ t.Errorf("selector %q: status %d, body %s", sel, status, buf)
+ continue
+ }
+ w, h := pngSize(t, buf)
+ if w != screenshotSmallWidth || h != screenshotSmallHeight {
+ t.Errorf("selector %q: image is %dx%d, want %dx%d", sel, w, h, screenshotSmallWidth, screenshotSmallHeight)
+ }
+ if bytes.Equal(buf, full) {
+ t.Errorf("selector %q: image is byte-identical to the no-selector capture — the selector was ignored", sel)
+ }
+ }
+}
+
+// An element with no box cannot be clipped. Returning the whole viewport would
+// be the original defect wearing a different hat, so it has to be an error.
+func TestScreenshotSelectorThatCannotBeClippedErrors(t *testing.T) {
+ f := newScreenshotFixture(t)
+
+ status, body := f.shoot(t, "&selector=%23empty")
+ if status == http.StatusOK {
+ w, h := pngSize(t, body)
+ t.Fatalf("zero-size element returned a %dx%d image, want an error", w, h)
+ }
+ if status < 400 || status > 599 {
+ t.Errorf("status = %d, want a 4xx/5xx error", status)
+ }
+}
+
+// Documented precedence: a selector already clips to an element, so it wins and
+// beyondViewport is dropped. The combination must still yield the element's box,
+// not a document-sized capture.
+func TestScreenshotSelectorWinsOverBeyondViewport(t *testing.T) {
+ f := newScreenshotFixture(t)
+
+ if req := parseScreenshotRequest(httptest.NewRequest(http.MethodGet, "/screenshot?beyondViewport=true&selector=%23small", nil)); req.beyondViewport {
+ t.Error("beyondViewport must be dropped when a selector is set")
+ }
+
+ status, buf := f.shoot(t, "&selector=%23small&beyondViewport=true")
+ if status != http.StatusOK {
+ t.Fatalf("status %d, body %s", status, buf)
+ }
+ w, h := pngSize(t, buf)
+ if w != screenshotSmallWidth || h != screenshotSmallHeight {
+ t.Errorf("selector+beyondViewport image is %dx%d, want the element's %dx%d", w, h, screenshotSmallWidth, screenshotSmallHeight)
+ }
+}