High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix: screenshot selector is accepted and silently ignored, always returning the full viewport (PIN-033)
Luigi Agostid396b3baabd6
diff --git a/internal/bridge/screenshot.go b/internal/bridge/screenshot.goindex fc6f327f..98bcdd50 100644--- a/internal/bridge/screenshot.go+++ b/internal/bridge/screenshot.go@@ -142,16 +142,16 @@ func scaledScreenshotClip(opts ScreenshotOpts, viewportWidth, viewportHeight, do// waiting for a fresh compositor surface frame. On idle pages in headed browsers// (e.g. Cloak) the surface stops swapping frames, so the default fromSurface=true// blocks until the action deadline (~30s); reading from the view avoids that for-// plain captures. But capture-beyond-viewport and any render-time rescale-// (clip.Scale != 1) both need the page recomposited at a new size, which only-// happens with fromSurface=true — forcing it off there silently drops the-// scale/beyond-viewport effect. So keep it off only for the plain/native-scale-// path.+// plain captures.+//+// Anything that changes the captured region needs the page recomposited, which+// only happens with fromSurface=true: capture-beyond-viewport, a render-time+// rescale, and the clip itself. Reading the view ignores the clip outright and+// returns the whole viewport, so every non-nil clip takes the surface path —+// keeping it off for a native-scale clip is what made a selector capture come+// back byte-identical to an unclipped one.func captureFromSurface(beyondViewport bool, clip *page.Viewport) bool {- if beyondViewport {- return true- }- return clip != nil && clip.Scale != 0 && clip.Scale != 1+ return beyondViewport || clip != nil}// captureScreenshotWithoutActivation wakes a background renderer beforediff --git a/internal/bridge/screenshot_clip_test.go b/internal/bridge/screenshot_clip_test.gonew file mode 100644index 00000000..a74a1144--- /dev/null+++ b/internal/bridge/screenshot_clip_test.go@@ -0,0 +1,194 @@+package bridge++import (+ "bytes"+ "context"+ "encoding/base64"+ "image"+ _ "image/png"+ "testing"+ "time"++ "github.com/chromedp/cdproto/page"+ "github.com/chromedp/chromedp"+ "github.com/pinchtab/pinchtab/internal/testbrowser"+)++// A single element of known size at a known offset, materially smaller than any+// viewport, so the expected clipped dimensions are exact rather than a ratio.+const clipFixtureHTML = `<body style="margin:0;background:#fff">+<div id="small" style="position:absolute;left:40px;top:60px;width:120px;height:60px;background:#c00"></div>+<div id="empty" style="position:absolute;left:0;top:0;width:0;height:0"></div>+<iframe id="f" style="position:absolute;left:400px;top:300px;border:0" width="300" height="200"+ srcdoc="<body style='margin:0'><div id='inner' style='position:absolute;left:10px;top:20px;width:80px;height:40px;background:#00c'></div></body>"></iframe>+</body>`++const clipFixtureSmallWidth, clipFixtureSmallHeight = 120, 60++func newClipFixture(t *testing.T) context.Context {+ t.Helper()+ chromePath := testbrowser.Path(t)++ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(+ chromedp.DefaultExecAllocatorOptions[:],+ chromedp.ExecPath(chromePath),+ chromedp.UserDataDir(t.TempDir()),+ chromedp.Flag("headless", true),+ chromedp.Flag("no-sandbox", true),+ )...)+ ctx, cancelBrowser := chromedp.NewContext(alloc)+ ctx, cancelTimeout := context.WithTimeout(ctx, 60*time.Second)+ t.Cleanup(func() {+ cancelTimeout()+ cancelBrowser()+ cancelAlloc()+ })++ dataURL := "data:text/html;base64," + base64.StdEncoding.EncodeToString([]byte(clipFixtureHTML))+ if err := chromedp.Run(ctx,+ chromedp.Navigate(dataURL),+ chromedp.WaitVisible("#small", chromedp.ByID),+ ); err != nil {+ t.Fatal(err)+ }+ return ctx+}++func imageSize(t *testing.T, buf []byte) (int, int) {+ t.Helper()+ cfg, _, err := image.DecodeConfig(bytes.NewReader(buf))+ if err != nil {+ t.Fatalf("decode screenshot: %v", err)+ }+ return cfg.Width, cfg.Height+}++func nodeIDForSelector(t *testing.T, ctx context.Context, css string) int64 {+ t.Helper()+ nodeID, err := ResolveCSSToNodeID(ctx, css)+ if err != nil {+ t.Fatalf("resolve %q: %v", css, err)+ }+ return nodeID+}++// A CSS selector does not cross an iframe boundary, so the in-frame element has+// to be resolved in the child frame's own execution context.+func inFrameNodeID(t *testing.T, ctx context.Context, css string) int64 {+ t.Helper()+ tree, err := FetchFrameTree(ctx)+ if err != nil {+ t.Fatal(err)+ }+ for _, child := range tree.ChildFrames {+ if nodeID, err := ResolveCSSToNodeIDInFrame(ctx, child.Frame.ID, css); err == nil {+ return nodeID+ }+ }+ t.Fatalf("no child frame resolves %q (%d children)", css, len(tree.ChildFrames))+ return 0+}++func capturePNG(t *testing.T, ctx context.Context, clip *page.Viewport) []byte {+ t.Helper()+ buf, err := CaptureScreenshot(ctx, ScreenshotOpts{Format: ScreenshotFormatPng, Clip: clip})+ if err != nil {+ t.Fatalf("capture: %v", err)+ }+ return buf+}++// The reported defect: a clip was computed correctly and handed to CDP, and the+// image came back the full viewport anyway — byte-identical to a capture that+// asked for no clip at all.+func TestCaptureScreenshotHonoursClipDimensions(t *testing.T) {+ ctx := newClipFixture(t)++ clip, err := ScreenshotClipForNode(ctx, nodeIDForSelector(t, ctx, "#small"))+ if err != nil {+ t.Fatalf("clip for #small: %v", err)+ }+ if clip.Width != clipFixtureSmallWidth || clip.Height != clipFixtureSmallHeight {+ t.Fatalf("clip = %+v, want %dx%d — the clip itself is wrong, before any capture",+ clip, clipFixtureSmallWidth, clipFixtureSmallHeight)+ }++ clipped := capturePNG(t, ctx, clip)+ w, h := imageSize(t, clipped)+ if w != clipFixtureSmallWidth || h != clipFixtureSmallHeight {+ t.Errorf("clipped capture is %dx%d, want %dx%d", w, h, clipFixtureSmallWidth, clipFixtureSmallHeight)+ }++ full := capturePNG(t, ctx, nil)+ fw, fh := imageSize(t, full)+ if fw == w && fh == h {+ t.Fatalf("clipped and unclipped captures are both %dx%d — the fixture cannot tell them apart", w, h)+ }+ if bytes.Equal(clipped, full) {+ t.Error("clipped and unclipped captures are byte-identical: the clip was accepted and ignored")+ }+}++// ScreenshotClipForNode walks frameElement ancestors adding each frame's offset,+// so an in-frame element must clip the region it occupies in the PAGE. Measured+// inside its own frame the element is at (10,20); in the page it is at (410,320).+func TestScreenshotClipForNodeAppliesFrameOffset(t *testing.T) {+ ctx := newClipFixture(t)++ clip, err := ScreenshotClipForNode(ctx, inFrameNodeID(t, ctx, "#inner"))+ if err != nil {+ t.Fatalf("clip for in-frame element: %v", err)+ }++ const wantX, wantY, wantW, wantH = 410, 320, 80, 40+ if clip.X != wantX || clip.Y != wantY {+ t.Errorf("in-frame clip origin = (%.0f,%.0f), want (%d,%d) — frame offset not applied (frame-local origin is (10,20))",+ clip.X, clip.Y, wantX, wantY)+ }+ if clip.Width != wantW || clip.Height != wantH {+ t.Errorf("in-frame clip size = %.0fx%.0f, want %dx%d", clip.Width, clip.Height, wantW, wantH)+ }++ buf := capturePNG(t, ctx, clip)+ if w, h := imageSize(t, buf); w != wantW || h != wantH {+ t.Errorf("in-frame clipped capture is %dx%d, want %dx%d", w, h, wantW, wantH)+ }+}++// An element with no box cannot be clipped; that must be an error rather than a+// silent full-viewport capture.+func TestScreenshotClipForNodeRejectsEmptyElement(t *testing.T) {+ ctx := newClipFixture(t)++ if clip, err := ScreenshotClipForNode(ctx, nodeIDForSelector(t, ctx, "#empty")); err == nil {+ t.Fatalf("zero-size element produced clip %+v, want an error", clip)+ }+}++// /capture shares this engine and was strictly worse than /screenshot: it+// reported coordinateSpace "clip" and a clip rect while returning the full+// viewport, so a vision model was told to map refs into a 120x60 space while+// looking at a whole page. The image must match the clip the response advertises.+func TestPairedCaptureImageMatchesReportedClip(t *testing.T) {+ ctx := newClipFixture(t)++ clip, err := ScreenshotClipForNode(ctx, nodeIDForSelector(t, ctx, "#small"))+ if err != nil {+ t.Fatal(err)+ }+ res, err := PairedCapture(ctx, CaptureOpts{+ MaxDepth: -1,+ Image: ScreenshotOpts{Format: ScreenshotFormatPng, Clip: clip},+ })+ if err != nil {+ t.Fatal(err)+ }++ if res.CoordinateSpace != "clip" || res.Clip == nil {+ t.Fatalf("CoordinateSpace = %q, Clip = %+v; want a clip capture", res.CoordinateSpace, res.Clip)+ }+ w, h := imageSize(t, res.ImageBytes)+ if float64(w) != res.Clip.Width || float64(h) != res.Clip.Height {+ t.Errorf("image is %dx%d but the response advertises a %.0fx%.0f clip", w, h, res.Clip.Width, res.Clip.Height)+ }+}diff --git a/internal/bridge/screenshot_test.go b/internal/bridge/screenshot_test.goindex 79feeb1b..d2b8e01c 100644--- a/internal/bridge/screenshot_test.go+++ b/internal/bridge/screenshot_test.go@@ -112,8 +112,8 @@ func TestCaptureFromSurface(t *testing.T) {}{{name: "plain viewport capture", want: false},{name: "beyond viewport forces surface", beyondViewport: true, want: true},- {name: "native-scale clip stays off", clip: &page.Viewport{Scale: 1}, want: false},- {name: "zero-scale clip treated as native", clip: &page.Viewport{Scale: 0}, want: false},+ {name: "native-scale clip forces surface", clip: &page.Viewport{Scale: 1}, want: true},+ {name: "zero-scale clip forces surface", clip: &page.Viewport{Scale: 0}, want: true},{name: "downscaled clip forces surface", clip: &page.Viewport{Scale: 0.25}, want: true},{name: "upscaled clip forces surface", clip: &page.Viewport{Scale: 2}, want: true},}diff --git a/internal/handlers/screenshot_selector_test.go b/internal/handlers/screenshot_selector_test.gonew file mode 100644index 00000000..c552aa88--- /dev/null+++ b/internal/handlers/screenshot_selector_test.go@@ -0,0 +1,177 @@+package handlers++import (+ "bytes"+ "context"+ "encoding/base64"+ "encoding/json"+ "image"+ _ "image/png"+ "net/http"+ "net/http/httptest"+ "testing"+ "time"++ "github.com/chromedp/chromedp"+ "github.com/pinchtab/pinchtab/internal/bridge"+ "github.com/pinchtab/pinchtab/internal/config"+ "github.com/pinchtab/pinchtab/internal/testbrowser"+)++// One element of a known size, far smaller than any viewport, so the expected+// clipped dimensions are exact rather than a ratio.+const screenshotFixtureHTML = `<body style="margin:0;background:#fff">+<div id="small" style="position:absolute;left:40px;top:60px;width:120px;height:60px;background:#c00">small</div>+<div id="empty" style="position:absolute;left:0;top:0;width:0;height:0"></div>+</body>`++const (+ screenshotSmallWidth = 120+ screenshotSmallHeight = 60+ screenshotTabID = "tab-shot"+)++type screenshotFixture struct {+ handlers *Handlers+ ctx context.Context+}++func newScreenshotFixture(t *testing.T) screenshotFixture {+ t.Helper()+ chromePath := testbrowser.Path(t)++ dataURL := "data:text/html;base64," + base64.StdEncoding.EncodeToString([]byte(screenshotFixtureHTML))+ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(+ chromedp.DefaultExecAllocatorOptions[:],+ chromedp.ExecPath(chromePath),+ chromedp.UserDataDir(t.TempDir()),+ chromedp.Flag("headless", true),+ chromedp.Flag("no-sandbox", true),+ )...)+ ctx, cancelBrowser := chromedp.NewContext(alloc)+ ctx, cancelTimeout := context.WithTimeout(ctx, 60*time.Second)+ t.Cleanup(func() {+ cancelTimeout()+ cancelBrowser()+ cancelAlloc()+ })++ if err := chromedp.Run(ctx,+ chromedp.Navigate(dataURL),+ chromedp.WaitVisible("#small", chromedp.ByID),+ ); err != nil {+ t.Fatal(err)+ }++ cfg := &config.RuntimeConfig{ActionTimeout: 10 * time.Second, DefaultBrowser: config.BrowserChrome, StateDir: t.TempDir()}+ b := bridge.New(context.Background(), ctx, cfg)+ b.RegisterTab(screenshotTabID, ctx)++ return screenshotFixture{handlers: New(b, cfg, nil, nil, nil), ctx: ctx}+}++// shoot drives the real endpoint and returns the decoded PNG plus its status.+func (f screenshotFixture) shoot(t *testing.T, query string) (int, []byte) {+ t.Helper()+ rec := httptest.NewRecorder()+ f.handlers.HandleScreenshot(rec, httptest.NewRequest(http.MethodGet, "/screenshot?format=png&tabId="+screenshotTabID+query, nil))+ if rec.Code != http.StatusOK {+ return rec.Code, rec.Body.Bytes()+ }+ var resp struct {+ Base64 string `json:"base64"`+ }+ if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {+ t.Fatalf("decode response: %v (body %s)", err, rec.Body.String())+ }+ buf, err := base64.StdEncoding.DecodeString(resp.Base64)+ if err != nil {+ t.Fatalf("decode base64 image: %v", err)+ }+ return rec.Code, buf+}++func pngSize(t *testing.T, buf []byte) (int, int) {+ t.Helper()+ cfg, _, err := image.DecodeConfig(bytes.NewReader(buf))+ if err != nil {+ t.Fatalf("decode png: %v", err)+ }+ return cfg.Width, cfg.Height+}++// The reported defect, at the endpoint: a selector was accepted, resolved, and+// silently ignored — every capture came back the full viewport and byte-identical+// to one that asked for no selector at all.+func TestScreenshotSelectorClipsToElement(t *testing.T) {+ f := newScreenshotFixture(t)++ statusFull, full := f.shoot(t, "")+ if statusFull != http.StatusOK {+ t.Fatalf("no-selector capture: status %d, body %s", statusFull, full)+ }+ fullW, fullH := pngSize(t, full)+ if fullW == screenshotSmallWidth && fullH == screenshotSmallHeight {+ t.Fatalf("viewport is %dx%d, the same as the target element — the fixture cannot tell them apart", fullW, fullH)+ }++ // A CSS selector and a ref for the same element must give the same answer:+ // the ref path resolves through the snapshot cache and the CSS path does not.+ nodeID, err := bridge.ResolveCSSToNodeID(f.ctx, "#small")+ if err != nil {+ t.Fatal(err)+ }+ f.handlers.Bridge.SetRefCache(screenshotTabID, &bridge.RefCache{Targets: map[string]bridge.RefTarget{+ "e0": {BackendNodeID: nodeID},+ }})++ for _, sel := range []string{"%23small", "css:%23small", "e0", "ref:e0"} {+ status, buf := f.shoot(t, "&selector="+sel)+ if status != http.StatusOK {+ t.Errorf("selector %q: status %d, body %s", sel, status, buf)+ continue+ }+ w, h := pngSize(t, buf)+ if w != screenshotSmallWidth || h != screenshotSmallHeight {+ t.Errorf("selector %q: image is %dx%d, want %dx%d", sel, w, h, screenshotSmallWidth, screenshotSmallHeight)+ }+ if bytes.Equal(buf, full) {+ t.Errorf("selector %q: image is byte-identical to the no-selector capture — the selector was ignored", sel)+ }+ }+}++// An element with no box cannot be clipped. Returning the whole viewport would+// be the original defect wearing a different hat, so it has to be an error.+func TestScreenshotSelectorThatCannotBeClippedErrors(t *testing.T) {+ f := newScreenshotFixture(t)++ status, body := f.shoot(t, "&selector=%23empty")+ if status == http.StatusOK {+ w, h := pngSize(t, body)+ t.Fatalf("zero-size element returned a %dx%d image, want an error", w, h)+ }+ if status < 400 || status > 599 {+ t.Errorf("status = %d, want a 4xx/5xx error", status)+ }+}++// Documented precedence: a selector already clips to an element, so it wins and+// beyondViewport is dropped. The combination must still yield the element's box,+// not a document-sized capture.+func TestScreenshotSelectorWinsOverBeyondViewport(t *testing.T) {+ f := newScreenshotFixture(t)++ if req := parseScreenshotRequest(httptest.NewRequest(http.MethodGet, "/screenshot?beyondViewport=true&selector=%23small", nil)); req.beyondViewport {+ t.Error("beyondViewport must be dropped when a selector is set")+ }++ status, buf := f.shoot(t, "&selector=%23small&beyondViewport=true")+ if status != http.StatusOK {+ t.Fatalf("status %d, body %s", status, buf)+ }+ w, h := pngSize(t, buf)+ if w != screenshotSmallWidth || h != screenshotSmallHeight {+ t.Errorf("selector+beyondViewport image is %dx%d, want the element's %dx%d", w, h, screenshotSmallWidth, screenshotSmallHeight)+ }+}