Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix: a measured visible:false is erased from capture payloads by omitempty (PIN-057)

Luigi Agostie488f2f3d1a8
diff --git a/docs/reference/capture.md b/docs/reference/capture.md
index 599e4d7e..216aa2b6 100644
--- a/docs/reference/capture.md
+++ b/docs/reference/capture.md
@@ -60,6 +60,11 @@ pinchtab capture -s "#checkout-form"
"ref": "e4", "role": "textbox", "name": "Email",
"boundingBox": { "x": 520, "y": 312, "w": 280, "h": 36 },
"visible": true
+ },
+ {
+ "ref": "e9", "role": "button", "name": "Submit",
+ "boundingBox": { "x": 520, "y": 1480, "w": 96, "h": 40 },
+ "visible": false
}
]
}
@@ -80,9 +85,10 @@ an `expectedEpoch` query param to reject stale refs at use time.
## Bounding boxes and coordinate space
-When `withBounds=true` (the default), each snapshot node with a
-non-zero backend node id gets a `boundingBox` and a `visible` flag. The
-coordinate space depends on `selector` and `beyondViewport`:
+When `withBounds=true` (the default), each snapshot node with a non-zero
+backend node id gets a `boundingBox` and a `visible` flag, and a node that
+cannot be measured gets neither. The coordinate space depends on `selector`
+and `beyondViewport`:
- **`viewport`** (default): boxes are viewport-relative CSS pixels. The
image is the visible viewport. `image.devicePixelRatio` tells you the
@@ -95,7 +101,15 @@ coordinate space depends on `selector` and `beyondViewport`:
document.
`visible` is true when the box has positive area and intersects the
-viewport — a cheap heuristic, not a strict occlusion check.
+viewport — a cheap heuristic, not a strict occlusion check. A node
+scrolled past, in either direction, is measured and reports
+`"visible": false`.
+
+**Absent means not measured, never "no".** `visible` appears exactly when
+`boundingBox` does, so the key is missing only where no measurement was
+taken: `withBounds=false`, a node with no backend node id, or a node whose
+box query failed. Treat a missing `visible` as unknown and a present
+`false` as off-screen; they are different answers.
## Useful flags
@@ -111,7 +125,7 @@ viewport — a cheap heuristic, not a strict occlusion check.
| `depth` | Snapshot tree depth limit |
| `output` | `file` (default), `inline` (base64 in JSON), or `raw` (bytes only — drops the snapshot) |
| `wait` | `stable` (default) waits for `Page.lifecycleEvent` quiescence (250ms silence / 750ms ceiling); `load` polls `document.readyState` until `complete` (2s ceiling); `none` skips the wait |
-| `withBounds` | `true` (default) — populate `boundingBox` + `visible` on every snapshot node |
+| `withBounds` | `true` (default) — populate `boundingBox` + `visible` on every measurable snapshot node; `false` omits both keys everywhere |
| `beyondViewport` | `true` — capture the full scrollable document; coordinate space becomes `document` |
| `scale` | Rescale the output bitmap. Default `1`. `0.5` halves each axis (quarter the pixels) |
| `requirePair` | `true` returns 409 if `pairing.navigated` would be true |
diff --git a/internal/bridge/observe/bounds.go b/internal/bridge/observe/bounds.go
index 63a96426..6736d972 100644
--- a/internal/bridge/observe/bounds.go
+++ b/internal/bridge/observe/bounds.go
@@ -88,7 +88,7 @@ func AnnotateBounds(ctx context.Context, nodes []A11yNode, pageCoords bool, vp V
box.Y += vp.ScrollY
}
nodes[i].BoundingBox = &box
- nodes[i].Visible = visible
+ nodes[i].Visible = &visible
}
return nil
}
diff --git a/internal/bridge/observe/bounds_test.go b/internal/bridge/observe/bounds_test.go
index 91656f02..b3724bc4 100644
--- a/internal/bridge/observe/bounds_test.go
+++ b/internal/bridge/observe/bounds_test.go
@@ -134,7 +134,10 @@ func (f scrollFixture) annotate(t *testing.T, pageCoords bool) (BoundingBox, boo
if nodes[0].BoundingBox == nil {
t.Fatal("AnnotateBounds produced no bounding box for the target")
}
- return *nodes[0].BoundingBox, nodes[0].Visible, vp
+ if nodes[0].Visible == nil {
+ t.Fatal("AnnotateBounds measured the target but left visible unset")
+ }
+ return *nodes[0].BoundingBox, *nodes[0].Visible, vp
}
// getBoxModel reports the content box and getBoundingClientRect the border box,
@@ -266,3 +269,46 @@ func TestElementBorderBoxIsViewportRelativeWhenScrolled(t *testing.T) {
assertNear(t, "border-box x", box.X, rect.X)
assertNear(t, "border-box y", box.Y, rect.Y)
}
+
+// Every unmeasured path for real, beside a measured one, in a single snapshot:
+// a node the bounds pass skips (NodeID == 0), a node whose getBoxModel call
+// fails (a backend id no document holds), and the target that measures fine.
+// The invariant has to hold across the mix, since it is what lets a client read
+// an absent "visible" as "not measured" rather than "off-screen".
+func TestAnnotateBoundsPairsVisibleWithBoundsAcrossUnmeasuredNodes(t *testing.T) {
+ f := newScrollFixture(t, 300, 400)
+
+ vp, err := FetchLayout(f.ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ nodes := []A11yNode{
+ {Ref: "e0", NodeID: 0},
+ {Ref: "e1", NodeID: f.nodeID},
+ {Ref: "e2", NodeID: 1 << 40},
+ }
+ if err := AnnotateBounds(f.ctx, nodes, false, vp); err != nil {
+ t.Fatal(err)
+ }
+
+ encoded := make([]map[string]any, 0, len(nodes))
+ for _, node := range nodes {
+ encoded = append(encoded, marshalNode(t, node))
+ }
+ assertVisiblePairsWithBounds(t, encoded)
+
+ if _, ok := encoded[1]["visible"]; !ok {
+ t.Fatalf("the measured target lost its visible key: %v", encoded[1])
+ }
+ for _, i := range []int{0, 2} {
+ if _, ok := encoded[i]["visible"]; ok {
+ t.Errorf("unmeasured node %d published a visible key: %v", i, encoded[i])
+ }
+ }
+
+ // The target sits below the fold at this scroll, so its measured answer is
+ // false — the value that used to vanish.
+ if encoded[1]["visible"] != false {
+ t.Errorf("target visible = %v, want a measured false", encoded[1]["visible"])
+ }
+}
diff --git a/internal/bridge/observe/snapshot.go b/internal/bridge/observe/snapshot.go
index 7b9c64a1..63216c62 100644
--- a/internal/bridge/observe/snapshot.go
+++ b/internal/bridge/observe/snapshot.go
@@ -28,6 +28,10 @@ func IsSensitiveAutocomplete(value string) bool {
return false
}
+// A11yNode is one snapshot node. Visible is a pointer because nil (absent on
+// the wire) means the bounds pass never measured this node, which is a
+// different statement from a measured false: it is set exactly when
+// BoundingBox is.
type A11yNode struct {
Ref string `json:"ref"`
Role string `json:"role"`
@@ -52,7 +56,7 @@ type A11yNode struct {
ChildFrameURL string `json:"childFrameUrl,omitempty"`
ChildFrameName string `json:"childFrameName,omitempty"`
BoundingBox *BoundingBox `json:"boundingBox,omitempty"`
- Visible bool `json:"visible,omitempty"`
+ Visible *bool `json:"visible,omitempty"`
}
// BoundingBox is a CSS-pixel rectangle for a snapshot node. Populated only by
diff --git a/internal/bridge/observe/snapshot_visible_test.go b/internal/bridge/observe/snapshot_visible_test.go
new file mode 100644
index 00000000..06f905b1
--- /dev/null
+++ b/internal/bridge/observe/snapshot_visible_test.go
@@ -0,0 +1,85 @@
+package observe
+
+import (
+ "encoding/json"
+ "testing"
+)
+
+func marshalNode(t *testing.T, node A11yNode) map[string]any {
+ t.Helper()
+ raw, err := json.Marshal(node)
+ if err != nil {
+ t.Fatalf("marshal node: %v", err)
+ }
+ var decoded map[string]any
+ if err := json.Unmarshal(raw, &decoded); err != nil {
+ t.Fatalf("decode node: %v", err)
+ }
+ return decoded
+}
+
+// assertVisiblePairsWithBounds is the invariant the whole card rests on: the
+// wire says "visible" exactly when it says "boundingBox", so absence of the key
+// means "not measured" and nothing else.
+func assertVisiblePairsWithBounds(t *testing.T, nodes []map[string]any) {
+ t.Helper()
+ if len(nodes) == 0 {
+ t.Fatal("no nodes to check the visible/boundingBox invariant over")
+ }
+ for i, node := range nodes {
+ _, hasVisible := node["visible"]
+ _, hasBounds := node["boundingBox"]
+ if hasVisible != hasBounds {
+ t.Errorf("node %d (%v): visible present=%v but boundingBox present=%v", i, node["ref"], hasVisible, hasBounds)
+ }
+ }
+}
+
+// A measured node that sits off-screen must serialise its false, which is the
+// value omitempty on a plain bool erased.
+func TestMeasuredOffScreenNodeSerialisesVisibleFalse(t *testing.T) {
+ offScreen := false
+ node := marshalNode(t, A11yNode{Ref: "e1", BoundingBox: &BoundingBox{Y: -2410, W: 80, H: 20}, Visible: &offScreen})
+
+ value, ok := node["visible"]
+ if !ok {
+ t.Fatalf("a measured off-screen node lost its visible key: %v", node)
+ }
+ if value != false {
+ t.Fatalf("visible = %v, want false", value)
+ }
+}
+
+func TestUnmeasuredNodeSerialisesWithoutVisibleKey(t *testing.T) {
+ node := marshalNode(t, A11yNode{Ref: "e1"})
+ if _, ok := node["visible"]; ok {
+ t.Fatalf("an unmeasured node published a visible key: %v", node)
+ }
+
+ onScreen := true
+ measured := marshalNode(t, A11yNode{Ref: "e2", BoundingBox: &BoundingBox{W: 80, H: 20}, Visible: &onScreen})
+ if measured["visible"] != true {
+ t.Fatalf("visible = %v, want true", measured["visible"])
+ }
+
+ assertVisiblePairsWithBounds(t, []map[string]any{node, measured})
+}
+
+// The other three bools come from the accessibility tree for every node, so for
+// them absent already means false and omitempty is correct. Converting them for
+// symmetry would widen the wire change for no gain, so the tags are pinned.
+func TestAccessibilityTreeBoolsKeepOmitempty(t *testing.T) {
+ node := marshalNode(t, A11yNode{Ref: "e1"})
+ for _, key := range []string{"disabled", "focused", "hidden"} {
+ if _, ok := node[key]; ok {
+ t.Errorf("%q must stay omitempty: a false value belongs off the wire", key)
+ }
+ }
+
+ enabled := marshalNode(t, A11yNode{Ref: "e1", Disabled: true, Focused: true, Hidden: true})
+ for _, key := range []string{"disabled", "focused", "hidden"} {
+ if enabled[key] != true {
+ t.Errorf("%q = %v, want true", key, enabled[key])
+ }
+ }
+}
diff --git a/internal/handlers/capture_test.go b/internal/handlers/capture_test.go
index 28cb787e..f6d88930 100644
--- a/internal/handlers/capture_test.go
+++ b/internal/handlers/capture_test.go
@@ -1,12 +1,20 @@
package handlers
import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
"net/http"
"net/http/httptest"
+ "os"
"strings"
"testing"
+ "time"
+ "github.com/chromedp/chromedp"
+ "github.com/pinchtab/pinchtab/internal/bridge"
"github.com/pinchtab/pinchtab/internal/config"
+ "github.com/pinchtab/pinchtab/internal/testbrowser"
)
func TestHandleCapture_NoTab(t *testing.T) {
@@ -114,3 +122,155 @@ func TestHandleCapture_OpenAPIExposes(t *testing.T) {
t.Fatalf("expected /openapi.json to list /capture, got %s", w.Body.String())
}
}
+
+// capturePairFixtureHTML puts one button above the fold and one far below it, so
+// a single scroll leaves one measured-and-on-screen and one
+// measured-and-off-screen — the two states the wire could not tell apart while
+// the field was an omitempty bool. Scrolled past ABOVE is the direction the
+// original report checked last, and it must serialise false like any other
+// off-screen node.
+const capturePairFixtureHTML = `<body style="margin:0;height:4000px">
+<button id="top" style="position:absolute;top:0;left:0">TopBtn</button>
+<button id="bottom" style="position:absolute;top:2500px;left:0">BotBtn</button>
+</body>`
+
+const capturePairTabID = "tab-capture-visible"
+
+type captureFixture struct {
+ handlers *Handlers
+}
+
+func newCaptureFixture(t *testing.T) captureFixture {
+ t.Helper()
+ chromePath := testbrowser.Path(t)
+
+ dataURL := "data:text/html;base64," + base64.StdEncoding.EncodeToString([]byte(capturePairFixtureHTML))
+ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(
+ chromedp.DefaultExecAllocatorOptions[:],
+ chromedp.ExecPath(chromePath),
+ chromedp.UserDataDir(t.TempDir()),
+ chromedp.Flag("headless", true),
+ chromedp.Flag("no-sandbox", true),
+ )...)
+ ctx, cancelBrowser := chromedp.NewContext(alloc)
+ ctx, cancelTimeout := context.WithTimeout(ctx, 60*time.Second)
+ t.Cleanup(func() {
+ cancelTimeout()
+ cancelBrowser()
+ cancelAlloc()
+ })
+
+ var scrolled bool
+ if err := chromedp.Run(ctx,
+ chromedp.Navigate(dataURL),
+ chromedp.WaitVisible("#bottom", chromedp.ByID),
+ chromedp.Evaluate(`(() => { window.scrollTo(0, 2423); return window.scrollY > 2000; })()`, &scrolled),
+ ); err != nil {
+ t.Fatal(err)
+ }
+ if !scrolled {
+ t.Fatal("fixture did not scroll; both buttons would report the same visibility")
+ }
+
+ cfg := &config.RuntimeConfig{ActionTimeout: 10 * time.Second, DefaultBrowser: config.BrowserChrome, StateDir: t.TempDir()}
+ b := bridge.New(context.Background(), ctx, cfg)
+ b.RegisterTab(capturePairTabID, ctx)
+
+ return captureFixture{handlers: New(b, cfg, nil, nil, nil)}
+}
+
+// capture drives the real endpoint and returns the snapshot nodes as they were
+// SERIALISED, not as Go structs: the defect lived entirely in the encoding, so
+// decoding into a typed struct would restore what the wire had lost.
+func (f captureFixture) capture(t *testing.T, query string) []map[string]any {
+ t.Helper()
+ rec := httptest.NewRecorder()
+ f.handlers.HandleCapture(rec, httptest.NewRequest(http.MethodGet, "/capture?output=inline&format=png&tabId="+capturePairTabID+query, nil))
+ if rec.Code != http.StatusOK {
+ if strings.Contains(rec.Body.String(), "Unable to capture screenshot") {
+ t.Skipf("local Chrome cannot capture screenshots: %s", rec.Body.String())
+ }
+ t.Fatalf("capture status %d: %s", rec.Code, rec.Body.String())
+ }
+ var resp struct {
+ Snapshot struct {
+ Nodes []map[string]any `json:"nodes"`
+ } `json:"snapshot"`
+ }
+ if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
+ t.Fatalf("decode capture response: %v (body %s)", err, rec.Body.String())
+ }
+ if len(resp.Snapshot.Nodes) == 0 {
+ t.Fatal("capture returned no snapshot nodes")
+ }
+ return resp.Snapshot.Nodes
+}
+
+func nodeByName(t *testing.T, nodes []map[string]any, name string) map[string]any {
+ t.Helper()
+ for _, node := range nodes {
+ if node["name"] == name {
+ return node
+ }
+ }
+ t.Fatalf("no node named %q in the capture snapshot (%d nodes)", name, len(nodes))
+ return nil
+}
+
+// The reported defect on the wire: the off-screen button carried no visible key
+// at all, so a client could not tell "measured, off-screen" from "never
+// measured". Both buttons are measured here, so both must state their answer.
+func TestCaptureSerialisesMeasuredVisibleFalse(t *testing.T) {
+ nodes := newCaptureFixture(t).capture(t, "&bounds=true")
+
+ offScreen := nodeByName(t, nodes, "TopBtn")
+ value, ok := offScreen["visible"]
+ if !ok {
+ t.Fatalf("the scrolled-past button published no visible key: %v", offScreen)
+ }
+ if value != false {
+ t.Errorf("scrolled-past button visible = %v, want false", value)
+ }
+
+ onScreen := nodeByName(t, nodes, "BotBtn")
+ if onScreen["visible"] != true {
+ t.Errorf("on-screen button visible = %v, want true", onScreen["visible"])
+ }
+
+ for i, node := range nodes {
+ _, hasVisible := node["visible"]
+ _, hasBounds := node["boundingBox"]
+ if hasVisible != hasBounds {
+ t.Errorf("node %d (%v): visible present=%v but boundingBox present=%v", i, node["ref"], hasVisible, hasBounds)
+ }
+ }
+}
+
+// bounds=false measures nothing, so no node may claim a visibility it was never
+// asked to compute.
+func TestCaptureWithoutBoundsPublishesNoVisibleKey(t *testing.T) {
+ nodes := newCaptureFixture(t).capture(t, "&bounds=false")
+
+ for i, node := range nodes {
+ if _, ok := node["visible"]; ok {
+ t.Errorf("node %d (%v) published visible without a bounds pass: %v", i, node["ref"], node)
+ }
+ if _, ok := node["boundingBox"]; ok {
+ t.Errorf("node %d (%v) published boundingBox with bounds=false: %v", i, node["ref"], node)
+ }
+ }
+}
+
+// The two tests above are the real proof, but they need a Chrome that can
+// screenshot, and they skip where it cannot. Their claim only transfers to the
+// wire while the envelope embeds the snapshot nodes verbatim: a hand-built node
+// map could drop visible again with every observe-level test still green.
+func TestCaptureEnvelopeEmbedsSnapshotNodesVerbatim(t *testing.T) {
+ src, err := os.ReadFile("capture.go")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(src), `"nodes": result.Nodes`) {
+ t.Fatal("the capture envelope no longer embeds result.Nodes directly; re-mapping nodes into a literal drops whatever field the map forgets")
+ }
+}