Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

feat: add click modes for occluded targets

Boschf411f4f2b5ae
diff --git a/docs/reference/click.md b/docs/reference/click.md
index 31fa0132..d5c25b60 100644
--- a/docs/reference/click.md
+++ b/docs/reference/click.md
@@ -25,6 +25,7 @@ OK
| `--dialog-text` | Prompt response text (with `--dialog-action accept`) |
| `--x`, `--y` | Click at specific coordinates |
| `--humanize` | Use humanized bezier+jitter input path (overrides instance config) |
+| `--mode dom\|dispatch` | Override the default click behavior. Omit `--mode` for the normal click path, use `dom` for `element.click()`, or `dispatch` for synthetic click events on the target |
| `--json` | Full JSON response |
| `--tab` | Target specific tab |
@@ -37,7 +38,9 @@ pinchtab click "text:Submit" # Click by text
pinchtab click e5 --snap # Click and show new snapshot
pinchtab click e5 --wait-nav # Click and wait for navigation
pinchtab click e5 --dialog-action accept # Auto-accept alert/confirm
-pinchtab click --x 100 --y 200 # Click at coordinates
+pinchtab click e5 --mode dom # Activate target directly despite occlusion
+pinchtab click e5 --mode dispatch # Dispatch click events on target despite occlusion
+pinchtab click --x 100 --y 200 # Click at coordinates
```
## Notes
@@ -48,7 +51,10 @@ pinchtab click --x 100 --y 200 # Click at coordinates
- Use [`/frame`](./frame.md) before selector-based iframe actions
- Missing selectors fail immediately; use `pinchtab wait` first for dynamic UI (see [`commands.md`](../commands.md))
- The API also accepts `selector` field: `{"kind":"click","selector":"#login"}`
-- Raw input is the default. To opt a click into the slower humanized path for a page that needs it, pass `humanize:true` in the action JSON or set `instanceDefaults.humanize:true`.
+- Click behavior works like this: omit `mode` for the normal click path, use `mode:"dom"` for `element.click()`, or `mode:"dispatch"` for synthetic click events.
+- `mode:"dom"` and `mode:"dispatch"` are mainly useful when the target is occluded by an overlay or popup.
+- `mode` and `humanize` are separate knobs: `mode` changes how the click is delivered to the target, while `humanize:true` keeps the normal pointer-click path but makes it slower and more human-like.
+- To opt a click into the slower humanized path for a page that needs it, pass `humanize:true` in the action JSON or set `instanceDefaults.humanize:true`.
## Related Pages
diff --git a/internal/bridge/action_pointer.go b/internal/bridge/action_pointer.go
index f715de2c..3caaee23 100644
--- a/internal/bridge/action_pointer.go
+++ b/internal/bridge/action_pointer.go
@@ -18,6 +18,7 @@ var mouseDownByCoordinateAction = MouseDownByCoordinate
var mouseUpByCoordinateAction = MouseUpByCoordinate
var clickByNodeIDAction = ClickByNodeID
var jsClickByBackendNodeAction = JSClickByBackendNode
+var dispatchClickByBackendNodeAction = JSDispatchClickByBackendNode
var doubleClickByNodeIDAction = DoubleClickByNodeID
var jsDoubleClickByBackendNodeAction = JSDoubleClickByBackendNode
@@ -49,6 +50,19 @@ func clickByNodeIDWithJSFallback(ctx context.Context, nodeID int64) error {
return err
}
+func clickByNodeIDWithMode(ctx context.Context, nodeID int64, mode string) error {
+ switch strings.ToLower(strings.TrimSpace(mode)) {
+ case "", "default":
+ return clickByNodeIDWithJSFallback(ctx, nodeID)
+ case "dom":
+ return jsClickByBackendNodeAction(ctx, nodeID)
+ case "dispatch":
+ return dispatchClickByBackendNodeAction(ctx, nodeID)
+ default:
+ return fmt.Errorf("invalid click mode: %s", mode)
+ }
+}
+
func doubleClickByNodeIDWithJSFallback(ctx context.Context, nodeID int64) error {
trustedCtx, cancel := context.WithTimeout(ctx, trustedNodeClickTimeout)
err := doubleClickByNodeIDAction(trustedCtx, nodeID)
@@ -233,12 +247,12 @@ func (b *Bridge) actionClick(ctx context.Context, req ActionRequest) (result map
if auto != nil {
auto.prepareNode(clickCtx, int64(node.BackendNodeID))
}
- err = clickByNodeIDWithJSFallback(clickCtx, int64(node.BackendNodeID))
+ err = clickByNodeIDWithMode(clickCtx, int64(node.BackendNodeID), req.Mode)
} else if req.NodeID > 0 {
if auto != nil {
auto.prepareNode(clickCtx, req.NodeID)
}
- err = clickByNodeIDWithJSFallback(clickCtx, req.NodeID)
+ err = clickByNodeIDWithMode(clickCtx, req.NodeID, req.Mode)
} else if req.HasXY {
err = ClickByCoordinate(clickCtx, req.X, req.Y)
} else {
diff --git a/internal/bridge/action_request.go b/internal/bridge/action_request.go
index 5c5e7739..bf5c11fa 100644
--- a/internal/bridge/action_request.go
+++ b/internal/bridge/action_request.go
@@ -51,6 +51,7 @@ type ActionRequest struct {
Y float64 `json:"y,omitempty"`
HasXY bool `json:"hasXY,omitempty"`
Button string `json:"button,omitempty"`
+ Mode string `json:"mode,omitempty"`
ScrollX int `json:"scrollX"`
ScrollY int `json:"scrollY"`
diff --git a/internal/bridge/cdpops/pointer.go b/internal/bridge/cdpops/pointer.go
index c39a0533..b5270b01 100644
--- a/internal/bridge/cdpops/pointer.go
+++ b/internal/bridge/cdpops/pointer.go
@@ -459,6 +459,26 @@ const jsClickFn = `function() {
}
}`
+const jsDispatchClickFn = `function() {
+ var el = this;
+ var r = el.getBoundingClientRect();
+ var cx = r.left + r.width / 2;
+ var cy = r.top + r.height / 2;
+ var init = {
+ clientX: cx, clientY: cy, screenX: cx, screenY: cy,
+ button: 0, buttons: 1,
+ bubbles: true, cancelable: true, view: window
+ };
+ if (typeof el.focus === 'function') {
+ try { el.focus({preventScroll: true}); } catch (e) {}
+ }
+ el.dispatchEvent(new PointerEvent('pointerdown', Object.assign({}, init, {pointerId: 1, pointerType: 'mouse', isPrimary: true})));
+ el.dispatchEvent(new MouseEvent('mousedown', init));
+ el.dispatchEvent(new PointerEvent('pointerup', Object.assign({}, init, {pointerId: 1, pointerType: 'mouse', isPrimary: true, buttons: 0})));
+ el.dispatchEvent(new MouseEvent('mouseup', Object.assign({}, init, {buttons: 0})));
+ el.dispatchEvent(new MouseEvent('click', Object.assign({}, init, {buttons: 0, detail: 1})));
+}`
+
const jsDoubleClickFn = `function() {
var el = this;
var r = el.getBoundingClientRect();
@@ -509,16 +529,29 @@ func resolveBackendNodeObjectID(ctx context.Context, backendNodeID int64) (strin
// the JS path runs the same handler chain (mousedown, mouseup, click) plus
// the browser's default action (el.click()) without the ack tax.
func JSClickByBackendNode(ctx context.Context, backendNodeID int64) error {
- if _, _, err := PointerPointForNode(ctx, backendNodeID, true); err != nil {
+ objectID, err := resolveBackendNodeObjectID(ctx, backendNodeID)
+ if err != nil {
return err
}
+ return chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
+ return chromedp.FromContext(ctx).Target.Execute(ctx, "Runtime.callFunctionOn", map[string]any{
+ "functionDeclaration": jsClickFn,
+ "objectId": objectID,
+ }, nil)
+ }))
+}
+
+// JSDispatchClickByBackendNode dispatches synthetic pointer/mouse events on the
+// target element without invoking element.click(). This bypasses occlusion while
+// staying closer to the browser event sequence than a DOM click.
+func JSDispatchClickByBackendNode(ctx context.Context, backendNodeID int64) error {
objectID, err := resolveBackendNodeObjectID(ctx, backendNodeID)
if err != nil {
return err
}
return chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
return chromedp.FromContext(ctx).Target.Execute(ctx, "Runtime.callFunctionOn", map[string]any{
- "functionDeclaration": jsClickFn,
+ "functionDeclaration": jsDispatchClickFn,
"objectId": objectID,
}, nil)
}))
diff --git a/internal/bridge/cdpops_facade.go b/internal/bridge/cdpops_facade.go
index 7491bf62..611a6130 100644
--- a/internal/bridge/cdpops_facade.go
+++ b/internal/bridge/cdpops_facade.go
@@ -63,6 +63,10 @@ func JSClickByBackendNode(ctx context.Context, nodeID int64) error {
return bridgecdpops.JSClickByBackendNode(ctx, nodeID)
}
+func JSDispatchClickByBackendNode(ctx context.Context, nodeID int64) error {
+ return bridgecdpops.JSDispatchClickByBackendNode(ctx, nodeID)
+}
+
func DoubleClickByCoordinate(ctx context.Context, x, y float64) error {
return bridgecdpops.DoubleClickByCoordinate(ctx, x, y)
}
diff --git a/internal/cli/actions/actions_element.go b/internal/cli/actions/actions_element.go
index 01c658b6..4a2d99be 100644
--- a/internal/cli/actions/actions_element.go
+++ b/internal/cli/actions/actions_element.go
@@ -46,6 +46,9 @@ func Action(client *http.Client, base, token, kind, selectorArg string, cmd *cob
if waitNav {
body["waitNav"] = true
}
+ if v, _ := cmd.Flags().GetString("mode"); v != "" {
+ body["mode"] = v
+ }
// --dismiss-banners only fires when --wait-nav is set; without nav,
// banners haven't changed and the dismissal pass would be wasted.
if waitNav {
diff --git a/internal/cli/actions/actions_element_test.go b/internal/cli/actions/actions_element_test.go
index 485d05d5..90332d32 100644
--- a/internal/cli/actions/actions_element_test.go
+++ b/internal/cli/actions/actions_element_test.go
@@ -20,6 +20,7 @@ func newActionCmd() *cobra.Command {
cmd.Flags().Int("dy", 0, "")
cmd.Flags().String("dialog-action", "", "")
cmd.Flags().String("dialog-text", "", "")
+ cmd.Flags().String("mode", "", "")
return cmd
}
@@ -101,6 +102,21 @@ func TestClickDismissBannersWithoutWaitNavIsNoop(t *testing.T) {
}
}
+func TestClickMode(t *testing.T) {
+ m := newMockServer()
+ defer m.close()
+ client := m.server.Client()
+
+ cmd := newActionCmd()
+ _ = cmd.Flags().Set("mode", "dom")
+ Action(client, m.base(), "", "click", "e5", cmd)
+ var body map[string]any
+ _ = json.Unmarshal([]byte(m.lastBody), &body)
+ if body["mode"] != "dom" {
+ t.Errorf("expected mode=dom, got %v", body["mode"])
+ }
+}
+
func TestClickDialogAction(t *testing.T) {
m := newMockServer()
defer m.close()
diff --git a/skills/pinchtab/SKILL.md b/skills/pinchtab/SKILL.md
index da27ac83..02d310f6 100644
--- a/skills/pinchtab/SKILL.md
+++ b/skills/pinchtab/SKILL.md
@@ -30,6 +30,9 @@ CLI-first browser skill. Use `pinchtab` commands.
1. Create a session: `export PINCHTAB_SESSION=$(pinchtab session create --agent-id myagent)` — do this once before any browser command.
2. Navigate: `pinchtab nav <url> --snap` — auto-starts the local server if needed, then returns tab ID + interactive snapshot in one call.
3. Interact: `pinchtab click <ref> --snap-diff` — returns OK + only changed elements (most token-efficient).
+ - Click behavior: omit `--mode` for the normal click path, use `--mode dom`, or use `--mode dispatch`.
+ - Occlusion workaround: `pinchtab click <ref> --mode dom` or `pinchtab click <ref> --mode dispatch`
+ - `--humanize` is separate: it keeps the normal pointer-click path, but makes it slower and more human-like.
4. For read-only observation: `pinchtab text` when you won't act on refs.
**Key optimization**: Use `--snap-diff` on `click`, `fill`, `select`, `back`, `forward`, `reload` to get only added/changed/removed elements — most token-efficient for multi-step flows. Use `--snap` when you need the full snapshot (e.g., first navigation, or after major page changes). Use `--text` when you need prose content for verification (skips snap, returns page text directly).
@@ -174,7 +177,7 @@ Guidance:
All interaction commands accept unified selectors (see Selectors above).
```bash
-pinchtab click <selector> # flags: --snap, --snap-diff, --text, --wait-nav, --dismiss-banners (with --wait-nav), --x/--y (coords), --dialog-action accept|dismiss [--dialog-text "..."]
+pinchtab click <selector> # flags: --snap, --snap-diff, --text, --wait-nav, --dismiss-banners (with --wait-nav), --x/--y (coords), --mode dom|dispatch, --humanize, --dialog-action accept|dismiss [--dialog-text "..."]
pinchtab dblclick <selector>
pinchtab mouse move|down|up <selector|x y> # --button left|middle|right
pinchtab mouse wheel <ms> --dx <n> --dy <n>
@@ -192,6 +195,9 @@ Rules:
- Default output is `OK`; use `--json` for recovery metadata. Errors go to stderr as `ERROR: <cmd>: <reason>`.
- **Prefer `--snap-diff`** with `click`, `fill`, `select`, `back`, `forward`, `reload` — returns `OK` + only changed elements. Use `--snap` when you need the full snapshot (first nav, major page change).
- Prefer `fill` for form entry; `type` only when the site depends on keystroke events.
+- Click behavior: omit `--mode` for the normal click path, use `click --mode dom` for `element.click()`, or `click --mode dispatch` for synthetic click events.
+- `click --mode dom` and `click --mode dispatch` are mainly for bypassing occlusion.
+- `click --humanize` is different from `--mode`: it still uses pointer input, just with slower human-like motion/timing.
- `click --wait-nav` when a click navigates. May return `{"success":true}` or `Error 409: unexpected page navigation` — treat 409 as success and verify with fresh `snap`/`text`.
- `--dismiss-banners` on `nav`/`back`/`forward`/`reload` (and on `click --wait-nav`) runs a best-effort pass that clicks a visible Accept all / Got it / OK / Close / Dismiss button, or removes obvious cookie/consent/dialog/overlay containers. Use when a fresh page-load shows a modal that blocks interaction (typical symptom: `Error 500: action click: element is occluded`). Heuristic — can misfire on pages that label legitimate UI as `overlay` or `modal`; not a substitute for an explicit selector when one is known.
- Use low-level `mouse` only for drag handles, canvas widgets, or exact pointer sequences.
diff --git a/skills/pinchtab/references/api.md b/skills/pinchtab/references/api.md
index 665b6815..2cff0844 100644
--- a/skills/pinchtab/references/api.md
+++ b/skills/pinchtab/references/api.md
@@ -86,10 +86,16 @@ Returns flat JSON array of nodes with `ref`, `role`, `name`, `depth`, `value`, `
```bash
# CLI: pinchtab click e5 / pinchtab type e12 hello / pinchtab press Enter
-# Click by ref
+# Click by ref (normal click path; omit mode)
curl -X POST /action -H 'Content-Type: application/json' \
-d '{"kind": "click", "ref": "e5"}'
+# Bypass occlusion on the target element
+curl -X POST /action -H 'Content-Type: application/json' \
+ -d '{"kind": "click", "ref": "e5", "mode": "dom"}'
+curl -X POST /action -H 'Content-Type: application/json' \
+ -d '{"kind": "click", "ref": "e5", "mode": "dispatch"}'
+
# Type into focused element (click first, then type)
curl -X POST /action -H 'Content-Type: application/json' \
-d '{"kind": "click", "ref": "e12"}'
@@ -147,6 +153,9 @@ curl -X POST /action -H 'Content-Type: application/json' \
Notes:
+- click behavior works like this: omit `mode` for the normal click path, use `mode:"dom"` for `element.click()`, or `mode:"dispatch"` for synthetic click events
+- `mode:"dom"` and `mode:"dispatch"` are mainly for bypassing occlusion
+- `mode` is separate from `humanize:true`, which keeps the normal pointer path but makes it slower and more human-like
- selector-based click and double-click paths resolve through backend node IDs before dispatching pointer events
- low-level pointer actions accept `ref`, `selector`, `nodeId`, or `x`/`y`
- `mouse-down` and `mouse-up` accept `button` with `left`, `right`, or `middle`
diff --git a/skills/pinchtab/references/commands.md b/skills/pinchtab/references/commands.md
index 3d8ee4c5..c1f769f1 100644
--- a/skills/pinchtab/references/commands.md
+++ b/skills/pinchtab/references/commands.md
@@ -83,7 +83,9 @@ Unscoped commands resolve the current tab by caller identity. Session-authentica
Click an element by its accessibility ref (from `snap`).
```bash
-pinchtab click e5
+pinchtab click e5 # normal click path (omit --mode)
+pinchtab click e5 --mode dom # bypass occlusion with element.click()
+pinchtab click e5 --mode dispatch # bypass occlusion with synthetic events
pinchtab click e5 --snap-diff # click + return only changed elements
pinchtab click e5 --snap # click + return full snapshot
pinchtab click e5 --tab <tabId>
diff --git a/tests/e2e/fixtures/occluded-click.html b/tests/e2e/fixtures/occluded-click.html
new file mode 100644
index 00000000..8ffc4f3e
--- /dev/null
+++ b/tests/e2e/fixtures/occluded-click.html
@@ -0,0 +1,93 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <title>Occluded Click Fixture</title>
+ <style>
+ body {
+ font-family: system-ui, sans-serif;
+ margin: 0;
+ padding: 32px;
+ min-height: 100vh;
+ background: #f5f7fb;
+ }
+ #target-wrap {
+ position: relative;
+ width: 360px;
+ height: 220px;
+ margin-top: 48px;
+ }
+ #target {
+ position: absolute;
+ left: 40px;
+ top: 40px;
+ width: 240px;
+ height: 80px;
+ border: none;
+ border-radius: 12px;
+ background: #2563eb;
+ color: white;
+ font-size: 20px;
+ font-weight: 600;
+ cursor: pointer;
+ z-index: 1;
+ }
+ #occluder {
+ position: absolute;
+ left: 40px;
+ top: 40px;
+ width: 240px;
+ height: 80px;
+ border-radius: 12px;
+ background: rgba(255, 255, 255, 0.01);
+ z-index: 2;
+ }
+ #status {
+ margin-top: 24px;
+ font-size: 18px;
+ font-weight: 600;
+ }
+ #coords {
+ margin-top: 8px;
+ color: #334155;
+ font-family: ui-monospace, monospace;
+ white-space: pre-wrap;
+ }
+ </style>
+</head>
+<body>
+ <h1>Occluded Click Fixture</h1>
+ <p>The button is visible but covered by a transparent overlay. Ref-based click should report occlusion; coordinate fallback should still work.</p>
+
+ <div id="target-wrap">
+ <button id="target" type="button">Proceed</button>
+ <div id="occluder" aria-hidden="true"></div>
+ </div>
+
+ <div id="status">not-clicked</div>
+ <pre id="coords"></pre>
+
+ <script>
+ window.occludedClickState = { clicked: false, clicks: 0, lastClientX: null, lastClientY: null };
+ const target = document.getElementById('target');
+ const status = document.getElementById('status');
+ const coords = document.getElementById('coords');
+
+ function render() {
+ status.textContent = window.occludedClickState.clicked ? 'clicked' : 'not-clicked';
+ coords.textContent = JSON.stringify(window.occludedClickState, null, 2);
+ }
+
+ target.addEventListener('click', (event) => {
+ window.occludedClickState.clicked = true;
+ window.occludedClickState.clicks += 1;
+ window.occludedClickState.lastClientX = Math.round(event.clientX || 0);
+ window.occludedClickState.lastClientY = Math.round(event.clientY || 0);
+ render();
+ });
+
+ render();
+ </script>
+</body>
+</html>
diff --git a/tests/e2e/scenarios/api/actions-extended.sh b/tests/e2e/scenarios/api/actions-extended.sh
index e360c3b8..c1f6ada8 100644
--- a/tests/e2e/scenarios/api/actions-extended.sh
+++ b/tests/e2e/scenarios/api/actions-extended.sh
@@ -116,6 +116,49 @@ assert_json_eq "$RESULT" '.result' '240' "wheel delta Y accumulated"
end_test
+# ─────────────────────────────────────────────────────────────────
+start_test "pinchtab occluded click supports dom and dispatch modes"
+
+pt_post /navigate "{\"url\":\"${FIXTURES_URL}/occluded-click.html\"}"
+assert_ok "navigate"
+
+pt_get "/snapshot?filter=interactive"
+assert_ok "snapshot"
+require_ref "button" "Proceed" TARGET_REF && {
+ pt_post /action "{\"kind\":\"click\",\"ref\":\"${TARGET_REF}\"}"
+ assert_http_status "500" "occluded ref click returns action failure"
+ assert_json_contains "$RESULT" '.error' 'element is occluded' "occluded error surfaced"
+
+ pt_get "/box?ref=${TARGET_REF}"
+ assert_ok "box lookup by ref"
+ assert_result_jq '.box.width > 0 and .box.height > 0' "box returns positive dimensions" "box dimensions missing"
+
+ pt_post /action "{\"kind\":\"mouse-move\",\"ref\":\"${TARGET_REF}\"}"
+ assert_ok "mouse-move by ref yields coordinates"
+ assert_result_jq '.result.x > 0 and .result.y > 0' "mouse-move exposes positive coordinates" "mouse-move coordinates missing"
+
+ pt_post /action "{\"kind\":\"click\",\"ref\":\"${TARGET_REF}\",\"mode\":\"dom\"}"
+ assert_ok "dom mode click succeeds despite occlusion"
+
+ pt_post /evaluate '{"expression":"window.occludedClickState"}'
+ assert_ok "evaluate click state after dom mode"
+ assert_json_eq "$RESULT" '.result.clicked' 'true' "dom mode triggered click"
+ assert_json_eq "$RESULT" '.result.clicks' '1' "dom mode fired one click"
+
+ pt_post /evaluate '{"expression":"window.occludedClickState = { clicked: false, clicks: 0, lastClientX: null, lastClientY: null }; window.occludedClickState"}'
+ assert_ok "reset click state"
+
+ pt_post /action "{\"kind\":\"click\",\"ref\":\"${TARGET_REF}\",\"mode\":\"dispatch\"}"
+ assert_ok "dispatch mode click succeeds despite occlusion"
+
+ pt_post /evaluate '{"expression":"window.occludedClickState"}'
+ assert_ok "evaluate click state after dispatch mode"
+ assert_json_eq "$RESULT" '.result.clicked' 'true' "dispatch mode triggered click"
+ assert_json_eq "$RESULT" '.result.clicks' '1' "dispatch mode fired one click"
+}
+
+end_test
+
# ─────────────────────────────────────────────────────────────────
start_test "pinchtab mouse current-pointer sequence"