High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
feat: add click modes for occluded targets
Boschf411f4f2b5ae
diff --git a/docs/reference/click.md b/docs/reference/click.mdindex 31fa0132..d5c25b60 100644--- a/docs/reference/click.md+++ b/docs/reference/click.md@@ -25,6 +25,7 @@ OK| `--dialog-text` | Prompt response text (with `--dialog-action accept`) || `--x`, `--y` | Click at specific coordinates || `--humanize` | Use humanized bezier+jitter input path (overrides instance config) |+| `--mode dom\|dispatch` | Override the default click behavior. Omit `--mode` for the normal click path, use `dom` for `element.click()`, or `dispatch` for synthetic click events on the target || `--json` | Full JSON response || `--tab` | Target specific tab |@@ -37,7 +38,9 @@ pinchtab click "text:Submit" # Click by textpinchtab click e5 --snap # Click and show new snapshotpinchtab click e5 --wait-nav # Click and wait for navigationpinchtab click e5 --dialog-action accept # Auto-accept alert/confirm-pinchtab click --x 100 --y 200 # Click at coordinates+pinchtab click e5 --mode dom # Activate target directly despite occlusion+pinchtab click e5 --mode dispatch # Dispatch click events on target despite occlusion+pinchtab click --x 100 --y 200 # Click at coordinates```## Notes@@ -48,7 +51,10 @@ pinchtab click --x 100 --y 200 # Click at coordinates- Use [`/frame`](./frame.md) before selector-based iframe actions- Missing selectors fail immediately; use `pinchtab wait` first for dynamic UI (see [`commands.md`](../commands.md))- The API also accepts `selector` field: `{"kind":"click","selector":"#login"}`-- Raw input is the default. To opt a click into the slower humanized path for a page that needs it, pass `humanize:true` in the action JSON or set `instanceDefaults.humanize:true`.+- Click behavior works like this: omit `mode` for the normal click path, use `mode:"dom"` for `element.click()`, or `mode:"dispatch"` for synthetic click events.+- `mode:"dom"` and `mode:"dispatch"` are mainly useful when the target is occluded by an overlay or popup.+- `mode` and `humanize` are separate knobs: `mode` changes how the click is delivered to the target, while `humanize:true` keeps the normal pointer-click path but makes it slower and more human-like.+- To opt a click into the slower humanized path for a page that needs it, pass `humanize:true` in the action JSON or set `instanceDefaults.humanize:true`.## Related Pagesdiff --git a/internal/bridge/action_pointer.go b/internal/bridge/action_pointer.goindex f715de2c..3caaee23 100644--- a/internal/bridge/action_pointer.go+++ b/internal/bridge/action_pointer.go@@ -18,6 +18,7 @@ var mouseDownByCoordinateAction = MouseDownByCoordinatevar mouseUpByCoordinateAction = MouseUpByCoordinatevar clickByNodeIDAction = ClickByNodeIDvar jsClickByBackendNodeAction = JSClickByBackendNode+var dispatchClickByBackendNodeAction = JSDispatchClickByBackendNodevar doubleClickByNodeIDAction = DoubleClickByNodeIDvar jsDoubleClickByBackendNodeAction = JSDoubleClickByBackendNode@@ -49,6 +50,19 @@ func clickByNodeIDWithJSFallback(ctx context.Context, nodeID int64) error {return err}+func clickByNodeIDWithMode(ctx context.Context, nodeID int64, mode string) error {+ switch strings.ToLower(strings.TrimSpace(mode)) {+ case "", "default":+ return clickByNodeIDWithJSFallback(ctx, nodeID)+ case "dom":+ return jsClickByBackendNodeAction(ctx, nodeID)+ case "dispatch":+ return dispatchClickByBackendNodeAction(ctx, nodeID)+ default:+ return fmt.Errorf("invalid click mode: %s", mode)+ }+}+func doubleClickByNodeIDWithJSFallback(ctx context.Context, nodeID int64) error {trustedCtx, cancel := context.WithTimeout(ctx, trustedNodeClickTimeout)err := doubleClickByNodeIDAction(trustedCtx, nodeID)@@ -233,12 +247,12 @@ func (b *Bridge) actionClick(ctx context.Context, req ActionRequest) (result mapif auto != nil {auto.prepareNode(clickCtx, int64(node.BackendNodeID))}- err = clickByNodeIDWithJSFallback(clickCtx, int64(node.BackendNodeID))+ err = clickByNodeIDWithMode(clickCtx, int64(node.BackendNodeID), req.Mode)} else if req.NodeID > 0 {if auto != nil {auto.prepareNode(clickCtx, req.NodeID)}- err = clickByNodeIDWithJSFallback(clickCtx, req.NodeID)+ err = clickByNodeIDWithMode(clickCtx, req.NodeID, req.Mode)} else if req.HasXY {err = ClickByCoordinate(clickCtx, req.X, req.Y)} else {diff --git a/internal/bridge/action_request.go b/internal/bridge/action_request.goindex 5c5e7739..bf5c11fa 100644--- a/internal/bridge/action_request.go+++ b/internal/bridge/action_request.go@@ -51,6 +51,7 @@ type ActionRequest struct {Y float64 `json:"y,omitempty"`HasXY bool `json:"hasXY,omitempty"`Button string `json:"button,omitempty"`+ Mode string `json:"mode,omitempty"`ScrollX int `json:"scrollX"`ScrollY int `json:"scrollY"`diff --git a/internal/bridge/cdpops/pointer.go b/internal/bridge/cdpops/pointer.goindex c39a0533..b5270b01 100644--- a/internal/bridge/cdpops/pointer.go+++ b/internal/bridge/cdpops/pointer.go@@ -459,6 +459,26 @@ const jsClickFn = `function() {}}`+const jsDispatchClickFn = `function() {+ var el = this;+ var r = el.getBoundingClientRect();+ var cx = r.left + r.width / 2;+ var cy = r.top + r.height / 2;+ var init = {+ clientX: cx, clientY: cy, screenX: cx, screenY: cy,+ button: 0, buttons: 1,+ bubbles: true, cancelable: true, view: window+ };+ if (typeof el.focus === 'function') {+ try { el.focus({preventScroll: true}); } catch (e) {}+ }+ el.dispatchEvent(new PointerEvent('pointerdown', Object.assign({}, init, {pointerId: 1, pointerType: 'mouse', isPrimary: true})));+ el.dispatchEvent(new MouseEvent('mousedown', init));+ el.dispatchEvent(new PointerEvent('pointerup', Object.assign({}, init, {pointerId: 1, pointerType: 'mouse', isPrimary: true, buttons: 0})));+ el.dispatchEvent(new MouseEvent('mouseup', Object.assign({}, init, {buttons: 0})));+ el.dispatchEvent(new MouseEvent('click', Object.assign({}, init, {buttons: 0, detail: 1})));+}`+const jsDoubleClickFn = `function() {var el = this;var r = el.getBoundingClientRect();@@ -509,16 +529,29 @@ func resolveBackendNodeObjectID(ctx context.Context, backendNodeID int64) (strin// the JS path runs the same handler chain (mousedown, mouseup, click) plus// the browser's default action (el.click()) without the ack tax.func JSClickByBackendNode(ctx context.Context, backendNodeID int64) error {- if _, _, err := PointerPointForNode(ctx, backendNodeID, true); err != nil {+ objectID, err := resolveBackendNodeObjectID(ctx, backendNodeID)+ if err != nil {return err}+ return chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {+ return chromedp.FromContext(ctx).Target.Execute(ctx, "Runtime.callFunctionOn", map[string]any{+ "functionDeclaration": jsClickFn,+ "objectId": objectID,+ }, nil)+ }))+}++// JSDispatchClickByBackendNode dispatches synthetic pointer/mouse events on the+// target element without invoking element.click(). This bypasses occlusion while+// staying closer to the browser event sequence than a DOM click.+func JSDispatchClickByBackendNode(ctx context.Context, backendNodeID int64) error {objectID, err := resolveBackendNodeObjectID(ctx, backendNodeID)if err != nil {return err}return chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {return chromedp.FromContext(ctx).Target.Execute(ctx, "Runtime.callFunctionOn", map[string]any{- "functionDeclaration": jsClickFn,+ "functionDeclaration": jsDispatchClickFn,"objectId": objectID,}, nil)}))diff --git a/internal/bridge/cdpops_facade.go b/internal/bridge/cdpops_facade.goindex 7491bf62..611a6130 100644--- a/internal/bridge/cdpops_facade.go+++ b/internal/bridge/cdpops_facade.go@@ -63,6 +63,10 @@ func JSClickByBackendNode(ctx context.Context, nodeID int64) error {return bridgecdpops.JSClickByBackendNode(ctx, nodeID)}+func JSDispatchClickByBackendNode(ctx context.Context, nodeID int64) error {+ return bridgecdpops.JSDispatchClickByBackendNode(ctx, nodeID)+}+func DoubleClickByCoordinate(ctx context.Context, x, y float64) error {return bridgecdpops.DoubleClickByCoordinate(ctx, x, y)}diff --git a/internal/cli/actions/actions_element.go b/internal/cli/actions/actions_element.goindex 01c658b6..4a2d99be 100644--- a/internal/cli/actions/actions_element.go+++ b/internal/cli/actions/actions_element.go@@ -46,6 +46,9 @@ func Action(client *http.Client, base, token, kind, selectorArg string, cmd *cobif waitNav {body["waitNav"] = true}+ if v, _ := cmd.Flags().GetString("mode"); v != "" {+ body["mode"] = v+ }// --dismiss-banners only fires when --wait-nav is set; without nav,// banners haven't changed and the dismissal pass would be wasted.if waitNav {diff --git a/internal/cli/actions/actions_element_test.go b/internal/cli/actions/actions_element_test.goindex 485d05d5..90332d32 100644--- a/internal/cli/actions/actions_element_test.go+++ b/internal/cli/actions/actions_element_test.go@@ -20,6 +20,7 @@ func newActionCmd() *cobra.Command {cmd.Flags().Int("dy", 0, "")cmd.Flags().String("dialog-action", "", "")cmd.Flags().String("dialog-text", "", "")+ cmd.Flags().String("mode", "", "")return cmd}@@ -101,6 +102,21 @@ func TestClickDismissBannersWithoutWaitNavIsNoop(t *testing.T) {}}+func TestClickMode(t *testing.T) {+ m := newMockServer()+ defer m.close()+ client := m.server.Client()++ cmd := newActionCmd()+ _ = cmd.Flags().Set("mode", "dom")+ Action(client, m.base(), "", "click", "e5", cmd)+ var body map[string]any+ _ = json.Unmarshal([]byte(m.lastBody), &body)+ if body["mode"] != "dom" {+ t.Errorf("expected mode=dom, got %v", body["mode"])+ }+}+func TestClickDialogAction(t *testing.T) {m := newMockServer()defer m.close()diff --git a/skills/pinchtab/SKILL.md b/skills/pinchtab/SKILL.mdindex da27ac83..02d310f6 100644--- a/skills/pinchtab/SKILL.md+++ b/skills/pinchtab/SKILL.md@@ -30,6 +30,9 @@ CLI-first browser skill. Use `pinchtab` commands.1. Create a session: `export PINCHTAB_SESSION=$(pinchtab session create --agent-id myagent)` — do this once before any browser command.2. Navigate: `pinchtab nav <url> --snap` — auto-starts the local server if needed, then returns tab ID + interactive snapshot in one call.3. Interact: `pinchtab click <ref> --snap-diff` — returns OK + only changed elements (most token-efficient).+ - Click behavior: omit `--mode` for the normal click path, use `--mode dom`, or use `--mode dispatch`.+ - Occlusion workaround: `pinchtab click <ref> --mode dom` or `pinchtab click <ref> --mode dispatch`+ - `--humanize` is separate: it keeps the normal pointer-click path, but makes it slower and more human-like.4. For read-only observation: `pinchtab text` when you won't act on refs.**Key optimization**: Use `--snap-diff` on `click`, `fill`, `select`, `back`, `forward`, `reload` to get only added/changed/removed elements — most token-efficient for multi-step flows. Use `--snap` when you need the full snapshot (e.g., first navigation, or after major page changes). Use `--text` when you need prose content for verification (skips snap, returns page text directly).@@ -174,7 +177,7 @@ Guidance:All interaction commands accept unified selectors (see Selectors above).```bash-pinchtab click <selector> # flags: --snap, --snap-diff, --text, --wait-nav, --dismiss-banners (with --wait-nav), --x/--y (coords), --dialog-action accept|dismiss [--dialog-text "..."]+pinchtab click <selector> # flags: --snap, --snap-diff, --text, --wait-nav, --dismiss-banners (with --wait-nav), --x/--y (coords), --mode dom|dispatch, --humanize, --dialog-action accept|dismiss [--dialog-text "..."]pinchtab dblclick <selector>pinchtab mouse move|down|up <selector|x y> # --button left|middle|rightpinchtab mouse wheel <ms> --dx <n> --dy <n>@@ -192,6 +195,9 @@ Rules:- Default output is `OK`; use `--json` for recovery metadata. Errors go to stderr as `ERROR: <cmd>: <reason>`.- **Prefer `--snap-diff`** with `click`, `fill`, `select`, `back`, `forward`, `reload` — returns `OK` + only changed elements. Use `--snap` when you need the full snapshot (first nav, major page change).- Prefer `fill` for form entry; `type` only when the site depends on keystroke events.+- Click behavior: omit `--mode` for the normal click path, use `click --mode dom` for `element.click()`, or `click --mode dispatch` for synthetic click events.+- `click --mode dom` and `click --mode dispatch` are mainly for bypassing occlusion.+- `click --humanize` is different from `--mode`: it still uses pointer input, just with slower human-like motion/timing.- `click --wait-nav` when a click navigates. May return `{"success":true}` or `Error 409: unexpected page navigation` — treat 409 as success and verify with fresh `snap`/`text`.- `--dismiss-banners` on `nav`/`back`/`forward`/`reload` (and on `click --wait-nav`) runs a best-effort pass that clicks a visible Accept all / Got it / OK / Close / Dismiss button, or removes obvious cookie/consent/dialog/overlay containers. Use when a fresh page-load shows a modal that blocks interaction (typical symptom: `Error 500: action click: element is occluded`). Heuristic — can misfire on pages that label legitimate UI as `overlay` or `modal`; not a substitute for an explicit selector when one is known.- Use low-level `mouse` only for drag handles, canvas widgets, or exact pointer sequences.diff --git a/skills/pinchtab/references/api.md b/skills/pinchtab/references/api.mdindex 665b6815..2cff0844 100644--- a/skills/pinchtab/references/api.md+++ b/skills/pinchtab/references/api.md@@ -86,10 +86,16 @@ Returns flat JSON array of nodes with `ref`, `role`, `name`, `depth`, `value`, ````bash# CLI: pinchtab click e5 / pinchtab type e12 hello / pinchtab press Enter-# Click by ref+# Click by ref (normal click path; omit mode)curl -X POST /action -H 'Content-Type: application/json' \-d '{"kind": "click", "ref": "e5"}'+# Bypass occlusion on the target element+curl -X POST /action -H 'Content-Type: application/json' \+ -d '{"kind": "click", "ref": "e5", "mode": "dom"}'+curl -X POST /action -H 'Content-Type: application/json' \+ -d '{"kind": "click", "ref": "e5", "mode": "dispatch"}'+# Type into focused element (click first, then type)curl -X POST /action -H 'Content-Type: application/json' \-d '{"kind": "click", "ref": "e12"}'@@ -147,6 +153,9 @@ curl -X POST /action -H 'Content-Type: application/json' \Notes:+- click behavior works like this: omit `mode` for the normal click path, use `mode:"dom"` for `element.click()`, or `mode:"dispatch"` for synthetic click events+- `mode:"dom"` and `mode:"dispatch"` are mainly for bypassing occlusion+- `mode` is separate from `humanize:true`, which keeps the normal pointer path but makes it slower and more human-like- selector-based click and double-click paths resolve through backend node IDs before dispatching pointer events- low-level pointer actions accept `ref`, `selector`, `nodeId`, or `x`/`y`- `mouse-down` and `mouse-up` accept `button` with `left`, `right`, or `middle`diff --git a/skills/pinchtab/references/commands.md b/skills/pinchtab/references/commands.mdindex 3d8ee4c5..c1f769f1 100644--- a/skills/pinchtab/references/commands.md+++ b/skills/pinchtab/references/commands.md@@ -83,7 +83,9 @@ Unscoped commands resolve the current tab by caller identity. Session-authenticaClick an element by its accessibility ref (from `snap`).```bash-pinchtab click e5+pinchtab click e5 # normal click path (omit --mode)+pinchtab click e5 --mode dom # bypass occlusion with element.click()+pinchtab click e5 --mode dispatch # bypass occlusion with synthetic eventspinchtab click e5 --snap-diff # click + return only changed elementspinchtab click e5 --snap # click + return full snapshotpinchtab click e5 --tab <tabId>diff --git a/tests/e2e/fixtures/occluded-click.html b/tests/e2e/fixtures/occluded-click.htmlnew file mode 100644index 00000000..8ffc4f3e--- /dev/null+++ b/tests/e2e/fixtures/occluded-click.html@@ -0,0 +1,93 @@+<!DOCTYPE html>+<html lang="en">+<head>+ <meta charset="UTF-8">+ <meta name="viewport" content="width=device-width, initial-scale=1.0">+ <title>Occluded Click Fixture</title>+ <style>+ body {+ font-family: system-ui, sans-serif;+ margin: 0;+ padding: 32px;+ min-height: 100vh;+ background: #f5f7fb;+ }+ #target-wrap {+ position: relative;+ width: 360px;+ height: 220px;+ margin-top: 48px;+ }+ #target {+ position: absolute;+ left: 40px;+ top: 40px;+ width: 240px;+ height: 80px;+ border: none;+ border-radius: 12px;+ background: #2563eb;+ color: white;+ font-size: 20px;+ font-weight: 600;+ cursor: pointer;+ z-index: 1;+ }+ #occluder {+ position: absolute;+ left: 40px;+ top: 40px;+ width: 240px;+ height: 80px;+ border-radius: 12px;+ background: rgba(255, 255, 255, 0.01);+ z-index: 2;+ }+ #status {+ margin-top: 24px;+ font-size: 18px;+ font-weight: 600;+ }+ #coords {+ margin-top: 8px;+ color: #334155;+ font-family: ui-monospace, monospace;+ white-space: pre-wrap;+ }+ </style>+</head>+<body>+ <h1>Occluded Click Fixture</h1>+ <p>The button is visible but covered by a transparent overlay. Ref-based click should report occlusion; coordinate fallback should still work.</p>++ <div id="target-wrap">+ <button id="target" type="button">Proceed</button>+ <div id="occluder" aria-hidden="true"></div>+ </div>++ <div id="status">not-clicked</div>+ <pre id="coords"></pre>++ <script>+ window.occludedClickState = { clicked: false, clicks: 0, lastClientX: null, lastClientY: null };+ const target = document.getElementById('target');+ const status = document.getElementById('status');+ const coords = document.getElementById('coords');++ function render() {+ status.textContent = window.occludedClickState.clicked ? 'clicked' : 'not-clicked';+ coords.textContent = JSON.stringify(window.occludedClickState, null, 2);+ }++ target.addEventListener('click', (event) => {+ window.occludedClickState.clicked = true;+ window.occludedClickState.clicks += 1;+ window.occludedClickState.lastClientX = Math.round(event.clientX || 0);+ window.occludedClickState.lastClientY = Math.round(event.clientY || 0);+ render();+ });++ render();+ </script>+</body>+</html>diff --git a/tests/e2e/scenarios/api/actions-extended.sh b/tests/e2e/scenarios/api/actions-extended.shindex e360c3b8..c1f6ada8 100644--- a/tests/e2e/scenarios/api/actions-extended.sh+++ b/tests/e2e/scenarios/api/actions-extended.sh@@ -116,6 +116,49 @@ assert_json_eq "$RESULT" '.result' '240' "wheel delta Y accumulated"end_test+# ─────────────────────────────────────────────────────────────────+start_test "pinchtab occluded click supports dom and dispatch modes"++pt_post /navigate "{\"url\":\"${FIXTURES_URL}/occluded-click.html\"}"+assert_ok "navigate"++pt_get "/snapshot?filter=interactive"+assert_ok "snapshot"+require_ref "button" "Proceed" TARGET_REF && {+ pt_post /action "{\"kind\":\"click\",\"ref\":\"${TARGET_REF}\"}"+ assert_http_status "500" "occluded ref click returns action failure"+ assert_json_contains "$RESULT" '.error' 'element is occluded' "occluded error surfaced"++ pt_get "/box?ref=${TARGET_REF}"+ assert_ok "box lookup by ref"+ assert_result_jq '.box.width > 0 and .box.height > 0' "box returns positive dimensions" "box dimensions missing"++ pt_post /action "{\"kind\":\"mouse-move\",\"ref\":\"${TARGET_REF}\"}"+ assert_ok "mouse-move by ref yields coordinates"+ assert_result_jq '.result.x > 0 and .result.y > 0' "mouse-move exposes positive coordinates" "mouse-move coordinates missing"++ pt_post /action "{\"kind\":\"click\",\"ref\":\"${TARGET_REF}\",\"mode\":\"dom\"}"+ assert_ok "dom mode click succeeds despite occlusion"++ pt_post /evaluate '{"expression":"window.occludedClickState"}'+ assert_ok "evaluate click state after dom mode"+ assert_json_eq "$RESULT" '.result.clicked' 'true' "dom mode triggered click"+ assert_json_eq "$RESULT" '.result.clicks' '1' "dom mode fired one click"++ pt_post /evaluate '{"expression":"window.occludedClickState = { clicked: false, clicks: 0, lastClientX: null, lastClientY: null }; window.occludedClickState"}'+ assert_ok "reset click state"++ pt_post /action "{\"kind\":\"click\",\"ref\":\"${TARGET_REF}\",\"mode\":\"dispatch\"}"+ assert_ok "dispatch mode click succeeds despite occlusion"++ pt_post /evaluate '{"expression":"window.occludedClickState"}'+ assert_ok "evaluate click state after dispatch mode"+ assert_json_eq "$RESULT" '.result.clicked' 'true' "dispatch mode triggered click"+ assert_json_eq "$RESULT" '.result.clicks' '1' "dispatch mode fired one click"+}++end_test+# ─────────────────────────────────────────────────────────────────start_test "pinchtab mouse current-pointer sequence"