Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix: scale screencast clicks into the css viewport

Bosch5abb8db35f21
diff --git a/dashboard/src/components/screencast/useScreencastInput.ts b/dashboard/src/components/screencast/useScreencastInput.ts
index d05c573d..2ed75eff 100644
--- a/dashboard/src/components/screencast/useScreencastInput.ts
+++ b/dashboard/src/components/screencast/useScreencastInput.ts
@@ -24,6 +24,10 @@ export function useScreencastInput({
return {
x: ((e.clientX - rect.left) / rect.width) * canvas.width,
y: ((e.clientY - rect.top) / rect.height) * canvas.height,
+ // Tell the server which frame size these coords are in, so it can scale
+ // them into the live CSS viewport (the frame is larger on HiDPI).
+ frameW: canvas.width,
+ frameH: canvas.height,
};
},
[canvasRef],
@@ -41,6 +45,8 @@ export function useScreencastInput({
x: Math.round(coords.x),
y: Math.round(coords.y),
hasXY: true,
+ frameW: coords.frameW,
+ frameH: coords.frameH,
});
} catch (err) {
console.error("click failed", err);
@@ -61,6 +67,8 @@ export function useScreencastInput({
x: Math.round(coords.x),
y: Math.round(coords.y),
scrollY: Math.round(e.deltaY),
+ frameW: coords.frameW,
+ frameH: coords.frameH,
});
} catch (err) {
console.error("scroll failed", err);
diff --git a/internal/bridge/action_pointer.go b/internal/bridge/action_pointer.go
index 3392eba0..e0c1f01f 100644
--- a/internal/bridge/action_pointer.go
+++ b/internal/bridge/action_pointer.go
@@ -221,7 +221,25 @@ func dialogBlocking(dm *DialogManager, tabID string) error {
}
}
+// scaleScreencastCoords rescales req.X/Y from the screencast frame pixel space
+// (req.FrameW/FrameH) into the live CSS viewport. Dashboard input maps a click on
+// the frame to frame-pixel coordinates; on HiDPI the frame is larger than the CSS
+// viewport (e.g. 2x), so without this the click would land at the wrong position
+// and miss its target. No-op when FrameW/FrameH are unset (coords already CSS px).
+func scaleScreencastCoords(ctx context.Context, req *ActionRequest) {
+ if !req.HasXY || req.FrameW <= 0 || req.FrameH <= 0 {
+ return
+ }
+ vw, vh := fetchViewportSize(ctx)
+ if vw <= 0 || vh <= 0 {
+ return
+ }
+ req.X = req.X * vw / req.FrameW
+ req.Y = req.Y * vh / req.FrameH
+}
+
func (b *Bridge) actionClick(ctx context.Context, req ActionRequest) (result map[string]any, err error) {
+ scaleScreencastCoords(ctx, &req)
if b.effectiveHumanize(req) {
return b.actionHumanizedClick(ctx, req)
}
@@ -522,6 +540,7 @@ func (b *Bridge) actionMouseWheel(ctx context.Context, req ActionRequest) (map[s
}
func (b *Bridge) actionScroll(ctx context.Context, req ActionRequest) (map[string]any, error) {
+ scaleScreencastCoords(ctx, &req)
if req.NodeID > 0 {
return map[string]any{"scrolled": true}, ScrollByNodeID(ctx, req.NodeID)
}
diff --git a/internal/bridge/action_request.go b/internal/bridge/action_request.go
index d345d168..c9783212 100644
--- a/internal/bridge/action_request.go
+++ b/internal/bridge/action_request.go
@@ -53,6 +53,15 @@ type ActionRequest struct {
Button string `json:"button,omitempty"`
Mode string `json:"mode,omitempty"`
+ // FrameW/FrameH are the pixel dimensions of the screencast frame the caller
+ // mapped the coordinates against (the dashboard maps a click to frame-pixel
+ // space). When set, X/Y are scaled from that frame space into the live CSS
+ // viewport before dispatch, so a HiDPI frame (e.g. 2x the CSS viewport) does
+ // not land clicks at twice their intended position. Zero means X/Y are
+ // already CSS pixels.
+ FrameW float64 `json:"frameW,omitempty"`
+ FrameH float64 `json:"frameH,omitempty"`
+
ScrollX int `json:"scrollX"`
ScrollY int `json:"scrollY"`
// DeltaX/DeltaY are explicit mouse-wheel deltas for low-level
diff --git a/tests/e2e/fixtures/screencast-click-targets.html b/tests/e2e/fixtures/screencast-click-targets.html
new file mode 100644
index 00000000..8d4480df
--- /dev/null
+++ b/tests/e2e/fixtures/screencast-click-targets.html
@@ -0,0 +1,56 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <title>E2E Test - Screencast Click Targets</title>
+ <style>
+ html, body { margin: 0; padding: 0; height: 100%; width: 100%;
+ font-family: sans-serif; background: #fafafa; overflow: hidden; }
+ .t { position: absolute; width: 120px; height: 56px; box-sizing: border-box;
+ border: 2px solid #2d6cdf; border-radius: 8px; background: #e8f0fe;
+ color: #17315f; display: flex; align-items: center; justify-content: center;
+ font-size: 16px; font-weight: 700; cursor: pointer; user-select: none; }
+ /* Targets spread across corners, edges, and centre. Right/bottom anchored
+ ones adapt to the viewport; the test reads each rect at runtime. */
+ #tl { top: 16px; left: 16px; }
+ #tr { top: 16px; right: 16px; }
+ #ml { top: 45%; left: 16px; }
+ #c { top: 45%; left: 50%; transform: translateX(-50%); }
+ #mr { top: 45%; right: 16px; }
+ #bl { bottom: 16px; left: 16px; }
+ #br { bottom: 16px; right: 16px; }
+ #inp { position: absolute; top: 72%; left: 50%; transform: translateX(-50%);
+ width: 240px; height: 48px; font-size: 18px; box-sizing: border-box; }
+ #status { position: absolute; top: 8px; left: 50%; transform: translateX(-50%);
+ font-size: 14px; color: #555; }
+ </style>
+</head>
+<body>
+ <button class="t" id="tl">TL</button>
+ <button class="t" id="tr">TR</button>
+ <button class="t" id="ml">ML</button>
+ <button class="t" id="c">C</button>
+ <button class="t" id="mr">MR</button>
+ <button class="t" id="bl">BL</button>
+ <button class="t" id="br">BR</button>
+ <input class="t" id="inp" placeholder="type here">
+ <div id="status">no click yet</div>
+
+ <script>
+ // Records which element a (coordinate-mapped) click actually landed on, so an
+ // e2e can verify the dashboard's frame-pixel → CSS-viewport scaling lands
+ // clicks on the intended target across different frame scales and positions.
+ window.__lastClick = "";
+ window.__lastFocus = "";
+ const status = document.getElementById("status");
+ document.querySelectorAll(".t").forEach((el) => {
+ el.addEventListener("click", () => {
+ window.__lastClick = el.id;
+ status.textContent = "clicked " + el.id;
+ });
+ el.addEventListener("focus", () => { window.__lastFocus = el.id; });
+ });
+ </script>
+</body>
+</html>
diff --git a/tests/e2e/scenarios/api/screencast-input-basic.sh b/tests/e2e/scenarios/api/screencast-input-basic.sh
new file mode 100644
index 00000000..c2fe9450
--- /dev/null
+++ b/tests/e2e/scenarios/api/screencast-input-basic.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+# screencast-input.sh — API scenarios for dashboard screencast input mapping.
+#
+# The dashboard maps a click on the screencast canvas to FRAME-pixel coordinates
+# and sends them with the frame dimensions (frameW/frameH). The server must scale
+# those into the live CSS viewport so the click lands on the intended element,
+# regardless of the frame's pixel scale (HiDPI frames are 2x+ the CSS viewport).
+# These tests exercise that mapping across multiple frame scales and target
+# positions, so it stays correct on every browser provider the suite runs against.
+
+GROUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${GROUP_DIR}/../../helpers/api.sh"
+
+SC_TAB_ID=""
+
+sc_navigate() {
+ local url="$1"
+ local body
+ if [ -n "$SC_TAB_ID" ]; then
+ body=$(jq -nc --arg url "$url" --arg tabId "$SC_TAB_ID" '{url:$url, tabId:$tabId}')
+ else
+ body=$(jq -nc --arg url "$url" '{url:$url}')
+ fi
+ pt_post /navigate -d "$body"
+ assert_ok "navigate"
+ local tab_id
+ tab_id=$(echo "$RESULT" | jq -r '.tabId // empty')
+ [ -n "$tab_id" ] && [ "$tab_id" != "null" ] && SC_TAB_ID="$tab_id"
+}
+
+# sc_eval <expression> → echoes the evaluated .result value
+sc_eval() {
+ local body
+ body=$(jq -nc --arg tabId "$SC_TAB_ID" --arg expression "$1" '{tabId:$tabId, expression:$expression}')
+ pt_post /evaluate "$body" >/dev/null
+ echo "$RESULT" | jq -r '.result // empty'
+}
+
+# sc_click_frame <x> <y> [frameW] [frameH] — click in frame-pixel space
+sc_click_frame() {
+ local x="$1" y="$2" fw="${3:-}" fh="${4:-}"
+ local body
+ if [ -n "$fw" ]; then
+ body=$(jq -nc --arg t "$SC_TAB_ID" --argjson x "$x" --argjson y "$y" --argjson fw "$fw" --argjson fh "$fh" \
+ '{kind:"click", tabId:$t, x:$x, y:$y, hasXY:true, frameW:$fw, frameH:$fh}')
+ else
+ body=$(jq -nc --arg t "$SC_TAB_ID" --argjson x "$x" --argjson y "$y" \
+ '{kind:"click", tabId:$t, x:$x, y:$y, hasXY:true}')
+ fi
+ pt_post /action "$body" >/dev/null
+}
+
+# ─────────────────────────────────────────────────────────────────
+start_test "screencast click mapping: every target at every frame scale"
+
+sc_navigate "${FIXTURES_URL}/screencast-click-targets.html"
+
+VP=$(sc_eval 'JSON.stringify([Math.round(window.innerWidth), Math.round(window.innerHeight)])')
+CSSW=$(echo "$VP" | jq -r '.[0]')
+CSSH=$(echo "$VP" | jq -r '.[1]')
+echo " css viewport: ${CSSW}x${CSSH}"
+
+# Frame scales to simulate: 1x (no scaling), and HiDPI/odd ratios.
+SCALES="1 1.5 2 3"
+TARGETS="tl tr ml c mr bl br"
+
+for id in $TARGETS; do
+ CENTER=$(sc_eval "var r=document.getElementById('${id}').getBoundingClientRect(); JSON.stringify([Math.round(r.left+r.width/2), Math.round(r.top+r.height/2)])")
+ CX=$(echo "$CENTER" | jq -r '.[0]')
+ CY=$(echo "$CENTER" | jq -r '.[1]')
+ for s in $SCALES; do
+ read -r FX FY FW FH < <(python3 -c "cx,cy,cw,ch,s=$CX,$CY,$CSSW,$CSSH,$s; print(round(cx*s), round(cy*s), round(cw*s), round(ch*s))")
+ sc_eval 'window.__lastClick=""; "ok"' >/dev/null
+ sc_click_frame "$FX" "$FY" "$FW" "$FH"
+ GOT=$(sc_eval 'window.__lastClick')
+ if [ "$GOT" = "$id" ]; then
+ pass_assert "scale ${s}x → click frame(${FX},${FY}) landed on '${id}'"
+ else
+ fail_assert "scale ${s}x → click frame(${FX},${FY}) landed on '${GOT}', want '${id}'"
+ fi
+ done
+done
+
+end_test
+
+# ─────────────────────────────────────────────────────────────────
+start_test "screencast click focuses an input field (HiDPI frame)"
+
+sc_navigate "${FIXTURES_URL}/screencast-click-targets.html"
+CSSW=$(sc_eval 'Math.round(window.innerWidth)')
+CSSH=$(sc_eval 'Math.round(window.innerHeight)')
+CENTER=$(sc_eval "var r=document.getElementById('inp').getBoundingClientRect(); JSON.stringify([Math.round(r.left+r.width/2), Math.round(r.top+r.height/2)])")
+CX=$(echo "$CENTER" | jq -r '.[0]')
+CY=$(echo "$CENTER" | jq -r '.[1]')
+
+# Simulate a 2x HiDPI frame.
+read -r FX FY FW FH < <(python3 -c "cx,cy,cw,ch=$CX,$CY,$CSSW,$CSSH; print(cx*2, cy*2, cw*2, ch*2)")
+sc_eval 'document.getElementById("inp").blur(); "ok"' >/dev/null
+sc_click_frame "$FX" "$FY" "$FW" "$FH"
+ACTIVE=$(sc_eval 'document.activeElement.id')
+if [ "$ACTIVE" = "inp" ]; then
+ pass_assert "2x-frame click focused the input"
+else
+ fail_assert "2x-frame click left focus on '${ACTIVE}', want 'inp'"
+fi
+
+end_test
+
+# ─────────────────────────────────────────────────────────────────
+start_test "screencast click without frameW treats coords as CSS pixels"
+
+sc_navigate "${FIXTURES_URL}/screencast-click-targets.html"
+CENTER=$(sc_eval "var r=document.getElementById('c').getBoundingClientRect(); JSON.stringify([Math.round(r.left+r.width/2), Math.round(r.top+r.height/2)])")
+CX=$(echo "$CENTER" | jq -r '.[0]')
+CY=$(echo "$CENTER" | jq -r '.[1]')
+sc_eval 'window.__lastClick=""; "ok"' >/dev/null
+sc_click_frame "$CX" "$CY" # no frameW/frameH → coords are already CSS px
+GOT=$(sc_eval 'window.__lastClick')
+if [ "$GOT" = "c" ]; then
+ pass_assert "CSS-pixel click (no frameW) landed on centre target"
+else
+ fail_assert "CSS-pixel click landed on '${GOT}', want 'c'"
+fi
+
+end_test
diff --git a/tests/e2e/scenarios/manifest.json b/tests/e2e/scenarios/manifest.json
index 66fc6c91..a07b75ff 100644
--- a/tests/e2e/scenarios/manifest.json
+++ b/tests/e2e/scenarios/manifest.json
@@ -25,6 +25,9 @@
"api/auto-switch-smoke.sh": {
"tags": ["tabs", "actions", "auto-switch", "smoke"]
},
+ "api/screencast-input-basic.sh": {
+ "tags": ["actions", "screencast", "input", "coordinates", "multi-browser", "pr"]
+ },
"api/network-route-extended.sh": {
"services": ["pinchtab", "pinchtab-full", "fixtures"],
"ready": ["E2E_SERVER", "E2E_FULL_SERVER"],