High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix: scale screencast clicks into the css viewport
Bosch5abb8db35f21
diff --git a/dashboard/src/components/screencast/useScreencastInput.ts b/dashboard/src/components/screencast/useScreencastInput.tsindex d05c573d..2ed75eff 100644--- a/dashboard/src/components/screencast/useScreencastInput.ts+++ b/dashboard/src/components/screencast/useScreencastInput.ts@@ -24,6 +24,10 @@ export function useScreencastInput({return {x: ((e.clientX - rect.left) / rect.width) * canvas.width,y: ((e.clientY - rect.top) / rect.height) * canvas.height,+ // Tell the server which frame size these coords are in, so it can scale+ // them into the live CSS viewport (the frame is larger on HiDPI).+ frameW: canvas.width,+ frameH: canvas.height,};},[canvasRef],@@ -41,6 +45,8 @@ export function useScreencastInput({x: Math.round(coords.x),y: Math.round(coords.y),hasXY: true,+ frameW: coords.frameW,+ frameH: coords.frameH,});} catch (err) {console.error("click failed", err);@@ -61,6 +67,8 @@ export function useScreencastInput({x: Math.round(coords.x),y: Math.round(coords.y),scrollY: Math.round(e.deltaY),+ frameW: coords.frameW,+ frameH: coords.frameH,});} catch (err) {console.error("scroll failed", err);diff --git a/internal/bridge/action_pointer.go b/internal/bridge/action_pointer.goindex 3392eba0..e0c1f01f 100644--- a/internal/bridge/action_pointer.go+++ b/internal/bridge/action_pointer.go@@ -221,7 +221,25 @@ func dialogBlocking(dm *DialogManager, tabID string) error {}}+// scaleScreencastCoords rescales req.X/Y from the screencast frame pixel space+// (req.FrameW/FrameH) into the live CSS viewport. Dashboard input maps a click on+// the frame to frame-pixel coordinates; on HiDPI the frame is larger than the CSS+// viewport (e.g. 2x), so without this the click would land at the wrong position+// and miss its target. No-op when FrameW/FrameH are unset (coords already CSS px).+func scaleScreencastCoords(ctx context.Context, req *ActionRequest) {+ if !req.HasXY || req.FrameW <= 0 || req.FrameH <= 0 {+ return+ }+ vw, vh := fetchViewportSize(ctx)+ if vw <= 0 || vh <= 0 {+ return+ }+ req.X = req.X * vw / req.FrameW+ req.Y = req.Y * vh / req.FrameH+}+func (b *Bridge) actionClick(ctx context.Context, req ActionRequest) (result map[string]any, err error) {+ scaleScreencastCoords(ctx, &req)if b.effectiveHumanize(req) {return b.actionHumanizedClick(ctx, req)}@@ -522,6 +540,7 @@ func (b *Bridge) actionMouseWheel(ctx context.Context, req ActionRequest) (map[s}func (b *Bridge) actionScroll(ctx context.Context, req ActionRequest) (map[string]any, error) {+ scaleScreencastCoords(ctx, &req)if req.NodeID > 0 {return map[string]any{"scrolled": true}, ScrollByNodeID(ctx, req.NodeID)}diff --git a/internal/bridge/action_request.go b/internal/bridge/action_request.goindex d345d168..c9783212 100644--- a/internal/bridge/action_request.go+++ b/internal/bridge/action_request.go@@ -53,6 +53,15 @@ type ActionRequest struct {Button string `json:"button,omitempty"`Mode string `json:"mode,omitempty"`+ // FrameW/FrameH are the pixel dimensions of the screencast frame the caller+ // mapped the coordinates against (the dashboard maps a click to frame-pixel+ // space). When set, X/Y are scaled from that frame space into the live CSS+ // viewport before dispatch, so a HiDPI frame (e.g. 2x the CSS viewport) does+ // not land clicks at twice their intended position. Zero means X/Y are+ // already CSS pixels.+ FrameW float64 `json:"frameW,omitempty"`+ FrameH float64 `json:"frameH,omitempty"`+ScrollX int `json:"scrollX"`ScrollY int `json:"scrollY"`// DeltaX/DeltaY are explicit mouse-wheel deltas for low-leveldiff --git a/tests/e2e/fixtures/screencast-click-targets.html b/tests/e2e/fixtures/screencast-click-targets.htmlnew file mode 100644index 00000000..8d4480df--- /dev/null+++ b/tests/e2e/fixtures/screencast-click-targets.html@@ -0,0 +1,56 @@+<!DOCTYPE html>+<html lang="en">+<head>+ <meta charset="UTF-8">+ <meta name="viewport" content="width=device-width, initial-scale=1.0">+ <title>E2E Test - Screencast Click Targets</title>+ <style>+ html, body { margin: 0; padding: 0; height: 100%; width: 100%;+ font-family: sans-serif; background: #fafafa; overflow: hidden; }+ .t { position: absolute; width: 120px; height: 56px; box-sizing: border-box;+ border: 2px solid #2d6cdf; border-radius: 8px; background: #e8f0fe;+ color: #17315f; display: flex; align-items: center; justify-content: center;+ font-size: 16px; font-weight: 700; cursor: pointer; user-select: none; }+ /* Targets spread across corners, edges, and centre. Right/bottom anchored+ ones adapt to the viewport; the test reads each rect at runtime. */+ #tl { top: 16px; left: 16px; }+ #tr { top: 16px; right: 16px; }+ #ml { top: 45%; left: 16px; }+ #c { top: 45%; left: 50%; transform: translateX(-50%); }+ #mr { top: 45%; right: 16px; }+ #bl { bottom: 16px; left: 16px; }+ #br { bottom: 16px; right: 16px; }+ #inp { position: absolute; top: 72%; left: 50%; transform: translateX(-50%);+ width: 240px; height: 48px; font-size: 18px; box-sizing: border-box; }+ #status { position: absolute; top: 8px; left: 50%; transform: translateX(-50%);+ font-size: 14px; color: #555; }+ </style>+</head>+<body>+ <button class="t" id="tl">TL</button>+ <button class="t" id="tr">TR</button>+ <button class="t" id="ml">ML</button>+ <button class="t" id="c">C</button>+ <button class="t" id="mr">MR</button>+ <button class="t" id="bl">BL</button>+ <button class="t" id="br">BR</button>+ <input class="t" id="inp" placeholder="type here">+ <div id="status">no click yet</div>++ <script>+ // Records which element a (coordinate-mapped) click actually landed on, so an+ // e2e can verify the dashboard's frame-pixel → CSS-viewport scaling lands+ // clicks on the intended target across different frame scales and positions.+ window.__lastClick = "";+ window.__lastFocus = "";+ const status = document.getElementById("status");+ document.querySelectorAll(".t").forEach((el) => {+ el.addEventListener("click", () => {+ window.__lastClick = el.id;+ status.textContent = "clicked " + el.id;+ });+ el.addEventListener("focus", () => { window.__lastFocus = el.id; });+ });+ </script>+</body>+</html>diff --git a/tests/e2e/scenarios/api/screencast-input-basic.sh b/tests/e2e/scenarios/api/screencast-input-basic.shnew file mode 100644index 00000000..c2fe9450--- /dev/null+++ b/tests/e2e/scenarios/api/screencast-input-basic.sh@@ -0,0 +1,125 @@+#!/bin/bash+# screencast-input.sh — API scenarios for dashboard screencast input mapping.+#+# The dashboard maps a click on the screencast canvas to FRAME-pixel coordinates+# and sends them with the frame dimensions (frameW/frameH). The server must scale+# those into the live CSS viewport so the click lands on the intended element,+# regardless of the frame's pixel scale (HiDPI frames are 2x+ the CSS viewport).+# These tests exercise that mapping across multiple frame scales and target+# positions, so it stays correct on every browser provider the suite runs against.++GROUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+source "${GROUP_DIR}/../../helpers/api.sh"++SC_TAB_ID=""++sc_navigate() {+ local url="$1"+ local body+ if [ -n "$SC_TAB_ID" ]; then+ body=$(jq -nc --arg url "$url" --arg tabId "$SC_TAB_ID" '{url:$url, tabId:$tabId}')+ else+ body=$(jq -nc --arg url "$url" '{url:$url}')+ fi+ pt_post /navigate -d "$body"+ assert_ok "navigate"+ local tab_id+ tab_id=$(echo "$RESULT" | jq -r '.tabId // empty')+ [ -n "$tab_id" ] && [ "$tab_id" != "null" ] && SC_TAB_ID="$tab_id"+}++# sc_eval <expression> → echoes the evaluated .result value+sc_eval() {+ local body+ body=$(jq -nc --arg tabId "$SC_TAB_ID" --arg expression "$1" '{tabId:$tabId, expression:$expression}')+ pt_post /evaluate "$body" >/dev/null+ echo "$RESULT" | jq -r '.result // empty'+}++# sc_click_frame <x> <y> [frameW] [frameH] — click in frame-pixel space+sc_click_frame() {+ local x="$1" y="$2" fw="${3:-}" fh="${4:-}"+ local body+ if [ -n "$fw" ]; then+ body=$(jq -nc --arg t "$SC_TAB_ID" --argjson x "$x" --argjson y "$y" --argjson fw "$fw" --argjson fh "$fh" \+ '{kind:"click", tabId:$t, x:$x, y:$y, hasXY:true, frameW:$fw, frameH:$fh}')+ else+ body=$(jq -nc --arg t "$SC_TAB_ID" --argjson x "$x" --argjson y "$y" \+ '{kind:"click", tabId:$t, x:$x, y:$y, hasXY:true}')+ fi+ pt_post /action "$body" >/dev/null+}++# ─────────────────────────────────────────────────────────────────+start_test "screencast click mapping: every target at every frame scale"++sc_navigate "${FIXTURES_URL}/screencast-click-targets.html"++VP=$(sc_eval 'JSON.stringify([Math.round(window.innerWidth), Math.round(window.innerHeight)])')+CSSW=$(echo "$VP" | jq -r '.[0]')+CSSH=$(echo "$VP" | jq -r '.[1]')+echo " css viewport: ${CSSW}x${CSSH}"++# Frame scales to simulate: 1x (no scaling), and HiDPI/odd ratios.+SCALES="1 1.5 2 3"+TARGETS="tl tr ml c mr bl br"++for id in $TARGETS; do+ CENTER=$(sc_eval "var r=document.getElementById('${id}').getBoundingClientRect(); JSON.stringify([Math.round(r.left+r.width/2), Math.round(r.top+r.height/2)])")+ CX=$(echo "$CENTER" | jq -r '.[0]')+ CY=$(echo "$CENTER" | jq -r '.[1]')+ for s in $SCALES; do+ read -r FX FY FW FH < <(python3 -c "cx,cy,cw,ch,s=$CX,$CY,$CSSW,$CSSH,$s; print(round(cx*s), round(cy*s), round(cw*s), round(ch*s))")+ sc_eval 'window.__lastClick=""; "ok"' >/dev/null+ sc_click_frame "$FX" "$FY" "$FW" "$FH"+ GOT=$(sc_eval 'window.__lastClick')+ if [ "$GOT" = "$id" ]; then+ pass_assert "scale ${s}x → click frame(${FX},${FY}) landed on '${id}'"+ else+ fail_assert "scale ${s}x → click frame(${FX},${FY}) landed on '${GOT}', want '${id}'"+ fi+ done+done++end_test++# ─────────────────────────────────────────────────────────────────+start_test "screencast click focuses an input field (HiDPI frame)"++sc_navigate "${FIXTURES_URL}/screencast-click-targets.html"+CSSW=$(sc_eval 'Math.round(window.innerWidth)')+CSSH=$(sc_eval 'Math.round(window.innerHeight)')+CENTER=$(sc_eval "var r=document.getElementById('inp').getBoundingClientRect(); JSON.stringify([Math.round(r.left+r.width/2), Math.round(r.top+r.height/2)])")+CX=$(echo "$CENTER" | jq -r '.[0]')+CY=$(echo "$CENTER" | jq -r '.[1]')++# Simulate a 2x HiDPI frame.+read -r FX FY FW FH < <(python3 -c "cx,cy,cw,ch=$CX,$CY,$CSSW,$CSSH; print(cx*2, cy*2, cw*2, ch*2)")+sc_eval 'document.getElementById("inp").blur(); "ok"' >/dev/null+sc_click_frame "$FX" "$FY" "$FW" "$FH"+ACTIVE=$(sc_eval 'document.activeElement.id')+if [ "$ACTIVE" = "inp" ]; then+ pass_assert "2x-frame click focused the input"+else+ fail_assert "2x-frame click left focus on '${ACTIVE}', want 'inp'"+fi++end_test++# ─────────────────────────────────────────────────────────────────+start_test "screencast click without frameW treats coords as CSS pixels"++sc_navigate "${FIXTURES_URL}/screencast-click-targets.html"+CENTER=$(sc_eval "var r=document.getElementById('c').getBoundingClientRect(); JSON.stringify([Math.round(r.left+r.width/2), Math.round(r.top+r.height/2)])")+CX=$(echo "$CENTER" | jq -r '.[0]')+CY=$(echo "$CENTER" | jq -r '.[1]')+sc_eval 'window.__lastClick=""; "ok"' >/dev/null+sc_click_frame "$CX" "$CY" # no frameW/frameH → coords are already CSS px+GOT=$(sc_eval 'window.__lastClick')+if [ "$GOT" = "c" ]; then+ pass_assert "CSS-pixel click (no frameW) landed on centre target"+else+ fail_assert "CSS-pixel click landed on '${GOT}', want 'c'"+fi++end_testdiff --git a/tests/e2e/scenarios/manifest.json b/tests/e2e/scenarios/manifest.jsonindex 66fc6c91..a07b75ff 100644--- a/tests/e2e/scenarios/manifest.json+++ b/tests/e2e/scenarios/manifest.json@@ -25,6 +25,9 @@"api/auto-switch-smoke.sh": {"tags": ["tabs", "actions", "auto-switch", "smoke"]},+ "api/screencast-input-basic.sh": {+ "tags": ["actions", "screencast", "input", "coordinates", "multi-browser", "pr"]+ },"api/network-route-extended.sh": {"services": ["pinchtab", "pinchtab-full", "fixtures"],"ready": ["E2E_SERVER", "E2E_FULL_SERVER"],