Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix: pinchtab_hover forwards a humanize flag actionHover never reads… (PIN-186)

Luigi Agostiac499e3add06
diff --git a/internal/bridge/action_pointer.go b/internal/bridge/action_pointer.go
index de99b1e4..5616fede 100644
--- a/internal/bridge/action_pointer.go
+++ b/internal/bridge/action_pointer.go
@@ -19,6 +19,8 @@ var mouseDownByCoordinateAction = MouseDownByCoordinate
var mouseUpByCoordinateAction = MouseUpByCoordinate
var clickByCoordinateAction = ClickByCoordinate
var clickElementAction = ClickElement
+var hoverElementAction = HoverElement
+var hoverCoordinateAction = Hover
var clickByNodeIDAction = ClickByNodeID
var jsClickByBackendNodeAction = JSClickByBackendNode
var dispatchClickByBackendNodeAction = JSDispatchClickByBackendNode
@@ -425,6 +427,9 @@ func (b *Bridge) actionDoubleClick(ctx context.Context, req ActionRequest) (resu
}
func (b *Bridge) actionHover(ctx context.Context, req ActionRequest) (map[string]any, error) {
+ if b.effectiveHumanize(req) {
+ return b.actionHumanizedHover(ctx, req)
+ }
if req.NodeID > 0 {
return map[string]any{"hovered": true}, HoverByNodeID(ctx, req.NodeID)
}
@@ -441,6 +446,28 @@ func (b *Bridge) actionHover(ctx context.Context, req ActionRequest) (map[string
return nil, fmt.Errorf("need selector, ref, nodeId, or x/y coordinates")
}
+func (b *Bridge) actionHumanizedHover(ctx context.Context, req ActionRequest) (map[string]any, error) {
+ var err error
+ switch {
+ case req.NodeID > 0:
+ err = hoverElementAction(ctx, cdp.BackendNodeID(req.NodeID))
+ case req.Selector != "":
+ node, nodeErr := firstNodeBySelector(ctx, req.Selector)
+ if nodeErr != nil {
+ return nil, nodeErr
+ }
+ err = hoverElementAction(ctx, node.BackendNodeID)
+ case req.HasXY:
+ err = hoverCoordinateAction(ctx, req.X, req.Y)
+ default:
+ return nil, fmt.Errorf("need selector, ref, nodeId, or x/y coordinates")
+ }
+ if err != nil {
+ return nil, err
+ }
+ return map[string]any{"hovered": true, "human": true}, nil
+}
+
func (b *Bridge) rememberPointerPosition(tabID string, x, y float64) {
if b == nil || tabID == "" {
return
diff --git a/internal/bridge/actions_test.go b/internal/bridge/actions_test.go
index 82bb88d3..6b5a57c0 100644
--- a/internal/bridge/actions_test.go
+++ b/internal/bridge/actions_test.go
@@ -2,13 +2,17 @@ package bridge
import (
"context"
+ "encoding/base64"
"encoding/json"
"errors"
"strings"
"testing"
+ "time"
"github.com/chromedp/cdproto/cdp"
+ "github.com/chromedp/chromedp"
"github.com/pinchtab/pinchtab/internal/config"
+ "github.com/pinchtab/pinchtab/internal/testbrowser"
)
func TestClickAction_UsesCoordinatePathIncludingZeroZero(t *testing.T) {
@@ -1169,3 +1173,140 @@ func TestScrollIntoViewAction_WithSelector_UsesCSSPath(t *testing.T) {
t.Fatalf("expected CSS path, got validation error: %v", err)
}
}
+
+func TestHoverAction_HumanizeRoutesEveryTargetForm(t *testing.T) {
+ origElement := hoverElementAction
+ origCoordinate := hoverCoordinateAction
+ t.Cleanup(func() {
+ hoverElementAction = origElement
+ hoverCoordinateAction = origCoordinate
+ })
+
+ var nodes []cdp.BackendNodeID
+ var points [][2]float64
+ hoverElementAction = func(ctx context.Context, backendNodeID cdp.BackendNodeID) error {
+ nodes = append(nodes, backendNodeID)
+ return nil
+ }
+ hoverCoordinateAction = func(ctx context.Context, x, y float64) error {
+ points = append(points, [2]float64{x, y})
+ return nil
+ }
+
+ tests := []struct {
+ name string
+ config *config.RuntimeConfig
+ req ActionRequest
+ wantNodes []cdp.BackendNodeID
+ wantPoints [][2]float64
+ wantHuman bool
+ }{
+ {
+ name: "nodeId with request opt-in",
+ config: &config.RuntimeConfig{},
+ req: ActionRequest{NodeID: 77, Humanize: boolPtr(true)},
+ wantNodes: []cdp.BackendNodeID{77},
+ wantHuman: true,
+ },
+ {
+ name: "coordinates with instance default",
+ config: &config.RuntimeConfig{Humanize: true},
+ req: ActionRequest{HasXY: true, X: 12, Y: 34},
+ wantPoints: [][2]float64{{12, 34}},
+ wantHuman: true,
+ },
+ {
+ name: "explicit false opts out of the humanized path",
+ config: &config.RuntimeConfig{Humanize: true},
+ req: ActionRequest{NodeID: 77, Humanize: boolPtr(false)},
+ wantHuman: false,
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ nodes, points = nil, nil
+ b := New(context.TODO(), nil, tc.config)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ res, err := b.Actions[ActionHover](ctx, tc.req)
+
+ if tc.wantHuman {
+ if err != nil {
+ t.Fatalf("humanized hover returned error: %v", err)
+ }
+ if hovered, _ := res["hovered"].(bool); !hovered {
+ t.Fatalf("result = %#v, want hovered=true", res)
+ }
+ if human, _ := res["human"].(bool); !human {
+ t.Fatalf("result = %#v, want human=true", res)
+ }
+ } else if _, ok := res["human"]; ok {
+ t.Fatalf("raw hover result = %#v, want no human key", res)
+ }
+
+ if len(nodes) != len(tc.wantNodes) || (len(nodes) == 1 && nodes[0] != tc.wantNodes[0]) {
+ t.Fatalf("element hovers = %v, want %v", nodes, tc.wantNodes)
+ }
+ if len(points) != len(tc.wantPoints) || (len(points) == 1 && points[0] != tc.wantPoints[0]) {
+ t.Fatalf("coordinate hovers = %v, want %v", points, tc.wantPoints)
+ }
+ })
+ }
+}
+
+func TestHoverAction_HumanizedStillRequiresATarget(t *testing.T) {
+ b := New(context.TODO(), nil, &config.RuntimeConfig{Humanize: true})
+
+ _, err := b.Actions[ActionHover](context.Background(), ActionRequest{})
+ if err == nil || !strings.Contains(err.Error(), "need selector") {
+ t.Fatalf("targetless humanized hover error = %v, want the target requirement", err)
+ }
+}
+
+// Browser-backed: the selector form resolves through firstNodeBySelector, which
+// has no browserless seam, so this is where selector routing and the real
+// trail landing on the target are pinned.
+func TestHoverAction_HumanizedSelectorLandsOnTheTarget(t *testing.T) {
+ chromePath := testbrowser.Path(t)
+ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(
+ chromedp.DefaultExecAllocatorOptions[:],
+ chromedp.ExecPath(chromePath),
+ chromedp.Flag("headless", true),
+ chromedp.Flag("no-sandbox", true),
+ )...)
+ defer cancelAlloc()
+ ctx, cancel := chromedp.NewContext(alloc)
+ defer cancel()
+ ctx, cancelTimeout := context.WithTimeout(ctx, 30*time.Second)
+ defer cancelTimeout()
+
+ html := `<style>#target { position: absolute; top: 100px; left: 100px; width: 120px; height: 40px; }</style>
+ <div id="target">hover me</div>
+ <script>
+ window.hovered = false;
+ document.getElementById("target").addEventListener("mouseover", () => window.hovered = true);
+ </script>`
+ dataURL := "data:text/html;base64," + base64.StdEncoding.EncodeToString([]byte(html))
+ if err := chromedp.Run(ctx, chromedp.Navigate(dataURL)); err != nil {
+ t.Fatal(err)
+ }
+
+ b := New(context.Background(), nil, &config.RuntimeConfig{Humanize: true})
+ res, err := b.Actions[ActionHover](ctx, ActionRequest{Selector: "#target"})
+ if err != nil {
+ t.Fatalf("humanized selector hover: %v", err)
+ }
+ if human, _ := res["human"].(bool); !human {
+ t.Fatalf("result = %#v, want human=true", res)
+ }
+
+ var hovered bool
+ if err := chromedp.Run(ctx, chromedp.Evaluate(`window.hovered`, &hovered)); err != nil {
+ t.Fatal(err)
+ }
+ if !hovered {
+ t.Fatal("humanized hover did not fire mouseover on the selector target")
+ }
+}
diff --git a/internal/bridge/input_human.go b/internal/bridge/input_human.go
index 9425e188..3e491c14 100644
--- a/internal/bridge/input_human.go
+++ b/internal/bridge/input_human.go
@@ -124,29 +124,41 @@ func MouseMove(ctx context.Context, fromX, fromY, toX, toY float64) error {
return nil
}
-func Click(ctx context.Context, x, y float64) error {
+// approachTarget walks the pointer to (x, y) from a random nearby start along
+// the bezier trail. Best-effort: the trail is only there for human-trail
+// realism, so a stall is logged and swallowed and the caller proceeds — its own
+// dispatch at (x, y) is what has to land. Only a cancelled outer context is
+// reported back.
+func approachTarget(ctx context.Context, x, y float64) error {
startOffsetX := (humanRand.Float64()-0.5)*200 + 50
startOffsetY := (humanRand.Float64()-0.5)*200 + 50
startX := x + startOffsetX
startY := y + startOffsetY
distance := math.Sqrt(startOffsetX*startOffsetX + startOffsetY*startOffsetY)
- if distance > 30 {
- // Best-effort: the initial random-start move and bezier trail are
- // only there for human-trail realism. If they stall, log and
- // proceed — the subsequent MousePressed sets the cursor at (x,y)
- // explicitly, so the click still lands.
- if err := dispatchMouseMovedBounded(ctx, startX, startY); err != nil {
- if ctx.Err() != nil {
- return ctx.Err()
- }
- slog.Debug("humanized click initial-move stalled, skipping bezier", "err", err)
- } else if err := MouseMove(ctx, startX, startY, x, y); err != nil {
- if ctx.Err() != nil {
- return ctx.Err()
- }
- slog.Debug("humanized click bezier trail failed, proceeding to click", "err", err)
+ if distance <= 30 {
+ return nil
+ }
+
+ if err := dispatchMouseMovedBounded(ctx, startX, startY); err != nil {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ slog.Debug("humanized initial-move stalled, skipping bezier", "err", err)
+ } else if err := MouseMove(ctx, startX, startY, x, y); err != nil {
+ if ctx.Err() != nil {
+ return ctx.Err()
}
+ slog.Debug("humanized bezier trail failed, proceeding to action", "err", err)
+ }
+ return nil
+}
+
+var approachTargetAction = approachTarget
+
+func Click(ctx context.Context, x, y float64) error {
+ if err := approachTargetAction(ctx, x, y); err != nil {
+ return err
}
time.Sleep(time.Duration(50+humanRand.Intn(150)) * time.Millisecond)
@@ -195,25 +207,38 @@ var boxModelForBackendNodeAction = func(ctx context.Context, backendNodeID cdp.B
return box, err
}
-var clickCoordinateHumanAction = Click
+var (
+ clickCoordinateHumanAction = Click
+ hoverCoordinateHumanAction = Hover
+ settleHoverAction = HoverByCoordinate
+)
-// The ID is the backendDOMNodeId from the accessibility tree, NOT a regular DOM nodeId.
-func ClickElement(ctx context.Context, backendNodeID cdp.BackendNodeID) error {
- // Scroll the target into the visual viewport first. The humanized click is
- // coordinate-based (Input.dispatchMouseEvent at the box-model center), so an
- // element below the fold or freshly revealed (lazy-loaded list item,
- // modal-triggered button) would otherwise produce coordinates that land on
- // whatever is currently visible there — the dispatch "succeeds" but the
- // intended click handler never fires. Best-effort: not every node scrolls.
+// Hover is the humanized sibling of Click: the same random-start bezier
+// approach, then the plain hover dispatch instead of press/release.
+func Hover(ctx context.Context, x, y float64) error {
+ if err := approachTargetAction(ctx, x, y); err != nil {
+ return err
+ }
+ return settleHoverAction(ctx, x, y)
+}
+
+// humanPointerPoint resolves the coordinate a humanized pointer action aims at.
+// It scrolls the target into the visual viewport first: the humanized path is
+// coordinate-based (Input.dispatchMouseEvent at the box-model center), so an
+// element below the fold or freshly revealed (lazy-loaded list item,
+// modal-triggered button) would otherwise produce coordinates that land on
+// whatever is currently visible there — the dispatch "succeeds" but the
+// intended handler never fires. Best-effort: not every node scrolls.
+func humanPointerPoint(ctx context.Context, backendNodeID cdp.BackendNodeID) (float64, float64, error) {
_ = scrollIntoViewIfNeededAction(ctx, backendNodeID)
box, err := boxModelForBackendNodeAction(ctx, backendNodeID)
if err != nil {
- return err
+ return 0, 0, err
}
if len(box.Content) < 8 {
- return fmt.Errorf("invalid box model")
+ return 0, 0, fmt.Errorf("invalid box model")
}
centerX := (box.Content[0] + box.Content[2]) / 2
@@ -222,7 +247,25 @@ func ClickElement(ctx context.Context, backendNodeID cdp.BackendNodeID) error {
offsetX := (humanRand.Float64() - 0.5) * 10
offsetY := (humanRand.Float64() - 0.5) * 10
- return clickCoordinateHumanAction(ctx, centerX+offsetX, centerY+offsetY)
+ return centerX + offsetX, centerY + offsetY, nil
+}
+
+// The ID is the backendDOMNodeId from the accessibility tree, NOT a regular DOM nodeId.
+func ClickElement(ctx context.Context, backendNodeID cdp.BackendNodeID) error {
+ x, y, err := humanPointerPoint(ctx, backendNodeID)
+ if err != nil {
+ return err
+ }
+ return clickCoordinateHumanAction(ctx, x, y)
+}
+
+// The ID is the backendDOMNodeId from the accessibility tree, NOT a regular DOM nodeId.
+func HoverElement(ctx context.Context, backendNodeID cdp.BackendNodeID) error {
+ x, y, err := humanPointerPoint(ctx, backendNodeID)
+ if err != nil {
+ return err
+ }
+ return hoverCoordinateHumanAction(ctx, x, y)
}
func Type(text string, fast bool) []chromedp.Action {
diff --git a/internal/bridge/input_human_test.go b/internal/bridge/input_human_test.go
index 463a24f7..29574e77 100644
--- a/internal/bridge/input_human_test.go
+++ b/internal/bridge/input_human_test.go
@@ -127,3 +127,122 @@ func TestClickElement_ScrollsBeforeReadingBoxModel(t *testing.T) {
t.Fatalf("calls = %v, want [scroll box]", calls)
}
}
+
+func TestHover_TrailIsBestEffort(t *testing.T) {
+ origSettle := settleHoverAction
+ t.Cleanup(func() { settleHoverAction = origSettle })
+
+ SetHumanRandSeed(7)
+
+ var settled [][2]float64
+ settleHoverAction = func(ctx context.Context, x, y float64) error {
+ settled = append(settled, [2]float64{x, y})
+ return nil
+ }
+
+ // No browser attached, so every trail dispatch fails; the hover must still land.
+ if err := Hover(context.Background(), 40, 60); err != nil {
+ t.Fatalf("Hover with a failing trail returned %v, want nil", err)
+ }
+ if len(settled) != 1 || settled[0] != [2]float64{40, 60} {
+ t.Fatalf("settled hovers = %v, want one at (40,60)", settled)
+ }
+}
+
+func TestHover_CancelledContextStopsBeforeSettling(t *testing.T) {
+ origSettle := settleHoverAction
+ t.Cleanup(func() { settleHoverAction = origSettle })
+
+ SetHumanRandSeed(7)
+
+ settleHoverAction = func(ctx context.Context, x, y float64) error {
+ t.Fatal("cancelled hover should not reach the settle dispatch")
+ return nil
+ }
+
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ if err := Hover(ctx, 40, 60); !errors.Is(err, context.Canceled) {
+ t.Fatalf("Hover error = %v, want context.Canceled", err)
+ }
+}
+
+func TestHoverElement_ReusesTheClickTargetPoint(t *testing.T) {
+ origScroll := scrollIntoViewIfNeededAction
+ origBoxModel := boxModelForBackendNodeAction
+ origHover := hoverCoordinateHumanAction
+ origClick := clickCoordinateHumanAction
+ t.Cleanup(func() {
+ scrollIntoViewIfNeededAction = origScroll
+ boxModelForBackendNodeAction = origBoxModel
+ hoverCoordinateHumanAction = origHover
+ clickCoordinateHumanAction = origClick
+ })
+
+ var calls []string
+ scrollIntoViewIfNeededAction = func(ctx context.Context, backendNodeID cdp.BackendNodeID) error {
+ calls = append(calls, "scroll")
+ return errors.New("scroll failed but should be best-effort")
+ }
+ boxModelForBackendNodeAction = func(ctx context.Context, backendNodeID cdp.BackendNodeID) (*dom.BoxModel, error) {
+ if backendNodeID != 99 {
+ t.Fatalf("box backendNodeID = %d, want 99", backendNodeID)
+ }
+ calls = append(calls, "box")
+ return &dom.BoxModel{Content: []float64{100, 200, 300, 200, 300, 260, 100, 260}}, nil
+ }
+
+ var hoverX, hoverY float64
+ hoverCoordinateHumanAction = func(ctx context.Context, x, y float64) error {
+ calls = append(calls, "hover")
+ hoverX, hoverY = x, y
+ return nil
+ }
+ var clickX, clickY float64
+ clickCoordinateHumanAction = func(ctx context.Context, x, y float64) error {
+ clickX, clickY = x, y
+ return nil
+ }
+
+ SetHumanRandSeed(11)
+ if err := HoverElement(context.Background(), 99); err != nil {
+ t.Fatalf("HoverElement returned %v", err)
+ }
+ hoverCalls := append([]string(nil), calls...)
+
+ SetHumanRandSeed(11)
+ if err := ClickElement(context.Background(), 99); err != nil {
+ t.Fatalf("ClickElement returned %v", err)
+ }
+
+ if hoverX != clickX || hoverY != clickY {
+ t.Fatalf("hover point (%v,%v) != click point (%v,%v); the two humanized paths must aim at the same box-model point", hoverX, hoverY, clickX, clickY)
+ }
+ if hoverX < 100 || hoverX > 300 || hoverY < 200 || hoverY > 260 {
+ t.Fatalf("hover point (%v,%v) is outside the box model", hoverX, hoverY)
+ }
+ if len(hoverCalls) != 3 || hoverCalls[0] != "scroll" || hoverCalls[1] != "box" || hoverCalls[2] != "hover" {
+ t.Fatalf("calls = %v, want [scroll box hover]", hoverCalls)
+ }
+}
+
+func TestHoverElement_PropagatesBoxModelFailure(t *testing.T) {
+ origBoxModel := boxModelForBackendNodeAction
+ origHover := hoverCoordinateHumanAction
+ t.Cleanup(func() {
+ boxModelForBackendNodeAction = origBoxModel
+ hoverCoordinateHumanAction = origHover
+ })
+
+ boxModelForBackendNodeAction = func(ctx context.Context, backendNodeID cdp.BackendNodeID) (*dom.BoxModel, error) {
+ return &dom.BoxModel{Content: []float64{1, 2}}, nil
+ }
+ hoverCoordinateHumanAction = func(ctx context.Context, x, y float64) error {
+ t.Fatal("hover should not run without a usable box model")
+ return nil
+ }
+
+ if err := HoverElement(context.Background(), 99); err == nil {
+ t.Fatal("expected an error for a short box model")
+ }
+}
diff --git a/internal/mcp/action_payload_contract_test.go b/internal/mcp/action_payload_contract_test.go
index b12aafb2..51d92a4e 100644
--- a/internal/mcp/action_payload_contract_test.go
+++ b/internal/mcp/action_payload_contract_test.go
@@ -101,9 +101,7 @@ var gatedByDeclaringSet = map[string]string{
// postedButUnread records keys a tool forwards that its action does not read — the same
// class as this card's defect, found by this census and filed separately rather than fixed
// here. Each entry names the card, and it reds again the moment the action starts reading.
-var postedButUnread = map[string]string{
- "hover/humanize": "PIN-186: pinchtab_hover forwards humanize but actionHover has no humanized variant, so the flag is ignored",
-}
+var postedButUnread = map[string]string{}
// declaringSets are the per-kind argument gates this handler uses, read from the package
// itself so the census follows them: a key inside `if humanizeAction[kind]` belongs to the