High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix: pinchtab_hover forwards a humanize flag actionHover never reads… (PIN-186)
Luigi Agostiac499e3add06
diff --git a/internal/bridge/action_pointer.go b/internal/bridge/action_pointer.goindex de99b1e4..5616fede 100644--- a/internal/bridge/action_pointer.go+++ b/internal/bridge/action_pointer.go@@ -19,6 +19,8 @@ var mouseDownByCoordinateAction = MouseDownByCoordinatevar mouseUpByCoordinateAction = MouseUpByCoordinatevar clickByCoordinateAction = ClickByCoordinatevar clickElementAction = ClickElement+var hoverElementAction = HoverElement+var hoverCoordinateAction = Hovervar clickByNodeIDAction = ClickByNodeIDvar jsClickByBackendNodeAction = JSClickByBackendNodevar dispatchClickByBackendNodeAction = JSDispatchClickByBackendNode@@ -425,6 +427,9 @@ func (b *Bridge) actionDoubleClick(ctx context.Context, req ActionRequest) (resu}func (b *Bridge) actionHover(ctx context.Context, req ActionRequest) (map[string]any, error) {+ if b.effectiveHumanize(req) {+ return b.actionHumanizedHover(ctx, req)+ }if req.NodeID > 0 {return map[string]any{"hovered": true}, HoverByNodeID(ctx, req.NodeID)}@@ -441,6 +446,28 @@ func (b *Bridge) actionHover(ctx context.Context, req ActionRequest) (map[stringreturn nil, fmt.Errorf("need selector, ref, nodeId, or x/y coordinates")}+func (b *Bridge) actionHumanizedHover(ctx context.Context, req ActionRequest) (map[string]any, error) {+ var err error+ switch {+ case req.NodeID > 0:+ err = hoverElementAction(ctx, cdp.BackendNodeID(req.NodeID))+ case req.Selector != "":+ node, nodeErr := firstNodeBySelector(ctx, req.Selector)+ if nodeErr != nil {+ return nil, nodeErr+ }+ err = hoverElementAction(ctx, node.BackendNodeID)+ case req.HasXY:+ err = hoverCoordinateAction(ctx, req.X, req.Y)+ default:+ return nil, fmt.Errorf("need selector, ref, nodeId, or x/y coordinates")+ }+ if err != nil {+ return nil, err+ }+ return map[string]any{"hovered": true, "human": true}, nil+}+func (b *Bridge) rememberPointerPosition(tabID string, x, y float64) {if b == nil || tabID == "" {returndiff --git a/internal/bridge/actions_test.go b/internal/bridge/actions_test.goindex 82bb88d3..6b5a57c0 100644--- a/internal/bridge/actions_test.go+++ b/internal/bridge/actions_test.go@@ -2,13 +2,17 @@ package bridgeimport ("context"+ "encoding/base64""encoding/json""errors""strings""testing"+ "time""github.com/chromedp/cdproto/cdp"+ "github.com/chromedp/chromedp""github.com/pinchtab/pinchtab/internal/config"+ "github.com/pinchtab/pinchtab/internal/testbrowser")func TestClickAction_UsesCoordinatePathIncludingZeroZero(t *testing.T) {@@ -1169,3 +1173,140 @@ func TestScrollIntoViewAction_WithSelector_UsesCSSPath(t *testing.T) {t.Fatalf("expected CSS path, got validation error: %v", err)}}++func TestHoverAction_HumanizeRoutesEveryTargetForm(t *testing.T) {+ origElement := hoverElementAction+ origCoordinate := hoverCoordinateAction+ t.Cleanup(func() {+ hoverElementAction = origElement+ hoverCoordinateAction = origCoordinate+ })++ var nodes []cdp.BackendNodeID+ var points [][2]float64+ hoverElementAction = func(ctx context.Context, backendNodeID cdp.BackendNodeID) error {+ nodes = append(nodes, backendNodeID)+ return nil+ }+ hoverCoordinateAction = func(ctx context.Context, x, y float64) error {+ points = append(points, [2]float64{x, y})+ return nil+ }++ tests := []struct {+ name string+ config *config.RuntimeConfig+ req ActionRequest+ wantNodes []cdp.BackendNodeID+ wantPoints [][2]float64+ wantHuman bool+ }{+ {+ name: "nodeId with request opt-in",+ config: &config.RuntimeConfig{},+ req: ActionRequest{NodeID: 77, Humanize: boolPtr(true)},+ wantNodes: []cdp.BackendNodeID{77},+ wantHuman: true,+ },+ {+ name: "coordinates with instance default",+ config: &config.RuntimeConfig{Humanize: true},+ req: ActionRequest{HasXY: true, X: 12, Y: 34},+ wantPoints: [][2]float64{{12, 34}},+ wantHuman: true,+ },+ {+ name: "explicit false opts out of the humanized path",+ config: &config.RuntimeConfig{Humanize: true},+ req: ActionRequest{NodeID: 77, Humanize: boolPtr(false)},+ wantHuman: false,+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ nodes, points = nil, nil+ b := New(context.TODO(), nil, tc.config)++ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ res, err := b.Actions[ActionHover](ctx, tc.req)++ if tc.wantHuman {+ if err != nil {+ t.Fatalf("humanized hover returned error: %v", err)+ }+ if hovered, _ := res["hovered"].(bool); !hovered {+ t.Fatalf("result = %#v, want hovered=true", res)+ }+ if human, _ := res["human"].(bool); !human {+ t.Fatalf("result = %#v, want human=true", res)+ }+ } else if _, ok := res["human"]; ok {+ t.Fatalf("raw hover result = %#v, want no human key", res)+ }++ if len(nodes) != len(tc.wantNodes) || (len(nodes) == 1 && nodes[0] != tc.wantNodes[0]) {+ t.Fatalf("element hovers = %v, want %v", nodes, tc.wantNodes)+ }+ if len(points) != len(tc.wantPoints) || (len(points) == 1 && points[0] != tc.wantPoints[0]) {+ t.Fatalf("coordinate hovers = %v, want %v", points, tc.wantPoints)+ }+ })+ }+}++func TestHoverAction_HumanizedStillRequiresATarget(t *testing.T) {+ b := New(context.TODO(), nil, &config.RuntimeConfig{Humanize: true})++ _, err := b.Actions[ActionHover](context.Background(), ActionRequest{})+ if err == nil || !strings.Contains(err.Error(), "need selector") {+ t.Fatalf("targetless humanized hover error = %v, want the target requirement", err)+ }+}++// Browser-backed: the selector form resolves through firstNodeBySelector, which+// has no browserless seam, so this is where selector routing and the real+// trail landing on the target are pinned.+func TestHoverAction_HumanizedSelectorLandsOnTheTarget(t *testing.T) {+ chromePath := testbrowser.Path(t)+ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(+ chromedp.DefaultExecAllocatorOptions[:],+ chromedp.ExecPath(chromePath),+ chromedp.Flag("headless", true),+ chromedp.Flag("no-sandbox", true),+ )...)+ defer cancelAlloc()+ ctx, cancel := chromedp.NewContext(alloc)+ defer cancel()+ ctx, cancelTimeout := context.WithTimeout(ctx, 30*time.Second)+ defer cancelTimeout()++ html := `<style>#target { position: absolute; top: 100px; left: 100px; width: 120px; height: 40px; }</style>+ <div id="target">hover me</div>+ <script>+ window.hovered = false;+ document.getElementById("target").addEventListener("mouseover", () => window.hovered = true);+ </script>`+ dataURL := "data:text/html;base64," + base64.StdEncoding.EncodeToString([]byte(html))+ if err := chromedp.Run(ctx, chromedp.Navigate(dataURL)); err != nil {+ t.Fatal(err)+ }++ b := New(context.Background(), nil, &config.RuntimeConfig{Humanize: true})+ res, err := b.Actions[ActionHover](ctx, ActionRequest{Selector: "#target"})+ if err != nil {+ t.Fatalf("humanized selector hover: %v", err)+ }+ if human, _ := res["human"].(bool); !human {+ t.Fatalf("result = %#v, want human=true", res)+ }++ var hovered bool+ if err := chromedp.Run(ctx, chromedp.Evaluate(`window.hovered`, &hovered)); err != nil {+ t.Fatal(err)+ }+ if !hovered {+ t.Fatal("humanized hover did not fire mouseover on the selector target")+ }+}diff --git a/internal/bridge/input_human.go b/internal/bridge/input_human.goindex 9425e188..3e491c14 100644--- a/internal/bridge/input_human.go+++ b/internal/bridge/input_human.go@@ -124,29 +124,41 @@ func MouseMove(ctx context.Context, fromX, fromY, toX, toY float64) error {return nil}-func Click(ctx context.Context, x, y float64) error {+// approachTarget walks the pointer to (x, y) from a random nearby start along+// the bezier trail. Best-effort: the trail is only there for human-trail+// realism, so a stall is logged and swallowed and the caller proceeds — its own+// dispatch at (x, y) is what has to land. Only a cancelled outer context is+// reported back.+func approachTarget(ctx context.Context, x, y float64) error {startOffsetX := (humanRand.Float64()-0.5)*200 + 50startOffsetY := (humanRand.Float64()-0.5)*200 + 50startX := x + startOffsetXstartY := y + startOffsetYdistance := math.Sqrt(startOffsetX*startOffsetX + startOffsetY*startOffsetY)- if distance > 30 {- // Best-effort: the initial random-start move and bezier trail are- // only there for human-trail realism. If they stall, log and- // proceed — the subsequent MousePressed sets the cursor at (x,y)- // explicitly, so the click still lands.- if err := dispatchMouseMovedBounded(ctx, startX, startY); err != nil {- if ctx.Err() != nil {- return ctx.Err()- }- slog.Debug("humanized click initial-move stalled, skipping bezier", "err", err)- } else if err := MouseMove(ctx, startX, startY, x, y); err != nil {- if ctx.Err() != nil {- return ctx.Err()- }- slog.Debug("humanized click bezier trail failed, proceeding to click", "err", err)+ if distance <= 30 {+ return nil+ }++ if err := dispatchMouseMovedBounded(ctx, startX, startY); err != nil {+ if ctx.Err() != nil {+ return ctx.Err()+ }+ slog.Debug("humanized initial-move stalled, skipping bezier", "err", err)+ } else if err := MouseMove(ctx, startX, startY, x, y); err != nil {+ if ctx.Err() != nil {+ return ctx.Err()}+ slog.Debug("humanized bezier trail failed, proceeding to action", "err", err)+ }+ return nil+}++var approachTargetAction = approachTarget++func Click(ctx context.Context, x, y float64) error {+ if err := approachTargetAction(ctx, x, y); err != nil {+ return err}time.Sleep(time.Duration(50+humanRand.Intn(150)) * time.Millisecond)@@ -195,25 +207,38 @@ var boxModelForBackendNodeAction = func(ctx context.Context, backendNodeID cdp.Breturn box, err}-var clickCoordinateHumanAction = Click+var (+ clickCoordinateHumanAction = Click+ hoverCoordinateHumanAction = Hover+ settleHoverAction = HoverByCoordinate+)-// The ID is the backendDOMNodeId from the accessibility tree, NOT a regular DOM nodeId.-func ClickElement(ctx context.Context, backendNodeID cdp.BackendNodeID) error {- // Scroll the target into the visual viewport first. The humanized click is- // coordinate-based (Input.dispatchMouseEvent at the box-model center), so an- // element below the fold or freshly revealed (lazy-loaded list item,- // modal-triggered button) would otherwise produce coordinates that land on- // whatever is currently visible there — the dispatch "succeeds" but the- // intended click handler never fires. Best-effort: not every node scrolls.+// Hover is the humanized sibling of Click: the same random-start bezier+// approach, then the plain hover dispatch instead of press/release.+func Hover(ctx context.Context, x, y float64) error {+ if err := approachTargetAction(ctx, x, y); err != nil {+ return err+ }+ return settleHoverAction(ctx, x, y)+}++// humanPointerPoint resolves the coordinate a humanized pointer action aims at.+// It scrolls the target into the visual viewport first: the humanized path is+// coordinate-based (Input.dispatchMouseEvent at the box-model center), so an+// element below the fold or freshly revealed (lazy-loaded list item,+// modal-triggered button) would otherwise produce coordinates that land on+// whatever is currently visible there — the dispatch "succeeds" but the+// intended handler never fires. Best-effort: not every node scrolls.+func humanPointerPoint(ctx context.Context, backendNodeID cdp.BackendNodeID) (float64, float64, error) {_ = scrollIntoViewIfNeededAction(ctx, backendNodeID)box, err := boxModelForBackendNodeAction(ctx, backendNodeID)if err != nil {- return err+ return 0, 0, err}if len(box.Content) < 8 {- return fmt.Errorf("invalid box model")+ return 0, 0, fmt.Errorf("invalid box model")}centerX := (box.Content[0] + box.Content[2]) / 2@@ -222,7 +247,25 @@ func ClickElement(ctx context.Context, backendNodeID cdp.BackendNodeID) error {offsetX := (humanRand.Float64() - 0.5) * 10offsetY := (humanRand.Float64() - 0.5) * 10- return clickCoordinateHumanAction(ctx, centerX+offsetX, centerY+offsetY)+ return centerX + offsetX, centerY + offsetY, nil+}++// The ID is the backendDOMNodeId from the accessibility tree, NOT a regular DOM nodeId.+func ClickElement(ctx context.Context, backendNodeID cdp.BackendNodeID) error {+ x, y, err := humanPointerPoint(ctx, backendNodeID)+ if err != nil {+ return err+ }+ return clickCoordinateHumanAction(ctx, x, y)+}++// The ID is the backendDOMNodeId from the accessibility tree, NOT a regular DOM nodeId.+func HoverElement(ctx context.Context, backendNodeID cdp.BackendNodeID) error {+ x, y, err := humanPointerPoint(ctx, backendNodeID)+ if err != nil {+ return err+ }+ return hoverCoordinateHumanAction(ctx, x, y)}func Type(text string, fast bool) []chromedp.Action {diff --git a/internal/bridge/input_human_test.go b/internal/bridge/input_human_test.goindex 463a24f7..29574e77 100644--- a/internal/bridge/input_human_test.go+++ b/internal/bridge/input_human_test.go@@ -127,3 +127,122 @@ func TestClickElement_ScrollsBeforeReadingBoxModel(t *testing.T) {t.Fatalf("calls = %v, want [scroll box]", calls)}}++func TestHover_TrailIsBestEffort(t *testing.T) {+ origSettle := settleHoverAction+ t.Cleanup(func() { settleHoverAction = origSettle })++ SetHumanRandSeed(7)++ var settled [][2]float64+ settleHoverAction = func(ctx context.Context, x, y float64) error {+ settled = append(settled, [2]float64{x, y})+ return nil+ }++ // No browser attached, so every trail dispatch fails; the hover must still land.+ if err := Hover(context.Background(), 40, 60); err != nil {+ t.Fatalf("Hover with a failing trail returned %v, want nil", err)+ }+ if len(settled) != 1 || settled[0] != [2]float64{40, 60} {+ t.Fatalf("settled hovers = %v, want one at (40,60)", settled)+ }+}++func TestHover_CancelledContextStopsBeforeSettling(t *testing.T) {+ origSettle := settleHoverAction+ t.Cleanup(func() { settleHoverAction = origSettle })++ SetHumanRandSeed(7)++ settleHoverAction = func(ctx context.Context, x, y float64) error {+ t.Fatal("cancelled hover should not reach the settle dispatch")+ return nil+ }++ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ if err := Hover(ctx, 40, 60); !errors.Is(err, context.Canceled) {+ t.Fatalf("Hover error = %v, want context.Canceled", err)+ }+}++func TestHoverElement_ReusesTheClickTargetPoint(t *testing.T) {+ origScroll := scrollIntoViewIfNeededAction+ origBoxModel := boxModelForBackendNodeAction+ origHover := hoverCoordinateHumanAction+ origClick := clickCoordinateHumanAction+ t.Cleanup(func() {+ scrollIntoViewIfNeededAction = origScroll+ boxModelForBackendNodeAction = origBoxModel+ hoverCoordinateHumanAction = origHover+ clickCoordinateHumanAction = origClick+ })++ var calls []string+ scrollIntoViewIfNeededAction = func(ctx context.Context, backendNodeID cdp.BackendNodeID) error {+ calls = append(calls, "scroll")+ return errors.New("scroll failed but should be best-effort")+ }+ boxModelForBackendNodeAction = func(ctx context.Context, backendNodeID cdp.BackendNodeID) (*dom.BoxModel, error) {+ if backendNodeID != 99 {+ t.Fatalf("box backendNodeID = %d, want 99", backendNodeID)+ }+ calls = append(calls, "box")+ return &dom.BoxModel{Content: []float64{100, 200, 300, 200, 300, 260, 100, 260}}, nil+ }++ var hoverX, hoverY float64+ hoverCoordinateHumanAction = func(ctx context.Context, x, y float64) error {+ calls = append(calls, "hover")+ hoverX, hoverY = x, y+ return nil+ }+ var clickX, clickY float64+ clickCoordinateHumanAction = func(ctx context.Context, x, y float64) error {+ clickX, clickY = x, y+ return nil+ }++ SetHumanRandSeed(11)+ if err := HoverElement(context.Background(), 99); err != nil {+ t.Fatalf("HoverElement returned %v", err)+ }+ hoverCalls := append([]string(nil), calls...)++ SetHumanRandSeed(11)+ if err := ClickElement(context.Background(), 99); err != nil {+ t.Fatalf("ClickElement returned %v", err)+ }++ if hoverX != clickX || hoverY != clickY {+ t.Fatalf("hover point (%v,%v) != click point (%v,%v); the two humanized paths must aim at the same box-model point", hoverX, hoverY, clickX, clickY)+ }+ if hoverX < 100 || hoverX > 300 || hoverY < 200 || hoverY > 260 {+ t.Fatalf("hover point (%v,%v) is outside the box model", hoverX, hoverY)+ }+ if len(hoverCalls) != 3 || hoverCalls[0] != "scroll" || hoverCalls[1] != "box" || hoverCalls[2] != "hover" {+ t.Fatalf("calls = %v, want [scroll box hover]", hoverCalls)+ }+}++func TestHoverElement_PropagatesBoxModelFailure(t *testing.T) {+ origBoxModel := boxModelForBackendNodeAction+ origHover := hoverCoordinateHumanAction+ t.Cleanup(func() {+ boxModelForBackendNodeAction = origBoxModel+ hoverCoordinateHumanAction = origHover+ })++ boxModelForBackendNodeAction = func(ctx context.Context, backendNodeID cdp.BackendNodeID) (*dom.BoxModel, error) {+ return &dom.BoxModel{Content: []float64{1, 2}}, nil+ }+ hoverCoordinateHumanAction = func(ctx context.Context, x, y float64) error {+ t.Fatal("hover should not run without a usable box model")+ return nil+ }++ if err := HoverElement(context.Background(), 99); err == nil {+ t.Fatal("expected an error for a short box model")+ }+}diff --git a/internal/mcp/action_payload_contract_test.go b/internal/mcp/action_payload_contract_test.goindex b12aafb2..51d92a4e 100644--- a/internal/mcp/action_payload_contract_test.go+++ b/internal/mcp/action_payload_contract_test.go@@ -101,9 +101,7 @@ var gatedByDeclaringSet = map[string]string{// postedButUnread records keys a tool forwards that its action does not read — the same// class as this card's defect, found by this census and filed separately rather than fixed// here. Each entry names the card, and it reds again the moment the action starts reading.-var postedButUnread = map[string]string{- "hover/humanize": "PIN-186: pinchtab_hover forwards humanize but actionHover has no humanized variant, so the flag is ignored",-}+var postedButUnread = map[string]string{}// declaringSets are the per-kind argument gates this handler uses, read from the package// itself so the census follows them: a key inside `if humanizeAction[kind]` belongs to the