Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix(click): use element's actual position instead of (0,0) (#87)

luigiagent82fec230f2c2
diff --git a/go.mod b/go.mod
index 1ac9e177..6cc01d9a 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,8 @@ go 1.25.0
require (
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d
github.com/chromedp/chromedp v0.14.2
+ github.com/gobwas/ws v1.4.0
+ gopkg.in/yaml.v3 v3.0.1
)
require (
@@ -12,7 +14,5 @@ require (
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
- github.com/gobwas/ws v1.4.0 // indirect
golang.org/x/sys v0.34.0 // indirect
- gopkg.in/yaml.v3 v3.0.1 // indirect
)
diff --git a/go.sum b/go.sum
index 7acd29ae..ba365b33 100644
--- a/go.sum
+++ b/go.sum
@@ -19,6 +19,7 @@ github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzb
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/internal/bridge/actions.go b/internal/bridge/actions.go
index 673d8d0a..cf5e7f60 100644
--- a/internal/bridge/actions.go
+++ b/internal/bridge/actions.go
@@ -126,7 +126,8 @@ func (b *Bridge) InitActionRegistry() {
},
ActionHumanClick: func(ctx context.Context, req ActionRequest) (map[string]any, error) {
if req.NodeID > 0 {
- if err := human.ClickElement(ctx, cdp.NodeID(req.NodeID)); err != nil {
+ // req.NodeID is a backendDOMNodeId from the accessibility tree
+ if err := human.ClickElement(ctx, cdp.BackendNodeID(req.NodeID)); err != nil {
return nil, err
}
return map[string]any{"clicked": true, "human": true}, nil
@@ -141,7 +142,8 @@ func (b *Bridge) InitActionRegistry() {
if len(nodes) == 0 {
return nil, fmt.Errorf("element not found: %s", req.Selector)
}
- if err := human.ClickElement(ctx, nodes[0].NodeID); err != nil {
+ // Use BackendNodeID from the DOM node
+ if err := human.ClickElement(ctx, nodes[0].BackendNodeID); err != nil {
return nil, err
}
return map[string]any{"clicked": true, "human": true}, nil
diff --git a/internal/bridge/cdp.go b/internal/bridge/cdp.go
index db84af5f..bd37a2b0 100644
--- a/internal/bridge/cdp.go
+++ b/internal/bridge/cdp.go
@@ -3,6 +3,7 @@ package bridge
import (
"context"
"encoding/json"
+ "fmt"
"time"
"github.com/chromedp/cdproto/network"
@@ -64,29 +65,77 @@ func SetResourceBlocking(ctx context.Context, patterns []string) error {
}
func ClickByNodeID(ctx context.Context, nodeID int64) error {
+ // Get element position via box model
+ x, y, err := getElementCenter(ctx, nodeID)
+ if err != nil {
+ return err
+ }
+
return chromedp.Run(ctx,
+ // Scroll element into view first
+ chromedp.ActionFunc(func(ctx context.Context) error {
+ return chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.scrollIntoViewIfNeeded", map[string]any{"backendNodeId": nodeID}, nil)
+ }),
+ // Focus the element
chromedp.ActionFunc(func(ctx context.Context) error {
return chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.focus", map[string]any{"backendNodeId": nodeID}, nil)
}),
+ // Mouse down at element center
chromedp.ActionFunc(func(ctx context.Context) error {
return chromedp.FromContext(ctx).Target.Execute(ctx, "Input.dispatchMouseEvent", map[string]any{
"type": "mousePressed",
"button": "left",
"clickCount": 1,
- "x": 0, "y": 0,
+ "x": x, "y": y,
}, nil)
}),
+ // Mouse up at element center
chromedp.ActionFunc(func(ctx context.Context) error {
return chromedp.FromContext(ctx).Target.Execute(ctx, "Input.dispatchMouseEvent", map[string]any{
"type": "mouseReleased",
"button": "left",
"clickCount": 1,
- "x": 0, "y": 0,
+ "x": x, "y": y,
}, nil)
}),
)
}
+// getElementCenter returns the center coordinates of an element using DOM.getBoxModel.
+func getElementCenter(ctx context.Context, backendNodeID int64) (x, y float64, err error) {
+ var result json.RawMessage
+ err = chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
+ return chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.getBoxModel", map[string]any{
+ "backendNodeId": backendNodeID,
+ }, &result)
+ }))
+ if err != nil {
+ return 0, 0, err
+ }
+
+ // Parse the box model response
+ // The "content" quad is [x1,y1, x2,y2, x3,y3, x4,y4] - four corners
+ var box struct {
+ Model struct {
+ Content []float64 `json:"content"`
+ } `json:"model"`
+ }
+ if err = json.Unmarshal(result, &box); err != nil {
+ return 0, 0, err
+ }
+
+ if len(box.Model.Content) < 4 {
+ return 0, 0, fmt.Errorf("invalid box model: expected at least 4 coordinates")
+ }
+
+ // Content quad: [x1,y1, x2,y2, x3,y3, x4,y4]
+ // Calculate center as average of all four corners
+ x = (box.Model.Content[0] + box.Model.Content[2] + box.Model.Content[4] + box.Model.Content[6]) / 4
+ y = (box.Model.Content[1] + box.Model.Content[3] + box.Model.Content[5] + box.Model.Content[7]) / 4
+
+ return x, y, nil
+}
+
func TypeByNodeID(ctx context.Context, nodeID int64, text string) error {
return chromedp.Run(ctx,
chromedp.ActionFunc(func(ctx context.Context) error {
@@ -97,11 +146,22 @@ func TypeByNodeID(ctx context.Context, nodeID int64, text string) error {
}
func HoverByNodeID(ctx context.Context, nodeID int64) error {
+ // Get element position via box model
+ x, y, err := getElementCenter(ctx, nodeID)
+ if err != nil {
+ return err
+ }
+
return chromedp.Run(ctx,
+ // Scroll element into view first
+ chromedp.ActionFunc(func(ctx context.Context) error {
+ return chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.scrollIntoViewIfNeeded", map[string]any{"backendNodeId": nodeID}, nil)
+ }),
+ // Move mouse to element center
chromedp.ActionFunc(func(ctx context.Context) error {
return chromedp.FromContext(ctx).Target.Execute(ctx, "Input.dispatchMouseEvent", map[string]any{
"type": "mouseMoved",
- "x": 0, "y": 0,
+ "x": x, "y": y,
}, nil)
}),
)
diff --git a/internal/bridge/cdp_test.go b/internal/bridge/cdp_test.go
index fd3e2469..e6fbf6db 100644
--- a/internal/bridge/cdp_test.go
+++ b/internal/bridge/cdp_test.go
@@ -47,3 +47,22 @@ func TestSelectByNodeID_UsesValue(t *testing.T) {
t.Error("expected error without browser connection, got nil (possible no-op)")
}
}
+
+func TestGetElementCenter_ParsesBoxModel(t *testing.T) {
+ // Test the box model parsing logic
+ // Content quad: [x1,y1, x2,y2, x3,y3, x4,y4]
+ // For a 100x50 box at position (200, 100):
+ // corners: (200,100), (300,100), (300,150), (200,150)
+ content := []float64{200, 100, 300, 100, 300, 150, 200, 150}
+
+ // Calculate expected center
+ expectedX := (content[0] + content[2] + content[4] + content[6]) / 4 // (200+300+300+200)/4 = 250
+ expectedY := (content[1] + content[3] + content[5] + content[7]) / 4 // (100+100+150+150)/4 = 125
+
+ if expectedX != 250 {
+ t.Errorf("expected X=250, got %f", expectedX)
+ }
+ if expectedY != 125 {
+ t.Errorf("expected Y=125, got %f", expectedY)
+ }
+}
diff --git a/internal/human/human.go b/internal/human/human.go
index 2ffb39ba..d6b837cd 100644
--- a/internal/human/human.go
+++ b/internal/human/human.go
@@ -131,12 +131,14 @@ func Click(ctx context.Context, x, y float64) error {
)
}
-func ClickElement(ctx context.Context, nodeID cdp.NodeID) error {
+// ClickElement clicks on an element identified by its backend DOM node ID.
+// This uses the backendDOMNodeId from the accessibility tree, NOT a regular DOM nodeId.
+func ClickElement(ctx context.Context, backendNodeID cdp.BackendNodeID) error {
var box *dom.BoxModel
if err := chromedp.Run(ctx,
chromedp.ActionFunc(func(ctx context.Context) error {
var err error
- box, err = dom.GetBoxModel().WithNodeID(nodeID).Do(ctx)
+ box, err = dom.GetBoxModel().WithBackendNodeID(backendNodeID).Do(ctx)
return err
}),
); err != nil {
diff --git a/tests/integration/actions_test.go b/tests/integration/actions_test.go
index b124f5bd..3a2f295a 100644
--- a/tests/integration/actions_test.go
+++ b/tests/integration/actions_test.go
@@ -33,6 +33,63 @@ func TestAction_Click(t *testing.T) {
}
}
+// A1b: Click actually triggers navigation (verifies fix for issue #80)
+// This test ensures clicks dispatch at the correct element position, not (0,0)
+func TestAction_Click_TriggersNavigation(t *testing.T) {
+ navigate(t, "https://example.com")
+ defer closeCurrentTab(t)
+ t.Logf("current tab: %s", currentTabID)
+
+ // Get snapshot to find the "More information..." link
+ _, snapBody := httpGet(t, "/snapshot?filter=interactive&tabId="+currentTabID)
+
+ var snapResp struct {
+ Nodes []struct {
+ Ref string `json:"ref"`
+ Role string `json:"role"`
+ Name string `json:"name"`
+ } `json:"nodes"`
+ }
+ if err := json.Unmarshal(snapBody, &snapResp); err != nil {
+ t.Fatalf("failed to parse snapshot: %v", err)
+ }
+
+ // Find any link (example.com has "More information..." link)
+ var linkRef string
+ for _, n := range snapResp.Nodes {
+ if n.Role == "link" {
+ linkRef = n.Ref
+ t.Logf("found link: ref=%s name=%q", n.Ref, n.Name)
+ break
+ }
+ }
+ if linkRef == "" {
+ t.Skip("no link found in snapshot")
+ }
+
+ // Click the link with waitNav to wait for navigation
+ code, body := httpPost(t, "/action", map[string]any{
+ "tabId": currentTabID,
+ "kind": "click",
+ "ref": linkRef,
+ "waitNav": true,
+ })
+ if code != 200 {
+ t.Fatalf("click failed with %d: %s", code, string(body))
+ }
+
+ // Verify navigation happened by checking the page content changed
+ // The "More information" link on example.com goes to iana.org
+ _, textBody := httpGet(t, "/text?tabId="+currentTabID)
+ text := string(textBody)
+
+ // After clicking, we should NOT be on example.com anymore
+ if strings.Contains(text, "Example Domain") && !strings.Contains(text, "IANA") {
+ t.Errorf("click did not trigger navigation - still on example.com (bug #80: click at wrong position)")
+ }
+ t.Logf("navigation verified: page content changed after click")
+}
+
// A4: Press key
func TestAction_Press(t *testing.T) {
navigate(t, "https://example.com")