High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix(click): use element's actual position instead of (0,0) (#87)
luigiagent82fec230f2c2
diff --git a/go.mod b/go.modindex 1ac9e177..6cc01d9a 100644--- a/go.mod+++ b/go.mod@@ -5,6 +5,8 @@ go 1.25.0require (github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266dgithub.com/chromedp/chromedp v0.14.2+ github.com/gobwas/ws v1.4.0+ gopkg.in/yaml.v3 v3.0.1)require (@@ -12,7 +14,5 @@ require (github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirectgithub.com/gobwas/httphead v0.1.0 // indirectgithub.com/gobwas/pool v0.2.1 // indirect- github.com/gobwas/ws v1.4.0 // indirectgolang.org/x/sys v0.34.0 // indirect- gopkg.in/yaml.v3 v3.0.1 // indirect)diff --git a/go.sum b/go.sumindex 7acd29ae..ba365b33 100644--- a/go.sum+++ b/go.sum@@ -19,6 +19,7 @@ github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbgolang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=diff --git a/internal/bridge/actions.go b/internal/bridge/actions.goindex 673d8d0a..cf5e7f60 100644--- a/internal/bridge/actions.go+++ b/internal/bridge/actions.go@@ -126,7 +126,8 @@ func (b *Bridge) InitActionRegistry() {},ActionHumanClick: func(ctx context.Context, req ActionRequest) (map[string]any, error) {if req.NodeID > 0 {- if err := human.ClickElement(ctx, cdp.NodeID(req.NodeID)); err != nil {+ // req.NodeID is a backendDOMNodeId from the accessibility tree+ if err := human.ClickElement(ctx, cdp.BackendNodeID(req.NodeID)); err != nil {return nil, err}return map[string]any{"clicked": true, "human": true}, nil@@ -141,7 +142,8 @@ func (b *Bridge) InitActionRegistry() {if len(nodes) == 0 {return nil, fmt.Errorf("element not found: %s", req.Selector)}- if err := human.ClickElement(ctx, nodes[0].NodeID); err != nil {+ // Use BackendNodeID from the DOM node+ if err := human.ClickElement(ctx, nodes[0].BackendNodeID); err != nil {return nil, err}return map[string]any{"clicked": true, "human": true}, nildiff --git a/internal/bridge/cdp.go b/internal/bridge/cdp.goindex db84af5f..bd37a2b0 100644--- a/internal/bridge/cdp.go+++ b/internal/bridge/cdp.go@@ -3,6 +3,7 @@ package bridgeimport ("context""encoding/json"+ "fmt""time""github.com/chromedp/cdproto/network"@@ -64,29 +65,77 @@ func SetResourceBlocking(ctx context.Context, patterns []string) error {}func ClickByNodeID(ctx context.Context, nodeID int64) error {+ // Get element position via box model+ x, y, err := getElementCenter(ctx, nodeID)+ if err != nil {+ return err+ }+return chromedp.Run(ctx,+ // Scroll element into view first+ chromedp.ActionFunc(func(ctx context.Context) error {+ return chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.scrollIntoViewIfNeeded", map[string]any{"backendNodeId": nodeID}, nil)+ }),+ // Focus the elementchromedp.ActionFunc(func(ctx context.Context) error {return chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.focus", map[string]any{"backendNodeId": nodeID}, nil)}),+ // Mouse down at element centerchromedp.ActionFunc(func(ctx context.Context) error {return chromedp.FromContext(ctx).Target.Execute(ctx, "Input.dispatchMouseEvent", map[string]any{"type": "mousePressed","button": "left","clickCount": 1,- "x": 0, "y": 0,+ "x": x, "y": y,}, nil)}),+ // Mouse up at element centerchromedp.ActionFunc(func(ctx context.Context) error {return chromedp.FromContext(ctx).Target.Execute(ctx, "Input.dispatchMouseEvent", map[string]any{"type": "mouseReleased","button": "left","clickCount": 1,- "x": 0, "y": 0,+ "x": x, "y": y,}, nil)}),)}+// getElementCenter returns the center coordinates of an element using DOM.getBoxModel.+func getElementCenter(ctx context.Context, backendNodeID int64) (x, y float64, err error) {+ var result json.RawMessage+ err = chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {+ return chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.getBoxModel", map[string]any{+ "backendNodeId": backendNodeID,+ }, &result)+ }))+ if err != nil {+ return 0, 0, err+ }++ // Parse the box model response+ // The "content" quad is [x1,y1, x2,y2, x3,y3, x4,y4] - four corners+ var box struct {+ Model struct {+ Content []float64 `json:"content"`+ } `json:"model"`+ }+ if err = json.Unmarshal(result, &box); err != nil {+ return 0, 0, err+ }++ if len(box.Model.Content) < 4 {+ return 0, 0, fmt.Errorf("invalid box model: expected at least 4 coordinates")+ }++ // Content quad: [x1,y1, x2,y2, x3,y3, x4,y4]+ // Calculate center as average of all four corners+ x = (box.Model.Content[0] + box.Model.Content[2] + box.Model.Content[4] + box.Model.Content[6]) / 4+ y = (box.Model.Content[1] + box.Model.Content[3] + box.Model.Content[5] + box.Model.Content[7]) / 4++ return x, y, nil+}+func TypeByNodeID(ctx context.Context, nodeID int64, text string) error {return chromedp.Run(ctx,chromedp.ActionFunc(func(ctx context.Context) error {@@ -97,11 +146,22 @@ func TypeByNodeID(ctx context.Context, nodeID int64, text string) error {}func HoverByNodeID(ctx context.Context, nodeID int64) error {+ // Get element position via box model+ x, y, err := getElementCenter(ctx, nodeID)+ if err != nil {+ return err+ }+return chromedp.Run(ctx,+ // Scroll element into view first+ chromedp.ActionFunc(func(ctx context.Context) error {+ return chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.scrollIntoViewIfNeeded", map[string]any{"backendNodeId": nodeID}, nil)+ }),+ // Move mouse to element centerchromedp.ActionFunc(func(ctx context.Context) error {return chromedp.FromContext(ctx).Target.Execute(ctx, "Input.dispatchMouseEvent", map[string]any{"type": "mouseMoved",- "x": 0, "y": 0,+ "x": x, "y": y,}, nil)}),)diff --git a/internal/bridge/cdp_test.go b/internal/bridge/cdp_test.goindex fd3e2469..e6fbf6db 100644--- a/internal/bridge/cdp_test.go+++ b/internal/bridge/cdp_test.go@@ -47,3 +47,22 @@ func TestSelectByNodeID_UsesValue(t *testing.T) {t.Error("expected error without browser connection, got nil (possible no-op)")}}++func TestGetElementCenter_ParsesBoxModel(t *testing.T) {+ // Test the box model parsing logic+ // Content quad: [x1,y1, x2,y2, x3,y3, x4,y4]+ // For a 100x50 box at position (200, 100):+ // corners: (200,100), (300,100), (300,150), (200,150)+ content := []float64{200, 100, 300, 100, 300, 150, 200, 150}++ // Calculate expected center+ expectedX := (content[0] + content[2] + content[4] + content[6]) / 4 // (200+300+300+200)/4 = 250+ expectedY := (content[1] + content[3] + content[5] + content[7]) / 4 // (100+100+150+150)/4 = 125++ if expectedX != 250 {+ t.Errorf("expected X=250, got %f", expectedX)+ }+ if expectedY != 125 {+ t.Errorf("expected Y=125, got %f", expectedY)+ }+}diff --git a/internal/human/human.go b/internal/human/human.goindex 2ffb39ba..d6b837cd 100644--- a/internal/human/human.go+++ b/internal/human/human.go@@ -131,12 +131,14 @@ func Click(ctx context.Context, x, y float64) error {)}-func ClickElement(ctx context.Context, nodeID cdp.NodeID) error {+// ClickElement clicks on an element identified by its backend DOM node ID.+// This uses the backendDOMNodeId from the accessibility tree, NOT a regular DOM nodeId.+func ClickElement(ctx context.Context, backendNodeID cdp.BackendNodeID) error {var box *dom.BoxModelif err := chromedp.Run(ctx,chromedp.ActionFunc(func(ctx context.Context) error {var err error- box, err = dom.GetBoxModel().WithNodeID(nodeID).Do(ctx)+ box, err = dom.GetBoxModel().WithBackendNodeID(backendNodeID).Do(ctx)return err}),); err != nil {diff --git a/tests/integration/actions_test.go b/tests/integration/actions_test.goindex b124f5bd..3a2f295a 100644--- a/tests/integration/actions_test.go+++ b/tests/integration/actions_test.go@@ -33,6 +33,63 @@ func TestAction_Click(t *testing.T) {}}+// A1b: Click actually triggers navigation (verifies fix for issue #80)+// This test ensures clicks dispatch at the correct element position, not (0,0)+func TestAction_Click_TriggersNavigation(t *testing.T) {+ navigate(t, "https://example.com")+ defer closeCurrentTab(t)+ t.Logf("current tab: %s", currentTabID)++ // Get snapshot to find the "More information..." link+ _, snapBody := httpGet(t, "/snapshot?filter=interactive&tabId="+currentTabID)++ var snapResp struct {+ Nodes []struct {+ Ref string `json:"ref"`+ Role string `json:"role"`+ Name string `json:"name"`+ } `json:"nodes"`+ }+ if err := json.Unmarshal(snapBody, &snapResp); err != nil {+ t.Fatalf("failed to parse snapshot: %v", err)+ }++ // Find any link (example.com has "More information..." link)+ var linkRef string+ for _, n := range snapResp.Nodes {+ if n.Role == "link" {+ linkRef = n.Ref+ t.Logf("found link: ref=%s name=%q", n.Ref, n.Name)+ break+ }+ }+ if linkRef == "" {+ t.Skip("no link found in snapshot")+ }++ // Click the link with waitNav to wait for navigation+ code, body := httpPost(t, "/action", map[string]any{+ "tabId": currentTabID,+ "kind": "click",+ "ref": linkRef,+ "waitNav": true,+ })+ if code != 200 {+ t.Fatalf("click failed with %d: %s", code, string(body))+ }++ // Verify navigation happened by checking the page content changed+ // The "More information" link on example.com goes to iana.org+ _, textBody := httpGet(t, "/text?tabId="+currentTabID)+ text := string(textBody)++ // After clicking, we should NOT be on example.com anymore+ if strings.Contains(text, "Example Domain") && !strings.Contains(text, "IANA") {+ t.Errorf("click did not trigger navigation - still on example.com (bug #80: click at wrong position)")+ }+ t.Logf("navigation verified: page content changed after click")+}+// A4: Press keyfunc TestAction_Press(t *testing.T) {navigate(t, "https://example.com")