Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

P5: hover/select/scroll actions, smart diff, text format, withElement helper, 21 tests

bot1b2f2dc31a2c
diff --git a/TODO.md b/TODO.md
index d9be21bb..294ba9e8 100644
--- a/TODO.md
+++ b/TODO.md
@@ -61,19 +61,23 @@ Split into 8 files (single package):
- [x] Add config tests — envOr, homeDir, constants
- [x] **18 tests passing** — `go test ./...`
-## P5: Bugs & Stealth
-- [ ] **Remove `--disable-blink-features=AutomationControlled` flag** — deprecated in Chrome 144+, triggers warning banner in headed mode, hurts stealth. JS injection already patches `navigator.webdriver`
+## Done ✅ (P5)
+- [x] **`withElement` helper** — generic `withElement(ctx, nodeID, jsFunc)` + `withElementArg` for all element actions
+- [x] **`hover` action** — mouseover/mouseenter via ref, selector
+- [x] **`select` action** — pick `<select>` option by value or text, fires change event
+- [x] **`scroll` action** — scroll to element (ref), by pixel amount (`scrollX`/`scrollY`), or default 800px down
+- [x] **Smart diff** — `?diff=true` returns added/changed/removed nodes since last snapshot
+- [x] **Wait for navigation** — `"waitNav": true` on click action, 1s delay for page load
+- [x] **Text format** — `?format=text` returns indented tree (~40-60% fewer tokens than JSON)
+- [x] **21 tests passing** — diff, text format, all existing tests green
+
+## P5 Remaining
-## P6: Features
-- [ ] **`/scroll` endpoint** — scroll to element or by amount. Needed for infinite-scroll pages (X, Reddit).
-- [ ] **`withElement` helper** — generic `withElement(ctx, nodeID, jsFunc)` for all element actions (click, type, scroll, hover, drag)
-- [ ] **Smart diff** — `?diff=true` returns only changes since last snapshot. Massive token savings on multi-step tasks
-- [ ] **Wait for navigation** — after click, wait for page load before returning
+## P5 Bugs & Stealth
+- [ ] **Remove `--disable-blink-features=AutomationControlled` flag** — deprecated in Chrome 144+, triggers warning banner in headed mode, hurts stealth. JS injection already patches `navigator.webdriver`
- [ ] **Better /text** — Readability-style extraction instead of raw innerText
- [ ] **Split handlers.go** — snapshot handler is complex enough for its own file
- [ ] **Wire interfaces into handlers** — Inject Browser/TabManager so handler tests don't need Chrome
-- [ ] **`hover` action** — mouseover for dropdowns/tooltips (common on X, Google)
-- [ ] **`select` action** — pick option from `<select>` dropdowns by value or text
## P7: Nice to Have
- [ ] **File-based output** — `?output=file` saves snapshot to disk, returns path (Playwright CLI approach)
@@ -83,7 +87,6 @@ Split into 8 files (single package):
- [ ] **Config file** — `~/.pinchtab/config.json`
- [ ] **LaunchAgent/systemd** — auto-start on boot
- [ ] **`/cookies` endpoint** — read/set cookies (useful for auth debugging)
-- [ ] **Snapshot text format** — `?format=text` returns indented tree instead of JSON (even cheaper tokens)
## Not Doing
- Plugin system
diff --git a/bridge.go b/bridge.go
index f0e7a5fe..bed67ea1 100644
--- a/bridge.go
+++ b/bridge.go
@@ -17,11 +17,12 @@ type TabEntry struct {
cancel context.CancelFunc
}
-// refCache stores the ref→backendNodeID mapping from the last snapshot per tab.
+// refCache stores the ref→backendNodeID mapping and the last snapshot nodes per tab.
// Refs are assigned during /snapshot and looked up during /action, avoiding
// a second a11y tree fetch that could drift.
type refCache struct {
- refs map[string]int64 // "e0" → backendNodeID
+ refs map[string]int64 // "e0" → backendNodeID
+ nodes []A11yNode // last snapshot for diff
}
// Bridge is the central state holder for the Chrome connection, tab contexts,
diff --git a/cdp.go b/cdp.go
index 1dd8a4d6..c32da5c3 100644
--- a/cdp.go
+++ b/cdp.go
@@ -35,10 +35,10 @@ func navigatePage(ctx context.Context, url string) error {
)
}
-// clickByNodeID resolves a backend DOM node to a JS object, scrolls it into
-// view, and calls .click(). Uses DOM.resolveNode + Runtime.callFunctionOn
-// which works on React/shadow DOM where CSS selectors fail.
-func clickByNodeID(ctx context.Context, backendNodeID int64) error {
+// withElement resolves a backendNodeID to a JS remote object, scrolls it into
+// view, and calls the given JS function on it. This is the generic helper for
+// all element-targeted actions (click, hover, select, etc.).
+func withElement(ctx context.Context, backendNodeID int64, jsFunc string) error {
return chromedp.Run(ctx,
chromedp.ActionFunc(func(ctx context.Context) error {
objectID, err := resolveNodeToObject(ctx, backendNodeID)
@@ -47,32 +47,50 @@ func clickByNodeID(ctx context.Context, backendNodeID int64) error {
}
callP := map[string]any{
"objectId": objectID,
- "functionDeclaration": "function() { this.scrollIntoViewIfNeeded(); this.click(); }",
+ "functionDeclaration": jsFunc,
"arguments": []any{},
}
if err := chromedp.FromContext(ctx).Target.Execute(ctx, "Runtime.callFunctionOn", callP, nil); err != nil {
- return fmt.Errorf("click callFunctionOn: %w", err)
+ return fmt.Errorf("callFunctionOn: %w", err)
}
return nil
}),
)
}
-// typeByNodeID scrolls an element into view, focuses it, and sends keyboard events.
-func typeByNodeID(ctx context.Context, backendNodeID int64, text string) error {
+// withElementArg is like withElement but passes a single string argument to the JS function.
+func withElementArg(ctx context.Context, backendNodeID int64, jsFunc string, arg string) error {
return chromedp.Run(ctx,
chromedp.ActionFunc(func(ctx context.Context) error {
objectID, err := resolveNodeToObject(ctx, backendNodeID)
if err != nil {
return err
}
- scrollP := map[string]any{
+ callP := map[string]any{
"objectId": objectID,
- "functionDeclaration": "function() { this.scrollIntoViewIfNeeded(); }",
- "arguments": []any{},
+ "functionDeclaration": jsFunc,
+ "arguments": []any{map[string]any{"value": arg}},
+ }
+ if err := chromedp.FromContext(ctx).Target.Execute(ctx, "Runtime.callFunctionOn", callP, nil); err != nil {
+ return fmt.Errorf("callFunctionOn: %w", err)
}
- _ = chromedp.FromContext(ctx).Target.Execute(ctx, "Runtime.callFunctionOn", scrollP, nil)
+ return nil
+ }),
+ )
+}
+// clickByNodeID scrolls an element into view and clicks it.
+func clickByNodeID(ctx context.Context, backendNodeID int64) error {
+ return withElement(ctx, backendNodeID, "function() { this.scrollIntoViewIfNeeded(); this.click(); }")
+}
+
+// typeByNodeID scrolls an element into view, focuses it, and sends keyboard events.
+func typeByNodeID(ctx context.Context, backendNodeID int64, text string) error {
+ if err := withElement(ctx, backendNodeID, "function() { this.scrollIntoViewIfNeeded(); }"); err != nil {
+ return err
+ }
+ return chromedp.Run(ctx,
+ chromedp.ActionFunc(func(ctx context.Context) error {
p := map[string]any{"backendNodeId": backendNodeID}
if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.focus", p, nil); err != nil {
return fmt.Errorf("DOM.focus: %w", err)
@@ -83,6 +101,31 @@ func typeByNodeID(ctx context.Context, backendNodeID int64, text string) error {
)
}
+// hoverByNodeID scrolls an element into view and dispatches a mouseover event.
+func hoverByNodeID(ctx context.Context, backendNodeID int64) error {
+ return withElement(ctx, backendNodeID, `function() {
+ this.scrollIntoViewIfNeeded();
+ this.dispatchEvent(new MouseEvent('mouseover', {bubbles: true}));
+ this.dispatchEvent(new MouseEvent('mouseenter', {bubbles: false}));
+ }`)
+}
+
+// selectByNodeID sets the value of a <select> element and fires change event.
+func selectByNodeID(ctx context.Context, backendNodeID int64, value string) error {
+ return withElementArg(ctx, backendNodeID, `function(val) {
+ this.scrollIntoViewIfNeeded();
+ const opt = Array.from(this.options).find(o => o.value === val || o.textContent.trim() === val);
+ if (opt) { this.value = opt.value; }
+ else { this.value = val; }
+ this.dispatchEvent(new Event('change', {bubbles: true}));
+ }`, value)
+}
+
+// scrollByNodeID scrolls an element into view.
+func scrollByNodeID(ctx context.Context, backendNodeID int64) error {
+ return withElement(ctx, backendNodeID, "function() { this.scrollIntoViewIfNeeded(); }")
+}
+
// resolveNodeToObject converts a backendNodeID to a JS remote object ID.
func resolveNodeToObject(ctx context.Context, backendNodeID int64) (string, error) {
p := map[string]any{"backendNodeId": backendNodeID}
diff --git a/config.go b/config.go
index 769d16fc..9036e2d9 100644
--- a/config.go
+++ b/config.go
@@ -18,11 +18,14 @@ const filterInteractive = "interactive"
// Action kinds for /action endpoint.
const (
- actionClick = "click"
- actionType = "type"
- actionFill = "fill"
- actionPress = "press"
- actionFocus = "focus"
+ actionClick = "click"
+ actionType = "type"
+ actionFill = "fill"
+ actionPress = "press"
+ actionFocus = "focus"
+ actionHover = "hover"
+ actionSelect = "select"
+ actionScroll = "scroll"
)
// Tab actions for /tab endpoint.
@@ -45,6 +48,7 @@ var (
actionTimeout = 15 * time.Second
navigateTimeout = 30 * time.Second
shutdownTimeout = 10 * time.Second
+ waitNavDelay = 1 * time.Second
)
func envOr(key, fallback string) string {
diff --git a/handlers.go b/handlers.go
index 41b4834f..a6742bc9 100644
--- a/handlers.go
+++ b/handlers.go
@@ -50,6 +50,8 @@ func (b *Bridge) handleTabs(w http.ResponseWriter, r *http.Request) {
func (b *Bridge) handleSnapshot(w http.ResponseWriter, r *http.Request) {
tabID := r.URL.Query().Get("tabId")
filter := r.URL.Query().Get("filter")
+ doDiff := r.URL.Query().Get("diff") == "true"
+ format := r.URL.Query().Get("format") // "text" for indented tree
maxDepthStr := r.URL.Query().Get("depth")
maxDepth := -1
if maxDepthStr != "" {
@@ -89,9 +91,19 @@ func (b *Bridge) handleSnapshot(w http.ResponseWriter, r *http.Request) {
flat, refs := buildSnapshot(treeResp.Nodes, filter, maxDepth)
- // Cache ref→nodeID mapping for this tab
+ // Get previous snapshot for diff before overwriting cache
+ var prevNodes []A11yNode
+ if doDiff {
+ b.mu.RLock()
+ if prev := b.snapshots[resolvedTabID]; prev != nil {
+ prevNodes = prev.nodes
+ }
+ b.mu.RUnlock()
+ }
+
+ // Cache ref→nodeID mapping and nodes for this tab
b.mu.Lock()
- b.snapshots[resolvedTabID] = &refCache{refs: refs}
+ b.snapshots[resolvedTabID] = &refCache{refs: refs, nodes: flat}
b.mu.Unlock()
var url, title string
@@ -100,6 +112,33 @@ func (b *Bridge) handleSnapshot(w http.ResponseWriter, r *http.Request) {
chromedp.Title(&title),
)
+ if doDiff && prevNodes != nil {
+ added, changed, removed := diffSnapshot(prevNodes, flat)
+ jsonResp(w, 200, map[string]any{
+ "url": url,
+ "title": title,
+ "diff": true,
+ "added": added,
+ "changed": changed,
+ "removed": removed,
+ "counts": map[string]int{
+ "added": len(added),
+ "changed": len(changed),
+ "removed": len(removed),
+ "total": len(flat),
+ },
+ })
+ return
+ }
+
+ if format == "text" {
+ w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+ w.WriteHeader(200)
+ fmt.Fprintf(w, "# %s\n# %s\n# %d nodes\n\n", title, url, len(flat))
+ w.Write([]byte(formatSnapshotText(flat)))
+ return
+ }
+
jsonResp(w, 200, map[string]any{
"url": url,
"title": title,
@@ -251,7 +290,11 @@ type actionRequest struct {
Selector string `json:"selector"`
Text string `json:"text"`
Key string `json:"key"`
+ Value string `json:"value"`
NodeID int64 `json:"nodeId"`
+ ScrollX int `json:"scrollX"`
+ ScrollY int `json:"scrollY"`
+ WaitNav bool `json:"waitNav"`
}
// ActionFunc handles a single action kind. Receives the full request for
@@ -261,13 +304,22 @@ type ActionFunc func(ctx context.Context, req actionRequest) (map[string]any, er
func (b *Bridge) actionRegistry() map[string]ActionFunc {
return map[string]ActionFunc{
actionClick: func(ctx context.Context, req actionRequest) (map[string]any, error) {
+ var err error
if req.Selector != "" {
- return map[string]any{"clicked": true}, chromedp.Run(ctx, chromedp.Click(req.Selector, chromedp.ByQuery))
+ err = chromedp.Run(ctx, chromedp.Click(req.Selector, chromedp.ByQuery))
+ } else if req.NodeID > 0 {
+ err = clickByNodeID(ctx, req.NodeID)
+ } else {
+ return nil, fmt.Errorf("need selector, ref, or nodeId")
}
- if req.NodeID > 0 {
- return map[string]any{"clicked": true}, clickByNodeID(ctx, req.NodeID)
+ if err != nil {
+ return nil, err
}
- return nil, fmt.Errorf("need selector, ref, or nodeId")
+ // Optional: wait for navigation after click (e.g. link clicks)
+ if req.WaitNav {
+ chromedp.Run(ctx, chromedp.Sleep(waitNavDelay))
+ }
+ return map[string]any{"clicked": true}, nil
},
actionType: func(ctx context.Context, req actionRequest) (map[string]any, error) {
if req.Text == "" {
@@ -310,6 +362,50 @@ func (b *Bridge) actionRegistry() map[string]ActionFunc {
}
return map[string]any{"focused": true}, nil
},
+ actionHover: func(ctx context.Context, req actionRequest) (map[string]any, error) {
+ if req.NodeID > 0 {
+ return map[string]any{"hovered": true}, hoverByNodeID(ctx, req.NodeID)
+ }
+ if req.Selector != "" {
+ return map[string]any{"hovered": true}, chromedp.Run(ctx,
+ chromedp.Evaluate(fmt.Sprintf(`document.querySelector(%q)?.dispatchEvent(new MouseEvent('mouseover', {bubbles:true}))`, req.Selector), nil),
+ )
+ }
+ return nil, fmt.Errorf("need selector or ref")
+ },
+ actionSelect: func(ctx context.Context, req actionRequest) (map[string]any, error) {
+ val := req.Value
+ if val == "" {
+ val = req.Text // fallback
+ }
+ if val == "" {
+ return nil, fmt.Errorf("value required for select")
+ }
+ if req.NodeID > 0 {
+ return map[string]any{"selected": val}, selectByNodeID(ctx, req.NodeID, val)
+ }
+ if req.Selector != "" {
+ return map[string]any{"selected": val}, chromedp.Run(ctx,
+ chromedp.SetValue(req.Selector, val, chromedp.ByQuery),
+ )
+ }
+ return nil, fmt.Errorf("need selector or ref")
+ },
+ actionScroll: func(ctx context.Context, req actionRequest) (map[string]any, error) {
+ // Scroll to element
+ if req.NodeID > 0 {
+ return map[string]any{"scrolled": true}, scrollByNodeID(ctx, req.NodeID)
+ }
+ // Scroll by pixel amount
+ if req.ScrollX != 0 || req.ScrollY != 0 {
+ js := fmt.Sprintf("window.scrollBy(%d, %d)", req.ScrollX, req.ScrollY)
+ return map[string]any{"scrolled": true, "x": req.ScrollX, "y": req.ScrollY},
+ chromedp.Run(ctx, chromedp.Evaluate(js, nil))
+ }
+ // Default: scroll down one viewport
+ return map[string]any{"scrolled": true, "y": 800},
+ chromedp.Run(ctx, chromedp.Evaluate("window.scrollBy(0, 800)", nil))
+ },
}
}
diff --git a/snapshot.go b/snapshot.go
index 74034527..fff2f09c 100644
--- a/snapshot.go
+++ b/snapshot.go
@@ -146,3 +146,67 @@ func buildSnapshot(nodes []rawAXNode, filter string, maxDepth int) ([]A11yNode,
return flat, refs
}
+
+// formatSnapshotText renders nodes as an indented text tree.
+// Much cheaper on tokens than JSON (~40-60% reduction).
+func formatSnapshotText(nodes []A11yNode) string {
+ var b strings.Builder
+ for _, n := range nodes {
+ for i := 0; i < n.Depth; i++ {
+ b.WriteString(" ")
+ }
+ b.WriteString(n.Ref)
+ b.WriteByte(' ')
+ b.WriteString(n.Role)
+ if n.Name != "" {
+ b.WriteString(` "`)
+ b.WriteString(n.Name)
+ b.WriteByte('"')
+ }
+ if n.Value != "" {
+ b.WriteString(` val="`)
+ b.WriteString(n.Value)
+ b.WriteByte('"')
+ }
+ if n.Focused {
+ b.WriteString(" [focused]")
+ }
+ if n.Disabled {
+ b.WriteString(" [disabled]")
+ }
+ b.WriteByte('\n')
+ }
+ return b.String()
+}
+
+// diffSnapshot returns only nodes that changed between prev and current snapshots.
+// A node is "changed" if it's new, removed, or has different name/value/focused/disabled.
+// Returns added, changed, and removed nodes.
+func diffSnapshot(prev, curr []A11yNode) (added, changed, removed []A11yNode) {
+ prevMap := make(map[string]A11yNode, len(prev))
+ for _, n := range prev {
+ key := fmt.Sprintf("%s:%s:%d", n.Role, n.Name, n.NodeID)
+ prevMap[key] = n
+ }
+
+ currMap := make(map[string]bool, len(curr))
+ for _, n := range curr {
+ key := fmt.Sprintf("%s:%s:%d", n.Role, n.Name, n.NodeID)
+ currMap[key] = true
+ old, existed := prevMap[key]
+ if !existed {
+ added = append(added, n)
+ } else if old.Value != n.Value || old.Focused != n.Focused || old.Disabled != n.Disabled {
+ changed = append(changed, n)
+ }
+ }
+
+ for _, n := range prev {
+ key := fmt.Sprintf("%s:%s:%d", n.Role, n.Name, n.NodeID)
+ if !currMap[key] {
+ removed = append(removed, n)
+ }
+ }
+
+ return
+}
diff --git a/snapshot_test.go b/snapshot_test.go
index 7b2be528..6be9c3bf 100644
--- a/snapshot_test.go
+++ b/snapshot_test.go
@@ -2,6 +2,7 @@ package main
import (
"encoding/json"
+ "strings"
"testing"
)
@@ -146,6 +147,65 @@ func TestBuildSnapshotInteractiveFilter(t *testing.T) {
}
}
+func TestFormatSnapshotText(t *testing.T) {
+ nodes := []A11yNode{
+ {Ref: "e0", Role: "WebArea", Name: "Page", Depth: 0},
+ {Ref: "e1", Role: "button", Name: "Submit", Depth: 1},
+ {Ref: "e2", Role: "textbox", Name: "Email", Depth: 1, Value: "[email protected]", Focused: true},
+ {Ref: "e3", Role: "button", Name: "Cancel", Depth: 1, Disabled: true},
+ }
+
+ text := formatSnapshotText(nodes)
+
+ if !strings.Contains(text, `e0 WebArea "Page"`) {
+ t.Error("missing root node")
+ }
+ if !strings.Contains(text, ` e1 button "Submit"`) {
+ t.Error("missing indented button")
+ }
+ if !strings.Contains(text, `val="[email protected]"`) {
+ t.Error("missing value")
+ }
+ if !strings.Contains(text, "[focused]") {
+ t.Error("missing focused flag")
+ }
+ if !strings.Contains(text, "[disabled]") {
+ t.Error("missing disabled flag")
+ }
+}
+
+func TestDiffSnapshot(t *testing.T) {
+ prev := []A11yNode{
+ {Ref: "e0", Role: "button", Name: "Submit", NodeID: 10},
+ {Ref: "e1", Role: "textbox", Name: "Email", NodeID: 20, Value: ""},
+ {Ref: "e2", Role: "link", Name: "Old Link", NodeID: 30},
+ }
+ curr := []A11yNode{
+ {Ref: "e0", Role: "button", Name: "Submit", NodeID: 10}, // unchanged
+ {Ref: "e1", Role: "textbox", Name: "Email", NodeID: 20, Value: "hi"}, // changed (value)
+ {Ref: "e3", Role: "link", Name: "New Link", NodeID: 40}, // added
+ }
+
+ added, changed, removed := diffSnapshot(prev, curr)
+
+ if len(added) != 1 || added[0].Name != "New Link" {
+ t.Errorf("expected 1 added (New Link), got %+v", added)
+ }
+ if len(changed) != 1 || changed[0].Name != "Email" {
+ t.Errorf("expected 1 changed (Email), got %+v", changed)
+ }
+ if len(removed) != 1 || removed[0].Name != "Old Link" {
+ t.Errorf("expected 1 removed (Old Link), got %+v", removed)
+ }
+}
+
+func TestDiffSnapshotEmpty(t *testing.T) {
+ added, changed, removed := diffSnapshot(nil, nil)
+ if len(added) != 0 || len(changed) != 0 || len(removed) != 0 {
+ t.Error("diff of two empty snapshots should be empty")
+ }
+}
+
func TestBuildSnapshotDepthFilter(t *testing.T) {
nodes := []rawAXNode{
{