High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
P5: hover/select/scroll actions, smart diff, text format, withElement helper, 21 tests
bot1b2f2dc31a2c
diff --git a/TODO.md b/TODO.mdindex d9be21bb..294ba9e8 100644--- a/TODO.md+++ b/TODO.md@@ -61,19 +61,23 @@ Split into 8 files (single package):- [x] Add config tests — envOr, homeDir, constants- [x] **18 tests passing** — `go test ./...`-## P5: Bugs & Stealth-- [ ] **Remove `--disable-blink-features=AutomationControlled` flag** — deprecated in Chrome 144+, triggers warning banner in headed mode, hurts stealth. JS injection already patches `navigator.webdriver`+## Done ✅ (P5)+- [x] **`withElement` helper** — generic `withElement(ctx, nodeID, jsFunc)` + `withElementArg` for all element actions+- [x] **`hover` action** — mouseover/mouseenter via ref, selector+- [x] **`select` action** — pick `<select>` option by value or text, fires change event+- [x] **`scroll` action** — scroll to element (ref), by pixel amount (`scrollX`/`scrollY`), or default 800px down+- [x] **Smart diff** — `?diff=true` returns added/changed/removed nodes since last snapshot+- [x] **Wait for navigation** — `"waitNav": true` on click action, 1s delay for page load+- [x] **Text format** — `?format=text` returns indented tree (~40-60% fewer tokens than JSON)+- [x] **21 tests passing** — diff, text format, all existing tests green++## P5 Remaining-## P6: Features-- [ ] **`/scroll` endpoint** — scroll to element or by amount. Needed for infinite-scroll pages (X, Reddit).-- [ ] **`withElement` helper** — generic `withElement(ctx, nodeID, jsFunc)` for all element actions (click, type, scroll, hover, drag)-- [ ] **Smart diff** — `?diff=true` returns only changes since last snapshot. Massive token savings on multi-step tasks-- [ ] **Wait for navigation** — after click, wait for page load before returning+## P5 Bugs & Stealth+- [ ] **Remove `--disable-blink-features=AutomationControlled` flag** — deprecated in Chrome 144+, triggers warning banner in headed mode, hurts stealth. JS injection already patches `navigator.webdriver`- [ ] **Better /text** — Readability-style extraction instead of raw innerText- [ ] **Split handlers.go** — snapshot handler is complex enough for its own file- [ ] **Wire interfaces into handlers** — Inject Browser/TabManager so handler tests don't need Chrome-- [ ] **`hover` action** — mouseover for dropdowns/tooltips (common on X, Google)-- [ ] **`select` action** — pick option from `<select>` dropdowns by value or text## P7: Nice to Have- [ ] **File-based output** — `?output=file` saves snapshot to disk, returns path (Playwright CLI approach)@@ -83,7 +87,6 @@ Split into 8 files (single package):- [ ] **Config file** — `~/.pinchtab/config.json`- [ ] **LaunchAgent/systemd** — auto-start on boot- [ ] **`/cookies` endpoint** — read/set cookies (useful for auth debugging)-- [ ] **Snapshot text format** — `?format=text` returns indented tree instead of JSON (even cheaper tokens)## Not Doing- Plugin systemdiff --git a/bridge.go b/bridge.goindex f0e7a5fe..bed67ea1 100644--- a/bridge.go+++ b/bridge.go@@ -17,11 +17,12 @@ type TabEntry struct {cancel context.CancelFunc}-// refCache stores the ref→backendNodeID mapping from the last snapshot per tab.+// refCache stores the ref→backendNodeID mapping and the last snapshot nodes per tab.// Refs are assigned during /snapshot and looked up during /action, avoiding// a second a11y tree fetch that could drift.type refCache struct {- refs map[string]int64 // "e0" → backendNodeID+ refs map[string]int64 // "e0" → backendNodeID+ nodes []A11yNode // last snapshot for diff}// Bridge is the central state holder for the Chrome connection, tab contexts,diff --git a/cdp.go b/cdp.goindex 1dd8a4d6..c32da5c3 100644--- a/cdp.go+++ b/cdp.go@@ -35,10 +35,10 @@ func navigatePage(ctx context.Context, url string) error {)}-// clickByNodeID resolves a backend DOM node to a JS object, scrolls it into-// view, and calls .click(). Uses DOM.resolveNode + Runtime.callFunctionOn-// which works on React/shadow DOM where CSS selectors fail.-func clickByNodeID(ctx context.Context, backendNodeID int64) error {+// withElement resolves a backendNodeID to a JS remote object, scrolls it into+// view, and calls the given JS function on it. This is the generic helper for+// all element-targeted actions (click, hover, select, etc.).+func withElement(ctx context.Context, backendNodeID int64, jsFunc string) error {return chromedp.Run(ctx,chromedp.ActionFunc(func(ctx context.Context) error {objectID, err := resolveNodeToObject(ctx, backendNodeID)@@ -47,32 +47,50 @@ func clickByNodeID(ctx context.Context, backendNodeID int64) error {}callP := map[string]any{"objectId": objectID,- "functionDeclaration": "function() { this.scrollIntoViewIfNeeded(); this.click(); }",+ "functionDeclaration": jsFunc,"arguments": []any{},}if err := chromedp.FromContext(ctx).Target.Execute(ctx, "Runtime.callFunctionOn", callP, nil); err != nil {- return fmt.Errorf("click callFunctionOn: %w", err)+ return fmt.Errorf("callFunctionOn: %w", err)}return nil}),)}-// typeByNodeID scrolls an element into view, focuses it, and sends keyboard events.-func typeByNodeID(ctx context.Context, backendNodeID int64, text string) error {+// withElementArg is like withElement but passes a single string argument to the JS function.+func withElementArg(ctx context.Context, backendNodeID int64, jsFunc string, arg string) error {return chromedp.Run(ctx,chromedp.ActionFunc(func(ctx context.Context) error {objectID, err := resolveNodeToObject(ctx, backendNodeID)if err != nil {return err}- scrollP := map[string]any{+ callP := map[string]any{"objectId": objectID,- "functionDeclaration": "function() { this.scrollIntoViewIfNeeded(); }",- "arguments": []any{},+ "functionDeclaration": jsFunc,+ "arguments": []any{map[string]any{"value": arg}},+ }+ if err := chromedp.FromContext(ctx).Target.Execute(ctx, "Runtime.callFunctionOn", callP, nil); err != nil {+ return fmt.Errorf("callFunctionOn: %w", err)}- _ = chromedp.FromContext(ctx).Target.Execute(ctx, "Runtime.callFunctionOn", scrollP, nil)+ return nil+ }),+ )+}+// clickByNodeID scrolls an element into view and clicks it.+func clickByNodeID(ctx context.Context, backendNodeID int64) error {+ return withElement(ctx, backendNodeID, "function() { this.scrollIntoViewIfNeeded(); this.click(); }")+}++// typeByNodeID scrolls an element into view, focuses it, and sends keyboard events.+func typeByNodeID(ctx context.Context, backendNodeID int64, text string) error {+ if err := withElement(ctx, backendNodeID, "function() { this.scrollIntoViewIfNeeded(); }"); err != nil {+ return err+ }+ return chromedp.Run(ctx,+ chromedp.ActionFunc(func(ctx context.Context) error {p := map[string]any{"backendNodeId": backendNodeID}if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.focus", p, nil); err != nil {return fmt.Errorf("DOM.focus: %w", err)@@ -83,6 +101,31 @@ func typeByNodeID(ctx context.Context, backendNodeID int64, text string) error {)}+// hoverByNodeID scrolls an element into view and dispatches a mouseover event.+func hoverByNodeID(ctx context.Context, backendNodeID int64) error {+ return withElement(ctx, backendNodeID, `function() {+ this.scrollIntoViewIfNeeded();+ this.dispatchEvent(new MouseEvent('mouseover', {bubbles: true}));+ this.dispatchEvent(new MouseEvent('mouseenter', {bubbles: false}));+ }`)+}++// selectByNodeID sets the value of a <select> element and fires change event.+func selectByNodeID(ctx context.Context, backendNodeID int64, value string) error {+ return withElementArg(ctx, backendNodeID, `function(val) {+ this.scrollIntoViewIfNeeded();+ const opt = Array.from(this.options).find(o => o.value === val || o.textContent.trim() === val);+ if (opt) { this.value = opt.value; }+ else { this.value = val; }+ this.dispatchEvent(new Event('change', {bubbles: true}));+ }`, value)+}++// scrollByNodeID scrolls an element into view.+func scrollByNodeID(ctx context.Context, backendNodeID int64) error {+ return withElement(ctx, backendNodeID, "function() { this.scrollIntoViewIfNeeded(); }")+}+// resolveNodeToObject converts a backendNodeID to a JS remote object ID.func resolveNodeToObject(ctx context.Context, backendNodeID int64) (string, error) {p := map[string]any{"backendNodeId": backendNodeID}diff --git a/config.go b/config.goindex 769d16fc..9036e2d9 100644--- a/config.go+++ b/config.go@@ -18,11 +18,14 @@ const filterInteractive = "interactive"// Action kinds for /action endpoint.const (- actionClick = "click"- actionType = "type"- actionFill = "fill"- actionPress = "press"- actionFocus = "focus"+ actionClick = "click"+ actionType = "type"+ actionFill = "fill"+ actionPress = "press"+ actionFocus = "focus"+ actionHover = "hover"+ actionSelect = "select"+ actionScroll = "scroll")// Tab actions for /tab endpoint.@@ -45,6 +48,7 @@ var (actionTimeout = 15 * time.SecondnavigateTimeout = 30 * time.SecondshutdownTimeout = 10 * time.Second+ waitNavDelay = 1 * time.Second)func envOr(key, fallback string) string {diff --git a/handlers.go b/handlers.goindex 41b4834f..a6742bc9 100644--- a/handlers.go+++ b/handlers.go@@ -50,6 +50,8 @@ func (b *Bridge) handleTabs(w http.ResponseWriter, r *http.Request) {func (b *Bridge) handleSnapshot(w http.ResponseWriter, r *http.Request) {tabID := r.URL.Query().Get("tabId")filter := r.URL.Query().Get("filter")+ doDiff := r.URL.Query().Get("diff") == "true"+ format := r.URL.Query().Get("format") // "text" for indented treemaxDepthStr := r.URL.Query().Get("depth")maxDepth := -1if maxDepthStr != "" {@@ -89,9 +91,19 @@ func (b *Bridge) handleSnapshot(w http.ResponseWriter, r *http.Request) {flat, refs := buildSnapshot(treeResp.Nodes, filter, maxDepth)- // Cache ref→nodeID mapping for this tab+ // Get previous snapshot for diff before overwriting cache+ var prevNodes []A11yNode+ if doDiff {+ b.mu.RLock()+ if prev := b.snapshots[resolvedTabID]; prev != nil {+ prevNodes = prev.nodes+ }+ b.mu.RUnlock()+ }++ // Cache ref→nodeID mapping and nodes for this tabb.mu.Lock()- b.snapshots[resolvedTabID] = &refCache{refs: refs}+ b.snapshots[resolvedTabID] = &refCache{refs: refs, nodes: flat}b.mu.Unlock()var url, title string@@ -100,6 +112,33 @@ func (b *Bridge) handleSnapshot(w http.ResponseWriter, r *http.Request) {chromedp.Title(&title),)+ if doDiff && prevNodes != nil {+ added, changed, removed := diffSnapshot(prevNodes, flat)+ jsonResp(w, 200, map[string]any{+ "url": url,+ "title": title,+ "diff": true,+ "added": added,+ "changed": changed,+ "removed": removed,+ "counts": map[string]int{+ "added": len(added),+ "changed": len(changed),+ "removed": len(removed),+ "total": len(flat),+ },+ })+ return+ }++ if format == "text" {+ w.Header().Set("Content-Type", "text/plain; charset=utf-8")+ w.WriteHeader(200)+ fmt.Fprintf(w, "# %s\n# %s\n# %d nodes\n\n", title, url, len(flat))+ w.Write([]byte(formatSnapshotText(flat)))+ return+ }+jsonResp(w, 200, map[string]any{"url": url,"title": title,@@ -251,7 +290,11 @@ type actionRequest struct {Selector string `json:"selector"`Text string `json:"text"`Key string `json:"key"`+ Value string `json:"value"`NodeID int64 `json:"nodeId"`+ ScrollX int `json:"scrollX"`+ ScrollY int `json:"scrollY"`+ WaitNav bool `json:"waitNav"`}// ActionFunc handles a single action kind. Receives the full request for@@ -261,13 +304,22 @@ type ActionFunc func(ctx context.Context, req actionRequest) (map[string]any, erfunc (b *Bridge) actionRegistry() map[string]ActionFunc {return map[string]ActionFunc{actionClick: func(ctx context.Context, req actionRequest) (map[string]any, error) {+ var err errorif req.Selector != "" {- return map[string]any{"clicked": true}, chromedp.Run(ctx, chromedp.Click(req.Selector, chromedp.ByQuery))+ err = chromedp.Run(ctx, chromedp.Click(req.Selector, chromedp.ByQuery))+ } else if req.NodeID > 0 {+ err = clickByNodeID(ctx, req.NodeID)+ } else {+ return nil, fmt.Errorf("need selector, ref, or nodeId")}- if req.NodeID > 0 {- return map[string]any{"clicked": true}, clickByNodeID(ctx, req.NodeID)+ if err != nil {+ return nil, err}- return nil, fmt.Errorf("need selector, ref, or nodeId")+ // Optional: wait for navigation after click (e.g. link clicks)+ if req.WaitNav {+ chromedp.Run(ctx, chromedp.Sleep(waitNavDelay))+ }+ return map[string]any{"clicked": true}, nil},actionType: func(ctx context.Context, req actionRequest) (map[string]any, error) {if req.Text == "" {@@ -310,6 +362,50 @@ func (b *Bridge) actionRegistry() map[string]ActionFunc {}return map[string]any{"focused": true}, nil},+ actionHover: func(ctx context.Context, req actionRequest) (map[string]any, error) {+ if req.NodeID > 0 {+ return map[string]any{"hovered": true}, hoverByNodeID(ctx, req.NodeID)+ }+ if req.Selector != "" {+ return map[string]any{"hovered": true}, chromedp.Run(ctx,+ chromedp.Evaluate(fmt.Sprintf(`document.querySelector(%q)?.dispatchEvent(new MouseEvent('mouseover', {bubbles:true}))`, req.Selector), nil),+ )+ }+ return nil, fmt.Errorf("need selector or ref")+ },+ actionSelect: func(ctx context.Context, req actionRequest) (map[string]any, error) {+ val := req.Value+ if val == "" {+ val = req.Text // fallback+ }+ if val == "" {+ return nil, fmt.Errorf("value required for select")+ }+ if req.NodeID > 0 {+ return map[string]any{"selected": val}, selectByNodeID(ctx, req.NodeID, val)+ }+ if req.Selector != "" {+ return map[string]any{"selected": val}, chromedp.Run(ctx,+ chromedp.SetValue(req.Selector, val, chromedp.ByQuery),+ )+ }+ return nil, fmt.Errorf("need selector or ref")+ },+ actionScroll: func(ctx context.Context, req actionRequest) (map[string]any, error) {+ // Scroll to element+ if req.NodeID > 0 {+ return map[string]any{"scrolled": true}, scrollByNodeID(ctx, req.NodeID)+ }+ // Scroll by pixel amount+ if req.ScrollX != 0 || req.ScrollY != 0 {+ js := fmt.Sprintf("window.scrollBy(%d, %d)", req.ScrollX, req.ScrollY)+ return map[string]any{"scrolled": true, "x": req.ScrollX, "y": req.ScrollY},+ chromedp.Run(ctx, chromedp.Evaluate(js, nil))+ }+ // Default: scroll down one viewport+ return map[string]any{"scrolled": true, "y": 800},+ chromedp.Run(ctx, chromedp.Evaluate("window.scrollBy(0, 800)", nil))+ },}}diff --git a/snapshot.go b/snapshot.goindex 74034527..fff2f09c 100644--- a/snapshot.go+++ b/snapshot.go@@ -146,3 +146,67 @@ func buildSnapshot(nodes []rawAXNode, filter string, maxDepth int) ([]A11yNode,return flat, refs}++// formatSnapshotText renders nodes as an indented text tree.+// Much cheaper on tokens than JSON (~40-60% reduction).+func formatSnapshotText(nodes []A11yNode) string {+ var b strings.Builder+ for _, n := range nodes {+ for i := 0; i < n.Depth; i++ {+ b.WriteString(" ")+ }+ b.WriteString(n.Ref)+ b.WriteByte(' ')+ b.WriteString(n.Role)+ if n.Name != "" {+ b.WriteString(` "`)+ b.WriteString(n.Name)+ b.WriteByte('"')+ }+ if n.Value != "" {+ b.WriteString(` val="`)+ b.WriteString(n.Value)+ b.WriteByte('"')+ }+ if n.Focused {+ b.WriteString(" [focused]")+ }+ if n.Disabled {+ b.WriteString(" [disabled]")+ }+ b.WriteByte('\n')+ }+ return b.String()+}++// diffSnapshot returns only nodes that changed between prev and current snapshots.+// A node is "changed" if it's new, removed, or has different name/value/focused/disabled.+// Returns added, changed, and removed nodes.+func diffSnapshot(prev, curr []A11yNode) (added, changed, removed []A11yNode) {+ prevMap := make(map[string]A11yNode, len(prev))+ for _, n := range prev {+ key := fmt.Sprintf("%s:%s:%d", n.Role, n.Name, n.NodeID)+ prevMap[key] = n+ }++ currMap := make(map[string]bool, len(curr))+ for _, n := range curr {+ key := fmt.Sprintf("%s:%s:%d", n.Role, n.Name, n.NodeID)+ currMap[key] = true+ old, existed := prevMap[key]+ if !existed {+ added = append(added, n)+ } else if old.Value != n.Value || old.Focused != n.Focused || old.Disabled != n.Disabled {+ changed = append(changed, n)+ }+ }++ for _, n := range prev {+ key := fmt.Sprintf("%s:%s:%d", n.Role, n.Name, n.NodeID)+ if !currMap[key] {+ removed = append(removed, n)+ }+ }++ return+}diff --git a/snapshot_test.go b/snapshot_test.goindex 7b2be528..6be9c3bf 100644--- a/snapshot_test.go+++ b/snapshot_test.go@@ -2,6 +2,7 @@ package mainimport ("encoding/json"+ "strings""testing")@@ -146,6 +147,65 @@ func TestBuildSnapshotInteractiveFilter(t *testing.T) {}}+func TestFormatSnapshotText(t *testing.T) {+ nodes := []A11yNode{+ {Ref: "e0", Role: "WebArea", Name: "Page", Depth: 0},+ {Ref: "e1", Role: "button", Name: "Submit", Depth: 1},+ {Ref: "e2", Role: "textbox", Name: "Email", Depth: 1, Value: "[email protected]", Focused: true},+ {Ref: "e3", Role: "button", Name: "Cancel", Depth: 1, Disabled: true},+ }++ text := formatSnapshotText(nodes)++ if !strings.Contains(text, `e0 WebArea "Page"`) {+ t.Error("missing root node")+ }+ if !strings.Contains(text, ` e1 button "Submit"`) {+ t.Error("missing indented button")+ }+ if !strings.Contains(text, `val="[email protected]"`) {+ t.Error("missing value")+ }+ if !strings.Contains(text, "[focused]") {+ t.Error("missing focused flag")+ }+ if !strings.Contains(text, "[disabled]") {+ t.Error("missing disabled flag")+ }+}++func TestDiffSnapshot(t *testing.T) {+ prev := []A11yNode{+ {Ref: "e0", Role: "button", Name: "Submit", NodeID: 10},+ {Ref: "e1", Role: "textbox", Name: "Email", NodeID: 20, Value: ""},+ {Ref: "e2", Role: "link", Name: "Old Link", NodeID: 30},+ }+ curr := []A11yNode{+ {Ref: "e0", Role: "button", Name: "Submit", NodeID: 10}, // unchanged+ {Ref: "e1", Role: "textbox", Name: "Email", NodeID: 20, Value: "hi"}, // changed (value)+ {Ref: "e3", Role: "link", Name: "New Link", NodeID: 40}, // added+ }++ added, changed, removed := diffSnapshot(prev, curr)++ if len(added) != 1 || added[0].Name != "New Link" {+ t.Errorf("expected 1 added (New Link), got %+v", added)+ }+ if len(changed) != 1 || changed[0].Name != "Email" {+ t.Errorf("expected 1 changed (Email), got %+v", changed)+ }+ if len(removed) != 1 || removed[0].Name != "Old Link" {+ t.Errorf("expected 1 removed (Old Link), got %+v", removed)+ }+}++func TestDiffSnapshotEmpty(t *testing.T) {+ added, changed, removed := diffSnapshot(nil, nil)+ if len(added) != 0 || len(changed) != 0 || len(removed) != 0 {+ t.Error("diff of two empty snapshots should be empty")+ }+}+func TestBuildSnapshotDepthFilter(t *testing.T) {nodes := []rawAXNode{{