High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
feat: basic iframe support
Luigi Agostid61801450c98
diff --git a/docs/reference/click.md b/docs/reference/click.mdindex 883ed706..e71bf641 100644--- a/docs/reference/click.md+++ b/docs/reference/click.md@@ -20,6 +20,7 @@ pinchtab click e5Notes:- element refs come from `/snapshot`+- refs returned for iframe descendants can be clicked directly; no manual frame switch is required- the raw action endpoint also accepts `selector`, for example `{"kind":"click","selector":"#login"}`- the CLI also accepts `#login`, `xpath://button`, `text:Submit`, and `find:login button`- `--wait-nav` exists on the top-level CLI commanddiff --git a/docs/reference/fill.md b/docs/reference/fill.mdindex ea459683..f1dfde66 100644--- a/docs/reference/fill.md+++ b/docs/reference/fill.md@@ -20,6 +20,7 @@ pinchtab fill e8 "[email protected]"Notes:- the top-level CLI accepts unified selector forms such as `e8`, `#email`, or `text:Email`+- refs returned for iframe descendants can be filled directly; no manual frame switch is required- for the raw HTTP action endpoint, use `selector` for CSS, XPath, text, or semantic selectors## Related Pagesdiff --git a/docs/reference/snapshot.md b/docs/reference/snapshot.mdindex 435194df..e4860c12 100644--- a/docs/reference/snapshot.md+++ b/docs/reference/snapshot.md@@ -2,6 +2,8 @@Get an accessibility snapshot of the current page, including element refs that can be reused by action commands.+Iframe content is detected automatically during snapshot capture. Same-origin iframe descendants are included beneath the iframe owner element, and their refs can be reused directly with action commands.+```bashcurl "http://localhost:9867/snapshot?filter=interactive"# CLI Alternativediff --git a/internal/bridge/action_resolve.go b/internal/bridge/action_resolve.goindex 1c86f9fe..63a0a24d 100644--- a/internal/bridge/action_resolve.go+++ b/internal/bridge/action_resolve.go@@ -316,8 +316,8 @@ func ResolveUnifiedSelector(ctx context.Context, sel selector.Selector, refCacheswitch sel.Kind {case selector.KindRef:if refCache != nil {- if nid, ok := refCache.Refs[sel.Value]; ok {- return nid, nil+ if target, ok := refCache.Lookup(sel.Value); ok {+ return target.BackendNodeID, nil}}return 0, fmt.Errorf("ref %s not found in snapshot cache", sel.Value)diff --git a/internal/bridge/bridge.go b/internal/bridge/bridge.goindex d5c47f65..46a917b9 100644--- a/internal/bridge/bridge.go+++ b/internal/bridge/bridge.go@@ -30,9 +30,50 @@ type TabEntry struct {ConsoleCaptureEnabled bool}+type RefTarget struct {+ BackendNodeID int64 `json:"backendNodeId"`+ FrameID string `json:"frameId,omitempty"`+ FrameURL string `json:"frameUrl,omitempty"`+ FrameName string `json:"frameName,omitempty"`+}+type RefCache struct {- Refs map[string]int64- Nodes []A11yNode+ Refs map[string]int64+ Targets map[string]RefTarget+ Nodes []A11yNode+}++func (c *RefCache) Lookup(ref string) (RefTarget, bool) {+ if c == nil {+ return RefTarget{}, false+ }+ if c.Targets != nil {+ if target, ok := c.Targets[ref]; ok {+ return target, true+ }+ }+ if c.Refs != nil {+ if nid, ok := c.Refs[ref]; ok {+ return RefTarget{BackendNodeID: nid}, true+ }+ }+ return RefTarget{}, false+}++func RefTargetsFromNodes(nodes []A11yNode) map[string]RefTarget {+ targets := make(map[string]RefTarget, len(nodes))+ for _, node := range nodes {+ if node.Ref == "" || node.NodeID == 0 {+ continue+ }+ targets[node.Ref] = RefTarget{+ BackendNodeID: node.NodeID,+ FrameID: node.FrameID,+ FrameURL: node.FrameURL,+ FrameName: node.FrameName,+ }+ }+ return targets}type Bridge struct {diff --git a/internal/bridge/bridge_test.go b/internal/bridge/bridge_test.goindex d0411f01..57900a1b 100644--- a/internal/bridge/bridge_test.go+++ b/internal/bridge/bridge_test.go@@ -71,6 +71,36 @@ func TestRefCacheLookup(t *testing.T) {}}+func TestRefTargetsFromNodes(t *testing.T) {+ nodes := []A11yNode{+ {+ Ref: "e0",+ NodeID: 100,+ FrameID: "frame-1",+ FrameURL: "https://example.com/embed",+ FrameName: "embed",+ },+ {+ Ref: "e1",+ NodeID: 200,+ },+ }++ targets := RefTargetsFromNodes(nodes)+ if len(targets) != 2 {+ t.Fatalf("expected 2 targets, got %d", len(targets))+ }+ if targets["e0"].BackendNodeID != 100 {+ t.Fatalf("e0 backend node = %d", targets["e0"].BackendNodeID)+ }+ if targets["e0"].FrameID != "frame-1" {+ t.Fatalf("e0 frame id = %q", targets["e0"].FrameID)+ }+ if targets["e1"].BackendNodeID != 200 {+ t.Fatalf("e1 backend node = %d", targets["e1"].BackendNodeID)+ }+}+func TestTabManagerRemoteAllocatorInitialization(t *testing.T) {// Test that TabManager can be initialized without a valid browser context.// This is the case for remote allocators where the browser context isdiff --git a/internal/bridge/cdpops/validation.go b/internal/bridge/cdpops/validation.goindex d636a090..385b40d3 100644--- a/internal/bridge/cdpops/validation.go+++ b/internal/bridge/cdpops/validation.go@@ -62,15 +62,38 @@ func PointerPointForNode(ctx context.Context, backendNodeID int64, requireTopMosconst probeJS = `function() {const r = this.getBoundingClientRect();const style = window.getComputedStyle(this);- const x = r.left + (r.width / 2);- const y = r.top + (r.height / 2);- const inViewport = x >= 0 && y >= 0 && x <= window.innerWidth && y <= window.innerHeight;+ const localX = r.left + (r.width / 2);+ const localY = r.top + (r.height / 2);+ let x = localX;+ let y = localY;+ let topWindow = window;+ try {+ let current = window;+ while (current && current.parent && current !== current.parent) {+ const frameEl = current.frameElement;+ if (!frameEl) {+ break;+ }+ const frameRect = frameEl.getBoundingClientRect();+ x += frameRect.left;+ y += frameRect.top;+ current = current.parent;+ topWindow = current;+ }+ } catch (e) {+ // Cross-origin ancestors can block frame traversal. In that case we keep+ // the frame-local coordinates and let higher layers decide whether that+ // target is safely actionable.+ }+ const viewportWidth = topWindow && topWindow.innerWidth ? topWindow.innerWidth : window.innerWidth;+ const viewportHeight = topWindow && topWindow.innerHeight ? topWindow.innerHeight : window.innerHeight;+ const inViewport = x >= 0 && y >= 0 && x <= viewportWidth && y <= viewportHeight;const visible = !!style && style.display !== 'none' && style.visibility !== 'hidden' && Number(style.opacity || '1') > 0;const pointerEvent = style ? String(style.pointerEvents || '') : '';let occluded = false;let topTag = '';- if (inViewport) {- const top = document.elementFromPoint(x, y);+ if (localX >= 0 && localY >= 0 && localX <= window.innerWidth && localY <= window.innerHeight) {+ const top = document.elementFromPoint(localX, localY);if (top) {topTag = String(top.tagName || '').toLowerCase();const related = top === this || this.contains(top) || top.contains(this);diff --git a/internal/bridge/observe/snapshot.go b/internal/bridge/observe/snapshot.goindex e3996424..65ab39c9 100644--- a/internal/bridge/observe/snapshot.go+++ b/internal/bridge/observe/snapshot.go@@ -6,19 +6,24 @@ import ("fmt""strings"+ "github.com/chromedp/cdproto/cdp"+ "github.com/chromedp/cdproto/dom""github.com/chromedp/chromedp")type A11yNode struct {- Ref string `json:"ref"`- Role string `json:"role"`- Name string `json:"name"`- Depth int `json:"depth"`- Value string `json:"value,omitempty"`- Disabled bool `json:"disabled,omitempty"`- Focused bool `json:"focused,omitempty"`- Hidden bool `json:"hidden,omitempty"`- NodeID int64 `json:"nodeId,omitempty"`+ Ref string `json:"ref"`+ Role string `json:"role"`+ Name string `json:"name"`+ Depth int `json:"depth"`+ Value string `json:"value,omitempty"`+ Disabled bool `json:"disabled,omitempty"`+ Focused bool `json:"focused,omitempty"`+ Hidden bool `json:"hidden,omitempty"`+ NodeID int64 `json:"nodeId,omitempty"`+ FrameID string `json:"frameId,omitempty"`+ FrameURL string `json:"frameUrl,omitempty"`+ FrameName string `json:"frameName,omitempty"`}type RawAXNode struct {@@ -30,16 +35,24 @@ type RawAXNode struct {Properties []RawAXProp `json:"properties"`ChildIDs []string `json:"childIds"`BackendDOMNodeID int64 `json:"backendDOMNodeId"`+ FrameID string `json:"-"`+ FrameURL string `json:"-"`+ FrameName string `json:"-"`+ FrameOwnerNodeID int64 `json:"-"`}type RawAXTreeResponse struct {Nodes []RawAXNode `json:"nodes"`}+type RawFrame struct {+ ID string `json:"id"`+ URL string `json:"url,omitempty"`+ Name string `json:"name,omitempty"`+}+type RawFrameTree struct {- Frame struct {- ID string `json:"id"`- } `json:"frame"`+ Frame RawFrame `json:"frame"`ChildFrames []RawFrameTree `json:"childFrames"`}@@ -59,6 +72,41 @@ func FrameIDs(tree RawFrameTree) []string {return ids}+// FrameMap returns frame metadata keyed by frame id.+func FrameMap(tree RawFrameTree) map[string]RawFrame {+ frames := make(map[string]RawFrame, 1+len(tree.ChildFrames))+ var walk func(RawFrameTree)+ walk = func(t RawFrameTree) {+ if t.Frame.ID != "" {+ frames[t.Frame.ID] = t.Frame+ }+ for _, child := range t.ChildFrames {+ walk(child)+ }+ }+ walk(tree)+ return frames+}++// FrameOwnerMap returns iframe owner backend node IDs keyed by child frame id.+func FrameOwnerMap(ctx context.Context, tree RawFrameTree) map[string]int64 {+ owners := make(map[string]int64, len(tree.ChildFrames))+ var walk func(RawFrameTree)+ walk = func(t RawFrameTree) {+ for _, child := range t.ChildFrames {+ if child.Frame.ID != "" {+ backendNodeID, _, err := dom.GetFrameOwner(cdp.FrameID(child.Frame.ID)).Do(ctx)+ if err == nil && backendNodeID != 0 {+ owners[child.Frame.ID] = int64(backendNodeID)+ }+ }+ walk(child)+ }+ }+ walk(tree)+ return owners+}+// FetchAXTree returns the merged accessibility tree for the current page and any child frames.func FetchAXTree(ctx context.Context) ([]RawAXNode, error) {var frameTreeResult json.RawMessage@@ -75,6 +123,8 @@ func FetchAXTree(ctx context.Context) ([]RawAXNode, error) {return fetchAXTreeForFrame(ctx, "")}+ frameMap := FrameMap(frameResp.FrameTree)+ ownerMap := FrameOwnerMap(ctx, frameResp.FrameTree)ids := FrameIDs(frameResp.FrameTree)if len(ids) == 0 {return fetchAXTreeForFrame(ctx, "")@@ -87,7 +137,12 @@ func FetchAXTree(ctx context.Context) ([]RawAXNode, error) {if err != nil {continue}+ frameMeta := frameMap[id]for _, n := range nodes {+ n.FrameID = id+ n.FrameURL = frameMeta.URL+ n.FrameName = frameMeta.Name+ n.FrameOwnerNodeID = ownerMap[id]key := n.NodeIDif key == "" {key = fmt.Sprintf("backend:%d:%s:%s", n.BackendDOMNodeID, n.Role.String(), n.Name.String())@@ -153,7 +208,7 @@ var InteractiveRoles = map[string]bool{"combobox": true, "listbox": true, "option": true, "checkbox": true,"radio": true, "switch": true, "slider": true, "spinbutton": true,"menuitem": true, "menuitemcheckbox": true, "menuitemradio": true,- "tab": true, "treeitem": true,+ "tab": true, "treeitem": true, "iframe": true, "Iframe": true,}const FilterInteractive = "interactive"@@ -171,26 +226,33 @@ func isAXNodeHidden(n RawAXNode) bool {}func BuildSnapshot(nodes []RawAXNode, filter string, maxDepth int) ([]A11yNode, map[string]int64) {+ nodeByID := make(map[string]RawAXNode, len(nodes))parentMap := make(map[string]string)+ childMap := make(map[string][]string, len(nodes))+ backendToAX := make(map[int64]string, len(nodes))+ frameRoots := make(map[string][]string, 4)+ frameOwners := make(map[string]int64, 4)+ frameOrder := make([]string, 0, 4)+ seenFrames := make(map[string]bool, 4)+for _, n := range nodes {+ nodeByID[n.NodeID] = n+ childMap[n.NodeID] = append(childMap[n.NodeID], n.ChildIDs...)+ if n.BackendDOMNodeID != 0 {+ backendToAX[n.BackendDOMNodeID] = n.NodeID+ }+ if !seenFrames[n.FrameID] {+ frameOrder = append(frameOrder, n.FrameID)+ seenFrames[n.FrameID] = true+ }+ if n.FrameOwnerNodeID != 0 && frameOwners[n.FrameID] == 0 {+ frameOwners[n.FrameID] = n.FrameOwnerNodeID+ }for _, childID := range n.ChildIDs {parentMap[childID] = n.NodeID}}maxAncestorWalk := max(len(parentMap)+1, 1)- depthOf := func(nodeID string) int {- d := 0- cur := nodeID- for range maxAncestorWalk {- p, ok := parentMap[cur]- if !ok {- break- }- d++- cur = p- }- return d- }// Build a set of AX node IDs that are hidden, including inherited hidden// status from ancestors. A child of a hidden node is also hidden.@@ -216,39 +278,65 @@ func BuildSnapshot(nodes []RawAXNode, filter string, maxDepth int) ([]A11yNode,return false}- flat := make([]A11yNode, 0)- refs := make(map[string]int64)- refID := 0-for _, n := range nodes {- if n.Ignored {+ parentID, ok := parentMap[n.NodeID]+ if !ok {+ frameRoots[n.FrameID] = append(frameRoots[n.FrameID], n.NodeID)continue}-- role := n.Role.String()- name := n.Name.String()-- if role == "none" || role == "generic" || role == "InlineTextBox" {- continue+ parentNode, ok := nodeByID[parentID]+ if !ok || parentNode.FrameID != n.FrameID {+ frameRoots[n.FrameID] = append(frameRoots[n.FrameID], n.NodeID)}- if name == "" && role == "StaticText" {- continue+ }++ rootFrameID := ""+ for _, frameID := range frameOrder {+ if frameOwners[frameID] == 0 {+ rootFrameID = frameID+ break}+ }+ if rootFrameID == "" && len(frameOrder) > 0 {+ rootFrameID = frameOrder[0]+ }- depth := depthOf(n.NodeID)- if maxDepth >= 0 && depth > maxDepth {+ topRoots := make([]string, 0, len(nodes))+ frameChildRoots := make(map[string][]string)+ for _, frameID := range frameOrder {+ roots := frameRoots[frameID]+ if len(roots) == 0 {continue}- if filter == FilterInteractive && !InteractiveRoles[role] {+ ownerBackendID := frameOwners[frameID]+ ownerAXID := backendToAX[ownerBackendID]+ if frameID == rootFrameID || ownerAXID == "" {+ topRoots = append(topRoots, roots...)continue}+ frameChildRoots[ownerAXID] = append(frameChildRoots[ownerAXID], roots...)+ }+ if len(topRoots) == 0 {+ for _, n := range nodes {+ topRoots = append(topRoots, n.NodeID)+ }+ }+ flat := make([]A11yNode, 0)+ refs := make(map[string]int64)+ refID := 0+ appendNode := func(n RawAXNode, depth int) {+ role := n.Role.String()+ name := n.Name.String()ref := fmt.Sprintf("e%d", refID)entry := A11yNode{- Ref: ref,- Role: role,- Name: name,- Depth: depth,+ Ref: ref,+ Role: role,+ Name: name,+ Depth: depth,+ FrameID: n.FrameID,+ FrameURL: n.FrameURL,+ FrameName: n.FrameName,}if v := n.Value.String(); v != "" {@@ -279,6 +367,73 @@ func BuildSnapshot(nodes []RawAXNode, filter string, maxDepth int) ([]A11yNode,refID++}+ isSkippableNode := func(n RawAXNode) bool {+ role := n.Role.String()+ name := n.Name.String()+ if n.Ignored {+ return true+ }+ if role == "none" || role == "generic" || role == "InlineTextBox" {+ return true+ }+ if name == "" && role == "StaticText" {+ return true+ }+ return false+ }++ isFlattenableFrameRoot := func(n RawAXNode) bool {+ role := n.Role.String()+ return n.FrameOwnerNodeID != 0 && (role == "WebArea" || role == "RootWebArea")+ }++ visited := make(map[string]bool, len(nodes))+ var visitFrameRoot func(string, int)+ var visit func(string, int)++ visit = func(nodeID string, depth int) {+ if nodeID == "" || visited[nodeID] {+ return+ }+ n, ok := nodeByID[nodeID]+ if !ok {+ return+ }+ visited[nodeID] = true+ if maxDepth >= 0 && depth > maxDepth {+ return+ }++ if !isSkippableNode(n) && (filter != FilterInteractive || InteractiveRoles[n.Role.String()]) {+ appendNode(n, depth)+ }++ for _, childID := range childMap[nodeID] {+ visit(childID, depth+1)+ }+ for _, frameRootID := range frameChildRoots[nodeID] {+ visitFrameRoot(frameRootID, depth+1)+ }+ }++ visitFrameRoot = func(nodeID string, depth int) {+ n, ok := nodeByID[nodeID]+ if !ok {+ return+ }+ if isFlattenableFrameRoot(n) {+ for _, childID := range childMap[nodeID] {+ visit(childID, depth)+ }+ return+ }+ visit(nodeID, depth)+ }++ for _, rootID := range topRoots {+ visit(rootID, 0)+ }+return flat, refs}diff --git a/internal/bridge/observe_facade.go b/internal/bridge/observe_facade.goindex c8c434d8..8d9217d3 100644--- a/internal/bridge/observe_facade.go+++ b/internal/bridge/observe_facade.go@@ -17,6 +17,7 @@ type A11yNode = bridgeobserve.A11yNodetype RawAXNode = bridgeobserve.RawAXNodetype RawAXValue = bridgeobserve.RawAXValuetype RawAXProp = bridgeobserve.RawAXProp+type RawFrame = bridgeobserve.RawFrametype rawFrameTree = bridgeobserve.RawFrameTreetype NetworkEntry = bridgeobserve.NetworkEntry@@ -29,6 +30,10 @@ func frameIDs(tree rawFrameTree) []string {return bridgeobserve.FrameIDs(tree)}+func FrameMap(tree rawFrameTree) map[string]RawFrame {+ return bridgeobserve.FrameMap(tree)+}+func FetchAXTree(ctx context.Context) ([]RawAXNode, error) {return bridgeobserve.FetchAXTree(ctx)}diff --git a/internal/bridge/snapshot_extra_test.go b/internal/bridge/snapshot_extra_test.goindex 282f1cf3..7d39c546 100644--- a/internal/bridge/snapshot_extra_test.go+++ b/internal/bridge/snapshot_extra_test.go@@ -239,16 +239,10 @@ func TestFrameIDs_Recursive(t *testing.T) {tree := rawFrameTree{}tree.Frame.ID = "root"tree.ChildFrames = []rawFrameTree{- {Frame: struct {- ID string `json:"id"`- }{ID: "child-1"}},+ {Frame: RawFrame{ID: "child-1"}},{- Frame: struct {- ID string `json:"id"`- }{ID: "child-2"},- ChildFrames: []rawFrameTree{{Frame: struct {- ID string `json:"id"`- }{ID: "grandchild"}}},+ Frame: RawFrame{ID: "child-2"},+ ChildFrames: []rawFrameTree{{Frame: RawFrame{ID: "grandchild"}}},},}got := frameIDs(tree)@@ -262,3 +256,22 @@ func TestFrameIDs_Recursive(t *testing.T) {}}}++func TestFrameMap_Recursive(t *testing.T) {+ tree := rawFrameTree{+ Frame: RawFrame{ID: "root", URL: "https://example.com", Name: "main"},+ ChildFrames: []rawFrameTree{+ {Frame: RawFrame{ID: "child-1", URL: "https://example.com/embed", Name: "embed"}},+ },+ }+ got := FrameMap(tree)+ if len(got) != 2 {+ t.Fatalf("expected 2 frames, got %d", len(got))+ }+ if got["root"].URL != "https://example.com" {+ t.Fatalf("root frame url = %q", got["root"].URL)+ }+ if got["child-1"].Name != "embed" {+ t.Fatalf("child frame name = %q", got["child-1"].Name)+ }+}diff --git a/internal/bridge/snapshot_test.go b/internal/bridge/snapshot_test.goindex a06cc91d..601b04c8 100644--- a/internal/bridge/snapshot_test.go+++ b/internal/bridge/snapshot_test.go@@ -318,3 +318,149 @@ func TestBuildSnapshotDepthFilter(t *testing.T) {t.Fatalf("expected 2 nodes at depth<=1, got %d: %+v", len(flat), flat)}}++func TestBuildSnapshotPreservesFrameMetadata(t *testing.T) {+ nodes := []RawAXNode{+ {+ NodeID: "frame-node",+ Role: &RawAXValue{Value: json.RawMessage(`"textbox"`)},+ Name: &RawAXValue{Value: json.RawMessage(`"Card number"`)},+ BackendDOMNodeID: 42,+ FrameID: "frame-payment",+ FrameURL: "https://payments.example/frame",+ FrameName: "payment-frame",+ },+ }++ flat, refs := BuildSnapshot(nodes, "", -1)+ if len(flat) != 1 {+ t.Fatalf("expected 1 node, got %d", len(flat))+ }+ if refs["e0"] != 42 {+ t.Fatalf("expected e0 to resolve to backend node 42, got %d", refs["e0"])+ }+ if flat[0].FrameID != "frame-payment" {+ t.Fatalf("frame id = %q, want %q", flat[0].FrameID, "frame-payment")+ }+ if flat[0].FrameURL != "https://payments.example/frame" {+ t.Fatalf("frame url = %q, want %q", flat[0].FrameURL, "https://payments.example/frame")+ }+ if flat[0].FrameName != "payment-frame" {+ t.Fatalf("frame name = %q, want %q", flat[0].FrameName, "payment-frame")+ }+}++func TestBuildSnapshotNestsFrameContentUnderOwner(t *testing.T) {+ nodes := []RawAXNode{+ {+ NodeID: "root",+ Role: &RawAXValue{Value: json.RawMessage(`"WebArea"`)},+ Name: &RawAXValue{Value: json.RawMessage(`"Outer"`)},+ ChildIDs: []string{"iframe"},+ BackendDOMNodeID: 1,+ FrameID: "main",+ },+ {+ NodeID: "iframe",+ Role: &RawAXValue{Value: json.RawMessage(`"Iframe"`)},+ Name: &RawAXValue{Value: json.RawMessage(`"payment-frame"`)},+ BackendDOMNodeID: 10,+ FrameID: "main",+ },+ {+ NodeID: "child-root",+ Role: &RawAXValue{Value: json.RawMessage(`"WebArea"`)},+ Name: &RawAXValue{Value: json.RawMessage(`"Inner"`)},+ ChildIDs: []string{"card", "pay"},+ BackendDOMNodeID: 11,+ FrameID: "child",+ FrameOwnerNodeID: 10,+ },+ {+ NodeID: "card",+ Role: &RawAXValue{Value: json.RawMessage(`"textbox"`)},+ Name: &RawAXValue{Value: json.RawMessage(`"Card number"`)},+ BackendDOMNodeID: 12,+ FrameID: "child",+ FrameOwnerNodeID: 10,+ },+ {+ NodeID: "pay",+ Role: &RawAXValue{Value: json.RawMessage(`"button"`)},+ Name: &RawAXValue{Value: json.RawMessage(`"Pay"`)},+ BackendDOMNodeID: 13,+ FrameID: "child",+ FrameOwnerNodeID: 10,+ },+ }++ flat, _ := BuildSnapshot(nodes, "", -1)+ if len(flat) != 4 {+ t.Fatalf("expected 4 visible nodes, got %d: %+v", len(flat), flat)+ }+ if flat[1].Name != "payment-frame" {+ t.Fatalf("second node = %q, want iframe owner", flat[1].Name)+ }+ if flat[2].Name != "Card number" || flat[2].Depth != 2 {+ t.Fatalf("textbox = %+v, want nested child at depth 2", flat[2])+ }+ if flat[3].Name != "Pay" || flat[3].Depth != 2 {+ t.Fatalf("button = %+v, want nested child at depth 2", flat[3])+ }+}++func TestBuildSnapshotInteractiveIncludesIframeOwnerAndChildActions(t *testing.T) {+ nodes := []RawAXNode{+ {+ NodeID: "root",+ Role: &RawAXValue{Value: json.RawMessage(`"WebArea"`)},+ Name: &RawAXValue{Value: json.RawMessage(`"Outer"`)},+ ChildIDs: []string{"iframe"},+ BackendDOMNodeID: 1,+ FrameID: "main",+ },+ {+ NodeID: "iframe",+ Role: &RawAXValue{Value: json.RawMessage(`"Iframe"`)},+ Name: &RawAXValue{Value: json.RawMessage(`"payment-frame"`)},+ BackendDOMNodeID: 10,+ FrameID: "main",+ },+ {+ NodeID: "child-root",+ Role: &RawAXValue{Value: json.RawMessage(`"WebArea"`)},+ Name: &RawAXValue{Value: json.RawMessage(`"Inner"`)},+ ChildIDs: []string{"card", "pay"},+ BackendDOMNodeID: 11,+ FrameID: "child",+ FrameOwnerNodeID: 10,+ },+ {+ NodeID: "card",+ Role: &RawAXValue{Value: json.RawMessage(`"textbox"`)},+ Name: &RawAXValue{Value: json.RawMessage(`"Card number"`)},+ BackendDOMNodeID: 12,+ FrameID: "child",+ FrameOwnerNodeID: 10,+ },+ {+ NodeID: "pay",+ Role: &RawAXValue{Value: json.RawMessage(`"button"`)},+ Name: &RawAXValue{Value: json.RawMessage(`"Pay"`)},+ BackendDOMNodeID: 13,+ FrameID: "child",+ FrameOwnerNodeID: 10,+ },+ }++ flat, _ := BuildSnapshot(nodes, FilterInteractive, -1)+ if len(flat) != 3 {+ t.Fatalf("expected iframe + 2 actionable children, got %d: %+v", len(flat), flat)+ }+ if flat[0].Role != "Iframe" {+ t.Fatalf("expected iframe owner in interactive snapshot, got %+v", flat[0])+ }+ if flat[1].Name != "Card number" || flat[2].Name != "Pay" {+ t.Fatalf("unexpected interactive descendants: %+v", flat)+ }+}diff --git a/internal/handlers/actions.go b/internal/handlers/actions.goindex 2a54570a..c739b5ed 100644--- a/internal/handlers/actions.go+++ b/internal/handlers/actions.go@@ -184,8 +184,8 @@ func (h *Handlers) HandleAction(w http.ResponseWriter, r *http.Request) {req.Selector = ""cache := h.Bridge.GetRefCache(resolvedTabID)if cache != nil {- if nid, ok := cache.Refs[sel.Value]; ok {- req.NodeID = nid+ if target, ok := cache.Lookup(sel.Value); ok {+ req.NodeID = target.BackendNodeID}}if req.NodeID == 0 {@@ -242,8 +242,8 @@ func (h *Handlers) HandleAction(w http.ResponseWriter, r *http.Request) {req.Ref = result.BestRefcache := h.Bridge.GetRefCache(resolvedTabID)if cache != nil {- if nid, ok := cache.Refs[result.BestRef]; ok {- req.NodeID = nid+ if target, ok := cache.Lookup(result.BestRef); ok {+ req.NodeID = target.BackendNodeID}}}@@ -303,8 +303,8 @@ func (h *Handlers) HandleAction(w http.ResponseWriter, r *http.Request) {recordStaleRefRetry()h.refreshRefCache(tCtx, resolvedTabID)if cache := h.Bridge.GetRefCache(resolvedTabID); cache != nil {- if nid, ok := cache.Refs[req.Ref]; ok {- req.NodeID = nid+ if target, ok := cache.Lookup(req.Ref); ok {+ req.NodeID = target.BackendNodeID}}}@@ -535,8 +535,8 @@ func (h *Handlers) handleActionsBatch(w http.ResponseWriter, r *http.Request, reaction.Selector = ""cache := h.Bridge.GetRefCache(resolvedTabID)if cache != nil {- if nid, ok := cache.Refs[sel.Value]; ok {- action.NodeID = nid+ if target, ok := cache.Lookup(sel.Value); ok {+ action.NodeID = target.BackendNodeID}}if action.NodeID == 0 {@@ -598,8 +598,8 @@ func (h *Handlers) handleActionsBatch(w http.ResponseWriter, r *http.Request, reaction.Ref = findResult.BestRefcache := h.Bridge.GetRefCache(resolvedTabID)if cache != nil {- if nid, ok := cache.Refs[findResult.BestRef]; ok {- action.NodeID = nid+ if target, ok := cache.Lookup(findResult.BestRef); ok {+ action.NodeID = target.BackendNodeID}}}@@ -689,8 +689,8 @@ func (h *Handlers) handleActionsBatch(w http.ResponseWriter, r *http.Request, rerecordStaleRefRetry()h.refreshRefCache(tCtx, resolvedTabID)if cache := h.Bridge.GetRefCache(resolvedTabID); cache != nil {- if nid, ok := cache.Refs[action.Ref]; ok {- action.NodeID = nid+ if target, ok := cache.Lookup(action.Ref); ok {+ action.NodeID = target.BackendNodeID}}}@@ -814,8 +814,8 @@ func (h *Handlers) HandleMacro(w http.ResponseWriter, r *http.Request) {step.Selector = ""cache := h.Bridge.GetRefCache(resolvedTabID)if cache != nil {- if nid, ok := cache.Refs[sel.Value]; ok {- step.NodeID = nid+ if target, ok := cache.Lookup(sel.Value); ok {+ step.NodeID = target.BackendNodeID}}if step.NodeID == 0 {@@ -880,8 +880,8 @@ func (h *Handlers) HandleMacro(w http.ResponseWriter, r *http.Request) {step.Ref = findResult.BestRefcache := h.Bridge.GetRefCache(resolvedTabID)if cache != nil {- if nid, ok := cache.Refs[findResult.BestRef]; ok {- step.NodeID = nid+ if target, ok := cache.Lookup(findResult.BestRef); ok {+ step.NodeID = target.BackendNodeID}}}@@ -956,8 +956,8 @@ func (h *Handlers) HandleMacro(w http.ResponseWriter, r *http.Request) {recordStaleRefRetry()h.refreshRefCache(tCtx, resolvedTabID)if cache := h.Bridge.GetRefCache(resolvedTabID); cache != nil {- if nid, ok := cache.Refs[step.Ref]; ok {- step.NodeID = nid+ if target, ok := cache.Lookup(step.Ref); ok {+ step.NodeID = target.BackendNodeID}}}@@ -1164,5 +1164,9 @@ func (h *Handlers) refreshRefCache(ctx context.Context, tabID string) {return}flat, refs := bridge.BuildSnapshot(nodes, bridge.FilterInteractive, -1)- h.Bridge.SetRefCache(tabID, &bridge.RefCache{Refs: refs, Nodes: flat})+ h.Bridge.SetRefCache(tabID, &bridge.RefCache{+ Refs: refs,+ Targets: bridge.RefTargetsFromNodes(flat),+ Nodes: flat,+ })}diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.goindex 1d72b288..100a3fc1 100644--- a/internal/handlers/handlers.go+++ b/internal/handlers/handlers.go@@ -73,8 +73,8 @@ func New(b bridge.BridgeAPI, cfg *config.RuntimeConfig, p bridge.ProfileService,if cache == nil {return 0, false}- nid, ok := cache.Refs[ref]- return nid, ok+ target, ok := cache.Lookup(ref)+ return target.BackendNodeID, ok},// DescriptorBuilderfunc(tabID string) []semantic.ElementDescriptor {diff --git a/internal/handlers/snapshot.go b/internal/handlers/snapshot.goindex 83b0b318..a79eadfe 100644--- a/internal/handlers/snapshot.go+++ b/internal/handlers/snapshot.go@@ -189,7 +189,11 @@ func (h *Handlers) HandleSnapshot(w http.ResponseWriter, r *http.Request) {}}- h.Bridge.SetRefCache(resolvedTabID, &bridge.RefCache{Refs: refs, Nodes: flat})+ h.Bridge.SetRefCache(resolvedTabID, &bridge.RefCache{+ Refs: refs,+ Targets: bridge.RefTargetsFromNodes(flat),+ Nodes: flat,+ })var url, title string_ = chromedp.Run(tCtx,diff --git a/tests/e2e/fixtures/iframe-content.html b/tests/e2e/fixtures/iframe-content.htmlnew file mode 100644index 00000000..bce47a9a--- /dev/null+++ b/tests/e2e/fixtures/iframe-content.html@@ -0,0 +1,23 @@+<!DOCTYPE html>+<html lang="en">+<head>+ <meta charset="UTF-8">+ <title>Iframe Payment Content</title>+</head>+<body>+ <h2>Payment</h2>+ <form id="payment-form">+ <label for="card-number">Card number</label>+ <input type="text" id="card-number" name="cardNumber">+ <button type="button" id="pay-button">Pay</button>+ </form>+ <p id="payment-result"></p>++ <script>+ document.getElementById('pay-button').addEventListener('click', function() {+ const value = document.getElementById('card-number').value;+ document.getElementById('payment-result').textContent = 'PAYMENT_SUBMITTED_' + value;+ });+ </script>+</body>+</html>diff --git a/tests/e2e/fixtures/iframe.html b/tests/e2e/fixtures/iframe.htmlnew file mode 100644index 00000000..822e7a84--- /dev/null+++ b/tests/e2e/fixtures/iframe.html@@ -0,0 +1,18 @@+<!DOCTYPE html>+<html lang="en">+<head>+ <meta charset="UTF-8">+ <title>Iframe Fixture</title>+</head>+<body>+ <h1>Iframe Fixture</h1>+ <p>The iframe below simulates an embedded payment form.</p>+ <iframe+ id="payment-frame"+ title="payment-frame"+ src="iframe-content.html"+ width="600"+ height="300"+ ></iframe>+</body>+</html>diff --git a/tests/e2e/scenarios/api/actions-extended.sh b/tests/e2e/scenarios/api/actions-extended.shindex 67144cc5..8172add4 100644--- a/tests/e2e/scenarios/api/actions-extended.sh+++ b/tests/e2e/scenarios/api/actions-extended.sh@@ -172,6 +172,29 @@ require_ref "textbox" "Notes" NOTES_REF && {end_test+# ─────────────────────────────────────────────────────────────────+start_test "iframe: snapshot refs support fill and click inside iframe"++navigate_fixture "iframe.html"+fresh_snapshot++assert_json_exists "$RESULT" '.nodes[] | select(.name == "payment-frame")' "snapshot includes iframe owner"++require_ref "textbox" "Card number" CARD_REF && \+require_ref "button" "Pay" PAY_REF && {+ pt_post /action -d "{\"kind\":\"fill\",\"ref\":\"${CARD_REF}\",\"text\":\"4111111111111111\"}"+ assert_ok "fill iframe textbox by ref"++ pt_post /action -d "{\"kind\":\"click\",\"ref\":\"${PAY_REF}\"}"+ assert_ok "click iframe button by ref"++ pt_post /evaluate -d '{"expression":"(() => { const frame = document.getElementById(\"payment-frame\"); const doc = frame && frame.contentDocument; const result = doc && doc.getElementById(\"payment-result\"); return result ? result.textContent : \"\"; })()"}'+ assert_ok "evaluate iframe result"+ assert_json_eq "$RESULT" '.result' 'PAYMENT_SUBMITTED_4111111111111111' "iframe form submitted via refs"+}++end_test+# Regression test for GitHub issue #236: press action was typing key names# as literal text instead of dispatching keyboard events.