Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

feat: basic iframe support

Luigi Agostid61801450c98
diff --git a/docs/reference/click.md b/docs/reference/click.md
index 883ed706..e71bf641 100644
--- a/docs/reference/click.md
+++ b/docs/reference/click.md
@@ -20,6 +20,7 @@ pinchtab click e5
Notes:
- element refs come from `/snapshot`
+- refs returned for iframe descendants can be clicked directly; no manual frame switch is required
- the raw action endpoint also accepts `selector`, for example `{"kind":"click","selector":"#login"}`
- the CLI also accepts `#login`, `xpath://button`, `text:Submit`, and `find:login button`
- `--wait-nav` exists on the top-level CLI command
diff --git a/docs/reference/fill.md b/docs/reference/fill.md
index ea459683..f1dfde66 100644
--- a/docs/reference/fill.md
+++ b/docs/reference/fill.md
@@ -20,6 +20,7 @@ pinchtab fill e8 "[email protected]"
Notes:
- the top-level CLI accepts unified selector forms such as `e8`, `#email`, or `text:Email`
+- refs returned for iframe descendants can be filled directly; no manual frame switch is required
- for the raw HTTP action endpoint, use `selector` for CSS, XPath, text, or semantic selectors
## Related Pages
diff --git a/docs/reference/snapshot.md b/docs/reference/snapshot.md
index 435194df..e4860c12 100644
--- a/docs/reference/snapshot.md
+++ b/docs/reference/snapshot.md
@@ -2,6 +2,8 @@
Get an accessibility snapshot of the current page, including element refs that can be reused by action commands.
+Iframe content is detected automatically during snapshot capture. Same-origin iframe descendants are included beneath the iframe owner element, and their refs can be reused directly with action commands.
+
```bash
curl "http://localhost:9867/snapshot?filter=interactive"
# CLI Alternative
diff --git a/internal/bridge/action_resolve.go b/internal/bridge/action_resolve.go
index 1c86f9fe..63a0a24d 100644
--- a/internal/bridge/action_resolve.go
+++ b/internal/bridge/action_resolve.go
@@ -316,8 +316,8 @@ func ResolveUnifiedSelector(ctx context.Context, sel selector.Selector, refCache
switch sel.Kind {
case selector.KindRef:
if refCache != nil {
- if nid, ok := refCache.Refs[sel.Value]; ok {
- return nid, nil
+ if target, ok := refCache.Lookup(sel.Value); ok {
+ return target.BackendNodeID, nil
}
}
return 0, fmt.Errorf("ref %s not found in snapshot cache", sel.Value)
diff --git a/internal/bridge/bridge.go b/internal/bridge/bridge.go
index d5c47f65..46a917b9 100644
--- a/internal/bridge/bridge.go
+++ b/internal/bridge/bridge.go
@@ -30,9 +30,50 @@ type TabEntry struct {
ConsoleCaptureEnabled bool
}
+type RefTarget struct {
+ BackendNodeID int64 `json:"backendNodeId"`
+ FrameID string `json:"frameId,omitempty"`
+ FrameURL string `json:"frameUrl,omitempty"`
+ FrameName string `json:"frameName,omitempty"`
+}
+
type RefCache struct {
- Refs map[string]int64
- Nodes []A11yNode
+ Refs map[string]int64
+ Targets map[string]RefTarget
+ Nodes []A11yNode
+}
+
+func (c *RefCache) Lookup(ref string) (RefTarget, bool) {
+ if c == nil {
+ return RefTarget{}, false
+ }
+ if c.Targets != nil {
+ if target, ok := c.Targets[ref]; ok {
+ return target, true
+ }
+ }
+ if c.Refs != nil {
+ if nid, ok := c.Refs[ref]; ok {
+ return RefTarget{BackendNodeID: nid}, true
+ }
+ }
+ return RefTarget{}, false
+}
+
+func RefTargetsFromNodes(nodes []A11yNode) map[string]RefTarget {
+ targets := make(map[string]RefTarget, len(nodes))
+ for _, node := range nodes {
+ if node.Ref == "" || node.NodeID == 0 {
+ continue
+ }
+ targets[node.Ref] = RefTarget{
+ BackendNodeID: node.NodeID,
+ FrameID: node.FrameID,
+ FrameURL: node.FrameURL,
+ FrameName: node.FrameName,
+ }
+ }
+ return targets
}
type Bridge struct {
diff --git a/internal/bridge/bridge_test.go b/internal/bridge/bridge_test.go
index d0411f01..57900a1b 100644
--- a/internal/bridge/bridge_test.go
+++ b/internal/bridge/bridge_test.go
@@ -71,6 +71,36 @@ func TestRefCacheLookup(t *testing.T) {
}
}
+func TestRefTargetsFromNodes(t *testing.T) {
+ nodes := []A11yNode{
+ {
+ Ref: "e0",
+ NodeID: 100,
+ FrameID: "frame-1",
+ FrameURL: "https://example.com/embed",
+ FrameName: "embed",
+ },
+ {
+ Ref: "e1",
+ NodeID: 200,
+ },
+ }
+
+ targets := RefTargetsFromNodes(nodes)
+ if len(targets) != 2 {
+ t.Fatalf("expected 2 targets, got %d", len(targets))
+ }
+ if targets["e0"].BackendNodeID != 100 {
+ t.Fatalf("e0 backend node = %d", targets["e0"].BackendNodeID)
+ }
+ if targets["e0"].FrameID != "frame-1" {
+ t.Fatalf("e0 frame id = %q", targets["e0"].FrameID)
+ }
+ if targets["e1"].BackendNodeID != 200 {
+ t.Fatalf("e1 backend node = %d", targets["e1"].BackendNodeID)
+ }
+}
+
func TestTabManagerRemoteAllocatorInitialization(t *testing.T) {
// Test that TabManager can be initialized without a valid browser context.
// This is the case for remote allocators where the browser context is
diff --git a/internal/bridge/cdpops/validation.go b/internal/bridge/cdpops/validation.go
index d636a090..385b40d3 100644
--- a/internal/bridge/cdpops/validation.go
+++ b/internal/bridge/cdpops/validation.go
@@ -62,15 +62,38 @@ func PointerPointForNode(ctx context.Context, backendNodeID int64, requireTopMos
const probeJS = `function() {
const r = this.getBoundingClientRect();
const style = window.getComputedStyle(this);
- const x = r.left + (r.width / 2);
- const y = r.top + (r.height / 2);
- const inViewport = x >= 0 && y >= 0 && x <= window.innerWidth && y <= window.innerHeight;
+ const localX = r.left + (r.width / 2);
+ const localY = r.top + (r.height / 2);
+ let x = localX;
+ let y = localY;
+ let topWindow = window;
+ try {
+ let current = window;
+ while (current && current.parent && current !== current.parent) {
+ const frameEl = current.frameElement;
+ if (!frameEl) {
+ break;
+ }
+ const frameRect = frameEl.getBoundingClientRect();
+ x += frameRect.left;
+ y += frameRect.top;
+ current = current.parent;
+ topWindow = current;
+ }
+ } catch (e) {
+ // Cross-origin ancestors can block frame traversal. In that case we keep
+ // the frame-local coordinates and let higher layers decide whether that
+ // target is safely actionable.
+ }
+ const viewportWidth = topWindow && topWindow.innerWidth ? topWindow.innerWidth : window.innerWidth;
+ const viewportHeight = topWindow && topWindow.innerHeight ? topWindow.innerHeight : window.innerHeight;
+ const inViewport = x >= 0 && y >= 0 && x <= viewportWidth && y <= viewportHeight;
const visible = !!style && style.display !== 'none' && style.visibility !== 'hidden' && Number(style.opacity || '1') > 0;
const pointerEvent = style ? String(style.pointerEvents || '') : '';
let occluded = false;
let topTag = '';
- if (inViewport) {
- const top = document.elementFromPoint(x, y);
+ if (localX >= 0 && localY >= 0 && localX <= window.innerWidth && localY <= window.innerHeight) {
+ const top = document.elementFromPoint(localX, localY);
if (top) {
topTag = String(top.tagName || '').toLowerCase();
const related = top === this || this.contains(top) || top.contains(this);
diff --git a/internal/bridge/observe/snapshot.go b/internal/bridge/observe/snapshot.go
index e3996424..65ab39c9 100644
--- a/internal/bridge/observe/snapshot.go
+++ b/internal/bridge/observe/snapshot.go
@@ -6,19 +6,24 @@ import (
"fmt"
"strings"
+ "github.com/chromedp/cdproto/cdp"
+ "github.com/chromedp/cdproto/dom"
"github.com/chromedp/chromedp"
)
type A11yNode struct {
- Ref string `json:"ref"`
- Role string `json:"role"`
- Name string `json:"name"`
- Depth int `json:"depth"`
- Value string `json:"value,omitempty"`
- Disabled bool `json:"disabled,omitempty"`
- Focused bool `json:"focused,omitempty"`
- Hidden bool `json:"hidden,omitempty"`
- NodeID int64 `json:"nodeId,omitempty"`
+ Ref string `json:"ref"`
+ Role string `json:"role"`
+ Name string `json:"name"`
+ Depth int `json:"depth"`
+ Value string `json:"value,omitempty"`
+ Disabled bool `json:"disabled,omitempty"`
+ Focused bool `json:"focused,omitempty"`
+ Hidden bool `json:"hidden,omitempty"`
+ NodeID int64 `json:"nodeId,omitempty"`
+ FrameID string `json:"frameId,omitempty"`
+ FrameURL string `json:"frameUrl,omitempty"`
+ FrameName string `json:"frameName,omitempty"`
}
type RawAXNode struct {
@@ -30,16 +35,24 @@ type RawAXNode struct {
Properties []RawAXProp `json:"properties"`
ChildIDs []string `json:"childIds"`
BackendDOMNodeID int64 `json:"backendDOMNodeId"`
+ FrameID string `json:"-"`
+ FrameURL string `json:"-"`
+ FrameName string `json:"-"`
+ FrameOwnerNodeID int64 `json:"-"`
}
type RawAXTreeResponse struct {
Nodes []RawAXNode `json:"nodes"`
}
+type RawFrame struct {
+ ID string `json:"id"`
+ URL string `json:"url,omitempty"`
+ Name string `json:"name,omitempty"`
+}
+
type RawFrameTree struct {
- Frame struct {
- ID string `json:"id"`
- } `json:"frame"`
+ Frame RawFrame `json:"frame"`
ChildFrames []RawFrameTree `json:"childFrames"`
}
@@ -59,6 +72,41 @@ func FrameIDs(tree RawFrameTree) []string {
return ids
}
+// FrameMap returns frame metadata keyed by frame id.
+func FrameMap(tree RawFrameTree) map[string]RawFrame {
+ frames := make(map[string]RawFrame, 1+len(tree.ChildFrames))
+ var walk func(RawFrameTree)
+ walk = func(t RawFrameTree) {
+ if t.Frame.ID != "" {
+ frames[t.Frame.ID] = t.Frame
+ }
+ for _, child := range t.ChildFrames {
+ walk(child)
+ }
+ }
+ walk(tree)
+ return frames
+}
+
+// FrameOwnerMap returns iframe owner backend node IDs keyed by child frame id.
+func FrameOwnerMap(ctx context.Context, tree RawFrameTree) map[string]int64 {
+ owners := make(map[string]int64, len(tree.ChildFrames))
+ var walk func(RawFrameTree)
+ walk = func(t RawFrameTree) {
+ for _, child := range t.ChildFrames {
+ if child.Frame.ID != "" {
+ backendNodeID, _, err := dom.GetFrameOwner(cdp.FrameID(child.Frame.ID)).Do(ctx)
+ if err == nil && backendNodeID != 0 {
+ owners[child.Frame.ID] = int64(backendNodeID)
+ }
+ }
+ walk(child)
+ }
+ }
+ walk(tree)
+ return owners
+}
+
// FetchAXTree returns the merged accessibility tree for the current page and any child frames.
func FetchAXTree(ctx context.Context) ([]RawAXNode, error) {
var frameTreeResult json.RawMessage
@@ -75,6 +123,8 @@ func FetchAXTree(ctx context.Context) ([]RawAXNode, error) {
return fetchAXTreeForFrame(ctx, "")
}
+ frameMap := FrameMap(frameResp.FrameTree)
+ ownerMap := FrameOwnerMap(ctx, frameResp.FrameTree)
ids := FrameIDs(frameResp.FrameTree)
if len(ids) == 0 {
return fetchAXTreeForFrame(ctx, "")
@@ -87,7 +137,12 @@ func FetchAXTree(ctx context.Context) ([]RawAXNode, error) {
if err != nil {
continue
}
+ frameMeta := frameMap[id]
for _, n := range nodes {
+ n.FrameID = id
+ n.FrameURL = frameMeta.URL
+ n.FrameName = frameMeta.Name
+ n.FrameOwnerNodeID = ownerMap[id]
key := n.NodeID
if key == "" {
key = fmt.Sprintf("backend:%d:%s:%s", n.BackendDOMNodeID, n.Role.String(), n.Name.String())
@@ -153,7 +208,7 @@ var InteractiveRoles = map[string]bool{
"combobox": true, "listbox": true, "option": true, "checkbox": true,
"radio": true, "switch": true, "slider": true, "spinbutton": true,
"menuitem": true, "menuitemcheckbox": true, "menuitemradio": true,
- "tab": true, "treeitem": true,
+ "tab": true, "treeitem": true, "iframe": true, "Iframe": true,
}
const FilterInteractive = "interactive"
@@ -171,26 +226,33 @@ func isAXNodeHidden(n RawAXNode) bool {
}
func BuildSnapshot(nodes []RawAXNode, filter string, maxDepth int) ([]A11yNode, map[string]int64) {
+ nodeByID := make(map[string]RawAXNode, len(nodes))
parentMap := make(map[string]string)
+ childMap := make(map[string][]string, len(nodes))
+ backendToAX := make(map[int64]string, len(nodes))
+ frameRoots := make(map[string][]string, 4)
+ frameOwners := make(map[string]int64, 4)
+ frameOrder := make([]string, 0, 4)
+ seenFrames := make(map[string]bool, 4)
+
for _, n := range nodes {
+ nodeByID[n.NodeID] = n
+ childMap[n.NodeID] = append(childMap[n.NodeID], n.ChildIDs...)
+ if n.BackendDOMNodeID != 0 {
+ backendToAX[n.BackendDOMNodeID] = n.NodeID
+ }
+ if !seenFrames[n.FrameID] {
+ frameOrder = append(frameOrder, n.FrameID)
+ seenFrames[n.FrameID] = true
+ }
+ if n.FrameOwnerNodeID != 0 && frameOwners[n.FrameID] == 0 {
+ frameOwners[n.FrameID] = n.FrameOwnerNodeID
+ }
for _, childID := range n.ChildIDs {
parentMap[childID] = n.NodeID
}
}
maxAncestorWalk := max(len(parentMap)+1, 1)
- depthOf := func(nodeID string) int {
- d := 0
- cur := nodeID
- for range maxAncestorWalk {
- p, ok := parentMap[cur]
- if !ok {
- break
- }
- d++
- cur = p
- }
- return d
- }
// Build a set of AX node IDs that are hidden, including inherited hidden
// status from ancestors. A child of a hidden node is also hidden.
@@ -216,39 +278,65 @@ func BuildSnapshot(nodes []RawAXNode, filter string, maxDepth int) ([]A11yNode,
return false
}
- flat := make([]A11yNode, 0)
- refs := make(map[string]int64)
- refID := 0
-
for _, n := range nodes {
- if n.Ignored {
+ parentID, ok := parentMap[n.NodeID]
+ if !ok {
+ frameRoots[n.FrameID] = append(frameRoots[n.FrameID], n.NodeID)
continue
}
-
- role := n.Role.String()
- name := n.Name.String()
-
- if role == "none" || role == "generic" || role == "InlineTextBox" {
- continue
+ parentNode, ok := nodeByID[parentID]
+ if !ok || parentNode.FrameID != n.FrameID {
+ frameRoots[n.FrameID] = append(frameRoots[n.FrameID], n.NodeID)
}
- if name == "" && role == "StaticText" {
- continue
+ }
+
+ rootFrameID := ""
+ for _, frameID := range frameOrder {
+ if frameOwners[frameID] == 0 {
+ rootFrameID = frameID
+ break
}
+ }
+ if rootFrameID == "" && len(frameOrder) > 0 {
+ rootFrameID = frameOrder[0]
+ }
- depth := depthOf(n.NodeID)
- if maxDepth >= 0 && depth > maxDepth {
+ topRoots := make([]string, 0, len(nodes))
+ frameChildRoots := make(map[string][]string)
+ for _, frameID := range frameOrder {
+ roots := frameRoots[frameID]
+ if len(roots) == 0 {
continue
}
- if filter == FilterInteractive && !InteractiveRoles[role] {
+ ownerBackendID := frameOwners[frameID]
+ ownerAXID := backendToAX[ownerBackendID]
+ if frameID == rootFrameID || ownerAXID == "" {
+ topRoots = append(topRoots, roots...)
continue
}
+ frameChildRoots[ownerAXID] = append(frameChildRoots[ownerAXID], roots...)
+ }
+ if len(topRoots) == 0 {
+ for _, n := range nodes {
+ topRoots = append(topRoots, n.NodeID)
+ }
+ }
+ flat := make([]A11yNode, 0)
+ refs := make(map[string]int64)
+ refID := 0
+ appendNode := func(n RawAXNode, depth int) {
+ role := n.Role.String()
+ name := n.Name.String()
ref := fmt.Sprintf("e%d", refID)
entry := A11yNode{
- Ref: ref,
- Role: role,
- Name: name,
- Depth: depth,
+ Ref: ref,
+ Role: role,
+ Name: name,
+ Depth: depth,
+ FrameID: n.FrameID,
+ FrameURL: n.FrameURL,
+ FrameName: n.FrameName,
}
if v := n.Value.String(); v != "" {
@@ -279,6 +367,73 @@ func BuildSnapshot(nodes []RawAXNode, filter string, maxDepth int) ([]A11yNode,
refID++
}
+ isSkippableNode := func(n RawAXNode) bool {
+ role := n.Role.String()
+ name := n.Name.String()
+ if n.Ignored {
+ return true
+ }
+ if role == "none" || role == "generic" || role == "InlineTextBox" {
+ return true
+ }
+ if name == "" && role == "StaticText" {
+ return true
+ }
+ return false
+ }
+
+ isFlattenableFrameRoot := func(n RawAXNode) bool {
+ role := n.Role.String()
+ return n.FrameOwnerNodeID != 0 && (role == "WebArea" || role == "RootWebArea")
+ }
+
+ visited := make(map[string]bool, len(nodes))
+ var visitFrameRoot func(string, int)
+ var visit func(string, int)
+
+ visit = func(nodeID string, depth int) {
+ if nodeID == "" || visited[nodeID] {
+ return
+ }
+ n, ok := nodeByID[nodeID]
+ if !ok {
+ return
+ }
+ visited[nodeID] = true
+ if maxDepth >= 0 && depth > maxDepth {
+ return
+ }
+
+ if !isSkippableNode(n) && (filter != FilterInteractive || InteractiveRoles[n.Role.String()]) {
+ appendNode(n, depth)
+ }
+
+ for _, childID := range childMap[nodeID] {
+ visit(childID, depth+1)
+ }
+ for _, frameRootID := range frameChildRoots[nodeID] {
+ visitFrameRoot(frameRootID, depth+1)
+ }
+ }
+
+ visitFrameRoot = func(nodeID string, depth int) {
+ n, ok := nodeByID[nodeID]
+ if !ok {
+ return
+ }
+ if isFlattenableFrameRoot(n) {
+ for _, childID := range childMap[nodeID] {
+ visit(childID, depth)
+ }
+ return
+ }
+ visit(nodeID, depth)
+ }
+
+ for _, rootID := range topRoots {
+ visit(rootID, 0)
+ }
+
return flat, refs
}
diff --git a/internal/bridge/observe_facade.go b/internal/bridge/observe_facade.go
index c8c434d8..8d9217d3 100644
--- a/internal/bridge/observe_facade.go
+++ b/internal/bridge/observe_facade.go
@@ -17,6 +17,7 @@ type A11yNode = bridgeobserve.A11yNode
type RawAXNode = bridgeobserve.RawAXNode
type RawAXValue = bridgeobserve.RawAXValue
type RawAXProp = bridgeobserve.RawAXProp
+type RawFrame = bridgeobserve.RawFrame
type rawFrameTree = bridgeobserve.RawFrameTree
type NetworkEntry = bridgeobserve.NetworkEntry
@@ -29,6 +30,10 @@ func frameIDs(tree rawFrameTree) []string {
return bridgeobserve.FrameIDs(tree)
}
+func FrameMap(tree rawFrameTree) map[string]RawFrame {
+ return bridgeobserve.FrameMap(tree)
+}
+
func FetchAXTree(ctx context.Context) ([]RawAXNode, error) {
return bridgeobserve.FetchAXTree(ctx)
}
diff --git a/internal/bridge/snapshot_extra_test.go b/internal/bridge/snapshot_extra_test.go
index 282f1cf3..7d39c546 100644
--- a/internal/bridge/snapshot_extra_test.go
+++ b/internal/bridge/snapshot_extra_test.go
@@ -239,16 +239,10 @@ func TestFrameIDs_Recursive(t *testing.T) {
tree := rawFrameTree{}
tree.Frame.ID = "root"
tree.ChildFrames = []rawFrameTree{
- {Frame: struct {
- ID string `json:"id"`
- }{ID: "child-1"}},
+ {Frame: RawFrame{ID: "child-1"}},
{
- Frame: struct {
- ID string `json:"id"`
- }{ID: "child-2"},
- ChildFrames: []rawFrameTree{{Frame: struct {
- ID string `json:"id"`
- }{ID: "grandchild"}}},
+ Frame: RawFrame{ID: "child-2"},
+ ChildFrames: []rawFrameTree{{Frame: RawFrame{ID: "grandchild"}}},
},
}
got := frameIDs(tree)
@@ -262,3 +256,22 @@ func TestFrameIDs_Recursive(t *testing.T) {
}
}
}
+
+func TestFrameMap_Recursive(t *testing.T) {
+ tree := rawFrameTree{
+ Frame: RawFrame{ID: "root", URL: "https://example.com", Name: "main"},
+ ChildFrames: []rawFrameTree{
+ {Frame: RawFrame{ID: "child-1", URL: "https://example.com/embed", Name: "embed"}},
+ },
+ }
+ got := FrameMap(tree)
+ if len(got) != 2 {
+ t.Fatalf("expected 2 frames, got %d", len(got))
+ }
+ if got["root"].URL != "https://example.com" {
+ t.Fatalf("root frame url = %q", got["root"].URL)
+ }
+ if got["child-1"].Name != "embed" {
+ t.Fatalf("child frame name = %q", got["child-1"].Name)
+ }
+}
diff --git a/internal/bridge/snapshot_test.go b/internal/bridge/snapshot_test.go
index a06cc91d..601b04c8 100644
--- a/internal/bridge/snapshot_test.go
+++ b/internal/bridge/snapshot_test.go
@@ -318,3 +318,149 @@ func TestBuildSnapshotDepthFilter(t *testing.T) {
t.Fatalf("expected 2 nodes at depth<=1, got %d: %+v", len(flat), flat)
}
}
+
+func TestBuildSnapshotPreservesFrameMetadata(t *testing.T) {
+ nodes := []RawAXNode{
+ {
+ NodeID: "frame-node",
+ Role: &RawAXValue{Value: json.RawMessage(`"textbox"`)},
+ Name: &RawAXValue{Value: json.RawMessage(`"Card number"`)},
+ BackendDOMNodeID: 42,
+ FrameID: "frame-payment",
+ FrameURL: "https://payments.example/frame",
+ FrameName: "payment-frame",
+ },
+ }
+
+ flat, refs := BuildSnapshot(nodes, "", -1)
+ if len(flat) != 1 {
+ t.Fatalf("expected 1 node, got %d", len(flat))
+ }
+ if refs["e0"] != 42 {
+ t.Fatalf("expected e0 to resolve to backend node 42, got %d", refs["e0"])
+ }
+ if flat[0].FrameID != "frame-payment" {
+ t.Fatalf("frame id = %q, want %q", flat[0].FrameID, "frame-payment")
+ }
+ if flat[0].FrameURL != "https://payments.example/frame" {
+ t.Fatalf("frame url = %q, want %q", flat[0].FrameURL, "https://payments.example/frame")
+ }
+ if flat[0].FrameName != "payment-frame" {
+ t.Fatalf("frame name = %q, want %q", flat[0].FrameName, "payment-frame")
+ }
+}
+
+func TestBuildSnapshotNestsFrameContentUnderOwner(t *testing.T) {
+ nodes := []RawAXNode{
+ {
+ NodeID: "root",
+ Role: &RawAXValue{Value: json.RawMessage(`"WebArea"`)},
+ Name: &RawAXValue{Value: json.RawMessage(`"Outer"`)},
+ ChildIDs: []string{"iframe"},
+ BackendDOMNodeID: 1,
+ FrameID: "main",
+ },
+ {
+ NodeID: "iframe",
+ Role: &RawAXValue{Value: json.RawMessage(`"Iframe"`)},
+ Name: &RawAXValue{Value: json.RawMessage(`"payment-frame"`)},
+ BackendDOMNodeID: 10,
+ FrameID: "main",
+ },
+ {
+ NodeID: "child-root",
+ Role: &RawAXValue{Value: json.RawMessage(`"WebArea"`)},
+ Name: &RawAXValue{Value: json.RawMessage(`"Inner"`)},
+ ChildIDs: []string{"card", "pay"},
+ BackendDOMNodeID: 11,
+ FrameID: "child",
+ FrameOwnerNodeID: 10,
+ },
+ {
+ NodeID: "card",
+ Role: &RawAXValue{Value: json.RawMessage(`"textbox"`)},
+ Name: &RawAXValue{Value: json.RawMessage(`"Card number"`)},
+ BackendDOMNodeID: 12,
+ FrameID: "child",
+ FrameOwnerNodeID: 10,
+ },
+ {
+ NodeID: "pay",
+ Role: &RawAXValue{Value: json.RawMessage(`"button"`)},
+ Name: &RawAXValue{Value: json.RawMessage(`"Pay"`)},
+ BackendDOMNodeID: 13,
+ FrameID: "child",
+ FrameOwnerNodeID: 10,
+ },
+ }
+
+ flat, _ := BuildSnapshot(nodes, "", -1)
+ if len(flat) != 4 {
+ t.Fatalf("expected 4 visible nodes, got %d: %+v", len(flat), flat)
+ }
+ if flat[1].Name != "payment-frame" {
+ t.Fatalf("second node = %q, want iframe owner", flat[1].Name)
+ }
+ if flat[2].Name != "Card number" || flat[2].Depth != 2 {
+ t.Fatalf("textbox = %+v, want nested child at depth 2", flat[2])
+ }
+ if flat[3].Name != "Pay" || flat[3].Depth != 2 {
+ t.Fatalf("button = %+v, want nested child at depth 2", flat[3])
+ }
+}
+
+func TestBuildSnapshotInteractiveIncludesIframeOwnerAndChildActions(t *testing.T) {
+ nodes := []RawAXNode{
+ {
+ NodeID: "root",
+ Role: &RawAXValue{Value: json.RawMessage(`"WebArea"`)},
+ Name: &RawAXValue{Value: json.RawMessage(`"Outer"`)},
+ ChildIDs: []string{"iframe"},
+ BackendDOMNodeID: 1,
+ FrameID: "main",
+ },
+ {
+ NodeID: "iframe",
+ Role: &RawAXValue{Value: json.RawMessage(`"Iframe"`)},
+ Name: &RawAXValue{Value: json.RawMessage(`"payment-frame"`)},
+ BackendDOMNodeID: 10,
+ FrameID: "main",
+ },
+ {
+ NodeID: "child-root",
+ Role: &RawAXValue{Value: json.RawMessage(`"WebArea"`)},
+ Name: &RawAXValue{Value: json.RawMessage(`"Inner"`)},
+ ChildIDs: []string{"card", "pay"},
+ BackendDOMNodeID: 11,
+ FrameID: "child",
+ FrameOwnerNodeID: 10,
+ },
+ {
+ NodeID: "card",
+ Role: &RawAXValue{Value: json.RawMessage(`"textbox"`)},
+ Name: &RawAXValue{Value: json.RawMessage(`"Card number"`)},
+ BackendDOMNodeID: 12,
+ FrameID: "child",
+ FrameOwnerNodeID: 10,
+ },
+ {
+ NodeID: "pay",
+ Role: &RawAXValue{Value: json.RawMessage(`"button"`)},
+ Name: &RawAXValue{Value: json.RawMessage(`"Pay"`)},
+ BackendDOMNodeID: 13,
+ FrameID: "child",
+ FrameOwnerNodeID: 10,
+ },
+ }
+
+ flat, _ := BuildSnapshot(nodes, FilterInteractive, -1)
+ if len(flat) != 3 {
+ t.Fatalf("expected iframe + 2 actionable children, got %d: %+v", len(flat), flat)
+ }
+ if flat[0].Role != "Iframe" {
+ t.Fatalf("expected iframe owner in interactive snapshot, got %+v", flat[0])
+ }
+ if flat[1].Name != "Card number" || flat[2].Name != "Pay" {
+ t.Fatalf("unexpected interactive descendants: %+v", flat)
+ }
+}
diff --git a/internal/handlers/actions.go b/internal/handlers/actions.go
index 2a54570a..c739b5ed 100644
--- a/internal/handlers/actions.go
+++ b/internal/handlers/actions.go
@@ -184,8 +184,8 @@ func (h *Handlers) HandleAction(w http.ResponseWriter, r *http.Request) {
req.Selector = ""
cache := h.Bridge.GetRefCache(resolvedTabID)
if cache != nil {
- if nid, ok := cache.Refs[sel.Value]; ok {
- req.NodeID = nid
+ if target, ok := cache.Lookup(sel.Value); ok {
+ req.NodeID = target.BackendNodeID
}
}
if req.NodeID == 0 {
@@ -242,8 +242,8 @@ func (h *Handlers) HandleAction(w http.ResponseWriter, r *http.Request) {
req.Ref = result.BestRef
cache := h.Bridge.GetRefCache(resolvedTabID)
if cache != nil {
- if nid, ok := cache.Refs[result.BestRef]; ok {
- req.NodeID = nid
+ if target, ok := cache.Lookup(result.BestRef); ok {
+ req.NodeID = target.BackendNodeID
}
}
}
@@ -303,8 +303,8 @@ func (h *Handlers) HandleAction(w http.ResponseWriter, r *http.Request) {
recordStaleRefRetry()
h.refreshRefCache(tCtx, resolvedTabID)
if cache := h.Bridge.GetRefCache(resolvedTabID); cache != nil {
- if nid, ok := cache.Refs[req.Ref]; ok {
- req.NodeID = nid
+ if target, ok := cache.Lookup(req.Ref); ok {
+ req.NodeID = target.BackendNodeID
}
}
}
@@ -535,8 +535,8 @@ func (h *Handlers) handleActionsBatch(w http.ResponseWriter, r *http.Request, re
action.Selector = ""
cache := h.Bridge.GetRefCache(resolvedTabID)
if cache != nil {
- if nid, ok := cache.Refs[sel.Value]; ok {
- action.NodeID = nid
+ if target, ok := cache.Lookup(sel.Value); ok {
+ action.NodeID = target.BackendNodeID
}
}
if action.NodeID == 0 {
@@ -598,8 +598,8 @@ func (h *Handlers) handleActionsBatch(w http.ResponseWriter, r *http.Request, re
action.Ref = findResult.BestRef
cache := h.Bridge.GetRefCache(resolvedTabID)
if cache != nil {
- if nid, ok := cache.Refs[findResult.BestRef]; ok {
- action.NodeID = nid
+ if target, ok := cache.Lookup(findResult.BestRef); ok {
+ action.NodeID = target.BackendNodeID
}
}
}
@@ -689,8 +689,8 @@ func (h *Handlers) handleActionsBatch(w http.ResponseWriter, r *http.Request, re
recordStaleRefRetry()
h.refreshRefCache(tCtx, resolvedTabID)
if cache := h.Bridge.GetRefCache(resolvedTabID); cache != nil {
- if nid, ok := cache.Refs[action.Ref]; ok {
- action.NodeID = nid
+ if target, ok := cache.Lookup(action.Ref); ok {
+ action.NodeID = target.BackendNodeID
}
}
}
@@ -814,8 +814,8 @@ func (h *Handlers) HandleMacro(w http.ResponseWriter, r *http.Request) {
step.Selector = ""
cache := h.Bridge.GetRefCache(resolvedTabID)
if cache != nil {
- if nid, ok := cache.Refs[sel.Value]; ok {
- step.NodeID = nid
+ if target, ok := cache.Lookup(sel.Value); ok {
+ step.NodeID = target.BackendNodeID
}
}
if step.NodeID == 0 {
@@ -880,8 +880,8 @@ func (h *Handlers) HandleMacro(w http.ResponseWriter, r *http.Request) {
step.Ref = findResult.BestRef
cache := h.Bridge.GetRefCache(resolvedTabID)
if cache != nil {
- if nid, ok := cache.Refs[findResult.BestRef]; ok {
- step.NodeID = nid
+ if target, ok := cache.Lookup(findResult.BestRef); ok {
+ step.NodeID = target.BackendNodeID
}
}
}
@@ -956,8 +956,8 @@ func (h *Handlers) HandleMacro(w http.ResponseWriter, r *http.Request) {
recordStaleRefRetry()
h.refreshRefCache(tCtx, resolvedTabID)
if cache := h.Bridge.GetRefCache(resolvedTabID); cache != nil {
- if nid, ok := cache.Refs[step.Ref]; ok {
- step.NodeID = nid
+ if target, ok := cache.Lookup(step.Ref); ok {
+ step.NodeID = target.BackendNodeID
}
}
}
@@ -1164,5 +1164,9 @@ func (h *Handlers) refreshRefCache(ctx context.Context, tabID string) {
return
}
flat, refs := bridge.BuildSnapshot(nodes, bridge.FilterInteractive, -1)
- h.Bridge.SetRefCache(tabID, &bridge.RefCache{Refs: refs, Nodes: flat})
+ h.Bridge.SetRefCache(tabID, &bridge.RefCache{
+ Refs: refs,
+ Targets: bridge.RefTargetsFromNodes(flat),
+ Nodes: flat,
+ })
}
diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go
index 1d72b288..100a3fc1 100644
--- a/internal/handlers/handlers.go
+++ b/internal/handlers/handlers.go
@@ -73,8 +73,8 @@ func New(b bridge.BridgeAPI, cfg *config.RuntimeConfig, p bridge.ProfileService,
if cache == nil {
return 0, false
}
- nid, ok := cache.Refs[ref]
- return nid, ok
+ target, ok := cache.Lookup(ref)
+ return target.BackendNodeID, ok
},
// DescriptorBuilder
func(tabID string) []semantic.ElementDescriptor {
diff --git a/internal/handlers/snapshot.go b/internal/handlers/snapshot.go
index 83b0b318..a79eadfe 100644
--- a/internal/handlers/snapshot.go
+++ b/internal/handlers/snapshot.go
@@ -189,7 +189,11 @@ func (h *Handlers) HandleSnapshot(w http.ResponseWriter, r *http.Request) {
}
}
- h.Bridge.SetRefCache(resolvedTabID, &bridge.RefCache{Refs: refs, Nodes: flat})
+ h.Bridge.SetRefCache(resolvedTabID, &bridge.RefCache{
+ Refs: refs,
+ Targets: bridge.RefTargetsFromNodes(flat),
+ Nodes: flat,
+ })
var url, title string
_ = chromedp.Run(tCtx,
diff --git a/tests/e2e/fixtures/iframe-content.html b/tests/e2e/fixtures/iframe-content.html
new file mode 100644
index 00000000..bce47a9a
--- /dev/null
+++ b/tests/e2e/fixtures/iframe-content.html
@@ -0,0 +1,23 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <title>Iframe Payment Content</title>
+</head>
+<body>
+ <h2>Payment</h2>
+ <form id="payment-form">
+ <label for="card-number">Card number</label>
+ <input type="text" id="card-number" name="cardNumber">
+ <button type="button" id="pay-button">Pay</button>
+ </form>
+ <p id="payment-result"></p>
+
+ <script>
+ document.getElementById('pay-button').addEventListener('click', function() {
+ const value = document.getElementById('card-number').value;
+ document.getElementById('payment-result').textContent = 'PAYMENT_SUBMITTED_' + value;
+ });
+ </script>
+</body>
+</html>
diff --git a/tests/e2e/fixtures/iframe.html b/tests/e2e/fixtures/iframe.html
new file mode 100644
index 00000000..822e7a84
--- /dev/null
+++ b/tests/e2e/fixtures/iframe.html
@@ -0,0 +1,18 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <title>Iframe Fixture</title>
+</head>
+<body>
+ <h1>Iframe Fixture</h1>
+ <p>The iframe below simulates an embedded payment form.</p>
+ <iframe
+ id="payment-frame"
+ title="payment-frame"
+ src="iframe-content.html"
+ width="600"
+ height="300"
+ ></iframe>
+</body>
+</html>
diff --git a/tests/e2e/scenarios/api/actions-extended.sh b/tests/e2e/scenarios/api/actions-extended.sh
index 67144cc5..8172add4 100644
--- a/tests/e2e/scenarios/api/actions-extended.sh
+++ b/tests/e2e/scenarios/api/actions-extended.sh
@@ -172,6 +172,29 @@ require_ref "textbox" "Notes" NOTES_REF && {
end_test
+# ─────────────────────────────────────────────────────────────────
+start_test "iframe: snapshot refs support fill and click inside iframe"
+
+navigate_fixture "iframe.html"
+fresh_snapshot
+
+assert_json_exists "$RESULT" '.nodes[] | select(.name == "payment-frame")' "snapshot includes iframe owner"
+
+require_ref "textbox" "Card number" CARD_REF && \
+require_ref "button" "Pay" PAY_REF && {
+ pt_post /action -d "{\"kind\":\"fill\",\"ref\":\"${CARD_REF}\",\"text\":\"4111111111111111\"}"
+ assert_ok "fill iframe textbox by ref"
+
+ pt_post /action -d "{\"kind\":\"click\",\"ref\":\"${PAY_REF}\"}"
+ assert_ok "click iframe button by ref"
+
+ pt_post /evaluate -d '{"expression":"(() => { const frame = document.getElementById(\"payment-frame\"); const doc = frame && frame.contentDocument; const result = doc && doc.getElementById(\"payment-result\"); return result ? result.textContent : \"\"; })()"}'
+ assert_ok "evaluate iframe result"
+ assert_json_eq "$RESULT" '.result' 'PAYMENT_SUBMITTED_4111111111111111' "iframe form submitted via refs"
+}
+
+end_test
+
# Regression test for GitHub issue #236: press action was typing key names
# as literal text instead of dispatching keyboard events.