Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix: include iframe AX nodes in snapshots

dddabtc0638fb68e6ef
diff --git a/internal/bridge/snapshot.go b/internal/bridge/snapshot.go
index dbdf9ee7..78be9bd3 100644
--- a/internal/bridge/snapshot.go
+++ b/internal/bridge/snapshot.go
@@ -1,9 +1,12 @@
package bridge
import (
+ "context"
"encoding/json"
"fmt"
"strings"
+
+ "github.com/chromedp/chromedp"
)
type A11yNode struct {
@@ -28,6 +31,97 @@ type RawAXNode struct {
BackendDOMNodeID int64 `json:"backendDOMNodeId"`
}
+type rawAXTreeResponse struct {
+ Nodes []RawAXNode `json:"nodes"`
+}
+
+type rawFrameTree struct {
+ Frame struct {
+ ID string `json:"id"`
+ } `json:"frame"`
+ ChildFrames []rawFrameTree `json:"childFrames"`
+}
+
+func frameIDs(tree rawFrameTree) []string {
+ ids := make([]string, 0, 1+len(tree.ChildFrames))
+ var walk func(rawFrameTree)
+ walk = func(t rawFrameTree) {
+ if t.Frame.ID != "" {
+ ids = append(ids, t.Frame.ID)
+ }
+ for _, child := range t.ChildFrames {
+ walk(child)
+ }
+ }
+ walk(tree)
+ return ids
+}
+
+// FetchAXTree returns the merged accessibility tree for the current page and any child frames.
+// If frame-aware enumeration fails, it falls back to the page-wide tree so snapshotting keeps working.
+func FetchAXTree(ctx context.Context) ([]RawAXNode, error) {
+ var frameTreeResult json.RawMessage
+ if err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
+ return chromedp.FromContext(ctx).Target.Execute(ctx, "Page.getFrameTree", nil, &frameTreeResult)
+ })); err != nil {
+ return fetchAXTreeForFrame(ctx, "")
+ }
+
+ var frameResp struct {
+ FrameTree rawFrameTree `json:"frameTree"`
+ }
+ if err := json.Unmarshal(frameTreeResult, &frameResp); err != nil {
+ return fetchAXTreeForFrame(ctx, "")
+ }
+
+ ids := frameIDs(frameResp.FrameTree)
+ if len(ids) == 0 {
+ return fetchAXTreeForFrame(ctx, "")
+ }
+
+ merged := make([]RawAXNode, 0, 256)
+ seen := make(map[string]bool, 256)
+ for _, id := range ids {
+ nodes, err := fetchAXTreeForFrame(ctx, id)
+ if err != nil {
+ continue
+ }
+ for _, n := range nodes {
+ key := n.NodeID
+ if key == "" {
+ key = fmt.Sprintf("backend:%d:%s:%s", n.BackendDOMNodeID, n.Role.String(), n.Name.String())
+ }
+ if seen[key] {
+ continue
+ }
+ seen[key] = true
+ merged = append(merged, n)
+ }
+ }
+ if len(merged) > 0 {
+ return merged, nil
+ }
+ return fetchAXTreeForFrame(ctx, "")
+}
+
+func fetchAXTreeForFrame(ctx context.Context, frameID string) ([]RawAXNode, error) {
+ params := map[string]any{}
+ if frameID != "" {
+ params["frameId"] = frameID
+ }
+ var rawResult json.RawMessage
+ if err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
+ return chromedp.FromContext(ctx).Target.Execute(ctx, "Accessibility.getFullAXTree", params, &rawResult)
+ })); err != nil {
+ return nil, err
+ }
+ var treeResp rawAXTreeResponse
+ if err := json.Unmarshal(rawResult, &treeResp); err != nil {
+ return nil, err
+ }
+ return treeResp.Nodes, nil
+}
+
type RawAXValue struct {
Type string `json:"type"`
Value json.RawMessage `json:"value"`
diff --git a/internal/bridge/snapshot_extra_test.go b/internal/bridge/snapshot_extra_test.go
index bdb0e768..282f1cf3 100644
--- a/internal/bridge/snapshot_extra_test.go
+++ b/internal/bridge/snapshot_extra_test.go
@@ -234,3 +234,31 @@ func searchString(s, substr string) bool {
}
return false
}
+
+func TestFrameIDs_Recursive(t *testing.T) {
+ tree := rawFrameTree{}
+ tree.Frame.ID = "root"
+ tree.ChildFrames = []rawFrameTree{
+ {Frame: struct {
+ ID string `json:"id"`
+ }{ID: "child-1"}},
+ {
+ Frame: struct {
+ ID string `json:"id"`
+ }{ID: "child-2"},
+ ChildFrames: []rawFrameTree{{Frame: struct {
+ ID string `json:"id"`
+ }{ID: "grandchild"}}},
+ },
+ }
+ got := frameIDs(tree)
+ want := []string{"root", "child-1", "child-2", "grandchild"}
+ if len(got) != len(want) {
+ t.Fatalf("len=%d want %d (%v)", len(got), len(want), got)
+ }
+ for i := range want {
+ if got[i] != want[i] {
+ t.Fatalf("frameIDs[%d]=%q want %q (all=%v)", i, got[i], want[i], got)
+ }
+ }
+}
diff --git a/internal/handlers/actions.go b/internal/handlers/actions.go
index 39f4b7e9..79d88b28 100644
--- a/internal/handlers/actions.go
+++ b/internal/handlers/actions.go
@@ -11,7 +11,6 @@ import (
"strings"
"time"
- "github.com/chromedp/chromedp"
"github.com/pinchtab/pinchtab/internal/bridge"
"github.com/pinchtab/pinchtab/internal/semantic"
"github.com/pinchtab/pinchtab/internal/web"
@@ -674,20 +673,10 @@ func shouldRetryStaleRef(err error) bool {
}
func (h *Handlers) refreshRefCache(ctx context.Context, tabID string) {
- var rawResult json.RawMessage
- if err := chromedp.Run(ctx,
- chromedp.ActionFunc(func(c context.Context) error {
- return chromedp.FromContext(c).Target.Execute(c, "Accessibility.getFullAXTree", nil, &rawResult)
- }),
- ); err != nil {
- return
- }
- var treeResp struct {
- Nodes []bridge.RawAXNode `json:"nodes"`
- }
- if err := json.Unmarshal(rawResult, &treeResp); err != nil {
+ nodes, err := bridge.FetchAXTree(ctx)
+ if err != nil {
return
}
- flat, refs := bridge.BuildSnapshot(treeResp.Nodes, bridge.FilterInteractive, -1)
+ flat, refs := bridge.BuildSnapshot(nodes, bridge.FilterInteractive, -1)
h.Bridge.SetRefCache(tabID, &bridge.RefCache{Refs: refs, Nodes: flat})
}
diff --git a/internal/handlers/snapshot.go b/internal/handlers/snapshot.go
index 78d90850..74106dac 100644
--- a/internal/handlers/snapshot.go
+++ b/internal/handlers/snapshot.go
@@ -125,24 +125,14 @@ func (h *Handlers) HandleSnapshot(w http.ResponseWriter, r *http.Request) {
}
}
- var rawResult json.RawMessage
- if err := chromedp.Run(tCtx,
- chromedp.ActionFunc(func(ctx context.Context) error {
- return chromedp.FromContext(ctx).Target.Execute(ctx,
- "Accessibility.getFullAXTree", nil, &rawResult)
- }),
- ); err != nil {
+ nodes, err := bridge.FetchAXTree(tCtx)
+ if err != nil {
web.Error(w, 500, fmt.Errorf("a11y tree: %w", err))
return
}
-
- var treeResp struct {
+ treeResp := struct {
Nodes []bridge.RawAXNode `json:"nodes"`
- }
- if err := json.Unmarshal(rawResult, &treeResp); err != nil {
- web.Error(w, 500, fmt.Errorf("parse a11y tree: %w", err))
- return
- }
+ }{Nodes: nodes}
if selector != "" {
var scopeNodeID int64