Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

refactor: action handlers around shared selector resolution

Luigi Agostiafa18e0f5ffc
diff --git a/internal/handlers/action_execution.go b/internal/handlers/action_execution.go
new file mode 100644
index 00000000..4d1e2556
--- /dev/null
+++ b/internal/handlers/action_execution.go
@@ -0,0 +1,202 @@
+package handlers
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "time"
+
+ "github.com/pinchtab/pinchtab/internal/bridge"
+ "github.com/pinchtab/pinchtab/internal/engine"
+ "github.com/pinchtab/semantic"
+ "github.com/pinchtab/semantic/recovery"
+)
+
+// cacheActionIntent stores the element's semantic identity in the
+// IntentCache so the recovery engine can reconstruct a query if the
+// ref becomes stale.
+func (h *Handlers) cacheActionIntent(tabID string, req bridge.ActionRequest) {
+ if h.Recovery == nil || req.Ref == "" {
+ return
+ }
+ // Don't overwrite an existing entry that has a real Query (from /find)
+ // with a descriptor-only entry.
+ if existing, ok := h.Recovery.IntentCache.Lookup(tabID, req.Ref); ok && existing.Query != "" {
+ return
+ }
+ desc := semantic.ElementDescriptor{Ref: req.Ref}
+ // Try to enrich from the current snapshot cache.
+ if cache := h.Bridge.GetRefCache(tabID); cache != nil {
+ for _, enriched := range semanticDescriptorsFromNodes(cache.Nodes) {
+ if enriched.Ref == req.Ref {
+ desc = enriched
+ break
+ }
+ }
+ }
+ h.Recovery.RecordIntent(tabID, req.Ref, recovery.IntentEntry{
+ Descriptor: desc,
+ CachedAt: time.Now(),
+ })
+}
+
+func (h *Handlers) executeAction(ctx context.Context, req bridge.ActionRequest) (map[string]any, string, error) {
+ req.Kind = bridge.CanonicalActionKind(req.Kind)
+ if h.shouldUseLiteAction(req) {
+ return h.executeLiteAction(ctx, req)
+ }
+
+ if err := h.ensureChrome(); err != nil {
+ return nil, "", fmt.Errorf("chrome initialization: %w", err)
+ }
+ result, err := h.Bridge.ExecuteAction(ctx, req.Kind, req)
+ return result, "", err
+}
+
+func (h *Handlers) shouldUseLiteAction(req bridge.ActionRequest) bool {
+ kind := bridge.CanonicalActionKind(req.Kind)
+ if h.effectiveActionHumanize(req) && (kind == bridge.ActionClick || kind == bridge.ActionType || kind == bridge.ActionKeyboardType) {
+ return false
+ }
+ capability, ok := actionCapability(kind)
+ if !ok {
+ return h.Router != nil && h.Router.Mode() == engine.ModeLite
+ }
+ return h.useLite(capability, "")
+}
+
+func (h *Handlers) effectiveActionHumanize(req bridge.ActionRequest) bool {
+ if req.Humanize != nil {
+ return *req.Humanize
+ }
+ if h != nil && h.Config != nil {
+ return h.Config.Humanize
+ }
+ return false
+}
+
+func (h *Handlers) executeLiteAction(ctx context.Context, req bridge.ActionRequest) (map[string]any, string, error) {
+ if h.Router == nil || h.Router.Lite() == nil {
+ return nil, "", fmt.Errorf("lite engine unavailable")
+ }
+ switch bridge.CanonicalActionKind(req.Kind) {
+ case bridge.ActionClick:
+ if req.Ref == "" {
+ return nil, "lite", fmt.Errorf("lite mode actions require ref from /snapshot")
+ }
+ if err := h.Router.Lite().Click(ctx, req.TabID, req.Ref); err != nil {
+ return nil, "lite", err
+ }
+ return map[string]any{"clicked": true}, "lite", nil
+ case bridge.ActionType, bridge.ActionFill:
+ if req.Ref == "" {
+ return nil, "lite", fmt.Errorf("lite mode actions require ref from /snapshot")
+ }
+ text := req.Text
+ if req.Kind == bridge.ActionFill && text == "" {
+ text = req.Value
+ }
+ if text == "" {
+ return nil, "lite", fmt.Errorf("text required for %s", req.Kind)
+ }
+ if err := h.Router.Lite().Type(ctx, req.TabID, req.Ref, text); err != nil {
+ return nil, "lite", err
+ }
+ return map[string]any{"typed": text}, "lite", nil
+ default:
+ return nil, "lite", fmt.Errorf("%w: %s", engine.ErrLiteNotSupported, req.Kind)
+ }
+}
+
+func actionCapability(kind string) (engine.Capability, bool) {
+ switch bridge.CanonicalActionKind(kind) {
+ case bridge.ActionClick:
+ return engine.CapClick, true
+ case bridge.ActionType, bridge.ActionFill:
+ return engine.CapType, true
+ default:
+ return "", false
+ }
+}
+
+const pointerRetryDelay = 50 * time.Millisecond
+
+func shouldRetryPointerAction(req bridge.ActionRequest, err error) bool {
+ if err == nil {
+ return false
+ }
+ kind := strings.ToLower(strings.TrimSpace(req.Kind))
+ switch kind {
+ case bridge.ActionClick, bridge.ActionDoubleClick, bridge.ActionHover, bridge.ActionDrag,
+ bridge.ActionMouseDown, bridge.ActionMouseUp, bridge.ActionMouseWheel:
+ // pointer action kinds
+ default:
+ return false
+ }
+
+ if errors.Is(err, bridge.ErrElementOccluded) ||
+ errors.Is(err, bridge.ErrElementBlocked) ||
+ errors.Is(err, bridge.ErrElementOffscreen) {
+ return true
+ }
+
+ return shouldRetryStaleRef(err)
+}
+
+func (h *Handlers) refreshActionNodeIDFromSelector(ctx context.Context, req *bridge.ActionRequest) {
+ if req == nil || req.NodeID > 0 || strings.TrimSpace(req.Selector) == "" {
+ return
+ }
+ nid, err := bridge.ResolveCSSToNodeID(ctx, req.Selector)
+ if err != nil {
+ return
+ }
+ req.NodeID = nid
+}
+
+func shouldRetryStaleRef(err error) bool {
+ if err == nil {
+ return false
+ }
+ if errors.Is(err, bridge.ErrElementStale) {
+ return true
+ }
+ // Fallback string matching is still needed for stale failures that can bypass
+ // bridge.ExecuteAction classification (for example, lite-engine paths or other
+ // non-bridge error surfaces that return raw backend-node messages).
+ e := strings.ToLower(err.Error())
+ return strings.Contains(e, "could not find node") || strings.Contains(e, "node with given id") || strings.Contains(e, "no node")
+}
+
+func (h *Handlers) refreshRefCache(ctx context.Context, tabID string) {
+ nodes, err := bridge.FetchAXTree(ctx)
+ if err != nil {
+ return
+ }
+ flat, refs := bridge.BuildSnapshot(nodes, bridge.FilterInteractive, -1)
+ _ = bridge.EnrichA11yNodesWithDOMMetadata(ctx, flat)
+ h.Bridge.SetRefCache(tabID, &bridge.RefCache{
+ Refs: refs,
+ Targets: bridge.RefTargetsFromNodes(flat),
+ Nodes: flat,
+ })
+}
+
+func isClickTimeoutWithPendingDialog(err error, kind, tabID string, b bridge.BridgeAPI) bool {
+ if err == nil || tabID == "" {
+ return false
+ }
+ if !errors.Is(err, context.DeadlineExceeded) {
+ return false
+ }
+ kind = bridge.CanonicalActionKind(kind)
+ if kind != bridge.ActionClick && kind != bridge.ActionDoubleClick {
+ return false
+ }
+ dm := b.GetDialogManager()
+ if dm == nil {
+ return false
+ }
+ return dm.GetPending(tabID) != nil
+}
diff --git a/internal/handlers/action_selector_resolution.go b/internal/handlers/action_selector_resolution.go
new file mode 100644
index 00000000..ada8cf82
--- /dev/null
+++ b/internal/handlers/action_selector_resolution.go
@@ -0,0 +1,100 @@
+package handlers
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+
+ "github.com/pinchtab/pinchtab/internal/bridge"
+ "github.com/pinchtab/pinchtab/internal/selector"
+)
+
+type actionSelectorResolution struct {
+ refMissing bool
+ status int
+}
+
+func (r actionSelectorResolution) httpStatus() int {
+ if r.status != 0 {
+ return r.status
+ }
+ return http.StatusBadRequest
+}
+
+func frameScopedSelectorError(kind string, err error) error {
+ return fmt.Errorf("%s in current frame: %w", kind, err)
+}
+
+func (h *Handlers) resolveActionRequestSelector(ctx context.Context, tabID string, useLiteAction bool, req *bridge.ActionRequest) (actionSelectorResolution, error) {
+ req.NormalizeSelector()
+ if useLiteAction || req.NodeID != 0 {
+ return actionSelectorResolution{}, nil
+ }
+ if req.Selector == "" {
+ if req.Ref != "" {
+ return actionSelectorResolution{refMissing: true}, nil
+ }
+ return actionSelectorResolution{}, nil
+ }
+
+ sel := selector.Parse(req.Selector)
+ if handled, err := h.applySemanticActionSelector(ctx, tabID, sel, req); handled {
+ if err != nil {
+ return actionSelectorResolution{status: semanticSelectorHTTPStatus(err)}, err
+ }
+ return actionSelectorResolution{}, nil
+ }
+
+ switch sel.Kind {
+ case selector.KindRef:
+ req.Ref = sel.Value
+ req.Selector = ""
+ cache := h.Bridge.GetRefCache(tabID)
+ if cache != nil {
+ if target, ok := cache.Lookup(sel.Value); ok {
+ req.NodeID = target.BackendNodeID
+ }
+ }
+ if req.NodeID == 0 {
+ return actionSelectorResolution{refMissing: true}, nil
+ }
+ case selector.KindCSS:
+ req.Ref = ""
+ nid, err := bridge.ResolveCSSToNodeIDInFrame(ctx, h.selectorFrameID(tabID), sel.Value)
+ if err != nil {
+ return actionSelectorResolution{}, frameScopedSelectorError("css selector", err)
+ }
+ req.NodeID = nid
+ req.Selector = ""
+ case selector.KindXPath:
+ nid, err := bridge.ResolveXPathToNodeIDInFrame(ctx, h.selectorFrameID(tabID), sel.Value)
+ if err != nil {
+ return actionSelectorResolution{}, frameScopedSelectorError("xpath selector", err)
+ }
+ req.NodeID = nid
+ req.Selector = ""
+ req.Ref = ""
+ case selector.KindText:
+ nid, err := bridge.ResolveTextToNodeIDInFrame(ctx, h.selectorFrameID(tabID), sel.Value)
+ if err != nil {
+ return actionSelectorResolution{}, frameScopedSelectorError("text selector", err)
+ }
+ req.NodeID = nid
+ req.Selector = ""
+ req.Ref = ""
+ case selector.KindRole, selector.KindLabel, selector.KindPlaceholder,
+ selector.KindAlt, selector.KindTitle, selector.KindTestID,
+ selector.KindFirst, selector.KindLast, selector.KindNth:
+ nid, err := bridge.ResolveUnifiedSelectorInFrame(ctx, sel, h.Bridge.GetRefCache(tabID), h.selectorFrameID(tabID))
+ if err != nil {
+ return actionSelectorResolution{}, frameScopedSelectorError("selector", err)
+ }
+ req.NodeID = nid
+ req.Selector = ""
+ req.Ref = ""
+ case selector.KindSemantic:
+ return actionSelectorResolution{}, fmt.Errorf("semantic selector requires a non-empty query")
+ }
+
+ return actionSelectorResolution{}, nil
+}
diff --git a/internal/handlers/action_types.go b/internal/handlers/action_types.go
new file mode 100644
index 00000000..ab81ac2d
--- /dev/null
+++ b/internal/handlers/action_types.go
@@ -0,0 +1,27 @@
+package handlers
+
+import "github.com/pinchtab/pinchtab/internal/bridge"
+
+type actionsRequest struct {
+ TabID string `json:"tabId"`
+ Owner string `json:"owner"`
+ Actions []bridge.ActionRequest `json:"actions"`
+ StopOnError bool `json:"stopOnError"`
+}
+
+type actionResult struct {
+ Index int `json:"index"`
+ Success bool `json:"success"`
+ Result map[string]any `json:"result,omitempty"`
+ Error string `json:"error,omitempty"`
+}
+
+func countSuccessful(results []actionResult) int {
+ count := 0
+ for _, r := range results {
+ if r.Success {
+ count++
+ }
+ }
+ return count
+}
diff --git a/internal/handlers/actions.go b/internal/handlers/actions.go
index 7e3dbb82..03f3978f 100644
--- a/internal/handlers/actions.go
+++ b/internal/handlers/actions.go
@@ -16,8 +16,6 @@ import (
"github.com/pinchtab/pinchtab/internal/bridge"
"github.com/pinchtab/pinchtab/internal/engine"
"github.com/pinchtab/pinchtab/internal/httpx"
- "github.com/pinchtab/pinchtab/internal/selector"
- "github.com/pinchtab/semantic"
"github.com/pinchtab/semantic/recovery"
)
@@ -31,10 +29,6 @@ func resolveOwner(r *http.Request, fallback string) string {
return strings.TrimSpace(fallback)
}
-func frameScopedSelectorError(kind string, err error) error {
- return fmt.Errorf("%s in current frame: %w", kind, err)
-}
-
func (h *Handlers) enforceTabLease(tabID, owner string) error {
if tabID == "" {
return nil
@@ -204,77 +198,13 @@ func (h *Handlers) HandleAction(w http.ResponseWriter, r *http.Request) {
go httpx.CancelOnClientDone(r.Context(), tCancel)
// Unified selector resolution: normalize legacy ref/selector fields
- // into the unified Selector, then resolve to a nodeID when possible.
- req.NormalizeSelector()
- refMissing := false
- if !useLiteAction && req.NodeID == 0 && req.Selector != "" {
- sel := selector.Parse(req.Selector)
- if handled, err := h.applySemanticActionSelector(tCtx, resolvedTabID, sel, &req); handled {
- if err != nil {
- httpx.Error(w, semanticSelectorHTTPStatus(err), err)
- return
- }
- } else {
- switch sel.Kind {
- case selector.KindRef:
- // Ensure Ref is set for downstream recovery/intent caching.
- req.Ref = sel.Value
- // Clear Selector so the bridge doesn't try to use the ref
- // string as a CSS selector (it checks Selector before NodeID).
- req.Selector = ""
- cache := h.Bridge.GetRefCache(resolvedTabID)
- if cache != nil {
- if target, ok := cache.Lookup(sel.Value); ok {
- req.NodeID = target.BackendNodeID
- }
- }
- if req.NodeID == 0 {
- refMissing = true
- }
- case selector.KindCSS:
- req.Ref = ""
- nid, err := bridge.ResolveCSSToNodeIDInFrame(tCtx, h.selectorFrameID(resolvedTabID), sel.Value)
- if err != nil {
- httpx.Error(w, 400, frameScopedSelectorError("css selector", err))
- return
- }
- req.NodeID = nid
- req.Selector = ""
- case selector.KindXPath:
- nid, err := bridge.ResolveXPathToNodeIDInFrame(tCtx, h.selectorFrameID(resolvedTabID), sel.Value)
- if err != nil {
- httpx.Error(w, 400, frameScopedSelectorError("xpath selector", err))
- return
- }
- req.NodeID = nid
- req.Selector = ""
- req.Ref = ""
- case selector.KindText:
- nid, err := bridge.ResolveTextToNodeIDInFrame(tCtx, h.selectorFrameID(resolvedTabID), sel.Value)
- if err != nil {
- httpx.Error(w, 400, frameScopedSelectorError("text selector", err))
- return
- }
- req.NodeID = nid
- req.Selector = ""
- req.Ref = ""
- case selector.KindRole, selector.KindLabel, selector.KindPlaceholder,
- selector.KindAlt, selector.KindTitle, selector.KindTestID,
- selector.KindFirst, selector.KindLast, selector.KindNth:
- nid, err := bridge.ResolveUnifiedSelectorInFrame(tCtx, sel, h.Bridge.GetRefCache(resolvedTabID), h.selectorFrameID(resolvedTabID))
- if err != nil {
- httpx.Error(w, 400, frameScopedSelectorError("selector", err))
- return
- }
- req.NodeID = nid
- req.Selector = ""
- req.Ref = ""
- case selector.KindSemantic:
- httpx.Error(w, 400, fmt.Errorf("semantic selector requires a non-empty query"))
- return
- }
- }
+ // and resolve browser/semantic selectors to node IDs when possible.
+ selectorResolution, err := h.resolveActionRequestSelector(tCtx, resolvedTabID, useLiteAction, &req)
+ if err != nil {
+ httpx.Error(w, selectorResolution.httpStatus(), err)
+ return
}
+ refMissing := selectorResolution.refMissing
// Cache intent before execution so recovery can reconstruct the query.
// Only cache when the ref IS in the snapshot — otherwise we'd overwrite
@@ -439,20 +369,6 @@ func (h *Handlers) HandleTabAction(w http.ResponseWriter, r *http.Request) {
h.HandleAction(w, wrapped)
}
-type actionsRequest struct {
- TabID string `json:"tabId"`
- Owner string `json:"owner"`
- Actions []bridge.ActionRequest `json:"actions"`
- StopOnError bool `json:"stopOnError"`
-}
-
-type actionResult struct {
- Index int `json:"index"`
- Success bool `json:"success"`
- Result map[string]any `json:"result,omitempty"`
- Error string `json:"error,omitempty"`
-}
-
func (h *Handlers) HandleActions(w http.ResponseWriter, r *http.Request) {
var req actionsRequest
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBodySize)).Decode(&req); err != nil {
@@ -576,120 +492,19 @@ func (h *Handlers) handleActionsBatch(w http.ResponseWriter, r *http.Request, re
tCtx, tCancel := context.WithTimeout(ctx, h.Config.ActionTimeout)
useLiteAction := h.shouldUseLiteAction(action)
- // Unified selector resolution for batch actions.
- action.NormalizeSelector()
- refMissing := false
- if !useLiteAction && action.NodeID == 0 && action.Selector != "" {
- sel := selector.Parse(action.Selector)
- if handled, resolveErr := h.applySemanticActionSelector(tCtx, resolvedTabID, sel, &action); handled {
- if resolveErr != nil {
- tCancel()
- results = append(results, actionResult{
- Index: i, Success: false,
- Error: resolveErr.Error(),
- })
- if req.StopOnError {
- break
- }
- continue
- }
- } else {
- switch sel.Kind {
- case selector.KindRef:
- action.Ref = sel.Value
- action.Selector = ""
- cache := h.Bridge.GetRefCache(resolvedTabID)
- if cache != nil {
- if target, ok := cache.Lookup(sel.Value); ok {
- action.NodeID = target.BackendNodeID
- }
- }
- if action.NodeID == 0 {
- refMissing = true
- }
- case selector.KindCSS:
- action.Ref = ""
- nid, resolveErr := bridge.ResolveCSSToNodeIDInFrame(tCtx, h.selectorFrameID(resolvedTabID), sel.Value)
- if resolveErr != nil {
- tCancel()
- results = append(results, actionResult{
- Index: i, Success: false,
- Error: frameScopedSelectorError("css selector", resolveErr).Error(),
- })
- if req.StopOnError {
- break
- }
- continue
- }
- action.NodeID = nid
- action.Selector = ""
- case selector.KindXPath:
- nid, resolveErr := bridge.ResolveXPathToNodeIDInFrame(tCtx, h.selectorFrameID(resolvedTabID), sel.Value)
- if resolveErr != nil {
- tCancel()
- results = append(results, actionResult{
- Index: i, Success: false,
- Error: frameScopedSelectorError("xpath selector", resolveErr).Error(),
- })
- if req.StopOnError {
- break
- }
- continue
- }
- action.NodeID = nid
- action.Selector = ""
- action.Ref = ""
- case selector.KindText:
- nid, resolveErr := bridge.ResolveTextToNodeIDInFrame(tCtx, h.selectorFrameID(resolvedTabID), sel.Value)
- if resolveErr != nil {
- tCancel()
- results = append(results, actionResult{
- Index: i, Success: false,
- Error: frameScopedSelectorError("text selector", resolveErr).Error(),
- })
- if req.StopOnError {
- break
- }
- continue
- }
- action.NodeID = nid
- action.Selector = ""
- action.Ref = ""
- case selector.KindRole, selector.KindLabel, selector.KindPlaceholder,
- selector.KindAlt, selector.KindTitle, selector.KindTestID,
- selector.KindFirst, selector.KindLast, selector.KindNth:
- nid, resolveErr := bridge.ResolveUnifiedSelectorInFrame(tCtx, sel, h.Bridge.GetRefCache(resolvedTabID), h.selectorFrameID(resolvedTabID))
- if resolveErr != nil {
- tCancel()
- results = append(results, actionResult{
- Index: i, Success: false,
- Error: frameScopedSelectorError("selector", resolveErr).Error(),
- })
- if req.StopOnError {
- break
- }
- continue
- }
- action.NodeID = nid
- action.Selector = ""
- action.Ref = ""
- case selector.KindSemantic:
- tCancel()
- results = append(results, actionResult{
- Index: i, Success: false,
- Error: "semantic selector requires a non-empty query",
- })
- if req.StopOnError {
- break
- }
- continue
- }
+ selectorResolution, resolveErr := h.resolveActionRequestSelector(tCtx, resolvedTabID, useLiteAction, &action)
+ if resolveErr != nil {
+ tCancel()
+ results = append(results, actionResult{
+ Index: i, Success: false,
+ Error: resolveErr.Error(),
+ })
+ if req.StopOnError {
+ break
}
- } else if !useLiteAction && action.Ref != "" && action.NodeID == 0 {
- // Legacy path: Ref set but NormalizeSelector didn't promote it
- // (shouldn't happen, but defensive).
- refMissing = true
+ continue
}
+ refMissing := selectorResolution.refMissing
if action.Kind == "" {
tCancel()
@@ -891,121 +706,20 @@ func (h *Handlers) HandleMacro(w http.ResponseWriter, r *http.Request) {
}
}
useLiteAction := h.shouldUseLiteAction(step)
- // Unified selector resolution for macro steps (mirrors HandleAction).
- step.NormalizeSelector()
- stepRefMissing := false
- if !useLiteAction && step.NodeID == 0 && step.Selector != "" {
- sel := selector.Parse(step.Selector)
- semanticCtx, semanticCancel := context.WithTimeout(ctx, stepTimeout)
- handled, resolveErr := h.applySemanticActionSelector(semanticCtx, resolvedTabID, sel, &step)
- semanticCancel()
- if handled {
- if resolveErr != nil {
- results = append(results, actionResult{
- Index: i, Success: false,
- Error: resolveErr.Error(),
- })
- if req.StopOnError {
- break
- }
- continue
- }
- } else {
- switch sel.Kind {
- case selector.KindRef:
- step.Ref = sel.Value
- step.Selector = ""
- cache := h.Bridge.GetRefCache(resolvedTabID)
- if cache != nil {
- if target, ok := cache.Lookup(sel.Value); ok {
- step.NodeID = target.BackendNodeID
- }
- }
- if step.NodeID == 0 {
- stepRefMissing = true
- }
- case selector.KindCSS:
- step.Ref = ""
- tCtx, cancel := context.WithTimeout(ctx, stepTimeout)
- nid, resolveErr := bridge.ResolveCSSToNodeIDInFrame(tCtx, h.selectorFrameID(resolvedTabID), sel.Value)
- cancel()
- if resolveErr != nil {
- results = append(results, actionResult{
- Index: i, Success: false,
- Error: frameScopedSelectorError("css selector", resolveErr).Error(),
- })
- if req.StopOnError {
- break
- }
- continue
- }
- step.NodeID = nid
- step.Selector = ""
- case selector.KindXPath:
- tCtx, cancel := context.WithTimeout(ctx, stepTimeout)
- nid, resolveErr := bridge.ResolveXPathToNodeIDInFrame(tCtx, h.selectorFrameID(resolvedTabID), sel.Value)
- cancel()
- if resolveErr != nil {
- results = append(results, actionResult{
- Index: i, Success: false,
- Error: frameScopedSelectorError("xpath selector", resolveErr).Error(),
- })
- if req.StopOnError {
- break
- }
- continue
- }
- step.NodeID = nid
- step.Selector = ""
- step.Ref = ""
- case selector.KindText:
- tCtx, cancel := context.WithTimeout(ctx, stepTimeout)
- nid, resolveErr := bridge.ResolveTextToNodeIDInFrame(tCtx, h.selectorFrameID(resolvedTabID), sel.Value)
- cancel()
- if resolveErr != nil {
- results = append(results, actionResult{
- Index: i, Success: false,
- Error: frameScopedSelectorError("text selector", resolveErr).Error(),
- })
- if req.StopOnError {
- break
- }
- continue
- }
- step.NodeID = nid
- step.Selector = ""
- step.Ref = ""
- case selector.KindRole, selector.KindLabel, selector.KindPlaceholder,
- selector.KindAlt, selector.KindTitle, selector.KindTestID,
- selector.KindFirst, selector.KindLast, selector.KindNth:
- tCtx, cancel := context.WithTimeout(ctx, stepTimeout)
- nid, resolveErr := bridge.ResolveUnifiedSelectorInFrame(tCtx, sel, h.Bridge.GetRefCache(resolvedTabID), h.selectorFrameID(resolvedTabID))
- cancel()
- if resolveErr != nil {
- results = append(results, actionResult{
- Index: i, Success: false,
- Error: frameScopedSelectorError("selector", resolveErr).Error(),
- })
- if req.StopOnError {
- break
- }
- continue
- }
- step.NodeID = nid
- step.Selector = ""
- step.Ref = ""
- case selector.KindSemantic:
- results = append(results, actionResult{
- Index: i, Success: false,
- Error: "semantic selector requires a non-empty query",
- })
- if req.StopOnError {
- break
- }
- continue
- }
+ selectorCtx, selectorCancel := context.WithTimeout(ctx, stepTimeout)
+ selectorResolution, resolveErr := h.resolveActionRequestSelector(selectorCtx, resolvedTabID, useLiteAction, &step)
+ selectorCancel()
+ if resolveErr != nil {
+ results = append(results, actionResult{
+ Index: i, Success: false,
+ Error: resolveErr.Error(),
+ })
+ if req.StopOnError {
+ break
}
+ continue
}
+ stepRefMissing := selectorResolution.refMissing
// Cache intent before execution so recovery can reconstruct the query.
// Only cache when the ref IS in the snapshot to avoid overwriting
@@ -1114,201 +828,3 @@ func (h *Handlers) HandleMacro(w http.ResponseWriter, r *http.Request) {
"failed": len(req.Steps) - successful,
})
}
-
-func countSuccessful(results []actionResult) int {
- count := 0
- for _, r := range results {
- if r.Success {
- count++
- }
- }
- return count
-}
-
-// cacheActionIntent stores the element's semantic identity in the
-// IntentCache so the recovery engine can reconstruct a query if the
-// ref becomes stale.
-func (h *Handlers) cacheActionIntent(tabID string, req bridge.ActionRequest) {
- if h.Recovery == nil || req.Ref == "" {
- return
- }
- // Don't overwrite an existing entry that has a real Query (from /find)
- // with a descriptor-only entry.
- if existing, ok := h.Recovery.IntentCache.Lookup(tabID, req.Ref); ok && existing.Query != "" {
- return
- }
- desc := semantic.ElementDescriptor{Ref: req.Ref}
- // Try to enrich from the current snapshot cache.
- if cache := h.Bridge.GetRefCache(tabID); cache != nil {
- for _, enriched := range semanticDescriptorsFromNodes(cache.Nodes) {
- if enriched.Ref == req.Ref {
- desc = enriched
- break
- }
- }
- }
- h.Recovery.RecordIntent(tabID, req.Ref, recovery.IntentEntry{
- Descriptor: desc,
- CachedAt: time.Now(),
- })
-}
-
-func (h *Handlers) executeAction(ctx context.Context, req bridge.ActionRequest) (map[string]any, string, error) {
- req.Kind = bridge.CanonicalActionKind(req.Kind)
- if h.shouldUseLiteAction(req) {
- return h.executeLiteAction(ctx, req)
- }
-
- if err := h.ensureChrome(); err != nil {
- return nil, "", fmt.Errorf("chrome initialization: %w", err)
- }
- result, err := h.Bridge.ExecuteAction(ctx, req.Kind, req)
- return result, "", err
-}
-
-func (h *Handlers) shouldUseLiteAction(req bridge.ActionRequest) bool {
- kind := bridge.CanonicalActionKind(req.Kind)
- if h.effectiveActionHumanize(req) && (kind == bridge.ActionClick || kind == bridge.ActionType || kind == bridge.ActionKeyboardType) {
- return false
- }
- capability, ok := actionCapability(kind)
- if !ok {
- return h.Router != nil && h.Router.Mode() == engine.ModeLite
- }
- return h.useLite(capability, "")
-}
-
-func (h *Handlers) effectiveActionHumanize(req bridge.ActionRequest) bool {
- if req.Humanize != nil {
- return *req.Humanize
- }
- if h != nil && h.Config != nil {
- return h.Config.Humanize
- }
- return false
-}
-
-func (h *Handlers) executeLiteAction(ctx context.Context, req bridge.ActionRequest) (map[string]any, string, error) {
- if h.Router == nil || h.Router.Lite() == nil {
- return nil, "", fmt.Errorf("lite engine unavailable")
- }
- switch bridge.CanonicalActionKind(req.Kind) {
- case bridge.ActionClick:
- if req.Ref == "" {
- return nil, "lite", fmt.Errorf("lite mode actions require ref from /snapshot")
- }
- if err := h.Router.Lite().Click(ctx, req.TabID, req.Ref); err != nil {
- return nil, "lite", err
- }
- return map[string]any{"clicked": true}, "lite", nil
- case bridge.ActionType, bridge.ActionFill:
- if req.Ref == "" {
- return nil, "lite", fmt.Errorf("lite mode actions require ref from /snapshot")
- }
- text := req.Text
- if req.Kind == bridge.ActionFill && text == "" {
- text = req.Value
- }
- if text == "" {
- return nil, "lite", fmt.Errorf("text required for %s", req.Kind)
- }
- if err := h.Router.Lite().Type(ctx, req.TabID, req.Ref, text); err != nil {
- return nil, "lite", err
- }
- return map[string]any{"typed": text}, "lite", nil
- default:
- return nil, "lite", fmt.Errorf("%w: %s", engine.ErrLiteNotSupported, req.Kind)
- }
-}
-
-func actionCapability(kind string) (engine.Capability, bool) {
- switch bridge.CanonicalActionKind(kind) {
- case bridge.ActionClick:
- return engine.CapClick, true
- case bridge.ActionType, bridge.ActionFill:
- return engine.CapType, true
- default:
- return "", false
- }
-}
-
-const pointerRetryDelay = 50 * time.Millisecond
-
-func shouldRetryPointerAction(req bridge.ActionRequest, err error) bool {
- if err == nil {
- return false
- }
- kind := strings.ToLower(strings.TrimSpace(req.Kind))
- switch kind {
- case bridge.ActionClick, bridge.ActionDoubleClick, bridge.ActionHover, bridge.ActionDrag,
- bridge.ActionMouseDown, bridge.ActionMouseUp, bridge.ActionMouseWheel:
- // pointer action kinds
- default:
- return false
- }
-
- if errors.Is(err, bridge.ErrElementOccluded) ||
- errors.Is(err, bridge.ErrElementBlocked) ||
- errors.Is(err, bridge.ErrElementOffscreen) {
- return true
- }
-
- return shouldRetryStaleRef(err)
-}
-
-func (h *Handlers) refreshActionNodeIDFromSelector(ctx context.Context, req *bridge.ActionRequest) {
- if req == nil || req.NodeID > 0 || strings.TrimSpace(req.Selector) == "" {
- return
- }
- nid, err := bridge.ResolveCSSToNodeID(ctx, req.Selector)
- if err != nil {
- return
- }
- req.NodeID = nid
-}
-
-func shouldRetryStaleRef(err error) bool {
- if err == nil {
- return false
- }
- if errors.Is(err, bridge.ErrElementStale) {
- return true
- }
- // Fallback string matching is still needed for stale failures that can bypass
- // bridge.ExecuteAction classification (for example, lite-engine paths or other
- // non-bridge error surfaces that return raw backend-node messages).
- e := strings.ToLower(err.Error())
- return strings.Contains(e, "could not find node") || strings.Contains(e, "node with given id") || strings.Contains(e, "no node")
-}
-
-func (h *Handlers) refreshRefCache(ctx context.Context, tabID string) {
- nodes, err := bridge.FetchAXTree(ctx)
- if err != nil {
- return
- }
- flat, refs := bridge.BuildSnapshot(nodes, bridge.FilterInteractive, -1)
- _ = bridge.EnrichA11yNodesWithDOMMetadata(ctx, flat)
- h.Bridge.SetRefCache(tabID, &bridge.RefCache{
- Refs: refs,
- Targets: bridge.RefTargetsFromNodes(flat),
- Nodes: flat,
- })
-}
-
-func isClickTimeoutWithPendingDialog(err error, kind, tabID string, b bridge.BridgeAPI) bool {
- if err == nil || tabID == "" {
- return false
- }
- if !errors.Is(err, context.DeadlineExceeded) {
- return false
- }
- kind = bridge.CanonicalActionKind(kind)
- if kind != bridge.ActionClick && kind != bridge.ActionDoubleClick {
- return false
- }
- dm := b.GetDialogManager()
- if dm == nil {
- return false
- }
- return dm.GetPending(tabID) != nil
-}