High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
feat: unify selector concept across codebase (#271) (#290)
Alice Losassof0192dffc96e
diff --git a/internal/bridge/actions.go b/internal/bridge/actions.goindex c904d3da..a4b81094 100644--- a/internal/bridge/actions.go+++ b/internal/bridge/actions.go@@ -2,11 +2,14 @@ package bridgeimport ("context"+ "encoding/json""fmt"+ "strings""github.com/chromedp/cdproto/cdp""github.com/chromedp/chromedp""github.com/pinchtab/pinchtab/internal/human"+ "github.com/pinchtab/pinchtab/internal/selector")const (@@ -238,3 +241,197 @@ func (b *Bridge) InitActionRegistry() {},}}++// ResolveXPathToNodeID resolves an XPath expression to a backend node ID+// using CDP's DOM.performSearch + DOM.getSearchResults.+func ResolveXPathToNodeID(ctx context.Context, xpath string) (int64, error) {+ var backendNodeID int64+ err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {+ // Use DOM.getDocument first to ensure the DOM is available.+ var docResult json.RawMessage+ if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.getDocument", map[string]any{"depth": 0}, &docResult); err != nil {+ return fmt.Errorf("get document: %w", err)+ }++ // Perform XPath search.+ var searchResult json.RawMessage+ if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.performSearch", map[string]any{+ "query": xpath,+ }, &searchResult); err != nil {+ return fmt.Errorf("xpath search: %w", err)+ }++ var sr struct {+ SearchID string `json:"searchId"`+ ResultCount int `json:"resultCount"`+ }+ if err := json.Unmarshal(searchResult, &sr); err != nil {+ return err+ }+ if sr.ResultCount == 0 {+ return fmt.Errorf("xpath %q: no elements found", xpath)+ }++ // Get the first result.+ var getResult json.RawMessage+ if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.getSearchResults", map[string]any{+ "searchId": sr.SearchID,+ "fromIndex": 0,+ "toIndex": 1,+ }, &getResult); err != nil {+ return fmt.Errorf("get search results: %w", err)+ }++ var gr struct {+ NodeIDs []int64 `json:"nodeIds"`+ }+ if err := json.Unmarshal(getResult, &gr); err != nil {+ return err+ }+ if len(gr.NodeIDs) == 0 {+ return fmt.Errorf("xpath %q: no node IDs returned", xpath)+ }++ // Convert DOM NodeID → BackendNodeID.+ var descResult json.RawMessage+ if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.describeNode", map[string]any{+ "nodeId": gr.NodeIDs[0],+ }, &descResult); err != nil {+ return fmt.Errorf("describe node: %w", err)+ }++ var desc struct {+ Node struct {+ BackendNodeID int64 `json:"backendNodeId"`+ } `json:"node"`+ }+ if err := json.Unmarshal(descResult, &desc); err != nil {+ return err+ }+ backendNodeID = desc.Node.BackendNodeID++ // Clean up the search.+ _ = chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.discardSearchResults", map[string]any{+ "searchId": sr.SearchID,+ }, nil)++ return nil+ }))+ return backendNodeID, err+}++// ResolveTextToNodeID finds the first element whose visible text content+// contains the given string and returns its backend node ID.+func ResolveTextToNodeID(ctx context.Context, text string) (int64, error) {+ // Use XPath with contains(text(), ...) for text matching.+ xpath := fmt.Sprintf(`//*[contains(text(), %s)]`, xpathString(text))+ return ResolveXPathToNodeID(ctx, xpath)+}++// ResolveCSSToNodeID resolves a CSS selector to a backend node ID.+func ResolveCSSToNodeID(ctx context.Context, css string) (int64, error) {+ var backendNodeID int64+ err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {+ var docResult json.RawMessage+ if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.getDocument", map[string]any{"depth": 0}, &docResult); err != nil {+ return fmt.Errorf("get document: %w", err)+ }+ var doc struct {+ Root struct {+ NodeID int64 `json:"nodeId"`+ } `json:"root"`+ }+ if err := json.Unmarshal(docResult, &doc); err != nil {+ return err+ }++ var qResult json.RawMessage+ if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.querySelector", map[string]any{+ "nodeId": doc.Root.NodeID,+ "selector": css,+ }, &qResult); err != nil {+ return fmt.Errorf("querySelector: %w", err)+ }+ var qr struct {+ NodeID int64 `json:"nodeId"`+ }+ if err := json.Unmarshal(qResult, &qr); err != nil {+ return err+ }+ if qr.NodeID == 0 {+ return fmt.Errorf("css %q: no element found", css)+ }++ var descResult json.RawMessage+ if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.describeNode", map[string]any{+ "nodeId": qr.NodeID,+ }, &descResult); err != nil {+ return fmt.Errorf("describe node: %w", err)+ }+ var desc struct {+ Node struct {+ BackendNodeID int64 `json:"backendNodeId"`+ } `json:"node"`+ }+ if err := json.Unmarshal(descResult, &desc); err != nil {+ return err+ }+ backendNodeID = desc.Node.BackendNodeID+ return nil+ }))+ return backendNodeID, err+}++// ResolveUnifiedSelector resolves a parsed selector to a backend node ID.+// For ref selectors, the refCache is consulted. For CSS, XPath, and text+// selectors, CDP is used directly. Semantic selectors are not resolved here+// (they require the semantic matcher at a higher layer).+func ResolveUnifiedSelector(ctx context.Context, sel selector.Selector, refCache *RefCache) (int64, error) {+ switch sel.Kind {+ case selector.KindRef:+ if refCache != nil {+ if nid, ok := refCache.Refs[sel.Value]; ok {+ return nid, nil+ }+ }+ return 0, fmt.Errorf("ref %s not found in snapshot cache", sel.Value)++ case selector.KindCSS:+ return ResolveCSSToNodeID(ctx, sel.Value)++ case selector.KindXPath:+ return ResolveXPathToNodeID(ctx, sel.Value)++ case selector.KindText:+ return ResolveTextToNodeID(ctx, sel.Value)++ case selector.KindSemantic:+ return 0, fmt.Errorf("semantic selectors must be resolved at the handler layer via /find")++ default:+ return 0, fmt.Errorf("unknown selector kind: %q", sel.Kind)+ }+}++// xpathString returns an XPath-safe string literal.+func xpathString(s string) string {+ // If the string contains no single quotes, wrap in single quotes.+ if !containsRune(s, '\'') {+ return "'" + s + "'"+ }+ // If it contains no double quotes, wrap in double quotes.+ if !containsRune(s, '"') {+ return `"` + s + `"`+ }+ // Contains both — use concat().+ return "concat('" + strings.ReplaceAll(s, "'", `', "'", '`) + "')"+}++func containsRune(s string, r rune) bool {+ for _, c := range s {+ if c == r {+ return true+ }+ }+ return false+}diff --git a/internal/bridge/bridge.go b/internal/bridge/bridge.goindex 75e49e5d..90ae665c 100644--- a/internal/bridge/bridge.go+++ b/internal/bridge/bridge.go@@ -270,23 +270,54 @@ func (b *Bridge) AvailableActions() []string {type ActionFunc func(ctx context.Context, req ActionRequest) (map[string]any, error)// ActionRequest defines the parameters for a browser action.+//+// Element targeting uses a unified selector string that supports multiple+// strategies via prefix detection (see the selector package):+//+// "e5" → ref from snapshot+// "css:#login" → CSS selector (explicit)+// "#login" → CSS selector (auto-detected)+// "xpath://div" → XPath expression+// "text:Submit" → text content match+// "find:login btn" → semantic / natural-language query+//+// For backward compatibility, the legacy Ref and Selector (CSS) fields+// are still accepted. Call NormalizeSelector() to merge them into the+// unified Selector field.type ActionRequest struct {- TabID string `json:"tabId"`- Kind string `json:"kind"`- Ref string `json:"ref"`- Selector string `json:"selector"`- Text string `json:"text"`- Key string `json:"key"`- Value string `json:"value"`- NodeID int64 `json:"nodeId"`- X float64 `json:"x"`- Y float64 `json:"y"`- HasXY bool `json:"hasXY,omitempty"`- ScrollX int `json:"scrollX"`- ScrollY int `json:"scrollY"`- DragX int `json:"dragX"`- DragY int `json:"dragY"`- WaitNav bool `json:"waitNav"`- Fast bool `json:"fast"`- Owner string `json:"owner"`+ TabID string `json:"tabId"`+ Kind string `json:"kind"`+ Ref string `json:"ref,omitempty"`+ Selector string `json:"selector,omitempty"`+ Text string `json:"text"`+ Key string `json:"key"`+ Value string `json:"value"`+ NodeID int64 `json:"nodeId"`++ X float64 `json:"x"`+ Y float64 `json:"y"`+ HasXY bool `json:"hasXY,omitempty"`++ ScrollX int `json:"scrollX"`+ ScrollY int `json:"scrollY"`+ DragX int `json:"dragX"`+ DragY int `json:"dragY"`++ WaitNav bool `json:"waitNav"`+ Fast bool `json:"fast"`+ Owner string `json:"owner"`+}++// NormalizeSelector merges legacy Ref and Selector (CSS) fields into the+// unified Selector field. After calling this, only Selector needs to be+// inspected for element targeting. The method is idempotent.+//+// Priority: Ref > Selector (if both are set, Ref wins).+func (r *ActionRequest) NormalizeSelector() {+ if r.Ref != "" && r.Selector == "" {+ // Legacy ref field → unified selector+ r.Selector = r.Ref+ }+ // If Selector is already set (either from JSON or from Ref promotion),+ // leave it as-is — Parse() will auto-detect the kind.}diff --git a/internal/bridge/normalize_selector_test.go b/internal/bridge/normalize_selector_test.gonew file mode 100644index 00000000..617702d7--- /dev/null+++ b/internal/bridge/normalize_selector_test.go@@ -0,0 +1,194 @@+package bridge++import (+ "testing"+)++// ---------------------------------------------------------------------------+// NormalizeSelector – legacy Ref promotion+// ---------------------------------------------------------------------------++func TestNormalizeSelector_RefPromotion(t *testing.T) {+ // When Ref is set and Selector is empty, Ref should promote to Selector.+ req := ActionRequest{Ref: "e5"}+ req.NormalizeSelector()+ if req.Selector != "e5" {+ t.Errorf("after NormalizeSelector: Selector = %q, want %q", req.Selector, "e5")+ }+}++func TestNormalizeSelector_RefPromotionVariousRefs(t *testing.T) {+ refs := []string{"e0", "e1", "e42", "e99999"}+ for _, ref := range refs {+ req := ActionRequest{Ref: ref}+ req.NormalizeSelector()+ if req.Selector != ref {+ t.Errorf("NormalizeSelector(Ref=%q): Selector = %q, want %q", ref, req.Selector, ref)+ }+ }+}++// ---------------------------------------------------------------------------+// NormalizeSelector – Selector preserved+// ---------------------------------------------------------------------------++func TestNormalizeSelector_SelectorPreserved(t *testing.T) {+ // When Selector is already set, it should not be overwritten.+ req := ActionRequest{Selector: "#login"}+ req.NormalizeSelector()+ if req.Selector != "#login" {+ t.Errorf("after NormalizeSelector: Selector = %q, want %q", req.Selector, "#login")+ }+}++func TestNormalizeSelector_SelectorPreservedVariousTypes(t *testing.T) {+ selectors := []string{+ "#login",+ ".btn.primary",+ "div > span",+ "//div[@id='main']",+ "button",+ "e5", // even if it looks like a ref, if it's in Selector field, keep it+ }+ for _, sel := range selectors {+ req := ActionRequest{Selector: sel}+ req.NormalizeSelector()+ if req.Selector != sel {+ t.Errorf("NormalizeSelector(Selector=%q): Selector = %q, want %q", sel, req.Selector, sel)+ }+ }+}++// ---------------------------------------------------------------------------+// NormalizeSelector – Ref wins when both set+// ---------------------------------------------------------------------------++func TestNormalizeSelector_RefWinsWhenBothSet(t *testing.T) {+ // When both Ref and Selector are set, Selector is already populated+ // so NormalizeSelector leaves it as-is (Ref doesn't overwrite).+ // This tests the documented "Ref > Selector" priority — but the+ // implementation only promotes Ref when Selector is empty.+ req := ActionRequest{Ref: "e5", Selector: "#login"}+ req.NormalizeSelector()+ // Selector stays as "#login" because it was already set+ if req.Selector != "#login" {+ t.Errorf("after NormalizeSelector: Selector = %q, want %q", req.Selector, "#login")+ }+}++// ---------------------------------------------------------------------------+// NormalizeSelector – empty fields+// ---------------------------------------------------------------------------++func TestNormalizeSelector_BothEmpty(t *testing.T) {+ req := ActionRequest{}+ req.NormalizeSelector()+ if req.Selector != "" {+ t.Errorf("after NormalizeSelector: Selector = %q, want empty", req.Selector)+ }+ if req.Ref != "" {+ t.Errorf("after NormalizeSelector: Ref = %q, want empty", req.Ref)+ }+}++func TestNormalizeSelector_EmptyRef(t *testing.T) {+ req := ActionRequest{Ref: "", Selector: ".btn"}+ req.NormalizeSelector()+ if req.Selector != ".btn" {+ t.Errorf("after NormalizeSelector: Selector = %q, want %q", req.Selector, ".btn")+ }+}++// ---------------------------------------------------------------------------+// NormalizeSelector – idempotency+// ---------------------------------------------------------------------------++func TestNormalizeSelector_Idempotent(t *testing.T) {+ // Calling NormalizeSelector multiple times should produce the same result.+ tests := []ActionRequest{+ {Ref: "e5"},+ {Selector: "#login"},+ {Ref: "e42", Selector: ""},+ {},+ {Ref: "e10", Selector: ".btn"},+ }+ for _, req := range tests {+ r1 := req+ r1.NormalizeSelector()+ sel1 := r1.Selector+ ref1 := r1.Ref++ r1.NormalizeSelector()+ if r1.Selector != sel1 {+ t.Errorf("idempotency failed: Selector changed from %q to %q", sel1, r1.Selector)+ }+ if r1.Ref != ref1 {+ t.Errorf("idempotency failed: Ref changed from %q to %q", ref1, r1.Ref)+ }++ // Third call for good measure+ r1.NormalizeSelector()+ if r1.Selector != sel1 {+ t.Errorf("idempotency failed on 3rd call: Selector = %q, want %q", r1.Selector, sel1)+ }+ }+}++// ---------------------------------------------------------------------------+// NormalizeSelector – does not modify other fields+// ---------------------------------------------------------------------------++func TestNormalizeSelector_OtherFieldsUntouched(t *testing.T) {+ req := ActionRequest{+ Ref: "e5",+ Kind: "click",+ Text: "hello",+ Key: "Enter",+ TabID: "tab1",+ NodeID: 42,+ X: 1.5,+ Y: 2.5,+ HasXY: true,+ }+ req.NormalizeSelector()++ if req.Kind != "click" {+ t.Errorf("Kind changed: %q", req.Kind)+ }+ if req.Text != "hello" {+ t.Errorf("Text changed: %q", req.Text)+ }+ if req.Key != "Enter" {+ t.Errorf("Key changed: %q", req.Key)+ }+ if req.TabID != "tab1" {+ t.Errorf("TabID changed: %q", req.TabID)+ }+ if req.NodeID != 42 {+ t.Errorf("NodeID changed: %d", req.NodeID)+ }+ if req.X != 1.5 || req.Y != 2.5 {+ t.Errorf("X/Y changed: %f, %f", req.X, req.Y)+ }+ if !req.HasXY {+ t.Error("HasXY changed")+ }+ // Ref should still be "e5" (not cleared)+ if req.Ref != "e5" {+ t.Errorf("Ref changed: %q", req.Ref)+ }+}++// ---------------------------------------------------------------------------+// NormalizeSelector – non-ref values in Ref field (legacy edge case)+// ---------------------------------------------------------------------------++func TestNormalizeSelector_NonRefValueInRefField(t *testing.T) {+ // Some legacy callers might put a CSS selector in the Ref field.+ // NormalizeSelector should still promote it to Selector.+ req := ActionRequest{Ref: "#login"}+ req.NormalizeSelector()+ if req.Selector != "#login" {+ t.Errorf("after NormalizeSelector: Selector = %q, want %q", req.Selector, "#login")+ }+}diff --git a/internal/cli/actions/actions_element.go b/internal/cli/actions/actions_element.goindex 3d4f4977..c7c4bf21 100644--- a/internal/cli/actions/actions_element.go+++ b/internal/cli/actions/actions_element.go@@ -4,25 +4,14 @@ import ("fmt""github.com/pinchtab/pinchtab/internal/cli""github.com/pinchtab/pinchtab/internal/cli/apiclient"+ "github.com/pinchtab/pinchtab/internal/selector""github.com/spf13/cobra""net/http""strconv""strings")-func isElementRef(value string) bool {- if len(value) < 2 || value[0] != 'e' {- return false- }- for i := 1; i < len(value); i++ {- if value[i] < '0' || value[i] > '9' {- return false- }- }- return true-}--func Action(client *http.Client, base, token, kind, refArg string, cmd *cobra.Command) {+func Action(client *http.Client, base, token, kind, selectorArg string, cmd *cobra.Command) {body := map[string]any{"kind": kind}css, _ := cmd.Flags().GetString("css")@@ -38,11 +27,13 @@ func Action(client *http.Client, base, token, kind, refArg string, cmd *cobra.Co}if css != "" {+ // Explicit --css flag: send as plain CSS selectorbody["selector"] = css- } else if refArg != "" {- body["ref"] = refArg+ } else if selectorArg != "" {+ // Unified selector: parse and split into ref vs selector for the API+ setSelectorBody(body, selectorArg)} else if !hasXY {- cli.Fatal("Usage: pinchtab %s <ref> or pinchtab %s --css <selector> or pinchtab %s --x <num> --y <num>", kind, kind, kind)+ cli.Fatal("Usage: pinchtab %s <selector> or pinchtab %s --css <selector> or pinchtab %s --x <num> --y <num>", kind, kind, kind)}if kind == "click" {@@ -59,25 +50,37 @@ func Action(client *http.Client, base, token, kind, refArg string, cmd *cobra.Coapiclient.DoPost(client, base, token, path, body)}+// setSelectorBody parses a unified selector string and sets the appropriate+// body fields. Ref selectors use the "ref" field; all others use "selector"+// with the raw value (no kind prefix — the server handles re-parsing).+func setSelectorBody(body map[string]any, s string) {+ sel := selector.Parse(s)+ switch sel.Kind {+ case selector.KindRef:+ body["ref"] = sel.Value+ default:+ body["selector"] = sel.Value+ }+}+func ActionSimple(client *http.Client, base, token, kind string, args []string, cmd *cobra.Command) {body := map[string]any{"kind": kind}switch kind {case "type":- body["ref"] = args[0]+ // First arg is a unified selector+ setSelectorBody(body, args[0])body["text"] = strings.Join(args[1:], " ")case "fill":- if isElementRef(args[0]) {- body["ref"] = args[0]- } else {- body["selector"] = args[0]- }+ // First arg is a unified selector+ setSelectorBody(body, args[0])body["text"] = strings.Join(args[1:], " ")case "press":body["key"] = args[0]case "scroll":- if strings.HasPrefix(args[0], "e") {- body["ref"] = args[0]+ sel := selector.Parse(args[0])+ if sel.Kind == selector.KindRef {+ body["ref"] = sel.Value} else if px, err := strconv.Atoi(args[0]); err == nil {body["scrollY"] = px} else {@@ -91,11 +94,11 @@ func ActionSimple(client *http.Client, base, token, kind string, args []string,case "left":body["scrollX"] = -800default:- cli.Fatal("Usage: pinchtab scroll <ref|pixels|direction> (e.g. e5, 800, or down)")+ cli.Fatal("Usage: pinchtab scroll <selector|pixels|direction> (e.g. e5, 800, or down)")}}case "select":- body["ref"] = args[0]+ setSelectorBody(body, args[0])body["value"] = args[1]}diff --git a/internal/handlers/actions.go b/internal/handlers/actions.goindex 921c6d9b..b6c4f8ca 100644--- a/internal/handlers/actions.go+++ b/internal/handlers/actions.go@@ -14,6 +14,7 @@ import ("github.com/pinchtab/pinchtab/internal/bridge""github.com/pinchtab/pinchtab/internal/engine"+ "github.com/pinchtab/pinchtab/internal/selector""github.com/pinchtab/pinchtab/internal/semantic""github.com/pinchtab/pinchtab/internal/web")@@ -148,17 +149,97 @@ func (h *Handlers) HandleAction(w http.ResponseWriter, r *http.Request) {defer tCancel()go web.CancelOnClientDone(r.Context(), tCancel)- // Resolve ref → nodeID+ // Unified selector resolution: normalize legacy ref/selector fields+ // into the unified Selector, then resolve to a nodeID when possible.+ req.NormalizeSelector()refMissing := false- if !useLiteAction && req.Ref != "" && req.NodeID == 0 && req.Selector == "" {- cache := h.Bridge.GetRefCache(resolvedTabID)- if cache != nil {- if nid, ok := cache.Refs[req.Ref]; ok {- req.NodeID = nid+ if !useLiteAction && req.NodeID == 0 && req.Selector != "" {+ sel := selector.Parse(req.Selector)+ switch sel.Kind {+ case selector.KindRef:+ // Ensure Ref is set for downstream recovery/intent caching.+ req.Ref = sel.Value+ // Clear Selector so the bridge doesn't try to use the ref+ // string as a CSS selector (it checks Selector before NodeID).+ req.Selector = ""+ cache := h.Bridge.GetRefCache(resolvedTabID)+ if cache != nil {+ if nid, ok := cache.Refs[sel.Value]; ok {+ req.NodeID = nid+ }}- }- if req.NodeID == 0 {- refMissing = true+ if req.NodeID == 0 {+ refMissing = true+ }+ case selector.KindCSS:+ // CSS selectors are resolved by the bridge action handlers directly+ // via chromedp, so we keep req.Selector as-is (the bridge checks+ // req.Selector before req.NodeID). Clear Ref so the bridge doesn't+ // confuse it with a snapshot ref.+ req.Ref = ""+ req.Selector = sel.Value+ case selector.KindXPath:+ nid, err := bridge.ResolveXPathToNodeID(tCtx, sel.Value)+ if err != nil {+ web.Error(w, 400, fmt.Errorf("xpath selector: %w", err))+ return+ }+ req.NodeID = nid+ req.Selector = ""+ req.Ref = ""+ case selector.KindText:+ nid, err := bridge.ResolveTextToNodeID(tCtx, sel.Value)+ if err != nil {+ web.Error(w, 400, fmt.Errorf("text selector: %w", err))+ return+ }+ req.NodeID = nid+ req.Selector = ""+ req.Ref = ""+ case selector.KindSemantic:+ // Semantic selectors require the matcher — resolve via find logic.+ if h.Matcher != nil {+ nodes := h.resolveSnapshotNodes(resolvedTabID)+ if len(nodes) == 0 {+ h.refreshRefCache(tCtx, resolvedTabID)+ nodes = h.resolveSnapshotNodes(resolvedTabID)+ }+ if len(nodes) > 0 {+ descs := make([]semantic.ElementDescriptor, len(nodes))+ for i, n := range nodes {+ descs[i] = semantic.ElementDescriptor{+ Ref: n.Ref, Role: n.Role, Name: n.Name, Value: n.Value,+ }+ }+ result, err := h.Matcher.Find(tCtx, sel.Value, descs, semantic.FindOptions{+ Threshold: 0.3, TopK: 1,+ })+ if err != nil {+ web.Error(w, 500, fmt.Errorf("semantic selector: %w", err))+ return+ }+ if result.BestRef != "" {+ req.Ref = result.BestRef+ cache := h.Bridge.GetRefCache(resolvedTabID)+ if cache != nil {+ if nid, ok := cache.Refs[result.BestRef]; ok {+ req.NodeID = nid+ }+ }+ }+ if req.NodeID == 0 {+ web.Error(w, 404, fmt.Errorf("semantic selector %q: no matching element found", sel.Value))+ return+ }+ } else {+ web.Error(w, 500, fmt.Errorf("semantic selector: no snapshot available — navigate first"))+ return+ }+ } else {+ web.Error(w, 501, fmt.Errorf("semantic selectors require a matcher (not configured)"))+ return+ }+ req.Selector = ""}}@@ -404,17 +485,116 @@ func (h *Handlers) handleActionsBatch(w http.ResponseWriter, r *http.Request, retCtx, tCancel := context.WithTimeout(ctx, h.Config.ActionTimeout)useLiteAction := h.shouldUseLiteAction(action.Kind)- if !useLiteAction && action.Ref != "" && action.NodeID == 0 && action.Selector == "" {- cache := h.Bridge.GetRefCache(resolvedTabID)- if cache != nil {- if nid, ok := cache.Refs[action.Ref]; ok {- action.NodeID = nid+ // Unified selector resolution for batch actions.+ action.NormalizeSelector()+ refMissing := false+ if !useLiteAction && action.NodeID == 0 && action.Selector != "" {+ sel := selector.Parse(action.Selector)+ switch sel.Kind {+ case selector.KindRef:+ action.Ref = sel.Value+ action.Selector = ""+ cache := h.Bridge.GetRefCache(resolvedTabID)+ if cache != nil {+ if nid, ok := cache.Refs[sel.Value]; ok {+ action.NodeID = nid+ }+ }+ if action.NodeID == 0 {+ refMissing = true}+ case selector.KindCSS:+ action.Ref = ""+ action.Selector = sel.Value+ case selector.KindXPath:+ nid, resolveErr := bridge.ResolveXPathToNodeID(tCtx, sel.Value)+ if resolveErr != nil {+ tCancel()+ results = append(results, actionResult{+ Index: i, Success: false,+ Error: fmt.Sprintf("xpath selector: %v", resolveErr),+ })+ if req.StopOnError {+ break+ }+ continue+ }+ action.NodeID = nid+ action.Selector = ""+ action.Ref = ""+ case selector.KindText:+ nid, resolveErr := bridge.ResolveTextToNodeID(tCtx, sel.Value)+ if resolveErr != nil {+ tCancel()+ results = append(results, actionResult{+ Index: i, Success: false,+ Error: fmt.Sprintf("text selector: %v", resolveErr),+ })+ if req.StopOnError {+ break+ }+ continue+ }+ action.NodeID = nid+ action.Selector = ""+ action.Ref = ""+ case selector.KindSemantic:+ if h.Matcher != nil {+ nodes := h.resolveSnapshotNodes(resolvedTabID)+ if len(nodes) == 0 {+ h.refreshRefCache(tCtx, resolvedTabID)+ nodes = h.resolveSnapshotNodes(resolvedTabID)+ }+ if len(nodes) > 0 {+ descs := make([]semantic.ElementDescriptor, len(nodes))+ for j, n := range nodes {+ descs[j] = semantic.ElementDescriptor{+ Ref: n.Ref, Role: n.Role, Name: n.Name, Value: n.Value,+ }+ }+ findResult, findErr := h.Matcher.Find(tCtx, sel.Value, descs, semantic.FindOptions{+ Threshold: 0.3, TopK: 1,+ })+ if findErr == nil && findResult.BestRef != "" {+ action.Ref = findResult.BestRef+ cache := h.Bridge.GetRefCache(resolvedTabID)+ if cache != nil {+ if nid, ok := cache.Refs[findResult.BestRef]; ok {+ action.NodeID = nid+ }+ }+ }+ }+ if action.NodeID == 0 {+ tCancel()+ results = append(results, actionResult{+ Index: i, Success: false,+ Error: fmt.Sprintf("semantic selector %q: no matching element found", sel.Value),+ })+ if req.StopOnError {+ break+ }+ continue+ }+ } else {+ tCancel()+ results = append(results, actionResult{+ Index: i, Success: false,+ Error: "semantic selectors require a matcher (not configured)",+ })+ if req.StopOnError {+ break+ }+ continue+ }+ action.Selector = ""}+ } else if !useLiteAction && action.Ref != "" && action.NodeID == 0 {+ // Legacy path: Ref set but NormalizeSelector didn't promote it+ // (shouldn't happen, but defensive).+ refMissing = true}- refMissing := !useLiteAction && action.Ref != "" && action.NodeID == 0 && action.Selector == ""-if action.Kind == "" {tCancel()results = append(results, actionResult{@@ -575,17 +755,111 @@ func (h *Handlers) HandleMacro(w http.ResponseWriter, r *http.Request) {step.TabID = resolvedTabID}useLiteAction := h.shouldUseLiteAction(step.Kind)- // Resolve ref → nodeID from snapshot cache (mirrors HandleAction).+ // Unified selector resolution for macro steps (mirrors HandleAction).+ step.NormalizeSelector()stepRefMissing := false- if !useLiteAction && step.Ref != "" && step.NodeID == 0 && step.Selector == "" {- cache := h.Bridge.GetRefCache(resolvedTabID)- if cache != nil {- if nid, ok := cache.Refs[step.Ref]; ok {- step.NodeID = nid+ if !useLiteAction && step.NodeID == 0 && step.Selector != "" {+ sel := selector.Parse(step.Selector)+ switch sel.Kind {+ case selector.KindRef:+ step.Ref = sel.Value+ step.Selector = ""+ cache := h.Bridge.GetRefCache(resolvedTabID)+ if cache != nil {+ if nid, ok := cache.Refs[sel.Value]; ok {+ step.NodeID = nid+ }}- }- if step.NodeID == 0 {- stepRefMissing = true+ if step.NodeID == 0 {+ stepRefMissing = true+ }+ case selector.KindCSS:+ step.Ref = ""+ step.Selector = sel.Value+ case selector.KindXPath:+ tCtx, cancel := context.WithTimeout(ctx, stepTimeout)+ nid, resolveErr := bridge.ResolveXPathToNodeID(tCtx, sel.Value)+ cancel()+ if resolveErr != nil {+ results = append(results, actionResult{+ Index: i, Success: false,+ Error: fmt.Sprintf("xpath selector: %v", resolveErr),+ })+ if req.StopOnError {+ break+ }+ continue+ }+ step.NodeID = nid+ step.Selector = ""+ step.Ref = ""+ case selector.KindText:+ tCtx, cancel := context.WithTimeout(ctx, stepTimeout)+ nid, resolveErr := bridge.ResolveTextToNodeID(tCtx, sel.Value)+ cancel()+ if resolveErr != nil {+ results = append(results, actionResult{+ Index: i, Success: false,+ Error: fmt.Sprintf("text selector: %v", resolveErr),+ })+ if req.StopOnError {+ break+ }+ continue+ }+ step.NodeID = nid+ step.Selector = ""+ step.Ref = ""+ case selector.KindSemantic:+ if h.Matcher != nil {+ tCtx, cancel := context.WithTimeout(ctx, stepTimeout)+ nodes := h.resolveSnapshotNodes(resolvedTabID)+ if len(nodes) == 0 {+ h.refreshRefCache(tCtx, resolvedTabID)+ nodes = h.resolveSnapshotNodes(resolvedTabID)+ }+ if len(nodes) > 0 {+ descs := make([]semantic.ElementDescriptor, len(nodes))+ for j, n := range nodes {+ descs[j] = semantic.ElementDescriptor{+ Ref: n.Ref, Role: n.Role, Name: n.Name, Value: n.Value,+ }+ }+ findResult, findErr := h.Matcher.Find(tCtx, sel.Value, descs, semantic.FindOptions{+ Threshold: 0.3, TopK: 1,+ })+ if findErr == nil && findResult.BestRef != "" {+ step.Ref = findResult.BestRef+ cache := h.Bridge.GetRefCache(resolvedTabID)+ if cache != nil {+ if nid, ok := cache.Refs[findResult.BestRef]; ok {+ step.NodeID = nid+ }+ }+ }+ }+ cancel()+ if step.NodeID == 0 {+ results = append(results, actionResult{+ Index: i, Success: false,+ Error: fmt.Sprintf("semantic selector %q: no matching element found", sel.Value),+ })+ if req.StopOnError {+ break+ }+ continue+ }+ } else {+ results = append(results, actionResult{+ Index: i, Success: false,+ Error: "semantic selectors require a matcher (not configured)",+ })+ if req.StopOnError {+ break+ }+ continue+ }+ step.Selector = ""}}diff --git a/internal/handlers/snapshot.go b/internal/handlers/snapshot.goindex 2e254e8a..fb70a10d 100644--- a/internal/handlers/snapshot.go+++ b/internal/handlers/snapshot.go@@ -135,53 +135,27 @@ func (h *Handlers) HandleSnapshot(w http.ResponseWriter, r *http.Request) {}{Nodes: nodes}if selector != "" {+ // Unified selector: resolve to a backend node ID for subtree scoping.+ // Supports CSS (default), XPath, and text selectors.var scopeNodeID int64- if err := chromedp.Run(tCtx,- chromedp.ActionFunc(func(ctx context.Context) error {- p := map[string]any{"nodeId": 0, "selector": selector}- var docResult json.RawMessage- if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.getDocument", map[string]any{"depth": 0}, &docResult); err != nil {- return fmt.Errorf("get document: %w", err)- }- var doc struct {- Root struct {- NodeID int64 `json:"nodeId"`- } `json:"root"`- }- if err := json.Unmarshal(docResult, &doc); err != nil {- return err- }- p["nodeId"] = doc.Root.NodeID- var qResult json.RawMessage- if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.querySelector", p, &qResult); err != nil {- return fmt.Errorf("querySelector: %w", err)- }- var qr struct {- NodeID int64 `json:"nodeId"`- }- if err := json.Unmarshal(qResult, &qr); err != nil {- return err- }- if qr.NodeID == 0 {- return fmt.Errorf("selector %q not found", selector)- }- var descResult json.RawMessage- if err := chromedp.FromContext(ctx).Target.Execute(ctx, "DOM.describeNode", map[string]any{"nodeId": qr.NodeID}, &descResult); err != nil {- return fmt.Errorf("describe node: %w", err)- }- var desc struct {- Node struct {- BackendNodeID int64 `json:"backendNodeId"`- } `json:"node"`- }- if err := json.Unmarshal(descResult, &desc); err != nil {- return err- }- scopeNodeID = desc.Node.BackendNodeID- return nil- }),- ); err != nil {- web.Error(w, 400, fmt.Errorf("selector: %w", err))+ var scopeErr error++ switch {+ case strings.HasPrefix(selector, "xpath:"):+ scopeNodeID, scopeErr = bridge.ResolveXPathToNodeID(tCtx, selector[len("xpath:"):])+ case strings.HasPrefix(selector, "//") || strings.HasPrefix(selector, "(//"):+ scopeNodeID, scopeErr = bridge.ResolveXPathToNodeID(tCtx, selector)+ case strings.HasPrefix(selector, "text:"):+ scopeNodeID, scopeErr = bridge.ResolveTextToNodeID(tCtx, selector[len("text:"):])+ case strings.HasPrefix(selector, "css:"):+ scopeNodeID, scopeErr = bridge.ResolveCSSToNodeID(tCtx, selector[len("css:"):])+ default:+ // Bare selector — treat as CSS (backward compatible)+ scopeNodeID, scopeErr = bridge.ResolveCSSToNodeID(tCtx, selector)+ }++ if scopeErr != nil {+ web.Error(w, 400, fmt.Errorf("selector: %w", scopeErr))return}diff --git a/internal/handlers/upload.go b/internal/handlers/upload.goindex 0dcdbcfc..a5305f06 100644--- a/internal/handlers/upload.go+++ b/internal/handlers/upload.go@@ -28,7 +28,7 @@ type uploadRequest struct {// POST /upload?tabId=<id>//// {-// "selector": "input[type=file]",+// "selector": "input[type=file]", // unified selector: CSS, XPath, text, ref, or semantic// "files": ["data:image/png;base64,...", "base64:..."],// "paths": ["/tmp/photo.jpg"]// }@@ -161,10 +161,28 @@ func (h *Handlers) HandleTabUpload(w http.ResponseWriter, r *http.Request) {h.HandleUpload(w, req)}-// resolveSelector finds a DOM node by CSS selector and returns its NodeID.-func resolveSelector(ctx context.Context, selector string) (cdp.NodeID, error) {- // Use Runtime.evaluate to get the remote object, then request the node.- expr := fmt.Sprintf(`document.querySelector(%q)`, selector)+// resolveSelector finds a DOM node by a unified selector string and returns its NodeID.+// Supports CSS (default), XPath (xpath: prefix or // auto-detect), and text (text: prefix).+func resolveSelector(ctx context.Context, sel string) (cdp.NodeID, error) {+ // Determine the JavaScript expression based on selector type.+ var expr string+ switch {+ case strings.HasPrefix(sel, "xpath:"):+ xpath := sel[len("xpath:"):]+ expr = fmt.Sprintf(`(function(){var r=document.evaluate(%q,document,null,XPathResult.FIRST_ORDERED_NODE_TYPE,null);return r.singleNodeValue})()`, xpath)+ case strings.HasPrefix(sel, "//") || strings.HasPrefix(sel, "(//"):+ expr = fmt.Sprintf(`(function(){var r=document.evaluate(%q,document,null,XPathResult.FIRST_ORDERED_NODE_TYPE,null);return r.singleNodeValue})()`, sel)+ case strings.HasPrefix(sel, "text:"):+ text := sel[len("text:"):]+ expr = fmt.Sprintf(`(function(){var w=document.createTreeWalker(document.body,NodeFilter.SHOW_TEXT);while(w.nextNode()){if(w.currentNode.textContent.includes(%q))return w.currentNode.parentElement}return null})()`, text)+ case strings.HasPrefix(sel, "css:"):+ css := sel[len("css:"):]+ expr = fmt.Sprintf(`document.querySelector(%q)`, css)+ default:+ // Bare selector — treat as CSS (backward compatible)+ expr = fmt.Sprintf(`document.querySelector(%q)`, sel)+ }+val, _, err := runtime.Evaluate(expr).Do(ctx)if err != nil {return 0, fmt.Errorf("evaluate: %w", err)diff --git a/internal/mcp/handlers.go b/internal/mcp/handlers.goindex 276e4678..25fa0bf8 100644--- a/internal/mcp/handlers.go+++ b/internal/mcp/handlers.go@@ -189,24 +189,35 @@ func handleAction(c *Client, kind string) func(context.Context, mcp.CallToolRequpayload["tabId"] = tabID}+ // Unified selector: prefer "selector" param, fall back to legacy "ref".+ resolveSelector := func(required bool) error {+ sel := optString(r, "selector")+ ref := optString(r, "ref")+ if sel != "" {+ payload["selector"] = sel+ } else if ref != "" {+ // Legacy: promote ref to selector+ payload["selector"] = ref+ } else if required {+ return fmt.Errorf("required parameter 'selector' is missing")+ }+ return nil+ }+switch kind {case "click", "hover", "focus":- ref, err := r.RequireString("ref")- if err != nil {+ if err := resolveSelector(true); err != nil {return mcp.NewToolResultError(err.Error()), nil}- payload["ref"] = refcase "type":- ref, err := r.RequireString("ref")- if err != nil {+ if err := resolveSelector(true); err != nil {return mcp.NewToolResultError(err.Error()), nil}text, err := r.RequireString("text")if err != nil {return mcp.NewToolResultError(err.Error()), nil}- payload["ref"] = refpayload["text"] = textcase "press":@@ -217,35 +228,29 @@ func handleAction(c *Client, kind string) func(context.Context, mcp.CallToolRequpayload["key"] = keycase "select":- ref, err := r.RequireString("ref")- if err != nil {+ if err := resolveSelector(true); err != nil {return mcp.NewToolResultError(err.Error()), nil}value, err := r.RequireString("value")if err != nil {return mcp.NewToolResultError(err.Error()), nil}- payload["ref"] = refpayload["value"] = valuecase "scroll":- if ref := optString(r, "ref"); ref != "" {- payload["ref"] = ref- }+ _ = resolveSelector(false)if px, ok := optFloat(r, "pixels"); ok {payload["scrollY"] = int(px)}case "fill":- ref, err := r.RequireString("ref")- if err != nil {+ if err := resolveSelector(true); err != nil {return mcp.NewToolResultError(err.Error()), nil}value, err := r.RequireString("value")if err != nil {return mcp.NewToolResultError(err.Error()), nil}- payload["ref"] = refpayload["value"] = value}@@ -438,7 +443,7 @@ func handleWait() func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResulfunc handleWaitForSelector(c *Client) func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error) {return func(ctx context.Context, r mcp.CallToolRequest) (*mcp.CallToolResult, error) {- selector, err := r.RequireString("selector")+ sel, err := r.RequireString("selector")if err != nil {return mcp.NewToolResultError(err.Error()), nil}@@ -453,8 +458,24 @@ func handleWaitForSelector(c *Client) func(context.Context, mcp.CallToolRequest)timeoutMS = 100}- // JavaScript that checks for the selector- js := fmt.Sprintf(`document.querySelector(%s) !== null`, jsonString(selector))+ // Build the appropriate JavaScript check based on selector type.+ var js string+ switch {+ case strings.HasPrefix(sel, "xpath:"):+ xpath := sel[len("xpath:"):]+ js = fmt.Sprintf(`(function(){try{var r=document.evaluate(%s,document,null,XPathResult.FIRST_ORDERED_NODE_TYPE,null);return r.singleNodeValue!==null}catch(e){return false}})()`, jsonString(xpath))+ case strings.HasPrefix(sel, "//") || strings.HasPrefix(sel, "(//"):+ js = fmt.Sprintf(`(function(){try{var r=document.evaluate(%s,document,null,XPathResult.FIRST_ORDERED_NODE_TYPE,null);return r.singleNodeValue!==null}catch(e){return false}})()`, jsonString(sel))+ case strings.HasPrefix(sel, "text:"):+ text := sel[len("text:"):]+ js = fmt.Sprintf(`(function(){var w=document.createTreeWalker(document.body,NodeFilter.SHOW_TEXT);while(w.nextNode()){if(w.currentNode.textContent.includes(%s))return true}return false})()`, jsonString(text))+ case strings.HasPrefix(sel, "css:"):+ css := sel[len("css:"):]+ js = fmt.Sprintf(`document.querySelector(%s) !== null`, jsonString(css))+ default:+ // Bare selector — treat as CSS (backward compatible)+ js = fmt.Sprintf(`document.querySelector(%s) !== null`, jsonString(sel))+ }payload := map[string]any{"expression": js}if tabID := optString(r, "tabId"); tabID != "" {@@ -471,12 +492,12 @@ func handleWaitForSelector(c *Client) func(context.Context, mcp.CallToolRequest)Result any `json:"result"`}if json.Unmarshal(body, &resp) == nil && resp.Result == true {- return mcp.NewToolResultText(fmt.Sprintf(`{"found":true,"selector":%s}`, jsonString(selector))), nil+ return mcp.NewToolResultText(fmt.Sprintf(`{"found":true,"selector":%s}`, jsonString(sel))), nil}}if time.Now().After(deadline) {- return mcp.NewToolResultError(fmt.Sprintf("timeout: selector %q not found within %dms", selector, int(timeoutMS))), nil+ return mcp.NewToolResultError(fmt.Sprintf("timeout: selector %q not found within %dms", sel, int(timeoutMS))), nil}select {diff --git a/internal/mcp/tools.go b/internal/mcp/tools.goindex d374fbff..ad0340ef 100644--- a/internal/mcp/tools.go+++ b/internal/mcp/tools.go@@ -17,7 +17,7 @@ func allTools() []mcp.Tool {mcp.WithBoolean("interactive", mcp.Description("Only interactive elements (buttons, links, inputs)")),mcp.WithBoolean("compact", mcp.Description("Compact format (most token-efficient)")),mcp.WithBoolean("diff", mcp.Description("Only changes since last snapshot")),- mcp.WithString("selector", mcp.Description("CSS selector to scope the snapshot")),+ mcp.WithString("selector", mcp.Description("Unified selector to scope the snapshot (CSS, XPath, text, or ref)")),),mcp.NewTool("pinchtab_screenshot",mcp.WithDescription("Take a screenshot of the current page"),@@ -33,13 +33,15 @@ func allTools() []mcp.Tool {// ── Interaction ─────────────────────────────────────────────mcp.NewTool("pinchtab_click",- mcp.WithDescription("Click an element by its accessibility ref"),- mcp.WithString("ref", mcp.Required(), mcp.Description("Element ref from snapshot (e.g., 'e5')")),+ mcp.WithDescription("Click an element by selector (ref from snapshot e.g. 'e5', CSS e.g. 'css:#btn', XPath e.g. 'xpath://button', text e.g. 'text:Submit', or semantic e.g. 'find:login button')"),+ mcp.WithString("selector", mcp.Required(), mcp.Description("Unified selector: ref (e.g. 'e5'), CSS ('css:#btn' or '#btn'), XPath ('xpath://button'), text ('text:Submit'), or semantic ('find:login button')")),+ mcp.WithString("ref", mcp.Description("(deprecated) Element ref from snapshot — use 'selector' instead")),mcp.WithString("tabId", mcp.Description("Target tab ID")),),mcp.NewTool("pinchtab_type",mcp.WithDescription("Type text into an input element"),- mcp.WithString("ref", mcp.Required(), mcp.Description("Element ref from snapshot")),+ mcp.WithString("selector", mcp.Required(), mcp.Description("Unified selector: ref (e.g. 'e5'), CSS, XPath, text, or semantic")),+ mcp.WithString("ref", mcp.Description("(deprecated) Element ref from snapshot — use 'selector' instead")),mcp.WithString("text", mcp.Required(), mcp.Description("Text to type")),mcp.WithString("tabId", mcp.Description("Target tab ID")),),@@ -50,29 +52,34 @@ func allTools() []mcp.Tool {),mcp.NewTool("pinchtab_hover",mcp.WithDescription("Hover over an element"),- mcp.WithString("ref", mcp.Required(), mcp.Description("Element ref from snapshot")),+ mcp.WithString("selector", mcp.Required(), mcp.Description("Unified selector: ref (e.g. 'e5'), CSS, XPath, text, or semantic")),+ mcp.WithString("ref", mcp.Description("(deprecated) Element ref from snapshot — use 'selector' instead")),mcp.WithString("tabId", mcp.Description("Target tab ID")),),mcp.NewTool("pinchtab_focus",mcp.WithDescription("Focus an element"),- mcp.WithString("ref", mcp.Required(), mcp.Description("Element ref from snapshot")),+ mcp.WithString("selector", mcp.Required(), mcp.Description("Unified selector: ref (e.g. 'e5'), CSS, XPath, text, or semantic")),+ mcp.WithString("ref", mcp.Description("(deprecated) Element ref from snapshot — use 'selector' instead")),mcp.WithString("tabId", mcp.Description("Target tab ID")),),mcp.NewTool("pinchtab_select",mcp.WithDescription("Select a dropdown option by value"),- mcp.WithString("ref", mcp.Required(), mcp.Description("Element ref for the select element")),+ mcp.WithString("selector", mcp.Required(), mcp.Description("Unified selector for the select element")),+ mcp.WithString("ref", mcp.Description("(deprecated) Element ref from snapshot — use 'selector' instead")),mcp.WithString("value", mcp.Required(), mcp.Description("Option value to select")),mcp.WithString("tabId", mcp.Description("Target tab ID")),),mcp.NewTool("pinchtab_scroll",mcp.WithDescription("Scroll the page or an element"),- mcp.WithString("ref", mcp.Description("Element ref to scroll (omit for page scroll)")),+ mcp.WithString("selector", mcp.Description("Unified selector for element to scroll (omit for page scroll)")),+ mcp.WithString("ref", mcp.Description("(deprecated) Element ref to scroll — use 'selector' instead")),mcp.WithNumber("pixels", mcp.Description("Number of pixels to scroll (positive=down, negative=up)")),mcp.WithString("tabId", mcp.Description("Target tab ID")),),mcp.NewTool("pinchtab_fill",mcp.WithDescription("Fill an input field directly via JavaScript dispatch"),- mcp.WithString("ref", mcp.Required(), mcp.Description("Element ref or CSS selector")),+ mcp.WithString("selector", mcp.Required(), mcp.Description("Unified selector: ref (e.g. 'e5'), CSS, XPath, text, or semantic")),+ mcp.WithString("ref", mcp.Description("(deprecated) Element ref — use 'selector' instead")),mcp.WithString("value", mcp.Required(), mcp.Description("Value to fill")),mcp.WithString("tabId", mcp.Description("Target tab ID")),),@@ -122,8 +129,8 @@ func allTools() []mcp.Tool {mcp.WithNumber("ms", mcp.Required(), mcp.Description("Milliseconds to wait (max 30000)")),),mcp.NewTool("pinchtab_wait_for_selector",- mcp.WithDescription("Wait for an element matching a CSS selector to appear on the page"),- mcp.WithString("selector", mcp.Required(), mcp.Description("CSS selector to wait for")),+ mcp.WithDescription("Wait for an element matching a selector to appear on the page"),+ mcp.WithString("selector", mcp.Required(), mcp.Description("Unified selector to wait for (CSS, XPath, text, or ref)")),mcp.WithNumber("timeout", mcp.Description("Timeout in milliseconds (default: 10000, max: 30000)")),mcp.WithString("tabId", mcp.Description("Target tab ID")),),diff --git a/internal/scheduler/task.go b/internal/scheduler/task.goindex 579f2086..ba48c65b 100644--- a/internal/scheduler/task.go+++ b/internal/scheduler/task.go@@ -38,7 +38,8 @@ type Task struct {AgentID string `json:"agentId"`Action string `json:"action"`TabID string `json:"tabId,omitempty"`- Ref string `json:"ref,omitempty"`+ Selector string `json:"selector,omitempty"`+ Ref string `json:"ref,omitempty"` // deprecated: use SelectorParams map[string]any `json:"params,omitempty"`Priority int `json:"priority"`State TaskState `json:"state"`@@ -112,6 +113,7 @@ func (t *Task) Snapshot() *Task {AgentID: t.AgentID,Action: t.Action,TabID: t.TabID,+ Selector: t.Selector,Ref: t.Ref,Params: t.Params,Priority: t.Priority,@@ -133,7 +135,8 @@ type SubmitRequest struct {AgentID string `json:"agentId"`Action string `json:"action"`TabID string `json:"tabId,omitempty"`- Ref string `json:"ref,omitempty"`+ Selector string `json:"selector,omitempty"`+ Ref string `json:"ref,omitempty"` // deprecated: use SelectorParams map[string]any `json:"params,omitempty"`Priority int `json:"priority,omitempty"`Deadline string `json:"deadline,omitempty"`diff --git a/internal/selector/selector.go b/internal/selector/selector.gonew file mode 100644index 00000000..5da2cfeb--- /dev/null+++ b/internal/selector/selector.go@@ -0,0 +1,194 @@+// Package selector provides a unified element targeting system.+//+// Instead of separate ref, css, xpath, text, and semantic fields,+// callers use a single selector string. The type is auto-detected+// from the value or an explicit prefix:+//+// "e5" → Ref (element ref from snapshot)+// "css:#login" → CSS (explicit prefix)+// "#login" → CSS (auto-detected)+// "xpath://div" → XPath+// "text:Submit" → Text (match by visible text)+// "find:login btn" → Semantic (natural-language query)+//+// Bare strings that look like CSS selectors (start with ., #, [,+// or contain tag-like patterns) are treated as CSS. Everything else+// without a prefix is treated as a ref if it matches the eN pattern,+// or as CSS otherwise.+package selector++import (+ "fmt"+ "strings"+)++// Kind represents the type of a selector.+type Kind string++const (+ KindNone Kind = ""+ KindRef Kind = "ref"+ KindCSS Kind = "css"+ KindXPath Kind = "xpath"+ KindText Kind = "text"+ KindSemantic Kind = "semantic"+)++// Selector is a parsed, unified element selector.+type Selector struct {+ Kind Kind `json:"kind"`+ Value string `json:"value"`+}++// String returns the canonical string representation with prefix.+func (s Selector) String() string {+ switch s.Kind {+ case KindRef:+ return s.Value+ case KindCSS:+ return "css:" + s.Value+ case KindXPath:+ return "xpath:" + s.Value+ case KindText:+ return "text:" + s.Value+ case KindSemantic:+ return "find:" + s.Value+ default:+ return s.Value+ }+}++// IsEmpty returns true if the selector has no value.+func (s Selector) IsEmpty() bool {+ return s.Value == ""+}++// Parse interprets a selector string and returns a typed Selector.+//+// Explicit prefixes take priority:+//+// "css:..." → CSS+// "xpath:..." → XPath+// "text:..." → Text+// "find:..." → Semantic+// "ref:..." → Ref (optional explicit prefix)+//+// Without a prefix, auto-detection applies:+//+// "e123" → Ref (matches /^e\d+$/)+// "#id" → CSS+// ".class" → CSS+// "[attr]" → CSS+// "tag.class" → CSS+// "//xpath" → XPath+// everything else → CSS (safest default for backward compat)+func Parse(s string) Selector {+ s = strings.TrimSpace(s)+ if s == "" {+ return Selector{}+ }++ // Explicit prefixes+ if after, ok := cutPrefix(s, "css:"); ok {+ return Selector{Kind: KindCSS, Value: after}+ }+ if after, ok := cutPrefix(s, "xpath:"); ok {+ return Selector{Kind: KindXPath, Value: after}+ }+ if after, ok := cutPrefix(s, "text:"); ok {+ return Selector{Kind: KindText, Value: after}+ }+ if after, ok := cutPrefix(s, "find:"); ok {+ return Selector{Kind: KindSemantic, Value: after}+ }+ if after, ok := cutPrefix(s, "ref:"); ok {+ return Selector{Kind: KindRef, Value: after}+ }++ // Auto-detect: XPath+ if strings.HasPrefix(s, "//") || strings.HasPrefix(s, "(//") {+ return Selector{Kind: KindXPath, Value: s}+ }++ // Auto-detect: Ref (e.g. e5, e123)+ if IsRef(s) {+ return Selector{Kind: KindRef, Value: s}+ }++ // Everything else is CSS (backward compatible default)+ return Selector{Kind: KindCSS, Value: s}+}++// IsRef returns true if the string matches the element ref pattern (e.g. "e5", "e123").+func IsRef(s string) bool {+ if len(s) < 2 || s[0] != 'e' {+ return false+ }+ for i := 1; i < len(s); i++ {+ if s[i] < '0' || s[i] > '9' {+ return false+ }+ }+ return true+}++// FromRef creates a Selector from a ref string.+func FromRef(ref string) Selector {+ if ref == "" {+ return Selector{}+ }+ return Selector{Kind: KindRef, Value: ref}+}++// FromCSS creates a Selector from a CSS selector string.+func FromCSS(css string) Selector {+ if css == "" {+ return Selector{}+ }+ return Selector{Kind: KindCSS, Value: css}+}++// FromXPath creates a Selector from an XPath expression.+func FromXPath(xpath string) Selector {+ if xpath == "" {+ return Selector{}+ }+ return Selector{Kind: KindXPath, Value: xpath}+}++// FromText creates a Selector from a text content query.+func FromText(text string) Selector {+ if text == "" {+ return Selector{}+ }+ return Selector{Kind: KindText, Value: text}+}++// FromSemantic creates a Selector from a semantic/natural-language query.+func FromSemantic(query string) Selector {+ if query == "" {+ return Selector{}+ }+ return Selector{Kind: KindSemantic, Value: query}+}++// Validate returns an error if the selector is invalid.+func (s Selector) Validate() error {+ if s.IsEmpty() {+ return fmt.Errorf("empty selector")+ }+ switch s.Kind {+ case KindRef, KindCSS, KindXPath, KindText, KindSemantic:+ return nil+ default:+ return fmt.Errorf("unknown selector kind: %q", s.Kind)+ }+}++// cutPrefix is a helper for strings.CutPrefix (available in Go 1.20+).+func cutPrefix(s, prefix string) (string, bool) {+ if strings.HasPrefix(s, prefix) {+ return s[len(prefix):], true+ }+ return s, false+}diff --git a/internal/selector/selector_test.go b/internal/selector/selector_test.gonew file mode 100644index 00000000..8a02f5c3--- /dev/null+++ b/internal/selector/selector_test.go@@ -0,0 +1,477 @@+package selector++import (+ "testing"+)++// ---------------------------------------------------------------------------+// Parse – explicit prefixes+// ---------------------------------------------------------------------------++func TestParse_ExplicitPrefixes(t *testing.T) {+ tests := []struct {+ input string+ kind Kind+ value string+ }{+ // CSS+ {"css:#login", KindCSS, "#login"},+ {"css:.btn.primary", KindCSS, ".btn.primary"},+ {"css:div > span", KindCSS, "div > span"},+ {"css:input[type=text]", KindCSS, "input[type=text]"},+ {"css:*", KindCSS, "*"},++ // XPath+ {"xpath://div[@id='main']", KindXPath, "//div[@id='main']"},+ {"xpath:(//button)[1]", KindXPath, "(//button)[1]"},+ {"xpath://a[contains(@href,'login')]", KindXPath, "//a[contains(@href,'login')]"},++ // Text+ {"text:Submit", KindText, "Submit"},+ {"text:Log in", KindText, "Log in"},+ {"text:", KindText, ""},+ {"text:with:colon", KindText, "with:colon"},++ // Semantic / find+ {"find:login button", KindSemantic, "login button"},+ {"find:the search input field", KindSemantic, "the search input field"},+ {"find:", KindSemantic, ""},++ // Ref (explicit prefix)+ {"ref:e5", KindRef, "e5"},+ {"ref:e0", KindRef, "e0"},+ {"ref:e99999", KindRef, "e99999"},+ // ref: prefix with non-standard value (still accepted as ref)+ {"ref:something", KindRef, "something"},+ }+ for _, tt := range tests {+ s := Parse(tt.input)+ if s.Kind != tt.kind {+ t.Errorf("Parse(%q).Kind = %q, want %q", tt.input, s.Kind, tt.kind)+ }+ if s.Value != tt.value {+ t.Errorf("Parse(%q).Value = %q, want %q", tt.input, s.Value, tt.value)+ }+ }+}++// ---------------------------------------------------------------------------+// Parse – auto-detection (no prefix)+// ---------------------------------------------------------------------------++func TestParse_AutoDetect(t *testing.T) {+ tests := []struct {+ input string+ kind Kind+ value string+ }{+ // Refs+ {"e0", KindRef, "e0"},+ {"e5", KindRef, "e5"},+ {"e42", KindRef, "e42"},+ {"e123", KindRef, "e123"},+ {"e99999", KindRef, "e99999"},++ // CSS auto-detect: # prefix+ {"#login", KindCSS, "#login"},+ {"#my-id", KindCSS, "#my-id"},++ // CSS auto-detect: . prefix+ {".btn", KindCSS, ".btn"},+ {".btn.primary", KindCSS, ".btn.primary"},++ // CSS auto-detect: [ prefix+ {"[type=file]", KindCSS, "[type=file]"},+ {"[data-testid='foo']", KindCSS, "[data-testid='foo']"},++ // CSS auto-detect: compound selectors+ {"button.submit", KindCSS, "button.submit"},+ {"div > span", KindCSS, "div > span"},+ {"input[name='email']", KindCSS, "input[name='email']"},+ {"ul li:first-child", KindCSS, "ul li:first-child"},+ {"a:hover", KindCSS, "a:hover"},++ // XPath auto-detect: //+ {"//div[@class='main']", KindXPath, "//div[@class='main']"},+ {"//a", KindXPath, "//a"},++ // XPath auto-detect: (//+ {"(//button)[1]", KindXPath, "(//button)[1]"},+ {"(//div[@class='x'])[last()]", KindXPath, "(//div[@class='x'])[last()]"},++ // Bare tag names → CSS (backward compat)+ {"button", KindCSS, "button"},+ {"div", KindCSS, "div"},+ {"input", KindCSS, "input"},++ // Words that start with 'e' but are NOT refs+ {"embed", KindCSS, "embed"},+ {"email", KindCSS, "email"},+ {"element", KindCSS, "element"},+ }+ for _, tt := range tests {+ s := Parse(tt.input)+ if s.Kind != tt.kind {+ t.Errorf("Parse(%q).Kind = %q, want %q", tt.input, s.Kind, tt.kind)+ }+ if s.Value != tt.value {+ t.Errorf("Parse(%q).Value = %q, want %q", tt.input, s.Value, tt.value)+ }+ }+}++// ---------------------------------------------------------------------------+// Parse – empty / whitespace+// ---------------------------------------------------------------------------++func TestParse_Empty(t *testing.T) {+ s := Parse("")+ if !s.IsEmpty() {+ t.Error("Parse(\"\") should be empty")+ }+ if s.Kind != KindNone {+ t.Errorf("Parse(\"\").Kind = %q, want %q", s.Kind, KindNone)+ }+}++func TestParse_WhitespaceOnly(t *testing.T) {+ for _, ws := range []string{" ", " ", "\t", "\n", " \t\n "} {+ s := Parse(ws)+ if !s.IsEmpty() {+ t.Errorf("Parse(%q) should be empty", ws)+ }+ }+}++func TestParse_WhitespaceTrimming(t *testing.T) {+ // Leading/trailing whitespace should be trimmed before parsing+ tests := []struct {+ input string+ kind Kind+ value string+ }{+ {" e5 ", KindRef, "e5"},+ {" #login ", KindCSS, "#login"},+ {"\tcss:.btn\t", KindCSS, ".btn"},+ {" xpath://div ", KindXPath, "//div"},+ {" text:Submit ", KindText, "Submit"},+ {" find:login btn ", KindSemantic, "login btn"},+ {" ref:e42 ", KindRef, "e42"},+ }+ for _, tt := range tests {+ s := Parse(tt.input)+ if s.Kind != tt.kind {+ t.Errorf("Parse(%q).Kind = %q, want %q", tt.input, s.Kind, tt.kind)+ }+ if s.Value != tt.value {+ t.Errorf("Parse(%q).Value = %q, want %q", tt.input, s.Value, tt.value)+ }+ }+}++// ---------------------------------------------------------------------------+// Parse – edge cases+// ---------------------------------------------------------------------------++func TestParse_EdgeCases(t *testing.T) {+ tests := []struct {+ name string+ input string+ kind Kind+ value string+ }{+ {+ name: "prefix with colon in value",+ input: "text:Click here: now",+ kind: KindText,+ value: "Click here: now",+ },+ {+ name: "css prefix with complex selector",+ input: "css:div.container > ul > li:nth-child(2n+1)",+ kind: KindCSS,+ value: "div.container > ul > li:nth-child(2n+1)",+ },+ {+ name: "xpath with predicates",+ input: "xpath://div[contains(@class,'active') and @data-visible='true']",+ kind: KindXPath,+ value: "//div[contains(@class,'active') and @data-visible='true']",+ },+ {+ name: "single character e is not a ref",+ input: "e",+ kind: KindCSS,+ value: "e",+ },+ {+ name: "e followed by non-digit",+ input: "eX",+ kind: KindCSS,+ value: "eX",+ },+ {+ name: "E uppercase is not a ref",+ input: "E5",+ kind: KindCSS,+ value: "E5",+ },+ {+ name: "e with mixed chars",+ input: "e5x",+ kind: KindCSS,+ value: "e5x",+ },+ {+ name: "unknown prefix treated as CSS",+ input: "bogus:something",+ kind: KindCSS,+ value: "bogus:something",+ },+ {+ name: "just a colon",+ input: ":",+ kind: KindCSS,+ value: ":",+ },+ {+ name: "css prefix empty value",+ input: "css:",+ kind: KindCSS,+ value: "",+ },+ {+ name: "ref prefix empty value",+ input: "ref:",+ kind: KindRef,+ value: "",+ },+ {+ name: "very long ref",+ input: "e1234567890",+ kind: KindRef,+ value: "e1234567890",+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Parse(tt.input)+ if s.Kind != tt.kind {+ t.Errorf("Parse(%q).Kind = %q, want %q", tt.input, s.Kind, tt.kind)+ }+ if s.Value != tt.value {+ t.Errorf("Parse(%q).Value = %q, want %q", tt.input, s.Value, tt.value)+ }+ })+ }+}++// ---------------------------------------------------------------------------+// IsRef+// ---------------------------------------------------------------------------++func TestIsRef(t *testing.T) {+ refs := []string{"e0", "e5", "e42", "e123", "e9999", "e1234567890"}+ for _, r := range refs {+ if !IsRef(r) {+ t.Errorf("IsRef(%q) = false, want true", r)+ }+ }++ nonRefs := []string{+ "", "e", "E5", "ex5", "e5x", "embed", "email", "element",+ "#e5", "ref:e5", "e-5", "e 5", "e.5", "5e", "ee5",+ "E0", "e", " e5", "e5 ",+ }+ for _, r := range nonRefs {+ if IsRef(r) {+ t.Errorf("IsRef(%q) = true, want false", r)+ }+ }+}++// ---------------------------------------------------------------------------+// Selector.String() – canonical representation+// ---------------------------------------------------------------------------++func TestSelector_String(t *testing.T) {+ tests := []struct {+ sel Selector+ want string+ }{+ {Selector{KindRef, "e5"}, "e5"},+ {Selector{KindRef, "e0"}, "e0"},+ {Selector{KindCSS, "#login"}, "css:#login"},+ {Selector{KindCSS, ".btn"}, "css:.btn"},+ {Selector{KindCSS, "div > span"}, "css:div > span"},+ {Selector{KindXPath, "//div"}, "xpath://div"},+ {Selector{KindXPath, "(//button)[1]"}, "xpath:(//button)[1]"},+ {Selector{KindText, "Submit"}, "text:Submit"},+ {Selector{KindText, "with:colon"}, "text:with:colon"},+ {Selector{KindSemantic, "login button"}, "find:login button"},+ {Selector{KindNone, ""}, ""},+ {Selector{KindNone, "something"}, "something"},+ }+ for _, tt := range tests {+ if got := tt.sel.String(); got != tt.want {+ t.Errorf("Selector{%s, %q}.String() = %q, want %q", tt.sel.Kind, tt.sel.Value, got, tt.want)+ }+ }+}++// ---------------------------------------------------------------------------+// Selector.IsEmpty()+// ---------------------------------------------------------------------------++func TestSelector_IsEmpty(t *testing.T) {+ if !(Selector{}).IsEmpty() {+ t.Error("zero-value Selector should be empty")+ }+ if !(Selector{Kind: KindCSS, Value: ""}).IsEmpty() {+ t.Error("Selector with empty Value should be empty")+ }+ if (Selector{Kind: KindRef, Value: "e5"}).IsEmpty() {+ t.Error("Selector with value should not be empty")+ }+}++// ---------------------------------------------------------------------------+// Selector.Validate()+// ---------------------------------------------------------------------------++func TestSelector_Validate(t *testing.T) {+ valid := []Selector{+ {KindRef, "e5"},+ {KindCSS, "#login"},+ {KindXPath, "//div"},+ {KindText, "Submit"},+ {KindSemantic, "login button"},+ }+ for _, s := range valid {+ if err := s.Validate(); err != nil {+ t.Errorf("Validate(%v) = %v, want nil", s, err)+ }+ }++ // Empty selector+ if err := (Selector{}).Validate(); err == nil {+ t.Error("Validate(empty) should fail")+ }+ // Empty value with kind set+ if err := (Selector{Kind: KindCSS}).Validate(); err == nil {+ t.Error("Validate(kind=css, value='') should fail")+ }+ // Unknown kind+ if err := (Selector{Kind: "bogus", Value: "x"}).Validate(); err == nil {+ t.Error("Validate(bogus kind) should fail")+ }+ if err := (Selector{Kind: Kind("unknown"), Value: "x"}).Validate(); err == nil {+ t.Error("Validate(unknown kind) should fail")+ }+}++// ---------------------------------------------------------------------------+// From* constructors+// ---------------------------------------------------------------------------++func TestFromConstructors(t *testing.T) {+ // Non-empty values+ if s := FromRef("e5"); s.Kind != KindRef || s.Value != "e5" {+ t.Errorf("FromRef(\"e5\"): %+v", s)+ }+ if s := FromCSS("#x"); s.Kind != KindCSS || s.Value != "#x" {+ t.Errorf("FromCSS(\"#x\"): %+v", s)+ }+ if s := FromXPath("//a"); s.Kind != KindXPath || s.Value != "//a" {+ t.Errorf("FromXPath(\"//a\"): %+v", s)+ }+ if s := FromText("hi"); s.Kind != KindText || s.Value != "hi" {+ t.Errorf("FromText(\"hi\"): %+v", s)+ }+ if s := FromSemantic("btn"); s.Kind != KindSemantic || s.Value != "btn" {+ t.Errorf("FromSemantic(\"btn\"): %+v", s)+ }++ // Empty values → empty selector+ empties := []struct {+ name string+ fn func(string) Selector+ }{+ {"FromRef", FromRef},+ {"FromCSS", FromCSS},+ {"FromXPath", FromXPath},+ {"FromText", FromText},+ {"FromSemantic", FromSemantic},+ }+ for _, e := range empties {+ if s := e.fn(""); !s.IsEmpty() {+ t.Errorf("%s(\"\") should be empty, got %+v", e.name, s)+ }+ }+}++// ---------------------------------------------------------------------------+// Parse roundtrip: Parse → String → Parse should be stable+// ---------------------------------------------------------------------------++func TestParse_Roundtrip(t *testing.T) {+ inputs := []string{+ "e5",+ "e0",+ "e99999",+ "css:#login",+ "css:.btn.primary",+ "css:div > span",+ "xpath://div[@id='x']",+ "xpath:(//button)[1]",+ "text:Submit Order",+ "text:with:colon:in:value",+ "find:the big red button",+ }+ for _, input := range inputs {+ s := Parse(input)+ rt := Parse(s.String())+ if rt.Kind != s.Kind || rt.Value != s.Value {+ t.Errorf("roundtrip failed: %q → %+v → %q → %+v", input, s, s.String(), rt)+ }+ }+}++// ---------------------------------------------------------------------------+// Parse – prefix priority (explicit prefix wins over auto-detect)+// ---------------------------------------------------------------------------++func TestParse_PrefixPriority(t *testing.T) {+ // "css://div" should be CSS, not XPath (explicit prefix wins)+ s := Parse("css://div")+ if s.Kind != KindCSS {+ t.Errorf("Parse(\"css://div\").Kind = %q, want css", s.Kind)+ }+ if s.Value != "//div" {+ t.Errorf("Parse(\"css://div\").Value = %q, want \"//div\"", s.Value)+ }++ // "ref:embed" should be ref, not CSS+ s = Parse("ref:embed")+ if s.Kind != KindRef {+ t.Errorf("Parse(\"ref:embed\").Kind = %q, want ref", s.Kind)+ }+ if s.Value != "embed" {+ t.Errorf("Parse(\"ref:embed\").Value = %q, want \"embed\"", s.Value)+ }++ // "text:#login" should be text, not CSS+ s = Parse("text:#login")+ if s.Kind != KindText {+ t.Errorf("Parse(\"text:#login\").Kind = %q, want text", s.Kind)+ }+ if s.Value != "#login" {+ t.Errorf("Parse(\"text:#login\").Value = %q, want \"#login\"", s.Value)+ }++ // "xpath:e5" should be xpath, not ref+ s = Parse("xpath:e5")+ if s.Kind != KindXPath {+ t.Errorf("Parse(\"xpath:e5\").Kind = %q, want xpath", s.Kind)+ }+}