High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
chore: update semantic find query support
Luigi Agosti565e1df186da
diff --git a/docs/reference/find.md b/docs/reference/find.mdindex e24a4cee..509468a8 100644--- a/docs/reference/find.md+++ b/docs/reference/find.md@@ -71,6 +71,46 @@ If `tabId` is omitted, PinchTab uses the active tab in the current bridge contexWhen `explain` is enabled, each match may also include lexical and embedding score details.+## Query Syntax++Beyond plain natural-language descriptions, the matcher understands two query modifiers:++### Negative Queries++Use `not`, `without`, `exclude`, `excluding`, `except`, `no`, or `ignore` to push elements away from the match:++```bash+# Picks Cancel over Submit+pinchtab find "button not submit"++# Compose multiple exclusions+pinchtab find "input no password no username"++# Exclude a phrase+pinchtab find "button without sign in"+```++Tokens before the trigger are positive; everything after is negative until the next trigger or end of query.++### Visual / Location Queries++Directional and relative phrases bias the match toward elements at the matching position on the page:++```bash+# Directional (top / bottom / left / right / corner)+pinchtab find "bottom button"+pinchtab find "button in top right corner"+pinchtab find "sidebar on the left"++# Anchor-relative (above / below / under / over)+pinchtab find "link below the search box"+pinchtab find "button above the footer"+```++When the accessibility snapshot has no coordinates, document order is used as a fallback for vertical position — so `"bottom button"` selects the last matching button in the snapshot. When element bounding boxes are available they take precedence.++Visual hints are applied by the combined matcher (the default). They do not affect the lexical-only matcher.+## Confidence Levels| Level | Score Range | Meaning |diff --git a/go.mod b/go.modindex a261c60e..fa5e669c 100644--- a/go.mod+++ b/go.mod@@ -9,11 +9,10 @@ require (github.com/gost-dom/browser v0.11.0github.com/mark3labs/mcp-go v0.45.0github.com/pinchtab/idpishield v0.1.4- github.com/pinchtab/semantic v0.1.1+ github.com/pinchtab/semantic v0.1.2github.com/shirou/gopsutil/v4 v4.26.2github.com/spf13/cobra v1.10.2golang.org/x/net v0.47.0- golang.org/x/text v0.35.0gopkg.in/yaml.v3 v3.0.1)@@ -41,4 +40,5 @@ require (github.com/yosida95/uritemplate/v3 v3.0.2 // indirectgithub.com/yusufpapurcu/wmi v1.2.4 // indirectgolang.org/x/sys v0.41.0 // indirect+ golang.org/x/text v0.35.0 // indirect)diff --git a/go.sum b/go.sumindex 7e1ecc94..9b0fddc4 100644--- a/go.sum+++ b/go.sum@@ -59,10 +59,8 @@ github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAgithub.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=github.com/pinchtab/idpishield v0.1.4 h1:zuIofkQ6c9tzdrYsaNWA10w78OuMcgiBlsCOz4Zk34E=github.com/pinchtab/idpishield v0.1.4/go.mod h1:i6MNCo1Y03KvpqblGAsGviox1WKpsn11gwOSeIlzYm0=-github.com/pinchtab/semantic v0.1.0 h1:jVBQmnddU1cCEKtpqk/t9QHrQ9y/VXMtZ0BI3VkLQxk=-github.com/pinchtab/semantic v0.1.0/go.mod h1:SPKIra6m26faxTxn4dxa1ex1TbGkZDKbJ2r5rCngWoQ=-github.com/pinchtab/semantic v0.1.1 h1:IRzRYIdQnmJyRPkjjlA3a+4UY/nnT020oxdRLdRdL80=-github.com/pinchtab/semantic v0.1.1/go.mod h1:SPKIra6m26faxTxn4dxa1ex1TbGkZDKbJ2r5rCngWoQ=+github.com/pinchtab/semantic v0.1.2 h1:aSSwNwFb+laEBW1/x44dhnGmqUn5xdWJsweLgbqu8WM=+github.com/pinchtab/semantic v0.1.2/go.mod h1:SPKIra6m26faxTxn4dxa1ex1TbGkZDKbJ2r5rCngWoQ=github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=diff --git a/internal/cli/actions/actions_find_test.go b/internal/cli/actions/actions_find_test.gonew file mode 100644index 00000000..7c8dcb7c--- /dev/null+++ b/internal/cli/actions/actions_find_test.go@@ -0,0 +1,57 @@+package actions++import (+ "encoding/json"+ "testing"++ "github.com/spf13/cobra"+)++func newFindCmd() *cobra.Command {+ cmd := &cobra.Command{}+ cmd.Flags().String("tab", "", "")+ cmd.Flags().String("threshold", "", "")+ cmd.Flags().Bool("explain", false, "")+ cmd.Flags().Bool("ref-only", false, "")+ cmd.Flags().Bool("json", false, "")+ return cmd+}++func TestFind_NegativeQueryForwarded(t *testing.T) {+ m := newMockServer()+ m.response = `{"best_ref":"e1","matches":[{"ref":"e1","role":"button","name":"Cancel"}]}`+ defer m.close()++ cmd := newFindCmd()+ _ = cmd.Flags().Set("ref-only", "true")+ Find(m.server.Client(), m.base(), "", "button not submit", cmd)++ if m.lastPath != "/find" {+ t.Errorf("expected /find, got %s", m.lastPath)+ }+ var body map[string]any+ _ = json.Unmarshal([]byte(m.lastBody), &body)+ if body["query"] != "button not submit" {+ t.Errorf("expected query passthrough, got %v", body["query"])+ }+}++func TestFind_VisualQueryWithTabRoute(t *testing.T) {+ m := newMockServer()+ m.response = `{"best_ref":"e2","matches":[{"ref":"e2","role":"button","name":"Action"}]}`+ defer m.close()++ cmd := newFindCmd()+ _ = cmd.Flags().Set("tab", "tab1")+ _ = cmd.Flags().Set("ref-only", "true")+ Find(m.server.Client(), m.base(), "", "bottom button", cmd)++ if m.lastPath != "/tabs/tab1/find" {+ t.Errorf("expected /tabs/tab1/find, got %s", m.lastPath)+ }+ var body map[string]any+ _ = json.Unmarshal([]byte(m.lastBody), &body)+ if body["query"] != "bottom button" {+ t.Errorf("expected visual query passthrough, got %v", body["query"])+ }+}diff --git a/internal/handlers/find_test.go b/internal/handlers/find_test.goindex 326d2447..82ab981f 100644--- a/internal/handlers/find_test.go+++ b/internal/handlers/find_test.go@@ -284,6 +284,69 @@ func TestHandleFind_ResponseMetrics(t *testing.T) {}}+func TestHandleFind_NegativeQuery(t *testing.T) {+ cache := &bridge.RefCache{+ Nodes: []bridge.A11yNode{+ {Ref: "e0", Role: "button", Name: "Submit"},+ {Ref: "e1", Role: "button", Name: "Cancel"},+ },+ Refs: map[string]int64{"e0": 1, "e1": 2},+ }++ h := newFindTestHandler(cache, false)++ body := `{"query": "button not submit", "threshold": 0.0, "topK": 3}`+ req := httptest.NewRequest("POST", "/find", bytes.NewReader([]byte(body)))+ w := httptest.NewRecorder()+ h.HandleFind(w, req)++ if w.Code != http.StatusOK {+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())+ }++ var resp findResponse+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {+ t.Fatalf("decode response: %v", err)+ }++ if resp.BestRef != "e1" {+ t.Errorf("expected best_ref=e1 (Cancel) for negative query, got %s", resp.BestRef)+ }+}++func TestHandleFind_VisualQuery_BottomButton(t *testing.T) {+ cache := &bridge.RefCache{+ Nodes: []bridge.A11yNode{+ {Ref: "e0", Role: "button", Name: "Action"},+ {Ref: "e1", Role: "button", Name: "Action"},+ {Ref: "e2", Role: "button", Name: "Action"},+ },+ Refs: map[string]int64{"e0": 1, "e1": 2, "e2": 3},+ }++ mb := &findMockBridge{refCache: cache}+ h := New(mb, &config.RuntimeConfig{ActionTimeout: 10 * time.Second}, nil, nil, nil)+ h.Matcher = semantic.NewCombinedMatcher(semantic.NewHashingEmbedder(128))++ body := `{"query": "bottom button", "threshold": 0.0, "topK": 3}`+ req := httptest.NewRequest("POST", "/find", bytes.NewReader([]byte(body)))+ w := httptest.NewRecorder()+ h.HandleFind(w, req)++ if w.Code != http.StatusOK {+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())+ }++ var resp findResponse+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {+ t.Fatalf("decode: %v", err)+ }++ if resp.BestRef != "e2" {+ t.Errorf("expected best_ref=e2 (last in document order) for 'bottom button', got %s", resp.BestRef)+ }+}+func TestHandleFind_EmbeddingMatcher(t *testing.T) {cache := &bridge.RefCache{Nodes: []bridge.A11yNode{diff --git a/tests/e2e/scenarios/cli/browser-extended.sh b/tests/e2e/scenarios/cli/browser-extended.shindex 2dab5028..155a10e1 100644--- a/tests/e2e/scenarios/cli/browser-extended.sh+++ b/tests/e2e/scenarios/cli/browser-extended.sh@@ -163,6 +163,35 @@ pt find "xyznonexistent99999"end_test+# ─────────────────────────────────────────────────────────────────+start_test "pinchtab find (negative query)"++pt_ok nav "${FIXTURES_URL}/find.html"+# Resolve "log in" first; the negative form "log in not sign in" must keep+# matching it, while "log in not log in" must not return the same ref.+pt_ok find "log in" --ref-only+LOGIN_REF="$PT_OUT"+pt_ok find "log in button not sign up" --ref-only+assert_output_contains "$LOGIN_REF" "negative query keeps Log In when excluding Sign Up"++pt_ok find "button not log in" --ref-only+assert_output_not_contains "$LOGIN_REF" "negative query 'button not log in' excludes Log In ref"++end_test++# ─────────────────────────────────────────────────────────────────+start_test "pinchtab find (visual query: top vs bottom button)"++pt_ok nav "${FIXTURES_URL}/find.html"+# "top button" and "bottom button" share the same base query but bias toward+# opposite ends of document order. The two refs must differ.+pt_ok find "top button" --ref-only+TOP_REF="$PT_OUT"+pt_ok find "bottom button" --ref-only+assert_output_not_contains "$TOP_REF" "visual hint produces different refs for top vs bottom"++end_test+# ─────────────────────────────────────────────────────────────────start_test "pinchtab snap --text"