Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

chore: update semantic find query support

Luigi Agosti565e1df186da
diff --git a/docs/reference/find.md b/docs/reference/find.md
index e24a4cee..509468a8 100644
--- a/docs/reference/find.md
+++ b/docs/reference/find.md
@@ -71,6 +71,46 @@ If `tabId` is omitted, PinchTab uses the active tab in the current bridge contex
When `explain` is enabled, each match may also include lexical and embedding score details.
+## Query Syntax
+
+Beyond plain natural-language descriptions, the matcher understands two query modifiers:
+
+### Negative Queries
+
+Use `not`, `without`, `exclude`, `excluding`, `except`, `no`, or `ignore` to push elements away from the match:
+
+```bash
+# Picks Cancel over Submit
+pinchtab find "button not submit"
+
+# Compose multiple exclusions
+pinchtab find "input no password no username"
+
+# Exclude a phrase
+pinchtab find "button without sign in"
+```
+
+Tokens before the trigger are positive; everything after is negative until the next trigger or end of query.
+
+### Visual / Location Queries
+
+Directional and relative phrases bias the match toward elements at the matching position on the page:
+
+```bash
+# Directional (top / bottom / left / right / corner)
+pinchtab find "bottom button"
+pinchtab find "button in top right corner"
+pinchtab find "sidebar on the left"
+
+# Anchor-relative (above / below / under / over)
+pinchtab find "link below the search box"
+pinchtab find "button above the footer"
+```
+
+When the accessibility snapshot has no coordinates, document order is used as a fallback for vertical position — so `"bottom button"` selects the last matching button in the snapshot. When element bounding boxes are available they take precedence.
+
+Visual hints are applied by the combined matcher (the default). They do not affect the lexical-only matcher.
+
## Confidence Levels
| Level | Score Range | Meaning |
diff --git a/go.mod b/go.mod
index a261c60e..fa5e669c 100644
--- a/go.mod
+++ b/go.mod
@@ -9,11 +9,10 @@ require (
github.com/gost-dom/browser v0.11.0
github.com/mark3labs/mcp-go v0.45.0
github.com/pinchtab/idpishield v0.1.4
- github.com/pinchtab/semantic v0.1.1
+ github.com/pinchtab/semantic v0.1.2
github.com/shirou/gopsutil/v4 v4.26.2
github.com/spf13/cobra v1.10.2
golang.org/x/net v0.47.0
- golang.org/x/text v0.35.0
gopkg.in/yaml.v3 v3.0.1
)
@@ -41,4 +40,5 @@ require (
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
golang.org/x/sys v0.41.0 // indirect
+ golang.org/x/text v0.35.0 // indirect
)
diff --git a/go.sum b/go.sum
index 7e1ecc94..9b0fddc4 100644
--- a/go.sum
+++ b/go.sum
@@ -59,10 +59,8 @@ github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhA
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
github.com/pinchtab/idpishield v0.1.4 h1:zuIofkQ6c9tzdrYsaNWA10w78OuMcgiBlsCOz4Zk34E=
github.com/pinchtab/idpishield v0.1.4/go.mod h1:i6MNCo1Y03KvpqblGAsGviox1WKpsn11gwOSeIlzYm0=
-github.com/pinchtab/semantic v0.1.0 h1:jVBQmnddU1cCEKtpqk/t9QHrQ9y/VXMtZ0BI3VkLQxk=
-github.com/pinchtab/semantic v0.1.0/go.mod h1:SPKIra6m26faxTxn4dxa1ex1TbGkZDKbJ2r5rCngWoQ=
-github.com/pinchtab/semantic v0.1.1 h1:IRzRYIdQnmJyRPkjjlA3a+4UY/nnT020oxdRLdRdL80=
-github.com/pinchtab/semantic v0.1.1/go.mod h1:SPKIra6m26faxTxn4dxa1ex1TbGkZDKbJ2r5rCngWoQ=
+github.com/pinchtab/semantic v0.1.2 h1:aSSwNwFb+laEBW1/x44dhnGmqUn5xdWJsweLgbqu8WM=
+github.com/pinchtab/semantic v0.1.2/go.mod h1:SPKIra6m26faxTxn4dxa1ex1TbGkZDKbJ2r5rCngWoQ=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
diff --git a/internal/cli/actions/actions_find_test.go b/internal/cli/actions/actions_find_test.go
new file mode 100644
index 00000000..7c8dcb7c
--- /dev/null
+++ b/internal/cli/actions/actions_find_test.go
@@ -0,0 +1,57 @@
+package actions
+
+import (
+ "encoding/json"
+ "testing"
+
+ "github.com/spf13/cobra"
+)
+
+func newFindCmd() *cobra.Command {
+ cmd := &cobra.Command{}
+ cmd.Flags().String("tab", "", "")
+ cmd.Flags().String("threshold", "", "")
+ cmd.Flags().Bool("explain", false, "")
+ cmd.Flags().Bool("ref-only", false, "")
+ cmd.Flags().Bool("json", false, "")
+ return cmd
+}
+
+func TestFind_NegativeQueryForwarded(t *testing.T) {
+ m := newMockServer()
+ m.response = `{"best_ref":"e1","matches":[{"ref":"e1","role":"button","name":"Cancel"}]}`
+ defer m.close()
+
+ cmd := newFindCmd()
+ _ = cmd.Flags().Set("ref-only", "true")
+ Find(m.server.Client(), m.base(), "", "button not submit", cmd)
+
+ if m.lastPath != "/find" {
+ t.Errorf("expected /find, got %s", m.lastPath)
+ }
+ var body map[string]any
+ _ = json.Unmarshal([]byte(m.lastBody), &body)
+ if body["query"] != "button not submit" {
+ t.Errorf("expected query passthrough, got %v", body["query"])
+ }
+}
+
+func TestFind_VisualQueryWithTabRoute(t *testing.T) {
+ m := newMockServer()
+ m.response = `{"best_ref":"e2","matches":[{"ref":"e2","role":"button","name":"Action"}]}`
+ defer m.close()
+
+ cmd := newFindCmd()
+ _ = cmd.Flags().Set("tab", "tab1")
+ _ = cmd.Flags().Set("ref-only", "true")
+ Find(m.server.Client(), m.base(), "", "bottom button", cmd)
+
+ if m.lastPath != "/tabs/tab1/find" {
+ t.Errorf("expected /tabs/tab1/find, got %s", m.lastPath)
+ }
+ var body map[string]any
+ _ = json.Unmarshal([]byte(m.lastBody), &body)
+ if body["query"] != "bottom button" {
+ t.Errorf("expected visual query passthrough, got %v", body["query"])
+ }
+}
diff --git a/internal/handlers/find_test.go b/internal/handlers/find_test.go
index 326d2447..82ab981f 100644
--- a/internal/handlers/find_test.go
+++ b/internal/handlers/find_test.go
@@ -284,6 +284,69 @@ func TestHandleFind_ResponseMetrics(t *testing.T) {
}
}
+func TestHandleFind_NegativeQuery(t *testing.T) {
+ cache := &bridge.RefCache{
+ Nodes: []bridge.A11yNode{
+ {Ref: "e0", Role: "button", Name: "Submit"},
+ {Ref: "e1", Role: "button", Name: "Cancel"},
+ },
+ Refs: map[string]int64{"e0": 1, "e1": 2},
+ }
+
+ h := newFindTestHandler(cache, false)
+
+ body := `{"query": "button not submit", "threshold": 0.0, "topK": 3}`
+ req := httptest.NewRequest("POST", "/find", bytes.NewReader([]byte(body)))
+ w := httptest.NewRecorder()
+ h.HandleFind(w, req)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+ }
+
+ var resp findResponse
+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+ t.Fatalf("decode response: %v", err)
+ }
+
+ if resp.BestRef != "e1" {
+ t.Errorf("expected best_ref=e1 (Cancel) for negative query, got %s", resp.BestRef)
+ }
+}
+
+func TestHandleFind_VisualQuery_BottomButton(t *testing.T) {
+ cache := &bridge.RefCache{
+ Nodes: []bridge.A11yNode{
+ {Ref: "e0", Role: "button", Name: "Action"},
+ {Ref: "e1", Role: "button", Name: "Action"},
+ {Ref: "e2", Role: "button", Name: "Action"},
+ },
+ Refs: map[string]int64{"e0": 1, "e1": 2, "e2": 3},
+ }
+
+ mb := &findMockBridge{refCache: cache}
+ h := New(mb, &config.RuntimeConfig{ActionTimeout: 10 * time.Second}, nil, nil, nil)
+ h.Matcher = semantic.NewCombinedMatcher(semantic.NewHashingEmbedder(128))
+
+ body := `{"query": "bottom button", "threshold": 0.0, "topK": 3}`
+ req := httptest.NewRequest("POST", "/find", bytes.NewReader([]byte(body)))
+ w := httptest.NewRecorder()
+ h.HandleFind(w, req)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+ }
+
+ var resp findResponse
+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+
+ if resp.BestRef != "e2" {
+ t.Errorf("expected best_ref=e2 (last in document order) for 'bottom button', got %s", resp.BestRef)
+ }
+}
+
func TestHandleFind_EmbeddingMatcher(t *testing.T) {
cache := &bridge.RefCache{
Nodes: []bridge.A11yNode{
diff --git a/tests/e2e/scenarios/cli/browser-extended.sh b/tests/e2e/scenarios/cli/browser-extended.sh
index 2dab5028..155a10e1 100644
--- a/tests/e2e/scenarios/cli/browser-extended.sh
+++ b/tests/e2e/scenarios/cli/browser-extended.sh
@@ -163,6 +163,35 @@ pt find "xyznonexistent99999"
end_test
+# ─────────────────────────────────────────────────────────────────
+start_test "pinchtab find (negative query)"
+
+pt_ok nav "${FIXTURES_URL}/find.html"
+# Resolve "log in" first; the negative form "log in not sign in" must keep
+# matching it, while "log in not log in" must not return the same ref.
+pt_ok find "log in" --ref-only
+LOGIN_REF="$PT_OUT"
+pt_ok find "log in button not sign up" --ref-only
+assert_output_contains "$LOGIN_REF" "negative query keeps Log In when excluding Sign Up"
+
+pt_ok find "button not log in" --ref-only
+assert_output_not_contains "$LOGIN_REF" "negative query 'button not log in' excludes Log In ref"
+
+end_test
+
+# ─────────────────────────────────────────────────────────────────
+start_test "pinchtab find (visual query: top vs bottom button)"
+
+pt_ok nav "${FIXTURES_URL}/find.html"
+# "top button" and "bottom button" share the same base query but bias toward
+# opposite ends of document order. The two refs must differ.
+pt_ok find "top button" --ref-only
+TOP_REF="$PT_OUT"
+pt_ok find "bottom button" --ref-only
+assert_output_not_contains "$TOP_REF" "visual hint produces different refs for top vs bottom"
+
+end_test
+
# ─────────────────────────────────────────────────────────────────
start_test "pinchtab snap --text"