Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

refactor: replace internal/semantic with github.com/pinchtab/semantic v0.1.0 (#378)

luigiagente04d166f34e5
diff --git a/go.mod b/go.mod
index 7e6200f1..387b3674 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
module github.com/pinchtab/pinchtab
-go 1.25.0
+go 1.26.0
require (
github.com/charmbracelet/lipgloss v1.1.0
@@ -39,6 +39,7 @@ require (
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/muesli/termenv v0.16.0 // indirect
+ github.com/pinchtab/semantic v0.1.0 // indirect
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/spf13/cast v1.7.1 // indirect
diff --git a/go.sum b/go.sum
index 7fdedb21..e5059105 100644
--- a/go.sum
+++ b/go.sum
@@ -77,6 +77,8 @@ github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A=
github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
+github.com/pinchtab/semantic v0.1.0 h1:jVBQmnddU1cCEKtpqk/t9QHrQ9y/VXMtZ0BI3VkLQxk=
+github.com/pinchtab/semantic v0.1.0/go.mod h1:SPKIra6m26faxTxn4dxa1ex1TbGkZDKbJ2r5rCngWoQ=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
diff --git a/internal/handlers/actions.go b/internal/handlers/actions.go
index 2add0a3c..2109367e 100644
--- a/internal/handlers/actions.go
+++ b/internal/handlers/actions.go
@@ -16,7 +16,8 @@ import (
"github.com/pinchtab/pinchtab/internal/engine"
"github.com/pinchtab/pinchtab/internal/httpx"
"github.com/pinchtab/pinchtab/internal/selector"
- "github.com/pinchtab/pinchtab/internal/semantic"
+ "github.com/pinchtab/semantic"
+ "github.com/pinchtab/semantic/recovery"
)
func resolveOwner(r *http.Request, fallback string) string {
@@ -261,7 +262,7 @@ func (h *Handlers) HandleAction(w http.ResponseWriter, r *http.Request) {
var result map[string]any
var engineName string
var actionErr error
- var recoveryResult *semantic.RecoveryResult
+ var recoveryResult *recovery.RecoveryResult
if refMissing && req.Ref != "" && h.Recovery != nil {
rr, actionRes, recoveryErr := h.Recovery.Attempt(
@@ -298,7 +299,7 @@ func (h *Handlers) HandleAction(w http.ResponseWriter, r *http.Request) {
if actionErr != nil && req.Ref != "" && h.Recovery != nil && h.Recovery.ShouldAttempt(actionErr, req.Ref) {
rr, actionRes, recoveryErr := h.Recovery.AttemptWithClassification(
tCtx, resolvedTabID, req.Ref, req.Kind,
- semantic.ClassifyFailure(actionErr),
+ recovery.ClassifyFailure(actionErr),
func(ctx context.Context, kind string, nodeID int64) (map[string]any, error) {
req.NodeID = nodeID
res, _, err := h.executeAction(ctx, req)
@@ -670,7 +671,7 @@ func (h *Handlers) handleActionsBatch(w http.ResponseWriter, r *http.Request, re
if err != nil && action.Ref != "" && h.Recovery != nil && h.Recovery.ShouldAttempt(err, action.Ref) {
rr, recRes, recErr := h.Recovery.AttemptWithClassification(
tCtx, resolvedTabID, action.Ref, action.Kind,
- semantic.ClassifyFailure(err),
+ recovery.ClassifyFailure(err),
func(ctx context.Context, kind string, nodeID int64) (map[string]any, error) {
action.NodeID = nodeID
res, _, err := h.executeAction(ctx, action)
@@ -933,7 +934,7 @@ func (h *Handlers) HandleMacro(w http.ResponseWriter, r *http.Request) {
if err != nil && step.Ref != "" && h.Recovery != nil && h.Recovery.ShouldAttempt(err, step.Ref) {
rr, recRes, recErr := h.Recovery.AttemptWithClassification(
tCtx, resolvedTabID, step.Ref, step.Kind,
- semantic.ClassifyFailure(err),
+ recovery.ClassifyFailure(err),
func(ctx context.Context, kind string, nodeID int64) (map[string]any, error) {
step.NodeID = nodeID
res, _, err := h.executeAction(ctx, step)
@@ -1001,7 +1002,7 @@ func (h *Handlers) cacheActionIntent(tabID string, req bridge.ActionRequest) {
}
}
}
- h.Recovery.RecordIntent(tabID, req.Ref, semantic.IntentEntry{
+ h.Recovery.RecordIntent(tabID, req.Ref, recovery.IntentEntry{
Descriptor: desc,
CachedAt: time.Now(),
})
diff --git a/internal/handlers/find.go b/internal/handlers/find.go
index 4e2b3295..1aa00949 100644
--- a/internal/handlers/find.go
+++ b/internal/handlers/find.go
@@ -12,7 +12,8 @@ import (
"github.com/pinchtab/pinchtab/internal/bridge"
"github.com/pinchtab/pinchtab/internal/httpx"
"github.com/pinchtab/pinchtab/internal/idpi"
- "github.com/pinchtab/pinchtab/internal/semantic"
+ "github.com/pinchtab/semantic"
+ "github.com/pinchtab/semantic/recovery"
)
type findRequest struct {
@@ -200,7 +201,7 @@ func (h *Handlers) HandleFind(w http.ResponseWriter, r *http.Request) {
break
}
}
- h.Recovery.RecordIntent(resolvedTabID, result.BestRef, semantic.IntentEntry{
+ h.Recovery.RecordIntent(resolvedTabID, result.BestRef, recovery.IntentEntry{
Query: req.Query,
Descriptor: bestDesc,
Score: result.BestScore,
diff --git a/internal/handlers/find_test.go b/internal/handlers/find_test.go
index 9696f584..3484ca9d 100644
--- a/internal/handlers/find_test.go
+++ b/internal/handlers/find_test.go
@@ -14,7 +14,7 @@ import (
"github.com/chromedp/cdproto/target"
"github.com/pinchtab/pinchtab/internal/bridge"
"github.com/pinchtab/pinchtab/internal/config"
- "github.com/pinchtab/pinchtab/internal/semantic"
+ "github.com/pinchtab/semantic"
)
// findMockBridge implements the subset of bridge.BridgeAPI required by HandleFind.
@@ -295,7 +295,7 @@ func TestHandleFind_EmbeddingMatcher(t *testing.T) {
mb := &findMockBridge{refCache: cache}
h := New(mb, &config.RuntimeConfig{ActionTimeout: 10 * time.Second}, nil, nil, nil)
- h.Matcher = semantic.NewEmbeddingMatcher(semantic.NewDummyEmbedder(64))
+ h.Matcher = semantic.NewEmbeddingMatcher(semantic.NewHashingEmbedder(64))
body := `{"query": "login button", "threshold": 0.0, "topK": 3}`
req := httptest.NewRequest("POST", "/find", bytes.NewReader([]byte(body)))
diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go
index 9f247a1c..29251bf1 100644
--- a/internal/handlers/handlers.go
+++ b/internal/handlers/handlers.go
@@ -12,7 +12,8 @@ import (
"github.com/pinchtab/pinchtab/internal/dashboard"
"github.com/pinchtab/pinchtab/internal/engine"
"github.com/pinchtab/pinchtab/internal/ids"
- "github.com/pinchtab/pinchtab/internal/semantic"
+ "github.com/pinchtab/semantic"
+ "github.com/pinchtab/semantic/recovery"
)
type Handlers struct {
@@ -23,15 +24,15 @@ type Handlers struct {
Orchestrator bridge.OrchestratorService
IdMgr *ids.Manager
Matcher semantic.ElementMatcher
- IntentCache *semantic.IntentCache
- Recovery *semantic.RecoveryEngine
+ IntentCache *recovery.IntentCache
+ Recovery *recovery.RecoveryEngine
Router *engine.Router // optional; nil ⇒ chrome-only
clipboard clipboardStore
}
func New(b bridge.BridgeAPI, cfg *config.RuntimeConfig, p bridge.ProfileService, d *dashboard.Dashboard, o bridge.OrchestratorService) *Handlers {
matcher := semantic.NewCombinedMatcher(semantic.NewHashingEmbedder(128))
- intentCache := semantic.NewIntentCache(200, 10*time.Minute)
+ intentCache := recovery.NewIntentCache(200, 10*time.Minute)
h := &Handlers{
Bridge: b,
@@ -46,8 +47,8 @@ func New(b bridge.BridgeAPI, cfg *config.RuntimeConfig, p bridge.ProfileService,
// Wire up the recovery engine with callbacks that delegate back to
// the handler's bridge without introducing circular imports.
- h.Recovery = semantic.NewRecoveryEngine(
- semantic.DefaultRecoveryConfig(),
+ h.Recovery = recovery.NewRecoveryEngine(
+ recovery.DefaultRecoveryConfig(),
matcher,
intentCache,
// SnapshotRefresher
diff --git a/internal/semantic/benchmark_study_test.go b/internal/semantic/benchmark_study_test.go
deleted file mode 100644
index 9a455788..00000000
--- a/internal/semantic/benchmark_study_test.go
+++ /dev/null
@@ -1,887 +0,0 @@
-package semantic
-
-// benchmark_study_test.go — Controlled benchmark study
-//
-// 20 queries × 10 page types comparing:
-// • LexicalMatcher (Jaccard + stopwords + role boost)
-// • EmbeddingMatcher (128-dim HashingEmbedder + cosine similarity)
-// • CombinedMatcher (0.6 lexical + 0.4 embedding)
-//
-// Metrics reported:
-// • Acc@1 — correct element is the top-ranked result
-// • Acc@3 — correct element appears in top-3 results
-// • Mean Latency (µs) per matcher
-//
-// Run:
-// go test ./internal/semantic/ -run TestBenchmarkStudy -v
-//
-// Or with benchmark timing detail:
-// go test ./internal/semantic/ -run TestBenchmarkStudy -v -count 5
-
-import (
- "context"
- "fmt"
- "sort"
- "strings"
- "testing"
- "time"
-)
-
-// -----------------------------------------------------------------------
-// Data structures
-// -----------------------------------------------------------------------
-
-// studyCase is a single (page, query, expected-ref) triple.
-type studyCase struct {
- page string // human-readable page name
- query string // natural language query
- expectedRef string // ref of the ground-truth element
- elements []ElementDescriptor
-}
-
-// studyResult records one matcher's answer for one case.
-type studyResult struct {
- matcherName string
- caseName string
- page string
- hit1 bool // Acc@1: best ref == expected
- hit3 bool // Acc@3: expected ref in top-3
- latencyNs int64
- bestRef string
- bestScore float64
-}
-
-// -----------------------------------------------------------------------
-// Ground-truth page element sets (10 pages × 2 queries each = 20 cases)
-// -----------------------------------------------------------------------
-
-func studyCases() []studyCase {
- // ---- Page 1: Login Form ------------------------------------------------
- login := []ElementDescriptor{
- {Ref: "e0", Role: "heading", Name: "Sign In"},
- {Ref: "e1", Role: "textbox", Name: "Email address"},
- {Ref: "e2", Role: "textbox", Name: "Password"},
- {Ref: "e3", Role: "checkbox", Name: "Remember me"},
- {Ref: "e4", Role: "button", Name: "Sign In"},
- {Ref: "e5", Role: "link", Name: "Forgot your password?"},
- {Ref: "e6", Role: "link", Name: "Create account"},
- {Ref: "e7", Role: "button", Name: "Continue with Google"},
- {Ref: "e8", Role: "button", Name: "Continue with Apple"},
- {Ref: "e9", Role: "img", Name: "Company logo"},
- }
-
- // ---- Page 2: Registration Form -----------------------------------------
- register := []ElementDescriptor{
- {Ref: "e0", Role: "heading", Name: "Create your account"},
- {Ref: "e1", Role: "textbox", Name: "First name"},
- {Ref: "e2", Role: "textbox", Name: "Last name"},
- {Ref: "e3", Role: "textbox", Name: "Email"},
- {Ref: "e4", Role: "textbox", Name: "Password"},
- {Ref: "e5", Role: "textbox", Name: "Confirm password"},
- {Ref: "e6", Role: "combobox", Name: "Date of birth"},
- {Ref: "e7", Role: "combobox", Name: "Country or region"},
- {Ref: "e8", Role: "checkbox", Name: "I agree to the Terms and Conditions"},
- {Ref: "e9", Role: "checkbox", Name: "Subscribe to marketing emails"},
- {Ref: "e10", Role: "button", Name: "Create account"},
- {Ref: "e11", Role: "link", Name: "Already have an account? Log in"},
- }
-
- // ---- Page 3: E-commerce Product Page -----------------------------------
- product := []ElementDescriptor{
- {Ref: "e0", Role: "heading", Name: "Wireless Noise-Cancelling Headphones"},
- {Ref: "e1", Role: "text", Name: "$299.99"},
- {Ref: "e2", Role: "combobox", Name: "Color", Value: "Midnight Black"},
- {Ref: "e3", Role: "spinbutton", Name: "Quantity", Value: "1"},
- {Ref: "e4", Role: "button", Name: "Add to cart"},
- {Ref: "e5", Role: "button", Name: "Buy now"},
- {Ref: "e6", Role: "button", Name: "Add to wishlist"},
- {Ref: "e7", Role: "tab", Name: "Description"},
- {Ref: "e8", Role: "tab", Name: "Reviews"},
- {Ref: "e9", Role: "tab", Name: "Specifications"},
- {Ref: "e10", Role: "img", Name: "Product image front view"},
- {Ref: "e11", Role: "text", Name: "Free shipping on orders over $50"},
- }
-
- // ---- Page 4: Navigation Header -----------------------------------------
- nav := []ElementDescriptor{
- {Ref: "e0", Role: "img", Name: "Site logo"},
- {Ref: "e1", Role: "link", Name: "Home"},
- {Ref: "e2", Role: "link", Name: "Products"},
- {Ref: "e3", Role: "link", Name: "Pricing"},
- {Ref: "e4", Role: "link", Name: "Blog"},
- {Ref: "e5", Role: "link", Name: "About Us"},
- {Ref: "e6", Role: "link", Name: "Contact"},
- {Ref: "e7", Role: "search", Name: "Search"},
- {Ref: "e8", Role: "button", Name: "Search"},
- {Ref: "e9", Role: "button", Name: "Open cart"},
- {Ref: "e10", Role: "link", Name: "Sign in"},
- {Ref: "e11", Role: "button", Name: "Open navigation menu"},
- }
-
- // ---- Page 5: Analytics Dashboard ---------------------------------------
- dashboard := []ElementDescriptor{
- {Ref: "e0", Role: "heading", Name: "Dashboard Overview"},
- {Ref: "e1", Role: "button", Name: "Export Report"},
- {Ref: "e2", Role: "button", Name: "Add Widget"},
- {Ref: "e3", Role: "combobox", Name: "Date range", Value: "Last 30 days"},
- {Ref: "e4", Role: "text", Name: "Total Revenue", Value: "$128,450"},
- {Ref: "e5", Role: "text", Name: "Active Users", Value: "8,302"},
- {Ref: "e6", Role: "text", Name: "Conversion Rate", Value: "3.4%"},
- {Ref: "e7", Role: "text", Name: "Avg Session Duration", Value: "4m 12s"},
- {Ref: "e8", Role: "button", Name: "Refresh Data"},
- {Ref: "e9", Role: "link", Name: "View detailed report"},
- {Ref: "e10", Role: "tab", Name: "Overview"},
- {Ref: "e11", Role: "tab", Name: "Revenue"},
- {Ref: "e12", Role: "tab", Name: "Users"},
- {Ref: "e13", Role: "button", Name: "Notifications"},
- }
-
- // ---- Page 6: Search Results Page ---------------------------------------
- search := []ElementDescriptor{
- {Ref: "e0", Role: "search", Name: "Search"},
- {Ref: "e1", Role: "button", Name: "Search"},
- {Ref: "e2", Role: "heading", Name: "Search Results for \"golang\""},
- {Ref: "e3", Role: "combobox", Name: "Sort by", Value: "Relevance"},
- {Ref: "e4", Role: "checkbox", Name: "Filter: Last 24 hours"},
- {Ref: "e5", Role: "checkbox", Name: "Filter: Images"},
- {Ref: "e6", Role: "checkbox", Name: "Filter: Videos"},
- {Ref: "e7", Role: "link", Name: "Next page"},
- {Ref: "e8", Role: "link", Name: "Previous page"},
- {Ref: "e9", Role: "button", Name: "Clear filters"},
- {Ref: "e10", Role: "text", Name: "About 4,230,000 results"},
- }
-
- // ---- Page 7: Admin Data Table ------------------------------------------
- table := []ElementDescriptor{
- {Ref: "e0", Role: "heading", Name: "Order Management"},
- {Ref: "e1", Role: "search", Name: "Search orders"},
- {Ref: "e2", Role: "button", Name: "Create order"},
- {Ref: "e3", Role: "button", Name: "Export to CSV"},
- {Ref: "e4", Role: "combobox", Name: "Status filter", Value: "All"},
- {Ref: "e5", Role: "combobox", Name: "Rows per page", Value: "25"},
- {Ref: "e6", Role: "columnheader", Name: "Order ID"},
- {Ref: "e7", Role: "columnheader", Name: "Customer"},
- {Ref: "e8", Role: "columnheader", Name: "Total"},
- {Ref: "e9", Role: "columnheader", Name: "Status"},
- {Ref: "e10", Role: "button", Name: "Previous page"},
- {Ref: "e11", Role: "button", Name: "Next page"},
- {Ref: "e12", Role: "button", Name: "Bulk delete"},
- {Ref: "e13", Role: "checkbox", Name: "Select all orders"},
- }
-
- // ---- Page 8: Confirmation Modal ----------------------------------------
- modal := []ElementDescriptor{
- {Ref: "e0", Role: "heading", Name: "Dashboard"},
- {Ref: "e1", Role: "button", Name: "New Project"},
- {Ref: "e2", Role: "dialog", Name: "Delete Project"},
- {Ref: "e3", Role: "heading", Name: "Delete Project"},
- {Ref: "e4", Role: "text", Name: "This will permanently delete the project and all its data. This action cannot be undone."},
- {Ref: "e5", Role: "textbox", Name: "Type project name to confirm"},
- {Ref: "e6", Role: "button", Name: "Delete project"},
- {Ref: "e7", Role: "button", Name: "Cancel"},
- {Ref: "e8", Role: "button", Name: "Close"},
- {Ref: "e9", Role: "navigation", Name: "Sidebar"},
- }
-
- // ---- Page 9: Settings / Preferences Page --------------------------------
- settings := []ElementDescriptor{
- {Ref: "e0", Role: "heading", Name: "Account Settings"},
- {Ref: "e1", Role: "textbox", Name: "Display name"},
- {Ref: "e2", Role: "textbox", Name: "Email address"},
- {Ref: "e3", Role: "textbox", Name: "Phone number"},
- {Ref: "e4", Role: "combobox", Name: "Language", Value: "English"},
- {Ref: "e5", Role: "combobox", Name: "Timezone", Value: "UTC-5"},
- {Ref: "e6", Role: "switch", Name: "Email notifications"},
- {Ref: "e7", Role: "switch", Name: "Push notifications"},
- {Ref: "e8", Role: "switch", Name: "Dark mode"},
- {Ref: "e9", Role: "button", Name: "Save changes"},
- {Ref: "e10", Role: "button", Name: "Cancel"},
- {Ref: "e11", Role: "button", Name: "Delete account"},
- {Ref: "e12", Role: "link", Name: "Change password"},
- }
-
- // ---- Page 10: Checkout / Payment Page ----------------------------------
- checkout := []ElementDescriptor{
- {Ref: "e0", Role: "heading", Name: "Checkout"},
- {Ref: "e1", Role: "textbox", Name: "Full name"},
- {Ref: "e2", Role: "textbox", Name: "Email"},
- {Ref: "e3", Role: "textbox", Name: "Shipping address"},
- {Ref: "e4", Role: "textbox", Name: "City"},
- {Ref: "e5", Role: "textbox", Name: "Postal code"},
- {Ref: "e6", Role: "combobox", Name: "Country"},
- {Ref: "e7", Role: "textbox", Name: "Card number"},
- {Ref: "e8", Role: "textbox", Name: "Expiry date"},
- {Ref: "e9", Role: "textbox", Name: "CVV"},
- {Ref: "e10", Role: "checkbox", Name: "Save card for future use"},
- {Ref: "e11", Role: "button", Name: "Place order"},
- {Ref: "e12", Role: "button", Name: "Back to cart"},
- {Ref: "e13", Role: "link", Name: "Apply coupon code"},
- }
-
- return []studyCase{
- // Page 1: Login — 2 queries
- {"Login Form", "sign in button", "e4", login},
- {"Login Form", "email input field", "e1", login},
-
- // Page 2: Registration — 2 queries
- {"Registration Form", "create account button", "e10", register},
- {"Registration Form", "confirm password field", "e5", register},
-
- // Page 3: E-commerce Product — 2 queries
- {"Product Page", "add to cart", "e4", product},
- {"Product Page", "product reviews tab", "e8", product},
-
- // Page 4: Navigation — 2 queries
- {"Navigation Header", "search box", "e7", nav},
- {"Navigation Header", "shopping cart", "e9", nav},
-
- // Page 5: Dashboard — 2 queries
- {"Analytics Dashboard", "export report", "e1", dashboard},
- {"Analytics Dashboard", "date range selector", "e3", dashboard},
-
- // Page 6: Search Results — 2 queries
- {"Search Results", "search input", "e0", search},
- {"Search Results", "next page link", "e7", search},
-
- // Page 7: Data Table — 2 queries
- {"Admin Data Table", "search orders", "e1", table},
- {"Admin Data Table", "export csv", "e3", table},
-
- // Page 8: Modal — 2 queries
- {"Confirmation Modal", "cancel button", "e7", modal},
- {"Confirmation Modal", "confirm deletion input", "e5", modal},
-
- // Page 9: Settings — 2 queries
- {"Settings Page", "save changes button", "e9", settings},
- {"Settings Page", "dark mode toggle", "e8", settings},
-
- // Page 10: Checkout — 2 queries
- {"Checkout Page", "place order button", "e11", checkout},
- {"Checkout Page", "card number field", "e7", checkout},
- }
-}
-
-// studyHardCases returns 10 intentionally-challenging query/element pairs
-// designed to reveal differentiation between matchers:
-//
-// Group A (query uses an abbreviation, 0 lexical word overlap):
-// expects embedding to win via character n-gram similarity.
-//
-// Group B (query uses a paraphrase / synonym):
-// expects both matchers to struggle, revealing the ceiling of
-// surface-form-only matching.
-//
-// Group C (ambiguous — multiple equally-plausible elements):
-// expects combined to win via score averaging.
-func studyHardCases() []studyCase {
- // Reuse page element sets from studyCases.
- cases := studyCases()
-
- // Helper: find element slice for a given page name.
- pageElems := map[string][]ElementDescriptor{}
- for _, c := range cases {
- if _, ok := pageElems[c.page]; !ok {
- pageElems[c.page] = c.elements
- }
- }
-
- product := pageElems["Product Page"]
- checkout := pageElems["Checkout Page"]
- login := pageElems["Login Form"]
- register := pageElems["Registration Form"]
- settings := pageElems["Settings Page"]
- table := pageElems["Admin Data Table"]
- modal := pageElems["Confirmation Modal"]
-
- return []studyCase{
- // ── Group A: Abbreviations (lexical = 0, embedding has n-gram overlap) ──
-
- // "specs" ↔ "Specifications" — shares "^sp","spe","pec","spec" (4-gram)
- {"Product Page [HARD]", "specs tab", "e9", product},
-
- // "qty" ↔ "Quantity" — shares "^q","ty","y$"
- {"Product Page [HARD]", "qty input", "e3", product},
-
- // "addr" ↔ "address" — shares "^a","ad","dd","dr"
- {"Checkout Page [HARD]", "addr field", "e3", checkout},
-
- // "pwd" ↔ "Password" — shares "^p","d$" (weak but present)
- {"Login Form [HARD]", "pwd textbox", "e2", login},
-
- // "notifs" ↔ "notifications" — shares "not","oti"
- {"Settings Page [HARD]", "toggle email notifs", "e6", settings},
-
- // ── Group B: Paraphrases / synonyms (no character overlap, both expected to struggle) ──
-
- // "download" ↔ "Export to CSV" — no shared characters
- {"Admin Data Table [HARD]", "download table data", "e3", table},
-
- // "proceed" ↔ "Place order" — no shared characters
- {"Checkout Page [HARD]", "proceed to payment", "e11", checkout},
-
- // "sign up" ↔ "Create account" — no shared characters
- {"Registration Form [HARD]", "sign up now", "e10", register},
-
- // "dismiss" ↔ "Cancel" — no shared characters
- {"Confirmation Modal [HARD]", "dismiss dialog", "e7", modal},
-
- // ── Group C: Ambiguous (multiple "button" elements, exact name helps) ──
-
- // "dark theme" ↔ "Dark mode" switch — "dark" word-matches exactly
- {"Settings Page [HARD]", "dark theme switch", "e8", settings},
- }
-}
-
-// -----------------------------------------------------------------------
-// Core evaluation logic
-// -----------------------------------------------------------------------
-
-func runMatcher(
- name string,
- matcher ElementMatcher,
- cases []studyCase,
-) []studyResult {
- ctx := context.Background()
- opts := FindOptions{Threshold: 0.0, TopK: 3}
- results := make([]studyResult, 0, len(cases))
-
- for _, c := range cases {
- start := time.Now()
- fr, err := matcher.Find(ctx, c.query, c.elements, opts)
- elapsed := time.Since(start).Nanoseconds()
-
- if err != nil {
- results = append(results, studyResult{
- matcherName: name,
- caseName: fmt.Sprintf("%s | %q", c.page, c.query),
- page: c.page,
- latencyNs: elapsed,
- })
- continue
- }
-
- // Acc@1
- hit1 := fr.BestRef == c.expectedRef
-
- // Acc@3 — expected ref in any of the top-3 matches
- hit3 := hit1
- if !hit3 {
- for _, m := range fr.Matches {
- if m.Ref == c.expectedRef {
- hit3 = true
- break
- }
- }
- }
-
- results = append(results, studyResult{
- matcherName: name,
- caseName: fmt.Sprintf("%s | %q", c.page, c.query),
- page: c.page,
- hit1: hit1,
- hit3: hit3,
- latencyNs: elapsed,
- bestRef: fr.BestRef,
- bestScore: fr.BestScore,
- })
- }
-
- return results
-}
-
-// -----------------------------------------------------------------------
-// Report generation helpers
-// -----------------------------------------------------------------------
-
-func percent(n, total int) string {
- if total == 0 {
- return " 0.0%"
- }
- return fmt.Sprintf("%5.1f%%", float64(n)/float64(total)*100)
-}
-
-func meanLatencyUs(results []studyResult) float64 {
- if len(results) == 0 {
- return 0
- }
- var sum int64
- for _, r := range results {
- sum += r.latencyNs
- }
- return float64(sum) / float64(len(results)) / 1000.0
-}
-
-func acc(results []studyResult, k int) (int, int) {
- hits := 0
- for _, r := range results {
- if k == 1 && r.hit1 {
- hits++
- }
- if k == 3 && r.hit3 {
- hits++
- }
- }
- return hits, len(results)
-}
-
-func printSeparator(t *testing.T, char string, width int) {
- t.Log(strings.Repeat(char, width))
-}
-
-// -----------------------------------------------------------------------
-// Main study test
-// -----------------------------------------------------------------------
-
-func TestBenchmarkStudy(t *testing.T) {
- cases := studyCases()
-
- matchers := []struct {
- name string
- matcher ElementMatcher
- }{
- {"Lexical", NewLexicalMatcher()},
- {"Embedding", NewEmbeddingMatcher(NewHashingEmbedder(128))},
- {"Combined", NewCombinedMatcher(NewHashingEmbedder(128))},
- }
-
- // Run all matchers
- allResults := make(map[string][]studyResult, len(matchers))
- for _, m := range matchers {
- allResults[m.name] = runMatcher(m.name, m.matcher, cases)
- }
-
- // ----------------------------------------------------------------
- // REPORT HEADER
- // ----------------------------------------------------------------
- const W = 72
- t.Log("")
- printSeparator(t, "═", W)
- t.Log(" SEMANTIC MATCHING — CONTROLLED BENCHMARK STUDY")
- t.Logf(" %d queries × %d page types | 3 matchers | TopK=3 | Threshold=0",
- len(cases), 10)
- printSeparator(t, "═", W)
-
- // ----------------------------------------------------------------
- // OVERALL SUMMARY TABLE
- // ----------------------------------------------------------------
- t.Log("")
- t.Log(" OVERALL RESULTS")
- t.Log("")
- t.Logf(" %-12s %7s %7s %12s", "Matcher", "Acc@1", "Acc@3", "Latency (µs)")
- t.Log(" " + strings.Repeat("-", 46))
-
- type summaryRow struct {
- name string
- acc1 int
- acc3 int
- total int
- latency float64
- }
- rows := make([]summaryRow, 0, len(matchers))
-
- for _, m := range matchers {
- res := allResults[m.name]
- h1, tot := acc(res, 1)
- h3, _ := acc(res, 3)
- lat := meanLatencyUs(res)
- rows = append(rows, summaryRow{m.name, h1, h3, tot, lat})
- t.Logf(" %-12s %s %s %10.1f µs",
- m.name, percent(h1, tot), percent(h3, tot), lat)
- }
-
- t.Log(" " + strings.Repeat("-", 46))
- t.Log("")
-
- // ----------------------------------------------------------------
- // PER-PAGE BREAKDOWN
- // ----------------------------------------------------------------
- // Collect unique page names in order
- pages := make([]string, 0, 10)
- seen := map[string]bool{}
- for _, c := range cases {
- if !seen[c.page] {
- pages = append(pages, c.page)
- seen[c.page] = true
- }
- }
-
- printSeparator(t, "─", W)
- t.Log(" PER-PAGE BREAKDOWN (Acc@1 / Acc@3) ")
- printSeparator(t, "─", W)
- t.Log("")
- t.Logf(" %-26s %-14s %-14s %-14s", "Page", "Lexical", "Embedding", "Combined")
- t.Log(" " + strings.Repeat("-", 66))
-
- for _, page := range pages {
- // Filter results for this page
- cols := make([]string, 0, 3)
- for _, m := range matchers {
- var pageRes []studyResult
- for _, r := range allResults[m.name] {
- if r.page == page {
- pageRes = append(pageRes, r)
- }
- }
- h1, tot := acc(pageRes, 1)
- h3, _ := acc(pageRes, 3)
- cols = append(cols, fmt.Sprintf("%s / %s", percent(h1, tot), percent(h3, tot)))
- }
- t.Logf(" %-26s %-14s %-14s %-14s", page, cols[0], cols[1], cols[2])
- }
- t.Log("")
-
- // ----------------------------------------------------------------
- // DETAILED CASE-BY-CASE TABLE
- // ----------------------------------------------------------------
- printSeparator(t, "─", W)
- t.Log(" CASE-BY-CASE RESULTS (✓ = hit, ✗ = miss) ")
- printSeparator(t, "─", W)
- t.Log("")
- t.Logf(" %-6s %-28s %-8s %-8s %-8s %-8s %-8s %-8s",
- "#", "Query", "Lex@1", "Lex@3", "Emb@1", "Emb@3", "Com@1", "Com@3")
- t.Log(" " + strings.Repeat("-", 88))
-
- mark := func(hit bool) string {
- if hit {
- return " ✓"
- }
- return " ✗"
- }
-
- for i, c := range cases {
- shortQ := c.query
- if len(shortQ) > 27 {
- shortQ = shortQ[:24] + "..."
- }
-
- lex := allResults["Lexical"][i]
- emb := allResults["Embedding"][i]
- com := allResults["Combined"][i]
-
- t.Logf(" %3d. %-28s %-8s %-8s %-8s %-8s %-8s %-8s",
- i+1, shortQ,
- mark(lex.hit1), mark(lex.hit3),
- mark(emb.hit1), mark(emb.hit3),
- mark(com.hit1), mark(com.hit3),
- )
- }
- t.Log("")
-
- // ----------------------------------------------------------------
- // MISSED CASES ANALYSIS
- // ----------------------------------------------------------------
- printSeparator(t, "─", W)
- t.Log(" MISSED CASES ANALYSIS (Acc@1 misses)")
- printSeparator(t, "─", W)
- t.Log("")
-
- for _, m := range matchers {
- missCount := 0
- for _, r := range allResults[m.name] {
- if !r.hit1 {
- missCount++
- }
- }
- if missCount == 0 {
- t.Logf(" %s: perfect score — no Acc@1 misses", m.name)
- continue
- }
- t.Logf(" %s misses (%d):", m.name, missCount)
- for i, r := range allResults[m.name] {
- if r.hit1 {
- continue
- }
- t.Logf(" [%2d] %-28s expected=%-4s got=%-4s score=%.3f",
- i+1, fmt.Sprintf("%q", cases[i].query),
- cases[i].expectedRef, r.bestRef, r.bestScore)
- }
- }
- t.Log("")
-
- // ----------------------------------------------------------------
- // MATCHER COMPARISON: cases where they DISAGREE
- // ----------------------------------------------------------------
- printSeparator(t, "─", W)
- t.Log(" DISAGREMENT ANALYSIS (where Combined beats both)")
- printSeparator(t, "─", W)
- t.Log("")
-
- improvements := 0
- for i := range cases {
- lex := allResults["Lexical"][i]
- emb := allResults["Embedding"][i]
- com := allResults["Combined"][i]
-
- if !lex.hit1 && !emb.hit1 && com.hit1 {
- improvements++
- t.Logf(" [%2d] %q — Combined rescued (lex✗ emb✗ com✓)",
- i+1, cases[i].query)
- }
- if lex.hit1 && emb.hit1 && !com.hit1 {
- t.Logf(" [%2d] %q — Combined degraded (lex✓ emb✓ com✗) !",
- i+1, cases[i].query)
- }
- }
- if improvements == 0 {
- t.Log(" No unique rescues by Combined (or none needed)")
- }
- t.Log("")
-
- // ----------------------------------------------------------------
- // LATENCY COMPARISON
- // ----------------------------------------------------------------
- printSeparator(t, "─", W)
- t.Log(" LATENCY SUMMARY")
- printSeparator(t, "─", W)
- t.Log("")
-
- // Sort by latency ascending
- sort.Slice(rows, func(i, j int) bool {
- return rows[i].latency < rows[j].latency
- })
-
- baseline := rows[0].latency
- for _, row := range rows {
- overhead := ""
- if row.latency > baseline {
- overhead = fmt.Sprintf(" (+%.1fx)", row.latency/baseline)
- }
- t.Logf(" %-12s %8.2f µs%s", row.name, row.latency, overhead)
- }
- t.Log("")
-
- // ----------------------------------------------------------------
- // FINAL VERDICT
- // ----------------------------------------------------------------
- printSeparator(t, "═", W)
- t.Log(" VERDICT")
- printSeparator(t, "─", W)
- t.Log("")
-
- // Find best Acc@1
- bestAcc1Name := ""
- bestAcc1 := -1
- for _, m := range matchers {
- h1, tot := acc(allResults[m.name], 1)
- pct := h1 * 100 / tot
- if pct > bestAcc1 {
- bestAcc1 = pct
- bestAcc1Name = m.name
- }
- }
-
- lexH1, tot := acc(allResults["Lexical"], 1)
- embH1, _ := acc(allResults["Embedding"], 1)
- comH1, _ := acc(allResults["Combined"], 1)
- lexH3, _ := acc(allResults["Lexical"], 3)
- embH3, _ := acc(allResults["Embedding"], 3)
- comH3, _ := acc(allResults["Combined"], 3)
-
- t.Logf(" Best Acc@1: %s (%d/%d = %s)", bestAcc1Name, comH1, tot, percent(comH1, tot))
- t.Log("")
- t.Logf(" Acc@1 — Lexical: %s Embedding: %s Combined: %s",
- percent(lexH1, tot), percent(embH1, tot), percent(comH1, tot))
- t.Logf(" Acc@3 — Lexical: %s Embedding: %s Combined: %s",
- percent(lexH3, tot), percent(embH3, tot), percent(comH3, tot))
- t.Log("")
- t.Logf(" Mean latency — Lexical: %.1fµs Embedding: %.1fµs Combined: %.1fµs",
- meanLatencyUs(allResults["Lexical"]),
- meanLatencyUs(allResults["Embedding"]),
- meanLatencyUs(allResults["Combined"]))
- t.Log("")
- printSeparator(t, "═", W)
- t.Log("")
-
- // ================================================================
- // PART II — HARD CASES (abbreviations, synonyms, paraphrases)
- // ================================================================
- hardCases := studyHardCases()
-
- allHard := make(map[string][]studyResult, len(matchers))
- for _, m := range matchers {
- allHard[m.name] = runMatcher(m.name, m.matcher, hardCases)
- }
-
- t.Log("")
- printSeparator(t, "═", W)
- t.Log(" PART II — HARD CASES (abbreviations, synonyms, paraphrases)")
- t.Logf(" %d queries | 3 matcher groups: A=abbrev B=synonym C=ambiguous",
- len(hardCases))
- printSeparator(t, "═", W)
-
- // Overall hard summary
- t.Log("")
- t.Log(" HARD OVERALL")
- t.Log("")
- t.Logf(" %-12s %7s %7s %12s", "Matcher", "Acc@1", "Acc@3", "Latency (µs)")
- t.Log(" " + strings.Repeat("-", 46))
- for _, m := range matchers {
- res := allHard[m.name]
- h1, tot := acc(res, 1)
- h3, _ := acc(res, 3)
- lat := meanLatencyUs(res)
- t.Logf(" %-12s %s %s %10.1f µs",
- m.name, percent(h1, tot), percent(h3, tot), lat)
- }
- t.Log(" " + strings.Repeat("-", 46))
- t.Log("")
-
- // Case-by-case hard results
- printSeparator(t, "─", W)
- t.Log(" HARD CASE-BY-CASE (✓ = hit, ✗ = miss)")
- printSeparator(t, "─", W)
- t.Log("")
- t.Logf(" %-3s %-3s %-30s %-8s %-8s %-8s %-8s %-8s %-8s",
- "#", "Grp", "Query", "Lex@1", "Lex@3", "Emb@1", "Emb@3", "Com@1", "Com@3")
- t.Log(" " + strings.Repeat("-", 90))
-
- groups := []string{"A", "A", "A", "A", "A", "B", "B", "B", "B", "C"}
- for i, c := range hardCases {
- grp := groups[i]
- shortQ := c.query
- if len(shortQ) > 29 {
- shortQ = shortQ[:26] + "..."
- }
- lex := allHard["Lexical"][i]
- emb := allHard["Embedding"][i]
- com := allHard["Combined"][i]
-
- t.Logf(" %3d %-3s %-30s %-8s %-8s %-8s %-8s %-8s %-8s",
- i+1, grp, shortQ,
- mark(lex.hit1), mark(lex.hit3),
- mark(emb.hit1), mark(emb.hit3),
- mark(com.hit1), mark(com.hit3),
- )
- }
- t.Log("")
-
- // Per-group analysis
- printSeparator(t, "─", W)
- t.Log(" GROUP ANALYSIS")
- printSeparator(t, "─", W)
- t.Log("")
- t.Logf(" %-20s %-14s %-14s %-14s", "Group", "Lexical Acc@1", "Embedding Acc@1", "Combined Acc@1")
- t.Log(" " + strings.Repeat("-", 66))
-
- groupDefs := []struct {
- label string
- ids []int // 0-indexed
- desc string
- }{
- {"A (abbreviations)", []int{0, 1, 2, 3, 4}, "expected: Emb ≥ Lex"},
- {"B (synonyms)", []int{5, 6, 7, 8}, "expected: both struggle"},
- {"C (ambiguous)", []int{9}, "expected: Combined wins"},
- }
-
- for _, gd := range groupDefs {
- for _, mname := range []string{"Lexical", "Embedding", "Combined"} {
- var subset []studyResult
- for _, idx := range gd.ids {
- subset = append(subset, allHard[mname][idx])
- }
- _ = subset
- }
- lexSub := filterByIdx(allHard["Lexical"], gd.ids)
- embSub := filterByIdx(allHard["Embedding"], gd.ids)
- comSub := filterByIdx(allHard["Combined"], gd.ids)
- lh1, lt := acc(lexSub, 1)
- eh1, et := acc(embSub, 1)
- ch1, ct := acc(comSub, 1)
- t.Logf(" %-20s %-14s %-15s %-14s %s",
- gd.label,
- percent(lh1, lt), percent(eh1, et), percent(ch1, ct),
- gd.desc)
- }
- t.Log("")
-
- // Combined vs Individual: misses that Combined rescues
- printSeparator(t, "─", W)
- t.Log(" COMBINED RESCUES (in hard cases)")
- printSeparator(t, "─", W)
- t.Log("")
- rescueCount := 0
- for i := range hardCases {
- lex := allHard["Lexical"][i]
- emb := allHard["Embedding"][i]
- com := allHard["Combined"][i]
- if !lex.hit1 && !emb.hit1 && com.hit1 {
- rescueCount++
- t.Logf(" RESCUE [%d] %q — Lex✗ Emb✗ Com✓", i+1, hardCases[i].query)
- }
- if !lex.hit1 && emb.hit1 && com.hit1 {
- t.Logf(" EMB WIN [%d] %q — Lex✗ Emb✓ Com✓", i+1, hardCases[i].query)
- }
- if lex.hit1 && !emb.hit1 && com.hit1 {
- t.Logf(" LEX WIN [%d] %q — Lex✓ Emb✗ Com✓", i+1, hardCases[i].query)
- }
- if lex.hit1 && !emb.hit1 && !com.hit1 {
- t.Logf(" COM FAIL[%d] %q — Lex✓ Emb✗ Com✗ ← embedding dragged score down", i+1, hardCases[i].query)
- }
- }
- if rescueCount == 0 {
- t.Log(" No unique Combined rescues in hard cases")
- }
- t.Log("")
-
- // Score comparison for hard cases
- printSeparator(t, "─", W)
- t.Log(" SCORE DETAIL (best score returned by each matcher per hard case)")
- printSeparator(t, "─", W)
- t.Log("")
- t.Logf(" %-3s %-30s %-6s %-8s %-8s %-8s %-8s",
- "#", "Query", "Want", "Lex score", "Emb score", "Com score", "Winner")
- t.Log(" " + strings.Repeat("-", 78))
- for i, c := range hardCases {
- lex := allHard["Lexical"][i]
- emb := allHard["Embedding"][i]
- com := allHard["Combined"][i]
- shortQ := c.query
- if len(shortQ) > 29 {
- shortQ = shortQ[:26] + "..."
- }
- var winner string
- if !lex.hit1 && emb.hit1 {
- winner = "Embedding"
- } else if lex.hit1 && !emb.hit1 {
- winner = "Lexical"
- } else if lex.hit1 && emb.hit1 {
- winner = "both"
- } else {
- winner = "none"
- }
- t.Logf(" %3d %-30s %-6s %9.3f %9.3f %9.3f %s",
- i+1, shortQ, c.expectedRef,
- lex.bestScore, emb.bestScore, com.bestScore, winner)
- }
- t.Log("")
-
- // Final combined verdict
- printSeparator(t, "═", W)
- t.Log(" FINAL VERDICT — EASY + HARD COMBINED")
- printSeparator(t, "─", W)
- t.Log("")
-
- allTotal := len(cases) + len(hardCases)
- for _, m := range matchers {
- combined := append(allResults[m.name], allHard[m.name]...)
- h1, _ := acc(combined, 1)
- h3, _ := acc(combined, 3)
- lat := meanLatencyUs(combined)
- t.Logf(" %-12s Acc@1=%s Acc@3=%s µs=%.1f (total %d queries)",
- m.name, percent(h1, allTotal), percent(h3, allTotal), lat, allTotal)
- }
- t.Log("")
- printSeparator(t, "═", W)
- t.Log("")
-}
-
-// filterByIdx returns results at specified 0-based indices.
-func filterByIdx(results []studyResult, indices []int) []studyResult {
- out := make([]studyResult, 0, len(indices))
- for _, i := range indices {
- if i < len(results) {
- out = append(out, results[i])
- }
- }
- return out
-}
diff --git a/internal/semantic/calibration.go b/internal/semantic/calibration.go
deleted file mode 100644
index 143821e9..00000000
--- a/internal/semantic/calibration.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package semantic
-
-// CalibrateConfidence maps a numeric similarity score to a human-readable
-// confidence label. This function is shared across all matcher strategies
-// to ensure consistent labelling.
-func CalibrateConfidence(score float64) string {
- switch {
- case score >= 0.8:
- return "high"
- case score >= 0.6:
- return "medium"
- default:
- return "low"
- }
-}
diff --git a/internal/semantic/combined_matcher.go b/internal/semantic/combined_matcher.go
deleted file mode 100644
index 4a7e915e..00000000
--- a/internal/semantic/combined_matcher.go
+++ /dev/null
@@ -1,192 +0,0 @@
-package semantic
-
-import (
- "context"
- "fmt"
- "sort"
-)
-
-// CombinedMatcher fuses results from a lexical matcher and an embedding
-// matcher. The final score for each element is a weighted average:
-//
-// score = lexicalWeight * lexicalScore + embeddingWeight * embeddingScore
-//
-// This gives the best of both worlds: exact word overlap from lexical
-// matching and fuzzy / sub-word similarity from embedding matching.
-type CombinedMatcher struct {
- lexical *LexicalMatcher
- embedding *EmbeddingMatcher
-
- // Weight factors (should sum to 1.0 for interpretable scores).
- LexicalWeight float64
- EmbeddingWeight float64
-}
-
-// NewCombinedMatcher creates a CombinedMatcher with default weights
-// (0.6 lexical, 0.4 embedding). The lexical component has higher weight
-// because it handles exact matches perfectly, while embeddings add value
-// for fuzzy / partial queries.
-func NewCombinedMatcher(embedder Embedder) *CombinedMatcher {
- return &CombinedMatcher{
- lexical: NewLexicalMatcher(),
- embedding: NewEmbeddingMatcher(embedder),
- LexicalWeight: 0.6,
- EmbeddingWeight: 0.4,
- }
-}
-
-// Strategy returns "combined:lexical+embedding:<embedder>".
-func (c *CombinedMatcher) Strategy() string {
- return "combined:lexical+" + c.embedding.Strategy()
-}
-
-// Find runs both lexical and embedding matchers, merges scores by ref,
-// applies weighted averaging, and returns the top-K candidates.
-func (c *CombinedMatcher) Find(ctx context.Context, query string, elements []ElementDescriptor, opts FindOptions) (FindResult, error) {
- if opts.TopK <= 0 {
- opts.TopK = 3
- }
-
- // Per-request weight overrides; fall back to matcher defaults.
- lexW := c.LexicalWeight
- embW := c.EmbeddingWeight
- if opts.LexicalWeight > 0 || opts.EmbeddingWeight > 0 {
- lexW = opts.LexicalWeight
- embW = opts.EmbeddingWeight
- }
-
- // Use a lower internal threshold to capture candidates from both matchers
- // that might miss the final threshold individually but pass when combined.
- internalOpts := FindOptions{
- Threshold: opts.Threshold * 0.5,
- TopK: len(elements), // get all candidates internally
- }
-
- // Run both matchers concurrently.
- type matcherResult struct {
- result FindResult
- err error
- }
- lexCh := make(chan matcherResult, 1)
- embCh := make(chan matcherResult, 1)
-
- go func() {
- defer func() {
- if p := recover(); p != nil {
- lexCh <- matcherResult{err: fmt.Errorf("lexical matcher panic: %v", p)}
- }
- }()
- r, err := c.lexical.Find(ctx, query, elements, internalOpts)
- lexCh <- matcherResult{r, err}
- }()
- go func() {
- defer func() {
- if p := recover(); p != nil {
- embCh <- matcherResult{err: fmt.Errorf("embedding matcher panic: %v", p)}
- }
- }()
- r, err := c.embedding.Find(ctx, query, elements, internalOpts)
- embCh <- matcherResult{r, err}
- }()
-
- lexRes := <-lexCh
- embRes := <-embCh
-
- if lexRes.err != nil {
- return FindResult{}, lexRes.err
- }
- if embRes.err != nil {
- return FindResult{}, embRes.err
- }
-
- // Build ref → score maps.
- lexScores := make(map[string]float64, len(lexRes.result.Matches))
- for _, m := range lexRes.result.Matches {
- lexScores[m.Ref] = m.Score
- }
-
- embScores := make(map[string]float64, len(embRes.result.Matches))
- for _, m := range embRes.result.Matches {
- embScores[m.Ref] = m.Score
- }
-
- // Build element lookup by ref for metadata.
- refToElem := make(map[string]ElementDescriptor, len(elements))
- for _, el := range elements {
- refToElem[el.Ref] = el
- }
-
- // Merge: collect all refs that appeared in either matcher.
- allRefs := make(map[string]bool)
- for ref := range lexScores {
- allRefs[ref] = true
- }
- for ref := range embScores {
- allRefs[ref] = true
- }
-
- type scored struct {
- ref string
- score float64
- el ElementDescriptor
- lexScore float64
- embScore float64
- }
-
- var candidates []scored
- for ref := range allRefs {
- ls := lexScores[ref]
- es := embScores[ref]
- combined := lexW*ls + embW*es
- if combined >= opts.Threshold {
- s := scored{
- ref: ref,
- score: combined,
- el: refToElem[ref],
- }
- if opts.Explain {
- s.lexScore = lexW * ls
- s.embScore = embW * es
- }
- candidates = append(candidates, s)
- }
- }
-
- // Sort descending by combined score.
- sort.Slice(candidates, func(i, j int) bool {
- return candidates[i].score > candidates[j].score
- })
-
- if len(candidates) > opts.TopK {
- candidates = candidates[:opts.TopK]
- }
-
- result := FindResult{
- Strategy: c.Strategy(),
- ElementCount: len(elements),
- }
-
- for _, cand := range candidates {
- em := ElementMatch{
- Ref: cand.ref,
- Score: cand.score,
- Role: cand.el.Role,
- Name: cand.el.Name,
- }
- if opts.Explain {
- em.Explain = &MatchExplain{
- LexicalScore: cand.lexScore,
- EmbeddingScore: cand.embScore,
- Composite: cand.el.Composite(),
- }
- }
- result.Matches = append(result.Matches, em)
- }
-
- if len(result.Matches) > 0 {
- result.BestRef = result.Matches[0].Ref
- result.BestScore = result.Matches[0].Score
- }
-
- return result, nil
-}
diff --git a/internal/semantic/descriptor.go b/internal/semantic/descriptor.go
deleted file mode 100644
index 4fe51652..00000000
--- a/internal/semantic/descriptor.go
+++ /dev/null
@@ -1,32 +0,0 @@
-// Package semantic provides lightweight lexical similarity matching
-// for accessibility tree elements. Zero external dependencies.
-package semantic
-
-import "strings"
-
-// ElementDescriptor builds a composite text description from an
-// accessibility tree node's properties for similarity comparison.
-type ElementDescriptor struct {
- Ref string
- Role string
- Name string
- Value string
-}
-
-// Composite returns a single string that captures the semantic identity
-// of an element, suitable for lexical similarity comparison.
-func (ed *ElementDescriptor) Composite() string {
- var parts []string
-
- if ed.Role != "" {
- parts = append(parts, ed.Role+":")
- }
- if ed.Name != "" {
- parts = append(parts, ed.Name)
- }
- if ed.Value != "" && ed.Value != ed.Name {
- parts = append(parts, "["+ed.Value+"]")
- }
-
- return strings.Join(parts, " ")
-}
diff --git a/internal/semantic/embedder.go b/internal/semantic/embedder.go
deleted file mode 100644
index 7079d2da..00000000
--- a/internal/semantic/embedder.go
+++ /dev/null
@@ -1,82 +0,0 @@
-package semantic
-
-// Embedder is the interface for converting text into dense vector
-// representations. Implementations range from lightweight deterministic
-// hashing (DummyEmbedder) to real ML models (future LocalEmbedder,
-// OpenAIEmbedder).
-type Embedder interface {
- // Embed converts a batch of text strings into float32 vectors.
- // All returned vectors must have the same dimensionality.
- Embed(texts []string) ([][]float32, error)
-
- // Strategy returns the name of the embedding strategy (e.g. "dummy", "local", "openai").
- Strategy() string
-}
-
-// DummyEmbedder generates deterministic fixed-dimension vectors using a
-// simple hash of each input string. Useful for architecture testing
-// without real ML dependencies.
-type DummyEmbedder struct {
- Dim int // vector dimensionality (default 64)
-}
-
-// NewDummyEmbedder creates a DummyEmbedder with the given dimensionality.
-func NewDummyEmbedder(dim int) *DummyEmbedder {
- if dim <= 0 {
- dim = 64
- }
- return &DummyEmbedder{Dim: dim}
-}
-
-// Strategy returns "dummy".
-func (d *DummyEmbedder) Strategy() string { return "dummy" }
-
-// Embed generates deterministic pseudo-vectors by hashing each character
-// of the input string into the vector dimensions. The resulting vectors
-// have useful properties: identical strings produce identical vectors,
-// and strings with shared tokens produce vectors with non-zero cosine
-// similarity.
-func (d *DummyEmbedder) Embed(texts []string) ([][]float32, error) {
- result := make([][]float32, len(texts))
- for i, text := range texts {
- result[i] = d.hashVec(text)
- }
- return result, nil
-}
-
-// hashVec produces a deterministic float32 vector from a string.
-func (d *DummyEmbedder) hashVec(s string) []float32 {
- vec := make([]float32, d.Dim)
- for i, c := range s {
- idx := (i*31 + int(c)) % d.Dim
- if idx < 0 {
- idx = -idx
- }
- vec[idx] += float32(c) / 128.0
- }
- // Normalize to unit length for cosine similarity.
- var norm float64
- for _, v := range vec {
- norm += float64(v) * float64(v)
- }
- if norm > 0 {
- invNorm := float32(1.0 / sqrt64(norm))
- for j := range vec {
- vec[j] *= invNorm
- }
- }
- return vec
-}
-
-// sqrt64 is a simple float64 square root to avoid importing math in this file.
-func sqrt64(x float64) float64 {
- if x <= 0 {
- return 0
- }
- // Newton's method, 20 iterations is ample for float64 precision.
- z := x / 2
- for i := 0; i < 20; i++ {
- z = (z + x/z) / 2
- }
- return z
-}
diff --git a/internal/semantic/embedding_matcher.go b/internal/semantic/embedding_matcher.go
deleted file mode 100644
index 3468806c..00000000
--- a/internal/semantic/embedding_matcher.go
+++ /dev/null
@@ -1,108 +0,0 @@
-package semantic
-
-import (
- "context"
- "math"
- "sort"
-)
-
-// EmbeddingMatcher implements ElementMatcher using vector embeddings
-// and cosine similarity. It delegates text → vector conversion to the
-// provided Embedder implementation.
-type EmbeddingMatcher struct {
- embedder Embedder
-}
-
-// NewEmbeddingMatcher creates an EmbeddingMatcher backed by the given Embedder.
-func NewEmbeddingMatcher(e Embedder) *EmbeddingMatcher {
- return &EmbeddingMatcher{embedder: e}
-}
-
-// Strategy returns "embedding:<embedder_strategy>".
-func (m *EmbeddingMatcher) Strategy() string {
- return "embedding:" + m.embedder.Strategy()
-}
-
-// Find embeds the query and all element descriptions, ranks by cosine
-// similarity, filters by threshold, and returns top-K matches.
-func (m *EmbeddingMatcher) Find(_ context.Context, query string, elements []ElementDescriptor, opts FindOptions) (FindResult, error) {
- if opts.TopK <= 0 {
- opts.TopK = 3
- }
-
- // Build composite descriptions.
- descs := make([]string, len(elements))
- for i, el := range elements {
- descs[i] = el.Composite()
- }
-
- // Embed query + all descriptions in a single batch.
- texts := append([]string{query}, descs...)
- vectors, err := m.embedder.Embed(texts)
- if err != nil {
- return FindResult{}, err
- }
-
- queryVec := vectors[0]
- elemVecs := vectors[1:]
-
- type scored struct {
- desc ElementDescriptor
- score float64
- }
-
- var candidates []scored
- for i, el := range elements {
- sim := CosineSimilarity(queryVec, elemVecs[i])
- if sim >= opts.Threshold {
- candidates = append(candidates, scored{desc: el, score: sim})
- }
- }
-
- sort.Slice(candidates, func(i, j int) bool {
- return candidates[i].score > candidates[j].score
- })
-
- if len(candidates) > opts.TopK {
- candidates = candidates[:opts.TopK]
- }
-
- result := FindResult{
- Strategy: m.Strategy(),
- ElementCount: len(elements),
- }
-
- for _, c := range candidates {
- result.Matches = append(result.Matches, ElementMatch{
- Ref: c.desc.Ref,
- Score: c.score,
- Role: c.desc.Role,
- Name: c.desc.Name,
- })
- }
-
- if len(result.Matches) > 0 {
- result.BestRef = result.Matches[0].Ref
- result.BestScore = result.Matches[0].Score
- }
-
- return result, nil
-}
-
-// CosineSimilarity computes cosine similarity between two float32 vectors.
-// Returns a value in [-1, 1]; for normalized vectors this is in [0, 1].
-func CosineSimilarity(a, b []float32) float64 {
- if len(a) != len(b) || len(a) == 0 {
- return 0
- }
- var dot, normA, normB float64
- for i := range a {
- dot += float64(a[i]) * float64(b[i])
- normA += float64(a[i]) * float64(a[i])
- normB += float64(b[i]) * float64(b[i])
- }
- if normA == 0 || normB == 0 {
- return 0
- }
- return dot / (math.Sqrt(normA) * math.Sqrt(normB))
-}
diff --git a/internal/semantic/failure.go b/internal/semantic/failure.go
deleted file mode 100644
index 35da3cc7..00000000
--- a/internal/semantic/failure.go
+++ /dev/null
@@ -1,168 +0,0 @@
-package semantic
-
-import "strings"
-
-// FailureType classifies the cause of an action failure so the recovery
-// engine can decide whether and how to attempt self-healing.
-type FailureType int
-
-const (
- // FailureUnknown — the error could not be classified; not recoverable.
- FailureUnknown FailureType = iota
-
- // FailureElementNotFound — the ref no longer maps to a DOM node.
- // Recoverable via semantic re-match on a fresh snapshot.
- FailureElementNotFound
-
- // FailureElementStale — the node exists but has been detached from
- // the live DOM (SPA re-render, modal overlay, etc.).
- FailureElementStale
-
- // FailureElementNotInteractable — the element is hidden, disabled,
- // or covered by an overlay and cannot receive input.
- FailureElementNotInteractable
-
- // FailureNavigation — the action triggered a page navigation; the
- // entire snapshot is invalid. Recoverable via fresh snapshot.
- FailureNavigation
-
- // FailureNetwork — a transport or Chrome-level error; not recoverable
- // by the recovery engine.
- FailureNetwork
-)
-
-// String returns a human-readable label for the failure type.
-func (f FailureType) String() string {
- switch f {
- case FailureElementNotFound:
- return "element_not_found"
- case FailureElementStale:
- return "element_stale"
- case FailureElementNotInteractable:
- return "element_not_interactable"
- case FailureNavigation:
- return "navigation"
- case FailureNetwork:
- return "network"
- default:
- return "unknown"
- }
-}
-
-// Recoverable reports whether the failure type is eligible for automatic
-// self-healing. Network and unknown failures are not recoverable.
-func (f FailureType) Recoverable() bool {
- switch f {
- case FailureElementNotFound,
- FailureElementStale,
- FailureElementNotInteractable,
- FailureNavigation:
- return true
- default:
- return false
- }
-}
-
-// ClassifyFailure inspects an error string and returns the most likely
-// FailureType. The classification is intentionally broad — it matches
-// error messages produced by Chrome DevTools Protocol, chromedp, and
-// PinchTab's own bridge layer so that recovery works regardless of which
-// layer reported the failure.
-func ClassifyFailure(err error) FailureType {
- if err == nil {
- return FailureUnknown
- }
- e := strings.ToLower(err.Error())
-
- // Element not found patterns.
- notFoundPatterns := []string{
- "could not find node",
- "node with given id",
- "no node",
- "ref not found",
- "node not found",
- "backend node id",
- "no node with given",
- }
- for _, p := range notFoundPatterns {
- if strings.Contains(e, p) {
- return FailureElementNotFound
- }
- }
-
- // Stale element / detached DOM patterns.
- // NOTE: "frame" detachment is classified as navigation, not stale.
- stalePatterns := []string{
- "stale",
- "orphan",
- "object reference",
- "node is detached",
- "execution context was destroyed",
- "context was destroyed",
- "target closed",
- }
- for _, p := range stalePatterns {
- if strings.Contains(e, p) {
- return FailureElementStale
- }
- }
- // "detached" alone is stale, but "frame" + "detached" is navigation.
- if strings.Contains(e, "detached") && !strings.Contains(e, "frame") {
- return FailureElementStale
- }
-
- // Not interactable patterns.
- interactablePatterns := []string{
- "not interactable",
- "not clickable",
- "element is not visible",
- "not visible",
- "element is disabled",
- "overlapped",
- "overlapping",
- "obscured",
- "pointer-events: none",
- "cannot focus",
- "outside of the viewport",
- "outside the viewport",
- }
- for _, p := range interactablePatterns {
- if strings.Contains(e, p) {
- return FailureElementNotInteractable
- }
- }
-
- // Navigation patterns.
- navPatterns := []string{
- "navigat",
- "page crashed",
- "page was destroyed",
- "inspected target",
- "frame was detached",
- "frame detached",
- }
- for _, p := range navPatterns {
- if strings.Contains(e, p) {
- return FailureNavigation
- }
- }
-
- // Network patterns.
- networkPatterns := []string{
- "net::",
- "connection refused",
- "could not connect",
- "timeout",
- "deadline exceeded",
- "websocket",
- "eof",
- "broken pipe",
- }
- for _, p := range networkPatterns {
- if strings.Contains(e, p) {
- return FailureNetwork
- }
- }
-
- return FailureUnknown
-}
diff --git a/internal/semantic/hashing_embedder.go b/internal/semantic/hashing_embedder.go
deleted file mode 100644
index 35eaedd0..00000000
--- a/internal/semantic/hashing_embedder.go
+++ /dev/null
@@ -1,171 +0,0 @@
-package semantic
-
-import (
- "hash/fnv"
- "math"
- "strings"
- "unicode"
-)
-
-// HashingEmbedder implements Embedder using a feature-hashing (hashing trick)
-// approach. It produces fixed-dimension vectors by hashing word unigrams and
-// character n-grams into a compact vector space. No vocabulary construction
-// is required, making each Embed call fully independent.
-//
-// Properties:
-// - Fixed vector dimensionality regardless of vocabulary size
-// - Captures sub-word similarity (e.g. "btn" ↔ "button")
-// - L2-normalized output for cosine similarity compatibility
-// - Zero external dependencies — pure Go
-type HashingEmbedder struct {
- dim int // vector dimensionality
- ngramMin int // minimum character n-gram length
- ngramMax int // maximum character n-gram length
- wordWeight float32 // weight factor for word-level features
- ngramWeight float32 // weight factor for n-gram features
-}
-
-// NewHashingEmbedder creates a HashingEmbedder with the given dimension.
-// Higher dimensions reduce hash collisions but use more memory.
-// Recommended: 128 for speed, 256 for accuracy.
-func NewHashingEmbedder(dim int) *HashingEmbedder {
- if dim <= 0 {
- dim = 128
- }
- return &HashingEmbedder{
- dim: dim,
- ngramMin: 2,
- ngramMax: 4,
- wordWeight: 1.0,
- ngramWeight: 0.5,
- }
-}
-
-// Strategy returns "hashing".
-func (h *HashingEmbedder) Strategy() string { return "hashing" }
-
-// Embed converts a batch of texts into hashed feature vectors.
-func (h *HashingEmbedder) Embed(texts []string) ([][]float32, error) {
- result := make([][]float32, len(texts))
- for i, text := range texts {
- result[i] = h.vectorize(text)
- }
- return result, nil
-}
-
-// vectorize converts a single text into a hashed feature vector combining
-// word-level, character n-gram, role-aware, and synonym features.
-func (h *HashingEmbedder) vectorize(text string) []float32 {
- vec := make([]float32, h.dim)
-
- // Normalize text
- text = strings.ToLower(text)
-
- // 1. Word-level features (captures exact word overlap)
- words := tokenizeForEmbedding(text)
- for _, word := range words {
- idx, sign := h.hashFeature("w:" + word)
- vec[idx] += sign * h.wordWeight
- }
-
- // 2. Character n-gram features (captures sub-word similarity)
- // e.g. "button" → "bu", "ut", "tt", "to", "on", "but", "utt", "tto", "ton"
- for _, word := range words {
- padded := "^" + word + "$" // boundary markers
- for n := h.ngramMin; n <= h.ngramMax; n++ {
- for i := 0; i <= len(padded)-n; i++ {
- ngram := padded[i : i+n]
- idx, sign := h.hashFeature("n:" + ngram)
- vec[idx] += sign * h.ngramWeight
- }
- }
- }
-
- // 3. Role-aware features: if a word is a known UI role, add an
- // extra feature to boost role-based matching
- for _, word := range words {
- if roleKeywords[word] {
- idx, sign := h.hashFeature("role:" + word)
- vec[idx] += sign * 0.8
- }
- }
-
- // 4. Synonym features: inject word-level features for known synonyms
- // at a reduced weight so "sign in" and "log in" share vector space.
- for _, word := range words {
- if syns, ok := synonymIndex[word]; ok {
- for syn := range syns {
- synTokens := strings.Fields(syn)
- for _, st := range synTokens {
- idx, sign := h.hashFeature("w:" + st)
- vec[idx] += sign * h.wordWeight * 0.3
- }
- }
- }
- }
-
- // 5. Multi-word synonym phrases: check consecutive word pairs/triples
- // so "look up" → "search" gets injected at the embedding level.
- for n := 2; n <= 3 && n <= len(words); n++ {
- for i := 0; i <= len(words)-n; i++ {
- phrase := strings.Join(words[i:i+n], " ")
- if syns, ok := synonymIndex[phrase]; ok {
- for syn := range syns {
- synTokens := strings.Fields(syn)
- for _, st := range synTokens {
- idx, sign := h.hashFeature("w:" + st)
- vec[idx] += sign * h.wordWeight * 0.3
- }
- }
- }
- }
- }
-
- // L2-normalize for cosine similarity
- h.normalize(vec)
- return vec
-}
-
-// hashFeature hashes a feature string into an index [0, dim) and a sign
-// (+1 or -1). The sign hash preserves inner-product properties (the
-// "signed hashing trick" per Weinberger et al. 2009).
-func (h *HashingEmbedder) hashFeature(feature string) (int, float32) {
- // Index hash
- hasher := fnv.New32a()
- hasher.Write([]byte(feature))
- idx := int(hasher.Sum32()) % h.dim
- if idx < 0 {
- idx = -idx
- }
-
- // Sign hash (use different seed by prepending marker)
- signHasher := fnv.New32()
- signHasher.Write([]byte("s:" + feature))
- sign := float32(1.0)
- if signHasher.Sum32()%2 == 1 {
- sign = -1.0
- }
-
- return idx, sign
-}
-
-// normalize L2-normalizes a vector in-place.
-func (h *HashingEmbedder) normalize(vec []float32) {
- var norm float64
- for _, v := range vec {
- norm += float64(v) * float64(v)
- }
- if norm > 0 {
- invNorm := float32(1.0 / math.Sqrt(norm))
- for i := range vec {
- vec[i] *= invNorm
- }
- }
-}
-
-// tokenizeForEmbedding splits text into lowercase tokens for embedding.
-func tokenizeForEmbedding(s string) []string {
- return strings.FieldsFunc(s, func(r rune) bool {
- return !unicode.IsLetter(r) && !unicode.IsDigit(r)
- })
-}
diff --git a/internal/semantic/improvements_test.go b/internal/semantic/improvements_test.go
deleted file mode 100644
index a02de05c..00000000
--- a/internal/semantic/improvements_test.go
+++ /dev/null
@@ -1,1360 +0,0 @@
-package semantic
-
-import (
- "context"
- "fmt"
- "strings"
- "testing"
-)
-
-// ===========================================================================
-// Synonym Expansion Tests
-// ===========================================================================
-
-func TestSynonymIndex_Bidirectional(t *testing.T) {
- // Every entry in uiSynonyms should be bidirectional.
- for canonical, synonyms := range uiSynonyms {
- for _, syn := range synonyms {
- if syns, ok := synonymIndex[syn]; !ok {
- t.Errorf("synonym %q (of %q) not in synonymIndex", syn, canonical)
- } else if !syns[canonical] {
- t.Errorf("synonymIndex[%q] does not map back to canonical %q", syn, canonical)
- }
- }
- }
-}
-
-func TestSynonymScore_SignInLogIn(t *testing.T) {
- qTokens := tokenize("sign in")
- dTokens := tokenize("Log in")
- score := synonymScore(qTokens, dTokens)
- if score < 0.3 {
- t.Errorf("expected synonym score >= 0.3 for 'sign in' vs 'Log in', got %f", score)
- }
-}
-
-func TestSynonymScore_RegisterCreateAccount(t *testing.T) {
- qTokens := tokenize("register")
- dTokens := tokenize("Create account")
- score := synonymScore(qTokens, dTokens)
- if score < 0.5 {
- t.Errorf("expected synonym score >= 0.5 for 'register' vs 'Create account', got %f", score)
- }
-}
-
-func TestSynonymScore_LookUpSearch(t *testing.T) {
- qTokens := tokenize("look up")
- dTokens := tokenize("Search")
- score := synonymScore(qTokens, dTokens)
- if score < 0.3 {
- t.Errorf("expected synonym score >= 0.3 for 'look up' vs 'Search', got %f", score)
- }
-}
-
-func TestSynonymScore_NavigationMainMenu(t *testing.T) {
- qTokens := tokenize("navigation")
- dTokens := tokenize("Main menu")
- score := synonymScore(qTokens, dTokens)
- if score < 0.3 {
- t.Errorf("expected synonym score >= 0.3 for 'navigation' vs 'Main menu', got %f", score)
- }
-}
-
-func TestSynonymScore_NoRelation(t *testing.T) {
- qTokens := tokenize("elephant")
- dTokens := tokenize("button")
- score := synonymScore(qTokens, dTokens)
- if score > 0.1 {
- t.Errorf("expected near-zero synonym score for unrelated terms, got %f", score)
- }
-}
-
-func TestExpandWithSynonyms_MultiWord(t *testing.T) {
- query := tokenize("sign in")
- desc := tokenize("log in")
- expanded := expandWithSynonyms(query, desc)
- // "sign in" is a synonym for "log in", so expansion should add "log" and "in"
- found := false
- for _, tok := range expanded {
- if tok == "log" {
- found = true
- break
- }
- }
- if !found {
- t.Errorf("expanding 'sign in' against desc 'log in' should add 'log', got: %v", expanded)
- }
-}
-
-func TestBuildPhrases(t *testing.T) {
- tokens := []string{"sign", "in", "button"}
- phrases := buildPhrases(tokens, 3)
- if len(phrases) == 0 {
- t.Fatal("expected at least one phrase")
- }
- found := false
- for _, p := range phrases {
- if p.text == "sign in" {
- found = true
- break
- }
- }
- if !found {
- texts := make([]string, len(phrases))
- for i, p := range phrases {
- texts[i] = p.text
- }
- t.Errorf("expected phrase 'sign in', got: %v", texts)
- }
-}
-
-// ===========================================================================
-// Context-Aware Stopword Tests
-// ===========================================================================
-
-func TestRemoveStopwordsContextAware_PreservesSignIn(t *testing.T) {
- tokens := tokenize("sign in")
- otherTokens := tokenize("log in button")
- filtered := removeStopwordsContextAware(tokens, otherTokens)
- // "in" should be preserved because it's part of "sign in" synonym phrase
- // or because it appears in the other side
- hasIn := false
- for _, tok := range filtered {
- if tok == "in" {
- hasIn = true
- }
- }
- if !hasIn {
- t.Errorf("expected 'in' to be preserved in context-aware removal for 'sign in', got: %v", filtered)
- }
-}
-
-func TestRemoveStopwordsContextAware_RemovesIrrelevantStopwords(t *testing.T) {
- tokens := tokenize("click the submit button")
- otherTokens := tokenize("button Submit")
- filtered := removeStopwordsContextAware(tokens, otherTokens)
- for _, tok := range filtered {
- if tok == "the" {
- t.Errorf("expected 'the' to be removed in context-aware removal, got: %v", filtered)
- }
- }
-}
-
-func TestRemoveStopwordsContextAware_PreservesSemanticStopwordInContext(t *testing.T) {
- // "not" is a semantic stopword — should be preserved if it appears in other set
- tokens := tokenize("not now")
- otherTokens := tokenize("Not now button")
- filtered := removeStopwordsContextAware(tokens, otherTokens)
- hasNot := false
- for _, tok := range filtered {
- if tok == "not" {
- hasNot = true
- }
- }
- if !hasNot {
- t.Errorf("expected 'not' to be preserved when it appears in other tokens, got: %v", filtered)
- }
-}
-
-// ===========================================================================
-// Prefix Matching Tests
-// ===========================================================================
-
-func TestTokenPrefixScore_BtnButton(t *testing.T) {
- // "btn" is NOT a string prefix of "button" (b-t-n vs b-u-t-t-o-n),
- // so prefix matching correctly returns 0. Abbreviation is handled by synonyms.
- qTokens := tokenize("btn")
- dTokens := tokenize("button")
- score := tokenPrefixScore(qTokens, dTokens)
- t.Logf("prefix score for 'btn' -> 'button' = %f (abbreviation handled by synonyms)", score)
- if score > 0.5 {
- t.Errorf("unexpected high prefix score for abbreviation 'btn' -> 'button', got %f", score)
- }
-}
-
-func TestTokenPrefixScore_NavNavigation(t *testing.T) {
- qTokens := tokenize("nav")
- dTokens := tokenize("navigation menu")
- score := tokenPrefixScore(qTokens, dTokens)
- if score < 0.2 {
- t.Errorf("expected prefix score >= 0.2 for 'nav' -> 'navigation', got %f", score)
- }
-}
-
-func TestTokenPrefixScore_NoPrefix(t *testing.T) {
- qTokens := tokenize("elephant")
- dTokens := tokenize("button")
- score := tokenPrefixScore(qTokens, dTokens)
- if score > 0.01 {
- t.Errorf("expected near-zero prefix score for unrelated terms, got %f", score)
- }
-}
-
-// ===========================================================================
-// LexicalScore with Improvements - Real-World Scenarios
-// ===========================================================================
-
-func TestLexicalScore_SignIn_vs_LogIn(t *testing.T) {
- // This was the #1 failing case from the real-world evaluation
- score := LexicalScore("sign in", "link: Log in")
- t.Logf("LexicalScore('sign in', 'link: Log in') = %f", score)
- if score < 0.15 {
- t.Errorf("expected improved score for 'sign in' vs 'Log in', got %f (was 0.207 before improvements)", score)
- }
-}
-
-func TestLexicalScore_Register_vs_CreateAccount(t *testing.T) {
- score := LexicalScore("register", "link: Create account")
- t.Logf("LexicalScore('register', 'link: Create account') = %f", score)
- if score < 0.10 {
- t.Errorf("expected improved score for 'register' vs 'Create account', got %f (was 0.134 before)", score)
- }
-}
-
-func TestLexicalScore_LookUp_vs_Search(t *testing.T) {
- score := LexicalScore("look up", "search: Search")
- t.Logf("LexicalScore('look up', 'search: Search') = %f", score)
- // Lexical-only score is 0.15; combined matcher finds it at 0.215 which is sufficient.
- if score < 0.10 {
- t.Errorf("expected improved score for 'look up' vs 'Search', got %f", score)
- }
-}
-
-func TestLexicalScore_Navigation_vs_MainMenu(t *testing.T) {
- score := LexicalScore("navigation", "menu: Main menu")
- t.Logf("LexicalScore('navigation', 'menu: Main menu') = %f", score)
- if score < 0.15 {
- t.Errorf("expected improved score for 'navigation' vs 'Main menu', got %f (was 0.206 before)", score)
- }
-}
-
-func TestLexicalScore_Download_vs_Export(t *testing.T) {
- score := LexicalScore("download report", "button: Export")
- t.Logf("LexicalScore('download report', 'button: Export') = %f", score)
- if score < 0.10 {
- t.Errorf("expected improved score for 'download' vs 'Export', got %f", score)
- }
-}
-
-func TestLexicalScore_Proceed_vs_PlaceOrder(t *testing.T) {
- score := LexicalScore("proceed to payment", "button: Place order")
- t.Logf("LexicalScore('proceed to payment', 'button: Place order') = %f", score)
- // This is a hard case — "proceed" maps to "next/continue", "payment" maps to checkout family
-}
-
-func TestLexicalScore_Dismiss_vs_Close(t *testing.T) {
- score := LexicalScore("dismiss dialog", "button: Close")
- t.Logf("LexicalScore('dismiss dialog', 'button: Close') = %f", score)
- if score < 0.10 {
- t.Errorf("expected improved score for 'dismiss' vs 'Close', got %f", score)
- }
-}
-
-func TestLexicalScore_PrefixAbbreviation(t *testing.T) {
- score := LexicalScore("btn submit", "button: Submit")
- t.Logf("LexicalScore('btn submit', 'button: Submit') = %f", score)
- if score < 0.3 {
- t.Errorf("expected good score for 'btn submit' vs 'button: Submit', got %f", score)
- }
-}
-
-func TestLexicalScore_StillExactMatch(t *testing.T) {
- score := LexicalScore("submit button", "button: Submit")
- if score < 0.5 {
- t.Errorf("expected high score for exact match after improvements, got %f", score)
- }
-}
-
-func TestLexicalScore_StillRejectsUnrelated(t *testing.T) {
- score := LexicalScore("download pdf", "button: Login")
- if score > 0.35 {
- t.Errorf("expected low score for unrelated query after improvements, got %f", score)
- }
-}
-
-// ===========================================================================
-// Combined Matcher with Improvements - Real-World Evaluation Scenarios
-// ===========================================================================
-
-// buildRealWorldElements creates elements mimicking real website structures
-func buildRealWorldElements() map[string][]ElementDescriptor {
- return map[string][]ElementDescriptor{
- "wikipedia": {
- {Ref: "e1", Role: "search", Name: "Search Wikipedia"},
- {Ref: "e2", Role: "button", Name: "Search"},
- {Ref: "e3", Role: "link", Name: "Main page"},
- {Ref: "e4", Role: "link", Name: "Contents"},
- {Ref: "e5", Role: "link", Name: "Current events"},
- {Ref: "e6", Role: "link", Name: "Random article"},
- {Ref: "e7", Role: "link", Name: "About Wikipedia"},
- {Ref: "e8", Role: "link", Name: "Log in"},
- {Ref: "e9", Role: "link", Name: "Create account"},
- {Ref: "e10", Role: "navigation", Name: "Main menu"},
- {Ref: "e11", Role: "link", Name: "Talk"},
- {Ref: "e12", Role: "link", Name: "Contributions"},
- {Ref: "e13", Role: "heading", Name: "Wikipedia, the free encyclopedia"},
- {Ref: "e14", Role: "link", Name: "(Top)"},
- {Ref: "e15", Role: "link", Name: "Languages"},
- },
- "github_login": {
- {Ref: "e1", Role: "link", Name: "Homepage"},
- {Ref: "e2", Role: "heading", Name: "Sign in to GitHub"},
- {Ref: "e3", Role: "textbox", Name: "Username or email address"},
- {Ref: "e4", Role: "textbox", Name: "Password"},
- {Ref: "e5", Role: "button", Name: "Sign in"},
- {Ref: "e6", Role: "link", Name: "Forgot password?"},
- {Ref: "e7", Role: "link", Name: "Create an account"},
- {Ref: "e8", Role: "link", Name: "Terms"},
- {Ref: "e9", Role: "link", Name: "Privacy"},
- {Ref: "e10", Role: "link", Name: "Docs"},
- {Ref: "e11", Role: "link", Name: "Contact GitHub Support"},
- },
- "google": {
- {Ref: "e1", Role: "combobox", Name: "Search"},
- {Ref: "e2", Role: "button", Name: "Google Search"},
- {Ref: "e3", Role: "button", Name: "I'm Feeling Lucky"},
- {Ref: "e4", Role: "link", Name: "Gmail"},
- {Ref: "e5", Role: "link", Name: "Images"},
- {Ref: "e6", Role: "link", Name: "Sign in"},
- {Ref: "e7", Role: "link", Name: "About"},
- {Ref: "e8", Role: "link", Name: "Store"},
- {Ref: "e9", Role: "link", Name: "Advertising"},
- {Ref: "e10", Role: "link", Name: "Privacy"},
- {Ref: "e11", Role: "link", Name: "Settings"},
- },
- "ecommerce": {
- {Ref: "e1", Role: "search", Name: "Search products"},
- {Ref: "e2", Role: "link", Name: "Home"},
- {Ref: "e3", Role: "link", Name: "Cart"},
- {Ref: "e4", Role: "button", Name: "Add to Cart"},
- {Ref: "e5", Role: "link", Name: "Sign in"},
- {Ref: "e6", Role: "link", Name: "Register"},
- {Ref: "e7", Role: "button", Name: "Buy Now"},
- {Ref: "e8", Role: "button", Name: "Place Order"},
- {Ref: "e9", Role: "link", Name: "Checkout"},
- {Ref: "e10", Role: "button", Name: "Apply Coupon"},
- {Ref: "e11", Role: "textbox", Name: "Quantity"},
- {Ref: "e12", Role: "button", Name: "Export Orders"},
- {Ref: "e13", Role: "navigation", Name: "Main navigation"},
- {Ref: "e14", Role: "link", Name: "My Account"},
- {Ref: "e15", Role: "button", Name: "Close"},
- },
- }
-}
-
-// ===========================================================================
-// CATEGORY 1: Exact Match Tests
-// ===========================================================================
-
-func TestCombined_ExactMatch_Wikipedia(t *testing.T) {
- sites := buildRealWorldElements()
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- tests := []struct {
- query string
- wantRef string
- wantDesc string
- }{
- {"Search Wikipedia", "e1", "Search Wikipedia"},
- {"Log in", "e8", "Log in"},
- {"Create account", "e9", "Create account"},
- {"Main menu", "e10", "Main menu"},
- {"Search button", "e2", "Search"},
- }
-
- for _, tt := range tests {
- t.Run(tt.query, func(t *testing.T) {
- result, err := matcher.Find(context.Background(), tt.query, sites["wikipedia"], FindOptions{
- Threshold: 0.2,
- TopK: 3,
- })
- if err != nil {
- t.Fatalf("Find error: %v", err)
- }
- if result.BestRef != tt.wantRef {
- t.Errorf("query=%q: got BestRef=%s (score=%.3f), want %s (%s)",
- tt.query, result.BestRef, result.BestScore, tt.wantRef, tt.wantDesc)
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%.3f role=%s name=%s", m.Ref, m.Score, m.Role, m.Name)
- }
- }
- })
- }
-}
-
-// ===========================================================================
-// CATEGORY 2: Synonym Tests (the primary weakness)
-// ===========================================================================
-
-func TestCombined_Synonym_SignIn_LogIn(t *testing.T) {
- sites := buildRealWorldElements()
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- // "sign in" should find "Log in" on Wikipedia
- result, err := matcher.Find(context.Background(), "sign in", sites["wikipedia"], FindOptions{
- Threshold: 0.15,
- TopK: 5,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("Query='sign in': BestRef=%s Score=%.3f Confidence=%s", result.BestRef, result.BestScore, result.ConfidenceLabel())
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%.3f role=%s name=%s", m.Ref, m.Score, m.Role, m.Name)
- }
- // After improvements, "sign in" should match "Log in" (e8) with decent score
- if result.BestRef != "e8" {
- t.Errorf("expected 'sign in' to match 'Log in' (e8), got %s", result.BestRef)
- }
-}
-
-func TestCombined_Synonym_Register_CreateAccount(t *testing.T) {
- sites := buildRealWorldElements()
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- result, err := matcher.Find(context.Background(), "register", sites["wikipedia"], FindOptions{
- Threshold: 0.15,
- TopK: 5,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("Query='register': BestRef=%s Score=%.3f", result.BestRef, result.BestScore)
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%.3f role=%s name=%s", m.Ref, m.Score, m.Role, m.Name)
- }
- // "register" should match "Create account" (e9)
- foundInTop3 := false
- for _, m := range result.Matches {
- if m.Ref == "e9" {
- foundInTop3 = true
- break
- }
- }
- if !foundInTop3 {
- t.Errorf("expected 'register' to find 'Create account' (e9) in top matches")
- }
-}
-
-func TestCombined_Synonym_LookUp_Search(t *testing.T) {
- sites := buildRealWorldElements()
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- result, err := matcher.Find(context.Background(), "look up", sites["wikipedia"], FindOptions{
- Threshold: 0.15,
- TopK: 5,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("Query='look up': BestRef=%s Score=%.3f", result.BestRef, result.BestScore)
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%.3f role=%s name=%s", m.Ref, m.Score, m.Role, m.Name)
- }
- // "look up" should match "Search Wikipedia" (e1) or "Search" (e2)
- foundSearch := false
- for _, m := range result.Matches {
- if m.Ref == "e1" || m.Ref == "e2" {
- foundSearch = true
- break
- }
- }
- if !foundSearch {
- t.Errorf("expected 'look up' to find Search element in top matches")
- }
-}
-
-func TestCombined_Synonym_Navigation_MainMenu(t *testing.T) {
- sites := buildRealWorldElements()
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- result, err := matcher.Find(context.Background(), "navigation", sites["wikipedia"], FindOptions{
- Threshold: 0.15,
- TopK: 5,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("Query='navigation': BestRef=%s Score=%.3f", result.BestRef, result.BestScore)
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%.3f role=%s name=%s", m.Ref, m.Score, m.Role, m.Name)
- }
- // "navigation" should match "Main menu" (e10 has role="navigation")
- if result.BestRef != "e10" {
- t.Errorf("expected 'navigation' to match 'Main menu' (e10), got %s", result.BestRef)
- }
-}
-
-func TestCombined_Synonym_Login_SignIn(t *testing.T) {
- sites := buildRealWorldElements()
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- // GitHub login page: "login" should find "Sign in" button
- result, err := matcher.Find(context.Background(), "login", sites["github_login"], FindOptions{
- Threshold: 0.15,
- TopK: 5,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("Query='login' on GitHub: BestRef=%s Score=%.3f", result.BestRef, result.BestScore)
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%.3f role=%s name=%s", m.Ref, m.Score, m.Role, m.Name)
- }
- // Should find "Sign in" button (e5) or heading "Sign in to GitHub" (e2)
- foundSignIn := false
- for _, m := range result.Matches {
- if m.Ref == "e5" || m.Ref == "e2" {
- foundSignIn = true
- break
- }
- }
- if !foundSignIn {
- t.Errorf("expected 'login' to find 'Sign in' element on GitHub login page")
- }
-}
-
-func TestCombined_Synonym_Purchase_Checkout(t *testing.T) {
- sites := buildRealWorldElements()
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- result, err := matcher.Find(context.Background(), "purchase", sites["ecommerce"], FindOptions{
- Threshold: 0.15,
- TopK: 5,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("Query='purchase': BestRef=%s Score=%.3f", result.BestRef, result.BestScore)
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%.3f role=%s name=%s", m.Ref, m.Score, m.Role, m.Name)
- }
- // "purchase" should match Checkout, Buy Now, or Place Order
- foundPurchase := false
- for _, m := range result.Matches {
- if m.Ref == "e7" || m.Ref == "e8" || m.Ref == "e9" {
- foundPurchase = true
- break
- }
- }
- if !foundPurchase {
- t.Errorf("expected 'purchase' to find checkout/buy/order related element")
- }
-}
-
-func TestCombined_Synonym_ProceedToPayment_PlaceOrder(t *testing.T) {
- sites := buildRealWorldElements()
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- result, err := matcher.Find(context.Background(), "proceed to payment", sites["ecommerce"], FindOptions{
- Threshold: 0.15,
- TopK: 5,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("Query='proceed to payment': BestRef=%s Score=%.3f", result.BestRef, result.BestScore)
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%.3f role=%s name=%s", m.Ref, m.Score, m.Role, m.Name)
- }
- if result.BestRef != "e8" {
- t.Fatalf("expected 'proceed to payment' to match 'Place Order' (e8), got %s", result.BestRef)
- }
-}
-
-func TestCombined_Synonym_Dismiss_Close(t *testing.T) {
- sites := buildRealWorldElements()
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- result, err := matcher.Find(context.Background(), "dismiss", sites["ecommerce"], FindOptions{
- Threshold: 0.15,
- TopK: 5,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("Query='dismiss': BestRef=%s Score=%.3f", result.BestRef, result.BestScore)
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%.3f role=%s name=%s", m.Ref, m.Score, m.Role, m.Name)
- }
- // "dismiss" should match "Close" (e15)
- if result.BestRef != "e15" {
- t.Errorf("expected 'dismiss' to match 'Close' (e15), got %s", result.BestRef)
- }
-}
-
-func TestCombined_Synonym_Download_Export(t *testing.T) {
- sites := buildRealWorldElements()
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- result, err := matcher.Find(context.Background(), "download orders", sites["ecommerce"], FindOptions{
- Threshold: 0.15,
- TopK: 5,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("Query='download orders': BestRef=%s Score=%.3f", result.BestRef, result.BestScore)
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%.3f role=%s name=%s", m.Ref, m.Score, m.Role, m.Name)
- }
- // "download orders" should match "Export Orders" (e12)
- if result.BestRef != "e12" {
- t.Errorf("expected 'download orders' to match 'Export Orders' (e12), got %s", result.BestRef)
- }
-}
-
-// ===========================================================================
-// CATEGORY 3: Paraphrase Tests
-// ===========================================================================
-
-func TestCombined_Paraphrase_ForgotPassword(t *testing.T) {
- sites := buildRealWorldElements()
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- result, err := matcher.Find(context.Background(), "reset password", sites["github_login"], FindOptions{
- Threshold: 0.15,
- TopK: 5,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("Query='reset password': BestRef=%s Score=%.3f", result.BestRef, result.BestScore)
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%.3f role=%s name=%s", m.Ref, m.Score, m.Role, m.Name)
- }
- // "reset password" should match "Forgot password?" (e6)
- if result.BestRef != "e6" {
- t.Errorf("expected 'reset password' to match 'Forgot password?' (e6), got %s", result.BestRef)
- }
-}
-
-func TestCombined_Paraphrase_ShoppingBag(t *testing.T) {
- sites := buildRealWorldElements()
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- result, err := matcher.Find(context.Background(), "shopping bag", sites["ecommerce"], FindOptions{
- Threshold: 0.15,
- TopK: 5,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("Query='shopping bag': BestRef=%s Score=%.3f", result.BestRef, result.BestScore)
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%.3f role=%s name=%s", m.Ref, m.Score, m.Role, m.Name)
- }
- // "shopping bag" should match "Cart" (e3) via synonym: cart -> bag
- foundCart := false
- for _, m := range result.Matches {
- if m.Ref == "e3" || m.Ref == "e4" {
- foundCart = true
- break
- }
- }
- if !foundCart {
- t.Errorf("expected 'shopping bag' to find Cart element")
- }
-}
-
-// ===========================================================================
-// CATEGORY 4: Partial Match / Abbreviation Tests
-// ===========================================================================
-
-func TestCombined_Partial_Btn(t *testing.T) {
- elements := []ElementDescriptor{
- {Ref: "e1", Role: "button", Name: "Submit"},
- {Ref: "e2", Role: "link", Name: "Home"},
- {Ref: "e3", Role: "textbox", Name: "Email"},
- }
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- result, err := matcher.Find(context.Background(), "submit btn", elements, FindOptions{
- Threshold: 0.15,
- TopK: 3,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("Query='submit btn': BestRef=%s Score=%.3f", result.BestRef, result.BestScore)
- if result.BestRef != "e1" {
- t.Errorf("expected 'submit btn' to match 'Submit' button (e1), got %s", result.BestRef)
- }
-}
-
-func TestCombined_Partial_Nav(t *testing.T) {
- sites := buildRealWorldElements()
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- result, err := matcher.Find(context.Background(), "nav menu", sites["ecommerce"], FindOptions{
- Threshold: 0.15,
- TopK: 5,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("Query='nav menu': BestRef=%s Score=%.3f", result.BestRef, result.BestScore)
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%.3f role=%s name=%s", m.Ref, m.Score, m.Role, m.Name)
- }
- // "nav menu" should match "Main navigation" (e13) via prefix + synonym
- if result.BestRef != "e13" {
- t.Errorf("expected 'nav menu' to match 'Main navigation' (e13), got %s", result.BestRef)
- }
-}
-
-func TestCombined_Partial_Qty(t *testing.T) {
- sites := buildRealWorldElements()
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- result, err := matcher.Find(context.Background(), "qty", sites["ecommerce"], FindOptions{
- Threshold: 0.15,
- TopK: 5,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("Query='qty': BestRef=%s Score=%.3f", result.BestRef, result.BestScore)
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%.3f role=%s name=%s", m.Ref, m.Score, m.Role, m.Name)
- }
- // "qty" should match "Quantity" (e11) via synonym
- if result.BestRef != "e11" {
- t.Errorf("expected 'qty' to match 'Quantity' (e11), got %s", result.BestRef)
- }
-}
-
-// ===========================================================================
-// CATEGORY 5: Edge Cases
-// ===========================================================================
-
-func TestCombined_EdgeCase_EmptyQuery(t *testing.T) {
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
- elements := []ElementDescriptor{{Ref: "e1", Role: "button", Name: "Submit"}}
-
- result, err := matcher.Find(context.Background(), "", elements, FindOptions{
- Threshold: 0.1,
- TopK: 3,
- })
- if err != nil {
- t.Fatal(err)
- }
- if len(result.Matches) > 0 {
- t.Errorf("expected no matches for empty query, got %d", len(result.Matches))
- }
-}
-
-func TestCombined_EdgeCase_GibberishQuery(t *testing.T) {
- sites := buildRealWorldElements()
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- result, err := matcher.Find(context.Background(), "xyzzy plugh qwerty", sites["wikipedia"], FindOptions{
- Threshold: 0.3,
- TopK: 3,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("Gibberish query: matches=%d best_score=%.3f", len(result.Matches), result.BestScore)
- // Should return no matches at threshold 0.3
- if len(result.Matches) > 0 {
- t.Errorf("expected no matches for gibberish query at threshold 0.3, got %d", len(result.Matches))
- }
-}
-
-func TestCombined_EdgeCase_AllStopwords(t *testing.T) {
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
- elements := []ElementDescriptor{
- {Ref: "e1", Role: "button", Name: "Submit"},
- {Ref: "e2", Role: "link", Name: "The"},
- }
-
- result, err := matcher.Find(context.Background(), "the a is", elements, FindOptions{
- Threshold: 0.1,
- TopK: 3,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("All-stopwords query: matches=%d", len(result.Matches))
-}
-
-func TestCombined_EdgeCase_VeryLongQuery(t *testing.T) {
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
- elements := []ElementDescriptor{
- {Ref: "e1", Role: "button", Name: "Submit"},
- }
-
- longQuery := "I want to find the submit button that is located on the bottom right of the page and click on it to submit the form"
- result, err := matcher.Find(context.Background(), longQuery, elements, FindOptions{
- Threshold: 0.1,
- TopK: 3,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("Long query: matches=%d best_score=%.3f", len(result.Matches), result.BestScore)
- if result.BestRef != "e1" {
- t.Errorf("expected long query to still find 'Submit' button, got %s", result.BestRef)
- }
-}
-
-func TestCombined_EdgeCase_SingleCharQuery(t *testing.T) {
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
- elements := []ElementDescriptor{
- {Ref: "e1", Role: "link", Name: "X"},
- {Ref: "e2", Role: "button", Name: "Close"},
- }
-
- result, err := matcher.Find(context.Background(), "x", elements, FindOptions{
- Threshold: 0.1,
- TopK: 3,
- })
- if err != nil {
- t.Fatal(err)
- }
- t.Logf("Single char 'x': BestRef=%s Score=%.3f", result.BestRef, result.BestScore)
-}
-
-// ===========================================================================
-// CATEGORY 6: Role Boost Accumulation Test (Bug Fix)
-// ===========================================================================
-
-func TestLexicalScore_MultipleRoleKeywordsAccumulate(t *testing.T) {
- // "search input" has two role keywords: "search" and "input"
- // Should get cumulative boost, not just one
- scoreMulti := LexicalScore("search input", "search: Email Input")
- scoreSingle := LexicalScore("search something", "search: Email Input")
-
- t.Logf("Multi-role score: %f, Single-role score: %f", scoreMulti, scoreSingle)
- if scoreMulti <= scoreSingle {
- t.Errorf("expected multi-role query to score higher than single-role, got multi=%f single=%f", scoreMulti, scoreSingle)
- }
-}
-
-// ===========================================================================
-// COMPREHENSIVE EVALUATION - Reproduces the exact tests from the issue
-// ===========================================================================
-
-func TestComprehensiveEvaluation(t *testing.T) {
- sites := buildRealWorldElements()
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- type evalCase struct {
- category string
- query string
- site string
- wantRef string
- wantName string
- }
-
- cases := []evalCase{
- // Exact matches
- {"exact", "Search Wikipedia", "wikipedia", "e1", "Search Wikipedia"},
- {"exact", "Log in", "wikipedia", "e8", "Log in"},
- {"exact", "Create account", "wikipedia", "e9", "Create account"},
- {"exact", "Sign in", "github_login", "e5", "Sign in"},
- {"exact", "Google Search", "google", "e2", "Google Search"},
-
- // Synonyms (the main weakness)
- {"synonym", "sign in", "wikipedia", "e8", "Log in"},
- {"synonym", "register", "wikipedia", "e9", "Create account"},
- {"synonym", "look up", "wikipedia", "e1", "Search Wikipedia"},
- {"synonym", "navigation", "wikipedia", "e10", "Main menu"},
- {"synonym", "login button", "github_login", "e5", "Sign in"},
- {"synonym", "authenticate", "github_login", "e5", "Sign in"},
- {"synonym", "dismiss", "ecommerce", "e15", "Close"},
- {"synonym", "download orders", "ecommerce", "e12", "Export Orders"},
-
- // Paraphrases
- {"paraphrase", "reset password", "github_login", "e6", "Forgot password?"},
- {"paraphrase", "email field", "github_login", "e3", "Username or email address"},
-
- // Partial / abbreviations
- {"partial", "qty", "ecommerce", "e11", "Quantity"},
- }
-
- results := make(map[string][]bool)
- var totalPass, totalFail int
-
- for _, tc := range cases {
- t.Run(fmt.Sprintf("%s/%s", tc.category, tc.query), func(t *testing.T) {
- result, err := matcher.Find(context.Background(), tc.query, sites[tc.site], FindOptions{
- Threshold: 0.1,
- TopK: 5,
- })
- if err != nil {
- t.Fatal(err)
- }
-
- pass := false
- for i, m := range result.Matches {
- if i >= 3 {
- break
- }
- if m.Ref == tc.wantRef {
- pass = true
- break
- }
- }
-
- results[tc.category] = append(results[tc.category], pass)
- if pass {
- totalPass++
- t.Logf("PASS: query=%q -> %s (score=%.3f)", tc.query, tc.wantName, result.BestScore)
- } else {
- totalFail++
- t.Logf("MISS: query=%q wanted %s (%s), got BestRef=%s (score=%.3f)",
- tc.query, tc.wantRef, tc.wantName, result.BestRef, result.BestScore)
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%.3f name=%s", m.Ref, m.Score, m.Name)
- }
- }
- })
- }
-
- // Summary
- t.Logf("\n=== EVALUATION SUMMARY ===")
- t.Logf("Total: %d/%d (%.1f%%)", totalPass, totalPass+totalFail, 100*float64(totalPass)/float64(totalPass+totalFail))
- for cat, res := range results {
- passed := 0
- for _, r := range res {
- if r {
- passed++
- }
- }
- t.Logf(" %s: %d/%d (%.0f%%)", cat, passed, len(res), 100*float64(passed)/float64(len(res)))
- }
-}
-
-// ===========================================================================
-// Hashing Embedder Synonym Feature Tests
-// ===========================================================================
-
-func TestHashingEmbedder_SynonymVectorsCloser(t *testing.T) {
- embedder := NewHashingEmbedder(128)
-
- vecs, err := embedder.Embed([]string{"sign in", "log in", "elephant"})
- if err != nil {
- t.Fatal(err)
- }
-
- // Cosine similarity between "sign in" and "log in" should be higher
- // than between "sign in" and "elephant"
- simSynonym := cosineSim(vecs[0], vecs[1])
- simUnrelated := cosineSim(vecs[0], vecs[2])
-
- t.Logf("sim('sign in', 'log in') = %.4f", simSynonym)
- t.Logf("sim('sign in', 'elephant') = %.4f", simUnrelated)
-
- if simSynonym <= simUnrelated {
- t.Errorf("expected synonym embedding similarity (%.4f) > unrelated (%.4f)", simSynonym, simUnrelated)
- }
-}
-
-func TestHashingEmbedder_AbbrVectorsCloser(t *testing.T) {
- embedder := NewHashingEmbedder(128)
-
- vecs, err := embedder.Embed([]string{"btn", "button", "elephant"})
- if err != nil {
- t.Fatal(err)
- }
-
- simAbbr := cosineSim(vecs[0], vecs[1])
- simUnrelated := cosineSim(vecs[0], vecs[2])
-
- t.Logf("sim('btn', 'button') = %.4f", simAbbr)
- t.Logf("sim('btn', 'elephant') = %.4f", simUnrelated)
-
- if simAbbr <= simUnrelated {
- t.Errorf("expected abbreviation embedding similarity (%.4f) > unrelated (%.4f)", simAbbr, simUnrelated)
- }
-}
-
-// cosineSim computes cosine similarity between two float32 vectors.
-func cosineSim(a, b []float32) float64 {
- var dot, normA, normB float64
- for i := range a {
- dot += float64(a[i]) * float64(b[i])
- normA += float64(a[i]) * float64(a[i])
- normB += float64(b[i]) * float64(b[i])
- }
- if normA == 0 || normB == 0 {
- return 0
- }
- return dot / (sqrt64(normA) * sqrt64(normB))
-}
-
-// ===========================================================================
-// Score Distribution Analysis Test
-// ===========================================================================
-
-func TestScoreDistribution_BeforeVsExpected(t *testing.T) {
- // This test documents the expected improvement in scores
- // for the queries that were failing in the real-world evaluation
- type scoreCase struct {
- query string
- desc string
- minScore float64
- label string
- }
-
- cases := []scoreCase{
- // These had very low scores before (from the issue)
- {"sign in", "link: Log in", 0.15, "synonym: sign in -> Log in"},
- {"register", "link: Create account", 0.10, "synonym: register -> Create account"},
- {"look up", "search: Search", 0.10, "synonym: look up -> Search"},
- {"navigation", "navigation: Main menu", 0.15, "synonym: navigation -> Main menu"},
- {"login button", "button: Sign in", 0.15, "synonym: login -> Sign in"},
- {"dismiss", "button: Close", 0.10, "synonym: dismiss -> Close"},
- {"download", "button: Export", 0.10, "synonym: download -> Export"},
-
- // Prefix/abbreviation cases
- {"btn submit", "button: Submit", 0.30, "prefix: btn -> button"},
- {"nav", "navigation: Main navigation", 0.15, "prefix: nav -> navigation"},
-
- // These should still work well (exact matches)
- {"submit button", "button: Submit", 0.50, "exact: submit button"},
- {"search box", "search: Search", 0.30, "exact: search"},
- {"email input", "textbox: Email", 0.20, "exact-ish: email input"},
- }
-
- for _, tc := range cases {
- t.Run(tc.label, func(t *testing.T) {
- score := LexicalScore(tc.query, tc.desc)
- status := "PASS"
- if score < tc.minScore {
- status = "FAIL"
- }
- t.Logf("[%s] LexicalScore(%q, %q) = %.4f (min: %.2f)", status, tc.query, tc.desc, score, tc.minScore)
- if score < tc.minScore {
- t.Errorf("score %.4f below minimum %.2f", score, tc.minScore)
- }
- })
- }
-}
-
-// ===========================================================================
-// Stopword Edge Cases
-// ===========================================================================
-
-func TestStopword_InPreservedInSignIn(t *testing.T) {
- // "in" should NOT be removed from "sign in" because it forms a synonym phrase
- q := tokenize("sign in button")
- d := tokenize("button Log in")
- filtered := removeStopwordsContextAware(q, d)
-
- hasIn := false
- for _, tok := range filtered {
- if tok == "in" {
- hasIn = true
- }
- }
- if !hasIn {
- t.Errorf("'in' should be preserved in 'sign in' context, filtered=%v", filtered)
- }
-}
-
-func TestStopword_UpPreservedInSignUp(t *testing.T) {
- q := tokenize("sign up now")
- d := tokenize("Register button")
- filtered := removeStopwordsContextAware(q, d)
-
- hasUp := false
- for _, tok := range filtered {
- if tok == "up" {
- hasUp = true
- }
- }
- if !hasUp {
- t.Errorf("'up' should be preserved in 'sign up' context, filtered=%v", filtered)
- }
-}
-
-func TestStopword_NotPreservedInNotNow(t *testing.T) {
- q := tokenize("not now")
- d := tokenize("button Not now")
- filtered := removeStopwordsContextAware(q, d)
-
- hasNot := false
- for _, tok := range filtered {
- if tok == "not" {
- hasNot = true
- }
- }
- if !hasNot {
- t.Errorf("'not' should be preserved when it appears in other tokens, filtered=%v", filtered)
- }
-}
-
-// ===========================================================================
-// Benchmark: Synonym Expansion Overhead
-// ===========================================================================
-
-func BenchmarkSynonymScore(b *testing.B) {
- qTokens := tokenize("sign in button")
- dTokens := tokenize("button Log in")
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- synonymScore(qTokens, dTokens)
- }
-}
-
-func BenchmarkExpandWithSynonyms(b *testing.B) {
- qTokens := tokenize("register now")
- dTokens := tokenize("link Create account")
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- expandWithSynonyms(qTokens, dTokens)
- }
-}
-
-func BenchmarkLexicalScore_WithSynonyms(b *testing.B) {
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- LexicalScore("sign in button", "button: Log in")
- }
-}
-
-func BenchmarkLexicalScore_ExactMatch(b *testing.B) {
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- LexicalScore("submit button", "button: Submit")
- }
-}
-
-func BenchmarkCombinedMatcher_SynonymQuery(b *testing.B) {
- elements := buildRealWorldElements()["wikipedia"]
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
- ctx := context.Background()
- opts := FindOptions{Threshold: 0.15, TopK: 3}
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- result, err := matcher.Find(ctx, "sign in", elements, opts)
- if err != nil {
- b.Fatalf("Find error: %v", err)
- }
- _ = result
- }
-}
-
-// ===========================================================================
-// Multi-Site Comprehensive Evaluation (scoring table)
-// ===========================================================================
-
-func TestMultiSiteEvaluation(t *testing.T) {
- sites := buildRealWorldElements()
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- type testCase struct {
- category string
- query string
- site string
- wantRef string
- wantName string
- }
-
- allCases := []testCase{
- // === EXACT MATCHES ===
- {"exact", "Search Wikipedia", "wikipedia", "e1", "Search Wikipedia"},
- {"exact", "Log in", "wikipedia", "e8", "Log in"},
- {"exact", "Create account", "wikipedia", "e9", "Create account"},
- {"exact", "Sign in", "github_login", "e5", "Sign in"},
- {"exact", "Password", "github_login", "e4", "Password"},
- {"exact", "Google Search", "google", "e2", "Google Search"},
- {"exact", "Cart", "ecommerce", "e3", "Cart"},
- {"exact", "Add to Cart", "ecommerce", "e4", "Add to Cart"},
-
- // === SYNONYMS ===
- {"synonym", "sign in", "wikipedia", "e8", "Log in"},
- {"synonym", "register", "wikipedia", "e9", "Create account"},
- {"synonym", "look up", "wikipedia", "e1", "Search Wikipedia"},
- {"synonym", "navigation", "wikipedia", "e10", "Main menu"},
- {"synonym", "login", "github_login", "e5", "Sign in"},
- {"synonym", "authenticate", "github_login", "e5", "Sign in"},
- {"synonym", "dismiss", "ecommerce", "e15", "Close"},
- {"synonym", "download orders", "ecommerce", "e12", "Export Orders"},
- {"synonym", "purchase", "ecommerce", "e7", "Buy Now"},
-
- // === PARAPHRASES ===
- {"paraphrase", "reset password", "github_login", "e6", "Forgot password?"},
- {"paraphrase", "search input", "google", "e1", "Search"},
- {"paraphrase", "email field", "github_login", "e3", "Username or email address"},
- {"paraphrase", "shopping bag", "ecommerce", "e3", "Cart"},
-
- // === PARTIAL/ABBREVIATIONS ===
- {"partial", "qty", "ecommerce", "e11", "Quantity"},
- {"partial", "nav menu", "ecommerce", "e13", "Main navigation"},
-
- // === EDGE CASES ===
- {"edge", "top right login link", "wikipedia", "e8", "Log in"},
- }
-
- catResults := make(map[string]struct{ pass, total int })
-
- for _, tc := range allCases {
- t.Run(fmt.Sprintf("%s_%s_%s", tc.site, tc.category, strings.ReplaceAll(tc.query, " ", "_")), func(t *testing.T) {
- result, err := matcher.Find(context.Background(), tc.query, sites[tc.site], FindOptions{
- Threshold: 0.1,
- TopK: 5,
- })
- if err != nil {
- t.Fatal(err)
- }
-
- pass := false
- for i, m := range result.Matches {
- if i >= 3 {
- break
- }
- if m.Ref == tc.wantRef {
- pass = true
- break
- }
- }
-
- cr := catResults[tc.category]
- cr.total++
- if pass {
- cr.pass++
- t.Logf("query=%q -> %s score=%.3f", tc.query, tc.wantName, result.BestScore)
- } else {
- t.Logf("MISS query=%q wanted=%s (%s) got=%s score=%.3f",
- tc.query, tc.wantRef, tc.wantName, result.BestRef, result.BestScore)
- for _, m := range result.Matches {
- t.Logf(" ref=%s score=%.3f name=%s", m.Ref, m.Score, m.Name)
- }
- }
- catResults[tc.category] = cr
- })
- }
-
- // Print summary table
- t.Logf("\n╔══════════════════════════════════════════════════╗")
- t.Logf("║ MULTI-SITE EVALUATION SUMMARY ║")
- t.Logf("╠══════════════════════════════════════════════════╣")
- totalP, totalT := 0, 0
- for _, cat := range []string{"exact", "synonym", "paraphrase", "partial", "edge"} {
- cr := catResults[cat]
- pct := 0.0
- if cr.total > 0 {
- pct = 100 * float64(cr.pass) / float64(cr.total)
- }
- t.Logf("║ %-14s %d/%d (%.0f%%) ║", cat, cr.pass, cr.total, pct)
- totalP += cr.pass
- totalT += cr.total
- }
- pct := 0.0
- if totalT > 0 {
- pct = 100 * float64(totalP) / float64(totalT)
- }
- t.Logf("╠══════════════════════════════════════════════════╣")
- t.Logf("║ TOTAL %d/%d (%.0f%%) ║", totalP, totalT, pct)
- t.Logf("╚══════════════════════════════════════════════════╝")
-}
-
-// ---------------------------------------------------------------------------
-// Round-2 bug-fix tests
-// ---------------------------------------------------------------------------
-
-func TestStopword_OnPreservedInLogOn(t *testing.T) {
- query := removeStopwordsContextAware(
- tokenize("log on"),
- tokenize("button: Sign in [login]"),
- )
- found := false
- for _, tok := range query {
- if tok == "on" {
- found = true
- }
- }
- if !found {
- t.Errorf("'on' should be preserved in 'log on' context, got %v", query)
- }
-}
-
-func TestSynonymScore_NoDuplicateCounting(t *testing.T) {
- // "sign in" should match the phrase "sign in" in synonymIndex.
- // The score should NOT double-count "sign" and "in" individually
- // on top of the phrase match.
- score := synonymScore(
- []string{"sign", "in"},
- []string{"login", "button"},
- )
- // Phrase "sign in" → synonym "login" → present in desc → 1 match.
- // len(queryTokens) = 2, but only 1 phrase matched (both indices consumed).
- // Score = 1/2 = 0.5.
- if score > 0.55 {
- t.Errorf("synonymScore should not double-count phrase components, got %.3f", score)
- }
- if score < 0.45 {
- t.Errorf("synonymScore should recognise 'sign in' vs 'login', got %.3f", score)
- }
-}
-
-func TestExpandWithSynonyms_NoDuplicateTokens(t *testing.T) {
- expanded := expandWithSynonyms(
- []string{"sign", "in"},
- []string{"login", "button"},
- )
- seen := make(map[string]int)
- for _, tok := range expanded {
- seen[tok]++
- }
- for tok, cnt := range seen {
- if cnt > 1 {
- t.Errorf("token %q appears %d times in expanded set, expected at most 1", tok, cnt)
- }
- }
-}
-
-func TestSynonymIndex_LogOnBidirectional(t *testing.T) {
- // "log on" should map to login-family and vice versa.
- syns, ok := synonymIndex["log on"]
- if !ok {
- t.Fatal("synonymIndex should contain 'log on'")
- }
- if _, has := syns["login"]; !has {
- t.Error("'log on' should map to 'login'")
- }
-
- loginSyns, ok := synonymIndex["login"]
- if !ok {
- t.Fatal("synonymIndex should contain 'login'")
- }
- if _, has := loginSyns["log on"]; !has {
- t.Error("'login' should map back to 'log on'")
- }
-}
-
-func TestEmbedder_PhraseAwareSynonymInjection(t *testing.T) {
- emb := NewHashingEmbedder(256)
-
- // "look up bar" and "search bar" should be closer than
- // "look up bar" and "weather bar" because "look up" → "search" synonym.
- vecs, err := emb.Embed([]string{
- "textbox: Look up bar",
- "textbox: Search bar",
- "textbox: Weather bar",
- })
- if err != nil {
- t.Fatalf("Embed failed: %v", err)
- }
- lookUp, search, weather := vecs[0], vecs[1], vecs[2]
-
- simSyn := cosineSim(lookUp, search)
- simUnrelated := cosineSim(lookUp, weather)
-
- if simSyn <= simUnrelated {
- t.Errorf("phrase-aware synonym injection should make 'look up' closer to 'search' "+
- "(got syn=%.4f, unrelated=%.4f)", simSyn, simUnrelated)
- }
-}
-
-func TestLexicalScore_LogOn_vs_SignIn(t *testing.T) {
- desc := "link: Sign in"
- score := LexicalScore("log on", desc)
- if score < 0.10 {
- t.Errorf("'log on' vs '%s' should have meaningful score, got %.4f", desc, score)
- }
-}
diff --git a/internal/semantic/intent_cache.go b/internal/semantic/intent_cache.go
deleted file mode 100644
index 6061f384..00000000
--- a/internal/semantic/intent_cache.go
+++ /dev/null
@@ -1,123 +0,0 @@
-package semantic
-
-import (
- "sync"
- "time"
-)
-
-// IntentEntry captures the semantic identity of an element at the time an
-// action was requested. When the element's ref becomes stale, the
-// recovery engine uses this cached intent to build a semantic search
-// query against the fresh snapshot.
-type IntentEntry struct {
- // Query is the original natural-language query (if the action was
- // preceded by /find). Otherwise empty.
- Query string
-
- // Descriptor holds role, name, and value of the element at
- // action time.
- Descriptor ElementDescriptor
-
- // Score and Confidence from the last /find (if available).
- Score float64
- Confidence string
- Strategy string
-
- // CachedAt is the wall-clock time the entry was stored.
- CachedAt time.Time
-}
-
-// IntentCache is a thread-safe, per-tab LRU cache of element intents.
-// Key hierarchy: tabID → ref → IntentEntry.
-type IntentCache struct {
- mu sync.RWMutex
- tabs map[string]map[string]IntentEntry
- maxRefs int // max entries per tab
- ttl time.Duration // entry expiry
-}
-
-// NewIntentCache returns an IntentCache that evicts entries older than
-// ttl and limits each tab to maxRefs entries.
-func NewIntentCache(maxRefs int, ttl time.Duration) *IntentCache {
- if maxRefs <= 0 {
- maxRefs = 200
- }
- if ttl <= 0 {
- ttl = 10 * time.Minute
- }
- return &IntentCache{
- tabs: make(map[string]map[string]IntentEntry),
- maxRefs: maxRefs,
- ttl: ttl,
- }
-}
-
-// Store records (or updates) the intent for a given (tabID, ref) pair.
-func (c *IntentCache) Store(tabID, ref string, entry IntentEntry) {
- c.mu.Lock()
- defer c.mu.Unlock()
-
- if entry.CachedAt.IsZero() {
- entry.CachedAt = time.Now()
- }
-
- tab, ok := c.tabs[tabID]
- if !ok {
- tab = make(map[string]IntentEntry)
- c.tabs[tabID] = tab
- }
-
- // Evict if at capacity.
- if len(tab) >= c.maxRefs {
- oldest := ""
- var oldestT time.Time
- for r, e := range tab {
- if oldest == "" || e.CachedAt.Before(oldestT) {
- oldest = r
- oldestT = e.CachedAt
- }
- }
- if oldest != "" {
- delete(tab, oldest)
- }
- }
-
- tab[ref] = entry
-}
-
-// Lookup returns the cached intent for the ref, or (IntentEntry{}, false).
-func (c *IntentCache) Lookup(tabID, ref string) (IntentEntry, bool) {
- c.mu.RLock()
- defer c.mu.RUnlock()
-
- tab, ok := c.tabs[tabID]
- if !ok {
- return IntentEntry{}, false
- }
- entry, ok := tab[ref]
- if !ok {
- return IntentEntry{}, false
- }
- if time.Since(entry.CachedAt) > c.ttl {
- return IntentEntry{}, false
- }
- return entry, true
-}
-
-// InvalidateTab removes all cached intents for a tab.
-func (c *IntentCache) InvalidateTab(tabID string) {
- c.mu.Lock()
- defer c.mu.Unlock()
- delete(c.tabs, tabID)
-}
-
-// Size returns the total number of cached entries across all tabs.
-func (c *IntentCache) Size() int {
- c.mu.RLock()
- defer c.mu.RUnlock()
- n := 0
- for _, tab := range c.tabs {
- n += len(tab)
- }
- return n
-}
diff --git a/internal/semantic/lexical_matcher.go b/internal/semantic/lexical_matcher.go
deleted file mode 100644
index 4bf90948..00000000
--- a/internal/semantic/lexical_matcher.go
+++ /dev/null
@@ -1,258 +0,0 @@
-package semantic
-
-import (
- "context"
- "sort"
- "strings"
- "unicode"
-)
-
-const (
- // roleBoostPerMatch is added per overlapping role keyword (capped).
- roleBoostPerMatch = 0.12
- // roleBoostCap prevents role boost from dominating the score.
- roleBoostCap = 0.25
- // synonymBoostWeight controls how much synonym matches contribute.
- synonymBoostWeight = 0.30
- // prefixMatchWeight controls how much prefix matches contribute.
- prefixMatchWeight = 0.20
-)
-
-// LexicalMatcher implements ElementMatcher using Jaccard similarity
-// with stopword removal, token frequency weighting, and role-aware boosting.
-// Zero external dependencies.
-type LexicalMatcher struct{}
-
-// NewLexicalMatcher creates a new LexicalMatcher.
-func NewLexicalMatcher() *LexicalMatcher {
- return &LexicalMatcher{}
-}
-
-// Strategy returns "lexical".
-func (m *LexicalMatcher) Strategy() string { return "lexical" }
-
-// Find scores all elements against the query using lexical similarity,
-// filters by threshold, sorts descending, and returns the top-K matches.
-func (m *LexicalMatcher) Find(_ context.Context, query string, elements []ElementDescriptor, opts FindOptions) (FindResult, error) {
- if opts.TopK <= 0 {
- opts.TopK = 3
- }
-
- type scored struct {
- desc ElementDescriptor
- score float64
- }
-
- var candidates []scored
- for _, el := range elements {
- composite := el.Composite()
- score := LexicalScore(query, composite)
- if score >= opts.Threshold {
- candidates = append(candidates, scored{desc: el, score: score})
- }
- }
-
- sort.Slice(candidates, func(i, j int) bool {
- return candidates[i].score > candidates[j].score
- })
-
- if len(candidates) > opts.TopK {
- candidates = candidates[:opts.TopK]
- }
-
- result := FindResult{
- Strategy: "lexical",
- ElementCount: len(elements),
- }
-
- for _, c := range candidates {
- result.Matches = append(result.Matches, ElementMatch{
- Ref: c.desc.Ref,
- Score: c.score,
- Role: c.desc.Role,
- Name: c.desc.Name,
- })
- }
-
- if len(result.Matches) > 0 {
- result.BestRef = result.Matches[0].Ref
- result.BestScore = result.Matches[0].Score
- }
-
- return result, nil
-}
-
-// --- lexical scoring internals ---
-
-// tokenize splits a string into lowercase tokens, removing punctuation.
-func tokenize(s string) []string {
- s = strings.ToLower(s)
- return strings.FieldsFunc(s, func(r rune) bool {
- return !unicode.IsLetter(r) && !unicode.IsDigit(r)
- })
-}
-
-// tokenFreq returns token → count map.
-func tokenFreq(tokens []string) map[string]int {
- m := make(map[string]int, len(tokens))
- for _, t := range tokens {
- m[t]++
- }
- return m
-}
-
-// tokenSet converts a slice of tokens to a set (map).
-func tokenSet(tokens []string) map[string]bool {
- m := make(map[string]bool, len(tokens))
- for _, t := range tokens {
- m[t] = true
- }
- return m
-}
-
-// roleKeywords are element roles that carry strong semantic signal.
-var roleKeywords = map[string]bool{
- "button": true,
- "input": true,
- "link": true,
- "submit": true,
- "form": true,
- "textbox": true,
- "checkbox": true,
- "radio": true,
- "select": true,
- "option": true,
- "tab": true,
- "menu": true,
- "search": true,
-}
-
-// LexicalScore computes a similarity between a query and an element
-// description using Jaccard overlap on tokens with:
-// - lowercase normalization
-// - context-aware stopword removal
-// - token frequency weighting (repeated tokens count proportionally)
-// - role keyword boost (cumulative, capped at roleBoostCap)
-// - synonym expansion (lightweight static synonym table)
-// - prefix matching for abbreviations (e.g. "btn" → "button")
-// - token importance weighting (rarer tokens score higher)
-//
-// Returns a value in [0, 1].
-func LexicalScore(query, desc string) float64 {
- rawQTokens := tokenize(query)
- rawDTokens := tokenize(desc)
-
- qTokens := removeStopwordsContextAware(rawQTokens, rawDTokens)
- dTokens := removeStopwordsContextAware(rawDTokens, rawQTokens)
-
- if len(qTokens) == 0 || len(dTokens) == 0 {
- return 0
- }
-
- // --- 1. Base Jaccard with frequency weighting ---
- qFreq := tokenFreq(qTokens)
- dFreq := tokenFreq(dTokens)
-
- var intersectW float64
- for t, qc := range qFreq {
- if dc, ok := dFreq[t]; ok {
- minC := qc
- if dc < minC {
- minC = dc
- }
- intersectW += float64(minC)
- }
- }
-
- allTokens := tokenSet(append(qTokens, dTokens...))
- var unionW float64
- for t := range allTokens {
- qc := qFreq[t]
- dc := dFreq[t]
- maxC := qc
- if dc > maxC {
- maxC = dc
- }
- unionW += float64(maxC)
- }
-
- if unionW == 0 {
- return 0
- }
-
- jaccard := intersectW / unionW
-
- // --- 2. Synonym boost ---
- synScore := synonymScore(qTokens, dTokens) * synonymBoostWeight
-
- // --- 3. Prefix matching boost ---
- prefixScore := tokenPrefixScore(qTokens, dTokens) * prefixMatchWeight
-
- // --- 4. Role keyword boost (cumulative, capped) ---
- roleBoost := 0.0
- qSet := tokenSet(qTokens)
- dSet := tokenSet(dTokens)
- for t := range qSet {
- if roleKeywords[t] && dSet[t] {
- roleBoost += roleBoostPerMatch
- }
- }
- for t := range qSet {
- if roleKeywords[t] {
- continue // already checked direct match
- }
- if syns, ok := synonymIndex[t]; ok {
- for syn := range syns {
- if roleKeywords[syn] && dSet[syn] {
- roleBoost += roleBoostPerMatch * 0.8
- break
- }
- }
- }
- }
- if roleBoost > roleBoostCap {
- roleBoost = roleBoostCap
- }
-
- score := jaccard + synScore + prefixScore + roleBoost
- if score > 1.0 {
- score = 1.0
- }
- return score
-}
-
-// tokenPrefixScore scores prefix overlap between query and description
-// tokens. Handles abbreviations like "btn" → "button", "nav" → "navigation".
-func tokenPrefixScore(qTokens, dTokens []string) float64 {
- if len(qTokens) == 0 {
- return 0
- }
-
- var total float64
- for _, qt := range qTokens {
- if len(qt) < 2 {
- continue
- }
- bestMatch := 0.0
- for _, dt := range dTokens {
- if qt == dt {
- continue // already counted by Jaccard
- }
- if len(dt) > len(qt) && strings.HasPrefix(dt, qt) {
- ratio := float64(len(qt)) / float64(len(dt))
- if ratio > bestMatch {
- bestMatch = ratio
- }
- }
- if len(qt) > len(dt) && strings.HasPrefix(qt, dt) {
- ratio := float64(len(dt)) / float64(len(qt))
- if ratio*0.7 > bestMatch { // penalize reverse prefix slightly
- bestMatch = ratio * 0.7
- }
- }
- }
- total += bestMatch
- }
-
- return total / float64(len(qTokens))
-}
diff --git a/internal/semantic/matcher.go b/internal/semantic/matcher.go
deleted file mode 100644
index b8bdc258..00000000
--- a/internal/semantic/matcher.go
+++ /dev/null
@@ -1,62 +0,0 @@
-package semantic
-
-import "context"
-
-// ElementMatcher is the interface for all element matching strategies.
-// Implementations include LexicalMatcher (Jaccard-based, zero deps) and
-// EmbeddingMatcher (cosine similarity on vector embeddings).
-type ElementMatcher interface {
- // Find scores elements against a natural language query and returns
- // the top-K matches above the threshold.
- Find(ctx context.Context, query string, elements []ElementDescriptor, opts FindOptions) (FindResult, error)
-
- // Strategy returns the name of the matching strategy (e.g. "lexical", "embedding").
- Strategy() string
-}
-
-// FindOptions controls matching behaviour.
-type FindOptions struct {
- Threshold float64
- TopK int
-
- // Per-request weight overrides (optional). If both are zero the
- // matcher's default weights are used.
- LexicalWeight float64
- EmbeddingWeight float64
-
- // Explain enables verbose per-match scoring breakdown.
- Explain bool
-}
-
-// FindResult holds the output of a Find operation.
-type FindResult struct {
- Matches []ElementMatch
- BestRef string
- BestScore float64
- Strategy string
- ElementCount int // total elements evaluated
-}
-
-// ConfidenceLabel returns a human-readable confidence level for the best
-// match score. Delegates to CalibrateConfidence for consistent labelling.
-func (r *FindResult) ConfidenceLabel() string {
- return CalibrateConfidence(r.BestScore)
-}
-
-// ElementMatch holds a scored match result.
-type ElementMatch struct {
- Ref string `json:"ref"`
- Score float64 `json:"score"`
- Role string `json:"role,omitempty"`
- Name string `json:"name,omitempty"`
-
- // Explain is populated when FindOptions.Explain is true.
- Explain *MatchExplain `json:"explain,omitempty"`
-}
-
-// MatchExplain exposes the per-strategy score breakdown for debugging.
-type MatchExplain struct {
- LexicalScore float64 `json:"lexical_score"`
- EmbeddingScore float64 `json:"embedding_score"`
- Composite string `json:"composite"`
-}
diff --git a/internal/semantic/recovery.go b/internal/semantic/recovery.go
deleted file mode 100644
index 10cdeecd..00000000
--- a/internal/semantic/recovery.go
+++ /dev/null
@@ -1,375 +0,0 @@
-package semantic
-
-import (
- "context"
- "fmt"
- "time"
-)
-
-// RecoveryConfig tunes the self-healing behaviour.
-type RecoveryConfig struct {
- // Enabled globally enables/disables recovery. Default true.
- Enabled bool
-
- // MaxRetries is the maximum number of recovery re-match attempts
- // before giving up. Default 1.
- MaxRetries int
-
- // MinConfidence is the minimum score the semantic re-match must
- // achieve for the recovery attempt to proceed. Default 0.4.
- MinConfidence float64
-
- // PreferHighConfidence when true will only auto-recover if the
- // confidence label is "high" or "medium". Default false (will
- // attempt recovery even at "low" confidence).
- PreferHighConfidence bool
-}
-
-// DefaultRecoveryConfig returns a production-ready configuration.
-func DefaultRecoveryConfig() RecoveryConfig {
- return RecoveryConfig{
- Enabled: true,
- MaxRetries: 1,
- MinConfidence: 0.4,
- PreferHighConfidence: false,
- }
-}
-
-// RecoveryResult captures what the recovery engine did. It is embedded
-// into the action response when recovery was attempted.
-type RecoveryResult struct {
- // Recovered is true if the action succeeded after re-matching.
- Recovered bool `json:"recovered"`
-
- // OriginalRef is the ref the agent originally requested.
- OriginalRef string `json:"original_ref"`
-
- // NewRef is the ref that the semantic re-match found.
- NewRef string `json:"new_ref,omitempty"`
-
- // Score is the semantic similarity score of the new match.
- Score float64 `json:"score,omitempty"`
-
- // Confidence is "high", "medium", or "low".
- Confidence string `json:"confidence,omitempty"`
-
- // Strategy is the matcher strategy that produced the new ref.
- Strategy string `json:"strategy,omitempty"`
-
- // FailureType classifies the original error.
- FailureType string `json:"failure_type"`
-
- // Attempts is how many re-match attempts were made.
- Attempts int `json:"attempts"`
-
- // LatencyMs is the total wall-clock time spent on recovery.
- LatencyMs int64 `json:"latency_ms"`
-
- // Error is non-empty when recovery was attempted but failed.
- Error string `json:"error,omitempty"`
-}
-
-// SnapshotRefresher is a callback the handler provides so the recovery
-// engine can force a fresh CDP accessibility tree fetch without importing
-// the bridge or chromedp packages.
-type SnapshotRefresher func(ctx context.Context, tabID string) error
-
-// NodeIDResolver maps a ref string to a node ID from the current
-// snapshot cache. Returns (nodeID, true) or (0, false).
-type NodeIDResolver func(tabID, ref string) (int64, bool)
-
-// ActionExecutor runs a single action and returns the result or error.
-// This is the same signature as Bridge.ExecuteAction.
-type ActionExecutor func(ctx context.Context, kind string, nodeID int64) (map[string]any, error)
-
-// DescriptorBuilder converts raw snapshot data into ElementDescriptors.
-// The handler provides this so the recovery engine stays decoupled from
-// bridge internals.
-type DescriptorBuilder func(tabID string) []ElementDescriptor
-
-// RecoveryEngine orchestrates self-healing when an action fails because
-// the target element's ref is stale or the DOM has changed.
-//
-// Integration pattern (used in handlers/actions.go):
-//
-// result, err := bridge.ExecuteAction(...)
-// if err != nil && recovery.ShouldAttempt(err, ref) {
-// rr := recovery.Attempt(ctx, tabID, ref, kind, ...)
-// ... use rr ...
-// }
-type RecoveryEngine struct {
- Config RecoveryConfig
- Matcher ElementMatcher
- IntentCache *IntentCache
- Refresh SnapshotRefresher
- ResolveNode NodeIDResolver
- BuildDescs DescriptorBuilder
-}
-
-// NewRecoveryEngine creates a RecoveryEngine with the given dependencies.
-func NewRecoveryEngine(
- cfg RecoveryConfig,
- matcher ElementMatcher,
- cache *IntentCache,
- refresh SnapshotRefresher,
- resolve NodeIDResolver,
- buildDescs DescriptorBuilder,
-) *RecoveryEngine {
- return &RecoveryEngine{
- Config: cfg,
- Matcher: matcher,
- IntentCache: cache,
- Refresh: refresh,
- ResolveNode: resolve,
- BuildDescs: buildDescs,
- }
-}
-
-// ShouldAttempt returns true when recovery is enabled and the failure
-// type is recoverable.
-func (re *RecoveryEngine) ShouldAttempt(err error, ref string) bool {
- if !re.Config.Enabled || ref == "" {
- return false
- }
- ft := ClassifyFailure(err)
- return ft.Recoverable()
-}
-
-// Attempt tries to semantically re-locate a stale element and re-execute
-// the action. It returns a RecoveryResult and, if successful, the action's
-// result payload.
-//
-// Flow:
-// 1. Classify the failure.
-// 2. Reconstruct a search query from the IntentCache or the ref's last
-// known descriptor.
-// 3. Force a fresh snapshot via the Refresh callback.
-// 4. Build new descriptors and run the Matcher.
-// 5. If a match exceeds MinConfidence, resolve its nodeID and call the
-// ActionExecutor.
-// 6. Return the RecoveryResult.
-func (re *RecoveryEngine) Attempt(
- ctx context.Context,
- tabID string,
- ref string,
- kind string,
- exec ActionExecutor,
-) (RecoveryResult, map[string]any, error) {
- start := time.Now()
-
- ft := ClassifyFailure(fmt.Errorf("recovery trigger"))
- // We always accept the call at this point since ShouldAttempt already
- // gated entry. Re-classify for the result struct.
- rr := RecoveryResult{
- OriginalRef: ref,
- FailureType: ft.String(),
- }
-
- // 1. Build the search query from cached intent.
- query := re.reconstructQuery(tabID, ref)
- if query == "" {
- rr.Error = "no cached intent for ref " + ref
- rr.LatencyMs = time.Since(start).Milliseconds()
- return rr, nil, fmt.Errorf("recovery: %s", rr.Error)
- }
-
- // 2. Iterate up to MaxRetries.
- maxRetries := re.Config.MaxRetries
- if maxRetries <= 0 {
- maxRetries = 1
- }
-
- var lastErr error
- for attempt := 1; attempt <= maxRetries; attempt++ {
- rr.Attempts = attempt
-
- // 3. Force a fresh snapshot.
- if re.Refresh != nil {
- if err := re.Refresh(ctx, tabID); err != nil {
- lastErr = fmt.Errorf("refresh snapshot: %w", err)
- continue
- }
- }
-
- // 4. Build descriptors from the fresh snapshot.
- descs := re.BuildDescs(tabID)
- if len(descs) == 0 {
- lastErr = fmt.Errorf("empty snapshot after refresh")
- continue
- }
-
- // 5. Run the semantic matcher.
- result, err := re.Matcher.Find(ctx, query, descs, FindOptions{
- Threshold: re.Config.MinConfidence,
- TopK: 1,
- })
- if err != nil {
- lastErr = fmt.Errorf("matcher: %w", err)
- continue
- }
- if result.BestRef == "" || result.BestScore < re.Config.MinConfidence {
- lastErr = fmt.Errorf("no match above threshold %.2f (best: %.2f)",
- re.Config.MinConfidence, result.BestScore)
- continue
- }
-
- // Confidence gate.
- conf := CalibrateConfidence(result.BestScore)
- if re.Config.PreferHighConfidence && conf == "low" {
- lastErr = fmt.Errorf("match confidence too low: %s (%.2f)",
- conf, result.BestScore)
- continue
- }
-
- rr.NewRef = result.BestRef
- rr.Score = result.BestScore
- rr.Confidence = conf
- rr.Strategy = result.Strategy
-
- // 6. Resolve the new ref → nodeID.
- nodeID, ok := re.ResolveNode(tabID, result.BestRef)
- if !ok {
- lastErr = fmt.Errorf("new ref %s not in cache after refresh", result.BestRef)
- continue
- }
-
- // 7. Re-execute the action.
- actionResult, execErr := exec(ctx, kind, nodeID)
- rr.LatencyMs = time.Since(start).Milliseconds()
- if execErr != nil {
- lastErr = execErr
- continue
- }
-
- rr.Recovered = true
- return rr, actionResult, nil
- }
-
- rr.LatencyMs = time.Since(start).Milliseconds()
- if lastErr != nil {
- rr.Error = lastErr.Error()
- }
- return rr, nil, lastErr
-}
-
-// AttemptWithClassification is like Attempt but accepts the pre-classified
-// failure type so callers that already called ClassifyFailure don't
-// re-compute it.
-func (re *RecoveryEngine) AttemptWithClassification(
- ctx context.Context,
- tabID string,
- ref string,
- kind string,
- ft FailureType,
- exec ActionExecutor,
-) (RecoveryResult, map[string]any, error) {
- start := time.Now()
-
- rr := RecoveryResult{
- OriginalRef: ref,
- FailureType: ft.String(),
- }
-
- query := re.reconstructQuery(tabID, ref)
- if query == "" {
- rr.Error = "no cached intent for ref " + ref
- rr.LatencyMs = time.Since(start).Milliseconds()
- return rr, nil, fmt.Errorf("recovery: %s", rr.Error)
- }
-
- maxRetries := re.Config.MaxRetries
- if maxRetries <= 0 {
- maxRetries = 1
- }
-
- var lastErr error
- for attempt := 1; attempt <= maxRetries; attempt++ {
- rr.Attempts = attempt
-
- if re.Refresh != nil {
- if err := re.Refresh(ctx, tabID); err != nil {
- lastErr = fmt.Errorf("refresh snapshot: %w", err)
- continue
- }
- }
-
- descs := re.BuildDescs(tabID)
- if len(descs) == 0 {
- lastErr = fmt.Errorf("empty snapshot after refresh")
- continue
- }
-
- result, err := re.Matcher.Find(ctx, query, descs, FindOptions{
- Threshold: re.Config.MinConfidence,
- TopK: 1,
- })
- if err != nil {
- lastErr = fmt.Errorf("matcher: %w", err)
- continue
- }
- if result.BestRef == "" || result.BestScore < re.Config.MinConfidence {
- lastErr = fmt.Errorf("no match above threshold %.2f (best: %.2f)",
- re.Config.MinConfidence, result.BestScore)
- continue
- }
-
- conf := CalibrateConfidence(result.BestScore)
- if re.Config.PreferHighConfidence && conf == "low" {
- lastErr = fmt.Errorf("match confidence too low: %s (%.2f)",
- conf, result.BestScore)
- continue
- }
-
- rr.NewRef = result.BestRef
- rr.Score = result.BestScore
- rr.Confidence = conf
- rr.Strategy = result.Strategy
-
- nodeID, ok := re.ResolveNode(tabID, result.BestRef)
- if !ok {
- lastErr = fmt.Errorf("new ref %s not in cache after refresh", result.BestRef)
- continue
- }
-
- actionResult, execErr := exec(ctx, kind, nodeID)
- rr.LatencyMs = time.Since(start).Milliseconds()
- if execErr != nil {
- lastErr = execErr
- continue
- }
-
- rr.Recovered = true
- return rr, actionResult, nil
- }
-
- rr.LatencyMs = time.Since(start).Milliseconds()
- if lastErr != nil {
- rr.Error = lastErr.Error()
- }
- return rr, nil, lastErr
-}
-
-// reconstructQuery builds the best possible search query from the
-// IntentCache. It prefers an explicit query string, falls back to the
-// element's composite descriptor.
-func (re *RecoveryEngine) reconstructQuery(tabID, ref string) string {
- if re.IntentCache == nil {
- return ""
- }
- entry, ok := re.IntentCache.Lookup(tabID, ref)
- if !ok {
- return ""
- }
- if entry.Query != "" {
- return entry.Query
- }
- return entry.Descriptor.Composite()
-}
-
-// RecordIntent is a convenience for the handler layer to cache element
-// intent after a successful /find or before an action execution.
-func (re *RecoveryEngine) RecordIntent(tabID, ref string, entry IntentEntry) {
- if re.IntentCache != nil {
- re.IntentCache.Store(tabID, ref, entry)
- }
-}
diff --git a/internal/semantic/recovery_test.go b/internal/semantic/recovery_test.go
deleted file mode 100644
index c59ace68..00000000
--- a/internal/semantic/recovery_test.go
+++ /dev/null
@@ -1,1284 +0,0 @@
-package semantic
-
-import (
- "context"
- "fmt"
- "strings"
- "testing"
- "time"
-)
-
-// ===========================================================================
-// FailureType classification tests
-// ===========================================================================
-
-func TestClassifyFailure_NilError(t *testing.T) {
- ft := ClassifyFailure(nil)
- if ft != FailureUnknown {
- t.Errorf("ClassifyFailure(nil) = %v, want FailureUnknown", ft)
- }
-}
-
-func TestClassifyFailure_ElementNotFound(t *testing.T) {
- patterns := []string{
- "could not find node with id 42",
- "node with given id not found",
- "no node for backendNodeId 123",
- "ref not found: e15",
- "Node not found for the given backend node id",
- "backend node id cannot be resolved",
- "no node with given id",
- }
- for _, p := range patterns {
- ft := ClassifyFailure(fmt.Errorf("%s", p))
- if ft != FailureElementNotFound {
- t.Errorf("ClassifyFailure(%q) = %v, want FailureElementNotFound", p, ft)
- }
- }
-}
-
-func TestClassifyFailure_ElementStale(t *testing.T) {
- patterns := []string{
- "stale element reference",
- "node is detached from the document",
- "execution context was destroyed",
- "orphan node detected",
- "object reference not set",
- }
- for _, p := range patterns {
- ft := ClassifyFailure(fmt.Errorf("%s", p))
- if ft != FailureElementStale {
- t.Errorf("ClassifyFailure(%q) = %v, want FailureElementStale", p, ft)
- }
- }
-}
-
-func TestClassifyFailure_NotInteractable(t *testing.T) {
- patterns := []string{
- "node is not visible on the page",
- "element is not interactable at point (100, 200)",
- "overlapping element covers the target",
- "element is disabled",
- "cannot focus the target element",
- "Element is outside of the viewport",
- }
- for _, p := range patterns {
- ft := ClassifyFailure(fmt.Errorf("%s", p))
- if ft != FailureElementNotInteractable {
- t.Errorf("ClassifyFailure(%q) = %v, want FailureElementNotInteractable", p, ft)
- }
- }
-}
-
-func TestClassifyFailure_Navigation(t *testing.T) {
- patterns := []string{
- "page navigated while action was pending",
- "Frame was detached during execution",
- "inspected target navigated or closed",
- "Navigated away from page",
- }
- for _, p := range patterns {
- ft := ClassifyFailure(fmt.Errorf("%s", p))
- if ft != FailureNavigation {
- t.Errorf("ClassifyFailure(%q) = %v, want FailureNavigation", p, ft)
- }
- }
-}
-
-func TestClassifyFailure_Network(t *testing.T) {
- patterns := []string{
- "connection refused to remote debugging port",
- "websocket connection closed unexpectedly",
- "could not connect to Chrome",
- "timeout waiting for response from browser",
- }
- for _, p := range patterns {
- ft := ClassifyFailure(fmt.Errorf("%s", p))
- if ft != FailureNetwork {
- t.Errorf("ClassifyFailure(%q) = %v, want FailureNetwork", p, ft)
- }
- }
-}
-
-func TestClassifyFailure_Unknown(t *testing.T) {
- patterns := []string{
- "something completely unexpected happened",
- "random error",
- }
- for _, p := range patterns {
- ft := ClassifyFailure(fmt.Errorf("%s", p))
- if ft != FailureUnknown {
- t.Errorf("ClassifyFailure(%q) = %v, want FailureUnknown", p, ft)
- }
- }
-}
-
-func TestFailureType_String(t *testing.T) {
- cases := map[FailureType]string{
- FailureUnknown: "unknown",
- FailureElementNotFound: "element_not_found",
- FailureElementStale: "element_stale",
- FailureElementNotInteractable: "element_not_interactable",
- FailureNavigation: "navigation",
- FailureNetwork: "network",
- }
- for ft, want := range cases {
- if ft.String() != want {
- t.Errorf("FailureType(%d).String() = %q, want %q", ft, ft.String(), want)
- }
- }
-}
-
-func TestFailureType_Recoverable(t *testing.T) {
- recoverable := []FailureType{
- FailureElementNotFound,
- FailureElementStale,
- FailureElementNotInteractable,
- FailureNavigation,
- }
- for _, ft := range recoverable {
- if !ft.Recoverable() {
- t.Errorf("%v.Recoverable() = false, want true", ft)
- }
- }
- nonRecoverable := []FailureType{
- FailureUnknown,
- FailureNetwork,
- }
- for _, ft := range nonRecoverable {
- if ft.Recoverable() {
- t.Errorf("%v.Recoverable() = true, want false", ft)
- }
- }
-}
-
-// ===========================================================================
-// IntentCache tests
-// ===========================================================================
-
-func TestIntentCache_StoreAndLookup(t *testing.T) {
- c := NewIntentCache(100, 5*time.Minute)
-
- entry := IntentEntry{
- Query: "submit button",
- Descriptor: ElementDescriptor{Ref: "e1", Role: "button", Name: "Submit"},
- Score: 0.95,
- Confidence: "high",
- Strategy: "combined",
- }
- c.Store("tab1", "e1", entry)
-
- got, ok := c.Lookup("tab1", "e1")
- if !ok {
- t.Fatal("Lookup returned false for stored entry")
- }
- if got.Query != "submit button" {
- t.Errorf("Query = %q, want %q", got.Query, "submit button")
- }
- if got.Descriptor.Role != "button" {
- t.Errorf("Descriptor.Role = %q, want %q", got.Descriptor.Role, "button")
- }
- if got.Score != 0.95 {
- t.Errorf("Score = %f, want 0.95", got.Score)
- }
-}
-
-func TestIntentCache_LookupMiss(t *testing.T) {
- c := NewIntentCache(100, 5*time.Minute)
-
- _, ok := c.Lookup("tab1", "e1")
- if ok {
- t.Error("Lookup returned true for missing entry")
- }
- // Wrong tab.
- c.Store("tab1", "e1", IntentEntry{Query: "test"})
- _, ok = c.Lookup("tab2", "e1")
- if ok {
- t.Error("Lookup returned true for wrong tab")
- }
-}
-
-func TestIntentCache_TTLExpiry(t *testing.T) {
- c := NewIntentCache(100, 50*time.Millisecond)
-
- c.Store("tab1", "e1", IntentEntry{Query: "test", CachedAt: time.Now()})
-
- // Should be found immediately.
- _, ok := c.Lookup("tab1", "e1")
- if !ok {
- t.Fatal("Lookup should find fresh entry")
- }
-
- // Wait for expiry.
- time.Sleep(60 * time.Millisecond)
- _, ok = c.Lookup("tab1", "e1")
- if ok {
- t.Error("Lookup should return false after TTL expiry")
- }
-}
-
-func TestIntentCache_LRUEviction(t *testing.T) {
- c := NewIntentCache(3, 5*time.Minute)
-
- // Fill to capacity.
- c.Store("tab1", "e1", IntentEntry{Query: "first", CachedAt: time.Now().Add(-3 * time.Minute)})
- c.Store("tab1", "e2", IntentEntry{Query: "second", CachedAt: time.Now().Add(-2 * time.Minute)})
- c.Store("tab1", "e3", IntentEntry{Query: "third", CachedAt: time.Now().Add(-1 * time.Minute)})
-
- // Add one more — oldest (e1) should be evicted.
- c.Store("tab1", "e4", IntentEntry{Query: "fourth"})
-
- _, ok := c.Lookup("tab1", "e1")
- if ok {
- t.Error("e1 should have been evicted (oldest)")
- }
- _, ok = c.Lookup("tab1", "e4")
- if !ok {
- t.Error("e4 should be present after eviction")
- }
-}
-
-func TestIntentCache_InvalidateTab(t *testing.T) {
- c := NewIntentCache(100, 5*time.Minute)
-
- c.Store("tab1", "e1", IntentEntry{Query: "test"})
- c.Store("tab1", "e2", IntentEntry{Query: "test2"})
- c.Store("tab2", "e1", IntentEntry{Query: "other"})
-
- c.InvalidateTab("tab1")
-
- _, ok := c.Lookup("tab1", "e1")
- if ok {
- t.Error("tab1/e1 should be gone after InvalidateTab")
- }
- _, ok = c.Lookup("tab2", "e1")
- if !ok {
- t.Error("tab2/e1 should still be present")
- }
-}
-
-func TestIntentCache_Size(t *testing.T) {
- c := NewIntentCache(100, 5*time.Minute)
- if c.Size() != 0 {
- t.Errorf("Size = %d, want 0", c.Size())
- }
-
- c.Store("tab1", "e1", IntentEntry{Query: "a"})
- c.Store("tab1", "e2", IntentEntry{Query: "b"})
- c.Store("tab2", "e3", IntentEntry{Query: "c"})
- if c.Size() != 3 {
- t.Errorf("Size = %d, want 3", c.Size())
- }
-}
-
-func TestIntentCache_UpdateExisting(t *testing.T) {
- c := NewIntentCache(100, 5*time.Minute)
-
- c.Store("tab1", "e1", IntentEntry{Query: "old query"})
- c.Store("tab1", "e1", IntentEntry{Query: "new query"})
-
- got, ok := c.Lookup("tab1", "e1")
- if !ok {
- t.Fatal("Lookup should find updated entry")
- }
- if got.Query != "new query" {
- t.Errorf("Query = %q, want %q", got.Query, "new query")
- }
- if c.Size() != 1 {
- t.Errorf("Size = %d, want 1 (update should not add)", c.Size())
- }
-}
-
-func TestIntentCache_DefaultValues(t *testing.T) {
- // Zero/negative values should be replaced with defaults.
- c := NewIntentCache(0, 0)
- if c.maxRefs != 200 {
- t.Errorf("maxRefs = %d, want 200", c.maxRefs)
- }
- if c.ttl != 10*time.Minute {
- t.Errorf("ttl = %v, want 10m", c.ttl)
- }
-}
-
-// ===========================================================================
-// RecoveryEngine tests
-// ===========================================================================
-
-// mockMatcher is a configurable test double for ElementMatcher.
-type mockMatcher struct {
- findFn func(ctx context.Context, query string, descs []ElementDescriptor, opts FindOptions) (FindResult, error)
-}
-
-func (m *mockMatcher) Find(ctx context.Context, query string, descs []ElementDescriptor, opts FindOptions) (FindResult, error) {
- if m.findFn != nil {
- return m.findFn(ctx, query, descs, opts)
- }
- return FindResult{}, fmt.Errorf("mockMatcher: Find not configured")
-}
-
-func (m *mockMatcher) Strategy() string { return "mock" }
-
-func TestRecoveryEngine_ShouldAttempt(t *testing.T) {
- re := &RecoveryEngine{Config: DefaultRecoveryConfig()}
-
- if re.ShouldAttempt(nil, "e1") {
- t.Error("ShouldAttempt(nil, e1) = true, want false")
- }
- if re.ShouldAttempt(fmt.Errorf("could not find node"), "") {
- t.Error("ShouldAttempt(err, '') = true, want false (empty ref)")
- }
- if re.ShouldAttempt(fmt.Errorf("could not find node"), "e1") != true {
- t.Error("ShouldAttempt(notFound, e1) = false, want true")
- }
- if re.ShouldAttempt(fmt.Errorf("stale element"), "e1") != true {
- t.Error("ShouldAttempt(stale, e1) = false, want true")
- }
- if re.ShouldAttempt(fmt.Errorf("websocket connection closed"), "e1") {
- t.Error("ShouldAttempt(network, e1) = true, want false (not recoverable)")
- }
- if re.ShouldAttempt(fmt.Errorf("random error xyz"), "e1") {
- t.Error("ShouldAttempt(unknown, e1) = true, want false")
- }
-}
-
-func TestRecoveryEngine_ShouldAttempt_Disabled(t *testing.T) {
- cfg := DefaultRecoveryConfig()
- cfg.Enabled = false
- re := &RecoveryEngine{Config: cfg}
-
- if re.ShouldAttempt(fmt.Errorf("could not find node"), "e1") {
- t.Error("ShouldAttempt should return false when disabled")
- }
-}
-
-func TestRecoveryEngine_Attempt_NoCachedIntent(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- re := NewRecoveryEngine(
- DefaultRecoveryConfig(),
- &mockMatcher{},
- cache,
- nil, nil, nil,
- )
-
- rr, res, err := re.Attempt(context.Background(), "tab1", "e99", "click", nil)
- if err == nil {
- t.Error("expected error when no intent cached")
- }
- if rr.Recovered {
- t.Error("Recovered should be false")
- }
- if res != nil {
- t.Error("result should be nil")
- }
- if !strings.Contains(rr.Error, "no cached intent") {
- t.Errorf("Error = %q, want to contain 'no cached intent'", rr.Error)
- }
-}
-
-func TestRecoveryEngine_Attempt_Success(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- cache.Store("tab1", "e5", IntentEntry{
- Query: "submit button",
- Descriptor: ElementDescriptor{Ref: "e5", Role: "button", Name: "Submit"},
- })
-
- matcher := &mockMatcher{
- findFn: func(_ context.Context, query string, descs []ElementDescriptor, opts FindOptions) (FindResult, error) {
- if query != "submit button" {
- return FindResult{}, fmt.Errorf("unexpected query: %s", query)
- }
- return FindResult{
- BestRef: "e12",
- BestScore: 0.88,
- Matches: []ElementMatch{
- {Ref: "e12", Role: "button", Name: "Submit Form", Score: 0.88},
- },
- Strategy: "combined",
- ElementCount: 3,
- }, nil
- },
- }
-
- refreshCalled := false
- re := NewRecoveryEngine(
- DefaultRecoveryConfig(),
- matcher,
- cache,
- func(ctx context.Context, tabID string) error {
- refreshCalled = true
- return nil
- },
- func(tabID, ref string) (int64, bool) {
- if ref == "e12" {
- return 42, true
- }
- return 0, false
- },
- func(tabID string) []ElementDescriptor {
- return []ElementDescriptor{
- {Ref: "e10", Role: "link", Name: "Home"},
- {Ref: "e12", Role: "button", Name: "Submit Form"},
- {Ref: "e14", Role: "textbox", Name: "Email"},
- }
- },
- )
-
- actionCalled := false
- rr, res, err := re.Attempt(context.Background(), "tab1", "e5", "click",
- func(ctx context.Context, kind string, nodeID int64) (map[string]any, error) {
- actionCalled = true
- if kind != "click" {
- return nil, fmt.Errorf("wrong kind: %s", kind)
- }
- if nodeID != 42 {
- return nil, fmt.Errorf("wrong nodeID: %d", nodeID)
- }
- return map[string]any{"clicked": true}, nil
- },
- )
-
- if err != nil {
- t.Fatalf("unexpected error: %v", err)
- }
- if !rr.Recovered {
- t.Error("Recovered should be true")
- }
- if !refreshCalled {
- t.Error("snapshot refresh was not called")
- }
- if !actionCalled {
- t.Error("action executor was not called")
- }
- if rr.NewRef != "e12" {
- t.Errorf("NewRef = %q, want %q", rr.NewRef, "e12")
- }
- if rr.Score != 0.88 {
- t.Errorf("Score = %f, want 0.88", rr.Score)
- }
- if rr.OriginalRef != "e5" {
- t.Errorf("OriginalRef = %q, want %q", rr.OriginalRef, "e5")
- }
- if res["clicked"] != true {
- t.Errorf("action result missing clicked=true")
- }
- if rr.LatencyMs < 0 {
- t.Errorf("LatencyMs = %d, should be >= 0", rr.LatencyMs)
- }
-}
-
-func TestRecoveryEngine_Attempt_ScoreBelowThreshold(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- cache.Store("tab1", "e1", IntentEntry{
- Query: "submit button",
- })
-
- matcher := &mockMatcher{
- findFn: func(_ context.Context, _ string, _ []ElementDescriptor, _ FindOptions) (FindResult, error) {
- return FindResult{
- BestRef: "e2",
- BestScore: 0.25, // Below default MinConfidence (0.4)
- }, nil
- },
- }
-
- re := NewRecoveryEngine(
- DefaultRecoveryConfig(),
- matcher,
- cache,
- func(_ context.Context, _ string) error { return nil },
- func(_, _ string) (int64, bool) { return 10, true },
- func(_ string) []ElementDescriptor {
- return []ElementDescriptor{{Ref: "e2", Role: "button", Name: "Cancel"}}
- },
- )
-
- rr, _, err := re.Attempt(context.Background(), "tab1", "e1", "click",
- func(_ context.Context, _ string, _ int64) (map[string]any, error) {
- t.Error("action should not be called when score is below threshold")
- return nil, nil
- },
- )
-
- if err == nil {
- t.Error("expected error for low score")
- }
- if rr.Recovered {
- t.Error("should not recover with score below threshold")
- }
- if !strings.Contains(rr.Error, "no match above threshold") {
- t.Errorf("Error = %q, want to contain threshold message", rr.Error)
- }
-}
-
-func TestRecoveryEngine_Attempt_ActionFailsOnReMatch(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- cache.Store("tab1", "e1", IntentEntry{Query: "login button"})
-
- matcher := &mockMatcher{
- findFn: func(_ context.Context, _ string, _ []ElementDescriptor, _ FindOptions) (FindResult, error) {
- return FindResult{
- BestRef: "e8",
- BestScore: 0.9,
- Strategy: "combined",
- }, nil
- },
- }
-
- re := NewRecoveryEngine(
- DefaultRecoveryConfig(),
- matcher,
- cache,
- func(_ context.Context, _ string) error { return nil },
- func(_, ref string) (int64, bool) {
- if ref == "e8" {
- return 77, true
- }
- return 0, false
- },
- func(_ string) []ElementDescriptor {
- return []ElementDescriptor{{Ref: "e8", Role: "button", Name: "Login"}}
- },
- )
-
- rr, _, err := re.Attempt(context.Background(), "tab1", "e1", "click",
- func(_ context.Context, kind string, nodeID int64) (map[string]any, error) {
- return nil, fmt.Errorf("element is disabled")
- },
- )
-
- if err == nil {
- t.Error("expected error when action fails after re-match")
- }
- if rr.Recovered {
- t.Error("should not be recovered when re-executed action fails")
- }
-}
-
-func TestRecoveryEngine_Attempt_EmptySnapshotAfterRefresh(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- cache.Store("tab1", "e1", IntentEntry{Query: "submit"})
-
- re := NewRecoveryEngine(
- DefaultRecoveryConfig(),
- &mockMatcher{},
- cache,
- func(_ context.Context, _ string) error { return nil },
- nil,
- func(_ string) []ElementDescriptor { return nil }, // empty snapshot
- )
-
- rr, _, err := re.Attempt(context.Background(), "tab1", "e1", "click", nil)
- if err == nil {
- t.Error("expected error for empty snapshot")
- }
- if !strings.Contains(rr.Error, "empty snapshot") {
- t.Errorf("Error = %q, want to contain 'empty snapshot'", rr.Error)
- }
-}
-
-func TestRecoveryEngine_Attempt_RefreshFails(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- cache.Store("tab1", "e1", IntentEntry{Query: "submit"})
-
- re := NewRecoveryEngine(
- DefaultRecoveryConfig(),
- &mockMatcher{},
- cache,
- func(_ context.Context, _ string) error { return fmt.Errorf("CDP timeout") },
- nil, nil,
- )
-
- rr, _, err := re.Attempt(context.Background(), "tab1", "e1", "click", nil)
- if err == nil {
- t.Error("expected error when refresh fails")
- }
- if !strings.Contains(rr.Error, "refresh snapshot") {
- t.Errorf("Error = %q, want to contain 'refresh snapshot'", rr.Error)
- }
-}
-
-func TestRecoveryEngine_Attempt_MatcherError(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- cache.Store("tab1", "e1", IntentEntry{Query: "submit"})
-
- matcher := &mockMatcher{
- findFn: func(_ context.Context, _ string, _ []ElementDescriptor, _ FindOptions) (FindResult, error) {
- return FindResult{}, fmt.Errorf("internal matcher error")
- },
- }
-
- re := NewRecoveryEngine(
- DefaultRecoveryConfig(),
- matcher,
- cache,
- func(_ context.Context, _ string) error { return nil },
- nil,
- func(_ string) []ElementDescriptor {
- return []ElementDescriptor{{Ref: "e2", Role: "button", Name: "Submit"}}
- },
- )
-
- rr, _, err := re.Attempt(context.Background(), "tab1", "e1", "click", nil)
- if err == nil {
- t.Error("expected error when matcher fails")
- }
- if !strings.Contains(rr.Error, "matcher") {
- t.Errorf("Error = %q, want to contain 'matcher'", rr.Error)
- }
-}
-
-func TestRecoveryEngine_Attempt_NewRefNotInCache(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- cache.Store("tab1", "e1", IntentEntry{Query: "submit"})
-
- matcher := &mockMatcher{
- findFn: func(_ context.Context, _ string, _ []ElementDescriptor, _ FindOptions) (FindResult, error) {
- return FindResult{BestRef: "e99", BestScore: 0.9, Strategy: "combined"}, nil
- },
- }
-
- re := NewRecoveryEngine(
- DefaultRecoveryConfig(),
- matcher,
- cache,
- func(_ context.Context, _ string) error { return nil },
- func(_, _ string) (int64, bool) { return 0, false }, // ref not in cache
- func(_ string) []ElementDescriptor {
- return []ElementDescriptor{{Ref: "e99", Role: "button", Name: "Submit"}}
- },
- )
-
- rr, _, err := re.Attempt(context.Background(), "tab1", "e1", "click", nil)
- if err == nil {
- t.Error("expected error when new ref not in node cache")
- }
- if !strings.Contains(rr.Error, "not in cache after refresh") {
- t.Errorf("Error = %q, want to contain 'not in cache after refresh'", rr.Error)
- }
-}
-
-func TestRecoveryEngine_AttemptWithClassification(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- cache.Store("tab1", "e3", IntentEntry{
- Query: "email input",
- Descriptor: ElementDescriptor{Ref: "e3", Role: "textbox", Name: "Email"},
- })
-
- matcher := &mockMatcher{
- findFn: func(_ context.Context, query string, _ []ElementDescriptor, _ FindOptions) (FindResult, error) {
- return FindResult{
- BestRef: "e20",
- BestScore: 0.85,
- Strategy: "combined",
- }, nil
- },
- }
-
- re := NewRecoveryEngine(
- DefaultRecoveryConfig(),
- matcher,
- cache,
- func(_ context.Context, _ string) error { return nil },
- func(_, ref string) (int64, bool) {
- if ref == "e20" {
- return 55, true
- }
- return 0, false
- },
- func(_ string) []ElementDescriptor {
- return []ElementDescriptor{{Ref: "e20", Role: "textbox", Name: "Email Address"}}
- },
- )
-
- rr, res, err := re.AttemptWithClassification(
- context.Background(), "tab1", "e3", "fill",
- FailureElementStale,
- func(_ context.Context, kind string, nodeID int64) (map[string]any, error) {
- return map[string]any{"filled": true}, nil
- },
- )
-
- if err != nil {
- t.Fatalf("unexpected error: %v", err)
- }
- if !rr.Recovered {
- t.Error("should be recovered")
- }
- if rr.FailureType != "element_stale" {
- t.Errorf("FailureType = %q, want %q", rr.FailureType, "element_stale")
- }
- if res["filled"] != true {
- t.Error("action result missing filled=true")
- }
-}
-
-func TestRecoveryEngine_PreferHighConfidence_RejectsLow(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- cache.Store("tab1", "e1", IntentEntry{Query: "submit"})
-
- matcher := &mockMatcher{
- findFn: func(_ context.Context, _ string, _ []ElementDescriptor, _ FindOptions) (FindResult, error) {
- return FindResult{
- BestRef: "e2",
- BestScore: 0.5, // CalibrateConfidence(0.5) = "low"
- Strategy: "combined",
- }, nil
- },
- }
-
- cfg := DefaultRecoveryConfig()
- cfg.PreferHighConfidence = true
-
- re := NewRecoveryEngine(
- cfg,
- matcher,
- cache,
- func(_ context.Context, _ string) error { return nil },
- func(_, _ string) (int64, bool) { return 10, true },
- func(_ string) []ElementDescriptor {
- return []ElementDescriptor{{Ref: "e2", Role: "button", Name: "Submit"}}
- },
- )
-
- rr, _, err := re.Attempt(context.Background(), "tab1", "e1", "click", nil)
- if err == nil {
- t.Error("expected error when PreferHighConfidence rejects low-confidence match")
- }
- if rr.Recovered {
- t.Error("should not recover with low confidence when PreferHighConfidence=true")
- }
- if !strings.Contains(rr.Error, "confidence too low") {
- t.Errorf("Error = %q, want 'confidence too low'", rr.Error)
- }
-}
-
-func TestRecoveryEngine_ReconstructQuery_FallbackToComposite(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- // Store entry with no Query, only a descriptor.
- cache.Store("tab1", "e1", IntentEntry{
- Descriptor: ElementDescriptor{Ref: "e1", Role: "button", Name: "Sign In"},
- })
-
- querySeen := ""
- matcher := &mockMatcher{
- findFn: func(_ context.Context, query string, _ []ElementDescriptor, _ FindOptions) (FindResult, error) {
- querySeen = query
- return FindResult{
- BestRef: "e10",
- BestScore: 0.9,
- Strategy: "combined",
- }, nil
- },
- }
-
- re := NewRecoveryEngine(
- DefaultRecoveryConfig(),
- matcher,
- cache,
- func(_ context.Context, _ string) error { return nil },
- func(_, ref string) (int64, bool) {
- if ref == "e10" {
- return 33, true
- }
- return 0, false
- },
- func(_ string) []ElementDescriptor {
- return []ElementDescriptor{{Ref: "e10", Role: "button", Name: "Sign In"}}
- },
- )
-
- _, _, _ = re.Attempt(context.Background(), "tab1", "e1", "click",
- func(_ context.Context, _ string, _ int64) (map[string]any, error) {
- return map[string]any{"ok": true}, nil
- },
- )
-
- // The query should be the Composite() of the descriptor: "button: Sign In"
- desc := ElementDescriptor{Ref: "e1", Role: "button", Name: "Sign In"}
- expected := desc.Composite()
- if querySeen != expected {
- t.Errorf("reconstructed query = %q, want %q", querySeen, expected)
- }
-}
-
-func TestRecoveryEngine_RecordIntent(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- re := NewRecoveryEngine(
- DefaultRecoveryConfig(),
- &mockMatcher{},
- cache,
- nil, nil, nil,
- )
-
- re.RecordIntent("tab1", "e5", IntentEntry{
- Query: "search box",
- Descriptor: ElementDescriptor{Ref: "e5", Role: "textbox", Name: "Search"},
- })
-
- entry, ok := cache.Lookup("tab1", "e5")
- if !ok {
- t.Fatal("RecordIntent should store in IntentCache")
- }
- if entry.Query != "search box" {
- t.Errorf("Query = %q, want %q", entry.Query, "search box")
- }
-}
-
-func TestRecoveryEngine_RecordIntent_NilCache(t *testing.T) {
- re := &RecoveryEngine{Config: DefaultRecoveryConfig()}
- // Should not panic with nil IntentCache.
- re.RecordIntent("tab1", "e1", IntentEntry{Query: "test"})
-}
-
-// ===========================================================================
-// Realistic scenario tests — simulating real website interactions
-// ===========================================================================
-
-// Scenario: SPA form re-render. User found "Submit" button via /find,
-// then a React re-render assigned new refs. The click on the old ref
-// fails with "could not find node". Recovery should semantically
-// re-locate the Submit button.
-func TestRecovery_Scenario_SPAFormReRender(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- // Simulate /find having cached the intent for "Submit" button.
- cache.Store("tab-react-app", "e15", IntentEntry{
- Query: "submit button",
- Descriptor: ElementDescriptor{Ref: "e15", Role: "button", Name: "Submit"},
- Score: 0.95,
- Confidence: "high",
- Strategy: "combined",
- })
-
- // After re-render, the fresh snapshot has different refs.
- freshDescs := []ElementDescriptor{
- {Ref: "e30", Role: "heading", Name: "Contact Form"},
- {Ref: "e31", Role: "textbox", Name: "Full Name"},
- {Ref: "e32", Role: "textbox", Name: "Email Address"},
- {Ref: "e33", Role: "textbox", Name: "Message"},
- {Ref: "e34", Role: "button", Name: "Submit"},
- {Ref: "e35", Role: "button", Name: "Cancel"},
- }
-
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- re := NewRecoveryEngine(
- DefaultRecoveryConfig(),
- matcher,
- cache,
- func(_ context.Context, _ string) error { return nil },
- func(_, ref string) (int64, bool) {
- nodeMap := map[string]int64{
- "e30": 100, "e31": 101, "e32": 102,
- "e33": 103, "e34": 104, "e35": 105,
- }
- nid, ok := nodeMap[ref]
- return nid, ok
- },
- func(_ string) []ElementDescriptor { return freshDescs },
- )
-
- err := fmt.Errorf("could not find node with id 15")
- if !re.ShouldAttempt(err, "e15") {
- t.Fatal("ShouldAttempt should be true for 'could not find node'")
- }
-
- rr, res, recErr := re.AttemptWithClassification(
- context.Background(), "tab-react-app", "e15", "click",
- ClassifyFailure(err),
- func(_ context.Context, kind string, nodeID int64) (map[string]any, error) {
- if nodeID != 104 {
- return nil, fmt.Errorf("wrong node: %d (expected Submit=104)", nodeID)
- }
- return map[string]any{"clicked": true}, nil
- },
- )
-
- if recErr != nil {
- t.Fatalf("recovery failed: %v", recErr)
- }
- if !rr.Recovered {
- t.Error("should have recovered")
- }
- if rr.NewRef != "e34" {
- t.Errorf("NewRef = %q, want e34 (Submit button)", rr.NewRef)
- }
- if rr.FailureType != "element_not_found" {
- t.Errorf("FailureType = %q, want element_not_found", rr.FailureType)
- }
- if res["clicked"] != true {
- t.Error("action result should contain clicked=true")
- }
-}
-
-// Scenario: E-commerce checkout — the "Place Order" button becomes
-// stale after cart updates via AJAX. Recovery should find the replaced
-// button.
-func TestRecovery_Scenario_EcommerceCheckout(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- cache.Store("tab-shop", "e50", IntentEntry{
- Query: "place order button",
- Descriptor: ElementDescriptor{Ref: "e50", Role: "button", Name: "Place Order"},
- Score: 0.92,
- Confidence: "high",
- })
-
- freshDescs := []ElementDescriptor{
- {Ref: "e70", Role: "heading", Name: "Your Cart (3 items)"},
- {Ref: "e71", Role: "button", Name: "Update Cart"},
- {Ref: "e72", Role: "button", Name: "Apply Coupon"},
- {Ref: "e73", Role: "button", Name: "Place Order"},
- {Ref: "e74", Role: "link", Name: "Continue Shopping"},
- }
-
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- re := NewRecoveryEngine(
- DefaultRecoveryConfig(),
- matcher,
- cache,
- func(_ context.Context, _ string) error { return nil },
- func(_, ref string) (int64, bool) {
- if ref == "e73" {
- return 200, true
- }
- return 0, false
- },
- func(_ string) []ElementDescriptor { return freshDescs },
- )
-
- rr, res, err := re.AttemptWithClassification(
- context.Background(), "tab-shop", "e50", "click",
- FailureElementStale,
- func(_ context.Context, _ string, nodeID int64) (map[string]any, error) {
- return map[string]any{"orderPlaced": true}, nil
- },
- )
-
- if err != nil {
- t.Fatalf("recovery failed: %v", err)
- }
- if !rr.Recovered {
- t.Error("should recover")
- }
- if rr.NewRef != "e73" {
- t.Errorf("NewRef = %q, want e73", rr.NewRef)
- }
- if res["orderPlaced"] != true {
- t.Error("should have orderPlaced=true")
- }
-}
-
-// Scenario: Login page — typing into an email field that was replaced
-// by a password field after an SPA navigation.
-func TestRecovery_Scenario_LoginFormNavigation(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- cache.Store("tab-login", "e20", IntentEntry{
- Query: "password input field",
- Descriptor: ElementDescriptor{
- Ref: "e20", Role: "textbox", Name: "Password",
- },
- })
-
- freshDescs := []ElementDescriptor{
- {Ref: "e40", Role: "heading", Name: "Log In"},
- {Ref: "e41", Role: "textbox", Name: "Username or Email"},
- {Ref: "e42", Role: "textbox", Name: "Password"},
- {Ref: "e43", Role: "button", Name: "Log In"},
- {Ref: "e44", Role: "link", Name: "Forgot Password?"},
- }
-
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- re := NewRecoveryEngine(
- DefaultRecoveryConfig(),
- matcher,
- cache,
- func(_ context.Context, _ string) error { return nil },
- func(_, ref string) (int64, bool) {
- m := map[string]int64{"e40": 1, "e41": 2, "e42": 3, "e43": 4, "e44": 5}
- nid, ok := m[ref]
- return nid, ok
- },
- func(_ string) []ElementDescriptor { return freshDescs },
- )
-
- rr, res, err := re.AttemptWithClassification(
- context.Background(), "tab-login", "e20", "fill",
- FailureElementNotFound,
- func(_ context.Context, kind string, nodeID int64) (map[string]any, error) {
- if nodeID != 3 { // Password field = e42 = nodeID 3
- return nil, fmt.Errorf("filled wrong field, nodeID=%d", nodeID)
- }
- return map[string]any{"filled": true, "kind": kind}, nil
- },
- )
-
- if err != nil {
- t.Fatalf("recovery failed: %v", err)
- }
- if !rr.Recovered {
- t.Error("should recover")
- }
- if rr.NewRef != "e42" {
- t.Errorf("NewRef = %q, want e42 (Password field)", rr.NewRef)
- }
- if res["filled"] != true {
- t.Error("should have filled=true")
- }
-}
-
-// Scenario: Google search page — "Google Search" button re-rendered
-// with slightly different name "Search" after dynamic page update.
-func TestRecovery_Scenario_GoogleSearchButton(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- cache.Store("tab-google", "e10", IntentEntry{
- Query: "google search button",
- Descriptor: ElementDescriptor{Ref: "e10", Role: "button", Name: "Google Search"},
- })
-
- freshDescs := []ElementDescriptor{
- {Ref: "e80", Role: "textbox", Name: "Search"},
- {Ref: "e81", Role: "button", Name: "Google Search"},
- {Ref: "e82", Role: "button", Name: "I'm Feeling Lucky"},
- }
-
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- re := NewRecoveryEngine(
- DefaultRecoveryConfig(),
- matcher,
- cache,
- func(_ context.Context, _ string) error { return nil },
- func(_, ref string) (int64, bool) {
- if ref == "e81" {
- return 300, true
- }
- return 0, false
- },
- func(_ string) []ElementDescriptor { return freshDescs },
- )
-
- rr, _, err := re.AttemptWithClassification(
- context.Background(), "tab-google", "e10", "click",
- FailureElementStale,
- func(_ context.Context, _ string, nodeID int64) (map[string]any, error) {
- return map[string]any{"clicked": true}, nil
- },
- )
-
- if err != nil {
- t.Fatalf("recovery failed: %v", err)
- }
- if !rr.Recovered {
- t.Error("should recover")
- }
- if rr.NewRef != "e81" {
- t.Errorf("NewRef = %q, want e81 (Google Search button)", rr.NewRef)
- }
-}
-
-// Scenario: Dashboard with many similar buttons. The recovery should
-// still find the right one using the cached query.
-func TestRecovery_Scenario_DashboardSimilarButtons(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- cache.Store("tab-dash", "e5", IntentEntry{
- Query: "delete account",
- Descriptor: ElementDescriptor{
- Ref: "e5", Role: "button", Name: "Delete Account",
- },
- })
-
- freshDescs := []ElementDescriptor{
- {Ref: "e100", Role: "button", Name: "Delete Comment"},
- {Ref: "e101", Role: "button", Name: "Delete Post"},
- {Ref: "e102", Role: "button", Name: "Delete Account"},
- {Ref: "e103", Role: "button", Name: "Edit Account"},
- {Ref: "e104", Role: "button", Name: "Save Settings"},
- }
-
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- re := NewRecoveryEngine(
- DefaultRecoveryConfig(),
- matcher,
- cache,
- func(_ context.Context, _ string) error { return nil },
- func(_, ref string) (int64, bool) {
- m := map[string]int64{
- "e100": 10, "e101": 11, "e102": 12, "e103": 13, "e104": 14,
- }
- nid, ok := m[ref]
- return nid, ok
- },
- func(_ string) []ElementDescriptor { return freshDescs },
- )
-
- rr, _, err := re.AttemptWithClassification(
- context.Background(), "tab-dash", "e5", "click",
- FailureElementNotFound,
- func(_ context.Context, _ string, nodeID int64) (map[string]any, error) {
- if nodeID != 12 {
- return nil, fmt.Errorf("wrong button, nodeID=%d want 12 (Delete Account)", nodeID)
- }
- return map[string]any{"deleted": true}, nil
- },
- )
-
- if err != nil {
- t.Fatalf("recovery failed: %v", err)
- }
- if !rr.Recovered {
- t.Error("should recover with correct 'Delete Account' button")
- }
- if rr.NewRef != "e102" {
- t.Errorf("NewRef = %q, want e102 (Delete Account)", rr.NewRef)
- }
-}
-
-// Scenario: Navigation link that moved position in a CMS after page
-// edit. Descriptor-only recovery (no query stored).
-func TestRecovery_Scenario_CMSNavigationLink(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- // No query — only descriptor. Recovery falls back to Composite().
- cache.Store("tab-cms", "e7", IntentEntry{
- Descriptor: ElementDescriptor{Ref: "e7", Role: "link", Name: "About Us"},
- })
-
- freshDescs := []ElementDescriptor{
- {Ref: "e200", Role: "link", Name: "Home"},
- {Ref: "e201", Role: "link", Name: "Services"},
- {Ref: "e202", Role: "link", Name: "About Us"},
- {Ref: "e203", Role: "link", Name: "Contact"},
- {Ref: "e204", Role: "link", Name: "Blog"},
- }
-
- matcher := NewCombinedMatcher(NewHashingEmbedder(128))
-
- re := NewRecoveryEngine(
- DefaultRecoveryConfig(),
- matcher,
- cache,
- func(_ context.Context, _ string) error { return nil },
- func(_, ref string) (int64, bool) {
- m := map[string]int64{
- "e200": 50, "e201": 51, "e202": 52, "e203": 53, "e204": 54,
- }
- nid, ok := m[ref]
- return nid, ok
- },
- func(_ string) []ElementDescriptor { return freshDescs },
- )
-
- rr, _, err := re.AttemptWithClassification(
- context.Background(), "tab-cms", "e7", "click",
- FailureElementNotFound,
- func(_ context.Context, _ string, nodeID int64) (map[string]any, error) {
- return map[string]any{"navigated": true}, nil
- },
- )
-
- if err != nil {
- t.Fatalf("recovery failed: %v", err)
- }
- if !rr.Recovered {
- t.Error("should recover")
- }
- if rr.NewRef != "e202" {
- t.Errorf("NewRef = %q, want e202 (About Us link)", rr.NewRef)
- }
-}
-
-// Scenario: Unrecoverable network error — recovery should NOT attempt.
-func TestRecovery_Scenario_NetworkError_NoAttempt(t *testing.T) {
- re := &RecoveryEngine{Config: DefaultRecoveryConfig()}
-
- err := fmt.Errorf("websocket: connection refused to remote debugging port")
- if re.ShouldAttempt(err, "e1") {
- t.Error("ShouldAttempt should be false for network errors")
- }
-}
-
-// Scenario: Unknown error — recovery should NOT attempt.
-func TestRecovery_Scenario_UnknownError_NoAttempt(t *testing.T) {
- re := &RecoveryEngine{Config: DefaultRecoveryConfig()}
-
- err := fmt.Errorf("some completely unexpected internal error")
- if re.ShouldAttempt(err, "e1") {
- t.Error("ShouldAttempt should be false for unknown errors")
- }
-}
-
-// Scenario: MaxRetries > 1. First attempt fails (matcher returns low
-// score), second attempt succeeds after the snapshot changes.
-func TestRecovery_Scenario_MultipleRetries(t *testing.T) {
- cache := NewIntentCache(100, 5*time.Minute)
- cache.Store("tab1", "e1", IntentEntry{Query: "save button"})
-
- attempt := 0
- matcher := &mockMatcher{
- findFn: func(_ context.Context, _ string, _ []ElementDescriptor, _ FindOptions) (FindResult, error) {
- attempt++
- if attempt == 1 {
- // First attempt: score too low.
- return FindResult{BestRef: "e10", BestScore: 0.2}, nil
- }
- // Second attempt: match found.
- return FindResult{BestRef: "e11", BestScore: 0.95, Strategy: "combined"}, nil
- },
- }
-
- cfg := DefaultRecoveryConfig()
- cfg.MaxRetries = 3
-
- re := NewRecoveryEngine(
- cfg,
- matcher,
- cache,
- func(_ context.Context, _ string) error { return nil },
- func(_, ref string) (int64, bool) {
- if ref == "e11" {
- return 99, true
- }
- return 0, false
- },
- func(_ string) []ElementDescriptor {
- return []ElementDescriptor{{Ref: "e11", Role: "button", Name: "Save"}}
- },
- )
-
- rr, _, err := re.Attempt(context.Background(), "tab1", "e1", "click",
- func(_ context.Context, _ string, _ int64) (map[string]any, error) {
- return map[string]any{"saved": true}, nil
- },
- )
-
- if err != nil {
- t.Fatalf("recovery failed: %v", err)
- }
- if !rr.Recovered {
- t.Error("should recover on second attempt")
- }
- if rr.Attempts != 2 {
- t.Errorf("Attempts = %d, want 2", rr.Attempts)
- }
-}
-
-// ===========================================================================
-// DefaultRecoveryConfig tests
-// ===========================================================================
-
-func TestDefaultRecoveryConfig(t *testing.T) {
- cfg := DefaultRecoveryConfig()
- if !cfg.Enabled {
- t.Error("default should be enabled")
- }
- if cfg.MaxRetries != 1 {
- t.Errorf("MaxRetries = %d, want 1", cfg.MaxRetries)
- }
- if cfg.MinConfidence != 0.4 {
- t.Errorf("MinConfidence = %f, want 0.4", cfg.MinConfidence)
- }
- if cfg.PreferHighConfidence {
- t.Error("PreferHighConfidence should be false by default")
- }
-}
diff --git a/internal/semantic/semantic_test.go b/internal/semantic/semantic_test.go
deleted file mode 100644
index 65a7fcbe..00000000
--- a/internal/semantic/semantic_test.go
+++ /dev/null
@@ -1,981 +0,0 @@
-package semantic
-
-import (
- "context"
- "fmt"
- "math"
- "testing"
-)
-
-// ===========================================================================
-// ElementDescriptor tests
-// ===========================================================================
-
-func TestComposite(t *testing.T) {
- tests := []struct {
- name string
- desc ElementDescriptor
- want string
- }{
- {
- name: "role and name",
- desc: ElementDescriptor{Ref: "e0", Role: "button", Name: "Submit"},
- want: "button: Submit",
- },
- {
- name: "role name and value",
- desc: ElementDescriptor{Ref: "e1", Role: "textbox", Name: "Email", Value: "[email protected]"},
- want: "textbox: Email [[email protected]]",
- },
- {
- name: "name only",
- desc: ElementDescriptor{Ref: "e2", Name: "Heading"},
- want: "Heading",
- },
- {
- name: "empty",
- desc: ElementDescriptor{Ref: "e3"},
- want: "",
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got := tt.desc.Composite()
- if got != tt.want {
- t.Errorf("Composite() = %q, want %q", got, tt.want)
- }
- })
- }
-}
-
-// ===========================================================================
-// CalibrateConfidence tests
-// ===========================================================================
-
-func TestCalibrateConfidence(t *testing.T) {
- cases := []struct {
- score float64
- want string
- }{
- {1.0, "high"},
- {0.85, "high"},
- {0.8, "high"},
- {0.79, "medium"},
- {0.6, "medium"},
- {0.59, "low"},
- {0.0, "low"},
- }
- for _, c := range cases {
- got := CalibrateConfidence(c.score)
- if got != c.want {
- t.Errorf("CalibrateConfidence(%f) = %q, want %q", c.score, got, c.want)
- }
- }
-}
-
-// ===========================================================================
-// Stopword tests
-// ===========================================================================
-
-func TestIsStopword(t *testing.T) {
- if !isStopword("the") {
- t.Error("expected 'the' to be a stopword")
- }
- if isStopword("button") {
- t.Error("expected 'button' not to be a stopword")
- }
-}
-
-func TestRemoveStopwords(t *testing.T) {
- tokens := []string{"click", "the", "submit", "button"}
- filtered := removeStopwords(tokens)
- if len(filtered) != 3 {
- t.Errorf("expected 3 tokens after stopword removal, got %d: %v", len(filtered), filtered)
- }
-
- // When ALL tokens are stopwords, the original should be preserved.
- allStop := []string{"the", "a", "is", "was"}
- kept := removeStopwords(allStop)
- if len(kept) != len(allStop) {
- t.Errorf("expected original tokens when all are stopwords, got %d", len(kept))
- }
-}
-
-// ===========================================================================
-// LexicalScore tests
-// ===========================================================================
-
-func TestLexicalScore_ExactMatch(t *testing.T) {
- score := LexicalScore("submit button", "button: Submit")
- if score < 0.5 {
- t.Errorf("expected high score for exact match, got %f", score)
- }
-}
-
-func TestLexicalScore_NoOverlap(t *testing.T) {
- score := LexicalScore("download pdf", "button: Login")
- if score > 0.3 {
- t.Errorf("expected low score for no overlap, got %f", score)
- }
-}
-
-func TestLexicalScore_RoleBoost(t *testing.T) {
- // "button" is a role keyword; if it appears in both, a boost is applied.
- withRole := LexicalScore("submit button", "button: Submit")
- withoutRole := LexicalScore("submit action", "link: Submit")
- if withRole <= withoutRole {
- t.Errorf("expected role boost to increase score: withRole=%f, withoutRole=%f", withRole, withoutRole)
- }
-}
-
-func TestLexicalScore_StopwordRemoval(t *testing.T) {
- // "the" is a stopword — it should be removed so both queries score similarly.
- s1 := LexicalScore("click the button", "button: Click")
- s2 := LexicalScore("click button", "button: Click")
- diff := math.Abs(s1 - s2)
- if diff > 0.01 {
- t.Errorf("stopwords should not affect score significantly: s1=%f, s2=%f, diff=%f", s1, s2, diff)
- }
-}
-
-// ===========================================================================
-// LexicalMatcher (ElementMatcher interface) tests
-// ===========================================================================
-
-func TestLexicalMatcher_Find(t *testing.T) {
- m := NewLexicalMatcher()
-
- if m.Strategy() != "lexical" {
- t.Errorf("expected strategy=lexical, got %s", m.Strategy())
- }
-
- elements := []ElementDescriptor{
- {Ref: "e0", Role: "button", Name: "Log In"},
- {Ref: "e1", Role: "link", Name: "Sign Up"},
- {Ref: "e2", Role: "textbox", Name: "Email Address"},
- }
-
- result, err := m.Find(context.Background(), "log in button", elements, FindOptions{
- Threshold: 0.1,
- TopK: 3,
- })
- if err != nil {
- t.Fatalf("Find returned error: %v", err)
- }
-
- if result.ElementCount != 3 {
- t.Errorf("expected ElementCount=3, got %d", result.ElementCount)
- }
- if result.BestRef != "e0" {
- t.Errorf("expected BestRef=e0, got %s", result.BestRef)
- }
- if result.BestScore <= 0 {
- t.Errorf("expected positive BestScore, got %f", result.BestScore)
- }
-}
-
-func TestLexicalMatcher_ThresholdFiltering(t *testing.T) {
- m := NewLexicalMatcher()
-
- elements := []ElementDescriptor{
- {Ref: "e0", Role: "button", Name: "Submit"},
- {Ref: "e1", Role: "link", Name: "Home"},
- }
-
- result, err := m.Find(context.Background(), "submit button", elements, FindOptions{
- Threshold: 0.99,
- TopK: 5,
- })
- if err != nil {
- t.Fatalf("Find returned error: %v", err)
- }
-
- // Very high threshold — most likely nothing passes.
- for _, m := range result.Matches {
- if m.Score < 0.99 {
- t.Errorf("match %s has score %f below threshold", m.Ref, m.Score)
- }
- }
-}
-
-// ===========================================================================
-// DummyEmbedder tests
-// ===========================================================================
-
-func TestDummyEmbedder_Deterministic(t *testing.T) {
- e := NewDummyEmbedder(64)
-
- v1, err := e.Embed([]string{"hello world"})
- if err != nil {
- t.Fatalf("Embed error: %v", err)
- }
- v2, err := e.Embed([]string{"hello world"})
- if err != nil {
- t.Fatalf("Embed error: %v", err)
- }
-
- if len(v1[0]) != 64 {
- t.Errorf("expected dim=64, got %d", len(v1[0]))
- }
-
- for i := range v1[0] {
- if v1[0][i] != v2[0][i] {
- t.Fatalf("DummyEmbedder is not deterministic at dim %d", i)
- }
- }
-}
-
-func TestDummyEmbedder_Strategy(t *testing.T) {
- e := NewDummyEmbedder(32)
- if e.Strategy() != "dummy" {
- t.Errorf("expected strategy=dummy, got %s", e.Strategy())
- }
-}
-
-func TestDummyEmbedder_DefaultDim(t *testing.T) {
- e := NewDummyEmbedder(0)
- if e.Dim != 64 {
- t.Errorf("expected default dim=64, got %d", e.Dim)
- }
-}
-
-func TestDummyEmbedder_NormalizedOutput(t *testing.T) {
- e := NewDummyEmbedder(64)
- vecs, err := e.Embed([]string{"test string"})
- if err != nil {
- t.Fatalf("Embed error: %v", err)
- }
-
- var norm float64
- for _, v := range vecs[0] {
- norm += float64(v) * float64(v)
- }
- norm = math.Sqrt(norm)
- if math.Abs(norm-1.0) > 0.01 {
- t.Errorf("expected unit-norm vector, got norm=%f", norm)
- }
-}
-
-// ===========================================================================
-// CosineSimilarity tests
-// ===========================================================================
-
-func TestCosineSimilarity_Identical(t *testing.T) {
- v := []float32{1, 0, 0, 0}
- sim := CosineSimilarity(v, v)
- if math.Abs(sim-1.0) > 1e-6 {
- t.Errorf("identical vectors should have similarity 1.0, got %f", sim)
- }
-}
-
-func TestCosineSimilarity_Orthogonal(t *testing.T) {
- a := []float32{1, 0, 0, 0}
- b := []float32{0, 1, 0, 0}
- sim := CosineSimilarity(a, b)
- if math.Abs(sim) > 1e-6 {
- t.Errorf("orthogonal vectors should have similarity ~0, got %f", sim)
- }
-}
-
-func TestCosineSimilarity_Empty(t *testing.T) {
- sim := CosineSimilarity(nil, nil)
- if sim != 0 {
- t.Errorf("empty vectors should have similarity 0, got %f", sim)
- }
-}
-
-func TestCosineSimilarity_DifferentLengths(t *testing.T) {
- a := []float32{1, 0}
- b := []float32{1, 0, 0}
- sim := CosineSimilarity(a, b)
- if sim != 0 {
- t.Errorf("different-length vectors should return 0, got %f", sim)
- }
-}
-
-// ===========================================================================
-// EmbeddingMatcher tests
-// ===========================================================================
-
-func TestEmbeddingMatcher_Strategy(t *testing.T) {
- m := NewEmbeddingMatcher(NewDummyEmbedder(64))
- want := "embedding:dummy"
- if m.Strategy() != want {
- t.Errorf("expected strategy=%s, got %s", want, m.Strategy())
- }
-}
-
-func TestEmbeddingMatcher_Find(t *testing.T) {
- m := NewEmbeddingMatcher(NewDummyEmbedder(64))
-
- elements := []ElementDescriptor{
- {Ref: "e0", Role: "button", Name: "Login"},
- {Ref: "e1", Role: "textbox", Name: "Username"},
- {Ref: "e2", Role: "link", Name: "Forgot Password"},
- }
-
- result, err := m.Find(context.Background(), "login button", elements, FindOptions{
- Threshold: 0.0,
- TopK: 3,
- })
- if err != nil {
- t.Fatalf("Find returned error: %v", err)
- }
-
- if result.ElementCount != 3 {
- t.Errorf("expected ElementCount=3, got %d", result.ElementCount)
- }
- if result.Strategy != "embedding:dummy" {
- t.Errorf("expected strategy=embedding:dummy, got %s", result.Strategy)
- }
- if len(result.Matches) == 0 {
- t.Error("expected at least one match")
- }
- // BestScore should be in valid range
- if result.BestScore < 0 || result.BestScore > 1 {
- t.Errorf("BestScore out of [0,1] range: %f", result.BestScore)
- }
-}
-
-func TestEmbeddingMatcher_ThresholdFiltering(t *testing.T) {
- m := NewEmbeddingMatcher(NewDummyEmbedder(64))
-
- elements := []ElementDescriptor{
- {Ref: "e0", Role: "button", Name: "Submit"},
- {Ref: "e1", Role: "link", Name: "Cancel"},
- }
-
- result, err := m.Find(context.Background(), "xyz completely unrelated", elements, FindOptions{
- Threshold: 0.99,
- TopK: 5,
- })
- if err != nil {
- t.Fatalf("Find returned error: %v", err)
- }
-
- for _, m := range result.Matches {
- if m.Score < 0.99 {
- t.Errorf("match %s score %f below threshold 0.99", m.Ref, m.Score)
- }
- }
-}
-
-// ===========================================================================
-// FindResult.ConfidenceLabel tests
-// ===========================================================================
-
-func TestFindResult_ConfidenceLabel(t *testing.T) {
- r := &FindResult{BestScore: 0.9}
- if r.ConfidenceLabel() != "high" {
- t.Errorf("expected high, got %s", r.ConfidenceLabel())
- }
-
- r.BestScore = 0.65
- if r.ConfidenceLabel() != "medium" {
- t.Errorf("expected medium, got %s", r.ConfidenceLabel())
- }
-
- r.BestScore = 0.1
- if r.ConfidenceLabel() != "low" {
- t.Errorf("expected low, got %s", r.ConfidenceLabel())
- }
-}
-
-// ===========================================================================
-// Phase 3: HashingEmbedder tests
-// ===========================================================================
-
-func TestHashingEmbedder_Strategy(t *testing.T) {
- e := NewHashingEmbedder(128)
- if e.Strategy() != "hashing" {
- t.Errorf("expected strategy=hashing, got %s", e.Strategy())
- }
-}
-
-func TestHashingEmbedder_DefaultDim(t *testing.T) {
- e := NewHashingEmbedder(0)
- if e.dim != 128 {
- t.Errorf("expected default dim=128, got %d", e.dim)
- }
-}
-
-func TestHashingEmbedder_Deterministic(t *testing.T) {
- e := NewHashingEmbedder(128)
- v1, err := e.Embed([]string{"click the submit button"})
- if err != nil {
- t.Fatalf("Embed error: %v", err)
- }
- v2, err := e.Embed([]string{"click the submit button"})
- if err != nil {
- t.Fatalf("Embed error: %v", err)
- }
-
- if len(v1[0]) != 128 {
- t.Errorf("expected dim=128, got %d", len(v1[0]))
- }
- for i := range v1[0] {
- if v1[0][i] != v2[0][i] {
- t.Fatalf("HashingEmbedder not deterministic at dim %d: %f != %f", i, v1[0][i], v2[0][i])
- }
- }
-}
-
-func TestHashingEmbedder_Normalized(t *testing.T) {
- e := NewHashingEmbedder(128)
- vecs, err := e.Embed([]string{"button submit", "textbox username"})
- if err != nil {
- t.Fatalf("Embed error: %v", err)
- }
-
- for i, vec := range vecs {
- var norm float64
- for _, v := range vec {
- norm += float64(v) * float64(v)
- }
- norm = math.Sqrt(norm)
- if math.Abs(norm-1.0) > 0.01 {
- t.Errorf("vector %d not unit-norm: norm=%f", i, norm)
- }
- }
-}
-
-func TestHashingEmbedder_EmptyInput(t *testing.T) {
- e := NewHashingEmbedder(64)
- vecs, err := e.Embed([]string{""})
- if err != nil {
- t.Fatalf("Embed error: %v", err)
- }
- if len(vecs[0]) != 64 {
- t.Errorf("expected dim=64, got %d", len(vecs[0]))
- }
- // Empty string should produce a zero vector (no features to hash).
- var sum float64
- for _, v := range vecs[0] {
- sum += float64(v) * float64(v)
- }
- if sum > 0 {
- t.Error("empty input should produce zero vector")
- }
-}
-
-func TestHashingEmbedder_SimilarTexts(t *testing.T) {
- e := NewHashingEmbedder(256) // higher dim for less collision
-
- vecs, err := e.Embed([]string{
- "submit button", // 0
- "submit form", // 1 – shares "submit"
- "download report", // 2 – unrelated
- })
- if err != nil {
- t.Fatalf("Embed error: %v", err)
- }
-
- simSameWord := CosineSimilarity(vecs[0], vecs[1]) // share "submit"
- simUnrelated := CosineSimilarity(vecs[0], vecs[2]) // no shared words
-
- if simSameWord <= simUnrelated {
- t.Errorf("texts sharing 'submit' should be more similar: same=%f, unrelated=%f",
- simSameWord, simUnrelated)
- }
-}
-
-func TestHashingEmbedder_SubwordSimilarity(t *testing.T) {
- e := NewHashingEmbedder(256)
-
- // Character n-grams should give nonzero similarity between "button" and "btn".
- vecs, err := e.Embed([]string{
- "button", // full word
- "btn", // abbreviation – shares "bt" bigram via n-grams
- "search", // unrelated
- })
- if err != nil {
- t.Fatalf("Embed error: %v", err)
- }
-
- simAbbrev := CosineSimilarity(vecs[0], vecs[1])
- simUnrelated := CosineSimilarity(vecs[0], vecs[2])
-
- // The abbreviation similarity might be small, but should be greater
- // than an unrelated word due to shared character n-grams.
- if simAbbrev <= simUnrelated {
- t.Errorf("abbreviation should be more similar: abbrev=%f, unrelated=%f",
- simAbbrev, simUnrelated)
- }
-}
-
-func TestHashingEmbedder_RoleFeatures(t *testing.T) {
- e := NewHashingEmbedder(128)
-
- // "button" is a role keyword; it should get an extra role feature.
- vecs, err := e.Embed([]string{
- "button submit",
- "button cancel",
- "textbox email",
- })
- if err != nil {
- t.Fatalf("Embed error: %v", err)
- }
-
- // Both have "button" role features — should be more similar to each other
- // than to "textbox email" which has a different role keyword.
- simSameRole := CosineSimilarity(vecs[0], vecs[1])
- simDiffRole := CosineSimilarity(vecs[0], vecs[2])
-
- if simSameRole <= simDiffRole {
- t.Errorf("same-role elements should be more similar: same=%f, diff=%f",
- simSameRole, simDiffRole)
- }
-}
-
-func TestHashingEmbedder_BatchConsistency(t *testing.T) {
- e := NewHashingEmbedder(128)
-
- texts := []string{"login button", "search box", "navigation menu"}
- batchVecs, err := e.Embed(texts)
- if err != nil {
- t.Fatalf("batch embed error: %v", err)
- }
-
- // Each text embedded individually should match the batch result.
- for i, text := range texts {
- singleVecs, err := e.Embed([]string{text})
- if err != nil {
- t.Fatalf("single embed error: %v", err)
- }
- for j := range singleVecs[0] {
- if singleVecs[0][j] != batchVecs[i][j] {
- t.Errorf("batch[%d] != single at dim %d: %f != %f", i, j, batchVecs[i][j], singleVecs[0][j])
- break
- }
- }
- }
-}
-
-// ===========================================================================
-// Phase 3: CombinedMatcher tests
-// ===========================================================================
-
-func TestCombinedMatcher_Strategy(t *testing.T) {
- m := NewCombinedMatcher(NewHashingEmbedder(128))
- want := "combined:lexical+embedding:hashing"
- if m.Strategy() != want {
- t.Errorf("expected strategy=%s, got %s", want, m.Strategy())
- }
-}
-
-func TestCombinedMatcher_Find(t *testing.T) {
- m := NewCombinedMatcher(NewHashingEmbedder(128))
-
- elements := []ElementDescriptor{
- {Ref: "e0", Role: "button", Name: "Log In"},
- {Ref: "e1", Role: "link", Name: "Sign Up"},
- {Ref: "e2", Role: "textbox", Name: "Email Address"},
- }
-
- result, err := m.Find(context.Background(), "log in button", elements, FindOptions{
- Threshold: 0.1,
- TopK: 3,
- })
- if err != nil {
- t.Fatalf("Find returned error: %v", err)
- }
-
- if result.ElementCount != 3 {
- t.Errorf("expected ElementCount=3, got %d", result.ElementCount)
- }
- if result.BestRef != "e0" {
- t.Errorf("expected BestRef=e0, got %s", result.BestRef)
- }
- if result.BestScore <= 0 {
- t.Errorf("expected positive BestScore, got %f", result.BestScore)
- }
- if result.Strategy != "combined:lexical+embedding:hashing" {
- t.Errorf("expected combined strategy, got %s", result.Strategy)
- }
-}
-
-func TestCombinedMatcher_ThresholdFiltering(t *testing.T) {
- m := NewCombinedMatcher(NewHashingEmbedder(128))
-
- elements := []ElementDescriptor{
- {Ref: "e0", Role: "button", Name: "Submit"},
- {Ref: "e1", Role: "link", Name: "Home"},
- }
-
- result, err := m.Find(context.Background(), "submit button", elements, FindOptions{
- Threshold: 0.99,
- TopK: 5,
- })
- if err != nil {
- t.Fatalf("Find returned error: %v", err)
- }
-
- for _, match := range result.Matches {
- if match.Score < 0.99 {
- t.Errorf("match %s has score %f below threshold", match.Ref, match.Score)
- }
- }
-}
-
-func TestCombinedMatcher_TopK(t *testing.T) {
- m := NewCombinedMatcher(NewHashingEmbedder(128))
-
- elements := []ElementDescriptor{
- {Ref: "e0", Role: "button", Name: "Submit"},
- {Ref: "e1", Role: "button", Name: "Cancel"},
- {Ref: "e2", Role: "button", Name: "Reset"},
- {Ref: "e3", Role: "link", Name: "Home"},
- {Ref: "e4", Role: "textbox", Name: "Name"},
- }
-
- result, err := m.Find(context.Background(), "button", elements, FindOptions{
- Threshold: 0.01,
- TopK: 2,
- })
- if err != nil {
- t.Fatalf("Find returned error: %v", err)
- }
-
- if len(result.Matches) > 2 {
- t.Errorf("expected at most 2 matches (TopK=2), got %d", len(result.Matches))
- }
-}
-
-func TestCombinedMatcher_ScoresDescending(t *testing.T) {
- m := NewCombinedMatcher(NewHashingEmbedder(128))
-
- elements := []ElementDescriptor{
- {Ref: "e0", Role: "button", Name: "Login"},
- {Ref: "e1", Role: "textbox", Name: "Username"},
- {Ref: "e2", Role: "link", Name: "Forgot Password"},
- {Ref: "e3", Role: "heading", Name: "Welcome Page"},
- }
-
- result, err := m.Find(context.Background(), "login button", elements, FindOptions{
- Threshold: 0.01,
- TopK: 10,
- })
- if err != nil {
- t.Fatalf("Find returned error: %v", err)
- }
-
- for i := 1; i < len(result.Matches); i++ {
- if result.Matches[i].Score > result.Matches[i-1].Score {
- t.Errorf("matches not sorted descending: [%d]=%f > [%d]=%f",
- i, result.Matches[i].Score, i-1, result.Matches[i-1].Score)
- }
- }
-}
-
-func TestCombinedMatcher_WeightsApplied(t *testing.T) {
- m := NewCombinedMatcher(NewHashingEmbedder(128))
-
- // Override weights to emphasize embedding.
- m.LexicalWeight = 0.2
- m.EmbeddingWeight = 0.8
-
- elements := []ElementDescriptor{
- {Ref: "e0", Role: "button", Name: "Log In"},
- {Ref: "e1", Role: "link", Name: "Sign Up"},
- }
-
- result, err := m.Find(context.Background(), "log in", elements, FindOptions{
- Threshold: 0.01,
- TopK: 3,
- })
- if err != nil {
- t.Fatalf("Find returned error: %v", err)
- }
-
- // With embedding-heavy weights the score should differ from default.
- // Just verify the matcher runs and returns valid results.
- if result.ElementCount != 2 {
- t.Errorf("expected ElementCount=2, got %d", result.ElementCount)
- }
- if result.BestRef != "e0" {
- t.Errorf("expected BestRef=e0, got %s", result.BestRef)
- }
-}
-
-func TestCombinedMatcher_NoElements(t *testing.T) {
- m := NewCombinedMatcher(NewHashingEmbedder(128))
-
- result, err := m.Find(context.Background(), "anything", nil, FindOptions{
- Threshold: 0.1,
- TopK: 3,
- })
- if err != nil {
- t.Fatalf("Find returned error: %v", err)
- }
-
- if len(result.Matches) != 0 {
- t.Errorf("expected no matches for empty elements, got %d", len(result.Matches))
- }
- if result.BestRef != "" {
- t.Errorf("expected empty BestRef, got %s", result.BestRef)
- }
-}
-
-// ===========================================================================
-// Phase 3: Complex UI test scenarios
-// ===========================================================================
-
-// complexFormElements returns a realistic form page with 15+ elements.
-func complexFormElements() []ElementDescriptor {
- return []ElementDescriptor{
- {Ref: "e0", Role: "heading", Name: "Registration Form"},
- {Ref: "e1", Role: "textbox", Name: "First Name"},
- {Ref: "e2", Role: "textbox", Name: "Last Name"},
- {Ref: "e3", Role: "textbox", Name: "Email Address"},
- {Ref: "e4", Role: "textbox", Name: "Password", Value: ""},
- {Ref: "e5", Role: "textbox", Name: "Confirm Password"},
- {Ref: "e6", Role: "combobox", Name: "Country"},
- {Ref: "e7", Role: "checkbox", Name: "I agree to the Terms of Service"},
- {Ref: "e8", Role: "checkbox", Name: "Subscribe to newsletter"},
- {Ref: "e9", Role: "button", Name: "Submit Registration"},
- {Ref: "e10", Role: "button", Name: "Cancel"},
- {Ref: "e11", Role: "link", Name: "Already have an account? Log in"},
- {Ref: "e12", Role: "link", Name: "Privacy Policy"},
- {Ref: "e13", Role: "link", Name: "Terms of Service"},
- {Ref: "e14", Role: "img", Name: "Company Logo"},
- {Ref: "e15", Role: "navigation", Name: "Main Navigation"},
- }
-}
-
-// complexTableElements returns a data table with columns and actions.
-func complexTableElements() []ElementDescriptor {
- return []ElementDescriptor{
- {Ref: "e0", Role: "heading", Name: "User Management"},
- {Ref: "e1", Role: "search", Name: "Search Users"},
- {Ref: "e2", Role: "button", Name: "Add New User"},
- {Ref: "e3", Role: "button", Name: "Export CSV"},
- {Ref: "e4", Role: "table", Name: "Users Table"},
- {Ref: "e5", Role: "columnheader", Name: "Name"},
- {Ref: "e6", Role: "columnheader", Name: "Email"},
- {Ref: "e7", Role: "columnheader", Name: "Role"},
- {Ref: "e8", Role: "columnheader", Name: "Status"},
- {Ref: "e9", Role: "columnheader", Name: "Actions"},
- {Ref: "e10", Role: "cell", Name: "John Doe", Value: "[email protected]"},
- {Ref: "e11", Role: "button", Name: "Edit", Value: "John Doe"},
- {Ref: "e12", Role: "button", Name: "Delete", Value: "John Doe"},
- {Ref: "e13", Role: "cell", Name: "Jane Smith", Value: "[email protected]"},
- {Ref: "e14", Role: "button", Name: "Edit", Value: "Jane Smith"},
- {Ref: "e15", Role: "button", Name: "Delete", Value: "Jane Smith"},
- {Ref: "e16", Role: "button", Name: "Previous Page"},
- {Ref: "e17", Role: "button", Name: "Next Page"},
- {Ref: "e18", Role: "combobox", Name: "Rows per page", Value: "10"},
- }
-}
-
-// complexModalElements returns a page with a modal dialog overlay.
-func complexModalElements() []ElementDescriptor {
- return []ElementDescriptor{
- {Ref: "e0", Role: "heading", Name: "Dashboard"},
- {Ref: "e1", Role: "button", Name: "Settings"},
- {Ref: "e2", Role: "button", Name: "Notifications"},
- {Ref: "e3", Role: "dialog", Name: "Confirm Delete"},
- {Ref: "e4", Role: "heading", Name: "Are you sure?"},
- {Ref: "e5", Role: "text", Name: "This action cannot be undone. The item will be permanently deleted."},
- {Ref: "e6", Role: "button", Name: "Yes, Delete"},
- {Ref: "e7", Role: "button", Name: "Cancel"},
- {Ref: "e8", Role: "button", Name: "Close Dialog"},
- {Ref: "e9", Role: "navigation", Name: "Sidebar Menu"},
- {Ref: "e10", Role: "link", Name: "Home"},
- {Ref: "e11", Role: "link", Name: "Reports"},
- {Ref: "e12", Role: "link", Name: "Settings"},
- }
-}
-
-func TestCombinedMatcher_ComplexForm(t *testing.T) {
- m := NewCombinedMatcher(NewHashingEmbedder(128))
- elements := complexFormElements()
-
- tests := []struct {
- query string
- wantRef string
- desc string
- }{
- {"submit registration", "e9", "should find the submit button"},
- {"email field", "e3", "should find email textbox"},
- {"terms checkbox", "e7", "should find terms of service checkbox"},
- {"password input", "e4", "should find password field"},
- {"cancel button", "e10", "should find cancel button"},
- {"log in link", "e11", "should find the login link"},
- {"country dropdown", "e6", "should find country combobox"},
- }
-
- for _, tt := range tests {
- t.Run(tt.desc, func(t *testing.T) {
- result, err := m.Find(context.Background(), tt.query, elements, FindOptions{
- Threshold: 0.01,
- TopK: 3,
- })
- if err != nil {
- t.Fatalf("Find error: %v", err)
- }
- if result.BestRef != tt.wantRef {
- t.Errorf("query=%q: expected BestRef=%s, got %s (score=%f)",
- tt.query, tt.wantRef, result.BestRef, result.BestScore)
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%f role=%s name=%s", m.Ref, m.Score, m.Role, m.Name)
- }
- }
- })
- }
-}
-
-func TestCombinedMatcher_ComplexTable(t *testing.T) {
- m := NewCombinedMatcher(NewHashingEmbedder(128))
- elements := complexTableElements()
-
- tests := []struct {
- query string
- wantRef string
- desc string
- }{
- {"search users", "e1", "should find the search box"},
- {"add new user", "e2", "should find the add button"},
- {"export csv", "e3", "should find the export button"},
- {"next page", "e17", "should find the next page button"},
- }
-
- for _, tt := range tests {
- t.Run(tt.desc, func(t *testing.T) {
- result, err := m.Find(context.Background(), tt.query, elements, FindOptions{
- Threshold: 0.01,
- TopK: 3,
- })
- if err != nil {
- t.Fatalf("Find error: %v", err)
- }
- if result.BestRef != tt.wantRef {
- t.Errorf("query=%q: expected BestRef=%s, got %s (score=%f)",
- tt.query, tt.wantRef, result.BestRef, result.BestScore)
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%f role=%s name=%s", m.Ref, m.Score, m.Role, m.Name)
- }
- }
- })
- }
-}
-
-func TestCombinedMatcher_ComplexModal(t *testing.T) {
- m := NewCombinedMatcher(NewHashingEmbedder(128))
- elements := complexModalElements()
-
- tests := []struct {
- query string
- wantRef string
- desc string
- }{
- {"delete button", "e6", "should find the yes delete button in modal"},
- {"close dialog", "e8", "should find the close dialog button"},
- {"cancel", "e7", "should find the cancel button in modal"},
- {"settings button", "e1", "should find the settings button"},
- }
-
- for _, tt := range tests {
- t.Run(tt.desc, func(t *testing.T) {
- result, err := m.Find(context.Background(), tt.query, elements, FindOptions{
- Threshold: 0.01,
- TopK: 3,
- })
- if err != nil {
- t.Fatalf("Find error: %v", err)
- }
- if result.BestRef != tt.wantRef {
- t.Errorf("query=%q: expected BestRef=%s, got %s (score=%f)",
- tt.query, tt.wantRef, result.BestRef, result.BestScore)
- for _, m := range result.Matches {
- t.Logf(" match: ref=%s score=%f role=%s name=%s", m.Ref, m.Score, m.Role, m.Name)
- }
- }
- })
- }
-}
-
-// ===========================================================================
-// Phase 3: Benchmark tests
-// ===========================================================================
-
-func BenchmarkLexicalMatcher_Find(b *testing.B) {
- m := NewLexicalMatcher()
- elements := complexFormElements()
- opts := FindOptions{Threshold: 0.1, TopK: 3}
- ctx := context.Background()
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- _, _ = m.Find(ctx, "submit registration button", elements, opts)
- }
-}
-
-func BenchmarkHashingEmbedder_Embed(b *testing.B) {
- e := NewHashingEmbedder(128)
- texts := []string{"submit registration button"}
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- _, _ = e.Embed(texts)
- }
-}
-
-func BenchmarkHashingEmbedder_EmbedBatch(b *testing.B) {
- e := NewHashingEmbedder(128)
- elements := complexFormElements()
- texts := make([]string, len(elements))
- for i, el := range elements {
- texts[i] = el.Composite()
- }
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- _, _ = e.Embed(texts)
- }
-}
-
-func BenchmarkEmbeddingMatcher_Find(b *testing.B) {
- m := NewEmbeddingMatcher(NewHashingEmbedder(128))
- elements := complexFormElements()
- opts := FindOptions{Threshold: 0.1, TopK: 3}
- ctx := context.Background()
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- _, _ = m.Find(ctx, "submit registration button", elements, opts)
- }
-}
-
-func BenchmarkCombinedMatcher_Find(b *testing.B) {
- m := NewCombinedMatcher(NewHashingEmbedder(128))
- elements := complexFormElements()
- opts := FindOptions{Threshold: 0.1, TopK: 3}
- ctx := context.Background()
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- _, _ = m.Find(ctx, "submit registration button", elements, opts)
- }
-}
-
-func BenchmarkCombinedMatcher_LargeElementSet(b *testing.B) {
- m := NewCombinedMatcher(NewHashingEmbedder(128))
- // Build a large element set (100 elements) simulating a complex page.
- elements := make([]ElementDescriptor, 100)
- roles := []string{"button", "link", "textbox", "heading", "img", "checkbox", "combobox"}
- for i := 0; i < 100; i++ {
- elements[i] = ElementDescriptor{
- Ref: fmt.Sprintf("e%d", i),
- Role: roles[i%len(roles)],
- Name: fmt.Sprintf("Element %d action item", i),
- }
- }
- opts := FindOptions{Threshold: 0.1, TopK: 5}
- ctx := context.Background()
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- _, _ = m.Find(ctx, "click the action button number 42", elements, opts)
- }
-}
diff --git a/internal/semantic/stopwords.go b/internal/semantic/stopwords.go
deleted file mode 100644
index e3261e5b..00000000
--- a/internal/semantic/stopwords.go
+++ /dev/null
@@ -1,123 +0,0 @@
-package semantic
-
-// stopwords is a set of common English words that carry little semantic
-// meaning and should be excluded from lexical matching to improve
-// signal-to-noise ratio.
-var stopwords = map[string]bool{
- "the": true, "a": true, "an": true, "is": true, "are": true,
- "was": true, "were": true, "be": true, "been": true, "being": true,
- "have": true, "has": true, "had": true, "do": true, "does": true,
- "did": true, "will": true, "would": true, "could": true, "should": true,
- "may": true, "might": true, "shall": true, "can": true,
- "to": true, "of": true, "for": true,
- "with": true, "at": true, "by": true, "from": true, "as": true,
- "into": true, "through": true, "about": true, "above": true,
- "after": true, "before": true, "between": true, "under": true,
- "and": true, "but": true, "nor": true,
- "so": true, "yet": true, "both": true, "either": true, "neither": true,
- "this": true, "that": true, "these": true, "those": true,
- "it": true, "its": true, "i": true, "me": true, "my": true,
- "we": true, "our": true, "you": true, "your": true, "he": true,
- "she": true, "his": true, "her": true, "they": true, "their": true,
-}
-
-// semanticStopwords are words that are normally stopwords but carry
-// meaningful signal in certain UI contexts (e.g. "in" in "sign in",
-// "not" in "do not"). They are removed ONLY if they don't appear in
-// the other side's token set (context-aware removal).
-var semanticStopwords = map[string]bool{
- "in": true, // "sign in", "log in"
- "up": true, // "sign up", "look up"
- "out": true, // "log out", "sign out"
- "on": true, // "log on"
- "off": true, // "log off"
- "not": true, // "do not", "not now"
- "no": true, // negation carries meaning
- "or": true, // disjunction in UI labels
- "ok": true, // acceptance button
-}
-
-// isStopword returns true if the token is a common English stopword.
-func isStopword(token string) bool {
- return stopwords[token]
-}
-
-// isSemanticStopword returns true for words that are semi-stopwords:
-// normally low-value but can carry meaning in UI context.
-func isSemanticStopword(token string) bool {
- return semanticStopwords[token]
-}
-
-// removeStopwords filters out stopwords from a token list.
-// If removal would empty the list, the original tokens are returned
-// to avoid zero-signal matching.
-func removeStopwords(tokens []string) []string {
- filtered := make([]string, 0, len(tokens))
- for _, t := range tokens {
- if !isStopword(t) {
- filtered = append(filtered, t)
- }
- }
- if len(filtered) == 0 {
- return tokens
- }
- return filtered
-}
-
-// removeStopwordsContextAware performs context-aware stopword removal.
-// A word is preserved if:
-// 1. It is not a stopword, OR
-// 2. It IS a semantic stopword AND it appears in the other set of
-// tokens (meaning it carries matching signal), OR
-// 3. It forms part of a known synonym phrase with adjacent tokens
-// (e.g. "sign" + "in" → "sign in" is a synonym entry).
-//
-// Falls back to returning original tokens if everything would be removed.
-func removeStopwordsContextAware(tokens []string, otherTokens []string) []string {
- otherSet := make(map[string]bool, len(otherTokens))
- for _, t := range otherTokens {
- otherSet[t] = true
- }
-
- phraseTokens := make(map[int]bool)
- for n := 2; n <= 3 && n <= len(tokens); n++ {
- for i := 0; i <= len(tokens)-n; i++ {
- joined := ""
- for j := i; j < i+n; j++ {
- if j > i {
- joined += " "
- }
- joined += tokens[j]
- }
- if _, ok := synonymIndex[joined]; ok {
- for j := i; j < i+n; j++ {
- phraseTokens[j] = true
- }
- }
- }
- }
-
- filtered := make([]string, 0, len(tokens))
- for i, t := range tokens {
- switch {
- case !isStopword(t) && !isSemanticStopword(t):
- // Not a stopword at all — always keep.
- filtered = append(filtered, t)
- case phraseTokens[i]:
- // Part of a known synonym phrase — keep it.
- filtered = append(filtered, t)
- case isSemanticStopword(t) && otherSet[t]:
- // Semantic stopword that appears in the other side — keep.
- filtered = append(filtered, t)
- case isSemanticStopword(t) && !isStopword(t):
- // Semantic-only word not in the hard stopword list — keep.
- filtered = append(filtered, t)
- // Pure stopwords are dropped (default case).
- }
- }
-
- if len(filtered) == 0 {
- return tokens
- }
- return filtered
-}
diff --git a/internal/semantic/synonyms.go b/internal/semantic/synonyms.go
deleted file mode 100644
index f196c56e..00000000
--- a/internal/semantic/synonyms.go
+++ /dev/null
@@ -1,281 +0,0 @@
-package semantic
-
-import "strings"
-
-// uiSynonyms maps common UI action/element terms to their synonyms.
-// During lexical scoring the query tokens are expanded with synonyms
-// so that "sign in" can match "log in", "register" can match
-// "create account", etc.
-//
-// Keys are canonical terms (lowercase). Values are slices of
-// equivalent terms that should be treated as matches. The mapping
-// is bidirectional: if "login" maps to "signin", then "signin"
-// also maps back to "login" via the reverse index built at init.
-var uiSynonyms = map[string][]string{
- // Authentication & account actions
- "login": {"signin", "log in", "sign in", "authenticate", "logon", "log on"},
- "logout": {"signout", "log out", "sign out", "logoff"},
- "register": {"signup", "sign up", "create account", "join", "enroll"},
- "password": {"passcode", "passphrase", "pwd"},
- "username": {"userid", "user name", "user id", "login name"},
- "email": {"e-mail", "mail", "email address"},
- "forgot": {"reset", "recover", "lost"},
-
- // Navigation & search
- "search": {"find", "lookup", "look up", "query", "filter"},
- "menu": {"navigation", "nav", "sidebar", "hamburger"},
- "home": {"homepage", "main page", "start", "landing"},
- "back": {"return", "go back", "previous"},
- "next": {"continue", "proceed", "forward", "advance"},
- "previous": {"prev", "back", "prior"},
- "close": {"dismiss", "exit", "x", "cancel"},
- "open": {"expand", "show", "reveal"},
- "settings": {"preferences", "options", "configuration", "config"},
-
- // Form actions
- "submit": {"send", "confirm", "apply", "save", "done", "go"},
- "cancel": {"abort", "discard", "nevermind"},
- "edit": {"modify", "change", "update"},
- "delete": {"remove", "erase", "trash", "discard"},
- "add": {"create", "new", "insert", "plus"},
- "upload": {"attach", "choose file", "browse"},
- "download": {"export", "save as", "get"},
-
- // UI elements
- "button": {"btn", "cta"},
- "input": {"field", "textbox", "text box", "text field"},
- "dropdown": {"select", "combobox", "combo box", "picker", "listbox"},
- "checkbox": {"check box", "tick", "toggle"},
- "link": {"anchor", "hyperlink", "href"},
- "tab": {"panel", "pane"},
- "modal": {"dialog", "dialogue", "popup", "pop up", "overlay"},
- "notification": {"alert", "toast", "banner", "message"},
- "tooltip": {"hint", "info", "help text"},
- "avatar": {"profile picture", "profile pic", "user image", "photo"},
-
- // Shopping & e-commerce
- "cart": {"basket", "bag", "shopping cart"},
- "checkout": {"pay", "payment", "purchase", "buy", "place order", "order", "proceed to payment"},
- "price": {"cost", "amount", "total"},
- "quantity": {"qty", "count", "amount"},
-
- // Content
- "image": {"img", "picture", "photo", "icon"},
- "video": {"clip", "media", "player"},
- "title": {"heading", "header", "headline"},
- "description": {"desc", "summary", "subtitle", "caption"},
- "list": {"items", "collection", "grid"},
-
- // Common actions
- "click": {"press", "tap", "hit", "select"},
- "scroll": {"swipe", "slide"},
- "drag": {"move", "reorder"},
- "copy": {"duplicate", "clone"},
- "paste": {"insert"},
- "undo": {"revert", "rollback"},
- "redo": {"repeat"},
- "refresh": {"reload", "update"},
- "share": {"send", "forward"},
- "like": {"favorite", "favourite", "heart", "star", "upvote"},
- "accept": {"agree", "allow", "ok", "okay", "yes", "confirm"},
- "reject": {"deny", "decline", "refuse", "no"},
-}
-
-// synonymIndex is the flattened bidirectional lookup: given any token
-// it returns all equivalent tokens including those obtained by walking
-// the reverse mapping.
-var synonymIndex map[string]map[string]bool
-
-func init() {
- synonymIndex = buildSynonymIndex(uiSynonyms)
-}
-
-// buildSynonymIndex builds a bidirectional synonym lookup from the
-// canonical synonym table.
-func buildSynonymIndex(table map[string][]string) map[string]map[string]bool {
- idx := make(map[string]map[string]bool)
-
- ensure := func(key string) {
- if idx[key] == nil {
- idx[key] = make(map[string]bool)
- }
- }
-
- for canonical, synonyms := range table {
- ensure(canonical)
- for _, syn := range synonyms {
- idx[canonical][syn] = true
- }
-
- for _, syn := range synonyms {
- ensure(syn)
- idx[syn][canonical] = true
- for _, other := range synonyms {
- if other != syn {
- idx[syn][other] = true
- }
- }
- }
- }
-
- // Handle multi-word entries: also index individual words to
- // the compound form. E.g. "sign in" indexes under "sign" and "in"
- // as joined tokens so that tokenized "sign" + "in" can resolve.
- // This is handled during expansion, not here.
-
- return idx
-}
-
-// expandWithSynonyms takes a set of query tokens and returns an expanded
-// set that includes synonym matches. Multi-word synonyms are split into
-// individual tokens during expansion.
-//
-// The expansion is conservative: only ONE synonym expansion per token is
-// applied (the one that appears in the target description tokens) to
-// avoid combinatorial explosion.
-func expandWithSynonyms(queryTokens []string, descTokens []string) []string {
- descSet := make(map[string]bool, len(descTokens))
- for _, dt := range descTokens {
- descSet[dt] = true
- }
-
- // Also join consecutive query tokens to check multi-word entries.
- // E.g. query ["sign", "in"] -> check "sign in"
- queryPhrases := buildPhrases(queryTokens, 3) // up to 3-word phrases
-
- expanded := make([]string, 0, len(queryTokens)*2)
- usedIndices := make(map[int]bool) // track which query tokens were consumed by phrase expansion
-
- // First pass: try multi-word phrase expansion.
- for _, phrase := range queryPhrases {
- if syns, ok := synonymIndex[phrase.text]; ok {
- for syn := range syns {
- synTokens := strings.Fields(syn)
- for _, st := range synTokens {
- if descSet[st] {
- // This synonym has tokens in the description — add them.
- expanded = append(expanded, synTokens...)
- for idx := phrase.startIdx; idx <= phrase.endIdx; idx++ {
- usedIndices[idx] = true
- }
- break
- }
- }
- }
- }
- }
-
- // Second pass: single-token expansion for tokens not consumed by phrases.
- for i, qt := range queryTokens {
- if usedIndices[i] {
- continue
- }
- expanded = append(expanded, qt)
- if syns, ok := synonymIndex[qt]; ok {
- for syn := range syns {
- synTokens := strings.Fields(syn)
- for _, st := range synTokens {
- if descSet[st] {
- expanded = append(expanded, synTokens...)
- break
- }
- }
- }
- }
- }
-
- return expanded
-}
-
-// phrase represents a multi-word phrase built from consecutive tokens.
-type phrase struct {
- text string
- startIdx int
- endIdx int
-}
-
-// buildPhrases generates all consecutive n-gram phrases (up to maxN words)
-// from a token list.
-func buildPhrases(tokens []string, maxN int) []phrase {
- var phrases []phrase
- for n := 2; n <= maxN && n <= len(tokens); n++ {
- for i := 0; i <= len(tokens)-n; i++ {
- phrases = append(phrases, phrase{
- text: strings.Join(tokens[i:i+n], " "),
- startIdx: i,
- endIdx: i + n - 1,
- })
- }
- }
- return phrases
-}
-
-// synonymScore computes an additional similarity contribution from
-// synonym matching between query and description tokens. Returns a
-// value in [0, 1] representing how much of the query was satisfied
-// via synonyms.
-func synonymScore(queryTokens, descTokens []string) float64 {
- if len(queryTokens) == 0 || len(descTokens) == 0 {
- return 0
- }
-
- descSet := make(map[string]bool, len(descTokens))
- for _, dt := range descTokens {
- descSet[dt] = true
- }
-
- matched := 0
- consumedIdx := make(map[int]bool)
-
- // Phrase matching first (higher priority — avoids double-counting components).
- queryPhrases := buildPhrases(queryTokens, 3)
- for _, p := range queryPhrases {
- if syns, ok := synonymIndex[p.text]; ok {
- for syn := range syns {
- synTokens := strings.Fields(syn)
- allPresent := true
- for _, st := range synTokens {
- if !descSet[st] {
- allPresent = false
- break
- }
- }
- if allPresent {
- matched++
- for idx := p.startIdx; idx <= p.endIdx; idx++ {
- consumedIdx[idx] = true
- }
- break
- }
- }
- }
- }
-
- // Single-token matching for tokens not consumed by phrases.
- for i, qt := range queryTokens {
- if consumedIdx[i] {
- continue
- }
- if descSet[qt] {
- continue
- }
- if syns, ok := synonymIndex[qt]; ok {
- for syn := range syns {
- synTokens := strings.Fields(syn)
- allPresent := true
- for _, st := range synTokens {
- if !descSet[st] {
- allPresent = false
- break
- }
- }
- if allPresent {
- matched++
- break
- }
- }
- }
- }
-
- return float64(matched) / float64(len(queryTokens))
-}