Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

12.8 KB
Raw Blame
// Package semantic provides the adapter between the autosolver system
// and the pinchtab/semantic package. This is the ONLY file in the
// autosolver module that imports github.com/pinchtab/semantic.
package semantic

import (
	"context"
	"fmt"
	"strings"

	"github.com/pinchtab/pinchtab/internal/autosolver"
	"github.com/pinchtab/semantic"
)

// Adapter wraps the pinchtab/semantic ElementMatcher to implement
// autosolver.SemanticEngine. It translates between the autosolver
// type system and the semantic package's types.
type Adapter struct {
	matcher semantic.ElementMatcher
}

// NewAdapter creates a semantic adapter from an existing ElementMatcher.
func NewAdapter(m semantic.ElementMatcher) *Adapter {
	return &Adapter{matcher: m}
}

// captchaRules and flowRules broaden autosolver.DetectChallengeIntent for
// pages it does not classify. Patterns the canonical detector already answers
// must not appear here: it runs first, so they would be unreachable.
var (
	captchaRules = []autosolver.IntentRule{
		{Patterns: []string{"challenge", "verify"}, Type: autosolver.IntentCaptcha, Confidence: 0.8, Details: "captcha detected via semantic analysis"},
	}

	flowRules = []autosolver.IntentRule{
		{Patterns: []string{"log in", "login", "sign in"}, Type: autosolver.IntentLogin, Confidence: 0.7, Details: "login page detected via semantic title analysis"},
		{Patterns: []string{"sign up", "signup", "register", "create account"}, Type: autosolver.IntentSignup, Confidence: 0.7, Details: "signup page detected via semantic title analysis"},
		{Patterns: []string{"getting started", "welcome", "onboarding", "complete your profile", "step 1"}, Type: autosolver.IntentOnboarding, Confidence: 0.65, Details: "onboarding flow detected via semantic title analysis"},
		{Patterns: []string{"continue", "next step", "choose option", "select plan", "wizard"}, Type: autosolver.IntentNavigation, Confidence: 0.6, Details: "navigation flow detected via semantic title analysis"},
		{Patterns: []string{"application form", "contact form", "checkout", "survey", "questionnaire"}, Type: autosolver.IntentForm, Confidence: 0.6, Details: "form flow detected via semantic title analysis"},
	}
)

// DetectIntent classifies the current page state by analyzing the page
// title and URL using semantic matching against known patterns.
func (a *Adapter) DetectIntent(ctx context.Context, page autosolver.Page) (*autosolver.Intent, error) {
	title := strings.ToLower(page.Title())
	url := page.URL()
	html, _ := page.HTML()
	if challenge := autosolver.DetectChallengeIntent(title, url, html); challenge != nil {
		return challenge, nil
	}

	if intent := autosolver.MatchIntentRules(captchaRules, title, strings.ToLower(url)); intent != nil {
		return intent, nil
	}

	if a.matcher != nil {
		intent, err := a.detectViaSemanticMatch(ctx, page)
		if err == nil && intent != nil {
			return intent, nil
		}
	}

	if intent := autosolver.MatchIntentRules(flowRules, title); intent != nil {
		return intent, nil
	}

	return &autosolver.Intent{
		Type:       autosolver.IntentNormal,
		Confidence: 0.6,
		Details:    "no challenge indicators detected",
	}, nil
}

// FindElement uses the semantic matcher to locate a UI element by
// natural-language description.
func (a *Adapter) FindElement(ctx context.Context, page autosolver.Page, query string) (*autosolver.ElementMatch, error) {
	if a.matcher == nil {
		return nil, nil
	}

	// Get page HTML and extract element descriptors.
	// For now we use a simplified approach — the full integration
	// would use the accessibility snapshot like handlers/find.go does.
	html, err := page.HTML()
	if err != nil {
		return nil, err
	}

	descs := extractDescriptors(html)
	if len(descs) == 0 {
		return nil, nil
	}

	result, err := a.matcher.Find(ctx, query, descs, semantic.FindOptions{
		Threshold: 0.3,
		TopK:      1,
	})
	if err != nil {
		return nil, err
	}

	if result.BestRef == "" || len(result.Matches) == 0 {
		return nil, nil
	}

	best := result.Matches[0]
	// Semantic refs are selector-compatible in this adapter, so keep both
	// fields aligned for executors that prefer CSS selectors.
	return &autosolver.ElementMatch{
		Ref:        best.Ref,
		Selector:   best.Ref,
		Role:       best.Role,
		Name:       best.Name,
		Score:      best.Score,
		Confidence: result.ConfidenceLabel(),
	}, nil
}

// SuggestAction determines the next action based on the detected intent
// and available page elements.
func (a *Adapter) SuggestAction(ctx context.Context, page autosolver.Page, intent *autosolver.Intent) (*autosolver.SuggestedAction, error) {
	if intent == nil {
		return &autosolver.SuggestedAction{
			Action: autosolver.ActionNone,
			Reason: "no intent detected",
		}, nil
	}

	switch intent.Type {
	case autosolver.IntentCaptcha:
		return a.suggestCaptchaAction(ctx, page, intent)
	case autosolver.IntentLogin:
		return a.suggestLoginAction(ctx, page)
	case autosolver.IntentSignup:
		return a.suggestSignupAction(ctx, page)
	case autosolver.IntentOnboarding:
		return a.suggestOnboardingAction(ctx, page)
	case autosolver.IntentNavigation:
		return a.suggestNavigationAction(ctx, page)
	case autosolver.IntentForm:
		return a.suggestFormAction(ctx, page)
	case autosolver.IntentBlocked:
		return &autosolver.SuggestedAction{
			Action: autosolver.ActionWait,
			Reason: "page is blocked; waiting for challenge resolution",
		}, nil
	default:
		return &autosolver.SuggestedAction{
			Action: autosolver.ActionNone,
			Reason: "no action needed for current page state",
		}, nil
	}
}

func (a *Adapter) suggestCaptchaAction(ctx context.Context, page autosolver.Page, intent *autosolver.Intent) (*autosolver.SuggestedAction, error) {
	match, err := a.FindElement(ctx, page, "captcha checkbox verify button")
	if err != nil || match == nil {
		return &autosolver.SuggestedAction{
			Action: autosolver.ActionWait,
			Reason: "captcha element not found; waiting for it to appear",
		}, nil
	}

	return &autosolver.SuggestedAction{
		Action:   autosolver.ActionClick,
		Selector: match.Selector,
		X:        match.X,
		Y:        match.Y,
		Reason:   "clicking captcha checkbox",
	}, nil
}

func (a *Adapter) suggestLoginAction(ctx context.Context, page autosolver.Page) (*autosolver.SuggestedAction, error) {
	match, err := a.FindElement(ctx, page, "username email input field")
	if err != nil || match == nil {
		return &autosolver.SuggestedAction{
			Action: autosolver.ActionNone,
			Reason: "login form elements not found",
		}, nil
	}

	return &autosolver.SuggestedAction{
		Action:   autosolver.ActionClick,
		Selector: match.Selector,
		Reason:   "focusing username/email input field",
	}, nil
}

func (a *Adapter) suggestSignupAction(ctx context.Context, page autosolver.Page) (*autosolver.SuggestedAction, error) {
	match, err := a.FindElement(ctx, page, "name email registration input field")
	if err != nil || match == nil {
		return &autosolver.SuggestedAction{
			Action: autosolver.ActionNone,
			Reason: "signup form elements not found",
		}, nil
	}

	return &autosolver.SuggestedAction{
		Action:   autosolver.ActionClick,
		Selector: match.Selector,
		Reason:   "focusing first registration input field",
	}, nil
}

func (a *Adapter) suggestOnboardingAction(ctx context.Context, page autosolver.Page) (*autosolver.SuggestedAction, error) {
	match, err := a.FindElement(ctx, page, "next continue skip done button")
	if err != nil || match == nil {
		return &autosolver.SuggestedAction{
			Action: autosolver.ActionWait,
			Reason: "onboarding controls not found yet",
		}, nil
	}

	return &autosolver.SuggestedAction{
		Action:   autosolver.ActionClick,
		Selector: match.Selector,
		Reason:   "advancing onboarding flow",
	}, nil
}

func (a *Adapter) suggestNavigationAction(ctx context.Context, page autosolver.Page) (*autosolver.SuggestedAction, error) {
	match, err := a.FindElement(ctx, page, "primary navigation button next continue link")
	if err != nil || match == nil {
		return &autosolver.SuggestedAction{
			Action: autosolver.ActionNone,
			Reason: "navigation controls not found",
		}, nil
	}

	return &autosolver.SuggestedAction{
		Action:   autosolver.ActionClick,
		Selector: match.Selector,
		Reason:   "advancing navigation flow",
	}, nil
}

func (a *Adapter) suggestFormAction(ctx context.Context, page autosolver.Page) (*autosolver.SuggestedAction, error) {
	match, err := a.FindElement(ctx, page, "required input field form control")
	if err != nil || match == nil {
		return &autosolver.SuggestedAction{
			Action: autosolver.ActionNone,
			Reason: "form fields not found",
		}, nil
	}

	return &autosolver.SuggestedAction{
		Action:   autosolver.ActionClick,
		Selector: match.Selector,
		Reason:   "focusing a required form field",
	}, nil
}

// detectViaSemanticMatch uses the semantic matcher to classify page
// elements and infer the page intent.
func (a *Adapter) detectViaSemanticMatch(ctx context.Context, page autosolver.Page) (*autosolver.Intent, error) {
	match, err := a.FindElement(ctx, page, "login submit button")
	if err == nil && match != nil && match.Score > 0.6 {
		return &autosolver.Intent{
			Type:       autosolver.IntentLogin,
			Confidence: match.Score,
			Details:    "login form detected via semantic element matching",
		}, nil
	}

	match, err = a.FindElement(ctx, page, "register create account button")
	if err == nil && match != nil && match.Score > 0.6 {
		return &autosolver.Intent{
			Type:       autosolver.IntentSignup,
			Confidence: match.Score,
			Details:    "signup form detected via semantic element matching",
		}, nil
	}

	match, err = a.FindElement(ctx, page, "next continue done button")
	if err == nil && match != nil && match.Score > 0.6 {
		return &autosolver.Intent{
			Type:       autosolver.IntentOnboarding,
			Confidence: match.Score,
			Details:    "onboarding flow detected via semantic element matching",
		}, nil
	}

	match, err = a.FindElement(ctx, page, "primary navigation button link")
	if err == nil && match != nil && match.Score > 0.6 {
		return &autosolver.Intent{
			Type:       autosolver.IntentNavigation,
			Confidence: match.Score,
			Details:    "navigation flow detected via semantic element matching",
		}, nil
	}

	match, err = a.FindElement(ctx, page, "required input field submit button")
	if err == nil && match != nil && match.Score > 0.6 {
		return &autosolver.Intent{
			Type:       autosolver.IntentForm,
			Confidence: match.Score,
			Details:    "form flow detected via semantic element matching",
		}, nil
	}

	return nil, nil
}

// extractDescriptors parses HTML to build minimal element descriptors
// for semantic matching. This is a simplified version — the production
// integration uses the full accessibility snapshot.
func extractDescriptors(html string) []semantic.ElementDescriptor {
	var descs []semantic.ElementDescriptor

	patterns := []struct {
		searchTag string
		selTag    string
		role      string
	}{
		{"<input", "input", "textbox"},
		{"<button", "button", "button"},
		{"<a ", "a", "link"},
		{"<select", "select", "combobox"},
		{"<textarea", "textarea", "textbox"},
	}

	lower := strings.ToLower(html)
	ref := 0
	for _, p := range patterns {
		idx := 0
		for {
			pos := strings.Index(lower[idx:], p.searchTag)
			if pos == -1 {
				break
			}
			idx += pos + len(p.searchTag)

			end := strings.Index(lower[idx:], ">")
			if end == -1 {
				break
			}
			snippet := html[idx : idx+end]
			selector := buildSelector(p.selTag, snippet)

			name := extractAttr(snippet, "placeholder")
			if name == "" {
				name = extractAttr(snippet, "aria-label")
			}
			if name == "" {
				name = extractAttr(snippet, "name")
			}
			if name == "" {
				name = selector
			}

			ref++
			descs = append(descs, semantic.ElementDescriptor{
				Ref:  selector,
				Role: p.role,
				Name: name,
			})

			if ref > 50 {
				return descs
			}
		}
	}
	return descs
}

func buildSelector(tag, snippet string) string {
	id := extractAttr(snippet, "id")
	if id != "" {
		return fmt.Sprintf("#%s", id)
	}

	name := extractAttr(snippet, "name")
	if name != "" {
		return fmt.Sprintf(`%s[name="%s"]`, tag, escapeAttrValue(name))
	}

	placeholder := extractAttr(snippet, "placeholder")
	if placeholder != "" {
		return fmt.Sprintf(`%s[placeholder="%s"]`, tag, escapeAttrValue(placeholder))
	}

	aria := extractAttr(snippet, "aria-label")
	if aria != "" {
		return fmt.Sprintf(`%s[aria-label="%s"]`, tag, escapeAttrValue(aria))
	}

	return tag
}

func escapeAttrValue(v string) string {
	v = strings.ReplaceAll(v, `\\`, `\\\\`)
	v = strings.ReplaceAll(v, `"`, `\\"`)
	return v
}

func extractAttr(snippet, attr string) string {
	lower := strings.ToLower(snippet)
	idx := strings.Index(lower, attr+"=")
	if idx == -1 {
		return ""
	}
	rest := snippet[idx+len(attr)+1:]
	if len(rest) == 0 {
		return ""
	}

	quote := rest[0]
	if quote != '"' && quote != '\'' {
		// Unquoted attribute — take until space or end.
		end := strings.IndexByte(rest, ' ')
		if end == -1 {
			return rest
		}
		return rest[:end]
	}

	end := strings.IndexByte(rest[1:], quote)
	if end == -1 {
		return ""
	}
	return rest[1 : 1+end]
}