High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix: the LLM prompt gets unstripped HTML cut mid-rune while the trim helper sits unused PIN-042
Luigi Agostiec7b2e75388e
diff --git a/internal/autosolver/autosolver.go b/internal/autosolver/autosolver.goindex 4f7db09a..0d72bdd8 100644--- a/internal/autosolver/autosolver.go+++ b/internal/autosolver/autosolver.go@@ -6,6 +6,8 @@ import ("log/slog""sort""time"++ "github.com/pinchtab/pinchtab/internal/htmltrim")// AutoSolver orchestrates the challenge-detection and solving pipeline.@@ -695,15 +697,10 @@ func (as *AutoSolver) tryLLM(ctx context.Context, page Page, executor ActionExecreturn false, entry}- // Trim HTML to reduce token usage (max ~4000 chars).- if len(html) > 4000 {- html = html[:4000]- }-resp, err := as.llm.SuggestNextAction(ctx, LLMRequest{PageTitle: page.Title(),PageURL: page.URL(),- TrimmedHTML: html,+ TrimmedHTML: htmltrim.TrimHTML(html),DetectedType: IntentUnknown,PrevAttempts: history,})diff --git a/internal/autosolver/autosolver_test.go b/internal/autosolver/autosolver_test.goindex cd2ee4e7..3bb620dc 100644--- a/internal/autosolver/autosolver_test.go+++ b/internal/autosolver/autosolver_test.go@@ -3,8 +3,12 @@ package autosolverimport ("context""fmt"+ "strings""testing""time"+ "unicode/utf8"++ "github.com/pinchtab/pinchtab/internal/sanitize")type mockPage struct {@@ -718,6 +722,49 @@ func TestSolve_LLMFallback(t *testing.T) {}}+func TestSolve_LLMRequestCarriesStrippedRuneSafeHTML(t *testing.T) {+ cfg := DefaultConfig()+ cfg.MaxAttempts = 1+ cfg.LLMFallback = true+ cfg.RetryBaseDelay = time.Millisecond++ llm := &mockLLM{resp: &LLMResponse{Action: ActionNone, Confidence: 0.8}}+ as := New(cfg, nil, llm)++ html := "<html><head><style>body{color:red}</style><script>var secret=1;</script></head><body>" ++ strings.Repeat("héllo wörld ", 1000) + "</body></html>"+ page := &mockPage{title: "Just a moment...", url: "https://example.com", html: html}++ if _, err := as.Solve(context.Background(), page, &mockExecutor{}); err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if len(llm.requests) != 1 {+ t.Fatalf("expected a single LLM request, got %d", len(llm.requests))+ }++ sent := llm.requests[0].TrimmedHTML+ for _, unwanted := range []string{"<style", "<script", "color:red", "var secret"} {+ if strings.Contains(sent, unwanted) {+ t.Errorf("TrimmedHTML still contains %q", unwanted)+ }+ }+ if len(sent) >= len(html) {+ t.Errorf("TrimmedHTML was not capped: %d bytes", len(sent))+ }+ if !utf8.ValidString(sent) {+ t.Errorf("TrimmedHTML is not valid UTF-8")+ }+ if strings.ContainsRune(sent, utf8.RuneError) {+ t.Errorf("TrimmedHTML contains U+FFFD absent from the source")+ }+ if strings.HasSuffix(sent, sanitize.TruncationSuffix) {+ t.Errorf("TrimmedHTML got a truncation marker appended: %q", sent[len(sent)-8:])+ }+ if !strings.HasPrefix(sent, "<html><head></head><body>héllo wörld") {+ t.Errorf("TrimmedHTML lost the page body: %q", sent[:40])+ }+}+func TestSolve_ContextCancellation(t *testing.T) {cfg := DefaultConfig()cfg.MaxAttempts = 10diff --git a/internal/autosolver/llm/llm.go b/internal/autosolver/llm/llm.goindex ce9098b6..cf4c6652 100644--- a/internal/autosolver/llm/llm.go+++ b/internal/autosolver/llm/llm.go@@ -44,7 +44,7 @@ func NewProvider(cfg ProviderConfig) *Provider {//// The prompt is designed to be token-efficient:// - Page title + URL (always included)-// - Trimmed HTML (scripts/styles removed, max ~4000 chars)+// - Trimmed HTML (scripts/styles removed, capped in bytes by htmltrim)// - Previous attempt summary (what failed and why)// - Structured output format (action type + parameters)func (p *Provider) SuggestNextAction(ctx context.Context, req autosolver.LLMRequest) (*autosolver.LLMResponse, error) {diff --git a/internal/autosolver/llm/trim.go b/internal/autosolver/llm/trim.godeleted file mode 100644index 0d35a72e..00000000--- a/internal/autosolver/llm/trim.go+++ /dev/null@@ -1,65 +0,0 @@-package llm--import (- "regexp"- "strings"-)--// TrimHTML strips non-essential content from HTML to reduce token usage.-// It removes script tags, style tags, comments, excessive whitespace,-// and non-interactive elements that don't contribute to page understanding.-//-// The goal is to reduce a full page (~50-200KB) to ~4KB while preserving-// interactive elements (forms, buttons, inputs, links) and their attributes.-func TrimHTML(html string) string {- // Remove script tags and their content.- html = reScript.ReplaceAllString(html, "")-- // Remove style tags and their content.- html = reStyle.ReplaceAllString(html, "")-- // Remove HTML comments.- html = reComment.ReplaceAllString(html, "")-- // Remove SVG elements (often large and not useful for automation).- html = reSVG.ReplaceAllString(html, "")-- // Remove data URIs (base64 images inflate size dramatically).- html = reDataURI.ReplaceAllString(html, `""`)-- // Collapse excessive whitespace.- html = reWhitespace.ReplaceAllString(html, " ")-- // Collapse multiple newlines.- html = reNewlines.ReplaceAllString(html, "\n")-- // Trim leading/trailing whitespace on each line.- lines := strings.Split(html, "\n")- var trimmed []string- for _, line := range lines {- line = strings.TrimSpace(line)- if line != "" {- trimmed = append(trimmed, line)- }- }- html = strings.Join(trimmed, "\n")-- // Hard cap to prevent excessive token usage.- if len(html) > maxTrimmedSize {- html = html[:maxTrimmedSize]- }-- return html-}--const maxTrimmedSize = 4000--var (- reScript = regexp.MustCompile(`(?is)<script[^>]*>.*?</script>`)- reStyle = regexp.MustCompile(`(?is)<style[^>]*>.*?</style>`)- reComment = regexp.MustCompile(`(?s)<!--.*?-->`)- reSVG = regexp.MustCompile(`(?is)<svg[^>]*>.*?</svg>`)- reDataURI = regexp.MustCompile(`"data:[^"]*"`)- reWhitespace = regexp.MustCompile(`[ \t]+`)- reNewlines = regexp.MustCompile(`\n{3,}`)-)diff --git a/internal/htmltrim/trim.go b/internal/htmltrim/trim.gonew file mode 100644index 00000000..050aa1cc--- /dev/null+++ b/internal/htmltrim/trim.go@@ -0,0 +1,47 @@+// Package htmltrim reduces page HTML to what an LLM prompt needs: the+// interactive markup, without the scripts, styles and binary payloads that+// dominate a full page's token cost.+package htmltrim++import (+ "regexp"+ "strings"++ "github.com/pinchtab/pinchtab/internal/sanitize"+)++// maxTrimmedBytes caps the result in bytes, cut on a rune boundary so the+// prompt is a byte-exact prefix of valid UTF-8.+const maxTrimmedBytes = 4000++// TrimHTML strips scripts, styles, comments, SVG, data URIs and excess+// whitespace, then caps the result at maxTrimmedBytes.+func TrimHTML(html string) string {+ html = reScript.ReplaceAllString(html, "")+ html = reStyle.ReplaceAllString(html, "")+ html = reComment.ReplaceAllString(html, "")+ html = reSVG.ReplaceAllString(html, "")+ html = reDataURI.ReplaceAllString(html, `""`)+ html = reWhitespace.ReplaceAllString(html, " ")+ html = reNewlines.ReplaceAllString(html, "\n")++ var trimmed []string+ for _, line := range strings.Split(html, "\n") {+ line = strings.TrimSpace(line)+ if line != "" {+ trimmed = append(trimmed, line)+ }+ }++ return sanitize.PrefixUTF8Bytes(strings.Join(trimmed, "\n"), maxTrimmedBytes)+}++var (+ reScript = regexp.MustCompile(`(?is)<script[^>]*>.*?</script>`)+ reStyle = regexp.MustCompile(`(?is)<style[^>]*>.*?</style>`)+ reComment = regexp.MustCompile(`(?s)<!--.*?-->`)+ reSVG = regexp.MustCompile(`(?is)<svg[^>]*>.*?</svg>`)+ reDataURI = regexp.MustCompile(`"data:[^"]*"`)+ reWhitespace = regexp.MustCompile(`[ \t]+`)+ reNewlines = regexp.MustCompile(`\n{3,}`)+)diff --git a/internal/htmltrim/trim_test.go b/internal/htmltrim/trim_test.gonew file mode 100644index 00000000..683be7d2--- /dev/null+++ b/internal/htmltrim/trim_test.go@@ -0,0 +1,55 @@+package htmltrim++import (+ "strings"+ "testing"+ "unicode/utf8"+)++func TestTrimHTMLStripsScriptsAndStyles(t *testing.T) {+ html := `<html><head><style>body{color:red}</style><script>var secret=1;</script></head>` ++ `<body><!-- a comment --><svg><path d="M0 0"/></svg><img src="data:image/png;base64,AAAA">` ++ `<button id="go">Go</button></body></html>`++ got := TrimHTML(html)++ for _, unwanted := range []string{"<style", "<script", "color:red", "var secret", "<!--", "<svg", "base64"} {+ if strings.Contains(got, unwanted) {+ t.Errorf("trimmed HTML still contains %q:\n%s", unwanted, got)+ }+ }+ if !strings.Contains(got, `<button id="go">Go</button>`) {+ t.Errorf("trimmed HTML dropped the interactive element:\n%s", got)+ }+}++func TestTrimHTMLCapsOnRuneBoundary(t *testing.T) {+ body := strings.Repeat("é", maxTrimmedBytes)+ got := TrimHTML("<p>" + body + "</p>")++ if len(got) > maxTrimmedBytes {+ t.Fatalf("trimmed HTML is %d bytes, over the %d-byte cap", len(got), maxTrimmedBytes)+ }+ if len(got) < maxTrimmedBytes-utf8.UTFMax {+ t.Fatalf("trimmed HTML is %d bytes, far under the %d-byte cap", len(got), maxTrimmedBytes)+ }+ if !utf8.ValidString(got) {+ t.Fatalf("trimmed HTML is not valid UTF-8: %q", got[len(got)-8:])+ }+ if strings.ContainsRune(got, utf8.RuneError) {+ t.Fatalf("trimmed HTML contains U+FFFD absent from the source")+ }+ if strings.HasSuffix(got, ".") {+ t.Fatalf("trimmed HTML got a truncation marker appended: %q", got[len(got)-8:])+ }+ if !strings.HasPrefix("<p>"+body+"</p>", got) {+ t.Fatalf("trimmed HTML is not a byte-exact prefix of the source")+ }+}++func TestTrimHTMLLeavesShortInputUncapped(t *testing.T) {+ html := "<p>héllo</p>"+ if got := TrimHTML(html); got != html {+ t.Fatalf("TrimHTML(%q) = %q, want it unchanged", html, got)+ }+}diff --git a/internal/sanitize/text.go b/internal/sanitize/text.goindex ce8ced38..2e3505b3 100644--- a/internal/sanitize/text.go+++ b/internal/sanitize/text.go@@ -14,29 +14,37 @@ var (winAbs = regexp.MustCompile(`(^|[\s"'(=:])([A-Za-z]:\\(?:[^\s"'():;<>{}\[\]]+\\?)+)`))-func TruncateUTF8Bytes(s string, maxBytes int) string {+// PrefixUTF8Bytes returns the longest prefix of s that fits in maxBytes bytes+// without splitting a rune. Nothing is appended: callers that want a+// truncation marker use TruncateUTF8Bytes.+func PrefixUTF8Bytes(s string, maxBytes int) string {if maxBytes <= 0 {return ""}if len(s) <= maxBytes {return s}- if maxBytes <= len(TruncationSuffix) {- return TruncationSuffix[:maxBytes]- }-- limit := maxBytes - len(TruncationSuffix)cut := 0for i := range s {- if i > limit {+ if i > maxBytes {break}cut = i}- if cut == 0 && limit > 0 {- return TruncationSuffix+ return s[:cut]+}++func TruncateUTF8Bytes(s string, maxBytes int) string {+ if maxBytes <= 0 {+ return ""+ }+ if len(s) <= maxBytes {+ return s+ }+ if maxBytes <= len(TruncationSuffix) {+ return TruncationSuffix[:maxBytes]}- return s[:cut] + TruncationSuffix+ return PrefixUTF8Bytes(s, maxBytes-len(TruncationSuffix)) + TruncationSuffix}func StripANSI(s string) string {diff --git a/internal/sanitize/text_test.go b/internal/sanitize/text_test.goindex 5bbb07a3..56a05374 100644--- a/internal/sanitize/text_test.go+++ b/internal/sanitize/text_test.go@@ -1,6 +1,10 @@package sanitize-import "testing"+import (+ "strings"+ "testing"+ "unicode/utf8"+)func TestCleanErrorRedactsAbsolutePaths(t *testing.T) {tests := []struct {@@ -48,3 +52,36 @@ func TestCleanErrorRedactsAbsolutePaths(t *testing.T) {})}}++func TestPrefixUTF8BytesCutsOnRuneBoundaryWithoutMarker(t *testing.T) {+ const s = "héllo"++ tests := []struct {+ name string+ maxBytes int+ want string+ }{+ {name: "whole string fits", maxBytes: len(s), want: s},+ {name: "budget beyond the string", maxBytes: len(s) + 10, want: s},+ {name: "boundary before a two-byte rune", maxBytes: 1, want: "h"},+ {name: "budget lands inside a two-byte rune", maxBytes: 2, want: "h"},+ {name: "budget lands after a two-byte rune", maxBytes: 3, want: "hé"},+ {name: "zero budget", maxBytes: 0, want: ""},+ {name: "negative budget", maxBytes: -1, want: ""},+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := PrefixUTF8Bytes(s, tt.maxBytes)+ if got != tt.want {+ t.Fatalf("PrefixUTF8Bytes(%q, %d) = %q, want %q", s, tt.maxBytes, got, tt.want)+ }+ if !utf8.ValidString(got) {+ t.Fatalf("PrefixUTF8Bytes(%q, %d) is not valid UTF-8", s, tt.maxBytes)+ }+ if strings.Contains(got, TruncationSuffix) {+ t.Fatalf("PrefixUTF8Bytes appended a truncation marker: %q", got)+ }+ })+ }+}