Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix: maxChars slices bytes not characters, injecting U+FFFD into /text and /html responses PIN-022

Luigi Agostic33dd2d05064
diff --git a/internal/handlers/inspect.go b/internal/handlers/inspect.go
index c1e91f05..d38d600e 100644
--- a/internal/handlers/inspect.go
+++ b/internal/handlers/inspect.go
@@ -101,9 +101,8 @@ func (h *Handlers) handleInspect(w http.ResponseWriter, r *http.Request, kind in
}
if kind == inspectKindHTML {
resp.HTML = payload.HTML
- if maxChars := parsePositiveInt(r.URL.Query().Get("maxChars")); maxChars > 0 && len(resp.HTML) > maxChars {
- resp.HTML = resp.HTML[:maxChars]
- resp.Truncated = true
+ if maxChars := parsePositiveInt(r.URL.Query().Get("maxChars")); maxChars > 0 {
+ resp.HTML, resp.Truncated = truncateChars(resp.HTML, maxChars)
}
}
if kind == inspectKindStyles {
diff --git a/internal/handlers/text.go b/internal/handlers/text.go
index 2de6c5cc..7bfbd699 100644
--- a/internal/handlers/text.go
+++ b/internal/handlers/text.go
@@ -16,6 +16,7 @@ import (
"github.com/pinchtab/pinchtab/internal/config"
"github.com/pinchtab/pinchtab/internal/httpx"
"github.com/pinchtab/pinchtab/internal/selector"
+ "unicode/utf8"
)
// @Endpoint GET /text
@@ -166,7 +167,7 @@ func (h *Handlers) extractDocumentText(tCtx context.Context, mode, targetFrameID
if err != nil {
return textExtraction{}, err
}
- return textExtraction{Text: text, Mode: extractionRaw, RawLength: len(text), RawKnown: true}, nil
+ return textExtraction{Text: text, Mode: extractionRaw, RawLength: utf8.RuneCountInString(text), RawKnown: true}, nil
}
text, err := h.extractText(tCtx, assets.ReadabilityJS, targetFrameID)
@@ -178,10 +179,11 @@ func (h *Handlers) extractDocumentText(tCtx context.Context, mode, targetFrameID
if rawErr != nil {
return textExtraction{Text: text, Mode: extractionReadability}, nil
}
- if readabilityCollapsed(len(text), len(raw)) {
- return textExtraction{Text: raw, Mode: extractionReadabilityFallback, RawLength: len(raw), RawKnown: true}, nil
+ extractedLen, rawLen := utf8.RuneCountInString(text), utf8.RuneCountInString(raw)
+ if readabilityCollapsed(extractedLen, rawLen) {
+ return textExtraction{Text: raw, Mode: extractionReadabilityFallback, RawLength: rawLen, RawKnown: true}, nil
}
- return textExtraction{Text: text, Mode: extractionReadability, RawLength: len(raw), RawKnown: true}, nil
+ return textExtraction{Text: text, Mode: extractionReadability, RawLength: rawLen, RawKnown: true}, nil
}
func readabilityCollapsed(extractedLen, rawLen int) bool {
@@ -199,14 +201,30 @@ func (h *Handlers) extractText(tCtx context.Context, script, targetFrameID strin
return h.evalTextInFrame(tCtx, script, targetFrameID)
}
+// truncateChars cuts s to at most limit characters, reporting whether it cut.
+// Slicing by byte splits multi-byte characters and the orphaned bytes reach the
+// client as U+FFFD.
+func truncateChars(s string, limit int) (string, bool) {
+ if limit < 0 || utf8.RuneCountInString(s) <= limit {
+ return s, false
+ }
+ count := 0
+ for offset := range s {
+ if count == limit {
+ return s[:offset], true
+ }
+ count++
+ }
+ return s, false
+}
+
// writeTextResponse truncates, IDPI-scans, and writes the document text as
// plain text (format text/plain) or the JSON envelope.
func (h *Handlers) writeTextResponse(w http.ResponseWriter, r *http.Request, tCtx context.Context, extraction textExtraction, maxChars int, format string, route *browserops.RouteMetadata) {
text := extraction.Text
truncated := false
- if maxChars > -1 && len(text) > maxChars {
- text = text[:maxChars]
- truncated = true
+ if maxChars > -1 {
+ text, truncated = truncateChars(text, maxChars)
}
url, _ := h.Bridge.CurrentURL(tCtx)
@@ -237,7 +255,7 @@ func (h *Handlers) writeTextResponse(w http.ResponseWriter, r *http.Request, tCt
"truncated": truncated,
"route": route,
"extraction": extraction.Mode,
- "textLength": len(text),
+ "textLength": utf8.RuneCountInString(text),
}
if extraction.RawKnown {
resp["rawLength"] = extraction.RawLength
diff --git a/internal/handlers/text_test.go b/internal/handlers/text_test.go
index 15de105f..5209e4b4 100644
--- a/internal/handlers/text_test.go
+++ b/internal/handlers/text_test.go
@@ -10,6 +10,8 @@ import (
"github.com/pinchtab/pinchtab/internal/bridge"
"github.com/pinchtab/pinchtab/internal/config"
+ "strings"
+ "unicode/utf8"
)
func TestHandleText_NoTab(t *testing.T) {
@@ -113,13 +115,14 @@ func TestExtractDocumentText_ReadabilityCollapseFallsBackToRaw(t *testing.T) {
if extraction.Text != landingPageRawText {
t.Errorf("Text = %q, want the raw document text", extraction.Text)
}
- if !extraction.RawKnown || extraction.RawLength != len(landingPageRawText) {
- t.Errorf("RawLength = %d (known=%v), want %d", extraction.RawLength, extraction.RawKnown, len(landingPageRawText))
+ if !extraction.RawKnown || extraction.RawLength != utf8.RuneCountInString(landingPageRawText) {
+ t.Errorf("RawLength = %d (known=%v), want %d", extraction.RawLength, extraction.RawKnown, utf8.RuneCountInString(landingPageRawText))
}
}
func TestExtractDocumentText_ArticleKeepsReadabilityOutput(t *testing.T) {
- article := landingPageRawText[:len(landingPageRawText)*9/10]
+ landingRunes := []rune(landingPageRawText)
+ article := string(landingRunes[:len(landingRunes)*9/10])
m := textScriptBridge(t, article, landingPageRawText)
h := New(m, &config.RuntimeConfig{}, nil, nil, nil)
@@ -133,8 +136,8 @@ func TestExtractDocumentText_ArticleKeepsReadabilityOutput(t *testing.T) {
if extraction.Text != article {
t.Errorf("Text = %q, want the readability output", extraction.Text)
}
- if extraction.RawLength != len(landingPageRawText) {
- t.Errorf("RawLength = %d, want %d", extraction.RawLength, len(landingPageRawText))
+ if extraction.RawLength != utf8.RuneCountInString(landingPageRawText) {
+ t.Errorf("RawLength = %d, want %d", extraction.RawLength, utf8.RuneCountInString(landingPageRawText))
}
}
@@ -192,7 +195,7 @@ func TestWriteTextResponse_EnvelopeReportsExtractionAndLengths(t *testing.T) {
extraction := textExtraction{
Text: landingPageRawText,
Mode: extractionReadabilityFallback,
- RawLength: len(landingPageRawText),
+ RawLength: utf8.RuneCountInString(landingPageRawText),
RawKnown: true,
}
w := textResponseRecorder(t, extraction, -1, "")
@@ -201,11 +204,11 @@ func TestWriteTextResponse_EnvelopeReportsExtractionAndLengths(t *testing.T) {
if envelope["extraction"] != extractionReadabilityFallback {
t.Errorf("extraction = %v, want %q", envelope["extraction"], extractionReadabilityFallback)
}
- if envelope["textLength"] != float64(len(landingPageRawText)) {
- t.Errorf("textLength = %v, want %d", envelope["textLength"], len(landingPageRawText))
+ if envelope["textLength"] != float64(utf8.RuneCountInString(landingPageRawText)) {
+ t.Errorf("textLength = %v, want %d", envelope["textLength"], utf8.RuneCountInString(landingPageRawText))
}
- if envelope["rawLength"] != float64(len(landingPageRawText)) {
- t.Errorf("rawLength = %v, want %d", envelope["rawLength"], len(landingPageRawText))
+ if envelope["rawLength"] != float64(utf8.RuneCountInString(landingPageRawText)) {
+ t.Errorf("rawLength = %v, want %d", envelope["rawLength"], utf8.RuneCountInString(landingPageRawText))
}
if envelope["truncated"] != false {
t.Errorf("truncated = %v, want false: a fallback is not a truncation", envelope["truncated"])
@@ -213,7 +216,7 @@ func TestWriteTextResponse_EnvelopeReportsExtractionAndLengths(t *testing.T) {
}
func TestWriteTextResponse_TruncatedOnlyWhenMaxCharsCuts(t *testing.T) {
- extraction := textExtraction{Text: landingPageRawText, Mode: extractionReadabilityFallback, RawLength: len(landingPageRawText), RawKnown: true}
+ extraction := textExtraction{Text: landingPageRawText, Mode: extractionReadabilityFallback, RawLength: utf8.RuneCountInString(landingPageRawText), RawKnown: true}
cut := decodeTextEnvelope(t, textResponseRecorder(t, extraction, 50, ""))
if cut["truncated"] != true {
@@ -223,14 +226,14 @@ func TestWriteTextResponse_TruncatedOnlyWhenMaxCharsCuts(t *testing.T) {
t.Errorf("textLength = %v, want the returned length 50", cut["textLength"])
}
- whole := decodeTextEnvelope(t, textResponseRecorder(t, extraction, len(landingPageRawText)+1, ""))
+ whole := decodeTextEnvelope(t, textResponseRecorder(t, extraction, utf8.RuneCountInString(landingPageRawText)+1, ""))
if whole["truncated"] != false {
t.Errorf("truncated = %v, want false when maxChars does not cut", whole["truncated"])
}
}
func TestWriteTextResponse_PlainFormatKeepsBareBodyAndReportsModeInHeader(t *testing.T) {
- extraction := textExtraction{Text: landingPageRawText, Mode: extractionReadabilityFallback, RawLength: len(landingPageRawText), RawKnown: true}
+ extraction := textExtraction{Text: landingPageRawText, Mode: extractionReadabilityFallback, RawLength: utf8.RuneCountInString(landingPageRawText), RawKnown: true}
w := textResponseRecorder(t, extraction, -1, "text")
if w.Body.String() != landingPageRawText {
@@ -285,3 +288,28 @@ func TestExtractDocumentText_BaselineUsesTheSameFrameTraversal(t *testing.T) {
t.Errorf("baseline script = %q, want the raw text script", b.scripts[1])
}
}
+
+func TestExtractDocumentText_CoverageFloorCountsCharactersNotBytes(t *testing.T) {
+ raw := strings.Repeat("測", 200)
+ if len(raw) < readabilityCoverageFloorChars {
+ t.Fatalf("fixture must exceed the floor in bytes to discriminate: %d", len(raw))
+ }
+ if utf8.RuneCountInString(raw) >= readabilityCoverageFloorChars {
+ t.Fatalf("fixture must sit under the floor in characters: %d", utf8.RuneCountInString(raw))
+ }
+ collapsed := string([]rune(raw)[:5])
+
+ m := textScriptBridge(t, collapsed, raw)
+ h := New(m, &config.RuntimeConfig{}, nil, nil, nil)
+
+ extraction, err := h.extractDocumentText(context.Background(), "", "")
+ if err != nil {
+ t.Fatalf("extractDocumentText: %v", err)
+ }
+ if extraction.Mode != extractionReadability {
+ t.Fatalf("Mode = %q, want %q: a page under the character floor must not trigger the fallback", extraction.Mode, extractionReadability)
+ }
+ if extraction.RawLength != utf8.RuneCountInString(raw) {
+ t.Fatalf("RawLength = %d, want %d characters", extraction.RawLength, utf8.RuneCountInString(raw))
+ }
+}
diff --git a/internal/handlers/truncate_chars_test.go b/internal/handlers/truncate_chars_test.go
new file mode 100644
index 00000000..136f7f69
--- /dev/null
+++ b/internal/handlers/truncate_chars_test.go
@@ -0,0 +1,150 @@
+package handlers
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "strings"
+ "testing"
+ "unicode/utf8"
+
+ "github.com/pinchtab/pinchtab/internal/bridge"
+ "github.com/pinchtab/pinchtab/internal/config"
+)
+
+// 3-byte (—, U+2014) and 4-byte (🎯, U+1F3AF) characters sit next to ASCII so a
+// cut anywhere in the middle of the string lands inside one of them.
+const multiByteFixture = `Docs — guide 🎯 done`
+
+type htmlInspectBridge struct {
+ mockBridge
+ html string
+}
+
+func (m *htmlInspectBridge) EvaluateInFrame(ctx context.Context, frameID, expression string, result any, opts bridge.EvalOpts) error {
+ payload, ok := result.(*inspectPayload)
+ if !ok {
+ return nil
+ }
+ payload.Title = "fixture"
+ payload.URL = "https://example.test/"
+ payload.HTML = m.html
+ return nil
+}
+
+func htmlEnvelope(t *testing.T, html, query string) map[string]any {
+ t.Helper()
+ h := New(&htmlInspectBridge{html: html}, &config.RuntimeConfig{}, nil, nil, nil)
+ w := httptest.NewRecorder()
+ r := httptest.NewRequest("GET", "/html"+query, nil)
+ h.HandleHTML(w, r)
+ if w.Code != http.StatusOK {
+ t.Fatalf("status = %d body=%s", w.Code, w.Body.String())
+ }
+ var envelope map[string]any
+ if err := json.Unmarshal(w.Body.Bytes(), &envelope); err != nil {
+ t.Fatalf("decode: %v (body=%s)", err, w.Body.String())
+ }
+ return envelope
+}
+
+func assertNoAddedReplacementChars(t *testing.T, source, got string, limit int) {
+ t.Helper()
+ if !utf8.ValidString(got) {
+ t.Fatalf("limit %d produced invalid UTF-8: %q", limit, got)
+ }
+ if strings.Count(got, "�") > strings.Count(source, "�") {
+ t.Fatalf("limit %d added replacement characters: %q", limit, got)
+ }
+}
+
+func TestTruncateChars_CutsOnCharacterBoundariesAtEveryLimit(t *testing.T) {
+ runes := []rune(multiByteFixture)
+ if len(runes) == len(multiByteFixture) {
+ t.Fatalf("fixture has no multi-byte characters: %q", multiByteFixture)
+ }
+
+ for limit := 0; limit <= len(runes)+2; limit++ {
+ got, truncated := truncateChars(multiByteFixture, limit)
+
+ assertNoAddedReplacementChars(t, multiByteFixture, got, limit)
+ want := multiByteFixture
+ if limit < len(runes) {
+ want = string(runes[:limit])
+ }
+ if got != want {
+ t.Fatalf("limit %d: got %q, want %q", limit, got, want)
+ }
+ if wantTruncated := limit < len(runes); truncated != wantTruncated {
+ t.Fatalf("limit %d: truncated = %v, want %v", limit, truncated, wantTruncated)
+ }
+ if count := utf8.RuneCountInString(got); count > limit && limit < len(runes) {
+ t.Fatalf("limit %d returned %d characters", limit, count)
+ }
+ }
+}
+
+func TestTruncateChars_NegativeLimitKeepsTheWholeString(t *testing.T) {
+ got, truncated := truncateChars(multiByteFixture, -1)
+ if got != multiByteFixture || truncated {
+ t.Fatalf("truncateChars(-1) = %q, %v; want the whole string uncut", got, truncated)
+ }
+}
+
+func TestWriteTextResponse_MaxCharsAcrossMultiByteCharacters(t *testing.T) {
+ extraction := textExtraction{
+ Text: multiByteFixture,
+ Mode: extractionRaw,
+ RawLength: utf8.RuneCountInString(multiByteFixture),
+ RawKnown: true,
+ }
+ runes := []rune(multiByteFixture)
+
+ for limit := 0; limit <= len(runes); limit++ {
+ envelope := decodeTextEnvelope(t, textResponseRecorder(t, extraction, limit, ""))
+ text, _ := envelope["text"].(string)
+
+ assertNoAddedReplacementChars(t, multiByteFixture, text, limit)
+ if got := utf8.RuneCountInString(text); got != limit {
+ t.Fatalf("maxChars %d returned %d characters: %q", limit, got, text)
+ }
+ if envelope["textLength"] != float64(utf8.RuneCountInString(text)) {
+ t.Fatalf("maxChars %d: textLength = %v, want %d", limit, envelope["textLength"], utf8.RuneCountInString(text))
+ }
+ }
+}
+
+func TestHandleHTML_MaxCharsAcrossMultiByteCharacters(t *testing.T) {
+ runes := []rune(multiByteFixture)
+
+ for limit := 1; limit <= len(runes); limit++ {
+ envelope := htmlEnvelope(t, multiByteFixture, "?maxChars="+strconv.Itoa(limit))
+ html, _ := envelope["html"].(string)
+
+ assertNoAddedReplacementChars(t, multiByteFixture, html, limit)
+ if got := utf8.RuneCountInString(html); got != limit {
+ t.Fatalf("maxChars %d returned %d characters: %q", limit, got, html)
+ }
+ }
+}
+
+func TestMaxCharsZeroKeepsEachEndpointsOwnMeaning(t *testing.T) {
+ extraction := textExtraction{Text: multiByteFixture, Mode: extractionRaw, RawLength: utf8.RuneCountInString(multiByteFixture), RawKnown: true}
+ envelope := decodeTextEnvelope(t, textResponseRecorder(t, extraction, 0, ""))
+ if text, _ := envelope["text"].(string); text != "" {
+ t.Fatalf("/text maxChars=0 returned %q, want empty: zero truncates to nothing", text)
+ }
+ if envelope["truncated"] != true {
+ t.Fatalf("/text maxChars=0: truncated = %v, want true", envelope["truncated"])
+ }
+
+ htmlResp := htmlEnvelope(t, multiByteFixture, "?maxChars=0")
+ if html, _ := htmlResp["html"].(string); html != multiByteFixture {
+ t.Fatalf("/html maxChars=0 returned %q, want the whole document: zero means no cap", html)
+ }
+ if _, ok := htmlResp["truncated"]; ok {
+ t.Fatalf("/html maxChars=0 reported truncation: %v", htmlResp["truncated"])
+ }
+}