Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix: /text default Readability mode can drop 97% of a page and reports truncated:false PIN-012

Luigi Agostid0290661db7f
diff --git a/docs/reference/text.md b/docs/reference/text.md
index f7c5a6f0..011dfb58 100644
--- a/docs/reference/text.md
+++ b/docs/reference/text.md
@@ -38,6 +38,26 @@ pinchtab text # Plain text output
pinchtab text --json # JSON: {"url":"...","title":"...","text":"..."}
```
+## Extraction Mode
+
+Readability is an article heuristic, so on a landing page, dashboard, or docs
+index it can return one block instead of the page. `/text` measures its output
+against `document.body.innerText` and, when the coverage is too low, returns the
+raw text instead. The JSON envelope always says which extractor produced the
+text:
+
+| Field | Description |
+|-------|-------------|
+| `extraction` | `readability`, `raw` (explicitly requested), or `readability_fallback` (Readability collapsed, raw text returned) |
+| `textLength` | Length of the returned text |
+| `rawLength` | Length of `document.body.innerText`, so coverage is computable |
+
+With `format=text` the body stays bare and the mode is reported in the
+`X-PT-Text-Extraction` response header. `mode=raw` never runs the comparison.
+`truncated` keeps its meaning — text cut by `maxChars` — and is unaffected by a
+fallback. The CLI prints a one-line note on stderr when a fallback fired; stdout
+stays text-only.
+
## Examples
```bash
diff --git a/internal/cli/actions/actions_text.go b/internal/cli/actions/actions_text.go
index d3cc668c..37523bb2 100644
--- a/internal/cli/actions/actions_text.go
+++ b/internal/cli/actions/actions_text.go
@@ -57,11 +57,19 @@ func Text(client *http.Client, base, token string, cmd *cobra.Command, args []st
body := apiclient.DoGetRaw(client, base, token, "/text", params)
var result struct {
- Text string `json:"text"`
+ Text string `json:"text"`
+ Extraction string `json:"extraction"`
}
if err := json.Unmarshal(body, &result); err != nil {
output.Value(string(body))
return
}
+ if result.Extraction == extractionReadabilityFallback {
+ output.Hint("readability extracted only a fragment of this page; returned the full page text instead (pass --full to request it directly)")
+ }
output.Value(result.Text)
}
+
+// extractionReadabilityFallback mirrors the /text envelope value the server
+// echoes when readability collapsed and it returned the raw document instead.
+const extractionReadabilityFallback = "readability_fallback"
diff --git a/internal/cli/actions/actions_text_test.go b/internal/cli/actions/actions_text_test.go
index cba9d7e0..e8aa737a 100644
--- a/internal/cli/actions/actions_text_test.go
+++ b/internal/cli/actions/actions_text_test.go
@@ -1,6 +1,10 @@
package actions
import (
+ "bytes"
+ "encoding/json"
+ "io"
+ "os"
"strings"
"testing"
@@ -88,3 +92,81 @@ func TestTextTab(t *testing.T) {
t.Errorf("expected tabId=TAB1, got %s", m.lastQuery)
}
}
+
+func captureStderr(t *testing.T, fn func()) string {
+ t.Helper()
+
+ oldStderr := os.Stderr
+ reader, writer, err := os.Pipe()
+ if err != nil {
+ t.Fatalf("os.Pipe: %v", err)
+ }
+ os.Stderr = writer
+
+ var buf bytes.Buffer
+ done := make(chan error, 1)
+ go func() {
+ _, copyErr := io.Copy(&buf, reader)
+ _ = reader.Close()
+ done <- copyErr
+ }()
+
+ defer func() { os.Stderr = oldStderr }()
+
+ fn()
+
+ _ = writer.Close()
+ if err := <-done; err != nil {
+ t.Fatalf("io.Copy: %v", err)
+ }
+ return buf.String()
+}
+
+func runTextAgainst(t *testing.T, response string) (stdout, stderr string) {
+ t.Helper()
+ m := newMockServer()
+ m.response = response
+ defer m.close()
+
+ stderr = captureStderr(t, func() {
+ stdout = captureStdout(t, func() {
+ Text(m.server.Client(), m.base(), "", newTextCmd(), nil)
+ })
+ })
+ return stdout, stderr
+}
+
+func TestTextFallbackNotesOnStderrAndKeepsStdoutClean(t *testing.T) {
+ page := "PinchTab — browser control for AI agents\nInstall: curl -fsSL https://pinchtab.com/install.sh | sh"
+ body, err := json.Marshal(map[string]any{
+ "text": page,
+ "extraction": "readability_fallback",
+ "textLength": len(page),
+ "rawLength": len(page),
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ stdout, stderr := runTextAgainst(t, string(body))
+
+ if stdout != page+"\n" {
+ t.Errorf("stdout = %q, want the page text alone", stdout)
+ }
+ if !strings.Contains(stderr, "--full") {
+ t.Errorf("stderr = %q, want a note pointing at --full", stderr)
+ }
+}
+
+func TestTextNoNoteWhenReadabilityHeld(t *testing.T) {
+ body := `{"text":"Article body","extraction":"readability","textLength":12,"rawLength":13}`
+
+ stdout, stderr := runTextAgainst(t, body)
+
+ if stdout != "Article body\n" {
+ t.Errorf("stdout = %q, want the article text", stdout)
+ }
+ if stderr != "" {
+ t.Errorf("stderr = %q, want no note when readability held", stderr)
+ }
+}
diff --git a/internal/handlers/text.go b/internal/handlers/text.go
index b20d2730..2de6c5cc 100644
--- a/internal/handlers/text.go
+++ b/internal/handlers/text.go
@@ -68,13 +68,15 @@ func (h *Handlers) HandleText(w http.ResponseWriter, r *http.Request) {
}
var text string
+ var extraction textExtraction
if selectorParam != "" || refParam != "" {
text, err = h.extractElementText(tCtx, resolvedTabID, selectorParam, refParam, modalNodeID)
} else if modalOpen {
err = h.Bridge.CallFunctionOnNode(tCtx, modalNodeID,
`function() { return this.innerText || this.textContent || ''; }`, nil, &text)
+ extraction = textExtraction{Text: text, Mode: extractionRaw, RawLength: len(text), RawKnown: true}
} else {
- text, err = h.extractDocumentText(tCtx, mode, targetFrameID)
+ extraction, err = h.extractDocumentText(tCtx, mode, targetFrameID)
}
stable := true
@@ -101,7 +103,7 @@ func (h *Handlers) HandleText(w http.ResponseWriter, r *http.Request) {
if selectorParam != "" || refParam != "" {
h.writeElementTextResponse(w, r, tCtx, text)
} else {
- h.writeTextResponse(w, r, tCtx, text, maxChars, format, textRoute)
+ h.writeTextResponse(w, r, tCtx, extraction, maxChars, format, textRoute)
}
return
}
@@ -120,25 +122,87 @@ func (h *Handlers) writeElementTextResponse(w http.ResponseWriter, r *http.Reque
})
}
+const rawTextScript = `document.body.innerText`
+
+// Extraction modes echoed to the caller: which extractor produced the text.
+const (
+ extractionReadability = "readability"
+ extractionRaw = "raw"
+ extractionReadabilityFallback = "readability_fallback"
+)
+
+// Readability is an article heuristic; on a layout it does not recognise as an
+// article (landing page, dashboard, docs index) it returns the single block it
+// scored highest. Measured against document.body.innerText, a collapse retains
+// ~3% of the page while healthy article extraction retains ~91%, so any ratio in
+// the low tens of percent separates the two with wide margin on both sides.
+const readabilityCoverageRatio = 0.25
+
+// Under this many raw characters the ratio is noise — a short page legitimately
+// extracts to a couple of lines — so no fallback fires below it.
+const readabilityCoverageFloorChars = 400
+
+// headerTextExtraction reports the extraction mode on the format=text path,
+// which returns a bare body with nowhere to carry the signal.
+const headerTextExtraction = "X-PT-Text-Extraction"
+
+// textExtraction is the document text plus the story of how it was produced:
+// the mode that actually ran and the length of the raw document it was measured
+// against (RawKnown is false when the baseline extraction itself failed).
+type textExtraction struct {
+ Text string
+ Mode string
+ RawLength int
+ RawKnown bool
+}
+
// extractDocumentText reads the document's text (readability unless mode=="raw")
-// across all reachable frames, or scoped to targetFrameID when set. The
-// cross-frame path silently skips cross-origin frames and never errors.
-func (h *Handlers) extractDocumentText(tCtx context.Context, mode, targetFrameID string) (string, error) {
- script := `document.body.innerText`
- if mode != "raw" {
- script = assets.ReadabilityJS
+// across all reachable frames, or scoped to targetFrameID when set. Readability
+// output that covers too little of the raw document is discarded in favour of
+// the raw text, reported as extractionReadabilityFallback.
+func (h *Handlers) extractDocumentText(tCtx context.Context, mode, targetFrameID string) (textExtraction, error) {
+ if mode == "raw" {
+ text, err := h.extractText(tCtx, rawTextScript, targetFrameID)
+ if err != nil {
+ return textExtraction{}, err
+ }
+ return textExtraction{Text: text, Mode: extractionRaw, RawLength: len(text), RawKnown: true}, nil
}
+
+ text, err := h.extractText(tCtx, assets.ReadabilityJS, targetFrameID)
+ if err != nil {
+ return textExtraction{}, err
+ }
+
+ raw, rawErr := h.extractText(tCtx, rawTextScript, targetFrameID)
+ if rawErr != nil {
+ return textExtraction{Text: text, Mode: extractionReadability}, nil
+ }
+ if readabilityCollapsed(len(text), len(raw)) {
+ return textExtraction{Text: raw, Mode: extractionReadabilityFallback, RawLength: len(raw), RawKnown: true}, nil
+ }
+ return textExtraction{Text: text, Mode: extractionReadability, RawLength: len(raw), RawKnown: true}, nil
+}
+
+func readabilityCollapsed(extractedLen, rawLen int) bool {
+ return rawLen >= readabilityCoverageFloorChars && float64(extractedLen) < float64(rawLen)*readabilityCoverageRatio
+}
+
+// extractText runs one extraction script through the traversal the request
+// selected: every reachable frame (joined), or the scoped frame's isolated world
+// so the expression sees the iframe's `document`, not the parent's. Both sides of
+// the coverage ratio go through here, so they are always built identically.
+func (h *Handlers) extractText(tCtx context.Context, script, targetFrameID string) (string, error) {
if targetFrameID == "" {
return h.extractTextAllFrames(tCtx, script), nil
}
- // Frame-scoped path — evaluate in the frame's isolated world so the
- // expression sees the iframe's `document`, not the parent's.
return h.evalTextInFrame(tCtx, script, targetFrameID)
}
// writeTextResponse truncates, IDPI-scans, and writes the document text as
// plain text (format text/plain) or the JSON envelope.
-func (h *Handlers) writeTextResponse(w http.ResponseWriter, r *http.Request, tCtx context.Context, text string, maxChars int, format string, route *browserops.RouteMetadata) {
+func (h *Handlers) writeTextResponse(w http.ResponseWriter, r *http.Request, tCtx context.Context, extraction textExtraction, maxChars int, format string, route *browserops.RouteMetadata) {
+ text := extraction.Text
truncated := false
if maxChars > -1 && len(text) > maxChars {
text = text[:maxChars]
@@ -157,6 +221,7 @@ func (h *Handlers) writeTextResponse(w http.ResponseWriter, r *http.Request, tCt
}
result.SetHeaders(w)
text = result.Text
+ w.Header().Set(headerTextExtraction, extraction.Mode)
if format == "text" || format == "plain" {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
@@ -166,11 +231,16 @@ func (h *Handlers) writeTextResponse(w http.ResponseWriter, r *http.Request, tCt
}
resp := map[string]any{
- "url": url,
- "title": title,
- "text": text,
- "truncated": truncated,
- "route": route,
+ "url": url,
+ "title": title,
+ "text": text,
+ "truncated": truncated,
+ "route": route,
+ "extraction": extraction.Mode,
+ "textLength": len(text),
+ }
+ if extraction.RawKnown {
+ resp["rawLength"] = extraction.RawLength
}
if result.Warning != "" {
resp["idpiWarning"] = result.Warning
diff --git a/internal/handlers/text_test.go b/internal/handlers/text_test.go
index 90a4cc9b..15de105f 100644
--- a/internal/handlers/text_test.go
+++ b/internal/handlers/text_test.go
@@ -1,10 +1,14 @@
package handlers
import (
+ "context"
+ "encoding/json"
+ "fmt"
"net/http"
"net/http/httptest"
"testing"
+ "github.com/pinchtab/pinchtab/internal/bridge"
"github.com/pinchtab/pinchtab/internal/config"
)
@@ -58,3 +62,226 @@ func TestHandleTabText_NoTab(t *testing.T) {
t.Errorf("expected 404, got %d", w.Code)
}
}
+
+const landingPageRawText = `PinchTab — browser control for AI agents
+
+Install: curl -fsSL https://pinchtab.com/install.sh | sh
+
+Endpoints: /navigate /snapshot /text /action /capture /find /evaluate
+
+🎯 Accessibility Tree — structured tree with stable refs (e0, e1...) for click, type, and read. Deterministic with no coordinate guessing.
+🧭 Navigation — drive tabs, frames, and history without a headful browser in the loop.
+📄 Text extraction — readable page text with an explicit raw mode when the heuristic collapses.
+🖼️ Capture — screenshots and PDFs of the live page, viewport or full document.
+🔍 Find — semantic element lookup that survives markup churn.
+🧪 Evaluate — run JavaScript in the page or in a specific frame.
+🔐 Profiles — reuse a dedicated automation profile with explicit approval.
+🧰 Macros — batch several actions into one request.
+📊 Activity — every request recorded with the route that served it.`
+
+const collapsedReadabilityText = `🎯 Accessibility Tree — structured tree with stable refs (e0, e1...) for click, type, and read.`
+
+func textScriptBridge(t *testing.T, readability, raw string) *mockBridge {
+ t.Helper()
+ return &mockBridge{
+ evaluateFn: func(expression string, result any) error {
+ out, ok := result.(*string)
+ if !ok {
+ t.Fatalf("evaluate result is %T, want *string", result)
+ }
+ if expression == rawTextScript {
+ *out = raw
+ return nil
+ }
+ *out = readability
+ return nil
+ },
+ }
+}
+
+func TestExtractDocumentText_ReadabilityCollapseFallsBackToRaw(t *testing.T) {
+ m := textScriptBridge(t, collapsedReadabilityText, landingPageRawText)
+ h := New(m, &config.RuntimeConfig{}, nil, nil, nil)
+
+ extraction, err := h.extractDocumentText(context.Background(), "", "")
+ if err != nil {
+ t.Fatalf("extractDocumentText: %v", err)
+ }
+ if extraction.Mode != extractionReadabilityFallback {
+ t.Errorf("Mode = %q, want %q", extraction.Mode, extractionReadabilityFallback)
+ }
+ if extraction.Text != landingPageRawText {
+ t.Errorf("Text = %q, want the raw document text", extraction.Text)
+ }
+ if !extraction.RawKnown || extraction.RawLength != len(landingPageRawText) {
+ t.Errorf("RawLength = %d (known=%v), want %d", extraction.RawLength, extraction.RawKnown, len(landingPageRawText))
+ }
+}
+
+func TestExtractDocumentText_ArticleKeepsReadabilityOutput(t *testing.T) {
+ article := landingPageRawText[:len(landingPageRawText)*9/10]
+ m := textScriptBridge(t, article, landingPageRawText)
+ h := New(m, &config.RuntimeConfig{}, nil, nil, nil)
+
+ extraction, err := h.extractDocumentText(context.Background(), "", "")
+ if err != nil {
+ t.Fatalf("extractDocumentText: %v", err)
+ }
+ if extraction.Mode != extractionReadability {
+ t.Errorf("Mode = %q, want %q", extraction.Mode, extractionReadability)
+ }
+ if extraction.Text != article {
+ t.Errorf("Text = %q, want the readability output", extraction.Text)
+ }
+ if extraction.RawLength != len(landingPageRawText) {
+ t.Errorf("RawLength = %d, want %d", extraction.RawLength, len(landingPageRawText))
+ }
+}
+
+func TestExtractDocumentText_RawModeSkipsCoverageComparison(t *testing.T) {
+ m := textScriptBridge(t, collapsedReadabilityText, landingPageRawText)
+ h := New(m, &config.RuntimeConfig{}, nil, nil, nil)
+
+ extraction, err := h.extractDocumentText(context.Background(), "raw", "")
+ if err != nil {
+ t.Fatalf("extractDocumentText: %v", err)
+ }
+ if extraction.Mode != extractionRaw {
+ t.Errorf("Mode = %q, want %q", extraction.Mode, extractionRaw)
+ }
+ if len(m.evaluateExprs) != 1 || m.evaluateExprs[0] != rawTextScript {
+ t.Fatalf("mode=raw ran %d extractions (%v), want exactly the raw one", len(m.evaluateExprs), m.evaluateExprs)
+ }
+}
+
+func TestReadabilityCollapsed_FloorKeepsShortPages(t *testing.T) {
+ if readabilityCollapsed(1, readabilityCoverageFloorChars-1) {
+ t.Error("a page below the character floor must not trigger the fallback")
+ }
+ if !readabilityCollapsed(1, readabilityCoverageFloorChars) {
+ t.Error("a near-total collapse at the floor must trigger the fallback")
+ }
+ atRatio := int(float64(readabilityCoverageFloorChars) * readabilityCoverageRatio)
+ if readabilityCollapsed(atRatio, readabilityCoverageFloorChars) {
+ t.Error("coverage at the ratio must not trigger the fallback")
+ }
+}
+
+func textResponseRecorder(t *testing.T, extraction textExtraction, maxChars int, format string) *httptest.ResponseRecorder {
+ t.Helper()
+ h := New(&mockBridge{}, &config.RuntimeConfig{}, nil, nil, nil)
+ w := httptest.NewRecorder()
+ r := httptest.NewRequest("GET", "/text", nil)
+ h.writeTextResponse(w, r, context.Background(), extraction, maxChars, format, nil)
+ if w.Code != http.StatusOK {
+ t.Fatalf("status = %d body=%s", w.Code, w.Body.String())
+ }
+ return w
+}
+
+func decodeTextEnvelope(t *testing.T, w *httptest.ResponseRecorder) map[string]any {
+ t.Helper()
+ var envelope map[string]any
+ if err := json.Unmarshal(w.Body.Bytes(), &envelope); err != nil {
+ t.Fatalf("decode: %v (body=%s)", err, w.Body.String())
+ }
+ return envelope
+}
+
+func TestWriteTextResponse_EnvelopeReportsExtractionAndLengths(t *testing.T) {
+ extraction := textExtraction{
+ Text: landingPageRawText,
+ Mode: extractionReadabilityFallback,
+ RawLength: len(landingPageRawText),
+ RawKnown: true,
+ }
+ w := textResponseRecorder(t, extraction, -1, "")
+ envelope := decodeTextEnvelope(t, w)
+
+ if envelope["extraction"] != extractionReadabilityFallback {
+ t.Errorf("extraction = %v, want %q", envelope["extraction"], extractionReadabilityFallback)
+ }
+ if envelope["textLength"] != float64(len(landingPageRawText)) {
+ t.Errorf("textLength = %v, want %d", envelope["textLength"], len(landingPageRawText))
+ }
+ if envelope["rawLength"] != float64(len(landingPageRawText)) {
+ t.Errorf("rawLength = %v, want %d", envelope["rawLength"], len(landingPageRawText))
+ }
+ if envelope["truncated"] != false {
+ t.Errorf("truncated = %v, want false: a fallback is not a truncation", envelope["truncated"])
+ }
+}
+
+func TestWriteTextResponse_TruncatedOnlyWhenMaxCharsCuts(t *testing.T) {
+ extraction := textExtraction{Text: landingPageRawText, Mode: extractionReadabilityFallback, RawLength: len(landingPageRawText), RawKnown: true}
+
+ cut := decodeTextEnvelope(t, textResponseRecorder(t, extraction, 50, ""))
+ if cut["truncated"] != true {
+ t.Errorf("truncated = %v, want true when maxChars cuts", cut["truncated"])
+ }
+ if cut["textLength"] != float64(50) {
+ t.Errorf("textLength = %v, want the returned length 50", cut["textLength"])
+ }
+
+ whole := decodeTextEnvelope(t, textResponseRecorder(t, extraction, len(landingPageRawText)+1, ""))
+ if whole["truncated"] != false {
+ t.Errorf("truncated = %v, want false when maxChars does not cut", whole["truncated"])
+ }
+}
+
+func TestWriteTextResponse_PlainFormatKeepsBareBodyAndReportsModeInHeader(t *testing.T) {
+ extraction := textExtraction{Text: landingPageRawText, Mode: extractionReadabilityFallback, RawLength: len(landingPageRawText), RawKnown: true}
+ w := textResponseRecorder(t, extraction, -1, "text")
+
+ if w.Body.String() != landingPageRawText {
+ t.Errorf("body = %q, want the bare text", w.Body.String())
+ }
+ if got := w.Header().Get(headerTextExtraction); got != extractionReadabilityFallback {
+ t.Errorf("%s = %q, want %q", headerTextExtraction, got, extractionReadabilityFallback)
+ }
+}
+
+type frameTextBridge struct {
+ mockBridge
+ frames []string
+ scripts []string
+ readability string
+ raw string
+}
+
+func (b *frameTextBridge) EvaluateInFrame(_ context.Context, frameID, expression string, result any, _ bridge.EvalOpts) error {
+ b.frames = append(b.frames, frameID)
+ b.scripts = append(b.scripts, expression)
+ out, ok := result.(*string)
+ if !ok {
+ return fmt.Errorf("evaluate result is %T, want *string", result)
+ }
+ if expression == rawTextScript {
+ *out = b.raw
+ return nil
+ }
+ *out = b.readability
+ return nil
+}
+
+func TestExtractDocumentText_BaselineUsesTheSameFrameTraversal(t *testing.T) {
+ b := &frameTextBridge{readability: collapsedReadabilityText, raw: landingPageRawText}
+ h := New(b, &config.RuntimeConfig{}, nil, nil, nil)
+
+ extraction, err := h.extractDocumentText(context.Background(), "", "FRAME7")
+ if err != nil {
+ t.Fatalf("extractDocumentText: %v", err)
+ }
+ if extraction.Mode != extractionReadabilityFallback {
+ t.Fatalf("Mode = %q, want %q", extraction.Mode, extractionReadabilityFallback)
+ }
+ if len(b.frames) != 2 || b.frames[0] != "FRAME7" || b.frames[1] != "FRAME7" {
+ t.Fatalf("frames = %v, want both extractions scoped to FRAME7", b.frames)
+ }
+ if b.evaluateCalls != 0 {
+ t.Errorf("baseline escaped the frame scope: %d top-frame evaluates (%v)", b.evaluateCalls, b.evaluateExprs)
+ }
+ if b.scripts[1] != rawTextScript {
+ t.Errorf("baseline script = %q, want the raw text script", b.scripts[1])
+ }
+}