High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix: /text default Readability mode can drop 97% of a page and reports truncated:false PIN-012
Luigi Agostid0290661db7f
diff --git a/docs/reference/text.md b/docs/reference/text.mdindex f7c5a6f0..011dfb58 100644--- a/docs/reference/text.md+++ b/docs/reference/text.md@@ -38,6 +38,26 @@ pinchtab text # Plain text outputpinchtab text --json # JSON: {"url":"...","title":"...","text":"..."}```+## Extraction Mode++Readability is an article heuristic, so on a landing page, dashboard, or docs+index it can return one block instead of the page. `/text` measures its output+against `document.body.innerText` and, when the coverage is too low, returns the+raw text instead. The JSON envelope always says which extractor produced the+text:++| Field | Description |+|-------|-------------|+| `extraction` | `readability`, `raw` (explicitly requested), or `readability_fallback` (Readability collapsed, raw text returned) |+| `textLength` | Length of the returned text |+| `rawLength` | Length of `document.body.innerText`, so coverage is computable |++With `format=text` the body stays bare and the mode is reported in the+`X-PT-Text-Extraction` response header. `mode=raw` never runs the comparison.+`truncated` keeps its meaning — text cut by `maxChars` — and is unaffected by a+fallback. The CLI prints a one-line note on stderr when a fallback fired; stdout+stays text-only.+## Examples```bashdiff --git a/internal/cli/actions/actions_text.go b/internal/cli/actions/actions_text.goindex d3cc668c..37523bb2 100644--- a/internal/cli/actions/actions_text.go+++ b/internal/cli/actions/actions_text.go@@ -57,11 +57,19 @@ func Text(client *http.Client, base, token string, cmd *cobra.Command, args []stbody := apiclient.DoGetRaw(client, base, token, "/text", params)var result struct {- Text string `json:"text"`+ Text string `json:"text"`+ Extraction string `json:"extraction"`}if err := json.Unmarshal(body, &result); err != nil {output.Value(string(body))return}+ if result.Extraction == extractionReadabilityFallback {+ output.Hint("readability extracted only a fragment of this page; returned the full page text instead (pass --full to request it directly)")+ }output.Value(result.Text)}++// extractionReadabilityFallback mirrors the /text envelope value the server+// echoes when readability collapsed and it returned the raw document instead.+const extractionReadabilityFallback = "readability_fallback"diff --git a/internal/cli/actions/actions_text_test.go b/internal/cli/actions/actions_text_test.goindex cba9d7e0..e8aa737a 100644--- a/internal/cli/actions/actions_text_test.go+++ b/internal/cli/actions/actions_text_test.go@@ -1,6 +1,10 @@package actionsimport (+ "bytes"+ "encoding/json"+ "io"+ "os""strings""testing"@@ -88,3 +92,81 @@ func TestTextTab(t *testing.T) {t.Errorf("expected tabId=TAB1, got %s", m.lastQuery)}}++func captureStderr(t *testing.T, fn func()) string {+ t.Helper()++ oldStderr := os.Stderr+ reader, writer, err := os.Pipe()+ if err != nil {+ t.Fatalf("os.Pipe: %v", err)+ }+ os.Stderr = writer++ var buf bytes.Buffer+ done := make(chan error, 1)+ go func() {+ _, copyErr := io.Copy(&buf, reader)+ _ = reader.Close()+ done <- copyErr+ }()++ defer func() { os.Stderr = oldStderr }()++ fn()++ _ = writer.Close()+ if err := <-done; err != nil {+ t.Fatalf("io.Copy: %v", err)+ }+ return buf.String()+}++func runTextAgainst(t *testing.T, response string) (stdout, stderr string) {+ t.Helper()+ m := newMockServer()+ m.response = response+ defer m.close()++ stderr = captureStderr(t, func() {+ stdout = captureStdout(t, func() {+ Text(m.server.Client(), m.base(), "", newTextCmd(), nil)+ })+ })+ return stdout, stderr+}++func TestTextFallbackNotesOnStderrAndKeepsStdoutClean(t *testing.T) {+ page := "PinchTab — browser control for AI agents\nInstall: curl -fsSL https://pinchtab.com/install.sh | sh"+ body, err := json.Marshal(map[string]any{+ "text": page,+ "extraction": "readability_fallback",+ "textLength": len(page),+ "rawLength": len(page),+ })+ if err != nil {+ t.Fatal(err)+ }++ stdout, stderr := runTextAgainst(t, string(body))++ if stdout != page+"\n" {+ t.Errorf("stdout = %q, want the page text alone", stdout)+ }+ if !strings.Contains(stderr, "--full") {+ t.Errorf("stderr = %q, want a note pointing at --full", stderr)+ }+}++func TestTextNoNoteWhenReadabilityHeld(t *testing.T) {+ body := `{"text":"Article body","extraction":"readability","textLength":12,"rawLength":13}`++ stdout, stderr := runTextAgainst(t, body)++ if stdout != "Article body\n" {+ t.Errorf("stdout = %q, want the article text", stdout)+ }+ if stderr != "" {+ t.Errorf("stderr = %q, want no note when readability held", stderr)+ }+}diff --git a/internal/handlers/text.go b/internal/handlers/text.goindex b20d2730..2de6c5cc 100644--- a/internal/handlers/text.go+++ b/internal/handlers/text.go@@ -68,13 +68,15 @@ func (h *Handlers) HandleText(w http.ResponseWriter, r *http.Request) {}var text string+ var extraction textExtractionif selectorParam != "" || refParam != "" {text, err = h.extractElementText(tCtx, resolvedTabID, selectorParam, refParam, modalNodeID)} else if modalOpen {err = h.Bridge.CallFunctionOnNode(tCtx, modalNodeID,`function() { return this.innerText || this.textContent || ''; }`, nil, &text)+ extraction = textExtraction{Text: text, Mode: extractionRaw, RawLength: len(text), RawKnown: true}} else {- text, err = h.extractDocumentText(tCtx, mode, targetFrameID)+ extraction, err = h.extractDocumentText(tCtx, mode, targetFrameID)}stable := true@@ -101,7 +103,7 @@ func (h *Handlers) HandleText(w http.ResponseWriter, r *http.Request) {if selectorParam != "" || refParam != "" {h.writeElementTextResponse(w, r, tCtx, text)} else {- h.writeTextResponse(w, r, tCtx, text, maxChars, format, textRoute)+ h.writeTextResponse(w, r, tCtx, extraction, maxChars, format, textRoute)}return}@@ -120,25 +122,87 @@ func (h *Handlers) writeElementTextResponse(w http.ResponseWriter, r *http.Reque})}+const rawTextScript = `document.body.innerText`++// Extraction modes echoed to the caller: which extractor produced the text.+const (+ extractionReadability = "readability"+ extractionRaw = "raw"+ extractionReadabilityFallback = "readability_fallback"+)++// Readability is an article heuristic; on a layout it does not recognise as an+// article (landing page, dashboard, docs index) it returns the single block it+// scored highest. Measured against document.body.innerText, a collapse retains+// ~3% of the page while healthy article extraction retains ~91%, so any ratio in+// the low tens of percent separates the two with wide margin on both sides.+const readabilityCoverageRatio = 0.25++// Under this many raw characters the ratio is noise — a short page legitimately+// extracts to a couple of lines — so no fallback fires below it.+const readabilityCoverageFloorChars = 400++// headerTextExtraction reports the extraction mode on the format=text path,+// which returns a bare body with nowhere to carry the signal.+const headerTextExtraction = "X-PT-Text-Extraction"++// textExtraction is the document text plus the story of how it was produced:+// the mode that actually ran and the length of the raw document it was measured+// against (RawKnown is false when the baseline extraction itself failed).+type textExtraction struct {+ Text string+ Mode string+ RawLength int+ RawKnown bool+}+// extractDocumentText reads the document's text (readability unless mode=="raw")-// across all reachable frames, or scoped to targetFrameID when set. The-// cross-frame path silently skips cross-origin frames and never errors.-func (h *Handlers) extractDocumentText(tCtx context.Context, mode, targetFrameID string) (string, error) {- script := `document.body.innerText`- if mode != "raw" {- script = assets.ReadabilityJS+// across all reachable frames, or scoped to targetFrameID when set. Readability+// output that covers too little of the raw document is discarded in favour of+// the raw text, reported as extractionReadabilityFallback.+func (h *Handlers) extractDocumentText(tCtx context.Context, mode, targetFrameID string) (textExtraction, error) {+ if mode == "raw" {+ text, err := h.extractText(tCtx, rawTextScript, targetFrameID)+ if err != nil {+ return textExtraction{}, err+ }+ return textExtraction{Text: text, Mode: extractionRaw, RawLength: len(text), RawKnown: true}, nil}++ text, err := h.extractText(tCtx, assets.ReadabilityJS, targetFrameID)+ if err != nil {+ return textExtraction{}, err+ }++ raw, rawErr := h.extractText(tCtx, rawTextScript, targetFrameID)+ if rawErr != nil {+ return textExtraction{Text: text, Mode: extractionReadability}, nil+ }+ if readabilityCollapsed(len(text), len(raw)) {+ return textExtraction{Text: raw, Mode: extractionReadabilityFallback, RawLength: len(raw), RawKnown: true}, nil+ }+ return textExtraction{Text: text, Mode: extractionReadability, RawLength: len(raw), RawKnown: true}, nil+}++func readabilityCollapsed(extractedLen, rawLen int) bool {+ return rawLen >= readabilityCoverageFloorChars && float64(extractedLen) < float64(rawLen)*readabilityCoverageRatio+}++// extractText runs one extraction script through the traversal the request+// selected: every reachable frame (joined), or the scoped frame's isolated world+// so the expression sees the iframe's `document`, not the parent's. Both sides of+// the coverage ratio go through here, so they are always built identically.+func (h *Handlers) extractText(tCtx context.Context, script, targetFrameID string) (string, error) {if targetFrameID == "" {return h.extractTextAllFrames(tCtx, script), nil}- // Frame-scoped path — evaluate in the frame's isolated world so the- // expression sees the iframe's `document`, not the parent's.return h.evalTextInFrame(tCtx, script, targetFrameID)}// writeTextResponse truncates, IDPI-scans, and writes the document text as// plain text (format text/plain) or the JSON envelope.-func (h *Handlers) writeTextResponse(w http.ResponseWriter, r *http.Request, tCtx context.Context, text string, maxChars int, format string, route *browserops.RouteMetadata) {+func (h *Handlers) writeTextResponse(w http.ResponseWriter, r *http.Request, tCtx context.Context, extraction textExtraction, maxChars int, format string, route *browserops.RouteMetadata) {+ text := extraction.Texttruncated := falseif maxChars > -1 && len(text) > maxChars {text = text[:maxChars]@@ -157,6 +221,7 @@ func (h *Handlers) writeTextResponse(w http.ResponseWriter, r *http.Request, tCt}result.SetHeaders(w)text = result.Text+ w.Header().Set(headerTextExtraction, extraction.Mode)if format == "text" || format == "plain" {w.Header().Set("Content-Type", "text/plain; charset=utf-8")@@ -166,11 +231,16 @@ func (h *Handlers) writeTextResponse(w http.ResponseWriter, r *http.Request, tCt}resp := map[string]any{- "url": url,- "title": title,- "text": text,- "truncated": truncated,- "route": route,+ "url": url,+ "title": title,+ "text": text,+ "truncated": truncated,+ "route": route,+ "extraction": extraction.Mode,+ "textLength": len(text),+ }+ if extraction.RawKnown {+ resp["rawLength"] = extraction.RawLength}if result.Warning != "" {resp["idpiWarning"] = result.Warningdiff --git a/internal/handlers/text_test.go b/internal/handlers/text_test.goindex 90a4cc9b..15de105f 100644--- a/internal/handlers/text_test.go+++ b/internal/handlers/text_test.go@@ -1,10 +1,14 @@package handlersimport (+ "context"+ "encoding/json"+ "fmt""net/http""net/http/httptest""testing"+ "github.com/pinchtab/pinchtab/internal/bridge""github.com/pinchtab/pinchtab/internal/config")@@ -58,3 +62,226 @@ func TestHandleTabText_NoTab(t *testing.T) {t.Errorf("expected 404, got %d", w.Code)}}++const landingPageRawText = `PinchTab — browser control for AI agents++Install: curl -fsSL https://pinchtab.com/install.sh | sh++Endpoints: /navigate /snapshot /text /action /capture /find /evaluate++🎯 Accessibility Tree — structured tree with stable refs (e0, e1...) for click, type, and read. Deterministic with no coordinate guessing.+🧭 Navigation — drive tabs, frames, and history without a headful browser in the loop.+📄 Text extraction — readable page text with an explicit raw mode when the heuristic collapses.+🖼️ Capture — screenshots and PDFs of the live page, viewport or full document.+🔍 Find — semantic element lookup that survives markup churn.+🧪 Evaluate — run JavaScript in the page or in a specific frame.+🔐 Profiles — reuse a dedicated automation profile with explicit approval.+🧰 Macros — batch several actions into one request.+📊 Activity — every request recorded with the route that served it.`++const collapsedReadabilityText = `🎯 Accessibility Tree — structured tree with stable refs (e0, e1...) for click, type, and read.`++func textScriptBridge(t *testing.T, readability, raw string) *mockBridge {+ t.Helper()+ return &mockBridge{+ evaluateFn: func(expression string, result any) error {+ out, ok := result.(*string)+ if !ok {+ t.Fatalf("evaluate result is %T, want *string", result)+ }+ if expression == rawTextScript {+ *out = raw+ return nil+ }+ *out = readability+ return nil+ },+ }+}++func TestExtractDocumentText_ReadabilityCollapseFallsBackToRaw(t *testing.T) {+ m := textScriptBridge(t, collapsedReadabilityText, landingPageRawText)+ h := New(m, &config.RuntimeConfig{}, nil, nil, nil)++ extraction, err := h.extractDocumentText(context.Background(), "", "")+ if err != nil {+ t.Fatalf("extractDocumentText: %v", err)+ }+ if extraction.Mode != extractionReadabilityFallback {+ t.Errorf("Mode = %q, want %q", extraction.Mode, extractionReadabilityFallback)+ }+ if extraction.Text != landingPageRawText {+ t.Errorf("Text = %q, want the raw document text", extraction.Text)+ }+ if !extraction.RawKnown || extraction.RawLength != len(landingPageRawText) {+ t.Errorf("RawLength = %d (known=%v), want %d", extraction.RawLength, extraction.RawKnown, len(landingPageRawText))+ }+}++func TestExtractDocumentText_ArticleKeepsReadabilityOutput(t *testing.T) {+ article := landingPageRawText[:len(landingPageRawText)*9/10]+ m := textScriptBridge(t, article, landingPageRawText)+ h := New(m, &config.RuntimeConfig{}, nil, nil, nil)++ extraction, err := h.extractDocumentText(context.Background(), "", "")+ if err != nil {+ t.Fatalf("extractDocumentText: %v", err)+ }+ if extraction.Mode != extractionReadability {+ t.Errorf("Mode = %q, want %q", extraction.Mode, extractionReadability)+ }+ if extraction.Text != article {+ t.Errorf("Text = %q, want the readability output", extraction.Text)+ }+ if extraction.RawLength != len(landingPageRawText) {+ t.Errorf("RawLength = %d, want %d", extraction.RawLength, len(landingPageRawText))+ }+}++func TestExtractDocumentText_RawModeSkipsCoverageComparison(t *testing.T) {+ m := textScriptBridge(t, collapsedReadabilityText, landingPageRawText)+ h := New(m, &config.RuntimeConfig{}, nil, nil, nil)++ extraction, err := h.extractDocumentText(context.Background(), "raw", "")+ if err != nil {+ t.Fatalf("extractDocumentText: %v", err)+ }+ if extraction.Mode != extractionRaw {+ t.Errorf("Mode = %q, want %q", extraction.Mode, extractionRaw)+ }+ if len(m.evaluateExprs) != 1 || m.evaluateExprs[0] != rawTextScript {+ t.Fatalf("mode=raw ran %d extractions (%v), want exactly the raw one", len(m.evaluateExprs), m.evaluateExprs)+ }+}++func TestReadabilityCollapsed_FloorKeepsShortPages(t *testing.T) {+ if readabilityCollapsed(1, readabilityCoverageFloorChars-1) {+ t.Error("a page below the character floor must not trigger the fallback")+ }+ if !readabilityCollapsed(1, readabilityCoverageFloorChars) {+ t.Error("a near-total collapse at the floor must trigger the fallback")+ }+ atRatio := int(float64(readabilityCoverageFloorChars) * readabilityCoverageRatio)+ if readabilityCollapsed(atRatio, readabilityCoverageFloorChars) {+ t.Error("coverage at the ratio must not trigger the fallback")+ }+}++func textResponseRecorder(t *testing.T, extraction textExtraction, maxChars int, format string) *httptest.ResponseRecorder {+ t.Helper()+ h := New(&mockBridge{}, &config.RuntimeConfig{}, nil, nil, nil)+ w := httptest.NewRecorder()+ r := httptest.NewRequest("GET", "/text", nil)+ h.writeTextResponse(w, r, context.Background(), extraction, maxChars, format, nil)+ if w.Code != http.StatusOK {+ t.Fatalf("status = %d body=%s", w.Code, w.Body.String())+ }+ return w+}++func decodeTextEnvelope(t *testing.T, w *httptest.ResponseRecorder) map[string]any {+ t.Helper()+ var envelope map[string]any+ if err := json.Unmarshal(w.Body.Bytes(), &envelope); err != nil {+ t.Fatalf("decode: %v (body=%s)", err, w.Body.String())+ }+ return envelope+}++func TestWriteTextResponse_EnvelopeReportsExtractionAndLengths(t *testing.T) {+ extraction := textExtraction{+ Text: landingPageRawText,+ Mode: extractionReadabilityFallback,+ RawLength: len(landingPageRawText),+ RawKnown: true,+ }+ w := textResponseRecorder(t, extraction, -1, "")+ envelope := decodeTextEnvelope(t, w)++ if envelope["extraction"] != extractionReadabilityFallback {+ t.Errorf("extraction = %v, want %q", envelope["extraction"], extractionReadabilityFallback)+ }+ if envelope["textLength"] != float64(len(landingPageRawText)) {+ t.Errorf("textLength = %v, want %d", envelope["textLength"], len(landingPageRawText))+ }+ if envelope["rawLength"] != float64(len(landingPageRawText)) {+ t.Errorf("rawLength = %v, want %d", envelope["rawLength"], len(landingPageRawText))+ }+ if envelope["truncated"] != false {+ t.Errorf("truncated = %v, want false: a fallback is not a truncation", envelope["truncated"])+ }+}++func TestWriteTextResponse_TruncatedOnlyWhenMaxCharsCuts(t *testing.T) {+ extraction := textExtraction{Text: landingPageRawText, Mode: extractionReadabilityFallback, RawLength: len(landingPageRawText), RawKnown: true}++ cut := decodeTextEnvelope(t, textResponseRecorder(t, extraction, 50, ""))+ if cut["truncated"] != true {+ t.Errorf("truncated = %v, want true when maxChars cuts", cut["truncated"])+ }+ if cut["textLength"] != float64(50) {+ t.Errorf("textLength = %v, want the returned length 50", cut["textLength"])+ }++ whole := decodeTextEnvelope(t, textResponseRecorder(t, extraction, len(landingPageRawText)+1, ""))+ if whole["truncated"] != false {+ t.Errorf("truncated = %v, want false when maxChars does not cut", whole["truncated"])+ }+}++func TestWriteTextResponse_PlainFormatKeepsBareBodyAndReportsModeInHeader(t *testing.T) {+ extraction := textExtraction{Text: landingPageRawText, Mode: extractionReadabilityFallback, RawLength: len(landingPageRawText), RawKnown: true}+ w := textResponseRecorder(t, extraction, -1, "text")++ if w.Body.String() != landingPageRawText {+ t.Errorf("body = %q, want the bare text", w.Body.String())+ }+ if got := w.Header().Get(headerTextExtraction); got != extractionReadabilityFallback {+ t.Errorf("%s = %q, want %q", headerTextExtraction, got, extractionReadabilityFallback)+ }+}++type frameTextBridge struct {+ mockBridge+ frames []string+ scripts []string+ readability string+ raw string+}++func (b *frameTextBridge) EvaluateInFrame(_ context.Context, frameID, expression string, result any, _ bridge.EvalOpts) error {+ b.frames = append(b.frames, frameID)+ b.scripts = append(b.scripts, expression)+ out, ok := result.(*string)+ if !ok {+ return fmt.Errorf("evaluate result is %T, want *string", result)+ }+ if expression == rawTextScript {+ *out = b.raw+ return nil+ }+ *out = b.readability+ return nil+}++func TestExtractDocumentText_BaselineUsesTheSameFrameTraversal(t *testing.T) {+ b := &frameTextBridge{readability: collapsedReadabilityText, raw: landingPageRawText}+ h := New(b, &config.RuntimeConfig{}, nil, nil, nil)++ extraction, err := h.extractDocumentText(context.Background(), "", "FRAME7")+ if err != nil {+ t.Fatalf("extractDocumentText: %v", err)+ }+ if extraction.Mode != extractionReadabilityFallback {+ t.Fatalf("Mode = %q, want %q", extraction.Mode, extractionReadabilityFallback)+ }+ if len(b.frames) != 2 || b.frames[0] != "FRAME7" || b.frames[1] != "FRAME7" {+ t.Fatalf("frames = %v, want both extractions scoped to FRAME7", b.frames)+ }+ if b.evaluateCalls != 0 {+ t.Errorf("baseline escaped the frame scope: %d top-frame evaluates (%v)", b.evaluateCalls, b.evaluateExprs)+ }+ if b.scripts[1] != rawTextScript {+ t.Errorf("baseline script = %q, want the raw text script", b.scripts[1])+ }+}