RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(pdf): strip tiled watermark text from text-layer extraction (#18308)
Taranum Wasuee3360bccb3e
diff --git a/internal/deepdoc/parser/pdf/layout/layout.go b/internal/deepdoc/parser/pdf/layout/layout.goindex e0250d21f8..00bb4ec8cc 100644--- a/internal/deepdoc/parser/pdf/layout/layout.go+++ b/internal/deepdoc/parser/pdf/layout/layout.go@@ -233,6 +233,110 @@ func dedupNormText(s string) string {return b.String()}+// FilterWatermarkBoxes drops single-character ASCII boxes whose text repeats+// verbatim many times on the SAME page. This targets the "tiled watermark on a+// rotated glyph string" pattern reported in #18145:+//+// - Resume / template PDFs plant a watermark string (e.g. "ce1a60…ZH1c56f")+// repeated diagonally across the page. The string is rotated so each+// glyph fails the layout line-overlap check and ends up in its own+// single-character box.+// - Naive chunker then interleaves those garbage boxes into every chunk.+//+// This runs at the POST-PROCESS level (after char→box and OCR→box have both+// converged into the same `[]pdf.TextBox` slice), so it covers both the+// direct-text and OCR-merge code paths that the issue reproduces. An earlier+// attempt to filter at the char-stage in `CharsToBoxes` was a dead branch+// for the OCR-merge path and operated at the wrong granularity (multi-char+// tokens), per the review on PR #18308.+//+// Scope is intentionally tight so legitimate content is never dropped:+//+// - Only boxes whose Text is exactly one ASCII letter or digit, classified+// against raw Text (no normalization). This excludes multi-token+// repetition (e.g. repeated SKU "ABC123"), CJK glyphs (always ≥3 UTF-8+// bytes → len>1), punctuation, and whitespace.+// - Promoted only if the box's text appears ≥ watermarkBoxesMinOccurrences+// times on the page. A real "Q" or "1" appears at most once per page+// outside a watermark tiling.+// - Page-local, so a watermark on page 3 cannot affect page 1. This also+// keeps the O(n) cost bounded by page size.+//+// CJK is unconstrained: a single box's bytes may be ≥1 even when the rune+// count is 1, so `len(box.Text) == 1` already excludes CJK fonts (which are+// typically 3 bytes per glyph). A box that 姓名 — the canonical Chinese name+// field — survives verbatim even when the rest of the document is contaminated.+func FilterWatermarkBoxes(boxes []pdf.TextBox) []pdf.TextBox {+ if len(boxes) == 0 {+ return boxes+ }+ // Pass 1: per-page count of single-char ASCII boxes. Use the raw+ // box.Text for both classification and the promotion key so no+ // normalization collapses distinct bytes.+ type key struct {+ page int+ text string+ }+ counts := make(map[key]int, len(boxes))+ for _, b := range boxes {+ if !isSingleAsciiAlnum(b.Text) {+ continue+ }+ counts[key{b.PageNumber, b.Text}]+++ }+ // Promote any (page, text) that meets the watermark threshold.+ promoted := make(map[key]struct{}, len(counts))+ for k, n := range counts {+ if n >= watermarkBoxesMinOccurrences {+ promoted[k] = struct{}{}+ }+ }+ if len(promoted) == 0 {+ return boxes+ }+ // Pass 2: drop boxes whose (page, text) was promoted.+ out := make([]pdf.TextBox, 0, len(boxes))+ for _, b := range boxes {+ if isSingleAsciiAlnum(b.Text) {+ if _, drop := promoted[key{b.PageNumber, b.Text}]; drop {+ continue+ }+ }+ out = append(out, b)+ }+ return out+}++// watermarkBoxesMinOccurrences is the per-page repetition threshold for a+// single-char ASCII box to be considered a watermark glyph. Legitimate short+// content ("1.", "Q.", "A.") rarely appears more than once per page outside+// of dense content (table rows, list items), and even there the same label+// rarely reaches this threshold. A watermark tile places the same glyph at+// 10+ x-intercepts across the page, so the threshold is set conservatively+// above that floor to keep false-positive rate negligible.+const watermarkBoxesMinOccurrences = 4++// isSingleAsciiAlnum reports whether s is exactly one ASCII letter or digit+// (no whitespace, no CJK, no punctuation, no multibyte UTF-8). Used by+// FilterWatermarkBoxes to limit the watermark signal to the rotated-glyph+// shape rather than the broader "any short token" case, which would+// collide with repeated SKUs and part numbers.+func isSingleAsciiAlnum(s string) bool {+ if len(s) != 1 {+ return false+ }+ c := s[0]+ switch {+ case c >= 'A' && c <= 'Z':+ return true+ case c >= 'a' && c <= 'z':+ return true+ case c >= '0' && c <= '9':+ return true+ }+ return false+}+// dedupYTolerancePt tolerates a small Y-boundary overshoot of an OCR// double-detection fragment's BOTTOM edge (and the height-match slack for the// top-overshoot case, see boxInsideTolerant). Such fragments sit on the SAMEdiff --git a/internal/deepdoc/parser/pdf/layout/watermark_test.go b/internal/deepdoc/parser/pdf/layout/watermark_test.gonew file mode 100644index 0000000000..fc61f53290--- /dev/null+++ b/internal/deepdoc/parser/pdf/layout/watermark_test.go@@ -0,0 +1,199 @@+package layout++import (+ "testing"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// makeBox is a small constructor so the per-case tables read like the+// per-glyph reproductions that actually triggered #18145: one box per+// rotated watermark glyph, positioned in disjoint X/Y locations across+// the same page.+func makeBox(page int, x0, x1, top, bot float64, text string) pdf.TextBox {+ return pdf.TextBox{+ X0: x0,+ X1: x1,+ Top: top,+ Bottom: bot,+ Text: text,+ PageNumber: page,+ }+}++func TestFilterWatermarkBoxes_DropsRepeatedSingleChars(t *testing.T) {+ // 5× "Q" and 5× "1" on page 0 (≥ watermarkMinOccurrences) — these+ // are the rotated glyphs from the tiled watermark in #18145.+ var boxes []pdf.TextBox+ for i := 0; i < 5; i++ {+ y := float64(100 + i*20)+ boxes = append(boxes,+ makeBox(0, 50, 58, y, y+12, "Q"),+ makeBox(0, 80, 88, y, y+12, "1"),+ )+ }+ out := FilterWatermarkBoxes(boxes)+ for _, b := range out {+ if b.Text == "Q" || b.Text == "1" {+ t.Errorf("watermark glyph %q survived (page=%d)", b.Text, b.PageNumber)+ }+ }+ if len(out) != 0 {+ t.Errorf("expected all watermark glyphs dropped, got %d", len(out))+ }+}++func TestFilterWatermarkBoxes_KeepsLegitimateShortContent(t *testing.T) {+ // 1× "Q", 1× "1", 1× "A" on page 0 — all legitimate single-char uses+ // (list markers, table column labels). Each appears below the+ // threshold so NOTHING should be dropped.+ boxes := []pdf.TextBox{+ makeBox(0, 50, 58, 100, 112, "Q"),+ makeBox(0, 50, 58, 130, 142, "1"),+ makeBox(0, 50, 58, 160, 172, "A"),+ }+ out := FilterWatermarkBoxes(boxes)+ if len(out) != len(boxes) {+ t.Errorf("expected %d boxes kept, got %d", len(boxes), len(out))+ }+}++func TestFilterWatermarkBoxes_KeepsCJK(t *testing.T) {+ // 6× "姓" on page 0 — a real form-field label appearing repeatedly+ // in a Chinese resume. CJK glyphs are ≥3 UTF-8 bytes per rune, so+ // isSingleAsciiAlnum rejects them outright. Every box survives.+ var boxes []pdf.TextBox+ for i := 0; i < 6; i++ {+ boxes = append(boxes,+ makeBox(0, 50, 58, float64(100+i*20), float64(112+i*20), "姓"),+ )+ }+ out := FilterWatermarkBoxes(boxes)+ if len(out) != len(boxes) {+ t.Errorf("CJK boxes (%d) lost — watermark filter must not touch CJK", len(boxes)-len(out))+ }+}++func TestFilterWatermarkBoxes_KeepsRepeatedMultiCharTokens(t *testing.T) {+ // "ABC123" is a realistic repeated SKU / batch-code / part-number+ // pattern. Even when it appears 8× per page, the multi-char length+ // keeps it out of the watermark filter, so the data is not lost.+ var boxes []pdf.TextBox+ for i := 0; i < 8; i++ {+ boxes = append(boxes,+ makeBox(0, float64(50+i*30), float64(80+i*30), 100, 112, "ABC123"),+ )+ }+ out := FilterWatermarkBoxes(boxes)+ if len(out) != len(boxes) {+ t.Errorf("multi-char SKU '%s' should survive verbatim, lost %d", "ABC123", len(boxes)-len(out))+ }+}++func TestFilterWatermarkBoxes_PerPageScopeIsolatesByPageNumber(t *testing.T) {+ // 4× "Q" on page 0 (≥ threshold → watermark), 2× "Q" on page 1+ // (legitimate repetition below threshold).+ var boxes []pdf.TextBox+ for i := 0; i < 4; i++ {+ boxes = append(boxes, makeBox(0, float64(50+i*30), 58, 100, 112, "Q"))+ }+ for i := 0; i < 2; i++ {+ boxes = append(boxes, makeBox(1, float64(50+i*30), 58, 100, 112, "Q"))+ }+ out := FilterWatermarkBoxes(boxes)+ page0 := 0+ page1 := 0+ for _, b := range out {+ switch b.PageNumber {+ case 0:+ page0+++ case 1:+ page1+++ }+ }+ if page0 != 0 {+ t.Errorf("page 0 still has %d 'Q' boxes after filter", page0)+ }+ if page1 != 2 {+ t.Errorf("page 1 'Q' boxes lost: want 2, got %d", page1)+ }+}++func TestFilterWatermarkBoxes_HandlesEmptyAndNil(t *testing.T) {+ if got := FilterWatermarkBoxes(nil); len(got) != 0 {+ t.Errorf("nil input should give empty output, got %d", len(got))+ }+ if got := FilterWatermarkBoxes([]pdf.TextBox{}); len(got) != 0 {+ t.Errorf("empty input should give empty output, got %d", len(got))+ }+}++func TestFilterWatermarkBoxes_NoFalsePositiveOnMixed(t *testing.T) {+ // A page with one "Q", one "1", one "ABC123", one " ", one "," — no+ // single ASCII-alnum token hits the threshold, nothing dropped.+ boxes := []pdf.TextBox{+ makeBox(0, 50, 58, 100, 112, "Q"),+ makeBox(0, 50, 58, 130, 142, "1"),+ makeBox(0, 50, 100, 160, 172, "ABC123"),+ makeBox(0, 50, 58, 190, 202, " "),+ makeBox(0, 50, 58, 220, 232, ","),+ }+ out := FilterWatermarkBoxes(boxes)+ if len(out) != len(boxes) {+ t.Errorf("expected %d boxes kept, got %d (lost %d)", len(boxes), len(out), len(boxes)-len(out))+ }+}++// 4 raw "Q" boxes reach the watermark threshold and must drop; the+// 3 padded variants ("Q ", " Q", " Q ") are not single-character+// candidates (length > 1) and must survive.+func TestFilterWatermarkBoxes_WhitespacePaddedNotCollapsedWithBareSingleChar(t *testing.T) {+ boxes := []pdf.TextBox{+ makeBox(0, 50, 58, 100, 112, "Q"), // raw → candidate+ makeBox(0, 50, 58, 130, 142, "Q "), // padded → NOT a candidate under no-TrimSpace+ makeBox(0, 50, 58, 160, 172, " Q"), // padded+ makeBox(0, 50, 58, 190, 202, " Q "), // padded+ makeBox(0, 50, 58, 220, 232, "Q"), // raw → candidate+ makeBox(0, 50, 58, 250, 262, "Q"), // raw → candidate+ makeBox(0, 50, 58, 280, 292, "Q"), // raw → candidate+ }+ if got := len(boxes); got != 7 {+ t.Fatalf("sanity: want 7 boxes, got %d", got)+ }+ out := FilterWatermarkBoxes(boxes)+ rawQ, padded := 0, 0+ for _, b := range out {+ switch b.Text {+ case "Q":+ rawQ+++ case "Q ", " Q", " Q ":+ padded+++ }+ }+ if rawQ != 0 {+ t.Errorf("expected all raw 'Q' boxes dropped (4 promoted, ≥ threshold), %d survived", rawQ)+ }+ if padded != 3 {+ t.Errorf("expected all 3 whitespace-padded boxes kept, %d survived", padded)+ }+ if len(out) != 3 {+ t.Errorf("expected 3 surviving boxes, got %d", len(out))+ }+}++func TestIsSingleAsciiAlnum(t *testing.T) {+ cases := []struct {+ in string+ want bool+ }{+ {"A", true}, {"z", true}, {"0", true}, {"9", true},+ {"", false}, {"AA", false}, {"a1", false},+ {" ", false}, {",", false}, {"中", false},+ {"\n", false}, {"\x00", false},+ }+ for _, c := range cases {+ if got := isSingleAsciiAlnum(c.in); got != c.want {+ t.Errorf("isSingleAsciiAlnum(%q) = %v, want %v", c.in, got, c.want)+ }+ }+}diff --git a/internal/deepdoc/parser/pdf/parser.go b/internal/deepdoc/parser/pdf/parser.goindex 4581fe9f72..859d9407b2 100644--- a/internal/deepdoc/parser/pdf/parser.go+++ b/internal/deepdoc/parser/pdf/parser.go@@ -523,6 +523,14 @@ func (p *Parser) buildLayout(ctx context.Context,boxes = lyt.DedupIdenticalText(boxes)boxes = lyt.DedupSubstringOverlaps(boxes)+ // Drop single-character ASCII boxes that repeat verbatim many times on+ // the SAME page — these are the rotated watermark glyphs from+ // templated PDFs (issue #18145). Post-process placement so the same+ // signal covers both the char-path (passed through CharsToBoxes) and+ // the OCR-merge path (passed through ocrMergeChars); a layout-stage+ // filter would have only caught the former.+ boxes = lyt.FilterWatermarkBoxes(boxes)+boxes = lyt.TextMerge(boxes, medianHeights)result.Metrics.BoxesTextMerge = len(boxes)