Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(pdf): strip tiled watermark text from text-layer extraction (#18308)

Taranum Wasuee3360bccb3e
diff --git a/internal/deepdoc/parser/pdf/layout/layout.go b/internal/deepdoc/parser/pdf/layout/layout.go
index e0250d21f8..00bb4ec8cc 100644
--- a/internal/deepdoc/parser/pdf/layout/layout.go
+++ b/internal/deepdoc/parser/pdf/layout/layout.go
@@ -233,6 +233,110 @@ func dedupNormText(s string) string {
return b.String()
}
+// FilterWatermarkBoxes drops single-character ASCII boxes whose text repeats
+// verbatim many times on the SAME page. This targets the "tiled watermark on a
+// rotated glyph string" pattern reported in #18145:
+//
+// - Resume / template PDFs plant a watermark string (e.g. "ce1a60…ZH1c56f")
+// repeated diagonally across the page. The string is rotated so each
+// glyph fails the layout line-overlap check and ends up in its own
+// single-character box.
+// - Naive chunker then interleaves those garbage boxes into every chunk.
+//
+// This runs at the POST-PROCESS level (after char→box and OCR→box have both
+// converged into the same `[]pdf.TextBox` slice), so it covers both the
+// direct-text and OCR-merge code paths that the issue reproduces. An earlier
+// attempt to filter at the char-stage in `CharsToBoxes` was a dead branch
+// for the OCR-merge path and operated at the wrong granularity (multi-char
+// tokens), per the review on PR #18308.
+//
+// Scope is intentionally tight so legitimate content is never dropped:
+//
+// - Only boxes whose Text is exactly one ASCII letter or digit, classified
+// against raw Text (no normalization). This excludes multi-token
+// repetition (e.g. repeated SKU "ABC123"), CJK glyphs (always ≥3 UTF-8
+// bytes → len>1), punctuation, and whitespace.
+// - Promoted only if the box's text appears ≥ watermarkBoxesMinOccurrences
+// times on the page. A real "Q" or "1" appears at most once per page
+// outside a watermark tiling.
+// - Page-local, so a watermark on page 3 cannot affect page 1. This also
+// keeps the O(n) cost bounded by page size.
+//
+// CJK is unconstrained: a single box's bytes may be ≥1 even when the rune
+// count is 1, so `len(box.Text) == 1` already excludes CJK fonts (which are
+// typically 3 bytes per glyph). A box that 姓名 — the canonical Chinese name
+// field — survives verbatim even when the rest of the document is contaminated.
+func FilterWatermarkBoxes(boxes []pdf.TextBox) []pdf.TextBox {
+ if len(boxes) == 0 {
+ return boxes
+ }
+ // Pass 1: per-page count of single-char ASCII boxes. Use the raw
+ // box.Text for both classification and the promotion key so no
+ // normalization collapses distinct bytes.
+ type key struct {
+ page int
+ text string
+ }
+ counts := make(map[key]int, len(boxes))
+ for _, b := range boxes {
+ if !isSingleAsciiAlnum(b.Text) {
+ continue
+ }
+ counts[key{b.PageNumber, b.Text}]++
+ }
+ // Promote any (page, text) that meets the watermark threshold.
+ promoted := make(map[key]struct{}, len(counts))
+ for k, n := range counts {
+ if n >= watermarkBoxesMinOccurrences {
+ promoted[k] = struct{}{}
+ }
+ }
+ if len(promoted) == 0 {
+ return boxes
+ }
+ // Pass 2: drop boxes whose (page, text) was promoted.
+ out := make([]pdf.TextBox, 0, len(boxes))
+ for _, b := range boxes {
+ if isSingleAsciiAlnum(b.Text) {
+ if _, drop := promoted[key{b.PageNumber, b.Text}]; drop {
+ continue
+ }
+ }
+ out = append(out, b)
+ }
+ return out
+}
+
+// watermarkBoxesMinOccurrences is the per-page repetition threshold for a
+// single-char ASCII box to be considered a watermark glyph. Legitimate short
+// content ("1.", "Q.", "A.") rarely appears more than once per page outside
+// of dense content (table rows, list items), and even there the same label
+// rarely reaches this threshold. A watermark tile places the same glyph at
+// 10+ x-intercepts across the page, so the threshold is set conservatively
+// above that floor to keep false-positive rate negligible.
+const watermarkBoxesMinOccurrences = 4
+
+// isSingleAsciiAlnum reports whether s is exactly one ASCII letter or digit
+// (no whitespace, no CJK, no punctuation, no multibyte UTF-8). Used by
+// FilterWatermarkBoxes to limit the watermark signal to the rotated-glyph
+// shape rather than the broader "any short token" case, which would
+// collide with repeated SKUs and part numbers.
+func isSingleAsciiAlnum(s string) bool {
+ if len(s) != 1 {
+ return false
+ }
+ c := s[0]
+ switch {
+ case c >= 'A' && c <= 'Z':
+ return true
+ case c >= 'a' && c <= 'z':
+ return true
+ case c >= '0' && c <= '9':
+ return true
+ }
+ return false
+}
+
// dedupYTolerancePt tolerates a small Y-boundary overshoot of an OCR
// double-detection fragment's BOTTOM edge (and the height-match slack for the
// top-overshoot case, see boxInsideTolerant). Such fragments sit on the SAME
diff --git a/internal/deepdoc/parser/pdf/layout/watermark_test.go b/internal/deepdoc/parser/pdf/layout/watermark_test.go
new file mode 100644
index 0000000000..fc61f53290
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/layout/watermark_test.go
@@ -0,0 +1,199 @@
+package layout
+
+import (
+ "testing"
+
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+)
+
+// makeBox is a small constructor so the per-case tables read like the
+// per-glyph reproductions that actually triggered #18145: one box per
+// rotated watermark glyph, positioned in disjoint X/Y locations across
+// the same page.
+func makeBox(page int, x0, x1, top, bot float64, text string) pdf.TextBox {
+ return pdf.TextBox{
+ X0: x0,
+ X1: x1,
+ Top: top,
+ Bottom: bot,
+ Text: text,
+ PageNumber: page,
+ }
+}
+
+func TestFilterWatermarkBoxes_DropsRepeatedSingleChars(t *testing.T) {
+ // 5× "Q" and 5× "1" on page 0 (≥ watermarkMinOccurrences) — these
+ // are the rotated glyphs from the tiled watermark in #18145.
+ var boxes []pdf.TextBox
+ for i := 0; i < 5; i++ {
+ y := float64(100 + i*20)
+ boxes = append(boxes,
+ makeBox(0, 50, 58, y, y+12, "Q"),
+ makeBox(0, 80, 88, y, y+12, "1"),
+ )
+ }
+ out := FilterWatermarkBoxes(boxes)
+ for _, b := range out {
+ if b.Text == "Q" || b.Text == "1" {
+ t.Errorf("watermark glyph %q survived (page=%d)", b.Text, b.PageNumber)
+ }
+ }
+ if len(out) != 0 {
+ t.Errorf("expected all watermark glyphs dropped, got %d", len(out))
+ }
+}
+
+func TestFilterWatermarkBoxes_KeepsLegitimateShortContent(t *testing.T) {
+ // 1× "Q", 1× "1", 1× "A" on page 0 — all legitimate single-char uses
+ // (list markers, table column labels). Each appears below the
+ // threshold so NOTHING should be dropped.
+ boxes := []pdf.TextBox{
+ makeBox(0, 50, 58, 100, 112, "Q"),
+ makeBox(0, 50, 58, 130, 142, "1"),
+ makeBox(0, 50, 58, 160, 172, "A"),
+ }
+ out := FilterWatermarkBoxes(boxes)
+ if len(out) != len(boxes) {
+ t.Errorf("expected %d boxes kept, got %d", len(boxes), len(out))
+ }
+}
+
+func TestFilterWatermarkBoxes_KeepsCJK(t *testing.T) {
+ // 6× "姓" on page 0 — a real form-field label appearing repeatedly
+ // in a Chinese resume. CJK glyphs are ≥3 UTF-8 bytes per rune, so
+ // isSingleAsciiAlnum rejects them outright. Every box survives.
+ var boxes []pdf.TextBox
+ for i := 0; i < 6; i++ {
+ boxes = append(boxes,
+ makeBox(0, 50, 58, float64(100+i*20), float64(112+i*20), "姓"),
+ )
+ }
+ out := FilterWatermarkBoxes(boxes)
+ if len(out) != len(boxes) {
+ t.Errorf("CJK boxes (%d) lost — watermark filter must not touch CJK", len(boxes)-len(out))
+ }
+}
+
+func TestFilterWatermarkBoxes_KeepsRepeatedMultiCharTokens(t *testing.T) {
+ // "ABC123" is a realistic repeated SKU / batch-code / part-number
+ // pattern. Even when it appears 8× per page, the multi-char length
+ // keeps it out of the watermark filter, so the data is not lost.
+ var boxes []pdf.TextBox
+ for i := 0; i < 8; i++ {
+ boxes = append(boxes,
+ makeBox(0, float64(50+i*30), float64(80+i*30), 100, 112, "ABC123"),
+ )
+ }
+ out := FilterWatermarkBoxes(boxes)
+ if len(out) != len(boxes) {
+ t.Errorf("multi-char SKU '%s' should survive verbatim, lost %d", "ABC123", len(boxes)-len(out))
+ }
+}
+
+func TestFilterWatermarkBoxes_PerPageScopeIsolatesByPageNumber(t *testing.T) {
+ // 4× "Q" on page 0 (≥ threshold → watermark), 2× "Q" on page 1
+ // (legitimate repetition below threshold).
+ var boxes []pdf.TextBox
+ for i := 0; i < 4; i++ {
+ boxes = append(boxes, makeBox(0, float64(50+i*30), 58, 100, 112, "Q"))
+ }
+ for i := 0; i < 2; i++ {
+ boxes = append(boxes, makeBox(1, float64(50+i*30), 58, 100, 112, "Q"))
+ }
+ out := FilterWatermarkBoxes(boxes)
+ page0 := 0
+ page1 := 0
+ for _, b := range out {
+ switch b.PageNumber {
+ case 0:
+ page0++
+ case 1:
+ page1++
+ }
+ }
+ if page0 != 0 {
+ t.Errorf("page 0 still has %d 'Q' boxes after filter", page0)
+ }
+ if page1 != 2 {
+ t.Errorf("page 1 'Q' boxes lost: want 2, got %d", page1)
+ }
+}
+
+func TestFilterWatermarkBoxes_HandlesEmptyAndNil(t *testing.T) {
+ if got := FilterWatermarkBoxes(nil); len(got) != 0 {
+ t.Errorf("nil input should give empty output, got %d", len(got))
+ }
+ if got := FilterWatermarkBoxes([]pdf.TextBox{}); len(got) != 0 {
+ t.Errorf("empty input should give empty output, got %d", len(got))
+ }
+}
+
+func TestFilterWatermarkBoxes_NoFalsePositiveOnMixed(t *testing.T) {
+ // A page with one "Q", one "1", one "ABC123", one " ", one "," — no
+ // single ASCII-alnum token hits the threshold, nothing dropped.
+ boxes := []pdf.TextBox{
+ makeBox(0, 50, 58, 100, 112, "Q"),
+ makeBox(0, 50, 58, 130, 142, "1"),
+ makeBox(0, 50, 100, 160, 172, "ABC123"),
+ makeBox(0, 50, 58, 190, 202, " "),
+ makeBox(0, 50, 58, 220, 232, ","),
+ }
+ out := FilterWatermarkBoxes(boxes)
+ if len(out) != len(boxes) {
+ t.Errorf("expected %d boxes kept, got %d (lost %d)", len(boxes), len(out), len(boxes)-len(out))
+ }
+}
+
+// 4 raw "Q" boxes reach the watermark threshold and must drop; the
+// 3 padded variants ("Q ", " Q", " Q ") are not single-character
+// candidates (length > 1) and must survive.
+func TestFilterWatermarkBoxes_WhitespacePaddedNotCollapsedWithBareSingleChar(t *testing.T) {
+ boxes := []pdf.TextBox{
+ makeBox(0, 50, 58, 100, 112, "Q"), // raw → candidate
+ makeBox(0, 50, 58, 130, 142, "Q "), // padded → NOT a candidate under no-TrimSpace
+ makeBox(0, 50, 58, 160, 172, " Q"), // padded
+ makeBox(0, 50, 58, 190, 202, " Q "), // padded
+ makeBox(0, 50, 58, 220, 232, "Q"), // raw → candidate
+ makeBox(0, 50, 58, 250, 262, "Q"), // raw → candidate
+ makeBox(0, 50, 58, 280, 292, "Q"), // raw → candidate
+ }
+ if got := len(boxes); got != 7 {
+ t.Fatalf("sanity: want 7 boxes, got %d", got)
+ }
+ out := FilterWatermarkBoxes(boxes)
+ rawQ, padded := 0, 0
+ for _, b := range out {
+ switch b.Text {
+ case "Q":
+ rawQ++
+ case "Q ", " Q", " Q ":
+ padded++
+ }
+ }
+ if rawQ != 0 {
+ t.Errorf("expected all raw 'Q' boxes dropped (4 promoted, ≥ threshold), %d survived", rawQ)
+ }
+ if padded != 3 {
+ t.Errorf("expected all 3 whitespace-padded boxes kept, %d survived", padded)
+ }
+ if len(out) != 3 {
+ t.Errorf("expected 3 surviving boxes, got %d", len(out))
+ }
+}
+
+func TestIsSingleAsciiAlnum(t *testing.T) {
+ cases := []struct {
+ in string
+ want bool
+ }{
+ {"A", true}, {"z", true}, {"0", true}, {"9", true},
+ {"", false}, {"AA", false}, {"a1", false},
+ {" ", false}, {",", false}, {"中", false},
+ {"\n", false}, {"\x00", false},
+ }
+ for _, c := range cases {
+ if got := isSingleAsciiAlnum(c.in); got != c.want {
+ t.Errorf("isSingleAsciiAlnum(%q) = %v, want %v", c.in, got, c.want)
+ }
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/parser.go b/internal/deepdoc/parser/pdf/parser.go
index 4581fe9f72..859d9407b2 100644
--- a/internal/deepdoc/parser/pdf/parser.go
+++ b/internal/deepdoc/parser/pdf/parser.go
@@ -523,6 +523,14 @@ func (p *Parser) buildLayout(ctx context.Context,
boxes = lyt.DedupIdenticalText(boxes)
boxes = lyt.DedupSubstringOverlaps(boxes)
+ // Drop single-character ASCII boxes that repeat verbatim many times on
+ // the SAME page — these are the rotated watermark glyphs from
+ // templated PDFs (issue #18145). Post-process placement so the same
+ // signal covers both the char-path (passed through CharsToBoxes) and
+ // the OCR-merge path (passed through ocrMergeChars); a layout-stage
+ // filter would have only caught the former.
+ boxes = lyt.FilterWatermarkBoxes(boxes)
+
boxes = lyt.TextMerge(boxes, medianHeights)
result.Metrics.BoxesTextMerge = len(boxes)