Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): keep recovered PDF TOC leader lines out of the body guard (#21768)

jay77721dce85b304377
diff --git a/internal/deepdoc/parser/pdf/layout/toc_header_footer.go b/internal/deepdoc/parser/pdf/layout/toc_header_footer.go
index b0524fdfea..75ae45d9b3 100644
--- a/internal/deepdoc/parser/pdf/layout/toc_header_footer.go
+++ b/internal/deepdoc/parser/pdf/layout/toc_header_footer.go
@@ -53,6 +53,16 @@ const (
// entries — so it needs no separate marker count. Prose can accumulate the
// occasional run, which is why the threshold is three and not one.
tocMinMergedRuns = 3
+ // tocMinLongEntryLeader is the shortest leader run that lets a long box be
+ // read as one TOC entry line. Tab leaders run for many characters; prose
+ // that merely ends in an ellipsis and a number does not, and must keep
+ // counting as prose.
+ tocMinLongEntryLeader = 6
+ // tocMinShortEntryLeader is the same floor for short boxes. A short entry
+ // stays a marker rather than body evidence, but an ellipsis that is only
+ // sentence punctuation ("The result is ... 42") must not turn three prose
+ // lines into a TOC page.
+ tocMinShortEntryLeader = 4
// tocMaxLeadPages bounds how many leading pages without body text may
// precede the TOC. A cover, copyright page or frontispiece carries no body
// text and is skipped, so a TOC that is not physically page one is still
@@ -100,7 +110,7 @@ var (
// fragment. Counting these runs tells a TOC block that upstream merged into
// one long box apart from a paragraph. Digits accept the full-width forms
// pageNumberPattern accepts, for the same reason.
- tocEntryRunPattern = regexp.MustCompile(`(\.{2,}|…{2,}|⋯{2,}|·{2,})\s*[0-90-9]{1,4}\s*[IVXLCDMivxlcdm]{0,5}`)
+ tocEntryRunPattern = regexp.MustCompile(`(\.{2,}|…{2,}|⋯{2,}|·{2,})\s*([0-90-9]{1,4}\s*[IVXLCDMivxlcdm]{0,5}|[IVXLCDMivxlcdm]{1,7}\b)`)
// tocLeaderOnlyPattern matches a box that is nothing but leader
// characters (dots, ellipses, middots, spaces): the leader column of a
// dotted TOC entry. Such a box is long but carries no body text, so it
@@ -316,6 +326,7 @@ type pageShape struct {
markers int // short boxes opening with a chapter marker or holding a page number
merged bool // a long box carrying tocMinMergedRuns leader+page-number runs
body int // marker-less, leader-less short boxes longer than tocBodyRunes
+ entries int // long entry lines: count toward the list threshold, not as prose
}
func shapeOfPage(boxes []pdf.TextBox, indices []int) pageShape {
@@ -326,17 +337,34 @@ func shapeOfPage(boxes []pdf.TextBox, indices []int) pageShape {
continue
}
if utf8.RuneCountInString(text) > tocMaxProseRunes {
- s.prose++
+ // A leader column is typographic filler, never prose. OCR detection
+ // misses it, so the native-text recovery emits it as its own long
+ // box; counting it as prose kept every page it appeared on.
+ if tocLeaderOnlyPattern.MatchString(text) {
+ continue
+ }
// A merged TOC block is long by definition, so the run count is the
- // only evidence it can carry. Cap the search: the answer is a
- // three-way comparison, not the exact number of runs.
+ // only evidence it can carry. It is confirmed entries, not prose:
+ // counting it as prose let three merged blocks guard their own page.
if len(tocEntryRunPattern.FindAllStringIndex(text, tocMinMergedRuns)) >= tocMinMergedRuns {
s.merged = true
+ continue
+ }
+ // An entry line — optional title, leaders, page number — is a TOC
+ // entry, not a paragraph. It confirms the page like any other
+ // marker instead of contradicting it as body text. Require a real
+ // tab leader here: a prose sentence that happens to end in an
+ // ellipsis and a number must keep counting as prose.
+ if isTOCEntryLine(text, tocMinLongEntryLeader) {
+ s.markers++
+ s.entries++
+ continue
}
+ s.prose++
continue
}
s.short++
- if chapterEntryPattern.MatchString(text) || pageNumberPattern.MatchString(text) {
+ if chapterEntryPattern.MatchString(text) || pageNumberPattern.MatchString(text) || isTOCEntryLine(text, tocMinShortEntryLeader) {
s.markers++
continue
}
@@ -347,14 +375,52 @@ func shapeOfPage(boxes []pdf.TextBox, indices []int) pageShape {
return s
}
-// isTOC reports whether the page reads as a table of contents: a list of short
-// boxes carrying entry markers, or one long box that is itself a run of
-// entries, and no body text either way.
+// isTOCEntryLine reports whether text ends with one TOC entry's tail: a leader
+// run followed by the entry's page number. minLeader is the shortest leader run
+// accepted, so a long box only reads as an entry when its tab leader is real.
+// Space-separated leader segments count as one run (OCR often splits them).
+// The leader+number grammar is reuse of tocEntryRunPattern; prose that merely
+// contains an ellipsis does not end in a run and never matches.
+func isTOCEntryLine(text string, minLeader int) bool {
+ locs := tocEntryRunPattern.FindAllStringIndex(text, -1)
+ if len(locs) == 0 {
+ return false
+ }
+ loc := locs[len(locs)-1]
+ if strings.TrimSpace(text[loc[1]:]) != "" {
+ return false
+ }
+ // Walk back over the leader run so segments split by spaces all count.
+ start := loc[0]
+ for start > 0 {
+ r, size := utf8.DecodeLastRuneInString(text[:start])
+ if r == ' ' || r == '\t' || r == '.' || r == '…' || r == '⋯' || r == '·' {
+ start -= size
+ continue
+ }
+ break
+ }
+ leader := 0
+ for _, r := range text[start:loc[1]] {
+ switch r {
+ case '.', '…', '⋯', '·':
+ leader++
+ case ' ', '\t':
+ default:
+ return leader >= minLeader
+ }
+ }
+ return false
+}
+
+// isTOC reports whether the page reads as a table of contents: a list of entry
+// boxes carrying markers, or one long box that is itself a run of entries, and
+// no body text either way.
func (s pageShape) isTOC() bool {
if s.carriesProse() {
return false
}
- return s.merged || (s.short >= tocMinShortBoxes && s.markers >= tocMinEntries)
+ return s.merged || (s.short+s.entries >= tocMinShortBoxes && s.markers >= tocMinEntries)
}
// carriesProse reports whether the page carries body text. No signal may drop
diff --git a/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go b/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go
index 4fa60635c7..064b763911 100644
--- a/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go
+++ b/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go
@@ -667,6 +667,237 @@ func TestRemoveTOCBoxes_ShortLineBodyPageIsNotTOC(t *testing.T) {
}
}
+// TestRemoveTOCBoxes_LeaderPageNumberBoxesAreEntries pins the shape the
+// OCR-merge path produces after #21399: detection misses the leader dots, so
+// native-text recovery emits them as their own boxes (".... 1"). Those boxes
+// are longer than tocBodyRunes, so the body guard used to read the TOC page as
+// body text and keep it. They are entries and must count as markers.
+func TestRemoveTOCBoxes_LeaderPageNumberBoxesAreEntries(t *testing.T) {
+ boxes := []pdf.TextBox{tb("目录", 0, 72, 110, 100, 120)}
+ titles := []string{
+ "第一章 道可道", "第二章 天下皆知", "第三章 不尚贤", "第四章 道冲",
+ "第五章 守中", "第六章 谷神", "第七章 后其身而身先", "第八章 夫唯不争",
+ }
+ leaders := []string{
+ ".............................................. 1",
+ ".............................................. 5",
+ "............................................. 9",
+ "............ ................................ 13",
+ "............ ................................ 17",
+ "............ ................................. 21",
+ "........... ............................... 25",
+ "........... ................................... 29",
+ }
+ for i, title := range titles {
+ y := float64(160 + i*30)
+ boxes = append(boxes, tb(title, 0, 72, 200, y, y+15))
+ boxes = append(boxes, tb(leaders[i], 0, 72, 520, y, y+15))
+ }
+ boxes = append(boxes, bodyPageBoxes(1)...)
+ boxes = append(boxes, bodyPageBoxes(2)...)
+
+ got := removeTOCWholeDoc(boxes, nil)
+ if n := countPage(got, 0); n != 0 {
+ t.Fatalf("leader+page-number boxes read as entries, the TOC page must be dropped, %d boxes survived", n)
+ }
+ if n := countPage(got, 1); n != 3 {
+ t.Fatalf("body page 1 must survive, %d of 3 boxes", n)
+ }
+}
+
+// TestRemoveTOCBoxes_LongLeaderBoxesAreNotProse pins the real 道德经 shape:
+// OCR detection misses the leader dots, so native-text recovery emits each
+// leader column as its own 50–100 rune box, while titles and page numbers land
+// in separate boxes. The leader boxes are longer than tocMaxProseRunes but
+// carry no text; counting them as prose kept the whole TOC.
+func TestRemoveTOCBoxes_LongLeaderBoxesAreNotProse(t *testing.T) {
+ boxes := []pdf.TextBox{tb("目录", 0, 72, 110, 100, 120)}
+ titles := []string{
+ "第1章 “道”", "第2章 圣人居无为之事", "第3章 无为而治", "第4章 道冲",
+ "第5章 “守中”", "第6章 “谷神”", "第7章 后其身而身先", "第8章 夫唯不争",
+ }
+ for i, title := range titles {
+ y := float64(160 + i*30)
+ boxes = append(boxes,
+ tb(title, 0, 72, 220, y, y+15),
+ tb(strings.Repeat(".", 40+i*7), 0, 220, 500, y, y+15),
+ tb(fmt.Sprintf(".%d", 3+i*2), 0, 500, 520, y, y+15),
+ )
+ }
+ boxes = append(boxes, bodyPageBoxes(1)...)
+ boxes = append(boxes, bodyPageBoxes(2)...)
+
+ got := removeTOCWholeDoc(boxes, nil)
+ if n := countPage(got, 0); n != 0 {
+ t.Fatalf("a page of leader columns is a TOC page, %d boxes survived", n)
+ }
+ if n := countPage(got, 1); n != 3 {
+ t.Fatalf("body page 1 must survive, %d of 3 boxes", n)
+ }
+}
+
+// TestRemoveTOCBoxes_LongEntryLinesWithoutBookmark pins the second half of the
+// recovered-line fix: a bookmarkless TOC whose entries are each one long
+// "title + leaders + number" box has almost no short boxes, so the entry lines
+// themselves must count toward the list threshold. They have real tab leaders,
+// so they are entries rather than prose.
+func TestRemoveTOCBoxes_LongEntryLinesWithoutBookmark(t *testing.T) {
+ boxes := []pdf.TextBox{tb("目录", 0, 72, 110, 100, 120)}
+ for i := 0; i < 8; i++ {
+ y := float64(160 + i*30)
+ title := fmt.Sprintf("第%d章 ", i+1) + strings.Repeat("说明", 20)
+ boxes = append(boxes, tb(title+" "+strings.Repeat(".", 20)+fmt.Sprintf(" %d", 3+i*2), 0, 72, 520, y, y+15))
+ }
+ boxes = append(boxes, bodyPageBoxes(1)...)
+ boxes = append(boxes, bodyPageBoxes(2)...)
+
+ got := removeTOCWholeDoc(boxes, nil)
+ if n := countPage(got, 0); n != 0 {
+ t.Fatalf("long entry lines must count toward the TOC shape, %d boxes survived", n)
+ }
+}
+
+// TestRemoveTOCBoxes_LongProseWithEllipsisIsNotEntry keeps the long-box rule
+// safe: prose ending in an ellipsis and a number does not have a real tab
+// leader, so it still counts as prose and protects the page.
+func TestRemoveTOCBoxes_LongProseWithEllipsisIsNotEntry(t *testing.T) {
+ line := func(tail string) string {
+ return "The measured value is reported in the appendix table " +
+ strings.Repeat("with detail ", 4) + tail
+ }
+ boxes := []pdf.TextBox{
+ tb("目录", 0, 72, 110, 100, 120),
+ tb("第1章 绪论", 0, 72, 200, 160, 175),
+ tb("第2章 背景", 0, 72, 200, 190, 205),
+ tb("第3章 方法", 0, 72, 200, 220, 235),
+ tb(line("... 42"), 0, 72, 520, 250, 265),
+ tb(line("... 43"), 0, 72, 520, 280, 295),
+ tb(line("... 44"), 0, 72, 520, 310, 325),
+ }
+ boxes = append(boxes, bodyPageBoxes(1)...)
+ got := removeTOCWholeDoc(boxes, map[int]bool{0: true})
+ if n := countPage(got, 0); n != 7 {
+ t.Fatalf("prose with a short ellipsis must protect the page, %d of 7 boxes survived", n)
+ }
+}
+
+// TestRemoveTOCBoxes_RomanEntryLines pins Roman-only page numbers: "Preface
+// .... vii" has no decimal number, so the run grammar must accept a Roman
+// numeral on its own.
+func TestRemoveTOCBoxes_RomanEntryLines(t *testing.T) {
+ boxes := []pdf.TextBox{tb("Contents", 0, 72, 110, 100, 120)}
+ romans := []string{"i", "ii", "iii", "iv", "v", "vi", "vii", "viii"}
+ for i := 0; i < 8; i++ {
+ y := float64(160 + i*30)
+ title := "Preface " + strings.Repeat("notes ", 10)
+ boxes = append(boxes, tb(title+" "+strings.Repeat(".", 12)+" "+romans[i], 0, 72, 520, y, y+15))
+ }
+ boxes = append(boxes, bodyPageBoxes(1)...)
+ boxes = append(boxes, bodyPageBoxes(2)...)
+
+ got := removeTOCWholeDoc(boxes, nil)
+ if n := countPage(got, 0); n != 0 {
+ t.Fatalf("roman-numbered entry lines must be detected, %d boxes survived", n)
+ }
+}
+
+// TestRemoveTOCBoxes_MergedBlocksAreNotProse: several long boxes that each
+// hold multiple leader+number runs are confirmed entries. Counting them as
+// prose let three such blocks protect their own page.
+func TestRemoveTOCBoxes_MergedBlocksAreNotProse(t *testing.T) {
+ boxes := []pdf.TextBox{tb("目录", 0, 72, 110, 100, 120)}
+ block := "Chapter One ...... 3 Chapter Two ...... 4 Chapter Three ...... 5"
+ for i := 0; i < 3; i++ {
+ y := float64(160 + i*30)
+ boxes = append(boxes, tb(block, 0, 72, 520, y, y+15))
+ }
+ boxes = append(boxes, bodyPageBoxes(1)...)
+ boxes = append(boxes, bodyPageBoxes(2)...)
+
+ got := removeTOCWholeDoc(boxes, nil)
+ if n := countPage(got, 0); n != 0 {
+ t.Fatalf("merged TOC blocks must not guard their own page, %d boxes survived", n)
+ }
+}
+
+// TestRemoveTOCBoxes_SpacedLeaderRunsAreEntries: OCR often splits a leader run
+// into space-separated segments. The threshold must count the whole run, not
+// just the last segment before the page number.
+func TestRemoveTOCBoxes_SpacedLeaderRunsAreEntries(t *testing.T) {
+ boxes := []pdf.TextBox{tb("目录", 0, 72, 110, 100, 120)}
+ for i := 0; i < 8; i++ {
+ y := float64(160 + i*30)
+ title := fmt.Sprintf("第%d章 ", i+1) + strings.Repeat("说明", 20)
+ text := title + " " + strings.Repeat(".", 30) + " " + strings.Repeat(".", 4) + fmt.Sprintf(" %d", 3+i*2)
+ boxes = append(boxes, tb(text, 0, 72, 520, y, y+15))
+ }
+ boxes = append(boxes, bodyPageBoxes(1)...)
+ boxes = append(boxes, bodyPageBoxes(2)...)
+
+ got := removeTOCWholeDoc(boxes, nil)
+ if n := countPage(got, 0); n != 0 {
+ t.Fatalf("space-separated leader runs must count as one run, %d boxes survived", n)
+ }
+}
+
+// TestRemoveTOCBoxes_ShortProseWithEllipsisIsNotEntry: on a short box, a bare
+// "..." before a number is sentence punctuation, not a tab leader. Three such
+// prose lines must keep the page.
+func TestRemoveTOCBoxes_ShortProseWithEllipsisIsNotEntry(t *testing.T) {
+ boxes := []pdf.TextBox{
+ tb("目录", 0, 72, 110, 100, 120),
+ tb("第1章 绪论", 0, 72, 200, 160, 175),
+ tb("第2章 背景", 0, 72, 200, 190, 205),
+ tb("第3章 方法", 0, 72, 200, 220, 235),
+ tb("The measured result is ... 42", 0, 72, 520, 250, 265),
+ tb("The second result is ... 43", 0, 72, 520, 280, 295),
+ tb("The third result is ... 44", 0, 72, 520, 310, 325),
+ }
+ boxes = append(boxes, bodyPageBoxes(1)...)
+ got := removeTOCWholeDoc(boxes, map[int]bool{0: true})
+ if n := countPage(got, 0); n != 7 {
+ t.Fatalf("short prose ending in an ellipsis must protect the page, %d of 7 boxes survived", n)
+ }
+}
+
+// TestRemoveTOCBoxes_RomanRunNeedsWordBoundary: without a word boundary the
+// Roman alternative matches the first letter of ordinary words, so three
+// "... word" fragments in one prose box look like a merged TOC block and the
+// page is dropped. The prose must keep protecting itself.
+func TestRemoveTOCBoxes_RomanRunNeedsWordBoundary(t *testing.T) {
+ boxes := []pdf.TextBox{
+ tb("目录", 0, 72, 110, 100, 120),
+ tb("第1章 绪论", 0, 72, 200, 160, 175),
+ tb("第2章 背景", 0, 72, 200, 190, 205),
+ tb("第3章 方法", 0, 72, 200, 220, 235),
+ tb("The results were ... delivered ... discussed ... measured in the appendix.", 0, 72, 520, 250, 265),
+ tb("The second set was ... delivered ... discussed ... measured in the same way.", 0, 72, 520, 280, 295),
+ tb("The third set was ... delivered ... discussed ... measured before the review.", 0, 72, 520, 310, 325),
+ }
+ boxes = append(boxes, bodyPageBoxes(1)...)
+ got := removeTOCWholeDoc(boxes, nil)
+ if n := countPage(got, 0); n != 7 {
+ t.Fatalf("prose fragments must not form a merged TOC block, %d of 7 boxes survived", n)
+ }
+}
+
+// TestRemoveTOCBoxes_ProseWithEllipsisIsNotAnEntry keeps the entry pattern
+// from weakening the body guard: an ellipsis followed by words (not a page
+// number) is prose, and so is a page of such lines.
+func TestRemoveTOCBoxes_ProseWithEllipsisIsNotAnEntry(t *testing.T) {
+ boxes := []pdf.TextBox{
+ tb("第一章 绪论", 0, 72, 200, 100, 115),
+ tb("The argument continues here... and the reader should follow it carefully.", 0, 72, 520, 130, 145),
+ tb("A second sentence of prose... with an ellipsis and more words after it.", 0, 72, 520, 160, 175),
+ tb("A third sentence of prose... that only looks like an entry at a glance.", 0, 72, 520, 190, 205),
+ }
+ boxes = append(boxes, bodyPageBoxes(1)...)
+ got := removeTOCWholeDoc(boxes, nil)
+ if n := countPage(got, 0); n != 4 {
+ t.Fatalf("prose with an ellipsis must stay, %d of 4 boxes survived", n)
+ }
+}
+
func TestRemoveTOCText_Normalize(t *testing.T) {
cases := []struct{ in, want string }{
{"Hello World", "hello world"},