Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): drop merged-block and wrapped PDF TOC entries when remove_toc is enabled (#18793)

euvref96316fa5eee
diff --git a/internal/parser/parser/pdf_postprocess.go b/internal/parser/parser/pdf_postprocess.go
index cc1466ff3e..19f3cd8042 100644
--- a/internal/parser/parser/pdf_postprocess.go
+++ b/internal/parser/parser/pdf_postprocess.go
@@ -16,6 +16,39 @@ import (
var pdfHeaderFooterPattern = regexp.MustCompile(`(?i)header|footer|number`)
var pdfTOCTitlePattern = regexp.MustCompile(`(?i)^(contents|目录|目次|table of contents|致谢|acknowledge)$`)
+// pdfTOCEntryPattern matches table-of-contents entry lines: a leader run
+// (dots, ellipses, midline ellipses, or middots) followed by a
+// page number, optionally with a trailing roman-numeral fragment, e.g.
+// "第1章 道可道…………3", "Introduction ....... 12", "……18 I". It extends the
+// leader heuristic Python already uses in remove_toc_word
+// (rag/flow/parser/utils.py) to the PDF path on both sides, so a TOC without
+// a "目录"/"contents" heading is still dropped. Wide-space runs are NOT a
+// leader here, mirroring pdfTOCEntryAnywherePattern: they are too common in
+// prose, so a line merely ending in " 12" is not a TOC entry.
+var pdfTOCEntryPattern = regexp.MustCompile(`(\.{2,}|…{2,}|⋯{2,}|·{2,})\s*\d{1,4}\s*[IVXLCivxlc]{0,5}\s*$`)
+
+// pdfTOCEntryAnywherePattern is the non-anchored form of pdfTOCEntryPattern.
+// DeepDoc merges the lines of a text block into ONE section whose text joins
+// the lines with spaces, so a TOC page usually becomes a single section whose
+// trailing text may be a page marker or a footer watermark instead of an
+// entry — the anchored pattern alone never fires for such a section. Counting
+// explicit leader runs (wide-space leaders excluded, they are too common in
+// prose) identifies those merged TOC blocks.
+var pdfTOCEntryAnywherePattern = regexp.MustCompile(`(\.{2,}|…{2,}|⋯{2,}|·{2,})\s*\d{1,4}\s*[IVXLCivxlc]{0,5}`)
+
+// pdfTOCBarePageRefPattern matches a line that consists solely of a leader
+// run plus a page number (and optional roman-numeral fragment), e.g.
+// "…………………39", "………18 I". In wrapped TOCs the entry title and its leader/page
+// number are split onto separate lines, leaving such a bare reference behind
+// the title line.
+var pdfTOCBarePageRefPattern = regexp.MustCompile(`^[\s.·…⋯]*\d{1,4}\s*[IVXLCivxlc]{0,5}[\s.。·…⋯]*$`)
+
+// pdfTOCTitlePrefixPattern matches lines that open like a TOC entry title:
+// a chapter/section marker or a common front/back-matter heading. Used only
+// to pair a title line with the bare leader+page-number line that follows it,
+// so regular prose is never dropped.
+var pdfTOCTitlePrefixPattern = regexp.MustCompile(`(?i)^(第\s*[0-90-9一二三四五六七八九十百千]+\s*[章节篇部册卷回讲]|chapter\s+[0-9]+|appendix\s+[a-z0-9]+|section\s+[0-9]+|前言|序言|引言|导论|绪论|后记|结语|附录|索引|参考文献|结论|致谢|acknowledgements?|acknowledgments?)`)
+
type pdfPostProcessOptions struct {
outputFormat string
pageWidth float64
@@ -112,7 +145,10 @@ func sortSectionsByPosition(result *deepdoctype.ParseResult) {
// applyRemoveTOC mirrors Python parser.py:663-681 three-way dispatch:
// - No outlines → pattern-based remove_toc on all sections
-// - First outline on page 1 → outline-based remove_toc_pdf
+// - First outline on page 1 → outline-based remove_toc_pdf, then the entry
+// filter: the outline pass only drops pages when an outline title names
+// the TOC ("目录"/"contents"), so a headingless TOC must still run
+// through filterPDFTOCEntries instead of being left in place
// - First outline after page 1 → pattern-based on pages before the first outline
func applyRemoveTOC(result *deepdoctype.ParseResult) {
if result == nil {
@@ -126,6 +162,7 @@ func applyRemoveTOC(result *deepdoctype.ParseResult) {
firstOutlinePage := outlines[0].PageNumber
if firstOutlinePage <= 1 {
removePDFTOCByOutlines(result, outlines)
+ result.Sections = filterPDFTOCEntries(result.Sections)
return
}
splitAt := len(result.Sections)
@@ -176,9 +213,69 @@ func removePDFTOC(result *deepdoctype.ParseResult) {
break
}
}
+ // The anchored pass above needs a "目录"/"contents" section; a TOC whose
+ // pages carry no such heading survives it entirely. Drop the remaining
+ // leader+page-number entry lines, mirroring remove_toc in
+ // rag/flow/parser/utils.py.
+ sections = filterPDFTOCEntries(sections)
result.Sections = sections
}
+// filterPDFTOCEntries removes table-of-contents sections from the stream,
+// reusing the backing array. It goes beyond the per-item anchored match that
+// Python's remove_toc (rag/flow/parser/utils.py) applies, because the Go
+// DeepDoc layout merges a whole TOC block into one section and wrapped TOCs
+// split an entry's leader/page number onto its own line:
+//
+// - a section ending in a leader+page-number run is an entry (anchored
+// pattern), as before;
+// - a section containing two or more leader+page-number runs is a merged
+// TOC block even when its trailing text (page marker, roman numeral or
+// footer watermark) is not an entry itself;
+// - when a dropped section is a bare leader+page-number reference, the
+// preceding kept section is dropped too if it opens like a TOC entry
+// title ("第28章 ………" followed by "………39"), pairing the wrapped entry.
+//
+// Regular prose is kept: a lone leader+number run inside a paragraph matches
+// neither the anchored pattern nor the two-run threshold, and title pairing
+// only fires on a dropped bare page reference.
+func filterPDFTOCEntries(sections []deepdoctype.Section) []deepdoctype.Section {
+ filtered := make([]deepdoctype.Section, 0, len(sections))
+ for _, s := range sections {
+ text := sectionText(s)
+ if !isPDFTOCEntrySection(text) {
+ filtered = append(filtered, s)
+ continue
+ }
+ if pdfTOCBarePageRefPattern.MatchString(text) && len(filtered) > 0 {
+ if isPDFTOCTitleCandidate(sectionText(filtered[len(filtered)-1])) {
+ filtered = filtered[:len(filtered)-1]
+ }
+ }
+ }
+ return filtered
+}
+
+// isPDFTOCEntrySection reports whether the section text is (or is dominated
+// by) table-of-contents entries.
+func isPDFTOCEntrySection(text string) bool {
+ if pdfTOCEntryPattern.MatchString(text) {
+ return true
+ }
+ return len(pdfTOCEntryAnywherePattern.FindAllString(text, -1)) >= 2
+}
+
+// isPDFTOCTitleCandidate reports whether a section reads like the title line
+// of a TOC entry whose leader/page number lives on the following line: it
+// must open with a chapter/section marker and must not contain a leader run
+// itself.
+func isPDFTOCTitleCandidate(text string) bool {
+ if !pdfTOCTitlePrefixPattern.MatchString(text) {
+ return false
+ }
+ return !pdfTOCEntryAnywherePattern.MatchString(text)
+}
+
func sectionText(s deepdoctype.Section) string {
return strings.TrimSpace(s.Text)
}
diff --git a/internal/parser/parser/pdf_postprocess_test.go b/internal/parser/parser/pdf_postprocess_test.go
index 4a1e4fdea6..f15fcd1d17 100644
--- a/internal/parser/parser/pdf_postprocess_test.go
+++ b/internal/parser/parser/pdf_postprocess_test.go
@@ -1,6 +1,7 @@
package parser
import (
+ "slices"
"testing"
deepdoctype "ragflow/internal/deepdoc/parser/type"
@@ -119,6 +120,196 @@ func TestApplyPDFPostProcess_RemoveTOCByOutlines(t *testing.T) {
}
}
+// TestApplyPDFPostProcess_RemoveTOCEntryLines covers TOC pages that carry no
+// "目录"/"contents" heading: the removal must still drop the leader+page
+// number entry lines while keeping regular content, and only when removeTOC
+// is enabled.
+func TestApplyPDFPostProcess_RemoveTOCEntryLines(t *testing.T) {
+ sections := []deepdoctype.Section{
+ makePDFSection("《道德经》全文及翻译", "text", 1, 50, 550, 100, 120),
+ makePDFSection("木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879", "text", 1, 50, 550, 120, 140),
+ makePDFSection("前言:……………………………………………………………………1", "text", 2, 50, 550, 100, 120),
+ makePDFSection("第1章 “道”…………………………………………………………3", "text", 2, 50, 550, 120, 140),
+ makePDFSection("第2章 圣人居无为之事,行不言之教…………………………4", "text", 2, 50, 550, 140, 160),
+ makePDFSection("第13章 以身为天下,可寄/托天下……………………………18 I", "text", 3, 50, 550, 100, 120),
+ makePDFSection("第17章 太上,不知有之;功成事遂,百姓皆谓“我自然”..25", "text", 3, 50, 550, 120, 140),
+ makePDFSection("Introduction to the Dao ....... 12", "text", 3, 50, 550, 140, 160),
+ makePDFSection("II", "text", 3, 50, 550, 160, 180),
+ makePDFSection("前言:本文是《道德经》的白话翻译,供读者参考。", "text", 4, 50, 550, 100, 120),
+ makePDFSection("道可道,非常道。名可名,非常名。", "text", 4, 50, 550, 120, 140),
+ }
+
+ untouched := &deepdoctype.ParseResult{Sections: append([]deepdoctype.Section(nil), sections...)}
+ applyPDFPostProcess(untouched, pdfPostProcessOptions{removeTOC: false})
+ if len(untouched.Sections) != len(sections) {
+ t.Fatalf("len(Sections) = %d, want %d when removeTOC is false", len(untouched.Sections), len(sections))
+ }
+
+ result := &deepdoctype.ParseResult{Sections: append([]deepdoctype.Section(nil), sections...)}
+ applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})
+ want := []string{
+ "《道德经》全文及翻译",
+ "木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879",
+ "II",
+ "前言:本文是《道德经》的白话翻译,供读者参考。",
+ "道可道,非常道。名可名,非常名。",
+ }
+ if len(result.Sections) != len(want) {
+ t.Fatalf("len(Sections) = %d, want %d; got %v", len(result.Sections), len(want), sectionTexts(result.Sections))
+ }
+ for i := range want {
+ if got := result.Sections[i].Text; got != want[i] {
+ t.Fatalf("Sections[%d].Text = %q, want %q", i, got, want[i])
+ }
+ }
+}
+
+// TestApplyPDFPostProcess_RemoveTOCEntryLinesWithPageOneOutlines covers the
+// book-PDF shape: the first outline sits on page 1 and none of its titles
+// names the TOC ("目录"/"contents"), so removePDFTOCByOutlines finds no page
+// range and is a no-op. The leader+page-number entry filter must still run on
+// this dispatch branch, otherwise a TOC without a 目录 heading survives it.
+func TestApplyPDFPostProcess_RemoveTOCEntryLinesWithPageOneOutlines(t *testing.T) {
+ result := &deepdoctype.ParseResult{
+ Sections: []deepdoctype.Section{
+ makePDFSection("《道德经》全文及翻译", "text", 1, 50, 550, 100, 120),
+ makePDFSection("前言:……………………………………………………………………1", "text", 2, 50, 550, 100, 120),
+ makePDFSection("第1章 “道”…………………………………………………………3", "text", 2, 50, 550, 120, 140),
+ makePDFSection("Introduction to the Dao ....... 12", "text", 2, 50, 550, 140, 160),
+ makePDFSection("道可道,非常道。名可名,非常名。", "text", 3, 50, 550, 100, 120),
+ },
+ Outlines: []deepdoctype.Outline{
+ {Title: "前言", Level: 0, PageNumber: 1},
+ {Title: "Chapter 1", Level: 1, PageNumber: 3},
+ },
+ }
+ applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})
+ want := []string{"《道德经》全文及翻译", "道可道,非常道。名可名,非常名。"}
+ if !slices.Equal(sectionTexts(result.Sections), want) {
+ t.Fatalf("sections = %v, want %v", sectionTexts(result.Sections), want)
+ }
+}
+
+func sectionTexts(sections []deepdoctype.Section) []string {
+ texts := make([]string, 0, len(sections))
+ for _, s := range sections {
+ texts = append(texts, s.Text)
+ }
+ return texts
+}
+
+// TestFilterPDFTOCEntries_MergedBlockTrailingNonEntry covers a TOC page that
+// DeepDoc merges into ONE section whose trailing text is not an entry (a
+// footer watermark), so the anchored entry pattern alone never fires. The
+// block must still be dropped because it is dominated by leader+page-number
+// runs (issue: agent workflow PDF "remove original table of contents" not
+// taking effect).
+func TestFilterPDFTOCEntries_MergedBlockTrailingNonEntry(t *testing.T) {
+ sections := []deepdoctype.Section{
+ makePDFSection("《道德经》全文及翻译", "text", 1, 50, 550, 100, 120),
+ makePDFSection("前言:………………………………………………1 第1章 “道”………………………………………………3 第2章 圣人居无为之事,行不言之教……………………4 第3章 无为而治…………………………………………6 木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879", "text", 1, 50, 550, 120, 300),
+ makePDFSection("道可道,非常道。名可名,非常名。", "text", 3, 50, 550, 100, 120),
+ }
+ got := filterPDFTOCEntries(sections)
+ want := []string{"《道德经》全文及翻译", "道可道,非常道。名可名,非常名。"}
+ if !slices.Equal(sectionTexts(got), want) {
+ t.Fatalf("sections = %v, want %v", sectionTexts(got), want)
+ }
+}
+
+// TestFilterPDFTOCEntries_WrappedEntryTitlePairsPageRef covers wrapped TOC
+// entries whose leader+page number is split onto its own line/section: the
+// bare page reference drops the preceding title line, but a preceding
+// non-title line (prose, watermark) is kept, and a lone leader run inside
+// prose is not enough to drop anything.
+func TestFilterPDFTOCEntries_WrappedEntryTitlePairsPageRef(t *testing.T) {
+ sections := []deepdoctype.Section{
+ makePDFSection("第27章 不贵其师,不爱其资,虽智大迷,是谓要妙", "text", 2, 50, 550, 100, 120),
+ makePDFSection("…………………………………………………………………………………………39", "text", 2, 50, 550, 120, 140),
+ makePDFSection("木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879", "text", 2, 50, 550, 140, 160),
+ makePDFSection("第28章 朴散则为器,圣人用之,则为官长,故大制不割??", "text", 2, 50, 550, 160, 180),
+ makePDFSection("………39 II", "text", 2, 50, 550, 180, 200),
+ makePDFSection("第1章", "text", 3, 50, 550, 100, 120),
+ makePDFSection("道可道,非常道。名可名,非常名。", "text", 3, 50, 550, 120, 140),
+ }
+ got := filterPDFTOCEntries(sections)
+ want := []string{
+ "木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879",
+ "第1章",
+ "道可道,非常道。名可名,非常名。",
+ }
+ if !slices.Equal(sectionTexts(got), want) {
+ t.Fatalf("sections = %v, want %v", sectionTexts(got), want)
+ }
+}
+
+// TestFilterPDFTOCEntries_KeepsProseWithSingleLeaderRun pins the guard rails:
+// prose containing one leader+number mention stays, and a bare page
+// reference preceded by prose (no chapter/section title) keeps the prose.
+func TestFilterPDFTOCEntries_KeepsProseWithSingleLeaderRun(t *testing.T) {
+ sections := []deepdoctype.Section{
+ makePDFSection("更多说明见第三章……12 的表格。", "text", 1, 50, 550, 100, 120),
+ makePDFSection("………34", "text", 1, 50, 550, 120, 140),
+ makePDFSection("天下皆知美之为美,斯恶已。", "text", 2, 50, 550, 100, 120),
+ }
+ got := filterPDFTOCEntries(sections)
+ want := []string{
+ "更多说明见第三章……12 的表格。",
+ "天下皆知美之为美,斯恶已。",
+ }
+ if !slices.Equal(sectionTexts(got), want) {
+ t.Fatalf("sections = %v, want %v", sectionTexts(got), want)
+ }
+}
+
+// TestFilterPDFTOCEntries_KeepsProseEndingInWideSpaces pins that a wide-space
+// run is not a TOC leader: prose ending in " 12" survives, while the same
+// shape with a dot leader is a real entry and is dropped.
+func TestFilterPDFTOCEntries_KeepsProseEndingInWideSpaces(t *testing.T) {
+ sections := []deepdoctype.Section{
+ makePDFSection("See the installation guide for details 12", "text", 1, 50, 550, 100, 120),
+ makePDFSection("Troubleshooting ....... 27", "text", 2, 50, 550, 100, 120),
+ }
+ got := filterPDFTOCEntries(sections)
+ want := []string{"See the installation guide for details 12"}
+ if !slices.Equal(sectionTexts(got), want) {
+ t.Fatalf("sections = %v, want %v", sectionTexts(got), want)
+ }
+}
+
+// TestFilterPDFTOCEntries_TwoRunProseTradeOff pins the accepted trade-off of
+// the merged-block heuristic: prose quoting two leader+number references is
+// dropped, because a DeepDoc-merged TOC page has exactly that shape and the
+// anchored pattern alone never fires for it. A single run keeps the prose
+// (see KeepsProseWithSingleLeaderRun); raising the threshold would
+// under-delete real merged TOC blocks.
+func TestFilterPDFTOCEntries_TwoRunProseTradeOff(t *testing.T) {
+ sections := []deepdoctype.Section{
+ makePDFSection("见第三章……12 和第五章……34 的说明。", "text", 1, 50, 550, 100, 120),
+ }
+ got := filterPDFTOCEntries(sections)
+ if len(got) != 0 {
+ t.Fatalf("sections = %v, want empty: two leader+number runs read as a merged TOC block", sectionTexts(got))
+ }
+}
+
+// TestFilterPDFTOCEntries_ChapterTitleWithoutBarePageRefSurvives pins the
+// guard of the wrapped-entry pairing: a "第N章"/"Chapter N" line is only
+// dropped when the following section is a bare leader+page-number reference
+// (see WrappedEntryTitlePairsPageRef); followed by ordinary prose it is body
+// content and must stay.
+func TestFilterPDFTOCEntries_ChapterTitleWithoutBarePageRefSurvives(t *testing.T) {
+ sections := []deepdoctype.Section{
+ makePDFSection("第1章 道可道,非常道", "text", 3, 50, 550, 100, 120),
+ makePDFSection("道可道,非常道。名可名,非常名。", "text", 3, 50, 550, 120, 140),
+ }
+ got := filterPDFTOCEntries(sections)
+ want := []string{"第1章 道可道,非常道", "道可道,非常道。名可名,非常名。"}
+ if !slices.Equal(sectionTexts(got), want) {
+ t.Fatalf("sections = %v, want %v", sectionTexts(got), want)
+ }
+}
+
func TestApplyPDFPostProcess_ReordersMultiColumnText(t *testing.T) {
cases := []struct {
name string