RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): drop merged-block and wrapped PDF TOC entries when remove_toc is enabled (#18793)
euvref96316fa5eee
diff --git a/internal/parser/parser/pdf_postprocess.go b/internal/parser/parser/pdf_postprocess.goindex cc1466ff3e..19f3cd8042 100644--- a/internal/parser/parser/pdf_postprocess.go+++ b/internal/parser/parser/pdf_postprocess.go@@ -16,6 +16,39 @@ import (var pdfHeaderFooterPattern = regexp.MustCompile(`(?i)header|footer|number`)var pdfTOCTitlePattern = regexp.MustCompile(`(?i)^(contents|目录|目次|table of contents|致谢|acknowledge)$`)+// pdfTOCEntryPattern matches table-of-contents entry lines: a leader run+// (dots, ellipses, midline ellipses, or middots) followed by a+// page number, optionally with a trailing roman-numeral fragment, e.g.+// "第1章 道可道…………3", "Introduction ....... 12", "……18 I". It extends the+// leader heuristic Python already uses in remove_toc_word+// (rag/flow/parser/utils.py) to the PDF path on both sides, so a TOC without+// a "目录"/"contents" heading is still dropped. Wide-space runs are NOT a+// leader here, mirroring pdfTOCEntryAnywherePattern: they are too common in+// prose, so a line merely ending in " 12" is not a TOC entry.+var pdfTOCEntryPattern = regexp.MustCompile(`(\.{2,}|…{2,}|⋯{2,}|·{2,})\s*\d{1,4}\s*[IVXLCivxlc]{0,5}\s*$`)++// pdfTOCEntryAnywherePattern is the non-anchored form of pdfTOCEntryPattern.+// DeepDoc merges the lines of a text block into ONE section whose text joins+// the lines with spaces, so a TOC page usually becomes a single section whose+// trailing text may be a page marker or a footer watermark instead of an+// entry — the anchored pattern alone never fires for such a section. Counting+// explicit leader runs (wide-space leaders excluded, they are too common in+// prose) identifies those merged TOC blocks.+var pdfTOCEntryAnywherePattern = regexp.MustCompile(`(\.{2,}|…{2,}|⋯{2,}|·{2,})\s*\d{1,4}\s*[IVXLCivxlc]{0,5}`)++// pdfTOCBarePageRefPattern matches a line that consists solely of a leader+// run plus a page number (and optional roman-numeral fragment), e.g.+// "…………………39", "………18 I". In wrapped TOCs the entry title and its leader/page+// number are split onto separate lines, leaving such a bare reference behind+// the title line.+var pdfTOCBarePageRefPattern = regexp.MustCompile(`^[\s.·…⋯]*\d{1,4}\s*[IVXLCivxlc]{0,5}[\s.。·…⋯]*$`)++// pdfTOCTitlePrefixPattern matches lines that open like a TOC entry title:+// a chapter/section marker or a common front/back-matter heading. Used only+// to pair a title line with the bare leader+page-number line that follows it,+// so regular prose is never dropped.+var pdfTOCTitlePrefixPattern = regexp.MustCompile(`(?i)^(第\s*[0-90-9一二三四五六七八九十百千]+\s*[章节篇部册卷回讲]|chapter\s+[0-9]+|appendix\s+[a-z0-9]+|section\s+[0-9]+|前言|序言|引言|导论|绪论|后记|结语|附录|索引|参考文献|结论|致谢|acknowledgements?|acknowledgments?)`)+type pdfPostProcessOptions struct {outputFormat stringpageWidth float64@@ -112,7 +145,10 @@ func sortSectionsByPosition(result *deepdoctype.ParseResult) {// applyRemoveTOC mirrors Python parser.py:663-681 three-way dispatch:// - No outlines → pattern-based remove_toc on all sections-// - First outline on page 1 → outline-based remove_toc_pdf+// - First outline on page 1 → outline-based remove_toc_pdf, then the entry+// filter: the outline pass only drops pages when an outline title names+// the TOC ("目录"/"contents"), so a headingless TOC must still run+// through filterPDFTOCEntries instead of being left in place// - First outline after page 1 → pattern-based on pages before the first outlinefunc applyRemoveTOC(result *deepdoctype.ParseResult) {if result == nil {@@ -126,6 +162,7 @@ func applyRemoveTOC(result *deepdoctype.ParseResult) {firstOutlinePage := outlines[0].PageNumberif firstOutlinePage <= 1 {removePDFTOCByOutlines(result, outlines)+ result.Sections = filterPDFTOCEntries(result.Sections)return}splitAt := len(result.Sections)@@ -176,9 +213,69 @@ func removePDFTOC(result *deepdoctype.ParseResult) {break}}+ // The anchored pass above needs a "目录"/"contents" section; a TOC whose+ // pages carry no such heading survives it entirely. Drop the remaining+ // leader+page-number entry lines, mirroring remove_toc in+ // rag/flow/parser/utils.py.+ sections = filterPDFTOCEntries(sections)result.Sections = sections}+// filterPDFTOCEntries removes table-of-contents sections from the stream,+// reusing the backing array. It goes beyond the per-item anchored match that+// Python's remove_toc (rag/flow/parser/utils.py) applies, because the Go+// DeepDoc layout merges a whole TOC block into one section and wrapped TOCs+// split an entry's leader/page number onto its own line:+//+// - a section ending in a leader+page-number run is an entry (anchored+// pattern), as before;+// - a section containing two or more leader+page-number runs is a merged+// TOC block even when its trailing text (page marker, roman numeral or+// footer watermark) is not an entry itself;+// - when a dropped section is a bare leader+page-number reference, the+// preceding kept section is dropped too if it opens like a TOC entry+// title ("第28章 ………" followed by "………39"), pairing the wrapped entry.+//+// Regular prose is kept: a lone leader+number run inside a paragraph matches+// neither the anchored pattern nor the two-run threshold, and title pairing+// only fires on a dropped bare page reference.+func filterPDFTOCEntries(sections []deepdoctype.Section) []deepdoctype.Section {+ filtered := make([]deepdoctype.Section, 0, len(sections))+ for _, s := range sections {+ text := sectionText(s)+ if !isPDFTOCEntrySection(text) {+ filtered = append(filtered, s)+ continue+ }+ if pdfTOCBarePageRefPattern.MatchString(text) && len(filtered) > 0 {+ if isPDFTOCTitleCandidate(sectionText(filtered[len(filtered)-1])) {+ filtered = filtered[:len(filtered)-1]+ }+ }+ }+ return filtered+}++// isPDFTOCEntrySection reports whether the section text is (or is dominated+// by) table-of-contents entries.+func isPDFTOCEntrySection(text string) bool {+ if pdfTOCEntryPattern.MatchString(text) {+ return true+ }+ return len(pdfTOCEntryAnywherePattern.FindAllString(text, -1)) >= 2+}++// isPDFTOCTitleCandidate reports whether a section reads like the title line+// of a TOC entry whose leader/page number lives on the following line: it+// must open with a chapter/section marker and must not contain a leader run+// itself.+func isPDFTOCTitleCandidate(text string) bool {+ if !pdfTOCTitlePrefixPattern.MatchString(text) {+ return false+ }+ return !pdfTOCEntryAnywherePattern.MatchString(text)+}+func sectionText(s deepdoctype.Section) string {return strings.TrimSpace(s.Text)}diff --git a/internal/parser/parser/pdf_postprocess_test.go b/internal/parser/parser/pdf_postprocess_test.goindex 4a1e4fdea6..f15fcd1d17 100644--- a/internal/parser/parser/pdf_postprocess_test.go+++ b/internal/parser/parser/pdf_postprocess_test.go@@ -1,6 +1,7 @@package parserimport (+ "slices""testing"deepdoctype "ragflow/internal/deepdoc/parser/type"@@ -119,6 +120,196 @@ func TestApplyPDFPostProcess_RemoveTOCByOutlines(t *testing.T) {}}+// TestApplyPDFPostProcess_RemoveTOCEntryLines covers TOC pages that carry no+// "目录"/"contents" heading: the removal must still drop the leader+page+// number entry lines while keeping regular content, and only when removeTOC+// is enabled.+func TestApplyPDFPostProcess_RemoveTOCEntryLines(t *testing.T) {+ sections := []deepdoctype.Section{+ makePDFSection("《道德经》全文及翻译", "text", 1, 50, 550, 100, 120),+ makePDFSection("木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879", "text", 1, 50, 550, 120, 140),+ makePDFSection("前言:……………………………………………………………………1", "text", 2, 50, 550, 100, 120),+ makePDFSection("第1章 “道”…………………………………………………………3", "text", 2, 50, 550, 120, 140),+ makePDFSection("第2章 圣人居无为之事,行不言之教…………………………4", "text", 2, 50, 550, 140, 160),+ makePDFSection("第13章 以身为天下,可寄/托天下……………………………18 I", "text", 3, 50, 550, 100, 120),+ makePDFSection("第17章 太上,不知有之;功成事遂,百姓皆谓“我自然”..25", "text", 3, 50, 550, 120, 140),+ makePDFSection("Introduction to the Dao ....... 12", "text", 3, 50, 550, 140, 160),+ makePDFSection("II", "text", 3, 50, 550, 160, 180),+ makePDFSection("前言:本文是《道德经》的白话翻译,供读者参考。", "text", 4, 50, 550, 100, 120),+ makePDFSection("道可道,非常道。名可名,非常名。", "text", 4, 50, 550, 120, 140),+ }++ untouched := &deepdoctype.ParseResult{Sections: append([]deepdoctype.Section(nil), sections...)}+ applyPDFPostProcess(untouched, pdfPostProcessOptions{removeTOC: false})+ if len(untouched.Sections) != len(sections) {+ t.Fatalf("len(Sections) = %d, want %d when removeTOC is false", len(untouched.Sections), len(sections))+ }++ result := &deepdoctype.ParseResult{Sections: append([]deepdoctype.Section(nil), sections...)}+ applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})+ want := []string{+ "《道德经》全文及翻译",+ "木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879",+ "II",+ "前言:本文是《道德经》的白话翻译,供读者参考。",+ "道可道,非常道。名可名,非常名。",+ }+ if len(result.Sections) != len(want) {+ t.Fatalf("len(Sections) = %d, want %d; got %v", len(result.Sections), len(want), sectionTexts(result.Sections))+ }+ for i := range want {+ if got := result.Sections[i].Text; got != want[i] {+ t.Fatalf("Sections[%d].Text = %q, want %q", i, got, want[i])+ }+ }+}++// TestApplyPDFPostProcess_RemoveTOCEntryLinesWithPageOneOutlines covers the+// book-PDF shape: the first outline sits on page 1 and none of its titles+// names the TOC ("目录"/"contents"), so removePDFTOCByOutlines finds no page+// range and is a no-op. The leader+page-number entry filter must still run on+// this dispatch branch, otherwise a TOC without a 目录 heading survives it.+func TestApplyPDFPostProcess_RemoveTOCEntryLinesWithPageOneOutlines(t *testing.T) {+ result := &deepdoctype.ParseResult{+ Sections: []deepdoctype.Section{+ makePDFSection("《道德经》全文及翻译", "text", 1, 50, 550, 100, 120),+ makePDFSection("前言:……………………………………………………………………1", "text", 2, 50, 550, 100, 120),+ makePDFSection("第1章 “道”…………………………………………………………3", "text", 2, 50, 550, 120, 140),+ makePDFSection("Introduction to the Dao ....... 12", "text", 2, 50, 550, 140, 160),+ makePDFSection("道可道,非常道。名可名,非常名。", "text", 3, 50, 550, 100, 120),+ },+ Outlines: []deepdoctype.Outline{+ {Title: "前言", Level: 0, PageNumber: 1},+ {Title: "Chapter 1", Level: 1, PageNumber: 3},+ },+ }+ applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})+ want := []string{"《道德经》全文及翻译", "道可道,非常道。名可名,非常名。"}+ if !slices.Equal(sectionTexts(result.Sections), want) {+ t.Fatalf("sections = %v, want %v", sectionTexts(result.Sections), want)+ }+}++func sectionTexts(sections []deepdoctype.Section) []string {+ texts := make([]string, 0, len(sections))+ for _, s := range sections {+ texts = append(texts, s.Text)+ }+ return texts+}++// TestFilterPDFTOCEntries_MergedBlockTrailingNonEntry covers a TOC page that+// DeepDoc merges into ONE section whose trailing text is not an entry (a+// footer watermark), so the anchored entry pattern alone never fires. The+// block must still be dropped because it is dominated by leader+page-number+// runs (issue: agent workflow PDF "remove original table of contents" not+// taking effect).+func TestFilterPDFTOCEntries_MergedBlockTrailingNonEntry(t *testing.T) {+ sections := []deepdoctype.Section{+ makePDFSection("《道德经》全文及翻译", "text", 1, 50, 550, 100, 120),+ makePDFSection("前言:………………………………………………1 第1章 “道”………………………………………………3 第2章 圣人居无为之事,行不言之教……………………4 第3章 无为而治…………………………………………6 木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879", "text", 1, 50, 550, 120, 300),+ makePDFSection("道可道,非常道。名可名,非常名。", "text", 3, 50, 550, 100, 120),+ }+ got := filterPDFTOCEntries(sections)+ want := []string{"《道德经》全文及翻译", "道可道,非常道。名可名,非常名。"}+ if !slices.Equal(sectionTexts(got), want) {+ t.Fatalf("sections = %v, want %v", sectionTexts(got), want)+ }+}++// TestFilterPDFTOCEntries_WrappedEntryTitlePairsPageRef covers wrapped TOC+// entries whose leader+page number is split onto its own line/section: the+// bare page reference drops the preceding title line, but a preceding+// non-title line (prose, watermark) is kept, and a lone leader run inside+// prose is not enough to drop anything.+func TestFilterPDFTOCEntries_WrappedEntryTitlePairsPageRef(t *testing.T) {+ sections := []deepdoctype.Section{+ makePDFSection("第27章 不贵其师,不爱其资,虽智大迷,是谓要妙", "text", 2, 50, 550, 100, 120),+ makePDFSection("…………………………………………………………………………………………39", "text", 2, 50, 550, 120, 140),+ makePDFSection("木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879", "text", 2, 50, 550, 140, 160),+ makePDFSection("第28章 朴散则为器,圣人用之,则为官长,故大制不割??", "text", 2, 50, 550, 160, 180),+ makePDFSection("………39 II", "text", 2, 50, 550, 180, 200),+ makePDFSection("第1章", "text", 3, 50, 550, 100, 120),+ makePDFSection("道可道,非常道。名可名,非常名。", "text", 3, 50, 550, 120, 140),+ }+ got := filterPDFTOCEntries(sections)+ want := []string{+ "木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879",+ "第1章",+ "道可道,非常道。名可名,非常名。",+ }+ if !slices.Equal(sectionTexts(got), want) {+ t.Fatalf("sections = %v, want %v", sectionTexts(got), want)+ }+}++// TestFilterPDFTOCEntries_KeepsProseWithSingleLeaderRun pins the guard rails:+// prose containing one leader+number mention stays, and a bare page+// reference preceded by prose (no chapter/section title) keeps the prose.+func TestFilterPDFTOCEntries_KeepsProseWithSingleLeaderRun(t *testing.T) {+ sections := []deepdoctype.Section{+ makePDFSection("更多说明见第三章……12 的表格。", "text", 1, 50, 550, 100, 120),+ makePDFSection("………34", "text", 1, 50, 550, 120, 140),+ makePDFSection("天下皆知美之为美,斯恶已。", "text", 2, 50, 550, 100, 120),+ }+ got := filterPDFTOCEntries(sections)+ want := []string{+ "更多说明见第三章……12 的表格。",+ "天下皆知美之为美,斯恶已。",+ }+ if !slices.Equal(sectionTexts(got), want) {+ t.Fatalf("sections = %v, want %v", sectionTexts(got), want)+ }+}++// TestFilterPDFTOCEntries_KeepsProseEndingInWideSpaces pins that a wide-space+// run is not a TOC leader: prose ending in " 12" survives, while the same+// shape with a dot leader is a real entry and is dropped.+func TestFilterPDFTOCEntries_KeepsProseEndingInWideSpaces(t *testing.T) {+ sections := []deepdoctype.Section{+ makePDFSection("See the installation guide for details 12", "text", 1, 50, 550, 100, 120),+ makePDFSection("Troubleshooting ....... 27", "text", 2, 50, 550, 100, 120),+ }+ got := filterPDFTOCEntries(sections)+ want := []string{"See the installation guide for details 12"}+ if !slices.Equal(sectionTexts(got), want) {+ t.Fatalf("sections = %v, want %v", sectionTexts(got), want)+ }+}++// TestFilterPDFTOCEntries_TwoRunProseTradeOff pins the accepted trade-off of+// the merged-block heuristic: prose quoting two leader+number references is+// dropped, because a DeepDoc-merged TOC page has exactly that shape and the+// anchored pattern alone never fires for it. A single run keeps the prose+// (see KeepsProseWithSingleLeaderRun); raising the threshold would+// under-delete real merged TOC blocks.+func TestFilterPDFTOCEntries_TwoRunProseTradeOff(t *testing.T) {+ sections := []deepdoctype.Section{+ makePDFSection("见第三章……12 和第五章……34 的说明。", "text", 1, 50, 550, 100, 120),+ }+ got := filterPDFTOCEntries(sections)+ if len(got) != 0 {+ t.Fatalf("sections = %v, want empty: two leader+number runs read as a merged TOC block", sectionTexts(got))+ }+}++// TestFilterPDFTOCEntries_ChapterTitleWithoutBarePageRefSurvives pins the+// guard of the wrapped-entry pairing: a "第N章"/"Chapter N" line is only+// dropped when the following section is a bare leader+page-number reference+// (see WrappedEntryTitlePairsPageRef); followed by ordinary prose it is body+// content and must stay.+func TestFilterPDFTOCEntries_ChapterTitleWithoutBarePageRefSurvives(t *testing.T) {+ sections := []deepdoctype.Section{+ makePDFSection("第1章 道可道,非常道", "text", 3, 50, 550, 100, 120),+ makePDFSection("道可道,非常道。名可名,非常名。", "text", 3, 50, 550, 120, 140),+ }+ got := filterPDFTOCEntries(sections)+ want := []string{"第1章 道可道,非常道", "道可道,非常道。名可名,非常名。"}+ if !slices.Equal(sectionTexts(got), want) {+ t.Fatalf("sections = %v, want %v", sectionTexts(got), want)+ }+}+func TestApplyPDFPostProcess_ReordersMultiColumnText(t *testing.T) {cases := []struct {name string