RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): keep short-line body pages out of PDF TOC removal (#20988)
jay777215753d7b27f66
diff --git a/internal/deepdoc/parser/pdf/layout/toc_header_footer.go b/internal/deepdoc/parser/pdf/layout/toc_header_footer.goindex 7f9aebb90c..b0524fdfea 100644--- a/internal/deepdoc/parser/pdf/layout/toc_header_footer.go+++ b/internal/deepdoc/parser/pdf/layout/toc_header_footer.go@@ -26,6 +26,19 @@ const (// tocMaxLongBoxes is the maximum number of boxes longer than// tocMaxProseRunes a page may carry and still be classified as TOC.tocMaxLongBoxes = 2+ // tocBodyRunes is the length (in runes) above which a box that is+ // neither a chapter marker nor a leader run counts as body evidence.+ // The prose guard above keys on long boxes, which a corpus of short+ // lines (a legal code's articles, verse, lists) never produces — the+ // 民法典's article lines wrap at ~35 runes, so prose stayed 0 on every+ // page and a mixed TOC-tail + article page was read as a TOC page and+ // dropped whole. A body line is still longer than a TOC entry title+ // and does not open with a chapter marker, so a page carrying several+ // such boxes is a body page no signal may drop.+ tocBodyRunes = 20+ // tocMaxBodyBoxes is the maximum number of body-evidence boxes a page+ // may carry and still be classified as TOC.+ tocMaxBodyBoxes = 2// tocMinShortBoxes is the minimum number of short boxes a TOC page carries.// A TOC is a list, so its entries usually stay separate boxes. A page whose// entries were merged into one long box is covered by tocMinMergedRuns, and@@ -88,6 +101,11 @@ var (// one long box apart from a paragraph. Digits accept the full-width forms// pageNumberPattern accepts, for the same reason.tocEntryRunPattern = regexp.MustCompile(`(\.{2,}|…{2,}|⋯{2,}|·{2,})\s*[0-90-9]{1,4}\s*[IVXLCDMivxlcdm]{0,5}`)+ // tocLeaderOnlyPattern matches a box that is nothing but leader+ // characters (dots, ellipses, middots, spaces): the leader column of a+ // dotted TOC entry. Such a box is long but carries no body text, so it+ // must not count as body evidence.+ tocLeaderOnlyPattern = regexp.MustCompile(`^[.…⋯·\s]+$`))// ---------------------------------------------------------------------------@@ -191,10 +209,11 @@ func outlineTitle(title string) string {// and only consulted when the parse begins in the document's front matter.//// Both signals pass through one guard: a page carrying body text (more than-// tocMaxLongBoxes boxes longer than tocMaxProseRunes) is never dropped, and a-// document consisting only of TOC pages is left untouched. The detector stays-// deliberately conservative — missing a TOC page costs noise chunks, deleting a-// content page loses text.+// tocMaxLongBoxes boxes longer than tocMaxProseRunes, or more than+// tocMaxBodyBoxes marker-less boxes longer than tocBodyRunes) is never+// dropped, and a document consisting only of TOC pages is left untouched.+// The detector stays deliberately conservative — missing a TOC page costs+// noise chunks, deleting a content page loses text.func RemoveTOCBoxes(boxes []pdf.TextBox, outlinePages map[int]bool, firstParsedPage, totalPages int) []pdf.TextBox {if len(boxes) == 0 {return boxes@@ -250,7 +269,7 @@ func RemoveTOCBoxes(boxes []pdf.TextBox, outlinePages map[int]bool, firstParsedPselected[pg] = truecontinue}- if shapes[pg].carriesProse() || lead >= tocMaxLeadPages {+ if shapes[pg].carriesLongProse() || lead >= tocMaxLeadPages {break}lead++@@ -296,6 +315,7 @@ type pageShape struct {short intmarkers int // short boxes opening with a chapter marker or holding a page numbermerged bool // a long box carrying tocMinMergedRuns leader+page-number runs+ body int // marker-less, leader-less short boxes longer than tocBodyRunes}func shapeOfPage(boxes []pdf.TextBox, indices []int) pageShape {@@ -318,6 +338,10 @@ func shapeOfPage(boxes []pdf.TextBox, indices []int) pageShape {s.short++if chapterEntryPattern.MatchString(text) || pageNumberPattern.MatchString(text) {s.markers+++ continue+ }+ if utf8.RuneCountInString(text) > tocBodyRunes && !tocLeaderOnlyPattern.MatchString(text) {+ s.body++}}return s@@ -334,8 +358,18 @@ func (s pageShape) isTOC() bool {}// carriesProse reports whether the page carries body text. No signal may drop-// such a page: this is the single guard that bounds every removal.-func (s pageShape) carriesProse() bool { return s.prose > tocMaxLongBoxes }+// such a page: this is the single guard that bounds every removal. Body text+// is evidenced by long boxes (prose), or — for corpora whose lines are all+// short, such as a legal code's articles or verse — by several marker-less,+// leader-less boxes longer than tocBodyRunes.+func (s pageShape) carriesProse() bool { return s.carriesLongProse() || s.body > tocMaxBodyBoxes }++// carriesLongProse reports the long-box signal alone. The leading-run scan+// uses it rather than carriesProse: a cover or copyright page carries short+// lines (its metadata block) but no body text, and the body-evidence arm+// would stop the scan on such a page before it reaches the TOC. Every drop+// still passes carriesProse, so the body evidence keeps bounding removal.+func (s pageShape) carriesLongProse() bool { return s.prose > tocMaxLongBoxes }// ---------------------------------------------------------------------------// Running header / footer removal.diff --git a/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go b/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.goindex 6ee4951719..4fa60635c7 100644--- a/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go+++ b/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go@@ -635,6 +635,38 @@ func TestRemoveTOCBoxes_BareCJKChapterTitles(t *testing.T) {}}+// TestRemoveTOCBoxes_ShortLineBodyPageIsNotTOC pins the 民法典 case: a page+// that starts with the tail of the TOC and continues into body text whose+// lines are all short (a legal code's articles wrap well below+// tocMaxProseRunes) must not be read as a TOC page. Before the body-evidence+// arm of the guard such a page had prose=0, passed isTOC(), and was dropped+// whole — losing the body text it carried.+func TestRemoveTOCBoxes_ShortLineBodyPageIsNotTOC(t *testing.T) {+ boxes := []pdf.TextBox{+ tb("第六章 医疗损害责任", 0, 72, 200, 100, 115),+ tb("第七章 环境污染和生态破坏责任", 0, 72, 220, 130, 145),+ tb("第八章 高度危险责任", 0, 72, 200, 160, 175),+ tb("第九章 饲养动物损害责任", 0, 72, 200, 190, 205),+ tb("第十章 建筑物和物件损害责任", 0, 72, 220, 220, 235),+ tb("附则", 0, 72, 100, 250, 265),+ tb("第一编 总则", 0, 72, 200, 280, 295),+ tb("第一章 基本规定", 0, 72, 200, 310, 325),+ tb("第一条 【立法目的】为了保护民事主体的合法权益,调整", 0, 72, 520, 340, 355),+ tb("和经济秩序,适应中国特色社会主义发展要求,弘扬社会主", 0, 72, 520, 360, 375),+ tb("宪法,制定本法。", 0, 72, 200, 380, 395),+ tb("第二条 【调整范围】民法调整平等主体的自然人、法人和", 0, 72, 520, 400, 415),+ tb("身关系和财产关系。", 0, 72, 200, 420, 435),+ tb("第三条 【民事权益受法律保护】民事主体的人身权利、财", 0, 72, 520, 440, 455),+ }+ boxes = append(boxes, bodyPageBoxes(1)...)+ boxes = append(boxes, bodyPageBoxes(2)...)++ got := removeTOCWholeDoc(boxes, nil)+ if n := countPage(got, 0); n != 14 {+ t.Fatalf("a mixed TOC-tail + short-line body page must be kept, %d of 14 boxes survived", n)+ }+}+func TestRemoveTOCText_Normalize(t *testing.T) {cases := []struct{ in, want string }{{"Hello World", "hello world"},