RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): make PDF "remove TOC" and "remove header/footer" options work without DeepDoc inference (#18602)
euvre0cc348ad22d7
diff --git a/internal/parser/parser/pdf_postprocess.go b/internal/parser/parser/pdf_postprocess.goindex 19f3cd8042..da28424909 100644--- a/internal/parser/parser/pdf_postprocess.go+++ b/internal/parser/parser/pdf_postprocess.go@@ -73,6 +73,7 @@ func applyPDFPostProcess(result *deepdoctype.ParseResult, opts pdfPostProcessOptnormalizePDFLayoutTypes(result)if opts.removeHeaderFooter {filterPDFHeaderFooter(result)+ removePDFRunningHeaderFooter(result)}assignPDFDocTypeKeywords(result, opts.flattenMediaToText)}@@ -98,6 +99,117 @@ func filterPDFHeaderFooter(result *deepdoctype.ParseResult) {result.Sections = filtered}+const (+ // pdfHeaderZoneRatio / pdfFooterZoneRatio bound the page zones where+ // running headers/footers live: sections sitting entirely in the top+ // or bottom fraction of a page are candidates.+ pdfHeaderZoneRatio = 0.10+ pdfFooterZoneRatio = 0.90+ // pdfMinPagesForHeaderFooter guards against false positives on short+ // documents: cross-page repetition only means "running header/footer"+ // when the document has enough pages.+ pdfMinPagesForHeaderFooter = 3+)++var pdfRunningDigitPattern = regexp.MustCompile(`\d+`)++// normalizePDFRunningText collapses whitespace and replaces digit runs so+// that per-page variants of the same running header/footer ("- 2 -",+// "- 3 -", "Page 4 of 9") share one comparison key.+func normalizePDFRunningText(text string) string {+ t := strings.Join(strings.Fields(text), " ")+ t = pdfRunningDigitPattern.ReplaceAllString(t, "#")+ return strings.ToLower(strings.TrimSpace(t))+}++// removePDFRunningHeaderFooter drops sections whose normalized text repeats+// in the same page zone (top 10% or bottom 10%) on at least half of the+// document's pages — running headers, footers, and page numbers. Every+// rendered page counts toward the total, including blank or image-only+// pages that produced no sections. The+// layout-type filter above only works when a DeepDoc inference service+// (DEEPDOC_URL) types header/footer regions; in default deployments there+// is none, every section stays "text", and 页眉页脚 removal silently does+// nothing. This positional frequency pass makes the option work without an+// inference service. Repetition — not position alone — is the guard that+// keeps genuine body text: a section must recur on >= max(2, pages/2) pages+// in the same zone to be removed.+func removePDFRunningHeaderFooter(result *deepdoctype.ParseResult) {+ sections := result.Sections+ if len(sections) == 0 || len(result.PageHeight) == 0 {+ return+ }+ numPages := len(result.PageHeight)+ if numPages < pdfMinPagesForHeaderFooter {+ return+ }+ type zoneKey struct {+ header bool+ text string+ }+ keyPages := make(map[zoneKey]map[int]struct{})+ keySections := make(map[zoneKey][]int)+ for i := range sections {+ s := §ions[i]+ if layout := strings.TrimSpace(s.LayoutType); layout != "" && layout != deepdoctype.LayoutTypeText {+ continue+ }+ if len(s.Positions) == 0 || len(s.Positions[0].PageNumbers) == 0 {+ continue+ }+ page := s.Positions[0].PageNumbers[0]+ pageHeight := result.PageHeight[page]+ if pageHeight <= 0 {+ continue+ }+ top, bottom := s.Positions[0].Top, s.Positions[0].Bottom+ if bottom <= top {+ continue+ }+ var isHeader, isFooter bool+ switch {+ case bottom <= pageHeight*pdfHeaderZoneRatio:+ isHeader = true+ case top >= pageHeight*pdfFooterZoneRatio:+ isFooter = true+ }+ if !isHeader && !isFooter {+ continue+ }+ norm := normalizePDFRunningText(s.Text)+ if norm == "" {+ continue+ }+ key := zoneKey{header: isHeader, text: norm}+ if keyPages[key] == nil {+ keyPages[key] = make(map[int]struct{})+ }+ keyPages[key][page] = struct{}{}+ keySections[key] = append(keySections[key], i)+ }+ drop := make(map[int]struct{})+ for key, pages := range keyPages {+ n := len(pages)+ if n < 2 || n*2 < numPages {+ continue+ }+ for _, i := range keySections[key] {+ drop[i] = struct{}{}+ }+ }+ if len(drop) == 0 {+ return+ }+ filtered := sections[:0]+ for i, s := range sections {+ if _, ok := drop[i]; ok {+ continue+ }+ filtered = append(filtered, s)+ }+ result.Sections = filtered+}+func assignPDFDocTypeKeywords(result *deepdoctype.ParseResult, flatten bool) {for i := range result.Sections {section := &result.Sections[i]@@ -201,6 +313,16 @@ func removePDFTOC(result *deepdoctype.ParseResult) {if i >= len(sections) || prefix == "" {break}+ // The first entry is formatted (leader run plus page number): the+ // table is entry-shaped and the prefix scan below must not run —+ // with a single formatted entry that scan would delete body text up+ // to the next line sharing the entry's prefix. Drop the entry and+ // resume the title scan; filterPDFTOCEntries removes the remaining+ // entry lines.+ if isPDFTOCEntrySection(sectionText(sections[i])) {+ sections = append(sections[:i], sections[i+1:]...)+ continue+ }sections = append(sections[:i], sections[i+1:]...)if i >= len(sections) || prefix == "" {breakdiff --git a/internal/parser/parser/pdf_postprocess_test.go b/internal/parser/parser/pdf_postprocess_test.goindex f15fcd1d17..0a62f4817c 100644--- a/internal/parser/parser/pdf_postprocess_test.go+++ b/internal/parser/parser/pdf_postprocess_test.go@@ -2,6 +2,7 @@ package parserimport ("slices"+ "strings""testing"deepdoctype "ragflow/internal/deepdoc/parser/type"@@ -99,6 +100,278 @@ func TestApplyPDFPostProcess_HeaderFooterFilteringIsOptional(t *testing.T) {}}+func TestApplyPDFPostProcess_RemoveTOCRemovesAllDotLeaderEntries(t *testing.T) {+ result := &deepdoctype.ParseResult{+ Sections: []deepdoctype.Section{+ {Text: "Contents"},+ {Text: "Chapter One Introduction .............. 2"},+ {Text: "Chapter Two Methods ................... 3"},+ {Text: "Chapter Three Results ................. 4"},+ {Text: "Chapter One Introduction"},+ {Text: "Body text that must survive parsing."},+ {Text: "Chapter Two Methods"},+ {Text: "More body text."},+ },+ }+ applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})+ var kept []string+ for _, s := range result.Sections {+ kept = append(kept, s.Text)+ }+ want := []string{+ "Chapter One Introduction",+ "Body text that must survive parsing.",+ "Chapter Two Methods",+ "More body text.",+ }+ if len(kept) != len(want) {+ t.Fatalf("kept = %v, want %v", kept, want)+ }+ for i := range want {+ if kept[i] != want[i] {+ t.Fatalf("kept = %v, want %v", kept, want)+ }+ }+}++func TestApplyPDFPostProcess_RemoveTOCPrefixScanWithoutEntryPattern(t *testing.T) {+ // Pattern-less TOC entries take the prefix scan: title and first entry+ // go, then everything up to the first line sharing the first entry's+ // prefix (the real body heading).+ result := &deepdoctype.ParseResult{+ Sections: []deepdoctype.Section{+ {Text: "Contents"},+ {Text: "1. Introduction"},+ {Text: "2. Background"},+ {Text: "1. Introduction"},+ {Text: "Body text."},+ },+ }+ applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})+ var kept []string+ for _, s := range result.Sections {+ kept = append(kept, s.Text)+ }+ want := []string{"1. Introduction", "Body text."}+ if len(kept) != len(want) {+ t.Fatalf("kept = %v, want %v", kept, want)+ }+ for i := range want {+ if kept[i] != want[i] {+ t.Fatalf("kept = %v, want %v", kept, want)+ }+ }+}++func TestApplyPDFPostProcess_RemoveTOCSingleFormattedEntryKeepsBody(t *testing.T) {+ // Only the first entry after the TOC title is formatted. It must count+ // as a consumed entry so the prefix scan never runs: that scan would+ // otherwise delete every line up to the next "Cha"-prefixed heading+ // ("Chapter Two"), dropping the body text between the TOC and it.+ result := &deepdoctype.ParseResult{+ Sections: []deepdoctype.Section{+ {Text: "Contents"},+ {Text: "Chapter One ..... 2"},+ {Text: "Body A"},+ {Text: "Body B"},+ {Text: "Chapter Two"},+ },+ }+ applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})+ var kept []string+ for _, s := range result.Sections {+ kept = append(kept, s.Text)+ }+ want := []string{"Body A", "Body B", "Chapter Two"}+ if len(kept) != len(want) {+ t.Fatalf("kept = %v, want %v", kept, want)+ }+ for i := range want {+ if kept[i] != want[i] {+ t.Fatalf("kept = %v, want %v", kept, want)+ }+ }+}++func TestApplyPDFPostProcess_RemoveRunningHeaderFooterWithoutDLA(t *testing.T) {+ const pageHeight = 842.0+ result := &deepdoctype.ParseResult{+ PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight},+ Sections: []deepdoctype.Section{+ makePDFSection("Contents", "", 0, 72, 158, 40, 60),+ makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 1, 72, 210, 21, 30),+ makePDFSection("Chapter One Introduction", "", 1, 72, 206, 88, 100),+ makePDFSection("Body text one.", "", 1, 72, 363, 124, 136),+ makePDFSection("- 2 -", "", 1, 289, 305, 807, 817),+ makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 2, 72, 210, 21, 30),+ makePDFSection("Chapter Two Methods", "", 2, 72, 190, 88, 100),+ makePDFSection("Body text two.", "", 2, 72, 374, 124, 136),+ makePDFSection("- 3 -", "", 2, 289, 305, 807, 817),+ makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 3, 72, 210, 21, 30),+ makePDFSection("Chapter Three Results", "", 3, 72, 192, 88, 100),+ makePDFSection("Body text three.", "", 3, 72, 367, 124, 136),+ makePDFSection("- 4 -", "", 3, 289, 305, 807, 817),+ },+ }+ applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})+ for _, s := range result.Sections {+ if strings.Contains(s.Text, "CONFIDENTIAL") {+ t.Fatalf("running header survived: %q", s.Text)+ }+ if strings.HasPrefix(strings.TrimSpace(s.Text), "- ") {+ t.Fatalf("page-number footer survived: %q", s.Text)+ }+ }+ if len(result.Sections) != 7 {+ t.Fatalf("len(Sections) = %d, want 7 (body content preserved)", len(result.Sections))+ }+}++func TestApplyPDFPostProcess_RunningHeaderFooterKeepsUniqueZoneText(t *testing.T) {+ const pageHeight = 842.0+ result := &deepdoctype.ParseResult{+ PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight},+ Sections: []deepdoctype.Section{+ // Different top-of-page text on every page: not a running header.+ makePDFSection("Unique top line one", "", 0, 72, 210, 21, 30),+ makePDFSection("Body one.", "", 0, 72, 363, 124, 136),+ makePDFSection("Unique top line two", "", 1, 72, 210, 21, 30),+ makePDFSection("Body two.", "", 1, 72, 363, 124, 136),+ makePDFSection("Unique top line three", "", 2, 72, 210, 21, 30),+ makePDFSection("Body three.", "", 2, 72, 363, 124, 136),+ },+ }+ applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})+ if len(result.Sections) != 6 {+ t.Fatalf("len(Sections) = %d, want 6 (unique zone text must survive)", len(result.Sections))+ }+}++func TestApplyPDFPostProcess_RunningHeaderFooterSkipsShortDocuments(t *testing.T) {+ const pageHeight = 842.0+ result := &deepdoctype.ParseResult{+ PageHeight: map[int]float64{0: pageHeight, 1: pageHeight},+ Sections: []deepdoctype.Section{+ makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 0, 72, 210, 21, 30),+ makePDFSection("Body one.", "", 0, 72, 363, 124, 136),+ makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 1, 72, 210, 21, 30),+ makePDFSection("Body two.", "", 1, 72, 363, 124, 136),+ },+ }+ applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})+ if len(result.Sections) != 4 {+ t.Fatalf("len(Sections) = %d, want 4 (2-page doc must not be touched)", len(result.Sections))+ }+}++func TestApplyPDFPostProcess_RunningHeaderFooterCountsAllRenderedPages(t *testing.T) {+ const pageHeight = 842.0+ // Pages 3 and 4 are blank or image-only: they produced no sections but+ // still count toward the page total. A candidate repeated on 2 of the 5+ // rendered pages occurs on fewer than half of the document's pages and+ // must survive.+ result := &deepdoctype.ParseResult{+ PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight, 4: pageHeight},+ Sections: []deepdoctype.Section{+ makePDFSection("Chapter One Introduction", "", 0, 72, 206, 88, 100),+ makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 1, 72, 210, 21, 30),+ makePDFSection("Body text one.", "", 1, 72, 363, 124, 136),+ makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 2, 72, 210, 21, 30),+ makePDFSection("Body text two.", "", 2, 72, 374, 124, 136),+ },+ }+ applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})+ if len(result.Sections) != 5 {+ t.Fatalf("len(Sections) = %d, want 5 (2 of 5 rendered pages is below the repetition threshold)", len(result.Sections))+ }++ // The same header on 3 of the 5 rendered pages meets the threshold and+ // is removed even though two pages contributed no sections.+ result = &deepdoctype.ParseResult{+ PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight, 4: pageHeight},+ Sections: []deepdoctype.Section{+ makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 0, 72, 210, 21, 30),+ makePDFSection("Body text one.", "", 0, 72, 363, 124, 136),+ makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 1, 72, 210, 21, 30),+ makePDFSection("Body text two.", "", 1, 72, 363, 124, 136),+ makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 2, 72, 210, 21, 30),+ makePDFSection("Body text three.", "", 2, 72, 367, 124, 136),+ },+ }+ applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})+ if len(result.Sections) != 3 {+ t.Fatalf("len(Sections) = %d, want 3 (running header removed on 3 of 5 rendered pages)", len(result.Sections))+ }+ for _, s := range result.Sections {+ if strings.Contains(s.Text, "CONFIDENTIAL") {+ t.Fatalf("running header survived: %q", s.Text)+ }+ }+}++func TestApplyPDFPostProcess_RunningHeaderFooterRequiresFullZone(t *testing.T) {+ const pageHeight = 842.0+ // A section that merely starts in the top 10% but extends past it+ // (bottom 200 > 842*0.10) is body content, not a header candidate —+ // even when it repeats on every page. Same for a section that ends in+ // the bottom 10% but starts above it (top 700 < 842*0.90).+ result := &deepdoctype.ParseResult{+ PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight},+ Sections: []deepdoctype.Section{+ makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 0, 72, 210, 40, 200),+ makePDFSection("Long legal disclaimer note", "", 0, 72, 363, 700, 800),+ makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 1, 72, 210, 40, 200),+ makePDFSection("Long legal disclaimer note", "", 1, 72, 363, 700, 800),+ makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 2, 72, 210, 40, 200),+ makePDFSection("Long legal disclaimer note", "", 2, 72, 363, 700, 800),+ },+ }+ applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})+ if len(result.Sections) != 6 {+ t.Fatalf("len(Sections) = %d, want 6 (sections crossing the zone boundary must survive)", len(result.Sections))+ }+}++func TestApplyPDFPostProcess_RunningHeaderFooterNeedsRepetitionNotPosition(t *testing.T) {+ const pageHeight = 842.0+ // Body blocks sitting fully inside the top 10% of every page but never+ // repeating: position alone must not remove them — the guard requires+ // the same normalized text on >= half of the rendered pages.+ result := &deepdoctype.ParseResult{+ PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight},+ Sections: []deepdoctype.Section{+ makePDFSection("Quarterly results summary", "", 0, 72, 300, 21, 60),+ makePDFSection("Ordinary body text on page zero.", "", 0, 72, 363, 124, 136),+ makePDFSection("Annual meeting minutes", "", 1, 72, 300, 21, 60),+ makePDFSection("Ordinary body text on page one.", "", 1, 72, 363, 124, 136),+ makePDFSection("Risk assessment overview", "", 2, 72, 300, 21, 60),+ makePDFSection("Ordinary body text on page two.", "", 2, 72, 363, 124, 136),+ makePDFSection("Compliance audit findings", "", 3, 72, 300, 21, 60),+ makePDFSection("Ordinary body text on page three.", "", 3, 72, 363, 124, 136),+ },+ }+ applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})+ if len(result.Sections) != 8 {+ t.Fatalf("len(Sections) = %d, want 8 (in-zone but non-repeating text must survive)", len(result.Sections))+ }++ // Repetition counts distinct pages, not occurrences: the same text+ // twice on one page still covers a single page of the document.+ result = &deepdoctype.ParseResult{+ PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight},+ Sections: []deepdoctype.Section{+ makePDFSection("Repeated block", "", 0, 72, 300, 21, 40),+ makePDFSection("Repeated block", "", 0, 72, 300, 45, 64),+ makePDFSection("Body text page one.", "", 1, 72, 363, 124, 136),+ makePDFSection("Body text page two.", "", 2, 72, 363, 124, 136),+ },+ }+ applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})+ if len(result.Sections) != 4 {+ t.Fatalf("len(Sections) = %d, want 4 (two occurrences on one page count as one page)", len(result.Sections))+ }+}+func TestApplyPDFPostProcess_RemoveTOCByOutlines(t *testing.T) {result := &deepdoctype.ParseResult{Sections: []deepdoctype.Section{