Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): move PDF TOC and header/footer removal to box level (#19779)

jay777215f1a69ff0d98
diff --git a/internal/deepdoc/parser/pdf/layout/toc_header_footer.go b/internal/deepdoc/parser/pdf/layout/toc_header_footer.go
new file mode 100644
index 0000000000..068ed1a4c0
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/layout/toc_header_footer.go
@@ -0,0 +1,450 @@
+package layout
+
+import (
+ "regexp"
+ "sort"
+ "strings"
+ "unicode/utf8"
+
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+)
+
+// ---------------------------------------------------------------------------
+// Tunable thresholds.
+// ---------------------------------------------------------------------------
+
+const (
+ // tocMaxProseRunes is the per-box length (in runes) above which a box is
+ // treated as a prose paragraph. A page with substantial body text has many
+ // such boxes; a TOC page has at most a title or a long chapter heading.
+ // This is the primary guard against deleting content: any page with more
+ // than tocMaxLongBoxes long boxes is never classified as TOC.
+ tocMaxProseRunes = 60
+ // tocMaxLongBoxes is the maximum number of boxes longer than
+ // tocMaxProseRunes a page may carry and still be classified as TOC.
+ tocMaxLongBoxes = 2
+ // tocMinShortBoxes is the minimum number of short boxes a TOC page carries.
+ // A TOC is a list, so its entries usually stay separate boxes. A page whose
+ // entries were merged into one long box is covered by tocMinMergedRuns, and
+ // one merged without leader runs at all is covered by the outline signal.
+ tocMinShortBoxes = 4
+ // tocMinEntries is the minimum number of confirming entry markers (chapter
+ // markers plus page numbers) a page must carry to be classified as TOC.
+ tocMinEntries = 3
+ // tocMinMergedRuns is the number of leader+page-number runs a single long
+ // box must carry before that box is read as a whole TOC block rather than as
+ // prose. A merged block confirms itself — three leader runs are three
+ // entries — so it needs no separate marker count. Prose can accumulate the
+ // occasional run, which is why the threshold is three and not one.
+ tocMinMergedRuns = 3
+ // tocMaxLeadPages bounds how many leading pages without body text may
+ // precede the TOC. A cover, copyright page or frontispiece carries no body
+ // text and is skipped, so a TOC that is not physically page one is still
+ // reachable; the first page carrying body text ends the search.
+ tocMaxLeadPages = 3
+)
+
+// ---------------------------------------------------------------------------
+// Box-level patterns.
+// ---------------------------------------------------------------------------
+
+var (
+ // chapterEntryPattern matches a box that opens like a TOC entry title: a
+ // chapter/section/appendix marker (Chinese or Latin, digit or word form) or
+ // a common front/back-matter heading.
+ chapterEntryPattern = regexp.MustCompile(`(?i)^(第\s*[0-90-9一二三四五六七八九十百千]+\s*[章节篇部册卷回讲目]|chapter\s+(\d+|[a-z]+|one|two|three|four|five|six|seven|eight|nine|ten)|appendix\s+[a-z0-9]+|section\s+\d+|part\s+(\d+|[ivxlcdm]+|[a-z]+)|前言|序言|引言|导论|绪论|后记|结语|附录|索引|参考文献|结论|致谢|跋|序|凡例|目录|acknowledgements?|acknowledgments?)`)
+ // pageNumberPattern matches common page-number forms: pure digits, digits
+ // with leading dots (..18), roman numerals (III, IV), or dash-wrapped
+ // numbers (- 2 -). Full-width digits count as digits, matching the class
+ // chapterEntryPattern uses for 第N章 numbering: the target corpus is largely
+ // CJK, and a full-width page-number column still has to confirm a TOC page.
+ pageNumberPattern = regexp.MustCompile(`^[.…]*[0-90-9]+[.…]*$|^[.…]*[IVXLCDM]+[.…]*$|^\s*-\s*[0-90-9]+\s*-\s*$`)
+ // tocStartPattern matches an outline title that names the table of contents,
+ // which is the only thing that starts a TOC page range. A heading that is
+ // merely adjacent to a TOC — acknowledgements, references — does not: the
+ // section-level detector this pass replaces accepted 致谢 here, so a book
+ // without a 目录 bookmark had its acknowledgements page claimed as a TOC.
+ tocStartPattern = regexp.MustCompile(`(?i)^(contents|目录|目次|table of contents)$`)
+ // tocEndMarkerPattern matches the front/back-matter headings that are
+ // neither a TOC nor an entry, so they cannot bound a TOC range. The
+ // acknowledgements family is spelled out the way chapterEntryPattern spells
+ // it (singular and plural, both spellings), because the pattern this
+ // replaces only matched the bare word "acknowledge" and so never fired on
+ // the usual "Acknowledgements" bookmark.
+ tocEndMarkerPattern = regexp.MustCompile(`(?i)^(致谢|acknowledg(?:e|ements?|ments?))$`)
+ // tocEntryRunPattern matches one TOC entry's tail: a leader run (two or more
+ // dots, ellipses or middots, never wide spaces — those are too common in
+ // prose ending in a number), a page number and an optional roman-numeral
+ // fragment. Counting these runs tells a TOC block that upstream merged into
+ // one long box apart from a paragraph. Digits accept the full-width forms
+ // pageNumberPattern accepts, for the same reason.
+ tocEntryRunPattern = regexp.MustCompile(`(\.{2,}|…{2,}|⋯{2,}|·{2,})\s*[0-90-9]{1,4}\s*[IVXLCDMivxlcdm]{0,5}`)
+)
+
+// ---------------------------------------------------------------------------
+// Outline signal.
+// ---------------------------------------------------------------------------
+
+// TOCPageRangeFromOutlines returns the 0-based page indices the PDF bookmarks
+// identify as the table of contents: from the outline entry naming the TOC up
+// to — excluding — the next entry at the same level. It returns nil when the
+// document carries no usable bookmark.
+//
+// Only an entry that names the TOC itself starts a range (tocStartPattern).
+// When a front/back-matter heading (tocEndMarkerPattern) is the next same-level
+// entry the range is abandoned rather than closed, because such a heading does
+// not say where the entries stop — it follows the TOC in some books and sits at
+// the back of others — so the extent is left to the box shape signal instead of
+// spanning the gap and deleting whatever lies inside it. Only the page the TOC
+// starts on is claimed.
+//
+// Outline page numbers are 1-based — pdfium adds one to the 0-based destination
+// index (internal/deepdoc/parser/pdf/pdfium/pdfium.go) — while
+// TextBox.PageNumber is 0-based, because ParseRaw enumerates pages from zero.
+// The conversion happens here, once, so no caller has to know about it.
+func TOCPageRangeFromOutlines(outlines []pdf.Outline) map[int]bool {
+ for i, o := range outlines {
+ if !tocStartPattern.MatchString(outlineTitle(o.Title)) {
+ continue
+ }
+ first := o.PageNumber - 1
+ if first < 0 {
+ return nil
+ }
+ for _, next := range outlines[i+1:] {
+ if next.Level != o.Level {
+ continue
+ }
+ if tocEndMarkerPattern.MatchString(outlineTitle(next.Title)) {
+ break
+ }
+ pages := make(map[int]bool, 4)
+ for pg := first; pg < next.PageNumber-1; pg++ {
+ pages[pg] = true
+ }
+ if len(pages) == 0 {
+ return nil
+ }
+ return pages
+ }
+ // No usable following entry at the same level: the TOC page is known
+ // but its extent is not, so claim only that page and leave the rest to
+ // the box shape signal.
+ return map[int]bool{first: true}
+ }
+ return nil
+}
+
+// outlineTitle strips the "@@" anchor suffix pdfium appends to bookmark titles
+// and lower-cases the result, matching how the section-level detector compared.
+func outlineTitle(title string) string {
+ title = strings.TrimSpace(title)
+ if idx := strings.Index(title, "@@"); idx >= 0 {
+ title = strings.TrimSpace(title[:idx])
+ }
+ return strings.ToLower(title)
+}
+
+// ---------------------------------------------------------------------------
+// TOC removal.
+// ---------------------------------------------------------------------------
+
+// RemoveTOCBoxes drops whole pages that are detected as tables of contents.
+//
+// Detection works on signals that are destroyed by the later TextMerge /
+// BoxesToSections passes (leader dots get folded into adjacent boxes, per-entry
+// geometry collapses into a section), so this MUST run before TextMerge (see
+// Parser.buildLayout).
+//
+// coversDocumentStart reports whether boxes cover the document's first page.
+// The shape signal reads a TOC out of a leading run, so it is skipped when the
+// parse began mid-document (Config.Pages): a TOC-shaped page at the start of a
+// page range says nothing about the document, and reading it as a document
+// prefix would delete content from the middle of the book. The outline signal
+// carries absolute page numbers and is unaffected.
+//
+// Two signals select pages and the union is dropped:
+//
+// 1. outlinePages — the pages the PDF bookmarks identify as TOC (see
+// TOCPageRangeFromOutlines). Strongest, and the only one that still sees a
+// TOC whose entries were merged into one box with no leader runs left.
+// 2. Box shape — the leading run of pages that read as a TOC list: a page
+// carrying at least tocMinShortBoxes short boxes and tocMinEntries
+// confirming markers (chapter markers or page numbers), or one long box
+// that is itself tocMinMergedRuns leader+page-number runs, preceded only by
+// pages without body text and spanning as many consecutive pages as keep
+// satisfying that shape. Kept for documents that carry no usable bookmark,
+// and only consulted when coversDocumentStart.
+//
+// Both signals pass through one guard: a page carrying body text (more than
+// tocMaxLongBoxes boxes longer than tocMaxProseRunes) is never dropped, and a
+// document consisting only of TOC pages is left untouched. The detector stays
+// deliberately conservative — missing a TOC page costs noise chunks, deleting a
+// content page loses text.
+func RemoveTOCBoxes(boxes []pdf.TextBox, outlinePages map[int]bool, coversDocumentStart bool) []pdf.TextBox {
+ if len(boxes) == 0 {
+ return boxes
+ }
+
+ perPage := make(map[int][]int, 64)
+ for i := range boxes {
+ p := boxes[i].PageNumber
+ perPage[p] = append(perPage[p], i)
+ }
+
+ pages := make([]int, 0, len(perPage))
+ for pg := range perPage {
+ pages = append(pages, pg)
+ }
+ sort.Ints(pages)
+
+ shapes := make(map[int]pageShape, len(pages))
+ for _, pg := range pages {
+ shapes[pg] = shapeOfPage(boxes, perPage[pg])
+ }
+
+ selected := make(map[int]bool, len(pages))
+ for pg := range outlinePages {
+ if _, ok := perPage[pg]; ok {
+ selected[pg] = true
+ }
+ }
+
+ // Heuristic fallback, for documents that carry no usable bookmark. A TOC is
+ // a document prefix, so only a parse that covers the document's first page
+ // is eligible, and then only the leading pages are candidates: pages without
+ // body text (a cover, copyright page) may be skipped, and the run continues
+ // while pages keep satisfying isTOC(). It ends on the first page that does
+ // not — in practice the first page carrying body text, which is what keeps
+ // per-chapter pages that happen to hold several short headings (the Daodejing
+ // case) out of scope. The run is deliberately unbounded in length: a real
+ // TOC can span more pages than any fixed cap would allow, and truncating it
+ // silently keeps the remaining TOC pages in the output. Length is bounded
+ // instead by isTOC() itself and by the all-TOC guard below.
+ if coversDocumentStart {
+ inTOC, lead := false, 0
+ for _, pg := range pages {
+ if inTOC {
+ if !shapes[pg].isTOC() {
+ break
+ }
+ selected[pg] = true
+ continue
+ }
+ if shapes[pg].isTOC() {
+ inTOC = true
+ selected[pg] = true
+ continue
+ }
+ if shapes[pg].carriesProse() || lead >= tocMaxLeadPages {
+ break
+ }
+ lead++
+ }
+ }
+
+ drop := make(map[int]struct{}, len(boxes))
+ droppedPages := 0
+ for _, pg := range pages {
+ if !selected[pg] || shapes[pg].carriesProse() {
+ continue
+ }
+ droppedPages++
+ for _, i := range perPage[pg] {
+ drop[i] = struct{}{}
+ }
+ }
+ if droppedPages == 0 || droppedPages == len(pages) {
+ return boxes
+ }
+
+ out := make([]pdf.TextBox, 0, len(boxes)-len(drop))
+ for i := range boxes {
+ if _, ok := drop[i]; ok {
+ continue
+ }
+ out = append(out, boxes[i])
+ }
+ return out
+}
+
+// pageShape summarises the boxes of one page.
+type pageShape struct {
+ prose int // boxes longer than tocMaxProseRunes
+ short int
+ markers int // short boxes opening with a chapter marker or holding a page number
+ merged bool // a long box carrying tocMinMergedRuns leader+page-number runs
+}
+
+func shapeOfPage(boxes []pdf.TextBox, indices []int) pageShape {
+ var s pageShape
+ for _, i := range indices {
+ text := strings.TrimSpace(boxes[i].Text)
+ if text == "" {
+ continue
+ }
+ if utf8.RuneCountInString(text) > tocMaxProseRunes {
+ s.prose++
+ // A merged TOC block is long by definition, so the run count is the
+ // only evidence it can carry. Cap the search: the answer is a
+ // three-way comparison, not the exact number of runs.
+ if len(tocEntryRunPattern.FindAllStringIndex(text, tocMinMergedRuns)) >= tocMinMergedRuns {
+ s.merged = true
+ }
+ continue
+ }
+ s.short++
+ if chapterEntryPattern.MatchString(text) || pageNumberPattern.MatchString(text) {
+ s.markers++
+ }
+ }
+ return s
+}
+
+// isTOC reports whether the page reads as a table of contents: a list of short
+// boxes carrying entry markers, or one long box that is itself a run of
+// entries, and no body text either way.
+func (s pageShape) isTOC() bool {
+ if s.carriesProse() {
+ return false
+ }
+ return s.merged || (s.short >= tocMinShortBoxes && s.markers >= tocMinEntries)
+}
+
+// carriesProse reports whether the page carries body text. No signal may drop
+// such a page: this is the single guard that bounds every removal.
+func (s pageShape) carriesProse() bool { return s.prose > tocMaxLongBoxes }
+
+// ---------------------------------------------------------------------------
+// Running header / footer removal.
+// ---------------------------------------------------------------------------
+
+const (
+ // headerZoneRatio / footerZoneRatio bound the top/bottom page zones where a
+ // running header / footer lives (fractions of the page height).
+ headerZoneRatio = 0.10
+ footerZoneRatio = 0.90
+ // minHeaderFooterPages guards against false positives on short documents.
+ minHeaderFooterPages = 3
+)
+
+// RemoveHeaderFooterBoxes drops boxes that are running headers or footers.
+//
+// A box is a candidate when it sits entirely in the top headerZoneRatio or
+// bottom footerZoneRatio of its page and carries text-like content. Among
+// candidates, a (zone, normalized-text) pair that recurs on at least half of
+// the document's pages — rounded up, so an odd page count raises the bar rather
+// than lowering it — is treated as a running header / footer and removed.
+//
+// Like RemoveTOCBoxes this works on intact box geometry and must run before
+// TextMerge (which can fold a header box into the first body section).
+func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) []pdf.TextBox {
+ if len(pageHeights) < minHeaderFooterPages || len(boxes) == 0 {
+ return boxes
+ }
+
+ type zoneKey struct {
+ zone string
+ text string
+ }
+ keyPages := make(map[zoneKey]map[int]struct{})
+ keyIdx := make(map[zoneKey][]int)
+
+ for i := range boxes {
+ b := boxes[i]
+ // Only plain-text boxes participate; tables / figures in the margin are
+ // not running headers/footers.
+ if lt := strings.TrimSpace(b.LayoutType); lt != "" && lt != "text" {
+ continue
+ }
+ h, ok := pageHeights[b.PageNumber]
+ if !ok || h <= 0 {
+ continue
+ }
+ top, bottom := b.Top, b.Bottom
+ var zone string
+ switch {
+ case bottom <= h*headerZoneRatio:
+ zone = "header"
+ case top >= h*footerZoneRatio:
+ zone = "footer"
+ default:
+ continue
+ }
+ norm := normalizeRunningText(b.Text)
+ if norm == "" {
+ continue
+ }
+ key := zoneKey{zone: zone, text: norm}
+ if keyPages[key] == nil {
+ keyPages[key] = make(map[int]struct{})
+ }
+ keyPages[key][b.PageNumber] = struct{}{}
+ keyIdx[key] = append(keyIdx[key], i)
+ }
+
+ numPages := len(pageHeights)
+ // Half the pages, rounded up: rounding an odd page count down would let two
+ // pages of a five-page document pass as "half", which is the one direction
+ // this guard must not drift in.
+ minPages := (numPages + 1) / 2
+ if minPages < 2 {
+ minPages = 2
+ }
+
+ drop := make(map[int]struct{})
+ for key, pages := range keyPages {
+ if len(pages) < minPages {
+ continue
+ }
+ for _, i := range keyIdx[key] {
+ drop[i] = struct{}{}
+ }
+ }
+ if len(drop) == 0 {
+ return boxes
+ }
+
+ out := make([]pdf.TextBox, 0, len(boxes)-len(drop))
+ for i := range boxes {
+ if _, ok := drop[i]; ok {
+ continue
+ }
+ out = append(out, boxes[i])
+ }
+ return out
+}
+
+// normalizeRunningText collapses whitespace, replaces digit runs with a single
+// '#' and lower-cases the result, so per-page variants of the same running
+// header/footer ("- 2 -", "- 3 -", "Page 4 of 9", "第 1 页") share one
+// comparison key.
+func normalizeRunningText(text string) string {
+ t := strings.Join(strings.Fields(text), " ")
+ out := make([]rune, 0, len(t))
+ inDigit := false
+ for _, r := range t {
+ if isMaskableDigit(r) {
+ inDigit = true
+ continue
+ }
+ if inDigit {
+ out = append(out, '#')
+ inDigit = false
+ }
+ out = append(out, r)
+ }
+ if inDigit {
+ out = append(out, '#')
+ }
+ return strings.ToLower(strings.TrimSpace(string(out)))
+}
+
+// isMaskableDigit reports whether r is a digit whose per-page value must be
+// masked before running headers/footers are compared: ASCII and full-width
+// decimal digits. Full width is required for the same reason pageNumberPattern
+// accepts it — otherwise every page of "第 1 页" / "第 2 页" gets its own key
+// and the footer never reaches minPages.
+func isMaskableDigit(r rune) bool {
+ return (r >= '0' && r <= '9') || (r >= '0' && r <= '9')
+}
diff --git a/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go b/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go
new file mode 100644
index 0000000000..ed48433155
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go
@@ -0,0 +1,759 @@
+package layout
+
+import (
+ "strings"
+ "testing"
+
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+)
+
+func tb(text string, page int, x0, x1, top, bottom float64) pdf.TextBox {
+ return pdf.TextBox{
+ Text: text,
+ PageNumber: page,
+ X0: x0,
+ X1: x1,
+ Top: top,
+ Bottom: bottom,
+ }
+}
+
+func texts(boxes []pdf.TextBox) []string {
+ out := make([]string, 0, len(boxes))
+ for _, b := range boxes {
+ out = append(out, b.Text)
+ }
+ return out
+}
+
+func equalText(a, b []string) bool {
+ if len(a) != len(b) {
+ return false
+ }
+ for i := range a {
+ if a[i] != b[i] {
+ return false
+ }
+ }
+ return true
+}
+
+// TestRemoveTOCBoxes_DropsDottedTOCPage: a page made of leader-dot entries is
+// removed entirely, while a body page on a different page is untouched.
+func TestRemoveTOCBoxes_DropsDottedTOCPage(t *testing.T) {
+ boxes := []pdf.TextBox{
+ // page 0: TOC with leader dots in their own boxes.
+ tb("Contents", 0, 72, 150, 100, 120),
+ tb("Chapter One", 0, 72, 175, 160, 175),
+ tb("..........................", 0, 175, 400, 160, 175),
+ tb("3", 0, 500, 520, 160, 175),
+ tb("Chapter Two", 0, 72, 175, 190, 205),
+ tb("..........................", 0, 175, 400, 190, 205),
+ tb("4", 0, 500, 520, 190, 205),
+ tb("Chapter Three", 0, 72, 180, 220, 235),
+ tb("..........................", 0, 175, 400, 220, 235),
+ tb("5", 0, 500, 520, 220, 235),
+ // page 1: normal body with multiple prose paragraphs.
+ tb("The way that can be told of is not the eternal way; the name that can be named is not the eternal name. The nameless is the origin of heaven and earth.", 1, 72, 500, 160, 180),
+ tb("The named is the mother of ten thousand things. Ever desireless, one can see the mystery. Ever desiring, one can see the manifestations.", 1, 72, 500, 190, 210),
+ tb("These two spring from the same source but differ in name. This appears as darkness. Darkness within darkness. The gate to all mystery.", 1, 72, 500, 220, 240),
+ }
+ got := RemoveTOCBoxes(boxes, nil, true)
+ // Only the body paragraphs on page 1 should survive.
+ want := []string{
+ "The way that can be told of is not the eternal way; the name that can be named is not the eternal name. The nameless is the origin of heaven and earth.",
+ "The named is the mother of ten thousand things. Ever desireless, one can see the mystery. Ever desiring, one can see the manifestations.",
+ "These two spring from the same source but differ in name. This appears as darkness. Darkness within darkness. The gate to all mystery.",
+ }
+ if !equalText(texts(got), want) {
+ t.Fatalf("got %v, want %v", texts(got), want)
+ }
+}
+
+// TestRemoveTOCBoxes_KeepsBodyPage: a page without leader dots is never removed.
+func TestRemoveTOCBoxes_KeepsBodyPage(t *testing.T) {
+ boxes := []pdf.TextBox{
+ tb("Section 1.1 Overview", 0, 72, 200, 100, 120),
+ tb("This paragraph explains the background of the work in some detail so the reader can follow.", 0, 72, 500, 130, 150),
+ tb("It continues with more context and supporting material for the main argument.", 0, 72, 500, 160, 180),
+ }
+ got := RemoveTOCBoxes(boxes, nil, true)
+ if len(got) != len(boxes) {
+ t.Fatalf("body page should be untouched, got %d boxes, want %d", len(got), len(boxes))
+ }
+}
+
+// TestRemoveTOCBoxes_ThresholdBoundary: 2 entries (below min) keeps the page,
+// 3 entries removes it.
+func TestRemoveTOCBoxes_ThresholdBoundary(t *testing.T) {
+ mkPage := func(n int, page int) []pdf.TextBox {
+ var boxes []pdf.TextBox
+ boxes = append(boxes, tb("Contents", page, 72, 150, 100, 120))
+ for i := 0; i < n; i++ {
+ y := float64(160 + i*30)
+ boxes = append(boxes,
+ tb("Item", page, 72, 120, y, y+15),
+ tb("..........................", page, 175, 400, y, y+15),
+ tb(string(rune('0'+i+1)), page, 500, 520, y, y+15),
+ )
+ }
+ return boxes
+ }
+ below := append(mkPage(2, 0),
+ tb("Body text here that is long enough to matter and exceeds sixty runes for sure absolutely yes it does.", 1, 72, 400, 160, 180),
+ tb("Another paragraph of body text that is also quite long and exceeds the thirty rune threshold easily.", 1, 72, 400, 190, 210),
+ tb("A third paragraph of body text that ensures the book is not classified as compact by the detector.", 1, 72, 400, 220, 240),
+ )
+ if got := RemoveTOCBoxes(below, nil, true); len(got) != len(below) {
+ t.Fatalf("2 entries should keep page: got %d boxes, want %d", len(got), len(below))
+ }
+ above := append(mkPage(3, 0),
+ tb("Body text here that is long enough to matter and exceeds sixty runes for sure absolutely yes it does.", 1, 72, 400, 160, 180),
+ tb("Another paragraph of body text that is also quite long and exceeds the thirty rune threshold easily.", 1, 72, 400, 190, 210),
+ tb("A third paragraph of body text that ensures the book is not classified as compact by the detector.", 1, 72, 400, 220, 240),
+ )
+ if got := RemoveTOCBoxes(above, nil, true); len(got) != 3 {
+ t.Fatalf("3 entries should drop the whole TOC page: got %d boxes, want 3", len(got))
+ }
+}
+
+// TestRemoveTOCBoxes_LeaderMergedIntoBox: when the leader dots are merged into
+// the same box as the title+page-number, the entry is still detected.
+func TestRemoveTOCBoxes_LeaderMergedIntoBox(t *testing.T) {
+ boxes := []pdf.TextBox{
+ tb("Contents", 0, 72, 150, 100, 120),
+ tb("Chapter One .......................... 3", 0, 72, 520, 160, 175),
+ tb("Chapter Two .......................... 4", 0, 72, 520, 190, 205),
+ tb("Chapter Three .......................... 5", 0, 72, 520, 220, 235),
+ tb("Body paragraph that must survive on page 1 and is definitely longer than thirty runes.", 1, 72, 400, 160, 180),
+ tb("Another body paragraph that ensures the book has enough long boxes to not be compact.", 1, 72, 400, 190, 210),
+ }
+ got := RemoveTOCBoxes(boxes, nil, true)
+ want := []string{
+ "Body paragraph that must survive on page 1 and is definitely longer than thirty runes.",
+ "Another body paragraph that ensures the book has enough long boxes to not be compact.",
+ }
+ if !equalText(texts(got), want) {
+ t.Fatalf("got %v, want %v", texts(got), want)
+ }
+}
+
+// TestRemoveTOCBoxes_ChineseChapterMarkers: Chinese "第N章" style TOC.
+func TestRemoveTOCBoxes_ChineseChapterMarkers(t *testing.T) {
+ boxes := []pdf.TextBox{
+ tb("目录", 0, 72, 110, 100, 120),
+ tb("第一章 道可道", 0, 72, 200, 160, 175),
+ tb("..........................", 0, 200, 420, 160, 175),
+ tb("1", 0, 500, 520, 160, 175),
+ tb("第二章 天下皆知", 0, 72, 210, 190, 205),
+ tb("..........................", 0, 200, 420, 190, 205),
+ tb("5", 0, 500, 520, 190, 205),
+ tb("第三章 不尚贤", 0, 72, 200, 220, 235),
+ tb("..........................", 0, 200, 420, 220, 235),
+ tb("9", 0, 500, 520, 220, 235),
+ tb("正文内容,道可道,非常道。名可名,非常名。无名天地之始,有名万物之母。故常无欲以观其妙,常有欲以观其徼。此两者同出而异名,同谓之玄。玄之又玄,众妙之门。", 1, 72, 500, 160, 180),
+ tb("天下皆知美之为美,斯恶已。皆知善之为善,斯不善已。故有无相生,难易相成,长短相形,高下相倾,音声相和,前后相随。", 1, 72, 500, 190, 210),
+ tb("是以圣人处无为之事,行不言之教。万物作焉而不辞,生而不有,为而不恃,功成而弗居。夫唯弗居,是以不去。", 1, 72, 500, 220, 240),
+ }
+ got := RemoveTOCBoxes(boxes, nil, true)
+ want := []string{
+ "正文内容,道可道,非常道。名可名,非常名。无名天地之始,有名万物之母。故常无欲以观其妙,常有欲以观其徼。此两者同出而异名,同谓之玄。玄之又玄,众妙之门。",
+ "天下皆知美之为美,斯恶已。皆知善之为善,斯不善已。故有无相生,难易相成,长短相形,高下相倾,音声相和,前后相随。",
+ "是以圣人处无为之事,行不言之教。万物作焉而不辞,生而不有,为而不恃,功成而弗居。夫唯弗居,是以不去。",
+ }
+ if !equalText(texts(got), want) {
+ t.Fatalf("got %v, want %v", texts(got), want)
+ }
+}
+
+// TestRemoveTOCBoxes_BodyPageWithLongBoxIsNotTOC: a page with long prose boxes
+// is never mistaken for a TOC even if it happens to contain a short dot line.
+func TestRemoveTOCBoxes_BodyPageWithLongBoxIsNotTOC(t *testing.T) {
+ boxes := []pdf.TextBox{
+ tb("This is a long body paragraph that clearly belongs to the main content of the document and should never be classified as a table of contents entry under any reasonable heuristic.", 0, 72, 500, 100, 120),
+ tb("Another sentence here to make the point clear and ensure the page is recognized as a body page.", 0, 72, 500, 130, 150),
+ }
+ got := RemoveTOCBoxes(boxes, nil, true)
+ if len(got) != len(boxes) {
+ t.Fatalf("body page with long boxes must be kept, got %d, want %d", len(got), len(boxes))
+ }
+}
+
+// tocPageBoxes builds one page of dotted TOC entries: a heading plus n entries,
+// each entry as its own box, which is the shape the box signal expects.
+func tocPageBoxes(pg, n int) []pdf.TextBox {
+ out := []pdf.TextBox{tb("目录", pg, 72, 110, 100, 120)}
+ for i := 0; i < n; i++ {
+ y := float64(160 + i*30)
+ out = append(out,
+ tb("第一章 道可道", pg, 72, 200, y, y+15),
+ tb("..........................", pg, 200, 420, y, y+15),
+ tb(string(rune('0'+i+1)), pg, 500, 520, y, y+15),
+ )
+ }
+ return out
+}
+
+// bodyPageBoxes builds one page of prose, which no signal may drop.
+func bodyPageBoxes(pg int) []pdf.TextBox {
+ return []pdf.TextBox{
+ tb("The way that can be told of is not the eternal way and the name that can be named is not the eternal name.", pg, 72, 500, 160, 180),
+ tb("The named is the mother of ten thousand things and ever desireless one can see the mystery of it all here.", pg, 72, 500, 190, 210),
+ tb("These two spring from the same source but differ in name and this appears as darkness within darkness.", pg, 72, 500, 220, 240),
+ }
+}
+
+func countPage(boxes []pdf.TextBox, pg int) int {
+ n := 0
+ for _, b := range boxes {
+ if b.PageNumber == pg {
+ n++
+ }
+ }
+ return n
+}
+
+// TestTOCPageRangeFromOutlines pins the 1-based to 0-based conversion: pdfium
+// reports outline pages 1-based while TextBox.PageNumber is 0-based, and the
+// section-level detector this replaces compared the two directly.
+func TestTOCPageRangeFromOutlines(t *testing.T) {
+ toc := TOCPageRangeFromOutlines([]pdf.Outline{
+ {Title: "目录", Level: 0, PageNumber: 1},
+ {Title: "第一章", Level: 0, PageNumber: 4},
+ })
+ if len(toc) != 3 || !toc[0] || !toc[1] || !toc[2] || toc[3] {
+ t.Fatalf("outline pages 1-3 must map to 0-based {0,1,2} and stop before the content page, got %v", toc)
+ }
+ if got := TOCPageRangeFromOutlines(nil); got != nil {
+ t.Fatalf("no outlines must yield nil, got %v", got)
+ }
+ if got := TOCPageRangeFromOutlines([]pdf.Outline{{Title: "第一章", Level: 0, PageNumber: 2}}); got != nil {
+ t.Fatalf("a first bookmark that is a chapter names no TOC page, got %v", got)
+ }
+ // No following entry at the same level: only the TOC page itself is known.
+ if got := TOCPageRangeFromOutlines([]pdf.Outline{{Title: "Contents", Level: 0, PageNumber: 2}}); len(got) != 1 || !got[1] {
+ t.Fatalf("a lone TOC bookmark should claim printed page 2 only, got %v", got)
+ }
+ // The "@@" anchor suffix pdfium appends must not defeat the title match.
+ if got := TOCPageRangeFromOutlines([]pdf.Outline{
+ {Title: "Contents@@5", Level: 0, PageNumber: 1},
+ {Title: "Chapter One", Level: 0, PageNumber: 2},
+ }); len(got) != 1 || !got[0] {
+ t.Fatalf("an anchored TOC title should claim 0-based page 0, got %v", got)
+ }
+}
+
+// TestTOCPageRangeFromOutlines_AcknowledgementsDoesNotStart: only an entry that
+// names the TOC starts a range. A book with an acknowledgements bookmark and no
+// contents bookmark used to have its acknowledgements page claimed and dropped.
+func TestTOCPageRangeFromOutlines_AcknowledgementsDoesNotStart(t *testing.T) {
+ cases := []struct {
+ name string
+ in []pdf.Outline
+ }{
+ {"chinese, followed by back matter", []pdf.Outline{
+ {Title: "致谢", Level: 0, PageNumber: 5},
+ {Title: "参考文献", Level: 0, PageNumber: 8},
+ }},
+ {"chinese, alone", []pdf.Outline{
+ {Title: "致谢", Level: 0, PageNumber: 5},
+ }},
+ {"english plural", []pdf.Outline{
+ {Title: "Acknowledgements", Level: 0, PageNumber: 5},
+ {Title: "References", Level: 0, PageNumber: 8},
+ }},
+ }
+ for _, c := range cases {
+ if got := TOCPageRangeFromOutlines(c.in); got != nil {
+ t.Errorf("%s: an acknowledgements bookmark must not open a TOC range, got %v", c.name, got)
+ }
+ }
+}
+
+// TestTOCPageRangeFromOutlines_EndMarkerAbandonsRange: a front/back-matter
+// heading sitting next to the TOC does not say where the entries stop, so the
+// range is abandoned rather than extended to whatever entry follows it.
+func TestTOCPageRangeFromOutlines_EndMarkerAbandonsRange(t *testing.T) {
+ // Before: 致谢 was skipped and the range ran to 参考文献, claiming printed
+ // pages 2-11 including the acknowledgements themselves.
+ if got := TOCPageRangeFromOutlines([]pdf.Outline{
+ {Title: "目录", Level: 0, PageNumber: 2},
+ {Title: "致谢", Level: 0, PageNumber: 10},
+ {Title: "参考文献", Level: 0, PageNumber: 12},
+ }); len(got) != 1 || !got[1] {
+ t.Fatalf("the acknowledgements entry must abandon the range, claiming printed page 2 only, got %v", got)
+ }
+ // The English heading now matches at all; before, it was treated as an
+ // ordinary boundary and claimed printed pages 2-9.
+ if got := TOCPageRangeFromOutlines([]pdf.Outline{
+ {Title: "Contents", Level: 0, PageNumber: 2},
+ {Title: "Acknowledgements", Level: 0, PageNumber: 10},
+ }); len(got) != 1 || !got[1] {
+ t.Fatalf("the english end marker must abandon the range too, got %v", got)
+ }
+ // A chapter entry is still a boundary: the range stops just before it.
+ if got := TOCPageRangeFromOutlines([]pdf.Outline{
+ {Title: "目录", Level: 0, PageNumber: 2},
+ {Title: "第一章", Level: 0, PageNumber: 5},
+ {Title: "致谢", Level: 0, PageNumber: 20},
+ }); len(got) != 3 || !got[1] || !got[2] || !got[3] || got[4] {
+ t.Fatalf("a chapter entry must still close the range at printed page 5, got %v", got)
+ }
+}
+
+// TestRemoveTOCBoxes_OutlineCoversLeaderlessMergedBlock: a TOC whose entries
+// were merged into one box with no leader runs left is invisible to the box
+// signal — neither short boxes nor leader runs survive — and is covered by the
+// outline signal alone.
+func TestRemoveTOCBoxes_OutlineCoversLeaderlessMergedBlock(t *testing.T) {
+ merged := "目录 第一章 道可道非常道名可名非常名 3 第二章 天下皆知美之为美斯恶已皆知善 7 第三章 不尚贤使民不争不贵难得之货 15 第四章 道冲而用之或不盈渊兮似万物之宗 21"
+ boxes := []pdf.TextBox{
+ tb("目录", 0, 72, 110, 100, 120),
+ tb(merged, 0, 72, 520, 160, 200),
+ }
+ boxes = append(boxes, bodyPageBoxes(1)...)
+ boxes = append(boxes, bodyPageBoxes(2)...)
+
+ if got := RemoveTOCBoxes(boxes, nil, true); len(got) != len(boxes) {
+ t.Fatalf("box shape alone cannot see a leaderless merged TOC block, got %d of %d", len(got), len(boxes))
+ }
+ got := RemoveTOCBoxes(boxes, map[int]bool{0: true}, true)
+ if n := countPage(got, 0); n != 0 {
+ t.Fatalf("an outline-selected page must be dropped, %d boxes survived", n)
+ }
+}
+
+// TestRemoveTOCBoxes_MergedBlockWithLeaders: DeepDoc joins the lines of a text
+// block into one box, so a TOC page can arrive as a single long box. Its leader
+// runs are the only evidence left, and the page must still be dropped without
+// any bookmark.
+func TestRemoveTOCBoxes_MergedBlockWithLeaders(t *testing.T) {
+ merged := "目录 第一章 道可道非常道名可名非常名 ..........3 第二章 天下皆知美之为美斯恶已皆知善 ..........7 第三章 不尚贤使民不争不贵难得之货 ..........15"
+ boxes := []pdf.TextBox{
+ tb("目录", 0, 72, 110, 100, 120),
+ tb(merged, 0, 72, 520, 160, 200),
+ }
+ boxes = append(boxes, bodyPageBoxes(1)...)
+ boxes = append(boxes, bodyPageBoxes(2)...)
+
+ got := RemoveTOCBoxes(boxes, nil, true)
+ if n := countPage(got, 0); n != 0 {
+ t.Fatalf("a merged TOC block carrying leader runs must be dropped, %d boxes survived", n)
+ }
+ if n := countPage(got, 1); n != 3 {
+ t.Fatalf("the body page must be kept, %d of 3 boxes", n)
+ }
+}
+
+// TestRemoveTOCBoxes_MergedBlockNeedsThreeRuns: two leader runs in a long box
+// are not enough. A paragraph can pick up the odd "……12", so the threshold sits
+// above that rather than at the first run.
+func TestRemoveTOCBoxes_MergedBlockNeedsThreeRuns(t *testing.T) {
+ merged := "目录 第一章 道可道非常道名可名非常名 ..........3 第二章 天下皆知美之为美斯恶已皆知善 ..........7 第三章 不尚贤使民不争不贵难得之货"
+ boxes := []pdf.TextBox{
+ tb("目录", 0, 72, 110, 100, 120),
+ tb(merged, 0, 72, 520, 160, 200),
+ }
+ boxes = append(boxes, bodyPageBoxes(1)...)
+ boxes = append(boxes, bodyPageBoxes(2)...)
+
+ if got := RemoveTOCBoxes(boxes, nil, true); len(got) != len(boxes) {
+ t.Fatalf("two leader runs are below the merged-block threshold, got %d of %d", len(got), len(boxes))
+ }
+}
+
+// TestRemoveTOCBoxes_MergedBlockFullWidthPageNumbers: the leader-run count uses
+// the same digit class as pageNumberPattern, so a CJK TOC numbering its entries
+// in full-width digits is still recognised.
+func TestRemoveTOCBoxes_MergedBlockFullWidthPageNumbers(t *testing.T) {
+ merged := "目录 第一章 道可道非常道名可名非常名 …………3 第二章 天下皆知美之为美斯恶已皆知善 …………7 第三章 不尚贤使民不争不贵难得之货 …………15"
+ boxes := []pdf.TextBox{
+ tb("目录", 0, 72, 110, 100, 120),
+ tb(merged, 0, 72, 520, 160, 200),
+ }
+ boxes = append(boxes, bodyPageBoxes(1)...)
+ boxes = append(boxes, bodyPageBoxes(2)...)
+
+ got := RemoveTOCBoxes(boxes, nil, true)
+ if n := countPage(got, 0); n != 0 {
+ t.Fatalf("full-width page numbers in a merged TOC block must be counted, %d boxes survived", n)
+ }
+}
+
+// TestRemoveTOCBoxes_MergedBlockRespectsProseGuard: a page that carries body
+// text is kept even when one of its boxes is a merged TOC block. The merged
+// signal is the one that could plausibly be talked into deleting a prose page,
+// so the guard is asserted against it rather than against the short-box shape.
+func TestRemoveTOCBoxes_MergedBlockRespectsProseGuard(t *testing.T) {
+ merged := "目录 第一章 道可道 ..........3 第二章 天下皆知美之为美 ..........7 第三章 不尚贤使民不争 ..........15"
+ boxes := []pdf.TextBox{
+ tb(merged, 0, 72, 520, 100, 140),
+ tb("This is a long body paragraph that certainly belongs to the main content of the document and would be lost.", 0, 72, 520, 160, 180),
+ tb("This is a second long body paragraph that certainly belongs to the main content of the document as well.", 0, 72, 520, 190, 210),
+ tb("This is a third long body paragraph that certainly belongs to the main content of the document too.", 0, 72, 520, 220, 240),
+ }
+ boxes = append(boxes, bodyPageBoxes(1)...)
+ boxes = append(boxes, bodyPageBoxes(2)...)
+
+ got := RemoveTOCBoxes(boxes, nil, true)
+ if n := countPage(got, 0); n != 4 {
+ t.Fatalf("a page carrying prose must never be dropped, %d of 4 boxes survived", n)
+ }
+}
+
+// TestRemoveTOCBoxes_OutlineRespectsProseGuard: a stale or malformed bookmark
+// that names a content page must not delete body text, while the box signal
+// still drops the real TOC page.
+func TestRemoveTOCBoxes_OutlineRespectsProseGuard(t *testing.T) {
+ boxes := tocPageBoxes(0, 3)
+ boxes = append(boxes, bodyPageBoxes(1)...)
+ boxes = append(boxes, bodyPageBoxes(2)...)
+
+ got := RemoveTOCBoxes(boxes, map[int]bool{1: true}, true)
+ if n := countPage(got, 1); n != 3 {
+ t.Fatalf("a page carrying prose must never be dropped, %d of 3 boxes survived", n)
+ }
+ if n := countPage(got, 0); n != 0 {
+ t.Fatalf("the box signal must still drop the TOC page, %d boxes survived", n)
+ }
+}
+
+// TestRemoveTOCBoxes_KeepsTOCOnlyDocument: a source file that is nothing but a
+// TOC page keeps it rather than being emptied.
+func TestRemoveTOCBoxes_KeepsTOCOnlyDocument(t *testing.T) {
+ boxes := tocPageBoxes(0, 3)
+ if got := RemoveTOCBoxes(boxes, nil, true); len(got) != len(boxes) {
+ t.Fatalf("a document consisting only of a TOC must be left untouched, got %d of %d", len(got), len(boxes))
+ }
+}
+
+// TestRemoveTOCBoxes_MidDocumentParseSkipsShapeSignal: when the parse covers a
+// later page range, the first parsed page is not a document prefix. Reading it
+// as one deleted content from the middle of the book.
+func TestRemoveTOCBoxes_MidDocumentParseSkipsShapeSignal(t *testing.T) {
+ var boxes []pdf.TextBox
+ boxes = append(boxes, tocPageBoxes(10, 3)...)
+ boxes = append(boxes, bodyPageBoxes(11)...)
+
+ if got := RemoveTOCBoxes(boxes, nil, false); len(got) != len(boxes) {
+ t.Fatalf("a page range starting mid-document must not be read as a TOC prefix, dropped %d of %d",
+ len(boxes)-len(got), len(boxes))
+ }
+ if got := RemoveTOCBoxes(boxes, nil, true); len(got) == len(boxes) {
+ t.Fatalf("the same boxes as a document prefix must still drop the TOC page")
+ }
+ // The outline signal carries absolute page numbers, so the gate must not
+ // silence it.
+ got := RemoveTOCBoxes(boxes, map[int]bool{10: true}, false)
+ if n := countPage(got, 10); n != 0 {
+ t.Fatalf("an outline-selected page must be dropped even mid-document, %d boxes survived", n)
+ }
+ if n := countPage(got, 11); n != 3 {
+ t.Fatalf("the body page must survive, %d of 3 boxes", n)
+ }
+}
+
+// TestRemoveTOCBoxes_MultiPageTOC: one TOC spanning consecutive pages is dropped
+// as a whole, not just its first page.
+func TestRemoveTOCBoxes_MultiPageTOC(t *testing.T) {
+ boxes := tocPageBoxes(0, 3)
+ boxes = append(boxes, tocPageBoxes(1, 3)...)
+ boxes = append(boxes, bodyPageBoxes(2)...)
+
+ got := RemoveTOCBoxes(boxes, nil, true)
+ if n := countPage(got, 0); n != 0 {
+ t.Fatalf("TOC page 0 must be dropped, %d boxes survived", n)
+ }
+ if n := countPage(got, 1); n != 0 {
+ t.Fatalf("TOC page 1 must be dropped, %d boxes survived", n)
+ }
+ if n := countPage(got, 2); n != 3 {
+ t.Fatalf("body page 2 must survive, %d of 3 boxes", n)
+ }
+}
+
+// TestRemoveTOCBoxes_LongTOCRun: the run length follows the page shapes, not a
+// fixed cap. A real TOC can be longer than any constant small enough to be a
+// useful bound, and truncating the run leaks the remaining TOC pages into the
+// output — the tail page of a six-page TOC was kept while the cap was five.
+func TestRemoveTOCBoxes_LongTOCRun(t *testing.T) {
+ const tocPages = 6
+ var boxes []pdf.TextBox
+ for pg := 0; pg < tocPages; pg++ {
+ boxes = append(boxes, tocPageBoxes(pg, 3)...)
+ }
+ boxes = append(boxes, bodyPageBoxes(tocPages)...)
+
+ got := RemoveTOCBoxes(boxes, nil, true)
+ for pg := 0; pg < tocPages; pg++ {
+ if n := countPage(got, pg); n != 0 {
+ t.Fatalf("TOC page %d must be dropped, %d boxes survived", pg, n)
+ }
+ }
+ if n := countPage(got, tocPages); n != 3 {
+ t.Fatalf("the body page after the TOC must survive, %d of 3 boxes", n)
+ }
+}
+
+// TestRemoveTOCBoxes_TOCBehindCoverPage: a cover page carrying several short
+// boxes must not close the candidate window before the TOC is reached.
+func TestRemoveTOCBoxes_TOCBehindCoverPage(t *testing.T) {
+ cover := []pdf.TextBox{
+ tb("The Analects", 0, 150, 400, 200, 230),
+ tb("Translated by Someone", 0, 150, 400, 260, 280),
+ tb("Second Edition", 0, 150, 400, 300, 320),
+ tb("Some Press, 2024", 0, 150, 400, 340, 360),
+ }
+ boxes := append(cover, tocPageBoxes(1, 3)...)
+ boxes = append(boxes, bodyPageBoxes(2)...)
+
+ got := RemoveTOCBoxes(boxes, nil, true)
+ if n := countPage(got, 0); n != 4 {
+ t.Fatalf("the cover page must be kept, %d of 4 boxes", n)
+ }
+ if n := countPage(got, 1); n != 0 {
+ t.Fatalf("the TOC behind the cover must be dropped, %d boxes survived", n)
+ }
+ if n := countPage(got, 2); n != 3 {
+ t.Fatalf("body page 2 must survive, %d of 3 boxes", n)
+ }
+}
+
+// TestRemoveTOCBoxes_StopsAtProsePage: widening the candidate window must not
+// reach past a page of body text. A TOC-shaped page behind one is kept.
+func TestRemoveTOCBoxes_StopsAtProsePage(t *testing.T) {
+ boxes := []pdf.TextBox{tb("The Analects", 0, 150, 400, 200, 230)}
+ boxes = append(boxes, bodyPageBoxes(1)...)
+ boxes = append(boxes, tocPageBoxes(2, 3)...)
+ boxes = append(boxes, bodyPageBoxes(3)...)
+
+ got := RemoveTOCBoxes(boxes, nil, true)
+ if n := countPage(got, 2); n != 10 {
+ t.Fatalf("a TOC-shaped page behind a body page must be kept, %d of 10 boxes", n)
+ }
+}
+
+// TestRemoveTOCBoxes_BareCJKChapterTitles pins the shape reported in #19744: a
+// CJK TOC whose entries are bare chapter titles, with no leader dots and no page
+// numbers. The section-level prefix scan under-deleted these entries; the box
+// signal drops the page as a whole.
+func TestRemoveTOCBoxes_BareCJKChapterTitles(t *testing.T) {
+ boxes := []pdf.TextBox{
+ tb("目录", 0, 72, 110, 100, 120),
+ tb("第一章 绪论", 0, 72, 200, 160, 175),
+ tb("第二章 背景", 0, 72, 200, 190, 205),
+ tb("第三章 方法", 0, 72, 200, 220, 235),
+ tb("第四章 结论", 0, 72, 200, 250, 265),
+ }
+ boxes = append(boxes, tb("第一章 绪论", 1, 72, 200, 160, 175))
+ boxes = append(boxes, bodyPageBoxes(1)...)
+
+ got := RemoveTOCBoxes(boxes, nil, true)
+ if n := countPage(got, 0); n != 0 {
+ t.Fatalf("the bare-title TOC page must be dropped, %d boxes survived", n)
+ }
+ if n := countPage(got, 1); n != 4 {
+ t.Fatalf("the body page must be kept, %d of 4 boxes", n)
+ }
+}
+
+func TestRemoveTOCText_Normalize(t *testing.T) {
+ cases := []struct{ in, want string }{
+ {"Hello World", "hello world"},
+ {"- 12 -", "- # -"},
+ {"Chapter 1: Intro", "chapter #: intro"},
+ {"第 1 页", "第 # 页"},
+ }
+ for _, c := range cases {
+ if got := normalizeRunningText(c.in); got != c.want {
+ t.Fatalf("normalizeRunningText(%q) = %q, want %q", c.in, got, c.want)
+ }
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_DropsRepeatedHeader: identical header box on every
+// page is removed; body is kept.
+func TestRemoveHeaderFooterBoxes_DropsRepeatedHeader(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}
+ boxes := []pdf.TextBox{
+ // identical header in top 10% on every page
+ tb("THE WAY OF GO", 0, 72, 200, 30, 45),
+ tb("Body text on page zero that is long enough.", 0, 72, 400, 160, 180),
+ tb("THE WAY OF GO", 1, 72, 200, 30, 45),
+ tb("Body text on page one that is long enough.", 1, 72, 400, 160, 180),
+ tb("THE WAY OF GO", 2, 72, 200, 30, 45),
+ tb("Body text on page two that is long enough.", 2, 72, 400, 160, 180),
+ }
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ var headers, bodies int
+ for _, b := range got {
+ if strings.Contains(b.Text, "THE WAY") {
+ headers++
+ } else {
+ bodies++
+ }
+ }
+ if headers != 0 {
+ t.Fatalf("expected all repeated headers removed, got %d", headers)
+ }
+ if bodies != 3 {
+ t.Fatalf("expected 3 body boxes kept, got %d", bodies)
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_KeepsUniqueZoneText: per-page unique headers are
+// kept (the repetition guard must not delete genuine content).
+func TestRemoveHeaderFooterBoxes_KeepsUniqueZoneText(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}
+ boxes := []pdf.TextBox{
+ tb("Chapter One: Getting Started", 0, 72, 250, 30, 45),
+ tb("Body zero.", 0, 72, 400, 160, 180),
+ tb("Chapter Two: Types and Values", 1, 72, 250, 30, 45),
+ tb("Body one.", 1, 72, 400, 160, 180),
+ tb("Chapter Three: Concurrency", 2, 72, 260, 30, 45),
+ tb("Body two.", 2, 72, 400, 160, 180),
+ }
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ if len(got) != len(boxes) {
+ t.Fatalf("unique per-page headers must be kept, got %d, want %d", len(got), len(boxes))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_PageCountGuard: documents with < 3 pages are not
+// touched.
+func TestRemoveHeaderFooterBoxes_PageCountGuard(t *testing.T) {
+ heights := map[int]float64{0: 842, 1: 842}
+ boxes := []pdf.TextBox{
+ tb("Repeated", 0, 72, 150, 30, 45),
+ tb("Body zero.", 0, 72, 400, 160, 180),
+ tb("Repeated", 1, 72, 150, 30, 45),
+ tb("Body one.", 1, 72, 400, 160, 180),
+ }
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ if len(got) != len(boxes) {
+ t.Fatalf("short documents must be untouched, got %d, want %d", len(got), len(boxes))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_SkipsNonTextLayout: a table/figure box in the
+// header zone must not be treated as a running header.
+func TestRemoveHeaderFooterBoxes_SkipsNonTextLayout(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}
+ var boxes []pdf.TextBox
+ for pg := 0; pg < 3; pg++ {
+ boxes = append(boxes,
+ // Same text, same zone, but a table: never a running header.
+ pdf.TextBox{Text: "Running head", PageNumber: pg, X0: 72, X1: 200, Top: 30, Bottom: 45, LayoutType: "table"},
+ // Positive control in the same zone: this one must go.
+ tb("Running head", pg, 280, 400, 30, 45),
+ tb("Body text.", pg, 72, 400, 160, 180),
+ )
+ }
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+
+ var tableKept, textKept int
+ for _, b := range got {
+ switch {
+ case b.LayoutType == "table":
+ tableKept++
+ case b.Text == "Running head":
+ textKept++
+ }
+ }
+ if textKept != 0 {
+ t.Fatalf("the repeated text header must be removed, %d survived", textKept)
+ }
+ if tableKept != 3 {
+ t.Fatalf("table boxes in the header zone must be kept, %d of 3 survived", tableKept)
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_FullWidthPageNumbers: full-width page numbers get
+// a distinct key per page unless they are masked like ASCII digits, so the
+// footer never reaches minPages and survives every page.
+func TestRemoveHeaderFooterBoxes_FullWidthPageNumbers(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}
+ var boxes []pdf.TextBox
+ for pg := 0; pg < 3; pg++ {
+ boxes = append(boxes,
+ tb("正文内容。", pg, 72, 400, 160, 180),
+ tb("第 "+string(rune('0'+pg+1))+" 页", pg, 280, 340, 820, 835),
+ )
+ }
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ for _, b := range got {
+ if b.Top >= 800 {
+ t.Fatalf("full-width page-number footer %q must be removed", b.Text)
+ }
+ }
+ if len(got) != 3 {
+ t.Fatalf("expected only the 3 body boxes, got %d", len(got))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_HalfOfOddPageCount: "at least half the pages" has
+// to round up on an odd page count. Rounding down would let 2 pages of a
+// five-page document pass as half, and this repetition rule is the only guard
+// keeping genuine content out of the drop set.
+func TestRemoveHeaderFooterBoxes_HalfOfOddPageCount(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight, 4: pageHeight}
+
+ // 2 of 5 pages is below half: too rare to be a running header.
+ below := []pdf.TextBox{
+ tb("THE WAY OF GO", 0, 72, 200, 30, 45),
+ tb("Body zero.", 0, 72, 400, 160, 180),
+ tb("Body one.", 1, 72, 400, 160, 180),
+ tb("THE WAY OF GO", 2, 72, 200, 30, 45),
+ tb("Body two.", 2, 72, 400, 160, 180),
+ tb("Body three.", 3, 72, 400, 160, 180),
+ tb("Body four.", 4, 72, 400, 160, 180),
+ }
+ if got := RemoveHeaderFooterBoxes(below, heights); len(got) != len(below) {
+ t.Fatalf("2 of 5 pages is not half, the boxes must be kept: got %d, want %d", len(got), len(below))
+ }
+
+ // 3 of 5 pages is half: a running header.
+ above := []pdf.TextBox{
+ tb("THE WAY OF GO", 0, 72, 200, 30, 45),
+ tb("Body zero.", 0, 72, 400, 160, 180),
+ tb("Body one.", 1, 72, 400, 160, 180),
+ tb("THE WAY OF GO", 2, 72, 200, 30, 45),
+ tb("Body two.", 2, 72, 400, 160, 180),
+ tb("THE WAY OF GO", 3, 72, 200, 30, 45),
+ tb("Body three.", 3, 72, 400, 160, 180),
+ tb("Body four.", 4, 72, 400, 160, 180),
+ }
+ got := RemoveHeaderFooterBoxes(above, heights)
+ if len(got) != 5 {
+ t.Fatalf("3 of 5 pages is half, the headers must be dropped: got %d boxes, want 5", len(got))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_DropsPageNumberFooter: identical "- N -" footers
+// (digit-masked to the same key) are removed.
+func TestRemoveHeaderFooterBoxes_DropsPageNumberFooter(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}
+ boxes := []pdf.TextBox{
+ tb("Body zero.", 0, 72, 400, 160, 180),
+ tb("- 1 -", 0, 280, 320, 820, 835),
+ tb("Body one.", 1, 72, 400, 160, 180),
+ tb("- 2 -", 1, 280, 320, 820, 835),
+ tb("Body two.", 2, 72, 400, 160, 180),
+ tb("- 3 -", 2, 280, 320, 820, 835),
+ }
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ for _, b := range got {
+ if strings.Contains(b.Text, "- ") && strings.Contains(b.Text, " -") {
+ t.Fatalf("footer %q should have been removed", b.Text)
+ }
+ }
+ if len(got) != 3 {
+ t.Fatalf("expected only the 3 body boxes, got %d", len(got))
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/parser.go b/internal/deepdoc/parser/pdf/parser.go
index 69ea43f0c0..7c505d67a8 100644
--- a/internal/deepdoc/parser/pdf/parser.go
+++ b/internal/deepdoc/parser/pdf/parser.go
@@ -82,14 +82,7 @@ func NewTableBuilderFor(doc pdf.DocAnalyzer) pdf.TableBuilder {
// ParseRaw is the internal entry point: runs the core pipeline on an
// already-opened engine. Exported for tests that inject mock engines.
func (p *Parser) ParseRaw(ctx context.Context, engine pdf.PDFEngine, docAnalyzer pdf.DocAnalyzer) (*pdf.ParseResult, error) {
- outlines := p.extractOutlines(engine)
-
- result, err := p.processPages(ctx, engine, docAnalyzer)
- if err != nil {
- return nil, err
- }
- result.Outlines = outlines
- return result, nil
+ return p.processPages(ctx, engine, docAnalyzer)
}
// ── ParseRaw helper functions ───────────────────────────────────────────────
@@ -211,7 +204,7 @@ func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,
var dlaRegions []pdf.DLAPageRegions
if pageImg != nil && renderErr == nil {
- ocrBoxes, updatedChars, ocrUsed = p.processPageBoxes(ctx, pageImg, chars, pg, renderErr, isScanNoise, docAnalyzer)
+ ocrBoxes, updatedChars, ocrUsed = p.processPageBoxes(ctx, pageImg, chars, pg, renderErr, isScanNoise, docAnalyzer, pageZoom)
annotated, pageTables, dlaRegions = p.enrichOnePageWithDeepDoc(
ctx, pageImg, ocrBoxes, pg, renderErr, docAnalyzer, tb, pageZoom)
}
@@ -230,7 +223,7 @@ func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,
slog.Debug("per-page zoom retry", "page", pg, "zoom", retryZoom)
retryImg, retryRenderErr := p.renderAtDPI(ctx, engine, pg, retryZoom*72)
if retryRenderErr == nil && retryImg != nil {
- ocrBoxes, updatedChars, ocrUsed = p.processPageBoxes(ctx, retryImg, chars, pg, retryRenderErr, isScanNoise, docAnalyzer)
+ ocrBoxes, updatedChars, ocrUsed = p.processPageBoxes(ctx, retryImg, chars, pg, retryRenderErr, isScanNoise, docAnalyzer, retryZoom)
annotated, pageTables, dlaRegions = p.enrichOnePageWithDeepDoc(
ctx, retryImg, ocrBoxes, pg, retryRenderErr, docAnalyzer, tb, retryZoom)
pageImg = retryImg
@@ -302,7 +295,7 @@ func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,
// glyphs so downstream median-height/width stats stay meaningful.
// - bool: whether OCR was actually used to produce ocrBoxes.
func (p *Parser) processPageBoxes(ctx context.Context, pageImg image.Image, chars []pdf.TextChar, pg int,
- renderErr error, isScanNoise bool, docAnalyzer pdf.DocAnalyzer,
+ renderErr error, isScanNoise bool, docAnalyzer pdf.DocAnalyzer, zoom float64,
) ([]pdf.TextBox, []pdf.TextChar, bool) {
var ocrBoxes []pdf.TextBox
ocrUsed := false
@@ -310,14 +303,14 @@ func (p *Parser) processPageBoxes(ctx context.Context, pageImg image.Image, char
if renderErr == nil && pageImg != nil {
hasCleanChars := len(chars) > 0 && !isScanNoise && !util.IsGarbledPage(chars)
if hasCleanChars {
- ocrBoxes = p.ocrMergeChars(ctx, pageImg, chars, docAnalyzer, pg)
+ ocrBoxes = p.ocrMergeChars(ctx, pageImg, chars, docAnalyzer, pg, zoom)
ocrUsed = ocrBoxes != nil
} else {
label := "scan page"
if len(chars) > 0 && !isScanNoise {
label = "garbled page"
}
- ocrBoxes = p.ocrDetectAndRecognize(ctx, pageImg, docAnalyzer, pg, label)
+ ocrBoxes = p.ocrDetectAndRecognize(ctx, pageImg, docAnalyzer, pg, label, zoom)
ocrUsed = ocrBoxes != nil
if ocrUsed {
// Synthetic OCR chars feed downstream median calculations.
@@ -329,7 +322,7 @@ func (p *Parser) processPageBoxes(ctx context.Context, pageImg image.Image, char
}
} else if len(chars) > 0 {
// Detect failed but chars exist: try the merge path.
- ocrBoxes = p.ocrMergeChars(ctx, pageImg, chars, docAnalyzer, pg)
+ ocrBoxes = p.ocrMergeChars(ctx, pageImg, chars, docAnalyzer, pg, zoom)
ocrUsed = ocrBoxes != nil
}
}
@@ -429,10 +422,11 @@ func (p *Parser) runPageWorkers(ctx context.Context, engine pdf.PDFEngine,
// state. Document-wide layout, table merge/replace, cross-page figures, and
// metrics aggregation happen here so page workers never mutate shared state.
// pageResults are expected to be sorted by page number.
-func (p *Parser) assembleDocument(ctx context.Context, pages []int, pageResults []*pageResult) (*pdf.ParseResult, error) {
+func (p *Parser) assembleDocument(ctx context.Context, pages []int, pageResults []*pageResult, outlines []pdf.Outline) (*pdf.ParseResult, error) {
result := &pdf.ParseResult{
PageHeight: make(map[int]float64),
PageWidth: make(map[int]float64),
+ Outlines: outlines,
}
var boxes []pdf.TextBox
@@ -481,8 +475,11 @@ func (p *Parser) assembleDocument(ctx context.Context, pages []int, pageResults
return result, nil
}
+ // A TOC is a document prefix, so the box-shape signal is only meaningful
+ // when this parse covers the document's first page.
+ coversDocumentStart := len(pages) > 0 && pages[0] == 0
if err := p.buildLayout(ctx, result, boxes, pageChars,
- medianHeights, medianWidths, pageEnglish); err != nil {
+ medianHeights, medianWidths, pageEnglish, coversDocumentStart); err != nil {
return nil, fmt.Errorf("buildLayout: %w", err)
}
return result, nil
@@ -492,11 +489,15 @@ func (p *Parser) assembleDocument(ctx context.Context, pages []int, pageResults
// the global layout/table/figure pipeline. It is the only step that runs
// AssignColumn, TextMerge, FinalReadingOrderMerge, NaiveVerticalMerge, table
// merge, figure consolidation, BoxesToSections, and caption merge.
+//
+// coversDocumentStart reports whether `pages` started at the document's first
+// page, which is what the TOC box-shape signal requires (see RemoveTOCBoxes).
func (p *Parser) buildLayout(ctx context.Context,
result *pdf.ParseResult,
boxes []pdf.TextBox, pageChars map[int][]pdf.TextChar,
medianHeights, medianWidths map[int]float64,
pageEnglish map[int]bool,
+ coversDocumentStart bool,
) error {
result.Metrics.BoxesInitial = len(boxes)
@@ -524,6 +525,40 @@ func (p *Parser) buildLayout(ctx context.Context,
// filter would have only caught the former.
boxes = lyt.FilterWatermarkBoxes(boxes)
+ // Page-level content removal on intact box geometry. These MUST run before
+ // TextMerge: RemoveTOCBoxes relies on leader-dot boxes (which TextMerge
+ // folds into adjacent text) and RemoveHeaderFooterBoxes relies on header
+ // boxes that TextMerge would otherwise merge into the first body section.
+ //
+ // Header/footer removal runs FIRST because its evidence is cross-page: it
+ // counts how many of the document's pages carry the same margin text, while
+ // TOC removal deletes whole pages. Running the TOC pass first would take
+ // that margin text off the pages it deletes, dropping the count below the
+ // half-the-pages bar, so a document with both options on would keep a
+ // running header that either option removes on its own.
+ //
+ // The TOC pass is nearly indifferent to the order. It decides on page shapes
+ // and absolute bookmark numbers, and the only boxes header/footer removal
+ // takes from a TOC page are margin boxes, which never carried an entry
+ // marker. They do count towards tocMinShortBoxes, so a TOC page sitting
+ // exactly on that threshold can fall below it and be kept — a missed TOC
+ // page, which is the direction this detector errs in regardless.
+ //
+ // coversDocumentStart gates the TOC box-shape signal: a TOC is a document
+ // prefix, so a parse restricted to a later page range must not read its own
+ // first page as one. The outline signal uses absolute page numbers and needs
+ // no such gate.
+ boxesBefore := len(boxes)
+ if p.Config.RemoveHeaderFooter {
+ boxes = lyt.RemoveHeaderFooterBoxes(boxes, result.PageHeight)
+ result.Metrics.BoxesHeaderFooterRemoved = boxesBefore - len(boxes)
+ boxesBefore = len(boxes)
+ }
+ if p.Config.RemoveTOC {
+ boxes = lyt.RemoveTOCBoxes(boxes, lyt.TOCPageRangeFromOutlines(result.Outlines), coversDocumentStart)
+ result.Metrics.BoxesTOCRemoved = boxesBefore - len(boxes)
+ }
+
boxes = lyt.TextMerge(boxes, medianHeights)
result.Metrics.BoxesTextMerge = len(boxes)
@@ -560,10 +595,15 @@ func (p *Parser) processPages(ctx context.Context, engine pdf.PDFEngine, docAnal
if err != nil {
return nil, fmt.Errorf("page count: %w", err)
}
+ // The outlines are read before the layout is built: buildLayout uses them to
+ // select TOC pages on the intact box geometry (see Parser.buildLayout), so
+ // they cannot be attached to the result afterwards.
+ outlines := p.extractOutlines(engine)
if pageCount == 0 {
return &pdf.ParseResult{
PageHeight: make(map[int]float64),
PageWidth: make(map[int]float64),
+ Outlines: outlines,
}, nil
}
@@ -583,7 +623,7 @@ func (p *Parser) processPages(ctx context.Context, engine pdf.PDFEngine, docAnal
slog.Warn("runPageWorkers: some pages failed", "err", pageErr)
}
- result, err := p.assembleDocument(ctx, pages, pageResults)
+ result, err := p.assembleDocument(ctx, pages, pageResults, outlines)
if err != nil {
return nil, err
}
diff --git a/internal/deepdoc/parser/pdf/parser_ocr.go b/internal/deepdoc/parser/pdf/parser_ocr.go
index 7a137a85e3..d342062436 100644
--- a/internal/deepdoc/parser/pdf/parser_ocr.go
+++ b/internal/deepdoc/parser/pdf/parser_ocr.go
@@ -12,7 +12,7 @@ import (
"strings"
)
-func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image, doc pdf.DocAnalyzer, pageNum int, logLabel string) []pdf.TextBox {
+func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image, doc pdf.DocAnalyzer, pageNum int, logLabel string, zoom float64) []pdf.TextBox {
boxes, err := p.inferOCRDetect(ctx, doc, pageImg)
if err != nil || len(boxes) == 0 {
if err != nil {
@@ -21,12 +21,14 @@ func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image,
return nil
}
- // detectBoxes returns image-pixel coords; ocrMergeChars divides by
- // pdf.DlaScale before emitting boxes so downstream layout receives
- // PDF-point coordinates. ocrDetectAndRecognize must match the same
- // conversion so both OCR paths produce the same coordinate space.
- imgW := float64(pageImg.Bounds().Dx()) / pdf.DlaScale
- imgH := float64(pageImg.Bounds().Dy()) / pdf.DlaScale
+ // detectBoxes returns image-pixel coords; both OCR paths divide by the zoom
+ // the page was rendered at before emitting boxes so downstream layout
+ // receives PDF-point coordinates — the same units as ParseResult.PageHeight.
+ // That zoom is DlaScale for the default render and the retry zoom after a
+ // per-page re-render, so it cannot be a constant.
+ scale := ocrCoordinateScale(zoom)
+ imgW := float64(pageImg.Bounds().Dx()) / scale
+ imgH := float64(pageImg.Bounds().Dy()) / scale
// For each box, de-skew via WarpCrop (layer 1) and build the layer-2
// rotation candidates: short/wide crops get one candidate at 0 deg; tall
@@ -61,10 +63,10 @@ func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image,
{X: b.X3, Y: b.Y3},
})
// Convert detection bounds to PDF-point space (mirrors detectBoxes).
- px0 := float64(x0) / pdf.DlaScale
- py0 := float64(y0) / pdf.DlaScale
- px1 := float64(x1) / pdf.DlaScale
- py1 := float64(y1) / pdf.DlaScale
+ px0 := float64(x0) / scale
+ py0 := float64(y0) / scale
+ px1 := float64(x1) / scale
+ py1 := float64(y1) / scale
if px0 < 0 {
px0 = 0
}
@@ -231,8 +233,8 @@ type ocrDetectBox struct {
srcIdx int
}
-func (p *Parser) ocrMergeChars(ctx context.Context, pageImg image.Image, chars []pdf.TextChar, doc pdf.DocAnalyzer, pageNum int) []pdf.TextBox {
- boxes, scale, err := p.detectBoxes(ctx, pageImg, doc, pageNum)
+func (p *Parser) ocrMergeChars(ctx context.Context, pageImg image.Image, chars []pdf.TextChar, doc pdf.DocAnalyzer, pageNum int, zoom float64) []pdf.TextBox {
+ boxes, scale, err := p.detectBoxes(ctx, pageImg, doc, pageNum, zoom)
if err != nil || len(boxes) == 0 {
return nil
}
@@ -240,14 +242,30 @@ func (p *Parser) ocrMergeChars(ctx context.Context, pageImg image.Image, chars [
return p.buildTextBoxes(ctx, pageImg, boxes, boxChars, doc, scale, pageNum)
}
-func (p *Parser) detectBoxes(ctx context.Context, pageImg image.Image, doc pdf.DocAnalyzer, pageNum int) ([]ocrDetectBox, float64, error) {
+// ocrCoordinateScale returns the factor that converts image pixels into PDF
+// points for a page rendered at zoom, falling back to the default DLA scale
+// when the caller supplies none. Detection and recognition must both use it:
+// any other factor leaves the boxes in a scale of their own, and every
+// page-relative comparison downstream — the header/footer zone check in
+// particular — then compares unlike quantities.
+func ocrCoordinateScale(zoom float64) float64 {
+ if zoom > 0 {
+ return zoom
+ }
+ return pdf.DlaScale
+}
+
+func (p *Parser) detectBoxes(ctx context.Context, pageImg image.Image, doc pdf.DocAnalyzer, pageNum int, zoom float64) ([]ocrDetectBox, float64, error) {
ocrDetectBoxes, err := p.inferOCRDetect(ctx, doc, pageImg)
if err != nil || len(ocrDetectBoxes) == 0 {
return nil, 0, err
}
slog.Debug("ocrMergeChars detect", "page", pageNum, "boxes", len(ocrDetectBoxes))
- scale := pdf.DlaScale // 3.0
+ // The caller multiplies the returned boxes back by this scale to crop the
+ // original render, so passing the render zoom keeps the round trip exact
+ // and makes the emitted coordinates PDF points (see ocrDetectAndRecognize).
+ scale := ocrCoordinateScale(zoom)
imgBounds := pageImg.Bounds()
imgW := float64(imgBounds.Dx()) / scale
imgH := float64(imgBounds.Dy()) / scale
diff --git a/internal/deepdoc/parser/pdf/parser_ocr_test.go b/internal/deepdoc/parser/pdf/parser_ocr_test.go
index 63e1fe5d5d..22c8bfeeb8 100644
--- a/internal/deepdoc/parser/pdf/parser_ocr_test.go
+++ b/internal/deepdoc/parser/pdf/parser_ocr_test.go
@@ -25,7 +25,7 @@ func TestOCRMergeChars_FullCoverage(t *testing.T) {
{X0: 12, X1: 28, Top: 2, Bottom: 35, Text: "World"},
}
- boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0)
+ boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0, pdf.DlaScale)
if len(boxes) != 1 {
t.Fatalf("expected 1 box, got %d", len(boxes))
}
@@ -54,7 +54,7 @@ func TestOCRMergeChars_PartialCoverage(t *testing.T) {
{X0: 2, X1: 12, Top: 2, Bottom: 15, Text: "A"},
}
- boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0)
+ boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0, pdf.DlaScale)
if len(boxes) != 2 {
t.Fatalf("expected 2 boxes, got %d", len(boxes))
}
@@ -81,14 +81,14 @@ func TestOCRMergeChars_NoDetectBoxes(t *testing.T) {
}
ctx := t.Context()
- boxes := p.ocrMergeChars(ctx, testPageImg(), chars, mock, 0)
+ boxes := p.ocrMergeChars(ctx, testPageImg(), chars, mock, 0, pdf.DlaScale)
if boxes != nil {
t.Errorf("expected nil for no detect boxes, got %d boxes", len(boxes))
}
// Also test empty OCRBoxes
mock.OCRBoxes = []pdf.OCRBox{}
- boxes = p.ocrMergeChars(ctx, testPageImg(), chars, mock, 0)
+ boxes = p.ocrMergeChars(ctx, testPageImg(), chars, mock, 0, pdf.DlaScale)
if boxes != nil {
t.Errorf("expected nil for empty detect boxes, got %d boxes", len(boxes))
}
@@ -114,7 +114,7 @@ func TestOCRMergeChars_GarbledChars(t *testing.T) {
{X0: 22, X1: 28, Top: 2, Bottom: 35, Text: "a"}, // normal
}
- boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0)
+ boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0, pdf.DlaScale)
if len(boxes) != 1 {
t.Fatalf("expected 1 box, got %d", len(boxes))
}
@@ -143,7 +143,7 @@ func TestOCRMergeChars_HeightGate(t *testing.T) {
{X0: 2, X1: 10, Top: 2, Bottom: 3, Text: "tiny"},
}
- boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0)
+ boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0, pdf.DlaScale)
if len(boxes) != 1 {
t.Fatalf("expected 1 box (OCR fallback after height gate), got %d", len(boxes))
}
@@ -174,7 +174,7 @@ func TestOCRMergeChars_FontEncodingGarbled(t *testing.T) {
Text: "#", FontName: "DY1+SimSun", PageNumber: 0,
}
}
- boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0)
+ boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0, pdf.DlaScale)
if len(boxes) != 1 {
t.Fatalf("expected 1 OCR-fallback box, got %d", len(boxes))
}
@@ -253,7 +253,7 @@ func TestOCRMergeChars_MixedFontSizes(t *testing.T) {
{X0: 12, X1: 24, Top: 5, Bottom: 35, Text: "大"}, // larger font, lower baseline
{X0: 24, X1: 36, Top: 5, Bottom: 35, Text: "号"}, // same size as 大, rightmost
}
- boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0)
+ boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0, pdf.DlaScale)
if len(boxes) != 1 {
t.Fatalf("expected 1 box, got %d", len(boxes))
}
@@ -285,7 +285,7 @@ func TestOCRMergeChars_BoxOrder(t *testing.T) {
{X0: 2, X1: 10, Top: 16, Bottom: 19, Text: "B"}, // box 2 (middle)
{X0: 2, X1: 10, Top: 32, Bottom: 37, Text: "C"}, // box 3 (bottom)
}
- boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0)
+ boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0, pdf.DlaScale)
if len(boxes) != 3 {
t.Fatalf("expected 3 boxes, got %d", len(boxes))
}
@@ -320,7 +320,7 @@ func TestOCRMergeChars_OverlappingBoxes(t *testing.T) {
{X0: 12, X1: 18, Top: 2, Bottom: 12, Text: "乙"}, // overlap zone
{X0: 22, X1: 28, Top: 2, Bottom: 12, Text: "丙"}, // Box B only
}
- boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0)
+ boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0, pdf.DlaScale)
if len(boxes) != 2 {
t.Fatalf("expected 2 boxes, got %d", len(boxes))
}
@@ -399,3 +399,47 @@ func TestPdfOxideUnmappedGarbled_RealWorldGarbled(t *testing.T) {
t.Error("real-world garbled text with ### clusters should be detected")
}
}
+
+// TestOCRBoxesTrackRenderZoom pins the coordinate invariant of both OCR paths:
+// the page is rendered at `zoom`, so detection pixels divide by that same zoom
+// to become PDF points — the units ParseResult.PageHeight is in, since
+// parser.go computes it as renderHeight/zoom. Dividing by a constant instead
+// leaves the boxes retryZoom/DlaScale times too large after a per-page retry
+// render (9/3 with the default config), which is how body boxes end up looking
+// like margin boxes to the header/footer zone check.
+func TestOCRBoxesTrackRenderZoom(t *testing.T) {
+ img := testPageImg() // 90x120 pixels
+ mock := &MockDocAnalyzer{
+ Healthy: true,
+ OCRBoxes: []pdf.OCRBox{
+ {X0: 0, Y0: 0, X1: 90, Y1: 0, X2: 90, Y2: 120, X3: 0, Y3: 120},
+ },
+ OCRTexts: []pdf.OCRText{{Text: "OCR text", Confidence: 0.9}},
+ }
+ p := NewParser(pdf.DefaultParserConfig())
+
+ // DlaScale is the default render; 9.0 is what the retry render uses when
+ // Config.Zoom is 3 (the default), so both must land inside their own page.
+ for _, zoom := range []float64{pdf.DlaScale, 9.0} {
+ pageW := float64(img.Bounds().Dx()) / zoom
+ pageH := float64(img.Bounds().Dy()) / zoom
+
+ merged := p.ocrMergeChars(t.Context(), img, nil, mock, 0, zoom)
+ if len(merged) != 1 {
+ t.Fatalf("zoom %v: ocrMergeChars expected 1 box, got %d", zoom, len(merged))
+ }
+ if merged[0].X1 > pageW || merged[0].Bottom > pageH {
+ t.Fatalf("zoom %v: ocrMergeChars box (%v,%v,%v,%v) falls outside the %.2fx%.2fpt page of this render",
+ zoom, merged[0].X0, merged[0].Top, merged[0].X1, merged[0].Bottom, pageW, pageH)
+ }
+
+ scanned := p.ocrDetectAndRecognize(t.Context(), img, mock, 0, "scan page", zoom)
+ if len(scanned) != 1 {
+ t.Fatalf("zoom %v: ocrDetectAndRecognize expected 1 box, got %d", zoom, len(scanned))
+ }
+ if scanned[0].X1 > pageW || scanned[0].Bottom > pageH {
+ t.Fatalf("zoom %v: ocrDetectAndRecognize box (%v,%v,%v,%v) falls outside the %.2fx%.2fpt page of this render",
+ zoom, scanned[0].X0, scanned[0].Top, scanned[0].X1, scanned[0].Bottom, pageW, pageH)
+ }
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/parser_ocr_warp_test.go b/internal/deepdoc/parser/pdf/parser_ocr_warp_test.go
index 35f17b0b60..dc98f66a85 100644
--- a/internal/deepdoc/parser/pdf/parser_ocr_warp_test.go
+++ b/internal/deepdoc/parser/pdf/parser_ocr_warp_test.go
@@ -76,7 +76,7 @@ func TestOCRDetectAndRecognize_WarpsCrop(t *testing.T) {
boxes: []pdf.OCRBox{box},
texts: []pdf.OCRText{{Text: "x", Confidence: 0.9}},
}
- got := p.ocrDetectAndRecognize(t.Context(), page, cap, 0, "warp")
+ got := p.ocrDetectAndRecognize(t.Context(), page, cap, 0, "warp", pdf.DlaScale)
if len(got) != 1 {
t.Fatalf("expected 1 text box, got %d", len(got))
}
@@ -171,7 +171,7 @@ func TestOCRMergeChars_WarpsEmptyBoxCrop(t *testing.T) {
boxes: []pdf.OCRBox{box},
texts: []pdf.OCRText{{Text: "x", Confidence: 0.9}},
}
- got := p.ocrMergeChars(t.Context(), page, chars, cap, 0)
+ got := p.ocrMergeChars(t.Context(), page, chars, cap, 0, pdf.DlaScale)
if len(got) == 0 {
t.Fatal("expected at least one text box from the merge path")
}
diff --git a/internal/deepdoc/parser/pdf/parser_test.go b/internal/deepdoc/parser/pdf/parser_test.go
index c189c1f14f..78545b1afc 100644
--- a/internal/deepdoc/parser/pdf/parser_test.go
+++ b/internal/deepdoc/parser/pdf/parser_test.go
@@ -38,14 +38,14 @@ func TestOCR_Fallback(t *testing.T) {
dummyImg := image.NewRGBA(image.Rect(0, 0, 100, 100))
t.Run("nil image", func(t *testing.T) {
- if got := p.ocrDetectAndRecognize(t.Context(), nil, &MockDocAnalyzer{Healthy: true}, 0, "garbled page"); got != nil {
+ if got := p.ocrDetectAndRecognize(t.Context(), nil, &MockDocAnalyzer{Healthy: true}, 0, "garbled page", pdf.DlaScale); got != nil {
t.Error("nil image → nil")
}
})
t.Run("detect returns no boxes", func(t *testing.T) {
mock := &MockDocAnalyzer{Healthy: true, OCRBoxes: nil}
- if got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "garbled page"); got != nil {
+ if got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "garbled page", pdf.DlaScale); got != nil {
t.Error("no det boxes → nil")
}
})
@@ -56,7 +56,7 @@ func TestOCR_Fallback(t *testing.T) {
OCRBoxes: []pdf.OCRBox{{X0: 10, Y0: 20, X1: 90, Y1: 20, X2: 90, Y2: 40, X3: 10, Y3: 40}},
OCRTexts: []pdf.OCRText{{Text: "Hello", Confidence: 0.9}},
}
- got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "garbled page")
+ got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "garbled page", pdf.DlaScale)
if len(got) != 1 {
t.Fatalf("expected 1 pdf.TextChar, got %d", len(got))
}
@@ -71,7 +71,7 @@ func TestOCR_Fallback(t *testing.T) {
OCRBoxes: []pdf.OCRBox{{X0: 10, Y0: 20, X1: 90, Y1: 20, X2: 90, Y2: 40, X3: 10, Y3: 40}},
OCRTexts: []pdf.OCRText{{Text: "", Confidence: 0.1}},
}
- got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "garbled page")
+ got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "garbled page", pdf.DlaScale)
if len(got) != 0 {
t.Error("empty rec text → empty result")
}
@@ -87,14 +87,14 @@ func TestOCR_ScanPage(t *testing.T) {
dummyImg := image.NewRGBA(image.Rect(0, 0, 100, 100))
t.Run("nil image", func(t *testing.T) {
- if got := p.ocrDetectAndRecognize(t.Context(), nil, &MockDocAnalyzer{Healthy: true}, 0, "scan page"); got != nil {
+ if got := p.ocrDetectAndRecognize(t.Context(), nil, &MockDocAnalyzer{Healthy: true}, 0, "scan page", pdf.DlaScale); got != nil {
t.Error("nil image → nil")
}
})
t.Run("detect returns no boxes", func(t *testing.T) {
mock := &MockDocAnalyzer{Healthy: true, OCRBoxes: nil}
- if got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "scan page"); got != nil {
+ if got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "scan page", pdf.DlaScale); got != nil {
t.Error("no det boxes → nil")
}
})
@@ -108,7 +108,7 @@ func TestOCR_ScanPage(t *testing.T) {
},
OCRTexts: []pdf.OCRText{{Text: "Hello", Confidence: 0.9}, {Text: "World", Confidence: 0.8}},
}
- got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "scan page")
+ got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "scan page", pdf.DlaScale)
if len(got) < 1 {
t.Error("expected at least 1 pdf.TextChar")
}
@@ -120,7 +120,7 @@ func TestOCR_ScanPage(t *testing.T) {
OCRBoxes: []pdf.OCRBox{{X0: 10, Y0: 20, X1: 90, Y1: 20, X2: 90, Y2: 40, X3: 10, Y3: 40}},
OCRTexts: []pdf.OCRText{},
}
- got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "scan page")
+ got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "scan page", pdf.DlaScale)
if len(got) != 0 {
t.Error("no rec text → empty")
}
@@ -264,7 +264,7 @@ func TestOCR_Fallback_PUAGarbled(t *testing.T) {
OCRBoxes: []pdf.OCRBox{{X0: 10, Y0: 20, X1: 90, Y1: 20, X2: 90, Y2: 40, X3: 10, Y3: 40}},
OCRTexts: []pdf.OCRText{{Text: "PUA OCR text", Confidence: 0.9}},
}
- got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "garbled page")
+ got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "garbled page", pdf.DlaScale)
if len(got) != 1 || got[0].Text != "PUA OCR text" {
t.Errorf("PUA garbled should trigger OCR, got %v", got)
}
@@ -278,7 +278,7 @@ func TestOCR_MergeChars(t *testing.T) {
t.Run("nil image", func(t *testing.T) {
chars := []pdf.TextChar{{X0: 10, Top: 10, X1: 20, Bottom: 30, Text: "A", PageNumber: 0}}
- if boxes := p.ocrMergeChars(t.Context(), nil, chars, &MockDocAnalyzer{Healthy: true}, 0); boxes != nil {
+ if boxes := p.ocrMergeChars(t.Context(), nil, chars, &MockDocAnalyzer{Healthy: true}, 0, pdf.DlaScale); boxes != nil {
t.Error("nil image → nil")
}
})
@@ -286,7 +286,7 @@ func TestOCR_MergeChars(t *testing.T) {
t.Run("detect returns no boxes", func(t *testing.T) {
mock := &MockDocAnalyzer{Healthy: true, OCRBoxes: []pdf.OCRBox{}}
chars := []pdf.TextChar{{X0: 10, Top: 10, X1: 20, Bottom: 30, Text: "A", PageNumber: 0}}
- if boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0); boxes != nil {
+ if boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale); boxes != nil {
t.Error("no detect boxes → nil")
}
})
@@ -298,7 +298,7 @@ func TestOCR_MergeChars(t *testing.T) {
OCRTexts: []pdf.OCRText{{Text: "Hello OCR", Confidence: 0.9}},
}
chars := []pdf.TextChar{{X0: 10, X1: 30, Top: 10, Bottom: 30, Text: "Hello", PageNumber: 0}}
- boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0)
+ boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale)
if len(boxes) != 1 {
t.Fatalf("expected 1 box, got %d", len(boxes))
}
@@ -315,7 +315,7 @@ func TestOCR_MergeChars(t *testing.T) {
OCRTexts: []pdf.OCRText{{Text: "OCR", Confidence: 0.9}},
}
chars := []pdf.TextChar{{X0: 10, X1: 20, Top: 10, Bottom: 20, Text: "A", PageNumber: 0}}
- boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0)
+ boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale)
if len(boxes) != 1 {
t.Fatalf("expected 1 box (OCR), got %d", len(boxes))
}
@@ -331,7 +331,7 @@ func TestOCR_MergeChars(t *testing.T) {
OCRTexts: []pdf.OCRText{},
}
chars := []pdf.TextChar{{X0: 10, X1: 20, Top: 10, Bottom: 20, Text: "A", PageNumber: 0}}
- boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0)
+ boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale)
if len(boxes) != 0 {
t.Fatalf("expected 0 boxes (empty OCR), got %d", len(boxes))
}
@@ -350,7 +350,7 @@ func TestOCR_MergeChars(t *testing.T) {
},
}
chars := []pdf.TextChar{{X0: 10, X1: 30, Top: 10, Bottom: 30, Text: "Hello", PageNumber: 0}}
- boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0)
+ boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale)
if len(boxes) != 2 {
t.Fatalf("expected 2 boxes, got %d", len(boxes))
}
@@ -378,7 +378,7 @@ func TestOCR_MergeChars(t *testing.T) {
{X0: 70, X1: 90, Top: 110, Bottom: 130, Text: "c", PageNumber: 0},
{X0: 10, X1: 30, Top: 10, Bottom: 30, Text: "a", PageNumber: 0},
}
- boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0)
+ boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale)
if len(boxes) != 2 {
t.Fatalf("expected 2 detect boxes, got %d", len(boxes))
}
@@ -405,7 +405,7 @@ func TestOCR_MergeChars(t *testing.T) {
{X0: 10, X1: 30, Top: 30, Bottom: 50, Text: "A", PageNumber: 0},
{X0: 40, X1: 60, Top: 20, Bottom: 120, Text: "B", PageNumber: 0},
}
- boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0)
+ boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale)
if len(boxes) != 1 {
t.Fatalf("expected 1 box, got %d", len(boxes))
}
@@ -428,7 +428,7 @@ func TestOCR_MergeChars(t *testing.T) {
{X0: 30, X1: 40, Top: 10, Bottom: 20, Text: "", PageNumber: 0},
{X0: 50, X1: 60, Top: 10, Bottom: 20, Text: "a", PageNumber: 0},
}
- boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0)
+ boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale)
if len(boxes) != 1 {
t.Fatalf("expected 1 box, got %d", len(boxes))
}
@@ -450,7 +450,7 @@ func TestOCR_MergeChars(t *testing.T) {
{Text: "d", X0: 10, X1: 20, Top: 10, Bottom: 25, PageNumber: 0},
{Text: "o", X0: 21, X1: 30, Top: 10, Bottom: 25, PageNumber: 0},
}
- boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0)
+ boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale)
if len(boxes) != 1 {
t.Fatalf("expected 1 box, got %d", len(boxes))
}
diff --git a/internal/deepdoc/parser/pdf/toc_box_removal_test.go b/internal/deepdoc/parser/pdf/toc_box_removal_test.go
new file mode 100644
index 0000000000..d916362413
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/toc_box_removal_test.go
@@ -0,0 +1,191 @@
+package pdf
+
+import (
+ "strings"
+ "testing"
+
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+)
+
+// outlineEngine is a MockEngine that also reports bookmarks; the base stub
+// hardcodes Outlines to nil.
+type outlineEngine struct {
+ MockEngine
+ outlines []pdf.Outline
+}
+
+func (e *outlineEngine) Outlines() ([]pdf.Outline, error) { return e.outlines, nil }
+
+// tocDocEngine builds a two-page document: page 0 is a TOC carrying two short
+// lines, page 1 is body text. Two short boxes stay below tocMinShortBoxes, so
+// the box signal cannot select page 0 and the bookmark is the only signal that
+// can.
+func tocDocEngine() *outlineEngine {
+ return &outlineEngine{
+ MockEngine: MockEngine{
+ NumPages: 2,
+ Chars: map[int][]pdf.TextChar{
+ 0: {
+ {X0: 50, X1: 550, Top: 100, Bottom: 112, Text: "目录", PageNumber: 0},
+ {X0: 50, X1: 550, Top: 300, Bottom: 312, Text: "第一章 道可道", PageNumber: 0},
+ },
+ 1: {
+ {X0: 50, X1: 550, Top: 100, Bottom: 112, Text: "body text", PageNumber: 1},
+ },
+ },
+ },
+ outlines: []pdf.Outline{
+ {Title: "目录", Level: 0, PageNumber: 1},
+ {Title: "第一章", Level: 0, PageNumber: 2},
+ },
+ }
+}
+
+func sectionsOnPage(result *pdf.ParseResult, pg int) int {
+ n := 0
+ for _, s := range result.Sections {
+ for _, p := range s.Positions {
+ for _, pn := range p.PageNumbers {
+ if pn == pg {
+ n++
+ }
+ }
+ }
+ }
+ return n
+}
+
+// TestParseRaw_RemoveTOCDropsPageSelectedByOutline drives the option end to end:
+// the bookmark names the TOC page, so its boxes must never reach the sections.
+func TestParseRaw_RemoveTOCDropsPageSelectedByOutline(t *testing.T) {
+ cfg := pdf.DefaultParserConfig()
+ cfg.RemoveTOC = true
+
+ result, err := NewParser(cfg).ParseRaw(t.Context(), tocDocEngine(), &MockDocAnalyzer{Healthy: true})
+ if err != nil {
+ t.Fatalf("ParseRaw: %v", err)
+ }
+ if result.Metrics.BoxesTOCRemoved == 0 {
+ t.Fatal("expected the bookmark-selected TOC page boxes to be removed")
+ }
+ if n := sectionsOnPage(result, 0); n != 0 {
+ t.Fatalf("page 0 is the TOC and must be gone, %d sections still reference it", n)
+ }
+ if n := sectionsOnPage(result, 1); n == 0 {
+ t.Fatal("body page 1 must survive TOC removal")
+ }
+}
+
+// TestParseRaw_RemoveTOCOffKeepsPage is the control: the same document keeps its
+// TOC page when the option is off.
+func TestParseRaw_RemoveTOCOffKeepsPage(t *testing.T) {
+ result, err := NewParser(pdf.DefaultParserConfig()).ParseRaw(t.Context(), tocDocEngine(), &MockDocAnalyzer{Healthy: true})
+ if err != nil {
+ t.Fatalf("ParseRaw: %v", err)
+ }
+ if result.Metrics.BoxesTOCRemoved != 0 {
+ t.Fatalf("RemoveTOC is off, got %d boxes removed", result.Metrics.BoxesTOCRemoved)
+ }
+ if n := sectionsOnPage(result, 0); n == 0 {
+ t.Fatal("with RemoveTOC off the TOC page must stay")
+ }
+}
+
+// fixtureHeader is the running header the combined fixture repeats. It carries
+// no digits, so every page normalizes to the same comparison key.
+const fixtureHeader = "BOOK TITLE"
+
+// combinedRemovalEngine builds a document where both passes have something to
+// do: pages 0 and 1 are TOC-shaped (five short boxes, four of them markers),
+// pages 2-5 carry prose, and the same header repeats in the top zone on pages
+// 0-3. Four of six pages is above the half-the-pages bar, so the header is
+// removable only while the pages carrying it are still in the box set. Render
+// height 1000 at DlaScale gives a 333pt page, which puts the header (bottom 5)
+// inside the top 10% and the body lines (top 100) outside it.
+func combinedRemovalEngine() *MockEngine {
+ line := func(pg int, top float64, text string) pdf.TextChar {
+ return pdf.TextChar{X0: 50, X1: 900, Top: top, Bottom: top + 12, Text: text, PageNumber: pg}
+ }
+ header := func(pg int) pdf.TextChar {
+ return pdf.TextChar{X0: 50, X1: 200, Top: 1, Bottom: 5, Text: fixtureHeader, PageNumber: pg}
+ }
+ // Long enough to count as prose (> tocMaxProseRunes runes).
+ const body = "This paragraph of body text is comfortably longer than sixty runes, so the page it sits on counts as prose."
+ tocLine := func(pg int, top float64, text string) pdf.TextChar { return line(pg, top, text) }
+
+ chars := map[int][]pdf.TextChar{}
+ for pg := 0; pg < 6; pg++ {
+ if pg < 2 {
+ chars[pg] = []pdf.TextChar{
+ header(pg),
+ tocLine(pg, 100, "目录"),
+ tocLine(pg, 130, "第一章 道可道"),
+ tocLine(pg, 160, "第二章 天下皆知"),
+ tocLine(pg, 190, "1"),
+ }
+ continue
+ }
+ page := []pdf.TextChar{line(pg, 100, body), line(pg, 130, body), line(pg, 160, body)}
+ if pg < 4 {
+ page = append([]pdf.TextChar{header(pg)}, page...)
+ }
+ chars[pg] = page
+ }
+ return &MockEngine{NumPages: 6, RenderW: 1000, RenderH: 1000, Chars: chars}
+}
+
+// TestParseRaw_HeaderFooterRemovesHeaderAlone is the control for the test below:
+// the fixture's header really is removable, so a combined run that keeps it is
+// reporting an interaction and not a header no run could remove.
+func TestParseRaw_HeaderFooterRemovesHeaderAlone(t *testing.T) {
+ cfg := pdf.DefaultParserConfig()
+ cfg.RemoveHeaderFooter = true
+
+ result, err := NewParser(cfg).ParseRaw(t.Context(), combinedRemovalEngine(), &MockDocAnalyzer{Healthy: true})
+ if err != nil {
+ t.Fatalf("ParseRaw: %v", err)
+ }
+ if result.Metrics.BoxesHeaderFooterRemoved == 0 {
+ t.Fatal("the repeated header must be removed when RemoveHeaderFooter is on")
+ }
+ for _, s := range result.Sections {
+ if strings.Contains(s.Text, fixtureHeader) {
+ t.Fatalf("header %q survived on its own", s.Text)
+ }
+ }
+}
+
+// TestParseRaw_TOCAndHeaderFooterBothApply pins the order the two passes run in.
+// Header/footer removal counts cross-page repetition, and TOC removal deletes
+// whole pages — so if TOC ran first it would take the header's evidence with it
+// and leave the header on the pages that survive, making the two options
+// together weaker than either one alone.
+func TestParseRaw_TOCAndHeaderFooterBothApply(t *testing.T) {
+ cfg := pdf.DefaultParserConfig()
+ cfg.RemoveTOC = true
+ cfg.RemoveHeaderFooter = true
+
+ result, err := NewParser(cfg).ParseRaw(t.Context(), combinedRemovalEngine(), &MockDocAnalyzer{Healthy: true})
+ if err != nil {
+ t.Fatalf("ParseRaw: %v", err)
+ }
+ if result.Metrics.BoxesTOCRemoved == 0 {
+ t.Fatal("the TOC pages must still be dropped when both options are on")
+ }
+ if result.Metrics.BoxesHeaderFooterRemoved == 0 {
+ t.Fatal("the running header must still be removed when both options are on")
+ }
+ for pg := 0; pg < 2; pg++ {
+ if n := sectionsOnPage(result, pg); n != 0 {
+ t.Fatalf("TOC page %d must be gone, %d sections still reference it", pg, n)
+ }
+ }
+ for _, s := range result.Sections {
+ if strings.Contains(s.Text, fixtureHeader) {
+ t.Fatalf("header %q survived the combined run", s.Text)
+ }
+ }
+ if n := sectionsOnPage(result, 4); n == 0 {
+ t.Fatal("body page 4 must survive both passes")
+ }
+}
diff --git a/internal/deepdoc/parser/type/types.go b/internal/deepdoc/parser/type/types.go
index 8345afbebf..aa76f4a697 100644
--- a/internal/deepdoc/parser/type/types.go
+++ b/internal/deepdoc/parser/type/types.go
@@ -14,11 +14,17 @@ import (
// PipelineMetrics records diagnostic counts at each pipeline stage.
type PipelineMetrics struct {
- BoxesInitial int
- BoxesTextMerge int
- BoxesVertMerge int
- BoxesFinal int
- TablesCount int
+ BoxesInitial int
+ // BoxesTOCRemoved / BoxesHeaderFooterRemoved count the boxes dropped by the
+ // optional box-level content removal passes, so the box budget can be
+ // reconciled: BoxesInitial - BoxesTOCRemoved - BoxesHeaderFooterRemoved >=
+ // BoxesTextMerge.
+ BoxesTOCRemoved int
+ BoxesHeaderFooterRemoved int
+ BoxesTextMerge int
+ BoxesVertMerge int
+ BoxesFinal int
+ TablesCount int
}
// ParseResult encapsulates all outputs from a single Parse() call.
@@ -237,6 +243,17 @@ type ParserConfig struct {
// nil/empty means parse all pages. Ranges beyond the document are clamped
// at parse time; fully out-of-range ranges are skipped.
Pages [][]int
+ // RemoveTOC enables box-level table-of-contents page removal in
+ // Parser.buildLayout. Detection relies on leader-dot boxes and per-box
+ // geometry that are destroyed by the later TextMerge pass, so it is gated
+ // onto the box-level pipeline there rather than the section-level
+ // post-process.
+ RemoveTOC bool
+ // RemoveHeaderFooter enables box-level running header / footer removal in
+ // Parser.buildLayout. It operates on intact box geometry (page zones and
+ // cross-page text repetition) before TextMerge can fold a header box into
+ // a body section.
+ RemoveHeaderFooter bool
}
// DefaultParserConfig returns a ParserConfig with sensible defaults.
diff --git a/internal/parser/parser/pdf_parser_cgo.go b/internal/parser/parser/pdf_parser_cgo.go
index 8a957ae7c2..349de5b63c 100644
--- a/internal/parser/parser/pdf_parser_cgo.go
+++ b/internal/parser/parser/pdf_parser_cgo.go
@@ -35,14 +35,14 @@ func (p *PDFParser) ParseWithResult(ctx context.Context, filename string, data [
}
cfg := deepdoctype.DefaultParserConfig()
cfg.Pages = p.Pages
+ cfg.RemoveTOC = p.RemoveTOC
+ cfg.RemoveHeaderFooter = p.RemoveHeaderFooter
parser := deepdocpdf.NewParser(cfg)
res := parsePDFWithDeepDocOptions(ctx, filename, data, pdfPostProcessOptions{
outputFormat: p.OutputFormat,
zoom: cfg.Zoom,
enableMultiColumn: p.EnableMultiColumn,
flattenMediaToText: p.FlattenMediaToText,
- removeTOC: p.RemoveTOC,
- removeHeaderFooter: p.RemoveHeaderFooter,
}, parser.Parse)
if res.Err != nil && errors.Is(res.Err, deepdocpdf.ErrNoPDFData) {
return ParseResult{Err: fmt.Errorf("%w: %s", ErrPDFEngineUnavailable, filename)}
diff --git a/internal/parser/parser/pdf_parser_common_test.go b/internal/parser/parser/pdf_parser_common_test.go
index 61c09ea99b..280772a7c8 100644
--- a/internal/parser/parser/pdf_parser_common_test.go
+++ b/internal/parser/parser/pdf_parser_common_test.go
@@ -220,81 +220,6 @@ func TestPDFParseResultToJSON_DefaultKeepsHeaderFooterLikePython(t *testing.T) {
}
}
-func TestPDFParseResultToJSONWithOptions_FiltersHeaderFooterWhenEnabled(t *testing.T) {
- parsed := &deepdoctype.ParseResult{
- Sections: []deepdoctype.Section{
- {Text: "Header", LayoutType: "header"},
- {
- Text: "Body",
- LayoutType: "",
- Positions: []deepdoctype.Position{{
- PageNumbers: []int{0},
- Left: 10,
- Right: 20,
- Top: 30,
- Bottom: 40,
- }},
- },
- {Text: "Footer", LayoutType: "footer"},
- },
- }
-
- res := pdfParseResultToJSONWithOptions("filtered.pdf", parsed, pdfPostProcessOptions{removeHeaderFooter: true})
- if res.Err != nil {
- t.Fatalf("pdfParseResultToJSONWithOptions: %v", res.Err)
- }
- if len(res.JSON) != 1 {
- t.Fatalf("JSON len = %d, want 1", len(res.JSON))
- }
- if got, want := res.JSON[0]["text"], "Body"; got != want {
- t.Fatalf("JSON[0].text = %v, want %v", got, want)
- }
-}
-
-func TestPDFParseResultToJSONWithOptions_RemovesTOCByOutline(t *testing.T) {
- parsed := &deepdoctype.ParseResult{
- Sections: []deepdoctype.Section{
- {
- Text: "Contents",
- LayoutType: "text",
- Positions: []deepdoctype.Position{{
- PageNumbers: []int{1},
- Left: 10,
- Right: 20,
- Top: 30,
- Bottom: 40,
- }},
- },
- {
- Text: "Body",
- LayoutType: "text",
- Positions: []deepdoctype.Position{{
- PageNumbers: []int{3},
- Left: 10,
- Right: 20,
- Top: 30,
- Bottom: 40,
- }},
- },
- },
- Outlines: []deepdoctype.Outline{
- {Title: "目录", Level: 0, PageNumber: 1},
- {Title: "Chapter 1", Level: 0, PageNumber: 3},
- },
- }
-
- res := pdfParseResultToJSONWithOptions("toc.pdf", parsed, pdfPostProcessOptions{removeTOC: true})
- if res.Err != nil {
- t.Fatalf("pdfParseResultToJSONWithOptions: %v", res.Err)
- }
- if len(res.JSON) != 1 {
- t.Fatalf("JSON len = %d, want 1", len(res.JSON))
- }
- if got, want := res.JSON[0]["text"], "Body"; got != want {
- t.Fatalf("JSON[0].text = %v, want %v", got, want)
- }
-}
-
func TestPDFParser_ConfigureFromSetup(t *testing.T) {
p := NewPDFParser()
p.ConfigureFromSetup(map[string]any{
diff --git a/internal/parser/parser/pdf_postprocess.go b/internal/parser/parser/pdf_postprocess.go
index da28424909..4ca4297b32 100644
--- a/internal/parser/parser/pdf_postprocess.go
+++ b/internal/parser/parser/pdf_postprocess.go
@@ -2,61 +2,23 @@ package parser
import (
"math"
- "regexp"
"sort"
"strings"
deepdoctype "ragflow/internal/deepdoc/parser/type"
)
-// Substring match to mirror Python's remove_header_footer:
-// re.search(r"(header|footer|number)", raw_layout, re.I) (rag/flow/parser/parser.py:754).
-// Python matches any layout type CONTAINING one of these words, not just the
-// exact token, so a composite label like "page-footer" is also stripped.
-var pdfHeaderFooterPattern = regexp.MustCompile(`(?i)header|footer|number`)
-var pdfTOCTitlePattern = regexp.MustCompile(`(?i)^(contents|目录|目次|table of contents|致谢|acknowledge)$`)
-
-// pdfTOCEntryPattern matches table-of-contents entry lines: a leader run
-// (dots, ellipses, midline ellipses, or middots) followed by a
-// page number, optionally with a trailing roman-numeral fragment, e.g.
-// "第1章 道可道…………3", "Introduction ....... 12", "……18 I". It extends the
-// leader heuristic Python already uses in remove_toc_word
-// (rag/flow/parser/utils.py) to the PDF path on both sides, so a TOC without
-// a "目录"/"contents" heading is still dropped. Wide-space runs are NOT a
-// leader here, mirroring pdfTOCEntryAnywherePattern: they are too common in
-// prose, so a line merely ending in " 12" is not a TOC entry.
-var pdfTOCEntryPattern = regexp.MustCompile(`(\.{2,}|…{2,}|⋯{2,}|·{2,})\s*\d{1,4}\s*[IVXLCivxlc]{0,5}\s*$`)
-
-// pdfTOCEntryAnywherePattern is the non-anchored form of pdfTOCEntryPattern.
-// DeepDoc merges the lines of a text block into ONE section whose text joins
-// the lines with spaces, so a TOC page usually becomes a single section whose
-// trailing text may be a page marker or a footer watermark instead of an
-// entry — the anchored pattern alone never fires for such a section. Counting
-// explicit leader runs (wide-space leaders excluded, they are too common in
-// prose) identifies those merged TOC blocks.
-var pdfTOCEntryAnywherePattern = regexp.MustCompile(`(\.{2,}|…{2,}|⋯{2,}|·{2,})\s*\d{1,4}\s*[IVXLCivxlc]{0,5}`)
-
-// pdfTOCBarePageRefPattern matches a line that consists solely of a leader
-// run plus a page number (and optional roman-numeral fragment), e.g.
-// "…………………39", "………18 I". In wrapped TOCs the entry title and its leader/page
-// number are split onto separate lines, leaving such a bare reference behind
-// the title line.
-var pdfTOCBarePageRefPattern = regexp.MustCompile(`^[\s.·…⋯]*\d{1,4}\s*[IVXLCivxlc]{0,5}[\s.。·…⋯]*$`)
-
-// pdfTOCTitlePrefixPattern matches lines that open like a TOC entry title:
-// a chapter/section marker or a common front/back-matter heading. Used only
-// to pair a title line with the bare leader+page-number line that follows it,
-// so regular prose is never dropped.
-var pdfTOCTitlePrefixPattern = regexp.MustCompile(`(?i)^(第\s*[0-90-9一二三四五六七八九十百千]+\s*[章节篇部册卷回讲]|chapter\s+[0-9]+|appendix\s+[a-z0-9]+|section\s+[0-9]+|前言|序言|引言|导论|绪论|后记|结语|附录|索引|参考文献|结论|致谢|acknowledgements?|acknowledgments?)`)
-
+// pdfPostProcessOptions carries the parser backend's post-process switches.
+// TOC and header/footer removal are handled earlier, at box level inside the
+// DeepDoc layout pipeline (Parser.buildLayout), where the leader-dot and
+// per-box geometry signals those detectors rely on are still intact; they are
+// intentionally not repeated here.
type pdfPostProcessOptions struct {
outputFormat string
pageWidth float64
zoom float64
enableMultiColumn bool
flattenMediaToText bool
- removeTOC bool
- removeHeaderFooter bool
}
func applyPDFPostProcess(result *deepdoctype.ParseResult, opts pdfPostProcessOptions) {
@@ -67,14 +29,7 @@ func applyPDFPostProcess(result *deepdoctype.ParseResult, opts pdfPostProcessOpt
if opts.enableMultiColumn && opts.pageWidth > 0 {
reorderPDFMultiColumn(result, opts.pageWidth, opts.zoom)
}
- if opts.removeTOC {
- applyRemoveTOC(result)
- }
normalizePDFLayoutTypes(result)
- if opts.removeHeaderFooter {
- filterPDFHeaderFooter(result)
- removePDFRunningHeaderFooter(result)
- }
assignPDFDocTypeKeywords(result, opts.flattenMediaToText)
}
@@ -88,128 +43,6 @@ func normalizePDFLayoutTypes(result *deepdoctype.ParseResult) {
}
}
-func filterPDFHeaderFooter(result *deepdoctype.ParseResult) {
- filtered := result.Sections[:0]
- for _, s := range result.Sections {
- if pdfHeaderFooterPattern.MatchString(strings.TrimSpace(s.LayoutType)) {
- continue
- }
- filtered = append(filtered, s)
- }
- result.Sections = filtered
-}
-
-const (
- // pdfHeaderZoneRatio / pdfFooterZoneRatio bound the page zones where
- // running headers/footers live: sections sitting entirely in the top
- // or bottom fraction of a page are candidates.
- pdfHeaderZoneRatio = 0.10
- pdfFooterZoneRatio = 0.90
- // pdfMinPagesForHeaderFooter guards against false positives on short
- // documents: cross-page repetition only means "running header/footer"
- // when the document has enough pages.
- pdfMinPagesForHeaderFooter = 3
-)
-
-var pdfRunningDigitPattern = regexp.MustCompile(`\d+`)
-
-// normalizePDFRunningText collapses whitespace and replaces digit runs so
-// that per-page variants of the same running header/footer ("- 2 -",
-// "- 3 -", "Page 4 of 9") share one comparison key.
-func normalizePDFRunningText(text string) string {
- t := strings.Join(strings.Fields(text), " ")
- t = pdfRunningDigitPattern.ReplaceAllString(t, "#")
- return strings.ToLower(strings.TrimSpace(t))
-}
-
-// removePDFRunningHeaderFooter drops sections whose normalized text repeats
-// in the same page zone (top 10% or bottom 10%) on at least half of the
-// document's pages — running headers, footers, and page numbers. Every
-// rendered page counts toward the total, including blank or image-only
-// pages that produced no sections. The
-// layout-type filter above only works when a DeepDoc inference service
-// (DEEPDOC_URL) types header/footer regions; in default deployments there
-// is none, every section stays "text", and 页眉页脚 removal silently does
-// nothing. This positional frequency pass makes the option work without an
-// inference service. Repetition — not position alone — is the guard that
-// keeps genuine body text: a section must recur on >= max(2, pages/2) pages
-// in the same zone to be removed.
-func removePDFRunningHeaderFooter(result *deepdoctype.ParseResult) {
- sections := result.Sections
- if len(sections) == 0 || len(result.PageHeight) == 0 {
- return
- }
- numPages := len(result.PageHeight)
- if numPages < pdfMinPagesForHeaderFooter {
- return
- }
- type zoneKey struct {
- header bool
- text string
- }
- keyPages := make(map[zoneKey]map[int]struct{})
- keySections := make(map[zoneKey][]int)
- for i := range sections {
- s := &sections[i]
- if layout := strings.TrimSpace(s.LayoutType); layout != "" && layout != deepdoctype.LayoutTypeText {
- continue
- }
- if len(s.Positions) == 0 || len(s.Positions[0].PageNumbers) == 0 {
- continue
- }
- page := s.Positions[0].PageNumbers[0]
- pageHeight := result.PageHeight[page]
- if pageHeight <= 0 {
- continue
- }
- top, bottom := s.Positions[0].Top, s.Positions[0].Bottom
- if bottom <= top {
- continue
- }
- var isHeader, isFooter bool
- switch {
- case bottom <= pageHeight*pdfHeaderZoneRatio:
- isHeader = true
- case top >= pageHeight*pdfFooterZoneRatio:
- isFooter = true
- }
- if !isHeader && !isFooter {
- continue
- }
- norm := normalizePDFRunningText(s.Text)
- if norm == "" {
- continue
- }
- key := zoneKey{header: isHeader, text: norm}
- if keyPages[key] == nil {
- keyPages[key] = make(map[int]struct{})
- }
- keyPages[key][page] = struct{}{}
- keySections[key] = append(keySections[key], i)
- }
- drop := make(map[int]struct{})
- for key, pages := range keyPages {
- n := len(pages)
- if n < 2 || n*2 < numPages {
- continue
- }
- for _, i := range keySections[key] {
- drop[i] = struct{}{}
- }
- }
- if len(drop) == 0 {
- return
- }
- filtered := sections[:0]
- for i, s := range sections {
- if _, ok := drop[i]; ok {
- continue
- }
- filtered = append(filtered, s)
- }
- result.Sections = filtered
-}
-
func assignPDFDocTypeKeywords(result *deepdoctype.ParseResult, flatten bool) {
for i := range result.Sections {
section := &result.Sections[i]
@@ -255,209 +88,6 @@ func sortSectionsByPosition(result *deepdoctype.ParseResult) {
})
}
-// applyRemoveTOC mirrors Python parser.py:663-681 three-way dispatch:
-// - No outlines → pattern-based remove_toc on all sections
-// - First outline on page 1 → outline-based remove_toc_pdf, then the entry
-// filter: the outline pass only drops pages when an outline title names
-// the TOC ("目录"/"contents"), so a headingless TOC must still run
-// through filterPDFTOCEntries instead of being left in place
-// - First outline after page 1 → pattern-based on pages before the first outline
-func applyRemoveTOC(result *deepdoctype.ParseResult) {
- if result == nil {
- return
- }
- outlines := result.Outlines
- if len(outlines) == 0 {
- removePDFTOC(result)
- return
- }
- firstOutlinePage := outlines[0].PageNumber
- if firstOutlinePage <= 1 {
- removePDFTOCByOutlines(result, outlines)
- result.Sections = filterPDFTOCEntries(result.Sections)
- return
- }
- splitAt := len(result.Sections)
- for i, s := range result.Sections {
- if firstSectionPage(s) >= firstOutlinePage {
- splitAt = i
- break
- }
- }
- beforeSplit := &deepdoctype.ParseResult{Sections: result.Sections[:splitAt]}
- removePDFTOC(beforeSplit)
- result.Sections = append(beforeSplit.Sections, result.Sections[splitAt:]...)
-}
-
-func removePDFTOC(result *deepdoctype.ParseResult) {
- sections := result.Sections
- i := 0
- for i < len(sections) {
- text := sectionText(sections[i])
- if !pdfTOCTitlePattern.MatchString(strings.ToLower(strings.TrimSpace(text))) {
- i++
- continue
- }
- sections = append(sections[:i], sections[i+1:]...)
- if i >= len(sections) {
- break
- }
- prefix := sectionTextPrefix(sections[i], 3)
- for prefix == "" {
- sections = append(sections[:i], sections[i+1:]...)
- if i >= len(sections) {
- break
- }
- prefix = sectionTextPrefix(sections[i], 3)
- }
- if i >= len(sections) || prefix == "" {
- break
- }
- // The first entry is formatted (leader run plus page number): the
- // table is entry-shaped and the prefix scan below must not run —
- // with a single formatted entry that scan would delete body text up
- // to the next line sharing the entry's prefix. Drop the entry and
- // resume the title scan; filterPDFTOCEntries removes the remaining
- // entry lines.
- if isPDFTOCEntrySection(sectionText(sections[i])) {
- sections = append(sections[:i], sections[i+1:]...)
- continue
- }
- sections = append(sections[:i], sections[i+1:]...)
- if i >= len(sections) || prefix == "" {
- break
- }
- for j := i; j < len(sections) && j < i+128; j++ {
- if !strings.HasPrefix(sectionText(sections[j]), prefix) {
- continue
- }
- sections = append(sections[:i], sections[j:]...)
- break
- }
- }
- // The anchored pass above needs a "目录"/"contents" section; a TOC whose
- // pages carry no such heading survives it entirely. Drop the remaining
- // leader+page-number entry lines, mirroring remove_toc in
- // rag/flow/parser/utils.py.
- sections = filterPDFTOCEntries(sections)
- result.Sections = sections
-}
-
-// filterPDFTOCEntries removes table-of-contents sections from the stream,
-// reusing the backing array. It goes beyond the per-item anchored match that
-// Python's remove_toc (rag/flow/parser/utils.py) applies, because the Go
-// DeepDoc layout merges a whole TOC block into one section and wrapped TOCs
-// split an entry's leader/page number onto its own line:
-//
-// - a section ending in a leader+page-number run is an entry (anchored
-// pattern), as before;
-// - a section containing two or more leader+page-number runs is a merged
-// TOC block even when its trailing text (page marker, roman numeral or
-// footer watermark) is not an entry itself;
-// - when a dropped section is a bare leader+page-number reference, the
-// preceding kept section is dropped too if it opens like a TOC entry
-// title ("第28章 ………" followed by "………39"), pairing the wrapped entry.
-//
-// Regular prose is kept: a lone leader+number run inside a paragraph matches
-// neither the anchored pattern nor the two-run threshold, and title pairing
-// only fires on a dropped bare page reference.
-func filterPDFTOCEntries(sections []deepdoctype.Section) []deepdoctype.Section {
- filtered := make([]deepdoctype.Section, 0, len(sections))
- for _, s := range sections {
- text := sectionText(s)
- if !isPDFTOCEntrySection(text) {
- filtered = append(filtered, s)
- continue
- }
- if pdfTOCBarePageRefPattern.MatchString(text) && len(filtered) > 0 {
- if isPDFTOCTitleCandidate(sectionText(filtered[len(filtered)-1])) {
- filtered = filtered[:len(filtered)-1]
- }
- }
- }
- return filtered
-}
-
-// isPDFTOCEntrySection reports whether the section text is (or is dominated
-// by) table-of-contents entries.
-func isPDFTOCEntrySection(text string) bool {
- if pdfTOCEntryPattern.MatchString(text) {
- return true
- }
- return len(pdfTOCEntryAnywherePattern.FindAllString(text, -1)) >= 2
-}
-
-// isPDFTOCTitleCandidate reports whether a section reads like the title line
-// of a TOC entry whose leader/page number lives on the following line: it
-// must open with a chapter/section marker and must not contain a leader run
-// itself.
-func isPDFTOCTitleCandidate(text string) bool {
- if !pdfTOCTitlePrefixPattern.MatchString(text) {
- return false
- }
- return !pdfTOCEntryAnywherePattern.MatchString(text)
-}
-
-func sectionText(s deepdoctype.Section) string {
- return strings.TrimSpace(s.Text)
-}
-
-func sectionTextPrefix(s deepdoctype.Section, n int) string {
- text := sectionText(s)
- if len(text) < n {
- return text
- }
- return text[:n]
-}
-func removePDFTOCByOutlines(result *deepdoctype.ParseResult, outlines []deepdoctype.Outline) {
- if result == nil || len(outlines) == 0 {
- return
- }
- tocPage, contentPage := findPDFTOCPageRange(outlines)
- if contentPage <= tocPage {
- return
- }
- filtered := result.Sections[:0]
- for _, s := range result.Sections {
- page := firstSectionPage(s)
- if page >= tocPage && page < contentPage {
- continue
- }
- filtered = append(filtered, s)
- }
- result.Sections = filtered
-}
-
-func findPDFTOCPageRange(outlines []deepdoctype.Outline) (tocPage, contentPage int) {
-outer:
- for i, o := range outlines {
- title := strings.TrimSpace(o.Title)
- if idx := strings.Index(title, "@@"); idx >= 0 {
- title = strings.TrimSpace(title[:idx])
- }
- if !pdfTOCTitlePattern.MatchString(strings.ToLower(title)) {
- continue
- }
- tocPage = o.PageNumber
- for _, next := range outlines[i+1:] {
- if next.Level != o.Level {
- continue
- }
- nextTitle := strings.TrimSpace(next.Title)
- if idx := strings.Index(nextTitle, "@@"); idx >= 0 {
- nextTitle = strings.TrimSpace(nextTitle[:idx])
- }
- if pdfTOCTitlePattern.MatchString(strings.ToLower(nextTitle)) {
- continue
- }
- contentPage = next.PageNumber
- break outer
- }
- break
- }
- return
-}
-
func reorderPDFMultiColumn(result *deepdoctype.ParseResult, pageWidth, _ float64) {
if result == nil || len(result.Sections) < 2 {
return
diff --git a/internal/parser/parser/pdf_postprocess_test.go b/internal/parser/parser/pdf_postprocess_test.go
index 0a62f4817c..bef11936a0 100644
--- a/internal/parser/parser/pdf_postprocess_test.go
+++ b/internal/parser/parser/pdf_postprocess_test.go
@@ -1,8 +1,6 @@
package parser
import (
- "slices"
- "strings"
"testing"
deepdoctype "ragflow/internal/deepdoc/parser/type"
@@ -79,510 +77,6 @@ func TestApplyPDFPostProcess_FlattenMediaKeepsImagesButMarksText(t *testing.T) {
}
}
-func TestApplyPDFPostProcess_HeaderFooterFilteringIsOptional(t *testing.T) {
- result := &deepdoctype.ParseResult{
- Sections: []deepdoctype.Section{
- {Text: "header", LayoutType: "header"},
- {Text: "body", LayoutType: "text"},
- },
- }
- applyPDFPostProcess(result, pdfPostProcessOptions{})
- if len(result.Sections) != 2 {
- t.Fatalf("len(Sections) = %d, want 2 when removeHeaderFooter is false", len(result.Sections))
- }
-
- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})
- if len(result.Sections) != 1 {
- t.Fatalf("len(Sections) = %d, want 1 when removeHeaderFooter is true", len(result.Sections))
- }
- if got, want := result.Sections[0].Text, "body"; got != want {
- t.Fatalf("remaining section = %q, want %q", got, want)
- }
-}
-
-func TestApplyPDFPostProcess_RemoveTOCRemovesAllDotLeaderEntries(t *testing.T) {
- result := &deepdoctype.ParseResult{
- Sections: []deepdoctype.Section{
- {Text: "Contents"},
- {Text: "Chapter One Introduction .............. 2"},
- {Text: "Chapter Two Methods ................... 3"},
- {Text: "Chapter Three Results ................. 4"},
- {Text: "Chapter One Introduction"},
- {Text: "Body text that must survive parsing."},
- {Text: "Chapter Two Methods"},
- {Text: "More body text."},
- },
- }
- applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})
- var kept []string
- for _, s := range result.Sections {
- kept = append(kept, s.Text)
- }
- want := []string{
- "Chapter One Introduction",
- "Body text that must survive parsing.",
- "Chapter Two Methods",
- "More body text.",
- }
- if len(kept) != len(want) {
- t.Fatalf("kept = %v, want %v", kept, want)
- }
- for i := range want {
- if kept[i] != want[i] {
- t.Fatalf("kept = %v, want %v", kept, want)
- }
- }
-}
-
-func TestApplyPDFPostProcess_RemoveTOCPrefixScanWithoutEntryPattern(t *testing.T) {
- // Pattern-less TOC entries take the prefix scan: title and first entry
- // go, then everything up to the first line sharing the first entry's
- // prefix (the real body heading).
- result := &deepdoctype.ParseResult{
- Sections: []deepdoctype.Section{
- {Text: "Contents"},
- {Text: "1. Introduction"},
- {Text: "2. Background"},
- {Text: "1. Introduction"},
- {Text: "Body text."},
- },
- }
- applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})
- var kept []string
- for _, s := range result.Sections {
- kept = append(kept, s.Text)
- }
- want := []string{"1. Introduction", "Body text."}
- if len(kept) != len(want) {
- t.Fatalf("kept = %v, want %v", kept, want)
- }
- for i := range want {
- if kept[i] != want[i] {
- t.Fatalf("kept = %v, want %v", kept, want)
- }
- }
-}
-
-func TestApplyPDFPostProcess_RemoveTOCSingleFormattedEntryKeepsBody(t *testing.T) {
- // Only the first entry after the TOC title is formatted. It must count
- // as a consumed entry so the prefix scan never runs: that scan would
- // otherwise delete every line up to the next "Cha"-prefixed heading
- // ("Chapter Two"), dropping the body text between the TOC and it.
- result := &deepdoctype.ParseResult{
- Sections: []deepdoctype.Section{
- {Text: "Contents"},
- {Text: "Chapter One ..... 2"},
- {Text: "Body A"},
- {Text: "Body B"},
- {Text: "Chapter Two"},
- },
- }
- applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})
- var kept []string
- for _, s := range result.Sections {
- kept = append(kept, s.Text)
- }
- want := []string{"Body A", "Body B", "Chapter Two"}
- if len(kept) != len(want) {
- t.Fatalf("kept = %v, want %v", kept, want)
- }
- for i := range want {
- if kept[i] != want[i] {
- t.Fatalf("kept = %v, want %v", kept, want)
- }
- }
-}
-
-func TestApplyPDFPostProcess_RemoveRunningHeaderFooterWithoutDLA(t *testing.T) {
- const pageHeight = 842.0
- result := &deepdoctype.ParseResult{
- PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight},
- Sections: []deepdoctype.Section{
- makePDFSection("Contents", "", 0, 72, 158, 40, 60),
- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 1, 72, 210, 21, 30),
- makePDFSection("Chapter One Introduction", "", 1, 72, 206, 88, 100),
- makePDFSection("Body text one.", "", 1, 72, 363, 124, 136),
- makePDFSection("- 2 -", "", 1, 289, 305, 807, 817),
- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 2, 72, 210, 21, 30),
- makePDFSection("Chapter Two Methods", "", 2, 72, 190, 88, 100),
- makePDFSection("Body text two.", "", 2, 72, 374, 124, 136),
- makePDFSection("- 3 -", "", 2, 289, 305, 807, 817),
- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 3, 72, 210, 21, 30),
- makePDFSection("Chapter Three Results", "", 3, 72, 192, 88, 100),
- makePDFSection("Body text three.", "", 3, 72, 367, 124, 136),
- makePDFSection("- 4 -", "", 3, 289, 305, 807, 817),
- },
- }
- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})
- for _, s := range result.Sections {
- if strings.Contains(s.Text, "CONFIDENTIAL") {
- t.Fatalf("running header survived: %q", s.Text)
- }
- if strings.HasPrefix(strings.TrimSpace(s.Text), "- ") {
- t.Fatalf("page-number footer survived: %q", s.Text)
- }
- }
- if len(result.Sections) != 7 {
- t.Fatalf("len(Sections) = %d, want 7 (body content preserved)", len(result.Sections))
- }
-}
-
-func TestApplyPDFPostProcess_RunningHeaderFooterKeepsUniqueZoneText(t *testing.T) {
- const pageHeight = 842.0
- result := &deepdoctype.ParseResult{
- PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight},
- Sections: []deepdoctype.Section{
- // Different top-of-page text on every page: not a running header.
- makePDFSection("Unique top line one", "", 0, 72, 210, 21, 30),
- makePDFSection("Body one.", "", 0, 72, 363, 124, 136),
- makePDFSection("Unique top line two", "", 1, 72, 210, 21, 30),
- makePDFSection("Body two.", "", 1, 72, 363, 124, 136),
- makePDFSection("Unique top line three", "", 2, 72, 210, 21, 30),
- makePDFSection("Body three.", "", 2, 72, 363, 124, 136),
- },
- }
- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})
- if len(result.Sections) != 6 {
- t.Fatalf("len(Sections) = %d, want 6 (unique zone text must survive)", len(result.Sections))
- }
-}
-
-func TestApplyPDFPostProcess_RunningHeaderFooterSkipsShortDocuments(t *testing.T) {
- const pageHeight = 842.0
- result := &deepdoctype.ParseResult{
- PageHeight: map[int]float64{0: pageHeight, 1: pageHeight},
- Sections: []deepdoctype.Section{
- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 0, 72, 210, 21, 30),
- makePDFSection("Body one.", "", 0, 72, 363, 124, 136),
- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 1, 72, 210, 21, 30),
- makePDFSection("Body two.", "", 1, 72, 363, 124, 136),
- },
- }
- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})
- if len(result.Sections) != 4 {
- t.Fatalf("len(Sections) = %d, want 4 (2-page doc must not be touched)", len(result.Sections))
- }
-}
-
-func TestApplyPDFPostProcess_RunningHeaderFooterCountsAllRenderedPages(t *testing.T) {
- const pageHeight = 842.0
- // Pages 3 and 4 are blank or image-only: they produced no sections but
- // still count toward the page total. A candidate repeated on 2 of the 5
- // rendered pages occurs on fewer than half of the document's pages and
- // must survive.
- result := &deepdoctype.ParseResult{
- PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight, 4: pageHeight},
- Sections: []deepdoctype.Section{
- makePDFSection("Chapter One Introduction", "", 0, 72, 206, 88, 100),
- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 1, 72, 210, 21, 30),
- makePDFSection("Body text one.", "", 1, 72, 363, 124, 136),
- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 2, 72, 210, 21, 30),
- makePDFSection("Body text two.", "", 2, 72, 374, 124, 136),
- },
- }
- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})
- if len(result.Sections) != 5 {
- t.Fatalf("len(Sections) = %d, want 5 (2 of 5 rendered pages is below the repetition threshold)", len(result.Sections))
- }
-
- // The same header on 3 of the 5 rendered pages meets the threshold and
- // is removed even though two pages contributed no sections.
- result = &deepdoctype.ParseResult{
- PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight, 4: pageHeight},
- Sections: []deepdoctype.Section{
- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 0, 72, 210, 21, 30),
- makePDFSection("Body text one.", "", 0, 72, 363, 124, 136),
- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 1, 72, 210, 21, 30),
- makePDFSection("Body text two.", "", 1, 72, 363, 124, 136),
- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 2, 72, 210, 21, 30),
- makePDFSection("Body text three.", "", 2, 72, 367, 124, 136),
- },
- }
- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})
- if len(result.Sections) != 3 {
- t.Fatalf("len(Sections) = %d, want 3 (running header removed on 3 of 5 rendered pages)", len(result.Sections))
- }
- for _, s := range result.Sections {
- if strings.Contains(s.Text, "CONFIDENTIAL") {
- t.Fatalf("running header survived: %q", s.Text)
- }
- }
-}
-
-func TestApplyPDFPostProcess_RunningHeaderFooterRequiresFullZone(t *testing.T) {
- const pageHeight = 842.0
- // A section that merely starts in the top 10% but extends past it
- // (bottom 200 > 842*0.10) is body content, not a header candidate —
- // even when it repeats on every page. Same for a section that ends in
- // the bottom 10% but starts above it (top 700 < 842*0.90).
- result := &deepdoctype.ParseResult{
- PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight},
- Sections: []deepdoctype.Section{
- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 0, 72, 210, 40, 200),
- makePDFSection("Long legal disclaimer note", "", 0, 72, 363, 700, 800),
- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 1, 72, 210, 40, 200),
- makePDFSection("Long legal disclaimer note", "", 1, 72, 363, 700, 800),
- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 2, 72, 210, 40, 200),
- makePDFSection("Long legal disclaimer note", "", 2, 72, 363, 700, 800),
- },
- }
- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})
- if len(result.Sections) != 6 {
- t.Fatalf("len(Sections) = %d, want 6 (sections crossing the zone boundary must survive)", len(result.Sections))
- }
-}
-
-func TestApplyPDFPostProcess_RunningHeaderFooterNeedsRepetitionNotPosition(t *testing.T) {
- const pageHeight = 842.0
- // Body blocks sitting fully inside the top 10% of every page but never
- // repeating: position alone must not remove them — the guard requires
- // the same normalized text on >= half of the rendered pages.
- result := &deepdoctype.ParseResult{
- PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight},
- Sections: []deepdoctype.Section{
- makePDFSection("Quarterly results summary", "", 0, 72, 300, 21, 60),
- makePDFSection("Ordinary body text on page zero.", "", 0, 72, 363, 124, 136),
- makePDFSection("Annual meeting minutes", "", 1, 72, 300, 21, 60),
- makePDFSection("Ordinary body text on page one.", "", 1, 72, 363, 124, 136),
- makePDFSection("Risk assessment overview", "", 2, 72, 300, 21, 60),
- makePDFSection("Ordinary body text on page two.", "", 2, 72, 363, 124, 136),
- makePDFSection("Compliance audit findings", "", 3, 72, 300, 21, 60),
- makePDFSection("Ordinary body text on page three.", "", 3, 72, 363, 124, 136),
- },
- }
- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})
- if len(result.Sections) != 8 {
- t.Fatalf("len(Sections) = %d, want 8 (in-zone but non-repeating text must survive)", len(result.Sections))
- }
-
- // Repetition counts distinct pages, not occurrences: the same text
- // twice on one page still covers a single page of the document.
- result = &deepdoctype.ParseResult{
- PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight},
- Sections: []deepdoctype.Section{
- makePDFSection("Repeated block", "", 0, 72, 300, 21, 40),
- makePDFSection("Repeated block", "", 0, 72, 300, 45, 64),
- makePDFSection("Body text page one.", "", 1, 72, 363, 124, 136),
- makePDFSection("Body text page two.", "", 2, 72, 363, 124, 136),
- },
- }
- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})
- if len(result.Sections) != 4 {
- t.Fatalf("len(Sections) = %d, want 4 (two occurrences on one page count as one page)", len(result.Sections))
- }
-}
-
-func TestApplyPDFPostProcess_RemoveTOCByOutlines(t *testing.T) {
- result := &deepdoctype.ParseResult{
- Sections: []deepdoctype.Section{
- makePDFSection("目录", "text", 1, 50, 550, 100, 120),
- makePDFSection("章节列表", "text", 2, 50, 550, 120, 140),
- makePDFSection("正文", "text", 3, 50, 550, 100, 120),
- },
- Outlines: []deepdoctype.Outline{
- {Title: "目录", Level: 0, PageNumber: 1},
- {Title: "第一章", Level: 0, PageNumber: 3},
- },
- }
- applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})
- if len(result.Sections) != 1 {
- t.Fatalf("len(Sections) = %d, want 1", len(result.Sections))
- }
- if got, want := result.Sections[0].Text, "正文"; got != want {
- t.Fatalf("remaining section = %q, want %q", got, want)
- }
-}
-
-// TestApplyPDFPostProcess_RemoveTOCEntryLines covers TOC pages that carry no
-// "目录"/"contents" heading: the removal must still drop the leader+page
-// number entry lines while keeping regular content, and only when removeTOC
-// is enabled.
-func TestApplyPDFPostProcess_RemoveTOCEntryLines(t *testing.T) {
- sections := []deepdoctype.Section{
- makePDFSection("《道德经》全文及翻译", "text", 1, 50, 550, 100, 120),
- makePDFSection("木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879", "text", 1, 50, 550, 120, 140),
- makePDFSection("前言:……………………………………………………………………1", "text", 2, 50, 550, 100, 120),
- makePDFSection("第1章 “道”…………………………………………………………3", "text", 2, 50, 550, 120, 140),
- makePDFSection("第2章 圣人居无为之事,行不言之教…………………………4", "text", 2, 50, 550, 140, 160),
- makePDFSection("第13章 以身为天下,可寄/托天下……………………………18 I", "text", 3, 50, 550, 100, 120),
- makePDFSection("第17章 太上,不知有之;功成事遂,百姓皆谓“我自然”..25", "text", 3, 50, 550, 120, 140),
- makePDFSection("Introduction to the Dao ....... 12", "text", 3, 50, 550, 140, 160),
- makePDFSection("II", "text", 3, 50, 550, 160, 180),
- makePDFSection("前言:本文是《道德经》的白话翻译,供读者参考。", "text", 4, 50, 550, 100, 120),
- makePDFSection("道可道,非常道。名可名,非常名。", "text", 4, 50, 550, 120, 140),
- }
-
- untouched := &deepdoctype.ParseResult{Sections: append([]deepdoctype.Section(nil), sections...)}
- applyPDFPostProcess(untouched, pdfPostProcessOptions{removeTOC: false})
- if len(untouched.Sections) != len(sections) {
- t.Fatalf("len(Sections) = %d, want %d when removeTOC is false", len(untouched.Sections), len(sections))
- }
-
- result := &deepdoctype.ParseResult{Sections: append([]deepdoctype.Section(nil), sections...)}
- applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})
- want := []string{
- "《道德经》全文及翻译",
- "木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879",
- "II",
- "前言:本文是《道德经》的白话翻译,供读者参考。",
- "道可道,非常道。名可名,非常名。",
- }
- if len(result.Sections) != len(want) {
- t.Fatalf("len(Sections) = %d, want %d; got %v", len(result.Sections), len(want), sectionTexts(result.Sections))
- }
- for i := range want {
- if got := result.Sections[i].Text; got != want[i] {
- t.Fatalf("Sections[%d].Text = %q, want %q", i, got, want[i])
- }
- }
-}
-
-// TestApplyPDFPostProcess_RemoveTOCEntryLinesWithPageOneOutlines covers the
-// book-PDF shape: the first outline sits on page 1 and none of its titles
-// names the TOC ("目录"/"contents"), so removePDFTOCByOutlines finds no page
-// range and is a no-op. The leader+page-number entry filter must still run on
-// this dispatch branch, otherwise a TOC without a 目录 heading survives it.
-func TestApplyPDFPostProcess_RemoveTOCEntryLinesWithPageOneOutlines(t *testing.T) {
- result := &deepdoctype.ParseResult{
- Sections: []deepdoctype.Section{
- makePDFSection("《道德经》全文及翻译", "text", 1, 50, 550, 100, 120),
- makePDFSection("前言:……………………………………………………………………1", "text", 2, 50, 550, 100, 120),
- makePDFSection("第1章 “道”…………………………………………………………3", "text", 2, 50, 550, 120, 140),
- makePDFSection("Introduction to the Dao ....... 12", "text", 2, 50, 550, 140, 160),
- makePDFSection("道可道,非常道。名可名,非常名。", "text", 3, 50, 550, 100, 120),
- },
- Outlines: []deepdoctype.Outline{
- {Title: "前言", Level: 0, PageNumber: 1},
- {Title: "Chapter 1", Level: 1, PageNumber: 3},
- },
- }
- applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})
- want := []string{"《道德经》全文及翻译", "道可道,非常道。名可名,非常名。"}
- if !slices.Equal(sectionTexts(result.Sections), want) {
- t.Fatalf("sections = %v, want %v", sectionTexts(result.Sections), want)
- }
-}
-
-func sectionTexts(sections []deepdoctype.Section) []string {
- texts := make([]string, 0, len(sections))
- for _, s := range sections {
- texts = append(texts, s.Text)
- }
- return texts
-}
-
-// TestFilterPDFTOCEntries_MergedBlockTrailingNonEntry covers a TOC page that
-// DeepDoc merges into ONE section whose trailing text is not an entry (a
-// footer watermark), so the anchored entry pattern alone never fires. The
-// block must still be dropped because it is dominated by leader+page-number
-// runs (issue: agent workflow PDF "remove original table of contents" not
-// taking effect).
-func TestFilterPDFTOCEntries_MergedBlockTrailingNonEntry(t *testing.T) {
- sections := []deepdoctype.Section{
- makePDFSection("《道德经》全文及翻译", "text", 1, 50, 550, 100, 120),
- makePDFSection("前言:………………………………………………1 第1章 “道”………………………………………………3 第2章 圣人居无为之事,行不言之教……………………4 第3章 无为而治…………………………………………6 木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879", "text", 1, 50, 550, 120, 300),
- makePDFSection("道可道,非常道。名可名,非常名。", "text", 3, 50, 550, 100, 120),
- }
- got := filterPDFTOCEntries(sections)
- want := []string{"《道德经》全文及翻译", "道可道,非常道。名可名,非常名。"}
- if !slices.Equal(sectionTexts(got), want) {
- t.Fatalf("sections = %v, want %v", sectionTexts(got), want)
- }
-}
-
-// TestFilterPDFTOCEntries_WrappedEntryTitlePairsPageRef covers wrapped TOC
-// entries whose leader+page number is split onto its own line/section: the
-// bare page reference drops the preceding title line, but a preceding
-// non-title line (prose, watermark) is kept, and a lone leader run inside
-// prose is not enough to drop anything.
-func TestFilterPDFTOCEntries_WrappedEntryTitlePairsPageRef(t *testing.T) {
- sections := []deepdoctype.Section{
- makePDFSection("第27章 不贵其师,不爱其资,虽智大迷,是谓要妙", "text", 2, 50, 550, 100, 120),
- makePDFSection("…………………………………………………………………………………………39", "text", 2, 50, 550, 120, 140),
- makePDFSection("木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879", "text", 2, 50, 550, 140, 160),
- makePDFSection("第28章 朴散则为器,圣人用之,则为官长,故大制不割??", "text", 2, 50, 550, 160, 180),
- makePDFSection("………39 II", "text", 2, 50, 550, 180, 200),
- makePDFSection("第1章", "text", 3, 50, 550, 100, 120),
- makePDFSection("道可道,非常道。名可名,非常名。", "text", 3, 50, 550, 120, 140),
- }
- got := filterPDFTOCEntries(sections)
- want := []string{
- "木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879",
- "第1章",
- "道可道,非常道。名可名,非常名。",
- }
- if !slices.Equal(sectionTexts(got), want) {
- t.Fatalf("sections = %v, want %v", sectionTexts(got), want)
- }
-}
-
-// TestFilterPDFTOCEntries_KeepsProseWithSingleLeaderRun pins the guard rails:
-// prose containing one leader+number mention stays, and a bare page
-// reference preceded by prose (no chapter/section title) keeps the prose.
-func TestFilterPDFTOCEntries_KeepsProseWithSingleLeaderRun(t *testing.T) {
- sections := []deepdoctype.Section{
- makePDFSection("更多说明见第三章……12 的表格。", "text", 1, 50, 550, 100, 120),
- makePDFSection("………34", "text", 1, 50, 550, 120, 140),
- makePDFSection("天下皆知美之为美,斯恶已。", "text", 2, 50, 550, 100, 120),
- }
- got := filterPDFTOCEntries(sections)
- want := []string{
- "更多说明见第三章……12 的表格。",
- "天下皆知美之为美,斯恶已。",
- }
- if !slices.Equal(sectionTexts(got), want) {
- t.Fatalf("sections = %v, want %v", sectionTexts(got), want)
- }
-}
-
-// TestFilterPDFTOCEntries_KeepsProseEndingInWideSpaces pins that a wide-space
-// run is not a TOC leader: prose ending in " 12" survives, while the same
-// shape with a dot leader is a real entry and is dropped.
-func TestFilterPDFTOCEntries_KeepsProseEndingInWideSpaces(t *testing.T) {
- sections := []deepdoctype.Section{
- makePDFSection("See the installation guide for details 12", "text", 1, 50, 550, 100, 120),
- makePDFSection("Troubleshooting ....... 27", "text", 2, 50, 550, 100, 120),
- }
- got := filterPDFTOCEntries(sections)
- want := []string{"See the installation guide for details 12"}
- if !slices.Equal(sectionTexts(got), want) {
- t.Fatalf("sections = %v, want %v", sectionTexts(got), want)
- }
-}
-
-// TestFilterPDFTOCEntries_TwoRunProseTradeOff pins the accepted trade-off of
-// the merged-block heuristic: prose quoting two leader+number references is
-// dropped, because a DeepDoc-merged TOC page has exactly that shape and the
-// anchored pattern alone never fires for it. A single run keeps the prose
-// (see KeepsProseWithSingleLeaderRun); raising the threshold would
-// under-delete real merged TOC blocks.
-func TestFilterPDFTOCEntries_TwoRunProseTradeOff(t *testing.T) {
- sections := []deepdoctype.Section{
- makePDFSection("见第三章……12 和第五章……34 的说明。", "text", 1, 50, 550, 100, 120),
- }
- got := filterPDFTOCEntries(sections)
- if len(got) != 0 {
- t.Fatalf("sections = %v, want empty: two leader+number runs read as a merged TOC block", sectionTexts(got))
- }
-}
-
-// TestFilterPDFTOCEntries_ChapterTitleWithoutBarePageRefSurvives pins the
-// guard of the wrapped-entry pairing: a "第N章"/"Chapter N" line is only
-// dropped when the following section is a bare leader+page-number reference
-// (see WrappedEntryTitlePairsPageRef); followed by ordinary prose it is body
-// content and must stay.
-func TestFilterPDFTOCEntries_ChapterTitleWithoutBarePageRefSurvives(t *testing.T) {
- sections := []deepdoctype.Section{
- makePDFSection("第1章 道可道,非常道", "text", 3, 50, 550, 100, 120),
- makePDFSection("道可道,非常道。名可名,非常名。", "text", 3, 50, 550, 120, 140),
- }
- got := filterPDFTOCEntries(sections)
- want := []string{"第1章 道可道,非常道", "道可道,非常道。名可名,非常名。"}
- if !slices.Equal(sectionTexts(got), want) {
- t.Fatalf("sections = %v, want %v", sectionTexts(got), want)
- }
-}
-
func TestApplyPDFPostProcess_ReordersMultiColumnText(t *testing.T) {
cases := []struct {
name string
@@ -605,38 +99,3 @@ func TestApplyPDFPostProcess_ReordersMultiColumnText(t *testing.T) {
}
}
}
-
-// TestFilterPDFHeaderFooter_SubstringMatch pins #5: Python's remove_header_footer
-// uses a substring match re.search(r"(header|footer|number)", ...) (rag/flow/parser/parser.py:754),
-// while Go used an anchored exact match ^(header|footer|number)$. A layout type
-// that merely CONTAINS one of those words (e.g. "page-footer") must be stripped to
-// match Python, not silently kept.
-func TestFilterPDFHeaderFooter_SubstringMatch(t *testing.T) {
- result := &deepdoctype.ParseResult{
- Sections: []deepdoctype.Section{
- {Text: "real header", LayoutType: "header"},
- {Text: "real footer", LayoutType: "footer"},
- {Text: "page 1", LayoutType: "number"},
- {Text: "a page-footer note", LayoutType: "page-footer"}, // composite -> substring match
- {Text: "body text", LayoutType: "text"},
- },
- }
- filterPDFHeaderFooter(result)
-
- kept := map[string]bool{}
- for _, s := range result.Sections {
- kept[s.LayoutType] = true
- }
- for _, lt := range []string{"header", "footer", "number"} {
- if kept[lt] {
- t.Errorf("#5 header/footer: %q should be stripped", lt)
- }
- }
- // Composite "page-footer" must be stripped by substring match (Python-equivalent).
- if kept["page-footer"] {
- t.Errorf("#5 header/footer: composite layout type %q should be stripped by substring match", "page-footer")
- }
- if !kept["text"] {
- t.Errorf("#5 header/footer: body text %q should be kept", "text")
- }
-}