RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): move PDF TOC and header/footer removal to box level (#19779)
jay777215f1a69ff0d98
diff --git a/internal/deepdoc/parser/pdf/layout/toc_header_footer.go b/internal/deepdoc/parser/pdf/layout/toc_header_footer.gonew file mode 100644index 0000000000..068ed1a4c0--- /dev/null+++ b/internal/deepdoc/parser/pdf/layout/toc_header_footer.go@@ -0,0 +1,450 @@+package layout++import (+ "regexp"+ "sort"+ "strings"+ "unicode/utf8"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// ---------------------------------------------------------------------------+// Tunable thresholds.+// ---------------------------------------------------------------------------++const (+ // tocMaxProseRunes is the per-box length (in runes) above which a box is+ // treated as a prose paragraph. A page with substantial body text has many+ // such boxes; a TOC page has at most a title or a long chapter heading.+ // This is the primary guard against deleting content: any page with more+ // than tocMaxLongBoxes long boxes is never classified as TOC.+ tocMaxProseRunes = 60+ // tocMaxLongBoxes is the maximum number of boxes longer than+ // tocMaxProseRunes a page may carry and still be classified as TOC.+ tocMaxLongBoxes = 2+ // tocMinShortBoxes is the minimum number of short boxes a TOC page carries.+ // A TOC is a list, so its entries usually stay separate boxes. A page whose+ // entries were merged into one long box is covered by tocMinMergedRuns, and+ // one merged without leader runs at all is covered by the outline signal.+ tocMinShortBoxes = 4+ // tocMinEntries is the minimum number of confirming entry markers (chapter+ // markers plus page numbers) a page must carry to be classified as TOC.+ tocMinEntries = 3+ // tocMinMergedRuns is the number of leader+page-number runs a single long+ // box must carry before that box is read as a whole TOC block rather than as+ // prose. A merged block confirms itself — three leader runs are three+ // entries — so it needs no separate marker count. Prose can accumulate the+ // occasional run, which is why the threshold is three and not one.+ tocMinMergedRuns = 3+ // tocMaxLeadPages bounds how many leading pages without body text may+ // precede the TOC. A cover, copyright page or frontispiece carries no body+ // text and is skipped, so a TOC that is not physically page one is still+ // reachable; the first page carrying body text ends the search.+ tocMaxLeadPages = 3+)++// ---------------------------------------------------------------------------+// Box-level patterns.+// ---------------------------------------------------------------------------++var (+ // chapterEntryPattern matches a box that opens like a TOC entry title: a+ // chapter/section/appendix marker (Chinese or Latin, digit or word form) or+ // a common front/back-matter heading.+ chapterEntryPattern = regexp.MustCompile(`(?i)^(第\s*[0-90-9一二三四五六七八九十百千]+\s*[章节篇部册卷回讲目]|chapter\s+(\d+|[a-z]+|one|two|three|four|five|six|seven|eight|nine|ten)|appendix\s+[a-z0-9]+|section\s+\d+|part\s+(\d+|[ivxlcdm]+|[a-z]+)|前言|序言|引言|导论|绪论|后记|结语|附录|索引|参考文献|结论|致谢|跋|序|凡例|目录|acknowledgements?|acknowledgments?)`)+ // pageNumberPattern matches common page-number forms: pure digits, digits+ // with leading dots (..18), roman numerals (III, IV), or dash-wrapped+ // numbers (- 2 -). Full-width digits count as digits, matching the class+ // chapterEntryPattern uses for 第N章 numbering: the target corpus is largely+ // CJK, and a full-width page-number column still has to confirm a TOC page.+ pageNumberPattern = regexp.MustCompile(`^[.…]*[0-90-9]+[.…]*$|^[.…]*[IVXLCDM]+[.…]*$|^\s*-\s*[0-90-9]+\s*-\s*$`)+ // tocStartPattern matches an outline title that names the table of contents,+ // which is the only thing that starts a TOC page range. A heading that is+ // merely adjacent to a TOC — acknowledgements, references — does not: the+ // section-level detector this pass replaces accepted 致谢 here, so a book+ // without a 目录 bookmark had its acknowledgements page claimed as a TOC.+ tocStartPattern = regexp.MustCompile(`(?i)^(contents|目录|目次|table of contents)$`)+ // tocEndMarkerPattern matches the front/back-matter headings that are+ // neither a TOC nor an entry, so they cannot bound a TOC range. The+ // acknowledgements family is spelled out the way chapterEntryPattern spells+ // it (singular and plural, both spellings), because the pattern this+ // replaces only matched the bare word "acknowledge" and so never fired on+ // the usual "Acknowledgements" bookmark.+ tocEndMarkerPattern = regexp.MustCompile(`(?i)^(致谢|acknowledg(?:e|ements?|ments?))$`)+ // tocEntryRunPattern matches one TOC entry's tail: a leader run (two or more+ // dots, ellipses or middots, never wide spaces — those are too common in+ // prose ending in a number), a page number and an optional roman-numeral+ // fragment. Counting these runs tells a TOC block that upstream merged into+ // one long box apart from a paragraph. Digits accept the full-width forms+ // pageNumberPattern accepts, for the same reason.+ tocEntryRunPattern = regexp.MustCompile(`(\.{2,}|…{2,}|⋯{2,}|·{2,})\s*[0-90-9]{1,4}\s*[IVXLCDMivxlcdm]{0,5}`)+)++// ---------------------------------------------------------------------------+// Outline signal.+// ---------------------------------------------------------------------------++// TOCPageRangeFromOutlines returns the 0-based page indices the PDF bookmarks+// identify as the table of contents: from the outline entry naming the TOC up+// to — excluding — the next entry at the same level. It returns nil when the+// document carries no usable bookmark.+//+// Only an entry that names the TOC itself starts a range (tocStartPattern).+// When a front/back-matter heading (tocEndMarkerPattern) is the next same-level+// entry the range is abandoned rather than closed, because such a heading does+// not say where the entries stop — it follows the TOC in some books and sits at+// the back of others — so the extent is left to the box shape signal instead of+// spanning the gap and deleting whatever lies inside it. Only the page the TOC+// starts on is claimed.+//+// Outline page numbers are 1-based — pdfium adds one to the 0-based destination+// index (internal/deepdoc/parser/pdf/pdfium/pdfium.go) — while+// TextBox.PageNumber is 0-based, because ParseRaw enumerates pages from zero.+// The conversion happens here, once, so no caller has to know about it.+func TOCPageRangeFromOutlines(outlines []pdf.Outline) map[int]bool {+ for i, o := range outlines {+ if !tocStartPattern.MatchString(outlineTitle(o.Title)) {+ continue+ }+ first := o.PageNumber - 1+ if first < 0 {+ return nil+ }+ for _, next := range outlines[i+1:] {+ if next.Level != o.Level {+ continue+ }+ if tocEndMarkerPattern.MatchString(outlineTitle(next.Title)) {+ break+ }+ pages := make(map[int]bool, 4)+ for pg := first; pg < next.PageNumber-1; pg++ {+ pages[pg] = true+ }+ if len(pages) == 0 {+ return nil+ }+ return pages+ }+ // No usable following entry at the same level: the TOC page is known+ // but its extent is not, so claim only that page and leave the rest to+ // the box shape signal.+ return map[int]bool{first: true}+ }+ return nil+}++// outlineTitle strips the "@@" anchor suffix pdfium appends to bookmark titles+// and lower-cases the result, matching how the section-level detector compared.+func outlineTitle(title string) string {+ title = strings.TrimSpace(title)+ if idx := strings.Index(title, "@@"); idx >= 0 {+ title = strings.TrimSpace(title[:idx])+ }+ return strings.ToLower(title)+}++// ---------------------------------------------------------------------------+// TOC removal.+// ---------------------------------------------------------------------------++// RemoveTOCBoxes drops whole pages that are detected as tables of contents.+//+// Detection works on signals that are destroyed by the later TextMerge /+// BoxesToSections passes (leader dots get folded into adjacent boxes, per-entry+// geometry collapses into a section), so this MUST run before TextMerge (see+// Parser.buildLayout).+//+// coversDocumentStart reports whether boxes cover the document's first page.+// The shape signal reads a TOC out of a leading run, so it is skipped when the+// parse began mid-document (Config.Pages): a TOC-shaped page at the start of a+// page range says nothing about the document, and reading it as a document+// prefix would delete content from the middle of the book. The outline signal+// carries absolute page numbers and is unaffected.+//+// Two signals select pages and the union is dropped:+//+// 1. outlinePages — the pages the PDF bookmarks identify as TOC (see+// TOCPageRangeFromOutlines). Strongest, and the only one that still sees a+// TOC whose entries were merged into one box with no leader runs left.+// 2. Box shape — the leading run of pages that read as a TOC list: a page+// carrying at least tocMinShortBoxes short boxes and tocMinEntries+// confirming markers (chapter markers or page numbers), or one long box+// that is itself tocMinMergedRuns leader+page-number runs, preceded only by+// pages without body text and spanning as many consecutive pages as keep+// satisfying that shape. Kept for documents that carry no usable bookmark,+// and only consulted when coversDocumentStart.+//+// Both signals pass through one guard: a page carrying body text (more than+// tocMaxLongBoxes boxes longer than tocMaxProseRunes) is never dropped, and a+// document consisting only of TOC pages is left untouched. The detector stays+// deliberately conservative — missing a TOC page costs noise chunks, deleting a+// content page loses text.+func RemoveTOCBoxes(boxes []pdf.TextBox, outlinePages map[int]bool, coversDocumentStart bool) []pdf.TextBox {+ if len(boxes) == 0 {+ return boxes+ }++ perPage := make(map[int][]int, 64)+ for i := range boxes {+ p := boxes[i].PageNumber+ perPage[p] = append(perPage[p], i)+ }++ pages := make([]int, 0, len(perPage))+ for pg := range perPage {+ pages = append(pages, pg)+ }+ sort.Ints(pages)++ shapes := make(map[int]pageShape, len(pages))+ for _, pg := range pages {+ shapes[pg] = shapeOfPage(boxes, perPage[pg])+ }++ selected := make(map[int]bool, len(pages))+ for pg := range outlinePages {+ if _, ok := perPage[pg]; ok {+ selected[pg] = true+ }+ }++ // Heuristic fallback, for documents that carry no usable bookmark. A TOC is+ // a document prefix, so only a parse that covers the document's first page+ // is eligible, and then only the leading pages are candidates: pages without+ // body text (a cover, copyright page) may be skipped, and the run continues+ // while pages keep satisfying isTOC(). It ends on the first page that does+ // not — in practice the first page carrying body text, which is what keeps+ // per-chapter pages that happen to hold several short headings (the Daodejing+ // case) out of scope. The run is deliberately unbounded in length: a real+ // TOC can span more pages than any fixed cap would allow, and truncating it+ // silently keeps the remaining TOC pages in the output. Length is bounded+ // instead by isTOC() itself and by the all-TOC guard below.+ if coversDocumentStart {+ inTOC, lead := false, 0+ for _, pg := range pages {+ if inTOC {+ if !shapes[pg].isTOC() {+ break+ }+ selected[pg] = true+ continue+ }+ if shapes[pg].isTOC() {+ inTOC = true+ selected[pg] = true+ continue+ }+ if shapes[pg].carriesProse() || lead >= tocMaxLeadPages {+ break+ }+ lead+++ }+ }++ drop := make(map[int]struct{}, len(boxes))+ droppedPages := 0+ for _, pg := range pages {+ if !selected[pg] || shapes[pg].carriesProse() {+ continue+ }+ droppedPages+++ for _, i := range perPage[pg] {+ drop[i] = struct{}{}+ }+ }+ if droppedPages == 0 || droppedPages == len(pages) {+ return boxes+ }++ out := make([]pdf.TextBox, 0, len(boxes)-len(drop))+ for i := range boxes {+ if _, ok := drop[i]; ok {+ continue+ }+ out = append(out, boxes[i])+ }+ return out+}++// pageShape summarises the boxes of one page.+type pageShape struct {+ prose int // boxes longer than tocMaxProseRunes+ short int+ markers int // short boxes opening with a chapter marker or holding a page number+ merged bool // a long box carrying tocMinMergedRuns leader+page-number runs+}++func shapeOfPage(boxes []pdf.TextBox, indices []int) pageShape {+ var s pageShape+ for _, i := range indices {+ text := strings.TrimSpace(boxes[i].Text)+ if text == "" {+ continue+ }+ if utf8.RuneCountInString(text) > tocMaxProseRunes {+ s.prose+++ // A merged TOC block is long by definition, so the run count is the+ // only evidence it can carry. Cap the search: the answer is a+ // three-way comparison, not the exact number of runs.+ if len(tocEntryRunPattern.FindAllStringIndex(text, tocMinMergedRuns)) >= tocMinMergedRuns {+ s.merged = true+ }+ continue+ }+ s.short+++ if chapterEntryPattern.MatchString(text) || pageNumberPattern.MatchString(text) {+ s.markers+++ }+ }+ return s+}++// isTOC reports whether the page reads as a table of contents: a list of short+// boxes carrying entry markers, or one long box that is itself a run of+// entries, and no body text either way.+func (s pageShape) isTOC() bool {+ if s.carriesProse() {+ return false+ }+ return s.merged || (s.short >= tocMinShortBoxes && s.markers >= tocMinEntries)+}++// carriesProse reports whether the page carries body text. No signal may drop+// such a page: this is the single guard that bounds every removal.+func (s pageShape) carriesProse() bool { return s.prose > tocMaxLongBoxes }++// ---------------------------------------------------------------------------+// Running header / footer removal.+// ---------------------------------------------------------------------------++const (+ // headerZoneRatio / footerZoneRatio bound the top/bottom page zones where a+ // running header / footer lives (fractions of the page height).+ headerZoneRatio = 0.10+ footerZoneRatio = 0.90+ // minHeaderFooterPages guards against false positives on short documents.+ minHeaderFooterPages = 3+)++// RemoveHeaderFooterBoxes drops boxes that are running headers or footers.+//+// A box is a candidate when it sits entirely in the top headerZoneRatio or+// bottom footerZoneRatio of its page and carries text-like content. Among+// candidates, a (zone, normalized-text) pair that recurs on at least half of+// the document's pages — rounded up, so an odd page count raises the bar rather+// than lowering it — is treated as a running header / footer and removed.+//+// Like RemoveTOCBoxes this works on intact box geometry and must run before+// TextMerge (which can fold a header box into the first body section).+func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) []pdf.TextBox {+ if len(pageHeights) < minHeaderFooterPages || len(boxes) == 0 {+ return boxes+ }++ type zoneKey struct {+ zone string+ text string+ }+ keyPages := make(map[zoneKey]map[int]struct{})+ keyIdx := make(map[zoneKey][]int)++ for i := range boxes {+ b := boxes[i]+ // Only plain-text boxes participate; tables / figures in the margin are+ // not running headers/footers.+ if lt := strings.TrimSpace(b.LayoutType); lt != "" && lt != "text" {+ continue+ }+ h, ok := pageHeights[b.PageNumber]+ if !ok || h <= 0 {+ continue+ }+ top, bottom := b.Top, b.Bottom+ var zone string+ switch {+ case bottom <= h*headerZoneRatio:+ zone = "header"+ case top >= h*footerZoneRatio:+ zone = "footer"+ default:+ continue+ }+ norm := normalizeRunningText(b.Text)+ if norm == "" {+ continue+ }+ key := zoneKey{zone: zone, text: norm}+ if keyPages[key] == nil {+ keyPages[key] = make(map[int]struct{})+ }+ keyPages[key][b.PageNumber] = struct{}{}+ keyIdx[key] = append(keyIdx[key], i)+ }++ numPages := len(pageHeights)+ // Half the pages, rounded up: rounding an odd page count down would let two+ // pages of a five-page document pass as "half", which is the one direction+ // this guard must not drift in.+ minPages := (numPages + 1) / 2+ if minPages < 2 {+ minPages = 2+ }++ drop := make(map[int]struct{})+ for key, pages := range keyPages {+ if len(pages) < minPages {+ continue+ }+ for _, i := range keyIdx[key] {+ drop[i] = struct{}{}+ }+ }+ if len(drop) == 0 {+ return boxes+ }++ out := make([]pdf.TextBox, 0, len(boxes)-len(drop))+ for i := range boxes {+ if _, ok := drop[i]; ok {+ continue+ }+ out = append(out, boxes[i])+ }+ return out+}++// normalizeRunningText collapses whitespace, replaces digit runs with a single+// '#' and lower-cases the result, so per-page variants of the same running+// header/footer ("- 2 -", "- 3 -", "Page 4 of 9", "第 1 页") share one+// comparison key.+func normalizeRunningText(text string) string {+ t := strings.Join(strings.Fields(text), " ")+ out := make([]rune, 0, len(t))+ inDigit := false+ for _, r := range t {+ if isMaskableDigit(r) {+ inDigit = true+ continue+ }+ if inDigit {+ out = append(out, '#')+ inDigit = false+ }+ out = append(out, r)+ }+ if inDigit {+ out = append(out, '#')+ }+ return strings.ToLower(strings.TrimSpace(string(out)))+}++// isMaskableDigit reports whether r is a digit whose per-page value must be+// masked before running headers/footers are compared: ASCII and full-width+// decimal digits. Full width is required for the same reason pageNumberPattern+// accepts it — otherwise every page of "第 1 页" / "第 2 页" gets its own key+// and the footer never reaches minPages.+func isMaskableDigit(r rune) bool {+ return (r >= '0' && r <= '9') || (r >= '0' && r <= '9')+}diff --git a/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go b/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.gonew file mode 100644index 0000000000..ed48433155--- /dev/null+++ b/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go@@ -0,0 +1,759 @@+package layout++import (+ "strings"+ "testing"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++func tb(text string, page int, x0, x1, top, bottom float64) pdf.TextBox {+ return pdf.TextBox{+ Text: text,+ PageNumber: page,+ X0: x0,+ X1: x1,+ Top: top,+ Bottom: bottom,+ }+}++func texts(boxes []pdf.TextBox) []string {+ out := make([]string, 0, len(boxes))+ for _, b := range boxes {+ out = append(out, b.Text)+ }+ return out+}++func equalText(a, b []string) bool {+ if len(a) != len(b) {+ return false+ }+ for i := range a {+ if a[i] != b[i] {+ return false+ }+ }+ return true+}++// TestRemoveTOCBoxes_DropsDottedTOCPage: a page made of leader-dot entries is+// removed entirely, while a body page on a different page is untouched.+func TestRemoveTOCBoxes_DropsDottedTOCPage(t *testing.T) {+ boxes := []pdf.TextBox{+ // page 0: TOC with leader dots in their own boxes.+ tb("Contents", 0, 72, 150, 100, 120),+ tb("Chapter One", 0, 72, 175, 160, 175),+ tb("..........................", 0, 175, 400, 160, 175),+ tb("3", 0, 500, 520, 160, 175),+ tb("Chapter Two", 0, 72, 175, 190, 205),+ tb("..........................", 0, 175, 400, 190, 205),+ tb("4", 0, 500, 520, 190, 205),+ tb("Chapter Three", 0, 72, 180, 220, 235),+ tb("..........................", 0, 175, 400, 220, 235),+ tb("5", 0, 500, 520, 220, 235),+ // page 1: normal body with multiple prose paragraphs.+ tb("The way that can be told of is not the eternal way; the name that can be named is not the eternal name. The nameless is the origin of heaven and earth.", 1, 72, 500, 160, 180),+ tb("The named is the mother of ten thousand things. Ever desireless, one can see the mystery. Ever desiring, one can see the manifestations.", 1, 72, 500, 190, 210),+ tb("These two spring from the same source but differ in name. This appears as darkness. Darkness within darkness. The gate to all mystery.", 1, 72, 500, 220, 240),+ }+ got := RemoveTOCBoxes(boxes, nil, true)+ // Only the body paragraphs on page 1 should survive.+ want := []string{+ "The way that can be told of is not the eternal way; the name that can be named is not the eternal name. The nameless is the origin of heaven and earth.",+ "The named is the mother of ten thousand things. Ever desireless, one can see the mystery. Ever desiring, one can see the manifestations.",+ "These two spring from the same source but differ in name. This appears as darkness. Darkness within darkness. The gate to all mystery.",+ }+ if !equalText(texts(got), want) {+ t.Fatalf("got %v, want %v", texts(got), want)+ }+}++// TestRemoveTOCBoxes_KeepsBodyPage: a page without leader dots is never removed.+func TestRemoveTOCBoxes_KeepsBodyPage(t *testing.T) {+ boxes := []pdf.TextBox{+ tb("Section 1.1 Overview", 0, 72, 200, 100, 120),+ tb("This paragraph explains the background of the work in some detail so the reader can follow.", 0, 72, 500, 130, 150),+ tb("It continues with more context and supporting material for the main argument.", 0, 72, 500, 160, 180),+ }+ got := RemoveTOCBoxes(boxes, nil, true)+ if len(got) != len(boxes) {+ t.Fatalf("body page should be untouched, got %d boxes, want %d", len(got), len(boxes))+ }+}++// TestRemoveTOCBoxes_ThresholdBoundary: 2 entries (below min) keeps the page,+// 3 entries removes it.+func TestRemoveTOCBoxes_ThresholdBoundary(t *testing.T) {+ mkPage := func(n int, page int) []pdf.TextBox {+ var boxes []pdf.TextBox+ boxes = append(boxes, tb("Contents", page, 72, 150, 100, 120))+ for i := 0; i < n; i++ {+ y := float64(160 + i*30)+ boxes = append(boxes,+ tb("Item", page, 72, 120, y, y+15),+ tb("..........................", page, 175, 400, y, y+15),+ tb(string(rune('0'+i+1)), page, 500, 520, y, y+15),+ )+ }+ return boxes+ }+ below := append(mkPage(2, 0),+ tb("Body text here that is long enough to matter and exceeds sixty runes for sure absolutely yes it does.", 1, 72, 400, 160, 180),+ tb("Another paragraph of body text that is also quite long and exceeds the thirty rune threshold easily.", 1, 72, 400, 190, 210),+ tb("A third paragraph of body text that ensures the book is not classified as compact by the detector.", 1, 72, 400, 220, 240),+ )+ if got := RemoveTOCBoxes(below, nil, true); len(got) != len(below) {+ t.Fatalf("2 entries should keep page: got %d boxes, want %d", len(got), len(below))+ }+ above := append(mkPage(3, 0),+ tb("Body text here that is long enough to matter and exceeds sixty runes for sure absolutely yes it does.", 1, 72, 400, 160, 180),+ tb("Another paragraph of body text that is also quite long and exceeds the thirty rune threshold easily.", 1, 72, 400, 190, 210),+ tb("A third paragraph of body text that ensures the book is not classified as compact by the detector.", 1, 72, 400, 220, 240),+ )+ if got := RemoveTOCBoxes(above, nil, true); len(got) != 3 {+ t.Fatalf("3 entries should drop the whole TOC page: got %d boxes, want 3", len(got))+ }+}++// TestRemoveTOCBoxes_LeaderMergedIntoBox: when the leader dots are merged into+// the same box as the title+page-number, the entry is still detected.+func TestRemoveTOCBoxes_LeaderMergedIntoBox(t *testing.T) {+ boxes := []pdf.TextBox{+ tb("Contents", 0, 72, 150, 100, 120),+ tb("Chapter One .......................... 3", 0, 72, 520, 160, 175),+ tb("Chapter Two .......................... 4", 0, 72, 520, 190, 205),+ tb("Chapter Three .......................... 5", 0, 72, 520, 220, 235),+ tb("Body paragraph that must survive on page 1 and is definitely longer than thirty runes.", 1, 72, 400, 160, 180),+ tb("Another body paragraph that ensures the book has enough long boxes to not be compact.", 1, 72, 400, 190, 210),+ }+ got := RemoveTOCBoxes(boxes, nil, true)+ want := []string{+ "Body paragraph that must survive on page 1 and is definitely longer than thirty runes.",+ "Another body paragraph that ensures the book has enough long boxes to not be compact.",+ }+ if !equalText(texts(got), want) {+ t.Fatalf("got %v, want %v", texts(got), want)+ }+}++// TestRemoveTOCBoxes_ChineseChapterMarkers: Chinese "第N章" style TOC.+func TestRemoveTOCBoxes_ChineseChapterMarkers(t *testing.T) {+ boxes := []pdf.TextBox{+ tb("目录", 0, 72, 110, 100, 120),+ tb("第一章 道可道", 0, 72, 200, 160, 175),+ tb("..........................", 0, 200, 420, 160, 175),+ tb("1", 0, 500, 520, 160, 175),+ tb("第二章 天下皆知", 0, 72, 210, 190, 205),+ tb("..........................", 0, 200, 420, 190, 205),+ tb("5", 0, 500, 520, 190, 205),+ tb("第三章 不尚贤", 0, 72, 200, 220, 235),+ tb("..........................", 0, 200, 420, 220, 235),+ tb("9", 0, 500, 520, 220, 235),+ tb("正文内容,道可道,非常道。名可名,非常名。无名天地之始,有名万物之母。故常无欲以观其妙,常有欲以观其徼。此两者同出而异名,同谓之玄。玄之又玄,众妙之门。", 1, 72, 500, 160, 180),+ tb("天下皆知美之为美,斯恶已。皆知善之为善,斯不善已。故有无相生,难易相成,长短相形,高下相倾,音声相和,前后相随。", 1, 72, 500, 190, 210),+ tb("是以圣人处无为之事,行不言之教。万物作焉而不辞,生而不有,为而不恃,功成而弗居。夫唯弗居,是以不去。", 1, 72, 500, 220, 240),+ }+ got := RemoveTOCBoxes(boxes, nil, true)+ want := []string{+ "正文内容,道可道,非常道。名可名,非常名。无名天地之始,有名万物之母。故常无欲以观其妙,常有欲以观其徼。此两者同出而异名,同谓之玄。玄之又玄,众妙之门。",+ "天下皆知美之为美,斯恶已。皆知善之为善,斯不善已。故有无相生,难易相成,长短相形,高下相倾,音声相和,前后相随。",+ "是以圣人处无为之事,行不言之教。万物作焉而不辞,生而不有,为而不恃,功成而弗居。夫唯弗居,是以不去。",+ }+ if !equalText(texts(got), want) {+ t.Fatalf("got %v, want %v", texts(got), want)+ }+}++// TestRemoveTOCBoxes_BodyPageWithLongBoxIsNotTOC: a page with long prose boxes+// is never mistaken for a TOC even if it happens to contain a short dot line.+func TestRemoveTOCBoxes_BodyPageWithLongBoxIsNotTOC(t *testing.T) {+ boxes := []pdf.TextBox{+ tb("This is a long body paragraph that clearly belongs to the main content of the document and should never be classified as a table of contents entry under any reasonable heuristic.", 0, 72, 500, 100, 120),+ tb("Another sentence here to make the point clear and ensure the page is recognized as a body page.", 0, 72, 500, 130, 150),+ }+ got := RemoveTOCBoxes(boxes, nil, true)+ if len(got) != len(boxes) {+ t.Fatalf("body page with long boxes must be kept, got %d, want %d", len(got), len(boxes))+ }+}++// tocPageBoxes builds one page of dotted TOC entries: a heading plus n entries,+// each entry as its own box, which is the shape the box signal expects.+func tocPageBoxes(pg, n int) []pdf.TextBox {+ out := []pdf.TextBox{tb("目录", pg, 72, 110, 100, 120)}+ for i := 0; i < n; i++ {+ y := float64(160 + i*30)+ out = append(out,+ tb("第一章 道可道", pg, 72, 200, y, y+15),+ tb("..........................", pg, 200, 420, y, y+15),+ tb(string(rune('0'+i+1)), pg, 500, 520, y, y+15),+ )+ }+ return out+}++// bodyPageBoxes builds one page of prose, which no signal may drop.+func bodyPageBoxes(pg int) []pdf.TextBox {+ return []pdf.TextBox{+ tb("The way that can be told of is not the eternal way and the name that can be named is not the eternal name.", pg, 72, 500, 160, 180),+ tb("The named is the mother of ten thousand things and ever desireless one can see the mystery of it all here.", pg, 72, 500, 190, 210),+ tb("These two spring from the same source but differ in name and this appears as darkness within darkness.", pg, 72, 500, 220, 240),+ }+}++func countPage(boxes []pdf.TextBox, pg int) int {+ n := 0+ for _, b := range boxes {+ if b.PageNumber == pg {+ n+++ }+ }+ return n+}++// TestTOCPageRangeFromOutlines pins the 1-based to 0-based conversion: pdfium+// reports outline pages 1-based while TextBox.PageNumber is 0-based, and the+// section-level detector this replaces compared the two directly.+func TestTOCPageRangeFromOutlines(t *testing.T) {+ toc := TOCPageRangeFromOutlines([]pdf.Outline{+ {Title: "目录", Level: 0, PageNumber: 1},+ {Title: "第一章", Level: 0, PageNumber: 4},+ })+ if len(toc) != 3 || !toc[0] || !toc[1] || !toc[2] || toc[3] {+ t.Fatalf("outline pages 1-3 must map to 0-based {0,1,2} and stop before the content page, got %v", toc)+ }+ if got := TOCPageRangeFromOutlines(nil); got != nil {+ t.Fatalf("no outlines must yield nil, got %v", got)+ }+ if got := TOCPageRangeFromOutlines([]pdf.Outline{{Title: "第一章", Level: 0, PageNumber: 2}}); got != nil {+ t.Fatalf("a first bookmark that is a chapter names no TOC page, got %v", got)+ }+ // No following entry at the same level: only the TOC page itself is known.+ if got := TOCPageRangeFromOutlines([]pdf.Outline{{Title: "Contents", Level: 0, PageNumber: 2}}); len(got) != 1 || !got[1] {+ t.Fatalf("a lone TOC bookmark should claim printed page 2 only, got %v", got)+ }+ // The "@@" anchor suffix pdfium appends must not defeat the title match.+ if got := TOCPageRangeFromOutlines([]pdf.Outline{+ {Title: "Contents@@5", Level: 0, PageNumber: 1},+ {Title: "Chapter One", Level: 0, PageNumber: 2},+ }); len(got) != 1 || !got[0] {+ t.Fatalf("an anchored TOC title should claim 0-based page 0, got %v", got)+ }+}++// TestTOCPageRangeFromOutlines_AcknowledgementsDoesNotStart: only an entry that+// names the TOC starts a range. A book with an acknowledgements bookmark and no+// contents bookmark used to have its acknowledgements page claimed and dropped.+func TestTOCPageRangeFromOutlines_AcknowledgementsDoesNotStart(t *testing.T) {+ cases := []struct {+ name string+ in []pdf.Outline+ }{+ {"chinese, followed by back matter", []pdf.Outline{+ {Title: "致谢", Level: 0, PageNumber: 5},+ {Title: "参考文献", Level: 0, PageNumber: 8},+ }},+ {"chinese, alone", []pdf.Outline{+ {Title: "致谢", Level: 0, PageNumber: 5},+ }},+ {"english plural", []pdf.Outline{+ {Title: "Acknowledgements", Level: 0, PageNumber: 5},+ {Title: "References", Level: 0, PageNumber: 8},+ }},+ }+ for _, c := range cases {+ if got := TOCPageRangeFromOutlines(c.in); got != nil {+ t.Errorf("%s: an acknowledgements bookmark must not open a TOC range, got %v", c.name, got)+ }+ }+}++// TestTOCPageRangeFromOutlines_EndMarkerAbandonsRange: a front/back-matter+// heading sitting next to the TOC does not say where the entries stop, so the+// range is abandoned rather than extended to whatever entry follows it.+func TestTOCPageRangeFromOutlines_EndMarkerAbandonsRange(t *testing.T) {+ // Before: 致谢 was skipped and the range ran to 参考文献, claiming printed+ // pages 2-11 including the acknowledgements themselves.+ if got := TOCPageRangeFromOutlines([]pdf.Outline{+ {Title: "目录", Level: 0, PageNumber: 2},+ {Title: "致谢", Level: 0, PageNumber: 10},+ {Title: "参考文献", Level: 0, PageNumber: 12},+ }); len(got) != 1 || !got[1] {+ t.Fatalf("the acknowledgements entry must abandon the range, claiming printed page 2 only, got %v", got)+ }+ // The English heading now matches at all; before, it was treated as an+ // ordinary boundary and claimed printed pages 2-9.+ if got := TOCPageRangeFromOutlines([]pdf.Outline{+ {Title: "Contents", Level: 0, PageNumber: 2},+ {Title: "Acknowledgements", Level: 0, PageNumber: 10},+ }); len(got) != 1 || !got[1] {+ t.Fatalf("the english end marker must abandon the range too, got %v", got)+ }+ // A chapter entry is still a boundary: the range stops just before it.+ if got := TOCPageRangeFromOutlines([]pdf.Outline{+ {Title: "目录", Level: 0, PageNumber: 2},+ {Title: "第一章", Level: 0, PageNumber: 5},+ {Title: "致谢", Level: 0, PageNumber: 20},+ }); len(got) != 3 || !got[1] || !got[2] || !got[3] || got[4] {+ t.Fatalf("a chapter entry must still close the range at printed page 5, got %v", got)+ }+}++// TestRemoveTOCBoxes_OutlineCoversLeaderlessMergedBlock: a TOC whose entries+// were merged into one box with no leader runs left is invisible to the box+// signal — neither short boxes nor leader runs survive — and is covered by the+// outline signal alone.+func TestRemoveTOCBoxes_OutlineCoversLeaderlessMergedBlock(t *testing.T) {+ merged := "目录 第一章 道可道非常道名可名非常名 3 第二章 天下皆知美之为美斯恶已皆知善 7 第三章 不尚贤使民不争不贵难得之货 15 第四章 道冲而用之或不盈渊兮似万物之宗 21"+ boxes := []pdf.TextBox{+ tb("目录", 0, 72, 110, 100, 120),+ tb(merged, 0, 72, 520, 160, 200),+ }+ boxes = append(boxes, bodyPageBoxes(1)...)+ boxes = append(boxes, bodyPageBoxes(2)...)++ if got := RemoveTOCBoxes(boxes, nil, true); len(got) != len(boxes) {+ t.Fatalf("box shape alone cannot see a leaderless merged TOC block, got %d of %d", len(got), len(boxes))+ }+ got := RemoveTOCBoxes(boxes, map[int]bool{0: true}, true)+ if n := countPage(got, 0); n != 0 {+ t.Fatalf("an outline-selected page must be dropped, %d boxes survived", n)+ }+}++// TestRemoveTOCBoxes_MergedBlockWithLeaders: DeepDoc joins the lines of a text+// block into one box, so a TOC page can arrive as a single long box. Its leader+// runs are the only evidence left, and the page must still be dropped without+// any bookmark.+func TestRemoveTOCBoxes_MergedBlockWithLeaders(t *testing.T) {+ merged := "目录 第一章 道可道非常道名可名非常名 ..........3 第二章 天下皆知美之为美斯恶已皆知善 ..........7 第三章 不尚贤使民不争不贵难得之货 ..........15"+ boxes := []pdf.TextBox{+ tb("目录", 0, 72, 110, 100, 120),+ tb(merged, 0, 72, 520, 160, 200),+ }+ boxes = append(boxes, bodyPageBoxes(1)...)+ boxes = append(boxes, bodyPageBoxes(2)...)++ got := RemoveTOCBoxes(boxes, nil, true)+ if n := countPage(got, 0); n != 0 {+ t.Fatalf("a merged TOC block carrying leader runs must be dropped, %d boxes survived", n)+ }+ if n := countPage(got, 1); n != 3 {+ t.Fatalf("the body page must be kept, %d of 3 boxes", n)+ }+}++// TestRemoveTOCBoxes_MergedBlockNeedsThreeRuns: two leader runs in a long box+// are not enough. A paragraph can pick up the odd "……12", so the threshold sits+// above that rather than at the first run.+func TestRemoveTOCBoxes_MergedBlockNeedsThreeRuns(t *testing.T) {+ merged := "目录 第一章 道可道非常道名可名非常名 ..........3 第二章 天下皆知美之为美斯恶已皆知善 ..........7 第三章 不尚贤使民不争不贵难得之货"+ boxes := []pdf.TextBox{+ tb("目录", 0, 72, 110, 100, 120),+ tb(merged, 0, 72, 520, 160, 200),+ }+ boxes = append(boxes, bodyPageBoxes(1)...)+ boxes = append(boxes, bodyPageBoxes(2)...)++ if got := RemoveTOCBoxes(boxes, nil, true); len(got) != len(boxes) {+ t.Fatalf("two leader runs are below the merged-block threshold, got %d of %d", len(got), len(boxes))+ }+}++// TestRemoveTOCBoxes_MergedBlockFullWidthPageNumbers: the leader-run count uses+// the same digit class as pageNumberPattern, so a CJK TOC numbering its entries+// in full-width digits is still recognised.+func TestRemoveTOCBoxes_MergedBlockFullWidthPageNumbers(t *testing.T) {+ merged := "目录 第一章 道可道非常道名可名非常名 …………3 第二章 天下皆知美之为美斯恶已皆知善 …………7 第三章 不尚贤使民不争不贵难得之货 …………15"+ boxes := []pdf.TextBox{+ tb("目录", 0, 72, 110, 100, 120),+ tb(merged, 0, 72, 520, 160, 200),+ }+ boxes = append(boxes, bodyPageBoxes(1)...)+ boxes = append(boxes, bodyPageBoxes(2)...)++ got := RemoveTOCBoxes(boxes, nil, true)+ if n := countPage(got, 0); n != 0 {+ t.Fatalf("full-width page numbers in a merged TOC block must be counted, %d boxes survived", n)+ }+}++// TestRemoveTOCBoxes_MergedBlockRespectsProseGuard: a page that carries body+// text is kept even when one of its boxes is a merged TOC block. The merged+// signal is the one that could plausibly be talked into deleting a prose page,+// so the guard is asserted against it rather than against the short-box shape.+func TestRemoveTOCBoxes_MergedBlockRespectsProseGuard(t *testing.T) {+ merged := "目录 第一章 道可道 ..........3 第二章 天下皆知美之为美 ..........7 第三章 不尚贤使民不争 ..........15"+ boxes := []pdf.TextBox{+ tb(merged, 0, 72, 520, 100, 140),+ tb("This is a long body paragraph that certainly belongs to the main content of the document and would be lost.", 0, 72, 520, 160, 180),+ tb("This is a second long body paragraph that certainly belongs to the main content of the document as well.", 0, 72, 520, 190, 210),+ tb("This is a third long body paragraph that certainly belongs to the main content of the document too.", 0, 72, 520, 220, 240),+ }+ boxes = append(boxes, bodyPageBoxes(1)...)+ boxes = append(boxes, bodyPageBoxes(2)...)++ got := RemoveTOCBoxes(boxes, nil, true)+ if n := countPage(got, 0); n != 4 {+ t.Fatalf("a page carrying prose must never be dropped, %d of 4 boxes survived", n)+ }+}++// TestRemoveTOCBoxes_OutlineRespectsProseGuard: a stale or malformed bookmark+// that names a content page must not delete body text, while the box signal+// still drops the real TOC page.+func TestRemoveTOCBoxes_OutlineRespectsProseGuard(t *testing.T) {+ boxes := tocPageBoxes(0, 3)+ boxes = append(boxes, bodyPageBoxes(1)...)+ boxes = append(boxes, bodyPageBoxes(2)...)++ got := RemoveTOCBoxes(boxes, map[int]bool{1: true}, true)+ if n := countPage(got, 1); n != 3 {+ t.Fatalf("a page carrying prose must never be dropped, %d of 3 boxes survived", n)+ }+ if n := countPage(got, 0); n != 0 {+ t.Fatalf("the box signal must still drop the TOC page, %d boxes survived", n)+ }+}++// TestRemoveTOCBoxes_KeepsTOCOnlyDocument: a source file that is nothing but a+// TOC page keeps it rather than being emptied.+func TestRemoveTOCBoxes_KeepsTOCOnlyDocument(t *testing.T) {+ boxes := tocPageBoxes(0, 3)+ if got := RemoveTOCBoxes(boxes, nil, true); len(got) != len(boxes) {+ t.Fatalf("a document consisting only of a TOC must be left untouched, got %d of %d", len(got), len(boxes))+ }+}++// TestRemoveTOCBoxes_MidDocumentParseSkipsShapeSignal: when the parse covers a+// later page range, the first parsed page is not a document prefix. Reading it+// as one deleted content from the middle of the book.+func TestRemoveTOCBoxes_MidDocumentParseSkipsShapeSignal(t *testing.T) {+ var boxes []pdf.TextBox+ boxes = append(boxes, tocPageBoxes(10, 3)...)+ boxes = append(boxes, bodyPageBoxes(11)...)++ if got := RemoveTOCBoxes(boxes, nil, false); len(got) != len(boxes) {+ t.Fatalf("a page range starting mid-document must not be read as a TOC prefix, dropped %d of %d",+ len(boxes)-len(got), len(boxes))+ }+ if got := RemoveTOCBoxes(boxes, nil, true); len(got) == len(boxes) {+ t.Fatalf("the same boxes as a document prefix must still drop the TOC page")+ }+ // The outline signal carries absolute page numbers, so the gate must not+ // silence it.+ got := RemoveTOCBoxes(boxes, map[int]bool{10: true}, false)+ if n := countPage(got, 10); n != 0 {+ t.Fatalf("an outline-selected page must be dropped even mid-document, %d boxes survived", n)+ }+ if n := countPage(got, 11); n != 3 {+ t.Fatalf("the body page must survive, %d of 3 boxes", n)+ }+}++// TestRemoveTOCBoxes_MultiPageTOC: one TOC spanning consecutive pages is dropped+// as a whole, not just its first page.+func TestRemoveTOCBoxes_MultiPageTOC(t *testing.T) {+ boxes := tocPageBoxes(0, 3)+ boxes = append(boxes, tocPageBoxes(1, 3)...)+ boxes = append(boxes, bodyPageBoxes(2)...)++ got := RemoveTOCBoxes(boxes, nil, true)+ if n := countPage(got, 0); n != 0 {+ t.Fatalf("TOC page 0 must be dropped, %d boxes survived", n)+ }+ if n := countPage(got, 1); n != 0 {+ t.Fatalf("TOC page 1 must be dropped, %d boxes survived", n)+ }+ if n := countPage(got, 2); n != 3 {+ t.Fatalf("body page 2 must survive, %d of 3 boxes", n)+ }+}++// TestRemoveTOCBoxes_LongTOCRun: the run length follows the page shapes, not a+// fixed cap. A real TOC can be longer than any constant small enough to be a+// useful bound, and truncating the run leaks the remaining TOC pages into the+// output — the tail page of a six-page TOC was kept while the cap was five.+func TestRemoveTOCBoxes_LongTOCRun(t *testing.T) {+ const tocPages = 6+ var boxes []pdf.TextBox+ for pg := 0; pg < tocPages; pg++ {+ boxes = append(boxes, tocPageBoxes(pg, 3)...)+ }+ boxes = append(boxes, bodyPageBoxes(tocPages)...)++ got := RemoveTOCBoxes(boxes, nil, true)+ for pg := 0; pg < tocPages; pg++ {+ if n := countPage(got, pg); n != 0 {+ t.Fatalf("TOC page %d must be dropped, %d boxes survived", pg, n)+ }+ }+ if n := countPage(got, tocPages); n != 3 {+ t.Fatalf("the body page after the TOC must survive, %d of 3 boxes", n)+ }+}++// TestRemoveTOCBoxes_TOCBehindCoverPage: a cover page carrying several short+// boxes must not close the candidate window before the TOC is reached.+func TestRemoveTOCBoxes_TOCBehindCoverPage(t *testing.T) {+ cover := []pdf.TextBox{+ tb("The Analects", 0, 150, 400, 200, 230),+ tb("Translated by Someone", 0, 150, 400, 260, 280),+ tb("Second Edition", 0, 150, 400, 300, 320),+ tb("Some Press, 2024", 0, 150, 400, 340, 360),+ }+ boxes := append(cover, tocPageBoxes(1, 3)...)+ boxes = append(boxes, bodyPageBoxes(2)...)++ got := RemoveTOCBoxes(boxes, nil, true)+ if n := countPage(got, 0); n != 4 {+ t.Fatalf("the cover page must be kept, %d of 4 boxes", n)+ }+ if n := countPage(got, 1); n != 0 {+ t.Fatalf("the TOC behind the cover must be dropped, %d boxes survived", n)+ }+ if n := countPage(got, 2); n != 3 {+ t.Fatalf("body page 2 must survive, %d of 3 boxes", n)+ }+}++// TestRemoveTOCBoxes_StopsAtProsePage: widening the candidate window must not+// reach past a page of body text. A TOC-shaped page behind one is kept.+func TestRemoveTOCBoxes_StopsAtProsePage(t *testing.T) {+ boxes := []pdf.TextBox{tb("The Analects", 0, 150, 400, 200, 230)}+ boxes = append(boxes, bodyPageBoxes(1)...)+ boxes = append(boxes, tocPageBoxes(2, 3)...)+ boxes = append(boxes, bodyPageBoxes(3)...)++ got := RemoveTOCBoxes(boxes, nil, true)+ if n := countPage(got, 2); n != 10 {+ t.Fatalf("a TOC-shaped page behind a body page must be kept, %d of 10 boxes", n)+ }+}++// TestRemoveTOCBoxes_BareCJKChapterTitles pins the shape reported in #19744: a+// CJK TOC whose entries are bare chapter titles, with no leader dots and no page+// numbers. The section-level prefix scan under-deleted these entries; the box+// signal drops the page as a whole.+func TestRemoveTOCBoxes_BareCJKChapterTitles(t *testing.T) {+ boxes := []pdf.TextBox{+ tb("目录", 0, 72, 110, 100, 120),+ tb("第一章 绪论", 0, 72, 200, 160, 175),+ tb("第二章 背景", 0, 72, 200, 190, 205),+ tb("第三章 方法", 0, 72, 200, 220, 235),+ tb("第四章 结论", 0, 72, 200, 250, 265),+ }+ boxes = append(boxes, tb("第一章 绪论", 1, 72, 200, 160, 175))+ boxes = append(boxes, bodyPageBoxes(1)...)++ got := RemoveTOCBoxes(boxes, nil, true)+ if n := countPage(got, 0); n != 0 {+ t.Fatalf("the bare-title TOC page must be dropped, %d boxes survived", n)+ }+ if n := countPage(got, 1); n != 4 {+ t.Fatalf("the body page must be kept, %d of 4 boxes", n)+ }+}++func TestRemoveTOCText_Normalize(t *testing.T) {+ cases := []struct{ in, want string }{+ {"Hello World", "hello world"},+ {"- 12 -", "- # -"},+ {"Chapter 1: Intro", "chapter #: intro"},+ {"第 1 页", "第 # 页"},+ }+ for _, c := range cases {+ if got := normalizeRunningText(c.in); got != c.want {+ t.Fatalf("normalizeRunningText(%q) = %q, want %q", c.in, got, c.want)+ }+ }+}++// TestRemoveHeaderFooterBoxes_DropsRepeatedHeader: identical header box on every+// page is removed; body is kept.+func TestRemoveHeaderFooterBoxes_DropsRepeatedHeader(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}+ boxes := []pdf.TextBox{+ // identical header in top 10% on every page+ tb("THE WAY OF GO", 0, 72, 200, 30, 45),+ tb("Body text on page zero that is long enough.", 0, 72, 400, 160, 180),+ tb("THE WAY OF GO", 1, 72, 200, 30, 45),+ tb("Body text on page one that is long enough.", 1, 72, 400, 160, 180),+ tb("THE WAY OF GO", 2, 72, 200, 30, 45),+ tb("Body text on page two that is long enough.", 2, 72, 400, 160, 180),+ }+ got := RemoveHeaderFooterBoxes(boxes, heights)+ var headers, bodies int+ for _, b := range got {+ if strings.Contains(b.Text, "THE WAY") {+ headers+++ } else {+ bodies+++ }+ }+ if headers != 0 {+ t.Fatalf("expected all repeated headers removed, got %d", headers)+ }+ if bodies != 3 {+ t.Fatalf("expected 3 body boxes kept, got %d", bodies)+ }+}++// TestRemoveHeaderFooterBoxes_KeepsUniqueZoneText: per-page unique headers are+// kept (the repetition guard must not delete genuine content).+func TestRemoveHeaderFooterBoxes_KeepsUniqueZoneText(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}+ boxes := []pdf.TextBox{+ tb("Chapter One: Getting Started", 0, 72, 250, 30, 45),+ tb("Body zero.", 0, 72, 400, 160, 180),+ tb("Chapter Two: Types and Values", 1, 72, 250, 30, 45),+ tb("Body one.", 1, 72, 400, 160, 180),+ tb("Chapter Three: Concurrency", 2, 72, 260, 30, 45),+ tb("Body two.", 2, 72, 400, 160, 180),+ }+ got := RemoveHeaderFooterBoxes(boxes, heights)+ if len(got) != len(boxes) {+ t.Fatalf("unique per-page headers must be kept, got %d, want %d", len(got), len(boxes))+ }+}++// TestRemoveHeaderFooterBoxes_PageCountGuard: documents with < 3 pages are not+// touched.+func TestRemoveHeaderFooterBoxes_PageCountGuard(t *testing.T) {+ heights := map[int]float64{0: 842, 1: 842}+ boxes := []pdf.TextBox{+ tb("Repeated", 0, 72, 150, 30, 45),+ tb("Body zero.", 0, 72, 400, 160, 180),+ tb("Repeated", 1, 72, 150, 30, 45),+ tb("Body one.", 1, 72, 400, 160, 180),+ }+ got := RemoveHeaderFooterBoxes(boxes, heights)+ if len(got) != len(boxes) {+ t.Fatalf("short documents must be untouched, got %d, want %d", len(got), len(boxes))+ }+}++// TestRemoveHeaderFooterBoxes_SkipsNonTextLayout: a table/figure box in the+// header zone must not be treated as a running header.+func TestRemoveHeaderFooterBoxes_SkipsNonTextLayout(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}+ var boxes []pdf.TextBox+ for pg := 0; pg < 3; pg++ {+ boxes = append(boxes,+ // Same text, same zone, but a table: never a running header.+ pdf.TextBox{Text: "Running head", PageNumber: pg, X0: 72, X1: 200, Top: 30, Bottom: 45, LayoutType: "table"},+ // Positive control in the same zone: this one must go.+ tb("Running head", pg, 280, 400, 30, 45),+ tb("Body text.", pg, 72, 400, 160, 180),+ )+ }+ got := RemoveHeaderFooterBoxes(boxes, heights)++ var tableKept, textKept int+ for _, b := range got {+ switch {+ case b.LayoutType == "table":+ tableKept+++ case b.Text == "Running head":+ textKept+++ }+ }+ if textKept != 0 {+ t.Fatalf("the repeated text header must be removed, %d survived", textKept)+ }+ if tableKept != 3 {+ t.Fatalf("table boxes in the header zone must be kept, %d of 3 survived", tableKept)+ }+}++// TestRemoveHeaderFooterBoxes_FullWidthPageNumbers: full-width page numbers get+// a distinct key per page unless they are masked like ASCII digits, so the+// footer never reaches minPages and survives every page.+func TestRemoveHeaderFooterBoxes_FullWidthPageNumbers(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}+ var boxes []pdf.TextBox+ for pg := 0; pg < 3; pg++ {+ boxes = append(boxes,+ tb("正文内容。", pg, 72, 400, 160, 180),+ tb("第 "+string(rune('0'+pg+1))+" 页", pg, 280, 340, 820, 835),+ )+ }+ got := RemoveHeaderFooterBoxes(boxes, heights)+ for _, b := range got {+ if b.Top >= 800 {+ t.Fatalf("full-width page-number footer %q must be removed", b.Text)+ }+ }+ if len(got) != 3 {+ t.Fatalf("expected only the 3 body boxes, got %d", len(got))+ }+}++// TestRemoveHeaderFooterBoxes_HalfOfOddPageCount: "at least half the pages" has+// to round up on an odd page count. Rounding down would let 2 pages of a+// five-page document pass as half, and this repetition rule is the only guard+// keeping genuine content out of the drop set.+func TestRemoveHeaderFooterBoxes_HalfOfOddPageCount(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight, 4: pageHeight}++ // 2 of 5 pages is below half: too rare to be a running header.+ below := []pdf.TextBox{+ tb("THE WAY OF GO", 0, 72, 200, 30, 45),+ tb("Body zero.", 0, 72, 400, 160, 180),+ tb("Body one.", 1, 72, 400, 160, 180),+ tb("THE WAY OF GO", 2, 72, 200, 30, 45),+ tb("Body two.", 2, 72, 400, 160, 180),+ tb("Body three.", 3, 72, 400, 160, 180),+ tb("Body four.", 4, 72, 400, 160, 180),+ }+ if got := RemoveHeaderFooterBoxes(below, heights); len(got) != len(below) {+ t.Fatalf("2 of 5 pages is not half, the boxes must be kept: got %d, want %d", len(got), len(below))+ }++ // 3 of 5 pages is half: a running header.+ above := []pdf.TextBox{+ tb("THE WAY OF GO", 0, 72, 200, 30, 45),+ tb("Body zero.", 0, 72, 400, 160, 180),+ tb("Body one.", 1, 72, 400, 160, 180),+ tb("THE WAY OF GO", 2, 72, 200, 30, 45),+ tb("Body two.", 2, 72, 400, 160, 180),+ tb("THE WAY OF GO", 3, 72, 200, 30, 45),+ tb("Body three.", 3, 72, 400, 160, 180),+ tb("Body four.", 4, 72, 400, 160, 180),+ }+ got := RemoveHeaderFooterBoxes(above, heights)+ if len(got) != 5 {+ t.Fatalf("3 of 5 pages is half, the headers must be dropped: got %d boxes, want 5", len(got))+ }+}++// TestRemoveHeaderFooterBoxes_DropsPageNumberFooter: identical "- N -" footers+// (digit-masked to the same key) are removed.+func TestRemoveHeaderFooterBoxes_DropsPageNumberFooter(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}+ boxes := []pdf.TextBox{+ tb("Body zero.", 0, 72, 400, 160, 180),+ tb("- 1 -", 0, 280, 320, 820, 835),+ tb("Body one.", 1, 72, 400, 160, 180),+ tb("- 2 -", 1, 280, 320, 820, 835),+ tb("Body two.", 2, 72, 400, 160, 180),+ tb("- 3 -", 2, 280, 320, 820, 835),+ }+ got := RemoveHeaderFooterBoxes(boxes, heights)+ for _, b := range got {+ if strings.Contains(b.Text, "- ") && strings.Contains(b.Text, " -") {+ t.Fatalf("footer %q should have been removed", b.Text)+ }+ }+ if len(got) != 3 {+ t.Fatalf("expected only the 3 body boxes, got %d", len(got))+ }+}diff --git a/internal/deepdoc/parser/pdf/parser.go b/internal/deepdoc/parser/pdf/parser.goindex 69ea43f0c0..7c505d67a8 100644--- a/internal/deepdoc/parser/pdf/parser.go+++ b/internal/deepdoc/parser/pdf/parser.go@@ -82,14 +82,7 @@ func NewTableBuilderFor(doc pdf.DocAnalyzer) pdf.TableBuilder {// ParseRaw is the internal entry point: runs the core pipeline on an// already-opened engine. Exported for tests that inject mock engines.func (p *Parser) ParseRaw(ctx context.Context, engine pdf.PDFEngine, docAnalyzer pdf.DocAnalyzer) (*pdf.ParseResult, error) {- outlines := p.extractOutlines(engine)-- result, err := p.processPages(ctx, engine, docAnalyzer)- if err != nil {- return nil, err- }- result.Outlines = outlines- return result, nil+ return p.processPages(ctx, engine, docAnalyzer)}// ── ParseRaw helper functions ───────────────────────────────────────────────@@ -211,7 +204,7 @@ func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,var dlaRegions []pdf.DLAPageRegionsif pageImg != nil && renderErr == nil {- ocrBoxes, updatedChars, ocrUsed = p.processPageBoxes(ctx, pageImg, chars, pg, renderErr, isScanNoise, docAnalyzer)+ ocrBoxes, updatedChars, ocrUsed = p.processPageBoxes(ctx, pageImg, chars, pg, renderErr, isScanNoise, docAnalyzer, pageZoom)annotated, pageTables, dlaRegions = p.enrichOnePageWithDeepDoc(ctx, pageImg, ocrBoxes, pg, renderErr, docAnalyzer, tb, pageZoom)}@@ -230,7 +223,7 @@ func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,slog.Debug("per-page zoom retry", "page", pg, "zoom", retryZoom)retryImg, retryRenderErr := p.renderAtDPI(ctx, engine, pg, retryZoom*72)if retryRenderErr == nil && retryImg != nil {- ocrBoxes, updatedChars, ocrUsed = p.processPageBoxes(ctx, retryImg, chars, pg, retryRenderErr, isScanNoise, docAnalyzer)+ ocrBoxes, updatedChars, ocrUsed = p.processPageBoxes(ctx, retryImg, chars, pg, retryRenderErr, isScanNoise, docAnalyzer, retryZoom)annotated, pageTables, dlaRegions = p.enrichOnePageWithDeepDoc(ctx, retryImg, ocrBoxes, pg, retryRenderErr, docAnalyzer, tb, retryZoom)pageImg = retryImg@@ -302,7 +295,7 @@ func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,// glyphs so downstream median-height/width stats stay meaningful.// - bool: whether OCR was actually used to produce ocrBoxes.func (p *Parser) processPageBoxes(ctx context.Context, pageImg image.Image, chars []pdf.TextChar, pg int,- renderErr error, isScanNoise bool, docAnalyzer pdf.DocAnalyzer,+ renderErr error, isScanNoise bool, docAnalyzer pdf.DocAnalyzer, zoom float64,) ([]pdf.TextBox, []pdf.TextChar, bool) {var ocrBoxes []pdf.TextBoxocrUsed := false@@ -310,14 +303,14 @@ func (p *Parser) processPageBoxes(ctx context.Context, pageImg image.Image, charif renderErr == nil && pageImg != nil {hasCleanChars := len(chars) > 0 && !isScanNoise && !util.IsGarbledPage(chars)if hasCleanChars {- ocrBoxes = p.ocrMergeChars(ctx, pageImg, chars, docAnalyzer, pg)+ ocrBoxes = p.ocrMergeChars(ctx, pageImg, chars, docAnalyzer, pg, zoom)ocrUsed = ocrBoxes != nil} else {label := "scan page"if len(chars) > 0 && !isScanNoise {label = "garbled page"}- ocrBoxes = p.ocrDetectAndRecognize(ctx, pageImg, docAnalyzer, pg, label)+ ocrBoxes = p.ocrDetectAndRecognize(ctx, pageImg, docAnalyzer, pg, label, zoom)ocrUsed = ocrBoxes != nilif ocrUsed {// Synthetic OCR chars feed downstream median calculations.@@ -329,7 +322,7 @@ func (p *Parser) processPageBoxes(ctx context.Context, pageImg image.Image, char}} else if len(chars) > 0 {// Detect failed but chars exist: try the merge path.- ocrBoxes = p.ocrMergeChars(ctx, pageImg, chars, docAnalyzer, pg)+ ocrBoxes = p.ocrMergeChars(ctx, pageImg, chars, docAnalyzer, pg, zoom)ocrUsed = ocrBoxes != nil}}@@ -429,10 +422,11 @@ func (p *Parser) runPageWorkers(ctx context.Context, engine pdf.PDFEngine,// state. Document-wide layout, table merge/replace, cross-page figures, and// metrics aggregation happen here so page workers never mutate shared state.// pageResults are expected to be sorted by page number.-func (p *Parser) assembleDocument(ctx context.Context, pages []int, pageResults []*pageResult) (*pdf.ParseResult, error) {+func (p *Parser) assembleDocument(ctx context.Context, pages []int, pageResults []*pageResult, outlines []pdf.Outline) (*pdf.ParseResult, error) {result := &pdf.ParseResult{PageHeight: make(map[int]float64),PageWidth: make(map[int]float64),+ Outlines: outlines,}var boxes []pdf.TextBox@@ -481,8 +475,11 @@ func (p *Parser) assembleDocument(ctx context.Context, pages []int, pageResultsreturn result, nil}+ // A TOC is a document prefix, so the box-shape signal is only meaningful+ // when this parse covers the document's first page.+ coversDocumentStart := len(pages) > 0 && pages[0] == 0if err := p.buildLayout(ctx, result, boxes, pageChars,- medianHeights, medianWidths, pageEnglish); err != nil {+ medianHeights, medianWidths, pageEnglish, coversDocumentStart); err != nil {return nil, fmt.Errorf("buildLayout: %w", err)}return result, nil@@ -492,11 +489,15 @@ func (p *Parser) assembleDocument(ctx context.Context, pages []int, pageResults// the global layout/table/figure pipeline. It is the only step that runs// AssignColumn, TextMerge, FinalReadingOrderMerge, NaiveVerticalMerge, table// merge, figure consolidation, BoxesToSections, and caption merge.+//+// coversDocumentStart reports whether `pages` started at the document's first+// page, which is what the TOC box-shape signal requires (see RemoveTOCBoxes).func (p *Parser) buildLayout(ctx context.Context,result *pdf.ParseResult,boxes []pdf.TextBox, pageChars map[int][]pdf.TextChar,medianHeights, medianWidths map[int]float64,pageEnglish map[int]bool,+ coversDocumentStart bool,) error {result.Metrics.BoxesInitial = len(boxes)@@ -524,6 +525,40 @@ func (p *Parser) buildLayout(ctx context.Context,// filter would have only caught the former.boxes = lyt.FilterWatermarkBoxes(boxes)+ // Page-level content removal on intact box geometry. These MUST run before+ // TextMerge: RemoveTOCBoxes relies on leader-dot boxes (which TextMerge+ // folds into adjacent text) and RemoveHeaderFooterBoxes relies on header+ // boxes that TextMerge would otherwise merge into the first body section.+ //+ // Header/footer removal runs FIRST because its evidence is cross-page: it+ // counts how many of the document's pages carry the same margin text, while+ // TOC removal deletes whole pages. Running the TOC pass first would take+ // that margin text off the pages it deletes, dropping the count below the+ // half-the-pages bar, so a document with both options on would keep a+ // running header that either option removes on its own.+ //+ // The TOC pass is nearly indifferent to the order. It decides on page shapes+ // and absolute bookmark numbers, and the only boxes header/footer removal+ // takes from a TOC page are margin boxes, which never carried an entry+ // marker. They do count towards tocMinShortBoxes, so a TOC page sitting+ // exactly on that threshold can fall below it and be kept — a missed TOC+ // page, which is the direction this detector errs in regardless.+ //+ // coversDocumentStart gates the TOC box-shape signal: a TOC is a document+ // prefix, so a parse restricted to a later page range must not read its own+ // first page as one. The outline signal uses absolute page numbers and needs+ // no such gate.+ boxesBefore := len(boxes)+ if p.Config.RemoveHeaderFooter {+ boxes = lyt.RemoveHeaderFooterBoxes(boxes, result.PageHeight)+ result.Metrics.BoxesHeaderFooterRemoved = boxesBefore - len(boxes)+ boxesBefore = len(boxes)+ }+ if p.Config.RemoveTOC {+ boxes = lyt.RemoveTOCBoxes(boxes, lyt.TOCPageRangeFromOutlines(result.Outlines), coversDocumentStart)+ result.Metrics.BoxesTOCRemoved = boxesBefore - len(boxes)+ }+boxes = lyt.TextMerge(boxes, medianHeights)result.Metrics.BoxesTextMerge = len(boxes)@@ -560,10 +595,15 @@ func (p *Parser) processPages(ctx context.Context, engine pdf.PDFEngine, docAnalif err != nil {return nil, fmt.Errorf("page count: %w", err)}+ // The outlines are read before the layout is built: buildLayout uses them to+ // select TOC pages on the intact box geometry (see Parser.buildLayout), so+ // they cannot be attached to the result afterwards.+ outlines := p.extractOutlines(engine)if pageCount == 0 {return &pdf.ParseResult{PageHeight: make(map[int]float64),PageWidth: make(map[int]float64),+ Outlines: outlines,}, nil}@@ -583,7 +623,7 @@ func (p *Parser) processPages(ctx context.Context, engine pdf.PDFEngine, docAnalslog.Warn("runPageWorkers: some pages failed", "err", pageErr)}- result, err := p.assembleDocument(ctx, pages, pageResults)+ result, err := p.assembleDocument(ctx, pages, pageResults, outlines)if err != nil {return nil, err}diff --git a/internal/deepdoc/parser/pdf/parser_ocr.go b/internal/deepdoc/parser/pdf/parser_ocr.goindex 7a137a85e3..d342062436 100644--- a/internal/deepdoc/parser/pdf/parser_ocr.go+++ b/internal/deepdoc/parser/pdf/parser_ocr.go@@ -12,7 +12,7 @@ import ("strings")-func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image, doc pdf.DocAnalyzer, pageNum int, logLabel string) []pdf.TextBox {+func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image, doc pdf.DocAnalyzer, pageNum int, logLabel string, zoom float64) []pdf.TextBox {boxes, err := p.inferOCRDetect(ctx, doc, pageImg)if err != nil || len(boxes) == 0 {if err != nil {@@ -21,12 +21,14 @@ func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image,return nil}- // detectBoxes returns image-pixel coords; ocrMergeChars divides by- // pdf.DlaScale before emitting boxes so downstream layout receives- // PDF-point coordinates. ocrDetectAndRecognize must match the same- // conversion so both OCR paths produce the same coordinate space.- imgW := float64(pageImg.Bounds().Dx()) / pdf.DlaScale- imgH := float64(pageImg.Bounds().Dy()) / pdf.DlaScale+ // detectBoxes returns image-pixel coords; both OCR paths divide by the zoom+ // the page was rendered at before emitting boxes so downstream layout+ // receives PDF-point coordinates — the same units as ParseResult.PageHeight.+ // That zoom is DlaScale for the default render and the retry zoom after a+ // per-page re-render, so it cannot be a constant.+ scale := ocrCoordinateScale(zoom)+ imgW := float64(pageImg.Bounds().Dx()) / scale+ imgH := float64(pageImg.Bounds().Dy()) / scale// For each box, de-skew via WarpCrop (layer 1) and build the layer-2// rotation candidates: short/wide crops get one candidate at 0 deg; tall@@ -61,10 +63,10 @@ func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image,{X: b.X3, Y: b.Y3},})// Convert detection bounds to PDF-point space (mirrors detectBoxes).- px0 := float64(x0) / pdf.DlaScale- py0 := float64(y0) / pdf.DlaScale- px1 := float64(x1) / pdf.DlaScale- py1 := float64(y1) / pdf.DlaScale+ px0 := float64(x0) / scale+ py0 := float64(y0) / scale+ px1 := float64(x1) / scale+ py1 := float64(y1) / scaleif px0 < 0 {px0 = 0}@@ -231,8 +233,8 @@ type ocrDetectBox struct {srcIdx int}-func (p *Parser) ocrMergeChars(ctx context.Context, pageImg image.Image, chars []pdf.TextChar, doc pdf.DocAnalyzer, pageNum int) []pdf.TextBox {- boxes, scale, err := p.detectBoxes(ctx, pageImg, doc, pageNum)+func (p *Parser) ocrMergeChars(ctx context.Context, pageImg image.Image, chars []pdf.TextChar, doc pdf.DocAnalyzer, pageNum int, zoom float64) []pdf.TextBox {+ boxes, scale, err := p.detectBoxes(ctx, pageImg, doc, pageNum, zoom)if err != nil || len(boxes) == 0 {return nil}@@ -240,14 +242,30 @@ func (p *Parser) ocrMergeChars(ctx context.Context, pageImg image.Image, chars [return p.buildTextBoxes(ctx, pageImg, boxes, boxChars, doc, scale, pageNum)}-func (p *Parser) detectBoxes(ctx context.Context, pageImg image.Image, doc pdf.DocAnalyzer, pageNum int) ([]ocrDetectBox, float64, error) {+// ocrCoordinateScale returns the factor that converts image pixels into PDF+// points for a page rendered at zoom, falling back to the default DLA scale+// when the caller supplies none. Detection and recognition must both use it:+// any other factor leaves the boxes in a scale of their own, and every+// page-relative comparison downstream — the header/footer zone check in+// particular — then compares unlike quantities.+func ocrCoordinateScale(zoom float64) float64 {+ if zoom > 0 {+ return zoom+ }+ return pdf.DlaScale+}++func (p *Parser) detectBoxes(ctx context.Context, pageImg image.Image, doc pdf.DocAnalyzer, pageNum int, zoom float64) ([]ocrDetectBox, float64, error) {ocrDetectBoxes, err := p.inferOCRDetect(ctx, doc, pageImg)if err != nil || len(ocrDetectBoxes) == 0 {return nil, 0, err}slog.Debug("ocrMergeChars detect", "page", pageNum, "boxes", len(ocrDetectBoxes))- scale := pdf.DlaScale // 3.0+ // The caller multiplies the returned boxes back by this scale to crop the+ // original render, so passing the render zoom keeps the round trip exact+ // and makes the emitted coordinates PDF points (see ocrDetectAndRecognize).+ scale := ocrCoordinateScale(zoom)imgBounds := pageImg.Bounds()imgW := float64(imgBounds.Dx()) / scaleimgH := float64(imgBounds.Dy()) / scalediff --git a/internal/deepdoc/parser/pdf/parser_ocr_test.go b/internal/deepdoc/parser/pdf/parser_ocr_test.goindex 63e1fe5d5d..22c8bfeeb8 100644--- a/internal/deepdoc/parser/pdf/parser_ocr_test.go+++ b/internal/deepdoc/parser/pdf/parser_ocr_test.go@@ -25,7 +25,7 @@ func TestOCRMergeChars_FullCoverage(t *testing.T) {{X0: 12, X1: 28, Top: 2, Bottom: 35, Text: "World"},}- boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0)+ boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0, pdf.DlaScale)if len(boxes) != 1 {t.Fatalf("expected 1 box, got %d", len(boxes))}@@ -54,7 +54,7 @@ func TestOCRMergeChars_PartialCoverage(t *testing.T) {{X0: 2, X1: 12, Top: 2, Bottom: 15, Text: "A"},}- boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0)+ boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0, pdf.DlaScale)if len(boxes) != 2 {t.Fatalf("expected 2 boxes, got %d", len(boxes))}@@ -81,14 +81,14 @@ func TestOCRMergeChars_NoDetectBoxes(t *testing.T) {}ctx := t.Context()- boxes := p.ocrMergeChars(ctx, testPageImg(), chars, mock, 0)+ boxes := p.ocrMergeChars(ctx, testPageImg(), chars, mock, 0, pdf.DlaScale)if boxes != nil {t.Errorf("expected nil for no detect boxes, got %d boxes", len(boxes))}// Also test empty OCRBoxesmock.OCRBoxes = []pdf.OCRBox{}- boxes = p.ocrMergeChars(ctx, testPageImg(), chars, mock, 0)+ boxes = p.ocrMergeChars(ctx, testPageImg(), chars, mock, 0, pdf.DlaScale)if boxes != nil {t.Errorf("expected nil for empty detect boxes, got %d boxes", len(boxes))}@@ -114,7 +114,7 @@ func TestOCRMergeChars_GarbledChars(t *testing.T) {{X0: 22, X1: 28, Top: 2, Bottom: 35, Text: "a"}, // normal}- boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0)+ boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0, pdf.DlaScale)if len(boxes) != 1 {t.Fatalf("expected 1 box, got %d", len(boxes))}@@ -143,7 +143,7 @@ func TestOCRMergeChars_HeightGate(t *testing.T) {{X0: 2, X1: 10, Top: 2, Bottom: 3, Text: "tiny"},}- boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0)+ boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0, pdf.DlaScale)if len(boxes) != 1 {t.Fatalf("expected 1 box (OCR fallback after height gate), got %d", len(boxes))}@@ -174,7 +174,7 @@ func TestOCRMergeChars_FontEncodingGarbled(t *testing.T) {Text: "#", FontName: "DY1+SimSun", PageNumber: 0,}}- boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0)+ boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0, pdf.DlaScale)if len(boxes) != 1 {t.Fatalf("expected 1 OCR-fallback box, got %d", len(boxes))}@@ -253,7 +253,7 @@ func TestOCRMergeChars_MixedFontSizes(t *testing.T) {{X0: 12, X1: 24, Top: 5, Bottom: 35, Text: "大"}, // larger font, lower baseline{X0: 24, X1: 36, Top: 5, Bottom: 35, Text: "号"}, // same size as 大, rightmost}- boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0)+ boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0, pdf.DlaScale)if len(boxes) != 1 {t.Fatalf("expected 1 box, got %d", len(boxes))}@@ -285,7 +285,7 @@ func TestOCRMergeChars_BoxOrder(t *testing.T) {{X0: 2, X1: 10, Top: 16, Bottom: 19, Text: "B"}, // box 2 (middle){X0: 2, X1: 10, Top: 32, Bottom: 37, Text: "C"}, // box 3 (bottom)}- boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0)+ boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0, pdf.DlaScale)if len(boxes) != 3 {t.Fatalf("expected 3 boxes, got %d", len(boxes))}@@ -320,7 +320,7 @@ func TestOCRMergeChars_OverlappingBoxes(t *testing.T) {{X0: 12, X1: 18, Top: 2, Bottom: 12, Text: "乙"}, // overlap zone{X0: 22, X1: 28, Top: 2, Bottom: 12, Text: "丙"}, // Box B only}- boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0)+ boxes := p.ocrMergeChars(t.Context(), testPageImg(), chars, mock, 0, pdf.DlaScale)if len(boxes) != 2 {t.Fatalf("expected 2 boxes, got %d", len(boxes))}@@ -399,3 +399,47 @@ func TestPdfOxideUnmappedGarbled_RealWorldGarbled(t *testing.T) {t.Error("real-world garbled text with ### clusters should be detected")}}++// TestOCRBoxesTrackRenderZoom pins the coordinate invariant of both OCR paths:+// the page is rendered at `zoom`, so detection pixels divide by that same zoom+// to become PDF points — the units ParseResult.PageHeight is in, since+// parser.go computes it as renderHeight/zoom. Dividing by a constant instead+// leaves the boxes retryZoom/DlaScale times too large after a per-page retry+// render (9/3 with the default config), which is how body boxes end up looking+// like margin boxes to the header/footer zone check.+func TestOCRBoxesTrackRenderZoom(t *testing.T) {+ img := testPageImg() // 90x120 pixels+ mock := &MockDocAnalyzer{+ Healthy: true,+ OCRBoxes: []pdf.OCRBox{+ {X0: 0, Y0: 0, X1: 90, Y1: 0, X2: 90, Y2: 120, X3: 0, Y3: 120},+ },+ OCRTexts: []pdf.OCRText{{Text: "OCR text", Confidence: 0.9}},+ }+ p := NewParser(pdf.DefaultParserConfig())++ // DlaScale is the default render; 9.0 is what the retry render uses when+ // Config.Zoom is 3 (the default), so both must land inside their own page.+ for _, zoom := range []float64{pdf.DlaScale, 9.0} {+ pageW := float64(img.Bounds().Dx()) / zoom+ pageH := float64(img.Bounds().Dy()) / zoom++ merged := p.ocrMergeChars(t.Context(), img, nil, mock, 0, zoom)+ if len(merged) != 1 {+ t.Fatalf("zoom %v: ocrMergeChars expected 1 box, got %d", zoom, len(merged))+ }+ if merged[0].X1 > pageW || merged[0].Bottom > pageH {+ t.Fatalf("zoom %v: ocrMergeChars box (%v,%v,%v,%v) falls outside the %.2fx%.2fpt page of this render",+ zoom, merged[0].X0, merged[0].Top, merged[0].X1, merged[0].Bottom, pageW, pageH)+ }++ scanned := p.ocrDetectAndRecognize(t.Context(), img, mock, 0, "scan page", zoom)+ if len(scanned) != 1 {+ t.Fatalf("zoom %v: ocrDetectAndRecognize expected 1 box, got %d", zoom, len(scanned))+ }+ if scanned[0].X1 > pageW || scanned[0].Bottom > pageH {+ t.Fatalf("zoom %v: ocrDetectAndRecognize box (%v,%v,%v,%v) falls outside the %.2fx%.2fpt page of this render",+ zoom, scanned[0].X0, scanned[0].Top, scanned[0].X1, scanned[0].Bottom, pageW, pageH)+ }+ }+}diff --git a/internal/deepdoc/parser/pdf/parser_ocr_warp_test.go b/internal/deepdoc/parser/pdf/parser_ocr_warp_test.goindex 35f17b0b60..dc98f66a85 100644--- a/internal/deepdoc/parser/pdf/parser_ocr_warp_test.go+++ b/internal/deepdoc/parser/pdf/parser_ocr_warp_test.go@@ -76,7 +76,7 @@ func TestOCRDetectAndRecognize_WarpsCrop(t *testing.T) {boxes: []pdf.OCRBox{box},texts: []pdf.OCRText{{Text: "x", Confidence: 0.9}},}- got := p.ocrDetectAndRecognize(t.Context(), page, cap, 0, "warp")+ got := p.ocrDetectAndRecognize(t.Context(), page, cap, 0, "warp", pdf.DlaScale)if len(got) != 1 {t.Fatalf("expected 1 text box, got %d", len(got))}@@ -171,7 +171,7 @@ func TestOCRMergeChars_WarpsEmptyBoxCrop(t *testing.T) {boxes: []pdf.OCRBox{box},texts: []pdf.OCRText{{Text: "x", Confidence: 0.9}},}- got := p.ocrMergeChars(t.Context(), page, chars, cap, 0)+ got := p.ocrMergeChars(t.Context(), page, chars, cap, 0, pdf.DlaScale)if len(got) == 0 {t.Fatal("expected at least one text box from the merge path")}diff --git a/internal/deepdoc/parser/pdf/parser_test.go b/internal/deepdoc/parser/pdf/parser_test.goindex c189c1f14f..78545b1afc 100644--- a/internal/deepdoc/parser/pdf/parser_test.go+++ b/internal/deepdoc/parser/pdf/parser_test.go@@ -38,14 +38,14 @@ func TestOCR_Fallback(t *testing.T) {dummyImg := image.NewRGBA(image.Rect(0, 0, 100, 100))t.Run("nil image", func(t *testing.T) {- if got := p.ocrDetectAndRecognize(t.Context(), nil, &MockDocAnalyzer{Healthy: true}, 0, "garbled page"); got != nil {+ if got := p.ocrDetectAndRecognize(t.Context(), nil, &MockDocAnalyzer{Healthy: true}, 0, "garbled page", pdf.DlaScale); got != nil {t.Error("nil image → nil")}})t.Run("detect returns no boxes", func(t *testing.T) {mock := &MockDocAnalyzer{Healthy: true, OCRBoxes: nil}- if got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "garbled page"); got != nil {+ if got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "garbled page", pdf.DlaScale); got != nil {t.Error("no det boxes → nil")}})@@ -56,7 +56,7 @@ func TestOCR_Fallback(t *testing.T) {OCRBoxes: []pdf.OCRBox{{X0: 10, Y0: 20, X1: 90, Y1: 20, X2: 90, Y2: 40, X3: 10, Y3: 40}},OCRTexts: []pdf.OCRText{{Text: "Hello", Confidence: 0.9}},}- got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "garbled page")+ got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "garbled page", pdf.DlaScale)if len(got) != 1 {t.Fatalf("expected 1 pdf.TextChar, got %d", len(got))}@@ -71,7 +71,7 @@ func TestOCR_Fallback(t *testing.T) {OCRBoxes: []pdf.OCRBox{{X0: 10, Y0: 20, X1: 90, Y1: 20, X2: 90, Y2: 40, X3: 10, Y3: 40}},OCRTexts: []pdf.OCRText{{Text: "", Confidence: 0.1}},}- got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "garbled page")+ got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "garbled page", pdf.DlaScale)if len(got) != 0 {t.Error("empty rec text → empty result")}@@ -87,14 +87,14 @@ func TestOCR_ScanPage(t *testing.T) {dummyImg := image.NewRGBA(image.Rect(0, 0, 100, 100))t.Run("nil image", func(t *testing.T) {- if got := p.ocrDetectAndRecognize(t.Context(), nil, &MockDocAnalyzer{Healthy: true}, 0, "scan page"); got != nil {+ if got := p.ocrDetectAndRecognize(t.Context(), nil, &MockDocAnalyzer{Healthy: true}, 0, "scan page", pdf.DlaScale); got != nil {t.Error("nil image → nil")}})t.Run("detect returns no boxes", func(t *testing.T) {mock := &MockDocAnalyzer{Healthy: true, OCRBoxes: nil}- if got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "scan page"); got != nil {+ if got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "scan page", pdf.DlaScale); got != nil {t.Error("no det boxes → nil")}})@@ -108,7 +108,7 @@ func TestOCR_ScanPage(t *testing.T) {},OCRTexts: []pdf.OCRText{{Text: "Hello", Confidence: 0.9}, {Text: "World", Confidence: 0.8}},}- got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "scan page")+ got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "scan page", pdf.DlaScale)if len(got) < 1 {t.Error("expected at least 1 pdf.TextChar")}@@ -120,7 +120,7 @@ func TestOCR_ScanPage(t *testing.T) {OCRBoxes: []pdf.OCRBox{{X0: 10, Y0: 20, X1: 90, Y1: 20, X2: 90, Y2: 40, X3: 10, Y3: 40}},OCRTexts: []pdf.OCRText{},}- got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "scan page")+ got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "scan page", pdf.DlaScale)if len(got) != 0 {t.Error("no rec text → empty")}@@ -264,7 +264,7 @@ func TestOCR_Fallback_PUAGarbled(t *testing.T) {OCRBoxes: []pdf.OCRBox{{X0: 10, Y0: 20, X1: 90, Y1: 20, X2: 90, Y2: 40, X3: 10, Y3: 40}},OCRTexts: []pdf.OCRText{{Text: "PUA OCR text", Confidence: 0.9}},}- got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "garbled page")+ got := p.ocrDetectAndRecognize(t.Context(), dummyImg, mock, 0, "garbled page", pdf.DlaScale)if len(got) != 1 || got[0].Text != "PUA OCR text" {t.Errorf("PUA garbled should trigger OCR, got %v", got)}@@ -278,7 +278,7 @@ func TestOCR_MergeChars(t *testing.T) {t.Run("nil image", func(t *testing.T) {chars := []pdf.TextChar{{X0: 10, Top: 10, X1: 20, Bottom: 30, Text: "A", PageNumber: 0}}- if boxes := p.ocrMergeChars(t.Context(), nil, chars, &MockDocAnalyzer{Healthy: true}, 0); boxes != nil {+ if boxes := p.ocrMergeChars(t.Context(), nil, chars, &MockDocAnalyzer{Healthy: true}, 0, pdf.DlaScale); boxes != nil {t.Error("nil image → nil")}})@@ -286,7 +286,7 @@ func TestOCR_MergeChars(t *testing.T) {t.Run("detect returns no boxes", func(t *testing.T) {mock := &MockDocAnalyzer{Healthy: true, OCRBoxes: []pdf.OCRBox{}}chars := []pdf.TextChar{{X0: 10, Top: 10, X1: 20, Bottom: 30, Text: "A", PageNumber: 0}}- if boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0); boxes != nil {+ if boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale); boxes != nil {t.Error("no detect boxes → nil")}})@@ -298,7 +298,7 @@ func TestOCR_MergeChars(t *testing.T) {OCRTexts: []pdf.OCRText{{Text: "Hello OCR", Confidence: 0.9}},}chars := []pdf.TextChar{{X0: 10, X1: 30, Top: 10, Bottom: 30, Text: "Hello", PageNumber: 0}}- boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0)+ boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale)if len(boxes) != 1 {t.Fatalf("expected 1 box, got %d", len(boxes))}@@ -315,7 +315,7 @@ func TestOCR_MergeChars(t *testing.T) {OCRTexts: []pdf.OCRText{{Text: "OCR", Confidence: 0.9}},}chars := []pdf.TextChar{{X0: 10, X1: 20, Top: 10, Bottom: 20, Text: "A", PageNumber: 0}}- boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0)+ boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale)if len(boxes) != 1 {t.Fatalf("expected 1 box (OCR), got %d", len(boxes))}@@ -331,7 +331,7 @@ func TestOCR_MergeChars(t *testing.T) {OCRTexts: []pdf.OCRText{},}chars := []pdf.TextChar{{X0: 10, X1: 20, Top: 10, Bottom: 20, Text: "A", PageNumber: 0}}- boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0)+ boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale)if len(boxes) != 0 {t.Fatalf("expected 0 boxes (empty OCR), got %d", len(boxes))}@@ -350,7 +350,7 @@ func TestOCR_MergeChars(t *testing.T) {},}chars := []pdf.TextChar{{X0: 10, X1: 30, Top: 10, Bottom: 30, Text: "Hello", PageNumber: 0}}- boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0)+ boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale)if len(boxes) != 2 {t.Fatalf("expected 2 boxes, got %d", len(boxes))}@@ -378,7 +378,7 @@ func TestOCR_MergeChars(t *testing.T) {{X0: 70, X1: 90, Top: 110, Bottom: 130, Text: "c", PageNumber: 0},{X0: 10, X1: 30, Top: 10, Bottom: 30, Text: "a", PageNumber: 0},}- boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0)+ boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale)if len(boxes) != 2 {t.Fatalf("expected 2 detect boxes, got %d", len(boxes))}@@ -405,7 +405,7 @@ func TestOCR_MergeChars(t *testing.T) {{X0: 10, X1: 30, Top: 30, Bottom: 50, Text: "A", PageNumber: 0},{X0: 40, X1: 60, Top: 20, Bottom: 120, Text: "B", PageNumber: 0},}- boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0)+ boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale)if len(boxes) != 1 {t.Fatalf("expected 1 box, got %d", len(boxes))}@@ -428,7 +428,7 @@ func TestOCR_MergeChars(t *testing.T) {{X0: 30, X1: 40, Top: 10, Bottom: 20, Text: "", PageNumber: 0},{X0: 50, X1: 60, Top: 10, Bottom: 20, Text: "a", PageNumber: 0},}- boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0)+ boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale)if len(boxes) != 1 {t.Fatalf("expected 1 box, got %d", len(boxes))}@@ -450,7 +450,7 @@ func TestOCR_MergeChars(t *testing.T) {{Text: "d", X0: 10, X1: 20, Top: 10, Bottom: 25, PageNumber: 0},{Text: "o", X0: 21, X1: 30, Top: 10, Bottom: 25, PageNumber: 0},}- boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0)+ boxes := p.ocrMergeChars(t.Context(), dummyImg, chars, mock, 0, pdf.DlaScale)if len(boxes) != 1 {t.Fatalf("expected 1 box, got %d", len(boxes))}diff --git a/internal/deepdoc/parser/pdf/toc_box_removal_test.go b/internal/deepdoc/parser/pdf/toc_box_removal_test.gonew file mode 100644index 0000000000..d916362413--- /dev/null+++ b/internal/deepdoc/parser/pdf/toc_box_removal_test.go@@ -0,0 +1,191 @@+package pdf++import (+ "strings"+ "testing"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// outlineEngine is a MockEngine that also reports bookmarks; the base stub+// hardcodes Outlines to nil.+type outlineEngine struct {+ MockEngine+ outlines []pdf.Outline+}++func (e *outlineEngine) Outlines() ([]pdf.Outline, error) { return e.outlines, nil }++// tocDocEngine builds a two-page document: page 0 is a TOC carrying two short+// lines, page 1 is body text. Two short boxes stay below tocMinShortBoxes, so+// the box signal cannot select page 0 and the bookmark is the only signal that+// can.+func tocDocEngine() *outlineEngine {+ return &outlineEngine{+ MockEngine: MockEngine{+ NumPages: 2,+ Chars: map[int][]pdf.TextChar{+ 0: {+ {X0: 50, X1: 550, Top: 100, Bottom: 112, Text: "目录", PageNumber: 0},+ {X0: 50, X1: 550, Top: 300, Bottom: 312, Text: "第一章 道可道", PageNumber: 0},+ },+ 1: {+ {X0: 50, X1: 550, Top: 100, Bottom: 112, Text: "body text", PageNumber: 1},+ },+ },+ },+ outlines: []pdf.Outline{+ {Title: "目录", Level: 0, PageNumber: 1},+ {Title: "第一章", Level: 0, PageNumber: 2},+ },+ }+}++func sectionsOnPage(result *pdf.ParseResult, pg int) int {+ n := 0+ for _, s := range result.Sections {+ for _, p := range s.Positions {+ for _, pn := range p.PageNumbers {+ if pn == pg {+ n+++ }+ }+ }+ }+ return n+}++// TestParseRaw_RemoveTOCDropsPageSelectedByOutline drives the option end to end:+// the bookmark names the TOC page, so its boxes must never reach the sections.+func TestParseRaw_RemoveTOCDropsPageSelectedByOutline(t *testing.T) {+ cfg := pdf.DefaultParserConfig()+ cfg.RemoveTOC = true++ result, err := NewParser(cfg).ParseRaw(t.Context(), tocDocEngine(), &MockDocAnalyzer{Healthy: true})+ if err != nil {+ t.Fatalf("ParseRaw: %v", err)+ }+ if result.Metrics.BoxesTOCRemoved == 0 {+ t.Fatal("expected the bookmark-selected TOC page boxes to be removed")+ }+ if n := sectionsOnPage(result, 0); n != 0 {+ t.Fatalf("page 0 is the TOC and must be gone, %d sections still reference it", n)+ }+ if n := sectionsOnPage(result, 1); n == 0 {+ t.Fatal("body page 1 must survive TOC removal")+ }+}++// TestParseRaw_RemoveTOCOffKeepsPage is the control: the same document keeps its+// TOC page when the option is off.+func TestParseRaw_RemoveTOCOffKeepsPage(t *testing.T) {+ result, err := NewParser(pdf.DefaultParserConfig()).ParseRaw(t.Context(), tocDocEngine(), &MockDocAnalyzer{Healthy: true})+ if err != nil {+ t.Fatalf("ParseRaw: %v", err)+ }+ if result.Metrics.BoxesTOCRemoved != 0 {+ t.Fatalf("RemoveTOC is off, got %d boxes removed", result.Metrics.BoxesTOCRemoved)+ }+ if n := sectionsOnPage(result, 0); n == 0 {+ t.Fatal("with RemoveTOC off the TOC page must stay")+ }+}++// fixtureHeader is the running header the combined fixture repeats. It carries+// no digits, so every page normalizes to the same comparison key.+const fixtureHeader = "BOOK TITLE"++// combinedRemovalEngine builds a document where both passes have something to+// do: pages 0 and 1 are TOC-shaped (five short boxes, four of them markers),+// pages 2-5 carry prose, and the same header repeats in the top zone on pages+// 0-3. Four of six pages is above the half-the-pages bar, so the header is+// removable only while the pages carrying it are still in the box set. Render+// height 1000 at DlaScale gives a 333pt page, which puts the header (bottom 5)+// inside the top 10% and the body lines (top 100) outside it.+func combinedRemovalEngine() *MockEngine {+ line := func(pg int, top float64, text string) pdf.TextChar {+ return pdf.TextChar{X0: 50, X1: 900, Top: top, Bottom: top + 12, Text: text, PageNumber: pg}+ }+ header := func(pg int) pdf.TextChar {+ return pdf.TextChar{X0: 50, X1: 200, Top: 1, Bottom: 5, Text: fixtureHeader, PageNumber: pg}+ }+ // Long enough to count as prose (> tocMaxProseRunes runes).+ const body = "This paragraph of body text is comfortably longer than sixty runes, so the page it sits on counts as prose."+ tocLine := func(pg int, top float64, text string) pdf.TextChar { return line(pg, top, text) }++ chars := map[int][]pdf.TextChar{}+ for pg := 0; pg < 6; pg++ {+ if pg < 2 {+ chars[pg] = []pdf.TextChar{+ header(pg),+ tocLine(pg, 100, "目录"),+ tocLine(pg, 130, "第一章 道可道"),+ tocLine(pg, 160, "第二章 天下皆知"),+ tocLine(pg, 190, "1"),+ }+ continue+ }+ page := []pdf.TextChar{line(pg, 100, body), line(pg, 130, body), line(pg, 160, body)}+ if pg < 4 {+ page = append([]pdf.TextChar{header(pg)}, page...)+ }+ chars[pg] = page+ }+ return &MockEngine{NumPages: 6, RenderW: 1000, RenderH: 1000, Chars: chars}+}++// TestParseRaw_HeaderFooterRemovesHeaderAlone is the control for the test below:+// the fixture's header really is removable, so a combined run that keeps it is+// reporting an interaction and not a header no run could remove.+func TestParseRaw_HeaderFooterRemovesHeaderAlone(t *testing.T) {+ cfg := pdf.DefaultParserConfig()+ cfg.RemoveHeaderFooter = true++ result, err := NewParser(cfg).ParseRaw(t.Context(), combinedRemovalEngine(), &MockDocAnalyzer{Healthy: true})+ if err != nil {+ t.Fatalf("ParseRaw: %v", err)+ }+ if result.Metrics.BoxesHeaderFooterRemoved == 0 {+ t.Fatal("the repeated header must be removed when RemoveHeaderFooter is on")+ }+ for _, s := range result.Sections {+ if strings.Contains(s.Text, fixtureHeader) {+ t.Fatalf("header %q survived on its own", s.Text)+ }+ }+}++// TestParseRaw_TOCAndHeaderFooterBothApply pins the order the two passes run in.+// Header/footer removal counts cross-page repetition, and TOC removal deletes+// whole pages — so if TOC ran first it would take the header's evidence with it+// and leave the header on the pages that survive, making the two options+// together weaker than either one alone.+func TestParseRaw_TOCAndHeaderFooterBothApply(t *testing.T) {+ cfg := pdf.DefaultParserConfig()+ cfg.RemoveTOC = true+ cfg.RemoveHeaderFooter = true++ result, err := NewParser(cfg).ParseRaw(t.Context(), combinedRemovalEngine(), &MockDocAnalyzer{Healthy: true})+ if err != nil {+ t.Fatalf("ParseRaw: %v", err)+ }+ if result.Metrics.BoxesTOCRemoved == 0 {+ t.Fatal("the TOC pages must still be dropped when both options are on")+ }+ if result.Metrics.BoxesHeaderFooterRemoved == 0 {+ t.Fatal("the running header must still be removed when both options are on")+ }+ for pg := 0; pg < 2; pg++ {+ if n := sectionsOnPage(result, pg); n != 0 {+ t.Fatalf("TOC page %d must be gone, %d sections still reference it", pg, n)+ }+ }+ for _, s := range result.Sections {+ if strings.Contains(s.Text, fixtureHeader) {+ t.Fatalf("header %q survived the combined run", s.Text)+ }+ }+ if n := sectionsOnPage(result, 4); n == 0 {+ t.Fatal("body page 4 must survive both passes")+ }+}diff --git a/internal/deepdoc/parser/type/types.go b/internal/deepdoc/parser/type/types.goindex 8345afbebf..aa76f4a697 100644--- a/internal/deepdoc/parser/type/types.go+++ b/internal/deepdoc/parser/type/types.go@@ -14,11 +14,17 @@ import (// PipelineMetrics records diagnostic counts at each pipeline stage.type PipelineMetrics struct {- BoxesInitial int- BoxesTextMerge int- BoxesVertMerge int- BoxesFinal int- TablesCount int+ BoxesInitial int+ // BoxesTOCRemoved / BoxesHeaderFooterRemoved count the boxes dropped by the+ // optional box-level content removal passes, so the box budget can be+ // reconciled: BoxesInitial - BoxesTOCRemoved - BoxesHeaderFooterRemoved >=+ // BoxesTextMerge.+ BoxesTOCRemoved int+ BoxesHeaderFooterRemoved int+ BoxesTextMerge int+ BoxesVertMerge int+ BoxesFinal int+ TablesCount int}// ParseResult encapsulates all outputs from a single Parse() call.@@ -237,6 +243,17 @@ type ParserConfig struct {// nil/empty means parse all pages. Ranges beyond the document are clamped// at parse time; fully out-of-range ranges are skipped.Pages [][]int+ // RemoveTOC enables box-level table-of-contents page removal in+ // Parser.buildLayout. Detection relies on leader-dot boxes and per-box+ // geometry that are destroyed by the later TextMerge pass, so it is gated+ // onto the box-level pipeline there rather than the section-level+ // post-process.+ RemoveTOC bool+ // RemoveHeaderFooter enables box-level running header / footer removal in+ // Parser.buildLayout. It operates on intact box geometry (page zones and+ // cross-page text repetition) before TextMerge can fold a header box into+ // a body section.+ RemoveHeaderFooter bool}// DefaultParserConfig returns a ParserConfig with sensible defaults.diff --git a/internal/parser/parser/pdf_parser_cgo.go b/internal/parser/parser/pdf_parser_cgo.goindex 8a957ae7c2..349de5b63c 100644--- a/internal/parser/parser/pdf_parser_cgo.go+++ b/internal/parser/parser/pdf_parser_cgo.go@@ -35,14 +35,14 @@ func (p *PDFParser) ParseWithResult(ctx context.Context, filename string, data [}cfg := deepdoctype.DefaultParserConfig()cfg.Pages = p.Pages+ cfg.RemoveTOC = p.RemoveTOC+ cfg.RemoveHeaderFooter = p.RemoveHeaderFooterparser := deepdocpdf.NewParser(cfg)res := parsePDFWithDeepDocOptions(ctx, filename, data, pdfPostProcessOptions{outputFormat: p.OutputFormat,zoom: cfg.Zoom,enableMultiColumn: p.EnableMultiColumn,flattenMediaToText: p.FlattenMediaToText,- removeTOC: p.RemoveTOC,- removeHeaderFooter: p.RemoveHeaderFooter,}, parser.Parse)if res.Err != nil && errors.Is(res.Err, deepdocpdf.ErrNoPDFData) {return ParseResult{Err: fmt.Errorf("%w: %s", ErrPDFEngineUnavailable, filename)}diff --git a/internal/parser/parser/pdf_parser_common_test.go b/internal/parser/parser/pdf_parser_common_test.goindex 61c09ea99b..280772a7c8 100644--- a/internal/parser/parser/pdf_parser_common_test.go+++ b/internal/parser/parser/pdf_parser_common_test.go@@ -220,81 +220,6 @@ func TestPDFParseResultToJSON_DefaultKeepsHeaderFooterLikePython(t *testing.T) {}}-func TestPDFParseResultToJSONWithOptions_FiltersHeaderFooterWhenEnabled(t *testing.T) {- parsed := &deepdoctype.ParseResult{- Sections: []deepdoctype.Section{- {Text: "Header", LayoutType: "header"},- {- Text: "Body",- LayoutType: "",- Positions: []deepdoctype.Position{{- PageNumbers: []int{0},- Left: 10,- Right: 20,- Top: 30,- Bottom: 40,- }},- },- {Text: "Footer", LayoutType: "footer"},- },- }-- res := pdfParseResultToJSONWithOptions("filtered.pdf", parsed, pdfPostProcessOptions{removeHeaderFooter: true})- if res.Err != nil {- t.Fatalf("pdfParseResultToJSONWithOptions: %v", res.Err)- }- if len(res.JSON) != 1 {- t.Fatalf("JSON len = %d, want 1", len(res.JSON))- }- if got, want := res.JSON[0]["text"], "Body"; got != want {- t.Fatalf("JSON[0].text = %v, want %v", got, want)- }-}--func TestPDFParseResultToJSONWithOptions_RemovesTOCByOutline(t *testing.T) {- parsed := &deepdoctype.ParseResult{- Sections: []deepdoctype.Section{- {- Text: "Contents",- LayoutType: "text",- Positions: []deepdoctype.Position{{- PageNumbers: []int{1},- Left: 10,- Right: 20,- Top: 30,- Bottom: 40,- }},- },- {- Text: "Body",- LayoutType: "text",- Positions: []deepdoctype.Position{{- PageNumbers: []int{3},- Left: 10,- Right: 20,- Top: 30,- Bottom: 40,- }},- },- },- Outlines: []deepdoctype.Outline{- {Title: "目录", Level: 0, PageNumber: 1},- {Title: "Chapter 1", Level: 0, PageNumber: 3},- },- }-- res := pdfParseResultToJSONWithOptions("toc.pdf", parsed, pdfPostProcessOptions{removeTOC: true})- if res.Err != nil {- t.Fatalf("pdfParseResultToJSONWithOptions: %v", res.Err)- }- if len(res.JSON) != 1 {- t.Fatalf("JSON len = %d, want 1", len(res.JSON))- }- if got, want := res.JSON[0]["text"], "Body"; got != want {- t.Fatalf("JSON[0].text = %v, want %v", got, want)- }-}-func TestPDFParser_ConfigureFromSetup(t *testing.T) {p := NewPDFParser()p.ConfigureFromSetup(map[string]any{diff --git a/internal/parser/parser/pdf_postprocess.go b/internal/parser/parser/pdf_postprocess.goindex da28424909..4ca4297b32 100644--- a/internal/parser/parser/pdf_postprocess.go+++ b/internal/parser/parser/pdf_postprocess.go@@ -2,61 +2,23 @@ package parserimport ("math"- "regexp""sort""strings"deepdoctype "ragflow/internal/deepdoc/parser/type")-// Substring match to mirror Python's remove_header_footer:-// re.search(r"(header|footer|number)", raw_layout, re.I) (rag/flow/parser/parser.py:754).-// Python matches any layout type CONTAINING one of these words, not just the-// exact token, so a composite label like "page-footer" is also stripped.-var pdfHeaderFooterPattern = regexp.MustCompile(`(?i)header|footer|number`)-var pdfTOCTitlePattern = regexp.MustCompile(`(?i)^(contents|目录|目次|table of contents|致谢|acknowledge)$`)--// pdfTOCEntryPattern matches table-of-contents entry lines: a leader run-// (dots, ellipses, midline ellipses, or middots) followed by a-// page number, optionally with a trailing roman-numeral fragment, e.g.-// "第1章 道可道…………3", "Introduction ....... 12", "……18 I". It extends the-// leader heuristic Python already uses in remove_toc_word-// (rag/flow/parser/utils.py) to the PDF path on both sides, so a TOC without-// a "目录"/"contents" heading is still dropped. Wide-space runs are NOT a-// leader here, mirroring pdfTOCEntryAnywherePattern: they are too common in-// prose, so a line merely ending in " 12" is not a TOC entry.-var pdfTOCEntryPattern = regexp.MustCompile(`(\.{2,}|…{2,}|⋯{2,}|·{2,})\s*\d{1,4}\s*[IVXLCivxlc]{0,5}\s*$`)--// pdfTOCEntryAnywherePattern is the non-anchored form of pdfTOCEntryPattern.-// DeepDoc merges the lines of a text block into ONE section whose text joins-// the lines with spaces, so a TOC page usually becomes a single section whose-// trailing text may be a page marker or a footer watermark instead of an-// entry — the anchored pattern alone never fires for such a section. Counting-// explicit leader runs (wide-space leaders excluded, they are too common in-// prose) identifies those merged TOC blocks.-var pdfTOCEntryAnywherePattern = regexp.MustCompile(`(\.{2,}|…{2,}|⋯{2,}|·{2,})\s*\d{1,4}\s*[IVXLCivxlc]{0,5}`)--// pdfTOCBarePageRefPattern matches a line that consists solely of a leader-// run plus a page number (and optional roman-numeral fragment), e.g.-// "…………………39", "………18 I". In wrapped TOCs the entry title and its leader/page-// number are split onto separate lines, leaving such a bare reference behind-// the title line.-var pdfTOCBarePageRefPattern = regexp.MustCompile(`^[\s.·…⋯]*\d{1,4}\s*[IVXLCivxlc]{0,5}[\s.。·…⋯]*$`)--// pdfTOCTitlePrefixPattern matches lines that open like a TOC entry title:-// a chapter/section marker or a common front/back-matter heading. Used only-// to pair a title line with the bare leader+page-number line that follows it,-// so regular prose is never dropped.-var pdfTOCTitlePrefixPattern = regexp.MustCompile(`(?i)^(第\s*[0-90-9一二三四五六七八九十百千]+\s*[章节篇部册卷回讲]|chapter\s+[0-9]+|appendix\s+[a-z0-9]+|section\s+[0-9]+|前言|序言|引言|导论|绪论|后记|结语|附录|索引|参考文献|结论|致谢|acknowledgements?|acknowledgments?)`)-+// pdfPostProcessOptions carries the parser backend's post-process switches.+// TOC and header/footer removal are handled earlier, at box level inside the+// DeepDoc layout pipeline (Parser.buildLayout), where the leader-dot and+// per-box geometry signals those detectors rely on are still intact; they are+// intentionally not repeated here.type pdfPostProcessOptions struct {outputFormat stringpageWidth float64zoom float64enableMultiColumn boolflattenMediaToText bool- removeTOC bool- removeHeaderFooter bool}func applyPDFPostProcess(result *deepdoctype.ParseResult, opts pdfPostProcessOptions) {@@ -67,14 +29,7 @@ func applyPDFPostProcess(result *deepdoctype.ParseResult, opts pdfPostProcessOptif opts.enableMultiColumn && opts.pageWidth > 0 {reorderPDFMultiColumn(result, opts.pageWidth, opts.zoom)}- if opts.removeTOC {- applyRemoveTOC(result)- }normalizePDFLayoutTypes(result)- if opts.removeHeaderFooter {- filterPDFHeaderFooter(result)- removePDFRunningHeaderFooter(result)- }assignPDFDocTypeKeywords(result, opts.flattenMediaToText)}@@ -88,128 +43,6 @@ func normalizePDFLayoutTypes(result *deepdoctype.ParseResult) {}}-func filterPDFHeaderFooter(result *deepdoctype.ParseResult) {- filtered := result.Sections[:0]- for _, s := range result.Sections {- if pdfHeaderFooterPattern.MatchString(strings.TrimSpace(s.LayoutType)) {- continue- }- filtered = append(filtered, s)- }- result.Sections = filtered-}--const (- // pdfHeaderZoneRatio / pdfFooterZoneRatio bound the page zones where- // running headers/footers live: sections sitting entirely in the top- // or bottom fraction of a page are candidates.- pdfHeaderZoneRatio = 0.10- pdfFooterZoneRatio = 0.90- // pdfMinPagesForHeaderFooter guards against false positives on short- // documents: cross-page repetition only means "running header/footer"- // when the document has enough pages.- pdfMinPagesForHeaderFooter = 3-)--var pdfRunningDigitPattern = regexp.MustCompile(`\d+`)--// normalizePDFRunningText collapses whitespace and replaces digit runs so-// that per-page variants of the same running header/footer ("- 2 -",-// "- 3 -", "Page 4 of 9") share one comparison key.-func normalizePDFRunningText(text string) string {- t := strings.Join(strings.Fields(text), " ")- t = pdfRunningDigitPattern.ReplaceAllString(t, "#")- return strings.ToLower(strings.TrimSpace(t))-}--// removePDFRunningHeaderFooter drops sections whose normalized text repeats-// in the same page zone (top 10% or bottom 10%) on at least half of the-// document's pages — running headers, footers, and page numbers. Every-// rendered page counts toward the total, including blank or image-only-// pages that produced no sections. The-// layout-type filter above only works when a DeepDoc inference service-// (DEEPDOC_URL) types header/footer regions; in default deployments there-// is none, every section stays "text", and 页眉页脚 removal silently does-// nothing. This positional frequency pass makes the option work without an-// inference service. Repetition — not position alone — is the guard that-// keeps genuine body text: a section must recur on >= max(2, pages/2) pages-// in the same zone to be removed.-func removePDFRunningHeaderFooter(result *deepdoctype.ParseResult) {- sections := result.Sections- if len(sections) == 0 || len(result.PageHeight) == 0 {- return- }- numPages := len(result.PageHeight)- if numPages < pdfMinPagesForHeaderFooter {- return- }- type zoneKey struct {- header bool- text string- }- keyPages := make(map[zoneKey]map[int]struct{})- keySections := make(map[zoneKey][]int)- for i := range sections {- s := §ions[i]- if layout := strings.TrimSpace(s.LayoutType); layout != "" && layout != deepdoctype.LayoutTypeText {- continue- }- if len(s.Positions) == 0 || len(s.Positions[0].PageNumbers) == 0 {- continue- }- page := s.Positions[0].PageNumbers[0]- pageHeight := result.PageHeight[page]- if pageHeight <= 0 {- continue- }- top, bottom := s.Positions[0].Top, s.Positions[0].Bottom- if bottom <= top {- continue- }- var isHeader, isFooter bool- switch {- case bottom <= pageHeight*pdfHeaderZoneRatio:- isHeader = true- case top >= pageHeight*pdfFooterZoneRatio:- isFooter = true- }- if !isHeader && !isFooter {- continue- }- norm := normalizePDFRunningText(s.Text)- if norm == "" {- continue- }- key := zoneKey{header: isHeader, text: norm}- if keyPages[key] == nil {- keyPages[key] = make(map[int]struct{})- }- keyPages[key][page] = struct{}{}- keySections[key] = append(keySections[key], i)- }- drop := make(map[int]struct{})- for key, pages := range keyPages {- n := len(pages)- if n < 2 || n*2 < numPages {- continue- }- for _, i := range keySections[key] {- drop[i] = struct{}{}- }- }- if len(drop) == 0 {- return- }- filtered := sections[:0]- for i, s := range sections {- if _, ok := drop[i]; ok {- continue- }- filtered = append(filtered, s)- }- result.Sections = filtered-}-func assignPDFDocTypeKeywords(result *deepdoctype.ParseResult, flatten bool) {for i := range result.Sections {section := &result.Sections[i]@@ -255,209 +88,6 @@ func sortSectionsByPosition(result *deepdoctype.ParseResult) {})}-// applyRemoveTOC mirrors Python parser.py:663-681 three-way dispatch:-// - No outlines → pattern-based remove_toc on all sections-// - First outline on page 1 → outline-based remove_toc_pdf, then the entry-// filter: the outline pass only drops pages when an outline title names-// the TOC ("目录"/"contents"), so a headingless TOC must still run-// through filterPDFTOCEntries instead of being left in place-// - First outline after page 1 → pattern-based on pages before the first outline-func applyRemoveTOC(result *deepdoctype.ParseResult) {- if result == nil {- return- }- outlines := result.Outlines- if len(outlines) == 0 {- removePDFTOC(result)- return- }- firstOutlinePage := outlines[0].PageNumber- if firstOutlinePage <= 1 {- removePDFTOCByOutlines(result, outlines)- result.Sections = filterPDFTOCEntries(result.Sections)- return- }- splitAt := len(result.Sections)- for i, s := range result.Sections {- if firstSectionPage(s) >= firstOutlinePage {- splitAt = i- break- }- }- beforeSplit := &deepdoctype.ParseResult{Sections: result.Sections[:splitAt]}- removePDFTOC(beforeSplit)- result.Sections = append(beforeSplit.Sections, result.Sections[splitAt:]...)-}--func removePDFTOC(result *deepdoctype.ParseResult) {- sections := result.Sections- i := 0- for i < len(sections) {- text := sectionText(sections[i])- if !pdfTOCTitlePattern.MatchString(strings.ToLower(strings.TrimSpace(text))) {- i++- continue- }- sections = append(sections[:i], sections[i+1:]...)- if i >= len(sections) {- break- }- prefix := sectionTextPrefix(sections[i], 3)- for prefix == "" {- sections = append(sections[:i], sections[i+1:]...)- if i >= len(sections) {- break- }- prefix = sectionTextPrefix(sections[i], 3)- }- if i >= len(sections) || prefix == "" {- break- }- // The first entry is formatted (leader run plus page number): the- // table is entry-shaped and the prefix scan below must not run —- // with a single formatted entry that scan would delete body text up- // to the next line sharing the entry's prefix. Drop the entry and- // resume the title scan; filterPDFTOCEntries removes the remaining- // entry lines.- if isPDFTOCEntrySection(sectionText(sections[i])) {- sections = append(sections[:i], sections[i+1:]...)- continue- }- sections = append(sections[:i], sections[i+1:]...)- if i >= len(sections) || prefix == "" {- break- }- for j := i; j < len(sections) && j < i+128; j++ {- if !strings.HasPrefix(sectionText(sections[j]), prefix) {- continue- }- sections = append(sections[:i], sections[j:]...)- break- }- }- // The anchored pass above needs a "目录"/"contents" section; a TOC whose- // pages carry no such heading survives it entirely. Drop the remaining- // leader+page-number entry lines, mirroring remove_toc in- // rag/flow/parser/utils.py.- sections = filterPDFTOCEntries(sections)- result.Sections = sections-}--// filterPDFTOCEntries removes table-of-contents sections from the stream,-// reusing the backing array. It goes beyond the per-item anchored match that-// Python's remove_toc (rag/flow/parser/utils.py) applies, because the Go-// DeepDoc layout merges a whole TOC block into one section and wrapped TOCs-// split an entry's leader/page number onto its own line:-//-// - a section ending in a leader+page-number run is an entry (anchored-// pattern), as before;-// - a section containing two or more leader+page-number runs is a merged-// TOC block even when its trailing text (page marker, roman numeral or-// footer watermark) is not an entry itself;-// - when a dropped section is a bare leader+page-number reference, the-// preceding kept section is dropped too if it opens like a TOC entry-// title ("第28章 ………" followed by "………39"), pairing the wrapped entry.-//-// Regular prose is kept: a lone leader+number run inside a paragraph matches-// neither the anchored pattern nor the two-run threshold, and title pairing-// only fires on a dropped bare page reference.-func filterPDFTOCEntries(sections []deepdoctype.Section) []deepdoctype.Section {- filtered := make([]deepdoctype.Section, 0, len(sections))- for _, s := range sections {- text := sectionText(s)- if !isPDFTOCEntrySection(text) {- filtered = append(filtered, s)- continue- }- if pdfTOCBarePageRefPattern.MatchString(text) && len(filtered) > 0 {- if isPDFTOCTitleCandidate(sectionText(filtered[len(filtered)-1])) {- filtered = filtered[:len(filtered)-1]- }- }- }- return filtered-}--// isPDFTOCEntrySection reports whether the section text is (or is dominated-// by) table-of-contents entries.-func isPDFTOCEntrySection(text string) bool {- if pdfTOCEntryPattern.MatchString(text) {- return true- }- return len(pdfTOCEntryAnywherePattern.FindAllString(text, -1)) >= 2-}--// isPDFTOCTitleCandidate reports whether a section reads like the title line-// of a TOC entry whose leader/page number lives on the following line: it-// must open with a chapter/section marker and must not contain a leader run-// itself.-func isPDFTOCTitleCandidate(text string) bool {- if !pdfTOCTitlePrefixPattern.MatchString(text) {- return false- }- return !pdfTOCEntryAnywherePattern.MatchString(text)-}--func sectionText(s deepdoctype.Section) string {- return strings.TrimSpace(s.Text)-}--func sectionTextPrefix(s deepdoctype.Section, n int) string {- text := sectionText(s)- if len(text) < n {- return text- }- return text[:n]-}-func removePDFTOCByOutlines(result *deepdoctype.ParseResult, outlines []deepdoctype.Outline) {- if result == nil || len(outlines) == 0 {- return- }- tocPage, contentPage := findPDFTOCPageRange(outlines)- if contentPage <= tocPage {- return- }- filtered := result.Sections[:0]- for _, s := range result.Sections {- page := firstSectionPage(s)- if page >= tocPage && page < contentPage {- continue- }- filtered = append(filtered, s)- }- result.Sections = filtered-}--func findPDFTOCPageRange(outlines []deepdoctype.Outline) (tocPage, contentPage int) {-outer:- for i, o := range outlines {- title := strings.TrimSpace(o.Title)- if idx := strings.Index(title, "@@"); idx >= 0 {- title = strings.TrimSpace(title[:idx])- }- if !pdfTOCTitlePattern.MatchString(strings.ToLower(title)) {- continue- }- tocPage = o.PageNumber- for _, next := range outlines[i+1:] {- if next.Level != o.Level {- continue- }- nextTitle := strings.TrimSpace(next.Title)- if idx := strings.Index(nextTitle, "@@"); idx >= 0 {- nextTitle = strings.TrimSpace(nextTitle[:idx])- }- if pdfTOCTitlePattern.MatchString(strings.ToLower(nextTitle)) {- continue- }- contentPage = next.PageNumber- break outer- }- break- }- return-}-func reorderPDFMultiColumn(result *deepdoctype.ParseResult, pageWidth, _ float64) {if result == nil || len(result.Sections) < 2 {returndiff --git a/internal/parser/parser/pdf_postprocess_test.go b/internal/parser/parser/pdf_postprocess_test.goindex 0a62f4817c..bef11936a0 100644--- a/internal/parser/parser/pdf_postprocess_test.go+++ b/internal/parser/parser/pdf_postprocess_test.go@@ -1,8 +1,6 @@package parserimport (- "slices"- "strings""testing"deepdoctype "ragflow/internal/deepdoc/parser/type"@@ -79,510 +77,6 @@ func TestApplyPDFPostProcess_FlattenMediaKeepsImagesButMarksText(t *testing.T) {}}-func TestApplyPDFPostProcess_HeaderFooterFilteringIsOptional(t *testing.T) {- result := &deepdoctype.ParseResult{- Sections: []deepdoctype.Section{- {Text: "header", LayoutType: "header"},- {Text: "body", LayoutType: "text"},- },- }- applyPDFPostProcess(result, pdfPostProcessOptions{})- if len(result.Sections) != 2 {- t.Fatalf("len(Sections) = %d, want 2 when removeHeaderFooter is false", len(result.Sections))- }-- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})- if len(result.Sections) != 1 {- t.Fatalf("len(Sections) = %d, want 1 when removeHeaderFooter is true", len(result.Sections))- }- if got, want := result.Sections[0].Text, "body"; got != want {- t.Fatalf("remaining section = %q, want %q", got, want)- }-}--func TestApplyPDFPostProcess_RemoveTOCRemovesAllDotLeaderEntries(t *testing.T) {- result := &deepdoctype.ParseResult{- Sections: []deepdoctype.Section{- {Text: "Contents"},- {Text: "Chapter One Introduction .............. 2"},- {Text: "Chapter Two Methods ................... 3"},- {Text: "Chapter Three Results ................. 4"},- {Text: "Chapter One Introduction"},- {Text: "Body text that must survive parsing."},- {Text: "Chapter Two Methods"},- {Text: "More body text."},- },- }- applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})- var kept []string- for _, s := range result.Sections {- kept = append(kept, s.Text)- }- want := []string{- "Chapter One Introduction",- "Body text that must survive parsing.",- "Chapter Two Methods",- "More body text.",- }- if len(kept) != len(want) {- t.Fatalf("kept = %v, want %v", kept, want)- }- for i := range want {- if kept[i] != want[i] {- t.Fatalf("kept = %v, want %v", kept, want)- }- }-}--func TestApplyPDFPostProcess_RemoveTOCPrefixScanWithoutEntryPattern(t *testing.T) {- // Pattern-less TOC entries take the prefix scan: title and first entry- // go, then everything up to the first line sharing the first entry's- // prefix (the real body heading).- result := &deepdoctype.ParseResult{- Sections: []deepdoctype.Section{- {Text: "Contents"},- {Text: "1. Introduction"},- {Text: "2. Background"},- {Text: "1. Introduction"},- {Text: "Body text."},- },- }- applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})- var kept []string- for _, s := range result.Sections {- kept = append(kept, s.Text)- }- want := []string{"1. Introduction", "Body text."}- if len(kept) != len(want) {- t.Fatalf("kept = %v, want %v", kept, want)- }- for i := range want {- if kept[i] != want[i] {- t.Fatalf("kept = %v, want %v", kept, want)- }- }-}--func TestApplyPDFPostProcess_RemoveTOCSingleFormattedEntryKeepsBody(t *testing.T) {- // Only the first entry after the TOC title is formatted. It must count- // as a consumed entry so the prefix scan never runs: that scan would- // otherwise delete every line up to the next "Cha"-prefixed heading- // ("Chapter Two"), dropping the body text between the TOC and it.- result := &deepdoctype.ParseResult{- Sections: []deepdoctype.Section{- {Text: "Contents"},- {Text: "Chapter One ..... 2"},- {Text: "Body A"},- {Text: "Body B"},- {Text: "Chapter Two"},- },- }- applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})- var kept []string- for _, s := range result.Sections {- kept = append(kept, s.Text)- }- want := []string{"Body A", "Body B", "Chapter Two"}- if len(kept) != len(want) {- t.Fatalf("kept = %v, want %v", kept, want)- }- for i := range want {- if kept[i] != want[i] {- t.Fatalf("kept = %v, want %v", kept, want)- }- }-}--func TestApplyPDFPostProcess_RemoveRunningHeaderFooterWithoutDLA(t *testing.T) {- const pageHeight = 842.0- result := &deepdoctype.ParseResult{- PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight},- Sections: []deepdoctype.Section{- makePDFSection("Contents", "", 0, 72, 158, 40, 60),- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 1, 72, 210, 21, 30),- makePDFSection("Chapter One Introduction", "", 1, 72, 206, 88, 100),- makePDFSection("Body text one.", "", 1, 72, 363, 124, 136),- makePDFSection("- 2 -", "", 1, 289, 305, 807, 817),- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 2, 72, 210, 21, 30),- makePDFSection("Chapter Two Methods", "", 2, 72, 190, 88, 100),- makePDFSection("Body text two.", "", 2, 72, 374, 124, 136),- makePDFSection("- 3 -", "", 2, 289, 305, 807, 817),- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 3, 72, 210, 21, 30),- makePDFSection("Chapter Three Results", "", 3, 72, 192, 88, 100),- makePDFSection("Body text three.", "", 3, 72, 367, 124, 136),- makePDFSection("- 4 -", "", 3, 289, 305, 807, 817),- },- }- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})- for _, s := range result.Sections {- if strings.Contains(s.Text, "CONFIDENTIAL") {- t.Fatalf("running header survived: %q", s.Text)- }- if strings.HasPrefix(strings.TrimSpace(s.Text), "- ") {- t.Fatalf("page-number footer survived: %q", s.Text)- }- }- if len(result.Sections) != 7 {- t.Fatalf("len(Sections) = %d, want 7 (body content preserved)", len(result.Sections))- }-}--func TestApplyPDFPostProcess_RunningHeaderFooterKeepsUniqueZoneText(t *testing.T) {- const pageHeight = 842.0- result := &deepdoctype.ParseResult{- PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight},- Sections: []deepdoctype.Section{- // Different top-of-page text on every page: not a running header.- makePDFSection("Unique top line one", "", 0, 72, 210, 21, 30),- makePDFSection("Body one.", "", 0, 72, 363, 124, 136),- makePDFSection("Unique top line two", "", 1, 72, 210, 21, 30),- makePDFSection("Body two.", "", 1, 72, 363, 124, 136),- makePDFSection("Unique top line three", "", 2, 72, 210, 21, 30),- makePDFSection("Body three.", "", 2, 72, 363, 124, 136),- },- }- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})- if len(result.Sections) != 6 {- t.Fatalf("len(Sections) = %d, want 6 (unique zone text must survive)", len(result.Sections))- }-}--func TestApplyPDFPostProcess_RunningHeaderFooterSkipsShortDocuments(t *testing.T) {- const pageHeight = 842.0- result := &deepdoctype.ParseResult{- PageHeight: map[int]float64{0: pageHeight, 1: pageHeight},- Sections: []deepdoctype.Section{- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 0, 72, 210, 21, 30),- makePDFSection("Body one.", "", 0, 72, 363, 124, 136),- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 1, 72, 210, 21, 30),- makePDFSection("Body two.", "", 1, 72, 363, 124, 136),- },- }- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})- if len(result.Sections) != 4 {- t.Fatalf("len(Sections) = %d, want 4 (2-page doc must not be touched)", len(result.Sections))- }-}--func TestApplyPDFPostProcess_RunningHeaderFooterCountsAllRenderedPages(t *testing.T) {- const pageHeight = 842.0- // Pages 3 and 4 are blank or image-only: they produced no sections but- // still count toward the page total. A candidate repeated on 2 of the 5- // rendered pages occurs on fewer than half of the document's pages and- // must survive.- result := &deepdoctype.ParseResult{- PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight, 4: pageHeight},- Sections: []deepdoctype.Section{- makePDFSection("Chapter One Introduction", "", 0, 72, 206, 88, 100),- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 1, 72, 210, 21, 30),- makePDFSection("Body text one.", "", 1, 72, 363, 124, 136),- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 2, 72, 210, 21, 30),- makePDFSection("Body text two.", "", 2, 72, 374, 124, 136),- },- }- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})- if len(result.Sections) != 5 {- t.Fatalf("len(Sections) = %d, want 5 (2 of 5 rendered pages is below the repetition threshold)", len(result.Sections))- }-- // The same header on 3 of the 5 rendered pages meets the threshold and- // is removed even though two pages contributed no sections.- result = &deepdoctype.ParseResult{- PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight, 4: pageHeight},- Sections: []deepdoctype.Section{- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 0, 72, 210, 21, 30),- makePDFSection("Body text one.", "", 0, 72, 363, 124, 136),- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 1, 72, 210, 21, 30),- makePDFSection("Body text two.", "", 1, 72, 363, 124, 136),- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 2, 72, 210, 21, 30),- makePDFSection("Body text three.", "", 2, 72, 367, 124, 136),- },- }- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})- if len(result.Sections) != 3 {- t.Fatalf("len(Sections) = %d, want 3 (running header removed on 3 of 5 rendered pages)", len(result.Sections))- }- for _, s := range result.Sections {- if strings.Contains(s.Text, "CONFIDENTIAL") {- t.Fatalf("running header survived: %q", s.Text)- }- }-}--func TestApplyPDFPostProcess_RunningHeaderFooterRequiresFullZone(t *testing.T) {- const pageHeight = 842.0- // A section that merely starts in the top 10% but extends past it- // (bottom 200 > 842*0.10) is body content, not a header candidate —- // even when it repeats on every page. Same for a section that ends in- // the bottom 10% but starts above it (top 700 < 842*0.90).- result := &deepdoctype.ParseResult{- PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight},- Sections: []deepdoctype.Section{- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 0, 72, 210, 40, 200),- makePDFSection("Long legal disclaimer note", "", 0, 72, 363, 700, 800),- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 1, 72, 210, 40, 200),- makePDFSection("Long legal disclaimer note", "", 1, 72, 363, 700, 800),- makePDFSection("CONFIDENTIAL DRAFT HEADER", "", 2, 72, 210, 40, 200),- makePDFSection("Long legal disclaimer note", "", 2, 72, 363, 700, 800),- },- }- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})- if len(result.Sections) != 6 {- t.Fatalf("len(Sections) = %d, want 6 (sections crossing the zone boundary must survive)", len(result.Sections))- }-}--func TestApplyPDFPostProcess_RunningHeaderFooterNeedsRepetitionNotPosition(t *testing.T) {- const pageHeight = 842.0- // Body blocks sitting fully inside the top 10% of every page but never- // repeating: position alone must not remove them — the guard requires- // the same normalized text on >= half of the rendered pages.- result := &deepdoctype.ParseResult{- PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight},- Sections: []deepdoctype.Section{- makePDFSection("Quarterly results summary", "", 0, 72, 300, 21, 60),- makePDFSection("Ordinary body text on page zero.", "", 0, 72, 363, 124, 136),- makePDFSection("Annual meeting minutes", "", 1, 72, 300, 21, 60),- makePDFSection("Ordinary body text on page one.", "", 1, 72, 363, 124, 136),- makePDFSection("Risk assessment overview", "", 2, 72, 300, 21, 60),- makePDFSection("Ordinary body text on page two.", "", 2, 72, 363, 124, 136),- makePDFSection("Compliance audit findings", "", 3, 72, 300, 21, 60),- makePDFSection("Ordinary body text on page three.", "", 3, 72, 363, 124, 136),- },- }- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})- if len(result.Sections) != 8 {- t.Fatalf("len(Sections) = %d, want 8 (in-zone but non-repeating text must survive)", len(result.Sections))- }-- // Repetition counts distinct pages, not occurrences: the same text- // twice on one page still covers a single page of the document.- result = &deepdoctype.ParseResult{- PageHeight: map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight},- Sections: []deepdoctype.Section{- makePDFSection("Repeated block", "", 0, 72, 300, 21, 40),- makePDFSection("Repeated block", "", 0, 72, 300, 45, 64),- makePDFSection("Body text page one.", "", 1, 72, 363, 124, 136),- makePDFSection("Body text page two.", "", 2, 72, 363, 124, 136),- },- }- applyPDFPostProcess(result, pdfPostProcessOptions{removeHeaderFooter: true})- if len(result.Sections) != 4 {- t.Fatalf("len(Sections) = %d, want 4 (two occurrences on one page count as one page)", len(result.Sections))- }-}--func TestApplyPDFPostProcess_RemoveTOCByOutlines(t *testing.T) {- result := &deepdoctype.ParseResult{- Sections: []deepdoctype.Section{- makePDFSection("目录", "text", 1, 50, 550, 100, 120),- makePDFSection("章节列表", "text", 2, 50, 550, 120, 140),- makePDFSection("正文", "text", 3, 50, 550, 100, 120),- },- Outlines: []deepdoctype.Outline{- {Title: "目录", Level: 0, PageNumber: 1},- {Title: "第一章", Level: 0, PageNumber: 3},- },- }- applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})- if len(result.Sections) != 1 {- t.Fatalf("len(Sections) = %d, want 1", len(result.Sections))- }- if got, want := result.Sections[0].Text, "正文"; got != want {- t.Fatalf("remaining section = %q, want %q", got, want)- }-}--// TestApplyPDFPostProcess_RemoveTOCEntryLines covers TOC pages that carry no-// "目录"/"contents" heading: the removal must still drop the leader+page-// number entry lines while keeping regular content, and only when removeTOC-// is enabled.-func TestApplyPDFPostProcess_RemoveTOCEntryLines(t *testing.T) {- sections := []deepdoctype.Section{- makePDFSection("《道德经》全文及翻译", "text", 1, 50, 550, 100, 120),- makePDFSection("木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879", "text", 1, 50, 550, 120, 140),- makePDFSection("前言:……………………………………………………………………1", "text", 2, 50, 550, 100, 120),- makePDFSection("第1章 “道”…………………………………………………………3", "text", 2, 50, 550, 120, 140),- makePDFSection("第2章 圣人居无为之事,行不言之教…………………………4", "text", 2, 50, 550, 140, 160),- makePDFSection("第13章 以身为天下,可寄/托天下……………………………18 I", "text", 3, 50, 550, 100, 120),- makePDFSection("第17章 太上,不知有之;功成事遂,百姓皆谓“我自然”..25", "text", 3, 50, 550, 120, 140),- makePDFSection("Introduction to the Dao ....... 12", "text", 3, 50, 550, 140, 160),- makePDFSection("II", "text", 3, 50, 550, 160, 180),- makePDFSection("前言:本文是《道德经》的白话翻译,供读者参考。", "text", 4, 50, 550, 100, 120),- makePDFSection("道可道,非常道。名可名,非常名。", "text", 4, 50, 550, 120, 140),- }-- untouched := &deepdoctype.ParseResult{Sections: append([]deepdoctype.Section(nil), sections...)}- applyPDFPostProcess(untouched, pdfPostProcessOptions{removeTOC: false})- if len(untouched.Sections) != len(sections) {- t.Fatalf("len(Sections) = %d, want %d when removeTOC is false", len(untouched.Sections), len(sections))- }-- result := &deepdoctype.ParseResult{Sections: append([]deepdoctype.Section(nil), sections...)}- applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})- want := []string{- "《道德经》全文及翻译",- "木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879",- "II",- "前言:本文是《道德经》的白话翻译,供读者参考。",- "道可道,非常道。名可名,非常名。",- }- if len(result.Sections) != len(want) {- t.Fatalf("len(Sections) = %d, want %d; got %v", len(result.Sections), len(want), sectionTexts(result.Sections))- }- for i := range want {- if got := result.Sections[i].Text; got != want[i] {- t.Fatalf("Sections[%d].Text = %q, want %q", i, got, want[i])- }- }-}--// TestApplyPDFPostProcess_RemoveTOCEntryLinesWithPageOneOutlines covers the-// book-PDF shape: the first outline sits on page 1 and none of its titles-// names the TOC ("目录"/"contents"), so removePDFTOCByOutlines finds no page-// range and is a no-op. The leader+page-number entry filter must still run on-// this dispatch branch, otherwise a TOC without a 目录 heading survives it.-func TestApplyPDFPostProcess_RemoveTOCEntryLinesWithPageOneOutlines(t *testing.T) {- result := &deepdoctype.ParseResult{- Sections: []deepdoctype.Section{- makePDFSection("《道德经》全文及翻译", "text", 1, 50, 550, 100, 120),- makePDFSection("前言:……………………………………………………………………1", "text", 2, 50, 550, 100, 120),- makePDFSection("第1章 “道”…………………………………………………………3", "text", 2, 50, 550, 120, 140),- makePDFSection("Introduction to the Dao ....... 12", "text", 2, 50, 550, 140, 160),- makePDFSection("道可道,非常道。名可名,非常名。", "text", 3, 50, 550, 100, 120),- },- Outlines: []deepdoctype.Outline{- {Title: "前言", Level: 0, PageNumber: 1},- {Title: "Chapter 1", Level: 1, PageNumber: 3},- },- }- applyPDFPostProcess(result, pdfPostProcessOptions{removeTOC: true})- want := []string{"《道德经》全文及翻译", "道可道,非常道。名可名,非常名。"}- if !slices.Equal(sectionTexts(result.Sections), want) {- t.Fatalf("sections = %v, want %v", sectionTexts(result.Sections), want)- }-}--func sectionTexts(sections []deepdoctype.Section) []string {- texts := make([]string, 0, len(sections))- for _, s := range sections {- texts = append(texts, s.Text)- }- return texts-}--// TestFilterPDFTOCEntries_MergedBlockTrailingNonEntry covers a TOC page that-// DeepDoc merges into ONE section whose trailing text is not an entry (a-// footer watermark), so the anchored entry pattern alone never fires. The-// block must still be dropped because it is dominated by leader+page-number-// runs (issue: agent workflow PDF "remove original table of contents" not-// taking effect).-func TestFilterPDFTOCEntries_MergedBlockTrailingNonEntry(t *testing.T) {- sections := []deepdoctype.Section{- makePDFSection("《道德经》全文及翻译", "text", 1, 50, 550, 100, 120),- makePDFSection("前言:………………………………………………1 第1章 “道”………………………………………………3 第2章 圣人居无为之事,行不言之教……………………4 第3章 无为而治…………………………………………6 木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879", "text", 1, 50, 550, 120, 300),- makePDFSection("道可道,非常道。名可名,非常名。", "text", 3, 50, 550, 100, 120),- }- got := filterPDFTOCEntries(sections)- want := []string{"《道德经》全文及翻译", "道可道,非常道。名可名,非常名。"}- if !slices.Equal(sectionTexts(got), want) {- t.Fatalf("sections = %v, want %v", sectionTexts(got), want)- }-}--// TestFilterPDFTOCEntries_WrappedEntryTitlePairsPageRef covers wrapped TOC-// entries whose leader+page number is split onto its own line/section: the-// bare page reference drops the preceding title line, but a preceding-// non-title line (prose, watermark) is kept, and a lone leader run inside-// prose is not enough to drop anything.-func TestFilterPDFTOCEntries_WrappedEntryTitlePairsPageRef(t *testing.T) {- sections := []deepdoctype.Section{- makePDFSection("第27章 不贵其师,不爱其资,虽智大迷,是谓要妙", "text", 2, 50, 550, 100, 120),- makePDFSection("…………………………………………………………………………………………39", "text", 2, 50, 550, 120, 140),- makePDFSection("木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879", "text", 2, 50, 550, 140, 160),- makePDFSection("第28章 朴散则为器,圣人用之,则为官长,故大制不割??", "text", 2, 50, 550, 160, 180),- makePDFSection("………39 II", "text", 2, 50, 550, 180, 200),- makePDFSection("第1章", "text", 3, 50, 550, 100, 120),- makePDFSection("道可道,非常道。名可名,非常名。", "text", 3, 50, 550, 120, 140),- }- got := filterPDFTOCEntries(sections)- want := []string{- "木瓜树 更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879",- "第1章",- "道可道,非常道。名可名,非常名。",- }- if !slices.Equal(sectionTexts(got), want) {- t.Fatalf("sections = %v, want %v", sectionTexts(got), want)- }-}--// TestFilterPDFTOCEntries_KeepsProseWithSingleLeaderRun pins the guard rails:-// prose containing one leader+number mention stays, and a bare page-// reference preceded by prose (no chapter/section title) keeps the prose.-func TestFilterPDFTOCEntries_KeepsProseWithSingleLeaderRun(t *testing.T) {- sections := []deepdoctype.Section{- makePDFSection("更多说明见第三章……12 的表格。", "text", 1, 50, 550, 100, 120),- makePDFSection("………34", "text", 1, 50, 550, 120, 140),- makePDFSection("天下皆知美之为美,斯恶已。", "text", 2, 50, 550, 100, 120),- }- got := filterPDFTOCEntries(sections)- want := []string{- "更多说明见第三章……12 的表格。",- "天下皆知美之为美,斯恶已。",- }- if !slices.Equal(sectionTexts(got), want) {- t.Fatalf("sections = %v, want %v", sectionTexts(got), want)- }-}--// TestFilterPDFTOCEntries_KeepsProseEndingInWideSpaces pins that a wide-space-// run is not a TOC leader: prose ending in " 12" survives, while the same-// shape with a dot leader is a real entry and is dropped.-func TestFilterPDFTOCEntries_KeepsProseEndingInWideSpaces(t *testing.T) {- sections := []deepdoctype.Section{- makePDFSection("See the installation guide for details 12", "text", 1, 50, 550, 100, 120),- makePDFSection("Troubleshooting ....... 27", "text", 2, 50, 550, 100, 120),- }- got := filterPDFTOCEntries(sections)- want := []string{"See the installation guide for details 12"}- if !slices.Equal(sectionTexts(got), want) {- t.Fatalf("sections = %v, want %v", sectionTexts(got), want)- }-}--// TestFilterPDFTOCEntries_TwoRunProseTradeOff pins the accepted trade-off of-// the merged-block heuristic: prose quoting two leader+number references is-// dropped, because a DeepDoc-merged TOC page has exactly that shape and the-// anchored pattern alone never fires for it. A single run keeps the prose-// (see KeepsProseWithSingleLeaderRun); raising the threshold would-// under-delete real merged TOC blocks.-func TestFilterPDFTOCEntries_TwoRunProseTradeOff(t *testing.T) {- sections := []deepdoctype.Section{- makePDFSection("见第三章……12 和第五章……34 的说明。", "text", 1, 50, 550, 100, 120),- }- got := filterPDFTOCEntries(sections)- if len(got) != 0 {- t.Fatalf("sections = %v, want empty: two leader+number runs read as a merged TOC block", sectionTexts(got))- }-}--// TestFilterPDFTOCEntries_ChapterTitleWithoutBarePageRefSurvives pins the-// guard of the wrapped-entry pairing: a "第N章"/"Chapter N" line is only-// dropped when the following section is a bare leader+page-number reference-// (see WrappedEntryTitlePairsPageRef); followed by ordinary prose it is body-// content and must stay.-func TestFilterPDFTOCEntries_ChapterTitleWithoutBarePageRefSurvives(t *testing.T) {- sections := []deepdoctype.Section{- makePDFSection("第1章 道可道,非常道", "text", 3, 50, 550, 100, 120),- makePDFSection("道可道,非常道。名可名,非常名。", "text", 3, 50, 550, 120, 140),- }- got := filterPDFTOCEntries(sections)- want := []string{"第1章 道可道,非常道", "道可道,非常道。名可名,非常名。"}- if !slices.Equal(sectionTexts(got), want) {- t.Fatalf("sections = %v, want %v", sectionTexts(got), want)- }-}-func TestApplyPDFPostProcess_ReordersMultiColumnText(t *testing.T) {cases := []struct {name string@@ -605,38 +99,3 @@ func TestApplyPDFPostProcess_ReordersMultiColumnText(t *testing.T) {}}}--// TestFilterPDFHeaderFooter_SubstringMatch pins #5: Python's remove_header_footer-// uses a substring match re.search(r"(header|footer|number)", ...) (rag/flow/parser/parser.py:754),-// while Go used an anchored exact match ^(header|footer|number)$. A layout type-// that merely CONTAINS one of those words (e.g. "page-footer") must be stripped to-// match Python, not silently kept.-func TestFilterPDFHeaderFooter_SubstringMatch(t *testing.T) {- result := &deepdoctype.ParseResult{- Sections: []deepdoctype.Section{- {Text: "real header", LayoutType: "header"},- {Text: "real footer", LayoutType: "footer"},- {Text: "page 1", LayoutType: "number"},- {Text: "a page-footer note", LayoutType: "page-footer"}, // composite -> substring match- {Text: "body text", LayoutType: "text"},- },- }- filterPDFHeaderFooter(result)-- kept := map[string]bool{}- for _, s := range result.Sections {- kept[s.LayoutType] = true- }- for _, lt := range []string{"header", "footer", "number"} {- if kept[lt] {- t.Errorf("#5 header/footer: %q should be stripped", lt)- }- }- // Composite "page-footer" must be stripped by substring match (Python-equivalent).- if kept["page-footer"] {- t.Errorf("#5 header/footer: composite layout type %q should be stripped by substring match", "page-footer")- }- if !kept["text"] {- t.Errorf("#5 header/footer: body text %q should be kept", "text")- }-}