RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(parser): upgrade PDF header/footer removal with 3D cascaded fusion (#19957)
jay77721c779b15d9cd0
diff --git a/internal/deepdoc/parser/pdf/layout/toc_header_footer.go b/internal/deepdoc/parser/pdf/layout/toc_header_footer.goindex 068ed1a4c0..62ebefbb31 100644--- a/internal/deepdoc/parser/pdf/layout/toc_header_footer.go+++ b/internal/deepdoc/parser/pdf/layout/toc_header_footer.go@@ -1,6 +1,7 @@package layoutimport (+ "log/slog""regexp""sort""strings"@@ -324,51 +325,272 @@ const (// running header / footer lives (fractions of the page height).headerZoneRatio = 0.10footerZoneRatio = 0.90- // minHeaderFooterPages guards against false positives on short documents.+ // headerMaxZoneRatio / footerMinZoneRatio define the expanded margin zones+ // when a clear whitespace gap separates the candidate from body content.+ headerMaxZoneRatio = 0.14+ footerMinZoneRatio = 0.86+ // minWhitespaceGapPt is the minimum vertical whitespace gap required to+ // justify expanding the header/footer zone beyond the base 10% ratio.+ minWhitespaceGapPt = 18.0+ // minHeaderFooterPages guards against cross-page false positives on short+ // documents. Deterministic page-number patterns still fire on shorter ones.minHeaderFooterPages = 3+ // minParityCoverageRatio is the minimum fraction of even or odd pages that+ // a running header/footer text must appear on to qualify under the parity track.+ //+ // In formal book typography (LaTeX book documentclass, InDesign book templates),+ // running headers are standardly suppressed on chapter opening pages (\thispagestyle{plain}),+ // blank verso pages before new chapters, and full-page illustrations/tables.+ // As a result, a running header in a real book often covers only 40%-48% of the+ // even or odd pages. A 0.40 threshold captures alternating bilateral book headers+ // while the evenPages >= 3 / oddPages >= 3 count guards against accidental matches.+ minParityCoverageRatio = 0.40)-// RemoveHeaderFooterBoxes drops boxes that are running headers or footers.+var (+ // strictDecoratedPagePattern matches page numbers with clear formatting:+ // - Dash/dot/tilde-wrapped: - 1 -, — 1 —, – 1 –, · 1 ·, • 1 •, ~ 1 ~+ // - Bracket/paren-wrapped: [1], (1)+ // - "Page X", "Page X of Y", "p. X", "X / Y", "X of Y"+ // - Chinese: 第 1 页, 第 1 页 共 10 页, 第 1 页/共 10 页, 第一页, 第一页 共十页+ // - Roman with dashes: - iv -, — iv —, · iv ·+ strictDecoratedPagePattern = regexp.MustCompile(`(?i)^(\s*[-—–~·•]+\s*[0-90-9]+\s*[-—–~·•]+|\s*\[\s*[0-90-9]+\s*\]|\s*\(\s*[0-90-9]+\s*\)|page\s+[0-90-9]+(\s*(of|/)\s*[0-90-9]+)?|p\.\s*[0-90-9]+|[0-90-9]+\s*/\s*[0-90-9]+|[0-90-9]+\s+of\s+[0-90-9]+|第\s*[0-90-9一二三四五六七八九十百]+\s*页(\s*(/\s*共|[,,]\s*共?|[/共])\s*[0-90-9一二三四五六七八九十百]+\s*页?)?|\s*[-—–~·•]+\s*((l|xl|x{1,3})(ix|iv|v?i{1,3}|v)?|ix|iv|v?i{1,3}|v)\s*[-—–~·•]+)$`)++ // strictBareNumberPattern matches lone Arabic digits or Roman numerals: 1, 23, iv, VII.+ strictBareNumberPattern = regexp.MustCompile(`(?i)^([0-90-9]{1,4}|((l|xl|x{1,3})(ix|iv|v?i{1,3}|v)?|ix|iv|v?i{1,3}|v))$`)+)++// computePageGaps computes for candidate margin boxes on a page:+// - gapBelow: distance from b.Bottom to the nearest other box on the page with Top >= b.Bottom - 2.0.+// - gapAbove: distance from b.Top to the nearest other box on the page with Bottom <= b.Top + 2.0.//-// A box is a candidate when it sits entirely in the top headerZoneRatio or-// bottom footerZoneRatio of its page and carries text-like content. Among-// candidates, a (zone, normalized-text) pair that recurs on at least half of-// the document's pages — rounded up, so an odd page count raises the bar rather-// than lowering it — is treated as a running header / footer and removed.+// Performance optimization: only boxes falling into the outer margin zones (top 14% or bottom 14%)+// ever query whitespace gaps. Skipping body-interior boxes reduces the outer loop from M to K (K <= 5),+// lowering complexity from O(M^2) to O(K*M) on dense multi-column pages.+func computePageGaps(boxes []pdf.TextBox, indices []int, pageHeight float64) (map[int]float64, map[int]float64) {+ gapBelow := make(map[int]float64, len(indices))+ gapAbove := make(map[int]float64, len(indices))++ for _, i := range indices {+ bi := boxes[i]+ if bi.Bottom > pageHeight*headerMaxZoneRatio && bi.Top < pageHeight*footerMinZoneRatio {+ continue+ }+ minDistBelow := pageHeight - bi.Bottom+ if minDistBelow < 0 {+ minDistBelow = 0+ }+ maxDistAbove := bi.Top+ if maxDistAbove < 0 {+ maxDistAbove = 0+ }++ for _, j := range indices {+ if i == j {+ continue+ }+ bj := boxes[j]+ if bj.Top >= bi.Bottom-2.0 {+ dist := bj.Top - bi.Bottom+ if dist < 0 {+ dist = 0+ }+ if dist < minDistBelow {+ minDistBelow = dist+ }+ }+ if bj.Bottom <= bi.Top+2.0 {+ dist := bi.Top - bj.Bottom+ if dist < 0 {+ dist = 0+ }+ if dist < maxDistAbove {+ maxDistAbove = dist+ }+ }+ }+ gapBelow[i] = minDistBelow+ gapAbove[i] = maxDistAbove+ }+ return gapBelow, gapAbove+}++// classifyZone determines whether a box sits in a header or footer zone.+func classifyZone(b pdf.TextBox, pageHeight float64, gapAbove, gapBelow float64) string {+ if b.Bottom <= pageHeight*headerZoneRatio {+ return "header"+ }+ if b.Bottom <= pageHeight*headerMaxZoneRatio && gapBelow >= minWhitespaceGapPt {+ return "header"+ }+ if b.Top >= pageHeight*footerZoneRatio {+ return "footer"+ }+ if b.Top >= pageHeight*footerMinZoneRatio && gapAbove >= minWhitespaceGapPt {+ return "footer"+ }+ return ""+}++// isNonTextLayout reports whether a layout type represents non-text elements+// like tables or figures that must not be treated as running headers/footers.+func isNonTextLayout(lt string) bool {+ lt = strings.TrimSpace(lt)+ return lt == "table" || lt == "figure" || lt == "equation" || lt == "image"+}++// isDeterministicPageNumber reports whether text is unambiguously a page number.+func isDeterministicPageNumber(text string, zone string, gapAbove, gapBelow float64, numPages int) bool {+ t := strings.TrimSpace(text)+ if t == "" {+ return false+ }+ if strictDecoratedPagePattern.MatchString(t) {+ return true+ }+ if strictBareNumberPattern.MatchString(t) {+ // Bare numbers must have clear vertical whitespace separation:+ // - in header: gapBelow >= 18pt to avoid dropping section headings/chapter numbers.+ // - in footer: gapAbove >= 18pt to avoid dropping footnote numbers, table cells, or trailing text.+ if zone == "header" && gapBelow < minWhitespaceGapPt {+ return false+ }+ if zone == "footer" && gapAbove < minWhitespaceGapPt {+ return false+ }+ // For bare numeric digits, verify the value is within a reasonable range of the document page count.+ // Protects isolated years (e.g. "2024"), IDs, or footnote indexes on short/medium documents.+ var val int+ isDigits := true+ for _, r := range t {+ if r >= '0' && r <= '9' {+ val = val*10 + int(r-'0')+ } else if r >= '0' && r <= '9' {+ val = val*10 + int(r-'0')+ } else {+ isDigits = false+ break+ }+ }+ if isDigits && len(t) > 0 {+ maxAllowed := numPages + 5+ if maxAllowed < 20 {+ maxAllowed = 20+ }+ if val > maxAllowed {+ return false+ }+ }+ return true+ }+ return false+}++type boxMeta struct {+ idx int+ page int+ top float64+}++type zoneKey struct {+ zone string+ text string+}++// RemoveHeaderFooterBoxes drops boxes that are running headers or footers.//-// Like RemoveTOCBoxes this works on intact box geometry and must run before-// TextMerge (which can fold a header box into the first body section).+// It uses a 3D Cascaded Fusion scheme:+// 1. Tier 1: Deterministic Pattern & Semantic Tagging. Explicit DLA header/footer+// tags in margin zones and unambiguously formatted page numbers are removed+// immediately (even on 1- or 2-page documents).+// 2. Tier 2: Dual-Track Recurrence Engine. Evaluates recurrence across:+// - Global half-pages threshold ((numPages + 1) / 2).+// - Parity partition (even/odd) to eliminate bilateral alternating headers.+// - Locality track (consecutive pages with stable Y) to remove chapter-varying headers.+// 3. Tier 3: Adaptive Whitespace Gap. Automatically extends the candidate zone+// from 10% to up to 14% when separated from body content by >= 18pt of blank space.func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) []pdf.TextBox {- if len(pageHeights) < minHeaderFooterPages || len(boxes) == 0 {+ if len(pageHeights) == 0 || len(boxes) == 0 {return boxes}- type zoneKey struct {- zone string- text string+ perPage := make(map[int][]int, len(pageHeights))+ for i := range boxes {+ perPage[boxes[i].PageNumber] = append(perPage[boxes[i].PageNumber], i)+ }++ allGapBelow := make(map[int]float64, len(boxes))+ allGapAbove := make(map[int]float64, len(boxes))+ for p, idxs := range perPage {+ h, ok := pageHeights[p]+ if !ok || h <= 0 {+ continue+ }+ gb, ga := computePageGaps(boxes, idxs, h)+ for idx, g := range gb {+ allGapBelow[idx] = g+ }+ for idx, g := range ga {+ allGapAbove[idx] = g+ }}- keyPages := make(map[zoneKey]map[int]struct{})- keyIdx := make(map[zoneKey][]int)+ numPages := len(pageHeights)+ drop := make(map[int]struct{}, len(boxes))++ // Tier 1: Deterministic page numbers & DLA semantic tags.for i := range boxes {b := boxes[i]- // Only plain-text boxes participate; tables / figures in the margin are- // not running headers/footers.- if lt := strings.TrimSpace(b.LayoutType); lt != "" && lt != "text" {+ if isNonTextLayout(b.LayoutType) {continue}h, ok := pageHeights[b.PageNumber]if !ok || h <= 0 {continue}- top, bottom := b.Top, b.Bottom- var zone string- switch {- case bottom <= h*headerZoneRatio:- zone = "header"- case top >= h*footerZoneRatio:- zone = "footer"- default:+ zone := classifyZone(b, h, allGapAbove[i], allGapBelow[i])+ if zone == "" {+ continue+ }++ lt := strings.TrimSpace(b.LayoutType)+ if (zone == "header" && lt == "header") || (zone == "footer" && lt == "footer") {+ slog.Debug("header_footer: dropped by DLA label", "page", b.PageNumber, "zone", zone, "layoutType", lt, "textLen", utf8.RuneCountInString(b.Text))+ drop[i] = struct{}{}+ continue+ }++ if isDeterministicPageNumber(b.Text, zone, allGapAbove[i], allGapBelow[i], numPages) {+ slog.Debug("header_footer: dropped by page-number pattern", "page", b.PageNumber, "zone", zone, "textLen", utf8.RuneCountInString(b.Text))+ drop[i] = struct{}{}+ continue+ }+ }++ if numPages < minHeaderFooterPages {+ if len(drop) == 0 {+ return boxes+ }+ slog.Debug("header_footer: removal completed (short document)", "total_boxes", len(boxes), "dropped_boxes", len(drop))+ return applyDrop(boxes, drop)+ }++ // Tier 2: Dual-Track Recurrence Engine.+ keyMetas := make(map[zoneKey][]boxMeta)+ for i := range boxes {+ if _, ok := drop[i]; ok {+ continue+ }+ b := boxes[i]+ if isNonTextLayout(b.LayoutType) {+ continue+ }+ h, ok := pageHeights[b.PageNumber]+ if !ok || h <= 0 {+ continue+ }+ zone := classifyZone(b, h, allGapAbove[i], allGapBelow[i])+ if zone == "" {continue}norm := normalizeRunningText(b.Text)@@ -376,35 +598,167 @@ func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) [continue}key := zoneKey{zone: zone, text: norm}- if keyPages[key] == nil {- keyPages[key] = make(map[int]struct{})+ keyMetas[key] = append(keyMetas[key], boxMeta{idx: i, page: b.PageNumber, top: b.Top})+ }++ evenTotal, oddTotal := 0, 0+ for p := range pageHeights {+ if p%2 == 0 {+ evenTotal+++ } else {+ oddTotal++}- keyPages[key][b.PageNumber] = struct{}{}- keyIdx[key] = append(keyIdx[key], i)}- numPages := len(pageHeights)- // Half the pages, rounded up: rounding an odd page count down would let two- // pages of a five-page document pass as "half", which is the one direction- // this guard must not drift in.- minPages := (numPages + 1) / 2- if minPages < 2 {- minPages = 2+ minGlobalPages := (numPages + 1) / 2+ if minGlobalPages < 2 {+ minGlobalPages = 2}- drop := make(map[int]struct{})- for key, pages := range keyPages {- if len(pages) < minPages {+ for key, metas := range keyMetas {+ pageSet := make(map[int]bool, len(metas))+ evenPages, oddPages := 0, 0+ for _, m := range metas {+ if !pageSet[m.page] {+ pageSet[m.page] = true+ if m.page%2 == 0 {+ evenPages+++ } else {+ oddPages+++ }+ }+ }++ distinctPages := len(pageSet)+ var reason string++ // 1. Global coverage: at least half the pages (rounded up).+ if distinctPages >= minGlobalPages {+ reason = "global_frequency"+ } else if evenTotal >= 3 && evenPages >= 3 && float64(evenPages)/float64(evenTotal) >= minParityCoverageRatio {+ // 2. Parity partition: at least minParityCoverageRatio of even pages.+ reason = "parity_even"+ } else if oddTotal >= 3 && oddPages >= 3 && float64(oddPages)/float64(oddTotal) >= minParityCoverageRatio {+ // 2. Parity partition: at least minParityCoverageRatio of odd pages.+ reason = "parity_odd"+ }++ if reason != "" {+ slog.Debug("header_footer: dropped by recurrence", "zone", key.zone, "textLen", utf8.RuneCountInString(key.text), "reason", reason, "pages", distinctPages)+ for _, m := range metas {+ drop[m.idx] = struct{}{}+ }continue}- for _, i := range keyIdx[key] {- drop[i] = struct{}{}++ // 3. Locality track: stable consecutive runs of at least 3 pages with stable Y.+ // Only drop occurrences that actually belong to qualifying consecutive runs,+ // preventing distant isolated occurrences of the same text from being deleted.+ if len(metas) >= 3 && utf8.RuneCountInString(key.text) <= 60 {+ runIndices := findStableConsecutiveRunIndices(metas, 3, 4.0)+ if len(runIndices) > 0 {+ slog.Debug("header_footer: dropped by recurrence", "zone", key.zone, "textLen", utf8.RuneCountInString(key.text), "reason", "local_consecutive_run", "boxes", len(runIndices))+ for _, idx := range runIndices {+ drop[idx] = struct{}{}+ }+ }}}+if len(drop) == 0 {return boxes}+ slog.Debug("header_footer: removal completed", "total_boxes", len(boxes), "dropped_boxes", len(drop))+ return applyDrop(boxes, drop)+}++// hasStableConsecutiveRun checks if metas has a run of at least minRun+// consecutive pages where the top variation is within maxDy.+func hasStableConsecutiveRun(metas []boxMeta, minRun int, maxDy float64) bool {+ return len(findStableConsecutiveRunIndices(metas, minRun, maxDy)) > 0+}+// findStableConsecutiveRunIndices finds all box indices that belong to any qualifying+// consecutive run of at least minRun pages where the top variation is within maxDy.+// Restricting removal to these specific indices ensures isolated occurrences outside+// the run (e.g. on later pages) are not accidentally dropped.+func findStableConsecutiveRunIndices(metas []boxMeta, minRun int, maxDy float64) []int {+ if len(metas) < minRun {+ return nil+ }+ sort.Slice(metas, func(i, j int) bool {+ if metas[i].page != metas[j].page {+ return metas[i].page < metas[j].page+ }+ return metas[i].top < metas[j].top+ })++ type pageOccurrence struct {+ page int+ top float64+ indices []int+ }+ var distinct []pageOccurrence+ for _, m := range metas {+ if len(distinct) == 0 || distinct[len(distinct)-1].page != m.page {+ distinct = append(distinct, pageOccurrence{page: m.page, top: m.top, indices: []int{m.idx}})+ } else {+ distinct[len(distinct)-1].indices = append(distinct[len(distinct)-1].indices, m.idx)+ }+ }++ if len(distinct) < minRun {+ return nil+ }++ qualifyingPages := make(map[int]bool)++ left := 0+ for right := 0; right < len(distinct); right++ {+ // Reset window if consecutive page sequence is broken+ if right > 0 && distinct[right].page != distinct[right-1].page+1 {+ left = right+ }+ // Check all qualifying consecutive windows ending at right+ for (right - left + 1) >= minRun {+ curMin := distinct[left].top+ curMax := distinct[left].top+ for k := left + 1; k <= right; k++ {+ if distinct[k].top < curMin {+ curMin = distinct[k].top+ }+ if distinct[k].top > curMax {+ curMax = distinct[k].top+ }+ }+ if (curMax - curMin) <= maxDy {+ for k := left; k <= right; k++ {+ qualifyingPages[distinct[k].page] = true+ }+ break+ }+ left+++ }+ }++ if len(qualifyingPages) == 0 {+ return nil+ }++ var result []int+ for _, p := range distinct {+ if qualifyingPages[p.page] {+ result = append(result, p.indices...)+ }+ }+ return result+}++// applyDrop filters out the dropped boxes and returns the survivors.+func applyDrop(boxes []pdf.TextBox, drop map[int]struct{}) []pdf.TextBox {+ if len(drop) == 0 {+ return boxes+ }out := make([]pdf.TextBox, 0, len(boxes)-len(drop))for i := range boxes {if _, ok := drop[i]; ok {@@ -424,6 +778,9 @@ func normalizeRunningText(text string) string {out := make([]rune, 0, len(t))inDigit := falsefor _, r := range t {+ if r == '—' || r == '–' || r == '-' {+ r = '-'+ }if isMaskableDigit(r) {inDigit = truecontinuediff --git a/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go b/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.goindex ed48433155..2c4e66be31 100644--- a/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go+++ b/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go@@ -1,6 +1,7 @@package layoutimport (+ "fmt""strings""testing"@@ -757,3 +758,456 @@ func TestRemoveHeaderFooterBoxes_DropsPageNumberFooter(t *testing.T) {t.Fatalf("expected only the 3 body boxes, got %d", len(got))}}++// TestRemoveHeaderFooterBoxes_AlternatingEvenOdd verifies bilateral alternating+// headers where even pages have the book title and odd pages have the chapter title.+func TestRemoveHeaderFooterBoxes_AlternatingEvenOdd(t *testing.T) {+ pageHeight := 842.0+ heights := make(map[int]float64, 8)+ var boxes []pdf.TextBox+ for pg := 0; pg < 8; pg++ {+ heights[pg] = pageHeight+ if pg%2 == 0 {+ boxes = append(boxes, tb("DEEP LEARNING BOOK", pg, 72, 250, 30, 45))+ } else {+ boxes = append(boxes, tb("CHAPTER THREE: CONVOLUTION", pg, 250, 500, 30, 45))+ }+ boxes = append(boxes, tb("Body text on this page that represents substantial content.", pg, 72, 450, 160, 180))+ }++ got := RemoveHeaderFooterBoxes(boxes, heights)+ for _, b := range got {+ if strings.Contains(b.Text, "DEEP LEARNING") || strings.Contains(b.Text, "CHAPTER THREE") {+ t.Fatalf("alternating header %q should have been removed", b.Text)+ }+ }+ if len(got) != 8 {+ t.Fatalf("expected 8 body boxes remaining, got %d", len(got))+ }+}++// TestRemoveHeaderFooterBoxes_ChapterVaryingConsecutiveRun verifies that chapter-varying+// running headers appearing on consecutive pages with stable Y coordinates are removed.+func TestRemoveHeaderFooterBoxes_ChapterVaryingConsecutiveRun(t *testing.T) {+ pageHeight := 842.0+ heights := make(map[int]float64, 10)+ for pg := 0; pg < 10; pg++ {+ heights[pg] = pageHeight+ }++ var boxes []pdf.TextBox+ // Pages 2, 3, 4: Chapter 1 (only 3 of 10 pages, 30% < 50%)+ for pg := 2; pg <= 4; pg++ {+ boxes = append(boxes, tb("Chapter 1: Overview", pg, 72, 220, 35.0, 48.0))+ boxes = append(boxes, tb("Body paragraph for chapter 1.", pg, 72, 450, 160, 180))+ }+ // Pages 5, 6, 7: Chapter 2 (only 3 of 10 pages, 30% < 50%)+ for pg := 5; pg <= 7; pg++ {+ boxes = append(boxes, tb("Chapter 2: Methods", pg, 72, 220, 35.1, 48.1))+ boxes = append(boxes, tb("Body paragraph for chapter 2.", pg, 72, 450, 160, 180))+ }++ got := RemoveHeaderFooterBoxes(boxes, heights)+ for _, b := range got {+ if strings.Contains(b.Text, "Chapter 1:") || strings.Contains(b.Text, "Chapter 2:") {+ t.Fatalf("chapter header %q should have been removed by locality run", b.Text)+ }+ }+ if len(got) != 6 {+ t.Fatalf("expected 6 body boxes remaining, got %d", len(got))+ }+}++// TestRemoveHeaderFooterBoxes_WhitespaceGapExpandedZone verifies that a header sitting+// beyond the base 10% ratio (at 11.3%) is removed when separated from body text by >= 18pt gap.+func TestRemoveHeaderFooterBoxes_WhitespaceGapExpandedZone(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight}+ var boxes []pdf.TextBox+ for pg := 0; pg < 4; pg++ {+ // Bottom at 95.0 on 842.0 is 11.28% (> 10%)+ boxes = append(boxes, tb("COMPANY CONFIDENTIAL HEADER", pg, 72, 300, 40, 95))+ // Body starts at 135.0 (Gap = 135 - 95 = 40pt >= 18pt)+ boxes = append(boxes, tb("First line of body text sitting comfortably below the gap.", pg, 72, 450, 135, 155))+ }++ got := RemoveHeaderFooterBoxes(boxes, heights)+ for _, b := range got {+ if strings.Contains(b.Text, "COMPANY CONFIDENTIAL") {+ t.Fatalf("expanded zone header %q should have been removed", b.Text)+ }+ }+ if len(got) != 4 {+ t.Fatalf("expected 4 body boxes, got %d", len(got))+ }+}++// TestRemoveHeaderFooterBoxes_WhitespaceGapProtectsTightBody verifies that text at 11%+// without a whitespace gap to content below is NOT removed as a header.+func TestRemoveHeaderFooterBoxes_WhitespaceGapProtectsTightBody(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight}+ var boxes []pdf.TextBox+ for pg := 0; pg < 4; pg++ {+ // Top at 75, Bottom at 92 (10.9% > 10%)+ boxes = append(boxes, tb("Section Heading At Top", pg, 72, 250, 75, 92))+ // Content right below at Top: 98 (Gap = 98 - 92 = 6pt < 18pt)+ boxes = append(boxes, tb("Tight paragraph line directly following heading.", pg, 72, 450, 98, 115))+ }++ got := RemoveHeaderFooterBoxes(boxes, heights)+ var headings int+ for _, b := range got {+ if strings.Contains(b.Text, "Section Heading") {+ headings+++ }+ }+ if headings != 4 {+ t.Fatalf("tight heading must be preserved by whitespace gap protection, got %d of 4", headings)+ }+}++// TestRemoveHeaderFooterBoxes_ShortDocumentPageNumbers verifies that formatted page+// numbers are removed even on 1- or 2-page documents.+func TestRemoveHeaderFooterBoxes_ShortDocumentPageNumbers(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight}+ boxes := []pdf.TextBox{+ tb("Invoice Body Page 1", 0, 72, 400, 160, 180),+ tb("Page 1 of 2", 0, 250, 350, 810, 825),+ tb("Invoice Body Page 2", 1, 72, 400, 160, 180),+ tb("Page 2 of 2", 1, 250, 350, 810, 825),+ }++ got := RemoveHeaderFooterBoxes(boxes, heights)+ for _, b := range got {+ if strings.Contains(b.Text, "of 2") {+ t.Fatalf("page number %q must be removed on 2-page doc", b.Text)+ }+ }+ if len(got) != 2 {+ t.Fatalf("expected only the 2 body boxes, got %d", len(got))+ }+}++// TestRemoveHeaderFooterBoxes_DLASemanticTagDirectDrop verifies that boxes explicitly+// tagged as LayoutType header/footer in margin zones are dropped immediately.+func TestRemoveHeaderFooterBoxes_DLASemanticTagDirectDrop(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight}+ boxes := []pdf.TextBox{+ {Text: "Header Tagged By DLA", PageNumber: 0, X0: 72, X1: 300, Top: 30, Bottom: 45, LayoutType: "header"},+ tb("Body text here.", 0, 72, 400, 160, 180),+ {Text: "Footer Tagged By DLA", PageNumber: 0, X0: 72, X1: 300, Top: 820, Bottom: 835, LayoutType: "footer"},+ }++ got := RemoveHeaderFooterBoxes(boxes, heights)+ if len(got) != 1 || got[0].Text != "Body text here." {+ t.Fatalf("DLA tagged header and footer must be dropped, got %v", got)+ }+}++// TestRemoveHeaderFooterBoxes_LayoutTypeTitleAllowed verifies that a running header+// misclassified by DLA as LayoutTypeTitle is still processed and removed.+func TestRemoveHeaderFooterBoxes_LayoutTypeTitleAllowed(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}+ boxes := []pdf.TextBox{+ {Text: "PROCEEDINGS OF ACM", PageNumber: 0, X0: 72, X1: 300, Top: 30, Bottom: 45, LayoutType: "title"},+ tb("Body on page 0.", 0, 72, 400, 160, 180),+ {Text: "PROCEEDINGS OF ACM", PageNumber: 1, X0: 72, X1: 300, Top: 30, Bottom: 45, LayoutType: "title"},+ tb("Body on page 1.", 1, 72, 400, 160, 180),+ {Text: "PROCEEDINGS OF ACM", PageNumber: 2, X0: 72, X1: 300, Top: 30, Bottom: 45, LayoutType: "title"},+ tb("Body on page 2.", 2, 72, 400, 160, 180),+ }++ got := RemoveHeaderFooterBoxes(boxes, heights)+ for _, b := range got {+ if strings.Contains(b.Text, "PROCEEDINGS") {+ t.Fatalf("header %q with LayoutType 'title' should have been removed", b.Text)+ }+ }+ if len(got) != 3 {+ t.Fatalf("expected 3 body boxes, got %d", len(got))+ }+}++// TestRemoveHeaderFooterBoxes_RomanNumerals verifies roman numeral page numbers.+func TestRemoveHeaderFooterBoxes_RomanNumerals(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}+ boxes := []pdf.TextBox{+ tb("Preface body zero.", 0, 72, 400, 160, 180),+ tb("- iv -", 0, 280, 320, 820, 835),+ tb("Preface body one.", 1, 72, 400, 160, 180),+ tb("- v -", 1, 280, 320, 820, 835),+ tb("Preface body two.", 2, 72, 400, 160, 180),+ tb("- vi -", 2, 280, 320, 820, 835),+ }++ got := RemoveHeaderFooterBoxes(boxes, heights)+ for _, b := range got {+ if strings.Contains(b.Text, "- ") {+ t.Fatalf("roman numeral footer %q should have been removed", b.Text)+ }+ }+ if len(got) != 3 {+ t.Fatalf("expected only 3 body boxes, got %d", len(got))+ }+}++// TestRemoveHeaderFooterBoxes_ChineseNumerals verifies Chinese page numbers.+func TestRemoveHeaderFooterBoxes_ChineseNumerals(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}+ boxes := []pdf.TextBox{+ tb("正文第一页。", 0, 72, 400, 160, 180),+ tb("第 一 页", 0, 280, 350, 820, 835),+ tb("正文第二页。", 1, 72, 400, 160, 180),+ tb("第 二 页", 1, 280, 350, 820, 835),+ tb("正文第三页。", 2, 72, 400, 160, 180),+ tb("第 三 页", 2, 280, 350, 820, 835),+ }++ got := RemoveHeaderFooterBoxes(boxes, heights)+ for _, b := range got {+ if strings.HasPrefix(b.Text, "第 ") {+ t.Fatalf("Chinese page number footer %q should have been removed", b.Text)+ }+ }+ if len(got) != 3 {+ t.Fatalf("expected only 3 body boxes, got %d", len(got))+ }+}++// TestStrictDecoratedPagePattern_ChineseSlashGong tests Chinese page-number formats+// with "/共", "共", and comma separators.+func TestStrictDecoratedPagePattern_ChineseSlashGong(t *testing.T) {+ cases := []string{+ "第1页/共10页",+ "第 1 页 / 共 10 页",+ "第1页/共10",+ "第1页 共10页",+ "第1页/10页",+ "第1页,共10页",+ "第1页,共10页",+ "第一页/共十页",+ "第一页 共十页",+ "第 1 页",+ "第一页",+ }+ for _, c := range cases {+ if !strictDecoratedPagePattern.MatchString(c) {+ t.Errorf("expected %q to match strictDecoratedPagePattern, but did not", c)+ }+ }+}++// TestRemoveHeaderFooterBoxes_ChineseSlashGong verifies that "第1页/共10页" is removed.+func TestRemoveHeaderFooterBoxes_ChineseSlashGong(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}+ boxes := []pdf.TextBox{+ tb("正文第一页内容。", 0, 72, 400, 160, 180),+ tb("第1页/共10页", 0, 280, 380, 820, 835),+ tb("正文第二页内容。", 1, 72, 400, 160, 180),+ tb("第2页/共10页", 1, 280, 380, 820, 835),+ tb("正文第三页内容。", 2, 72, 400, 160, 180),+ tb("第3页/共10页", 2, 280, 380, 820, 835),+ }++ got := RemoveHeaderFooterBoxes(boxes, heights)+ for _, b := range got {+ if strings.Contains(b.Text, "共10页") {+ t.Fatalf("Chinese page footer %q should have been removed", b.Text)+ }+ }+ if len(got) != 3 {+ t.Fatalf("expected 3 body boxes, got %d", len(got))+ }+}++// TestHasStableConsecutiveRun_SlidingWindow tests sliding window behavior in hasStableConsecutiveRun,+// specifically verifying that initial outliers do not block subsequent qualifying runs.+func TestHasStableConsecutiveRun_SlidingWindow(t *testing.T) {+ // Case 1: Outlier at page 1 (top 30), stable run at pages 2, 3, 4 (top 40)+ metas1 := []boxMeta{+ {page: 1, top: 30},+ {page: 2, top: 40},+ {page: 3, top: 40},+ {page: 4, top: 40},+ }+ if !hasStableConsecutiveRun(metas1, 3, 4.0) {+ t.Errorf("expected true for pages 2..4 stable run despite page 1 outlier")+ }++ // Case 2: Stable run at pages 1, 2, 3 (top 40), outlier at page 4 (top 30)+ metas2 := []boxMeta{+ {page: 1, top: 40},+ {page: 2, top: 40},+ {page: 3, top: 40},+ {page: 4, top: 30},+ }+ if !hasStableConsecutiveRun(metas2, 3, 4.0) {+ t.Errorf("expected true for pages 1..3 stable run")+ }++ // Case 3: Divergent tops across all pages (no 3 consecutive pages within 4.0)+ metas3 := []boxMeta{+ {page: 1, top: 10},+ {page: 2, top: 30},+ {page: 3, top: 50},+ {page: 4, top: 70},+ }+ if hasStableConsecutiveRun(metas3, 3, 4.0) {+ t.Errorf("expected false for divergent tops")+ }++ // Case 4: Page gap breaks consecutive run+ metas4 := []boxMeta{+ {page: 1, top: 40},+ {page: 2, top: 40},+ {page: 4, top: 40},+ {page: 5, top: 40},+ }+ if hasStableConsecutiveRun(metas4, 3, 4.0) {+ t.Errorf("expected false when no consecutive run reaches minRun=3")+ }++ // Case 5: Run after a page gap meets minRun=3+ metas5 := []boxMeta{+ {page: 1, top: 40},+ {page: 2, top: 40},+ {page: 4, top: 40},+ {page: 5, top: 40},+ {page: 6, top: 40},+ }+ if !hasStableConsecutiveRun(metas5, 3, 4.0) {+ t.Errorf("expected true for pages 4..6 run after page gap")+ }+}++// TestRemoveHeaderFooterBoxes_LocalRunPreservesDistantIsolatedBox verifies that a local+// consecutive run (pages 1, 2, 3) removes those running headers while preserving a distant,+// isolated occurrence of the exact same text on page 10.+func TestRemoveHeaderFooterBoxes_LocalRunPreservesDistantIsolatedBox(t *testing.T) {+ pageHeight := 842.0+ heights := make(map[int]float64, 12)+ for pg := 0; pg < 12; pg++ {+ heights[pg] = pageHeight+ }++ var boxes []pdf.TextBox+ // Running chapter header on consecutive pages 1, 2, 3 (top = 35)+ for pg := 1; pg <= 3; pg++ {+ boxes = append(boxes, tb("Chapter 1: Foundations", pg, 72, 250, 35, 48))+ boxes = append(boxes, tb(fmt.Sprintf("Body text on page %d.", pg), pg, 72, 450, 160, 180))+ }+ // Pages 4..9 have other body text+ for pg := 4; pg <= 9; pg++ {+ boxes = append(boxes, tb(fmt.Sprintf("Regular body on page %d.", pg), pg, 72, 450, 160, 180))+ }+ // Page 10 has an isolated text box in the margin with the SAME text "Chapter 1: Foundations",+ // but it is NOT part of a consecutive run (isolated single occurrence).+ isolatedBox := tb("Chapter 1: Foundations", 10, 72, 250, 35, 48)+ boxes = append(boxes, isolatedBox)+ boxes = append(boxes, tb("Body text on page 10 referencing chapter 1.", 10, 72, 450, 160, 180))++ got := RemoveHeaderFooterBoxes(boxes, heights)++ // Pages 1, 2, 3 headers must be removed:+ for _, b := range got {+ if (b.PageNumber >= 1 && b.PageNumber <= 3) && strings.Contains(b.Text, "Chapter 1: Foundations") {+ t.Errorf("running header on page %d should have been removed", b.PageNumber)+ }+ }++ // Page 10 isolated box must be PRESERVED:+ foundIsolated := false+ for _, b := range got {+ if b.PageNumber == 10 && b.Text == "Chapter 1: Foundations" {+ foundIsolated = true+ break+ }+ }+ if !foundIsolated {+ t.Errorf("distant isolated box on page 10 should be preserved, but was dropped!")+ }+}++// TestRemoveHeaderFooterBoxes_BareFooterPageNumberWithGapDropped verifies that a bare+// Arabic page number in the footer zone with clear whitespace above is cleanly removed.+func TestRemoveHeaderFooterBoxes_BareFooterPageNumberWithGapDropped(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight}+ boxes := []pdf.TextBox{+ tb("Page 0 body content.", 0, 72, 400, 160, 200),+ tb("1", 0, 290, 310, 810, 825), // gapAbove = 810 - 200 = 610pt >> 18pt+ tb("Page 1 body content.", 1, 72, 400, 160, 200),+ tb("2", 1, 290, 310, 810, 825),+ }++ got := RemoveHeaderFooterBoxes(boxes, heights)+ for _, b := range got {+ if b.Text == "1" || b.Text == "2" {+ t.Fatalf("bare page number %q with clear whitespace above must be dropped", b.Text)+ }+ }+ if len(got) != 2 {+ t.Fatalf("expected 2 body boxes remaining, got %d", len(got))+ }+}++// TestRemoveHeaderFooterBoxes_FootnoteTightNumberPreserved verifies that a tight bare+// number in the bottom 10% (e.g. footnote index with gapAbove < 18pt) is protected.+func TestRemoveHeaderFooterBoxes_FootnoteTightNumberPreserved(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight}+ boxes := []pdf.TextBox{+ tb("Main body paragraph here.", 0, 72, 400, 100, 200),+ tb("Footnote explanation text at bottom.", 0, 72, 400, 790, 804), // bottom = 804+ tb("1", 0, 72, 85, 810, 822), // top = 810, gapAbove = 6pt < 18pt!+ }++ got := RemoveHeaderFooterBoxes(boxes, heights)+ foundFootnoteNum := false+ for _, b := range got {+ if b.Text == "1" {+ foundFootnoteNum = true+ break+ }+ }+ if !foundFootnoteNum {+ t.Fatalf("tight footnote number '1' (gapAbove < 18pt) must be preserved, but was dropped!")+ }+ if len(got) != 3 {+ t.Fatalf("expected all 3 boxes to be kept, got %d", len(got))+ }+}++// TestRemoveHeaderFooterBoxes_FooterYearPreservedOnShortDocument verifies that a 4-digit+// year (e.g. "2024") in the footer margin on a 2-page document is NOT misidentified as a page number.+func TestRemoveHeaderFooterBoxes_FooterYearPreservedOnShortDocument(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight}+ boxes := []pdf.TextBox{+ tb("Legal Agreement Page 1", 0, 72, 400, 160, 200),+ tb("2024", 0, 280, 320, 810, 825), // gapAbove >> 18pt, but value 2024 > 20+ tb("Legal Agreement Page 2", 1, 72, 400, 160, 200),+ }++ got := RemoveHeaderFooterBoxes(boxes, heights)+ foundYear := false+ for _, b := range got {+ if b.Text == "2024" {+ foundYear = true+ break+ }+ }+ if !foundYear {+ t.Fatalf("footer year '2024' (value > maxAllowed) must be preserved, but was dropped!")+ }+ if len(got) != 3 {+ t.Fatalf("expected all 3 boxes to be kept, got %d", len(got))+ }+}