Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(parser): upgrade PDF header/footer removal with 3D cascaded fusion (#19957)

jay77721c779b15d9cd0
diff --git a/internal/deepdoc/parser/pdf/layout/toc_header_footer.go b/internal/deepdoc/parser/pdf/layout/toc_header_footer.go
index 068ed1a4c0..62ebefbb31 100644
--- a/internal/deepdoc/parser/pdf/layout/toc_header_footer.go
+++ b/internal/deepdoc/parser/pdf/layout/toc_header_footer.go
@@ -1,6 +1,7 @@
package layout
import (
+ "log/slog"
"regexp"
"sort"
"strings"
@@ -324,51 +325,272 @@ const (
// running header / footer lives (fractions of the page height).
headerZoneRatio = 0.10
footerZoneRatio = 0.90
- // minHeaderFooterPages guards against false positives on short documents.
+ // headerMaxZoneRatio / footerMinZoneRatio define the expanded margin zones
+ // when a clear whitespace gap separates the candidate from body content.
+ headerMaxZoneRatio = 0.14
+ footerMinZoneRatio = 0.86
+ // minWhitespaceGapPt is the minimum vertical whitespace gap required to
+ // justify expanding the header/footer zone beyond the base 10% ratio.
+ minWhitespaceGapPt = 18.0
+ // minHeaderFooterPages guards against cross-page false positives on short
+ // documents. Deterministic page-number patterns still fire on shorter ones.
minHeaderFooterPages = 3
+ // minParityCoverageRatio is the minimum fraction of even or odd pages that
+ // a running header/footer text must appear on to qualify under the parity track.
+ //
+ // In formal book typography (LaTeX book documentclass, InDesign book templates),
+ // running headers are standardly suppressed on chapter opening pages (\thispagestyle{plain}),
+ // blank verso pages before new chapters, and full-page illustrations/tables.
+ // As a result, a running header in a real book often covers only 40%-48% of the
+ // even or odd pages. A 0.40 threshold captures alternating bilateral book headers
+ // while the evenPages >= 3 / oddPages >= 3 count guards against accidental matches.
+ minParityCoverageRatio = 0.40
)
-// RemoveHeaderFooterBoxes drops boxes that are running headers or footers.
+var (
+ // strictDecoratedPagePattern matches page numbers with clear formatting:
+ // - Dash/dot/tilde-wrapped: - 1 -, — 1 —, – 1 –, · 1 ·, • 1 •, ~ 1 ~
+ // - Bracket/paren-wrapped: [1], (1)
+ // - "Page X", "Page X of Y", "p. X", "X / Y", "X of Y"
+ // - Chinese: 第 1 页, 第 1 页 共 10 页, 第 1 页/共 10 页, 第一页, 第一页 共十页
+ // - Roman with dashes: - iv -, — iv —, · iv ·
+ strictDecoratedPagePattern = regexp.MustCompile(`(?i)^(\s*[-—–~·•]+\s*[0-90-9]+\s*[-—–~·•]+|\s*\[\s*[0-90-9]+\s*\]|\s*\(\s*[0-90-9]+\s*\)|page\s+[0-90-9]+(\s*(of|/)\s*[0-90-9]+)?|p\.\s*[0-90-9]+|[0-90-9]+\s*/\s*[0-90-9]+|[0-90-9]+\s+of\s+[0-90-9]+|第\s*[0-90-9一二三四五六七八九十百]+\s*页(\s*(/\s*共|[,,]\s*共?|[/共])\s*[0-90-9一二三四五六七八九十百]+\s*页?)?|\s*[-—–~·•]+\s*((l|xl|x{1,3})(ix|iv|v?i{1,3}|v)?|ix|iv|v?i{1,3}|v)\s*[-—–~·•]+)$`)
+
+ // strictBareNumberPattern matches lone Arabic digits or Roman numerals: 1, 23, iv, VII.
+ strictBareNumberPattern = regexp.MustCompile(`(?i)^([0-90-9]{1,4}|((l|xl|x{1,3})(ix|iv|v?i{1,3}|v)?|ix|iv|v?i{1,3}|v))$`)
+)
+
+// computePageGaps computes for candidate margin boxes on a page:
+// - gapBelow: distance from b.Bottom to the nearest other box on the page with Top >= b.Bottom - 2.0.
+// - gapAbove: distance from b.Top to the nearest other box on the page with Bottom <= b.Top + 2.0.
//
-// A box is a candidate when it sits entirely in the top headerZoneRatio or
-// bottom footerZoneRatio of its page and carries text-like content. Among
-// candidates, a (zone, normalized-text) pair that recurs on at least half of
-// the document's pages — rounded up, so an odd page count raises the bar rather
-// than lowering it — is treated as a running header / footer and removed.
+// Performance optimization: only boxes falling into the outer margin zones (top 14% or bottom 14%)
+// ever query whitespace gaps. Skipping body-interior boxes reduces the outer loop from M to K (K <= 5),
+// lowering complexity from O(M^2) to O(K*M) on dense multi-column pages.
+func computePageGaps(boxes []pdf.TextBox, indices []int, pageHeight float64) (map[int]float64, map[int]float64) {
+ gapBelow := make(map[int]float64, len(indices))
+ gapAbove := make(map[int]float64, len(indices))
+
+ for _, i := range indices {
+ bi := boxes[i]
+ if bi.Bottom > pageHeight*headerMaxZoneRatio && bi.Top < pageHeight*footerMinZoneRatio {
+ continue
+ }
+ minDistBelow := pageHeight - bi.Bottom
+ if minDistBelow < 0 {
+ minDistBelow = 0
+ }
+ maxDistAbove := bi.Top
+ if maxDistAbove < 0 {
+ maxDistAbove = 0
+ }
+
+ for _, j := range indices {
+ if i == j {
+ continue
+ }
+ bj := boxes[j]
+ if bj.Top >= bi.Bottom-2.0 {
+ dist := bj.Top - bi.Bottom
+ if dist < 0 {
+ dist = 0
+ }
+ if dist < minDistBelow {
+ minDistBelow = dist
+ }
+ }
+ if bj.Bottom <= bi.Top+2.0 {
+ dist := bi.Top - bj.Bottom
+ if dist < 0 {
+ dist = 0
+ }
+ if dist < maxDistAbove {
+ maxDistAbove = dist
+ }
+ }
+ }
+ gapBelow[i] = minDistBelow
+ gapAbove[i] = maxDistAbove
+ }
+ return gapBelow, gapAbove
+}
+
+// classifyZone determines whether a box sits in a header or footer zone.
+func classifyZone(b pdf.TextBox, pageHeight float64, gapAbove, gapBelow float64) string {
+ if b.Bottom <= pageHeight*headerZoneRatio {
+ return "header"
+ }
+ if b.Bottom <= pageHeight*headerMaxZoneRatio && gapBelow >= minWhitespaceGapPt {
+ return "header"
+ }
+ if b.Top >= pageHeight*footerZoneRatio {
+ return "footer"
+ }
+ if b.Top >= pageHeight*footerMinZoneRatio && gapAbove >= minWhitespaceGapPt {
+ return "footer"
+ }
+ return ""
+}
+
+// isNonTextLayout reports whether a layout type represents non-text elements
+// like tables or figures that must not be treated as running headers/footers.
+func isNonTextLayout(lt string) bool {
+ lt = strings.TrimSpace(lt)
+ return lt == "table" || lt == "figure" || lt == "equation" || lt == "image"
+}
+
+// isDeterministicPageNumber reports whether text is unambiguously a page number.
+func isDeterministicPageNumber(text string, zone string, gapAbove, gapBelow float64, numPages int) bool {
+ t := strings.TrimSpace(text)
+ if t == "" {
+ return false
+ }
+ if strictDecoratedPagePattern.MatchString(t) {
+ return true
+ }
+ if strictBareNumberPattern.MatchString(t) {
+ // Bare numbers must have clear vertical whitespace separation:
+ // - in header: gapBelow >= 18pt to avoid dropping section headings/chapter numbers.
+ // - in footer: gapAbove >= 18pt to avoid dropping footnote numbers, table cells, or trailing text.
+ if zone == "header" && gapBelow < minWhitespaceGapPt {
+ return false
+ }
+ if zone == "footer" && gapAbove < minWhitespaceGapPt {
+ return false
+ }
+ // For bare numeric digits, verify the value is within a reasonable range of the document page count.
+ // Protects isolated years (e.g. "2024"), IDs, or footnote indexes on short/medium documents.
+ var val int
+ isDigits := true
+ for _, r := range t {
+ if r >= '0' && r <= '9' {
+ val = val*10 + int(r-'0')
+ } else if r >= '0' && r <= '9' {
+ val = val*10 + int(r-'0')
+ } else {
+ isDigits = false
+ break
+ }
+ }
+ if isDigits && len(t) > 0 {
+ maxAllowed := numPages + 5
+ if maxAllowed < 20 {
+ maxAllowed = 20
+ }
+ if val > maxAllowed {
+ return false
+ }
+ }
+ return true
+ }
+ return false
+}
+
+type boxMeta struct {
+ idx int
+ page int
+ top float64
+}
+
+type zoneKey struct {
+ zone string
+ text string
+}
+
+// RemoveHeaderFooterBoxes drops boxes that are running headers or footers.
//
-// Like RemoveTOCBoxes this works on intact box geometry and must run before
-// TextMerge (which can fold a header box into the first body section).
+// It uses a 3D Cascaded Fusion scheme:
+// 1. Tier 1: Deterministic Pattern & Semantic Tagging. Explicit DLA header/footer
+// tags in margin zones and unambiguously formatted page numbers are removed
+// immediately (even on 1- or 2-page documents).
+// 2. Tier 2: Dual-Track Recurrence Engine. Evaluates recurrence across:
+// - Global half-pages threshold ((numPages + 1) / 2).
+// - Parity partition (even/odd) to eliminate bilateral alternating headers.
+// - Locality track (consecutive pages with stable Y) to remove chapter-varying headers.
+// 3. Tier 3: Adaptive Whitespace Gap. Automatically extends the candidate zone
+// from 10% to up to 14% when separated from body content by >= 18pt of blank space.
func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) []pdf.TextBox {
- if len(pageHeights) < minHeaderFooterPages || len(boxes) == 0 {
+ if len(pageHeights) == 0 || len(boxes) == 0 {
return boxes
}
- type zoneKey struct {
- zone string
- text string
+ perPage := make(map[int][]int, len(pageHeights))
+ for i := range boxes {
+ perPage[boxes[i].PageNumber] = append(perPage[boxes[i].PageNumber], i)
+ }
+
+ allGapBelow := make(map[int]float64, len(boxes))
+ allGapAbove := make(map[int]float64, len(boxes))
+ for p, idxs := range perPage {
+ h, ok := pageHeights[p]
+ if !ok || h <= 0 {
+ continue
+ }
+ gb, ga := computePageGaps(boxes, idxs, h)
+ for idx, g := range gb {
+ allGapBelow[idx] = g
+ }
+ for idx, g := range ga {
+ allGapAbove[idx] = g
+ }
}
- keyPages := make(map[zoneKey]map[int]struct{})
- keyIdx := make(map[zoneKey][]int)
+ numPages := len(pageHeights)
+ drop := make(map[int]struct{}, len(boxes))
+
+ // Tier 1: Deterministic page numbers & DLA semantic tags.
for i := range boxes {
b := boxes[i]
- // Only plain-text boxes participate; tables / figures in the margin are
- // not running headers/footers.
- if lt := strings.TrimSpace(b.LayoutType); lt != "" && lt != "text" {
+ if isNonTextLayout(b.LayoutType) {
continue
}
h, ok := pageHeights[b.PageNumber]
if !ok || h <= 0 {
continue
}
- top, bottom := b.Top, b.Bottom
- var zone string
- switch {
- case bottom <= h*headerZoneRatio:
- zone = "header"
- case top >= h*footerZoneRatio:
- zone = "footer"
- default:
+ zone := classifyZone(b, h, allGapAbove[i], allGapBelow[i])
+ if zone == "" {
+ continue
+ }
+
+ lt := strings.TrimSpace(b.LayoutType)
+ if (zone == "header" && lt == "header") || (zone == "footer" && lt == "footer") {
+ slog.Debug("header_footer: dropped by DLA label", "page", b.PageNumber, "zone", zone, "layoutType", lt, "textLen", utf8.RuneCountInString(b.Text))
+ drop[i] = struct{}{}
+ continue
+ }
+
+ if isDeterministicPageNumber(b.Text, zone, allGapAbove[i], allGapBelow[i], numPages) {
+ slog.Debug("header_footer: dropped by page-number pattern", "page", b.PageNumber, "zone", zone, "textLen", utf8.RuneCountInString(b.Text))
+ drop[i] = struct{}{}
+ continue
+ }
+ }
+
+ if numPages < minHeaderFooterPages {
+ if len(drop) == 0 {
+ return boxes
+ }
+ slog.Debug("header_footer: removal completed (short document)", "total_boxes", len(boxes), "dropped_boxes", len(drop))
+ return applyDrop(boxes, drop)
+ }
+
+ // Tier 2: Dual-Track Recurrence Engine.
+ keyMetas := make(map[zoneKey][]boxMeta)
+ for i := range boxes {
+ if _, ok := drop[i]; ok {
+ continue
+ }
+ b := boxes[i]
+ if isNonTextLayout(b.LayoutType) {
+ continue
+ }
+ h, ok := pageHeights[b.PageNumber]
+ if !ok || h <= 0 {
+ continue
+ }
+ zone := classifyZone(b, h, allGapAbove[i], allGapBelow[i])
+ if zone == "" {
continue
}
norm := normalizeRunningText(b.Text)
@@ -376,35 +598,167 @@ func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) [
continue
}
key := zoneKey{zone: zone, text: norm}
- if keyPages[key] == nil {
- keyPages[key] = make(map[int]struct{})
+ keyMetas[key] = append(keyMetas[key], boxMeta{idx: i, page: b.PageNumber, top: b.Top})
+ }
+
+ evenTotal, oddTotal := 0, 0
+ for p := range pageHeights {
+ if p%2 == 0 {
+ evenTotal++
+ } else {
+ oddTotal++
}
- keyPages[key][b.PageNumber] = struct{}{}
- keyIdx[key] = append(keyIdx[key], i)
}
- numPages := len(pageHeights)
- // Half the pages, rounded up: rounding an odd page count down would let two
- // pages of a five-page document pass as "half", which is the one direction
- // this guard must not drift in.
- minPages := (numPages + 1) / 2
- if minPages < 2 {
- minPages = 2
+ minGlobalPages := (numPages + 1) / 2
+ if minGlobalPages < 2 {
+ minGlobalPages = 2
}
- drop := make(map[int]struct{})
- for key, pages := range keyPages {
- if len(pages) < minPages {
+ for key, metas := range keyMetas {
+ pageSet := make(map[int]bool, len(metas))
+ evenPages, oddPages := 0, 0
+ for _, m := range metas {
+ if !pageSet[m.page] {
+ pageSet[m.page] = true
+ if m.page%2 == 0 {
+ evenPages++
+ } else {
+ oddPages++
+ }
+ }
+ }
+
+ distinctPages := len(pageSet)
+ var reason string
+
+ // 1. Global coverage: at least half the pages (rounded up).
+ if distinctPages >= minGlobalPages {
+ reason = "global_frequency"
+ } else if evenTotal >= 3 && evenPages >= 3 && float64(evenPages)/float64(evenTotal) >= minParityCoverageRatio {
+ // 2. Parity partition: at least minParityCoverageRatio of even pages.
+ reason = "parity_even"
+ } else if oddTotal >= 3 && oddPages >= 3 && float64(oddPages)/float64(oddTotal) >= minParityCoverageRatio {
+ // 2. Parity partition: at least minParityCoverageRatio of odd pages.
+ reason = "parity_odd"
+ }
+
+ if reason != "" {
+ slog.Debug("header_footer: dropped by recurrence", "zone", key.zone, "textLen", utf8.RuneCountInString(key.text), "reason", reason, "pages", distinctPages)
+ for _, m := range metas {
+ drop[m.idx] = struct{}{}
+ }
continue
}
- for _, i := range keyIdx[key] {
- drop[i] = struct{}{}
+
+ // 3. Locality track: stable consecutive runs of at least 3 pages with stable Y.
+ // Only drop occurrences that actually belong to qualifying consecutive runs,
+ // preventing distant isolated occurrences of the same text from being deleted.
+ if len(metas) >= 3 && utf8.RuneCountInString(key.text) <= 60 {
+ runIndices := findStableConsecutiveRunIndices(metas, 3, 4.0)
+ if len(runIndices) > 0 {
+ slog.Debug("header_footer: dropped by recurrence", "zone", key.zone, "textLen", utf8.RuneCountInString(key.text), "reason", "local_consecutive_run", "boxes", len(runIndices))
+ for _, idx := range runIndices {
+ drop[idx] = struct{}{}
+ }
+ }
}
}
+
if len(drop) == 0 {
return boxes
}
+ slog.Debug("header_footer: removal completed", "total_boxes", len(boxes), "dropped_boxes", len(drop))
+ return applyDrop(boxes, drop)
+}
+
+// hasStableConsecutiveRun checks if metas has a run of at least minRun
+// consecutive pages where the top variation is within maxDy.
+func hasStableConsecutiveRun(metas []boxMeta, minRun int, maxDy float64) bool {
+ return len(findStableConsecutiveRunIndices(metas, minRun, maxDy)) > 0
+}
+// findStableConsecutiveRunIndices finds all box indices that belong to any qualifying
+// consecutive run of at least minRun pages where the top variation is within maxDy.
+// Restricting removal to these specific indices ensures isolated occurrences outside
+// the run (e.g. on later pages) are not accidentally dropped.
+func findStableConsecutiveRunIndices(metas []boxMeta, minRun int, maxDy float64) []int {
+ if len(metas) < minRun {
+ return nil
+ }
+ sort.Slice(metas, func(i, j int) bool {
+ if metas[i].page != metas[j].page {
+ return metas[i].page < metas[j].page
+ }
+ return metas[i].top < metas[j].top
+ })
+
+ type pageOccurrence struct {
+ page int
+ top float64
+ indices []int
+ }
+ var distinct []pageOccurrence
+ for _, m := range metas {
+ if len(distinct) == 0 || distinct[len(distinct)-1].page != m.page {
+ distinct = append(distinct, pageOccurrence{page: m.page, top: m.top, indices: []int{m.idx}})
+ } else {
+ distinct[len(distinct)-1].indices = append(distinct[len(distinct)-1].indices, m.idx)
+ }
+ }
+
+ if len(distinct) < minRun {
+ return nil
+ }
+
+ qualifyingPages := make(map[int]bool)
+
+ left := 0
+ for right := 0; right < len(distinct); right++ {
+ // Reset window if consecutive page sequence is broken
+ if right > 0 && distinct[right].page != distinct[right-1].page+1 {
+ left = right
+ }
+ // Check all qualifying consecutive windows ending at right
+ for (right - left + 1) >= minRun {
+ curMin := distinct[left].top
+ curMax := distinct[left].top
+ for k := left + 1; k <= right; k++ {
+ if distinct[k].top < curMin {
+ curMin = distinct[k].top
+ }
+ if distinct[k].top > curMax {
+ curMax = distinct[k].top
+ }
+ }
+ if (curMax - curMin) <= maxDy {
+ for k := left; k <= right; k++ {
+ qualifyingPages[distinct[k].page] = true
+ }
+ break
+ }
+ left++
+ }
+ }
+
+ if len(qualifyingPages) == 0 {
+ return nil
+ }
+
+ var result []int
+ for _, p := range distinct {
+ if qualifyingPages[p.page] {
+ result = append(result, p.indices...)
+ }
+ }
+ return result
+}
+
+// applyDrop filters out the dropped boxes and returns the survivors.
+func applyDrop(boxes []pdf.TextBox, drop map[int]struct{}) []pdf.TextBox {
+ if len(drop) == 0 {
+ return boxes
+ }
out := make([]pdf.TextBox, 0, len(boxes)-len(drop))
for i := range boxes {
if _, ok := drop[i]; ok {
@@ -424,6 +778,9 @@ func normalizeRunningText(text string) string {
out := make([]rune, 0, len(t))
inDigit := false
for _, r := range t {
+ if r == '—' || r == '–' || r == '-' {
+ r = '-'
+ }
if isMaskableDigit(r) {
inDigit = true
continue
diff --git a/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go b/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go
index ed48433155..2c4e66be31 100644
--- a/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go
+++ b/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go
@@ -1,6 +1,7 @@
package layout
import (
+ "fmt"
"strings"
"testing"
@@ -757,3 +758,456 @@ func TestRemoveHeaderFooterBoxes_DropsPageNumberFooter(t *testing.T) {
t.Fatalf("expected only the 3 body boxes, got %d", len(got))
}
}
+
+// TestRemoveHeaderFooterBoxes_AlternatingEvenOdd verifies bilateral alternating
+// headers where even pages have the book title and odd pages have the chapter title.
+func TestRemoveHeaderFooterBoxes_AlternatingEvenOdd(t *testing.T) {
+ pageHeight := 842.0
+ heights := make(map[int]float64, 8)
+ var boxes []pdf.TextBox
+ for pg := 0; pg < 8; pg++ {
+ heights[pg] = pageHeight
+ if pg%2 == 0 {
+ boxes = append(boxes, tb("DEEP LEARNING BOOK", pg, 72, 250, 30, 45))
+ } else {
+ boxes = append(boxes, tb("CHAPTER THREE: CONVOLUTION", pg, 250, 500, 30, 45))
+ }
+ boxes = append(boxes, tb("Body text on this page that represents substantial content.", pg, 72, 450, 160, 180))
+ }
+
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ for _, b := range got {
+ if strings.Contains(b.Text, "DEEP LEARNING") || strings.Contains(b.Text, "CHAPTER THREE") {
+ t.Fatalf("alternating header %q should have been removed", b.Text)
+ }
+ }
+ if len(got) != 8 {
+ t.Fatalf("expected 8 body boxes remaining, got %d", len(got))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_ChapterVaryingConsecutiveRun verifies that chapter-varying
+// running headers appearing on consecutive pages with stable Y coordinates are removed.
+func TestRemoveHeaderFooterBoxes_ChapterVaryingConsecutiveRun(t *testing.T) {
+ pageHeight := 842.0
+ heights := make(map[int]float64, 10)
+ for pg := 0; pg < 10; pg++ {
+ heights[pg] = pageHeight
+ }
+
+ var boxes []pdf.TextBox
+ // Pages 2, 3, 4: Chapter 1 (only 3 of 10 pages, 30% < 50%)
+ for pg := 2; pg <= 4; pg++ {
+ boxes = append(boxes, tb("Chapter 1: Overview", pg, 72, 220, 35.0, 48.0))
+ boxes = append(boxes, tb("Body paragraph for chapter 1.", pg, 72, 450, 160, 180))
+ }
+ // Pages 5, 6, 7: Chapter 2 (only 3 of 10 pages, 30% < 50%)
+ for pg := 5; pg <= 7; pg++ {
+ boxes = append(boxes, tb("Chapter 2: Methods", pg, 72, 220, 35.1, 48.1))
+ boxes = append(boxes, tb("Body paragraph for chapter 2.", pg, 72, 450, 160, 180))
+ }
+
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ for _, b := range got {
+ if strings.Contains(b.Text, "Chapter 1:") || strings.Contains(b.Text, "Chapter 2:") {
+ t.Fatalf("chapter header %q should have been removed by locality run", b.Text)
+ }
+ }
+ if len(got) != 6 {
+ t.Fatalf("expected 6 body boxes remaining, got %d", len(got))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_WhitespaceGapExpandedZone verifies that a header sitting
+// beyond the base 10% ratio (at 11.3%) is removed when separated from body text by >= 18pt gap.
+func TestRemoveHeaderFooterBoxes_WhitespaceGapExpandedZone(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight}
+ var boxes []pdf.TextBox
+ for pg := 0; pg < 4; pg++ {
+ // Bottom at 95.0 on 842.0 is 11.28% (> 10%)
+ boxes = append(boxes, tb("COMPANY CONFIDENTIAL HEADER", pg, 72, 300, 40, 95))
+ // Body starts at 135.0 (Gap = 135 - 95 = 40pt >= 18pt)
+ boxes = append(boxes, tb("First line of body text sitting comfortably below the gap.", pg, 72, 450, 135, 155))
+ }
+
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ for _, b := range got {
+ if strings.Contains(b.Text, "COMPANY CONFIDENTIAL") {
+ t.Fatalf("expanded zone header %q should have been removed", b.Text)
+ }
+ }
+ if len(got) != 4 {
+ t.Fatalf("expected 4 body boxes, got %d", len(got))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_WhitespaceGapProtectsTightBody verifies that text at 11%
+// without a whitespace gap to content below is NOT removed as a header.
+func TestRemoveHeaderFooterBoxes_WhitespaceGapProtectsTightBody(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight}
+ var boxes []pdf.TextBox
+ for pg := 0; pg < 4; pg++ {
+ // Top at 75, Bottom at 92 (10.9% > 10%)
+ boxes = append(boxes, tb("Section Heading At Top", pg, 72, 250, 75, 92))
+ // Content right below at Top: 98 (Gap = 98 - 92 = 6pt < 18pt)
+ boxes = append(boxes, tb("Tight paragraph line directly following heading.", pg, 72, 450, 98, 115))
+ }
+
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ var headings int
+ for _, b := range got {
+ if strings.Contains(b.Text, "Section Heading") {
+ headings++
+ }
+ }
+ if headings != 4 {
+ t.Fatalf("tight heading must be preserved by whitespace gap protection, got %d of 4", headings)
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_ShortDocumentPageNumbers verifies that formatted page
+// numbers are removed even on 1- or 2-page documents.
+func TestRemoveHeaderFooterBoxes_ShortDocumentPageNumbers(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight}
+ boxes := []pdf.TextBox{
+ tb("Invoice Body Page 1", 0, 72, 400, 160, 180),
+ tb("Page 1 of 2", 0, 250, 350, 810, 825),
+ tb("Invoice Body Page 2", 1, 72, 400, 160, 180),
+ tb("Page 2 of 2", 1, 250, 350, 810, 825),
+ }
+
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ for _, b := range got {
+ if strings.Contains(b.Text, "of 2") {
+ t.Fatalf("page number %q must be removed on 2-page doc", b.Text)
+ }
+ }
+ if len(got) != 2 {
+ t.Fatalf("expected only the 2 body boxes, got %d", len(got))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_DLASemanticTagDirectDrop verifies that boxes explicitly
+// tagged as LayoutType header/footer in margin zones are dropped immediately.
+func TestRemoveHeaderFooterBoxes_DLASemanticTagDirectDrop(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight}
+ boxes := []pdf.TextBox{
+ {Text: "Header Tagged By DLA", PageNumber: 0, X0: 72, X1: 300, Top: 30, Bottom: 45, LayoutType: "header"},
+ tb("Body text here.", 0, 72, 400, 160, 180),
+ {Text: "Footer Tagged By DLA", PageNumber: 0, X0: 72, X1: 300, Top: 820, Bottom: 835, LayoutType: "footer"},
+ }
+
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ if len(got) != 1 || got[0].Text != "Body text here." {
+ t.Fatalf("DLA tagged header and footer must be dropped, got %v", got)
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_LayoutTypeTitleAllowed verifies that a running header
+// misclassified by DLA as LayoutTypeTitle is still processed and removed.
+func TestRemoveHeaderFooterBoxes_LayoutTypeTitleAllowed(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}
+ boxes := []pdf.TextBox{
+ {Text: "PROCEEDINGS OF ACM", PageNumber: 0, X0: 72, X1: 300, Top: 30, Bottom: 45, LayoutType: "title"},
+ tb("Body on page 0.", 0, 72, 400, 160, 180),
+ {Text: "PROCEEDINGS OF ACM", PageNumber: 1, X0: 72, X1: 300, Top: 30, Bottom: 45, LayoutType: "title"},
+ tb("Body on page 1.", 1, 72, 400, 160, 180),
+ {Text: "PROCEEDINGS OF ACM", PageNumber: 2, X0: 72, X1: 300, Top: 30, Bottom: 45, LayoutType: "title"},
+ tb("Body on page 2.", 2, 72, 400, 160, 180),
+ }
+
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ for _, b := range got {
+ if strings.Contains(b.Text, "PROCEEDINGS") {
+ t.Fatalf("header %q with LayoutType 'title' should have been removed", b.Text)
+ }
+ }
+ if len(got) != 3 {
+ t.Fatalf("expected 3 body boxes, got %d", len(got))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_RomanNumerals verifies roman numeral page numbers.
+func TestRemoveHeaderFooterBoxes_RomanNumerals(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}
+ boxes := []pdf.TextBox{
+ tb("Preface body zero.", 0, 72, 400, 160, 180),
+ tb("- iv -", 0, 280, 320, 820, 835),
+ tb("Preface body one.", 1, 72, 400, 160, 180),
+ tb("- v -", 1, 280, 320, 820, 835),
+ tb("Preface body two.", 2, 72, 400, 160, 180),
+ tb("- vi -", 2, 280, 320, 820, 835),
+ }
+
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ for _, b := range got {
+ if strings.Contains(b.Text, "- ") {
+ t.Fatalf("roman numeral footer %q should have been removed", b.Text)
+ }
+ }
+ if len(got) != 3 {
+ t.Fatalf("expected only 3 body boxes, got %d", len(got))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_ChineseNumerals verifies Chinese page numbers.
+func TestRemoveHeaderFooterBoxes_ChineseNumerals(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}
+ boxes := []pdf.TextBox{
+ tb("正文第一页。", 0, 72, 400, 160, 180),
+ tb("第 一 页", 0, 280, 350, 820, 835),
+ tb("正文第二页。", 1, 72, 400, 160, 180),
+ tb("第 二 页", 1, 280, 350, 820, 835),
+ tb("正文第三页。", 2, 72, 400, 160, 180),
+ tb("第 三 页", 2, 280, 350, 820, 835),
+ }
+
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ for _, b := range got {
+ if strings.HasPrefix(b.Text, "第 ") {
+ t.Fatalf("Chinese page number footer %q should have been removed", b.Text)
+ }
+ }
+ if len(got) != 3 {
+ t.Fatalf("expected only 3 body boxes, got %d", len(got))
+ }
+}
+
+// TestStrictDecoratedPagePattern_ChineseSlashGong tests Chinese page-number formats
+// with "/共", "共", and comma separators.
+func TestStrictDecoratedPagePattern_ChineseSlashGong(t *testing.T) {
+ cases := []string{
+ "第1页/共10页",
+ "第 1 页 / 共 10 页",
+ "第1页/共10",
+ "第1页 共10页",
+ "第1页/10页",
+ "第1页,共10页",
+ "第1页,共10页",
+ "第一页/共十页",
+ "第一页 共十页",
+ "第 1 页",
+ "第一页",
+ }
+ for _, c := range cases {
+ if !strictDecoratedPagePattern.MatchString(c) {
+ t.Errorf("expected %q to match strictDecoratedPagePattern, but did not", c)
+ }
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_ChineseSlashGong verifies that "第1页/共10页" is removed.
+func TestRemoveHeaderFooterBoxes_ChineseSlashGong(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}
+ boxes := []pdf.TextBox{
+ tb("正文第一页内容。", 0, 72, 400, 160, 180),
+ tb("第1页/共10页", 0, 280, 380, 820, 835),
+ tb("正文第二页内容。", 1, 72, 400, 160, 180),
+ tb("第2页/共10页", 1, 280, 380, 820, 835),
+ tb("正文第三页内容。", 2, 72, 400, 160, 180),
+ tb("第3页/共10页", 2, 280, 380, 820, 835),
+ }
+
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ for _, b := range got {
+ if strings.Contains(b.Text, "共10页") {
+ t.Fatalf("Chinese page footer %q should have been removed", b.Text)
+ }
+ }
+ if len(got) != 3 {
+ t.Fatalf("expected 3 body boxes, got %d", len(got))
+ }
+}
+
+// TestHasStableConsecutiveRun_SlidingWindow tests sliding window behavior in hasStableConsecutiveRun,
+// specifically verifying that initial outliers do not block subsequent qualifying runs.
+func TestHasStableConsecutiveRun_SlidingWindow(t *testing.T) {
+ // Case 1: Outlier at page 1 (top 30), stable run at pages 2, 3, 4 (top 40)
+ metas1 := []boxMeta{
+ {page: 1, top: 30},
+ {page: 2, top: 40},
+ {page: 3, top: 40},
+ {page: 4, top: 40},
+ }
+ if !hasStableConsecutiveRun(metas1, 3, 4.0) {
+ t.Errorf("expected true for pages 2..4 stable run despite page 1 outlier")
+ }
+
+ // Case 2: Stable run at pages 1, 2, 3 (top 40), outlier at page 4 (top 30)
+ metas2 := []boxMeta{
+ {page: 1, top: 40},
+ {page: 2, top: 40},
+ {page: 3, top: 40},
+ {page: 4, top: 30},
+ }
+ if !hasStableConsecutiveRun(metas2, 3, 4.0) {
+ t.Errorf("expected true for pages 1..3 stable run")
+ }
+
+ // Case 3: Divergent tops across all pages (no 3 consecutive pages within 4.0)
+ metas3 := []boxMeta{
+ {page: 1, top: 10},
+ {page: 2, top: 30},
+ {page: 3, top: 50},
+ {page: 4, top: 70},
+ }
+ if hasStableConsecutiveRun(metas3, 3, 4.0) {
+ t.Errorf("expected false for divergent tops")
+ }
+
+ // Case 4: Page gap breaks consecutive run
+ metas4 := []boxMeta{
+ {page: 1, top: 40},
+ {page: 2, top: 40},
+ {page: 4, top: 40},
+ {page: 5, top: 40},
+ }
+ if hasStableConsecutiveRun(metas4, 3, 4.0) {
+ t.Errorf("expected false when no consecutive run reaches minRun=3")
+ }
+
+ // Case 5: Run after a page gap meets minRun=3
+ metas5 := []boxMeta{
+ {page: 1, top: 40},
+ {page: 2, top: 40},
+ {page: 4, top: 40},
+ {page: 5, top: 40},
+ {page: 6, top: 40},
+ }
+ if !hasStableConsecutiveRun(metas5, 3, 4.0) {
+ t.Errorf("expected true for pages 4..6 run after page gap")
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_LocalRunPreservesDistantIsolatedBox verifies that a local
+// consecutive run (pages 1, 2, 3) removes those running headers while preserving a distant,
+// isolated occurrence of the exact same text on page 10.
+func TestRemoveHeaderFooterBoxes_LocalRunPreservesDistantIsolatedBox(t *testing.T) {
+ pageHeight := 842.0
+ heights := make(map[int]float64, 12)
+ for pg := 0; pg < 12; pg++ {
+ heights[pg] = pageHeight
+ }
+
+ var boxes []pdf.TextBox
+ // Running chapter header on consecutive pages 1, 2, 3 (top = 35)
+ for pg := 1; pg <= 3; pg++ {
+ boxes = append(boxes, tb("Chapter 1: Foundations", pg, 72, 250, 35, 48))
+ boxes = append(boxes, tb(fmt.Sprintf("Body text on page %d.", pg), pg, 72, 450, 160, 180))
+ }
+ // Pages 4..9 have other body text
+ for pg := 4; pg <= 9; pg++ {
+ boxes = append(boxes, tb(fmt.Sprintf("Regular body on page %d.", pg), pg, 72, 450, 160, 180))
+ }
+ // Page 10 has an isolated text box in the margin with the SAME text "Chapter 1: Foundations",
+ // but it is NOT part of a consecutive run (isolated single occurrence).
+ isolatedBox := tb("Chapter 1: Foundations", 10, 72, 250, 35, 48)
+ boxes = append(boxes, isolatedBox)
+ boxes = append(boxes, tb("Body text on page 10 referencing chapter 1.", 10, 72, 450, 160, 180))
+
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+
+ // Pages 1, 2, 3 headers must be removed:
+ for _, b := range got {
+ if (b.PageNumber >= 1 && b.PageNumber <= 3) && strings.Contains(b.Text, "Chapter 1: Foundations") {
+ t.Errorf("running header on page %d should have been removed", b.PageNumber)
+ }
+ }
+
+ // Page 10 isolated box must be PRESERVED:
+ foundIsolated := false
+ for _, b := range got {
+ if b.PageNumber == 10 && b.Text == "Chapter 1: Foundations" {
+ foundIsolated = true
+ break
+ }
+ }
+ if !foundIsolated {
+ t.Errorf("distant isolated box on page 10 should be preserved, but was dropped!")
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_BareFooterPageNumberWithGapDropped verifies that a bare
+// Arabic page number in the footer zone with clear whitespace above is cleanly removed.
+func TestRemoveHeaderFooterBoxes_BareFooterPageNumberWithGapDropped(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight}
+ boxes := []pdf.TextBox{
+ tb("Page 0 body content.", 0, 72, 400, 160, 200),
+ tb("1", 0, 290, 310, 810, 825), // gapAbove = 810 - 200 = 610pt >> 18pt
+ tb("Page 1 body content.", 1, 72, 400, 160, 200),
+ tb("2", 1, 290, 310, 810, 825),
+ }
+
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ for _, b := range got {
+ if b.Text == "1" || b.Text == "2" {
+ t.Fatalf("bare page number %q with clear whitespace above must be dropped", b.Text)
+ }
+ }
+ if len(got) != 2 {
+ t.Fatalf("expected 2 body boxes remaining, got %d", len(got))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_FootnoteTightNumberPreserved verifies that a tight bare
+// number in the bottom 10% (e.g. footnote index with gapAbove < 18pt) is protected.
+func TestRemoveHeaderFooterBoxes_FootnoteTightNumberPreserved(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight}
+ boxes := []pdf.TextBox{
+ tb("Main body paragraph here.", 0, 72, 400, 100, 200),
+ tb("Footnote explanation text at bottom.", 0, 72, 400, 790, 804), // bottom = 804
+ tb("1", 0, 72, 85, 810, 822), // top = 810, gapAbove = 6pt < 18pt!
+ }
+
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ foundFootnoteNum := false
+ for _, b := range got {
+ if b.Text == "1" {
+ foundFootnoteNum = true
+ break
+ }
+ }
+ if !foundFootnoteNum {
+ t.Fatalf("tight footnote number '1' (gapAbove < 18pt) must be preserved, but was dropped!")
+ }
+ if len(got) != 3 {
+ t.Fatalf("expected all 3 boxes to be kept, got %d", len(got))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_FooterYearPreservedOnShortDocument verifies that a 4-digit
+// year (e.g. "2024") in the footer margin on a 2-page document is NOT misidentified as a page number.
+func TestRemoveHeaderFooterBoxes_FooterYearPreservedOnShortDocument(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight}
+ boxes := []pdf.TextBox{
+ tb("Legal Agreement Page 1", 0, 72, 400, 160, 200),
+ tb("2024", 0, 280, 320, 810, 825), // gapAbove >> 18pt, but value 2024 > 20
+ tb("Legal Agreement Page 2", 1, 72, 400, 160, 200),
+ }
+
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ foundYear := false
+ for _, b := range got {
+ if b.Text == "2024" {
+ foundYear = true
+ break
+ }
+ }
+ if !foundYear {
+ t.Fatalf("footer year '2024' (value > maxAllowed) must be preserved, but was dropped!")
+ }
+ if len(got) != 3 {
+ t.Fatalf("expected all 3 boxes to be kept, got %d", len(got))
+ }
+}