Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): catch tight page numbers and site-promo headers/footers (#20019)

jay77721dae80165322d
diff --git a/internal/deepdoc/parser/pdf/layout/toc_header_footer.go b/internal/deepdoc/parser/pdf/layout/toc_header_footer.go
index d68b312935..4f1dce8dae 100644
--- a/internal/deepdoc/parser/pdf/layout/toc_header_footer.go
+++ b/internal/deepdoc/parser/pdf/layout/toc_header_footer.go
@@ -347,6 +347,28 @@ const (
// even or odd pages. A 0.40 threshold captures alternating bilateral book headers
// while the evenPages >= 3 / oddPages >= 3 count guards against accidental matches.
minParityCoverageRatio = 0.40
+ // seqBandTopRatio / seqBandBottomRatio bound the wide bands in which the
+ // sequence-backed page-number track looks for bare numeric / Roman-numeral
+ // boxes. The bands are much looser than the header/footer zones because the
+ // track's evidence is not geometric: only boxes that form an incrementing
+ // run across consecutive pages at a stable Y are removed, which tight body
+ // text cannot fake.
+ seqBandTopRatio = 0.20
+ seqBandBottomRatio = 0.80
+ // seqMinRun / seqMaxDy mirror the locality track: at least 3 consecutive
+ // pages, total top-edge span across the run within 4pt.
+ seqMinRun = 3
+ seqMaxDy = 4.0
+ // promoCompanionMaxRunes bounds the length of a margin line that may be
+ // learned as a running-header ad companion of a dropped site-promo box.
+ // A piracy header is a short slogan, so anything longer is treated as body
+ // text and never learned.
+ promoCompanionMaxRunes = 24
+ // minCompanionPages is how many distinct pages must carry the same band
+ // peer of a dropped site-promo box before that peer's text is treated as a
+ // running-header ad and propagated to every page. Requiring several pages
+ // keeps a one-off line that merely shares a page with an ad intact.
+ minCompanionPages = 3
)
var (
@@ -360,6 +382,23 @@ var (
// strictBareNumberPattern matches lone Arabic digits or Roman numerals: 1, 23, iv, VII.
strictBareNumberPattern = regexp.MustCompile(`(?i)^([0-90-9]{1,4}|((l|xl|x{1,3})(ix|iv|v?i{1,3}|v)?|ix|iv|v?i{1,3}|v))$`)
+
+ // canonicalRomanPattern accepts the full canonical Roman-numeral grammar
+ // (up to M) so the sequence track can parse page numbers beyond the XXX
+ // range strictBareNumberPattern's header/footer idiom covers. Non-canonical
+ // forms (IIII) are rejected by the pattern itself; romanValue only computes
+ // the subtractive sum of a spelling this regex already accepted.
+ canonicalRomanPattern = regexp.MustCompile(`(?i)^(m{0,3})(cm|cd|d?c{0,3})(xc|xl|l?x{0,3})(ix|iv|v?i{0,3})$`)
+
+ // sitePromoPattern matches margin lines that advertise a download site —
+ // a URL core with at most a short non-URL wrapper on either side. Such a
+ // line is a running header/footer regardless of page count: piracy-supply
+ // PDFs change the wrapper text on some pages ("更多的书籍免费下载 http://…",
+ // "欢迎访问!http://…"), so those variants reach no recurrence track, while
+ // the URL itself is deterministic evidence. Only whole-box, length-capped
+ // boxes in the margin bands use it, so body prose containing a URL is never
+ // touched.
+ sitePromoPattern = regexp.MustCompile(`(?i)^.{0,20}(https?://\S+|www\.[\w-]+(\.[\w-]+)+|(forum|bbs|tieba|phpwind)\.[\w-]+\.[a-z]{2,}\S*|\S*fromuid=\S+).{0,20}$`)
)
// computePageGaps computes for candidate margin boxes on a page:
@@ -441,6 +480,164 @@ func isNonTextLayout(lt string) bool {
return lt == "table" || lt == "figure" || lt == "equation" || lt == "image"
}
+// parseDecimalValue parses a string of lone ASCII or full-width digits.
+func parseDecimalValue(t string) (int, bool) {
+ if t == "" {
+ return 0, false
+ }
+ val := 0
+ for _, r := range t {
+ switch {
+ case r >= '0' && r <= '9':
+ val = val*10 + int(r-'0')
+ case r >= '0' && r <= '9':
+ val = val*10 + int(r-'0')
+ default:
+ return 0, false
+ }
+ }
+ return val, true
+}
+
+// romanValue parses a canonical Roman numeral (I..MMMCMXCIX) and returns its
+// value. The canonical pattern admits only canonical spellings, so non-canonical
+// forms such as "IIII" are rejected before the subtractive sum is computed.
+func romanValue(t string) (int, bool) {
+ if !canonicalRomanPattern.MatchString(t) {
+ return 0, false
+ }
+ values := map[rune]int{'m': 1000, 'd': 500, 'c': 100, 'l': 50, 'x': 10, 'v': 5, 'i': 1}
+ val := 0
+ prev := 0
+ for _, r := range strings.ToLower(t) {
+ v := values[r]
+ if v == 0 {
+ return 0, false
+ }
+ if v > prev {
+ val += v - 2*prev
+ } else {
+ val += v
+ }
+ prev = v
+ }
+ if val <= 0 {
+ return 0, false
+ }
+ return val, true
+}
+
+// parseBareNumberValue reports the numeric value of a box whose entire text is
+// one bare number: lone digits (ASCII or full-width) or a canonical Roman
+// numeral. "12", "45", "iv", "XIV" parse; "12.", "- 3 -", "2024." do not.
+func parseBareNumberValue(text string) (int, bool) {
+ t := strings.TrimSpace(text)
+ if v, ok := parseDecimalValue(t); ok {
+ return v, true
+ }
+ return romanValue(t)
+}
+
+// pageNumberCeiling bounds which bare numeric values can be page numbers: a
+// document of N pages never carries page number > N+5, and the floor of 20
+// keeps short documents (years like 2024 on a 2-page footer) protected.
+func pageNumberCeiling(numPages int) int {
+ maxAllowed := numPages + 5
+ if maxAllowed < 20 {
+ maxAllowed = 20
+ }
+ return maxAllowed
+}
+
+// isSitePromo reports whether the whole box text advertises a download site.
+func isSitePromo(text string) bool {
+ t := strings.Join(strings.Fields(text), " ")
+ if t == "" || utf8.RuneCountInString(t) > 60 {
+ return false
+ }
+ return sitePromoPattern.MatchString(t)
+}
+
+// bandOf reports which expanded margin band ("header"/"footer") a box sits in,
+// or "" when it is outside both. It uses the same zone bounds the site-promo
+// tier applies, so a box learned as an ad companion and a box later matched
+// against it are judged by one rule.
+func bandOf(b pdf.TextBox, pageHeight float64) string {
+ switch {
+ case b.Bottom <= pageHeight*headerMaxZoneRatio:
+ return "header"
+ case b.Top >= pageHeight*footerMinZoneRatio:
+ return "footer"
+ }
+ return ""
+}
+
+// promoSpot is the page and band where a site-promo box was dropped, recorded
+// so the companion pass can harvest its in-band peers as running-header ads.
+type promoSpot struct {
+ page int
+ band string
+}
+
+// promoCompanionDrops returns indices of margin-band boxes that are the slogan
+// half of an advertisement already seen in the same band next to a dropped
+// site-promo box. It harvests the short, non-title, in-band peers of every
+// promo spot, then flags every band box whose collapsed text was harvested on
+// at least minCompanionPages distinct pages — so a piracy header whose URL line
+// is omitted or split on some pages still goes. The peers are keyed on
+// collapseText (whitespace + case only, digits preserved) so two lines that
+// differ only by a number are never merged. `skip` holds indices already doomed
+// by an earlier tier and is consulted, never mutated.
+func promoCompanionDrops(boxes []pdf.TextBox, perPage map[int][]int, pageHeights map[int]float64, skip map[int]struct{}, promoSpots []promoSpot) []int {
+ companionPages := make(map[string]map[int]bool)
+ for _, sp := range promoSpots {
+ h := pageHeights[sp.page]
+ for _, j := range perPage[sp.page] {
+ if _, dropped := skip[j]; dropped {
+ continue
+ }
+ b := boxes[j]
+ if isNonTextLayout(b.LayoutType) || strings.TrimSpace(b.LayoutType) == "title" {
+ continue
+ }
+ if bandOf(b, h) != sp.band {
+ continue
+ }
+ key := collapseText(b.Text)
+ if key == "" || utf8.RuneCountInString(key) > promoCompanionMaxRunes {
+ continue
+ }
+ if companionPages[key] == nil {
+ companionPages[key] = make(map[int]bool)
+ }
+ companionPages[key][sp.page] = true
+ }
+ }
+ var out []int
+ for key, pages := range companionPages {
+ if len(pages) < minCompanionPages {
+ continue
+ }
+ for i := range boxes {
+ if _, dropped := skip[i]; dropped {
+ continue
+ }
+ b := boxes[i]
+ if isNonTextLayout(b.LayoutType) || strings.TrimSpace(b.LayoutType) == "title" {
+ continue
+ }
+ h, ok := pageHeights[b.PageNumber]
+ if !ok || h <= 0 {
+ continue
+ }
+ if bandOf(b, h) != "" && collapseText(b.Text) == key {
+ out = append(out, i)
+ }
+ }
+ }
+ return out
+}
+
// isDeterministicPageNumber reports whether text is unambiguously a page number.
func isDeterministicPageNumber(text string, zone string, gapAbove, gapBelow float64, numPages int) bool {
t := strings.TrimSpace(text)
@@ -462,24 +659,8 @@ func isDeterministicPageNumber(text string, zone string, gapAbove, gapBelow floa
}
// For bare numeric digits, verify the value is within a reasonable range of the document page count.
// Protects isolated years (e.g. "2024"), IDs, or footnote indexes on short/medium documents.
- var val int
- isDigits := true
- for _, r := range t {
- if r >= '0' && r <= '9' {
- val = val*10 + int(r-'0')
- } else if r >= '0' && r <= '9' {
- val = val*10 + int(r-'0')
- } else {
- isDigits = false
- break
- }
- }
- if isDigits && len(t) > 0 {
- maxAllowed := numPages + 5
- if maxAllowed < 20 {
- maxAllowed = 20
- }
- if val > maxAllowed {
+ if val, ok := parseDecimalValue(t); ok {
+ if val > pageNumberCeiling(numPages) {
return false
}
}
@@ -499,18 +680,143 @@ type zoneKey struct {
text string
}
+// pageNumCand is one bare-number box inside a wide margin band, a candidate
+// for the sequence-backed page-number track.
+type pageNumCand struct {
+ idx int
+ page int
+ value int
+ top float64
+ band string
+}
+
+// collectPageNumberCandidates gathers boxes whose entire text is one bare
+// number (digits or Roman) sitting in the wide margin bands, bounded by the
+// page-count ceiling. It walks EVERY box, including ones earlier tiers already
+// dropped: the +1 chain must stay contiguous when the gap-backed tiers took
+// the roomy pages and only the tight ones remain to this track.
+func collectPageNumberCandidates(boxes []pdf.TextBox, pageHeights map[int]float64, ceiling int) []pageNumCand {
+ var cands []pageNumCand
+ for i := range boxes {
+ b := boxes[i]
+ if isNonTextLayout(b.LayoutType) {
+ continue
+ }
+ h, ok := pageHeights[b.PageNumber]
+ if !ok || h <= 0 {
+ continue
+ }
+ band := ""
+ switch {
+ case b.Top >= h*seqBandBottomRatio:
+ band = "footer"
+ case b.Bottom <= h*seqBandTopRatio:
+ band = "header"
+ default:
+ continue
+ }
+ val, ok := parseBareNumberValue(b.Text)
+ if !ok || val > ceiling {
+ continue
+ }
+ cands = append(cands, pageNumCand{idx: i, page: b.PageNumber, value: val, top: b.Top, band: band})
+ }
+ return cands
+}
+
+// findPageNumberSequenceDrops returns the indices of candidates that take part
+// in a run of at least seqMinRun consecutive pages whose bare numbers step by
+// exactly one (up or down) while the run's total top-edge span stays within
+// seqMaxDy — the same full-window criterion the locality track applies, so a
+// per-hop check never lets drift accumulate past the threshold. Such a run is
+// a page-number series on sequence evidence alone — enough to remove tight
+// footers the geometric zone gate rejected; isolated or non-stepping numbers
+// are left alone.
+func findPageNumberSequenceDrops(cands []pageNumCand) []int {
+ byBand := make(map[string][]pageNumCand, 2)
+ for _, c := range cands {
+ byBand[c.band] = append(byBand[c.band], c)
+ }
+ dropped := make(map[int]bool, len(cands))
+ for _, group := range byBand {
+ if len(group) < seqMinRun {
+ continue
+ }
+ sort.Slice(group, func(i, j int) bool {
+ if group[i].page != group[j].page {
+ return group[i].page < group[j].page
+ }
+ return group[i].top < group[j].top
+ })
+ for _, dir := range []int{1, -1} {
+ n := len(group)
+ dp := make([]int, n)
+ prev := make([]int, n)
+ minTop := make([]float64, n)
+ maxTop := make([]float64, n)
+ for i := range group {
+ dp[i], prev[i] = 1, -1
+ minTop[i], maxTop[i] = group[i].top, group[i].top
+ }
+ for i := 0; i < n; i++ {
+ for j := 0; j < i; j++ {
+ if group[j].page != group[i].page-1 || group[j].value != group[i].value-dir {
+ continue
+ }
+ newMin, newMax := minTop[j], maxTop[j]
+ if group[i].top < newMin {
+ newMin = group[i].top
+ }
+ if group[i].top > newMax {
+ newMax = group[i].top
+ }
+ if newMax-newMin > seqMaxDy {
+ continue
+ }
+ // Among equal-length chains prefer the tighter Y span, so a
+ // wide early detour cannot block a later legal extension.
+ if dp[j]+1 > dp[i] || (dp[j]+1 == dp[i] && newMax-newMin < maxTop[i]-minTop[i]) {
+ dp[i] = dp[j] + 1
+ prev[i] = j
+ minTop[i], maxTop[i] = newMin, newMax
+ }
+ }
+ }
+ for i := 0; i < n; i++ {
+ if dp[i] < seqMinRun {
+ continue
+ }
+ for k := i; k >= 0; k = prev[k] {
+ dropped[group[k].idx] = true
+ }
+ }
+ }
+ }
+ out := make([]int, 0, len(dropped))
+ for idx := range dropped {
+ out = append(out, idx)
+ }
+ return out
+}
+
// RemoveHeaderFooterBoxes drops boxes that are running headers or footers.
//
// It uses a 3D Cascaded Fusion scheme:
// 1. Tier 1: Deterministic Pattern & Semantic Tagging. Explicit DLA header/footer
-// tags in margin zones and unambiguously formatted page numbers are removed
-// immediately (even on 1- or 2-page documents).
+// tags in margin zones, unambiguously formatted page numbers, and margin
+// lines advertising a download site are removed immediately (even on 1- or
+// 2-page documents).
// 2. Tier 2: Dual-Track Recurrence Engine. Evaluates recurrence across:
// - Global half-pages threshold ((numPages + 1) / 2).
// - Parity partition (even/odd) to eliminate bilateral alternating headers.
// - Locality track (consecutive pages with stable Y) to remove chapter-varying headers.
// 3. Tier 3: Adaptive Whitespace Gap. Automatically extends the candidate zone
-// from 10% to up to 14% when separated from body content by >= 18pt of blank space.
+// from 10% to up to 14% when separated from body content by >= 18pt of blank
+// space.
+// 4. Sequence track: bare numeric / Roman-numeral boxes in wide margin bands
+// are removed when they step by one across >= 3 consecutive pages at a
+// stable Y, catching page numbers that sit too tight under body text for
+// the geometric gate in any tier above.
func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) []pdf.TextBox {
if len(pageHeights) == 0 || len(boxes) == 0 {
return boxes
@@ -539,6 +845,9 @@ func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) [
numPages := len(pageHeights)
drop := make(map[int]struct{}, len(boxes))
+ // promoSpots records where a site-promo box was dropped in Tier 1 so the
+ // companion pass can learn its band peers' text as running-header ads.
+ var promoSpots []promoSpot
// Tier 1: Deterministic page numbers & DLA semantic tags.
for i := range boxes {
@@ -550,6 +859,22 @@ func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) [
if !ok || h <= 0 {
continue
}
+
+ // A URL ad in either expanded margin zone (top 14% / bottom 14%,
+ // narrower than the sequence track's 20% bands) is an advertising
+ // header/footer on sight — no whitespace gap and no recurrence required,
+ // and this also fires on documents too short for the recurrence tracks.
+ if isSitePromo(b.Text) && (b.Bottom <= h*headerMaxZoneRatio || b.Top >= h*footerMinZoneRatio) {
+ slog.Debug("header_footer: dropped by site promo", "page", b.PageNumber, "textLen", utf8.RuneCountInString(b.Text))
+ drop[i] = struct{}{}
+ if b.Bottom <= h*headerMaxZoneRatio {
+ promoSpots = append(promoSpots, promoSpot{page: b.PageNumber, band: "header"})
+ } else {
+ promoSpots = append(promoSpots, promoSpot{page: b.PageNumber, band: "footer"})
+ }
+ continue
+ }
+
zone := classifyZone(b, h, allGapAbove[i], allGapBelow[i])
if zone == "" {
continue
@@ -582,6 +907,14 @@ func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) [
return applyDrop(boxes, drop)
}
+ // Promo-companion propagation. Runs after Tier 1 and before the recurrence
+ // engine, so the boxes it removes never enter the recurrence statistics
+ // (which already skip dropped indices) and cannot perturb any "#"-masked count.
+ for _, idx := range promoCompanionDrops(boxes, perPage, pageHeights, drop, promoSpots) {
+ slog.Debug("header_footer: dropped by promo companion", "page", boxes[idx].PageNumber, "zone", bandOf(boxes[idx], pageHeights[boxes[idx].PageNumber]), "textLen", utf8.RuneCountInString(boxes[idx].Text))
+ drop[idx] = struct{}{}
+ }
+
// Tier 2: Dual-Track Recurrence Engine.
keyMetas := make(map[zoneKey][]boxMeta)
for i := range boxes {
@@ -676,6 +1009,21 @@ func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) [
}
}
+ // Sequence-backed page numbers. Word-derived books sit their page numbers
+ // a few points under body text — below the 90% footer line AND below the
+ // expanded band's whitespace requirement — so the geometric gate rejects
+ // them before any tier can see a pattern or count a recurrence. A run of
+ // bare numbers stepping by one across consecutive pages at a stable Y is a
+ // page-number series on sequence evidence alone. This runs after the
+ // recurrence engine so it only adds drops: the recurrence counts, which
+ // include these boxes under their masked key, stay untouched.
+ if seqDrops := findPageNumberSequenceDrops(collectPageNumberCandidates(boxes, pageHeights, pageNumberCeiling(numPages))); len(seqDrops) > 0 {
+ slog.Debug("header_footer: dropped by page-number sequence", "boxes", len(seqDrops))
+ for _, idx := range seqDrops {
+ drop[idx] = struct{}{}
+ }
+ }
+
if len(drop) == 0 {
return boxes
}
@@ -684,12 +1032,6 @@ func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) [
return applyDrop(boxes, drop)
}
-// hasStableConsecutiveRun checks if metas has a run of at least minRun
-// consecutive pages where the top variation is within maxDy.
-func hasStableConsecutiveRun(metas []boxMeta, minRun int, maxDy float64) bool {
- return len(findStableConsecutiveRunIndices(metas, minRun, maxDy)) > 0
-}
-
// findStableConsecutiveRunIndices finds all box indices that belong to any qualifying
// consecutive run of at least minRun pages where the top variation is within maxDy.
// Restricting removal to these specific indices ensures isolated occurrences outside
@@ -781,6 +1123,14 @@ func applyDrop(boxes []pdf.TextBox, drop map[int]struct{}) []pdf.TextBox {
return out
}
+// collapseText lower-cases and whitespace-collapses text without masking
+// digits, for comparison keys that must not treat two differently-numbered
+// lines as identical (unlike normalizeRunningText, which is for running
+// headers/footers whose per-page number must be folded away).
+func collapseText(text string) string {
+ return strings.ToLower(strings.Join(strings.Fields(text), " "))
+}
+
// normalizeRunningText collapses whitespace, replaces digit runs with a single
// '#' and lower-cases the result, so per-page variants of the same running
// header/footer ("- 2 -", "- 3 -", "Page 4 of 9", "第 1 页") share one
diff --git a/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go b/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go
index 2c4e66be31..0785f1e349 100644
--- a/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go
+++ b/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go
@@ -1027,9 +1027,10 @@ func TestRemoveHeaderFooterBoxes_ChineseSlashGong(t *testing.T) {
}
}
-// TestHasStableConsecutiveRun_SlidingWindow tests sliding window behavior in hasStableConsecutiveRun,
-// specifically verifying that initial outliers do not block subsequent qualifying runs.
-func TestHasStableConsecutiveRun_SlidingWindow(t *testing.T) {
+// TestFindStableConsecutiveRunIndices_SlidingWindow tests sliding window behavior in
+// findStableConsecutiveRunIndices, specifically verifying that initial outliers do
+// not block subsequent qualifying runs.
+func TestFindStableConsecutiveRunIndices_SlidingWindow(t *testing.T) {
// Case 1: Outlier at page 1 (top 30), stable run at pages 2, 3, 4 (top 40)
metas1 := []boxMeta{
{page: 1, top: 30},
@@ -1037,8 +1038,8 @@ func TestHasStableConsecutiveRun_SlidingWindow(t *testing.T) {
{page: 3, top: 40},
{page: 4, top: 40},
}
- if !hasStableConsecutiveRun(metas1, 3, 4.0) {
- t.Errorf("expected true for pages 2..4 stable run despite page 1 outlier")
+ if len(findStableConsecutiveRunIndices(metas1, 3, 4.0)) == 0 {
+ t.Errorf("expected pages 2..4 stable run despite page 1 outlier")
}
// Case 2: Stable run at pages 1, 2, 3 (top 40), outlier at page 4 (top 30)
@@ -1048,8 +1049,8 @@ func TestHasStableConsecutiveRun_SlidingWindow(t *testing.T) {
{page: 3, top: 40},
{page: 4, top: 30},
}
- if !hasStableConsecutiveRun(metas2, 3, 4.0) {
- t.Errorf("expected true for pages 1..3 stable run")
+ if len(findStableConsecutiveRunIndices(metas2, 3, 4.0)) == 0 {
+ t.Errorf("expected pages 1..3 stable run")
}
// Case 3: Divergent tops across all pages (no 3 consecutive pages within 4.0)
@@ -1059,8 +1060,8 @@ func TestHasStableConsecutiveRun_SlidingWindow(t *testing.T) {
{page: 3, top: 50},
{page: 4, top: 70},
}
- if hasStableConsecutiveRun(metas3, 3, 4.0) {
- t.Errorf("expected false for divergent tops")
+ if len(findStableConsecutiveRunIndices(metas3, 3, 4.0)) != 0 {
+ t.Errorf("expected no run for divergent tops")
}
// Case 4: Page gap breaks consecutive run
@@ -1070,8 +1071,8 @@ func TestHasStableConsecutiveRun_SlidingWindow(t *testing.T) {
{page: 4, top: 40},
{page: 5, top: 40},
}
- if hasStableConsecutiveRun(metas4, 3, 4.0) {
- t.Errorf("expected false when no consecutive run reaches minRun=3")
+ if len(findStableConsecutiveRunIndices(metas4, 3, 4.0)) != 0 {
+ t.Errorf("expected no run when no consecutive run reaches minRun=3")
}
// Case 5: Run after a page gap meets minRun=3
@@ -1082,8 +1083,8 @@ func TestHasStableConsecutiveRun_SlidingWindow(t *testing.T) {
{page: 5, top: 40},
{page: 6, top: 40},
}
- if !hasStableConsecutiveRun(metas5, 3, 4.0) {
- t.Errorf("expected true for pages 4..6 run after page gap")
+ if len(findStableConsecutiveRunIndices(metas5, 3, 4.0)) == 0 {
+ t.Errorf("expected pages 4..6 run after page gap")
}
}
@@ -1211,3 +1212,342 @@ func TestRemoveHeaderFooterBoxes_FooterYearPreservedOnShortDocument(t *testing.T
t.Fatalf("expected all 3 boxes to be kept, got %d", len(got))
}
}
+
+// tightFooter builds one page for the sequence-track tests: a body line ending
+// at bottom, a bare-number box 5pt under it at Top=755 (below the 757.8 footer
+// line, inside the 0.86 expanded band without its gap) — geometry the zone gate
+// rejects, leaving the +1 sequence as the only evidence.
+func tightFooter(pg int, numText string) []pdf.TextBox {
+ return []pdf.TextBox{
+ tb(fmt.Sprintf("Body paragraph on page %d with a little extra length to stay unique.", pg), pg, 72, 500, 100, 750),
+ tb(numText, pg, 400, 420, 755, 766),
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_TightSequenceFootersRemoved verifies that bare page
+// numbers sitting tight under body text — rejected by both the base footer line
+// and the expanded band's whitespace requirement — are removed by the sequence
+// track when they step by one across consecutive pages.
+func TestRemoveHeaderFooterBoxes_TightSequenceFootersRemoved(t *testing.T) {
+ pageHeight := 842.0
+ t.Run("arabic", func(t *testing.T) {
+ heights := make(map[int]float64, 10)
+ var boxes []pdf.TextBox
+ for pg := 0; pg < 10; pg++ {
+ heights[pg] = pageHeight
+ boxes = append(boxes, tightFooter(pg, fmt.Sprintf("%d", pg+1))...)
+ }
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ for _, b := range got {
+ if _, ok := parseBareNumberValue(b.Text); ok {
+ t.Fatalf("page number %q survived the sequence track", b.Text)
+ }
+ }
+ if len(got) != 10 {
+ t.Fatalf("expected 10 body boxes, got %d", len(got))
+ }
+ })
+ t.Run("roman", func(t *testing.T) {
+ heights := make(map[int]float64, 6)
+ boxes := []pdf.TextBox{}
+ romans := []string{"I", "II", "III", "IV", "V", "VI"}
+ for pg, r := range romans {
+ heights[pg] = pageHeight
+ boxes = append(boxes, tightFooter(pg, r)...)
+ }
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ for _, b := range got {
+ if _, ok := parseBareNumberValue(b.Text); ok {
+ t.Fatalf("Roman page number %q survived the sequence track", b.Text)
+ }
+ }
+ if len(got) != 6 {
+ t.Fatalf("expected 6 body boxes, got %d", len(got))
+ }
+ })
+}
+
+// TestRemoveHeaderFooterBoxes_NumbersNoSequencePreserved verifies the sequence
+// track's guards: numbers that do not step by one, and runs shorter than 3
+// consecutive pages, are left alone.
+func TestRemoveHeaderFooterBoxes_NumbersNoSequencePreserved(t *testing.T) {
+ pageHeight := 842.0
+ t.Run("non_stepping", func(t *testing.T) {
+ heights := make(map[int]float64, 5)
+ var boxes []pdf.TextBox
+ nums := []string{"3", "7", "12", "15", "19"}
+ for pg, n := range nums {
+ heights[pg] = pageHeight
+ boxes = append(boxes, tightFooter(pg, n)...)
+ }
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ if len(got) != 10 {
+ t.Fatalf("non-stepping numbers must be preserved, got %d kept boxes", len(got))
+ }
+ })
+ t.Run("run_too_short", func(t *testing.T) {
+ // +1 steps exist (1,2 and 4,5) but each chain is broken before length 3.
+ heights := make(map[int]float64, 6)
+ var boxes []pdf.TextBox
+ for pg := 0; pg < 6; pg++ {
+ heights[pg] = pageHeight
+ }
+ boxes = append(boxes, tightFooter(0, "1")...)
+ boxes = append(boxes, tightFooter(1, "2")...)
+ boxes = append(boxes, tightFooter(3, "4")...)
+ boxes = append(boxes, tightFooter(4, "5")...)
+ boxes = append(boxes, tb("Plain body line, no number box on this page.", 2, 72, 500, 100, 750))
+ boxes = append(boxes, tb("Plain body line here too.", 5, 72, 500, 100, 750))
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ if len(got) != 10 {
+ t.Fatalf("2-page number chains must be preserved, got %d kept boxes", len(got))
+ }
+ })
+}
+
+// TestRemoveHeaderFooterBoxes_SequenceDriftSpanGuard verifies the sequence
+// track bounds TOTAL Y span across a chain, not just each hop: per-hop drift
+// that accumulates past seqMaxDy must not remove the run, while a chain whose
+// full span stays inside the threshold still does.
+func TestRemoveHeaderFooterBoxes_SequenceDriftSpanGuard(t *testing.T) {
+ pageHeight := 842.0
+ footerAt := func(pg int, numText string, top float64) []pdf.TextBox {
+ return []pdf.TextBox{
+ tb(fmt.Sprintf("Body paragraph on page %d with a little extra length to stay unique.", pg), pg, 72, 500, 100, 674),
+ tb(numText, pg, 400, 420, top, top+11), // Top 680-ish: wide band, zone stays empty (gap 6pt, above 724.1 line only)
+ }
+ }
+ t.Run("accumulating_drift_preserved", func(t *testing.T) {
+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}
+ var boxes []pdf.TextBox
+ tops := []float64{680, 684, 688} // each hop 4pt, total span 8pt > seqMaxDy
+ for pg, top := range tops {
+ boxes = append(boxes, footerAt(pg, fmt.Sprintf("%d", pg+1), top)...)
+ }
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ if len(got) != 6 {
+ t.Fatalf("chain drifting 8pt in total must be preserved, got %d kept boxes", len(got))
+ }
+ })
+ t.Run("within_span_removed", func(t *testing.T) {
+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}
+ var boxes []pdf.TextBox
+ tops := []float64{680, 681.5, 683} // total span 3pt <= seqMaxDy
+ for pg, top := range tops {
+ boxes = append(boxes, footerAt(pg, fmt.Sprintf("%d", pg+1), top)...)
+ }
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ for _, b := range got {
+ if _, ok := parseBareNumberValue(b.Text); ok {
+ t.Fatalf("page number %q survived the sequence track", b.Text)
+ }
+ }
+ if len(got) != 3 {
+ t.Fatalf("chain within total-drift span must be removed, got %d kept boxes", len(got))
+ }
+ })
+}
+
+// TestRemoveHeaderFooterBoxes_YearFooterSequenceCeilingPreserved verifies that a
+// stepping year sequence in the footer band ("2024","2025","2026") is protected
+// by the page-count ceiling even though it would otherwise satisfy the +1 chain.
+func TestRemoveHeaderFooterBoxes_YearFooterSequenceCeilingPreserved(t *testing.T) {
+ pageHeight := 842.0
+ heights := make(map[int]float64, 3)
+ var boxes []pdf.TextBox
+ years := []string{"2024", "2025", "2026"}
+ for pg, y := range years {
+ heights[pg] = pageHeight
+ boxes = append(boxes, tightFooter(pg, y)...)
+ }
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ keptYears := make(map[string]bool, len(years))
+ for _, b := range got {
+ keptYears[b.Text] = true
+ }
+ for _, y := range years {
+ if !keptYears[y] {
+ t.Fatalf("year footer %q must be preserved: its value exceeds the page-number ceiling", y)
+ }
+ }
+ if len(got) != 6 {
+ t.Fatalf("year footers exceed the page-number ceiling and must be preserved, got %d", len(got))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_TightBottomTextHeadingPreserved verifies the wide
+// sequence bands never admit non-numeric text: a repeated section heading 6pt
+// above the page bottom must survive even though its text repeats on every page.
+func TestRemoveHeaderFooterBoxes_TightBottomTextHeadingPreserved(t *testing.T) {
+ pageHeight := 842.0
+ heights := make(map[int]float64, 4)
+ var boxes []pdf.TextBox
+ for pg := 0; pg < 4; pg++ {
+ heights[pg] = pageHeight
+ boxes = append(boxes, tb("Body text filling most of the page above the heading.", pg, 72, 500, 100, 724))
+ boxes = append(boxes, tb("Quarterly Report Summary", pg, 72, 300, 730, 742)) // gapAbove = 6pt, repeats identically
+ }
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ var headings int
+ for _, b := range got {
+ if b.Text == "Quarterly Report Summary" {
+ headings++
+ }
+ }
+ if headings != 4 || len(got) != 8 {
+ t.Fatalf("tight repeated text headings must be preserved, got %d of 4 headings (%d boxes)", headings, len(got))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_SitePromoVariantInHeaderRemoved verifies that margin
+// lines advertising a download site are removed even when the wrapper text varies
+// page to page and no recurrence track can see them.
+func TestRemoveHeaderFooterBoxes_SitePromoVariantInHeaderRemoved(t *testing.T) {
+ pageHeight := 842.0
+ heights := make(map[int]float64, 4)
+ var boxes []pdf.TextBox
+ for pg := 0; pg < 4; pg++ {
+ heights[pg] = pageHeight
+ // The body is written out per page; only the promo lines matter.
+ boxes = append(boxes, tb(fmt.Sprintf("Substantive body content of page %d goes here.", pg), pg, 72, 450, 160, 700))
+ if pg < 2 {
+ boxes = append(boxes, tb("更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879", pg, 60, 300, 40, 52))
+ } else {
+ boxes = append(boxes, tb("欢迎访问!http://forum.law58.cn/?fromuid=381879", pg, 60, 300, 40, 52))
+ }
+ }
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ for _, b := range got {
+ if strings.Contains(b.Text, "law58") {
+ t.Fatalf("site promo %q survived despite varying wrappers", b.Text)
+ }
+ }
+ if len(got) != 4 {
+ t.Fatalf("expected 4 body boxes, got %d", len(got))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_BodyUrlKept verifies the site-promo rule cannot
+// reach into the body: a mid-page URL box and a long prose line containing a
+// URL are both kept.
+func TestRemoveHeaderFooterBoxes_BodyUrlKept(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight}
+ boxes := []pdf.TextBox{
+ // Mid-page short URL: inside no band at all.
+ tb("http://a.cn/x", 0, 72, 130, 400, 415),
+ // > 60 runes: length-capped out even though it starts in the top band.
+ tb("See http://example.com/very/long/path/for/the/annual/errata/annex which continues well beyond sixty runes.", 1, 72, 500, 40, 55),
+ tb("Ordinary body text.", 1, 72, 300, 500, 520),
+ }
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ if len(got) != 3 {
+ t.Fatalf("body URLs must be preserved, got %d", len(got))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_DropsPromoCompanion verifies that a fixed slogan
+// riding in the same margin band as a dropped site-promo ad is learned from the
+// pages where they co-occur and propagated to every page, so the ad is gone
+// even where its URL line was never detected.
+func TestRemoveHeaderFooterBoxes_DropsPromoCompanion(t *testing.T) {
+ pageHeight := 842.0 // header band: bottom <= 117.9; body starts well below.
+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight}
+ const slogan = "木瓜树 更多的书籍免费下载"
+ var boxes []pdf.TextBox
+ for pg := 0; pg < 3; pg++ { // pages 0,1,2: ad URL + slogan companion
+ boxes = append(boxes,
+ tb("http://forum.law58.cn/?fromuid=381879", pg, 60, 300, 40, 52),
+ tb(slogan, pg, 60, 300, 60, 72),
+ tb(fmt.Sprintf("Substantive body content of page %d.", pg), pg, 72, 450, 200, 700),
+ )
+ }
+ // page 3 carries only the slogan line, no URL.
+ boxes = append(boxes,
+ tb(slogan, 3, 60, 300, 60, 72),
+ tb("Substantive body content of page 3.", 3, 72, 450, 200, 700),
+ )
+
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ for _, b := range got {
+ if strings.Contains(b.Text, "law58") {
+ t.Fatalf("site promo survived: %q", b.Text)
+ }
+ if strings.Contains(b.Text, "木瓜树") {
+ t.Fatalf("promo companion slogan survived: %q", b.Text)
+ }
+ }
+ if len(got) != 4 {
+ t.Fatalf("expected 4 body boxes, got %d", len(got))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_PromoCompanionKeepsTitle verifies a LayoutType
+// "title" line that merely shares the header band with an ad is never harvested
+// as a companion, so legitimate running section titles survive.
+func TestRemoveHeaderFooterBoxes_PromoCompanionKeepsTitle(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}
+ titles := []string{"[图甲]", "[图乙]", "[图丙]"} // unique per page: no recurrence to lean on
+ var boxes []pdf.TextBox
+ for pg := 0; pg < 3; pg++ {
+ boxes = append(boxes,
+ tb("http://forum.law58.cn/?fromuid=381879", pg, 60, 300, 40, 52),
+ pdf.TextBox{Text: titles[pg], PageNumber: pg, X0: 60, X1: 200, Top: 60, Bottom: 72, LayoutType: "title"},
+ tb(fmt.Sprintf("Body content of page %d here.", pg), pg, 72, 450, 200, 700),
+ )
+ }
+
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ seenTitle := 0
+ for _, b := range got {
+ if strings.Contains(b.Text, "law58") {
+ t.Fatalf("site promo survived: %q", b.Text)
+ }
+ if strings.HasPrefix(b.Text, "[图") {
+ seenTitle++
+ }
+ }
+ if seenTitle != 3 {
+ t.Fatalf("all 3 title lines must survive as companions excluded, got %d", seenTitle)
+ }
+ if len(got) != 6 {
+ t.Fatalf("expected 3 titles + 3 bodies, got %d", len(got))
+ }
+}
+
+// TestRemoveHeaderFooterBoxes_PromoCompanionKeepsRarePeer verifies a short
+// non-title band peer that co-occurs with the ad on only one page is not
+// harvested — the page-count floor stops one-off neighbours.
+func TestRemoveHeaderFooterBoxes_PromoCompanionKeepsRarePeer(t *testing.T) {
+ pageHeight := 842.0
+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}
+ const slogan = "木瓜树 更多的书籍免费下载"
+ var boxes []pdf.TextBox
+ for pg := 0; pg < 3; pg++ {
+ boxes = append(boxes,
+ tb("http://forum.law58.cn/?fromuid=381879", pg, 60, 300, 40, 52),
+ tb(slogan, pg, 60, 300, 60, 72),
+ tb(fmt.Sprintf("Body content of page %d here.", pg), pg, 72, 450, 200, 700),
+ )
+ }
+ // A short slogan-like peer present only on page 0, next to its ad.
+ boxes = append(boxes, tb("扫码加入读书群", 0, 60, 300, 80, 92))
+
+ got := RemoveHeaderFooterBoxes(boxes, heights)
+ var keptRare int
+ for _, b := range got {
+ if strings.Contains(b.Text, "law58") || strings.Contains(b.Text, "木瓜树") {
+ t.Fatalf("promo/slogan survived: %q", b.Text)
+ }
+ if b.Text == "扫码加入读书群" {
+ keptRare++
+ }
+ }
+ if keptRare != 1 {
+ t.Fatalf("rare single-page peer must survive, got %d", keptRare)
+ }
+ if len(got) != 4 {
+ t.Fatalf("expected 3 bodies + 1 rare peer, got %d", len(got))
+ }
+}