RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): catch tight page numbers and site-promo headers/footers (#20019)
jay77721dae80165322d
diff --git a/internal/deepdoc/parser/pdf/layout/toc_header_footer.go b/internal/deepdoc/parser/pdf/layout/toc_header_footer.goindex d68b312935..4f1dce8dae 100644--- a/internal/deepdoc/parser/pdf/layout/toc_header_footer.go+++ b/internal/deepdoc/parser/pdf/layout/toc_header_footer.go@@ -347,6 +347,28 @@ const (// even or odd pages. A 0.40 threshold captures alternating bilateral book headers// while the evenPages >= 3 / oddPages >= 3 count guards against accidental matches.minParityCoverageRatio = 0.40+ // seqBandTopRatio / seqBandBottomRatio bound the wide bands in which the+ // sequence-backed page-number track looks for bare numeric / Roman-numeral+ // boxes. The bands are much looser than the header/footer zones because the+ // track's evidence is not geometric: only boxes that form an incrementing+ // run across consecutive pages at a stable Y are removed, which tight body+ // text cannot fake.+ seqBandTopRatio = 0.20+ seqBandBottomRatio = 0.80+ // seqMinRun / seqMaxDy mirror the locality track: at least 3 consecutive+ // pages, total top-edge span across the run within 4pt.+ seqMinRun = 3+ seqMaxDy = 4.0+ // promoCompanionMaxRunes bounds the length of a margin line that may be+ // learned as a running-header ad companion of a dropped site-promo box.+ // A piracy header is a short slogan, so anything longer is treated as body+ // text and never learned.+ promoCompanionMaxRunes = 24+ // minCompanionPages is how many distinct pages must carry the same band+ // peer of a dropped site-promo box before that peer's text is treated as a+ // running-header ad and propagated to every page. Requiring several pages+ // keeps a one-off line that merely shares a page with an ad intact.+ minCompanionPages = 3)var (@@ -360,6 +382,23 @@ var (// strictBareNumberPattern matches lone Arabic digits or Roman numerals: 1, 23, iv, VII.strictBareNumberPattern = regexp.MustCompile(`(?i)^([0-90-9]{1,4}|((l|xl|x{1,3})(ix|iv|v?i{1,3}|v)?|ix|iv|v?i{1,3}|v))$`)++ // canonicalRomanPattern accepts the full canonical Roman-numeral grammar+ // (up to M) so the sequence track can parse page numbers beyond the XXX+ // range strictBareNumberPattern's header/footer idiom covers. Non-canonical+ // forms (IIII) are rejected by the pattern itself; romanValue only computes+ // the subtractive sum of a spelling this regex already accepted.+ canonicalRomanPattern = regexp.MustCompile(`(?i)^(m{0,3})(cm|cd|d?c{0,3})(xc|xl|l?x{0,3})(ix|iv|v?i{0,3})$`)++ // sitePromoPattern matches margin lines that advertise a download site —+ // a URL core with at most a short non-URL wrapper on either side. Such a+ // line is a running header/footer regardless of page count: piracy-supply+ // PDFs change the wrapper text on some pages ("更多的书籍免费下载 http://…",+ // "欢迎访问!http://…"), so those variants reach no recurrence track, while+ // the URL itself is deterministic evidence. Only whole-box, length-capped+ // boxes in the margin bands use it, so body prose containing a URL is never+ // touched.+ sitePromoPattern = regexp.MustCompile(`(?i)^.{0,20}(https?://\S+|www\.[\w-]+(\.[\w-]+)+|(forum|bbs|tieba|phpwind)\.[\w-]+\.[a-z]{2,}\S*|\S*fromuid=\S+).{0,20}$`))// computePageGaps computes for candidate margin boxes on a page:@@ -441,6 +480,164 @@ func isNonTextLayout(lt string) bool {return lt == "table" || lt == "figure" || lt == "equation" || lt == "image"}+// parseDecimalValue parses a string of lone ASCII or full-width digits.+func parseDecimalValue(t string) (int, bool) {+ if t == "" {+ return 0, false+ }+ val := 0+ for _, r := range t {+ switch {+ case r >= '0' && r <= '9':+ val = val*10 + int(r-'0')+ case r >= '0' && r <= '9':+ val = val*10 + int(r-'0')+ default:+ return 0, false+ }+ }+ return val, true+}++// romanValue parses a canonical Roman numeral (I..MMMCMXCIX) and returns its+// value. The canonical pattern admits only canonical spellings, so non-canonical+// forms such as "IIII" are rejected before the subtractive sum is computed.+func romanValue(t string) (int, bool) {+ if !canonicalRomanPattern.MatchString(t) {+ return 0, false+ }+ values := map[rune]int{'m': 1000, 'd': 500, 'c': 100, 'l': 50, 'x': 10, 'v': 5, 'i': 1}+ val := 0+ prev := 0+ for _, r := range strings.ToLower(t) {+ v := values[r]+ if v == 0 {+ return 0, false+ }+ if v > prev {+ val += v - 2*prev+ } else {+ val += v+ }+ prev = v+ }+ if val <= 0 {+ return 0, false+ }+ return val, true+}++// parseBareNumberValue reports the numeric value of a box whose entire text is+// one bare number: lone digits (ASCII or full-width) or a canonical Roman+// numeral. "12", "45", "iv", "XIV" parse; "12.", "- 3 -", "2024." do not.+func parseBareNumberValue(text string) (int, bool) {+ t := strings.TrimSpace(text)+ if v, ok := parseDecimalValue(t); ok {+ return v, true+ }+ return romanValue(t)+}++// pageNumberCeiling bounds which bare numeric values can be page numbers: a+// document of N pages never carries page number > N+5, and the floor of 20+// keeps short documents (years like 2024 on a 2-page footer) protected.+func pageNumberCeiling(numPages int) int {+ maxAllowed := numPages + 5+ if maxAllowed < 20 {+ maxAllowed = 20+ }+ return maxAllowed+}++// isSitePromo reports whether the whole box text advertises a download site.+func isSitePromo(text string) bool {+ t := strings.Join(strings.Fields(text), " ")+ if t == "" || utf8.RuneCountInString(t) > 60 {+ return false+ }+ return sitePromoPattern.MatchString(t)+}++// bandOf reports which expanded margin band ("header"/"footer") a box sits in,+// or "" when it is outside both. It uses the same zone bounds the site-promo+// tier applies, so a box learned as an ad companion and a box later matched+// against it are judged by one rule.+func bandOf(b pdf.TextBox, pageHeight float64) string {+ switch {+ case b.Bottom <= pageHeight*headerMaxZoneRatio:+ return "header"+ case b.Top >= pageHeight*footerMinZoneRatio:+ return "footer"+ }+ return ""+}++// promoSpot is the page and band where a site-promo box was dropped, recorded+// so the companion pass can harvest its in-band peers as running-header ads.+type promoSpot struct {+ page int+ band string+}++// promoCompanionDrops returns indices of margin-band boxes that are the slogan+// half of an advertisement already seen in the same band next to a dropped+// site-promo box. It harvests the short, non-title, in-band peers of every+// promo spot, then flags every band box whose collapsed text was harvested on+// at least minCompanionPages distinct pages — so a piracy header whose URL line+// is omitted or split on some pages still goes. The peers are keyed on+// collapseText (whitespace + case only, digits preserved) so two lines that+// differ only by a number are never merged. `skip` holds indices already doomed+// by an earlier tier and is consulted, never mutated.+func promoCompanionDrops(boxes []pdf.TextBox, perPage map[int][]int, pageHeights map[int]float64, skip map[int]struct{}, promoSpots []promoSpot) []int {+ companionPages := make(map[string]map[int]bool)+ for _, sp := range promoSpots {+ h := pageHeights[sp.page]+ for _, j := range perPage[sp.page] {+ if _, dropped := skip[j]; dropped {+ continue+ }+ b := boxes[j]+ if isNonTextLayout(b.LayoutType) || strings.TrimSpace(b.LayoutType) == "title" {+ continue+ }+ if bandOf(b, h) != sp.band {+ continue+ }+ key := collapseText(b.Text)+ if key == "" || utf8.RuneCountInString(key) > promoCompanionMaxRunes {+ continue+ }+ if companionPages[key] == nil {+ companionPages[key] = make(map[int]bool)+ }+ companionPages[key][sp.page] = true+ }+ }+ var out []int+ for key, pages := range companionPages {+ if len(pages) < minCompanionPages {+ continue+ }+ for i := range boxes {+ if _, dropped := skip[i]; dropped {+ continue+ }+ b := boxes[i]+ if isNonTextLayout(b.LayoutType) || strings.TrimSpace(b.LayoutType) == "title" {+ continue+ }+ h, ok := pageHeights[b.PageNumber]+ if !ok || h <= 0 {+ continue+ }+ if bandOf(b, h) != "" && collapseText(b.Text) == key {+ out = append(out, i)+ }+ }+ }+ return out+}+// isDeterministicPageNumber reports whether text is unambiguously a page number.func isDeterministicPageNumber(text string, zone string, gapAbove, gapBelow float64, numPages int) bool {t := strings.TrimSpace(text)@@ -462,24 +659,8 @@ func isDeterministicPageNumber(text string, zone string, gapAbove, gapBelow floa}// For bare numeric digits, verify the value is within a reasonable range of the document page count.// Protects isolated years (e.g. "2024"), IDs, or footnote indexes on short/medium documents.- var val int- isDigits := true- for _, r := range t {- if r >= '0' && r <= '9' {- val = val*10 + int(r-'0')- } else if r >= '0' && r <= '9' {- val = val*10 + int(r-'0')- } else {- isDigits = false- break- }- }- if isDigits && len(t) > 0 {- maxAllowed := numPages + 5- if maxAllowed < 20 {- maxAllowed = 20- }- if val > maxAllowed {+ if val, ok := parseDecimalValue(t); ok {+ if val > pageNumberCeiling(numPages) {return false}}@@ -499,18 +680,143 @@ type zoneKey struct {text string}+// pageNumCand is one bare-number box inside a wide margin band, a candidate+// for the sequence-backed page-number track.+type pageNumCand struct {+ idx int+ page int+ value int+ top float64+ band string+}++// collectPageNumberCandidates gathers boxes whose entire text is one bare+// number (digits or Roman) sitting in the wide margin bands, bounded by the+// page-count ceiling. It walks EVERY box, including ones earlier tiers already+// dropped: the +1 chain must stay contiguous when the gap-backed tiers took+// the roomy pages and only the tight ones remain to this track.+func collectPageNumberCandidates(boxes []pdf.TextBox, pageHeights map[int]float64, ceiling int) []pageNumCand {+ var cands []pageNumCand+ for i := range boxes {+ b := boxes[i]+ if isNonTextLayout(b.LayoutType) {+ continue+ }+ h, ok := pageHeights[b.PageNumber]+ if !ok || h <= 0 {+ continue+ }+ band := ""+ switch {+ case b.Top >= h*seqBandBottomRatio:+ band = "footer"+ case b.Bottom <= h*seqBandTopRatio:+ band = "header"+ default:+ continue+ }+ val, ok := parseBareNumberValue(b.Text)+ if !ok || val > ceiling {+ continue+ }+ cands = append(cands, pageNumCand{idx: i, page: b.PageNumber, value: val, top: b.Top, band: band})+ }+ return cands+}++// findPageNumberSequenceDrops returns the indices of candidates that take part+// in a run of at least seqMinRun consecutive pages whose bare numbers step by+// exactly one (up or down) while the run's total top-edge span stays within+// seqMaxDy — the same full-window criterion the locality track applies, so a+// per-hop check never lets drift accumulate past the threshold. Such a run is+// a page-number series on sequence evidence alone — enough to remove tight+// footers the geometric zone gate rejected; isolated or non-stepping numbers+// are left alone.+func findPageNumberSequenceDrops(cands []pageNumCand) []int {+ byBand := make(map[string][]pageNumCand, 2)+ for _, c := range cands {+ byBand[c.band] = append(byBand[c.band], c)+ }+ dropped := make(map[int]bool, len(cands))+ for _, group := range byBand {+ if len(group) < seqMinRun {+ continue+ }+ sort.Slice(group, func(i, j int) bool {+ if group[i].page != group[j].page {+ return group[i].page < group[j].page+ }+ return group[i].top < group[j].top+ })+ for _, dir := range []int{1, -1} {+ n := len(group)+ dp := make([]int, n)+ prev := make([]int, n)+ minTop := make([]float64, n)+ maxTop := make([]float64, n)+ for i := range group {+ dp[i], prev[i] = 1, -1+ minTop[i], maxTop[i] = group[i].top, group[i].top+ }+ for i := 0; i < n; i++ {+ for j := 0; j < i; j++ {+ if group[j].page != group[i].page-1 || group[j].value != group[i].value-dir {+ continue+ }+ newMin, newMax := minTop[j], maxTop[j]+ if group[i].top < newMin {+ newMin = group[i].top+ }+ if group[i].top > newMax {+ newMax = group[i].top+ }+ if newMax-newMin > seqMaxDy {+ continue+ }+ // Among equal-length chains prefer the tighter Y span, so a+ // wide early detour cannot block a later legal extension.+ if dp[j]+1 > dp[i] || (dp[j]+1 == dp[i] && newMax-newMin < maxTop[i]-minTop[i]) {+ dp[i] = dp[j] + 1+ prev[i] = j+ minTop[i], maxTop[i] = newMin, newMax+ }+ }+ }+ for i := 0; i < n; i++ {+ if dp[i] < seqMinRun {+ continue+ }+ for k := i; k >= 0; k = prev[k] {+ dropped[group[k].idx] = true+ }+ }+ }+ }+ out := make([]int, 0, len(dropped))+ for idx := range dropped {+ out = append(out, idx)+ }+ return out+}+// RemoveHeaderFooterBoxes drops boxes that are running headers or footers.//// It uses a 3D Cascaded Fusion scheme:// 1. Tier 1: Deterministic Pattern & Semantic Tagging. Explicit DLA header/footer-// tags in margin zones and unambiguously formatted page numbers are removed-// immediately (even on 1- or 2-page documents).+// tags in margin zones, unambiguously formatted page numbers, and margin+// lines advertising a download site are removed immediately (even on 1- or+// 2-page documents).// 2. Tier 2: Dual-Track Recurrence Engine. Evaluates recurrence across:// - Global half-pages threshold ((numPages + 1) / 2).// - Parity partition (even/odd) to eliminate bilateral alternating headers.// - Locality track (consecutive pages with stable Y) to remove chapter-varying headers.// 3. Tier 3: Adaptive Whitespace Gap. Automatically extends the candidate zone-// from 10% to up to 14% when separated from body content by >= 18pt of blank space.+// from 10% to up to 14% when separated from body content by >= 18pt of blank+// space.+// 4. Sequence track: bare numeric / Roman-numeral boxes in wide margin bands+// are removed when they step by one across >= 3 consecutive pages at a+// stable Y, catching page numbers that sit too tight under body text for+// the geometric gate in any tier above.func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) []pdf.TextBox {if len(pageHeights) == 0 || len(boxes) == 0 {return boxes@@ -539,6 +845,9 @@ func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) [numPages := len(pageHeights)drop := make(map[int]struct{}, len(boxes))+ // promoSpots records where a site-promo box was dropped in Tier 1 so the+ // companion pass can learn its band peers' text as running-header ads.+ var promoSpots []promoSpot// Tier 1: Deterministic page numbers & DLA semantic tags.for i := range boxes {@@ -550,6 +859,22 @@ func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) [if !ok || h <= 0 {continue}++ // A URL ad in either expanded margin zone (top 14% / bottom 14%,+ // narrower than the sequence track's 20% bands) is an advertising+ // header/footer on sight — no whitespace gap and no recurrence required,+ // and this also fires on documents too short for the recurrence tracks.+ if isSitePromo(b.Text) && (b.Bottom <= h*headerMaxZoneRatio || b.Top >= h*footerMinZoneRatio) {+ slog.Debug("header_footer: dropped by site promo", "page", b.PageNumber, "textLen", utf8.RuneCountInString(b.Text))+ drop[i] = struct{}{}+ if b.Bottom <= h*headerMaxZoneRatio {+ promoSpots = append(promoSpots, promoSpot{page: b.PageNumber, band: "header"})+ } else {+ promoSpots = append(promoSpots, promoSpot{page: b.PageNumber, band: "footer"})+ }+ continue+ }+zone := classifyZone(b, h, allGapAbove[i], allGapBelow[i])if zone == "" {continue@@ -582,6 +907,14 @@ func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) [return applyDrop(boxes, drop)}+ // Promo-companion propagation. Runs after Tier 1 and before the recurrence+ // engine, so the boxes it removes never enter the recurrence statistics+ // (which already skip dropped indices) and cannot perturb any "#"-masked count.+ for _, idx := range promoCompanionDrops(boxes, perPage, pageHeights, drop, promoSpots) {+ slog.Debug("header_footer: dropped by promo companion", "page", boxes[idx].PageNumber, "zone", bandOf(boxes[idx], pageHeights[boxes[idx].PageNumber]), "textLen", utf8.RuneCountInString(boxes[idx].Text))+ drop[idx] = struct{}{}+ }+// Tier 2: Dual-Track Recurrence Engine.keyMetas := make(map[zoneKey][]boxMeta)for i := range boxes {@@ -676,6 +1009,21 @@ func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) [}}+ // Sequence-backed page numbers. Word-derived books sit their page numbers+ // a few points under body text — below the 90% footer line AND below the+ // expanded band's whitespace requirement — so the geometric gate rejects+ // them before any tier can see a pattern or count a recurrence. A run of+ // bare numbers stepping by one across consecutive pages at a stable Y is a+ // page-number series on sequence evidence alone. This runs after the+ // recurrence engine so it only adds drops: the recurrence counts, which+ // include these boxes under their masked key, stay untouched.+ if seqDrops := findPageNumberSequenceDrops(collectPageNumberCandidates(boxes, pageHeights, pageNumberCeiling(numPages))); len(seqDrops) > 0 {+ slog.Debug("header_footer: dropped by page-number sequence", "boxes", len(seqDrops))+ for _, idx := range seqDrops {+ drop[idx] = struct{}{}+ }+ }+if len(drop) == 0 {return boxes}@@ -684,12 +1032,6 @@ func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64) [return applyDrop(boxes, drop)}-// hasStableConsecutiveRun checks if metas has a run of at least minRun-// consecutive pages where the top variation is within maxDy.-func hasStableConsecutiveRun(metas []boxMeta, minRun int, maxDy float64) bool {- return len(findStableConsecutiveRunIndices(metas, minRun, maxDy)) > 0-}-// findStableConsecutiveRunIndices finds all box indices that belong to any qualifying// consecutive run of at least minRun pages where the top variation is within maxDy.// Restricting removal to these specific indices ensures isolated occurrences outside@@ -781,6 +1123,14 @@ func applyDrop(boxes []pdf.TextBox, drop map[int]struct{}) []pdf.TextBox {return out}+// collapseText lower-cases and whitespace-collapses text without masking+// digits, for comparison keys that must not treat two differently-numbered+// lines as identical (unlike normalizeRunningText, which is for running+// headers/footers whose per-page number must be folded away).+func collapseText(text string) string {+ return strings.ToLower(strings.Join(strings.Fields(text), " "))+}+// normalizeRunningText collapses whitespace, replaces digit runs with a single// '#' and lower-cases the result, so per-page variants of the same running// header/footer ("- 2 -", "- 3 -", "Page 4 of 9", "第 1 页") share onediff --git a/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go b/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.goindex 2c4e66be31..0785f1e349 100644--- a/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go+++ b/internal/deepdoc/parser/pdf/layout/toc_header_footer_test.go@@ -1027,9 +1027,10 @@ func TestRemoveHeaderFooterBoxes_ChineseSlashGong(t *testing.T) {}}-// TestHasStableConsecutiveRun_SlidingWindow tests sliding window behavior in hasStableConsecutiveRun,-// specifically verifying that initial outliers do not block subsequent qualifying runs.-func TestHasStableConsecutiveRun_SlidingWindow(t *testing.T) {+// TestFindStableConsecutiveRunIndices_SlidingWindow tests sliding window behavior in+// findStableConsecutiveRunIndices, specifically verifying that initial outliers do+// not block subsequent qualifying runs.+func TestFindStableConsecutiveRunIndices_SlidingWindow(t *testing.T) {// Case 1: Outlier at page 1 (top 30), stable run at pages 2, 3, 4 (top 40)metas1 := []boxMeta{{page: 1, top: 30},@@ -1037,8 +1038,8 @@ func TestHasStableConsecutiveRun_SlidingWindow(t *testing.T) {{page: 3, top: 40},{page: 4, top: 40},}- if !hasStableConsecutiveRun(metas1, 3, 4.0) {- t.Errorf("expected true for pages 2..4 stable run despite page 1 outlier")+ if len(findStableConsecutiveRunIndices(metas1, 3, 4.0)) == 0 {+ t.Errorf("expected pages 2..4 stable run despite page 1 outlier")}// Case 2: Stable run at pages 1, 2, 3 (top 40), outlier at page 4 (top 30)@@ -1048,8 +1049,8 @@ func TestHasStableConsecutiveRun_SlidingWindow(t *testing.T) {{page: 3, top: 40},{page: 4, top: 30},}- if !hasStableConsecutiveRun(metas2, 3, 4.0) {- t.Errorf("expected true for pages 1..3 stable run")+ if len(findStableConsecutiveRunIndices(metas2, 3, 4.0)) == 0 {+ t.Errorf("expected pages 1..3 stable run")}// Case 3: Divergent tops across all pages (no 3 consecutive pages within 4.0)@@ -1059,8 +1060,8 @@ func TestHasStableConsecutiveRun_SlidingWindow(t *testing.T) {{page: 3, top: 50},{page: 4, top: 70},}- if hasStableConsecutiveRun(metas3, 3, 4.0) {- t.Errorf("expected false for divergent tops")+ if len(findStableConsecutiveRunIndices(metas3, 3, 4.0)) != 0 {+ t.Errorf("expected no run for divergent tops")}// Case 4: Page gap breaks consecutive run@@ -1070,8 +1071,8 @@ func TestHasStableConsecutiveRun_SlidingWindow(t *testing.T) {{page: 4, top: 40},{page: 5, top: 40},}- if hasStableConsecutiveRun(metas4, 3, 4.0) {- t.Errorf("expected false when no consecutive run reaches minRun=3")+ if len(findStableConsecutiveRunIndices(metas4, 3, 4.0)) != 0 {+ t.Errorf("expected no run when no consecutive run reaches minRun=3")}// Case 5: Run after a page gap meets minRun=3@@ -1082,8 +1083,8 @@ func TestHasStableConsecutiveRun_SlidingWindow(t *testing.T) {{page: 5, top: 40},{page: 6, top: 40},}- if !hasStableConsecutiveRun(metas5, 3, 4.0) {- t.Errorf("expected true for pages 4..6 run after page gap")+ if len(findStableConsecutiveRunIndices(metas5, 3, 4.0)) == 0 {+ t.Errorf("expected pages 4..6 run after page gap")}}@@ -1211,3 +1212,342 @@ func TestRemoveHeaderFooterBoxes_FooterYearPreservedOnShortDocument(t *testing.Tt.Fatalf("expected all 3 boxes to be kept, got %d", len(got))}}++// tightFooter builds one page for the sequence-track tests: a body line ending+// at bottom, a bare-number box 5pt under it at Top=755 (below the 757.8 footer+// line, inside the 0.86 expanded band without its gap) — geometry the zone gate+// rejects, leaving the +1 sequence as the only evidence.+func tightFooter(pg int, numText string) []pdf.TextBox {+ return []pdf.TextBox{+ tb(fmt.Sprintf("Body paragraph on page %d with a little extra length to stay unique.", pg), pg, 72, 500, 100, 750),+ tb(numText, pg, 400, 420, 755, 766),+ }+}++// TestRemoveHeaderFooterBoxes_TightSequenceFootersRemoved verifies that bare page+// numbers sitting tight under body text — rejected by both the base footer line+// and the expanded band's whitespace requirement — are removed by the sequence+// track when they step by one across consecutive pages.+func TestRemoveHeaderFooterBoxes_TightSequenceFootersRemoved(t *testing.T) {+ pageHeight := 842.0+ t.Run("arabic", func(t *testing.T) {+ heights := make(map[int]float64, 10)+ var boxes []pdf.TextBox+ for pg := 0; pg < 10; pg++ {+ heights[pg] = pageHeight+ boxes = append(boxes, tightFooter(pg, fmt.Sprintf("%d", pg+1))...)+ }+ got := RemoveHeaderFooterBoxes(boxes, heights)+ for _, b := range got {+ if _, ok := parseBareNumberValue(b.Text); ok {+ t.Fatalf("page number %q survived the sequence track", b.Text)+ }+ }+ if len(got) != 10 {+ t.Fatalf("expected 10 body boxes, got %d", len(got))+ }+ })+ t.Run("roman", func(t *testing.T) {+ heights := make(map[int]float64, 6)+ boxes := []pdf.TextBox{}+ romans := []string{"I", "II", "III", "IV", "V", "VI"}+ for pg, r := range romans {+ heights[pg] = pageHeight+ boxes = append(boxes, tightFooter(pg, r)...)+ }+ got := RemoveHeaderFooterBoxes(boxes, heights)+ for _, b := range got {+ if _, ok := parseBareNumberValue(b.Text); ok {+ t.Fatalf("Roman page number %q survived the sequence track", b.Text)+ }+ }+ if len(got) != 6 {+ t.Fatalf("expected 6 body boxes, got %d", len(got))+ }+ })+}++// TestRemoveHeaderFooterBoxes_NumbersNoSequencePreserved verifies the sequence+// track's guards: numbers that do not step by one, and runs shorter than 3+// consecutive pages, are left alone.+func TestRemoveHeaderFooterBoxes_NumbersNoSequencePreserved(t *testing.T) {+ pageHeight := 842.0+ t.Run("non_stepping", func(t *testing.T) {+ heights := make(map[int]float64, 5)+ var boxes []pdf.TextBox+ nums := []string{"3", "7", "12", "15", "19"}+ for pg, n := range nums {+ heights[pg] = pageHeight+ boxes = append(boxes, tightFooter(pg, n)...)+ }+ got := RemoveHeaderFooterBoxes(boxes, heights)+ if len(got) != 10 {+ t.Fatalf("non-stepping numbers must be preserved, got %d kept boxes", len(got))+ }+ })+ t.Run("run_too_short", func(t *testing.T) {+ // +1 steps exist (1,2 and 4,5) but each chain is broken before length 3.+ heights := make(map[int]float64, 6)+ var boxes []pdf.TextBox+ for pg := 0; pg < 6; pg++ {+ heights[pg] = pageHeight+ }+ boxes = append(boxes, tightFooter(0, "1")...)+ boxes = append(boxes, tightFooter(1, "2")...)+ boxes = append(boxes, tightFooter(3, "4")...)+ boxes = append(boxes, tightFooter(4, "5")...)+ boxes = append(boxes, tb("Plain body line, no number box on this page.", 2, 72, 500, 100, 750))+ boxes = append(boxes, tb("Plain body line here too.", 5, 72, 500, 100, 750))+ got := RemoveHeaderFooterBoxes(boxes, heights)+ if len(got) != 10 {+ t.Fatalf("2-page number chains must be preserved, got %d kept boxes", len(got))+ }+ })+}++// TestRemoveHeaderFooterBoxes_SequenceDriftSpanGuard verifies the sequence+// track bounds TOTAL Y span across a chain, not just each hop: per-hop drift+// that accumulates past seqMaxDy must not remove the run, while a chain whose+// full span stays inside the threshold still does.+func TestRemoveHeaderFooterBoxes_SequenceDriftSpanGuard(t *testing.T) {+ pageHeight := 842.0+ footerAt := func(pg int, numText string, top float64) []pdf.TextBox {+ return []pdf.TextBox{+ tb(fmt.Sprintf("Body paragraph on page %d with a little extra length to stay unique.", pg), pg, 72, 500, 100, 674),+ tb(numText, pg, 400, 420, top, top+11), // Top 680-ish: wide band, zone stays empty (gap 6pt, above 724.1 line only)+ }+ }+ t.Run("accumulating_drift_preserved", func(t *testing.T) {+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}+ var boxes []pdf.TextBox+ tops := []float64{680, 684, 688} // each hop 4pt, total span 8pt > seqMaxDy+ for pg, top := range tops {+ boxes = append(boxes, footerAt(pg, fmt.Sprintf("%d", pg+1), top)...)+ }+ got := RemoveHeaderFooterBoxes(boxes, heights)+ if len(got) != 6 {+ t.Fatalf("chain drifting 8pt in total must be preserved, got %d kept boxes", len(got))+ }+ })+ t.Run("within_span_removed", func(t *testing.T) {+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}+ var boxes []pdf.TextBox+ tops := []float64{680, 681.5, 683} // total span 3pt <= seqMaxDy+ for pg, top := range tops {+ boxes = append(boxes, footerAt(pg, fmt.Sprintf("%d", pg+1), top)...)+ }+ got := RemoveHeaderFooterBoxes(boxes, heights)+ for _, b := range got {+ if _, ok := parseBareNumberValue(b.Text); ok {+ t.Fatalf("page number %q survived the sequence track", b.Text)+ }+ }+ if len(got) != 3 {+ t.Fatalf("chain within total-drift span must be removed, got %d kept boxes", len(got))+ }+ })+}++// TestRemoveHeaderFooterBoxes_YearFooterSequenceCeilingPreserved verifies that a+// stepping year sequence in the footer band ("2024","2025","2026") is protected+// by the page-count ceiling even though it would otherwise satisfy the +1 chain.+func TestRemoveHeaderFooterBoxes_YearFooterSequenceCeilingPreserved(t *testing.T) {+ pageHeight := 842.0+ heights := make(map[int]float64, 3)+ var boxes []pdf.TextBox+ years := []string{"2024", "2025", "2026"}+ for pg, y := range years {+ heights[pg] = pageHeight+ boxes = append(boxes, tightFooter(pg, y)...)+ }+ got := RemoveHeaderFooterBoxes(boxes, heights)+ keptYears := make(map[string]bool, len(years))+ for _, b := range got {+ keptYears[b.Text] = true+ }+ for _, y := range years {+ if !keptYears[y] {+ t.Fatalf("year footer %q must be preserved: its value exceeds the page-number ceiling", y)+ }+ }+ if len(got) != 6 {+ t.Fatalf("year footers exceed the page-number ceiling and must be preserved, got %d", len(got))+ }+}++// TestRemoveHeaderFooterBoxes_TightBottomTextHeadingPreserved verifies the wide+// sequence bands never admit non-numeric text: a repeated section heading 6pt+// above the page bottom must survive even though its text repeats on every page.+func TestRemoveHeaderFooterBoxes_TightBottomTextHeadingPreserved(t *testing.T) {+ pageHeight := 842.0+ heights := make(map[int]float64, 4)+ var boxes []pdf.TextBox+ for pg := 0; pg < 4; pg++ {+ heights[pg] = pageHeight+ boxes = append(boxes, tb("Body text filling most of the page above the heading.", pg, 72, 500, 100, 724))+ boxes = append(boxes, tb("Quarterly Report Summary", pg, 72, 300, 730, 742)) // gapAbove = 6pt, repeats identically+ }+ got := RemoveHeaderFooterBoxes(boxes, heights)+ var headings int+ for _, b := range got {+ if b.Text == "Quarterly Report Summary" {+ headings+++ }+ }+ if headings != 4 || len(got) != 8 {+ t.Fatalf("tight repeated text headings must be preserved, got %d of 4 headings (%d boxes)", headings, len(got))+ }+}++// TestRemoveHeaderFooterBoxes_SitePromoVariantInHeaderRemoved verifies that margin+// lines advertising a download site are removed even when the wrapper text varies+// page to page and no recurrence track can see them.+func TestRemoveHeaderFooterBoxes_SitePromoVariantInHeaderRemoved(t *testing.T) {+ pageHeight := 842.0+ heights := make(map[int]float64, 4)+ var boxes []pdf.TextBox+ for pg := 0; pg < 4; pg++ {+ heights[pg] = pageHeight+ // The body is written out per page; only the promo lines matter.+ boxes = append(boxes, tb(fmt.Sprintf("Substantive body content of page %d goes here.", pg), pg, 72, 450, 160, 700))+ if pg < 2 {+ boxes = append(boxes, tb("更多的书籍免费下载 http://forum.law58.cn/?fromuid=381879", pg, 60, 300, 40, 52))+ } else {+ boxes = append(boxes, tb("欢迎访问!http://forum.law58.cn/?fromuid=381879", pg, 60, 300, 40, 52))+ }+ }+ got := RemoveHeaderFooterBoxes(boxes, heights)+ for _, b := range got {+ if strings.Contains(b.Text, "law58") {+ t.Fatalf("site promo %q survived despite varying wrappers", b.Text)+ }+ }+ if len(got) != 4 {+ t.Fatalf("expected 4 body boxes, got %d", len(got))+ }+}++// TestRemoveHeaderFooterBoxes_BodyUrlKept verifies the site-promo rule cannot+// reach into the body: a mid-page URL box and a long prose line containing a+// URL are both kept.+func TestRemoveHeaderFooterBoxes_BodyUrlKept(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight}+ boxes := []pdf.TextBox{+ // Mid-page short URL: inside no band at all.+ tb("http://a.cn/x", 0, 72, 130, 400, 415),+ // > 60 runes: length-capped out even though it starts in the top band.+ tb("See http://example.com/very/long/path/for/the/annual/errata/annex which continues well beyond sixty runes.", 1, 72, 500, 40, 55),+ tb("Ordinary body text.", 1, 72, 300, 500, 520),+ }+ got := RemoveHeaderFooterBoxes(boxes, heights)+ if len(got) != 3 {+ t.Fatalf("body URLs must be preserved, got %d", len(got))+ }+}++// TestRemoveHeaderFooterBoxes_DropsPromoCompanion verifies that a fixed slogan+// riding in the same margin band as a dropped site-promo ad is learned from the+// pages where they co-occur and propagated to every page, so the ad is gone+// even where its URL line was never detected.+func TestRemoveHeaderFooterBoxes_DropsPromoCompanion(t *testing.T) {+ pageHeight := 842.0 // header band: bottom <= 117.9; body starts well below.+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight, 3: pageHeight}+ const slogan = "木瓜树 更多的书籍免费下载"+ var boxes []pdf.TextBox+ for pg := 0; pg < 3; pg++ { // pages 0,1,2: ad URL + slogan companion+ boxes = append(boxes,+ tb("http://forum.law58.cn/?fromuid=381879", pg, 60, 300, 40, 52),+ tb(slogan, pg, 60, 300, 60, 72),+ tb(fmt.Sprintf("Substantive body content of page %d.", pg), pg, 72, 450, 200, 700),+ )+ }+ // page 3 carries only the slogan line, no URL.+ boxes = append(boxes,+ tb(slogan, 3, 60, 300, 60, 72),+ tb("Substantive body content of page 3.", 3, 72, 450, 200, 700),+ )++ got := RemoveHeaderFooterBoxes(boxes, heights)+ for _, b := range got {+ if strings.Contains(b.Text, "law58") {+ t.Fatalf("site promo survived: %q", b.Text)+ }+ if strings.Contains(b.Text, "木瓜树") {+ t.Fatalf("promo companion slogan survived: %q", b.Text)+ }+ }+ if len(got) != 4 {+ t.Fatalf("expected 4 body boxes, got %d", len(got))+ }+}++// TestRemoveHeaderFooterBoxes_PromoCompanionKeepsTitle verifies a LayoutType+// "title" line that merely shares the header band with an ad is never harvested+// as a companion, so legitimate running section titles survive.+func TestRemoveHeaderFooterBoxes_PromoCompanionKeepsTitle(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}+ titles := []string{"[图甲]", "[图乙]", "[图丙]"} // unique per page: no recurrence to lean on+ var boxes []pdf.TextBox+ for pg := 0; pg < 3; pg++ {+ boxes = append(boxes,+ tb("http://forum.law58.cn/?fromuid=381879", pg, 60, 300, 40, 52),+ pdf.TextBox{Text: titles[pg], PageNumber: pg, X0: 60, X1: 200, Top: 60, Bottom: 72, LayoutType: "title"},+ tb(fmt.Sprintf("Body content of page %d here.", pg), pg, 72, 450, 200, 700),+ )+ }++ got := RemoveHeaderFooterBoxes(boxes, heights)+ seenTitle := 0+ for _, b := range got {+ if strings.Contains(b.Text, "law58") {+ t.Fatalf("site promo survived: %q", b.Text)+ }+ if strings.HasPrefix(b.Text, "[图") {+ seenTitle+++ }+ }+ if seenTitle != 3 {+ t.Fatalf("all 3 title lines must survive as companions excluded, got %d", seenTitle)+ }+ if len(got) != 6 {+ t.Fatalf("expected 3 titles + 3 bodies, got %d", len(got))+ }+}++// TestRemoveHeaderFooterBoxes_PromoCompanionKeepsRarePeer verifies a short+// non-title band peer that co-occurs with the ad on only one page is not+// harvested — the page-count floor stops one-off neighbours.+func TestRemoveHeaderFooterBoxes_PromoCompanionKeepsRarePeer(t *testing.T) {+ pageHeight := 842.0+ heights := map[int]float64{0: pageHeight, 1: pageHeight, 2: pageHeight}+ const slogan = "木瓜树 更多的书籍免费下载"+ var boxes []pdf.TextBox+ for pg := 0; pg < 3; pg++ {+ boxes = append(boxes,+ tb("http://forum.law58.cn/?fromuid=381879", pg, 60, 300, 40, 52),+ tb(slogan, pg, 60, 300, 60, 72),+ tb(fmt.Sprintf("Body content of page %d here.", pg), pg, 72, 450, 200, 700),+ )+ }+ // A short slogan-like peer present only on page 0, next to its ad.+ boxes = append(boxes, tb("扫码加入读书群", 0, 60, 300, 80, 92))++ got := RemoveHeaderFooterBoxes(boxes, heights)+ var keptRare int+ for _, b := range got {+ if strings.Contains(b.Text, "law58") || strings.Contains(b.Text, "木瓜树") {+ t.Fatalf("promo/slogan survived: %q", b.Text)+ }+ if b.Text == "扫码加入读书群" {+ keptRare+++ }+ }+ if keptRare != 1 {+ t.Fatalf("rare single-page peer must survive, got %d", keptRare)+ }+ if len(got) != 4 {+ t.Fatalf("expected 3 bodies + 1 rare peer, got %d", len(got))+ }+}