RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
package layout
import (
"regexp"
"sort"
"strings"
"unicode/utf8"
"go.uber.org/zap"
"ragflow/internal/common"
pdf "ragflow/internal/deepdoc/parser/pdf/type"
)
// ---------------------------------------------------------------------------
// Tunable thresholds.
// ---------------------------------------------------------------------------
const (
// tocMaxProseRunes is the per-box length (in runes) above which a box is
// treated as a prose paragraph. A page with substantial body text has many
// such boxes; a TOC page has at most a title or a long chapter heading.
// This is the primary guard against deleting content: any page with more
// than tocMaxLongBoxes long boxes is never classified as TOC.
tocMaxProseRunes = 60
// tocMaxLongBoxes is the maximum number of boxes longer than
// tocMaxProseRunes a page may carry and still be classified as TOC.
tocMaxLongBoxes = 2
// tocBodyRunes is the length (in runes) above which a box that is
// neither a chapter marker nor a leader run counts as body evidence.
// The prose guard above keys on long boxes, which a corpus of short
// lines (a legal code's articles, verse, lists) never produces — the
// 民法典's article lines wrap at ~35 runes, so prose stayed 0 on every
// page and a mixed TOC-tail + article page was read as a TOC page and
// dropped whole. A body line is still longer than a TOC entry title
// and does not open with a chapter marker, so a page carrying several
// such boxes is a body page no signal may drop.
tocBodyRunes = 20
// tocMaxBodyBoxes is the maximum number of body-evidence boxes a page
// may carry and still be classified as TOC.
tocMaxBodyBoxes = 2
// tocMinShortBoxes is the minimum number of short boxes a TOC page carries.
// A TOC is a list, so its entries usually stay separate boxes. A page whose
// entries were merged into one long box is covered by tocMinMergedRuns, and
// one merged without leader runs at all is covered by the outline signal.
tocMinShortBoxes = 4
// tocMinEntries is the minimum number of confirming entry markers (chapter
// markers plus page numbers) a page must carry to be classified as TOC.
tocMinEntries = 3
// tocMinMergedRuns is the number of leader+page-number runs a single long
// box must carry before that box is read as a whole TOC block rather than as
// prose. A merged block confirms itself — three leader runs are three
// entries — so it needs no separate marker count. Prose can accumulate the
// occasional run, which is why the threshold is three and not one.
tocMinMergedRuns = 3
// tocMinLongEntryLeader is the shortest leader run that lets a long box be
// read as one TOC entry line. Tab leaders run for many characters; prose
// that merely ends in an ellipsis and a number does not, and must keep
// counting as prose.
tocMinLongEntryLeader = 6
// tocMinShortEntryLeader is the same floor for short boxes. A short entry
// stays a marker rather than body evidence, but an ellipsis that is only
// sentence punctuation ("The result is ... 42") must not turn three prose
// lines into a TOC page.
tocMinShortEntryLeader = 4
// tocMaxLeadPages bounds how many leading pages without body text may
// precede the TOC. A cover, copyright page or frontispiece carries no body
// text and is skipped, so a TOC that is not physically page one is still
// reachable; the first page carrying body text ends the search.
tocMaxLeadPages = 3
// tocShapeFrontMatterMaxPage is the last 0-based page a parse may start on
// for the no-bookmark box-shape fallback to run. A TOC is a document prefix,
// so only parses beginning in the front matter may read their leading run as
// one; 10 tolerates cover/copyright/preface pages and ranges that start a few
// pages before the TOC, while still blocking a mid-book "TOC-shaped" page.
tocShapeFrontMatterMaxPage = 10
)
// ---------------------------------------------------------------------------
// Box-level patterns.
// ---------------------------------------------------------------------------
var (
// chapterEntryPattern matches a box that opens like a TOC entry title: a
// chapter/section/appendix marker (Chinese or Latin, digit or word form) or
// a common front/back-matter heading.
chapterEntryPattern = regexp.MustCompile(`(?i)^(第\s*[0-90-9一二三四五六七八九十百千]+\s*[章节篇部册卷回讲目]|chapter\s+(\d+|[a-z]+|one|two|three|four|five|six|seven|eight|nine|ten)|appendix\s+[a-z0-9]+|section\s+\d+|part\s+(\d+|[ivxlcdm]+|[a-z]+)|前言|序言|引言|导论|绪论|后记|结语|附录|索引|参考文献|结论|致谢|跋|序|凡例|目录|acknowledgements?|acknowledgments?)`)
// pageNumberPattern matches common page-number forms: pure digits, digits
// with leading dots (..18), roman numerals (III, IV), or dash-wrapped
// numbers (- 2 -). Full-width digits count as digits, matching the class
// chapterEntryPattern uses for 第N章 numbering: the target corpus is largely
// CJK, and a full-width page-number column still has to confirm a TOC page.
pageNumberPattern = regexp.MustCompile(`^[.…]*[0-90-9]+[.…]*$|^[.…]*[IVXLCDM]+[.…]*$|^\s*-\s*[0-90-9]+\s*-\s*$`)
// tocStartPattern matches an outline title that names the table of contents,
// which is the only thing that starts a TOC page range. A heading that is
// merely adjacent to a TOC — acknowledgements, references — does not: the
// section-level detector this pass replaces accepted 致谢 here, so a book
// without a 目录 bookmark had its acknowledgements page claimed as a TOC.
tocStartPattern = regexp.MustCompile(`(?i)^(contents|目录|目次|table of contents)$`)
// tocEndMarkerPattern matches the front/back-matter headings that are
// neither a TOC nor an entry, so they cannot bound a TOC range. The
// acknowledgements family is spelled out the way chapterEntryPattern spells
// it (singular and plural, both spellings), because the pattern this
// replaces only matched the bare word "acknowledge" and so never fired on
// the usual "Acknowledgements" bookmark.
tocEndMarkerPattern = regexp.MustCompile(`(?i)^(致谢|acknowledg(?:e|ements?|ments?))$`)
// tocEntryRunPattern matches one TOC entry's tail: a leader run (two or more
// dots, ellipses or middots, never wide spaces — those are too common in
// prose ending in a number), a page number and an optional roman-numeral
// fragment. Counting these runs tells a TOC block that upstream merged into
// one long box apart from a paragraph. Digits accept the full-width forms
// pageNumberPattern accepts, for the same reason.
tocEntryRunPattern = regexp.MustCompile(`(\.{2,}|…{2,}|⋯{2,}|·{2,})\s*([0-90-9]{1,4}\s*[IVXLCDMivxlcdm]{0,5}|[IVXLCDMivxlcdm]{1,7}\b)`)
// tocLeaderOnlyPattern matches a box that is nothing but leader
// characters (dots, ellipses, middots, spaces): the leader column of a
// dotted TOC entry. Such a box is long but carries no body text, so it
// must not count as body evidence.
tocLeaderOnlyPattern = regexp.MustCompile(`^[.…⋯·\s]+$`)
)
// ---------------------------------------------------------------------------
// Outline signal.
// ---------------------------------------------------------------------------
// TOCPageRangeFromOutlines returns the 0-based page indices the PDF bookmarks
// identify as the table of contents: from the outline entry naming the TOC up
// to — excluding — the next entry at the same level. It returns nil when the
// document carries no usable bookmark.
//
// Only an entry that names the TOC itself starts a range (tocStartPattern).
// When a front/back-matter heading (tocEndMarkerPattern) is the next same-level
// entry the range is abandoned rather than closed, because such a heading does
// not say where the entries stop — it follows the TOC in some books and sits at
// the back of others — so the extent is left to the box shape signal instead of
// spanning the gap and deleting whatever lies inside it. Only the page the TOC
// starts on is claimed.
//
// Outline page numbers are 1-based — pdfium adds one to the 0-based destination
// index (internal/deepdoc/parser/pdf/pdfium/pdfium.go) — while
// TextBox.PageNumber is 0-based, because ParseRaw enumerates pages from zero.
// The conversion happens here, once, so no caller has to know about it.
func TOCPageRangeFromOutlines(outlines []pdf.Outline) map[int]bool {
for i, o := range outlines {
if !tocStartPattern.MatchString(outlineTitle(o.Title)) {
continue
}
first := o.PageNumber - 1
if first < 0 {
return nil
}
for _, next := range outlines[i+1:] {
if next.Level != o.Level {
continue
}
if tocEndMarkerPattern.MatchString(outlineTitle(next.Title)) {
break
}
pages := make(map[int]bool, 4)
for pg := first; pg < next.PageNumber-1; pg++ {
pages[pg] = true
}
if len(pages) == 0 {
return nil
}
return pages
}
// No usable following entry at the same level: the TOC page is known
// but its extent is not, so claim only that page and leave the rest to
// the box shape signal.
return map[int]bool{first: true}
}
return nil
}
// outlineTitle strips the "@@" anchor suffix pdfium appends to bookmark titles
// and lower-cases the result, matching how the section-level detector compared.
func outlineTitle(title string) string {
title = strings.TrimSpace(title)
if idx := strings.Index(title, "@@"); idx >= 0 {
title = strings.TrimSpace(title[:idx])
}
return strings.ToLower(title)
}
// ---------------------------------------------------------------------------
// TOC removal.
// ---------------------------------------------------------------------------
// RemoveTOCBoxes drops whole pages that are detected as tables of contents.
//
// Detection works on signals that are destroyed by the later TextMerge /
// BoxesToSections passes (leader dots get folded into adjacent boxes, per-entry
// geometry collapses into a section), so this MUST run before TextMerge (see
// Parser.buildLayout).
//
// firstParsedPage is the first 0-based page in the parsed subset (Config.Pages),
// or -1 when unknown. The shape signal reads a TOC out of a leading run, so it
// only runs when the parse begins in the document's front matter
// (firstParsedPage <= tocShapeFrontMatterMaxPage): a TOC is a document prefix,
// and reading a mid-book page range's leading run as one would delete content
// from the middle of the book. The outline signal carries absolute page numbers
// and is unaffected.
//
// totalPages is the document's full engine page count. The all-TOC guard below
// is document-level: only a document that is nothing but TOC pages is left
// untouched, not a parsed subset that happens to be all TOC.
//
// Two signals select pages and the union is dropped:
//
// 1. outlinePages — the pages the PDF bookmarks identify as TOC (see
// TOCPageRangeFromOutlines). Strongest, and the only one that still sees a
// TOC whose entries were merged into one box with no leader runs left.
// 2. Box shape — the leading run of pages that read as a TOC list: a page
// carrying at least tocMinShortBoxes short boxes and tocMinEntries
// confirming markers (chapter markers or page numbers), or one long box
// that is itself tocMinMergedRuns leader+page-number runs, preceded only by
// pages without body text and spanning as many consecutive pages as keep
// satisfying that shape. Kept for documents that carry no usable bookmark,
// and only consulted when the parse begins in the document's front matter.
//
// Both signals pass through one guard: a page carrying body text (more than
// tocMaxLongBoxes boxes longer than tocMaxProseRunes, or more than
// tocMaxBodyBoxes marker-less boxes longer than tocBodyRunes) is never
// dropped, and a document consisting only of TOC pages is left untouched.
// The detector stays deliberately conservative — missing a TOC page costs
// noise chunks, deleting a content page loses text.
func RemoveTOCBoxes(boxes []pdf.TextBox, outlinePages map[int]bool, firstParsedPage, totalPages int) []pdf.TextBox {
if len(boxes) == 0 {
return boxes
}
perPage := make(map[int][]int, 64)
for i := range boxes {
p := boxes[i].PageNumber
perPage[p] = append(perPage[p], i)
}
pages := make([]int, 0, len(perPage))
for pg := range perPage {
pages = append(pages, pg)
}
sort.Ints(pages)
shapes := make(map[int]pageShape, len(pages))
for _, pg := range pages {
shapes[pg] = shapeOfPage(boxes, perPage[pg])
}
selected := make(map[int]bool, len(pages))
for pg := range outlinePages {
if _, ok := perPage[pg]; ok {
selected[pg] = true
}
}
// Heuristic fallback, for documents that carry no usable bookmark. A TOC is
// a document prefix, so only a parse that begins in the document's front
// matter is eligible, and then only the leading pages are candidates: pages
// without body text (a cover, copyright page) may be skipped, and the run
// continues while pages keep satisfying isTOC(). It ends on the first page that does
// not — in practice the first page carrying body text, which is what keeps
// per-chapter pages that happen to hold several short headings (the Daodejing
// case) out of scope. The run is deliberately unbounded in length: a real
// TOC can span more pages than any fixed cap would allow, and truncating it
// silently keeps the remaining TOC pages in the output. Length is bounded
// instead by isTOC() itself and by the all-TOC guard below.
if firstParsedPage >= 0 && firstParsedPage <= tocShapeFrontMatterMaxPage {
inTOC, lead := false, 0
for _, pg := range pages {
if inTOC {
if !shapes[pg].isTOC() {
break
}
selected[pg] = true
continue
}
if shapes[pg].isTOC() {
inTOC = true
selected[pg] = true
continue
}
if shapes[pg].carriesLongProse() || lead >= tocMaxLeadPages {
break
}
lead++
}
}
// The all-TOC guard is document-level, not subset-level: a document that is
// nothing but TOC pages is left untouched, while a parsed range that happens
// to be all TOC still drops. Floor at the observed subset so a caller that
// omits totalPages (0) degrades to the previous behavior.
docPages := totalPages
if docPages < len(pages) {
docPages = len(pages)
}
drop := make(map[int]struct{}, len(boxes))
droppedPages := 0
for _, pg := range pages {
if !selected[pg] || shapes[pg].carriesProse() {
continue
}
droppedPages++
for _, i := range perPage[pg] {
drop[i] = struct{}{}
}
}
if droppedPages == 0 || droppedPages == docPages {
return boxes
}
out := make([]pdf.TextBox, 0, len(boxes)-len(drop))
for i := range boxes {
if _, ok := drop[i]; ok {
continue
}
out = append(out, boxes[i])
}
return out
}
// pageShape summarises the boxes of one page.
type pageShape struct {
prose int // boxes longer than tocMaxProseRunes
short int
markers int // short boxes opening with a chapter marker or holding a page number
merged bool // a long box carrying tocMinMergedRuns leader+page-number runs
body int // marker-less, leader-less short boxes longer than tocBodyRunes
entries int // long entry lines: count toward the list threshold, not as prose
}
func shapeOfPage(boxes []pdf.TextBox, indices []int) pageShape {
var s pageShape
for _, i := range indices {
text := strings.TrimSpace(boxes[i].Text)
if text == "" {
continue
}
if utf8.RuneCountInString(text) > tocMaxProseRunes {
// A leader column is typographic filler, never prose. OCR detection
// misses it, so the native-text recovery emits it as its own long
// box; counting it as prose kept every page it appeared on.
if tocLeaderOnlyPattern.MatchString(text) {
continue
}
// A merged TOC block is long by definition, so the run count is the
// only evidence it can carry. It is confirmed entries, not prose:
// counting it as prose let three merged blocks guard their own page.
if len(tocEntryRunPattern.FindAllStringIndex(text, tocMinMergedRuns)) >= tocMinMergedRuns {
s.merged = true
continue
}
// An entry line — optional title, leaders, page number — is a TOC
// entry, not a paragraph. It confirms the page like any other
// marker instead of contradicting it as body text. Require a real
// tab leader here: a prose sentence that happens to end in an
// ellipsis and a number must keep counting as prose.
if isTOCEntryLine(text, tocMinLongEntryLeader) {
s.markers++
s.entries++
continue
}
s.prose++
continue
}
s.short++
if chapterEntryPattern.MatchString(text) || pageNumberPattern.MatchString(text) || isTOCEntryLine(text, tocMinShortEntryLeader) {
s.markers++
continue
}
if utf8.RuneCountInString(text) > tocBodyRunes && !tocLeaderOnlyPattern.MatchString(text) {
s.body++
}
}
return s
}
// isTOCEntryLine reports whether text ends with one TOC entry's tail: a leader
// run followed by the entry's page number. minLeader is the shortest leader run
// accepted, so a long box only reads as an entry when its tab leader is real.
// Space-separated leader segments count as one run (OCR often splits them).
// The leader+number grammar is reuse of tocEntryRunPattern; prose that merely
// contains an ellipsis does not end in a run and never matches.
func isTOCEntryLine(text string, minLeader int) bool {
locs := tocEntryRunPattern.FindAllStringIndex(text, -1)
if len(locs) == 0 {
return false
}
loc := locs[len(locs)-1]
if strings.TrimSpace(text[loc[1]:]) != "" {
return false
}
// Walk back over the leader run so segments split by spaces all count.
start := loc[0]
for start > 0 {
r, size := utf8.DecodeLastRuneInString(text[:start])
if r == ' ' || r == '\t' || r == '.' || r == '…' || r == '⋯' || r == '·' {
start -= size
continue
}
break
}
leader := 0
for _, r := range text[start:loc[1]] {
switch r {
case '.', '…', '⋯', '·':
leader++
case ' ', '\t':
default:
return leader >= minLeader
}
}
return false
}
// isTOC reports whether the page reads as a table of contents: a list of entry
// boxes carrying markers, or one long box that is itself a run of entries, and
// no body text either way.
func (s pageShape) isTOC() bool {
if s.carriesProse() {
return false
}
return s.merged || (s.short+s.entries >= tocMinShortBoxes && s.markers >= tocMinEntries)
}
// carriesProse reports whether the page carries body text. No signal may drop
// such a page: this is the single guard that bounds every removal. Body text
// is evidenced by long boxes (prose), or — for corpora whose lines are all
// short, such as a legal code's articles or verse — by several marker-less,
// leader-less boxes longer than tocBodyRunes.
func (s pageShape) carriesProse() bool { return s.carriesLongProse() || s.body > tocMaxBodyBoxes }
// carriesLongProse reports the long-box signal alone. The leading-run scan
// uses it rather than carriesProse: a cover or copyright page carries short
// lines (its metadata block) but no body text, and the body-evidence arm
// would stop the scan on such a page before it reaches the TOC. Every drop
// still passes carriesProse, so the body evidence keeps bounding removal.
func (s pageShape) carriesLongProse() bool { return s.prose > tocMaxLongBoxes }
// ---------------------------------------------------------------------------
// Running header / footer removal.
// ---------------------------------------------------------------------------
const (
// headerZoneRatio / footerZoneRatio bound the top/bottom page zones where a
// running header / footer lives (fractions of the page height).
headerZoneRatio = 0.10
footerZoneRatio = 0.90
// headerMaxZoneRatio / footerMinZoneRatio define the expanded margin zones
// when a clear whitespace gap separates the candidate from body content.
headerMaxZoneRatio = 0.14
footerMinZoneRatio = 0.86
// minWhitespaceGapPt is the minimum vertical whitespace gap required to
// justify expanding the header/footer zone beyond the base 10% ratio.
minWhitespaceGapPt = 18.0
// minHeaderFooterPages guards against cross-page false positives on short
// documents. Deterministic page-number patterns still fire on shorter ones.
minHeaderFooterPages = 3
// minParityCoverageRatio is the minimum fraction of even or odd pages that
// a running header/footer text must appear on to qualify under the parity track.
//
// In formal book typography (LaTeX book documentclass, InDesign book templates),
// running headers are standardly suppressed on chapter opening pages (\thispagestyle{plain}),
// blank verso pages before new chapters, and full-page illustrations/tables.
// As a result, a running header in a real book often covers only 40%-48% of the
// even or odd pages. A 0.40 threshold captures alternating bilateral book headers
// while the evenPages >= 3 / oddPages >= 3 count guards against accidental matches.
minParityCoverageRatio = 0.40
// seqBandTopRatio / seqBandBottomRatio bound the wide bands in which the
// sequence-backed page-number track looks for bare numeric / Roman-numeral
// boxes. The bands are much looser than the header/footer zones because the
// track's evidence is not geometric: only boxes that form an incrementing
// run across consecutive pages at a stable Y are removed, which tight body
// text cannot fake.
seqBandTopRatio = 0.20
seqBandBottomRatio = 0.80
// seqMinRun / seqMaxDy mirror the locality track: at least 3 consecutive
// pages, total top-edge span across the run within 4pt.
seqMinRun = 3
seqMaxDy = 4.0
// promoCompanionMaxRunes bounds the length of a margin line that may be
// learned as a running-header ad companion of a dropped site-promo box.
// A piracy header is a short slogan, so anything longer is treated as body
// text and never learned.
promoCompanionMaxRunes = 24
// minCompanionPages is how many distinct pages must carry the same band
// peer of a dropped site-promo box before that peer's text is treated as a
// running-header ad and propagated to every page. Requiring several pages
// keeps a one-off line that merely shares a page with an ad intact.
minCompanionPages = 3
)
var (
// strictDecoratedPagePattern matches page numbers with clear formatting:
// - Dash/dot/tilde-wrapped: - 1 -, — 1 —, – 1 –, · 1 ·, • 1 •, ~ 1 ~
// - Bracket/paren-wrapped: [1], (1)
// - "Page X", "Page X of Y", "p. X", "X / Y", "X of Y"
// - Chinese: 第 1 页, 第 1 页 共 10 页, 第 1 页/共 10 页, 第一页, 第一页 共十页
// - Roman with dashes: - iv -, — iv —, · iv ·
strictDecoratedPagePattern = regexp.MustCompile(`(?i)^(\s*[-—–~·•]+\s*[0-90-9]+\s*[-—–~·•]+|\s*\[\s*[0-90-9]+\s*\]|\s*\(\s*[0-90-9]+\s*\)|page\s+[0-90-9]+(\s*(of|/)\s*[0-90-9]+)?|p\.\s*[0-90-9]+|[0-90-9]+\s*/\s*[0-90-9]+|[0-90-9]+\s+of\s+[0-90-9]+|第\s*[0-90-9一二三四五六七八九十百]+\s*页(\s*(/\s*共|[,,]\s*共?|[/共])\s*[0-90-9一二三四五六七八九十百]+\s*页?)?|\s*[-—–~·•]+\s*((l|xl|x{1,3})(ix|iv|v?i{1,3}|v)?|ix|iv|v?i{1,3}|v)\s*[-—–~·•]+)$`)
// strictBareNumberPattern matches lone Arabic digits or Roman numerals: 1, 23, iv, VII.
strictBareNumberPattern = regexp.MustCompile(`(?i)^([0-90-9]{1,4}|((l|xl|x{1,3})(ix|iv|v?i{1,3}|v)?|ix|iv|v?i{1,3}|v))$`)
// canonicalRomanPattern accepts the full canonical Roman-numeral grammar
// (up to M) so the sequence track can parse page numbers beyond the XXX
// range strictBareNumberPattern's header/footer idiom covers. Non-canonical
// forms (IIII) are rejected by the pattern itself; romanValue only computes
// the subtractive sum of a spelling this regex already accepted.
canonicalRomanPattern = regexp.MustCompile(`(?i)^(m{0,3})(cm|cd|d?c{0,3})(xc|xl|l?x{0,3})(ix|iv|v?i{0,3})$`)
// sitePromoPattern matches margin lines that advertise a download site —
// a URL core with at most a short non-URL wrapper on either side. Such a
// line is a running header/footer regardless of page count: piracy-supply
// PDFs change the wrapper text on some pages ("更多的书籍免费下载 http://…",
// "欢迎访问!http://…"), so those variants reach no recurrence track, while
// the URL itself is deterministic evidence. Only whole-box, length-capped
// boxes in the margin bands use it, so body prose containing a URL is never
// touched.
sitePromoPattern = regexp.MustCompile(`(?i)^.{0,20}(https?://\S+|www\.[\w-]+(\.[\w-]+)+|(forum|bbs|tieba|phpwind)\.[\w-]+\.[a-z]{2,}\S*|\S*fromuid=\S+).{0,20}$`)
)
// computePageGaps computes for candidate margin boxes on a page:
// - gapBelow: distance from b.Bottom to the nearest other box on the page with Top >= b.Bottom - 2.0.
// - gapAbove: distance from b.Top to the nearest other box on the page with Bottom <= b.Top + 2.0.
//
// Performance optimization: only boxes falling into the outer margin zones (top 14% or bottom 14%)
// ever query whitespace gaps. Skipping body-interior boxes reduces the outer loop from M to K (K <= 5),
// lowering complexity from O(M^2) to O(K*M) on dense multi-column pages.
func computePageGaps(boxes []pdf.TextBox, indices []int, pageHeight float64) (map[int]float64, map[int]float64) {
gapBelow := make(map[int]float64, len(indices))
gapAbove := make(map[int]float64, len(indices))
for _, i := range indices {
bi := boxes[i]
if bi.Bottom > pageHeight*headerMaxZoneRatio && bi.Top < pageHeight*footerMinZoneRatio {
continue
}
minDistBelow := pageHeight - bi.Bottom
if minDistBelow < 0 {
minDistBelow = 0
}
maxDistAbove := bi.Top
if maxDistAbove < 0 {
maxDistAbove = 0
}
for _, j := range indices {
if i == j {
continue
}
bj := boxes[j]
if bj.Top >= bi.Bottom-2.0 {
dist := bj.Top - bi.Bottom
if dist < 0 {
dist = 0
}
if dist < minDistBelow {
minDistBelow = dist
}
}
if bj.Bottom <= bi.Top+2.0 {
dist := bi.Top - bj.Bottom
if dist < 0 {
dist = 0
}
if dist < maxDistAbove {
maxDistAbove = dist
}
}
}
gapBelow[i] = minDistBelow
gapAbove[i] = maxDistAbove
}
return gapBelow, gapAbove
}
// classifyZone determines whether a box sits in a header or footer zone.
func classifyZone(b pdf.TextBox, pageHeight float64, gapAbove, gapBelow float64) string {
if b.Bottom <= pageHeight*headerZoneRatio {
return "header"
}
if b.Bottom <= pageHeight*headerMaxZoneRatio && gapBelow >= minWhitespaceGapPt {
return "header"
}
if b.Top >= pageHeight*footerZoneRatio {
return "footer"
}
if b.Top >= pageHeight*footerMinZoneRatio && gapAbove >= minWhitespaceGapPt {
return "footer"
}
return ""
}
// isNonTextLayout reports whether a layout type represents non-text elements
// like tables or figures that must not be treated as running headers/footers.
func isNonTextLayout(lt string) bool {
lt = strings.TrimSpace(lt)
return lt == "table" || lt == "figure" || lt == "equation" || lt == "image"
}
// parseDecimalValue parses a string of lone ASCII or full-width digits.
func parseDecimalValue(t string) (int, bool) {
if t == "" {
return 0, false
}
val := 0
for _, r := range t {
switch {
case r >= '0' && r <= '9':
val = val*10 + int(r-'0')
case r >= '0' && r <= '9':
val = val*10 + int(r-'0')
default:
return 0, false
}
}
return val, true
}
// romanValue parses a canonical Roman numeral (I..MMMCMXCIX) and returns its
// value. The canonical pattern admits only canonical spellings, so non-canonical
// forms such as "IIII" are rejected before the subtractive sum is computed.
func romanValue(t string) (int, bool) {
if !canonicalRomanPattern.MatchString(t) {
return 0, false
}
values := map[rune]int{'m': 1000, 'd': 500, 'c': 100, 'l': 50, 'x': 10, 'v': 5, 'i': 1}
val := 0
prev := 0
for _, r := range strings.ToLower(t) {
v := values[r]
if v == 0 {
return 0, false
}
if v > prev {
val += v - 2*prev
} else {
val += v
}
prev = v
}
if val <= 0 {
return 0, false
}
return val, true
}
// parseBareNumberValue reports the numeric value of a box whose entire text is
// one bare number: lone digits (ASCII or full-width) or a canonical Roman
// numeral. "12", "45", "iv", "XIV" parse; "12.", "- 3 -", "2024." do not.
func parseBareNumberValue(text string) (int, bool) {
t := strings.TrimSpace(text)
if v, ok := parseDecimalValue(t); ok {
return v, true
}
return romanValue(t)
}
// pageNumberCeiling bounds which bare numeric values can be page numbers: a
// document of N pages never carries page number > N+5, and the floor of 20
// keeps short documents (years like 2024 on a 2-page footer) protected.
func pageNumberCeiling(numPages int) int {
maxAllowed := numPages + 5
if maxAllowed < 20 {
maxAllowed = 20
}
return maxAllowed
}
// isSitePromo reports whether the whole box text advertises a download site.
func isSitePromo(text string) bool {
t := strings.Join(strings.Fields(text), " ")
if t == "" || utf8.RuneCountInString(t) > 60 {
return false
}
return sitePromoPattern.MatchString(t)
}
// bandOf reports which expanded margin band ("header"/"footer") a box sits in,
// or "" when it is outside both. It uses the same zone bounds the site-promo
// tier applies, so a box learned as an ad companion and a box later matched
// against it are judged by one rule.
func bandOf(b pdf.TextBox, pageHeight float64) string {
switch {
case b.Bottom <= pageHeight*headerMaxZoneRatio:
return "header"
case b.Top >= pageHeight*footerMinZoneRatio:
return "footer"
}
return ""
}
// promoSpot is the page and band where a site-promo box was dropped, recorded
// so the companion pass can harvest its in-band peers as running-header ads.
type promoSpot struct {
page int
band string
}
// promoCompanionDrops returns indices of margin-band boxes that are the slogan
// half of an advertisement already seen in the same band next to a dropped
// site-promo box. It harvests the short, non-title, in-band peers of every
// promo spot, then flags every band box whose collapsed text was harvested on
// at least minCompanionPages distinct pages — so a piracy header whose URL line
// is omitted or split on some pages still goes. The peers are keyed on
// collapseText (whitespace + case only, digits preserved) so two lines that
// differ only by a number are never merged. `skip` holds indices already doomed
// by an earlier tier and is consulted, never mutated.
func promoCompanionDrops(boxes []pdf.TextBox, perPage map[int][]int, pageHeights map[int]float64, skip map[int]struct{}, promoSpots []promoSpot) []int {
companionPages := make(map[string]map[int]bool)
for _, sp := range promoSpots {
h := pageHeights[sp.page]
for _, j := range perPage[sp.page] {
if _, dropped := skip[j]; dropped {
continue
}
b := boxes[j]
if isNonTextLayout(b.LayoutType) || strings.TrimSpace(b.LayoutType) == "title" {
continue
}
if bandOf(b, h) != sp.band {
continue
}
key := collapseText(b.Text)
if key == "" || utf8.RuneCountInString(key) > promoCompanionMaxRunes {
continue
}
if companionPages[key] == nil {
companionPages[key] = make(map[int]bool)
}
companionPages[key][sp.page] = true
}
}
var out []int
for key, pages := range companionPages {
if len(pages) < minCompanionPages {
continue
}
for i := range boxes {
if _, dropped := skip[i]; dropped {
continue
}
b := boxes[i]
if isNonTextLayout(b.LayoutType) || strings.TrimSpace(b.LayoutType) == "title" {
continue
}
h, ok := pageHeights[b.PageNumber]
if !ok || h <= 0 {
continue
}
if bandOf(b, h) != "" && collapseText(b.Text) == key {
out = append(out, i)
}
}
}
return out
}
// isDeterministicPageNumber reports whether text is unambiguously a page number.
func isDeterministicPageNumber(text string, zone string, gapAbove, gapBelow float64, numPages int) bool {
t := strings.TrimSpace(text)
if t == "" {
return false
}
if strictDecoratedPagePattern.MatchString(t) {
return true
}
if strictBareNumberPattern.MatchString(t) {
// Bare numbers must have clear vertical whitespace separation:
// - in header: gapBelow >= 18pt to avoid dropping section headings/chapter numbers.
// - in footer: gapAbove >= 18pt to avoid dropping footnote numbers, table cells, or trailing text.
if zone == "header" && gapBelow < minWhitespaceGapPt {
return false
}
if zone == "footer" && gapAbove < minWhitespaceGapPt {
return false
}
// For bare numeric digits, verify the value is within a reasonable range of the document page count.
// Protects isolated years (e.g. "2024"), IDs, or footnote indexes on short/medium documents.
if val, ok := parseDecimalValue(t); ok {
if val > pageNumberCeiling(numPages) {
return false
}
}
return true
}
return false
}
type boxMeta struct {
idx int
page int
top float64
}
type zoneKey struct {
zone string
text string
}
// pageNumCand is one bare-number box inside a wide margin band, a candidate
// for the sequence-backed page-number track.
type pageNumCand struct {
idx int
page int
value int
top float64
band string
}
// collectPageNumberCandidates gathers boxes whose entire text is one bare
// number (digits or Roman) sitting in the wide margin bands, bounded by the
// page-count ceiling. It walks EVERY box, including ones earlier tiers already
// dropped: the +1 chain must stay contiguous when the gap-backed tiers took
// the roomy pages and only the tight ones remain to this track.
func collectPageNumberCandidates(boxes []pdf.TextBox, pageHeights map[int]float64, ceiling int) []pageNumCand {
var cands []pageNumCand
for i := range boxes {
b := boxes[i]
if isNonTextLayout(b.LayoutType) {
continue
}
h, ok := pageHeights[b.PageNumber]
if !ok || h <= 0 {
continue
}
band := ""
switch {
case b.Top >= h*seqBandBottomRatio:
band = "footer"
case b.Bottom <= h*seqBandTopRatio:
band = "header"
default:
continue
}
val, ok := parseBareNumberValue(b.Text)
if !ok || val > ceiling {
continue
}
cands = append(cands, pageNumCand{idx: i, page: b.PageNumber, value: val, top: b.Top, band: band})
}
return cands
}
// findPageNumberSequenceDrops returns the indices of candidates that take part
// in a run of at least seqMinRun consecutive pages whose bare numbers step by
// exactly one (up or down) while the run's total top-edge span stays within
// seqMaxDy — the same full-window criterion the locality track applies, so a
// per-hop check never lets drift accumulate past the threshold. Such a run is
// a page-number series on sequence evidence alone — enough to remove tight
// footers the geometric zone gate rejected; isolated or non-stepping numbers
// are left alone.
func findPageNumberSequenceDrops(cands []pageNumCand) []int {
byBand := make(map[string][]pageNumCand, 2)
for _, c := range cands {
byBand[c.band] = append(byBand[c.band], c)
}
dropped := make(map[int]bool, len(cands))
for _, group := range byBand {
if len(group) < seqMinRun {
continue
}
sort.Slice(group, func(i, j int) bool {
if group[i].page != group[j].page {
return group[i].page < group[j].page
}
return group[i].top < group[j].top
})
for _, dir := range []int{1, -1} {
n := len(group)
dp := make([]int, n)
prev := make([]int, n)
minTop := make([]float64, n)
maxTop := make([]float64, n)
for i := range group {
dp[i], prev[i] = 1, -1
minTop[i], maxTop[i] = group[i].top, group[i].top
}
for i := 0; i < n; i++ {
for j := 0; j < i; j++ {
if group[j].page != group[i].page-1 || group[j].value != group[i].value-dir {
continue
}
newMin, newMax := minTop[j], maxTop[j]
if group[i].top < newMin {
newMin = group[i].top
}
if group[i].top > newMax {
newMax = group[i].top
}
if newMax-newMin > seqMaxDy {
continue
}
// Among equal-length chains prefer the tighter Y span, so a
// wide early detour cannot block a later legal extension.
if dp[j]+1 > dp[i] || (dp[j]+1 == dp[i] && newMax-newMin < maxTop[i]-minTop[i]) {
dp[i] = dp[j] + 1
prev[i] = j
minTop[i], maxTop[i] = newMin, newMax
}
}
}
for i := 0; i < n; i++ {
if dp[i] < seqMinRun {
continue
}
for k := i; k >= 0; k = prev[k] {
dropped[group[k].idx] = true
}
}
}
}
out := make([]int, 0, len(dropped))
for idx := range dropped {
out = append(out, idx)
}
return out
}
// RemoveHeaderFooterBoxes drops boxes that are running headers or footers.
//
// It uses a 3D Cascaded Fusion scheme:
// 1. Tier 1: Deterministic Pattern & Semantic Tagging. Explicit DLA header/footer
// tags in margin zones, unambiguously formatted page numbers, and margin
// lines advertising a download site are removed immediately (even on 1- or
// 2-page documents).
// 2. Tier 2: Dual-Track Recurrence Engine. Evaluates recurrence across:
// - Global half-pages threshold ((numPages + 1) / 2).
// - Parity partition (even/odd) to eliminate bilateral alternating headers.
// - Locality track (consecutive pages with stable Y) to remove chapter-varying headers.
// 3. Tier 3: Adaptive Whitespace Gap. Automatically extends the candidate zone
// from 10% to up to 14% when separated from body content by >= 18pt of blank
// space.
// 4. Sequence track: bare numeric / Roman-numeral boxes in wide margin bands
// are removed when they step by one across >= 3 consecutive pages at a
// stable Y, catching page numbers that sit too tight under body text for
// the geometric gate in any tier above.
//
// totalPages is the document's full page count (engine.PageCount). The
// document-level judgments — the bare-page-number ceiling, the short-document
// guard, the global half-pages threshold, and the sequence-track ceiling — are
// computed from it, not from len(pageHeights): when Config.Pages parses a
// subset, pageHeights covers only those pages, while the page numbers are
// absolute and the document size is unchanged. The parity and locality tracks
// stay scope-relative (they judge the pages actually observed).
func RemoveHeaderFooterBoxes(boxes []pdf.TextBox, pageHeights map[int]float64, totalPages int) []pdf.TextBox {
if len(pageHeights) == 0 || len(boxes) == 0 {
return boxes
}
perPage := make(map[int][]int, len(pageHeights))
for i := range boxes {
perPage[boxes[i].PageNumber] = append(perPage[boxes[i].PageNumber], i)
}
allGapBelow := make(map[int]float64, len(boxes))
allGapAbove := make(map[int]float64, len(boxes))
for p, idxs := range perPage {
h, ok := pageHeights[p]
if !ok || h <= 0 {
continue
}
gb, ga := computePageGaps(boxes, idxs, h)
for idx, g := range gb {
allGapBelow[idx] = g
}
for idx, g := range ga {
allGapAbove[idx] = g
}
}
// Document-level basis. Never smaller than the observed subset so a caller
// that omits totalPages (0) degrades to the previous subset behavior and a
// bogus small value cannot shrink the thresholds below what was parsed.
docPages := totalPages
if docPages < len(pageHeights) {
docPages = len(pageHeights)
}
drop := make(map[int]struct{}, len(boxes))
// promoSpots records where a site-promo box was dropped in Tier 1 so the
// companion pass can learn its band peers' text as running-header ads.
var promoSpots []promoSpot
// Tier 1: Deterministic page numbers & DLA semantic tags.
for i := range boxes {
b := boxes[i]
if isNonTextLayout(b.LayoutType) {
continue
}
h, ok := pageHeights[b.PageNumber]
if !ok || h <= 0 {
continue
}
// A URL ad in either expanded margin zone (top 14% / bottom 14%,
// narrower than the sequence track's 20% bands) is an advertising
// header/footer on sight — no whitespace gap and no recurrence required,
// and this also fires on documents too short for the recurrence tracks.
if isSitePromo(b.Text) && (b.Bottom <= h*headerMaxZoneRatio || b.Top >= h*footerMinZoneRatio) {
common.Debug("header_footer: dropped by site promo",
zap.Int("page", b.PageNumber), zap.Int("textLen", utf8.RuneCountInString(b.Text)))
drop[i] = struct{}{}
if b.Bottom <= h*headerMaxZoneRatio {
promoSpots = append(promoSpots, promoSpot{page: b.PageNumber, band: "header"})
} else {
promoSpots = append(promoSpots, promoSpot{page: b.PageNumber, band: "footer"})
}
continue
}
zone := classifyZone(b, h, allGapAbove[i], allGapBelow[i])
if zone == "" {
continue
}
lt := strings.TrimSpace(b.LayoutType)
if (zone == "header" && lt == "header") || (zone == "footer" && lt == "footer") {
common.Debug("header_footer: dropped by DLA label",
zap.Int("page", b.PageNumber), zap.String("zone", zone),
zap.String("layoutType", lt), zap.Int("textLen", utf8.RuneCountInString(b.Text)))
drop[i] = struct{}{}
continue
}
if isDeterministicPageNumber(b.Text, zone, allGapAbove[i], allGapBelow[i], docPages) {
common.Debug("header_footer: dropped by page-number pattern",
zap.Int("page", b.PageNumber), zap.String("zone", zone),
zap.Int("textLen", utf8.RuneCountInString(b.Text)))
drop[i] = struct{}{}
continue
}
}
if docPages < minHeaderFooterPages {
if len(drop) == 0 {
return boxes
}
common.Debug("header_footer: removal completed (short document)",
zap.Int("total_boxes", len(boxes)), zap.Int("dropped_boxes", len(drop)))
return applyDrop(boxes, drop)
}
// Promo-companion propagation. Runs after Tier 1 and before the recurrence
// engine, so the boxes it removes never enter the recurrence statistics
// (which already skip dropped indices) and cannot perturb any "#"-masked count.
for _, idx := range promoCompanionDrops(boxes, perPage, pageHeights, drop, promoSpots) {
common.Debug("header_footer: dropped by promo companion",
zap.Int("page", boxes[idx].PageNumber), zap.String("zone", bandOf(boxes[idx], pageHeights[boxes[idx].PageNumber])), zap.Int("textLen", utf8.RuneCountInString(boxes[idx].Text)))
drop[idx] = struct{}{}
}
// Tier 2: Dual-Track Recurrence Engine.
keyMetas := make(map[zoneKey][]boxMeta)
for i := range boxes {
if _, ok := drop[i]; ok {
continue
}
b := boxes[i]
if isNonTextLayout(b.LayoutType) {
continue
}
h, ok := pageHeights[b.PageNumber]
if !ok || h <= 0 {
continue
}
zone := classifyZone(b, h, allGapAbove[i], allGapBelow[i])
if zone == "" {
continue
}
norm := normalizeRunningText(b.Text)
if norm == "" {
continue
}
key := zoneKey{zone: zone, text: norm}
keyMetas[key] = append(keyMetas[key], boxMeta{idx: i, page: b.PageNumber, top: b.Top})
}
evenTotal, oddTotal := 0, 0
for p := range pageHeights {
if p%2 == 0 {
evenTotal++
} else {
oddTotal++
}
}
minGlobalPages := (docPages + 1) / 2
if minGlobalPages < 2 {
minGlobalPages = 2
}
for key, metas := range keyMetas {
pageSet := make(map[int]bool, len(metas))
evenPages, oddPages := 0, 0
for _, m := range metas {
if !pageSet[m.page] {
pageSet[m.page] = true
if m.page%2 == 0 {
evenPages++
} else {
oddPages++
}
}
}
distinctPages := len(pageSet)
var reason string
// 1. Global coverage: at least half the pages (rounded up).
if distinctPages >= minGlobalPages {
reason = "global_frequency"
} else if evenTotal >= 3 && evenPages >= 3 && float64(evenPages)/float64(evenTotal) >= minParityCoverageRatio {
// 2. Parity partition: at least minParityCoverageRatio of even pages.
reason = "parity_even"
} else if oddTotal >= 3 && oddPages >= 3 && float64(oddPages)/float64(oddTotal) >= minParityCoverageRatio {
// 2. Parity partition: at least minParityCoverageRatio of odd pages.
reason = "parity_odd"
}
if reason != "" {
common.Debug("header_footer: dropped by recurrence",
zap.String("zone", key.zone), zap.Int("textLen", utf8.RuneCountInString(key.text)),
zap.String("reason", reason), zap.Int("pages", distinctPages))
for _, m := range metas {
drop[m.idx] = struct{}{}
}
continue
}
// 3. Locality track: stable consecutive runs of at least 3 pages with stable Y.
// Only drop occurrences that actually belong to qualifying consecutive runs,
// preventing distant isolated occurrences of the same text from being deleted.
if len(metas) >= 3 && utf8.RuneCountInString(key.text) <= 60 {
runIndices := findStableConsecutiveRunIndices(metas, 3, 4.0)
if len(runIndices) > 0 {
common.Debug("header_footer: dropped by recurrence",
zap.String("zone", key.zone), zap.Int("textLen", utf8.RuneCountInString(key.text)),
zap.String("reason", "local_consecutive_run"), zap.Int("boxes", len(runIndices)))
for _, idx := range runIndices {
drop[idx] = struct{}{}
}
}
}
}
// Sequence-backed page numbers. Word-derived books sit their page numbers
// a few points under body text — below the 90% footer line AND below the
// expanded band's whitespace requirement — so the geometric gate rejects
// them before any tier can see a pattern or count a recurrence. A run of
// bare numbers stepping by one across consecutive pages at a stable Y is a
// page-number series on sequence evidence alone. This runs after the
// recurrence engine so it only adds drops: the recurrence counts, which
// include these boxes under their masked key, stay untouched.
if seqDrops := findPageNumberSequenceDrops(collectPageNumberCandidates(boxes, pageHeights, pageNumberCeiling(docPages))); len(seqDrops) > 0 {
common.Debug("header_footer: dropped by page-number sequence", zap.Int("boxes", len(seqDrops)))
for _, idx := range seqDrops {
drop[idx] = struct{}{}
}
}
if len(drop) == 0 {
return boxes
}
common.Debug("header_footer: removal completed",
zap.Int("total_boxes", len(boxes)), zap.Int("dropped_boxes", len(drop)))
return applyDrop(boxes, drop)
}
// findStableConsecutiveRunIndices finds all box indices that belong to any qualifying
// consecutive run of at least minRun pages where the top variation is within maxDy.
// Restricting removal to these specific indices ensures isolated occurrences outside
// the run (e.g. on later pages) are not accidentally dropped.
func findStableConsecutiveRunIndices(metas []boxMeta, minRun int, maxDy float64) []int {
if len(metas) < minRun {
return nil
}
sort.Slice(metas, func(i, j int) bool {
if metas[i].page != metas[j].page {
return metas[i].page < metas[j].page
}
return metas[i].top < metas[j].top
})
type pageOccurrence struct {
page int
top float64
indices []int
}
var distinct []pageOccurrence
for _, m := range metas {
if len(distinct) == 0 || distinct[len(distinct)-1].page != m.page {
distinct = append(distinct, pageOccurrence{page: m.page, top: m.top, indices: []int{m.idx}})
} else {
distinct[len(distinct)-1].indices = append(distinct[len(distinct)-1].indices, m.idx)
}
}
if len(distinct) < minRun {
return nil
}
qualifyingPages := make(map[int]bool)
left := 0
for right := 0; right < len(distinct); right++ {
// Reset window if consecutive page sequence is broken
if right > 0 && distinct[right].page != distinct[right-1].page+1 {
left = right
}
// Check all qualifying consecutive windows ending at right
for (right - left + 1) >= minRun {
curMin := distinct[left].top
curMax := distinct[left].top
for k := left + 1; k <= right; k++ {
if distinct[k].top < curMin {
curMin = distinct[k].top
}
if distinct[k].top > curMax {
curMax = distinct[k].top
}
}
if (curMax - curMin) <= maxDy {
for k := left; k <= right; k++ {
qualifyingPages[distinct[k].page] = true
}
break
}
left++
}
}
if len(qualifyingPages) == 0 {
return nil
}
var result []int
for _, p := range distinct {
if qualifyingPages[p.page] {
result = append(result, p.indices...)
}
}
return result
}
// applyDrop filters out the dropped boxes and returns the survivors.
func applyDrop(boxes []pdf.TextBox, drop map[int]struct{}) []pdf.TextBox {
if len(drop) == 0 {
return boxes
}
out := make([]pdf.TextBox, 0, len(boxes)-len(drop))
for i := range boxes {
if _, ok := drop[i]; ok {
continue
}
out = append(out, boxes[i])
}
return out
}
// collapseText lower-cases and whitespace-collapses text without masking
// digits, for comparison keys that must not treat two differently-numbered
// lines as identical (unlike normalizeRunningText, which is for running
// headers/footers whose per-page number must be folded away).
func collapseText(text string) string {
return strings.ToLower(strings.Join(strings.Fields(text), " "))
}
// normalizeRunningText collapses whitespace, replaces digit runs with a single
// '#' and lower-cases the result, so per-page variants of the same running
// header/footer ("- 2 -", "- 3 -", "Page 4 of 9", "第 1 页") share one
// comparison key.
func normalizeRunningText(text string) string {
t := strings.Join(strings.Fields(text), " ")
out := make([]rune, 0, len(t))
inDigit := false
for _, r := range t {
if r == '—' || r == '–' || r == '-' {
r = '-'
}
if isMaskableDigit(r) {
inDigit = true
continue
}
if inDigit {
out = append(out, '#')
inDigit = false
}
out = append(out, r)
}
if inDigit {
out = append(out, '#')
}
return strings.ToLower(strings.TrimSpace(string(out)))
}
// isMaskableDigit reports whether r is a digit whose per-page value must be
// masked before running headers/footers are compared: ASCII and full-width
// decimal digits. Full width is required for the same reason pageNumberPattern
// accepts it — otherwise every page of "第 1 页" / "第 2 页" gets its own key
// and the footer never reaches minPages.
func isMaskableDigit(r rune) bool {
return (r >= '0' && r <= '9') || (r >= '0' && r <= '9')
}