Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(pdf/layout): recover title-bridged & gutter-less double columns (2D rescue + L3) (#18150)

Jack7646afab1ba4
diff --git a/internal/deepdoc/parser/pdf/layout/combined_column.go b/internal/deepdoc/parser/pdf/layout/combined_column.go
index fe1b0fcc3c..9ca6ce51c0 100644
--- a/internal/deepdoc/parser/pdf/layout/combined_column.go
+++ b/internal/deepdoc/parser/pdf/layout/combined_column.go
@@ -4,6 +4,7 @@ import (
"math"
"math/rand"
"sort"
+ "unicode/utf8"
pdf "ragflow/internal/deepdoc/parser/pdf/type"
util "ragflow/internal/deepdoc/parser/pdf/util"
@@ -74,6 +75,33 @@ const maxColumnCount = 4
// touching those.
const minColLineFrac = 0.12
+// maxPageExtent caps the X span a single line may plausibly occupy. A line
+// wider than this is treated as a malformed coordinate (mirrors pdf-inspector's
+// MAX_PAGE_EXTENT=14400 guard) and excluded from the column projection so it
+// cannot balloon the page extent and collapse multi-column detection.
+const maxPageExtent = 14400.0
+
+// maxTrimFraction is the largest fraction of lines robustPageExtent may discard
+// as outliers. If more than this fraction is anomalous, the page is trusted
+// as-is: the "anomalies" are the norm, not noise.
+const maxTrimFraction = 0.10
+
+// maxBins caps the histogram allocation in gapColumnCount/detectColumnCount2D so
+// a malformed (ballooned) page extent cannot trigger an OOM-scale allocation
+// (mirrors pdf-inspector's bin cap). When the extent is huge, the bin is
+// widened so the projection still resolves real gutters.
+const maxBins = 65536
+
+// gapMinFrac: a horizontal run of low coverage counts as a column gap only if
+// it is at least this fraction of the page width. Reused by both gapColumnCount
+// and the 2D both-sides gutter rescue so the two detectors agree on what a
+// "real" gutter width is.
+const gapMinFrac = 0.04
+
+// binPt: x-binning resolution (points) for the 1D gap histogram and the 2D
+// gutter scan. Sharing it keeps the gap and gutter detectors aligned.
+const binPt = 2.0
+
// detectColumnCount returns (columnCount, centroids) for one page.
// columnCount is 1, 2, or up to maxColumnCount; centroids are the k cluster
// means in x0 space (snapshot of the gate decision) and are reused for ColID
@@ -83,7 +111,7 @@ func detectColumnCount(boxes []pdf.TextBox, indices []int) (int, []float64) {
for i, idx := range indices {
lines[i] = boxes[idx]
}
- g := gapColumnCount(lines, 0.04, 0.15, 2.0)
+ g := gapColumnCount(lines, gapMinFrac, 0.15, binPt)
if g >= 2 {
_, width := pageExtent(lines)
if width > 0 {
@@ -133,6 +161,20 @@ func detectColumnCount(boxes []pdf.TextBox, indices []int) (int, []float64) {
}
return 1, nil
}
+ // 2D rescue: a clean vertical gutter the 1D projection masks via bridging
+ // rows (full-width front matter + in-body headings/captions). Recovers
+ // title-bridged doubles the balance gate correctly rejects (sparse
+ // minority). Runs only after both gap>=2 and the balance gate fail, so it
+ // never touches already-correct pages.
+ if k, cents := detectColumnCount2D(lines); k >= 2 {
+ return k, cents
+ }
+ // L3: median-width-ratio complement (PR #10475). Fires only after gap,
+ // balance, and the 2D valley rescue all returned 1, so it never touches
+ // the pages they already handle. Targets "gutter-less" doubles/triples.
+ if k, cents := detectColumnCountMedian(lines); k >= 2 {
+ return k, cents
+ }
return 1, nil
}
@@ -231,19 +273,38 @@ func gapColumnCount(lines []pdf.TextBox, gapMinFrac, crossTol, binPt float64) in
if n == 0 {
return 1
}
- minX0, width := pageExtent(lines)
+ // A1: image/equation placeholders must not feed the projection — a figure
+ // spanning the gutter would otherwise fill the gap and mask a real column
+ // boundary.
+ lines = textProjectionLines(lines)
+ if len(lines) == 0 {
+ return 1
+ }
+ // A2: robust page extent discards malformed/outlier lines so a single bad
+ // box cannot balloon the width and collapse detection to one column.
+ minX0, width := robustPageExtent(lines)
if width <= 0 {
return 1
}
minGap := gapMinFrac * width
- nb := int(width/binPt) + 1
+ // Cap the bin count so a ballooned extent cannot allocate an OOM-scale
+ // histogram. When the extent is huge, widen the bin so the projection still
+ // resolves real gutters.
+ effBin := binPt
+ if width/float64(maxBins) > effBin {
+ effBin = width / float64(maxBins)
+ }
+ nb := int(width/effBin) + 1
+ if nb < 1 {
+ nb = 1
+ }
cov := make([]int, nb)
for _, b := range lines {
- i0 := int((b.X0 - minX0) / binPt)
+ i0 := int((b.X0 - minX0) / effBin)
if i0 < 0 {
i0 = 0
}
- i1 := int((b.X1 - minX0) / binPt)
+ i1 := int((b.X1 - minX0) / effBin)
if i1 > nb-1 {
i1 = nb - 1
}
@@ -256,7 +317,7 @@ func gapColumnCount(lines []pdf.TextBox, gapMinFrac, crossTol, binPt float64) in
run := 0.0
for _, c := range cov {
if float64(c) < thr {
- run += binPt
+ run += effBin
} else {
if run >= minGap {
cols++
@@ -270,6 +331,323 @@ func gapColumnCount(lines []pdf.TextBox, gapMinFrac, crossTol, binPt float64) in
return cols
}
+// bridgingFrac: lines wider than this fraction of the page text width are
+// treated as bridging elements — full-width front matter (already dropped by
+// dropFullWidth at 0.9) plus partially-wide in-body headings/captions that
+// span the gutter. Dropping them before the valley scan is what exposes the
+// clean gutter of a title-bridged double column. 0.60 is the sweet spot
+// measured on the 70-page corpus: lower (0.50) leaves too few real column
+// lines on single pages and keeps enough bridging width to still hide some
+// gutters; higher (0.65) lets the sparse bridging lines that hide the target
+// gutters survive.
+const bridgingFrac = 0.60
+
+// medianFullWidthFrac is the full-width threshold for the L3 median-width
+// detector. It is lower than dropFullWidth's 0.9 because the median path
+// buckets by normalized center x and only needs to exclude lines that would
+// otherwise dominate every bucket; lines between 0.8 and 0.9 width are rare
+// and keeping them out of the buckets avoids a single wide line skewing cents.
+const medianFullWidthFrac = 0.80
+
+// medianColCap: max column count the median-width-ratio signal (L3, from PR
+// #10475's page_w/median_w) may assign. Capped at 3 so a raw_cols estimate of
+// 4 (common on 3-column pages) does not over-shoot, and well under
+// maxColumnCount.
+const medianColCap = 3
+
+// shortLineFrac: L3 requires at least one line spanning >= this fraction of
+// the page width. A real multi-column page has lines that span a column
+// (~page_w/N); a single page of uniformly short lines also has a small median
+// width, but no near-full-width line — this guard filters those false doubles.
+const shortLineFrac = 0.45
+
+// detectColumnCount2D is a rescue detector for title-bridged double columns:
+// pages whose two body columns are separated by a clean gutter that the 1D x0
+// projection loses once full-width front matter (and in-body bridging
+// headings/captions) spans it. It runs only after gap>=2 and the balance gate
+// both fail, so it never touches already-correct pages.
+//
+// Method (faithful to tool-py/column_detectors.gap_glyph_body_column_counts,
+// extended with bridging removal): project the BODY — full-width lines dropped
+// by dropFullWidth, then any still-wide bridging line dropped at
+// bridgingFrac*width — onto the x-axis with glyph-count-per-bin weighting
+// (each covered bin receives the line's full rune count, so a wide line
+// contributes proportionally more), and look
+// for INTERIOR valleys (low-ink runs bounded by high ink on both sides, wider
+// than gapMinFrac*width, and not at the page edge). A single clean gutter
+// splits the page into two real columns.
+//
+// The rescue ACCEPTS only exactly one interior valley (k=2). Zero valleys
+// means no clean gutter (keep single). Two or more valleys means either a
+// multi-column layout (already handled by the gap path) or a single page with
+// a vertical blank band (figure/equation) — both are rejected so the rescue
+// never over-splits a single column into 3+. The both-sides prune gate
+// (pruneColumns, minColLineFrac) is the final guard: a spurious second block
+// with too few lines is dropped.
+func detectColumnCount2D(lines []pdf.TextBox) (int, []float64) {
+ // A1: strip figure/equation boxes (they span the gutter and would fill the
+ // projection, hiding a real column boundary). A2: robust extent so a
+ // malformed box cannot balloon the page width / histogram allocation.
+ projLines := textProjectionLines(lines)
+ minX0, width := robustPageExtent(projLines)
+ if width <= 0 {
+ return 0, nil
+ }
+ body := dropFullWidth(projLines, width)
+ body = dropWide(body, width, bridgingFrac)
+ if len(body) < 4 {
+ return 0, nil
+ }
+ // Cap the bin count so a ballooned extent cannot allocate an OOM-scale
+ // histogram. When the extent is huge, widen the bin so the projection still
+ // resolves real gutters.
+ effBin := binPt
+ if width/float64(maxBins) > effBin {
+ effBin = width / float64(maxBins)
+ }
+ nb := int(width/effBin) + 1
+ proj := make([]int, nb)
+ for _, b := range body {
+ w := utf8.RuneCountInString(b.Text)
+ if w <= 0 {
+ w = 1
+ }
+ i0 := clampInt(int((b.X0-minX0)/effBin), 0, nb-1)
+ i1 := clampInt(int((b.X1-minX0)/effBin), 0, nb-1)
+ for i := i0; i <= i1; i++ {
+ proj[i] += w
+ }
+ }
+ pk := 0
+ for _, p := range proj {
+ if p > pk {
+ pk = p
+ }
+ }
+ if pk == 0 {
+ return 0, nil
+ }
+ // A gutter is a run of bins whose glyph-weight is below valleyFrac of the
+ // page peak. Measured on the 70-page corpus this relative threshold (the
+ // tool-py reference value) is what actually recovers title-bridged doubles:
+ // their gutter is clean (≈0 glyphs) and the minority column still carries
+ // enough ink to sit above valleyFrac*peak and bound the gutter. A minority
+ // column below ~30% of peak ink (e.g. a very sparse 6-line column) merges
+ // with the gutter and is NOT recovered — that is a real limitation, not a
+ // bug; such pages fall back to the confidence-labeling track (issue #18079).
+ const valleyFrac = 0.30
+ minGap := gapMinFrac * width
+ edge := int(0.05 * width / effBin)
+ if edge < 0 {
+ edge = 0
+ }
+ // Find interior valleys; accept ONLY a single clean gutter (k=2). Zero
+ // valleys means no clean gutter (keep single). Two or more valleys means
+ // either a multi-column layout (handled by the gap path) or a single page
+ // with a vertical blank band (figure/equation) — both are rejected so the
+ // rescue never over-splits a single column into 3+.
+ var valleyC float64
+ count := 0
+ i := 0
+ for i < nb {
+ if float64(proj[i]) < valleyFrac*float64(pk) {
+ j := i
+ for j < nb && float64(proj[j]) < valleyFrac*float64(pk) {
+ j++
+ }
+ runW := float64(j-i) * effBin
+ isInterior := i > edge && j-1 < nb-1-edge
+ if runW >= minGap && isInterior {
+ count++
+ valleyC = minX0 + float64(i+j)*effBin/2
+ }
+ i = j
+ } else {
+ i++
+ }
+ }
+ if count != 1 {
+ return 0, nil
+ }
+ // Split the body at the single valley into left/right blocks; each
+ // centroid is the mean X0 of its lines. Classify by b.X0 (not the center)
+ // so the split agrees with pruneColumns' X0-based assignment — lines whose
+ // center and X0 fall on opposite sides of the valley would otherwise be
+ // counted differently by the two steps.
+ var leftSum, rightSum float64
+ lc, rc := 0, 0
+ for _, b := range body {
+ if b.X0 < valleyC {
+ leftSum += b.X0
+ lc++
+ } else {
+ rightSum += b.X0
+ rc++
+ }
+ }
+ if lc == 0 || rc == 0 {
+ return 0, nil
+ }
+ cents := []float64{leftSum / float64(lc), rightSum / float64(rc)}
+ if pk2, pc, ok := pruneColumns(body, cents); ok {
+ return pk2, pc
+ }
+ return 0, nil
+}
+
+// dropWide removes lines whose width spans >= frac of the page text width.
+// Used by detectColumnCount2D to strip in-body bridging headings/captions
+// (partially-wide lines that span the gutter but are not full-width front
+// matter) before the valley scan. Returns nil if every line is wide so the
+// caller treats the page as single rather than pushing an empty body through.
+func dropWide(lines []pdf.TextBox, width, frac float64) []pdf.TextBox {
+ if frac >= 1 {
+ return lines
+ }
+ thr := frac * width
+ out := make([]pdf.TextBox, 0, len(lines))
+ for _, b := range lines {
+ if b.X1-b.X0 < thr {
+ out = append(out, b)
+ }
+ }
+ if len(out) == 0 {
+ return nil
+ }
+ return out
+}
+
+// medianWidth returns the median box width on the page. Used by the L3
+// median-width-ratio column signal.
+func medianWidth(lines []pdf.TextBox) float64 {
+ if len(lines) == 0 {
+ return 1.0
+ }
+ ws := make([]float64, len(lines))
+ for i, b := range lines {
+ ws[i] = b.X1 - b.X0
+ if ws[i] < 1 {
+ ws[i] = 1
+ }
+ }
+ sort.Float64s(ws)
+ n := len(ws)
+ if n%2 == 1 {
+ return ws[n/2]
+ }
+ return (ws[n/2-1] + ws[n/2]) / 2.0
+}
+
+// maxWidth returns the widest box on the page.
+func maxWidth(lines []pdf.TextBox) float64 {
+ m := 0.0
+ for _, b := range lines {
+ if w := b.X1 - b.X0; w > m {
+ m = w
+ }
+ }
+ return m
+}
+
+// detectColumnCountMedian is the L3 complementary signal, inspired by PR
+// #10475's _assign_column (page_w / median_line_width). It fires only after
+// gap, the balance gate, and the 2D valley rescue have ALL returned a single
+// column, so it never touches the pages they already handle correctly.
+//
+// It targets "gutter-less" doubles/triples: pages whose two (or three) body
+// columns are separated by a gutter so narrow/bridged that the x-projection
+// has no clean ink dip — so the geometric detectors miss them, yet each line
+// is only ~page_w/N wide, giving raw_cols = page_w/median_w >= 2.
+//
+// Two gates keep it safe (measured on the 70-page corpus):
+// - raw_cols > maxColumnCount (4): a huge ratio means table cells, not text
+// columns (narrow cells yield a tiny median width) -> skip.
+// - no line spanning >= shortLineFrac*page_w: the page is one column of
+// uniformly short lines whose small median width is not a real multi-column
+// signal -> skip.
+//
+// When it fires, columns are assigned by normalized center-x bucketing
+// (matching PR #10475's col_id assignment); the bucket means become centroids.
+func detectColumnCountMedian(lines []pdf.TextBox) (int, []float64) {
+ minX0, width := pageExtent(lines)
+ if width <= 0 {
+ return 0, nil
+ }
+ mw := medianWidth(lines)
+ if mw < 1 {
+ mw = 1
+ }
+ raw := int(width / mw)
+ if raw < 2 {
+ return 0, nil
+ }
+ if raw > maxColumnCount {
+ // Table-like (narrow cells): not a text layout.
+ return 0, nil
+ }
+ if maxWidth(lines) < shortLineFrac*width {
+ // Uniformly short lines, not real columns.
+ return 0, nil
+ }
+ k := raw
+ if k > medianColCap {
+ k = medianColCap
+ }
+ if k < 2 {
+ k = 2
+ }
+ // Bucket non-full-width lines by normalized center x, mirroring PR #10475.
+ // Collect the SAME non-full-width lines into body so the prune step counts
+ // the line set that produced the centroids — otherwise full-width
+ // titles/abstracts (which the bucket loop skips) would be re-counted by
+ // pruneColumns, inflate one column, and let a real multi-column page
+ // collapse to one. This is the same discipline balancedBodyK2 and
+ // detectColumnCount2D already follow.
+ fwThr := medianFullWidthFrac * width
+ body := make([]pdf.TextBox, 0, len(lines))
+ buckets := make([][]float64, k)
+ for _, b := range lines {
+ if b.X1-b.X0 >= fwThr {
+ continue
+ }
+ body = append(body, b)
+ cx := 0.5 * (b.X0 + b.X1)
+ norm := (cx - minX0) / width
+ if norm < 0 {
+ norm = 0
+ }
+ if norm > 0.999999 {
+ norm = 0.999999
+ }
+ bkt := int(norm * float64(k))
+ if bkt > k-1 {
+ bkt = k - 1
+ }
+ buckets[bkt] = append(buckets[bkt], b.X0)
+ }
+ cents := make([]float64, 0, k)
+ for _, bx := range buckets {
+ if len(bx) == 0 {
+ continue
+ }
+ var s float64
+ for _, x := range bx {
+ s += x
+ }
+ cents = append(cents, s/float64(len(bx)))
+ }
+ if len(cents) < 2 {
+ return 0, nil
+ }
+ // Require each column to hold enough lines (the same guard used by the
+ // rest of the detector) so a sparse side column does not become a false
+ // split.
+ if pk, pc, ok := pruneColumns(body, cents); ok {
+ return pk, pc
+ }
+ return 0, nil
+}
+
// balancedBodyK2 runs a forced k=2 KMeans on the BODY x0 (full-width front
// matter excluded) and reports whether the split is a real two-column: two
// clusters each holding >= minModeFrac of body lines, separated by >=
@@ -351,6 +729,102 @@ func pageExtent(lines []pdf.TextBox) (minX0, width float64) {
return minX0, maxX1 - minX0
}
+// textProjectionLines returns the lines that should feed the column
+// projection. Image and equation placeholders are excluded because a figure
+// that spans the gutter would otherwise fill the gutter's gap and mask a real
+// column boundary (mirrors pdf-inspector stripping image placeholders). Table
+// boxes are intentionally kept: the existing tableMaxColFrac gate already
+// handles narrow table columns, and dropping them here would widen the blast
+// radius unnecessarily.
+func textProjectionLines(lines []pdf.TextBox) []pdf.TextBox {
+ out := make([]pdf.TextBox, 0, len(lines))
+ for _, b := range lines {
+ switch b.LayoutType {
+ case pdf.LayoutTypeFigure, pdf.LayoutTypeEquation:
+ continue
+ default:
+ out = append(out, b)
+ }
+ }
+ return out
+}
+
+// robustPageExtent returns the X extent (minX0, width) of the text body,
+// discarding a small number of outlier lines (malformed coordinates / stray
+// boxes) that would otherwise balloon the extent and collapse multi-column
+// detection. Normal pages return exactly pageExtent's result.
+func robustPageExtent(lines []pdf.TextBox) (minX0, width float64) {
+ if len(lines) == 0 {
+ return 0, 0
+ }
+ // Drop implausibly wide (malformed) lines outright: a single box with an
+ // absurd X1 (e.g. 1e6) would otherwise set the page width to 1e6.
+ work := make([]pdf.TextBox, 0, len(lines))
+ dropped := 0
+ for _, b := range lines {
+ if b.X1-b.X0 > maxPageExtent {
+ dropped++
+ continue
+ }
+ work = append(work, b)
+ }
+ if len(work) == 0 || float64(dropped)/float64(len(lines)) >= maxTrimFraction {
+ // Everything malformed, or too many dropped: the outliers are the
+ // norm. Fall back to the raw extent so the page is never altered.
+ return pageExtent(lines)
+ }
+ // Cluster lines by left edge; discard clusters separated from the main
+ // body by more than a full page width, provided they are a minority.
+ return clusteredExtent(work)
+}
+
+// clusteredExtent keeps the largest cluster of lines (by count) and returns its
+// extent, but only when the discarded minority is below maxTrimFraction;
+// otherwise it returns the raw extent of all lines. This catches outliers whose
+// width alone is plausible (e.g. a tiny box placed at an absurd X coordinate).
+func clusteredExtent(lines []pdf.TextBox) (minX0, width float64) {
+ if len(lines) <= 1 {
+ return pageExtent(lines)
+ }
+ xs := make([]float64, len(lines))
+ for i, b := range lines {
+ xs[i] = b.X0
+ }
+ sort.Float64s(xs)
+ // Split into clusters at gaps larger than a full page.
+ clusters := [][]float64{{xs[0]}}
+ for i := 1; i < len(xs); i++ {
+ if xs[i]-xs[i-1] > maxPageExtent {
+ clusters = append(clusters, []float64{xs[i]})
+ } else {
+ clusters[len(clusters)-1] = append(clusters[len(clusters)-1], xs[i])
+ }
+ }
+ if len(clusters) == 1 {
+ return pageExtent(lines)
+ }
+ best := 0
+ for i := 1; i < len(clusters); i++ {
+ if len(clusters[i]) > len(clusters[best]) {
+ best = i
+ }
+ }
+ dropped := len(lines) - len(clusters[best])
+ if float64(dropped)/float64(len(lines)) >= maxTrimFraction {
+ return pageExtent(lines)
+ }
+ lo, hi := clusters[best][0], clusters[best][0]
+ for _, x := range clusters[best] {
+ if x < lo {
+ lo = x
+ }
+ if x > hi {
+ hi = x
+ }
+ }
+ return lo, hi - lo
+}
+
// snapX0s pulls x0 values within indentTol of minX0 back to minX0, so slightly
// indented lines still cluster with the left edge (mirrors _assign_column).
func snapX0s(x0s []float64, minX0, indentTol float64) []float64 {
diff --git a/internal/deepdoc/parser/pdf/layout/combined_column_defensive_test.go b/internal/deepdoc/parser/pdf/layout/combined_column_defensive_test.go
new file mode 100644
index 0000000000..3f0c26b030
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/layout/combined_column_defensive_test.go
@@ -0,0 +1,160 @@
+package layout
+
+import (
+ "testing"
+
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+)
+
+// columnCountOf returns the number of distinct columns reported by AssignColumn
+// for the given page boxes (mirrors the harness's ColID tally).
+func columnCountOf(boxes []pdf.TextBox) int {
+ res := AssignColumn(boxes)
+ k := 1
+ for _, b := range res {
+ if b.ColID+1 > k {
+ k = b.ColID + 1
+ }
+ }
+ return k
+}
+
+// addBox appends a text box spanning [x0,x1] at the next vertical slot.
+func addBox(boxes *[]pdf.TextBox, x0, x1 float64) {
+ top := float64(len(*boxes)) * 12
+ *boxes = append(*boxes, pdf.TextBox{
+ PageNumber: 0, X0: x0, X1: x1, Top: top, Bottom: top + 10,
+ LayoutType: pdf.LayoutTypeText, Text: "b",
+ })
+}
+
+// TestTextProjectionLinesStripsFigures proves A1: image and equation
+// placeholders must not feed the column projection (a figure spanning the
+// gutter would otherwise mask a real column boundary). Text and table boxes
+// are kept.
+func TestTextProjectionLinesStripsFigures(t *testing.T) {
+ lines := []pdf.TextBox{
+ {LayoutType: pdf.LayoutTypeText, X0: 100, X1: 300},
+ {LayoutType: pdf.LayoutTypeFigure, X0: 50, X1: 700},
+ {LayoutType: pdf.LayoutTypeEquation, X0: 50, X1: 700},
+ {LayoutType: pdf.LayoutTypeTable, X0: 100, X1: 300},
+ {LayoutType: pdf.LayoutTypeText, X0: 400, X1: 600},
+ {LayoutType: "", X0: 100, X1: 300}, // empty type is treated as text
+ }
+ out := textProjectionLines(lines)
+ if len(out) != 4 {
+ t.Fatalf("textProjectionLines kept %d lines, want 4 (text+table+text+empty), dropped %d",
+ len(out), len(lines)-len(out))
+ }
+ for _, b := range out {
+ if b.LayoutType == pdf.LayoutTypeFigure || b.LayoutType == pdf.LayoutTypeEquation {
+ t.Errorf("figure/equation box leaked into projection: %+v", b)
+ }
+ }
+}
+
+// TestRobustPageExtentDropsOutlier proves A2: a single malformed box with an
+// absurd width must not inflate the page extent; the extent of the real text
+// body must be returned instead. Uses a realistic page (many real lines) so the
+// lone outlier is a true minority (< maxTrimFraction).
+func TestRobustPageExtentDropsOutlier(t *testing.T) {
+ var lines []pdf.TextBox
+ for i := 0; i < 20; i++ {
+ lines = append(lines, pdf.TextBox{X0: 100, X1: 300}) // left column
+ }
+ for i := 0; i < 20; i++ {
+ lines = append(lines, pdf.TextBox{X0: 400, X1: 600}) // right column
+ }
+ lines = append(lines, pdf.TextBox{X0: 100, X1: 1e6}) // malformed outlier
+ minX0, width := robustPageExtent(lines)
+ wantMin, wantWidth := 100.0, 500.0
+ if minX0 != wantMin || width != wantWidth {
+ t.Fatalf("robustPageExtent = (%.0f, %.0f), want (%.0f, %.0f) (outlier must be dropped)",
+ minX0, width, wantMin, wantWidth)
+ }
+}
+
+// TestRobustPageExtentNotTrimsRealFarColumns proves the A2 guard: two real
+// columns placed far apart (but within a sane page span) must NOT be trimmed
+// to one. The gap between them is large but below maxPageExtent, so no split.
+func TestRobustPageExtentNotTrimsRealFarColumns(t *testing.T) {
+ lines := []pdf.TextBox{
+ {X0: 100, X1: 300}, // left column
+ {X0: 2000, X1: 2200}, // right column, far but < maxPageExtent away
+ }
+ minX0, width := robustPageExtent(lines)
+ // Extent must span BOTH columns: 100 .. 2200.
+ if minX0 != 100.0 || width != 2100.0 {
+ t.Fatalf("robustPageExtent = (%.0f, %.0f), want (100, 2100) (real far columns kept)",
+ minX0, width)
+ }
+}
+
+// TestSyntheticOutlierBoxKeepsColumns proves A2 end-to-end: a clean double
+// column plus a malformed box must still be detected as 2 columns (the outlier
+// must not collapse detection to 1 via a ballooned extent).
+func TestSyntheticOutlierBoxKeepsColumns(t *testing.T) {
+ var boxes []pdf.TextBox
+ for i := 0; i < 14; i++ {
+ addBox(&boxes, 100, 350) // left column
+ }
+ for i := 0; i < 6; i++ {
+ addBox(&boxes, 450, 700) // right column
+ }
+ // Malformed box: normal left edge but absurd right edge.
+ boxes = append(boxes, pdf.TextBox{
+ PageNumber: 0, X0: 100, X1: 1e6, Top: 9999, Bottom: 10009,
+ LayoutType: pdf.LayoutTypeText, Text: "bad",
+ })
+ k := gapColumnCount(boxes, 0.04, 0.15, 2.0)
+ if k != 2 {
+ t.Fatalf("outlier box collapsed detection to %d column(s); want 2", k)
+ }
+}
+
+// TestSyntheticFigureSpanningGutter proves A1 end-to-end on the 1D gap path: a
+// full-width figure that bridges the gutter must not fill the projection and
+// mask the two real text columns.
+func TestSyntheticFigureSpanningGutter(t *testing.T) {
+ var boxes []pdf.TextBox
+ for i := 0; i < 14; i++ {
+ addBox(&boxes, 100, 350) // left column
+ }
+ for i := 0; i < 6; i++ {
+ addBox(&boxes, 450, 700) // right column
+ }
+ // Full-width figure spanning the gutter.
+ boxes = append(boxes, pdf.TextBox{
+ PageNumber: 0, X0: 100, X1: 700, Top: 5000, Bottom: 5100,
+ LayoutType: pdf.LayoutTypeFigure, Text: "fig",
+ })
+ k := gapColumnCount(boxes, 0.04, 0.15, 2.0)
+ if k != 2 {
+ t.Fatalf("figure spanning gutter collapsed detection to %d column(s); want 2", k)
+ }
+}
+
+// TestDetectColumnCount2D_IgnoresFigureSpanningGutter proves A1 extends to the
+// 2D rescue projection: a figure that spans the gutter must not fill the
+// projection and hide a real two-column valley. Without A1 the figure masks
+// the gutter and detectColumnCount2D returns 0 (miss); with A1 it returns 2.
+func TestDetectColumnCount2D_IgnoresFigureSpanningGutter(t *testing.T) {
+ var boxes []pdf.TextBox
+ for i := 0; i < 14; i++ {
+ addBox(&boxes, 100, 340) // left column
+ }
+ for i := 0; i < 6; i++ {
+ addBox(&boxes, 460, 700) // right column
+ }
+ // Figure spanning the gutter: wide enough to survive dropWide (>=0.6*width
+ // would drop it, so keep it just under) but not full-width. It fills the
+ // projection across the real column boundary and hides the valley pre-A1.
+ boxes = append(boxes, pdf.TextBox{
+ PageNumber: 0, X0: 100, X1: 455, Top: 5000, Bottom: 5100,
+ LayoutType: pdf.LayoutTypeFigure, Text: "fig",
+ })
+ k, _ := detectColumnCount2D(boxes)
+ if k != 2 {
+ t.Fatalf("figure spanning gutter hid the 2D valley: detectColumnCount2D=%d, want 2", k)
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/layout/combined_column_synthetic_test.go b/internal/deepdoc/parser/pdf/layout/combined_column_synthetic_test.go
index a81a0c1254..2db81a100c 100644
--- a/internal/deepdoc/parser/pdf/layout/combined_column_synthetic_test.go
+++ b/internal/deepdoc/parser/pdf/layout/combined_column_synthetic_test.go
@@ -87,23 +87,91 @@ func TestSyntheticSparseSecondColumn(t *testing.T) {
}
}
-// TODO #18079: a 2D spatial column detector should split this into 2.
+// TestSyntheticTitleBridgedDouble models a title-bridged double column: a
+// CLEAN vertical gutter plus a full-width title block at the very top that
+// bridges the gutter. The two body columns do NOT overlap in x0.
//
-// The page has a full-width title block at the top that bridges the gutter,
-// and below it two side-by-side columns where the right column is sparse and
-// its x0 overlaps the left column's x0 range. The x0-based balance gate
-// therefore rejects it (minority < 30%, x0 overlap) and the page is reported
-// as single. Skipped until #18079 lands; then unskip and assert got == 2.
+// The heading bridge is NOT a discriminant signal. What recovers the page is:
+// (1) dropFullWidth drops the full-width title, and (2) detectColumnCount2D
+// drops the still-wide bridging line at bridgingFrac*width, exposing the clean
+// gutter; an interior-valley scan then finds exactly one gutter -> 2 columns.
+//
+// Geometry here: title [50,450] at top (bridges gutter only at the top);
+// left body column [50,240] (30 lines); right body column [260,450] (12 lines)
+// — a clean 20-unit gutter at 240–260 with no x0 overlap.
+//
+// Right-column line count is set to 12 on purpose:
+// - body = 30 left + 12 right = 42; 12/42 = 0.286 < 0.30 (minModeFrac) so
+// the 1D balance gate correctly rejects it (minority too small), and
+// crossTol=0.15 collapses the gutter in 1D projection -> reports 1 until
+// the 2D rescue lands.
+// - The right column carries 12/30 = 0.40 of the page peak glyph ink, i.e.
+// above valleyFrac*peak (0.30), so it BOUNDS the gutter and the valley
+// scan recovers it -> 2 columns. NOTE: this is the real capability of the
+// rescue — it needs the minority column above ~30% of peak ink. A truly
+// sparse column (e.g. 6 lines = 0.20 of peak) merges with the gutter and
+// is NOT recovered; those fall back to the confidence-labeling track.
+// - 12 >= 0.12*42 = 5.04, so the both-sides prune gate (minColLineFrac)
+// accepts it. TestSyntheticSparseSecondColumn (right=3, 3 < 0.12*33) stays
+// 1, so the recover/stay-1 split is carried by right-column count.
func TestSyntheticTitleBridgedDouble(t *testing.T) {
- t.Skip("TODO #18079: title-bridged double should be detected as 2 columns")
boxes := concat(
concat(
- stackedColumn(50, 440, 3, 10, 10), // full-width title (bridges the gutter)
- stackedColumn(50, 240, 30, 40, 10), // left column
+ stackedColumn(50, 450, 3, 10, 10), // full-width title (bridges gutter only at top)
+ stackedColumn(50, 240, 30, 40, 10), // left body column [50,240]
),
- stackedColumn(200, 440, 4, 40, 20), // right column: sparse, x0 overlaps left
+ stackedColumn(260, 450, 12, 40, 20), // right body column [260,450]: 12 lines, clean gutter 240–260
)
if got := columnCount(boxes); got != 2 {
t.Errorf("title-bridged double: got %d, want 2", got)
}
}
+
+// TestSyntheticMedianWidthDouble locks the L3 signal (PR #10475's
+// page_w/median_w): a gutter-less double whose two columns are ADJACENT in x0
+// (no whitespace gutter), so there is no clean ink dip for gap/balance/2D-
+// rescue to find, yet each line is only ~half the page wide (raw_cols =
+// page_w/median_w = 2).
+//
+// Geometry: left body column [50,250] (30 lines), right body column [250,450]
+// (12 lines) — adjacent at x=250, so the x-projection is one continuous ink
+// band with no >= gapMinFrac run -> gap=1. Balance rejects it (right is
+// 12/42 = 0.286 < minModeFrac 0.30). The 2D rescue also fails (no clean
+// interior valley). Only the median-width ratio (raw_cols=2, a line spans 200
+// >= 0.45*400=180) recovers it -> 2 columns.
+//
+// This pins the #18079 acceptance target that the x0-based detectors alone
+// cannot reach: a real double with no geometric gutter.
+func TestSyntheticMedianWidthDouble(t *testing.T) {
+ boxes := concat(
+ stackedColumn(50, 250, 30, 10, 10), // left body column [50,250]
+ stackedColumn(250, 450, 12, 10, 20), // right body column [250,450]: adjacent, no gutter
+ )
+ if got := columnCount(boxes); got != 2 {
+ t.Errorf("median-width gutter-less double: got %d, want 2", got)
+ }
+}
+
+// TestSyntheticMedianWidthDoubleWithTitle locks the L3 median detector's prune
+// discipline: it must prune on the SAME non-full-width lines that produced the
+// centroids, not the full line set. A gutter-less double with a full-width
+// title exercises the path where, before the fix, the title (counted by
+// pruneColumns into the left column) inflated n and collapsed the sparse right
+// column to a single column. After the fix the detector recovers 2.
+//
+// Geometry: left [50,250] (29 lines), right [250,450] (4 lines) — adjacent,
+// no gutter; plus one full-width title [50,450]. Without the title the right
+// column is 4/33 = 0.121 >= minColLineFrac (0.12) -> 2; with the title counted
+// in, 4/34 = 0.118 < 0.12 -> 1 (the pre-fix bug).
+func TestSyntheticMedianWidthDoubleWithTitle(t *testing.T) {
+ boxes := concat(
+ concat(
+ stackedColumn(50, 250, 29, 10, 10), // left body column [50,250]
+ stackedColumn(250, 450, 4, 10, 20), // right body column [250,450]: adjacent, no gutter
+ ),
+ stackedColumn(50, 450, 1, 10, 10), // full-width title [50,450]
+ )
+ if got := columnCount(boxes); got != 2 {
+ t.Errorf("median-width gutter-less double with title: got %d, want 2", got)
+ }
+}