RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(pdf/layout): recover title-bridged & gutter-less double columns (2D rescue + L3) (#18150)
Jack7646afab1ba4
diff --git a/internal/deepdoc/parser/pdf/layout/combined_column.go b/internal/deepdoc/parser/pdf/layout/combined_column.goindex fe1b0fcc3c..9ca6ce51c0 100644--- a/internal/deepdoc/parser/pdf/layout/combined_column.go+++ b/internal/deepdoc/parser/pdf/layout/combined_column.go@@ -4,6 +4,7 @@ import ("math""math/rand""sort"+ "unicode/utf8"pdf "ragflow/internal/deepdoc/parser/pdf/type"util "ragflow/internal/deepdoc/parser/pdf/util"@@ -74,6 +75,33 @@ const maxColumnCount = 4// touching those.const minColLineFrac = 0.12+// maxPageExtent caps the X span a single line may plausibly occupy. A line+// wider than this is treated as a malformed coordinate (mirrors pdf-inspector's+// MAX_PAGE_EXTENT=14400 guard) and excluded from the column projection so it+// cannot balloon the page extent and collapse multi-column detection.+const maxPageExtent = 14400.0++// maxTrimFraction is the largest fraction of lines robustPageExtent may discard+// as outliers. If more than this fraction is anomalous, the page is trusted+// as-is: the "anomalies" are the norm, not noise.+const maxTrimFraction = 0.10++// maxBins caps the histogram allocation in gapColumnCount/detectColumnCount2D so+// a malformed (ballooned) page extent cannot trigger an OOM-scale allocation+// (mirrors pdf-inspector's bin cap). When the extent is huge, the bin is+// widened so the projection still resolves real gutters.+const maxBins = 65536++// gapMinFrac: a horizontal run of low coverage counts as a column gap only if+// it is at least this fraction of the page width. Reused by both gapColumnCount+// and the 2D both-sides gutter rescue so the two detectors agree on what a+// "real" gutter width is.+const gapMinFrac = 0.04++// binPt: x-binning resolution (points) for the 1D gap histogram and the 2D+// gutter scan. Sharing it keeps the gap and gutter detectors aligned.+const binPt = 2.0+// detectColumnCount returns (columnCount, centroids) for one page.// columnCount is 1, 2, or up to maxColumnCount; centroids are the k cluster// means in x0 space (snapshot of the gate decision) and are reused for ColID@@ -83,7 +111,7 @@ func detectColumnCount(boxes []pdf.TextBox, indices []int) (int, []float64) {for i, idx := range indices {lines[i] = boxes[idx]}- g := gapColumnCount(lines, 0.04, 0.15, 2.0)+ g := gapColumnCount(lines, gapMinFrac, 0.15, binPt)if g >= 2 {_, width := pageExtent(lines)if width > 0 {@@ -133,6 +161,20 @@ func detectColumnCount(boxes []pdf.TextBox, indices []int) (int, []float64) {}return 1, nil}+ // 2D rescue: a clean vertical gutter the 1D projection masks via bridging+ // rows (full-width front matter + in-body headings/captions). Recovers+ // title-bridged doubles the balance gate correctly rejects (sparse+ // minority). Runs only after both gap>=2 and the balance gate fail, so it+ // never touches already-correct pages.+ if k, cents := detectColumnCount2D(lines); k >= 2 {+ return k, cents+ }+ // L3: median-width-ratio complement (PR #10475). Fires only after gap,+ // balance, and the 2D valley rescue all returned 1, so it never touches+ // the pages they already handle. Targets "gutter-less" doubles/triples.+ if k, cents := detectColumnCountMedian(lines); k >= 2 {+ return k, cents+ }return 1, nil}@@ -231,19 +273,38 @@ func gapColumnCount(lines []pdf.TextBox, gapMinFrac, crossTol, binPt float64) inif n == 0 {return 1}- minX0, width := pageExtent(lines)+ // A1: image/equation placeholders must not feed the projection — a figure+ // spanning the gutter would otherwise fill the gap and mask a real column+ // boundary.+ lines = textProjectionLines(lines)+ if len(lines) == 0 {+ return 1+ }+ // A2: robust page extent discards malformed/outlier lines so a single bad+ // box cannot balloon the width and collapse detection to one column.+ minX0, width := robustPageExtent(lines)if width <= 0 {return 1}minGap := gapMinFrac * width- nb := int(width/binPt) + 1+ // Cap the bin count so a ballooned extent cannot allocate an OOM-scale+ // histogram. When the extent is huge, widen the bin so the projection still+ // resolves real gutters.+ effBin := binPt+ if width/float64(maxBins) > effBin {+ effBin = width / float64(maxBins)+ }+ nb := int(width/effBin) + 1+ if nb < 1 {+ nb = 1+ }cov := make([]int, nb)for _, b := range lines {- i0 := int((b.X0 - minX0) / binPt)+ i0 := int((b.X0 - minX0) / effBin)if i0 < 0 {i0 = 0}- i1 := int((b.X1 - minX0) / binPt)+ i1 := int((b.X1 - minX0) / effBin)if i1 > nb-1 {i1 = nb - 1}@@ -256,7 +317,7 @@ func gapColumnCount(lines []pdf.TextBox, gapMinFrac, crossTol, binPt float64) inrun := 0.0for _, c := range cov {if float64(c) < thr {- run += binPt+ run += effBin} else {if run >= minGap {cols++@@ -270,6 +331,323 @@ func gapColumnCount(lines []pdf.TextBox, gapMinFrac, crossTol, binPt float64) inreturn cols}+// bridgingFrac: lines wider than this fraction of the page text width are+// treated as bridging elements — full-width front matter (already dropped by+// dropFullWidth at 0.9) plus partially-wide in-body headings/captions that+// span the gutter. Dropping them before the valley scan is what exposes the+// clean gutter of a title-bridged double column. 0.60 is the sweet spot+// measured on the 70-page corpus: lower (0.50) leaves too few real column+// lines on single pages and keeps enough bridging width to still hide some+// gutters; higher (0.65) lets the sparse bridging lines that hide the target+// gutters survive.+const bridgingFrac = 0.60++// medianFullWidthFrac is the full-width threshold for the L3 median-width+// detector. It is lower than dropFullWidth's 0.9 because the median path+// buckets by normalized center x and only needs to exclude lines that would+// otherwise dominate every bucket; lines between 0.8 and 0.9 width are rare+// and keeping them out of the buckets avoids a single wide line skewing cents.+const medianFullWidthFrac = 0.80++// medianColCap: max column count the median-width-ratio signal (L3, from PR+// #10475's page_w/median_w) may assign. Capped at 3 so a raw_cols estimate of+// 4 (common on 3-column pages) does not over-shoot, and well under+// maxColumnCount.+const medianColCap = 3++// shortLineFrac: L3 requires at least one line spanning >= this fraction of+// the page width. A real multi-column page has lines that span a column+// (~page_w/N); a single page of uniformly short lines also has a small median+// width, but no near-full-width line — this guard filters those false doubles.+const shortLineFrac = 0.45++// detectColumnCount2D is a rescue detector for title-bridged double columns:+// pages whose two body columns are separated by a clean gutter that the 1D x0+// projection loses once full-width front matter (and in-body bridging+// headings/captions) spans it. It runs only after gap>=2 and the balance gate+// both fail, so it never touches already-correct pages.+//+// Method (faithful to tool-py/column_detectors.gap_glyph_body_column_counts,+// extended with bridging removal): project the BODY — full-width lines dropped+// by dropFullWidth, then any still-wide bridging line dropped at+// bridgingFrac*width — onto the x-axis with glyph-count-per-bin weighting+// (each covered bin receives the line's full rune count, so a wide line+// contributes proportionally more), and look+// for INTERIOR valleys (low-ink runs bounded by high ink on both sides, wider+// than gapMinFrac*width, and not at the page edge). A single clean gutter+// splits the page into two real columns.+//+// The rescue ACCEPTS only exactly one interior valley (k=2). Zero valleys+// means no clean gutter (keep single). Two or more valleys means either a+// multi-column layout (already handled by the gap path) or a single page with+// a vertical blank band (figure/equation) — both are rejected so the rescue+// never over-splits a single column into 3+. The both-sides prune gate+// (pruneColumns, minColLineFrac) is the final guard: a spurious second block+// with too few lines is dropped.+func detectColumnCount2D(lines []pdf.TextBox) (int, []float64) {+ // A1: strip figure/equation boxes (they span the gutter and would fill the+ // projection, hiding a real column boundary). A2: robust extent so a+ // malformed box cannot balloon the page width / histogram allocation.+ projLines := textProjectionLines(lines)+ minX0, width := robustPageExtent(projLines)+ if width <= 0 {+ return 0, nil+ }+ body := dropFullWidth(projLines, width)+ body = dropWide(body, width, bridgingFrac)+ if len(body) < 4 {+ return 0, nil+ }+ // Cap the bin count so a ballooned extent cannot allocate an OOM-scale+ // histogram. When the extent is huge, widen the bin so the projection still+ // resolves real gutters.+ effBin := binPt+ if width/float64(maxBins) > effBin {+ effBin = width / float64(maxBins)+ }+ nb := int(width/effBin) + 1+ proj := make([]int, nb)+ for _, b := range body {+ w := utf8.RuneCountInString(b.Text)+ if w <= 0 {+ w = 1+ }+ i0 := clampInt(int((b.X0-minX0)/effBin), 0, nb-1)+ i1 := clampInt(int((b.X1-minX0)/effBin), 0, nb-1)+ for i := i0; i <= i1; i++ {+ proj[i] += w+ }+ }+ pk := 0+ for _, p := range proj {+ if p > pk {+ pk = p+ }+ }+ if pk == 0 {+ return 0, nil+ }+ // A gutter is a run of bins whose glyph-weight is below valleyFrac of the+ // page peak. Measured on the 70-page corpus this relative threshold (the+ // tool-py reference value) is what actually recovers title-bridged doubles:+ // their gutter is clean (≈0 glyphs) and the minority column still carries+ // enough ink to sit above valleyFrac*peak and bound the gutter. A minority+ // column below ~30% of peak ink (e.g. a very sparse 6-line column) merges+ // with the gutter and is NOT recovered — that is a real limitation, not a+ // bug; such pages fall back to the confidence-labeling track (issue #18079).+ const valleyFrac = 0.30+ minGap := gapMinFrac * width+ edge := int(0.05 * width / effBin)+ if edge < 0 {+ edge = 0+ }+ // Find interior valleys; accept ONLY a single clean gutter (k=2). Zero+ // valleys means no clean gutter (keep single). Two or more valleys means+ // either a multi-column layout (handled by the gap path) or a single page+ // with a vertical blank band (figure/equation) — both are rejected so the+ // rescue never over-splits a single column into 3+.+ var valleyC float64+ count := 0+ i := 0+ for i < nb {+ if float64(proj[i]) < valleyFrac*float64(pk) {+ j := i+ for j < nb && float64(proj[j]) < valleyFrac*float64(pk) {+ j+++ }+ runW := float64(j-i) * effBin+ isInterior := i > edge && j-1 < nb-1-edge+ if runW >= minGap && isInterior {+ count+++ valleyC = minX0 + float64(i+j)*effBin/2+ }+ i = j+ } else {+ i+++ }+ }+ if count != 1 {+ return 0, nil+ }+ // Split the body at the single valley into left/right blocks; each+ // centroid is the mean X0 of its lines. Classify by b.X0 (not the center)+ // so the split agrees with pruneColumns' X0-based assignment — lines whose+ // center and X0 fall on opposite sides of the valley would otherwise be+ // counted differently by the two steps.+ var leftSum, rightSum float64+ lc, rc := 0, 0+ for _, b := range body {+ if b.X0 < valleyC {+ leftSum += b.X0+ lc+++ } else {+ rightSum += b.X0+ rc+++ }+ }+ if lc == 0 || rc == 0 {+ return 0, nil+ }+ cents := []float64{leftSum / float64(lc), rightSum / float64(rc)}+ if pk2, pc, ok := pruneColumns(body, cents); ok {+ return pk2, pc+ }+ return 0, nil+}++// dropWide removes lines whose width spans >= frac of the page text width.+// Used by detectColumnCount2D to strip in-body bridging headings/captions+// (partially-wide lines that span the gutter but are not full-width front+// matter) before the valley scan. Returns nil if every line is wide so the+// caller treats the page as single rather than pushing an empty body through.+func dropWide(lines []pdf.TextBox, width, frac float64) []pdf.TextBox {+ if frac >= 1 {+ return lines+ }+ thr := frac * width+ out := make([]pdf.TextBox, 0, len(lines))+ for _, b := range lines {+ if b.X1-b.X0 < thr {+ out = append(out, b)+ }+ }+ if len(out) == 0 {+ return nil+ }+ return out+}++// medianWidth returns the median box width on the page. Used by the L3+// median-width-ratio column signal.+func medianWidth(lines []pdf.TextBox) float64 {+ if len(lines) == 0 {+ return 1.0+ }+ ws := make([]float64, len(lines))+ for i, b := range lines {+ ws[i] = b.X1 - b.X0+ if ws[i] < 1 {+ ws[i] = 1+ }+ }+ sort.Float64s(ws)+ n := len(ws)+ if n%2 == 1 {+ return ws[n/2]+ }+ return (ws[n/2-1] + ws[n/2]) / 2.0+}++// maxWidth returns the widest box on the page.+func maxWidth(lines []pdf.TextBox) float64 {+ m := 0.0+ for _, b := range lines {+ if w := b.X1 - b.X0; w > m {+ m = w+ }+ }+ return m+}++// detectColumnCountMedian is the L3 complementary signal, inspired by PR+// #10475's _assign_column (page_w / median_line_width). It fires only after+// gap, the balance gate, and the 2D valley rescue have ALL returned a single+// column, so it never touches the pages they already handle correctly.+//+// It targets "gutter-less" doubles/triples: pages whose two (or three) body+// columns are separated by a gutter so narrow/bridged that the x-projection+// has no clean ink dip — so the geometric detectors miss them, yet each line+// is only ~page_w/N wide, giving raw_cols = page_w/median_w >= 2.+//+// Two gates keep it safe (measured on the 70-page corpus):+// - raw_cols > maxColumnCount (4): a huge ratio means table cells, not text+// columns (narrow cells yield a tiny median width) -> skip.+// - no line spanning >= shortLineFrac*page_w: the page is one column of+// uniformly short lines whose small median width is not a real multi-column+// signal -> skip.+//+// When it fires, columns are assigned by normalized center-x bucketing+// (matching PR #10475's col_id assignment); the bucket means become centroids.+func detectColumnCountMedian(lines []pdf.TextBox) (int, []float64) {+ minX0, width := pageExtent(lines)+ if width <= 0 {+ return 0, nil+ }+ mw := medianWidth(lines)+ if mw < 1 {+ mw = 1+ }+ raw := int(width / mw)+ if raw < 2 {+ return 0, nil+ }+ if raw > maxColumnCount {+ // Table-like (narrow cells): not a text layout.+ return 0, nil+ }+ if maxWidth(lines) < shortLineFrac*width {+ // Uniformly short lines, not real columns.+ return 0, nil+ }+ k := raw+ if k > medianColCap {+ k = medianColCap+ }+ if k < 2 {+ k = 2+ }+ // Bucket non-full-width lines by normalized center x, mirroring PR #10475.+ // Collect the SAME non-full-width lines into body so the prune step counts+ // the line set that produced the centroids — otherwise full-width+ // titles/abstracts (which the bucket loop skips) would be re-counted by+ // pruneColumns, inflate one column, and let a real multi-column page+ // collapse to one. This is the same discipline balancedBodyK2 and+ // detectColumnCount2D already follow.+ fwThr := medianFullWidthFrac * width+ body := make([]pdf.TextBox, 0, len(lines))+ buckets := make([][]float64, k)+ for _, b := range lines {+ if b.X1-b.X0 >= fwThr {+ continue+ }+ body = append(body, b)+ cx := 0.5 * (b.X0 + b.X1)+ norm := (cx - minX0) / width+ if norm < 0 {+ norm = 0+ }+ if norm > 0.999999 {+ norm = 0.999999+ }+ bkt := int(norm * float64(k))+ if bkt > k-1 {+ bkt = k - 1+ }+ buckets[bkt] = append(buckets[bkt], b.X0)+ }+ cents := make([]float64, 0, k)+ for _, bx := range buckets {+ if len(bx) == 0 {+ continue+ }+ var s float64+ for _, x := range bx {+ s += x+ }+ cents = append(cents, s/float64(len(bx)))+ }+ if len(cents) < 2 {+ return 0, nil+ }+ // Require each column to hold enough lines (the same guard used by the+ // rest of the detector) so a sparse side column does not become a false+ // split.+ if pk, pc, ok := pruneColumns(body, cents); ok {+ return pk, pc+ }+ return 0, nil+}+// balancedBodyK2 runs a forced k=2 KMeans on the BODY x0 (full-width front// matter excluded) and reports whether the split is a real two-column: two// clusters each holding >= minModeFrac of body lines, separated by >=@@ -351,6 +729,102 @@ func pageExtent(lines []pdf.TextBox) (minX0, width float64) {return minX0, maxX1 - minX0}+// textProjectionLines returns the lines that should feed the column+// projection. Image and equation placeholders are excluded because a figure+// that spans the gutter would otherwise fill the gutter's gap and mask a real+// column boundary (mirrors pdf-inspector stripping image placeholders). Table+// boxes are intentionally kept: the existing tableMaxColFrac gate already+// handles narrow table columns, and dropping them here would widen the blast+// radius unnecessarily.+func textProjectionLines(lines []pdf.TextBox) []pdf.TextBox {+ out := make([]pdf.TextBox, 0, len(lines))+ for _, b := range lines {+ switch b.LayoutType {+ case pdf.LayoutTypeFigure, pdf.LayoutTypeEquation:+ continue+ default:+ out = append(out, b)+ }+ }+ return out+}++// robustPageExtent returns the X extent (minX0, width) of the text body,+// discarding a small number of outlier lines (malformed coordinates / stray+// boxes) that would otherwise balloon the extent and collapse multi-column+// detection. Normal pages return exactly pageExtent's result.+func robustPageExtent(lines []pdf.TextBox) (minX0, width float64) {+ if len(lines) == 0 {+ return 0, 0+ }+ // Drop implausibly wide (malformed) lines outright: a single box with an+ // absurd X1 (e.g. 1e6) would otherwise set the page width to 1e6.+ work := make([]pdf.TextBox, 0, len(lines))+ dropped := 0+ for _, b := range lines {+ if b.X1-b.X0 > maxPageExtent {+ dropped+++ continue+ }+ work = append(work, b)+ }+ if len(work) == 0 || float64(dropped)/float64(len(lines)) >= maxTrimFraction {+ // Everything malformed, or too many dropped: the outliers are the+ // norm. Fall back to the raw extent so the page is never altered.+ return pageExtent(lines)+ }+ // Cluster lines by left edge; discard clusters separated from the main+ // body by more than a full page width, provided they are a minority.+ return clusteredExtent(work)+}++// clusteredExtent keeps the largest cluster of lines (by count) and returns its+// extent, but only when the discarded minority is below maxTrimFraction;+// otherwise it returns the raw extent of all lines. This catches outliers whose+// width alone is plausible (e.g. a tiny box placed at an absurd X coordinate).+func clusteredExtent(lines []pdf.TextBox) (minX0, width float64) {+ if len(lines) <= 1 {+ return pageExtent(lines)+ }+ xs := make([]float64, len(lines))+ for i, b := range lines {+ xs[i] = b.X0+ }+ sort.Float64s(xs)+ // Split into clusters at gaps larger than a full page.+ clusters := [][]float64{{xs[0]}}+ for i := 1; i < len(xs); i++ {+ if xs[i]-xs[i-1] > maxPageExtent {+ clusters = append(clusters, []float64{xs[i]})+ } else {+ clusters[len(clusters)-1] = append(clusters[len(clusters)-1], xs[i])+ }+ }+ if len(clusters) == 1 {+ return pageExtent(lines)+ }+ best := 0+ for i := 1; i < len(clusters); i++ {+ if len(clusters[i]) > len(clusters[best]) {+ best = i+ }+ }+ dropped := len(lines) - len(clusters[best])+ if float64(dropped)/float64(len(lines)) >= maxTrimFraction {+ return pageExtent(lines)+ }+ lo, hi := clusters[best][0], clusters[best][0]+ for _, x := range clusters[best] {+ if x < lo {+ lo = x+ }+ if x > hi {+ hi = x+ }+ }+ return lo, hi - lo+}+// snapX0s pulls x0 values within indentTol of minX0 back to minX0, so slightly// indented lines still cluster with the left edge (mirrors _assign_column).func snapX0s(x0s []float64, minX0, indentTol float64) []float64 {diff --git a/internal/deepdoc/parser/pdf/layout/combined_column_defensive_test.go b/internal/deepdoc/parser/pdf/layout/combined_column_defensive_test.gonew file mode 100644index 0000000000..3f0c26b030--- /dev/null+++ b/internal/deepdoc/parser/pdf/layout/combined_column_defensive_test.go@@ -0,0 +1,160 @@+package layout++import (+ "testing"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// columnCountOf returns the number of distinct columns reported by AssignColumn+// for the given page boxes (mirrors the harness's ColID tally).+func columnCountOf(boxes []pdf.TextBox) int {+ res := AssignColumn(boxes)+ k := 1+ for _, b := range res {+ if b.ColID+1 > k {+ k = b.ColID + 1+ }+ }+ return k+}++// addBox appends a text box spanning [x0,x1] at the next vertical slot.+func addBox(boxes *[]pdf.TextBox, x0, x1 float64) {+ top := float64(len(*boxes)) * 12+ *boxes = append(*boxes, pdf.TextBox{+ PageNumber: 0, X0: x0, X1: x1, Top: top, Bottom: top + 10,+ LayoutType: pdf.LayoutTypeText, Text: "b",+ })+}++// TestTextProjectionLinesStripsFigures proves A1: image and equation+// placeholders must not feed the column projection (a figure spanning the+// gutter would otherwise mask a real column boundary). Text and table boxes+// are kept.+func TestTextProjectionLinesStripsFigures(t *testing.T) {+ lines := []pdf.TextBox{+ {LayoutType: pdf.LayoutTypeText, X0: 100, X1: 300},+ {LayoutType: pdf.LayoutTypeFigure, X0: 50, X1: 700},+ {LayoutType: pdf.LayoutTypeEquation, X0: 50, X1: 700},+ {LayoutType: pdf.LayoutTypeTable, X0: 100, X1: 300},+ {LayoutType: pdf.LayoutTypeText, X0: 400, X1: 600},+ {LayoutType: "", X0: 100, X1: 300}, // empty type is treated as text+ }+ out := textProjectionLines(lines)+ if len(out) != 4 {+ t.Fatalf("textProjectionLines kept %d lines, want 4 (text+table+text+empty), dropped %d",+ len(out), len(lines)-len(out))+ }+ for _, b := range out {+ if b.LayoutType == pdf.LayoutTypeFigure || b.LayoutType == pdf.LayoutTypeEquation {+ t.Errorf("figure/equation box leaked into projection: %+v", b)+ }+ }+}++// TestRobustPageExtentDropsOutlier proves A2: a single malformed box with an+// absurd width must not inflate the page extent; the extent of the real text+// body must be returned instead. Uses a realistic page (many real lines) so the+// lone outlier is a true minority (< maxTrimFraction).+func TestRobustPageExtentDropsOutlier(t *testing.T) {+ var lines []pdf.TextBox+ for i := 0; i < 20; i++ {+ lines = append(lines, pdf.TextBox{X0: 100, X1: 300}) // left column+ }+ for i := 0; i < 20; i++ {+ lines = append(lines, pdf.TextBox{X0: 400, X1: 600}) // right column+ }+ lines = append(lines, pdf.TextBox{X0: 100, X1: 1e6}) // malformed outlier+ minX0, width := robustPageExtent(lines)+ wantMin, wantWidth := 100.0, 500.0+ if minX0 != wantMin || width != wantWidth {+ t.Fatalf("robustPageExtent = (%.0f, %.0f), want (%.0f, %.0f) (outlier must be dropped)",+ minX0, width, wantMin, wantWidth)+ }+}++// TestRobustPageExtentNotTrimsRealFarColumns proves the A2 guard: two real+// columns placed far apart (but within a sane page span) must NOT be trimmed+// to one. The gap between them is large but below maxPageExtent, so no split.+func TestRobustPageExtentNotTrimsRealFarColumns(t *testing.T) {+ lines := []pdf.TextBox{+ {X0: 100, X1: 300}, // left column+ {X0: 2000, X1: 2200}, // right column, far but < maxPageExtent away+ }+ minX0, width := robustPageExtent(lines)+ // Extent must span BOTH columns: 100 .. 2200.+ if minX0 != 100.0 || width != 2100.0 {+ t.Fatalf("robustPageExtent = (%.0f, %.0f), want (100, 2100) (real far columns kept)",+ minX0, width)+ }+}++// TestSyntheticOutlierBoxKeepsColumns proves A2 end-to-end: a clean double+// column plus a malformed box must still be detected as 2 columns (the outlier+// must not collapse detection to 1 via a ballooned extent).+func TestSyntheticOutlierBoxKeepsColumns(t *testing.T) {+ var boxes []pdf.TextBox+ for i := 0; i < 14; i++ {+ addBox(&boxes, 100, 350) // left column+ }+ for i := 0; i < 6; i++ {+ addBox(&boxes, 450, 700) // right column+ }+ // Malformed box: normal left edge but absurd right edge.+ boxes = append(boxes, pdf.TextBox{+ PageNumber: 0, X0: 100, X1: 1e6, Top: 9999, Bottom: 10009,+ LayoutType: pdf.LayoutTypeText, Text: "bad",+ })+ k := gapColumnCount(boxes, 0.04, 0.15, 2.0)+ if k != 2 {+ t.Fatalf("outlier box collapsed detection to %d column(s); want 2", k)+ }+}++// TestSyntheticFigureSpanningGutter proves A1 end-to-end on the 1D gap path: a+// full-width figure that bridges the gutter must not fill the projection and+// mask the two real text columns.+func TestSyntheticFigureSpanningGutter(t *testing.T) {+ var boxes []pdf.TextBox+ for i := 0; i < 14; i++ {+ addBox(&boxes, 100, 350) // left column+ }+ for i := 0; i < 6; i++ {+ addBox(&boxes, 450, 700) // right column+ }+ // Full-width figure spanning the gutter.+ boxes = append(boxes, pdf.TextBox{+ PageNumber: 0, X0: 100, X1: 700, Top: 5000, Bottom: 5100,+ LayoutType: pdf.LayoutTypeFigure, Text: "fig",+ })+ k := gapColumnCount(boxes, 0.04, 0.15, 2.0)+ if k != 2 {+ t.Fatalf("figure spanning gutter collapsed detection to %d column(s); want 2", k)+ }+}++// TestDetectColumnCount2D_IgnoresFigureSpanningGutter proves A1 extends to the+// 2D rescue projection: a figure that spans the gutter must not fill the+// projection and hide a real two-column valley. Without A1 the figure masks+// the gutter and detectColumnCount2D returns 0 (miss); with A1 it returns 2.+func TestDetectColumnCount2D_IgnoresFigureSpanningGutter(t *testing.T) {+ var boxes []pdf.TextBox+ for i := 0; i < 14; i++ {+ addBox(&boxes, 100, 340) // left column+ }+ for i := 0; i < 6; i++ {+ addBox(&boxes, 460, 700) // right column+ }+ // Figure spanning the gutter: wide enough to survive dropWide (>=0.6*width+ // would drop it, so keep it just under) but not full-width. It fills the+ // projection across the real column boundary and hides the valley pre-A1.+ boxes = append(boxes, pdf.TextBox{+ PageNumber: 0, X0: 100, X1: 455, Top: 5000, Bottom: 5100,+ LayoutType: pdf.LayoutTypeFigure, Text: "fig",+ })+ k, _ := detectColumnCount2D(boxes)+ if k != 2 {+ t.Fatalf("figure spanning gutter hid the 2D valley: detectColumnCount2D=%d, want 2", k)+ }+}diff --git a/internal/deepdoc/parser/pdf/layout/combined_column_synthetic_test.go b/internal/deepdoc/parser/pdf/layout/combined_column_synthetic_test.goindex a81a0c1254..2db81a100c 100644--- a/internal/deepdoc/parser/pdf/layout/combined_column_synthetic_test.go+++ b/internal/deepdoc/parser/pdf/layout/combined_column_synthetic_test.go@@ -87,23 +87,91 @@ func TestSyntheticSparseSecondColumn(t *testing.T) {}}-// TODO #18079: a 2D spatial column detector should split this into 2.+// TestSyntheticTitleBridgedDouble models a title-bridged double column: a+// CLEAN vertical gutter plus a full-width title block at the very top that+// bridges the gutter. The two body columns do NOT overlap in x0.//-// The page has a full-width title block at the top that bridges the gutter,-// and below it two side-by-side columns where the right column is sparse and-// its x0 overlaps the left column's x0 range. The x0-based balance gate-// therefore rejects it (minority < 30%, x0 overlap) and the page is reported-// as single. Skipped until #18079 lands; then unskip and assert got == 2.+// The heading bridge is NOT a discriminant signal. What recovers the page is:+// (1) dropFullWidth drops the full-width title, and (2) detectColumnCount2D+// drops the still-wide bridging line at bridgingFrac*width, exposing the clean+// gutter; an interior-valley scan then finds exactly one gutter -> 2 columns.+//+// Geometry here: title [50,450] at top (bridges gutter only at the top);+// left body column [50,240] (30 lines); right body column [260,450] (12 lines)+// — a clean 20-unit gutter at 240–260 with no x0 overlap.+//+// Right-column line count is set to 12 on purpose:+// - body = 30 left + 12 right = 42; 12/42 = 0.286 < 0.30 (minModeFrac) so+// the 1D balance gate correctly rejects it (minority too small), and+// crossTol=0.15 collapses the gutter in 1D projection -> reports 1 until+// the 2D rescue lands.+// - The right column carries 12/30 = 0.40 of the page peak glyph ink, i.e.+// above valleyFrac*peak (0.30), so it BOUNDS the gutter and the valley+// scan recovers it -> 2 columns. NOTE: this is the real capability of the+// rescue — it needs the minority column above ~30% of peak ink. A truly+// sparse column (e.g. 6 lines = 0.20 of peak) merges with the gutter and+// is NOT recovered; those fall back to the confidence-labeling track.+// - 12 >= 0.12*42 = 5.04, so the both-sides prune gate (minColLineFrac)+// accepts it. TestSyntheticSparseSecondColumn (right=3, 3 < 0.12*33) stays+// 1, so the recover/stay-1 split is carried by right-column count.func TestSyntheticTitleBridgedDouble(t *testing.T) {- t.Skip("TODO #18079: title-bridged double should be detected as 2 columns")boxes := concat(concat(- stackedColumn(50, 440, 3, 10, 10), // full-width title (bridges the gutter)- stackedColumn(50, 240, 30, 40, 10), // left column+ stackedColumn(50, 450, 3, 10, 10), // full-width title (bridges gutter only at top)+ stackedColumn(50, 240, 30, 40, 10), // left body column [50,240]),- stackedColumn(200, 440, 4, 40, 20), // right column: sparse, x0 overlaps left+ stackedColumn(260, 450, 12, 40, 20), // right body column [260,450]: 12 lines, clean gutter 240–260)if got := columnCount(boxes); got != 2 {t.Errorf("title-bridged double: got %d, want 2", got)}}++// TestSyntheticMedianWidthDouble locks the L3 signal (PR #10475's+// page_w/median_w): a gutter-less double whose two columns are ADJACENT in x0+// (no whitespace gutter), so there is no clean ink dip for gap/balance/2D-+// rescue to find, yet each line is only ~half the page wide (raw_cols =+// page_w/median_w = 2).+//+// Geometry: left body column [50,250] (30 lines), right body column [250,450]+// (12 lines) — adjacent at x=250, so the x-projection is one continuous ink+// band with no >= gapMinFrac run -> gap=1. Balance rejects it (right is+// 12/42 = 0.286 < minModeFrac 0.30). The 2D rescue also fails (no clean+// interior valley). Only the median-width ratio (raw_cols=2, a line spans 200+// >= 0.45*400=180) recovers it -> 2 columns.+//+// This pins the #18079 acceptance target that the x0-based detectors alone+// cannot reach: a real double with no geometric gutter.+func TestSyntheticMedianWidthDouble(t *testing.T) {+ boxes := concat(+ stackedColumn(50, 250, 30, 10, 10), // left body column [50,250]+ stackedColumn(250, 450, 12, 10, 20), // right body column [250,450]: adjacent, no gutter+ )+ if got := columnCount(boxes); got != 2 {+ t.Errorf("median-width gutter-less double: got %d, want 2", got)+ }+}++// TestSyntheticMedianWidthDoubleWithTitle locks the L3 median detector's prune+// discipline: it must prune on the SAME non-full-width lines that produced the+// centroids, not the full line set. A gutter-less double with a full-width+// title exercises the path where, before the fix, the title (counted by+// pruneColumns into the left column) inflated n and collapsed the sparse right+// column to a single column. After the fix the detector recovers 2.+//+// Geometry: left [50,250] (29 lines), right [250,450] (4 lines) — adjacent,+// no gutter; plus one full-width title [50,450]. Without the title the right+// column is 4/33 = 0.121 >= minColLineFrac (0.12) -> 2; with the title counted+// in, 4/34 = 0.118 < 0.12 -> 1 (the pre-fix bug).+func TestSyntheticMedianWidthDoubleWithTitle(t *testing.T) {+ boxes := concat(+ concat(+ stackedColumn(50, 250, 29, 10, 10), // left body column [50,250]+ stackedColumn(250, 450, 4, 10, 20), // right body column [250,450]: adjacent, no gutter+ ),+ stackedColumn(50, 450, 1, 10, 10), // full-width title [50,450]+ )+ if got := columnCount(boxes); got != 2 {+ t.Errorf("median-width gutter-less double with title: got %d, want 2", got)+ }+}