Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(pdf): align DLA region handling with Python reference (#18295)

Jack272645a27aba
diff --git a/internal/deepdoc/parser/pdf/batch_smoke_test.go b/internal/deepdoc/parser/pdf/batch_smoke_test.go
index 6b8678d7bc..e25e6f99d5 100644
--- a/internal/deepdoc/parser/pdf/batch_smoke_test.go
+++ b/internal/deepdoc/parser/pdf/batch_smoke_test.go
@@ -21,6 +21,7 @@ import (
inf "ragflow/internal/deepdoc/parser/pdf/inference"
lyt "ragflow/internal/deepdoc/parser/pdf/layout"
+ "ragflow/internal/deepdoc/parser/pdf/table"
"ragflow/internal/deepdoc/parser/pdf/tool"
pdf "ragflow/internal/deepdoc/parser/pdf/type"
)
@@ -91,7 +92,7 @@ func variantFromEnv() string {
}
type outputDirs struct {
- text, tables, dla string
+ text, tables, dla, tsr string
}
func mkOutputDirs(variant string) outputDirs {
@@ -99,10 +100,12 @@ func mkOutputDirs(variant string) outputDirs {
text: filepath.Join("testdata", "output", "go", variant, "text"),
tables: filepath.Join("testdata", "output", "go", variant, "tables"),
dla: filepath.Join("testdata", "output", "go", variant, "dla"),
+ tsr: filepath.Join("testdata", "output", "go", variant, "tsr_raw"),
}
os.MkdirAll(d.text, 0755)
os.MkdirAll(d.tables, 0755)
os.MkdirAll(d.dla, 0755)
+ os.MkdirAll(d.tsr, 0755)
return d
}
@@ -344,9 +347,68 @@ func writeOutputs(dirs outputDirs, name string, parsed *pdf.ParseResult, res *pa
}
// ── DLA layout intermediates ──
+ // DLA dump: post-filter regions (NMS + confidence filter + Y-sort +
+ // cleanup), matching Python's page_layout. This makes the Go dump
+ // comparable with the Python parity dump, which also writes post-filter
+ // regions.
+ //
+ // Coordinate space: Go dumps image-pixel coordinates (no scale division;
+ // see FilteredDLARegions), whereas Python's page_layout divides by
+ // scale_factor (typically 3, layout_recognizer.py:90-93). So the two
+ // dumps differ by ~3x in absolute coordinates — this is expected, and the
+ // parity comparison is count-only (CompareDLAWithPython), so it does not
+ // affect the match.
+ //
+ // Real DLA regions always carry a score, so cleanupLayouts' box-fallback
+ // never triggers and nil boxes are safe here (count-wise). If a score-0
+ // region were ever emitted, the area tie-break would fall back to keeping
+ // the first region rather than mirroring Python's area-based choice.
if parsed.DLARegions != nil {
- if b, _ := json.MarshalIndent(parsed.DLARegions, "", " "); b != nil {
+ filteredPages := make([]pdf.DLAPageRegions, 0, len(parsed.DLARegions))
+ for _, pr := range parsed.DLARegions {
+ filteredPages = append(filteredPages, pdf.DLAPageRegions{
+ Page: pr.Page,
+ Regions: table.FilteredDLARegions(pr.Regions, nil),
+ })
+ }
+ if b, _ := json.MarshalIndent(filteredPages, "", " "); b != nil {
os.WriteFile(filepath.Join(dirs.dla, name+".json"), b, 0644)
}
}
+
+ // ── TSR raw cells ── (matching Python's tsr_raw dump from parser.tb_cpns).
+ // Flat list of per-cell records so CompareTSRRawWithPython can diff labels
+ // and table counts against the Python reference.
+ type tsrRawCellDump struct {
+ TableIndex int `json:"table_index"`
+ Page int `json:"page"`
+ Label string `json:"label"`
+ X0 float64 `json:"x0"`
+ Y0 float64 `json:"y0"`
+ X1 float64 `json:"x1"`
+ Y1 float64 `json:"y1"`
+ Text string `json:"text"`
+ }
+ var tsrCells []tsrRawCellDump
+ for ti, t := range parsed.Tables {
+ page := 0
+ if len(t.Positions) > 0 && len(t.Positions[0].PageNumbers) > 0 {
+ page = t.Positions[0].PageNumbers[0]
+ }
+ for _, c := range t.Cells {
+ tsrCells = append(tsrCells, tsrRawCellDump{
+ TableIndex: ti,
+ Page: page,
+ Label: c.Label,
+ X0: c.X0,
+ Y0: c.Y0,
+ X1: c.X1,
+ Y1: c.Y1,
+ Text: c.Text,
+ })
+ }
+ }
+ if b, _ := json.MarshalIndent(tsrCells, "", " "); b != nil {
+ os.WriteFile(filepath.Join(dirs.tsr, name+".json"), b, 0644)
+ }
}
diff --git a/internal/deepdoc/parser/pdf/table/table_annotate.go b/internal/deepdoc/parser/pdf/table/table_annotate.go
index 87d3719cd6..7335e61ca0 100644
--- a/internal/deepdoc/parser/pdf/table/table_annotate.go
+++ b/internal/deepdoc/parser/pdf/table/table_annotate.go
@@ -3,6 +3,7 @@ package table
import (
"fmt"
"math"
+ "sort"
pdf "ragflow/internal/deepdoc/parser/pdf/type"
"ragflow/internal/deepdoc/parser/pdf/util"
@@ -57,63 +58,300 @@ func MatchTableRegions(boxes []pdf.TextBox, regions []pdf.DLARegion, scale float
// ── layout annotation ──────────────────────────────────────────────────
-// annotateBoxLayouts sets LayoutType and LayoutNo on each box, matching
+// annRegion is a layout region in PDF space, used internally by
+// AnnotateBoxLayouts. It carries the fields needed for cleanup, sort, and
+// annotation bookkeeping.
+type annRegion struct {
+ x0, y0, x1, y1 float64
+ label string
+ score float64
+ visited bool
+ typeIndex int
+}
+
+// regionIntersect returns the intersection area of two regions, or 0.
+func regionIntersect(a, b annRegion) float64 {
+ ix0 := math.Max(a.x0, b.x0)
+ iy0 := math.Max(a.y0, b.y0)
+ ix1 := math.Min(a.x1, b.x1)
+ iy1 := math.Min(a.y1, b.y1)
+ if ix0 < ix1 && iy0 < iy1 {
+ return (ix1 - ix0) * (iy1 - iy0)
+ }
+ return 0
+}
+
+// regionArea returns the area of a region, or 0 if degenerate.
+func regionArea(a annRegion) float64 {
+ w := a.x1 - a.x0
+ h := a.y1 - a.y0
+ if w <= 0 || h <= 0 {
+ return 0
+ }
+ return w * h
+}
+
+// overlapRatio returns intersection / area(a), matching
+// Recognizer.overlapped_area with ratio=True (recognizer.py:106-122).
+func overlapRatio(a, b annRegion) float64 {
+ ar := regionArea(a)
+ if ar <= 0 {
+ return 0
+ }
+ return regionIntersect(a, b) / ar
+}
+
+// annNotOverlapped mirrors recognizer.py:126-127 (annRegion variant).
+func annNotOverlapped(a, b annRegion) bool {
+ return a.x1 < b.x0 || a.x0 > b.x1 || a.y1 < b.y0 || a.y0 > b.y1
+}
+
+func imin(a, b int) int {
+ if a < b {
+ return a
+ }
+ return b
+}
+
+// sortYFirstly orders regions top-to-bottom (and left-to-right within a
+// vertical threshold), matching Recognizer.sort_Y_firstly (recognizer.py:54)
+// which LayoutRecognizer.__call__ applies before annotation
+// (layout_recognizer.py:99).
+func sortYFirstly(regs []annRegion) {
+ if len(regs) == 0 {
+ return
+ }
+ avgH := 0.0
+ for _, r := range regs {
+ avgH += r.y1 - r.y0
+ }
+ thr := avgH / float64(len(regs)) / 2
+ sort.SliceStable(regs, func(i, j int) bool {
+ di := regs[i].y0 - regs[j].y0
+ if di < -thr {
+ return true
+ }
+ if di > thr {
+ return false
+ }
+ return regs[i].x0 < regs[j].x0
+ })
+}
+
+// cleanupLayouts de-duplicates overlapping same-type regions, matching
+// Recognizer.layouts_cleanup (recognizer.py:124) called by
+// LayoutRecognizer.__call__ at layout_recognizer.py:100. A pair of same-type
+// regions whose overlap exceeds thr (0.7) in either direction collapses to a
+// single region: the higher-score one, or - when scores are absent - the one
+// covering more text-box area. far=2 limits comparison to nearby regions.
+func cleanupLayouts(regs []annRegion, boxes []pdf.TextBox) []annRegion {
+ const far = 2
+ const thr = 0.7
+ i := 0
+ for i+1 < len(regs) {
+ j := i + 1
+ for j < imin(i+far, len(regs)) && (regs[j].label != regs[i].label || annNotOverlapped(regs[i], regs[j])) {
+ j++
+ }
+ if j >= imin(i+far, len(regs)) {
+ i++
+ continue
+ }
+ if overlapRatio(regs[i], regs[j]) < thr && overlapRatio(regs[j], regs[i]) < thr {
+ i++
+ continue
+ }
+ // Collapse the pair. Python layouts_cleanup keeps the HIGHER-score
+ // region; on equal scores it keeps the later one (j) via pop(i).
+ // Match that exactly so equal-confidence pairs converge with Python.
+ drop := j
+ if regs[i].score > 0 && regs[j].score > 0 {
+ if regs[i].score > regs[j].score {
+ drop = j
+ } else {
+ drop = i
+ }
+ } else {
+ areaI, areaJ := 0.0, 0.0
+ for _, b := range boxes {
+ tb := annRegion{x0: b.X0, y0: b.Top, x1: b.X1, y1: b.Bottom}
+ if !annNotOverlapped(tb, regs[i]) {
+ areaI += regionIntersect(tb, regs[i])
+ }
+ if !annNotOverlapped(tb, regs[j]) {
+ areaJ += regionIntersect(tb, regs[j])
+ }
+ }
+ if areaJ > areaI {
+ drop = i
+ }
+ }
+ regs = append(regs[:drop], regs[drop+1:]...)
+ }
+ return regs
+}
+
+// nmsDLARegions applies per-class non-maximum suppression to raw DLA regions,
+// mirroring Python's layout model postprocess (layout_recognizer.py:246,
+// operators.py:667 nms with iou_thresh). For each label, detections are sorted
+// by confidence descending; the top one is kept and any other same-label
+// detection whose IoU (using the +1 overlap convention from operators.py:685)
+// exceeds iouThresh is suppressed. This runs on the raw, pre-scale detections
+// just as Python's postprocess does, before cleanup/annotation.
+//
+// It is idempotent with a server-side NMS: applying it again to already-suppressed
+// boxes yields the same set, so it safely converges Go to Python regardless of
+// where suppression happens upstream.
+func nmsDLARegions(regions []pdf.DLARegion, iouThresh float64) []pdf.DLARegion {
+ if len(regions) == 0 {
+ return regions
+ }
+ byLabel := map[string][]int{}
+ for i, r := range regions {
+ byLabel[r.Label] = append(byLabel[r.Label], i)
+ }
+ suppressed := make([]bool, len(regions))
+ for _, idxs := range byLabel {
+ // Highest confidence first (greedy NMS keeps the top box). On equal
+ // confidence, break the tie by original index so the result is
+ // deterministic for identical inputs — otherwise the same PDF could
+ // keep a different region (and emit different LayoutNo) on each run.
+ sort.SliceStable(idxs, func(a, b int) bool {
+ ca, cb := regions[idxs[a]].Confidence, regions[idxs[b]].Confidence
+ if ca != cb {
+ return ca > cb
+ }
+ return idxs[a] < idxs[b]
+ })
+ for k := 0; k < len(idxs); k++ {
+ i := idxs[k]
+ if suppressed[i] {
+ continue
+ }
+ for m := k + 1; m < len(idxs); m++ {
+ j := idxs[m]
+ if suppressed[j] {
+ continue
+ }
+ if nmsIoU(regions[i], regions[j]) > iouThresh {
+ suppressed[j] = true
+ }
+ }
+ }
+ }
+ out := make([]pdf.DLARegion, 0, len(regions))
+ for i, r := range regions {
+ if !suppressed[i] {
+ out = append(out, r)
+ }
+ }
+ return out
+}
+
+// nmsIoU computes IoU using the +1 overlap convention from operators.py:685-688
+// (w = max(0, x22-x11+1), h = max(0, y22-y11+1)) while area uses no +1. This
+// must match Python exactly so borderline suppressions (around the 0.45 threshold)
+// align.
+func nmsIoU(a, b pdf.DLARegion) float64 {
+ w := math.Max(0, math.Min(a.X1, b.X1)-math.Max(a.X0, b.X0)+1)
+ h := math.Max(0, math.Min(a.Y1, b.Y1)-math.Max(a.Y0, b.Y0)+1)
+ inter := w * h
+ areaA := (a.X1 - a.X0) * (a.Y1 - a.Y0)
+ areaB := (b.X1 - b.X0) * (b.Y1 - b.Y0)
+ if areaA <= 0 || areaB <= 0 {
+ return 0
+ }
+ return inter / (areaA + areaB - inter)
+}
+
+// AnnotateBoxLayouts sets LayoutType and LayoutNo on each box, matching
// Python's LayoutRecognizer.__call__ which assigns layout types in priority
-// order (footer→header→…→equation) with an overlap threshold of 40% of the
+// order (footer->header->...->equation) with an overlap threshold of 40% of the
// box's area.
//
-// Python: _layouts_rec (pdf_parser.py:827) → LayoutRecognizer.__call__ →
+// Python: _layouts_rec (pdf_parser.py:827) -> LayoutRecognizer.__call__ ->
//
// for lt in priority_order: findLayout(lt)
//
// Each findLayout(ty): for each unannotated box, find the DLA region of
-// type ty with max overlap ≥ 0.4 × box_area. First type to match wins.
+// type ty with max overlap >= 0.4 * box_area. First type to match wins.
//
// CID-pattern boxes (e.g. "(cid:123)") are skipped as garbage.
-// annotateBoxLayouts assigns LayoutType and LayoutNo to boxes based on DLA
+// AnnotateBoxLayouts assigns LayoutType and LayoutNo to boxes based on DLA
// regions. Returns the filtered slice (Python pops CID-garbled boxes and
-// garbage-layout boxes at wrong positions — Go mirrors with compact).
+// garbage-layout boxes at wrong positions - Go mirrors with compact).
// Also creates synthetic figure boxes for unmatched figure/equation regions.
-func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale float64, pageImgHeight float64) []pdf.TextBox {
+//
+// Before annotation, regions are de-duplicated (layouts_cleanup) and sorted
+// top-to-bottom (sort_Y_firstly) to match Python, and unmatched figure and
+// equation regions receive SEPARATE synthetic namespaces (figure-N /
+// equation-N) so they never collide.
+// FilteredDLARegions returns the DLA regions after per-class NMS, the
+// confidence filter, Y-sort, and cleanup — i.e. the exact set fed to
+// annotation. It mirrors Python's page_layout (layout_recognizer.py:84-100):
+// - nmsDLARegions(0.45) == layout model postprocess (operators.py:667)
+// - keep if score >= 0.4 OR type not garbage == layout_recognizer.py:97
+// - sortYFirstly == layout_recognizer.py:99 (sort_Y_firstly)
+// - cleanupLayouts == layout_recognizer.py:100 (layouts_cleanup)
+//
+// Regions are returned in image-pixel space (no scale division) so callers
+// that only need the region set — e.g. the parity harness dumping post-filter
+// regions for comparison with Python's page_layout — get a stable comparison
+// point regardless of render DPI. cleanupLayouts only consults boxes when both
+// compared regions have score 0, which never happens for real DLA output, so
+// passing nil boxes from the harness is safe.
+func FilteredDLARegions(regions []pdf.DLARegion, boxes []pdf.TextBox) []pdf.DLARegion {
+ regions = nmsDLARegions(regions, 0.45)
if len(regions) == 0 {
- return boxes
+ return nil
+ }
+ kept := regions[:0]
+ for _, r := range regions {
+ if r.Confidence >= 0.4 || !isGarbageLayoutType(r.Label) {
+ kept = append(kept, r)
+ }
}
+ ars := make([]annRegion, len(kept))
+ for i, r := range kept {
+ ars[i] = annRegion{x0: r.X0, y0: r.Y0, x1: r.X1, y1: r.Y1, label: r.Label, score: r.Confidence}
+ }
+ sortYFirstly(ars)
+ ars = cleanupLayouts(ars, boxes)
+ out := make([]pdf.DLARegion, len(ars))
+ for i, a := range ars {
+ out[i] = pdf.DLARegion{X0: a.x0, Y0: a.y0, X1: a.x1, Y1: a.y1, Label: a.label, Confidence: a.score}
+ }
+ return out
+}
- // Scale all regions to PDF space once.
- type scaledRegion struct {
- x0, y0, x1, y1 float64
- label string
+func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale float64, pageImgHeight float64) []pdf.TextBox {
+ // NMS, confidence filter, Y-sort, and cleanup — the exact region set fed
+ // to annotation. This mirrors Python's layout_recognizer.py:84-100
+ // (filter + sort_Y_firstly + layouts_cleanup) and is the same pipeline the
+ // parity harness uses (via FilteredDLARegions) to dump post-filter regions
+ // comparable with Python's page_layout.
+ filtered := FilteredDLARegions(regions, boxes)
+ if len(filtered) == 0 {
+ return boxes
}
- scaled := make([]scaledRegion, len(regions))
- for i, r := range regions {
- scaled[i] = scaledRegion{
+
+ // Scale filtered regions from image-pixel space to PDF space.
+ cands := make([]annRegion, 0, len(filtered))
+ for _, r := range filtered {
+ cands = append(cands, annRegion{
x0: r.X0 / scale, y0: r.Y0 / scale,
x1: r.X1 / scale, y1: r.Y1 / scale,
- label: r.Label,
- }
- }
-
- // DLA confidence filter — matches Python's `score >= 0.4`.
- regionOK := make([]bool, len(regions))
- for i, r := range regions {
- regionOK[i] = r.Confidence >= 0.4 || !isGarbageLayoutType(r.Label)
+ label: r.Label, score: r.Confidence,
+ })
}
- // Pre-compute per-type index for each region (Python: matched index within
- // filtered layouts_of_type list). "text" regions get 0,1,2... independent
- // of "figure" regions.
- typeIndex := make([]int, len(regions))
+ // Per-type index in the cleaned, Y-sorted list (Python: ii in lts_).
typeCounters := make(map[string]int)
- for j, r := range scaled {
- if regionOK[j] {
- typeIndex[j] = typeCounters[r.label]
- typeCounters[r.label]++
- }
+ for j := range cands {
+ cands[j].typeIndex = typeCounters[cands[j].label]
+ typeCounters[cands[j].label]++
}
- // Track visited regions (Python: layout["visited"] = True).
- visited := make([]bool, len(regions))
-
// Marks for Python-style pop removal.
dropped := make([]bool, len(boxes))
@@ -139,9 +377,10 @@ func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale floa
continue
}
bestOverlap := 0.0
+ bestRegionOverlap := 0.0
bestJ := -1
- for j, r := range scaled {
- if r.label != ty || !regionOK[j] {
+ for j, r := range cands {
+ if r.label != ty {
continue
}
ix0 := math.Max(r.x0, boxes[i].X0)
@@ -149,20 +388,32 @@ func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale floa
ix1 := math.Min(r.x1, boxes[i].X1)
iy1 := math.Min(r.y1, boxes[i].Bottom)
if ix0 < ix1 && iy0 < iy1 {
- ov := (ix1 - ix0) * (iy1 - iy0) / boxArea
- if ov > bestOverlap {
+ inter := (ix1 - ix0) * (iy1 - iy0)
+ ov := inter / boxArea // fraction of the box covered (Python's ov)
+ rArea := (r.x1 - r.x0) * (r.y1 - r.y0)
+ ovRegion := 0.0
+ if rArea > 0 {
+ ovRegion = inter / rArea // fraction of the region covered (Python's _ov)
+ }
+ // Mirror Python's (ov, _ov) tuple comparison
+ // (recognizer.py:255-269): primary key is the box
+ // coverage ratio; on a tie the region-coverage ratio
+ // wins (prefer the region the box sits more "inside"
+ // of); on a full tie keep the first (topmost) region.
+ if ov > bestOverlap || (ov == bestOverlap && ovRegion > bestRegionOverlap) {
bestOverlap = ov
+ bestRegionOverlap = ovRegion
bestJ = j
}
}
}
if bestJ >= 0 && bestOverlap >= 0.4 {
- // Garbage layout not at page edge → pop (Python: bxs.pop(i)).
+ // Garbage layout not at page edge -> pop (Python: bxs.pop(i)).
if isGarbageLayoutType(ty) && pageImgHeight > 0 && !garbageKeepFeat(ty, boxes[i], pageImgHeight/scale) {
dropped[i] = true
continue
}
- visited[bestJ] = true
+ cands[bestJ].visited = true
// Python: equation mapped to "figure" for layout_type
if ty == pdf.LayoutTypeEquation {
boxes[i].LayoutType = pdf.LayoutTypeFigure
@@ -170,7 +421,7 @@ func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale floa
boxes[i].LayoutType = ty
}
// Python: f"{layout_type}-{matched}" where matched is per-type index
- boxes[i].LayoutNo = fmt.Sprintf("%s-%d", ty, typeIndex[bestJ])
+ boxes[i].LayoutNo = fmt.Sprintf("%s-%d", ty, cands[bestJ].typeIndex)
}
}
}
@@ -196,25 +447,28 @@ func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale floa
boxes = compacted
// Synthetic figure boxes for unmatched figure/equation regions (Python:
- // dla_cli.py:187-195). Use a fresh per-type counter for synthetic boxes.
- synthIdx := 0
- for j, r := range scaled {
- if !regionOK[j] || visited[j] {
+ // layout_recognizer.py:145-155). Python numbers each unmatched region with
+ // its index WITHIN the per-type list that also includes already-visited
+ // regions (enumerate([lt for lt in lts if lt["type"] == ty])), so we reuse
+ // the per-type typeIndex computed above rather than a separate
+ // unvisited-only counter. Python keeps figure-N / equation-N in SEPARATE
+ // namespaces, so the typeIndex is keyed by the original type label.
+ for j := range cands {
+ if cands[j].visited {
continue
}
- if r.label != pdf.LayoutTypeFigure && r.label != pdf.LayoutTypeEquation {
+ if cands[j].label != pdf.LayoutTypeFigure && cands[j].label != pdf.LayoutTypeEquation {
continue
}
boxes = append(boxes, pdf.TextBox{
- X0: r.x0,
- X1: r.x1,
- Top: r.y0,
- Bottom: r.y1,
+ X0: cands[j].x0,
+ X1: cands[j].x1,
+ Top: cands[j].y0,
+ Bottom: cands[j].y1,
Text: "",
LayoutType: pdf.LayoutTypeFigure,
- LayoutNo: fmt.Sprintf("figure-%d", synthIdx),
+ LayoutNo: fmt.Sprintf("%s-%d", cands[j].label, cands[j].typeIndex),
})
- synthIdx++
}
return boxes
@@ -232,7 +486,7 @@ func isGarbageLayoutType(ty string) bool {
// garbageKeepFeat matches Python's keep_feats in LayoutRecognizer.__call__:
// footer near page bottom (>90% of page height) or header near page top (<10%)
-// are real page decorations — keep them. Others are DLA noise.
+// are real page decorations - keep them. Others are DLA noise.
func garbageKeepFeat(ty string, box pdf.TextBox, pageImgHeight float64) bool {
switch ty {
case pdf.LayoutTypeFooter:
diff --git a/internal/deepdoc/parser/pdf/table/table_annotate_parity_test.go b/internal/deepdoc/parser/pdf/table/table_annotate_parity_test.go
new file mode 100644
index 0000000000..23cb815e1a
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/table/table_annotate_parity_test.go
@@ -0,0 +1,297 @@
+// Tests pinning the Python-equivalent behavior of AnnotateBoxLayouts.
+//
+// These assert the layout-annotation semantics that Python's
+// LayoutRecognizer.__call__ (deepdoc/vision/layout_recognizer.py:68) produces,
+// and that the Go implementation now replicates (GREEN):
+//
+// - #1 layouts_cleanup: Python de-dupes overlapping same-type regions
+// (recognizer.py:124-160, called at layout_recognizer.py:100) BEFORE
+// annotation. Go matches via cleanupLayouts, so no extra synthetic
+// figure/equation boxes are emitted.
+// - #2 sort_Y_firstly: Python sorts regions top-to-bottom (recognizer.py:54,
+// layout_recognizer.py:99) before numbering them, so layoutno indices
+// follow reading order. Go matches via sortYFirstly.
+// - #3 synthetic namespace: Python numbers unmatched figure/equation regions
+// in SEPARATE counters -> "figure-N" / "equation-N"
+// (layout_recognizer.py:145-155). Go matches via a per-type typeIndex
+// keyed by the original type label.
+//
+// These are pure-Go unit tests (no model server, no external service): they
+// feed crafted pdf.DLARegion slices and assert the annotated pdf.TextBox
+// output. Coordinates use scale=1 so region pixels == PDF space.
+
+package table
+
+import (
+ "testing"
+
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+)
+
+// countFigureBoxes returns the number of boxes whose LayoutType is "figure".
+func countFigureBoxes(boxes []pdf.TextBox) int {
+ n := 0
+ for _, b := range boxes {
+ if b.LayoutType == pdf.LayoutTypeFigure {
+ n++
+ }
+ }
+ return n
+}
+
+// countSyntheticFigures returns the number of figure boxes with no text
+// (the synthetic placeholders AnnotateBoxLayouts appends for unvisited
+// figure/equation regions).
+func countSyntheticFigures(boxes []pdf.TextBox) int {
+ n := 0
+ for _, b := range boxes {
+ if b.LayoutType == pdf.LayoutTypeFigure && b.Text == "" {
+ n++
+ }
+ }
+ return n
+}
+
+// TestAnnotateBoxLayouts_DuplicateFigureRegions_Merged pins #1: two heavily
+// overlapping same-type (figure) regions must be treated as ONE, so only one
+// figure box (the annotated text box) results and NO synthetic figure box is
+// produced. Python merges via layouts_cleanup; Go now mirrors that by fusing
+// the overlapping pair into one region, so the duplicate synthetic figure is
+// avoided.
+func TestAnnotateBoxLayouts_DuplicateFigureRegions_Merged(t *testing.T) {
+ box := pdf.TextBox{X0: 0, X1: 100, Top: 0, Bottom: 50, Text: "Some caption text", PageNumber: 0}
+ regions := []pdf.DLARegion{
+ {X0: 0, Y0: 0, X1: 100, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeFigure},
+ {X0: 0, Y0: 0, X1: 100, Y1: 50, Confidence: 0.8, Label: pdf.LayoutTypeFigure}, // identical -> IoU 1.0
+ }
+
+ out := AnnotateBoxLayouts([]pdf.TextBox{box}, regions, 1.0, 100.0)
+
+ if got := countFigureBoxes(out); got != 1 {
+ t.Errorf("#1 layouts_cleanup: expected 1 figure box after de-duplicating overlapping figure regions, got %d", got)
+ }
+ if got := countSyntheticFigures(out); got != 0 {
+ t.Errorf("#1 layouts_cleanup: expected 0 synthetic figure boxes (duplicate region must be merged), got %d", got)
+ }
+}
+
+// TestAnnotateBoxLayouts_SameTypeOutOfYOrder_LayoutNoFollowsY pins #2: when
+// same-type regions arrive in non-reading order (here the bottom region is
+// first in the wire list), the layoutno index must still follow top-to-bottom
+// order, exactly like Python's sort_Y_firstly. The box unambiguously overlaps
+// only the bottom region, so the winner is the same in both languages; only
+// the assigned index differs.
+func TestAnnotateBoxLayouts_SameTypeOutOfYOrder_LayoutNoFollowsY(t *testing.T) {
+ // Wire order: bottom region first (NOT top-to-bottom).
+ regions := []pdf.DLARegion{
+ {X0: 0, Y0: 30, X1: 100, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeTitle}, // bottom strip
+ {X0: 0, Y0: 0, X1: 100, Y1: 20, Confidence: 0.9, Label: pdf.LayoutTypeTitle}, // top strip
+ }
+ // Box overlaps only the bottom region -> unambiguous winner = bottom region.
+ box := pdf.TextBox{X0: 0, X1: 100, Top: 30, Bottom: 50, Text: "Title text", PageNumber: 0}
+
+ out := AnnotateBoxLayouts([]pdf.TextBox{box}, regions, 1.0, 100.0)
+
+ var annotated *pdf.TextBox
+ for i := range out {
+ if out[i].Text == "Title text" {
+ annotated = &out[i]
+ }
+ }
+ if annotated == nil {
+ t.Fatal("#2 sort_Y_firstly: text box was not annotated as title")
+ }
+ // Python Y-sorts -> [top(ii=0), bottom(ii=1)] -> winner bottom => "title-1".
+ if annotated.LayoutNo != "title-1" {
+ t.Errorf("#2 sort_Y_firstly: expected layoutno \"title-1\" (top-to-bottom numbering), got %q", annotated.LayoutNo)
+ }
+}
+
+// TestAnnotateBoxLayouts_SyntheticFigureEquation_SeparateNamespaces pins #3:
+// an unmatched figure region and an unmatched equation region must yield
+// distinct synthetic layoutnos "figure-0" and "equation-0". Go now keeps
+// per-type synthetic counters so figures and equations are numbered
+// independently instead of sharing a single figure-N counter.
+func TestAnnotateBoxLayouts_SyntheticFigureEquation_SeparateNamespaces(t *testing.T) {
+ regions := []pdf.DLARegion{
+ {X0: 0, Y0: 0, X1: 50, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeFigure},
+ {X0: 60, Y0: 0, X1: 110, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeEquation},
+ }
+
+ out := AnnotateBoxLayouts([]pdf.TextBox{}, regions, 1.0, 100.0)
+
+ layoutNos := map[string]bool{}
+ for _, b := range out {
+ layoutNos[b.LayoutNo] = true
+ }
+ if !layoutNos["figure-0"] {
+ t.Errorf("#3 synthetic namespace: expected a synthetic box with layoutno \"figure-0\"")
+ }
+ if !layoutNos["equation-0"] {
+ t.Errorf("#3 synthetic namespace: expected a synthetic box with layoutno \"equation-0\" (separate counter from figure); equation was folded into the shared figure counter")
+ }
+}
+
+// TestAnnotateBoxLayouts_TieBreakRegionCoverage pins #4: when two same-type
+// regions cover the box by the SAME fraction (ov tie), Python's
+// find_overlapped_with_threshold (recognizer.py:255-269) breaks the tie by the
+// region-coverage ratio (_ov = box∩region / region area), preferring the region
+// the box sits more "inside" of. Go mirrors the (ov, _ov) tuple comparison so
+// the higher-_ov region wins the tie instead of the first Y-sorted candidate.
+//
+// Layout: wide top region A and narrow bottom region B. The box spans both
+// vertically with EQUAL absolute intersection, so ov_A == ov_B. B is smaller,
+// so _ov_B > _ov_A and Python picks B. In Y-sorted order [A, B], B is per-type
+// index 1, so the expected layoutno is "table-1".
+func TestAnnotateBoxLayouts_TieBreakRegionCoverage(t *testing.T) {
+ regions := []pdf.DLARegion{
+ {X0: 0, Y0: 0, X1: 200, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeTable}, // A: wide, top
+ {X0: 0, Y0: 60, X1: 50, Y1: 110, Confidence: 0.9, Label: pdf.LayoutTypeTable}, // B: narrow, bottom
+ }
+ box := pdf.TextBox{X0: 0, X1: 50, Top: 0, Bottom: 110, Text: "tbl", PageNumber: 0}
+
+ out := AnnotateBoxLayouts([]pdf.TextBox{box}, regions, 1.0, 0)
+
+ var annotated *pdf.TextBox
+ for i := range out {
+ if out[i].Text == "tbl" {
+ annotated = &out[i]
+ }
+ }
+ if annotated == nil {
+ t.Fatal("#4 tie-break: box not annotated as table")
+ }
+ if annotated.LayoutNo != "table-1" {
+ t.Errorf("#4 tie-break: expected layoutno \"table-1\" (smaller region B wins the _ov tie), got %q", annotated.LayoutNo)
+ }
+}
+
+// TestAnnotateBoxLayouts_NMSDedupOverlappingRegions pins #6: Python's layout
+// model postprocess applies per-class NMS with IoU 0.45 (layout_recognizer.py:246,
+// operators.py:667) on the RAW detections BEFORE annotation. Go must do the same
+// on the raw regions; otherwise same-label detections overlapping between 0.45
+// and 0.7 survive (cleanupLayouts only merges at thr=0.7) and emit extra
+// synthetic figure boxes.
+//
+// Two same-label figure regions with IoU ~0.54: >0.45 so NMS suppresses the
+// lower-score one; <0.7 so cleanupLayouts would NOT merge them. With NMS there
+// is exactly one figure region -> one synthetic figure box.
+func TestAnnotateBoxLayouts_NMSDedupOverlappingRegions(t *testing.T) {
+ regions := []pdf.DLARegion{
+ {X0: 0, Y0: 0, X1: 100, Y1: 100, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // A
+ {X0: 35, Y0: 0, X1: 135, Y1: 100, Confidence: 0.8, Label: pdf.LayoutTypeFigure}, // B overlaps A (shift 35)
+ }
+ // No text box overlaps -> without NMS both become synthetic figures.
+ // Overlap is ~0.65 (no-+1 IoU) so cleanupLayouts (thr=0.7) does NOT merge,
+ // but Python's NMS uses +1 IoU (~0.50) > 0.45 and suppresses the lower-score B.
+ out := AnnotateBoxLayouts([]pdf.TextBox{}, regions, 1.0, 100.0)
+
+ if got := countFigureBoxes(out); got != 1 {
+ t.Errorf("#6 NMS: expected 1 figure box after per-class NMS merges overlapping detections (IoU ~0.5), got %d", got)
+ }
+}
+
+// TestAnnotateBoxLayouts_NMSDeterministic pins that equal-confidence
+// same-label detections collapse to the SAME region on every run. Before the
+// sort.SliceStable + original-index tie-break, sort.Slice ordered equal scores
+// nondeterministically, so the survivor (and its synthetic LayoutNo) could flip
+// between otherwise-identical runs — breaking reproducibility and the golden
+// comparison.
+func TestAnnotateBoxLayouts_NMSDeterministic(t *testing.T) {
+ regions := []pdf.DLARegion{
+ {X0: 0, Y0: 0, X1: 100, Y1: 100, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // A (lower original index)
+ {X0: 35, Y0: 0, X1: 135, Y1: 100, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // B overlaps A, equal score
+ }
+ var baseline []pdf.TextBox
+ for run := 0; run < 30; run++ {
+ out := AnnotateBoxLayouts([]pdf.TextBox{}, regions, 1.0, 100.0)
+ if got := countFigureBoxes(out); got != 1 {
+ t.Fatalf("run %d: expected exactly 1 figure box after NMS, got %d", run, got)
+ }
+ if run == 0 {
+ baseline = out
+ continue
+ }
+ if len(out) != len(baseline) {
+ t.Fatalf("run %d: box count changed across runs (%d vs %d)", run, len(out), len(baseline))
+ }
+ for i := range out {
+ if out[i].X0 != baseline[i].X0 || out[i].X1 != baseline[i].X1 ||
+ out[i].Top != baseline[i].Top || out[i].Bottom != baseline[i].Bottom ||
+ out[i].LayoutNo != baseline[i].LayoutNo {
+ t.Fatalf("run %d: nondeterministic output (box %d differs from run 0)", run, i)
+ }
+ }
+ }
+ // Lower-index tie-break keeps region A.
+ if baseline[0].X0 != 0 || baseline[0].X1 != 100 {
+ t.Errorf("expected the lower-index region A (x0=0,x1=100) to survive, got x0=%v,x1=%v", baseline[0].X0, baseline[0].X1)
+ }
+}
+
+// TestAnnotateBoxLayouts_CleanupEqualScoreKeepsLater pins that when two
+// same-type regions overlap beyond cleanup's 0.7 threshold with EQUAL
+// confidence, Go keeps the LATER one (j) — matching Python layouts_cleanup
+// (pop(i) on equal scores). Before the fix Go kept the earlier region.
+//
+// NMS (IoU 0.45) must not pre-remove either: A sits fully inside B, so their
+// +1 IoU is tiny (<0.45) while their overlap ratio (inter/area of B) exceeds
+// 0.7, so only cleanup is exercised.
+func TestAnnotateBoxLayouts_CleanupEqualScoreKeepsLater(t *testing.T) {
+ regions := []pdf.DLARegion{
+ {X0: 0, Y0: 0, X1: 400, Y1: 400, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // B: large, on top (Y0=0)
+ {X0: 100, Y0: 200, X1: 200, Y1: 300, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // A: small, inside B, lower (Y0=200)
+ }
+ // No text box overlaps -> both unvisited -> cleanup collapses to one synthetic.
+ out := AnnotateBoxLayouts([]pdf.TextBox{}, regions, 1.0, 100.0)
+ if got := countFigureBoxes(out); got != 1 {
+ t.Fatalf("expected 1 figure box after cleanup merge, got %d", got)
+ }
+ // Y-sort puts B (Y0=0) first, A (Y0=200) second; equal score -> keep later (A).
+ if out[0].X0 != 100 || out[0].X1 != 200 || out[0].Top != 200 || out[0].Bottom != 300 {
+ t.Errorf("cleanup equal-score: expected later region A (x0=100,x1=200,top=200,bottom=300) to survive, got x0=%v,x1=%v,top=%v,bottom=%v",
+ out[0].X0, out[0].X1, out[0].Top, out[0].Bottom)
+ }
+}
+
+// TestAnnotateBoxLayouts_SyntheticVisitedInterleaved pins that an unmatched
+// figure region is numbered by its index in the per-type list that ALSO
+// includes already-visited figure regions (Python's enumerate over the full
+// type-filtered list). Before the fix Go used a separate unvisited-only
+// counter, so the unmatched figure became figure-0 instead of figure-1.
+func TestAnnotateBoxLayouts_SyntheticVisitedInterleaved(t *testing.T) {
+ boxes := []pdf.TextBox{
+ {X0: 0, X1: 100, Top: 0, Bottom: 50, Text: "caption for A"},
+ }
+ regions := []pdf.DLARegion{
+ {X0: 0, Y0: 0, X1: 100, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // A: matched to text box -> visited
+ {X0: 200, Y0: 0, X1: 300, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // B: unmatched -> synthetic
+ }
+ out := AnnotateBoxLayouts(boxes, regions, 1.0, 100.0)
+
+ var annotated, synthetic *pdf.TextBox
+ for i := range out {
+ if out[i].Text == "caption for A" {
+ annotated = &out[i]
+ }
+ if out[i].LayoutType == pdf.LayoutTypeFigure && out[i].Text == "" {
+ synthetic = &out[i]
+ }
+ }
+ if annotated == nil {
+ t.Fatal("visited figure A: text box was not annotated as figure")
+ }
+ if annotated.LayoutNo != "figure-0" {
+ t.Errorf("visited figure A: expected LayoutNo figure-0, got %q", annotated.LayoutNo)
+ }
+ if synthetic == nil {
+ t.Fatal("expected a synthetic figure box for unmatched B")
+ }
+ // B is the 2nd figure in Y order (after visited A), so Python numbers it figure-1.
+ if synthetic.LayoutNo != "figure-1" {
+ t.Errorf("unmatched figure B: expected figure-1 (index in full per-type list), got %q", synthetic.LayoutNo)
+ }
+ if synthetic.X0 != 200 || synthetic.X1 != 300 {
+ t.Errorf("unmatched figure B: expected x0=200,x1=300, got x0=%v,x1=%v", synthetic.X0, synthetic.X1)
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/table/table_annotate_test.go b/internal/deepdoc/parser/pdf/table/table_annotate_test.go
index 3ac34468f4..6ffd615b99 100644
--- a/internal/deepdoc/parser/pdf/table/table_annotate_test.go
+++ b/internal/deepdoc/parser/pdf/table/table_annotate_test.go
@@ -154,6 +154,34 @@ func TestAnnotateBoxLayouts_ConfidenceFilter(t *testing.T) {
}
}
+// TestFilteredDLARegions pins the post-filter region set the parity harness
+// dumps for comparison with Python's page_layout. The confidence filter keeps
+// a region when score >= 0.4 OR its type is not garbage — so a low-confidence
+// *non-garbage* region (e.g. text at 0.1) is KEPT, exactly matching Python's
+// `score >= 0.4 or type not in garbage_layouts` (layout_recognizer.py:97).
+// Returned regions stay in image-pixel space (no scale division).
+func TestFilteredDLARegions(t *testing.T) {
+ regions := []pdf.DLARegion{
+ {X0: 0, Y0: 0, X1: 300, Y1: 150, Label: "footer", Confidence: 0.2}, // low-conf garbage → dropped
+ {X0: 0, Y0: 200, X1: 300, Y1: 350, Label: "text", Confidence: 0.1}, // low-conf non-garbage → kept
+ {X0: 0, Y0: 400, X1: 500, Y1: 460, Label: "reference", Confidence: 0.5}, // >=0.4 garbage → kept
+ }
+ got := FilteredDLARegions(regions, nil)
+ if len(got) != 2 {
+ t.Fatalf("FilteredDLARegions() = %d regions, want 2 (got %+v)", len(got), got)
+ }
+ labels := map[string]bool{}
+ for _, r := range got {
+ labels[r.Label] = true
+ if r.Confidence == 0.2 {
+ t.Errorf("low-confidence footer should have been filtered out")
+ }
+ }
+ if !labels["text"] || !labels["reference"] {
+ t.Errorf("expected text(low-conf) and reference(>=0.4) kept, got labels %v", labels)
+ }
+}
+
func TestAnnotateBoxLayouts_GarbageFooterRejected(t *testing.T) {
// Footer at page bottom: Bottom(290) > 270 (90% of 300px→PDF height 100→90% of 100=90)
// → real footer decoration → garbage → pop (Python: bxs.pop(i)).
@@ -278,8 +306,11 @@ func TestAnnotateBoxLayouts_SyntheticFigure(t *testing.T) {
if b.LayoutType == "figure" && b.Text == "" {
if b.LayoutNo == "figure-0" {
foundFig0 = true
- if b.X0 != 100 || b.X1 != 200 {
- t.Errorf("synthetic figure-0: expected x0=100,x1=200 (300/3,600/3), got x0=%v,x1=%v", b.X0, b.X1)
+ // After sort_Y_firstly, the top figure region (Y0=0 -> PDF
+ // y0=0, x0=200 from 600/3, x1=300 from 900/3) is figure-0,
+ // matching Python's top-to-bottom numbering.
+ if b.X0 != 200 || b.X1 != 300 {
+ t.Errorf("synthetic figure-0: expected x0=200,x1=300 (top region after Y-sort), got x0=%v,x1=%v", b.X0, b.X1)
}
}
if b.LayoutNo == "figure-1" {
diff --git a/internal/deepdoc/parser/pdf/tool/compare.go b/internal/deepdoc/parser/pdf/tool/compare.go
index c62ea849be..c8c0c88a77 100644
--- a/internal/deepdoc/parser/pdf/tool/compare.go
+++ b/internal/deepdoc/parser/pdf/tool/compare.go
@@ -47,6 +47,9 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul
var diffs []Diff
matched, mismatched := 0, 0
+ // suppressedAny tracks whether any PDF hit the Py stage-metric phase
+ // gap (see the guard below) so we can warn once in the summary.
+ suppressedAny := false
for _, r := range goResults {
py, ok := pyMap[r.File]
@@ -74,6 +77,23 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul
if py.Sections > 0 {
d.SectionsDiffPct = math.Abs(float64(r.Sections-py.Sections)) / float64(py.Sections) * 100
}
+
+ // Phase-consistency guard: the Python harness cannot report
+ // post-merge pipeline stages (boxes_text_merge / boxes_vertical_merge
+ // / sections) because the production parser does not expose them, so
+ // its #@meta falls back to the raw final box count for all three.
+ // Comparing those against Go's real merged stages would emit
+ // misleading percentages (e.g. 67% on "sections"). Mark them N/A.
+ if py.BoxesInitial > 0 &&
+ py.BoxesTextMerge == py.BoxesInitial &&
+ py.BoxesVertMerge == py.BoxesInitial &&
+ py.Sections == py.BoxesInitial {
+ d.BoxesTMDiffPct = -1
+ d.BoxesVMDiffPct = -1
+ d.SectionsDiffPct = -1
+ suppressedAny = true
+ }
+
if py.TextLen > 0 {
d.TextLenDiffPct = math.Abs(float64(r.TextLen-py.TextLen)) / float64(py.TextLen) * 100
}
@@ -103,7 +123,18 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul
len(diffs), len(goResults), r.File, 100-d.CharSim, 100-d.LcsSim, 100-d.RawCharSim, 100-d.RawLcsSim)
}
- sort.Slice(diffs, func(i, j int) bool { return diffs[i].SectionsDiffPct < diffs[j].SectionsDiffPct })
+ // Sort worst-first by SectionsDiffPct, but push N/A (-1) rows to the
+ // bottom so the per-PDF table reads cleanly.
+ sort.Slice(diffs, func(i, j int) bool {
+ a, b := diffs[i].SectionsDiffPct, diffs[j].SectionsDiffPct
+ if a < 0 {
+ a = math.MaxFloat64
+ }
+ if b < 0 {
+ b = math.MaxFloat64
+ }
+ return a < b
+ })
log.Logf("\n=== Go vs Python (%d PDFs) ===", len(diffs))
log.Logf("Pages match: %d/%d", matched, matched+mismatched)
@@ -117,10 +148,10 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul
gr := goMap[d.File]
goStages := fmt.Sprintf("%3d->%3d->%3d->%3d", gr.BoxesInitial, gr.BoxesTextMerg, gr.BoxesVertMerg, gr.Sections)
pyStages := fmt.Sprintf("%3d->%3d->%3d->%3d", py.BoxesInitial, py.BoxesTextMerge, py.BoxesVertMerge, py.Sections)
- log.Logf("%-40s %-18s %-18s %4.0f%% %4.0f%% %4.0f%% %4.0f%% %4.0f%% %+4d %.0f%% %.0f%% %.0f%% %.0f%%",
+ log.Logf("%-40s %-18s %-18s %5s %5s %5s %5s %5.0f%% %+4d %.0f%% %.0f%% %.0f%% %.0f%%",
d.File, goStages, pyStages,
- d.BoxesInitDiffPct, d.BoxesTMDiffPct, d.BoxesVMDiffPct,
- d.SectionsDiffPct, d.TextLenDiffPct, d.TablesDiff,
+ pctStr(d.BoxesInitDiffPct), pctStr(d.BoxesTMDiffPct), pctStr(d.BoxesVMDiffPct),
+ pctStr(d.SectionsDiffPct), d.TextLenDiffPct, d.TablesDiff,
100-d.CharSim, 100-d.LcsSim,
100-d.RawCharSim, 100-d.RawLcsSim)
}
@@ -134,17 +165,31 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul
median, mean, max, min float64
over5, over10 int
}
- computeStats := func(get func(Diff) float64) stats {
- sort.Slice(diffs, func(i, j int) bool { return get(diffs[i]) < get(diffs[j]) })
- s := stats{min: 1e9}
- if n%2 == 0 {
- s.median = (get(diffs[n/2-1]) + get(diffs[n/2])) / 2
+ // computeStats returns summary statistics over the valid (>=0) samples.
+ // It returns ok=false when every sample is N/A, so the caller can print
+ // "N/A" instead of bogus numbers.
+ computeStats := func(get func(Diff) float64) (s stats, ok bool) {
+ s.min = math.MaxFloat64
+ var vals []float64
+ for _, d := range diffs {
+ v := get(d)
+ if v < 0 {
+ continue
+ }
+ vals = append(vals, v)
+ }
+ if len(vals) == 0 {
+ return s, false
+ }
+ sort.Float64s(vals)
+ m := len(vals)
+ if m%2 == 0 {
+ s.median = (vals[m/2-1] + vals[m/2]) / 2
} else {
- s.median = get(diffs[n/2])
+ s.median = vals[m/2]
}
var sum float64
- for _, d := range diffs {
- v := get(d)
+ for _, v := range vals {
sum += v
if v > s.max {
s.max = v
@@ -159,8 +204,8 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul
s.over10++
}
}
- s.mean = sum / float64(n)
- return s
+ s.mean = sum / float64(m)
+ return s, true
}
label := func(name string, s stats) string {
@@ -169,15 +214,59 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul
}
log.Logf("\nSummary (n=%d):", n)
- log.Logf(" %s", label("BoxesInit ", computeStats(func(d Diff) float64 { return d.BoxesInitDiffPct })))
- log.Logf(" %s", label("TextMerge", computeStats(func(d Diff) float64 { return d.BoxesTMDiffPct })))
- log.Logf(" %s", label("VertMerge", computeStats(func(d Diff) float64 { return d.BoxesVMDiffPct })))
- log.Logf(" %s", label("Sections ", computeStats(func(d Diff) float64 { return d.SectionsDiffPct })))
- log.Logf(" %s", label("TextLen ", computeStats(func(d Diff) float64 { return d.TextLenDiffPct })))
- log.Logf(" %s", label("CharDiff ", computeStats(func(d Diff) float64 { return 100 - d.CharSim })))
- log.Logf(" %s", label("LcsDiff ", computeStats(func(d Diff) float64 { return 100 - d.LcsSim })))
- log.Logf(" %s", label("RawCharDiff", computeStats(func(d Diff) float64 { return 100 - d.RawCharSim })))
- log.Logf(" %s", label("RawLcsDiff ", computeStats(func(d Diff) float64 { return 100 - d.RawLcsSim })))
+ if s, ok := computeStats(func(d Diff) float64 { return d.BoxesInitDiffPct }); ok {
+ log.Logf(" %s", label("BoxesInit ", s))
+ } else {
+ log.Logf(" BoxesInit N/A")
+ }
+ if s, ok := computeStats(func(d Diff) float64 { return d.BoxesTMDiffPct }); ok {
+ log.Logf(" %s", label("TextMerge", s))
+ } else {
+ log.Logf(" TextMerge N/A (Python harness does not expose post-merge stage)")
+ }
+ if s, ok := computeStats(func(d Diff) float64 { return d.BoxesVMDiffPct }); ok {
+ log.Logf(" %s", label("VertMerge", s))
+ } else {
+ log.Logf(" VertMerge N/A (Python harness does not expose post-merge stage)")
+ }
+ if s, ok := computeStats(func(d Diff) float64 { return d.SectionsDiffPct }); ok {
+ log.Logf(" %s", label("Sections ", s))
+ } else {
+ log.Logf(" Sections N/A (Python harness does not expose post-merge stage)")
+ }
+ if s, ok := computeStats(func(d Diff) float64 { return d.TextLenDiffPct }); ok {
+ log.Logf(" %s", label("TextLen ", s))
+ } else {
+ log.Logf(" TextLen N/A")
+ }
+ if s, ok := computeStats(func(d Diff) float64 { return 100 - d.CharSim }); ok {
+ log.Logf(" %s", label("CharDiff ", s))
+ } else {
+ log.Logf(" CharDiff N/A")
+ }
+ if s, ok := computeStats(func(d Diff) float64 { return 100 - d.LcsSim }); ok {
+ log.Logf(" %s", label("LcsDiff ", s))
+ } else {
+ log.Logf(" LcsDiff N/A")
+ }
+ if s, ok := computeStats(func(d Diff) float64 { return 100 - d.RawCharSim }); ok {
+ log.Logf(" %s", label("RawCharDiff", s))
+ } else {
+ log.Logf(" RawCharDiff N/A")
+ }
+ if s, ok := computeStats(func(d Diff) float64 { return 100 - d.RawLcsSim }); ok {
+ log.Logf(" %s", label("RawLcsDiff ", s))
+ } else {
+ log.Logf(" RawLcsDiff N/A")
+ }
+
+ if suppressedAny {
+ log.Logf("\nNOTE: 'N/A' in TM%%/VM%%/Sec%% means the Python harness could not")
+ log.Logf("expose post-merge pipeline stage counts (the production parser does")
+ log.Logf("not record boxes_text_merge / boxes_vertical_merge / sections), so its")
+ log.Logf("#@meta falls back to the raw final box count. Those columns are not")
+ log.Logf("comparable and were excluded from the summary statistics.")
+ }
// Auto-generate xlsx report with timestamp.
mode := filepath.Base(filepath.Dir(goTextDir)) // "ocr"
@@ -289,11 +378,15 @@ func WriteExcel(path string, diffs []Diff) error {
for col := 2; col <= 9; col++ {
cell := cellName(col, r)
v := vals[col-1]
- f.SetCellValue(sheet, cell, v)
- // Color: green <5, yellow 5-20, red >=20.
if col == 7 { // TabsD is a count, not percentage
+ f.SetCellValue(sheet, cell, v)
+ continue
+ }
+ if v < 0 { // N/A: leave the cell blank, no color.
continue
}
+ f.SetCellValue(sheet, cell, v)
+ // Color: green <5, yellow 5-20, red >=20.
abs := math.Abs(v)
switch {
case abs < 5:
@@ -646,3 +739,12 @@ func abs(x int) int {
}
return x
}
+
+// pctStr renders a diff-percentage for table output. Negative values encode
+// "N/A" (e.g. a Python stage metric that could not be reported).
+func pctStr(v float64) string {
+ if v < 0 {
+ return "N/A"
+ }
+ return fmt.Sprintf("%.0f%%", v)
+}
diff --git a/internal/deepdoc/parser/pdf/tool/compare_test.go b/internal/deepdoc/parser/pdf/tool/compare_test.go
index bd2dee92f8..1fc7951c5c 100644
--- a/internal/deepdoc/parser/pdf/tool/compare_test.go
+++ b/internal/deepdoc/parser/pdf/tool/compare_test.go
@@ -7,6 +7,8 @@ import (
"os"
"path/filepath"
"testing"
+
+ "ragflow/internal/common"
)
// TestBatchCompareWithPython compares Go output against Python reference
diff --git a/internal/parser/parser/pdf_postprocess.go b/internal/parser/parser/pdf_postprocess.go
index 7a0ff99b2f..cc1466ff3e 100644
--- a/internal/parser/parser/pdf_postprocess.go
+++ b/internal/parser/parser/pdf_postprocess.go
@@ -9,7 +9,11 @@ import (
deepdoctype "ragflow/internal/deepdoc/parser/type"
)
-var pdfHeaderFooterPattern = regexp.MustCompile(`(?i)^(header|footer|number)$`)
+// Substring match to mirror Python's remove_header_footer:
+// re.search(r"(header|footer|number)", raw_layout, re.I) (rag/flow/parser/parser.py:754).
+// Python matches any layout type CONTAINING one of these words, not just the
+// exact token, so a composite label like "page-footer" is also stripped.
+var pdfHeaderFooterPattern = regexp.MustCompile(`(?i)header|footer|number`)
var pdfTOCTitlePattern = regexp.MustCompile(`(?i)^(contents|目录|目次|table of contents|致谢|acknowledge)$`)
type pdfPostProcessOptions struct {
diff --git a/internal/parser/parser/pdf_postprocess_test.go b/internal/parser/parser/pdf_postprocess_test.go
index dbb62bb12c..4a1e4fdea6 100644
--- a/internal/parser/parser/pdf_postprocess_test.go
+++ b/internal/parser/parser/pdf_postprocess_test.go
@@ -141,3 +141,38 @@ func TestApplyPDFPostProcess_ReordersMultiColumnText(t *testing.T) {
}
}
}
+
+// TestFilterPDFHeaderFooter_SubstringMatch pins #5: Python's remove_header_footer
+// uses a substring match re.search(r"(header|footer|number)", ...) (rag/flow/parser/parser.py:754),
+// while Go used an anchored exact match ^(header|footer|number)$. A layout type
+// that merely CONTAINS one of those words (e.g. "page-footer") must be stripped to
+// match Python, not silently kept.
+func TestFilterPDFHeaderFooter_SubstringMatch(t *testing.T) {
+ result := &deepdoctype.ParseResult{
+ Sections: []deepdoctype.Section{
+ {Text: "real header", LayoutType: "header"},
+ {Text: "real footer", LayoutType: "footer"},
+ {Text: "page 1", LayoutType: "number"},
+ {Text: "a page-footer note", LayoutType: "page-footer"}, // composite -> substring match
+ {Text: "body text", LayoutType: "text"},
+ },
+ }
+ filterPDFHeaderFooter(result)
+
+ kept := map[string]bool{}
+ for _, s := range result.Sections {
+ kept[s.LayoutType] = true
+ }
+ for _, lt := range []string{"header", "footer", "number"} {
+ if kept[lt] {
+ t.Errorf("#5 header/footer: %q should be stripped", lt)
+ }
+ }
+ // Composite "page-footer" must be stripped by substring match (Python-equivalent).
+ if kept["page-footer"] {
+ t.Errorf("#5 header/footer: composite layout type %q should be stripped by substring match", "page-footer")
+ }
+ if !kept["text"] {
+ t.Errorf("#5 header/footer: body text %q should be kept", "text")
+ }
+}