RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(pdf): align DLA region handling with Python reference (#18295)
Jack272645a27aba
diff --git a/internal/deepdoc/parser/pdf/batch_smoke_test.go b/internal/deepdoc/parser/pdf/batch_smoke_test.goindex 6b8678d7bc..e25e6f99d5 100644--- a/internal/deepdoc/parser/pdf/batch_smoke_test.go+++ b/internal/deepdoc/parser/pdf/batch_smoke_test.go@@ -21,6 +21,7 @@ import (inf "ragflow/internal/deepdoc/parser/pdf/inference"lyt "ragflow/internal/deepdoc/parser/pdf/layout"+ "ragflow/internal/deepdoc/parser/pdf/table""ragflow/internal/deepdoc/parser/pdf/tool"pdf "ragflow/internal/deepdoc/parser/pdf/type")@@ -91,7 +92,7 @@ func variantFromEnv() string {}type outputDirs struct {- text, tables, dla string+ text, tables, dla, tsr string}func mkOutputDirs(variant string) outputDirs {@@ -99,10 +100,12 @@ func mkOutputDirs(variant string) outputDirs {text: filepath.Join("testdata", "output", "go", variant, "text"),tables: filepath.Join("testdata", "output", "go", variant, "tables"),dla: filepath.Join("testdata", "output", "go", variant, "dla"),+ tsr: filepath.Join("testdata", "output", "go", variant, "tsr_raw"),}os.MkdirAll(d.text, 0755)os.MkdirAll(d.tables, 0755)os.MkdirAll(d.dla, 0755)+ os.MkdirAll(d.tsr, 0755)return d}@@ -344,9 +347,68 @@ func writeOutputs(dirs outputDirs, name string, parsed *pdf.ParseResult, res *pa}// ── DLA layout intermediates ──+ // DLA dump: post-filter regions (NMS + confidence filter + Y-sort ++ // cleanup), matching Python's page_layout. This makes the Go dump+ // comparable with the Python parity dump, which also writes post-filter+ // regions.+ //+ // Coordinate space: Go dumps image-pixel coordinates (no scale division;+ // see FilteredDLARegions), whereas Python's page_layout divides by+ // scale_factor (typically 3, layout_recognizer.py:90-93). So the two+ // dumps differ by ~3x in absolute coordinates — this is expected, and the+ // parity comparison is count-only (CompareDLAWithPython), so it does not+ // affect the match.+ //+ // Real DLA regions always carry a score, so cleanupLayouts' box-fallback+ // never triggers and nil boxes are safe here (count-wise). If a score-0+ // region were ever emitted, the area tie-break would fall back to keeping+ // the first region rather than mirroring Python's area-based choice.if parsed.DLARegions != nil {- if b, _ := json.MarshalIndent(parsed.DLARegions, "", " "); b != nil {+ filteredPages := make([]pdf.DLAPageRegions, 0, len(parsed.DLARegions))+ for _, pr := range parsed.DLARegions {+ filteredPages = append(filteredPages, pdf.DLAPageRegions{+ Page: pr.Page,+ Regions: table.FilteredDLARegions(pr.Regions, nil),+ })+ }+ if b, _ := json.MarshalIndent(filteredPages, "", " "); b != nil {os.WriteFile(filepath.Join(dirs.dla, name+".json"), b, 0644)}}++ // ── TSR raw cells ── (matching Python's tsr_raw dump from parser.tb_cpns).+ // Flat list of per-cell records so CompareTSRRawWithPython can diff labels+ // and table counts against the Python reference.+ type tsrRawCellDump struct {+ TableIndex int `json:"table_index"`+ Page int `json:"page"`+ Label string `json:"label"`+ X0 float64 `json:"x0"`+ Y0 float64 `json:"y0"`+ X1 float64 `json:"x1"`+ Y1 float64 `json:"y1"`+ Text string `json:"text"`+ }+ var tsrCells []tsrRawCellDump+ for ti, t := range parsed.Tables {+ page := 0+ if len(t.Positions) > 0 && len(t.Positions[0].PageNumbers) > 0 {+ page = t.Positions[0].PageNumbers[0]+ }+ for _, c := range t.Cells {+ tsrCells = append(tsrCells, tsrRawCellDump{+ TableIndex: ti,+ Page: page,+ Label: c.Label,+ X0: c.X0,+ Y0: c.Y0,+ X1: c.X1,+ Y1: c.Y1,+ Text: c.Text,+ })+ }+ }+ if b, _ := json.MarshalIndent(tsrCells, "", " "); b != nil {+ os.WriteFile(filepath.Join(dirs.tsr, name+".json"), b, 0644)+ }}diff --git a/internal/deepdoc/parser/pdf/table/table_annotate.go b/internal/deepdoc/parser/pdf/table/table_annotate.goindex 87d3719cd6..7335e61ca0 100644--- a/internal/deepdoc/parser/pdf/table/table_annotate.go+++ b/internal/deepdoc/parser/pdf/table/table_annotate.go@@ -3,6 +3,7 @@ package tableimport ("fmt""math"+ "sort"pdf "ragflow/internal/deepdoc/parser/pdf/type""ragflow/internal/deepdoc/parser/pdf/util"@@ -57,63 +58,300 @@ func MatchTableRegions(boxes []pdf.TextBox, regions []pdf.DLARegion, scale float// ── layout annotation ──────────────────────────────────────────────────-// annotateBoxLayouts sets LayoutType and LayoutNo on each box, matching+// annRegion is a layout region in PDF space, used internally by+// AnnotateBoxLayouts. It carries the fields needed for cleanup, sort, and+// annotation bookkeeping.+type annRegion struct {+ x0, y0, x1, y1 float64+ label string+ score float64+ visited bool+ typeIndex int+}++// regionIntersect returns the intersection area of two regions, or 0.+func regionIntersect(a, b annRegion) float64 {+ ix0 := math.Max(a.x0, b.x0)+ iy0 := math.Max(a.y0, b.y0)+ ix1 := math.Min(a.x1, b.x1)+ iy1 := math.Min(a.y1, b.y1)+ if ix0 < ix1 && iy0 < iy1 {+ return (ix1 - ix0) * (iy1 - iy0)+ }+ return 0+}++// regionArea returns the area of a region, or 0 if degenerate.+func regionArea(a annRegion) float64 {+ w := a.x1 - a.x0+ h := a.y1 - a.y0+ if w <= 0 || h <= 0 {+ return 0+ }+ return w * h+}++// overlapRatio returns intersection / area(a), matching+// Recognizer.overlapped_area with ratio=True (recognizer.py:106-122).+func overlapRatio(a, b annRegion) float64 {+ ar := regionArea(a)+ if ar <= 0 {+ return 0+ }+ return regionIntersect(a, b) / ar+}++// annNotOverlapped mirrors recognizer.py:126-127 (annRegion variant).+func annNotOverlapped(a, b annRegion) bool {+ return a.x1 < b.x0 || a.x0 > b.x1 || a.y1 < b.y0 || a.y0 > b.y1+}++func imin(a, b int) int {+ if a < b {+ return a+ }+ return b+}++// sortYFirstly orders regions top-to-bottom (and left-to-right within a+// vertical threshold), matching Recognizer.sort_Y_firstly (recognizer.py:54)+// which LayoutRecognizer.__call__ applies before annotation+// (layout_recognizer.py:99).+func sortYFirstly(regs []annRegion) {+ if len(regs) == 0 {+ return+ }+ avgH := 0.0+ for _, r := range regs {+ avgH += r.y1 - r.y0+ }+ thr := avgH / float64(len(regs)) / 2+ sort.SliceStable(regs, func(i, j int) bool {+ di := regs[i].y0 - regs[j].y0+ if di < -thr {+ return true+ }+ if di > thr {+ return false+ }+ return regs[i].x0 < regs[j].x0+ })+}++// cleanupLayouts de-duplicates overlapping same-type regions, matching+// Recognizer.layouts_cleanup (recognizer.py:124) called by+// LayoutRecognizer.__call__ at layout_recognizer.py:100. A pair of same-type+// regions whose overlap exceeds thr (0.7) in either direction collapses to a+// single region: the higher-score one, or - when scores are absent - the one+// covering more text-box area. far=2 limits comparison to nearby regions.+func cleanupLayouts(regs []annRegion, boxes []pdf.TextBox) []annRegion {+ const far = 2+ const thr = 0.7+ i := 0+ for i+1 < len(regs) {+ j := i + 1+ for j < imin(i+far, len(regs)) && (regs[j].label != regs[i].label || annNotOverlapped(regs[i], regs[j])) {+ j+++ }+ if j >= imin(i+far, len(regs)) {+ i+++ continue+ }+ if overlapRatio(regs[i], regs[j]) < thr && overlapRatio(regs[j], regs[i]) < thr {+ i+++ continue+ }+ // Collapse the pair. Python layouts_cleanup keeps the HIGHER-score+ // region; on equal scores it keeps the later one (j) via pop(i).+ // Match that exactly so equal-confidence pairs converge with Python.+ drop := j+ if regs[i].score > 0 && regs[j].score > 0 {+ if regs[i].score > regs[j].score {+ drop = j+ } else {+ drop = i+ }+ } else {+ areaI, areaJ := 0.0, 0.0+ for _, b := range boxes {+ tb := annRegion{x0: b.X0, y0: b.Top, x1: b.X1, y1: b.Bottom}+ if !annNotOverlapped(tb, regs[i]) {+ areaI += regionIntersect(tb, regs[i])+ }+ if !annNotOverlapped(tb, regs[j]) {+ areaJ += regionIntersect(tb, regs[j])+ }+ }+ if areaJ > areaI {+ drop = i+ }+ }+ regs = append(regs[:drop], regs[drop+1:]...)+ }+ return regs+}++// nmsDLARegions applies per-class non-maximum suppression to raw DLA regions,+// mirroring Python's layout model postprocess (layout_recognizer.py:246,+// operators.py:667 nms with iou_thresh). For each label, detections are sorted+// by confidence descending; the top one is kept and any other same-label+// detection whose IoU (using the +1 overlap convention from operators.py:685)+// exceeds iouThresh is suppressed. This runs on the raw, pre-scale detections+// just as Python's postprocess does, before cleanup/annotation.+//+// It is idempotent with a server-side NMS: applying it again to already-suppressed+// boxes yields the same set, so it safely converges Go to Python regardless of+// where suppression happens upstream.+func nmsDLARegions(regions []pdf.DLARegion, iouThresh float64) []pdf.DLARegion {+ if len(regions) == 0 {+ return regions+ }+ byLabel := map[string][]int{}+ for i, r := range regions {+ byLabel[r.Label] = append(byLabel[r.Label], i)+ }+ suppressed := make([]bool, len(regions))+ for _, idxs := range byLabel {+ // Highest confidence first (greedy NMS keeps the top box). On equal+ // confidence, break the tie by original index so the result is+ // deterministic for identical inputs — otherwise the same PDF could+ // keep a different region (and emit different LayoutNo) on each run.+ sort.SliceStable(idxs, func(a, b int) bool {+ ca, cb := regions[idxs[a]].Confidence, regions[idxs[b]].Confidence+ if ca != cb {+ return ca > cb+ }+ return idxs[a] < idxs[b]+ })+ for k := 0; k < len(idxs); k++ {+ i := idxs[k]+ if suppressed[i] {+ continue+ }+ for m := k + 1; m < len(idxs); m++ {+ j := idxs[m]+ if suppressed[j] {+ continue+ }+ if nmsIoU(regions[i], regions[j]) > iouThresh {+ suppressed[j] = true+ }+ }+ }+ }+ out := make([]pdf.DLARegion, 0, len(regions))+ for i, r := range regions {+ if !suppressed[i] {+ out = append(out, r)+ }+ }+ return out+}++// nmsIoU computes IoU using the +1 overlap convention from operators.py:685-688+// (w = max(0, x22-x11+1), h = max(0, y22-y11+1)) while area uses no +1. This+// must match Python exactly so borderline suppressions (around the 0.45 threshold)+// align.+func nmsIoU(a, b pdf.DLARegion) float64 {+ w := math.Max(0, math.Min(a.X1, b.X1)-math.Max(a.X0, b.X0)+1)+ h := math.Max(0, math.Min(a.Y1, b.Y1)-math.Max(a.Y0, b.Y0)+1)+ inter := w * h+ areaA := (a.X1 - a.X0) * (a.Y1 - a.Y0)+ areaB := (b.X1 - b.X0) * (b.Y1 - b.Y0)+ if areaA <= 0 || areaB <= 0 {+ return 0+ }+ return inter / (areaA + areaB - inter)+}++// AnnotateBoxLayouts sets LayoutType and LayoutNo on each box, matching// Python's LayoutRecognizer.__call__ which assigns layout types in priority-// order (footer→header→…→equation) with an overlap threshold of 40% of the+// order (footer->header->...->equation) with an overlap threshold of 40% of the// box's area.//-// Python: _layouts_rec (pdf_parser.py:827) → LayoutRecognizer.__call__ →+// Python: _layouts_rec (pdf_parser.py:827) -> LayoutRecognizer.__call__ ->//// for lt in priority_order: findLayout(lt)//// Each findLayout(ty): for each unannotated box, find the DLA region of-// type ty with max overlap ≥ 0.4 × box_area. First type to match wins.+// type ty with max overlap >= 0.4 * box_area. First type to match wins.//// CID-pattern boxes (e.g. "(cid:123)") are skipped as garbage.-// annotateBoxLayouts assigns LayoutType and LayoutNo to boxes based on DLA+// AnnotateBoxLayouts assigns LayoutType and LayoutNo to boxes based on DLA// regions. Returns the filtered slice (Python pops CID-garbled boxes and-// garbage-layout boxes at wrong positions — Go mirrors with compact).+// garbage-layout boxes at wrong positions - Go mirrors with compact).// Also creates synthetic figure boxes for unmatched figure/equation regions.-func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale float64, pageImgHeight float64) []pdf.TextBox {+//+// Before annotation, regions are de-duplicated (layouts_cleanup) and sorted+// top-to-bottom (sort_Y_firstly) to match Python, and unmatched figure and+// equation regions receive SEPARATE synthetic namespaces (figure-N /+// equation-N) so they never collide.+// FilteredDLARegions returns the DLA regions after per-class NMS, the+// confidence filter, Y-sort, and cleanup — i.e. the exact set fed to+// annotation. It mirrors Python's page_layout (layout_recognizer.py:84-100):+// - nmsDLARegions(0.45) == layout model postprocess (operators.py:667)+// - keep if score >= 0.4 OR type not garbage == layout_recognizer.py:97+// - sortYFirstly == layout_recognizer.py:99 (sort_Y_firstly)+// - cleanupLayouts == layout_recognizer.py:100 (layouts_cleanup)+//+// Regions are returned in image-pixel space (no scale division) so callers+// that only need the region set — e.g. the parity harness dumping post-filter+// regions for comparison with Python's page_layout — get a stable comparison+// point regardless of render DPI. cleanupLayouts only consults boxes when both+// compared regions have score 0, which never happens for real DLA output, so+// passing nil boxes from the harness is safe.+func FilteredDLARegions(regions []pdf.DLARegion, boxes []pdf.TextBox) []pdf.DLARegion {+ regions = nmsDLARegions(regions, 0.45)if len(regions) == 0 {- return boxes+ return nil+ }+ kept := regions[:0]+ for _, r := range regions {+ if r.Confidence >= 0.4 || !isGarbageLayoutType(r.Label) {+ kept = append(kept, r)+ }}+ ars := make([]annRegion, len(kept))+ for i, r := range kept {+ ars[i] = annRegion{x0: r.X0, y0: r.Y0, x1: r.X1, y1: r.Y1, label: r.Label, score: r.Confidence}+ }+ sortYFirstly(ars)+ ars = cleanupLayouts(ars, boxes)+ out := make([]pdf.DLARegion, len(ars))+ for i, a := range ars {+ out[i] = pdf.DLARegion{X0: a.x0, Y0: a.y0, X1: a.x1, Y1: a.y1, Label: a.label, Confidence: a.score}+ }+ return out+}- // Scale all regions to PDF space once.- type scaledRegion struct {- x0, y0, x1, y1 float64- label string+func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale float64, pageImgHeight float64) []pdf.TextBox {+ // NMS, confidence filter, Y-sort, and cleanup — the exact region set fed+ // to annotation. This mirrors Python's layout_recognizer.py:84-100+ // (filter + sort_Y_firstly + layouts_cleanup) and is the same pipeline the+ // parity harness uses (via FilteredDLARegions) to dump post-filter regions+ // comparable with Python's page_layout.+ filtered := FilteredDLARegions(regions, boxes)+ if len(filtered) == 0 {+ return boxes}- scaled := make([]scaledRegion, len(regions))- for i, r := range regions {- scaled[i] = scaledRegion{++ // Scale filtered regions from image-pixel space to PDF space.+ cands := make([]annRegion, 0, len(filtered))+ for _, r := range filtered {+ cands = append(cands, annRegion{x0: r.X0 / scale, y0: r.Y0 / scale,x1: r.X1 / scale, y1: r.Y1 / scale,- label: r.Label,- }- }-- // DLA confidence filter — matches Python's `score >= 0.4`.- regionOK := make([]bool, len(regions))- for i, r := range regions {- regionOK[i] = r.Confidence >= 0.4 || !isGarbageLayoutType(r.Label)+ label: r.Label, score: r.Confidence,+ })}- // Pre-compute per-type index for each region (Python: matched index within- // filtered layouts_of_type list). "text" regions get 0,1,2... independent- // of "figure" regions.- typeIndex := make([]int, len(regions))+ // Per-type index in the cleaned, Y-sorted list (Python: ii in lts_).typeCounters := make(map[string]int)- for j, r := range scaled {- if regionOK[j] {- typeIndex[j] = typeCounters[r.label]- typeCounters[r.label]++- }+ for j := range cands {+ cands[j].typeIndex = typeCounters[cands[j].label]+ typeCounters[cands[j].label]++}- // Track visited regions (Python: layout["visited"] = True).- visited := make([]bool, len(regions))-// Marks for Python-style pop removal.dropped := make([]bool, len(boxes))@@ -139,9 +377,10 @@ func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale floacontinue}bestOverlap := 0.0+ bestRegionOverlap := 0.0bestJ := -1- for j, r := range scaled {- if r.label != ty || !regionOK[j] {+ for j, r := range cands {+ if r.label != ty {continue}ix0 := math.Max(r.x0, boxes[i].X0)@@ -149,20 +388,32 @@ func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale floaix1 := math.Min(r.x1, boxes[i].X1)iy1 := math.Min(r.y1, boxes[i].Bottom)if ix0 < ix1 && iy0 < iy1 {- ov := (ix1 - ix0) * (iy1 - iy0) / boxArea- if ov > bestOverlap {+ inter := (ix1 - ix0) * (iy1 - iy0)+ ov := inter / boxArea // fraction of the box covered (Python's ov)+ rArea := (r.x1 - r.x0) * (r.y1 - r.y0)+ ovRegion := 0.0+ if rArea > 0 {+ ovRegion = inter / rArea // fraction of the region covered (Python's _ov)+ }+ // Mirror Python's (ov, _ov) tuple comparison+ // (recognizer.py:255-269): primary key is the box+ // coverage ratio; on a tie the region-coverage ratio+ // wins (prefer the region the box sits more "inside"+ // of); on a full tie keep the first (topmost) region.+ if ov > bestOverlap || (ov == bestOverlap && ovRegion > bestRegionOverlap) {bestOverlap = ov+ bestRegionOverlap = ovRegionbestJ = j}}}if bestJ >= 0 && bestOverlap >= 0.4 {- // Garbage layout not at page edge → pop (Python: bxs.pop(i)).+ // Garbage layout not at page edge -> pop (Python: bxs.pop(i)).if isGarbageLayoutType(ty) && pageImgHeight > 0 && !garbageKeepFeat(ty, boxes[i], pageImgHeight/scale) {dropped[i] = truecontinue}- visited[bestJ] = true+ cands[bestJ].visited = true// Python: equation mapped to "figure" for layout_typeif ty == pdf.LayoutTypeEquation {boxes[i].LayoutType = pdf.LayoutTypeFigure@@ -170,7 +421,7 @@ func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale floaboxes[i].LayoutType = ty}// Python: f"{layout_type}-{matched}" where matched is per-type index- boxes[i].LayoutNo = fmt.Sprintf("%s-%d", ty, typeIndex[bestJ])+ boxes[i].LayoutNo = fmt.Sprintf("%s-%d", ty, cands[bestJ].typeIndex)}}}@@ -196,25 +447,28 @@ func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale floaboxes = compacted// Synthetic figure boxes for unmatched figure/equation regions (Python:- // dla_cli.py:187-195). Use a fresh per-type counter for synthetic boxes.- synthIdx := 0- for j, r := range scaled {- if !regionOK[j] || visited[j] {+ // layout_recognizer.py:145-155). Python numbers each unmatched region with+ // its index WITHIN the per-type list that also includes already-visited+ // regions (enumerate([lt for lt in lts if lt["type"] == ty])), so we reuse+ // the per-type typeIndex computed above rather than a separate+ // unvisited-only counter. Python keeps figure-N / equation-N in SEPARATE+ // namespaces, so the typeIndex is keyed by the original type label.+ for j := range cands {+ if cands[j].visited {continue}- if r.label != pdf.LayoutTypeFigure && r.label != pdf.LayoutTypeEquation {+ if cands[j].label != pdf.LayoutTypeFigure && cands[j].label != pdf.LayoutTypeEquation {continue}boxes = append(boxes, pdf.TextBox{- X0: r.x0,- X1: r.x1,- Top: r.y0,- Bottom: r.y1,+ X0: cands[j].x0,+ X1: cands[j].x1,+ Top: cands[j].y0,+ Bottom: cands[j].y1,Text: "",LayoutType: pdf.LayoutTypeFigure,- LayoutNo: fmt.Sprintf("figure-%d", synthIdx),+ LayoutNo: fmt.Sprintf("%s-%d", cands[j].label, cands[j].typeIndex),})- synthIdx++}return boxes@@ -232,7 +486,7 @@ func isGarbageLayoutType(ty string) bool {// garbageKeepFeat matches Python's keep_feats in LayoutRecognizer.__call__:// footer near page bottom (>90% of page height) or header near page top (<10%)-// are real page decorations — keep them. Others are DLA noise.+// are real page decorations - keep them. Others are DLA noise.func garbageKeepFeat(ty string, box pdf.TextBox, pageImgHeight float64) bool {switch ty {case pdf.LayoutTypeFooter:diff --git a/internal/deepdoc/parser/pdf/table/table_annotate_parity_test.go b/internal/deepdoc/parser/pdf/table/table_annotate_parity_test.gonew file mode 100644index 0000000000..23cb815e1a--- /dev/null+++ b/internal/deepdoc/parser/pdf/table/table_annotate_parity_test.go@@ -0,0 +1,297 @@+// Tests pinning the Python-equivalent behavior of AnnotateBoxLayouts.+//+// These assert the layout-annotation semantics that Python's+// LayoutRecognizer.__call__ (deepdoc/vision/layout_recognizer.py:68) produces,+// and that the Go implementation now replicates (GREEN):+//+// - #1 layouts_cleanup: Python de-dupes overlapping same-type regions+// (recognizer.py:124-160, called at layout_recognizer.py:100) BEFORE+// annotation. Go matches via cleanupLayouts, so no extra synthetic+// figure/equation boxes are emitted.+// - #2 sort_Y_firstly: Python sorts regions top-to-bottom (recognizer.py:54,+// layout_recognizer.py:99) before numbering them, so layoutno indices+// follow reading order. Go matches via sortYFirstly.+// - #3 synthetic namespace: Python numbers unmatched figure/equation regions+// in SEPARATE counters -> "figure-N" / "equation-N"+// (layout_recognizer.py:145-155). Go matches via a per-type typeIndex+// keyed by the original type label.+//+// These are pure-Go unit tests (no model server, no external service): they+// feed crafted pdf.DLARegion slices and assert the annotated pdf.TextBox+// output. Coordinates use scale=1 so region pixels == PDF space.++package table++import (+ "testing"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// countFigureBoxes returns the number of boxes whose LayoutType is "figure".+func countFigureBoxes(boxes []pdf.TextBox) int {+ n := 0+ for _, b := range boxes {+ if b.LayoutType == pdf.LayoutTypeFigure {+ n+++ }+ }+ return n+}++// countSyntheticFigures returns the number of figure boxes with no text+// (the synthetic placeholders AnnotateBoxLayouts appends for unvisited+// figure/equation regions).+func countSyntheticFigures(boxes []pdf.TextBox) int {+ n := 0+ for _, b := range boxes {+ if b.LayoutType == pdf.LayoutTypeFigure && b.Text == "" {+ n+++ }+ }+ return n+}++// TestAnnotateBoxLayouts_DuplicateFigureRegions_Merged pins #1: two heavily+// overlapping same-type (figure) regions must be treated as ONE, so only one+// figure box (the annotated text box) results and NO synthetic figure box is+// produced. Python merges via layouts_cleanup; Go now mirrors that by fusing+// the overlapping pair into one region, so the duplicate synthetic figure is+// avoided.+func TestAnnotateBoxLayouts_DuplicateFigureRegions_Merged(t *testing.T) {+ box := pdf.TextBox{X0: 0, X1: 100, Top: 0, Bottom: 50, Text: "Some caption text", PageNumber: 0}+ regions := []pdf.DLARegion{+ {X0: 0, Y0: 0, X1: 100, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeFigure},+ {X0: 0, Y0: 0, X1: 100, Y1: 50, Confidence: 0.8, Label: pdf.LayoutTypeFigure}, // identical -> IoU 1.0+ }++ out := AnnotateBoxLayouts([]pdf.TextBox{box}, regions, 1.0, 100.0)++ if got := countFigureBoxes(out); got != 1 {+ t.Errorf("#1 layouts_cleanup: expected 1 figure box after de-duplicating overlapping figure regions, got %d", got)+ }+ if got := countSyntheticFigures(out); got != 0 {+ t.Errorf("#1 layouts_cleanup: expected 0 synthetic figure boxes (duplicate region must be merged), got %d", got)+ }+}++// TestAnnotateBoxLayouts_SameTypeOutOfYOrder_LayoutNoFollowsY pins #2: when+// same-type regions arrive in non-reading order (here the bottom region is+// first in the wire list), the layoutno index must still follow top-to-bottom+// order, exactly like Python's sort_Y_firstly. The box unambiguously overlaps+// only the bottom region, so the winner is the same in both languages; only+// the assigned index differs.+func TestAnnotateBoxLayouts_SameTypeOutOfYOrder_LayoutNoFollowsY(t *testing.T) {+ // Wire order: bottom region first (NOT top-to-bottom).+ regions := []pdf.DLARegion{+ {X0: 0, Y0: 30, X1: 100, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeTitle}, // bottom strip+ {X0: 0, Y0: 0, X1: 100, Y1: 20, Confidence: 0.9, Label: pdf.LayoutTypeTitle}, // top strip+ }+ // Box overlaps only the bottom region -> unambiguous winner = bottom region.+ box := pdf.TextBox{X0: 0, X1: 100, Top: 30, Bottom: 50, Text: "Title text", PageNumber: 0}++ out := AnnotateBoxLayouts([]pdf.TextBox{box}, regions, 1.0, 100.0)++ var annotated *pdf.TextBox+ for i := range out {+ if out[i].Text == "Title text" {+ annotated = &out[i]+ }+ }+ if annotated == nil {+ t.Fatal("#2 sort_Y_firstly: text box was not annotated as title")+ }+ // Python Y-sorts -> [top(ii=0), bottom(ii=1)] -> winner bottom => "title-1".+ if annotated.LayoutNo != "title-1" {+ t.Errorf("#2 sort_Y_firstly: expected layoutno \"title-1\" (top-to-bottom numbering), got %q", annotated.LayoutNo)+ }+}++// TestAnnotateBoxLayouts_SyntheticFigureEquation_SeparateNamespaces pins #3:+// an unmatched figure region and an unmatched equation region must yield+// distinct synthetic layoutnos "figure-0" and "equation-0". Go now keeps+// per-type synthetic counters so figures and equations are numbered+// independently instead of sharing a single figure-N counter.+func TestAnnotateBoxLayouts_SyntheticFigureEquation_SeparateNamespaces(t *testing.T) {+ regions := []pdf.DLARegion{+ {X0: 0, Y0: 0, X1: 50, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeFigure},+ {X0: 60, Y0: 0, X1: 110, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeEquation},+ }++ out := AnnotateBoxLayouts([]pdf.TextBox{}, regions, 1.0, 100.0)++ layoutNos := map[string]bool{}+ for _, b := range out {+ layoutNos[b.LayoutNo] = true+ }+ if !layoutNos["figure-0"] {+ t.Errorf("#3 synthetic namespace: expected a synthetic box with layoutno \"figure-0\"")+ }+ if !layoutNos["equation-0"] {+ t.Errorf("#3 synthetic namespace: expected a synthetic box with layoutno \"equation-0\" (separate counter from figure); equation was folded into the shared figure counter")+ }+}++// TestAnnotateBoxLayouts_TieBreakRegionCoverage pins #4: when two same-type+// regions cover the box by the SAME fraction (ov tie), Python's+// find_overlapped_with_threshold (recognizer.py:255-269) breaks the tie by the+// region-coverage ratio (_ov = box∩region / region area), preferring the region+// the box sits more "inside" of. Go mirrors the (ov, _ov) tuple comparison so+// the higher-_ov region wins the tie instead of the first Y-sorted candidate.+//+// Layout: wide top region A and narrow bottom region B. The box spans both+// vertically with EQUAL absolute intersection, so ov_A == ov_B. B is smaller,+// so _ov_B > _ov_A and Python picks B. In Y-sorted order [A, B], B is per-type+// index 1, so the expected layoutno is "table-1".+func TestAnnotateBoxLayouts_TieBreakRegionCoverage(t *testing.T) {+ regions := []pdf.DLARegion{+ {X0: 0, Y0: 0, X1: 200, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeTable}, // A: wide, top+ {X0: 0, Y0: 60, X1: 50, Y1: 110, Confidence: 0.9, Label: pdf.LayoutTypeTable}, // B: narrow, bottom+ }+ box := pdf.TextBox{X0: 0, X1: 50, Top: 0, Bottom: 110, Text: "tbl", PageNumber: 0}++ out := AnnotateBoxLayouts([]pdf.TextBox{box}, regions, 1.0, 0)++ var annotated *pdf.TextBox+ for i := range out {+ if out[i].Text == "tbl" {+ annotated = &out[i]+ }+ }+ if annotated == nil {+ t.Fatal("#4 tie-break: box not annotated as table")+ }+ if annotated.LayoutNo != "table-1" {+ t.Errorf("#4 tie-break: expected layoutno \"table-1\" (smaller region B wins the _ov tie), got %q", annotated.LayoutNo)+ }+}++// TestAnnotateBoxLayouts_NMSDedupOverlappingRegions pins #6: Python's layout+// model postprocess applies per-class NMS with IoU 0.45 (layout_recognizer.py:246,+// operators.py:667) on the RAW detections BEFORE annotation. Go must do the same+// on the raw regions; otherwise same-label detections overlapping between 0.45+// and 0.7 survive (cleanupLayouts only merges at thr=0.7) and emit extra+// synthetic figure boxes.+//+// Two same-label figure regions with IoU ~0.54: >0.45 so NMS suppresses the+// lower-score one; <0.7 so cleanupLayouts would NOT merge them. With NMS there+// is exactly one figure region -> one synthetic figure box.+func TestAnnotateBoxLayouts_NMSDedupOverlappingRegions(t *testing.T) {+ regions := []pdf.DLARegion{+ {X0: 0, Y0: 0, X1: 100, Y1: 100, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // A+ {X0: 35, Y0: 0, X1: 135, Y1: 100, Confidence: 0.8, Label: pdf.LayoutTypeFigure}, // B overlaps A (shift 35)+ }+ // No text box overlaps -> without NMS both become synthetic figures.+ // Overlap is ~0.65 (no-+1 IoU) so cleanupLayouts (thr=0.7) does NOT merge,+ // but Python's NMS uses +1 IoU (~0.50) > 0.45 and suppresses the lower-score B.+ out := AnnotateBoxLayouts([]pdf.TextBox{}, regions, 1.0, 100.0)++ if got := countFigureBoxes(out); got != 1 {+ t.Errorf("#6 NMS: expected 1 figure box after per-class NMS merges overlapping detections (IoU ~0.5), got %d", got)+ }+}++// TestAnnotateBoxLayouts_NMSDeterministic pins that equal-confidence+// same-label detections collapse to the SAME region on every run. Before the+// sort.SliceStable + original-index tie-break, sort.Slice ordered equal scores+// nondeterministically, so the survivor (and its synthetic LayoutNo) could flip+// between otherwise-identical runs — breaking reproducibility and the golden+// comparison.+func TestAnnotateBoxLayouts_NMSDeterministic(t *testing.T) {+ regions := []pdf.DLARegion{+ {X0: 0, Y0: 0, X1: 100, Y1: 100, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // A (lower original index)+ {X0: 35, Y0: 0, X1: 135, Y1: 100, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // B overlaps A, equal score+ }+ var baseline []pdf.TextBox+ for run := 0; run < 30; run++ {+ out := AnnotateBoxLayouts([]pdf.TextBox{}, regions, 1.0, 100.0)+ if got := countFigureBoxes(out); got != 1 {+ t.Fatalf("run %d: expected exactly 1 figure box after NMS, got %d", run, got)+ }+ if run == 0 {+ baseline = out+ continue+ }+ if len(out) != len(baseline) {+ t.Fatalf("run %d: box count changed across runs (%d vs %d)", run, len(out), len(baseline))+ }+ for i := range out {+ if out[i].X0 != baseline[i].X0 || out[i].X1 != baseline[i].X1 ||+ out[i].Top != baseline[i].Top || out[i].Bottom != baseline[i].Bottom ||+ out[i].LayoutNo != baseline[i].LayoutNo {+ t.Fatalf("run %d: nondeterministic output (box %d differs from run 0)", run, i)+ }+ }+ }+ // Lower-index tie-break keeps region A.+ if baseline[0].X0 != 0 || baseline[0].X1 != 100 {+ t.Errorf("expected the lower-index region A (x0=0,x1=100) to survive, got x0=%v,x1=%v", baseline[0].X0, baseline[0].X1)+ }+}++// TestAnnotateBoxLayouts_CleanupEqualScoreKeepsLater pins that when two+// same-type regions overlap beyond cleanup's 0.7 threshold with EQUAL+// confidence, Go keeps the LATER one (j) — matching Python layouts_cleanup+// (pop(i) on equal scores). Before the fix Go kept the earlier region.+//+// NMS (IoU 0.45) must not pre-remove either: A sits fully inside B, so their+// +1 IoU is tiny (<0.45) while their overlap ratio (inter/area of B) exceeds+// 0.7, so only cleanup is exercised.+func TestAnnotateBoxLayouts_CleanupEqualScoreKeepsLater(t *testing.T) {+ regions := []pdf.DLARegion{+ {X0: 0, Y0: 0, X1: 400, Y1: 400, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // B: large, on top (Y0=0)+ {X0: 100, Y0: 200, X1: 200, Y1: 300, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // A: small, inside B, lower (Y0=200)+ }+ // No text box overlaps -> both unvisited -> cleanup collapses to one synthetic.+ out := AnnotateBoxLayouts([]pdf.TextBox{}, regions, 1.0, 100.0)+ if got := countFigureBoxes(out); got != 1 {+ t.Fatalf("expected 1 figure box after cleanup merge, got %d", got)+ }+ // Y-sort puts B (Y0=0) first, A (Y0=200) second; equal score -> keep later (A).+ if out[0].X0 != 100 || out[0].X1 != 200 || out[0].Top != 200 || out[0].Bottom != 300 {+ t.Errorf("cleanup equal-score: expected later region A (x0=100,x1=200,top=200,bottom=300) to survive, got x0=%v,x1=%v,top=%v,bottom=%v",+ out[0].X0, out[0].X1, out[0].Top, out[0].Bottom)+ }+}++// TestAnnotateBoxLayouts_SyntheticVisitedInterleaved pins that an unmatched+// figure region is numbered by its index in the per-type list that ALSO+// includes already-visited figure regions (Python's enumerate over the full+// type-filtered list). Before the fix Go used a separate unvisited-only+// counter, so the unmatched figure became figure-0 instead of figure-1.+func TestAnnotateBoxLayouts_SyntheticVisitedInterleaved(t *testing.T) {+ boxes := []pdf.TextBox{+ {X0: 0, X1: 100, Top: 0, Bottom: 50, Text: "caption for A"},+ }+ regions := []pdf.DLARegion{+ {X0: 0, Y0: 0, X1: 100, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // A: matched to text box -> visited+ {X0: 200, Y0: 0, X1: 300, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // B: unmatched -> synthetic+ }+ out := AnnotateBoxLayouts(boxes, regions, 1.0, 100.0)++ var annotated, synthetic *pdf.TextBox+ for i := range out {+ if out[i].Text == "caption for A" {+ annotated = &out[i]+ }+ if out[i].LayoutType == pdf.LayoutTypeFigure && out[i].Text == "" {+ synthetic = &out[i]+ }+ }+ if annotated == nil {+ t.Fatal("visited figure A: text box was not annotated as figure")+ }+ if annotated.LayoutNo != "figure-0" {+ t.Errorf("visited figure A: expected LayoutNo figure-0, got %q", annotated.LayoutNo)+ }+ if synthetic == nil {+ t.Fatal("expected a synthetic figure box for unmatched B")+ }+ // B is the 2nd figure in Y order (after visited A), so Python numbers it figure-1.+ if synthetic.LayoutNo != "figure-1" {+ t.Errorf("unmatched figure B: expected figure-1 (index in full per-type list), got %q", synthetic.LayoutNo)+ }+ if synthetic.X0 != 200 || synthetic.X1 != 300 {+ t.Errorf("unmatched figure B: expected x0=200,x1=300, got x0=%v,x1=%v", synthetic.X0, synthetic.X1)+ }+}diff --git a/internal/deepdoc/parser/pdf/table/table_annotate_test.go b/internal/deepdoc/parser/pdf/table/table_annotate_test.goindex 3ac34468f4..6ffd615b99 100644--- a/internal/deepdoc/parser/pdf/table/table_annotate_test.go+++ b/internal/deepdoc/parser/pdf/table/table_annotate_test.go@@ -154,6 +154,34 @@ func TestAnnotateBoxLayouts_ConfidenceFilter(t *testing.T) {}}+// TestFilteredDLARegions pins the post-filter region set the parity harness+// dumps for comparison with Python's page_layout. The confidence filter keeps+// a region when score >= 0.4 OR its type is not garbage — so a low-confidence+// *non-garbage* region (e.g. text at 0.1) is KEPT, exactly matching Python's+// `score >= 0.4 or type not in garbage_layouts` (layout_recognizer.py:97).+// Returned regions stay in image-pixel space (no scale division).+func TestFilteredDLARegions(t *testing.T) {+ regions := []pdf.DLARegion{+ {X0: 0, Y0: 0, X1: 300, Y1: 150, Label: "footer", Confidence: 0.2}, // low-conf garbage → dropped+ {X0: 0, Y0: 200, X1: 300, Y1: 350, Label: "text", Confidence: 0.1}, // low-conf non-garbage → kept+ {X0: 0, Y0: 400, X1: 500, Y1: 460, Label: "reference", Confidence: 0.5}, // >=0.4 garbage → kept+ }+ got := FilteredDLARegions(regions, nil)+ if len(got) != 2 {+ t.Fatalf("FilteredDLARegions() = %d regions, want 2 (got %+v)", len(got), got)+ }+ labels := map[string]bool{}+ for _, r := range got {+ labels[r.Label] = true+ if r.Confidence == 0.2 {+ t.Errorf("low-confidence footer should have been filtered out")+ }+ }+ if !labels["text"] || !labels["reference"] {+ t.Errorf("expected text(low-conf) and reference(>=0.4) kept, got labels %v", labels)+ }+}+func TestAnnotateBoxLayouts_GarbageFooterRejected(t *testing.T) {// Footer at page bottom: Bottom(290) > 270 (90% of 300px→PDF height 100→90% of 100=90)// → real footer decoration → garbage → pop (Python: bxs.pop(i)).@@ -278,8 +306,11 @@ func TestAnnotateBoxLayouts_SyntheticFigure(t *testing.T) {if b.LayoutType == "figure" && b.Text == "" {if b.LayoutNo == "figure-0" {foundFig0 = true- if b.X0 != 100 || b.X1 != 200 {- t.Errorf("synthetic figure-0: expected x0=100,x1=200 (300/3,600/3), got x0=%v,x1=%v", b.X0, b.X1)+ // After sort_Y_firstly, the top figure region (Y0=0 -> PDF+ // y0=0, x0=200 from 600/3, x1=300 from 900/3) is figure-0,+ // matching Python's top-to-bottom numbering.+ if b.X0 != 200 || b.X1 != 300 {+ t.Errorf("synthetic figure-0: expected x0=200,x1=300 (top region after Y-sort), got x0=%v,x1=%v", b.X0, b.X1)}}if b.LayoutNo == "figure-1" {diff --git a/internal/deepdoc/parser/pdf/tool/compare.go b/internal/deepdoc/parser/pdf/tool/compare.goindex c62ea849be..c8c0c88a77 100644--- a/internal/deepdoc/parser/pdf/tool/compare.go+++ b/internal/deepdoc/parser/pdf/tool/compare.go@@ -47,6 +47,9 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResulvar diffs []Diffmatched, mismatched := 0, 0+ // suppressedAny tracks whether any PDF hit the Py stage-metric phase+ // gap (see the guard below) so we can warn once in the summary.+ suppressedAny := falsefor _, r := range goResults {py, ok := pyMap[r.File]@@ -74,6 +77,23 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResulif py.Sections > 0 {d.SectionsDiffPct = math.Abs(float64(r.Sections-py.Sections)) / float64(py.Sections) * 100}++ // Phase-consistency guard: the Python harness cannot report+ // post-merge pipeline stages (boxes_text_merge / boxes_vertical_merge+ // / sections) because the production parser does not expose them, so+ // its #@meta falls back to the raw final box count for all three.+ // Comparing those against Go's real merged stages would emit+ // misleading percentages (e.g. 67% on "sections"). Mark them N/A.+ if py.BoxesInitial > 0 &&+ py.BoxesTextMerge == py.BoxesInitial &&+ py.BoxesVertMerge == py.BoxesInitial &&+ py.Sections == py.BoxesInitial {+ d.BoxesTMDiffPct = -1+ d.BoxesVMDiffPct = -1+ d.SectionsDiffPct = -1+ suppressedAny = true+ }+if py.TextLen > 0 {d.TextLenDiffPct = math.Abs(float64(r.TextLen-py.TextLen)) / float64(py.TextLen) * 100}@@ -103,7 +123,18 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResullen(diffs), len(goResults), r.File, 100-d.CharSim, 100-d.LcsSim, 100-d.RawCharSim, 100-d.RawLcsSim)}- sort.Slice(diffs, func(i, j int) bool { return diffs[i].SectionsDiffPct < diffs[j].SectionsDiffPct })+ // Sort worst-first by SectionsDiffPct, but push N/A (-1) rows to the+ // bottom so the per-PDF table reads cleanly.+ sort.Slice(diffs, func(i, j int) bool {+ a, b := diffs[i].SectionsDiffPct, diffs[j].SectionsDiffPct+ if a < 0 {+ a = math.MaxFloat64+ }+ if b < 0 {+ b = math.MaxFloat64+ }+ return a < b+ })log.Logf("\n=== Go vs Python (%d PDFs) ===", len(diffs))log.Logf("Pages match: %d/%d", matched, matched+mismatched)@@ -117,10 +148,10 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResulgr := goMap[d.File]goStages := fmt.Sprintf("%3d->%3d->%3d->%3d", gr.BoxesInitial, gr.BoxesTextMerg, gr.BoxesVertMerg, gr.Sections)pyStages := fmt.Sprintf("%3d->%3d->%3d->%3d", py.BoxesInitial, py.BoxesTextMerge, py.BoxesVertMerge, py.Sections)- log.Logf("%-40s %-18s %-18s %4.0f%% %4.0f%% %4.0f%% %4.0f%% %4.0f%% %+4d %.0f%% %.0f%% %.0f%% %.0f%%",+ log.Logf("%-40s %-18s %-18s %5s %5s %5s %5s %5.0f%% %+4d %.0f%% %.0f%% %.0f%% %.0f%%",d.File, goStages, pyStages,- d.BoxesInitDiffPct, d.BoxesTMDiffPct, d.BoxesVMDiffPct,- d.SectionsDiffPct, d.TextLenDiffPct, d.TablesDiff,+ pctStr(d.BoxesInitDiffPct), pctStr(d.BoxesTMDiffPct), pctStr(d.BoxesVMDiffPct),+ pctStr(d.SectionsDiffPct), d.TextLenDiffPct, d.TablesDiff,100-d.CharSim, 100-d.LcsSim,100-d.RawCharSim, 100-d.RawLcsSim)}@@ -134,17 +165,31 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResulmedian, mean, max, min float64over5, over10 int}- computeStats := func(get func(Diff) float64) stats {- sort.Slice(diffs, func(i, j int) bool { return get(diffs[i]) < get(diffs[j]) })- s := stats{min: 1e9}- if n%2 == 0 {- s.median = (get(diffs[n/2-1]) + get(diffs[n/2])) / 2+ // computeStats returns summary statistics over the valid (>=0) samples.+ // It returns ok=false when every sample is N/A, so the caller can print+ // "N/A" instead of bogus numbers.+ computeStats := func(get func(Diff) float64) (s stats, ok bool) {+ s.min = math.MaxFloat64+ var vals []float64+ for _, d := range diffs {+ v := get(d)+ if v < 0 {+ continue+ }+ vals = append(vals, v)+ }+ if len(vals) == 0 {+ return s, false+ }+ sort.Float64s(vals)+ m := len(vals)+ if m%2 == 0 {+ s.median = (vals[m/2-1] + vals[m/2]) / 2} else {- s.median = get(diffs[n/2])+ s.median = vals[m/2]}var sum float64- for _, d := range diffs {- v := get(d)+ for _, v := range vals {sum += vif v > s.max {s.max = v@@ -159,8 +204,8 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResuls.over10++}}- s.mean = sum / float64(n)- return s+ s.mean = sum / float64(m)+ return s, true}label := func(name string, s stats) string {@@ -169,15 +214,59 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul}log.Logf("\nSummary (n=%d):", n)- log.Logf(" %s", label("BoxesInit ", computeStats(func(d Diff) float64 { return d.BoxesInitDiffPct })))- log.Logf(" %s", label("TextMerge", computeStats(func(d Diff) float64 { return d.BoxesTMDiffPct })))- log.Logf(" %s", label("VertMerge", computeStats(func(d Diff) float64 { return d.BoxesVMDiffPct })))- log.Logf(" %s", label("Sections ", computeStats(func(d Diff) float64 { return d.SectionsDiffPct })))- log.Logf(" %s", label("TextLen ", computeStats(func(d Diff) float64 { return d.TextLenDiffPct })))- log.Logf(" %s", label("CharDiff ", computeStats(func(d Diff) float64 { return 100 - d.CharSim })))- log.Logf(" %s", label("LcsDiff ", computeStats(func(d Diff) float64 { return 100 - d.LcsSim })))- log.Logf(" %s", label("RawCharDiff", computeStats(func(d Diff) float64 { return 100 - d.RawCharSim })))- log.Logf(" %s", label("RawLcsDiff ", computeStats(func(d Diff) float64 { return 100 - d.RawLcsSim })))+ if s, ok := computeStats(func(d Diff) float64 { return d.BoxesInitDiffPct }); ok {+ log.Logf(" %s", label("BoxesInit ", s))+ } else {+ log.Logf(" BoxesInit N/A")+ }+ if s, ok := computeStats(func(d Diff) float64 { return d.BoxesTMDiffPct }); ok {+ log.Logf(" %s", label("TextMerge", s))+ } else {+ log.Logf(" TextMerge N/A (Python harness does not expose post-merge stage)")+ }+ if s, ok := computeStats(func(d Diff) float64 { return d.BoxesVMDiffPct }); ok {+ log.Logf(" %s", label("VertMerge", s))+ } else {+ log.Logf(" VertMerge N/A (Python harness does not expose post-merge stage)")+ }+ if s, ok := computeStats(func(d Diff) float64 { return d.SectionsDiffPct }); ok {+ log.Logf(" %s", label("Sections ", s))+ } else {+ log.Logf(" Sections N/A (Python harness does not expose post-merge stage)")+ }+ if s, ok := computeStats(func(d Diff) float64 { return d.TextLenDiffPct }); ok {+ log.Logf(" %s", label("TextLen ", s))+ } else {+ log.Logf(" TextLen N/A")+ }+ if s, ok := computeStats(func(d Diff) float64 { return 100 - d.CharSim }); ok {+ log.Logf(" %s", label("CharDiff ", s))+ } else {+ log.Logf(" CharDiff N/A")+ }+ if s, ok := computeStats(func(d Diff) float64 { return 100 - d.LcsSim }); ok {+ log.Logf(" %s", label("LcsDiff ", s))+ } else {+ log.Logf(" LcsDiff N/A")+ }+ if s, ok := computeStats(func(d Diff) float64 { return 100 - d.RawCharSim }); ok {+ log.Logf(" %s", label("RawCharDiff", s))+ } else {+ log.Logf(" RawCharDiff N/A")+ }+ if s, ok := computeStats(func(d Diff) float64 { return 100 - d.RawLcsSim }); ok {+ log.Logf(" %s", label("RawLcsDiff ", s))+ } else {+ log.Logf(" RawLcsDiff N/A")+ }++ if suppressedAny {+ log.Logf("\nNOTE: 'N/A' in TM%%/VM%%/Sec%% means the Python harness could not")+ log.Logf("expose post-merge pipeline stage counts (the production parser does")+ log.Logf("not record boxes_text_merge / boxes_vertical_merge / sections), so its")+ log.Logf("#@meta falls back to the raw final box count. Those columns are not")+ log.Logf("comparable and were excluded from the summary statistics.")+ }// Auto-generate xlsx report with timestamp.mode := filepath.Base(filepath.Dir(goTextDir)) // "ocr"@@ -289,11 +378,15 @@ func WriteExcel(path string, diffs []Diff) error {for col := 2; col <= 9; col++ {cell := cellName(col, r)v := vals[col-1]- f.SetCellValue(sheet, cell, v)- // Color: green <5, yellow 5-20, red >=20.if col == 7 { // TabsD is a count, not percentage+ f.SetCellValue(sheet, cell, v)+ continue+ }+ if v < 0 { // N/A: leave the cell blank, no color.continue}+ f.SetCellValue(sheet, cell, v)+ // Color: green <5, yellow 5-20, red >=20.abs := math.Abs(v)switch {case abs < 5:@@ -646,3 +739,12 @@ func abs(x int) int {}return x}++// pctStr renders a diff-percentage for table output. Negative values encode+// "N/A" (e.g. a Python stage metric that could not be reported).+func pctStr(v float64) string {+ if v < 0 {+ return "N/A"+ }+ return fmt.Sprintf("%.0f%%", v)+}diff --git a/internal/deepdoc/parser/pdf/tool/compare_test.go b/internal/deepdoc/parser/pdf/tool/compare_test.goindex bd2dee92f8..1fc7951c5c 100644--- a/internal/deepdoc/parser/pdf/tool/compare_test.go+++ b/internal/deepdoc/parser/pdf/tool/compare_test.go@@ -7,6 +7,8 @@ import ("os""path/filepath""testing"++ "ragflow/internal/common")// TestBatchCompareWithPython compares Go output against Python referencediff --git a/internal/parser/parser/pdf_postprocess.go b/internal/parser/parser/pdf_postprocess.goindex 7a0ff99b2f..cc1466ff3e 100644--- a/internal/parser/parser/pdf_postprocess.go+++ b/internal/parser/parser/pdf_postprocess.go@@ -9,7 +9,11 @@ import (deepdoctype "ragflow/internal/deepdoc/parser/type")-var pdfHeaderFooterPattern = regexp.MustCompile(`(?i)^(header|footer|number)$`)+// Substring match to mirror Python's remove_header_footer:+// re.search(r"(header|footer|number)", raw_layout, re.I) (rag/flow/parser/parser.py:754).+// Python matches any layout type CONTAINING one of these words, not just the+// exact token, so a composite label like "page-footer" is also stripped.+var pdfHeaderFooterPattern = regexp.MustCompile(`(?i)header|footer|number`)var pdfTOCTitlePattern = regexp.MustCompile(`(?i)^(contents|目录|目次|table of contents|致谢|acknowledge)$`)type pdfPostProcessOptions struct {diff --git a/internal/parser/parser/pdf_postprocess_test.go b/internal/parser/parser/pdf_postprocess_test.goindex dbb62bb12c..4a1e4fdea6 100644--- a/internal/parser/parser/pdf_postprocess_test.go+++ b/internal/parser/parser/pdf_postprocess_test.go@@ -141,3 +141,38 @@ func TestApplyPDFPostProcess_ReordersMultiColumnText(t *testing.T) {}}}++// TestFilterPDFHeaderFooter_SubstringMatch pins #5: Python's remove_header_footer+// uses a substring match re.search(r"(header|footer|number)", ...) (rag/flow/parser/parser.py:754),+// while Go used an anchored exact match ^(header|footer|number)$. A layout type+// that merely CONTAINS one of those words (e.g. "page-footer") must be stripped to+// match Python, not silently kept.+func TestFilterPDFHeaderFooter_SubstringMatch(t *testing.T) {+ result := &deepdoctype.ParseResult{+ Sections: []deepdoctype.Section{+ {Text: "real header", LayoutType: "header"},+ {Text: "real footer", LayoutType: "footer"},+ {Text: "page 1", LayoutType: "number"},+ {Text: "a page-footer note", LayoutType: "page-footer"}, // composite -> substring match+ {Text: "body text", LayoutType: "text"},+ },+ }+ filterPDFHeaderFooter(result)++ kept := map[string]bool{}+ for _, s := range result.Sections {+ kept[s.LayoutType] = true+ }+ for _, lt := range []string{"header", "footer", "number"} {+ if kept[lt] {+ t.Errorf("#5 header/footer: %q should be stripped", lt)+ }+ }+ // Composite "page-footer" must be stripped by substring match (Python-equivalent).+ if kept["page-footer"] {+ t.Errorf("#5 header/footer: composite layout type %q should be stripped by substring match", "page-footer")+ }+ if !kept["text"] {+ t.Errorf("#5 header/footer: body text %q should be kept", "text")+ }+}