RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(deepdoc): trim garbage-gate comments, fix subtest fact error, dedupe constants (#18377)
Jackcb012380fe06
diff --git a/internal/deepdoc/parser/pdf/inference/client.go b/internal/deepdoc/parser/pdf/inference/client.goindex 451c2fdde1..5dc646f4c6 100644--- a/internal/deepdoc/parser/pdf/inference/client.go+++ b/internal/deepdoc/parser/pdf/inference/client.go@@ -76,24 +76,6 @@ type bboxesResponse struct {BBoxes [][]float64 `json:"bboxes"`}-// dlaGarbageLayouts mirrors Python LayoutRecognizer's garbage gate-// (deepdoc/vision/layout_recognizer.py:97 and :379), which drops any region-// whose type is in garbage_layouts=["footer","header","reference"] AND whose-// confidence is below 0.4. We apply the same gate at the DLA source so every-// consumer of Client.DLA (not just the table-annotation path, which re-applies-// it downstream) sees the Python-aligned region set.-//-// Of the three types, only "reference" is reachable with the OSS default-// 10-class DLA taxonomy (DefaultDLALabels has no footer/header classes), so in-// practice this gate only fires on low-confidence references. footer/header are-// included defensively to match Python's full garbage set for any deployment-// whose DLA label taxonomy emits them.-var dlaGarbageLayouts = map[string]bool{- string(pdf.LayoutTypeFooter): true,- string(pdf.LayoutTypeHeader): true,- string(pdf.LayoutTypeReference): true,-}-// DLA analyzes a full page image and returns labeled regions.func (c *Client) DLA(ctx context.Context, pageImage image.Image) ([]pdf.DLARegion, error) {data, err := util.EncodePNG(pageImage)@@ -114,8 +96,9 @@ func (c *Client) DLA(ctx context.Context, pageImage image.Image) ([]pdf.DLARegioif clsID := int(b[5]); clsID >= 0 && clsID < len(labels) {label = labels[clsID]}- // Drop low-confidence garbage-layout regions (Python parity: 0.4 gate).- if dlaGarbageLayouts[label] && b[4] < 0.4 {+ // Drop low-confidence garbage-layout regions (Python parity: 0.4 gate,+ // deepdoc/vision/layout_recognizer.py:97).+ if pdf.GarbageLayoutTypes[label] && b[4] < pdf.GarbageLayoutScoreThreshold {continue}regions = append(regions, pdf.DLARegion{diff --git a/internal/deepdoc/parser/pdf/inference/client_dla_garbage_test.go b/internal/deepdoc/parser/pdf/inference/client_dla_garbage_test.goindex 199de8e5c4..090ce44fba 100644--- a/internal/deepdoc/parser/pdf/inference/client_dla_garbage_test.go+++ b/internal/deepdoc/parser/pdf/inference/client_dla_garbage_test.go@@ -9,14 +9,14 @@ import ()// TestDeepDocHTTP_DLA_GarbageGate pins the Python-parity 0.4 garbage gate-// (LayoutRecognizer.__call__, deepdoc/vision/layout_recognizer.py:97 and :379):-// a region whose layout type is a garbage layout (footer/header/reference) and-// whose confidence is strictly below 0.4 is dropped; everything else is kept.+// (deepdoc/vision/layout_recognizer.py:97): a region whose layout type is a+// garbage layout (footer/header/reference) and whose confidence is strictly+// below 0.4 is dropped; everything else is kept.//// The OSS default 10-class DLA taxonomy only emits "reference" as a garbage-// type, but footer/header are covered defensively (see dlaGarbageLayouts in-// client.go). Each subtest drives a mock /predict/dla backend returning one-// bbox and asserts the resulting region set.+// type, but footer/header are covered defensively via pdf.GarbageLayoutTypes.+// Each subtest drives a mock /predict/dla backend returning one bbox and+// asserts the resulting region set.func TestDeepDocHTTP_DLA_GarbageGate(t *testing.T) {newClient := func(t *testing.T, bboxes [][]float64) *Client {t.Helper()@@ -78,7 +78,8 @@ func TestDeepDocHTTP_DLA_GarbageGate(t *testing.T) {t.Run("low_conf_garbage_and_text", func(t *testing.T) {// A low-confidence reference is dropped while an unrelated text region- // is kept — Python keeps only the high-confidence non-garbage region.+ // (non-garbage) is kept — the gate only removes garbage-layout regions+ // below 0.4; non-garbage regions pass through at any confidence.client := newClient(t, [][]float64{{50, 10, 500, 50, 0.30, referenceClass}, // reference, low confidence -> dropped{50, 100, 500, 300, 0.90, 1}, // text, high confidence -> keptdiff --git a/internal/deepdoc/parser/pdf/table/table_annotate.go b/internal/deepdoc/parser/pdf/table/table_annotate.goindex 7335e61ca0..911a97460a 100644--- a/internal/deepdoc/parser/pdf/table/table_annotate.go+++ b/internal/deepdoc/parser/pdf/table/table_annotate.go@@ -307,7 +307,7 @@ func FilteredDLARegions(regions []pdf.DLARegion, boxes []pdf.TextBox) []pdf.DLAR}kept := regions[:0]for _, r := range regions {- if r.Confidence >= 0.4 || !isGarbageLayoutType(r.Label) {+ if r.Confidence >= pdf.GarbageLayoutScoreThreshold || !pdf.GarbageLayoutTypes[r.Label] {kept = append(kept, r)}}@@ -409,7 +409,7 @@ func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale floa}if bestJ >= 0 && bestOverlap >= 0.4 {// Garbage layout not at page edge -> pop (Python: bxs.pop(i)).- if isGarbageLayoutType(ty) && pageImgHeight > 0 && !garbageKeepFeat(ty, boxes[i], pageImgHeight/scale) {+ if pdf.GarbageLayoutTypes[ty] && pageImgHeight > 0 && !garbageKeepFeat(ty, boxes[i], pageImgHeight/scale) {dropped[i] = truecontinue}@@ -475,15 +475,6 @@ func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale floa}// ── garbage layout helpers ────────────────────────────────────────────-// garbageLayoutTypes matches Python's self.garbage_layouts.-var garbageLayoutTypes = map[string]bool{- pdf.LayoutTypeFooter: true, pdf.LayoutTypeHeader: true, pdf.LayoutTypeReference: true,-}--func isGarbageLayoutType(ty string) bool {- return garbageLayoutTypes[ty]-}-// garbageKeepFeat matches Python's keep_feats in LayoutRecognizer.__call__:// footer near page bottom (>90% of page height) or header near page top (<10%)// are real page decorations - keep them. Others are DLA noise.diff --git a/internal/deepdoc/parser/pdf/type/types.go b/internal/deepdoc/parser/pdf/type/types.goindex ae0cd7447e..f0eed00b10 100644--- a/internal/deepdoc/parser/pdf/type/types.go+++ b/internal/deepdoc/parser/pdf/type/types.go@@ -44,6 +44,8 @@ const (LayoutTypeHeader = doctype.LayoutTypeHeaderDLALabelFigureCaption = doctype.DLALabelFigureCaptionDLALabelTableCaption = doctype.DLALabelTableCaption++ GarbageLayoutScoreThreshold = doctype.GarbageLayoutScoreThreshold)// ── Re-export functions and variables ──────────────────────────────────────@@ -52,4 +54,5 @@ var (CollectFigures = doctype.CollectFiguresDefaultParserConfig = doctype.DefaultParserConfigIsCJK = doctype.IsCJK+ GarbageLayoutTypes = doctype.GarbageLayoutTypes)diff --git a/internal/deepdoc/parser/type/types.go b/internal/deepdoc/parser/type/types.goindex 5c9f79d56f..428f23b838 100644--- a/internal/deepdoc/parser/type/types.go+++ b/internal/deepdoc/parser/type/types.go@@ -251,6 +251,20 @@ const (DLALabelTableCaption = "table caption")+// GarbageLayoutScoreThreshold is the minimum confidence a garbage-layout+// region must reach to survive; below it the region is dropped. Mirrors+// Python's garbage gate in LayoutRecognizer (deepdoc/vision/layout_recognizer.py:97).+const GarbageLayoutScoreThreshold = 0.4++// GarbageLayoutTypes are layout types dropped when their confidence is below+// GarbageLayoutScoreThreshold. Mirrors Python's self.garbage_layouts+// = ["footer", "header", "reference"].+var GarbageLayoutTypes = map[string]bool{+ LayoutTypeFooter: true,+ LayoutTypeHeader: true,+ LayoutTypeReference: true,+}+// ── Interfaces ────────────────────────────────────────────────────────────// DocAnalyzer abstracts DeepDoc vision operations.