RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(deepdoc): drop low-confidence garbage DLA regions to match Python (#18365)
Jack7ceafa07f8d0
diff --git a/internal/deepdoc/parser/pdf/inference/client.go b/internal/deepdoc/parser/pdf/inference/client.goindex c2b8fb95bf..451c2fdde1 100644--- a/internal/deepdoc/parser/pdf/inference/client.go+++ b/internal/deepdoc/parser/pdf/inference/client.go@@ -76,6 +76,24 @@ type bboxesResponse struct {BBoxes [][]float64 `json:"bboxes"`}+// dlaGarbageLayouts mirrors Python LayoutRecognizer's garbage gate+// (deepdoc/vision/layout_recognizer.py:97 and :379), which drops any region+// whose type is in garbage_layouts=["footer","header","reference"] AND whose+// confidence is below 0.4. We apply the same gate at the DLA source so every+// consumer of Client.DLA (not just the table-annotation path, which re-applies+// it downstream) sees the Python-aligned region set.+//+// Of the three types, only "reference" is reachable with the OSS default+// 10-class DLA taxonomy (DefaultDLALabels has no footer/header classes), so in+// practice this gate only fires on low-confidence references. footer/header are+// included defensively to match Python's full garbage set for any deployment+// whose DLA label taxonomy emits them.+var dlaGarbageLayouts = map[string]bool{+ string(pdf.LayoutTypeFooter): true,+ string(pdf.LayoutTypeHeader): true,+ string(pdf.LayoutTypeReference): true,+}+// DLA analyzes a full page image and returns labeled regions.func (c *Client) DLA(ctx context.Context, pageImage image.Image) ([]pdf.DLARegion, error) {data, err := util.EncodePNG(pageImage)@@ -96,6 +114,10 @@ func (c *Client) DLA(ctx context.Context, pageImage image.Image) ([]pdf.DLARegioif clsID := int(b[5]); clsID >= 0 && clsID < len(labels) {label = labels[clsID]}+ // Drop low-confidence garbage-layout regions (Python parity: 0.4 gate).+ if dlaGarbageLayouts[label] && b[4] < 0.4 {+ continue+ }regions = append(regions, pdf.DLARegion{X0: b[0], Y0: b[1], X1: b[2], Y1: b[3],Confidence: b[4],diff --git a/internal/deepdoc/parser/pdf/inference/client_dla_garbage_test.go b/internal/deepdoc/parser/pdf/inference/client_dla_garbage_test.gonew file mode 100644index 0000000000..199de8e5c4--- /dev/null+++ b/internal/deepdoc/parser/pdf/inference/client_dla_garbage_test.go@@ -0,0 +1,97 @@+package inference++import (+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "testing"+)++// TestDeepDocHTTP_DLA_GarbageGate pins the Python-parity 0.4 garbage gate+// (LayoutRecognizer.__call__, deepdoc/vision/layout_recognizer.py:97 and :379):+// a region whose layout type is a garbage layout (footer/header/reference) and+// whose confidence is strictly below 0.4 is dropped; everything else is kept.+//+// The OSS default 10-class DLA taxonomy only emits "reference" as a garbage+// type, but footer/header are covered defensively (see dlaGarbageLayouts in+// client.go). Each subtest drives a mock /predict/dla backend returning one+// bbox and asserts the resulting region set.+func TestDeepDocHTTP_DLA_GarbageGate(t *testing.T) {+ newClient := func(t *testing.T, bboxes [][]float64) *Client {+ t.Helper()+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/predict/dla" {+ t.Errorf("path = %q, want /predict/dla", r.URL.Path)+ }+ if err := json.NewEncoder(w).Encode(map[string]any{"bboxes": bboxes}); err != nil {+ // Handler runs in its own goroutine; fail the test, don't panic.+ t.Errorf("encode response: %v", err)+ }+ }))+ t.Cleanup(srv.Close)+ return mustNewDeepDocClient(t, srv.URL)+ }++ // bbox = [x0, y0, x1, y1, confidence, classId]; classId 2 = "reference".+ const referenceClass = 2++ t.Run("low_conf_reference_dropped", func(t *testing.T) {+ client := newClient(t, [][]float64{+ {50, 10, 500, 50, 0.30, referenceClass}, // reference, low confidence+ })+ regions, err := client.DLA(context.Background(), testImage())+ if err != nil {+ t.Fatal(err)+ }+ if len(regions) != 0 {+ t.Fatalf("got %d regions, want 0 (low-confidence 'reference' must be dropped by the 0.4 garbage gate)", len(regions))+ }+ })++ t.Run("high_conf_reference_kept", func(t *testing.T) {+ client := newClient(t, [][]float64{+ {50, 10, 500, 50, 0.90, referenceClass}, // reference, high confidence+ })+ regions, err := client.DLA(context.Background(), testImage())+ if err != nil {+ t.Fatal(err)+ }+ if len(regions) != 1 || regions[0].Label != "reference" {+ t.Fatalf("got %v, want exactly one 'reference' region (conf >= 0.4 is kept)", regions)+ }+ })++ t.Run("boundary_conf_0.4_kept", func(t *testing.T) {+ // Gate is strict (< 0.4); confidence exactly 0.4 is the boundary and kept.+ client := newClient(t, [][]float64{+ {50, 10, 500, 50, 0.40, referenceClass}, // reference, exactly 0.4+ })+ regions, err := client.DLA(context.Background(), testImage())+ if err != nil {+ t.Fatal(err)+ }+ if len(regions) != 1 || regions[0].Label != "reference" {+ t.Fatalf("got %v, want exactly one 'reference' region (conf == 0.4 is the gate boundary and kept)", regions)+ }+ })++ t.Run("low_conf_garbage_and_text", func(t *testing.T) {+ // A low-confidence reference is dropped while an unrelated text region+ // is kept — Python keeps only the high-confidence non-garbage region.+ client := newClient(t, [][]float64{+ {50, 10, 500, 50, 0.30, referenceClass}, // reference, low confidence -> dropped+ {50, 100, 500, 300, 0.90, 1}, // text, high confidence -> kept+ })+ regions, err := client.DLA(context.Background(), testImage())+ if err != nil {+ t.Fatal(err)+ }+ if len(regions) != 1 {+ t.Fatalf("got %d regions, want 1 (low-confidence 'reference' dropped, 'text' kept)", len(regions))+ }+ if regions[0].Label != "text" {+ t.Errorf("regions[0].Label = %q, want 'text'", regions[0].Label)+ }+ })+}