RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(deepdoc): correct DLA channel order (BGR->RGB) and keep per-class NMS to match Python serving (#20637)
xugangqiang9980851eecd1
diff --git a/internal/deepdoc/native/dla.go b/internal/deepdoc/native/dla.goindex 63c5c06705..5cf976427d 100644--- a/internal/deepdoc/native/dla.go+++ b/internal/deepdoc/native/dla.go@@ -90,8 +90,16 @@ func dlaGeom(img *Image) (newW, newH int, dw, dh float64) {// dlaLetterbox places the already-resized BGR raster (newH*newW*3, row-major)// into the dlaInputSize canvas with 114-filled borders and returns the CHW-// float blob (/255) the YOLOv10 layout model consumes. Only the resize source-// differs from the production Python reference (Go bilinearResize vs cv2).+// float blob (/255) the YOLOv10 layout model consumes.+//+// Channel order MUST be RGB (channel 0 = red). The Go decoder yields a BGR+// raster (img.ToBGR), but the Python reference deepdoc pipeline applies+// cv2.cvtColor(BGR2RGB) inside its letterbox before the HWC->CHW transpose, so+// the layout model is trained and inferred on RGB input. Swapping the channels+// here keeps Go aligned with that reference. Feeding BGR instead (channel 0 =+// blue) silently flips the colour channels and collapses the confidence of any+// box over a coloured region, while leaving grayscale pages byte-identical+// (hence the bug went unnoticed on text-heavy scans).func dlaLetterbox(resized []byte, newW, newH int, dw, dh float64) []float32 {top := int(math.Round(dh - 0.1))left := int(math.Round(dw - 0.1))@@ -103,14 +111,15 @@ func dlaLetterbox(resized []byte, newW, newH int, dw, dh float64) []float32 {inY, inX := y-top, x-leftif inY >= 0 && inY < newH && inX >= 0 && inX < newW {o := (inY*newW + inX) * 3- cb = float32(resized[o])- cg = float32(resized[o+1])+ // resized is BGR; the model wants RGB (Python ref does BGR2RGB).cr = float32(resized[o+2])+ cg = float32(resized[o+1])+ cb = float32(resized[o])}- // CHW; model expects BGR, so channel 0 = blue, 2 = red.- blob[0*dlaInputSize*dlaInputSize+y*dlaInputSize+x] = cb / 255.0+ // CHW; model expects RGB, so channel 0 = red, 2 = blue.+ blob[0*dlaInputSize*dlaInputSize+y*dlaInputSize+x] = cr / 255.0blob[1*dlaInputSize*dlaInputSize+y*dlaInputSize+x] = cg / 255.0- blob[2*dlaInputSize*dlaInputSize+y*dlaInputSize+x] = cr / 255.0+ blob[2*dlaInputSize*dlaInputSize+y*dlaInputSize+x] = cb / 255.0}}return blob@@ -156,24 +165,45 @@ func dlaPostprocess(out []float32, sf [4]float32) DLAResult {})}- byClass := map[int][]int{}+ // The layout model (layout.onnx and layout.ort are the SAME graph — the .ort+ // is just the FlatBuffer-serialized form, with no NMS op baked in) does NOT+ // run NMS in its graph: output0 is a set of raw top-k candidate boxes+ // (xyxy, score, class). The OSS reference pipeline (deepdoc's+ // LayoutRecognizer4YOLOv10.postprocess / ref_dla.py) applies a per-class+ // NMS(iou=0.45) on top of these candidates before handing boxes to the+ // caller — that post-NMS output is the wire contract the caller consumes.+ // The golden files are generated from that same Python serving post-process,+ // so to match the caller's view this code re-runs the identical per-class+ // nms(0.45) (the operators.py +1 convention). This is the single NMS the+ // serving applies — there is no second NMS downstream (native_analyzer maps+ // the boxes through as-is), so it is NOT a double-NMS.+ res := DLAResult{}+ // Per-class NMS(0.45, +1) — mirrors OSS Python serving's postprocess; the+ // golden is generated from that post-NMS output.+ byCls := map[int][]int{}for i, c := range cands {- byClass[c.cls] = append(byClass[c.cls], i)+ byCls[c.cls] = append(byCls[c.cls], i)}- res := DLAResult{}- for cls, idxs := range byClass {- sub := make([]nmsBox, len(idxs))+ keep := map[int]bool{}+ for _, idxs := range byCls {+ bs := make([]nmsBox, len(idxs))for k, i := range idxs {- sub[k] = cands[i].nmsBox+ bs[k] = cands[i].nmsBox}- for _, keep := range nms(sub, 0.45, true) {- res.Boxes = append(res.Boxes, DLABox{- X0: round2(sub[keep].X0), Y0: round2(sub[keep].Y0),- X1: round2(sub[keep].X1), Y1: round2(sub[keep].Y1),- Score: round4(sub[keep].Score), Class: cls,- })+ for _, k := range nms(bs, 0.45, true) {+ keep[idxs[k]] = true}}+ for i, c := range cands {+ if !keep[i] {+ continue+ }+ res.Boxes = append(res.Boxes, DLABox{+ X0: round2(c.nmsBox.X0), Y0: round2(c.nmsBox.Y0),+ X1: round2(c.nmsBox.X1), Y1: round2(c.nmsBox.Y1),+ Score: round4(c.nmsBox.Score), Class: c.cls,+ })+ }// Re-map class ids through the OSS label->Go index map.mapped := res.Boxes[:0]for _, b := range res.Boxes {@@ -194,6 +224,7 @@ func dlaPostprocess(out []float32, sf [4]float32) DLAResult {mapped = append(mapped, b)}res.Boxes = mapped+// Deterministic ordering: dlaPostprocess iterates a class->index map, whose// iteration order is unspecified in Go. Sort so identical detections always// serialize identically (e.g. for stable Wire() across runs / session reuse).diff --git a/internal/deepdoc/native/dla_letterbox_test.go b/internal/deepdoc/native/dla_letterbox_test.gonew file mode 100644index 0000000000..a3545ddc1a--- /dev/null+++ b/internal/deepdoc/native/dla_letterbox_test.go@@ -0,0 +1,44 @@+//go:build cgo++package native++import "testing"++// TestDlaLetterboxChannelOrder verifies the DLA preprocessing feeds the layout+// model pixels in RGB order (channel 0 = red), matching the Python reference+// deepdoc pipeline. The Python LayoutRecognizer letterbox applies+// cv2.cvtColor(BGR2RGB) before the HWC->CHW transpose, so the YOLOv10 layout+// model is trained and inferred on RGB input. The Go image decoder yields a BGR+// raster (img.ToBGR), therefore the channel swap to RGB must happen inside+// dlaLetterbox.+//+// Regression guard: feeding BGR (channel 0 = blue) instead of RGB silently+// flips the colour channels. Grayscale pages are byte-identical so detection is+// unaffected, but a coloured region (e.g. a figure) sees swapped channels and+// the model's confidence for that box collapses -- the box can then fall under+// the 0.4 encode gate and vanish from the output.+func TestDlaLetterboxChannelOrder(t *testing.T) {+ const (+ blue = 30+ green = 20+ red = 10+ )+ // resized is BGR row-major, exactly as produced by+ // img.ToBGR() followed by bilinearResize.+ resized := []byte{blue, green, red}+ blob := dlaLetterbox(resized, 1, 1, 0, 0)++ // CHW layout: offset for channel c at (y,x) is c*W*H + y*W + x.+ at := func(c int) float32 {+ return blob[c*dlaInputSize*dlaInputSize+0*dlaInputSize+0]+ }+ if got := at(0); got != float32(red)/255.0 {+ t.Fatalf("channel 0 (must be RED) = %v, want %v -- RGB order required; Python ref does BGR2RGB", got, float32(red)/255.0)+ }+ if got := at(1); got != float32(green)/255.0 {+ t.Fatalf("channel 1 (must be GREEN) = %v, want %v", got, float32(green)/255.0)+ }+ if got := at(2); got != float32(blue)/255.0 {+ t.Fatalf("channel 2 (must be BLUE) = %v, want %v", got, float32(blue)/255.0)+ }+}diff --git a/internal/deepdoc/native/dla_postprocess_test.go b/internal/deepdoc/native/dla_postprocess_test.gonew file mode 100644index 0000000000..4bfaa9b538--- /dev/null+++ b/internal/deepdoc/native/dla_postprocess_test.go@@ -0,0 +1,44 @@+//go:build cgo && integration++package native++import (+ "context"+ "encoding/json"+ "os"+ "path/filepath"+ "testing"+)++// TestDLARealEnPaperMatchesGolden pins the dla_real_en_paper layout output to+// the golden generated from the OSS Python serving post-process (cv2+// preprocessing + per-class nms(0.45), class ids remapped through DLA_CLASS_MAP+// to match Go's Wire()). The layout .ort graph contains NO NMS op — output0 is+// raw candidate boxes — but the serving pipeline applies per-class nms(0.45)+// before handing boxes to the caller, and that post-NMS output is the wire+// contract. Go's dlaPostprocess re-runs that same nms so its caller-visible+// boxes match; this is the single NMS the serving applies (native_analyzer maps+// the boxes through as-is, no second NMS), so it is not a double-NMS. The+// golden is regenerated from real Python in-process inference, so any drift in+// Go's preprocessing, NMS, or class remap shows up here as a per-coordinate+// diff beyond CmpTolCoord.+func TestDLARealEnPaperMatchesGolden(t *testing.T) {+ skipIfNoModels(t)+ img, err := Decode(filepath.Join("testdata", "dla_real_en_paper.png"))+ if err != nil {+ t.Fatalf("decode: %v", err)+ }+ res, err := RunDLA(context.Background(), os.Getenv("MODEL_DIR"), img)+ if err != nil {+ t.Fatalf("RunDLA: %v", err)+ }+ var got struct {+ Boxes [][]float64 `json:"bboxes"`+ }+ if err := json.Unmarshal([]byte(res.Wire()), &got); err != nil {+ t.Fatalf("parse Go wire: %v", err)+ }+ gold := LoadGoldenBoxes(t, filepath.Join("testdata", "dla_real_en_paper.dla.golden.json"))+ // CompareBoxes fails on any per-coordinate diff beyond CmpTolCoord (3.5px).+ CompareBoxes(t, gold, got.Boxes)+}diff --git a/internal/deepdoc/native/testdata.ref b/internal/deepdoc/native/testdata.refindex 59110447ef..64e632677e 100644--- a/internal/deepdoc/native/testdata.ref+++ b/internal/deepdoc/native/testdata.ref@@ -1 +1 @@-deepdoc-native-testdata-v1+deepdoc-native-testdata-v2