RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(deepdoc): drop per-cell OCR in Go table parser to align with Python (#18405)
Jackfa386ddf101c
diff --git a/internal/deepdoc/parser/pdf/parser_concurrency.go b/internal/deepdoc/parser/pdf/parser_concurrency.goindex dccf9705d0..a0912835ed 100644--- a/internal/deepdoc/parser/pdf/parser_concurrency.go+++ b/internal/deepdoc/parser/pdf/parser_concurrency.go@@ -205,8 +205,8 @@ func (p *Parser) inferOCRDetect(ctx context.Context, doc pdf.DocAnalyzer, pageIm}// inferOCRRecognize routes doc.OCRRecognize through the inference-// limiter. Per-region OCR fallback paths (buildTextBoxes, ocrTableCells)-// should use this wrapper so the per-region fan-out is bounded.+// limiter. Per-region OCR fallback paths (buildTextBoxes) should use this+// wrapper so the per-region fan-out is bounded.func (p *Parser) inferOCRRecognize(ctx context.Context, doc pdf.DocAnalyzer, cropped image.Image) ([]pdf.OCRText, error) {if doc == nil || !doc.Health() {return nil, nildiff --git a/internal/deepdoc/parser/pdf/parser_ocr.go b/internal/deepdoc/parser/pdf/parser_ocr.goindex 85db246fe7..4e3515c1b3 100644--- a/internal/deepdoc/parser/pdf/parser_ocr.go+++ b/internal/deepdoc/parser/pdf/parser_ocr.go@@ -279,47 +279,6 @@ func charBoxOverlapRatio(c pdf.TextChar, x0, x1, y0, y1 float64) float64 {return inter / charArea}-// ocrTableCells fills empty TSR cells via OCR recognition.-func (p *Parser) ocrTableCells(ctx context.Context, cells []pdf.TSRCell, tableImg image.Image, doc pdf.DocAnalyzer) {- if doc == nil || tableImg == nil || len(cells) == 0 {- return- }- for i := range cells {- if cells[i].Text != "" {- continue- }- x0 := int(math.Max(0, cells[i].X0))- y0 := int(math.Max(0, cells[i].Y0))- x1 := int(math.Min(float64(tableImg.Bounds().Dx()), cells[i].X1))- y1 := int(math.Min(float64(tableImg.Bounds().Dy()), cells[i].Y1))- if x0 >= x1 || y0 >= y1 {- continue- }- // De-skew via WarpCrop and recognize via ocrRecognizeWithRotation like- // the other OCR paths. Table cells are axis-aligned, so WarpCrop- // early-exits to FastCrop and ocrRecognizeWithRotation recognizes once- // at 0 deg; the bounds-clamp / non-finite guard is still inherited.- cropped := util.WarpCrop(tableImg, [4]util.Pt{- {X: float64(x0), Y: float64(y0)},- {X: float64(x1), Y: float64(y0)},- {X: float64(x1), Y: float64(y1)},- {X: float64(x0), Y: float64(y1)},- })- texts, err := p.ocrRecognizeWithRotation(ctx, doc, cropped)- if err != nil {- slog.Warn("table cell OCR failed", "err", err)- continue- }- var parts []string- for _, t := range texts {- if t.Text != "" {- parts = append(parts, t.Text)- }- }- cells[i].Text = strings.TrimSpace(strings.Join(parts, " "))- }-}-// buildTextBoxes assembles detect box text from embedded chars and fills empty boxes via single-image OCR.// Each region that lacks embedded text is cropped and recognized with a// direct doc.OCRRecognize call so empty-box fallback runs through thediff --git a/internal/deepdoc/parser/pdf/parser_test.go b/internal/deepdoc/parser/pdf/parser_test.goindex 797e961113..fcef7c4845 100644--- a/internal/deepdoc/parser/pdf/parser_test.go+++ b/internal/deepdoc/parser/pdf/parser_test.go@@ -127,82 +127,6 @@ func TestOCR_ScanPage(t *testing.T) {})}-// ── OCR table cell ─────────────────────────────────────────────────────--func TestOCR_TableCell(t *testing.T) {- p := newTestParser()- t.Run("fill single empty cell", func(t *testing.T) {- cells := []pdf.TSRCell{- {X0: 0, Y0: 0, X1: 100, Y1: 50, Text: ""},- {X0: 100, Y0: 0, X1: 200, Y1: 50, Text: "已有"},- }- mock := &MockDocAnalyzer{Healthy: true, OCRTexts: []pdf.OCRText{{Text: "识别结果", Confidence: 0.9}}}- dummy := image.NewRGBA(image.Rect(0, 0, 200, 50))-- p.ocrTableCells(t.Context(), cells, dummy, mock)-- if cells[0].Text != "识别结果" {- t.Errorf("empty cell not filled: %q", cells[0].Text)- }- if cells[1].Text != "已有" {- t.Errorf("filled cell changed: %q", cells[1].Text)- }- })-- t.Run("all cells already filled — no OCR", func(t *testing.T) {- cells := []pdf.TSRCell{- {X0: 0, Y0: 0, X1: 100, Y1: 50, Text: "A"},- {X0: 100, Y0: 0, X1: 200, Y1: 50, Text: "B"},- }- p.ocrTableCells(t.Context(), cells, nil, nil) // should not panic- if cells[0].Text != "A" || cells[1].Text != "B" {- t.Error("filled cells should not change")- }- })-- t.Run("empty cells list", func(t *testing.T) {- ctx := t.Context()- p.ocrTableCells(ctx, nil, nil, nil) // should not panic- p.ocrTableCells(ctx, []pdf.TSRCell{}, nil, nil)- })-- t.Run("no DeepDoc — skip", func(t *testing.T) {- cells := []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 50, Text: ""}}- p.ocrTableCells(t.Context(), cells, nil, nil)- if cells[0].Text != "" {- t.Error("without DeepDoc, cell should stay empty")- }- })-- t.Run("no cropped image — skip", func(t *testing.T) {- cells := []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 50, Text: ""}}- mock := &MockDocAnalyzer{Healthy: true, OCRTexts: []pdf.OCRText{{Text: "x", Confidence: 0.5}}}- p.ocrTableCells(t.Context(), cells, nil, mock)- if cells[0].Text != "" {- t.Error("without image, cell should stay empty")- }- })-- t.Run("OCR returns empty string", func(t *testing.T) {- cells := []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 50, Text: ""}}- mock := &MockDocAnalyzer{Healthy: true, OCRTexts: []pdf.OCRText{}}- dummy := image.NewRGBA(image.Rect(0, 0, 100, 50))- p.ocrTableCells(t.Context(), cells, dummy, mock)- if cells[0].Text != "" {- t.Error("empty OCR result → cell stays empty")- }- })-- t.Run("cell out of image bounds", func(t *testing.T) {- cells := []pdf.TSRCell{{X0: 500, Y0: 500, X1: 600, Y1: 600, Text: ""}}- mock := &MockDocAnalyzer{Healthy: true, OCRTexts: []pdf.OCRText{{Text: "out of bounds", Confidence: 0.9}}}- dummy := image.NewRGBA(image.Rect(0, 0, 100, 100))- // Should not panic — gracefully degrade- p.ocrTableCells(t.Context(), cells, dummy, mock)- t.Logf("out-of-bounds cell: text=%q", cells[0].Text)- })-}-func garbledSample() []pdf.TextChar {punctuation := []string{"!", "#", "$", "%", "&", "*", "+", "-", ".", "/",":", ";", "<", ">", "=", "?", "@", "^", "_", "~"}diff --git a/internal/deepdoc/parser/pdf/table_extract.go b/internal/deepdoc/parser/pdf/table_extract.goindex 465d4c9c09..e6ad5cb628 100644--- a/internal/deepdoc/parser/pdf/table_extract.go+++ b/internal/deepdoc/parser/pdf/table_extract.go@@ -102,9 +102,6 @@ func (p *Parser) processOneTable(ctx context.Context, pageImg image.Image, boxesvar boxInCrop []pdf.TextBoxif tsrErr == nil && len(cells) > 0 {if bestAngle != 0 {- if !p.Config.SkipOCR {- p.ocrTableCells(ctx, cells, tsrImg, docAnalyzer)- }for i := range cells {cells[i].X0, cells[i].Y0, cells[i].X1, cells[i].Y1 = util.MapRotatedRectToOriginal(cells[i].X0, cells[i].Y0, cells[i].X1, cells[i].Y1, bestAngle, origW, origH)@@ -149,16 +146,6 @@ func (p *Parser) processOneTable(ctx context.Context, pageImg image.Image, boxesidx++}}- if bestAngle == 0 && !p.Config.SkipOCR {- p.ocrTableCells(ctx, flat, tsrImg, docAnalyzer)- idx = 0- for ri := range grid {- for ci := range grid[ri] {- grid[ri][ci].Text = flat[idx].Text- idx++- }- }- }}}item := pdf.TableItem{diff --git a/internal/deepdoc/parser/pdf/table_extract_test.go b/internal/deepdoc/parser/pdf/table_extract_test.goindex d51bc2e0ce..82a18147c3 100644--- a/internal/deepdoc/parser/pdf/table_extract_test.go+++ b/internal/deepdoc/parser/pdf/table_extract_test.go@@ -2,6 +2,7 @@ package pdfimport ("context"+ "fmt""image""math""testing"@@ -161,3 +162,56 @@ func TestProcessOneTable_CropOffUsesFixedMargin(t *testing.T) {t.Errorf("cropOffY = %v, want %v (region.Y0 - fixed 30px margin)", item.CropOffY, wantOffY)}}++// ocrFillingDoc is like orientationScoringDoc but its OCRRecognize returns+// text for any cropped image. It exists so a test can prove Go does NOT+// perform per-cell OCR on empty TSR cells: even though the OCR engine would+// happily fill any cropped cell, the cell must stay empty. This guards the+// alignment target (Python only fills cells from page-level OCR boxes matched+// via construct_table; it never crops individual cells for recognition).+type ocrFillingDoc struct {+ orientationScoringDoc+}++func (d *ocrFillingDoc) OCRRecognize(_ context.Context, _ image.Image) ([]pdf.OCRText, error) {+ return []pdf.OCRText{{Text: "OCR-FILL", Confidence: 0.9}}, nil+}++// TestProcessOneTable_NoPerCellOCR is a regression guard for the removal of+// ocrTableCells (per-cell OCR). An empty TSR cell with no overlapping+// page-level OCR box must remain empty regardless of table auto-rotation:+// the former rotated path (bestAngle != 0) and the non-rotated path+// (bestAngle == 0) both used to fill such cells via per-cell OCR.+func TestProcessOneTable_NoPerCellOCR(t *testing.T) {+ doc := &ocrFillingDoc{}+ for _, autoRotate := range []bool{false, true} {+ t.Run(fmt.Sprintf("autoRotate=%v", autoRotate), func(t *testing.T) {+ cfg := pdf.DefaultParserConfig()+ cfg.AutoRotateTables = &autoRotate+ cfg.SkipOCR = false+ p := NewParser(cfg)++ pageImg := image.NewRGBA(image.Rect(0, 0, 320, 220))+ // No page-level OCR box overlaps the cell, so FillCellTextFromBoxes+ // leaves it empty; per-cell OCR must not fill it either.+ boxes := []pdf.TextBox{}+ match := tbl.TableMatch{+ Region: pdf.DLARegion{X0: 10, Y0: 10, X1: 210, Y1: 110, Label: pdf.LayoutTypeTable},+ BoxIdx: []int{},+ }+ builder := &staticTableBuilder{+ cells: []pdf.TSRCell{+ {X0: 10, Y0: 20, X1: 60, Y1: 80, Label: "table row", Text: ""},+ },+ }++ item := p.processOneTable(context.Background(), pageImg, boxes, 0, doc, builder, match, pdf.DlaScale)+ if len(item.Cells) != 1 {+ t.Fatalf("cells = %d, want 1", len(item.Cells))+ }+ if item.Cells[0].Text != "" {+ t.Errorf("empty cell filled by per-cell OCR: %q; Go must align with Python, which skips per-cell OCR", item.Cells[0].Text)+ }+ })+ }+}