Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(ingestion): recognize image tables through the DeepDoc page pipeline (#21911)

jay777219ed193674cb8
diff --git a/internal/deepdoc/parser/image/parser.go b/internal/deepdoc/parser/image/parser.go
new file mode 100644
index 0000000000..5ac0942e5d
--- /dev/null
+++ b/internal/deepdoc/parser/image/parser.go
@@ -0,0 +1,187 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Package imageparser runs a standalone image through the DeepDoc page
+// pipeline. An image is a one-page document with no embedded characters, so
+// OCR, layout analysis, table structure recognition, reading order, and table
+// construction all reuse the PDF page path.
+package imageparser
+
+import (
+ "context"
+ "errors"
+ "image"
+ "math"
+ "strings"
+
+ deepdocpdf "ragflow/internal/deepdoc/parser/pdf"
+ util "ragflow/internal/deepdoc/parser/pdf/util"
+ deepdoctype "ragflow/internal/deepdoc/parser/type"
+ "ragflow/internal/entity"
+ "ragflow/internal/parser/tableutil"
+)
+
+// BlockKind identifies one structured block in reading order.
+type BlockKind uint8
+
+const (
+ BlockText BlockKind = iota
+ BlockTable
+ BlockImage
+)
+
+// Block is one text, table, or image block. Image is the table or figure crop
+// for the chunk preview; it is nil for text blocks.
+type Block struct {
+ Kind BlockKind
+ Text string
+ Table *entity.TableData
+ Image image.Image
+}
+
+// pageEngine adapts a decoded image to the PDFEngine interface. ExtractChars is
+// empty so the page takes the full-OCR path, and RawData is nil so the cgo
+// renderer falls back to RenderPageImage instead of rasterising PDF bytes.
+type pageEngine struct {
+ img image.Image
+}
+
+var errNoImagePage = errors.New("image page engine: no image page")
+
+func (e *pageEngine) PageCount() (int, error) { return 1, nil }
+
+func (e *pageEngine) ExtractChars(int) ([]deepdoctype.TextChar, error) { return nil, nil }
+
+func (e *pageEngine) RenderPage(int, float64) ([]byte, error) { return nil, errNoImagePage }
+
+func (e *pageEngine) RenderPageImage(page int, _ float64) (image.Image, error) {
+ if page != 0 || e.img == nil {
+ return nil, errNoImagePage
+ }
+ return e.img, nil
+}
+
+func (e *pageEngine) RawData() []byte { return nil }
+
+func (e *pageEngine) Outlines() ([]deepdoctype.Outline, error) { return nil, nil }
+
+func (e *pageEngine) Close() error { return nil }
+
+// Parse runs the image through the page pipeline and returns text and table
+// blocks in reading order. The analyzer performs OCR, layout recognition, and
+// table structure recognition.
+func Parse(ctx context.Context, img image.Image, analyzer deepdoctype.DocAnalyzer) ([]Block, error) {
+ if img == nil {
+ return nil, errors.New("image parse: nil image")
+ }
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ if analyzer == nil || !analyzer.Health() {
+ return nil, errors.New("local OCR analyzer is unavailable")
+ }
+
+ cfg := deepdoctype.DefaultParserConfig()
+ // The image is already the page render. A higher-DPI retry would reuse the
+ // same pixels while changing the page's coordinate scale.
+ cfg.Zoom = 0
+
+ result, err := deepdocpdf.NewParser(cfg).ParseRaw(ctx, &pageEngine{img: img}, analyzer)
+ if err != nil {
+ return nil, err
+ }
+ defer result.Close()
+ return blocksFromSections(img, result.Sections), nil
+}
+
+// blocksFromSections converts the page pipeline's sections into image blocks.
+// Table sections carry table HTML, figure sections carry a crop region, and
+// text sections keep their OCR text.
+func blocksFromSections(img image.Image, sections []deepdoctype.Section) []Block {
+ blocks := make([]Block, 0, len(sections))
+ for _, section := range sections {
+ switch section.LayoutType {
+ case deepdoctype.LayoutTypeTable:
+ table, err := tableutil.ParseTableHTML(section.Text)
+ if err == nil && table != nil && len(table.Grid) > 0 {
+ blocks = append(blocks, Block{
+ Kind: BlockTable,
+ Table: table,
+ Image: cropSection(img, section.Positions),
+ })
+ continue
+ }
+ // A DLA table region can keep its plain OCR text when TSR produced
+ // no cells: no table HTML replaces the boxes, but they still carry
+ // LayoutType=table. Preserve that text instead of dropping it.
+ lower := strings.ToLower(section.Text)
+ if strings.Contains(lower, "<table") || strings.Contains(lower, "<tr") {
+ continue
+ }
+
+ case deepdoctype.LayoutTypeFigure:
+ if crop := cropSection(img, section.Positions); crop != nil {
+ blocks = append(blocks, Block{
+ Kind: BlockImage,
+ Text: strings.TrimSpace(section.Text),
+ Image: crop,
+ })
+ continue
+ }
+ }
+
+ if text := strings.TrimSpace(section.Text); text != "" {
+ blocks = append(blocks, Block{Kind: BlockText, Text: text})
+ }
+ }
+ return blocks
+}
+
+// cropSection crops the first position, which is the table region. Caption
+// positions appended by MergeCaptions follow it.
+func cropSection(img image.Image, positions []deepdoctype.Position) image.Image {
+ if img == nil || len(positions) == 0 {
+ return nil
+ }
+ position := positions[0]
+ return cropImage(img,
+ position.Left*deepdoctype.DlaScale,
+ position.Top*deepdoctype.DlaScale,
+ position.Right*deepdoctype.DlaScale,
+ position.Bottom*deepdoctype.DlaScale)
+}
+
+func cropImage(img image.Image, x0, y0, x1, y1 float64) image.Image {
+ bounds := img.Bounds()
+ left := clampInt(int(math.Floor(x0)), bounds.Min.X, bounds.Max.X)
+ top := clampInt(int(math.Floor(y0)), bounds.Min.Y, bounds.Max.Y)
+ right := clampInt(int(math.Ceil(x1)), bounds.Min.X, bounds.Max.X)
+ bottom := clampInt(int(math.Ceil(y1)), bounds.Min.Y, bounds.Max.Y)
+ if left >= right || top >= bottom {
+ return nil
+ }
+ return util.FastCrop(img, left, top, right, bottom)
+}
+
+func clampInt(value, low, high int) int {
+ if value < low {
+ return low
+ }
+ if value > high {
+ return high
+ }
+ return value
+}
diff --git a/internal/deepdoc/parser/image/parser_test.go b/internal/deepdoc/parser/image/parser_test.go
new file mode 100644
index 0000000000..c19f6eb9d4
--- /dev/null
+++ b/internal/deepdoc/parser/image/parser_test.go
@@ -0,0 +1,380 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package imageparser
+
+import (
+ "context"
+ "image"
+ "image/color"
+ "image/draw"
+ "strings"
+ "testing"
+
+ deepdoctype "ragflow/internal/deepdoc/parser/type"
+)
+
+type testAnalyzer struct {
+ boxes []deepdoctype.OCRBox
+ texts map[color.RGBA]string
+ regions []deepdoctype.DLARegion
+ cells []deepdoctype.TSRCell
+ unhealthy bool
+}
+
+func (a *testAnalyzer) DLA(context.Context, image.Image) ([]deepdoctype.DLARegion, error) {
+ return a.regions, nil
+}
+
+func (a *testAnalyzer) TSR(context.Context, image.Image) ([]deepdoctype.TSRCell, error) {
+ return a.cells, nil
+}
+
+func (a *testAnalyzer) OCRDetect(context.Context, image.Image) ([]deepdoctype.OCRBox, error) {
+ return a.boxes, nil
+}
+
+func (a *testAnalyzer) OCRRecognize(_ context.Context, img image.Image) ([]deepdoctype.OCRText, error) {
+ bounds := img.Bounds()
+ center := color.RGBAModel.Convert(img.At(
+ bounds.Min.X+bounds.Dx()/2,
+ bounds.Min.Y+bounds.Dy()/2,
+ )).(color.RGBA)
+ text, ok := a.texts[center]
+ if !ok {
+ return nil, nil
+ }
+ return []deepdoctype.OCRText{{Text: text, Confidence: 0.9}}, nil
+}
+
+func (a *testAnalyzer) Health() bool { return !a.unhealthy }
+
+type testCell struct {
+ rect image.Rectangle
+ text string
+ ink color.RGBA
+}
+
+func newTestImage(width, height int, cells []testCell) (image.Image, []deepdoctype.OCRBox, map[color.RGBA]string) {
+ img := image.NewRGBA(image.Rect(0, 0, width, height))
+ draw.Draw(img, img.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src)
+ boxes := make([]deepdoctype.OCRBox, 0, len(cells))
+ texts := make(map[color.RGBA]string, len(cells))
+ for _, cell := range cells {
+ draw.Draw(img, cell.rect, &image.Uniform{C: cell.ink}, image.Point{}, draw.Src)
+ boxes = append(boxes, deepdoctype.OCRBox{
+ X0: float64(cell.rect.Min.X), Y0: float64(cell.rect.Min.Y),
+ X1: float64(cell.rect.Max.X), Y1: float64(cell.rect.Min.Y),
+ X2: float64(cell.rect.Max.X), Y2: float64(cell.rect.Max.Y),
+ X3: float64(cell.rect.Min.X), Y3: float64(cell.rect.Max.Y),
+ })
+ texts[cell.ink] = cell.text
+ }
+ return img, boxes, texts
+}
+
+func tableCells() []deepdoctype.TSRCell {
+ return []deepdoctype.TSRCell{
+ {X0: 0, Y0: 0, X1: 240, Y1: 120, Label: "table"},
+ {X0: 10, Y0: 10, X1: 110, Y1: 50, Label: "table column"},
+ {X0: 110, Y0: 10, X1: 230, Y1: 50, Label: "table column"},
+ {X0: 10, Y0: 10, X1: 230, Y1: 50, Label: "table row"},
+ {X0: 10, Y0: 60, X1: 230, Y1: 100, Label: "table row"},
+ {X0: 10, Y0: 10, X1: 230, Y1: 50, Label: "table column header"},
+ }
+}
+
+func TestPageEngine(t *testing.T) {
+ img := image.NewRGBA(image.Rect(0, 0, 20, 10))
+ engine := &pageEngine{img: img}
+
+ if count, err := engine.PageCount(); err != nil || count != 1 {
+ t.Fatalf("PageCount() = %d, %v", count, err)
+ }
+ if chars, err := engine.ExtractChars(0); err != nil || chars != nil {
+ t.Fatalf("ExtractChars() = %v, %v", chars, err)
+ }
+ rendered, err := engine.RenderPageImage(0, 216)
+ if err != nil || rendered != img {
+ t.Fatalf("RenderPageImage() = %v, %v", rendered, err)
+ }
+ if _, err := engine.RenderPageImage(1, 216); err == nil {
+ t.Fatal("expected an error for a missing page")
+ }
+ if _, err := engine.RenderPage(0, 216); err == nil {
+ t.Fatal("expected an error for PDF byte rendering")
+ }
+ if engine.RawData() != nil {
+ t.Fatal("RawData() must be nil so the renderer falls back to the image")
+ }
+ if outlines, err := engine.Outlines(); err != nil || outlines != nil {
+ t.Fatalf("Outlines() = %v, %v", outlines, err)
+ }
+ if err := engine.Close(); err != nil {
+ t.Fatalf("Close() = %v", err)
+ }
+}
+
+func TestParseFindsTable(t *testing.T) {
+ img, boxes, texts := newTestImage(240, 140, []testCell{
+ {image.Rect(20, 20, 60, 40), "Name", color.RGBA{R: 255, A: 255}},
+ {image.Rect(120, 20, 160, 40), "Value", color.RGBA{G: 255, A: 255}},
+ {image.Rect(20, 80, 50, 100), "A", color.RGBA{B: 255, A: 255}},
+ {image.Rect(120, 80, 150, 100), "42", color.RGBA{R: 255, G: 255, A: 255}},
+ })
+ analyzer := &testAnalyzer{
+ boxes: boxes,
+ texts: texts,
+ regions: []deepdoctype.DLARegion{{X0: 0, Y0: 0, X1: 240, Y1: 120, Label: "table", Confidence: 0.9}},
+ cells: tableCells(),
+ }
+
+ blocks, err := Parse(t.Context(), img, analyzer)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(blocks) != 1 || blocks[0].Kind != BlockTable {
+ t.Fatalf("blocks = %+v, want one table block", blocks)
+ }
+ table := blocks[0].Table
+ if table == nil || len(table.Grid) != 2 || len(table.Grid[0]) != 2 {
+ t.Fatalf("table = %+v", table)
+ }
+ if table.HeaderRows != 1 {
+ t.Fatalf("header rows = %d, want 1", table.HeaderRows)
+ }
+ if got := table.Grid[0][0].Text; got != "Name" {
+ t.Errorf("cell[0][0] = %q", got)
+ }
+ if got := table.Grid[1][1].Text; got != "42" {
+ t.Errorf("cell[1][1] = %q", got)
+ }
+ if blocks[0].Image == nil {
+ t.Error("table preview is nil")
+ }
+}
+
+func TestParseKeepsTextWhenNoTable(t *testing.T) {
+ img, boxes, texts := newTestImage(240, 140, []testCell{
+ {image.Rect(20, 20, 60, 40), "Name", color.RGBA{R: 255, A: 255}},
+ {image.Rect(120, 20, 160, 40), "Value", color.RGBA{G: 255, A: 255}},
+ {image.Rect(20, 80, 50, 100), "A", color.RGBA{B: 255, A: 255}},
+ {image.Rect(120, 80, 150, 100), "42", color.RGBA{R: 255, G: 255, A: 255}},
+ })
+ blocks, err := Parse(t.Context(), img, &testAnalyzer{boxes: boxes, texts: texts})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(blocks) == 0 {
+ t.Fatal("expected text blocks")
+ }
+ var text strings.Builder
+ for _, block := range blocks {
+ if block.Kind != BlockText {
+ t.Fatalf("block = %+v, want text", block)
+ }
+ text.WriteString(block.Text)
+ text.WriteByte('\n')
+ }
+ for _, want := range []string{"Name", "Value", "A", "42"} {
+ if !strings.Contains(text.String(), want) {
+ t.Errorf("text %q does not contain %q", text.String(), want)
+ }
+ }
+}
+
+func TestParseMergesTableCaption(t *testing.T) {
+ img, boxes, texts := newTestImage(240, 180, []testCell{
+ {image.Rect(20, 45, 80, 55), "Table 1", color.RGBA{R: 128, G: 128, B: 128, A: 255}},
+ {image.Rect(40, 70, 80, 90), "Name", color.RGBA{R: 255, A: 255}},
+ {image.Rect(150, 70, 190, 90), "Value", color.RGBA{G: 255, A: 255}},
+ {image.Rect(40, 120, 70, 140), "A", color.RGBA{B: 255, A: 255}},
+ {image.Rect(150, 120, 180, 140), "42", color.RGBA{R: 255, G: 255, A: 255}},
+ })
+ analyzer := &testAnalyzer{
+ boxes: boxes,
+ texts: texts,
+ regions: []deepdoctype.DLARegion{{X0: 0, Y0: 60, X1: 240, Y1: 160, Label: "table", Confidence: 0.9}},
+ cells: []deepdoctype.TSRCell{
+ {X0: 0, Y0: 0, X1: 240, Y1: 130, Label: "table"},
+ {X0: 10, Y0: 40, X1: 110, Y1: 110, Label: "table column"},
+ {X0: 110, Y0: 40, X1: 230, Y1: 110, Label: "table column"},
+ {X0: 10, Y0: 40, X1: 230, Y1: 60, Label: "table row"},
+ {X0: 10, Y0: 90, X1: 230, Y1: 110, Label: "table row"},
+ {X0: 10, Y0: 40, X1: 230, Y1: 60, Label: "table column header"},
+ },
+ }
+
+ blocks, err := Parse(t.Context(), img, analyzer)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(blocks) != 1 || blocks[0].Kind != BlockTable {
+ t.Fatalf("blocks = %+v, want one table block", blocks)
+ }
+ if got := blocks[0].Table.Caption; got != "Table 1" {
+ t.Fatalf("caption = %q", got)
+ }
+ if got := blocks[0].Table.Grid[0][0].Text; got != "Name" {
+ t.Fatalf("header = %q", got)
+ }
+}
+
+func TestParseKeepsSingleColumnTable(t *testing.T) {
+ img, boxes, texts := newTestImage(240, 140, []testCell{
+ {image.Rect(20, 20, 60, 40), "Name", color.RGBA{R: 255, A: 255}},
+ {image.Rect(20, 80, 50, 100), "A", color.RGBA{B: 255, A: 255}},
+ })
+ analyzer := &testAnalyzer{
+ boxes: boxes,
+ texts: texts,
+ regions: []deepdoctype.DLARegion{{X0: 0, Y0: 0, X1: 240, Y1: 120, Label: "table", Confidence: 0.9}},
+ cells: []deepdoctype.TSRCell{
+ {X0: 0, Y0: 0, X1: 240, Y1: 120, Label: "table"},
+ {X0: 10, Y0: 10, X1: 230, Y1: 50, Label: "table row"},
+ {X0: 10, Y0: 60, X1: 230, Y1: 100, Label: "table row"},
+ },
+ }
+
+ blocks, err := Parse(t.Context(), img, analyzer)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(blocks) != 1 || blocks[0].Kind != BlockTable {
+ t.Fatalf("blocks = %+v, want one table block", blocks)
+ }
+ table := blocks[0].Table
+ if table == nil || len(table.Grid) != 2 || len(table.Grid[0]) != 1 {
+ t.Fatalf("table = %+v", table)
+ }
+ if table.Grid[0][0].Text != "Name" || table.Grid[1][0].Text != "A" {
+ t.Fatalf("table grid = %+v", table.Grid)
+ }
+ if blocks[0].Image == nil {
+ t.Error("table preview is nil")
+ }
+}
+
+func TestParseEmitsFigureImage(t *testing.T) {
+ img, boxes, texts := newTestImage(240, 140, []testCell{
+ {image.Rect(20, 20, 60, 40), "Chart", color.RGBA{R: 10, G: 20, B: 30, A: 255}},
+ })
+ analyzer := &testAnalyzer{
+ boxes: boxes,
+ texts: texts,
+ regions: []deepdoctype.DLARegion{{X0: 0, Y0: 0, X1: 240, Y1: 120, Label: "figure", Confidence: 0.9}},
+ }
+
+ blocks, err := Parse(t.Context(), img, analyzer)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(blocks) != 1 || blocks[0].Kind != BlockImage {
+ t.Fatalf("blocks = %+v, want one image block", blocks)
+ }
+ if blocks[0].Image == nil {
+ t.Fatal("figure crop is nil")
+ }
+}
+
+func TestParseTableAndFigure(t *testing.T) {
+ img, boxes, texts := newTestImage(300, 140, []testCell{
+ {image.Rect(20, 20, 60, 40), "Name", color.RGBA{R: 255, A: 255}},
+ {image.Rect(20, 80, 50, 100), "A", color.RGBA{B: 255, A: 255}},
+ {image.Rect(220, 20, 260, 40), "Chart", color.RGBA{R: 10, G: 20, B: 30, A: 255}},
+ })
+ analyzer := &testAnalyzer{
+ boxes: boxes,
+ texts: texts,
+ regions: []deepdoctype.DLARegion{
+ {X0: 0, Y0: 0, X1: 180, Y1: 120, Label: "table", Confidence: 0.9},
+ {X0: 200, Y0: 0, X1: 300, Y1: 120, Label: "figure", Confidence: 0.9},
+ },
+ cells: []deepdoctype.TSRCell{
+ {X0: 0, Y0: 0, X1: 180, Y1: 120, Label: "table"},
+ {X0: 10, Y0: 10, X1: 170, Y1: 50, Label: "table row"},
+ {X0: 10, Y0: 60, X1: 170, Y1: 100, Label: "table row"},
+ },
+ }
+
+ blocks, err := Parse(t.Context(), img, analyzer)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var tables, images int
+ for _, block := range blocks {
+ switch block.Kind {
+ case BlockTable:
+ tables++
+ case BlockImage:
+ images++
+ if block.Image == nil {
+ t.Fatal("figure crop is nil")
+ }
+ }
+ }
+ if tables != 1 || images != 1 {
+ t.Fatalf("blocks = %+v, want one table and one image", blocks)
+ }
+}
+
+func TestParseKeepsTextWhenTableStructureMissing(t *testing.T) {
+ img, boxes, texts := newTestImage(240, 140, []testCell{
+ {image.Rect(20, 20, 60, 40), "Name", color.RGBA{R: 255, A: 255}},
+ {image.Rect(20, 80, 50, 100), "A", color.RGBA{B: 255, A: 255}},
+ })
+ analyzer := &testAnalyzer{
+ boxes: boxes,
+ texts: texts,
+ regions: []deepdoctype.DLARegion{{X0: 0, Y0: 0, X1: 240, Y1: 120, Label: "table", Confidence: 0.9}},
+ // No TSR cells: the table region keeps its plain OCR text.
+ }
+
+ blocks, err := Parse(t.Context(), img, analyzer)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, block := range blocks {
+ if block.Kind == BlockTable {
+ t.Fatalf("table structure is missing; got a table block: %+v", block)
+ }
+ }
+ var text strings.Builder
+ for _, block := range blocks {
+ if block.Kind == BlockText {
+ text.WriteString(block.Text)
+ text.WriteByte('\n')
+ }
+ }
+ for _, want := range []string{"Name", "A"} {
+ if !strings.Contains(text.String(), want) {
+ t.Fatalf("text %q does not contain %q", text.String(), want)
+ }
+ }
+}
+
+func TestParseNilImage(t *testing.T) {
+ if _, err := Parse(context.Background(), nil, &testAnalyzer{}); err == nil {
+ t.Fatal("expected an error for a nil image")
+ }
+}
+
+func TestParseUnhealthyAnalyzer(t *testing.T) {
+ img := image.NewRGBA(image.Rect(0, 0, 10, 10))
+ if _, err := Parse(context.Background(), img, &testAnalyzer{unhealthy: true}); err == nil {
+ t.Fatal("expected an error for an unhealthy analyzer")
+ }
+}
diff --git a/internal/ingestion/component/image_table_test.go b/internal/ingestion/component/image_table_test.go
new file mode 100644
index 0000000000..3d6d3de82a
--- /dev/null
+++ b/internal/ingestion/component/image_table_test.go
@@ -0,0 +1,141 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "bytes"
+ "context"
+ "image"
+ "image/png"
+ "testing"
+
+ imageparser "ragflow/internal/deepdoc/parser/image"
+ deepdoctype "ragflow/internal/deepdoc/parser/type"
+ "ragflow/internal/entity"
+ "ragflow/internal/parser/parser"
+ "ragflow/internal/utility"
+)
+
+type pictureTableAnalyzer struct {
+ deepdoctype.DocAnalyzer
+}
+
+func (*pictureTableAnalyzer) Health() bool { return true }
+
+func (*pictureTableAnalyzer) DLA(context.Context, image.Image) ([]deepdoctype.DLARegion, error) {
+ return []deepdoctype.DLARegion{{X0: 0, Y0: 0, X1: 240, Y1: 120, Label: "table", Confidence: 0.9}}, nil
+}
+
+func (*pictureTableAnalyzer) TSR(context.Context, image.Image) ([]deepdoctype.TSRCell, error) {
+ return []deepdoctype.TSRCell{
+ {X0: 0, Y0: 0, X1: 240, Y1: 120, Label: "table"},
+ {X0: 10, Y0: 10, X1: 110, Y1: 50, Label: "table column"},
+ {X0: 110, Y0: 10, X1: 230, Y1: 50, Label: "table column"},
+ {X0: 10, Y0: 10, X1: 230, Y1: 50, Label: "table row"},
+ {X0: 10, Y0: 60, X1: 230, Y1: 100, Label: "table row"},
+ {X0: 10, Y0: 10, X1: 230, Y1: 50, Label: "table column header"},
+ }, nil
+}
+
+func (*pictureTableAnalyzer) OCRDetect(context.Context, image.Image) ([]deepdoctype.OCRBox, error) {
+ return []deepdoctype.OCRBox{
+ {X0: 20, Y0: 20, X1: 60, Y1: 20, X2: 60, Y2: 40, X3: 20, Y3: 40},
+ {X0: 120, Y0: 20, X1: 170, Y1: 20, X2: 170, Y2: 40, X3: 120, Y3: 40},
+ {X0: 20, Y0: 90, X1: 50, Y1: 90, X2: 50, Y2: 110, X3: 20, Y3: 110},
+ {X0: 120, Y0: 90, X1: 180, Y1: 90, X2: 180, Y2: 110, X3: 120, Y3: 110},
+ }, nil
+}
+
+func (*pictureTableAnalyzer) OCRRecognize(_ context.Context, img image.Image) ([]deepdoctype.OCRText, error) {
+ text := map[int]string{30: "A", 40: "Name", 50: "Value", 60: "42"}[img.Bounds().Dx()]
+ return []deepdoctype.OCRText{{Text: text, Confidence: 0.99}}, nil
+}
+
+func tablePicturePNG(t *testing.T) []byte {
+ t.Helper()
+ var data bytes.Buffer
+ if err := png.Encode(&data, image.NewRGBA(image.Rect(0, 0, 240, 140))); err != nil {
+ t.Fatal(err)
+ }
+ return data.Bytes()
+}
+
+func TestMaybeDispatchImageEmitsTableItem(t *testing.T) {
+ original := deepdoctype.NativeDocAnalyzerFactory
+ t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = original })
+ analyzer := &pictureTableAnalyzer{}
+ deepdoctype.NativeDocAnalyzerFactory = func() (deepdoctype.DocAnalyzer, bool) { return analyzer, true }
+
+ result, handled, err := maybeDispatchImage(t.Context(), parser.NewPictureParser(), nil, utility.FileTypeVISUAL, "table.png", tablePicturePNG(t), map[string]any{"tenant_id": "t1"}, defaultSetups(), false, visionSettings{})
+ if err != nil {
+ t.Fatalf("maybeDispatchImage: %v", err)
+ }
+ if !handled {
+ t.Fatal("expected the image branch to handle the file")
+ }
+ if len(result.JSON) != 1 {
+ t.Fatalf("JSON len = %d, want one table item: %+v", len(result.JSON), result.JSON)
+ }
+ item := result.JSON[0].ToMap()
+ if item["doc_type_kwd"] != "table" {
+ t.Fatalf("doc_type_kwd = %v, want table", item["doc_type_kwd"])
+ }
+ table, ok := item["table"].(*entity.TableData)
+ if !ok || table == nil {
+ t.Fatalf("table payload = %#v", item["table"])
+ }
+ if table.HeaderRows != 1 || len(table.Grid) != 2 || len(table.Grid[0]) != 2 {
+ t.Fatalf("table shape = %+v", table)
+ }
+ if table.Grid[0][0].Text != "Name" || table.Grid[1][1].Text != "42" {
+ t.Fatalf("table grid = %+v", table.Grid)
+ }
+ if item["image"] == nil || item["image"] == "" {
+ t.Fatal("table item lost its crop preview")
+ }
+ if _, has := item["positions"]; has {
+ t.Fatal("image table item must not carry PDF positions")
+ }
+ if len(result.Warnings) != 0 {
+ t.Fatalf("warnings = %v", result.Warnings)
+ }
+}
+
+func TestImageBlockItemsEmitsFigureItem(t *testing.T) {
+ img := image.NewRGBA(image.Rect(0, 0, 10, 10))
+ items := imageBlockItems([]imageparser.Block{{
+ Kind: imageparser.BlockImage,
+ Text: "chart",
+ Image: img,
+ }})
+ if len(items) != 1 {
+ t.Fatalf("items = %d, want one figure item", len(items))
+ }
+ item := items[0].ToMap()
+ if item["doc_type_kwd"] != "image" {
+ t.Fatalf("doc_type_kwd = %v, want image", item["doc_type_kwd"])
+ }
+ if item["text"] != "chart" {
+ t.Fatalf("text = %v, want chart", item["text"])
+ }
+ if item["layout"] != "figure" {
+ t.Fatalf("layout = %v, want figure", item["layout"])
+ }
+ if item["image"] == nil || item["image"] == "" {
+ t.Fatal("figure item lost its crop preview")
+ }
+}
diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.go
index 72706e3d12..7bb9b79e2a 100644
--- a/internal/ingestion/component/media_dispatch.go
+++ b/internal/ingestion/component/media_dispatch.go
@@ -14,25 +14,27 @@
// limitations under the License.
//
-// Media dispatch runs image OCR and optional vision enhancement, audio
-// transcription, and video dispatch at the component layer.
+// Media dispatch runs image OCR, table structure recognition, optional vision
+// enhancement, audio transcription, and video dispatch at the component layer.
package component
import (
"bytes"
"context"
+ "encoding/base64"
"fmt"
"image"
+ "image/png"
"os"
"path/filepath"
- "sort"
"strings"
"go.uber.org/zap"
"ragflow/internal/common"
- deepdocpdf "ragflow/internal/deepdoc/parser/pdf"
+ imageparser "ragflow/internal/deepdoc/parser/image"
+ deepdoctype "ragflow/internal/deepdoc/parser/type"
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
"ragflow/internal/ingestion/component/progress"
@@ -77,8 +79,10 @@ func maybeDispatchVideo(
fmt.Errorf("Parser: video parsing is not yet supported; underlying video analysis capability is pending")
}
-// maybeDispatchImage extracts local or selected-model OCR (native text for SVG),
-// preserves the original image and optionally appends global vision enhancement.
+// maybeDispatchImage extracts local or selected-model OCR (native text for SVG)
+// and emits structured table/text/figure items when the page pipeline finds
+// them. Without a table it keeps the single whole-image item; with a table it
+// emits text and table/figure crops instead, matching PDF page output.
func maybeDispatchImage(
ctx context.Context,
p parser.ParseResultProducer,
@@ -118,6 +122,7 @@ func maybeDispatchImage(
isSVG := isSVGFilename(filename)
var text string
var ocrErr error
+ var blocks []imageparser.Block
if isSVG {
text, err = extractSVGText(ctx, binary)
if err != nil {
@@ -131,7 +136,14 @@ func maybeDispatchImage(
}
if useOCR {
if ocrModelID == "" {
- text, ocrErr = extractImageText(ctx, img)
+ var analyzer deepdoctype.DocAnalyzer
+ analyzer, ocrErr = parser.GetDocAnalyzer()
+ if ocrErr == nil {
+ blocks, ocrErr = imageparser.Parse(ctx, img, analyzer)
+ if ocrErr == nil {
+ text = imageBlockText(blocks)
+ }
+ }
} else {
// Selected OCR uses original bytes, so media admission need not
// cover database resolution or remote inference.
@@ -148,15 +160,21 @@ func maybeDispatchImage(
if len(parsed.JSON) == 0 {
return parsed, true, fmt.Errorf("parser: image parser returned no image item")
}
- imageData, _ := parsed.JSON[0].ToMap()["image"].(string)
- m := parsed.JSON[0].ToMap()
- m["text"] = text
- parsed.JSON[0] = parser.DocItemFromMap(m)
+ imageItem := parsed.JSON[0].ToMap()
+ imageData, _ := imageItem["image"].(string)
+ hasTables := imageHasTables(blocks)
+ var items []parser.DocItem
+ if hasTables {
+ items = imageBlockItems(blocks)
+ } else {
+ imageItem["text"] = text
+ items = []parser.DocItem{parser.DocItemFromMap(imageItem)}
+ }
if ocrErr != nil {
parsed.Warnings = append(parsed.Warnings, fmt.Sprintf("image OCR unavailable: %v", ocrErr))
} else if isSVG && text == "" {
parsed.Warnings = append(parsed.Warnings, "image SVG contains no text")
- } else if useOCR && strings.TrimSpace(text) == "" {
+ } else if useOCR && strings.TrimSpace(text) == "" && !hasTables {
parsed.Warnings = append(parsed.Warnings, "image OCR returned no text")
}
if !useOCR && !enableVisionEnhancement {
@@ -185,10 +203,18 @@ func maybeDispatchImage(
description, warnings := describeImage(ctx, db, imageData, getStringOr(inputs, "tenant_id", ""), setup, inputs, imageVision)
parsed.Warnings = append(parsed.Warnings, warnings...)
if description != "" {
- m := parsed.JSON[0].ToMap()
- appendItemText(m, description)
- parsed.JSON[0] = parser.DocItemFromMap(m)
- } else if strings.TrimSpace(text) == "" {
+ if hasTables {
+ if imageData != "" {
+ items = append(items, parser.ImageItem{Text: description, Image: imageData})
+ } else {
+ items = append(items, parser.NewTextJSONItem(description))
+ }
+ } else {
+ imageItem["text"] = text
+ appendItemText(imageItem, description)
+ items[0] = parser.DocItemFromMap(imageItem)
+ }
+ } else if strings.TrimSpace(text) == "" && !hasTables {
// Enhancement was permitted and the model produced nothing usable,
// so the item still has no text at all. Without this the chunk is
// dropped downstream with no trace, which is what the two other
@@ -197,6 +223,7 @@ func maybeDispatchImage(
"image has no searchable text: the vision model returned no description, so the Tokenizer will discard this item")
}
}
+ parsed.JSON = items
return parsed, true, nil
}
@@ -227,28 +254,70 @@ func decodeDispatchImage(data []byte, decodeRaster bool) (image.Image, error) {
return img, nil
}
-func extractImageText(ctx context.Context, img image.Image) (string, error) {
- analyzer, err := parser.GetDocAnalyzer()
- if err != nil {
- return "", err
- }
- if analyzer == nil || !analyzer.Health() {
- return "", fmt.Errorf("local OCR analyzer is unavailable")
- }
- boxes := deepdocpdf.OCRImage(ctx, img, analyzer, 1.0)
- sort.SliceStable(boxes, func(i, j int) bool {
- if boxes[i].Top == boxes[j].Top {
- return boxes[i].X0 < boxes[j].X0
+// imageBlockText is the flat OCR text used by the single-image fallback and
+// the vision-description path. Table content stays in the table items.
+func imageBlockText(blocks []imageparser.Block) string {
+ texts := make([]string, 0, len(blocks))
+ for _, block := range blocks {
+ if block.Kind != imageparser.BlockText && block.Kind != imageparser.BlockImage {
+ continue
}
- return boxes[i].Top < boxes[j].Top
- })
- texts := make([]string, 0, len(boxes))
- for _, box := range boxes {
- if text := strings.TrimSpace(box.Text); text != "" {
+ if text := strings.TrimSpace(block.Text); text != "" {
texts = append(texts, text)
}
}
- return strings.Join(texts, "\n"), nil
+ return strings.Join(texts, "\n")
+}
+
+func imageHasTables(blocks []imageparser.Block) bool {
+ for _, block := range blocks {
+ if block.Kind == imageparser.BlockTable && block.Table != nil {
+ return true
+ }
+ }
+ return false
+}
+
+func imageBlockItems(blocks []imageparser.Block) []parser.DocItem {
+ items := make([]parser.DocItem, 0, len(blocks))
+ for _, block := range blocks {
+ switch block.Kind {
+ case imageparser.BlockText:
+ if strings.TrimSpace(block.Text) == "" {
+ continue
+ }
+ items = append(items, parser.TextItem{Text: block.Text})
+ case imageparser.BlockTable:
+ if block.Table == nil {
+ continue
+ }
+ extra := map[string]any{"layout": "table"}
+ if uri := encodeMediaImage(block.Image); uri != "" {
+ extra["image"] = uri
+ }
+ items = append(items, parser.TableItem{Table: block.Table, Extra: extra})
+ case imageparser.BlockImage:
+ if uri := encodeMediaImage(block.Image); uri != "" {
+ items = append(items, parser.ImageItem{
+ Text: block.Text,
+ Image: uri,
+ Extra: map[string]any{"layout": "figure"},
+ })
+ }
+ }
+ }
+ return items
+}
+
+func encodeMediaImage(img image.Image) string {
+ if img == nil {
+ return ""
+ }
+ var buf bytes.Buffer
+ if err := png.Encode(&buf, img); err != nil {
+ return ""
+ }
+ return "data:image/png;base64," + base64.StdEncoding.EncodeToString(buf.Bytes())
}
func describeImage(
diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.go
index 96a668795a..2628157a54 100644
--- a/internal/ingestion/component/media_dispatch_test.go
+++ b/internal/ingestion/component/media_dispatch_test.go
@@ -50,6 +50,14 @@ type pictureOCRAnalyzer struct {
func (*pictureOCRAnalyzer) Health() bool { return true }
+func (*pictureOCRAnalyzer) DLA(context.Context, image.Image) ([]deepdoctype.DLARegion, error) {
+ return nil, nil
+}
+
+func (*pictureOCRAnalyzer) TSR(context.Context, image.Image) ([]deepdoctype.TSRCell, error) {
+ return nil, nil
+}
+
func (a *pictureOCRAnalyzer) OCRDetect(ctx context.Context, img image.Image) ([]deepdoctype.OCRBox, error) {
if a.empty {
return nil, nil