Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): restore picture OCR independently of VLM enhancement (#20424)

jay77721285b2c290320
diff --git a/internal/deepdoc/parser/pdf/parser_ocr.go b/internal/deepdoc/parser/pdf/parser_ocr.go
index 5b046fbf35..456d2e9cdd 100644
--- a/internal/deepdoc/parser/pdf/parser_ocr.go
+++ b/internal/deepdoc/parser/pdf/parser_ocr.go
@@ -24,6 +24,13 @@ import (
// only to its own local max width — exactly what the Python reference does.
const recBatchNum = 16
+// OCRImage detects and recognizes text using the same de-skewing, rotation
+// selection, and batched recognition as PDF pages. zoom converts image pixels
+// to output coordinates; standalone images use 1.
+func OCRImage(ctx context.Context, img image.Image, doc pdf.DocAnalyzer, zoom float64) []pdf.TextBox {
+ return (&Parser{}).ocrDetectAndRecognize(ctx, img, doc, 0, "image", zoom)
+}
+
func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image, doc pdf.DocAnalyzer, pageNum int, logLabel string, zoom float64) []pdf.TextBox {
boxes, err := p.inferOCRDetect(ctx, doc, pageImg)
if err != nil || len(boxes) == 0 {
diff --git a/internal/ingestion/component/dispatch_model.go b/internal/ingestion/component/dispatch_model.go
index 8aad0b2665..649fd683f6 100644
--- a/internal/ingestion/component/dispatch_model.go
+++ b/internal/ingestion/component/dispatch_model.go
@@ -47,8 +47,7 @@ var resolveModelConfig = defaultResolveModelConfig
// configuredMediaModelID extracts a per-call model reference from a parser setup.
// Image parsing stores the VLM model reference in parse_method when it is not
-// "ocr" (mirroring Python rag/flow/parser/parser.py:_image); "ocr" selects the
-// tenant default vision model, and Go does not run OCR for standalone images.
+// an OCR method. OCR backend selection is independent of the VLM model.
// Other media families use vlm.llm_id, matching the frontend parser form.
func configuredMediaModelID(setup schema.ParserSetup, family string) string {
if family == "image" {
diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.go
index 8511798f06..cada98751d 100644
--- a/internal/ingestion/component/media_dispatch.go
+++ b/internal/ingestion/component/media_dispatch.go
@@ -14,21 +14,25 @@
// limitations under the License.
//
-// Media dispatch: image, audio, and video branches that require model access
-// (IMAGE2TEXT, SPEECH2TEXT) at the component layer.
+// Media dispatch runs image OCR and optional vision enhancement, audio
+// transcription, and video dispatch at the component layer.
package component
import (
+ "bytes"
"context"
"fmt"
+ "image"
"os"
"path/filepath"
+ "sort"
"strings"
"go.uber.org/zap"
"ragflow/internal/common"
+ deepdocpdf "ragflow/internal/deepdoc/parser/pdf"
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
"ragflow/internal/ingestion/component/schema"
@@ -71,7 +75,7 @@ func maybeDispatchVideo(
fmt.Errorf("Parser: video parsing is not yet supported; underlying video analysis capability is pending")
}
-// Image dispatch: IMAGE2TEXT vision describe ---
+// Image dispatch: OCR followed by optional IMAGE2TEXT enhancement.
func maybeDispatchImage(
ctx context.Context,
@@ -90,50 +94,108 @@ func maybeDispatchImage(
if !ok {
return parser.ParseResult{}, false, nil
}
- tenantID := getStringOr(inputs, "tenant_id", "")
- if !enableVisionEnhancement {
- return parser.ParseResult{}, true, fmt.Errorf("parser: image has no searchable text because vision enhancement is disabled")
+ method := getStringOr(setup, "parse_method", "")
+ useOCR := method == "" || strings.EqualFold(method, "ocr")
+ release, err := parser.AcquireImageMedia(ctx)
+ if err != nil {
+ return parser.ParseResult{}, true, err
+ }
+ defer release()
+ img, err := decodeDispatchImage(binary, useOCR)
+ if err != nil {
+ return parser.ParseResult{}, true, err
+ }
+ var text string
+ if useOCR {
+ text, err = extractImageText(ctx, img)
}
+ release()
parsed := dispatchParse(ctx, fileType, filename, binary, setups)
if parsed.Err != nil {
return parsed, true, parsed.Err
}
if len(parsed.JSON) == 0 {
- return parser.ParseResult{}, true, fmt.Errorf("parser: image parser returned no image item")
+ return parsed, true, fmt.Errorf("parser: image parser returned no image item")
}
+ parsed.OutputFormat = "json"
imageData, _ := parsed.JSON[0]["image"].(string)
- if imageData == "" {
- return parser.ParseResult{}, true, fmt.Errorf("parser: image parser returned no image payload")
- }
- result, handled, err := maybeDispatchImageVLM(ctx, db, imageData, tenantID, setup, inputs)
- result.Warnings = append(result.Warnings, parsed.Warnings...)
- result.File = parsed.File
- if err == nil && len(result.JSON) > 0 {
- text, _ := result.JSON[0]["text"].(string)
- if strings.TrimSpace(text) == "" {
- if len(result.Warnings) > 0 {
- return result, true, fmt.Errorf("parser: image has no searchable text: %s", result.Warnings[0])
- }
- return result, true, fmt.Errorf("parser: vision enhancement returned no searchable text")
+ parsed.JSON[0]["text"] = text
+ if err != nil {
+ parsed.Warnings = append(parsed.Warnings, fmt.Sprintf("image OCR unavailable: %v", err))
+ } else if useOCR && strings.TrimSpace(text) == "" {
+ parsed.Warnings = append(parsed.Warnings, "image OCR returned no text")
+ }
+ if err := ctx.Err(); err != nil {
+ return parsed, true, err
+ }
+ if enableVisionEnhancement {
+ description, warnings := describeImage(ctx, db, imageData, getStringOr(inputs, "tenant_id", ""), setup, inputs)
+ parsed.Warnings = append(parsed.Warnings, warnings...)
+ if description != "" {
+ appendItemText(parsed.JSON[0], description)
+ }
+ }
+ return parsed, true, nil
+}
+
+func decodeDispatchImage(data []byte, decodeRaster bool) (image.Image, error) {
+ if len(data) == 0 || len(data) > parser.MaxImagePayloadBytes {
+ return nil, fmt.Errorf("parser: image payload exceeds size limits")
+ }
+ config, _, err := image.DecodeConfig(bytes.NewReader(data))
+ if err != nil {
+ return nil, fmt.Errorf("parser: decode image: %w", err)
+ }
+ if config.Width <= 0 || config.Height <= 0 || config.Width > parser.MaxImageEdge || config.Height > parser.MaxImageEdge || int64(config.Width)*int64(config.Height) > parser.MaxImagePixels {
+ return nil, fmt.Errorf("parser: image dimensions %dx%d exceed limits", config.Width, config.Height)
+ }
+ // VLM consumes the original bytes; only local OCR needs a decoded raster.
+ if !decodeRaster {
+ return nil, nil
+ }
+ img, _, err := image.Decode(bytes.NewReader(data))
+ if err != nil {
+ return nil, fmt.Errorf("parser: decode image: %w", err)
+ }
+ return img, nil
+}
+
+func extractImageText(ctx context.Context, img image.Image) (string, error) {
+ analyzer, err := parser.GetDocAnalyzer()
+ if err != nil {
+ return "", err
+ }
+ if analyzer == nil || !analyzer.Health() {
+ return "", fmt.Errorf("local OCR analyzer is unavailable")
+ }
+ boxes := deepdocpdf.OCRImage(ctx, img, analyzer, 1.0)
+ sort.SliceStable(boxes, func(i, j int) bool {
+ if boxes[i].Top == boxes[j].Top {
+ return boxes[i].X0 < boxes[j].X0
+ }
+ return boxes[i].Top < boxes[j].Top
+ })
+ texts := make([]string, 0, len(boxes))
+ for _, box := range boxes {
+ if text := strings.TrimSpace(box.Text); text != "" {
+ texts = append(texts, text)
}
}
- return result, handled, err
+ return strings.Join(texts, "\n"), nil
}
-func maybeDispatchImageVLM(
+func describeImage(
ctx context.Context,
db *gorm.DB,
dataURI string,
tenantID string,
setup schema.ParserSetup,
inputs map[string]any,
-) (parser.ParseResult, bool, error) {
+) (string, []string) {
// --- Optional VLM description ---
lang := resolveVisionLanguage(inputs, getStringOr(setup, "lang", ""))
if tenantID == "" {
- result := imageDispatchResult("", dataURI)
- result.Warnings = append(result.Warnings, "image VLM enhancement skipped: tenant ID is missing")
- return result, true, nil
+ return "", []string{"image VLM enhancement skipped: tenant ID is missing"}
}
// Use the configured image VLM or the tenant default.
@@ -156,9 +218,7 @@ func maybeDispatchImageVLM(
err = fmt.Errorf("no usable vision model")
}
if err != nil {
- result := imageDispatchResult("", dataURI)
- result.Warnings = append(result.Warnings, fmt.Sprintf("image VLM enhancement skipped: model unavailable: %v", err))
- return result, true, nil
+ return "", []string{fmt.Sprintf("image VLM enhancement skipped: model unavailable: %v", err)}
}
prompt := defaultImageVisionPrompt(lang)
@@ -178,31 +238,14 @@ func maybeDispatchImageVLM(
vision := true
resp, err := driver.ChatWithMessages(ctx, modelName, messages, apiConfig, &modelModule.ChatConfig{Vision: &vision}, nil)
if err != nil {
- result := imageDispatchResult("", dataURI)
- result.Warnings = append(result.Warnings, fmt.Sprintf("image VLM enhancement failed: %v", err))
- return result, true, nil
+ return "", []string{fmt.Sprintf("image VLM enhancement failed: %v", err)}
}
vlmText := ""
if resp != nil && resp.Answer != nil {
vlmText = strings.TrimSpace(*resp.Answer)
}
- return imageDispatchResult(vlmText, dataURI), true, nil
-}
-
-// imageDispatchResult builds the structured JSON payload for the image
-// family: a single item carrying the VLM description, the image attachment
-// (data URI), and doc_type_kwd "image". Mirrors Python
-// rag/app/picture.py:71-72.
-func imageDispatchResult(text, dataURI string) parser.ParseResult {
- return parser.ParseResult{
- OutputFormat: "json",
- JSON: []map[string]any{{
- "text": text,
- "image": dataURI,
- "doc_type_kwd": "image",
- }},
- }
+ return vlmText, nil
}
// Audio dispatch: SPEECH2TEXT transcription ---
diff --git a/internal/ingestion/component/media_dispatch_integration_test.go b/internal/ingestion/component/media_dispatch_integration_test.go
new file mode 100644
index 0000000000..485d69f0ab
--- /dev/null
+++ b/internal/ingestion/component/media_dispatch_integration_test.go
@@ -0,0 +1,44 @@
+//go:build cgo && integration
+
+package component
+
+import (
+ _ "embed"
+ "os"
+ "strings"
+ "testing"
+
+ nativeanalyzer "ragflow/internal/deepdoc/parser/pdf/inference/native_analyzer"
+ deepdoctype "ragflow/internal/deepdoc/parser/type"
+ "ragflow/internal/utility"
+)
+
+//go:embed testdata/picture_ocr.png
+var pictureOCRFixture []byte
+
+func TestMaybeDispatchImageNativeOCRWithoutVision(t *testing.T) {
+ modelDir := os.Getenv("MODEL_DIR")
+ if modelDir == "" {
+ t.Skip("MODEL_DIR required for native OCR integration")
+ }
+ original := deepdoctype.NativeDocAnalyzerFactory
+ t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = original })
+ if err := nativeanalyzer.Register(modelDir, nativeanalyzer.DefaultDropScore); err != nil {
+ t.Fatal(err)
+ }
+ result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL,
+ "picture_ocr.png", pictureOCRFixture, nil, defaultSetups(), false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !handled || len(result.JSON) != 1 {
+ t.Fatalf("result = %+v, handled = %v", result, handled)
+ }
+ text, _ := result.JSON[0]["text"].(string)
+ if strings.Join(strings.Fields(strings.ToUpper(text)), "") != "HELLOWORLD" {
+ t.Fatalf("native OCR text = %q, want HELLO WORLD: warnings = %v", text, result.Warnings)
+ }
+ if result.JSON[0]["image"] == "" || result.JSON[0]["doc_type_kwd"] != "image" {
+ t.Fatalf("missing image attachment: %+v", result.JSON[0])
+ }
+}
diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.go
index c94ac8e784..c701a80772 100644
--- a/internal/ingestion/component/media_dispatch_test.go
+++ b/internal/ingestion/component/media_dispatch_test.go
@@ -19,29 +19,106 @@ import (
"bytes"
"context"
"encoding/base64"
+ "encoding/json"
+ "errors"
"image"
+ "image/png"
+ "io"
+ "os"
"strings"
"sync"
"testing"
+ "time"
"gorm.io/gorm"
"ragflow/internal/common"
"ragflow/internal/dao"
+ deepdocpdf "ragflow/internal/deepdoc/parser/pdf"
+ deepdoctype "ragflow/internal/deepdoc/parser/type"
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
"ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/parser"
"ragflow/internal/utility"
)
-func TestMaybeDispatchImageWithoutVisionReportsNoContent(t *testing.T) {
- result, handled, err := maybeDispatchImage(t.Context(), dao.DB, utility.FileTypeVISUAL,
- "photo.png", []byte("image bytes"), nil, defaultSetups(), false)
- if !handled {
- t.Fatal("image was not handled")
+type pictureOCRAnalyzer struct {
+ deepdoctype.DocAnalyzer
+ empty bool
+}
+
+func (*pictureOCRAnalyzer) Health() bool { return true }
+
+func (a *pictureOCRAnalyzer) OCRDetect(ctx context.Context, img image.Image) ([]deepdoctype.OCRBox, error) {
+ if a.empty {
+ return nil, nil
}
- if err == nil || !strings.Contains(err.Error(), "vision enhancement") {
- t.Fatalf("error = %v, want explicit vision requirement; result = %+v", err, result)
+ return []deepdoctype.OCRBox{{X0: 1, Y0: 1, X1: 9, Y1: 1, X2: 9, Y2: 9, X3: 1, Y3: 9}}, nil
+}
+
+func (*pictureOCRAnalyzer) OCRRecognize(ctx context.Context, img image.Image) ([]deepdoctype.OCRText, error) {
+ return []deepdoctype.OCRText{{Text: "OCR text"}}, nil
+}
+
+func TestMaybeDispatchImageOCRIndependentOfVision(t *testing.T) {
+ templateData, err := os.ReadFile("../pipeline/template/ingestion_pipeline_picture.json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ var template struct {
+ DSL struct {
+ Components map[string]struct {
+ Obj struct{ Params map[string]any }
+ }
+ }
+ }
+ if err := json.Unmarshal(templateData, &template); err != nil {
+ t.Fatal(err)
+ }
+ params := template.DSL.Components["Parser:ViewsCaptureLight"].Obj.Params
+ original := deepdoctype.NativeDocAnalyzerFactory
+ deepdoctype.NativeDocAnalyzerFactory = func() (deepdoctype.DocAnalyzer, bool) { return &pictureOCRAnalyzer{}, true }
+ t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = original })
+ data := picturePNG(t)
+ for _, tc := range []struct {
+ name string
+ method string
+ enabled bool
+ tenant string
+ want string
+ }{
+ {"disabled", "ocr", false, "", "OCR text"},
+ {"enabled", "ocr", true, "t1", "OCR text\ncaptured"},
+ {"unavailable", "ocr", true, "", "OCR text"},
+ {"default", "", false, "", "OCR text"},
+ {"vlm-only", "custom-vlm", false, "", ""},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ originalResolver := resolveTenantModelByType
+ resolveTenantModelByType = func(context.Context, *gorm.DB, string, entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ return &imagePromptCaptureDriver{}, "vision", &modelModule.APIConfig{}, 0, nil
+ }
+ t.Cleanup(func() { resolveTenantModelByType = originalResolver })
+ params["image"].(map[string]any)["parse_method"] = tc.method
+ component, err := NewParserComponent(params)
+ if err != nil {
+ t.Fatal(err)
+ }
+ pc := component.(*ParserComponent)
+ pc.enableVisionEnhancement = tc.enabled
+ result, err := pc.Invoke(t.Context(), nil, map[string]any{"name": "photo.png", "file_type": "image", "binary": data, "tenant_id": tc.tenant})
+ if err != nil {
+ t.Fatal(err)
+ }
+ items, _ := result["json"].([]map[string]any)
+ if len(items) != 1 || items[0]["text"] != tc.want {
+ t.Fatalf("items = %+v, want text %q", items, tc.want)
+ }
+ if items[0]["image"] == "" || items[0]["doc_type_kwd"] != "image" {
+ t.Fatalf("missing image attachment: %+v", items)
+ }
+ })
}
}
@@ -117,7 +194,7 @@ func TestMaybeDispatchImage_UsesSystemPrompt(t *testing.T) {
dao.DB,
utility.FileTypeVISUAL,
"test.png",
- []byte("not-a-real-image"),
+ picturePNG(t),
map[string]any{"tenant_id": "t1"},
setups,
true,
@@ -169,7 +246,7 @@ func TestMaybeDispatchImage_DefaultPromptUsesDatasetLanguage(t *testing.T) {
dao.DB,
utility.FileTypeVISUAL,
"test.png",
- []byte("not-a-real-image"),
+ picturePNG(t),
map[string]any{"tenant_id": "t1", "lang": "Japanese"},
setups,
true,
@@ -212,7 +289,7 @@ func TestMaybeDispatchImage_ReturnsJSONWithImage(t *testing.T) {
dao.DB,
utility.FileTypeVISUAL,
"test.png",
- []byte("not-a-real-image"),
+ picturePNG(t),
map[string]any{"tenant_id": "t1"},
setups,
true,
@@ -264,7 +341,7 @@ func TestMaybeDispatchImage_HardcodesJSONOutput(t *testing.T) {
dao.DB,
utility.FileTypeVISUAL,
"test.png",
- []byte("not-a-real-image"),
+ picturePNG(t),
map[string]any{"tenant_id": "t1"},
setups,
true,
@@ -456,7 +533,7 @@ func TestMaybeDispatchImage_UsesConfiguredVLMModel(t *testing.T) {
dao.DB,
utility.FileTypeVISUAL,
"test.png",
- []byte("not-a-real-image"),
+ picturePNG(t),
map[string]any{"tenant_id": "t1"},
setups,
true,
@@ -533,14 +610,10 @@ func TestMaybeDispatchAudio_UsesConfiguredModel(t *testing.T) {
}
}
-// TestImageDecoders_RegisteredFormats validates that image decoders for WebP, BMP,
-// TIFF, PNG, JPEG, and GIF are registered by vision_image.go and can decode
-// their respective binary payloads via image.Decode without importing the decoder
-// packages directly in the test file.
+// TestImageDecoders_RegisteredFormats checks that the component package can
+// decode WebP, BMP, TIFF, PNG, JPEG, and GIF payloads.
func TestImageDecoders_RegisteredFormats(t *testing.T) {
- // Fixed binary fixtures for image formats decoded via decoders registered in
- // vision_image.go (neither standard library nor x/image decoders are imported
- // in this test file).
+ // Fixed binary fixtures for supported image formats.
const (
webpB64 = "UklGRiQAAABXRUJQVlA4IBgAAAAwAQCdASoBAAEAD8D+JaQAA3AA/ua1AAA="
bmpB64 = "Qk1GAAAAAAAAADYAAAAoAAAAAgAAAAIAAAABACAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD/AP8AAP//AAAAAA=="
@@ -581,3 +654,125 @@ func TestImageDecoders_RegisteredFormats(t *testing.T) {
})
}
}
+
+func TestMaybeDispatchImageWithoutOCRTextKeepsImage(t *testing.T) {
+ original := deepdoctype.NativeDocAnalyzerFactory
+ t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = original })
+ for _, available := range []bool{false, true} {
+ deepdoctype.NativeDocAnalyzerFactory = func() (deepdoctype.DocAnalyzer, bool) { return &pictureOCRAnalyzer{empty: true}, available }
+ for _, method := range []string{"ocr", ""} {
+ setups := defaultSetups()
+ setups["image"]["parse_method"] = method
+ result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t), nil, setups, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !handled || len(result.JSON) != 1 || result.JSON[0]["text"] != "" || result.JSON[0]["image"] == "" || len(result.Warnings) == 0 {
+ t.Fatalf("method %q, available %v: result = %+v, handled = %v", method, available, result, handled)
+ }
+ }
+ }
+}
+
+type unhealthyPictureOCRAnalyzer struct {
+ deepdoctype.DocAnalyzer
+}
+
+func (*unhealthyPictureOCRAnalyzer) Health() bool { return false }
+
+func TestMaybeDispatchImageUnhealthyOCRKeepsImage(t *testing.T) {
+ original := deepdoctype.NativeDocAnalyzerFactory
+ t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = original })
+ // OCR methods are unimplemented: calling them would panic.
+ deepdoctype.NativeDocAnalyzerFactory = func() (deepdoctype.DocAnalyzer, bool) {
+ return &unhealthyPictureOCRAnalyzer{}, true
+ }
+ result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t), nil, defaultSetups(), false)
+ if err != nil || result.Err != nil {
+ t.Fatalf("unhealthy OCR should degrade without an error: err = %v, result.Err = %v", err, result.Err)
+ }
+ if !handled || len(result.JSON) != 1 || result.JSON[0]["text"] != "" || result.JSON[0]["image"] == "" {
+ t.Fatalf("unhealthy OCR should retain an image without text: result = %+v, handled = %v", result, handled)
+ }
+ if !strings.Contains(strings.Join(result.Warnings, "\n"), "image OCR unavailable: local OCR analyzer is unavailable") {
+ t.Fatalf("missing unhealthy OCR warning: %v", result.Warnings)
+ }
+}
+
+func TestMaybeDispatchImageRejectsUndecodableBytes(t *testing.T) {
+ for _, method := range []string{"ocr", "custom-vlm"} {
+ setups := defaultSetups()
+ setups["image"]["parse_method"] = method
+ _, _, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", []byte("bad image"), nil, setups, false)
+ if err == nil || !strings.Contains(err.Error(), "decode") {
+ t.Fatalf("method %q: error = %v, want decode error", method, err)
+ }
+ }
+}
+
+func TestMaybeDispatchImageReleasesAdmissionAfterPanic(t *testing.T) {
+ original := deepdoctype.NativeDocAnalyzerFactory
+ t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = original })
+ deepdoctype.NativeDocAnalyzerFactory = func() (deepdoctype.DocAnalyzer, bool) {
+ panic("OCR analyzer panic")
+ }
+ data := picturePNG(t)
+ func() {
+ defer func() {
+ if recover() == nil {
+ t.Error("expected OCR analyzer panic")
+ }
+ }()
+ _, _, _ = maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", data, nil, defaultSetups(), false)
+ }()
+ ctx, cancel := context.WithTimeout(t.Context(), time.Second)
+ defer cancel()
+ for range deepdocpdf.DeepDocConcurrency() {
+ release, err := parser.AcquireImageMedia(ctx)
+ if err != nil {
+ t.Fatalf("image admission leaked after panic: %v", err)
+ }
+ defer release()
+ }
+}
+
+func TestMaybeDispatchImageDecodesRasterOnlyForOCR(t *testing.T) {
+ const magic = "picture-dispatch-raster-test"
+ image.RegisterFormat(magic, magic,
+ func(io.Reader) (image.Image, error) { return nil, errors.New("raster decoding requested") },
+ func(io.Reader) (image.Config, error) { return image.Config{Width: 12, Height: 12}, nil })
+ original := resolveModelConfig
+ t.Cleanup(func() { resolveModelConfig = original })
+ resolveModelConfig = func(context.Context, *gorm.DB, string, entity.ModelType, string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ return &imagePromptCaptureDriver{}, "vision", &modelModule.APIConfig{}, 0, nil
+ }
+ for _, method := range []string{"custom-vlm", "ocr"} {
+ t.Run(method, func(t *testing.T) {
+ setups := defaultSetups()
+ setups["image"]["parse_method"] = method
+ result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL,
+ "photo.png", []byte(magic), map[string]any{"tenant_id": "t1"}, setups, true)
+ if method == "ocr" {
+ if err == nil || !strings.Contains(err.Error(), "raster decoding requested") {
+ t.Fatalf("error = %v, want raster decode error", err)
+ }
+ return
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !handled || len(result.JSON) != 1 || result.JSON[0]["text"] != "captured" || result.JSON[0]["image"] == "" {
+ t.Fatalf("VLM result = %+v, handled = %v", result, handled)
+ }
+ })
+ }
+}
+
+func picturePNG(t *testing.T) []byte {
+ t.Helper()
+ var data bytes.Buffer
+ if err := png.Encode(&data, image.NewRGBA(image.Rect(0, 0, 12, 12))); err != nil {
+ t.Fatal(err)
+ }
+ return data.Bytes()
+}
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index 72dd2cdd34..303a3645b0 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -240,13 +240,12 @@ func (c *ParserComponent) Check() error {
}
}
}
- // image family (parser.py:283-287). The legacy "ocr" setup value selects
- // the tenant default vision model; image OCR is not run by this path.
+ // Image OCR runs independently of optional vision enhancement.
if img, ok := c.setups["image"]; ok {
pm, _ := img["parse_method"].(string)
// A model selected for optional image enhancement needs a language
// only when enhancement is enabled.
- if c.enableVisionEnhancement && pm != "ocr" {
+ if c.enableVisionEnhancement && !strings.EqualFold(pm, "ocr") && pm != "" {
if lang, _ := img["lang"].(string); lang == "" {
return errors.New("image VLM language does not support empty value")
}
@@ -315,7 +314,7 @@ func defaultSetups() map[string]schema.ParserSetup {
"llm_id": "",
"lang": "Chinese",
"system_prompt": "",
- "suffix": []string{"jpg", "jpeg", "png", "gif"},
+ "suffix": []string{"jpg", "jpeg", "png", "gif", "bmp", "tif", "tiff", "webp"},
"output_format": "json",
},
"email": {
@@ -481,8 +480,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
}
var handledImage bool
if !handledVision && !handledMedia {
- // Image/Picture dispatch: optional IMAGE2TEXT vision description.
- // Mirrors Python's rag/app/picture.py:chunk() image branch.
+ // Image dispatch: OCR with independently controlled VLM enhancement.
dispatched, handledImage, visionErr = maybeDispatchImage(ctx, db, fileTypeExt, filename, binary, inputs, setups, c.enableVisionEnhancement)
if visionErr != nil {
return nil, visionErr
diff --git a/internal/ingestion/component/testdata/picture_ocr.png b/internal/ingestion/component/testdata/picture_ocr.png
new file mode 100644
index 0000000000..a1a76dfd5c
Binary files /dev/null and b/internal/ingestion/component/testdata/picture_ocr.png differ