Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): fetch a PDF source before image admission in VLM enhancement (#20846)

jay77721c80d15b37c01
diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.go
index d635a115d9..f97cb7b95b 100644
--- a/internal/ingestion/component/vision_enhancement.go
+++ b/internal/ingestion/component/vision_enhancement.go
@@ -60,8 +60,8 @@ const (
visionMediaItemBudget = 10 * time.Second
)
-// isUsableVisionImage reports whether raw carries a valid image data URI or a
-// base64 payload that can be sent to a vision model.
+// isUsableVisionImage checks the encoded size and rejects raster dimensions
+// beyond the limits shared by Parser and VLM materialization.
func isUsableVisionImage(raw string) bool {
raw = strings.TrimSpace(raw)
if raw == "" {
@@ -107,10 +107,11 @@ func isValidBase64(s string, stripWhitespace bool) bool {
return false
}
-// visionImageCropper materializes the VLM payload for a parsed item. Under
-// cgo it crops PDF sections on demand; inline images stay encoded. Close
-// releases any re-acquired native engine.
+// visionImageCropper prepares any source document before image admission,
+// materializes one item's VLM payload under image admission, and releases
+// source resources when the batch finishes.
type visionImageCropper interface {
+ Prepare(ctx context.Context, item map[string]any) error
Crop(ctx context.Context, item map[string]any) (*visionImage, error)
Close() error
}
@@ -204,14 +205,18 @@ func maybeDispatchVisionEnhancement(
return dispatched, false, nil
}
- // Materialize one resource at a time. The VLM semaphore is acquired before
- // materialization, so at most visionEnhancementConcurrency encoded payloads
- // remain live while model requests run.
+ // Prepare PDF sources outside image admission so slow storage reads do not
+ // block unrelated image materialization. The separate VLM semaphore bounds
+ // how many encoded payloads remain live while model requests run.
cropper, cerr := visionImageCropperFactory(ctx, db, inputs)
if cerr != nil {
dispatched.Warnings = append(dispatched.Warnings, fmt.Sprintf("vision enhancement skipped: %v", cerr))
return dispatched, false, nil
}
+ // Close is invoked explicitly after materialization so the engine and the
+ // source bytes are released while model requests are still in flight. Keep
+ // a deferred Close as a panic/early-return safety net: Close is idempotent
+ // (engine is nilled, sourceErr cleared), so the second call is a no-op.
defer cropper.Close()
modified := false
descriptions := make([]string, len(items))
@@ -240,13 +245,26 @@ func maybeDispatchVisionEnhancement(
}
var resource *visionImage
func() {
+ item := dispatched.JSON[itemIdx]
+ if err := cropper.Prepare(ctx, item); err != nil {
+ if ctx.Err() == nil {
+ if payload, _ := item["image"].(string); isUsableVisionImage(payload) {
+ resource = &visionImage{VLMData: payload, VLMDataValidated: true}
+ }
+ }
+ if resource == nil && ctx.Err() == nil {
+ failures[slot] = fmt.Sprintf("image source preparation failed: %v", err)
+ }
+ return
+ }
+
itemCtx, cancelItem := context.WithTimeout(ctx, visionMediaItemBudget)
defer cancelItem()
release, err := parser.AcquireImageMedia(itemCtx)
if err != nil {
if ctx.Err() == nil && vlmReady {
- if payload, _ := dispatched.JSON[itemIdx]["image"].(string); payload != "" {
- resource = &visionImage{VLMData: payload}
+ if payload, _ := item["image"].(string); isUsableVisionImage(payload) {
+ resource = &visionImage{VLMData: payload, VLMDataValidated: true}
}
}
if resource == nil && ctx.Err() == nil {
@@ -255,7 +273,6 @@ func maybeDispatchVisionEnhancement(
return
}
defer release()
- item := dispatched.JSON[itemIdx]
resource, err = cropper.Crop(itemCtx, item)
if err != nil || resource == nil {
cropErr := err
@@ -329,6 +346,10 @@ func maybeDispatchVisionEnhancement(
}
}(slot, resource.VLMData)
}
+ if err := cropper.Close(); err != nil {
+ dispatched.Warnings = append(dispatched.Warnings, fmt.Sprintf("vision enhancement cleanup failed: %v", err))
+ common.Warn("vision enhancement: failed to close image cropper", zap.Error(err))
+ }
wg.Wait()
if err := ctx.Err(); err != nil {
return dispatched, modified, err
diff --git a/internal/ingestion/component/vision_enhancement_cgo.go b/internal/ingestion/component/vision_enhancement_cgo.go
index 95354da940..0f5dbe2a1e 100644
--- a/internal/ingestion/component/vision_enhancement_cgo.go
+++ b/internal/ingestion/component/vision_enhancement_cgo.go
@@ -80,8 +80,8 @@ type visionPDFCropper struct {
}
// newVisionImageCropper builds the on-demand cropper. It never touches storage
-// here; the source PDF is re-acquired lazily on the first Crop call that needs
-// it.
+// here; the source PDF is re-acquired lazily by Prepare or Crop when an item
+// has PDF positions but no inline image.
func newVisionImageCropper(ctx context.Context, db *gorm.DB, inputs map[string]any) (visionImageCropper, error) {
if ctx == nil {
ctx = context.Background()
@@ -96,6 +96,26 @@ func newVisionImageCropper(ctx context.Context, db *gorm.DB, inputs map[string]a
}, nil
}
+// pdfSourcePositions returns the cropped positions when item requires the
+// source PDF: no inlined image but usable PDF positions. Prepare and Crop
+// share this predicate so a future condition change cannot make Prepare skip a
+// fetch that Crop then performs under image admission, silently reintroducing
+// the admission block.
+func pdfSourcePositions(item map[string]any) ([]deepdoctype.Position, bool) {
+ if img, _ := item["image"].(string); img != "" {
+ return nil, false
+ }
+ matrix, ok := parser.ExtractPDFPositions(item)
+ if !ok {
+ return nil, false
+ }
+ positions := util.PositionsFromMatrix(matrix)
+ if len(positions) == 0 {
+ return nil, false
+ }
+ return positions, true
+}
+
func (c *visionPDFCropper) Crop(ctx context.Context, item map[string]any) (*visionImage, error) {
if err := ctx.Err(); err != nil {
return nil, err
@@ -105,14 +125,10 @@ func (c *visionPDFCropper) Crop(ctx context.Context, item map[string]any) (*visi
if img, _ := item["image"].(string); img != "" {
return materializeInlineVisionImage(img)
}
- matrix, ok := parser.ExtractPDFPositions(item)
+ positions, ok := pdfSourcePositions(item)
if !ok {
return nil, nil
}
- positions := util.PositionsFromMatrix(matrix)
- if len(positions) == 0 {
- return nil, nil
- }
if err := c.ensureEngine(); err != nil {
// Best-effort: a missing/unreadable source PDF means no vision
// description for this item, not a hard failure.
@@ -166,6 +182,19 @@ func (c *visionPDFCropper) Crop(ctx context.Context, item map[string]any) (*visi
return &visionImage{Raster: raster}, nil
}
+func (c *visionPDFCropper) Prepare(ctx context.Context, item map[string]any) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ if _, ok := pdfSourcePositions(item); !ok {
+ return nil
+ }
+ if err := c.ensureEngine(); err != nil {
+ return err
+ }
+ return ctx.Err()
+}
+
func pdfPagesRasterWithinVisionLimits(engine deepdoctype.PDFEngine, pageNums map[int]struct{}) bool {
sizer, ok := engine.(visionPageSizer)
if !ok {
@@ -263,12 +292,14 @@ func (c *visionPDFCropper) Close() error {
c.mu.Lock()
defer c.mu.Unlock()
c.cancelSource()
+ var closeErr error
+ if c.engine != nil {
+ closeErr = c.engine.Close()
+ c.engine = nil
+ }
if c.sourceErr != nil && c.invocationCtx.Err() == nil {
common.Warn("vision enhancement: source PDF unavailable for VLM crop", zap.Error(c.sourceErr))
c.sourceErr = nil
}
- if c.engine != nil {
- return c.engine.Close()
- }
- return nil
+ return closeErr
}
diff --git a/internal/ingestion/component/vision_enhancement_nocgo.go b/internal/ingestion/component/vision_enhancement_nocgo.go
index 07d51a5641..ad92f9d0c2 100644
--- a/internal/ingestion/component/vision_enhancement_nocgo.go
+++ b/internal/ingestion/component/vision_enhancement_nocgo.go
@@ -35,6 +35,10 @@ func newVisionImageCropper(_ context.Context, _ *gorm.DB, _ map[string]any) (vis
return visionPDFCropperNoop{}, nil
}
+func (visionPDFCropperNoop) Prepare(ctx context.Context, _ map[string]any) error {
+ return ctx.Err()
+}
+
func (visionPDFCropperNoop) Crop(_ context.Context, item map[string]any) (*visionImage, error) {
if img, _ := item["image"].(string); img != "" {
return materializeInlineVisionImage(img)
diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.go
index aa6e7b88c2..9b0e04490e 100644
--- a/internal/ingestion/component/vision_enhancement_test.go
+++ b/internal/ingestion/component/vision_enhancement_test.go
@@ -48,12 +48,37 @@ type visionEnhanceFakeDriver struct {
type failingVisionImageCropper struct{ err error }
+func (failingVisionImageCropper) Prepare(context.Context, map[string]any) error { return nil }
+
func (c failingVisionImageCropper) Crop(context.Context, map[string]any) (*visionImage, error) {
return nil, c.err
}
func (failingVisionImageCropper) Close() error { return nil }
+type admissionCheckingVisionCropper struct {
+ prepared bool
+ payload string
+}
+
+func (c *admissionCheckingVisionCropper) Prepare(ctx context.Context, _ map[string]any) error {
+ acquireCtx, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
+ defer cancel()
+ release, err := parser.AcquireImageMedia(acquireCtx)
+ if err != nil {
+ return fmt.Errorf("source preparation ran while image materialization was admitted: %w", err)
+ }
+ release()
+ c.prepared = true
+ return nil
+}
+
+func (c *admissionCheckingVisionCropper) Crop(context.Context, map[string]any) (*visionImage, error) {
+ return &visionImage{VLMData: c.payload, VLMDataValidated: true}, nil
+}
+
+func (*admissionCheckingVisionCropper) Close() error { return nil }
+
type visionEnhanceCaptureInvoker struct {
mu sync.Mutex
images []string
@@ -218,6 +243,35 @@ func TestVisionEnhancement_AppendsVLMToExistingText(t *testing.T) {
}
}
+func TestVisionEnhancementPreparesPDFSourceBeforeImageAdmission(t *testing.T) {
+ invoker := &visionEnhanceCaptureInvoker{}
+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
+
+ cropper := &admissionCheckingVisionCropper{payload: visionTestPNGBase64(t)}
+ originalFactory := visionImageCropperFactory
+ visionImageCropperFactory = func(context.Context, *gorm.DB, map[string]any) (visionImageCropper, error) {
+ return cropper, nil
+ }
+ t.Cleanup(func() { visionImageCropperFactory = originalFactory })
+
+ dispatched := parser.ParseResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{{
+ "image": cropper.payload,
+ "doc_type_kwd": "image",
+ }},
+ }
+ if _, _, err := maybeDispatchVisionEnhancement(
+ t.Context(), dao.DB, utility.FileTypePDF, dispatched,
+ map[string]any{"tenant_id": "t1"}, visionSettings{}, fractionWindow{},
+ ); err != nil {
+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
+ }
+ if !cropper.prepared {
+ t.Fatal("PDF source preparation was not called before image admission")
+ }
+}
+
func TestVisionEnhancement_ReportsVLMInvocationFailure(t *testing.T) {
swapVisionGlobals(t, fakeResolver, func(context.Context, modelModule.ModelDriver, string, []modelModule.Message, *modelModule.APIConfig) (*modelModule.ChatResponse, error) {
return nil, errors.New("VLM unavailable")
diff --git a/internal/ingestion/component/vision_source_admission_test.go b/internal/ingestion/component/vision_source_admission_test.go
new file mode 100644
index 0000000000..992a5bec3d
--- /dev/null
+++ b/internal/ingestion/component/vision_source_admission_test.go
@@ -0,0 +1,105 @@
+//go:build cgo
+
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "context"
+ "strings"
+ "testing"
+ "time"
+
+ "ragflow/internal/dao"
+ "ragflow/internal/parser/parser"
+ "ragflow/internal/utility"
+)
+
+// TestVisionSourceFetchDoesNotBlockImageAdmission pins the invariant that
+// acquiring the source document for one document's PDF crops must not be done
+// while holding the process-wide image admission slot.
+//
+// The two budgets are what make the ordering matter rather than merely being
+// untidy: the fetch is bounded by visionPDFSourceBudget (90s) and runs on the
+// invocation context, while a waiter only holds visionMediaItemBudget (10s) for
+// the image slot. Holding the slot across the fetch therefore makes an
+// unrelated document's image materialization fail on a timeout that no amount
+// of capacity can fix — the waiter is outlasted, not outnumbered.
+//
+// Two concurrent calls model two documents parsed in the same process, which
+// requires ingestor.max_concurrent_workers >= 2.
+func TestVisionSourceFetchDoesNotBlockImageAdmission(t *testing.T) {
+ if testing.Short() {
+ t.Skip("needs the stub fetch to outlast visionMediaItemBudget")
+ }
+
+ invoker := &visionEnhanceCaptureInvoker{}
+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
+
+ originalFetcher := visionSourceFetcher
+ fetchStarted := make(chan struct{}, 1)
+ visionSourceFetcher = func(ctx context.Context, _, _ string) ([]byte, error) {
+ select {
+ case fetchStarted <- struct{}{}:
+ default:
+ }
+ select {
+ case <-time.After(visionMediaItemBudget + 2*time.Second):
+ case <-ctx.Done():
+ }
+ // Non-PDF bytes: ensureEngine stops before opening an engine, so the
+ // test exercises source acquisition without the native backend.
+ return []byte("not a pdf"), nil
+ }
+ t.Cleanup(func() { visionSourceFetcher = originalFetcher })
+
+ newItem := func() map[string]any {
+ return map[string]any{
+ "doc_type_kwd": "image",
+ "text": "existing caption",
+ "positions": [][]any{{[]any{1}, 50.0, 300.0, 200.0, 400.0}},
+ }
+ }
+ run := func(inputs map[string]any) []string {
+ dispatched := parser.ParseResult{OutputFormat: "json", JSON: []map[string]any{newItem()}}
+ res, _, err := maybeDispatchVisionEnhancement(
+ context.Background(), dao.DB, utility.FileTypePDF, dispatched,
+ inputs, visionSettings{}, fractionWindow{},
+ )
+ if err != nil {
+ t.Errorf("maybeDispatchVisionEnhancement: %v", err)
+ }
+ return res.Warnings
+ }
+
+ // A's fetch is in flight — and must not be occupying the image slot.
+ sourceHolder := make(chan []string, 1)
+ go func() { sourceHolder <- run(map[string]any{"tenant_id": "t1", "bucket": "b", "path": "A.pdf"}) }()
+ select {
+ case <-fetchStarted:
+ case <-time.After(visionMediaItemBudget):
+ t.Fatal("source fetch never started")
+ }
+
+ // B materializes images while A's fetch is still running.
+ for _, warning := range run(map[string]any{"tenant_id": "t1", "bucket": "b", "path": "B.pdf"}) {
+ if strings.Contains(warning, "media admission failed") {
+ t.Fatalf("image admission was blocked by another document's source fetch: %s", warning)
+ }
+ }
+ <-sourceHolder
+}