RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): fetch a PDF source before image admission in VLM enhancement (#20846)
jay77721c80d15b37c01
diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.goindex d635a115d9..f97cb7b95b 100644--- a/internal/ingestion/component/vision_enhancement.go+++ b/internal/ingestion/component/vision_enhancement.go@@ -60,8 +60,8 @@ const (visionMediaItemBudget = 10 * time.Second)-// isUsableVisionImage reports whether raw carries a valid image data URI or a-// base64 payload that can be sent to a vision model.+// isUsableVisionImage checks the encoded size and rejects raster dimensions+// beyond the limits shared by Parser and VLM materialization.func isUsableVisionImage(raw string) bool {raw = strings.TrimSpace(raw)if raw == "" {@@ -107,10 +107,11 @@ func isValidBase64(s string, stripWhitespace bool) bool {return false}-// visionImageCropper materializes the VLM payload for a parsed item. Under-// cgo it crops PDF sections on demand; inline images stay encoded. Close-// releases any re-acquired native engine.+// visionImageCropper prepares any source document before image admission,+// materializes one item's VLM payload under image admission, and releases+// source resources when the batch finishes.type visionImageCropper interface {+ Prepare(ctx context.Context, item map[string]any) errorCrop(ctx context.Context, item map[string]any) (*visionImage, error)Close() error}@@ -204,14 +205,18 @@ func maybeDispatchVisionEnhancement(return dispatched, false, nil}- // Materialize one resource at a time. The VLM semaphore is acquired before- // materialization, so at most visionEnhancementConcurrency encoded payloads- // remain live while model requests run.+ // Prepare PDF sources outside image admission so slow storage reads do not+ // block unrelated image materialization. The separate VLM semaphore bounds+ // how many encoded payloads remain live while model requests run.cropper, cerr := visionImageCropperFactory(ctx, db, inputs)if cerr != nil {dispatched.Warnings = append(dispatched.Warnings, fmt.Sprintf("vision enhancement skipped: %v", cerr))return dispatched, false, nil}+ // Close is invoked explicitly after materialization so the engine and the+ // source bytes are released while model requests are still in flight. Keep+ // a deferred Close as a panic/early-return safety net: Close is idempotent+ // (engine is nilled, sourceErr cleared), so the second call is a no-op.defer cropper.Close()modified := falsedescriptions := make([]string, len(items))@@ -240,13 +245,26 @@ func maybeDispatchVisionEnhancement(}var resource *visionImagefunc() {+ item := dispatched.JSON[itemIdx]+ if err := cropper.Prepare(ctx, item); err != nil {+ if ctx.Err() == nil {+ if payload, _ := item["image"].(string); isUsableVisionImage(payload) {+ resource = &visionImage{VLMData: payload, VLMDataValidated: true}+ }+ }+ if resource == nil && ctx.Err() == nil {+ failures[slot] = fmt.Sprintf("image source preparation failed: %v", err)+ }+ return+ }+itemCtx, cancelItem := context.WithTimeout(ctx, visionMediaItemBudget)defer cancelItem()release, err := parser.AcquireImageMedia(itemCtx)if err != nil {if ctx.Err() == nil && vlmReady {- if payload, _ := dispatched.JSON[itemIdx]["image"].(string); payload != "" {- resource = &visionImage{VLMData: payload}+ if payload, _ := item["image"].(string); isUsableVisionImage(payload) {+ resource = &visionImage{VLMData: payload, VLMDataValidated: true}}}if resource == nil && ctx.Err() == nil {@@ -255,7 +273,6 @@ func maybeDispatchVisionEnhancement(return}defer release()- item := dispatched.JSON[itemIdx]resource, err = cropper.Crop(itemCtx, item)if err != nil || resource == nil {cropErr := err@@ -329,6 +346,10 @@ func maybeDispatchVisionEnhancement(}}(slot, resource.VLMData)}+ if err := cropper.Close(); err != nil {+ dispatched.Warnings = append(dispatched.Warnings, fmt.Sprintf("vision enhancement cleanup failed: %v", err))+ common.Warn("vision enhancement: failed to close image cropper", zap.Error(err))+ }wg.Wait()if err := ctx.Err(); err != nil {return dispatched, modified, errdiff --git a/internal/ingestion/component/vision_enhancement_cgo.go b/internal/ingestion/component/vision_enhancement_cgo.goindex 95354da940..0f5dbe2a1e 100644--- a/internal/ingestion/component/vision_enhancement_cgo.go+++ b/internal/ingestion/component/vision_enhancement_cgo.go@@ -80,8 +80,8 @@ type visionPDFCropper struct {}// newVisionImageCropper builds the on-demand cropper. It never touches storage-// here; the source PDF is re-acquired lazily on the first Crop call that needs-// it.+// here; the source PDF is re-acquired lazily by Prepare or Crop when an item+// has PDF positions but no inline image.func newVisionImageCropper(ctx context.Context, db *gorm.DB, inputs map[string]any) (visionImageCropper, error) {if ctx == nil {ctx = context.Background()@@ -96,6 +96,26 @@ func newVisionImageCropper(ctx context.Context, db *gorm.DB, inputs map[string]a}, nil}+// pdfSourcePositions returns the cropped positions when item requires the+// source PDF: no inlined image but usable PDF positions. Prepare and Crop+// share this predicate so a future condition change cannot make Prepare skip a+// fetch that Crop then performs under image admission, silently reintroducing+// the admission block.+func pdfSourcePositions(item map[string]any) ([]deepdoctype.Position, bool) {+ if img, _ := item["image"].(string); img != "" {+ return nil, false+ }+ matrix, ok := parser.ExtractPDFPositions(item)+ if !ok {+ return nil, false+ }+ positions := util.PositionsFromMatrix(matrix)+ if len(positions) == 0 {+ return nil, false+ }+ return positions, true+}+func (c *visionPDFCropper) Crop(ctx context.Context, item map[string]any) (*visionImage, error) {if err := ctx.Err(); err != nil {return nil, err@@ -105,14 +125,10 @@ func (c *visionPDFCropper) Crop(ctx context.Context, item map[string]any) (*visiif img, _ := item["image"].(string); img != "" {return materializeInlineVisionImage(img)}- matrix, ok := parser.ExtractPDFPositions(item)+ positions, ok := pdfSourcePositions(item)if !ok {return nil, nil}- positions := util.PositionsFromMatrix(matrix)- if len(positions) == 0 {- return nil, nil- }if err := c.ensureEngine(); err != nil {// Best-effort: a missing/unreadable source PDF means no vision// description for this item, not a hard failure.@@ -166,6 +182,19 @@ func (c *visionPDFCropper) Crop(ctx context.Context, item map[string]any) (*visireturn &visionImage{Raster: raster}, nil}+func (c *visionPDFCropper) Prepare(ctx context.Context, item map[string]any) error {+ if err := ctx.Err(); err != nil {+ return err+ }+ if _, ok := pdfSourcePositions(item); !ok {+ return nil+ }+ if err := c.ensureEngine(); err != nil {+ return err+ }+ return ctx.Err()+}+func pdfPagesRasterWithinVisionLimits(engine deepdoctype.PDFEngine, pageNums map[int]struct{}) bool {sizer, ok := engine.(visionPageSizer)if !ok {@@ -263,12 +292,14 @@ func (c *visionPDFCropper) Close() error {c.mu.Lock()defer c.mu.Unlock()c.cancelSource()+ var closeErr error+ if c.engine != nil {+ closeErr = c.engine.Close()+ c.engine = nil+ }if c.sourceErr != nil && c.invocationCtx.Err() == nil {common.Warn("vision enhancement: source PDF unavailable for VLM crop", zap.Error(c.sourceErr))c.sourceErr = nil}- if c.engine != nil {- return c.engine.Close()- }- return nil+ return closeErr}diff --git a/internal/ingestion/component/vision_enhancement_nocgo.go b/internal/ingestion/component/vision_enhancement_nocgo.goindex 07d51a5641..ad92f9d0c2 100644--- a/internal/ingestion/component/vision_enhancement_nocgo.go+++ b/internal/ingestion/component/vision_enhancement_nocgo.go@@ -35,6 +35,10 @@ func newVisionImageCropper(_ context.Context, _ *gorm.DB, _ map[string]any) (visreturn visionPDFCropperNoop{}, nil}+func (visionPDFCropperNoop) Prepare(ctx context.Context, _ map[string]any) error {+ return ctx.Err()+}+func (visionPDFCropperNoop) Crop(_ context.Context, item map[string]any) (*visionImage, error) {if img, _ := item["image"].(string); img != "" {return materializeInlineVisionImage(img)diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.goindex aa6e7b88c2..9b0e04490e 100644--- a/internal/ingestion/component/vision_enhancement_test.go+++ b/internal/ingestion/component/vision_enhancement_test.go@@ -48,12 +48,37 @@ type visionEnhanceFakeDriver struct {type failingVisionImageCropper struct{ err error }+func (failingVisionImageCropper) Prepare(context.Context, map[string]any) error { return nil }+func (c failingVisionImageCropper) Crop(context.Context, map[string]any) (*visionImage, error) {return nil, c.err}func (failingVisionImageCropper) Close() error { return nil }+type admissionCheckingVisionCropper struct {+ prepared bool+ payload string+}++func (c *admissionCheckingVisionCropper) Prepare(ctx context.Context, _ map[string]any) error {+ acquireCtx, cancel := context.WithTimeout(ctx, 100*time.Millisecond)+ defer cancel()+ release, err := parser.AcquireImageMedia(acquireCtx)+ if err != nil {+ return fmt.Errorf("source preparation ran while image materialization was admitted: %w", err)+ }+ release()+ c.prepared = true+ return nil+}++func (c *admissionCheckingVisionCropper) Crop(context.Context, map[string]any) (*visionImage, error) {+ return &visionImage{VLMData: c.payload, VLMDataValidated: true}, nil+}++func (*admissionCheckingVisionCropper) Close() error { return nil }+type visionEnhanceCaptureInvoker struct {mu sync.Muteximages []string@@ -218,6 +243,35 @@ func TestVisionEnhancement_AppendsVLMToExistingText(t *testing.T) {}}+func TestVisionEnhancementPreparesPDFSourceBeforeImageAdmission(t *testing.T) {+ invoker := &visionEnhanceCaptureInvoker{}+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)++ cropper := &admissionCheckingVisionCropper{payload: visionTestPNGBase64(t)}+ originalFactory := visionImageCropperFactory+ visionImageCropperFactory = func(context.Context, *gorm.DB, map[string]any) (visionImageCropper, error) {+ return cropper, nil+ }+ t.Cleanup(func() { visionImageCropperFactory = originalFactory })++ dispatched := parser.ParseResult{+ OutputFormat: "json",+ JSON: []map[string]any{{+ "image": cropper.payload,+ "doc_type_kwd": "image",+ }},+ }+ if _, _, err := maybeDispatchVisionEnhancement(+ t.Context(), dao.DB, utility.FileTypePDF, dispatched,+ map[string]any{"tenant_id": "t1"}, visionSettings{}, fractionWindow{},+ ); err != nil {+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)+ }+ if !cropper.prepared {+ t.Fatal("PDF source preparation was not called before image admission")+ }+}+func TestVisionEnhancement_ReportsVLMInvocationFailure(t *testing.T) {swapVisionGlobals(t, fakeResolver, func(context.Context, modelModule.ModelDriver, string, []modelModule.Message, *modelModule.APIConfig) (*modelModule.ChatResponse, error) {return nil, errors.New("VLM unavailable")diff --git a/internal/ingestion/component/vision_source_admission_test.go b/internal/ingestion/component/vision_source_admission_test.gonew file mode 100644index 0000000000..992a5bec3d--- /dev/null+++ b/internal/ingestion/component/vision_source_admission_test.go@@ -0,0 +1,105 @@+//go:build cgo++//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "context"+ "strings"+ "testing"+ "time"++ "ragflow/internal/dao"+ "ragflow/internal/parser/parser"+ "ragflow/internal/utility"+)++// TestVisionSourceFetchDoesNotBlockImageAdmission pins the invariant that+// acquiring the source document for one document's PDF crops must not be done+// while holding the process-wide image admission slot.+//+// The two budgets are what make the ordering matter rather than merely being+// untidy: the fetch is bounded by visionPDFSourceBudget (90s) and runs on the+// invocation context, while a waiter only holds visionMediaItemBudget (10s) for+// the image slot. Holding the slot across the fetch therefore makes an+// unrelated document's image materialization fail on a timeout that no amount+// of capacity can fix — the waiter is outlasted, not outnumbered.+//+// Two concurrent calls model two documents parsed in the same process, which+// requires ingestor.max_concurrent_workers >= 2.+func TestVisionSourceFetchDoesNotBlockImageAdmission(t *testing.T) {+ if testing.Short() {+ t.Skip("needs the stub fetch to outlast visionMediaItemBudget")+ }++ invoker := &visionEnhanceCaptureInvoker{}+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)++ originalFetcher := visionSourceFetcher+ fetchStarted := make(chan struct{}, 1)+ visionSourceFetcher = func(ctx context.Context, _, _ string) ([]byte, error) {+ select {+ case fetchStarted <- struct{}{}:+ default:+ }+ select {+ case <-time.After(visionMediaItemBudget + 2*time.Second):+ case <-ctx.Done():+ }+ // Non-PDF bytes: ensureEngine stops before opening an engine, so the+ // test exercises source acquisition without the native backend.+ return []byte("not a pdf"), nil+ }+ t.Cleanup(func() { visionSourceFetcher = originalFetcher })++ newItem := func() map[string]any {+ return map[string]any{+ "doc_type_kwd": "image",+ "text": "existing caption",+ "positions": [][]any{{[]any{1}, 50.0, 300.0, 200.0, 400.0}},+ }+ }+ run := func(inputs map[string]any) []string {+ dispatched := parser.ParseResult{OutputFormat: "json", JSON: []map[string]any{newItem()}}+ res, _, err := maybeDispatchVisionEnhancement(+ context.Background(), dao.DB, utility.FileTypePDF, dispatched,+ inputs, visionSettings{}, fractionWindow{},+ )+ if err != nil {+ t.Errorf("maybeDispatchVisionEnhancement: %v", err)+ }+ return res.Warnings+ }++ // A's fetch is in flight — and must not be occupying the image slot.+ sourceHolder := make(chan []string, 1)+ go func() { sourceHolder <- run(map[string]any{"tenant_id": "t1", "bucket": "b", "path": "A.pdf"}) }()+ select {+ case <-fetchStarted:+ case <-time.After(visionMediaItemBudget):+ t.Fatal("source fetch never started")+ }++ // B materializes images while A's fetch is still running.+ for _, warning := range run(map[string]any{"tenant_id": "t1", "bucket": "b", "path": "B.pdf"}) {+ if strings.Contains(warning, "media admission failed") {+ t.Fatalf("image admission was blocked by another document's source fetch: %s", warning)+ }+ }+ <-sourceHolder+}