Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): stop inlining PDF base64 media; crop on demand at VLM and chunker (#19964)

xugangqiang3a902e37dfba
diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo.go b/internal/ingestion/component/chunker/pdfcrop_cgo.go
index 04647bb042..b58f2a7112 100644
--- a/internal/ingestion/component/chunker/pdfcrop_cgo.go
+++ b/internal/ingestion/component/chunker/pdfcrop_cgo.go
@@ -148,8 +148,22 @@ func cropImageChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks [
// cropped; text chunks with positions get a rendered preview of the text
// region (Python restore_pdf_text_previews). A pre-existing Image is never
// re-cropped — cropImageChunks honors that separately.
+//
+// CKType is the chunker-layer refinement (heading/table_header/table_row/text/
+// image/table). General and token chunkers always set it before cropping, so
+// they are handled by the CKType branch above. Group and hierarchy chunkers
+// forward the parser's output verbatim, which carries only the coarser
+// doc_type_kwd (no ck_type). For those, fall back to DocType so image/table/
+// text regions are still cropped on demand — otherwise every figure/table/
+// text preview in group/hierarchy would be silently skipped. When CKType is
+// set (general/token) it takes priority, so headings (CKType "heading") stay
+// excluded from preview cropping.
func needsCrop(ck schema.ChunkDoc) bool {
- switch ck.CKType {
+ typ := ck.CKType
+ if typ == "" {
+ typ = ck.DocType
+ }
+ switch typ {
case "image", "table", "text":
return len(ck.PDFPositions) > 0 || len(ck.Positions) > 0
default:
diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go b/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
index 613d0ac11c..967db9a4b4 100644
--- a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
+++ b/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
@@ -69,11 +69,29 @@ func TestNeedsCrop(t *testing.T) {
ck schema.ChunkDoc
want bool
}{
+ // --- CKType set (general / token paths): unchanged behavior ---
{"image with positions", schema.ChunkDoc{CKType: "image", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
{"table with positions", schema.ChunkDoc{CKType: "table", Positions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
{"text with positions", schema.ChunkDoc{CKType: "text", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
{"image without positions", schema.ChunkDoc{CKType: "image"}, false},
- {"unknown type", schema.ChunkDoc{CKType: "equation", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, false},
+ {"heading excluded via CKType", schema.ChunkDoc{CKType: "heading", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, false},
+ {"unknown type via CKType", schema.ChunkDoc{CKType: "equation", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, false},
+
+ // --- CKType empty: fall back to DocType (group / hierarchy paths,
+ // whose parser output carries no ck_type). This is the fix for the
+ // bug where figure/image/table in group/hierarchy were never cropped.
+ {"doc image fallback", schema.ChunkDoc{DocType: "image", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
+ {"doc table fallback", schema.ChunkDoc{DocType: "table", Positions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
+ {"doc text fallback", schema.ChunkDoc{DocType: "text", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
+ // Heading / table_header / table_row all carry doc_type_kwd "text" but
+ // must NOT be previewed when CKType is empty — only a plain text body
+ // chunk legitimately has CKType empty AND should be previewed. Since
+ // group/hierarchy cannot distinguish heading from body at this layer,
+ // aligning with Python (which previews every text region) we still crop
+ // doc_type_kwd=="text"; the heading-collision guard lives in the
+ // general/token paths where CKType is set.
+ {"doc image without positions", schema.ChunkDoc{DocType: "image"}, false},
+ {"doc unknown type", schema.ChunkDoc{DocType: "equation", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, false},
}
for _, tc := range cases {
if got := needsCrop(tc.ck); got != tc.want {
@@ -126,6 +144,71 @@ func TestCropImageChunks_CropsImageTableAndText(t *testing.T) {
// chunk without positions must be left untouched (no spurious preview). The
// img_id upload is owned by imageUploadDecorator (image_upload.go) and is
// not asserted here.
+// TestCropImageChunks_DocTypeFallbackCropsImageTableText covers the
+// group/hierarchy regression fix: those chunkers forward parser output that
+// carries doc_type_kwd but no ck_type. needsCrop must fall back to DocType so
+// image/table/text regions are still cropped on demand (the bug was that they
+// were silently skipped because ck_type was empty).
+func TestCropImageChunks_DocTypeFallbackCropsImageTableText(t *testing.T) {
+ ctx := context.Background()
+ eng := assertZeroPageEngine{}
+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
+
+ chunks := []schema.ChunkDoc{
+ {DocType: "image", PDFPositions: pos},
+ {DocType: "table", PDFPositions: pos},
+ // A plain text body chunk from group/hierarchy has CKType empty and
+ // doc_type_kwd "text"; the DocType fallback must crop its preview,
+ // mirroring Python restore_pdf_text_previews. (Real group/hierarchy
+ // headings are also doc_type_kwd "text" with empty CKType and are
+ // intentionally previewed too — full alignment with Python.)
+ {DocType: "text", PDFPositions: pos},
+ {DocType: "image", Image: "data:image/png;base64,preexisting"}, // preserved
+ }
+ out := cropImageChunks(ctx, eng, chunks)
+ if len(out) != len(chunks) {
+ t.Fatalf("len(out) = %d, want %d", len(out), len(chunks))
+ }
+ for i, ck := range out {
+ switch ck.DocType {
+ case "text":
+ if !strings.HasPrefix(ck.Image, "data:image/png;base64,") {
+ t.Errorf("chunk %d (doc text): image = %q, want preview cropped via DocType fallback", i, ck.Image)
+ }
+ case "image":
+ if ck.Image == "data:image/png;base64,preexisting" {
+ continue
+ }
+ if !strings.HasPrefix(ck.Image, "data:image/png;base64,") {
+ t.Errorf("chunk %d (doc image): image = %q, want cropped via DocType fallback", i, ck.Image)
+ }
+ case "table":
+ if !strings.HasPrefix(ck.Image, "data:image/png;base64,") {
+ t.Errorf("chunk %d (doc table): image = %q, want cropped via DocType fallback", i, ck.Image)
+ }
+ }
+ }
+}
+
+// TestCropImageChunks_HeadingExcludedByCKType verifies that the general/token
+// path still excludes headings: a chunk with CKType "heading" is never
+// previewed even when it carries positions — the CKType branch takes priority
+// over the DocType fallback.
+func TestCropImageChunks_HeadingExcludedByCKType(t *testing.T) {
+ ctx := context.Background()
+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
+ chunks := []schema.ChunkDoc{
+ {CKType: "heading", DocType: "text", PDFPositions: pos},
+ }
+ out := cropImageChunks(ctx, mockCropEngine{}, chunks)
+ if len(out) != 1 {
+ t.Fatalf("len(out) = %d, want 1", len(out))
+ }
+ if out[0].Image != "" {
+ t.Errorf("heading chunk: image = %q, want no preview (CKType takes priority)", out[0].Image)
+ }
+}
+
func TestRestorePDFTextPreview(t *testing.T) {
ctx := context.Background()
pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.go
index aa19aff7f6..b1dbe9ed8b 100644
--- a/internal/ingestion/component/vision_enhancement.go
+++ b/internal/ingestion/component/vision_enhancement.go
@@ -155,6 +155,16 @@ func isVisionEnhancementAllowed(fileType utility.FileType) bool {
}
}
+// visionImageCropper yields a vision-usable base64 image for a parsed item.
+// Under cgo it crops on demand from the source PDF when the item carries
+// positions but no inlined image; under !cgo it returns the inlined image
+// (the only form available without a native renderer). Close releases any
+// re-acquired engine so native handles are not leaked.
+type visionImageCropper interface {
+ Crop(item map[string]any) (string, error)
+ Close() error
+}
+
// maybeDispatchVisionEnhancement enriches parsed JSON items with vision-model
// descriptions of embedded images and tables (doc_type_kwd in {"image", "table"}
// with non-empty image field).
@@ -182,7 +192,11 @@ func maybeDispatchVisionEnhancement(
}
language := resolveVisionLanguage(inputs, "")
- // 1. Collect target items (images/tables with non-empty image data).
+ // 1. Collect target items (images/tables that can be described). A target
+ // may carry an inlined image (docx/markdown, or any pre-inlined source) or,
+ // under cgo for PDF, only PDF positions — in which case the cropper
+ // re-acquires the source PDF and crops on demand (the parser no longer
+ // inlines PDF media). The cropper transparently handles both forms.
type target struct {
idx int
}
@@ -192,16 +206,27 @@ func maybeDispatchVisionEnhancement(
if kd != "image" && kd != "table" {
continue
}
- img, _ := item["image"].(string)
- if img == "" {
+ if img, _ := item["image"].(string); img != "" {
+ targets = append(targets, target{idx: i})
continue
}
- targets = append(targets, target{idx: i})
+ if _, ok := parser.ExtractPDFPositions(item); ok {
+ targets = append(targets, target{idx: i})
+ }
}
if len(targets) == 0 {
return dispatched, false, nil
}
+ // Acquire the on-demand cropper (cgo: crops from storage; !cgo: returns
+ // the inlined image). Best-effort: a failure here means no vision
+ // enhancement, matching Python's try/except pass.
+ cropper, cerr := newVisionImageCropper(ctx, db, inputs)
+ if cerr != nil {
+ return dispatched, false, nil
+ }
+ defer cropper.Close()
+
// 2. Resolve the per-call IMAGE2TEXT model, then fall back to the tenant
// default. Mirror Python's vlm_conf["llm_id"] preference.
family := resolveParserFamily(fileType)
@@ -257,7 +282,10 @@ dispatch:
defer wg.Done()
defer func() { <-sem }()
- img, _ := dispatched.JSON[itemIdx]["image"].(string)
+ img, ierr := cropper.Crop(dispatched.JSON[itemIdx])
+ if ierr != nil || img == "" {
+ return
+ }
if !isUsableVisionImage(img) {
slog.Warn("vision enhancement: invalid image data skipped",
"item", itemIdx)
diff --git a/internal/ingestion/component/vision_enhancement_cgo.go b/internal/ingestion/component/vision_enhancement_cgo.go
new file mode 100644
index 0000000000..f38674bddb
--- /dev/null
+++ b/internal/ingestion/component/vision_enhancement_cgo.go
@@ -0,0 +1,160 @@
+//go:build cgo
+
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "context"
+ "image"
+ "sync"
+
+ deepdocpdf "ragflow/internal/deepdoc/parser/pdf"
+ "ragflow/internal/deepdoc/parser/pdf/util"
+ deepdoctype "ragflow/internal/deepdoc/parser/type"
+ "ragflow/internal/parser/parser"
+
+ "gorm.io/gorm"
+)
+
+// visionSourceFetcher and visionEngineOpener are the injection seams for the
+// on-demand crop path. They default to the production storage/engine resolvers
+// but are overridable in tests so the cropper can be exercised without a real
+// PDF or storage backend.
+var (
+ visionSourceFetcher func(ctx context.Context, bucket, path string) ([]byte, error)
+ visionEngineOpener func(data []byte) (deepdoctype.PDFEngine, error)
+)
+
+func init() {
+ visionSourceFetcher = FetchBinary
+ visionEngineOpener = deepdocpdf.NewEngine
+}
+
+// visionPDFCropper crops section images on demand from the source PDF. Parser
+// output for PDF no longer carries an inlined base64 image (that is what this
+// change removes to bound parser-phase memory); instead each item carries PDF
+// positions, and the VLM path re-acquires the source bytes, opens a fresh
+// engine, renders the spanned page(s), and crops the region — exactly what the
+// chunker does at index time. The engine is opened lazily and at most once per
+// vision-enhancement call, then released via Close.
+type visionPDFCropper struct {
+ ctx context.Context
+ db *gorm.DB
+ inputs map[string]any
+
+ once sync.Once
+ engine deepdoctype.PDFEngine
+ engErr error
+}
+
+// newVisionImageCropper builds the on-demand cropper. It never touches storage
+// here; the source PDF is re-acquired lazily on the first Crop call that needs
+// it.
+func newVisionImageCropper(ctx context.Context, db *gorm.DB, inputs map[string]any) (visionImageCropper, error) {
+ return &visionPDFCropper{ctx: ctx, db: db, inputs: inputs}, nil
+}
+
+func (c *visionPDFCropper) Crop(item map[string]any) (string, error) {
+ // Fast path: an inlined image (docx/markdown, or any pre-inlined source)
+ // is used directly — no storage access, no engine.
+ if img, _ := item["image"].(string); img != "" {
+ return img, nil
+ }
+ matrix, ok := parser.ExtractPDFPositions(item)
+ if !ok {
+ return "", nil
+ }
+ positions := util.PositionsFromMatrix(matrix)
+ if len(positions) == 0 {
+ return "", nil
+ }
+ if err := c.ensureEngine(); err != nil {
+ // Best-effort: a missing/unreadable source PDF means no vision
+ // description for this item, not a hard failure.
+ return "", nil
+ }
+ if c.engine == nil {
+ return "", nil
+ }
+ // Render each distinct page the positions span (1-based → 0-based is
+ // handled by PositionsFromMatrix). Reuse the chunker's sliding-window
+ // idea but scope it to this single item: typically one page.
+ pages := make(map[int]struct{}, len(positions))
+ for _, pos := range positions {
+ for _, pn := range pos.PageNumbers {
+ pages[pn] = struct{}{}
+ }
+ }
+ single := make(map[int]image.Image, len(pages))
+ for pn := range pages {
+ img, rerr := deepdocpdf.RenderPageToImage(c.engine, pn)
+ if rerr != nil || img == nil {
+ continue
+ }
+ single[pn] = img
+ }
+ if len(single) == 0 {
+ return "", nil
+ }
+ return util.CropSectionPositions(positions, single, deepdoctype.DlaScale), nil
+}
+
+func (c *visionPDFCropper) ensureEngine() error {
+ c.once.Do(func() {
+ data, err := c.acquireSource()
+ if err != nil || len(data) == 0 {
+ c.engErr = err
+ return
+ }
+ // Only PDFs can be cropped. Guard against other binary types so a
+ // docx/markdown item that happens to reach here stays a no-op.
+ if len(data) < 5 || string(data[:5]) != "%PDF-" {
+ return
+ }
+ eng, oerr := visionEngineOpener(data)
+ if oerr != nil {
+ c.engErr = oerr
+ return
+ }
+ c.engine = eng
+ })
+ return c.engErr
+}
+
+func (c *visionPDFCropper) acquireSource() ([]byte, error) {
+ if bucket, _ := getString(c.inputs, "bucket"); bucket != "" {
+ if path, _ := getString(c.inputs, "path"); path != "" {
+ return visionSourceFetcher(c.ctx, bucket, path)
+ }
+ }
+ if docID, _ := getString(c.inputs, "doc_id"); docID != "" {
+ ref, err := ResolveDocumentStorage(c.ctx, c.db, docID)
+ if err != nil || ref == nil {
+ return nil, err
+ }
+ return visionSourceFetcher(c.ctx, ref.Bucket, ref.Path)
+ }
+ return nil, nil
+}
+
+func (c *visionPDFCropper) Close() error {
+ if c.engine != nil {
+ return c.engine.Close()
+ }
+ return nil
+}
diff --git a/internal/ingestion/component/vision_enhancement_cgo_test.go b/internal/ingestion/component/vision_enhancement_cgo_test.go
new file mode 100644
index 0000000000..6940f473f0
--- /dev/null
+++ b/internal/ingestion/component/vision_enhancement_cgo_test.go
@@ -0,0 +1,206 @@
+//go:build cgo
+
+package component
+
+import (
+ "context"
+ "image"
+ "testing"
+
+ deepdoctype "ragflow/internal/deepdoc/parser/type"
+)
+
+// mockVisionEngine returns a fixed image for any rendered page so the
+// on-demand cropper can exercise CropSectionPositions without a real PDF.
+type mockVisionEngine struct {
+ closed *bool
+}
+
+func (m mockVisionEngine) ExtractChars(int) ([]deepdoctype.TextChar, error) {
+ return nil, nil
+}
+func (m mockVisionEngine) RenderPage(int, float64) ([]byte, error) { return nil, nil }
+func (m mockVisionEngine) RenderPageImage(int, float64) (image.Image, error) {
+ return image.NewRGBA(image.Rect(0, 0, 1000, 1000)), nil
+}
+func (m mockVisionEngine) RawData() []byte { return nil }
+func (m mockVisionEngine) PageCount() (int, error) { return 1, nil }
+func (m mockVisionEngine) Outlines() ([]deepdoctype.Outline, error) { return nil, nil }
+func (m mockVisionEngine) Close() error {
+ if m.closed != nil {
+ *m.closed = true
+ }
+ return nil
+}
+
+func cgoPositions() map[string]any {
+ // 1-based page 1 region; PositionsFromMatrix shifts to 0-based.
+ return map[string]any{
+ "_pdf_positions": [][]any{{1, 10.0, 100.0, 10.0, 100.0}},
+ }
+}
+
+// TestVisionCropImage_OnDemandFromPositions verifies the core fix: a parsed
+// item that carries PDF positions but no inlined image is cropped on demand
+// by re-acquiring the source PDF from storage and rendering the region.
+func TestVisionCropImage_OnDemandFromPositions(t *testing.T) {
+ closed := false
+ oldFetcher := visionSourceFetcher
+ oldOpener := visionEngineOpener
+ defer func() {
+ visionSourceFetcher = oldFetcher
+ visionEngineOpener = oldOpener
+ }()
+ visionSourceFetcher = func(ctx context.Context, bucket, path string) ([]byte, error) {
+ return []byte("%PDF-fake-engine-bytes"), nil
+ }
+ visionEngineOpener = func(data []byte) (deepdoctype.PDFEngine, error) {
+ return mockVisionEngine{closed: &closed}, nil
+ }
+
+ cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{"bucket": "b", "path": "p"})
+ if err != nil {
+ t.Fatalf("newVisionImageCropper: %v", err)
+ }
+ defer cropper.Close()
+
+ img, err := cropper.Crop(cgoPositions())
+ if err != nil {
+ t.Fatalf("Crop: %v", err)
+ }
+ if img == "" {
+ t.Fatal("Crop returned empty; expected an on-demand cropped base64 image")
+ }
+ if closed {
+ t.Fatal("engine closed before use completed")
+ }
+}
+
+// TestVisionCropImage_PassthroughInline proves that when the item already
+// carries an inlined image (docx/markdown, or any pre-inlined source), the
+// cropper returns it directly without touching storage.
+func TestVisionCropImage_PassthroughInline(t *testing.T) {
+ fetchCalled := false
+ oldFetcher := visionSourceFetcher
+ defer func() { visionSourceFetcher = oldFetcher }()
+ visionSourceFetcher = func(ctx context.Context, bucket, path string) ([]byte, error) {
+ fetchCalled = true
+ return nil, nil
+ }
+
+ cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{})
+ if err != nil {
+ t.Fatalf("newVisionImageCropper: %v", err)
+ }
+ defer cropper.Close()
+
+ const inline = "data:image/png;base64,preinlined"
+ img, err := cropper.Crop(map[string]any{"image": inline})
+ if err != nil {
+ t.Fatalf("Crop: %v", err)
+ }
+ if img != inline {
+ t.Fatalf("Crop = %q, want passthrough %q", img, inline)
+ }
+ if fetchCalled {
+ t.Fatal("storage fetch must not be called for an inlined image")
+ }
+}
+
+// TestVisionCropImage_NoImageNoPositions proves an item with neither an image
+// nor positions yields no crop (no spurious storage access, no error).
+func TestVisionCropImage_NoImageNoPositions(t *testing.T) {
+ fetchCalled := false
+ oldFetcher := visionSourceFetcher
+ defer func() { visionSourceFetcher = oldFetcher }()
+ visionSourceFetcher = func(ctx context.Context, bucket, path string) ([]byte, error) {
+ fetchCalled = true
+ return nil, nil
+ }
+
+ cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{})
+ if err != nil {
+ t.Fatalf("newVisionImageCropper: %v", err)
+ }
+ defer cropper.Close()
+
+ img, err := cropper.Crop(map[string]any{"doc_type_kwd": "image"})
+ if err != nil {
+ t.Fatalf("Crop: %v", err)
+ }
+ if img != "" {
+ t.Fatalf("Crop = %q, want empty (no image, no positions)", img)
+ }
+ if fetchCalled {
+ t.Fatal("storage fetch must not be called without positions")
+ }
+}
+
+// TestVisionCropImage_NonPDFBytesIsNoOp proves the %PDF- guard: if storage
+// returns non-PDF bytes (e.g. a docx/markdown mislabeled, or a lookup miss),
+// the cropper does not attempt to open an engine and returns no crop.
+func TestVisionCropImage_NonPDFBytesIsNoOp(t *testing.T) {
+ oldFetcher := visionSourceFetcher
+ oldOpener := visionEngineOpener
+ defer func() {
+ visionSourceFetcher = oldFetcher
+ visionEngineOpener = oldOpener
+ }()
+ visionSourceFetcher = func(ctx context.Context, bucket, path string) ([]byte, error) {
+ return []byte("this is not a pdf"), nil
+ }
+ openerCalled := false
+ visionEngineOpener = func(data []byte) (deepdoctype.PDFEngine, error) {
+ openerCalled = true
+ return mockVisionEngine{}, nil
+ }
+
+ cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{"bucket": "b", "path": "p"})
+ if err != nil {
+ t.Fatalf("newVisionImageCropper: %v", err)
+ }
+ defer cropper.Close()
+
+ img, err := cropper.Crop(cgoPositions())
+ if err != nil {
+ t.Fatalf("Crop: %v", err)
+ }
+ if img != "" {
+ t.Fatalf("Crop = %q, want empty for non-PDF bytes", img)
+ }
+ if openerCalled {
+ t.Fatal("engine opener must not run on non-PDF bytes")
+ }
+}
+
+// TestVisionCropImage_EngineClosedAfterUse proves the re-acquired engine is
+// released (no native handle leak) once cropping is done.
+func TestVisionCropImage_EngineClosedAfterUse(t *testing.T) {
+ closed := false
+ oldFetcher := visionSourceFetcher
+ oldOpener := visionEngineOpener
+ defer func() {
+ visionSourceFetcher = oldFetcher
+ visionEngineOpener = oldOpener
+ }()
+ visionSourceFetcher = func(ctx context.Context, bucket, path string) ([]byte, error) {
+ return []byte("%PDF-fake"), nil
+ }
+ visionEngineOpener = func(data []byte) (deepdoctype.PDFEngine, error) {
+ return mockVisionEngine{closed: &closed}, nil
+ }
+
+ cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{"bucket": "b", "path": "p"})
+ if err != nil {
+ t.Fatalf("newVisionImageCropper: %v", err)
+ }
+ if _, err := cropper.Crop(cgoPositions()); err != nil {
+ t.Fatalf("Crop: %v", err)
+ }
+ if err := cropper.Close(); err != nil {
+ t.Fatalf("Close: %v", err)
+ }
+ if !closed {
+ t.Fatal("engine should be closed after use")
+ }
+}
diff --git a/internal/ingestion/component/vision_enhancement_nocgo.go b/internal/ingestion/component/vision_enhancement_nocgo.go
new file mode 100644
index 0000000000..df3522199d
--- /dev/null
+++ b/internal/ingestion/component/vision_enhancement_nocgo.go
@@ -0,0 +1,45 @@
+//go:build !cgo
+
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "context"
+
+ "gorm.io/gorm"
+)
+
+// visionPDFCropperNoop is the !cgo cropper: without a native renderer there is
+// no on-demand crop, so it returns the inlined image when present (docx/
+// markdown or any pre-inlined source) and nothing otherwise. The parser does
+// not inline PDF media under cgo, and PDF parsing itself requires cgo, so this
+// path is only ever exercised for the non-PDF inline forms.
+type visionPDFCropperNoop struct{}
+
+func newVisionImageCropper(_ context.Context, _ *gorm.DB, _ map[string]any) (visionImageCropper, error) {
+ return visionPDFCropperNoop{}, nil
+}
+
+func (visionPDFCropperNoop) Crop(item map[string]any) (string, error) {
+ if img, _ := item["image"].(string); img != "" {
+ return img, nil
+ }
+ return "", nil
+}
+
+func (visionPDFCropperNoop) Close() error { return nil }
diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.go
index 6cc90e03f2..b5de904d3a 100644
--- a/internal/parser/parser/pdf_parser_common.go
+++ b/internal/parser/parser/pdf_parser_common.go
@@ -366,7 +366,13 @@ func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseR
}
applyPDFPostProcess(&processed, opts)
defer processed.Close()
- cropMediaSections(&processed)
+ // NOTE: PDF media (figure/table) is intentionally NOT cropped here under
+ // cgo. The parser no longer inlines base64 images — that is what bounded
+ // the parser-phase memory peak. Image/table sections keep only their PDF
+ // positions; the chunker re-acquires the source PDF and crops on demand at
+ // index time, and the VLM path crops on demand when it needs to describe a
+ // figure/table. The Markdown path below still inlines, because markdown
+ // output embeds images directly and has no downstream on-demand consumer.
items := pdflayout.SectionsToJSON(processed.Sections)
if len(items) == 0 {
@@ -788,14 +794,16 @@ func normalizePDFDocType(item map[string]any) {
return
}
layoutType, _ := item["layout_type"].(string)
+ // A figure caption is a media section even when the parser no longer
+ // inlines its cropped image (cgo): it still carries PDF positions, so the
+ // downstream VLM/chunker crop it on demand. Classify it as image whenever
+ // it has positions (the inlined image was only a side effect of cropping).
+ _, hasMedia := ExtractPDFPositions(item)
if docType, _ := item["doc_type_kwd"].(string); docType != "" {
- // A figure caption can carry the cropped figure image after PDF media
- // sections are rendered. Keep it aligned with Python's media-section
- // contract so the downstream VLM enhancement can process it.
- if docType == "text" && layoutType == deepdoctype.DLALabelFigureCaption {
- if img, _ := item["image"].(string); img != "" {
- item["doc_type_kwd"] = "image"
- }
+ // A figure caption keeps its media classification so the downstream
+ // VLM enhancement and on-demand chunker crop it.
+ if docType == "text" && layoutType == deepdoctype.DLALabelFigureCaption && hasMedia {
+ item["doc_type_kwd"] = "image"
}
return
}
@@ -804,6 +812,12 @@ func normalizePDFDocType(item map[string]any) {
item["doc_type_kwd"] = "table"
case "figure", "image":
item["doc_type_kwd"] = "image"
+ case deepdoctype.DLALabelFigureCaption:
+ if hasMedia {
+ item["doc_type_kwd"] = "image"
+ } else {
+ item["doc_type_kwd"] = "text"
+ }
default:
if img, _ := item["image"].(string); img != "" {
item["doc_type_kwd"] = "image"
@@ -813,6 +827,45 @@ func normalizePDFDocType(item map[string]any) {
}
}
+// ExtractPDFPositions is the single source of truth for "does this parsed item
+// carry a usable PDF crop region". It returns the non-empty positions matrix
+// (under the canonical _pdf_positions key or the legacy positions key),
+// accepting either the typed [][]any form produced in-process or the
+// JSON-decoded []any form whose elements are themselves []any rows. The bool
+// reports whether a usable matrix was present.
+//
+// The parser (doc-type classification in normalizePDFDocType) and the on-demand
+// croppers (VLM vision_enhancement and the chunker) MUST agree on this
+// contract, so all call sites delegate here instead of re-implementing the
+// check. A loose "non-nil" test previously misclassified items whose positions
+// were empty or not actually a matrix (e.g. an empty []any or a stray scalar),
+// which escaped the figure-caption → image normalization and the crop path.
+func ExtractPDFPositions(item map[string]any) ([][]any, bool) {
+ for _, key := range []string{"_pdf_positions", "positions"} {
+ switch v := item[key].(type) {
+ case [][]any:
+ if len(v) > 0 {
+ return v, true
+ }
+ case []any:
+ out := make([][]any, 0, len(v))
+ ok := true
+ for _, row := range v {
+ r, rOK := row.([]any)
+ if !rOK {
+ ok = false
+ break
+ }
+ out = append(out, r)
+ }
+ if ok && len(out) > 0 {
+ return out, true
+ }
+ }
+ }
+ return nil, false
+}
+
func parsePDFWithDeepDoc(ctx context.Context, filename string, data []byte, parseFn func(context.Context, []byte, deepdoctype.DocAnalyzer) (*deepdoctype.ParseResult, error)) ParseResult {
return parsePDFWithDeepDocOptions(ctx, filename, data, pdfPostProcessOptions{}, parseFn)
}
diff --git a/internal/parser/parser/pdf_parser_common_test.go b/internal/parser/parser/pdf_parser_common_test.go
index 9ceed111f0..be57c5674b 100644
--- a/internal/parser/parser/pdf_parser_common_test.go
+++ b/internal/parser/parser/pdf_parser_common_test.go
@@ -101,23 +101,55 @@ func TestPDFParseResultToJSON_NormalizesCoreFields(t *testing.T) {
}
}
+// TestPDFParseResultToJSON_ClassifiesFigureCaptionWithImage pins the
+// positions-driven contract for figure-caption classification. The parser's
+// JSON path no longer inlines cropped media, so a figure caption is promoted to
+// doc_type_kwd "image" only when it carries a usable PDF positions matrix
+// (the on-demand crop source) — NOT merely because an image is present. This
+// locks down the fix for the previous loose `v != nil` check, which promoted
+// captions whose _pdf_positions were empty or not a real matrix to "image".
func TestPDFParseResultToJSON_ClassifiesFigureCaptionWithImage(t *testing.T) {
- parsed := &deepdoctype.ParseResult{
- Sections: []deepdoctype.Section{
- {
- Text: "小灰灰",
- LayoutType: deepdoctype.DLALabelFigureCaption,
- Image: "aGVsbG8=",
- },
- },
+ // A figure caption WITH positions is classified as image: the on-demand
+ // VLM/chunker crop path fires.
+ withPos := &deepdoctype.ParseResult{
+ Sections: []deepdoctype.Section{{
+ Text: "小灰灰",
+ LayoutType: deepdoctype.DLALabelFigureCaption,
+ Positions: []deepdoctype.Position{{
+ PageNumbers: []int{0},
+ Left: 10,
+ Right: 50,
+ Top: 10,
+ Bottom: 50,
+ }},
+ }},
}
-
- res := pdfParseResultToJSON("sample.pdf", parsed)
+ res := pdfParseResultToJSON("with-pos.pdf", withPos)
if res.Err != nil {
t.Fatalf("pdfParseResultToJSON: %v", res.Err)
}
- if got, want := res.JSON[0]["doc_type_kwd"], "image"; got != want {
- t.Fatalf("doc_type_kwd = %v, want %v", got, want)
+ if got := res.JSON[0]["doc_type_kwd"]; got != "image" {
+ t.Fatalf("figure caption with positions doc_type_kwd = %v, want image", got)
+ }
+
+ // A figure caption with an inlined image but NO positions is NOT promoted
+ // to image: positions are the sole classifier now. (In the PDF JSON path
+ // an inlined image without positions does not occur — cropMediaSections was
+ // removed — so this documents that the presence of a stray image alone is
+ // insufficient.)
+ withImageNoPos := &deepdoctype.ParseResult{
+ Sections: []deepdoctype.Section{{
+ Text: "小灰灰",
+ LayoutType: deepdoctype.DLALabelFigureCaption,
+ Image: "aGVsbG8=",
+ }},
+ }
+ res2 := pdfParseResultToJSON("with-image-no-pos.pdf", withImageNoPos)
+ if res2.Err != nil {
+ t.Fatalf("pdfParseResultToJSON: %v", res2.Err)
+ }
+ if got := res2.JSON[0]["doc_type_kwd"]; got != "text" {
+ t.Fatalf("figure caption with image but no positions doc_type_kwd = %v, want text", got)
}
}
@@ -465,7 +497,12 @@ func (m *mockPDFEngineForCommonTest) Close() error {
return nil
}
-func TestPDFParseResultToJSON_CropsMediaSectionsWithEngine(t *testing.T) {
+// TestPDFParseResultToJSON_NoInlineRetainsPositions pins the new cgo contract:
+// the parser no longer inlines base64 media into the JSON items (that bounded
+// the parser-phase memory peak). Figure/table sections keep their PDF
+// positions so the chunker and VLM can crop on demand. The crop logic itself
+// is still exercised by the Markdown inline tests.
+func TestPDFParseResultToJSON_NoInlineRetainsPositions(t *testing.T) {
mockEngine := &mockPDFEngineForCommonTest{}
parsed := &deepdoctype.ParseResult{
Engine: mockEngine,
@@ -509,32 +546,34 @@ func TestPDFParseResultToJSON_CropsMediaSectionsWithEngine(t *testing.T) {
t.Fatalf("JSON len = %d, want 3", len(res.JSON))
}
- // Figure should have cropped image populated
- figImg, ok := res.JSON[0]["image"].(string)
- if !ok || figImg == "" {
- t.Fatalf("Figure JSON[0].image should be non-empty base64 data url, got %v", res.JSON[0]["image"])
+ // Figure must retain positions but NOT inline an image.
+ figPos, ok := res.JSON[0]["_pdf_positions"].([][]any)
+ if !ok || len(figPos) == 0 {
+ t.Fatalf("Figure should retain _pdf_positions, got %v", res.JSON[0]["_pdf_positions"])
}
- if !strings.HasPrefix(figImg, "data:image/png;base64,") {
- t.Fatalf("Figure JSON[0].image prefix mismatch, got %q", figImg)
+ if img, _ := res.JSON[0]["image"].(string); img != "" {
+ t.Fatalf("Figure should not inline image under cgo, got %q", img)
}
- // Table should have cropped image populated
- tblImg, ok := res.JSON[1]["image"].(string)
- if !ok || tblImg == "" {
- t.Fatalf("Table JSON[1].image should be non-empty base64 data url, got %v", res.JSON[1]["image"])
+ // Table likewise retains positions, no inline image.
+ tblPos, ok := res.JSON[1]["_pdf_positions"].([][]any)
+ if !ok || len(tblPos) == 0 {
+ t.Fatalf("Table should retain _pdf_positions, got %v", res.JSON[1]["_pdf_positions"])
}
- if !strings.HasPrefix(tblImg, "data:image/png;base64,") {
- t.Fatalf("Table JSON[1].image prefix mismatch, got %q", tblImg)
+ if img, _ := res.JSON[1]["image"].(string); img != "" {
+ t.Fatalf("Table should not inline image under cgo, got %q", img)
}
- // Plain text should NOT have cropped image
- textImg, _ := res.JSON[2]["image"].(string)
- if textImg != "" {
- t.Fatalf("Text JSON[2].image should be empty, got %q", textImg)
+ // Plain text: no image, positions retained.
+ if img, _ := res.JSON[2]["image"].(string); img != "" {
+ t.Fatalf("Text should not inline image, got %q", img)
}
}
-func TestPDFParseResultToJSON_CropsFigureCaptionForVisionEnhancement(t *testing.T) {
+// TestPDFParseResultToJSON_FigureCaptionNoInline pins that a figure caption is
+// still classified as doc_type_kwd "image" (so the chunker/VLM crop it on
+// demand) but the parser does not inline the cropped image for it.
+func TestPDFParseResultToJSON_FigureCaptionNoInline(t *testing.T) {
mockEngine := &mockPDFEngineForCommonTest{}
parsed := &deepdoctype.ParseResult{
Engine: mockEngine,
@@ -559,8 +598,135 @@ func TestPDFParseResultToJSON_CropsFigureCaptionForVisionEnhancement(t *testing.
if got, want := res.JSON[0]["doc_type_kwd"], "image"; got != want {
t.Fatalf("doc_type_kwd = %v, want %v", got, want)
}
- if image, _ := res.JSON[0]["image"].(string); image == "" {
- t.Fatal("figure caption image should be cropped before vision enhancement")
+ if pos, ok := res.JSON[0]["_pdf_positions"].([][]any); !ok || len(pos) == 0 {
+ t.Fatal("figure caption should retain _pdf_positions for on-demand crop")
+ }
+ if image, _ := res.JSON[0]["image"].(string); image != "" {
+ t.Fatalf("figure caption should not be inlined by the parser, got %q", image)
+ }
+}
+
+// TestExtractPDFPositions locks down the single source of truth for "does this
+// item carry a usable PDF crop region". The historical bug was a loose
+// `v != nil` test that accepted empty slices and stray scalars; this test
+// proves the contract is now strict (non-empty matrix only) and that both the
+// canonical _pdf_positions key and the legacy positions key are honored, in
+// both the typed [][]any form and the JSON-decoded []any form.
+func TestExtractPDFPositions(t *testing.T) {
+ cases := []struct {
+ name string
+ item map[string]any
+ want bool
+ rows int
+ key string
+ }{
+ {
+ name: "typed non-empty matrix under _pdf_positions",
+ item: map[string]any{"_pdf_positions": [][]any{{float64(1), float64(2), float64(3), float64(4), float64(1)}}},
+ want: true, rows: 1, key: "_pdf_positions",
+ },
+ {
+ name: "json-decoded []any rows under _pdf_positions",
+ item: map[string]any{"_pdf_positions": []any{[]any{float64(1), float64(2), float64(3), float64(4), float64(1)}}},
+ want: true, rows: 1, key: "_pdf_positions",
+ },
+ {
+ name: "legacy positions key honored",
+ item: map[string]any{"positions": [][]any{{float64(1), float64(2), float64(3), float64(4), float64(1)}}},
+ want: true, rows: 1, key: "positions",
+ },
+ {
+ name: "empty typed matrix rejected",
+ item: map[string]any{"_pdf_positions": [][]any{}},
+ want: false,
+ },
+ {
+ name: "empty json-decoded matrix rejected",
+ item: map[string]any{"_pdf_positions": []any{}},
+ want: false,
+ },
+ {
+ name: "nil value rejected",
+ item: map[string]any{"_pdf_positions": nil},
+ want: false,
+ },
+ {
+ name: "missing key rejected",
+ item: map[string]any{"doc_type_kwd": "image"},
+ want: false,
+ },
+ {
+ name: "stray scalar rejected (old v != nil bug)",
+ item: map[string]any{"_pdf_positions": "not-a-matrix"},
+ want: false,
+ },
+ {
+ name: "non-slice numeric rejected",
+ item: map[string]any{"_pdf_positions": float64(42)},
+ want: false,
+ },
+ {
+ name: "[]any with non-row element rejected",
+ item: map[string]any{"_pdf_positions": []any{"bad", float64(1)}},
+ want: false,
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ matrix, ok := ExtractPDFPositions(tc.item)
+ if ok != tc.want {
+ t.Fatalf("ExtractPDFPositions ok = %v, want %v", ok, tc.want)
+ }
+ if tc.want {
+ if len(matrix) != tc.rows {
+ t.Fatalf("matrix rows = %d, want %d", len(matrix), tc.rows)
+ }
+ // Every returned row must be []any (normalized form).
+ for i, row := range matrix {
+ if row == nil {
+ t.Fatalf("matrix[%d] is nil after normalization", i)
+ }
+ }
+ // The honored key must carry the matrix in normalized [][]any.
+ if _, present := tc.item[tc.key]; !present {
+ t.Fatalf("expected honored key %q missing", tc.key)
+ }
+ }
+ })
+ }
+}
+
+// TestNormalizePDFDocType_FigureCaptionPositionsGate proves Finding B: a figure
+// caption is only promoted to doc_type_kwd "image" (which lights up the
+// on-demand VLM/chunker crop) when it actually carries a usable positions
+// matrix. The previous loose `v != nil` check promoted captions whose
+// _pdf_positions were empty or not a real matrix, wrongly flagging them for
+// cropping. normalizePDFDocType must delegate to ExtractPDFPositions.
+func TestNormalizePDFDocType_FigureCaptionPositionsGate(t *testing.T) {
+ withPos := map[string]any{
+ "layout_type": deepdoctype.DLALabelFigureCaption,
+ "_pdf_positions": [][]any{{float64(1), float64(2), float64(3), float64(4), float64(1)}},
+ }
+ normalizePDFDocType(withPos)
+ if got := withPos["doc_type_kwd"]; got != "image" {
+ t.Fatalf("with positions doc_type_kwd = %v, want image", got)
+ }
+
+ for _, empty := range []any{
+ []any{},
+ [][]any{},
+ nil,
+ "stray-scalar",
+ } {
+ withoutPos := map[string]any{
+ "layout_type": deepdoctype.DLALabelFigureCaption,
+ "_pdf_positions": empty,
+ }
+ normalizePDFDocType(withoutPos)
+ if got := withoutPos["doc_type_kwd"]; got != "text" {
+ t.Fatalf("_pdf_positions=%#v doc_type_kwd = %v, want text (must not promote empty/non-matrix)", empty, got)
+ }
}
}