RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): stop inlining PDF base64 media; crop on demand at VLM and chunker (#19964)
xugangqiang3a902e37dfba
diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo.go b/internal/ingestion/component/chunker/pdfcrop_cgo.goindex 04647bb042..b58f2a7112 100644--- a/internal/ingestion/component/chunker/pdfcrop_cgo.go+++ b/internal/ingestion/component/chunker/pdfcrop_cgo.go@@ -148,8 +148,22 @@ func cropImageChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks [// cropped; text chunks with positions get a rendered preview of the text// region (Python restore_pdf_text_previews). A pre-existing Image is never// re-cropped — cropImageChunks honors that separately.+//+// CKType is the chunker-layer refinement (heading/table_header/table_row/text/+// image/table). General and token chunkers always set it before cropping, so+// they are handled by the CKType branch above. Group and hierarchy chunkers+// forward the parser's output verbatim, which carries only the coarser+// doc_type_kwd (no ck_type). For those, fall back to DocType so image/table/+// text regions are still cropped on demand — otherwise every figure/table/+// text preview in group/hierarchy would be silently skipped. When CKType is+// set (general/token) it takes priority, so headings (CKType "heading") stay+// excluded from preview cropping.func needsCrop(ck schema.ChunkDoc) bool {- switch ck.CKType {+ typ := ck.CKType+ if typ == "" {+ typ = ck.DocType+ }+ switch typ {case "image", "table", "text":return len(ck.PDFPositions) > 0 || len(ck.Positions) > 0default:diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go b/internal/ingestion/component/chunker/pdfcrop_cgo_test.goindex 613d0ac11c..967db9a4b4 100644--- a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go+++ b/internal/ingestion/component/chunker/pdfcrop_cgo_test.go@@ -69,11 +69,29 @@ func TestNeedsCrop(t *testing.T) {ck schema.ChunkDocwant bool}{+ // --- CKType set (general / token paths): unchanged behavior ---{"image with positions", schema.ChunkDoc{CKType: "image", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},{"table with positions", schema.ChunkDoc{CKType: "table", Positions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},{"text with positions", schema.ChunkDoc{CKType: "text", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},{"image without positions", schema.ChunkDoc{CKType: "image"}, false},- {"unknown type", schema.ChunkDoc{CKType: "equation", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, false},+ {"heading excluded via CKType", schema.ChunkDoc{CKType: "heading", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, false},+ {"unknown type via CKType", schema.ChunkDoc{CKType: "equation", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, false},++ // --- CKType empty: fall back to DocType (group / hierarchy paths,+ // whose parser output carries no ck_type). This is the fix for the+ // bug where figure/image/table in group/hierarchy were never cropped.+ {"doc image fallback", schema.ChunkDoc{DocType: "image", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},+ {"doc table fallback", schema.ChunkDoc{DocType: "table", Positions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},+ {"doc text fallback", schema.ChunkDoc{DocType: "text", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},+ // Heading / table_header / table_row all carry doc_type_kwd "text" but+ // must NOT be previewed when CKType is empty — only a plain text body+ // chunk legitimately has CKType empty AND should be previewed. Since+ // group/hierarchy cannot distinguish heading from body at this layer,+ // aligning with Python (which previews every text region) we still crop+ // doc_type_kwd=="text"; the heading-collision guard lives in the+ // general/token paths where CKType is set.+ {"doc image without positions", schema.ChunkDoc{DocType: "image"}, false},+ {"doc unknown type", schema.ChunkDoc{DocType: "equation", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, false},}for _, tc := range cases {if got := needsCrop(tc.ck); got != tc.want {@@ -126,6 +144,71 @@ func TestCropImageChunks_CropsImageTableAndText(t *testing.T) {// chunk without positions must be left untouched (no spurious preview). The// img_id upload is owned by imageUploadDecorator (image_upload.go) and is// not asserted here.+// TestCropImageChunks_DocTypeFallbackCropsImageTableText covers the+// group/hierarchy regression fix: those chunkers forward parser output that+// carries doc_type_kwd but no ck_type. needsCrop must fall back to DocType so+// image/table/text regions are still cropped on demand (the bug was that they+// were silently skipped because ck_type was empty).+func TestCropImageChunks_DocTypeFallbackCropsImageTableText(t *testing.T) {+ ctx := context.Background()+ eng := assertZeroPageEngine{}+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})++ chunks := []schema.ChunkDoc{+ {DocType: "image", PDFPositions: pos},+ {DocType: "table", PDFPositions: pos},+ // A plain text body chunk from group/hierarchy has CKType empty and+ // doc_type_kwd "text"; the DocType fallback must crop its preview,+ // mirroring Python restore_pdf_text_previews. (Real group/hierarchy+ // headings are also doc_type_kwd "text" with empty CKType and are+ // intentionally previewed too — full alignment with Python.)+ {DocType: "text", PDFPositions: pos},+ {DocType: "image", Image: "data:image/png;base64,preexisting"}, // preserved+ }+ out := cropImageChunks(ctx, eng, chunks)+ if len(out) != len(chunks) {+ t.Fatalf("len(out) = %d, want %d", len(out), len(chunks))+ }+ for i, ck := range out {+ switch ck.DocType {+ case "text":+ if !strings.HasPrefix(ck.Image, "data:image/png;base64,") {+ t.Errorf("chunk %d (doc text): image = %q, want preview cropped via DocType fallback", i, ck.Image)+ }+ case "image":+ if ck.Image == "data:image/png;base64,preexisting" {+ continue+ }+ if !strings.HasPrefix(ck.Image, "data:image/png;base64,") {+ t.Errorf("chunk %d (doc image): image = %q, want cropped via DocType fallback", i, ck.Image)+ }+ case "table":+ if !strings.HasPrefix(ck.Image, "data:image/png;base64,") {+ t.Errorf("chunk %d (doc table): image = %q, want cropped via DocType fallback", i, ck.Image)+ }+ }+ }+}++// TestCropImageChunks_HeadingExcludedByCKType verifies that the general/token+// path still excludes headings: a chunk with CKType "heading" is never+// previewed even when it carries positions — the CKType branch takes priority+// over the DocType fallback.+func TestCropImageChunks_HeadingExcludedByCKType(t *testing.T) {+ ctx := context.Background()+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})+ chunks := []schema.ChunkDoc{+ {CKType: "heading", DocType: "text", PDFPositions: pos},+ }+ out := cropImageChunks(ctx, mockCropEngine{}, chunks)+ if len(out) != 1 {+ t.Fatalf("len(out) = %d, want 1", len(out))+ }+ if out[0].Image != "" {+ t.Errorf("heading chunk: image = %q, want no preview (CKType takes priority)", out[0].Image)+ }+}+func TestRestorePDFTextPreview(t *testing.T) {ctx := context.Background()pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.goindex aa19aff7f6..b1dbe9ed8b 100644--- a/internal/ingestion/component/vision_enhancement.go+++ b/internal/ingestion/component/vision_enhancement.go@@ -155,6 +155,16 @@ func isVisionEnhancementAllowed(fileType utility.FileType) bool {}}+// visionImageCropper yields a vision-usable base64 image for a parsed item.+// Under cgo it crops on demand from the source PDF when the item carries+// positions but no inlined image; under !cgo it returns the inlined image+// (the only form available without a native renderer). Close releases any+// re-acquired engine so native handles are not leaked.+type visionImageCropper interface {+ Crop(item map[string]any) (string, error)+ Close() error+}+// maybeDispatchVisionEnhancement enriches parsed JSON items with vision-model// descriptions of embedded images and tables (doc_type_kwd in {"image", "table"}// with non-empty image field).@@ -182,7 +192,11 @@ func maybeDispatchVisionEnhancement(}language := resolveVisionLanguage(inputs, "")- // 1. Collect target items (images/tables with non-empty image data).+ // 1. Collect target items (images/tables that can be described). A target+ // may carry an inlined image (docx/markdown, or any pre-inlined source) or,+ // under cgo for PDF, only PDF positions — in which case the cropper+ // re-acquires the source PDF and crops on demand (the parser no longer+ // inlines PDF media). The cropper transparently handles both forms.type target struct {idx int}@@ -192,16 +206,27 @@ func maybeDispatchVisionEnhancement(if kd != "image" && kd != "table" {continue}- img, _ := item["image"].(string)- if img == "" {+ if img, _ := item["image"].(string); img != "" {+ targets = append(targets, target{idx: i})continue}- targets = append(targets, target{idx: i})+ if _, ok := parser.ExtractPDFPositions(item); ok {+ targets = append(targets, target{idx: i})+ }}if len(targets) == 0 {return dispatched, false, nil}+ // Acquire the on-demand cropper (cgo: crops from storage; !cgo: returns+ // the inlined image). Best-effort: a failure here means no vision+ // enhancement, matching Python's try/except pass.+ cropper, cerr := newVisionImageCropper(ctx, db, inputs)+ if cerr != nil {+ return dispatched, false, nil+ }+ defer cropper.Close()+// 2. Resolve the per-call IMAGE2TEXT model, then fall back to the tenant// default. Mirror Python's vlm_conf["llm_id"] preference.family := resolveParserFamily(fileType)@@ -257,7 +282,10 @@ dispatch:defer wg.Done()defer func() { <-sem }()- img, _ := dispatched.JSON[itemIdx]["image"].(string)+ img, ierr := cropper.Crop(dispatched.JSON[itemIdx])+ if ierr != nil || img == "" {+ return+ }if !isUsableVisionImage(img) {slog.Warn("vision enhancement: invalid image data skipped","item", itemIdx)diff --git a/internal/ingestion/component/vision_enhancement_cgo.go b/internal/ingestion/component/vision_enhancement_cgo.gonew file mode 100644index 0000000000..f38674bddb--- /dev/null+++ b/internal/ingestion/component/vision_enhancement_cgo.go@@ -0,0 +1,160 @@+//go:build cgo++//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "context"+ "image"+ "sync"++ deepdocpdf "ragflow/internal/deepdoc/parser/pdf"+ "ragflow/internal/deepdoc/parser/pdf/util"+ deepdoctype "ragflow/internal/deepdoc/parser/type"+ "ragflow/internal/parser/parser"++ "gorm.io/gorm"+)++// visionSourceFetcher and visionEngineOpener are the injection seams for the+// on-demand crop path. They default to the production storage/engine resolvers+// but are overridable in tests so the cropper can be exercised without a real+// PDF or storage backend.+var (+ visionSourceFetcher func(ctx context.Context, bucket, path string) ([]byte, error)+ visionEngineOpener func(data []byte) (deepdoctype.PDFEngine, error)+)++func init() {+ visionSourceFetcher = FetchBinary+ visionEngineOpener = deepdocpdf.NewEngine+}++// visionPDFCropper crops section images on demand from the source PDF. Parser+// output for PDF no longer carries an inlined base64 image (that is what this+// change removes to bound parser-phase memory); instead each item carries PDF+// positions, and the VLM path re-acquires the source bytes, opens a fresh+// engine, renders the spanned page(s), and crops the region — exactly what the+// chunker does at index time. The engine is opened lazily and at most once per+// vision-enhancement call, then released via Close.+type visionPDFCropper struct {+ ctx context.Context+ db *gorm.DB+ inputs map[string]any++ once sync.Once+ engine deepdoctype.PDFEngine+ engErr error+}++// newVisionImageCropper builds the on-demand cropper. It never touches storage+// here; the source PDF is re-acquired lazily on the first Crop call that needs+// it.+func newVisionImageCropper(ctx context.Context, db *gorm.DB, inputs map[string]any) (visionImageCropper, error) {+ return &visionPDFCropper{ctx: ctx, db: db, inputs: inputs}, nil+}++func (c *visionPDFCropper) Crop(item map[string]any) (string, error) {+ // Fast path: an inlined image (docx/markdown, or any pre-inlined source)+ // is used directly — no storage access, no engine.+ if img, _ := item["image"].(string); img != "" {+ return img, nil+ }+ matrix, ok := parser.ExtractPDFPositions(item)+ if !ok {+ return "", nil+ }+ positions := util.PositionsFromMatrix(matrix)+ if len(positions) == 0 {+ return "", nil+ }+ if err := c.ensureEngine(); err != nil {+ // Best-effort: a missing/unreadable source PDF means no vision+ // description for this item, not a hard failure.+ return "", nil+ }+ if c.engine == nil {+ return "", nil+ }+ // Render each distinct page the positions span (1-based → 0-based is+ // handled by PositionsFromMatrix). Reuse the chunker's sliding-window+ // idea but scope it to this single item: typically one page.+ pages := make(map[int]struct{}, len(positions))+ for _, pos := range positions {+ for _, pn := range pos.PageNumbers {+ pages[pn] = struct{}{}+ }+ }+ single := make(map[int]image.Image, len(pages))+ for pn := range pages {+ img, rerr := deepdocpdf.RenderPageToImage(c.engine, pn)+ if rerr != nil || img == nil {+ continue+ }+ single[pn] = img+ }+ if len(single) == 0 {+ return "", nil+ }+ return util.CropSectionPositions(positions, single, deepdoctype.DlaScale), nil+}++func (c *visionPDFCropper) ensureEngine() error {+ c.once.Do(func() {+ data, err := c.acquireSource()+ if err != nil || len(data) == 0 {+ c.engErr = err+ return+ }+ // Only PDFs can be cropped. Guard against other binary types so a+ // docx/markdown item that happens to reach here stays a no-op.+ if len(data) < 5 || string(data[:5]) != "%PDF-" {+ return+ }+ eng, oerr := visionEngineOpener(data)+ if oerr != nil {+ c.engErr = oerr+ return+ }+ c.engine = eng+ })+ return c.engErr+}++func (c *visionPDFCropper) acquireSource() ([]byte, error) {+ if bucket, _ := getString(c.inputs, "bucket"); bucket != "" {+ if path, _ := getString(c.inputs, "path"); path != "" {+ return visionSourceFetcher(c.ctx, bucket, path)+ }+ }+ if docID, _ := getString(c.inputs, "doc_id"); docID != "" {+ ref, err := ResolveDocumentStorage(c.ctx, c.db, docID)+ if err != nil || ref == nil {+ return nil, err+ }+ return visionSourceFetcher(c.ctx, ref.Bucket, ref.Path)+ }+ return nil, nil+}++func (c *visionPDFCropper) Close() error {+ if c.engine != nil {+ return c.engine.Close()+ }+ return nil+}diff --git a/internal/ingestion/component/vision_enhancement_cgo_test.go b/internal/ingestion/component/vision_enhancement_cgo_test.gonew file mode 100644index 0000000000..6940f473f0--- /dev/null+++ b/internal/ingestion/component/vision_enhancement_cgo_test.go@@ -0,0 +1,206 @@+//go:build cgo++package component++import (+ "context"+ "image"+ "testing"++ deepdoctype "ragflow/internal/deepdoc/parser/type"+)++// mockVisionEngine returns a fixed image for any rendered page so the+// on-demand cropper can exercise CropSectionPositions without a real PDF.+type mockVisionEngine struct {+ closed *bool+}++func (m mockVisionEngine) ExtractChars(int) ([]deepdoctype.TextChar, error) {+ return nil, nil+}+func (m mockVisionEngine) RenderPage(int, float64) ([]byte, error) { return nil, nil }+func (m mockVisionEngine) RenderPageImage(int, float64) (image.Image, error) {+ return image.NewRGBA(image.Rect(0, 0, 1000, 1000)), nil+}+func (m mockVisionEngine) RawData() []byte { return nil }+func (m mockVisionEngine) PageCount() (int, error) { return 1, nil }+func (m mockVisionEngine) Outlines() ([]deepdoctype.Outline, error) { return nil, nil }+func (m mockVisionEngine) Close() error {+ if m.closed != nil {+ *m.closed = true+ }+ return nil+}++func cgoPositions() map[string]any {+ // 1-based page 1 region; PositionsFromMatrix shifts to 0-based.+ return map[string]any{+ "_pdf_positions": [][]any{{1, 10.0, 100.0, 10.0, 100.0}},+ }+}++// TestVisionCropImage_OnDemandFromPositions verifies the core fix: a parsed+// item that carries PDF positions but no inlined image is cropped on demand+// by re-acquiring the source PDF from storage and rendering the region.+func TestVisionCropImage_OnDemandFromPositions(t *testing.T) {+ closed := false+ oldFetcher := visionSourceFetcher+ oldOpener := visionEngineOpener+ defer func() {+ visionSourceFetcher = oldFetcher+ visionEngineOpener = oldOpener+ }()+ visionSourceFetcher = func(ctx context.Context, bucket, path string) ([]byte, error) {+ return []byte("%PDF-fake-engine-bytes"), nil+ }+ visionEngineOpener = func(data []byte) (deepdoctype.PDFEngine, error) {+ return mockVisionEngine{closed: &closed}, nil+ }++ cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{"bucket": "b", "path": "p"})+ if err != nil {+ t.Fatalf("newVisionImageCropper: %v", err)+ }+ defer cropper.Close()++ img, err := cropper.Crop(cgoPositions())+ if err != nil {+ t.Fatalf("Crop: %v", err)+ }+ if img == "" {+ t.Fatal("Crop returned empty; expected an on-demand cropped base64 image")+ }+ if closed {+ t.Fatal("engine closed before use completed")+ }+}++// TestVisionCropImage_PassthroughInline proves that when the item already+// carries an inlined image (docx/markdown, or any pre-inlined source), the+// cropper returns it directly without touching storage.+func TestVisionCropImage_PassthroughInline(t *testing.T) {+ fetchCalled := false+ oldFetcher := visionSourceFetcher+ defer func() { visionSourceFetcher = oldFetcher }()+ visionSourceFetcher = func(ctx context.Context, bucket, path string) ([]byte, error) {+ fetchCalled = true+ return nil, nil+ }++ cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{})+ if err != nil {+ t.Fatalf("newVisionImageCropper: %v", err)+ }+ defer cropper.Close()++ const inline = "data:image/png;base64,preinlined"+ img, err := cropper.Crop(map[string]any{"image": inline})+ if err != nil {+ t.Fatalf("Crop: %v", err)+ }+ if img != inline {+ t.Fatalf("Crop = %q, want passthrough %q", img, inline)+ }+ if fetchCalled {+ t.Fatal("storage fetch must not be called for an inlined image")+ }+}++// TestVisionCropImage_NoImageNoPositions proves an item with neither an image+// nor positions yields no crop (no spurious storage access, no error).+func TestVisionCropImage_NoImageNoPositions(t *testing.T) {+ fetchCalled := false+ oldFetcher := visionSourceFetcher+ defer func() { visionSourceFetcher = oldFetcher }()+ visionSourceFetcher = func(ctx context.Context, bucket, path string) ([]byte, error) {+ fetchCalled = true+ return nil, nil+ }++ cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{})+ if err != nil {+ t.Fatalf("newVisionImageCropper: %v", err)+ }+ defer cropper.Close()++ img, err := cropper.Crop(map[string]any{"doc_type_kwd": "image"})+ if err != nil {+ t.Fatalf("Crop: %v", err)+ }+ if img != "" {+ t.Fatalf("Crop = %q, want empty (no image, no positions)", img)+ }+ if fetchCalled {+ t.Fatal("storage fetch must not be called without positions")+ }+}++// TestVisionCropImage_NonPDFBytesIsNoOp proves the %PDF- guard: if storage+// returns non-PDF bytes (e.g. a docx/markdown mislabeled, or a lookup miss),+// the cropper does not attempt to open an engine and returns no crop.+func TestVisionCropImage_NonPDFBytesIsNoOp(t *testing.T) {+ oldFetcher := visionSourceFetcher+ oldOpener := visionEngineOpener+ defer func() {+ visionSourceFetcher = oldFetcher+ visionEngineOpener = oldOpener+ }()+ visionSourceFetcher = func(ctx context.Context, bucket, path string) ([]byte, error) {+ return []byte("this is not a pdf"), nil+ }+ openerCalled := false+ visionEngineOpener = func(data []byte) (deepdoctype.PDFEngine, error) {+ openerCalled = true+ return mockVisionEngine{}, nil+ }++ cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{"bucket": "b", "path": "p"})+ if err != nil {+ t.Fatalf("newVisionImageCropper: %v", err)+ }+ defer cropper.Close()++ img, err := cropper.Crop(cgoPositions())+ if err != nil {+ t.Fatalf("Crop: %v", err)+ }+ if img != "" {+ t.Fatalf("Crop = %q, want empty for non-PDF bytes", img)+ }+ if openerCalled {+ t.Fatal("engine opener must not run on non-PDF bytes")+ }+}++// TestVisionCropImage_EngineClosedAfterUse proves the re-acquired engine is+// released (no native handle leak) once cropping is done.+func TestVisionCropImage_EngineClosedAfterUse(t *testing.T) {+ closed := false+ oldFetcher := visionSourceFetcher+ oldOpener := visionEngineOpener+ defer func() {+ visionSourceFetcher = oldFetcher+ visionEngineOpener = oldOpener+ }()+ visionSourceFetcher = func(ctx context.Context, bucket, path string) ([]byte, error) {+ return []byte("%PDF-fake"), nil+ }+ visionEngineOpener = func(data []byte) (deepdoctype.PDFEngine, error) {+ return mockVisionEngine{closed: &closed}, nil+ }++ cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{"bucket": "b", "path": "p"})+ if err != nil {+ t.Fatalf("newVisionImageCropper: %v", err)+ }+ if _, err := cropper.Crop(cgoPositions()); err != nil {+ t.Fatalf("Crop: %v", err)+ }+ if err := cropper.Close(); err != nil {+ t.Fatalf("Close: %v", err)+ }+ if !closed {+ t.Fatal("engine should be closed after use")+ }+}diff --git a/internal/ingestion/component/vision_enhancement_nocgo.go b/internal/ingestion/component/vision_enhancement_nocgo.gonew file mode 100644index 0000000000..df3522199d--- /dev/null+++ b/internal/ingestion/component/vision_enhancement_nocgo.go@@ -0,0 +1,45 @@+//go:build !cgo++//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "context"++ "gorm.io/gorm"+)++// visionPDFCropperNoop is the !cgo cropper: without a native renderer there is+// no on-demand crop, so it returns the inlined image when present (docx/+// markdown or any pre-inlined source) and nothing otherwise. The parser does+// not inline PDF media under cgo, and PDF parsing itself requires cgo, so this+// path is only ever exercised for the non-PDF inline forms.+type visionPDFCropperNoop struct{}++func newVisionImageCropper(_ context.Context, _ *gorm.DB, _ map[string]any) (visionImageCropper, error) {+ return visionPDFCropperNoop{}, nil+}++func (visionPDFCropperNoop) Crop(item map[string]any) (string, error) {+ if img, _ := item["image"].(string); img != "" {+ return img, nil+ }+ return "", nil+}++func (visionPDFCropperNoop) Close() error { return nil }diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.goindex 6cc90e03f2..b5de904d3a 100644--- a/internal/parser/parser/pdf_parser_common.go+++ b/internal/parser/parser/pdf_parser_common.go@@ -366,7 +366,13 @@ func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseR}applyPDFPostProcess(&processed, opts)defer processed.Close()- cropMediaSections(&processed)+ // NOTE: PDF media (figure/table) is intentionally NOT cropped here under+ // cgo. The parser no longer inlines base64 images — that is what bounded+ // the parser-phase memory peak. Image/table sections keep only their PDF+ // positions; the chunker re-acquires the source PDF and crops on demand at+ // index time, and the VLM path crops on demand when it needs to describe a+ // figure/table. The Markdown path below still inlines, because markdown+ // output embeds images directly and has no downstream on-demand consumer.items := pdflayout.SectionsToJSON(processed.Sections)if len(items) == 0 {@@ -788,14 +794,16 @@ func normalizePDFDocType(item map[string]any) {return}layoutType, _ := item["layout_type"].(string)+ // A figure caption is a media section even when the parser no longer+ // inlines its cropped image (cgo): it still carries PDF positions, so the+ // downstream VLM/chunker crop it on demand. Classify it as image whenever+ // it has positions (the inlined image was only a side effect of cropping).+ _, hasMedia := ExtractPDFPositions(item)if docType, _ := item["doc_type_kwd"].(string); docType != "" {- // A figure caption can carry the cropped figure image after PDF media- // sections are rendered. Keep it aligned with Python's media-section- // contract so the downstream VLM enhancement can process it.- if docType == "text" && layoutType == deepdoctype.DLALabelFigureCaption {- if img, _ := item["image"].(string); img != "" {- item["doc_type_kwd"] = "image"- }+ // A figure caption keeps its media classification so the downstream+ // VLM enhancement and on-demand chunker crop it.+ if docType == "text" && layoutType == deepdoctype.DLALabelFigureCaption && hasMedia {+ item["doc_type_kwd"] = "image"}return}@@ -804,6 +812,12 @@ func normalizePDFDocType(item map[string]any) {item["doc_type_kwd"] = "table"case "figure", "image":item["doc_type_kwd"] = "image"+ case deepdoctype.DLALabelFigureCaption:+ if hasMedia {+ item["doc_type_kwd"] = "image"+ } else {+ item["doc_type_kwd"] = "text"+ }default:if img, _ := item["image"].(string); img != "" {item["doc_type_kwd"] = "image"@@ -813,6 +827,45 @@ func normalizePDFDocType(item map[string]any) {}}+// ExtractPDFPositions is the single source of truth for "does this parsed item+// carry a usable PDF crop region". It returns the non-empty positions matrix+// (under the canonical _pdf_positions key or the legacy positions key),+// accepting either the typed [][]any form produced in-process or the+// JSON-decoded []any form whose elements are themselves []any rows. The bool+// reports whether a usable matrix was present.+//+// The parser (doc-type classification in normalizePDFDocType) and the on-demand+// croppers (VLM vision_enhancement and the chunker) MUST agree on this+// contract, so all call sites delegate here instead of re-implementing the+// check. A loose "non-nil" test previously misclassified items whose positions+// were empty or not actually a matrix (e.g. an empty []any or a stray scalar),+// which escaped the figure-caption → image normalization and the crop path.+func ExtractPDFPositions(item map[string]any) ([][]any, bool) {+ for _, key := range []string{"_pdf_positions", "positions"} {+ switch v := item[key].(type) {+ case [][]any:+ if len(v) > 0 {+ return v, true+ }+ case []any:+ out := make([][]any, 0, len(v))+ ok := true+ for _, row := range v {+ r, rOK := row.([]any)+ if !rOK {+ ok = false+ break+ }+ out = append(out, r)+ }+ if ok && len(out) > 0 {+ return out, true+ }+ }+ }+ return nil, false+}+func parsePDFWithDeepDoc(ctx context.Context, filename string, data []byte, parseFn func(context.Context, []byte, deepdoctype.DocAnalyzer) (*deepdoctype.ParseResult, error)) ParseResult {return parsePDFWithDeepDocOptions(ctx, filename, data, pdfPostProcessOptions{}, parseFn)}diff --git a/internal/parser/parser/pdf_parser_common_test.go b/internal/parser/parser/pdf_parser_common_test.goindex 9ceed111f0..be57c5674b 100644--- a/internal/parser/parser/pdf_parser_common_test.go+++ b/internal/parser/parser/pdf_parser_common_test.go@@ -101,23 +101,55 @@ func TestPDFParseResultToJSON_NormalizesCoreFields(t *testing.T) {}}+// TestPDFParseResultToJSON_ClassifiesFigureCaptionWithImage pins the+// positions-driven contract for figure-caption classification. The parser's+// JSON path no longer inlines cropped media, so a figure caption is promoted to+// doc_type_kwd "image" only when it carries a usable PDF positions matrix+// (the on-demand crop source) — NOT merely because an image is present. This+// locks down the fix for the previous loose `v != nil` check, which promoted+// captions whose _pdf_positions were empty or not a real matrix to "image".func TestPDFParseResultToJSON_ClassifiesFigureCaptionWithImage(t *testing.T) {- parsed := &deepdoctype.ParseResult{- Sections: []deepdoctype.Section{- {- Text: "小灰灰",- LayoutType: deepdoctype.DLALabelFigureCaption,- Image: "aGVsbG8=",- },- },+ // A figure caption WITH positions is classified as image: the on-demand+ // VLM/chunker crop path fires.+ withPos := &deepdoctype.ParseResult{+ Sections: []deepdoctype.Section{{+ Text: "小灰灰",+ LayoutType: deepdoctype.DLALabelFigureCaption,+ Positions: []deepdoctype.Position{{+ PageNumbers: []int{0},+ Left: 10,+ Right: 50,+ Top: 10,+ Bottom: 50,+ }},+ }},}-- res := pdfParseResultToJSON("sample.pdf", parsed)+ res := pdfParseResultToJSON("with-pos.pdf", withPos)if res.Err != nil {t.Fatalf("pdfParseResultToJSON: %v", res.Err)}- if got, want := res.JSON[0]["doc_type_kwd"], "image"; got != want {- t.Fatalf("doc_type_kwd = %v, want %v", got, want)+ if got := res.JSON[0]["doc_type_kwd"]; got != "image" {+ t.Fatalf("figure caption with positions doc_type_kwd = %v, want image", got)+ }++ // A figure caption with an inlined image but NO positions is NOT promoted+ // to image: positions are the sole classifier now. (In the PDF JSON path+ // an inlined image without positions does not occur — cropMediaSections was+ // removed — so this documents that the presence of a stray image alone is+ // insufficient.)+ withImageNoPos := &deepdoctype.ParseResult{+ Sections: []deepdoctype.Section{{+ Text: "小灰灰",+ LayoutType: deepdoctype.DLALabelFigureCaption,+ Image: "aGVsbG8=",+ }},+ }+ res2 := pdfParseResultToJSON("with-image-no-pos.pdf", withImageNoPos)+ if res2.Err != nil {+ t.Fatalf("pdfParseResultToJSON: %v", res2.Err)+ }+ if got := res2.JSON[0]["doc_type_kwd"]; got != "text" {+ t.Fatalf("figure caption with image but no positions doc_type_kwd = %v, want text", got)}}@@ -465,7 +497,12 @@ func (m *mockPDFEngineForCommonTest) Close() error {return nil}-func TestPDFParseResultToJSON_CropsMediaSectionsWithEngine(t *testing.T) {+// TestPDFParseResultToJSON_NoInlineRetainsPositions pins the new cgo contract:+// the parser no longer inlines base64 media into the JSON items (that bounded+// the parser-phase memory peak). Figure/table sections keep their PDF+// positions so the chunker and VLM can crop on demand. The crop logic itself+// is still exercised by the Markdown inline tests.+func TestPDFParseResultToJSON_NoInlineRetainsPositions(t *testing.T) {mockEngine := &mockPDFEngineForCommonTest{}parsed := &deepdoctype.ParseResult{Engine: mockEngine,@@ -509,32 +546,34 @@ func TestPDFParseResultToJSON_CropsMediaSectionsWithEngine(t *testing.T) {t.Fatalf("JSON len = %d, want 3", len(res.JSON))}- // Figure should have cropped image populated- figImg, ok := res.JSON[0]["image"].(string)- if !ok || figImg == "" {- t.Fatalf("Figure JSON[0].image should be non-empty base64 data url, got %v", res.JSON[0]["image"])+ // Figure must retain positions but NOT inline an image.+ figPos, ok := res.JSON[0]["_pdf_positions"].([][]any)+ if !ok || len(figPos) == 0 {+ t.Fatalf("Figure should retain _pdf_positions, got %v", res.JSON[0]["_pdf_positions"])}- if !strings.HasPrefix(figImg, "data:image/png;base64,") {- t.Fatalf("Figure JSON[0].image prefix mismatch, got %q", figImg)+ if img, _ := res.JSON[0]["image"].(string); img != "" {+ t.Fatalf("Figure should not inline image under cgo, got %q", img)}- // Table should have cropped image populated- tblImg, ok := res.JSON[1]["image"].(string)- if !ok || tblImg == "" {- t.Fatalf("Table JSON[1].image should be non-empty base64 data url, got %v", res.JSON[1]["image"])+ // Table likewise retains positions, no inline image.+ tblPos, ok := res.JSON[1]["_pdf_positions"].([][]any)+ if !ok || len(tblPos) == 0 {+ t.Fatalf("Table should retain _pdf_positions, got %v", res.JSON[1]["_pdf_positions"])}- if !strings.HasPrefix(tblImg, "data:image/png;base64,") {- t.Fatalf("Table JSON[1].image prefix mismatch, got %q", tblImg)+ if img, _ := res.JSON[1]["image"].(string); img != "" {+ t.Fatalf("Table should not inline image under cgo, got %q", img)}- // Plain text should NOT have cropped image- textImg, _ := res.JSON[2]["image"].(string)- if textImg != "" {- t.Fatalf("Text JSON[2].image should be empty, got %q", textImg)+ // Plain text: no image, positions retained.+ if img, _ := res.JSON[2]["image"].(string); img != "" {+ t.Fatalf("Text should not inline image, got %q", img)}}-func TestPDFParseResultToJSON_CropsFigureCaptionForVisionEnhancement(t *testing.T) {+// TestPDFParseResultToJSON_FigureCaptionNoInline pins that a figure caption is+// still classified as doc_type_kwd "image" (so the chunker/VLM crop it on+// demand) but the parser does not inline the cropped image for it.+func TestPDFParseResultToJSON_FigureCaptionNoInline(t *testing.T) {mockEngine := &mockPDFEngineForCommonTest{}parsed := &deepdoctype.ParseResult{Engine: mockEngine,@@ -559,8 +598,135 @@ func TestPDFParseResultToJSON_CropsFigureCaptionForVisionEnhancement(t *testing.if got, want := res.JSON[0]["doc_type_kwd"], "image"; got != want {t.Fatalf("doc_type_kwd = %v, want %v", got, want)}- if image, _ := res.JSON[0]["image"].(string); image == "" {- t.Fatal("figure caption image should be cropped before vision enhancement")+ if pos, ok := res.JSON[0]["_pdf_positions"].([][]any); !ok || len(pos) == 0 {+ t.Fatal("figure caption should retain _pdf_positions for on-demand crop")+ }+ if image, _ := res.JSON[0]["image"].(string); image != "" {+ t.Fatalf("figure caption should not be inlined by the parser, got %q", image)+ }+}++// TestExtractPDFPositions locks down the single source of truth for "does this+// item carry a usable PDF crop region". The historical bug was a loose+// `v != nil` test that accepted empty slices and stray scalars; this test+// proves the contract is now strict (non-empty matrix only) and that both the+// canonical _pdf_positions key and the legacy positions key are honored, in+// both the typed [][]any form and the JSON-decoded []any form.+func TestExtractPDFPositions(t *testing.T) {+ cases := []struct {+ name string+ item map[string]any+ want bool+ rows int+ key string+ }{+ {+ name: "typed non-empty matrix under _pdf_positions",+ item: map[string]any{"_pdf_positions": [][]any{{float64(1), float64(2), float64(3), float64(4), float64(1)}}},+ want: true, rows: 1, key: "_pdf_positions",+ },+ {+ name: "json-decoded []any rows under _pdf_positions",+ item: map[string]any{"_pdf_positions": []any{[]any{float64(1), float64(2), float64(3), float64(4), float64(1)}}},+ want: true, rows: 1, key: "_pdf_positions",+ },+ {+ name: "legacy positions key honored",+ item: map[string]any{"positions": [][]any{{float64(1), float64(2), float64(3), float64(4), float64(1)}}},+ want: true, rows: 1, key: "positions",+ },+ {+ name: "empty typed matrix rejected",+ item: map[string]any{"_pdf_positions": [][]any{}},+ want: false,+ },+ {+ name: "empty json-decoded matrix rejected",+ item: map[string]any{"_pdf_positions": []any{}},+ want: false,+ },+ {+ name: "nil value rejected",+ item: map[string]any{"_pdf_positions": nil},+ want: false,+ },+ {+ name: "missing key rejected",+ item: map[string]any{"doc_type_kwd": "image"},+ want: false,+ },+ {+ name: "stray scalar rejected (old v != nil bug)",+ item: map[string]any{"_pdf_positions": "not-a-matrix"},+ want: false,+ },+ {+ name: "non-slice numeric rejected",+ item: map[string]any{"_pdf_positions": float64(42)},+ want: false,+ },+ {+ name: "[]any with non-row element rejected",+ item: map[string]any{"_pdf_positions": []any{"bad", float64(1)}},+ want: false,+ },+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ matrix, ok := ExtractPDFPositions(tc.item)+ if ok != tc.want {+ t.Fatalf("ExtractPDFPositions ok = %v, want %v", ok, tc.want)+ }+ if tc.want {+ if len(matrix) != tc.rows {+ t.Fatalf("matrix rows = %d, want %d", len(matrix), tc.rows)+ }+ // Every returned row must be []any (normalized form).+ for i, row := range matrix {+ if row == nil {+ t.Fatalf("matrix[%d] is nil after normalization", i)+ }+ }+ // The honored key must carry the matrix in normalized [][]any.+ if _, present := tc.item[tc.key]; !present {+ t.Fatalf("expected honored key %q missing", tc.key)+ }+ }+ })+ }+}++// TestNormalizePDFDocType_FigureCaptionPositionsGate proves Finding B: a figure+// caption is only promoted to doc_type_kwd "image" (which lights up the+// on-demand VLM/chunker crop) when it actually carries a usable positions+// matrix. The previous loose `v != nil` check promoted captions whose+// _pdf_positions were empty or not a real matrix, wrongly flagging them for+// cropping. normalizePDFDocType must delegate to ExtractPDFPositions.+func TestNormalizePDFDocType_FigureCaptionPositionsGate(t *testing.T) {+ withPos := map[string]any{+ "layout_type": deepdoctype.DLALabelFigureCaption,+ "_pdf_positions": [][]any{{float64(1), float64(2), float64(3), float64(4), float64(1)}},+ }+ normalizePDFDocType(withPos)+ if got := withPos["doc_type_kwd"]; got != "image" {+ t.Fatalf("with positions doc_type_kwd = %v, want image", got)+ }++ for _, empty := range []any{+ []any{},+ [][]any{},+ nil,+ "stray-scalar",+ } {+ withoutPos := map[string]any{+ "layout_type": deepdoctype.DLALabelFigureCaption,+ "_pdf_positions": empty,+ }+ normalizePDFDocType(withoutPos)+ if got := withoutPos["doc_type_kwd"]; got != "text" {+ t.Fatalf("_pdf_positions=%#v doc_type_kwd = %v, want text (must not promote empty/non-matrix)", empty, got)+ }}}