Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): prevent PDF child chunks from overwriting each other (#21773)

jay777211191d500a6cc
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index fa47e951aa..da99a68ae7 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -271,6 +271,12 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
}
flushBody()
+ for i := range chunks {
+ if itemDocType(chunks[i]) == "text" {
+ chunks[i].Text = removeTag(chunks[i].Text)
+ }
+ }
+ chunks = splitGeneralChildren(chunks, childrenPattern)
engine, err := newPDFEngineFromUpstream(ctx, db, upstream)
if err != nil {
common.Warn("GeneralChunker: could not open PDF for on-demand cropping", zap.Error(err))
@@ -279,7 +285,7 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
defer engine.Close()
chunks = cropImageChunks(ctx, engine, chunks)
}
- chunks = finalizeGeneralChunks(chunks, childrenPattern)
+ chunks = finalizeGeneralChunks(chunks, nil)
if err := ctx.Err(); err != nil {
return nil, fmt.Errorf("GeneralChunker: %w", err)
}
@@ -1459,11 +1465,6 @@ func finalizeGeneralChunks(chunks []schema.ChunkDoc, childrenPattern *regexp.Reg
visible := make([]schema.ChunkDoc, 0, len(chunks))
for _, chunk := range chunks {
chunk.Text = removeTag(chunk.Text)
- // Keep a chunk whenever it still carries retrievable content. A
- // streamed crop upload sets ImgID and clears Image, so an uploaded
- // media chunk (text may be only a position tag, or empty) must be
- // retained via its ImgID — otherwise the already-uploaded image is
- // dropped and its MinIO object orphaned.
if strings.TrimSpace(chunk.Text) == "" && !hasMediaPayload(chunk) {
continue
}
diff --git a/internal/ingestion/component/chunker/general_pdf_children_test.go b/internal/ingestion/component/chunker/general_pdf_children_test.go
new file mode 100644
index 0000000000..fc20c2af01
--- /dev/null
+++ b/internal/ingestion/component/chunker/general_pdf_children_test.go
@@ -0,0 +1,186 @@
+//go:build cgo
+
+package chunker
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "image/color"
+ "image/png"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/ingestion/component"
+ "ragflow/internal/ingestion/task/indexdoc"
+ "ragflow/internal/storage"
+)
+
+func TestGeneralPDFChildren(t *testing.T) {
+ // Red, green and blue markers occupy the alpha, beta and gamma regions.
+ data := newGeneralPDFChildPreviewFixture(t)
+ for _, tc := range []struct {
+ name string
+ children []string
+ want []string
+ failUpload bool
+ }{
+ {"children", []string{"#"}, []string{"alpha#", "beta#", "gamma"}, false},
+ {"no children", nil, []string{"alpha#beta#gamma"}, false},
+ {"retry upload", []string{"#"}, []string{"alpha#", "beta#", "gamma"}, true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ ctx := withIngestionGlobals(t, "preview-test", "doc-1")
+ store := storage.NewMemoryStorage()
+ if err := store.Put(ctx, "preview-test", "source.pdf", data); err != nil {
+ t.Fatal(err)
+ }
+ previousStorage, previousUploader := component.SetStorageFactoryOverride, ChunkImageUploader
+ component.SetStorageFactoryOverride = func() storage.Storage { return store }
+ ChunkImageUploader = component.DefaultImageUploader
+ if tc.failUpload {
+ var failed atomic.Bool
+ ChunkImageUploader = func(ctx context.Context, kbID, chunkID string, data []byte) (string, error) {
+ if failed.CompareAndSwap(false, true) {
+ return "", errors.New("temporary upload failure")
+ }
+ return component.DefaultImageUploader(ctx, kbID, chunkID, data)
+ }
+ }
+ t.Cleanup(func() {
+ component.SetStorageFactoryOverride = previousStorage
+ ChunkImageUploader = previousUploader
+ })
+ inputs := map[string]any{
+ "name": "source.pdf", "file_type": "pdf", "output_format": "json",
+ "bucket": "preview-test", "path": "source.pdf",
+ "json": []map[string]any{{
+ "text": "alpha#beta#gamma", "doc_type_kwd": "text",
+ "positions": [][]float64{{1, 90, 400, 200, 500}},
+ }},
+ }
+ run := func(children []string) []map[string]any {
+ t.Helper()
+ inner, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 512,
+ "delimiters": []string{"#"},
+ "children_delimiters": children,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ chunker := &imageUploadDecorator{inner: inner}
+ out, err := chunker.Invoke(ctx, nil, inputs)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return outputChunks(t, out)
+ }
+ chunks := run(tc.children)
+ if len(chunks) != len(tc.want) {
+ t.Fatalf("got %d chunks, want %d: %#v", len(chunks), len(tc.want), chunks)
+ }
+ heights := make([]int, len(chunks))
+ for i, chunk := range chunks {
+ text, _ := chunk["text"].(string)
+ if text != tc.want[i] {
+ t.Fatalf("chunk %d text = %q, want %q", i, text, tc.want[i])
+ }
+ id, _ := chunk["id"].(string)
+ if id != common.ChunkID("doc-1", text) {
+ t.Errorf("chunk %d has stale id %q for %q", i, id, text)
+ }
+ if chunk["img_id"] != "preview-test-"+id {
+ t.Errorf("chunk %d image reference = %v", i, chunk["img_id"])
+ }
+ if chunk["image"] != nil {
+ t.Errorf("chunk %d retains uploaded image bytes", i)
+ }
+ preview, err := store.Get(ctx, "preview-test", id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ img, err := png.Decode(bytes.NewReader(preview))
+ if err != nil {
+ t.Fatal(err)
+ }
+ heights[i] = img.Bounds().Dy()
+ markers := []color.RGBA{{255, 0, 0, 255}, {0, 255, 0, 255}, {0, 0, 255, 255}}
+ found := make(map[color.RGBA]bool)
+ for y := img.Bounds().Min.Y; y < img.Bounds().Max.Y; y++ {
+ for x := img.Bounds().Min.X; x < img.Bounds().Max.X; x++ {
+ found[color.RGBAModel.Convert(img.At(x, y)).(color.RGBA)] = true
+ }
+ }
+ for marker, pixel := range markers {
+ want := len(tc.children) == 0 || marker == i
+ if found[pixel] != want {
+ t.Errorf("child %d undimmed marker %d present = %v, want %v", i, marker, found[pixel], want)
+ }
+ }
+ }
+ if objects := store.(*storage.MemoryStorage).Inspect(); len(objects) != len(tc.want)+1 {
+ t.Errorf("stored %d objects, want source PDF plus %d child previews", len(objects), len(tc.want))
+ }
+ if _, err := indexdoc.ProcessChunksForPipeline(chunks, "doc-1", "source.pdf", time.Unix(0, 0)); err != nil {
+ t.Fatal(err)
+ }
+ parents := indexdoc.MaterializeParentChunks("preview-test", chunks)
+ wantParents := 0
+ if len(tc.children) > 0 {
+ wantParents = 1
+ }
+ if len(parents) != wantParents {
+ t.Fatalf("parent rows = %d, want %d", len(parents), wantParents)
+ }
+ rowsByID := make(map[string]map[string]any)
+ for _, row := range append(parents, chunks...) {
+ id, _ := row["id"].(string)
+ if id == "" || rowsByID[id] != nil {
+ t.Fatalf("parent/child rows have empty or duplicate ID %q", id)
+ }
+ rowsByID[id] = row
+ }
+ for i, chunk := range chunks {
+ if chunk["id"] != common.ChunkID("doc-1", tc.want[i]) || chunk["content_with_weight"] != tc.want[i] {
+ t.Errorf("child %d identity or text changed during index normalization: %#v", i, chunk)
+ }
+ if wantParents == 0 {
+ if chunk["mom_id"] != nil {
+ t.Errorf("unsplit chunk has unexpected mom_id %v", chunk["mom_id"])
+ }
+ continue
+ }
+ if chunk["mom_id"] != parents[0]["id"] {
+ t.Errorf("child %d mom_id = %v, want %v", i, chunk["mom_id"], parents[0]["id"])
+ }
+ }
+ if wantParents > 0 {
+ if parents[0]["content_with_weight"] != "alpha#beta#gamma" || parents[0]["available_int"] != 0 {
+ t.Errorf("parent must retain the full text as a hidden row: %#v", parents[0])
+ }
+ }
+ if len(chunks) > 1 {
+ parent := run(nil)
+ if len(parent) != 1 {
+ t.Fatalf("unsplit chunks = %d, want 1", len(parent))
+ }
+ preview, err := store.Get(ctx, "preview-test", parent[0]["id"].(string))
+ if err != nil {
+ t.Fatal(err)
+ }
+ img, err := png.DecodeConfig(bytes.NewReader(preview))
+ if err != nil {
+ t.Fatal(err)
+ }
+ for i, height := range heights {
+ if height >= img.Height {
+ t.Errorf("chunk %d preview height = %d, want smaller than unsplit preview %d", i, height, img.Height)
+ }
+ }
+ }
+ })
+ }
+}
diff --git a/internal/ingestion/component/chunker/general_pdf_fixture_test.go b/internal/ingestion/component/chunker/general_pdf_fixture_test.go
new file mode 100644
index 0000000000..b2a2a13842
--- /dev/null
+++ b/internal/ingestion/component/chunker/general_pdf_fixture_test.go
@@ -0,0 +1,52 @@
+//go:build cgo
+
+package chunker
+
+import (
+ "bytes"
+ "compress/zlib"
+ "fmt"
+ "testing"
+)
+
+// newGeneralPDFChildPreviewFixture builds a one-page PDF with three solid
+// markers so preview tests do not need an opaque binary fixture.
+func newGeneralPDFChildPreviewFixture(t *testing.T) []byte {
+ t.Helper()
+
+ content := []byte(
+ "1 0 0 rg\n110 582 20 20 re f\n" +
+ "0 1 0 rg\n110 472 20 20 re f\n" +
+ "0 0 1 rg\n110 372 20 20 re f\n",
+ )
+ var compressed bytes.Buffer
+ writer := zlib.NewWriter(&compressed)
+ if _, err := writer.Write(content); err != nil {
+ t.Fatal(err)
+ }
+ if err := writer.Close(); err != nil {
+ t.Fatal(err)
+ }
+
+ var pdf bytes.Buffer
+ pdf.WriteString("%PDF-1.4\n%\xe2\xe3\xcf\xd3\n")
+ offsets := make([]int, 5)
+ writeObject := func(number int, body string) {
+ t.Helper()
+ offsets[number] = pdf.Len()
+ fmt.Fprintf(&pdf, "%d 0 obj\n%s\nendobj\n", number, body)
+ }
+ writeObject(1, "<< /Type /Catalog /Pages 2 0 R >>")
+ writeObject(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>")
+ writeObject(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 842] /Resources << >> /Contents 4 0 R >>")
+ writeObject(4, fmt.Sprintf("<< /Length %d /Filter /FlateDecode >>\nstream\n%s\nendstream", compressed.Len(), compressed.Bytes()))
+
+ xref := pdf.Len()
+ pdf.WriteString("xref\n0 5\n")
+ pdf.WriteString("0000000000 65535 f \n")
+ for number := 1; number < len(offsets); number++ {
+ fmt.Fprintf(&pdf, "%010d 00000 n \n", offsets[number])
+ }
+ fmt.Fprintf(&pdf, "trailer\n<< /Size 5 /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", xref)
+ return pdf.Bytes()
+}
diff --git a/internal/ingestion/component/chunker/general_pdf_media_test.go b/internal/ingestion/component/chunker/general_pdf_media_test.go
new file mode 100644
index 0000000000..0e4bd3ae47
--- /dev/null
+++ b/internal/ingestion/component/chunker/general_pdf_media_test.go
@@ -0,0 +1,108 @@
+//go:build cgo
+
+package chunker
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "image/png"
+ "sync/atomic"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/ingestion/component"
+ "ragflow/internal/storage"
+)
+
+func TestGeneralPDFContextOnlyMedia(t *testing.T) {
+ data := newGeneralPDFChildPreviewFixture(t)
+ for _, kind := range []string{"image", "table"} {
+ for _, scenario := range []string{"crop", "retry upload", "no context", "no source", "missing source", "invalid pdf", "missing page"} {
+ t.Run(kind+"/"+scenario, func(t *testing.T) {
+ ctx := withIngestionGlobals(t, "context-test", "doc-1")
+ store := storage.NewMemoryStorage()
+ previousStorage, previousUploader := component.SetStorageFactoryOverride, ChunkImageUploader
+ component.SetStorageFactoryOverride = func() storage.Storage { return store }
+ ChunkImageUploader = component.DefaultImageUploader
+ t.Cleanup(func() {
+ component.SetStorageFactoryOverride = previousStorage
+ ChunkImageUploader = previousUploader
+ })
+ if scenario == "retry upload" {
+ var failed atomic.Bool
+ ChunkImageUploader = func(ctx context.Context, kbID, chunkID string, data []byte) (string, error) {
+ if chunkID != common.ChunkID("doc-1", "before.") && failed.CompareAndSwap(false, true) {
+ return "", errors.New("temporary upload failure")
+ }
+ return component.DefaultImageUploader(ctx, kbID, chunkID, data)
+ }
+ }
+ page := 1
+ if scenario == "missing page" {
+ page = 99
+ }
+ inputs := map[string]any{
+ "name": "source.pdf", "file_type": "pdf", "output_format": "json",
+ "json": []map[string]any{
+ {"text": "before.", "doc_type_kwd": "text", "positions": [][]int{{1, 90, 400, 100, 120}}},
+ {"text": "", "doc_type_kwd": kind, "positions": [][]int{{page, 90, 400, 200, 300}}},
+ },
+ }
+ if scenario != "no source" {
+ inputs["bucket"], inputs["path"] = "context-test", "source.pdf"
+ if scenario != "missing source" {
+ pdf := data
+ if scenario == "invalid pdf" {
+ pdf = []byte("%PDF-invalid")
+ }
+ if err := store.Put(ctx, "context-test", "source.pdf", pdf); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ contextSize := 64
+ if scenario == "no context" {
+ contextSize = 0
+ }
+ inner, err := NewGeneralChunker(map[string]any{"image_context_size": contextSize, "table_context_size": contextSize})
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := (&imageUploadDecorator{inner: inner}).Invoke(ctx, nil, inputs)
+ if err != nil {
+ t.Fatal(err)
+ }
+ chunks := outputChunks(t, out)
+ want := 1
+ if scenario == "crop" || scenario == "retry upload" {
+ want = 2
+ }
+ if len(chunks) != want {
+ t.Fatalf("chunks = %d, want %d: %#v", len(chunks), want, chunks)
+ }
+ if want == 1 {
+ if chunks[0]["text"] != "before." || chunks[0]["doc_type_kwd"] != "text" {
+ t.Fatalf("unexpected remaining text: %#v", chunks)
+ }
+ return
+ }
+ media := chunks[1]
+ if media["doc_type_kwd"] != kind || media["text"] != "before." {
+ t.Errorf("media lost its type or context: %#v", media)
+ }
+ id, _ := media["id"].(string)
+ if media["img_id"] != "context-test-"+id || media["image"] != nil {
+ t.Errorf("media was not uploaded correctly: %#v", media)
+ }
+ preview, err := store.Get(ctx, "context-test", id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := png.DecodeConfig(bytes.NewReader(preview)); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+ }
+}