RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): prevent PDF child chunks from overwriting each other (#21773)
jay777211191d500a6cc
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex fa47e951aa..da99a68ae7 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -271,6 +271,12 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups}flushBody()+ for i := range chunks {+ if itemDocType(chunks[i]) == "text" {+ chunks[i].Text = removeTag(chunks[i].Text)+ }+ }+ chunks = splitGeneralChildren(chunks, childrenPattern)engine, err := newPDFEngineFromUpstream(ctx, db, upstream)if err != nil {common.Warn("GeneralChunker: could not open PDF for on-demand cropping", zap.Error(err))@@ -279,7 +285,7 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, upsdefer engine.Close()chunks = cropImageChunks(ctx, engine, chunks)}- chunks = finalizeGeneralChunks(chunks, childrenPattern)+ chunks = finalizeGeneralChunks(chunks, nil)if err := ctx.Err(); err != nil {return nil, fmt.Errorf("GeneralChunker: %w", err)}@@ -1459,11 +1465,6 @@ func finalizeGeneralChunks(chunks []schema.ChunkDoc, childrenPattern *regexp.Regvisible := make([]schema.ChunkDoc, 0, len(chunks))for _, chunk := range chunks {chunk.Text = removeTag(chunk.Text)- // Keep a chunk whenever it still carries retrievable content. A- // streamed crop upload sets ImgID and clears Image, so an uploaded- // media chunk (text may be only a position tag, or empty) must be- // retained via its ImgID — otherwise the already-uploaded image is- // dropped and its MinIO object orphaned.if strings.TrimSpace(chunk.Text) == "" && !hasMediaPayload(chunk) {continue}diff --git a/internal/ingestion/component/chunker/general_pdf_children_test.go b/internal/ingestion/component/chunker/general_pdf_children_test.gonew file mode 100644index 0000000000..fc20c2af01--- /dev/null+++ b/internal/ingestion/component/chunker/general_pdf_children_test.go@@ -0,0 +1,186 @@+//go:build cgo++package chunker++import (+ "bytes"+ "context"+ "errors"+ "image/color"+ "image/png"+ "sync/atomic"+ "testing"+ "time"++ "ragflow/internal/common"+ "ragflow/internal/ingestion/component"+ "ragflow/internal/ingestion/task/indexdoc"+ "ragflow/internal/storage"+)++func TestGeneralPDFChildren(t *testing.T) {+ // Red, green and blue markers occupy the alpha, beta and gamma regions.+ data := newGeneralPDFChildPreviewFixture(t)+ for _, tc := range []struct {+ name string+ children []string+ want []string+ failUpload bool+ }{+ {"children", []string{"#"}, []string{"alpha#", "beta#", "gamma"}, false},+ {"no children", nil, []string{"alpha#beta#gamma"}, false},+ {"retry upload", []string{"#"}, []string{"alpha#", "beta#", "gamma"}, true},+ } {+ t.Run(tc.name, func(t *testing.T) {+ ctx := withIngestionGlobals(t, "preview-test", "doc-1")+ store := storage.NewMemoryStorage()+ if err := store.Put(ctx, "preview-test", "source.pdf", data); err != nil {+ t.Fatal(err)+ }+ previousStorage, previousUploader := component.SetStorageFactoryOverride, ChunkImageUploader+ component.SetStorageFactoryOverride = func() storage.Storage { return store }+ ChunkImageUploader = component.DefaultImageUploader+ if tc.failUpload {+ var failed atomic.Bool+ ChunkImageUploader = func(ctx context.Context, kbID, chunkID string, data []byte) (string, error) {+ if failed.CompareAndSwap(false, true) {+ return "", errors.New("temporary upload failure")+ }+ return component.DefaultImageUploader(ctx, kbID, chunkID, data)+ }+ }+ t.Cleanup(func() {+ component.SetStorageFactoryOverride = previousStorage+ ChunkImageUploader = previousUploader+ })+ inputs := map[string]any{+ "name": "source.pdf", "file_type": "pdf", "output_format": "json",+ "bucket": "preview-test", "path": "source.pdf",+ "json": []map[string]any{{+ "text": "alpha#beta#gamma", "doc_type_kwd": "text",+ "positions": [][]float64{{1, 90, 400, 200, 500}},+ }},+ }+ run := func(children []string) []map[string]any {+ t.Helper()+ inner, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 512,+ "delimiters": []string{"#"},+ "children_delimiters": children,+ })+ if err != nil {+ t.Fatal(err)+ }+ chunker := &imageUploadDecorator{inner: inner}+ out, err := chunker.Invoke(ctx, nil, inputs)+ if err != nil {+ t.Fatal(err)+ }+ return outputChunks(t, out)+ }+ chunks := run(tc.children)+ if len(chunks) != len(tc.want) {+ t.Fatalf("got %d chunks, want %d: %#v", len(chunks), len(tc.want), chunks)+ }+ heights := make([]int, len(chunks))+ for i, chunk := range chunks {+ text, _ := chunk["text"].(string)+ if text != tc.want[i] {+ t.Fatalf("chunk %d text = %q, want %q", i, text, tc.want[i])+ }+ id, _ := chunk["id"].(string)+ if id != common.ChunkID("doc-1", text) {+ t.Errorf("chunk %d has stale id %q for %q", i, id, text)+ }+ if chunk["img_id"] != "preview-test-"+id {+ t.Errorf("chunk %d image reference = %v", i, chunk["img_id"])+ }+ if chunk["image"] != nil {+ t.Errorf("chunk %d retains uploaded image bytes", i)+ }+ preview, err := store.Get(ctx, "preview-test", id)+ if err != nil {+ t.Fatal(err)+ }+ img, err := png.Decode(bytes.NewReader(preview))+ if err != nil {+ t.Fatal(err)+ }+ heights[i] = img.Bounds().Dy()+ markers := []color.RGBA{{255, 0, 0, 255}, {0, 255, 0, 255}, {0, 0, 255, 255}}+ found := make(map[color.RGBA]bool)+ for y := img.Bounds().Min.Y; y < img.Bounds().Max.Y; y++ {+ for x := img.Bounds().Min.X; x < img.Bounds().Max.X; x++ {+ found[color.RGBAModel.Convert(img.At(x, y)).(color.RGBA)] = true+ }+ }+ for marker, pixel := range markers {+ want := len(tc.children) == 0 || marker == i+ if found[pixel] != want {+ t.Errorf("child %d undimmed marker %d present = %v, want %v", i, marker, found[pixel], want)+ }+ }+ }+ if objects := store.(*storage.MemoryStorage).Inspect(); len(objects) != len(tc.want)+1 {+ t.Errorf("stored %d objects, want source PDF plus %d child previews", len(objects), len(tc.want))+ }+ if _, err := indexdoc.ProcessChunksForPipeline(chunks, "doc-1", "source.pdf", time.Unix(0, 0)); err != nil {+ t.Fatal(err)+ }+ parents := indexdoc.MaterializeParentChunks("preview-test", chunks)+ wantParents := 0+ if len(tc.children) > 0 {+ wantParents = 1+ }+ if len(parents) != wantParents {+ t.Fatalf("parent rows = %d, want %d", len(parents), wantParents)+ }+ rowsByID := make(map[string]map[string]any)+ for _, row := range append(parents, chunks...) {+ id, _ := row["id"].(string)+ if id == "" || rowsByID[id] != nil {+ t.Fatalf("parent/child rows have empty or duplicate ID %q", id)+ }+ rowsByID[id] = row+ }+ for i, chunk := range chunks {+ if chunk["id"] != common.ChunkID("doc-1", tc.want[i]) || chunk["content_with_weight"] != tc.want[i] {+ t.Errorf("child %d identity or text changed during index normalization: %#v", i, chunk)+ }+ if wantParents == 0 {+ if chunk["mom_id"] != nil {+ t.Errorf("unsplit chunk has unexpected mom_id %v", chunk["mom_id"])+ }+ continue+ }+ if chunk["mom_id"] != parents[0]["id"] {+ t.Errorf("child %d mom_id = %v, want %v", i, chunk["mom_id"], parents[0]["id"])+ }+ }+ if wantParents > 0 {+ if parents[0]["content_with_weight"] != "alpha#beta#gamma" || parents[0]["available_int"] != 0 {+ t.Errorf("parent must retain the full text as a hidden row: %#v", parents[0])+ }+ }+ if len(chunks) > 1 {+ parent := run(nil)+ if len(parent) != 1 {+ t.Fatalf("unsplit chunks = %d, want 1", len(parent))+ }+ preview, err := store.Get(ctx, "preview-test", parent[0]["id"].(string))+ if err != nil {+ t.Fatal(err)+ }+ img, err := png.DecodeConfig(bytes.NewReader(preview))+ if err != nil {+ t.Fatal(err)+ }+ for i, height := range heights {+ if height >= img.Height {+ t.Errorf("chunk %d preview height = %d, want smaller than unsplit preview %d", i, height, img.Height)+ }+ }+ }+ })+ }+}diff --git a/internal/ingestion/component/chunker/general_pdf_fixture_test.go b/internal/ingestion/component/chunker/general_pdf_fixture_test.gonew file mode 100644index 0000000000..b2a2a13842--- /dev/null+++ b/internal/ingestion/component/chunker/general_pdf_fixture_test.go@@ -0,0 +1,52 @@+//go:build cgo++package chunker++import (+ "bytes"+ "compress/zlib"+ "fmt"+ "testing"+)++// newGeneralPDFChildPreviewFixture builds a one-page PDF with three solid+// markers so preview tests do not need an opaque binary fixture.+func newGeneralPDFChildPreviewFixture(t *testing.T) []byte {+ t.Helper()++ content := []byte(+ "1 0 0 rg\n110 582 20 20 re f\n" ++ "0 1 0 rg\n110 472 20 20 re f\n" ++ "0 0 1 rg\n110 372 20 20 re f\n",+ )+ var compressed bytes.Buffer+ writer := zlib.NewWriter(&compressed)+ if _, err := writer.Write(content); err != nil {+ t.Fatal(err)+ }+ if err := writer.Close(); err != nil {+ t.Fatal(err)+ }++ var pdf bytes.Buffer+ pdf.WriteString("%PDF-1.4\n%\xe2\xe3\xcf\xd3\n")+ offsets := make([]int, 5)+ writeObject := func(number int, body string) {+ t.Helper()+ offsets[number] = pdf.Len()+ fmt.Fprintf(&pdf, "%d 0 obj\n%s\nendobj\n", number, body)+ }+ writeObject(1, "<< /Type /Catalog /Pages 2 0 R >>")+ writeObject(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>")+ writeObject(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 842] /Resources << >> /Contents 4 0 R >>")+ writeObject(4, fmt.Sprintf("<< /Length %d /Filter /FlateDecode >>\nstream\n%s\nendstream", compressed.Len(), compressed.Bytes()))++ xref := pdf.Len()+ pdf.WriteString("xref\n0 5\n")+ pdf.WriteString("0000000000 65535 f \n")+ for number := 1; number < len(offsets); number++ {+ fmt.Fprintf(&pdf, "%010d 00000 n \n", offsets[number])+ }+ fmt.Fprintf(&pdf, "trailer\n<< /Size 5 /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", xref)+ return pdf.Bytes()+}diff --git a/internal/ingestion/component/chunker/general_pdf_media_test.go b/internal/ingestion/component/chunker/general_pdf_media_test.gonew file mode 100644index 0000000000..0e4bd3ae47--- /dev/null+++ b/internal/ingestion/component/chunker/general_pdf_media_test.go@@ -0,0 +1,108 @@+//go:build cgo++package chunker++import (+ "bytes"+ "context"+ "errors"+ "image/png"+ "sync/atomic"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/ingestion/component"+ "ragflow/internal/storage"+)++func TestGeneralPDFContextOnlyMedia(t *testing.T) {+ data := newGeneralPDFChildPreviewFixture(t)+ for _, kind := range []string{"image", "table"} {+ for _, scenario := range []string{"crop", "retry upload", "no context", "no source", "missing source", "invalid pdf", "missing page"} {+ t.Run(kind+"/"+scenario, func(t *testing.T) {+ ctx := withIngestionGlobals(t, "context-test", "doc-1")+ store := storage.NewMemoryStorage()+ previousStorage, previousUploader := component.SetStorageFactoryOverride, ChunkImageUploader+ component.SetStorageFactoryOverride = func() storage.Storage { return store }+ ChunkImageUploader = component.DefaultImageUploader+ t.Cleanup(func() {+ component.SetStorageFactoryOverride = previousStorage+ ChunkImageUploader = previousUploader+ })+ if scenario == "retry upload" {+ var failed atomic.Bool+ ChunkImageUploader = func(ctx context.Context, kbID, chunkID string, data []byte) (string, error) {+ if chunkID != common.ChunkID("doc-1", "before.") && failed.CompareAndSwap(false, true) {+ return "", errors.New("temporary upload failure")+ }+ return component.DefaultImageUploader(ctx, kbID, chunkID, data)+ }+ }+ page := 1+ if scenario == "missing page" {+ page = 99+ }+ inputs := map[string]any{+ "name": "source.pdf", "file_type": "pdf", "output_format": "json",+ "json": []map[string]any{+ {"text": "before.", "doc_type_kwd": "text", "positions": [][]int{{1, 90, 400, 100, 120}}},+ {"text": "", "doc_type_kwd": kind, "positions": [][]int{{page, 90, 400, 200, 300}}},+ },+ }+ if scenario != "no source" {+ inputs["bucket"], inputs["path"] = "context-test", "source.pdf"+ if scenario != "missing source" {+ pdf := data+ if scenario == "invalid pdf" {+ pdf = []byte("%PDF-invalid")+ }+ if err := store.Put(ctx, "context-test", "source.pdf", pdf); err != nil {+ t.Fatal(err)+ }+ }+ }+ contextSize := 64+ if scenario == "no context" {+ contextSize = 0+ }+ inner, err := NewGeneralChunker(map[string]any{"image_context_size": contextSize, "table_context_size": contextSize})+ if err != nil {+ t.Fatal(err)+ }+ out, err := (&imageUploadDecorator{inner: inner}).Invoke(ctx, nil, inputs)+ if err != nil {+ t.Fatal(err)+ }+ chunks := outputChunks(t, out)+ want := 1+ if scenario == "crop" || scenario == "retry upload" {+ want = 2+ }+ if len(chunks) != want {+ t.Fatalf("chunks = %d, want %d: %#v", len(chunks), want, chunks)+ }+ if want == 1 {+ if chunks[0]["text"] != "before." || chunks[0]["doc_type_kwd"] != "text" {+ t.Fatalf("unexpected remaining text: %#v", chunks)+ }+ return+ }+ media := chunks[1]+ if media["doc_type_kwd"] != kind || media["text"] != "before." {+ t.Errorf("media lost its type or context: %#v", media)+ }+ id, _ := media["id"].(string)+ if media["img_id"] != "context-test-"+id || media["image"] != nil {+ t.Errorf("media was not uploaded correctly: %#v", media)+ }+ preview, err := store.Get(ctx, "context-test", id)+ if err != nil {+ t.Fatal(err)+ }+ if _, err := png.DecodeConfig(bytes.NewReader(preview)); err != nil {+ t.Fatal(err)+ }+ })+ }+ }+}