RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(chunker): stream cropped previews to storage instead of batching (#20165)
xugangqiange6f2c2587d60
diff --git a/internal/ingestion/component/chunker/a1a2_regression_test.go b/internal/ingestion/component/chunker/a1a2_regression_test.gonew file mode 100644index 0000000000..110f8dfa60--- /dev/null+++ b/internal/ingestion/component/chunker/a1a2_regression_test.go@@ -0,0 +1,120 @@+//go:build cgo++// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package chunker++import (+ "context"+ "fmt"+ "strings"+ "testing"++ "go.uber.org/zap"+ "go.uber.org/zap/zapcore"+ "go.uber.org/zap/zaptest/observer"++ "ragflow/internal/common"+ "ragflow/internal/ingestion/component/schema"+)++// TestFinalizeGeneralChunks_KeepsUploadedTagOnlyChunk is the regression test+// for A1: a media chunk whose text is ONLY a parser position tag (so+// removeTag yields "") but which already has a non-empty ImgID (set by the+// streaming crop upload) must survive finalizeGeneralChunks. The streaming+// upload clears out[i].Image immediately, so the old keep condition+// `Text=="" && Image==""` would drop it — discarding an already-uploaded+// image and orphaning the MinIO object. The keep condition must also consult+// ImgID.+func TestFinalizeGeneralChunks_KeepsUploadedTagOnlyChunk(t *testing.T) {+ in := []schema.ChunkDoc{+ {+ DocType: "image",+ Text: "@@1\t2##", // pure position tag; removeTag -> ""+ Image: "", // cleared by streaming upload+ ImgID: "kb1-deadbeef", // uploaded, non-empty+ },+ }+ got := finalizeGeneralChunks(in, nil)+ if len(got) != 1 {+ t.Fatalf("len(got) = %d, want 1 (uploaded tag-only chunk must survive finalize)", len(got))+ }+ if got[0].ImgID != "kb1-deadbeef" {+ t.Errorf("ImgID = %q, want preserved kb1-deadbeef", got[0].ImgID)+ }++ // Control: a genuinely empty chunk (no ImgID) must still be dropped.+ empty := []schema.ChunkDoc{{DocType: "image", Text: "", Image: "", ImgID: ""}}+ if got2 := finalizeGeneralChunks(empty, nil); len(got2) != 0 {+ t.Errorf("len(empty finalize) = %d, want 0 (true empty chunk dropped)", len(got2))+ }+}++// TestCropImageChunks_UploadFailureRedactsChunkText is the regression test for+// A2: when a streaming preview upload fails, cropImageChunks logs a warning,+// but that warning must NOT carry the document's chunk text (CWE-532 —+// sensitive data into logs). The text field must be redacted from the log.+func TestCropImageChunks_UploadFailureRedactsChunkText(t *testing.T) {+ core, observedLogs := observer.New(zapcore.WarnLevel)+ oldLogger := common.Logger+ common.Logger = zap.New(core)+ t.Cleanup(func() { common.Logger = oldLogger })++ ctx := withIngestionGlobals(t, "kb1", "doc1")++ orig := ChunkImageUploader+ ChunkImageUploader = func(_ context.Context, _, _ string, _ []byte) (string, error) {+ return "", fmt.Errorf("boom")+ }+ t.Cleanup(func() { ChunkImageUploader = orig })++ eng := mockCropEngine{}+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})+ secret := "SECRET_DOC_TEXT_9f3a"+ chunks := []schema.ChunkDoc{{CKType: "image", Text: secret, PDFPositions: pos}}+ cropImageChunks(ctx, eng, chunks)++ for _, e := range observedLogs.All() {+ for _, f := range e.Context {+ if f.Key == "chunk" {+ t.Errorf("upload-failure log still carries a 'chunk' field: %v", f)+ }+ if sv, ok := f.Interface.(string); ok && strings.Contains(sv, secret) {+ t.Errorf("upload-failure log leaks document text %q via field %q", secret, f.Key)+ }+ }+ }+}++// TestCanonicalChunkIDStripsPositionTags is the regression test for the+// streaming-upload key consistency Blocker: canonicalChunkID (used both by+// cropImageChunks for the MinIO object key and, via the decorator, as+// ck["id"]) must be derived from the tag-STRIPPED text. For the common+// no-media-context image/table chunk the parser text still carries a position+// tag (@@…##); keying the upload on the raw text would store the object+// under a key that never matches the canonical id the retrieval path looks+// up by.+func TestCanonicalChunkIDStripsPositionTags(t *testing.T) {+ in := schema.ChunkDoc{DocType: "image", Text: "abc@@1\t2##"}++ want := common.ChunkID("doc1", removeTag(in.Text)) // tag-stripped+ if got := canonicalChunkID("doc1", in); got != want {+ t.Errorf("canonicalChunkID = %q, want tag-stripped id %q", got, want)+ }+ // The raw, tag-bearing text must NOT be the id.+ if got := canonicalChunkID("doc1", in); got == common.ChunkID("doc1", in.Text) {+ t.Errorf("canonicalChunkID equals tag-bearing id; must strip position tags")+ }+}diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.goindex 4fb3c426ed..b4eea7c327 100644--- a/internal/ingestion/component/chunker/common.go+++ b/internal/ingestion/component/chunker/common.go@@ -22,6 +22,7 @@ import ("strings""ragflow/internal/agent/runtime"+ "ragflow/internal/common""ragflow/internal/ingestion/component/schema""ragflow/internal/parser/chunk""ragflow/internal/tokenizer"@@ -233,6 +234,32 @@ func chunkOutputs(chunks []schema.ChunkDoc) map[string]any {}}+// canonicalChunkText returns the normalized text a chunk id is derived+// from. It folds media context (when present) and strips position tags,+// matching exactly the text the decorator keys on after+// finalizeGeneralChunks runs removeTag and chunkOutputs materializes+// context. Routing every chunk id through this one function — instead of+// recomputing the text at each consumption point — guarantees the streamed+// crop-upload key and the decorator's ck["id"] can never diverge, including+// for image/table chunks whose text still carries position tags when there+// is no media context.+func canonicalChunkText(ck schema.ChunkDoc) string {+ return removeTag(materializeMediaContext(ck).Text)+}++// canonicalChunkID returns the deterministic chunk id: the single identity+// used both as the MinIO object key and as ck["id"]. The id formula (ChunkID+// over canonicalChunkText) is centralized here, so the text normalization+// that feeds the hash lives in exactly one place. The decorator in+// register.go re-derives ck["id"] from the same already-finalized text as a+// fallback for chunks that bypassed the streamed crop-upload path; the two+// routes cannot disagree because a chunk is either stamped by crop-upload+// (and the decorator reuses that id) or computed by the decorator fallback —+// never both — and both hash the normalized chunk body.+func canonicalChunkID(docID string, ck schema.ChunkDoc) string {+ return common.ChunkID(docID, canonicalChunkText(ck))+}+// materializeMediaContext folds a media chunk's surrounding context into its// body and clears the two fields that carried it. Python's chunker emits the// same shape — its finalize builds remove_tag(context_above + text +diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex 2f70203ce1..f4549fccf1 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -1134,7 +1134,12 @@ func finalizeGeneralChunks(chunks []schema.ChunkDoc, childrenPattern *regexp.Regvisible := make([]schema.ChunkDoc, 0, len(chunks))for _, chunk := range chunks {chunk.Text = removeTag(chunk.Text)- if strings.TrimSpace(chunk.Text) == "" && chunk.Image == "" {+ // Keep a chunk whenever it still carries retrievable content. A+ // streamed crop upload sets ImgID and clears Image, so an uploaded+ // media chunk (text may be only a position tag, or empty) must be+ // retained via its ImgID — otherwise the already-uploaded image is+ // dropped and its MinIO object orphaned.+ if strings.TrimSpace(chunk.Text) == "" && chunk.Image == "" && chunk.ImgID == "" {continue}visible = append(visible, chunk)diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo.go b/internal/ingestion/component/chunker/pdfcrop_cgo.goindex c25775b052..f6d6c44718 100644--- a/internal/ingestion/component/chunker/pdfcrop_cgo.go+++ b/internal/ingestion/component/chunker/pdfcrop_cgo.go@@ -12,6 +12,7 @@ package chunkerimport ("context"+ "encoding/base64""encoding/json""image""os"@@ -174,6 +175,44 @@ func cropImageChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks [return}out[i].Image = "data:image/png;base64," + img++ // Stream the freshly cropped preview to object storage and drop+ // the in-memory base64 immediately, instead of carrying every+ // chunk's image until the later batch upload pass+ // (imageUploadDecorator). The batch pass is idempotent: it skips+ // any chunk whose img_id is already set, so an upload that fails+ // here simply falls through to that retry path. kb_id is empty+ // only in canvas debug (dry-run) mode, where no persist stage+ // runs and the decorator's debug branch drops the raw bytes.+ if kbID, docID := resolveImageUploadContext(ctx, nil); kbID != "" {+ if raw, derr := base64.StdEncoding.DecodeString(img); derr == nil {+ // Key the streamed upload under the chunk's canonical id,+ // so the MinIO object is stored under exactly the key the+ // decorator (imageUploadDecorator) later exposes as+ // ck["id"] and the persist/retrieval path looks it up by.+ // canonicalChunkText is the single source for that id text:+ // it folds media context and strips position tags, so the+ // value equals the tag-stripped (finalized) text the+ // decorator derives — for every chunk type, including+ // image/table chunks whose text still carries position tags+ // when there is no media context. crop depends only on+ // positions, so reading the canonical text here does not+ // alter the cropped image or the chunker's later output+ // text.+ chunkID := canonicalChunkID(docID, out[i])+ if imgID, uerr := uploadOneImage(ctx, ChunkImageUploader, kbID, chunkID, raw); uerr == nil {+ out[i].ImgID = imgID+ out[i].Image = ""+ out[i].ID = chunkID+ } else {+ // Note: the document text is intentionally NOT logged+ // here (CWE-532). The upload is retried at the persist+ // stage, so the error is enough to diagnose.+ common.Warn("cropImageChunks: preview upload failed; will retry at persist stage",+ zap.Error(uerr))+ }+ }+ }}(i, ck, positions)}wg.Wait()diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go b/internal/ingestion/component/chunker/pdfcrop_cgo_test.goindex 967db9a4b4..e9b4736c11 100644--- a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go+++ b/internal/ingestion/component/chunker/pdfcrop_cgo_test.go@@ -8,8 +8,11 @@ import ("fmt""image""strings"+ "sync""testing"+ "ragflow/internal/agent/runtime"+ "ragflow/internal/common"deepdoctype "ragflow/internal/deepdoc/parser/type""ragflow/internal/ingestion/component/schema")@@ -262,3 +265,248 @@ func TestCropImageChunks_RenderFailureSkipsChunk(t *testing.T) {t.Errorf("chunk image = %q, want data:image/png;base64, prefix", out[0].Image)}}++// recordingUploader captures every upload invocation so tests can assert on+// the streaming-upload behavior of cropImageChunks.+type recordingUploader struct {+ mu sync.Mutex+ calls []uploadCall+}++type uploadCall struct {+ kbID string+ chunkID string+ dataLen int+}++func (r *recordingUploader) upload(ctx context.Context, kbID, chunkID string, data []byte) (string, error) {+ r.mu.Lock()+ r.calls = append(r.calls, uploadCall{kbID: kbID, chunkID: chunkID, dataLen: len(data)})+ r.mu.Unlock()+ // Mirror the production key format used by imageUploadDecorator so the+ // test can confirm cropImageChunks produces the same reference.+ return kbID + "-" + chunkID, nil+}++// withIngestionGlobals returns a ctx carrying kb_id / doc_id in the run-level+// CanvasState.Globals bag, exactly as the production ingestion pipeline+// seeds them via SeedIngestionGlobals before the chunker runs.+func withIngestionGlobals(t *testing.T, kbID, docID string) context.Context {+ t.Helper()+ st := runtime.NewCanvasState("test-run", "test-session")+ st.SetGlobal("kb_id", kbID)+ st.SetGlobal("doc_id", docID)+ return runtime.WithState(context.Background(), st)+}++// TestCropImageChunks_StreamingUpload verifies that, when a KB is present,+// cropImageChunks uploads each freshly cropped preview immediately and drops+// the in-memory base64, instead of holding every chunk's image until the+// later batch upload pass. This is the memory fix: peak Go-heap retention+// during the chunker stage of a large PDF collapses from "whole document" to+// "one chunk".+func TestCropImageChunks_StreamingUpload(t *testing.T) {+ ctx := withIngestionGlobals(t, "kb1", "doc1")++ rec := &recordingUploader{}+ orig := ChunkImageUploader+ ChunkImageUploader = rec.upload+ t.Cleanup(func() { ChunkImageUploader = orig })++ eng := mockCropEngine{}+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})+ chunks := []schema.ChunkDoc{+ {CKType: "image", Text: "img-chunk", PDFPositions: pos},+ {CKType: "table", Text: "tbl-chunk", PDFPositions: pos},+ {CKType: "text", Text: "txt-chunk", PDFPositions: pos},+ }+ out := cropImageChunks(ctx, eng, chunks)+ if len(out) != len(chunks) {+ t.Fatalf("len(out) = %d, want %d", len(out), len(chunks))+ }++ for i, ck := range out {+ if ck.ImgID == "" {+ t.Errorf("chunk %d (%s): ImgID empty, want uploaded id", i, ck.CKType)+ }+ if ck.Image != "" {+ t.Errorf("chunk %d (%s): Image = %q, want cleared after upload", i, ck.CKType, ck.Image)+ }+ }+ if len(rec.calls) != len(chunks) {+ t.Fatalf("upload calls = %d, want %d", len(rec.calls), len(chunks))+ }+ // cropImageChunks fans out per chunk in goroutines, so the order of+ // upload calls is non-deterministic; assert the SET of uploaded ids+ // matches the expected per-chunk ids rather than the call order.+ wantIDs := make(map[string]struct{}, len(chunks))+ for _, ck := range chunks {+ wantIDs[common.ChunkID("doc1", ck.Text)] = struct{}{}+ }+ for _, c := range rec.calls {+ if c.dataLen == 0 {+ t.Errorf("uploaded empty bytes")+ }+ if _, ok := wantIDs[c.chunkID]; !ok {+ t.Errorf("call uploaded unexpected chunkID %q", c.chunkID)+ }+ }+ // The stored img_id per chunk must equal what the uploader returns+ // ("<kb_id>-<chunkID>"); keyed by input position, which is deterministic+ // even though the upload order is not.+ for i, ck := range out {+ wantImgID := "kb1-" + common.ChunkID("doc1", chunks[i].Text)+ if ck.ImgID != wantImgID {+ t.Errorf("chunk %d: ImgID = %q, want %q", i, ck.ImgID, wantImgID)+ }+ }+}++// TestCropImageChunks_NoUploadWhenKBAbsent locks the canvas-debug (dry-run)+// path: with no CanvasState (and thus no kb_id), cropImageChunks must NOT+// upload and must retain the base64 preview in memory — the decorator's+// debug branch is responsible for dropping it, and no persist stage will run.+func TestCropImageChunks_NoUploadWhenKBAbsent(t *testing.T) {+ ctx := context.Background() // no CanvasState → kb_id == ""++ rec := &recordingUploader{}+ orig := ChunkImageUploader+ ChunkImageUploader = rec.upload+ t.Cleanup(func() { ChunkImageUploader = orig })++ eng := mockCropEngine{}+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})+ chunks := []schema.ChunkDoc{{CKType: "image", PDFPositions: pos}}+ out := cropImageChunks(ctx, eng, chunks)++ if len(rec.calls) != 0 {+ t.Fatalf("upload calls = %d, want 0 (kb_id absent)", len(rec.calls))+ }+ if !strings.HasPrefix(out[0].Image, "data:image/png;base64,") {+ t.Errorf("Image not retained: %q", out[0].Image)+ }+ if out[0].ImgID != "" {+ t.Errorf("ImgID = %q, want empty", out[0].ImgID)+ }+}++// TestCropImageChunks_UploadFailureFallsThroughToBatchPass verifies that a+// failed streaming upload keeps the base64 preview so the later idempotent+// batch upload pass (imageUploadDecorator) can retry it.+func TestCropImageChunks_UploadFailureFallsThroughToBatchPass(t *testing.T) {+ ctx := withIngestionGlobals(t, "kb1", "doc1")++ orig := ChunkImageUploader+ ChunkImageUploader = func(_ context.Context, _, _ string, _ []byte) (string, error) {+ return "", fmt.Errorf("boom")+ }+ t.Cleanup(func() { ChunkImageUploader = orig })++ eng := mockCropEngine{}+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})+ chunks := []schema.ChunkDoc{{CKType: "image", Text: "img-chunk", PDFPositions: pos}}+ out := cropImageChunks(ctx, eng, chunks)++ if out[0].ImgID != "" {+ t.Errorf("ImgID = %q, want empty after failed upload", out[0].ImgID)+ }+ if !strings.HasPrefix(out[0].Image, "data:image/png;base64,") {+ t.Errorf("Image not retained after failed upload: %q", out[0].Image)+ }+}++// TestCropImageChunks_StreamingUploadUsesFinalizedText verifies that the+// streaming upload keys the image under the chunk's FINALIZED text — after+// removeTag + context fold — rather than the raw pre-finalization text. This+// is what makes the stored img_id match the canonical chunk id that+// imageUploadDecorator assigns later (register.go) and Python's convention.+// Without this, media chunks whose text changes during finalization (context+// folding or position-tag stripping) would be stored under a different key+// than their canonical id.+func TestCropImageChunks_StreamingUploadUsesFinalizedText(t *testing.T) {+ ctx := withIngestionGlobals(t, "kb1", "doc1")++ rec := &recordingUploader{}+ orig := ChunkImageUploader+ ChunkImageUploader = rec.upload+ t.Cleanup(func() { ChunkImageUploader = orig })++ eng := mockCropEngine{}+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})+ // ContextAbove is folded into the body by materializeMediaContext, so the+ // finalized text differs from the raw Text.+ chunks := []schema.ChunkDoc{+ {CKType: "image", Text: "body", ContextAbove: "ABOVE ", PDFPositions: pos},+ }+ out := cropImageChunks(ctx, eng, chunks)++ if len(out) != 1 {+ t.Fatalf("len(out) = %d, want 1", len(out))+ }+ if len(rec.calls) != 1 {+ t.Fatalf("upload calls = %d, want 1", len(rec.calls))+ }++ // Finalized text = removeTag(ContextAbove + removeTag(Text) + ContextBelow)+ // = "ABOVE body".+ wantID := common.ChunkID("doc1", "ABOVE body")+ rawID := common.ChunkID("doc1", "body")++ if rec.calls[0].chunkID == rawID {+ t.Errorf("upload keyed by raw text %q; want finalized-text key %q", rawID, wantID)+ }+ if rec.calls[0].chunkID != wantID {+ t.Errorf("upload chunkID = %q, want %q", rec.calls[0].chunkID, wantID)+ }+ if out[0].ImgID != "kb1-"+wantID {+ t.Errorf("ImgID = %q, want %q", out[0].ImgID, "kb1-"+wantID)+ }+}++// TestCropImageChunks_StreamingUploadUsesCanonicalID is the regression test+// for the most common chunk shape: an image/table chunk whose Text still+// carries parser position tags (@@…##) but has NO media context+// (ContextAbove/Below empty). For that shape materializeMediaContext+// short-circuits and leaves the tags in place, so the raw text still carries+// them. The canonical chunk id (what imageUploadDecorator assigns later in+// register.go as ck["id"] and what retrieval looks the object up by) is+// derived from the TAG-STRIPPED (finalized) text. The streamed upload key+// MUST therefore also be the tag-stripped canonical id — keying on the raw,+// tag-bearing text would store the object under a key that never matches+// ck["id"]. canonicalChunkID is the single source for that id, so the two+// can never diverge.+func TestCropImageChunks_StreamingUploadUsesCanonicalID(t *testing.T) {+ ctx := withIngestionGlobals(t, "kb1", "doc1")++ rec := &recordingUploader{}+ orig := ChunkImageUploader+ ChunkImageUploader = rec.upload+ t.Cleanup(func() { ChunkImageUploader = orig })++ eng := mockCropEngine{}+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})+ // Text carries a parser position tag (@@x\t y##), no media context.+ in := schema.ChunkDoc{CKType: "image", Text: "abc@@1\t2##", PDFPositions: pos}+ out := cropImageChunks(ctx, eng, []schema.ChunkDoc{in})++ if len(out) != 1 {+ t.Fatalf("len(out) = %d, want 1", len(out))+ }+ if len(rec.calls) != 1 {+ t.Fatalf("upload calls = %d, want 1", len(rec.calls))+ }++ // The streamed upload key must be the canonical (tag-STRIPPED) id,+ // exactly what imageUploadDecorator later exposes as ck["id"].+ wantID := canonicalChunkID("doc1", in)+ if rec.calls[0].chunkID != wantID {+ t.Errorf("upload chunkID = %q, want canonical %q", rec.calls[0].chunkID, wantID)+ }+ // Regression guard: it must NOT be the raw tag-bearing variant.+ if rec.calls[0].chunkID == common.ChunkID("doc1", in.Text) {+ t.Errorf("upload keyed by raw tag-bearing text %q; must use canonical (tag-stripped) id", common.ChunkID("doc1", in.Text))+ }+ if out[0].ImgID != "kb1-"+wantID {+ t.Errorf("ImgID = %q, want %q", out[0].ImgID, "kb1-"+wantID)+ }+}diff --git a/internal/ingestion/component/chunker/register.go b/internal/ingestion/component/chunker/register.goindex 585c9197de..dc875ce72c 100644--- a/internal/ingestion/component/chunker/register.go+++ b/internal/ingestion/component/chunker/register.go@@ -90,9 +90,20 @@ func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs m// Compute and write the deterministic chunk id (component.ChunkID) for// every chunk. This happens here — before any upload — so uploadChunkImage- // can read ck["id"] without deriving it itself. Downstream, the persist- // stage reuses the same formula as a fallback when ck["id"] is absent.+ // can read ck["id"] without deriving it itself. The id formula itself+ // lives in canonicalChunkID (common.go); this loop only re-derives it as+ // a fallback for chunks whose id was not already stamped upstream.+ //+ // A chunk id may already be stamped (e.g. by the streamed crop-upload+ // path, which keys MinIO by the canonical id). Reuse it so the id is+ // computed exactly once; both routes normalize the text through+ // canonicalChunkText, so the crop-time key and this id are identical+ // and cannot diverge. For chunks that bypassed crop, fall back to+ // deriving it here from the already-finalized text.for _, ck := range chunks {+ if id, ok := ck["id"]; ok && id != "" {+ continue+ }text, err := requireChunkText(ck)if err != nil {return nil, errdiff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.goindex c60b8c7ce9..6648dd6b6d 100644--- a/internal/ingestion/component/schema/chunk_types.go+++ b/internal/ingestion/component/schema/chunk_types.go@@ -105,6 +105,7 @@ type ChunkDoc struct {TKNums *int `json:"tk_nums,omitempty"`Mom string `json:"mom,omitempty"`ImgID string `json:"img_id,omitempty"`+ ID string `json:"id,omitempty"`Layout string `json:"layout,omitempty"`LayoutType string `json:"layout_type,omitempty"`LayoutNo string `json:"layoutno,omitempty"`