Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(chunker): stream cropped previews to storage instead of batching (#20165)

xugangqiange6f2c2587d60
diff --git a/internal/ingestion/component/chunker/a1a2_regression_test.go b/internal/ingestion/component/chunker/a1a2_regression_test.go
new file mode 100644
index 0000000000..110f8dfa60
--- /dev/null
+++ b/internal/ingestion/component/chunker/a1a2_regression_test.go
@@ -0,0 +1,120 @@
+//go:build cgo
+
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package chunker
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "testing"
+
+ "go.uber.org/zap"
+ "go.uber.org/zap/zapcore"
+ "go.uber.org/zap/zaptest/observer"
+
+ "ragflow/internal/common"
+ "ragflow/internal/ingestion/component/schema"
+)
+
+// TestFinalizeGeneralChunks_KeepsUploadedTagOnlyChunk is the regression test
+// for A1: a media chunk whose text is ONLY a parser position tag (so
+// removeTag yields "") but which already has a non-empty ImgID (set by the
+// streaming crop upload) must survive finalizeGeneralChunks. The streaming
+// upload clears out[i].Image immediately, so the old keep condition
+// `Text=="" && Image==""` would drop it — discarding an already-uploaded
+// image and orphaning the MinIO object. The keep condition must also consult
+// ImgID.
+func TestFinalizeGeneralChunks_KeepsUploadedTagOnlyChunk(t *testing.T) {
+ in := []schema.ChunkDoc{
+ {
+ DocType: "image",
+ Text: "@@1\t2##", // pure position tag; removeTag -> ""
+ Image: "", // cleared by streaming upload
+ ImgID: "kb1-deadbeef", // uploaded, non-empty
+ },
+ }
+ got := finalizeGeneralChunks(in, nil)
+ if len(got) != 1 {
+ t.Fatalf("len(got) = %d, want 1 (uploaded tag-only chunk must survive finalize)", len(got))
+ }
+ if got[0].ImgID != "kb1-deadbeef" {
+ t.Errorf("ImgID = %q, want preserved kb1-deadbeef", got[0].ImgID)
+ }
+
+ // Control: a genuinely empty chunk (no ImgID) must still be dropped.
+ empty := []schema.ChunkDoc{{DocType: "image", Text: "", Image: "", ImgID: ""}}
+ if got2 := finalizeGeneralChunks(empty, nil); len(got2) != 0 {
+ t.Errorf("len(empty finalize) = %d, want 0 (true empty chunk dropped)", len(got2))
+ }
+}
+
+// TestCropImageChunks_UploadFailureRedactsChunkText is the regression test for
+// A2: when a streaming preview upload fails, cropImageChunks logs a warning,
+// but that warning must NOT carry the document's chunk text (CWE-532 —
+// sensitive data into logs). The text field must be redacted from the log.
+func TestCropImageChunks_UploadFailureRedactsChunkText(t *testing.T) {
+ core, observedLogs := observer.New(zapcore.WarnLevel)
+ oldLogger := common.Logger
+ common.Logger = zap.New(core)
+ t.Cleanup(func() { common.Logger = oldLogger })
+
+ ctx := withIngestionGlobals(t, "kb1", "doc1")
+
+ orig := ChunkImageUploader
+ ChunkImageUploader = func(_ context.Context, _, _ string, _ []byte) (string, error) {
+ return "", fmt.Errorf("boom")
+ }
+ t.Cleanup(func() { ChunkImageUploader = orig })
+
+ eng := mockCropEngine{}
+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
+ secret := "SECRET_DOC_TEXT_9f3a"
+ chunks := []schema.ChunkDoc{{CKType: "image", Text: secret, PDFPositions: pos}}
+ cropImageChunks(ctx, eng, chunks)
+
+ for _, e := range observedLogs.All() {
+ for _, f := range e.Context {
+ if f.Key == "chunk" {
+ t.Errorf("upload-failure log still carries a 'chunk' field: %v", f)
+ }
+ if sv, ok := f.Interface.(string); ok && strings.Contains(sv, secret) {
+ t.Errorf("upload-failure log leaks document text %q via field %q", secret, f.Key)
+ }
+ }
+ }
+}
+
+// TestCanonicalChunkIDStripsPositionTags is the regression test for the
+// streaming-upload key consistency Blocker: canonicalChunkID (used both by
+// cropImageChunks for the MinIO object key and, via the decorator, as
+// ck["id"]) must be derived from the tag-STRIPPED text. For the common
+// no-media-context image/table chunk the parser text still carries a position
+// tag (@@…##); keying the upload on the raw text would store the object
+// under a key that never matches the canonical id the retrieval path looks
+// up by.
+func TestCanonicalChunkIDStripsPositionTags(t *testing.T) {
+ in := schema.ChunkDoc{DocType: "image", Text: "abc@@1\t2##"}
+
+ want := common.ChunkID("doc1", removeTag(in.Text)) // tag-stripped
+ if got := canonicalChunkID("doc1", in); got != want {
+ t.Errorf("canonicalChunkID = %q, want tag-stripped id %q", got, want)
+ }
+ // The raw, tag-bearing text must NOT be the id.
+ if got := canonicalChunkID("doc1", in); got == common.ChunkID("doc1", in.Text) {
+ t.Errorf("canonicalChunkID equals tag-bearing id; must strip position tags")
+ }
+}
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go
index 4fb3c426ed..b4eea7c327 100644
--- a/internal/ingestion/component/chunker/common.go
+++ b/internal/ingestion/component/chunker/common.go
@@ -22,6 +22,7 @@ import (
"strings"
"ragflow/internal/agent/runtime"
+ "ragflow/internal/common"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/chunk"
"ragflow/internal/tokenizer"
@@ -233,6 +234,32 @@ func chunkOutputs(chunks []schema.ChunkDoc) map[string]any {
}
}
+// canonicalChunkText returns the normalized text a chunk id is derived
+// from. It folds media context (when present) and strips position tags,
+// matching exactly the text the decorator keys on after
+// finalizeGeneralChunks runs removeTag and chunkOutputs materializes
+// context. Routing every chunk id through this one function — instead of
+// recomputing the text at each consumption point — guarantees the streamed
+// crop-upload key and the decorator's ck["id"] can never diverge, including
+// for image/table chunks whose text still carries position tags when there
+// is no media context.
+func canonicalChunkText(ck schema.ChunkDoc) string {
+ return removeTag(materializeMediaContext(ck).Text)
+}
+
+// canonicalChunkID returns the deterministic chunk id: the single identity
+// used both as the MinIO object key and as ck["id"]. The id formula (ChunkID
+// over canonicalChunkText) is centralized here, so the text normalization
+// that feeds the hash lives in exactly one place. The decorator in
+// register.go re-derives ck["id"] from the same already-finalized text as a
+// fallback for chunks that bypassed the streamed crop-upload path; the two
+// routes cannot disagree because a chunk is either stamped by crop-upload
+// (and the decorator reuses that id) or computed by the decorator fallback —
+// never both — and both hash the normalized chunk body.
+func canonicalChunkID(docID string, ck schema.ChunkDoc) string {
+ return common.ChunkID(docID, canonicalChunkText(ck))
+}
+
// materializeMediaContext folds a media chunk's surrounding context into its
// body and clears the two fields that carried it. Python's chunker emits the
// same shape — its finalize builds remove_tag(context_above + text +
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index 2f70203ce1..f4549fccf1 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -1134,7 +1134,12 @@ func finalizeGeneralChunks(chunks []schema.ChunkDoc, childrenPattern *regexp.Reg
visible := make([]schema.ChunkDoc, 0, len(chunks))
for _, chunk := range chunks {
chunk.Text = removeTag(chunk.Text)
- if strings.TrimSpace(chunk.Text) == "" && chunk.Image == "" {
+ // Keep a chunk whenever it still carries retrievable content. A
+ // streamed crop upload sets ImgID and clears Image, so an uploaded
+ // media chunk (text may be only a position tag, or empty) must be
+ // retained via its ImgID — otherwise the already-uploaded image is
+ // dropped and its MinIO object orphaned.
+ if strings.TrimSpace(chunk.Text) == "" && chunk.Image == "" && chunk.ImgID == "" {
continue
}
visible = append(visible, chunk)
diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo.go b/internal/ingestion/component/chunker/pdfcrop_cgo.go
index c25775b052..f6d6c44718 100644
--- a/internal/ingestion/component/chunker/pdfcrop_cgo.go
+++ b/internal/ingestion/component/chunker/pdfcrop_cgo.go
@@ -12,6 +12,7 @@ package chunker
import (
"context"
+ "encoding/base64"
"encoding/json"
"image"
"os"
@@ -174,6 +175,44 @@ func cropImageChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks [
return
}
out[i].Image = "data:image/png;base64," + img
+
+ // Stream the freshly cropped preview to object storage and drop
+ // the in-memory base64 immediately, instead of carrying every
+ // chunk's image until the later batch upload pass
+ // (imageUploadDecorator). The batch pass is idempotent: it skips
+ // any chunk whose img_id is already set, so an upload that fails
+ // here simply falls through to that retry path. kb_id is empty
+ // only in canvas debug (dry-run) mode, where no persist stage
+ // runs and the decorator's debug branch drops the raw bytes.
+ if kbID, docID := resolveImageUploadContext(ctx, nil); kbID != "" {
+ if raw, derr := base64.StdEncoding.DecodeString(img); derr == nil {
+ // Key the streamed upload under the chunk's canonical id,
+ // so the MinIO object is stored under exactly the key the
+ // decorator (imageUploadDecorator) later exposes as
+ // ck["id"] and the persist/retrieval path looks it up by.
+ // canonicalChunkText is the single source for that id text:
+ // it folds media context and strips position tags, so the
+ // value equals the tag-stripped (finalized) text the
+ // decorator derives — for every chunk type, including
+ // image/table chunks whose text still carries position tags
+ // when there is no media context. crop depends only on
+ // positions, so reading the canonical text here does not
+ // alter the cropped image or the chunker's later output
+ // text.
+ chunkID := canonicalChunkID(docID, out[i])
+ if imgID, uerr := uploadOneImage(ctx, ChunkImageUploader, kbID, chunkID, raw); uerr == nil {
+ out[i].ImgID = imgID
+ out[i].Image = ""
+ out[i].ID = chunkID
+ } else {
+ // Note: the document text is intentionally NOT logged
+ // here (CWE-532). The upload is retried at the persist
+ // stage, so the error is enough to diagnose.
+ common.Warn("cropImageChunks: preview upload failed; will retry at persist stage",
+ zap.Error(uerr))
+ }
+ }
+ }
}(i, ck, positions)
}
wg.Wait()
diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go b/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
index 967db9a4b4..e9b4736c11 100644
--- a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
+++ b/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
@@ -8,8 +8,11 @@ import (
"fmt"
"image"
"strings"
+ "sync"
"testing"
+ "ragflow/internal/agent/runtime"
+ "ragflow/internal/common"
deepdoctype "ragflow/internal/deepdoc/parser/type"
"ragflow/internal/ingestion/component/schema"
)
@@ -262,3 +265,248 @@ func TestCropImageChunks_RenderFailureSkipsChunk(t *testing.T) {
t.Errorf("chunk image = %q, want data:image/png;base64, prefix", out[0].Image)
}
}
+
+// recordingUploader captures every upload invocation so tests can assert on
+// the streaming-upload behavior of cropImageChunks.
+type recordingUploader struct {
+ mu sync.Mutex
+ calls []uploadCall
+}
+
+type uploadCall struct {
+ kbID string
+ chunkID string
+ dataLen int
+}
+
+func (r *recordingUploader) upload(ctx context.Context, kbID, chunkID string, data []byte) (string, error) {
+ r.mu.Lock()
+ r.calls = append(r.calls, uploadCall{kbID: kbID, chunkID: chunkID, dataLen: len(data)})
+ r.mu.Unlock()
+ // Mirror the production key format used by imageUploadDecorator so the
+ // test can confirm cropImageChunks produces the same reference.
+ return kbID + "-" + chunkID, nil
+}
+
+// withIngestionGlobals returns a ctx carrying kb_id / doc_id in the run-level
+// CanvasState.Globals bag, exactly as the production ingestion pipeline
+// seeds them via SeedIngestionGlobals before the chunker runs.
+func withIngestionGlobals(t *testing.T, kbID, docID string) context.Context {
+ t.Helper()
+ st := runtime.NewCanvasState("test-run", "test-session")
+ st.SetGlobal("kb_id", kbID)
+ st.SetGlobal("doc_id", docID)
+ return runtime.WithState(context.Background(), st)
+}
+
+// TestCropImageChunks_StreamingUpload verifies that, when a KB is present,
+// cropImageChunks uploads each freshly cropped preview immediately and drops
+// the in-memory base64, instead of holding every chunk's image until the
+// later batch upload pass. This is the memory fix: peak Go-heap retention
+// during the chunker stage of a large PDF collapses from "whole document" to
+// "one chunk".
+func TestCropImageChunks_StreamingUpload(t *testing.T) {
+ ctx := withIngestionGlobals(t, "kb1", "doc1")
+
+ rec := &recordingUploader{}
+ orig := ChunkImageUploader
+ ChunkImageUploader = rec.upload
+ t.Cleanup(func() { ChunkImageUploader = orig })
+
+ eng := mockCropEngine{}
+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
+ chunks := []schema.ChunkDoc{
+ {CKType: "image", Text: "img-chunk", PDFPositions: pos},
+ {CKType: "table", Text: "tbl-chunk", PDFPositions: pos},
+ {CKType: "text", Text: "txt-chunk", PDFPositions: pos},
+ }
+ out := cropImageChunks(ctx, eng, chunks)
+ if len(out) != len(chunks) {
+ t.Fatalf("len(out) = %d, want %d", len(out), len(chunks))
+ }
+
+ for i, ck := range out {
+ if ck.ImgID == "" {
+ t.Errorf("chunk %d (%s): ImgID empty, want uploaded id", i, ck.CKType)
+ }
+ if ck.Image != "" {
+ t.Errorf("chunk %d (%s): Image = %q, want cleared after upload", i, ck.CKType, ck.Image)
+ }
+ }
+ if len(rec.calls) != len(chunks) {
+ t.Fatalf("upload calls = %d, want %d", len(rec.calls), len(chunks))
+ }
+ // cropImageChunks fans out per chunk in goroutines, so the order of
+ // upload calls is non-deterministic; assert the SET of uploaded ids
+ // matches the expected per-chunk ids rather than the call order.
+ wantIDs := make(map[string]struct{}, len(chunks))
+ for _, ck := range chunks {
+ wantIDs[common.ChunkID("doc1", ck.Text)] = struct{}{}
+ }
+ for _, c := range rec.calls {
+ if c.dataLen == 0 {
+ t.Errorf("uploaded empty bytes")
+ }
+ if _, ok := wantIDs[c.chunkID]; !ok {
+ t.Errorf("call uploaded unexpected chunkID %q", c.chunkID)
+ }
+ }
+ // The stored img_id per chunk must equal what the uploader returns
+ // ("<kb_id>-<chunkID>"); keyed by input position, which is deterministic
+ // even though the upload order is not.
+ for i, ck := range out {
+ wantImgID := "kb1-" + common.ChunkID("doc1", chunks[i].Text)
+ if ck.ImgID != wantImgID {
+ t.Errorf("chunk %d: ImgID = %q, want %q", i, ck.ImgID, wantImgID)
+ }
+ }
+}
+
+// TestCropImageChunks_NoUploadWhenKBAbsent locks the canvas-debug (dry-run)
+// path: with no CanvasState (and thus no kb_id), cropImageChunks must NOT
+// upload and must retain the base64 preview in memory — the decorator's
+// debug branch is responsible for dropping it, and no persist stage will run.
+func TestCropImageChunks_NoUploadWhenKBAbsent(t *testing.T) {
+ ctx := context.Background() // no CanvasState → kb_id == ""
+
+ rec := &recordingUploader{}
+ orig := ChunkImageUploader
+ ChunkImageUploader = rec.upload
+ t.Cleanup(func() { ChunkImageUploader = orig })
+
+ eng := mockCropEngine{}
+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
+ chunks := []schema.ChunkDoc{{CKType: "image", PDFPositions: pos}}
+ out := cropImageChunks(ctx, eng, chunks)
+
+ if len(rec.calls) != 0 {
+ t.Fatalf("upload calls = %d, want 0 (kb_id absent)", len(rec.calls))
+ }
+ if !strings.HasPrefix(out[0].Image, "data:image/png;base64,") {
+ t.Errorf("Image not retained: %q", out[0].Image)
+ }
+ if out[0].ImgID != "" {
+ t.Errorf("ImgID = %q, want empty", out[0].ImgID)
+ }
+}
+
+// TestCropImageChunks_UploadFailureFallsThroughToBatchPass verifies that a
+// failed streaming upload keeps the base64 preview so the later idempotent
+// batch upload pass (imageUploadDecorator) can retry it.
+func TestCropImageChunks_UploadFailureFallsThroughToBatchPass(t *testing.T) {
+ ctx := withIngestionGlobals(t, "kb1", "doc1")
+
+ orig := ChunkImageUploader
+ ChunkImageUploader = func(_ context.Context, _, _ string, _ []byte) (string, error) {
+ return "", fmt.Errorf("boom")
+ }
+ t.Cleanup(func() { ChunkImageUploader = orig })
+
+ eng := mockCropEngine{}
+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
+ chunks := []schema.ChunkDoc{{CKType: "image", Text: "img-chunk", PDFPositions: pos}}
+ out := cropImageChunks(ctx, eng, chunks)
+
+ if out[0].ImgID != "" {
+ t.Errorf("ImgID = %q, want empty after failed upload", out[0].ImgID)
+ }
+ if !strings.HasPrefix(out[0].Image, "data:image/png;base64,") {
+ t.Errorf("Image not retained after failed upload: %q", out[0].Image)
+ }
+}
+
+// TestCropImageChunks_StreamingUploadUsesFinalizedText verifies that the
+// streaming upload keys the image under the chunk's FINALIZED text — after
+// removeTag + context fold — rather than the raw pre-finalization text. This
+// is what makes the stored img_id match the canonical chunk id that
+// imageUploadDecorator assigns later (register.go) and Python's convention.
+// Without this, media chunks whose text changes during finalization (context
+// folding or position-tag stripping) would be stored under a different key
+// than their canonical id.
+func TestCropImageChunks_StreamingUploadUsesFinalizedText(t *testing.T) {
+ ctx := withIngestionGlobals(t, "kb1", "doc1")
+
+ rec := &recordingUploader{}
+ orig := ChunkImageUploader
+ ChunkImageUploader = rec.upload
+ t.Cleanup(func() { ChunkImageUploader = orig })
+
+ eng := mockCropEngine{}
+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
+ // ContextAbove is folded into the body by materializeMediaContext, so the
+ // finalized text differs from the raw Text.
+ chunks := []schema.ChunkDoc{
+ {CKType: "image", Text: "body", ContextAbove: "ABOVE ", PDFPositions: pos},
+ }
+ out := cropImageChunks(ctx, eng, chunks)
+
+ if len(out) != 1 {
+ t.Fatalf("len(out) = %d, want 1", len(out))
+ }
+ if len(rec.calls) != 1 {
+ t.Fatalf("upload calls = %d, want 1", len(rec.calls))
+ }
+
+ // Finalized text = removeTag(ContextAbove + removeTag(Text) + ContextBelow)
+ // = "ABOVE body".
+ wantID := common.ChunkID("doc1", "ABOVE body")
+ rawID := common.ChunkID("doc1", "body")
+
+ if rec.calls[0].chunkID == rawID {
+ t.Errorf("upload keyed by raw text %q; want finalized-text key %q", rawID, wantID)
+ }
+ if rec.calls[0].chunkID != wantID {
+ t.Errorf("upload chunkID = %q, want %q", rec.calls[0].chunkID, wantID)
+ }
+ if out[0].ImgID != "kb1-"+wantID {
+ t.Errorf("ImgID = %q, want %q", out[0].ImgID, "kb1-"+wantID)
+ }
+}
+
+// TestCropImageChunks_StreamingUploadUsesCanonicalID is the regression test
+// for the most common chunk shape: an image/table chunk whose Text still
+// carries parser position tags (@@…##) but has NO media context
+// (ContextAbove/Below empty). For that shape materializeMediaContext
+// short-circuits and leaves the tags in place, so the raw text still carries
+// them. The canonical chunk id (what imageUploadDecorator assigns later in
+// register.go as ck["id"] and what retrieval looks the object up by) is
+// derived from the TAG-STRIPPED (finalized) text. The streamed upload key
+// MUST therefore also be the tag-stripped canonical id — keying on the raw,
+// tag-bearing text would store the object under a key that never matches
+// ck["id"]. canonicalChunkID is the single source for that id, so the two
+// can never diverge.
+func TestCropImageChunks_StreamingUploadUsesCanonicalID(t *testing.T) {
+ ctx := withIngestionGlobals(t, "kb1", "doc1")
+
+ rec := &recordingUploader{}
+ orig := ChunkImageUploader
+ ChunkImageUploader = rec.upload
+ t.Cleanup(func() { ChunkImageUploader = orig })
+
+ eng := mockCropEngine{}
+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
+ // Text carries a parser position tag (@@x\t y##), no media context.
+ in := schema.ChunkDoc{CKType: "image", Text: "abc@@1\t2##", PDFPositions: pos}
+ out := cropImageChunks(ctx, eng, []schema.ChunkDoc{in})
+
+ if len(out) != 1 {
+ t.Fatalf("len(out) = %d, want 1", len(out))
+ }
+ if len(rec.calls) != 1 {
+ t.Fatalf("upload calls = %d, want 1", len(rec.calls))
+ }
+
+ // The streamed upload key must be the canonical (tag-STRIPPED) id,
+ // exactly what imageUploadDecorator later exposes as ck["id"].
+ wantID := canonicalChunkID("doc1", in)
+ if rec.calls[0].chunkID != wantID {
+ t.Errorf("upload chunkID = %q, want canonical %q", rec.calls[0].chunkID, wantID)
+ }
+ // Regression guard: it must NOT be the raw tag-bearing variant.
+ if rec.calls[0].chunkID == common.ChunkID("doc1", in.Text) {
+ t.Errorf("upload keyed by raw tag-bearing text %q; must use canonical (tag-stripped) id", common.ChunkID("doc1", in.Text))
+ }
+ if out[0].ImgID != "kb1-"+wantID {
+ t.Errorf("ImgID = %q, want %q", out[0].ImgID, "kb1-"+wantID)
+ }
+}
diff --git a/internal/ingestion/component/chunker/register.go b/internal/ingestion/component/chunker/register.go
index 585c9197de..dc875ce72c 100644
--- a/internal/ingestion/component/chunker/register.go
+++ b/internal/ingestion/component/chunker/register.go
@@ -90,9 +90,20 @@ func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs m
// Compute and write the deterministic chunk id (component.ChunkID) for
// every chunk. This happens here — before any upload — so uploadChunkImage
- // can read ck["id"] without deriving it itself. Downstream, the persist
- // stage reuses the same formula as a fallback when ck["id"] is absent.
+ // can read ck["id"] without deriving it itself. The id formula itself
+ // lives in canonicalChunkID (common.go); this loop only re-derives it as
+ // a fallback for chunks whose id was not already stamped upstream.
+ //
+ // A chunk id may already be stamped (e.g. by the streamed crop-upload
+ // path, which keys MinIO by the canonical id). Reuse it so the id is
+ // computed exactly once; both routes normalize the text through
+ // canonicalChunkText, so the crop-time key and this id are identical
+ // and cannot diverge. For chunks that bypassed crop, fall back to
+ // deriving it here from the already-finalized text.
for _, ck := range chunks {
+ if id, ok := ck["id"]; ok && id != "" {
+ continue
+ }
text, err := requireChunkText(ck)
if err != nil {
return nil, err
diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.go
index c60b8c7ce9..6648dd6b6d 100644
--- a/internal/ingestion/component/schema/chunk_types.go
+++ b/internal/ingestion/component/schema/chunk_types.go
@@ -105,6 +105,7 @@ type ChunkDoc struct {
TKNums *int `json:"tk_nums,omitempty"`
Mom string `json:"mom,omitempty"`
ImgID string `json:"img_id,omitempty"`
+ ID string `json:"id,omitempty"`
Layout string `json:"layout,omitempty"`
LayoutType string `json:"layout_type,omitempty"`
LayoutNo string `json:"layoutno,omitempty"`