RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): keep PDF positions on all cap-split sub-chunks so chunks don't lose their preview images (#18772)
euvre5756ec49d258
diff --git a/internal/ingestion/component/chunker/title_cap.go b/internal/ingestion/component/chunker/title_cap.goindex 17b7c6b8c8..7a272fde5b 100644--- a/internal/ingestion/component/chunker/title_cap.go+++ b/internal/ingestion/component/chunker/title_cap.go@@ -20,12 +20,11 @@ package chunker// A built chunk that exceeds `chunk_token_cap` tokens is re-split on sentence// boundaries into <= cap sub-chunks; a single boundary-less run that still// exceeds the cap is hard-split so the ceiling always holds. Table/image-// chunks are atomic and never split. The first sub-chunk keeps the source-// chunk's merged PDF position matrix (Plan A); subsequent sub-chunks carry an-// empty array.+// chunks are atomic and never split. Every sub-chunk keeps the source chunk's+// merged PDF position matrix so each one still gets its page-region preview+// image and position highlight.import (- "encoding/json""strings""unicode/utf8"@@ -165,8 +164,10 @@ func enforceTitleTokenCap(chunks []map[string]any, cap int) []map[string]any {// splitTitleChunkByCap re-splits one oversized text chunk into <= cap// sub-chunks. Sentences are tried first (greedy grouping); any remaining-// over-cap segment is hard-split. The first sub-chunk keeps the source-// chunk's merged PDF position matrix (Plan A); the rest carry an empty array.+// over-cap segment is hard-split. Every sub-chunk keeps the source chunk's+// merged PDF position matrix (the coordinates are coarse, source-chunk+// level), so the on-demand crop pass can attach a preview image to each+// sub-chunk.func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {text := toString(chunk["text"])sentences := titleSentenceSplit(text)@@ -203,32 +204,19 @@ func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {}out := make([]map[string]any, 0, len(finalGroups))- for i, g := range finalGroups {+ for _, g := range finalGroups {sub := shallowCopyChunk(chunk)sub["text"] = g- // Plan A: only the first sub-chunk keeps the position matrix; later- // sub-chunks carry an empty matrix of the same type, or drop the key- // when the source type is unknown.- if i > 0 {- if empty, ok := emptyPositionsLike(chunk["_pdf_positions"]); ok {- sub["_pdf_positions"] = empty- } else {- delete(sub, "_pdf_positions")- }- if empty, ok := emptyPositionsLike(chunk["positions"]); ok {- sub["positions"] = empty- } else {- delete(sub, "positions")- }- }out = append(out, sub)}return out}// shallowCopyChunk returns a shallow copy of c. Coordinate matrices-// ([][]float64) are shared by reference, but splitTitleChunkByCap replaces-// (never mutates) them for sub-chunks, so the source chunk is never aliased.+// ([][]float64) are deliberately shared by reference between the source+// chunk and its cap-split sub-chunks: nothing in the split or the downstream+// read-only passes (on-demand crop, position indexing) mutates a matrix, so+// the sharing is intentional and safe.func shallowCopyChunk(c map[string]any) map[string]any {m := make(map[string]any, len(c))for k, v := range c {@@ -237,22 +225,6 @@ func shallowCopyChunk(c map[string]any) map[string]any {return m}-// emptyPositionsLike returns an empty coordinate array of the same Go type as-// the source value (mergePositionMatrix produces [][]float64; tests may feed-// json.RawMessage), so a Plan-A sub-chunk's position field stays type-consistent-// with the first sub-chunk's. The second result is false for unknown types;-// the caller then DELETES the key instead of storing a nil-valued one.-func emptyPositionsLike(v any) (any, bool) {- switch v.(type) {- case [][]float64:- return [][]float64{}, true- case json.RawMessage:- return json.RawMessage("[]"), true- default:- return nil, false- }-}-// toStringOrDefault returns the string value of v, or def when v is absent /// nil / not a string. Used to treat a missing doc_type_kwd as "text".func toStringOrDefault(v any, def string) string {diff --git a/internal/ingestion/component/chunker/title_token_cap_test.go b/internal/ingestion/component/chunker/title_token_cap_test.goindex 1c2b9a67e9..5939f96963 100644--- a/internal/ingestion/component/chunker/title_token_cap_test.go+++ b/internal/ingestion/component/chunker/title_token_cap_test.go@@ -273,7 +273,7 @@ func TestHardSplitByTokens_EnglishRemainderStaysWhole(t *testing.T) {}}-func TestEnforceTitleTokenCap_PlanAPositions(t *testing.T) {+func TestEnforceTitleTokenCap_SubChunksKeepPositions(t *testing.T) {charTokenizer()defer restoreTokenizer()pos := [][]float64{{1, 10, 200, 50, 80}}@@ -285,30 +285,27 @@ func TestEnforceTitleTokenCap_PlanAPositions(t *testing.T) {if len(got) <= 1 {t.Fatalf("expected split, got %d", len(got))}- // First sub-chunk keeps the original position matrix.- first, ok := got[0]["positions"].([][]float64)- if !ok || len(first) == 0 || first[0][0] != 1 {- t.Errorf("first sub-chunk positions = %#v, want [[1 10 200 50 80]]", got[0]["positions"])- }- // Remaining sub-chunks carry an empty matrix (Plan A).- for i := 1; i < len(got); i++ {+ // Every sub-chunk keeps the original position matrix: the on-demand crop+ // pass and the position index derive from it.+ for i := range got {v, ok := got[i]["positions"].([][]float64)- if !ok {- t.Errorf("sub-chunk %d positions type = %T, want [][]float64", i, got[i]["positions"])+ if !ok || len(v) == 0 {+ t.Errorf("sub-chunk %d positions = %#v, want [[1 10 200 50 80]]", i, got[i]["positions"])continue}- if len(v) != 0 {- t.Errorf("sub-chunk %d positions = %#v, want empty", i, v)+ if v[0][0] != 1 {+ t.Errorf("sub-chunk %d positions = %#v, want the original matrix", i, v)}}}-func TestEnforceTitleTokenCap_PlanA_UnknownPositionsType(t *testing.T) {+func TestEnforceTitleTokenCap_UnknownPositionsTypeCarriedAsIs(t *testing.T) {charTokenizer()defer restoreTokenizer()body := strings.Join(joinSentences(12), "")- // An unknown position value type (not [][]float64 / json.RawMessage): the- // key must be DROPPED from later sub-chunks, never stored as a nil value.+ // An unknown position value type (not [][]float64 / json.RawMessage) is+ // shallow-copied to every sub-chunk verbatim; the cap split never inspects+ // or normalizes position payloads.chunks := []map[string]any{{"text": body, "positions": "not-a-matrix"},}@@ -316,13 +313,10 @@ func TestEnforceTitleTokenCap_PlanA_UnknownPositionsType(t *testing.T) {if len(got) <= 1 {t.Fatalf("expected split, got %d", len(got))}- // First sub-chunk keeps the value via shallow copy.- if _, ok := got[0]["positions"]; !ok {- t.Error("first sub-chunk lost its positions key")- }- for i := 1; i < len(got); i++ {- if _, ok := got[i]["positions"]; ok {- t.Errorf("sub-chunk %d kept unknown-type positions key, want deleted", i)+ for i := range got {+ v, ok := got[i]["positions"].(string)+ if !ok || v != "not-a-matrix" {+ t.Errorf("sub-chunk %d positions = %#v, want the source value carried as-is", i, got[i]["positions"])}}}@@ -508,7 +502,7 @@ func TestTitleCap_GroupPipeline_MultiRecordMerged(t *testing.T) {assertCapInvariants(t, chunks, 10, want.String())}-func TestTitleCap_HierarchyPipeline_PlanAPositions(t *testing.T) {+func TestTitleCap_HierarchyPipeline_SubChunksKeepPositions(t *testing.T) {charTokenizer()defer restoreTokenizer()body := strings.Join(joinSentences(12), "")@@ -527,21 +521,16 @@ func TestTitleCap_HierarchyPipeline_PlanAPositions(t *testing.T) {if len(chunks) <= 1 {t.Fatalf("expected split, got %d", len(chunks))}- first := chunks[0]["positions"]- if first == nil {- t.Fatalf("first sub-chunk missing positions")- }- if fm, ok := first.([][]float64); !ok || len(fm) == 0 || fm[0][0] != 1 {- t.Errorf("first positions = %#v, want [[1 10 200 50 80]]", first)- }- for i := 1; i < len(chunks); i++ {+ // Every sub-chunk keeps the source position matrix verbatim so the crop+ // pass can attach a preview image to each of them.+ for i := range chunks {v, ok := chunks[i]["positions"]if !ok {t.Errorf("sub-chunk %d missing positions", i)continue}- if vm, ok := v.([][]float64); !ok || len(vm) != 0 {- t.Errorf("sub-chunk %d positions = %#v, want empty [][]float64", i, v)+ if vm, ok := v.([][]float64); !ok || len(vm) == 0 || vm[0][0] != 1 {+ t.Errorf("sub-chunk %d positions = %#v, want [[1 10 200 50 80]]", i, v)}}}diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex e66f48d091..670a161774 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -1031,9 +1031,9 @@ func mergeByTokenSizeFromJSON(perItem [][]schema.ChunkDoc, chunkTokens int, over// so every returned text unit is <= target tokens. Sentence boundaries are// tried first (delimiters preserved, lossless); a piece that still exceeds// target is hard-split on token count (char-prefix via TrimContentToTokenLimit,-// also lossless). PDF positions follow Plan A: the original (coarse)-// coordinates attach to the first sub-piece only. Non-text units pass through-// unchanged.+// also lossless). Every sub-piece keeps the source unit's (coarse) PDF+// positions so each piece still gets its page-region preview image and+// highlight. Non-text units pass through unchanged.func expandOversizedUnits(units []schema.ChunkDoc, target int) []schema.ChunkDoc {if target <= 0 {return units@@ -1064,8 +1064,8 @@ func expandOversizedUnits(units []schema.ChunkDoc, target int) []schema.ChunkDoc// Sentence boundaries are preferred; a boundary-less run that still exceeds// target is hard token-split. Delimiters are preserved so the concatenated// pieces reproduce the unit text exactly (lossless). Every piece keeps the-// source unit's DocType, and PDF positions follow Plan A (original coordinates-// on the first piece only).+// source unit's DocType and (coarse) PDF positions, built from a clone of the+// source so all ChunkDoc fields survive the split.func splitOversizedText(ck schema.ChunkDoc, target int) []schema.ChunkDoc {text := ck.Text// Preserve a leading "\n" glue (text-path units are built as "\n"+part).@@ -1131,14 +1131,16 @@ func splitOversizedText(ck schema.ChunkDoc, target int) []schema.ChunkDoc {pieces[0].TKNums = intPtr(n)}}- // Plan A: the first sub-piece inherits the source unit's metadata (coarse- // positions + item attributes); later sub-pieces keep only the basics.- // Build it from a clone so every ChunkDoc field survives the split.- head := cloneChunkDoc(ck)- head.Text = pieces[0].Text- head.TKNums = pieces[0].TKNums- head.CKType = "text"- pieces[0] = head+ // Every sub-piece inherits the source unit's metadata (coarse positions ++ // item attributes); each is built from a clone so every ChunkDoc field+ // survives the split and each piece keeps its page-region preview.+ for i := range pieces {+ inherited := cloneChunkDoc(ck)+ inherited.Text = pieces[i].Text+ inherited.TKNums = pieces[i].TKNums+ inherited.CKType = "text"+ pieces[i] = inherited+ }return pieces}diff --git a/internal/ingestion/component/chunker/token_hardcap_test.go b/internal/ingestion/component/chunker/token_hardcap_test.goindex c71e15d498..ad01cf31fa 100644--- a/internal/ingestion/component/chunker/token_hardcap_test.go+++ b/internal/ingestion/component/chunker/token_hardcap_test.go@@ -131,10 +131,10 @@ func TestMergeUnits_ExpandsOversizedSentence(t *testing.T) {}}-// TestMergeUnits_PlanAPositions verifies Plan A: original (coarse) positions-// attach to the first sub-chunk of an expanded unit only; later sub-chunks-// carry none.-func TestMergeUnits_PlanAPositions(t *testing.T) {+// TestMergeUnits_SubChunksKeepPositions verifies that the original (coarse)+// positions attach to every sub-chunk of an expanded unit, so each piece+// still gets its page-region preview image.+func TestMergeUnits_SubChunksKeepPositions(t *testing.T) {const target = 30body := strings.Repeat("word ", 100)pos := []byte(`[{"page":1}]`)@@ -144,12 +144,9 @@ func TestMergeUnits_PlanAPositions(t *testing.T) {if len(got) < 2 {t.Fatalf("want >=2 chunks, got %d", len(got))}- if len(got[0].PDFPositions) == 0 {- t.Errorf("first sub-chunk must carry the original positions")- }- for i := 1; i < len(got); i++ {- if len(got[i].PDFPositions) != 0 {- t.Errorf("sub-chunk %d must carry no positions", i)+ for i := range got {+ if string(got[i].PDFPositions) != string(pos) {+ t.Errorf("sub-chunk %d must carry the original positions, got %q", i, got[i].PDFPositions)}}}@@ -255,11 +252,11 @@ func TestMergeUnits_ExpansionPreservesWhitespaceFragments(t *testing.T) {}}-// TestMergeUnits_ExpandedFirstPieceCarriesMetadata verifies Plan A metadata-// inheritance: the first sub-piece of an expanded unit keeps every field the-// source unit carried (coarse positions plus item attributes), while later-// sub-pieces keep only the basics.-func TestMergeUnits_ExpandedFirstPieceCarriesMetadata(t *testing.T) {+// TestMergeUnits_ExpandedPiecesCarryMetadata verifies metadata inheritance:+// every sub-piece of an expanded unit keeps the fields the source unit+// carried (coarse positions plus item attributes), built from a clone of the+// source unit.+func TestMergeUnits_ExpandedPiecesCarryMetadata(t *testing.T) {const target = 30body := strings.Repeat("word ", 100)unit := schema.ChunkDoc{@@ -280,28 +277,21 @@ func TestMergeUnits_ExpandedFirstPieceCarriesMetadata(t *testing.T) {if len(got) < 2 {t.Fatalf("oversized unit must be split, got %d chunk(s)", len(got))}- head := got[0]- if head.Mom != "parent-id" || head.ImgID != "img-1" || head.Layout != "text" || head.Image != "raw-image" {- t.Errorf("first sub-piece lost item metadata: %#v", head)- }- if head.PageNumber == nil || *head.PageNumber != 3 {- t.Errorf("first sub-piece lost PageNumber: %#v", head.PageNumber)- }- if len(head.TagKwd) != 1 || head.TagKwd[0] != "t1" {- t.Errorf("first sub-piece lost TagKwd: %#v", head.TagKwd)- }- if head.ChunkOrderInt == nil || *head.ChunkOrderInt != 7 {- t.Errorf("first sub-piece lost ChunkOrderInt: %#v", head.ChunkOrderInt)- }- if len(head.PDFPositions) == 0 {- t.Errorf("first sub-piece must carry the original positions")- }- for i := 1; i < len(got); i++ {- if len(got[i].PDFPositions) != 0 {- t.Errorf("sub-piece %d must carry no positions", i)+ for i, ck := range got {+ if ck.Mom != "parent-id" || ck.ImgID != "img-1" || ck.Layout != "text" || ck.Image != "raw-image" {+ t.Errorf("sub-piece %d lost item metadata: %#v", i, ck)+ }+ if ck.PageNumber == nil || *ck.PageNumber != 3 {+ t.Errorf("sub-piece %d lost PageNumber: %#v", i, ck.PageNumber)+ }+ if len(ck.TagKwd) != 1 || ck.TagKwd[0] != "t1" {+ t.Errorf("sub-piece %d lost TagKwd: %#v", i, ck.TagKwd)+ }+ if ck.ChunkOrderInt == nil || *ck.ChunkOrderInt != 7 {+ t.Errorf("sub-piece %d lost ChunkOrderInt: %#v", i, ck.ChunkOrderInt)}- if got[i].Mom != "" || got[i].ImgID != "" || got[i].Layout != "" {- t.Errorf("sub-piece %d must not carry item metadata", i)+ if len(ck.PDFPositions) == 0 {+ t.Errorf("sub-piece %d must carry the original positions", i)}}}diff --git a/rag/flow/chunker/title_chunker/common.py b/rag/flow/chunker/title_chunker/common.pyindex fcdbfdae01..c212e1461d 100644--- a/rag/flow/chunker/title_chunker/common.py+++ b/rag/flow/chunker/title_chunker/common.py@@ -151,8 +151,8 @@ class BaseTitleChunker(ABC):"""Re-split one oversized text chunk into <= cap sub-chunks.Sentence boundaries are tried first; any remaining over-cap segment is- hard-split. PDF coordinates follow plan A: the original (coarse,- page-level) positions attach to the first sub-chunk only.+ hard-split. Every sub-chunk keeps the original (coarse, page-level)+ positions so each still gets its preview image and highlight."""text = chunk.get("text") or ""sentences = self._split_text_by_sentences(text)@@ -183,13 +183,15 @@ class BaseTitleChunker(ABC):has_positions = PDF_POSITIONS_KEY in chunkorig_positions = chunk.get(PDF_POSITIONS_KEY)out = []- for i, group in enumerate(final_groups):+ for group in final_groups:sub = dict(chunk)sub["text"] = group- # Plan A: the original (coarse, page-level) coordinates attach to the- # first sub-chunk only; subsequent sub-chunks carry none.+ # Every sub-chunk keeps the original (coarse, page-level)+ # coordinates so the preview-image/position restore pass covers+ # all sub-chunks. Each sub-chunk owns a deep copy so no two of+ # them alias the same position list.if has_positions:- sub[PDF_POSITIONS_KEY] = orig_positions if i == 0 else []+ sub[PDF_POSITIONS_KEY] = deepcopy(orig_positions)out.append(sub)return outdiff --git a/rag/flow/tests/test_title_chunker_token_cap.py b/rag/flow/tests/test_title_chunker_token_cap.pyindex c925ab2e61..3239dac485 100644--- a/rag/flow/tests/test_title_chunker_token_cap.py+++ b/rag/flow/tests/test_title_chunker_token_cap.py@@ -354,9 +354,9 @@ def test_boundaryless_run_is_hard_split():# --------------------------------------------------------------------------- #-# 7. PDF positions: plan A - first sub-chunk keeps coordinates, rest empty #+# 7. PDF positions: every sub-chunk keeps the source coordinates ## --------------------------------------------------------------------------- #-def test_position_plan_a_first_subchunk_keeps_coordinates():+def test_position_all_subchunks_keep_coordinates():body = "。".join(f"S{i:02d}" for i in range(12)) + "。"items = [{"text": body, "doc_type_kwd": "text", "positions": [[1, 10, 200, 50, 80]]}]chunks = _run(@@ -368,9 +368,48 @@ def test_position_plan_a_first_subchunk_keeps_coordinates():)assert len(chunks) > 1- assert chunks[0].get("position_int") == [[1, 10, 200, 50, 80]], "first sub-chunk must keep coordinates"- for ck in chunks[1:]:- assert "position_int" not in ck, "only the first sub-chunk should carry coordinates"+ for ck in chunks:+ assert ck.get("position_int") == [[1, 10, 200, 50, 80]], "every sub-chunk must keep the source coordinates"+++# --------------------------------------------------------------------------- #+# 7d. sub-chunk position lists are independent deep copies (no aliasing) #+# --------------------------------------------------------------------------- #+def test_split_subchunk_positions_not_aliased():+ # _split_text_chunk_by_cap must give every sub-chunk its own deep copy of+ # the source position list: sharing one list object across sub-chunks+ # would let a future in-place mutation silently corrupt all of them.+ body = "。".join(f"S{i:02d}" for i in range(12)) + "。"+ source_positions = [[1, 10, 200, 50, 80], [2, 15, 190, 60, 90]]++ with _load_title_chunker_with_stubs() as (common_module, _hierarchy_module, _group_module):+ key = common_module.PDF_POSITIONS_KEY++ class _Splitter(common_module.BaseTitleChunker):+ # _split_text_chunk_by_cap uses no instance state, so skip the+ # process/from_upstream wiring of the real constructor.+ def __init__(self):+ pass++ def resolve_levels(self, line_records):+ return None++ def build_chunks(self, line_records, resolved):+ return []++ chunker = _Splitter()+ chunk = {"text": body, "doc_type_kwd": "text", key: source_positions}+ subs = common_module.BaseTitleChunker._split_text_chunk_by_cap(chunker, chunk, 20)++ assert len(subs) > 1+ for sub in subs:+ assert sub[key] == source_positions, "every sub-chunk keeps the source coordinates"+ assert sub[key] is not source_positions, "sub-chunk must not alias the source position list"+ # Deep independence: mutating one sub-chunk's nested coordinates must not+ # leak into the source chunk or any other sub-chunk.+ subs[0][key][0][0] = 999+ assert source_positions[0][0] == 1+ assert all(sub[key][0][0] == 1 for sub in subs[1:])# --------------------------------------------------------------------------- #