Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): keep PDF positions on all cap-split sub-chunks so chunks don't lose their preview images (#18772)

euvre5756ec49d258
diff --git a/internal/ingestion/component/chunker/title_cap.go b/internal/ingestion/component/chunker/title_cap.go
index 17b7c6b8c8..7a272fde5b 100644
--- a/internal/ingestion/component/chunker/title_cap.go
+++ b/internal/ingestion/component/chunker/title_cap.go
@@ -20,12 +20,11 @@ package chunker
// A built chunk that exceeds `chunk_token_cap` tokens is re-split on sentence
// boundaries into <= cap sub-chunks; a single boundary-less run that still
// exceeds the cap is hard-split so the ceiling always holds. Table/image
-// chunks are atomic and never split. The first sub-chunk keeps the source
-// chunk's merged PDF position matrix (Plan A); subsequent sub-chunks carry an
-// empty array.
+// chunks are atomic and never split. Every sub-chunk keeps the source chunk's
+// merged PDF position matrix so each one still gets its page-region preview
+// image and position highlight.
import (
- "encoding/json"
"strings"
"unicode/utf8"
@@ -165,8 +164,10 @@ func enforceTitleTokenCap(chunks []map[string]any, cap int) []map[string]any {
// splitTitleChunkByCap re-splits one oversized text chunk into <= cap
// sub-chunks. Sentences are tried first (greedy grouping); any remaining
-// over-cap segment is hard-split. The first sub-chunk keeps the source
-// chunk's merged PDF position matrix (Plan A); the rest carry an empty array.
+// over-cap segment is hard-split. Every sub-chunk keeps the source chunk's
+// merged PDF position matrix (the coordinates are coarse, source-chunk
+// level), so the on-demand crop pass can attach a preview image to each
+// sub-chunk.
func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {
text := toString(chunk["text"])
sentences := titleSentenceSplit(text)
@@ -203,32 +204,19 @@ func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {
}
out := make([]map[string]any, 0, len(finalGroups))
- for i, g := range finalGroups {
+ for _, g := range finalGroups {
sub := shallowCopyChunk(chunk)
sub["text"] = g
- // Plan A: only the first sub-chunk keeps the position matrix; later
- // sub-chunks carry an empty matrix of the same type, or drop the key
- // when the source type is unknown.
- if i > 0 {
- if empty, ok := emptyPositionsLike(chunk["_pdf_positions"]); ok {
- sub["_pdf_positions"] = empty
- } else {
- delete(sub, "_pdf_positions")
- }
- if empty, ok := emptyPositionsLike(chunk["positions"]); ok {
- sub["positions"] = empty
- } else {
- delete(sub, "positions")
- }
- }
out = append(out, sub)
}
return out
}
// shallowCopyChunk returns a shallow copy of c. Coordinate matrices
-// ([][]float64) are shared by reference, but splitTitleChunkByCap replaces
-// (never mutates) them for sub-chunks, so the source chunk is never aliased.
+// ([][]float64) are deliberately shared by reference between the source
+// chunk and its cap-split sub-chunks: nothing in the split or the downstream
+// read-only passes (on-demand crop, position indexing) mutates a matrix, so
+// the sharing is intentional and safe.
func shallowCopyChunk(c map[string]any) map[string]any {
m := make(map[string]any, len(c))
for k, v := range c {
@@ -237,22 +225,6 @@ func shallowCopyChunk(c map[string]any) map[string]any {
return m
}
-// emptyPositionsLike returns an empty coordinate array of the same Go type as
-// the source value (mergePositionMatrix produces [][]float64; tests may feed
-// json.RawMessage), so a Plan-A sub-chunk's position field stays type-consistent
-// with the first sub-chunk's. The second result is false for unknown types;
-// the caller then DELETES the key instead of storing a nil-valued one.
-func emptyPositionsLike(v any) (any, bool) {
- switch v.(type) {
- case [][]float64:
- return [][]float64{}, true
- case json.RawMessage:
- return json.RawMessage("[]"), true
- default:
- return nil, false
- }
-}
-
// toStringOrDefault returns the string value of v, or def when v is absent /
// nil / not a string. Used to treat a missing doc_type_kwd as "text".
func toStringOrDefault(v any, def string) string {
diff --git a/internal/ingestion/component/chunker/title_token_cap_test.go b/internal/ingestion/component/chunker/title_token_cap_test.go
index 1c2b9a67e9..5939f96963 100644
--- a/internal/ingestion/component/chunker/title_token_cap_test.go
+++ b/internal/ingestion/component/chunker/title_token_cap_test.go
@@ -273,7 +273,7 @@ func TestHardSplitByTokens_EnglishRemainderStaysWhole(t *testing.T) {
}
}
-func TestEnforceTitleTokenCap_PlanAPositions(t *testing.T) {
+func TestEnforceTitleTokenCap_SubChunksKeepPositions(t *testing.T) {
charTokenizer()
defer restoreTokenizer()
pos := [][]float64{{1, 10, 200, 50, 80}}
@@ -285,30 +285,27 @@ func TestEnforceTitleTokenCap_PlanAPositions(t *testing.T) {
if len(got) <= 1 {
t.Fatalf("expected split, got %d", len(got))
}
- // First sub-chunk keeps the original position matrix.
- first, ok := got[0]["positions"].([][]float64)
- if !ok || len(first) == 0 || first[0][0] != 1 {
- t.Errorf("first sub-chunk positions = %#v, want [[1 10 200 50 80]]", got[0]["positions"])
- }
- // Remaining sub-chunks carry an empty matrix (Plan A).
- for i := 1; i < len(got); i++ {
+ // Every sub-chunk keeps the original position matrix: the on-demand crop
+ // pass and the position index derive from it.
+ for i := range got {
v, ok := got[i]["positions"].([][]float64)
- if !ok {
- t.Errorf("sub-chunk %d positions type = %T, want [][]float64", i, got[i]["positions"])
+ if !ok || len(v) == 0 {
+ t.Errorf("sub-chunk %d positions = %#v, want [[1 10 200 50 80]]", i, got[i]["positions"])
continue
}
- if len(v) != 0 {
- t.Errorf("sub-chunk %d positions = %#v, want empty", i, v)
+ if v[0][0] != 1 {
+ t.Errorf("sub-chunk %d positions = %#v, want the original matrix", i, v)
}
}
}
-func TestEnforceTitleTokenCap_PlanA_UnknownPositionsType(t *testing.T) {
+func TestEnforceTitleTokenCap_UnknownPositionsTypeCarriedAsIs(t *testing.T) {
charTokenizer()
defer restoreTokenizer()
body := strings.Join(joinSentences(12), "")
- // An unknown position value type (not [][]float64 / json.RawMessage): the
- // key must be DROPPED from later sub-chunks, never stored as a nil value.
+ // An unknown position value type (not [][]float64 / json.RawMessage) is
+ // shallow-copied to every sub-chunk verbatim; the cap split never inspects
+ // or normalizes position payloads.
chunks := []map[string]any{
{"text": body, "positions": "not-a-matrix"},
}
@@ -316,13 +313,10 @@ func TestEnforceTitleTokenCap_PlanA_UnknownPositionsType(t *testing.T) {
if len(got) <= 1 {
t.Fatalf("expected split, got %d", len(got))
}
- // First sub-chunk keeps the value via shallow copy.
- if _, ok := got[0]["positions"]; !ok {
- t.Error("first sub-chunk lost its positions key")
- }
- for i := 1; i < len(got); i++ {
- if _, ok := got[i]["positions"]; ok {
- t.Errorf("sub-chunk %d kept unknown-type positions key, want deleted", i)
+ for i := range got {
+ v, ok := got[i]["positions"].(string)
+ if !ok || v != "not-a-matrix" {
+ t.Errorf("sub-chunk %d positions = %#v, want the source value carried as-is", i, got[i]["positions"])
}
}
}
@@ -508,7 +502,7 @@ func TestTitleCap_GroupPipeline_MultiRecordMerged(t *testing.T) {
assertCapInvariants(t, chunks, 10, want.String())
}
-func TestTitleCap_HierarchyPipeline_PlanAPositions(t *testing.T) {
+func TestTitleCap_HierarchyPipeline_SubChunksKeepPositions(t *testing.T) {
charTokenizer()
defer restoreTokenizer()
body := strings.Join(joinSentences(12), "")
@@ -527,21 +521,16 @@ func TestTitleCap_HierarchyPipeline_PlanAPositions(t *testing.T) {
if len(chunks) <= 1 {
t.Fatalf("expected split, got %d", len(chunks))
}
- first := chunks[0]["positions"]
- if first == nil {
- t.Fatalf("first sub-chunk missing positions")
- }
- if fm, ok := first.([][]float64); !ok || len(fm) == 0 || fm[0][0] != 1 {
- t.Errorf("first positions = %#v, want [[1 10 200 50 80]]", first)
- }
- for i := 1; i < len(chunks); i++ {
+ // Every sub-chunk keeps the source position matrix verbatim so the crop
+ // pass can attach a preview image to each of them.
+ for i := range chunks {
v, ok := chunks[i]["positions"]
if !ok {
t.Errorf("sub-chunk %d missing positions", i)
continue
}
- if vm, ok := v.([][]float64); !ok || len(vm) != 0 {
- t.Errorf("sub-chunk %d positions = %#v, want empty [][]float64", i, v)
+ if vm, ok := v.([][]float64); !ok || len(vm) == 0 || vm[0][0] != 1 {
+ t.Errorf("sub-chunk %d positions = %#v, want [[1 10 200 50 80]]", i, v)
}
}
}
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index e66f48d091..670a161774 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -1031,9 +1031,9 @@ func mergeByTokenSizeFromJSON(perItem [][]schema.ChunkDoc, chunkTokens int, over
// so every returned text unit is <= target tokens. Sentence boundaries are
// tried first (delimiters preserved, lossless); a piece that still exceeds
// target is hard-split on token count (char-prefix via TrimContentToTokenLimit,
-// also lossless). PDF positions follow Plan A: the original (coarse)
-// coordinates attach to the first sub-piece only. Non-text units pass through
-// unchanged.
+// also lossless). Every sub-piece keeps the source unit's (coarse) PDF
+// positions so each piece still gets its page-region preview image and
+// highlight. Non-text units pass through unchanged.
func expandOversizedUnits(units []schema.ChunkDoc, target int) []schema.ChunkDoc {
if target <= 0 {
return units
@@ -1064,8 +1064,8 @@ func expandOversizedUnits(units []schema.ChunkDoc, target int) []schema.ChunkDoc
// Sentence boundaries are preferred; a boundary-less run that still exceeds
// target is hard token-split. Delimiters are preserved so the concatenated
// pieces reproduce the unit text exactly (lossless). Every piece keeps the
-// source unit's DocType, and PDF positions follow Plan A (original coordinates
-// on the first piece only).
+// source unit's DocType and (coarse) PDF positions, built from a clone of the
+// source so all ChunkDoc fields survive the split.
func splitOversizedText(ck schema.ChunkDoc, target int) []schema.ChunkDoc {
text := ck.Text
// Preserve a leading "\n" glue (text-path units are built as "\n"+part).
@@ -1131,14 +1131,16 @@ func splitOversizedText(ck schema.ChunkDoc, target int) []schema.ChunkDoc {
pieces[0].TKNums = intPtr(n)
}
}
- // Plan A: the first sub-piece inherits the source unit's metadata (coarse
- // positions + item attributes); later sub-pieces keep only the basics.
- // Build it from a clone so every ChunkDoc field survives the split.
- head := cloneChunkDoc(ck)
- head.Text = pieces[0].Text
- head.TKNums = pieces[0].TKNums
- head.CKType = "text"
- pieces[0] = head
+ // Every sub-piece inherits the source unit's metadata (coarse positions +
+ // item attributes); each is built from a clone so every ChunkDoc field
+ // survives the split and each piece keeps its page-region preview.
+ for i := range pieces {
+ inherited := cloneChunkDoc(ck)
+ inherited.Text = pieces[i].Text
+ inherited.TKNums = pieces[i].TKNums
+ inherited.CKType = "text"
+ pieces[i] = inherited
+ }
return pieces
}
diff --git a/internal/ingestion/component/chunker/token_hardcap_test.go b/internal/ingestion/component/chunker/token_hardcap_test.go
index c71e15d498..ad01cf31fa 100644
--- a/internal/ingestion/component/chunker/token_hardcap_test.go
+++ b/internal/ingestion/component/chunker/token_hardcap_test.go
@@ -131,10 +131,10 @@ func TestMergeUnits_ExpandsOversizedSentence(t *testing.T) {
}
}
-// TestMergeUnits_PlanAPositions verifies Plan A: original (coarse) positions
-// attach to the first sub-chunk of an expanded unit only; later sub-chunks
-// carry none.
-func TestMergeUnits_PlanAPositions(t *testing.T) {
+// TestMergeUnits_SubChunksKeepPositions verifies that the original (coarse)
+// positions attach to every sub-chunk of an expanded unit, so each piece
+// still gets its page-region preview image.
+func TestMergeUnits_SubChunksKeepPositions(t *testing.T) {
const target = 30
body := strings.Repeat("word ", 100)
pos := []byte(`[{"page":1}]`)
@@ -144,12 +144,9 @@ func TestMergeUnits_PlanAPositions(t *testing.T) {
if len(got) < 2 {
t.Fatalf("want >=2 chunks, got %d", len(got))
}
- if len(got[0].PDFPositions) == 0 {
- t.Errorf("first sub-chunk must carry the original positions")
- }
- for i := 1; i < len(got); i++ {
- if len(got[i].PDFPositions) != 0 {
- t.Errorf("sub-chunk %d must carry no positions", i)
+ for i := range got {
+ if string(got[i].PDFPositions) != string(pos) {
+ t.Errorf("sub-chunk %d must carry the original positions, got %q", i, got[i].PDFPositions)
}
}
}
@@ -255,11 +252,11 @@ func TestMergeUnits_ExpansionPreservesWhitespaceFragments(t *testing.T) {
}
}
-// TestMergeUnits_ExpandedFirstPieceCarriesMetadata verifies Plan A metadata
-// inheritance: the first sub-piece of an expanded unit keeps every field the
-// source unit carried (coarse positions plus item attributes), while later
-// sub-pieces keep only the basics.
-func TestMergeUnits_ExpandedFirstPieceCarriesMetadata(t *testing.T) {
+// TestMergeUnits_ExpandedPiecesCarryMetadata verifies metadata inheritance:
+// every sub-piece of an expanded unit keeps the fields the source unit
+// carried (coarse positions plus item attributes), built from a clone of the
+// source unit.
+func TestMergeUnits_ExpandedPiecesCarryMetadata(t *testing.T) {
const target = 30
body := strings.Repeat("word ", 100)
unit := schema.ChunkDoc{
@@ -280,28 +277,21 @@ func TestMergeUnits_ExpandedFirstPieceCarriesMetadata(t *testing.T) {
if len(got) < 2 {
t.Fatalf("oversized unit must be split, got %d chunk(s)", len(got))
}
- head := got[0]
- if head.Mom != "parent-id" || head.ImgID != "img-1" || head.Layout != "text" || head.Image != "raw-image" {
- t.Errorf("first sub-piece lost item metadata: %#v", head)
- }
- if head.PageNumber == nil || *head.PageNumber != 3 {
- t.Errorf("first sub-piece lost PageNumber: %#v", head.PageNumber)
- }
- if len(head.TagKwd) != 1 || head.TagKwd[0] != "t1" {
- t.Errorf("first sub-piece lost TagKwd: %#v", head.TagKwd)
- }
- if head.ChunkOrderInt == nil || *head.ChunkOrderInt != 7 {
- t.Errorf("first sub-piece lost ChunkOrderInt: %#v", head.ChunkOrderInt)
- }
- if len(head.PDFPositions) == 0 {
- t.Errorf("first sub-piece must carry the original positions")
- }
- for i := 1; i < len(got); i++ {
- if len(got[i].PDFPositions) != 0 {
- t.Errorf("sub-piece %d must carry no positions", i)
+ for i, ck := range got {
+ if ck.Mom != "parent-id" || ck.ImgID != "img-1" || ck.Layout != "text" || ck.Image != "raw-image" {
+ t.Errorf("sub-piece %d lost item metadata: %#v", i, ck)
+ }
+ if ck.PageNumber == nil || *ck.PageNumber != 3 {
+ t.Errorf("sub-piece %d lost PageNumber: %#v", i, ck.PageNumber)
+ }
+ if len(ck.TagKwd) != 1 || ck.TagKwd[0] != "t1" {
+ t.Errorf("sub-piece %d lost TagKwd: %#v", i, ck.TagKwd)
+ }
+ if ck.ChunkOrderInt == nil || *ck.ChunkOrderInt != 7 {
+ t.Errorf("sub-piece %d lost ChunkOrderInt: %#v", i, ck.ChunkOrderInt)
}
- if got[i].Mom != "" || got[i].ImgID != "" || got[i].Layout != "" {
- t.Errorf("sub-piece %d must not carry item metadata", i)
+ if len(ck.PDFPositions) == 0 {
+ t.Errorf("sub-piece %d must carry the original positions", i)
}
}
}
diff --git a/rag/flow/chunker/title_chunker/common.py b/rag/flow/chunker/title_chunker/common.py
index fcdbfdae01..c212e1461d 100644
--- a/rag/flow/chunker/title_chunker/common.py
+++ b/rag/flow/chunker/title_chunker/common.py
@@ -151,8 +151,8 @@ class BaseTitleChunker(ABC):
"""Re-split one oversized text chunk into <= cap sub-chunks.
Sentence boundaries are tried first; any remaining over-cap segment is
- hard-split. PDF coordinates follow plan A: the original (coarse,
- page-level) positions attach to the first sub-chunk only.
+ hard-split. Every sub-chunk keeps the original (coarse, page-level)
+ positions so each still gets its preview image and highlight.
"""
text = chunk.get("text") or ""
sentences = self._split_text_by_sentences(text)
@@ -183,13 +183,15 @@ class BaseTitleChunker(ABC):
has_positions = PDF_POSITIONS_KEY in chunk
orig_positions = chunk.get(PDF_POSITIONS_KEY)
out = []
- for i, group in enumerate(final_groups):
+ for group in final_groups:
sub = dict(chunk)
sub["text"] = group
- # Plan A: the original (coarse, page-level) coordinates attach to the
- # first sub-chunk only; subsequent sub-chunks carry none.
+ # Every sub-chunk keeps the original (coarse, page-level)
+ # coordinates so the preview-image/position restore pass covers
+ # all sub-chunks. Each sub-chunk owns a deep copy so no two of
+ # them alias the same position list.
if has_positions:
- sub[PDF_POSITIONS_KEY] = orig_positions if i == 0 else []
+ sub[PDF_POSITIONS_KEY] = deepcopy(orig_positions)
out.append(sub)
return out
diff --git a/rag/flow/tests/test_title_chunker_token_cap.py b/rag/flow/tests/test_title_chunker_token_cap.py
index c925ab2e61..3239dac485 100644
--- a/rag/flow/tests/test_title_chunker_token_cap.py
+++ b/rag/flow/tests/test_title_chunker_token_cap.py
@@ -354,9 +354,9 @@ def test_boundaryless_run_is_hard_split():
# --------------------------------------------------------------------------- #
-# 7. PDF positions: plan A - first sub-chunk keeps coordinates, rest empty #
+# 7. PDF positions: every sub-chunk keeps the source coordinates #
# --------------------------------------------------------------------------- #
-def test_position_plan_a_first_subchunk_keeps_coordinates():
+def test_position_all_subchunks_keep_coordinates():
body = "。".join(f"S{i:02d}" for i in range(12)) + "。"
items = [{"text": body, "doc_type_kwd": "text", "positions": [[1, 10, 200, 50, 80]]}]
chunks = _run(
@@ -368,9 +368,48 @@ def test_position_plan_a_first_subchunk_keeps_coordinates():
)
assert len(chunks) > 1
- assert chunks[0].get("position_int") == [[1, 10, 200, 50, 80]], "first sub-chunk must keep coordinates"
- for ck in chunks[1:]:
- assert "position_int" not in ck, "only the first sub-chunk should carry coordinates"
+ for ck in chunks:
+ assert ck.get("position_int") == [[1, 10, 200, 50, 80]], "every sub-chunk must keep the source coordinates"
+
+
+# --------------------------------------------------------------------------- #
+# 7d. sub-chunk position lists are independent deep copies (no aliasing) #
+# --------------------------------------------------------------------------- #
+def test_split_subchunk_positions_not_aliased():
+ # _split_text_chunk_by_cap must give every sub-chunk its own deep copy of
+ # the source position list: sharing one list object across sub-chunks
+ # would let a future in-place mutation silently corrupt all of them.
+ body = "。".join(f"S{i:02d}" for i in range(12)) + "。"
+ source_positions = [[1, 10, 200, 50, 80], [2, 15, 190, 60, 90]]
+
+ with _load_title_chunker_with_stubs() as (common_module, _hierarchy_module, _group_module):
+ key = common_module.PDF_POSITIONS_KEY
+
+ class _Splitter(common_module.BaseTitleChunker):
+ # _split_text_chunk_by_cap uses no instance state, so skip the
+ # process/from_upstream wiring of the real constructor.
+ def __init__(self):
+ pass
+
+ def resolve_levels(self, line_records):
+ return None
+
+ def build_chunks(self, line_records, resolved):
+ return []
+
+ chunker = _Splitter()
+ chunk = {"text": body, "doc_type_kwd": "text", key: source_positions}
+ subs = common_module.BaseTitleChunker._split_text_chunk_by_cap(chunker, chunk, 20)
+
+ assert len(subs) > 1
+ for sub in subs:
+ assert sub[key] == source_positions, "every sub-chunk keeps the source coordinates"
+ assert sub[key] is not source_positions, "sub-chunk must not alias the source position list"
+ # Deep independence: mutating one sub-chunk's nested coordinates must not
+ # leak into the source chunk or any other sub-chunk.
+ subs[0][key][0][0] = 999
+ assert source_positions[0][0] == 1
+ assert all(sub[key][0][0] == 1 for sub in subs[1:])
# --------------------------------------------------------------------------- #