Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): preserve separators between JSON items (#20857)

lorenzozane0d1553f5427c
diff --git a/internal/ingestion/component/chunker/json_global_merge_test.go b/internal/ingestion/component/chunker/json_global_merge_test.go
index 2e51662905..0e2d619fc9 100644
--- a/internal/ingestion/component/chunker/json_global_merge_test.go
+++ b/internal/ingestion/component/chunker/json_global_merge_test.go
@@ -27,8 +27,8 @@ package chunker
// TestJSONGlobalMergeMatchesPython pins the TokenChunker json path's
// cross-item merge: adjacent text chunks across JSON items collapse into one
-// chunk when they jointly fit chunk_token_size, matching Python's
-// _merge_text_chunks_by_token_size.
+// chunk when they jointly fit chunk_token_size, with a separator at the item
+// boundary like Python's _merge_text_chunks_by_token_size.
import (
"context"
"strings"
@@ -83,6 +83,9 @@ func TestJSONGlobalMergeMatchesPython(t *testing.T) {
if i1 >= i2 {
t.Errorf("merged chunk reordered items (want %q before %q):\n%q", first, second, text)
}
+ if i2 == 0 || text[i2-1] != '\n' {
+ t.Errorf("items are missing a line break at their boundary:\n%q", text)
+ }
}
// TestJSONSingleItemSubSplitUnderHardCap pins the TokenChunker json path's
diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_multi_item_40.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_multi_item_40.json
index f2e58ed502..e0f21797fe 100644
--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_multi_item_40.json
+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_multi_item_40.json
@@ -3,31 +3,35 @@
"chunks": [
{
"doc_type_kwd": "text",
- "text": "Section 1. RAGFlow chunks documents for retrieval. This is sentence two of item 1.Section 2. RAGFlow chunks documents for retrieval. This is sentence two of item 2.Section 3. RAGFlow chunks documents for retrieval. This is sentence two of item 3.Section 4. RAGFlow chunks documents for retrieval. This is sentence two of item 4.Section 5. RAGFlow chunks documents for retrieval. This is sentence two of item 5.Section 6. RAGFlow chunks documents for retrieval. This is sentence two of item 6."
+ "text": "Section 1. RAGFlow chunks documents for retrieval. This is sentence two of item 1.\nSection 2. RAGFlow chunks documents for retrieval. This is sentence two of item 2.\nSection 3. RAGFlow chunks documents for retrieval. This is sentence two of item 3.\nSection 4. RAGFlow chunks documents for retrieval. This is sentence two of item 4.\nSection 5. RAGFlow chunks documents for retrieval. This is sentence two of item 5."
},
{
"doc_type_kwd": "text",
- "text": "Section 7. RAGFlow chunks documents for retrieval. This is sentence two of item 7.Section 8. RAGFlow chunks documents for retrieval. This is sentence two of item 8.Section 9. RAGFlow chunks documents for retrieval. This is sentence two of item 9.Section 10. RAGFlow chunks documents for retrieval. This is sentence two of item 10.Section 11. RAGFlow chunks documents for retrieval. This is sentence two of item 11.Section 12. RAGFlow chunks documents for retrieval. This is sentence two of item 12."
+ "text": "\nSection 6. RAGFlow chunks documents for retrieval. This is sentence two of item 6.\nSection 7. RAGFlow chunks documents for retrieval. This is sentence two of item 7.\nSection 8. RAGFlow chunks documents for retrieval. This is sentence two of item 8.\nSection 9. RAGFlow chunks documents for retrieval. This is sentence two of item 9.\nSection 10. RAGFlow chunks documents for retrieval. This is sentence two of item 10."
},
{
"doc_type_kwd": "text",
- "text": "Section 13. RAGFlow chunks documents for retrieval. This is sentence two of item 13.Section 14. RAGFlow chunks documents for retrieval. This is sentence two of item 14.Section 15. RAGFlow chunks documents for retrieval. This is sentence two of item 15.Section 16. RAGFlow chunks documents for retrieval. This is sentence two of item 16.Section 17. RAGFlow chunks documents for retrieval. This is sentence two of item 17.Section 18. RAGFlow chunks documents for retrieval. This is sentence two of item 18."
+ "text": "\nSection 11. RAGFlow chunks documents for retrieval. This is sentence two of item 11.\nSection 12. RAGFlow chunks documents for retrieval. This is sentence two of item 12.\nSection 13. RAGFlow chunks documents for retrieval. This is sentence two of item 13.\nSection 14. RAGFlow chunks documents for retrieval. This is sentence two of item 14.\nSection 15. RAGFlow chunks documents for retrieval. This is sentence two of item 15."
},
{
"doc_type_kwd": "text",
- "text": "Section 19. RAGFlow chunks documents for retrieval. This is sentence two of item 19.Section 20. RAGFlow chunks documents for retrieval. This is sentence two of item 20.Section 21. RAGFlow chunks documents for retrieval. This is sentence two of item 21.Section 22. RAGFlow chunks documents for retrieval. This is sentence two of item 22.Section 23. RAGFlow chunks documents for retrieval. This is sentence two of item 23.Section 24. RAGFlow chunks documents for retrieval. This is sentence two of item 24."
+ "text": "\nSection 16. RAGFlow chunks documents for retrieval. This is sentence two of item 16.\nSection 17. RAGFlow chunks documents for retrieval. This is sentence two of item 17.\nSection 18. RAGFlow chunks documents for retrieval. This is sentence two of item 18.\nSection 19. RAGFlow chunks documents for retrieval. This is sentence two of item 19.\nSection 20. RAGFlow chunks documents for retrieval. This is sentence two of item 20."
},
{
"doc_type_kwd": "text",
- "text": "Section 25. RAGFlow chunks documents for retrieval. This is sentence two of item 25.Section 26. RAGFlow chunks documents for retrieval. This is sentence two of item 26.Section 27. RAGFlow chunks documents for retrieval. This is sentence two of item 27.Section 28. RAGFlow chunks documents for retrieval. This is sentence two of item 28.Section 29. RAGFlow chunks documents for retrieval. This is sentence two of item 29.Section 30. RAGFlow chunks documents for retrieval. This is sentence two of item 30."
+ "text": "\nSection 21. RAGFlow chunks documents for retrieval. This is sentence two of item 21.\nSection 22. RAGFlow chunks documents for retrieval. This is sentence two of item 22.\nSection 23. RAGFlow chunks documents for retrieval. This is sentence two of item 23.\nSection 24. RAGFlow chunks documents for retrieval. This is sentence two of item 24.\nSection 25. RAGFlow chunks documents for retrieval. This is sentence two of item 25."
},
{
"doc_type_kwd": "text",
- "text": "Section 31. RAGFlow chunks documents for retrieval. This is sentence two of item 31.Section 32. RAGFlow chunks documents for retrieval. This is sentence two of item 32.Section 33. RAGFlow chunks documents for retrieval. This is sentence two of item 33.Section 34. RAGFlow chunks documents for retrieval. This is sentence two of item 34.Section 35. RAGFlow chunks documents for retrieval. This is sentence two of item 35.Section 36. RAGFlow chunks documents for retrieval. This is sentence two of item 36."
+ "text": "\nSection 26. RAGFlow chunks documents for retrieval. This is sentence two of item 26.\nSection 27. RAGFlow chunks documents for retrieval. This is sentence two of item 27.\nSection 28. RAGFlow chunks documents for retrieval. This is sentence two of item 28.\nSection 29. RAGFlow chunks documents for retrieval. This is sentence two of item 29.\nSection 30. RAGFlow chunks documents for retrieval. This is sentence two of item 30."
},
{
"doc_type_kwd": "text",
- "text": "Section 37. RAGFlow chunks documents for retrieval. This is sentence two of item 37.Section 38. RAGFlow chunks documents for retrieval. This is sentence two of item 38.Section 39. RAGFlow chunks documents for retrieval. This is sentence two of item 39.Section 40. RAGFlow chunks documents for retrieval. This is sentence two of item 40."
+ "text": "\nSection 31. RAGFlow chunks documents for retrieval. This is sentence two of item 31.\nSection 32. RAGFlow chunks documents for retrieval. This is sentence two of item 32.\nSection 33. RAGFlow chunks documents for retrieval. This is sentence two of item 33.\nSection 34. RAGFlow chunks documents for retrieval. This is sentence two of item 34.\nSection 35. RAGFlow chunks documents for retrieval. This is sentence two of item 35."
+ },
+ {
+ "doc_type_kwd": "text",
+ "text": "\nSection 36. RAGFlow chunks documents for retrieval. This is sentence two of item 36.\nSection 37. RAGFlow chunks documents for retrieval. This is sentence two of item 37.\nSection 38. RAGFlow chunks documents for retrieval. This is sentence two of item 38.\nSection 39. RAGFlow chunks documents for retrieval. This is sentence two of item 39.\nSection 40. RAGFlow chunks documents for retrieval. This is sentence two of item 40."
}
]
}
diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_two_item.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_two_item.json
deleted file mode 100644
index 9dd4baf861..0000000000
--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_two_item.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "case_id": "token__json_two_item",
- "chunks": [
- {
- "doc_type_kwd": "text",
- "text": "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omegaanother long item with many words that should be merged when the token budget allows merging across items on the python side but stays separate on the go side"
- }
- ]
-}
diff --git a/internal/ingestion/component/chunker/testdata/parity/known_diffs.json b/internal/ingestion/component/chunker/testdata/parity/known_diffs.json
index 9d4bc5a591..c0c3ff696c 100644
--- a/internal/ingestion/component/chunker/testdata/parity/known_diffs.json
+++ b/internal/ingestion/component/chunker/testdata/parity/known_diffs.json
@@ -48,16 +48,6 @@
"permanent": false,
"tracking": "Go TokenChunker retains the children (bare) delimiter on the chunk it ends (splitByDelim keep mode); Python's _split_text_by_pattern drops it. The user-selected lossless contract (PR #20276 follow-up) keeps bare (non-backtick) delimiters, so this diverges intentionally; resolve by dropping the children delimiter on the Python side.",
"reason": "Go TokenChunker retains the children delimiter '。' on each child chunk text; Python's _split_text_by_pattern discards it. Bare (non-backtick) delimiters are preserved losslessly by design, so the divergence is intentional. NOTE: the backtick-wrapped (custom) primary delimiters html/json/markdown/text_backtick were previously enrolled here but now DROP like Python and have been removed."
- },
- {
- "id": "token-json-interitem-join",
- "tag": "go_intentional",
- "kind": "chunk_text",
- "applies_to": ["token__json_two_item"],
- "owner_fix_side": "python",
- "permanent": false,
- "tracking": "Go TokenChunker merges JSON items with a '' join (each item already carries its own retained delimiter, so '' reproduces the source); Python's _merge_text_chunks_by_token_size inserts '\\n' between items. Align the Go join on the Python side to resolve.",
- "reason": "Go TokenChunker concatenates the two JSON items with no inter-item separator ('alpha ... omegaanother long item ...'), while Python inserts a '\\n' ('alpha ... omega\\nanother long item ...'). The source JSON items carry no separator, so '' is the faithful lossless join; the divergence is intentional pending a Python-side join alignment. (Enrolled under the old delimiter-retention rule by mistake; split out here.)"
}
]
}
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index 9cbfa38610..40c39a6194 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -75,6 +75,7 @@ import (
"regexp"
"strings"
"sync"
+ "unicode"
"unicode/utf8"
"go.uber.org/zap"
@@ -601,6 +602,9 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s
"_ERROR": fmt.Sprintf("TokenChunker: %v", err),
}
}
+ if !customDelim {
+ separateItemBoundaries(perItem)
+ }
// Surrounding media context is attached on the flattened chunk list,
// mirroring Python's _attach_context_to_media_chunks (token_chunker.py:537,
@@ -625,12 +629,9 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s
flat = attachMediaContext(flat, c.param.TableContextSize, c.param.ImageContextSize)
// Python _merge_text_chunks_by_token_size merges adjacent text
// chunks across JSON items into one global token budget. The join
- // separator is "" because each unit already carries its own retained
- // delimiter (splitByDelim keeps it), so concatenating reproduces the
- // source exactly. A "\n" separator would have been needed when the
- // delimiter was dropped, but would now double-insert it; the upstream
- // JSON items themselves carry no inter-item separator, so "" is the
- // faithful lossless join.
+ // separator stays empty because delimiters within each item are
+ // retained. separateItemBoundaries adds separators only between
+ // adjacent text items that have none at their edges.
flat = flatten(mergeByTokenSizeFromJSON([][]schema.ChunkDoc{flat}, c.param.ChunkTokenSize, c.param.OverlappedPercent))
}
@@ -722,6 +723,34 @@ func splitTextParserSentences(item schema.ChunkDoc) []schema.ChunkDoc {
// JSON-payload internals
// ---------------------------------------------------------------------------
+func separateItemBoundaries(perItem [][]schema.ChunkDoc) {
+ var previous *schema.ChunkDoc
+ for i := range perItem {
+ units := perItem[i]
+ if len(units) == 0 {
+ continue
+ }
+ firstIndex, lastIndex := 0, len(units)-1
+ for firstIndex <= lastIndex && units[firstIndex].CKType == "text" && units[firstIndex].Text == "" {
+ firstIndex++
+ }
+ for lastIndex >= firstIndex && units[lastIndex].CKType == "text" && units[lastIndex].Text == "" {
+ lastIndex--
+ }
+ if firstIndex > lastIndex {
+ continue
+ }
+ first := &units[firstIndex]
+ if previous != nil && previous.CKType == "text" && first.CKType == "text" &&
+ strings.TrimRightFunc(previous.Text, unicode.IsSpace) == previous.Text &&
+ strings.TrimLeftFunc(first.Text, unicode.IsSpace) == first.Text {
+ first.Text = "\n" + first.Text
+ first.TKNums = intPtr(tokenizeStr(first.Text))
+ }
+ previous = &units[lastIndex]
+ }
+}
+
// chunkFromItem mirrors _build_json_chunks for a single item. keepDelim
// follows the same policy as the text path: a bare delimiter is retained so
// the chunk reproduces the source exactly, but a custom (backtick) delimiter
diff --git a/internal/ingestion/component/chunker/token_item_separator_test.go b/internal/ingestion/component/chunker/token_item_separator_test.go
new file mode 100644
index 0000000000..4d9719932e
--- /dev/null
+++ b/internal/ingestion/component/chunker/token_item_separator_test.go
@@ -0,0 +1,86 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package chunker
+
+import (
+ "context"
+ "testing"
+)
+
+func TestTokenChunkerSeparatesAdjacentJSONItems(t *testing.T) {
+ items := []string{
+ "The installer requires Python 3.10 or later.",
+ "## Configuration",
+ "Edit config.yaml before the first start.",
+ }
+ if got, want := invokeTokenChunkerJSON(t, items...), "The installer requires Python 3.10 or later.\n## Configuration\nEdit config.yaml before the first start."; got != want {
+ t.Errorf("chunk text = %q, want %q", got, want)
+ }
+}
+
+func TestTokenChunkerPreservesExistingJSONItemWhitespace(t *testing.T) {
+ tests := []struct {
+ name string
+ items []string
+ want string
+ }{
+ {name: "previous item ends in whitespace", items: []string{"first\n", "second"}, want: "first\nsecond"},
+ {name: "next item starts in whitespace", items: []string{"first", " second"}, want: "first second"},
+ {name: "delimiter inside one item", items: []string{"first\nsecond"}, want: "first\nsecond"},
+ {name: "empty item between text items", items: []string{"first", "", "second"}, want: "first\nsecond"},
+ {name: "leading empty item", items: []string{"", "second"}, want: "second"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := invokeTokenChunkerJSON(t, tt.items...); got != tt.want {
+ t.Errorf("chunk text = %q, want %q", got, tt.want)
+ }
+ })
+ }
+}
+
+func invokeTokenChunkerJSON(t *testing.T, texts ...string) string {
+ t.Helper()
+ comp, err := NewTokenChunker(map[string]any{"chunk_token_size": float64(512)})
+ if err != nil {
+ t.Fatalf("construct TokenChunker: %v", err)
+ }
+ items := make([]map[string]any, len(texts))
+ for i, text := range texts {
+ items[i] = map[string]any{"text": text, "doc_type_kwd": "text"}
+ }
+ out, err := comp.Invoke(context.Background(), nil, map[string]any{
+ "name": "readme.md",
+ "output_format": "json",
+ "json": items,
+ })
+ if err != nil {
+ t.Fatalf("invoke TokenChunker: %v", err)
+ }
+ if msg, _ := out["_ERROR"].(string); msg != "" {
+ t.Fatalf("TokenChunker returned _ERROR: %s", msg)
+ }
+
+ chunks, ok := out["chunks"].([]map[string]any)
+ if !ok || len(chunks) != 1 {
+ t.Fatalf("chunks = %#v, want one merged chunk", out["chunks"])
+ }
+ text, ok := chunks[0]["text"].(string)
+ if !ok {
+ t.Fatalf("chunk text = %T, want string", chunks[0]["text"])
+ }
+ return text
+}