Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): eliminate O(n^2) coordinate extension and redundant merge copies (#20132)

xugangqiange81fb8dcb8b7
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index 85e503b6d3..3e363e110a 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -596,6 +596,10 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s
// leaves every media chunk without neighbours, because one item rarely
// yields more than one chunk.
flat := flatten(perItem)
+ // perItem is no longer needed once flattened: drop the [][]ChunkDoc
+ // scaffolding (its sub-slice backing arrays) so it can be collected. The
+ // ChunkDoc values now live solely in flat.
+ perItem = nil
// Python's naive_merge: custom (backtick) delimiters produce one
// chunk per segment — no token-size merge (naive_merge:1194-1213).
@@ -623,7 +627,13 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s
// Crop image/table chunks on demand when a PDF engine is available.
flat = cropImageChunks(ctx, engine, flat)
- out := make([]schema.ChunkDoc, 0, len(flat))
+ // Fuse the keep-filter and the media-context materialisation into a single
+ // pass. The previous code built `out` (a full []ChunkDoc copy of the kept
+ // chunks) and then chunkOutputs copied it again into `materialized`. Building
+ // `materialized` directly removes one full-document-scale slice allocation
+ // from the chunker's peak (the 3266-page PDF holds millions of fine-grained
+ // ChunkDocs at this point).
+ materialized := make([]schema.ChunkDoc, 0, len(flat))
for _, m := range flat {
// Strip parser position tags from the final text:
// the merge paths may carry @@...## markers that must not leak into
@@ -641,12 +651,17 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s
if strings.TrimSpace(removeTag(schema.ContextualText(m))) == "" {
continue
}
- out = append(out, m)
+ materialized = append(materialized, materializeMediaContext(m))
}
- if len(out) == 0 {
+ // flat has been consumed; release it before we build the output map.
+ flat = nil
+ if len(materialized) == 0 {
return emptyOutputs()
}
- return chunkOutputs(out)
+ return map[string]any{
+ "output_format": "chunks",
+ "chunks": schema.ChunkDocsToMaps(materialized),
+ }
}
// isTextParserSentenceFallback restores the semantic sentence units that the
@@ -1621,15 +1636,59 @@ func cloneChunkDoc(in schema.ChunkDoc) schema.ChunkDoc {
// extendRawJSONArray concatenates two JSON array payloads, mirroring
// Python's `merged[prev][KEY].extend(current[KEY])`. Either operand may be
// empty; the result is always a valid JSON array (or an empty raw message).
+//
// It is used to accumulate PDF coordinate lists (`_pdf_positions`,
// `positions`) when text chunks are merged (diffs 2.5 / 2.3).
+//
+// Fast path (compact arrays): the positions that flow through here are always
+// json.Marshal'd parser output, i.e. compact `[...]` arrays with no internal
+// whitespace. Instead of the old per-step json.Unmarshal + json.Marshal
+// (which re-serialised the ENTIRE growing array on every merge step — O(n^2)
+// bytes plus reflection — and dominated the chunker's wall time / heap peak on
+// large PDFs, see the 10k-unit benchmark: ~11 GB allocated per merge), we strip
+// the outer brackets and append the element bytes directly. Appending into `a`'s
+// (re)growing buffer is O(n) total. The merged coordinate list is semantically
+// identical (extend, not replace) and byte-for-byte equal to the old path for
+// compact input.
+//
+// Slow path: for any non-compact-array / malformed operand we fall back to the
+// exact prior behaviour (unmarshal + marshal), so output never diverges from the
+// historical contract.
+//
+// Mutation contract (append-like): the fast path grows `a`'s backing buffer in
+// place to accumulate the result, so the returned slice may alias / reuse `a`'s
+// storage and `a` must be treated as consumed after the call. Callers that pass
+// a shared or reused buffer as `a` must copy it first; the merge path satisfies
+// this naturally because `pdfTail`/`posTail` are dedicated accumulators that own
+// their buffers. When `a` is empty the result is a fresh COPY of `b` (never `b`
+// itself) so the first extension never aliases the caller's source coordinate
+// bytes.
func extendRawJSONArray(a, b json.RawMessage) json.RawMessage {
if len(a) == 0 {
- return b
+ // Return a COPY of b, not b itself: the caller stores the result and
+ // the fast path may later mutate its backing array in place. Returning
+ // b directly would alias the caller's (possibly shared) source bytes,
+ // e.g. the overlap branch re-extends the same growing buffer and would
+ // corrupt the original unit coordinate list (see #18148 overlap chain).
+ return append(json.RawMessage(nil), b...)
}
if len(b) == 0 {
+ return append(json.RawMessage(nil), a...)
+ }
+ if a[0] == '[' && a[len(a)-1] == ']' && b[0] == '[' && b[len(b)-1] == ']' {
+ innerA := a[1 : len(a)-1]
+ innerB := b[1 : len(b)-1]
+ // Drop a's trailing ']' and append b's elements followed by a fresh
+ // ']'. append reuses/grows a's buffer (amortised O(n) total).
+ a = a[:len(a)-1]
+ if len(innerA) > 0 && len(innerB) > 0 {
+ a = append(a, ',')
+ }
+ a = append(a, innerB...)
+ a = append(a, ']')
return a
}
+ // Slow path (unchanged semantics).
var arrA, arrB []json.RawMessage
if err := json.Unmarshal(a, &arrA); err != nil {
return b
diff --git a/internal/ingestion/component/chunker/token_merge_alloc_test.go b/internal/ingestion/component/chunker/token_merge_alloc_test.go
new file mode 100644
index 0000000000..2ac85421fa
--- /dev/null
+++ b/internal/ingestion/component/chunker/token_merge_alloc_test.go
@@ -0,0 +1,169 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package chunker
+
+import (
+ "encoding/json"
+ "fmt"
+ "strings"
+ "testing"
+
+ "ragflow/internal/ingestion/component/schema"
+)
+
+// slowConcat mirrors the PRE-FIX extendRawJSONArray (unmarshal + marshal per
+// step) so the fast-path rewrite can be locked to byte-identical output. It is
+// the reference the production function must stay equal to.
+func slowConcat(a, b json.RawMessage) json.RawMessage {
+ if len(a) == 0 {
+ return append(json.RawMessage(nil), b...)
+ }
+ if len(b) == 0 {
+ return append(json.RawMessage(nil), a...)
+ }
+ var arrA, arrB []json.RawMessage
+ if err := json.Unmarshal(a, &arrA); err != nil {
+ return b
+ }
+ if err := json.Unmarshal(b, &arrB); err != nil {
+ return a
+ }
+ arrA = append(arrA, arrB...)
+ out, err := json.Marshal(arrA)
+ if err != nil {
+ return a
+ }
+ return out
+}
+
+// TestExtendRawJSONArray_BehaviorIdenticalToSlow locks the O(n^2)->O(n) rewrite
+// of extendRawJSONArray to byte-identical output versus the old unmarshal/marshal
+// path across nil/empty/nested-array inputs. This is what keeps the merged
+// chunk's extended _pdf_positions / positions byte-for-byte identical to before
+// (parity-safe).
+func TestExtendRawJSONArray_BehaviorIdenticalToSlow(t *testing.T) {
+ cases := []struct{ a, b string }{
+ {"", "[[1,2]]"},
+ {"[[1,2]]", ""},
+ {"[]", "[[1,2]]"},
+ {"[[1,2]]", "[]"},
+ {"[[1,2]]", "[[3,4]]"},
+ {"[[1,2],[3,4]]", "[[5,6]]"},
+ {"[[[1]]]", "[[[2]]]"},
+ {"[[1,2]]", "[[3,4],[5,6]]"},
+ {"[[1,2,3]]", "[[4,5,6],[7,8,9]]"},
+ }
+ for _, c := range cases {
+ // Compute the mutation-free reference FIRST: extendRawJSONArray's
+ // fast path grows its accumulator `a` in place (append contract), so
+ // `a` must not be reused after the call. slowConcat never mutates its
+ // inputs, so evaluating it before fast keeps the comparison valid.
+ a := json.RawMessage(c.a)
+ b := json.RawMessage(c.b)
+ slow := slowConcat(a, b)
+ fast := extendRawJSONArray(a, b)
+ if string(fast) != string(slow) {
+ t.Errorf("extendRawJSONArray(%q,%q): fast=%q slow=%q", c.a, c.b, fast, slow)
+ }
+ }
+}
+
+// TestExtendRawJSONArray_NonArrayFallsBackToSlow locks the defensive fallback:
+// when an operand is not a compact JSON array, the function must still behave
+// exactly like the old path (which tolerated malformed payloads).
+func TestExtendRawJSONArray_NonArrayFallsBackToSlow(t *testing.T) {
+ cases := []struct{ a, b string }{
+ {"[1,2", "[[3,4]]"},
+ {"[[1,2]]", "3,4]"},
+ {"notjson", "[[3,4]]"},
+ }
+ for _, c := range cases {
+ // Compute the mutation-free reference FIRST (see
+ // TestExtendRawJSONArray_BehaviorIdenticalToSlow for why).
+ a := json.RawMessage(c.a)
+ b := json.RawMessage(c.b)
+ slow := slowConcat(a, b)
+ fast := extendRawJSONArray(a, b)
+ if string(fast) != string(slow) {
+ t.Errorf("non-array fallback mismatch (%q,%q): fast=%q slow=%q", c.a, c.b, fast, slow)
+ }
+ }
+}
+
+// benchMergeUnitsJSON builds n text units each carrying a NON-TRIVIAL
+// _pdf_positions array (~20 boxes, ~0.5 KB) and merges them with the real
+// JSON-path default chunk budget (chunkTokens=512). That budget caps each merge
+// run at ~100 units, so the only quadratic cost left in the hot path is the
+// per-step coordinate-list extension in extendRawJSONArray — which the fast
+// path turns from O(n^2) (unmarshal+marshal the whole growing array every step)
+// into O(n). The text path (prev.Text concat + re-tokenize guard) is bounded by
+// chunkTokens and is identical before/after the fix, so it does not mask the
+// improvement.
+func benchMergeUnitsJSON(b *testing.B, n int) {
+ pos := make([]byte, 0, 512)
+ pos = append(pos, '[')
+ for i := 0; i < 20; i++ {
+ if i > 0 {
+ pos = append(pos, ',')
+ }
+ pos = append(pos, fmt.Sprintf("[%d,0,%d,0,%d]", i, i*10, i*5)...)
+ }
+ pos = append(pos, ']')
+ units := make([]schema.ChunkDoc, n)
+ for i := 0; i < n; i++ {
+ units[i] = schema.ChunkDoc{
+ Text: strings.Repeat("word ", 5),
+ CKType: "text",
+ TKNums: intPtr(5),
+ PDFPositions: append(json.RawMessage(nil), pos...),
+ }
+ }
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ _ = mergeByTokenSizeFromJSON([][]schema.ChunkDoc{units}, 512, 0)
+ }
+}
+
+func BenchmarkMergeUnitsJSON_1k(b *testing.B) { benchMergeUnitsJSON(b, 1000) }
+func BenchmarkMergeUnitsJSON_3k(b *testing.B) { benchMergeUnitsJSON(b, 3000) }
+func BenchmarkMergeUnitsJSON_10k(b *testing.B) { benchMergeUnitsJSON(b, 10000) }
+
+// benchExtendRawJSONArrayChain isolates extendRawJSONArray from the rest of the
+// merge path: it chains N extensions of a ~0.5 KB coordinate array. Before the
+// fix each step unmarshal+marshal'd the entire growing array (O(n^2) bytes +
+// reflection); the fast path appends into the buffer (O(n)). It is the direct,
+// noise-free proof of the fix.
+func benchExtendRawJSONArrayChain(b *testing.B, n int) {
+ pos := make([]byte, 0, 512)
+ pos = append(pos, '[')
+ for i := 0; i < 20; i++ {
+ if i > 0 {
+ pos = append(pos, ',')
+ }
+ pos = append(pos, fmt.Sprintf("[%d,0,%d,0,%d]", i, i*10, i*5)...)
+ }
+ pos = append(pos, ']')
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ var acc json.RawMessage
+ for j := 0; j < n; j++ {
+ acc = extendRawJSONArray(acc, pos)
+ }
+ }
+}
+
+func BenchmarkExtendRawJSONArrayChain_1k(b *testing.B) { benchExtendRawJSONArrayChain(b, 1000) }
+func BenchmarkExtendRawJSONArrayChain_10k(b *testing.B) { benchExtendRawJSONArrayChain(b, 10000) }