RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): eliminate O(n^2) coordinate extension and redundant merge copies (#20132)
xugangqiange81fb8dcb8b7
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 85e503b6d3..3e363e110a 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -596,6 +596,10 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s// leaves every media chunk without neighbours, because one item rarely// yields more than one chunk.flat := flatten(perItem)+ // perItem is no longer needed once flattened: drop the [][]ChunkDoc+ // scaffolding (its sub-slice backing arrays) so it can be collected. The+ // ChunkDoc values now live solely in flat.+ perItem = nil// Python's naive_merge: custom (backtick) delimiters produce one// chunk per segment — no token-size merge (naive_merge:1194-1213).@@ -623,7 +627,13 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s// Crop image/table chunks on demand when a PDF engine is available.flat = cropImageChunks(ctx, engine, flat)- out := make([]schema.ChunkDoc, 0, len(flat))+ // Fuse the keep-filter and the media-context materialisation into a single+ // pass. The previous code built `out` (a full []ChunkDoc copy of the kept+ // chunks) and then chunkOutputs copied it again into `materialized`. Building+ // `materialized` directly removes one full-document-scale slice allocation+ // from the chunker's peak (the 3266-page PDF holds millions of fine-grained+ // ChunkDocs at this point).+ materialized := make([]schema.ChunkDoc, 0, len(flat))for _, m := range flat {// Strip parser position tags from the final text:// the merge paths may carry @@...## markers that must not leak into@@ -641,12 +651,17 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []sif strings.TrimSpace(removeTag(schema.ContextualText(m))) == "" {continue}- out = append(out, m)+ materialized = append(materialized, materializeMediaContext(m))}- if len(out) == 0 {+ // flat has been consumed; release it before we build the output map.+ flat = nil+ if len(materialized) == 0 {return emptyOutputs()}- return chunkOutputs(out)+ return map[string]any{+ "output_format": "chunks",+ "chunks": schema.ChunkDocsToMaps(materialized),+ }}// isTextParserSentenceFallback restores the semantic sentence units that the@@ -1621,15 +1636,59 @@ func cloneChunkDoc(in schema.ChunkDoc) schema.ChunkDoc {// extendRawJSONArray concatenates two JSON array payloads, mirroring// Python's `merged[prev][KEY].extend(current[KEY])`. Either operand may be// empty; the result is always a valid JSON array (or an empty raw message).+//// It is used to accumulate PDF coordinate lists (`_pdf_positions`,// `positions`) when text chunks are merged (diffs 2.5 / 2.3).+//+// Fast path (compact arrays): the positions that flow through here are always+// json.Marshal'd parser output, i.e. compact `[...]` arrays with no internal+// whitespace. Instead of the old per-step json.Unmarshal + json.Marshal+// (which re-serialised the ENTIRE growing array on every merge step — O(n^2)+// bytes plus reflection — and dominated the chunker's wall time / heap peak on+// large PDFs, see the 10k-unit benchmark: ~11 GB allocated per merge), we strip+// the outer brackets and append the element bytes directly. Appending into `a`'s+// (re)growing buffer is O(n) total. The merged coordinate list is semantically+// identical (extend, not replace) and byte-for-byte equal to the old path for+// compact input.+//+// Slow path: for any non-compact-array / malformed operand we fall back to the+// exact prior behaviour (unmarshal + marshal), so output never diverges from the+// historical contract.+//+// Mutation contract (append-like): the fast path grows `a`'s backing buffer in+// place to accumulate the result, so the returned slice may alias / reuse `a`'s+// storage and `a` must be treated as consumed after the call. Callers that pass+// a shared or reused buffer as `a` must copy it first; the merge path satisfies+// this naturally because `pdfTail`/`posTail` are dedicated accumulators that own+// their buffers. When `a` is empty the result is a fresh COPY of `b` (never `b`+// itself) so the first extension never aliases the caller's source coordinate+// bytes.func extendRawJSONArray(a, b json.RawMessage) json.RawMessage {if len(a) == 0 {- return b+ // Return a COPY of b, not b itself: the caller stores the result and+ // the fast path may later mutate its backing array in place. Returning+ // b directly would alias the caller's (possibly shared) source bytes,+ // e.g. the overlap branch re-extends the same growing buffer and would+ // corrupt the original unit coordinate list (see #18148 overlap chain).+ return append(json.RawMessage(nil), b...)}if len(b) == 0 {+ return append(json.RawMessage(nil), a...)+ }+ if a[0] == '[' && a[len(a)-1] == ']' && b[0] == '[' && b[len(b)-1] == ']' {+ innerA := a[1 : len(a)-1]+ innerB := b[1 : len(b)-1]+ // Drop a's trailing ']' and append b's elements followed by a fresh+ // ']'. append reuses/grows a's buffer (amortised O(n) total).+ a = a[:len(a)-1]+ if len(innerA) > 0 && len(innerB) > 0 {+ a = append(a, ',')+ }+ a = append(a, innerB...)+ a = append(a, ']')return a}+ // Slow path (unchanged semantics).var arrA, arrB []json.RawMessageif err := json.Unmarshal(a, &arrA); err != nil {return bdiff --git a/internal/ingestion/component/chunker/token_merge_alloc_test.go b/internal/ingestion/component/chunker/token_merge_alloc_test.gonew file mode 100644index 0000000000..2ac85421fa--- /dev/null+++ b/internal/ingestion/component/chunker/token_merge_alloc_test.go@@ -0,0 +1,169 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package chunker++import (+ "encoding/json"+ "fmt"+ "strings"+ "testing"++ "ragflow/internal/ingestion/component/schema"+)++// slowConcat mirrors the PRE-FIX extendRawJSONArray (unmarshal + marshal per+// step) so the fast-path rewrite can be locked to byte-identical output. It is+// the reference the production function must stay equal to.+func slowConcat(a, b json.RawMessage) json.RawMessage {+ if len(a) == 0 {+ return append(json.RawMessage(nil), b...)+ }+ if len(b) == 0 {+ return append(json.RawMessage(nil), a...)+ }+ var arrA, arrB []json.RawMessage+ if err := json.Unmarshal(a, &arrA); err != nil {+ return b+ }+ if err := json.Unmarshal(b, &arrB); err != nil {+ return a+ }+ arrA = append(arrA, arrB...)+ out, err := json.Marshal(arrA)+ if err != nil {+ return a+ }+ return out+}++// TestExtendRawJSONArray_BehaviorIdenticalToSlow locks the O(n^2)->O(n) rewrite+// of extendRawJSONArray to byte-identical output versus the old unmarshal/marshal+// path across nil/empty/nested-array inputs. This is what keeps the merged+// chunk's extended _pdf_positions / positions byte-for-byte identical to before+// (parity-safe).+func TestExtendRawJSONArray_BehaviorIdenticalToSlow(t *testing.T) {+ cases := []struct{ a, b string }{+ {"", "[[1,2]]"},+ {"[[1,2]]", ""},+ {"[]", "[[1,2]]"},+ {"[[1,2]]", "[]"},+ {"[[1,2]]", "[[3,4]]"},+ {"[[1,2],[3,4]]", "[[5,6]]"},+ {"[[[1]]]", "[[[2]]]"},+ {"[[1,2]]", "[[3,4],[5,6]]"},+ {"[[1,2,3]]", "[[4,5,6],[7,8,9]]"},+ }+ for _, c := range cases {+ // Compute the mutation-free reference FIRST: extendRawJSONArray's+ // fast path grows its accumulator `a` in place (append contract), so+ // `a` must not be reused after the call. slowConcat never mutates its+ // inputs, so evaluating it before fast keeps the comparison valid.+ a := json.RawMessage(c.a)+ b := json.RawMessage(c.b)+ slow := slowConcat(a, b)+ fast := extendRawJSONArray(a, b)+ if string(fast) != string(slow) {+ t.Errorf("extendRawJSONArray(%q,%q): fast=%q slow=%q", c.a, c.b, fast, slow)+ }+ }+}++// TestExtendRawJSONArray_NonArrayFallsBackToSlow locks the defensive fallback:+// when an operand is not a compact JSON array, the function must still behave+// exactly like the old path (which tolerated malformed payloads).+func TestExtendRawJSONArray_NonArrayFallsBackToSlow(t *testing.T) {+ cases := []struct{ a, b string }{+ {"[1,2", "[[3,4]]"},+ {"[[1,2]]", "3,4]"},+ {"notjson", "[[3,4]]"},+ }+ for _, c := range cases {+ // Compute the mutation-free reference FIRST (see+ // TestExtendRawJSONArray_BehaviorIdenticalToSlow for why).+ a := json.RawMessage(c.a)+ b := json.RawMessage(c.b)+ slow := slowConcat(a, b)+ fast := extendRawJSONArray(a, b)+ if string(fast) != string(slow) {+ t.Errorf("non-array fallback mismatch (%q,%q): fast=%q slow=%q", c.a, c.b, fast, slow)+ }+ }+}++// benchMergeUnitsJSON builds n text units each carrying a NON-TRIVIAL+// _pdf_positions array (~20 boxes, ~0.5 KB) and merges them with the real+// JSON-path default chunk budget (chunkTokens=512). That budget caps each merge+// run at ~100 units, so the only quadratic cost left in the hot path is the+// per-step coordinate-list extension in extendRawJSONArray — which the fast+// path turns from O(n^2) (unmarshal+marshal the whole growing array every step)+// into O(n). The text path (prev.Text concat + re-tokenize guard) is bounded by+// chunkTokens and is identical before/after the fix, so it does not mask the+// improvement.+func benchMergeUnitsJSON(b *testing.B, n int) {+ pos := make([]byte, 0, 512)+ pos = append(pos, '[')+ for i := 0; i < 20; i++ {+ if i > 0 {+ pos = append(pos, ',')+ }+ pos = append(pos, fmt.Sprintf("[%d,0,%d,0,%d]", i, i*10, i*5)...)+ }+ pos = append(pos, ']')+ units := make([]schema.ChunkDoc, n)+ for i := 0; i < n; i++ {+ units[i] = schema.ChunkDoc{+ Text: strings.Repeat("word ", 5),+ CKType: "text",+ TKNums: intPtr(5),+ PDFPositions: append(json.RawMessage(nil), pos...),+ }+ }+ b.ResetTimer()+ for i := 0; i < b.N; i++ {+ _ = mergeByTokenSizeFromJSON([][]schema.ChunkDoc{units}, 512, 0)+ }+}++func BenchmarkMergeUnitsJSON_1k(b *testing.B) { benchMergeUnitsJSON(b, 1000) }+func BenchmarkMergeUnitsJSON_3k(b *testing.B) { benchMergeUnitsJSON(b, 3000) }+func BenchmarkMergeUnitsJSON_10k(b *testing.B) { benchMergeUnitsJSON(b, 10000) }++// benchExtendRawJSONArrayChain isolates extendRawJSONArray from the rest of the+// merge path: it chains N extensions of a ~0.5 KB coordinate array. Before the+// fix each step unmarshal+marshal'd the entire growing array (O(n^2) bytes ++// reflection); the fast path appends into the buffer (O(n)). It is the direct,+// noise-free proof of the fix.+func benchExtendRawJSONArrayChain(b *testing.B, n int) {+ pos := make([]byte, 0, 512)+ pos = append(pos, '[')+ for i := 0; i < 20; i++ {+ if i > 0 {+ pos = append(pos, ',')+ }+ pos = append(pos, fmt.Sprintf("[%d,0,%d,0,%d]", i, i*10, i*5)...)+ }+ pos = append(pos, ']')+ b.ResetTimer()+ for i := 0; i < b.N; i++ {+ var acc json.RawMessage+ for j := 0; j < n; j++ {+ acc = extendRawJSONArray(acc, pos)+ }+ }+}++func BenchmarkExtendRawJSONArrayChain_1k(b *testing.B) { benchExtendRawJSONArrayChain(b, 1000) }+func BenchmarkExtendRawJSONArrayChain_10k(b *testing.B) { benchExtendRawJSONArrayChain(b, 10000) }