RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunk): count merge budget in runes, not bytes (#20690)
Zheng Haoran637224db9ea5
diff --git a/internal/parser/chunk/options.go b/internal/parser/chunk/options.goindex 902e77c8f0..89dcea87a7 100644--- a/internal/parser/chunk/options.go+++ b/internal/parser/chunk/options.go@@ -37,8 +37,8 @@ type ChunkOptions struct {// strategy degrades to "sentence" inside the operator.SplitStrategy string- // Postprocess configuration. Zero values mean "do not run that- // step"; non-zero values enable it.+ // MergeTargetSize > 0 enables the merge step; its value is the+ // per-chunk budget in runes (Unicode code points).MergeTargetSize int// FilterMinLength > 0 drops chunks shorter than that (rune count).diff --git a/internal/parser/chunk/postprocess.go b/internal/parser/chunk/postprocess.goindex 585cc49452..1838b1b943 100644--- a/internal/parser/chunk/postprocess.go+++ b/internal/parser/chunk/postprocess.go@@ -134,11 +134,17 @@ func (o *PostprocessOperator) mergeChunks(chunks []ChunkData) []ChunkData {var merged []ChunkDatavar buf strings.Buildervar bufMeta map[string]interface{}+ // bufRunes tracks the buffer length in runes: target and the per-chunk+ // overflow check are rune-based, so the running total must be too.+ // buf.Len() is bytes and inflates 3x for CJK, which flushes early and+ // leaves every merged chunk far below target_size.+ var bufRunes intfirstIndex := 0for i, c := range chunks {+ cRunes := len([]rune(c.Content))// If this single chunk already exceeds target, flush first then add- if len([]rune(c.Content)) >= target {+ if cRunes >= target {if buf.Len() > 0 {merged = append(merged, ChunkData{Content: buf.String(),@@ -147,6 +153,7 @@ func (o *PostprocessOperator) mergeChunks(chunks []ChunkData) []ChunkData {})buf.Reset()bufMeta = nil+ bufRunes = 0}merged = append(merged, c)firstIndex = i + 1@@ -155,12 +162,12 @@ func (o *PostprocessOperator) mergeChunks(chunks []ChunkData) []ChunkData {if buf.Len() == 0 {buf.WriteString(c.Content)+ bufRunes = cRunesbufMeta = c.MetadatafirstIndex = c.Index} else {- nextLen := len([]rune(c.Content))// If adding this chunk would exceed target, flush current and start new- if buf.Len()+nextLen+1 > target {+ if bufRunes+cRunes+1 > target {merged = append(merged, ChunkData{Content: buf.String(),Index: firstIndex,@@ -168,11 +175,13 @@ func (o *PostprocessOperator) mergeChunks(chunks []ChunkData) []ChunkData {})buf.Reset()buf.WriteString(c.Content)+ bufRunes = cRunesbufMeta = c.MetadatafirstIndex = c.Index} else {buf.WriteString(" ")buf.WriteString(c.Content)+ bufRunes += cRunes + 1// Merge metadata (last wins for overlapping keys)if c.Metadata != nil && bufMeta == nil {bufMeta = make(map[string]interface{})diff --git a/internal/parser/chunk/postprocess_test.go b/internal/parser/chunk/postprocess_test.goindex cb237ff95c..8a468e3a48 100644--- a/internal/parser/chunk/postprocess_test.go+++ b/internal/parser/chunk/postprocess_test.go@@ -130,3 +130,63 @@ func TestNewPostprocessOperatorParsesFilterFlags(t *testing.T) {t.Errorf("String() missing filter flags:\n%s", s)}}++// runMerge builds a postprocess operator whose only stage is the merge, runs it+// over the given chunk contents, and returns the resulting contents in order.+func runMerge(t *testing.T, target int, contents ...string) []string {+ t.Helper()+ op, err := NewPostprocessOperator(map[string]interface{}{+ "merge": map[string]interface{}{"target_size": float64(target)},+ })+ if err != nil {+ t.Fatalf("NewPostprocessOperator returned error: %v", err)+ }+ ctx := &ChunkContext{SplitChunks: make([]ChunkData, len(contents))}+ for i, c := range contents {+ ctx.SplitChunks[i] = ChunkData{Content: c, Index: i}+ }+ if err := op.Execute(ctx); err != nil {+ t.Fatalf("Execute returned error: %v", err)+ }+ out := make([]string, len(ctx.ResultChunks))+ for i, c := range ctx.ResultChunks {+ out[i] = c.Content+ }+ return out+}++// TestMergeChunksTargetCountsRunes pins the merge budget unit: target_size and+// the per-chunk overflow check are documented in runes (ChunkOptions.+// FilterMinLength, len([]rune(c.Content)) >= target), so the running total must+// accumulate runes too. Mixing builder byte length with rune length makes CJK+// text flush early: every merged chunk stays far below target_size.+func TestMergeChunksTargetCountsRunes(t *testing.T) {+ // Three 4-rune chunks; joining separator adds 1 rune per join.+ // Budget 12 runes: chunk1+chunk2 = 4+1+4 = 9 fits; adding chunk3 would+ // reach 14, so the merge must flush after two chunks.+ got := runMerge(t, 12, "甲乙丙丁", "戊己庚辛", "壬癸子丑")+ want := []string{"甲乙丙丁 戊己庚辛", "壬癸子丑"}+ if !eq(got, want) {+ t.Fatalf("merge budget must count runes, not bytes: got %q, want %q", got, want)+ }+}++// TestMergeChunksASCIIBehaviourUnchanged asserts the rune-accounting fix leaves+// ASCII (1 byte per rune) behaviour identical.+func TestMergeChunksASCIIBehaviourUnchanged(t *testing.T) {+ got := runMerge(t, 12, "abcd", "efgh", "ijkl")+ want := []string{"abcd efgh", "ijkl"}+ if !eq(got, want) {+ t.Fatalf("ASCII merge behaviour changed: got %q, want %q", got, want)+ }+}++// TestMergeChunksOversizeUnitStandsAloneCJK keeps the single-chunk overflow+// path covered for multi-byte text; the overflow check is rune-based already.+func TestMergeChunksOversizeUnitStandsAloneCJK(t *testing.T) {+ got := runMerge(t, 4, "甲乙丙丁", "戊己庚辛")+ want := []string{"甲乙丙丁", "戊己庚辛"}+ if !eq(got, want) {+ t.Fatalf("oversize CJK unit must stand alone: got %q, want %q", got, want)+ }+}