Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunk): count merge budget in runes, not bytes (#20690)

Zheng Haoran637224db9ea5
diff --git a/internal/parser/chunk/options.go b/internal/parser/chunk/options.go
index 902e77c8f0..89dcea87a7 100644
--- a/internal/parser/chunk/options.go
+++ b/internal/parser/chunk/options.go
@@ -37,8 +37,8 @@ type ChunkOptions struct {
// strategy degrades to "sentence" inside the operator.
SplitStrategy string
- // Postprocess configuration. Zero values mean "do not run that
- // step"; non-zero values enable it.
+ // MergeTargetSize > 0 enables the merge step; its value is the
+ // per-chunk budget in runes (Unicode code points).
MergeTargetSize int
// FilterMinLength > 0 drops chunks shorter than that (rune count).
diff --git a/internal/parser/chunk/postprocess.go b/internal/parser/chunk/postprocess.go
index 585cc49452..1838b1b943 100644
--- a/internal/parser/chunk/postprocess.go
+++ b/internal/parser/chunk/postprocess.go
@@ -134,11 +134,17 @@ func (o *PostprocessOperator) mergeChunks(chunks []ChunkData) []ChunkData {
var merged []ChunkData
var buf strings.Builder
var bufMeta map[string]interface{}
+ // bufRunes tracks the buffer length in runes: target and the per-chunk
+ // overflow check are rune-based, so the running total must be too.
+ // buf.Len() is bytes and inflates 3x for CJK, which flushes early and
+ // leaves every merged chunk far below target_size.
+ var bufRunes int
firstIndex := 0
for i, c := range chunks {
+ cRunes := len([]rune(c.Content))
// If this single chunk already exceeds target, flush first then add
- if len([]rune(c.Content)) >= target {
+ if cRunes >= target {
if buf.Len() > 0 {
merged = append(merged, ChunkData{
Content: buf.String(),
@@ -147,6 +153,7 @@ func (o *PostprocessOperator) mergeChunks(chunks []ChunkData) []ChunkData {
})
buf.Reset()
bufMeta = nil
+ bufRunes = 0
}
merged = append(merged, c)
firstIndex = i + 1
@@ -155,12 +162,12 @@ func (o *PostprocessOperator) mergeChunks(chunks []ChunkData) []ChunkData {
if buf.Len() == 0 {
buf.WriteString(c.Content)
+ bufRunes = cRunes
bufMeta = c.Metadata
firstIndex = c.Index
} else {
- nextLen := len([]rune(c.Content))
// If adding this chunk would exceed target, flush current and start new
- if buf.Len()+nextLen+1 > target {
+ if bufRunes+cRunes+1 > target {
merged = append(merged, ChunkData{
Content: buf.String(),
Index: firstIndex,
@@ -168,11 +175,13 @@ func (o *PostprocessOperator) mergeChunks(chunks []ChunkData) []ChunkData {
})
buf.Reset()
buf.WriteString(c.Content)
+ bufRunes = cRunes
bufMeta = c.Metadata
firstIndex = c.Index
} else {
buf.WriteString(" ")
buf.WriteString(c.Content)
+ bufRunes += cRunes + 1
// Merge metadata (last wins for overlapping keys)
if c.Metadata != nil && bufMeta == nil {
bufMeta = make(map[string]interface{})
diff --git a/internal/parser/chunk/postprocess_test.go b/internal/parser/chunk/postprocess_test.go
index cb237ff95c..8a468e3a48 100644
--- a/internal/parser/chunk/postprocess_test.go
+++ b/internal/parser/chunk/postprocess_test.go
@@ -130,3 +130,63 @@ func TestNewPostprocessOperatorParsesFilterFlags(t *testing.T) {
t.Errorf("String() missing filter flags:\n%s", s)
}
}
+
+// runMerge builds a postprocess operator whose only stage is the merge, runs it
+// over the given chunk contents, and returns the resulting contents in order.
+func runMerge(t *testing.T, target int, contents ...string) []string {
+ t.Helper()
+ op, err := NewPostprocessOperator(map[string]interface{}{
+ "merge": map[string]interface{}{"target_size": float64(target)},
+ })
+ if err != nil {
+ t.Fatalf("NewPostprocessOperator returned error: %v", err)
+ }
+ ctx := &ChunkContext{SplitChunks: make([]ChunkData, len(contents))}
+ for i, c := range contents {
+ ctx.SplitChunks[i] = ChunkData{Content: c, Index: i}
+ }
+ if err := op.Execute(ctx); err != nil {
+ t.Fatalf("Execute returned error: %v", err)
+ }
+ out := make([]string, len(ctx.ResultChunks))
+ for i, c := range ctx.ResultChunks {
+ out[i] = c.Content
+ }
+ return out
+}
+
+// TestMergeChunksTargetCountsRunes pins the merge budget unit: target_size and
+// the per-chunk overflow check are documented in runes (ChunkOptions.
+// FilterMinLength, len([]rune(c.Content)) >= target), so the running total must
+// accumulate runes too. Mixing builder byte length with rune length makes CJK
+// text flush early: every merged chunk stays far below target_size.
+func TestMergeChunksTargetCountsRunes(t *testing.T) {
+ // Three 4-rune chunks; joining separator adds 1 rune per join.
+ // Budget 12 runes: chunk1+chunk2 = 4+1+4 = 9 fits; adding chunk3 would
+ // reach 14, so the merge must flush after two chunks.
+ got := runMerge(t, 12, "甲乙丙丁", "戊己庚辛", "壬癸子丑")
+ want := []string{"甲乙丙丁 戊己庚辛", "壬癸子丑"}
+ if !eq(got, want) {
+ t.Fatalf("merge budget must count runes, not bytes: got %q, want %q", got, want)
+ }
+}
+
+// TestMergeChunksASCIIBehaviourUnchanged asserts the rune-accounting fix leaves
+// ASCII (1 byte per rune) behaviour identical.
+func TestMergeChunksASCIIBehaviourUnchanged(t *testing.T) {
+ got := runMerge(t, 12, "abcd", "efgh", "ijkl")
+ want := []string{"abcd efgh", "ijkl"}
+ if !eq(got, want) {
+ t.Fatalf("ASCII merge behaviour changed: got %q, want %q", got, want)
+ }
+}
+
+// TestMergeChunksOversizeUnitStandsAloneCJK keeps the single-chunk overflow
+// path covered for multi-byte text; the overflow check is rune-based already.
+func TestMergeChunksOversizeUnitStandsAloneCJK(t *testing.T) {
+ got := runMerge(t, 4, "甲乙丙丁", "戊己庚辛")
+ want := []string{"甲乙丙丁", "戊己庚辛"}
+ if !eq(got, want) {
+ t.Fatalf("oversize CJK unit must stand alone: got %q, want %q", got, want)
+ }
+}