Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(common,internal): round instead of truncate in normalize_overlapped_percent (closes #17418) (#17452)

Harsh Kashyap6cb43f74c4ec
diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go
index bb0c432ff2..c3cdff1e25 100644
--- a/internal/ingestion/component/chunker/token_test.go
+++ b/internal/ingestion/component/chunker/token_test.go
@@ -395,7 +395,11 @@ func TestNormalizeOverlappedPercent(t *testing.T) {
}{
{"zero", 0, 0},
{"fraction 0.1 -> 10", 0.1, 10},
+ {"fraction 0.29 -> 29 (round, was 28 trunc)", 0.29, 29},
+ {"fraction 0.3 -> 30", 0.3, 30},
{"fraction 0.5 -> 50", 0.5, 50},
+ {"fraction 0.57 -> 57 (round, was 56 trunc)", 0.57, 57},
+ {"fraction 0.93 -> 90 (clamp after round)", 0.93, 90},
{"fraction 0.95 -> 90 (clamp)", 0.95, 90},
{"percent 15", 15, 15},
{"int truncation 33.3 -> 33", 33.3, 33},
@@ -413,6 +417,8 @@ func TestNormalizeOverlappedPercent(t *testing.T) {
{"huge 1e300 -> 90", 1e300, 90},
{"huge -1e300 -> 0", -1e300, 0},
{"huge math.MaxFloat64 -> 90", math.MaxFloat64, 90},
+ {"huge math.MaxFloat64 +1 -> 90 (clamp pre-round)", math.MaxFloat64 + 1, 90},
+ {`numeric string fraction "0.29" -> 29`, "0.29", 29},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
@@ -491,6 +497,8 @@ func TestTokenChunkerParam_ValidateOverlappedRange(t *testing.T) {
wantErr bool
}{
{"fraction 0.3 -> 30", 0.3, 30, false},
+ {"fraction 0.29 -> 29 (round, was 28 trunc)", 0.29, 29, false},
+ {"fraction 0.57 -> 57 (round, was 56 trunc)", 0.57, 57, false},
{"percent 0", 0, 0, false},
{"percent 30", 30, 30, false},
{"percent 90 (boundary)", 90, 90, false},
diff --git a/internal/ingestion/component/schema/chunker.go b/internal/ingestion/component/schema/chunker.go
index 6e9453dea8..826fdceabd 100644
--- a/internal/ingestion/component/schema/chunker.go
+++ b/internal/ingestion/component/schema/chunker.go
@@ -293,7 +293,12 @@ func clampOverlappedPercent(value float64) float64 {
if value > 90 {
value = 90
}
- value = float64(int(value))
+ // Round rather than truncate so fraction inputs like 0.29 normalize
+ // to 29 (user expectation) rather than 28. Mirrors Python's
+ // common/float_utils.py:50-58 fix for #17418. We still clamp
+ // before rounding so out-of-int-range values (e.g. 1e300) land on
+ // 90, not an implementation-defined int.
+ value = math.Round(value)
return value
}
@@ -308,7 +313,10 @@ func (p *TokenChunkerParam) Validate() error {
// rather than silently clamped — eliminating the latent footgun where a
// fraction passed to a struct bypassed normalization.
if v := p.OverlappedPercent; 0 < v && v < 1 {
- p.OverlappedPercent = float64(int(v * 100))
+ // Round rather than truncate so fraction inputs like 0.29 normalize
+ // to 29 (user expectation) rather than 28. Mirrors Python's
+ // common/float_utils.py:50-58 fix for #17418.
+ p.OverlappedPercent = math.Round(v * 100)
}
switch p.DelimiterMode {
case "token_size", "delimiter":