RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(common,internal): round instead of truncate in normalize_overlapped_percent (closes #17418) (#17452)
Harsh Kashyap6cb43f74c4ec
diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.goindex bb0c432ff2..c3cdff1e25 100644--- a/internal/ingestion/component/chunker/token_test.go+++ b/internal/ingestion/component/chunker/token_test.go@@ -395,7 +395,11 @@ func TestNormalizeOverlappedPercent(t *testing.T) {}{{"zero", 0, 0},{"fraction 0.1 -> 10", 0.1, 10},+ {"fraction 0.29 -> 29 (round, was 28 trunc)", 0.29, 29},+ {"fraction 0.3 -> 30", 0.3, 30},{"fraction 0.5 -> 50", 0.5, 50},+ {"fraction 0.57 -> 57 (round, was 56 trunc)", 0.57, 57},+ {"fraction 0.93 -> 90 (clamp after round)", 0.93, 90},{"fraction 0.95 -> 90 (clamp)", 0.95, 90},{"percent 15", 15, 15},{"int truncation 33.3 -> 33", 33.3, 33},@@ -413,6 +417,8 @@ func TestNormalizeOverlappedPercent(t *testing.T) {{"huge 1e300 -> 90", 1e300, 90},{"huge -1e300 -> 0", -1e300, 0},{"huge math.MaxFloat64 -> 90", math.MaxFloat64, 90},+ {"huge math.MaxFloat64 +1 -> 90 (clamp pre-round)", math.MaxFloat64 + 1, 90},+ {`numeric string fraction "0.29" -> 29`, "0.29", 29},}for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {@@ -491,6 +497,8 @@ func TestTokenChunkerParam_ValidateOverlappedRange(t *testing.T) {wantErr bool}{{"fraction 0.3 -> 30", 0.3, 30, false},+ {"fraction 0.29 -> 29 (round, was 28 trunc)", 0.29, 29, false},+ {"fraction 0.57 -> 57 (round, was 56 trunc)", 0.57, 57, false},{"percent 0", 0, 0, false},{"percent 30", 30, 30, false},{"percent 90 (boundary)", 90, 90, false},diff --git a/internal/ingestion/component/schema/chunker.go b/internal/ingestion/component/schema/chunker.goindex 6e9453dea8..826fdceabd 100644--- a/internal/ingestion/component/schema/chunker.go+++ b/internal/ingestion/component/schema/chunker.go@@ -293,7 +293,12 @@ func clampOverlappedPercent(value float64) float64 {if value > 90 {value = 90}- value = float64(int(value))+ // Round rather than truncate so fraction inputs like 0.29 normalize+ // to 29 (user expectation) rather than 28. Mirrors Python's+ // common/float_utils.py:50-58 fix for #17418. We still clamp+ // before rounding so out-of-int-range values (e.g. 1e300) land on+ // 90, not an implementation-defined int.+ value = math.Round(value)return value}@@ -308,7 +313,10 @@ func (p *TokenChunkerParam) Validate() error {// rather than silently clamped — eliminating the latent footgun where a// fraction passed to a struct bypassed normalization.if v := p.OverlappedPercent; 0 < v && v < 1 {- p.OverlappedPercent = float64(int(v * 100))+ // Round rather than truncate so fraction inputs like 0.29 normalize+ // to 29 (user expectation) rather than 28. Mirrors Python's+ // common/float_utils.py:50-58 fix for #17418.+ p.OverlappedPercent = math.Round(v * 100)}switch p.DelimiterMode {case "token_size", "delimiter":