Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(chunker): preserve explicit zero chunk_overlap on ingest (#4276)

Li-john10211bc28444f4a7
diff --git a/internal/application/service/knowledge_process_config_test.go b/internal/application/service/knowledge_process_config_test.go
index 2c3f16ae4..731f882fc 100644
--- a/internal/application/service/knowledge_process_config_test.go
+++ b/internal/application/service/knowledge_process_config_test.go
@@ -642,3 +642,17 @@ func TestResolveProcessConfig_SummaryEnabled(t *testing.T) {
})
}
}
+
+func TestBuildSplitterConfigFromChunking_PreservesWikiOnlyZeroOverlap(t *testing.T) {
+ t.Parallel()
+
+ // Mirrors frontend WIKI_ONLY_CHUNKING_PRESET (chunkSize 2048, chunkOverlap 0).
+ kb := &types.KnowledgeBase{
+ ChunkingConfig: types.ChunkingConfig{ChunkSize: 2048, ChunkOverlap: 0},
+ }
+ eff := ResolveProcessConfig(kb, nil)
+ cfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
+
+ require.Equal(t, 2048, cfg.ChunkSize)
+ require.Equal(t, 0, cfg.ChunkOverlap)
+}
diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.go
index 99a86aa14..98f3b597e 100644
--- a/internal/infrastructure/chunker/splitter.go
+++ b/internal/infrastructure/chunker/splitter.go
@@ -95,12 +95,12 @@ type SplitterConfig struct {
// (Go DefaultConfig: 64, knowledge.go buildSplitterConfig: 50, Python
// docreader: 100). All consolidated to 80 here.
//
-// Existing knowledge bases that stored ChunkOverlap=0 in the DB pick
-// this 80 up on next re-index; their previously-indexed embeddings will
-// not match new ones bit-for-bit. Recall stays similar but search
-// ranking can shift slightly. To freeze the old behavior on a per-KB
-// basis, explicitly set ChunkingConfig.ChunkOverlap to 64 before
-// re-indexing.
+// An empty SplitterConfig / ChunkingConfig (both size and overlap unset)
+// still receives DefaultChunkOverlap on normalize. An explicit
+// ChunkOverlap of 0 with a configured ChunkSize is preserved — wiki-only
+// KBs and atomic FAQ-style content rely on true zero overlap. KBs that
+// previously depended on stored 0 being rewritten to 80 should set
+// ChunkOverlap to 80 (or another positive value) before re-indexing.
const (
DefaultChunkSize = 512
DefaultChunkOverlap = 80
diff --git a/internal/infrastructure/chunker/strategy.go b/internal/infrastructure/chunker/strategy.go
index 578c98b8f..6a6e91308 100644
--- a/internal/infrastructure/chunker/strategy.go
+++ b/internal/infrastructure/chunker/strategy.go
@@ -183,15 +183,27 @@ func splitParentChild(text string, parentCfg, childCfg SplitterConfig, withDiagn
return ParentChildResult{Parents: newParents, Children: children}, diag
}
-// NormalizeSplitterConfig applies the same base splitter defaults used by
-// knowledge ingestion before parent-child derivation or single-pass splitting.
-func NormalizeSplitterConfig(cfg SplitterConfig) SplitterConfig {
- if cfg.ChunkSize <= 0 {
+// defaultChunkSizeAndOverlap fills unset size/overlap fields.
+//
+// An empty config (both fields still at Go zero values) still receives the
+// package defaults. When ChunkSize is already set, ChunkOverlap == 0 is kept
+// as an explicit "no overlap" request — matching the wiki-only KB preset and
+// the DefaultChunkOverlap docs ("Use 0 for strictly atomic data").
+func defaultChunkSizeAndOverlap(cfg SplitterConfig) SplitterConfig {
+ sizeUnset := cfg.ChunkSize <= 0
+ if sizeUnset {
cfg.ChunkSize = DefaultChunkSize
}
- if cfg.ChunkOverlap <= 0 {
+ if cfg.ChunkOverlap < 0 || (cfg.ChunkOverlap == 0 && sizeUnset) {
cfg.ChunkOverlap = DefaultChunkOverlap
}
+ return cfg
+}
+
+// NormalizeSplitterConfig applies the same base splitter defaults used by
+// knowledge ingestion before parent-child derivation or single-pass splitting.
+func NormalizeSplitterConfig(cfg SplitterConfig) SplitterConfig {
+ cfg = defaultChunkSizeAndOverlap(cfg)
if len(cfg.Separators) == 0 {
cfg.Separators = []string{"\n\n", "\n", "。"}
}
@@ -312,19 +324,14 @@ func runTier(tier StrategyTier, text string, cfg SplitterConfig, profile *DocPro
}
// ensureDefaults fills in zero-value config fields with sane defaults.
-// NormalizeSplitterConfig also uses these defaults so service callers and
-// direct callers of this package get the same numbers.
+// Size/overlap defaulting is shared with NormalizeSplitterConfig via
+// defaultChunkSizeAndOverlap so service callers and direct callers stay aligned.
//
// When cfg.TokenLimit is set, ChunkSize is clamped to the character budget
// that fits within that token limit (with a 10% safety factor). This makes
// chunks safe for embedding APIs that have hard token caps.
func ensureDefaults(cfg SplitterConfig) SplitterConfig {
- if cfg.ChunkSize <= 0 {
- cfg.ChunkSize = DefaultChunkSize
- }
- if cfg.ChunkOverlap <= 0 {
- cfg.ChunkOverlap = DefaultChunkOverlap
- }
+ cfg = defaultChunkSizeAndOverlap(cfg)
if len(cfg.Separators) == 0 {
cfg.Separators = []string{"\n\n", "\n", "。"}
}
diff --git a/internal/infrastructure/chunker/strategy_test.go b/internal/infrastructure/chunker/strategy_test.go
index 4c565ec00..9bae7e011 100644
--- a/internal/infrastructure/chunker/strategy_test.go
+++ b/internal/infrastructure/chunker/strategy_test.go
@@ -352,3 +352,31 @@ func TestValidateChunks_TolerantTinyTail(t *testing.T) {
t.Errorf("tiny last chunk should be tolerated, got: %s", v.Reason)
}
}
+
+func TestNormalizeSplitterConfig_PreservesExplicitZeroOverlap(t *testing.T) {
+ // Wiki-only / atomic FAQ presets store ChunkSize>0 with ChunkOverlap=0.
+ cfg := NormalizeSplitterConfig(SplitterConfig{ChunkSize: 2048, ChunkOverlap: 0})
+ if cfg.ChunkSize != 2048 {
+ t.Fatalf("chunk size: got %d want 2048", cfg.ChunkSize)
+ }
+ if cfg.ChunkOverlap != 0 {
+ t.Fatalf("explicit zero overlap rewritten to %d", cfg.ChunkOverlap)
+ }
+}
+
+func TestEnsureDefaults_PreservesExplicitZeroOverlap(t *testing.T) {
+ cfg := ensureDefaults(SplitterConfig{ChunkSize: 512, ChunkOverlap: 0})
+ if cfg.ChunkOverlap != 0 {
+ t.Fatalf("explicit zero overlap rewritten to %d", cfg.ChunkOverlap)
+ }
+ if cfg.ChunkSize != 512 {
+ t.Fatalf("chunk size: got %d want 512", cfg.ChunkSize)
+ }
+}
+
+func TestNormalizeSplitterConfig_EmptyStillDefaultsOverlap(t *testing.T) {
+ cfg := NormalizeSplitterConfig(SplitterConfig{})
+ if cfg.ChunkOverlap != DefaultChunkOverlap {
+ t.Fatalf("empty config overlap: got %d want %d", cfg.ChunkOverlap, DefaultChunkOverlap)
+ }
+}