Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(chunker): preserve explicit zero chunk_overlap on ingest (#4276)
Li-john10211bc28444f4a7
diff --git a/internal/application/service/knowledge_process_config_test.go b/internal/application/service/knowledge_process_config_test.goindex 2c3f16ae4..731f882fc 100644--- a/internal/application/service/knowledge_process_config_test.go+++ b/internal/application/service/knowledge_process_config_test.go@@ -642,3 +642,17 @@ func TestResolveProcessConfig_SummaryEnabled(t *testing.T) {})}}++func TestBuildSplitterConfigFromChunking_PreservesWikiOnlyZeroOverlap(t *testing.T) {+ t.Parallel()++ // Mirrors frontend WIKI_ONLY_CHUNKING_PRESET (chunkSize 2048, chunkOverlap 0).+ kb := &types.KnowledgeBase{+ ChunkingConfig: types.ChunkingConfig{ChunkSize: 2048, ChunkOverlap: 0},+ }+ eff := ResolveProcessConfig(kb, nil)+ cfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)++ require.Equal(t, 2048, cfg.ChunkSize)+ require.Equal(t, 0, cfg.ChunkOverlap)+}diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.goindex 99a86aa14..98f3b597e 100644--- a/internal/infrastructure/chunker/splitter.go+++ b/internal/infrastructure/chunker/splitter.go@@ -95,12 +95,12 @@ type SplitterConfig struct {// (Go DefaultConfig: 64, knowledge.go buildSplitterConfig: 50, Python// docreader: 100). All consolidated to 80 here.//-// Existing knowledge bases that stored ChunkOverlap=0 in the DB pick-// this 80 up on next re-index; their previously-indexed embeddings will-// not match new ones bit-for-bit. Recall stays similar but search-// ranking can shift slightly. To freeze the old behavior on a per-KB-// basis, explicitly set ChunkingConfig.ChunkOverlap to 64 before-// re-indexing.+// An empty SplitterConfig / ChunkingConfig (both size and overlap unset)+// still receives DefaultChunkOverlap on normalize. An explicit+// ChunkOverlap of 0 with a configured ChunkSize is preserved — wiki-only+// KBs and atomic FAQ-style content rely on true zero overlap. KBs that+// previously depended on stored 0 being rewritten to 80 should set+// ChunkOverlap to 80 (or another positive value) before re-indexing.const (DefaultChunkSize = 512DefaultChunkOverlap = 80diff --git a/internal/infrastructure/chunker/strategy.go b/internal/infrastructure/chunker/strategy.goindex 578c98b8f..6a6e91308 100644--- a/internal/infrastructure/chunker/strategy.go+++ b/internal/infrastructure/chunker/strategy.go@@ -183,15 +183,27 @@ func splitParentChild(text string, parentCfg, childCfg SplitterConfig, withDiagnreturn ParentChildResult{Parents: newParents, Children: children}, diag}-// NormalizeSplitterConfig applies the same base splitter defaults used by-// knowledge ingestion before parent-child derivation or single-pass splitting.-func NormalizeSplitterConfig(cfg SplitterConfig) SplitterConfig {- if cfg.ChunkSize <= 0 {+// defaultChunkSizeAndOverlap fills unset size/overlap fields.+//+// An empty config (both fields still at Go zero values) still receives the+// package defaults. When ChunkSize is already set, ChunkOverlap == 0 is kept+// as an explicit "no overlap" request — matching the wiki-only KB preset and+// the DefaultChunkOverlap docs ("Use 0 for strictly atomic data").+func defaultChunkSizeAndOverlap(cfg SplitterConfig) SplitterConfig {+ sizeUnset := cfg.ChunkSize <= 0+ if sizeUnset {cfg.ChunkSize = DefaultChunkSize}- if cfg.ChunkOverlap <= 0 {+ if cfg.ChunkOverlap < 0 || (cfg.ChunkOverlap == 0 && sizeUnset) {cfg.ChunkOverlap = DefaultChunkOverlap}+ return cfg+}++// NormalizeSplitterConfig applies the same base splitter defaults used by+// knowledge ingestion before parent-child derivation or single-pass splitting.+func NormalizeSplitterConfig(cfg SplitterConfig) SplitterConfig {+ cfg = defaultChunkSizeAndOverlap(cfg)if len(cfg.Separators) == 0 {cfg.Separators = []string{"\n\n", "\n", "。"}}@@ -312,19 +324,14 @@ func runTier(tier StrategyTier, text string, cfg SplitterConfig, profile *DocPro}// ensureDefaults fills in zero-value config fields with sane defaults.-// NormalizeSplitterConfig also uses these defaults so service callers and-// direct callers of this package get the same numbers.+// Size/overlap defaulting is shared with NormalizeSplitterConfig via+// defaultChunkSizeAndOverlap so service callers and direct callers stay aligned.//// When cfg.TokenLimit is set, ChunkSize is clamped to the character budget// that fits within that token limit (with a 10% safety factor). This makes// chunks safe for embedding APIs that have hard token caps.func ensureDefaults(cfg SplitterConfig) SplitterConfig {- if cfg.ChunkSize <= 0 {- cfg.ChunkSize = DefaultChunkSize- }- if cfg.ChunkOverlap <= 0 {- cfg.ChunkOverlap = DefaultChunkOverlap- }+ cfg = defaultChunkSizeAndOverlap(cfg)if len(cfg.Separators) == 0 {cfg.Separators = []string{"\n\n", "\n", "。"}}diff --git a/internal/infrastructure/chunker/strategy_test.go b/internal/infrastructure/chunker/strategy_test.goindex 4c565ec00..9bae7e011 100644--- a/internal/infrastructure/chunker/strategy_test.go+++ b/internal/infrastructure/chunker/strategy_test.go@@ -352,3 +352,31 @@ func TestValidateChunks_TolerantTinyTail(t *testing.T) {t.Errorf("tiny last chunk should be tolerated, got: %s", v.Reason)}}++func TestNormalizeSplitterConfig_PreservesExplicitZeroOverlap(t *testing.T) {+ // Wiki-only / atomic FAQ presets store ChunkSize>0 with ChunkOverlap=0.+ cfg := NormalizeSplitterConfig(SplitterConfig{ChunkSize: 2048, ChunkOverlap: 0})+ if cfg.ChunkSize != 2048 {+ t.Fatalf("chunk size: got %d want 2048", cfg.ChunkSize)+ }+ if cfg.ChunkOverlap != 0 {+ t.Fatalf("explicit zero overlap rewritten to %d", cfg.ChunkOverlap)+ }+}++func TestEnsureDefaults_PreservesExplicitZeroOverlap(t *testing.T) {+ cfg := ensureDefaults(SplitterConfig{ChunkSize: 512, ChunkOverlap: 0})+ if cfg.ChunkOverlap != 0 {+ t.Fatalf("explicit zero overlap rewritten to %d", cfg.ChunkOverlap)+ }+ if cfg.ChunkSize != 512 {+ t.Fatalf("chunk size: got %d want 512", cfg.ChunkSize)+ }+}++func TestNormalizeSplitterConfig_EmptyStillDefaultsOverlap(t *testing.T) {+ cfg := NormalizeSplitterConfig(SplitterConfig{})+ if cfg.ChunkOverlap != DefaultChunkOverlap {+ t.Fatalf("empty config overlap: got %d want %d", cfg.ChunkOverlap, DefaultChunkOverlap)+ }+}