RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): strip the title-family params ManualChunker cannot honour (#20139) (#20220)
Harsh Kashyap5227bb24f6b5
diff --git a/internal/ingestion/component/chunker/manual.go b/internal/ingestion/component/chunker/manual.goindex a88dba6da3..ef0a7fb9e2 100644--- a/internal/ingestion/component/chunker/manual.go+++ b/internal/ingestion/component/chunker/manual.go@@ -25,9 +25,11 @@ import ("sort""ragflow/internal/agent/runtime"+ "ragflow/internal/common""ragflow/internal/ingestion/component/globals""ragflow/internal/parser/parser"+ "go.uber.org/zap""gorm.io/gorm")@@ -45,18 +47,29 @@ type ManualChunkerComponent struct {// params as TitleChunker (method is pinned to "group"); the position resort is// automatic based on whether the upstream payload carries coordinates.func NewManualChunker(params map[string]any) (runtime.Component, error) {- // method is pinned to "group": ManualChunker's entire value-add is the- // physical-position resort, which only fires inside the group path.- // A caller must not downgrade it to "naive"/"title" (that would skip the- // resort and silently diverge from Python's manual.py), so method is- // deliberately ignored here even if passed in params.- conf := map[string]any{"method": "group"}+ // method, chunk_token_cap, and include_heading_content are pinned for+ // ManualChunker. The chunker's entire value-add is the physical-position+ // resort inside the group method, so method must stay "group". The cap+ // and the heading-content flag are title-chunker-family params that the+ // group path does not honour (see chunkFromRecords: it passes tokenCap=0+ // and only the hierarchy path reads include_heading_content). The+ // operator form is not promised to apply them; we strip them here+ // instead of accepting and silently ignoring them downstream. See issue+ // #20139.+ conf := map[string]any{"method": "group", "chunk_token_cap": 0}+ dropped := make([]string, 0, 3)for k, v := range params {- if k == "method" {+ switch k {+ case "method", "chunk_token_cap", "include_heading_content":+ dropped = append(dropped, k)continue}conf[k] = v}+ if len(dropped) > 0 {+ common.Warn("ManualChunker: ignoring operator-form params that do not apply to this chunker",+ zap.Strings("dropped", dropped))+ }p := defaultsTitle()p.Update(conf)if err := p.TitleChunkerParam.Validate(); err != nil {diff --git a/internal/ingestion/component/chunker/manual_test.go b/internal/ingestion/component/chunker/manual_test.goindex 097d366d48..ddff0c42ff 100644--- a/internal/ingestion/component/chunker/manual_test.go+++ b/internal/ingestion/component/chunker/manual_test.go@@ -430,3 +430,97 @@ func TestManualChunker_MixedPositionedAndPlain(t *testing.T) {t.Fatalf("coordinate-free records lost original order in %q", joined)}}++// TestNewManualChunker_StripsUnsupportedTitleParams pins the issue #20139+// contract: the three title-family params that the operator form accepts but+// ManualChunker cannot honour (`method`, `chunk_token_cap`,+// `include_heading_content`) must not be propagated to `c.param`, regardless+// of what the caller passed. The fix strips them inside NewManualChunker+// with a `common.Warn` so the silent-ignore failure mode becomes a visible+// log line. Runtime semantics are unchanged because the group path would+// have ignored them anyway.+func TestNewManualChunker_StripsUnsupportedTitleParams(t *testing.T) {+ cases := []struct {+ name string+ params map[string]any+ }{+ {+ name: "method downgrade attempt is dropped",+ params: map[string]any{+ "method": "naive",+ "levels": [][]string{{`^# `}},+ "chunk_token_cap": 128,+ "include_heading_content": true,+ },+ },+ {+ name: "method upgrade attempt is dropped",+ params: map[string]any{+ "method": "hierarchy",+ "levels": [][]string{{`^# `}},+ "chunk_token_cap": 256,+ "include_heading_content": false,+ },+ },+ {+ name: "only cap and heading flag set",+ params: map[string]any{+ "levels": [][]string{{`^# `}},+ "chunk_token_cap": 1024,+ "include_heading_content": true,+ },+ },+ {+ name: "no unsupported params is the baseline",+ params: map[string]any{"levels": [][]string{{`^# `}}},+ },+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ c := mustManual(t, tc.params)+ if got, want := c.param.TitleChunkerParam.Method, "group"; got != want {+ t.Errorf("Method = %q, want %q (ManualChunker pins method=group)", got, want)+ }+ if got := c.param.TitleChunkerParam.ChunkTokenCap; got != 0 {+ t.Errorf("ChunkTokenCap = %d, want 0 (stripped before Update)", got)+ }+ if got := c.param.TitleChunkerParam.IncludeHeadingContent; got != false {+ t.Errorf("IncludeHeadingContent = %v, want false (stripped before Update)", got)+ }+ })+ }+}++// TestNewManualChunker_KeepsSupportedParamsAroundPinnedOnes pins that the+// three pinned keys do not bleed into the rest of the conf: levels, hierarchy+// and root_chunk_as_heading still flow through to `c.param` exactly as before.+func TestNewManualChunker_KeepsSupportedParamsAroundPinnedOnes(t *testing.T) {+ c := mustManual(t, map[string]any{+ "method": "naive", // dropped+ "chunk_token_cap": 999, // dropped+ "include_heading_content": true, // dropped+ "levels": [][]string{{`^# `}, {`^## `}},+ "hierarchy": 2,+ "root_chunk_as_heading": true,+ })+ if len(c.param.TitleChunkerParam.Levels) != 2 {+ t.Errorf("Levels len = %d, want 2", len(c.param.TitleChunkerParam.Levels))+ } else {+ if got := c.param.TitleChunkerParam.Levels[0]; len(got) == 0 || got[0] != "^# " {+ t.Errorf("Levels[0] = %#v, want [\"^# \"]", got)+ }+ if got := c.param.TitleChunkerParam.Levels[1]; len(got) == 0 || got[0] != "^## " {+ t.Errorf("Levels[1] = %#v, want [\"^## \"]", got)+ }+ }+ if c.param.TitleChunkerParam.Hierarchy == nil || *c.param.TitleChunkerParam.Hierarchy != 2 {+ t.Errorf("Hierarchy = %v, want pointer to 2", c.param.TitleChunkerParam.Hierarchy)+ }+ if !c.param.TitleChunkerParam.RootChunkAsHeading {+ t.Errorf("RootChunkAsHeading = false, want true")+ }+ if c.param.TitleChunkerParam.Method != "group" {+ t.Errorf("Method = %q, want group (still pinned even with other params set)", c.param.TitleChunkerParam.Method)+ }+}