Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): strip the title-family params ManualChunker cannot honour (#20139) (#20220)

Harsh Kashyap5227bb24f6b5
diff --git a/internal/ingestion/component/chunker/manual.go b/internal/ingestion/component/chunker/manual.go
index a88dba6da3..ef0a7fb9e2 100644
--- a/internal/ingestion/component/chunker/manual.go
+++ b/internal/ingestion/component/chunker/manual.go
@@ -25,9 +25,11 @@ import (
"sort"
"ragflow/internal/agent/runtime"
+ "ragflow/internal/common"
"ragflow/internal/ingestion/component/globals"
"ragflow/internal/parser/parser"
+ "go.uber.org/zap"
"gorm.io/gorm"
)
@@ -45,18 +47,29 @@ type ManualChunkerComponent struct {
// params as TitleChunker (method is pinned to "group"); the position resort is
// automatic based on whether the upstream payload carries coordinates.
func NewManualChunker(params map[string]any) (runtime.Component, error) {
- // method is pinned to "group": ManualChunker's entire value-add is the
- // physical-position resort, which only fires inside the group path.
- // A caller must not downgrade it to "naive"/"title" (that would skip the
- // resort and silently diverge from Python's manual.py), so method is
- // deliberately ignored here even if passed in params.
- conf := map[string]any{"method": "group"}
+ // method, chunk_token_cap, and include_heading_content are pinned for
+ // ManualChunker. The chunker's entire value-add is the physical-position
+ // resort inside the group method, so method must stay "group". The cap
+ // and the heading-content flag are title-chunker-family params that the
+ // group path does not honour (see chunkFromRecords: it passes tokenCap=0
+ // and only the hierarchy path reads include_heading_content). The
+ // operator form is not promised to apply them; we strip them here
+ // instead of accepting and silently ignoring them downstream. See issue
+ // #20139.
+ conf := map[string]any{"method": "group", "chunk_token_cap": 0}
+ dropped := make([]string, 0, 3)
for k, v := range params {
- if k == "method" {
+ switch k {
+ case "method", "chunk_token_cap", "include_heading_content":
+ dropped = append(dropped, k)
continue
}
conf[k] = v
}
+ if len(dropped) > 0 {
+ common.Warn("ManualChunker: ignoring operator-form params that do not apply to this chunker",
+ zap.Strings("dropped", dropped))
+ }
p := defaultsTitle()
p.Update(conf)
if err := p.TitleChunkerParam.Validate(); err != nil {
diff --git a/internal/ingestion/component/chunker/manual_test.go b/internal/ingestion/component/chunker/manual_test.go
index 097d366d48..ddff0c42ff 100644
--- a/internal/ingestion/component/chunker/manual_test.go
+++ b/internal/ingestion/component/chunker/manual_test.go
@@ -430,3 +430,97 @@ func TestManualChunker_MixedPositionedAndPlain(t *testing.T) {
t.Fatalf("coordinate-free records lost original order in %q", joined)
}
}
+
+// TestNewManualChunker_StripsUnsupportedTitleParams pins the issue #20139
+// contract: the three title-family params that the operator form accepts but
+// ManualChunker cannot honour (`method`, `chunk_token_cap`,
+// `include_heading_content`) must not be propagated to `c.param`, regardless
+// of what the caller passed. The fix strips them inside NewManualChunker
+// with a `common.Warn` so the silent-ignore failure mode becomes a visible
+// log line. Runtime semantics are unchanged because the group path would
+// have ignored them anyway.
+func TestNewManualChunker_StripsUnsupportedTitleParams(t *testing.T) {
+ cases := []struct {
+ name string
+ params map[string]any
+ }{
+ {
+ name: "method downgrade attempt is dropped",
+ params: map[string]any{
+ "method": "naive",
+ "levels": [][]string{{`^# `}},
+ "chunk_token_cap": 128,
+ "include_heading_content": true,
+ },
+ },
+ {
+ name: "method upgrade attempt is dropped",
+ params: map[string]any{
+ "method": "hierarchy",
+ "levels": [][]string{{`^# `}},
+ "chunk_token_cap": 256,
+ "include_heading_content": false,
+ },
+ },
+ {
+ name: "only cap and heading flag set",
+ params: map[string]any{
+ "levels": [][]string{{`^# `}},
+ "chunk_token_cap": 1024,
+ "include_heading_content": true,
+ },
+ },
+ {
+ name: "no unsupported params is the baseline",
+ params: map[string]any{"levels": [][]string{{`^# `}}},
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ c := mustManual(t, tc.params)
+ if got, want := c.param.TitleChunkerParam.Method, "group"; got != want {
+ t.Errorf("Method = %q, want %q (ManualChunker pins method=group)", got, want)
+ }
+ if got := c.param.TitleChunkerParam.ChunkTokenCap; got != 0 {
+ t.Errorf("ChunkTokenCap = %d, want 0 (stripped before Update)", got)
+ }
+ if got := c.param.TitleChunkerParam.IncludeHeadingContent; got != false {
+ t.Errorf("IncludeHeadingContent = %v, want false (stripped before Update)", got)
+ }
+ })
+ }
+}
+
+// TestNewManualChunker_KeepsSupportedParamsAroundPinnedOnes pins that the
+// three pinned keys do not bleed into the rest of the conf: levels, hierarchy
+// and root_chunk_as_heading still flow through to `c.param` exactly as before.
+func TestNewManualChunker_KeepsSupportedParamsAroundPinnedOnes(t *testing.T) {
+ c := mustManual(t, map[string]any{
+ "method": "naive", // dropped
+ "chunk_token_cap": 999, // dropped
+ "include_heading_content": true, // dropped
+ "levels": [][]string{{`^# `}, {`^## `}},
+ "hierarchy": 2,
+ "root_chunk_as_heading": true,
+ })
+ if len(c.param.TitleChunkerParam.Levels) != 2 {
+ t.Errorf("Levels len = %d, want 2", len(c.param.TitleChunkerParam.Levels))
+ } else {
+ if got := c.param.TitleChunkerParam.Levels[0]; len(got) == 0 || got[0] != "^# " {
+ t.Errorf("Levels[0] = %#v, want [\"^# \"]", got)
+ }
+ if got := c.param.TitleChunkerParam.Levels[1]; len(got) == 0 || got[0] != "^## " {
+ t.Errorf("Levels[1] = %#v, want [\"^## \"]", got)
+ }
+ }
+ if c.param.TitleChunkerParam.Hierarchy == nil || *c.param.TitleChunkerParam.Hierarchy != 2 {
+ t.Errorf("Hierarchy = %v, want pointer to 2", c.param.TitleChunkerParam.Hierarchy)
+ }
+ if !c.param.TitleChunkerParam.RootChunkAsHeading {
+ t.Errorf("RootChunkAsHeading = false, want true")
+ }
+ if c.param.TitleChunkerParam.Method != "group" {
+ t.Errorf("Method = %q, want group (still pinned even with other params set)", c.param.TitleChunkerParam.Method)
+ }
+}