Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: route TokenChunker delimiter_mode "one" to OneChunker in Go ingestion (#18121)

euvrefe963e69ba1b
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go
index c0d57170bb..230df7fbf4 100644
--- a/internal/ingestion/component/chunker/common.go
+++ b/internal/ingestion/component/chunker/common.go
@@ -36,6 +36,12 @@ import (
func newChunkerByName(name string, params map[string]any) (runtime.Component, error) {
switch name {
case ComponentNameTokenChunker:
+ // The DSL contract (shared by the web UI and the Python runtime)
+ // expresses single-chunk mode as TokenChunker delimiter_mode "one";
+ // in Go that behaviour lives in the OneChunker component.
+ if mode, _ := params["delimiter_mode"].(string); mode == "one" {
+ return NewOneChunker(params)
+ }
return NewTokenChunker(params)
case ComponentNameTitleChunker:
return NewTitleChunker(params)
diff --git a/internal/ingestion/component/chunker/one_test.go b/internal/ingestion/component/chunker/one_test.go
index 5fda780c96..f96adf3db5 100644
--- a/internal/ingestion/component/chunker/one_test.go
+++ b/internal/ingestion/component/chunker/one_test.go
@@ -39,6 +39,20 @@ func oneChunksOf(t *testing.T, inputs map[string]any) []map[string]any {
return chunks
}
+// TestNewChunkerByName_TokenChunkerOneMode pins the DSL-contract
+// translation: a TokenChunker component whose delimiter_mode is "one"
+// (what the web UI and the Python runtime emit) must build a
+// OneChunker, not fail schema validation.
+func TestNewChunkerByName_TokenChunkerOneMode(t *testing.T) {
+ comp, err := newChunkerByName(ComponentNameTokenChunker, map[string]any{"delimiter_mode": "one"})
+ if err != nil {
+ t.Fatalf("newChunkerByName: %v", err)
+ }
+ if _, ok := comp.(*OneChunkerComponent); !ok {
+ t.Fatalf("component type = %T, want *OneChunkerComponent", comp)
+ }
+}
+
// TestOneChunker_Text emits exactly one chunk for a text payload,
// faithful to rag/app/one.py (whole file = one chunk).
func TestOneChunker_Text(t *testing.T) {