RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: route TokenChunker delimiter_mode "one" to OneChunker in Go ingestion (#18121)
euvrefe963e69ba1b
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.goindex c0d57170bb..230df7fbf4 100644--- a/internal/ingestion/component/chunker/common.go+++ b/internal/ingestion/component/chunker/common.go@@ -36,6 +36,12 @@ import (func newChunkerByName(name string, params map[string]any) (runtime.Component, error) {switch name {case ComponentNameTokenChunker:+ // The DSL contract (shared by the web UI and the Python runtime)+ // expresses single-chunk mode as TokenChunker delimiter_mode "one";+ // in Go that behaviour lives in the OneChunker component.+ if mode, _ := params["delimiter_mode"].(string); mode == "one" {+ return NewOneChunker(params)+ }return NewTokenChunker(params)case ComponentNameTitleChunker:return NewTitleChunker(params)diff --git a/internal/ingestion/component/chunker/one_test.go b/internal/ingestion/component/chunker/one_test.goindex 5fda780c96..f96adf3db5 100644--- a/internal/ingestion/component/chunker/one_test.go+++ b/internal/ingestion/component/chunker/one_test.go@@ -39,6 +39,20 @@ func oneChunksOf(t *testing.T, inputs map[string]any) []map[string]any {return chunks}+// TestNewChunkerByName_TokenChunkerOneMode pins the DSL-contract+// translation: a TokenChunker component whose delimiter_mode is "one"+// (what the web UI and the Python runtime emit) must build a+// OneChunker, not fail schema validation.+func TestNewChunkerByName_TokenChunkerOneMode(t *testing.T) {+ comp, err := newChunkerByName(ComponentNameTokenChunker, map[string]any{"delimiter_mode": "one"})+ if err != nil {+ t.Fatalf("newChunkerByName: %v", err)+ }+ if _, ok := comp.(*OneChunkerComponent); !ok {+ t.Fatalf("component type = %T, want *OneChunkerComponent", comp)+ }+}+// TestOneChunker_Text emits exactly one chunk for a text payload,// faithful to rag/app/one.py (whole file = one chunk).func TestOneChunker_Text(t *testing.T) {