Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(ingestion): centralize component names and identity checks (#22081)

jay777213a6b6dca78a0
diff --git a/internal/common/component_identity.go b/internal/common/component_identity.go
new file mode 100644
index 0000000000..f97645da05
--- /dev/null
+++ b/internal/common/component_identity.go
@@ -0,0 +1,79 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package common
+
+import (
+ "sort"
+ "strings"
+)
+
+// hasComponentIDPrefix reports whether id is "<componentName>:<non-empty instance>".
+func hasComponentIDPrefix(id, componentName string) bool {
+ kind, instance, ok := strings.Cut(id, ":")
+ return ok && kind == componentName && instance != ""
+}
+
+// IsExtractorComponentID reports whether id names an Extractor node in a
+// component-scoped parser_config or pipeline DSL. Only the canonical
+// "Extractor:<instance>" form is accepted.
+func IsExtractorComponentID(id string) bool {
+ return hasComponentIDPrefix(id, ComponentNameExtractor)
+}
+
+// IsExtractorComponent reports whether the component name or component ID
+// identifies an Extractor.
+func IsExtractorComponent(id, componentName string) bool {
+ return componentName == ComponentNameExtractor || IsExtractorComponentID(id)
+}
+
+// IsParserComponentID reports whether id names a Parser node in a
+// component-scoped parser_config or pipeline DSL.
+func IsParserComponentID(id string) bool {
+ return hasComponentIDPrefix(id, ComponentNameParser)
+}
+
+// IsParserComponent reports whether the component name or component ID
+// identifies a Parser.
+func IsParserComponent(id, componentName string) bool {
+ return componentName == ComponentNameParser || IsParserComponentID(id)
+}
+
+// IsCompilerComponentID reports whether id names a Compiler node in a
+// component-scoped parser_config or pipeline DSL.
+func IsCompilerComponentID(id string) bool {
+ return hasComponentIDPrefix(id, ComponentNameCompiler)
+}
+
+// IsComponentKey reports whether id is a canonical component-scoped
+// parser_config key: "<ComponentName>:<non-empty instance>".
+func IsComponentKey(id string) bool {
+ kind, instance, ok := strings.Cut(id, ":")
+ return ok && instance != "" && IsComponentName(kind)
+}
+
+// SortedExtractorKeys returns the parser_config keys that name Extractor nodes,
+// sorted so callers resolve multiple Extractor nodes deterministically.
+func SortedExtractorKeys(parserConfig map[string]any) []string {
+ keys := make([]string, 0)
+ for key := range parserConfig {
+ if IsExtractorComponentID(key) {
+ keys = append(keys, key)
+ }
+ }
+ sort.Strings(keys)
+ return keys
+}
diff --git a/internal/common/component_identity_test.go b/internal/common/component_identity_test.go
new file mode 100644
index 0000000000..2f53313f8f
--- /dev/null
+++ b/internal/common/component_identity_test.go
@@ -0,0 +1,210 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package common
+
+import (
+ "reflect"
+ "testing"
+)
+
+func TestIsExtractorComponentID(t *testing.T) {
+ tests := []struct {
+ id string
+ want bool
+ }{
+ {id: "Extractor", want: false},
+ {id: "extractor", want: false},
+ {id: "Extractor:AutoExtractDefault", want: true},
+ {id: "extractor:node", want: false},
+ {id: "EXTRACTOR:Node", want: false},
+ {id: "Extractor:", want: false},
+ {id: "extractor_0", want: false},
+ {id: "Extractor_0", want: false},
+ {id: "extractors", want: false},
+ {id: "my_extractor", want: false},
+ {id: "Parser:Extractor", want: false},
+ {id: "", want: false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.id, func(t *testing.T) {
+ if got := IsExtractorComponentID(tt.id); got != tt.want {
+ t.Fatalf("IsExtractorComponentID(%q) = %v, want %v", tt.id, got, tt.want)
+ }
+ })
+ }
+}
+
+func TestIsParserComponentID(t *testing.T) {
+ tests := []struct {
+ id string
+ want bool
+ }{
+ {id: "Parser:HipSignsRhyme", want: true},
+ {id: "Parser:", want: false},
+ {id: "parser:node", want: false},
+ {id: "PARSER:Node", want: false},
+ {id: "Parser", want: false},
+ {id: "parser_0", want: false},
+ {id: "Extractor:Parser", want: false},
+ {id: "", want: false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.id, func(t *testing.T) {
+ if got := IsParserComponentID(tt.id); got != tt.want {
+ t.Fatalf("IsParserComponentID(%q) = %v, want %v", tt.id, got, tt.want)
+ }
+ })
+ }
+}
+
+func TestIsCompilerComponentID(t *testing.T) {
+ tests := []struct {
+ id string
+ want bool
+ }{
+ {id: "Compiler:CurvySunsLove", want: true},
+ {id: "Compiler:", want: false},
+ {id: "compiler:node", want: false},
+ {id: "COMPILER:Node", want: false},
+ {id: "compiler_0", want: false},
+ {id: "Compiler", want: false},
+ {id: "compilers", want: false},
+ {id: "Parser:Compiler", want: false},
+ {id: "", want: false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.id, func(t *testing.T) {
+ if got := IsCompilerComponentID(tt.id); got != tt.want {
+ t.Fatalf("IsCompilerComponentID(%q) = %v, want %v", tt.id, got, tt.want)
+ }
+ })
+ }
+}
+
+func TestIsComponentKey(t *testing.T) {
+ tests := []struct {
+ id string
+ want bool
+ }{
+ {id: "Extractor:AutoExtractDefault", want: true},
+ {id: "Parser:HipSignsRhyme", want: true},
+ {id: "Parser:", want: false},
+ {id: "GeneralChunker:SixApplesFall", want: true},
+ {id: "File:node", want: true},
+ {id: "Extractor:", want: false},
+ {id: ":foo", want: false},
+ {id: "foo:bar", want: false},
+ {id: "extractor:0", want: false},
+ {id: "extractor_0", want: false},
+ {id: "compiler_0", want: false},
+ {id: "Extractor", want: false},
+ {id: "chunk_token_num", want: false},
+ {id: "parser_0", want: false},
+ {id: "", want: false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.id, func(t *testing.T) {
+ if got := IsComponentKey(tt.id); got != tt.want {
+ t.Fatalf("IsComponentKey(%q) = %v, want %v", tt.id, got, tt.want)
+ }
+ })
+ }
+}
+
+func TestIsComponentName(t *testing.T) {
+ names := []string{
+ ComponentNameFile,
+ ComponentNameParser,
+ ComponentNameExtractor,
+ ComponentNameCompiler,
+ ComponentNameTokenizer,
+ ComponentNameGeneralChunker,
+ ComponentNameTokenChunker,
+ ComponentNameTitleChunker,
+ ComponentNameManualChunker,
+ ComponentNameOneChunker,
+ ComponentNamePageChunker,
+ ComponentNameQAChunker,
+ ComponentNameTableChunker,
+ ComponentNameGroupTitleChunker,
+ ComponentNameHierarchyTitleChunker,
+ }
+ for _, name := range names {
+ if !IsComponentName(name) {
+ t.Errorf("IsComponentName(%q) = false, want true", name)
+ }
+ }
+ for _, name := range []string{"", "Chunker", "extractor", "Extractor "} {
+ if IsComponentName(name) {
+ t.Errorf("IsComponentName(%q) = true, want false", name)
+ }
+ }
+}
+
+func TestSortedExtractorKeys(t *testing.T) {
+ parserConfig := map[string]any{
+ "Extractor:B": nil,
+ "Extractor:A": nil,
+ "Extractor:": nil,
+ "Extractor": nil,
+ "extractor_0": nil,
+ "Parser:X": nil,
+ }
+ want := []string{"Extractor:A", "Extractor:B"}
+ if got := SortedExtractorKeys(parserConfig); !reflect.DeepEqual(got, want) {
+ t.Fatalf("SortedExtractorKeys = %#v, want %#v", got, want)
+ }
+}
+
+func TestIsExtractorComponent(t *testing.T) {
+ tests := []struct {
+ id string
+ name string
+ want bool
+ }{
+ {id: "node-1", name: "Extractor", want: true},
+ {id: "Extractor:node-1", name: "", want: true},
+ {id: "extractor_0", name: "", want: false},
+ {id: "node-1", name: "Parser", want: false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name+"/"+tt.id, func(t *testing.T) {
+ if got := IsExtractorComponent(tt.id, tt.name); got != tt.want {
+ t.Fatalf("IsExtractorComponent(%q, %q) = %v, want %v", tt.id, tt.name, got, tt.want)
+ }
+ })
+ }
+}
+
+func TestIsParserComponent(t *testing.T) {
+ tests := []struct {
+ id string
+ name string
+ want bool
+ }{
+ {id: "node-1", name: "Parser", want: true},
+ {id: "Parser:node-1", name: "", want: true},
+ {id: "node-1", name: "Extractor", want: false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name+"/"+tt.id, func(t *testing.T) {
+ if got := IsParserComponent(tt.id, tt.name); got != tt.want {
+ t.Fatalf("IsParserComponent(%q, %q) = %v, want %v", tt.id, tt.name, got, tt.want)
+ }
+ })
+ }
+}
diff --git a/internal/common/component_names.go b/internal/common/component_names.go
new file mode 100644
index 0000000000..e40e2fbcbf
--- /dev/null
+++ b/internal/common/component_names.go
@@ -0,0 +1,61 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package common
+
+// Canonical ingestion component names.
+const (
+ ComponentNameFile = "File"
+ ComponentNameParser = "Parser"
+ ComponentNameExtractor = "Extractor"
+ ComponentNameCompiler = "Compiler"
+ ComponentNameTokenizer = "Tokenizer"
+
+ ComponentNameGeneralChunker = "GeneralChunker"
+ ComponentNameTokenChunker = "TokenChunker"
+ ComponentNameTitleChunker = "TitleChunker"
+ ComponentNameManualChunker = "ManualChunker"
+ ComponentNameOneChunker = "OneChunker"
+ ComponentNamePageChunker = "PageChunker"
+ ComponentNameQAChunker = "QAChunker"
+ ComponentNameTableChunker = "TableChunker"
+ ComponentNameGroupTitleChunker = "GroupTitleChunker"
+ ComponentNameHierarchyTitleChunker = "HierarchyTitleChunker"
+)
+
+// IsComponentName reports whether name is a canonical ingestion component name.
+func IsComponentName(name string) bool {
+ switch name {
+ case ComponentNameFile,
+ ComponentNameParser,
+ ComponentNameExtractor,
+ ComponentNameCompiler,
+ ComponentNameTokenizer,
+ ComponentNameGeneralChunker,
+ ComponentNameTokenChunker,
+ ComponentNameTitleChunker,
+ ComponentNameManualChunker,
+ ComponentNameOneChunker,
+ ComponentNamePageChunker,
+ ComponentNameQAChunker,
+ ComponentNameTableChunker,
+ ComponentNameGroupTitleChunker,
+ ComponentNameHierarchyTitleChunker:
+ return true
+ default:
+ return false
+ }
+}
diff --git a/internal/common/metadata_utils.go b/internal/common/metadata_utils.go
index 0a4d8f8db8..3d461ec9c2 100644
--- a/internal/common/metadata_utils.go
+++ b/internal/common/metadata_utils.go
@@ -18,7 +18,6 @@ package common
import (
"regexp"
- "sort"
"strconv"
"strings"
)
@@ -441,15 +440,7 @@ func ExtractorMetadataConfig(parserConfig map[string]any) (map[string]any, bool)
if parserConfig == nil {
return nil, false
}
- var extractorKeys []string
- for k := range parserConfig {
- lower := strings.ToLower(k)
- if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") {
- extractorKeys = append(extractorKeys, k)
- }
- }
- sort.Strings(extractorKeys)
- for _, k := range extractorKeys {
+ for _, k := range SortedExtractorKeys(parserConfig) {
node, ok := parserConfig[k].(map[string]any)
if !ok {
continue
diff --git a/internal/common/parser_config.go b/internal/common/parser_config.go
index 92f0163e68..4dc93c0b6d 100644
--- a/internal/common/parser_config.go
+++ b/internal/common/parser_config.go
@@ -105,7 +105,7 @@ func ExtractPipelineDefaults(dsl map[string]interface{}) map[string]interface{}
continue
}
name, _ := obj["component_name"].(string)
- if name == "" || name == "File" {
+ if name == "" || name == ComponentNameFile {
continue
}
params, _ := obj["params"].(map[string]interface{})
diff --git a/internal/dao/ingestion_parser_config_migration.go b/internal/dao/ingestion_parser_config_migration.go
index 0630165877..b1c781ea74 100644
--- a/internal/dao/ingestion_parser_config_migration.go
+++ b/internal/dao/ingestion_parser_config_migration.go
@@ -1164,12 +1164,9 @@ func migrateExtractorConfig(config map[string]any, parserID string, pipelineID *
targets = append(targets, newNode)
}
} else {
- for id, raw := range config {
- id = strings.ToLower(id)
- if strings.HasPrefix(id, "extractor:") || strings.HasPrefix(id, "extractor_") {
- if node, ok := raw.(map[string]any); ok {
- targets = append(targets, node)
- }
+ for _, id := range common.SortedExtractorKeys(config) {
+ if node, ok := config[id].(map[string]any); ok {
+ targets = append(targets, node)
}
}
if len(targets) == 0 {
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index d871355d56..cbb26b92db 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -44,8 +44,6 @@ import (
"ragflow/internal/parser/tableutil"
)
-const ComponentNameGeneralChunker = "GeneralChunker"
-
var contextSentencePattern = regexp.MustCompile(`([。!??;!\n]|\. )`)
type generalChunkerParam struct {
diff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.go
index 38c8e7d238..6cd1cdd116 100644
--- a/internal/ingestion/component/chunker/group.go
+++ b/internal/ingestion/component/chunker/group.go
@@ -51,8 +51,6 @@ import (
"ragflow/internal/tokenizer"
)
-const ComponentNameGroupTitleChunker = "GroupTitleChunker"
-
// MIN_GROUP_TOKENS / MAX_GROUP_TOKENS mirror the python constants in
// group_chunker.py:22-23. They drive the merge heuristic for adjacent
// text records.
diff --git a/internal/ingestion/component/chunker/hierarchy.go b/internal/ingestion/component/chunker/hierarchy.go
index 63e3b9d009..b59134c17d 100644
--- a/internal/ingestion/component/chunker/hierarchy.go
+++ b/internal/ingestion/component/chunker/hierarchy.go
@@ -51,8 +51,6 @@ import (
"ragflow/internal/ingestion/component/globals"
)
-const ComponentNameHierarchyTitleChunker = "HierarchyTitleChunker"
-
// chunkNode mirrors python `_ChunkNode` (hierarchy_chunker.py:22-32).
type chunkNode struct {
level int
diff --git a/internal/ingestion/component/chunker/manual.go b/internal/ingestion/component/chunker/manual.go
index bfdf4d8a69..563390b8dd 100644
--- a/internal/ingestion/component/chunker/manual.go
+++ b/internal/ingestion/component/chunker/manual.go
@@ -33,8 +33,6 @@ import (
"gorm.io/gorm"
)
-const ComponentNameManualChunker = "ManualChunker"
-
// ManualChunkerComponent is the standalone manual-layout chunker. It shares
// the GroupTitleChunker grouping body (chunkFromRecords) and only differs by
// the optional physical-position pre-sort.
diff --git a/internal/ingestion/component/chunker/names.go b/internal/ingestion/component/chunker/names.go
new file mode 100644
index 0000000000..fdb5544cd0
--- /dev/null
+++ b/internal/ingestion/component/chunker/names.go
@@ -0,0 +1,33 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunker
+
+import "ragflow/internal/common"
+
+// Chunker component names re-exported from the canonical component registry.
+const (
+ ComponentNameGeneralChunker = common.ComponentNameGeneralChunker
+ ComponentNameTokenChunker = common.ComponentNameTokenChunker
+ ComponentNameTitleChunker = common.ComponentNameTitleChunker
+ ComponentNameManualChunker = common.ComponentNameManualChunker
+ ComponentNameOneChunker = common.ComponentNameOneChunker
+ ComponentNamePageChunker = common.ComponentNamePageChunker
+ ComponentNameQAChunker = common.ComponentNameQAChunker
+ ComponentNameTableChunker = common.ComponentNameTableChunker
+ ComponentNameGroupTitleChunker = common.ComponentNameGroupTitleChunker
+ ComponentNameHierarchyTitleChunker = common.ComponentNameHierarchyTitleChunker
+)
diff --git a/internal/ingestion/component/chunker/one.go b/internal/ingestion/component/chunker/one.go
index 1eeb11eed7..b569811480 100644
--- a/internal/ingestion/component/chunker/one.go
+++ b/internal/ingestion/component/chunker/one.go
@@ -38,8 +38,6 @@ import (
"gorm.io/gorm"
)
-const ComponentNameOneChunker = "OneChunker"
-
type oneChunkerParam struct{}
func (p *oneChunkerParam) Update(conf map[string]any) {}
diff --git a/internal/ingestion/component/chunker/page.go b/internal/ingestion/component/chunker/page.go
index f0428c38f4..deb6bc81f3 100644
--- a/internal/ingestion/component/chunker/page.go
+++ b/internal/ingestion/component/chunker/page.go
@@ -34,8 +34,6 @@ import (
"gorm.io/gorm"
)
-const ComponentNamePageChunker = "PageChunker"
-
type pageChunkerParam struct{}
func (p *pageChunkerParam) Update(conf map[string]any) {}
diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.go
index 520607e58d..6e9dbc7254 100644
--- a/internal/ingestion/component/chunker/qa.go
+++ b/internal/ingestion/component/chunker/qa.go
@@ -48,8 +48,6 @@ import (
"ragflow/internal/tokenizer"
)
-const ComponentNameQAChunker = "QAChunker"
-
type qaChunkerParam struct {
Lang string `json:"lang,omitempty"`
}
diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.go
index c662b275af..6e451ce7b5 100644
--- a/internal/ingestion/component/chunker/table.go
+++ b/internal/ingestion/component/chunker/table.go
@@ -41,8 +41,6 @@ import (
"gorm.io/gorm"
)
-const ComponentNameTableChunker = "TableChunker"
-
type tableChunkerParam struct{}
func (p *tableChunkerParam) Update(conf map[string]any) {}
diff --git a/internal/ingestion/component/chunker/title.go b/internal/ingestion/component/chunker/title.go
index ad658cbe5e..629d44e598 100644
--- a/internal/ingestion/component/chunker/title.go
+++ b/internal/ingestion/component/chunker/title.go
@@ -82,8 +82,6 @@ import (
"gorm.io/gorm"
)
-const ComponentNameTitleChunker = "TitleChunker"
-
type titleChunkerParam struct {
schema.TitleChunkerParam
}
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index 7989563828..f5793a56e0 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -90,8 +90,6 @@ import (
"ragflow/internal/parser/chunk"
)
-const ComponentNameTokenChunker = "TokenChunker"
-
type tokenChunkerParam struct {
schema.TokenChunkerParam
}
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go
index 99db93bfe9..2e049e027d 100644
--- a/internal/ingestion/component/extractor.go
+++ b/internal/ingestion/component/extractor.go
@@ -66,8 +66,8 @@ import (
)
// componentNameExtractor is the canonical Extractor component name; the value
-// lives in the schema package so the pipeline layer can reference the same
-// identity.
+// lives in common and is re-exported by the schema package so both layers
+// reference the same identity.
const componentNameExtractor = schema.ComponentNameExtractor
// defaultExtractorChatTimeout bounds one Extractor LLM call, including
diff --git a/internal/ingestion/component/file.go b/internal/ingestion/component/file.go
index bbadda72a8..ebd98e0d69 100644
--- a/internal/ingestion/component/file.go
+++ b/internal/ingestion/component/file.go
@@ -50,7 +50,7 @@ import (
"gorm.io/gorm"
)
-const ComponentNameFile = "File"
+const ComponentNameFile = schema.ComponentNameFile
// FileComponent resolves document/file metadata and forwards enough
// identity for downstream Parser to fetch bytes.
diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go
index 1fab7e5d2d..5740c14877 100644
--- a/internal/ingestion/component/knowledge_compiler/component.go
+++ b/internal/ingestion/component/knowledge_compiler/component.go
@@ -47,8 +47,8 @@ var chunkerOutputs = map[string]string{
// (rag/flow/compiler/compiler.py registers component_name = "Compiler"), so a
// canvas saved by the Python frontend and Go's built-in ingestion templates
// both reference the node as "Compiler" and resolve to the same component.
-// The value itself lives in the schema package so the pipeline layer can
-// reference the same identity.
+// The value itself lives in common and is re-exported by the schema package so
+// the pipeline layer references the same identity.
const componentNameCompiler = schema.ComponentNameCompiler
// KnowledgeCompilerComponent is the runtime.Component surface. Param is set at
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index 71cd9ca5b1..0e27602f67 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -96,7 +96,8 @@ import (
)
// ComponentNameParser is the canonical Parser component name; the value lives
-// in the schema package so the pipeline layer can reference the same identity.
+// in common and is re-exported by the schema package so both layers reference
+// the same identity.
const ComponentNameParser = schema.ComponentNameParser
// pageFormFeed is the byte that text-page mode treats as a
diff --git a/internal/ingestion/component/schema/component_names.go b/internal/ingestion/component/schema/component_names.go
index 631d78dd02..809c41e82b 100644
--- a/internal/ingestion/component/schema/component_names.go
+++ b/internal/ingestion/component/schema/component_names.go
@@ -16,16 +16,19 @@
package schema
-// Canonical ingestion component names. These are the identities components
-// register under in runtime.DefaultRegistry and the values a DSL
-// component_name carries. Layers that must recognise a component by name
-// across the component/pipeline boundary (dispatch, param whitelists,
-// operator-facing failure messages) reference these constants instead of
-// repeating the literals; the schema package is the shared dependency both
-// sides already import. More names can join this set as other layers need
-// them.
+import "ragflow/internal/common"
+
+// Canonical ingestion component names re-exported from common for callers that
+// need them across the pipeline/component boundary.
const (
- ComponentNameParser = "Parser"
- ComponentNameExtractor = "Extractor"
- ComponentNameCompiler = "Compiler"
+ ComponentNameFile = common.ComponentNameFile
+ ComponentNameParser = common.ComponentNameParser
+ ComponentNameExtractor = common.ComponentNameExtractor
+ ComponentNameCompiler = common.ComponentNameCompiler
+ ComponentNameTokenizer = common.ComponentNameTokenizer
+
+ ComponentNameGeneralChunker = common.ComponentNameGeneralChunker
+ ComponentNameTokenChunker = common.ComponentNameTokenChunker
+ ComponentNameTitleChunker = common.ComponentNameTitleChunker
+ ComponentNameManualChunker = common.ComponentNameManualChunker
)
diff --git a/internal/ingestion/component/tag_query.go b/internal/ingestion/component/tag_query.go
index 14737f2386..db7760d568 100644
--- a/internal/ingestion/component/tag_query.go
+++ b/internal/ingestion/component/tag_query.go
@@ -22,6 +22,7 @@ import (
"sort"
"strings"
+ "ragflow/internal/common"
"ragflow/internal/entity"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/tokenizer"
@@ -87,20 +88,9 @@ func TagFileIDFromParserConfig(parserConfig map[string]any) string {
if v := tagFileIDFromTagsMap(parserConfig["tags"]); v != "" {
return v
}
- // Several Extractor nodes can each declare their own tags block, so the
- // winner has to be picked in a stable order: Go randomizes map iteration,
- // and an unstable pick would let the dataset's tag source change between
- // process restarts (different chunks tagged, different retrieval
- // features). Sorting makes it deterministic; "Extractor" sorts before
- // "Extractor:<name>" because it is a prefix, so the unscoped node wins.
- keys := make([]string, 0, len(parserConfig))
- for key := range parserConfig {
- if key == "Extractor" || strings.HasPrefix(key, "Extractor:") {
- keys = append(keys, key)
- }
- }
- sort.Strings(keys)
- for _, key := range keys {
+ // Several Extractor nodes can each declare their own tags block. Pick the
+ // winner deterministically by sorted component ID.
+ for _, key := range common.SortedExtractorKeys(parserConfig) {
extractorMap, ok := asStringAnyMap(parserConfig[key])
if !ok {
continue
diff --git a/internal/ingestion/component/tag_query_test.go b/internal/ingestion/component/tag_query_test.go
index 1fa2b4884e..902919ae4e 100644
--- a/internal/ingestion/component/tag_query_test.go
+++ b/internal/ingestion/component/tag_query_test.go
@@ -56,6 +56,33 @@ func TestTagFileIDFromParserConfig(t *testing.T) {
},
want: "file-3",
},
+ {
+ name: "lowercase extractor form is not canonical",
+ parserConfig: map[string]any{
+ "extractor:AutoExtractDefault": map[string]any{
+ "tags": map[string]any{"tag_file_id": "file-5"},
+ },
+ },
+ want: "",
+ },
+ {
+ name: "bare extractor form is not scoped",
+ parserConfig: map[string]any{
+ "Extractor": map[string]any{
+ "tags": map[string]any{"tag_file_id": "file-6"},
+ },
+ },
+ want: "",
+ },
+ {
+ name: "underscore extractor form is not scoped",
+ parserConfig: map[string]any{
+ "extractor_0": map[string]any{
+ "tags": map[string]any{"tag_file_id": "file-7"},
+ },
+ },
+ want: "",
+ },
{
name: "entity.JSONMap nested value",
parserConfig: map[string]any{
@@ -85,29 +112,17 @@ func TestTagFileIDFromParserConfig(t *testing.T) {
func TestTagFileIDFromParserConfigPicksStableExtractorNode(t *testing.T) {
// Several Extractor nodes can each declare their own tags block. Go
- // randomizes map iteration, so an unsorted scan would return a different
- // tag file - and thus tag the dataset from a different tag source - on
- // every call and every process restart.
- parserConfig := map[string]any{
- "Extractor": map[string]any{"tags": map[string]any{"tag_file_id": "file-unscoped"}},
- }
+ // randomizes map iteration, so the chosen tag source must not depend on
+ // iteration order.
+ parserConfig := map[string]any{}
for i := 0; i < 8; i++ {
parserConfig[fmt.Sprintf("Extractor:node-%d", i)] = map[string]any{
"tags": map[string]any{"tag_file_id": fmt.Sprintf("file-%d", i)},
}
}
- // The unscoped "Extractor" node wins: it is a prefix of the scoped keys,
- // so it sorts first.
- for i := 0; i < 200; i++ {
- if got := TagFileIDFromParserConfig(parserConfig); got != "file-unscoped" {
- t.Fatalf("TagFileIDFromParserConfig with unscoped node = %q, want %q", got, "file-unscoped")
- }
- }
- // Without it the lowest-sorting scoped node wins, still stably.
- delete(parserConfig, "Extractor")
for i := 0; i < 200; i++ {
if got := TagFileIDFromParserConfig(parserConfig); got != "file-0" {
- t.Fatalf("TagFileIDFromParserConfig across scoped nodes = %q, want %q", got, "file-0")
+ t.Fatalf("TagFileIDFromParserConfig = %q, want %q", got, "file-0")
}
}
}
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go
index bf3c5ba428..b14345754b 100644
--- a/internal/ingestion/component/tokenizer.go
+++ b/internal/ingestion/component/tokenizer.go
@@ -98,7 +98,7 @@ import (
"ragflow/internal/tokenizer"
)
-const ComponentNameTokenizer = "Tokenizer"
+const ComponentNameTokenizer = schema.ComponentNameTokenizer
// titleExtRE strips a trailing file-extension (e.g. ".pdf") from the
// upstream document name before tokenizing it. Mirrors the python
diff --git a/internal/ingestion/pipeline/component_params.go b/internal/ingestion/pipeline/component_params.go
index a06fab7d89..9f51a3e769 100644
--- a/internal/ingestion/pipeline/component_params.go
+++ b/internal/ingestion/pipeline/component_params.go
@@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
+ "ragflow/internal/common"
"ragflow/internal/ingestion/component/schema"
)
@@ -42,7 +43,10 @@ func ExtractAllComponentParams(dslJSON []byte) ([]ComponentParamsSchema, error)
extractorCount := 0
for cpnID, comp := range dsl.Components {
- if isExtractorComponent(cpnID, comp.Obj.ComponentName) {
+ if cpnID != common.ComponentNameFile && !common.IsComponentKey(cpnID) {
+ return nil, fmt.Errorf("ExtractAllComponentParams: component ID %q must be <name>:<instance>; re-save the pipeline with canonical component IDs", cpnID)
+ }
+ if common.IsExtractorComponent(cpnID, comp.Obj.ComponentName) {
extractorCount++
}
}
@@ -62,7 +66,7 @@ func ExtractAllComponentParams(dslJSON []byte) ([]ComponentParamsSchema, error)
// Canvases saved by the Python-era frontend nest the Parser's
// per-family setups under a "setups" key; lift them so the extracted
// defaults (and the parser_config built from them) stay flat.
- if isParserComponent(cpnID, comp.Obj.ComponentName) {
+ if common.IsParserComponent(cpnID, comp.Obj.ComponentName) {
params = schema.FlattenLegacyParserSetups(params)
}
out = append(out, ComponentParamsSchema{
diff --git a/internal/ingestion/pipeline/component_params_test.go b/internal/ingestion/pipeline/component_params_test.go
index 58c754ebdb..103239958e 100644
--- a/internal/ingestion/pipeline/component_params_test.go
+++ b/internal/ingestion/pipeline/component_params_test.go
@@ -196,6 +196,20 @@ func TestExtractAllComponentParams_MissingComponents(t *testing.T) {
}
}
+func TestExtractAllComponentParams_RejectsUnscopedComponentIDs(t *testing.T) {
+ for _, id := range []string{"Extractor", "extractor_0", "parser_0"} {
+ dslJSON := []byte(`{"components":{"` + id + `":{"obj":{"component_name":"Extractor","params":{}}}}}`)
+ if _, err := ExtractAllComponentParams(dslJSON); err == nil {
+ t.Errorf("ExtractAllComponentParams(%q) expected error for unscoped component ID", id)
+ }
+ }
+ // File is the one bare component ID used by built-in ingestion templates.
+ dslJSON := []byte(`{"components":{"File":{"obj":{"component_name":"File","params":{}}}}}`)
+ if _, err := ExtractAllComponentParams(dslJSON); err != nil {
+ t.Fatalf("ExtractAllComponentParams(File): %v", err)
+ }
+}
+
func componentIDs(schemas []ComponentParamsSchema) []string {
ids := make([]string, len(schemas))
for i, s := range schemas {
diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.go
index 12af493f2e..9ba5d63f08 100644
--- a/internal/ingestion/pipeline/pipeline.go
+++ b/internal/ingestion/pipeline/pipeline.go
@@ -149,7 +149,7 @@ func ValidatePipeline(cnv *canvas.Canvas) error {
}
extractorCount := 0
for id, comp := range cnv.Components {
- if isExtractorComponent(id, comp.Obj.ComponentName) {
+ if common.IsExtractorComponent(id, comp.Obj.ComponentName) {
extractorCount++
}
}
diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.go
index 000d44016e..9570e8fffa 100644
--- a/internal/ingestion/pipeline/pipeline_params.go
+++ b/internal/ingestion/pipeline/pipeline_params.go
@@ -83,14 +83,14 @@ var titleFamilyParamKeys = unionParamKeys(
)
var componentParamSchemaKeys = map[string]map[string]struct{}{
- "titlechunker": titleFamilyParamKeys,
+ strings.ToLower(schema.ComponentNameTitleChunker): titleFamilyParamKeys,
// ManualChunker reuses TitleChunkerParam. It pins method=group and is exempt
// from the token cap, so method, chunk_token_cap and include_heading_content
// (hierarchy-only) are accepted here but ignored by the component — the
// operator form omits them for the same reason.
- "manualchunker": titleFamilyParamKeys,
- "tokenchunker": extractJSONTags(schema.TokenChunkerParam{}),
- "tokenizer": extractJSONTags(schema.TokenizerParam{}),
+ strings.ToLower(schema.ComponentNameManualChunker): titleFamilyParamKeys,
+ strings.ToLower(schema.ComponentNameTokenChunker): extractJSONTags(schema.TokenChunkerParam{}),
+ strings.ToLower(schema.ComponentNameTokenizer): extractJSONTags(schema.TokenizerParam{}),
}
// extractJSONTags returns all top-level json tag names from a struct.
@@ -119,38 +119,6 @@ func extractJSONTags(v any) map[string]struct{} {
return tags
}
-// isExtractorComponent returns true if the component name or component ID
-// indicates an Extractor component.
-func isExtractorComponent(cpnID, componentName string) bool {
- lowerName := strings.ToLower(componentName)
- lowerID := strings.ToLower(cpnID)
- return lowerName == strings.ToLower(schema.ComponentNameExtractor) ||
- strings.HasPrefix(lowerID, "extractor:") ||
- strings.HasPrefix(lowerID, "extractor_")
-}
-
-// isCompilerComponent returns true if the component ID indicates a Compiler
-// component, whose LLM-backed runtime parameters follow the same surface as
-// the Extractor's.
-func isCompilerComponent(cpnID string) bool {
- lowerID := strings.ToLower(cpnID)
- return strings.HasPrefix(lowerID, "compiler:") ||
- strings.HasPrefix(lowerID, "compiler_")
-}
-
-// IsParserComponentID reports whether a component ID names a Parser component.
-// A parser_config map is keyed by component ID, where the component name is not
-// available, so this is the config-side half of isParserComponent.
-func IsParserComponentID(cpnID string) bool {
- return strings.HasPrefix(strings.ToLower(cpnID), "parser:")
-}
-
-// isParserComponent returns true if the component name or component ID
-// indicates a Parser component.
-func isParserComponent(cpnID, componentName string) bool {
- return strings.EqualFold(componentName, schema.ComponentNameParser) || IsParserComponentID(cpnID)
-}
-
// NormalizeParserConfigSetups lifts legacy nested Parser "setups" entries in
// a parser_config-shaped map ({cpnID: params}) into the flat per-family
// shape. The runtime merges such a map over the DSL-baked component params
@@ -163,7 +131,7 @@ func NormalizeParserConfigSetups(parserConfig map[string]any) map[string]any {
return nil
}
for cpnID, raw := range parserConfig {
- if !isParserComponent(cpnID, "") {
+ if !common.IsParserComponentID(cpnID) {
continue
}
params, ok := raw.(map[string]any)
@@ -177,22 +145,19 @@ func NormalizeParserConfigSetups(parserConfig map[string]any) map[string]any {
// getComponentParamWhitelist returns the dynamic parameter whitelist for a component type.
func getComponentParamWhitelist(cpnID string) (map[string]struct{}, bool) {
- if isExtractorComponent(cpnID, "") {
+ if common.IsExtractorComponentID(cpnID) {
return extractorValidParamKeys, true
}
- if isCompilerComponent(cpnID) {
+ if common.IsCompilerComponentID(cpnID) {
return llmRuntimeParamKeys, true
}
return nil, false
}
-// cleanComponentParams filters rawConfig against the extracted DSL schemas.
-// Keys containing ':' are treated as component IDs; they are kept only when both
-// the cpnID AND the param name exist in one of the accepted-key sources: the DSL
-// schema (the params the template bakes), the schema-derived per-component table
-// (componentParamSchemaKeys), or the component's dynamic parameter schema (e.g.
-// Extractor modular features). Keys without ':' (legacy flat fields) are dropped
-// with a warning.
+// cleanComponentParams keeps only component-scoped keys recognized by
+// common.IsComponentKey, and only params accepted by the DSL schema, the
+// per-component table, or the component's dynamic schema. Flat fields are
+// dropped with a warning.
func cleanComponentParams(schemas []ComponentParamsSchema, rawConfig map[string]interface{}) map[string]interface{} {
validCPNs := make(map[string]map[string]struct{}, len(schemas))
componentNames := make(map[string]string, len(schemas))
@@ -201,10 +166,10 @@ func cleanComponentParams(schemas []ComponentParamsSchema, rawConfig map[string]
for k := range s.ParamsDefaults {
keys[k] = struct{}{}
}
- if s.ComponentName == "GeneralChunker" {
+ if s.ComponentName == schema.ComponentNameGeneralChunker {
keys["delimiters"] = struct{}{}
}
- if isParserComponent(s.CpnID, s.ComponentName) {
+ if common.IsParserComponent(s.CpnID, s.ComponentName) {
keys["enable_vision_enhancement"] = struct{}{}
keys["vlm"] = struct{}{}
}
@@ -225,7 +190,7 @@ func cleanComponentParams(schemas []ComponentParamsSchema, rawConfig map[string]
result := make(map[string]interface{}, len(rawConfig))
for key, val := range rawConfig {
- if !strings.Contains(key, ":") {
+ if !common.IsComponentKey(key) {
common.Warn("cleanComponentParams: dropping legacy flat field",
zap.String("key", key))
continue
@@ -240,18 +205,18 @@ func cleanComponentParams(schemas []ComponentParamsSchema, rawConfig map[string]
if !ok {
continue
}
- if componentNames[key] == "GeneralChunker" {
+ if componentNames[key] == schema.ComponentNameGeneralChunker {
params = normalizeGeneralComponentParams(params)
}
// Legacy parser_config rows (or canvases) saved by the Python-era
// frontend nest the Parser's setups under a "setups" key; lift them
// before filtering, otherwise the families are dropped as unknown
// params against the flat DSL schema keys.
- if isParserComponent(key, componentNames[key]) {
+ if common.IsParserComponent(key, componentNames[key]) {
params = schema.FlattenLegacyParserSetups(params)
}
dynamicWhitelist, hasDynamic := getComponentParamWhitelist(key)
- if isExtractorComponent(key, "") {
+ if common.IsExtractorComponentID(key) {
params = NormalizeExtractorParams(params)
}
cleaned := make(map[string]any, len(params))
@@ -430,5 +395,6 @@ func ApplyParentChildChunkerConfig(componentConfig entity.JSONMap, rawConfig map
// chunks into children.
func IsChunkerComponent(componentID string) bool {
lowerID := strings.ToLower(componentID)
- return strings.HasPrefix(lowerID, "generalchunker:") || strings.HasPrefix(lowerID, "tokenchunker:")
+ return strings.HasPrefix(lowerID, strings.ToLower(schema.ComponentNameGeneralChunker)+":") ||
+ strings.HasPrefix(lowerID, strings.ToLower(schema.ComponentNameTokenChunker)+":")
}
diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.go
index 533bc7468c..376595d99c 100644
--- a/internal/ingestion/pipeline/pipeline_params_test.go
+++ b/internal/ingestion/pipeline/pipeline_params_test.go
@@ -34,9 +34,9 @@ func generalDSL(t *testing.T) []byte {
},
},
},
- "Chunker:LegalReadersDecide": map[string]any{
+ "GeneralChunker:LegalReadersDecide": map[string]any{
"obj": map[string]any{
- "component_name": "Chunker",
+ "component_name": "GeneralChunker",
"params": map[string]any{
"outputs": map[string]any{},
"chunk_size": float64(512),
@@ -165,9 +165,9 @@ func TestBuildParserConfig_ShallowMerge_NestedParam(t *testing.T) {
// A component with a nested-map param "chunk" that has sub-keys.
dsl := map[string]any{
"components": map[string]any{
- "Chunker:xyz": map[string]any{
+ "GeneralChunker:xyz": map[string]any{
"obj": map[string]any{
- "component_name": "Chunker",
+ "component_name": "GeneralChunker",
"params": map[string]any{
"outputs": map[string]any{},
"chunk": map[string]any{
@@ -186,15 +186,15 @@ func TestBuildParserConfig_ShallowMerge_NestedParam(t *testing.T) {
// User only overrides one sub-key of "chunk".
overrides := map[string]any{
- "Chunker:xyz": map[string]any{
+ "GeneralChunker:xyz": map[string]any{
"chunk": map[string]any{"size": float64(1024)},
},
}
result := buildTestParserConfig(t, dslJSON, overrides)
- chunker, ok := result["Chunker:xyz"].(map[string]any)
+ chunker, ok := result["GeneralChunker:xyz"].(map[string]any)
if !ok {
- t.Fatal("expected Chunker:xyz in result")
+ t.Fatal("expected GeneralChunker:xyz in result")
}
chunk, ok := chunker["chunk"].(map[string]any)
if !ok {
@@ -348,14 +348,14 @@ func TestNormalizeParserConfigSetups_EmptyNestedMap(t *testing.T) {
func TestBuildParserConfig_ScalarOverridePreservesOtherDefaults(t *testing.T) {
dslJSON := generalDSL(t)
overrides := map[string]any{
- "Chunker:LegalReadersDecide": map[string]any{
+ "GeneralChunker:LegalReadersDecide": map[string]any{
"chunk_size": float64(1024),
},
}
result := buildTestParserConfig(t, dslJSON, overrides)
- chunker, ok := result["Chunker:LegalReadersDecide"].(map[string]any)
+ chunker, ok := result["GeneralChunker:LegalReadersDecide"].(map[string]any)
if !ok {
- t.Fatal("expected Chunker:LegalReadersDecide in result")
+ t.Fatal("expected GeneralChunker:LegalReadersDecide in result")
}
if chunker["chunk_size"] != float64(1024) {
t.Errorf("expected chunk_size=1024, got %v", chunker["chunk_size"])
@@ -398,8 +398,8 @@ func TestBuildParserConfig_AllComponentsPresent(t *testing.T) {
if _, ok := result["Parser:HipSignsRhyme"]; !ok {
t.Error("expected Parser:HipSignsRhyme")
}
- if _, ok := result["Chunker:LegalReadersDecide"]; !ok {
- t.Error("expected Chunker:LegalReadersDecide")
+ if _, ok := result["GeneralChunker:LegalReadersDecide"]; !ok {
+ t.Error("expected GeneralChunker:LegalReadersDecide")
}
}
@@ -470,7 +470,7 @@ func TestComponentParamsDefaults_Basic(t *testing.T) {
if _, ok := parser["pdf"]; !ok {
t.Error("expected pdf to be present")
}
- chunker := result["Chunker:LegalReadersDecide"]
+ chunker := result["GeneralChunker:LegalReadersDecide"]
if chunker["chunk_size"] != float64(512) {
t.Errorf("expected chunk_size=512, got %v", chunker["chunk_size"])
}
diff --git a/internal/ingestion/pipeline/template_output_format_test.go b/internal/ingestion/pipeline/template_output_format_test.go
index fe14309cfb..68617b00d9 100644
--- a/internal/ingestion/pipeline/template_output_format_test.go
+++ b/internal/ingestion/pipeline/template_output_format_test.go
@@ -21,6 +21,8 @@ import (
"path/filepath"
"strings"
"testing"
+
+ "ragflow/internal/common"
)
// parserSetupParamsKeys are the Parser params keys that are NOT per-family
@@ -108,7 +110,7 @@ func assertNoParserOutputFormatInGraph(t *testing.T, ref string, root map[string
for _, rawNode := range nodes {
node, _ := rawNode.(map[string]any)
id, _ := node["id"].(string)
- if !IsParserComponentID(id) {
+ if !common.IsParserComponentID(id) {
continue
}
data, _ := node["data"].(map[string]any)
diff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.go
index 2728bee4ed..636f3b671b 100644
--- a/internal/ingestion/task/indexdoc/process.go
+++ b/internal/ingestion/task/indexdoc/process.go
@@ -18,7 +18,6 @@ package indexdoc
import (
"fmt"
- "strings"
"time"
"ragflow/internal/common"
@@ -376,7 +375,7 @@ func resolveTableColumnConfig(parserConfig map[string]interface{}) (mode string,
return mode, roles, rawNames
}
for cid, raw := range parserConfig {
- if !strings.HasPrefix(cid, "Parser:") {
+ if !common.IsParserComponentID(cid) {
continue
}
comp, _ := raw.(map[string]interface{})
diff --git a/internal/service/component_scoped_parser_config.go b/internal/service/component_scoped_parser_config.go
index c6d876ab01..004f78b1ce 100644
--- a/internal/service/component_scoped_parser_config.go
+++ b/internal/service/component_scoped_parser_config.go
@@ -3,6 +3,7 @@ package service
import (
"strings"
+ "ragflow/internal/common"
"ragflow/internal/entity"
)
@@ -33,9 +34,8 @@ func ApplyComponentScopedParserConfig(
if !ok {
continue
}
- cpnLower := strings.ToLower(cpnID)
switch {
- case strings.HasPrefix(cpnLower, "extractor:") || strings.HasPrefix(cpnLower, "extractor_"):
+ case common.IsExtractorComponentID(cpnID):
if value, _ := params["llm_id"].(string); strings.TrimSpace(value) == "" && strings.TrimSpace(llmID) != "" {
params["llm_id"] = llmID
}
@@ -54,7 +54,7 @@ func ApplyComponentScopedParserConfig(
"built_in_metadata": []any{},
}
}
- case strings.HasPrefix(cpnLower, "compiler:") || strings.HasPrefix(cpnLower, "compiler_"):
+ case common.IsCompilerComponentID(cpnID):
if value, _ := params["llm_id"].(string); strings.TrimSpace(value) == "" && strings.TrimSpace(llmID) != "" {
params["llm_id"] = llmID
}
diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.go
index 58db80de31..7e739f71aa 100644
--- a/internal/service/dataset/helpers.go
+++ b/internal/service/dataset/helpers.go
@@ -8,6 +8,7 @@ import (
"strings"
"unicode/utf8"
+ "ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
pipelinepkg "ragflow/internal/ingestion/pipeline"
@@ -202,23 +203,19 @@ func validateDatasetParserConfigSize(parserConfig map[string]interface{}) error
return nil
}
-// DropUnscopedParserConfigKeys removes top-level parser_config keys that are
-// not component-scoped (i.e. do not contain ":"), since the Go backend only
-// consumes keys keyed by a component id (e.g. "Parser:abc123" or
-// "Extractor:AutoExtractDefault"). Flat keys are legacy transport artifacts
-// that downstream consumers (BuildParserConfig) never read, so they are
-// dropped rather than rejected. Only top-level keys are affected; a key nested
-// inside a component node (e.g. "Extractor:AutoExtractDefault.metadata") keeps
-// its existing name. It mutates the provided map in place and returns the names
-// of the dropped keys so callers can log the silent drop. A nil or empty map is
-// returned untouched with a nil result.
+// DropUnscopedParserConfigKeys removes top-level parser_config keys that
+// common.IsComponentKey does not recognize as component-scoped. Flat keys are
+// transport artifacts that BuildParserConfig never reads, so they are dropped
+// rather than rejected. Only top-level keys are affected; nested keys keep
+// their names. It mutates parserConfig and returns the dropped names so callers
+// can log them. A nil or empty map is returned untouched.
func DropUnscopedParserConfigKeys(parserConfig map[string]any) []string {
if len(parserConfig) == 0 {
return nil
}
var dropped []string
for key := range parserConfig {
- if !strings.Contains(key, ":") {
+ if !common.IsComponentKey(key) {
dropped = append(dropped, key)
}
}
@@ -369,10 +366,10 @@ func preserveDatasetParserConfigState(next, existing entity.JSONMap, incoming ma
}
var mm map[string]any
if incoming != nil {
- mm = extractorNodeMetadata(incoming)
+ mm = extractorNodeMetadata(incoming, next)
}
if mm == nil && existing != nil {
- mm = extractorNodeMetadata(existing)
+ mm = extractorNodeMetadata(existing, next)
}
if mm != nil {
next["metadata"] = mm
@@ -447,18 +444,19 @@ func preserveDatasetParserConfigState(next, existing entity.JSONMap, incoming ma
}
// extractorNodeMetadata returns the modular metadata object ({enabled, metadata,
-// built_in_metadata}) from the first Extractor node in a parser_config, or nil
-// if none is present. Metadata is component-scoped under Extractor nodes; this
-// helper reads it back for preservation across partial updates.
-func extractorNodeMetadata(parserConfig map[string]interface{}) map[string]any {
+// built_in_metadata}) from the first Extractor node present in validConfig, or
+// nil if none is present. validConfig is the rebuilt parser_config for the
+// current DSL; keys outside it are ignored so stale nodes cannot override the
+// active configuration.
+func extractorNodeMetadata(parserConfig, validConfig map[string]interface{}) map[string]any {
if parserConfig == nil {
return nil
}
- for cpnID, raw := range parserConfig {
- lower := strings.ToLower(cpnID)
- if !strings.HasPrefix(lower, "extractor:") && !strings.HasPrefix(lower, "extractor_") {
+ for _, cpnID := range common.SortedExtractorKeys(parserConfig) {
+ if _, ok := validConfig[cpnID]; !ok {
continue
}
+ raw := parserConfig[cpnID]
params, ok := raw.(map[string]any)
if !ok {
continue
diff --git a/internal/service/dataset/helpers_test.go b/internal/service/dataset/helpers_test.go
index a52d810ac2..8ec5f364d1 100644
--- a/internal/service/dataset/helpers_test.go
+++ b/internal/service/dataset/helpers_test.go
@@ -227,6 +227,9 @@ func TestDropUnscopedParserConfigKeys_NilAndEmpty(t *testing.T) {
func TestDropUnscopedParserConfigKeys_DropsFlatKeys(t *testing.T) {
cfg := map[string]any{
"File": map[string]any{},
+ "Extractor": map[string]any{},
+ "extractor_0": map[string]any{},
+ "compiler_0": map[string]any{},
"chunk_token_num": float64(128),
"delimiter": "\n",
"metadata": map[string]any{},
@@ -236,7 +239,7 @@ func TestDropUnscopedParserConfigKeys_DropsFlatKeys(t *testing.T) {
}
dropped := DropUnscopedParserConfigKeys(cfg)
sort.Strings(dropped)
- want := []string{"File", "chunk_token_num", "delimiter", "metadata", "parent_child"}
+ want := []string{"Extractor", "File", "chunk_token_num", "compiler_0", "delimiter", "extractor_0", "metadata", "parent_child"}
if !reflect.DeepEqual(dropped, want) {
t.Fatalf("dropped = %#v, want %#v", dropped, want)
}
@@ -510,7 +513,9 @@ func TestPreserveDatasetParserConfigState_FallsBackWhenIncomingNotMap(t *testing
"metadata": incomingMetadata,
},
}
- got := preserveDatasetParserConfigState(entity.JSONMap{}, existing, incoming)
+ got := preserveDatasetParserConfigState(entity.JSONMap{
+ "Extractor:AutoExtractDefault": map[string]any{},
+ }, existing, incoming)
meta, ok := got["metadata"].(map[string]any)
if !ok {
t.Fatalf("expected existing modular metadata preserved, got %#v", got["metadata"])
@@ -523,6 +528,38 @@ func TestPreserveDatasetParserConfigState_FallsBackWhenIncomingNotMap(t *testing
}
}
+func TestPreserveDatasetParserConfigState_IgnoresStaleExtractor(t *testing.T) {
+ existing := entity.JSONMap{
+ "Extractor:AutoExtractDefault": map[string]any{
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{map[string]any{"key": "current_field", "type": "string"}},
+ "built_in_metadata": []any{},
+ },
+ },
+ }
+ incoming := map[string]interface{}{
+ "Extractor:Old": map[string]any{
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{map[string]any{"key": "stale_field", "type": "string"}},
+ "built_in_metadata": []any{},
+ },
+ },
+ }
+ got := preserveDatasetParserConfigState(entity.JSONMap{
+ "Extractor:AutoExtractDefault": map[string]any{},
+ }, existing, incoming)
+ meta, ok := got["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("expected current modular metadata, got %#v", got["metadata"])
+ }
+ fields, ok := meta["metadata"].([]any)
+ if !ok || len(fields) != 1 || fields[0].(map[string]any)["key"] != "current_field" {
+ t.Fatalf("expected current_field to win over stale Extractor, got %#v", meta["metadata"])
+ }
+}
+
func TestPreserveDatasetParserConfigState_UsesValidIncomingMap(t *testing.T) {
existing := entity.JSONMap{
"Extractor:AutoExtractDefault": map[string]any{
@@ -542,7 +579,9 @@ func TestPreserveDatasetParserConfigState_UsesValidIncomingMap(t *testing.T) {
},
},
}
- got := preserveDatasetParserConfigState(entity.JSONMap{}, existing, incoming)
+ got := preserveDatasetParserConfigState(entity.JSONMap{
+ "Extractor:AutoExtractDefault": map[string]any{},
+ }, existing, incoming)
meta, ok := got["metadata"].(map[string]any)
if !ok {
t.Fatalf("expected modular metadata map, got %#v", got["metadata"])
diff --git a/internal/service/dataset/metadata.go b/internal/service/dataset/metadata.go
index 33e5e98ace..6d90140cb8 100644
--- a/internal/service/dataset/metadata.go
+++ b/internal/service/dataset/metadata.go
@@ -192,11 +192,8 @@ func modularMetadataConfig(parserConfig map[string]any) (bool, bool, []any, []an
builtIn := anyOrEmptyList(metaObj["built_in_metadata"])
return true, enabled, metadata, builtIn
}
- for cpnID, raw := range parserConfig {
- lower := strings.ToLower(cpnID)
- if !strings.HasPrefix(lower, "extractor:") && !strings.HasPrefix(lower, "extractor_") {
- continue
- }
+ for _, cpnID := range common.SortedExtractorKeys(parserConfig) {
+ raw := parserConfig[cpnID]
params, ok := raw.(map[string]any)
if !ok {
continue
diff --git a/internal/service/dataset/parser_config_roundtrip_test.go b/internal/service/dataset/parser_config_roundtrip_test.go
new file mode 100644
index 0000000000..48e36eb82d
--- /dev/null
+++ b/internal/service/dataset/parser_config_roundtrip_test.go
@@ -0,0 +1,53 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dataset
+
+import (
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/ingestion/component"
+ pipelinepkg "ragflow/internal/ingestion/pipeline"
+)
+
+func TestParserConfigCanonicalExtractorRoundTrip(t *testing.T) {
+ dsl := []byte(`{"components":{"Extractor:AutoExtractDefault":{"obj":{"component_name":"Extractor","params":{"metadata":{"enabled":true,"metadata":[],"built_in_metadata":[]},"tags":{"top_n":5,"tag_file_id":""}}}}}}`)
+ raw := map[string]any{
+ "chunk_token_num": float64(128),
+ "Extractor:AutoExtractDefault": map[string]any{
+ "tags": map[string]any{"top_n": 5, "tag_file_id": "file-round-trip"},
+ },
+ }
+ if dropped, err := ValidateParserConfig(raw); err != nil {
+ t.Fatalf("ValidateParserConfig: %v", err)
+ } else if len(dropped) != 1 || dropped[0] != "chunk_token_num" {
+ t.Fatalf("dropped = %#v, want [chunk_token_num]", dropped)
+ }
+
+ config, err := pipelinepkg.BuildParserConfig(dsl, raw)
+ if err != nil {
+ t.Fatalf("BuildParserConfig: %v", err)
+ }
+ if got := component.TagFileIDFromParserConfig(map[string]any(config)); got != "file-round-trip" {
+ t.Fatalf("TagFileIDFromParserConfig = %q, want file-round-trip", got)
+ }
+ if meta, ok := common.ExtractorMetadataConfig(map[string]any(config)); !ok {
+ t.Fatal("ExtractorMetadataConfig did not find the canonical Extractor node")
+ } else if enabled, _ := meta["enabled"].(bool); !enabled {
+ t.Fatalf("metadata enabled = %v, want true", meta["enabled"])
+ }
+}
diff --git a/internal/service/dataset/update_test.go b/internal/service/dataset/update_test.go
index 064d289b44..7bf5a11718 100644
--- a/internal/service/dataset/update_test.go
+++ b/internal/service/dataset/update_test.go
@@ -1356,8 +1356,7 @@ func TestUpdateDatasetUsesTenantModelIDForExtractorDefault(t *testing.T) {
func extractorNodeMetadataInTest(t *testing.T, cfg map[string]interface{}) map[string]interface{} {
t.Helper()
for cpnID, raw := range cfg {
- lower := strings.ToLower(cpnID)
- if !strings.HasPrefix(lower, "extractor:") && !strings.HasPrefix(lower, "extractor_") {
+ if !common.IsExtractorComponentID(cpnID) {
continue
}
params, ok := raw.(map[string]interface{})
diff --git a/internal/service/document/document_applykb_chain_test.go b/internal/service/document/document_applykb_chain_test.go
index 3dc6027ac0..88a5579804 100644
--- a/internal/service/document/document_applykb_chain_test.go
+++ b/internal/service/document/document_applykb_chain_test.go
@@ -15,7 +15,6 @@ package document
import (
"encoding/json"
"sort"
- "strings"
"testing"
"gorm.io/gorm"
@@ -42,8 +41,7 @@ func firstExtractorMetadata(t *testing.T, parserConfig entity.JSONMap) map[strin
t.Helper()
var keys []string
for k := range parserConfig {
- lower := strings.ToLower(k)
- if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") {
+ if common.IsExtractorComponentID(k) {
keys = append(keys, k)
}
}
diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.go
index 2e120ef3c6..8421e5aad3 100644
--- a/internal/service/document/document_parse.go
+++ b/internal/service/document/document_parse.go
@@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"ragflow/internal/service"
- "sort"
"strconv"
"strings"
"sync"
@@ -198,7 +197,7 @@ func (s *DocumentService) applyKnowledgebaseParserConfig(ctx context.Context, do
if metaOK {
merged["metadata"] = deepCopyValue(meta)
} else {
- for _, cpnID := range sortedExtractorKeys(merged) {
+ for _, cpnID := range common.SortedExtractorKeys(map[string]any(merged)) {
params, ok := nodeParams(merged[cpnID])
if !ok {
continue
@@ -227,21 +226,6 @@ func (s *DocumentService) applyKnowledgebaseParserConfig(ctx context.Context, do
return nil
}
-// sortedExtractorKeys returns the component IDs in parserConfig that name
-// Extractor nodes, in sorted order so multi-Extractor configs resolve
-// deterministically instead of depending on Go map iteration order.
-func sortedExtractorKeys(parserConfig entity.JSONMap) []string {
- var keys []string
- for k := range parserConfig {
- lower := strings.ToLower(k)
- if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") {
- keys = append(keys, k)
- }
- }
- sort.Strings(keys)
- return keys
-}
-
// nodeParams returns a writable view of a component node's params, accepting
// both the decoded (map[string]any) and entity (entity.JSONMap) shapes.
// The returned map shares storage with the input, so writes are visible.
@@ -281,7 +265,7 @@ func resolveDatasetModularMetadata(parserConfig entity.JSONMap) (map[string]any,
if mm, ok := parserConfig["metadata"].(entity.JSONMap); ok {
return map[string]any(mm), true, true
}
- for _, cpnID := range sortedExtractorKeys(parserConfig) {
+ for _, cpnID := range common.SortedExtractorKeys(map[string]any(parserConfig)) {
params, ok := nodeParams(parserConfig[cpnID])
if !ok {
continue
@@ -301,7 +285,7 @@ func resolveDatasetLLMID(parserConfig entity.JSONMap) string {
if s, ok := parserConfig["llm_id"].(string); ok && strings.TrimSpace(s) != "" {
return s
}
- for _, cpnID := range sortedExtractorKeys(parserConfig) {
+ for _, cpnID := range common.SortedExtractorKeys(map[string]any(parserConfig)) {
params, ok := nodeParams(parserConfig[cpnID])
if !ok {
continue
diff --git a/internal/service/document/document_parser_selection.go b/internal/service/document/document_parser_selection.go
index 8b8176dc58..18e69b6600 100644
--- a/internal/service/document/document_parser_selection.go
+++ b/internal/service/document/document_parser_selection.go
@@ -66,7 +66,7 @@ func parserParamsDeclaresFamily(params map[string]any, family string) bool {
// config, or a malformed one): the config cannot answer then.
func configDeclaredFamily(config entity.JSONMap, family string) (declared, found bool) {
for cpnID, raw := range config {
- if !pipelinepkg.IsParserComponentID(cpnID) {
+ if !common.IsParserComponentID(cpnID) {
continue
}
params, ok := raw.(map[string]any)
@@ -94,7 +94,7 @@ func templateDeclaredFamily(ctx context.Context, parserID, family string) (decla
return false, false
}
for cpnID, params := range defaults {
- if !pipelinepkg.IsParserComponentID(cpnID) {
+ if !common.IsParserComponentID(cpnID) {
continue
}
return parserParamsDeclaresFamily(params, family), true
diff --git a/internal/service/document/document_parser_selection_test.go b/internal/service/document/document_parser_selection_test.go
index e89a0c6a06..e495a47a40 100644
--- a/internal/service/document/document_parser_selection_test.go
+++ b/internal/service/document/document_parser_selection_test.go
@@ -3,9 +3,9 @@ package document
import (
"mime/multipart"
"reflect"
- "strings"
"testing"
+ "ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/permission"
@@ -763,7 +763,7 @@ func parserFamilySetup(t *testing.T, doc map[string]interface{}, family string)
t.Fatalf("%v parser_config = %T, want map", doc["name"], doc["parser_config"])
}
for cpnID, raw := range cfg {
- if !strings.HasPrefix(cpnID, "Parser:") {
+ if !common.IsParserComponentID(cpnID) {
continue
}
params, _ := raw.(map[string]interface{})