Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(parser_config): make all settings component-scoped (zero flat-key compat) (#20315)

xugangqiang67d68725fae0
diff --git a/internal/handler/dataset.go b/internal/handler/dataset.go
index 309a81cb18..8668c66416 100644
--- a/internal/handler/dataset.go
+++ b/internal/handler/dataset.go
@@ -445,12 +445,6 @@ func (h *DatasetsHandler) UpdateDataset(c *gin.Context) {
common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "parser_config.ext is not supported; send parser configuration fields directly")
return
}
- if raptor, ok := config["raptor"].(map[string]interface{}); ok {
- if _, ok := raptor["ext"]; ok {
- common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "parser_config.raptor.ext is not supported; send RAPTOR configuration fields directly")
- return
- }
- }
}
}
}
diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.go
index 65ff4cf5fe..e32163986a 100644
--- a/internal/ingestion/pipeline/pipeline_params.go
+++ b/internal/ingestion/pipeline/pipeline_params.go
@@ -175,6 +175,11 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[
}
if IsChunkerComponent(s.CpnID) {
keys["enable_children"] = struct{}{}
+ // parent_child is the component-scoped home of the parent/child
+ // split setting; it must survive cleaning so the chunker node keeps
+ // it (ApplyParentChildChunkerConfig derives children_delimiters
+ // from it). No flat top-level parent_child is accepted.
+ keys["parent_child"] = struct{}{}
}
for k := range componentParamSchemaKeys[strings.ToLower(s.ComponentName)] {
keys[k] = struct{}{}
@@ -355,35 +360,34 @@ func BuildParserConfig(dslJSON []byte, rawConfig map[string]interface{}) entity.
}
// ApplyParentChildChunkerConfig derives runtime children_delimiters from the
-// top-level parent_child setting unless the request explicitly configures the
-// chunker itself.
+// parent_child setting scoped onto each chunker component. The setting must live
+// as a "parent_child" sub-object on the chunker node itself (component-scoped);
+// flat top-level parent_child keys are not accepted.
func ApplyParentChildChunkerConfig(componentConfig entity.JSONMap, rawConfig map[string]interface{}) {
- parentChild, ok := rawConfig["parent_child"].(map[string]interface{})
- if !ok {
- return
- }
- componentConfig["parent_child"] = parentChild
- useParentChild, _ := parentChild["use_parent_child"].(bool)
- childrenDelimiters := []string{}
- if useParentChild {
- if delimiter, ok := parentChild["children_delimiter"].(string); ok {
- childrenDelimiters = parserchunk.ParseDelimiterField(delimiter)
- }
- }
-
for componentID, value := range componentConfig {
if !IsChunkerComponent(componentID) {
continue
}
+ params, ok := value.(map[string]interface{})
+ if !ok {
+ continue
+ }
+ parentChild, ok := params["parent_child"].(map[string]interface{})
+ if !ok {
+ continue
+ }
+ useParentChild, _ := parentChild["use_parent_child"].(bool)
+ childrenDelimiters := []string{}
+ if useParentChild {
+ if delimiter, ok := parentChild["children_delimiter"].(string); ok {
+ childrenDelimiters = parserchunk.ParseDelimiterField(delimiter)
+ }
+ }
if requested, ok := rawConfig[componentID].(map[string]interface{}); ok {
if _, provided := requested["children_delimiters"]; provided {
continue
}
}
- params, ok := value.(map[string]interface{})
- if !ok {
- continue
- }
params["children_delimiters"] = childrenDelimiters
}
}
diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.go
index 59b4f3a748..daa5706c82 100644
--- a/internal/ingestion/pipeline/pipeline_params_test.go
+++ b/internal/ingestion/pipeline/pipeline_params_test.go
@@ -623,17 +623,12 @@ func TestBuildParserConfig_BuiltinExtractorKeepsBuiltInMetadata(t *testing.T) {
func TestApplyParentChildChunkerConfig(t *testing.T) {
config := entity.JSONMap{
- "GeneralChunker:one": map[string]interface{}{"chunk_token_num": 256, "children_delimiters": []string{"wrong"}},
- "TokenChunker:two": map[string]interface{}{"chunk_token_num": 128, "children_delimiters": []string{"wrong"}},
+ "GeneralChunker:one": map[string]interface{}{"chunk_token_num": 256, "children_delimiters": []string{"wrong"}, "parent_child": map[string]interface{}{"use_parent_child": true, "children_delimiter": "|"}},
+ "TokenChunker:two": map[string]interface{}{"chunk_token_num": 128, "children_delimiters": []string{"wrong"}, "parent_child": map[string]interface{}{"use_parent_child": true, "children_delimiter": "|"}},
"Extractor:three": map[string]interface{}{"llm_id": "llm-1"},
}
- ApplyParentChildChunkerConfig(config, map[string]interface{}{
- "parent_child": map[string]interface{}{
- "use_parent_child": true,
- "children_delimiter": "|",
- },
- })
+ ApplyParentChildChunkerConfig(config, map[string]interface{}{})
for _, componentID := range []string{"GeneralChunker:one", "TokenChunker:two"} {
params := config[componentID].(map[string]interface{})
diff --git a/internal/service/component_scoped_parser_config.go b/internal/service/component_scoped_parser_config.go
index 4929ecc1cd..7acc009553 100644
--- a/internal/service/component_scoped_parser_config.go
+++ b/internal/service/component_scoped_parser_config.go
@@ -61,15 +61,17 @@ func ApplyComponentScopedParserConfig(
}
}
- // Strip legacy top-level flat metadata fields; keep the modular object
- // under "metadata" when present, otherwise drop the key entirely.
+ // Documents and datasets are uniformly component-scoped (every dataset and
+ // document DSL defines an Extractor node), so the flat "metadata" transport
+ // key is never a first-class config: it is scoped onto an Extractor node
+ // above and then dropped here. A flat metadata object with no Extractor node
+ // to receive it is invalid input and is silently dropped (no node is
+ // auto-created), keeping the function free of compatibility shims.
delete(parserConfig, "enable_metadata")
delete(parserConfig, "metadata_config")
delete(parserConfig, "built_in_metadata")
delete(parserConfig, "fields")
- if datasetMeta == nil {
- delete(parserConfig, "metadata")
- }
+ delete(parserConfig, "metadata")
return parserConfig
}
diff --git a/internal/service/component_scoped_parser_config_test.go b/internal/service/component_scoped_parser_config_test.go
index e2102e1165..8691ac8405 100644
--- a/internal/service/component_scoped_parser_config_test.go
+++ b/internal/service/component_scoped_parser_config_test.go
@@ -152,6 +152,74 @@ func TestApplyComponentScopedParserConfig_DatasetMetadataIsAuthoritative(t *test
}
}
+func TestApplyComponentScopedParserConfig_RemovesFlatMetadataWithoutExtractor(t *testing.T) {
+ // Strict contract: a flat "metadata" transport key with no Extractor node to
+ // receive it is invalid input. The flat key is dropped and no Extractor node
+ // is auto-created — the scoping path carries no compatibility shim.
+ parserConfig := entity.JSONMap{
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{"key": "author", "type": "string"},
+ },
+ "built_in_metadata": []any{},
+ },
+ "GeneralChunker:SixApplesFall": map[string]any{"chunk_token_size": float64(256)},
+ }
+
+ got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")
+
+ if _, ok := got["metadata"]; ok {
+ t.Fatalf("flat top-level metadata must be removed, got %#v", got["metadata"])
+ }
+ if _, ok := got["Extractor:AutoExtractDefault"]; ok {
+ t.Fatalf("must not auto-create Extractor node, got %#v", got)
+ }
+ // The chunker node is untouched and the config stays component-scoped.
+ if _, ok := got["GeneralChunker:SixApplesFall"].(map[string]any); !ok {
+ t.Fatalf("expected chunker node preserved, got %#v", got["GeneralChunker:SixApplesFall"])
+ }
+}
+
+func TestApplyComponentScopedParserConfig_ScopesFlatMetadataOntoExistingExtractor(t *testing.T) {
+ // Realistic case: a DSL-derived config already has an Extractor node. The
+ // flat modular "metadata" transport key must be scoped onto that node (and
+ // the tenant llm_id filled in) and removed from the top level.
+ parserConfig := entity.JSONMap{
+ "Extractor:AutoExtractDefault": map[string]any{
+ "llm_id": "",
+ },
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{"key": "author", "type": "string"},
+ },
+ "built_in_metadata": []any{},
+ },
+ }
+
+ got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")
+
+ if _, ok := got["metadata"]; ok {
+ t.Fatalf("flat top-level metadata must be removed, got %#v", got["metadata"])
+ }
+ extractor, ok := got["Extractor:AutoExtractDefault"].(map[string]any)
+ if !ok {
+ t.Fatalf("expected Extractor:AutoExtractDefault node, got %#v", got)
+ }
+ meta, ok := extractor["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("expected metadata scoped onto Extractor node, got %#v", extractor["metadata"])
+ }
+ fields, _ := meta["metadata"].([]any)
+ if len(fields) != 1 || fields[0].(map[string]any)["key"] != "author" {
+ t.Fatalf("expected author field scoped, got %#v", meta["metadata"])
+ }
+ if extractor["llm_id"] != "tenant-llm" {
+ t.Fatalf("expected llm_id filled from tenant, got %#v", extractor["llm_id"])
+ }
+}
+
func TestApplyComponentScopedParserConfig_PreservesNodeOnlyMetadata(t *testing.T) {
parserConfig := entity.JSONMap{
"Extractor:CanvasNode": map[string]any{
diff --git a/internal/service/dataset/create_test.go b/internal/service/dataset/create_test.go
index 618660c576..e59f32f2df 100644
--- a/internal/service/dataset/create_test.go
+++ b/internal/service/dataset/create_test.go
@@ -80,15 +80,32 @@ func TestCreateDataset_DefaultsParentChildConfig(t *testing.T) {
if !ok {
t.Fatalf("parser_config type = %T, want entity.JSONMap", result["parser_config"])
}
- parentChild, ok := config["parent_child"].(map[string]interface{})
+ chunker := findChunkerNodeInTest(t, map[string]interface{}(config))
+ parentChild, ok := chunker["parent_child"].(map[string]interface{})
if !ok {
- t.Fatalf("parent_child = %#v, want default map", config["parent_child"])
+ t.Fatalf("chunker parent_child = %#v, want default map", chunker["parent_child"])
}
if parentChild["use_parent_child"] != false || parentChild["children_delimiter"] != "\n" {
t.Fatalf("parent_child = %#v, want disabled defaults", parentChild)
}
}
+// findChunkerNodeInTest returns the first chunker component params from a
+// parser_config (the component-scoped home of the parent_child setting).
+func findChunkerNodeInTest(t *testing.T, config map[string]interface{}) map[string]interface{} {
+ t.Helper()
+ for cpnID, raw := range config {
+ lower := strings.ToLower(cpnID)
+ if strings.HasPrefix(lower, "generalchunker:") || strings.HasPrefix(lower, "tokenchunker:") {
+ if params, ok := raw.(map[string]interface{}); ok {
+ return params
+ }
+ }
+ }
+ t.Fatalf("no chunker node found in parser_config: %#v", config)
+ return nil
+}
+
func TestCreateDataset_BuiltinParserDoesNotRequireParseType(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
@@ -152,9 +169,11 @@ func TestCreateDataset_ParentChildConfigReachesGeneralChunker(t *testing.T) {
ParserID: &parserID,
ParseType: &parseType,
ParserConfig: map[string]interface{}{
- "parent_child": map[string]interface{}{
- "use_parent_child": true,
- "children_delimiter": "|",
+ "GeneralChunker:SixApplesFall": map[string]interface{}{
+ "parent_child": map[string]interface{}{
+ "use_parent_child": true,
+ "children_delimiter": "|",
+ },
},
},
}, "tenant-1")
diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.go
index d022fc2a7d..2835e89fd9 100644
--- a/internal/service/dataset/crud.go
+++ b/internal/service/dataset/crud.go
@@ -128,21 +128,27 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD
// Preserve the public default shape when parser_config is empty. The
// parent_child block remains the single source of truth; chunker
// children_delimiters are derived below only when it is configured.
- var parentChild map[string]interface{}
- if req.ParserConfig != nil {
- if pc, ok := req.ParserConfig["parent_child"].(map[string]interface{}); ok {
- parentChild = pc
- }
- }
+ parentChild := resolveParentChild(req.ParserConfig)
if parentChild == nil {
parentChild = map[string]interface{}{
"use_parent_child": false,
"children_delimiter": "\n",
}
}
- parserConfig["parent_child"] = parentChild
+ // Scope parent_child onto every chunker node (component-scoped); no flat key.
+ for componentID, value := range parserConfig {
+ if !pipelinepkg.IsChunkerComponent(componentID) {
+ continue
+ }
+ params, ok := value.(map[string]interface{})
+ if !ok {
+ params = map[string]interface{}{}
+ parserConfig[componentID] = params
+ }
+ params["parent_child"] = parentChild
+ }
- parentChildConfig := map[string]interface{}{"parent_child": parentChild}
+ parentChildConfig := map[string]interface{}{}
if req.ParserConfig != nil {
for componentID, value := range req.ParserConfig {
if pipelinepkg.IsChunkerComponent(componentID) {
diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.go
index 2e02d799c4..8d2ab4801b 100644
--- a/internal/service/dataset/helpers.go
+++ b/internal/service/dataset/helpers.go
@@ -198,183 +198,28 @@ func validateDatasetParserConfigSize(parserConfig map[string]interface{}) error
return nil
}
+// validateDatasetParserConfig enforces the Go backend's "component-scoped only"
+// contract: every top-level parser_config key must be keyed by a component id
+// (it contains ":"), never a flat (non-component-scoped) key. The Python backend
+// is fully retired (SystemHandler.Language always returns "go"), so the dataset
+// parser_config is always built from a DSL whose params live on component nodes
+// (e.g. chunk size is "chunk_token_size" on the chunker node, not the legacy
+// flat "chunk_token_num"). Any flat key would be silently dropped by
+// CleanComponentParams downstream and is never consumed, so we reject it loudly
+// here instead.
+//
+// ValidateDocumentParserConfig delegates to this same check, so documents and
+// datasets share one contract: no flat keys are ever permitted on either path.
func validateDatasetParserConfig(parserConfig map[string]interface{}) error {
- for key := range parserConfig {
- if strings.Contains(key, ":") {
- return nil // Component-scoped DSL parameters are validated by BuildParserConfig.
- }
+ if len(parserConfig) == 0 {
+ return nil
}
- allowed := map[string]bool{"layout_recognize": true, "chunk_token_num": true, "delimiter": true, "auto_keywords": true, "auto_questions": true, "html4excel": true, "image_context_size": true, "table_context_size": true, "topn_tags": true, "llm_id": true, "parent_child": true, "children_delimiter": true, "tag_kb_ids": true, "filename_embd_weight": true, "task_page_size": true, "pages": true, "graphrag": true, "raptor": true}
for key := range parserConfig {
- if !allowed[key] {
- return fmt.Errorf("Extra inputs are not permitted: %s", key)
- }
- }
- intBounds := map[string][2]float64{"auto_keywords": {0, 32}, "auto_questions": {0, 10}, "chunk_token_num": {1, 2048}, "topn_tags": {1, 10}, "task_page_size": {1, 100000000}}
- for key, bounds := range intBounds {
- if value, ok := parserConfig[key]; ok {
- if value == nil && key == "task_page_size" {
- continue
- }
- n, ok := value.(float64)
- if !ok || n != float64(int64(n)) {
- return errors.New("Input should be a valid integer")
- }
- if n < bounds[0] {
- return fmt.Errorf("Input should be greater than or equal to %v", int(bounds[0]))
- }
- if n > bounds[1] {
- return fmt.Errorf("Input should be less than or equal to %v", int(bounds[1]))
- }
- }
- }
- if value, ok := parserConfig["delimiter"]; ok {
- s, ok := value.(string)
- if !ok {
- return errors.New("Input should be a valid string")
- }
- if len(s) == 0 {
- return errors.New("String should have at least 1 character")
- }
- }
- if value, ok := parserConfig["html4excel"]; ok {
- if _, ok := value.(bool); !ok {
- return errors.New("Input should be a valid boolean")
- }
- }
- if value, ok := parserConfig["tag_kb_ids"]; ok {
- list, ok := value.([]interface{})
- if !ok {
- return errors.New("Input should be a valid list")
- }
- for _, item := range list {
- if _, ok := item.(string); !ok {
- return errors.New("Input should be a valid string")
- }
- }
- }
- if value, ok := parserConfig["pages"]; ok {
- if value == nil {
- return nil
- }
- list, ok := value.([]interface{})
- if !ok {
- return errors.New("Input should be a valid list")
- }
- for _, item := range list {
- row, ok := item.([]interface{})
- if !ok || len(row) != 2 {
- return errors.New("Input should be a valid list")
- }
- for _, bound := range row {
- n, ok := bound.(float64)
- if !ok || n != float64(int64(n)) {
- return errors.New("Input should be a valid integer")
- }
- }
- }
- }
- if value, ok := parserConfig["filename_embd_weight"]; ok {
- n, ok := value.(float64)
- if !ok {
- return errors.New("Input should be a valid number")
- }
- if n < 0 {
- return errors.New("Input should be greater than or equal to 0")
- }
- if n > 1 {
- return errors.New("Input should be less than or equal to 1")
- }
- }
- for _, key := range []string{"raptor", "graphrag", "parent_child"} {
- value, ok := parserConfig[key]
- if !ok {
- continue
- }
- obj, ok := value.(map[string]interface{})
- if !ok {
- return errors.New("Input should be a valid dictionary")
- }
- if key == "graphrag" {
- if v, exists := obj["use_graphrag"]; exists {
- if _, ok := v.(bool); !ok {
- return errors.New("Input should be a valid boolean")
- }
- }
- if v, exists := obj["entity_types"]; exists {
- list, ok := v.([]interface{})
- if !ok {
- return errors.New("Input should be a valid list")
- }
- for _, item := range list {
- if _, ok := item.(string); !ok {
- return errors.New("Input should be a valid string")
- }
- }
- }
- if v, exists := obj["method"]; exists {
- method, ok := v.(string)
- if !ok || (method != "light" && method != "general" && method != "ner") {
- return errors.New("Input should be 'light', 'general' or 'ner'")
- }
- }
- for _, name := range []string{"community", "resolution"} {
- if v, exists := obj[name]; exists {
- if _, ok := v.(bool); !ok {
- return errors.New("Input should be a valid boolean")
- }
- }
- }
- }
- if key == "raptor" {
- if v, exists := obj["use_raptor"]; exists {
- if _, ok := v.(bool); !ok {
- return errors.New("Input should be a valid boolean")
- }
- }
- if v, exists := obj["prompt"]; exists {
- if s, ok := v.(string); !ok || strings.TrimSpace(s) == "" {
- return errors.New("String should have at least 1 character")
- }
- }
- for name, bounds := range map[string][2]float64{"max_token": {1, 2048}, "max_cluster": {1, 1024}, "random_seed": {0, 9223372036854775807}} {
- if v, exists := obj[name]; exists {
- n, ok := v.(float64)
- if !ok || n != float64(int64(n)) {
- return errors.New("Input should be a valid integer")
- }
- if n < bounds[0] {
- return fmt.Errorf("Input should be greater than or equal to %v", int(bounds[0]))
- }
- if n > bounds[1] {
- return fmt.Errorf("Input should be less than or equal to %v", int(bounds[1]))
- }
- }
- }
- if v, exists := obj["clustering_threshold"]; exists {
- n, ok := v.(float64)
- if !ok {
- return errors.New("Input should be a valid number")
- }
- if n < 0 {
- return errors.New("Input should be greater than or equal to 0")
- }
- if n > 1 {
- return errors.New("Input should be less than or equal to 1")
- }
- }
- }
- if key == "parent_child" {
- if v, exists := obj["use_parent_child"]; exists {
- if _, ok := v.(bool); !ok {
- return errors.New("Input should be a valid boolean")
- }
- }
- if v, exists := obj["children_delimiter"]; exists {
- if s, ok := v.(string); !ok || s == "" {
- return errors.New("String should have at least 1 character")
- }
- }
+ if !strings.Contains(key, ":") {
+ return fmt.Errorf(
+ "parser_config key %q must be component-scoped (e.g. under an Extractor or GeneralChunker node), not a flat top-level key",
+ key,
+ )
}
}
return nil
@@ -385,17 +230,14 @@ func ValidateParserConfig(parserConfig map[string]interface{}) error {
return validateDatasetParserConfig(parserConfig)
}
-// ValidateDocumentParserConfig validates known public parser_config fields.
-// Documents retain unknown parser settings for parser-specific consumers.
+// ValidateDocumentParserConfig validates the parser_config attached to a
+// document. Documents follow the same component-scoped contract as datasets:
+// every key must be scoped under a node id (e.g. "Extractor:AutoExtractDefault"
+// or "GeneralChunker:SixApplesFall"). A document's Extractor/GeneralChunker
+// nodes come from the same pipeline DSL as the dataset, so there is no flat-key
+// fallback to keep. validateDatasetParserConfig enforces the no-flat-key rule.
func ValidateDocumentParserConfig(parserConfig map[string]interface{}) error {
- known := map[string]bool{"layout_recognize": true, "chunk_token_num": true, "delimiter": true, "auto_keywords": true, "auto_questions": true, "html4excel": true, "image_context_size": true, "table_context_size": true, "topn_tags": true, "llm_id": true, "parent_child": true, "children_delimiter": true, "tag_kb_ids": true, "filename_embd_weight": true, "task_page_size": true, "pages": true, "graphrag": true, "raptor": true}
- config := make(map[string]interface{}, len(parserConfig))
- for key, value := range parserConfig {
- if known[key] || strings.Contains(key, ":") {
- config[key] = value
- }
- }
- return validateDatasetParserConfig(config)
+ return validateDatasetParserConfig(parserConfig)
}
// NormalizeDatasetID validates the dataset ID format and returns its
@@ -513,31 +355,30 @@ func preserveDatasetParserConfigState(next, existing entity.JSONMap, incoming ma
}
var mm map[string]any
if incoming != nil {
- if v, ok := incoming["metadata"].(map[string]any); ok {
- mm = v
- }
+ mm = extractorNodeMetadata(incoming)
}
if mm == nil && existing != nil {
- if v, ok := existing["metadata"].(map[string]any); ok {
- mm = v
- }
+ mm = extractorNodeMetadata(existing)
}
if mm != nil {
next["metadata"] = mm
}
- var parentChild map[string]any
- if incoming != nil {
- if value, ok := incoming["parent_child"].(map[string]any); ok {
- parentChild = value
- }
- }
- if parentChild == nil && existing != nil {
- if value, ok := existing["parent_child"].(map[string]any); ok {
- parentChild = value
- }
- }
+ // Resolve the dataset-level parent_child setting (component-scoped on a
+ // chunker node) and scope it onto every chunker node in next. There is no
+ // flat aggregation key.
+ parentChild := resolveParentChild(incoming, existing)
if parentChild != nil {
- next["parent_child"] = parentChild
+ for componentID, value := range next {
+ if !pipelinepkg.IsChunkerComponent(componentID) {
+ continue
+ }
+ params, ok := value.(map[string]interface{})
+ if !ok {
+ params = map[string]interface{}{}
+ next[componentID] = params
+ }
+ params["parent_child"] = parentChild
+ }
}
requestedChildren := make(map[string]interface{})
for componentID, value := range incoming {
@@ -556,14 +397,14 @@ func preserveDatasetParserConfigState(next, existing entity.JSONMap, incoming ma
}
// Re-derive delimiters from parent_child, then keep explicit chunker edits
// (or an existing chunker setting on a partial update) over that fallback.
- parentChildConfig := map[string]interface{}{"parent_child": parentChild}
+ parentChildConfig := map[string]interface{}{}
for componentID, value := range incoming {
if pipelinepkg.IsChunkerComponent(componentID) {
parentChildConfig[componentID] = value
}
}
pipelinepkg.ApplyParentChildChunkerConfig(next, parentChildConfig)
- _, parentChildUpdated := incoming["parent_child"]
+ parentChildUpdated := parentChild != nil
for componentID, value := range next {
if !pipelinepkg.IsChunkerComponent(componentID) {
continue
@@ -591,6 +432,55 @@ func preserveDatasetParserConfigState(next, existing entity.JSONMap, incoming ma
return next
}
+// extractorNodeMetadata returns the modular metadata object ({enabled, metadata,
+// built_in_metadata}) from the first Extractor node in a parser_config, or nil
+// if none is present. Metadata is component-scoped under Extractor nodes; this
+// helper reads it back for preservation across partial updates.
+func extractorNodeMetadata(parserConfig map[string]interface{}) map[string]any {
+ if parserConfig == nil {
+ return nil
+ }
+ for cpnID, raw := range parserConfig {
+ lower := strings.ToLower(cpnID)
+ if !strings.HasPrefix(lower, "extractor:") && !strings.HasPrefix(lower, "extractor_") {
+ continue
+ }
+ params, ok := raw.(map[string]any)
+ if !ok {
+ continue
+ }
+ if meta, ok := params["metadata"].(map[string]any); ok {
+ return meta
+ }
+ }
+ return nil
+}
+
+// resolveParentChild extracts the dataset-level parent_child setting from one or
+// more parser_configs. It reads the component-scoped "parent_child" sub-object on
+// a chunker node only; flat top-level keys are not accepted. Returns nil when
+// absent.
+func resolveParentChild(configs ...map[string]interface{}) map[string]any {
+ for _, cfg := range configs {
+ if cfg == nil {
+ continue
+ }
+ for componentID, raw := range cfg {
+ if !pipelinepkg.IsChunkerComponent(componentID) {
+ continue
+ }
+ params, ok := raw.(map[string]any)
+ if !ok {
+ continue
+ }
+ if pc, ok := params["parent_child"].(map[string]any); ok {
+ return pc
+ }
+ }
+ }
+ return nil
+}
+
func parserConfigJSONMap(value interface{}) entity.JSONMap {
switch typed := value.(type) {
case nil:
diff --git a/internal/service/dataset/helpers_test.go b/internal/service/dataset/helpers_test.go
index b112a42868..4ae3e07e6f 100644
--- a/internal/service/dataset/helpers_test.go
+++ b/internal/service/dataset/helpers_test.go
@@ -205,40 +205,113 @@ func TestValidateDatasetParserConfigSize_OverLimit(t *testing.T) {
}
}
-func TestValidateDatasetParserConfig_AllowsNullableOptionalFields(t *testing.T) {
+func TestValidateDatasetParserConfig_RejectsFlatNullableOptionalFields(t *testing.T) {
+ // A flat key is rejected regardless of its value (even nil): datasets are
+ // component-scoped only, so "task_page_size"/"pages" must live on a node.
for _, config := range []map[string]interface{}{
{"task_page_size": nil},
{"pages": nil},
} {
- if err := validateDatasetParserConfig(config); err != nil {
- t.Fatalf("validateDatasetParserConfig(%#v): %v", config, err)
+ if err := validateDatasetParserConfig(config); err == nil {
+ t.Fatalf("validateDatasetParserConfig(%#v): expected error for flat key, got nil", config)
}
}
}
func TestValidateDatasetParserConfig_DelimiterType(t *testing.T) {
err := validateDatasetParserConfig(map[string]interface{}{"delimiter": float64(1)})
- if err == nil || err.Error() != "Input should be a valid string" {
- t.Fatalf("err=%v", err)
+ if err == nil {
+ t.Fatal("expected error for flat delimiter key")
}
}
-func TestValidateDocumentParserConfig_AllowsUnknownFields(t *testing.T) {
- if err := ValidateDocumentParserConfig(map[string]interface{}{"parser_specific": "value"}); err != nil {
- t.Fatalf("err=%v", err)
+func TestValidateDocumentParserConfig_RejectsAnyFlatKey(t *testing.T) {
+ // Documents follow the same component-scoped contract as datasets: no flat
+ // top-level key is accepted (the Go backend scopes every setting under a
+ // node id such as "Extractor:AutoExtractDefault" or "GeneralChunker:SixApplesFall").
+ for _, flat := range []map[string]interface{}{
+ {"parser_specific": "value"},
+ {"delimiter": float64(1)},
+ {"children_delimiter": "|"},
+ {"metadata": map[string]interface{}{}},
+ {"parent_child": map[string]interface{}{}},
+ } {
+ if err := ValidateDocumentParserConfig(flat); err == nil {
+ t.Fatalf("expected error for flat document config %#v", flat)
+ }
}
- if err := ValidateDocumentParserConfig(map[string]interface{}{"delimiter": float64(1)}); err == nil {
- t.Fatal("expected known-field validation error")
+ // Component-scoped keys always pass.
+ if err := ValidateDocumentParserConfig(map[string]interface{}{
+ "Extractor:AutoExtractDefault": map[string]interface{}{"metadata": map[string]interface{}{}},
+ "GeneralChunker:SixApplesFall": map[string]interface{}{"chunk_token_size": float64(512)},
+ "GeneralChunker:SixApplesFallPC": map[string]interface{}{"parent_child": map[string]interface{}{}},
+ }); err != nil {
+ t.Fatalf("expected nil for component-scoped keys, got %v", err)
}
}
-func TestValidateParserConfigAcceptsFlatParentChildDelimiter(t *testing.T) {
+func TestValidateParserConfigRejectsFlatParentChildDelimiter(t *testing.T) {
config := map[string]interface{}{"children_delimiter": "|"}
- if err := validateDatasetParserConfig(config); err != nil {
- t.Fatalf("flat children_delimiter should remain accepted for compatibility: %v", err)
+ // Datasets reject every flat key, including parser-level ones.
+ if err := validateDatasetParserConfig(config); err == nil {
+ t.Fatal("expected error for flat children_delimiter on dataset")
}
- if err := ValidateDocumentParserConfig(config); err != nil {
- t.Fatalf("document parser config should accept children_delimiter: %v", err)
+ // Documents share the same no-flat-key contract.
+ if err := ValidateDocumentParserConfig(config); err == nil {
+ t.Fatal("expected error for flat children_delimiter on document")
+ }
+}
+
+func TestValidateDatasetParserConfig_RejectsAnyFlatKey(t *testing.T) {
+ for _, flat := range []map[string]interface{}{
+ {"chunk_token_num": float64(128)},
+ {"delimiter": "\n"},
+ {"children_delimiter": "|"},
+ {"layout_recognize": "DeepDOC"},
+ {"auto_keywords": float64(0)},
+ {"task_page_size": float64(1)},
+ {"pages": nil},
+ {"parser_specific": "value"},
+ } {
+ if err := validateDatasetParserConfig(flat); err == nil {
+ t.Fatalf("expected error for flat dataset config %#v", flat)
+ }
+ }
+ // Component-scoped keys (containing ":") always pass.
+ if err := validateDatasetParserConfig(map[string]interface{}{
+ "GeneralChunker:SixApplesFall": map[string]interface{}{"chunk_token_size": float64(512)},
+ }); err != nil {
+ t.Fatalf("expected nil for component-scoped keys, got %v", err)
+ }
+}
+
+func TestValidateParserConfigRejectsFlatMetadataAndParentChild(t *testing.T) {
+ if err := validateDatasetParserConfig(map[string]interface{}{"metadata": map[string]interface{}{}}); err == nil {
+ t.Fatal("expected error for flat metadata key")
+ }
+ if err := ValidateDocumentParserConfig(map[string]interface{}{"parent_child": map[string]interface{}{}}); err == nil {
+ t.Fatal("expected error for flat parent_child key")
+ }
+ // Component-scoped (cpnID-keyed) forms are accepted.
+ if err := validateDatasetParserConfig(map[string]interface{}{
+ "Extractor:AutoExtractDefault": map[string]interface{}{"metadata": map[string]interface{}{}},
+ "GeneralChunker:SixApplesFall": map[string]interface{}{"parent_child": map[string]interface{}{}},
+ }); err != nil {
+ t.Fatalf("expected nil for component-scoped keys, got %v", err)
+ }
+ // Parser-level flat keys are no longer accepted on datasets: the Go backend
+ // is component-scoped only (chunk size is "chunk_token_size" on the chunker
+ // node, not the legacy flat "chunk_token_num").
+ for _, flat := range []map[string]interface{}{
+ {"chunk_token_num": float64(128)},
+ {"delimiter": "\n"},
+ {"children_delimiter": "|"},
+ {"layout_recognize": "DeepDOC"},
+ {"parser_specific": "value"},
+ } {
+ if err := validateDatasetParserConfig(flat); err == nil {
+ t.Fatalf("expected error for flat dataset config %#v", flat)
+ }
}
}
@@ -368,10 +441,12 @@ func TestNormalizeMetadataConfigFields_TrimsKey(t *testing.T) {
func TestPreserveDatasetParserConfigState_FallsBackWhenIncomingNotMap(t *testing.T) {
existing := entity.JSONMap{
- "metadata": map[string]any{
- "enabled": true,
- "metadata": []any{map[string]any{"key": "existing_field", "type": "string"}},
- "built_in_metadata": []any{},
+ "Extractor:AutoExtractDefault": map[string]any{
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{map[string]any{"key": "existing_field", "type": "string"}},
+ "built_in_metadata": []any{},
+ },
},
}
cases := map[string]interface{}{
@@ -380,7 +455,11 @@ func TestPreserveDatasetParserConfigState_FallsBackWhenIncomingNotMap(t *testing
}
for name, incomingMetadata := range cases {
t.Run(name, func(t *testing.T) {
- incoming := map[string]interface{}{"metadata": incomingMetadata}
+ incoming := map[string]interface{}{
+ "Extractor:AutoExtractDefault": map[string]any{
+ "metadata": incomingMetadata,
+ },
+ }
got := preserveDatasetParserConfigState(entity.JSONMap{}, existing, incoming)
meta, ok := got["metadata"].(map[string]any)
if !ok {
@@ -396,17 +475,21 @@ func TestPreserveDatasetParserConfigState_FallsBackWhenIncomingNotMap(t *testing
func TestPreserveDatasetParserConfigState_UsesValidIncomingMap(t *testing.T) {
existing := entity.JSONMap{
- "metadata": map[string]any{
- "enabled": false,
- "metadata": []any{},
- "built_in_metadata": []any{},
+ "Extractor:AutoExtractDefault": map[string]any{
+ "metadata": map[string]any{
+ "enabled": false,
+ "metadata": []any{},
+ "built_in_metadata": []any{},
+ },
},
}
incoming := map[string]interface{}{
- "metadata": map[string]any{
- "enabled": true,
- "metadata": []any{map[string]any{"key": "incoming_field", "type": "string"}},
- "built_in_metadata": []any{},
+ "Extractor:AutoExtractDefault": map[string]any{
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{map[string]any{"key": "incoming_field", "type": "string"}},
+ "built_in_metadata": []any{},
+ },
},
}
got := preserveDatasetParserConfigState(entity.JSONMap{}, existing, incoming)
diff --git a/internal/service/dataset/metadata.go b/internal/service/dataset/metadata.go
index f3243345d8..b7854e7247 100644
--- a/internal/service/dataset/metadata.go
+++ b/internal/service/dataset/metadata.go
@@ -173,20 +173,40 @@ func (d *DatasetService) UpdateMetadataConfig(ctx context.Context, datasetID, te
}
// modularMetadataConfig reads the modular dataset-level metadata object
-// ({"enabled", "metadata", "built_in_metadata"}) from parser_config. Missing
-// or malformed config yields a disabled, empty result.
+// ({"enabled", "metadata", "built_in_metadata"}) from parser_config. The
+// dataset-level flat "metadata" key is authoritative when present; the
+// component-scoped copy under an Extractor node is used as a fallback (the
+// stored form after scoping). Configs without either (e.g. minimal fixtures)
+// yield a disabled, empty result.
func modularMetadataConfig(parserConfig map[string]any) (bool, bool, []any, []any) {
if parserConfig == nil {
return false, false, []any{}, []any{}
}
- metaObj, ok := parserConfig["metadata"].(map[string]any)
- if !ok {
- return false, false, []any{}, []any{}
+ if metaObj, ok := parserConfig["metadata"].(map[string]any); ok {
+ enabled, _ := metaObj["enabled"].(bool)
+ metadata := anyOrEmptyList(metaObj["metadata"])
+ builtIn := anyOrEmptyList(metaObj["built_in_metadata"])
+ return true, enabled, metadata, builtIn
+ }
+ for cpnID, raw := range parserConfig {
+ lower := strings.ToLower(cpnID)
+ if !strings.HasPrefix(lower, "extractor:") && !strings.HasPrefix(lower, "extractor_") {
+ continue
+ }
+ params, ok := raw.(map[string]any)
+ if !ok {
+ continue
+ }
+ metaObj, ok := params["metadata"].(map[string]any)
+ if !ok {
+ continue
+ }
+ enabled, _ := metaObj["enabled"].(bool)
+ metadata := anyOrEmptyList(metaObj["metadata"])
+ builtIn := anyOrEmptyList(metaObj["built_in_metadata"])
+ return true, enabled, metadata, builtIn
}
- enabled, _ := metaObj["enabled"].(bool)
- metadata := anyOrEmptyList(metaObj["metadata"])
- builtIn := anyOrEmptyList(metaObj["built_in_metadata"])
- return true, enabled, metadata, builtIn
+ return false, false, []any{}, []any{}
}
func anyOrEmptyList(value any) []any {
diff --git a/internal/service/dataset/metadata_config_test.go b/internal/service/dataset/metadata_config_test.go
index ae6aa37599..afb97bfae6 100644
--- a/internal/service/dataset/metadata_config_test.go
+++ b/internal/service/dataset/metadata_config_test.go
@@ -59,7 +59,7 @@ func insertDatasetMetadataConfigKB(t *testing.T, datasetID, tenantID string) {
CreatedBy: tenantID,
Permission: string(entity.TenantPermissionMe),
ParserID: "naive",
- ParserConfig: entity.JSONMap{},
+ ParserConfig: entity.JSONMap{"Extractor:AutoExtractDefault": map[string]any{}},
Status: sptr("1"),
}
if err := dao.DB.Create(kb).Error; err != nil {
@@ -83,6 +83,14 @@ func insertDatasetMetadataConfigTeamMember(t *testing.T, userID, tenantID string
func insertDatasetMetadataConfigDoc(t *testing.T, docID, datasetID string, parserConfig entity.JSONMap) {
t.Helper()
+ if parserConfig == nil {
+ parserConfig = entity.JSONMap{}
+ }
+ // A realistic document parser_config always carries an Extractor node (it
+ // derives from a DSL), so ensure one is present for the metadata-update path.
+ if _, ok := parserConfig["Extractor:AutoExtractDefault"].(map[string]any); !ok {
+ parserConfig["Extractor:AutoExtractDefault"] = map[string]any{}
+ }
doc := &entity.Document{
ID: docID,
KbID: datasetID,
@@ -128,9 +136,13 @@ func TestDatasetServiceUpdateDocumentMetadataConfig(t *testing.T) {
t.Fatalf("existing parser_config fields should be preserved: %#v", doc.ParserConfig)
}
- updatedMetadata, ok := doc.ParserConfig["metadata"].(map[string]interface{})
+ docExtractor, ok := doc.ParserConfig["Extractor:AutoExtractDefault"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("expected Extractor:AutoExtractDefault node, got %#v", doc.ParserConfig)
+ }
+ updatedMetadata, ok := docExtractor["metadata"].(map[string]interface{})
if !ok {
- t.Fatalf("expected metadata map, got %#v", doc.ParserConfig["metadata"])
+ t.Fatalf("expected metadata map, got %#v", docExtractor["metadata"])
}
if updatedMetadata["author"] != "Alice" || updatedMetadata["year"] != float64(2026) {
t.Fatalf("unexpected metadata: %#v", updatedMetadata)
@@ -140,7 +152,11 @@ func TestDatasetServiceUpdateDocumentMetadataConfig(t *testing.T) {
if err != nil {
t.Fatalf("failed to fetch persisted document: %v", err)
}
- if persisted.ParserConfig["metadata"] == nil {
+ persistedExtractor, ok := persisted.ParserConfig["Extractor:AutoExtractDefault"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("metadata was not persisted: %#v", persisted.ParserConfig)
+ }
+ if persistedExtractor["metadata"] == nil {
t.Fatalf("metadata was not persisted: %#v", persisted.ParserConfig)
}
}
@@ -222,7 +238,7 @@ func TestDatasetServiceUpdateDocumentMetadataConfigAllowsTeamMember(t *testing.T
if code != common.CodeSuccess {
t.Fatalf("expected success code, got %d", code)
}
- if doc.ParserConfig["metadata"] == nil {
+ if doc.ParserConfig["Extractor:AutoExtractDefault"] == nil {
t.Fatalf("metadata was not updated: %#v", doc.ParserConfig)
}
}
@@ -377,9 +393,16 @@ func TestDatasetServiceUpdateMetadataConfigSyncsExtractorSchema(t *testing.T) {
if _, ok := persisted.ParserConfig["enable_metadata"]; ok {
t.Fatalf("enable_metadata should be absent, got %#v", persisted.ParserConfig["enable_metadata"])
}
- metaObj, ok := persisted.ParserConfig["metadata"].(map[string]interface{})
+ if _, ok := persisted.ParserConfig["metadata"]; ok {
+ t.Fatalf("top-level flat metadata should be absent, got %#v", persisted.ParserConfig["metadata"])
+ }
+ extractor, ok = persisted.ParserConfig["Extractor:AutoExtractDefault"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("expected extractor component params, got %#v", persisted.ParserConfig["Extractor:AutoExtractDefault"])
+ }
+ metaObj, ok := extractor["metadata"].(map[string]interface{})
if !ok {
- t.Fatalf("expected modular metadata map, got %#v", persisted.ParserConfig["metadata"])
+ t.Fatalf("expected modular metadata map under extractor node, got %#v", extractor["metadata"])
}
if enabled, ok := metaObj["enabled"].(bool); !ok || enabled {
t.Fatalf("expected modular metadata.enabled == false after emptying fields, got %#v", metaObj["enabled"])
@@ -496,9 +519,13 @@ func TestUpdateMetadataConfig_ExplicitEnabledPreservedWhenEmpty(t *testing.T) {
if err != nil {
t.Fatalf("get kb: %v", err)
}
- metaObj, ok := persisted.ParserConfig["metadata"].(map[string]interface{})
+ extractor, ok := persisted.ParserConfig["Extractor:AutoExtractDefault"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("expected Extractor:AutoExtractDefault node, got %#v", persisted.ParserConfig)
+ }
+ metaObj, ok := extractor["metadata"].(map[string]interface{})
if !ok {
- t.Fatalf("expected modular metadata, got %#v", persisted.ParserConfig["metadata"])
+ t.Fatalf("expected modular metadata, got %#v", extractor["metadata"])
}
if persistedEnabled, _ := metaObj["enabled"].(bool); !persistedEnabled {
t.Fatalf("expected metadata.enabled=true, got %#v", metaObj["enabled"])
diff --git a/internal/service/dataset/update_test.go b/internal/service/dataset/update_test.go
index 9f541322ed..73b94edb84 100644
--- a/internal/service/dataset/update_test.go
+++ b/internal/service/dataset/update_test.go
@@ -108,9 +108,11 @@ func TestUpdateDataset_ParentChildConfigReachesGeneralChunker(t *testing.T) {
_, code, err := testDatasetUpdateService(t).UpdateDataset(t.Context(), "kb-1", "tenant-1", service.UpdateDatasetRequest{
ParserConfig: map[string]interface{}{
- "parent_child": map[string]interface{}{
- "use_parent_child": true,
- "children_delimiter": "|",
+ "GeneralChunker:SixApplesFall": map[string]interface{}{
+ "parent_child": map[string]interface{}{
+ "use_parent_child": true,
+ "children_delimiter": "|",
+ },
},
},
})
@@ -128,9 +130,12 @@ func TestUpdateDataset_ParentChildConfigReachesGeneralChunker(t *testing.T) {
if got, ok := chunker["children_delimiters"].([]interface{}); !ok || len(got) != 1 || got[0] != "|" {
t.Fatalf("children_delimiters = %#v, want [\"|\"]", chunker["children_delimiters"])
}
- parentChild, ok := persisted.ParserConfig["parent_child"].(map[string]interface{})
+ parentChild, ok := chunker["parent_child"].(map[string]interface{})
if !ok || parentChild["use_parent_child"] != true || parentChild["children_delimiter"] != "|" {
- t.Fatalf("parent_child = %#v, want persisted public setting", persisted.ParserConfig["parent_child"])
+ t.Fatalf("chunker parent_child = %#v, want persisted public setting", chunker["parent_child"])
+ }
+ if _, ok := persisted.ParserConfig["parent_child"]; ok {
+ t.Fatalf("top-level flat parent_child should be absent, got %#v", persisted.ParserConfig["parent_child"])
}
}
@@ -140,9 +145,11 @@ func TestUpdateDatasetPreservesParentChildChunkerRuntimeConfig(t *testing.T) {
insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original")
existingConfig := entity.JSONMap{
- "parent_child": map[string]interface{}{
- "use_parent_child": true,
- "children_delimiter": "|",
+ "GeneralChunker:SixApplesFall": map[string]interface{}{
+ "parent_child": map[string]interface{}{
+ "use_parent_child": true,
+ "children_delimiter": "|",
+ },
},
}
if err := db.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", existingConfig).Error; err != nil {
@@ -150,7 +157,10 @@ func TestUpdateDatasetPreservesParentChildChunkerRuntimeConfig(t *testing.T) {
}
_, code, err := testDatasetUpdateService(t).UpdateDataset(t.Context(), "kb-1", "tenant-1", service.UpdateDatasetRequest{
- ParserConfig: map[string]interface{}{"chunk_token_num": float64(256)},
+ // Component-scoped override (flat parser-level keys are rejected now).
+ ParserConfig: map[string]interface{}{
+ "GeneralChunker:SixApplesFall": map[string]interface{}{"chunk_token_size": float64(256)},
+ },
})
if err != nil || code != common.CodeSuccess {
t.Fatalf("UpdateDataset err=%v code=%d", err, code)
@@ -1185,6 +1195,27 @@ func TestUpdateDataset_AcceptsValidComponentParams_Builtin(t *testing.T) {
}
}
+// extractorNodeMetadata returns the metadata object stored on the first
+// Extractor node of a parser_config (the component-scoped form).
+func extractorNodeMetadataInTest(t *testing.T, cfg map[string]interface{}) map[string]interface{} {
+ t.Helper()
+ for cpnID, raw := range cfg {
+ lower := strings.ToLower(cpnID)
+ if !strings.HasPrefix(lower, "extractor:") && !strings.HasPrefix(lower, "extractor_") {
+ continue
+ }
+ params, ok := raw.(map[string]interface{})
+ if !ok {
+ continue
+ }
+ if meta, ok := params["metadata"].(map[string]interface{}); ok {
+ return meta
+ }
+ }
+ t.Fatalf("no Extractor node with metadata found in parser_config: %#v", cfg)
+ return nil
+}
+
func TestUpdateDataset_PreservesIncomingMetadataWhenCleaningParserConfig(t *testing.T) {
db := setupDatasetUpdateTestDB(t)
pushServiceDB(t, db)
@@ -1203,10 +1234,12 @@ func TestUpdateDataset_PreservesIncomingMetadataWhenCleaningParserConfig(t *test
"Parser:HipSignsRhyme": map[string]interface{}{
"pdf": map[string]interface{}{"parse_method": "deepdoc"},
},
- "metadata": map[string]interface{}{
- "enabled": true,
- "metadata": incomingMetadata,
- "built_in_metadata": incomingBuiltInMetadata,
+ "Extractor:AutoExtractDefault": map[string]interface{}{
+ "metadata": map[string]interface{}{
+ "enabled": true,
+ "metadata": incomingMetadata,
+ "built_in_metadata": incomingBuiltInMetadata,
+ },
},
}
@@ -1222,12 +1255,16 @@ func TestUpdateDataset_PreservesIncomingMetadataWhenCleaningParserConfig(t *test
if err != nil {
t.Fatalf("get updated kb: %v", err)
}
- if !reflect.DeepEqual(persisted.ParserConfig["metadata"], map[string]interface{}{
+ meta := extractorNodeMetadataInTest(t, map[string]interface{}(persisted.ParserConfig))
+ if !reflect.DeepEqual(meta, map[string]interface{}{
"enabled": true,
"metadata": incomingMetadata,
"built_in_metadata": incomingBuiltInMetadata,
}) {
- t.Fatalf("modular metadata was not preserved: %#v", persisted.ParserConfig["metadata"])
+ t.Fatalf("modular metadata was not preserved on extractor node: %#v", meta)
+ }
+ if _, ok := persisted.ParserConfig["metadata"]; ok {
+ t.Fatalf("top-level flat metadata should be absent: %#v", persisted.ParserConfig["metadata"])
}
if _, ok := persisted.ParserConfig["enable_metadata"]; ok {
t.Fatalf("enable_metadata should be absent: %#v", persisted.ParserConfig["enable_metadata"])
@@ -1250,10 +1287,12 @@ func TestUpdateDataset_PreservesExistingMetadataWhenParserConfigOmitsIt(t *testi
"type": "string",
}}
if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", entity.JSONMap{
- "metadata": map[string]interface{}{
- "enabled": true,
- "metadata": existingMetadata,
- "built_in_metadata": []interface{}{map[string]interface{}{"key": "document_name", "type": "string"}},
+ "Extractor:AutoExtractDefault": map[string]interface{}{
+ "metadata": map[string]interface{}{
+ "enabled": true,
+ "metadata": existingMetadata,
+ "built_in_metadata": []interface{}{map[string]interface{}{"key": "document_name", "type": "string"}},
+ },
},
}).Error; err != nil {
t.Fatalf("seed parser_config: %v", err)
@@ -1275,12 +1314,16 @@ func TestUpdateDataset_PreservesExistingMetadataWhenParserConfigOmitsIt(t *testi
if err != nil {
t.Fatalf("get updated kb: %v", err)
}
- if !reflect.DeepEqual(persisted.ParserConfig["metadata"], map[string]interface{}{
+ meta := extractorNodeMetadataInTest(t, map[string]interface{}(persisted.ParserConfig))
+ if !reflect.DeepEqual(meta, map[string]interface{}{
"enabled": true,
"metadata": existingMetadata,
"built_in_metadata": []interface{}{map[string]interface{}{"key": "document_name", "type": "string"}},
}) {
- t.Fatalf("existing modular metadata was not preserved: %#v", persisted.ParserConfig["metadata"])
+ t.Fatalf("existing modular metadata was not preserved on extractor node: %#v", meta)
+ }
+ if _, ok := persisted.ParserConfig["metadata"]; ok {
+ t.Fatalf("top-level flat metadata should be absent: %#v", persisted.ParserConfig["metadata"])
}
if _, ok := persisted.ParserConfig["enable_metadata"]; ok {
t.Fatalf("enable_metadata should be absent: %#v", persisted.ParserConfig["enable_metadata"])
diff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.go
index 04c0dd8893..830c1b5300 100644
--- a/internal/service/document/document_dataset_update.go
+++ b/internal/service/document/document_dataset_update.go
@@ -468,9 +468,6 @@ func (s *DocumentService) updateDocumentParserConfig(ctx context.Context, docume
}
merged := common.DeepMergeMaps(doc.ParserConfig, config)
- if _, ok := config["raptor"]; !ok {
- delete(merged, "raptor")
- }
pipelinepkg.ApplyParentChildChunkerConfig(merged, config)
return s.documentDAO.UpdateByID(ctx, dao.DB, documentID, map[string]interface{}{
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index 3ee329dfca..94ffd75255 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -3479,9 +3479,11 @@ func TestUpdateDatasetDocumentParentChildConfigSurvivesDSLFailure(t *testing.T)
ParseType: &parseType,
PipelineID: &pipelineID,
ParserConfig: map[string]any{
- "parent_child": map[string]any{
- "use_parent_child": true,
- "children_delimiter": "|",
+ "GeneralChunker:SixApplesFall": map[string]any{
+ "parent_child": map[string]any{
+ "use_parent_child": true,
+ "children_delimiter": "|",
+ },
},
},
}, map[string]bool{"pipeline_id": true, "parse_type": true, "parser_config": true})
@@ -3537,9 +3539,11 @@ func TestUpdateDatasetDocumentParentChildConfigReachesGeneralChunker(t *testing.
resp, code, err := testDocumentService(t).UpdateDatasetDocument(t.Context(), "tenant-1", "kb-1", "doc-1", &UpdateDatasetDocumentRequest{
ParserConfig: map[string]any{
- "parent_child": map[string]any{
- "use_parent_child": true,
- "children_delimiter": "|",
+ "GeneralChunker:SixApplesFall": map[string]any{
+ "parent_child": map[string]any{
+ "use_parent_child": true,
+ "children_delimiter": "|",
+ },
},
},
}, map[string]bool{"parser_config": true})
@@ -3553,9 +3557,12 @@ func TestUpdateDatasetDocumentParentChildConfigReachesGeneralChunker(t *testing.
if got, ok := chunker["children_delimiters"].([]interface{}); !ok || len(got) != 1 || got[0] != "|" {
t.Fatalf("children_delimiters = %#v, want [|]", chunker["children_delimiters"])
}
- parentChild, ok := resp.ParserConfig["parent_child"].(map[string]interface{})
+ parentChild, ok := chunker["parent_child"].(map[string]interface{})
if !ok || parentChild["use_parent_child"] != true || parentChild["children_delimiter"] != "|" {
- t.Fatalf("parent_child = %#v, want persisted public setting", resp.ParserConfig["parent_child"])
+ t.Fatalf("chunker parent_child = %#v, want persisted public setting", chunker["parent_child"])
+ }
+ if _, ok := resp.ParserConfig["parent_child"]; ok {
+ t.Fatalf("top-level flat parent_child should be absent, got %#v", resp.ParserConfig["parent_child"])
}
}
diff --git a/test/testcases/restful_api/test_datasets.py b/test/testcases/restful_api/test_datasets.py
index 067afe99e4..50af4cae41 100644
--- a/test/testcases/restful_api/test_datasets.py
+++ b/test/testcases/restful_api/test_datasets.py
@@ -365,6 +365,11 @@ def test_dataset_update_parser_config_valid_matrix_contract(rest_client, clear_d
)
assert update_res.status_code == 200
update_payload = update_res.json()
+ if IS_GO_PROXY:
+ # Go enforces the component-scoped contract: a flat top-level
+ # parser_config key (one without ":") is rejected with code 101.
+ assert update_payload["code"] == 101, update_payload
+ return
assert update_payload["code"] == 0, update_payload
list_res = rest_client.get("/datasets", params={"id": dataset_id})
@@ -372,11 +377,6 @@ def test_dataset_update_parser_config_valid_matrix_contract(rest_client, clear_d
list_payload = list_res.json()
assert list_payload["code"] == 0, list_payload
actual_parser_config = list_payload["data"][0]["parser_config"]
- if IS_GO_PROXY:
- assert isinstance(actual_parser_config, dict) and actual_parser_config, list_payload
- assert "raptor" not in actual_parser_config, list_payload
- assert "graphrag" not in actual_parser_config, list_payload
- return
for key, expected_value in parser_config.items():
if key in {"graphrag", "raptor"}:
assert key not in actual_parser_config, list_payload
@@ -1081,6 +1081,11 @@ def test_dataset_update_parser_config_invalid_contract(rest_client, clear_datase
assert res.status_code == 200
payload = res.json()
assert payload["code"] == 101, payload
+ if IS_GO_PROXY:
+ # Go rejects any flat top-level key wholesale with a
+ # "must be component-scoped" message, so the Python field-level
+ # validation messages below do not apply.
+ continue
assert expected_message in payload["message"], payload
@@ -1317,6 +1322,8 @@ def test_dataset_create_embedding_model_format_contract(rest_client, clear_datas
@pytest.mark.p2
def test_dataset_create_parser_config_missing_raptor_and_graphrag(rest_client, clear_datasets):
+ if IS_GO_PROXY:
+ pytest.skip("Go CreateDataset does not accept legacy flat parser_config")
payload = {
"name": "test_parser_config_missing_fields",
"parser_config": {"chunk_token_num": 1024},
@@ -1503,6 +1510,8 @@ def test_dataset_create_parser_config_valid_matrix_contract(rest_client, clear_d
ids=["only_raptor", "only_graphrag", "both_fields"],
)
def test_dataset_create_parser_config_bugfix_contract(rest_client, clear_datasets, name, parser_config):
+ if IS_GO_PROXY:
+ pytest.skip("Go CreateDataset does not accept legacy flat parser_config")
res = rest_client.post("/datasets", json={"name": name, "parser_config": parser_config})
assert res.status_code == 200
body = res.json()
@@ -1523,6 +1532,8 @@ def test_dataset_create_parser_config_bugfix_contract(rest_client, clear_dataset
ids=["qa", "manual", "paper", "book", "laws", "presentation"],
)
def test_dataset_create_parser_config_different_chunk_methods_contract(rest_client, clear_datasets, chunk_method):
+ if IS_GO_PROXY:
+ pytest.skip("Go CreateDataset does not accept legacy flat parser_config")
payload = {
"name": f"test_parser_config_{chunk_method}",
PARSER_ID_FIELD: chunk_method,
@@ -1746,6 +1757,8 @@ def test_dataset_create_permission_and_chunk_method_contract(rest_client, clear_
@pytest.mark.p2
def test_dataset_create_parser_config_invalid_contract(rest_client, clear_datasets):
+ if IS_GO_PROXY:
+ pytest.skip("Go CreateDataset does not accept legacy flat parser_config")
invalid_cases = [
("auto_keywords_min_limit", {"auto_keywords": -1}, "Input should be greater than or equal to 0"),
("auto_keywords_max_limit", {"auto_keywords": 33}, "Input should be less than or equal to 32"),
@@ -2373,7 +2386,10 @@ def test_dataset_metadata_config_get_and_update_contract(rest_client, create_dat
],
}
if IS_GO_PROXY:
- normalized_update_payload["enabled"] = True
+ # Go keeps the dataset-level metadata.enabled flag: it is not derived
+ # from whether fields are present, so a fresh dataset (flag disabled)
+ # stays disabled until enabled explicitly.
+ normalized_update_payload["enabled"] = False
update_res = rest_client.put(f"/datasets/{dataset_id}/metadata/config", json=update_payload)
assert update_res.status_code == 200
update_body = update_res.json()
diff --git a/test/testcases/restful_api/test_documents.py b/test/testcases/restful_api/test_documents.py
index 62efff3853..f1a6a36c1d 100644
--- a/test/testcases/restful_api/test_documents.py
+++ b/test/testcases/restful_api/test_documents.py
@@ -14,16 +14,19 @@
# limitations under the License.
#
-from concurrent.futures import ThreadPoolExecutor, as_completed
import string
+import uuid
+from concurrent.futures import ThreadPoolExecutor, as_completed
from contextlib import ExitStack
from pathlib import Path
-import uuid
-from openpyxl import Workbook
import pytest
import requests
+from openpyxl import Workbook
from requests_toolbelt import MultipartEncoder
+from utils import wait_for
+from utils.file_utils import create_txt_file
+
from test.testcases.configs import DEFAULT_PARSER_CONFIG, DOCUMENT_NAME_LIMIT, HOST_ADDRESS, INVALID_API_TOKEN, INVALID_ID_32, IS_GO_PROXY, VERSION
from test.testcases.restful_api.helpers.assertions import assert_auth_error
from test.testcases.restful_api.helpers.client import RestClient
@@ -39,8 +42,6 @@ from test.testcases.utils.file_utils import (
create_pdf_file,
create_ppt_file,
)
-from utils import wait_for
-from utils.file_utils import create_txt_file
@pytest.mark.p1
@@ -1044,7 +1045,12 @@ def test_document_metadata_config_contract(rest_client, create_document):
update_body = update_res.json()
assert update_body["code"] == 0, update_body
parser_config = update_body["data"]["parser_config"]
- assert parser_config["metadata"] == update_payload["metadata"], update_body
+ if IS_GO_PROXY:
+ # Go scopes metadata onto the Extractor node rather than a flat key.
+ scoped = parser_config.get("Extractor:AutoExtractDefault", {})
+ assert scoped.get("metadata") == update_payload["metadata"], update_body
+ else:
+ assert parser_config["metadata"] == update_payload["metadata"], update_body
@pytest.mark.p2
diff --git a/web/src/hooks/__tests__/parser-config-utils.test.ts b/web/src/hooks/__tests__/parser-config-utils.test.ts
new file mode 100644
index 0000000000..1076ea8a61
--- /dev/null
+++ b/web/src/hooks/__tests__/parser-config-utils.test.ts
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { normalizeParserConfig } from '@/hooks/parser-config-utils';
+
+// Drive the backend variant the same way the existing frontend tests do.
+let mockIsGoBackend = true;
+jest.mock('@/utils/backend-runtime', () => ({
+ getBackendLanguage: () => (mockIsGoBackend ? 'go' : 'python'),
+ subscribeBackendLanguage: jest.fn(),
+}));
+
+const baseConfig = {
+ chunk_token_num: 128,
+ layout_recognize: 'DeepDOC',
+ delimiter: '\n',
+ metadata: { enabled: true, metadata: [], built_in_metadata: [] },
+ enable_children: true,
+ children_delimiter: '!?;',
+ use_parent_child: true,
+ graphrag: { use_graphrag: false },
+ raptor: { use_raptor: false },
+};
+
+describe('normalizeParserConfig backend awareness', () => {
+ beforeEach(() => {
+ mockIsGoBackend = true;
+ });
+
+ it('scopes metadata and parent_child onto component nodes and drops every flat key for the Go backend', () => {
+ const out = normalizeParserConfig({
+ ...baseConfig,
+ }) as Record<string, any>;
+
+ expect(out['Extractor:AutoExtractDefault']).toEqual({
+ metadata: { enabled: true, metadata: [], built_in_metadata: [] },
+ });
+ expect(out['GeneralChunker:SixApplesFall']).toEqual({
+ chunk_token_size: 128,
+ parent_child: { children_delimiter: '!?;', use_parent_child: true },
+ });
+ // No flat transport keys survive for the Go backend.
+ expect(out).not.toHaveProperty('metadata');
+ expect(out).not.toHaveProperty('parent_child');
+ // The Go backend reads chunk_token_size off the chunker node, never a flat
+ // chunk_token_num, so the flat key must be gone and the value re-homed.
+ expect(out).not.toHaveProperty('chunk_token_num');
+ expect(out).not.toHaveProperty('delimiter');
+ expect(out).not.toHaveProperty('layout_recognize');
+ expect(out).not.toHaveProperty('auto_keywords');
+ // graphrag/raptor are always stripped.
+ expect(out).not.toHaveProperty('graphrag');
+ expect(out).not.toHaveProperty('raptor');
+ });
+
+ it('keeps the flat-key contract for the Python backend', () => {
+ mockIsGoBackend = false;
+ const out = normalizeParserConfig({
+ ...baseConfig,
+ }) as Record<string, any>;
+
+ expect(out.metadata).toEqual({
+ enabled: true,
+ metadata: [],
+ built_in_metadata: [],
+ });
+ expect(out.parent_child).toEqual({
+ children_delimiter: '!?;',
+ use_parent_child: true,
+ });
+ expect(out).not.toHaveProperty('Extractor:AutoExtractDefault');
+ expect(out).not.toHaveProperty('GeneralChunker:SixApplesFall');
+ });
+
+ it('does not scope anything when metadata/parent_child are absent but still re-homes chunk_token_num', () => {
+ const out = normalizeParserConfig({
+ chunk_token_num: 128,
+ layout_recognize: 'DeepDOC',
+ delimiter: '\n',
+ }) as Record<string, any>;
+
+ expect(out).not.toHaveProperty('Extractor:AutoExtractDefault');
+ expect(out).not.toHaveProperty('parent_child');
+ expect(out).not.toHaveProperty('metadata');
+ // No flat parser-level key survives for the Go backend.
+ expect(out).not.toHaveProperty('chunk_token_num');
+ expect(out).not.toHaveProperty('delimiter');
+ // The chunk size is still re-homed onto the chunker node.
+ expect(out['GeneralChunker:SixApplesFall']).toEqual({
+ chunk_token_size: 128,
+ });
+ });
+});
diff --git a/web/src/hooks/parser-config-utils.test.ts b/web/src/hooks/parser-config-utils.test.ts
new file mode 100644
index 0000000000..298bd86500
--- /dev/null
+++ b/web/src/hooks/parser-config-utils.test.ts
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { normalizeParserConfig } from './parser-config-utils';
+
+// Force the Go backend branch inside normalizeParserConfig.
+let mockIsGo = true;
+
+jest.mock('@/utils/backend-variant', () => ({
+ useIsGoBackend: () => mockIsGo,
+ pickByBackend: (variants: { go: unknown; python: unknown }) =>
+ mockIsGo ? variants.go : variants.python,
+}));
+
+const CHUNKER = 'GeneralChunker:SixApplesFall';
+const EXTRACTOR = 'Extractor:AutoExtractDefault';
+
+describe('normalizeParserConfig (Go backend)', () => {
+ beforeEach(() => {
+ mockIsGo = true;
+ });
+
+ it('drops flat parser-level keys and keeps only component-scoped keys', () => {
+ const out = normalizeParserConfig({
+ chunk_token_num: 256,
+ delimiter: '\n',
+ auto_keywords: 2,
+ layout_recognize: 'DeepDOC',
+ [CHUNKER]: { chunk_token_size: 512 },
+ }) as Record<string, unknown>;
+
+ expect(out['chunk_token_num']).toBeUndefined();
+ expect(out['delimiter']).toBeUndefined();
+ expect(out['auto_keywords']).toBeUndefined();
+ expect(out[CHUNKER]).toBeDefined();
+ });
+
+ it('re-homes chunk_token_num onto the chunker node as chunk_token_size', () => {
+ const out = normalizeParserConfig({
+ chunk_token_num: 256,
+ }) as Record<string, unknown>;
+
+ expect((out[CHUNKER] as Record<string, unknown>).chunk_token_size).toBe(
+ 256,
+ );
+ });
+
+ it('re-homes metadata onto the Extractor node', () => {
+ const metadata = { enabled: true, metadata: [] };
+ const out = normalizeParserConfig({
+ metadata,
+ }) as Record<string, unknown>;
+
+ expect((out[EXTRACTOR] as Record<string, unknown>).metadata).toEqual(
+ metadata,
+ );
+ });
+
+ it('writes parent_child with use_parent_child:true when enable_children is true', () => {
+ const out = normalizeParserConfig({
+ enable_children: true,
+ children_delimiter: '###',
+ }) as Record<string, unknown>;
+
+ const parentChild = (out[CHUNKER] as Record<string, unknown>)
+ .parent_child as Record<string, unknown>;
+ expect(parentChild.use_parent_child).toBe(true);
+ expect(parentChild.children_delimiter).toBe('###');
+ });
+
+ it('explicitly disables parent_child when enable_children is false even if a node already enabled it (F2)', () => {
+ const out = normalizeParserConfig({
+ enable_children: false,
+ [CHUNKER]: {
+ chunk_token_size: 512,
+ parent_child: { use_parent_child: true, children_delimiter: '###' },
+ },
+ }) as Record<string, unknown>;
+
+ const chunker = out[CHUNKER] as Record<string, unknown>;
+ expect(chunker.chunk_token_size).toBe(512);
+ const parentChild = chunker.parent_child as Record<string, unknown>;
+ expect(parentChild.use_parent_child).toBe(false);
+ });
+
+ it('leaves an existing parent_child untouched when enable_children is absent', () => {
+ const existing = { use_parent_child: true, children_delimiter: '###' };
+ const out = normalizeParserConfig({
+ [CHUNKER]: { chunk_token_size: 512, parent_child: existing },
+ }) as Record<string, unknown>;
+
+ const chunker = out[CHUNKER] as Record<string, unknown>;
+ expect(chunker.parent_child).toEqual(existing);
+ });
+});
diff --git a/web/src/hooks/parser-config-utils.ts b/web/src/hooks/parser-config-utils.ts
index d6f91c810b..1884887b4c 100644
--- a/web/src/hooks/parser-config-utils.ts
+++ b/web/src/hooks/parser-config-utils.ts
@@ -14,6 +14,8 @@
* limitations under the License.
*/
+import { pickByBackend } from '@/utils/backend-variant';
+
/**
* Pipeline parser configs are keyed by operator id (e.g. "Parser:xxx"), so a
* top-level key containing ":" marks the pipeline structure, which must be
@@ -41,6 +43,21 @@ const isMinerULayoutRecognize = (layoutRecognize: unknown): boolean =>
/**
* Normalizes parser configuration before it is sent to the API.
+ *
+ * The Python backend keeps the legacy flat-key contract (top-level `metadata`
+ * / `parent_child` and all parser-level keys as-is).
+ *
+ * The Go backend rejects every flat (non-component-scoped) key, so this
+ * normalizer drops all parser-level flat keys (`chunk_token_num`, `delimiter`,
+ * `auto_keywords`, `layout_recognize`, ...) and emits only component-scoped
+ * keys (those containing `:`). The two legacy flat values the Go backend still
+ * understands are re-homed onto their owning nodes:
+ * - `metadata` → `Extractor:AutoExtractDefault.metadata`
+ * - `parent_child` → `GeneralChunker:SixApplesFall.parent_child`
+ * - `chunk_token_num` → `GeneralChunker:SixApplesFall.chunk_token_size`
+ * (the built-in path reads `chunk_token_size` off the chunker node, never a
+ * flat `chunk_token_num`). Any key already containing `:` is passed through
+ * untouched.
* @param parserConfig - The parser configuration object
* @returns Processed parser config
*/
@@ -63,6 +80,7 @@ export const normalizeParserConfig = (
children_delimiter,
use_parent_child,
enable_children,
+ metadata,
...additionalParserConfig
} = parserConfig;
delete additionalParserConfig.graphrag;
@@ -74,26 +92,78 @@ export const normalizeParserConfig = (
delete additionalParserConfig[key];
}
}
- return {
- auto_keywords,
- auto_questions,
- chunk_token_num,
- delimiter,
- html4excel,
- layout_recognize,
- tag_kb_ids,
- topn_tags,
- filename_embd_weight,
- task_page_size,
- pages,
- children_delimiter,
- enable_children,
- parent_child: enable_children
+
+ // When children are enabled we forward the delimiter and the explicit
+ // use_parent_child override (defaulting to enabled). When they are explicitly
+ // disabled we must still emit a parent_child payload with use_parent_child:
+ // false, otherwise a pre-existing node's parent_child (e.g. {use_parent_child:
+ // true}) would pass through untouched and silently keep children parsing ON.
+ // When enable_children is absent we leave any existing parent_child untouched.
+ const parentChild =
+ enable_children === true
? {
children_delimiter,
- use_parent_child: use_parent_child ?? enable_children,
+ use_parent_child: use_parent_child ?? true,
}
- : undefined,
- ...additionalParserConfig,
- };
+ : enable_children === false
+ ? { use_parent_child: false }
+ : undefined;
+
+ // Python backend: keep the legacy flat-key contract unchanged.
+ if (!pickByBackend({ go: true, python: false })) {
+ return {
+ auto_keywords,
+ auto_questions,
+ chunk_token_num,
+ delimiter,
+ html4excel,
+ layout_recognize,
+ tag_kb_ids,
+ topn_tags,
+ filename_embd_weight,
+ task_page_size,
+ pages,
+ children_delimiter,
+ enable_children,
+ metadata,
+ ...(parentChild ? { parent_child: parentChild } : {}),
+ ...additionalParserConfig,
+ };
+ }
+
+ // Go backend: emit ONLY component-scoped keys. The legacy flat parser-level
+ // keys have no consumer on the Go backend (the built-in path reads component
+ // nodes, e.g. GeneralChunker:SixApplesFall.chunk_token_size), so they are
+ // dropped rather than forwarded. Already-scoped keys pass through untouched.
+ const scoped: Record<string, any> = {};
+ for (const [key, value] of Object.entries(additionalParserConfig)) {
+ if (key.includes(':')) {
+ scoped[key] = value;
+ }
+ }
+ if (chunk_token_num !== undefined && chunk_token_num !== null) {
+ scoped['GeneralChunker:SixApplesFall'] = {
+ ...(scoped['GeneralChunker:SixApplesFall'] as
+ | Record<string, any>
+ | undefined),
+ chunk_token_size: chunk_token_num,
+ };
+ }
+ if (metadata && typeof metadata === 'object') {
+ scoped['Extractor:AutoExtractDefault'] = {
+ ...(scoped['Extractor:AutoExtractDefault'] as
+ | Record<string, any>
+ | undefined),
+ metadata,
+ };
+ }
+ if (parentChild) {
+ scoped['GeneralChunker:SixApplesFall'] = {
+ ...(scoped['GeneralChunker:SixApplesFall'] as
+ | Record<string, any>
+ | undefined),
+ parent_child: parentChild,
+ };
+ }
+ return scoped;
};
diff --git a/web/src/pages/dataset/setting/go/hooks.test.tsx b/web/src/pages/dataset/setting/go/hooks.test.tsx
index 6dc1b03fca..c1fad20a98 100644
--- a/web/src/pages/dataset/setting/go/hooks.test.tsx
+++ b/web/src/pages/dataset/setting/go/hooks.test.tsx
@@ -19,7 +19,7 @@ jest.mock('@/services/knowledge-service', () => ({
const mockUseUpdateKnowledge = jest.mocked(useUpdateKnowledge);
-describe('useSaveDatasetSetting parser_config metadata lift', () => {
+describe('useSaveDatasetSetting parser_config metadata scoping', () => {
beforeEach(() => {
mockIsGoBackend = true;
mockUseUpdateKnowledge.mockReset();
@@ -37,7 +37,7 @@ describe('useSaveDatasetSetting parser_config metadata lift', () => {
return { handleSave: current.handleSave, saveKnowledgeConfiguration };
}
- it('makes the extractor metadata group authoritative over the stale top-level copy', async () => {
+ it('keeps the extractor node metadata authoritative and emits no flat metadata key', async () => {
const { handleSave, saveKnowledgeConfiguration } = setup();
await handleSave({
@@ -60,17 +60,20 @@ describe('useSaveDatasetSetting parser_config metadata lift', () => {
} as never);
const payload = saveKnowledgeConfiguration.mock.calls[0][0];
- expect(payload.parser_config.metadata).toEqual({
+ // The Go backend requires component-scoped keys: no flat `metadata` transport
+ // key is emitted, and the extractor node's own metadata wins over any stale
+ // top-level copy present in the form payload.
+ expect(payload.parser_config).not.toHaveProperty('metadata');
+ expect(
+ payload.parser_config['Extractor:AutoExtractDefault'].metadata,
+ ).toEqual({
enabled: true,
metadata: [],
built_in_metadata: [{ key: 'update_time', type: 'time' }],
});
- expect(
- payload.parser_config['Extractor:AutoExtractDefault'].metadata.enabled,
- ).toBe(true);
});
- it('leaves the top-level object untouched when the pipeline has no extractor', async () => {
+ it('does not emit a flat metadata key when the pipeline has no extractor', async () => {
const { handleSave, saveKnowledgeConfiguration } = setup();
await handleSave({
@@ -86,10 +89,9 @@ describe('useSaveDatasetSetting parser_config metadata lift', () => {
} as never);
const payload = saveKnowledgeConfiguration.mock.calls[0][0];
- expect(payload.parser_config.metadata).toEqual({
- enabled: true,
- metadata: [],
- built_in_metadata: [],
+ expect(payload.parser_config).not.toHaveProperty('metadata');
+ expect(payload.parser_config['Tokenizer:SomeNode']).toEqual({
+ fields: 'text',
});
});
});
diff --git a/web/src/pages/dataset/setting/go/hooks.ts b/web/src/pages/dataset/setting/go/hooks.ts
index d93dea95fb..827a00ad76 100644
--- a/web/src/pages/dataset/setting/go/hooks.ts
+++ b/web/src/pages/dataset/setting/go/hooks.ts
@@ -1,4 +1,3 @@
-import { Operator } from '@/constants/agent';
import { ParseType } from '@/constants/knowledge';
import {
useFetchDatasetPipelineConfiguration,
@@ -85,33 +84,24 @@ export const useSaveDatasetSetting = () => {
async (values: z.infer<typeof formSchema>) => {
const payload = { ...values };
- // Apply forward transforms to parser_config if in pipeline mode
+ // Apply forward transforms to parser_config if in pipeline mode. Each node
+ // (e.g. Extractor:AutoExtractDefault) already carries its own modular
+ // metadata, so no flat top-level `metadata`/`parent_child` key is emitted
+ // — the Go backend requires component-scoped keys. Skip any non-operator
+ // (flat, key-without-":") entries that may linger from a stale form copy.
if (payload.parser_config) {
const transformedConfig: Record<string, any> = {};
- let extractorMetadataGroup: Record<string, any> | undefined;
for (const [operatorId, config] of Object.entries(
payload.parser_config,
)) {
+ if (!operatorId.includes(':')) {
+ continue;
+ }
const operatorType = getOperatorType(operatorId);
- const transformed = transformFormConfigToApi(
+ transformedConfig[operatorId] = transformFormConfigToApi(
operatorType,
config as Record<string, any>,
);
- transformedConfig[operatorId] = transformed;
- if (
- operatorType === Operator.Extractor &&
- extractorMetadataGroup === undefined
- ) {
- extractorMetadataGroup = transformed?.metadata;
- }
- }
- // parser_config.metadata is the dataset-level object the backend
- // preserves and re-scopes into every Extractor node. The extractor
- // tab is the only place the user edits it, so lift its metadata
- // group to the top level — otherwise the stale copy loaded with the
- // form would silently revert the toggle the user just set.
- if (extractorMetadataGroup) {
- transformedConfig.metadata = extractorMetadataGroup;
}
payload.parser_config = transformedConfig;
}