Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(ingestion): unify auto-metadata on modular metadata and hard-delete legacy flats (BuiltInMetadata carried, not LLM) (#18511)

jay777212022efa9fcef
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go
index 63b11a7b43..d2cdcf04cd 100644
--- a/internal/ingestion/component/extractor.go
+++ b/internal/ingestion/component/extractor.go
@@ -265,6 +265,7 @@ func NewExtractorComponent(params map[string]any) (runtime.Component, error) {
if v, ok := metaRaw["metadata"]; ok {
p.Metadata.Metadata = parseMetadataFieldDefs(v)
}
+ // BuiltInMetadata is carried for persistence/replay; LLM extraction uses Metadata only.
if v, ok := metaRaw["built_in_metadata"]; ok {
p.Metadata.BuiltInMetadata = parseMetadataFieldDefs(v)
}
diff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.go
index e5aff9f85b..7aca970979 100644
--- a/internal/ingestion/component/extractor_test.go
+++ b/internal/ingestion/component/extractor_test.go
@@ -1491,6 +1491,9 @@ func TestExtractorModularMetadataConfig(t *testing.T) {
if ext.Param.Metadata.Metadata[0].Key != "author" || ext.Param.Metadata.Metadata[0].Description != "The author name" || len(ext.Param.Metadata.Metadata[0].Enum) != 2 {
t.Errorf("Metadata field 0 mismatch: %+v", ext.Param.Metadata.Metadata[0])
}
+ if ext.Param.Metadata.Metadata[1].Key != "year" {
+ t.Errorf("Metadata field 1 mismatch: %+v", ext.Param.Metadata.Metadata[1])
+ }
if len(ext.Param.Metadata.BuiltInMetadata) != 1 || ext.Param.Metadata.BuiltInMetadata[0].Key != "file_name" {
t.Errorf("BuiltInMetadata mismatch: %+v", ext.Param.Metadata.BuiltInMetadata)
}
@@ -1859,3 +1862,72 @@ func TestExtractor_ParseMetadataFieldDefs_MapSlice(t *testing.T) {
t.Errorf("parseMetadataFieldDefs failed for []common.MetadataFieldDef: %#v", directDefs)
}
}
+
+func TestExtractorBuiltInDoesNotCallLLM(t *testing.T) {
+ // 1b39355c regressed by making runEnableMetadata fire when only built_in_metadata was configured.
+ // With the modular shape, BuiltInMetadata must never trigger an LLM call; it is applied by the finalizer.
+ stub := withStubChatInvoker(t)
+ params := map[string]any{
+ "llm_id": "llm-1",
+ "metadata": map[string]any{
+ "enabled": true,
+ "built_in_metadata": []any{
+ map[string]any{"key": "file_name", "type": "string"},
+ map[string]any{"key": "update_time", "type": "time"},
+ },
+ "metadata": []any{},
+ },
+ }
+ comp, err := NewExtractorComponent(params)
+ if err != nil {
+ t.Fatalf("NewExtractorComponent: %v", err)
+ }
+ in := map[string]any{
+ "chunks": []map[string]any{{"text": "hello world"}},
+ }
+ out, err := comp.Invoke(t.Context(), nil, in)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if stub.calls.Load() != 0 {
+ t.Fatalf("built_in-only must not call LLM, got %d calls, requests=%v", stub.calls.Load(), stub.requests)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ if len(chunks) != 1 {
+ t.Fatalf("expected 1 chunk, got %v", out)
+ }
+ if _, ok := chunks[0]["metadata"]; ok {
+ t.Fatalf("built_in must not produce chunk metadata, got %v", chunks[0]["metadata"])
+ }
+}
+
+func TestExtractorEnabledFalseDoesNotCallLLM(t *testing.T) {
+ stub := withStubChatInvoker(t)
+ params := map[string]any{
+ "llm_id": "llm-1",
+ "metadata": map[string]any{
+ "enabled": false,
+ "metadata": []any{
+ map[string]any{"key": "author", "type": "string"},
+ },
+ "built_in_metadata": []any{
+ map[string]any{"key": "file_name", "type": "string"},
+ },
+ },
+ }
+ comp, err := NewExtractorComponent(params)
+ if err != nil {
+ t.Fatalf("NewExtractorComponent: %v", err)
+ }
+ in := map[string]any{"chunks": []map[string]any{{"text": "hello"}}}
+ out, err := comp.Invoke(t.Context(), nil, in)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if stub.calls.Load() != 0 {
+ t.Fatalf("enabled=false must not call LLM, got %d", stub.calls.Load())
+ }
+ if chunks, _ := out["chunks"].([]map[string]any); len(chunks) != 1 {
+ t.Fatalf("expected 1 chunk, got %v", out)
+ }
+}
diff --git a/internal/ingestion/component/schema/extractor.go b/internal/ingestion/component/schema/extractor.go
index bb5dc4e08d..7357b0aaa8 100644
--- a/internal/ingestion/component/schema/extractor.go
+++ b/internal/ingestion/component/schema/extractor.go
@@ -99,6 +99,8 @@ type SummaryExtractConfig struct {
}
// MetadataExtractConfig configures structured metadata extraction.
+// BuiltInMetadata is carried for persistence/replay; it is NOT LLM-extracted.
+// Deterministic file_name/update_time is applied via PipelineResult -> doc_state.applyBuiltInMetadata.
type MetadataExtractConfig struct {
Enabled bool `json:"enabled"`
Metadata []common.MetadataFieldDef `json:"metadata,omitempty"`
diff --git a/internal/ingestion/service/doc_state_test.go b/internal/ingestion/service/doc_state_test.go
index 86ad8c4be4..f47ebc1c5c 100644
--- a/internal/ingestion/service/doc_state_test.go
+++ b/internal/ingestion/service/doc_state_test.go
@@ -255,3 +255,30 @@ func TestDocStateUpdater_BuiltInMetadataEmptyConfig(t *testing.T) {
t.Errorf("empty built-in config must be a no-op, got %v", svc.metaData)
}
}
+
+func TestDocStateUpdater_BuiltInNotWrittenWhenEnabledFalse(t *testing.T) {
+ svc := &stubDocStateSvc{metaData: map[string]any{}}
+ u := &docStateUpdater{docSvc: svc}
+ ctx := t.Context()
+ u.apply(ctx, &taskpkg.PipelineResult{
+ DocID: "doc-1",
+ DocName: "report.pdf",
+ AutoMetadataEnabled: false,
+ BuiltInMetadataConfig: []any{
+ map[string]any{"key": "file_name", "type": "string"},
+ map[string]any{"key": "update_time", "type": "time"},
+ },
+ Metadata: map[string]any{"author": "Alice"},
+ ChunkCount: 1,
+ TokenConsumption: 1,
+ })
+ if _, ok := svc.metaData["file_name"]; ok {
+ t.Fatalf("built_in must not be written when enabled=false, got %v", svc.metaData)
+ }
+ if _, ok := svc.metaData["update_time"]; ok {
+ t.Fatalf("built_in must not be written when enabled=false, got %v", svc.metaData)
+ }
+ if svc.metaData["author"] != "Alice" {
+ t.Fatalf("custom metadata should still be written even when built_in is off, got %v", svc.metaData)
+ }
+}
diff --git a/internal/ingestion/service/e2e_metadata_chain_test.go b/internal/ingestion/service/e2e_metadata_chain_test.go
new file mode 100644
index 0000000000..22c3d668e2
--- /dev/null
+++ b/internal/ingestion/service/e2e_metadata_chain_test.go
@@ -0,0 +1,73 @@
+package service
+
+import (
+ "context"
+ "testing"
+
+ "ragflow/internal/entity"
+ taskpkg "ragflow/internal/ingestion/task"
+ "ragflow/internal/service"
+)
+
+// TestE2E_EnabledFalseDoesNotWriteBuiltIn verifies the full chain's gate:
+// dataset metadata enabled=false -> pipeline AutoMetadataEnabled=false -> doc_state does not write built_in.
+func TestE2E_EnabledFalseDoesNotWriteBuiltIn(t *testing.T) {
+ // Directly exercise the doc_state gate which is the last step of the chain.
+ // The earlier steps (dataset UpdateMetadataConfig -> ApplyComponentScoped -> pipeline builtInMetadataFromParserConfig)
+ // are covered by unit tests; this E2E asserts the gate is not bypassed.
+ svc := &stubDocStateSvc{metaData: map[string]any{"existing": "keep"}}
+ u := &docStateUpdater{docSvc: svc}
+ ctx := t.Context()
+ u.apply(ctx, &taskpkg.PipelineResult{
+ DocID: "doc-e2e",
+ DocName: "a.pdf",
+ AutoMetadataEnabled: false,
+ BuiltInMetadataConfig: []any{map[string]any{"key": "file_name", "type": "string"}, map[string]any{"key": "update_time", "type": "time"}},
+ Metadata: map[string]any{"author": "Bob"},
+ ChunkCount: 1,
+ })
+ if _, ok := svc.metaData["file_name"]; ok {
+ t.Fatalf("must not write file_name when enabled=false, got %v", svc.metaData)
+ }
+ if svc.metaData["author"] != "Bob" {
+ t.Fatalf("custom metadata should still be written, got %v", svc.metaData)
+ }
+ if svc.metaData["existing"] != "keep" {
+ t.Fatalf("existing must be kept, got %v", svc.metaData)
+ }
+}
+
+func TestE2E_ApplyComponentScopedThenPipelineThenDocState(t *testing.T) {
+ // ParserConfig as it would be after DatasetService.UpdateMetadataConfig
+ parserConfig := entity.JSONMap{
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{map[string]any{"key": "author", "type": "string"}},
+ "built_in_metadata": []any{map[string]any{"key": "file_name", "type": "string"}, map[string]any{"key": "update_time", "type": "time"}},
+ },
+ "Extractor:AutoExtractDefault": map[string]any{},
+ }
+ // Scope to node
+ got := service.ApplyComponentScopedParserConfig(parserConfig, "llm-1")
+ node, _ := got["Extractor:AutoExtractDefault"].(map[string]any)
+ meta, _ := node["metadata"].(map[string]any)
+ if enabled, _ := meta["enabled"].(bool); !enabled {
+ t.Fatalf("scoped enabled must be true, got %v", meta["enabled"])
+ }
+ if _, ok := meta["built_in_metadata"]; !ok {
+ t.Fatalf("scoped built_in missing, got %v", meta)
+ }
+ // Pipeline would read built_in -> doc_state would write it
+ svc := &stubDocStateSvc{metaData: map[string]any{}}
+ u := &docStateUpdater{docSvc: svc}
+ u.apply(context.Background(), &taskpkg.PipelineResult{
+ DocID: "doc-1", KbID: "kb-1", DocName: "report.pdf",
+ AutoMetadataEnabled: true,
+ BuiltInMetadataConfig: []any{map[string]any{"key": "file_name", "type": "string"}},
+ Metadata: map[string]any{"author": "Alice"},
+ ChunkCount: 1,
+ })
+ if svc.metaData["file_name"] != "report.pdf" {
+ t.Fatalf("file_name not written, got %v", svc.metaData)
+ }
+}
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index 3d224c8121..40a20cebe5 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -303,9 +303,10 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
}
// builtInMetadataFromParserConfig extracts the built-in metadata config
-// (update_time / file_name) and whether auto-metadata is enabled. Supports
-// modular metadata_config sub-objects as well as legacy flat fields on
-// Extractor nodes and top-level parser_config.
+// (update_time / file_name) and whether auto-metadata is enabled from the
+// component-scoped Extractor node's modular metadata config. Legacy flat
+// fields (enable_metadata / metadata_config / built_in_metadata at either the
+// top level or on the node) are intentionally not supported.
func builtInMetadataFromParserConfig(parserConfig entity.JSONMap) ([]any, bool) {
var extractorKeys []string
for k := range parserConfig {
@@ -320,37 +321,29 @@ func builtInMetadataFromParserConfig(parserConfig entity.JSONMap) ([]any, bool)
nodeRaw := parserConfig[k]
if node, ok := nodeRaw.(map[string]any); ok {
if metaObj, ok := node["metadata"].(map[string]any); ok {
- enabled := parserConfigBool(metaObj["enabled"])
- if arr, ok := metaObj["built_in_metadata"].([]any); ok && len(arr) > 0 {
- return arr, enabled
- }
- }
- if metaConf, ok := node["metadata_config"].(map[string]any); ok {
- enabled := parserConfigBool(metaConf["enabled"])
- if arr, ok := metaConf["built_in_metadata"].([]any); ok && len(arr) > 0 {
- return arr, enabled
- }
- }
- enabled := parserConfigBool(node["enable_metadata"])
- if arr, ok := node["built_in_metadata"].([]any); ok && len(arr) > 0 {
- return arr, enabled
+ arr := metadataFieldSlice(metaObj["built_in_metadata"])
+ return arr, parserConfigBool(metaObj["enabled"])
}
}
}
- if metaObj, ok := parserConfig["metadata"].(map[string]any); ok {
- if arr, ok := metaObj["built_in_metadata"].([]any); ok && len(arr) > 0 {
- return arr, true
- }
- }
- if metaConf, ok := parserConfig["metadata_config"].(map[string]any); ok {
- if arr, ok := metaConf["built_in_metadata"].([]any); ok && len(arr) > 0 {
- return arr, true
+ return nil, false
+}
+
+// metadataFieldSlice normalizes a built_in_metadata / metadata value that may
+// arrive as []interface{} (DB round-trip) or []map[string]interface{} (in-memory
+// construction) into a []any.
+func metadataFieldSlice(value any) []any {
+ if list, ok := value.([]any); ok {
+ return list
+ }
+ if list, ok := value.([]map[string]any); ok {
+ out := make([]any, 0, len(list))
+ for _, item := range list {
+ out = append(out, item)
}
+ return out
}
- if arr, ok := parserConfig["built_in_metadata"].([]any); ok && len(arr) > 0 {
- return arr, true
- }
- return nil, false
+ return nil
}
// parserConfigBool coerces a parser_config boolean-like value (bool / number)
diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.go
index b9dbb572a6..c6bc388c62 100644
--- a/internal/ingestion/task/pipeline_executor_defaults_test.go
+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go
@@ -186,38 +186,24 @@ func TestBuildComponentParams_GoldenCoversAllTemplates(t *testing.T) {
func TestBuiltInMetadataFromParserConfig_ExtractorNodeParams(t *testing.T) {
pc := entity.JSONMap{
"Extractor:AutoExtractDefault": map[string]any{
- "enable_metadata": 1,
- "built_in_metadata": []any{
- map[string]any{"key": "update_time", "type": "time"},
- map[string]any{"key": "file_name", "type": "string"},
+ "metadata": map[string]any{
+ "enabled": true,
+ "built_in_metadata": []any{
+ map[string]any{"key": "update_time", "type": "time"},
+ map[string]any{"key": "file_name", "type": "string"},
+ },
},
},
}
cfg, enabled := builtInMetadataFromParserConfig(pc)
if !enabled {
- t.Fatal("auto metadata should be enabled from extractor node params")
+ t.Fatal("auto metadata should be enabled from extractor node modular config")
}
if len(cfg) != 2 {
t.Fatalf("built-in config len = %d, want 2", len(cfg))
}
}
-func TestBuiltInMetadataFromParserConfig_TopLevelFallback(t *testing.T) {
- pc := entity.JSONMap{
- "enable_metadata": true,
- "built_in_metadata": []any{
- map[string]any{"key": "file_name", "type": "string"},
- },
- }
- cfg, enabled := builtInMetadataFromParserConfig(pc)
- if !enabled {
- t.Fatal("auto metadata should be enabled from top-level config")
- }
- if len(cfg) != 1 {
- t.Fatalf("built-in config len = %d, want 1", len(cfg))
- }
-}
-
func TestBuiltInMetadataFromParserConfig_None(t *testing.T) {
cfg, enabled := builtInMetadataFromParserConfig(entity.JSONMap{})
if enabled || len(cfg) != 0 {
@@ -241,13 +227,15 @@ func TestBuiltInMetadataFromParserConfig_MultiExtractorDeterministic(t *testing.
// Extractor:A must deterministically win regardless of Go map iteration order.
pc := entity.JSONMap{
"Extractor:B": map[string]any{
- "enable_metadata": 1,
- "built_in_metadata": []any{
- map[string]any{"key": "file_name", "type": "string"},
+ "metadata": map[string]any{
+ "enabled": true,
+ "built_in_metadata": []any{
+ map[string]any{"key": "file_name", "type": "string"},
+ },
},
},
"Extractor:A": map[string]any{
- "metadata_config": map[string]any{
+ "metadata": map[string]any{
"enabled": true,
"built_in_metadata": []any{
map[string]any{"key": "update_time", "type": "time"},
@@ -270,3 +258,57 @@ func TestBuiltInMetadataFromParserConfig_MultiExtractorDeterministic(t *testing.
}
}
}
+
+func TestBuiltInMetadataFromParserConfig_FirstMetadataNodeWins(t *testing.T) {
+ // The alphabetically-first Extractor node that declares metadata is the
+ // authoritative source, even when its built_in_metadata is empty.
+ pc := entity.JSONMap{
+ "Extractor:A": map[string]any{
+ "metadata": map[string]any{
+ "enabled": true,
+ "built_in_metadata": []any{},
+ },
+ },
+ "Extractor:B": map[string]any{
+ "metadata": map[string]any{
+ "enabled": true,
+ "built_in_metadata": []any{
+ map[string]any{"key": "file_name", "type": "string"},
+ },
+ },
+ },
+ }
+
+ cfg, enabled := builtInMetadataFromParserConfig(pc)
+ if !enabled {
+ t.Fatalf("expected enabled=true from first metadata node, got false")
+ }
+ if len(cfg) != 0 {
+ t.Fatalf("expected empty built-in config from first metadata node, got %#v", cfg)
+ }
+}
+
+func TestBuiltInMetadataFromParserConfig_MapSliceBuiltIn(t *testing.T) {
+ pc := entity.JSONMap{
+ "Extractor:A": map[string]any{
+ "metadata": map[string]any{
+ "enabled": true,
+ "built_in_metadata": []map[string]any{
+ {"key": "update_time", "type": "time"},
+ },
+ },
+ },
+ }
+
+ cfg, enabled := builtInMetadataFromParserConfig(pc)
+ if !enabled {
+ t.Fatalf("expected enabled=true, got false")
+ }
+ if len(cfg) != 1 {
+ t.Fatalf("expected 1 built-in item from []map[string]any, got %d", len(cfg))
+ }
+ item, ok := cfg[0].(map[string]any)
+ if !ok || item["key"] != "update_time" {
+ t.Fatalf("expected update_time item, got %#v", cfg[0])
+ }
+}
diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.go
index 62a50f0bc4..80222d12f6 100644
--- a/internal/ingestion/task/pipeline_executor_test.go
+++ b/internal/ingestion/task/pipeline_executor_test.go
@@ -575,25 +575,3 @@ func TestCountOriginalChunkIDs(t *testing.T) {
t.Fatalf("compiler products: got %d, want 2", n)
}
}
-
-func TestBuiltInMetadata_ZeroExtractorFallback(t *testing.T) {
- // Scenario: User constructed a pipeline with 0 Extractor components,
- // but dataset parser_config has top-level built_in_metadata.
- parserConfig := map[string]any{
- "Parser:HipSignsRhyme": map[string]any{
- "setups": map[string]any{},
- },
- "built_in_metadata": []any{
- map[string]any{"key": "file_name", "type": "string"},
- map[string]any{"key": "update_time", "type": "time"},
- },
- }
-
- arr, enabled := builtInMetadataFromParserConfig(parserConfig)
- if !enabled {
- t.Errorf("expected builtInMetadataFromParserConfig enabled=true for zero-extractor pipeline with top-level built_in_metadata")
- }
- if len(arr) != 2 {
- t.Errorf("expected 2 built-in metadata items, got %d", len(arr))
- }
-}
diff --git a/internal/service/component_scoped_parser_config.go b/internal/service/component_scoped_parser_config.go
index cf78bc993f..4929ecc1cd 100644
--- a/internal/service/component_scoped_parser_config.go
+++ b/internal/service/component_scoped_parser_config.go
@@ -6,9 +6,15 @@ import (
"ragflow/internal/entity"
)
-// ApplyComponentScopedParserConfig fills dataset-scoped component params onto a
-// parser_config map without reintroducing top-level flat fields. It mutates the
-// provided map in place and returns it for convenience.
+// ApplyComponentScopedParserConfig scopes a dataset-level modular metadata
+// config (parserConfig["metadata"] = {enabled, metadata, built_in_metadata})
+// onto every Extractor node and strips the legacy flat metadata fields
+// (enable_metadata / metadata_config / built_in_metadata / fields) both at the
+// top level and on nodes. Legacy flat metadata forms are intentionally not
+// supported. When a dataset-level modular metadata config is present it is
+// authoritative and replaces each Extractor node's metadata; when absent, the
+// node's own modular metadata is preserved. It mutates the provided map in
+// place and returns it for convenience.
func ApplyComponentScopedParserConfig(
parserConfig entity.JSONMap,
llmID string,
@@ -17,25 +23,16 @@ func ApplyComponentScopedParserConfig(
parserConfig = entity.JSONMap{}
}
- enableMetadata := parserConfigTruthy(parserConfig["enable_metadata"])
- if !enableMetadata {
- if metaMap, ok := parserConfig["metadata"].(map[string]any); ok {
- enableMetadata = parserConfigTruthy(metaMap["enabled"])
- }
- }
- if !enableMetadata {
- enableMetadata = parserConfigTruthy(parserConfig["enabled"])
+ var datasetMeta map[string]any
+ if mm, ok := parserConfig["metadata"].(map[string]any); ok {
+ datasetMeta = cloneJSONMap(mm)
}
- metadataFields := extractFieldList(parserConfig, "fields", "metadata")
- builtInMetadataFields := extractFieldList(parserConfig, "built_in_metadata", "")
-
for cpnID, raw := range parserConfig {
params, ok := raw.(map[string]any)
if !ok {
continue
}
-
cpnLower := strings.ToLower(cpnID)
switch {
case strings.HasPrefix(cpnLower, "extractor:") || strings.HasPrefix(cpnLower, "extractor_"):
@@ -43,45 +40,20 @@ func ApplyComponentScopedParserConfig(
params["llm_id"] = llmID
}
delete(params, "enable_metadata")
-
- hasTopLevelFields := len(metadataFields) > 0 || len(builtInMetadataFields) > 0
-
- if hasTopLevelFields {
+ delete(params, "metadata_config")
+ delete(params, "built_in_metadata")
+ delete(params, "fields")
+ if datasetMeta != nil {
+ params["metadata"] = cloneJSONMap(datasetMeta)
+ continue
+ }
+ if _, ok := params["metadata"].(map[string]any); !ok {
params["metadata"] = map[string]any{
- "enabled": enableMetadata,
- "metadata": metadataFields,
- "built_in_metadata": builtInMetadataFields,
- }
- } else {
- nodeCustom := extractFieldList(params, "fields", "metadata")
- nodeBuiltIn := extractFieldList(params, "built_in_metadata", "")
- nodeEnabled := false
-
- if metaMap, ok := params["metadata"].(map[string]any); ok && metaMap["enabled"] != nil {
- nodeEnabled = parserConfigTruthy(metaMap["enabled"])
- } else if mcMap, ok := params["metadata_config"].(map[string]any); ok && mcMap["enabled"] != nil {
- nodeEnabled = parserConfigTruthy(mcMap["enabled"])
- } else if params["enable_metadata"] != nil {
- nodeEnabled = parserConfigTruthy(params["enable_metadata"])
- } else if enableMetadata {
- nodeEnabled = true
- }
-
- if len(nodeCustom) > 0 || len(nodeBuiltIn) > 0 || nodeEnabled {
- params["metadata"] = map[string]any{
- "enabled": nodeEnabled,
- "metadata": nodeCustom,
- "built_in_metadata": nodeBuiltIn,
- }
- } else {
- params["metadata"] = map[string]any{
- "enabled": false,
- "metadata": []any{},
- "built_in_metadata": []any{},
- }
+ "enabled": false,
+ "metadata": []any{},
+ "built_in_metadata": []any{},
}
}
- delete(params, "metadata_config")
case strings.HasPrefix(cpnLower, "compiler:") || strings.HasPrefix(cpnLower, "compiler_"):
if value, _ := params["llm_id"].(string); strings.TrimSpace(value) == "" && strings.TrimSpace(llmID) != "" {
params["llm_id"] = llmID
@@ -89,145 +61,23 @@ func ApplyComponentScopedParserConfig(
}
}
- return parserConfig
-}
-
-func mergeMetadataFields(parserConfig entity.JSONMap) []any {
- custom := extractFieldList(parserConfig, "fields", "metadata")
- builtIn := extractFieldList(parserConfig, "built_in_metadata", "")
- out := make([]any, 0, len(custom)+len(builtIn))
- out = append(out, custom...)
- out = append(out, builtIn...)
- return out
-}
-
-func extractFieldList(parserConfig entity.JSONMap, primaryKey, fallbackKey string) []any {
- if parserConfig == nil {
- return []any{}
- }
- if raw, ok := parserConfig[primaryKey]; ok && raw != nil {
- if slice := extractValidFields(raw); len(slice) > 0 || primaryKey == "fields" {
- return slice
- }
+ // Strip legacy top-level flat metadata fields; keep the modular object
+ // under "metadata" when present, otherwise drop the key entirely.
+ delete(parserConfig, "enable_metadata")
+ delete(parserConfig, "metadata_config")
+ delete(parserConfig, "built_in_metadata")
+ delete(parserConfig, "fields")
+ if datasetMeta == nil {
+ delete(parserConfig, "metadata")
}
- if fallbackKey != "" {
- if raw, ok := parserConfig[fallbackKey]; ok && raw != nil {
- if slice := extractValidFields(raw); len(slice) > 0 {
- return slice
- }
- }
- }
- for _, subKey := range []string{"metadata", "metadata_config"} {
- if metaMap, ok := parserConfig[subKey].(map[string]any); ok {
- if raw, ok := metaMap[primaryKey]; ok && raw != nil {
- if slice := extractValidFields(raw); len(slice) > 0 {
- return slice
- }
- }
- if fallbackKey != "" {
- if raw, ok := metaMap[fallbackKey]; ok && raw != nil {
- if slice := extractValidFields(raw); len(slice) > 0 {
- return slice
- }
- }
- }
- }
- }
- return []any{}
-}
-func extractValidFields(value any) []any {
- items := anySlice(value)
- if items == nil {
- return []any{}
- }
- out := make([]any, 0, len(items))
- for _, item := range items {
- field, ok := item.(map[string]any)
- if !ok {
- continue
- }
- name, _ := field["key"].(string)
- if strings.TrimSpace(name) == "" {
- continue
- }
- out = append(out, field)
- }
- return out
-}
-
-func anySlice(value any) []any {
- switch typed := value.(type) {
- case []any:
- return typed
- case []map[string]any:
- out := make([]any, 0, len(typed))
- for _, item := range typed {
- out = append(out, item)
- }
- return out
- case []map[string]string:
- out := make([]any, 0, len(typed))
- for _, item := range typed {
- m := make(map[string]any, len(item))
- for k, v := range item {
- m[k] = v
- }
- out = append(out, m)
- }
- return out
- case []MetadataConfigField:
- out := make([]any, 0, len(typed))
- for _, item := range typed {
- out = append(out, map[string]any{
- "key": item.Key,
- "type": item.Type,
- "description": item.Description,
- "enum": item.Enum,
- })
- }
- return out
- default:
- return nil
- }
-}
-
-func hasMetadataConfigShape(parserConfig entity.JSONMap) bool {
- if parserConfig == nil {
- return false
- }
- if _, ok := parserConfig["enable_metadata"]; ok {
- return true
- }
- if _, ok := parserConfig["enabled"]; ok {
- return true
- }
- for _, key := range []string{"fields", "metadata", "built_in_metadata"} {
- if anySlice(parserConfig[key]) != nil {
- return true
- }
- }
- if _, ok := parserConfig["metadata"].(map[string]any); ok {
- return true
- }
- return false
+ return parserConfig
}
-func parserConfigTruthy(value any) bool {
- switch typed := value.(type) {
- case bool:
- return typed
- case string:
- switch typed {
- case "true", "True", "TRUE", "1":
- return true
- }
- case float64:
- return typed > 0
- case int:
- return typed > 0
- case int64:
- return typed > 0
+func cloneJSONMap(in map[string]any) map[string]any {
+ out := make(map[string]any, len(in))
+ for k, v := range in {
+ out[k] = v
}
- return false
+ return out
}
diff --git a/internal/service/component_scoped_parser_config_test.go b/internal/service/component_scoped_parser_config_test.go
index 0ffb3a6248..e2102e1165 100644
--- a/internal/service/component_scoped_parser_config_test.go
+++ b/internal/service/component_scoped_parser_config_test.go
@@ -7,31 +7,27 @@ import (
"ragflow/internal/entity"
)
-func TestApplyComponentScopedParserConfig_SyncsExtractorAndCompiler(t *testing.T) {
+func TestApplyComponentScopedParserConfig_ScopesDatasetMetadata(t *testing.T) {
parserConfig := entity.JSONMap{
- "enable_metadata": true,
- "metadata": []any{
- map[string]any{"key": "author", "type": "string"},
- },
- "built_in_metadata": []any{
- map[string]any{"key": "document_name", "type": "string"},
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{"key": "author", "type": "string"},
+ },
+ "built_in_metadata": []any{
+ map[string]any{"key": "document_name", "type": "string"},
+ },
},
"Extractor:AutoExtractDefault": map[string]any{},
"Compiler:KnownSwiftLions": map[string]any{},
}
- got := ApplyComponentScopedParserConfig(
- parserConfig,
- "llm-default",
- )
+ got := ApplyComponentScopedParserConfig(parserConfig, "llm-default")
extractor := got["Extractor:AutoExtractDefault"].(map[string]any)
if extractor["llm_id"] != "llm-default" {
t.Fatalf("extractor llm_id = %#v, want llm-default", extractor["llm_id"])
}
- if _, ok := extractor["enable_metadata"]; ok {
- t.Fatalf("extractor enable_metadata should not be present, got %#v", extractor["enable_metadata"])
- }
wantMetadata := map[string]any{
"enabled": true,
"metadata": []any{
@@ -49,189 +45,124 @@ func TestApplyComponentScopedParserConfig_SyncsExtractorAndCompiler(t *testing.T
if compiler["llm_id"] != "llm-default" {
t.Fatalf("compiler llm_id = %#v, want llm-default", compiler["llm_id"])
}
- if _, ok := compiler["embedding_model"]; ok {
- t.Fatalf("compiler embedding_model = %#v, want absent", compiler["embedding_model"])
- }
- if _, ok := compiler["tenant_id"]; ok {
- t.Fatalf("compiler tenant_id = %#v, want absent", compiler["tenant_id"])
- }
- if _, ok := compiler["dataset_id"]; ok {
- t.Fatalf("compiler dataset_id = %#v, want absent", compiler["dataset_id"])
- }
-}
-
-func TestApplyComponentScopedParserConfig_PreservesExplicitExtractorLLMID(t *testing.T) {
- parserConfig := entity.JSONMap{
- "Extractor:Custom": map[string]any{
- "llm_id": "custom-llm",
- },
- }
-
- got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")
- extractor := got["Extractor:Custom"].(map[string]any)
- if extractor["llm_id"] != "custom-llm" {
- t.Fatalf("extractor llm_id = %#v, want custom-llm", extractor["llm_id"])
- }
}
-func TestApplyComponentScopedParserConfig_AcceptsTypedMetadataSlices(t *testing.T) {
+func TestApplyComponentScopedParserConfig_StripsLegacyFlatFields(t *testing.T) {
parserConfig := entity.JSONMap{
"enable_metadata": true,
- "metadata": []map[string]interface{}{
- {"key": "author", "type": "string"},
+ "built_in_metadata": []any{
+ map[string]any{"key": "document_name", "type": "string"},
},
- "built_in_metadata": []map[string]interface{}{
- {"key": "document_name", "type": "string"},
+ "fields": []any{
+ map[string]any{"key": "author", "type": "string"},
+ },
+ "Extractor:AutoExtractDefault": map[string]any{
+ "enable_metadata": 1,
+ "metadata_config": map[string]any{"enabled": true},
+ "built_in_metadata": []any{map[string]any{"key": "stale", "type": "string"}},
+ "fields": []any{map[string]any{"key": "stale", "type": "string"}},
},
- "Extractor:AutoExtractDefault": map[string]any{},
}
got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")
+ for _, key := range []string{"enable_metadata", "built_in_metadata", "fields", "metadata_config"} {
+ if _, ok := got[key]; ok {
+ t.Fatalf("top-level %s should be stripped, got %#v", key, got[key])
+ }
+ }
extractor := got["Extractor:AutoExtractDefault"].(map[string]any)
-
+ for _, key := range []string{"enable_metadata", "built_in_metadata", "fields", "metadata_config"} {
+ if _, ok := extractor[key]; ok {
+ t.Fatalf("node %s should be stripped, got %#v", key, extractor[key])
+ }
+ }
wantMetadata := map[string]any{
- "enabled": true,
- "metadata": []any{
- map[string]interface{}{"key": "author", "type": "string"},
- },
- "built_in_metadata": []any{
- map[string]interface{}{"key": "document_name", "type": "string"},
- },
+ "enabled": false,
+ "metadata": []any{},
+ "built_in_metadata": []any{},
}
if !reflect.DeepEqual(extractor["metadata"], wantMetadata) {
t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantMetadata)
}
}
-func TestApplyComponentScopedParserConfig_ClearsExtractorMetadataWhenDisabled(t *testing.T) {
+func TestApplyComponentScopedParserConfig_PreservesExplicitExtractorLLMID(t *testing.T) {
parserConfig := entity.JSONMap{
- "enable_metadata": false,
- "metadata": []map[string]interface{}{},
- "built_in_metadata": []map[string]interface{}{
- {"key": "document_name", "type": "string"},
- },
- "Extractor:AutoExtractDefault": map[string]any{
- "enable_metadata": 1,
- "metadata": map[string]any{
- "enabled": true,
- "metadata": []any{
- map[string]any{"key": "stale", "type": "string"},
- },
- },
+ "Extractor:Custom": map[string]any{
+ "llm_id": "custom-llm",
},
}
got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")
- extractor := got["Extractor:AutoExtractDefault"].(map[string]any)
-
- if _, ok := extractor["enable_metadata"]; ok {
- t.Fatalf("extractor enable_metadata should not be present, got %#v", extractor["enable_metadata"])
- }
- wantMetadata := map[string]any{
- "enabled": false,
- "metadata": []any{},
- "built_in_metadata": []any{
- map[string]interface{}{"key": "document_name", "type": "string"},
- },
- }
- if !reflect.DeepEqual(extractor["metadata"], wantMetadata) {
- t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantMetadata)
+ extractor := got["Extractor:Custom"].(map[string]any)
+ if extractor["llm_id"] != "custom-llm" {
+ t.Fatalf("extractor llm_id = %#v, want custom-llm", extractor["llm_id"])
}
}
-func TestApplyComponentScopedParserConfig_DoesNotTreatDocumentMetadataValuesAsSchema(t *testing.T) {
+func TestApplyComponentScopedParserConfig_NoDatasetMetadataDisablesNode(t *testing.T) {
parserConfig := entity.JSONMap{
- "metadata": map[string]interface{}{
- "author": "Alice",
- },
"Extractor:AutoExtractDefault": map[string]any{},
}
got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")
extractor := got["Extractor:AutoExtractDefault"].(map[string]any)
-
- wantMetadata := map[string]any{
+ want := map[string]any{
"enabled": false,
"metadata": []any{},
"built_in_metadata": []any{},
}
- if !reflect.DeepEqual(extractor["metadata"], wantMetadata) {
- t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantMetadata)
+ if !reflect.DeepEqual(extractor["metadata"], want) {
+ t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], want)
}
}
-func TestMergeMetadataFields_MergesFieldsAndBuiltIn(t *testing.T) {
+func TestApplyComponentScopedParserConfig_DatasetMetadataIsAuthoritative(t *testing.T) {
parserConfig := entity.JSONMap{
- "metadata": []any{
- map[string]any{"key": "author", "type": "string"},
- },
- "built_in_metadata": []any{
- map[string]any{"key": "document_name", "type": "string"},
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{"key": "dataset_field", "type": "string"},
+ },
+ "built_in_metadata": []any{},
},
- }
-
- got := mergeMetadataFields(parserConfig)
- want := []any{
- map[string]any{"key": "author", "type": "string"},
- map[string]any{"key": "document_name", "type": "string"},
- }
- if !reflect.DeepEqual(got, want) {
- t.Fatalf("mergeMetadataFields = %#v, want %#v", got, want)
- }
-}
-
-func TestApplyComponentScopedParserConfig_PreservesNodeOnlyMetadataOnDatasetSave(t *testing.T) {
- parserConfig := entity.JSONMap{
"Extractor:CanvasNode": map[string]any{
- "llm_id": "model-1",
"metadata": map[string]any{
- "enabled": true,
+ "enabled": false,
"metadata": []any{
- map[string]any{"key": "node_author", "type": "string"},
- },
- "built_in_metadata": []any{
- map[string]any{"key": "node_doc_title", "type": "string"},
+ map[string]any{"key": "node_field", "type": "string"},
},
+ "built_in_metadata": []any{},
},
},
}
got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")
node := got["Extractor:CanvasNode"].(map[string]any)
-
meta, ok := node["metadata"].(map[string]any)
if !ok {
t.Fatalf("expected node metadata map, got %T: %#v", node["metadata"], node["metadata"])
}
if enabled, _ := meta["enabled"].(bool); !enabled {
- t.Fatalf("expected node metadata.enabled = true, got false")
+ t.Fatalf("expected dataset metadata enabled=true, got false")
}
-
fields, ok := meta["metadata"].([]any)
- if !ok || len(fields) != 1 {
- t.Fatalf("expected 1 custom field preserved, got %#v", meta["metadata"])
- }
- if fields[0].(map[string]any)["key"] != "node_author" {
- t.Fatalf("expected node_author preserved, got %#v", fields[0])
- }
- builtIn, ok := meta["built_in_metadata"].([]any)
- if !ok || len(builtIn) != 1 {
- t.Fatalf("expected 1 built-in field preserved, got %#v", meta["built_in_metadata"])
- }
- if builtIn[0].(map[string]any)["key"] != "node_doc_title" {
- t.Fatalf("expected node_doc_title preserved, got %#v", builtIn[0])
+ if !ok || len(fields) != 1 || fields[0].(map[string]any)["key"] != "dataset_field" {
+ t.Fatalf("expected dataset_field to replace node_field, got %#v", meta["metadata"])
}
}
-func TestApplyComponentScopedParserConfig_PreservesNodeOnlyMetadataConfigShape(t *testing.T) {
+func TestApplyComponentScopedParserConfig_PreservesNodeOnlyMetadata(t *testing.T) {
parserConfig := entity.JSONMap{
- "enable_metadata": false,
"Extractor:CanvasNode": map[string]any{
"llm_id": "model-1",
- "metadata_config": map[string]any{
+ "metadata": map[string]any{
"enabled": true,
"metadata": []any{
- map[string]any{"key": "custom_field", "type": "string"},
+ map[string]any{"key": "node_author", "type": "string"},
+ },
+ "built_in_metadata": []any{
+ map[string]any{"key": "node_doc_title", "type": "string"},
},
},
},
@@ -239,10 +170,6 @@ func TestApplyComponentScopedParserConfig_PreservesNodeOnlyMetadataConfigShape(t
got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")
node := got["Extractor:CanvasNode"].(map[string]any)
-
- if _, ok := node["metadata_config"]; ok {
- t.Fatalf("metadata_config should be deleted from node, got %#v", node["metadata_config"])
- }
meta, ok := node["metadata"].(map[string]any)
if !ok {
t.Fatalf("expected node metadata map, got %T: %#v", node["metadata"], node["metadata"])
@@ -251,10 +178,11 @@ func TestApplyComponentScopedParserConfig_PreservesNodeOnlyMetadataConfigShape(t
t.Fatalf("expected node metadata.enabled = true, got false")
}
fields, ok := meta["metadata"].([]any)
- if !ok || len(fields) != 1 {
- t.Fatalf("expected 1 custom field preserved, got %#v", meta["metadata"])
+ if !ok || len(fields) != 1 || fields[0].(map[string]any)["key"] != "node_author" {
+ t.Fatalf("expected node_author preserved, got %#v", meta["metadata"])
}
- if fields[0].(map[string]any)["key"] != "custom_field" {
- t.Fatalf("expected custom_field preserved, got %#v", fields[0])
+ builtIn, ok := meta["built_in_metadata"].([]any)
+ if !ok || len(builtIn) != 1 || builtIn[0].(map[string]any)["key"] != "node_doc_title" {
+ t.Fatalf("expected node_doc_title preserved, got %#v", meta["built_in_metadata"])
}
}
diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.go
index 28aac5e863..4dcc22b4f2 100644
--- a/internal/service/dataset/helpers.go
+++ b/internal/service/dataset/helpers.go
@@ -253,19 +253,20 @@ func preserveDatasetParserConfigMetadata(next, existing entity.JSONMap, incoming
if next == nil {
next = entity.JSONMap{}
}
- for _, key := range []string{"fields", "metadata", "built_in_metadata", "enable_metadata"} {
- if incoming != nil {
- if value, ok := incoming[key]; ok {
- next[key] = value
- continue
- }
+ var mm map[string]any
+ if incoming != nil {
+ if v, ok := incoming["metadata"].(map[string]any); ok {
+ mm = v
}
- if existing != nil {
- if value, ok := existing[key]; ok {
- next[key] = value
- }
+ }
+ if mm == nil && existing != nil {
+ if v, ok := existing["metadata"].(map[string]any); ok {
+ mm = v
}
}
+ if mm != nil {
+ next["metadata"] = mm
+ }
return next
}
diff --git a/internal/service/dataset/helpers_test.go b/internal/service/dataset/helpers_test.go
index def8a42086..c714e4c53a 100644
--- a/internal/service/dataset/helpers_test.go
+++ b/internal/service/dataset/helpers_test.go
@@ -4,6 +4,7 @@ import (
"strings"
"testing"
+ "ragflow/internal/entity"
"ragflow/internal/service"
)
@@ -395,3 +396,57 @@ func TestNormalizeMetadataConfigFields_TrimsKey(t *testing.T) {
t.Errorf("expected trimmed key 'my_field', got %v", result[0]["key"])
}
}
+
+func TestPreserveDatasetParserConfigMetadata_FallsBackWhenIncomingNotMap(t *testing.T) {
+ existing := entity.JSONMap{
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{map[string]any{"key": "existing_field", "type": "string"}},
+ "built_in_metadata": []any{},
+ },
+ }
+ cases := map[string]interface{}{
+ "null": nil,
+ "array": []any{},
+ }
+ for name, incomingMetadata := range cases {
+ t.Run(name, func(t *testing.T) {
+ incoming := map[string]interface{}{"metadata": incomingMetadata}
+ got := preserveDatasetParserConfigMetadata(entity.JSONMap{}, existing, incoming)
+ meta, ok := got["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("expected existing modular metadata preserved, got %#v", got["metadata"])
+ }
+ fields, ok := meta["metadata"].([]any)
+ if !ok || len(fields) != 1 || fields[0].(map[string]any)["key"] != "existing_field" {
+ t.Fatalf("expected existing_field preserved, got %#v", meta["metadata"])
+ }
+ })
+ }
+}
+
+func TestPreserveDatasetParserConfigMetadata_UsesValidIncomingMap(t *testing.T) {
+ existing := entity.JSONMap{
+ "metadata": map[string]any{
+ "enabled": false,
+ "metadata": []any{},
+ "built_in_metadata": []any{},
+ },
+ }
+ incoming := map[string]interface{}{
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{map[string]any{"key": "incoming_field", "type": "string"}},
+ "built_in_metadata": []any{},
+ },
+ }
+ got := preserveDatasetParserConfigMetadata(entity.JSONMap{}, existing, incoming)
+ meta, ok := got["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("expected modular metadata map, got %#v", got["metadata"])
+ }
+ fields, ok := meta["metadata"].([]any)
+ if !ok || len(fields) != 1 || fields[0].(map[string]any)["key"] != "incoming_field" {
+ t.Fatalf("expected incoming_field to be used, got %#v", meta["metadata"])
+ }
+}
diff --git a/internal/service/dataset/metadata.go b/internal/service/dataset/metadata.go
index 13c929fee0..f3243345d8 100644
--- a/internal/service/dataset/metadata.go
+++ b/internal/service/dataset/metadata.go
@@ -38,14 +38,22 @@ func (d *DatasetService) UpdateDocumentMetadataConfig(ctx context.Context, userI
parserConfig = entity.JSONMap{}
}
parserConfig["metadata"] = metadata
- if kb, kbErr := d.kbDAO.GetByID(ctx, dao.DB, datasetID); kbErr == nil && kb != nil {
- if tenant, tenantErr := d.tenantDAO.GetByID(ctx, dao.DB, kb.TenantID); tenantErr == nil && tenant != nil {
- parserConfig = service.ApplyComponentScopedParserConfig(
- parserConfig,
- tenant.LLMID,
- )
- }
+ kb, kbErr := d.kbDAO.GetByID(ctx, dao.DB, datasetID)
+ if kbErr != nil {
+ return nil, common.CodeServerError, errors.New("database operation failed")
+ }
+ if kb == nil {
+ return nil, common.CodeServerError, errors.New("database operation failed")
+ }
+ tenant, tenantErr := d.tenantDAO.GetByID(ctx, dao.DB, kb.TenantID)
+ if tenantErr != nil {
+ return nil, common.CodeServerError, errors.New("database operation failed")
}
+ llmID := ""
+ if tenant != nil {
+ llmID = tenant.LLMID
+ }
+ parserConfig = service.ApplyComponentScopedParserConfig(parserConfig, llmID)
if err = d.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{"parser_config": parserConfig}); err != nil {
return nil, common.CodeServerError, errors.New("database operation failed")
@@ -77,11 +85,12 @@ func (d *DatasetService) GetMetadataConfig(ctx context.Context, datasetID, tenan
return nil, common.CodeDataError, fmt.Errorf("user '%s' lacks permission for dataset '%s'", tenantID, datasetID)
}
- metadata := parserConfigValueOrEmptyList(kb.ParserConfig, "metadata")
+ _, enabled, metadata, builtInMetadata := modularMetadataConfig(kb.ParserConfig)
return map[string]interface{}{
+ "enabled": enabled,
"metadata": metadata,
- "built_in_metadata": parserConfigValueOrEmptyList(kb.ParserConfig, "built_in_metadata"),
+ "built_in_metadata": builtInMetadata,
}, common.CodeSuccess, nil
}
@@ -122,33 +131,77 @@ func (d *DatasetService) UpdateMetadataConfig(ctx context.Context, datasetID, te
if parserConfig == nil {
parserConfig = entity.JSONMap{}
}
- parserConfig["metadata"] = metadata
- parserConfig["built_in_metadata"] = builtInMetadata
- // enable_metadata state machine:
- // - If req.Enabled is explicitly set: use *req.Enabled.
- // - If nil (uninitialized dataset): auto-enable if metadata fields exist.
- // - If false (explicitly disabled by user): preserve false even when fields are added.
- // - If fields are empty: auto-disable to avoid invoking extractor on empty schema.
- if req.Enabled != nil {
- parserConfig["enable_metadata"] = *req.Enabled
- } else if parserConfig["enable_metadata"] == nil {
- parserConfig["enable_metadata"] = len(metadata) > 0 || len(builtInMetadata) > 0
- } else if len(metadata) == 0 && len(builtInMetadata) == 0 {
- parserConfig["enable_metadata"] = false
- }
- if tenant, tenantErr := d.tenantDAO.GetByID(ctx, dao.DB, kb.TenantID); tenantErr == nil && tenant != nil {
- parserConfig = service.ApplyComponentScopedParserConfig(
- parserConfig,
- tenant.LLMID,
- )
+ present, currentEnabled, _, _ := modularMetadataConfig(parserConfig)
+ enabled := false
+ switch {
+ case req.Enabled != nil:
+ enabled = *req.Enabled
+ case present:
+ enabled = currentEnabled
+ default:
+ enabled = len(metadata) > 0 || len(builtInMetadata) > 0
+ }
+ if req.Enabled == nil && len(metadata) == 0 && len(builtInMetadata) == 0 {
+ enabled = false
+ }
+ parserConfig["metadata"] = map[string]any{
+ "enabled": enabled,
+ "metadata": metadata,
+ "built_in_metadata": builtInMetadata,
+ }
+ delete(parserConfig, "enable_metadata")
+ delete(parserConfig, "built_in_metadata")
+ tenant, tenantErr := d.tenantDAO.GetByID(ctx, dao.DB, kb.TenantID)
+ if tenantErr != nil {
+ return nil, common.CodeServerError, errors.New("database operation failed")
}
+ llmID := ""
+ if tenant != nil {
+ llmID = tenant.LLMID
+ }
+ parserConfig = service.ApplyComponentScopedParserConfig(parserConfig, llmID)
if err = d.kbDAO.UpdateByID(ctx, dao.DB, kb.ID, map[string]interface{}{"parser_config": parserConfig}); err != nil {
return nil, common.CodeServerError, errors.New("update auto-metadata error.(Database error)")
}
return map[string]interface{}{
+ "enabled": enabled,
"metadata": metadata,
"built_in_metadata": builtInMetadata,
}, common.CodeSuccess, nil
}
+
+// modularMetadataConfig reads the modular dataset-level metadata object
+// ({"enabled", "metadata", "built_in_metadata"}) from parser_config. Missing
+// or malformed config yields a disabled, empty result.
+func modularMetadataConfig(parserConfig map[string]any) (bool, bool, []any, []any) {
+ if parserConfig == nil {
+ return false, false, []any{}, []any{}
+ }
+ metaObj, ok := parserConfig["metadata"].(map[string]any)
+ if !ok {
+ return false, false, []any{}, []any{}
+ }
+ enabled, _ := metaObj["enabled"].(bool)
+ metadata := anyOrEmptyList(metaObj["metadata"])
+ builtIn := anyOrEmptyList(metaObj["built_in_metadata"])
+ return true, enabled, metadata, builtIn
+}
+
+func anyOrEmptyList(value any) []any {
+ if value == nil {
+ return []any{}
+ }
+ if list, ok := value.([]any); ok {
+ return list
+ }
+ if list, ok := value.([]map[string]any); ok {
+ out := make([]any, 0, len(list))
+ for _, item := range list {
+ out = append(out, item)
+ }
+ return out
+ }
+ return []any{}
+}
diff --git a/internal/service/dataset/metadata_config_test.go b/internal/service/dataset/metadata_config_test.go
index e5074c251b..ae6aa37599 100644
--- a/internal/service/dataset/metadata_config_test.go
+++ b/internal/service/dataset/metadata_config_test.go
@@ -102,6 +102,7 @@ func insertDatasetMetadataConfigDoc(t *testing.T, docID, datasetID string, parse
func TestDatasetServiceUpdateDocumentMetadataConfig(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
+ insertCreateDatasetTenant(t, "user-1")
insertDatasetMetadataConfigKB(t, "kb-1", "user-1")
insertDatasetMetadataConfigDoc(t, "doc-1", "kb-1", entity.JSONMap{"pages": []interface{}{1, 2}})
@@ -197,6 +198,7 @@ func TestDatasetServiceUpdateDocumentMetadataConfigRejectsNonOwner(t *testing.T)
func TestDatasetServiceUpdateDocumentMetadataConfigAllowsTeamMember(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
+ insertCreateDatasetTenant(t, "owner-1")
insertDatasetMetadataConfigKB(t, "kb-1", "owner-1")
if err := dao.DB.Model(&entity.Knowledgebase{}).
Where("id = ?", "kb-1").
@@ -233,13 +235,18 @@ func TestDatasetServiceUpdateMetadataConfigSyncsExtractorSchema(t *testing.T) {
if err := dao.DB.Model(&entity.Knowledgebase{}).
Where("id = ?", "kb-1").
Update("parser_config", entity.JSONMap{
- "enable_metadata": false,
+ "metadata": map[string]any{
+ "enabled": false,
+ "metadata": []any{},
+ "built_in_metadata": []any{},
+ },
"Extractor:AutoExtractDefault": map[string]any{
"metadata": map[string]any{
"enabled": true,
- "fields": []any{
+ "metadata": []any{
map[string]any{"key": "stale", "type": "string"},
},
+ "built_in_metadata": []any{},
},
},
}).Error; err != nil {
@@ -284,14 +291,18 @@ func TestDatasetServiceUpdateMetadataConfigSyncsExtractorSchema(t *testing.T) {
t.Fatalf("expected modular metadata map in extractor, got %#v", extractor["metadata"])
}
if enabled, ok := extractorMeta["enabled"].(bool); !ok || enabled {
- t.Fatalf("extractor metadata.enabled = %#v, want false when top-level flag stays disabled", extractorMeta["enabled"])
+ t.Fatalf("extractor metadata.enabled = %#v, want false when dataset-level flag stays disabled", extractorMeta["enabled"])
}
// State 2: When enabled, updating fields updates component with enabled = true
if err := dao.DB.Model(&entity.Knowledgebase{}).
Where("id = ?", "kb-1").
Update("parser_config", entity.JSONMap{
- "enable_metadata": true,
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{},
+ "built_in_metadata": []any{},
+ },
"Extractor:AutoExtractDefault": map[string]any{},
}).Error; err != nil {
t.Fatalf("reset parser_config: %v", err)
@@ -363,8 +374,15 @@ func TestDatasetServiceUpdateMetadataConfigSyncsExtractorSchema(t *testing.T) {
if err != nil {
t.Fatalf("failed to fetch persisted dataset: %v", err)
}
- if enabled, ok := persisted.ParserConfig["enable_metadata"].(bool); !ok || enabled {
- t.Fatalf("expected parser_config.enable_metadata == false after emptying fields, got %#v", persisted.ParserConfig["enable_metadata"])
+ if _, ok := persisted.ParserConfig["enable_metadata"]; ok {
+ t.Fatalf("enable_metadata should be absent, got %#v", persisted.ParserConfig["enable_metadata"])
+ }
+ metaObj, ok := persisted.ParserConfig["metadata"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("expected modular metadata map, got %#v", persisted.ParserConfig["metadata"])
+ }
+ if enabled, ok := metaObj["enabled"].(bool); !ok || enabled {
+ t.Fatalf("expected modular metadata.enabled == false after emptying fields, got %#v", metaObj["enabled"])
}
}
@@ -420,3 +438,97 @@ func TestDatasetServiceGetMetadataConfig(t *testing.T) {
t.Fatalf("expected key file_name, got %#v", builtIn[0])
}
}
+
+func TestUpdateMetadataConfig_TenantLookupFailureDoesNotPersist(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ // kb-1 points at a tenant that does not exist, so tenantDAO.GetByID fails.
+ insertDatasetMetadataConfigKB(t, "kb-1", "missing-tenant")
+
+ ctx := t.Context()
+ result, code, err := (&DatasetService{
+ kbDAO: dao.NewKnowledgebaseDAO(),
+ tenantDAO: dao.NewTenantDAO(),
+ }).UpdateMetadataConfig(ctx, "kb-1", "missing-tenant", &service.MetadataConfigRequest{
+ Metadata: []service.MetadataConfigField{
+ {Key: "author", Type: "string"},
+ },
+ BuiltInMetadata: []service.MetadataConfigField{},
+ })
+ if code != common.CodeServerError || err == nil {
+ t.Fatalf("expected server error on tenant lookup failure, got code=%d err=%v result=%#v", code, err, result)
+ }
+
+ persisted, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")
+ if err != nil {
+ t.Fatalf("get kb: %v", err)
+ }
+ if _, ok := persisted.ParserConfig["metadata"]; ok {
+ t.Fatalf("parser config must not be persisted on tenant lookup failure, got %#v", persisted.ParserConfig)
+ }
+}
+
+func TestUpdateMetadataConfig_ExplicitEnabledPreservedWhenEmpty(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertCreateDatasetTenant(t, "tenant-1")
+ insertDatasetMetadataConfigKB(t, "kb-1", "tenant-1")
+
+ enabled := true
+ ctx := t.Context()
+ result, code, err := (&DatasetService{
+ kbDAO: dao.NewKnowledgebaseDAO(),
+ tenantDAO: dao.NewTenantDAO(),
+ }).UpdateMetadataConfig(ctx, "kb-1", "tenant-1", &service.MetadataConfigRequest{
+ Enabled: &enabled,
+ Metadata: []service.MetadataConfigField{},
+ BuiltInMetadata: []service.MetadataConfigField{},
+ })
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("UpdateMetadataConfig failed: code=%d err=%v", code, err)
+ }
+ respEnabled, ok := result["enabled"].(bool)
+ if !ok || !respEnabled {
+ t.Fatalf("expected enabled=true in response, got %#v", result["enabled"])
+ }
+
+ persisted, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")
+ if err != nil {
+ t.Fatalf("get kb: %v", err)
+ }
+ metaObj, ok := persisted.ParserConfig["metadata"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("expected modular metadata, got %#v", persisted.ParserConfig["metadata"])
+ }
+ if persistedEnabled, _ := metaObj["enabled"].(bool); !persistedEnabled {
+ t.Fatalf("expected metadata.enabled=true, got %#v", metaObj["enabled"])
+ }
+}
+
+func TestUpdateDocumentMetadataConfig_KBAndTenantLookupFailure(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertDatasetMetadataConfigKB(t, "kb-ok", "owner-1")
+ insertDatasetMetadataConfigDoc(t, "doc-ok", "kb-ok", entity.JSONMap{})
+ ctx := t.Context()
+
+ // Missing KB -> database layer should surface as CodeServerError / not found
+ _, code, err := testDatasetServiceForDocumentMetadataConfig(t).UpdateDocumentMetadataConfig(
+ ctx, "owner-1", "kb-missing", "doc-ok",
+ map[string]interface{}{"metadata": map[string]interface{}{"a": "b"}},
+ )
+ if code == common.CodeSuccess || err == nil {
+ t.Fatalf("expected error for missing kb, got code=%v err=%v", code, err)
+ }
+
+ // KB exists but tenant lookup fails (kb.TenantID points to missing tenant)
+ insertDatasetMetadataConfigKB(t, "kb-bad-tenant", "missing-tenant")
+ insertDatasetMetadataConfigDoc(t, "doc2", "kb-bad-tenant", entity.JSONMap{})
+ _, code, err = testDatasetServiceForDocumentMetadataConfig(t).UpdateDocumentMetadataConfig(
+ ctx, "missing-tenant", "kb-bad-tenant", "doc2",
+ map[string]interface{}{"metadata": map[string]interface{}{"a": "b"}},
+ )
+ if code != common.CodeServerError || err == nil {
+ t.Fatalf("expected server error for tenant lookup failure, got code=%v err=%v", code, err)
+ }
+}
diff --git a/internal/service/dataset/update_test.go b/internal/service/dataset/update_test.go
index dd2b396112..bcea8c9032 100644
--- a/internal/service/dataset/update_test.go
+++ b/internal/service/dataset/update_test.go
@@ -1067,9 +1067,11 @@ func TestUpdateDataset_PreservesIncomingMetadataWhenCleaningParserConfig(t *test
"Parser:HipSignsRhyme": map[string]interface{}{
"pdf": map[string]interface{}{"parse_method": "deepdoc"},
},
- "metadata": incomingMetadata,
- "built_in_metadata": incomingBuiltInMetadata,
- "enable_metadata": true,
+ "metadata": map[string]interface{}{
+ "enabled": true,
+ "metadata": incomingMetadata,
+ "built_in_metadata": incomingBuiltInMetadata,
+ },
}
_, code, err := testDatasetUpdateService(t).UpdateDataset(t.Context(), "kb-1", "tenant-1", service.UpdateDatasetRequest{
@@ -1084,14 +1086,18 @@ func TestUpdateDataset_PreservesIncomingMetadataWhenCleaningParserConfig(t *test
if err != nil {
t.Fatalf("get updated kb: %v", err)
}
- if !reflect.DeepEqual(persisted.ParserConfig["metadata"], incomingMetadata) {
- t.Fatalf("metadata was not preserved: %#v", persisted.ParserConfig["metadata"])
+ if !reflect.DeepEqual(persisted.ParserConfig["metadata"], map[string]interface{}{
+ "enabled": true,
+ "metadata": incomingMetadata,
+ "built_in_metadata": incomingBuiltInMetadata,
+ }) {
+ t.Fatalf("modular metadata was not preserved: %#v", persisted.ParserConfig["metadata"])
}
- if !reflect.DeepEqual(persisted.ParserConfig["built_in_metadata"], incomingBuiltInMetadata) {
- t.Fatalf("built_in_metadata was not preserved: %#v", persisted.ParserConfig["built_in_metadata"])
+ if _, ok := persisted.ParserConfig["enable_metadata"]; ok {
+ t.Fatalf("enable_metadata should be absent: %#v", persisted.ParserConfig["enable_metadata"])
}
- if persisted.ParserConfig["enable_metadata"] != true {
- t.Fatalf("enable_metadata was not preserved: %#v", persisted.ParserConfig["enable_metadata"])
+ if _, ok := persisted.ParserConfig["built_in_metadata"]; ok {
+ t.Fatalf("built_in_metadata should be absent: %#v", persisted.ParserConfig["built_in_metadata"])
}
if _, ok := persisted.ParserConfig["Parser:HipSignsRhyme"].(map[string]interface{}); !ok {
t.Fatalf("component parser_config missing: %#v", persisted.ParserConfig)
@@ -1107,14 +1113,12 @@ func TestUpdateDataset_PreservesExistingMetadataWhenParserConfigOmitsIt(t *testi
"key": "category",
"type": "string",
}}
- existingBuiltInMetadata := []interface{}{map[string]interface{}{
- "key": "document_name",
- "type": "string",
- }}
if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", entity.JSONMap{
- "metadata": existingMetadata,
- "built_in_metadata": existingBuiltInMetadata,
- "enable_metadata": true,
+ "metadata": map[string]interface{}{
+ "enabled": true,
+ "metadata": existingMetadata,
+ "built_in_metadata": []interface{}{map[string]interface{}{"key": "document_name", "type": "string"}},
+ },
}).Error; err != nil {
t.Fatalf("seed parser_config: %v", err)
}
@@ -1135,14 +1139,18 @@ func TestUpdateDataset_PreservesExistingMetadataWhenParserConfigOmitsIt(t *testi
if err != nil {
t.Fatalf("get updated kb: %v", err)
}
- if !reflect.DeepEqual(persisted.ParserConfig["metadata"], existingMetadata) {
- t.Fatalf("existing metadata was not preserved: %#v", persisted.ParserConfig["metadata"])
+ if !reflect.DeepEqual(persisted.ParserConfig["metadata"], map[string]interface{}{
+ "enabled": true,
+ "metadata": existingMetadata,
+ "built_in_metadata": []interface{}{map[string]interface{}{"key": "document_name", "type": "string"}},
+ }) {
+ t.Fatalf("existing modular metadata was not preserved: %#v", persisted.ParserConfig["metadata"])
}
- if !reflect.DeepEqual(persisted.ParserConfig["built_in_metadata"], existingBuiltInMetadata) {
- t.Fatalf("existing built_in_metadata was not preserved: %#v", persisted.ParserConfig["built_in_metadata"])
+ if _, ok := persisted.ParserConfig["enable_metadata"]; ok {
+ t.Fatalf("enable_metadata should be absent: %#v", persisted.ParserConfig["enable_metadata"])
}
- if persisted.ParserConfig["enable_metadata"] != true {
- t.Fatalf("existing enable_metadata was not preserved: %#v", persisted.ParserConfig["enable_metadata"])
+ if _, ok := persisted.ParserConfig["built_in_metadata"]; ok {
+ t.Fatalf("built_in_metadata should be absent: %#v", persisted.ParserConfig["built_in_metadata"])
}
}
diff --git a/test/testcases/restful_api/test_datasets.py b/test/testcases/restful_api/test_datasets.py
index b1f8d70cad..f67b97f71d 100644
--- a/test/testcases/restful_api/test_datasets.py
+++ b/test/testcases/restful_api/test_datasets.py
@@ -2287,7 +2287,8 @@ def test_dataset_metadata_config_get_and_update_contract(rest_client, create_dat
assert success_res.status_code == 200
success_payload = success_res.json()
assert success_payload["code"] == 0, success_payload
- assert success_payload["data"] == {"metadata": [], "built_in_metadata": []}, success_payload
+ expected_empty = {"enabled": False, "metadata": [], "built_in_metadata": []} if IS_GO_PROXY else {"metadata": [], "built_in_metadata": []}
+ assert success_payload["data"] == expected_empty, success_payload
for scenario_name, client in (("missing token", RestClient(token=None)), ("invalid token", RestClient(token=INVALID_API_TOKEN))):
get_res = client.get(f"/datasets/{dataset_id}/metadata/config")
@@ -2327,6 +2328,8 @@ def test_dataset_metadata_config_get_and_update_contract(rest_client, create_dat
{"key": "size", "type": "number", "description": "File size", "enum": None},
],
}
+ if IS_GO_PROXY:
+ normalized_update_payload["enabled"] = True
update_res = rest_client.put(f"/datasets/{dataset_id}/metadata/config", json=update_payload)
assert update_res.status_code == 200
update_body = update_res.json()
@@ -2343,7 +2346,7 @@ def test_dataset_metadata_config_get_and_update_contract(rest_client, create_dat
assert missing_payload_res.status_code == 200
missing_payload = missing_payload_res.json()
assert missing_payload["code"] == 0, missing_payload
- assert missing_payload["data"] == {"metadata": [], "built_in_metadata": []}, missing_payload
+ assert missing_payload["data"] == expected_empty, missing_payload
invalid_update_dataset_res = rest_client.put(
"/datasets/invalid_dataset_id/metadata/config",