RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(ingestion): unify auto-metadata on modular metadata and hard-delete legacy flats (BuiltInMetadata carried, not LLM) (#18511)
jay777212022efa9fcef
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.goindex 63b11a7b43..d2cdcf04cd 100644--- a/internal/ingestion/component/extractor.go+++ b/internal/ingestion/component/extractor.go@@ -265,6 +265,7 @@ func NewExtractorComponent(params map[string]any) (runtime.Component, error) {if v, ok := metaRaw["metadata"]; ok {p.Metadata.Metadata = parseMetadataFieldDefs(v)}+ // BuiltInMetadata is carried for persistence/replay; LLM extraction uses Metadata only.if v, ok := metaRaw["built_in_metadata"]; ok {p.Metadata.BuiltInMetadata = parseMetadataFieldDefs(v)}diff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.goindex e5aff9f85b..7aca970979 100644--- a/internal/ingestion/component/extractor_test.go+++ b/internal/ingestion/component/extractor_test.go@@ -1491,6 +1491,9 @@ func TestExtractorModularMetadataConfig(t *testing.T) {if ext.Param.Metadata.Metadata[0].Key != "author" || ext.Param.Metadata.Metadata[0].Description != "The author name" || len(ext.Param.Metadata.Metadata[0].Enum) != 2 {t.Errorf("Metadata field 0 mismatch: %+v", ext.Param.Metadata.Metadata[0])}+ if ext.Param.Metadata.Metadata[1].Key != "year" {+ t.Errorf("Metadata field 1 mismatch: %+v", ext.Param.Metadata.Metadata[1])+ }if len(ext.Param.Metadata.BuiltInMetadata) != 1 || ext.Param.Metadata.BuiltInMetadata[0].Key != "file_name" {t.Errorf("BuiltInMetadata mismatch: %+v", ext.Param.Metadata.BuiltInMetadata)}@@ -1859,3 +1862,72 @@ func TestExtractor_ParseMetadataFieldDefs_MapSlice(t *testing.T) {t.Errorf("parseMetadataFieldDefs failed for []common.MetadataFieldDef: %#v", directDefs)}}++func TestExtractorBuiltInDoesNotCallLLM(t *testing.T) {+ // 1b39355c regressed by making runEnableMetadata fire when only built_in_metadata was configured.+ // With the modular shape, BuiltInMetadata must never trigger an LLM call; it is applied by the finalizer.+ stub := withStubChatInvoker(t)+ params := map[string]any{+ "llm_id": "llm-1",+ "metadata": map[string]any{+ "enabled": true,+ "built_in_metadata": []any{+ map[string]any{"key": "file_name", "type": "string"},+ map[string]any{"key": "update_time", "type": "time"},+ },+ "metadata": []any{},+ },+ }+ comp, err := NewExtractorComponent(params)+ if err != nil {+ t.Fatalf("NewExtractorComponent: %v", err)+ }+ in := map[string]any{+ "chunks": []map[string]any{{"text": "hello world"}},+ }+ out, err := comp.Invoke(t.Context(), nil, in)+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if stub.calls.Load() != 0 {+ t.Fatalf("built_in-only must not call LLM, got %d calls, requests=%v", stub.calls.Load(), stub.requests)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) != 1 {+ t.Fatalf("expected 1 chunk, got %v", out)+ }+ if _, ok := chunks[0]["metadata"]; ok {+ t.Fatalf("built_in must not produce chunk metadata, got %v", chunks[0]["metadata"])+ }+}++func TestExtractorEnabledFalseDoesNotCallLLM(t *testing.T) {+ stub := withStubChatInvoker(t)+ params := map[string]any{+ "llm_id": "llm-1",+ "metadata": map[string]any{+ "enabled": false,+ "metadata": []any{+ map[string]any{"key": "author", "type": "string"},+ },+ "built_in_metadata": []any{+ map[string]any{"key": "file_name", "type": "string"},+ },+ },+ }+ comp, err := NewExtractorComponent(params)+ if err != nil {+ t.Fatalf("NewExtractorComponent: %v", err)+ }+ in := map[string]any{"chunks": []map[string]any{{"text": "hello"}}}+ out, err := comp.Invoke(t.Context(), nil, in)+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if stub.calls.Load() != 0 {+ t.Fatalf("enabled=false must not call LLM, got %d", stub.calls.Load())+ }+ if chunks, _ := out["chunks"].([]map[string]any); len(chunks) != 1 {+ t.Fatalf("expected 1 chunk, got %v", out)+ }+}diff --git a/internal/ingestion/component/schema/extractor.go b/internal/ingestion/component/schema/extractor.goindex bb5dc4e08d..7357b0aaa8 100644--- a/internal/ingestion/component/schema/extractor.go+++ b/internal/ingestion/component/schema/extractor.go@@ -99,6 +99,8 @@ type SummaryExtractConfig struct {}// MetadataExtractConfig configures structured metadata extraction.+// BuiltInMetadata is carried for persistence/replay; it is NOT LLM-extracted.+// Deterministic file_name/update_time is applied via PipelineResult -> doc_state.applyBuiltInMetadata.type MetadataExtractConfig struct {Enabled bool `json:"enabled"`Metadata []common.MetadataFieldDef `json:"metadata,omitempty"`diff --git a/internal/ingestion/service/doc_state_test.go b/internal/ingestion/service/doc_state_test.goindex 86ad8c4be4..f47ebc1c5c 100644--- a/internal/ingestion/service/doc_state_test.go+++ b/internal/ingestion/service/doc_state_test.go@@ -255,3 +255,30 @@ func TestDocStateUpdater_BuiltInMetadataEmptyConfig(t *testing.T) {t.Errorf("empty built-in config must be a no-op, got %v", svc.metaData)}}++func TestDocStateUpdater_BuiltInNotWrittenWhenEnabledFalse(t *testing.T) {+ svc := &stubDocStateSvc{metaData: map[string]any{}}+ u := &docStateUpdater{docSvc: svc}+ ctx := t.Context()+ u.apply(ctx, &taskpkg.PipelineResult{+ DocID: "doc-1",+ DocName: "report.pdf",+ AutoMetadataEnabled: false,+ BuiltInMetadataConfig: []any{+ map[string]any{"key": "file_name", "type": "string"},+ map[string]any{"key": "update_time", "type": "time"},+ },+ Metadata: map[string]any{"author": "Alice"},+ ChunkCount: 1,+ TokenConsumption: 1,+ })+ if _, ok := svc.metaData["file_name"]; ok {+ t.Fatalf("built_in must not be written when enabled=false, got %v", svc.metaData)+ }+ if _, ok := svc.metaData["update_time"]; ok {+ t.Fatalf("built_in must not be written when enabled=false, got %v", svc.metaData)+ }+ if svc.metaData["author"] != "Alice" {+ t.Fatalf("custom metadata should still be written even when built_in is off, got %v", svc.metaData)+ }+}diff --git a/internal/ingestion/service/e2e_metadata_chain_test.go b/internal/ingestion/service/e2e_metadata_chain_test.gonew file mode 100644index 0000000000..22c3d668e2--- /dev/null+++ b/internal/ingestion/service/e2e_metadata_chain_test.go@@ -0,0 +1,73 @@+package service++import (+ "context"+ "testing"++ "ragflow/internal/entity"+ taskpkg "ragflow/internal/ingestion/task"+ "ragflow/internal/service"+)++// TestE2E_EnabledFalseDoesNotWriteBuiltIn verifies the full chain's gate:+// dataset metadata enabled=false -> pipeline AutoMetadataEnabled=false -> doc_state does not write built_in.+func TestE2E_EnabledFalseDoesNotWriteBuiltIn(t *testing.T) {+ // Directly exercise the doc_state gate which is the last step of the chain.+ // The earlier steps (dataset UpdateMetadataConfig -> ApplyComponentScoped -> pipeline builtInMetadataFromParserConfig)+ // are covered by unit tests; this E2E asserts the gate is not bypassed.+ svc := &stubDocStateSvc{metaData: map[string]any{"existing": "keep"}}+ u := &docStateUpdater{docSvc: svc}+ ctx := t.Context()+ u.apply(ctx, &taskpkg.PipelineResult{+ DocID: "doc-e2e",+ DocName: "a.pdf",+ AutoMetadataEnabled: false,+ BuiltInMetadataConfig: []any{map[string]any{"key": "file_name", "type": "string"}, map[string]any{"key": "update_time", "type": "time"}},+ Metadata: map[string]any{"author": "Bob"},+ ChunkCount: 1,+ })+ if _, ok := svc.metaData["file_name"]; ok {+ t.Fatalf("must not write file_name when enabled=false, got %v", svc.metaData)+ }+ if svc.metaData["author"] != "Bob" {+ t.Fatalf("custom metadata should still be written, got %v", svc.metaData)+ }+ if svc.metaData["existing"] != "keep" {+ t.Fatalf("existing must be kept, got %v", svc.metaData)+ }+}++func TestE2E_ApplyComponentScopedThenPipelineThenDocState(t *testing.T) {+ // ParserConfig as it would be after DatasetService.UpdateMetadataConfig+ parserConfig := entity.JSONMap{+ "metadata": map[string]any{+ "enabled": true,+ "metadata": []any{map[string]any{"key": "author", "type": "string"}},+ "built_in_metadata": []any{map[string]any{"key": "file_name", "type": "string"}, map[string]any{"key": "update_time", "type": "time"}},+ },+ "Extractor:AutoExtractDefault": map[string]any{},+ }+ // Scope to node+ got := service.ApplyComponentScopedParserConfig(parserConfig, "llm-1")+ node, _ := got["Extractor:AutoExtractDefault"].(map[string]any)+ meta, _ := node["metadata"].(map[string]any)+ if enabled, _ := meta["enabled"].(bool); !enabled {+ t.Fatalf("scoped enabled must be true, got %v", meta["enabled"])+ }+ if _, ok := meta["built_in_metadata"]; !ok {+ t.Fatalf("scoped built_in missing, got %v", meta)+ }+ // Pipeline would read built_in -> doc_state would write it+ svc := &stubDocStateSvc{metaData: map[string]any{}}+ u := &docStateUpdater{docSvc: svc}+ u.apply(context.Background(), &taskpkg.PipelineResult{+ DocID: "doc-1", KbID: "kb-1", DocName: "report.pdf",+ AutoMetadataEnabled: true,+ BuiltInMetadataConfig: []any{map[string]any{"key": "file_name", "type": "string"}},+ Metadata: map[string]any{"author": "Alice"},+ ChunkCount: 1,+ })+ if svc.metaData["file_name"] != "report.pdf" {+ t.Fatalf("file_name not written, got %v", svc.metaData)+ }+}diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex 3d224c8121..40a20cebe5 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -303,9 +303,10 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map}// builtInMetadataFromParserConfig extracts the built-in metadata config-// (update_time / file_name) and whether auto-metadata is enabled. Supports-// modular metadata_config sub-objects as well as legacy flat fields on-// Extractor nodes and top-level parser_config.+// (update_time / file_name) and whether auto-metadata is enabled from the+// component-scoped Extractor node's modular metadata config. Legacy flat+// fields (enable_metadata / metadata_config / built_in_metadata at either the+// top level or on the node) are intentionally not supported.func builtInMetadataFromParserConfig(parserConfig entity.JSONMap) ([]any, bool) {var extractorKeys []stringfor k := range parserConfig {@@ -320,37 +321,29 @@ func builtInMetadataFromParserConfig(parserConfig entity.JSONMap) ([]any, bool)nodeRaw := parserConfig[k]if node, ok := nodeRaw.(map[string]any); ok {if metaObj, ok := node["metadata"].(map[string]any); ok {- enabled := parserConfigBool(metaObj["enabled"])- if arr, ok := metaObj["built_in_metadata"].([]any); ok && len(arr) > 0 {- return arr, enabled- }- }- if metaConf, ok := node["metadata_config"].(map[string]any); ok {- enabled := parserConfigBool(metaConf["enabled"])- if arr, ok := metaConf["built_in_metadata"].([]any); ok && len(arr) > 0 {- return arr, enabled- }- }- enabled := parserConfigBool(node["enable_metadata"])- if arr, ok := node["built_in_metadata"].([]any); ok && len(arr) > 0 {- return arr, enabled+ arr := metadataFieldSlice(metaObj["built_in_metadata"])+ return arr, parserConfigBool(metaObj["enabled"])}}}- if metaObj, ok := parserConfig["metadata"].(map[string]any); ok {- if arr, ok := metaObj["built_in_metadata"].([]any); ok && len(arr) > 0 {- return arr, true- }- }- if metaConf, ok := parserConfig["metadata_config"].(map[string]any); ok {- if arr, ok := metaConf["built_in_metadata"].([]any); ok && len(arr) > 0 {- return arr, true+ return nil, false+}++// metadataFieldSlice normalizes a built_in_metadata / metadata value that may+// arrive as []interface{} (DB round-trip) or []map[string]interface{} (in-memory+// construction) into a []any.+func metadataFieldSlice(value any) []any {+ if list, ok := value.([]any); ok {+ return list+ }+ if list, ok := value.([]map[string]any); ok {+ out := make([]any, 0, len(list))+ for _, item := range list {+ out = append(out, item)}+ return out}- if arr, ok := parserConfig["built_in_metadata"].([]any); ok && len(arr) > 0 {- return arr, true- }- return nil, false+ return nil}// parserConfigBool coerces a parser_config boolean-like value (bool / number)diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.goindex b9dbb572a6..c6bc388c62 100644--- a/internal/ingestion/task/pipeline_executor_defaults_test.go+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go@@ -186,38 +186,24 @@ func TestBuildComponentParams_GoldenCoversAllTemplates(t *testing.T) {func TestBuiltInMetadataFromParserConfig_ExtractorNodeParams(t *testing.T) {pc := entity.JSONMap{"Extractor:AutoExtractDefault": map[string]any{- "enable_metadata": 1,- "built_in_metadata": []any{- map[string]any{"key": "update_time", "type": "time"},- map[string]any{"key": "file_name", "type": "string"},+ "metadata": map[string]any{+ "enabled": true,+ "built_in_metadata": []any{+ map[string]any{"key": "update_time", "type": "time"},+ map[string]any{"key": "file_name", "type": "string"},+ },},},}cfg, enabled := builtInMetadataFromParserConfig(pc)if !enabled {- t.Fatal("auto metadata should be enabled from extractor node params")+ t.Fatal("auto metadata should be enabled from extractor node modular config")}if len(cfg) != 2 {t.Fatalf("built-in config len = %d, want 2", len(cfg))}}-func TestBuiltInMetadataFromParserConfig_TopLevelFallback(t *testing.T) {- pc := entity.JSONMap{- "enable_metadata": true,- "built_in_metadata": []any{- map[string]any{"key": "file_name", "type": "string"},- },- }- cfg, enabled := builtInMetadataFromParserConfig(pc)- if !enabled {- t.Fatal("auto metadata should be enabled from top-level config")- }- if len(cfg) != 1 {- t.Fatalf("built-in config len = %d, want 1", len(cfg))- }-}-func TestBuiltInMetadataFromParserConfig_None(t *testing.T) {cfg, enabled := builtInMetadataFromParserConfig(entity.JSONMap{})if enabled || len(cfg) != 0 {@@ -241,13 +227,15 @@ func TestBuiltInMetadataFromParserConfig_MultiExtractorDeterministic(t *testing.// Extractor:A must deterministically win regardless of Go map iteration order.pc := entity.JSONMap{"Extractor:B": map[string]any{- "enable_metadata": 1,- "built_in_metadata": []any{- map[string]any{"key": "file_name", "type": "string"},+ "metadata": map[string]any{+ "enabled": true,+ "built_in_metadata": []any{+ map[string]any{"key": "file_name", "type": "string"},+ },},},"Extractor:A": map[string]any{- "metadata_config": map[string]any{+ "metadata": map[string]any{"enabled": true,"built_in_metadata": []any{map[string]any{"key": "update_time", "type": "time"},@@ -270,3 +258,57 @@ func TestBuiltInMetadataFromParserConfig_MultiExtractorDeterministic(t *testing.}}}++func TestBuiltInMetadataFromParserConfig_FirstMetadataNodeWins(t *testing.T) {+ // The alphabetically-first Extractor node that declares metadata is the+ // authoritative source, even when its built_in_metadata is empty.+ pc := entity.JSONMap{+ "Extractor:A": map[string]any{+ "metadata": map[string]any{+ "enabled": true,+ "built_in_metadata": []any{},+ },+ },+ "Extractor:B": map[string]any{+ "metadata": map[string]any{+ "enabled": true,+ "built_in_metadata": []any{+ map[string]any{"key": "file_name", "type": "string"},+ },+ },+ },+ }++ cfg, enabled := builtInMetadataFromParserConfig(pc)+ if !enabled {+ t.Fatalf("expected enabled=true from first metadata node, got false")+ }+ if len(cfg) != 0 {+ t.Fatalf("expected empty built-in config from first metadata node, got %#v", cfg)+ }+}++func TestBuiltInMetadataFromParserConfig_MapSliceBuiltIn(t *testing.T) {+ pc := entity.JSONMap{+ "Extractor:A": map[string]any{+ "metadata": map[string]any{+ "enabled": true,+ "built_in_metadata": []map[string]any{+ {"key": "update_time", "type": "time"},+ },+ },+ },+ }++ cfg, enabled := builtInMetadataFromParserConfig(pc)+ if !enabled {+ t.Fatalf("expected enabled=true, got false")+ }+ if len(cfg) != 1 {+ t.Fatalf("expected 1 built-in item from []map[string]any, got %d", len(cfg))+ }+ item, ok := cfg[0].(map[string]any)+ if !ok || item["key"] != "update_time" {+ t.Fatalf("expected update_time item, got %#v", cfg[0])+ }+}diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.goindex 62a50f0bc4..80222d12f6 100644--- a/internal/ingestion/task/pipeline_executor_test.go+++ b/internal/ingestion/task/pipeline_executor_test.go@@ -575,25 +575,3 @@ func TestCountOriginalChunkIDs(t *testing.T) {t.Fatalf("compiler products: got %d, want 2", n)}}--func TestBuiltInMetadata_ZeroExtractorFallback(t *testing.T) {- // Scenario: User constructed a pipeline with 0 Extractor components,- // but dataset parser_config has top-level built_in_metadata.- parserConfig := map[string]any{- "Parser:HipSignsRhyme": map[string]any{- "setups": map[string]any{},- },- "built_in_metadata": []any{- map[string]any{"key": "file_name", "type": "string"},- map[string]any{"key": "update_time", "type": "time"},- },- }-- arr, enabled := builtInMetadataFromParserConfig(parserConfig)- if !enabled {- t.Errorf("expected builtInMetadataFromParserConfig enabled=true for zero-extractor pipeline with top-level built_in_metadata")- }- if len(arr) != 2 {- t.Errorf("expected 2 built-in metadata items, got %d", len(arr))- }-}diff --git a/internal/service/component_scoped_parser_config.go b/internal/service/component_scoped_parser_config.goindex cf78bc993f..4929ecc1cd 100644--- a/internal/service/component_scoped_parser_config.go+++ b/internal/service/component_scoped_parser_config.go@@ -6,9 +6,15 @@ import ("ragflow/internal/entity")-// ApplyComponentScopedParserConfig fills dataset-scoped component params onto a-// parser_config map without reintroducing top-level flat fields. It mutates the-// provided map in place and returns it for convenience.+// ApplyComponentScopedParserConfig scopes a dataset-level modular metadata+// config (parserConfig["metadata"] = {enabled, metadata, built_in_metadata})+// onto every Extractor node and strips the legacy flat metadata fields+// (enable_metadata / metadata_config / built_in_metadata / fields) both at the+// top level and on nodes. Legacy flat metadata forms are intentionally not+// supported. When a dataset-level modular metadata config is present it is+// authoritative and replaces each Extractor node's metadata; when absent, the+// node's own modular metadata is preserved. It mutates the provided map in+// place and returns it for convenience.func ApplyComponentScopedParserConfig(parserConfig entity.JSONMap,llmID string,@@ -17,25 +23,16 @@ func ApplyComponentScopedParserConfig(parserConfig = entity.JSONMap{}}- enableMetadata := parserConfigTruthy(parserConfig["enable_metadata"])- if !enableMetadata {- if metaMap, ok := parserConfig["metadata"].(map[string]any); ok {- enableMetadata = parserConfigTruthy(metaMap["enabled"])- }- }- if !enableMetadata {- enableMetadata = parserConfigTruthy(parserConfig["enabled"])+ var datasetMeta map[string]any+ if mm, ok := parserConfig["metadata"].(map[string]any); ok {+ datasetMeta = cloneJSONMap(mm)}- metadataFields := extractFieldList(parserConfig, "fields", "metadata")- builtInMetadataFields := extractFieldList(parserConfig, "built_in_metadata", "")-for cpnID, raw := range parserConfig {params, ok := raw.(map[string]any)if !ok {continue}-cpnLower := strings.ToLower(cpnID)switch {case strings.HasPrefix(cpnLower, "extractor:") || strings.HasPrefix(cpnLower, "extractor_"):@@ -43,45 +40,20 @@ func ApplyComponentScopedParserConfig(params["llm_id"] = llmID}delete(params, "enable_metadata")-- hasTopLevelFields := len(metadataFields) > 0 || len(builtInMetadataFields) > 0-- if hasTopLevelFields {+ delete(params, "metadata_config")+ delete(params, "built_in_metadata")+ delete(params, "fields")+ if datasetMeta != nil {+ params["metadata"] = cloneJSONMap(datasetMeta)+ continue+ }+ if _, ok := params["metadata"].(map[string]any); !ok {params["metadata"] = map[string]any{- "enabled": enableMetadata,- "metadata": metadataFields,- "built_in_metadata": builtInMetadataFields,- }- } else {- nodeCustom := extractFieldList(params, "fields", "metadata")- nodeBuiltIn := extractFieldList(params, "built_in_metadata", "")- nodeEnabled := false-- if metaMap, ok := params["metadata"].(map[string]any); ok && metaMap["enabled"] != nil {- nodeEnabled = parserConfigTruthy(metaMap["enabled"])- } else if mcMap, ok := params["metadata_config"].(map[string]any); ok && mcMap["enabled"] != nil {- nodeEnabled = parserConfigTruthy(mcMap["enabled"])- } else if params["enable_metadata"] != nil {- nodeEnabled = parserConfigTruthy(params["enable_metadata"])- } else if enableMetadata {- nodeEnabled = true- }-- if len(nodeCustom) > 0 || len(nodeBuiltIn) > 0 || nodeEnabled {- params["metadata"] = map[string]any{- "enabled": nodeEnabled,- "metadata": nodeCustom,- "built_in_metadata": nodeBuiltIn,- }- } else {- params["metadata"] = map[string]any{- "enabled": false,- "metadata": []any{},- "built_in_metadata": []any{},- }+ "enabled": false,+ "metadata": []any{},+ "built_in_metadata": []any{},}}- delete(params, "metadata_config")case strings.HasPrefix(cpnLower, "compiler:") || strings.HasPrefix(cpnLower, "compiler_"):if value, _ := params["llm_id"].(string); strings.TrimSpace(value) == "" && strings.TrimSpace(llmID) != "" {params["llm_id"] = llmID@@ -89,145 +61,23 @@ func ApplyComponentScopedParserConfig(}}- return parserConfig-}--func mergeMetadataFields(parserConfig entity.JSONMap) []any {- custom := extractFieldList(parserConfig, "fields", "metadata")- builtIn := extractFieldList(parserConfig, "built_in_metadata", "")- out := make([]any, 0, len(custom)+len(builtIn))- out = append(out, custom...)- out = append(out, builtIn...)- return out-}--func extractFieldList(parserConfig entity.JSONMap, primaryKey, fallbackKey string) []any {- if parserConfig == nil {- return []any{}- }- if raw, ok := parserConfig[primaryKey]; ok && raw != nil {- if slice := extractValidFields(raw); len(slice) > 0 || primaryKey == "fields" {- return slice- }+ // Strip legacy top-level flat metadata fields; keep the modular object+ // under "metadata" when present, otherwise drop the key entirely.+ delete(parserConfig, "enable_metadata")+ delete(parserConfig, "metadata_config")+ delete(parserConfig, "built_in_metadata")+ delete(parserConfig, "fields")+ if datasetMeta == nil {+ delete(parserConfig, "metadata")}- if fallbackKey != "" {- if raw, ok := parserConfig[fallbackKey]; ok && raw != nil {- if slice := extractValidFields(raw); len(slice) > 0 {- return slice- }- }- }- for _, subKey := range []string{"metadata", "metadata_config"} {- if metaMap, ok := parserConfig[subKey].(map[string]any); ok {- if raw, ok := metaMap[primaryKey]; ok && raw != nil {- if slice := extractValidFields(raw); len(slice) > 0 {- return slice- }- }- if fallbackKey != "" {- if raw, ok := metaMap[fallbackKey]; ok && raw != nil {- if slice := extractValidFields(raw); len(slice) > 0 {- return slice- }- }- }- }- }- return []any{}-}-func extractValidFields(value any) []any {- items := anySlice(value)- if items == nil {- return []any{}- }- out := make([]any, 0, len(items))- for _, item := range items {- field, ok := item.(map[string]any)- if !ok {- continue- }- name, _ := field["key"].(string)- if strings.TrimSpace(name) == "" {- continue- }- out = append(out, field)- }- return out-}--func anySlice(value any) []any {- switch typed := value.(type) {- case []any:- return typed- case []map[string]any:- out := make([]any, 0, len(typed))- for _, item := range typed {- out = append(out, item)- }- return out- case []map[string]string:- out := make([]any, 0, len(typed))- for _, item := range typed {- m := make(map[string]any, len(item))- for k, v := range item {- m[k] = v- }- out = append(out, m)- }- return out- case []MetadataConfigField:- out := make([]any, 0, len(typed))- for _, item := range typed {- out = append(out, map[string]any{- "key": item.Key,- "type": item.Type,- "description": item.Description,- "enum": item.Enum,- })- }- return out- default:- return nil- }-}--func hasMetadataConfigShape(parserConfig entity.JSONMap) bool {- if parserConfig == nil {- return false- }- if _, ok := parserConfig["enable_metadata"]; ok {- return true- }- if _, ok := parserConfig["enabled"]; ok {- return true- }- for _, key := range []string{"fields", "metadata", "built_in_metadata"} {- if anySlice(parserConfig[key]) != nil {- return true- }- }- if _, ok := parserConfig["metadata"].(map[string]any); ok {- return true- }- return false+ return parserConfig}-func parserConfigTruthy(value any) bool {- switch typed := value.(type) {- case bool:- return typed- case string:- switch typed {- case "true", "True", "TRUE", "1":- return true- }- case float64:- return typed > 0- case int:- return typed > 0- case int64:- return typed > 0+func cloneJSONMap(in map[string]any) map[string]any {+ out := make(map[string]any, len(in))+ for k, v := range in {+ out[k] = v}- return false+ return out}diff --git a/internal/service/component_scoped_parser_config_test.go b/internal/service/component_scoped_parser_config_test.goindex 0ffb3a6248..e2102e1165 100644--- a/internal/service/component_scoped_parser_config_test.go+++ b/internal/service/component_scoped_parser_config_test.go@@ -7,31 +7,27 @@ import ("ragflow/internal/entity")-func TestApplyComponentScopedParserConfig_SyncsExtractorAndCompiler(t *testing.T) {+func TestApplyComponentScopedParserConfig_ScopesDatasetMetadata(t *testing.T) {parserConfig := entity.JSONMap{- "enable_metadata": true,- "metadata": []any{- map[string]any{"key": "author", "type": "string"},- },- "built_in_metadata": []any{- map[string]any{"key": "document_name", "type": "string"},+ "metadata": map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]any{"key": "author", "type": "string"},+ },+ "built_in_metadata": []any{+ map[string]any{"key": "document_name", "type": "string"},+ },},"Extractor:AutoExtractDefault": map[string]any{},"Compiler:KnownSwiftLions": map[string]any{},}- got := ApplyComponentScopedParserConfig(- parserConfig,- "llm-default",- )+ got := ApplyComponentScopedParserConfig(parserConfig, "llm-default")extractor := got["Extractor:AutoExtractDefault"].(map[string]any)if extractor["llm_id"] != "llm-default" {t.Fatalf("extractor llm_id = %#v, want llm-default", extractor["llm_id"])}- if _, ok := extractor["enable_metadata"]; ok {- t.Fatalf("extractor enable_metadata should not be present, got %#v", extractor["enable_metadata"])- }wantMetadata := map[string]any{"enabled": true,"metadata": []any{@@ -49,189 +45,124 @@ func TestApplyComponentScopedParserConfig_SyncsExtractorAndCompiler(t *testing.Tif compiler["llm_id"] != "llm-default" {t.Fatalf("compiler llm_id = %#v, want llm-default", compiler["llm_id"])}- if _, ok := compiler["embedding_model"]; ok {- t.Fatalf("compiler embedding_model = %#v, want absent", compiler["embedding_model"])- }- if _, ok := compiler["tenant_id"]; ok {- t.Fatalf("compiler tenant_id = %#v, want absent", compiler["tenant_id"])- }- if _, ok := compiler["dataset_id"]; ok {- t.Fatalf("compiler dataset_id = %#v, want absent", compiler["dataset_id"])- }-}--func TestApplyComponentScopedParserConfig_PreservesExplicitExtractorLLMID(t *testing.T) {- parserConfig := entity.JSONMap{- "Extractor:Custom": map[string]any{- "llm_id": "custom-llm",- },- }-- got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")- extractor := got["Extractor:Custom"].(map[string]any)- if extractor["llm_id"] != "custom-llm" {- t.Fatalf("extractor llm_id = %#v, want custom-llm", extractor["llm_id"])- }}-func TestApplyComponentScopedParserConfig_AcceptsTypedMetadataSlices(t *testing.T) {+func TestApplyComponentScopedParserConfig_StripsLegacyFlatFields(t *testing.T) {parserConfig := entity.JSONMap{"enable_metadata": true,- "metadata": []map[string]interface{}{- {"key": "author", "type": "string"},+ "built_in_metadata": []any{+ map[string]any{"key": "document_name", "type": "string"},},- "built_in_metadata": []map[string]interface{}{- {"key": "document_name", "type": "string"},+ "fields": []any{+ map[string]any{"key": "author", "type": "string"},+ },+ "Extractor:AutoExtractDefault": map[string]any{+ "enable_metadata": 1,+ "metadata_config": map[string]any{"enabled": true},+ "built_in_metadata": []any{map[string]any{"key": "stale", "type": "string"}},+ "fields": []any{map[string]any{"key": "stale", "type": "string"}},},- "Extractor:AutoExtractDefault": map[string]any{},}got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")+ for _, key := range []string{"enable_metadata", "built_in_metadata", "fields", "metadata_config"} {+ if _, ok := got[key]; ok {+ t.Fatalf("top-level %s should be stripped, got %#v", key, got[key])+ }+ }extractor := got["Extractor:AutoExtractDefault"].(map[string]any)-+ for _, key := range []string{"enable_metadata", "built_in_metadata", "fields", "metadata_config"} {+ if _, ok := extractor[key]; ok {+ t.Fatalf("node %s should be stripped, got %#v", key, extractor[key])+ }+ }wantMetadata := map[string]any{- "enabled": true,- "metadata": []any{- map[string]interface{}{"key": "author", "type": "string"},- },- "built_in_metadata": []any{- map[string]interface{}{"key": "document_name", "type": "string"},- },+ "enabled": false,+ "metadata": []any{},+ "built_in_metadata": []any{},}if !reflect.DeepEqual(extractor["metadata"], wantMetadata) {t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantMetadata)}}-func TestApplyComponentScopedParserConfig_ClearsExtractorMetadataWhenDisabled(t *testing.T) {+func TestApplyComponentScopedParserConfig_PreservesExplicitExtractorLLMID(t *testing.T) {parserConfig := entity.JSONMap{- "enable_metadata": false,- "metadata": []map[string]interface{}{},- "built_in_metadata": []map[string]interface{}{- {"key": "document_name", "type": "string"},- },- "Extractor:AutoExtractDefault": map[string]any{- "enable_metadata": 1,- "metadata": map[string]any{- "enabled": true,- "metadata": []any{- map[string]any{"key": "stale", "type": "string"},- },- },+ "Extractor:Custom": map[string]any{+ "llm_id": "custom-llm",},}got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")- extractor := got["Extractor:AutoExtractDefault"].(map[string]any)-- if _, ok := extractor["enable_metadata"]; ok {- t.Fatalf("extractor enable_metadata should not be present, got %#v", extractor["enable_metadata"])- }- wantMetadata := map[string]any{- "enabled": false,- "metadata": []any{},- "built_in_metadata": []any{- map[string]interface{}{"key": "document_name", "type": "string"},- },- }- if !reflect.DeepEqual(extractor["metadata"], wantMetadata) {- t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantMetadata)+ extractor := got["Extractor:Custom"].(map[string]any)+ if extractor["llm_id"] != "custom-llm" {+ t.Fatalf("extractor llm_id = %#v, want custom-llm", extractor["llm_id"])}}-func TestApplyComponentScopedParserConfig_DoesNotTreatDocumentMetadataValuesAsSchema(t *testing.T) {+func TestApplyComponentScopedParserConfig_NoDatasetMetadataDisablesNode(t *testing.T) {parserConfig := entity.JSONMap{- "metadata": map[string]interface{}{- "author": "Alice",- },"Extractor:AutoExtractDefault": map[string]any{},}got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")extractor := got["Extractor:AutoExtractDefault"].(map[string]any)-- wantMetadata := map[string]any{+ want := map[string]any{"enabled": false,"metadata": []any{},"built_in_metadata": []any{},}- if !reflect.DeepEqual(extractor["metadata"], wantMetadata) {- t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantMetadata)+ if !reflect.DeepEqual(extractor["metadata"], want) {+ t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], want)}}-func TestMergeMetadataFields_MergesFieldsAndBuiltIn(t *testing.T) {+func TestApplyComponentScopedParserConfig_DatasetMetadataIsAuthoritative(t *testing.T) {parserConfig := entity.JSONMap{- "metadata": []any{- map[string]any{"key": "author", "type": "string"},- },- "built_in_metadata": []any{- map[string]any{"key": "document_name", "type": "string"},+ "metadata": map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]any{"key": "dataset_field", "type": "string"},+ },+ "built_in_metadata": []any{},},- }-- got := mergeMetadataFields(parserConfig)- want := []any{- map[string]any{"key": "author", "type": "string"},- map[string]any{"key": "document_name", "type": "string"},- }- if !reflect.DeepEqual(got, want) {- t.Fatalf("mergeMetadataFields = %#v, want %#v", got, want)- }-}--func TestApplyComponentScopedParserConfig_PreservesNodeOnlyMetadataOnDatasetSave(t *testing.T) {- parserConfig := entity.JSONMap{"Extractor:CanvasNode": map[string]any{- "llm_id": "model-1","metadata": map[string]any{- "enabled": true,+ "enabled": false,"metadata": []any{- map[string]any{"key": "node_author", "type": "string"},- },- "built_in_metadata": []any{- map[string]any{"key": "node_doc_title", "type": "string"},+ map[string]any{"key": "node_field", "type": "string"},},+ "built_in_metadata": []any{},},},}got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")node := got["Extractor:CanvasNode"].(map[string]any)-meta, ok := node["metadata"].(map[string]any)if !ok {t.Fatalf("expected node metadata map, got %T: %#v", node["metadata"], node["metadata"])}if enabled, _ := meta["enabled"].(bool); !enabled {- t.Fatalf("expected node metadata.enabled = true, got false")+ t.Fatalf("expected dataset metadata enabled=true, got false")}-fields, ok := meta["metadata"].([]any)- if !ok || len(fields) != 1 {- t.Fatalf("expected 1 custom field preserved, got %#v", meta["metadata"])- }- if fields[0].(map[string]any)["key"] != "node_author" {- t.Fatalf("expected node_author preserved, got %#v", fields[0])- }- builtIn, ok := meta["built_in_metadata"].([]any)- if !ok || len(builtIn) != 1 {- t.Fatalf("expected 1 built-in field preserved, got %#v", meta["built_in_metadata"])- }- if builtIn[0].(map[string]any)["key"] != "node_doc_title" {- t.Fatalf("expected node_doc_title preserved, got %#v", builtIn[0])+ if !ok || len(fields) != 1 || fields[0].(map[string]any)["key"] != "dataset_field" {+ t.Fatalf("expected dataset_field to replace node_field, got %#v", meta["metadata"])}}-func TestApplyComponentScopedParserConfig_PreservesNodeOnlyMetadataConfigShape(t *testing.T) {+func TestApplyComponentScopedParserConfig_PreservesNodeOnlyMetadata(t *testing.T) {parserConfig := entity.JSONMap{- "enable_metadata": false,"Extractor:CanvasNode": map[string]any{"llm_id": "model-1",- "metadata_config": map[string]any{+ "metadata": map[string]any{"enabled": true,"metadata": []any{- map[string]any{"key": "custom_field", "type": "string"},+ map[string]any{"key": "node_author", "type": "string"},+ },+ "built_in_metadata": []any{+ map[string]any{"key": "node_doc_title", "type": "string"},},},},@@ -239,10 +170,6 @@ func TestApplyComponentScopedParserConfig_PreservesNodeOnlyMetadataConfigShape(tgot := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")node := got["Extractor:CanvasNode"].(map[string]any)-- if _, ok := node["metadata_config"]; ok {- t.Fatalf("metadata_config should be deleted from node, got %#v", node["metadata_config"])- }meta, ok := node["metadata"].(map[string]any)if !ok {t.Fatalf("expected node metadata map, got %T: %#v", node["metadata"], node["metadata"])@@ -251,10 +178,11 @@ func TestApplyComponentScopedParserConfig_PreservesNodeOnlyMetadataConfigShape(tt.Fatalf("expected node metadata.enabled = true, got false")}fields, ok := meta["metadata"].([]any)- if !ok || len(fields) != 1 {- t.Fatalf("expected 1 custom field preserved, got %#v", meta["metadata"])+ if !ok || len(fields) != 1 || fields[0].(map[string]any)["key"] != "node_author" {+ t.Fatalf("expected node_author preserved, got %#v", meta["metadata"])}- if fields[0].(map[string]any)["key"] != "custom_field" {- t.Fatalf("expected custom_field preserved, got %#v", fields[0])+ builtIn, ok := meta["built_in_metadata"].([]any)+ if !ok || len(builtIn) != 1 || builtIn[0].(map[string]any)["key"] != "node_doc_title" {+ t.Fatalf("expected node_doc_title preserved, got %#v", meta["built_in_metadata"])}}diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.goindex 28aac5e863..4dcc22b4f2 100644--- a/internal/service/dataset/helpers.go+++ b/internal/service/dataset/helpers.go@@ -253,19 +253,20 @@ func preserveDatasetParserConfigMetadata(next, existing entity.JSONMap, incomingif next == nil {next = entity.JSONMap{}}- for _, key := range []string{"fields", "metadata", "built_in_metadata", "enable_metadata"} {- if incoming != nil {- if value, ok := incoming[key]; ok {- next[key] = value- continue- }+ var mm map[string]any+ if incoming != nil {+ if v, ok := incoming["metadata"].(map[string]any); ok {+ mm = v}- if existing != nil {- if value, ok := existing[key]; ok {- next[key] = value- }+ }+ if mm == nil && existing != nil {+ if v, ok := existing["metadata"].(map[string]any); ok {+ mm = v}}+ if mm != nil {+ next["metadata"] = mm+ }return next}diff --git a/internal/service/dataset/helpers_test.go b/internal/service/dataset/helpers_test.goindex def8a42086..c714e4c53a 100644--- a/internal/service/dataset/helpers_test.go+++ b/internal/service/dataset/helpers_test.go@@ -4,6 +4,7 @@ import ("strings""testing"+ "ragflow/internal/entity""ragflow/internal/service")@@ -395,3 +396,57 @@ func TestNormalizeMetadataConfigFields_TrimsKey(t *testing.T) {t.Errorf("expected trimmed key 'my_field', got %v", result[0]["key"])}}++func TestPreserveDatasetParserConfigMetadata_FallsBackWhenIncomingNotMap(t *testing.T) {+ existing := entity.JSONMap{+ "metadata": map[string]any{+ "enabled": true,+ "metadata": []any{map[string]any{"key": "existing_field", "type": "string"}},+ "built_in_metadata": []any{},+ },+ }+ cases := map[string]interface{}{+ "null": nil,+ "array": []any{},+ }+ for name, incomingMetadata := range cases {+ t.Run(name, func(t *testing.T) {+ incoming := map[string]interface{}{"metadata": incomingMetadata}+ got := preserveDatasetParserConfigMetadata(entity.JSONMap{}, existing, incoming)+ meta, ok := got["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("expected existing modular metadata preserved, got %#v", got["metadata"])+ }+ fields, ok := meta["metadata"].([]any)+ if !ok || len(fields) != 1 || fields[0].(map[string]any)["key"] != "existing_field" {+ t.Fatalf("expected existing_field preserved, got %#v", meta["metadata"])+ }+ })+ }+}++func TestPreserveDatasetParserConfigMetadata_UsesValidIncomingMap(t *testing.T) {+ existing := entity.JSONMap{+ "metadata": map[string]any{+ "enabled": false,+ "metadata": []any{},+ "built_in_metadata": []any{},+ },+ }+ incoming := map[string]interface{}{+ "metadata": map[string]any{+ "enabled": true,+ "metadata": []any{map[string]any{"key": "incoming_field", "type": "string"}},+ "built_in_metadata": []any{},+ },+ }+ got := preserveDatasetParserConfigMetadata(entity.JSONMap{}, existing, incoming)+ meta, ok := got["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("expected modular metadata map, got %#v", got["metadata"])+ }+ fields, ok := meta["metadata"].([]any)+ if !ok || len(fields) != 1 || fields[0].(map[string]any)["key"] != "incoming_field" {+ t.Fatalf("expected incoming_field to be used, got %#v", meta["metadata"])+ }+}diff --git a/internal/service/dataset/metadata.go b/internal/service/dataset/metadata.goindex 13c929fee0..f3243345d8 100644--- a/internal/service/dataset/metadata.go+++ b/internal/service/dataset/metadata.go@@ -38,14 +38,22 @@ func (d *DatasetService) UpdateDocumentMetadataConfig(ctx context.Context, userIparserConfig = entity.JSONMap{}}parserConfig["metadata"] = metadata- if kb, kbErr := d.kbDAO.GetByID(ctx, dao.DB, datasetID); kbErr == nil && kb != nil {- if tenant, tenantErr := d.tenantDAO.GetByID(ctx, dao.DB, kb.TenantID); tenantErr == nil && tenant != nil {- parserConfig = service.ApplyComponentScopedParserConfig(- parserConfig,- tenant.LLMID,- )- }+ kb, kbErr := d.kbDAO.GetByID(ctx, dao.DB, datasetID)+ if kbErr != nil {+ return nil, common.CodeServerError, errors.New("database operation failed")+ }+ if kb == nil {+ return nil, common.CodeServerError, errors.New("database operation failed")+ }+ tenant, tenantErr := d.tenantDAO.GetByID(ctx, dao.DB, kb.TenantID)+ if tenantErr != nil {+ return nil, common.CodeServerError, errors.New("database operation failed")}+ llmID := ""+ if tenant != nil {+ llmID = tenant.LLMID+ }+ parserConfig = service.ApplyComponentScopedParserConfig(parserConfig, llmID)if err = d.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{"parser_config": parserConfig}); err != nil {return nil, common.CodeServerError, errors.New("database operation failed")@@ -77,11 +85,12 @@ func (d *DatasetService) GetMetadataConfig(ctx context.Context, datasetID, tenanreturn nil, common.CodeDataError, fmt.Errorf("user '%s' lacks permission for dataset '%s'", tenantID, datasetID)}- metadata := parserConfigValueOrEmptyList(kb.ParserConfig, "metadata")+ _, enabled, metadata, builtInMetadata := modularMetadataConfig(kb.ParserConfig)return map[string]interface{}{+ "enabled": enabled,"metadata": metadata,- "built_in_metadata": parserConfigValueOrEmptyList(kb.ParserConfig, "built_in_metadata"),+ "built_in_metadata": builtInMetadata,}, common.CodeSuccess, nil}@@ -122,33 +131,77 @@ func (d *DatasetService) UpdateMetadataConfig(ctx context.Context, datasetID, teif parserConfig == nil {parserConfig = entity.JSONMap{}}- parserConfig["metadata"] = metadata- parserConfig["built_in_metadata"] = builtInMetadata- // enable_metadata state machine:- // - If req.Enabled is explicitly set: use *req.Enabled.- // - If nil (uninitialized dataset): auto-enable if metadata fields exist.- // - If false (explicitly disabled by user): preserve false even when fields are added.- // - If fields are empty: auto-disable to avoid invoking extractor on empty schema.- if req.Enabled != nil {- parserConfig["enable_metadata"] = *req.Enabled- } else if parserConfig["enable_metadata"] == nil {- parserConfig["enable_metadata"] = len(metadata) > 0 || len(builtInMetadata) > 0- } else if len(metadata) == 0 && len(builtInMetadata) == 0 {- parserConfig["enable_metadata"] = false- }- if tenant, tenantErr := d.tenantDAO.GetByID(ctx, dao.DB, kb.TenantID); tenantErr == nil && tenant != nil {- parserConfig = service.ApplyComponentScopedParserConfig(- parserConfig,- tenant.LLMID,- )+ present, currentEnabled, _, _ := modularMetadataConfig(parserConfig)+ enabled := false+ switch {+ case req.Enabled != nil:+ enabled = *req.Enabled+ case present:+ enabled = currentEnabled+ default:+ enabled = len(metadata) > 0 || len(builtInMetadata) > 0+ }+ if req.Enabled == nil && len(metadata) == 0 && len(builtInMetadata) == 0 {+ enabled = false+ }+ parserConfig["metadata"] = map[string]any{+ "enabled": enabled,+ "metadata": metadata,+ "built_in_metadata": builtInMetadata,+ }+ delete(parserConfig, "enable_metadata")+ delete(parserConfig, "built_in_metadata")+ tenant, tenantErr := d.tenantDAO.GetByID(ctx, dao.DB, kb.TenantID)+ if tenantErr != nil {+ return nil, common.CodeServerError, errors.New("database operation failed")}+ llmID := ""+ if tenant != nil {+ llmID = tenant.LLMID+ }+ parserConfig = service.ApplyComponentScopedParserConfig(parserConfig, llmID)if err = d.kbDAO.UpdateByID(ctx, dao.DB, kb.ID, map[string]interface{}{"parser_config": parserConfig}); err != nil {return nil, common.CodeServerError, errors.New("update auto-metadata error.(Database error)")}return map[string]interface{}{+ "enabled": enabled,"metadata": metadata,"built_in_metadata": builtInMetadata,}, common.CodeSuccess, nil}++// modularMetadataConfig reads the modular dataset-level metadata object+// ({"enabled", "metadata", "built_in_metadata"}) from parser_config. Missing+// or malformed config yields a disabled, empty result.+func modularMetadataConfig(parserConfig map[string]any) (bool, bool, []any, []any) {+ if parserConfig == nil {+ return false, false, []any{}, []any{}+ }+ metaObj, ok := parserConfig["metadata"].(map[string]any)+ if !ok {+ return false, false, []any{}, []any{}+ }+ enabled, _ := metaObj["enabled"].(bool)+ metadata := anyOrEmptyList(metaObj["metadata"])+ builtIn := anyOrEmptyList(metaObj["built_in_metadata"])+ return true, enabled, metadata, builtIn+}++func anyOrEmptyList(value any) []any {+ if value == nil {+ return []any{}+ }+ if list, ok := value.([]any); ok {+ return list+ }+ if list, ok := value.([]map[string]any); ok {+ out := make([]any, 0, len(list))+ for _, item := range list {+ out = append(out, item)+ }+ return out+ }+ return []any{}+}diff --git a/internal/service/dataset/metadata_config_test.go b/internal/service/dataset/metadata_config_test.goindex e5074c251b..ae6aa37599 100644--- a/internal/service/dataset/metadata_config_test.go+++ b/internal/service/dataset/metadata_config_test.go@@ -102,6 +102,7 @@ func insertDatasetMetadataConfigDoc(t *testing.T, docID, datasetID string, parsefunc TestDatasetServiceUpdateDocumentMetadataConfig(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)+ insertCreateDatasetTenant(t, "user-1")insertDatasetMetadataConfigKB(t, "kb-1", "user-1")insertDatasetMetadataConfigDoc(t, "doc-1", "kb-1", entity.JSONMap{"pages": []interface{}{1, 2}})@@ -197,6 +198,7 @@ func TestDatasetServiceUpdateDocumentMetadataConfigRejectsNonOwner(t *testing.T)func TestDatasetServiceUpdateDocumentMetadataConfigAllowsTeamMember(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)+ insertCreateDatasetTenant(t, "owner-1")insertDatasetMetadataConfigKB(t, "kb-1", "owner-1")if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").@@ -233,13 +235,18 @@ func TestDatasetServiceUpdateMetadataConfigSyncsExtractorSchema(t *testing.T) {if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", entity.JSONMap{- "enable_metadata": false,+ "metadata": map[string]any{+ "enabled": false,+ "metadata": []any{},+ "built_in_metadata": []any{},+ },"Extractor:AutoExtractDefault": map[string]any{"metadata": map[string]any{"enabled": true,- "fields": []any{+ "metadata": []any{map[string]any{"key": "stale", "type": "string"},},+ "built_in_metadata": []any{},},},}).Error; err != nil {@@ -284,14 +291,18 @@ func TestDatasetServiceUpdateMetadataConfigSyncsExtractorSchema(t *testing.T) {t.Fatalf("expected modular metadata map in extractor, got %#v", extractor["metadata"])}if enabled, ok := extractorMeta["enabled"].(bool); !ok || enabled {- t.Fatalf("extractor metadata.enabled = %#v, want false when top-level flag stays disabled", extractorMeta["enabled"])+ t.Fatalf("extractor metadata.enabled = %#v, want false when dataset-level flag stays disabled", extractorMeta["enabled"])}// State 2: When enabled, updating fields updates component with enabled = trueif err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", entity.JSONMap{- "enable_metadata": true,+ "metadata": map[string]any{+ "enabled": true,+ "metadata": []any{},+ "built_in_metadata": []any{},+ },"Extractor:AutoExtractDefault": map[string]any{},}).Error; err != nil {t.Fatalf("reset parser_config: %v", err)@@ -363,8 +374,15 @@ func TestDatasetServiceUpdateMetadataConfigSyncsExtractorSchema(t *testing.T) {if err != nil {t.Fatalf("failed to fetch persisted dataset: %v", err)}- if enabled, ok := persisted.ParserConfig["enable_metadata"].(bool); !ok || enabled {- t.Fatalf("expected parser_config.enable_metadata == false after emptying fields, got %#v", persisted.ParserConfig["enable_metadata"])+ if _, ok := persisted.ParserConfig["enable_metadata"]; ok {+ t.Fatalf("enable_metadata should be absent, got %#v", persisted.ParserConfig["enable_metadata"])+ }+ metaObj, ok := persisted.ParserConfig["metadata"].(map[string]interface{})+ if !ok {+ t.Fatalf("expected modular metadata map, got %#v", persisted.ParserConfig["metadata"])+ }+ if enabled, ok := metaObj["enabled"].(bool); !ok || enabled {+ t.Fatalf("expected modular metadata.enabled == false after emptying fields, got %#v", metaObj["enabled"])}}@@ -420,3 +438,97 @@ func TestDatasetServiceGetMetadataConfig(t *testing.T) {t.Fatalf("expected key file_name, got %#v", builtIn[0])}}++func TestUpdateMetadataConfig_TenantLookupFailureDoesNotPersist(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ // kb-1 points at a tenant that does not exist, so tenantDAO.GetByID fails.+ insertDatasetMetadataConfigKB(t, "kb-1", "missing-tenant")++ ctx := t.Context()+ result, code, err := (&DatasetService{+ kbDAO: dao.NewKnowledgebaseDAO(),+ tenantDAO: dao.NewTenantDAO(),+ }).UpdateMetadataConfig(ctx, "kb-1", "missing-tenant", &service.MetadataConfigRequest{+ Metadata: []service.MetadataConfigField{+ {Key: "author", Type: "string"},+ },+ BuiltInMetadata: []service.MetadataConfigField{},+ })+ if code != common.CodeServerError || err == nil {+ t.Fatalf("expected server error on tenant lookup failure, got code=%d err=%v result=%#v", code, err, result)+ }++ persisted, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")+ if err != nil {+ t.Fatalf("get kb: %v", err)+ }+ if _, ok := persisted.ParserConfig["metadata"]; ok {+ t.Fatalf("parser config must not be persisted on tenant lookup failure, got %#v", persisted.ParserConfig)+ }+}++func TestUpdateMetadataConfig_ExplicitEnabledPreservedWhenEmpty(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertCreateDatasetTenant(t, "tenant-1")+ insertDatasetMetadataConfigKB(t, "kb-1", "tenant-1")++ enabled := true+ ctx := t.Context()+ result, code, err := (&DatasetService{+ kbDAO: dao.NewKnowledgebaseDAO(),+ tenantDAO: dao.NewTenantDAO(),+ }).UpdateMetadataConfig(ctx, "kb-1", "tenant-1", &service.MetadataConfigRequest{+ Enabled: &enabled,+ Metadata: []service.MetadataConfigField{},+ BuiltInMetadata: []service.MetadataConfigField{},+ })+ if err != nil || code != common.CodeSuccess {+ t.Fatalf("UpdateMetadataConfig failed: code=%d err=%v", code, err)+ }+ respEnabled, ok := result["enabled"].(bool)+ if !ok || !respEnabled {+ t.Fatalf("expected enabled=true in response, got %#v", result["enabled"])+ }++ persisted, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")+ if err != nil {+ t.Fatalf("get kb: %v", err)+ }+ metaObj, ok := persisted.ParserConfig["metadata"].(map[string]interface{})+ if !ok {+ t.Fatalf("expected modular metadata, got %#v", persisted.ParserConfig["metadata"])+ }+ if persistedEnabled, _ := metaObj["enabled"].(bool); !persistedEnabled {+ t.Fatalf("expected metadata.enabled=true, got %#v", metaObj["enabled"])+ }+}++func TestUpdateDocumentMetadataConfig_KBAndTenantLookupFailure(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertDatasetMetadataConfigKB(t, "kb-ok", "owner-1")+ insertDatasetMetadataConfigDoc(t, "doc-ok", "kb-ok", entity.JSONMap{})+ ctx := t.Context()++ // Missing KB -> database layer should surface as CodeServerError / not found+ _, code, err := testDatasetServiceForDocumentMetadataConfig(t).UpdateDocumentMetadataConfig(+ ctx, "owner-1", "kb-missing", "doc-ok",+ map[string]interface{}{"metadata": map[string]interface{}{"a": "b"}},+ )+ if code == common.CodeSuccess || err == nil {+ t.Fatalf("expected error for missing kb, got code=%v err=%v", code, err)+ }++ // KB exists but tenant lookup fails (kb.TenantID points to missing tenant)+ insertDatasetMetadataConfigKB(t, "kb-bad-tenant", "missing-tenant")+ insertDatasetMetadataConfigDoc(t, "doc2", "kb-bad-tenant", entity.JSONMap{})+ _, code, err = testDatasetServiceForDocumentMetadataConfig(t).UpdateDocumentMetadataConfig(+ ctx, "missing-tenant", "kb-bad-tenant", "doc2",+ map[string]interface{}{"metadata": map[string]interface{}{"a": "b"}},+ )+ if code != common.CodeServerError || err == nil {+ t.Fatalf("expected server error for tenant lookup failure, got code=%v err=%v", code, err)+ }+}diff --git a/internal/service/dataset/update_test.go b/internal/service/dataset/update_test.goindex dd2b396112..bcea8c9032 100644--- a/internal/service/dataset/update_test.go+++ b/internal/service/dataset/update_test.go@@ -1067,9 +1067,11 @@ func TestUpdateDataset_PreservesIncomingMetadataWhenCleaningParserConfig(t *test"Parser:HipSignsRhyme": map[string]interface{}{"pdf": map[string]interface{}{"parse_method": "deepdoc"},},- "metadata": incomingMetadata,- "built_in_metadata": incomingBuiltInMetadata,- "enable_metadata": true,+ "metadata": map[string]interface{}{+ "enabled": true,+ "metadata": incomingMetadata,+ "built_in_metadata": incomingBuiltInMetadata,+ },}_, code, err := testDatasetUpdateService(t).UpdateDataset(t.Context(), "kb-1", "tenant-1", service.UpdateDatasetRequest{@@ -1084,14 +1086,18 @@ func TestUpdateDataset_PreservesIncomingMetadataWhenCleaningParserConfig(t *testif err != nil {t.Fatalf("get updated kb: %v", err)}- if !reflect.DeepEqual(persisted.ParserConfig["metadata"], incomingMetadata) {- t.Fatalf("metadata was not preserved: %#v", persisted.ParserConfig["metadata"])+ if !reflect.DeepEqual(persisted.ParserConfig["metadata"], map[string]interface{}{+ "enabled": true,+ "metadata": incomingMetadata,+ "built_in_metadata": incomingBuiltInMetadata,+ }) {+ t.Fatalf("modular metadata was not preserved: %#v", persisted.ParserConfig["metadata"])}- if !reflect.DeepEqual(persisted.ParserConfig["built_in_metadata"], incomingBuiltInMetadata) {- t.Fatalf("built_in_metadata was not preserved: %#v", persisted.ParserConfig["built_in_metadata"])+ if _, ok := persisted.ParserConfig["enable_metadata"]; ok {+ t.Fatalf("enable_metadata should be absent: %#v", persisted.ParserConfig["enable_metadata"])}- if persisted.ParserConfig["enable_metadata"] != true {- t.Fatalf("enable_metadata was not preserved: %#v", persisted.ParserConfig["enable_metadata"])+ if _, ok := persisted.ParserConfig["built_in_metadata"]; ok {+ t.Fatalf("built_in_metadata should be absent: %#v", persisted.ParserConfig["built_in_metadata"])}if _, ok := persisted.ParserConfig["Parser:HipSignsRhyme"].(map[string]interface{}); !ok {t.Fatalf("component parser_config missing: %#v", persisted.ParserConfig)@@ -1107,14 +1113,12 @@ func TestUpdateDataset_PreservesExistingMetadataWhenParserConfigOmitsIt(t *testi"key": "category","type": "string",}}- existingBuiltInMetadata := []interface{}{map[string]interface{}{- "key": "document_name",- "type": "string",- }}if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", entity.JSONMap{- "metadata": existingMetadata,- "built_in_metadata": existingBuiltInMetadata,- "enable_metadata": true,+ "metadata": map[string]interface{}{+ "enabled": true,+ "metadata": existingMetadata,+ "built_in_metadata": []interface{}{map[string]interface{}{"key": "document_name", "type": "string"}},+ },}).Error; err != nil {t.Fatalf("seed parser_config: %v", err)}@@ -1135,14 +1139,18 @@ func TestUpdateDataset_PreservesExistingMetadataWhenParserConfigOmitsIt(t *testiif err != nil {t.Fatalf("get updated kb: %v", err)}- if !reflect.DeepEqual(persisted.ParserConfig["metadata"], existingMetadata) {- t.Fatalf("existing metadata was not preserved: %#v", persisted.ParserConfig["metadata"])+ if !reflect.DeepEqual(persisted.ParserConfig["metadata"], map[string]interface{}{+ "enabled": true,+ "metadata": existingMetadata,+ "built_in_metadata": []interface{}{map[string]interface{}{"key": "document_name", "type": "string"}},+ }) {+ t.Fatalf("existing modular metadata was not preserved: %#v", persisted.ParserConfig["metadata"])}- if !reflect.DeepEqual(persisted.ParserConfig["built_in_metadata"], existingBuiltInMetadata) {- t.Fatalf("existing built_in_metadata was not preserved: %#v", persisted.ParserConfig["built_in_metadata"])+ if _, ok := persisted.ParserConfig["enable_metadata"]; ok {+ t.Fatalf("enable_metadata should be absent: %#v", persisted.ParserConfig["enable_metadata"])}- if persisted.ParserConfig["enable_metadata"] != true {- t.Fatalf("existing enable_metadata was not preserved: %#v", persisted.ParserConfig["enable_metadata"])+ if _, ok := persisted.ParserConfig["built_in_metadata"]; ok {+ t.Fatalf("built_in_metadata should be absent: %#v", persisted.ParserConfig["built_in_metadata"])}}diff --git a/test/testcases/restful_api/test_datasets.py b/test/testcases/restful_api/test_datasets.pyindex b1f8d70cad..f67b97f71d 100644--- a/test/testcases/restful_api/test_datasets.py+++ b/test/testcases/restful_api/test_datasets.py@@ -2287,7 +2287,8 @@ def test_dataset_metadata_config_get_and_update_contract(rest_client, create_datassert success_res.status_code == 200success_payload = success_res.json()assert success_payload["code"] == 0, success_payload- assert success_payload["data"] == {"metadata": [], "built_in_metadata": []}, success_payload+ expected_empty = {"enabled": False, "metadata": [], "built_in_metadata": []} if IS_GO_PROXY else {"metadata": [], "built_in_metadata": []}+ assert success_payload["data"] == expected_empty, success_payloadfor scenario_name, client in (("missing token", RestClient(token=None)), ("invalid token", RestClient(token=INVALID_API_TOKEN))):get_res = client.get(f"/datasets/{dataset_id}/metadata/config")@@ -2327,6 +2328,8 @@ def test_dataset_metadata_config_get_and_update_contract(rest_client, create_dat{"key": "size", "type": "number", "description": "File size", "enum": None},],}+ if IS_GO_PROXY:+ normalized_update_payload["enabled"] = Trueupdate_res = rest_client.put(f"/datasets/{dataset_id}/metadata/config", json=update_payload)assert update_res.status_code == 200update_body = update_res.json()@@ -2343,7 +2346,7 @@ def test_dataset_metadata_config_get_and_update_contract(rest_client, create_datassert missing_payload_res.status_code == 200missing_payload = missing_payload_res.json()assert missing_payload["code"] == 0, missing_payload- assert missing_payload["data"] == {"metadata": [], "built_in_metadata": []}, missing_payload+ assert missing_payload["data"] == expected_empty, missing_payloadinvalid_update_dataset_res = rest_client.put("/datasets/invalid_dataset_id/metadata/config",