RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: restrict metadata field key (#19999)
Lynnf3eed31decb5
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.goindex 64602a7016..d0e5a6af6a 100644--- a/internal/ingestion/component/extractor.go+++ b/internal/ingestion/component/extractor.go@@ -166,6 +166,7 @@ Propose questions about a given piece of text content.autoMetadataPrompt = `## Role: Metadata extraction expert.## Rules:+ - Fixed Key Set: Output ONLY the keys that appear in the Schema. Never add, rename, translate, pluralize or re-case a key, and never invent an extra key even if the Content clearly carries that information. Any key that is not in the Schema MUST be omitted.- Strict Evidence Only: Extract a value ONLY if it is explicitly mentioned in the Content.- Enum Filter: For any field with an 'enum' list, the list acts as a strict filter. If no element from the list (or its direct synonym) is found in the Content, you MUST NOT extract that field.- No Meta-Inference: Do not infer values based on the document's nature, format, or category. If the text does not literally state the information, treat it as missing.@@ -1059,6 +1060,11 @@ func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB,// Non-JSON or empty response — nothing to extract, not an error.return nil}+ parsed = filterMetadataToDeclaredKeys(parsed, c.Param.Metadata.Metadata)+ if len(parsed) == 0 {+ // Every key fell outside the declared field set — nothing to keep.+ return nil+ }setMetadataLLMCache(ctx, in.cache, modelID, schemaStr, chunkID, parsed)}// Merge into the chunk metadata map, preserving existing keys.@@ -1081,6 +1087,33 @@ func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB,return nil}+// filterMetadataToDeclaredKeys drops every key that is not part of the+// declared metadata field set.+//+// The rendered schema carries additionalProperties:false, but that is only a+// hint to the model: a model that renames, pluralizes or invents a field still+// returns a parseable object, and such a key would otherwise widen the+// dataset's metadata schema through mergeChunkMetadata. The declared set is+// therefore enforced here, in code.+func filterMetadataToDeclaredKeys(parsed map[string]any, fields []common.MetadataFieldDef) map[string]any {+ allowed := make(map[string]struct{}, len(fields))+ for _, f := range fields {+ allowed[f.Key] = struct{}{}+ }+ kept := make(map[string]any, len(parsed))+ for k, v := range parsed {+ if _, ok := allowed[k]; !ok {+ common.Debug("extractor stage",+ zap.String("component", componentNameExtractor),+ zap.String("dropped_metadata_key", k),+ )+ continue+ }+ kept[k] = v+ }+ return kept+}+// extractorChunkText resolves the body an extraction is run against.//// Canonical text wins. Media-only chunks may use their explicit surroundingdiff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.goindex 9c9ab461c0..b961329963 100644--- a/internal/ingestion/component/extractor_test.go+++ b/internal/ingestion/component/extractor_test.go@@ -603,6 +603,49 @@ func TestExtractorComponent_runEnableMetadata_MergesIntoChunkMetadata(t *testing}}+// TestExtractorComponent_runEnableMetadata_DropsUndeclaredKeys verifies a model+// that invents or re-cases a field cannot widen the dataset's metadata schema:+// only keys from the declared field set survive.+func TestExtractorComponent_runEnableMetadata_DropsUndeclaredKeys(t *testing.T) {+ withStubChatInvoker(t, stubResponse{Content: `{"category":"finance","authors":["李白"],"Category":"finance"}`})+ c := newMetadataExtractor(common.MetadataFieldDef{Key: "category", Type: "string"})+ ck := map[string]any{}+ if err := c.runEnableMetadata(t.Context(), nil, extractorInputs{llmID: "m"}, ck, "chunk text"); err != nil {+ t.Fatalf("runEnableMetadata: %v", err)+ }+ meta, ok := ck["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("ck[metadata] missing or wrong type: %T", ck["metadata"])+ }+ if meta["category"] != "finance" {+ t.Errorf("metadata[category] = %v, want finance", meta["category"])+ }+ if _, leaked := meta["authors"]; leaked {+ t.Errorf("undeclared key authors leaked into metadata: %v", meta)+ }+ if _, leaked := meta["Category"]; leaked {+ t.Errorf("re-cased key Category leaked into metadata: %v", meta)+ }+}++// TestExtractorComponent_runEnableMetadata_AllKeysUndeclared verifies a response+// whose every key falls outside the declared set leaves existing metadata intact.+func TestExtractorComponent_runEnableMetadata_AllKeysUndeclared(t *testing.T) {+ withStubChatInvoker(t, stubResponse{Content: `{"authors":["李白"]}`})+ c := newMetadataExtractor(common.MetadataFieldDef{Key: "category", Type: "string"})+ ck := map[string]any{"metadata": map[string]any{"preexisting": "keep"}}+ if err := c.runEnableMetadata(t.Context(), nil, extractorInputs{llmID: "m"}, ck, "chunk text"); err != nil {+ t.Fatalf("runEnableMetadata: %v", err)+ }+ meta, ok := ck["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("ck[metadata] missing or wrong type: %T", ck["metadata"])+ }+ if len(meta) != 1 || meta["preexisting"] != "keep" {+ t.Errorf("metadata = %v, want only preexisting=keep", meta)+ }+}+// TestExtractorComponent_runEnableMetadata_StripsJSONFence verifies the// extraction path tolerates a fenced ```json response.func TestExtractorComponent_runEnableMetadata_StripsJSONFence(t *testing.T) {diff --git a/rag/prompts/meta_data.md b/rag/prompts/meta_data.mdindex d976b2cef9..3041290643 100644--- a/rag/prompts/meta_data.md+++ b/rag/prompts/meta_data.md@@ -1,14 +1,15 @@## Role: Metadata extraction expert.## Rules:- - Strict Evidence Only: Extract a value ONLY if it is explicitly mentioned in the Content.+ - Fixed Key Set: Output ONLY the keys that appear in the Schema. Never add, rename, translate, pluralize or re-case a key, and never invent an extra key even if the Content clearly carries that information. Any key that is not in the Schema MUST be omitted.+ - Strict Evidence Only: Extract a value ONLY if it is explicitly mentioned in the Content.- Enum Filter: For any field with an 'enum' list, the list acts as a strict filter. If no element from the list (or its direct synonym) is found in the Content, you MUST NOT extract that field.- No Meta-Inference: Do not infer values based on the document's nature, format, or category. If the text does not literally state the information, treat it as missing.- Zero-Hallucination: Never invent information or pick a "likely" value from the enum to fill a field.- - Empty Result: If no matches are found for any field, or if the content is irrelevant, output ONLY {}.+ - Empty Result: If no matches are found for any field, or if the content is irrelevant, output ONLY {}.- Output: ONLY a valid JSON string. No Markdown, no notes.## Schema for extraction:{{ schema }}## Content to analyze:-{{ content }}\ No newline at end of file+{{ content }}