Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: restrict metadata field key (#19999)

Lynnf3eed31decb5
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go
index 64602a7016..d0e5a6af6a 100644
--- a/internal/ingestion/component/extractor.go
+++ b/internal/ingestion/component/extractor.go
@@ -166,6 +166,7 @@ Propose questions about a given piece of text content.
autoMetadataPrompt = `## Role: Metadata extraction expert.
## Rules:
+ - Fixed Key Set: Output ONLY the keys that appear in the Schema. Never add, rename, translate, pluralize or re-case a key, and never invent an extra key even if the Content clearly carries that information. Any key that is not in the Schema MUST be omitted.
- Strict Evidence Only: Extract a value ONLY if it is explicitly mentioned in the Content.
- Enum Filter: For any field with an 'enum' list, the list acts as a strict filter. If no element from the list (or its direct synonym) is found in the Content, you MUST NOT extract that field.
- No Meta-Inference: Do not infer values based on the document's nature, format, or category. If the text does not literally state the information, treat it as missing.
@@ -1059,6 +1060,11 @@ func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB,
// Non-JSON or empty response — nothing to extract, not an error.
return nil
}
+ parsed = filterMetadataToDeclaredKeys(parsed, c.Param.Metadata.Metadata)
+ if len(parsed) == 0 {
+ // Every key fell outside the declared field set — nothing to keep.
+ return nil
+ }
setMetadataLLMCache(ctx, in.cache, modelID, schemaStr, chunkID, parsed)
}
// Merge into the chunk metadata map, preserving existing keys.
@@ -1081,6 +1087,33 @@ func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB,
return nil
}
+// filterMetadataToDeclaredKeys drops every key that is not part of the
+// declared metadata field set.
+//
+// The rendered schema carries additionalProperties:false, but that is only a
+// hint to the model: a model that renames, pluralizes or invents a field still
+// returns a parseable object, and such a key would otherwise widen the
+// dataset's metadata schema through mergeChunkMetadata. The declared set is
+// therefore enforced here, in code.
+func filterMetadataToDeclaredKeys(parsed map[string]any, fields []common.MetadataFieldDef) map[string]any {
+ allowed := make(map[string]struct{}, len(fields))
+ for _, f := range fields {
+ allowed[f.Key] = struct{}{}
+ }
+ kept := make(map[string]any, len(parsed))
+ for k, v := range parsed {
+ if _, ok := allowed[k]; !ok {
+ common.Debug("extractor stage",
+ zap.String("component", componentNameExtractor),
+ zap.String("dropped_metadata_key", k),
+ )
+ continue
+ }
+ kept[k] = v
+ }
+ return kept
+}
+
// extractorChunkText resolves the body an extraction is run against.
//
// Canonical text wins. Media-only chunks may use their explicit surrounding
diff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.go
index 9c9ab461c0..b961329963 100644
--- a/internal/ingestion/component/extractor_test.go
+++ b/internal/ingestion/component/extractor_test.go
@@ -603,6 +603,49 @@ func TestExtractorComponent_runEnableMetadata_MergesIntoChunkMetadata(t *testing
}
}
+// TestExtractorComponent_runEnableMetadata_DropsUndeclaredKeys verifies a model
+// that invents or re-cases a field cannot widen the dataset's metadata schema:
+// only keys from the declared field set survive.
+func TestExtractorComponent_runEnableMetadata_DropsUndeclaredKeys(t *testing.T) {
+ withStubChatInvoker(t, stubResponse{Content: `{"category":"finance","authors":["李白"],"Category":"finance"}`})
+ c := newMetadataExtractor(common.MetadataFieldDef{Key: "category", Type: "string"})
+ ck := map[string]any{}
+ if err := c.runEnableMetadata(t.Context(), nil, extractorInputs{llmID: "m"}, ck, "chunk text"); err != nil {
+ t.Fatalf("runEnableMetadata: %v", err)
+ }
+ meta, ok := ck["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("ck[metadata] missing or wrong type: %T", ck["metadata"])
+ }
+ if meta["category"] != "finance" {
+ t.Errorf("metadata[category] = %v, want finance", meta["category"])
+ }
+ if _, leaked := meta["authors"]; leaked {
+ t.Errorf("undeclared key authors leaked into metadata: %v", meta)
+ }
+ if _, leaked := meta["Category"]; leaked {
+ t.Errorf("re-cased key Category leaked into metadata: %v", meta)
+ }
+}
+
+// TestExtractorComponent_runEnableMetadata_AllKeysUndeclared verifies a response
+// whose every key falls outside the declared set leaves existing metadata intact.
+func TestExtractorComponent_runEnableMetadata_AllKeysUndeclared(t *testing.T) {
+ withStubChatInvoker(t, stubResponse{Content: `{"authors":["李白"]}`})
+ c := newMetadataExtractor(common.MetadataFieldDef{Key: "category", Type: "string"})
+ ck := map[string]any{"metadata": map[string]any{"preexisting": "keep"}}
+ if err := c.runEnableMetadata(t.Context(), nil, extractorInputs{llmID: "m"}, ck, "chunk text"); err != nil {
+ t.Fatalf("runEnableMetadata: %v", err)
+ }
+ meta, ok := ck["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("ck[metadata] missing or wrong type: %T", ck["metadata"])
+ }
+ if len(meta) != 1 || meta["preexisting"] != "keep" {
+ t.Errorf("metadata = %v, want only preexisting=keep", meta)
+ }
+}
+
// TestExtractorComponent_runEnableMetadata_StripsJSONFence verifies the
// extraction path tolerates a fenced ```json response.
func TestExtractorComponent_runEnableMetadata_StripsJSONFence(t *testing.T) {
diff --git a/rag/prompts/meta_data.md b/rag/prompts/meta_data.md
index d976b2cef9..3041290643 100644
--- a/rag/prompts/meta_data.md
+++ b/rag/prompts/meta_data.md
@@ -1,14 +1,15 @@
## Role: Metadata extraction expert.
## Rules:
- - Strict Evidence Only: Extract a value ONLY if it is explicitly mentioned in the Content.
+ - Fixed Key Set: Output ONLY the keys that appear in the Schema. Never add, rename, translate, pluralize or re-case a key, and never invent an extra key even if the Content clearly carries that information. Any key that is not in the Schema MUST be omitted.
+ - Strict Evidence Only: Extract a value ONLY if it is explicitly mentioned in the Content.
- Enum Filter: For any field with an 'enum' list, the list acts as a strict filter. If no element from the list (or its direct synonym) is found in the Content, you MUST NOT extract that field.
- No Meta-Inference: Do not infer values based on the document's nature, format, or category. If the text does not literally state the information, treat it as missing.
- Zero-Hallucination: Never invent information or pick a "likely" value from the enum to fill a field.
- - Empty Result: If no matches are found for any field, or if the content is irrelevant, output ONLY {}.
+ - Empty Result: If no matches are found for any field, or if the content is irrelevant, output ONLY {}.
- Output: ONLY a valid JSON string. No Markdown, no notes.
## Schema for extraction:
{{ schema }}
## Content to analyze:
-{{ content }}
\ No newline at end of file
+{{ content }}