Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(ingestion): end-to-end Auto metadata in Go (extractor merge, builtin UI, built-in fields) (#18273)

jay77721452720a62dce
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go
index 01a5f92d54..6c1e3cb9de 100644
--- a/internal/ingestion/component/extractor.go
+++ b/internal/ingestion/component/extractor.go
@@ -671,7 +671,13 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
if callErr != nil {
return callErr
}
- in.chunks = []map[string]any{{in.fieldName: ans}}
+ ck := map[string]any{}
+ if in.fieldName == "metadata" {
+ mergeExtractionIntoMetadata(ck, ans)
+ } else {
+ ck[in.fieldName] = ans
+ }
+ in.chunks = []map[string]any{ck}
return nil
}
// Auto keyword / question / metadata extraction runs with
@@ -714,7 +720,11 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
if callErr != nil {
return fmt.Errorf("chunk %d: %w", i, callErr)
}
- ck[in.fieldName] = ans
+ if in.fieldName == "metadata" {
+ mergeExtractionIntoMetadata(ck, ans)
+ } else {
+ ck[in.fieldName] = ans
+ }
}
return nil
}); err != nil {
@@ -730,6 +740,29 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
}, nil
}
+// mergeExtractionIntoMetadata merges the field_name="metadata" extraction
+// result into the chunk's metadata map (enable_metadata's output) instead of
+// overwriting it. ck["metadata"] stays a map[string]any — the unified contract
+// for document metadata produced by this component. A non-JSON / empty result
+// is ignored (no string fallback), so it can never clobber or corrupt the
+// metadata map. On overlapping keys the field_name result wins (it runs later).
+func mergeExtractionIntoMetadata(ck map[string]any, ans string) {
+ parsed, ok := tryParseJSONObject(ans)
+ if !ok {
+ return
+ }
+ existing, _ := ck["metadata"].(map[string]any)
+ if existing == nil {
+ existing = make(map[string]any, len(parsed))
+ }
+ for k, v := range parsed {
+ if v != nil {
+ existing[k] = v
+ }
+ }
+ ck["metadata"] = existing
+}
+
// runAutoKeywords extracts keywords for the current chunk and stores
// them on ck["important_kwd"]. mu may be nil (sequential path); when
// non-nil it serializes the shared chunk-map accesses so concurrent
diff --git a/internal/ingestion/component/extractor_metadata_regression_test.go b/internal/ingestion/component/extractor_metadata_regression_test.go
new file mode 100644
index 0000000000..172f329a6f
--- /dev/null
+++ b/internal/ingestion/component/extractor_metadata_regression_test.go
@@ -0,0 +1,202 @@
+package component
+
+import (
+ "testing"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/ingestion/task/indexdoc"
+)
+
+// TestRepro_FieldNameMetadata_AggregatesMap guards the unified contract:
+// field_name="metadata" (the pipeline/canvas way to produce document metadata)
+// writes a parsed map into ck["metadata"] — not a raw string — so the
+// aggregation layer, which is strict on map, merges it into document metadata.
+func TestRepro_FieldNameMetadata_AggregatesMap(t *testing.T) {
+ withStubChatInvoker(t,
+ stubResponse{Content: `{"category":"finance","region":"east"}`},
+ )
+ c := &ExtractorComponent{Param: schema.ExtractorParam{
+ FieldName: "metadata",
+ LLMID: "gpt-4o-mini",
+ Prompt: "Extract metadata as JSON:",
+ }}
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "chunks": []map[string]any{{"text": "first text"}},
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := out["chunks"].([]map[string]any)
+ meta, ok := chunks[0]["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("chunk[0].metadata = %T, want map[string]any (field_name=metadata must produce a map, not a string)", chunks[0]["metadata"])
+ }
+ if meta["category"] != "finance" || meta["region"] != "east" {
+ t.Errorf("metadata = %v, want category=finance region=east", meta)
+ }
+
+ // Feed through the aggregation path, exactly as the executor does.
+ metadata, err := indexdoc.ProcessChunksForPipeline(chunks, "doc-1", "doc.pdf", time.Now())
+ if err != nil {
+ t.Fatalf("ProcessChunksForPipeline: %v", err)
+ }
+ if metadata["category"] != "finance" {
+ t.Errorf("category = %v, want finance", metadata["category"])
+ }
+ if metadata["region"] != "east" {
+ t.Errorf("region = %v, want east", metadata["region"])
+ }
+}
+
+// TestRepro_BothEnabled_MergesMetadata guards the unified contract: when both
+// enable_metadata (auto, schema-driven) and field_name="metadata" (pipeline
+// custom) are configured, the extractor MERGES them into one ck["metadata"]
+// map instead of the field_name result overwriting the auto-metadata. Both
+// contribute to the document metadata.
+func TestRepro_BothEnabled_MergesMetadata(t *testing.T) {
+ withStubChatInvoker(t,
+ // call 1: enable_metadata (runAutoExtractions)
+ stubResponse{Content: `{"category":"finance","region":"east"}`},
+ // call 2: field_name="metadata" primary extraction
+ stubResponse{Content: `{"author":"zhang","date":"2024-01-01"}`},
+ )
+ c := &ExtractorComponent{Param: schema.ExtractorParam{
+ FieldName: "metadata",
+ LLMID: "gpt-4o-mini",
+ Prompt: "Extract more metadata as JSON:",
+ EnableMetadata: 1,
+ Metadata: []common.MetadataFieldDef{
+ {Key: "category", Type: "string"},
+ {Key: "region", Type: "string"},
+ },
+ }}
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "chunks": []map[string]any{{"text": "content"}},
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := out["chunks"].([]map[string]any)
+ meta, ok := chunks[0]["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("chunk[0].metadata = %T, want merged map; auto-metadata was overwritten", chunks[0]["metadata"])
+ }
+ for _, k := range []string{"category", "region", "author", "date"} {
+ if _, has := meta[k]; !has {
+ t.Errorf("merged metadata missing %q: %v", k, meta)
+ }
+ }
+
+ metadata, err := indexdoc.ProcessChunksForPipeline(chunks, "doc-1", "doc.pdf", time.Now())
+ if err != nil {
+ t.Fatalf("ProcessChunksForPipeline: %v", err)
+ }
+ for _, k := range []string{"category", "region", "author", "date"} {
+ if _, has := metadata[k]; !has {
+ t.Errorf("aggregated doc metadata missing %q: %v", k, metadata)
+ }
+ }
+}
+
+// TestRepro_OverlapFieldNameWins pins the overlap rule: when enable_metadata
+// and field_name="metadata" extract the same key, the field_name (custom,
+// later) result wins.
+func TestRepro_OverlapFieldNameWins(t *testing.T) {
+ withStubChatInvoker(t,
+ // call 1: enable_metadata
+ stubResponse{Content: `{"category":"finance","region":"east"}`},
+ // call 2: field_name="metadata"
+ stubResponse{Content: `{"category":"law","author":"X"}`},
+ )
+ c := &ExtractorComponent{Param: schema.ExtractorParam{
+ FieldName: "metadata",
+ LLMID: "gpt-4o-mini",
+ Prompt: "Extract metadata as JSON:",
+ EnableMetadata: 1,
+ Metadata: []common.MetadataFieldDef{
+ {Key: "category", Type: "string"},
+ {Key: "region", Type: "string"},
+ },
+ }}
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "chunks": []map[string]any{{"text": "content"}},
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := out["chunks"].([]map[string]any)
+ meta := chunks[0]["metadata"].(map[string]any)
+ if meta["category"] != "law" {
+ t.Errorf("category = %v, want law (field_name wins on overlap)", meta["category"])
+ }
+ if meta["region"] != "east" {
+ t.Errorf("region = %v, want east (auto field preserved)", meta["region"])
+ }
+}
+
+// TestRepro_FieldNameNonJSON_KeepsEnableMetadata verifies a non-JSON
+// field_name="metadata" result does not clobber the enable_metadata map and
+// never writes a raw string.
+func TestRepro_FieldNameNonJSON_KeepsEnableMetadata(t *testing.T) {
+ withStubChatInvoker(t,
+ // call 1: enable_metadata
+ stubResponse{Content: `{"category":"finance"}`},
+ // call 2: field_name="metadata" -> non-JSON
+ stubResponse{Content: "could not find metadata"},
+ )
+ c := &ExtractorComponent{Param: schema.ExtractorParam{
+ FieldName: "metadata",
+ LLMID: "gpt-4o-mini",
+ Prompt: "Extract metadata as JSON:",
+ EnableMetadata: 1,
+ Metadata: []common.MetadataFieldDef{
+ {Key: "category", Type: "string"},
+ },
+ }}
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "chunks": []map[string]any{{"text": "content"}},
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := out["chunks"].([]map[string]any)
+ meta, ok := chunks[0]["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("chunk[0].metadata = %T, want map (auto-metadata preserved, no string written)", chunks[0]["metadata"])
+ }
+ if meta["category"] != "finance" {
+ t.Errorf("category = %v, want finance (enable_metadata map preserved)", meta["category"])
+ }
+}
+
+// TestRepro_NoChunksMetadata_RoutesThroughMap guards the zero-chunk fast path:
+// field_name="metadata" with no input chunks must route through the same
+// parse-and-merge as the chunked path, writing a map (not a raw string) so the
+// strict aggregation layer does not drop the document metadata.
+func TestRepro_NoChunksMetadata_RoutesThroughMap(t *testing.T) {
+ withStubChatInvoker(t,
+ stubResponse{Content: `{"category":"finance","region":"east"}`},
+ )
+ c := &ExtractorComponent{Param: schema.ExtractorParam{
+ FieldName: "metadata",
+ LLMID: "gpt-4o-mini",
+ Prompt: "Extract metadata as JSON:",
+ }}
+ out, err := c.Invoke(t.Context(), nil, map[string]any{})
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := out["chunks"].([]map[string]any)
+ if len(chunks) != 1 {
+ t.Fatalf("chunks len = %d, want 1", len(chunks))
+ }
+ meta, ok := chunks[0]["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("chunk[0].metadata = %T, want map[string]any (zero-chunk path must not write a raw string)", chunks[0]["metadata"])
+ }
+ if meta["category"] != "finance" || meta["region"] != "east" {
+ t.Errorf("metadata = %v, want category=finance region=east", meta)
+ }
+}
diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.go
index 01be82d3b5..8010e58150 100644
--- a/internal/ingestion/pipeline/pipeline_params.go
+++ b/internal/ingestion/pipeline/pipeline_params.go
@@ -53,10 +53,17 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[
for pk, pv := range params {
if _, ok := validKeys[pk]; ok {
cleaned[pk] = pv
- } else {
- common.Warn("CleanComponentParams: dropping unknown param",
- zap.String("cpnID", key), zap.String("param", pk))
+ continue
}
+ // Auto metadata params are valid extractor params even when a
+ // builtin template does not declare them (the frontend stores
+ // them on the node params); keep them through the filter.
+ if isExtractorComponent(key) && isAutoMetadataParam(pk) {
+ cleaned[pk] = pv
+ continue
+ }
+ common.Warn("CleanComponentParams: dropping unknown param",
+ zap.String("cpnID", key), zap.String("param", pk))
}
if len(cleaned) > 0 {
result[key] = cleaned
@@ -149,3 +156,20 @@ func ResolveComponentParamsDefaultsFromIDs(parserID string, pipelineID *string)
}
return out, nil
}
+
+// isExtractorComponent reports whether a component id is an Extractor node
+// (component ids are "<ComponentName>:<name>" or "<ComponentName>_<name>").
+func isExtractorComponent(cpnID string) bool {
+ lower := strings.ToLower(cpnID)
+ return strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_")
+}
+
+// isAutoMetadataParam reports whether a param belongs to the Auto metadata
+// feature (enable_metadata toggle + metadata / built_in_metadata schemas).
+func isAutoMetadataParam(param string) bool {
+ switch param {
+ case "enable_metadata", "metadata", "built_in_metadata":
+ return true
+ }
+ return false
+}
diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.go
index 52fe83fc41..4dc776912d 100644
--- a/internal/ingestion/pipeline/pipeline_params_test.go
+++ b/internal/ingestion/pipeline/pipeline_params_test.go
@@ -308,3 +308,49 @@ func TestResolveComponentParamsDefaults_ResultIsMutable(t *testing.T) {
// ensure entity.JSONMap is used (import used).
var _ entity.JSONMap
+
+func TestBuildParserConfig_BuiltinExtractorKeepsBuiltInMetadata(t *testing.T) {
+ registry, err := DefaultRegistry()
+ if err != nil {
+ t.Fatalf("DefaultRegistry: %v", err)
+ }
+ checked := 0
+ for _, ref := range registry.Refs() {
+ tpl, ok := registry.Get(ref)
+ if !ok {
+ t.Fatalf("registry.Get(%q) failed", ref)
+ }
+ dslJSON, err := json.Marshal(tpl.DSL)
+ if err != nil {
+ t.Fatalf("marshal DSL %q: %v", ref, err)
+ }
+ schemas, err := ExtractAllComponentParams(dslJSON)
+ if err != nil {
+ t.Fatalf("ExtractAllComponentParams %q: %v", ref, err)
+ }
+ for _, s := range schemas {
+ if s.ComponentName != "Extractor" {
+ continue
+ }
+ checked++
+ overrides := map[string]any{
+ s.CpnID: map[string]any{
+ "built_in_metadata": []any{
+ map[string]any{"key": "update_time", "type": "time"},
+ },
+ },
+ }
+ result := BuildParserConfig(dslJSON, overrides)
+ params, ok := result[s.CpnID].(map[string]any)
+ if !ok {
+ t.Fatalf("template %q: expected component %q in result", ref, s.CpnID)
+ }
+ if _, ok := params["built_in_metadata"]; !ok {
+ t.Errorf("template %q: built_in_metadata dropped by CleanComponentParams; add \"built_in_metadata\": [] to the extractor node params", ref)
+ }
+ }
+ }
+ if checked == 0 {
+ t.Fatal("expected at least one builtin template with an Extractor component")
+ }
+}
diff --git a/internal/ingestion/service/doc_state.go b/internal/ingestion/service/doc_state.go
index e4a293028b..a953f09c8a 100644
--- a/internal/ingestion/service/doc_state.go
+++ b/internal/ingestion/service/doc_state.go
@@ -19,6 +19,7 @@ package service
import (
"context"
"fmt"
+ "time"
"ragflow/internal/common"
taskpkg "ragflow/internal/ingestion/task"
@@ -61,6 +62,16 @@ func (u *docStateUpdater) apply(ctx context.Context, r *taskpkg.PipelineResult)
common.Warn(fmt.Sprintf("failed to update document metadata: %v", err))
}
}
+ // Built-in metadata (update_time / file_name) is applied on top of the
+ // LLM-extracted metadata, mirroring Python apply_built_in_metadata
+ // (task_executor_refactor/chunk_post_processor.py): it runs when
+ // auto-metadata is enabled and built-in fields are configured, and its
+ // values overwrite whatever is already stored.
+ if r.AutoMetadataEnabled && len(r.BuiltInMetadataConfig) > 0 {
+ if err := applyBuiltInMetadata(ctx, u.docSvc, r.DocID, r.DocName, r.BuiltInMetadataConfig); err != nil {
+ common.Warn(fmt.Sprintf("failed to apply built-in metadata: %v", err))
+ }
+ }
if err := u.docSvc.ApplyDocCounts(ctx, r.DocID, r.KbID, r.ChunkCount, r.TokenConsumption, r.Duration); err != nil {
common.Warn(fmt.Sprintf("failed to apply doc counts: %v", err))
}
@@ -86,3 +97,40 @@ func mergeDocMetadata(ctx context.Context, svc docStateSvc, docID string, metada
merged = common.SplitCombinedMetadataValues(merged)
return svc.SetDocumentMetadata(ctx, docID, merged)
}
+
+// applyBuiltInMetadata writes the configured built-in metadata fields into the
+// document's metadata, overwriting existing values. Mirrors Python
+// apply_built_in_metadata (task_executor_refactor/chunk_post_processor.py):
+// - update_time -> current timestamp "2006-01-02 15:04:05"
+// - file_name -> the document name
+func applyBuiltInMetadata(ctx context.Context, svc docStateSvc, docID, docName string, config []any) error {
+ builtIn := make(map[string]any, 2)
+ for _, raw := range config {
+ item, ok := raw.(map[string]any)
+ if !ok {
+ continue
+ }
+ key, _ := item["key"].(string)
+ switch key {
+ case "update_time":
+ builtIn["update_time"] = time.Now().Format("2006-01-02 15:04:05")
+ case "file_name":
+ if docName != "" {
+ builtIn["file_name"] = docName
+ }
+ }
+ }
+ if len(builtIn) == 0 {
+ return nil
+ }
+ existing, err := svc.GetDocumentMetadataByID(ctx, docID)
+ if err != nil {
+ return err
+ }
+ if existing == nil {
+ existing = map[string]any{}
+ }
+ merged := utility.UpdateMetadataTo(existing, builtIn)
+ merged = common.SplitCombinedMetadataValues(merged)
+ return svc.SetDocumentMetadata(ctx, docID, merged)
+}
diff --git a/internal/ingestion/service/doc_state_test.go b/internal/ingestion/service/doc_state_test.go
index 786b64fe24..86ad8c4be4 100644
--- a/internal/ingestion/service/doc_state_test.go
+++ b/internal/ingestion/service/doc_state_test.go
@@ -19,6 +19,7 @@ package service
import (
"context"
"testing"
+ "time"
taskpkg "ragflow/internal/ingestion/task"
)
@@ -176,3 +177,81 @@ func TestDocStateUpdater_IncrementArgs(t *testing.T) {
t.Fatalf("duration=%f, want 0", svc.gotDuration)
}
}
+
+func TestDocStateUpdater_AppliesBuiltInMetadata(t *testing.T) {
+ svc := &stubDocStateSvc{metaData: map[string]any{"category": "finance"}}
+ u := &docStateUpdater{docSvc: svc}
+ ctx := t.Context()
+ u.apply(ctx, &taskpkg.PipelineResult{
+ DocID: "doc-1",
+ DocName: "annual-report.pdf",
+ AutoMetadataEnabled: true,
+ BuiltInMetadataConfig: []any{
+ map[string]any{"key": "update_time", "type": "time"},
+ map[string]any{"key": "file_name", "type": "string"},
+ },
+ ChunkCount: 1, TokenConsumption: 10,
+ })
+ if svc.metaData["file_name"] != "annual-report.pdf" {
+ t.Errorf("file_name = %v, want annual-report.pdf", svc.metaData["file_name"])
+ }
+ updateTime, ok := svc.metaData["update_time"].(string)
+ if !ok {
+ t.Fatalf("update_time = %T, want string", svc.metaData["update_time"])
+ }
+ if _, err := time.Parse("2006-01-02 15:04:05", updateTime); err != nil {
+ t.Errorf("update_time = %q, want 2006-01-02 15:04:05 format", updateTime)
+ }
+ if svc.metaData["category"] != "finance" {
+ t.Errorf("existing metadata must be preserved: category = %v", svc.metaData["category"])
+ }
+}
+
+func TestDocStateUpdater_BuiltInMetadataGatedOff(t *testing.T) {
+ svc := &stubDocStateSvc{metaData: map[string]any{}}
+ u := &docStateUpdater{docSvc: svc}
+ ctx := t.Context()
+ u.apply(ctx, &taskpkg.PipelineResult{
+ DocID: "doc-1",
+ DocName: "annual-report.pdf",
+ AutoMetadataEnabled: false,
+ BuiltInMetadataConfig: []any{map[string]any{"key": "file_name", "type": "string"}},
+ ChunkCount: 1, TokenConsumption: 10,
+ })
+ if _, ok := svc.metaData["file_name"]; ok {
+ t.Errorf("built-in metadata must not be applied when auto-metadata is disabled: %v", svc.metaData)
+ }
+}
+
+func TestDocStateUpdater_BuiltInMetadataOverwritesExisting(t *testing.T) {
+ svc := &stubDocStateSvc{metaData: map[string]any{"update_time": "2020-01-01 00:00:00"}}
+ u := &docStateUpdater{docSvc: svc}
+ ctx := t.Context()
+ u.apply(ctx, &taskpkg.PipelineResult{
+ DocID: "doc-1",
+ DocName: "annual-report.pdf",
+ AutoMetadataEnabled: true,
+ BuiltInMetadataConfig: []any{map[string]any{"key": "update_time", "type": "time"}},
+ ChunkCount: 1, TokenConsumption: 10,
+ })
+ if svc.metaData["update_time"] == "2020-01-01 00:00:00" {
+ t.Errorf("built-in update_time must overwrite the stored value: %v", svc.metaData["update_time"])
+ }
+ if _, ok := svc.metaData["update_time"].(string); !ok {
+ t.Fatalf("update_time = %T, want string", svc.metaData["update_time"])
+ }
+}
+
+func TestDocStateUpdater_BuiltInMetadataEmptyConfig(t *testing.T) {
+ svc := &stubDocStateSvc{metaData: map[string]any{}}
+ u := &docStateUpdater{docSvc: svc}
+ ctx := t.Context()
+ u.apply(ctx, &taskpkg.PipelineResult{
+ DocID: "doc-1", DocName: "annual-report.pdf",
+ AutoMetadataEnabled: true,
+ ChunkCount: 1, TokenConsumption: 10,
+ })
+ if len(svc.metaData) != 0 {
+ t.Errorf("empty built-in config must be a no-op, got %v", svc.metaData)
+ }
+}
diff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.go
index 511d5f8502..65c27a57e3 100644
--- a/internal/ingestion/task/indexdoc/process.go
+++ b/internal/ingestion/task/indexdoc/process.go
@@ -127,9 +127,17 @@ func mergeChunkMetadata(metadata map[string]any, ck map[string]any) map[string]a
if !exists {
return metadata
}
- if metaMap, ok := metaVal.(map[string]any); ok {
- metadata = utility.UpdateMetadataTo(metadata, metaMap)
+ metaMap, ok := metaVal.(map[string]any)
+ if !ok {
+ // Contract: ck["metadata"] is produced by the Extractor component and
+ // is always a map[string]any (the merge of enable_metadata + the
+ // field_name="metadata" extraction). A non-map value signals an
+ // upstream bug, not a value to guess-parse — record and drop it.
+ common.Warn(fmt.Sprintf("mergeChunkMetadata: chunk metadata is %T, want map[string]any; dropping", metaVal))
+ delete(ck, "metadata")
+ return metadata
}
+ metadata = utility.UpdateMetadataTo(metadata, metaMap)
delete(ck, "metadata")
return metadata
}
diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.go
index 591f659444..3276e2d12b 100644
--- a/internal/ingestion/task/indexdoc/process_test.go
+++ b/internal/ingestion/task/indexdoc/process_test.go
@@ -178,6 +178,58 @@ func TestProcessChunksForPipeline_QuestionsProcessing(t *testing.T) {
}
}
+// TestProcessChunksForPipeline_MetadataMapAggregated pins the normal contract:
+// ck["metadata"] produced by the Extractor (the merge of enable_metadata +
+// field_name="metadata") is a map[string]any and is aggregated into the
+// returned doc-level metadata.
+func TestProcessChunksForPipeline_MetadataMapAggregated(t *testing.T) {
+ chunks := []map[string]any{
+ {"text": "hello", "metadata": map[string]any{"category": "finance", "region": "east"}},
+ }
+ metadata, err := ProcessChunksForPipeline(chunks, "doc-1", "test-doc.pdf", time.Now())
+ if err != nil {
+ t.Fatalf("ProcessChunksForPipeline: %v", err)
+ }
+ if metadata["category"] != "finance" {
+ t.Errorf("category = %v, want finance", metadata["category"])
+ }
+ if metadata["region"] != "east" {
+ t.Errorf("region = %v, want east", metadata["region"])
+ }
+ // The consumed metadata key must not leak onto the persisted chunk.
+ if _, exists := chunks[0]["metadata"]; exists {
+ t.Error("metadata key should be removed from the chunk after aggregation")
+ }
+}
+
+// TestProcessChunksForPipeline_MetadataNonMapDropped pins the strict contract:
+// ck["metadata"] is Extractor-owned and always a map[string]any. A non-map
+// value (e.g. a JSON string, as field_name="metadata" used to emit before the
+// extractor unified to map) is a contract violation — it is dropped with a
+// warning, never guess-parsed, so an upstream bug surfaces instead of silently
+// producing document metadata.
+func TestProcessChunksForPipeline_MetadataNonMapDropped(t *testing.T) {
+ for name, value := range map[string]any{
+ "json_string": `{"category":"finance","region":"east"}`,
+ "fenced": "```json\n{\"category\":\"law\"}\n```",
+ "not_json": "this is not json",
+ } {
+ t.Run(name, func(t *testing.T) {
+ chunks := []map[string]any{{"text": "hello", "metadata": value}}
+ metadata, err := ProcessChunksForPipeline(chunks, "doc-1", "test-doc.pdf", time.Now())
+ if err != nil {
+ t.Fatalf("ProcessChunksForPipeline: %v", err)
+ }
+ if len(metadata) != 0 {
+ t.Errorf("metadata = %v, want empty (non-map metadata dropped)", metadata)
+ }
+ if _, exists := chunks[0]["metadata"]; exists {
+ t.Error("metadata key should be removed from the chunk after aggregation")
+ }
+ })
+ }
+}
+
func TestProcessChunksForPipeline_KeywordsProcessing(t *testing.T) {
chunks := []map[string]any{{"text": "hello", "keywords": "kw1,kw2;kw3"}}
_, err := ProcessChunksForPipeline(chunks, "doc-1", "test-doc.pdf", time.Now())
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index 7139ad1a8b..fca13f455e 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -49,6 +49,12 @@ type PipelineResult struct {
ChunkCount int
TokenConsumption int
Duration float64 // pipeline wall-clock seconds
+ // DocName, BuiltInMetadataConfig and AutoMetadataEnabled carry what the
+ // document-state finalizer needs to apply built-in metadata
+ // (update_time / file_name), mirroring Python apply_built_in_metadata.
+ DocName string
+ BuiltInMetadataConfig []any
+ AutoMetadataEnabled bool
// MessageID is the polling key for the debug-run log. The front-end reads
// it from the run response and polls GET /agents/:id/logs/:message_id to
// render progress; it is empty for non-debug (persist) runs.
@@ -277,16 +283,64 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
chunkCount := countDistinctChunkIDs(chunks)
+ builtInMetadata, autoMetaEnabled := builtInMetadataFromParserConfig(
+ s.taskCtx.Doc.ParserConfig,
+ )
+
return &PipelineResult{
- DocID: s.taskCtx.Doc.ID,
- KbID: s.taskCtx.Doc.KbID,
- Metadata: metadata,
- ChunkCount: chunkCount,
- TokenConsumption: embeddingTokenConsumption,
- Duration: time.Since(start).Seconds(),
+ DocID: s.taskCtx.Doc.ID,
+ KbID: s.taskCtx.Doc.KbID,
+ Metadata: metadata,
+ ChunkCount: chunkCount,
+ TokenConsumption: embeddingTokenConsumption,
+ Duration: time.Since(start).Seconds(),
+ DocName: docNameValue(s.taskCtx.Doc.Name),
+ BuiltInMetadataConfig: builtInMetadata,
+ AutoMetadataEnabled: autoMetaEnabled,
}, nil
}
+// builtInMetadataFromParserConfig extracts the built-in metadata config
+// (update_time / file_name) and whether auto-metadata is enabled. The frontend
+// stores both on the builtin extractor node params (Extractor:AutoExtractDefault)
+// via the operator form; the top-level parser_config shape (Python dataset
+// settings) is supported as a fallback.
+func builtInMetadataFromParserConfig(parserConfig entity.JSONMap) ([]any, bool) {
+ if nodeRaw, ok := parserConfig["Extractor:AutoExtractDefault"]; ok {
+ if node, ok := nodeRaw.(map[string]any); ok {
+ enabled := parserConfigBool(node["enable_metadata"])
+ if arr, ok := node["built_in_metadata"].([]any); ok && len(arr) > 0 {
+ return arr, enabled
+ }
+ }
+ }
+ if arr, ok := parserConfig["built_in_metadata"].([]any); ok && len(arr) > 0 {
+ return arr, parserConfigBool(parserConfig["enable_metadata"])
+ }
+ return nil, false
+}
+
+// parserConfigBool coerces a parser_config boolean-like value (bool / number)
+// to bool, mirroring the frontend's enable_metadata handling.
+func parserConfigBool(v any) bool {
+ switch typed := v.(type) {
+ case bool:
+ return typed
+ case float64:
+ return typed > 0
+ case int:
+ return typed > 0
+ }
+ return false
+}
+
+func docNameValue(name *string) string {
+ if name == nil {
+ return ""
+ }
+ return *name
+}
+
// countDistinctChunkIDs returns the number of distinct chunk IDs in the slice.
// After ProcessChunksForPipeline, every chunk carries an "id" field computed
// from xxhash(text+docID). Chunks with identical text share the same id and
diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.go
index b4148ed099..6f52677cba 100644
--- a/internal/ingestion/task/pipeline_executor_defaults_test.go
+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go
@@ -182,3 +182,55 @@ func TestBuildComponentParams_GoldenCoversAllTemplates(t *testing.T) {
}
}
}
+
+func TestBuiltInMetadataFromParserConfig_ExtractorNodeParams(t *testing.T) {
+ pc := entity.JSONMap{
+ "Extractor:AutoExtractDefault": map[string]any{
+ "enable_metadata": 1,
+ "built_in_metadata": []any{
+ map[string]any{"key": "update_time", "type": "time"},
+ map[string]any{"key": "file_name", "type": "string"},
+ },
+ },
+ }
+ cfg, enabled := builtInMetadataFromParserConfig(pc)
+ if !enabled {
+ t.Fatal("auto metadata should be enabled from extractor node params")
+ }
+ if len(cfg) != 2 {
+ t.Fatalf("built-in config len = %d, want 2", len(cfg))
+ }
+}
+
+func TestBuiltInMetadataFromParserConfig_TopLevelFallback(t *testing.T) {
+ pc := entity.JSONMap{
+ "enable_metadata": true,
+ "built_in_metadata": []any{
+ map[string]any{"key": "file_name", "type": "string"},
+ },
+ }
+ cfg, enabled := builtInMetadataFromParserConfig(pc)
+ if !enabled {
+ t.Fatal("auto metadata should be enabled from top-level config")
+ }
+ if len(cfg) != 1 {
+ t.Fatalf("built-in config len = %d, want 1", len(cfg))
+ }
+}
+
+func TestBuiltInMetadataFromParserConfig_None(t *testing.T) {
+ cfg, enabled := builtInMetadataFromParserConfig(entity.JSONMap{})
+ if enabled || len(cfg) != 0 {
+ t.Fatalf("got enabled=%v cfg=%v, want disabled empty", enabled, cfg)
+ }
+}
+
+func TestDocNameValue(t *testing.T) {
+ name := "report.pdf"
+ if got := docNameValue(&name); got != "report.pdf" {
+ t.Errorf("docNameValue = %q, want report.pdf", got)
+ }
+ if got := docNameValue(nil); got != "" {
+ t.Errorf("docNameValue(nil) = %q, want empty", got)
+ }
+}
diff --git a/web/src/pages/agent/form/extractor-form/index.tsx b/web/src/pages/agent/form/extractor-form/index.tsx
index 599e06f097..23d5337333 100644
--- a/web/src/pages/agent/form/extractor-form/index.tsx
+++ b/web/src/pages/agent/form/extractor-form/index.tsx
@@ -10,12 +10,25 @@ import { RAGFlowFormItem } from '@/components/ragflow-form';
import { SliderInputFormField } from '@/components/slider-input-form-field';
import { AsyncTreeSelect } from '@/components/ui/async-tree-select';
import { Form } from '@/components/ui/form';
+import { Button } from '@/components/ui/button';
+import { Switch } from '@/components/ui/switch';
import { PromptEditor } from '@/pages/agent/form/components/prompt-editor';
+import { MetadataType } from '@/pages/dataset/components/metedata/constant';
+import {
+ useManageMetadata,
+ util,
+} from '@/pages/dataset/components/metedata/hooks/use-manage-modal';
+import {
+ IBuiltInMetadataItem,
+ IMetaDataReturnJSONSettings,
+} from '@/pages/dataset/components/metedata/interface';
+import { ManageMetadataModal } from '@/pages/dataset/components/metedata/manage-modal';
import { isGoBackend } from '@/utils/backend-runtime';
import { buildOptions } from '@/utils/form';
import { zodResolver } from '@hookform/resolvers/zod';
-import { memo } from 'react';
-import { useForm } from 'react-hook-form';
+import { Settings } from 'lucide-react';
+import { memo, useCallback } from 'react';
+import { useForm, useFormContext } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import { z } from 'zod';
import {
@@ -43,11 +56,110 @@ export const FormSchema = z.object({
auto_questions: z.number().optional(),
auto_tags: z.number().optional(),
tag_file_id: z.string().optional(),
+ // Builtin auto-metadata (mirrors Python's Auto metadata): enable_metadata
+ // toggle + metadata / built_in_metadata field schema, consumed by the Go
+ // extractor's runEnableMetadata at parse time.
+ enable_metadata: z.number().optional(),
+ metadata: z.any().optional(),
+ built_in_metadata: z.any().optional(),
...LlmSettingSchema,
});
export type ExtractorFormSchemaType = z.infer<typeof FormSchema>;
+// Builtin auto-extract node id hardcoded in every builtin ingestion DSL
+// template (internal/ingestion/pipeline/template/*.json). Only this node
+// shows the Auto metadata option; canvas / user-pipeline extractor nodes do
+// not.
+const BuiltinAutoExtractNodeId = 'Extractor:AutoExtractDefault';
+
+// ExtractorAutoMetadata mirrors Python's dataset "Auto metadata" control: an
+// enable_metadata switch plus a field-schema editor (custom + built-in).
+// Values are stored on the node params (enable_metadata / metadata /
+// built_in_metadata) and drive the Go extractor's runEnableMetadata.
+function ExtractorAutoMetadata() {
+ const { t } = useTranslation();
+ const form = useFormContext<ExtractorFormSchemaType>();
+ const {
+ manageMetadataVisible,
+ showManageMetadataModal,
+ hideManageMetadataModal,
+ tableData,
+ config: metadataConfig,
+ } = useManageMetadata();
+
+ const handleOpen = useCallback(() => {
+ showManageMetadataModal({
+ metadata: util.metaDataSettingJSONToMetaDataTableData(
+ form.getValues('metadata') || [],
+ ),
+ isCanAdd: true,
+ type: MetadataType.Setting,
+ builtInMetadata: form.getValues('built_in_metadata') || [],
+ });
+ }, [form, showManageMetadataModal]);
+
+ const handleSave = useCallback(
+ (data?: {
+ metadata?: IMetaDataReturnJSONSettings;
+ builtInMetadata?: IBuiltInMetadataItem[];
+ }) => {
+ form.setValue('metadata', data?.metadata || []);
+ form.setValue('built_in_metadata', data?.builtInMetadata || []);
+ form.setValue('enable_metadata', 1);
+ },
+ [form],
+ );
+
+ return (
+ <>
+ <RAGFlowFormItem
+ label={t('knowledgeConfiguration.autoMetadata')}
+ name="enable_metadata"
+ >
+ {(field) => (
+ <div className="flex items-center justify-between">
+ <Button
+ type="button"
+ variant="ghost"
+ onClick={handleOpen}
+ data-testid="extractor-metadata-open-modal-btn"
+ >
+ <div className="flex items-center gap-2">
+ <Settings />
+ {t('knowledgeConfiguration.settings')}
+ </div>
+ </Button>
+ <Switch
+ checked={field.value === 1 || field.value === true}
+ onCheckedChange={(checked) => field.onChange(checked ? 1 : 0)}
+ data-testid="extractor-metadata-switch"
+ />
+ </div>
+ )}
+ </RAGFlowFormItem>
+ {manageMetadataVisible && (
+ <ManageMetadataModal
+ title={t('knowledgeDetails.metadata.metadataGenerationSettings')}
+ visible={manageMetadataVisible}
+ hideModal={hideManageMetadataModal}
+ tableData={tableData}
+ isCanAdd={metadataConfig.isCanAdd}
+ isDeleteSingleValue={metadataConfig.isDeleteSingleValue}
+ type={metadataConfig.type}
+ otherData={metadataConfig.record}
+ isShowDescription
+ isShowValueSwitch
+ isVerticalShowValue={false}
+ builtInMetadata={metadataConfig.builtInMetadata}
+ secondTitle={metadataConfig.secondTitle}
+ success={handleSave}
+ />
+ )}
+ </>
+ );
+}
+
const outputList = buildOutputList(initialExtractorValues.outputs);
const ExtractorForm = ({
@@ -130,6 +242,9 @@ const ExtractorForm = ({
)}
</RAGFlowFormItem>
+ {(node?.data as Record<string, any>)?.operatorId ===
+ BuiltinAutoExtractNodeId && <ExtractorAutoMetadata />}
+
<RAGFlowFormItem label={t('flow.systemPrompt')} name="sys_prompt">
<PromptEditor
placeholder={t('flow.messagePlaceholder')}