RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(ingestion): end-to-end Auto metadata in Go (extractor merge, builtin UI, built-in fields) (#18273)
jay77721452720a62dce
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.goindex 01a5f92d54..6c1e3cb9de 100644--- a/internal/ingestion/component/extractor.go+++ b/internal/ingestion/component/extractor.go@@ -671,7 +671,13 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs mapif callErr != nil {return callErr}- in.chunks = []map[string]any{{in.fieldName: ans}}+ ck := map[string]any{}+ if in.fieldName == "metadata" {+ mergeExtractionIntoMetadata(ck, ans)+ } else {+ ck[in.fieldName] = ans+ }+ in.chunks = []map[string]any{ck}return nil}// Auto keyword / question / metadata extraction runs with@@ -714,7 +720,11 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs mapif callErr != nil {return fmt.Errorf("chunk %d: %w", i, callErr)}- ck[in.fieldName] = ans+ if in.fieldName == "metadata" {+ mergeExtractionIntoMetadata(ck, ans)+ } else {+ ck[in.fieldName] = ans+ }}return nil}); err != nil {@@ -730,6 +740,29 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map}, nil}+// mergeExtractionIntoMetadata merges the field_name="metadata" extraction+// result into the chunk's metadata map (enable_metadata's output) instead of+// overwriting it. ck["metadata"] stays a map[string]any — the unified contract+// for document metadata produced by this component. A non-JSON / empty result+// is ignored (no string fallback), so it can never clobber or corrupt the+// metadata map. On overlapping keys the field_name result wins (it runs later).+func mergeExtractionIntoMetadata(ck map[string]any, ans string) {+ parsed, ok := tryParseJSONObject(ans)+ if !ok {+ return+ }+ existing, _ := ck["metadata"].(map[string]any)+ if existing == nil {+ existing = make(map[string]any, len(parsed))+ }+ for k, v := range parsed {+ if v != nil {+ existing[k] = v+ }+ }+ ck["metadata"] = existing+}+// runAutoKeywords extracts keywords for the current chunk and stores// them on ck["important_kwd"]. mu may be nil (sequential path); when// non-nil it serializes the shared chunk-map accesses so concurrentdiff --git a/internal/ingestion/component/extractor_metadata_regression_test.go b/internal/ingestion/component/extractor_metadata_regression_test.gonew file mode 100644index 0000000000..172f329a6f--- /dev/null+++ b/internal/ingestion/component/extractor_metadata_regression_test.go@@ -0,0 +1,202 @@+package component++import (+ "testing"+ "time"++ "ragflow/internal/common"+ "ragflow/internal/ingestion/component/schema"+ "ragflow/internal/ingestion/task/indexdoc"+)++// TestRepro_FieldNameMetadata_AggregatesMap guards the unified contract:+// field_name="metadata" (the pipeline/canvas way to produce document metadata)+// writes a parsed map into ck["metadata"] — not a raw string — so the+// aggregation layer, which is strict on map, merges it into document metadata.+func TestRepro_FieldNameMetadata_AggregatesMap(t *testing.T) {+ withStubChatInvoker(t,+ stubResponse{Content: `{"category":"finance","region":"east"}`},+ )+ c := &ExtractorComponent{Param: schema.ExtractorParam{+ FieldName: "metadata",+ LLMID: "gpt-4o-mini",+ Prompt: "Extract metadata as JSON:",+ }}+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "chunks": []map[string]any{{"text": "first text"}},+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := out["chunks"].([]map[string]any)+ meta, ok := chunks[0]["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("chunk[0].metadata = %T, want map[string]any (field_name=metadata must produce a map, not a string)", chunks[0]["metadata"])+ }+ if meta["category"] != "finance" || meta["region"] != "east" {+ t.Errorf("metadata = %v, want category=finance region=east", meta)+ }++ // Feed through the aggregation path, exactly as the executor does.+ metadata, err := indexdoc.ProcessChunksForPipeline(chunks, "doc-1", "doc.pdf", time.Now())+ if err != nil {+ t.Fatalf("ProcessChunksForPipeline: %v", err)+ }+ if metadata["category"] != "finance" {+ t.Errorf("category = %v, want finance", metadata["category"])+ }+ if metadata["region"] != "east" {+ t.Errorf("region = %v, want east", metadata["region"])+ }+}++// TestRepro_BothEnabled_MergesMetadata guards the unified contract: when both+// enable_metadata (auto, schema-driven) and field_name="metadata" (pipeline+// custom) are configured, the extractor MERGES them into one ck["metadata"]+// map instead of the field_name result overwriting the auto-metadata. Both+// contribute to the document metadata.+func TestRepro_BothEnabled_MergesMetadata(t *testing.T) {+ withStubChatInvoker(t,+ // call 1: enable_metadata (runAutoExtractions)+ stubResponse{Content: `{"category":"finance","region":"east"}`},+ // call 2: field_name="metadata" primary extraction+ stubResponse{Content: `{"author":"zhang","date":"2024-01-01"}`},+ )+ c := &ExtractorComponent{Param: schema.ExtractorParam{+ FieldName: "metadata",+ LLMID: "gpt-4o-mini",+ Prompt: "Extract more metadata as JSON:",+ EnableMetadata: 1,+ Metadata: []common.MetadataFieldDef{+ {Key: "category", Type: "string"},+ {Key: "region", Type: "string"},+ },+ }}+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "chunks": []map[string]any{{"text": "content"}},+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := out["chunks"].([]map[string]any)+ meta, ok := chunks[0]["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("chunk[0].metadata = %T, want merged map; auto-metadata was overwritten", chunks[0]["metadata"])+ }+ for _, k := range []string{"category", "region", "author", "date"} {+ if _, has := meta[k]; !has {+ t.Errorf("merged metadata missing %q: %v", k, meta)+ }+ }++ metadata, err := indexdoc.ProcessChunksForPipeline(chunks, "doc-1", "doc.pdf", time.Now())+ if err != nil {+ t.Fatalf("ProcessChunksForPipeline: %v", err)+ }+ for _, k := range []string{"category", "region", "author", "date"} {+ if _, has := metadata[k]; !has {+ t.Errorf("aggregated doc metadata missing %q: %v", k, metadata)+ }+ }+}++// TestRepro_OverlapFieldNameWins pins the overlap rule: when enable_metadata+// and field_name="metadata" extract the same key, the field_name (custom,+// later) result wins.+func TestRepro_OverlapFieldNameWins(t *testing.T) {+ withStubChatInvoker(t,+ // call 1: enable_metadata+ stubResponse{Content: `{"category":"finance","region":"east"}`},+ // call 2: field_name="metadata"+ stubResponse{Content: `{"category":"law","author":"X"}`},+ )+ c := &ExtractorComponent{Param: schema.ExtractorParam{+ FieldName: "metadata",+ LLMID: "gpt-4o-mini",+ Prompt: "Extract metadata as JSON:",+ EnableMetadata: 1,+ Metadata: []common.MetadataFieldDef{+ {Key: "category", Type: "string"},+ {Key: "region", Type: "string"},+ },+ }}+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "chunks": []map[string]any{{"text": "content"}},+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := out["chunks"].([]map[string]any)+ meta := chunks[0]["metadata"].(map[string]any)+ if meta["category"] != "law" {+ t.Errorf("category = %v, want law (field_name wins on overlap)", meta["category"])+ }+ if meta["region"] != "east" {+ t.Errorf("region = %v, want east (auto field preserved)", meta["region"])+ }+}++// TestRepro_FieldNameNonJSON_KeepsEnableMetadata verifies a non-JSON+// field_name="metadata" result does not clobber the enable_metadata map and+// never writes a raw string.+func TestRepro_FieldNameNonJSON_KeepsEnableMetadata(t *testing.T) {+ withStubChatInvoker(t,+ // call 1: enable_metadata+ stubResponse{Content: `{"category":"finance"}`},+ // call 2: field_name="metadata" -> non-JSON+ stubResponse{Content: "could not find metadata"},+ )+ c := &ExtractorComponent{Param: schema.ExtractorParam{+ FieldName: "metadata",+ LLMID: "gpt-4o-mini",+ Prompt: "Extract metadata as JSON:",+ EnableMetadata: 1,+ Metadata: []common.MetadataFieldDef{+ {Key: "category", Type: "string"},+ },+ }}+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "chunks": []map[string]any{{"text": "content"}},+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := out["chunks"].([]map[string]any)+ meta, ok := chunks[0]["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("chunk[0].metadata = %T, want map (auto-metadata preserved, no string written)", chunks[0]["metadata"])+ }+ if meta["category"] != "finance" {+ t.Errorf("category = %v, want finance (enable_metadata map preserved)", meta["category"])+ }+}++// TestRepro_NoChunksMetadata_RoutesThroughMap guards the zero-chunk fast path:+// field_name="metadata" with no input chunks must route through the same+// parse-and-merge as the chunked path, writing a map (not a raw string) so the+// strict aggregation layer does not drop the document metadata.+func TestRepro_NoChunksMetadata_RoutesThroughMap(t *testing.T) {+ withStubChatInvoker(t,+ stubResponse{Content: `{"category":"finance","region":"east"}`},+ )+ c := &ExtractorComponent{Param: schema.ExtractorParam{+ FieldName: "metadata",+ LLMID: "gpt-4o-mini",+ Prompt: "Extract metadata as JSON:",+ }}+ out, err := c.Invoke(t.Context(), nil, map[string]any{})+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := out["chunks"].([]map[string]any)+ if len(chunks) != 1 {+ t.Fatalf("chunks len = %d, want 1", len(chunks))+ }+ meta, ok := chunks[0]["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("chunk[0].metadata = %T, want map[string]any (zero-chunk path must not write a raw string)", chunks[0]["metadata"])+ }+ if meta["category"] != "finance" || meta["region"] != "east" {+ t.Errorf("metadata = %v, want category=finance region=east", meta)+ }+}diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.goindex 01be82d3b5..8010e58150 100644--- a/internal/ingestion/pipeline/pipeline_params.go+++ b/internal/ingestion/pipeline/pipeline_params.go@@ -53,10 +53,17 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[for pk, pv := range params {if _, ok := validKeys[pk]; ok {cleaned[pk] = pv- } else {- common.Warn("CleanComponentParams: dropping unknown param",- zap.String("cpnID", key), zap.String("param", pk))+ continue}+ // Auto metadata params are valid extractor params even when a+ // builtin template does not declare them (the frontend stores+ // them on the node params); keep them through the filter.+ if isExtractorComponent(key) && isAutoMetadataParam(pk) {+ cleaned[pk] = pv+ continue+ }+ common.Warn("CleanComponentParams: dropping unknown param",+ zap.String("cpnID", key), zap.String("param", pk))}if len(cleaned) > 0 {result[key] = cleaned@@ -149,3 +156,20 @@ func ResolveComponentParamsDefaultsFromIDs(parserID string, pipelineID *string)}return out, nil}++// isExtractorComponent reports whether a component id is an Extractor node+// (component ids are "<ComponentName>:<name>" or "<ComponentName>_<name>").+func isExtractorComponent(cpnID string) bool {+ lower := strings.ToLower(cpnID)+ return strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_")+}++// isAutoMetadataParam reports whether a param belongs to the Auto metadata+// feature (enable_metadata toggle + metadata / built_in_metadata schemas).+func isAutoMetadataParam(param string) bool {+ switch param {+ case "enable_metadata", "metadata", "built_in_metadata":+ return true+ }+ return false+}diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.goindex 52fe83fc41..4dc776912d 100644--- a/internal/ingestion/pipeline/pipeline_params_test.go+++ b/internal/ingestion/pipeline/pipeline_params_test.go@@ -308,3 +308,49 @@ func TestResolveComponentParamsDefaults_ResultIsMutable(t *testing.T) {// ensure entity.JSONMap is used (import used).var _ entity.JSONMap++func TestBuildParserConfig_BuiltinExtractorKeepsBuiltInMetadata(t *testing.T) {+ registry, err := DefaultRegistry()+ if err != nil {+ t.Fatalf("DefaultRegistry: %v", err)+ }+ checked := 0+ for _, ref := range registry.Refs() {+ tpl, ok := registry.Get(ref)+ if !ok {+ t.Fatalf("registry.Get(%q) failed", ref)+ }+ dslJSON, err := json.Marshal(tpl.DSL)+ if err != nil {+ t.Fatalf("marshal DSL %q: %v", ref, err)+ }+ schemas, err := ExtractAllComponentParams(dslJSON)+ if err != nil {+ t.Fatalf("ExtractAllComponentParams %q: %v", ref, err)+ }+ for _, s := range schemas {+ if s.ComponentName != "Extractor" {+ continue+ }+ checked+++ overrides := map[string]any{+ s.CpnID: map[string]any{+ "built_in_metadata": []any{+ map[string]any{"key": "update_time", "type": "time"},+ },+ },+ }+ result := BuildParserConfig(dslJSON, overrides)+ params, ok := result[s.CpnID].(map[string]any)+ if !ok {+ t.Fatalf("template %q: expected component %q in result", ref, s.CpnID)+ }+ if _, ok := params["built_in_metadata"]; !ok {+ t.Errorf("template %q: built_in_metadata dropped by CleanComponentParams; add \"built_in_metadata\": [] to the extractor node params", ref)+ }+ }+ }+ if checked == 0 {+ t.Fatal("expected at least one builtin template with an Extractor component")+ }+}diff --git a/internal/ingestion/service/doc_state.go b/internal/ingestion/service/doc_state.goindex e4a293028b..a953f09c8a 100644--- a/internal/ingestion/service/doc_state.go+++ b/internal/ingestion/service/doc_state.go@@ -19,6 +19,7 @@ package serviceimport ("context""fmt"+ "time""ragflow/internal/common"taskpkg "ragflow/internal/ingestion/task"@@ -61,6 +62,16 @@ func (u *docStateUpdater) apply(ctx context.Context, r *taskpkg.PipelineResult)common.Warn(fmt.Sprintf("failed to update document metadata: %v", err))}}+ // Built-in metadata (update_time / file_name) is applied on top of the+ // LLM-extracted metadata, mirroring Python apply_built_in_metadata+ // (task_executor_refactor/chunk_post_processor.py): it runs when+ // auto-metadata is enabled and built-in fields are configured, and its+ // values overwrite whatever is already stored.+ if r.AutoMetadataEnabled && len(r.BuiltInMetadataConfig) > 0 {+ if err := applyBuiltInMetadata(ctx, u.docSvc, r.DocID, r.DocName, r.BuiltInMetadataConfig); err != nil {+ common.Warn(fmt.Sprintf("failed to apply built-in metadata: %v", err))+ }+ }if err := u.docSvc.ApplyDocCounts(ctx, r.DocID, r.KbID, r.ChunkCount, r.TokenConsumption, r.Duration); err != nil {common.Warn(fmt.Sprintf("failed to apply doc counts: %v", err))}@@ -86,3 +97,40 @@ func mergeDocMetadata(ctx context.Context, svc docStateSvc, docID string, metadamerged = common.SplitCombinedMetadataValues(merged)return svc.SetDocumentMetadata(ctx, docID, merged)}++// applyBuiltInMetadata writes the configured built-in metadata fields into the+// document's metadata, overwriting existing values. Mirrors Python+// apply_built_in_metadata (task_executor_refactor/chunk_post_processor.py):+// - update_time -> current timestamp "2006-01-02 15:04:05"+// - file_name -> the document name+func applyBuiltInMetadata(ctx context.Context, svc docStateSvc, docID, docName string, config []any) error {+ builtIn := make(map[string]any, 2)+ for _, raw := range config {+ item, ok := raw.(map[string]any)+ if !ok {+ continue+ }+ key, _ := item["key"].(string)+ switch key {+ case "update_time":+ builtIn["update_time"] = time.Now().Format("2006-01-02 15:04:05")+ case "file_name":+ if docName != "" {+ builtIn["file_name"] = docName+ }+ }+ }+ if len(builtIn) == 0 {+ return nil+ }+ existing, err := svc.GetDocumentMetadataByID(ctx, docID)+ if err != nil {+ return err+ }+ if existing == nil {+ existing = map[string]any{}+ }+ merged := utility.UpdateMetadataTo(existing, builtIn)+ merged = common.SplitCombinedMetadataValues(merged)+ return svc.SetDocumentMetadata(ctx, docID, merged)+}diff --git a/internal/ingestion/service/doc_state_test.go b/internal/ingestion/service/doc_state_test.goindex 786b64fe24..86ad8c4be4 100644--- a/internal/ingestion/service/doc_state_test.go+++ b/internal/ingestion/service/doc_state_test.go@@ -19,6 +19,7 @@ package serviceimport ("context""testing"+ "time"taskpkg "ragflow/internal/ingestion/task")@@ -176,3 +177,81 @@ func TestDocStateUpdater_IncrementArgs(t *testing.T) {t.Fatalf("duration=%f, want 0", svc.gotDuration)}}++func TestDocStateUpdater_AppliesBuiltInMetadata(t *testing.T) {+ svc := &stubDocStateSvc{metaData: map[string]any{"category": "finance"}}+ u := &docStateUpdater{docSvc: svc}+ ctx := t.Context()+ u.apply(ctx, &taskpkg.PipelineResult{+ DocID: "doc-1",+ DocName: "annual-report.pdf",+ AutoMetadataEnabled: true,+ BuiltInMetadataConfig: []any{+ map[string]any{"key": "update_time", "type": "time"},+ map[string]any{"key": "file_name", "type": "string"},+ },+ ChunkCount: 1, TokenConsumption: 10,+ })+ if svc.metaData["file_name"] != "annual-report.pdf" {+ t.Errorf("file_name = %v, want annual-report.pdf", svc.metaData["file_name"])+ }+ updateTime, ok := svc.metaData["update_time"].(string)+ if !ok {+ t.Fatalf("update_time = %T, want string", svc.metaData["update_time"])+ }+ if _, err := time.Parse("2006-01-02 15:04:05", updateTime); err != nil {+ t.Errorf("update_time = %q, want 2006-01-02 15:04:05 format", updateTime)+ }+ if svc.metaData["category"] != "finance" {+ t.Errorf("existing metadata must be preserved: category = %v", svc.metaData["category"])+ }+}++func TestDocStateUpdater_BuiltInMetadataGatedOff(t *testing.T) {+ svc := &stubDocStateSvc{metaData: map[string]any{}}+ u := &docStateUpdater{docSvc: svc}+ ctx := t.Context()+ u.apply(ctx, &taskpkg.PipelineResult{+ DocID: "doc-1",+ DocName: "annual-report.pdf",+ AutoMetadataEnabled: false,+ BuiltInMetadataConfig: []any{map[string]any{"key": "file_name", "type": "string"}},+ ChunkCount: 1, TokenConsumption: 10,+ })+ if _, ok := svc.metaData["file_name"]; ok {+ t.Errorf("built-in metadata must not be applied when auto-metadata is disabled: %v", svc.metaData)+ }+}++func TestDocStateUpdater_BuiltInMetadataOverwritesExisting(t *testing.T) {+ svc := &stubDocStateSvc{metaData: map[string]any{"update_time": "2020-01-01 00:00:00"}}+ u := &docStateUpdater{docSvc: svc}+ ctx := t.Context()+ u.apply(ctx, &taskpkg.PipelineResult{+ DocID: "doc-1",+ DocName: "annual-report.pdf",+ AutoMetadataEnabled: true,+ BuiltInMetadataConfig: []any{map[string]any{"key": "update_time", "type": "time"}},+ ChunkCount: 1, TokenConsumption: 10,+ })+ if svc.metaData["update_time"] == "2020-01-01 00:00:00" {+ t.Errorf("built-in update_time must overwrite the stored value: %v", svc.metaData["update_time"])+ }+ if _, ok := svc.metaData["update_time"].(string); !ok {+ t.Fatalf("update_time = %T, want string", svc.metaData["update_time"])+ }+}++func TestDocStateUpdater_BuiltInMetadataEmptyConfig(t *testing.T) {+ svc := &stubDocStateSvc{metaData: map[string]any{}}+ u := &docStateUpdater{docSvc: svc}+ ctx := t.Context()+ u.apply(ctx, &taskpkg.PipelineResult{+ DocID: "doc-1", DocName: "annual-report.pdf",+ AutoMetadataEnabled: true,+ ChunkCount: 1, TokenConsumption: 10,+ })+ if len(svc.metaData) != 0 {+ t.Errorf("empty built-in config must be a no-op, got %v", svc.metaData)+ }+}diff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.goindex 511d5f8502..65c27a57e3 100644--- a/internal/ingestion/task/indexdoc/process.go+++ b/internal/ingestion/task/indexdoc/process.go@@ -127,9 +127,17 @@ func mergeChunkMetadata(metadata map[string]any, ck map[string]any) map[string]aif !exists {return metadata}- if metaMap, ok := metaVal.(map[string]any); ok {- metadata = utility.UpdateMetadataTo(metadata, metaMap)+ metaMap, ok := metaVal.(map[string]any)+ if !ok {+ // Contract: ck["metadata"] is produced by the Extractor component and+ // is always a map[string]any (the merge of enable_metadata + the+ // field_name="metadata" extraction). A non-map value signals an+ // upstream bug, not a value to guess-parse — record and drop it.+ common.Warn(fmt.Sprintf("mergeChunkMetadata: chunk metadata is %T, want map[string]any; dropping", metaVal))+ delete(ck, "metadata")+ return metadata}+ metadata = utility.UpdateMetadataTo(metadata, metaMap)delete(ck, "metadata")return metadata}diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.goindex 591f659444..3276e2d12b 100644--- a/internal/ingestion/task/indexdoc/process_test.go+++ b/internal/ingestion/task/indexdoc/process_test.go@@ -178,6 +178,58 @@ func TestProcessChunksForPipeline_QuestionsProcessing(t *testing.T) {}}+// TestProcessChunksForPipeline_MetadataMapAggregated pins the normal contract:+// ck["metadata"] produced by the Extractor (the merge of enable_metadata ++// field_name="metadata") is a map[string]any and is aggregated into the+// returned doc-level metadata.+func TestProcessChunksForPipeline_MetadataMapAggregated(t *testing.T) {+ chunks := []map[string]any{+ {"text": "hello", "metadata": map[string]any{"category": "finance", "region": "east"}},+ }+ metadata, err := ProcessChunksForPipeline(chunks, "doc-1", "test-doc.pdf", time.Now())+ if err != nil {+ t.Fatalf("ProcessChunksForPipeline: %v", err)+ }+ if metadata["category"] != "finance" {+ t.Errorf("category = %v, want finance", metadata["category"])+ }+ if metadata["region"] != "east" {+ t.Errorf("region = %v, want east", metadata["region"])+ }+ // The consumed metadata key must not leak onto the persisted chunk.+ if _, exists := chunks[0]["metadata"]; exists {+ t.Error("metadata key should be removed from the chunk after aggregation")+ }+}++// TestProcessChunksForPipeline_MetadataNonMapDropped pins the strict contract:+// ck["metadata"] is Extractor-owned and always a map[string]any. A non-map+// value (e.g. a JSON string, as field_name="metadata" used to emit before the+// extractor unified to map) is a contract violation — it is dropped with a+// warning, never guess-parsed, so an upstream bug surfaces instead of silently+// producing document metadata.+func TestProcessChunksForPipeline_MetadataNonMapDropped(t *testing.T) {+ for name, value := range map[string]any{+ "json_string": `{"category":"finance","region":"east"}`,+ "fenced": "```json\n{\"category\":\"law\"}\n```",+ "not_json": "this is not json",+ } {+ t.Run(name, func(t *testing.T) {+ chunks := []map[string]any{{"text": "hello", "metadata": value}}+ metadata, err := ProcessChunksForPipeline(chunks, "doc-1", "test-doc.pdf", time.Now())+ if err != nil {+ t.Fatalf("ProcessChunksForPipeline: %v", err)+ }+ if len(metadata) != 0 {+ t.Errorf("metadata = %v, want empty (non-map metadata dropped)", metadata)+ }+ if _, exists := chunks[0]["metadata"]; exists {+ t.Error("metadata key should be removed from the chunk after aggregation")+ }+ })+ }+}+func TestProcessChunksForPipeline_KeywordsProcessing(t *testing.T) {chunks := []map[string]any{{"text": "hello", "keywords": "kw1,kw2;kw3"}}_, err := ProcessChunksForPipeline(chunks, "doc-1", "test-doc.pdf", time.Now())diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex 7139ad1a8b..fca13f455e 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -49,6 +49,12 @@ type PipelineResult struct {ChunkCount intTokenConsumption intDuration float64 // pipeline wall-clock seconds+ // DocName, BuiltInMetadataConfig and AutoMetadataEnabled carry what the+ // document-state finalizer needs to apply built-in metadata+ // (update_time / file_name), mirroring Python apply_built_in_metadata.+ DocName string+ BuiltInMetadataConfig []any+ AutoMetadataEnabled bool// MessageID is the polling key for the debug-run log. The front-end reads// it from the run response and polls GET /agents/:id/logs/:message_id to// render progress; it is empty for non-debug (persist) runs.@@ -277,16 +283,64 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput mapchunkCount := countDistinctChunkIDs(chunks)+ builtInMetadata, autoMetaEnabled := builtInMetadataFromParserConfig(+ s.taskCtx.Doc.ParserConfig,+ )+return &PipelineResult{- DocID: s.taskCtx.Doc.ID,- KbID: s.taskCtx.Doc.KbID,- Metadata: metadata,- ChunkCount: chunkCount,- TokenConsumption: embeddingTokenConsumption,- Duration: time.Since(start).Seconds(),+ DocID: s.taskCtx.Doc.ID,+ KbID: s.taskCtx.Doc.KbID,+ Metadata: metadata,+ ChunkCount: chunkCount,+ TokenConsumption: embeddingTokenConsumption,+ Duration: time.Since(start).Seconds(),+ DocName: docNameValue(s.taskCtx.Doc.Name),+ BuiltInMetadataConfig: builtInMetadata,+ AutoMetadataEnabled: autoMetaEnabled,}, nil}+// builtInMetadataFromParserConfig extracts the built-in metadata config+// (update_time / file_name) and whether auto-metadata is enabled. The frontend+// stores both on the builtin extractor node params (Extractor:AutoExtractDefault)+// via the operator form; the top-level parser_config shape (Python dataset+// settings) is supported as a fallback.+func builtInMetadataFromParserConfig(parserConfig entity.JSONMap) ([]any, bool) {+ if nodeRaw, ok := parserConfig["Extractor:AutoExtractDefault"]; ok {+ if node, ok := nodeRaw.(map[string]any); ok {+ enabled := parserConfigBool(node["enable_metadata"])+ if arr, ok := node["built_in_metadata"].([]any); ok && len(arr) > 0 {+ return arr, enabled+ }+ }+ }+ if arr, ok := parserConfig["built_in_metadata"].([]any); ok && len(arr) > 0 {+ return arr, parserConfigBool(parserConfig["enable_metadata"])+ }+ return nil, false+}++// parserConfigBool coerces a parser_config boolean-like value (bool / number)+// to bool, mirroring the frontend's enable_metadata handling.+func parserConfigBool(v any) bool {+ switch typed := v.(type) {+ case bool:+ return typed+ case float64:+ return typed > 0+ case int:+ return typed > 0+ }+ return false+}++func docNameValue(name *string) string {+ if name == nil {+ return ""+ }+ return *name+}+// countDistinctChunkIDs returns the number of distinct chunk IDs in the slice.// After ProcessChunksForPipeline, every chunk carries an "id" field computed// from xxhash(text+docID). Chunks with identical text share the same id anddiff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.goindex b4148ed099..6f52677cba 100644--- a/internal/ingestion/task/pipeline_executor_defaults_test.go+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go@@ -182,3 +182,55 @@ func TestBuildComponentParams_GoldenCoversAllTemplates(t *testing.T) {}}}++func TestBuiltInMetadataFromParserConfig_ExtractorNodeParams(t *testing.T) {+ pc := entity.JSONMap{+ "Extractor:AutoExtractDefault": map[string]any{+ "enable_metadata": 1,+ "built_in_metadata": []any{+ map[string]any{"key": "update_time", "type": "time"},+ map[string]any{"key": "file_name", "type": "string"},+ },+ },+ }+ cfg, enabled := builtInMetadataFromParserConfig(pc)+ if !enabled {+ t.Fatal("auto metadata should be enabled from extractor node params")+ }+ if len(cfg) != 2 {+ t.Fatalf("built-in config len = %d, want 2", len(cfg))+ }+}++func TestBuiltInMetadataFromParserConfig_TopLevelFallback(t *testing.T) {+ pc := entity.JSONMap{+ "enable_metadata": true,+ "built_in_metadata": []any{+ map[string]any{"key": "file_name", "type": "string"},+ },+ }+ cfg, enabled := builtInMetadataFromParserConfig(pc)+ if !enabled {+ t.Fatal("auto metadata should be enabled from top-level config")+ }+ if len(cfg) != 1 {+ t.Fatalf("built-in config len = %d, want 1", len(cfg))+ }+}++func TestBuiltInMetadataFromParserConfig_None(t *testing.T) {+ cfg, enabled := builtInMetadataFromParserConfig(entity.JSONMap{})+ if enabled || len(cfg) != 0 {+ t.Fatalf("got enabled=%v cfg=%v, want disabled empty", enabled, cfg)+ }+}++func TestDocNameValue(t *testing.T) {+ name := "report.pdf"+ if got := docNameValue(&name); got != "report.pdf" {+ t.Errorf("docNameValue = %q, want report.pdf", got)+ }+ if got := docNameValue(nil); got != "" {+ t.Errorf("docNameValue(nil) = %q, want empty", got)+ }+}diff --git a/web/src/pages/agent/form/extractor-form/index.tsx b/web/src/pages/agent/form/extractor-form/index.tsxindex 599e06f097..23d5337333 100644--- a/web/src/pages/agent/form/extractor-form/index.tsx+++ b/web/src/pages/agent/form/extractor-form/index.tsx@@ -10,12 +10,25 @@ import { RAGFlowFormItem } from '@/components/ragflow-form';import { SliderInputFormField } from '@/components/slider-input-form-field';import { AsyncTreeSelect } from '@/components/ui/async-tree-select';import { Form } from '@/components/ui/form';+import { Button } from '@/components/ui/button';+import { Switch } from '@/components/ui/switch';import { PromptEditor } from '@/pages/agent/form/components/prompt-editor';+import { MetadataType } from '@/pages/dataset/components/metedata/constant';+import {+ useManageMetadata,+ util,+} from '@/pages/dataset/components/metedata/hooks/use-manage-modal';+import {+ IBuiltInMetadataItem,+ IMetaDataReturnJSONSettings,+} from '@/pages/dataset/components/metedata/interface';+import { ManageMetadataModal } from '@/pages/dataset/components/metedata/manage-modal';import { isGoBackend } from '@/utils/backend-runtime';import { buildOptions } from '@/utils/form';import { zodResolver } from '@hookform/resolvers/zod';-import { memo } from 'react';-import { useForm } from 'react-hook-form';+import { Settings } from 'lucide-react';+import { memo, useCallback } from 'react';+import { useForm, useFormContext } from 'react-hook-form';import { useTranslation } from 'react-i18next';import { z } from 'zod';import {@@ -43,11 +56,110 @@ export const FormSchema = z.object({auto_questions: z.number().optional(),auto_tags: z.number().optional(),tag_file_id: z.string().optional(),+ // Builtin auto-metadata (mirrors Python's Auto metadata): enable_metadata+ // toggle + metadata / built_in_metadata field schema, consumed by the Go+ // extractor's runEnableMetadata at parse time.+ enable_metadata: z.number().optional(),+ metadata: z.any().optional(),+ built_in_metadata: z.any().optional(),...LlmSettingSchema,});export type ExtractorFormSchemaType = z.infer<typeof FormSchema>;+// Builtin auto-extract node id hardcoded in every builtin ingestion DSL+// template (internal/ingestion/pipeline/template/*.json). Only this node+// shows the Auto metadata option; canvas / user-pipeline extractor nodes do+// not.+const BuiltinAutoExtractNodeId = 'Extractor:AutoExtractDefault';++// ExtractorAutoMetadata mirrors Python's dataset "Auto metadata" control: an+// enable_metadata switch plus a field-schema editor (custom + built-in).+// Values are stored on the node params (enable_metadata / metadata /+// built_in_metadata) and drive the Go extractor's runEnableMetadata.+function ExtractorAutoMetadata() {+ const { t } = useTranslation();+ const form = useFormContext<ExtractorFormSchemaType>();+ const {+ manageMetadataVisible,+ showManageMetadataModal,+ hideManageMetadataModal,+ tableData,+ config: metadataConfig,+ } = useManageMetadata();++ const handleOpen = useCallback(() => {+ showManageMetadataModal({+ metadata: util.metaDataSettingJSONToMetaDataTableData(+ form.getValues('metadata') || [],+ ),+ isCanAdd: true,+ type: MetadataType.Setting,+ builtInMetadata: form.getValues('built_in_metadata') || [],+ });+ }, [form, showManageMetadataModal]);++ const handleSave = useCallback(+ (data?: {+ metadata?: IMetaDataReturnJSONSettings;+ builtInMetadata?: IBuiltInMetadataItem[];+ }) => {+ form.setValue('metadata', data?.metadata || []);+ form.setValue('built_in_metadata', data?.builtInMetadata || []);+ form.setValue('enable_metadata', 1);+ },+ [form],+ );++ return (+ <>+ <RAGFlowFormItem+ label={t('knowledgeConfiguration.autoMetadata')}+ name="enable_metadata"+ >+ {(field) => (+ <div className="flex items-center justify-between">+ <Button+ type="button"+ variant="ghost"+ onClick={handleOpen}+ data-testid="extractor-metadata-open-modal-btn"+ >+ <div className="flex items-center gap-2">+ <Settings />+ {t('knowledgeConfiguration.settings')}+ </div>+ </Button>+ <Switch+ checked={field.value === 1 || field.value === true}+ onCheckedChange={(checked) => field.onChange(checked ? 1 : 0)}+ data-testid="extractor-metadata-switch"+ />+ </div>+ )}+ </RAGFlowFormItem>+ {manageMetadataVisible && (+ <ManageMetadataModal+ title={t('knowledgeDetails.metadata.metadataGenerationSettings')}+ visible={manageMetadataVisible}+ hideModal={hideManageMetadataModal}+ tableData={tableData}+ isCanAdd={metadataConfig.isCanAdd}+ isDeleteSingleValue={metadataConfig.isDeleteSingleValue}+ type={metadataConfig.type}+ otherData={metadataConfig.record}+ isShowDescription+ isShowValueSwitch+ isVerticalShowValue={false}+ builtInMetadata={metadataConfig.builtInMetadata}+ secondTitle={metadataConfig.secondTitle}+ success={handleSave}+ />+ )}+ </>+ );+}+const outputList = buildOutputList(initialExtractorValues.outputs);const ExtractorForm = ({@@ -130,6 +242,9 @@ const ExtractorForm = ({)}</RAGFlowFormItem>+ {(node?.data as Record<string, any>)?.operatorId ===+ BuiltinAutoExtractNodeId && <ExtractorAutoMetadata />}+<RAGFlowFormItem label={t('flow.systemPrompt')} name="sys_prompt"><PromptEditorplaceholder={t('flow.messagePlaceholder')}