RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(ingestion,web): modularize extractor configuration with sub-tabs, independent prompts, and metadata integration (#18383)
jay77721c71991bb7aeb
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.goindex db76fd8936..417c61d9ff 100644--- a/internal/ingestion/component/extractor.go+++ b/internal/ingestion/component/extractor.go@@ -207,6 +207,18 @@ Propose questions about a given piece of text content.## Content to analyze:%s`++ autoSummaryPrompt = `## Role+You are a precise and faithful text summarizer.++## Task+Create a concise and faithful summary of the provided text content.++## Requirements+- The summary MUST strictly rely on the provided text without hallucinating.+- The summary MUST be in the same language as the original text content.+- Be concise and focus on the main ideas, omitting redundant details.+- Output summary ONLY.`)// ExtractorComponent performs LLM-based extraction over a chunk@@ -265,49 +277,136 @@ func NewExtractorComponent(params map[string]any) (runtime.Component, error) {}}}- if v, ok := params["auto_keywords"]; ok {- p.AutoKeywords = mapInt(v)+ // 1. Keywords (modular or flat)+ if kwRaw, ok := params["keywords"].(map[string]any); ok {+ if v, ok := kwRaw["top_n"]; ok {+ p.Keywords.TopN = mapInt(v)+ }+ if v, ok := kwRaw["system_prompt"].(string); ok {+ p.Keywords.SystemPrompt = v+ } else if v, ok := kwRaw["sys_prompt"].(string); ok {+ p.Keywords.SystemPrompt = v+ }+ } else if v, ok := params["auto_keywords"]; ok {+ p.Keywords.TopN = mapInt(v)+ if v, ok := params["keywords_sys_prompt"].(string); ok {+ p.Keywords.SystemPrompt = v+ }}- if v, ok := params["auto_questions"]; ok {- p.AutoQuestions = mapInt(v)+ p.AutoKeywords = p.Keywords.TopN++ // 2. Questions (modular or flat)+ if qRaw, ok := params["questions"].(map[string]any); ok {+ if v, ok := qRaw["top_n"]; ok {+ p.Questions.TopN = mapInt(v)+ }+ if v, ok := qRaw["system_prompt"].(string); ok {+ p.Questions.SystemPrompt = v+ } else if v, ok := qRaw["sys_prompt"].(string); ok {+ p.Questions.SystemPrompt = v+ }+ } else if v, ok := params["auto_questions"]; ok {+ p.Questions.TopN = mapInt(v)+ if v, ok := params["questions_sys_prompt"].(string); ok {+ p.Questions.SystemPrompt = v+ }}- if v, ok := params["auto_tags"]; ok {- p.AutoTags = mapInt(v)+ p.AutoQuestions = p.Questions.TopN++ // 3. Tags (modular or flat)+ if tagRaw, ok := params["tags"].(map[string]any); ok {+ if v, ok := tagRaw["top_n"]; ok {+ p.Tags.TopN = mapInt(v)+ }+ if v, ok := tagRaw["tag_file_id"].(string); ok {+ p.Tags.TagFileID = v+ }+ } else {+ if v, ok := params["auto_tags"]; ok {+ p.Tags.TopN = mapInt(v)+ }+ if v, ok := params["tag_file_id"].(string); ok {+ p.Tags.TagFileID = v+ }}- if v, ok := params["enable_metadata"]; ok {- p.EnableMetadata = mapInt(v)+ p.AutoTags = p.Tags.TopN+ p.TagFileID = p.Tags.TagFileID++ // 4. Summary (modular or flat)+ if sumRaw, ok := params["summary"].(map[string]any); ok {+ if v, ok := sumRaw["enabled"].(bool); ok {+ p.Summary.Enabled = v+ } else if v, ok := sumRaw["enabled"]; ok {+ p.Summary.Enabled = mapInt(v) == 1+ }+ if v, ok := sumRaw["system_prompt"].(string); ok {+ p.Summary.SystemPrompt = v+ } else if v, ok := sumRaw["sys_prompt"].(string); ok {+ p.Summary.SystemPrompt = v+ }+ } else {+ if v, ok := params["enable_summary"].(bool); ok {+ p.Summary.Enabled = v+ } else if v, ok := params["enable_summary"]; ok {+ p.Summary.Enabled = mapInt(v) == 1+ } else if p.FieldName == "summary" {+ p.Summary.Enabled = true+ }+ p.Summary.SystemPrompt = p.SystemPrompt}- if v, ok := params["metadata"].([]any); ok {- fields := make([]common.MetadataFieldDef, 0, len(v))- for _, f := range v {- m, ok := f.(map[string]any)- if !ok {- continue- }- key, _ := m["key"].(string)- if key = strings.TrimSpace(key); key == "" {- continue- }- def := common.MetadataFieldDef{Key: key}- if t, ok := m["type"].(string); ok {- def.Type = t- }- if d, ok := m["description"].(string); ok {- def.Description = d- }- if e, ok := m["enum"].([]any); ok {- for _, ev := range e {- if s, ok := ev.(string); ok {- def.Enum = append(def.Enum, s)- }- }- }- fields = append(fields, def)+ if p.Summary.Enabled {+ p.EnableSummary = 1+ if p.FieldName == "" {+ p.FieldName = "summary"+ }+ if p.Summary.SystemPrompt != "" && p.SystemPrompt == "" {+ p.SystemPrompt = p.Summary.SystemPrompt+ }+ } else {+ p.EnableSummary = 0+ // When summary is explicitly disabled, avoid legacy "summary" field_name default triggering open-ended calls+ if p.FieldName == "summary" {+ p.FieldName = ""}- p.Metadata = fields}- if v, ok := params["tag_file_id"].(string); ok {- p.TagFileID = v++ // Metadata sub-config parsing with legacy flat fallback+ if metaRaw, ok := params["metadata_config"].(map[string]any); ok {+ if v, ok := metaRaw["enabled"].(bool); ok {+ p.MetadataConfig.Enabled = v+ } else if v, ok := metaRaw["enabled"]; ok {+ p.MetadataConfig.Enabled = mapInt(v) == 1+ }+ if v, ok := metaRaw["metadata"]; ok {+ p.MetadataConfig.Metadata = parseMetadataFieldDefs(v)+ }+ if v, ok := metaRaw["built_in_metadata"]; ok {+ p.MetadataConfig.BuiltInMetadata = parseMetadataFieldDefs(v)+ }+ if p.MetadataConfig.Enabled {+ p.EnableMetadata = 1+ } else {+ p.EnableMetadata = 0+ }+ p.Metadata = p.MetadataConfig.Metadata+ } else {+ if v, ok := params["enable_metadata"].(bool); ok {+ if v {+ p.EnableMetadata = 1+ } else {+ p.EnableMetadata = 0+ }+ } else if v, ok := params["enable_metadata"]; ok {+ p.EnableMetadata = mapInt(v)+ }+ if v, ok := params["metadata"]; ok {+ p.Metadata = parseMetadataFieldDefs(v)+ }+ if v, ok := params["built_in_metadata"]; ok {+ p.MetadataConfig.BuiltInMetadata = parseMetadataFieldDefs(v)+ }+ p.MetadataConfig.Enabled = p.EnableMetadata > 0+ p.MetadataConfig.Metadata = p.Metadata}}if err := p.Validate(); err != nil {@@ -468,10 +567,10 @@ func (e *einoExtractorChatInvoker) Chat(ctx context.Context, req extractorChatReapiKey := req.APIKeycfg := &models.APIConfig{ApiKey: &apiKey}cm := models.NewChatModel(d, &modelName, cfg)- var chatCfg *models.ChatConfig+ chatCfg := &models.ChatConfig{}if req.Temperature != nil {temp := *req.Temperature- chatCfg = &models.ChatConfig{Temperature: &temp}+ chatCfg.Temperature = &temp}wrapper := models.NewEinoChatModel(cm, chatCfg)// Honour ctx cancel up front so the caller's WithTimeout(...)@@ -485,6 +584,7 @@ func (e *einoExtractorChatInvoker) Chat(ctx context.Context, req extractorChatRecommon.Error(fmt.Sprintf("error when chat with message: %v", req.Messages), err)return nil, err}+ common.Debug(fmt.Sprintf("extractor: chat completed for model %s, response_length=%d", modelName, len(out.Content)))return &extractorChatResponse{Content: out.Content}, nil}@@ -672,19 +772,18 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map// Without this, a template containing {text} would be forwarded// to the model unsubstituted.callIn := in+ if in.fieldName == "summary" && strings.TrimSpace(callIn.systemPrompt) == "" && strings.TrimSpace(c.Param.Summary.SystemPrompt) == "" {+ callIn.systemPrompt = autoSummaryPrompt+ }callIn.systemPrompt, callIn.prompt = renderExtractorPrompts(- in.systemPrompt, in.prompt, map[string]any{}, "",+ callIn.systemPrompt, in.prompt, map[string]any{}, "",)ans, callErr := c.callText(timeoutCtx, db, callIn, "")if callErr != nil {return callErr}ck := map[string]any{}- if in.fieldName == "metadata" {- mergeExtractionIntoMetadata(ck, ans)- } else {- ck[in.fieldName] = ans- }+ ck[in.fieldName] = ansin.chunks = []map[string]any{ck}return nil}@@ -712,16 +811,15 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map// chunk field values, mirroring Python's// string_format at extractor.py:103.callIn := in- callIn.systemPrompt, callIn.prompt = renderExtractorPrompts(in.systemPrompt, in.prompt, ck, text)+ if in.fieldName == "summary" && strings.TrimSpace(callIn.systemPrompt) == "" && strings.TrimSpace(c.Param.Summary.SystemPrompt) == "" {+ callIn.systemPrompt = autoSummaryPrompt+ }+ callIn.systemPrompt, callIn.prompt = renderExtractorPrompts(callIn.systemPrompt, in.prompt, ck, text)ans, callErr := c.callText(timeoutCtx, db, callIn, "")if callErr != nil {return fmt.Errorf("chunk %d: %w", i, callErr)}- if in.fieldName == "metadata" {- mergeExtractionIntoMetadata(ck, ans)- } else {- ck[in.fieldName] = ans- }+ ck[in.fieldName] = ans}return nil}); err != nil {@@ -737,29 +835,6 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map}, nil}-// mergeExtractionIntoMetadata merges the field_name="metadata" extraction-// result into the chunk's metadata map (enable_metadata's output) instead of-// overwriting it. ck["metadata"] stays a map[string]any — the unified contract-// for document metadata produced by this component. A non-JSON / empty result-// is ignored (no string fallback), so it can never clobber or corrupt the-// metadata map. On overlapping keys the field_name result wins (it runs later).-func mergeExtractionIntoMetadata(ck map[string]any, ans string) {- parsed, ok := tryParseJSONObject(ans)- if !ok {- return- }- existing, _ := ck["metadata"].(map[string]any)- if existing == nil {- existing = make(map[string]any, len(parsed))- }- for k, v := range parsed {- if v != nil {- existing[k] = v- }- }- ck["metadata"] = existing-}-// runAutoKeywords extracts keywords for the current chunk and stores// them on ck["important_kwd"]. mu may be nil (sequential path); when// non-nil it serializes the shared chunk-map accesses so concurrent@@ -771,11 +846,22 @@ func (c *ExtractorComponent) runAutoKeywords(ctx context.Context, db *gorm.DB, iif exists {return nil}+ topN := c.Param.Keywords.TopN+ if topN <= 0 {+ topN = c.Param.AutoKeywords+ }+ var sysPrompt, userPrompt string+ if strings.TrimSpace(c.Param.Keywords.SystemPrompt) != "" {+ sysPrompt, userPrompt = renderExtractorPrompts(c.Param.Keywords.SystemPrompt, "Output: ", ck, chunkText)+ } else {+ sysPrompt = fmt.Sprintf(autoKeywordPrompt, topN, chunkText)+ userPrompt = "Output: "+ }kwTemp := extractorTemperaturekwIn := extractorInputs{llmID: in.llmID,- systemPrompt: fmt.Sprintf(autoKeywordPrompt, c.Param.AutoKeywords, chunkText),- prompt: "Output: ",+ systemPrompt: sysPrompt,+ prompt: userPrompt,temperature: &kwTemp,}resultStr, err := c.callText(ctx, db, kwIn, "")@@ -806,11 +892,22 @@ func (c *ExtractorComponent) runAutoQuestions(ctx context.Context, db *gorm.DB,if exists {return nil}+ topN := c.Param.Questions.TopN+ if topN <= 0 {+ topN = c.Param.AutoQuestions+ }+ var sysPrompt, userPrompt string+ if strings.TrimSpace(c.Param.Questions.SystemPrompt) != "" {+ sysPrompt, userPrompt = renderExtractorPrompts(c.Param.Questions.SystemPrompt, "Output: ", ck, chunkText)+ } else {+ sysPrompt = fmt.Sprintf(autoQuestionPrompt, topN, chunkText)+ userPrompt = "Output: "+ }qTemp := extractorTemperatureqIn := extractorInputs{llmID: in.llmID,- systemPrompt: fmt.Sprintf(autoQuestionPrompt, c.Param.AutoQuestions, chunkText),- prompt: "Output: ",+ systemPrompt: sysPrompt,+ prompt: userPrompt,temperature: &qTemp,}resultStr, err := c.callText(ctx, db, qIn, "")@@ -1701,10 +1798,42 @@ func tryParseJSONObject(s string) (map[string]any, bool) {return out, true}-// init registers Extractor under CategoryIngestion (per plan §4-// Phase 2.5). Metadata is derived from the Inputs()/Outputs()-// methods on ExtractorComponent so the API layer (Phase 4) can-// enumerate the catalog without instantiating the component.+// parseMetadataFieldDefs converts an any value (typically []any of maps)+// to a typed []common.MetadataFieldDef slice.+func parseMetadataFieldDefs(v any) []common.MetadataFieldDef {+ arr, ok := v.([]any)+ if !ok {+ return nil+ }+ fields := make([]common.MetadataFieldDef, 0, len(arr))+ for _, f := range arr {+ m, ok := f.(map[string]any)+ if !ok {+ continue+ }+ key, _ := m["key"].(string)+ if key = strings.TrimSpace(key); key == "" {+ continue+ }+ def := common.MetadataFieldDef{Key: key}+ if t, ok := m["type"].(string); ok {+ def.Type = t+ }+ if d, ok := m["description"].(string); ok {+ def.Description = d+ }+ if e, ok := m["enum"].([]any); ok {+ for _, ev := range e {+ if s, ok := ev.(string); ok {+ def.Enum = append(def.Enum, s)+ }+ }+ }+ fields = append(fields, def)+ }+ return fields+}+// mapInt converts a JSON-compatible value to int.func mapInt(v interface{}) int {switch n := v.(type) {diff --git a/internal/ingestion/component/extractor_metadata_regression_test.go b/internal/ingestion/component/extractor_metadata_regression_test.godeleted file mode 100644index 172f329a6f..0000000000--- a/internal/ingestion/component/extractor_metadata_regression_test.go+++ /dev/null@@ -1,202 +0,0 @@-package component--import (- "testing"- "time"-- "ragflow/internal/common"- "ragflow/internal/ingestion/component/schema"- "ragflow/internal/ingestion/task/indexdoc"-)--// TestRepro_FieldNameMetadata_AggregatesMap guards the unified contract:-// field_name="metadata" (the pipeline/canvas way to produce document metadata)-// writes a parsed map into ck["metadata"] — not a raw string — so the-// aggregation layer, which is strict on map, merges it into document metadata.-func TestRepro_FieldNameMetadata_AggregatesMap(t *testing.T) {- withStubChatInvoker(t,- stubResponse{Content: `{"category":"finance","region":"east"}`},- )- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "metadata",- LLMID: "gpt-4o-mini",- Prompt: "Extract metadata as JSON:",- }}- out, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{"text": "first text"}},- })- if err != nil {- t.Fatalf("Invoke: %v", err)- }- chunks := out["chunks"].([]map[string]any)- meta, ok := chunks[0]["metadata"].(map[string]any)- if !ok {- t.Fatalf("chunk[0].metadata = %T, want map[string]any (field_name=metadata must produce a map, not a string)", chunks[0]["metadata"])- }- if meta["category"] != "finance" || meta["region"] != "east" {- t.Errorf("metadata = %v, want category=finance region=east", meta)- }-- // Feed through the aggregation path, exactly as the executor does.- metadata, err := indexdoc.ProcessChunksForPipeline(chunks, "doc-1", "doc.pdf", time.Now())- if err != nil {- t.Fatalf("ProcessChunksForPipeline: %v", err)- }- if metadata["category"] != "finance" {- t.Errorf("category = %v, want finance", metadata["category"])- }- if metadata["region"] != "east" {- t.Errorf("region = %v, want east", metadata["region"])- }-}--// TestRepro_BothEnabled_MergesMetadata guards the unified contract: when both-// enable_metadata (auto, schema-driven) and field_name="metadata" (pipeline-// custom) are configured, the extractor MERGES them into one ck["metadata"]-// map instead of the field_name result overwriting the auto-metadata. Both-// contribute to the document metadata.-func TestRepro_BothEnabled_MergesMetadata(t *testing.T) {- withStubChatInvoker(t,- // call 1: enable_metadata (runAutoExtractions)- stubResponse{Content: `{"category":"finance","region":"east"}`},- // call 2: field_name="metadata" primary extraction- stubResponse{Content: `{"author":"zhang","date":"2024-01-01"}`},- )- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "metadata",- LLMID: "gpt-4o-mini",- Prompt: "Extract more metadata as JSON:",- EnableMetadata: 1,- Metadata: []common.MetadataFieldDef{- {Key: "category", Type: "string"},- {Key: "region", Type: "string"},- },- }}- out, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{"text": "content"}},- })- if err != nil {- t.Fatalf("Invoke: %v", err)- }- chunks := out["chunks"].([]map[string]any)- meta, ok := chunks[0]["metadata"].(map[string]any)- if !ok {- t.Fatalf("chunk[0].metadata = %T, want merged map; auto-metadata was overwritten", chunks[0]["metadata"])- }- for _, k := range []string{"category", "region", "author", "date"} {- if _, has := meta[k]; !has {- t.Errorf("merged metadata missing %q: %v", k, meta)- }- }-- metadata, err := indexdoc.ProcessChunksForPipeline(chunks, "doc-1", "doc.pdf", time.Now())- if err != nil {- t.Fatalf("ProcessChunksForPipeline: %v", err)- }- for _, k := range []string{"category", "region", "author", "date"} {- if _, has := metadata[k]; !has {- t.Errorf("aggregated doc metadata missing %q: %v", k, metadata)- }- }-}--// TestRepro_OverlapFieldNameWins pins the overlap rule: when enable_metadata-// and field_name="metadata" extract the same key, the field_name (custom,-// later) result wins.-func TestRepro_OverlapFieldNameWins(t *testing.T) {- withStubChatInvoker(t,- // call 1: enable_metadata- stubResponse{Content: `{"category":"finance","region":"east"}`},- // call 2: field_name="metadata"- stubResponse{Content: `{"category":"law","author":"X"}`},- )- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "metadata",- LLMID: "gpt-4o-mini",- Prompt: "Extract metadata as JSON:",- EnableMetadata: 1,- Metadata: []common.MetadataFieldDef{- {Key: "category", Type: "string"},- {Key: "region", Type: "string"},- },- }}- out, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{"text": "content"}},- })- if err != nil {- t.Fatalf("Invoke: %v", err)- }- chunks := out["chunks"].([]map[string]any)- meta := chunks[0]["metadata"].(map[string]any)- if meta["category"] != "law" {- t.Errorf("category = %v, want law (field_name wins on overlap)", meta["category"])- }- if meta["region"] != "east" {- t.Errorf("region = %v, want east (auto field preserved)", meta["region"])- }-}--// TestRepro_FieldNameNonJSON_KeepsEnableMetadata verifies a non-JSON-// field_name="metadata" result does not clobber the enable_metadata map and-// never writes a raw string.-func TestRepro_FieldNameNonJSON_KeepsEnableMetadata(t *testing.T) {- withStubChatInvoker(t,- // call 1: enable_metadata- stubResponse{Content: `{"category":"finance"}`},- // call 2: field_name="metadata" -> non-JSON- stubResponse{Content: "could not find metadata"},- )- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "metadata",- LLMID: "gpt-4o-mini",- Prompt: "Extract metadata as JSON:",- EnableMetadata: 1,- Metadata: []common.MetadataFieldDef{- {Key: "category", Type: "string"},- },- }}- out, err := c.Invoke(t.Context(), nil, map[string]any{- "chunks": []map[string]any{{"text": "content"}},- })- if err != nil {- t.Fatalf("Invoke: %v", err)- }- chunks := out["chunks"].([]map[string]any)- meta, ok := chunks[0]["metadata"].(map[string]any)- if !ok {- t.Fatalf("chunk[0].metadata = %T, want map (auto-metadata preserved, no string written)", chunks[0]["metadata"])- }- if meta["category"] != "finance" {- t.Errorf("category = %v, want finance (enable_metadata map preserved)", meta["category"])- }-}--// TestRepro_NoChunksMetadata_RoutesThroughMap guards the zero-chunk fast path:-// field_name="metadata" with no input chunks must route through the same-// parse-and-merge as the chunked path, writing a map (not a raw string) so the-// strict aggregation layer does not drop the document metadata.-func TestRepro_NoChunksMetadata_RoutesThroughMap(t *testing.T) {- withStubChatInvoker(t,- stubResponse{Content: `{"category":"finance","region":"east"}`},- )- c := &ExtractorComponent{Param: schema.ExtractorParam{- FieldName: "metadata",- LLMID: "gpt-4o-mini",- Prompt: "Extract metadata as JSON:",- }}- out, err := c.Invoke(t.Context(), nil, map[string]any{})- if err != nil {- t.Fatalf("Invoke: %v", err)- }- chunks := out["chunks"].([]map[string]any)- if len(chunks) != 1 {- t.Fatalf("chunks len = %d, want 1", len(chunks))- }- meta, ok := chunks[0]["metadata"].(map[string]any)- if !ok {- t.Fatalf("chunk[0].metadata = %T, want map[string]any (zero-chunk path must not write a raw string)", chunks[0]["metadata"])- }- if meta["category"] != "finance" || meta["region"] != "east" {- t.Errorf("metadata = %v, want category=finance region=east", meta)- }-}diff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.goindex e4f8ca77e3..fd738a0a0e 100644--- a/internal/ingestion/component/extractor_test.go+++ b/internal/ingestion/component/extractor_test.go@@ -19,6 +19,7 @@ package componentimport ("context""errors"+ "fmt""strings""sync""sync/atomic"@@ -2328,3 +2329,488 @@ func TestRenderExtractorPrompts_TableDriven(t *testing.T) {})}}++func TestExtractorModularParams(t *testing.T) {+ params := map[string]any{+ "llm_id": "test-llm-1",+ "keywords": map[string]any{+ "top_n": 5,+ "system_prompt": "Custom keywords prompt for {text}",+ },+ "questions": map[string]any{+ "top_n": 3,+ "system_prompt": "Custom questions prompt for {text}",+ },+ "tags": map[string]any{+ "top_n": 2,+ "tag_file_id": "tag_file_abc",+ },+ "summary": map[string]any{+ "enabled": true,+ "system_prompt": "Custom summary prompt for {text}",+ },+ }++ comp, err := NewExtractorComponent(params)+ if err != nil {+ t.Fatalf("NewExtractorComponent failed: %v", err)+ }++ ext, ok := comp.(*ExtractorComponent)+ if !ok {+ t.Fatalf("expected *ExtractorComponent, got %T", comp)+ }++ if ext.Param.Keywords.TopN != 5 || ext.Param.Keywords.SystemPrompt != "Custom keywords prompt for {text}" {+ t.Errorf("Keywords config mismatch: %+v", ext.Param.Keywords)+ }+ if ext.Param.AutoKeywords != 5 {+ t.Errorf("AutoKeywords flat sync mismatch: %d", ext.Param.AutoKeywords)+ }++ if ext.Param.Questions.TopN != 3 || ext.Param.Questions.SystemPrompt != "Custom questions prompt for {text}" {+ t.Errorf("Questions config mismatch: %+v", ext.Param.Questions)+ }+ if ext.Param.AutoQuestions != 3 {+ t.Errorf("AutoQuestions flat sync mismatch: %d", ext.Param.AutoQuestions)+ }++ if ext.Param.Tags.TopN != 2 || ext.Param.Tags.TagFileID != "tag_file_abc" {+ t.Errorf("Tags config mismatch: %+v", ext.Param.Tags)+ }+ if ext.Param.AutoTags != 2 || ext.Param.TagFileID != "tag_file_abc" {+ t.Errorf("AutoTags flat sync mismatch: %d, %s", ext.Param.AutoTags, ext.Param.TagFileID)+ }++ if !ext.Param.Summary.Enabled || ext.Param.Summary.SystemPrompt != "Custom summary prompt for {text}" {+ t.Errorf("Summary config mismatch: %+v", ext.Param.Summary)+ }+ if ext.Param.FieldName != "summary" {+ t.Errorf("FieldName mismatch: %s", ext.Param.FieldName)+ }+}++func TestExtractorLegacyFlatParamsFallback(t *testing.T) {+ params := map[string]any{+ "llm_id": "test-llm-1",+ "auto_keywords": 4,+ "auto_questions": 2,+ "auto_tags": 1,+ "tag_file_id": "tag_file_legacy",+ "field_name": "summary",+ "sys_prompt": "Legacy summary prompt",+ }++ comp, err := NewExtractorComponent(params)+ if err != nil {+ t.Fatalf("NewExtractorComponent failed: %v", err)+ }++ ext, ok := comp.(*ExtractorComponent)+ if !ok {+ t.Fatalf("expected *ExtractorComponent, got %T", comp)+ }++ if ext.Param.Keywords.TopN != 4 || ext.Param.AutoKeywords != 4 {+ t.Errorf("Keywords fallback mismatch: %d / %d", ext.Param.Keywords.TopN, ext.Param.AutoKeywords)+ }+ if ext.Param.Questions.TopN != 2 || ext.Param.AutoQuestions != 2 {+ t.Errorf("Questions fallback mismatch: %d / %d", ext.Param.Questions.TopN, ext.Param.AutoQuestions)+ }+ if ext.Param.Tags.TopN != 1 || ext.Param.Tags.TagFileID != "tag_file_legacy" {+ t.Errorf("Tags fallback mismatch: %+v", ext.Param.Tags)+ }+ if !ext.Param.Summary.Enabled || ext.Param.Summary.SystemPrompt != "Legacy summary prompt" {+ t.Errorf("Summary fallback mismatch: %+v", ext.Param.Summary)+ }+}++func TestExtractorModularPromptsExecution(t *testing.T) {+ stub := withStubChatInvoker(t,+ stubResponse{Content: "kw1, kw2"},+ stubResponse{Content: "question 1?\nquestion 2?"},+ stubResponse{Content: "Summary of chunk"},+ )++ params := map[string]any{+ "llm_id": "llm-1",+ "keywords": map[string]any{+ "top_n": 2,+ "system_prompt": "Custom KW Prompt: {text}",+ },+ "questions": map[string]any{+ "top_n": 2,+ "system_prompt": "Custom Q Prompt: {text}",+ },+ "summary": map[string]any{+ "enabled": true,+ "system_prompt": "Custom Sum Prompt: {text}",+ },+ }++ comp, err := NewExtractorComponent(params)+ if err != nil {+ t.Fatalf("NewExtractorComponent: %v", err)+ }++ in := map[string]any{+ "chunks": []map[string]any{+ {"content_with_weight": "Hello world content"},+ },+ }++ out, err := comp.Invoke(t.Context(), nil, in)+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }++ chunks, ok := out["chunks"].([]map[string]any)+ if !ok || len(chunks) != 1 {+ t.Fatalf("expected 1 chunk output, got %v", out)+ }++ ck := chunks[0]+ if kwds, ok := ck["important_kwd"].([]string); !ok || len(kwds) != 2 {+ t.Errorf("expected important_kwd = [kw1, kw2], got %v", ck["important_kwd"])+ }+ if qs, ok := ck["question_kwd"].([]string); !ok || len(qs) != 2 {+ t.Errorf("expected question_kwd = [question 1?, question 2?], got %v", ck["question_kwd"])+ }+ if sum, ok := ck["summary"].(string); !ok || sum != "Summary of chunk" {+ t.Errorf("expected summary = 'Summary of chunk', got %v", ck["summary"])+ }++ // Verify custom prompts were rendered into messages and passed to the chat invoker+ reqs := stub.requests+ if len(reqs) != 3 {+ t.Fatalf("expected 3 LLM calls, got %d", len(reqs))+ }++ var foundKW, foundQ, foundSum bool+ for _, r := range reqs {+ for _, msg := range r.Messages {+ if strings.Contains(msg.Content, "Custom KW Prompt: Hello world content") {+ foundKW = true+ }+ if strings.Contains(msg.Content, "Custom Q Prompt: Hello world content") {+ foundQ = true+ }+ if strings.Contains(msg.Content, "Custom Sum Prompt: Hello world content") {+ foundSum = true+ }+ }+ }++ if !foundKW {+ t.Errorf("expected custom keyword prompt to be used in LLM call, got reqs: %+v", reqs)+ }+ if !foundQ {+ t.Errorf("expected custom question prompt to be used in LLM call, got reqs: %+v", reqs)+ }+ if !foundSum {+ t.Errorf("expected custom summary prompt to be used in LLM call, got reqs: %+v", reqs)+ }+}++func TestExtractorModularMetadataConfig(t *testing.T) {+ params := map[string]any{+ "llm_id": "llm-1",+ "metadata_config": map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]any{+ "key": "author",+ "type": "string",+ "description": "The author name",+ "enum": []any{"Alice", "Bob"},+ },+ map[string]any{+ "key": "year",+ "type": "integer",+ },+ },+ "built_in_metadata": []any{+ map[string]any{+ "key": "file_name",+ "type": "string",+ },+ },+ },+ }++ comp, err := NewExtractorComponent(params)+ if err != nil {+ t.Fatalf("NewExtractorComponent: %v", err)+ }++ ext, ok := comp.(*ExtractorComponent)+ if !ok {+ t.Fatalf("expected *ExtractorComponent, got %T", comp)+ }++ if !ext.Param.MetadataConfig.Enabled || ext.Param.EnableMetadata != 1 {+ t.Errorf("Metadata enabled mismatch: %+v / %d", ext.Param.MetadataConfig.Enabled, ext.Param.EnableMetadata)+ }+ if len(ext.Param.MetadataConfig.Metadata) != 2 || len(ext.Param.Metadata) != 2 {+ t.Fatalf("Metadata fields mismatch: %d / %d", len(ext.Param.MetadataConfig.Metadata), len(ext.Param.Metadata))+ }+ if ext.Param.Metadata[0].Key != "author" || ext.Param.Metadata[0].Description != "The author name" || len(ext.Param.Metadata[0].Enum) != 2 {+ t.Errorf("Metadata field 0 mismatch: %+v", ext.Param.Metadata[0])+ }+ if len(ext.Param.MetadataConfig.BuiltInMetadata) != 1 || ext.Param.MetadataConfig.BuiltInMetadata[0].Key != "file_name" {+ t.Errorf("BuiltInMetadata mismatch: %+v", ext.Param.MetadataConfig.BuiltInMetadata)+ }+}++func TestExtractorModularMetadataExecution(t *testing.T) {+ withStubChatInvoker(t,+ stubResponse{Content: `{"author": "Alice", "year": 2026}`},+ )++ params := map[string]any{+ "llm_id": "llm-1",+ "metadata_config": map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]any{+ "key": "author",+ "type": "string",+ },+ map[string]any{+ "key": "year",+ "type": "integer",+ },+ },+ },+ }++ comp, err := NewExtractorComponent(params)+ if err != nil {+ t.Fatalf("NewExtractorComponent: %v", err)+ }++ in := map[string]any{+ "chunks": []map[string]any{+ {"content_with_weight": "Written by Alice in 2026."},+ },+ }++ out, err := comp.Invoke(t.Context(), nil, in)+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }++ chunks, ok := out["chunks"].([]map[string]any)+ if !ok || len(chunks) != 1 {+ t.Fatalf("expected 1 chunk output, got %v", out)+ }++ ck := chunks[0]+ meta, ok := ck["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("expected chunk metadata map, got %T: %v", ck["metadata"], ck["metadata"])+ }+ if meta["author"] != "Alice" {+ t.Errorf("expected metadata.author = Alice, got %v", meta["author"])+ }+ // year can be float64 or int from JSON+ if fmt.Sprintf("%v", meta["year"]) != "2026" {+ t.Errorf("expected metadata.year = 2026, got %v", meta["year"])+ }+}++func TestExtractorDefaultSummaryPromptInjection(t *testing.T) {+ stub := withStubChatInvoker(t, stubResponse{Content: "A concise summary."})++ params := map[string]any{+ "llm_id": "llm-1",+ "summary": map[string]any{+ "enabled": true,+ // system_prompt left empty to test default prompt injection+ },+ }++ comp, err := NewExtractorComponent(params)+ if err != nil {+ t.Fatalf("NewExtractorComponent: %v", err)+ }++ in := map[string]any{+ "chunks": []map[string]any{+ {"content_with_weight": "This is a detailed paragraph about artificial intelligence."},+ },+ }++ out, err := comp.Invoke(t.Context(), nil, in)+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }++ chunks, ok := out["chunks"].([]map[string]any)+ if !ok || len(chunks) != 1 {+ t.Fatalf("expected 1 chunk output, got %v", out)+ }++ if chunks[0]["summary"] != "A concise summary." {+ t.Errorf("expected summary = 'A concise summary.', got %v", chunks[0]["summary"])+ }++ if stub.Calls() != 1 {+ t.Fatalf("expected 1 LLM call, got %d", stub.Calls())+ }++ lastReq := stub.lastRequest()+ msgs := lastReq.Messages+ if len(msgs) != 2 {+ t.Fatalf("expected 2 messages (system + user), got %d: %+v", len(msgs), msgs)+ }++ if msgs[0].Role != "system" || !strings.Contains(msgs[0].Content, "You are a precise and faithful text summarizer") {+ t.Errorf("expected autoSummaryPrompt in system message, got: %+v", msgs[0])+ }++ if msgs[1].Role != "user" || !strings.Contains(msgs[1].Content, "This is a detailed paragraph about artificial intelligence.") {+ t.Errorf("expected chunk text in user message, got: %+v", msgs[1])+ }+}++func TestExtractorDisabledSummarySkipsCall(t *testing.T) {+ stub := withStubChatInvoker(t, stubResponse{Content: "Not expected"})++ params := map[string]any{+ "llm_id": "llm-1",+ "summary": map[string]any{+ "enabled": false,+ },+ "field_name": "summary", // legacy default that should be ignored when enabled=false+ }++ comp, err := NewExtractorComponent(params)+ if err != nil {+ t.Fatalf("NewExtractorComponent: %v", err)+ }++ in := map[string]any{+ "chunks": []map[string]any{+ {"content_with_weight": "Some text."},+ },+ }++ out, err := comp.Invoke(t.Context(), nil, in)+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }++ if stub.Calls() != 0 {+ t.Errorf("expected 0 LLM calls when summary is disabled, got %d", stub.Calls())+ }++ chunks, ok := out["chunks"].([]map[string]any)+ if !ok || len(chunks) != 1 {+ t.Fatalf("expected 1 chunk output, got %v", out)+ }+ if _, has := chunks[0]["summary"]; has {+ t.Errorf("expected no summary key in chunk, got %v", chunks[0]["summary"])+ }+}++func TestExtractor_NestedPrecedenceOverFlat(t *testing.T) {+ // Case A: Nested explicitly disabled (enabled=false) while legacy flat has enabled=1/5.+ // Nested MUST win and disable the features.+ paramsDisabled := map[string]any{+ "metadata_config": map[string]any{+ "enabled": false,+ "metadata": []any{+ map[string]any{"key": "category", "type": "string"},+ },+ },+ "enable_metadata": 1,+ "summary": map[string]any{+ "enabled": false,+ },+ "enable_summary": 1,+ "keywords": map[string]any{+ "top_n": 0,+ },+ "auto_keywords": 5,+ "questions": map[string]any{+ "top_n": 0,+ },+ "auto_questions": 3,+ "tags": map[string]any{+ "top_n": 0,+ },+ "auto_tags": 2,+ }++ compRawA, err := NewExtractorComponent(paramsDisabled)+ if err != nil {+ t.Fatalf("NewExtractorComponent A: %v", err)+ }+ compA := compRawA.(*ExtractorComponent)+ if compA.Param.MetadataConfig.Enabled != false || compA.Param.EnableMetadata != 0 {+ t.Errorf("expected metadata disabled, got nested=%v flat=%v", compA.Param.MetadataConfig.Enabled, compA.Param.EnableMetadata)+ }+ if compA.Param.Summary.Enabled != false || compA.Param.EnableSummary != 0 {+ t.Errorf("expected summary disabled, got nested=%v flat=%v", compA.Param.Summary.Enabled, compA.Param.EnableSummary)+ }+ if compA.Param.Keywords.TopN != 0 || compA.Param.AutoKeywords != 0 {+ t.Errorf("expected keywords disabled (0), got nested=%v flat=%v", compA.Param.Keywords.TopN, compA.Param.AutoKeywords)+ }+ if compA.Param.Questions.TopN != 0 || compA.Param.AutoQuestions != 0 {+ t.Errorf("expected questions disabled (0), got nested=%v flat=%v", compA.Param.Questions.TopN, compA.Param.AutoQuestions)+ }+ if compA.Param.Tags.TopN != 0 || compA.Param.AutoTags != 0 {+ t.Errorf("expected tags disabled (0), got nested=%v flat=%v", compA.Param.Tags.TopN, compA.Param.AutoTags)+ }++ // Case B: Nested explicitly enabled (enabled=true) while legacy flat has 0.+ // Nested MUST win and enable the features.+ paramsEnabled := map[string]any{+ "metadata_config": map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]any{"key": "author", "type": "string"},+ },+ },+ "enable_metadata": 0,+ "summary": map[string]any{+ "enabled": true,+ },+ "enable_summary": 0,+ "keywords": map[string]any{+ "top_n": 4,+ },+ "auto_keywords": 0,+ "questions": map[string]any{+ "top_n": 2,+ },+ "auto_questions": 0,+ "tags": map[string]any{+ "top_n": 3,+ "tag_file_id": "file-123",+ },+ "auto_tags": 0,+ }++ compRawB, err := NewExtractorComponent(paramsEnabled)+ if err != nil {+ t.Fatalf("NewExtractorComponent B: %v", err)+ }+ compB := compRawB.(*ExtractorComponent)+ if compB.Param.MetadataConfig.Enabled != true || compB.Param.EnableMetadata != 1 {+ t.Errorf("expected metadata enabled, got nested=%v flat=%v", compB.Param.MetadataConfig.Enabled, compB.Param.EnableMetadata)+ }+ if compB.Param.Summary.Enabled != true || compB.Param.EnableSummary != 1 {+ t.Errorf("expected summary enabled, got nested=%v flat=%v", compB.Param.Summary.Enabled, compB.Param.EnableSummary)+ }+ if compB.Param.Keywords.TopN != 4 || compB.Param.AutoKeywords != 4 {+ t.Errorf("expected keywords 4, got nested=%v flat=%v", compB.Param.Keywords.TopN, compB.Param.AutoKeywords)+ }+ if compB.Param.Questions.TopN != 2 || compB.Param.AutoQuestions != 2 {+ t.Errorf("expected questions 2, got nested=%v flat=%v", compB.Param.Questions.TopN, compB.Param.AutoQuestions)+ }+ if compB.Param.Tags.TopN != 3 || compB.Param.AutoTags != 3 {+ t.Errorf("expected tags 3, got nested=%v flat=%v", compB.Param.Tags.TopN, compB.Param.AutoTags)+ }+}diff --git a/internal/ingestion/component/schema/extractor.go b/internal/ingestion/component/schema/extractor.goindex 783defc253..4b123e73bf 100644--- a/internal/ingestion/component/schema/extractor.go+++ b/internal/ingestion/component/schema/extractor.go@@ -74,18 +74,52 @@ type ExtractorFromUpstream struct {// chunk from the LLM call).func (ExtractorFromUpstream) Validate() error { return nil }+// KeywordExtractConfig configures automatic keyword extraction.+type KeywordExtractConfig struct {+ TopN int `json:"top_n,omitempty"`+ SystemPrompt string `json:"system_prompt,omitempty"`+}++// QuestionExtractConfig configures automatic question generation.+type QuestionExtractConfig struct {+ TopN int `json:"top_n,omitempty"`+ SystemPrompt string `json:"system_prompt,omitempty"`+}++// TagExtractConfig configures automatic tag extraction.+type TagExtractConfig struct {+ TopN int `json:"top_n,omitempty"`+ TagFileID string `json:"tag_file_id,omitempty"`+}++// SummaryExtractConfig configures summary / enhanced context extraction.+type SummaryExtractConfig struct {+ Enabled bool `json:"enabled,omitempty"`+ SystemPrompt string `json:"system_prompt,omitempty"`+}++// MetadataExtractConfig configures structured metadata extraction.+type MetadataExtractConfig struct {+ Enabled bool `json:"enabled,omitempty"`+ Metadata []common.MetadataFieldDef `json:"metadata,omitempty"`+ BuiltInMetadata []common.MetadataFieldDef `json:"built_in_metadata,omitempty"`+}+// ExtractorParam is the static configuration for the Extractor-// component. Mirrors rag/flow/extractor/extractor.py:ExtractorParam,-// which extends both ProcessParamBase and LLMParam. The LLM fields-// (`llm_id`, `parameters`, `system_prompt`, `prompt`, `messages`,-// etc.) live on the agent-side `LLMParam`; the Go port captures only-// the Extractor-specific field plus a pointer to the LLM config so-// the wiring is explicit.+// component. Supports modular sub-configs (Keywords, Questions, Tags,+// Summary, Metadata) as well as legacy flat fields for backward compatibility.type ExtractorParam struct {+ // Modular sub-configs+ Keywords KeywordExtractConfig `json:"keywords,omitempty"`+ Questions QuestionExtractConfig `json:"questions,omitempty"`+ Tags TagExtractConfig `json:"tags,omitempty"`+ Summary SummaryExtractConfig `json:"summary,omitempty"`+ MetadataConfig MetadataExtractConfig `json:"metadata_config,omitempty"`+// FieldName is the chunk key the LLM extraction result is written// to (Python: `self._param.field_name`). Optional — when empty,// auto_keywords or auto_questions may still be used.- FieldName string `json:"field_name"`+ FieldName string `json:"field_name,omitempty"`// LLMID identifies the LLM model used for extraction. This is the// agent-side LLMParam.llm_id; on the ingestion side it is@@ -99,29 +133,24 @@ type ExtractorParam struct {Prompt string `json:"prompt,omitempty"`// AutoKeywords enables automatic keyword extraction with a fixed- // prompt. The value determines the top-N count.+ // prompt. The value determines the top-N count. Legacy flat field.AutoKeywords int `json:"auto_keywords,omitempty"`// AutoQuestions enables automatic question generation with a fixed- // prompt. The value determines the top-N count.+ // prompt. The value determines the top-N count. Legacy flat field.AutoQuestions int `json:"auto_questions,omitempty"`- // AutoTags enables tag assignment on chunks. When > 0, the- // component runs a two-phase tagger: Phase 1 uses Jaccard- // matching against tag source examples; Phase 2 uses the LLM- // for unmatched chunks. The value determines the top-N tags.+ // AutoTags enables tag assignment on chunks. Legacy flat field.AutoTags int `json:"auto_tags,omitempty"`// TagFileID references a tag-definition file stored in object- // storage. Used only when AutoTags > 0 and no inline tag- // source text is wired in.- TagFileID string `json:"tag_file_id"`-- // EnableMetadata enables automatic structured metadata extraction- // with a fixed prompt. When > 0, runEnableMetadata asks the LLM to- // fill the fields listed in Metadata as a JSON object and- // merges the result into chunk["metadata"] (mirrors Python's- // gen_metadata_task path; see doc_metadata_go_port_research.md).+ // storage. Legacy flat field.+ TagFileID string `json:"tag_file_id,omitempty"`++ // EnableSummary enables summary extraction. Legacy flat field.+ EnableSummary int `json:"enable_summary,omitempty"`++ // EnableMetadata enables automatic structured metadata extraction. Legacy flat field.EnableMetadata int `json:"enable_metadata,omitempty"`// Metadata lists the target field definitions for@@ -142,6 +171,7 @@ func (ExtractorParam) Defaults() ExtractorParam {AutoQuestions: 0,AutoTags: 0,TagFileID: "",+ EnableSummary: 0,EnableMetadata: 0,Metadata: nil,}diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.goindex 8010e58150..d6784fe80c 100644--- a/internal/ingestion/pipeline/pipeline_params.go+++ b/internal/ingestion/pipeline/pipeline_params.go@@ -55,10 +55,10 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[cleaned[pk] = pvcontinue}- // Auto metadata params are valid extractor params even when a- // builtin template does not declare them (the frontend stores+ // Extractor params are valid even when a builtin template does+ // not declare them explicitly in params: {} (the frontend stores// them on the node params); keep them through the filter.- if isExtractorComponent(key) && isAutoMetadataParam(pk) {+ if isExtractorComponent(key) && isExtractorParam(pk) {cleaned[pk] = pvcontinue}@@ -164,11 +164,19 @@ func isExtractorComponent(cpnID string) bool {return strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_")}-// isAutoMetadataParam reports whether a param belongs to the Auto metadata-// feature (enable_metadata toggle + metadata / built_in_metadata schemas).-func isAutoMetadataParam(param string) bool {+// isExtractorParam reports whether a param belongs to the Extractor component+// (modular sub-configs, legacy flat fields, LLM settings, or UI state flags).+func isExtractorParam(param string) bool {switch param {- case "enable_metadata", "metadata", "built_in_metadata":+ case "keywords", "questions", "tags", "summary", "metadata_config",+ "auto_keywords", "keywords_sys_prompt",+ "auto_questions", "questions_sys_prompt",+ "auto_tags", "tag_file_id",+ "enable_summary", "sys_prompt", "system_prompt", "prompt", "prompts", "field_name",+ "enable_metadata", "metadata", "built_in_metadata",+ "llm_id", "temperature", "top_p", "max_tokens", "presence_penalty", "frequency_penalty",+ "temperatureEnabled", "topPEnabled", "maxTokensEnabled", "presencePenaltyEnabled", "frequencyPenaltyEnabled",+ "outputs":return true}return falsediff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.goindex 4dc776912d..1f544dc05d 100644--- a/internal/ingestion/pipeline/pipeline_params_test.go+++ b/internal/ingestion/pipeline/pipeline_params_test.go@@ -354,3 +354,61 @@ func TestBuildParserConfig_BuiltinExtractorKeepsBuiltInMetadata(t *testing.T) {t.Fatal("expected at least one builtin template with an Extractor component")}}++func TestCleanComponentParams_KeepsModularExtractorParams(t *testing.T) {+ // Builtin template with empty Extractor params: {}+ dsl := map[string]any{+ "components": map[string]any{+ "Extractor:AutoExtractDefault": map[string]any{+ "obj": map[string]any{+ "component_name": "Extractor",+ "params": map[string]any{},+ },+ },+ },+ }+ dslJSON, err := json.Marshal(dsl)+ if err != nil {+ t.Fatalf("marshal dsl: %v", err)+ }++ overrides := map[string]any{+ "Extractor:AutoExtractDefault": map[string]any{+ "summary": map[string]any{+ "enabled": true,+ "system_prompt": "summary prompt",+ },+ "metadata_config": map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]any{"key": "category", "type": "string"},+ },+ },+ "keywords": map[string]any{+ "top_n": 5,+ },+ "enable_summary": 1,+ "llm_id": "gpt-4",+ "temperature": 0.7,+ },+ }++ result := CleanComponentParams(dslJSON, overrides)+ ext, ok := result["Extractor:AutoExtractDefault"].(map[string]any)+ if !ok {+ t.Fatalf("expected Extractor:AutoExtractDefault in result, got: %v", result)+ }++ if sum, ok := ext["summary"].(map[string]any); !ok || sum["enabled"] != true {+ t.Errorf("expected summary.enabled == true, got: %v", ext["summary"])+ }+ if meta, ok := ext["metadata_config"].(map[string]any); !ok || meta["enabled"] != true {+ t.Errorf("expected metadata_config.enabled == true, got: %v", ext["metadata_config"])+ }+ if kw, ok := ext["keywords"].(map[string]any); !ok || kw["top_n"] != 5 {+ t.Errorf("expected keywords.top_n == 5, got: %v", ext["keywords"])+ }+ if ext["llm_id"] != "gpt-4" {+ t.Errorf("expected llm_id == gpt-4, got: %v", ext["llm_id"])+ }+}diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex 4b36b0f781..a8bbe30d55 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -301,19 +301,39 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map}// builtInMetadataFromParserConfig extracts the built-in metadata config-// (update_time / file_name) and whether auto-metadata is enabled. The frontend-// stores both on the builtin extractor node params (Extractor:AutoExtractDefault)-// via the operator form; the top-level parser_config shape (Python dataset-// settings) is supported as a fallback.+// (update_time / file_name) and whether auto-metadata is enabled. Supports+// modular metadata_config sub-objects as well as legacy flat fields on+// Extractor nodes and top-level parser_config.func builtInMetadataFromParserConfig(parserConfig entity.JSONMap) ([]any, bool) {- if nodeRaw, ok := parserConfig["Extractor:AutoExtractDefault"]; ok {+ var extractorKeys []string+ for k := range parserConfig {+ if strings.HasPrefix(k, "Extractor") {+ extractorKeys = append(extractorKeys, k)+ }+ }+ sort.Strings(extractorKeys)++ for _, k := range extractorKeys {+ nodeRaw := parserConfig[k]if node, ok := nodeRaw.(map[string]any); ok {+ if metaConf, ok := node["metadata_config"].(map[string]any); ok {+ enabled := parserConfigBool(metaConf["enabled"])+ if arr, ok := metaConf["built_in_metadata"].([]any); ok && len(arr) > 0 {+ return arr, enabled+ }+ }enabled := parserConfigBool(node["enable_metadata"])if arr, ok := node["built_in_metadata"].([]any); ok && len(arr) > 0 {return arr, enabled}}}+ if metaConf, ok := parserConfig["metadata_config"].(map[string]any); ok {+ enabled := parserConfigBool(metaConf["enabled"])+ if arr, ok := metaConf["built_in_metadata"].([]any); ok && len(arr) > 0 {+ return arr, enabled+ }+ }if arr, ok := parserConfig["built_in_metadata"].([]any); ok && len(arr) > 0 {return arr, parserConfigBool(parserConfig["enable_metadata"])}diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.goindex 6f52677cba..6d9fcca7be 100644--- a/internal/ingestion/task/pipeline_executor_defaults_test.go+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go@@ -234,3 +234,39 @@ func TestDocNameValue(t *testing.T) {t.Errorf("docNameValue(nil) = %q, want empty", got)}}++func TestBuiltInMetadataFromParserConfig_MultiExtractorDeterministic(t *testing.T) {+ // Multiple Extractor nodes with different built_in_metadata.+ // Since keys are sorted alphabetically ("Extractor:A" before "Extractor:B"),+ // Extractor:A must deterministically win regardless of Go map iteration order.+ pc := entity.JSONMap{+ "Extractor:B": map[string]any{+ "enable_metadata": 1,+ "built_in_metadata": []any{+ map[string]any{"key": "file_name", "type": "string"},+ },+ },+ "Extractor:A": map[string]any{+ "metadata_config": map[string]any{+ "enabled": true,+ "built_in_metadata": []any{+ map[string]any{"key": "update_time", "type": "time"},+ },+ },+ },+ }++ for i := 0; i < 20; i++ {+ cfg, enabled := builtInMetadataFromParserConfig(pc)+ if !enabled {+ t.Fatalf("iteration %d: expected enabled=true", i)+ }+ if len(cfg) != 1 {+ t.Fatalf("iteration %d: expected 1 item, got %d", i, len(cfg))+ }+ item, ok := cfg[0].(map[string]any)+ if !ok || item["key"] != "update_time" {+ t.Fatalf("iteration %d: expected Extractor:A item update_time, got %v", i, cfg[0])+ }+ }+}diff --git a/web/src/locales/en.ts b/web/src/locales/en.tsindex 595d831b1b..fb5f9908b2 100644--- a/web/src/locales/en.ts+++ b/web/src/locales/en.ts@@ -3332,8 +3332,11 @@ The Indexer will store the content in the corresponding data structures for thesummary: 'Summary',keywords: 'Keywords',questions: 'Questions',+ tags: 'Tags',metadata: 'Metadata',fieldName: 'Result destination',+ enableSummary: 'Enable Summary',+ useBuiltInTemplate: 'Use built-in template',prompts: {system: {keywords: `Rolediff --git a/web/src/locales/zh.ts b/web/src/locales/zh.tsindex c68fcdbb58..8ee02c6f71 100644--- a/web/src/locales/zh.ts+++ b/web/src/locales/zh.ts@@ -2914,8 +2914,11 @@ Tokenizer 会根据所选方式将内容存储为对应的数据结构。`,summary: '增强上下文',keywords: '关键词',questions: '问题',+ tags: '标签',metadata: '元数据',fieldName: '结果目的地',+ enableSummary: '启用增强上下文',+ useBuiltInTemplate: '使用内置模板',prompts: {system: {keywords: `角色diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsxindex 3f0d8a491b..e42c264ab8 100644--- a/web/src/pages/agent/constant/pipeline.tsx+++ b/web/src/pages/agent/constant/pipeline.tsx@@ -92,13 +92,6 @@ export const InitialOutputFormatMap = {[FileType.Audio]: AudioOutputFormat.Text,};-export enum ContextGeneratorFieldName {- Summary = 'summary',- Keywords = 'keywords',- Questions = 'questions',- Metadata = 'metadata',-}-export const FileId = 'File'; // BeginIdexport enum TokenizerSearchMethod {@@ -350,11 +343,39 @@ export const initialTitleChunkerValues = {export const initialExtractorValues = {...initialLlmBaseValues,- field_name: ContextGeneratorFieldName.Summary,+ keywords: {+ top_n: 0,+ system_prompt: '',+ },+ questions: {+ top_n: 0,+ system_prompt: '',+ },+ tags: {+ top_n: 0,+ tag_file_id: '',+ },+ summary: {+ enabled: false,+ system_prompt: '',+ },+ metadata_config: {+ enabled: false,+ metadata: [],+ built_in_metadata: [],+ },+ metadata: [],+ built_in_metadata: [],+ field_name: '',auto_keywords: 0,auto_questions: 0,- auto_tags: 1,+ auto_tags: 0,tag_file_id: '',+ enable_summary: 0,+ enable_metadata: 0,+ keywords_sys_prompt: '',+ questions_sys_prompt: '',+ sys_prompt: '',outputs: {chunks: { type: 'Array<Object>', value: [] },},diff --git a/web/src/pages/agent/form/components/prompt-editor/index.tsx b/web/src/pages/agent/form/components/prompt-editor/index.tsxindex 5ec8cb4c9b..4879e08dbe 100644--- a/web/src/pages/agent/form/components/prompt-editor/index.tsx+++ b/web/src/pages/agent/form/components/prompt-editor/index.tsx@@ -62,6 +62,7 @@ const Nodes: Array<Klass<LexicalNode>> = [type PromptContentProps = {enablePathQueryAutoMerge: boolean;showToolbar?: boolean;+ showMergePath?: boolean;multiLine?: boolean;onBlur?: () => void;onEnablePathQueryAutoMergeChange: (checked: boolean) => void;@@ -70,6 +71,7 @@ type PromptContentProps = {type IProps = {enablePathQueryAutoMerge?: boolean;showToolbar?: boolean;+ showMergePath?: boolean;multiLine?: boolean;value?: string;onChange?: (value?: string) => void;@@ -81,6 +83,7 @@ type IProps = {function PromptContent({enablePathQueryAutoMerge,showToolbar = true,+ showMergePath = true,multiLine = true,onBlur,onEnablePathQueryAutoMergeChange,@@ -128,28 +131,30 @@ function PromptContent({<p>{t('flow.insertVariableTip')}</p></TooltipContent></Tooltip>- <Tooltip>- <TooltipTrigger asChild>- <label className="flex cursor-pointer items-center rounded-sm border border-border bg-bg-base/95 px-1 py-0.5 shadow-sm backdrop-blur-sm">- <span className="sr-only">{t('flow.mergePath')}</span>- <div className="origin-right scale-75">- <Switch- checked={enablePathQueryAutoMerge}- onCheckedChange={onEnablePathQueryAutoMergeChange}- aria-label={t('flow.mergePath')}- />- </div>- </label>- </TooltipTrigger>- <TooltipContent>- <p>{t('flow.mergePath')}</p>- <p>{t('flow.mergePathTip')}</p>- </TooltipContent>- </Tooltip>+ {showMergePath && (+ <Tooltip>+ <TooltipTrigger asChild>+ <label className="flex cursor-pointer items-center rounded-sm border border-border bg-bg-base/95 px-1 py-0.5 shadow-sm backdrop-blur-sm">+ <span className="sr-only">{t('flow.mergePath')}</span>+ <div className="origin-right scale-75">+ <Switch+ checked={enablePathQueryAutoMerge}+ onCheckedChange={onEnablePathQueryAutoMergeChange}+ aria-label={t('flow.mergePath')}+ />+ </div>+ </label>+ </TooltipTrigger>+ <TooltipContent>+ <p>{t('flow.mergePath')}</p>+ <p>{t('flow.mergePathTip')}</p>+ </TooltipContent>+ </Tooltip>+ )}</div>)}<div className="relative">- {!showToolbar && (+ {!showToolbar && showMergePath && (<div className="absolute inset-y-0 right-2 z-10 flex items-center"><Tooltip><TooltipTrigger asChild>@@ -173,8 +178,9 @@ function PromptContent({)}<ContentEditableclassName={cn(- 'relative px-2 py-1 pr-14 focus-visible:outline-none max-h-[50vh] overflow-auto text-sm',+ 'relative px-2 py-1 focus-visible:outline-none max-h-[50vh] overflow-auto text-sm',{+ 'pr-14': !showToolbar && showMergePath,'min-h-40': multiLine,},)}@@ -193,6 +199,7 @@ export const PromptEditor = forwardRef(function PromptEditor(onBlur,placeholder,showToolbar = true,+ showMergePath = true,multiLine = true,enablePathQueryAutoMerge = true,extraOptions,@@ -222,11 +229,7 @@ export const PromptEditor = forwardRef(function PromptEditor(const onValueChange = useCallback((editorState: EditorState) => {editorState?.read(() => {- // const listNodes = $nodesOfType(VariableNode); // to be removed- // const allNodes = $dfs();-const text = $getRoot().getTextContent();-onChange?.(text);});},@@ -241,6 +244,7 @@ export const PromptEditor = forwardRef(function PromptEditor(<PromptContentenablePathQueryAutoMerge={isPathQueryAutoMergeEnabled}showToolbar={showToolbar}+ showMergePath={showMergePath}multiLine={multiLine}onBlur={onBlur}onEnablePathQueryAutoMergeChange={setIsPathQueryAutoMergeEnabled}diff --git a/web/src/pages/agent/form/extractor-form/index.tsx b/web/src/pages/agent/form/extractor-form/index.tsxindex 23d5337333..0eae00437e 100644--- a/web/src/pages/agent/form/extractor-form/index.tsx+++ b/web/src/pages/agent/form/extractor-form/index.tsx@@ -1,17 +1,34 @@+/*+ * Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+import {AutoKeywordsFormField,AutoQuestionsFormField,} from '@/components/auto-keywords-form-field';-import { ConfirmDeleteDialog } from '@/components/confirm-delete-dialog';import { LargeModelFormField } from '@/components/large-model-form-field';import { LlmSettingSchema } from '@/components/llm-setting-items/next';-import { SelectWithSearch } from '@/components/originui/select-with-search';import { RAGFlowFormItem } from '@/components/ragflow-form';import { SliderInputFormField } from '@/components/slider-input-form-field';import { AsyncTreeSelect } from '@/components/ui/async-tree-select';-import { Form } from '@/components/ui/form';import { Button } from '@/components/ui/button';+import { Form } from '@/components/ui/form';import { Switch } from '@/components/ui/switch';+import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';+import { FormLayout } from '@/constants/form';+import { RAGFlowNodeType } from '@/interfaces/database/agent';import { PromptEditor } from '@/pages/agent/form/components/prompt-editor';import { MetadataType } from '@/pages/dataset/components/metedata/constant';import {@@ -23,42 +40,67 @@ import {IMetaDataReturnJSONSettings,} from '@/pages/dataset/components/metedata/interface';import { ManageMetadataModal } from '@/pages/dataset/components/metedata/manage-modal';-import { isGoBackend } from '@/utils/backend-runtime';-import { buildOptions } from '@/utils/form';import { zodResolver } from '@hookform/resolvers/zod';import { Settings } from 'lucide-react';-import { memo, useCallback } from 'react';+import { memo, useCallback, useEffect, useMemo, useState } from 'react';import { useForm, useFormContext } from 'react-hook-form';import { useTranslation } from 'react-i18next';import { z } from 'zod';-import {- ContextGeneratorFieldName,- initialExtractorValues,-} from '../../constant/pipeline';+import { initialExtractorValues } from '../../constant/pipeline';import { useOwnerTenantId } from '../../context';-import { useBuildNodeOutputOptions } from '../../hooks/use-build-options';import { useFormChangeCallback } from '../../hooks/use-form-change-callback';-import { useFormValues } from '../../hooks/use-form-values';import { useWatchFormChange } from '../../hooks/use-watch-form-change';import { INextOperatorForm } from '../../interface';import { buildOutputList } from '../../utils/build-output-list';+import { transformExtractorConfigToForm } from '@/utils/pipeline-operator';import { FormWrapper } from '../components/form-wrapper';import { Output } from '../components/output';-import { useSwitchPrompt } from './use-switch-prompt';import { canSelectTagFile, useTagFileTree } from './use-tag-file-tree';-import { FormLayout } from '@/constants/form';export const FormSchema = z.object({- field_name: z.string(),- sys_prompt: z.string(),+ keywords: z+ .object({+ top_n: z.number().optional(),+ system_prompt: z.string().optional(),+ })+ .optional(),+ questions: z+ .object({+ top_n: z.number().optional(),+ system_prompt: z.string().optional(),+ })+ .optional(),+ tags: z+ .object({+ top_n: z.number().optional(),+ tag_file_id: z.string().optional(),+ })+ .optional(),+ summary: z+ .object({+ enabled: z.union([z.number(), z.boolean()]).optional(),+ system_prompt: z.string().optional(),+ })+ .optional(),+ metadata_config: z+ .object({+ enabled: z.union([z.number(), z.boolean()]).optional(),+ metadata: z.any().optional(),+ built_in_metadata: z.any().optional(),+ })+ .optional(),++ // Legacy flat fields for backward compatibility+ field_name: z.string().optional(),+ sys_prompt: z.string().optional(),prompts: z.string().optional(),+ keywords_sys_prompt: z.string().optional(),+ questions_sys_prompt: z.string().optional(),auto_keywords: z.number().optional(),auto_questions: z.number().optional(),auto_tags: z.number().optional(),tag_file_id: z.string().optional(),- // Builtin auto-metadata (mirrors Python's Auto metadata): enable_metadata- // toggle + metadata / built_in_metadata field schema, consumed by the Go- // extractor's runEnableMetadata at parse time.+ enable_summary: z.union([z.number(), z.boolean()]).optional(),enable_metadata: z.number().optional(),metadata: z.any().optional(),built_in_metadata: z.any().optional(),@@ -67,11 +109,13 @@ export const FormSchema = z.object({export type ExtractorFormSchemaType = z.infer<typeof FormSchema>;-// Builtin auto-extract node id hardcoded in every builtin ingestion DSL-// template (internal/ingestion/pipeline/template/*.json). Only this node-// shows the Auto metadata option; canvas / user-pipeline extractor nodes do-// not.-const BuiltinAutoExtractNodeId = 'Extractor:AutoExtractDefault';+enum ExtractorSubTab {+ Keywords = 'keywords',+ Questions = 'questions',+ Tags = 'tags',+ Summary = 'summary',+ Metadata = 'metadata',+}// ExtractorAutoMetadata mirrors Python's dataset "Auto metadata" control: an// enable_metadata switch plus a field-schema editor (custom + built-in).@@ -91,11 +135,16 @@ function ExtractorAutoMetadata() {const handleOpen = useCallback(() => {showManageMetadataModal({metadata: util.metaDataSettingJSONToMetaDataTableData(- form.getValues('metadata') || [],+ form.getValues('metadata_config.metadata') ||+ form.getValues('metadata') ||+ [],),isCanAdd: true,type: MetadataType.Setting,- builtInMetadata: form.getValues('built_in_metadata') || [],+ builtInMetadata:+ form.getValues('metadata_config.built_in_metadata') ||+ form.getValues('built_in_metadata') ||+ [],});}, [form, showManageMetadataModal]);@@ -104,9 +153,19 @@ function ExtractorAutoMetadata() {metadata?: IMetaDataReturnJSONSettings;builtInMetadata?: IBuiltInMetadataItem[];}) => {- form.setValue('metadata', data?.metadata || []);- form.setValue('built_in_metadata', data?.builtInMetadata || []);- form.setValue('enable_metadata', 1);+ const metaList = data?.metadata || [];+ const builtInList = data?.builtInMetadata || [];+ form.setValue('metadata_config.metadata', metaList, {+ shouldDirty: true,+ });+ form.setValue('metadata_config.built_in_metadata', builtInList, {+ shouldDirty: true,+ });+ form.setValue('metadata_config.enabled', true, { shouldDirty: true });+ // Also keep flat fields for backward compatibility+ form.setValue('metadata', metaList, { shouldDirty: true });+ form.setValue('built_in_metadata', builtInList, { shouldDirty: true });+ form.setValue('enable_metadata', 1, { shouldDirty: true });},[form],);@@ -115,7 +174,7 @@ function ExtractorAutoMetadata() {<><RAGFlowFormItemlabel={t('knowledgeConfiguration.autoMetadata')}- name="enable_metadata"+ name="metadata_config.enabled">{(field) => (<div className="flex items-center justify-between">@@ -132,7 +191,12 @@ function ExtractorAutoMetadata() {</Button><Switchchecked={field.value === 1 || field.value === true}- onCheckedChange={(checked) => field.onChange(checked ? 1 : 0)}+ onCheckedChange={(checked) => {+ field.onChange(checked);+ form.setValue('enable_metadata', checked ? 1 : 0, {+ shouldDirty: true,+ });+ }}data-testid="extractor-metadata-switch"/></div>@@ -162,38 +226,67 @@ function ExtractorAutoMetadata() {const outputList = buildOutputList(initialExtractorValues.outputs);+const useNormalizedExtractorFormValues = (node?: RAGFlowNodeType) => {+ return useMemo(() => {+ const raw = (node?.data?.form as Record<string, any>) || {};+ return {+ ...initialExtractorValues,+ ...transformExtractorConfigToForm(raw),+ };+ }, [node?.data?.form]);+};+const ExtractorForm = ({node,onValuesChange,hideOutputs,}: INextOperatorForm) => {- const defaultValues = useFormValues(initialExtractorValues, node);+ const defaultValues = useNormalizedExtractorFormValues(node);const { t } = useTranslation();const form = useForm<ExtractorFormSchemaType>({defaultValues,resolver: zodResolver(FormSchema),- // mode: 'onChange',});- const promptOptions = useBuildNodeOutputOptions(node?.id);-- const options = buildOptions(ContextGeneratorFieldName, t, 'flow');+ useEffect(() => {+ form.reset(defaultValues);+ }, [defaultValues, form]);- const {- handleFieldNameChange,- confirmSwitch,- hideModal,- visible,- cancelSwitch,- } = useSwitchPrompt(form);+ const [activeTab, setActiveTab] = useState<ExtractorSubTab>(+ ExtractorSubTab.Keywords,+ );useWatchFormChange(node?.id, form);useFormChangeCallback(form, onValuesChange);const ownerTenantId = useOwnerTenantId();- const { treeData, loadData } = useTagFileTree(form.watch('tag_file_id'));+ const tagFileIdWatch =+ form.watch('tags.tag_file_id') || form.watch('tag_file_id');+ const { treeData, loadData } = useTagFileTree(tagFileIdWatch);++ useEffect(() => {+ if (!form.getValues('keywords.system_prompt')) {+ form.setValue(+ 'keywords.system_prompt',+ t('flow.prompts.system.keywords'),+ );+ }+ if (!form.getValues('questions.system_prompt')) {+ form.setValue(+ 'questions.system_prompt',+ t('flow.prompts.system.questions'),+ );+ }+ if (!form.getValues('summary.system_prompt')) {+ form.setValue('summary.system_prompt', t('flow.prompts.system.summary'));+ }+ }, [form, t]);++ const handleTabChange = useCallback((tab: string) => {+ setActiveTab(tab as ExtractorSubTab);+ }, []);return (<Form {...form}>@@ -201,20 +294,74 @@ const ExtractorForm = ({<LargeModelFormFieldownerTenantId={ownerTenantId}></LargeModelFormField>- <AutoKeywordsFormField name="auto_keywords"></AutoKeywordsFormField>- <AutoQuestionsFormField name="auto_questions"></AutoQuestionsFormField>- {isGoBackend() && (- <>++ <Tabs+ value={activeTab}+ onValueChange={handleTabChange}+ className="w-full"+ >+ <TabsList className="w-full justify-start">+ <TabsTrigger value={ExtractorSubTab.Keywords}>+ {t('flow.keywords')}+ </TabsTrigger>+ <TabsTrigger value={ExtractorSubTab.Questions}>+ {t('flow.questions')}+ </TabsTrigger>+ <TabsTrigger value={ExtractorSubTab.Tags}>+ {t('flow.tags') || t('knowledgeDetails.autoTags')}+ </TabsTrigger>+ <TabsTrigger value={ExtractorSubTab.Summary}>+ {t('flow.summary')}+ </TabsTrigger>+ <TabsTrigger value={ExtractorSubTab.Metadata}>+ {t('flow.metadata')}+ </TabsTrigger>+ </TabsList>++ <TabsContent+ value={ExtractorSubTab.Keywords}+ className="space-y-4 pt-2"+ >+ <AutoKeywordsFormField name="keywords.top_n" />+ <RAGFlowFormItem+ label={t('flow.systemPrompt')}+ name="keywords.system_prompt"+ >+ <PromptEditor+ placeholder={t('flow.messagePlaceholder')}+ showToolbar={false}+ showMergePath={false}+ />+ </RAGFlowFormItem>+ </TabsContent>++ <TabsContent+ value={ExtractorSubTab.Questions}+ className="space-y-4 pt-2"+ >+ <AutoQuestionsFormField name="questions.top_n" />+ <RAGFlowFormItem+ label={t('flow.systemPrompt')}+ name="questions.system_prompt"+ >+ <PromptEditor+ placeholder={t('flow.messagePlaceholder')}+ showToolbar={false}+ showMergePath={false}+ />+ </RAGFlowFormItem>+ </TabsContent>++ <TabsContent value={ExtractorSubTab.Tags} className="space-y-4 pt-2"><SliderInputFormField- name="auto_tags"+ name="tags.top_n"label={t('knowledgeDetails.autoTags')}- min={1}+ min={0}max={10}- defaultValue={1}+ defaultValue={0}layout={FormLayout.Vertical}- ></SliderInputFormField>-- <RAGFlowFormItem label={t('flow.tagFile')} name="tag_file_id">+ />+ <RAGFlowFormItem label={t('flow.tagFile')} name="tags.tag_file_id">{(field) => (<AsyncTreeSelecttreeData={treeData}@@ -222,55 +369,59 @@ const ExtractorForm = ({onChange={field.onChange}loadData={loadData}canSelect={canSelectTagFile}- ></AsyncTreeSelect>+ />)}</RAGFlowFormItem>- </>- )}-- <RAGFlowFormItem label={t('flow.fieldName')} name="field_name">- {(field) => (- <SelectWithSearch- onChange={(value) => {- field.onChange(value);- handleFieldNameChange(value);- }}- value={field.value}- placeholder={t('dataFlowPlaceholder')}- options={options}- ></SelectWithSearch>- )}- </RAGFlowFormItem>-- {(node?.data as Record<string, any>)?.operatorId ===- BuiltinAutoExtractNodeId && <ExtractorAutoMetadata />}+ </TabsContent>- <RAGFlowFormItem label={t('flow.systemPrompt')} name="sys_prompt">- <PromptEditor- placeholder={t('flow.messagePlaceholder')}- showToolbar={true}- baseOptions={promptOptions}- ></PromptEditor>- </RAGFlowFormItem>+ <TabsContent+ value={ExtractorSubTab.Summary}+ className="space-y-4 pt-2"+ >+ <RAGFlowFormItem+ label={t('flow.enableSummary')}+ name="summary.enabled"+ horizontal+ valueClassName="w-auto flex justify-end"+ >+ {(field) => (+ <Switch+ checked={field.value === 1 || field.value === true}+ onCheckedChange={(checked) => {+ field.onChange(checked);+ form.setValue('field_name', checked ? 'summary' : '', {+ shouldDirty: true,+ });+ form.setValue('enable_summary', checked ? 1 : 0, {+ shouldDirty: true,+ });+ }}+ data-testid="extractor-summary-switch"+ />+ )}+ </RAGFlowFormItem>+ <RAGFlowFormItem+ label={t('flow.systemPrompt')}+ name="summary.system_prompt"+ >+ <PromptEditor+ placeholder={t('flow.messagePlaceholder')}+ showToolbar={false}+ showMergePath={false}+ />+ </RAGFlowFormItem>+ </TabsContent>- <RAGFlowFormItem label={t('flow.userPrompt')} name="prompts">- <PromptEditor- showToolbar={true}- baseOptions={promptOptions}- ></PromptEditor>- </RAGFlowFormItem>+ <TabsContent+ value={ExtractorSubTab.Metadata}+ className="space-y-4 pt-2"+ >+ <ExtractorAutoMetadata />+ </TabsContent>+ </Tabs>{!hideOutputs && <Output list={outputList}></Output>}</FormWrapper>- {visible && (- <ConfirmDeleteDialog- title={t('flow.switchPromptMessage')}- open- onOpenChange={hideModal}- onOk={confirmSwitch}- onCancel={cancelSwitch}- ></ConfirmDeleteDialog>- )}</Form>);};diff --git a/web/src/pages/agent/form/extractor-form/use-switch-prompt.ts b/web/src/pages/agent/form/extractor-form/use-switch-prompt.tsdeleted file mode 100644index 1efe7b409c..0000000000--- a/web/src/pages/agent/form/extractor-form/use-switch-prompt.ts+++ /dev/null@@ -1,68 +0,0 @@-import { useSetModalState } from '@/hooks/common-hooks';-import { useCallback, useRef } from 'react';-import { useTranslation } from 'react-i18next';--type SwitchPromptField = 'field_name' | 'sys_prompt' | 'prompts';--type SwitchPromptForm = {- getValues(name: 'field_name'): string;- setValue(- name: SwitchPromptField,- value: string,- options?: { shouldDirty?: boolean; shouldValidate?: boolean },- ): void;-};--export function useSwitchPrompt(form: SwitchPromptForm) {- const { visible, showModal, hideModal } = useSetModalState();- const { t } = useTranslation();- const previousFieldNames = useRef<string[]>([form.getValues('field_name')]);-- const setPromptValue = useCallback(- (field: SwitchPromptField, key: string, value: string) => {- form.setValue(field, t(`flow.prompts.${key}.${value}`), {- shouldDirty: true,- shouldValidate: true,- });- },- [form, t],- );-- const handleFieldNameChange = useCallback(- (value: string) => {- if (value) {- const names = previousFieldNames.current;- if (names.length > 1) {- names.shift();- }- names.push(value);- showModal();- }- },- [showModal],- );-- const confirmSwitch = useCallback(() => {- const value = form.getValues('field_name');- setPromptValue('sys_prompt', 'system', value);- setPromptValue('prompts', 'user', value);- }, [form, setPromptValue]);-- const cancelSwitch = useCallback(() => {- const previousValue = previousFieldNames.current.at(-2);- if (previousValue) {- form.setValue('field_name', previousValue, {- shouldDirty: true,- shouldValidate: true,- });- }- }, [form]);-- return {- handleFieldNameChange,- confirmSwitch,- hideModal,- visible,- cancelSwitch,- };-}diff --git a/web/src/pages/agent/hooks/use-form-change-callback.ts b/web/src/pages/agent/hooks/use-form-change-callback.tsindex 8b4b95d6b0..7ac000532c 100644--- a/web/src/pages/agent/hooks/use-form-change-callback.ts+++ b/web/src/pages/agent/hooks/use-form-change-callback.ts@@ -9,7 +9,7 @@ export function useFormChangeCallback(useEffect(() => {if (onValuesChange) {- onValuesChange(values);+ onValuesChange(form.getValues());}- }, [onValuesChange, values]);+ }, [form, onValuesChange, values]);}diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.tsindex 35fd85444e..b432f1a565 100644--- a/web/src/pages/agent/utils.ts+++ b/web/src/pages/agent/utils.ts@@ -400,7 +400,78 @@ export function transformTitleChunkerParams(}export function transformExtractorParams(params: ExtractorFormSchemaType) {- return { ...params, prompts: [{ content: params.prompts, role: 'user' }] };+ const isMetadataEnabled =+ params.metadata_config?.enabled !== undefined+ ? Boolean(params.metadata_config?.enabled)+ : params.enable_metadata === 1 || params.enable_metadata === true;++ const isSummaryEnabled =+ params.summary?.enabled !== undefined+ ? Boolean(params.summary?.enabled)+ : params.enable_summary === 1 ||+ params.enable_summary === true ||+ params.field_name === 'summary';++ const metadataList =+ params.metadata_config?.metadata ?? params.metadata ?? [];+ const builtInMetadataList =+ params.metadata_config?.built_in_metadata ??+ params.built_in_metadata ??+ [];++ const summarySysPrompt =+ params.summary?.system_prompt ?? params.sys_prompt ?? '';++ const keywordsTopN = params.keywords?.top_n ?? params.auto_keywords ?? 0;+ const keywordsSysPrompt =+ params.keywords?.system_prompt ?? params.keywords_sys_prompt ?? '';++ const questionsTopN = params.questions?.top_n ?? params.auto_questions ?? 0;+ const questionsSysPrompt =+ params.questions?.system_prompt ?? params.questions_sys_prompt ?? '';++ const tagsTopN = params.tags?.top_n ?? params.auto_tags ?? 0;+ const tagFileId = params.tags?.tag_file_id ?? params.tag_file_id ?? '';++ return {+ ...params,+ prompts: [{ content: params.prompts, role: 'user' }],+ auto_keywords: keywordsTopN,+ keywords_sys_prompt: keywordsSysPrompt,+ keywords: {+ top_n: keywordsTopN,+ system_prompt: keywordsSysPrompt,+ },+ auto_questions: questionsTopN,+ questions_sys_prompt: questionsSysPrompt,+ questions: {+ top_n: questionsTopN,+ system_prompt: questionsSysPrompt,+ },+ auto_tags: tagsTopN,+ tag_file_id: tagFileId,+ tags: {+ top_n: tagsTopN,+ tag_file_id: tagFileId,+ },+ enable_summary: isSummaryEnabled ? 1 : 0,+ sys_prompt: summarySysPrompt,+ field_name: isSummaryEnabled+ ? (params.field_name || 'summary')+ : (params.field_name === 'summary' ? '' : (params.field_name || '')),+ summary: {+ enabled: isSummaryEnabled,+ system_prompt: summarySysPrompt,+ },+ enable_metadata: isMetadataEnabled ? 1 : 0,+ metadata: metadataList,+ built_in_metadata: builtInMetadataList,+ metadata_config: {+ enabled: isMetadataEnabled,+ metadata: metadataList,+ built_in_metadata: builtInMetadataList,+ },+ };}function transformDataOperationsParams(params: DataOperationsFormSchemaType) {diff --git a/web/src/pages/agent/utils/tests/extractor-transform.test.ts b/web/src/pages/agent/utils/tests/extractor-transform.test.tsnew file mode 100644index 0000000000..800a3b35d4--- /dev/null+++ b/web/src/pages/agent/utils/tests/extractor-transform.test.ts@@ -0,0 +1,115 @@+import { transformExtractorConfigToForm } from '@/utils/pipeline-operator';+import { transformExtractorParams } from '../../utils';++describe('Extractor parameter transformations & precedence', () => {+ describe('transformExtractorParams', () => {+ it('synchronizes nested modular configs to flat fields and preserves nested objects', () => {+ const input: any = {+ summary: {+ enabled: true,+ system_prompt: 'Custom summary prompt',+ },+ metadata_config: {+ enabled: true,+ metadata: [{ key: 'category', type: 'string' }],+ built_in_metadata: [{ key: 'update_time', type: 'time' }],+ },+ keywords: {+ top_n: 5,+ system_prompt: 'KW prompt',+ },+ questions: {+ top_n: 3,+ system_prompt: 'Q prompt',+ },+ tags: {+ top_n: 2,+ tag_file_id: 'tag-123',+ },+ llm_id: 'gpt-4',+ };++ const result = transformExtractorParams(input);++ expect(result.enable_summary).toBe(1);+ expect(result.summary).toEqual({+ enabled: true,+ system_prompt: 'Custom summary prompt',+ });+ expect(result.enable_metadata).toBe(1);+ expect(result.metadata_config.enabled).toBe(true);+ expect(result.metadata_config.metadata).toHaveLength(1);+ expect(result.auto_keywords).toBe(5);+ expect(result.auto_questions).toBe(3);+ expect(result.auto_tags).toBe(2);+ expect(result.tag_file_id).toBe('tag-123');+ });++ it('gives nested modular enabled: false precedence over legacy flat enable_*: 1', () => {+ const input: any = {+ summary: {+ enabled: false,+ system_prompt: '',+ },+ enable_summary: 1,+ metadata_config: {+ enabled: false,+ metadata: [],+ built_in_metadata: [],+ },+ enable_metadata: 1,+ };++ const result = transformExtractorParams(input);++ expect(result.summary.enabled).toBe(false);+ expect(result.enable_summary).toBe(0);+ expect(result.metadata_config.enabled).toBe(false);+ expect(result.enable_metadata).toBe(0);+ });++ it('preserves custom field_name when summary is disabled', () => {+ const input: any = {+ summary: {+ enabled: false,+ system_prompt: '',+ },+ field_name: 'custom_chunk_field',+ };++ const result = transformExtractorParams(input);+ expect(result.field_name).toBe('custom_chunk_field');+ });+ });++ describe('transformExtractorConfigToForm', () => {+ it('normalizes legacy flat API format into nested form schema', () => {+ const config = {+ enable_summary: 1,+ sys_prompt: 'Old summary prompt',+ enable_metadata: 1,+ metadata: [{ key: 'author', type: 'string' }],+ built_in_metadata: [{ key: 'file_name', type: 'string' }],+ auto_keywords: 4,+ auto_questions: 2,+ auto_tags: 1,+ tag_file_id: 'tag-file-1',+ };++ const result = transformExtractorConfigToForm(config);++ expect(result.summary).toEqual({+ enabled: true,+ system_prompt: 'Old summary prompt',+ });+ expect(result.metadata_config).toEqual({+ enabled: true,+ metadata: [{ key: 'author', type: 'string' }],+ built_in_metadata: [{ key: 'file_name', type: 'string' }],+ });+ expect(result.keywords.top_n).toBe(4);+ expect(result.questions.top_n).toBe(2);+ expect(result.tags.top_n).toBe(1);+ });+ });+});diff --git a/web/src/utils/pipeline-operator.ts b/web/src/utils/pipeline-operator.tsindex e436ca61d0..efaac7b69d 100644--- a/web/src/utils/pipeline-operator.ts+++ b/web/src/utils/pipeline-operator.ts@@ -86,7 +86,7 @@ function transformLevelsToRules(* DSL: { prompts: [{ content: "text", role: "user" }] }* Form: { prompts: "text" }*/-function transformExtractorConfigToForm(+export function transformExtractorConfigToForm(config: Record<string, any> | undefined,): Record<string, any> {if (!config) return {};@@ -95,6 +95,61 @@ function transformExtractorConfigToForm(if (Array.isArray(config.prompts) && config.prompts.length > 0) {result.prompts = config.prompts[0]?.content ?? '';}++ const isSummaryEnabled =+ config.summary?.enabled !== undefined+ ? Boolean(config.summary?.enabled)+ : config.enable_summary === 1 || config.enable_summary === true;++ const isMetadataEnabled =+ config.metadata_config?.enabled !== undefined+ ? Boolean(config.metadata_config?.enabled)+ : config.enable_metadata === 1 || config.enable_metadata === true;++ result.keywords = {+ top_n:+ config.keywords?.top_n ??+ config.auto_keywords ??+ initialExtractorValues.keywords.top_n,+ system_prompt:+ config.keywords?.system_prompt ?? config.keywords_sys_prompt ?? '',+ };+ result.questions = {+ top_n:+ config.questions?.top_n ??+ config.auto_questions ??+ initialExtractorValues.questions.top_n,+ system_prompt:+ config.questions?.system_prompt ?? config.questions_sys_prompt ?? '',+ };+ result.tags = {+ top_n:+ config.tags?.top_n ??+ config.auto_tags ??+ initialExtractorValues.tags.top_n,+ tag_file_id: config.tags?.tag_file_id ?? config.tag_file_id ?? '',+ };+ result.summary = {+ enabled: isSummaryEnabled,+ system_prompt: config.summary?.system_prompt ?? config.sys_prompt ?? '',+ };+ result.enable_summary = isSummaryEnabled ? 1 : 0;+ result.field_name = isSummaryEnabled+ ? (config.field_name || 'summary')+ : (config.field_name === 'summary' ? '' : (config.field_name || ''));++ result.metadata_config = {+ enabled: isMetadataEnabled,+ metadata: config.metadata_config?.metadata ?? config.metadata ?? [],+ built_in_metadata:+ config.metadata_config?.built_in_metadata ??+ config.built_in_metadata ??+ [],+ };+ result.enable_metadata = isMetadataEnabled ? 1 : 0;+ result.metadata = result.metadata_config.metadata;+ result.built_in_metadata = result.metadata_config.built_in_metadata;+return result;}