Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(ingestion,web): modularize extractor configuration with sub-tabs, independent prompts, and metadata integration (#18383)

jay77721c71991bb7aeb
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go
index db76fd8936..417c61d9ff 100644
--- a/internal/ingestion/component/extractor.go
+++ b/internal/ingestion/component/extractor.go
@@ -207,6 +207,18 @@ Propose questions about a given piece of text content.
## Content to analyze:
%s`
+
+ autoSummaryPrompt = `## Role
+You are a precise and faithful text summarizer.
+
+## Task
+Create a concise and faithful summary of the provided text content.
+
+## Requirements
+- The summary MUST strictly rely on the provided text without hallucinating.
+- The summary MUST be in the same language as the original text content.
+- Be concise and focus on the main ideas, omitting redundant details.
+- Output summary ONLY.`
)
// ExtractorComponent performs LLM-based extraction over a chunk
@@ -265,49 +277,136 @@ func NewExtractorComponent(params map[string]any) (runtime.Component, error) {
}
}
}
- if v, ok := params["auto_keywords"]; ok {
- p.AutoKeywords = mapInt(v)
+ // 1. Keywords (modular or flat)
+ if kwRaw, ok := params["keywords"].(map[string]any); ok {
+ if v, ok := kwRaw["top_n"]; ok {
+ p.Keywords.TopN = mapInt(v)
+ }
+ if v, ok := kwRaw["system_prompt"].(string); ok {
+ p.Keywords.SystemPrompt = v
+ } else if v, ok := kwRaw["sys_prompt"].(string); ok {
+ p.Keywords.SystemPrompt = v
+ }
+ } else if v, ok := params["auto_keywords"]; ok {
+ p.Keywords.TopN = mapInt(v)
+ if v, ok := params["keywords_sys_prompt"].(string); ok {
+ p.Keywords.SystemPrompt = v
+ }
}
- if v, ok := params["auto_questions"]; ok {
- p.AutoQuestions = mapInt(v)
+ p.AutoKeywords = p.Keywords.TopN
+
+ // 2. Questions (modular or flat)
+ if qRaw, ok := params["questions"].(map[string]any); ok {
+ if v, ok := qRaw["top_n"]; ok {
+ p.Questions.TopN = mapInt(v)
+ }
+ if v, ok := qRaw["system_prompt"].(string); ok {
+ p.Questions.SystemPrompt = v
+ } else if v, ok := qRaw["sys_prompt"].(string); ok {
+ p.Questions.SystemPrompt = v
+ }
+ } else if v, ok := params["auto_questions"]; ok {
+ p.Questions.TopN = mapInt(v)
+ if v, ok := params["questions_sys_prompt"].(string); ok {
+ p.Questions.SystemPrompt = v
+ }
}
- if v, ok := params["auto_tags"]; ok {
- p.AutoTags = mapInt(v)
+ p.AutoQuestions = p.Questions.TopN
+
+ // 3. Tags (modular or flat)
+ if tagRaw, ok := params["tags"].(map[string]any); ok {
+ if v, ok := tagRaw["top_n"]; ok {
+ p.Tags.TopN = mapInt(v)
+ }
+ if v, ok := tagRaw["tag_file_id"].(string); ok {
+ p.Tags.TagFileID = v
+ }
+ } else {
+ if v, ok := params["auto_tags"]; ok {
+ p.Tags.TopN = mapInt(v)
+ }
+ if v, ok := params["tag_file_id"].(string); ok {
+ p.Tags.TagFileID = v
+ }
}
- if v, ok := params["enable_metadata"]; ok {
- p.EnableMetadata = mapInt(v)
+ p.AutoTags = p.Tags.TopN
+ p.TagFileID = p.Tags.TagFileID
+
+ // 4. Summary (modular or flat)
+ if sumRaw, ok := params["summary"].(map[string]any); ok {
+ if v, ok := sumRaw["enabled"].(bool); ok {
+ p.Summary.Enabled = v
+ } else if v, ok := sumRaw["enabled"]; ok {
+ p.Summary.Enabled = mapInt(v) == 1
+ }
+ if v, ok := sumRaw["system_prompt"].(string); ok {
+ p.Summary.SystemPrompt = v
+ } else if v, ok := sumRaw["sys_prompt"].(string); ok {
+ p.Summary.SystemPrompt = v
+ }
+ } else {
+ if v, ok := params["enable_summary"].(bool); ok {
+ p.Summary.Enabled = v
+ } else if v, ok := params["enable_summary"]; ok {
+ p.Summary.Enabled = mapInt(v) == 1
+ } else if p.FieldName == "summary" {
+ p.Summary.Enabled = true
+ }
+ p.Summary.SystemPrompt = p.SystemPrompt
}
- if v, ok := params["metadata"].([]any); ok {
- fields := make([]common.MetadataFieldDef, 0, len(v))
- for _, f := range v {
- m, ok := f.(map[string]any)
- if !ok {
- continue
- }
- key, _ := m["key"].(string)
- if key = strings.TrimSpace(key); key == "" {
- continue
- }
- def := common.MetadataFieldDef{Key: key}
- if t, ok := m["type"].(string); ok {
- def.Type = t
- }
- if d, ok := m["description"].(string); ok {
- def.Description = d
- }
- if e, ok := m["enum"].([]any); ok {
- for _, ev := range e {
- if s, ok := ev.(string); ok {
- def.Enum = append(def.Enum, s)
- }
- }
- }
- fields = append(fields, def)
+ if p.Summary.Enabled {
+ p.EnableSummary = 1
+ if p.FieldName == "" {
+ p.FieldName = "summary"
+ }
+ if p.Summary.SystemPrompt != "" && p.SystemPrompt == "" {
+ p.SystemPrompt = p.Summary.SystemPrompt
+ }
+ } else {
+ p.EnableSummary = 0
+ // When summary is explicitly disabled, avoid legacy "summary" field_name default triggering open-ended calls
+ if p.FieldName == "summary" {
+ p.FieldName = ""
}
- p.Metadata = fields
}
- if v, ok := params["tag_file_id"].(string); ok {
- p.TagFileID = v
+
+ // Metadata sub-config parsing with legacy flat fallback
+ if metaRaw, ok := params["metadata_config"].(map[string]any); ok {
+ if v, ok := metaRaw["enabled"].(bool); ok {
+ p.MetadataConfig.Enabled = v
+ } else if v, ok := metaRaw["enabled"]; ok {
+ p.MetadataConfig.Enabled = mapInt(v) == 1
+ }
+ if v, ok := metaRaw["metadata"]; ok {
+ p.MetadataConfig.Metadata = parseMetadataFieldDefs(v)
+ }
+ if v, ok := metaRaw["built_in_metadata"]; ok {
+ p.MetadataConfig.BuiltInMetadata = parseMetadataFieldDefs(v)
+ }
+ if p.MetadataConfig.Enabled {
+ p.EnableMetadata = 1
+ } else {
+ p.EnableMetadata = 0
+ }
+ p.Metadata = p.MetadataConfig.Metadata
+ } else {
+ if v, ok := params["enable_metadata"].(bool); ok {
+ if v {
+ p.EnableMetadata = 1
+ } else {
+ p.EnableMetadata = 0
+ }
+ } else if v, ok := params["enable_metadata"]; ok {
+ p.EnableMetadata = mapInt(v)
+ }
+ if v, ok := params["metadata"]; ok {
+ p.Metadata = parseMetadataFieldDefs(v)
+ }
+ if v, ok := params["built_in_metadata"]; ok {
+ p.MetadataConfig.BuiltInMetadata = parseMetadataFieldDefs(v)
+ }
+ p.MetadataConfig.Enabled = p.EnableMetadata > 0
+ p.MetadataConfig.Metadata = p.Metadata
}
}
if err := p.Validate(); err != nil {
@@ -468,10 +567,10 @@ func (e *einoExtractorChatInvoker) Chat(ctx context.Context, req extractorChatRe
apiKey := req.APIKey
cfg := &models.APIConfig{ApiKey: &apiKey}
cm := models.NewChatModel(d, &modelName, cfg)
- var chatCfg *models.ChatConfig
+ chatCfg := &models.ChatConfig{}
if req.Temperature != nil {
temp := *req.Temperature
- chatCfg = &models.ChatConfig{Temperature: &temp}
+ chatCfg.Temperature = &temp
}
wrapper := models.NewEinoChatModel(cm, chatCfg)
// Honour ctx cancel up front so the caller's WithTimeout(...)
@@ -485,6 +584,7 @@ func (e *einoExtractorChatInvoker) Chat(ctx context.Context, req extractorChatRe
common.Error(fmt.Sprintf("error when chat with message: %v", req.Messages), err)
return nil, err
}
+ common.Debug(fmt.Sprintf("extractor: chat completed for model %s, response_length=%d", modelName, len(out.Content)))
return &extractorChatResponse{Content: out.Content}, nil
}
@@ -672,19 +772,18 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
// Without this, a template containing {text} would be forwarded
// to the model unsubstituted.
callIn := in
+ if in.fieldName == "summary" && strings.TrimSpace(callIn.systemPrompt) == "" && strings.TrimSpace(c.Param.Summary.SystemPrompt) == "" {
+ callIn.systemPrompt = autoSummaryPrompt
+ }
callIn.systemPrompt, callIn.prompt = renderExtractorPrompts(
- in.systemPrompt, in.prompt, map[string]any{}, "",
+ callIn.systemPrompt, in.prompt, map[string]any{}, "",
)
ans, callErr := c.callText(timeoutCtx, db, callIn, "")
if callErr != nil {
return callErr
}
ck := map[string]any{}
- if in.fieldName == "metadata" {
- mergeExtractionIntoMetadata(ck, ans)
- } else {
- ck[in.fieldName] = ans
- }
+ ck[in.fieldName] = ans
in.chunks = []map[string]any{ck}
return nil
}
@@ -712,16 +811,15 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
// chunk field values, mirroring Python's
// string_format at extractor.py:103.
callIn := in
- callIn.systemPrompt, callIn.prompt = renderExtractorPrompts(in.systemPrompt, in.prompt, ck, text)
+ if in.fieldName == "summary" && strings.TrimSpace(callIn.systemPrompt) == "" && strings.TrimSpace(c.Param.Summary.SystemPrompt) == "" {
+ callIn.systemPrompt = autoSummaryPrompt
+ }
+ callIn.systemPrompt, callIn.prompt = renderExtractorPrompts(callIn.systemPrompt, in.prompt, ck, text)
ans, callErr := c.callText(timeoutCtx, db, callIn, "")
if callErr != nil {
return fmt.Errorf("chunk %d: %w", i, callErr)
}
- if in.fieldName == "metadata" {
- mergeExtractionIntoMetadata(ck, ans)
- } else {
- ck[in.fieldName] = ans
- }
+ ck[in.fieldName] = ans
}
return nil
}); err != nil {
@@ -737,29 +835,6 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
}, nil
}
-// mergeExtractionIntoMetadata merges the field_name="metadata" extraction
-// result into the chunk's metadata map (enable_metadata's output) instead of
-// overwriting it. ck["metadata"] stays a map[string]any — the unified contract
-// for document metadata produced by this component. A non-JSON / empty result
-// is ignored (no string fallback), so it can never clobber or corrupt the
-// metadata map. On overlapping keys the field_name result wins (it runs later).
-func mergeExtractionIntoMetadata(ck map[string]any, ans string) {
- parsed, ok := tryParseJSONObject(ans)
- if !ok {
- return
- }
- existing, _ := ck["metadata"].(map[string]any)
- if existing == nil {
- existing = make(map[string]any, len(parsed))
- }
- for k, v := range parsed {
- if v != nil {
- existing[k] = v
- }
- }
- ck["metadata"] = existing
-}
-
// runAutoKeywords extracts keywords for the current chunk and stores
// them on ck["important_kwd"]. mu may be nil (sequential path); when
// non-nil it serializes the shared chunk-map accesses so concurrent
@@ -771,11 +846,22 @@ func (c *ExtractorComponent) runAutoKeywords(ctx context.Context, db *gorm.DB, i
if exists {
return nil
}
+ topN := c.Param.Keywords.TopN
+ if topN <= 0 {
+ topN = c.Param.AutoKeywords
+ }
+ var sysPrompt, userPrompt string
+ if strings.TrimSpace(c.Param.Keywords.SystemPrompt) != "" {
+ sysPrompt, userPrompt = renderExtractorPrompts(c.Param.Keywords.SystemPrompt, "Output: ", ck, chunkText)
+ } else {
+ sysPrompt = fmt.Sprintf(autoKeywordPrompt, topN, chunkText)
+ userPrompt = "Output: "
+ }
kwTemp := extractorTemperature
kwIn := extractorInputs{
llmID: in.llmID,
- systemPrompt: fmt.Sprintf(autoKeywordPrompt, c.Param.AutoKeywords, chunkText),
- prompt: "Output: ",
+ systemPrompt: sysPrompt,
+ prompt: userPrompt,
temperature: &kwTemp,
}
resultStr, err := c.callText(ctx, db, kwIn, "")
@@ -806,11 +892,22 @@ func (c *ExtractorComponent) runAutoQuestions(ctx context.Context, db *gorm.DB,
if exists {
return nil
}
+ topN := c.Param.Questions.TopN
+ if topN <= 0 {
+ topN = c.Param.AutoQuestions
+ }
+ var sysPrompt, userPrompt string
+ if strings.TrimSpace(c.Param.Questions.SystemPrompt) != "" {
+ sysPrompt, userPrompt = renderExtractorPrompts(c.Param.Questions.SystemPrompt, "Output: ", ck, chunkText)
+ } else {
+ sysPrompt = fmt.Sprintf(autoQuestionPrompt, topN, chunkText)
+ userPrompt = "Output: "
+ }
qTemp := extractorTemperature
qIn := extractorInputs{
llmID: in.llmID,
- systemPrompt: fmt.Sprintf(autoQuestionPrompt, c.Param.AutoQuestions, chunkText),
- prompt: "Output: ",
+ systemPrompt: sysPrompt,
+ prompt: userPrompt,
temperature: &qTemp,
}
resultStr, err := c.callText(ctx, db, qIn, "")
@@ -1701,10 +1798,42 @@ func tryParseJSONObject(s string) (map[string]any, bool) {
return out, true
}
-// init registers Extractor under CategoryIngestion (per plan §4
-// Phase 2.5). Metadata is derived from the Inputs()/Outputs()
-// methods on ExtractorComponent so the API layer (Phase 4) can
-// enumerate the catalog without instantiating the component.
+// parseMetadataFieldDefs converts an any value (typically []any of maps)
+// to a typed []common.MetadataFieldDef slice.
+func parseMetadataFieldDefs(v any) []common.MetadataFieldDef {
+ arr, ok := v.([]any)
+ if !ok {
+ return nil
+ }
+ fields := make([]common.MetadataFieldDef, 0, len(arr))
+ for _, f := range arr {
+ m, ok := f.(map[string]any)
+ if !ok {
+ continue
+ }
+ key, _ := m["key"].(string)
+ if key = strings.TrimSpace(key); key == "" {
+ continue
+ }
+ def := common.MetadataFieldDef{Key: key}
+ if t, ok := m["type"].(string); ok {
+ def.Type = t
+ }
+ if d, ok := m["description"].(string); ok {
+ def.Description = d
+ }
+ if e, ok := m["enum"].([]any); ok {
+ for _, ev := range e {
+ if s, ok := ev.(string); ok {
+ def.Enum = append(def.Enum, s)
+ }
+ }
+ }
+ fields = append(fields, def)
+ }
+ return fields
+}
+
// mapInt converts a JSON-compatible value to int.
func mapInt(v interface{}) int {
switch n := v.(type) {
diff --git a/internal/ingestion/component/extractor_metadata_regression_test.go b/internal/ingestion/component/extractor_metadata_regression_test.go
deleted file mode 100644
index 172f329a6f..0000000000
--- a/internal/ingestion/component/extractor_metadata_regression_test.go
+++ /dev/null
@@ -1,202 +0,0 @@
-package component
-
-import (
- "testing"
- "time"
-
- "ragflow/internal/common"
- "ragflow/internal/ingestion/component/schema"
- "ragflow/internal/ingestion/task/indexdoc"
-)
-
-// TestRepro_FieldNameMetadata_AggregatesMap guards the unified contract:
-// field_name="metadata" (the pipeline/canvas way to produce document metadata)
-// writes a parsed map into ck["metadata"] — not a raw string — so the
-// aggregation layer, which is strict on map, merges it into document metadata.
-func TestRepro_FieldNameMetadata_AggregatesMap(t *testing.T) {
- withStubChatInvoker(t,
- stubResponse{Content: `{"category":"finance","region":"east"}`},
- )
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "metadata",
- LLMID: "gpt-4o-mini",
- Prompt: "Extract metadata as JSON:",
- }}
- out, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{"text": "first text"}},
- })
- if err != nil {
- t.Fatalf("Invoke: %v", err)
- }
- chunks := out["chunks"].([]map[string]any)
- meta, ok := chunks[0]["metadata"].(map[string]any)
- if !ok {
- t.Fatalf("chunk[0].metadata = %T, want map[string]any (field_name=metadata must produce a map, not a string)", chunks[0]["metadata"])
- }
- if meta["category"] != "finance" || meta["region"] != "east" {
- t.Errorf("metadata = %v, want category=finance region=east", meta)
- }
-
- // Feed through the aggregation path, exactly as the executor does.
- metadata, err := indexdoc.ProcessChunksForPipeline(chunks, "doc-1", "doc.pdf", time.Now())
- if err != nil {
- t.Fatalf("ProcessChunksForPipeline: %v", err)
- }
- if metadata["category"] != "finance" {
- t.Errorf("category = %v, want finance", metadata["category"])
- }
- if metadata["region"] != "east" {
- t.Errorf("region = %v, want east", metadata["region"])
- }
-}
-
-// TestRepro_BothEnabled_MergesMetadata guards the unified contract: when both
-// enable_metadata (auto, schema-driven) and field_name="metadata" (pipeline
-// custom) are configured, the extractor MERGES them into one ck["metadata"]
-// map instead of the field_name result overwriting the auto-metadata. Both
-// contribute to the document metadata.
-func TestRepro_BothEnabled_MergesMetadata(t *testing.T) {
- withStubChatInvoker(t,
- // call 1: enable_metadata (runAutoExtractions)
- stubResponse{Content: `{"category":"finance","region":"east"}`},
- // call 2: field_name="metadata" primary extraction
- stubResponse{Content: `{"author":"zhang","date":"2024-01-01"}`},
- )
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "metadata",
- LLMID: "gpt-4o-mini",
- Prompt: "Extract more metadata as JSON:",
- EnableMetadata: 1,
- Metadata: []common.MetadataFieldDef{
- {Key: "category", Type: "string"},
- {Key: "region", Type: "string"},
- },
- }}
- out, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{"text": "content"}},
- })
- if err != nil {
- t.Fatalf("Invoke: %v", err)
- }
- chunks := out["chunks"].([]map[string]any)
- meta, ok := chunks[0]["metadata"].(map[string]any)
- if !ok {
- t.Fatalf("chunk[0].metadata = %T, want merged map; auto-metadata was overwritten", chunks[0]["metadata"])
- }
- for _, k := range []string{"category", "region", "author", "date"} {
- if _, has := meta[k]; !has {
- t.Errorf("merged metadata missing %q: %v", k, meta)
- }
- }
-
- metadata, err := indexdoc.ProcessChunksForPipeline(chunks, "doc-1", "doc.pdf", time.Now())
- if err != nil {
- t.Fatalf("ProcessChunksForPipeline: %v", err)
- }
- for _, k := range []string{"category", "region", "author", "date"} {
- if _, has := metadata[k]; !has {
- t.Errorf("aggregated doc metadata missing %q: %v", k, metadata)
- }
- }
-}
-
-// TestRepro_OverlapFieldNameWins pins the overlap rule: when enable_metadata
-// and field_name="metadata" extract the same key, the field_name (custom,
-// later) result wins.
-func TestRepro_OverlapFieldNameWins(t *testing.T) {
- withStubChatInvoker(t,
- // call 1: enable_metadata
- stubResponse{Content: `{"category":"finance","region":"east"}`},
- // call 2: field_name="metadata"
- stubResponse{Content: `{"category":"law","author":"X"}`},
- )
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "metadata",
- LLMID: "gpt-4o-mini",
- Prompt: "Extract metadata as JSON:",
- EnableMetadata: 1,
- Metadata: []common.MetadataFieldDef{
- {Key: "category", Type: "string"},
- {Key: "region", Type: "string"},
- },
- }}
- out, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{"text": "content"}},
- })
- if err != nil {
- t.Fatalf("Invoke: %v", err)
- }
- chunks := out["chunks"].([]map[string]any)
- meta := chunks[0]["metadata"].(map[string]any)
- if meta["category"] != "law" {
- t.Errorf("category = %v, want law (field_name wins on overlap)", meta["category"])
- }
- if meta["region"] != "east" {
- t.Errorf("region = %v, want east (auto field preserved)", meta["region"])
- }
-}
-
-// TestRepro_FieldNameNonJSON_KeepsEnableMetadata verifies a non-JSON
-// field_name="metadata" result does not clobber the enable_metadata map and
-// never writes a raw string.
-func TestRepro_FieldNameNonJSON_KeepsEnableMetadata(t *testing.T) {
- withStubChatInvoker(t,
- // call 1: enable_metadata
- stubResponse{Content: `{"category":"finance"}`},
- // call 2: field_name="metadata" -> non-JSON
- stubResponse{Content: "could not find metadata"},
- )
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "metadata",
- LLMID: "gpt-4o-mini",
- Prompt: "Extract metadata as JSON:",
- EnableMetadata: 1,
- Metadata: []common.MetadataFieldDef{
- {Key: "category", Type: "string"},
- },
- }}
- out, err := c.Invoke(t.Context(), nil, map[string]any{
- "chunks": []map[string]any{{"text": "content"}},
- })
- if err != nil {
- t.Fatalf("Invoke: %v", err)
- }
- chunks := out["chunks"].([]map[string]any)
- meta, ok := chunks[0]["metadata"].(map[string]any)
- if !ok {
- t.Fatalf("chunk[0].metadata = %T, want map (auto-metadata preserved, no string written)", chunks[0]["metadata"])
- }
- if meta["category"] != "finance" {
- t.Errorf("category = %v, want finance (enable_metadata map preserved)", meta["category"])
- }
-}
-
-// TestRepro_NoChunksMetadata_RoutesThroughMap guards the zero-chunk fast path:
-// field_name="metadata" with no input chunks must route through the same
-// parse-and-merge as the chunked path, writing a map (not a raw string) so the
-// strict aggregation layer does not drop the document metadata.
-func TestRepro_NoChunksMetadata_RoutesThroughMap(t *testing.T) {
- withStubChatInvoker(t,
- stubResponse{Content: `{"category":"finance","region":"east"}`},
- )
- c := &ExtractorComponent{Param: schema.ExtractorParam{
- FieldName: "metadata",
- LLMID: "gpt-4o-mini",
- Prompt: "Extract metadata as JSON:",
- }}
- out, err := c.Invoke(t.Context(), nil, map[string]any{})
- if err != nil {
- t.Fatalf("Invoke: %v", err)
- }
- chunks := out["chunks"].([]map[string]any)
- if len(chunks) != 1 {
- t.Fatalf("chunks len = %d, want 1", len(chunks))
- }
- meta, ok := chunks[0]["metadata"].(map[string]any)
- if !ok {
- t.Fatalf("chunk[0].metadata = %T, want map[string]any (zero-chunk path must not write a raw string)", chunks[0]["metadata"])
- }
- if meta["category"] != "finance" || meta["region"] != "east" {
- t.Errorf("metadata = %v, want category=finance region=east", meta)
- }
-}
diff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.go
index e4f8ca77e3..fd738a0a0e 100644
--- a/internal/ingestion/component/extractor_test.go
+++ b/internal/ingestion/component/extractor_test.go
@@ -19,6 +19,7 @@ package component
import (
"context"
"errors"
+ "fmt"
"strings"
"sync"
"sync/atomic"
@@ -2328,3 +2329,488 @@ func TestRenderExtractorPrompts_TableDriven(t *testing.T) {
})
}
}
+
+func TestExtractorModularParams(t *testing.T) {
+ params := map[string]any{
+ "llm_id": "test-llm-1",
+ "keywords": map[string]any{
+ "top_n": 5,
+ "system_prompt": "Custom keywords prompt for {text}",
+ },
+ "questions": map[string]any{
+ "top_n": 3,
+ "system_prompt": "Custom questions prompt for {text}",
+ },
+ "tags": map[string]any{
+ "top_n": 2,
+ "tag_file_id": "tag_file_abc",
+ },
+ "summary": map[string]any{
+ "enabled": true,
+ "system_prompt": "Custom summary prompt for {text}",
+ },
+ }
+
+ comp, err := NewExtractorComponent(params)
+ if err != nil {
+ t.Fatalf("NewExtractorComponent failed: %v", err)
+ }
+
+ ext, ok := comp.(*ExtractorComponent)
+ if !ok {
+ t.Fatalf("expected *ExtractorComponent, got %T", comp)
+ }
+
+ if ext.Param.Keywords.TopN != 5 || ext.Param.Keywords.SystemPrompt != "Custom keywords prompt for {text}" {
+ t.Errorf("Keywords config mismatch: %+v", ext.Param.Keywords)
+ }
+ if ext.Param.AutoKeywords != 5 {
+ t.Errorf("AutoKeywords flat sync mismatch: %d", ext.Param.AutoKeywords)
+ }
+
+ if ext.Param.Questions.TopN != 3 || ext.Param.Questions.SystemPrompt != "Custom questions prompt for {text}" {
+ t.Errorf("Questions config mismatch: %+v", ext.Param.Questions)
+ }
+ if ext.Param.AutoQuestions != 3 {
+ t.Errorf("AutoQuestions flat sync mismatch: %d", ext.Param.AutoQuestions)
+ }
+
+ if ext.Param.Tags.TopN != 2 || ext.Param.Tags.TagFileID != "tag_file_abc" {
+ t.Errorf("Tags config mismatch: %+v", ext.Param.Tags)
+ }
+ if ext.Param.AutoTags != 2 || ext.Param.TagFileID != "tag_file_abc" {
+ t.Errorf("AutoTags flat sync mismatch: %d, %s", ext.Param.AutoTags, ext.Param.TagFileID)
+ }
+
+ if !ext.Param.Summary.Enabled || ext.Param.Summary.SystemPrompt != "Custom summary prompt for {text}" {
+ t.Errorf("Summary config mismatch: %+v", ext.Param.Summary)
+ }
+ if ext.Param.FieldName != "summary" {
+ t.Errorf("FieldName mismatch: %s", ext.Param.FieldName)
+ }
+}
+
+func TestExtractorLegacyFlatParamsFallback(t *testing.T) {
+ params := map[string]any{
+ "llm_id": "test-llm-1",
+ "auto_keywords": 4,
+ "auto_questions": 2,
+ "auto_tags": 1,
+ "tag_file_id": "tag_file_legacy",
+ "field_name": "summary",
+ "sys_prompt": "Legacy summary prompt",
+ }
+
+ comp, err := NewExtractorComponent(params)
+ if err != nil {
+ t.Fatalf("NewExtractorComponent failed: %v", err)
+ }
+
+ ext, ok := comp.(*ExtractorComponent)
+ if !ok {
+ t.Fatalf("expected *ExtractorComponent, got %T", comp)
+ }
+
+ if ext.Param.Keywords.TopN != 4 || ext.Param.AutoKeywords != 4 {
+ t.Errorf("Keywords fallback mismatch: %d / %d", ext.Param.Keywords.TopN, ext.Param.AutoKeywords)
+ }
+ if ext.Param.Questions.TopN != 2 || ext.Param.AutoQuestions != 2 {
+ t.Errorf("Questions fallback mismatch: %d / %d", ext.Param.Questions.TopN, ext.Param.AutoQuestions)
+ }
+ if ext.Param.Tags.TopN != 1 || ext.Param.Tags.TagFileID != "tag_file_legacy" {
+ t.Errorf("Tags fallback mismatch: %+v", ext.Param.Tags)
+ }
+ if !ext.Param.Summary.Enabled || ext.Param.Summary.SystemPrompt != "Legacy summary prompt" {
+ t.Errorf("Summary fallback mismatch: %+v", ext.Param.Summary)
+ }
+}
+
+func TestExtractorModularPromptsExecution(t *testing.T) {
+ stub := withStubChatInvoker(t,
+ stubResponse{Content: "kw1, kw2"},
+ stubResponse{Content: "question 1?\nquestion 2?"},
+ stubResponse{Content: "Summary of chunk"},
+ )
+
+ params := map[string]any{
+ "llm_id": "llm-1",
+ "keywords": map[string]any{
+ "top_n": 2,
+ "system_prompt": "Custom KW Prompt: {text}",
+ },
+ "questions": map[string]any{
+ "top_n": 2,
+ "system_prompt": "Custom Q Prompt: {text}",
+ },
+ "summary": map[string]any{
+ "enabled": true,
+ "system_prompt": "Custom Sum Prompt: {text}",
+ },
+ }
+
+ comp, err := NewExtractorComponent(params)
+ if err != nil {
+ t.Fatalf("NewExtractorComponent: %v", err)
+ }
+
+ in := map[string]any{
+ "chunks": []map[string]any{
+ {"content_with_weight": "Hello world content"},
+ },
+ }
+
+ out, err := comp.Invoke(t.Context(), nil, in)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+
+ chunks, ok := out["chunks"].([]map[string]any)
+ if !ok || len(chunks) != 1 {
+ t.Fatalf("expected 1 chunk output, got %v", out)
+ }
+
+ ck := chunks[0]
+ if kwds, ok := ck["important_kwd"].([]string); !ok || len(kwds) != 2 {
+ t.Errorf("expected important_kwd = [kw1, kw2], got %v", ck["important_kwd"])
+ }
+ if qs, ok := ck["question_kwd"].([]string); !ok || len(qs) != 2 {
+ t.Errorf("expected question_kwd = [question 1?, question 2?], got %v", ck["question_kwd"])
+ }
+ if sum, ok := ck["summary"].(string); !ok || sum != "Summary of chunk" {
+ t.Errorf("expected summary = 'Summary of chunk', got %v", ck["summary"])
+ }
+
+ // Verify custom prompts were rendered into messages and passed to the chat invoker
+ reqs := stub.requests
+ if len(reqs) != 3 {
+ t.Fatalf("expected 3 LLM calls, got %d", len(reqs))
+ }
+
+ var foundKW, foundQ, foundSum bool
+ for _, r := range reqs {
+ for _, msg := range r.Messages {
+ if strings.Contains(msg.Content, "Custom KW Prompt: Hello world content") {
+ foundKW = true
+ }
+ if strings.Contains(msg.Content, "Custom Q Prompt: Hello world content") {
+ foundQ = true
+ }
+ if strings.Contains(msg.Content, "Custom Sum Prompt: Hello world content") {
+ foundSum = true
+ }
+ }
+ }
+
+ if !foundKW {
+ t.Errorf("expected custom keyword prompt to be used in LLM call, got reqs: %+v", reqs)
+ }
+ if !foundQ {
+ t.Errorf("expected custom question prompt to be used in LLM call, got reqs: %+v", reqs)
+ }
+ if !foundSum {
+ t.Errorf("expected custom summary prompt to be used in LLM call, got reqs: %+v", reqs)
+ }
+}
+
+func TestExtractorModularMetadataConfig(t *testing.T) {
+ params := map[string]any{
+ "llm_id": "llm-1",
+ "metadata_config": map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{
+ "key": "author",
+ "type": "string",
+ "description": "The author name",
+ "enum": []any{"Alice", "Bob"},
+ },
+ map[string]any{
+ "key": "year",
+ "type": "integer",
+ },
+ },
+ "built_in_metadata": []any{
+ map[string]any{
+ "key": "file_name",
+ "type": "string",
+ },
+ },
+ },
+ }
+
+ comp, err := NewExtractorComponent(params)
+ if err != nil {
+ t.Fatalf("NewExtractorComponent: %v", err)
+ }
+
+ ext, ok := comp.(*ExtractorComponent)
+ if !ok {
+ t.Fatalf("expected *ExtractorComponent, got %T", comp)
+ }
+
+ if !ext.Param.MetadataConfig.Enabled || ext.Param.EnableMetadata != 1 {
+ t.Errorf("Metadata enabled mismatch: %+v / %d", ext.Param.MetadataConfig.Enabled, ext.Param.EnableMetadata)
+ }
+ if len(ext.Param.MetadataConfig.Metadata) != 2 || len(ext.Param.Metadata) != 2 {
+ t.Fatalf("Metadata fields mismatch: %d / %d", len(ext.Param.MetadataConfig.Metadata), len(ext.Param.Metadata))
+ }
+ if ext.Param.Metadata[0].Key != "author" || ext.Param.Metadata[0].Description != "The author name" || len(ext.Param.Metadata[0].Enum) != 2 {
+ t.Errorf("Metadata field 0 mismatch: %+v", ext.Param.Metadata[0])
+ }
+ if len(ext.Param.MetadataConfig.BuiltInMetadata) != 1 || ext.Param.MetadataConfig.BuiltInMetadata[0].Key != "file_name" {
+ t.Errorf("BuiltInMetadata mismatch: %+v", ext.Param.MetadataConfig.BuiltInMetadata)
+ }
+}
+
+func TestExtractorModularMetadataExecution(t *testing.T) {
+ withStubChatInvoker(t,
+ stubResponse{Content: `{"author": "Alice", "year": 2026}`},
+ )
+
+ params := map[string]any{
+ "llm_id": "llm-1",
+ "metadata_config": map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{
+ "key": "author",
+ "type": "string",
+ },
+ map[string]any{
+ "key": "year",
+ "type": "integer",
+ },
+ },
+ },
+ }
+
+ comp, err := NewExtractorComponent(params)
+ if err != nil {
+ t.Fatalf("NewExtractorComponent: %v", err)
+ }
+
+ in := map[string]any{
+ "chunks": []map[string]any{
+ {"content_with_weight": "Written by Alice in 2026."},
+ },
+ }
+
+ out, err := comp.Invoke(t.Context(), nil, in)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+
+ chunks, ok := out["chunks"].([]map[string]any)
+ if !ok || len(chunks) != 1 {
+ t.Fatalf("expected 1 chunk output, got %v", out)
+ }
+
+ ck := chunks[0]
+ meta, ok := ck["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("expected chunk metadata map, got %T: %v", ck["metadata"], ck["metadata"])
+ }
+ if meta["author"] != "Alice" {
+ t.Errorf("expected metadata.author = Alice, got %v", meta["author"])
+ }
+ // year can be float64 or int from JSON
+ if fmt.Sprintf("%v", meta["year"]) != "2026" {
+ t.Errorf("expected metadata.year = 2026, got %v", meta["year"])
+ }
+}
+
+func TestExtractorDefaultSummaryPromptInjection(t *testing.T) {
+ stub := withStubChatInvoker(t, stubResponse{Content: "A concise summary."})
+
+ params := map[string]any{
+ "llm_id": "llm-1",
+ "summary": map[string]any{
+ "enabled": true,
+ // system_prompt left empty to test default prompt injection
+ },
+ }
+
+ comp, err := NewExtractorComponent(params)
+ if err != nil {
+ t.Fatalf("NewExtractorComponent: %v", err)
+ }
+
+ in := map[string]any{
+ "chunks": []map[string]any{
+ {"content_with_weight": "This is a detailed paragraph about artificial intelligence."},
+ },
+ }
+
+ out, err := comp.Invoke(t.Context(), nil, in)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+
+ chunks, ok := out["chunks"].([]map[string]any)
+ if !ok || len(chunks) != 1 {
+ t.Fatalf("expected 1 chunk output, got %v", out)
+ }
+
+ if chunks[0]["summary"] != "A concise summary." {
+ t.Errorf("expected summary = 'A concise summary.', got %v", chunks[0]["summary"])
+ }
+
+ if stub.Calls() != 1 {
+ t.Fatalf("expected 1 LLM call, got %d", stub.Calls())
+ }
+
+ lastReq := stub.lastRequest()
+ msgs := lastReq.Messages
+ if len(msgs) != 2 {
+ t.Fatalf("expected 2 messages (system + user), got %d: %+v", len(msgs), msgs)
+ }
+
+ if msgs[0].Role != "system" || !strings.Contains(msgs[0].Content, "You are a precise and faithful text summarizer") {
+ t.Errorf("expected autoSummaryPrompt in system message, got: %+v", msgs[0])
+ }
+
+ if msgs[1].Role != "user" || !strings.Contains(msgs[1].Content, "This is a detailed paragraph about artificial intelligence.") {
+ t.Errorf("expected chunk text in user message, got: %+v", msgs[1])
+ }
+}
+
+func TestExtractorDisabledSummarySkipsCall(t *testing.T) {
+ stub := withStubChatInvoker(t, stubResponse{Content: "Not expected"})
+
+ params := map[string]any{
+ "llm_id": "llm-1",
+ "summary": map[string]any{
+ "enabled": false,
+ },
+ "field_name": "summary", // legacy default that should be ignored when enabled=false
+ }
+
+ comp, err := NewExtractorComponent(params)
+ if err != nil {
+ t.Fatalf("NewExtractorComponent: %v", err)
+ }
+
+ in := map[string]any{
+ "chunks": []map[string]any{
+ {"content_with_weight": "Some text."},
+ },
+ }
+
+ out, err := comp.Invoke(t.Context(), nil, in)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+
+ if stub.Calls() != 0 {
+ t.Errorf("expected 0 LLM calls when summary is disabled, got %d", stub.Calls())
+ }
+
+ chunks, ok := out["chunks"].([]map[string]any)
+ if !ok || len(chunks) != 1 {
+ t.Fatalf("expected 1 chunk output, got %v", out)
+ }
+ if _, has := chunks[0]["summary"]; has {
+ t.Errorf("expected no summary key in chunk, got %v", chunks[0]["summary"])
+ }
+}
+
+func TestExtractor_NestedPrecedenceOverFlat(t *testing.T) {
+ // Case A: Nested explicitly disabled (enabled=false) while legacy flat has enabled=1/5.
+ // Nested MUST win and disable the features.
+ paramsDisabled := map[string]any{
+ "metadata_config": map[string]any{
+ "enabled": false,
+ "metadata": []any{
+ map[string]any{"key": "category", "type": "string"},
+ },
+ },
+ "enable_metadata": 1,
+ "summary": map[string]any{
+ "enabled": false,
+ },
+ "enable_summary": 1,
+ "keywords": map[string]any{
+ "top_n": 0,
+ },
+ "auto_keywords": 5,
+ "questions": map[string]any{
+ "top_n": 0,
+ },
+ "auto_questions": 3,
+ "tags": map[string]any{
+ "top_n": 0,
+ },
+ "auto_tags": 2,
+ }
+
+ compRawA, err := NewExtractorComponent(paramsDisabled)
+ if err != nil {
+ t.Fatalf("NewExtractorComponent A: %v", err)
+ }
+ compA := compRawA.(*ExtractorComponent)
+ if compA.Param.MetadataConfig.Enabled != false || compA.Param.EnableMetadata != 0 {
+ t.Errorf("expected metadata disabled, got nested=%v flat=%v", compA.Param.MetadataConfig.Enabled, compA.Param.EnableMetadata)
+ }
+ if compA.Param.Summary.Enabled != false || compA.Param.EnableSummary != 0 {
+ t.Errorf("expected summary disabled, got nested=%v flat=%v", compA.Param.Summary.Enabled, compA.Param.EnableSummary)
+ }
+ if compA.Param.Keywords.TopN != 0 || compA.Param.AutoKeywords != 0 {
+ t.Errorf("expected keywords disabled (0), got nested=%v flat=%v", compA.Param.Keywords.TopN, compA.Param.AutoKeywords)
+ }
+ if compA.Param.Questions.TopN != 0 || compA.Param.AutoQuestions != 0 {
+ t.Errorf("expected questions disabled (0), got nested=%v flat=%v", compA.Param.Questions.TopN, compA.Param.AutoQuestions)
+ }
+ if compA.Param.Tags.TopN != 0 || compA.Param.AutoTags != 0 {
+ t.Errorf("expected tags disabled (0), got nested=%v flat=%v", compA.Param.Tags.TopN, compA.Param.AutoTags)
+ }
+
+ // Case B: Nested explicitly enabled (enabled=true) while legacy flat has 0.
+ // Nested MUST win and enable the features.
+ paramsEnabled := map[string]any{
+ "metadata_config": map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{"key": "author", "type": "string"},
+ },
+ },
+ "enable_metadata": 0,
+ "summary": map[string]any{
+ "enabled": true,
+ },
+ "enable_summary": 0,
+ "keywords": map[string]any{
+ "top_n": 4,
+ },
+ "auto_keywords": 0,
+ "questions": map[string]any{
+ "top_n": 2,
+ },
+ "auto_questions": 0,
+ "tags": map[string]any{
+ "top_n": 3,
+ "tag_file_id": "file-123",
+ },
+ "auto_tags": 0,
+ }
+
+ compRawB, err := NewExtractorComponent(paramsEnabled)
+ if err != nil {
+ t.Fatalf("NewExtractorComponent B: %v", err)
+ }
+ compB := compRawB.(*ExtractorComponent)
+ if compB.Param.MetadataConfig.Enabled != true || compB.Param.EnableMetadata != 1 {
+ t.Errorf("expected metadata enabled, got nested=%v flat=%v", compB.Param.MetadataConfig.Enabled, compB.Param.EnableMetadata)
+ }
+ if compB.Param.Summary.Enabled != true || compB.Param.EnableSummary != 1 {
+ t.Errorf("expected summary enabled, got nested=%v flat=%v", compB.Param.Summary.Enabled, compB.Param.EnableSummary)
+ }
+ if compB.Param.Keywords.TopN != 4 || compB.Param.AutoKeywords != 4 {
+ t.Errorf("expected keywords 4, got nested=%v flat=%v", compB.Param.Keywords.TopN, compB.Param.AutoKeywords)
+ }
+ if compB.Param.Questions.TopN != 2 || compB.Param.AutoQuestions != 2 {
+ t.Errorf("expected questions 2, got nested=%v flat=%v", compB.Param.Questions.TopN, compB.Param.AutoQuestions)
+ }
+ if compB.Param.Tags.TopN != 3 || compB.Param.AutoTags != 3 {
+ t.Errorf("expected tags 3, got nested=%v flat=%v", compB.Param.Tags.TopN, compB.Param.AutoTags)
+ }
+}
diff --git a/internal/ingestion/component/schema/extractor.go b/internal/ingestion/component/schema/extractor.go
index 783defc253..4b123e73bf 100644
--- a/internal/ingestion/component/schema/extractor.go
+++ b/internal/ingestion/component/schema/extractor.go
@@ -74,18 +74,52 @@ type ExtractorFromUpstream struct {
// chunk from the LLM call).
func (ExtractorFromUpstream) Validate() error { return nil }
+// KeywordExtractConfig configures automatic keyword extraction.
+type KeywordExtractConfig struct {
+ TopN int `json:"top_n,omitempty"`
+ SystemPrompt string `json:"system_prompt,omitempty"`
+}
+
+// QuestionExtractConfig configures automatic question generation.
+type QuestionExtractConfig struct {
+ TopN int `json:"top_n,omitempty"`
+ SystemPrompt string `json:"system_prompt,omitempty"`
+}
+
+// TagExtractConfig configures automatic tag extraction.
+type TagExtractConfig struct {
+ TopN int `json:"top_n,omitempty"`
+ TagFileID string `json:"tag_file_id,omitempty"`
+}
+
+// SummaryExtractConfig configures summary / enhanced context extraction.
+type SummaryExtractConfig struct {
+ Enabled bool `json:"enabled,omitempty"`
+ SystemPrompt string `json:"system_prompt,omitempty"`
+}
+
+// MetadataExtractConfig configures structured metadata extraction.
+type MetadataExtractConfig struct {
+ Enabled bool `json:"enabled,omitempty"`
+ Metadata []common.MetadataFieldDef `json:"metadata,omitempty"`
+ BuiltInMetadata []common.MetadataFieldDef `json:"built_in_metadata,omitempty"`
+}
+
// ExtractorParam is the static configuration for the Extractor
-// component. Mirrors rag/flow/extractor/extractor.py:ExtractorParam,
-// which extends both ProcessParamBase and LLMParam. The LLM fields
-// (`llm_id`, `parameters`, `system_prompt`, `prompt`, `messages`,
-// etc.) live on the agent-side `LLMParam`; the Go port captures only
-// the Extractor-specific field plus a pointer to the LLM config so
-// the wiring is explicit.
+// component. Supports modular sub-configs (Keywords, Questions, Tags,
+// Summary, Metadata) as well as legacy flat fields for backward compatibility.
type ExtractorParam struct {
+ // Modular sub-configs
+ Keywords KeywordExtractConfig `json:"keywords,omitempty"`
+ Questions QuestionExtractConfig `json:"questions,omitempty"`
+ Tags TagExtractConfig `json:"tags,omitempty"`
+ Summary SummaryExtractConfig `json:"summary,omitempty"`
+ MetadataConfig MetadataExtractConfig `json:"metadata_config,omitempty"`
+
// FieldName is the chunk key the LLM extraction result is written
// to (Python: `self._param.field_name`). Optional — when empty,
// auto_keywords or auto_questions may still be used.
- FieldName string `json:"field_name"`
+ FieldName string `json:"field_name,omitempty"`
// LLMID identifies the LLM model used for extraction. This is the
// agent-side LLMParam.llm_id; on the ingestion side it is
@@ -99,29 +133,24 @@ type ExtractorParam struct {
Prompt string `json:"prompt,omitempty"`
// AutoKeywords enables automatic keyword extraction with a fixed
- // prompt. The value determines the top-N count.
+ // prompt. The value determines the top-N count. Legacy flat field.
AutoKeywords int `json:"auto_keywords,omitempty"`
// AutoQuestions enables automatic question generation with a fixed
- // prompt. The value determines the top-N count.
+ // prompt. The value determines the top-N count. Legacy flat field.
AutoQuestions int `json:"auto_questions,omitempty"`
- // AutoTags enables tag assignment on chunks. When > 0, the
- // component runs a two-phase tagger: Phase 1 uses Jaccard
- // matching against tag source examples; Phase 2 uses the LLM
- // for unmatched chunks. The value determines the top-N tags.
+ // AutoTags enables tag assignment on chunks. Legacy flat field.
AutoTags int `json:"auto_tags,omitempty"`
// TagFileID references a tag-definition file stored in object
- // storage. Used only when AutoTags > 0 and no inline tag
- // source text is wired in.
- TagFileID string `json:"tag_file_id"`
-
- // EnableMetadata enables automatic structured metadata extraction
- // with a fixed prompt. When > 0, runEnableMetadata asks the LLM to
- // fill the fields listed in Metadata as a JSON object and
- // merges the result into chunk["metadata"] (mirrors Python's
- // gen_metadata_task path; see doc_metadata_go_port_research.md).
+ // storage. Legacy flat field.
+ TagFileID string `json:"tag_file_id,omitempty"`
+
+ // EnableSummary enables summary extraction. Legacy flat field.
+ EnableSummary int `json:"enable_summary,omitempty"`
+
+ // EnableMetadata enables automatic structured metadata extraction. Legacy flat field.
EnableMetadata int `json:"enable_metadata,omitempty"`
// Metadata lists the target field definitions for
@@ -142,6 +171,7 @@ func (ExtractorParam) Defaults() ExtractorParam {
AutoQuestions: 0,
AutoTags: 0,
TagFileID: "",
+ EnableSummary: 0,
EnableMetadata: 0,
Metadata: nil,
}
diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.go
index 8010e58150..d6784fe80c 100644
--- a/internal/ingestion/pipeline/pipeline_params.go
+++ b/internal/ingestion/pipeline/pipeline_params.go
@@ -55,10 +55,10 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[
cleaned[pk] = pv
continue
}
- // Auto metadata params are valid extractor params even when a
- // builtin template does not declare them (the frontend stores
+ // Extractor params are valid even when a builtin template does
+ // not declare them explicitly in params: {} (the frontend stores
// them on the node params); keep them through the filter.
- if isExtractorComponent(key) && isAutoMetadataParam(pk) {
+ if isExtractorComponent(key) && isExtractorParam(pk) {
cleaned[pk] = pv
continue
}
@@ -164,11 +164,19 @@ func isExtractorComponent(cpnID string) bool {
return strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_")
}
-// isAutoMetadataParam reports whether a param belongs to the Auto metadata
-// feature (enable_metadata toggle + metadata / built_in_metadata schemas).
-func isAutoMetadataParam(param string) bool {
+// isExtractorParam reports whether a param belongs to the Extractor component
+// (modular sub-configs, legacy flat fields, LLM settings, or UI state flags).
+func isExtractorParam(param string) bool {
switch param {
- case "enable_metadata", "metadata", "built_in_metadata":
+ case "keywords", "questions", "tags", "summary", "metadata_config",
+ "auto_keywords", "keywords_sys_prompt",
+ "auto_questions", "questions_sys_prompt",
+ "auto_tags", "tag_file_id",
+ "enable_summary", "sys_prompt", "system_prompt", "prompt", "prompts", "field_name",
+ "enable_metadata", "metadata", "built_in_metadata",
+ "llm_id", "temperature", "top_p", "max_tokens", "presence_penalty", "frequency_penalty",
+ "temperatureEnabled", "topPEnabled", "maxTokensEnabled", "presencePenaltyEnabled", "frequencyPenaltyEnabled",
+ "outputs":
return true
}
return false
diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.go
index 4dc776912d..1f544dc05d 100644
--- a/internal/ingestion/pipeline/pipeline_params_test.go
+++ b/internal/ingestion/pipeline/pipeline_params_test.go
@@ -354,3 +354,61 @@ func TestBuildParserConfig_BuiltinExtractorKeepsBuiltInMetadata(t *testing.T) {
t.Fatal("expected at least one builtin template with an Extractor component")
}
}
+
+func TestCleanComponentParams_KeepsModularExtractorParams(t *testing.T) {
+ // Builtin template with empty Extractor params: {}
+ dsl := map[string]any{
+ "components": map[string]any{
+ "Extractor:AutoExtractDefault": map[string]any{
+ "obj": map[string]any{
+ "component_name": "Extractor",
+ "params": map[string]any{},
+ },
+ },
+ },
+ }
+ dslJSON, err := json.Marshal(dsl)
+ if err != nil {
+ t.Fatalf("marshal dsl: %v", err)
+ }
+
+ overrides := map[string]any{
+ "Extractor:AutoExtractDefault": map[string]any{
+ "summary": map[string]any{
+ "enabled": true,
+ "system_prompt": "summary prompt",
+ },
+ "metadata_config": map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{"key": "category", "type": "string"},
+ },
+ },
+ "keywords": map[string]any{
+ "top_n": 5,
+ },
+ "enable_summary": 1,
+ "llm_id": "gpt-4",
+ "temperature": 0.7,
+ },
+ }
+
+ result := CleanComponentParams(dslJSON, overrides)
+ ext, ok := result["Extractor:AutoExtractDefault"].(map[string]any)
+ if !ok {
+ t.Fatalf("expected Extractor:AutoExtractDefault in result, got: %v", result)
+ }
+
+ if sum, ok := ext["summary"].(map[string]any); !ok || sum["enabled"] != true {
+ t.Errorf("expected summary.enabled == true, got: %v", ext["summary"])
+ }
+ if meta, ok := ext["metadata_config"].(map[string]any); !ok || meta["enabled"] != true {
+ t.Errorf("expected metadata_config.enabled == true, got: %v", ext["metadata_config"])
+ }
+ if kw, ok := ext["keywords"].(map[string]any); !ok || kw["top_n"] != 5 {
+ t.Errorf("expected keywords.top_n == 5, got: %v", ext["keywords"])
+ }
+ if ext["llm_id"] != "gpt-4" {
+ t.Errorf("expected llm_id == gpt-4, got: %v", ext["llm_id"])
+ }
+}
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index 4b36b0f781..a8bbe30d55 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -301,19 +301,39 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
}
// builtInMetadataFromParserConfig extracts the built-in metadata config
-// (update_time / file_name) and whether auto-metadata is enabled. The frontend
-// stores both on the builtin extractor node params (Extractor:AutoExtractDefault)
-// via the operator form; the top-level parser_config shape (Python dataset
-// settings) is supported as a fallback.
+// (update_time / file_name) and whether auto-metadata is enabled. Supports
+// modular metadata_config sub-objects as well as legacy flat fields on
+// Extractor nodes and top-level parser_config.
func builtInMetadataFromParserConfig(parserConfig entity.JSONMap) ([]any, bool) {
- if nodeRaw, ok := parserConfig["Extractor:AutoExtractDefault"]; ok {
+ var extractorKeys []string
+ for k := range parserConfig {
+ if strings.HasPrefix(k, "Extractor") {
+ extractorKeys = append(extractorKeys, k)
+ }
+ }
+ sort.Strings(extractorKeys)
+
+ for _, k := range extractorKeys {
+ nodeRaw := parserConfig[k]
if node, ok := nodeRaw.(map[string]any); ok {
+ if metaConf, ok := node["metadata_config"].(map[string]any); ok {
+ enabled := parserConfigBool(metaConf["enabled"])
+ if arr, ok := metaConf["built_in_metadata"].([]any); ok && len(arr) > 0 {
+ return arr, enabled
+ }
+ }
enabled := parserConfigBool(node["enable_metadata"])
if arr, ok := node["built_in_metadata"].([]any); ok && len(arr) > 0 {
return arr, enabled
}
}
}
+ if metaConf, ok := parserConfig["metadata_config"].(map[string]any); ok {
+ enabled := parserConfigBool(metaConf["enabled"])
+ if arr, ok := metaConf["built_in_metadata"].([]any); ok && len(arr) > 0 {
+ return arr, enabled
+ }
+ }
if arr, ok := parserConfig["built_in_metadata"].([]any); ok && len(arr) > 0 {
return arr, parserConfigBool(parserConfig["enable_metadata"])
}
diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.go
index 6f52677cba..6d9fcca7be 100644
--- a/internal/ingestion/task/pipeline_executor_defaults_test.go
+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go
@@ -234,3 +234,39 @@ func TestDocNameValue(t *testing.T) {
t.Errorf("docNameValue(nil) = %q, want empty", got)
}
}
+
+func TestBuiltInMetadataFromParserConfig_MultiExtractorDeterministic(t *testing.T) {
+ // Multiple Extractor nodes with different built_in_metadata.
+ // Since keys are sorted alphabetically ("Extractor:A" before "Extractor:B"),
+ // Extractor:A must deterministically win regardless of Go map iteration order.
+ pc := entity.JSONMap{
+ "Extractor:B": map[string]any{
+ "enable_metadata": 1,
+ "built_in_metadata": []any{
+ map[string]any{"key": "file_name", "type": "string"},
+ },
+ },
+ "Extractor:A": map[string]any{
+ "metadata_config": map[string]any{
+ "enabled": true,
+ "built_in_metadata": []any{
+ map[string]any{"key": "update_time", "type": "time"},
+ },
+ },
+ },
+ }
+
+ for i := 0; i < 20; i++ {
+ cfg, enabled := builtInMetadataFromParserConfig(pc)
+ if !enabled {
+ t.Fatalf("iteration %d: expected enabled=true", i)
+ }
+ if len(cfg) != 1 {
+ t.Fatalf("iteration %d: expected 1 item, got %d", i, len(cfg))
+ }
+ item, ok := cfg[0].(map[string]any)
+ if !ok || item["key"] != "update_time" {
+ t.Fatalf("iteration %d: expected Extractor:A item update_time, got %v", i, cfg[0])
+ }
+ }
+}
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index 595d831b1b..fb5f9908b2 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -3332,8 +3332,11 @@ The Indexer will store the content in the corresponding data structures for the
summary: 'Summary',
keywords: 'Keywords',
questions: 'Questions',
+ tags: 'Tags',
metadata: 'Metadata',
fieldName: 'Result destination',
+ enableSummary: 'Enable Summary',
+ useBuiltInTemplate: 'Use built-in template',
prompts: {
system: {
keywords: `Role
diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
index c68fcdbb58..8ee02c6f71 100644
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -2914,8 +2914,11 @@ Tokenizer 会根据所选方式将内容存储为对应的数据结构。`,
summary: '增强上下文',
keywords: '关键词',
questions: '问题',
+ tags: '标签',
metadata: '元数据',
fieldName: '结果目的地',
+ enableSummary: '启用增强上下文',
+ useBuiltInTemplate: '使用内置模板',
prompts: {
system: {
keywords: `角色
diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsx
index 3f0d8a491b..e42c264ab8 100644
--- a/web/src/pages/agent/constant/pipeline.tsx
+++ b/web/src/pages/agent/constant/pipeline.tsx
@@ -92,13 +92,6 @@ export const InitialOutputFormatMap = {
[FileType.Audio]: AudioOutputFormat.Text,
};
-export enum ContextGeneratorFieldName {
- Summary = 'summary',
- Keywords = 'keywords',
- Questions = 'questions',
- Metadata = 'metadata',
-}
-
export const FileId = 'File'; // BeginId
export enum TokenizerSearchMethod {
@@ -350,11 +343,39 @@ export const initialTitleChunkerValues = {
export const initialExtractorValues = {
...initialLlmBaseValues,
- field_name: ContextGeneratorFieldName.Summary,
+ keywords: {
+ top_n: 0,
+ system_prompt: '',
+ },
+ questions: {
+ top_n: 0,
+ system_prompt: '',
+ },
+ tags: {
+ top_n: 0,
+ tag_file_id: '',
+ },
+ summary: {
+ enabled: false,
+ system_prompt: '',
+ },
+ metadata_config: {
+ enabled: false,
+ metadata: [],
+ built_in_metadata: [],
+ },
+ metadata: [],
+ built_in_metadata: [],
+ field_name: '',
auto_keywords: 0,
auto_questions: 0,
- auto_tags: 1,
+ auto_tags: 0,
tag_file_id: '',
+ enable_summary: 0,
+ enable_metadata: 0,
+ keywords_sys_prompt: '',
+ questions_sys_prompt: '',
+ sys_prompt: '',
outputs: {
chunks: { type: 'Array<Object>', value: [] },
},
diff --git a/web/src/pages/agent/form/components/prompt-editor/index.tsx b/web/src/pages/agent/form/components/prompt-editor/index.tsx
index 5ec8cb4c9b..4879e08dbe 100644
--- a/web/src/pages/agent/form/components/prompt-editor/index.tsx
+++ b/web/src/pages/agent/form/components/prompt-editor/index.tsx
@@ -62,6 +62,7 @@ const Nodes: Array<Klass<LexicalNode>> = [
type PromptContentProps = {
enablePathQueryAutoMerge: boolean;
showToolbar?: boolean;
+ showMergePath?: boolean;
multiLine?: boolean;
onBlur?: () => void;
onEnablePathQueryAutoMergeChange: (checked: boolean) => void;
@@ -70,6 +71,7 @@ type PromptContentProps = {
type IProps = {
enablePathQueryAutoMerge?: boolean;
showToolbar?: boolean;
+ showMergePath?: boolean;
multiLine?: boolean;
value?: string;
onChange?: (value?: string) => void;
@@ -81,6 +83,7 @@ type IProps = {
function PromptContent({
enablePathQueryAutoMerge,
showToolbar = true,
+ showMergePath = true,
multiLine = true,
onBlur,
onEnablePathQueryAutoMergeChange,
@@ -128,28 +131,30 @@ function PromptContent({
<p>{t('flow.insertVariableTip')}</p>
</TooltipContent>
</Tooltip>
- <Tooltip>
- <TooltipTrigger asChild>
- <label className="flex cursor-pointer items-center rounded-sm border border-border bg-bg-base/95 px-1 py-0.5 shadow-sm backdrop-blur-sm">
- <span className="sr-only">{t('flow.mergePath')}</span>
- <div className="origin-right scale-75">
- <Switch
- checked={enablePathQueryAutoMerge}
- onCheckedChange={onEnablePathQueryAutoMergeChange}
- aria-label={t('flow.mergePath')}
- />
- </div>
- </label>
- </TooltipTrigger>
- <TooltipContent>
- <p>{t('flow.mergePath')}</p>
- <p>{t('flow.mergePathTip')}</p>
- </TooltipContent>
- </Tooltip>
+ {showMergePath && (
+ <Tooltip>
+ <TooltipTrigger asChild>
+ <label className="flex cursor-pointer items-center rounded-sm border border-border bg-bg-base/95 px-1 py-0.5 shadow-sm backdrop-blur-sm">
+ <span className="sr-only">{t('flow.mergePath')}</span>
+ <div className="origin-right scale-75">
+ <Switch
+ checked={enablePathQueryAutoMerge}
+ onCheckedChange={onEnablePathQueryAutoMergeChange}
+ aria-label={t('flow.mergePath')}
+ />
+ </div>
+ </label>
+ </TooltipTrigger>
+ <TooltipContent>
+ <p>{t('flow.mergePath')}</p>
+ <p>{t('flow.mergePathTip')}</p>
+ </TooltipContent>
+ </Tooltip>
+ )}
</div>
)}
<div className="relative">
- {!showToolbar && (
+ {!showToolbar && showMergePath && (
<div className="absolute inset-y-0 right-2 z-10 flex items-center">
<Tooltip>
<TooltipTrigger asChild>
@@ -173,8 +178,9 @@ function PromptContent({
)}
<ContentEditable
className={cn(
- 'relative px-2 py-1 pr-14 focus-visible:outline-none max-h-[50vh] overflow-auto text-sm',
+ 'relative px-2 py-1 focus-visible:outline-none max-h-[50vh] overflow-auto text-sm',
{
+ 'pr-14': !showToolbar && showMergePath,
'min-h-40': multiLine,
},
)}
@@ -193,6 +199,7 @@ export const PromptEditor = forwardRef(function PromptEditor(
onBlur,
placeholder,
showToolbar = true,
+ showMergePath = true,
multiLine = true,
enablePathQueryAutoMerge = true,
extraOptions,
@@ -222,11 +229,7 @@ export const PromptEditor = forwardRef(function PromptEditor(
const onValueChange = useCallback(
(editorState: EditorState) => {
editorState?.read(() => {
- // const listNodes = $nodesOfType(VariableNode); // to be removed
- // const allNodes = $dfs();
-
const text = $getRoot().getTextContent();
-
onChange?.(text);
});
},
@@ -241,6 +244,7 @@ export const PromptEditor = forwardRef(function PromptEditor(
<PromptContent
enablePathQueryAutoMerge={isPathQueryAutoMergeEnabled}
showToolbar={showToolbar}
+ showMergePath={showMergePath}
multiLine={multiLine}
onBlur={onBlur}
onEnablePathQueryAutoMergeChange={setIsPathQueryAutoMergeEnabled}
diff --git a/web/src/pages/agent/form/extractor-form/index.tsx b/web/src/pages/agent/form/extractor-form/index.tsx
index 23d5337333..0eae00437e 100644
--- a/web/src/pages/agent/form/extractor-form/index.tsx
+++ b/web/src/pages/agent/form/extractor-form/index.tsx
@@ -1,17 +1,34 @@
+/*
+ * Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
import {
AutoKeywordsFormField,
AutoQuestionsFormField,
} from '@/components/auto-keywords-form-field';
-import { ConfirmDeleteDialog } from '@/components/confirm-delete-dialog';
import { LargeModelFormField } from '@/components/large-model-form-field';
import { LlmSettingSchema } from '@/components/llm-setting-items/next';
-import { SelectWithSearch } from '@/components/originui/select-with-search';
import { RAGFlowFormItem } from '@/components/ragflow-form';
import { SliderInputFormField } from '@/components/slider-input-form-field';
import { AsyncTreeSelect } from '@/components/ui/async-tree-select';
-import { Form } from '@/components/ui/form';
import { Button } from '@/components/ui/button';
+import { Form } from '@/components/ui/form';
import { Switch } from '@/components/ui/switch';
+import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
+import { FormLayout } from '@/constants/form';
+import { RAGFlowNodeType } from '@/interfaces/database/agent';
import { PromptEditor } from '@/pages/agent/form/components/prompt-editor';
import { MetadataType } from '@/pages/dataset/components/metedata/constant';
import {
@@ -23,42 +40,67 @@ import {
IMetaDataReturnJSONSettings,
} from '@/pages/dataset/components/metedata/interface';
import { ManageMetadataModal } from '@/pages/dataset/components/metedata/manage-modal';
-import { isGoBackend } from '@/utils/backend-runtime';
-import { buildOptions } from '@/utils/form';
import { zodResolver } from '@hookform/resolvers/zod';
import { Settings } from 'lucide-react';
-import { memo, useCallback } from 'react';
+import { memo, useCallback, useEffect, useMemo, useState } from 'react';
import { useForm, useFormContext } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import { z } from 'zod';
-import {
- ContextGeneratorFieldName,
- initialExtractorValues,
-} from '../../constant/pipeline';
+import { initialExtractorValues } from '../../constant/pipeline';
import { useOwnerTenantId } from '../../context';
-import { useBuildNodeOutputOptions } from '../../hooks/use-build-options';
import { useFormChangeCallback } from '../../hooks/use-form-change-callback';
-import { useFormValues } from '../../hooks/use-form-values';
import { useWatchFormChange } from '../../hooks/use-watch-form-change';
import { INextOperatorForm } from '../../interface';
import { buildOutputList } from '../../utils/build-output-list';
+import { transformExtractorConfigToForm } from '@/utils/pipeline-operator';
import { FormWrapper } from '../components/form-wrapper';
import { Output } from '../components/output';
-import { useSwitchPrompt } from './use-switch-prompt';
import { canSelectTagFile, useTagFileTree } from './use-tag-file-tree';
-import { FormLayout } from '@/constants/form';
export const FormSchema = z.object({
- field_name: z.string(),
- sys_prompt: z.string(),
+ keywords: z
+ .object({
+ top_n: z.number().optional(),
+ system_prompt: z.string().optional(),
+ })
+ .optional(),
+ questions: z
+ .object({
+ top_n: z.number().optional(),
+ system_prompt: z.string().optional(),
+ })
+ .optional(),
+ tags: z
+ .object({
+ top_n: z.number().optional(),
+ tag_file_id: z.string().optional(),
+ })
+ .optional(),
+ summary: z
+ .object({
+ enabled: z.union([z.number(), z.boolean()]).optional(),
+ system_prompt: z.string().optional(),
+ })
+ .optional(),
+ metadata_config: z
+ .object({
+ enabled: z.union([z.number(), z.boolean()]).optional(),
+ metadata: z.any().optional(),
+ built_in_metadata: z.any().optional(),
+ })
+ .optional(),
+
+ // Legacy flat fields for backward compatibility
+ field_name: z.string().optional(),
+ sys_prompt: z.string().optional(),
prompts: z.string().optional(),
+ keywords_sys_prompt: z.string().optional(),
+ questions_sys_prompt: z.string().optional(),
auto_keywords: z.number().optional(),
auto_questions: z.number().optional(),
auto_tags: z.number().optional(),
tag_file_id: z.string().optional(),
- // Builtin auto-metadata (mirrors Python's Auto metadata): enable_metadata
- // toggle + metadata / built_in_metadata field schema, consumed by the Go
- // extractor's runEnableMetadata at parse time.
+ enable_summary: z.union([z.number(), z.boolean()]).optional(),
enable_metadata: z.number().optional(),
metadata: z.any().optional(),
built_in_metadata: z.any().optional(),
@@ -67,11 +109,13 @@ export const FormSchema = z.object({
export type ExtractorFormSchemaType = z.infer<typeof FormSchema>;
-// Builtin auto-extract node id hardcoded in every builtin ingestion DSL
-// template (internal/ingestion/pipeline/template/*.json). Only this node
-// shows the Auto metadata option; canvas / user-pipeline extractor nodes do
-// not.
-const BuiltinAutoExtractNodeId = 'Extractor:AutoExtractDefault';
+enum ExtractorSubTab {
+ Keywords = 'keywords',
+ Questions = 'questions',
+ Tags = 'tags',
+ Summary = 'summary',
+ Metadata = 'metadata',
+}
// ExtractorAutoMetadata mirrors Python's dataset "Auto metadata" control: an
// enable_metadata switch plus a field-schema editor (custom + built-in).
@@ -91,11 +135,16 @@ function ExtractorAutoMetadata() {
const handleOpen = useCallback(() => {
showManageMetadataModal({
metadata: util.metaDataSettingJSONToMetaDataTableData(
- form.getValues('metadata') || [],
+ form.getValues('metadata_config.metadata') ||
+ form.getValues('metadata') ||
+ [],
),
isCanAdd: true,
type: MetadataType.Setting,
- builtInMetadata: form.getValues('built_in_metadata') || [],
+ builtInMetadata:
+ form.getValues('metadata_config.built_in_metadata') ||
+ form.getValues('built_in_metadata') ||
+ [],
});
}, [form, showManageMetadataModal]);
@@ -104,9 +153,19 @@ function ExtractorAutoMetadata() {
metadata?: IMetaDataReturnJSONSettings;
builtInMetadata?: IBuiltInMetadataItem[];
}) => {
- form.setValue('metadata', data?.metadata || []);
- form.setValue('built_in_metadata', data?.builtInMetadata || []);
- form.setValue('enable_metadata', 1);
+ const metaList = data?.metadata || [];
+ const builtInList = data?.builtInMetadata || [];
+ form.setValue('metadata_config.metadata', metaList, {
+ shouldDirty: true,
+ });
+ form.setValue('metadata_config.built_in_metadata', builtInList, {
+ shouldDirty: true,
+ });
+ form.setValue('metadata_config.enabled', true, { shouldDirty: true });
+ // Also keep flat fields for backward compatibility
+ form.setValue('metadata', metaList, { shouldDirty: true });
+ form.setValue('built_in_metadata', builtInList, { shouldDirty: true });
+ form.setValue('enable_metadata', 1, { shouldDirty: true });
},
[form],
);
@@ -115,7 +174,7 @@ function ExtractorAutoMetadata() {
<>
<RAGFlowFormItem
label={t('knowledgeConfiguration.autoMetadata')}
- name="enable_metadata"
+ name="metadata_config.enabled"
>
{(field) => (
<div className="flex items-center justify-between">
@@ -132,7 +191,12 @@ function ExtractorAutoMetadata() {
</Button>
<Switch
checked={field.value === 1 || field.value === true}
- onCheckedChange={(checked) => field.onChange(checked ? 1 : 0)}
+ onCheckedChange={(checked) => {
+ field.onChange(checked);
+ form.setValue('enable_metadata', checked ? 1 : 0, {
+ shouldDirty: true,
+ });
+ }}
data-testid="extractor-metadata-switch"
/>
</div>
@@ -162,38 +226,67 @@ function ExtractorAutoMetadata() {
const outputList = buildOutputList(initialExtractorValues.outputs);
+const useNormalizedExtractorFormValues = (node?: RAGFlowNodeType) => {
+ return useMemo(() => {
+ const raw = (node?.data?.form as Record<string, any>) || {};
+ return {
+ ...initialExtractorValues,
+ ...transformExtractorConfigToForm(raw),
+ };
+ }, [node?.data?.form]);
+};
+
const ExtractorForm = ({
node,
onValuesChange,
hideOutputs,
}: INextOperatorForm) => {
- const defaultValues = useFormValues(initialExtractorValues, node);
+ const defaultValues = useNormalizedExtractorFormValues(node);
const { t } = useTranslation();
const form = useForm<ExtractorFormSchemaType>({
defaultValues,
resolver: zodResolver(FormSchema),
- // mode: 'onChange',
});
- const promptOptions = useBuildNodeOutputOptions(node?.id);
-
- const options = buildOptions(ContextGeneratorFieldName, t, 'flow');
+ useEffect(() => {
+ form.reset(defaultValues);
+ }, [defaultValues, form]);
- const {
- handleFieldNameChange,
- confirmSwitch,
- hideModal,
- visible,
- cancelSwitch,
- } = useSwitchPrompt(form);
+ const [activeTab, setActiveTab] = useState<ExtractorSubTab>(
+ ExtractorSubTab.Keywords,
+ );
useWatchFormChange(node?.id, form);
useFormChangeCallback(form, onValuesChange);
const ownerTenantId = useOwnerTenantId();
- const { treeData, loadData } = useTagFileTree(form.watch('tag_file_id'));
+ const tagFileIdWatch =
+ form.watch('tags.tag_file_id') || form.watch('tag_file_id');
+ const { treeData, loadData } = useTagFileTree(tagFileIdWatch);
+
+ useEffect(() => {
+ if (!form.getValues('keywords.system_prompt')) {
+ form.setValue(
+ 'keywords.system_prompt',
+ t('flow.prompts.system.keywords'),
+ );
+ }
+ if (!form.getValues('questions.system_prompt')) {
+ form.setValue(
+ 'questions.system_prompt',
+ t('flow.prompts.system.questions'),
+ );
+ }
+ if (!form.getValues('summary.system_prompt')) {
+ form.setValue('summary.system_prompt', t('flow.prompts.system.summary'));
+ }
+ }, [form, t]);
+
+ const handleTabChange = useCallback((tab: string) => {
+ setActiveTab(tab as ExtractorSubTab);
+ }, []);
return (
<Form {...form}>
@@ -201,20 +294,74 @@ const ExtractorForm = ({
<LargeModelFormField
ownerTenantId={ownerTenantId}
></LargeModelFormField>
- <AutoKeywordsFormField name="auto_keywords"></AutoKeywordsFormField>
- <AutoQuestionsFormField name="auto_questions"></AutoQuestionsFormField>
- {isGoBackend() && (
- <>
+
+ <Tabs
+ value={activeTab}
+ onValueChange={handleTabChange}
+ className="w-full"
+ >
+ <TabsList className="w-full justify-start">
+ <TabsTrigger value={ExtractorSubTab.Keywords}>
+ {t('flow.keywords')}
+ </TabsTrigger>
+ <TabsTrigger value={ExtractorSubTab.Questions}>
+ {t('flow.questions')}
+ </TabsTrigger>
+ <TabsTrigger value={ExtractorSubTab.Tags}>
+ {t('flow.tags') || t('knowledgeDetails.autoTags')}
+ </TabsTrigger>
+ <TabsTrigger value={ExtractorSubTab.Summary}>
+ {t('flow.summary')}
+ </TabsTrigger>
+ <TabsTrigger value={ExtractorSubTab.Metadata}>
+ {t('flow.metadata')}
+ </TabsTrigger>
+ </TabsList>
+
+ <TabsContent
+ value={ExtractorSubTab.Keywords}
+ className="space-y-4 pt-2"
+ >
+ <AutoKeywordsFormField name="keywords.top_n" />
+ <RAGFlowFormItem
+ label={t('flow.systemPrompt')}
+ name="keywords.system_prompt"
+ >
+ <PromptEditor
+ placeholder={t('flow.messagePlaceholder')}
+ showToolbar={false}
+ showMergePath={false}
+ />
+ </RAGFlowFormItem>
+ </TabsContent>
+
+ <TabsContent
+ value={ExtractorSubTab.Questions}
+ className="space-y-4 pt-2"
+ >
+ <AutoQuestionsFormField name="questions.top_n" />
+ <RAGFlowFormItem
+ label={t('flow.systemPrompt')}
+ name="questions.system_prompt"
+ >
+ <PromptEditor
+ placeholder={t('flow.messagePlaceholder')}
+ showToolbar={false}
+ showMergePath={false}
+ />
+ </RAGFlowFormItem>
+ </TabsContent>
+
+ <TabsContent value={ExtractorSubTab.Tags} className="space-y-4 pt-2">
<SliderInputFormField
- name="auto_tags"
+ name="tags.top_n"
label={t('knowledgeDetails.autoTags')}
- min={1}
+ min={0}
max={10}
- defaultValue={1}
+ defaultValue={0}
layout={FormLayout.Vertical}
- ></SliderInputFormField>
-
- <RAGFlowFormItem label={t('flow.tagFile')} name="tag_file_id">
+ />
+ <RAGFlowFormItem label={t('flow.tagFile')} name="tags.tag_file_id">
{(field) => (
<AsyncTreeSelect
treeData={treeData}
@@ -222,55 +369,59 @@ const ExtractorForm = ({
onChange={field.onChange}
loadData={loadData}
canSelect={canSelectTagFile}
- ></AsyncTreeSelect>
+ />
)}
</RAGFlowFormItem>
- </>
- )}
-
- <RAGFlowFormItem label={t('flow.fieldName')} name="field_name">
- {(field) => (
- <SelectWithSearch
- onChange={(value) => {
- field.onChange(value);
- handleFieldNameChange(value);
- }}
- value={field.value}
- placeholder={t('dataFlowPlaceholder')}
- options={options}
- ></SelectWithSearch>
- )}
- </RAGFlowFormItem>
-
- {(node?.data as Record<string, any>)?.operatorId ===
- BuiltinAutoExtractNodeId && <ExtractorAutoMetadata />}
+ </TabsContent>
- <RAGFlowFormItem label={t('flow.systemPrompt')} name="sys_prompt">
- <PromptEditor
- placeholder={t('flow.messagePlaceholder')}
- showToolbar={true}
- baseOptions={promptOptions}
- ></PromptEditor>
- </RAGFlowFormItem>
+ <TabsContent
+ value={ExtractorSubTab.Summary}
+ className="space-y-4 pt-2"
+ >
+ <RAGFlowFormItem
+ label={t('flow.enableSummary')}
+ name="summary.enabled"
+ horizontal
+ valueClassName="w-auto flex justify-end"
+ >
+ {(field) => (
+ <Switch
+ checked={field.value === 1 || field.value === true}
+ onCheckedChange={(checked) => {
+ field.onChange(checked);
+ form.setValue('field_name', checked ? 'summary' : '', {
+ shouldDirty: true,
+ });
+ form.setValue('enable_summary', checked ? 1 : 0, {
+ shouldDirty: true,
+ });
+ }}
+ data-testid="extractor-summary-switch"
+ />
+ )}
+ </RAGFlowFormItem>
+ <RAGFlowFormItem
+ label={t('flow.systemPrompt')}
+ name="summary.system_prompt"
+ >
+ <PromptEditor
+ placeholder={t('flow.messagePlaceholder')}
+ showToolbar={false}
+ showMergePath={false}
+ />
+ </RAGFlowFormItem>
+ </TabsContent>
- <RAGFlowFormItem label={t('flow.userPrompt')} name="prompts">
- <PromptEditor
- showToolbar={true}
- baseOptions={promptOptions}
- ></PromptEditor>
- </RAGFlowFormItem>
+ <TabsContent
+ value={ExtractorSubTab.Metadata}
+ className="space-y-4 pt-2"
+ >
+ <ExtractorAutoMetadata />
+ </TabsContent>
+ </Tabs>
{!hideOutputs && <Output list={outputList}></Output>}
</FormWrapper>
- {visible && (
- <ConfirmDeleteDialog
- title={t('flow.switchPromptMessage')}
- open
- onOpenChange={hideModal}
- onOk={confirmSwitch}
- onCancel={cancelSwitch}
- ></ConfirmDeleteDialog>
- )}
</Form>
);
};
diff --git a/web/src/pages/agent/form/extractor-form/use-switch-prompt.ts b/web/src/pages/agent/form/extractor-form/use-switch-prompt.ts
deleted file mode 100644
index 1efe7b409c..0000000000
--- a/web/src/pages/agent/form/extractor-form/use-switch-prompt.ts
+++ /dev/null
@@ -1,68 +0,0 @@
-import { useSetModalState } from '@/hooks/common-hooks';
-import { useCallback, useRef } from 'react';
-import { useTranslation } from 'react-i18next';
-
-type SwitchPromptField = 'field_name' | 'sys_prompt' | 'prompts';
-
-type SwitchPromptForm = {
- getValues(name: 'field_name'): string;
- setValue(
- name: SwitchPromptField,
- value: string,
- options?: { shouldDirty?: boolean; shouldValidate?: boolean },
- ): void;
-};
-
-export function useSwitchPrompt(form: SwitchPromptForm) {
- const { visible, showModal, hideModal } = useSetModalState();
- const { t } = useTranslation();
- const previousFieldNames = useRef<string[]>([form.getValues('field_name')]);
-
- const setPromptValue = useCallback(
- (field: SwitchPromptField, key: string, value: string) => {
- form.setValue(field, t(`flow.prompts.${key}.${value}`), {
- shouldDirty: true,
- shouldValidate: true,
- });
- },
- [form, t],
- );
-
- const handleFieldNameChange = useCallback(
- (value: string) => {
- if (value) {
- const names = previousFieldNames.current;
- if (names.length > 1) {
- names.shift();
- }
- names.push(value);
- showModal();
- }
- },
- [showModal],
- );
-
- const confirmSwitch = useCallback(() => {
- const value = form.getValues('field_name');
- setPromptValue('sys_prompt', 'system', value);
- setPromptValue('prompts', 'user', value);
- }, [form, setPromptValue]);
-
- const cancelSwitch = useCallback(() => {
- const previousValue = previousFieldNames.current.at(-2);
- if (previousValue) {
- form.setValue('field_name', previousValue, {
- shouldDirty: true,
- shouldValidate: true,
- });
- }
- }, [form]);
-
- return {
- handleFieldNameChange,
- confirmSwitch,
- hideModal,
- visible,
- cancelSwitch,
- };
-}
diff --git a/web/src/pages/agent/hooks/use-form-change-callback.ts b/web/src/pages/agent/hooks/use-form-change-callback.ts
index 8b4b95d6b0..7ac000532c 100644
--- a/web/src/pages/agent/hooks/use-form-change-callback.ts
+++ b/web/src/pages/agent/hooks/use-form-change-callback.ts
@@ -9,7 +9,7 @@ export function useFormChangeCallback(
useEffect(() => {
if (onValuesChange) {
- onValuesChange(values);
+ onValuesChange(form.getValues());
}
- }, [onValuesChange, values]);
+ }, [form, onValuesChange, values]);
}
diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.ts
index 35fd85444e..b432f1a565 100644
--- a/web/src/pages/agent/utils.ts
+++ b/web/src/pages/agent/utils.ts
@@ -400,7 +400,78 @@ export function transformTitleChunkerParams(
}
export function transformExtractorParams(params: ExtractorFormSchemaType) {
- return { ...params, prompts: [{ content: params.prompts, role: 'user' }] };
+ const isMetadataEnabled =
+ params.metadata_config?.enabled !== undefined
+ ? Boolean(params.metadata_config?.enabled)
+ : params.enable_metadata === 1 || params.enable_metadata === true;
+
+ const isSummaryEnabled =
+ params.summary?.enabled !== undefined
+ ? Boolean(params.summary?.enabled)
+ : params.enable_summary === 1 ||
+ params.enable_summary === true ||
+ params.field_name === 'summary';
+
+ const metadataList =
+ params.metadata_config?.metadata ?? params.metadata ?? [];
+ const builtInMetadataList =
+ params.metadata_config?.built_in_metadata ??
+ params.built_in_metadata ??
+ [];
+
+ const summarySysPrompt =
+ params.summary?.system_prompt ?? params.sys_prompt ?? '';
+
+ const keywordsTopN = params.keywords?.top_n ?? params.auto_keywords ?? 0;
+ const keywordsSysPrompt =
+ params.keywords?.system_prompt ?? params.keywords_sys_prompt ?? '';
+
+ const questionsTopN = params.questions?.top_n ?? params.auto_questions ?? 0;
+ const questionsSysPrompt =
+ params.questions?.system_prompt ?? params.questions_sys_prompt ?? '';
+
+ const tagsTopN = params.tags?.top_n ?? params.auto_tags ?? 0;
+ const tagFileId = params.tags?.tag_file_id ?? params.tag_file_id ?? '';
+
+ return {
+ ...params,
+ prompts: [{ content: params.prompts, role: 'user' }],
+ auto_keywords: keywordsTopN,
+ keywords_sys_prompt: keywordsSysPrompt,
+ keywords: {
+ top_n: keywordsTopN,
+ system_prompt: keywordsSysPrompt,
+ },
+ auto_questions: questionsTopN,
+ questions_sys_prompt: questionsSysPrompt,
+ questions: {
+ top_n: questionsTopN,
+ system_prompt: questionsSysPrompt,
+ },
+ auto_tags: tagsTopN,
+ tag_file_id: tagFileId,
+ tags: {
+ top_n: tagsTopN,
+ tag_file_id: tagFileId,
+ },
+ enable_summary: isSummaryEnabled ? 1 : 0,
+ sys_prompt: summarySysPrompt,
+ field_name: isSummaryEnabled
+ ? (params.field_name || 'summary')
+ : (params.field_name === 'summary' ? '' : (params.field_name || '')),
+ summary: {
+ enabled: isSummaryEnabled,
+ system_prompt: summarySysPrompt,
+ },
+ enable_metadata: isMetadataEnabled ? 1 : 0,
+ metadata: metadataList,
+ built_in_metadata: builtInMetadataList,
+ metadata_config: {
+ enabled: isMetadataEnabled,
+ metadata: metadataList,
+ built_in_metadata: builtInMetadataList,
+ },
+ };
}
function transformDataOperationsParams(params: DataOperationsFormSchemaType) {
diff --git a/web/src/pages/agent/utils/tests/extractor-transform.test.ts b/web/src/pages/agent/utils/tests/extractor-transform.test.ts
new file mode 100644
index 0000000000..800a3b35d4
--- /dev/null
+++ b/web/src/pages/agent/utils/tests/extractor-transform.test.ts
@@ -0,0 +1,115 @@
+import { transformExtractorConfigToForm } from '@/utils/pipeline-operator';
+import { transformExtractorParams } from '../../utils';
+
+describe('Extractor parameter transformations & precedence', () => {
+ describe('transformExtractorParams', () => {
+ it('synchronizes nested modular configs to flat fields and preserves nested objects', () => {
+ const input: any = {
+ summary: {
+ enabled: true,
+ system_prompt: 'Custom summary prompt',
+ },
+ metadata_config: {
+ enabled: true,
+ metadata: [{ key: 'category', type: 'string' }],
+ built_in_metadata: [{ key: 'update_time', type: 'time' }],
+ },
+ keywords: {
+ top_n: 5,
+ system_prompt: 'KW prompt',
+ },
+ questions: {
+ top_n: 3,
+ system_prompt: 'Q prompt',
+ },
+ tags: {
+ top_n: 2,
+ tag_file_id: 'tag-123',
+ },
+ llm_id: 'gpt-4',
+ };
+
+ const result = transformExtractorParams(input);
+
+ expect(result.enable_summary).toBe(1);
+ expect(result.summary).toEqual({
+ enabled: true,
+ system_prompt: 'Custom summary prompt',
+ });
+ expect(result.enable_metadata).toBe(1);
+ expect(result.metadata_config.enabled).toBe(true);
+ expect(result.metadata_config.metadata).toHaveLength(1);
+ expect(result.auto_keywords).toBe(5);
+ expect(result.auto_questions).toBe(3);
+ expect(result.auto_tags).toBe(2);
+ expect(result.tag_file_id).toBe('tag-123');
+ });
+
+ it('gives nested modular enabled: false precedence over legacy flat enable_*: 1', () => {
+ const input: any = {
+ summary: {
+ enabled: false,
+ system_prompt: '',
+ },
+ enable_summary: 1,
+ metadata_config: {
+ enabled: false,
+ metadata: [],
+ built_in_metadata: [],
+ },
+ enable_metadata: 1,
+ };
+
+ const result = transformExtractorParams(input);
+
+ expect(result.summary.enabled).toBe(false);
+ expect(result.enable_summary).toBe(0);
+ expect(result.metadata_config.enabled).toBe(false);
+ expect(result.enable_metadata).toBe(0);
+ });
+
+ it('preserves custom field_name when summary is disabled', () => {
+ const input: any = {
+ summary: {
+ enabled: false,
+ system_prompt: '',
+ },
+ field_name: 'custom_chunk_field',
+ };
+
+ const result = transformExtractorParams(input);
+ expect(result.field_name).toBe('custom_chunk_field');
+ });
+ });
+
+ describe('transformExtractorConfigToForm', () => {
+ it('normalizes legacy flat API format into nested form schema', () => {
+ const config = {
+ enable_summary: 1,
+ sys_prompt: 'Old summary prompt',
+ enable_metadata: 1,
+ metadata: [{ key: 'author', type: 'string' }],
+ built_in_metadata: [{ key: 'file_name', type: 'string' }],
+ auto_keywords: 4,
+ auto_questions: 2,
+ auto_tags: 1,
+ tag_file_id: 'tag-file-1',
+ };
+
+ const result = transformExtractorConfigToForm(config);
+
+ expect(result.summary).toEqual({
+ enabled: true,
+ system_prompt: 'Old summary prompt',
+ });
+ expect(result.metadata_config).toEqual({
+ enabled: true,
+ metadata: [{ key: 'author', type: 'string' }],
+ built_in_metadata: [{ key: 'file_name', type: 'string' }],
+ });
+ expect(result.keywords.top_n).toBe(4);
+ expect(result.questions.top_n).toBe(2);
+ expect(result.tags.top_n).toBe(1);
+ });
+ });
+});
diff --git a/web/src/utils/pipeline-operator.ts b/web/src/utils/pipeline-operator.ts
index e436ca61d0..efaac7b69d 100644
--- a/web/src/utils/pipeline-operator.ts
+++ b/web/src/utils/pipeline-operator.ts
@@ -86,7 +86,7 @@ function transformLevelsToRules(
* DSL: { prompts: [{ content: "text", role: "user" }] }
* Form: { prompts: "text" }
*/
-function transformExtractorConfigToForm(
+export function transformExtractorConfigToForm(
config: Record<string, any> | undefined,
): Record<string, any> {
if (!config) return {};
@@ -95,6 +95,61 @@ function transformExtractorConfigToForm(
if (Array.isArray(config.prompts) && config.prompts.length > 0) {
result.prompts = config.prompts[0]?.content ?? '';
}
+
+ const isSummaryEnabled =
+ config.summary?.enabled !== undefined
+ ? Boolean(config.summary?.enabled)
+ : config.enable_summary === 1 || config.enable_summary === true;
+
+ const isMetadataEnabled =
+ config.metadata_config?.enabled !== undefined
+ ? Boolean(config.metadata_config?.enabled)
+ : config.enable_metadata === 1 || config.enable_metadata === true;
+
+ result.keywords = {
+ top_n:
+ config.keywords?.top_n ??
+ config.auto_keywords ??
+ initialExtractorValues.keywords.top_n,
+ system_prompt:
+ config.keywords?.system_prompt ?? config.keywords_sys_prompt ?? '',
+ };
+ result.questions = {
+ top_n:
+ config.questions?.top_n ??
+ config.auto_questions ??
+ initialExtractorValues.questions.top_n,
+ system_prompt:
+ config.questions?.system_prompt ?? config.questions_sys_prompt ?? '',
+ };
+ result.tags = {
+ top_n:
+ config.tags?.top_n ??
+ config.auto_tags ??
+ initialExtractorValues.tags.top_n,
+ tag_file_id: config.tags?.tag_file_id ?? config.tag_file_id ?? '',
+ };
+ result.summary = {
+ enabled: isSummaryEnabled,
+ system_prompt: config.summary?.system_prompt ?? config.sys_prompt ?? '',
+ };
+ result.enable_summary = isSummaryEnabled ? 1 : 0;
+ result.field_name = isSummaryEnabled
+ ? (config.field_name || 'summary')
+ : (config.field_name === 'summary' ? '' : (config.field_name || ''));
+
+ result.metadata_config = {
+ enabled: isMetadataEnabled,
+ metadata: config.metadata_config?.metadata ?? config.metadata ?? [],
+ built_in_metadata:
+ config.metadata_config?.built_in_metadata ??
+ config.built_in_metadata ??
+ [],
+ };
+ result.enable_metadata = isMetadataEnabled ? 1 : 0;
+ result.metadata = result.metadata_config.metadata;
+ result.built_in_metadata = result.metadata_config.built_in_metadata;
+
return result;
}