Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(agent): migrate chunk_summary and title_chunker templates to modular Extractor params (#20754)

britz113a3dba5613
diff --git a/internal/agent/templates/chunk_summary.json b/internal/agent/templates/chunk_summary.json
index 7be45c4e80..f862f68c1c 100644
--- a/internal/agent/templates/chunk_summary.json
+++ b/internal/agent/templates/chunk_summary.json
@@ -21,7 +21,6 @@
"obj": {
"component_name": "Extractor",
"params": {
- "field_name": "summary",
"frequencyPenaltyEnabled": true,
"frequency_penalty": 0.7,
"llm_id": "",
@@ -35,16 +34,12 @@
},
"presencePenaltyEnabled": true,
"presence_penalty": 0.4,
- "prompts": [
- {
- "content": "Text to Summarize:\n{TokenChunker:ModernPetsKneel@chunks}",
- "role": "user"
- }
- ],
- "sys_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff.",
+ "summary": {
+ "enabled": true,
+ "system_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff."
+ },
"temperature": 0.1,
"temperatureEnabled": true,
- "tenant_llm_id": null,
"topPEnabled": true,
"top_p": 0.3
}
@@ -492,7 +487,6 @@
{
"data": {
"form": {
- "field_name": "summary",
"frequencyPenaltyEnabled": true,
"frequency_penalty": 0.7,
"llm_id": "",
@@ -506,11 +500,12 @@
},
"presencePenaltyEnabled": true,
"presence_penalty": 0.4,
- "prompts": "Text to Summarize:\n{TokenChunker:ModernPetsKneel@chunks}",
- "sys_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff.",
+ "summary": {
+ "enabled": true,
+ "system_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff."
+ },
"temperature": 0.1,
"temperatureEnabled": true,
- "tenant_llm_id": null,
"topPEnabled": true,
"top_p": 0.3
},
diff --git a/internal/agent/templates/title_chunker.json b/internal/agent/templates/title_chunker.json
index cc9bd7eb7c..294cd87276 100644
--- a/internal/agent/templates/title_chunker.json
+++ b/internal/agent/templates/title_chunker.json
@@ -21,7 +21,6 @@
"obj": {
"component_name": "Extractor",
"params": {
- "field_name": "summary",
"frequencyPenaltyEnabled": true,
"frequency_penalty": 0.7,
"llm_id": "",
@@ -35,16 +34,12 @@
},
"presencePenaltyEnabled": true,
"presence_penalty": 0.4,
- "prompts": [
- {
- "content": "Text to Summarize:\n{TitleChunker:WideResultsTeach@chunks}",
- "role": "user"
- }
- ],
- "sys_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff.",
+ "summary": {
+ "enabled": true,
+ "system_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff."
+ },
"temperature": 0.1,
"temperatureEnabled": true,
- "tenant_llm_id": null,
"topPEnabled": true,
"top_p": 0.3
}
@@ -567,7 +562,6 @@
{
"data": {
"form": {
- "field_name": "summary",
"frequencyPenaltyEnabled": true,
"frequency_penalty": 0.7,
"llm_id": "",
@@ -581,11 +575,12 @@
},
"presencePenaltyEnabled": true,
"presence_penalty": 0.4,
- "prompts": "Text to Summarize:\n{TitleChunker:WideResultsTeach@chunks}",
- "sys_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff.",
+ "summary": {
+ "enabled": true,
+ "system_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff."
+ },
"temperature": 0.1,
"temperatureEnabled": true,
- "tenant_llm_id": null,
"topPEnabled": true,
"top_p": 0.3
},
diff --git a/internal/ingestion/pipeline/agent_template_extractor_contract_test.go b/internal/ingestion/pipeline/agent_template_extractor_contract_test.go
new file mode 100644
index 0000000000..56e6c965e4
--- /dev/null
+++ b/internal/ingestion/pipeline/agent_template_extractor_contract_test.go
@@ -0,0 +1,180 @@
+package pipeline
+
+import (
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "reflect"
+ "testing"
+)
+
+// TestAgentTemplateExtractorsUseModularParams guards against #20716: the
+// Extractor component only reads the modular sub-configs
+// (keywords/questions/tags/summary/metadata) declared by schema.ExtractorParam.
+// A template still baking the legacy flat fields (field_name/prompts/
+// sys_prompt/tenant_llm_id) ships an Extractor whose prompt configuration is
+// silently ignored at runtime.
+func TestAgentTemplateExtractorsUseModularParams(t *testing.T) {
+ // The agent templates carry a canvas form (data.form) that mirrors the DSL
+ // params; the ingestion_pipeline_* templates keep their params only in the
+ // DSL component, so only the former must have a matching form.
+ dirs := []struct {
+ path string
+ requireForm bool
+ }{
+ {filepath.Join("..", "..", "agent", "templates"), true},
+ {filepath.Join("template"), false},
+ }
+ legacyKeys := []string{"field_name", "prompts", "sys_prompt", "tenant_llm_id"}
+ modularKeys := []string{"keywords", "questions", "tags", "summary", "metadata"}
+
+ // The agent templates preset an active sub-task; the ingestion_pipeline_*
+ // templates ship every sub-task disabled and let the operator enable them
+ // on the canvas, so only the former must be active out of the box.
+ mustBeActive := map[string]bool{
+ "advanced_ingestion_pipeline.json": true,
+ "chunk_summary.json": true,
+ "title_chunker.json": true,
+ }
+
+ // subTaskActive mirrors the runtime gates in internal/ingestion/component:
+ // summary/metadata run when enabled is true; keywords/questions/tags follow
+ // resolveExtractToggle — an explicit "enabled" key is authoritative, and only
+ // a legacy group without it derives its state from top_n > 0.
+ subTaskActive := func(name string, v any) bool {
+ m, ok := v.(map[string]any)
+ if !ok {
+ return false
+ }
+ switch name {
+ case "keywords", "questions", "tags":
+ if enabled, ok := m["enabled"]; ok {
+ if b, ok := enabled.(bool); ok {
+ return b
+ }
+ n, ok := enabled.(float64)
+ return ok && n == 1
+ }
+ n, ok := m["top_n"].(float64)
+ return ok && n > 0
+ default:
+ e, ok := m["enabled"].(bool)
+ return ok && e
+ }
+ }
+
+ seen := 0
+ for _, dir := range dirs {
+ files, err := filepath.Glob(filepath.Join(dir.path, "*.json"))
+ if err != nil {
+ t.Fatalf("glob %s: %v", dir.path, err)
+ }
+ for _, f := range files {
+ raw, err := os.ReadFile(f)
+ if err != nil {
+ t.Fatalf("read %s: %v", f, err)
+ }
+ var doc map[string]any
+ if err := json.Unmarshal(raw, &doc); err != nil {
+ t.Fatalf("unmarshal %s: %v", f, err)
+ }
+ if doc["canvas_type"] != "Ingestion Pipeline" {
+ continue
+ }
+ base := filepath.Base(f)
+ components, _ := doc["dsl"].(map[string]any)["components"].(map[string]any)
+
+ // Canvas nodes mirror the component params in data.form for the
+ // agent templates; collect them by id so the two copies can be
+ // compared.
+ nodeForms := map[string]map[string]any{}
+ if nodes, ok := doc["dsl"].(map[string]any)["graph"].(map[string]any)["nodes"].([]any); ok {
+ for _, n := range nodes {
+ node, _ := n.(map[string]any)
+ id, _ := node["id"].(string)
+ form, _ := node["data"].(map[string]any)["form"].(map[string]any)
+ if id != "" && form != nil {
+ nodeForms[id] = form
+ }
+ }
+ }
+ for cpnID, comp := range components {
+ obj, _ := comp.(map[string]any)["obj"].(map[string]any)
+ if obj["component_name"] != "Extractor" {
+ continue
+ }
+ seen++
+ params, _ := obj["params"].(map[string]any)
+
+ if form, ok := nodeForms[cpnID]; ok {
+ if !reflect.DeepEqual(form, params) {
+ t.Errorf("%s: Extractor %s params and canvas data.form diverge; both must be updated together", f, cpnID)
+ }
+ } else if dir.requireForm {
+ t.Errorf("%s: Extractor %s has no matching canvas data.form node to compare against params", f, cpnID)
+ }
+ for _, k := range legacyKeys {
+ if _, found := params[k]; found {
+ t.Errorf("%s: Extractor %s still uses legacy param %q; migrate it to the modular format", f, cpnID, k)
+ }
+ }
+
+ hasModular := false
+ activeCount := 0
+ for _, k := range modularKeys {
+ v, ok := params[k]
+ if !ok || v == nil {
+ continue
+ }
+ hasModular = true
+ if subTaskActive(k, v) {
+ activeCount++
+ }
+ }
+ if !hasModular {
+ t.Errorf("%s: Extractor %s configures no modular sub-task (keywords/questions/tags/summary/metadata)", f, cpnID)
+ }
+ if mustBeActive[base] && activeCount == 0 {
+ t.Errorf("%s: Extractor %s has every modular sub-task disabled; the template's purpose requires at least one active", f, cpnID)
+ }
+
+ // The params must also survive CleanComponentParams: no
+ // dropped keys and any active sub-task left active with its
+ // prompt intact.
+ dslRaw, _ := json.Marshal(doc["dsl"])
+ cleaned := CleanComponentParams(dslRaw, map[string]any{cpnID: params})
+ got, _ := cleaned[cpnID].(map[string]any)
+ if got == nil {
+ t.Errorf("%s: Extractor %s dropped by CleanComponentParams", f, cpnID)
+ continue
+ }
+ for k := range params {
+ if _, kept := got[k]; !kept {
+ t.Errorf("%s: Extractor %s param %q dropped by CleanComponentParams", f, cpnID, k)
+ }
+ }
+ for _, k := range modularKeys {
+ v, ok := params[k]
+ if !ok || !subTaskActive(k, v) {
+ continue
+ }
+ if !subTaskActive(k, got[k]) {
+ t.Errorf("%s: Extractor %s sub-task %q no longer active after CleanComponentParams: %#v", f, cpnID, k, got[k])
+ continue
+ }
+ if k == "summary" {
+ cleanedSummary, _ := got[k].(map[string]any)
+ prompt, _ := cleanedSummary["system_prompt"].(string)
+ if prompt == "" {
+ t.Errorf("%s: Extractor %s summary.system_prompt empty after CleanComponentParams", f, cpnID)
+ }
+ }
+ }
+ }
+ }
+ }
+ if seen == 0 {
+ t.Fatal("no Ingestion Pipeline templates with an Extractor found; test paths stale?")
+ }
+ t.Logf("checked %d Extractor components across builtin ingestion templates", seen)
+}