RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: Limit the pipeline canvas to a maximum of one extractor operator. (#20711)
balibabu0aa00d0d9da0
diff --git a/internal/agent/templates/advanced_ingestion_pipeline.json b/internal/agent/templates/advanced_ingestion_pipeline.jsonindex 68ac23a2a8..f4215787c2 100644--- a/internal/agent/templates/advanced_ingestion_pipeline.json+++ b/internal/agent/templates/advanced_ingestion_pipeline.json@@ -14,160 +14,81 @@"canvas_category": "dataflow_canvas","dsl": {"components": {- "Extractor:CurlyEmusJam": {+ "Extractor:LazyCarpetsKiss": {"downstream": ["Tokenizer:WittySunsListen"],"obj": {"component_name": "Extractor","params": {- "field_name": "metadata",- "frequencyPenaltyEnabled": true,- "frequency_penalty": 0.7,"llm_id": "",- "maxTokensEnabled": false,- "max_tokens": 256,- "outputs": {- "chunks": {- "type": "Array<Object>",- "value": []- }- },- "presencePenaltyEnabled": true,- "presence_penalty": 0.4,- "prompts": [- {- "content": "Content:\n{Extractor:SmartWindowsHammer@chunks}",- "role": "user"- }- ],- "sys_prompt": "Extract important structured information from the given content. Output ONLY a valid JSON string with no additional text. If no important structured information is found, output an empty JSON object: {}.\n\nImportant structured information may include: names, dates, locations, events, key facts, numerical data, or other extractable entities.","temperature": 0.1,- "temperatureEnabled": true,- "tenant_llm_id": null,- "topPEnabled": true,- "top_p": 0.3- }- },- "upstream": [- "Extractor:SmartWindowsHammer"- ]- },- "Extractor:LazyCarpetsKiss": {- "downstream": [- "Extractor:LovelyPearsRest"- ],- "obj": {- "component_name": "Extractor",- "params": {- "field_name": "summary",- "frequencyPenaltyEnabled": true,+ "top_p": 0.3,"frequency_penalty": 0.7,- "llm_id": "",- "maxTokensEnabled": false,- "max_tokens": 256,- "outputs": {- "chunks": {- "type": "Array<Object>",- "value": []- }- },- "presencePenaltyEnabled": true,"presence_penalty": 0.4,- "prompts": [- {- "content": "Text to Summarize:\n{TokenChunker:BumpyStarsPress@chunks}",- "role": "user"- }- ],- "sys_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff.",- "temperature": 0.1,- "temperatureEnabled": true,- "tenant_llm_id": null,- "topPEnabled": true,- "top_p": 0.3- }- },- "upstream": [- "TokenChunker:BumpyStarsPress"- ]- },- "Extractor:LovelyPearsRest": {- "downstream": [- "Extractor:SmartWindowsHammer"- ],- "obj": {- "component_name": "Extractor",- "params": {- "field_name": "keywords",- "frequencyPenaltyEnabled": true,- "frequency_penalty": 0.7,- "llm_id": "",- "maxTokensEnabled": false,"max_tokens": 256,- "outputs": {- "chunks": {- "type": "Array<Object>",- "value": []- }- },- "presencePenaltyEnabled": true,- "presence_penalty": 0.4,- "prompts": [- {- "content": "Text Content\n{Extractor:LazyCarpetsKiss@chunks}",- "role": "user"- }- ],- "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nExtract the most important keywords/phrases of a given piece of text content.\n\nRequirements\n- Summarize the text content, and give the top 5 important keywords/phrases.\n- The keywords MUST be in the same language as the given piece of text content.\n- The keywords are delimited by ENGLISH COMMA.\n- Output keywords ONLY.",- "temperature": 0.1,"temperatureEnabled": true,- "tenant_llm_id": null,"topPEnabled": true,- "top_p": 0.3- }- },- "upstream": [- "Extractor:LazyCarpetsKiss"- ]- },- "Extractor:SmartWindowsHammer": {- "downstream": [- "Extractor:CurlyEmusJam"- ],- "obj": {- "component_name": "Extractor",- "params": {- "field_name": "questions",+ "presencePenaltyEnabled": true,"frequencyPenaltyEnabled": true,- "frequency_penalty": 0.7,- "llm_id": "","maxTokensEnabled": false,- "max_tokens": 256,+ "keywords": {+ "top_n": 5,+ "system_prompt": "Role\nYou are a text analyzer.\n\nTask\nExtract the most important keywords/phrases of a given piece of text content.\n\nRequirements\n- Summarize the text content, and give the top {{ topn }} important keywords/phrases.\n- The keywords MUST be in the same language as the given piece of text content.\n- The keywords are delimited by ENGLISH COMMA.\n- Output keywords ONLY."+ },+ "questions": {+ "top_n": 3,+ "system_prompt": "Role\nYou are a text analyzer.\n\nTask\nPropose {{ topn }} questions about a given piece of text content.\n\nRequirements\n- Understand and summarize the text content, and propose the top {{ topn }} important questions.\n- The questions SHOULD NOT have overlapping meanings.\n- The questions SHOULD cover the main content of the text as much as possible.\n- The questions MUST be in the same language as the given piece of text content.\n- One question per line.\n- Output questions ONLY."+ },+ "tags": {+ "top_n": 0,+ "tag_file_id": ""+ },+ "summary": {+ "enabled": true,+ "system_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff."+ },+ "metadata": {+ "enabled": true,+ "metadata": [+ {+ "key": "names",+ "type": "string",+ "description": "Names of people, organizations, or products mentioned in the content"+ },+ {+ "key": "dates",+ "type": "string",+ "description": "Important dates or time periods mentioned in the content"+ },+ {+ "key": "locations",+ "type": "string",+ "description": "Geographic locations mentioned in the content"+ },+ {+ "key": "events",+ "type": "string",+ "description": "Key events described in the content"+ },+ {+ "key": "key_facts",+ "type": "string",+ "description": "Important facts or numerical data stated in the content"+ }+ ],+ "built_in_metadata": []+ },"outputs": {"chunks": {"type": "Array<Object>","value": []}- },- "presencePenaltyEnabled": true,- "presence_penalty": 0.4,- "prompts": [- {- "content": "Text Content\n{Extractor:LovelyPearsRest@chunks}",- "role": "user"- }- ],- "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nPropose 3 questions about a given piece of text content.\n\nRequirements\n- Understand and summarize the text content, and propose the top 3 important questions.\n- The questions SHOULD NOT have overlapping meanings.\n- The questions SHOULD cover the main content of the text as much as possible.\n- The questions MUST be in the same language as the given piece of text content.\n- One question per line.\n- Output questions ONLY.",- "temperature": 0.1,- "temperatureEnabled": true,- "tenant_llm_id": null,- "topPEnabled": true,- "top_p": 0.3+ }}},"upstream": [- "Extractor:LovelyPearsRest"+ "TokenChunker:BumpyStarsPress"]},"File": {@@ -368,7 +289,7 @@}},"upstream": [- "Extractor:CurlyEmusJam"+ "Extractor:LazyCarpetsKiss"]}},@@ -402,41 +323,9 @@"data": {"isHovered": false},- "id": "xy-edge__Extractor:LazyCarpetsKissstart-Extractor:LovelyPearsRestend",+ "id": "xy-edge__Extractor:LazyCarpetsKissstart-Tokenizer:WittySunsListenend","source": "Extractor:LazyCarpetsKiss","sourceHandle": "start",- "target": "Extractor:LovelyPearsRest",- "targetHandle": "end"- },- {- "data": {- "isHovered": false- },- "id": "xy-edge__Extractor:LovelyPearsReststart-Extractor:SmartWindowsHammerend",- "selected": false,- "source": "Extractor:LovelyPearsRest",- "sourceHandle": "start",- "target": "Extractor:SmartWindowsHammer",- "targetHandle": "end"- },- {- "data": {- "isHovered": false- },- "id": "xy-edge__Extractor:SmartWindowsHammerstart-Extractor:CurlyEmusJamend",- "selected": false,- "source": "Extractor:SmartWindowsHammer",- "sourceHandle": "start",- "target": "Extractor:CurlyEmusJam",- "targetHandle": "end"- },- {- "data": {- "isHovered": false- },- "id": "xy-edge__Extractor:CurlyEmusJamstart-Tokenizer:WittySunsListenend",- "source": "Extractor:CurlyEmusJam",- "sourceHandle": "start","target": "Tokenizer:WittySunsListen","targetHandle": "end"}@@ -611,170 +500,84 @@{"data": {"form": {- "field_name": "summary",- "frequencyPenaltyEnabled": true,- "frequency_penalty": 0.7,"llm_id": "",- "maxTokensEnabled": false,- "max_tokens": 256,- "outputs": {- "chunks": {- "type": "Array<Object>",- "value": []- }- },- "presencePenaltyEnabled": true,- "presence_penalty": 0.4,- "prompts": "Text to Summarize:\n{TokenChunker:BumpyStarsPress@chunks}",- "sys_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff.","temperature": 0.1,- "temperatureEnabled": true,- "tenant_llm_id": null,- "topPEnabled": true,- "top_p": 0.3- },- "label": "Extractor",- "name": "Summarization"- },- "id": "Extractor:LazyCarpetsKiss",- "measured": {- "height": 90,- "width": 200- },- "position": {- "x": 916.9952409420641,- "y": 195.39629819663406- },- "selected": false,- "sourcePosition": "right",- "targetPosition": "left",- "type": "contextNode"- },- {- "data": {- "form": {- "field_name": "keywords",- "frequencyPenaltyEnabled": true,+ "top_p": 0.3,"frequency_penalty": 0.7,- "llm_id": "",- "maxTokensEnabled": false,- "max_tokens": 256,- "outputs": {- "chunks": {- "type": "Array<Object>",- "value": []- }- },- "presencePenaltyEnabled": true,"presence_penalty": 0.4,- "prompts": "Text Content\n{Extractor:LazyCarpetsKiss@chunks}",- "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nExtract the most important keywords/phrases of a given piece of text content.\n\nRequirements\n- Summarize the text content, and give the top 5 important keywords/phrases.\n- The keywords MUST be in the same language as the given piece of text content.\n- The keywords are delimited by ENGLISH COMMA.\n- Output keywords ONLY.",- "temperature": 0.1,- "temperatureEnabled": true,- "tenant_llm_id": null,- "topPEnabled": true,- "top_p": 0.3- },- "label": "Extractor",- "name": "Auto Keyword"- },- "dragging": false,- "id": "Extractor:LovelyPearsRest",- "measured": {- "height": 90,- "width": 200- },- "position": {- "x": 983.5410692821999,- "y": 301.1557383781162- },- "selected": false,- "sourcePosition": "right",- "targetPosition": "left",- "type": "contextNode"- },- {- "data": {- "form": {- "field_name": "questions",- "frequencyPenaltyEnabled": true,- "frequency_penalty": 0.7,- "llm_id": "",- "maxTokensEnabled": false,"max_tokens": 256,- "outputs": {- "chunks": {- "type": "Array<Object>",- "value": []- }- },- "presencePenaltyEnabled": true,- "presence_penalty": 0.4,- "prompts": "Text Content\n{Extractor:LovelyPearsRest@chunks}",- "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nPropose 3 questions about a given piece of text content.\n\nRequirements\n- Understand and summarize the text content, and propose the top 3 important questions.\n- The questions SHOULD NOT have overlapping meanings.\n- The questions SHOULD cover the main content of the text as much as possible.\n- The questions MUST be in the same language as the given piece of text content.\n- One question per line.\n- Output questions ONLY.",- "temperature": 0.1,"temperatureEnabled": true,- "tenant_llm_id": null,"topPEnabled": true,- "top_p": 0.3- },- "label": "Extractor",- "name": "Auto Question"- },- "dragging": false,- "id": "Extractor:SmartWindowsHammer",- "measured": {- "height": 90,- "width": 200- },- "position": {- "x": 1021.1009769800036,- "y": 421.67760363913044- },- "selected": false,- "sourcePosition": "right",- "targetPosition": "left",- "type": "contextNode"- },- {- "data": {- "form": {- "field_name": "metadata",+ "presencePenaltyEnabled": true,"frequencyPenaltyEnabled": true,- "frequency_penalty": 0.7,- "llm_id": "","maxTokensEnabled": false,- "max_tokens": 256,+ "keywords": {+ "top_n": 5,+ "system_prompt": "Role\nYou are a text analyzer.\n\nTask\nExtract the most important keywords/phrases of a given piece of text content.\n\nRequirements\n- Summarize the text content, and give the top {{ topn }} important keywords/phrases.\n- The keywords MUST be in the same language as the given piece of text content.\n- The keywords are delimited by ENGLISH COMMA.\n- Output keywords ONLY."+ },+ "questions": {+ "top_n": 3,+ "system_prompt": "Role\nYou are a text analyzer.\n\nTask\nPropose {{ topn }} questions about a given piece of text content.\n\nRequirements\n- Understand and summarize the text content, and propose the top {{ topn }} important questions.\n- The questions SHOULD NOT have overlapping meanings.\n- The questions SHOULD cover the main content of the text as much as possible.\n- The questions MUST be in the same language as the given piece of text content.\n- One question per line.\n- Output questions ONLY."+ },+ "tags": {+ "top_n": 0,+ "tag_file_id": ""+ },+ "summary": {+ "enabled": true,+ "system_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff."+ },+ "metadata": {+ "enabled": true,+ "metadata": [+ {+ "key": "names",+ "type": "string",+ "description": "Names of people, organizations, or products mentioned in the content"+ },+ {+ "key": "dates",+ "type": "string",+ "description": "Important dates or time periods mentioned in the content"+ },+ {+ "key": "locations",+ "type": "string",+ "description": "Geographic locations mentioned in the content"+ },+ {+ "key": "events",+ "type": "string",+ "description": "Key events described in the content"+ },+ {+ "key": "key_facts",+ "type": "string",+ "description": "Important facts or numerical data stated in the content"+ }+ ],+ "built_in_metadata": []+ },"outputs": {"chunks": {"type": "Array<Object>","value": []}- },- "presencePenaltyEnabled": true,- "presence_penalty": 0.4,- "prompts": "Content:\n{Extractor:SmartWindowsHammer@chunks}",- "sys_prompt": "Extract important structured information from the given content. Output ONLY a valid JSON string with no additional text. If no important structured information is found, output an empty JSON object: {}.\n\nImportant structured information may include: names, dates, locations, events, key facts, numerical data, or other extractable entities.",- "temperature": 0.1,- "temperatureEnabled": true,- "tenant_llm_id": null,- "topPEnabled": true,- "top_p": 0.3+ }},"label": "Extractor",- "name": "Auto Metadata"+ "name": "Auto Extract"},- "dragging": false,- "id": "Extractor:CurlyEmusJam",+ "id": "Extractor:LazyCarpetsKiss","measured": {"height": 90,"width": 200},"position": {- "x": 1065.7115140232393,- "y": 527.4370438206126+ "x": 916.9952409420641,+ "y": 195.39629819663406},- "selected": true,+ "selected": false,"sourcePosition": "right","targetPosition": "left","type": "contextNode"diff --git a/web/AGENTS.md b/web/AGENTS.mdindex 9511f4b6e5..8d379255fe 100644--- a/web/AGENTS.md+++ b/web/AGENTS.md@@ -134,6 +134,7 @@ The folder `src/components/ui/` is the project's **shared UI library** — it co- Components: `EditableTextarea`, `RAGFlowFormItem`- Constants: `InitialMockData`, `DefaultPlaceholder`- Avoid camelCase or SCREAMING_SNAKE_CASE for components and top-level constants.+- **Place domain constant lists in the page's `constant/` directory** (e.g. `pages/agent/constant/pipeline.tsx`) when they belong to an existing family of constants there (like `ChunkerOperators`, `SingleOperators`), instead of declaring them locally next to their consumer. Keeping related lists in one file makes their overlaps and differences visible and gives new entries a single obvious home.- **Name things semantically, not generically.** A name should say what the value holds or what the callback does, so call sites read naturally. Don't leak a generic underlying-primitive API into your own component's public props — rename at the wrapper boundary (e.g., a tabs wrapper exposes `activeTab` / `onTabChange`, mapped internally to Radix's `value` / `onValueChange`), and align with what callers already name the state. Same for variables and functions: `hiddenNodeIds` over `nodeIdSet`, `pickByBackend` over `resolveVariant`.- Avoid duplicating component structures in JSX; favor render props or reusable components.diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsxindex 4852cb1590..aa8652cae3 100644--- a/web/src/pages/agent/constant/pipeline.tsx+++ b/web/src/pages/agent/constant/pipeline.tsx@@ -429,12 +429,23 @@ export const NoDebugOperatorsList = [Operator.File];// legacy Python operator and is deliberately excluded.export const ChunkerOperators = [Operator.TokenChunker, Operator.TitleChunker];+// Operators offered in the flat top-level list of the pipeline "next step"+// menu, distinct from the chunker group. Single-instance filtering happens+// per entry in buildPipelineNextOperators.+export const TopLevelOperators = [+ Operator.Parser,+ Operator.Tokenizer,+ Operator.Compiler,+ Operator.Extractor,+];+// Pipeline operators limited to a single instance per canvas. Title/Token// chunkers are separate entries here (each blocks its own duplication) but// also share one slot: see the chunker group rule in buildPipelineNextOperators.export const SingleOperators = [Operator.Tokenizer,Operator.Compiler,+ Operator.Extractor,Operator.GeneralChunker,Operator.TokenChunker,Operator.TitleChunker,diff --git a/web/src/pages/agent/utils/__tests__/pipeline-connection.test.ts b/web/src/pages/agent/utils/__tests__/pipeline-connection.test.tsnew file mode 100644index 0000000000..9c3d572d61--- /dev/null+++ b/web/src/pages/agent/utils/__tests__/pipeline-connection.test.ts@@ -0,0 +1,59 @@+import { Operator } from '@/constants/agent';+import {+ buildPipelineNextOperators,+ hasPipelineNextOperators,+ isSingleInstanceOperator,+} from '../pipeline-connection';++const noOperators = () => false;+const onlyExtractor = (operator: Operator) => operator === Operator.Extractor;++describe('buildPipelineNextOperators', () => {+ it('offers Extractor when the canvas has none', () => {+ const { operators } = buildPipelineNextOperators(+ Operator.TokenChunker,+ noOperators,+ );+ expect(operators).toContain(Operator.Extractor);+ });++ it('hides Extractor when one is already on the canvas', () => {+ const { operators } = buildPipelineNextOperators(+ Operator.TokenChunker,+ onlyExtractor,+ );+ expect(operators).not.toContain(Operator.Extractor);+ // Other operators are unaffected.+ expect(operators).toEqual(+ expect.arrayContaining([+ Operator.Parser,+ Operator.Tokenizer,+ Operator.Compiler,+ ]),+ );+ });++ it('still hides the chunker group from an Extractor source', () => {+ const { operators, showChunker } = buildPipelineNextOperators(+ Operator.Extractor,+ onlyExtractor,+ );+ expect(operators).not.toContain(Operator.Extractor);+ expect(showChunker).toBe(false);+ });+});++describe('hasPipelineNextOperators', () => {+ it('reports an empty menu when only single-instance operators remain', () => {+ const everythingTaken = () => true;+ expect(hasPipelineNextOperators(Operator.Extractor, everythingTaken)).toBe(+ false,+ );+ });+});++describe('isSingleInstanceOperator', () => {+ it('treats Extractor as single-instance so duplication is disabled', () => {+ expect(isSingleInstanceOperator(Operator.Extractor)).toBe(true);+ });+});diff --git a/web/src/pages/agent/utils/pipeline-connection.ts b/web/src/pages/agent/utils/pipeline-connection.tsindex ba1647f924..04dc99098a 100644--- a/web/src/pages/agent/utils/pipeline-connection.ts+++ b/web/src/pages/agent/utils/pipeline-connection.ts@@ -1,5 +1,9 @@import { Operator } from '@/constants/agent';-import { ChunkerOperators, SingleOperators } from '../constant/pipeline';+import {+ ChunkerOperators,+ SingleOperators,+ TopLevelOperators,+} from '../constant/pipeline';export function isChunkerOperator(operator: Operator): boolean {return (ChunkerOperators as Operator[]).includes(operator);@@ -15,8 +19,8 @@ export function isSingleInstanceOperator(operator: Operator): boolean {}export interface PipelineNextOperators {- // Operators offered in the flat top-level list (Parser, Tokenizer,- // Extractor, Compiler), minus the single-instance ones already on canvas.+ // Operators offered in the flat top-level list, minus the single-instance+ // ones already on canvas.operators: Operator[];// The chunker variants, offered only while the single chunker slot is free.chunkerOperators: Operator[];@@ -38,16 +42,14 @@ export function buildPipelineNextOperators(const operators: Operator[] = [];// Pipelines require a Parser to feed a chunker, so from a Parser node the- // menu offers only the chunker group.+ // menu offers only the chunker group. Extractor joins the filtered list+ // because the Go pipeline allows at most one per canvas (ValidatePipeline).if (source !== Operator.Parser) {- [Operator.Parser, Operator.Tokenizer, Operator.Compiler].forEach(- (operator) => {- if (!hasOperator(operator)) {- operators.push(operator);- }- },- );- operators.push(Operator.Extractor);+ TopLevelOperators.forEach((operator) => {+ if (!hasOperator(operator)) {+ operators.push(operator);+ }+ });}// The chunker group occupies a single slot: any chunker already on the