RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
/*
* Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Operator } from '@/constants/agent';
import { DSL, RAGFlowNodeType } from '@/interfaces/database/agent';
import {
getInitialExtractorValues,
initialCompilationValues,
initialExtractorValues,
initialGeneralChunkerValues,
initialParserValues,
initialTitleChunkerValues,
initialTokenChunkerValues,
initialTokenizerValues,
} from '@/pages/agent/constant/pipeline';
import {
normalizeTokenChunkerFormValues,
transformCompilationParams,
transformExtractorParams,
transformGeneralChunkerParams,
transformParserParams,
transformTitleChunkerParams,
transformTokenChunkerParams,
} from '@/pages/agent/utils';
import { cloneDeep, isEmpty, omit } from 'lodash';
export const FileNodeId = 'File';
export function getOperatorType(operatorId: string): Operator {
return (operatorId.split(':')[0] || operatorId) as Operator;
}
// The dataset-level metadata group stored at parser_config.metadata. The group
// is shaped {enabled, metadata, built_in_metadata}; the boolean enabled flag
// and the built_in_metadata array identify it among other values.
function isDatasetMetadataGroup(value: any): boolean {
return (
typeof value === 'object' &&
value !== null &&
!Array.isArray(value) &&
typeof value.enabled === 'boolean' &&
Array.isArray(value.built_in_metadata)
);
}
export function transformParserConfigSetups(
setups: Record<string, any> | undefined,
): any[] {
if (!setups || typeof setups !== 'object') {
return [];
}
// parser_config entries saved by the Python-era frontend (or derived from
// a Python canvas) nest the per-family setups under a "setups" key; the Go
// shape lists file families at the top level. Lift the nested group —
// top-level families win — and drop the non-family protocol keys so they
// never render as a bogus file format. The vision options
// (enable_vision_enhancement / vlm) are global params-level keys, not
// families, so they are dropped here too (the caller lifts them back onto
// the form's top level).
let source = setups;
const nested = source.setups;
if (nested && typeof nested === 'object' && !Array.isArray(nested)) {
source = { ...nested, ...omit(source, ['setups']) };
}
return Object.entries(
omit(source, [
'outputs',
'allowed_output_format',
'enable_vision_enhancement',
'vlm',
]),
)
.map(([fileFormat, config]) => {
const { pages, ...rest } = (config ?? {}) as Record<string, any>;
const normalizedPages = Array.isArray(pages)
? pages.map(([from, to]: [number, number]) => ({ from, to }))
: pages;
return {
fileFormat,
...rest,
...(normalizedPages !== undefined ? { pages: normalizedPages } : {}),
} as Record<string, any>;
})
.sort((a, b) => (a.order_index ?? Infinity) - (b.order_index ?? Infinity));
}
function transformLevelsToRules(
levels: any[],
): Array<{ levels: Array<{ expression: string }> }> {
if (!Array.isArray(levels)) return [];
return levels
.map((levelGroup) => {
if (Array.isArray(levelGroup)) {
const filteredExpressions = levelGroup.filter(
(expr: string) => expr && expr.trim() !== '',
);
if (filteredExpressions.length === 0) return null;
return {
levels: filteredExpressions.map((expression: string) => ({
expression,
})),
};
}
return { levels: [{ expression: '' }] };
})
.filter((rule) => rule !== null);
}
/**
* Converts Extractor config from API/DSL format to form format: flattens the
* DSL prompts array into the form's single field and normalizes the nested
* per-feature groups the extractor schema reads (schema.ExtractorParam).
* Legacy flat keys (auto_keywords, keywords_sys_prompt, enable_summary, ...)
* are accepted as read fallbacks and never re-emitted.
*
* DSL: { prompts: [{ content: "text", role: "user" }] }
* Form: { prompts: "text" }
*/
export function transformExtractorConfigToForm(
config: Record<string, any> | undefined,
): Record<string, any> {
if (!config) return {};
const result = { ...config };
if (Array.isArray(config.prompts) && config.prompts.length > 0) {
result.prompts = config.prompts[0]?.content ?? '';
}
const isSummaryEnabled =
config.summary?.enabled !== undefined
? Boolean(config.summary?.enabled)
: config.enable_summary === 1 || config.enable_summary === true;
// The metadata group is stored under "metadata" — the shape the Go
// Extractor reads (schema.ExtractorParam). Accept two historical shapes:
// the transitional "metadata_config" key, and legacy flat nodes carrying
// enable_metadata + metadata[] + built_in_metadata[] (flat "metadata" is
// an array there, which distinguishes it from the group object).
const metadataGroup =
config.metadata !== undefined && !Array.isArray(config.metadata)
? config.metadata
: config.metadata_config;
const isMetadataEnabled =
metadataGroup?.enabled !== undefined
? Boolean(metadataGroup.enabled)
: config.enable_metadata === 1 || config.enable_metadata === true;
// The enabled switch is authoritative when the group carries it; legacy
// configs without it derive enabled from top_n > 0, matching the Go
// extractor's resolveExtractToggle.
const keywordsTopN =
config.keywords?.top_n ??
config.auto_keywords ??
initialExtractorValues.keywords.top_n;
const questionsTopN =
config.questions?.top_n ??
config.auto_questions ??
initialExtractorValues.questions.top_n;
const tagsTopN =
config.tags?.top_n ?? config.auto_tags ?? initialExtractorValues.tags.top_n;
result.keywords = {
enabled:
config.keywords?.enabled !== undefined
? Boolean(config.keywords.enabled)
: keywordsTopN > 0,
top_n: keywordsTopN,
system_prompt:
config.keywords?.system_prompt ?? config.keywords_sys_prompt ?? '',
};
result.questions = {
enabled:
config.questions?.enabled !== undefined
? Boolean(config.questions.enabled)
: questionsTopN > 0,
top_n: questionsTopN,
system_prompt:
config.questions?.system_prompt ?? config.questions_sys_prompt ?? '',
};
result.tags = {
enabled:
config.tags?.enabled !== undefined
? Boolean(config.tags.enabled)
: tagsTopN > 0,
top_n: tagsTopN,
tag_file_id: config.tags?.tag_file_id ?? config.tag_file_id ?? '',
};
result.summary = {
enabled: isSummaryEnabled,
system_prompt: config.summary?.system_prompt ?? config.sys_prompt ?? '',
};
result.metadata = {
enabled: isMetadataEnabled,
metadata:
metadataGroup?.metadata ??
(Array.isArray(config.metadata) ? config.metadata : []),
built_in_metadata:
metadataGroup?.built_in_metadata ?? config.built_in_metadata ?? [],
};
// Drop the historical keys so they don't linger in form state and get
// re-emitted into the DSL on save — the extractor only reads the group.
delete result.metadata_config;
delete result.enable_metadata;
delete result.built_in_metadata;
return result;
}
/**
* Converts TokenChunker config from API/DSL format to form format.
* DSL: { delimiters: ["\n"], overlapped_percent: 0.1, table_context_size: 10, image_context_size: 20 }
* Form: { delimiters: [{ value: "\n" }], overlapped_percent: 10, image_table_context_window: 15, enable_children: false }
*/
function transformTokenChunkerConfigToForm(
config: Record<string, any> | undefined,
{ seedLegacyDelimiter = true } = {},
): Record<string, any> {
if (!config) return {};
const result = { ...config };
// Convert string array delimiters to object arrays. The legacy re-seed and
// mode normalization happen in normalizeTokenChunkerFormValues below, which
// still sees the raw delimiter_mode.
if (Array.isArray(config.delimiters)) {
result.delimiters = config.delimiters.map((d: string) => ({ value: d }));
}
if (Array.isArray(config.children_delimiters)) {
result.children_delimiters = config.children_delimiters.map(
(d: string) => ({ value: d }),
);
}
// Convert overlapped_percent from 0-1 scale to 0-100 scale
if (typeof config.overlapped_percent === 'number') {
result.overlapped_percent = Math.round(config.overlapped_percent * 100);
}
// Merge table_context_size and image_context_size into image_table_context_window
const tableSize = Number(config.table_context_size ?? 0);
const imageSize = Number(config.image_context_size ?? 0);
result.image_table_context_window = Math.max(tableSize, imageSize);
const normalized = normalizeTokenChunkerFormValues(result, {
seedLegacyDelimiter,
});
// Clean up DSL-only fields not in form schema
delete normalized.table_context_size;
delete normalized.image_context_size;
return normalized;
}
function transformGeneralChunkerConfigToForm(
config: Record<string, any> | undefined,
): Record<string, any> {
// The general chunker never had the legacy 'token_size' tab, so an empty
// saved list is always a deliberate choice — skip the legacy re-seed.
const result = transformTokenChunkerConfigToForm(config, {
seedLegacyDelimiter: false,
});
result.table_context_size = Number(config?.table_context_size ?? 0);
result.image_context_size = Number(config?.image_context_size ?? 0);
delete result.image_table_context_window;
delete result.delimiter_mode;
return result;
}
/**
* Converts TitleChunker config from API/DSL format to form format.
* DSL: { method: "hierarchy", hierarchy: "3", levels: [...], chunk_token_cap, include_heading_content, root_chunk_as_heading }
* Form: { method, hierarchyHierarchy, hierarchyGroup, hierarchyChunkTokenCap, groupChunkTokenCap, include_heading_content, root_chunk_as_heading, hierarchyRules, groupRules }
*/
function transformTitleChunkerConfigToForm(
config: Record<string, any> | undefined,
): Record<string, any> {
if (!config) return {};
const result = { ...config };
const method = config.method ?? 'hierarchy';
// Convert legacy `hierarchy` (single field) to split fields
let hierarchy = config.hierarchy;
if (typeof hierarchy === 'number') {
hierarchy = String(hierarchy);
}
if (method === 'hierarchy') {
result.hierarchyHierarchy = config.hierarchyHierarchy || hierarchy || '3';
result.hierarchyGroup = config.hierarchyGroup || '0';
} else {
result.hierarchyHierarchy = config.hierarchyHierarchy || hierarchy || '3';
result.hierarchyGroup = config.hierarchyGroup || hierarchy || '0';
}
// Convert `levels` to `hierarchyRules`/`groupRules`
if (config.levels && Array.isArray(config.levels) && !config.hierarchyRules) {
result.hierarchyRules = transformLevelsToRules(config.levels);
}
if (config.levels && Array.isArray(config.levels) && !config.groupRules) {
result.groupRules = transformLevelsToRules(config.levels);
}
// Split the backend-facing shared `chunk_token_cap` into the per-method
// fields the form edits (mirroring the rules/hierarchy split above).
if (config.chunk_token_cap !== undefined) {
if (result.hierarchyChunkTokenCap === undefined) {
result.hierarchyChunkTokenCap = config.chunk_token_cap;
}
if (result.groupChunkTokenCap === undefined) {
result.groupChunkTokenCap = config.chunk_token_cap;
}
}
// Clean up DSL-only fields
delete result.hierarchy;
delete result.levels;
delete result.rules;
delete result.chunk_token_cap;
return result;
}
/**
* Tokenizer is a passthrough — both API and form formats are identical.
*/
function transformTokenizerConfigToForm(
config: Record<string, any> | undefined,
): Record<string, any> {
return config ?? {};
}
/**
* Dispatches reverse transform by operator type.
* Converts DSL-format config to form-format config.
*/
export function transformApiConfigToForm(
operatorType: string,
config: Record<string, any> | undefined,
): Record<string, any> {
switch (operatorType) {
case Operator.Parser:
// Lift the global vision options back onto the form's top level; omit
// them when absent so an explicit undefined cannot clobber template or
// default baselines downstream (Object.assign/spread copies undefined).
return {
setups: transformParserConfigSetups(config),
...(config?.enable_vision_enhancement !== undefined
? { enable_vision_enhancement: config.enable_vision_enhancement }
: {}),
...(config?.vlm !== undefined ? { vlm: config.vlm } : {}),
};
case Operator.Extractor:
return transformExtractorConfigToForm(config);
case Operator.Tokenizer:
return transformTokenizerConfigToForm(config);
case Operator.TokenChunker:
return transformTokenChunkerConfigToForm(config);
case Operator.GeneralChunker:
return transformGeneralChunkerConfigToForm(config);
case Operator.TitleChunker:
case Operator.ManualChunker:
return transformTitleChunkerConfigToForm(config);
default:
return config ?? {};
}
}
/**
* Converts a saved parser_config (API format, keyed by operator id) to the
* form format used by the operator tabs. Configs without pipeline keys
* (built-in parse type) are returned as-is.
*
* The dataset-level metadata group is authoritative for Extractor nodes,
* mirroring buildOperatorNode, so the values seeded into the outer form match
* what the operator tabs initialize from.
*/
export function transformSavedParserConfigToForm(
parserConfig?: Record<string, any>,
): Record<string, any> | undefined {
if (
!parserConfig ||
typeof parserConfig !== 'object' ||
Array.isArray(parserConfig) ||
!Object.keys(parserConfig).some((key) => key.includes(':'))
) {
return parserConfig;
}
const formParserConfig: Record<string, any> = {};
for (const [operatorId, config] of Object.entries(parserConfig)) {
const operatorType = getOperatorType(operatorId);
const apiConfig =
operatorType === Operator.Extractor &&
isDatasetMetadataGroup(parserConfig.metadata)
? { ...config, metadata: parserConfig.metadata }
: (config as Record<string, any>);
formParserConfig[operatorId] = transformApiConfigToForm(
operatorType,
apiConfig,
);
}
return formParserConfig;
}
/**
* Dispatches forward transform by operator type.
* Converts form-format config to DSL format for saving.
*/
export function transformFormConfigToApi(
operatorType: string,
config: Record<string, any> | undefined,
): Record<string, any> {
if (!config) return {};
switch (operatorType) {
case Operator.Parser:
return transformParserParams(config as any);
case Operator.Extractor:
return transformExtractorParams(config as any);
case Operator.Compiler:
return transformCompilationParams(config as Record<string, any>);
case Operator.Tokenizer:
return config; // passthrough for Tokenizer
case Operator.TokenChunker:
return transformTokenChunkerParams(config as any);
case Operator.GeneralChunker:
return transformGeneralChunkerParams(config as any);
case Operator.TitleChunker:
return transformTitleChunkerParams(config as any);
case Operator.ManualChunker:
// These fields are UI-only for the title chunker and are not read by
// the backend ManualChunker component (manual.go pins method=group,
// ignores the token cap).
return transformTitleChunkerParams(
omit(config, [
'include_heading_content',
'chunk_token_cap',
'hierarchyChunkTokenCap',
'groupChunkTokenCap',
]) as any,
);
default:
return config;
}
}
export function normalizeOperatorForm(
operatorId: string,
rawForm: Record<string, any> | undefined,
): Record<string, any> {
const operatorType = getOperatorType(operatorId);
switch (operatorType) {
case Operator.Parser: {
return {
...cloneDeep(initialParserValues),
...rawForm,
setups: rawForm?.setups?.length
? rawForm.setups
: cloneDeep(initialParserValues.setups),
};
}
case Operator.TitleChunker:
case Operator.ManualChunker:
return {
...cloneDeep(initialTitleChunkerValues),
...rawForm,
};
case Operator.TokenChunker:
return {
...cloneDeep(initialTokenChunkerValues),
...rawForm,
};
case Operator.GeneralChunker:
return {
...cloneDeep(initialGeneralChunkerValues),
...rawForm,
};
case Operator.Extractor:
return {
...cloneDeep(getInitialExtractorValues()),
...rawForm,
};
case Operator.Compiler:
return {
...cloneDeep(initialCompilationValues),
...rawForm,
};
case Operator.Tokenizer:
return {
...cloneDeep(initialTokenizerValues),
...rawForm,
};
default:
return rawForm ?? {};
}
}
export function buildOperatorNode(
dslNode: RAGFlowNodeType,
pipelineParserConfig: Record<string, any> = {},
): RAGFlowNodeType {
const operatorId = dslNode.id;
const operatorType = getOperatorType(operatorId);
// Form-format config from DSL graph
const configFromDsl = dslNode.data?.form;
const rawForm = {
...configFromDsl, // template defaults from DSL (form format)
};
if (!isEmpty(pipelineParserConfig)) {
let apiConfig = pipelineParserConfig[operatorId];
// Dataset-level auto-metadata lives at parser_config.metadata; the
// backend preserves that object and re-scopes it into every Extractor
// node on write, so it is the authoritative source for the metadata
// toggle's initial state — prefer it over a possibly stale per-node copy.
if (
operatorType === Operator.Extractor &&
isDatasetMetadataGroup(pipelineParserConfig.metadata)
) {
apiConfig = {
...apiConfig,
metadata: pipelineParserConfig.metadata,
};
}
// user overrides from API (now also form format)
Object.assign(
rawForm,
transformApiConfigToForm(operatorType, apiConfig), // Convert API config to form format, then merge (DSL template is baseline, API overrides)
);
}
return {
...dslNode,
id: '',
data: {
...dslNode.data,
label: operatorType,
operatorId,
form: normalizeOperatorForm(operatorId, rawForm),
},
};
}
export function buildPipelineOperatorNodes(
dsl?: DSL,
pipelineParserConfig: Record<string, any> = {},
): RAGFlowNodeType[] {
if (!dsl?.graph?.nodes || !dsl?.graph?.edges) {
return [];
}
// Build source → target map from edges (pipeline is linear)
const sourceToTarget = new Map<string, string>();
for (const edge of dsl.graph.edges) {
sourceToTarget.set(edge.source, edge.target);
}
// Follow the chain starting from File, collecting node IDs in order
const orderedIds: string[] = [];
let currentId: string | undefined = FileNodeId;
while (currentId) {
orderedIds.push(currentId);
currentId = sourceToTarget.get(currentId);
}
// Build a lookup from node ID → node
const nodeById = new Map<string, RAGFlowNodeType>();
for (const node of dsl.graph.nodes) {
nodeById.set(node.id, node);
}
// Map ordered IDs to nodes, excluding File
return orderedIds
.filter((id) => id !== FileNodeId)
.map((id) => nodeById.get(id))
.filter((node): node is RAGFlowNodeType => node !== undefined)
.map((node) => buildOperatorNode(node, pipelineParserConfig));
}
/**
* Builds a parser_config record from operator nodes, keyed by operatorId.
* Each entry is the node's merged form (DSL template defaults + overrides),
* i.e. exactly what the operator tabs display.
*/
export function buildParserConfigFromNodes(
operatorNodes: RAGFlowNodeType[],
): Record<string, any> {
return Object.fromEntries(
operatorNodes.map((node) => [
(node.data as Record<string, any>)?.operatorId || node.data?.label || '',
node.data?.form ?? {},
]),
);
}