RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
import { RAGFlowNodeType } from '@/interfaces/database/agent';
import { FileType } from '@/constants/file';
import { normalizeParserConfig } from '@/hooks/parser-config-utils';
import {
transformApiConfigToForm,
transformSavedParserConfigToForm,
} from '@/utils/pipeline-operator';
import { Operator } from './constant';
import { FormSchema } from './form/parser-form/schema';
import {
generateNodeNamesWithIncreasingIndex,
isEmptyMessageContent,
normalizeTokenChunkerFormValues,
receiveMessageError,
shouldSeedDefaultDelimiter,
transformGeneralChunkerParams,
transformParserParams,
transformTokenChunkerParams,
} from './utils';
describe('transformParserParams MinerU settings', () => {
it('round-trips component-scoped choices through validation, save and reload', () => {
const config = {
'Parser:primary': {
pdf: {
parse_method: 'mineru-model-id',
mineru_formula_enable: false,
mineru_table_enable: true,
mineru_parse_method: 'ocr',
mineru_lang: 'Japanese',
},
},
'Parser:secondary': {
pdf: {
parse_method: 'mineru-model-id',
mineru_formula_enable: true,
mineru_table_enable: false,
},
},
};
const form = transformSavedParserConfigToForm(config)!;
const saved = Object.fromEntries(
Object.entries(form).map(([id, values]) => [
id,
transformParserParams(FormSchema.parse(values)),
]),
);
const stored = JSON.parse(JSON.stringify(normalizeParserConfig(saved)));
expect(stored).toMatchObject(config);
expect(transformSavedParserConfigToForm(stored)).toMatchObject(form);
});
it.each([
{ modelId: 'model-b', ocrEnabled: true },
{ modelId: '', ocrEnabled: true },
{ modelId: 'model-b', ocrEnabled: false },
{ modelId: '', ocrEnabled: false },
])(
'round-trips MinerU PDF controls alongside image OCR $modelId with OCR enabled=$ocrEnabled',
({ modelId, ocrEnabled }) => {
const config = {
'Parser:primary': {
enable_vision_enhancement: !ocrEnabled,
vlm: {
llm_id: 'vision-model-id',
system_prompt: 'Describe visible details.',
},
pdf: {
parse_method: 'mineru-model-id',
mineru_formula_enable: false,
mineru_table_enable: true,
mineru_parse_method: 'ocr',
mineru_lang: 'Japanese',
},
image: {
ocr_enabled: ocrEnabled,
ocr_model_id: modelId,
},
},
};
// Exercise the saved-config boundary and schema, not live OCR execution.
const form = transformSavedParserConfigToForm(config)!;
const saved = {
'Parser:primary': transformParserParams(
FormSchema.parse(form['Parser:primary']),
),
};
const stored = JSON.parse(JSON.stringify(normalizeParserConfig(saved)));
expect(stored).toMatchObject(config);
expect(stored['Parser:primary'].pdf).not.toHaveProperty('ocr_enabled');
expect(stored['Parser:primary'].pdf).not.toHaveProperty('ocr_model_id');
for (const key of [
'mineru_formula_enable',
'mineru_table_enable',
'mineru_parse_method',
'mineru_lang',
]) {
expect(stored['Parser:primary'].image).not.toHaveProperty(key);
}
const reloaded = transformSavedParserConfigToForm(stored)!;
const savedAgain = {
'Parser:primary': transformParserParams(
FormSchema.parse(reloaded['Parser:primary']),
),
};
expect(
JSON.parse(JSON.stringify(normalizeParserConfig(savedAgain))),
).toEqual(stored);
},
);
it.each([undefined, true, false])(
'preserves independent formula/table choices and engine settings: %s',
(formulaEnabled) => {
for (const tableEnabled of [undefined, true, false]) {
const result = transformParserParams({
setups: [
{
fileFormat: FileType.PDF,
parse_method: 'mineru-model-id',
mineru_parse_method: 'ocr',
mineru_lang: 'Japanese',
...(formulaEnabled === undefined
? {}
: { mineru_formula_enable: formulaEnabled }),
...(tableEnabled === undefined
? {}
: { mineru_table_enable: tableEnabled }),
},
],
} as any);
const saved = JSON.parse(JSON.stringify(result)).pdf;
expect(saved).toMatchObject({
parse_method: 'mineru-model-id',
mineru_parse_method: 'ocr',
mineru_lang: 'Japanese',
});
if (formulaEnabled === undefined) {
expect(saved).not.toHaveProperty('mineru_formula_enable');
} else {
expect(saved.mineru_formula_enable).toBe(formulaEnabled);
}
if (tableEnabled === undefined) {
expect(saved).not.toHaveProperty('mineru_table_enable');
} else {
expect(saved.mineru_table_enable).toBe(tableEnabled);
}
}
},
);
});
describe('transformParserParams vision settings', () => {
it.each(['model-b', ''])(
'saves the explicit image OCR selection %j',
(modelId) => {
const result: Record<string, any> = transformParserParams({
enable_vision_enhancement: false,
setups: [
{
fileFormat: FileType.Image,
ocr_enabled: true,
ocr_model_id: modelId,
},
],
} as any);
expect(result.image).toMatchObject({
ocr_enabled: true,
ocr_model_id: modelId,
});
expect(result.vlm.llm_id).toBe('');
const reloaded = transformApiConfigToForm(
Operator.Parser,
JSON.parse(JSON.stringify(result)),
);
const savedAgain: Record<string, any> = transformParserParams(
reloaded as any,
);
expect(savedAgain.image).toEqual(result.image);
},
);
it('saves the normalized global settings when the node form was not opened', () => {
const result = transformParserParams({
setups: [
{ fileFormat: FileType.PDF, vlm: { llm_id: 'model-B' } },
{ fileFormat: FileType.Audio, vlm: { llm_id: 'asr-model' } },
],
} as any);
expect(result.enable_vision_enhancement).toBe(true);
expect(result.vlm).toEqual({
llm_id: 'model-B',
system_prompt: '',
});
expect(result).not.toHaveProperty(`${FileType.PDF}.vlm`);
expect(result).toHaveProperty(`${FileType.Audio}.vlm`, {
llm_id: 'asr-model',
});
});
it.each([true, false])('preserves model B with enhancement %s', (enabled) => {
const result = transformParserParams({
enable_vision_enhancement: enabled,
vlm: { llm_id: 'model-B' },
setups: [{ fileFormat: FileType.Image, parse_method: 'ocr' }],
} as any);
expect(result.enable_vision_enhancement).toBe(enabled);
expect(result.vlm).toEqual({
llm_id: 'model-B',
system_prompt: '',
});
});
it('writes the image OCR switch without retired vision fields', () => {
// The saved shape is the contract the Go parser reads, so pin it: the
// switch out, and the retired per-family keys never written back.
const result: Record<string, any> = transformParserParams({
enable_vision_enhancement: false,
vlm: { llm_id: '', system_prompt: '' },
setups: [
{
fileFormat: FileType.Image,
ocr_enabled: false,
parse_method: 'stale-model@provider',
lang: 'French',
system_prompt: 'stale',
output_format: 'json',
},
],
} as any);
expect(result.image).toMatchObject({ ocr_enabled: false });
expect(result.image).not.toHaveProperty('parse_method');
expect(result.image).not.toHaveProperty('lang');
expect(result.image).not.toHaveProperty('system_prompt');
// output_format is retired: the Go Parser normalizes every family to
// JSON, so a stored legacy value must never be written back.
expect(result.image).not.toHaveProperty('output_format');
});
});
describe('transformTokenChunkerParams', () => {
it('keeps overlapped_percent and delimiters when delimiter_mode is one', () => {
// Regression: saving with delimiter_mode='one' used to zero these fields,
// so a reload showed overlapped_percent=0 and delimiters=["\n"].
const result = transformTokenChunkerParams({
delimiter_mode: 'one',
chunk_token_size: 512,
overlapped_percent: 9,
image_table_context_window: 81,
delimiters: [{ value: '\n' }, { value: '!' }, { value: '。' }],
children_delimiters: [],
enable_children: false,
} as any);
expect(result.overlapped_percent).toBeCloseTo(0.09);
expect(result.delimiters).toEqual(['\n', '!', '。']);
expect(result.delimiter_mode).toBe('one');
});
it('converts form values to api format in delimiter mode', () => {
const result = transformTokenChunkerParams({
delimiter_mode: 'delimiter',
chunk_token_size: 512,
overlapped_percent: 30,
image_table_context_window: 81,
delimiters: [{ value: '\n' }, { value: '' }],
children_delimiters: [{ value: '|' }],
enable_children: true,
} as any);
expect(result.overlapped_percent).toBeCloseTo(0.3);
expect(result.delimiters).toEqual(['\n', '']);
expect(result.children_delimiters).toEqual(['|']);
expect(result.table_context_size).toBe(81);
expect(result.image_context_size).toBe(81);
});
it('emits an explicit empty delimiters list when the user removed all rows', () => {
// The backend falls back to its own default (["\n"]) when the key is
// absent, so the save path must emit [] to keep pure token-size chunking.
const result = transformTokenChunkerParams({
delimiter_mode: 'delimiter',
chunk_token_size: 512,
overlapped_percent: 0,
image_table_context_window: 0,
delimiters: [],
children_delimiters: [],
enable_children: false,
} as any);
expect(result.delimiters).toEqual([]);
});
it('migrates a legacy node on save even when the form was never opened', () => {
// Nodes saved under the removed 'token_size' tab persisted an empty list;
// the save path applies the same migration the form would.
const result = transformTokenChunkerParams({
delimiter_mode: 'token_size',
chunk_token_size: 512,
overlapped_percent: 0,
image_table_context_window: 0,
delimiters: [],
children_delimiters: [],
} as any);
expect(result.delimiter_mode).toBe('delimiter');
expect(result.delimiters).toEqual(['\n']);
});
it('keeps legacy children delimiters instead of wiping them', () => {
// Nodes saved before the enable_children toggle existed carry children
// delimiters without the flag; derive it instead of dropping the values.
const result = transformTokenChunkerParams({
delimiter_mode: 'delimiter',
chunk_token_size: 512,
overlapped_percent: 0,
image_table_context_window: 0,
delimiters: [{ value: '\n' }],
children_delimiters: [{ value: '|' }],
} as any);
expect(result.enable_children).toBe(true);
expect(result.children_delimiters).toEqual(['|']);
});
});
describe('transformGeneralChunkerParams', () => {
it('never re-seeds the general chunker delimiter list on save', () => {
const result = transformGeneralChunkerParams({
chunk_token_size: 512,
overlapped_percent: 0,
delimiters: [],
children_delimiters: [],
enable_children: false,
table_context_size: 0,
image_context_size: 0,
} as any);
expect(result.delimiters).toEqual([]);
expect(result).not.toHaveProperty('delimiter_mode');
});
});
describe('normalizeTokenChunkerFormValues', () => {
it('normalizes a legacy token_size node and re-seeds the delimiter row', () => {
const result = normalizeTokenChunkerFormValues({
delimiter_mode: 'token_size',
delimiters: [],
children_delimiters: [],
});
expect(result.delimiter_mode).toBe('delimiter');
expect(result.delimiters).toEqual([{ value: '\n' }]);
});
it('keeps a deliberate empty list saved by the current form', () => {
const result = normalizeTokenChunkerFormValues({
delimiter_mode: 'one',
delimiters: [],
});
expect(result.delimiter_mode).toBe('one');
expect(result.delimiters).toEqual([]);
});
it('derives enable_children from a non-empty children list', () => {
const result = normalizeTokenChunkerFormValues({
children_delimiters: [{ value: '\n\n' }],
});
expect(result.enable_children).toBe(true);
});
it('respects an explicit enable_children flag over derivation', () => {
const result = normalizeTokenChunkerFormValues({
enable_children: false,
children_delimiters: [{ value: '\n\n' }],
});
expect(result.enable_children).toBe(false);
});
it('is idempotent', () => {
const once = normalizeTokenChunkerFormValues({
delimiters: [],
children_delimiters: [{ value: '|' }],
});
expect(normalizeTokenChunkerFormValues(once)).toEqual(once);
});
it('skips the legacy re-seed when seedLegacyDelimiter is false', () => {
const result = normalizeTokenChunkerFormValues(
{ delimiters: [] },
{ seedLegacyDelimiter: false },
);
expect(result.delimiters).toEqual([]);
});
});
describe('shouldSeedDefaultDelimiter', () => {
it('seeds only for legacy nodes with an empty list', () => {
// Legacy 'token_size' tab nodes always persisted an empty list.
expect(shouldSeedDefaultDelimiter('token_size', [])).toBe(true);
// Older DSLs may not carry a mode at all.
expect(shouldSeedDefaultDelimiter(undefined, [])).toBe(true);
expect(shouldSeedDefaultDelimiter(undefined, undefined)).toBe(true);
});
it('never seeds when the current form saved the node', () => {
// An explicit mode means the empty list is a deliberate user choice
// (pure token-size chunking) and must survive a reload.
expect(shouldSeedDefaultDelimiter('delimiter', [])).toBe(false);
expect(shouldSeedDefaultDelimiter('one', [])).toBe(false);
});
it('never seeds a non-empty list', () => {
expect(shouldSeedDefaultDelimiter('token_size', ['\n'])).toBe(false);
expect(shouldSeedDefaultDelimiter(undefined, [{ value: '\n' }])).toBe(
false,
);
});
});
describe('receiveMessageError', () => {
it('accepts successful SSE events without an application code', () => {
expect(
receiveMessageError({
response: { status: 200 },
data: { event: 'workflow_finished' },
}),
).toBe(false);
});
it('rejects application errors returned with HTTP 200', () => {
expect(
receiveMessageError({ response: { status: 200 }, data: { code: 102 } }),
).toBe(true);
expect(
receiveMessageError({ response: { status: 200 }, data: { code: 0 } }),
).toBe(false);
});
});
describe('Message component content validation', () => {
describe('isEmptyMessageContent', () => {
it('rejects missing, malformed and blank content', () => {
expect(isEmptyMessageContent()).toBe(true);
expect(isEmptyMessageContent(null)).toBe(true);
expect(isEmptyMessageContent('')).toBe(true);
expect(isEmptyMessageContent(' \t ')).toBe(true);
expect(isEmptyMessageContent([])).toBe(true);
expect(isEmptyMessageContent(['', ' \t '])).toBe(true);
// Non-string entries never satisfy the backend either.
expect(isEmptyMessageContent([123])).toBe(true);
});
it('accepts strings and arrays without blank entries', () => {
expect(isEmptyMessageContent('hello')).toBe(false);
expect(isEmptyMessageContent(['hi'])).toBe(false);
expect(isEmptyMessageContent(['', '{begin@query}'])).toBe(true);
expect(isEmptyMessageContent([' text '])).toBe(false);
});
});
});
describe('generateNodeNamesWithIncreasingIndex', () => {
const createNamedNode = (name: string) =>
({
id: `${Operator.Retrieval}:${name}`,
type: 'ragNode',
position: { x: 0, y: 0 },
data: { label: Operator.Retrieval, name, form: {} },
}) as RAGFlowNodeType;
it('uses the bare name for the first operator of a type', () => {
expect(generateNodeNamesWithIncreasingIndex('Retrieval', [])).toBe(
'Retrieval',
);
});
it('appends an index only from the second operator on', () => {
expect(
generateNodeNamesWithIncreasingIndex('Retrieval', [
createNamedNode('Retrieval'),
]),
).toBe('Retrieval_1');
expect(
generateNodeNamesWithIncreasingIndex('Retrieval', [
createNamedNode('Retrieval'),
createNamedNode('Retrieval_1'),
]),
).toBe('Retrieval_2');
});
it('fills the gap between existing indexes', () => {
expect(
generateNodeNamesWithIncreasingIndex('Retrieval', [
createNamedNode('Retrieval'),
createNamedNode('Retrieval_2'),
]),
).toBe('Retrieval_1');
});
it('does not backfill index 0 when only suffixed names exist', () => {
expect(
generateNodeNamesWithIncreasingIndex('Retrieval', [
createNamedNode('Retrieval_1'),
]),
).toBe('Retrieval_2');
});
it('treats a legacy _0 name as the first operator', () => {
expect(
generateNodeNamesWithIncreasingIndex('Retrieval', [
createNamedNode('Retrieval_0'),
]),
).toBe('Retrieval_1');
});
it('ignores nodes of other types and non-indexed names', () => {
expect(
generateNodeNamesWithIncreasingIndex('Retrieval', [
createNamedNode('Message'),
createNamedNode('Retrieval_beta'),
createNamedNode('Retrieval_1_extra'),
]),
).toBe('Retrieval');
});
});