Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: Unable to delete the last item in the token chunker's delimiter configuration. (#20376)

balibabu1eb3307ce247
diff --git a/web/src/pages/agent/form/token-chunker-form/index.tsx b/web/src/pages/agent/form/token-chunker-form/index.tsx
index 03acf402d9..974e249f98 100644
--- a/web/src/pages/agent/form/token-chunker-form/index.tsx
+++ b/web/src/pages/agent/form/token-chunker-form/index.tsx
@@ -9,7 +9,6 @@ import { Form } from '@/components/ui/form';
import { Switch } from '@/components/ui/switch';
import { FormTooltip } from '@/components/ui/tooltip';
import { zodResolver } from '@hookform/resolvers/zod';
-import { isEmpty } from 'lodash';
import { Info, Trash2 } from 'lucide-react';
import { memo } from 'react';
import { useFieldArray, useForm, useWatch } from 'react-hook-form';
@@ -27,6 +26,7 @@ import {
getChunkerChildrenDelimiterPreview,
getChunkerDelimiterPreview,
getChunkerDelimiterTipKey,
+ normalizeTokenChunkerFormValues,
} from '../../utils';
import { buildOutputList } from '../../utils/build-output-list';
import { FormWrapper } from '../components/form-wrapper';
@@ -73,17 +73,11 @@ const TokenChunkerForm = ({
const defaultValues = useFormValues(initialValues, node);
const { t } = useTranslation();
- // Normalize legacy values: 'token_size' (removed tab) and empty fall back
- // to 'delimiter'; nodes saved in the removed tab may carry empty delimiters,
- // so seed the default '\n' row.
- const formDefaultValues = {
- ...defaultValues,
- delimiter_mode:
- defaultValues.delimiter_mode === 'one' ? 'one' : 'delimiter',
- delimiters: isEmpty(defaultValues.delimiters)
- ? [{ value: '\n' }]
- : defaultValues.delimiters,
- };
+ // The general chunker never had the legacy 'token_size' tab, so it never
+ // re-seeds an empty delimiter list.
+ const formDefaultValues = normalizeTokenChunkerFormValues(defaultValues, {
+ seedLegacyDelimiter: !isGeneralChunker,
+ });
const form = useForm<TokenChunkerFormSchemaType>({
defaultValues: formDefaultValues,
diff --git a/web/src/pages/agent/utils.test.ts b/web/src/pages/agent/utils.test.ts
index d9953034be..bf6057590f 100644
--- a/web/src/pages/agent/utils.test.ts
+++ b/web/src/pages/agent/utils.test.ts
@@ -3,7 +3,10 @@ import { Operator } from './constant';
import {
generateNodeNamesWithIncreasingIndex,
isEmptyMessageContent,
+ normalizeTokenChunkerFormValues,
receiveMessageError,
+ shouldSeedDefaultDelimiter,
+ transformGeneralChunkerParams,
transformTokenChunkerParams,
} from './utils';
@@ -43,6 +46,153 @@ describe('transformTokenChunkerParams', () => {
expect(result.table_context_size).toBe(81);
expect(result.image_context_size).toBe(81);
});
+
+ it('emits an explicit empty delimiters list when the user removed all rows', () => {
+ // The backend falls back to its own default (["\n"]) when the key is
+ // absent, so the save path must emit [] to keep pure token-size chunking.
+ const result = transformTokenChunkerParams({
+ delimiter_mode: 'delimiter',
+ chunk_token_size: 512,
+ overlapped_percent: 0,
+ image_table_context_window: 0,
+ delimiters: [],
+ children_delimiters: [],
+ enable_children: false,
+ } as any);
+
+ expect(result.delimiters).toEqual([]);
+ });
+
+ it('migrates a legacy node on save even when the form was never opened', () => {
+ // Nodes saved under the removed 'token_size' tab persisted an empty list;
+ // the save path applies the same migration the form would.
+ const result = transformTokenChunkerParams({
+ delimiter_mode: 'token_size',
+ chunk_token_size: 512,
+ overlapped_percent: 0,
+ image_table_context_window: 0,
+ delimiters: [],
+ children_delimiters: [],
+ } as any);
+
+ expect(result.delimiter_mode).toBe('delimiter');
+ expect(result.delimiters).toEqual(['\n']);
+ });
+
+ it('keeps legacy children delimiters instead of wiping them', () => {
+ // Nodes saved before the enable_children toggle existed carry children
+ // delimiters without the flag; derive it instead of dropping the values.
+ const result = transformTokenChunkerParams({
+ delimiter_mode: 'delimiter',
+ chunk_token_size: 512,
+ overlapped_percent: 0,
+ image_table_context_window: 0,
+ delimiters: [{ value: '\n' }],
+ children_delimiters: [{ value: '|' }],
+ } as any);
+
+ expect(result.enable_children).toBe(true);
+ expect(result.children_delimiters).toEqual(['|']);
+ });
+});
+
+describe('transformGeneralChunkerParams', () => {
+ it('never re-seeds the general chunker delimiter list on save', () => {
+ const result = transformGeneralChunkerParams({
+ chunk_token_size: 512,
+ overlapped_percent: 0,
+ delimiters: [],
+ children_delimiters: [],
+ enable_children: false,
+ table_context_size: 0,
+ image_context_size: 0,
+ } as any);
+
+ expect(result.delimiters).toEqual([]);
+ expect(result).not.toHaveProperty('delimiter_mode');
+ });
+});
+
+describe('normalizeTokenChunkerFormValues', () => {
+ it('normalizes a legacy token_size node and re-seeds the delimiter row', () => {
+ const result = normalizeTokenChunkerFormValues({
+ delimiter_mode: 'token_size',
+ delimiters: [],
+ children_delimiters: [],
+ });
+
+ expect(result.delimiter_mode).toBe('delimiter');
+ expect(result.delimiters).toEqual([{ value: '\n' }]);
+ });
+
+ it('keeps a deliberate empty list saved by the current form', () => {
+ const result = normalizeTokenChunkerFormValues({
+ delimiter_mode: 'one',
+ delimiters: [],
+ });
+
+ expect(result.delimiter_mode).toBe('one');
+ expect(result.delimiters).toEqual([]);
+ });
+
+ it('derives enable_children from a non-empty children list', () => {
+ const result = normalizeTokenChunkerFormValues({
+ children_delimiters: [{ value: '\n\n' }],
+ });
+
+ expect(result.enable_children).toBe(true);
+ });
+
+ it('respects an explicit enable_children flag over derivation', () => {
+ const result = normalizeTokenChunkerFormValues({
+ enable_children: false,
+ children_delimiters: [{ value: '\n\n' }],
+ });
+
+ expect(result.enable_children).toBe(false);
+ });
+
+ it('is idempotent', () => {
+ const once = normalizeTokenChunkerFormValues({
+ delimiters: [],
+ children_delimiters: [{ value: '|' }],
+ });
+
+ expect(normalizeTokenChunkerFormValues(once)).toEqual(once);
+ });
+
+ it('skips the legacy re-seed when seedLegacyDelimiter is false', () => {
+ const result = normalizeTokenChunkerFormValues(
+ { delimiters: [] },
+ { seedLegacyDelimiter: false },
+ );
+
+ expect(result.delimiters).toEqual([]);
+ });
+});
+
+describe('shouldSeedDefaultDelimiter', () => {
+ it('seeds only for legacy nodes with an empty list', () => {
+ // Legacy 'token_size' tab nodes always persisted an empty list.
+ expect(shouldSeedDefaultDelimiter('token_size', [])).toBe(true);
+ // Older DSLs may not carry a mode at all.
+ expect(shouldSeedDefaultDelimiter(undefined, [])).toBe(true);
+ expect(shouldSeedDefaultDelimiter(undefined, undefined)).toBe(true);
+ });
+
+ it('never seeds when the current form saved the node', () => {
+ // An explicit mode means the empty list is a deliberate user choice
+ // (pure token-size chunking) and must survive a reload.
+ expect(shouldSeedDefaultDelimiter('delimiter', [])).toBe(false);
+ expect(shouldSeedDefaultDelimiter('one', [])).toBe(false);
+ });
+
+ it('never seeds a non-empty list', () => {
+ expect(shouldSeedDefaultDelimiter('token_size', ['\n'])).toBe(false);
+ expect(shouldSeedDefaultDelimiter(undefined, [{ value: '\n' }])).toBe(
+ false,
+ );
+ });
});
describe('receiveMessageError', () => {
diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.ts
index 9aada74c9c..5f1c8c0832 100644
--- a/web/src/pages/agent/utils.ts
+++ b/web/src/pages/agent/utils.ts
@@ -345,24 +345,70 @@ export function transformParserParams(params: ParserFormSchemaType) {
});
}
+// Decides whether an empty delimiter list should be re-seeded with the
+// default '\n' row. Only legacy nodes need it: nodes saved under the removed
+// 'token_size' tab always persisted an empty list (older DSLs may not even
+// carry a mode). Nodes written by the current form have an explicit
+// 'delimiter'/'one' mode, so an empty list there is a deliberate choice —
+// pure token-size chunking, which both backends support — and must survive.
+export function shouldSeedDefaultDelimiter(
+ delimiterMode: string | undefined,
+ delimiters: unknown,
+): boolean {
+ return (
+ isEmpty(delimiters) &&
+ delimiterMode !== 'delimiter' &&
+ delimiterMode !== 'one'
+ );
+}
+
+// Migrates legacy token chunker values to the current form shape. Idempotent,
+// so every boundary (form defaults, canvas save, dataset load) can apply it:
+// - 'token_size' (removed tab) or an absent mode falls back to 'delimiter';
+// - only legacy nodes get the empty delimiter list re-seeded (the general
+// chunker never had the legacy tab, so callers pass seedLegacyDelimiter:
+// false — an empty list there is always a deliberate choice);
+// - enable_children predates the toggle: derive it from the children list.
+export function normalizeTokenChunkerFormValues<T extends Record<string, any>>(
+ values: T,
+ { seedLegacyDelimiter = true } = {},
+) {
+ return {
+ ...values,
+ delimiter_mode: values.delimiter_mode === 'one' ? 'one' : 'delimiter',
+ delimiters:
+ seedLegacyDelimiter &&
+ shouldSeedDefaultDelimiter(values.delimiter_mode, values.delimiters)
+ ? [{ value: '\n' }]
+ : (values.delimiters ?? []),
+ enable_children:
+ values.enable_children ??
+ (Array.isArray(values.children_delimiters) &&
+ values.children_delimiters.length > 0),
+ children_delimiters: values.children_delimiters ?? [],
+ };
+}
+
export function transformTokenChunkerParams(
params: TokenChunkerFormSchemaType,
+ { seedLegacyDelimiter = true } = {},
) {
- const { image_table_context_window, ...rest } = params;
+ const { image_table_context_window, ...rest } =
+ normalizeTokenChunkerFormValues(params, { seedLegacyDelimiter });
const imageTableContextWindow = Number(image_table_context_window || 0);
return {
...rest,
// Keep the configured values in 'one' mode too: the chunker ignores them
// while merging everything into a single chunk, but zeroing them here
// wiped the user's settings on every save (they reloaded as 0 / ["\n"]).
- overlapped_percent: Number(params.overlapped_percent) / 100,
- delimiters: transformObjectArrayToPureArray(params.delimiters, 'value'),
+ overlapped_percent: Number(rest.overlapped_percent) / 100,
+ delimiters: transformObjectArrayToPureArray(rest.delimiters, 'value'),
table_context_size: imageTableContextWindow,
image_context_size: imageTableContextWindow,
// Unset children delimiters if this option is not enabled
- children_delimiters: params.enable_children
- ? transformObjectArrayToPureArray(params.children_delimiters, 'value')
+ children_delimiters: rest.enable_children
+ ? transformObjectArrayToPureArray(rest.children_delimiters, 'value')
: [],
};
}
@@ -397,10 +443,14 @@ export function getChunkerChildrenDelimiterPreview(
export function transformGeneralChunkerParams(
params: TokenChunkerFormSchemaType,
) {
- const result = transformTokenChunkerParams(params);
+ // The general chunker never had the legacy 'token_size' tab, so an empty
+ // delimiter list is always a deliberate choice — skip the legacy re-seed.
+ const result = omit(
+ transformTokenChunkerParams(params, { seedLegacyDelimiter: false }),
+ ['delimiter_mode'],
+ );
result.table_context_size = Number(params.table_context_size || 0);
result.image_context_size = Number(params.image_context_size || 0);
- delete result.delimiter_mode;
return result;
}
diff --git a/web/src/utils/__tests__/pipeline-operator.test.ts b/web/src/utils/__tests__/pipeline-operator.test.ts
index b9566b1329..56fdf941c7 100644
--- a/web/src/utils/__tests__/pipeline-operator.test.ts
+++ b/web/src/utils/__tests__/pipeline-operator.test.ts
@@ -49,6 +49,78 @@ describe('GeneralChunker operator bridge', () => {
expect(api.image_context_size).toBe(3);
expect(api).not.toHaveProperty('delimiter_mode');
});
+
+ it('keeps an empty delimiter list empty (no legacy re-seed)', () => {
+ // The general chunker never had the 'token_size' tab, so an empty saved
+ // list is always a deliberate choice (pure token-size chunking).
+ const form = transformApiConfigToForm('GeneralChunker', {
+ chunk_token_size: 512,
+ delimiters: [],
+ overlapped_percent: 0,
+ });
+ expect(form.delimiters).toEqual([]);
+ });
+});
+
+describe('TokenChunker delimiter seeding on load', () => {
+ it('keeps an empty list saved by the current form empty', () => {
+ const form = transformApiConfigToForm('TokenChunker', {
+ delimiter_mode: 'delimiter',
+ chunk_token_size: 512,
+ delimiters: [],
+ overlapped_percent: 0,
+ });
+ expect(form.delimiters).toEqual([]);
+ expect(form.delimiter_mode).toBe('delimiter');
+ });
+
+ it('keeps an empty list in one mode empty', () => {
+ const form = transformApiConfigToForm('TokenChunker', {
+ delimiter_mode: 'one',
+ chunk_token_size: 512,
+ delimiters: [],
+ overlapped_percent: 0,
+ });
+ expect(form.delimiters).toEqual([]);
+ expect(form.delimiter_mode).toBe('one');
+ });
+
+ it('re-seeds the default row for legacy token_size nodes', () => {
+ // Nodes saved under the removed 'token_size' tab persisted an empty list;
+ // seed the default '\n' row so the merged 'delimiter' tab is not blank.
+ const form = transformApiConfigToForm('TokenChunker', {
+ delimiter_mode: 'token_size',
+ chunk_token_size: 512,
+ delimiters: [],
+ overlapped_percent: 0,
+ });
+ expect(form.delimiters).toEqual([{ value: '\n' }]);
+ expect(form.delimiter_mode).toBe('delimiter');
+ });
+
+ it('re-seeds the default row when the mode is absent (older DSLs)', () => {
+ const form = transformApiConfigToForm('TokenChunker', {
+ chunk_token_size: 512,
+ delimiters: [],
+ overlapped_percent: 0,
+ });
+ expect(form.delimiters).toEqual([{ value: '\n' }]);
+ expect(form.delimiter_mode).toBe('delimiter');
+ });
+
+ it('derives enable_children from a non-empty children list', () => {
+ // Configs saved before the enable_children toggle existed carry children
+ // delimiters without the flag.
+ const form = transformApiConfigToForm('TokenChunker', {
+ delimiter_mode: 'delimiter',
+ chunk_token_size: 512,
+ delimiters: ['\n'],
+ children_delimiters: ['|'],
+ overlapped_percent: 0,
+ });
+ expect(form.enable_children).toBe(true);
+ expect(form.children_delimiters).toEqual([{ value: '|' }]);
+ });
});
describe('Parser nested setups compatibility', () => {
diff --git a/web/src/utils/pipeline-operator.ts b/web/src/utils/pipeline-operator.ts
index 880b409efd..7e8ee60213 100644
--- a/web/src/utils/pipeline-operator.ts
+++ b/web/src/utils/pipeline-operator.ts
@@ -27,6 +27,7 @@ import {
initialTokenizerValues,
} from '@/pages/agent/constant/pipeline';
import {
+ normalizeTokenChunkerFormValues,
transformCompilationParams,
transformExtractorParams,
transformGeneralChunkerParams,
@@ -220,18 +221,17 @@ export function transformExtractorConfigToForm(
*/
function transformTokenChunkerConfigToForm(
config: Record<string, any> | undefined,
+ { seedLegacyDelimiter = true } = {},
): Record<string, any> {
if (!config) return {};
const result = { ...config };
- // Convert string array delimiters to object array; seed the default '\n'
- // row when the saved list is empty (legacy token_size nodes saved []).
+ // Convert string array delimiters to object arrays. The legacy re-seed and
+ // mode normalization happen in normalizeTokenChunkerFormValues below, which
+ // still sees the raw delimiter_mode.
if (Array.isArray(config.delimiters)) {
result.delimiters = config.delimiters.map((d: string) => ({ value: d }));
- if (result.delimiters.length === 0) {
- result.delimiters = [{ value: '\n' }];
- }
}
if (Array.isArray(config.children_delimiters)) {
result.children_delimiters = config.children_delimiters.map(
@@ -249,28 +249,25 @@ function transformTokenChunkerConfigToForm(
const imageSize = Number(config.image_context_size ?? 0);
result.image_table_context_window = Math.max(tableSize, imageSize);
- // Normalize delimiter_mode: the 'token_size' tab was removed from the form,
- // so legacy configs (explicit 'token_size' or absent) load as 'delimiter'.
- result.delimiter_mode = config.delimiter_mode === 'one' ? 'one' : 'delimiter';
-
- // Derive enable_children from presence of children_delimiters
- if (config.enable_children === undefined) {
- result.enable_children =
- Array.isArray(config.children_delimiters) &&
- config.children_delimiters.length > 0;
- }
+ const normalized = normalizeTokenChunkerFormValues(result, {
+ seedLegacyDelimiter,
+ });
// Clean up DSL-only fields not in form schema
- delete result.table_context_size;
- delete result.image_context_size;
+ delete normalized.table_context_size;
+ delete normalized.image_context_size;
- return result;
+ return normalized;
}
function transformGeneralChunkerConfigToForm(
config: Record<string, any> | undefined,
): Record<string, any> {
- const result = transformTokenChunkerConfigToForm(config);
+ // The general chunker never had the legacy 'token_size' tab, so an empty
+ // saved list is always a deliberate choice — skip the legacy re-seed.
+ const result = transformTokenChunkerConfigToForm(config, {
+ seedLegacyDelimiter: false,
+ });
result.table_context_size = Number(config?.table_context_size ?? 0);
result.image_context_size = Number(config?.image_context_size ?? 0);
delete result.image_table_context_window;