Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: Move the content of the "Token Size" tab from the token chunker operator to the "Delimiter" tab. #17888 (#17975)

balibabu398d81f587fa
diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsx
index 03dbb7e6f5..3b26171529 100644
--- a/web/src/pages/agent/constant/pipeline.tsx
+++ b/web/src/pages/agent/constant/pipeline.tsx
@@ -262,7 +262,7 @@ export const initialTokenChunkerValues = {
outputs: {
chunks: { type: 'Array<Object>', value: [] },
},
- delimiter_mode: 'token_size',
+ delimiter_mode: 'delimiter',
chunk_token_size: 512,
overlapped_percent: 0,
delimiters: [{ value: '\n' }],
diff --git a/web/src/pages/agent/form/token-chunker-form/index.tsx b/web/src/pages/agent/form/token-chunker-form/index.tsx
index b61af84747..1b17fb480c 100644
--- a/web/src/pages/agent/form/token-chunker-form/index.tsx
+++ b/web/src/pages/agent/form/token-chunker-form/index.tsx
@@ -6,6 +6,7 @@ import { BlockButton, Button } from '@/components/ui/button';
import { Form, FormControl, FormField, FormItem } from '@/components/ui/form';
import { Switch } from '@/components/ui/switch';
import { zodResolver } from '@hookform/resolvers/zod';
+import { isEmpty } from 'lodash';
import { Info, Trash2 } from 'lucide-react';
import { memo } from 'react';
import { useFieldArray, useForm } from 'react-hook-form';
@@ -37,7 +38,7 @@ export const FormSchema = z.object({
}),
),
overlapped_percent: z.number(),
- delimiter_mode: z.enum(['token_size', 'delimiter', 'one']).optional(),
+ delimiter_mode: z.enum(['delimiter', 'one']).optional(),
});
export type TokenChunkerFormSchemaType = z.infer<typeof FormSchema>;
@@ -50,9 +51,16 @@ const TokenChunkerForm = ({
const defaultValues = useFormValues(initialTokenChunkerValues, node);
const { t } = useTranslation();
+ // Normalize legacy values: 'token_size' (removed tab) and empty fall back
+ // to 'delimiter'; nodes saved in the removed tab may carry empty delimiters,
+ // so seed the default '\n' row.
const formDefaultValues = {
...defaultValues,
- delimiter_mode: defaultValues.delimiter_mode || 'token_size',
+ delimiter_mode:
+ defaultValues.delimiter_mode === 'one' ? 'one' : 'delimiter',
+ delimiters: isEmpty(defaultValues.delimiters)
+ ? [{ value: '\n' }]
+ : defaultValues.delimiters,
};
const form = useForm<TokenChunkerFormSchemaType>({
@@ -85,14 +93,13 @@ const TokenChunkerForm = ({
type: FormFieldType.Segmented,
label: '',
options: [
- { label: 'Token Size', value: 'token_size' },
{ label: t('flow.delimiters'), value: 'delimiter' },
{ label: t('flow.one'), value: 'one' },
],
}}
/>
- {delimiterMode === 'token_size' && (
+ {delimiterMode === 'delimiter' && (
<>
<SliderInputFormField
name="chunk_token_size"
@@ -113,11 +120,6 @@ const TokenChunkerForm = ({
label={t('knowledgeConfiguration.imageTableContextWindow')}
tooltip={t('knowledgeConfiguration.imageTableContextWindowTip')}
/>
- </>
- )}
-
- {delimiterMode === 'delimiter' && (
- <>
<section>
<span className="mb-2 inline-block">{t('flow.delimiters')}</span>
<div className="space-y-4">
diff --git a/web/src/utils/pipeline-operator.ts b/web/src/utils/pipeline-operator.ts
index a4305114e2..e436ca61d0 100644
--- a/web/src/utils/pipeline-operator.ts
+++ b/web/src/utils/pipeline-operator.ts
@@ -110,9 +110,13 @@ function transformTokenChunkerConfigToForm(
const result = { ...config };
- // Convert string array delimiters to object array
+ // Convert string array delimiters to object array; seed the default '\n'
+ // row when the saved list is empty (legacy token_size nodes saved []).
if (Array.isArray(config.delimiters)) {
result.delimiters = config.delimiters.map((d: string) => ({ value: d }));
+ if (result.delimiters.length === 0) {
+ result.delimiters = [{ value: '\n' }];
+ }
}
if (Array.isArray(config.children_delimiters)) {
result.children_delimiters = config.children_delimiters.map(
@@ -130,12 +134,9 @@ function transformTokenChunkerConfigToForm(
const imageSize = Number(config.image_context_size ?? 0);
result.image_table_context_window = Math.max(tableSize, imageSize);
- // Derive delimiter_mode from data
- if (config.delimiter_mode === undefined) {
- const hasDelimiters =
- Array.isArray(config.delimiters) && config.delimiters.length > 0;
- result.delimiter_mode = hasDelimiters ? 'delimiter' : 'token_size';
- }
+ // Normalize delimiter_mode: the 'token_size' tab was removed from the form,
+ // so legacy configs (explicit 'token_size' or absent) load as 'delimiter'.
+ result.delimiter_mode = config.delimiter_mode === 'one' ? 'one' : 'delimiter';
// Derive enable_children from presence of children_delimiters
if (config.enable_children === undefined) {