RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: Move the content of the "Token Size" tab from the token chunker operator to the "Delimiter" tab. #17888 (#17975)
balibabu398d81f587fa
diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsxindex 03dbb7e6f5..3b26171529 100644--- a/web/src/pages/agent/constant/pipeline.tsx+++ b/web/src/pages/agent/constant/pipeline.tsx@@ -262,7 +262,7 @@ export const initialTokenChunkerValues = {outputs: {chunks: { type: 'Array<Object>', value: [] },},- delimiter_mode: 'token_size',+ delimiter_mode: 'delimiter',chunk_token_size: 512,overlapped_percent: 0,delimiters: [{ value: '\n' }],diff --git a/web/src/pages/agent/form/token-chunker-form/index.tsx b/web/src/pages/agent/form/token-chunker-form/index.tsxindex b61af84747..1b17fb480c 100644--- a/web/src/pages/agent/form/token-chunker-form/index.tsx+++ b/web/src/pages/agent/form/token-chunker-form/index.tsx@@ -6,6 +6,7 @@ import { BlockButton, Button } from '@/components/ui/button';import { Form, FormControl, FormField, FormItem } from '@/components/ui/form';import { Switch } from '@/components/ui/switch';import { zodResolver } from '@hookform/resolvers/zod';+import { isEmpty } from 'lodash';import { Info, Trash2 } from 'lucide-react';import { memo } from 'react';import { useFieldArray, useForm } from 'react-hook-form';@@ -37,7 +38,7 @@ export const FormSchema = z.object({}),),overlapped_percent: z.number(),- delimiter_mode: z.enum(['token_size', 'delimiter', 'one']).optional(),+ delimiter_mode: z.enum(['delimiter', 'one']).optional(),});export type TokenChunkerFormSchemaType = z.infer<typeof FormSchema>;@@ -50,9 +51,16 @@ const TokenChunkerForm = ({const defaultValues = useFormValues(initialTokenChunkerValues, node);const { t } = useTranslation();+ // Normalize legacy values: 'token_size' (removed tab) and empty fall back+ // to 'delimiter'; nodes saved in the removed tab may carry empty delimiters,+ // so seed the default '\n' row.const formDefaultValues = {...defaultValues,- delimiter_mode: defaultValues.delimiter_mode || 'token_size',+ delimiter_mode:+ defaultValues.delimiter_mode === 'one' ? 'one' : 'delimiter',+ delimiters: isEmpty(defaultValues.delimiters)+ ? [{ value: '\n' }]+ : defaultValues.delimiters,};const form = useForm<TokenChunkerFormSchemaType>({@@ -85,14 +93,13 @@ const TokenChunkerForm = ({type: FormFieldType.Segmented,label: '',options: [- { label: 'Token Size', value: 'token_size' },{ label: t('flow.delimiters'), value: 'delimiter' },{ label: t('flow.one'), value: 'one' },],}}/>- {delimiterMode === 'token_size' && (+ {delimiterMode === 'delimiter' && (<><SliderInputFormFieldname="chunk_token_size"@@ -113,11 +120,6 @@ const TokenChunkerForm = ({label={t('knowledgeConfiguration.imageTableContextWindow')}tooltip={t('knowledgeConfiguration.imageTableContextWindowTip')}/>- </>- )}-- {delimiterMode === 'delimiter' && (- <><section><span className="mb-2 inline-block">{t('flow.delimiters')}</span><div className="space-y-4">diff --git a/web/src/utils/pipeline-operator.ts b/web/src/utils/pipeline-operator.tsindex a4305114e2..e436ca61d0 100644--- a/web/src/utils/pipeline-operator.ts+++ b/web/src/utils/pipeline-operator.ts@@ -110,9 +110,13 @@ function transformTokenChunkerConfigToForm(const result = { ...config };- // Convert string array delimiters to object array+ // Convert string array delimiters to object array; seed the default '\n'+ // row when the saved list is empty (legacy token_size nodes saved []).if (Array.isArray(config.delimiters)) {result.delimiters = config.delimiters.map((d: string) => ({ value: d }));+ if (result.delimiters.length === 0) {+ result.delimiters = [{ value: '\n' }];+ }}if (Array.isArray(config.children_delimiters)) {result.children_delimiters = config.children_delimiters.map(@@ -130,12 +134,9 @@ function transformTokenChunkerConfigToForm(const imageSize = Number(config.image_context_size ?? 0);result.image_table_context_window = Math.max(tableSize, imageSize);- // Derive delimiter_mode from data- if (config.delimiter_mode === undefined) {- const hasDelimiters =- Array.isArray(config.delimiters) && config.delimiters.length > 0;- result.delimiter_mode = hasDelimiters ? 'delimiter' : 'token_size';- }+ // Normalize delimiter_mode: the 'token_size' tab was removed from the form,+ // so legacy configs (explicit 'token_size' or absent) load as 'delimiter'.+ result.delimiter_mode = config.delimiter_mode === 'one' ? 'one' : 'delimiter';// Derive enable_children from presence of children_delimitersif (config.enable_children === undefined) {