RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: Unable to delete the last item in the token chunker's delimiter configuration. (#20376)
balibabu1eb3307ce247
diff --git a/web/src/pages/agent/form/token-chunker-form/index.tsx b/web/src/pages/agent/form/token-chunker-form/index.tsxindex 03acf402d9..974e249f98 100644--- a/web/src/pages/agent/form/token-chunker-form/index.tsx+++ b/web/src/pages/agent/form/token-chunker-form/index.tsx@@ -9,7 +9,6 @@ import { Form } from '@/components/ui/form';import { Switch } from '@/components/ui/switch';import { FormTooltip } from '@/components/ui/tooltip';import { zodResolver } from '@hookform/resolvers/zod';-import { isEmpty } from 'lodash';import { Info, Trash2 } from 'lucide-react';import { memo } from 'react';import { useFieldArray, useForm, useWatch } from 'react-hook-form';@@ -27,6 +26,7 @@ import {getChunkerChildrenDelimiterPreview,getChunkerDelimiterPreview,getChunkerDelimiterTipKey,+ normalizeTokenChunkerFormValues,} from '../../utils';import { buildOutputList } from '../../utils/build-output-list';import { FormWrapper } from '../components/form-wrapper';@@ -73,17 +73,11 @@ const TokenChunkerForm = ({const defaultValues = useFormValues(initialValues, node);const { t } = useTranslation();- // Normalize legacy values: 'token_size' (removed tab) and empty fall back- // to 'delimiter'; nodes saved in the removed tab may carry empty delimiters,- // so seed the default '\n' row.- const formDefaultValues = {- ...defaultValues,- delimiter_mode:- defaultValues.delimiter_mode === 'one' ? 'one' : 'delimiter',- delimiters: isEmpty(defaultValues.delimiters)- ? [{ value: '\n' }]- : defaultValues.delimiters,- };+ // The general chunker never had the legacy 'token_size' tab, so it never+ // re-seeds an empty delimiter list.+ const formDefaultValues = normalizeTokenChunkerFormValues(defaultValues, {+ seedLegacyDelimiter: !isGeneralChunker,+ });const form = useForm<TokenChunkerFormSchemaType>({defaultValues: formDefaultValues,diff --git a/web/src/pages/agent/utils.test.ts b/web/src/pages/agent/utils.test.tsindex d9953034be..bf6057590f 100644--- a/web/src/pages/agent/utils.test.ts+++ b/web/src/pages/agent/utils.test.ts@@ -3,7 +3,10 @@ import { Operator } from './constant';import {generateNodeNamesWithIncreasingIndex,isEmptyMessageContent,+ normalizeTokenChunkerFormValues,receiveMessageError,+ shouldSeedDefaultDelimiter,+ transformGeneralChunkerParams,transformTokenChunkerParams,} from './utils';@@ -43,6 +46,153 @@ describe('transformTokenChunkerParams', () => {expect(result.table_context_size).toBe(81);expect(result.image_context_size).toBe(81);});++ it('emits an explicit empty delimiters list when the user removed all rows', () => {+ // The backend falls back to its own default (["\n"]) when the key is+ // absent, so the save path must emit [] to keep pure token-size chunking.+ const result = transformTokenChunkerParams({+ delimiter_mode: 'delimiter',+ chunk_token_size: 512,+ overlapped_percent: 0,+ image_table_context_window: 0,+ delimiters: [],+ children_delimiters: [],+ enable_children: false,+ } as any);++ expect(result.delimiters).toEqual([]);+ });++ it('migrates a legacy node on save even when the form was never opened', () => {+ // Nodes saved under the removed 'token_size' tab persisted an empty list;+ // the save path applies the same migration the form would.+ const result = transformTokenChunkerParams({+ delimiter_mode: 'token_size',+ chunk_token_size: 512,+ overlapped_percent: 0,+ image_table_context_window: 0,+ delimiters: [],+ children_delimiters: [],+ } as any);++ expect(result.delimiter_mode).toBe('delimiter');+ expect(result.delimiters).toEqual(['\n']);+ });++ it('keeps legacy children delimiters instead of wiping them', () => {+ // Nodes saved before the enable_children toggle existed carry children+ // delimiters without the flag; derive it instead of dropping the values.+ const result = transformTokenChunkerParams({+ delimiter_mode: 'delimiter',+ chunk_token_size: 512,+ overlapped_percent: 0,+ image_table_context_window: 0,+ delimiters: [{ value: '\n' }],+ children_delimiters: [{ value: '|' }],+ } as any);++ expect(result.enable_children).toBe(true);+ expect(result.children_delimiters).toEqual(['|']);+ });+});++describe('transformGeneralChunkerParams', () => {+ it('never re-seeds the general chunker delimiter list on save', () => {+ const result = transformGeneralChunkerParams({+ chunk_token_size: 512,+ overlapped_percent: 0,+ delimiters: [],+ children_delimiters: [],+ enable_children: false,+ table_context_size: 0,+ image_context_size: 0,+ } as any);++ expect(result.delimiters).toEqual([]);+ expect(result).not.toHaveProperty('delimiter_mode');+ });+});++describe('normalizeTokenChunkerFormValues', () => {+ it('normalizes a legacy token_size node and re-seeds the delimiter row', () => {+ const result = normalizeTokenChunkerFormValues({+ delimiter_mode: 'token_size',+ delimiters: [],+ children_delimiters: [],+ });++ expect(result.delimiter_mode).toBe('delimiter');+ expect(result.delimiters).toEqual([{ value: '\n' }]);+ });++ it('keeps a deliberate empty list saved by the current form', () => {+ const result = normalizeTokenChunkerFormValues({+ delimiter_mode: 'one',+ delimiters: [],+ });++ expect(result.delimiter_mode).toBe('one');+ expect(result.delimiters).toEqual([]);+ });++ it('derives enable_children from a non-empty children list', () => {+ const result = normalizeTokenChunkerFormValues({+ children_delimiters: [{ value: '\n\n' }],+ });++ expect(result.enable_children).toBe(true);+ });++ it('respects an explicit enable_children flag over derivation', () => {+ const result = normalizeTokenChunkerFormValues({+ enable_children: false,+ children_delimiters: [{ value: '\n\n' }],+ });++ expect(result.enable_children).toBe(false);+ });++ it('is idempotent', () => {+ const once = normalizeTokenChunkerFormValues({+ delimiters: [],+ children_delimiters: [{ value: '|' }],+ });++ expect(normalizeTokenChunkerFormValues(once)).toEqual(once);+ });++ it('skips the legacy re-seed when seedLegacyDelimiter is false', () => {+ const result = normalizeTokenChunkerFormValues(+ { delimiters: [] },+ { seedLegacyDelimiter: false },+ );++ expect(result.delimiters).toEqual([]);+ });+});++describe('shouldSeedDefaultDelimiter', () => {+ it('seeds only for legacy nodes with an empty list', () => {+ // Legacy 'token_size' tab nodes always persisted an empty list.+ expect(shouldSeedDefaultDelimiter('token_size', [])).toBe(true);+ // Older DSLs may not carry a mode at all.+ expect(shouldSeedDefaultDelimiter(undefined, [])).toBe(true);+ expect(shouldSeedDefaultDelimiter(undefined, undefined)).toBe(true);+ });++ it('never seeds when the current form saved the node', () => {+ // An explicit mode means the empty list is a deliberate user choice+ // (pure token-size chunking) and must survive a reload.+ expect(shouldSeedDefaultDelimiter('delimiter', [])).toBe(false);+ expect(shouldSeedDefaultDelimiter('one', [])).toBe(false);+ });++ it('never seeds a non-empty list', () => {+ expect(shouldSeedDefaultDelimiter('token_size', ['\n'])).toBe(false);+ expect(shouldSeedDefaultDelimiter(undefined, [{ value: '\n' }])).toBe(+ false,+ );+ });});describe('receiveMessageError', () => {diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.tsindex 9aada74c9c..5f1c8c0832 100644--- a/web/src/pages/agent/utils.ts+++ b/web/src/pages/agent/utils.ts@@ -345,24 +345,70 @@ export function transformParserParams(params: ParserFormSchemaType) {});}+// Decides whether an empty delimiter list should be re-seeded with the+// default '\n' row. Only legacy nodes need it: nodes saved under the removed+// 'token_size' tab always persisted an empty list (older DSLs may not even+// carry a mode). Nodes written by the current form have an explicit+// 'delimiter'/'one' mode, so an empty list there is a deliberate choice —+// pure token-size chunking, which both backends support — and must survive.+export function shouldSeedDefaultDelimiter(+ delimiterMode: string | undefined,+ delimiters: unknown,+): boolean {+ return (+ isEmpty(delimiters) &&+ delimiterMode !== 'delimiter' &&+ delimiterMode !== 'one'+ );+}++// Migrates legacy token chunker values to the current form shape. Idempotent,+// so every boundary (form defaults, canvas save, dataset load) can apply it:+// - 'token_size' (removed tab) or an absent mode falls back to 'delimiter';+// - only legacy nodes get the empty delimiter list re-seeded (the general+// chunker never had the legacy tab, so callers pass seedLegacyDelimiter:+// false — an empty list there is always a deliberate choice);+// - enable_children predates the toggle: derive it from the children list.+export function normalizeTokenChunkerFormValues<T extends Record<string, any>>(+ values: T,+ { seedLegacyDelimiter = true } = {},+) {+ return {+ ...values,+ delimiter_mode: values.delimiter_mode === 'one' ? 'one' : 'delimiter',+ delimiters:+ seedLegacyDelimiter &&+ shouldSeedDefaultDelimiter(values.delimiter_mode, values.delimiters)+ ? [{ value: '\n' }]+ : (values.delimiters ?? []),+ enable_children:+ values.enable_children ??+ (Array.isArray(values.children_delimiters) &&+ values.children_delimiters.length > 0),+ children_delimiters: values.children_delimiters ?? [],+ };+}+export function transformTokenChunkerParams(params: TokenChunkerFormSchemaType,+ { seedLegacyDelimiter = true } = {},) {- const { image_table_context_window, ...rest } = params;+ const { image_table_context_window, ...rest } =+ normalizeTokenChunkerFormValues(params, { seedLegacyDelimiter });const imageTableContextWindow = Number(image_table_context_window || 0);return {...rest,// Keep the configured values in 'one' mode too: the chunker ignores them// while merging everything into a single chunk, but zeroing them here// wiped the user's settings on every save (they reloaded as 0 / ["\n"]).- overlapped_percent: Number(params.overlapped_percent) / 100,- delimiters: transformObjectArrayToPureArray(params.delimiters, 'value'),+ overlapped_percent: Number(rest.overlapped_percent) / 100,+ delimiters: transformObjectArrayToPureArray(rest.delimiters, 'value'),table_context_size: imageTableContextWindow,image_context_size: imageTableContextWindow,// Unset children delimiters if this option is not enabled- children_delimiters: params.enable_children- ? transformObjectArrayToPureArray(params.children_delimiters, 'value')+ children_delimiters: rest.enable_children+ ? transformObjectArrayToPureArray(rest.children_delimiters, 'value'): [],};}@@ -397,10 +443,14 @@ export function getChunkerChildrenDelimiterPreview(export function transformGeneralChunkerParams(params: TokenChunkerFormSchemaType,) {- const result = transformTokenChunkerParams(params);+ // The general chunker never had the legacy 'token_size' tab, so an empty+ // delimiter list is always a deliberate choice — skip the legacy re-seed.+ const result = omit(+ transformTokenChunkerParams(params, { seedLegacyDelimiter: false }),+ ['delimiter_mode'],+ );result.table_context_size = Number(params.table_context_size || 0);result.image_context_size = Number(params.image_context_size || 0);- delete result.delimiter_mode;return result;}diff --git a/web/src/utils/__tests__/pipeline-operator.test.ts b/web/src/utils/__tests__/pipeline-operator.test.tsindex b9566b1329..56fdf941c7 100644--- a/web/src/utils/__tests__/pipeline-operator.test.ts+++ b/web/src/utils/__tests__/pipeline-operator.test.ts@@ -49,6 +49,78 @@ describe('GeneralChunker operator bridge', () => {expect(api.image_context_size).toBe(3);expect(api).not.toHaveProperty('delimiter_mode');});++ it('keeps an empty delimiter list empty (no legacy re-seed)', () => {+ // The general chunker never had the 'token_size' tab, so an empty saved+ // list is always a deliberate choice (pure token-size chunking).+ const form = transformApiConfigToForm('GeneralChunker', {+ chunk_token_size: 512,+ delimiters: [],+ overlapped_percent: 0,+ });+ expect(form.delimiters).toEqual([]);+ });+});++describe('TokenChunker delimiter seeding on load', () => {+ it('keeps an empty list saved by the current form empty', () => {+ const form = transformApiConfigToForm('TokenChunker', {+ delimiter_mode: 'delimiter',+ chunk_token_size: 512,+ delimiters: [],+ overlapped_percent: 0,+ });+ expect(form.delimiters).toEqual([]);+ expect(form.delimiter_mode).toBe('delimiter');+ });++ it('keeps an empty list in one mode empty', () => {+ const form = transformApiConfigToForm('TokenChunker', {+ delimiter_mode: 'one',+ chunk_token_size: 512,+ delimiters: [],+ overlapped_percent: 0,+ });+ expect(form.delimiters).toEqual([]);+ expect(form.delimiter_mode).toBe('one');+ });++ it('re-seeds the default row for legacy token_size nodes', () => {+ // Nodes saved under the removed 'token_size' tab persisted an empty list;+ // seed the default '\n' row so the merged 'delimiter' tab is not blank.+ const form = transformApiConfigToForm('TokenChunker', {+ delimiter_mode: 'token_size',+ chunk_token_size: 512,+ delimiters: [],+ overlapped_percent: 0,+ });+ expect(form.delimiters).toEqual([{ value: '\n' }]);+ expect(form.delimiter_mode).toBe('delimiter');+ });++ it('re-seeds the default row when the mode is absent (older DSLs)', () => {+ const form = transformApiConfigToForm('TokenChunker', {+ chunk_token_size: 512,+ delimiters: [],+ overlapped_percent: 0,+ });+ expect(form.delimiters).toEqual([{ value: '\n' }]);+ expect(form.delimiter_mode).toBe('delimiter');+ });++ it('derives enable_children from a non-empty children list', () => {+ // Configs saved before the enable_children toggle existed carry children+ // delimiters without the flag.+ const form = transformApiConfigToForm('TokenChunker', {+ delimiter_mode: 'delimiter',+ chunk_token_size: 512,+ delimiters: ['\n'],+ children_delimiters: ['|'],+ overlapped_percent: 0,+ });+ expect(form.enable_children).toBe(true);+ expect(form.children_delimiters).toEqual([{ value: '|' }]);+ });});describe('Parser nested setups compatibility', () => {diff --git a/web/src/utils/pipeline-operator.ts b/web/src/utils/pipeline-operator.tsindex 880b409efd..7e8ee60213 100644--- a/web/src/utils/pipeline-operator.ts+++ b/web/src/utils/pipeline-operator.ts@@ -27,6 +27,7 @@ import {initialTokenizerValues,} from '@/pages/agent/constant/pipeline';import {+ normalizeTokenChunkerFormValues,transformCompilationParams,transformExtractorParams,transformGeneralChunkerParams,@@ -220,18 +221,17 @@ export function transformExtractorConfigToForm(*/function transformTokenChunkerConfigToForm(config: Record<string, any> | undefined,+ { seedLegacyDelimiter = true } = {},): Record<string, any> {if (!config) return {};const result = { ...config };- // Convert string array delimiters to object array; seed the default '\n'- // row when the saved list is empty (legacy token_size nodes saved []).+ // Convert string array delimiters to object arrays. The legacy re-seed and+ // mode normalization happen in normalizeTokenChunkerFormValues below, which+ // still sees the raw delimiter_mode.if (Array.isArray(config.delimiters)) {result.delimiters = config.delimiters.map((d: string) => ({ value: d }));- if (result.delimiters.length === 0) {- result.delimiters = [{ value: '\n' }];- }}if (Array.isArray(config.children_delimiters)) {result.children_delimiters = config.children_delimiters.map(@@ -249,28 +249,25 @@ function transformTokenChunkerConfigToForm(const imageSize = Number(config.image_context_size ?? 0);result.image_table_context_window = Math.max(tableSize, imageSize);- // Normalize delimiter_mode: the 'token_size' tab was removed from the form,- // so legacy configs (explicit 'token_size' or absent) load as 'delimiter'.- result.delimiter_mode = config.delimiter_mode === 'one' ? 'one' : 'delimiter';-- // Derive enable_children from presence of children_delimiters- if (config.enable_children === undefined) {- result.enable_children =- Array.isArray(config.children_delimiters) &&- config.children_delimiters.length > 0;- }+ const normalized = normalizeTokenChunkerFormValues(result, {+ seedLegacyDelimiter,+ });// Clean up DSL-only fields not in form schema- delete result.table_context_size;- delete result.image_context_size;+ delete normalized.table_context_size;+ delete normalized.image_context_size;- return result;+ return normalized;}function transformGeneralChunkerConfigToForm(config: Record<string, any> | undefined,): Record<string, any> {- const result = transformTokenChunkerConfigToForm(config);+ // The general chunker never had the legacy 'token_size' tab, so an empty+ // saved list is always a deliberate choice — skip the legacy re-seed.+ const result = transformTokenChunkerConfigToForm(config, {+ seedLegacyDelimiter: false,+ });result.table_context_size = Number(config?.table_context_size ?? 0);result.image_context_size = Number(config?.image_context_size ?? 0);delete result.image_table_context_window;