Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(web): explain chunker delimiter semantics in the pipeline forms (#19676)

jay777214c013d85ba72
diff --git a/web/src/components/delimiter-preview.tsx b/web/src/components/delimiter-preview.tsx
index 97fc95cd1f..a71ccfd54c 100644
--- a/web/src/components/delimiter-preview.tsx
+++ b/web/src/components/delimiter-preview.tsx
@@ -17,7 +17,10 @@
import { useTranslation } from 'react-i18next';
import { Badge } from './ui/badge';
-import { parseDelimitersForDisplay } from '@/utils/delimiter-preview';
+import {
+ type ParsedDelimiter,
+ parseDelimitersForDisplay,
+} from '@/utils/delimiter-preview';
interface Props {
/** The current value of the delimiter field. */
@@ -36,8 +39,24 @@ interface Props {
* `⇥` for tab, `␣` for space, etc.).
*/
export function DelimiterPreview({ value }: Props) {
+ return <DelimiterBadges parsed={parseDelimitersForDisplay(value)} />;
+}
+
+/**
+ * Preview of the chunker operator's delimiter LIST (one delimiter per row).
+ * The caller parses the rows for the running backend, so the badges show what
+ * that backend will actually split at.
+ */
+export function DelimiterListPreview({
+ parsed,
+}: {
+ parsed: ParsedDelimiter[];
+}) {
+ return <DelimiterBadges parsed={parsed} />;
+}
+
+function DelimiterBadges({ parsed }: { parsed: ParsedDelimiter[] }) {
const { t } = useTranslation();
- const parsed = parseDelimitersForDisplay(value);
if (parsed.length === 0) {
return (
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index daba5365fd..8ba565aab5 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -2684,6 +2684,12 @@ Best for: Documents with flowing, contextually connected content — such as boo
tab: 'Tab',
space: 'Space',
delimiters: 'Delimiters',
+ delimitersTip:
+ 'One delimiter per row; multi-character delimiters can be typed as-is (e.g. ##). With backticks (e.g. `##`): hard split — every delimiter starts its own chunk and no token-size merge is applied. Without backticks: soft split — the delimiter is only a split point, and the pieces are still merged up to the chunk token size, so short documents may show no visible change.',
+ delimitersTipPython:
+ 'One delimiter per row. Only backtick-wrapped entries (e.g. `##`) take effect: every delimiter starts its own chunk and no token-size merge is applied. Entries without backticks are ignored.',
+ childrenDelimitersTip:
+ 'Child split: each parent chunk is split again at these delimiters into child chunks used for retrieval; the chunk token size does not apply.',
one: 'One',
oneChunkTitle: 'Note',
oneChunkDescription:
diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
index ee80787056..46b7bde72f 100644
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -2295,6 +2295,12 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取 Entities 和 R
tab: '制表符',
space: '空格',
delimiters: '分隔符',
+ delimitersTip:
+ '每行一个分隔符,多字符分隔符可直接填写(如 ##)。加反引号(如 `##`):强制切分,每个分隔符处独立成块,不再按 token 大小合并;不加反引号:只作为切分点,切出的段落仍会按 chunk_token_size 合并,因此短文本可能看不出效果。',
+ delimitersTipPython:
+ '每行一个分隔符。只有用反引号包裹的条目(如 `##`)生效:每个分隔符处独立成块,不再按 token 大小合并;不加反引号的条目会被忽略。',
+ childrenDelimitersTip:
+ '子块切分:每个父块会再按这些分隔符切成子块(子块用于检索),不受 chunk_token_size 影响。',
one: 'One',
oneChunkTitle: 'Note',
oneChunkDescription:
diff --git a/web/src/pages/agent/__tests__/chunker-delimiter-hints.test.ts b/web/src/pages/agent/__tests__/chunker-delimiter-hints.test.ts
new file mode 100644
index 0000000000..d5b2072766
--- /dev/null
+++ b/web/src/pages/agent/__tests__/chunker-delimiter-hints.test.ts
@@ -0,0 +1,67 @@
+// Kept out of the co-located utils.test.ts: files containing jest.mock are
+// transformed through the babel path (see jest-esbuild-transformer.cjs), which
+// cannot handle an imported type used in a type annotation.
+import {
+ getChunkerChildrenDelimiterPreview,
+ getChunkerDelimiterPreview,
+ getChunkerDelimiterTipKey,
+} from '../utils';
+
+// Dispatch on the mutable flag so each scenario can exercise either the
+// Go or the Python branch without mounting React.
+let mockIsGoBackend = false;
+jest.mock('@/utils/backend-variant', () => ({
+ pickByBackend: ({ go, python }: { go: unknown; python: unknown }) =>
+ mockIsGoBackend ? go : python,
+}));
+
+describe('getChunkerDelimiterTipKey', () => {
+ afterEach(() => {
+ mockIsGoBackend = false;
+ });
+
+ it('points at the Go delimiter tip on the Go backend', () => {
+ mockIsGoBackend = true;
+ expect(getChunkerDelimiterTipKey()).toBe('flow.delimitersTip');
+ });
+
+ it('points at the Python delimiter tip on the Python backend', () => {
+ expect(getChunkerDelimiterTipKey()).toBe('flow.delimitersTipPython');
+ });
+});
+
+describe('getChunkerDelimiterPreview', () => {
+ afterEach(() => {
+ mockIsGoBackend = false;
+ });
+
+ it('previews every non-empty row as one delimiter on the Go backend', () => {
+ mockIsGoBackend = true;
+ expect(
+ getChunkerDelimiterPreview(['\n', '`##`']).map((d) => d.raw),
+ ).toEqual(['##', '\n']);
+ });
+
+ it('previews only backtick-wrapped rows on the Python backend', () => {
+ expect(
+ getChunkerDelimiterPreview(['\n', '`##`']).map((d) => d.raw),
+ ).toEqual(['##']);
+ });
+});
+
+describe('getChunkerChildrenDelimiterPreview', () => {
+ afterEach(() => {
+ mockIsGoBackend = false;
+ });
+
+ it('keeps bare rows on both backends', () => {
+ expect(
+ getChunkerChildrenDelimiterPreview(['\n', '`##`']).map((d) => d.raw),
+ ).toEqual(['##', '\n']);
+
+ mockIsGoBackend = true;
+ expect(
+ getChunkerChildrenDelimiterPreview(['\n', '`##`']).map((d) => d.raw),
+ ).toEqual(['##', '\n']);
+ });
+});
diff --git a/web/src/pages/agent/form/token-chunker-form/index.tsx b/web/src/pages/agent/form/token-chunker-form/index.tsx
index e8dd32ba03..03acf402d9 100644
--- a/web/src/pages/agent/form/token-chunker-form/index.tsx
+++ b/web/src/pages/agent/form/token-chunker-form/index.tsx
@@ -1,4 +1,5 @@
import { DelimiterInput } from '@/components/delimiter-form-field';
+import { DelimiterListPreview } from '@/components/delimiter-preview';
import { FormFieldType, RenderField } from '@/components/dynamic-form';
import { useSyncExternalFormErrors } from '@/components/pipeline-operator-tabs/use-sync-external-form-errors';
import { RAGFlowFormItem } from '@/components/ragflow-form';
@@ -6,11 +7,12 @@ import { SliderInputFormField } from '@/components/slider-input-form-field';
import { BlockButton, Button } from '@/components/ui/button';
import { Form } from '@/components/ui/form';
import { Switch } from '@/components/ui/switch';
+import { FormTooltip } from '@/components/ui/tooltip';
import { zodResolver } from '@hookform/resolvers/zod';
import { isEmpty } from 'lodash';
import { Info, Trash2 } from 'lucide-react';
import { memo } from 'react';
-import { useFieldArray, useForm } from 'react-hook-form';
+import { useFieldArray, useForm, useWatch } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import { z } from 'zod';
import {
@@ -21,6 +23,11 @@ import { useFormChangeCallback } from '../../hooks/use-form-change-callback';
import { useFormValues } from '../../hooks/use-form-values';
import { useWatchFormChange } from '../../hooks/use-watch-form-change';
import { INextOperatorForm } from '../../interface';
+import {
+ getChunkerChildrenDelimiterPreview,
+ getChunkerDelimiterPreview,
+ getChunkerDelimiterTipKey,
+} from '../../utils';
import { buildOutputList } from '../../utils/build-output-list';
import { FormWrapper } from '../components/form-wrapper';
import { Output } from '../components/output';
@@ -94,11 +101,22 @@ const TokenChunkerForm = ({
control: form.control,
});
+ const delimiterValues = useWatch({ control: form.control, name });
+
const childrenDelimiters = useFieldArray({
name: 'children_delimiters',
control: form.control,
});
+ const childrenDelimiterValues = useWatch({
+ control: form.control,
+ name: 'children_delimiters',
+ });
+
+ const childrenPreview = getChunkerChildrenDelimiterPreview(
+ (childrenDelimiterValues ?? []).map((delimiter) => delimiter?.value),
+ );
+
useWatchFormChange(node?.id, form);
useFormChangeCallback(form, onValuesChange);
@@ -165,7 +183,10 @@ const TokenChunkerForm = ({
/>
)}
<section>
- <span className="mb-2 inline-block">{t('flow.delimiters')}</span>
+ <span className="mb-2 inline-flex items-center">
+ {t('flow.delimiters')}
+ <FormTooltip tooltip={t(getChunkerDelimiterTipKey())} />
+ </span>
<div className="space-y-4">
{fields.map((field, index) => (
<div key={field.id} className="flex items-center gap-2">
@@ -188,6 +209,11 @@ const TokenChunkerForm = ({
</div>
))}
</div>
+ <DelimiterListPreview
+ parsed={getChunkerDelimiterPreview(
+ (delimiterValues ?? []).map((delimiter) => delimiter?.value),
+ )}
+ />
</section>
<BlockButton type="button" onClick={() => append({ value: '\n' })}>
{t('common.add')}
@@ -212,7 +238,10 @@ const TokenChunkerForm = ({
{delimiterMode !== 'one' && (
<fieldset>
<div className="mb-2 flex justify-between items-center gap-1">
- <span>{t('flow.enableChildrenDelimiters')}</span>
+ <span className="inline-flex items-center">
+ {t('flow.enableChildrenDelimiters')}
+ <FormTooltip tooltip={t('flow.childrenDelimitersTip')} />
+ </span>
<RAGFlowFormItem name="enable_children">
{(field) => (
@@ -253,6 +282,9 @@ const TokenChunkerForm = ({
>
{t('common.add')}
</BlockButton>
+ {childrenPreview.length > 0 && (
+ <DelimiterListPreview parsed={childrenPreview} />
+ )}
</div>
)}
</fieldset>
diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.ts
index 0005405f0e..77ce72caef 100644
--- a/web/src/pages/agent/utils.ts
+++ b/web/src/pages/agent/utils.ts
@@ -10,6 +10,7 @@ import {
} from '@/interfaces/database/agent';
import { pickByBackend } from '@/utils/backend-variant';
import { buildSelectOptions } from '@/utils/component-util';
+import { parseDelimiterListForDisplay } from '@/utils/delimiter-preview';
import { buildOptions, removeUselessFieldsFromValues } from '@/utils/form';
import { Edge, Node, XYPosition } from '@xyflow/react';
import { humanId } from 'human-id';
@@ -367,6 +368,33 @@ export function transformTokenChunkerParams(
};
}
+// The two backends honor the chunker's delimiter list differently: the Go
+// chunker treats a bare entry as a soft split point (the split pieces are still
+// merged up to the chunk token size) while the Python flow TokenChunker only
+// activates backtick-wrapped entries. The shared chunker form shows the tip
+// that matches the running backend.
+export function getChunkerDelimiterTipKey() {
+ return pickByBackend({
+ go: 'flow.delimitersTip',
+ python: 'flow.delimitersTipPython',
+ });
+}
+
+export function getChunkerDelimiterPreview(values: (string | undefined)[]) {
+ return pickByBackend({
+ go: parseDelimiterListForDisplay(values, { keepBare: true }),
+ python: parseDelimiterListForDisplay(values, { keepBare: false }),
+ });
+}
+
+// The child split activates every non-empty entry on both backends, so the
+// child preview never drops bare rows.
+export function getChunkerChildrenDelimiterPreview(
+ values: (string | undefined)[],
+) {
+ return parseDelimiterListForDisplay(values, { keepBare: true });
+}
+
export function transformGeneralChunkerParams(
params: TokenChunkerFormSchemaType,
) {
diff --git a/web/src/utils/__tests__/delimiter-preview.test.ts b/web/src/utils/__tests__/delimiter-preview.test.ts
index 76facd72cc..2b1d0e8d10 100644
--- a/web/src/utils/__tests__/delimiter-preview.test.ts
+++ b/web/src/utils/__tests__/delimiter-preview.test.ts
@@ -1,4 +1,7 @@
-import { parseDelimitersForDisplay } from '../delimiter-preview';
+import {
+ parseDelimiterListForDisplay,
+ parseDelimitersForDisplay,
+} from '../delimiter-preview';
describe('parseDelimitersForDisplay', () => {
it('returns empty for undefined or empty input', () => {
@@ -46,3 +49,51 @@ describe('parseDelimitersForDisplay', () => {
expect(item.display).toBe('↵');
});
});
+
+describe('parseDelimiterListForDisplay', () => {
+ it('returns empty for undefined, empty or blank rows', () => {
+ expect(parseDelimiterListForDisplay(undefined, { keepBare: true })).toEqual(
+ [],
+ );
+ expect(parseDelimiterListForDisplay([], { keepBare: true })).toEqual([]);
+ expect(
+ parseDelimiterListForDisplay(['', undefined], { keepBare: true }),
+ ).toEqual([]);
+ });
+
+ it('keeps every non-empty row as one delimiter when keepBare', () => {
+ // A multi-character row stays one delimiter — unlike the single-string
+ // field, where "##" would be two '#' delimiters.
+ const got = parseDelimiterListForDisplay(['\n', '!', '##'], {
+ keepBare: true,
+ }).map((d) => d.raw);
+ expect(got).toEqual(['##', '\n', '!']);
+ });
+
+ it('strips the backticks from a wrapped row', () => {
+ const got = parseDelimiterListForDisplay(['`##`', '`END`'], {
+ keepBare: true,
+ }).map((d) => d.raw);
+ expect(got).toEqual(['END', '##']);
+ });
+
+ it('ignores bare rows for the Python backend', () => {
+ const got = parseDelimiterListForDisplay(['\n', '!', '`##`'], {
+ keepBare: false,
+ }).map((d) => d.raw);
+ expect(got).toEqual(['##']);
+ });
+
+ it('dedupes a bare row and its backtick-wrapped twin', () => {
+ const got = parseDelimiterListForDisplay(['#', '`#`', '###'], {
+ keepBare: true,
+ }).map((d) => d.raw);
+ expect(got).toEqual(['###', '#']);
+ });
+
+ it('applies whitespace glyphs to a list row', () => {
+ const [item] = parseDelimiterListForDisplay(['\n'], { keepBare: true });
+ expect(item.raw).toBe('\n');
+ expect(item.display).toBe('↵');
+ });
+});
diff --git a/web/src/utils/delimiter-preview.ts b/web/src/utils/delimiter-preview.ts
index f86a6f2a8c..6f287d0641 100644
--- a/web/src/utils/delimiter-preview.ts
+++ b/web/src/utils/delimiter-preview.ts
@@ -95,3 +95,42 @@ export function parseDelimitersForDisplay(
// Stable sort longest-first (matches backend parse_delimiter_field).
return result.sort((a, b) => b.raw.length - a.raw.length);
}
+
+/**
+ * Parse the chunker operator's delimiter LIST (one delimiter per row) for
+ * display — a different field shape from the single delimiter string above,
+ * where each row is already one delimiter.
+ *
+ * A backtick-wrapped row contributes its inner text; any other non-empty row is
+ * one delimiter exactly as typed (multi-character included) when `keepBare` is
+ * true. `keepBare` is false for the Python flow TokenChunker, which only
+ * activates backtick-wrapped entries. Mirrors the backend list compiler:
+ * deduplicated, longest-first.
+ */
+export function parseDelimiterListForDisplay(
+ values: (string | undefined)[] | undefined,
+ { keepBare }: { keepBare: boolean },
+): ParsedDelimiter[] {
+ if (!values) return [];
+
+ const result: ParsedDelimiter[] = [];
+ const seen = new Set<string>();
+
+ const push = (raw: string) => {
+ if (!raw || seen.has(raw)) return;
+ seen.add(raw);
+ result.push({ raw, display: toDisplay(raw) });
+ };
+
+ for (const value of values) {
+ if (!value) continue;
+ if (value.length >= 2 && value.startsWith('`') && value.endsWith('`')) {
+ push(value.slice(1, -1));
+ continue;
+ }
+ if (keepBare) push(value);
+ }
+
+ // Stable sort longest-first (matches the backend list compiler).
+ return result.sort((a, b) => b.raw.length - a.raw.length);
+}