RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(web): explain chunker delimiter semantics in the pipeline forms (#19676)
jay777214c013d85ba72
diff --git a/web/src/components/delimiter-preview.tsx b/web/src/components/delimiter-preview.tsxindex 97fc95cd1f..a71ccfd54c 100644--- a/web/src/components/delimiter-preview.tsx+++ b/web/src/components/delimiter-preview.tsx@@ -17,7 +17,10 @@import { useTranslation } from 'react-i18next';import { Badge } from './ui/badge';-import { parseDelimitersForDisplay } from '@/utils/delimiter-preview';+import {+ type ParsedDelimiter,+ parseDelimitersForDisplay,+} from '@/utils/delimiter-preview';interface Props {/** The current value of the delimiter field. */@@ -36,8 +39,24 @@ interface Props {* `⇥` for tab, `␣` for space, etc.).*/export function DelimiterPreview({ value }: Props) {+ return <DelimiterBadges parsed={parseDelimitersForDisplay(value)} />;+}++/**+ * Preview of the chunker operator's delimiter LIST (one delimiter per row).+ * The caller parses the rows for the running backend, so the badges show what+ * that backend will actually split at.+ */+export function DelimiterListPreview({+ parsed,+}: {+ parsed: ParsedDelimiter[];+}) {+ return <DelimiterBadges parsed={parsed} />;+}++function DelimiterBadges({ parsed }: { parsed: ParsedDelimiter[] }) {const { t } = useTranslation();- const parsed = parseDelimitersForDisplay(value);if (parsed.length === 0) {return (diff --git a/web/src/locales/en.ts b/web/src/locales/en.tsindex daba5365fd..8ba565aab5 100644--- a/web/src/locales/en.ts+++ b/web/src/locales/en.ts@@ -2684,6 +2684,12 @@ Best for: Documents with flowing, contextually connected content — such as bootab: 'Tab',space: 'Space',delimiters: 'Delimiters',+ delimitersTip:+ 'One delimiter per row; multi-character delimiters can be typed as-is (e.g. ##). With backticks (e.g. `##`): hard split — every delimiter starts its own chunk and no token-size merge is applied. Without backticks: soft split — the delimiter is only a split point, and the pieces are still merged up to the chunk token size, so short documents may show no visible change.',+ delimitersTipPython:+ 'One delimiter per row. Only backtick-wrapped entries (e.g. `##`) take effect: every delimiter starts its own chunk and no token-size merge is applied. Entries without backticks are ignored.',+ childrenDelimitersTip:+ 'Child split: each parent chunk is split again at these delimiters into child chunks used for retrieval; the chunk token size does not apply.',one: 'One',oneChunkTitle: 'Note',oneChunkDescription:diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.tsindex ee80787056..46b7bde72f 100644--- a/web/src/locales/zh.ts+++ b/web/src/locales/zh.ts@@ -2295,6 +2295,12 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取 Entities 和 Rtab: '制表符',space: '空格',delimiters: '分隔符',+ delimitersTip:+ '每行一个分隔符,多字符分隔符可直接填写(如 ##)。加反引号(如 `##`):强制切分,每个分隔符处独立成块,不再按 token 大小合并;不加反引号:只作为切分点,切出的段落仍会按 chunk_token_size 合并,因此短文本可能看不出效果。',+ delimitersTipPython:+ '每行一个分隔符。只有用反引号包裹的条目(如 `##`)生效:每个分隔符处独立成块,不再按 token 大小合并;不加反引号的条目会被忽略。',+ childrenDelimitersTip:+ '子块切分:每个父块会再按这些分隔符切成子块(子块用于检索),不受 chunk_token_size 影响。',one: 'One',oneChunkTitle: 'Note',oneChunkDescription:diff --git a/web/src/pages/agent/__tests__/chunker-delimiter-hints.test.ts b/web/src/pages/agent/__tests__/chunker-delimiter-hints.test.tsnew file mode 100644index 0000000000..d5b2072766--- /dev/null+++ b/web/src/pages/agent/__tests__/chunker-delimiter-hints.test.ts@@ -0,0 +1,67 @@+// Kept out of the co-located utils.test.ts: files containing jest.mock are+// transformed through the babel path (see jest-esbuild-transformer.cjs), which+// cannot handle an imported type used in a type annotation.+import {+ getChunkerChildrenDelimiterPreview,+ getChunkerDelimiterPreview,+ getChunkerDelimiterTipKey,+} from '../utils';++// Dispatch on the mutable flag so each scenario can exercise either the+// Go or the Python branch without mounting React.+let mockIsGoBackend = false;+jest.mock('@/utils/backend-variant', () => ({+ pickByBackend: ({ go, python }: { go: unknown; python: unknown }) =>+ mockIsGoBackend ? go : python,+}));++describe('getChunkerDelimiterTipKey', () => {+ afterEach(() => {+ mockIsGoBackend = false;+ });++ it('points at the Go delimiter tip on the Go backend', () => {+ mockIsGoBackend = true;+ expect(getChunkerDelimiterTipKey()).toBe('flow.delimitersTip');+ });++ it('points at the Python delimiter tip on the Python backend', () => {+ expect(getChunkerDelimiterTipKey()).toBe('flow.delimitersTipPython');+ });+});++describe('getChunkerDelimiterPreview', () => {+ afterEach(() => {+ mockIsGoBackend = false;+ });++ it('previews every non-empty row as one delimiter on the Go backend', () => {+ mockIsGoBackend = true;+ expect(+ getChunkerDelimiterPreview(['\n', '`##`']).map((d) => d.raw),+ ).toEqual(['##', '\n']);+ });++ it('previews only backtick-wrapped rows on the Python backend', () => {+ expect(+ getChunkerDelimiterPreview(['\n', '`##`']).map((d) => d.raw),+ ).toEqual(['##']);+ });+});++describe('getChunkerChildrenDelimiterPreview', () => {+ afterEach(() => {+ mockIsGoBackend = false;+ });++ it('keeps bare rows on both backends', () => {+ expect(+ getChunkerChildrenDelimiterPreview(['\n', '`##`']).map((d) => d.raw),+ ).toEqual(['##', '\n']);++ mockIsGoBackend = true;+ expect(+ getChunkerChildrenDelimiterPreview(['\n', '`##`']).map((d) => d.raw),+ ).toEqual(['##', '\n']);+ });+});diff --git a/web/src/pages/agent/form/token-chunker-form/index.tsx b/web/src/pages/agent/form/token-chunker-form/index.tsxindex e8dd32ba03..03acf402d9 100644--- a/web/src/pages/agent/form/token-chunker-form/index.tsx+++ b/web/src/pages/agent/form/token-chunker-form/index.tsx@@ -1,4 +1,5 @@import { DelimiterInput } from '@/components/delimiter-form-field';+import { DelimiterListPreview } from '@/components/delimiter-preview';import { FormFieldType, RenderField } from '@/components/dynamic-form';import { useSyncExternalFormErrors } from '@/components/pipeline-operator-tabs/use-sync-external-form-errors';import { RAGFlowFormItem } from '@/components/ragflow-form';@@ -6,11 +7,12 @@ import { SliderInputFormField } from '@/components/slider-input-form-field';import { BlockButton, Button } from '@/components/ui/button';import { Form } from '@/components/ui/form';import { Switch } from '@/components/ui/switch';+import { FormTooltip } from '@/components/ui/tooltip';import { zodResolver } from '@hookform/resolvers/zod';import { isEmpty } from 'lodash';import { Info, Trash2 } from 'lucide-react';import { memo } from 'react';-import { useFieldArray, useForm } from 'react-hook-form';+import { useFieldArray, useForm, useWatch } from 'react-hook-form';import { useTranslation } from 'react-i18next';import { z } from 'zod';import {@@ -21,6 +23,11 @@ import { useFormChangeCallback } from '../../hooks/use-form-change-callback';import { useFormValues } from '../../hooks/use-form-values';import { useWatchFormChange } from '../../hooks/use-watch-form-change';import { INextOperatorForm } from '../../interface';+import {+ getChunkerChildrenDelimiterPreview,+ getChunkerDelimiterPreview,+ getChunkerDelimiterTipKey,+} from '../../utils';import { buildOutputList } from '../../utils/build-output-list';import { FormWrapper } from '../components/form-wrapper';import { Output } from '../components/output';@@ -94,11 +101,22 @@ const TokenChunkerForm = ({control: form.control,});+ const delimiterValues = useWatch({ control: form.control, name });+const childrenDelimiters = useFieldArray({name: 'children_delimiters',control: form.control,});+ const childrenDelimiterValues = useWatch({+ control: form.control,+ name: 'children_delimiters',+ });++ const childrenPreview = getChunkerChildrenDelimiterPreview(+ (childrenDelimiterValues ?? []).map((delimiter) => delimiter?.value),+ );+useWatchFormChange(node?.id, form);useFormChangeCallback(form, onValuesChange);@@ -165,7 +183,10 @@ const TokenChunkerForm = ({/>)}<section>- <span className="mb-2 inline-block">{t('flow.delimiters')}</span>+ <span className="mb-2 inline-flex items-center">+ {t('flow.delimiters')}+ <FormTooltip tooltip={t(getChunkerDelimiterTipKey())} />+ </span><div className="space-y-4">{fields.map((field, index) => (<div key={field.id} className="flex items-center gap-2">@@ -188,6 +209,11 @@ const TokenChunkerForm = ({</div>))}</div>+ <DelimiterListPreview+ parsed={getChunkerDelimiterPreview(+ (delimiterValues ?? []).map((delimiter) => delimiter?.value),+ )}+ /></section><BlockButton type="button" onClick={() => append({ value: '\n' })}>{t('common.add')}@@ -212,7 +238,10 @@ const TokenChunkerForm = ({{delimiterMode !== 'one' && (<fieldset><div className="mb-2 flex justify-between items-center gap-1">- <span>{t('flow.enableChildrenDelimiters')}</span>+ <span className="inline-flex items-center">+ {t('flow.enableChildrenDelimiters')}+ <FormTooltip tooltip={t('flow.childrenDelimitersTip')} />+ </span><RAGFlowFormItem name="enable_children">{(field) => (@@ -253,6 +282,9 @@ const TokenChunkerForm = ({>{t('common.add')}</BlockButton>+ {childrenPreview.length > 0 && (+ <DelimiterListPreview parsed={childrenPreview} />+ )}</div>)}</fieldset>diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.tsindex 0005405f0e..77ce72caef 100644--- a/web/src/pages/agent/utils.ts+++ b/web/src/pages/agent/utils.ts@@ -10,6 +10,7 @@ import {} from '@/interfaces/database/agent';import { pickByBackend } from '@/utils/backend-variant';import { buildSelectOptions } from '@/utils/component-util';+import { parseDelimiterListForDisplay } from '@/utils/delimiter-preview';import { buildOptions, removeUselessFieldsFromValues } from '@/utils/form';import { Edge, Node, XYPosition } from '@xyflow/react';import { humanId } from 'human-id';@@ -367,6 +368,33 @@ export function transformTokenChunkerParams(};}+// The two backends honor the chunker's delimiter list differently: the Go+// chunker treats a bare entry as a soft split point (the split pieces are still+// merged up to the chunk token size) while the Python flow TokenChunker only+// activates backtick-wrapped entries. The shared chunker form shows the tip+// that matches the running backend.+export function getChunkerDelimiterTipKey() {+ return pickByBackend({+ go: 'flow.delimitersTip',+ python: 'flow.delimitersTipPython',+ });+}++export function getChunkerDelimiterPreview(values: (string | undefined)[]) {+ return pickByBackend({+ go: parseDelimiterListForDisplay(values, { keepBare: true }),+ python: parseDelimiterListForDisplay(values, { keepBare: false }),+ });+}++// The child split activates every non-empty entry on both backends, so the+// child preview never drops bare rows.+export function getChunkerChildrenDelimiterPreview(+ values: (string | undefined)[],+) {+ return parseDelimiterListForDisplay(values, { keepBare: true });+}+export function transformGeneralChunkerParams(params: TokenChunkerFormSchemaType,) {diff --git a/web/src/utils/__tests__/delimiter-preview.test.ts b/web/src/utils/__tests__/delimiter-preview.test.tsindex 76facd72cc..2b1d0e8d10 100644--- a/web/src/utils/__tests__/delimiter-preview.test.ts+++ b/web/src/utils/__tests__/delimiter-preview.test.ts@@ -1,4 +1,7 @@-import { parseDelimitersForDisplay } from '../delimiter-preview';+import {+ parseDelimiterListForDisplay,+ parseDelimitersForDisplay,+} from '../delimiter-preview';describe('parseDelimitersForDisplay', () => {it('returns empty for undefined or empty input', () => {@@ -46,3 +49,51 @@ describe('parseDelimitersForDisplay', () => {expect(item.display).toBe('↵');});});++describe('parseDelimiterListForDisplay', () => {+ it('returns empty for undefined, empty or blank rows', () => {+ expect(parseDelimiterListForDisplay(undefined, { keepBare: true })).toEqual(+ [],+ );+ expect(parseDelimiterListForDisplay([], { keepBare: true })).toEqual([]);+ expect(+ parseDelimiterListForDisplay(['', undefined], { keepBare: true }),+ ).toEqual([]);+ });++ it('keeps every non-empty row as one delimiter when keepBare', () => {+ // A multi-character row stays one delimiter — unlike the single-string+ // field, where "##" would be two '#' delimiters.+ const got = parseDelimiterListForDisplay(['\n', '!', '##'], {+ keepBare: true,+ }).map((d) => d.raw);+ expect(got).toEqual(['##', '\n', '!']);+ });++ it('strips the backticks from a wrapped row', () => {+ const got = parseDelimiterListForDisplay(['`##`', '`END`'], {+ keepBare: true,+ }).map((d) => d.raw);+ expect(got).toEqual(['END', '##']);+ });++ it('ignores bare rows for the Python backend', () => {+ const got = parseDelimiterListForDisplay(['\n', '!', '`##`'], {+ keepBare: false,+ }).map((d) => d.raw);+ expect(got).toEqual(['##']);+ });++ it('dedupes a bare row and its backtick-wrapped twin', () => {+ const got = parseDelimiterListForDisplay(['#', '`#`', '###'], {+ keepBare: true,+ }).map((d) => d.raw);+ expect(got).toEqual(['###', '#']);+ });++ it('applies whitespace glyphs to a list row', () => {+ const [item] = parseDelimiterListForDisplay(['\n'], { keepBare: true });+ expect(item.raw).toBe('\n');+ expect(item.display).toBe('↵');+ });+});diff --git a/web/src/utils/delimiter-preview.ts b/web/src/utils/delimiter-preview.tsindex f86a6f2a8c..6f287d0641 100644--- a/web/src/utils/delimiter-preview.ts+++ b/web/src/utils/delimiter-preview.ts@@ -95,3 +95,42 @@ export function parseDelimitersForDisplay(// Stable sort longest-first (matches backend parse_delimiter_field).return result.sort((a, b) => b.raw.length - a.raw.length);}++/**+ * Parse the chunker operator's delimiter LIST (one delimiter per row) for+ * display — a different field shape from the single delimiter string above,+ * where each row is already one delimiter.+ *+ * A backtick-wrapped row contributes its inner text; any other non-empty row is+ * one delimiter exactly as typed (multi-character included) when `keepBare` is+ * true. `keepBare` is false for the Python flow TokenChunker, which only+ * activates backtick-wrapped entries. Mirrors the backend list compiler:+ * deduplicated, longest-first.+ */+export function parseDelimiterListForDisplay(+ values: (string | undefined)[] | undefined,+ { keepBare }: { keepBare: boolean },+): ParsedDelimiter[] {+ if (!values) return [];++ const result: ParsedDelimiter[] = [];+ const seen = new Set<string>();++ const push = (raw: string) => {+ if (!raw || seen.has(raw)) return;+ seen.add(raw);+ result.push({ raw, display: toDisplay(raw) });+ };++ for (const value of values) {+ if (!value) continue;+ if (value.length >= 2 && value.startsWith('`') && value.endsWith('`')) {+ push(value.slice(1, -1));+ continue;+ }+ if (keepBare) push(value);+ }++ // Stable sort longest-first (matches the backend list compiler).+ return result.sort((a, b) => b.raw.length - a.raw.length);+}