RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
/*
* Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Parses a "Delimiter for text" field value into a list of delimiters for
* display in the UI. Mirrors the canonical backend parser in
* `rag/nlp/delim.py` (`parse_delimiter_field`): CRLF normalization, bare
* and backtick-wrapped tokens, insertion-ordered dedupe, and longest-first
* stable sort. Whitespace glyph substitution is display-only.
*/
export interface ParsedDelimiter {
/** The original characters the user entered. */
raw: string;
/**
* The same content with whitespace characters replaced by visible
* Unicode glyphs so that whitespace-only delimiters (which are
* invisible in a single-line input) still appear in the preview.
*/
display: string;
}
/**
* Map of whitespace characters to visible glyphs. Only used for
* display; the value sent to the backend is the raw string.
*/
const WHITESPACE_GLYPHS: Record<string, string> = {
'\n': '↵', // DOWNWARDS ARROW WITH CORNER LEFTWARDS
'\t': '⇥', // RIGHTWARDS ARROW TO BAR
'\r': '␍', // CARRIAGE RETURN SYMBOL
' ': '␣', // OPEN BOX
'\f': '␌', // FORM FEED SYMBOL
'\v': '␋', // VERTICAL TAB SYMBOL
};
/**
* Render a raw delimiter string for display by substituting visible
* Unicode glyphs for each whitespace character. Non-whitespace
* characters pass through unchanged.
*/
function toDisplay(raw: string): string {
let out = '';
for (const ch of raw) {
out += ch in WHITESPACE_GLYPHS ? WHITESPACE_GLYPHS[ch] : ch;
}
return out;
}
/**
* Parse the delimiter field into a deduplicated, longest-first list that
* matches backend `parse_delimiter_field` semantics. Display glyphs are
* applied after parsing.
*
* Returns an empty array if the value is undefined or empty.
*/
export function parseDelimitersForDisplay(
value: string | undefined,
): ParsedDelimiter[] {
if (!value) return [];
// Match backend: \r\n → \n, then standalone \r → \n.
const normalizedValue = value.replaceAll('\r\n', '\n').replaceAll('\r', '\n');
const result: ParsedDelimiter[] = [];
const seen = new Set<string>();
const push = (raw: string) => {
if (!raw || seen.has(raw)) return;
seen.add(raw);
result.push({ raw, display: toDisplay(raw) });
};
let cursor = 0;
for (const match of normalizedValue.matchAll(/`([^`]+)`/g)) {
const start = match.index!;
const end = start + match[0].length;
for (const ch of normalizedValue.slice(cursor, start)) push(ch);
push(match[1]);
cursor = end;
}
for (const ch of normalizedValue.slice(cursor)) push(ch);
// Stable sort longest-first (matches backend parse_delimiter_field).
return result.sort((a, b) => b.raw.length - a.raw.length);
}
/**
* Parse the chunker operator's delimiter LIST (one delimiter per row) for
* display — a different field shape from the single delimiter string above,
* where each row is already one delimiter.
*
* A backtick-wrapped row contributes its inner text; any other non-empty row is
* one delimiter exactly as typed (multi-character included) when `keepBare` is
* true. `keepBare` is false for the Python flow TokenChunker, which only
* activates backtick-wrapped entries. Mirrors the backend list compiler:
* deduplicated, longest-first.
*/
export function parseDelimiterListForDisplay(
values: (string | undefined)[] | undefined,
{ keepBare }: { keepBare: boolean },
): ParsedDelimiter[] {
if (!values) return [];
const result: ParsedDelimiter[] = [];
const seen = new Set<string>();
const push = (raw: string) => {
if (!raw || seen.has(raw)) return;
seen.add(raw);
result.push({ raw, display: toDisplay(raw) });
};
for (const value of values) {
if (!value) continue;
if (value.length >= 2 && value.startsWith('`') && value.endsWith('`')) {
push(value.slice(1, -1));
continue;
}
if (keepBare) push(value);
}
// Stable sort longest-first (matches the backend list compiler).
return result.sort((a, b) => b.raw.length - a.raw.length);
}