RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
test(token_chunker): drop unsupported chunk_token_size=0 case from delimiter-mode test (#17997)
Jack869da9c7adf5
diff --git a/rag/flow/tests/test_token_chunker.py b/rag/flow/tests/test_token_chunker.pyindex a33732a9c0..f12acbd081 100644--- a/rag/flow/tests/test_token_chunker.py+++ b/rag/flow/tests/test_token_chunker.py@@ -426,10 +426,10 @@ def test_json_delimiter_mode_consecutive_delimiter_keeps_boundary():assert all("##" not in t for t in texts)-def test_text_delimiter_mode_zero_or_one_no_atom_split():- # chunk_token_size=0/1 must not atom-split delimiter segments into 1-token+def test_text_delimiter_mode_one_no_atom_split():+ # chunk_token_size=1 must not atom-split delimiter segments into 1-token# chunks; the delimiter path produces delimiter-boundary chunks regardless of cap.- for module, chunker in _build_json_chunker({"delimiter_mode": "delimiter", "delimiters": ["`|`"]}):+ for _module, chunker in _build_json_chunker({"delimiter_mode": "delimiter", "delimiters": ["`|`"]}):# text path: delimiter_mode is delimiter but a custom delimiter is# present, so the delimiter branch (_split_text_by_pattern) is used.kwargs = {@@ -437,9 +437,9 @@ def test_text_delimiter_mode_zero_or_one_no_atom_split():"output_format": "text","text": "aaa|bbb|ccc",}- for chunk_token_size in (0, 1):- setattr(chunker._param, "chunk_token_size", chunk_token_size)- asyncio.run(chunker._invoke(**kwargs))- chunks = chunker._outputs["chunks"]- texts = [c["text"] for c in chunks]- assert texts == ["aaa", "bbb", "ccc"], f"chunk_token_size={chunk_token_size} atom-split a delimiter segment: {texts}"+ chunk_token_size = 1+ setattr(chunker._param, "chunk_token_size", chunk_token_size)+ asyncio.run(chunker._invoke(**kwargs))+ chunks = chunker._outputs["chunks"]+ texts = [c["text"] for c in chunks]+ assert texts == ["aaa", "bbb", "ccc"], f"chunk_token_size={chunk_token_size} atom-split a delimiter segment: {texts}"