Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

test(token_chunker): drop unsupported chunk_token_size=0 case from delimiter-mode test (#17997)

Jack869da9c7adf5
diff --git a/rag/flow/tests/test_token_chunker.py b/rag/flow/tests/test_token_chunker.py
index a33732a9c0..f12acbd081 100644
--- a/rag/flow/tests/test_token_chunker.py
+++ b/rag/flow/tests/test_token_chunker.py
@@ -426,10 +426,10 @@ def test_json_delimiter_mode_consecutive_delimiter_keeps_boundary():
assert all("##" not in t for t in texts)
-def test_text_delimiter_mode_zero_or_one_no_atom_split():
- # chunk_token_size=0/1 must not atom-split delimiter segments into 1-token
+def test_text_delimiter_mode_one_no_atom_split():
+ # chunk_token_size=1 must not atom-split delimiter segments into 1-token
# chunks; the delimiter path produces delimiter-boundary chunks regardless of cap.
- for module, chunker in _build_json_chunker({"delimiter_mode": "delimiter", "delimiters": ["`|`"]}):
+ for _module, chunker in _build_json_chunker({"delimiter_mode": "delimiter", "delimiters": ["`|`"]}):
# text path: delimiter_mode is delimiter but a custom delimiter is
# present, so the delimiter branch (_split_text_by_pattern) is used.
kwargs = {
@@ -437,9 +437,9 @@ def test_text_delimiter_mode_zero_or_one_no_atom_split():
"output_format": "text",
"text": "aaa|bbb|ccc",
}
- for chunk_token_size in (0, 1):
- setattr(chunker._param, "chunk_token_size", chunk_token_size)
- asyncio.run(chunker._invoke(**kwargs))
- chunks = chunker._outputs["chunks"]
- texts = [c["text"] for c in chunks]
- assert texts == ["aaa", "bbb", "ccc"], f"chunk_token_size={chunk_token_size} atom-split a delimiter segment: {texts}"
+ chunk_token_size = 1
+ setattr(chunker._param, "chunk_token_size", chunk_token_size)
+ asyncio.run(chunker._invoke(**kwargs))
+ chunks = chunker._outputs["chunks"]
+ texts = [c["text"] for c in chunks]
+ assert texts == ["aaa", "bbb", "ccc"], f"chunk_token_size={chunk_token_size} atom-split a delimiter segment: {texts}"