Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(rag): drop empty and whitespace-only chunks in ApproxTokenChunker (#2687)
DiegoCluv78a97dcb7328a
diff --git a/src/agentscope/rag/_chunker/_approx_token_chunker.py b/src/agentscope/rag/_chunker/_approx_token_chunker.pyindex 3ecf52261..c8accba0c 100644--- a/src/agentscope/rag/_chunker/_approx_token_chunker.py+++ b/src/agentscope/rag/_chunker/_approx_token_chunker.py@@ -28,7 +28,7 @@ class ApproxTokenChunker(ChunkerBase):Sections carrying a :class:`~agentscope.message.DataBlock`(images, video, etc.) are passed through unchanged as a single- chunk.+ chunk. Empty or whitespace-only text produces no chunk... note:: Chunks never span across two input Sections, asrequired by :class:`ChunkerBase`.@@ -144,9 +144,11 @@ class ApproxTokenChunker(ChunkerBase):for section in sections:contents: list[TextBlock | DataBlock]if isinstance(section.content, TextBlock):+ # Blank pieces (e.g. empty scanned pages) are droppedcontents = [TextBlock(text=piece)for piece in self._split_text(section.content.text)+ if piece.strip()]else:# DataBlock pass-through: never slice multimodal datadiff --git a/src/agentscope/rag/_chunker/_base.py b/src/agentscope/rag/_chunker/_base.pyindex c80c16075..3a5cca802 100644--- a/src/agentscope/rag/_chunker/_base.py+++ b/src/agentscope/rag/_chunker/_base.py@@ -41,6 +41,9 @@ class ChunkerBase(ABC):- **DataBlock pass-through**: a Section whose content is a:class:`~agentscope.message.DataBlock` becomes a single Chunkwith the same content; multimodal data is never sliced.+ - **No empty-text chunks**: a Section whose text content is+ empty or whitespace-only produces no Chunk, so blank pages+ never reach the embedding step.- **Continuous indexing**: ``chunk_index`` runs from ``0`` to``total_chunks - 1`` across the entire output list, evenwhen the input contains many Sections.diff --git a/tests/rag_chunker_approx_token_test.py b/tests/rag_chunker_approx_token_test.pyindex 547996159..d405cb1db 100644--- a/tests/rag_chunker_approx_token_test.py+++ b/tests/rag_chunker_approx_token_test.py@@ -218,3 +218,48 @@ class ApproxTokenChunkerTest(IsolatedAsyncioTestCase):},],)++ async def test_blank_sections_are_dropped(self) -> None:+ """Empty and whitespace-only sections produce no chunk and leave+ no gap in the numbering."""+ chunker = ApproxTokenChunker(chunk_size=512, overlap=50)+ sections = [+ Section(content=TextBlock(text=""), source="scan.pdf"),+ Section(content=TextBlock(text="page two"), source="scan.pdf"),+ Section(content=TextBlock(text=" \n "), source="scan.pdf"),+ Section(content=TextBlock(text="page four"), source="scan.pdf"),+ ]++ chunks = await chunker.chunk(sections)++ self.assertListEqual(+ _dump_chunks(chunks),+ [+ {+ "content": {+ "type": "text",+ "text": "page two",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ "source": "scan.pdf",+ "chunk_index": 0,+ "total_chunks": 2,+ "metadata": {},+ },+ {+ "content": {+ "type": "text",+ "text": "page four",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ "source": "scan.pdf",+ "chunk_index": 1,+ "total_chunks": 2,+ "metadata": {},+ },+ ],+ )