Skip to content

Build and run agents you can see, understand and trust.

fix(rag): drop empty and whitespace-only chunks in ApproxTokenChunker (#2687)

DiegoCluv78a97dcb7328a
diff --git a/src/agentscope/rag/_chunker/_approx_token_chunker.py b/src/agentscope/rag/_chunker/_approx_token_chunker.py
index 3ecf52261..c8accba0c 100644
--- a/src/agentscope/rag/_chunker/_approx_token_chunker.py
+++ b/src/agentscope/rag/_chunker/_approx_token_chunker.py
@@ -28,7 +28,7 @@ class ApproxTokenChunker(ChunkerBase):
Sections carrying a :class:`~agentscope.message.DataBlock`
(images, video, etc.) are passed through unchanged as a single
- chunk.
+ chunk. Empty or whitespace-only text produces no chunk.
.. note:: Chunks never span across two input Sections, as
required by :class:`ChunkerBase`.
@@ -144,9 +144,11 @@ class ApproxTokenChunker(ChunkerBase):
for section in sections:
contents: list[TextBlock | DataBlock]
if isinstance(section.content, TextBlock):
+ # Blank pieces (e.g. empty scanned pages) are dropped
contents = [
TextBlock(text=piece)
for piece in self._split_text(section.content.text)
+ if piece.strip()
]
else:
# DataBlock pass-through: never slice multimodal data
diff --git a/src/agentscope/rag/_chunker/_base.py b/src/agentscope/rag/_chunker/_base.py
index c80c16075..3a5cca802 100644
--- a/src/agentscope/rag/_chunker/_base.py
+++ b/src/agentscope/rag/_chunker/_base.py
@@ -41,6 +41,9 @@ class ChunkerBase(ABC):
- **DataBlock pass-through**: a Section whose content is a
:class:`~agentscope.message.DataBlock` becomes a single Chunk
with the same content; multimodal data is never sliced.
+ - **No empty-text chunks**: a Section whose text content is
+ empty or whitespace-only produces no Chunk, so blank pages
+ never reach the embedding step.
- **Continuous indexing**: ``chunk_index`` runs from ``0`` to
``total_chunks - 1`` across the entire output list, even
when the input contains many Sections.
diff --git a/tests/rag_chunker_approx_token_test.py b/tests/rag_chunker_approx_token_test.py
index 547996159..d405cb1db 100644
--- a/tests/rag_chunker_approx_token_test.py
+++ b/tests/rag_chunker_approx_token_test.py
@@ -218,3 +218,48 @@ class ApproxTokenChunkerTest(IsolatedAsyncioTestCase):
},
],
)
+
+ async def test_blank_sections_are_dropped(self) -> None:
+ """Empty and whitespace-only sections produce no chunk and leave
+ no gap in the numbering."""
+ chunker = ApproxTokenChunker(chunk_size=512, overlap=50)
+ sections = [
+ Section(content=TextBlock(text=""), source="scan.pdf"),
+ Section(content=TextBlock(text="page two"), source="scan.pdf"),
+ Section(content=TextBlock(text=" \n "), source="scan.pdf"),
+ Section(content=TextBlock(text="page four"), source="scan.pdf"),
+ ]
+
+ chunks = await chunker.chunk(sections)
+
+ self.assertListEqual(
+ _dump_chunks(chunks),
+ [
+ {
+ "content": {
+ "type": "text",
+ "text": "page two",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ "source": "scan.pdf",
+ "chunk_index": 0,
+ "total_chunks": 2,
+ "metadata": {},
+ },
+ {
+ "content": {
+ "type": "text",
+ "text": "page four",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ "source": "scan.pdf",
+ "chunk_index": 1,
+ "total_chunks": 2,
+ "metadata": {},
+ },
+ ],
+ )