Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

fix(split): stop character/word/token splitters after the final chunk

44erxyz572f072ef5d3
diff --git a/semantica/split/methods.py b/semantica/split/methods.py
index 3ba339b7..2172e257 100644
--- a/semantica/split/methods.py
+++ b/semantica/split/methods.py
@@ -311,6 +311,9 @@ def split_by_tokens(
)
)
+ if end_idx >= len(tokens):
+ break
+
# Move to next chunk with overlap
start_idx = max(start_idx + 1, end_idx - chunk_overlap)
text_start = text_end - chunk_overlap * 4 # Approximate
@@ -530,6 +533,9 @@ def split_by_characters(
)
)
+ if end >= text_length:
+ break
+
start = max(start + 1, end - chunk_overlap)
return chunks
@@ -580,6 +586,9 @@ def split_by_words(
)
)
+ if end_idx >= len(words):
+ break
+
start_idx = max(start_idx + 1, end_idx - chunk_overlap)
return chunks
diff --git a/tests/split/test_split_tail_chunks.py b/tests/split/test_split_tail_chunks.py
new file mode 100644
index 00000000..3129ce65
--- /dev/null
+++ b/tests/split/test_split_tail_chunks.py
@@ -0,0 +1,48 @@
+"""The last chunk used to be followed by shrinking copies of the tail (#1813)."""
+
+import unittest
+
+from semantica.split.methods import (
+ split_by_characters,
+ split_by_tokens,
+ split_by_words,
+)
+
+
+class TestSplitTailChunks(unittest.TestCase):
+
+ def test_characters_short_text_is_one_chunk(self):
+ chunks = split_by_characters("word " * 6, chunk_size=50, chunk_overlap=10)
+ self.assertEqual([c.text for c in chunks], ["word " * 6])
+
+ def test_characters_keeps_overlap_and_stops_at_the_end(self):
+ text = "abcdefghij" * 10 # 100 chars
+ chunks = split_by_characters(text, chunk_size=40, chunk_overlap=10)
+ self.assertEqual(
+ [(c.start_index, c.end_index) for c in chunks],
+ [(0, 40), (30, 70), (60, 100)],
+ )
+
+ def test_characters_exact_fit(self):
+ chunks = split_by_characters("x" * 50, chunk_size=50, chunk_overlap=10)
+ self.assertEqual(len(chunks), 1)
+
+ def test_words_short_text_is_one_chunk(self):
+ chunks = split_by_words("word " * 6, chunk_size=50, chunk_overlap=10)
+ self.assertEqual([c.text for c in chunks], ["word word word word word word"])
+
+ def test_words_keeps_overlap_and_stops_at_the_end(self):
+ text = " ".join(f"w{i}" for i in range(10))
+ chunks = split_by_words(text, chunk_size=4, chunk_overlap=1)
+ self.assertEqual(
+ [c.text for c in chunks],
+ ["w0 w1 w2 w3", "w3 w4 w5 w6", "w6 w7 w8 w9"],
+ )
+
+ def test_tokens_short_text_is_one_chunk(self):
+ chunks = split_by_tokens("word " * 6, chunk_size=50, chunk_overlap=10)
+ self.assertEqual(len(chunks), 1)
+
+
+if __name__ == "__main__":
+ unittest.main()