Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
fix(split): stop character/word/token splitters after the final chunk
44erxyz572f072ef5d3
diff --git a/semantica/split/methods.py b/semantica/split/methods.pyindex 3ba339b7..2172e257 100644--- a/semantica/split/methods.py+++ b/semantica/split/methods.py@@ -311,6 +311,9 @@ def split_by_tokens())+ if end_idx >= len(tokens):+ break+# Move to next chunk with overlapstart_idx = max(start_idx + 1, end_idx - chunk_overlap)text_start = text_end - chunk_overlap * 4 # Approximate@@ -530,6 +533,9 @@ def split_by_characters())+ if end >= text_length:+ break+start = max(start + 1, end - chunk_overlap)return chunks@@ -580,6 +586,9 @@ def split_by_words())+ if end_idx >= len(words):+ break+start_idx = max(start_idx + 1, end_idx - chunk_overlap)return chunksdiff --git a/tests/split/test_split_tail_chunks.py b/tests/split/test_split_tail_chunks.pynew file mode 100644index 00000000..3129ce65--- /dev/null+++ b/tests/split/test_split_tail_chunks.py@@ -0,0 +1,48 @@+"""The last chunk used to be followed by shrinking copies of the tail (#1813)."""++import unittest++from semantica.split.methods import (+ split_by_characters,+ split_by_tokens,+ split_by_words,+)+++class TestSplitTailChunks(unittest.TestCase):++ def test_characters_short_text_is_one_chunk(self):+ chunks = split_by_characters("word " * 6, chunk_size=50, chunk_overlap=10)+ self.assertEqual([c.text for c in chunks], ["word " * 6])++ def test_characters_keeps_overlap_and_stops_at_the_end(self):+ text = "abcdefghij" * 10 # 100 chars+ chunks = split_by_characters(text, chunk_size=40, chunk_overlap=10)+ self.assertEqual(+ [(c.start_index, c.end_index) for c in chunks],+ [(0, 40), (30, 70), (60, 100)],+ )++ def test_characters_exact_fit(self):+ chunks = split_by_characters("x" * 50, chunk_size=50, chunk_overlap=10)+ self.assertEqual(len(chunks), 1)++ def test_words_short_text_is_one_chunk(self):+ chunks = split_by_words("word " * 6, chunk_size=50, chunk_overlap=10)+ self.assertEqual([c.text for c in chunks], ["word word word word word word"])++ def test_words_keeps_overlap_and_stops_at_the_end(self):+ text = " ".join(f"w{i}" for i in range(10))+ chunks = split_by_words(text, chunk_size=4, chunk_overlap=1)+ self.assertEqual(+ [c.text for c in chunks],+ ["w0 w1 w2 w3", "w3 w4 w5 w6", "w6 w7 w8 w9"],+ )++ def test_tokens_short_text_is_one_chunk(self):+ chunks = split_by_tokens("word " * 6, chunk_size=50, chunk_overlap=10)+ self.assertEqual(len(chunks), 1)+++if __name__ == "__main__":+ unittest.main()