Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
fix(split): address sliding-window follow-ups from #1920 (#2003)
Panfeng Yaob24e2418e2de
diff --git a/semantica/split/sliding_window_chunker.py b/semantica/split/sliding_window_chunker.pyindex 61bf250d..2caeca59 100644--- a/semantica/split/sliding_window_chunker.py+++ b/semantica/split/sliding_window_chunker.py@@ -86,7 +86,11 @@ class SlidingWindowChunker:chunk_size use character windows with stride capped at chunk_size.Returns:- list: List of chunks+ list: Chunks whose offsets match their returned text in the input.++ Notes:+ Logs a warning if fixed windows leave input characters uncovered because+ stride exceeds chunk_size and the text is longer than chunk_size."""tracking_id = self.progress_tracker.start_tracking(module="split",@@ -109,6 +113,12 @@ class SlidingWindowChunker:)chunks = self._chunk_with_boundaries(text)else:+ if self.stride > self.chunk_size and len(text) > self.chunk_size:+ self.logger.warning(+ "With preserve_boundaries=False, stride exceeds chunk_size, "+ "leaving some input characters uncovered. "+ "Use stride <= chunk_size for continuous coverage."+ )self.progress_tracker.update_tracking(tracking_id, message="Chunking with fixed-size windows...")@@ -145,7 +155,7 @@ class SlidingWindowChunker:metadata={"chunk_index": chunk_index,"chunk_size": len(chunk_text),- "has_overlap": chunk_index > 0,+ "has_overlap": bool(chunks) and start < chunks[-1].end_index,},))@@ -168,7 +178,11 @@ class SlidingWindowChunker:group_end = 0def append_chunk(start: int, end: int, boundary_preserved: bool) -> None:- chunk_text = text[start:end].strip()+ while start < end and text[start].isspace():+ start += 1+ while end > start and text[end - 1].isspace():+ end -= 1+ chunk_text = text[start:end]if chunk_text:has_overlap = bool(chunks) and start < chunks[-1].end_indexchunks.append(@@ -211,11 +225,11 @@ class SlidingWindowChunker:# without changing configured stride or carrying partial sentences.step = min(self.stride, self.chunk_size)for fragment_start in range(start, end_pos, step):- append_chunk(- fragment_start,- min(fragment_start + self.chunk_size, end_pos),- False,- )+ fragment_end = min(fragment_start + self.chunk_size, end_pos)+ append_chunk(fragment_start, fragment_end, False)+ # Later windows would only repeat an already covered suffix.+ if fragment_end == end_pos:+ breakelse:if group_start is None:group_start = startdiff --git a/tests/split/test_chunkers.py b/tests/split/test_chunkers.pyindex b86fa6a7..47e42017 100644--- a/tests/split/test_chunkers.py+++ b/tests/split/test_chunkers.py@@ -96,6 +96,55 @@ class TestSlidingWindowChunker:== chunk_size - overlap)+ @pytest.mark.parametrize(+ "config, expected_spans, expected_overlap",+ [+ pytest.param(+ {"overlap": 0},+ [(0, 8), (8, 16), (16, 20)],+ [False, False, False],+ id="zero-overlap",+ ),+ pytest.param(+ {"overlap": 3},+ [(0, 8), (5, 13), (10, 18), (15, 20)],+ [False, True, True, True],+ id="default-overlap",+ ),+ pytest.param(+ {"overlap": 3, "stride": 8},+ [(0, 8), (8, 16), (16, 20)],+ [False, False, False],+ id="custom-stride-touching-windows",+ ),+ pytest.param(+ {"overlap": 3, "stride": 12},+ [(0, 8), (12, 20)],+ [False, False],+ id="custom-stride-with-gaps",+ ),+ pytest.param(+ {"overlap": 0, "stride": 5},+ [(0, 8), (5, 13), (10, 18), (15, 20)],+ [False, True, True, True],+ id="custom-stride-overrides-zero-overlap",+ ),+ ],+ )+ def test_fixed_size_metadata_reports_actual_overlap(+ self, config, expected_spans, expected_overlap+ ):+ text = "abcdefghijklmnopqrst"+ chunker = SlidingWindowChunker(chunk_size=8, **config)++ chunks = chunker.chunk(text, preserve_boundaries=False)++ assert [(chunk.start_index, chunk.end_index) for chunk in chunks] == expected_spans+ assert [chunk.text for chunk in chunks] == [+ text[start:end] for start, end in expected_spans+ ]+ assert [chunk.metadata["has_overlap"] for chunk in chunks] == expected_overlap+def test_chunk_with_overlap_helper(self):text = "word " * 40chunker = SlidingWindowChunker(chunk_size=50, overlap=0)@@ -118,7 +167,9 @@ class TestSlidingWindowChunker:chunks = chunker.chunk_with_overlap("abcdefghij", overlap_size=4)- assert [chunk.start_index for chunk in chunks] == list(range(10))+ # The window [5, 10) covers the end; no overlap-only tails remain.+ assert [chunk.start_index for chunk in chunks] == list(range(6))+ assert chunks[-1].end_index == 10def test_chunk_with_overlap_restores_custom_stride(self):chunker = SlidingWindowChunker(chunk_size=10, overlap=2, stride=3)diff --git a/tests/split/test_sliding_window_sentence_overlap.py b/tests/split/test_sliding_window_sentence_overlap.pyindex 32c57f6e..8530c8f4 100644--- a/tests/split/test_sliding_window_sentence_overlap.py+++ b/tests/split/test_sliding_window_sentence_overlap.py@@ -10,6 +10,7 @@ Long-sentence fallback caps stride at chunk_size to preserve content coverage."""import json+import loggingimport subprocessimport sysimport textwrap@@ -22,14 +23,14 @@ from semantica.split.sliding_window_chunker import SlidingWindowChunkerdef _assert_source_coverage(text, chunks, chunk_size):- """Check actual content as well as reported offsets, allowing outer trimming."""+ """Check exact source offsets and coverage of all non-whitespace content."""assert chunkscovered = set()for chunk in chunks:assert 0 <= chunk.start_index < chunk.end_index <= len(text)assert chunk.textassert len(chunk.text) <= chunk_size- assert chunk.text.strip() == text[chunk.start_index : chunk.end_index].strip()+ assert chunk.text == text[chunk.start_index : chunk.end_index]covered.update(range(chunk.start_index, chunk.end_index))missing = [@@ -168,7 +169,76 @@ def test_long_sentence_keeps_character_overlap(preserve_boundaries):assert current.start_index > previous.start_indexshared = min(3, len(previous.text), len(current.text))assert previous.text[-shared:] == current.text[:shared]- # Do not prescribe the existing character-fallback behavior for final tails.+++ "text, expected",+ [+ ("abc def ghi.", [(0, 3), (4, 7), (8, 12)]),+ ("ab cd.", [(0, 2), (5, 8)]),+ ("ab cd.", [(0, 2), (8, 11)]),+ ("abc\tdef\u2003ghi.", [(0, 3), (4, 7), (8, 12)]),+ (" \tab cd. ", [(2, 4), (7, 10)]),+ ],+)+def test_long_sentence_fallback_offsets_match_trimmed_text(text, expected):+ chunks = SlidingWindowChunker(chunk_size=4, overlap=0).chunk(text)++ _assert_source_coverage(text, chunks, 4)+ assert [(chunk.start_index, chunk.end_index) for chunk in chunks] == expected+ assert [chunk.metadata["chunk_index"] for chunk in chunks] == list(+ range(len(chunks))+ )+ assert all(chunk.metadata["chunk_size"] == len(chunk.text) for chunk in chunks)+++ "text, expected, has_overlap",+ [+ ("abc def.", [(0, 3), (8, 12)], [False, False]),+ ("ab cdef ghi.", [(0, 7), (5, 12)], [False, True]),+ ],+)+def test_long_sentence_fallback_overlap_uses_trimmed_offsets(+ text, expected, has_overlap+):+ chunks = SlidingWindowChunker(chunk_size=8, overlap=3).chunk(text)++ _assert_source_coverage(text, chunks, 8)+ assert [(chunk.start_index, chunk.end_index) for chunk in chunks] == expected+ assert [chunk.metadata["has_overlap"] for chunk in chunks] == has_overlap+++ "text, overlap, stride, expected",+ [+ ("abcdefghijkl.", 3, None, [(0, 8), (5, 13)]),+ ("abcdefghijklmnop.", 3, None, [(0, 8), (5, 13), (10, 17)]),+ ("abcdefghijklm.", 3, None, [(0, 8), (5, 13), (10, 14)]),+ ("abcdefghi.", 7, None, [(0, 8), (1, 9), (2, 10)]),+ ("abcdefghijkl.", 0, 5, [(0, 8), (5, 13)]),+ ("abcdefghijkl", 3, None, [(0, 8), (5, 12)]),+ ],+ ids=[+ "exact-end",+ "partial-end",+ "required-tail",+ "high-overlap",+ "stride",+ "unterminated",+ ],+)+def test_long_sentence_fallback_stops_after_covering_sentence(+ text, overlap, stride, expected+):+ config = {"chunk_size": 8, "overlap": overlap}+ if stride is not None:+ config["stride"] = stride+ chunks = SlidingWindowChunker(**config).chunk(text)++ _assert_source_coverage(text, chunks, 8)+ assert [(chunk.start_index, chunk.end_index) for chunk in chunks] == expected+ assert all(not chunk.metadata["boundary_preserved"] for chunk in chunks)def test_long_sentence_fallback_returns_to_complete_sentence_groups():@@ -183,10 +253,12 @@ def test_long_sentence_fallback_returns_to_complete_sentence_groups():assert chunks[-1].metadata["boundary_preserved"] is Truefragments = chunks[1:-1]- assert len(fragments) >= 2- assert [(chunk.start_index, chunk.end_index) for chunk in fragments[:2]] == [+ assert [(chunk.start_index, chunk.end_index) for chunk in fragments] == [(3, 11),(8, 16),+ (13, 21),+ (18, 26),+ (23, 29),]for fragment in fragments:assert 3 <= fragment.start_index < fragment.end_index <= 3 + len(long_sentence)@@ -261,21 +333,58 @@ def test_long_sentence_fallback_caps_stride_to_preserve_coverage(stride, text):assert chunker.stride == stride-def test_fixed_windows_keep_wide_stride_after_long_sentence_fallback():- text = "abcdefghijklmnopqrstuvwxy."+ "text, expected_spans",+ [+ ("abcdefghi", [(0, 8)]),+ ("abcdefghijklmnopqrstuvwxy.", [(0, 8), (20, 26)]),+ ],+)+def test_fixed_windows_warn_and_keep_wide_stride_after_fallback(+ text, expected_spans, caplog+):chunker = SlidingWindowChunker(chunk_size=8, overlap=3, stride=20)chunker.chunk(text)- chunks = chunker.chunk(text, preserve_boundaries=False)+ with caplog.at_level(logging.WARNING, logger=chunker.logger.name):+ chunks = chunker.chunk(text, preserve_boundaries=False)- assert [(chunk.start_index, chunk.end_index) for chunk in chunks] == [- (0, 8),- (20, 26),+ recorded = [+ record for record in caplog.records if record.name == chunker.logger.name+ ]+ assert len(recorded) == 1+ assert recorded[0].levelno == logging.WARNING+ assert "stride" in recorded[0].message+ assert "chunk_size" in recorded[0].message+ assert [(chunk.start_index, chunk.end_index) for chunk in chunks] == expected_spans+ assert [chunk.text for chunk in chunks] == [+ text[start:end] for start, end in expected_spans]- assert [chunk.text for chunk in chunks] == [text[:8], text[20:]]assert chunker.stride == 20+ "text, stride, preserve_boundaries",+ [+ ("", 20, False),+ ("abcdefg", 20, False),+ ("abcdefgh", 20, False),+ ("abcdefghijklmnopqrstuvwxy.", 5, False),+ ("abcdefghijklmnopqrstuvwxy.", 8, False),+ ("abcdefghijklmnopqrstuvwxy.", 20, True),+ ("One.Two.Six.", 20, True),+ ],+)+def test_gap_free_chunking_does_not_warn(text, stride, preserve_boundaries, caplog):+ chunker = SlidingWindowChunker(chunk_size=8, overlap=3, stride=stride)+ with caplog.at_level(logging.WARNING, logger=chunker.logger.name):+ chunker.chunk(text, preserve_boundaries=preserve_boundaries)++ assert not [+ record for record in caplog.records if record.name == chunker.logger.name+ ]++@pytest.mark.parametrize("overlap", [0, 8], ids=["zero-overlap", "small-overlap"])def test_many_short_sentences_do_not_retain_all_offsets(overlap):# Isolate tracing from pytest's own allocations and any existing tracer.