Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

fix(split): address sliding-window follow-ups from #1920 (#2003)

Panfeng Yaob24e2418e2de
diff --git a/semantica/split/sliding_window_chunker.py b/semantica/split/sliding_window_chunker.py
index 61bf250d..2caeca59 100644
--- a/semantica/split/sliding_window_chunker.py
+++ b/semantica/split/sliding_window_chunker.py
@@ -86,7 +86,11 @@ class SlidingWindowChunker:
chunk_size use character windows with stride capped at chunk_size.
Returns:
- list: List of chunks
+ list: Chunks whose offsets match their returned text in the input.
+
+ Notes:
+ Logs a warning if fixed windows leave input characters uncovered because
+ stride exceeds chunk_size and the text is longer than chunk_size.
"""
tracking_id = self.progress_tracker.start_tracking(
module="split",
@@ -109,6 +113,12 @@ class SlidingWindowChunker:
)
chunks = self._chunk_with_boundaries(text)
else:
+ if self.stride > self.chunk_size and len(text) > self.chunk_size:
+ self.logger.warning(
+ "With preserve_boundaries=False, stride exceeds chunk_size, "
+ "leaving some input characters uncovered. "
+ "Use stride <= chunk_size for continuous coverage."
+ )
self.progress_tracker.update_tracking(
tracking_id, message="Chunking with fixed-size windows..."
)
@@ -145,7 +155,7 @@ class SlidingWindowChunker:
metadata={
"chunk_index": chunk_index,
"chunk_size": len(chunk_text),
- "has_overlap": chunk_index > 0,
+ "has_overlap": bool(chunks) and start < chunks[-1].end_index,
},
)
)
@@ -168,7 +178,11 @@ class SlidingWindowChunker:
group_end = 0
def append_chunk(start: int, end: int, boundary_preserved: bool) -> None:
- chunk_text = text[start:end].strip()
+ while start < end and text[start].isspace():
+ start += 1
+ while end > start and text[end - 1].isspace():
+ end -= 1
+ chunk_text = text[start:end]
if chunk_text:
has_overlap = bool(chunks) and start < chunks[-1].end_index
chunks.append(
@@ -211,11 +225,11 @@ class SlidingWindowChunker:
# without changing configured stride or carrying partial sentences.
step = min(self.stride, self.chunk_size)
for fragment_start in range(start, end_pos, step):
- append_chunk(
- fragment_start,
- min(fragment_start + self.chunk_size, end_pos),
- False,
- )
+ fragment_end = min(fragment_start + self.chunk_size, end_pos)
+ append_chunk(fragment_start, fragment_end, False)
+ # Later windows would only repeat an already covered suffix.
+ if fragment_end == end_pos:
+ break
else:
if group_start is None:
group_start = start
diff --git a/tests/split/test_chunkers.py b/tests/split/test_chunkers.py
index b86fa6a7..47e42017 100644
--- a/tests/split/test_chunkers.py
+++ b/tests/split/test_chunkers.py
@@ -96,6 +96,55 @@ class TestSlidingWindowChunker:
== chunk_size - overlap
)
+ @pytest.mark.parametrize(
+ "config, expected_spans, expected_overlap",
+ [
+ pytest.param(
+ {"overlap": 0},
+ [(0, 8), (8, 16), (16, 20)],
+ [False, False, False],
+ id="zero-overlap",
+ ),
+ pytest.param(
+ {"overlap": 3},
+ [(0, 8), (5, 13), (10, 18), (15, 20)],
+ [False, True, True, True],
+ id="default-overlap",
+ ),
+ pytest.param(
+ {"overlap": 3, "stride": 8},
+ [(0, 8), (8, 16), (16, 20)],
+ [False, False, False],
+ id="custom-stride-touching-windows",
+ ),
+ pytest.param(
+ {"overlap": 3, "stride": 12},
+ [(0, 8), (12, 20)],
+ [False, False],
+ id="custom-stride-with-gaps",
+ ),
+ pytest.param(
+ {"overlap": 0, "stride": 5},
+ [(0, 8), (5, 13), (10, 18), (15, 20)],
+ [False, True, True, True],
+ id="custom-stride-overrides-zero-overlap",
+ ),
+ ],
+ )
+ def test_fixed_size_metadata_reports_actual_overlap(
+ self, config, expected_spans, expected_overlap
+ ):
+ text = "abcdefghijklmnopqrst"
+ chunker = SlidingWindowChunker(chunk_size=8, **config)
+
+ chunks = chunker.chunk(text, preserve_boundaries=False)
+
+ assert [(chunk.start_index, chunk.end_index) for chunk in chunks] == expected_spans
+ assert [chunk.text for chunk in chunks] == [
+ text[start:end] for start, end in expected_spans
+ ]
+ assert [chunk.metadata["has_overlap"] for chunk in chunks] == expected_overlap
+
def test_chunk_with_overlap_helper(self):
text = "word " * 40
chunker = SlidingWindowChunker(chunk_size=50, overlap=0)
@@ -118,7 +167,9 @@ class TestSlidingWindowChunker:
chunks = chunker.chunk_with_overlap("abcdefghij", overlap_size=4)
- assert [chunk.start_index for chunk in chunks] == list(range(10))
+ # The window [5, 10) covers the end; no overlap-only tails remain.
+ assert [chunk.start_index for chunk in chunks] == list(range(6))
+ assert chunks[-1].end_index == 10
def test_chunk_with_overlap_restores_custom_stride(self):
chunker = SlidingWindowChunker(chunk_size=10, overlap=2, stride=3)
diff --git a/tests/split/test_sliding_window_sentence_overlap.py b/tests/split/test_sliding_window_sentence_overlap.py
index 32c57f6e..8530c8f4 100644
--- a/tests/split/test_sliding_window_sentence_overlap.py
+++ b/tests/split/test_sliding_window_sentence_overlap.py
@@ -10,6 +10,7 @@ Long-sentence fallback caps stride at chunk_size to preserve content coverage.
"""
import json
+import logging
import subprocess
import sys
import textwrap
@@ -22,14 +23,14 @@ from semantica.split.sliding_window_chunker import SlidingWindowChunker
def _assert_source_coverage(text, chunks, chunk_size):
- """Check actual content as well as reported offsets, allowing outer trimming."""
+ """Check exact source offsets and coverage of all non-whitespace content."""
assert chunks
covered = set()
for chunk in chunks:
assert 0 <= chunk.start_index < chunk.end_index <= len(text)
assert chunk.text
assert len(chunk.text) <= chunk_size
- assert chunk.text.strip() == text[chunk.start_index : chunk.end_index].strip()
+ assert chunk.text == text[chunk.start_index : chunk.end_index]
covered.update(range(chunk.start_index, chunk.end_index))
missing = [
@@ -168,7 +169,76 @@ def test_long_sentence_keeps_character_overlap(preserve_boundaries):
assert current.start_index > previous.start_index
shared = min(3, len(previous.text), len(current.text))
assert previous.text[-shared:] == current.text[:shared]
- # Do not prescribe the existing character-fallback behavior for final tails.
+
+
+ "text, expected",
+ [
+ ("abc def ghi.", [(0, 3), (4, 7), (8, 12)]),
+ ("ab cd.", [(0, 2), (5, 8)]),
+ ("ab cd.", [(0, 2), (8, 11)]),
+ ("abc\tdef\u2003ghi.", [(0, 3), (4, 7), (8, 12)]),
+ (" \tab cd. ", [(2, 4), (7, 10)]),
+ ],
+)
+def test_long_sentence_fallback_offsets_match_trimmed_text(text, expected):
+ chunks = SlidingWindowChunker(chunk_size=4, overlap=0).chunk(text)
+
+ _assert_source_coverage(text, chunks, 4)
+ assert [(chunk.start_index, chunk.end_index) for chunk in chunks] == expected
+ assert [chunk.metadata["chunk_index"] for chunk in chunks] == list(
+ range(len(chunks))
+ )
+ assert all(chunk.metadata["chunk_size"] == len(chunk.text) for chunk in chunks)
+
+
+ "text, expected, has_overlap",
+ [
+ ("abc def.", [(0, 3), (8, 12)], [False, False]),
+ ("ab cdef ghi.", [(0, 7), (5, 12)], [False, True]),
+ ],
+)
+def test_long_sentence_fallback_overlap_uses_trimmed_offsets(
+ text, expected, has_overlap
+):
+ chunks = SlidingWindowChunker(chunk_size=8, overlap=3).chunk(text)
+
+ _assert_source_coverage(text, chunks, 8)
+ assert [(chunk.start_index, chunk.end_index) for chunk in chunks] == expected
+ assert [chunk.metadata["has_overlap"] for chunk in chunks] == has_overlap
+
+
+ "text, overlap, stride, expected",
+ [
+ ("abcdefghijkl.", 3, None, [(0, 8), (5, 13)]),
+ ("abcdefghijklmnop.", 3, None, [(0, 8), (5, 13), (10, 17)]),
+ ("abcdefghijklm.", 3, None, [(0, 8), (5, 13), (10, 14)]),
+ ("abcdefghi.", 7, None, [(0, 8), (1, 9), (2, 10)]),
+ ("abcdefghijkl.", 0, 5, [(0, 8), (5, 13)]),
+ ("abcdefghijkl", 3, None, [(0, 8), (5, 12)]),
+ ],
+ ids=[
+ "exact-end",
+ "partial-end",
+ "required-tail",
+ "high-overlap",
+ "stride",
+ "unterminated",
+ ],
+)
+def test_long_sentence_fallback_stops_after_covering_sentence(
+ text, overlap, stride, expected
+):
+ config = {"chunk_size": 8, "overlap": overlap}
+ if stride is not None:
+ config["stride"] = stride
+ chunks = SlidingWindowChunker(**config).chunk(text)
+
+ _assert_source_coverage(text, chunks, 8)
+ assert [(chunk.start_index, chunk.end_index) for chunk in chunks] == expected
+ assert all(not chunk.metadata["boundary_preserved"] for chunk in chunks)
def test_long_sentence_fallback_returns_to_complete_sentence_groups():
@@ -183,10 +253,12 @@ def test_long_sentence_fallback_returns_to_complete_sentence_groups():
assert chunks[-1].metadata["boundary_preserved"] is True
fragments = chunks[1:-1]
- assert len(fragments) >= 2
- assert [(chunk.start_index, chunk.end_index) for chunk in fragments[:2]] == [
+ assert [(chunk.start_index, chunk.end_index) for chunk in fragments] == [
(3, 11),
(8, 16),
+ (13, 21),
+ (18, 26),
+ (23, 29),
]
for fragment in fragments:
assert 3 <= fragment.start_index < fragment.end_index <= 3 + len(long_sentence)
@@ -261,21 +333,58 @@ def test_long_sentence_fallback_caps_stride_to_preserve_coverage(stride, text):
assert chunker.stride == stride
-def test_fixed_windows_keep_wide_stride_after_long_sentence_fallback():
- text = "abcdefghijklmnopqrstuvwxy."
+ "text, expected_spans",
+ [
+ ("abcdefghi", [(0, 8)]),
+ ("abcdefghijklmnopqrstuvwxy.", [(0, 8), (20, 26)]),
+ ],
+)
+def test_fixed_windows_warn_and_keep_wide_stride_after_fallback(
+ text, expected_spans, caplog
+):
chunker = SlidingWindowChunker(chunk_size=8, overlap=3, stride=20)
chunker.chunk(text)
- chunks = chunker.chunk(text, preserve_boundaries=False)
+ with caplog.at_level(logging.WARNING, logger=chunker.logger.name):
+ chunks = chunker.chunk(text, preserve_boundaries=False)
- assert [(chunk.start_index, chunk.end_index) for chunk in chunks] == [
- (0, 8),
- (20, 26),
+ recorded = [
+ record for record in caplog.records if record.name == chunker.logger.name
+ ]
+ assert len(recorded) == 1
+ assert recorded[0].levelno == logging.WARNING
+ assert "stride" in recorded[0].message
+ assert "chunk_size" in recorded[0].message
+ assert [(chunk.start_index, chunk.end_index) for chunk in chunks] == expected_spans
+ assert [chunk.text for chunk in chunks] == [
+ text[start:end] for start, end in expected_spans
]
- assert [chunk.text for chunk in chunks] == [text[:8], text[20:]]
assert chunker.stride == 20
+ "text, stride, preserve_boundaries",
+ [
+ ("", 20, False),
+ ("abcdefg", 20, False),
+ ("abcdefgh", 20, False),
+ ("abcdefghijklmnopqrstuvwxy.", 5, False),
+ ("abcdefghijklmnopqrstuvwxy.", 8, False),
+ ("abcdefghijklmnopqrstuvwxy.", 20, True),
+ ("One.Two.Six.", 20, True),
+ ],
+)
+def test_gap_free_chunking_does_not_warn(text, stride, preserve_boundaries, caplog):
+ chunker = SlidingWindowChunker(chunk_size=8, overlap=3, stride=stride)
+ with caplog.at_level(logging.WARNING, logger=chunker.logger.name):
+ chunker.chunk(text, preserve_boundaries=preserve_boundaries)
+
+ assert not [
+ record for record in caplog.records if record.name == chunker.logger.name
+ ]
+
+
@pytest.mark.parametrize("overlap", [0, 8], ids=["zero-overlap", "small-overlap"])
def test_many_short_sentences_do_not_retain_all_offsets(overlap):
# Isolate tracing from pytest's own allocations and any existing tracer.