Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
fix(split): preserve content across sentence boundaries (#1920)
Panfeng Yao9a9bb482d847
diff --git a/semantica/split/methods.py b/semantica/split/methods.pyindex 2172e257..aa260578 100644--- a/semantica/split/methods.py+++ b/semantica/split/methods.py@@ -1620,9 +1620,13 @@ def split_sliding_window(Args:text: Input textchunk_size: Chunk size in characters- overlap: Overlap size in characters- stride: Stride size (default: chunk_size - overlap)- preserve_boundaries: Whether to preserve word/sentence boundaries+ overlap: Character overlap, or the maximum character budget for complete+ trailing sentences when preserve_boundaries is True+ stride: Character-window step (default: chunk_size - overlap). Applies+ to fixed windows and long-sentence fallback, not sentence grouping.+ Long-sentence fallback caps the step at chunk_size to avoid gaps.+ preserve_boundaries: Group complete sentences and repeat only whole+ trailing sentences that fit both the overlap budget and chunk_size**kwargs: Additional optionsReturns:diff --git a/semantica/split/sliding_window_chunker.py b/semantica/split/sliding_window_chunker.pyindex 40e1d6fb..61bf250d 100644--- a/semantica/split/sliding_window_chunker.py+++ b/semantica/split/sliding_window_chunker.py@@ -25,6 +25,8 @@ Author: Semantica ContributorsLicense: MIT"""+import re+from collections import dequefrom dataclasses import dataclass, fieldfrom typing import Any, Dict, List, Optional@@ -44,8 +46,12 @@ class SlidingWindowChunker:Args:**config: Configuration options:- chunk_size: Chunk size in characters- - overlap: Overlap size in characters (default: 0)- - stride: Stride size (default: chunk_size - overlap)+ - overlap: Character overlap for fixed windows, or a maximum+ character budget for complete trailing sentences (default: 0)+ - stride: Character-window step (default: chunk_size - overlap).+ Used by fixed windows and long-sentence fallback only; sentence+ grouping takes precedence over an explicitly configured stride.+ Long-sentence fallback caps the step at chunk_size to avoid gaps."""self.logger = get_logger("sliding_window_chunker")self.config = config@@ -74,7 +80,10 @@ class SlidingWindowChunker:Args:text: Input text to chunk**options: Chunking options:- - preserve_boundaries: Try to preserve word/sentence boundaries (default: True)+ - preserve_boundaries: Group complete sentences (default: True).+ Repeat only complete trailing sentences that fit the overlap+ budget and leave room for a new sentence. Sentences longer than+ chunk_size use character windows with stride capped at chunk_size.Returns:list: List of chunks@@ -147,61 +156,78 @@ class SlidingWindowChunker:return chunksdef _chunk_with_boundaries(self, text: str) -> List[Chunk]:- """Chunk text preserving word/sentence boundaries."""- chunks = []- text_length = len(text)-- start = 0- chunk_index = 0+ """Scan sentences once, retaining only reusable trailing sentence starts.- while start < text_length:- # Determine end position- end_pos = min(start + self.chunk_size, text_length)-- # If not at end, try to find a good boundary- if end_pos < text_length:- # Look for sentence boundary (., !, ?)- boundary_chars = [".", "!", "?", "\n"]- for boundary in boundary_chars:- last_boundary = text.rfind(boundary, start, end_pos)- if (- last_boundary > start + self.chunk_size * 0.5- ): # At least 50% of target size- end_pos = last_boundary + 1- break-- # If no sentence boundary, look for word boundary- if end_pos == min(start + self.chunk_size, text_length):- last_space = text.rfind(" ", start, end_pos)- if last_space > start + self.chunk_size * 0.5:- end_pos = last_space-- chunk_text = text[start:end_pos].strip()+ Overlap repeats complete trailing sentences within the character budget,+ leaving room for new content. Long sentences use character windows.+ """+ chunks = []+ sentence_starts = deque()+ cursor = 0+ group_start = None+ group_end = 0+ def append_chunk(start: int, end: int, boundary_preserved: bool) -> None:+ chunk_text = text[start:end].strip()if chunk_text:+ has_overlap = bool(chunks) and start < chunks[-1].end_indexchunks.append(Chunk(text=chunk_text,start_index=start,- end_index=end_pos,+ end_index=end,metadata={- "chunk_index": chunk_index,+ "chunk_index": len(chunks),"chunk_size": len(chunk_text),- "has_overlap": chunk_index > 0,- "boundary_preserved": end_pos < text_length,+ "has_overlap": has_overlap,+ "boundary_preserved": boundary_preserved,},))- # Move to next chunk with stride- start += self.stride-- # Ensure we don't go backwards- if start <= end_pos - self.overlap:- start = end_pos - self.overlap-- chunk_index += 1-+ # finditer is lazy; '$' also includes an unterminated final sentence.+ for boundary in re.finditer(r"[.!?\n]+|$", text):+ start, end_pos = cursor, boundary.end()+ cursor = end_pos+ while start < end_pos and text[start].isspace():+ start += 1+ while end_pos > start and text[end_pos - 1].isspace():+ end_pos -= 1+ if start == end_pos:+ continue++ if group_start is not None and end_pos - group_start > self.chunk_size:+ append_chunk(group_start, group_end, True)+ # Candidates already fit the overlap budget. Retain only a suffix+ # that also leaves room for this new sentence, ensuring progress.+ while sentence_starts and (+ end_pos - sentence_starts[0] > self.chunk_size+ ):+ sentence_starts.popleft()+ group_start = sentence_starts[0] if sentence_starts else None++ if end_pos - start > self.chunk_size:+ # A long sentence cannot share a group. Cap the step to avoid gaps,+ # without changing configured stride or carrying partial sentences.+ step = min(self.stride, self.chunk_size)+ for fragment_start in range(start, end_pos, step):+ append_chunk(+ fragment_start,+ min(fragment_start + self.chunk_size, end_pos),+ False,+ )+ else:+ if group_start is None:+ group_start = start+ group_end = end_pos+ sentence_starts.append(start)+ # Discard starts as soon as their suffix cannot be reused. The+ # group's first offset stays available even when overlap is zero.+ while sentence_starts and group_end - sentence_starts[0] > self.overlap:+ sentence_starts.popleft()++ if group_start is not None:+ append_chunk(group_start, group_end, True)return chunksdef chunk_with_overlap(diff --git a/semantica/split/split_usage.md b/semantica/split/split_usage.mdindex 7586df79..8d78bb64 100644--- a/semantica/split/split_usage.md+++ b/semantica/split/split_usage.md@@ -306,11 +306,10 @@ chunks = structural_chunker.chunk(text)# Sliding window chunkingsliding_chunker = SlidingWindowChunker(- window_size=1000,- step_size=800, # 200 overlap- min_chunk_size=100+ chunk_size=1000,+ overlap=200, # max budget for repeated trailing sentences; actual overlap may be smaller or zero)-chunks = sliding_chunker.chunk(text)+chunks = sliding_chunker.chunk(text, preserve_boundaries=True)# Table-specific chunkingtable_chunker = TableChunker(@@ -362,6 +361,18 @@ hierarchical_chunker = HierarchicalChunker(chunks = hierarchical_chunker.chunk(text)```+#### SlidingWindowChunker overlap and stride++With `preserve_boundaries=True` (the default), complete sentences are grouped up to `chunk_size`:++- `overlap` is a maximum character budget for complete trailing sentences, so actual overlap may be smaller or zero. Only the longest contiguous suffix of complete sentences that fits the budget and leaves room for a new sentence is repeated.+- Complete-sentence grouping takes precedence over an explicitly configured `stride`. Stride continues to control fixed character windows (`preserve_boundaries=False`) and the long-sentence fallback, which caps the step at `chunk_size`.+- Sentences longer than `chunk_size` are split into character windows rather than adjusted to word boundaries.+- `has_overlap` reports actual source-range overlap with the preceding emitted chunk, rather than simply being True for every chunk after the first.+- `boundary_preserved` is True for sentence groups, including an unterminated final sentence unit, and False for long-sentence character fallback. Previously, it indicated whether the chunk ended before the end of the input text.++With `preserve_boundaries=False`, windows start every `stride` characters, so a `stride` larger than `chunk_size` leaves text between windows uncovered. Keep `stride <= chunk_size` when every character must appear in some chunk.+## Using Methods### Getting Available Methodsdiff --git a/tests/split/test_chunkers.py b/tests/split/test_chunkers.pyindex 4193f0e3..b86fa6a7 100644--- a/tests/split/test_chunkers.py+++ b/tests/split/test_chunkers.py@@ -159,6 +159,36 @@ class TestSlidingWindowChunker:assert chunk.textassert chunk.metadata.get("chunk_index") is not None+ @pytest.mark.parametrize(+ "preserve_boundaries", [False, True], ids=["fixed-size", "sentence-boundaries"]+ )+ @pytest.mark.parametrize(+ "separator", [" ", ""], ids=["with-space", "without-space"]+ )+ def test_sentence_boundaries_do_not_drop_content(+ self, preserve_boundaries, separator+ ):+ """Regression #1800: shortening a window must not skip the following text."""+ text = f"First phrase.{separator}LOST suffix goes here."+ chunker = SlidingWindowChunker(chunk_size=20, overlap=2)++ chunks = chunker.chunk(text, preserve_boundaries=preserve_boundaries)++ covered_positions = set()+ for chunk in chunks:+ assert 0 <= chunk.start_index < chunk.end_index <= len(text)+ source_slice = text[chunk.start_index : chunk.end_index]+ assert chunk.text.strip() == source_slice.strip()+ covered_positions.update(range(chunk.start_index, chunk.end_index))++ # Boundary-aware chunks strip outer whitespace; check substantive content.+ missing = [+ (index, char)+ for index, char in enumerate(text)+ if not char.isspace() and index not in covered_positions+ ]+ assert not missing, f"Uncovered non-whitespace characters: {missing!r}"+# ---------------------------------------------------------------------------# StructuralChunkerdiff --git a/tests/split/test_sliding_window_sentence_overlap.py b/tests/split/test_sliding_window_sentence_overlap.pynew file mode 100644index 00000000..32c57f6e--- /dev/null+++ b/tests/split/test_sliding_window_sentence_overlap.py@@ -0,0 +1,340 @@+"""Acceptance tests for the sentence-overlap behavior for issue #1800.++Sentence grouping packs complete sentences up to chunk_size. Overlap is a+character budget for the longest contiguous suffix of complete sentences that+also leaves room for new content. Punctuation and internal separators count+toward that budget; outer whitespace may be stripped as in existing chunks.+Long sentences fall back to character windows. Stride controls character windows+only; sentence grouping takes precedence over an explicitly configured stride.+Long-sentence fallback caps stride at chunk_size to preserve content coverage.+"""++import json+import subprocess+import sys+import textwrap+from pathlib import Path++import pytest++from semantica.split.methods import split_sliding_window+from semantica.split.sliding_window_chunker import SlidingWindowChunker+++def _assert_source_coverage(text, chunks, chunk_size):+ """Check actual content as well as reported offsets, allowing outer trimming."""+ assert chunks+ covered = set()+ for chunk in chunks:+ assert 0 <= chunk.start_index < chunk.end_index <= len(text)+ assert chunk.text+ assert len(chunk.text) <= chunk_size+ assert chunk.text.strip() == text[chunk.start_index : chunk.end_index].strip()+ covered.update(range(chunk.start_index, chunk.end_index))++ missing = [+ (index, char)+ for index, char in enumerate(text)+ if not char.isspace() and index not in covered+ ]+ assert not missing, f"Uncovered non-whitespace characters: {missing!r}"+++def _assert_sentence_groups(text, chunk_size, overlap, expected):+ chunks = SlidingWindowChunker(chunk_size=chunk_size, overlap=overlap).chunk(+ text, preserve_boundaries=True+ )+ _assert_source_coverage(text, chunks, chunk_size)+ assert [chunk.text for chunk in chunks] == expected++ for previous, current in zip(chunks, chunks[1:]):+ assert current.start_index > previous.start_index+ assert (+ current.end_index > previous.end_index+ ), "Every group must add new content"+ shared = text[current.start_index : previous.end_index].strip()+ assert len(shared) <= overlap+++ "overlap, expected",+ [+ pytest.param(0, ["One.Two.Six.", "Ten.End."], id="zero-budget"),+ pytest.param(3, ["One.Two.Six.", "Ten.End."], id="last-sentence-does-not-fit"),+ pytest.param(4, ["One.Two.Six.", "Six.Ten.End."], id="one-sentence-exact-fit"),+ pytest.param(+ 7, ["One.Two.Six.", "Six.Ten.End."], id="no-partial-extra-sentence"+ ),+ pytest.param(+ 8,+ ["One.Two.Six.", "Two.Six.Ten.", "Six.Ten.End."],+ id="two-complete-sentences",+ ),+ ],+)+def test_sentence_overlap_uses_a_whole_sentence_budget(overlap, expected):+ # Each sentence occupies four characters, including its punctuation.+ _assert_sentence_groups("One.Two.Six.Ten.End.", 12, overlap, expected)+++def test_sentence_overlap_cannot_skip_the_last_sentence_to_fit_an_earlier_one():+ # "Go." fits the budget, but "Longest." does not: the overlap must be a suffix.+ _assert_sentence_groups("Go.Longest.End.", 12, 3, ["Go.Longest.", "End."])+++def test_sentence_overlap_shrinks_to_leave_room_for_a_new_sentence():+ # "Run.Sit." fits budget 8, but adding "Longer." would exceed chunk_size 12.+ _assert_sentence_groups(+ "Go.Run.Sit.Longer.End.", 12, 8, ["Go.Run.Sit.", "Sit.Longer.", "Longer.End."]+ )+++def test_sentence_overlap_is_zero_when_even_one_repeated_sentence_would_overflow():+ # "Sit." fits budget 4, but "Sit.Longword." would occupy 13 characters.+ _assert_sentence_groups(+ "Go.Run.Sit.Longword.End.", 12, 4, ["Go.Run.Sit.", "Longword.", "End."]+ )+++ "overlap, expected",+ [+ pytest.param(+ 8, ["One. Two. Six.", "Six. Ten. End."], id="separator-over-budget"+ ),+ pytest.param(+ 9,+ ["One. Two. Six.", "Two. Six. Ten.", "Six. Ten. End."],+ id="separator-included-in-budget",+ ),+ ],+)+def test_sentence_overlap_counts_internal_whitespace(overlap, expected):+ # "Two. Six." is nine characters, not eight; leading/trailing spaces are ignored.+ _assert_sentence_groups("One. Two. Six. Ten. End.", 14, overlap, expected)+++def test_sentence_overlap_handles_mixed_sentence_terminators():+ _assert_sentence_groups(+ "One!Two?Six.Ten!End?", 12, 4, ["One!Two?Six.", "Six.Ten!End?"]+ )+++def test_large_sentence_overlap_budget_still_makes_progress():+ # The first group is 11 characters and the budget is also 11. A naive+ # end - overlap update can get stuck at zero. Isolate that risk so pytest+ # can kill a non-terminating implementation instead of hanging indefinitely.+ script = textwrap.dedent("""+ import json+ from semantica.split.sliding_window_chunker import SlidingWindowChunker++ chunker = SlidingWindowChunker(chunk_size=12, overlap=11)+ chunker.progress_tracker.enabled = False+ chunks = chunker.chunk("Go.Run.Sit.Longer.End.", preserve_boundaries=True)+ print(json.dumps([+ [chunk.text, chunk.start_index, chunk.end_index] for chunk in chunks+ ]))+ """)+ result = subprocess.run(+ [sys.executable, "-c", script],+ cwd=Path(__file__).resolve().parents[2],+ capture_output=True,+ text=True,+ check=True,+ timeout=180, # covers a cold `import semantica` on slow CI runners+ )+ assert json.loads(result.stdout) == [+ ["Go.Run.Sit.", 0, 11],+ ["Sit.Longer.", 7, 18],+ ["Longer.End.", 11, 22],+ ]+++ "preserve_boundaries", [False, True], ids=["fixed", "fallback"]+)+def test_long_sentence_keeps_character_overlap(preserve_boundaries):+ text = "abcdefghijklmnopqrstuvwxy."+ chunks = SlidingWindowChunker(chunk_size=8, overlap=3).chunk(+ text, preserve_boundaries=preserve_boundaries+ )+ _assert_source_coverage(text, chunks, 8)+ assert len(chunks) >= 2+ assert [(chunk.start_index, chunk.end_index) for chunk in chunks[:2]] == [+ (0, 8),+ (5, 13),+ ]+ for previous, current in zip(chunks, chunks[1:]):+ assert current.start_index > previous.start_index+ shared = min(3, len(previous.text), len(current.text))+ assert previous.text[-shared:] == current.text[:shared]+ # Do not prescribe the existing character-fallback behavior for final tails.+++def test_long_sentence_fallback_returns_to_complete_sentence_groups():+ long_sentence = "abcdefghijklmnopqrstuvwxy."+ text = "Hi." + long_sentence + "Bye.End."+ chunks = SlidingWindowChunker(chunk_size=8, overlap=3).chunk(text)++ _assert_source_coverage(text, chunks, 8)+ assert chunks[0].text == "Hi."+ assert chunks[-1].text == "Bye.End."+ assert chunks[0].metadata["boundary_preserved"] is True+ assert chunks[-1].metadata["boundary_preserved"] is True++ fragments = chunks[1:-1]+ assert len(fragments) >= 2+ assert [(chunk.start_index, chunk.end_index) for chunk in fragments[:2]] == [+ (3, 11),+ (8, 16),+ ]+ for fragment in fragments:+ assert 3 <= fragment.start_index < fragment.end_index <= 3 + len(long_sentence)+ assert fragment.metadata["boundary_preserved"] is False+ # A fragment of the long sentence must not be repeated in the next sentence group.+ assert chunks[-1].metadata["has_overlap"] is False+++def test_unterminated_final_sentence_is_kept_as_a_complete_unit():+ _assert_sentence_groups("One.Two.Tail", 8, 4, ["One.Two.", "Two.Tail"])+++def test_sentence_groups_preserve_punctuation_runs_and_internal_newlines():+ _assert_sentence_groups(+ " One?!\n\nTwo.\nTail ", 11, 4, ["One?!\n\nTwo.", "Two.\nTail"]+ )++[email protected]("text", [" \t ", "\n\n \r\n"])+def test_whitespace_only_input_does_not_emit_sentence_groups(text):+ assert SlidingWindowChunker(chunk_size=8, overlap=3).chunk(text) == []++[email protected]("stride", [1, 20])+def test_sentence_grouping_takes_precedence_over_custom_stride(stride):+ text = "One.Two.Six.Ten.End."+ chunker = SlidingWindowChunker(chunk_size=12, overlap=4, stride=stride)+ chunks = chunker.chunk(text)++ _assert_source_coverage(text, chunks, 12)+ assert [chunk.text for chunk in chunks] == ["One.Two.Six.", "Six.Ten.End."]+ assert chunker.stride == stride+++ "preserve_boundaries", [False, True], ids=["fixed", "fallback"]+)+def test_character_windows_use_custom_stride(preserve_boundaries):+ text = "abcdefghijklmnopqrstuvwxy."+ chunks = SlidingWindowChunker(chunk_size=8, overlap=3, stride=2).chunk(+ text, preserve_boundaries=preserve_boundaries+ )++ _assert_source_coverage(text, chunks, 8)+ assert [(chunk.start_index, chunk.end_index) for chunk in chunks[:3]] == [+ (0, 8),+ (2, 10),+ (4, 12),+ ]+ # Explicit stride 2 determines six shared characters, overriding the default step.+ assert chunks[0].text[-6:] == chunks[1].text[:6]++[email protected]("stride", [8, 20, 64])+ "text",+ [+ "abcdefghijklmnopqrstuvwxy.",+ "Hi.abcdefghijklmnopqrstuvwxy.Bye.End.",+ "abcdefghijklmnopqrstuvwxy",+ ],+ ids=["long-sentence", "mixed-sentences", "unterminated"],+)+def test_long_sentence_fallback_caps_stride_to_preserve_coverage(stride, text):+ chunker = SlidingWindowChunker(chunk_size=8, overlap=3, stride=stride)++ chunks = chunker.chunk(text)++ _assert_source_coverage(text, chunks, 8)+ assert "".join(chunk.text for chunk in chunks) == text+ assert all(not chunk.metadata["has_overlap"] for chunk in chunks)+ assert chunker.stride == stride+++def test_fixed_windows_keep_wide_stride_after_long_sentence_fallback():+ text = "abcdefghijklmnopqrstuvwxy."+ chunker = SlidingWindowChunker(chunk_size=8, overlap=3, stride=20)+ chunker.chunk(text)++ chunks = chunker.chunk(text, preserve_boundaries=False)++ assert [(chunk.start_index, chunk.end_index) for chunk in chunks] == [+ (0, 8),+ (20, 26),+ ]+ assert [chunk.text for chunk in chunks] == [text[:8], text[20:]]+ assert chunker.stride == 20++[email protected]("overlap", [0, 8], ids=["zero-overlap", "small-overlap"])+def test_many_short_sentences_do_not_retain_all_offsets(overlap):+ # Isolate tracing from pytest's own allocations and any existing tracer.+ # A single large chunk isolates auxiliary memory from output-list growth.+ script = textwrap.dedent("""+ import json+ import sys+ import tracemalloc+ from semantica.split.sliding_window_chunker import SlidingWindowChunker++ text = "a." * 50_000+ chunker = SlidingWindowChunker(chunk_size=len(text), overlap=int(sys.argv[1]))+ chunker._chunk_with_boundaries("Warmup.")+ tracemalloc.stop()+ tracemalloc.start()+ try:+ chunks = chunker._chunk_with_boundaries(text)+ _, peak = tracemalloc.get_traced_memory()+ finally:+ tracemalloc.stop()+ assert len(chunks) == 1+ assert chunks[0].text == text+ assert (chunks[0].start_index, chunks[0].end_index) == (0, len(text))+ print(json.dumps({"peak": peak, "text_length": len(text)}))+ """)+ result = subprocess.run(+ [sys.executable, "-c", script, str(overlap)],+ cwd=Path(__file__).resolve().parents[2],+ capture_output=True,+ text=True,+ check=True,+ timeout=180, # covers a cold `import semantica` on slow CI runners+ )+ measurement = json.loads(result.stdout)+ # Allow several text copies plus fixed overhead, but not one Python offset+ # per sentence. Measure peak usage since the queue is freed before return.+ assert measurement["peak"] < 4 * measurement["text_length"] + 128 * 1024+++def test_sliding_window_method_covers_long_sentences_with_wide_stride():+ text = "abcdefghijklmnopqrstuvwxy."+ chunks = split_sliding_window(text, chunk_size=8, overlap=3, stride=20)++ _assert_source_coverage(text, chunks, 8)+ assert "".join(chunk.text for chunk in chunks) == text+ assert all(chunk.metadata["boundary_preserved"] is False for chunk in chunks)++[email protected]("overlap, expected", [(3, [False, False]), (4, [False, True])])+def test_sentence_overlap_metadata_reports_actual_overlap(overlap, expected):+ chunks = SlidingWindowChunker(chunk_size=12, overlap=overlap).chunk(+ "One.Two.Six.Ten.End."+ )+ assert [chunk.metadata["has_overlap"] for chunk in chunks] == expected+++def test_sliding_window_method_preserves_the_sentence_overlap_contract():+ chunks = split_sliding_window(+ "One.Two.Six.Ten.End.", chunk_size=12, overlap=4, preserve_boundaries=True+ )+ assert [chunk.text for chunk in chunks] == ["One.Two.Six.", "Six.Ten.End."]+ assert all(chunk.metadata["boundary_preserved"] for chunk in chunks)