Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
"""Acceptance tests for the sentence-overlap behavior for issue #1800.
Sentence grouping packs complete sentences up to chunk_size. Overlap is a
character budget for the longest contiguous suffix of complete sentences that
also leaves room for new content. Punctuation and internal separators count
toward that budget; outer whitespace may be stripped as in existing chunks.
Long sentences fall back to character windows. Stride controls character windows
only; sentence grouping takes precedence over an explicitly configured stride.
Long-sentence fallback caps stride at chunk_size to preserve content coverage.
"""
import json
import logging
import subprocess
import sys
import textwrap
from pathlib import Path
import pytest
from semantica.split.methods import split_sliding_window
from semantica.split.sliding_window_chunker import SlidingWindowChunker
def _assert_source_coverage(text, chunks, chunk_size):
"""Check exact source offsets and coverage of all non-whitespace content."""
assert chunks
covered = set()
for chunk in chunks:
assert 0 <= chunk.start_index < chunk.end_index <= len(text)
assert chunk.text
assert len(chunk.text) <= chunk_size
assert chunk.text == text[chunk.start_index : chunk.end_index]
covered.update(range(chunk.start_index, chunk.end_index))
missing = [
(index, char)
for index, char in enumerate(text)
if not char.isspace() and index not in covered
]
assert not missing, f"Uncovered non-whitespace characters: {missing!r}"
def _assert_sentence_groups(text, chunk_size, overlap, expected):
chunks = SlidingWindowChunker(chunk_size=chunk_size, overlap=overlap).chunk(
text, preserve_boundaries=True
)
_assert_source_coverage(text, chunks, chunk_size)
assert [chunk.text for chunk in chunks] == expected
for previous, current in zip(chunks, chunks[1:]):
assert current.start_index > previous.start_index
assert (
current.end_index > previous.end_index
), "Every group must add new content"
shared = text[current.start_index : previous.end_index].strip()
assert len(shared) <= overlap
@pytest.mark.parametrize(
"overlap, expected",
[
pytest.param(0, ["One.Two.Six.", "Ten.End."], id="zero-budget"),
pytest.param(3, ["One.Two.Six.", "Ten.End."], id="last-sentence-does-not-fit"),
pytest.param(4, ["One.Two.Six.", "Six.Ten.End."], id="one-sentence-exact-fit"),
pytest.param(
7, ["One.Two.Six.", "Six.Ten.End."], id="no-partial-extra-sentence"
),
pytest.param(
8,
["One.Two.Six.", "Two.Six.Ten.", "Six.Ten.End."],
id="two-complete-sentences",
),
],
)
def test_sentence_overlap_uses_a_whole_sentence_budget(overlap, expected):
# Each sentence occupies four characters, including its punctuation.
_assert_sentence_groups("One.Two.Six.Ten.End.", 12, overlap, expected)
def test_sentence_overlap_cannot_skip_the_last_sentence_to_fit_an_earlier_one():
# "Go." fits the budget, but "Longest." does not: the overlap must be a suffix.
_assert_sentence_groups("Go.Longest.End.", 12, 3, ["Go.Longest.", "End."])
def test_sentence_overlap_shrinks_to_leave_room_for_a_new_sentence():
# "Run.Sit." fits budget 8, but adding "Longer." would exceed chunk_size 12.
_assert_sentence_groups(
"Go.Run.Sit.Longer.End.", 12, 8, ["Go.Run.Sit.", "Sit.Longer.", "Longer.End."]
)
def test_sentence_overlap_is_zero_when_even_one_repeated_sentence_would_overflow():
# "Sit." fits budget 4, but "Sit.Longword." would occupy 13 characters.
_assert_sentence_groups(
"Go.Run.Sit.Longword.End.", 12, 4, ["Go.Run.Sit.", "Longword.", "End."]
)
@pytest.mark.parametrize(
"overlap, expected",
[
pytest.param(
8, ["One. Two. Six.", "Six. Ten. End."], id="separator-over-budget"
),
pytest.param(
9,
["One. Two. Six.", "Two. Six. Ten.", "Six. Ten. End."],
id="separator-included-in-budget",
),
],
)
def test_sentence_overlap_counts_internal_whitespace(overlap, expected):
# "Two. Six." is nine characters, not eight; leading/trailing spaces are ignored.
_assert_sentence_groups("One. Two. Six. Ten. End.", 14, overlap, expected)
def test_sentence_overlap_handles_mixed_sentence_terminators():
_assert_sentence_groups(
"One!Two?Six.Ten!End?", 12, 4, ["One!Two?Six.", "Six.Ten!End?"]
)
def test_large_sentence_overlap_budget_still_makes_progress():
# The first group is 11 characters and the budget is also 11. A naive
# end - overlap update can get stuck at zero. Isolate that risk so pytest
# can kill a non-terminating implementation instead of hanging indefinitely.
script = textwrap.dedent("""
import json
from semantica.split.sliding_window_chunker import SlidingWindowChunker
chunker = SlidingWindowChunker(chunk_size=12, overlap=11)
chunker.progress_tracker.enabled = False
chunks = chunker.chunk("Go.Run.Sit.Longer.End.", preserve_boundaries=True)
print(json.dumps([
[chunk.text, chunk.start_index, chunk.end_index] for chunk in chunks
]))
""")
result = subprocess.run(
[sys.executable, "-c", script],
cwd=Path(__file__).resolve().parents[2],
capture_output=True,
text=True,
check=True,
timeout=180, # covers a cold `import semantica` on slow CI runners
)
assert json.loads(result.stdout) == [
["Go.Run.Sit.", 0, 11],
["Sit.Longer.", 7, 18],
["Longer.End.", 11, 22],
]
@pytest.mark.parametrize(
"preserve_boundaries", [False, True], ids=["fixed", "fallback"]
)
def test_long_sentence_keeps_character_overlap(preserve_boundaries):
text = "abcdefghijklmnopqrstuvwxy."
chunks = SlidingWindowChunker(chunk_size=8, overlap=3).chunk(
text, preserve_boundaries=preserve_boundaries
)
_assert_source_coverage(text, chunks, 8)
assert len(chunks) >= 2
assert [(chunk.start_index, chunk.end_index) for chunk in chunks[:2]] == [
(0, 8),
(5, 13),
]
for previous, current in zip(chunks, chunks[1:]):
assert current.start_index > previous.start_index
shared = min(3, len(previous.text), len(current.text))
assert previous.text[-shared:] == current.text[:shared]
@pytest.mark.parametrize(
"text, expected",
[
("abc def ghi.", [(0, 3), (4, 7), (8, 12)]),
("ab cd.", [(0, 2), (5, 8)]),
("ab cd.", [(0, 2), (8, 11)]),
("abc\tdef\u2003ghi.", [(0, 3), (4, 7), (8, 12)]),
(" \tab cd. ", [(2, 4), (7, 10)]),
],
)
def test_long_sentence_fallback_offsets_match_trimmed_text(text, expected):
chunks = SlidingWindowChunker(chunk_size=4, overlap=0).chunk(text)
_assert_source_coverage(text, chunks, 4)
assert [(chunk.start_index, chunk.end_index) for chunk in chunks] == expected
assert [chunk.metadata["chunk_index"] for chunk in chunks] == list(
range(len(chunks))
)
assert all(chunk.metadata["chunk_size"] == len(chunk.text) for chunk in chunks)
@pytest.mark.parametrize(
"text, expected, has_overlap",
[
("abc def.", [(0, 3), (8, 12)], [False, False]),
("ab cdef ghi.", [(0, 7), (5, 12)], [False, True]),
],
)
def test_long_sentence_fallback_overlap_uses_trimmed_offsets(
text, expected, has_overlap
):
chunks = SlidingWindowChunker(chunk_size=8, overlap=3).chunk(text)
_assert_source_coverage(text, chunks, 8)
assert [(chunk.start_index, chunk.end_index) for chunk in chunks] == expected
assert [chunk.metadata["has_overlap"] for chunk in chunks] == has_overlap
@pytest.mark.parametrize(
"text, overlap, stride, expected",
[
("abcdefghijkl.", 3, None, [(0, 8), (5, 13)]),
("abcdefghijklmnop.", 3, None, [(0, 8), (5, 13), (10, 17)]),
("abcdefghijklm.", 3, None, [(0, 8), (5, 13), (10, 14)]),
("abcdefghi.", 7, None, [(0, 8), (1, 9), (2, 10)]),
("abcdefghijkl.", 0, 5, [(0, 8), (5, 13)]),
("abcdefghijkl", 3, None, [(0, 8), (5, 12)]),
],
ids=[
"exact-end",
"partial-end",
"required-tail",
"high-overlap",
"stride",
"unterminated",
],
)
def test_long_sentence_fallback_stops_after_covering_sentence(
text, overlap, stride, expected
):
config = {"chunk_size": 8, "overlap": overlap}
if stride is not None:
config["stride"] = stride
chunks = SlidingWindowChunker(**config).chunk(text)
_assert_source_coverage(text, chunks, 8)
assert [(chunk.start_index, chunk.end_index) for chunk in chunks] == expected
assert all(not chunk.metadata["boundary_preserved"] for chunk in chunks)
def test_long_sentence_fallback_returns_to_complete_sentence_groups():
long_sentence = "abcdefghijklmnopqrstuvwxy."
text = "Hi." + long_sentence + "Bye.End."
chunks = SlidingWindowChunker(chunk_size=8, overlap=3).chunk(text)
_assert_source_coverage(text, chunks, 8)
assert chunks[0].text == "Hi."
assert chunks[-1].text == "Bye.End."
assert chunks[0].metadata["boundary_preserved"] is True
assert chunks[-1].metadata["boundary_preserved"] is True
fragments = chunks[1:-1]
assert [(chunk.start_index, chunk.end_index) for chunk in fragments] == [
(3, 11),
(8, 16),
(13, 21),
(18, 26),
(23, 29),
]
for fragment in fragments:
assert 3 <= fragment.start_index < fragment.end_index <= 3 + len(long_sentence)
assert fragment.metadata["boundary_preserved"] is False
# A fragment of the long sentence must not be repeated in the next sentence group.
assert chunks[-1].metadata["has_overlap"] is False
def test_unterminated_final_sentence_is_kept_as_a_complete_unit():
_assert_sentence_groups("One.Two.Tail", 8, 4, ["One.Two.", "Two.Tail"])
def test_sentence_groups_preserve_punctuation_runs_and_internal_newlines():
_assert_sentence_groups(
" One?!\n\nTwo.\nTail ", 11, 4, ["One?!\n\nTwo.", "Two.\nTail"]
)
@pytest.mark.parametrize("text", [" \t ", "\n\n \r\n"])
def test_whitespace_only_input_does_not_emit_sentence_groups(text):
assert SlidingWindowChunker(chunk_size=8, overlap=3).chunk(text) == []
@pytest.mark.parametrize("stride", [1, 20])
def test_sentence_grouping_takes_precedence_over_custom_stride(stride):
text = "One.Two.Six.Ten.End."
chunker = SlidingWindowChunker(chunk_size=12, overlap=4, stride=stride)
chunks = chunker.chunk(text)
_assert_source_coverage(text, chunks, 12)
assert [chunk.text for chunk in chunks] == ["One.Two.Six.", "Six.Ten.End."]
assert chunker.stride == stride
@pytest.mark.parametrize(
"preserve_boundaries", [False, True], ids=["fixed", "fallback"]
)
def test_character_windows_use_custom_stride(preserve_boundaries):
text = "abcdefghijklmnopqrstuvwxy."
chunks = SlidingWindowChunker(chunk_size=8, overlap=3, stride=2).chunk(
text, preserve_boundaries=preserve_boundaries
)
_assert_source_coverage(text, chunks, 8)
assert [(chunk.start_index, chunk.end_index) for chunk in chunks[:3]] == [
(0, 8),
(2, 10),
(4, 12),
]
# Explicit stride 2 determines six shared characters, overriding the default step.
assert chunks[0].text[-6:] == chunks[1].text[:6]
@pytest.mark.parametrize("stride", [8, 20, 64])
@pytest.mark.parametrize(
"text",
[
"abcdefghijklmnopqrstuvwxy.",
"Hi.abcdefghijklmnopqrstuvwxy.Bye.End.",
"abcdefghijklmnopqrstuvwxy",
],
ids=["long-sentence", "mixed-sentences", "unterminated"],
)
def test_long_sentence_fallback_caps_stride_to_preserve_coverage(stride, text):
chunker = SlidingWindowChunker(chunk_size=8, overlap=3, stride=stride)
chunks = chunker.chunk(text)
_assert_source_coverage(text, chunks, 8)
assert "".join(chunk.text for chunk in chunks) == text
assert all(not chunk.metadata["has_overlap"] for chunk in chunks)
assert chunker.stride == stride
@pytest.mark.parametrize(
"text, expected_spans",
[
("abcdefghi", [(0, 8)]),
("abcdefghijklmnopqrstuvwxy.", [(0, 8), (20, 26)]),
],
)
def test_fixed_windows_warn_and_keep_wide_stride_after_fallback(
text, expected_spans, caplog
):
chunker = SlidingWindowChunker(chunk_size=8, overlap=3, stride=20)
chunker.chunk(text)
with caplog.at_level(logging.WARNING, logger=chunker.logger.name):
chunks = chunker.chunk(text, preserve_boundaries=False)
recorded = [
record for record in caplog.records if record.name == chunker.logger.name
]
assert len(recorded) == 1
assert recorded[0].levelno == logging.WARNING
assert "stride" in recorded[0].message
assert "chunk_size" in recorded[0].message
assert [(chunk.start_index, chunk.end_index) for chunk in chunks] == expected_spans
assert [chunk.text for chunk in chunks] == [
text[start:end] for start, end in expected_spans
]
assert chunker.stride == 20
@pytest.mark.parametrize(
"text, stride, preserve_boundaries",
[
("", 20, False),
("abcdefg", 20, False),
("abcdefgh", 20, False),
("abcdefghijklmnopqrstuvwxy.", 5, False),
("abcdefghijklmnopqrstuvwxy.", 8, False),
("abcdefghijklmnopqrstuvwxy.", 20, True),
("One.Two.Six.", 20, True),
],
)
def test_gap_free_chunking_does_not_warn(text, stride, preserve_boundaries, caplog):
chunker = SlidingWindowChunker(chunk_size=8, overlap=3, stride=stride)
with caplog.at_level(logging.WARNING, logger=chunker.logger.name):
chunker.chunk(text, preserve_boundaries=preserve_boundaries)
assert not [
record for record in caplog.records if record.name == chunker.logger.name
]
@pytest.mark.parametrize("overlap", [0, 8], ids=["zero-overlap", "small-overlap"])
def test_many_short_sentences_do_not_retain_all_offsets(overlap):
# Isolate tracing from pytest's own allocations and any existing tracer.
# A single large chunk isolates auxiliary memory from output-list growth.
script = textwrap.dedent("""
import json
import sys
import tracemalloc
from semantica.split.sliding_window_chunker import SlidingWindowChunker
text = "a." * 50_000
chunker = SlidingWindowChunker(chunk_size=len(text), overlap=int(sys.argv[1]))
chunker._chunk_with_boundaries("Warmup.")
tracemalloc.stop()
tracemalloc.start()
try:
chunks = chunker._chunk_with_boundaries(text)
_, peak = tracemalloc.get_traced_memory()
finally:
tracemalloc.stop()
assert len(chunks) == 1
assert chunks[0].text == text
assert (chunks[0].start_index, chunks[0].end_index) == (0, len(text))
print(json.dumps({"peak": peak, "text_length": len(text)}))
""")
result = subprocess.run(
[sys.executable, "-c", script, str(overlap)],
cwd=Path(__file__).resolve().parents[2],
capture_output=True,
text=True,
check=True,
timeout=180, # covers a cold `import semantica` on slow CI runners
)
measurement = json.loads(result.stdout)
# Allow several text copies plus fixed overhead, but not one Python offset
# per sentence. Measure peak usage since the queue is freed before return.
assert measurement["peak"] < 4 * measurement["text_length"] + 128 * 1024
def test_sliding_window_method_covers_long_sentences_with_wide_stride():
text = "abcdefghijklmnopqrstuvwxy."
chunks = split_sliding_window(text, chunk_size=8, overlap=3, stride=20)
_assert_source_coverage(text, chunks, 8)
assert "".join(chunk.text for chunk in chunks) == text
assert all(chunk.metadata["boundary_preserved"] is False for chunk in chunks)
@pytest.mark.parametrize("overlap, expected", [(3, [False, False]), (4, [False, True])])
def test_sentence_overlap_metadata_reports_actual_overlap(overlap, expected):
chunks = SlidingWindowChunker(chunk_size=12, overlap=overlap).chunk(
"One.Two.Six.Ten.End."
)
assert [chunk.metadata["has_overlap"] for chunk in chunks] == expected
def test_sliding_window_method_preserves_the_sentence_overlap_contract():
chunks = split_sliding_window(
"One.Two.Six.Ten.End.", chunk_size=12, overlap=4, preserve_boundaries=True
)
assert [chunk.text for chunk in chunks] == ["One.Two.Six.", "Six.Ten.End."]
assert all(chunk.metadata["boundary_preserved"] for chunk in chunks)