Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

test(split): cover untested chunker classes (#864) (#904)

pravit-amp64f6c5cba2ac
diff --git a/semantica/split/methods.py b/semantica/split/methods.py
index 3c3d1193..8c338dc6 100644
--- a/semantica/split/methods.py
+++ b/semantica/split/methods.py
@@ -1180,7 +1180,9 @@ def split_graph_based(
)
entities = ner_extractor.extract(text)
- relations = relation_extractor.extract(text)
+ # Relation extraction requires the extracted entities; failing to pass
+ # them raises and can trigger the broad fallback to recursive splitting.
+ relations = relation_extractor.extract(text, entities)
# Build graph
G = nx.Graph()
@@ -1623,8 +1625,9 @@ def split_sliding_window(
)
try:
+ effective_stride = stride if stride is not None else (chunk_size - overlap)
chunker = SlidingWindowChunker(
- chunk_size=chunk_size, overlap=overlap, stride=stride, **kwargs
+ chunk_size=chunk_size, overlap=overlap, stride=effective_stride, **kwargs
)
return chunker.chunk(text, preserve_boundaries=preserve_boundaries, **kwargs)
except Exception as e:
diff --git a/tests/split/test_chunkers.py b/tests/split/test_chunkers.py
new file mode 100644
index 00000000..7210a9de
--- /dev/null
+++ b/tests/split/test_chunkers.py
@@ -0,0 +1,592 @@
+"""Tests for previously untested split chunker classes (issue #864)."""
+
+import pytest
+
+from semantica.split.kg_chunkers import (
+ EntityAwareChunker,
+ GraphBasedChunker,
+ HierarchicalChunker,
+ OntologyAwareChunker,
+ RelationAwareChunker,
+)
+from semantica.split.methods import (
+ SEMANTIC_EXTRACT_AVAILABLE,
+ NETWORKX_AVAILABLE,
+ split_by_characters,
+ split_by_paragraphs,
+ split_by_sentences,
+ split_by_words,
+ split_entity_aware,
+ split_graph_based,
+ split_hierarchical,
+ split_ontology_aware,
+ split_recursive,
+ split_relation_aware,
+ split_sliding_window,
+ split_structural,
+)
+from semantica.split.semantic_chunker import Chunk
+from semantica.split.sliding_window_chunker import SlidingWindowChunker
+from semantica.split.structural_chunker import StructuralChunker, StructuralElement
+from semantica.split.table_chunker import TableChunk, TableChunker
+from semantica.utils.exceptions import ValidationError
+
+requires_semantic_extract = pytest.mark.skipif(
+ not SEMANTIC_EXTRACT_AVAILABLE,
+ reason="semantica.semantic_extract is not available",
+)
+requires_networkx = pytest.mark.skipif(
+ not NETWORKX_AVAILABLE,
+ reason="networkx is not available",
+)
+
+
+# ---------------------------------------------------------------------------
+# SlidingWindowChunker
+# ---------------------------------------------------------------------------
+
+
+class TestSlidingWindowChunker:
+ def test_init_defaults_and_validation(self):
+ chunker = SlidingWindowChunker(chunk_size=100, overlap=20)
+ assert chunker.chunk_size == 100
+ assert chunker.overlap == 20
+ assert chunker.stride == 80
+
+ with pytest.raises(ValidationError):
+ SlidingWindowChunker(chunk_size=0)
+ with pytest.raises(ValidationError):
+ SlidingWindowChunker(chunk_size=100, overlap=-1)
+ with pytest.raises(ValidationError):
+ SlidingWindowChunker(chunk_size=100, overlap=100)
+
+ def test_empty_text_returns_empty(self):
+ chunker = SlidingWindowChunker(chunk_size=50, overlap=10)
+ assert chunker.chunk("") == []
+
+ def test_fixed_size_overlap_invariant(self):
+ """Last `overlap` chars of chunk N appear at the start of chunk N+1."""
+ text = "abcdefghijklmnopqrstuvwxyz0123456789" * 3 # 108 chars
+ overlap = 10
+ chunk_size = 30
+ chunker = SlidingWindowChunker(
+ chunk_size=chunk_size, overlap=overlap, stride=chunk_size - overlap
+ )
+ chunks = chunker.chunk(text, preserve_boundaries=False)
+
+ assert len(chunks) >= 2
+ for i in range(len(chunks) - 1):
+ # Final chunk may be shorter than overlap; compare shared window only
+ shared = min(overlap, len(chunks[i].text), len(chunks[i + 1].text))
+ expected_overlap = chunks[i].text[-shared:]
+ actual_prefix = chunks[i + 1].text[:shared]
+ assert actual_prefix == expected_overlap, (
+ f"Overlap mismatch between chunk {i} and {i + 1}: "
+ f"{expected_overlap!r} != {actual_prefix!r}"
+ )
+
+ for i in range(len(chunks) - 1):
+ assert (
+ chunks[i + 1].start_index - chunks[i].start_index
+ == chunk_size - overlap
+ )
+
+ def test_chunk_with_overlap_helper(self):
+ text = "word " * 40
+ chunker = SlidingWindowChunker(chunk_size=50, overlap=0)
+ chunks = chunker.chunk_with_overlap(text, overlap_size=15)
+ assert len(chunks) >= 2
+ assert chunker.overlap == 0
+
+ def test_boundary_preservation_avoids_mid_word_when_possible(self):
+ text = (
+ "Alice went to the market. Bob bought apples. "
+ "Carol cooked dinner. Dave drove home."
+ )
+ chunker = SlidingWindowChunker(chunk_size=40, overlap=10)
+ chunks = chunker.chunk(text, preserve_boundaries=True)
+ assert len(chunks) >= 1
+ for chunk in chunks:
+ assert isinstance(chunk, Chunk)
+ assert chunk.text
+ assert chunk.metadata.get("chunk_index") is not None
+
+
+# ---------------------------------------------------------------------------
+# StructuralChunker
+# ---------------------------------------------------------------------------
+
+
+class TestStructuralChunker:
+ MARKDOWN_DOC = """# Introduction
+
+This is the intro paragraph about the project.
+
+## Details
+
+Here are more details about how it works.
+
+- item one
+- item two
+- item three
+
+## Conclusion
+
+Final thoughts on the subject.
+"""
+
+ def test_empty_text_returns_empty(self):
+ chunker = StructuralChunker(max_chunk_size=500)
+ assert chunker.chunk("") == []
+
+ def test_heading_based_splits(self):
+ chunker = StructuralChunker(respect_headers=True, max_chunk_size=200)
+ chunks = chunker.chunk(self.MARKDOWN_DOC)
+
+ assert len(chunks) >= 1
+ for chunk in chunks:
+ assert isinstance(chunk, Chunk)
+ assert chunk.metadata.get("structure_preserved") is True
+ assert "element_types" in chunk.metadata
+
+ all_types = []
+ for chunk in chunks:
+ all_types.extend(chunk.metadata["element_types"])
+ assert "heading" in all_types
+ assert "paragraph" in all_types
+
+ def test_heading_boundaries_separate_sections(self):
+ """Distinct top-level headings must not be merged into one chunk."""
+ doc = """# Alpha
+
+Content exclusively about alpha topic here.
+
+# Beta
+
+Content exclusively about beta topic here.
+"""
+ chunker = StructuralChunker(respect_headers=True, max_chunk_size=50)
+ chunks = chunker.chunk(doc)
+
+ assert len(chunks) >= 2
+ alpha_chunks = [c for c in chunks if "exclusively about alpha" in c.text]
+ beta_chunks = [c for c in chunks if "exclusively about beta" in c.text]
+ assert alpha_chunks, "Alpha section body missing from chunks"
+ assert beta_chunks, "Beta section body missing from chunks"
+
+ # Heading-boundary invariant: alpha and beta bodies stay in separate chunks
+ for chunk in chunks:
+ has_alpha = "exclusively about alpha" in chunk.text
+ has_beta = "exclusively about beta" in chunk.text
+ assert not (has_alpha and has_beta), (
+ f"Sections merged across heading boundary: {chunk.text!r}"
+ )
+
+ def test_extract_structure_detects_headings_and_lists(self):
+ chunker = StructuralChunker()
+ elements = chunker._extract_structure(self.MARKDOWN_DOC)
+ types = [e.type for e in elements]
+ assert "heading" in types
+ assert "list" in types
+ assert "paragraph" in types
+ assert all(isinstance(e, StructuralElement) for e in elements)
+
+ def test_code_block_preserved(self):
+ text = """# Code
+
+```python
+def hello():
+ return "world"
+```
+
+After the code.
+"""
+ chunker = StructuralChunker(max_chunk_size=2000)
+ elements = chunker._extract_structure(text)
+ types = [e.type for e in elements]
+ assert "code_block" in types
+ code = next(e for e in elements if e.type == "code_block")
+ assert "def hello" in code.text
+
+
+# ---------------------------------------------------------------------------
+# TableChunker
+# ---------------------------------------------------------------------------
+
+
+class TestTableChunker:
+ def _sample_table(self, n_rows: int = 10):
+ headers = ["Name", "Age", "City"]
+ rows = [[f"Person{i}", str(20 + i), f"City{i}"] for i in range(n_rows)]
+ return {"headers": headers, "rows": rows}
+
+ def test_rows_are_not_split_mid_row(self):
+ """Each chunk contains complete rows only — never a partial row."""
+ table = self._sample_table(10)
+ chunker = TableChunker(max_rows=3, preserve_headers=True)
+ chunks = chunker.chunk_table(table)
+
+ assert len(chunks) == 4 # 3+3+3+1
+ for chunk in chunks:
+ assert isinstance(chunk, TableChunk)
+ assert chunk.headers == ["Name", "Age", "City"]
+ for row in chunk.rows:
+ assert len(row) == 3
+ assert chunk.metadata["row_count"] == len(chunk.rows)
+
+ flattened = [row for c in chunks for row in c.rows]
+ assert flattened == table["rows"]
+
+ def test_markdown_table_chunk_does_not_split_rows(self):
+ md = """| Name | Age | City |
+| --- | --- | --- |
+| Alice | 30 | NYC |
+| Bob | 25 | LA |
+| Carol | 40 | SF |
+| Dave | 35 | CHI |
+"""
+ chunker = TableChunker(max_rows=2, preserve_headers=True)
+ chunks = chunker.chunk(md)
+
+ assert len(chunks) == 2
+ for chunk in chunks:
+ assert chunk.metadata["chunk_type"] == "table"
+ data_lines = [
+ line
+ for line in chunk.text.split("\n")
+ if line and "---" not in line and not line.startswith("Name")
+ ]
+ for line in data_lines:
+ cells = [c.strip() for c in line.split("|")]
+ assert len(cells) == 3
+
+ def test_non_table_text_returns_single_chunk(self):
+ chunker = TableChunker()
+ chunks = chunker.chunk("Just plain text without a table.")
+ assert len(chunks) == 1
+ assert chunks[0].metadata.get("error") == "No table found"
+
+ def test_extract_table_schema(self):
+ table = {
+ "headers": ["id", "active", "label"],
+ "rows": [
+ ["1", "true", "alpha"],
+ ["2", "false", "beta"],
+ ],
+ }
+ schema = TableChunker().extract_table_schema(table)
+ assert schema["column_count"] == 3
+ assert schema["row_count"] == 2
+ assert schema["column_types"]["id"] == "numeric"
+ assert schema["column_types"]["active"] == "boolean"
+ assert schema["column_types"]["label"] == "text"
+
+ def test_chunk_by_columns(self):
+ table = self._sample_table(3)
+ chunker = TableChunker(chunk_by_columns=True, preserve_headers=True)
+ chunks = chunker.chunk_table(table, max_columns=2)
+ assert len(chunks) == 2
+ assert chunks[0].headers == ["Name", "Age"]
+ assert chunks[1].headers == ["City"]
+ for chunk in chunks:
+ for row in chunk.rows:
+ assert len(row) == len(chunk.headers)
+
+
+# ---------------------------------------------------------------------------
+# EntityAwareChunker (real optional deps via importorskip / skipif)
+# ---------------------------------------------------------------------------
+
+
+class TestEntityAwareChunker:
+ def test_init(self):
+ chunker = EntityAwareChunker(
+ chunk_size=500, chunk_overlap=50, ner_method="pattern"
+ )
+ assert chunker.chunk_size == 500
+ assert chunker.ner_method == "pattern"
+ assert chunker.preserve_entities is True
+
+ def test_empty_text(self):
+ chunker = EntityAwareChunker(chunk_size=100, ner_method="pattern")
+ chunks = chunker.chunk("")
+ assert isinstance(chunks, list)
+
+ @requires_semantic_extract
+ def test_entity_boundaries_preserved_with_pattern_ner(self):
+ """Entity spans stay intact when using real pattern NER."""
+ pytest.importorskip("semantica.semantic_extract")
+ entity_text = "AppleInc"
+ # Use a contiguous token the pattern NER can latch onto
+ text = (
+ "Intro sentence one goes here. Intro sentence two goes here. "
+ f"{entity_text} was founded in Cupertino California recently. "
+ "More filler sentences keep the document long enough to chunk. "
+ "Yet another sentence about products and services worldwide. "
+ "Final sentence for padding the overall document length out."
+ )
+ chunks = split_entity_aware(
+ text,
+ chunk_size=90,
+ ner_method="pattern",
+ preserve_entities=True,
+ )
+ assert len(chunks) >= 1
+ containing = [c for c in chunks if entity_text in c.text]
+ assert containing, "Expected entity text to appear in at least one chunk"
+ for chunk in containing:
+ idx = chunk.text.index(entity_text)
+ assert chunk.text[idx : idx + len(entity_text)] == entity_text
+
+ @requires_semantic_extract
+ def test_entity_aware_chunker_with_pattern_ner(self):
+ pytest.importorskip("semantica.semantic_extract")
+ text = (
+ "Alice Johnson founded Acme Corporation in New York. "
+ "Bob Smith joined the company later. "
+ "They expanded operations across Europe and Asia. "
+ ) * 5
+ chunker = EntityAwareChunker(
+ chunk_size=120, ner_method="pattern", preserve_entities=True
+ )
+ chunks = chunker.chunk(text)
+ assert len(chunks) >= 1
+ assert all(isinstance(c, Chunk) for c in chunks)
+
+
+# ---------------------------------------------------------------------------
+# RelationAware / GraphBased / OntologyAware / Hierarchical
+# ---------------------------------------------------------------------------
+
+
+class TestRelationAwareChunker:
+ def test_init(self):
+ chunker = RelationAwareChunker(chunk_size=100, relation_method="pattern")
+ assert chunker.chunk_size == 100
+ assert chunker.relation_method == "pattern"
+
+ @requires_semantic_extract
+ def test_chunk_with_pattern_extractors(self):
+ pytest.importorskip("semantica.semantic_extract")
+ text = (
+ "Alice works at Acme. Bob reports to Alice. "
+ "Carol founded Acme in 2010. More padding text follows here. "
+ ) * 4
+ chunker = RelationAwareChunker(
+ chunk_size=100, relation_method="pattern", ner_method="pattern"
+ )
+ chunks = chunker.chunk(text)
+ assert isinstance(chunks, list)
+ assert len(chunks) >= 1
+ assert all(isinstance(c, Chunk) for c in chunks)
+
+
+class TestGraphBasedChunker:
+ def test_init(self):
+ chunker = GraphBasedChunker(
+ chunk_size=500, strategy="community", algorithm="louvain"
+ )
+ assert chunker.strategy == "community"
+ assert chunker.algorithm == "louvain"
+
+ @requires_semantic_extract
+ @requires_networkx
+ def test_chunk_with_real_optional_deps(self):
+ pytest.importorskip("networkx")
+ pytest.importorskip("semantica.semantic_extract")
+ text = (
+ "Alice met Bob at Acme Corporation yesterday afternoon. "
+ "Bob introduced Carol to the Acme engineering team. "
+ "Carol and Alice later discussed graph-based retrieval methods. "
+ ) * 3
+ chunker = GraphBasedChunker(
+ chunk_size=200,
+ strategy="community",
+ algorithm="louvain",
+ ner_method="pattern",
+ relation_method="pattern",
+ )
+ chunks = chunker.chunk(text)
+ assert len(chunks) >= 1
+ assert all(isinstance(c, Chunk) for c in chunks)
+ # Ensure the graph-based path actually ran (not fallback-to-recursive).
+ assert any(
+ c.metadata.get("method") == "graph_based" for c in chunks
+ ), "Expected at least one graph_based chunk"
+ assert any(
+ c.metadata.get("strategy") == "community"
+ and c.metadata.get("algorithm") == "louvain"
+ for c in chunks
+ ), "Expected graph_based chunk metadata to include strategy/algorithm"
+ assert not any(
+ c.metadata.get("method") == "recursive" for c in chunks
+ ), "Graph-based fallback to recursive was triggered"
+
+
+class TestOntologyAwareChunker:
+ def test_init(self):
+ chunker = OntologyAwareChunker(chunk_size=200, preserve_concepts=True)
+ assert chunker.chunk_size == 200
+ assert chunker.preserve_concepts is True
+
+ @requires_semantic_extract
+ def test_chunk_uses_entity_aware_path(self):
+ pytest.importorskip("semantica.semantic_extract")
+ text = "Concept Alpha relates to Concept Beta in the taxonomy. " * 8
+ chunker = OntologyAwareChunker(
+ chunk_size=120, preserve_concepts=True, ner_method="pattern"
+ )
+ chunks = chunker.chunk(text)
+ assert len(chunks) >= 1
+ assert all(isinstance(c, Chunk) for c in chunks)
+
+
+class TestHierarchicalChunker:
+ def test_hierarchical_markdown_sections(self):
+ text = """# Section One
+
+Paragraph under section one with enough content to matter.
+
+# Section Two
+
+Paragraph under section two also with sufficient content.
+"""
+ chunker = HierarchicalChunker(
+ levels=["section", "paragraph"], chunk_sizes=[2000, 500]
+ )
+ chunks = chunker.chunk(text)
+ assert len(chunks) >= 1
+ for chunk in chunks:
+ assert chunk.metadata.get("hierarchical") is True
+ assert chunk.metadata.get("levels") == ["section", "paragraph"]
+
+ def test_split_hierarchical_function(self):
+ text = "Para one.\n\nPara two.\n\nPara three."
+ chunks = split_hierarchical(text, levels=["paragraph"], chunk_sizes=[1000])
+ assert len(chunks) >= 1
+
+
+# ---------------------------------------------------------------------------
+# Exported method functions (public API smoke coverage)
+# ---------------------------------------------------------------------------
+
+
+class TestSplitMethodFunctions:
+ SAMPLE = (
+ "First sentence about knowledge graphs. "
+ "Second sentence covers entity extraction. "
+ "Third sentence discusses relation awareness. "
+ "Fourth sentence wraps up the example."
+ )
+
+ MARKDOWN = """# Intro
+
+Intro paragraph with enough text to matter for structural splitting.
+
+# Body
+
+Body paragraph under a distinct heading for separation checks.
+"""
+
+ def test_split_recursive(self):
+ chunks = split_recursive(self.SAMPLE, chunk_size=60)
+ assert len(chunks) >= 1
+ assert all(isinstance(c, Chunk) for c in chunks)
+
+ def test_split_by_sentences(self):
+ chunks = split_by_sentences(self.SAMPLE, chunk_size=80)
+ assert len(chunks) >= 1
+
+ def test_split_by_paragraphs(self):
+ text = "Para A content here.\n\nPara B content here.\n\nPara C content here."
+ chunks = split_by_paragraphs(text, chunk_size=50)
+ assert len(chunks) >= 1
+
+ def test_split_by_characters(self):
+ chunks = split_by_characters(self.SAMPLE, chunk_size=40)
+ assert len(chunks) >= 2
+
+ def test_split_by_words(self):
+ chunks = split_by_words(self.SAMPLE, chunk_size=10)
+ assert len(chunks) >= 1
+
+ def test_split_structural(self):
+ chunks = split_structural(
+ self.MARKDOWN, max_chunk_size=80, respect_headers=True
+ )
+ assert len(chunks) >= 2
+ assert all(isinstance(c, Chunk) for c in chunks)
+
+ def test_split_sliding_window(self):
+ chunks = split_sliding_window(
+ self.SAMPLE * 3,
+ chunk_size=40,
+ overlap=10,
+ preserve_boundaries=False,
+ )
+ assert len(chunks) >= 2
+ assert all(isinstance(c, Chunk) for c in chunks)
+ # Verify the sliding-window path was taken, not the recursive fallback.
+ # chunks[1].metadata["has_overlap"] is set only by SlidingWindowChunker.
+ assert chunks[1].metadata.get("has_overlap") is True, (
+ "Expected sliding-window chunks to carry has_overlap=True; "
+ "fallback to recursive may have occurred"
+ )
+ assert chunks[1].metadata.get("method") != "recursive", (
+ "Sliding-window fallback to recursive was triggered unexpectedly"
+ )
+
+ @requires_semantic_extract
+ def test_split_entity_aware(self):
+ pytest.importorskip("semantica.semantic_extract")
+ chunks = split_entity_aware(
+ self.SAMPLE * 3, chunk_size=80, ner_method="pattern"
+ )
+ assert len(chunks) >= 1
+
+ @requires_semantic_extract
+ def test_split_relation_aware(self):
+ pytest.importorskip("semantica.semantic_extract")
+ chunks = split_relation_aware(
+ self.SAMPLE * 3,
+ chunk_size=80,
+ relation_method="pattern",
+ ner_method="pattern",
+ )
+ assert len(chunks) >= 1
+
+ @requires_semantic_extract
+ @requires_networkx
+ def test_split_graph_based(self):
+ pytest.importorskip("networkx")
+ pytest.importorskip("semantica.semantic_extract")
+ chunks = split_graph_based(
+ self.SAMPLE * 3,
+ chunk_size=120,
+ strategy="community",
+ algorithm="louvain",
+ ner_method="pattern",
+ relation_method="pattern",
+ )
+ assert len(chunks) >= 1
+ assert all(isinstance(c, Chunk) for c in chunks)
+ # Ensure we didn't satisfy the test via the broad recursive fallback.
+ assert any(
+ c.metadata.get("method") == "graph_based" for c in chunks
+ ), "Expected at least one graph_based chunk"
+ assert any(
+ c.metadata.get("strategy") == "community"
+ and c.metadata.get("algorithm") == "louvain"
+ for c in chunks
+ ), "Expected graph_based chunk metadata to include strategy/algorithm"
+ assert not any(
+ c.metadata.get("method") == "recursive" for c in chunks
+ ), "Graph-based fallback to recursive was triggered"
+
+ @requires_semantic_extract
+ def test_split_ontology_aware(self):
+ pytest.importorskip("semantica.semantic_extract")
+ chunks = split_ontology_aware(
+ self.SAMPLE * 3, chunk_size=80, ner_method="pattern"
+ )
+ assert len(chunks) >= 1