Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
test(split): cover untested chunker classes (#864) (#904)
pravit-amp64f6c5cba2ac
diff --git a/semantica/split/methods.py b/semantica/split/methods.pyindex 3c3d1193..8c338dc6 100644--- a/semantica/split/methods.py+++ b/semantica/split/methods.py@@ -1180,7 +1180,9 @@ def split_graph_based()entities = ner_extractor.extract(text)- relations = relation_extractor.extract(text)+ # Relation extraction requires the extracted entities; failing to pass+ # them raises and can trigger the broad fallback to recursive splitting.+ relations = relation_extractor.extract(text, entities)# Build graphG = nx.Graph()@@ -1623,8 +1625,9 @@ def split_sliding_window()try:+ effective_stride = stride if stride is not None else (chunk_size - overlap)chunker = SlidingWindowChunker(- chunk_size=chunk_size, overlap=overlap, stride=stride, **kwargs+ chunk_size=chunk_size, overlap=overlap, stride=effective_stride, **kwargs)return chunker.chunk(text, preserve_boundaries=preserve_boundaries, **kwargs)except Exception as e:diff --git a/tests/split/test_chunkers.py b/tests/split/test_chunkers.pynew file mode 100644index 00000000..7210a9de--- /dev/null+++ b/tests/split/test_chunkers.py@@ -0,0 +1,592 @@+"""Tests for previously untested split chunker classes (issue #864)."""++import pytest++from semantica.split.kg_chunkers import (+ EntityAwareChunker,+ GraphBasedChunker,+ HierarchicalChunker,+ OntologyAwareChunker,+ RelationAwareChunker,+)+from semantica.split.methods import (+ SEMANTIC_EXTRACT_AVAILABLE,+ NETWORKX_AVAILABLE,+ split_by_characters,+ split_by_paragraphs,+ split_by_sentences,+ split_by_words,+ split_entity_aware,+ split_graph_based,+ split_hierarchical,+ split_ontology_aware,+ split_recursive,+ split_relation_aware,+ split_sliding_window,+ split_structural,+)+from semantica.split.semantic_chunker import Chunk+from semantica.split.sliding_window_chunker import SlidingWindowChunker+from semantica.split.structural_chunker import StructuralChunker, StructuralElement+from semantica.split.table_chunker import TableChunk, TableChunker+from semantica.utils.exceptions import ValidationError++requires_semantic_extract = pytest.mark.skipif(+ not SEMANTIC_EXTRACT_AVAILABLE,+ reason="semantica.semantic_extract is not available",+)+requires_networkx = pytest.mark.skipif(+ not NETWORKX_AVAILABLE,+ reason="networkx is not available",+)+++# ---------------------------------------------------------------------------+# SlidingWindowChunker+# ---------------------------------------------------------------------------+++class TestSlidingWindowChunker:+ def test_init_defaults_and_validation(self):+ chunker = SlidingWindowChunker(chunk_size=100, overlap=20)+ assert chunker.chunk_size == 100+ assert chunker.overlap == 20+ assert chunker.stride == 80++ with pytest.raises(ValidationError):+ SlidingWindowChunker(chunk_size=0)+ with pytest.raises(ValidationError):+ SlidingWindowChunker(chunk_size=100, overlap=-1)+ with pytest.raises(ValidationError):+ SlidingWindowChunker(chunk_size=100, overlap=100)++ def test_empty_text_returns_empty(self):+ chunker = SlidingWindowChunker(chunk_size=50, overlap=10)+ assert chunker.chunk("") == []++ def test_fixed_size_overlap_invariant(self):+ """Last `overlap` chars of chunk N appear at the start of chunk N+1."""+ text = "abcdefghijklmnopqrstuvwxyz0123456789" * 3 # 108 chars+ overlap = 10+ chunk_size = 30+ chunker = SlidingWindowChunker(+ chunk_size=chunk_size, overlap=overlap, stride=chunk_size - overlap+ )+ chunks = chunker.chunk(text, preserve_boundaries=False)++ assert len(chunks) >= 2+ for i in range(len(chunks) - 1):+ # Final chunk may be shorter than overlap; compare shared window only+ shared = min(overlap, len(chunks[i].text), len(chunks[i + 1].text))+ expected_overlap = chunks[i].text[-shared:]+ actual_prefix = chunks[i + 1].text[:shared]+ assert actual_prefix == expected_overlap, (+ f"Overlap mismatch between chunk {i} and {i + 1}: "+ f"{expected_overlap!r} != {actual_prefix!r}"+ )++ for i in range(len(chunks) - 1):+ assert (+ chunks[i + 1].start_index - chunks[i].start_index+ == chunk_size - overlap+ )++ def test_chunk_with_overlap_helper(self):+ text = "word " * 40+ chunker = SlidingWindowChunker(chunk_size=50, overlap=0)+ chunks = chunker.chunk_with_overlap(text, overlap_size=15)+ assert len(chunks) >= 2+ assert chunker.overlap == 0++ def test_boundary_preservation_avoids_mid_word_when_possible(self):+ text = (+ "Alice went to the market. Bob bought apples. "+ "Carol cooked dinner. Dave drove home."+ )+ chunker = SlidingWindowChunker(chunk_size=40, overlap=10)+ chunks = chunker.chunk(text, preserve_boundaries=True)+ assert len(chunks) >= 1+ for chunk in chunks:+ assert isinstance(chunk, Chunk)+ assert chunk.text+ assert chunk.metadata.get("chunk_index") is not None+++# ---------------------------------------------------------------------------+# StructuralChunker+# ---------------------------------------------------------------------------+++class TestStructuralChunker:+ MARKDOWN_DOC = """# Introduction++This is the intro paragraph about the project.++## Details++Here are more details about how it works.++- item one+- item two+- item three++## Conclusion++Final thoughts on the subject.+"""++ def test_empty_text_returns_empty(self):+ chunker = StructuralChunker(max_chunk_size=500)+ assert chunker.chunk("") == []++ def test_heading_based_splits(self):+ chunker = StructuralChunker(respect_headers=True, max_chunk_size=200)+ chunks = chunker.chunk(self.MARKDOWN_DOC)++ assert len(chunks) >= 1+ for chunk in chunks:+ assert isinstance(chunk, Chunk)+ assert chunk.metadata.get("structure_preserved") is True+ assert "element_types" in chunk.metadata++ all_types = []+ for chunk in chunks:+ all_types.extend(chunk.metadata["element_types"])+ assert "heading" in all_types+ assert "paragraph" in all_types++ def test_heading_boundaries_separate_sections(self):+ """Distinct top-level headings must not be merged into one chunk."""+ doc = """# Alpha++Content exclusively about alpha topic here.++# Beta++Content exclusively about beta topic here.+"""+ chunker = StructuralChunker(respect_headers=True, max_chunk_size=50)+ chunks = chunker.chunk(doc)++ assert len(chunks) >= 2+ alpha_chunks = [c for c in chunks if "exclusively about alpha" in c.text]+ beta_chunks = [c for c in chunks if "exclusively about beta" in c.text]+ assert alpha_chunks, "Alpha section body missing from chunks"+ assert beta_chunks, "Beta section body missing from chunks"++ # Heading-boundary invariant: alpha and beta bodies stay in separate chunks+ for chunk in chunks:+ has_alpha = "exclusively about alpha" in chunk.text+ has_beta = "exclusively about beta" in chunk.text+ assert not (has_alpha and has_beta), (+ f"Sections merged across heading boundary: {chunk.text!r}"+ )++ def test_extract_structure_detects_headings_and_lists(self):+ chunker = StructuralChunker()+ elements = chunker._extract_structure(self.MARKDOWN_DOC)+ types = [e.type for e in elements]+ assert "heading" in types+ assert "list" in types+ assert "paragraph" in types+ assert all(isinstance(e, StructuralElement) for e in elements)++ def test_code_block_preserved(self):+ text = """# Code++```python+def hello():+ return "world"+```++After the code.+"""+ chunker = StructuralChunker(max_chunk_size=2000)+ elements = chunker._extract_structure(text)+ types = [e.type for e in elements]+ assert "code_block" in types+ code = next(e for e in elements if e.type == "code_block")+ assert "def hello" in code.text+++# ---------------------------------------------------------------------------+# TableChunker+# ---------------------------------------------------------------------------+++class TestTableChunker:+ def _sample_table(self, n_rows: int = 10):+ headers = ["Name", "Age", "City"]+ rows = [[f"Person{i}", str(20 + i), f"City{i}"] for i in range(n_rows)]+ return {"headers": headers, "rows": rows}++ def test_rows_are_not_split_mid_row(self):+ """Each chunk contains complete rows only — never a partial row."""+ table = self._sample_table(10)+ chunker = TableChunker(max_rows=3, preserve_headers=True)+ chunks = chunker.chunk_table(table)++ assert len(chunks) == 4 # 3+3+3+1+ for chunk in chunks:+ assert isinstance(chunk, TableChunk)+ assert chunk.headers == ["Name", "Age", "City"]+ for row in chunk.rows:+ assert len(row) == 3+ assert chunk.metadata["row_count"] == len(chunk.rows)++ flattened = [row for c in chunks for row in c.rows]+ assert flattened == table["rows"]++ def test_markdown_table_chunk_does_not_split_rows(self):+ md = """| Name | Age | City |+| --- | --- | --- |+| Alice | 30 | NYC |+| Bob | 25 | LA |+| Carol | 40 | SF |+| Dave | 35 | CHI |+"""+ chunker = TableChunker(max_rows=2, preserve_headers=True)+ chunks = chunker.chunk(md)++ assert len(chunks) == 2+ for chunk in chunks:+ assert chunk.metadata["chunk_type"] == "table"+ data_lines = [+ line+ for line in chunk.text.split("\n")+ if line and "---" not in line and not line.startswith("Name")+ ]+ for line in data_lines:+ cells = [c.strip() for c in line.split("|")]+ assert len(cells) == 3++ def test_non_table_text_returns_single_chunk(self):+ chunker = TableChunker()+ chunks = chunker.chunk("Just plain text without a table.")+ assert len(chunks) == 1+ assert chunks[0].metadata.get("error") == "No table found"++ def test_extract_table_schema(self):+ table = {+ "headers": ["id", "active", "label"],+ "rows": [+ ["1", "true", "alpha"],+ ["2", "false", "beta"],+ ],+ }+ schema = TableChunker().extract_table_schema(table)+ assert schema["column_count"] == 3+ assert schema["row_count"] == 2+ assert schema["column_types"]["id"] == "numeric"+ assert schema["column_types"]["active"] == "boolean"+ assert schema["column_types"]["label"] == "text"++ def test_chunk_by_columns(self):+ table = self._sample_table(3)+ chunker = TableChunker(chunk_by_columns=True, preserve_headers=True)+ chunks = chunker.chunk_table(table, max_columns=2)+ assert len(chunks) == 2+ assert chunks[0].headers == ["Name", "Age"]+ assert chunks[1].headers == ["City"]+ for chunk in chunks:+ for row in chunk.rows:+ assert len(row) == len(chunk.headers)+++# ---------------------------------------------------------------------------+# EntityAwareChunker (real optional deps via importorskip / skipif)+# ---------------------------------------------------------------------------+++class TestEntityAwareChunker:+ def test_init(self):+ chunker = EntityAwareChunker(+ chunk_size=500, chunk_overlap=50, ner_method="pattern"+ )+ assert chunker.chunk_size == 500+ assert chunker.ner_method == "pattern"+ assert chunker.preserve_entities is True++ def test_empty_text(self):+ chunker = EntityAwareChunker(chunk_size=100, ner_method="pattern")+ chunks = chunker.chunk("")+ assert isinstance(chunks, list)++ @requires_semantic_extract+ def test_entity_boundaries_preserved_with_pattern_ner(self):+ """Entity spans stay intact when using real pattern NER."""+ pytest.importorskip("semantica.semantic_extract")+ entity_text = "AppleInc"+ # Use a contiguous token the pattern NER can latch onto+ text = (+ "Intro sentence one goes here. Intro sentence two goes here. "+ f"{entity_text} was founded in Cupertino California recently. "+ "More filler sentences keep the document long enough to chunk. "+ "Yet another sentence about products and services worldwide. "+ "Final sentence for padding the overall document length out."+ )+ chunks = split_entity_aware(+ text,+ chunk_size=90,+ ner_method="pattern",+ preserve_entities=True,+ )+ assert len(chunks) >= 1+ containing = [c for c in chunks if entity_text in c.text]+ assert containing, "Expected entity text to appear in at least one chunk"+ for chunk in containing:+ idx = chunk.text.index(entity_text)+ assert chunk.text[idx : idx + len(entity_text)] == entity_text++ @requires_semantic_extract+ def test_entity_aware_chunker_with_pattern_ner(self):+ pytest.importorskip("semantica.semantic_extract")+ text = (+ "Alice Johnson founded Acme Corporation in New York. "+ "Bob Smith joined the company later. "+ "They expanded operations across Europe and Asia. "+ ) * 5+ chunker = EntityAwareChunker(+ chunk_size=120, ner_method="pattern", preserve_entities=True+ )+ chunks = chunker.chunk(text)+ assert len(chunks) >= 1+ assert all(isinstance(c, Chunk) for c in chunks)+++# ---------------------------------------------------------------------------+# RelationAware / GraphBased / OntologyAware / Hierarchical+# ---------------------------------------------------------------------------+++class TestRelationAwareChunker:+ def test_init(self):+ chunker = RelationAwareChunker(chunk_size=100, relation_method="pattern")+ assert chunker.chunk_size == 100+ assert chunker.relation_method == "pattern"++ @requires_semantic_extract+ def test_chunk_with_pattern_extractors(self):+ pytest.importorskip("semantica.semantic_extract")+ text = (+ "Alice works at Acme. Bob reports to Alice. "+ "Carol founded Acme in 2010. More padding text follows here. "+ ) * 4+ chunker = RelationAwareChunker(+ chunk_size=100, relation_method="pattern", ner_method="pattern"+ )+ chunks = chunker.chunk(text)+ assert isinstance(chunks, list)+ assert len(chunks) >= 1+ assert all(isinstance(c, Chunk) for c in chunks)+++class TestGraphBasedChunker:+ def test_init(self):+ chunker = GraphBasedChunker(+ chunk_size=500, strategy="community", algorithm="louvain"+ )+ assert chunker.strategy == "community"+ assert chunker.algorithm == "louvain"++ @requires_semantic_extract+ @requires_networkx+ def test_chunk_with_real_optional_deps(self):+ pytest.importorskip("networkx")+ pytest.importorskip("semantica.semantic_extract")+ text = (+ "Alice met Bob at Acme Corporation yesterday afternoon. "+ "Bob introduced Carol to the Acme engineering team. "+ "Carol and Alice later discussed graph-based retrieval methods. "+ ) * 3+ chunker = GraphBasedChunker(+ chunk_size=200,+ strategy="community",+ algorithm="louvain",+ ner_method="pattern",+ relation_method="pattern",+ )+ chunks = chunker.chunk(text)+ assert len(chunks) >= 1+ assert all(isinstance(c, Chunk) for c in chunks)+ # Ensure the graph-based path actually ran (not fallback-to-recursive).+ assert any(+ c.metadata.get("method") == "graph_based" for c in chunks+ ), "Expected at least one graph_based chunk"+ assert any(+ c.metadata.get("strategy") == "community"+ and c.metadata.get("algorithm") == "louvain"+ for c in chunks+ ), "Expected graph_based chunk metadata to include strategy/algorithm"+ assert not any(+ c.metadata.get("method") == "recursive" for c in chunks+ ), "Graph-based fallback to recursive was triggered"+++class TestOntologyAwareChunker:+ def test_init(self):+ chunker = OntologyAwareChunker(chunk_size=200, preserve_concepts=True)+ assert chunker.chunk_size == 200+ assert chunker.preserve_concepts is True++ @requires_semantic_extract+ def test_chunk_uses_entity_aware_path(self):+ pytest.importorskip("semantica.semantic_extract")+ text = "Concept Alpha relates to Concept Beta in the taxonomy. " * 8+ chunker = OntologyAwareChunker(+ chunk_size=120, preserve_concepts=True, ner_method="pattern"+ )+ chunks = chunker.chunk(text)+ assert len(chunks) >= 1+ assert all(isinstance(c, Chunk) for c in chunks)+++class TestHierarchicalChunker:+ def test_hierarchical_markdown_sections(self):+ text = """# Section One++Paragraph under section one with enough content to matter.++# Section Two++Paragraph under section two also with sufficient content.+"""+ chunker = HierarchicalChunker(+ levels=["section", "paragraph"], chunk_sizes=[2000, 500]+ )+ chunks = chunker.chunk(text)+ assert len(chunks) >= 1+ for chunk in chunks:+ assert chunk.metadata.get("hierarchical") is True+ assert chunk.metadata.get("levels") == ["section", "paragraph"]++ def test_split_hierarchical_function(self):+ text = "Para one.\n\nPara two.\n\nPara three."+ chunks = split_hierarchical(text, levels=["paragraph"], chunk_sizes=[1000])+ assert len(chunks) >= 1+++# ---------------------------------------------------------------------------+# Exported method functions (public API smoke coverage)+# ---------------------------------------------------------------------------+++class TestSplitMethodFunctions:+ SAMPLE = (+ "First sentence about knowledge graphs. "+ "Second sentence covers entity extraction. "+ "Third sentence discusses relation awareness. "+ "Fourth sentence wraps up the example."+ )++ MARKDOWN = """# Intro++Intro paragraph with enough text to matter for structural splitting.++# Body++Body paragraph under a distinct heading for separation checks.+"""++ def test_split_recursive(self):+ chunks = split_recursive(self.SAMPLE, chunk_size=60)+ assert len(chunks) >= 1+ assert all(isinstance(c, Chunk) for c in chunks)++ def test_split_by_sentences(self):+ chunks = split_by_sentences(self.SAMPLE, chunk_size=80)+ assert len(chunks) >= 1++ def test_split_by_paragraphs(self):+ text = "Para A content here.\n\nPara B content here.\n\nPara C content here."+ chunks = split_by_paragraphs(text, chunk_size=50)+ assert len(chunks) >= 1++ def test_split_by_characters(self):+ chunks = split_by_characters(self.SAMPLE, chunk_size=40)+ assert len(chunks) >= 2++ def test_split_by_words(self):+ chunks = split_by_words(self.SAMPLE, chunk_size=10)+ assert len(chunks) >= 1++ def test_split_structural(self):+ chunks = split_structural(+ self.MARKDOWN, max_chunk_size=80, respect_headers=True+ )+ assert len(chunks) >= 2+ assert all(isinstance(c, Chunk) for c in chunks)++ def test_split_sliding_window(self):+ chunks = split_sliding_window(+ self.SAMPLE * 3,+ chunk_size=40,+ overlap=10,+ preserve_boundaries=False,+ )+ assert len(chunks) >= 2+ assert all(isinstance(c, Chunk) for c in chunks)+ # Verify the sliding-window path was taken, not the recursive fallback.+ # chunks[1].metadata["has_overlap"] is set only by SlidingWindowChunker.+ assert chunks[1].metadata.get("has_overlap") is True, (+ "Expected sliding-window chunks to carry has_overlap=True; "+ "fallback to recursive may have occurred"+ )+ assert chunks[1].metadata.get("method") != "recursive", (+ "Sliding-window fallback to recursive was triggered unexpectedly"+ )++ @requires_semantic_extract+ def test_split_entity_aware(self):+ pytest.importorskip("semantica.semantic_extract")+ chunks = split_entity_aware(+ self.SAMPLE * 3, chunk_size=80, ner_method="pattern"+ )+ assert len(chunks) >= 1++ @requires_semantic_extract+ def test_split_relation_aware(self):+ pytest.importorskip("semantica.semantic_extract")+ chunks = split_relation_aware(+ self.SAMPLE * 3,+ chunk_size=80,+ relation_method="pattern",+ ner_method="pattern",+ )+ assert len(chunks) >= 1++ @requires_semantic_extract+ @requires_networkx+ def test_split_graph_based(self):+ pytest.importorskip("networkx")+ pytest.importorskip("semantica.semantic_extract")+ chunks = split_graph_based(+ self.SAMPLE * 3,+ chunk_size=120,+ strategy="community",+ algorithm="louvain",+ ner_method="pattern",+ relation_method="pattern",+ )+ assert len(chunks) >= 1+ assert all(isinstance(c, Chunk) for c in chunks)+ # Ensure we didn't satisfy the test via the broad recursive fallback.+ assert any(+ c.metadata.get("method") == "graph_based" for c in chunks+ ), "Expected at least one graph_based chunk"+ assert any(+ c.metadata.get("strategy") == "community"+ and c.metadata.get("algorithm") == "louvain"+ for c in chunks+ ), "Expected graph_based chunk metadata to include strategy/algorithm"+ assert not any(+ c.metadata.get("method") == "recursive" for c in chunks+ ), "Graph-based fallback to recursive was triggered"++ @requires_semantic_extract+ def test_split_ontology_aware(self):+ pytest.importorskip("semantica.semantic_extract")+ chunks = split_ontology_aware(+ self.SAMPLE * 3, chunk_size=80, ner_method="pattern"+ )+ assert len(chunks) >= 1