Personal AI, On Personal Devices
Python
feat: add type-aware semantic chunker with section/paragraph/sentence splitting
krypticmouseefc979edf095
diff --git a/src/openjarvis/connectors/chunker.py b/src/openjarvis/connectors/chunker.pynew file mode 100644index 00000000..f0515497--- /dev/null+++ b/src/openjarvis/connectors/chunker.py@@ -0,0 +1,284 @@+"""Type-aware semantic chunker for Deep Research ingestion.++Splits text based on document type, never splitting mid-sentence.+Returns ``ChunkResult`` dataclass objects with section metadata and+inherited parent metadata.++Splitting strategy by doc_type+-------------------------------+- ``event``, ``contact`` : Always a single chunk; never split.+- ``email`` : Split on reply boundaries (``On … wrote:``),+ then sentence-split within each part.+- ``message`` : Split on double-newline boundaries, accumulate+ into chunks up to *max_tokens*.+- ``document``, ``note``,+ anything else : Split on ``## Heading`` section boundaries →+ paragraph boundaries (``\\n\\n``) within sections →+ sentence boundaries as a last resort.++Token counting uses whitespace splitting: ``len(text.split())``.+"""++from __future__ import annotations++import re+from dataclasses import dataclass, field+from typing import Any, Dict, List, Optional++# ---------------------------------------------------------------------------+# Public types+# ---------------------------------------------------------------------------++_SENTENCE_SPLIT_RE = re.compile(r'(?<=[.!?])\s+(?=[A-Z"])')+_SECTION_RE = re.compile(r"(?m)^##\s+(.+)$")+_REPLY_BOUNDARY_RE = re.compile(r"(?m)^On .+wrote:\s*$")+++@dataclass(slots=True)+class ChunkResult:+ """A single chunk produced by ``SemanticChunker.chunk()``."""++ content: str+ index: int = 0+ metadata: Dict[str, Any] = field(default_factory=dict)+++# ---------------------------------------------------------------------------+# Internal helpers+# ---------------------------------------------------------------------------+++def _count_tokens(text: str) -> int:+ """Approximate token count via whitespace splitting."""+ return len(text.split())+++def _split_sentences(text: str) -> List[str]:+ """Split *text* into sentences using the canonical regex.++ The regex splits after sentence-ending punctuation (``.``, ``!``, ``?``)+ followed by whitespace and a capital letter or a double-quote.+ """+ parts = _SENTENCE_SPLIT_RE.split(text)+ return [p.strip() for p in parts if p.strip()]+++def _accumulate(+ segments: List[str],+ *,+ max_tokens: int,+ sep: str = " ",+) -> List[str]:+ """Greedily merge *segments* into chunks up to *max_tokens* tokens.++ A segment that is already larger than *max_tokens* is placed in its own+ chunk; it is never split further by this function.+ """+ chunks: List[str] = []+ current_parts: List[str] = []+ current_tokens = 0++ for seg in segments:+ seg_tokens = _count_tokens(seg)+ if current_parts and current_tokens + seg_tokens > max_tokens:+ chunks.append(sep.join(current_parts))+ current_parts = [seg]+ current_tokens = seg_tokens+ else:+ current_parts.append(seg)+ current_tokens += seg_tokens++ if current_parts:+ chunks.append(sep.join(current_parts))++ return chunks+++def _sentence_chunks(text: str, *, max_tokens: int) -> List[str]:+ """Split *text* by sentences and accumulate into max_tokens chunks."""+ sentences = _split_sentences(text)+ if not sentences:+ stripped = text.strip()+ return [stripped] if stripped else []+ return _accumulate(sentences, max_tokens=max_tokens, sep=" ")+++def _paragraph_chunks(text: str, *, max_tokens: int) -> List[str]:+ """Split *text* on paragraph breaks (``\\n\\n``), then by sentences if needed."""+ paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]+ result: List[str] = []+ for para in paragraphs:+ if _count_tokens(para) <= max_tokens:+ result.append(para)+ else:+ result.extend(_sentence_chunks(para, max_tokens=max_tokens))+ return result+++# ---------------------------------------------------------------------------+# SemanticChunker+# ---------------------------------------------------------------------------+++class SemanticChunker:+ """Split text based on document type without breaking mid-sentence.++ Parameters+ ----------+ max_tokens:+ Soft upper limit on chunk size measured in whitespace-delimited tokens+ (i.e. ``len(text.split())``). Single unsplittable segments may exceed+ this limit.+ """++ def __init__(self, max_tokens: int = 512) -> None:+ self.max_tokens = max_tokens++ # ------------------------------------------------------------------+ # Public API+ # ------------------------------------------------------------------++ def chunk(+ self,+ text: str,+ *,+ doc_type: str = "document",+ metadata: Optional[Dict[str, Any]] = None,+ ) -> List[ChunkResult]:+ """Split *text* into ``ChunkResult`` objects.++ Parameters+ ----------+ text: The raw text to split.+ doc_type: Controls the splitting strategy (see module docstring).+ metadata: Parent metadata dict; copied into every chunk's ``metadata``.++ Returns+ -------+ A list of ``ChunkResult`` objects with sequential 0-based ``index``+ values. Returns an empty list if *text* is empty or whitespace-only.+ """+ if not text or not text.strip():+ return []++ parent_meta: Dict[str, Any] = dict(metadata or {})++ if doc_type in ("event", "contact"):+ raw_chunks = self._chunk_atomic(text)+ elif doc_type == "email":+ raw_chunks = self._chunk_email(text)+ elif doc_type == "message":+ raw_chunks = self._chunk_message(text)+ else:+ # "document", "note", or any unknown type+ raw_chunks = self._chunk_document(text)++ results: List[ChunkResult] = []+ for idx, (content, extra_meta) in enumerate(raw_chunks):+ merged: Dict[str, Any] = dict(parent_meta)+ merged.update(extra_meta)+ results.append(ChunkResult(content=content, index=idx, metadata=merged))++ return results++ # ------------------------------------------------------------------+ # Strategy implementations+ # ------------------------------------------------------------------++ def _chunk_atomic(self, text: str) -> List[tuple[str, Dict[str, Any]]]:+ """Return the entire text as a single chunk (event / contact)."""+ return [(text, {})]++ def _chunk_email(self, text: str) -> List[tuple[str, Dict[str, Any]]]:+ """Split on reply boundaries; sentence-split each part."""+ # Split the email into parts on "On ... wrote:" lines.+ # re.split with a capturing group keeps the boundary in results,+ # so we re-attach the header to the following segment.+ boundaries = _REPLY_BOUNDARY_RE.split(text)++ # Each boundary match is a separator; reassemble so the "On … wrote:"+ # line stays with the content that follows it (the quoted block).+ raw_parts: List[str] = []+ if boundaries:+ # The first element is the text before the first boundary (the+ # main reply body).+ raw_parts.append(boundaries[0])+ # Subsequent elements alternate: matched boundary, then text after.+ # Because we used split() (not findall), the boundaries themselves+ # are not in the list — only the text segments between them.+ # So boundaries[1:] are the segments after each matched header.+ # We need to re-find the headers to reassemble.+ headers = _REPLY_BOUNDARY_RE.findall(text)+ for header, body in zip(headers, boundaries[1:]):+ # We found the header text via findall; reconstruct the part.+ part = (header.strip() + "\n" + body).strip()+ raw_parts.append(part)++ chunks: List[tuple[str, Dict[str, Any]]] = []+ for part in raw_parts:+ part = part.strip()+ if not part:+ continue+ if _count_tokens(part) <= self.max_tokens:+ chunks.append((part, {}))+ else:+ for sub in _sentence_chunks(part, max_tokens=self.max_tokens):+ if sub:+ chunks.append((sub, {}))++ return chunks if chunks else [(text.strip(), {})]++ def _chunk_message(self, text: str) -> List[tuple[str, Dict[str, Any]]]:+ """Split on double-newline boundaries and accumulate up to max_tokens."""+ paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]+ raw_chunks = _accumulate(paragraphs, max_tokens=self.max_tokens, sep="\n\n")+ return [(c, {}) for c in raw_chunks if c]++ def _chunk_document(self, text: str) -> List[tuple[str, Dict[str, Any]]]:+ """Split on ## headings → paragraphs → sentences."""+ # Find all ## heading positions+ section_matches = list(_SECTION_RE.finditer(text))++ if not section_matches:+ # No headings — fall back to paragraph/sentence splitting+ raw_chunks = _paragraph_chunks(text, max_tokens=self.max_tokens)+ return [(c, {}) for c in raw_chunks if c]++ # Build (title, body_text) pairs for each section+ sections: List[tuple[str, str]] = []+ for i, m in enumerate(section_matches):+ title = m.group(1).strip()+ body_start = m.end()+ body_end = (+ section_matches[i + 1].start()+ if i + 1 < len(section_matches)+ else len(text)+ )+ body = text[body_start:body_end].strip()+ sections.append((title, body))++ # Check for preamble text before the first heading+ preamble = text[: section_matches[0].start()].strip()+ result: List[tuple[str, Dict[str, Any]]] = []++ if preamble:+ for c in _paragraph_chunks(preamble, max_tokens=self.max_tokens):+ if c:+ result.append((c, {}))++ for title, body in sections:+ section_meta: Dict[str, Any] = {"section": title}+ if not body:+ # Empty section — emit a placeholder chunk with just the title+ result.append((title, section_meta))+ continue++ para_chunks = _paragraph_chunks(body, max_tokens=self.max_tokens)+ for c in para_chunks:+ if c:+ result.append((c, dict(section_meta)))++ return result if result else [(text.strip(), {})]+++__all__ = ["ChunkResult", "SemanticChunker"]diff --git a/tests/connectors/test_chunker.py b/tests/connectors/test_chunker.pynew file mode 100644index 00000000..086d804a--- /dev/null+++ b/tests/connectors/test_chunker.py@@ -0,0 +1,264 @@+"""Tests for SemanticChunker — type-aware text splitting."""++from __future__ import annotations++import pytest++from openjarvis.connectors.chunker import ChunkResult, SemanticChunker++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------+++def chunker() -> SemanticChunker:+ return SemanticChunker(max_tokens=50)+++# ---------------------------------------------------------------------------+# 1. Short message stays as single chunk+# ---------------------------------------------------------------------------+++def test_short_message_single_chunk(chunker: SemanticChunker) -> None:+ """A message shorter than max_tokens is returned as a single chunk."""+ text = "Hello, world! How are you today?"+ results = chunker.chunk(text, doc_type="message")+ assert len(results) == 1+ assert results[0].content == text+ assert results[0].index == 0+++# ---------------------------------------------------------------------------+# 2. Long document splits on ## Heading sections, metadata has section key+# ---------------------------------------------------------------------------+++def test_document_splits_on_headings() -> None:+ """Documents with ## headings produce chunks with section metadata."""+ chunker = SemanticChunker(max_tokens=512)+ text = (+ "## Introduction\n"+ "This is the introduction paragraph. It explains the context.\n\n"+ "## Methods\n"+ "This section describes the methods used in the study.\n\n"+ "## Results\n"+ "Here are the results of the experiment."+ )+ results = chunker.chunk(text, doc_type="document")+ # Each ## section should produce at least one chunk+ sections = {r.metadata.get("section") for r in results}+ assert "Introduction" in sections+ assert "Methods" in sections+ assert "Results" in sections+++# ---------------------------------------------------------------------------+# 3. Within a section, splits on paragraph boundaries+# ---------------------------------------------------------------------------+++def test_document_splits_on_paragraphs() -> None:+ """Within a section, long content splits on double-newline paragraph breaks."""+ # Use a small max_tokens to force splitting within a section+ chunker = SemanticChunker(max_tokens=15)+ # Build a section with two paragraphs, each > 15 tokens+ para1 = " ".join(["word"] * 20) # 20 tokens+ para2 = " ".join(["text"] * 20) # 20 tokens+ text = f"## Section One\n{para1}\n\n{para2}"+ results = chunker.chunk(text, doc_type="document")+ # Both paragraphs should be separate chunks+ assert len(results) >= 2+ # All chunks from this section carry section metadata+ for r in results:+ assert r.metadata.get("section") == "Section One"+++# ---------------------------------------------------------------------------+# 4. Never splits mid-sentence (chunks end with . ? or ! except possibly last)+# ---------------------------------------------------------------------------+++def test_no_mid_sentence_splits() -> None:+ """Chunks (except possibly the last) must end with sentence-ending punctuation."""+ chunker = SemanticChunker(max_tokens=10)+ # Build text with clearly delimited sentences in a document section+ text = (+ "## Analysis\n"+ "The first result was positive. The second outcome was negative. "+ "The third finding was inconclusive. The final conclusion is pending."+ )+ results = chunker.chunk(text, doc_type="document")+ # All chunks except possibly the last should end with sentence punctuation+ for r in results[:-1]:+ stripped = r.content.rstrip()+ assert stripped[-1] in {".", "?", "!"}, (+ f"Non-final chunk does not end with sentence punctuation: {stripped!r}"+ )+++# ---------------------------------------------------------------------------+# 5. Email thread splits on reply boundaries+# ---------------------------------------------------------------------------+++def test_email_splits_on_reply_boundaries() -> None:+ """Emails split on 'On ... wrote:' reply headers."""+ chunker = SemanticChunker(max_tokens=512)+ text = (+ "Hi Alice, please see my comments below.\n\n"+ "On Mon, Jan 1, 2024, Alice Smith <[email protected]> wrote:\n"+ "> Original message here.\n"+ "> More original text.\n\n"+ "On Sun, Dec 31, 2023, Bob Jones <[email protected]> wrote:\n"+ "> Even earlier message content."+ )+ results = chunker.chunk(text, doc_type="email")+ # Should produce more than one chunk due to reply boundaries+ assert len(results) >= 2+++# ---------------------------------------------------------------------------+# 6. Event stays as single chunk+# ---------------------------------------------------------------------------+++def test_event_always_single_chunk() -> None:+ """Events are never split regardless of length."""+ chunker = SemanticChunker(max_tokens=5)+ text = " ".join(["word"] * 100) # 100 tokens, well above max_tokens=5+ results = chunker.chunk(text, doc_type="event")+ assert len(results) == 1+ assert results[0].content == text+++# ---------------------------------------------------------------------------+# 7. Contact stays as single chunk+# ---------------------------------------------------------------------------+++def test_contact_always_single_chunk() -> None:+ """Contacts are never split regardless of length."""+ chunker = SemanticChunker(max_tokens=5)+ text = " ".join(["info"] * 100) # 100 tokens, well above max_tokens=5+ results = chunker.chunk(text, doc_type="contact")+ assert len(results) == 1+ assert results[0].content == text+++# ---------------------------------------------------------------------------+# 8. Parent metadata inherited to all chunks+# ---------------------------------------------------------------------------+++def test_parent_metadata_inherited() -> None:+ """All chunks carry the parent metadata passed to chunk()."""+ chunker = SemanticChunker(max_tokens=10)+ parent_meta = {"source": "gmail", "doc_id": "abc-123", "priority": "high"}+ text = (+ "## Section A\n"+ "First sentence of section A. Second sentence of section A. "+ "Third sentence here. Fourth sentence concludes.\n\n"+ "## Section B\n"+ "First sentence of section B. Second sentence of section B."+ )+ results = chunker.chunk(text, doc_type="document", metadata=parent_meta)+ for r in results:+ assert r.metadata.get("source") == "gmail"+ assert r.metadata.get("doc_id") == "abc-123"+ assert r.metadata.get("priority") == "high"+++# ---------------------------------------------------------------------------+# 9. Chunks have sequential 0-based indexes+# ---------------------------------------------------------------------------+++def test_sequential_chunk_indexes() -> None:+ """Chunks are indexed sequentially from 0 across all splits."""+ chunker = SemanticChunker(max_tokens=10)+ text = (+ "## Alpha\n"+ "Sentence one ends here. Sentence two ends here. Sentence three ends here.\n\n"+ "## Beta\n"+ "Sentence four ends here. Sentence five ends here."+ )+ results = chunker.chunk(text, doc_type="document")+ for i, r in enumerate(results):+ assert r.index == i, f"Expected index {i}, got {r.index}"+++# ---------------------------------------------------------------------------+# Additional edge cases+# ---------------------------------------------------------------------------+++def test_note_doc_type_treated_as_document() -> None:+ """doc_type='note' uses the same document splitting strategy."""+ chunker = SemanticChunker(max_tokens=512)+ text = (+ "## My Note\n"+ "This is a note with section headings.\n\n"+ "## Another Section\n"+ "More content here."+ )+ results = chunker.chunk(text, doc_type="note")+ sections = {r.metadata.get("section") for r in results}+ assert "My Note" in sections+ assert "Another Section" in sections+++def test_unknown_doc_type_treated_as_document() -> None:+ """Unknown doc_type uses the document splitting strategy."""+ chunker = SemanticChunker(max_tokens=512)+ text = "## Header\nContent under the header.\n\n"+ results = chunker.chunk(text, doc_type="unknown_type")+ sections = {r.metadata.get("section") for r in results}+ assert "Header" in sections+++def test_empty_text_returns_empty_list() -> None:+ """Empty string input returns an empty list."""+ chunker = SemanticChunker(max_tokens=512)+ results = chunker.chunk("", doc_type="document")+ assert results == []+++def test_whitespace_only_text_returns_empty_list() -> None:+ """Whitespace-only input returns an empty list."""+ chunker = SemanticChunker(max_tokens=512)+ results = chunker.chunk(" \n\n \t ", doc_type="document")+ assert results == []+++def test_chunk_result_is_dataclass() -> None:+ """ChunkResult has the expected fields with correct defaults."""+ cr = ChunkResult(content="hello")+ assert cr.content == "hello"+ assert cr.index == 0+ assert cr.metadata == {}+++def test_message_accumulates_into_max_tokens() -> None:+ """Message chunks accumulate paragraphs up to max_tokens."""+ chunker = SemanticChunker(max_tokens=20)+ # Each paragraph is 8 tokens; two fit in 20 but three would exceed 20 (8+8+8=24)+ para = "one two three four five six seven eight" # 8 tokens+ text = f"{para}\n\n{para}\n\n{para}\n\n{para}"+ results = chunker.chunk(text, doc_type="message")+ # Should not fit all 4 paragraphs in one chunk (would be 32 tokens)+ assert len(results) >= 2+ # Each chunk should be within or just at max_tokens (greedy accumulation)+ for r in results[:-1]:+ assert len(r.content.split()) <= 20 * 2 # some flexibility for joining+++def test_document_no_headings_uses_paragraphs() -> None:+ """Documents without ## headings fall back to paragraph splitting."""+ chunker = SemanticChunker(max_tokens=15)+ para1 = " ".join(["alpha"] * 20) # 20 tokens, forces split+ para2 = " ".join(["beta"] * 20)+ text = f"{para1}\n\n{para2}"+ results = chunker.chunk(text, doc_type="document")+ assert len(results) >= 2