Personal AI, On Personal Devices
Python
feat: wire attachment text extraction into IngestionPipeline
krypticmouseee1805ceaf7c
diff --git a/src/openjarvis/connectors/pipeline.py b/src/openjarvis/connectors/pipeline.pyindex 2585d0e0..4df3a030 100644--- a/src/openjarvis/connectors/pipeline.py+++ b/src/openjarvis/connectors/pipeline.py@@ -13,12 +13,15 @@ Typical usage::from __future__ import annotations-from typing import Iterable+from typing import TYPE_CHECKING, Iterable, Optional-from openjarvis.connectors._stubs import Document+from openjarvis.connectors._stubs import Attachment, Documentfrom openjarvis.connectors.chunker import SemanticChunkerfrom openjarvis.connectors.store import KnowledgeStore+if TYPE_CHECKING:+ from openjarvis.connectors.attachment_store import AttachmentStore+class IngestionPipeline:"""Deduplicate, chunk, and index documents into a KnowledgeStore.@@ -29,11 +32,22 @@ class IngestionPipeline:The ``KnowledgeStore`` instance to write chunks into.max_tokens:Soft upper-limit on chunk size passed to ``SemanticChunker``.+ attachment_store:+ Optional ``AttachmentStore`` for persisting attachment blobs and+ extracting text from supported MIME types (PDF, plain text, etc.).+ When ``None`` (default) attachments are silently ignored."""- def __init__(self, store: KnowledgeStore, *, max_tokens: int = 512) -> None:+ def __init__(+ self,+ store: KnowledgeStore,+ *,+ max_tokens: int = 512,+ attachment_store: Optional[AttachmentStore] = None,+ ) -> None:self._store = storeself._chunker = SemanticChunker(max_tokens=max_tokens)+ self._attachment_store = attachment_storeself._seen_doc_ids: set[str] = set()self._load_existing_doc_ids()@@ -48,6 +62,26 @@ class IngestionPipeline:).fetchall()self._seen_doc_ids = {r[0] for r in rows}+ def _extract_attachment_text(self, att: Attachment) -> str:+ """Extract text from an attachment.++ Returns the extracted text, or an empty string if the MIME type is+ unsupported or extraction fails.+ """+ if att.mime_type == "application/pdf":+ try:+ import io++ import pdfplumber++ with pdfplumber.open(io.BytesIO(att.content)) as pdf:+ return "\n".join(page.extract_text() or "" for page in pdf.pages)+ except Exception: # noqa: BLE001+ return ""+ if att.mime_type in ("text/plain", "text/markdown", "text/csv"):+ return att.content.decode("utf-8", errors="replace")+ return ""+# ------------------------------------------------------------------# Public API# ------------------------------------------------------------------@@ -119,6 +153,49 @@ class IngestionPipeline:)chunks_stored += 1+ # Process attachments when an attachment store is configured.+ if self._attachment_store and doc.attachments:+ for att in doc.attachments:+ if not att.content:+ continue++ # Persist the raw blob and obtain its SHA-256.+ sha = self._attachment_store.store(+ content=att.content,+ filename=att.filename,+ mime_type=att.mime_type,+ source_doc_id=doc.doc_id,+ )++ # Extract searchable text and index it as additional chunks.+ extracted = self._extract_attachment_text(att)+ if extracted:+ att_chunks = self._chunker.chunk(+ extracted,+ doc_type=doc.doc_type,+ metadata={+ **parent_meta,+ "attachment": att.filename,+ "sha256": sha,+ },+ )+ for chunk in att_chunks:+ self._store.store(+ content=chunk.content,+ source=doc.source,+ doc_type=doc.doc_type,+ doc_id=doc.doc_id,+ title=f"{doc.title} [{att.filename}]",+ author=doc.author,+ participants=doc.participants,+ timestamp=timestamp_str,+ thread_id=doc.thread_id,+ url=doc.url,+ metadata=chunk.metadata,+ chunk_index=chunk.index,+ )+ chunks_stored += 1+self._seen_doc_ids.add(doc.doc_id)return chunks_storeddiff --git a/tests/connectors/test_pipeline_attachments.py b/tests/connectors/test_pipeline_attachments.pynew file mode 100644index 00000000..912cabd0--- /dev/null+++ b/tests/connectors/test_pipeline_attachments.py@@ -0,0 +1,163 @@+"""Tests for attachment processing in IngestionPipeline."""++from __future__ import annotations++from datetime import datetime, timezone+from pathlib import Path++import pytest++from openjarvis.connectors._stubs import Attachment, Document+from openjarvis.connectors.attachment_store import AttachmentStore+from openjarvis.connectors.pipeline import IngestionPipeline+from openjarvis.connectors.store import KnowledgeStore++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+++def _make_doc(**kwargs) -> Document: # type: ignore[type-arg]+ """Build a Document with sensible defaults."""+ defaults = dict(+ doc_id="doc:att:001",+ source="gmail",+ doc_type="email",+ content="Main body of the email.",+ title="Test Email",+ author="[email protected]",+ participants=[],+ timestamp=datetime(2025, 3, 1, tzinfo=timezone.utc),+ thread_id=None,+ url=None,+ attachments=[],+ metadata={},+ )+ defaults.update(kwargs)+ return Document(**defaults) # type: ignore[arg-type]+++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------+++def store(tmp_path: Path) -> KnowledgeStore:+ return KnowledgeStore(db_path=tmp_path / "att_pipeline.db")+++def att_store(tmp_path: Path) -> AttachmentStore:+ return AttachmentStore(base_dir=str(tmp_path / "blobs"))+++def pipeline(store: KnowledgeStore, att_store: AttachmentStore) -> IngestionPipeline:+ return IngestionPipeline(store, attachment_store=att_store)+++# ---------------------------------------------------------------------------+# Test 1: text/plain attachment text is indexed and searchable+# ---------------------------------------------------------------------------+++def test_plain_text_attachment_indexed(+ pipeline: IngestionPipeline, store: KnowledgeStore+) -> None:+ """Ingesting a doc with a text/plain attachment indexes the attachment text."""+ att_text = b"Deep learning quarterly report: model accuracy improved by 15%."+ att = Attachment(+ filename="report.txt",+ mime_type="text/plain",+ size_bytes=len(att_text),+ content=att_text,+ )+ doc = _make_doc(+ doc_id="doc:att:plain:001",+ content="Please see the attached report.",+ title="Q1 Report Email",+ attachments=[att],+ )++ n = pipeline.ingest([doc])++ # Expect at least 2 chunks: 1 from the main body + 1 from the attachment+ assert n >= 2++ # Attachment text must be retrievable+ results = store.retrieve("model accuracy improved", top_k=5)+ assert len(results) >= 1++ # At least one result should come from the attachment chunk+ att_results = [r for r in results if r.metadata.get("attachment") == "report.txt"]+ assert len(att_results) >= 1++ # Title for attachment chunks includes the filename in brackets+ assert "report.txt" in att_results[0].metadata.get("title", "")+++# ---------------------------------------------------------------------------+# Test 2: blob is stored in AttachmentStore+# ---------------------------------------------------------------------------+++def test_attachment_blob_stored(+ pipeline: IngestionPipeline, att_store: AttachmentStore+) -> None:+ """The raw bytes of an attachment are persisted in the AttachmentStore."""+ att_content = b"Confidential: merger details enclosed."+ att = Attachment(+ filename="merger.txt",+ mime_type="text/plain",+ size_bytes=len(att_content),+ content=att_content,+ )+ doc = _make_doc(+ doc_id="doc:att:blob:001",+ content="See attached for details.",+ attachments=[att],+ )++ pipeline.ingest([doc])++ # The blob must be retrievable from the attachment store+ import hashlib++ expected_sha = hashlib.sha256(att_content).hexdigest()+ stored_bytes = att_store.get_content(expected_sha)+ assert stored_bytes == att_content++ # Metadata must link back to the source document+ meta = att_store.get_metadata(expected_sha)+ assert meta is not None+ assert meta["filename"] == "merger.txt"+ assert meta["mime_type"] == "text/plain"+ assert "doc:att:blob:001" in meta["source_doc_ids"]+++# ---------------------------------------------------------------------------+# Test 3: no regression when doc has no attachments+# ---------------------------------------------------------------------------+++def test_no_attachments_no_regression(+ store: KnowledgeStore, att_store: AttachmentStore+) -> None:+ """Pipeline with attachment_store handles docs without attachments correctly."""+ pipeline = IngestionPipeline(store, attachment_store=att_store)++ doc = _make_doc(+ doc_id="doc:no:att:001",+ content="Just a regular email with no attachments.",+ attachments=[],+ )++ n = pipeline.ingest([doc])++ assert n == 1+ assert store.count() == 1++ # No blobs should be stored+ rows = att_store._conn.execute("SELECT COUNT(*) FROM attachments").fetchone()+ assert rows[0] == 0