Personal AI, On Personal Devices
Python
feat: add EmbeddingStore for ColBERTv2 disk persistence
krypticmouseef859e6a17a4
diff --git a/src/openjarvis/connectors/embedding_store.py b/src/openjarvis/connectors/embedding_store.pynew file mode 100644index 00000000..6f2e0373--- /dev/null+++ b/src/openjarvis/connectors/embedding_store.py@@ -0,0 +1,179 @@+"""EmbeddingStore --- disk-persistent ColBERT token-level embeddings.++Stores per-chunk ColBERTv2 token embeddings as individual ``.pt`` files on+disk with a SQLite index for O(1) chunk_id lookups. Designed to be used by+``ColBERTReranker`` so that document embeddings computed at ingest time can+be reused across queries instead of re-encoding on every search.++All ``torch`` usage is lazily imported and guarded behind ``try/except`` so+that the store degrades gracefully when PyTorch is not installed --- every+public method returns ``None`` or ``False`` instead of raising.+"""++from __future__ import annotations++import logging+import sqlite3+from pathlib import Path+from typing import TYPE_CHECKING, Optional++if TYPE_CHECKING:+ import torch # type: ignore[import]++logger = logging.getLogger(__name__)++# ---------------------------------------------------------------------------+# DDL for the lightweight SQLite index+# ---------------------------------------------------------------------------++_CREATE_INDEX_TABLE = """+CREATE TABLE IF NOT EXISTS embedding_index (+ chunk_id TEXT PRIMARY KEY,+ filename TEXT NOT NULL+);+"""+++class EmbeddingStore:+ """Stores ColBERT token-level embeddings on disk.++ Each ``chunk_id`` maps to a tensor of shape ``(num_tokens, 128)`` saved+ as a standard PyTorch ``.pt`` file. A small SQLite database provides an+ O(1) index from *chunk_id* to the tensor filename.++ Parameters+ ----------+ store_dir:+ Root directory for the embedding store. Defaults to+ ``~/.openjarvis/embeddings/``. The directory is created if it does+ not already exist.+ """++ def __init__(self, store_dir: str = "") -> None:+ if not store_dir:+ from openjarvis.core.config import DEFAULT_CONFIG_DIR++ store_dir = str(DEFAULT_CONFIG_DIR / "embeddings")++ self._store_dir = Path(store_dir)+ self._tensor_dir = self._store_dir / "tensors"+ self._tensor_dir.mkdir(parents=True, exist_ok=True)++ db_path = self._store_dir / "index.db"+ self._conn = sqlite3.connect(str(db_path), check_same_thread=False)+ self._conn.execute("PRAGMA journal_mode=WAL;")+ self._conn.executescript(_CREATE_INDEX_TABLE)+ self._conn.commit()++ # ------------------------------------------------------------------+ # Public API+ # ------------------------------------------------------------------++ def store(self, chunk_id: str, embedding: "torch.Tensor") -> None:+ """Save a chunk's embedding tensor to disk.++ Parameters+ ----------+ chunk_id:+ Unique identifier for the chunk (matches the KnowledgeStore PK).+ embedding:+ Tensor of shape ``(num_tokens, 128)`` from ColBERT encoding.+ """+ try:+ import torch as _torch # noqa: F401+ except ImportError:+ logger.debug("EmbeddingStore.store: torch not available, skipping.")+ return++ filename = f"{chunk_id}.pt"+ tensor_path = self._tensor_dir / filename++ _torch.save(embedding, str(tensor_path))++ self._conn.execute(+ "INSERT OR REPLACE INTO embedding_index (chunk_id, filename) VALUES (?, ?)",+ (chunk_id, filename),+ )+ self._conn.commit()++ def get(self, chunk_id: str) -> Optional["torch.Tensor"]:+ """Load a chunk's embedding from disk.++ Returns ``None`` if the chunk has no stored embedding or if+ ``torch`` is not installed.+ """+ try:+ import torch as _torch # noqa: F401+ except ImportError:+ return None++ row = self._conn.execute(+ "SELECT filename FROM embedding_index WHERE chunk_id = ?",+ (chunk_id,),+ ).fetchone()++ if row is None:+ return None++ tensor_path = self._tensor_dir / row[0]+ if not tensor_path.exists():+ # Stale index entry --- clean it up+ self._conn.execute(+ "DELETE FROM embedding_index WHERE chunk_id = ?", (chunk_id,)+ )+ self._conn.commit()+ return None++ try:+ return _torch.load(str(tensor_path), weights_only=True)+ except Exception as exc:+ logger.warning(+ "EmbeddingStore.get: failed to load tensor for %s (%s)", chunk_id, exc+ )+ return None++ def has(self, chunk_id: str) -> bool:+ """Check if embeddings exist for a chunk."""+ row = self._conn.execute(+ "SELECT 1 FROM embedding_index WHERE chunk_id = ?",+ (chunk_id,),+ ).fetchone()+ return row is not None++ def count(self) -> int:+ """Return the number of stored embeddings."""+ row = self._conn.execute("SELECT COUNT(*) FROM embedding_index").fetchone()+ return row[0] if row else 0++ def delete(self, chunk_id: str) -> bool:+ """Delete the embedding for a chunk.++ Returns ``True`` if the embedding existed and was removed.+ """+ row = self._conn.execute(+ "SELECT filename FROM embedding_index WHERE chunk_id = ?",+ (chunk_id,),+ ).fetchone()++ if row is None:+ return False++ tensor_path = self._tensor_dir / row[0]+ if tensor_path.exists():+ tensor_path.unlink()++ self._conn.execute(+ "DELETE FROM embedding_index WHERE chunk_id = ?", (chunk_id,)+ )+ self._conn.commit()+ return True++ def close(self) -> None:+ """Close the underlying SQLite connection."""+ try:+ self._conn.close()+ except Exception:+ pass+++__all__ = ["EmbeddingStore"]diff --git a/src/openjarvis/connectors/retriever.py b/src/openjarvis/connectors/retriever.pyindex e1f483c6..a82716d1 100644--- a/src/openjarvis/connectors/retriever.py+++ b/src/openjarvis/connectors/retriever.py@@ -16,11 +16,14 @@ from __future__ import annotationsimport loggingfrom abc import ABC, abstractmethod-from typing import List, Optional+from typing import TYPE_CHECKING, List, Optionalfrom openjarvis.connectors.store import KnowledgeStorefrom openjarvis.tools.storage._stubs import RetrievalResult+if TYPE_CHECKING:+ from openjarvis.connectors.embedding_store import EmbeddingStore+logger = logging.getLogger(__name__)# ---------------------------------------------------------------------------@@ -74,12 +77,22 @@ class ColBERTReranker(Reranker):checkpoint:Path or HuggingFace model ID for the ColBERT checkpoint.Defaults to ``"colbert-ir/colbertv2.0"``.+ embedding_store:+ Optional :class:`EmbeddingStore` for caching per-chunk ColBERT+ token embeddings on disk. When provided, the reranker looks up+ pre-computed embeddings before falling back to ``docFromText()``,+ and stores newly computed embeddings for future reuse."""- def __init__(self, checkpoint: str = "colbert-ir/colbertv2.0") -> None:+ def __init__(+ self,+ checkpoint: str = "colbert-ir/colbertv2.0",+ embedding_store: Optional["EmbeddingStore"] = None,+ ) -> None:self._checkpoint = checkpointself._model = Noneself._warned = False+ self._embedding_store = embedding_storedef _load_model(self) -> bool:"""Attempt to load the ColBERT model. Returns True on success."""@@ -138,24 +151,35 @@ class ColBERTReranker(Reranker):# Encode query: (Q, dim) where Q=query_maxlen (32)q_emb = self._model.queryFromText([query])[0]- doc_texts = [r.content for r in candidates]- # Score each candidate via MaxSim+ # Score each candidate via MaxSim, using cached embeddings+ # from the EmbeddingStore when available.scores = []- for text in doc_texts:- d_emb = self._model.docFromText([text], bsize=1)[0]- # d_emb shape: (1, T, dim) or (T, dim)- if d_emb.dim() == 3:- d_emb = d_emb.squeeze(0)+ for r in candidates:+ chunk_id = r.metadata.get("chunk_id", "")++ # Try cached embedding first+ d_emb = None+ if self._embedding_store and chunk_id:+ d_emb = self._embedding_store.get(chunk_id)++ if d_emb is None:+ # Encode on the fly+ d_emb = self._model.docFromText([r.content], bsize=1)[0]+ # d_emb shape: (1, T, dim) or (T, dim)+ if d_emb.dim() == 3:+ d_emb = d_emb.squeeze(0)+ # Cache for future queries+ if self._embedding_store and chunk_id:+ self._embedding_store.store(chunk_id, d_emb)+sim = torch.nn.functional.cosine_similarity(q_emb.unsqueeze(1), d_emb.unsqueeze(0), dim=2)maxsim = sim.max(dim=1).values.sum().item()scores.append(maxsim)- ranked = sorted(- zip(scores, candidates), key=lambda x: x[0], reverse=True- )+ ranked = sorted(zip(scores, candidates), key=lambda x: x[0], reverse=True)reranked = []for score, result in ranked[:top_k]:reranked.append(diff --git a/src/openjarvis/connectors/store.py b/src/openjarvis/connectors/store.pyindex 9e65ff2d..3d10312e 100644--- a/src/openjarvis/connectors/store.py+++ b/src/openjarvis/connectors/store.py@@ -118,6 +118,7 @@ class KnowledgeStore(MemoryBackend):def __init__(self, db_path: Union[str, Path] = "") -> None:if not db_path:from openjarvis.core.config import DEFAULT_CONFIG_DIR+db_path = DEFAULT_CONFIG_DIR / "knowledge.db"self._db_path = str(db_path)@@ -138,10 +139,7 @@ class KnowledgeStore(MemoryBackend):self._conn.execute("PRAGMA journal_mode=WAL;")self._conn.execute("PRAGMA foreign_keys=ON;")self._conn.executescript(- _CREATE_MAIN_TABLE- + _CREATE_FTS_TABLE- + _CREATE_TRIGGERS- + _CREATE_INDEXES+ _CREATE_MAIN_TABLE + _CREATE_FTS_TABLE + _CREATE_TRIGGERS + _CREATE_INDEXES)self._conn.commit()@@ -179,6 +177,7 @@ class KnowledgeStore(MemoryBackend):# Merge provenance fields into metadata for easy access in resultscombined_meta: Dict[str, Any] = dict(metadata or {})+ combined_meta["chunk_id"] = chunk_idcombined_meta["source"] = sourcecombined_meta["doc_type"] = doc_typecombined_meta["doc_id"] = doc_id@@ -310,6 +309,10 @@ class KnowledgeStore(MemoryBackend):results: List[RetrievalResult] = []for row in rows:meta = json.loads(row["metadata"]) if row["metadata"] else {}+ # Ensure chunk_id is always present in metadata (backfill for+ # rows stored before this field was added to combined_meta).+ if "chunk_id" not in meta:+ meta["chunk_id"] = row["id"]results.append(RetrievalResult(content=row["content"],diff --git a/tests/connectors/test_embedding_store.py b/tests/connectors/test_embedding_store.pynew file mode 100644index 00000000..d63c49e4--- /dev/null+++ b/tests/connectors/test_embedding_store.py@@ -0,0 +1,162 @@+"""Tests for EmbeddingStore — disk-persistent ColBERT token-level embeddings."""++from __future__ import annotations++from pathlib import Path+from unittest.mock import MagicMock, patch++import pytest++from openjarvis.connectors.embedding_store import EmbeddingStore++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------+++def emb_store(tmp_path: Path) -> EmbeddingStore:+ """An EmbeddingStore rooted at a temporary directory."""+ return EmbeddingStore(store_dir=str(tmp_path / "embeddings"))+++def _make_tensor(rows: int = 20, cols: int = 128):+ """Create a dummy tensor (rows, cols) mimicking ColBERT token embeddings."""+ import torch # type: ignore[import]++ return torch.randn(rows, cols)+++# ---------------------------------------------------------------------------+# Test 1: store_and_get — round-trip persistence+# ---------------------------------------------------------------------------+++def test_store_and_get(emb_store: EmbeddingStore) -> None:+ """Storing a tensor and getting it back preserves shape and values."""+ import torch # type: ignore[import]++ tensor = _make_tensor(25, 128)+ emb_store.store("chunk-abc", tensor)++ loaded = emb_store.get("chunk-abc")+ assert loaded is not None+ assert loaded.shape == (25, 128)+ assert torch.allclose(tensor, loaded)+++# ---------------------------------------------------------------------------+# Test 2: has — True after store, False for unknown+# ---------------------------------------------------------------------------+++def test_has(emb_store: EmbeddingStore) -> None:+ """has() returns True for stored chunks, False for unknown ones."""+ assert emb_store.has("nonexistent") is False++ tensor = _make_tensor()+ emb_store.store("chunk-1", tensor)++ assert emb_store.has("chunk-1") is True+ assert emb_store.has("chunk-2") is False+++# ---------------------------------------------------------------------------+# Test 3: count — 0 initially, increments on store+# ---------------------------------------------------------------------------+++def test_count(emb_store: EmbeddingStore) -> None:+ """count() tracks the number of stored embeddings."""+ assert emb_store.count() == 0++ emb_store.store("a", _make_tensor())+ assert emb_store.count() == 1++ emb_store.store("b", _make_tensor())+ assert emb_store.count() == 2+++# ---------------------------------------------------------------------------+# Test 4: delete — removes tensor, has() returns False after+# ---------------------------------------------------------------------------+++def test_delete(emb_store: EmbeddingStore) -> None:+ """delete() removes both the tensor file and the index entry."""+ tensor = _make_tensor()+ emb_store.store("chunk-del", tensor)+ assert emb_store.has("chunk-del") is True++ result = emb_store.delete("chunk-del")+ assert result is True+ assert emb_store.has("chunk-del") is False+ assert emb_store.get("chunk-del") is None+ assert emb_store.count() == 0+++def test_delete_nonexistent(emb_store: EmbeddingStore) -> None:+ """delete() returns False for a chunk that was never stored."""+ assert emb_store.delete("nope") is False+++# ---------------------------------------------------------------------------+# Test 5: get_nonexistent — returns None+# ---------------------------------------------------------------------------+++def test_get_nonexistent(emb_store: EmbeddingStore) -> None:+ """get() returns None for an unknown chunk_id."""+ assert emb_store.get("does-not-exist") is None+++# ---------------------------------------------------------------------------+# Test 6: graceful_without_torch — mock torch import failure+# ---------------------------------------------------------------------------+++def test_graceful_without_torch(tmp_path: Path) -> None:+ """When torch is not importable, all methods degrade gracefully."""+ store = EmbeddingStore(store_dir=str(tmp_path / "no_torch"))++ fake_tensor = MagicMock()++ # Patch the import inside embedding_store so torch appears unavailable+ import builtins++ real_import = builtins.__import__++ def _no_torch(name, *args, **kwargs):+ if name == "torch" or name.startswith("torch."):+ raise ImportError("mocked: no torch")+ return real_import(name, *args, **kwargs)++ with patch("builtins.__import__", side_effect=_no_torch):+ # store() should silently skip+ store.store("chunk-x", fake_tensor)+ assert store.count() == 0++ # get() should return None+ assert store.get("chunk-x") is None+++# ---------------------------------------------------------------------------+# Test 7: overwrite existing embedding+# ---------------------------------------------------------------------------+++def test_store_overwrites(emb_store: EmbeddingStore) -> None:+ """Storing the same chunk_id twice overwrites the previous embedding."""+ import torch # type: ignore[import]++ t1 = _make_tensor(10, 128)+ t2 = _make_tensor(15, 128)++ emb_store.store("chunk-ow", t1)+ emb_store.store("chunk-ow", t2)++ assert emb_store.count() == 1+ loaded = emb_store.get("chunk-ow")+ assert loaded is not None+ assert loaded.shape == (15, 128)+ assert torch.allclose(t2, loaded)diff --git a/tests/connectors/test_pipeline.py b/tests/connectors/test_pipeline.pyindex d642950b..fa70efce 100644--- a/tests/connectors/test_pipeline.py+++ b/tests/connectors/test_pipeline.py@@ -11,7 +11,6 @@ from openjarvis.connectors._stubs import Documentfrom openjarvis.connectors.pipeline import IngestionPipelinefrom openjarvis.connectors.store import KnowledgeStore-# ---------------------------------------------------------------------------# Helpers# ---------------------------------------------------------------------------diff --git a/tests/connectors/test_retriever.py b/tests/connectors/test_retriever.pyindex a46108b5..80d011c5 100644--- a/tests/connectors/test_retriever.py+++ b/tests/connectors/test_retriever.py@@ -8,7 +8,7 @@ from unittest.mock import MagicMockimport pytest-from openjarvis.connectors.retriever import Reranker, TwoStageRetriever+from openjarvis.connectors.retriever import ColBERTReranker, Reranker, TwoStageRetrieverfrom openjarvis.connectors.store import KnowledgeStorefrom openjarvis.tools.storage._stubs import RetrievalResult@@ -208,3 +208,98 @@ def test_retrieve_recall_k_larger_than_top_k(assert len(call_log) == 1# recall_k=50 vs top_k*3=6 → should use 50assert call_log[0] == 50+++# ---------------------------------------------------------------------------+# Test 9: reranker_uses_cached_embeddings — EmbeddingStore integration+# ---------------------------------------------------------------------------+++def test_reranker_uses_cached_embeddings() -> None:+ """ColBERTReranker checks EmbeddingStore.get() before calling docFromText().++ When a cached embedding is found, docFromText() should NOT be called for+ that candidate.+ """+ import torch # type: ignore[import]++ # Create a fake cached embedding (T=20 tokens, dim=128)+ cached_emb = torch.randn(20, 128)++ # Mock the embedding store+ mock_store = MagicMock()+ mock_store.get.return_value = cached_emb++ # Create a reranker with the mocked embedding store+ reranker = ColBERTReranker(embedding_store=mock_store)++ # Mock the ColBERT model so _load_model succeeds+ mock_model = MagicMock()+ # queryFromText returns a (Q, dim) tensor+ mock_model.queryFromText.return_value = [torch.randn(32, 128)]+ reranker._model = mock_model # bypass _load_model()++ candidates = [+ RetrievalResult(+ content="Some document about AI",+ score=1.0,+ source="gmail",+ metadata={"chunk_id": "chunk-123"},+ ),+ ]++ results = reranker.rerank("AI research", candidates, top_k=1)++ # Embedding store should have been consulted+ mock_store.get.assert_called_once_with("chunk-123")++ # docFromText should NOT have been called (cache hit)+ mock_model.docFromText.assert_not_called()++ # We should still get a result back+ assert len(results) == 1+++# ---------------------------------------------------------------------------+# Test 10: reranker_caches_new_embeddings — stores after compute+# ---------------------------------------------------------------------------+++def test_reranker_caches_new_embeddings() -> None:+ """When EmbeddingStore.get() returns None, docFromText() is called and+ the result is stored back via EmbeddingStore.store()."""+ import torch # type: ignore[import]++ mock_store = MagicMock()+ mock_store.get.return_value = None # cache miss++ reranker = ColBERTReranker(embedding_store=mock_store)++ mock_model = MagicMock()+ mock_model.queryFromText.return_value = [torch.randn(32, 128)]+ # docFromText returns (1, T, dim) which the reranker squeezes+ doc_emb = torch.randn(1, 20, 128)+ mock_model.docFromText.return_value = [doc_emb]+ reranker._model = mock_model++ candidates = [+ RetrievalResult(+ content="Document about neural nets",+ score=1.0,+ source="obsidian",+ metadata={"chunk_id": "chunk-456"},+ ),+ ]++ results = reranker.rerank("neural nets", candidates, top_k=1)++ # docFromText should have been called (cache miss)+ mock_model.docFromText.assert_called_once()++ # The computed embedding should be stored back+ mock_store.store.assert_called_once()+ call_args = mock_store.store.call_args+ assert call_args[0][0] == "chunk-456" # chunk_id+ assert call_args[0][1].shape == (20, 128) # squeezed tensor++ assert len(results) == 1diff --git a/tests/connectors/test_stubs.py b/tests/connectors/test_stubs.pyindex 23c99ef4..d5990c51 100644--- a/tests/connectors/test_stubs.py+++ b/tests/connectors/test_stubs.py@@ -3,7 +3,7 @@from __future__ import annotationsfrom datetime import datetime-from typing import Iterator, List, Optional+from typing import Iterator, Optionalfrom openjarvis.connectors._stubs import (Attachment,@@ -12,7 +12,6 @@ from openjarvis.connectors._stubs import (SyncStatus,)from openjarvis.core.registry import ConnectorRegistry-from openjarvis.tools._stubs import ToolSpecclass FakeConnector(BaseConnector):