Skip to content

Personal AI, On Personal Devices

Python

feat: add EmbeddingStore for ColBERTv2 disk persistence

krypticmouseef859e6a17a4
diff --git a/src/openjarvis/connectors/embedding_store.py b/src/openjarvis/connectors/embedding_store.py
new file mode 100644
index 00000000..6f2e0373
--- /dev/null
+++ b/src/openjarvis/connectors/embedding_store.py
@@ -0,0 +1,179 @@
+"""EmbeddingStore --- disk-persistent ColBERT token-level embeddings.
+
+Stores per-chunk ColBERTv2 token embeddings as individual ``.pt`` files on
+disk with a SQLite index for O(1) chunk_id lookups. Designed to be used by
+``ColBERTReranker`` so that document embeddings computed at ingest time can
+be reused across queries instead of re-encoding on every search.
+
+All ``torch`` usage is lazily imported and guarded behind ``try/except`` so
+that the store degrades gracefully when PyTorch is not installed --- every
+public method returns ``None`` or ``False`` instead of raising.
+"""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+ import torch # type: ignore[import]
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# DDL for the lightweight SQLite index
+# ---------------------------------------------------------------------------
+
+_CREATE_INDEX_TABLE = """
+CREATE TABLE IF NOT EXISTS embedding_index (
+ chunk_id TEXT PRIMARY KEY,
+ filename TEXT NOT NULL
+);
+"""
+
+
+class EmbeddingStore:
+ """Stores ColBERT token-level embeddings on disk.
+
+ Each ``chunk_id`` maps to a tensor of shape ``(num_tokens, 128)`` saved
+ as a standard PyTorch ``.pt`` file. A small SQLite database provides an
+ O(1) index from *chunk_id* to the tensor filename.
+
+ Parameters
+ ----------
+ store_dir:
+ Root directory for the embedding store. Defaults to
+ ``~/.openjarvis/embeddings/``. The directory is created if it does
+ not already exist.
+ """
+
+ def __init__(self, store_dir: str = "") -> None:
+ if not store_dir:
+ from openjarvis.core.config import DEFAULT_CONFIG_DIR
+
+ store_dir = str(DEFAULT_CONFIG_DIR / "embeddings")
+
+ self._store_dir = Path(store_dir)
+ self._tensor_dir = self._store_dir / "tensors"
+ self._tensor_dir.mkdir(parents=True, exist_ok=True)
+
+ db_path = self._store_dir / "index.db"
+ self._conn = sqlite3.connect(str(db_path), check_same_thread=False)
+ self._conn.execute("PRAGMA journal_mode=WAL;")
+ self._conn.executescript(_CREATE_INDEX_TABLE)
+ self._conn.commit()
+
+ # ------------------------------------------------------------------
+ # Public API
+ # ------------------------------------------------------------------
+
+ def store(self, chunk_id: str, embedding: "torch.Tensor") -> None:
+ """Save a chunk's embedding tensor to disk.
+
+ Parameters
+ ----------
+ chunk_id:
+ Unique identifier for the chunk (matches the KnowledgeStore PK).
+ embedding:
+ Tensor of shape ``(num_tokens, 128)`` from ColBERT encoding.
+ """
+ try:
+ import torch as _torch # noqa: F401
+ except ImportError:
+ logger.debug("EmbeddingStore.store: torch not available, skipping.")
+ return
+
+ filename = f"{chunk_id}.pt"
+ tensor_path = self._tensor_dir / filename
+
+ _torch.save(embedding, str(tensor_path))
+
+ self._conn.execute(
+ "INSERT OR REPLACE INTO embedding_index (chunk_id, filename) VALUES (?, ?)",
+ (chunk_id, filename),
+ )
+ self._conn.commit()
+
+ def get(self, chunk_id: str) -> Optional["torch.Tensor"]:
+ """Load a chunk's embedding from disk.
+
+ Returns ``None`` if the chunk has no stored embedding or if
+ ``torch`` is not installed.
+ """
+ try:
+ import torch as _torch # noqa: F401
+ except ImportError:
+ return None
+
+ row = self._conn.execute(
+ "SELECT filename FROM embedding_index WHERE chunk_id = ?",
+ (chunk_id,),
+ ).fetchone()
+
+ if row is None:
+ return None
+
+ tensor_path = self._tensor_dir / row[0]
+ if not tensor_path.exists():
+ # Stale index entry --- clean it up
+ self._conn.execute(
+ "DELETE FROM embedding_index WHERE chunk_id = ?", (chunk_id,)
+ )
+ self._conn.commit()
+ return None
+
+ try:
+ return _torch.load(str(tensor_path), weights_only=True)
+ except Exception as exc:
+ logger.warning(
+ "EmbeddingStore.get: failed to load tensor for %s (%s)", chunk_id, exc
+ )
+ return None
+
+ def has(self, chunk_id: str) -> bool:
+ """Check if embeddings exist for a chunk."""
+ row = self._conn.execute(
+ "SELECT 1 FROM embedding_index WHERE chunk_id = ?",
+ (chunk_id,),
+ ).fetchone()
+ return row is not None
+
+ def count(self) -> int:
+ """Return the number of stored embeddings."""
+ row = self._conn.execute("SELECT COUNT(*) FROM embedding_index").fetchone()
+ return row[0] if row else 0
+
+ def delete(self, chunk_id: str) -> bool:
+ """Delete the embedding for a chunk.
+
+ Returns ``True`` if the embedding existed and was removed.
+ """
+ row = self._conn.execute(
+ "SELECT filename FROM embedding_index WHERE chunk_id = ?",
+ (chunk_id,),
+ ).fetchone()
+
+ if row is None:
+ return False
+
+ tensor_path = self._tensor_dir / row[0]
+ if tensor_path.exists():
+ tensor_path.unlink()
+
+ self._conn.execute(
+ "DELETE FROM embedding_index WHERE chunk_id = ?", (chunk_id,)
+ )
+ self._conn.commit()
+ return True
+
+ def close(self) -> None:
+ """Close the underlying SQLite connection."""
+ try:
+ self._conn.close()
+ except Exception:
+ pass
+
+
+__all__ = ["EmbeddingStore"]
diff --git a/src/openjarvis/connectors/retriever.py b/src/openjarvis/connectors/retriever.py
index e1f483c6..a82716d1 100644
--- a/src/openjarvis/connectors/retriever.py
+++ b/src/openjarvis/connectors/retriever.py
@@ -16,11 +16,14 @@ from __future__ import annotations
import logging
from abc import ABC, abstractmethod
-from typing import List, Optional
+from typing import TYPE_CHECKING, List, Optional
from openjarvis.connectors.store import KnowledgeStore
from openjarvis.tools.storage._stubs import RetrievalResult
+if TYPE_CHECKING:
+ from openjarvis.connectors.embedding_store import EmbeddingStore
+
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
@@ -74,12 +77,22 @@ class ColBERTReranker(Reranker):
checkpoint:
Path or HuggingFace model ID for the ColBERT checkpoint.
Defaults to ``"colbert-ir/colbertv2.0"``.
+ embedding_store:
+ Optional :class:`EmbeddingStore` for caching per-chunk ColBERT
+ token embeddings on disk. When provided, the reranker looks up
+ pre-computed embeddings before falling back to ``docFromText()``,
+ and stores newly computed embeddings for future reuse.
"""
- def __init__(self, checkpoint: str = "colbert-ir/colbertv2.0") -> None:
+ def __init__(
+ self,
+ checkpoint: str = "colbert-ir/colbertv2.0",
+ embedding_store: Optional["EmbeddingStore"] = None,
+ ) -> None:
self._checkpoint = checkpoint
self._model = None
self._warned = False
+ self._embedding_store = embedding_store
def _load_model(self) -> bool:
"""Attempt to load the ColBERT model. Returns True on success."""
@@ -138,24 +151,35 @@ class ColBERTReranker(Reranker):
# Encode query: (Q, dim) where Q=query_maxlen (32)
q_emb = self._model.queryFromText([query])[0]
- doc_texts = [r.content for r in candidates]
- # Score each candidate via MaxSim
+ # Score each candidate via MaxSim, using cached embeddings
+ # from the EmbeddingStore when available.
scores = []
- for text in doc_texts:
- d_emb = self._model.docFromText([text], bsize=1)[0]
- # d_emb shape: (1, T, dim) or (T, dim)
- if d_emb.dim() == 3:
- d_emb = d_emb.squeeze(0)
+ for r in candidates:
+ chunk_id = r.metadata.get("chunk_id", "")
+
+ # Try cached embedding first
+ d_emb = None
+ if self._embedding_store and chunk_id:
+ d_emb = self._embedding_store.get(chunk_id)
+
+ if d_emb is None:
+ # Encode on the fly
+ d_emb = self._model.docFromText([r.content], bsize=1)[0]
+ # d_emb shape: (1, T, dim) or (T, dim)
+ if d_emb.dim() == 3:
+ d_emb = d_emb.squeeze(0)
+ # Cache for future queries
+ if self._embedding_store and chunk_id:
+ self._embedding_store.store(chunk_id, d_emb)
+
sim = torch.nn.functional.cosine_similarity(
q_emb.unsqueeze(1), d_emb.unsqueeze(0), dim=2
)
maxsim = sim.max(dim=1).values.sum().item()
scores.append(maxsim)
- ranked = sorted(
- zip(scores, candidates), key=lambda x: x[0], reverse=True
- )
+ ranked = sorted(zip(scores, candidates), key=lambda x: x[0], reverse=True)
reranked = []
for score, result in ranked[:top_k]:
reranked.append(
diff --git a/src/openjarvis/connectors/store.py b/src/openjarvis/connectors/store.py
index 9e65ff2d..3d10312e 100644
--- a/src/openjarvis/connectors/store.py
+++ b/src/openjarvis/connectors/store.py
@@ -118,6 +118,7 @@ class KnowledgeStore(MemoryBackend):
def __init__(self, db_path: Union[str, Path] = "") -> None:
if not db_path:
from openjarvis.core.config import DEFAULT_CONFIG_DIR
+
db_path = DEFAULT_CONFIG_DIR / "knowledge.db"
self._db_path = str(db_path)
@@ -138,10 +139,7 @@ class KnowledgeStore(MemoryBackend):
self._conn.execute("PRAGMA journal_mode=WAL;")
self._conn.execute("PRAGMA foreign_keys=ON;")
self._conn.executescript(
- _CREATE_MAIN_TABLE
- + _CREATE_FTS_TABLE
- + _CREATE_TRIGGERS
- + _CREATE_INDEXES
+ _CREATE_MAIN_TABLE + _CREATE_FTS_TABLE + _CREATE_TRIGGERS + _CREATE_INDEXES
)
self._conn.commit()
@@ -179,6 +177,7 @@ class KnowledgeStore(MemoryBackend):
# Merge provenance fields into metadata for easy access in results
combined_meta: Dict[str, Any] = dict(metadata or {})
+ combined_meta["chunk_id"] = chunk_id
combined_meta["source"] = source
combined_meta["doc_type"] = doc_type
combined_meta["doc_id"] = doc_id
@@ -310,6 +309,10 @@ class KnowledgeStore(MemoryBackend):
results: List[RetrievalResult] = []
for row in rows:
meta = json.loads(row["metadata"]) if row["metadata"] else {}
+ # Ensure chunk_id is always present in metadata (backfill for
+ # rows stored before this field was added to combined_meta).
+ if "chunk_id" not in meta:
+ meta["chunk_id"] = row["id"]
results.append(
RetrievalResult(
content=row["content"],
diff --git a/tests/connectors/test_embedding_store.py b/tests/connectors/test_embedding_store.py
new file mode 100644
index 00000000..d63c49e4
--- /dev/null
+++ b/tests/connectors/test_embedding_store.py
@@ -0,0 +1,162 @@
+"""Tests for EmbeddingStore — disk-persistent ColBERT token-level embeddings."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from openjarvis.connectors.embedding_store import EmbeddingStore
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+def emb_store(tmp_path: Path) -> EmbeddingStore:
+ """An EmbeddingStore rooted at a temporary directory."""
+ return EmbeddingStore(store_dir=str(tmp_path / "embeddings"))
+
+
+def _make_tensor(rows: int = 20, cols: int = 128):
+ """Create a dummy tensor (rows, cols) mimicking ColBERT token embeddings."""
+ import torch # type: ignore[import]
+
+ return torch.randn(rows, cols)
+
+
+# ---------------------------------------------------------------------------
+# Test 1: store_and_get — round-trip persistence
+# ---------------------------------------------------------------------------
+
+
+def test_store_and_get(emb_store: EmbeddingStore) -> None:
+ """Storing a tensor and getting it back preserves shape and values."""
+ import torch # type: ignore[import]
+
+ tensor = _make_tensor(25, 128)
+ emb_store.store("chunk-abc", tensor)
+
+ loaded = emb_store.get("chunk-abc")
+ assert loaded is not None
+ assert loaded.shape == (25, 128)
+ assert torch.allclose(tensor, loaded)
+
+
+# ---------------------------------------------------------------------------
+# Test 2: has — True after store, False for unknown
+# ---------------------------------------------------------------------------
+
+
+def test_has(emb_store: EmbeddingStore) -> None:
+ """has() returns True for stored chunks, False for unknown ones."""
+ assert emb_store.has("nonexistent") is False
+
+ tensor = _make_tensor()
+ emb_store.store("chunk-1", tensor)
+
+ assert emb_store.has("chunk-1") is True
+ assert emb_store.has("chunk-2") is False
+
+
+# ---------------------------------------------------------------------------
+# Test 3: count — 0 initially, increments on store
+# ---------------------------------------------------------------------------
+
+
+def test_count(emb_store: EmbeddingStore) -> None:
+ """count() tracks the number of stored embeddings."""
+ assert emb_store.count() == 0
+
+ emb_store.store("a", _make_tensor())
+ assert emb_store.count() == 1
+
+ emb_store.store("b", _make_tensor())
+ assert emb_store.count() == 2
+
+
+# ---------------------------------------------------------------------------
+# Test 4: delete — removes tensor, has() returns False after
+# ---------------------------------------------------------------------------
+
+
+def test_delete(emb_store: EmbeddingStore) -> None:
+ """delete() removes both the tensor file and the index entry."""
+ tensor = _make_tensor()
+ emb_store.store("chunk-del", tensor)
+ assert emb_store.has("chunk-del") is True
+
+ result = emb_store.delete("chunk-del")
+ assert result is True
+ assert emb_store.has("chunk-del") is False
+ assert emb_store.get("chunk-del") is None
+ assert emb_store.count() == 0
+
+
+def test_delete_nonexistent(emb_store: EmbeddingStore) -> None:
+ """delete() returns False for a chunk that was never stored."""
+ assert emb_store.delete("nope") is False
+
+
+# ---------------------------------------------------------------------------
+# Test 5: get_nonexistent — returns None
+# ---------------------------------------------------------------------------
+
+
+def test_get_nonexistent(emb_store: EmbeddingStore) -> None:
+ """get() returns None for an unknown chunk_id."""
+ assert emb_store.get("does-not-exist") is None
+
+
+# ---------------------------------------------------------------------------
+# Test 6: graceful_without_torch — mock torch import failure
+# ---------------------------------------------------------------------------
+
+
+def test_graceful_without_torch(tmp_path: Path) -> None:
+ """When torch is not importable, all methods degrade gracefully."""
+ store = EmbeddingStore(store_dir=str(tmp_path / "no_torch"))
+
+ fake_tensor = MagicMock()
+
+ # Patch the import inside embedding_store so torch appears unavailable
+ import builtins
+
+ real_import = builtins.__import__
+
+ def _no_torch(name, *args, **kwargs):
+ if name == "torch" or name.startswith("torch."):
+ raise ImportError("mocked: no torch")
+ return real_import(name, *args, **kwargs)
+
+ with patch("builtins.__import__", side_effect=_no_torch):
+ # store() should silently skip
+ store.store("chunk-x", fake_tensor)
+ assert store.count() == 0
+
+ # get() should return None
+ assert store.get("chunk-x") is None
+
+
+# ---------------------------------------------------------------------------
+# Test 7: overwrite existing embedding
+# ---------------------------------------------------------------------------
+
+
+def test_store_overwrites(emb_store: EmbeddingStore) -> None:
+ """Storing the same chunk_id twice overwrites the previous embedding."""
+ import torch # type: ignore[import]
+
+ t1 = _make_tensor(10, 128)
+ t2 = _make_tensor(15, 128)
+
+ emb_store.store("chunk-ow", t1)
+ emb_store.store("chunk-ow", t2)
+
+ assert emb_store.count() == 1
+ loaded = emb_store.get("chunk-ow")
+ assert loaded is not None
+ assert loaded.shape == (15, 128)
+ assert torch.allclose(t2, loaded)
diff --git a/tests/connectors/test_pipeline.py b/tests/connectors/test_pipeline.py
index d642950b..fa70efce 100644
--- a/tests/connectors/test_pipeline.py
+++ b/tests/connectors/test_pipeline.py
@@ -11,7 +11,6 @@ from openjarvis.connectors._stubs import Document
from openjarvis.connectors.pipeline import IngestionPipeline
from openjarvis.connectors.store import KnowledgeStore
-
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
diff --git a/tests/connectors/test_retriever.py b/tests/connectors/test_retriever.py
index a46108b5..80d011c5 100644
--- a/tests/connectors/test_retriever.py
+++ b/tests/connectors/test_retriever.py
@@ -8,7 +8,7 @@ from unittest.mock import MagicMock
import pytest
-from openjarvis.connectors.retriever import Reranker, TwoStageRetriever
+from openjarvis.connectors.retriever import ColBERTReranker, Reranker, TwoStageRetriever
from openjarvis.connectors.store import KnowledgeStore
from openjarvis.tools.storage._stubs import RetrievalResult
@@ -208,3 +208,98 @@ def test_retrieve_recall_k_larger_than_top_k(
assert len(call_log) == 1
# recall_k=50 vs top_k*3=6 → should use 50
assert call_log[0] == 50
+
+
+# ---------------------------------------------------------------------------
+# Test 9: reranker_uses_cached_embeddings — EmbeddingStore integration
+# ---------------------------------------------------------------------------
+
+
+def test_reranker_uses_cached_embeddings() -> None:
+ """ColBERTReranker checks EmbeddingStore.get() before calling docFromText().
+
+ When a cached embedding is found, docFromText() should NOT be called for
+ that candidate.
+ """
+ import torch # type: ignore[import]
+
+ # Create a fake cached embedding (T=20 tokens, dim=128)
+ cached_emb = torch.randn(20, 128)
+
+ # Mock the embedding store
+ mock_store = MagicMock()
+ mock_store.get.return_value = cached_emb
+
+ # Create a reranker with the mocked embedding store
+ reranker = ColBERTReranker(embedding_store=mock_store)
+
+ # Mock the ColBERT model so _load_model succeeds
+ mock_model = MagicMock()
+ # queryFromText returns a (Q, dim) tensor
+ mock_model.queryFromText.return_value = [torch.randn(32, 128)]
+ reranker._model = mock_model # bypass _load_model()
+
+ candidates = [
+ RetrievalResult(
+ content="Some document about AI",
+ score=1.0,
+ source="gmail",
+ metadata={"chunk_id": "chunk-123"},
+ ),
+ ]
+
+ results = reranker.rerank("AI research", candidates, top_k=1)
+
+ # Embedding store should have been consulted
+ mock_store.get.assert_called_once_with("chunk-123")
+
+ # docFromText should NOT have been called (cache hit)
+ mock_model.docFromText.assert_not_called()
+
+ # We should still get a result back
+ assert len(results) == 1
+
+
+# ---------------------------------------------------------------------------
+# Test 10: reranker_caches_new_embeddings — stores after compute
+# ---------------------------------------------------------------------------
+
+
+def test_reranker_caches_new_embeddings() -> None:
+ """When EmbeddingStore.get() returns None, docFromText() is called and
+ the result is stored back via EmbeddingStore.store()."""
+ import torch # type: ignore[import]
+
+ mock_store = MagicMock()
+ mock_store.get.return_value = None # cache miss
+
+ reranker = ColBERTReranker(embedding_store=mock_store)
+
+ mock_model = MagicMock()
+ mock_model.queryFromText.return_value = [torch.randn(32, 128)]
+ # docFromText returns (1, T, dim) which the reranker squeezes
+ doc_emb = torch.randn(1, 20, 128)
+ mock_model.docFromText.return_value = [doc_emb]
+ reranker._model = mock_model
+
+ candidates = [
+ RetrievalResult(
+ content="Document about neural nets",
+ score=1.0,
+ source="obsidian",
+ metadata={"chunk_id": "chunk-456"},
+ ),
+ ]
+
+ results = reranker.rerank("neural nets", candidates, top_k=1)
+
+ # docFromText should have been called (cache miss)
+ mock_model.docFromText.assert_called_once()
+
+ # The computed embedding should be stored back
+ mock_store.store.assert_called_once()
+ call_args = mock_store.store.call_args
+ assert call_args[0][0] == "chunk-456" # chunk_id
+ assert call_args[0][1].shape == (20, 128) # squeezed tensor
+
+ assert len(results) == 1
diff --git a/tests/connectors/test_stubs.py b/tests/connectors/test_stubs.py
index 23c99ef4..d5990c51 100644
--- a/tests/connectors/test_stubs.py
+++ b/tests/connectors/test_stubs.py
@@ -3,7 +3,7 @@
from __future__ import annotations
from datetime import datetime
-from typing import Iterator, List, Optional
+from typing import Iterator, Optional
from openjarvis.connectors._stubs import (
Attachment,
@@ -12,7 +12,6 @@ from openjarvis.connectors._stubs import (
SyncStatus,
)
from openjarvis.core.registry import ConnectorRegistry
-from openjarvis.tools._stubs import ToolSpec
class FakeConnector(BaseConnector):