Skip to content

Personal AI, On Personal Devices

Python

feat: add TwoStageRetriever with BM25 recall and pluggable ColBERT reranking

krypticmouse2967419691dc
diff --git a/src/openjarvis/connectors/retriever.py b/src/openjarvis/connectors/retriever.py
new file mode 100644
index 00000000..8275c431
--- /dev/null
+++ b/src/openjarvis/connectors/retriever.py
@@ -0,0 +1,260 @@
+"""TwoStageRetriever — BM25 recall from KnowledgeStore + optional reranking.
+
+Composes a fast BM25 first-stage recall (via ``KnowledgeStore``) with an
+optional second-stage ``Reranker`` for semantic reordering. The default
+``ColBERTReranker`` lazy-loads a ColBERT checkpoint and scores candidates via
+MaxSim; it degrades gracefully when ``colbert-ai`` is not installed.
+
+Typical usage::
+
+ store = KnowledgeStore()
+ retriever = TwoStageRetriever(store, reranker=ColBERTReranker())
+ results = retriever.retrieve("neural networks", top_k=10)
+"""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from typing import List, Optional
+
+from openjarvis.connectors.store import KnowledgeStore
+from openjarvis.tools.storage._stubs import RetrievalResult
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Abstract Reranker
+# ---------------------------------------------------------------------------
+
+
+class Reranker(ABC):
+ """Abstract base class for second-stage semantic rerankers."""
+
+ @abstractmethod
+ def rerank(
+ self,
+ query: str,
+ candidates: List[RetrievalResult],
+ *,
+ top_k: int = 10,
+ ) -> List[RetrievalResult]:
+ """Rerank *candidates* for *query* and return the top *top_k* results.
+
+ Parameters
+ ----------
+ query:
+ The original search query.
+ candidates:
+ List of ``RetrievalResult`` objects from the BM25 recall stage.
+ top_k:
+ Maximum number of results to return.
+
+ Returns
+ -------
+ List[RetrievalResult]
+ Reranked results, truncated to *top_k*.
+ """
+
+
+# ---------------------------------------------------------------------------
+# ColBERT reranker
+# ---------------------------------------------------------------------------
+
+
+class ColBERTReranker(Reranker):
+ """Semantic reranker backed by ColBERT MaxSim scoring.
+
+ Lazy-loads a ColBERT checkpoint on first use. If the ``colbert-ai``
+ package is not installed the reranker falls back to returning the
+ BM25-ordered candidates unchanged (with a warning logged once).
+
+ Parameters
+ ----------
+ checkpoint:
+ Path or HuggingFace model ID for the ColBERT checkpoint.
+ Defaults to ``"colbert-ir/colbertv2.0"``.
+ """
+
+ def __init__(self, checkpoint: str = "colbert-ir/colbertv2.0") -> None:
+ self._checkpoint = checkpoint
+ self._model = None
+ self._warned = False
+
+ def _load_model(self) -> bool:
+ """Attempt to load the ColBERT model. Returns True on success."""
+ if self._model is not None:
+ return True
+ try:
+ from colbert import Searcher # type: ignore[import]
+ from colbert.infra import ColBERTConfig # type: ignore[import]
+
+ config = ColBERTConfig(checkpoint=self._checkpoint)
+ self._model = Searcher(index=None, config=config)
+ return True
+ except Exception as exc:
+ if not self._warned:
+ logger.warning(
+ "ColBERTReranker: failed to load colbert-ai (%s). "
+ "Falling back to BM25 order.",
+ exc,
+ )
+ self._warned = True
+ return False
+
+ def rerank(
+ self,
+ query: str,
+ candidates: List[RetrievalResult],
+ *,
+ top_k: int = 10,
+ ) -> List[RetrievalResult]:
+ """Rerank *candidates* using ColBERT MaxSim scores.
+
+ Falls back to BM25 order if ``colbert-ai`` is unavailable.
+ """
+ if not candidates:
+ return []
+
+ if not self._load_model():
+ # Graceful degradation: return BM25 order
+ return candidates[:top_k]
+
+ try:
+ # Encode query and document texts
+ query_enc = self._model.encode_query(query) # (1, L, dim)
+ doc_texts = [r.content for r in candidates]
+ doc_encs = self._model.encode_passages(doc_texts) # (N, D, dim)
+
+ # MaxSim: for each doc, max over doc tokens, sum over query tokens
+ import torch # type: ignore[import]
+
+ scores = []
+ for d_enc in doc_encs:
+ # query_enc: (L, dim), d_enc: (D, dim)
+ sim = torch.matmul(query_enc, d_enc.T) # (L, D)
+ maxsim = sim.max(dim=-1).values.sum().item()
+ scores.append(maxsim)
+
+ ranked = sorted(
+ zip(scores, candidates), key=lambda x: x[0], reverse=True
+ )
+ reranked = []
+ for score, result in ranked[:top_k]:
+ reranked.append(
+ RetrievalResult(
+ content=result.content,
+ score=float(score),
+ source=result.source,
+ metadata=result.metadata,
+ )
+ )
+ return reranked
+
+ except Exception as exc:
+ logger.warning("ColBERTReranker.rerank failed (%s); using BM25 order.", exc)
+ return candidates[:top_k]
+
+
+# ---------------------------------------------------------------------------
+# TwoStageRetriever
+# ---------------------------------------------------------------------------
+
+
+class TwoStageRetriever:
+ """BM25 recall + optional semantic reranking for Deep Research.
+
+ Stage 1 retrieves ``max(recall_k, top_k * 3)`` candidates from the
+ ``KnowledgeStore`` using FTS5/BM25. Stage 2 optionally passes those
+ candidates through a ``Reranker`` to produce a semantically ordered
+ final list of ``top_k`` results.
+
+ Parameters
+ ----------
+ store:
+ The ``KnowledgeStore`` to query in Stage 1.
+ reranker:
+ An optional ``Reranker`` for Stage 2. When *None* the retriever
+ returns BM25-ordered results directly.
+ recall_k:
+ Number of candidates to fetch in Stage 1. The actual recall
+ size is ``max(recall_k, top_k * 3)`` so that the reranker always
+ has a meaningful pool to work with.
+ """
+
+ def __init__(
+ self,
+ store: KnowledgeStore,
+ reranker: Optional[Reranker] = None,
+ *,
+ recall_k: int = 100,
+ ) -> None:
+ self._store = store
+ self._reranker = reranker
+ self._recall_k = recall_k
+
+ def retrieve(
+ self,
+ query: str,
+ *,
+ top_k: int = 10,
+ source: str = "",
+ doc_type: str = "",
+ author: str = "",
+ since: str = "",
+ until: str = "",
+ ) -> List[RetrievalResult]:
+ """Run the two-stage retrieval pipeline.
+
+ Parameters
+ ----------
+ query:
+ Full-text search query.
+ top_k:
+ Maximum number of results to return.
+ source:
+ Restrict to chunks from this source (e.g. ``"gmail"``).
+ doc_type:
+ Restrict to chunks of this doc type (e.g. ``"email"``).
+ author:
+ Restrict to chunks authored by this person.
+ since:
+ Exclude chunks whose timestamp is earlier than this ISO string.
+ until:
+ Exclude chunks whose timestamp is later than this ISO string.
+
+ Returns
+ -------
+ List[RetrievalResult]
+ Up to *top_k* results, reranked when a reranker is configured.
+ """
+ # Determine Stage-1 recall size
+ stage1_k = max(self._recall_k, top_k * 3)
+
+ # Build optional filter kwargs (only pass non-empty values)
+ filter_kwargs = {}
+ if source:
+ filter_kwargs["source"] = source
+ if doc_type:
+ filter_kwargs["doc_type"] = doc_type
+ if author:
+ filter_kwargs["author"] = author
+ if since:
+ filter_kwargs["since"] = since
+ if until:
+ filter_kwargs["until"] = until
+
+ # Stage 1: BM25 recall
+ candidates = self._store.retrieve(query, top_k=stage1_k, **filter_kwargs)
+
+ if not candidates:
+ return []
+
+ # Stage 2: optional reranking
+ if self._reranker is not None and len(candidates) > top_k:
+ return self._reranker.rerank(query, candidates, top_k=top_k)
+
+ return candidates[:top_k]
+
+
+__all__ = ["ColBERTReranker", "Reranker", "TwoStageRetriever"]
diff --git a/tests/connectors/test_retriever.py b/tests/connectors/test_retriever.py
new file mode 100644
index 00000000..a46108b5
--- /dev/null
+++ b/tests/connectors/test_retriever.py
@@ -0,0 +1,210 @@
+"""Tests for TwoStageRetriever — BM25 recall + optional semantic reranking."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import List
+from unittest.mock import MagicMock
+
+import pytest
+
+from openjarvis.connectors.retriever import Reranker, TwoStageRetriever
+from openjarvis.connectors.store import KnowledgeStore
+from openjarvis.tools.storage._stubs import RetrievalResult
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+def store(tmp_path: Path) -> KnowledgeStore:
+ """KnowledgeStore pre-populated with 4 items across different sources/authors."""
+ ks = KnowledgeStore(db_path=tmp_path / "test_retriever.db")
+
+ ks.store(
+ content="AI research on neural networks and deep learning advances.",
+ source="gmail",
+ doc_type="email",
+ author="[email protected]",
+ )
+ ks.store(
+ content="AI research on distributed systems and cloud infrastructure.",
+ source="obsidian",
+ doc_type="note",
+ author="[email protected]",
+ )
+ ks.store(
+ content="AI research on natural language processing and transformers.",
+ source="gmail",
+ doc_type="email",
+ author="[email protected]",
+ )
+ ks.store(
+ content="AI research on reinforcement learning for robotics.",
+ source="slack",
+ doc_type="message",
+ author="[email protected]",
+ )
+
+ return ks
+
+
+def retriever(store: KnowledgeStore) -> TwoStageRetriever:
+ return TwoStageRetriever(store)
+
+
+# ---------------------------------------------------------------------------
+# Test 1: retrieve_returns_results — basic search works
+# ---------------------------------------------------------------------------
+
+
+def test_retrieve_returns_results(retriever: TwoStageRetriever) -> None:
+ """A matching query returns at least one result."""
+ results = retriever.retrieve("AI research")
+ assert isinstance(results, list)
+ assert len(results) >= 1
+ assert all(isinstance(r, RetrievalResult) for r in results)
+
+
+# ---------------------------------------------------------------------------
+# Test 2: retrieve_respects_top_k — limits results
+# ---------------------------------------------------------------------------
+
+
+def test_retrieve_respects_top_k(retriever: TwoStageRetriever) -> None:
+ """retrieve() returns at most top_k results."""
+ results = retriever.retrieve("AI research", top_k=2)
+ assert len(results) <= 2
+
+
+# ---------------------------------------------------------------------------
+# Test 3: retrieve_with_source_filter — filters pass through to store
+# ---------------------------------------------------------------------------
+
+
+def test_retrieve_with_source_filter(
+ store: KnowledgeStore,
+) -> None:
+ """source= filter is forwarded to the KnowledgeStore."""
+ ret = TwoStageRetriever(store)
+ results = ret.retrieve("AI research", source="gmail")
+ assert len(results) >= 1
+ for r in results:
+ assert r.metadata.get("source") == "gmail"
+
+
+# ---------------------------------------------------------------------------
+# Test 4: retrieve_with_author_filter — filters pass through
+# ---------------------------------------------------------------------------
+
+
+def test_retrieve_with_author_filter(
+ store: KnowledgeStore,
+) -> None:
+ """author= filter is forwarded to the KnowledgeStore."""
+ ret = TwoStageRetriever(store)
+ results = ret.retrieve("AI research", author="[email protected]")
+ assert len(results) >= 1
+ for r in results:
+ assert r.metadata.get("author") == "[email protected]"
+
+
+# ---------------------------------------------------------------------------
+# Test 5: retrieve_bm25_only_when_no_colbert — works without reranker
+# ---------------------------------------------------------------------------
+
+
+def test_retrieve_bm25_only_when_no_reranker(
+ store: KnowledgeStore,
+) -> None:
+ """Without a reranker the retriever returns BM25 results directly."""
+ ret = TwoStageRetriever(store, reranker=None)
+ results = ret.retrieve("AI research", top_k=3)
+ assert isinstance(results, list)
+ assert len(results) <= 3
+
+
+# ---------------------------------------------------------------------------
+# Test 6: retrieve_with_colbert_reranking — mock reranker is called
+# ---------------------------------------------------------------------------
+
+
+def test_retrieve_with_colbert_reranking(
+ store: KnowledgeStore,
+) -> None:
+ """When a reranker is provided it is called to reorder candidates."""
+ mock_reranker = MagicMock(spec=Reranker)
+ # The mock reranker returns a pair of results to distinguish from BM25 output
+ reranked = [
+ RetrievalResult(
+ content="Reranked result A",
+ score=0.99,
+ source="gmail",
+ metadata={"reranked": True},
+ ),
+ RetrievalResult(
+ content="Reranked result B",
+ score=0.95,
+ source="slack",
+ metadata={"reranked": True},
+ ),
+ ]
+ mock_reranker.rerank.return_value = reranked
+
+ # recall_k=4 fetches all 4 docs; top_k=2 so len(candidates)=4 > top_k=2
+ ret = TwoStageRetriever(store, reranker=mock_reranker, recall_k=4)
+ results = ret.retrieve("AI research", top_k=2)
+
+ # The reranker must have been called with top_k=2
+ mock_reranker.rerank.assert_called_once()
+ call_args = mock_reranker.rerank.call_args
+ assert call_args[1].get("top_k") == 2
+
+ # Results come from the reranker
+ assert results == reranked
+
+
+# ---------------------------------------------------------------------------
+# Test 7: retrieve_no_results — empty for nonexistent query
+# ---------------------------------------------------------------------------
+
+
+def test_retrieve_no_results(retriever: TwoStageRetriever) -> None:
+ """A query matching nothing returns an empty list."""
+ results = retriever.retrieve("xyzzy_nonexistent_zqjwkm")
+ assert results == []
+
+
+# ---------------------------------------------------------------------------
+# Test 8: retrieve_recall_k_larger_than_top_k — BM25 fetches more than final k
+# ---------------------------------------------------------------------------
+
+
+def test_retrieve_recall_k_larger_than_top_k(
+ store: KnowledgeStore,
+) -> None:
+ """Stage-1 recall fetches max(recall_k, top_k*3) candidates."""
+ call_log: List[int] = []
+
+ class SpyStore(KnowledgeStore):
+ def retrieve(self, query, *, top_k=5, **kwargs): # type: ignore[override]
+ call_log.append(top_k)
+ return super().retrieve(query, top_k=top_k, **kwargs)
+
+ spy = SpyStore(db_path=":memory:")
+ spy.store(
+ content="Deep learning research in computer vision tasks.",
+ source="gmail",
+ doc_type="email",
+ author="[email protected]",
+ )
+
+ ret = TwoStageRetriever(spy, recall_k=50)
+ ret.retrieve("deep learning research", top_k=2)
+
+ # Stage-1 must have requested more candidates than the final top_k
+ assert len(call_log) == 1
+ # recall_k=50 vs top_k*3=6 → should use 50
+ assert call_log[0] == 50