Personal AI, On Personal Devices
Python
feat: add TwoStageRetriever with BM25 recall and pluggable ColBERT reranking
krypticmouse2967419691dc
diff --git a/src/openjarvis/connectors/retriever.py b/src/openjarvis/connectors/retriever.pynew file mode 100644index 00000000..8275c431--- /dev/null+++ b/src/openjarvis/connectors/retriever.py@@ -0,0 +1,260 @@+"""TwoStageRetriever — BM25 recall from KnowledgeStore + optional reranking.++Composes a fast BM25 first-stage recall (via ``KnowledgeStore``) with an+optional second-stage ``Reranker`` for semantic reordering. The default+``ColBERTReranker`` lazy-loads a ColBERT checkpoint and scores candidates via+MaxSim; it degrades gracefully when ``colbert-ai`` is not installed.++Typical usage::++ store = KnowledgeStore()+ retriever = TwoStageRetriever(store, reranker=ColBERTReranker())+ results = retriever.retrieve("neural networks", top_k=10)+"""++from __future__ import annotations++import logging+from abc import ABC, abstractmethod+from typing import List, Optional++from openjarvis.connectors.store import KnowledgeStore+from openjarvis.tools.storage._stubs import RetrievalResult++logger = logging.getLogger(__name__)++# ---------------------------------------------------------------------------+# Abstract Reranker+# ---------------------------------------------------------------------------+++class Reranker(ABC):+ """Abstract base class for second-stage semantic rerankers."""++ @abstractmethod+ def rerank(+ self,+ query: str,+ candidates: List[RetrievalResult],+ *,+ top_k: int = 10,+ ) -> List[RetrievalResult]:+ """Rerank *candidates* for *query* and return the top *top_k* results.++ Parameters+ ----------+ query:+ The original search query.+ candidates:+ List of ``RetrievalResult`` objects from the BM25 recall stage.+ top_k:+ Maximum number of results to return.++ Returns+ -------+ List[RetrievalResult]+ Reranked results, truncated to *top_k*.+ """+++# ---------------------------------------------------------------------------+# ColBERT reranker+# ---------------------------------------------------------------------------+++class ColBERTReranker(Reranker):+ """Semantic reranker backed by ColBERT MaxSim scoring.++ Lazy-loads a ColBERT checkpoint on first use. If the ``colbert-ai``+ package is not installed the reranker falls back to returning the+ BM25-ordered candidates unchanged (with a warning logged once).++ Parameters+ ----------+ checkpoint:+ Path or HuggingFace model ID for the ColBERT checkpoint.+ Defaults to ``"colbert-ir/colbertv2.0"``.+ """++ def __init__(self, checkpoint: str = "colbert-ir/colbertv2.0") -> None:+ self._checkpoint = checkpoint+ self._model = None+ self._warned = False++ def _load_model(self) -> bool:+ """Attempt to load the ColBERT model. Returns True on success."""+ if self._model is not None:+ return True+ try:+ from colbert import Searcher # type: ignore[import]+ from colbert.infra import ColBERTConfig # type: ignore[import]++ config = ColBERTConfig(checkpoint=self._checkpoint)+ self._model = Searcher(index=None, config=config)+ return True+ except Exception as exc:+ if not self._warned:+ logger.warning(+ "ColBERTReranker: failed to load colbert-ai (%s). "+ "Falling back to BM25 order.",+ exc,+ )+ self._warned = True+ return False++ def rerank(+ self,+ query: str,+ candidates: List[RetrievalResult],+ *,+ top_k: int = 10,+ ) -> List[RetrievalResult]:+ """Rerank *candidates* using ColBERT MaxSim scores.++ Falls back to BM25 order if ``colbert-ai`` is unavailable.+ """+ if not candidates:+ return []++ if not self._load_model():+ # Graceful degradation: return BM25 order+ return candidates[:top_k]++ try:+ # Encode query and document texts+ query_enc = self._model.encode_query(query) # (1, L, dim)+ doc_texts = [r.content for r in candidates]+ doc_encs = self._model.encode_passages(doc_texts) # (N, D, dim)++ # MaxSim: for each doc, max over doc tokens, sum over query tokens+ import torch # type: ignore[import]++ scores = []+ for d_enc in doc_encs:+ # query_enc: (L, dim), d_enc: (D, dim)+ sim = torch.matmul(query_enc, d_enc.T) # (L, D)+ maxsim = sim.max(dim=-1).values.sum().item()+ scores.append(maxsim)++ ranked = sorted(+ zip(scores, candidates), key=lambda x: x[0], reverse=True+ )+ reranked = []+ for score, result in ranked[:top_k]:+ reranked.append(+ RetrievalResult(+ content=result.content,+ score=float(score),+ source=result.source,+ metadata=result.metadata,+ )+ )+ return reranked++ except Exception as exc:+ logger.warning("ColBERTReranker.rerank failed (%s); using BM25 order.", exc)+ return candidates[:top_k]+++# ---------------------------------------------------------------------------+# TwoStageRetriever+# ---------------------------------------------------------------------------+++class TwoStageRetriever:+ """BM25 recall + optional semantic reranking for Deep Research.++ Stage 1 retrieves ``max(recall_k, top_k * 3)`` candidates from the+ ``KnowledgeStore`` using FTS5/BM25. Stage 2 optionally passes those+ candidates through a ``Reranker`` to produce a semantically ordered+ final list of ``top_k`` results.++ Parameters+ ----------+ store:+ The ``KnowledgeStore`` to query in Stage 1.+ reranker:+ An optional ``Reranker`` for Stage 2. When *None* the retriever+ returns BM25-ordered results directly.+ recall_k:+ Number of candidates to fetch in Stage 1. The actual recall+ size is ``max(recall_k, top_k * 3)`` so that the reranker always+ has a meaningful pool to work with.+ """++ def __init__(+ self,+ store: KnowledgeStore,+ reranker: Optional[Reranker] = None,+ *,+ recall_k: int = 100,+ ) -> None:+ self._store = store+ self._reranker = reranker+ self._recall_k = recall_k++ def retrieve(+ self,+ query: str,+ *,+ top_k: int = 10,+ source: str = "",+ doc_type: str = "",+ author: str = "",+ since: str = "",+ until: str = "",+ ) -> List[RetrievalResult]:+ """Run the two-stage retrieval pipeline.++ Parameters+ ----------+ query:+ Full-text search query.+ top_k:+ Maximum number of results to return.+ source:+ Restrict to chunks from this source (e.g. ``"gmail"``).+ doc_type:+ Restrict to chunks of this doc type (e.g. ``"email"``).+ author:+ Restrict to chunks authored by this person.+ since:+ Exclude chunks whose timestamp is earlier than this ISO string.+ until:+ Exclude chunks whose timestamp is later than this ISO string.++ Returns+ -------+ List[RetrievalResult]+ Up to *top_k* results, reranked when a reranker is configured.+ """+ # Determine Stage-1 recall size+ stage1_k = max(self._recall_k, top_k * 3)++ # Build optional filter kwargs (only pass non-empty values)+ filter_kwargs = {}+ if source:+ filter_kwargs["source"] = source+ if doc_type:+ filter_kwargs["doc_type"] = doc_type+ if author:+ filter_kwargs["author"] = author+ if since:+ filter_kwargs["since"] = since+ if until:+ filter_kwargs["until"] = until++ # Stage 1: BM25 recall+ candidates = self._store.retrieve(query, top_k=stage1_k, **filter_kwargs)++ if not candidates:+ return []++ # Stage 2: optional reranking+ if self._reranker is not None and len(candidates) > top_k:+ return self._reranker.rerank(query, candidates, top_k=top_k)++ return candidates[:top_k]+++__all__ = ["ColBERTReranker", "Reranker", "TwoStageRetriever"]diff --git a/tests/connectors/test_retriever.py b/tests/connectors/test_retriever.pynew file mode 100644index 00000000..a46108b5--- /dev/null+++ b/tests/connectors/test_retriever.py@@ -0,0 +1,210 @@+"""Tests for TwoStageRetriever — BM25 recall + optional semantic reranking."""++from __future__ import annotations++from pathlib import Path+from typing import List+from unittest.mock import MagicMock++import pytest++from openjarvis.connectors.retriever import Reranker, TwoStageRetriever+from openjarvis.connectors.store import KnowledgeStore+from openjarvis.tools.storage._stubs import RetrievalResult++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------+++def store(tmp_path: Path) -> KnowledgeStore:+ """KnowledgeStore pre-populated with 4 items across different sources/authors."""+ ks = KnowledgeStore(db_path=tmp_path / "test_retriever.db")++ ks.store(+ content="AI research on neural networks and deep learning advances.",+ source="gmail",+ doc_type="email",+ author="[email protected]",+ )+ ks.store(+ content="AI research on distributed systems and cloud infrastructure.",+ source="obsidian",+ doc_type="note",+ author="[email protected]",+ )+ ks.store(+ content="AI research on natural language processing and transformers.",+ source="gmail",+ doc_type="email",+ author="[email protected]",+ )+ ks.store(+ content="AI research on reinforcement learning for robotics.",+ source="slack",+ doc_type="message",+ author="[email protected]",+ )++ return ks+++def retriever(store: KnowledgeStore) -> TwoStageRetriever:+ return TwoStageRetriever(store)+++# ---------------------------------------------------------------------------+# Test 1: retrieve_returns_results — basic search works+# ---------------------------------------------------------------------------+++def test_retrieve_returns_results(retriever: TwoStageRetriever) -> None:+ """A matching query returns at least one result."""+ results = retriever.retrieve("AI research")+ assert isinstance(results, list)+ assert len(results) >= 1+ assert all(isinstance(r, RetrievalResult) for r in results)+++# ---------------------------------------------------------------------------+# Test 2: retrieve_respects_top_k — limits results+# ---------------------------------------------------------------------------+++def test_retrieve_respects_top_k(retriever: TwoStageRetriever) -> None:+ """retrieve() returns at most top_k results."""+ results = retriever.retrieve("AI research", top_k=2)+ assert len(results) <= 2+++# ---------------------------------------------------------------------------+# Test 3: retrieve_with_source_filter — filters pass through to store+# ---------------------------------------------------------------------------+++def test_retrieve_with_source_filter(+ store: KnowledgeStore,+) -> None:+ """source= filter is forwarded to the KnowledgeStore."""+ ret = TwoStageRetriever(store)+ results = ret.retrieve("AI research", source="gmail")+ assert len(results) >= 1+ for r in results:+ assert r.metadata.get("source") == "gmail"+++# ---------------------------------------------------------------------------+# Test 4: retrieve_with_author_filter — filters pass through+# ---------------------------------------------------------------------------+++def test_retrieve_with_author_filter(+ store: KnowledgeStore,+) -> None:+ """author= filter is forwarded to the KnowledgeStore."""+ ret = TwoStageRetriever(store)+ results = ret.retrieve("AI research", author="[email protected]")+ assert len(results) >= 1+ for r in results:+ assert r.metadata.get("author") == "[email protected]"+++# ---------------------------------------------------------------------------+# Test 5: retrieve_bm25_only_when_no_colbert — works without reranker+# ---------------------------------------------------------------------------+++def test_retrieve_bm25_only_when_no_reranker(+ store: KnowledgeStore,+) -> None:+ """Without a reranker the retriever returns BM25 results directly."""+ ret = TwoStageRetriever(store, reranker=None)+ results = ret.retrieve("AI research", top_k=3)+ assert isinstance(results, list)+ assert len(results) <= 3+++# ---------------------------------------------------------------------------+# Test 6: retrieve_with_colbert_reranking — mock reranker is called+# ---------------------------------------------------------------------------+++def test_retrieve_with_colbert_reranking(+ store: KnowledgeStore,+) -> None:+ """When a reranker is provided it is called to reorder candidates."""+ mock_reranker = MagicMock(spec=Reranker)+ # The mock reranker returns a pair of results to distinguish from BM25 output+ reranked = [+ RetrievalResult(+ content="Reranked result A",+ score=0.99,+ source="gmail",+ metadata={"reranked": True},+ ),+ RetrievalResult(+ content="Reranked result B",+ score=0.95,+ source="slack",+ metadata={"reranked": True},+ ),+ ]+ mock_reranker.rerank.return_value = reranked++ # recall_k=4 fetches all 4 docs; top_k=2 so len(candidates)=4 > top_k=2+ ret = TwoStageRetriever(store, reranker=mock_reranker, recall_k=4)+ results = ret.retrieve("AI research", top_k=2)++ # The reranker must have been called with top_k=2+ mock_reranker.rerank.assert_called_once()+ call_args = mock_reranker.rerank.call_args+ assert call_args[1].get("top_k") == 2++ # Results come from the reranker+ assert results == reranked+++# ---------------------------------------------------------------------------+# Test 7: retrieve_no_results — empty for nonexistent query+# ---------------------------------------------------------------------------+++def test_retrieve_no_results(retriever: TwoStageRetriever) -> None:+ """A query matching nothing returns an empty list."""+ results = retriever.retrieve("xyzzy_nonexistent_zqjwkm")+ assert results == []+++# ---------------------------------------------------------------------------+# Test 8: retrieve_recall_k_larger_than_top_k — BM25 fetches more than final k+# ---------------------------------------------------------------------------+++def test_retrieve_recall_k_larger_than_top_k(+ store: KnowledgeStore,+) -> None:+ """Stage-1 recall fetches max(recall_k, top_k*3) candidates."""+ call_log: List[int] = []++ class SpyStore(KnowledgeStore):+ def retrieve(self, query, *, top_k=5, **kwargs): # type: ignore[override]+ call_log.append(top_k)+ return super().retrieve(query, top_k=top_k, **kwargs)++ spy = SpyStore(db_path=":memory:")+ spy.store(+ content="Deep learning research in computer vision tasks.",+ source="gmail",+ doc_type="email",+ author="[email protected]",+ )++ ret = TwoStageRetriever(spy, recall_k=50)+ ret.retrieve("deep learning research", top_k=2)++ # Stage-1 must have requested more candidates than the final top_k+ assert len(call_log) == 1+ # recall_k=50 vs top_k*3=6 → should use 50+ assert call_log[0] == 50