Personal AI, On Personal Devices
Python
fix: ColBERTReranker uses Checkpoint API with proper CPU config
krypticmousec3eb294b313b
diff --git a/src/openjarvis/connectors/retriever.py b/src/openjarvis/connectors/retriever.pyindex 8275c431..e1f483c6 100644--- a/src/openjarvis/connectors/retriever.py+++ b/src/openjarvis/connectors/retriever.py@@ -86,11 +86,24 @@ class ColBERTReranker(Reranker):if self._model is not None:return Truetry:- from colbert import Searcher # type: ignore[import]- from colbert.infra import ColBERTConfig # type: ignore[import]+ import os- config = ColBERTConfig(checkpoint=self._checkpoint)- self._model = Searcher(index=None, config=config)+ from colbert.infra.config import ColBERTConfig # type: ignore[import]+ from colbert.modeling.checkpoint import Checkpoint # type: ignore[import]++ # Force CPU if CUDA unavailable+ gpus = 0+ try:+ import torch # type: ignore[import]++ if torch.cuda.is_available():+ gpus = 1+ except Exception:+ pass++ os.environ.setdefault("CUDA_VISIBLE_DEVICES", "")+ config = ColBERTConfig(gpus=gpus)+ self._model = Checkpoint(self._checkpoint, colbert_config=config)return Trueexcept Exception as exc:if not self._warned:@@ -121,19 +134,23 @@ class ColBERTReranker(Reranker):return candidates[:top_k]try:- # Encode query and document texts- query_enc = self._model.encode_query(query) # (1, L, dim)- doc_texts = [r.content for r in candidates]- doc_encs = self._model.encode_passages(doc_texts) # (N, D, dim)-- # MaxSim: for each doc, max over doc tokens, sum over query tokensimport torch # type: ignore[import]+ # Encode query: (Q, dim) where Q=query_maxlen (32)+ q_emb = self._model.queryFromText([query])[0]+ doc_texts = [r.content for r in candidates]++ # Score each candidate via MaxSimscores = []- for d_enc in doc_encs:- # query_enc: (L, dim), d_enc: (D, dim)- sim = torch.matmul(query_enc, d_enc.T) # (L, D)- maxsim = sim.max(dim=-1).values.sum().item()+ for text in doc_texts:+ d_emb = self._model.docFromText([text], bsize=1)[0]+ # d_emb shape: (1, T, dim) or (T, dim)+ if d_emb.dim() == 3:+ d_emb = d_emb.squeeze(0)+ sim = torch.nn.functional.cosine_similarity(+ q_emb.unsqueeze(1), d_emb.unsqueeze(0), dim=2+ )+ maxsim = sim.max(dim=1).values.sum().item()scores.append(maxsim)ranked = sorted(