DSPy: The framework for programming—not prompting—language models
Python
Add support in PineconeRM to use a local embedding model
Karim Elmaaroufiae542f06d169
diff --git a/dspy/retrieve/pinecone_rm.py b/dspy/retrieve/pinecone_rm.pyindex 98412133c..fd888a178 100644--- a/dspy/retrieve/pinecone_rm.py+++ b/dspy/retrieve/pinecone_rm.py@@ -3,6 +3,9 @@ Retriever model for PineconeAuthor: Dhar Rawal (@drawal1)"""+import torch+from dsp.utils import dotdict+from transformers import AutoTokenizer, AutoModelfrom typing import Optional, List, Unionimport openaiimport dspy@@ -30,6 +33,7 @@ class PineconeRM(dspy.Retrieve):pinecone_index_name (str): The name of the Pinecone index to query against.pinecone_api_key (str, optional): The Pinecone API key. Defaults to None.pinecone_env (str, optional): The Pinecone environment. Defaults to None.+ local_embed_model (str, optional): The local embedding model to use. A popular default is "sentence-transformers/all-mpnet-base-v2".openai_embed_model (str, optional): The OpenAI embedding model to use. Defaults to "text-embedding-ada-002".openai_api_key (str, optional): The API key for OpenAI. Defaults to None.openai_org (str, optional): The organization for OpenAI. Defaults to None.@@ -57,22 +61,38 @@ class PineconeRM(dspy.Retrieve):pinecone_index_name: str,pinecone_api_key: Optional[str] = None,pinecone_env: Optional[str] = None,- openai_embed_model: str = "text-embedding-ada-002",+ local_embed_model: Optional[str] = None,+ openai_embed_model: Optional[str] = "text-embedding-ada-002",openai_api_key: Optional[str] = None,openai_org: Optional[str] = None,k: int = 3,):- self._openai_embed_model = openai_embed_model+ if local_embed_model is not None:+ self._local_embed_model = AutoModel.from_pretrained(local_embed_model)+ self._local_tokenizer = AutoTokenizer.from_pretrained(local_embed_model)+ self.use_local_model = True+ self.device = torch.device(+ 'cuda:0' if torch.cuda.is_available() else+ 'mps' if torch.backends.mps.is_available()+ else 'cpu'+ )+ elif openai_embed_model is not None:+ self._openai_embed_model = openai_embed_model+ self.use_local_model = False+ # If not provided, defaults to env vars OPENAI_API_KEY and OPENAI_ORGANIZATION+ if openai_api_key:+ openai.api_key = openai_api_key+ if openai_org:+ openai.organization = openai_org+ else:+ raise ValueError(+ "Either local_embed_model or openai_embed_model must be provided."+ )+self._pinecone_index = self._init_pinecone(pinecone_index_name, pinecone_api_key, pinecone_env)- # If not provided, defaults to env vars OPENAI_API_KEY and OPENAI_ORGANIZATION- if openai_api_key:- openai.api_key = openai_api_key- if openai_org:- openai.organization = openai_org-super().__init__(k=k)def _init_pinecone(@@ -119,13 +139,25 @@ class PineconeRM(dspy.Retrieve):)return pinecone.Index(index_name)-++ def _mean_pooling(+ self,+ model_output: torch.tensor,+ attention_mask: torch.tensor+ ) -> torch.tensor:+ token_embeddings = model_output[0] # First element of model_output contains all token embeddings+ input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()+ return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)+@backoff.on_exception(backoff.expo,(openai.error.RateLimitError, openai.error.ServiceUnavailableError),max_time=15,)- def _get_embeddings(self, queries: List[str]) -> List[List[float]]:+ def _get_embeddings(+ self,+ queries: List[str]+ ) -> List[List[float]]:"""Return query vector after creating embedding using OpenAIArgs:@@ -134,10 +166,23 @@ class PineconeRM(dspy.Retrieve):Returns:List[List[float]]: List of embeddings corresponding to each query."""- embedding = openai.Embedding.create(- input=queries, model=self._openai_embed_model- )- return [embedding["embedding"] for embedding in embedding["data"]]+ if not self.use_local_model:+ embedding = openai.Embedding.create(+ input=queries, model=self._openai_embed_model+ )+ return [embedding["embedding"] for embedding in embedding["data"]]++ # Use local model+ encoded_input = self._local_tokenizer(queries, padding=True, truncation=True, return_tensors="pt").to(self.device)+ with torch.no_grad():+ model_output = self._local_embed_model(**encoded_input.to(self.device))++ embeddings = self._mean_pooling(model_output, encoded_input['attention_mask'])+ normalized_embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=1)+ return normalized_embeddings.cpu().numpy().tolist()++ # we need a pooling strategy to get a single vector representation of the input+ # so the default is to take the mean of the hidden statesdef forward(self, query_or_queries: Union[str, List[str]]) -> dspy.Prediction:"""Search with pinecone for self.k top passages for query