Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Overhaul pinecone retrieval integration

Mark Snidal19c66a26d357
diff --git a/README.md b/README.md
index 6916259de..f8b18b26d 100644
--- a/README.md
+++ b/README.md
@@ -56,7 +56,11 @@ Or open our intro notebook in Google Colab: [<img align="center" src="https://co
> _Note: If you're looking for Demonstrate-Search-Predict (DSP), which is the previous version of DSPy, you can find it on the [v1](https://github.com/stanfordnlp/dspy/tree/v1) branch of this repo._
+For the optional Pinecone retrieval integration, include the `pinecone` extra:
+```
+pip install dspy-ai[pinecone]
+```
## 2) Syntax: You're in charge of the workflow—it's free-form Python code!
diff --git a/dspy/retrieve/pinecone_rm.py b/dspy/retrieve/pinecone_rm.py
index 0d0f1ce04..e21feb439 100644
--- a/dspy/retrieve/pinecone_rm.py
+++ b/dspy/retrieve/pinecone_rm.py
@@ -3,64 +3,171 @@ Retriever model for Pinecone
Author: Dhar Rawal (@drawal1)
"""
-import pinecone #you will need to install pinecone # type: ignore
-import openai # type: ignore
-import dspy # type: ignore
-
-PINECONE_API_KEY = 'YOUR_PINECONE_API_KEY'
-PINECONE_ENVIRONMENT = 'YOUR PINCONE ENVIRONMENT' # for example 'us-east4-gcp'
-INDEX_NAME = "YOUR PINECONE INDEX NAME" # You should have an index build already. See Pinecone docs
-EMBED_MODEL = "YOUR EMBEDDING MODEL" # For example 'text-embedding-ada-002' for OpenAI gpt-3.5-turbo
-
-def init_pinecone(pinecone_api_key, pinecone_env, index_name):
- """Initialize pinecone and load the index"""
- pinecone.init(
- api_key=pinecone_api_key, # find at app.pinecone.io
- environment=pinecone_env, # next to api key in console
- )
+from typing import Optional, List, Union
+import openai
+import dspy
+import backoff
+
+try:
+ import pinecone
+except ImportError:
+ pinecone = None
- return pinecone.Index(index_name)
+if pinecone is None:
+ raise ImportError(
+ "The pinecone library is required to use PineconeRM. Install it with `pip install dspy-ai[pinecone]`"
+ )
-PINECONE_INDEX = init_pinecone(PINECONE_API_KEY, PINECONE_ENVIRONMENT, INDEX_NAME)
class PineconeRM(dspy.Retrieve):
"""
- A class that uses Pinecone to retrieve the top passages for a given query.
+ A retrieval module that uses Pinecone to return the top passages for a given query.
- Args:
- openai_api_key (str): The API key for OpenAI.
- k (int, optional): The number of top passages to retrieve. Defaults to 3.
+ Assumes that the Pinecone index has been created and populated with the following metadata:
+ - text: The text of the passage
- Returns:
- dspy.Prediction: An object containing the retrieved passages.
+ Args:
+ pinecone_index_name (str): The name of the Pinecone index to query against.
+ pinecone_api_key (str, optional): The Pinecone API key. Defaults to None.
+ pinecone_env (str, optional): The Pinecone environment. Defaults to None.
+ openai_embed_model (str, optional): The OpenAI embedding model to use. Defaults to "text-embedding-ada-002".
+ openai_api_key (str, optional): The API key for OpenAI. Defaults to None.
+ openai_org (str, optional): The organization for OpenAI. Defaults to None.
+ k (int, optional): The number of top passages to retrieve. Defaults to 3.
+
+ Returns:
+ dspy.Prediction: An object containing the retrieved passages.
+
+ Examples:
+ Below is a code snippet that shows how to use this as the default retriver:
+ ```python
+ llm = dspy.OpenAI(model="gpt-3.5-turbo")
+ retriever_model = PineconeRM(openai.api_key)
+ dspy.settings.configure(lm=llm, rm=retriever_model)
+ ```
- Examples:
- Below is a code snippet that shows how to use this as the default retriver:
- ```python
- llm = dspy.OpenAI(model='gpt-3.5-turbo')
- retriever_model = PineconeRM(openai.api_key)
- dspy.settings.configure(lm=llm, rm=retriever_model)
- ```
-
- Below is a code snippet that shows how to use this in the forward() function of a module
- ```python
- self.retrieve = PineconeRM(k=num_passages)
- ```
+ Below is a code snippet that shows how to use this in the forward() function of a module
+ ```python
+ self.retrieve = PineconeRM(k=num_passages)
+ ```
"""
- def __init__(self, openai_api_key, k=3):
- self._openai_api_key = openai_api_key
+
+ def __init__(
+ self,
+ pinecone_index_name: str,
+ pinecone_api_key: Optional[str] = None,
+ pinecone_env: Optional[str] = None,
+ openai_embed_model: str = "text-embedding-ada-002",
+ openai_api_key: Optional[str] = None,
+ openai_org: Optional[str] = None,
+ k: int = 3,
+ ):
+ self._openai_embed_model = openai_embed_model
+ self._pinecone_index = self._init_pinecone(
+ pinecone_index_name, pinecone_api_key, pinecone_env
+ )
+
+ # If not provided, defaults to env vars OPENAI_API_KEY and OPENAI_ORGANIZATION
+ if openai_api_key:
+ openai.api_key = openai_api_key
+ if openai_org:
+ openai.organization = openai_org
+
super().__init__(k=k)
- def forward(self, query_or_queries):
- """ search with pinecone for self.k top passages for query"""
- # convert query_or_queries to a python list if it is not
- queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries
+ def _init_pinecone(
+ self,
+ index_name: str,
+ api_key: Optional[str] = None,
+ environment: Optional[str] = None,
+ ) -> pinecone.Index:
+ """Initialize pinecone and return the loaded index.
+
+ Args:
+ index_name (str): The name of the index to load.
+ api_key (str, optional): The Pinecone API key, defaults to env var PINECONE_API_KEY if not provided.
+ environment (str, optional): The environment (ie. `us-west1-gcp`. Defaults to env PINECONE_ENVIRONMENT.
+
+ Raises:
+ ValueError: If api_key or environment is not provided and not set as an environment variable.
+
+ Returns:
+ pinecone.Index: The loaded index.
+ """
+
+ # Pinecone init overrides default if kwargs are present, so we need to exclude if None
+ kwargs = {}
+ if api_key:
+ kwargs["api_key"] = api_key
+ if environment:
+ kwargs["environment"] = environment
+ pinecone.init(**kwargs)
+
+ return pinecone.Index(index_name)
+
+ @backoff.on_exception(
+ backoff.expo,
+ (openai.error.RateLimitError, openai.error.ServiceUnavailableError),
+ max_time=15,
+ )
+ def _get_embeddings(self, queries: List[str]) -> List[List[float]]:
+ """Return query vector after creating embedding using OpenAI
+
+ Args:
+ queries (list): List of query strings to embed.
+
+ Returns:
+ List[List[float]]: List of embeddings corresponding to each query.
+ """
+ embedding = openai.Embedding.create(
+ input=queries, model=self._openai_embed_model
+ )
+ return [embedding["embedding"] for embedding in embedding["data"]]
+
+ def forward(self, query_or_queries: Union[str, List[str]]) -> dspy.Prediction:
+ """Search with pinecone for self.k top passages for query
+
+ Args:
+ query_or_queries (Union[str, List[str]]): The query or queries to search for.
+
+ Returns:
+ dspy.Prediction: An object containing the retrieved passages.
+ """
+ queries = (
+ [query_or_queries]
+ if isinstance(query_or_queries, str)
+ else query_or_queries
+ )
+ queries = [q for q in queries if q] # Filter empty queries
+ embeddings = self._get_embeddings(queries)
+
+ # For single query, just look up the top k passages
+ if len(queries) == 1:
+ results_dict = self._pinecone_index.query(
+ embeddings[0], top_k=self.k, include_metadata=True
+ )
- embedding = openai.Embedding.create(input=queries, engine=EMBED_MODEL, openai_api_key=self._openai_api_key)
- query_vec = embedding['data'][0]['embedding']
+ # Sort results by score
+ sorted_results = sorted(
+ results_dict["matches"], key=lambda x: x["score"], reverse=True
+ )
+ passages = [result["metadata"]["text"] for result in sorted_results]
+ return dspy.Prediction(passages=passages)
- # retrieve relevant contexts from Pinecone (including the questions)
- results_dict = PINECONE_INDEX.query(query_vec, top_k=self.k, include_metadata=True)
+ # For multiple queries, query each and return the highest scoring passages
+ # If a passage is returned multiple times, the score is accumulated. For this reason we increase top_k by 3x
+ passage_scores = {}
+ for embedding in embeddings:
+ results_dict = self._pinecone_index.query(
+ embedding, top_k=self.k * 3, include_metadata=True
+ )
+ for result in results_dict["matches"]:
+ passage_scores[result["metadata"]["text"]] = (
+ passage_scores.get(result["metadata"]["text"], 0.0)
+ + result["score"]
+ )
- passages = [result['metadata']['text'] for result in results_dict['matches']]
- return dspy.Prediction(passages=passages)
+ sorted_passages = sorted(
+ passage_scores.items(), key=lambda x: x[1], reverse=True
+ )[: self.k]
+ return dspy.Prediction(passages=[passage for passage, _ in sorted_passages])
diff --git a/setup.py b/setup.py
index 7892662bc..a9f5c2b7c 100644
--- a/setup.py
+++ b/setup.py
@@ -21,6 +21,9 @@ setup(
packages=find_packages(include=['dsp.*', 'dspy.*', 'dsp', 'dspy']),
python_requires='>=3.9',
install_requires=requirements,
+ extras_require={
+ "pinecone": ["pinecone-client~=2.2.4"]
+ },
classifiers=[
"Development Status :: 3 - Alpha",
"Intended Audience :: Science/Research",