DSPy: The framework for programming—not prompting—language models
Python
feature(dspy): added retrieval module for Watson Discovery (#1100)
Andrés Wagner57527d41e44e
diff --git a/docs/api/retrieval_model_clients/WatsonDiscovery.md b/docs/api/retrieval_model_clients/WatsonDiscovery.mdnew file mode 100644index 000000000..c046a78fe--- /dev/null+++ b/docs/api/retrieval_model_clients/WatsonDiscovery.md@@ -0,0 +1,63 @@+---+sidebar_position: 1+---++# retrieve.WatsonDiscoveryRM++### Constructor++The constructor initializes the `WatsonDiscoveryRM` class instance and sets up the request parameters for interacting with Watson Discovery service at IBM Cloud.++```python+class WatsonDiscoveryRM:+ def __init__(+ self,+ apikey: str,+ url:str,+ version:str,+ project_id:str,+ collection_ids:list=[],+ k: int = 7,+ ):+```++**Parameters:**+- `apikey` (str): apikey for authentication purposes,+- `url` (str): endpoint URL that includes the service instance ID+- `version` (str): Release date of the version of the API you want to use. Specify dates in YYYY-MM-DD format.+- `project_id` (str): The Universally Unique Identifier (UUID) of the project.+- `collection_ids` (list): An array containing the collections on which the search will be executed.+- `k` (int, optional): The number of top passages to retrieve. Defaults to 7.++### Methods++#### `forward(self, query_or_queries: Union[str, list[str]], k: Optional[int]= None) -> dspy.Prediction:`++Search the Watson Discovery collection for the top `k` passages matching the given query or queries.++**Parameters:**+- `query_or_queries` (_Union[str, list[str]]_): The query or list of queries to search for.+- `k` (_Optional[int]_, _optional_): The number of results to retrieve. If not specified, defaults to the value set during initialization.++**Returns:**+- `dspy.Prediction`: Contains the retrieved passages, each represented as a `dotdict` with schema `[{"title":str, "long_text": str, "passage_score": float, "document_id": str, "collection_id": str, "start_offset": int, "end_offset": int, "field": str}]`++### Quickstart++```python+import dspy++retriever_model = WatsonDiscoveryRM(+ apikey = "Your API Key",+ url = "URL of the Watson Discovery Service",+ version = "2023-03-31",+ project_id = "Project Id",+ collection_ids = ["Collection ID"],+ k = 5+)++retrieval_response = retriever_model("Explore the significance of quantum computing",k=5)++for result in retrieval_response:+ print("Document:", result.long_text, "\n")+```diff --git a/dspy/retrieve/watson_discovery_rm.py b/dspy/retrieve/watson_discovery_rm.pynew file mode 100644index 000000000..1d0fcf840--- /dev/null+++ b/dspy/retrieve/watson_discovery_rm.py@@ -0,0 +1,107 @@+import json+from typing import Optional, Union++import requests+from requests.auth import HTTPBasicAuth++import dspy+from dsp.utils import dotdict+++class WatsonDiscoveryRM(dspy.Retrieve):+ """A retrieval module that uses Watson Discovery to return the top passages for a given query.++ Args:+ apikey (str): apikey for authentication purposes,+ url (str): endpoint URL that includes the service instance ID+ version (str): Release date of the version of the API you want to use. Specify dates in YYYY-MM-DD format.+ project_id (str): The Universally Unique Identifier (UUID) of the project.+ collection_ids (list): An array containing the collections on which the search will be executed.+ k (int, optional): The number of top passages to retrieve. Defaults to 5.++ Returns:+ dspy.Prediction: An object containing the retrieved passages.+ """++ def __init__(+ self,+ apikey: str,+ url:str,+ version:str,+ project_id:str,+ collection_ids:list=None,+ k: int = 5,+ ):+ if collection_ids is None:+ collection_ids = []+ self.apikey=apikey+ self.url=url,+ self.version:version+ self.project_id:project_id+ self.collection_ids=collection_ids+ self.k: k+ self.query_url=url + "/v2/projects/" + project_id + "/query?version=" + version++ super().__init__(k=k)++ def forward(self, query_or_queries: Union[str, list[str]], k: Optional[int]= None) -> dspy.Prediction:+ """Search with Watson Discovery for self.k top passages for query.++ Args:+ query_or_queries (Union[str, List[str]]): The query or queries to search for.+ k (int, optional): The number of top passages to retrieve.++ Returns:+ dspy.Prediction: An object containing the retrieved passages.+ """+ queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries+ queries = [q for q in queries if q] # Filter empty queries+ k = self.k if k is None else k++ response=[]+ try:+ for query in queries:+ payload = json.dumps({+ "collection_ids": self.collection_ids,+ "natural_language_query": "text:"+query,+ "passages": {+ "enabled": True,+ "count": k,+ "fields":["text"],+ "characters": 500,+ "find_answers": False,+ "per_document": False,+ },+ "highlight": False,+ })+ headers = {'Content-Type': 'application/json'}++ discovery_results = requests.request(+ "POST",+ url=self.query_url,+ headers=headers,+ auth=HTTPBasicAuth("apikey", self.apikey),+ data=payload,+ )++ discovery_results.raise_for_status()++ doc_dict={}+ for d in discovery_results.json()["results"]:+ doc_dict[d["document_id"]]=d["title"]++ for d in discovery_results.json()["passages"]:+ response.append(dotdict({+ "title":doc_dict[d["document_id"]],+ "long_text": doc_dict[d["document_id"]]+" | " + d["passage_text"],+ "passage_score": d["passage_score"],+ "document_id": d["document_id"],+ "collection_id": d["collection_id"],+ "start_offset": d["start_offset"],+ "end_offset": d["end_offset"],+ "field": d["field"],+ }))++ except requests.exceptions.RequestException as err:+ raise SystemExit(err) from err+ return response\ No newline at end of file