Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

feature(dspy): added retrieval module for Watson Discovery (#1100)

Andrés Wagner57527d41e44e
diff --git a/docs/api/retrieval_model_clients/WatsonDiscovery.md b/docs/api/retrieval_model_clients/WatsonDiscovery.md
new file mode 100644
index 000000000..c046a78fe
--- /dev/null
+++ b/docs/api/retrieval_model_clients/WatsonDiscovery.md
@@ -0,0 +1,63 @@
+---
+sidebar_position: 1
+---
+
+# retrieve.WatsonDiscoveryRM
+
+### Constructor
+
+The constructor initializes the `WatsonDiscoveryRM` class instance and sets up the request parameters for interacting with Watson Discovery service at IBM Cloud.
+
+```python
+class WatsonDiscoveryRM:
+ def __init__(
+ self,
+ apikey: str,
+ url:str,
+ version:str,
+ project_id:str,
+ collection_ids:list=[],
+ k: int = 7,
+ ):
+```
+
+**Parameters:**
+- `apikey` (str): apikey for authentication purposes,
+- `url` (str): endpoint URL that includes the service instance ID
+- `version` (str): Release date of the version of the API you want to use. Specify dates in YYYY-MM-DD format.
+- `project_id` (str): The Universally Unique Identifier (UUID) of the project.
+- `collection_ids` (list): An array containing the collections on which the search will be executed.
+- `k` (int, optional): The number of top passages to retrieve. Defaults to 7.
+
+### Methods
+
+#### `forward(self, query_or_queries: Union[str, list[str]], k: Optional[int]= None) -> dspy.Prediction:`
+
+Search the Watson Discovery collection for the top `k` passages matching the given query or queries.
+
+**Parameters:**
+- `query_or_queries` (_Union[str, list[str]]_): The query or list of queries to search for.
+- `k` (_Optional[int]_, _optional_): The number of results to retrieve. If not specified, defaults to the value set during initialization.
+
+**Returns:**
+- `dspy.Prediction`: Contains the retrieved passages, each represented as a `dotdict` with schema `[{"title":str, "long_text": str, "passage_score": float, "document_id": str, "collection_id": str, "start_offset": int, "end_offset": int, "field": str}]`
+
+### Quickstart
+
+```python
+import dspy
+
+retriever_model = WatsonDiscoveryRM(
+ apikey = "Your API Key",
+ url = "URL of the Watson Discovery Service",
+ version = "2023-03-31",
+ project_id = "Project Id",
+ collection_ids = ["Collection ID"],
+ k = 5
+)
+
+retrieval_response = retriever_model("Explore the significance of quantum computing",k=5)
+
+for result in retrieval_response:
+ print("Document:", result.long_text, "\n")
+```
diff --git a/dspy/retrieve/watson_discovery_rm.py b/dspy/retrieve/watson_discovery_rm.py
new file mode 100644
index 000000000..1d0fcf840
--- /dev/null
+++ b/dspy/retrieve/watson_discovery_rm.py
@@ -0,0 +1,107 @@
+import json
+from typing import Optional, Union
+
+import requests
+from requests.auth import HTTPBasicAuth
+
+import dspy
+from dsp.utils import dotdict
+
+
+class WatsonDiscoveryRM(dspy.Retrieve):
+ """A retrieval module that uses Watson Discovery to return the top passages for a given query.
+
+ Args:
+ apikey (str): apikey for authentication purposes,
+ url (str): endpoint URL that includes the service instance ID
+ version (str): Release date of the version of the API you want to use. Specify dates in YYYY-MM-DD format.
+ project_id (str): The Universally Unique Identifier (UUID) of the project.
+ collection_ids (list): An array containing the collections on which the search will be executed.
+ k (int, optional): The number of top passages to retrieve. Defaults to 5.
+
+ Returns:
+ dspy.Prediction: An object containing the retrieved passages.
+ """
+
+ def __init__(
+ self,
+ apikey: str,
+ url:str,
+ version:str,
+ project_id:str,
+ collection_ids:list=None,
+ k: int = 5,
+ ):
+ if collection_ids is None:
+ collection_ids = []
+ self.apikey=apikey
+ self.url=url,
+ self.version:version
+ self.project_id:project_id
+ self.collection_ids=collection_ids
+ self.k: k
+ self.query_url=url + "/v2/projects/" + project_id + "/query?version=" + version
+
+ super().__init__(k=k)
+
+ def forward(self, query_or_queries: Union[str, list[str]], k: Optional[int]= None) -> dspy.Prediction:
+ """Search with Watson Discovery for self.k top passages for query.
+
+ Args:
+ query_or_queries (Union[str, List[str]]): The query or queries to search for.
+ k (int, optional): The number of top passages to retrieve.
+
+ Returns:
+ dspy.Prediction: An object containing the retrieved passages.
+ """
+ queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries
+ queries = [q for q in queries if q] # Filter empty queries
+ k = self.k if k is None else k
+
+ response=[]
+ try:
+ for query in queries:
+ payload = json.dumps({
+ "collection_ids": self.collection_ids,
+ "natural_language_query": "text:"+query,
+ "passages": {
+ "enabled": True,
+ "count": k,
+ "fields":["text"],
+ "characters": 500,
+ "find_answers": False,
+ "per_document": False,
+ },
+ "highlight": False,
+ })
+ headers = {'Content-Type': 'application/json'}
+
+ discovery_results = requests.request(
+ "POST",
+ url=self.query_url,
+ headers=headers,
+ auth=HTTPBasicAuth("apikey", self.apikey),
+ data=payload,
+ )
+
+ discovery_results.raise_for_status()
+
+ doc_dict={}
+ for d in discovery_results.json()["results"]:
+ doc_dict[d["document_id"]]=d["title"]
+
+ for d in discovery_results.json()["passages"]:
+ response.append(dotdict({
+ "title":doc_dict[d["document_id"]],
+ "long_text": doc_dict[d["document_id"]]+" | " + d["passage_text"],
+ "passage_score": d["passage_score"],
+ "document_id": d["document_id"],
+ "collection_id": d["collection_id"],
+ "start_offset": d["start_offset"],
+ "end_offset": d["end_offset"],
+ "field": d["field"],
+ }))
+
+ except requests.exceptions.RequestException as err:
+ raise SystemExit(err) from err
+ return response
\ No newline at end of file