Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

added client integrations for Anyscale

Arnav Singhvi30e3c67bfb7f
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.py
index 482b53492..e7894c19a 100644
--- a/dsp/modules/__init__.py
+++ b/dsp/modules/__init__.py
@@ -7,4 +7,5 @@ from .cohere import *
from .sbert import *
from .pyserini import *
-from .hf_client import HFClientTGI
\ No newline at end of file
+from .hf_client import HFClientTGI
+from .hf_client import HFClientAnyscale
\ No newline at end of file
diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.py
index d8677ece4..9201b21cc 100644
--- a/dsp/modules/hf_client.py
+++ b/dsp/modules/hf_client.py
@@ -1,3 +1,4 @@
+import os
import random
import requests
from dsp.modules.hf import HFModel, openai_to_hf
@@ -84,6 +85,40 @@ def send_hftgi_request_v00(arg, **kwargs):
return requests.post(arg, **kwargs)
+class HFClientAnyscale(HFModel):
+ def __init__(self, model, **kwargs):
+ super().__init__(model=model, is_client=True)
+ self.session = requests.Session()
+ self.api_base = os.getenv("OPENAI_API_BASE")
+ self.token = os.getenv("OPENAI_API_KEY")
+ self.model = model
+ self.kwargs = {
+ "temperature": 0.7,
+ "n": 1,
+ **kwargs
+ }
+
+ def _generate(self, prompt, **kwargs):
+ url = f"{self.api_base}/chat/completions"
+ kwargs = {**self.kwargs, **kwargs}
+ temperature = kwargs.get("temperature")
+ messages = [{"role": "user", "content": prompt}]
+ body = {
+ "model": self.model,
+ "messages": messages,
+ "temperature": temperature
+ }
+ headers = {"Authorization": f"Bearer {self.token}"}
+ try:
+ with self.session.post(url, headers=headers, json=body) as resp:
+ resp_json = resp.json()
+ completions = [resp_json.get('choices', [])[0].get('message', {}).get('content', "")]
+ response = {"prompt": prompt, "choices": [{"text": c} for c in completions]}
+ return response
+ except Exception as e:
+ print(f"Failed to parse JSON response: {e}")
+ raise Exception("Received invalid JSON response from server")
+
class ChatModuleClient(HFModel):
def __init__(self, model, model_path):
super().__init__(model=model, is_client=True)
diff --git a/dspy/__init__.py b/dspy/__init__.py
index 1376a2c49..8bf8046d0 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -18,5 +18,6 @@ OpenAI = dsp.GPT3
ColBERTv2 = dsp.ColBERTv2
Pyserini = dsp.PyseriniRetriever
HFClientTGI = dsp.HFClientTGI
+HFClientAnyscale = dsp.HFClientAnyscale
ChatModuleClient = ChatModuleClient
HFModel = dsp.HFModel
\ No newline at end of file