Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

add minimum working support for vLLM

Carlo Nucera314aae082c2a
diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.py
index 12e6e200d..b36bff4d5 100644
--- a/dsp/modules/hf_client.py
+++ b/dsp/modules/hf_client.py
@@ -104,6 +104,48 @@ def send_hftgi_request_v01_wrapped(arg, url, ports, **kwargs):
def send_hftgi_request_v00(arg, **kwargs):
return requests.post(arg, **kwargs)
+
+class HFClientVLLM(HFModel):
+ def __init__(self, model, port, url="http://localhost", **kwargs):
+ super().__init__(model=model, is_client=True)
+ self.url = f"{url}:{port}"
+ self.headers = {"Content-Type": "application/json"}
+
+ def _generate(self, prompt, **kwargs):
+ kwargs = {**self.kwargs, **kwargs}
+
+ payload = {
+ "model": kwargs["model"],
+ "prompt": prompt,
+ "max_tokens": kwargs["max_tokens"],
+ "temperature": kwargs["temperature"],
+ }
+
+ response = send_hfvllm_request_v00(
+ f"{self.url}/v1/completions",
+ json=payload,
+ headers=self.headers,
+ )
+
+ try:
+ json_response = response.json()
+ completions = json_response["choices"]
+ response = {
+ "prompt": prompt,
+ "choices": [{"text": c["text"]} for c in completions],
+ }
+ return response
+
+ except Exception as e:
+ print("Failed to parse JSON response:", response.text)
+ raise Exception("Received invalid JSON response from server")
+
+
+def send_hfvllm_request_v00(arg, **kwargs):
+ return requests.post(arg, **kwargs)
+
+
class HFServerTGI:
def __init__(self, user_dir):
self.model_weights_dir = os.path.abspath(os.path.join(os.getcwd(), "text-generation-inference", user_dir))
diff --git a/dspy/__init__.py b/dspy/__init__.py
index 558163d13..0cbfe86b1 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -1,5 +1,5 @@
from dsp.modules.hf_client import ChatModuleClient
-from dsp.modules.hf_client import HFServerTGI
+from dsp.modules.hf_client import HFServerTGI, HFClientVLLM
from .signatures import *
from .retrieve import *
@@ -20,6 +20,7 @@ ColBERTv2 = dsp.ColBERTv2
Pyserini = dsp.PyseriniRetriever
HFClientTGI = dsp.HFClientTGI
+HFClientVLLM = HFClientVLLM
Anyscale = dsp.Anyscale
HFModel = dsp.HFModel