DSPy: The framework for programming—not prompting—language models
Python
add minimum working support for vLLM
Carlo Nucera314aae082c2a
diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.pyindex 12e6e200d..b36bff4d5 100644--- a/dsp/modules/hf_client.py+++ b/dsp/modules/hf_client.py@@ -104,6 +104,48 @@ def send_hftgi_request_v01_wrapped(arg, url, ports, **kwargs):def send_hftgi_request_v00(arg, **kwargs):return requests.post(arg, **kwargs)++class HFClientVLLM(HFModel):+ def __init__(self, model, port, url="http://localhost", **kwargs):+ super().__init__(model=model, is_client=True)+ self.url = f"{url}:{port}"+ self.headers = {"Content-Type": "application/json"}++ def _generate(self, prompt, **kwargs):+ kwargs = {**self.kwargs, **kwargs}++ payload = {+ "model": kwargs["model"],+ "prompt": prompt,+ "max_tokens": kwargs["max_tokens"],+ "temperature": kwargs["temperature"],+ }++ response = send_hfvllm_request_v00(+ f"{self.url}/v1/completions",+ json=payload,+ headers=self.headers,+ )++ try:+ json_response = response.json()+ completions = json_response["choices"]+ response = {+ "prompt": prompt,+ "choices": [{"text": c["text"]} for c in completions],+ }+ return response++ except Exception as e:+ print("Failed to parse JSON response:", response.text)+ raise Exception("Received invalid JSON response from server")+++def send_hfvllm_request_v00(arg, **kwargs):+ return requests.post(arg, **kwargs)++class HFServerTGI:def __init__(self, user_dir):self.model_weights_dir = os.path.abspath(os.path.join(os.getcwd(), "text-generation-inference", user_dir))diff --git a/dspy/__init__.py b/dspy/__init__.pyindex 558163d13..0cbfe86b1 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -1,5 +1,5 @@from dsp.modules.hf_client import ChatModuleClient-from dsp.modules.hf_client import HFServerTGI+from dsp.modules.hf_client import HFServerTGI, HFClientVLLMfrom .signatures import *from .retrieve import *@@ -20,6 +20,7 @@ ColBERTv2 = dsp.ColBERTv2Pyserini = dsp.PyseriniRetrieverHFClientTGI = dsp.HFClientTGI+HFClientVLLM = HFClientVLLMAnyscale = dsp.AnyscaleHFModel = dsp.HFModel