DSPy: The framework for programming—not prompting—language models
Python
added vllm chat completion endpoints and fix inspect_history
tjtanaa0b7289fca059
diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.pyindex caa4b6265..fcda7dd8a 100644--- a/dsp/modules/hf_client.py+++ b/dsp/modules/hf_client.py@@ -128,40 +128,77 @@ class HFClientVLLM(HFModel):self.headers = {"Content-Type": "application/json"}self.kwargs |= kwargs+ # kwargs needs to have model, port and url for the lm.copy() to work properly+ self.kwargs.update({+ 'model': model,+ 'port': port,+ 'url': url+ })def _generate(self, prompt, **kwargs):kwargs = {**self.kwargs, **kwargs}-- payload = {- "model": self.kwargs["model"],- "prompt": prompt,- **kwargs,- }-+ # get model_type+ model_type = kwargs.get("model_type",None)+# Round robin the urls.url = self.urls.pop(0)self.urls.append(url)++ if model_type == "chat":+ system_prompt = kwargs.get("system_prompt",None)+ messages = [{"role": "user", "content": prompt}]+ if system_prompt:+ messages.insert(0, {"role": "system", "content": system_prompt})+ payload = {+ "model": self.kwargs["model"],+ "messages": messages,+ **kwargs,+ }+ response = send_hfvllm_chat_request_v00(+ f"{url}/v1/chat/completions",+ json=payload,+ headers=self.headers,+ )++ try:+ json_response = response.json()+ completions = json_response["choices"]+ response = {+ "prompt": prompt,+ "choices": [{"text": c["message"]['content']} for c in completions],+ }+ return response- response = send_hfvllm_request_v00(- f"{url}/v1/completions",- json=payload,- headers=self.headers,- )-- try:- json_response = response.json()- completions = json_response["choices"]- response = {+ except Exception:+ print("Failed to parse JSON response:", response.text)+ raise Exception("Received invalid JSON response from server")+ else:+ payload = {+ "model": self.kwargs["model"],"prompt": prompt,- "choices": [{"text": c["text"]} for c in completions],+ **kwargs,}- return response++ response = send_hfvllm_request_v00(+ f"{url}/v1/completions",+ json=payload,+ headers=self.headers,+ )++ try:+ json_response = response.json()+ completions = json_response["choices"]+ response = {+ "prompt": prompt,+ "choices": [{"text": c["text"]} for c in completions],+ }+ return response- except Exception:- print("Failed to parse JSON response:", response.text)- raise Exception("Received invalid JSON response from server")+ except Exception:+ print("Failed to parse JSON response:", response.text)+ raise Exception("Received invalid JSON response from server")@CacheMemory.cache@@ -169,6 +206,11 @@ def send_hfvllm_request_v00(arg, **kwargs):return requests.post(arg, **kwargs)+def send_hfvllm_chat_request_v00(arg, **kwargs):+ return requests.post(arg, **kwargs)++class HFServerTGI:def __init__(self, user_dir):self.model_weights_dir = os.path.abspath(os.path.join(os.getcwd(), "text-generation-inference", user_dir))@@ -435,4 +477,4 @@ class HFClientSGLang(HFModel):@CacheMemory.cachedef send_hfsglang_request_v00(arg, **kwargs):- return requests.post(arg, **kwargs)\ No newline at end of file+ return requests.post(arg, **kwargs)diff --git a/dsp/modules/lm.py b/dsp/modules/lm.pyindex b1c26dc8d..ac65343b4 100644--- a/dsp/modules/lm.py+++ b/dsp/modules/lm.py@@ -63,13 +63,11 @@ class LM(ABC):if len(printed) >= n:break- for idx, (prompt, choices) in enumerate(reversed(printed)):- printing_value = ""-+ for idx, (prompt, choices) in enumerate(printed):# skip the first `skip` prompts- if (n - idx - 1) < skip:+ if (n - idx - 1) > skip:continue-+ printing_value = ""printing_value += "\n\n\n"printing_value += prompt