Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

added vllm chat completion endpoints and fix inspect_history

tjtanaa0b7289fca059
diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.py
index caa4b6265..fcda7dd8a 100644
--- a/dsp/modules/hf_client.py
+++ b/dsp/modules/hf_client.py
@@ -128,40 +128,77 @@ class HFClientVLLM(HFModel):
self.headers = {"Content-Type": "application/json"}
self.kwargs |= kwargs
+ # kwargs needs to have model, port and url for the lm.copy() to work properly
+ self.kwargs.update({
+ 'model': model,
+ 'port': port,
+ 'url': url
+ })
def _generate(self, prompt, **kwargs):
kwargs = {**self.kwargs, **kwargs}
-
- payload = {
- "model": self.kwargs["model"],
- "prompt": prompt,
- **kwargs,
- }
-
+ # get model_type
+ model_type = kwargs.get("model_type",None)
+
# Round robin the urls.
url = self.urls.pop(0)
self.urls.append(url)
+
+ if model_type == "chat":
+ system_prompt = kwargs.get("system_prompt",None)
+ messages = [{"role": "user", "content": prompt}]
+ if system_prompt:
+ messages.insert(0, {"role": "system", "content": system_prompt})
+ payload = {
+ "model": self.kwargs["model"],
+ "messages": messages,
+ **kwargs,
+ }
+ response = send_hfvllm_chat_request_v00(
+ f"{url}/v1/chat/completions",
+ json=payload,
+ headers=self.headers,
+ )
+
+ try:
+ json_response = response.json()
+ completions = json_response["choices"]
+ response = {
+ "prompt": prompt,
+ "choices": [{"text": c["message"]['content']} for c in completions],
+ }
+ return response
- response = send_hfvllm_request_v00(
- f"{url}/v1/completions",
- json=payload,
- headers=self.headers,
- )
-
- try:
- json_response = response.json()
- completions = json_response["choices"]
- response = {
+ except Exception:
+ print("Failed to parse JSON response:", response.text)
+ raise Exception("Received invalid JSON response from server")
+ else:
+ payload = {
+ "model": self.kwargs["model"],
"prompt": prompt,
- "choices": [{"text": c["text"]} for c in completions],
+ **kwargs,
}
- return response
+
+ response = send_hfvllm_request_v00(
+ f"{url}/v1/completions",
+ json=payload,
+ headers=self.headers,
+ )
+
+ try:
+ json_response = response.json()
+ completions = json_response["choices"]
+ response = {
+ "prompt": prompt,
+ "choices": [{"text": c["text"]} for c in completions],
+ }
+ return response
- except Exception:
- print("Failed to parse JSON response:", response.text)
- raise Exception("Received invalid JSON response from server")
+ except Exception:
+ print("Failed to parse JSON response:", response.text)
+ raise Exception("Received invalid JSON response from server")
@CacheMemory.cache
@@ -169,6 +206,11 @@ def send_hfvllm_request_v00(arg, **kwargs):
return requests.post(arg, **kwargs)
+def send_hfvllm_chat_request_v00(arg, **kwargs):
+ return requests.post(arg, **kwargs)
+
+
class HFServerTGI:
def __init__(self, user_dir):
self.model_weights_dir = os.path.abspath(os.path.join(os.getcwd(), "text-generation-inference", user_dir))
@@ -435,4 +477,4 @@ class HFClientSGLang(HFModel):
@CacheMemory.cache
def send_hfsglang_request_v00(arg, **kwargs):
- return requests.post(arg, **kwargs)
\ No newline at end of file
+ return requests.post(arg, **kwargs)
diff --git a/dsp/modules/lm.py b/dsp/modules/lm.py
index b1c26dc8d..ac65343b4 100644
--- a/dsp/modules/lm.py
+++ b/dsp/modules/lm.py
@@ -63,13 +63,11 @@ class LM(ABC):
if len(printed) >= n:
break
- for idx, (prompt, choices) in enumerate(reversed(printed)):
- printing_value = ""
-
+ for idx, (prompt, choices) in enumerate(printed):
# skip the first `skip` prompts
- if (n - idx - 1) < skip:
+ if (n - idx - 1) > skip:
continue
-
+ printing_value = ""
printing_value += "\n\n\n"
printing_value += prompt