Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

making the fix

Somesh1ebbbde38076
diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.py
index c850c46d8..4e9ed472e 100644
--- a/dsp/modules/hf_client.py
+++ b/dsp/modules/hf_client.py
@@ -218,11 +218,11 @@ class HFServerTGI:
docker_process.wait()
class Together(HFModel):
- def __init__(self, model, api_key , **kwargs):
+ def __init__(self, model, **kwargs):
super().__init__(model=model, is_client=True)
self.session = requests.Session()
- self.api_base = "https://api.together.xyz/v1/"
- self.token = api_key
+ self.api_base = os.getenv("TOGETHER_API_BASE")
+ self.token = os.getenv("TOGETHER_API_KEY")
self.model = model
self.use_inst_template = False
@@ -248,9 +248,11 @@ class Together(HFModel):
max_time=1000,
on_backoff=backoff_hdlr,
)
- def _generate(self, prompt, use_chat_api=True, **kwargs):
+ def _generate(self, prompt, use_chat_api=False, **kwargs):
url = f"{self.api_base}"
+
kwargs = {**self.kwargs, **kwargs}
+
stop = kwargs.get("stop")
temperature = kwargs.get("temperature")
max_tokens = kwargs.get("max_tokens", 150)
@@ -270,10 +272,10 @@ class Together(HFModel):
"messages": messages,
"temperature": temperature,
"max_tokens": max_tokens,
- # "top_p": top_p,
- # "top_k": top_k,
- # "repetition_penalty": repetition_penalty,
- # "stop": stop,
+ "top_p": top_p,
+ "top_k": top_k,
+ "repetition_penalty": repetition_penalty,
+ "stop": stop,
}
else:
body = {
@@ -281,37 +283,26 @@ class Together(HFModel):
"prompt": prompt,
"temperature": temperature,
"max_tokens": max_tokens,
- # "top_p": top_p,
- # "top_k": top_k,
- # "repetition_penalty": repetition_penalty,
- # "stop": stop,
+ "top_p": top_p,
+ "top_k": top_k,
+ "repetition_penalty": repetition_penalty,
+ "stop": stop,
}
- headers = {
- "Authorization": f"Bearer {self.token}",
- "Content-Type": "application/json"
- }
-
- try:
- # print(url, headers, body)
- # resp = requests.post(url, headers = headers , json = body)
- # print(resp.text)
- # return resp.json()
+ headers = {"Authorization": f"Bearer {self.token}"}
+ try:
with self.session.post(url, headers=headers, json=body) as resp:
- print(resp)
resp_json = resp.json()
- print(resp_json)
if use_chat_api:
- completions = [resp_json.get('choices', [])[0].get('message', {}).get('content', "")]
+ completions = [resp_json['output'].get('choices', [])[0].get('message', {}).get('content', "")]
else:
- completions = [resp_json.get('choices', [])[0].get('content', "")]
+ completions = [resp_json['output'].get('choices', [])[0].get('text', "")]
response = {"prompt": prompt, "choices": [{"text": c} for c in completions]}
return response
except Exception as e:
- pass
- # if resp_json:
- # print(f"resp_json:{resp_json}")
+ if resp_json:
+ print(f"resp_json:{resp_json}")
print(f"Failed to parse JSON response: {e}")
raise Exception("Received invalid JSON response from server")
@@ -444,4 +435,4 @@ class HFClientSGLang(HFModel):
@CacheMemory.cache
def send_hfsglang_request_v00(arg, **kwargs):
- return requests.post(arg, **kwargs)
+ return requests.post(arg, **kwargs)
\ No newline at end of file
diff --git a/dsp/modules/together_client.py b/dsp/modules/together_client.py
deleted file mode 100644
index 4dc29ad65..000000000
--- a/dsp/modules/together_client.py
+++ /dev/null
@@ -1,171 +0,0 @@
-import logging
-import os
-import json
-from typing import Any, Literal, Optional, Required
-import backoff
-import together
-import functools
-import openai
-
-
-
-import dsp
-from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_on
-from dsp.modules.lm import LM
-
-
-# Configure logging
-logging.basicConfig(
- level=logging.INFO,
- format="%(message)s",
- handlers=[logging.FileHandler("together_usage.log")],
-)
-
-def backoff_hdlr(details):
- """Handler from https://pypi.org/project/backoff/"""
- print(
- "Backing off {wait:0.1f} seconds after {tries} tries "
- "calling function {target} with kwargs "
- "{kwargs}".format(**details),
- )
-
-
-class TogetherLM(LM):
- """Wrapper around groq's API.
-
- Args:
- model (str, optional): groq supported LLM model to use. Defaults to "mixtral-8x7b-32768".
- api_key (Optional[str], optional): API provider Authentication token. use Defaults to None.
- **kwargs: Additional arguments to pass to the API provider.
- """
-
- def __init__(
- self,
- api_key: str,
- model: str = "mistralai/Mistral-7B-Instruct-v0.1",
- **kwargs,
- ):
- super().__init__(model)
- self.provider = "together"
- if api_key:
- self.client = openai.OpenAI(
- api_key=api_key,
- base_url="https://api.together.xyz/v1",
- )
-
- else:
- raise ValueError("api_key is required for groq")
-
-
- self.kwargs = {
- "temperature": 0.0,
- "max_tokens": 150,
- "top_p": 1,
- "frequency_penalty": 0,
- "presence_penalty": 0,
- "n": 1,
- **kwargs,
- }
- # models = self.client.models.list().data
- # if models is not None:
- # if model in [m.id for m in models]:
- self.kwargs["model"] = model
- self.history: list[dict[str, Any]] = []
-
-
- def log_usage(self, response):
- """Log the total tokens from the Groq API response."""
- usage_data = response.get("usage")
- if usage_data:
- total_tokens = usage_data.get("total_tokens")
- logging.info(f"{total_tokens}")
-
- def basic_request(self, prompt: str, **kwargs):
- raw_kwargs = kwargs
-
- kwargs = {**self.kwargs, **kwargs}
-
- kwargs["messages"] = [{"role": "user", "content": prompt}]
- response = self.chat_request(**kwargs)
-
- history = {
- "prompt": prompt,
- "response": response.choices[0].message.content,
- "kwargs": kwargs,
- "raw_kwargs": raw_kwargs,
- }
-
- self.history.append(history)
-
- return response
-
- @backoff.on_exception(
- backoff.expo,
- openai.RateLimitError,
- max_time=1000,
- on_backoff=backoff_hdlr,
- )
- def request(self, prompt: str, **kwargs):
- """Handles retreival of model completions whilst handling rate limiting and caching."""
- if "model_type" in kwargs:
- del kwargs["model_type"]
-
- return self.basic_request(prompt, **kwargs)
-
- def _get_choice_text(self, choice) -> str:
- return choice.message.content
-
- def chat_request(self, **kwargs):
- """Handles retreival of model completions whilst handling rate limiting and caching."""
- response = self.client.chat.completions.create(**kwargs)
- return response
-
- def __call__(
- self,
- prompt: str,
- only_completed: bool = True,
- return_sorted: bool = False,
- **kwargs,
- ) -> list[dict[str, Any]]:
- """Retrieves completions from model.
-
- Args:
- prompt (str): prompt to send to model
- only_completed (bool, optional): return only completed responses and ignores completion due to length. Defaults to True.
- return_sorted (bool, optional): sort the completion choices using the returned probabilities. Defaults to False.
-
- Returns:
- list[dict[str, Any]]: list of completion choices
- """
-
- assert only_completed, "for now"
- assert return_sorted is False, "for now"
- response = self.request(prompt, **kwargs)
-
- if dsp.settings.log_openai_usage:
- self.log_usage(response)
-
- choices = response.choices
-
- completions = [self._get_choice_text(c) for c in choices]
- if return_sorted and kwargs.get("n", 1) > 1:
- scored_completions = []
-
- for c in choices:
- tokens, logprobs = (
- c["logprobs"]["tokens"],
- c["logprobs"]["token_logprobs"],
- )
-
- if "<|endoftext|>" in tokens:
- index = tokens.index("<|endoftext|>") + 1
- tokens, logprobs = tokens[:index], logprobs[:index]
-
- avglog = sum(logprobs) / len(logprobs)
- scored_completions.append((avglog, self._get_choice_text(c)))
-
- scored_completions = sorted(scored_completions, reverse=True)
- completions = [c for _, c in scored_completions]
-
- return completions
-
diff --git a/dspy/__init__.py b/dspy/__init__.py
index 6a5fdf725..75d233281 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -27,7 +27,7 @@ HFClientTGI = dsp.HFClientTGI
HFClientVLLM = HFClientVLLM
Anyscale = dsp.Anyscale
-Together = dsp.TogetherLM
+Together = dsp.Together
HFModel = dsp.HFModel
OllamaLocal = dsp.OllamaLocal
Bedrock = dsp.Bedrock
diff --git a/poetry.lock b/poetry.lock
index dfa6004be..be2cbdc07 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
[[package]]
name = "aiohttp"
@@ -4095,6 +4095,7 @@ files = [
{file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
{file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
{file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
+ {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
{file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
{file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
{file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},