DSPy: The framework for programming—not prompting—language models
Python
making the fix
Somesh1ebbbde38076
diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.pyindex c850c46d8..4e9ed472e 100644--- a/dsp/modules/hf_client.py+++ b/dsp/modules/hf_client.py@@ -218,11 +218,11 @@ class HFServerTGI:docker_process.wait()class Together(HFModel):- def __init__(self, model, api_key , **kwargs):+ def __init__(self, model, **kwargs):super().__init__(model=model, is_client=True)self.session = requests.Session()- self.api_base = "https://api.together.xyz/v1/"- self.token = api_key+ self.api_base = os.getenv("TOGETHER_API_BASE")+ self.token = os.getenv("TOGETHER_API_KEY")self.model = modelself.use_inst_template = False@@ -248,9 +248,11 @@ class Together(HFModel):max_time=1000,on_backoff=backoff_hdlr,)- def _generate(self, prompt, use_chat_api=True, **kwargs):+ def _generate(self, prompt, use_chat_api=False, **kwargs):url = f"{self.api_base}"+kwargs = {**self.kwargs, **kwargs}+stop = kwargs.get("stop")temperature = kwargs.get("temperature")max_tokens = kwargs.get("max_tokens", 150)@@ -270,10 +272,10 @@ class Together(HFModel):"messages": messages,"temperature": temperature,"max_tokens": max_tokens,- # "top_p": top_p,- # "top_k": top_k,- # "repetition_penalty": repetition_penalty,- # "stop": stop,+ "top_p": top_p,+ "top_k": top_k,+ "repetition_penalty": repetition_penalty,+ "stop": stop,}else:body = {@@ -281,37 +283,26 @@ class Together(HFModel):"prompt": prompt,"temperature": temperature,"max_tokens": max_tokens,- # "top_p": top_p,- # "top_k": top_k,- # "repetition_penalty": repetition_penalty,- # "stop": stop,+ "top_p": top_p,+ "top_k": top_k,+ "repetition_penalty": repetition_penalty,+ "stop": stop,}- headers = {- "Authorization": f"Bearer {self.token}",- "Content-Type": "application/json"- }-- try:- # print(url, headers, body)- # resp = requests.post(url, headers = headers , json = body)- # print(resp.text)- # return resp.json()+ headers = {"Authorization": f"Bearer {self.token}"}+ try:with self.session.post(url, headers=headers, json=body) as resp:- print(resp)resp_json = resp.json()- print(resp_json)if use_chat_api:- completions = [resp_json.get('choices', [])[0].get('message', {}).get('content', "")]+ completions = [resp_json['output'].get('choices', [])[0].get('message', {}).get('content', "")]else:- completions = [resp_json.get('choices', [])[0].get('content', "")]+ completions = [resp_json['output'].get('choices', [])[0].get('text', "")]response = {"prompt": prompt, "choices": [{"text": c} for c in completions]}return responseexcept Exception as e:- pass- # if resp_json:- # print(f"resp_json:{resp_json}")+ if resp_json:+ print(f"resp_json:{resp_json}")print(f"Failed to parse JSON response: {e}")raise Exception("Received invalid JSON response from server")@@ -444,4 +435,4 @@ class HFClientSGLang(HFModel):@CacheMemory.cachedef send_hfsglang_request_v00(arg, **kwargs):- return requests.post(arg, **kwargs)+ return requests.post(arg, **kwargs)\ No newline at end of filediff --git a/dsp/modules/together_client.py b/dsp/modules/together_client.pydeleted file mode 100644index 4dc29ad65..000000000--- a/dsp/modules/together_client.py+++ /dev/null@@ -1,171 +0,0 @@-import logging-import os-import json-from typing import Any, Literal, Optional, Required-import backoff-import together-import functools-import openai----import dsp-from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_on-from dsp.modules.lm import LM---# Configure logging-logging.basicConfig(- level=logging.INFO,- format="%(message)s",- handlers=[logging.FileHandler("together_usage.log")],-)--def backoff_hdlr(details):- """Handler from https://pypi.org/project/backoff/"""- print(- "Backing off {wait:0.1f} seconds after {tries} tries "- "calling function {target} with kwargs "- "{kwargs}".format(**details),- )---class TogetherLM(LM):- """Wrapper around groq's API.-- Args:- model (str, optional): groq supported LLM model to use. Defaults to "mixtral-8x7b-32768".- api_key (Optional[str], optional): API provider Authentication token. use Defaults to None.- **kwargs: Additional arguments to pass to the API provider.- """-- def __init__(- self,- api_key: str,- model: str = "mistralai/Mistral-7B-Instruct-v0.1",- **kwargs,- ):- super().__init__(model)- self.provider = "together"- if api_key:- self.client = openai.OpenAI(- api_key=api_key,- base_url="https://api.together.xyz/v1",- )-- else:- raise ValueError("api_key is required for groq")--- self.kwargs = {- "temperature": 0.0,- "max_tokens": 150,- "top_p": 1,- "frequency_penalty": 0,- "presence_penalty": 0,- "n": 1,- **kwargs,- }- # models = self.client.models.list().data- # if models is not None:- # if model in [m.id for m in models]:- self.kwargs["model"] = model- self.history: list[dict[str, Any]] = []--- def log_usage(self, response):- """Log the total tokens from the Groq API response."""- usage_data = response.get("usage")- if usage_data:- total_tokens = usage_data.get("total_tokens")- logging.info(f"{total_tokens}")-- def basic_request(self, prompt: str, **kwargs):- raw_kwargs = kwargs-- kwargs = {**self.kwargs, **kwargs}-- kwargs["messages"] = [{"role": "user", "content": prompt}]- response = self.chat_request(**kwargs)-- history = {- "prompt": prompt,- "response": response.choices[0].message.content,- "kwargs": kwargs,- "raw_kwargs": raw_kwargs,- }-- self.history.append(history)-- return response-- @backoff.on_exception(- backoff.expo,- openai.RateLimitError,- max_time=1000,- on_backoff=backoff_hdlr,- )- def request(self, prompt: str, **kwargs):- """Handles retreival of model completions whilst handling rate limiting and caching."""- if "model_type" in kwargs:- del kwargs["model_type"]-- return self.basic_request(prompt, **kwargs)-- def _get_choice_text(self, choice) -> str:- return choice.message.content-- def chat_request(self, **kwargs):- """Handles retreival of model completions whilst handling rate limiting and caching."""- response = self.client.chat.completions.create(**kwargs)- return response-- def __call__(- self,- prompt: str,- only_completed: bool = True,- return_sorted: bool = False,- **kwargs,- ) -> list[dict[str, Any]]:- """Retrieves completions from model.-- Args:- prompt (str): prompt to send to model- only_completed (bool, optional): return only completed responses and ignores completion due to length. Defaults to True.- return_sorted (bool, optional): sort the completion choices using the returned probabilities. Defaults to False.-- Returns:- list[dict[str, Any]]: list of completion choices- """-- assert only_completed, "for now"- assert return_sorted is False, "for now"- response = self.request(prompt, **kwargs)-- if dsp.settings.log_openai_usage:- self.log_usage(response)-- choices = response.choices-- completions = [self._get_choice_text(c) for c in choices]- if return_sorted and kwargs.get("n", 1) > 1:- scored_completions = []-- for c in choices:- tokens, logprobs = (- c["logprobs"]["tokens"],- c["logprobs"]["token_logprobs"],- )-- if "<|endoftext|>" in tokens:- index = tokens.index("<|endoftext|>") + 1- tokens, logprobs = tokens[:index], logprobs[:index]-- avglog = sum(logprobs) / len(logprobs)- scored_completions.append((avglog, self._get_choice_text(c)))-- scored_completions = sorted(scored_completions, reverse=True)- completions = [c for _, c in scored_completions]-- return completions-diff --git a/dspy/__init__.py b/dspy/__init__.pyindex 6a5fdf725..75d233281 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -27,7 +27,7 @@ HFClientTGI = dsp.HFClientTGIHFClientVLLM = HFClientVLLMAnyscale = dsp.Anyscale-Together = dsp.TogetherLM+Together = dsp.TogetherHFModel = dsp.HFModelOllamaLocal = dsp.OllamaLocalBedrock = dsp.Bedrockdiff --git a/poetry.lock b/poetry.lockindex dfa6004be..be2cbdc07 100644--- a/poetry.lock+++ b/poetry.lock@@ -1,4 +1,4 @@-# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.+# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.[[package]]name = "aiohttp"@@ -4095,6 +4095,7 @@ files = [{file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},{file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},{file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},+ {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},{file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},{file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},{file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},