DSPy: The framework for programming—not prompting—language models
Python
Add code to acce together APIs
Insop Songab4762b0ddbc
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.pyindex 4c550358c..e3518b7b1 100644--- a/dsp/modules/__init__.py+++ b/dsp/modules/__init__.py@@ -11,4 +11,5 @@ from .clarifai import *from .hf_client import HFClientTGI-from .hf_client import Anyscale\ No newline at end of file+from .hf_client import Anyscale+from .hf_client import Together\ No newline at end of filediff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.pyindex 07aea1c1e..86e59acf1 100644--- a/dsp/modules/hf_client.py+++ b/dsp/modules/hf_client.py@@ -12,6 +12,17 @@ import time# from dsp.modules.adapter import TurboAdapter, DavinciAdapter, LlamaAdapter+import backoff++ERRORS = (Exception)++def backoff_hdlr(details):+ """Handler from https://pypi.org/project/backoff/"""+ print(+ "Backing off {wait:0.1f} seconds after {tries} tries "+ "calling function {target} with kwargs "+ "{kwargs}".format(**details)+ )class HFClientTGI(HFModel):def __init__(self, model, port, url="http://future-hgx-1", http_request_kwargs=None, **kwargs):@@ -194,6 +205,83 @@ class HFServerTGI:docker_process.terminate()docker_process.wait()+class Together(HFModel):+ def __init__(self, model, **kwargs):+ super().__init__(model=model, is_client=True)+ self.session = requests.Session()+ self.api_base = os.getenv("TOGETHER_API_BASE")+ self.token = os.getenv("TOGETHER_API_KEY")+ self.model = model+ self.kwargs = {+ "temperature": 0.0,+ "max_tokens": 512,+ "top_p": 1,+ "top_k": 20,+ "repetition_penalty": 1,+ "n": 1,+ **kwargs+ }++ @backoff.on_exception(+ backoff.expo,+ ERRORS,+ max_time=1000,+ on_backoff=backoff_hdlr,+ )+ def _generate(self, prompt, use_chat_api=False, **kwargs):+ url = f"{self.api_base}"++ kwargs = {**self.kwargs, **kwargs}++ temperature = kwargs.get("temperature")+ max_tokens = kwargs.get("max_tokens", 150)+ top_p = kwargs.get("top_p", 0.7)+ top_k = kwargs.get("top_k", 50)+ repetition_penalty = kwargs.get("repetition_penalty", 1)++ if use_chat_api:+ url = f"{self.api_base}/chat/completions"+ messages = [+ {"role": "system", "content": "You are a helpful assistant. You must continue the user text directly without *any* additional interjections."},+ {"role": "user", "content": prompt}+ ]+ body = {+ "model": self.model,+ "messages": messages,+ "temperature": temperature,+ "max_tokens": max_tokens,+ "top_p": top_p,+ "top_k": top_k,+ "repetition_penalty": repetition_penalty+ }+ else:+ body = {+ "model": self.model,+ "prompt": f"[INST]{prompt}[/INST]",+ "temperature": temperature,+ "max_tokens": max_tokens,+ "top_p": top_p,+ "top_k": top_k,+ "repetition_penalty": repetition_penalty,+ "stop": ["[/INST]", "</s>"]+ }++ headers = {"Authorization": f"Bearer {self.token}"}++ try:+ with self.session.post(url, headers=headers, json=body) as resp:+ resp_json = resp.json()+ if use_chat_api:+ completions = [resp_json['output'].get('choices', [])[0].get('message', {}).get('content', "")]+ else:+ completions = [resp_json['output'].get('choices', [])[0].get('text', "")]+ response = {"prompt": prompt, "choices": [{"text": c} for c in completions]}+ return response+ except Exception as e:+ if resp_json:+ print(f"resp_json:{resp_json}")+ print(f"Failed to parse JSON response: {e}")+ raise Exception("Received invalid JSON response from server")class Anyscale(HFModel):def __init__(self, model, **kwargs):super().__init__(model=model, is_client=True)diff --git a/dspy/__init__.py b/dspy/__init__.pyindex 353dd6ea6..9296a37d2 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -25,6 +25,7 @@ HFClientTGI = dsp.HFClientTGIHFClientVLLM = HFClientVLLMAnyscale = dsp.Anyscale+Together = dsp.TogetherHFModel = dsp.HFModelOllamaLocal = dsp.OllamaLocal