Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Add code to acce together APIs

Insop Songab4762b0ddbc
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.py
index 4c550358c..e3518b7b1 100644
--- a/dsp/modules/__init__.py
+++ b/dsp/modules/__init__.py
@@ -11,4 +11,5 @@ from .clarifai import *
from .hf_client import HFClientTGI
-from .hf_client import Anyscale
\ No newline at end of file
+from .hf_client import Anyscale
+from .hf_client import Together
\ No newline at end of file
diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.py
index 07aea1c1e..86e59acf1 100644
--- a/dsp/modules/hf_client.py
+++ b/dsp/modules/hf_client.py
@@ -12,6 +12,17 @@ import time
# from dsp.modules.adapter import TurboAdapter, DavinciAdapter, LlamaAdapter
+import backoff
+
+ERRORS = (Exception)
+
+def backoff_hdlr(details):
+ """Handler from https://pypi.org/project/backoff/"""
+ print(
+ "Backing off {wait:0.1f} seconds after {tries} tries "
+ "calling function {target} with kwargs "
+ "{kwargs}".format(**details)
+ )
class HFClientTGI(HFModel):
def __init__(self, model, port, url="http://future-hgx-1", http_request_kwargs=None, **kwargs):
@@ -194,6 +205,83 @@ class HFServerTGI:
docker_process.terminate()
docker_process.wait()
+class Together(HFModel):
+ def __init__(self, model, **kwargs):
+ super().__init__(model=model, is_client=True)
+ self.session = requests.Session()
+ self.api_base = os.getenv("TOGETHER_API_BASE")
+ self.token = os.getenv("TOGETHER_API_KEY")
+ self.model = model
+ self.kwargs = {
+ "temperature": 0.0,
+ "max_tokens": 512,
+ "top_p": 1,
+ "top_k": 20,
+ "repetition_penalty": 1,
+ "n": 1,
+ **kwargs
+ }
+
+ @backoff.on_exception(
+ backoff.expo,
+ ERRORS,
+ max_time=1000,
+ on_backoff=backoff_hdlr,
+ )
+ def _generate(self, prompt, use_chat_api=False, **kwargs):
+ url = f"{self.api_base}"
+
+ kwargs = {**self.kwargs, **kwargs}
+
+ temperature = kwargs.get("temperature")
+ max_tokens = kwargs.get("max_tokens", 150)
+ top_p = kwargs.get("top_p", 0.7)
+ top_k = kwargs.get("top_k", 50)
+ repetition_penalty = kwargs.get("repetition_penalty", 1)
+
+ if use_chat_api:
+ url = f"{self.api_base}/chat/completions"
+ messages = [
+ {"role": "system", "content": "You are a helpful assistant. You must continue the user text directly without *any* additional interjections."},
+ {"role": "user", "content": prompt}
+ ]
+ body = {
+ "model": self.model,
+ "messages": messages,
+ "temperature": temperature,
+ "max_tokens": max_tokens,
+ "top_p": top_p,
+ "top_k": top_k,
+ "repetition_penalty": repetition_penalty
+ }
+ else:
+ body = {
+ "model": self.model,
+ "prompt": f"[INST]{prompt}[/INST]",
+ "temperature": temperature,
+ "max_tokens": max_tokens,
+ "top_p": top_p,
+ "top_k": top_k,
+ "repetition_penalty": repetition_penalty,
+ "stop": ["[/INST]", "</s>"]
+ }
+
+ headers = {"Authorization": f"Bearer {self.token}"}
+
+ try:
+ with self.session.post(url, headers=headers, json=body) as resp:
+ resp_json = resp.json()
+ if use_chat_api:
+ completions = [resp_json['output'].get('choices', [])[0].get('message', {}).get('content', "")]
+ else:
+ completions = [resp_json['output'].get('choices', [])[0].get('text', "")]
+ response = {"prompt": prompt, "choices": [{"text": c} for c in completions]}
+ return response
+ except Exception as e:
+ if resp_json:
+ print(f"resp_json:{resp_json}")
+ print(f"Failed to parse JSON response: {e}")
+ raise Exception("Received invalid JSON response from server")
class Anyscale(HFModel):
def __init__(self, model, **kwargs):
super().__init__(model=model, is_client=True)
diff --git a/dspy/__init__.py b/dspy/__init__.py
index 353dd6ea6..9296a37d2 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -25,6 +25,7 @@ HFClientTGI = dsp.HFClientTGI
HFClientVLLM = HFClientVLLM
Anyscale = dsp.Anyscale
+Together = dsp.Together
HFModel = dsp.HFModel
OllamaLocal = dsp.OllamaLocal