DSPy: The framework for programming—not prompting—language models
Python
making predictinos fast
Somesh Fengade8d206e098168
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.pyindex 79e51402d..b52d663cd 100644--- a/dsp/modules/__init__.py+++ b/dsp/modules/__init__.py@@ -8,6 +8,7 @@ from .colbertv2 import ColBERTv2from .databricks import *from .google import *from .gpt3 import *+from .groq_client import *from .hf import HFModelfrom .hf_client import Anyscale, HFClientTGI, Togetherfrom .mistral import *@@ -15,3 +16,4 @@ from .ollama import *from .pyserini import *from .sbert import *from .sentence_vectorizer import *+diff --git a/dsp/modules/groq.py b/dsp/modules/groq.pydeleted file mode 100644index 8526f4978..000000000--- a/dsp/modules/groq.py+++ /dev/null@@ -1 +0,0 @@-# stub for groq\ No newline at end of filediff --git a/dsp/modules/groq_client.py b/dsp/modules/groq_client.pynew file mode 100644index 000000000..44c591538--- /dev/null+++ b/dsp/modules/groq_client.py@@ -0,0 +1,166 @@+import logging+import os+import json+from typing import Any, Literal, Optional+import backoff+from groq import Groq, AsyncGroq+import groq+import functools+++import dsp+from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_on+from dsp.modules.lm import LM+++# Configure logging+logging.basicConfig(+ level=logging.INFO,+ format="%(message)s",+ handlers=[logging.FileHandler("groq_usage.log")],+)++++def backoff_hdlr(details):+ """Handler from https://pypi.org/project/backoff/"""+ print(+ "Backing off {wait:0.1f} seconds after {tries} tries "+ "calling function {target} with kwargs "+ "{kwargs}".format(**details),+ )+++class GroqLM(LM):+ """Wrapper around groq's API.++ Args:+ model (str, optional): groq supported LLM model to use. Defaults to "mixtral-8x7b-32768".+ api_key (Optional[str], optional): API provider Authentication token. use Defaults to None.+ **kwargs: Additional arguments to pass to the API provider.+ """++ def __init__(+ self,+ model: str = "mixtral-8x7b-32768",+ api_key: Optional[str] = None,+ **kwargs,+ ):+ super().__init__(model)++ if api_key:+ self.api_key = api_key+ self.client = Groq(api_key = api_key)++ self.kwargs = {+ "temperature": 0.0,+ "max_tokens": 150,+ "top_p": 1,+ "frequency_penalty": 0,+ "presence_penalty": 0,+ "n": 1,+ **kwargs,+ }+ models = self.client.models.list().data+ if models is not None:+ if model in [m.id for m in models]:+ self.kwargs["model"] = model+ self.history: list[dict[str, Any]] = []+++ def log_usage(self, response):+ """Log the total tokens from the Groq API response."""+ usage_data = response.get("usage")+ if usage_data:+ total_tokens = usage_data.get("total_tokens")+ logging.info(f"{total_tokens}")++ def basic_request(self, prompt: str, **kwargs):+ raw_kwargs = kwargs++ kwargs = {**self.kwargs, **kwargs}++ kwargs["messages"] = [{"role": "user", "content": prompt}]+ response = self.chat_request(**kwargs)++ history = {+ "prompt": prompt,+ "response": response,+ "kwargs": kwargs,+ "raw_kwargs": raw_kwargs,+ }++ self.history.append(history)++ return response++ @backoff.on_exception(+ backoff.expo,+ groq.RateLimitError,+ max_time=1000,+ on_backoff=backoff_hdlr,+ )+ def request(self, prompt: str, **kwargs):+ """Handles retreival of model completions whilst handling rate limiting and caching."""+ if "model_type" in kwargs:+ del kwargs["model_type"]++ return self.basic_request(prompt, **kwargs)++ def _get_choice_text(self, choice) -> str:+ return choice.message.content++ def chat_request(self, **kwargs):+ """Handles retreival of model completions whilst handling rate limiting and caching."""+ response = self.client.chat.completions.create(**kwargs)+ return response++ def __call__(+ self,+ prompt: str,+ only_completed: bool = True,+ return_sorted: bool = False,+ **kwargs,+ ) -> list[dict[str, Any]]:+ """Retrieves completions from model.++ Args:+ prompt (str): prompt to send to model+ only_completed (bool, optional): return only completed responses and ignores completion due to length. Defaults to True.+ return_sorted (bool, optional): sort the completion choices using the returned probabilities. Defaults to False.++ Returns:+ list[dict[str, Any]]: list of completion choices+ """++ assert only_completed, "for now"+ assert return_sorted is False, "for now"+ response = self.request(prompt, **kwargs)++ if dsp.settings.log_openai_usage:+ self.log_usage(response)++ choices = response.choices++ completions = [self._get_choice_text(c) for c in choices]+ if return_sorted and kwargs.get("n", 1) > 1:+ scored_completions = []++ for c in choices:+ tokens, logprobs = (+ c["logprobs"]["tokens"],+ c["logprobs"]["token_logprobs"],+ )++ if "<|endoftext|>" in tokens:+ index = tokens.index("<|endoftext|>") + 1+ tokens, logprobs = tokens[:index], logprobs[:index]++ avglog = sum(logprobs) / len(logprobs)+ scored_completions.append((avglog, self._get_choice_text(c)))++ scored_completions = sorted(scored_completions, reverse=True)+ completions = [c for _, c in scored_completions]++ return completions+diff --git a/dspy/__init__.py b/dspy/__init__.pyindex f2edcbfce..558917744 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -21,10 +21,12 @@ ColBERTv2 = dsp.ColBERTv2Pyserini = dsp.PyseriniRetrieverClarifai = dsp.ClarifaiLLMGoogle = dsp.Google+GROQ = dsp.GroqLMHFClientTGI = dsp.HFClientTGIHFClientVLLM = HFClientVLLM+Anyscale = dsp.AnyscaleTogether = dsp.TogetherHFModel = dsp.HFModeldiff --git a/poetry.lock b/poetry.lockindex c0f89cf9b..625eefaa7 100644--- a/poetry.lock+++ b/poetry.lock@@ -3119,16 +3119,15 @@ files = [][package.dependencies]-anyio = ">=3.5.0,<5"-distro = ">=1.7.0,<2"-httpx = ">=0.23.0,<1"-pydantic = ">=1.9.0,<3"-sniffio = "*"-tqdm = ">4"-typing-extensions = ">=4.7,<5"+aiohttp = "*"+requests = ">=2.20"+tqdm = "*"[package.extras]-datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"]+datalib = ["numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"]+dev = ["black (>=21.6b0,<22.0)", "pytest (==6.*)", "pytest-asyncio", "pytest-mock"]+embeddings = ["matplotlib", "numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)", "plotly", "scikit-learn (>=1.0.2)", "scipy", "tenacity (>=8.0.1)"]+wandb = ["numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)", "wandb"][[package]]name = "opentelemetry-api"diff --git a/pyproject.toml b/pyproject.tomlindex 85ea9a169..8094beb2a 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -82,7 +82,7 @@ python = ">=3.9,<3.12"pydantic = ">=2.5.0,<=2.7"backoff = "^2.2.1"joblib = "^1.3.2"-openai = ">=0.28.1,<2.0.0"+openai = "0.28.1"pandas = "^2.1.1"regex = "^2023.10.3"ujson = "^5.8.0"