Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

(feature) Adding support for Mistral AI's API

Franck Stephane Ndzomga6f0f62bd3b07
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.py
index 241a349c5..d98edf39b 100644
--- a/dsp/modules/__init__.py
+++ b/dsp/modules/__init__.py
@@ -6,6 +6,7 @@ from .cohere import *
from .colbertv2 import ColBERTv2
from .databricks import *
from .google import *
+<<<<<<< Updated upstream
from .gpt3 import *
from .hf import HFModel
from .hf_client import Anyscale, HFClientTGI, Together
@@ -13,3 +14,11 @@ from .ollama import *
from .pyserini import *
from .sbert import *
from .sentence_vectorizer import *
+=======
+from .mistral import *
+
+
+from .hf_client import HFClientTGI
+from .hf_client import Anyscale
+from .hf_client import Together
+>>>>>>> Stashed changes
diff --git a/dsp/modules/lm.py b/dsp/modules/lm.py
index fb0b0aab5..93ba6a35c 100644
--- a/dsp/modules/lm.py
+++ b/dsp/modules/lm.py
@@ -53,6 +53,13 @@ class LM(ABC):
x['response'],
),
)
+ elif provider == "mistral":
+ printed.append(
+ (
+ prompt,
+ x['response'].choices
+ )
+ )
else:
printed.append(
(
@@ -69,6 +76,7 @@ class LM(ABC):
break
for idx, (prompt, choices) in enumerate(reversed(printed)):
+ print(printed)
# skip the first `skip` prompts
if (n - idx - 1) < skip:
continue
@@ -84,6 +92,8 @@ class LM(ABC):
text=choices
elif provider == "google":
text = choices[0].parts[0].text
+ elif provider == "mistral":
+ text = choices[0].message.content
else:
text = choices[0]["text"]
self.print_green(text, end="")
diff --git a/dsp/modules/mistral.py b/dsp/modules/mistral.py
new file mode 100644
index 000000000..7c36e0b80
--- /dev/null
+++ b/dsp/modules/mistral.py
@@ -0,0 +1,124 @@
+from typing import Any, Optional
+import backoff
+
+from dsp.modules.lm import LM
+
+try:
+ from mistralai.client import MistralClient
+ from mistralai.models.chat_completion import ChatMessage
+ from mistralai.models.chat_completion import ChatCompletionResponse
+ from mistralai.exceptions import MistralAPIException
+ mistralai_api_error = MistralAPIException
+except ImportError:
+ mistralai_api_error = Exception
+ print("Not loading Mistral AI because it is not installed.")
+
+
+def backoff_hdlr(details):
+ """Handler from https://pypi.org/project/backoff/"""
+ print(
+ "Backing off {wait:0.1f} seconds after {tries} tries "
+ "calling function {target} with kwargs "
+ "{kwargs}".format(**details)
+ )
+
+
+def giveup_hdlr(details):
+ """wrapper function that decides when to give up on retry"""
+ if "rate limits" in details.message:
+ return False
+ return True
+
+
+class Mistral(LM):
+ """Wrapper around Mistral AI's API.
+
+ Currently supported models include `mistral-small-latest`, `mistral-medium-latest`, `mistral-large-latest`.
+ """
+
+ def __init__(
+ self,
+ model: str = "mistral-medium-latest",
+ api_key: Optional[str] = None,
+ **kwargs
+ ):
+ """
+ Parameters
+ ----------
+ model : str
+ Which pre-trained model from Mistral AI to use?
+ Choices are [`mistral-small-latest`, `mistral-medium-latest`, `mistral-large-latest`]
+ api_key : str
+ The API key for Mistral AI.
+ **kwargs: dict
+ Additional arguments to pass to the API provider.
+ """
+ super().__init__(model)
+ self.client = MistralClient(api_key=api_key)
+ self.provider = "mistral"
+ self.kwargs = {
+ "model": model,
+ "temperature": 0.17,
+ "max_tokens": 150,
+ **kwargs
+ }
+
+ self.history: list[dict[str, Any]] = []
+
+ def basic_request(self, prompt: str, **kwargs) -> ChatCompletionResponse:
+ """Basic request to Mistral AI's API."""
+ raw_kwargs = kwargs
+ kwargs = {
+ **self.kwargs,
+ "messages": [ChatMessage(role="user", content=prompt)],
+ **kwargs,
+ }
+
+ # Mistral disallows "n" arguments
+ n = kwargs.pop("n", None)
+ if n is not None and n > 1 and kwargs['temperature'] == 0.0:
+ kwargs['temperature'] = 0.7
+
+ response = self.client.chat(**kwargs)
+
+ history = {
+ "prompt": prompt,
+ "response": response,
+ "kwargs": kwargs,
+ "raw_kwargs": raw_kwargs,
+ }
+ self.history.append(history)
+
+ return response
+
+ @backoff.on_exception(
+ backoff.expo,
+ (mistralai_api_error),
+ max_time=1000,
+ on_backoff=backoff_hdlr,
+ giveup=giveup_hdlr,
+ )
+ def request(self, prompt: str, **kwargs) -> ChatCompletionResponse:
+ """Handles retrieval of completions from Mistral AI whilst handling API errors."""
+ prompt = prompt + "Follow the format only once !"
+ return self.basic_request(prompt, **kwargs)
+
+ def __call__(
+ self,
+ prompt: str,
+ only_completed: bool = True,
+ return_sorted: bool = False,
+ **kwargs
+ ):
+
+ assert only_completed, "for now"
+ assert return_sorted is False, "for now"
+
+ n = kwargs.pop("n", 1)
+
+ completions = []
+ for _ in range(n):
+ response = self.request(prompt, **kwargs)
+ completions.append(response.choices[0].message.content)
+
+ return completions
diff --git a/dspy/__init__.py b/dspy/__init__.py
index 55dd6b6ff..567610af5 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -13,6 +13,7 @@ settings = dsp.settings
AzureOpenAI = dsp.AzureOpenAI
OpenAI = dsp.GPT3
+Mistral = dsp.Mistral
Databricks = dsp.Databricks
Cohere = dsp.Cohere
ColBERTv2 = dsp.ColBERTv2