Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

feature(modules): added support for aws providers (bedrock and sagemaker) and models (mistral, anthropic, meta)

Dhar Rawal702c04fc25f3
diff --git a/docs/api/language_model_clients/aws.md b/docs/api/language_model_clients/aws.md
new file mode 100644
index 000000000..afac264f3
--- /dev/null
+++ b/docs/api/language_model_clients/aws.md
@@ -0,0 +1,61 @@
+---
+sidebar_position: 9
+---
+
+# dsp.AWSMistral, dsp.AWSAnthropic, dsp.AWSMeta
+
+### Usage
+
+```python
+# Notes:
+# 1. Install boto3 to use AWS models.
+# 2. Configure your AWS credentials with the AWS CLI before using these models
+
+# initialize the bedrock aws provider
+bedrock = Bedrock(region_name="us-west-2")
+# For mixtral on Bedrock
+lm = AWSMistral(bedrock, "mistral.mixtral-8x7b-instruct-v0:1", **kwargs)
+# For haiku on Bedrock
+lm = AWSAnthropic(bedrock, "anthropic.claude-3-haiku-20240307-v1:0", **kwargs)
+# For llama2 on Bedrock
+lm = AWSMeta(bedrock, "meta.llama2-13b-chat-v1", **kwargs)
+
+# initialize the sagemaker aws provider
+sagemaker = Sagemaker(region_name="us-west-2")
+# For mistral on Sagemaker
+# Note: you need to create a Sagemaker endpoint for the mistral model first
+lm = AWSMistral(sagemaker, "<YOUR_MISTRAL_ENDPOINT_NAME>", **kwargs)
+
+```
+
+### Constructor
+
+The constructor initializes the base class `LM` and the `AWSProvider` class.
+
+```python
+class AWSMistral(AWSModel):
+ """Mistral family of models."""
+
+ def __init__(
+ self,
+ aws_provider: AWSProvider,
+ model: str,
+ max_context_size: int = 32768,
+ max_new_tokens: int = 1500,
+ **kwargs
+ ) -> None:
+```
+
+**Parameters:**
+- `aws_provider` (AWSProvider): The aws provider to use. One of `Bedrock` or `Sagemaker`.
+- `model` (_str_): Mistral AI pretrained models. Defaults to `mistral-medium-latest`.
+- `max_context_size` (_Optional[int]_, _optional_): Max context size for this model. Defaults to 32768.
+- `max_new_tokens` (_Optional[int]_, _optional_): Max new tokens possible for this model. Defaults to 1500.
+- `**kwargs`: Additional language model arguments to pass to the API provider.
+
+### Methods
+
+Refer to [`dspy.OpenAI`](https://dspy-docs.vercel.app/api/language_model_clients/OpenAI) documentation.
+
+
+`AWSAnthropic` and `AWSMeta` work exactly the same as `AWSMistral`.
\ No newline at end of file
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.py
index b7a739f50..935f65b40 100644
--- a/dsp/modules/__init__.py
+++ b/dsp/modules/__init__.py
@@ -1,6 +1,10 @@
from .anthropic import Claude
+from .aws_models import AWSAnthropic, AWSMeta, AWSMistral, AWSModel
+
+# Below is obsolete. It has been replaced with Bedrock class in dsp/modules/aws_providers.py
+# from .bedrock import *
+from .aws_providers import Bedrock, Sagemaker
from .azure_openai import AzureOpenAI
-from .bedrock import *
from .cache_utils import *
from .clarifai import *
from .cohere import *
@@ -17,4 +21,3 @@ from .ollama import *
from .pyserini import *
from .sbert import *
from .sentence_vectorizer import *
-
diff --git a/dsp/modules/aws_models.py b/dsp/modules/aws_models.py
new file mode 100644
index 000000000..3b9394889
--- /dev/null
+++ b/dsp/modules/aws_models.py
@@ -0,0 +1,300 @@
+"""AWS models for LMs."""
+
+from __future__ import annotations
+
+import json
+import logging
+from abc import abstractmethod
+from typing import Any
+
+from dsp.modules.aws_providers import AWSProvider, Bedrock, Sagemaker
+from dsp.modules.lm import LM
+
+# Heuristic translating number of chars to tokens
+# ~4 chars = 1 token
+CHARS2TOKENS: int = 4
+
+
+class AWSModel(LM):
+ """This class adds support for an AWS model.
+ It is an abstract class and should not be instantiated directly.
+ Instead, use one of the subclasses - AWSMistral, AWSAnthropic, or AWSMeta.
+ The subclasses implement the abstract methods _create_body and _call_model and work in conjunction with the AWSProvider classes Bedrock and Sagemaker.
+ Usage Example:
+ bedrock = Bedrock(region_name="us-west-2")
+ bedrock_mixtral = AWSMistral(bedrock, "mistral.mixtral-8x7b-instruct-v0:1", **kwargs)
+ bedrock_haiku = AWSAnthropic(bedrock, "anthropic.claude-3-haiku-20240307-v1:0", **kwargs)
+ bedrock_llama2 = AWSMeta(bedrock, "meta.llama2-13b-chat-v1", **kwargs)
+
+ sagemaker = Sagemaker(region_name="us-west-2")
+ sagemaker_model = AWSMistral(sagemaker, "<YOUR_ENDPOINT_NAME>", **kwargs)
+ """
+
+ def __init__(
+ self,
+ model: str,
+ max_context_size: int,
+ max_new_tokens: int,
+ **kwargs,
+ ) -> None:
+ """_summary_.
+
+ Args:
+ model (str, optional): An LM name, e.g., a bedrock name or an AWS endpoint.
+ max_context_size (int): The maximum context size in tokens.
+ max_new_tokens (int): The maximum number of tokens to be sampled from the LM.
+ """
+ super().__init__(model=model)
+ self._model_name: str = model
+ self._max_context_size: int = max_context_size
+ self._max_new_tokens: int = max_new_tokens
+
+ self.kwargs = {
+ **self.kwargs,
+ **kwargs,
+ }
+
+ @abstractmethod
+ def _create_body(self, prompt: str, **kwargs) -> tuple[int, dict[str, str | float]]:
+ pass
+
+ @abstractmethod
+ def _call_model(self, body: str) -> str | list[str]:
+ """Call model, get generated input without the formatted prompt."""
+
+ def _estimate_tokens(self, text: str) -> int:
+ return len(text)/CHARS2TOKENS
+
+ def _extract_input_parameters(
+ self,
+ body: dict[Any, Any],
+ ) -> dict[str, str | float | int]:
+ return body
+
+ def _format_prompt(self, raw_prompt: str) -> str:
+ return "\n\nHuman: " + raw_prompt + "\n\nAssistant:"
+
+ def _simple_api_call(self, formatted_prompt: str, **kwargs) -> str | list[str]:
+ n, body = self._create_body(formatted_prompt, **kwargs)
+ json_body = json.dumps(body)
+
+ if n > 1:
+ llm_out = [self._call_model(json_body) for _ in range(n)]
+ llm_out = [generated.replace(formatted_prompt, "") for generated in llm_out]
+ else:
+ llm_out = self._call_model(json_body)
+ llm_out = llm_out.replace(formatted_prompt, "")
+
+ self.history.append(
+ {"prompt": formatted_prompt, "response": llm_out, "kwargs": body},
+ )
+ return llm_out
+
+ def basic_request(self, prompt, **kwargs) -> str | list[str]:
+ """Query the endpoint."""
+ token_count = self._estimate_tokens(prompt)
+ if token_count > self._max_context_size:
+ logging.info("Error - input tokens %s exceeds max context %s", token_count, self._max_context_size)
+ raise ValueError(
+ f"Error - input tokens {token_count} exceeds max context {self._max_context_size}",
+ )
+
+ formatted_prompt: str = self._format_prompt(prompt)
+ return self._simple_api_call(formatted_prompt=formatted_prompt, **kwargs)
+
+ def __call__(
+ self,
+ prompt: str,
+ only_completed: bool = True,
+ return_sorted: bool = False,
+ **kwargs,
+ ) -> list[str]:
+ """Query the AWS LLM.
+
+ There is only support for only_completed=True and return_sorted=False
+ right now.
+ """
+ assert only_completed, "for now"
+ assert return_sorted is False, "for now"
+
+ generated = self.basic_request(prompt, **kwargs)
+ return [generated]
+
+
+class AWSMistral(AWSModel):
+ """Mistral family of models."""
+
+ def __init__(
+ self,
+ aws_provider: AWSProvider,
+ model: str,
+ max_context_size: int = 32768,
+ max_new_tokens: int = 1500,
+ **kwargs,
+ ) -> None:
+ """NOTE: Configure your AWS credentials with the AWS CLI before using this model!"""
+ super().__init__(
+ model=model,
+ max_context_size=max_context_size,
+ max_new_tokens=max_new_tokens,
+ **kwargs,
+ )
+ self.aws_provider = aws_provider
+ self.provider = aws_provider.get_provider_name()
+
+ self.kwargs["stop"] = "\n\n---"
+
+ def _format_prompt(self, raw_prompt: str) -> str:
+ return "<s> [INST] Human: " + raw_prompt + " [/INST] Assistant: "
+
+ def _create_body(self, prompt: str, **kwargs) -> tuple[int, dict[str, str | float]]:
+ base_args: dict[str, Any] = self.kwargs
+ for k, v in kwargs.items():
+ base_args[k] = v
+
+ n, base_args = self.aws_provider.sanitize_kwargs(base_args)
+
+ query_args: dict[str, str | float] = {}
+ if isinstance(self.aws_provider, Bedrock):
+ query_args["prompt"] = prompt
+ elif isinstance(self.aws_provider, Sagemaker):
+ query_args["parameters"] = base_args
+ query_args["inputs"] = prompt
+ else:
+ raise ValueError("Error - provider not recognized")
+
+ return (n, query_args)
+
+ def _call_model(self, body: str) -> str:
+ response = self.aws_provider.call_model(
+ model_id=self._model_name,
+ body=body,
+ )
+ if isinstance(self.aws_provider, Bedrock):
+ response_body = json.loads(response["body"].read())
+ completion = response_body["outputs"][0]["text"]
+ elif isinstance(self.aws_provider, Sagemaker):
+ response_body = json.loads(response["Body"].read())
+ completion = response_body[0]["generated_text"]
+ else:
+ raise ValueError("Error - provider not recognized")
+
+ completion = completion.split(self.kwargs["stop"])[0]
+ return completion
+
+
+class AWSAnthropic(AWSModel):
+ """Anthropic family of models."""
+
+ def __init__(
+ self,
+ aws_provider: AWSProvider,
+ model: str,
+ max_context_size: int = 200000,
+ max_new_tokens: int = 1500,
+ **kwargs,
+ ) -> None:
+ """NOTE: Configure your AWS credentials with the AWS CLI before using this model!"""
+ super().__init__(
+ model=model,
+ max_context_size=max_context_size,
+ max_new_tokens=max_new_tokens,
+ **kwargs,
+ )
+ self.provider = aws_provider
+
+ for k, v in kwargs.items():
+ self.kwargs[k] = v
+
+ def _create_body(self, prompt: str, **kwargs) -> tuple[int, dict[str, str | float]]:
+ base_args: dict[str, Any] = self.kwargs
+ for k, v in kwargs.items():
+ base_args[k] = v
+
+ n, query_args = self.provider.sanitize_kwargs(base_args)
+
+ # Anthropic models do not support the following parameters
+ query_args.pop("frequency_penalty", None)
+ query_args.pop("num_generations", None)
+ query_args.pop("presence_penalty", None)
+ query_args.pop("model", None)
+
+ # we are using the Claude messages API
+ # https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages.html
+ query_args["messages"] = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": prompt,
+ },
+ ],
+ },
+ ]
+ query_args["anthropic_version"] = "bedrock-2023-05-31"
+ return (n, query_args)
+
+ def _call_model(self, body: str) -> str:
+ response = self.provider.predictor.invoke_model(
+ modelId=self._model_name,
+ body=body,
+ )
+ response_body = json.loads(response["body"].read())
+ completion = response_body["content"][0]["text"]
+ return completion
+
+
+class AWSMeta(AWSModel):
+ """Llama2 family of models."""
+
+ def __init__(
+ self,
+ aws_provider: AWSProvider,
+ model: str,
+ max_context_size: int = 4096,
+ max_new_tokens: int = 1500,
+ **kwargs,
+ ) -> None:
+ """NOTE: Configure your AWS credentials with the AWS CLI before using this model!"""
+ super().__init__(
+ model=model,
+ max_context_size=max_context_size,
+ max_new_tokens=max_new_tokens,
+ **kwargs,
+ )
+ self.provider = aws_provider
+
+ for k, v in kwargs.items():
+ self.kwargs[k] = v
+
+ self.kwargs["max_gen_len"] = self.kwargs.pop("max_tokens")
+
+ def _create_body(self, prompt: str, **kwargs) -> tuple[int, dict[str, str | float]]:
+ base_args: dict[str, Any] = self.kwargs
+ for k, v in kwargs.items():
+ base_args[k] = v
+
+ n, query_args = self.provider.sanitize_kwargs(base_args)
+
+ # Meta models do not support the following parameters
+ query_args.pop("frequency_penalty", None)
+ query_args.pop("num_generations", None)
+ query_args.pop("presence_penalty", None)
+ query_args.pop("model", None)
+
+ query_args["prompt"] = prompt
+ return (n, query_args)
+
+ def _call_model(self, body: str) -> str:
+ response = self.provider.predictor.invoke_model(
+ modelId=self._model_name,
+ body=body,
+ )
+ response_body = json.loads(response["body"].read())
+ completion = response_body["generation"]
+
+ stop = "\n\n"
+ completion = completion.split(stop)[0]
+
+ return completion
diff --git a/dsp/modules/aws_providers.py b/dsp/modules/aws_providers.py
new file mode 100644
index 000000000..ae3f0d4c3
--- /dev/null
+++ b/dsp/modules/aws_providers.py
@@ -0,0 +1,128 @@
+"""AWS providers for LMs."""
+
+from abc import ABC, abstractmethod
+from typing import Any, Optional
+
+
+class AWSProvider(ABC):
+ """This abstract class adds support for AWS model providers such as Bedrock and SageMaker.
+ The subclasses such as Bedrock and Sagemaker implement the abstract method _call_model and work in conjunction with the AWSModel classes.
+ Usage Example:
+ bedrock = Bedrock(region_name="us-west-2")
+ bedrock_mixtral = AWSMistral(bedrock, "mistral.mixtral-8x7b-instruct-v0:1", **kwargs)
+ bedrock_haiku = AWSAnthropic(bedrock, "anthropic.claude-3-haiku-20240307-v1:0", **kwargs)
+ bedrock_llama2 = AWSMeta(bedrock, "meta.llama2-13b-chat-v1", **kwargs)
+
+ sagemaker = Sagemaker(region_name="us-west-2")
+ sagemaker_model = AWSMistral(sagemaker, "<YOUR_ENDPOINT_NAME>", **kwargs)
+ """
+
+ def __init__(
+ self,
+ region_name: str,
+ service_name: str,
+ profile_name: Optional[str] = None,
+ batch_n_enabled: bool = True,
+ ) -> None:
+ """_summary_.
+
+ Args:
+ region_name (str, optional): The AWS region where this LM is hosted.
+ service_name (str): Used in context of invoking the boto3 API.
+ profile_name (str, optional): boto3 credentials profile.
+ batch_n_enabled (bool): If False, call the LM N times rather than batching.
+ """
+ try:
+ import boto3 # pylint: disable=import-outside-toplevel
+ except ImportError as exc:
+ raise ImportError('Please install boto3 to use AWS models.') from exc
+
+ if profile_name is None:
+ self.predictor = boto3.client(service_name, region_name=region_name)
+ else:
+ self.predictor = boto3.Session(profile_name=profile_name).client(
+ service_name,
+ region_name=region_name,
+ )
+
+ self.batch_n_enabled = batch_n_enabled
+
+ def get_provider_name(self) -> str:
+ """Return the provider name."""
+ return self.__class__.__name__
+
+ @abstractmethod
+ def call_model(self, model_id: str, body: str) -> str:
+ """Call the model and return the response."""
+
+ def sanitize_kwargs(self, query_kwargs: dict[str, Any]) -> tuple[int, dict[str, Any]]:
+ """Ensure that input kwargs can be used by Bedrock or Sagemaker."""
+ if "temperature" in query_kwargs:
+ if query_kwargs["temperature"] > 0.99:
+ query_kwargs["temperature"] = 0.99
+ if query_kwargs["temperature"] < 0.01:
+ query_kwargs["temperature"] = 0.01
+
+ if "top_p" in query_kwargs:
+ if query_kwargs["top_p"] > 0.99:
+ query_kwargs["top_p"] = 0.99
+ if query_kwargs["top_p"] < 0.01:
+ query_kwargs["top_p"] = 0.01
+
+ n = -1
+ if not self.batch_n_enabled:
+ n = query_kwargs.pop('n', 1)
+ query_kwargs["num_generations"] = n
+
+ return n, query_kwargs
+
+
+class Bedrock(AWSProvider):
+ """This class adds support for Bedrock models."""
+
+ def __init__(
+ self,
+ region_name: str,
+ profile_name: Optional[str] = None,
+ batch_n_enabled: bool = False, # This has to be setup manually on Bedrock.
+ ) -> None:
+ """_summary_.
+
+ Args:
+ region_name (str, optional): The AWS region where this LM is hosted.
+ profile_name (str, optional): boto3 credentials profile.
+ """
+ super().__init__(region_name, "bedrock-runtime", profile_name, batch_n_enabled)
+
+ def call_model(self, model_id: str, body: str) -> str:
+ return self.predictor.invoke_model(
+ modelId=model_id,
+ body=body,
+ accept="application/json",
+ contentType="application/json",
+ )
+
+
+class Sagemaker(AWSProvider):
+ """This class adds support for Sagemaker models."""
+
+ def __init__(
+ self,
+ region_name: str,
+ profile_name: Optional[str] = None,
+ ) -> None:
+ """_summary_.
+
+ Args:
+ region_name (str, optional): The AWS region where this LM is hosted.
+ profile_name (str, optional): boto3 credentials profile.
+ """
+ super().__init__(region_name, "runtime.sagemaker", profile_name)
+
+ def call_model(self, model_id: str, body: str) -> str:
+ return self.predictor.invoke_endpoint(
+ EndpointName=model_id,
+ Body=body,
+ Accept="application/json",
+ ContentType="application/json",
+ )
diff --git a/dsp/modules/bedrock.py b/dsp/modules/bedrock.py
deleted file mode 100644
index 252c87fe8..000000000
--- a/dsp/modules/bedrock.py
+++ /dev/null
@@ -1,79 +0,0 @@
-from __future__ import annotations
-
-import json
-from typing import Any, Optional
-
-from dsp.modules.aws_lm import AWSLM
-
-
-class Bedrock(AWSLM):
- def __init__(
- self,
- region_name: str,
- model: str,
- profile_name: Optional[str] = None,
- input_output_ratio: int = 3,
- max_new_tokens: int = 1500,
- ) -> None:
- """Use an AWS Bedrock language model.
- NOTE: You must first configure your AWS credentials with the AWS CLI before using this model!
-
- Args:
- region_name (str, optional): The AWS region where this LM is hosted.
- model (str, optional): An AWS Bedrock LM name. You can find available models with the AWS CLI as follows: aws bedrock list-foundation-models --query "modelSummaries[*].modelId".
- temperature (float, optional): Default temperature for LM. Defaults to 0.
- input_output_ratio (int, optional): The rough size of the number of input tokens to output tokens in the worst case. Defaults to 3.
- max_new_tokens (int, optional): The maximum number of tokens to be sampled from the LM.
- """
- super().__init__(
- model=model,
- service_name="bedrock-runtime",
- region_name=region_name,
- profile_name=profile_name,
- truncate_long_prompts=False,
- input_output_ratio=input_output_ratio,
- max_new_tokens=max_new_tokens,
- batch_n=True, # Bedrock does not support the `n` parameter
- )
- self._validate_model(model)
- self.provider = "claude" if "claude" in model.lower() else "bedrock"
-
- def _validate_model(self, model: str) -> None:
- if "claude" not in model.lower():
- raise NotImplementedError("Only claude models are supported as of now")
-
- def _create_body(self, prompt: str, **kwargs) -> dict[str, str | float]:
- base_args: dict[str, Any] = {
- "max_tokens_to_sample": self._max_new_tokens,
- }
- for k, v in kwargs.items():
- base_args[k] = v
- query_args: dict[str, Any] = self._sanitize_kwargs(base_args)
- query_args["prompt"] = prompt
- # AWS Bedrock forbids these keys
- if "max_tokens" in query_args:
- max_tokens: int = query_args["max_tokens"]
- input_tokens: int = self._estimate_tokens(prompt)
- max_tokens_to_sample: int = max_tokens - input_tokens
- del query_args["max_tokens"]
- query_args["max_tokens_to_sample"] = max_tokens_to_sample
- return query_args
-
- def _call_model(self, body: str) -> str:
- response = self.predictor.invoke_model(
- modelId=self._model_name,
- body=body,
- accept="application/json",
- contentType="application/json",
- )
- response_body = json.loads(response["body"].read())
- completion = response_body["completion"]
- return completion
-
- def _extract_input_parameters(
- self, body: dict[Any, Any],
- ) -> dict[str, str | float | int]:
- return body
-
- def _format_prompt(self, raw_prompt: str) -> str:
- return "\n\nHuman: " + raw_prompt + "\n\nAssistant:"
diff --git a/dspy/__init__.py b/dspy/__init__.py
index 1c6539f59..2d5ba4f00 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -33,7 +33,13 @@ Anyscale = dsp.Anyscale
Together = dsp.Together
HFModel = dsp.HFModel
OllamaLocal = dsp.OllamaLocal
+
Bedrock = dsp.Bedrock
+Sagemaker = dsp.Sagemaker
+AWSModel = dsp.AWSModel
+AWSMistral = dsp.AWSMistral
+AWSAnthropic = dsp.AWSAnthropic
+AWSMeta = dsp.AWSMeta
configure = settings.configure
context = settings.context
diff --git a/pyproject.toml b/pyproject.toml
index 2dafeab07..f564865f2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -46,6 +46,7 @@ marqo = ["marqo"]
pinecone = ["pinecone-client~=2.2.4"]
weaviate = ["weaviate-client~=4.5.4"]
milvus = ["pymilvus~=2.3.7"]
+aws = ["boto3~=1.34.78"]
docs = [
"sphinx>=4.3.0",
"furo>=2023.3.27",
@@ -58,7 +59,6 @@ docs = [
"autodoc_pydantic",
"sphinx-reredirects>=0.1.2",
"sphinx-automodapi==0.16.0",
-
]
dev = ["pytest>=6.2.5"]
@@ -100,6 +100,7 @@ qdrant-client = { version = "^1.6.2", optional = true }
pinecone-client = { version = "^2.2.4", optional = true }
weaviate-client = { version = "^4.5.4", optional = true }
pymilvus = { version = "^2.3.6", optional = true }
+boto3 = { version = "^1.34.78", optional = true }
sphinx = { version = ">=4.3.0", optional = true }
furo = { version = ">=2023.3.27", optional = true }
docutils = { version = "<0.17", optional = true }
@@ -134,6 +135,7 @@ marqo = ["marqo"]
pinecone = ["pinecone-client"]
weaviate = ["weaviate-client"]
milvus = ["pymilvus"]
+aws = ["boto3"]
postgres = ["psycopg2", "pgvector"]
docs = [
"sphinx",
diff --git a/tests/modules/test_aws_models.py b/tests/modules/test_aws_models.py
new file mode 100644
index 000000000..89cedcc6f
--- /dev/null
+++ b/tests/modules/test_aws_models.py
@@ -0,0 +1,59 @@
+"""Tests for AWS models.
+Note: Requires configuration of your AWS credentials with the AWS CLI and creating sagemaker endpoints.
+TODO: Create mock fixtures for pytest to remove the need for AWS credentials and endpoints.
+"""
+
+import dsp
+import dspy
+
+def get_lm(lm_provider: str, model_path: str, **kwargs) -> dsp.modules.lm.LM:
+ """get the language model"""
+ # extract model vendor and name from model name
+ # Model path format is <MODEL_VENDOR>/<MODEL_NAME_OR_ENDPOINT>
+ model_vendor = model_path.split('/')[0]
+ model_name = model_path.split('/')[1]
+
+ if lm_provider == 'Bedrock':
+ bedrock = dspy.Bedrock(region_name="us-west-2")
+ if model_vendor == 'mistral':
+ return dspy.AWSMistral(bedrock, model_name, **kwargs)
+ elif model_vendor == 'anthropic':
+ return dspy.AWSAnthropic(bedrock, model_name, **kwargs)
+ elif model_vendor == 'meta':
+ return dspy.AWSMeta(bedrock, model_name, **kwargs)
+ else:
+ raise ValueError("Model vendor missing or unsupported: Model path format is <MODEL_VENDOR>/<MODEL_NAME_OR_ENDPOINT>")
+ elif lm_provider == 'Sagemaker':
+ sagemaker = dspy.Sagemaker(region_name="us-west-2")
+ if model_vendor == 'mistral':
+ return dspy.AWSMistral(sagemaker, model_name, **kwargs)
+ elif model_vendor == 'meta':
+ return dspy.AWSMeta(sagemaker, model_name, **kwargs)
+ else:
+ raise ValueError("Model vendor missing or unsupported: Model path format is <MODEL_VENDOR>/<MODEL_NAME_OR_ENDPOINT>")
+ else:
+ raise ValueError(f"Unsupported model: {model_name}")
+
+def run_tests():
+ """Test the providers and models"""
+ # Configure your AWS credentials with the AWS CLI before running this script
+ provider_model_tuples = [
+ ('Bedrock', 'mistral/mistral.mixtral-8x7b-instruct-v0:1'),
+ ('Bedrock', 'anthropic/anthropic.claude-3-haiku-20240307-v1:0'),
+ ('Bedrock', 'anthropic/anthropic.claude-3-sonnet-20240229-v1:0'),
+ ('Bedrock', 'meta/meta.llama2-70b-chat-v1'),
+ # ('Sagemaker', 'mistral/<YOUR_ENDPOINT_NAME>'), # REPLACE YOUR_ENDPOINT_NAME with your sagemaker endpoint
+ ]
+
+ predict_func = dspy.Predict("question -> answer")
+ for provider, model_path in provider_model_tuples:
+ print(f"Provider: {provider}, Model: {model_path}")
+ lm = get_lm(provider, model_path)
+ with dspy.context(lm=lm):
+ question = "What is the capital of France?"
+ answer = predict_func(question=question).answer
+ print(f"Question: {question}\nAnswer: {answer}\n\n")
+
+
+if __name__ == "__main__":
+ run_tests()