Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Revert "feature(modules): added support for aws providers (bedrock and sagemaker) and models (mistral, anthropic, meta)"

Dhar Rawal98abac2c1c9b
diff --git a/docs/api/language_model_clients/aws.md b/docs/api/language_model_clients/aws.md
deleted file mode 100644
index afac264f3..000000000
--- a/docs/api/language_model_clients/aws.md
+++ /dev/null
@@ -1,61 +0,0 @@
----
-sidebar_position: 9
----
-
-# dsp.AWSMistral, dsp.AWSAnthropic, dsp.AWSMeta
-
-### Usage
-
-```python
-# Notes:
-# 1. Install boto3 to use AWS models.
-# 2. Configure your AWS credentials with the AWS CLI before using these models
-
-# initialize the bedrock aws provider
-bedrock = Bedrock(region_name="us-west-2")
-# For mixtral on Bedrock
-lm = AWSMistral(bedrock, "mistral.mixtral-8x7b-instruct-v0:1", **kwargs)
-# For haiku on Bedrock
-lm = AWSAnthropic(bedrock, "anthropic.claude-3-haiku-20240307-v1:0", **kwargs)
-# For llama2 on Bedrock
-lm = AWSMeta(bedrock, "meta.llama2-13b-chat-v1", **kwargs)
-
-# initialize the sagemaker aws provider
-sagemaker = Sagemaker(region_name="us-west-2")
-# For mistral on Sagemaker
-# Note: you need to create a Sagemaker endpoint for the mistral model first
-lm = AWSMistral(sagemaker, "<YOUR_MISTRAL_ENDPOINT_NAME>", **kwargs)
-
-```
-
-### Constructor
-
-The constructor initializes the base class `LM` and the `AWSProvider` class.
-
-```python
-class AWSMistral(AWSModel):
- """Mistral family of models."""
-
- def __init__(
- self,
- aws_provider: AWSProvider,
- model: str,
- max_context_size: int = 32768,
- max_new_tokens: int = 1500,
- **kwargs
- ) -> None:
-```
-
-**Parameters:**
-- `aws_provider` (AWSProvider): The aws provider to use. One of `Bedrock` or `Sagemaker`.
-- `model` (_str_): Mistral AI pretrained models. Defaults to `mistral-medium-latest`.
-- `max_context_size` (_Optional[int]_, _optional_): Max context size for this model. Defaults to 32768.
-- `max_new_tokens` (_Optional[int]_, _optional_): Max new tokens possible for this model. Defaults to 1500.
-- `**kwargs`: Additional language model arguments to pass to the API provider.
-
-### Methods
-
-Refer to [`dspy.OpenAI`](https://dspy-docs.vercel.app/api/language_model_clients/OpenAI) documentation.
-
-
-`AWSAnthropic` and `AWSMeta` work exactly the same as `AWSMistral`.
\ No newline at end of file
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.py
index 935f65b40..b7a739f50 100644
--- a/dsp/modules/__init__.py
+++ b/dsp/modules/__init__.py
@@ -1,10 +1,6 @@
from .anthropic import Claude
-from .aws_models import AWSAnthropic, AWSMeta, AWSMistral, AWSModel
-
-# Below is obsolete. It has been replaced with Bedrock class in dsp/modules/aws_providers.py
-# from .bedrock import *
-from .aws_providers import Bedrock, Sagemaker
from .azure_openai import AzureOpenAI
+from .bedrock import *
from .cache_utils import *
from .clarifai import *
from .cohere import *
@@ -21,3 +17,4 @@ from .ollama import *
from .pyserini import *
from .sbert import *
from .sentence_vectorizer import *
+
diff --git a/dsp/modules/aws_models.py b/dsp/modules/aws_models.py
deleted file mode 100644
index 3b9394889..000000000
--- a/dsp/modules/aws_models.py
+++ /dev/null
@@ -1,300 +0,0 @@
-"""AWS models for LMs."""
-
-from __future__ import annotations
-
-import json
-import logging
-from abc import abstractmethod
-from typing import Any
-
-from dsp.modules.aws_providers import AWSProvider, Bedrock, Sagemaker
-from dsp.modules.lm import LM
-
-# Heuristic translating number of chars to tokens
-# ~4 chars = 1 token
-CHARS2TOKENS: int = 4
-
-
-class AWSModel(LM):
- """This class adds support for an AWS model.
- It is an abstract class and should not be instantiated directly.
- Instead, use one of the subclasses - AWSMistral, AWSAnthropic, or AWSMeta.
- The subclasses implement the abstract methods _create_body and _call_model and work in conjunction with the AWSProvider classes Bedrock and Sagemaker.
- Usage Example:
- bedrock = Bedrock(region_name="us-west-2")
- bedrock_mixtral = AWSMistral(bedrock, "mistral.mixtral-8x7b-instruct-v0:1", **kwargs)
- bedrock_haiku = AWSAnthropic(bedrock, "anthropic.claude-3-haiku-20240307-v1:0", **kwargs)
- bedrock_llama2 = AWSMeta(bedrock, "meta.llama2-13b-chat-v1", **kwargs)
-
- sagemaker = Sagemaker(region_name="us-west-2")
- sagemaker_model = AWSMistral(sagemaker, "<YOUR_ENDPOINT_NAME>", **kwargs)
- """
-
- def __init__(
- self,
- model: str,
- max_context_size: int,
- max_new_tokens: int,
- **kwargs,
- ) -> None:
- """_summary_.
-
- Args:
- model (str, optional): An LM name, e.g., a bedrock name or an AWS endpoint.
- max_context_size (int): The maximum context size in tokens.
- max_new_tokens (int): The maximum number of tokens to be sampled from the LM.
- """
- super().__init__(model=model)
- self._model_name: str = model
- self._max_context_size: int = max_context_size
- self._max_new_tokens: int = max_new_tokens
-
- self.kwargs = {
- **self.kwargs,
- **kwargs,
- }
-
- @abstractmethod
- def _create_body(self, prompt: str, **kwargs) -> tuple[int, dict[str, str | float]]:
- pass
-
- @abstractmethod
- def _call_model(self, body: str) -> str | list[str]:
- """Call model, get generated input without the formatted prompt."""
-
- def _estimate_tokens(self, text: str) -> int:
- return len(text)/CHARS2TOKENS
-
- def _extract_input_parameters(
- self,
- body: dict[Any, Any],
- ) -> dict[str, str | float | int]:
- return body
-
- def _format_prompt(self, raw_prompt: str) -> str:
- return "\n\nHuman: " + raw_prompt + "\n\nAssistant:"
-
- def _simple_api_call(self, formatted_prompt: str, **kwargs) -> str | list[str]:
- n, body = self._create_body(formatted_prompt, **kwargs)
- json_body = json.dumps(body)
-
- if n > 1:
- llm_out = [self._call_model(json_body) for _ in range(n)]
- llm_out = [generated.replace(formatted_prompt, "") for generated in llm_out]
- else:
- llm_out = self._call_model(json_body)
- llm_out = llm_out.replace(formatted_prompt, "")
-
- self.history.append(
- {"prompt": formatted_prompt, "response": llm_out, "kwargs": body},
- )
- return llm_out
-
- def basic_request(self, prompt, **kwargs) -> str | list[str]:
- """Query the endpoint."""
- token_count = self._estimate_tokens(prompt)
- if token_count > self._max_context_size:
- logging.info("Error - input tokens %s exceeds max context %s", token_count, self._max_context_size)
- raise ValueError(
- f"Error - input tokens {token_count} exceeds max context {self._max_context_size}",
- )
-
- formatted_prompt: str = self._format_prompt(prompt)
- return self._simple_api_call(formatted_prompt=formatted_prompt, **kwargs)
-
- def __call__(
- self,
- prompt: str,
- only_completed: bool = True,
- return_sorted: bool = False,
- **kwargs,
- ) -> list[str]:
- """Query the AWS LLM.
-
- There is only support for only_completed=True and return_sorted=False
- right now.
- """
- assert only_completed, "for now"
- assert return_sorted is False, "for now"
-
- generated = self.basic_request(prompt, **kwargs)
- return [generated]
-
-
-class AWSMistral(AWSModel):
- """Mistral family of models."""
-
- def __init__(
- self,
- aws_provider: AWSProvider,
- model: str,
- max_context_size: int = 32768,
- max_new_tokens: int = 1500,
- **kwargs,
- ) -> None:
- """NOTE: Configure your AWS credentials with the AWS CLI before using this model!"""
- super().__init__(
- model=model,
- max_context_size=max_context_size,
- max_new_tokens=max_new_tokens,
- **kwargs,
- )
- self.aws_provider = aws_provider
- self.provider = aws_provider.get_provider_name()
-
- self.kwargs["stop"] = "\n\n---"
-
- def _format_prompt(self, raw_prompt: str) -> str:
- return "<s> [INST] Human: " + raw_prompt + " [/INST] Assistant: "
-
- def _create_body(self, prompt: str, **kwargs) -> tuple[int, dict[str, str | float]]:
- base_args: dict[str, Any] = self.kwargs
- for k, v in kwargs.items():
- base_args[k] = v
-
- n, base_args = self.aws_provider.sanitize_kwargs(base_args)
-
- query_args: dict[str, str | float] = {}
- if isinstance(self.aws_provider, Bedrock):
- query_args["prompt"] = prompt
- elif isinstance(self.aws_provider, Sagemaker):
- query_args["parameters"] = base_args
- query_args["inputs"] = prompt
- else:
- raise ValueError("Error - provider not recognized")
-
- return (n, query_args)
-
- def _call_model(self, body: str) -> str:
- response = self.aws_provider.call_model(
- model_id=self._model_name,
- body=body,
- )
- if isinstance(self.aws_provider, Bedrock):
- response_body = json.loads(response["body"].read())
- completion = response_body["outputs"][0]["text"]
- elif isinstance(self.aws_provider, Sagemaker):
- response_body = json.loads(response["Body"].read())
- completion = response_body[0]["generated_text"]
- else:
- raise ValueError("Error - provider not recognized")
-
- completion = completion.split(self.kwargs["stop"])[0]
- return completion
-
-
-class AWSAnthropic(AWSModel):
- """Anthropic family of models."""
-
- def __init__(
- self,
- aws_provider: AWSProvider,
- model: str,
- max_context_size: int = 200000,
- max_new_tokens: int = 1500,
- **kwargs,
- ) -> None:
- """NOTE: Configure your AWS credentials with the AWS CLI before using this model!"""
- super().__init__(
- model=model,
- max_context_size=max_context_size,
- max_new_tokens=max_new_tokens,
- **kwargs,
- )
- self.provider = aws_provider
-
- for k, v in kwargs.items():
- self.kwargs[k] = v
-
- def _create_body(self, prompt: str, **kwargs) -> tuple[int, dict[str, str | float]]:
- base_args: dict[str, Any] = self.kwargs
- for k, v in kwargs.items():
- base_args[k] = v
-
- n, query_args = self.provider.sanitize_kwargs(base_args)
-
- # Anthropic models do not support the following parameters
- query_args.pop("frequency_penalty", None)
- query_args.pop("num_generations", None)
- query_args.pop("presence_penalty", None)
- query_args.pop("model", None)
-
- # we are using the Claude messages API
- # https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages.html
- query_args["messages"] = [
- {
- "role": "user",
- "content": [
- {
- "type": "text",
- "text": prompt,
- },
- ],
- },
- ]
- query_args["anthropic_version"] = "bedrock-2023-05-31"
- return (n, query_args)
-
- def _call_model(self, body: str) -> str:
- response = self.provider.predictor.invoke_model(
- modelId=self._model_name,
- body=body,
- )
- response_body = json.loads(response["body"].read())
- completion = response_body["content"][0]["text"]
- return completion
-
-
-class AWSMeta(AWSModel):
- """Llama2 family of models."""
-
- def __init__(
- self,
- aws_provider: AWSProvider,
- model: str,
- max_context_size: int = 4096,
- max_new_tokens: int = 1500,
- **kwargs,
- ) -> None:
- """NOTE: Configure your AWS credentials with the AWS CLI before using this model!"""
- super().__init__(
- model=model,
- max_context_size=max_context_size,
- max_new_tokens=max_new_tokens,
- **kwargs,
- )
- self.provider = aws_provider
-
- for k, v in kwargs.items():
- self.kwargs[k] = v
-
- self.kwargs["max_gen_len"] = self.kwargs.pop("max_tokens")
-
- def _create_body(self, prompt: str, **kwargs) -> tuple[int, dict[str, str | float]]:
- base_args: dict[str, Any] = self.kwargs
- for k, v in kwargs.items():
- base_args[k] = v
-
- n, query_args = self.provider.sanitize_kwargs(base_args)
-
- # Meta models do not support the following parameters
- query_args.pop("frequency_penalty", None)
- query_args.pop("num_generations", None)
- query_args.pop("presence_penalty", None)
- query_args.pop("model", None)
-
- query_args["prompt"] = prompt
- return (n, query_args)
-
- def _call_model(self, body: str) -> str:
- response = self.provider.predictor.invoke_model(
- modelId=self._model_name,
- body=body,
- )
- response_body = json.loads(response["body"].read())
- completion = response_body["generation"]
-
- stop = "\n\n"
- completion = completion.split(stop)[0]
-
- return completion
diff --git a/dsp/modules/aws_providers.py b/dsp/modules/aws_providers.py
deleted file mode 100644
index ae3f0d4c3..000000000
--- a/dsp/modules/aws_providers.py
+++ /dev/null
@@ -1,128 +0,0 @@
-"""AWS providers for LMs."""
-
-from abc import ABC, abstractmethod
-from typing import Any, Optional
-
-
-class AWSProvider(ABC):
- """This abstract class adds support for AWS model providers such as Bedrock and SageMaker.
- The subclasses such as Bedrock and Sagemaker implement the abstract method _call_model and work in conjunction with the AWSModel classes.
- Usage Example:
- bedrock = Bedrock(region_name="us-west-2")
- bedrock_mixtral = AWSMistral(bedrock, "mistral.mixtral-8x7b-instruct-v0:1", **kwargs)
- bedrock_haiku = AWSAnthropic(bedrock, "anthropic.claude-3-haiku-20240307-v1:0", **kwargs)
- bedrock_llama2 = AWSMeta(bedrock, "meta.llama2-13b-chat-v1", **kwargs)
-
- sagemaker = Sagemaker(region_name="us-west-2")
- sagemaker_model = AWSMistral(sagemaker, "<YOUR_ENDPOINT_NAME>", **kwargs)
- """
-
- def __init__(
- self,
- region_name: str,
- service_name: str,
- profile_name: Optional[str] = None,
- batch_n_enabled: bool = True,
- ) -> None:
- """_summary_.
-
- Args:
- region_name (str, optional): The AWS region where this LM is hosted.
- service_name (str): Used in context of invoking the boto3 API.
- profile_name (str, optional): boto3 credentials profile.
- batch_n_enabled (bool): If False, call the LM N times rather than batching.
- """
- try:
- import boto3 # pylint: disable=import-outside-toplevel
- except ImportError as exc:
- raise ImportError('Please install boto3 to use AWS models.') from exc
-
- if profile_name is None:
- self.predictor = boto3.client(service_name, region_name=region_name)
- else:
- self.predictor = boto3.Session(profile_name=profile_name).client(
- service_name,
- region_name=region_name,
- )
-
- self.batch_n_enabled = batch_n_enabled
-
- def get_provider_name(self) -> str:
- """Return the provider name."""
- return self.__class__.__name__
-
- @abstractmethod
- def call_model(self, model_id: str, body: str) -> str:
- """Call the model and return the response."""
-
- def sanitize_kwargs(self, query_kwargs: dict[str, Any]) -> tuple[int, dict[str, Any]]:
- """Ensure that input kwargs can be used by Bedrock or Sagemaker."""
- if "temperature" in query_kwargs:
- if query_kwargs["temperature"] > 0.99:
- query_kwargs["temperature"] = 0.99
- if query_kwargs["temperature"] < 0.01:
- query_kwargs["temperature"] = 0.01
-
- if "top_p" in query_kwargs:
- if query_kwargs["top_p"] > 0.99:
- query_kwargs["top_p"] = 0.99
- if query_kwargs["top_p"] < 0.01:
- query_kwargs["top_p"] = 0.01
-
- n = -1
- if not self.batch_n_enabled:
- n = query_kwargs.pop('n', 1)
- query_kwargs["num_generations"] = n
-
- return n, query_kwargs
-
-
-class Bedrock(AWSProvider):
- """This class adds support for Bedrock models."""
-
- def __init__(
- self,
- region_name: str,
- profile_name: Optional[str] = None,
- batch_n_enabled: bool = False, # This has to be setup manually on Bedrock.
- ) -> None:
- """_summary_.
-
- Args:
- region_name (str, optional): The AWS region where this LM is hosted.
- profile_name (str, optional): boto3 credentials profile.
- """
- super().__init__(region_name, "bedrock-runtime", profile_name, batch_n_enabled)
-
- def call_model(self, model_id: str, body: str) -> str:
- return self.predictor.invoke_model(
- modelId=model_id,
- body=body,
- accept="application/json",
- contentType="application/json",
- )
-
-
-class Sagemaker(AWSProvider):
- """This class adds support for Sagemaker models."""
-
- def __init__(
- self,
- region_name: str,
- profile_name: Optional[str] = None,
- ) -> None:
- """_summary_.
-
- Args:
- region_name (str, optional): The AWS region where this LM is hosted.
- profile_name (str, optional): boto3 credentials profile.
- """
- super().__init__(region_name, "runtime.sagemaker", profile_name)
-
- def call_model(self, model_id: str, body: str) -> str:
- return self.predictor.invoke_endpoint(
- EndpointName=model_id,
- Body=body,
- Accept="application/json",
- ContentType="application/json",
- )
diff --git a/dsp/modules/bedrock.py b/dsp/modules/bedrock.py
new file mode 100644
index 000000000..252c87fe8
--- /dev/null
+++ b/dsp/modules/bedrock.py
@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import json
+from typing import Any, Optional
+
+from dsp.modules.aws_lm import AWSLM
+
+
+class Bedrock(AWSLM):
+ def __init__(
+ self,
+ region_name: str,
+ model: str,
+ profile_name: Optional[str] = None,
+ input_output_ratio: int = 3,
+ max_new_tokens: int = 1500,
+ ) -> None:
+ """Use an AWS Bedrock language model.
+ NOTE: You must first configure your AWS credentials with the AWS CLI before using this model!
+
+ Args:
+ region_name (str, optional): The AWS region where this LM is hosted.
+ model (str, optional): An AWS Bedrock LM name. You can find available models with the AWS CLI as follows: aws bedrock list-foundation-models --query "modelSummaries[*].modelId".
+ temperature (float, optional): Default temperature for LM. Defaults to 0.
+ input_output_ratio (int, optional): The rough size of the number of input tokens to output tokens in the worst case. Defaults to 3.
+ max_new_tokens (int, optional): The maximum number of tokens to be sampled from the LM.
+ """
+ super().__init__(
+ model=model,
+ service_name="bedrock-runtime",
+ region_name=region_name,
+ profile_name=profile_name,
+ truncate_long_prompts=False,
+ input_output_ratio=input_output_ratio,
+ max_new_tokens=max_new_tokens,
+ batch_n=True, # Bedrock does not support the `n` parameter
+ )
+ self._validate_model(model)
+ self.provider = "claude" if "claude" in model.lower() else "bedrock"
+
+ def _validate_model(self, model: str) -> None:
+ if "claude" not in model.lower():
+ raise NotImplementedError("Only claude models are supported as of now")
+
+ def _create_body(self, prompt: str, **kwargs) -> dict[str, str | float]:
+ base_args: dict[str, Any] = {
+ "max_tokens_to_sample": self._max_new_tokens,
+ }
+ for k, v in kwargs.items():
+ base_args[k] = v
+ query_args: dict[str, Any] = self._sanitize_kwargs(base_args)
+ query_args["prompt"] = prompt
+ # AWS Bedrock forbids these keys
+ if "max_tokens" in query_args:
+ max_tokens: int = query_args["max_tokens"]
+ input_tokens: int = self._estimate_tokens(prompt)
+ max_tokens_to_sample: int = max_tokens - input_tokens
+ del query_args["max_tokens"]
+ query_args["max_tokens_to_sample"] = max_tokens_to_sample
+ return query_args
+
+ def _call_model(self, body: str) -> str:
+ response = self.predictor.invoke_model(
+ modelId=self._model_name,
+ body=body,
+ accept="application/json",
+ contentType="application/json",
+ )
+ response_body = json.loads(response["body"].read())
+ completion = response_body["completion"]
+ return completion
+
+ def _extract_input_parameters(
+ self, body: dict[Any, Any],
+ ) -> dict[str, str | float | int]:
+ return body
+
+ def _format_prompt(self, raw_prompt: str) -> str:
+ return "\n\nHuman: " + raw_prompt + "\n\nAssistant:"
diff --git a/dspy/__init__.py b/dspy/__init__.py
index 2d5ba4f00..1c6539f59 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -33,13 +33,7 @@ Anyscale = dsp.Anyscale
Together = dsp.Together
HFModel = dsp.HFModel
OllamaLocal = dsp.OllamaLocal
-
Bedrock = dsp.Bedrock
-Sagemaker = dsp.Sagemaker
-AWSModel = dsp.AWSModel
-AWSMistral = dsp.AWSMistral
-AWSAnthropic = dsp.AWSAnthropic
-AWSMeta = dsp.AWSMeta
configure = settings.configure
context = settings.context
diff --git a/pyproject.toml b/pyproject.toml
index f564865f2..2dafeab07 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -46,7 +46,6 @@ marqo = ["marqo"]
pinecone = ["pinecone-client~=2.2.4"]
weaviate = ["weaviate-client~=4.5.4"]
milvus = ["pymilvus~=2.3.7"]
-aws = ["boto3~=1.34.78"]
docs = [
"sphinx>=4.3.0",
"furo>=2023.3.27",
@@ -59,6 +58,7 @@ docs = [
"autodoc_pydantic",
"sphinx-reredirects>=0.1.2",
"sphinx-automodapi==0.16.0",
+
]
dev = ["pytest>=6.2.5"]
@@ -100,7 +100,6 @@ qdrant-client = { version = "^1.6.2", optional = true }
pinecone-client = { version = "^2.2.4", optional = true }
weaviate-client = { version = "^4.5.4", optional = true }
pymilvus = { version = "^2.3.6", optional = true }
-boto3 = { version = "^1.34.78", optional = true }
sphinx = { version = ">=4.3.0", optional = true }
furo = { version = ">=2023.3.27", optional = true }
docutils = { version = "<0.17", optional = true }
@@ -135,7 +134,6 @@ marqo = ["marqo"]
pinecone = ["pinecone-client"]
weaviate = ["weaviate-client"]
milvus = ["pymilvus"]
-aws = ["boto3"]
postgres = ["psycopg2", "pgvector"]
docs = [
"sphinx",
diff --git a/tests/modules/test_aws_models.py b/tests/modules/test_aws_models.py
deleted file mode 100644
index 89cedcc6f..000000000
--- a/tests/modules/test_aws_models.py
+++ /dev/null
@@ -1,59 +0,0 @@
-"""Tests for AWS models.
-Note: Requires configuration of your AWS credentials with the AWS CLI and creating sagemaker endpoints.
-TODO: Create mock fixtures for pytest to remove the need for AWS credentials and endpoints.
-"""
-
-import dsp
-import dspy
-
-def get_lm(lm_provider: str, model_path: str, **kwargs) -> dsp.modules.lm.LM:
- """get the language model"""
- # extract model vendor and name from model name
- # Model path format is <MODEL_VENDOR>/<MODEL_NAME_OR_ENDPOINT>
- model_vendor = model_path.split('/')[0]
- model_name = model_path.split('/')[1]
-
- if lm_provider == 'Bedrock':
- bedrock = dspy.Bedrock(region_name="us-west-2")
- if model_vendor == 'mistral':
- return dspy.AWSMistral(bedrock, model_name, **kwargs)
- elif model_vendor == 'anthropic':
- return dspy.AWSAnthropic(bedrock, model_name, **kwargs)
- elif model_vendor == 'meta':
- return dspy.AWSMeta(bedrock, model_name, **kwargs)
- else:
- raise ValueError("Model vendor missing or unsupported: Model path format is <MODEL_VENDOR>/<MODEL_NAME_OR_ENDPOINT>")
- elif lm_provider == 'Sagemaker':
- sagemaker = dspy.Sagemaker(region_name="us-west-2")
- if model_vendor == 'mistral':
- return dspy.AWSMistral(sagemaker, model_name, **kwargs)
- elif model_vendor == 'meta':
- return dspy.AWSMeta(sagemaker, model_name, **kwargs)
- else:
- raise ValueError("Model vendor missing or unsupported: Model path format is <MODEL_VENDOR>/<MODEL_NAME_OR_ENDPOINT>")
- else:
- raise ValueError(f"Unsupported model: {model_name}")
-
-def run_tests():
- """Test the providers and models"""
- # Configure your AWS credentials with the AWS CLI before running this script
- provider_model_tuples = [
- ('Bedrock', 'mistral/mistral.mixtral-8x7b-instruct-v0:1'),
- ('Bedrock', 'anthropic/anthropic.claude-3-haiku-20240307-v1:0'),
- ('Bedrock', 'anthropic/anthropic.claude-3-sonnet-20240229-v1:0'),
- ('Bedrock', 'meta/meta.llama2-70b-chat-v1'),
- # ('Sagemaker', 'mistral/<YOUR_ENDPOINT_NAME>'), # REPLACE YOUR_ENDPOINT_NAME with your sagemaker endpoint
- ]
-
- predict_func = dspy.Predict("question -> answer")
- for provider, model_path in provider_model_tuples:
- print(f"Provider: {provider}, Model: {model_path}")
- lm = get_lm(provider, model_path)
- with dspy.context(lm=lm):
- question = "What is the capital of France?"
- answer = predict_func(question=question).answer
- print(f"Question: {question}\nAnswer: {answer}\n\n")
-
-
-if __name__ == "__main__":
- run_tests()