DSPy: The framework for programming—not prompting—language models
Python
Revert "feature(modules): added support for aws providers (bedrock and sagemaker) and models (mistral, anthropic, meta)"
Dhar Rawal98abac2c1c9b
diff --git a/docs/api/language_model_clients/aws.md b/docs/api/language_model_clients/aws.mddeleted file mode 100644index afac264f3..000000000--- a/docs/api/language_model_clients/aws.md+++ /dev/null@@ -1,61 +0,0 @@-----sidebar_position: 9------# dsp.AWSMistral, dsp.AWSAnthropic, dsp.AWSMeta--### Usage--```python-# Notes:-# 1. Install boto3 to use AWS models.-# 2. Configure your AWS credentials with the AWS CLI before using these models--# initialize the bedrock aws provider-bedrock = Bedrock(region_name="us-west-2")-# For mixtral on Bedrock-lm = AWSMistral(bedrock, "mistral.mixtral-8x7b-instruct-v0:1", **kwargs)-# For haiku on Bedrock-lm = AWSAnthropic(bedrock, "anthropic.claude-3-haiku-20240307-v1:0", **kwargs)-# For llama2 on Bedrock-lm = AWSMeta(bedrock, "meta.llama2-13b-chat-v1", **kwargs)--# initialize the sagemaker aws provider-sagemaker = Sagemaker(region_name="us-west-2")-# For mistral on Sagemaker-# Note: you need to create a Sagemaker endpoint for the mistral model first-lm = AWSMistral(sagemaker, "<YOUR_MISTRAL_ENDPOINT_NAME>", **kwargs)--```--### Constructor--The constructor initializes the base class `LM` and the `AWSProvider` class.--```python-class AWSMistral(AWSModel):- """Mistral family of models."""-- def __init__(- self,- aws_provider: AWSProvider,- model: str,- max_context_size: int = 32768,- max_new_tokens: int = 1500,- **kwargs- ) -> None:-```--**Parameters:**-- `aws_provider` (AWSProvider): The aws provider to use. One of `Bedrock` or `Sagemaker`.-- `model` (_str_): Mistral AI pretrained models. Defaults to `mistral-medium-latest`.-- `max_context_size` (_Optional[int]_, _optional_): Max context size for this model. Defaults to 32768.-- `max_new_tokens` (_Optional[int]_, _optional_): Max new tokens possible for this model. Defaults to 1500.-- `**kwargs`: Additional language model arguments to pass to the API provider.--### Methods--Refer to [`dspy.OpenAI`](https://dspy-docs.vercel.app/api/language_model_clients/OpenAI) documentation.---`AWSAnthropic` and `AWSMeta` work exactly the same as `AWSMistral`.\ No newline at end of filediff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.pyindex 935f65b40..b7a739f50 100644--- a/dsp/modules/__init__.py+++ b/dsp/modules/__init__.py@@ -1,10 +1,6 @@from .anthropic import Claude-from .aws_models import AWSAnthropic, AWSMeta, AWSMistral, AWSModel--# Below is obsolete. It has been replaced with Bedrock class in dsp/modules/aws_providers.py-# from .bedrock import *-from .aws_providers import Bedrock, Sagemakerfrom .azure_openai import AzureOpenAI+from .bedrock import *from .cache_utils import *from .clarifai import *from .cohere import *@@ -21,3 +17,4 @@ from .ollama import *from .pyserini import *from .sbert import *from .sentence_vectorizer import *+diff --git a/dsp/modules/aws_models.py b/dsp/modules/aws_models.pydeleted file mode 100644index 3b9394889..000000000--- a/dsp/modules/aws_models.py+++ /dev/null@@ -1,300 +0,0 @@-"""AWS models for LMs."""--from __future__ import annotations--import json-import logging-from abc import abstractmethod-from typing import Any--from dsp.modules.aws_providers import AWSProvider, Bedrock, Sagemaker-from dsp.modules.lm import LM--# Heuristic translating number of chars to tokens-# ~4 chars = 1 token-CHARS2TOKENS: int = 4---class AWSModel(LM):- """This class adds support for an AWS model.- It is an abstract class and should not be instantiated directly.- Instead, use one of the subclasses - AWSMistral, AWSAnthropic, or AWSMeta.- The subclasses implement the abstract methods _create_body and _call_model and work in conjunction with the AWSProvider classes Bedrock and Sagemaker.- Usage Example:- bedrock = Bedrock(region_name="us-west-2")- bedrock_mixtral = AWSMistral(bedrock, "mistral.mixtral-8x7b-instruct-v0:1", **kwargs)- bedrock_haiku = AWSAnthropic(bedrock, "anthropic.claude-3-haiku-20240307-v1:0", **kwargs)- bedrock_llama2 = AWSMeta(bedrock, "meta.llama2-13b-chat-v1", **kwargs)-- sagemaker = Sagemaker(region_name="us-west-2")- sagemaker_model = AWSMistral(sagemaker, "<YOUR_ENDPOINT_NAME>", **kwargs)- """-- def __init__(- self,- model: str,- max_context_size: int,- max_new_tokens: int,- **kwargs,- ) -> None:- """_summary_.-- Args:- model (str, optional): An LM name, e.g., a bedrock name or an AWS endpoint.- max_context_size (int): The maximum context size in tokens.- max_new_tokens (int): The maximum number of tokens to be sampled from the LM.- """- super().__init__(model=model)- self._model_name: str = model- self._max_context_size: int = max_context_size- self._max_new_tokens: int = max_new_tokens-- self.kwargs = {- **self.kwargs,- **kwargs,- }-- @abstractmethod- def _create_body(self, prompt: str, **kwargs) -> tuple[int, dict[str, str | float]]:- pass-- @abstractmethod- def _call_model(self, body: str) -> str | list[str]:- """Call model, get generated input without the formatted prompt."""-- def _estimate_tokens(self, text: str) -> int:- return len(text)/CHARS2TOKENS-- def _extract_input_parameters(- self,- body: dict[Any, Any],- ) -> dict[str, str | float | int]:- return body-- def _format_prompt(self, raw_prompt: str) -> str:- return "\n\nHuman: " + raw_prompt + "\n\nAssistant:"-- def _simple_api_call(self, formatted_prompt: str, **kwargs) -> str | list[str]:- n, body = self._create_body(formatted_prompt, **kwargs)- json_body = json.dumps(body)-- if n > 1:- llm_out = [self._call_model(json_body) for _ in range(n)]- llm_out = [generated.replace(formatted_prompt, "") for generated in llm_out]- else:- llm_out = self._call_model(json_body)- llm_out = llm_out.replace(formatted_prompt, "")-- self.history.append(- {"prompt": formatted_prompt, "response": llm_out, "kwargs": body},- )- return llm_out-- def basic_request(self, prompt, **kwargs) -> str | list[str]:- """Query the endpoint."""- token_count = self._estimate_tokens(prompt)- if token_count > self._max_context_size:- logging.info("Error - input tokens %s exceeds max context %s", token_count, self._max_context_size)- raise ValueError(- f"Error - input tokens {token_count} exceeds max context {self._max_context_size}",- )-- formatted_prompt: str = self._format_prompt(prompt)- return self._simple_api_call(formatted_prompt=formatted_prompt, **kwargs)-- def __call__(- self,- prompt: str,- only_completed: bool = True,- return_sorted: bool = False,- **kwargs,- ) -> list[str]:- """Query the AWS LLM.-- There is only support for only_completed=True and return_sorted=False- right now.- """- assert only_completed, "for now"- assert return_sorted is False, "for now"-- generated = self.basic_request(prompt, **kwargs)- return [generated]---class AWSMistral(AWSModel):- """Mistral family of models."""-- def __init__(- self,- aws_provider: AWSProvider,- model: str,- max_context_size: int = 32768,- max_new_tokens: int = 1500,- **kwargs,- ) -> None:- """NOTE: Configure your AWS credentials with the AWS CLI before using this model!"""- super().__init__(- model=model,- max_context_size=max_context_size,- max_new_tokens=max_new_tokens,- **kwargs,- )- self.aws_provider = aws_provider- self.provider = aws_provider.get_provider_name()-- self.kwargs["stop"] = "\n\n---"-- def _format_prompt(self, raw_prompt: str) -> str:- return "<s> [INST] Human: " + raw_prompt + " [/INST] Assistant: "-- def _create_body(self, prompt: str, **kwargs) -> tuple[int, dict[str, str | float]]:- base_args: dict[str, Any] = self.kwargs- for k, v in kwargs.items():- base_args[k] = v-- n, base_args = self.aws_provider.sanitize_kwargs(base_args)-- query_args: dict[str, str | float] = {}- if isinstance(self.aws_provider, Bedrock):- query_args["prompt"] = prompt- elif isinstance(self.aws_provider, Sagemaker):- query_args["parameters"] = base_args- query_args["inputs"] = prompt- else:- raise ValueError("Error - provider not recognized")-- return (n, query_args)-- def _call_model(self, body: str) -> str:- response = self.aws_provider.call_model(- model_id=self._model_name,- body=body,- )- if isinstance(self.aws_provider, Bedrock):- response_body = json.loads(response["body"].read())- completion = response_body["outputs"][0]["text"]- elif isinstance(self.aws_provider, Sagemaker):- response_body = json.loads(response["Body"].read())- completion = response_body[0]["generated_text"]- else:- raise ValueError("Error - provider not recognized")-- completion = completion.split(self.kwargs["stop"])[0]- return completion---class AWSAnthropic(AWSModel):- """Anthropic family of models."""-- def __init__(- self,- aws_provider: AWSProvider,- model: str,- max_context_size: int = 200000,- max_new_tokens: int = 1500,- **kwargs,- ) -> None:- """NOTE: Configure your AWS credentials with the AWS CLI before using this model!"""- super().__init__(- model=model,- max_context_size=max_context_size,- max_new_tokens=max_new_tokens,- **kwargs,- )- self.provider = aws_provider-- for k, v in kwargs.items():- self.kwargs[k] = v-- def _create_body(self, prompt: str, **kwargs) -> tuple[int, dict[str, str | float]]:- base_args: dict[str, Any] = self.kwargs- for k, v in kwargs.items():- base_args[k] = v-- n, query_args = self.provider.sanitize_kwargs(base_args)-- # Anthropic models do not support the following parameters- query_args.pop("frequency_penalty", None)- query_args.pop("num_generations", None)- query_args.pop("presence_penalty", None)- query_args.pop("model", None)-- # we are using the Claude messages API- # https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages.html- query_args["messages"] = [- {- "role": "user",- "content": [- {- "type": "text",- "text": prompt,- },- ],- },- ]- query_args["anthropic_version"] = "bedrock-2023-05-31"- return (n, query_args)-- def _call_model(self, body: str) -> str:- response = self.provider.predictor.invoke_model(- modelId=self._model_name,- body=body,- )- response_body = json.loads(response["body"].read())- completion = response_body["content"][0]["text"]- return completion---class AWSMeta(AWSModel):- """Llama2 family of models."""-- def __init__(- self,- aws_provider: AWSProvider,- model: str,- max_context_size: int = 4096,- max_new_tokens: int = 1500,- **kwargs,- ) -> None:- """NOTE: Configure your AWS credentials with the AWS CLI before using this model!"""- super().__init__(- model=model,- max_context_size=max_context_size,- max_new_tokens=max_new_tokens,- **kwargs,- )- self.provider = aws_provider-- for k, v in kwargs.items():- self.kwargs[k] = v-- self.kwargs["max_gen_len"] = self.kwargs.pop("max_tokens")-- def _create_body(self, prompt: str, **kwargs) -> tuple[int, dict[str, str | float]]:- base_args: dict[str, Any] = self.kwargs- for k, v in kwargs.items():- base_args[k] = v-- n, query_args = self.provider.sanitize_kwargs(base_args)-- # Meta models do not support the following parameters- query_args.pop("frequency_penalty", None)- query_args.pop("num_generations", None)- query_args.pop("presence_penalty", None)- query_args.pop("model", None)-- query_args["prompt"] = prompt- return (n, query_args)-- def _call_model(self, body: str) -> str:- response = self.provider.predictor.invoke_model(- modelId=self._model_name,- body=body,- )- response_body = json.loads(response["body"].read())- completion = response_body["generation"]-- stop = "\n\n"- completion = completion.split(stop)[0]-- return completiondiff --git a/dsp/modules/aws_providers.py b/dsp/modules/aws_providers.pydeleted file mode 100644index ae3f0d4c3..000000000--- a/dsp/modules/aws_providers.py+++ /dev/null@@ -1,128 +0,0 @@-"""AWS providers for LMs."""--from abc import ABC, abstractmethod-from typing import Any, Optional---class AWSProvider(ABC):- """This abstract class adds support for AWS model providers such as Bedrock and SageMaker.- The subclasses such as Bedrock and Sagemaker implement the abstract method _call_model and work in conjunction with the AWSModel classes.- Usage Example:- bedrock = Bedrock(region_name="us-west-2")- bedrock_mixtral = AWSMistral(bedrock, "mistral.mixtral-8x7b-instruct-v0:1", **kwargs)- bedrock_haiku = AWSAnthropic(bedrock, "anthropic.claude-3-haiku-20240307-v1:0", **kwargs)- bedrock_llama2 = AWSMeta(bedrock, "meta.llama2-13b-chat-v1", **kwargs)-- sagemaker = Sagemaker(region_name="us-west-2")- sagemaker_model = AWSMistral(sagemaker, "<YOUR_ENDPOINT_NAME>", **kwargs)- """-- def __init__(- self,- region_name: str,- service_name: str,- profile_name: Optional[str] = None,- batch_n_enabled: bool = True,- ) -> None:- """_summary_.-- Args:- region_name (str, optional): The AWS region where this LM is hosted.- service_name (str): Used in context of invoking the boto3 API.- profile_name (str, optional): boto3 credentials profile.- batch_n_enabled (bool): If False, call the LM N times rather than batching.- """- try:- import boto3 # pylint: disable=import-outside-toplevel- except ImportError as exc:- raise ImportError('Please install boto3 to use AWS models.') from exc-- if profile_name is None:- self.predictor = boto3.client(service_name, region_name=region_name)- else:- self.predictor = boto3.Session(profile_name=profile_name).client(- service_name,- region_name=region_name,- )-- self.batch_n_enabled = batch_n_enabled-- def get_provider_name(self) -> str:- """Return the provider name."""- return self.__class__.__name__-- @abstractmethod- def call_model(self, model_id: str, body: str) -> str:- """Call the model and return the response."""-- def sanitize_kwargs(self, query_kwargs: dict[str, Any]) -> tuple[int, dict[str, Any]]:- """Ensure that input kwargs can be used by Bedrock or Sagemaker."""- if "temperature" in query_kwargs:- if query_kwargs["temperature"] > 0.99:- query_kwargs["temperature"] = 0.99- if query_kwargs["temperature"] < 0.01:- query_kwargs["temperature"] = 0.01-- if "top_p" in query_kwargs:- if query_kwargs["top_p"] > 0.99:- query_kwargs["top_p"] = 0.99- if query_kwargs["top_p"] < 0.01:- query_kwargs["top_p"] = 0.01-- n = -1- if not self.batch_n_enabled:- n = query_kwargs.pop('n', 1)- query_kwargs["num_generations"] = n-- return n, query_kwargs---class Bedrock(AWSProvider):- """This class adds support for Bedrock models."""-- def __init__(- self,- region_name: str,- profile_name: Optional[str] = None,- batch_n_enabled: bool = False, # This has to be setup manually on Bedrock.- ) -> None:- """_summary_.-- Args:- region_name (str, optional): The AWS region where this LM is hosted.- profile_name (str, optional): boto3 credentials profile.- """- super().__init__(region_name, "bedrock-runtime", profile_name, batch_n_enabled)-- def call_model(self, model_id: str, body: str) -> str:- return self.predictor.invoke_model(- modelId=model_id,- body=body,- accept="application/json",- contentType="application/json",- )---class Sagemaker(AWSProvider):- """This class adds support for Sagemaker models."""-- def __init__(- self,- region_name: str,- profile_name: Optional[str] = None,- ) -> None:- """_summary_.-- Args:- region_name (str, optional): The AWS region where this LM is hosted.- profile_name (str, optional): boto3 credentials profile.- """- super().__init__(region_name, "runtime.sagemaker", profile_name)-- def call_model(self, model_id: str, body: str) -> str:- return self.predictor.invoke_endpoint(- EndpointName=model_id,- Body=body,- Accept="application/json",- ContentType="application/json",- )diff --git a/dsp/modules/bedrock.py b/dsp/modules/bedrock.pynew file mode 100644index 000000000..252c87fe8--- /dev/null+++ b/dsp/modules/bedrock.py@@ -0,0 +1,79 @@+from __future__ import annotations++import json+from typing import Any, Optional++from dsp.modules.aws_lm import AWSLM+++class Bedrock(AWSLM):+ def __init__(+ self,+ region_name: str,+ model: str,+ profile_name: Optional[str] = None,+ input_output_ratio: int = 3,+ max_new_tokens: int = 1500,+ ) -> None:+ """Use an AWS Bedrock language model.+ NOTE: You must first configure your AWS credentials with the AWS CLI before using this model!++ Args:+ region_name (str, optional): The AWS region where this LM is hosted.+ model (str, optional): An AWS Bedrock LM name. You can find available models with the AWS CLI as follows: aws bedrock list-foundation-models --query "modelSummaries[*].modelId".+ temperature (float, optional): Default temperature for LM. Defaults to 0.+ input_output_ratio (int, optional): The rough size of the number of input tokens to output tokens in the worst case. Defaults to 3.+ max_new_tokens (int, optional): The maximum number of tokens to be sampled from the LM.+ """+ super().__init__(+ model=model,+ service_name="bedrock-runtime",+ region_name=region_name,+ profile_name=profile_name,+ truncate_long_prompts=False,+ input_output_ratio=input_output_ratio,+ max_new_tokens=max_new_tokens,+ batch_n=True, # Bedrock does not support the `n` parameter+ )+ self._validate_model(model)+ self.provider = "claude" if "claude" in model.lower() else "bedrock"++ def _validate_model(self, model: str) -> None:+ if "claude" not in model.lower():+ raise NotImplementedError("Only claude models are supported as of now")++ def _create_body(self, prompt: str, **kwargs) -> dict[str, str | float]:+ base_args: dict[str, Any] = {+ "max_tokens_to_sample": self._max_new_tokens,+ }+ for k, v in kwargs.items():+ base_args[k] = v+ query_args: dict[str, Any] = self._sanitize_kwargs(base_args)+ query_args["prompt"] = prompt+ # AWS Bedrock forbids these keys+ if "max_tokens" in query_args:+ max_tokens: int = query_args["max_tokens"]+ input_tokens: int = self._estimate_tokens(prompt)+ max_tokens_to_sample: int = max_tokens - input_tokens+ del query_args["max_tokens"]+ query_args["max_tokens_to_sample"] = max_tokens_to_sample+ return query_args++ def _call_model(self, body: str) -> str:+ response = self.predictor.invoke_model(+ modelId=self._model_name,+ body=body,+ accept="application/json",+ contentType="application/json",+ )+ response_body = json.loads(response["body"].read())+ completion = response_body["completion"]+ return completion++ def _extract_input_parameters(+ self, body: dict[Any, Any],+ ) -> dict[str, str | float | int]:+ return body++ def _format_prompt(self, raw_prompt: str) -> str:+ return "\n\nHuman: " + raw_prompt + "\n\nAssistant:"diff --git a/dspy/__init__.py b/dspy/__init__.pyindex 2d5ba4f00..1c6539f59 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -33,13 +33,7 @@ Anyscale = dsp.AnyscaleTogether = dsp.TogetherHFModel = dsp.HFModelOllamaLocal = dsp.OllamaLocal-Bedrock = dsp.Bedrock-Sagemaker = dsp.Sagemaker-AWSModel = dsp.AWSModel-AWSMistral = dsp.AWSMistral-AWSAnthropic = dsp.AWSAnthropic-AWSMeta = dsp.AWSMetaconfigure = settings.configurecontext = settings.contextdiff --git a/pyproject.toml b/pyproject.tomlindex f564865f2..2dafeab07 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -46,7 +46,6 @@ marqo = ["marqo"]pinecone = ["pinecone-client~=2.2.4"]weaviate = ["weaviate-client~=4.5.4"]milvus = ["pymilvus~=2.3.7"]-aws = ["boto3~=1.34.78"]docs = ["sphinx>=4.3.0","furo>=2023.3.27",@@ -59,6 +58,7 @@ docs = ["autodoc_pydantic","sphinx-reredirects>=0.1.2","sphinx-automodapi==0.16.0",+]dev = ["pytest>=6.2.5"]@@ -100,7 +100,6 @@ qdrant-client = { version = "^1.6.2", optional = true }pinecone-client = { version = "^2.2.4", optional = true }weaviate-client = { version = "^4.5.4", optional = true }pymilvus = { version = "^2.3.6", optional = true }-boto3 = { version = "^1.34.78", optional = true }sphinx = { version = ">=4.3.0", optional = true }furo = { version = ">=2023.3.27", optional = true }docutils = { version = "<0.17", optional = true }@@ -135,7 +134,6 @@ marqo = ["marqo"]pinecone = ["pinecone-client"]weaviate = ["weaviate-client"]milvus = ["pymilvus"]-aws = ["boto3"]postgres = ["psycopg2", "pgvector"]docs = ["sphinx",diff --git a/tests/modules/test_aws_models.py b/tests/modules/test_aws_models.pydeleted file mode 100644index 89cedcc6f..000000000--- a/tests/modules/test_aws_models.py+++ /dev/null@@ -1,59 +0,0 @@-"""Tests for AWS models.-Note: Requires configuration of your AWS credentials with the AWS CLI and creating sagemaker endpoints.-TODO: Create mock fixtures for pytest to remove the need for AWS credentials and endpoints.-"""--import dsp-import dspy--def get_lm(lm_provider: str, model_path: str, **kwargs) -> dsp.modules.lm.LM:- """get the language model"""- # extract model vendor and name from model name- # Model path format is <MODEL_VENDOR>/<MODEL_NAME_OR_ENDPOINT>- model_vendor = model_path.split('/')[0]- model_name = model_path.split('/')[1]-- if lm_provider == 'Bedrock':- bedrock = dspy.Bedrock(region_name="us-west-2")- if model_vendor == 'mistral':- return dspy.AWSMistral(bedrock, model_name, **kwargs)- elif model_vendor == 'anthropic':- return dspy.AWSAnthropic(bedrock, model_name, **kwargs)- elif model_vendor == 'meta':- return dspy.AWSMeta(bedrock, model_name, **kwargs)- else:- raise ValueError("Model vendor missing or unsupported: Model path format is <MODEL_VENDOR>/<MODEL_NAME_OR_ENDPOINT>")- elif lm_provider == 'Sagemaker':- sagemaker = dspy.Sagemaker(region_name="us-west-2")- if model_vendor == 'mistral':- return dspy.AWSMistral(sagemaker, model_name, **kwargs)- elif model_vendor == 'meta':- return dspy.AWSMeta(sagemaker, model_name, **kwargs)- else:- raise ValueError("Model vendor missing or unsupported: Model path format is <MODEL_VENDOR>/<MODEL_NAME_OR_ENDPOINT>")- else:- raise ValueError(f"Unsupported model: {model_name}")--def run_tests():- """Test the providers and models"""- # Configure your AWS credentials with the AWS CLI before running this script- provider_model_tuples = [- ('Bedrock', 'mistral/mistral.mixtral-8x7b-instruct-v0:1'),- ('Bedrock', 'anthropic/anthropic.claude-3-haiku-20240307-v1:0'),- ('Bedrock', 'anthropic/anthropic.claude-3-sonnet-20240229-v1:0'),- ('Bedrock', 'meta/meta.llama2-70b-chat-v1'),- # ('Sagemaker', 'mistral/<YOUR_ENDPOINT_NAME>'), # REPLACE YOUR_ENDPOINT_NAME with your sagemaker endpoint- ]-- predict_func = dspy.Predict("question -> answer")- for provider, model_path in provider_model_tuples:- print(f"Provider: {provider}, Model: {model_path}")- lm = get_lm(provider, model_path)- with dspy.context(lm=lm):- question = "What is the capital of France?"- answer = predict_func(question=question).answer- print(f"Question: {question}\nAnswer: {answer}\n\n")---if __name__ == "__main__":- run_tests()