DSPy: The framework for programming—not prompting—language models
Python
feat(bedrock-lm): Added a LM for Bedrock and a base AWS LM class that can later be used for Sagemaker.
Jamie Scharf04fad29321fb
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.pyindex e3518b7b1..1fb45bdac 100644--- a/dsp/modules/__init__.py+++ b/dsp/modules/__init__.py@@ -8,8 +8,9 @@ from .sbert import *from .pyserini import *from .ollama import *from .clarifai import *+from .bedrock import *from .hf_client import HFClientTGIfrom .hf_client import Anyscale-from .hf_client import Together\ No newline at end of file+from .hf_client import Togetherdiff --git a/dsp/modules/aws_lm.py b/dsp/modules/aws_lm.pynew file mode 100644index 000000000..dab5c92d6--- /dev/null+++ b/dsp/modules/aws_lm.py@@ -0,0 +1,147 @@+"""+A generalized AWS LLM.+"""++from abc import abstractmethod+import logging+from typing import Any, Literal+import json+import boto3+from dsp.modules.lm import LM++# Heuristic translating number of chars to tokens+# ~4 chars = 1 token+CHARS2TOKENS: int = 4+++class AWSLM(LM):+ """+ This class adds support for an AWS model+ """++ def __init__(+ self,+ model: str,+ region_name: str,+ service_name: str,+ max_new_tokens: int,+ truncate_long_prompts: bool = False,+ input_output_ratio: int = 3,+ ) -> None:+ """_summary_++ Args:++ service_name (str): Used in context of invoking the boto3 API.+ region_name (str, optional): The AWS region where this LM is hosted.+ model (str, optional): An LM name, e.g., a bedrock name or an AWS endpoint.+ max_new_tokens (int, optional): The maximum number of tokens to be sampled from the LM.+ input_output_ratio (int, optional): The rough size of the number of input tokens to output tokens in the worst case. Defaults to 3.+ temperature (float, optional): _description_. Defaults to 0.0.+ truncate_long_prompts (bool, optional): If True, remove extremely long inputs to context. Defaults to False.+ """+ super().__init__(model=model)+ # AWS doesn't have an equivalent of max_tokens so let's clarify+ # that the expected input is going to be about 2x as long as the output+ self.kwargs["max_tokens"] = max_new_tokens * input_output_ratio+ self._max_new_tokens: int = max_new_tokens+ self._model_name: str = model+ self._truncate_long_prompt_prompts: bool = truncate_long_prompts+ self.predictor = boto3.client(service_name, region_name=region_name)++ @abstractmethod+ def _create_body(self, prompt: str, **kwargs) -> dict[str, str | float]:+ pass++ def _sanitize_kwargs(self, query_kwargs: dict[str, Any]) -> dict[str, Any]:+ """Ensure that input kwargs can be used by Bedrock or Sagemaker."""+ base_args: dict[str, Any] = {"temperature": self.kwargs["temperature"]}++ for k, v in base_args.items():+ if k not in query_kwargs:+ query_kwargs[k] = v+ if query_kwargs["temperature"] > 1.0:+ query_kwargs["temperature"] = 0.99+ if query_kwargs["temperature"] < 0.01:+ query_kwargs["temperature"] = 0.01++ return query_kwargs++ @abstractmethod+ def _call_model(self, body: str) -> str:+ """Call model, get generated input without the formatted prompt"""+ pass++ @abstractmethod+ def _extract_input_parameters(+ self, body: dict[Any, Any]+ ) -> dict[str, str | float | int]:+ pass++ def basic_request(self, prompt, **kwargs) -> str:+ """Query the endpoint."""++ # Remove any texts that are too long+ formatted_prompt: str+ if self._truncate_long_prompt_prompts:+ truncated_prompt: str = self._truncate_prompt(prompt)+ formatted_prompt = self._format_prompt(truncated_prompt)+ else:+ formatted_prompt = self._format_prompt((prompt))+ body: dict[str, str | float] = self._create_body(formatted_prompt, **kwargs)+ json_body: str = json.dumps(body)++ generated: str = self._call_model(json_body)++ self.history.append(+ {"prompt": formatted_prompt, "response": generated, "kwargs": body}+ )++ return generated.replace(formatted_prompt, "")++ def _estimate_tokens(self, text: str) -> int:+ return len(text) * CHARS2TOKENS++ @abstractmethod+ def _format_prompt(self, raw_prompt: str) -> str:+ pass++ def _truncate_prompt(+ self,+ input_text: str,+ remove_beginning_or_ending: Literal["beginning", "ending"] = "beginning",+ max_input_tokens: int = 2500,+ ) -> str:+ """Reformat inputs such that they do not overflow context size limitation."""+ token_count = self._estimate_tokens(input_text)+ if token_count > self.kwargs["max_tokens"]:+ logging.info("Excessive prompt found in llm input")+ logging.info("Truncating texts to avoid error")+ max_chars: int = CHARS2TOKENS * max_input_tokens+ truncated_text: str+ if remove_beginning_or_ending == "ending":+ truncated_text = input_text[0:max_chars]+ else:+ truncated_text = input_text[-max_chars:]+ return truncated_text+ return input_text++ def __call__(+ self,+ prompt: str,+ only_completed: bool = True,+ return_sorted: bool = False,+ **kwargs,+ ) -> list[str]:+ """+ Query the AWS LLM.++ There is only support for only_completed=True and return_sorted=False+ right now.+ """+ if not only_completed:+ raise NotImplementedError("Error, only_completed not yet supported!")+ if return_sorted:+ raise NotImplementedError("Error, return_sorted not yet supported!")+ generated = self.basic_request(prompt, **kwargs)+ return [generated]diff --git a/dsp/modules/bedrock.py b/dsp/modules/bedrock.pynew file mode 100644index 000000000..f3b88c09e--- /dev/null+++ b/dsp/modules/bedrock.py@@ -0,0 +1,62 @@+from typing import Any+import json+from dsp.modules.aws_lm import AWSLM+++class Bedrock(AWSLM):+ def __init__(+ self,+ region_name: str,+ model: str,+ input_output_ratio: int = 3,+ max_new_tokens: int = 1500,+ ) -> None:+ """Use an AWS Bedrock language model.+ NOTE: You must first configure your AWS credentials with the AWS CLI before using this model!++ Args:+ region_name (str, optional): The AWS region where this LM is hosted.+ model (str, optional): An AWS Bedrock LM name. You can find available models with the AWS CLI as follows: aws bedrock list-foundation-models --query "modelSummaries[*].modelId".+ temperature (float, optional): Default temperature for LM. Defaults to 0.+ input_output_ratio (int, optional): The rough size of the number of input tokens to output tokens in the worst case. Defaults to 3.+ max_new_tokens (int, optional): The maximum number of tokens to be sampled from the LM.+ """+ super().__init__(+ model=model,+ service_name="bedrock-runtime",+ region_name=region_name,+ truncate_long_prompts=False,+ input_output_ratio=input_output_ratio,+ max_new_tokens=max_new_tokens,+ )++ def _create_body(self, prompt: str, **kwargs) -> dict[str, str | float]:+ base_args: dict[str, Any] = {+ "max_tokens_to_sample": self._max_new_tokens,+ }+ for k, v in kwargs.items():+ base_args[k] = v+ query_args: dict[str, Any] = self._sanitize_kwargs(base_args)+ query_args["prompt"] = prompt+ # AWS Bedrock forbids these keys++ return query_args++ def _call_model(self, body: str) -> str:+ response = self.predictor.invoke_model(+ modelId=self._model_name,+ body=body,+ accept="application/json",+ contentType="application/json",+ )+ response_body = json.loads(response["body"].read())+ completion = response_body["completion"]+ return completion++ def _extract_input_parameters(+ self, body: dict[Any, Any]+ ) -> dict[str, str | float | int]:+ return body++ def _format_prompt(self, raw_prompt: str) -> str:+ return "\n\nHuman: " + raw_prompt + "\n\nAssistant:"diff --git a/dspy/__init__.py b/dspy/__init__.pyindex 9296a37d2..c30295518 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -5,6 +5,7 @@ from .signatures import *from .retrieve import *from .predict import *from .primitives import *+# from .evaluation import *@@ -28,6 +29,7 @@ Anyscale = dsp.AnyscaleTogether = dsp.TogetherHFModel = dsp.HFModelOllamaLocal = dsp.OllamaLocal+Bedrock = dsp.Bedrockconfigure = settings.configure-context = settings.context\ No newline at end of file+context = settings.contextdiff --git a/pyproject.toml b/pyproject.tomlindex 204db0515..0dc3b3c96 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -29,6 +29,7 @@ dependencies = ["datasets~=2.14.6","requests~=2.31.0","optuna~=3.4.0",+ "boto3~=1.34"][project.optional-dependencies]