Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

feat(bedrock-lm): Added a LM for Bedrock and a base AWS LM class that can later be used for Sagemaker.

Jamie Scharf04fad29321fb
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.py
index e3518b7b1..1fb45bdac 100644
--- a/dsp/modules/__init__.py
+++ b/dsp/modules/__init__.py
@@ -8,8 +8,9 @@ from .sbert import *
from .pyserini import *
from .ollama import *
from .clarifai import *
+from .bedrock import *
from .hf_client import HFClientTGI
from .hf_client import Anyscale
-from .hf_client import Together
\ No newline at end of file
+from .hf_client import Together
diff --git a/dsp/modules/aws_lm.py b/dsp/modules/aws_lm.py
new file mode 100644
index 000000000..dab5c92d6
--- /dev/null
+++ b/dsp/modules/aws_lm.py
@@ -0,0 +1,147 @@
+"""
+A generalized AWS LLM.
+"""
+
+from abc import abstractmethod
+import logging
+from typing import Any, Literal
+import json
+import boto3
+from dsp.modules.lm import LM
+
+# Heuristic translating number of chars to tokens
+# ~4 chars = 1 token
+CHARS2TOKENS: int = 4
+
+
+class AWSLM(LM):
+ """
+ This class adds support for an AWS model
+ """
+
+ def __init__(
+ self,
+ model: str,
+ region_name: str,
+ service_name: str,
+ max_new_tokens: int,
+ truncate_long_prompts: bool = False,
+ input_output_ratio: int = 3,
+ ) -> None:
+ """_summary_
+
+ Args:
+
+ service_name (str): Used in context of invoking the boto3 API.
+ region_name (str, optional): The AWS region where this LM is hosted.
+ model (str, optional): An LM name, e.g., a bedrock name or an AWS endpoint.
+ max_new_tokens (int, optional): The maximum number of tokens to be sampled from the LM.
+ input_output_ratio (int, optional): The rough size of the number of input tokens to output tokens in the worst case. Defaults to 3.
+ temperature (float, optional): _description_. Defaults to 0.0.
+ truncate_long_prompts (bool, optional): If True, remove extremely long inputs to context. Defaults to False.
+ """
+ super().__init__(model=model)
+ # AWS doesn't have an equivalent of max_tokens so let's clarify
+ # that the expected input is going to be about 2x as long as the output
+ self.kwargs["max_tokens"] = max_new_tokens * input_output_ratio
+ self._max_new_tokens: int = max_new_tokens
+ self._model_name: str = model
+ self._truncate_long_prompt_prompts: bool = truncate_long_prompts
+ self.predictor = boto3.client(service_name, region_name=region_name)
+
+ @abstractmethod
+ def _create_body(self, prompt: str, **kwargs) -> dict[str, str | float]:
+ pass
+
+ def _sanitize_kwargs(self, query_kwargs: dict[str, Any]) -> dict[str, Any]:
+ """Ensure that input kwargs can be used by Bedrock or Sagemaker."""
+ base_args: dict[str, Any] = {"temperature": self.kwargs["temperature"]}
+
+ for k, v in base_args.items():
+ if k not in query_kwargs:
+ query_kwargs[k] = v
+ if query_kwargs["temperature"] > 1.0:
+ query_kwargs["temperature"] = 0.99
+ if query_kwargs["temperature"] < 0.01:
+ query_kwargs["temperature"] = 0.01
+
+ return query_kwargs
+
+ @abstractmethod
+ def _call_model(self, body: str) -> str:
+ """Call model, get generated input without the formatted prompt"""
+ pass
+
+ @abstractmethod
+ def _extract_input_parameters(
+ self, body: dict[Any, Any]
+ ) -> dict[str, str | float | int]:
+ pass
+
+ def basic_request(self, prompt, **kwargs) -> str:
+ """Query the endpoint."""
+
+ # Remove any texts that are too long
+ formatted_prompt: str
+ if self._truncate_long_prompt_prompts:
+ truncated_prompt: str = self._truncate_prompt(prompt)
+ formatted_prompt = self._format_prompt(truncated_prompt)
+ else:
+ formatted_prompt = self._format_prompt((prompt))
+ body: dict[str, str | float] = self._create_body(formatted_prompt, **kwargs)
+ json_body: str = json.dumps(body)
+
+ generated: str = self._call_model(json_body)
+
+ self.history.append(
+ {"prompt": formatted_prompt, "response": generated, "kwargs": body}
+ )
+
+ return generated.replace(formatted_prompt, "")
+
+ def _estimate_tokens(self, text: str) -> int:
+ return len(text) * CHARS2TOKENS
+
+ @abstractmethod
+ def _format_prompt(self, raw_prompt: str) -> str:
+ pass
+
+ def _truncate_prompt(
+ self,
+ input_text: str,
+ remove_beginning_or_ending: Literal["beginning", "ending"] = "beginning",
+ max_input_tokens: int = 2500,
+ ) -> str:
+ """Reformat inputs such that they do not overflow context size limitation."""
+ token_count = self._estimate_tokens(input_text)
+ if token_count > self.kwargs["max_tokens"]:
+ logging.info("Excessive prompt found in llm input")
+ logging.info("Truncating texts to avoid error")
+ max_chars: int = CHARS2TOKENS * max_input_tokens
+ truncated_text: str
+ if remove_beginning_or_ending == "ending":
+ truncated_text = input_text[0:max_chars]
+ else:
+ truncated_text = input_text[-max_chars:]
+ return truncated_text
+ return input_text
+
+ def __call__(
+ self,
+ prompt: str,
+ only_completed: bool = True,
+ return_sorted: bool = False,
+ **kwargs,
+ ) -> list[str]:
+ """
+ Query the AWS LLM.
+
+ There is only support for only_completed=True and return_sorted=False
+ right now.
+ """
+ if not only_completed:
+ raise NotImplementedError("Error, only_completed not yet supported!")
+ if return_sorted:
+ raise NotImplementedError("Error, return_sorted not yet supported!")
+ generated = self.basic_request(prompt, **kwargs)
+ return [generated]
diff --git a/dsp/modules/bedrock.py b/dsp/modules/bedrock.py
new file mode 100644
index 000000000..f3b88c09e
--- /dev/null
+++ b/dsp/modules/bedrock.py
@@ -0,0 +1,62 @@
+from typing import Any
+import json
+from dsp.modules.aws_lm import AWSLM
+
+
+class Bedrock(AWSLM):
+ def __init__(
+ self,
+ region_name: str,
+ model: str,
+ input_output_ratio: int = 3,
+ max_new_tokens: int = 1500,
+ ) -> None:
+ """Use an AWS Bedrock language model.
+ NOTE: You must first configure your AWS credentials with the AWS CLI before using this model!
+
+ Args:
+ region_name (str, optional): The AWS region where this LM is hosted.
+ model (str, optional): An AWS Bedrock LM name. You can find available models with the AWS CLI as follows: aws bedrock list-foundation-models --query "modelSummaries[*].modelId".
+ temperature (float, optional): Default temperature for LM. Defaults to 0.
+ input_output_ratio (int, optional): The rough size of the number of input tokens to output tokens in the worst case. Defaults to 3.
+ max_new_tokens (int, optional): The maximum number of tokens to be sampled from the LM.
+ """
+ super().__init__(
+ model=model,
+ service_name="bedrock-runtime",
+ region_name=region_name,
+ truncate_long_prompts=False,
+ input_output_ratio=input_output_ratio,
+ max_new_tokens=max_new_tokens,
+ )
+
+ def _create_body(self, prompt: str, **kwargs) -> dict[str, str | float]:
+ base_args: dict[str, Any] = {
+ "max_tokens_to_sample": self._max_new_tokens,
+ }
+ for k, v in kwargs.items():
+ base_args[k] = v
+ query_args: dict[str, Any] = self._sanitize_kwargs(base_args)
+ query_args["prompt"] = prompt
+ # AWS Bedrock forbids these keys
+
+ return query_args
+
+ def _call_model(self, body: str) -> str:
+ response = self.predictor.invoke_model(
+ modelId=self._model_name,
+ body=body,
+ accept="application/json",
+ contentType="application/json",
+ )
+ response_body = json.loads(response["body"].read())
+ completion = response_body["completion"]
+ return completion
+
+ def _extract_input_parameters(
+ self, body: dict[Any, Any]
+ ) -> dict[str, str | float | int]:
+ return body
+
+ def _format_prompt(self, raw_prompt: str) -> str:
+ return "\n\nHuman: " + raw_prompt + "\n\nAssistant:"
diff --git a/dspy/__init__.py b/dspy/__init__.py
index 9296a37d2..c30295518 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -5,6 +5,7 @@ from .signatures import *
from .retrieve import *
from .predict import *
from .primitives import *
+
# from .evaluation import *
@@ -28,6 +29,7 @@ Anyscale = dsp.Anyscale
Together = dsp.Together
HFModel = dsp.HFModel
OllamaLocal = dsp.OllamaLocal
+Bedrock = dsp.Bedrock
configure = settings.configure
-context = settings.context
\ No newline at end of file
+context = settings.context
diff --git a/pyproject.toml b/pyproject.toml
index 204db0515..0dc3b3c96 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -29,6 +29,7 @@ dependencies = [
"datasets~=2.14.6",
"requests~=2.31.0",
"optuna~=3.4.0",
+ "boto3~=1.34"
]
[project.optional-dependencies]