Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: enable bedrock for anthropic models (#8847)
Ari Webb9dbf428c1ff4
diff --git a/letta/llm_api/bedrock_client.py b/letta/llm_api/bedrock_client.pyindex d471424fe..a49dcfb25 100644--- a/letta/llm_api/bedrock_client.py+++ b/letta/llm_api/bedrock_client.py@@ -16,6 +16,18 @@ logger = get_logger(__name__)class BedrockClient(AnthropicClient):+ @staticmethod+ def get_inference_profile_id_from_handle(handle: str) -> str:+ """+ Extract the Bedrock inference profile ID from the LLMConfig handle.++ The handle format is: bedrock/us.anthropic.claude-opus-4-5-20250918-v1:0+ Returns: us.anthropic.claude-opus-4-5-20250918-v1:0+ """+ if "/" in handle:+ return handle.split("/", 1)[1]+ return handle+async def get_byok_overrides_async(self, llm_config: LLMConfig) -> tuple[str, str, str]:override_access_key_id, override_secret_access_key, override_default_region = None, None, Noneif llm_config.provider_category == ProviderCategory.byok:@@ -74,6 +86,13 @@ class BedrockClient(AnthropicClient):tool_return_truncation_chars: Optional[int] = None,) -> dict:data = super().build_request_data(agent_type, messages, llm_config, tools, force_tool_call, requires_subsequent_tool_call)++ # Swap the model name back to the Bedrock inference profile ID for the API call+ # The LLMConfig.model contains the Anthropic-style name (e.g., "claude-opus-4-5-20250918")+ # but Bedrock API needs the inference profile ID (e.g., "us.anthropic.claude-opus-4-5-20250918-v1:0")+ if llm_config.handle:+ data["model"] = self.get_inference_profile_id_from_handle(llm_config.handle)+# remove disallowed fieldsif "tool_choice" in data:del data["tool_choice"]["disable_parallel_tool_use"]diff --git a/letta/schemas/llm_config.py b/letta/schemas/llm_config.pyindex 58b1f22ca..5ce041f88 100644--- a/letta/schemas/llm_config.py+++ b/letta/schemas/llm_config.py@@ -182,7 +182,7 @@ class LLMConfig(BaseModel):if is_openai_reasoning_model(model):values["put_inner_thoughts_in_kwargs"] = False- if values.get("model_endpoint_type") == "anthropic" and (+ if values.get("model_endpoint_type") in ("anthropic", "bedrock") and (model.startswith("claude-3-7-sonnet")or model.startswith("claude-sonnet-4")or model.startswith("claude-opus-4")@@ -413,7 +413,7 @@ class LLMConfig(BaseModel):@classmethoddef is_anthropic_reasoning_model(cls, config: "LLMConfig") -> bool:- return config.model_endpoint_type == "anthropic" and (+ return config.model_endpoint_type in ("anthropic", "bedrock") and (config.model.startswith("claude-opus-4")or config.model.startswith("claude-sonnet-4")or config.model.startswith("claude-3-7-sonnet")diff --git a/letta/schemas/providers/bedrock.py b/letta/schemas/providers/bedrock.pyindex 4cfaff810..7f833ad85 100644--- a/letta/schemas/providers/bedrock.py+++ b/letta/schemas/providers/bedrock.py@@ -18,22 +18,46 @@ logger = get_logger(__name__)class BedrockProvider(Provider):provider_type: Literal[ProviderType.bedrock] = Field(ProviderType.bedrock, description="The type of the provider.")provider_category: ProviderCategory = Field(ProviderCategory.base, description="The category of the provider (base or byok)")- access_key: str = Field(..., description="AWS secret access key for Bedrock.")+ access_key: str | None = Field(None, description="AWS access key ID for Bedrock")+ api_key: str | None = Field(None, description="AWS secret access key for Bedrock")region: str = Field(..., description="AWS region for Bedrock")+ @staticmethod+ def extract_anthropic_model_name(inference_profile_id: str) -> str:+ """+ Extract the Anthropic-style model name from a Bedrock inference profile ID.++ Input format: us.anthropic.claude-opus-4-5-20250918-v1:0+ Output: claude-opus-4-5-20250918++ This allows Bedrock models to use the same model name format as regular Anthropic models,+ so all the existing model name checks (startswith("claude-"), etc.) work correctly.+ """+ # Remove region prefix (e.g., "us.anthropic." -> "claude-...")+ if ".anthropic." in inference_profile_id:+ model_part = inference_profile_id.split(".anthropic.")[1]+ else:+ model_part = inference_profile_id++ # Remove version suffix (e.g., "-v1:0" at the end)+ # Pattern: -v followed by digits, optionally followed by :digits+ import re++ model_name = re.sub(r"-v\d+(?::\d+)?$", "", model_part)+ return model_name+async def bedrock_get_model_list_async(self) -> list[dict]:+ """+ List Bedrock inference profiles using boto3.+ """from aioboto3.session import Sessiontry:- # Decrypt credentials before using- access_key = await self.access_key_enc.get_plaintext_async() if self.access_key_enc else None- secret_key = await self.api_key_enc.get_plaintext_async() if self.api_key_enc else None-session = Session()async with session.client("bedrock",- aws_access_key_id=access_key,- aws_secret_access_key=secret_key,+ aws_access_key_id=self.access_key,+ aws_secret_access_key=self.api_key,region_name=self.region,) as bedrock:response = await bedrock.list_inference_profiles()@@ -43,34 +67,43 @@ class BedrockProvider(Provider):raise easync def check_api_key(self):- """Check if the Bedrock credentials are valid"""+ """Check if the Bedrock credentials are valid by listing models"""from letta.errors import LLMAuthenticationErrortry:- # For BYOK providers, use the custom credentials- if self.provider_category == ProviderCategory.byok:- # If we can list models, the credentials are valid- await self.bedrock_get_model_list_async()- else:- # For base providers, use default credentials- bedrock_get_model_list(region_name=self.region)+ # If we can list models, the credentials are valid+ await self.bedrock_get_model_list_async()except Exception as e:raise LLMAuthenticationError(message=f"Failed to authenticate with Bedrock: {e}")async def list_llm_models_async(self) -> list[LLMConfig]:models = await self.bedrock_get_model_list_async()- configs = []+ # Deduplicate models by normalized name - prefer regional (us., eu.) over global+ seen_models: dict[str, tuple[str, dict]] = {} # model_name -> (inference_profile_id, model_summary)for model_summary in models:- model_arn = model_summary["inferenceProfileArn"]+ inference_profile_id = model_summary["inferenceProfileId"]+ model_name = self.extract_anthropic_model_name(inference_profile_id)++ if model_name not in seen_models:+ seen_models[model_name] = (inference_profile_id, model_summary)+ else:+ # Prefer regional profiles over global ones+ existing_id = seen_models[model_name][0]+ if existing_id.startswith("global.") and not inference_profile_id.startswith("global."):+ seen_models[model_name] = (inference_profile_id, model_summary)++ configs = []+ for model_name, (inference_profile_id, model_summary) in seen_models.items():configs.append(LLMConfig(- model=model_arn,+ model=model_name,model_endpoint_type=self.provider_type.value,model_endpoint=None,- context_window=self.get_model_context_window(model_arn),- handle=self.get_handle(model_arn),- max_tokens=self.get_default_max_output_tokens(model_arn),+ context_window=self.get_model_context_window(inference_profile_id),+ # Store the full inference profile ID in the handle for API calls+ handle=self.get_handle(inference_profile_id),+ max_tokens=self.get_default_max_output_tokens(inference_profile_id),provider_name=self.name,provider_category=self.provider_category,)@@ -82,15 +115,19 @@ class BedrockProvider(Provider):"""Get context window size for a specific model.- Bedrock doesn't provide this via API, so we maintain a mapping- 200k for anthropic: https://aws.amazon.com/bedrock/anthropic/+ Bedrock doesn't provide this via API, so we maintain a mapping."""- if model_name.startswith("anthropic"):+ model_lower = model_name.lower()+ if "anthropic" in model_lower or "claude" in model_lower:return 200_000else:- return 100_000 # default to 100k if unknown+ return 100_000 # default if unknowndef get_handle(self, model_name: str, is_embedding: bool = False, base_name: str | None = None) -> str:- logger.debug("Getting handle for model_name: %s", model_name)- model = model_name.split(".")[-1]- return f"{self.name}/{model}"+ """+ Create handle from inference profile ID.++ Input format: us.anthropic.claude-sonnet-4-20250514-v1:0+ Output: bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0+ """+ return f"{self.name}/{model_name}"diff --git a/letta/server/server.py b/letta/server/server.pyindex 196c35f84..a021bdfca 100644--- a/letta/server/server.py+++ b/letta/server/server.py@@ -296,6 +296,8 @@ class SyncServer(object):self._enabled_providers.append(BedrockProvider(name="bedrock",+ access_key=model_settings.aws_access_key_id,+ api_key=model_settings.aws_secret_access_key,region=model_settings.aws_default_region,))diff --git a/letta/settings.py b/letta/settings.pyindex 3caead226..365b9d95d 100644--- a/letta/settings.py+++ b/letta/settings.py@@ -144,7 +144,7 @@ class ModelSettings(BaseSettings):# Bedrockaws_access_key_id: Optional[str] = Noneaws_secret_access_key: Optional[str] = None- aws_default_region: Optional[str] = None+ aws_default_region: str = "us-east-1"bedrock_anthropic_version: Optional[str] = "bedrock-2023-05-31"# anthropicdiff --git a/tests/integration_test_send_message.py b/tests/integration_test_send_message.pyindex 53df36b00..c329180ab 100644--- a/tests/integration_test_send_message.py+++ b/tests/integration_test_send_message.py@@ -229,7 +229,8 @@ def is_reasoner_model(model_handle: str, model_settings: dict) -> bool:return model.startswith("o1") or model.startswith("o3") or model.startswith("o4") or model.startswith("gpt-5")# Anthropic reasoning models (from anthropic_client.py:608-616)- elif provider_type == "anthropic":+ # Also applies to Bedrock with Anthropic models+ elif provider_type in ("anthropic", "bedrock"):return (model.startswith("claude-3-7-sonnet")or model.startswith("claude-sonnet-4")diff --git a/tests/integration_test_send_message_v2.py b/tests/integration_test_send_message_v2.pyindex e383ecc80..124b1a5bd 100644--- a/tests/integration_test_send_message_v2.py+++ b/tests/integration_test_send_message_v2.py@@ -484,8 +484,22 @@ def is_reasoner_model(model_handle: str, model_settings: dict) -> bool:)# Z.ai models output reasoning by defaultis_zai_reasoning = model_settings.get("provider_type") == "zai"+ # Bedrock Anthropic reasoning models+ is_bedrock_reasoning = model_settings.get("provider_type") == "bedrock" and (+ "claude-3-7-sonnet" in model_handle+ or "claude-sonnet-4" in model_handle+ or "claude-opus-4" in model_handle+ or "claude-haiku-4-5" in model_handle+ )- return is_openai_reasoning or is_anthropic_reasoning or is_google_vertex_reasoning or is_google_ai_reasoning or is_zai_reasoning+ return (+ is_openai_reasoning+ or is_anthropic_reasoning+ or is_google_vertex_reasoning+ or is_google_ai_reasoning+ or is_zai_reasoning+ or is_bedrock_reasoning+ )# ------------------------------@@ -653,8 +667,8 @@ async def test_parallel_tool_calls(model_handle, model_settings = model_configprovider_type = model_settings.get("provider_type", "")- if provider_type not in ["anthropic", "openai", "google_ai", "google_vertex"]:- pytest.skip("Parallel tool calling test only applies to Anthropic, OpenAI, and Gemini models.")+ if provider_type not in ["anthropic", "openai", "google_ai", "google_vertex", "bedrock"]:+ pytest.skip("Parallel tool calling test only applies to Anthropic, OpenAI, Gemini, and Bedrock models.")if "gpt-5" in model_handle or "o3" in model_handle:pytest.skip("GPT-5 takes too long to test, o3 is bad at this task.")diff --git a/tests/model_settings/bedrock-claude-4-5-opus.json b/tests/model_settings/bedrock-claude-4-5-opus.jsonnew file mode 100644index 000000000..e9d3b10cd--- /dev/null+++ b/tests/model_settings/bedrock-claude-4-5-opus.json@@ -0,0 +1,9 @@+{+ "handle": "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0",+ "model_settings": {+ "provider_type": "bedrock",+ "temperature": 1.0,+ "max_output_tokens": 16000,+ "parallel_tool_calls": false+ }+}