Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

add bedrock provider, simplify abstract class (#7707)

Shahmir Varqhac9302e5f32fd
diff --git a/marimo/_server/ai/config.py b/marimo/_server/ai/config.py
index 72e2497f32..5686b36b04 100644
--- a/marimo/_server/ai/config.py
+++ b/marimo/_server/ai/config.py
@@ -211,7 +211,7 @@ class AnyProviderConfig:
ai_config = _get_ai_config(config, "bedrock")
key = _get_key(ai_config, "Bedrock")
return cls(
- base_url=_get_base_url(ai_config),
+ base_url=_get_base_url(ai_config, "Bedrock"),
api_key=key,
tools=_get_tools(config.get("mode", "manual")),
)
diff --git a/marimo/_server/ai/providers.py b/marimo/_server/ai/providers.py
index 5ebe027256..74a9f03d09 100644
--- a/marimo/_server/ai/providers.py
+++ b/marimo/_server/ai/providers.py
@@ -49,12 +49,6 @@ LONG_THINKING_TIMEOUT = 120
if TYPE_CHECKING:
# Used for Bedrock, unified interface for all models
from anthropic.types.beta import BetaThinkingConfigParam
- from litellm import ( # type: ignore[attr-defined]
- CustomStreamWrapper as LitellmStream,
- )
- from litellm.types.utils import (
- ModelResponseStream as LitellmStreamResponse,
- )
from openai import ( # type: ignore[import-not-found]
AsyncOpenAI,
AsyncStream as OpenAiStream,
@@ -64,10 +58,16 @@ if TYPE_CHECKING:
)
from pydantic_ai import Agent, DeferredToolRequests, FunctionToolset
from pydantic_ai.messages import ThinkingPart
+ from pydantic_ai.models import Model
+ from pydantic_ai.models.bedrock import BedrockConverseModel
+ from pydantic_ai.models.google import GoogleModel
from pydantic_ai.providers import Provider
from pydantic_ai.providers.anthropic import (
AnthropicProvider as PydanticAnthropic,
)
+ from pydantic_ai.providers.bedrock import (
+ BedrockProvider as PydanticBedrock,
+ )
from pydantic_ai.providers.google import GoogleProvider as PydanticGoogle
from pydantic_ai.ui.vercel_ai import VercelAIAdapter
from pydantic_ai.ui.vercel_ai.request_types import UIMessage, UIMessagePart
@@ -148,10 +148,26 @@ class PydanticProvider(ABC, Generic[ProviderT]):
"""Create a provider for the given config."""
@abstractmethod
+ def create_model(self, max_tokens: int) -> Model:
+ """Create a Pydantic AI model for the given max tokens."""
+
def create_agent(
- self, max_tokens: int, tools: list[ToolDefinition], system_prompt: str
+ self,
+ max_tokens: int,
+ tools: list[ToolDefinition],
+ system_prompt: str,
) -> Agent[None, DeferredToolRequests | str]:
"""Create a Pydantic AI agent"""
+ from pydantic_ai import Agent
+
+ model = self.create_model(max_tokens)
+ toolset, output_type = self._get_toolsets_and_output_type(tools)
+ return Agent(
+ model,
+ toolsets=[toolset] if tools else None,
+ instructions=system_prompt,
+ output_type=output_type,
+ )
def get_vercel_adapter(self) -> type[VercelAIAdapter[Any, Any]]:
"""Return the Vercel AI adapter for the given provider."""
@@ -290,27 +306,17 @@ class GoogleProvider(PydanticProvider["PydanticGoogle"]):
provider = PydanticGoogle()
return provider
- def create_agent(
- self, max_tokens: int, tools: list[ToolDefinition], system_prompt: str
- ) -> Agent[None, DeferredToolRequests | str]:
- from pydantic_ai import Agent
+ def create_model(self, max_tokens: int) -> GoogleModel:
from pydantic_ai.models.google import GoogleModel, GoogleModelSettings
- toolset, output_type = self._get_toolsets_and_output_type(tools)
-
- return Agent(
- GoogleModel(
- model_name=self.model,
- provider=self.provider,
- settings=GoogleModelSettings(
- max_tokens=max_tokens,
- # Works on non-thinking models too
- google_thinking_config={"include_thoughts": True},
- ),
+ return GoogleModel(
+ model_name=self.model,
+ provider=self.provider,
+ settings=GoogleModelSettings(
+ max_tokens=max_tokens,
+ # Works on non-thinking models too
+ google_thinking_config={"include_thoughts": True},
),
- toolsets=[toolset] if tools else None,
- instructions=system_prompt,
- output_type=output_type,
)
@@ -953,16 +959,12 @@ class AnthropicProvider(PydanticProvider["PydanticAnthropic"]):
return PydanticAnthropic(api_key=config.api_key)
- def create_agent(
- self, max_tokens: int, tools: list[ToolDefinition], system_prompt: str
- ) -> Agent[None, DeferredToolRequests | str]:
- from pydantic_ai import Agent
+ def create_model(self, max_tokens: int) -> Model:
from pydantic_ai.models.anthropic import (
AnthropicModel,
AnthropicModelSettings,
)
- toolset, output_type = self._get_toolsets_and_output_type(tools)
is_thinking_model = self.is_extended_thinking_model(self.model)
thinking_config: BetaThinkingConfigParam = {"type": "disabled"}
if is_thinking_model:
@@ -971,19 +973,14 @@ class AnthropicProvider(PydanticProvider["PydanticAnthropic"]):
"budget_tokens": self.DEFAULT_EXTENDED_THINKING_BUDGET_TOKENS,
}
- return Agent(
- AnthropicModel(
- model_name=self.model,
- provider=self.provider,
- settings=AnthropicModelSettings(
- max_tokens=max_tokens,
- temperature=self.get_temperature(),
- anthropic_thinking=thinking_config,
- ),
+ return AnthropicModel(
+ model_name=self.model,
+ provider=self.provider,
+ settings=AnthropicModelSettings(
+ max_tokens=max_tokens,
+ temperature=self.get_temperature(),
+ anthropic_thinking=thinking_config,
),
- toolsets=[toolset] if tools else None,
- instructions=system_prompt,
- output_type=output_type,
)
def is_extended_thinking_model(self, model: str) -> bool:
@@ -1088,12 +1085,7 @@ class AnthropicProvider(PydanticProvider["PydanticAnthropic"]):
return AnthropicVercelAIAdapter
-class BedrockProvider(
- CompletionProvider[
- "LitellmStreamResponse",
- "LitellmStream",
- ]
-):
+class BedrockProvider(PydanticProvider["PydanticBedrock"]):
def setup_credentials(self, config: AnyProviderConfig) -> None:
# Use profile name if provided, otherwise use API key
try:
@@ -1113,109 +1105,29 @@ class BedrockProvider(
detail="Error setting up AWS credentials",
) from e
- async def stream_completion(
- self,
- messages: list[ChatMessage],
- system_prompt: str,
- max_tokens: int,
- additional_tools: list[ToolDefinition],
- ) -> LitellmStream:
- DependencyManager.litellm.require(why="for AI assistance with Bedrock")
- DependencyManager.boto3.require(why="for AI assistance with Bedrock")
- from litellm import acompletion as litellm_completion
-
- self.setup_credentials(self.config)
- tools = self.config.tools
-
- config = {
- "model": self.model,
- "messages": cast(
- Any,
- convert_to_openai_messages(
- [ChatMessage(role="system", content=system_prompt)]
- + messages
- ),
- ),
- "max_completion_tokens": max_tokens,
- "stream": True,
- "timeout": TIMEOUT,
- }
- if tools:
- all_tools = tools + additional_tools
- config["tools"] = convert_to_openai_tools(all_tools)
-
- return await litellm_completion(**config)
-
- def extract_content(
- self,
- response: LitellmStreamResponse,
- tool_call_ids: Optional[list[str]] = None,
- ) -> Optional[ExtractedContentList]:
- tool_call_ids = tool_call_ids or []
- if (
- hasattr(response, "choices")
- and response.choices
- and response.choices[0].delta
- ):
- delta = response.choices[0].delta
-
- # Text content
- content = delta.content
- if content:
- return [(str(content), "text")]
-
- # Tool call: LiteLLM follows OpenAI format for tool calls
-
- if hasattr(delta, "tool_calls") and delta.tool_calls:
- tool_content: ExtractedContentList = []
-
- for tool_call in delta.tool_calls:
- tool_index: int = tool_call.index
-
- # Start of tool call
- # id is only present for the first tool call chunk
- if hasattr(tool_call, "id") and tool_call.id:
- tool_info = {
- "toolCallId": tool_call.id,
- "toolName": tool_call.function.name,
- }
- tool_content.append((tool_info, "tool_call_start"))
-
- # Delta of tool call
- # arguments is only present second chunk onwards
- if (
- hasattr(tool_call, "function")
- and tool_call.function
- and hasattr(tool_call.function, "arguments")
- and tool_call.function.arguments
- and tool_index < len(tool_call_ids)
- and tool_call_ids[tool_index]
- ):
- tool_delta = {
- "toolCallId": tool_call_ids[tool_index],
- "inputTextDelta": tool_call.function.arguments,
- }
- tool_content.append((tool_delta, "tool_call_delta"))
+ def create_provider(self, config: AnyProviderConfig) -> PydanticBedrock:
+ from pydantic_ai.providers.bedrock import (
+ BedrockProvider as PydanticBedrock,
+ )
- # return the tool content
- return tool_content
+ self.setup_credentials(config)
+ # For bedrock, the config sets the region name as the base_url
+ return PydanticBedrock(region_name=config.base_url)
- return None
+ def create_model(self, max_tokens: int) -> BedrockConverseModel:
+ from pydantic_ai.models.bedrock import (
+ BedrockConverseModel,
+ BedrockModelSettings,
+ )
- def get_finish_reason(
- self, response: LitellmStreamResponse
- ) -> Optional[FinishReason]:
- if (
- hasattr(response, "choices")
- and response.choices
- and response.choices[0].finish_reason
- ):
- return (
- "tool_calls"
- if response.choices[0].finish_reason == "tool_calls"
- else "stop"
- )
- return None
+ return BedrockConverseModel(
+ model_name=self.model,
+ provider=self.provider,
+ settings=BedrockModelSettings(
+ max_tokens=max_tokens,
+ # TODO: Add reasoning support
+ ),
+ )
def get_completion_provider(
@@ -1232,7 +1144,9 @@ def get_completion_provider(
model_id.model, config, [DependencyManager.google_ai]
)
elif model_id.provider == "bedrock":
- return BedrockProvider(model_id.model, config)
+ return BedrockProvider(
+ model_id.model, config, [DependencyManager.boto3]
+ )
elif model_id.provider == "azure":
return AzureOpenAIProvider(model_id.model, config)
elif model_id.provider == "openrouter":
diff --git a/pixi.lock b/pixi.lock
index a54a1bd6fc..44423a7696 100644
--- a/pixi.lock
+++ b/pixi.lock
@@ -115,7 +115,7 @@ environments:
- pypi: https://files.pythonhosted.org/packages/b2/70/00dc4246d9f3c69ecbb9bc36d5ad1a359884464a44711c665cb0afb1e9de/loro-1.10.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
- pypi: https://files.pythonhosted.org/packages/70/81/54e3ce63502cd085a0c556652a4e1b919c45a446bd1e5300e10c44c8c521/markdown-3.10-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/5c/a2/488517a43ccf5a4b6b6eca6dd4ede0bd82b043d1539dd6bb908a19f8efd3/msgspec-0.20.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl
- - pypi: https://files.pythonhosted.org/packages/79/3e/b8ecc67e178919671695f64374a7ba916cf0adbf86efedc6054f38b5b8ae/narwhals-2.14.0-py3-none-any.whl
+ - pypi: https://files.pythonhosted.org/packages/3d/2e/cf2ffeb386ac3763526151163ad7da9f1b586aac96d2b4f7de1eaebf0c61/narwhals-2.15.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/16/32/f8e3c85d1d5250232a5d3477a2a28cc291968ff175caeadaf3cc19ce0e4a/parso-0.8.5-py2.py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/29/f8/40e01c350ad9a2b3cb4e6adbcc8a83b17ee50dd5792102b6142385937db5/psutil-7.2.1-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl
- pypi: https://files.pythonhosted.org/packages/ea/10/47caf89cbb52e5bb764696fd52a8c591a2f0e851a93270c05a17f36000b5/pymdown_extensions-10.20-py3-none-any.whl
@@ -213,7 +213,7 @@ environments:
- pypi: https://files.pythonhosted.org/packages/78/e4/7b614260bf16c5e33c0bea6ac47ab0284efd21f89f2e5e4e15cd93bead40/loro-1.10.3-cp312-cp312-macosx_11_0_arm64.whl
- pypi: https://files.pythonhosted.org/packages/70/81/54e3ce63502cd085a0c556652a4e1b919c45a446bd1e5300e10c44c8c521/markdown-3.10-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/7f/ee/af51d090ada641d4b264992a486435ba3ef5b5634bc27e6eb002f71cef7d/msgspec-0.20.0-cp312-cp312-macosx_11_0_arm64.whl
- - pypi: https://files.pythonhosted.org/packages/79/3e/b8ecc67e178919671695f64374a7ba916cf0adbf86efedc6054f38b5b8ae/narwhals-2.14.0-py3-none-any.whl
+ - pypi: https://files.pythonhosted.org/packages/3d/2e/cf2ffeb386ac3763526151163ad7da9f1b586aac96d2b4f7de1eaebf0c61/narwhals-2.15.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/16/32/f8e3c85d1d5250232a5d3477a2a28cc291968ff175caeadaf3cc19ce0e4a/parso-0.8.5-py2.py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/c5/2c/78e4a789306a92ade5000da4f5de3255202c534acdadc3aac7b5458fadef/psutil-7.2.1-cp36-abi3-macosx_11_0_arm64.whl
- pypi: https://files.pythonhosted.org/packages/ea/10/47caf89cbb52e5bb764696fd52a8c591a2f0e851a93270c05a17f36000b5/pymdown_extensions-10.20-py3-none-any.whl
@@ -1133,7 +1133,7 @@ packages:
- pypi: ./
name: marimo
version: 0.18.4
- sha256: 1db0ec3fd3a5aaad887dcd9c1fe56a2791af8a584a5f266799cf9a3dd50f7a78
+ sha256: 64534d3c58bd4c81fc84e6f6b7afb039e1d1eee5b8e3ae047b9421d3be87363a
requires_dist:
- click>=8.0,<9
- jedi>=0.18.0
@@ -1166,6 +1166,7 @@ packages:
- mcp>=1.0.0 ; extra == 'mcp'
- pydantic>2 ; extra == 'mcp'
requires_python: '>=3.10'
+ editable: true
- pypi: https://files.pythonhosted.org/packages/70/81/54e3ce63502cd085a0c556652a4e1b919c45a446bd1e5300e10c44c8c521/markdown-3.10-py3-none-any.whl
name: markdown
version: '3.10'
@@ -1234,10 +1235,10 @@ packages:
- tomli-w ; extra == 'toml'
- pyyaml ; extra == 'yaml'
requires_python: '>=3.9'
-- pypi: https://files.pythonhosted.org/packages/79/3e/b8ecc67e178919671695f64374a7ba916cf0adbf86efedc6054f38b5b8ae/narwhals-2.14.0-py3-none-any.whl
+- pypi: https://files.pythonhosted.org/packages/3d/2e/cf2ffeb386ac3763526151163ad7da9f1b586aac96d2b4f7de1eaebf0c61/narwhals-2.15.0-py3-none-any.whl
name: narwhals
- version: 2.14.0
- sha256: b56796c9a00179bd757d15282c540024e1d5c910b19b8c9944d836566c030acf
+ version: 2.15.0
+ sha256: cbfe21ca19d260d9fd67f995ec75c44592d1f106933b03ddd375df7ac841f9d6
requires_dist:
- cudf>=24.10.0 ; extra == 'cudf'
- dask[dataframe]>=2024.8 ; extra == 'dask'
diff --git a/pyproject.toml b/pyproject.toml
index 071f0cdf21..ab8b50005f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -183,7 +183,7 @@ dependencies = [
"matplotlib>=3.8.0",
"sqlglot[rs]>=26.2.0",
"sqlalchemy>=2.0.40",
- "pydantic-ai-slim[google,anthropic]>=1.39.0",
+ "pydantic-ai-slim[google,anthropic,bedrock]>=1.39.0",
"openai>=1.55.3",
"loro>=1.5.0",
"pandas-stubs>=1.5.3.230321",
@@ -260,10 +260,11 @@ extra-dependencies = [
"ipython~=8.12.3",
# testing gen ai
"openai>=1.55.3",
- "pydantic-ai-slim[google,anthropic]>=1.39.0",
+ "pydantic-ai-slim[google,anthropic,bedrock]>=1.39.0",
# - google-auth uses cachetools, and cachetools<5.0.0 uses collections.MutableMapping (removed in Python 3.10)
"cachetools>=5.0.0",
"boto3>=1.38.46",
+ # for bedrock ui chat
"litellm>=1.70.0",
# exporting as ipynb
"nbformat>=5.10.4",
diff --git a/tests/_server/ai/test_ai_config.py b/tests/_server/ai/test_ai_config.py
index 255814c21c..ee428dad12 100644
--- a/tests/_server/ai/test_ai_config.py
+++ b/tests/_server/ai/test_ai_config.py
@@ -359,8 +359,7 @@ class TestAnyProviderConfig:
provider_config = AnyProviderConfig.for_bedrock(config)
assert provider_config.api_key == "profile:test-profile"
- # Note: base_url is None because _get_base_url doesn't get "Bedrock" name parameter
- assert provider_config.base_url is None
+ assert provider_config.base_url == "us-east-1"
def test_for_bedrock_with_credentials(self):
"""Test Bedrock configuration with AWS credentials."""
@@ -375,8 +374,7 @@ class TestAnyProviderConfig:
provider_config = AnyProviderConfig.for_bedrock(config)
assert provider_config.api_key == "test-access-key:test-secret-key"
- # Note: base_url is None because _get_base_url doesn't get "Bedrock" name parameter
- assert provider_config.base_url is None
+ assert provider_config.base_url == "us-west-2"
def test_for_model_openai(self) -> None:
"""Test for_model with OpenAI model."""
diff --git a/tests/_server/ai/test_providers.py b/tests/_server/ai/test_providers.py
index b625e0ec6a..1da50f1c92 100644
--- a/tests/_server/ai/test_providers.py
+++ b/tests/_server/ai/test_providers.py
@@ -79,7 +79,7 @@ def test_anyprovider_for_model(model_name: str, provider_name: str) -> None:
pytest.param(
"bedrock/anthropic.claude-3-sonnet-20240229",
BedrockProvider,
- None,
+ DependencyManager.boto3,
id="bedrock",
),
pytest.param(
@@ -95,7 +95,16 @@ def test_get_completion_provider(
if dependency and not dependency.has():
pytest.skip(f"{dependency.pkg} is not installed")
- config = AnyProviderConfig(api_key="test-key", base_url="http://test-url")
+ if provider_type == BedrockProvider:
+ # For Bedrock, we pass bedrock-required details through the config
+ config = AnyProviderConfig(
+ api_key="aws_access_key_id:aws_secret_access_key", # credentials
+ base_url="us-east-1", # region name
+ )
+ else:
+ config = AnyProviderConfig(
+ api_key="test-key", base_url="http://test-url"
+ )
provider = get_completion_provider(config, model_name)
assert isinstance(provider, provider_type)
@@ -191,32 +200,6 @@ async def test_azure_openai_provider() -> None:
assert endpoint == "https://unknown_domain.openai"
- "provider_type",
- [
- pytest.param(OpenAIProvider, id="openai"),
- pytest.param(BedrockProvider, id="bedrock"),
- ],
-)
-def test_extract_content_with_none_tool_call_ids(
- provider_type: type,
-) -> None:
- """Test extract_content handles None tool_call_ids without errors."""
- config = AnyProviderConfig(api_key="test-key", base_url="http://test")
- provider = provider_type("test-model", config)
-
- mock_response = MagicMock()
- mock_delta = MagicMock()
- mock_delta.content = "Hello"
- mock_delta.tool_calls = None
- mock_choice = MagicMock()
- mock_choice.delta = mock_delta
- mock_response.choices = [mock_choice]
-
- result = provider.extract_content(mock_response, None)
- assert result == [("Hello", "text")]
-
-
def test_openai_extract_content_multiple_tool_calls() -> None:
"""Test OpenAI extracts multiple tool calls correctly."""
config = AnyProviderConfig(api_key="test-key", base_url="http://test")
@@ -502,35 +485,3 @@ def test_openai_extract_content_tool_delta_out_of_bounds() -> None:
result = provider.extract_content(mock_response, [])
# Should return None or empty list since we can't process the delta
assert result is None or result == []
-
-
-def test_bedrock_extract_content_tool_delta_out_of_bounds() -> None:
- """Test Bedrock handles tool call delta with out-of-bounds index gracefully.
-
- This reproduces the issue reported in #7330 where some providers may send
- tool call deltas before the tool call start, causing an IndexError.
- """
- config = AnyProviderConfig(api_key="test-key", base_url="http://test")
- provider = BedrockProvider("bedrock/anthropic.claude-3-sonnet", config)
-
- mock_response = MagicMock()
- mock_delta = MagicMock()
- mock_delta.content = None
-
- # Create a tool call delta with index 0, but tool_call_ids is empty
- mock_tool_delta = MagicMock()
- mock_tool_delta.index = 0
- mock_tool_delta.id = None # No id for delta chunks
- mock_tool_delta.function = MagicMock()
- mock_tool_delta.function.name = None # No name for delta chunks
- mock_tool_delta.function.arguments = '{"location": "SF"}'
-
- mock_delta.tool_calls = [mock_tool_delta]
- mock_choice = MagicMock()
- mock_choice.delta = mock_delta
- mock_response.choices = [mock_choice]
-
- # Call with empty tool_call_ids - this should not crash
- result = provider.extract_content(mock_response, [])
- # Should return None or empty list since we can't process the delta
- assert result is None or result == []