Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: remove litellm (#8852)

Dylan Madisetti270c1b701e2d
diff --git a/examples/ai/chat/bedrock_example.py b/examples/ai/chat/bedrock_example.py
index 299d8e6cf6..7170132ced 100644
--- a/examples/ai/chat/bedrock_example.py
+++ b/examples/ai/chat/bedrock_example.py
@@ -2,7 +2,6 @@
# requires-python = ">=3.11"
# dependencies = [
# "boto3",
-# "litellm",
# "marimo",
# ]
# ///
diff --git a/marimo/_ai/llm/_impl.py b/marimo/_ai/llm/_impl.py
index 2405a569a4..f036716304 100644
--- a/marimo/_ai/llm/_impl.py
+++ b/marimo/_ai/llm/_impl.py
@@ -561,100 +561,104 @@ class bedrock(ChatModel):
system_message: str = DEFAULT_SYSTEM_MESSAGE,
region_name: str = "us-east-1",
profile_name: Optional[str] = None,
+ credentials: Optional[dict[str, str]] = None,
aws_access_key_id: Optional[str] = None,
aws_secret_access_key: Optional[str] = None,
):
if not model.startswith("bedrock/"):
model = f"bedrock/{model}"
self.model = model
+ self._model_id = model[len("bedrock/") :]
self.system_message = system_message
self.region_name = region_name
self.profile_name = profile_name
+ if credentials:
+ aws_access_key_id = credentials.get(
+ "aws_access_key_id", aws_access_key_id
+ )
+ aws_secret_access_key = credentials.get(
+ "aws_secret_access_key", aws_secret_access_key
+ )
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
- def _setup_credentials(self) -> None:
- # Use profile name if provided, otherwise use API key
+ def _create_client(self) -> Any:
+ DependencyManager.boto3.require(
+ "bedrock chat model requires boto3. `pip install boto3`"
+ )
+ import boto3
+
+ session_kwargs: dict[str, Any] = {}
if self.profile_name:
- os.environ["AWS_PROFILE"] = self.profile_name
- elif self.aws_access_key_id and self.aws_secret_access_key:
- os.environ["AWS_ACCESS_KEY_ID"] = self.aws_access_key_id
- os.environ["AWS_SECRET_ACCESS_KEY"] = self.aws_secret_access_key
- else:
- pass # Use default credential chain
+ session_kwargs["profile_name"] = self.profile_name
+ if self.aws_access_key_id and self.aws_secret_access_key:
+ session_kwargs["aws_access_key_id"] = self.aws_access_key_id
+ session_kwargs["aws_secret_access_key"] = (
+ self.aws_secret_access_key
+ )
- def _stream_response(
+ session = boto3.Session(**session_kwargs)
+ return session.client("bedrock-runtime", region_name=self.region_name)
+
+ def _build_converse_params(
self, messages: list[ChatMessage], config: ChatModelConfig
- ) -> Generator[str, None, None]:
- """Helper method for streaming - yields delta chunks.
+ ) -> dict[str, Any]:
+ converse_messages: list[dict[str, Any]] = []
+ for msg in messages:
+ content = str(msg.content) if msg.content else ""
+ converse_messages.append(
+ {"role": msg.role, "content": [{"text": content}]}
+ )
- Each yield is a new piece of content (delta) to be accumulated
- by the consumer. This follows the standard AWS Bedrock streaming pattern.
- """
- from litellm import completion as litellm_completion
+ params: dict[str, Any] = {
+ "modelId": self._model_id,
+ "messages": converse_messages,
+ "system": [{"text": self.system_message}],
+ }
- response = litellm_completion(
- model=self.model,
- messages=convert_to_openai_messages(
- [ChatMessage(role="system", content=self.system_message)]
- + messages
- ),
- max_tokens=config.max_tokens,
- temperature=config.temperature,
- top_p=config.top_p,
- frequency_penalty=config.frequency_penalty,
- presence_penalty=config.presence_penalty,
- stream=True,
- )
+ inference_config: dict[str, Any] = {}
+ if config.max_tokens is not None:
+ inference_config["maxTokens"] = config.max_tokens
+ if config.temperature is not None:
+ inference_config["temperature"] = config.temperature
+ if config.top_p is not None:
+ inference_config["topP"] = config.top_p
+ if inference_config:
+ params["inferenceConfig"] = inference_config
- for chunk in response:
- if chunk.choices and len(chunk.choices) > 0:
- delta = chunk.choices[0].delta
- if delta.content:
- yield delta.content
+ return params
+
+ def _stream_response(
+ self, client: Any, params: dict[str, Any]
+ ) -> Generator[str, None, None]:
+ """Stream response using the Bedrock Converse Stream API."""
+ response = client.converse_stream(**params)
+ for event in response["stream"]:
+ if "contentBlockDelta" in event:
+ delta = event["contentBlockDelta"].get("delta", {})
+ if "text" in delta:
+ yield delta["text"]
def __call__(
self, messages: list[ChatMessage], config: ChatModelConfig
) -> object:
- DependencyManager.boto3.require(
- "bedrock chat model requires boto3. `pip install boto3`"
- )
- DependencyManager.litellm.require(
- "bedrock chat model requires litellm. `pip install litellm`"
- )
- self._setup_credentials()
+ client = self._create_client()
+ params = self._build_converse_params(messages, config)
try:
- # Try streaming first, fall back to non-streaming on error
try:
- return self._stream_response(messages, config)
+ return self._stream_response(client, params)
except Exception as stream_error:
- # Fall back to non-streaming if streaming fails
if _looks_like_streaming_error(stream_error):
- from litellm import completion as litellm_completion
-
- response = litellm_completion(
- model=self.model,
- messages=convert_to_openai_messages(
- [
- ChatMessage(
- role="system",
- content=self.system_message,
- )
- ]
- + messages
- ),
- max_tokens=config.max_tokens,
- temperature=config.temperature,
- top_p=config.top_p,
- frequency_penalty=config.frequency_penalty,
- presence_penalty=config.presence_penalty,
- stream=False,
- )
- return response.choices[0].message.content or ""
+ response = client.converse(**params)
+ output = response.get("output", {})
+ message = output.get("message", {})
+ content = message.get("content", [])
+ if content and "text" in content[0]:
+ return content[0]["text"]
+ return ""
raise
except Exception as e:
- # Handle common AWS exceptions with helpful messages
error_msg = str(e)
if "AccessDenied" in error_msg:
@@ -676,7 +680,6 @@ class bedrock(ChatModel):
"Check that the model ID is correct and available in this region."
) from e
else:
- # Re-raise original exception if not handled
raise
diff --git a/marimo/_dependencies/dependencies.py b/marimo/_dependencies/dependencies.py
index a3637b633a..273c2a4134 100644
--- a/marimo/_dependencies/dependencies.py
+++ b/marimo/_dependencies/dependencies.py
@@ -246,7 +246,7 @@ class DependencyManager:
tomlkit = Dependency("tomlkit")
loro = Dependency("loro")
boto3 = Dependency("boto3")
- litellm = Dependency("litellm")
+
redshift_connector = Dependency("redshift_connector")
mcp = Dependency("mcp")
pydantic_ai = Dependency(
diff --git a/pyproject.toml b/pyproject.toml
index 42a85bb1f4..7cecdb542e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -196,7 +196,6 @@ test-optional = [
"cachetools>=5.0.0",
"boto3>=1.38.46",
# for bedrock ui chat
- "litellm>=1.70.0,<=1.82.6",
# exporting as ipynb
"nbformat>=5.10.4",
"nbconvert>=7.17.0",
@@ -231,9 +230,9 @@ typecheck = [
"fsspec>=2026.2.0",
"pydantic-ai-slim[google,anthropic,bedrock,openai]>=1.52.0",
"loro>=1.5.0",
+ "boto3-stubs>=1.38.46",
"pandas-stubs>=1.5.3.230321",
"pyiceberg>=0.9.0",
- "litellm>=1.70.0,<=1.82.6",
"python-dotenv>=1.0.1",
"jupytext>=1.17.2",
"types-Pillow>=10.2.0.20240520",
diff --git a/tests/_ai/llm/test_impl.py b/tests/_ai/llm/test_impl.py
index 1a4cbdee22..89e822ccb3 100644
--- a/tests/_ai/llm/test_impl.py
+++ b/tests/_ai/llm/test_impl.py
@@ -94,19 +94,28 @@ def mock_google_client():
@pytest.fixture
-def mock_litellm_completion():
- """Fixture for mocking the litellm completion function."""
- with patch("litellm.completion") as mock_litellm_completion:
- # Setup the response structure
- mock_response = MagicMock()
- mock_choice = MagicMock()
- mock_message = MagicMock()
- mock_message.content = "Test response"
- mock_choice.message = mock_message
- mock_response.choices = [mock_choice]
- mock_litellm_completion.return_value = mock_response
+def mock_bedrock_client():
+ """Fixture for mocking the boto3 bedrock-runtime client."""
+ with patch("boto3.Session") as mock_session_class:
+ mock_session = MagicMock()
+ mock_session_class.return_value = mock_session
+
+ mock_client = MagicMock()
+ mock_session.client.return_value = mock_client
- yield mock_litellm_completion
+ # Setup converse response
+ mock_client.converse.return_value = {
+ "output": {"message": {"content": [{"text": "Test response"}]}}
+ }
+
+ # Setup converse_stream response
+ mock_client.converse_stream.return_value = {
+ "stream": [
+ {"contentBlockDelta": {"delta": {"text": "Test response"}}}
+ ]
+ }
+
+ yield mock_client
@pytest.fixture
@@ -1009,8 +1018,8 @@ class TestAnthropic:
@pytest.mark.skipif(
- not DependencyManager.boto3.has() or not DependencyManager.litellm.has(),
- reason="boto3 or litellm is not installed",
+ not DependencyManager.boto3.has(),
+ reason="boto3 is not installed",
)
class TestBedrock:
"""Test the Bedrock model class"""
@@ -1023,7 +1032,6 @@ class TestBedrock:
region_name="us-east-1",
)
- # bedrock automatically prefixes with bedrock/ for litellm usage
assert model.model == "bedrock/anthropic.claude-3-sonnet-20240229"
assert model.system_message == "Test system message"
assert model.region_name == "us-east-1"
@@ -1031,6 +1039,11 @@ class TestBedrock:
assert model.aws_access_key_id is None
assert model.aws_secret_access_key is None
+ def test_init_with_bedrock_prefix(self):
+ """Test that the bedrock/ prefix is accepted and preserved"""
+ model = bedrock("bedrock/anthropic.claude-3-sonnet-20240229")
+ assert model.model == "bedrock/anthropic.claude-3-sonnet-20240229"
+
def test_init_with_credentials(self):
"""Test initialization with explicit credentials"""
model = bedrock(
@@ -1051,36 +1064,28 @@ class TestBedrock:
assert model.profile_name == "test-profile"
- @pytest.mark.xfail(
- reason="latest litellm and openai are not compatible",
- )
- def test_call(self, mock_litellm_completion, test_messages, test_config):
- """Test calling the bedrock class with LiteLLM client."""
+ def test_call(self, mock_bedrock_client, test_messages, test_config):
+ """Test calling the bedrock class with boto3 Converse API."""
model_name = "anthropic.claude-3-sonnet-20240229"
- # Create model with API key to avoid _require_api_key
model = bedrock(model_name)
- result = model(test_messages, test_config)
+ result = list(model(test_messages, test_config))
- # Verify result
- assert result == "Test response"
+ assert result == ["Test response"]
- # Verify API call
- mock_litellm_completion.assert_called_once()
- call_args = mock_litellm_completion.call_args[1]
- assert call_args["model"] == f"bedrock/{model_name}"
- assert len(call_args["messages"]) == 2
- assert call_args["messages"][0]["role"] == "system"
- assert call_args["messages"][0]["content"] == DEFAULT_SYSTEM_MESSAGE
- assert call_args["messages"][1]["role"] == "user"
- assert call_args["messages"][1]["content"] == "Test prompt"
- assert call_args["max_tokens"] == 100
- # Use pytest.approx for floating point comparisons
- assert call_args["temperature"] == pytest.approx(0.7)
- assert call_args["top_p"] == pytest.approx(0.9)
- assert call_args["frequency_penalty"] == pytest.approx(0.5)
- assert call_args["presence_penalty"] == pytest.approx(0.5)
+ mock_bedrock_client.converse_stream.assert_called_once()
+ call_args = mock_bedrock_client.converse_stream.call_args[1]
+ assert call_args["modelId"] == model_name
+ assert call_args["system"] == [{"text": DEFAULT_SYSTEM_MESSAGE}]
+ assert len(call_args["messages"]) == 1
+ assert call_args["messages"][0]["role"] == "user"
+ assert call_args["messages"][0]["content"] == [{"text": "Test prompt"}]
+ assert call_args["inferenceConfig"]["maxTokens"] == 100
+ assert call_args["inferenceConfig"]["temperature"] == pytest.approx(
+ 0.7
+ )
+ assert call_args["inferenceConfig"]["topP"] == pytest.approx(0.9)
@pytest.mark.skipif(