Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

api: remove stream argument for built-in AI models (#7222)

Akshay Agrawala58a1b82c80c
diff --git a/docs/api/inputs/chat.md b/docs/api/inputs/chat.md
index 344d953c40..997f535264 100644
--- a/docs/api/inputs/chat.md
+++ b/docs/api/inputs/chat.md
@@ -137,31 +137,7 @@ mo.ui.chat(
Chatbots can stream responses in real-time, creating a more interactive experience
similar to ChatGPT where you see the response appear word-by-word as it's generated.
-### With Built-in Models
-
-For built-in models (OpenAI, Anthropic, Google, Groq, Bedrock), set `stream=True` in the model constructor:
-
-```python
-import marimo as mo
-
-chat = mo.ui.chat(
- mo.ai.llm.openai(
- "gpt-4o",
- system_message="You are a helpful assistant.",
- stream=True, # Enable streaming
- ),
- show_configuration_controls=True
-)
-chat
-```
-
-This works for all built-in models:
-
-- `mo.ai.llm.openai("gpt-4o", stream=True)`
-- `mo.ai.llm.anthropic("claude-3-5-sonnet-20240620", stream=True)`
-- `mo.ai.llm.google("gemini-1.5-pro-latest", stream=True)`
-- `mo.ai.llm.groq("llama-3.1-70b-versatile", stream=True)`
-- `mo.ai.llm.bedrock("anthropic.claude-3-7-sonnet-20250219-v1:0", stream=True)`
+Responses from built-in models (OpenAI, Anthropic, Google, Groq, Bedrock) are streamed by default.
### With Custom Models
diff --git a/examples/ai/chat/anthropic_example.py b/examples/ai/chat/anthropic_example.py
index 460d066f99..d738feb24d 100644
--- a/examples/ai/chat/anthropic_example.py
+++ b/examples/ai/chat/anthropic_example.py
@@ -56,7 +56,6 @@ def _(key, mo):
"claude-3-7-sonnet-20250219",
system_message="You are a helpful assistant.",
api_key=key,
- # stream=True, # Uncomment to enable streaming responses
),
allow_attachments=[
"image/png",
diff --git a/examples/ai/chat/bedrock_example.py b/examples/ai/chat/bedrock_example.py
index c65575d452..acc1a54870 100644
--- a/examples/ai/chat/bedrock_example.py
+++ b/examples/ai/chat/bedrock_example.py
@@ -192,7 +192,6 @@ def _(config_form, max_tokens, mo, temperature):
"model": model,
"region_name": region,
"system_message": system_message,
- # "stream": True, # Uncomment to enable streaming responses
}
# Add profile if specified
diff --git a/examples/ai/chat/gemini.py b/examples/ai/chat/gemini.py
index bf758a1cbb..f23df4ef72 100644
--- a/examples/ai/chat/gemini.py
+++ b/examples/ai/chat/gemini.py
@@ -56,7 +56,6 @@ def _(key, mo):
"gemini-2.5-flash",
system_message="You are a helpful assistant.",
api_key=key,
- # stream=True, # Uncomment to enable streaming responses
),
prompts=[
"Hello",
diff --git a/examples/ai/chat/groq_example.py b/examples/ai/chat/groq_example.py
index ae10e553bc..b030f4fcc3 100644
--- a/examples/ai/chat/groq_example.py
+++ b/examples/ai/chat/groq_example.py
@@ -73,7 +73,6 @@ def _(key, mo):
model="llama-3.1-70b-versatile",
system_message="You are a helpful assistant.",
api_key=key,
- # stream=True, # Uncomment to enable streaming responses
),
prompts=[
"Hello",
diff --git a/examples/ai/chat/openai_example.py b/examples/ai/chat/openai_example.py
index 0e75e45dba..8556a126b0 100644
--- a/examples/ai/chat/openai_example.py
+++ b/examples/ai/chat/openai_example.py
@@ -56,7 +56,6 @@ def _(mo, openai_key):
"gpt-4o",
system_message="You are a helpful assistant.",
api_key=openai_key,
- # stream=True, # Uncomment to enable streaming responses
),
prompts=[
"Hello",
diff --git a/examples/ai/chat/streaming_openai.py b/examples/ai/chat/streaming_openai.py
index 4c19223c68..ea2c202f23 100644
--- a/examples/ai/chat/streaming_openai.py
+++ b/examples/ai/chat/streaming_openai.py
@@ -51,7 +51,6 @@ def _(api_key_input, mo):
"gpt-4o-mini",
system_message="You are a helpful assistant. Keep responses concise and friendly.",
api_key=api_key_input.value,
- stream=True, # Enable streaming
),
prompts=[
"Tell me a short joke",
@@ -84,13 +83,10 @@ def _(mo):
mo.ai.llm.openai(
"gpt-4o-mini",
api_key="your-key",
- stream=True, # Enable streaming!
)
)
```
- Set `stream=True` to enable streaming responses. 🚀
-
Other built-in models (`anthropic`, `google`, `groq`) work the same way.
""")
return
diff --git a/marimo/_ai/llm/_impl.py b/marimo/_ai/llm/_impl.py
index d3e77ffae6..c04bdd42da 100644
--- a/marimo/_ai/llm/_impl.py
+++ b/marimo/_ai/llm/_impl.py
@@ -58,10 +58,6 @@ class openai(ChatModel):
If not provided, the API key will be retrieved
from the OPENAI_API_KEY environment variable or the user's config.
base_url: The base URL to use
- stream: Whether to stream the response token-by-token as it's generated.
- When True, the response appears progressively (like ChatGPT), creating
- a more responsive user experience. When False, the complete response
- is returned all at once. Defaults to False for backward compatibility.
"""
def __init__(
@@ -71,13 +67,11 @@ class openai(ChatModel):
system_message: str = DEFAULT_SYSTEM_MESSAGE,
api_key: Optional[str] = None,
base_url: Optional[str] = None,
- stream: bool = False,
):
self.model = model
self.system_message = system_message
self.api_key = api_key
self.base_url = base_url
- self.stream = stream
@property
def _require_api_key(self) -> str:
@@ -168,17 +162,10 @@ class openai(ChatModel):
top_p=config.top_p,
frequency_penalty=config.frequency_penalty,
presence_penalty=config.presence_penalty,
- stream=self.stream,
+ stream=True,
)
- if self.stream:
- # Return generator for streaming
- return self._stream_response(response)
- else:
- # Non-streaming response - return string directly
- choice = cast(Any, response).choices[0]
- content = choice.message.content
- return content or ""
+ return self._stream_response(response)
class anthropic(ChatModel):
@@ -194,10 +181,6 @@ class anthropic(ChatModel):
from the ANTHROPIC_API_KEY environment variable
or the user's config.
base_url: The base URL to use
- stream: Whether to stream the response token-by-token as it's generated.
- When True, the response appears progressively (like ChatGPT), creating
- a more responsive user experience. When False, the complete response
- is returned all at once. Defaults to False for backward compatibility.
"""
def __init__(
@@ -207,13 +190,11 @@ class anthropic(ChatModel):
system_message: str = DEFAULT_SYSTEM_MESSAGE,
api_key: Optional[str] = None,
base_url: Optional[str] = None,
- stream: bool = False,
):
self.model = model
self.system_message = system_message
self.api_key = api_key
self.base_url = base_url
- self.stream = stream
@property
def _require_api_key(self) -> str:
@@ -273,7 +254,7 @@ class anthropic(ChatModel):
"system": self.system_message,
"max_tokens": config.max_tokens or 4096,
"messages": anthropic_messages,
- "stream": self.stream,
+ "stream": True,
}
if config.top_p is not None:
params["top_p"] = config.top_p
@@ -282,23 +263,7 @@ class anthropic(ChatModel):
if config.temperature is not None:
params["temperature"] = config.temperature
- if self.stream:
- # Return generator for streaming
- return self._stream_response(client, params)
- else:
- # Non-streaming response - return directly
- response = client.messages.create(**params)
-
- content = response.content
- if len(content) > 0:
- if content[0].type == "text":
- return content[0].text
- elif content[0].type == "tool_use":
- return content
- else:
- return ""
- else:
- return ""
+ return self._stream_response(client, params)
class google(ChatModel):
@@ -313,10 +278,6 @@ class google(ChatModel):
If not provided, the API key will be retrieved
from the GOOGLE_AI_API_KEY environment variable
or the user's config.
- stream: Whether to stream the response token-by-token as it's generated.
- When True, the response appears progressively (like ChatGPT), creating
- a more responsive user experience. When False, the complete response
- is returned all at once. Defaults to False for backward compatibility.
"""
def __init__(
@@ -325,12 +286,10 @@ class google(ChatModel):
*,
system_message: str = DEFAULT_SYSTEM_MESSAGE,
api_key: Optional[str] = None,
- stream: bool = False,
):
self.model = model
self.system_message = system_message
self.api_key = api_key
- self.stream = stream
@property
def _require_api_key(self) -> str:
@@ -358,6 +317,24 @@ class google(ChatModel):
"set GOOGLE_AI_API_KEY as an environment variable"
)
+ def _stream_response(
+ self, client: Any, google_messages: Any, generation_config: Any
+ ) -> Generator[str, None, None]:
+ """Helper method for streaming - separate to avoid mixing yield/return."""
+ accumulated = ""
+ response = client.models.generate_content_stream(
+ model=self.model,
+ contents=google_messages,
+ config=generation_config,
+ )
+ for chunk in response:
+ if chunk.text:
+ accumulated += chunk.text
+ yield accumulated
+ # Yield final accumulated result to ensure complete response
+ if accumulated:
+ yield accumulated
+
def __call__(
self, messages: list[ChatMessage], config: ChatModelConfig
) -> object:
@@ -381,39 +358,9 @@ class google(ChatModel):
"presence_penalty": config.presence_penalty,
}
- if self.stream:
- # Return generator for streaming
- return self._stream_response(
- client, google_messages, generation_config
- )
- else:
- # Non-streaming response - return directly
- response = client.models.generate_content(
- model=self.model,
- contents=google_messages,
- config=cast(Any, generation_config),
- )
-
- content = response.text
- return content or ""
-
- def _stream_response(
- self, client: Any, google_messages: Any, generation_config: Any
- ) -> Generator[str, None, None]:
- """Helper method for streaming - separate to avoid mixing yield/return."""
- accumulated = ""
- response = client.models.generate_content_stream(
- model=self.model,
- contents=google_messages,
- config=generation_config,
+ return self._stream_response(
+ client, google_messages, generation_config
)
- for chunk in response:
- if chunk.text:
- accumulated += chunk.text
- yield accumulated
- # Yield final accumulated result to ensure complete response
- if accumulated:
- yield accumulated
class groq(ChatModel):
@@ -428,10 +375,6 @@ class groq(ChatModel):
If not provided, the API key will be retrieved
from the GROQ_API_KEY environment variable or the user's config.
base_url: The base URL to use
- stream: Whether to stream the response token-by-token as it's generated.
- When True, the response appears progressively (like ChatGPT), creating
- a more responsive user experience. When False, the complete response
- is returned all at once. Defaults to False for backward compatibility.
"""
def __init__(
@@ -441,13 +384,11 @@ class groq(ChatModel):
system_message: str = DEFAULT_SYSTEM_MESSAGE,
api_key: Optional[str] = None,
base_url: Optional[str] = None,
- stream: bool = False,
):
self.model = model
self.system_message = system_message
self.api_key = api_key
self.base_url = base_url
- self.stream = stream
@property
def _require_api_key(self) -> str:
@@ -516,24 +457,7 @@ class groq(ChatModel):
+ messages
)
- if self.stream:
- # Return generator for streaming
- return self._stream_response(client, groq_messages, config)
- else:
- # Non-streaming response - return directly
- response = client.chat.completions.create(
- model=self.model,
- messages=groq_messages,
- max_tokens=config.max_tokens,
- temperature=config.temperature,
- top_p=config.top_p,
- stop=None,
- stream=False,
- )
-
- choice = response.choices[0]
- content = choice.message.content
- return content or ""
+ return self._stream_response(client, groq_messages, config)
class bedrock(ChatModel):
@@ -550,10 +474,6 @@ class bedrock(ChatModel):
Dict with keys: "aws_access_key_id" and "aws_secret_access_key"
If not provided, credentials will be retrieved from the environment
or the AWS configuration files.
- stream: Whether to stream the response token-by-token as it's generated.
- When True, the response appears progressively (like ChatGPT), creating
- a more responsive user experience. When False, the complete response
- is returned all at once. Defaults to False for backward compatibility.
"""
def __init__(
@@ -565,7 +485,6 @@ class bedrock(ChatModel):
profile_name: Optional[str] = None,
aws_access_key_id: Optional[str] = None,
aws_secret_access_key: Optional[str] = None,
- stream: bool = False,
):
if not model.startswith("bedrock/"):
model = f"bedrock/{model}"
@@ -575,7 +494,6 @@ class bedrock(ChatModel):
self.profile_name = profile_name
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
- self.stream = stream
def _setup_credentials(self) -> None:
# Use profile name if provided, otherwise use API key
@@ -627,35 +545,10 @@ class bedrock(ChatModel):
DependencyManager.litellm.require(
"bedrock chat model requires litellm. `pip install litellm`"
)
- from litellm import completion as litellm_completion
-
self._setup_credentials()
try:
- if self.stream:
- # Return generator for streaming
- return self._stream_response(messages, config)
- else:
- # Non-streaming response - return directly
- response = litellm_completion(
- model=self.model,
- messages=convert_to_openai_messages(
- [
- ChatMessage(
- role="system", content=self.system_message
- )
- ]
- + messages
- ),
- max_tokens=config.max_tokens,
- temperature=config.temperature,
- top_p=config.top_p,
- frequency_penalty=config.frequency_penalty,
- presence_penalty=config.presence_penalty,
- stream=False,
- )
- return response.choices[0].message.content
-
+ return self._stream_response(messages, config)
except Exception as e:
# Handle common AWS exceptions with helpful messages
error_msg = str(e)
diff --git a/tests/_ai/llm/test_impl.py b/tests/_ai/llm/test_impl.py
index 8680b43538..73b9b5f8d4 100644
--- a/tests/_ai/llm/test_impl.py
+++ b/tests/_ai/llm/test_impl.py
@@ -27,14 +27,16 @@ def mock_openai_client():
mock_client = MagicMock()
mock_openai_class.return_value = mock_client
- # Setup the response structure
- mock_response = MagicMock()
+ # Setup the streaming response structure
+ mock_chunk = MagicMock()
mock_choice = MagicMock()
- mock_message = MagicMock()
- mock_message.content = "Test response"
- mock_choice.message = mock_message
- mock_response.choices = [mock_choice]
- mock_client.chat.completions.create.return_value = mock_response
+ mock_delta = MagicMock()
+ mock_delta.content = "Test response"
+ mock_choice.delta = mock_delta
+ mock_chunk.choices = [mock_choice]
+
+ # Return an iterable for streaming
+ mock_client.chat.completions.create.return_value = [mock_chunk]
yield mock_client, mock_openai_class
@@ -97,14 +99,16 @@ def mock_azure_openai_client():
mock_client = MagicMock()
mock_azure_openai_class.return_value = mock_client
- # Setup the response structure
- mock_response = MagicMock()
+ # Setup the streaming response structure
+ mock_chunk = MagicMock()
mock_choice = MagicMock()
- mock_message = MagicMock()
- mock_message.content = "Test response"
- mock_choice.message = mock_message
- mock_response.choices = [mock_choice]
- mock_client.chat.completions.create.return_value = mock_response
+ mock_delta = MagicMock()
+ mock_delta.content = "Test response"
+ mock_choice.delta = mock_delta
+ mock_chunk.choices = [mock_choice]
+
+ # Return an iterable for streaming
+ mock_client.chat.completions.create.return_value = [mock_chunk]
yield mock_client, mock_azure_openai_class
@@ -258,7 +262,9 @@ class TestOpenAI:
# Create model with API key to avoid _require_api_key
model = openai("gpt-4", api_key="test-key")
- result = model(test_messages, test_config)
+ result_gen = model(test_messages, test_config)
+ # Consume the generator to get the final result
+ result = list(result_gen)[-1] if result_gen else ""
# Verify result
assert result == "Test response"
@@ -287,7 +293,7 @@ class TestOpenAI:
assert call_args["top_p"] == pytest.approx(0.9)
assert call_args["frequency_penalty"] == pytest.approx(0.5)
assert call_args["presence_penalty"] == pytest.approx(0.5)
- assert call_args["stream"] is False
+ assert call_args["stream"] is True
def test_call_with_base_url(
self, mock_openai_client, test_messages, test_config
@@ -300,7 +306,9 @@ class TestOpenAI:
"gpt-4", api_key="test-key", base_url="https://example.com"
)
- result = model(test_messages, test_config)
+ result_gen = model(test_messages, test_config)
+ # Consume the generator to get the final result
+ result = list(result_gen)[-1] if result_gen else ""
# Verify result
assert result == "Test response"
@@ -323,7 +331,9 @@ class TestOpenAI:
base_url="https://example.openai.azure.com/openai/deployments/gpt-4/chat/completions?api-version=2023-05-15",
)
- result = model(test_messages, test_config)
+ result_gen = model(test_messages, test_config)
+ # Consume the generator to get the final result
+ result = list(result_gen)[-1] if result_gen else ""
# Verify result
assert result == "Test response"
@@ -346,15 +356,19 @@ class TestOpenAI:
"""Test calling the openai class with an empty response."""
mock_client, _ = mock_openai_client
- # Modify the mock to return an empty content
- mock_client.chat.completions.create.return_value.choices[
- 0
- ].message.content = None
+ # For streaming, we need to mock a streaming response with no content
+ # Create an empty chunk
+ mock_chunk = MagicMock()
+ mock_chunk.choices = []
+ mock_client.chat.completions.create.return_value = [mock_chunk]
# Create model with API key
model = openai("gpt-4", api_key="test-key")
- result = model(test_messages, test_config)
+ result_gen = model(test_messages, test_config)
+ # Consume the generator to get the final result
+ result_list = list(result_gen)
+ result = result_list[-1] if result_list else ""
# Verify empty string is returned when content is None
assert result == ""
@@ -732,9 +746,11 @@ class TestGoogle:
mock_require_api_key.return_value = "test-key"
mock_client = MagicMock()
mock_genai_client_class.return_value = mock_client
- mock_response = MagicMock()
- mock_response.text = "Test response"
- mock_client.models.generate_content.return_value = mock_response
+
+ # Setup streaming response
+ mock_chunk = MagicMock()
+ mock_chunk.text = "Test response"
+ mock_client.models.generate_content_stream.return_value = [mock_chunk]
model = google("gemini-2.5-flash-preview-05-20")
# Patch the _require_api_key property to return the test key directly
@@ -749,13 +765,15 @@ class TestGoogle:
presence_penalty=0.5,
)
- result = model(messages, config)
+ result_gen = model(messages, config)
+ # Consume the generator to get the final result
+ result = list(result_gen)[-1] if result_gen else ""
assert result == "Test response"
mock_genai_client_class.assert_called_once_with(api_key="test-key")
- mock_client.models.generate_content.assert_called_once()
- call_args = mock_client.models.generate_content.call_args[1]
+ mock_client.models.generate_content_stream.assert_called_once()
+ call_args = mock_client.models.generate_content_stream.call_args[1]
assert call_args["model"] == "gemini-2.5-flash-preview-05-20"
config_arg = call_args["config"]
assert config_arg["system_instruction"] == DEFAULT_SYSTEM_MESSAGE
@@ -832,12 +850,13 @@ class TestAnthropic:
mock_require_api_key.return_value = "test-key"
mock_client = MagicMock()
mock_anthropic_class.return_value = mock_client
- mock_response = MagicMock()
- mock_content = MagicMock()
- mock_content.type = "text"
- mock_content.text = "Test response"
- mock_response.content = [mock_content]
- mock_client.messages.create.return_value = mock_response
+
+ # Setup streaming response using context manager
+ mock_stream = MagicMock()
+ mock_stream.__enter__ = MagicMock(return_value=mock_stream)
+ mock_stream.__exit__ = MagicMock(return_value=None)
+ mock_stream.text_stream = ["Test response"]
+ mock_client.messages.stream.return_value = mock_stream
model = anthropic("claude-3-opus-20240229")
# Patch the _require_api_key property to return the test key directly
@@ -850,43 +869,55 @@ class TestAnthropic:
top_k=10,
)
- result = model(messages, config)
+ result_gen = model(messages, config)
+ # Consume the generator to get the final result
+ result = list(result_gen)[-1] if result_gen else ""
assert result == "Test response"
mock_anthropic_class.assert_called_once_with(
api_key="test-key", base_url=None
)
- mock_client.messages.create.assert_called_once()
- call_args = mock_client.messages.create.call_args[1]
+ mock_client.messages.stream.assert_called_once()
+ call_args = mock_client.messages.stream.call_args[1]
assert call_args["model"] == "claude-3-opus-20240229"
assert call_args["system"] == DEFAULT_SYSTEM_MESSAGE
assert call_args["max_tokens"] == 100
assert call_args["temperature"] == 0.7
assert call_args["top_p"] == 0.9
assert call_args["top_k"] == 10
- assert call_args["stream"] is False
+ assert call_args["stream"] is True
@patch.object(anthropic, "_require_api_key")
@patch("anthropic.Anthropic")
def test_call_tool_use(
self, mock_anthropic_class: MagicMock, mock_require_api_key: MagicMock
) -> None:
- """Test calling the anthropic class with tool use response."""
+ """Test calling the anthropic class with tool use response.
+
+ Note: With streaming API, tool use may not be supported in the same way.
+ This test is kept for backwards compatibility but may need revision.
+ """
mock_require_api_key.return_value = "test-key"
mock_client = MagicMock()
mock_anthropic_class.return_value = mock_client
- mock_response = MagicMock()
- mock_content = MagicMock()
- mock_content.type = "tool_use"
- mock_response.content = [mock_content]
- mock_client.messages.create.return_value = mock_response
+
+ # Setup streaming response with empty text (tool use case)
+ mock_stream = MagicMock()
+ mock_stream.__enter__ = MagicMock(return_value=mock_stream)
+ mock_stream.__exit__ = MagicMock(return_value=None)
+ mock_stream.text_stream = [] # No text for tool use
+ mock_client.messages.stream.return_value = mock_stream
model = anthropic("claude-3-opus-20240229")
messages = [ChatMessage(role="user", content="Test prompt")]
config = ChatModelConfig()
- result = model(messages, config)
- assert result == [mock_content]
+ result_gen = model(messages, config)
+ # Consume the generator
+ result_list = list(result_gen)
+ # For empty text stream, expect empty result
+ result = result_list[-1] if result_list else ""
+ assert result == ""
@patch.object(anthropic, "_require_api_key")
@patch("anthropic.Anthropic")
@@ -897,15 +928,22 @@ class TestAnthropic:
mock_require_api_key.return_value = "test-key"
mock_client = MagicMock()
mock_anthropic_class.return_value = mock_client
- mock_response = MagicMock()
- mock_response.content = []
- mock_client.messages.create.return_value = mock_response
+
+ # Setup streaming response with no content
+ mock_stream = MagicMock()
+ mock_stream.__enter__ = MagicMock(return_value=mock_stream)
+ mock_stream.__exit__ = MagicMock(return_value=None)
+ mock_stream.text_stream = []
+ mock_client.messages.stream.return_value = mock_stream
model = anthropic("claude-3-opus-20240229")
messages = [ChatMessage(role="user", content="Test prompt")]
config = ChatModelConfig()
- result = model(messages, config)
+ result_gen = model(messages, config)
+ # Consume the generator
+ result_list = list(result_gen)
+ result = result_list[-1] if result_list else ""
assert result == ""
@patch.dict(os.environ, {"ANTHROPIC_API_KEY": "env-key"})