A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
api: remove stream argument for built-in AI models (#7222)
Akshay Agrawala58a1b82c80c
diff --git a/docs/api/inputs/chat.md b/docs/api/inputs/chat.mdindex 344d953c40..997f535264 100644--- a/docs/api/inputs/chat.md+++ b/docs/api/inputs/chat.md@@ -137,31 +137,7 @@ mo.ui.chat(Chatbots can stream responses in real-time, creating a more interactive experiencesimilar to ChatGPT where you see the response appear word-by-word as it's generated.-### With Built-in Models--For built-in models (OpenAI, Anthropic, Google, Groq, Bedrock), set `stream=True` in the model constructor:--```python-import marimo as mo--chat = mo.ui.chat(- mo.ai.llm.openai(- "gpt-4o",- system_message="You are a helpful assistant.",- stream=True, # Enable streaming- ),- show_configuration_controls=True-)-chat-```--This works for all built-in models:--- `mo.ai.llm.openai("gpt-4o", stream=True)`-- `mo.ai.llm.anthropic("claude-3-5-sonnet-20240620", stream=True)`-- `mo.ai.llm.google("gemini-1.5-pro-latest", stream=True)`-- `mo.ai.llm.groq("llama-3.1-70b-versatile", stream=True)`-- `mo.ai.llm.bedrock("anthropic.claude-3-7-sonnet-20250219-v1:0", stream=True)`+Responses from built-in models (OpenAI, Anthropic, Google, Groq, Bedrock) are streamed by default.### With Custom Modelsdiff --git a/examples/ai/chat/anthropic_example.py b/examples/ai/chat/anthropic_example.pyindex 460d066f99..d738feb24d 100644--- a/examples/ai/chat/anthropic_example.py+++ b/examples/ai/chat/anthropic_example.py@@ -56,7 +56,6 @@ def _(key, mo):"claude-3-7-sonnet-20250219",system_message="You are a helpful assistant.",api_key=key,- # stream=True, # Uncomment to enable streaming responses),allow_attachments=["image/png",diff --git a/examples/ai/chat/bedrock_example.py b/examples/ai/chat/bedrock_example.pyindex c65575d452..acc1a54870 100644--- a/examples/ai/chat/bedrock_example.py+++ b/examples/ai/chat/bedrock_example.py@@ -192,7 +192,6 @@ def _(config_form, max_tokens, mo, temperature):"model": model,"region_name": region,"system_message": system_message,- # "stream": True, # Uncomment to enable streaming responses}# Add profile if specifieddiff --git a/examples/ai/chat/gemini.py b/examples/ai/chat/gemini.pyindex bf758a1cbb..f23df4ef72 100644--- a/examples/ai/chat/gemini.py+++ b/examples/ai/chat/gemini.py@@ -56,7 +56,6 @@ def _(key, mo):"gemini-2.5-flash",system_message="You are a helpful assistant.",api_key=key,- # stream=True, # Uncomment to enable streaming responses),prompts=["Hello",diff --git a/examples/ai/chat/groq_example.py b/examples/ai/chat/groq_example.pyindex ae10e553bc..b030f4fcc3 100644--- a/examples/ai/chat/groq_example.py+++ b/examples/ai/chat/groq_example.py@@ -73,7 +73,6 @@ def _(key, mo):model="llama-3.1-70b-versatile",system_message="You are a helpful assistant.",api_key=key,- # stream=True, # Uncomment to enable streaming responses),prompts=["Hello",diff --git a/examples/ai/chat/openai_example.py b/examples/ai/chat/openai_example.pyindex 0e75e45dba..8556a126b0 100644--- a/examples/ai/chat/openai_example.py+++ b/examples/ai/chat/openai_example.py@@ -56,7 +56,6 @@ def _(mo, openai_key):"gpt-4o",system_message="You are a helpful assistant.",api_key=openai_key,- # stream=True, # Uncomment to enable streaming responses),prompts=["Hello",diff --git a/examples/ai/chat/streaming_openai.py b/examples/ai/chat/streaming_openai.pyindex 4c19223c68..ea2c202f23 100644--- a/examples/ai/chat/streaming_openai.py+++ b/examples/ai/chat/streaming_openai.py@@ -51,7 +51,6 @@ def _(api_key_input, mo):"gpt-4o-mini",system_message="You are a helpful assistant. Keep responses concise and friendly.",api_key=api_key_input.value,- stream=True, # Enable streaming),prompts=["Tell me a short joke",@@ -84,13 +83,10 @@ def _(mo):mo.ai.llm.openai("gpt-4o-mini",api_key="your-key",- stream=True, # Enable streaming!))```- Set `stream=True` to enable streaming responses. 🚀-Other built-in models (`anthropic`, `google`, `groq`) work the same way.""")returndiff --git a/marimo/_ai/llm/_impl.py b/marimo/_ai/llm/_impl.pyindex d3e77ffae6..c04bdd42da 100644--- a/marimo/_ai/llm/_impl.py+++ b/marimo/_ai/llm/_impl.py@@ -58,10 +58,6 @@ class openai(ChatModel):If not provided, the API key will be retrievedfrom the OPENAI_API_KEY environment variable or the user's config.base_url: The base URL to use- stream: Whether to stream the response token-by-token as it's generated.- When True, the response appears progressively (like ChatGPT), creating- a more responsive user experience. When False, the complete response- is returned all at once. Defaults to False for backward compatibility."""def __init__(@@ -71,13 +67,11 @@ class openai(ChatModel):system_message: str = DEFAULT_SYSTEM_MESSAGE,api_key: Optional[str] = None,base_url: Optional[str] = None,- stream: bool = False,):self.model = modelself.system_message = system_messageself.api_key = api_keyself.base_url = base_url- self.stream = stream@propertydef _require_api_key(self) -> str:@@ -168,17 +162,10 @@ class openai(ChatModel):top_p=config.top_p,frequency_penalty=config.frequency_penalty,presence_penalty=config.presence_penalty,- stream=self.stream,+ stream=True,)- if self.stream:- # Return generator for streaming- return self._stream_response(response)- else:- # Non-streaming response - return string directly- choice = cast(Any, response).choices[0]- content = choice.message.content- return content or ""+ return self._stream_response(response)class anthropic(ChatModel):@@ -194,10 +181,6 @@ class anthropic(ChatModel):from the ANTHROPIC_API_KEY environment variableor the user's config.base_url: The base URL to use- stream: Whether to stream the response token-by-token as it's generated.- When True, the response appears progressively (like ChatGPT), creating- a more responsive user experience. When False, the complete response- is returned all at once. Defaults to False for backward compatibility."""def __init__(@@ -207,13 +190,11 @@ class anthropic(ChatModel):system_message: str = DEFAULT_SYSTEM_MESSAGE,api_key: Optional[str] = None,base_url: Optional[str] = None,- stream: bool = False,):self.model = modelself.system_message = system_messageself.api_key = api_keyself.base_url = base_url- self.stream = stream@propertydef _require_api_key(self) -> str:@@ -273,7 +254,7 @@ class anthropic(ChatModel):"system": self.system_message,"max_tokens": config.max_tokens or 4096,"messages": anthropic_messages,- "stream": self.stream,+ "stream": True,}if config.top_p is not None:params["top_p"] = config.top_p@@ -282,23 +263,7 @@ class anthropic(ChatModel):if config.temperature is not None:params["temperature"] = config.temperature- if self.stream:- # Return generator for streaming- return self._stream_response(client, params)- else:- # Non-streaming response - return directly- response = client.messages.create(**params)-- content = response.content- if len(content) > 0:- if content[0].type == "text":- return content[0].text- elif content[0].type == "tool_use":- return content- else:- return ""- else:- return ""+ return self._stream_response(client, params)class google(ChatModel):@@ -313,10 +278,6 @@ class google(ChatModel):If not provided, the API key will be retrievedfrom the GOOGLE_AI_API_KEY environment variableor the user's config.- stream: Whether to stream the response token-by-token as it's generated.- When True, the response appears progressively (like ChatGPT), creating- a more responsive user experience. When False, the complete response- is returned all at once. Defaults to False for backward compatibility."""def __init__(@@ -325,12 +286,10 @@ class google(ChatModel):*,system_message: str = DEFAULT_SYSTEM_MESSAGE,api_key: Optional[str] = None,- stream: bool = False,):self.model = modelself.system_message = system_messageself.api_key = api_key- self.stream = stream@propertydef _require_api_key(self) -> str:@@ -358,6 +317,24 @@ class google(ChatModel):"set GOOGLE_AI_API_KEY as an environment variable")+ def _stream_response(+ self, client: Any, google_messages: Any, generation_config: Any+ ) -> Generator[str, None, None]:+ """Helper method for streaming - separate to avoid mixing yield/return."""+ accumulated = ""+ response = client.models.generate_content_stream(+ model=self.model,+ contents=google_messages,+ config=generation_config,+ )+ for chunk in response:+ if chunk.text:+ accumulated += chunk.text+ yield accumulated+ # Yield final accumulated result to ensure complete response+ if accumulated:+ yield accumulated+def __call__(self, messages: list[ChatMessage], config: ChatModelConfig) -> object:@@ -381,39 +358,9 @@ class google(ChatModel):"presence_penalty": config.presence_penalty,}- if self.stream:- # Return generator for streaming- return self._stream_response(- client, google_messages, generation_config- )- else:- # Non-streaming response - return directly- response = client.models.generate_content(- model=self.model,- contents=google_messages,- config=cast(Any, generation_config),- )-- content = response.text- return content or ""-- def _stream_response(- self, client: Any, google_messages: Any, generation_config: Any- ) -> Generator[str, None, None]:- """Helper method for streaming - separate to avoid mixing yield/return."""- accumulated = ""- response = client.models.generate_content_stream(- model=self.model,- contents=google_messages,- config=generation_config,+ return self._stream_response(+ client, google_messages, generation_config)- for chunk in response:- if chunk.text:- accumulated += chunk.text- yield accumulated- # Yield final accumulated result to ensure complete response- if accumulated:- yield accumulatedclass groq(ChatModel):@@ -428,10 +375,6 @@ class groq(ChatModel):If not provided, the API key will be retrievedfrom the GROQ_API_KEY environment variable or the user's config.base_url: The base URL to use- stream: Whether to stream the response token-by-token as it's generated.- When True, the response appears progressively (like ChatGPT), creating- a more responsive user experience. When False, the complete response- is returned all at once. Defaults to False for backward compatibility."""def __init__(@@ -441,13 +384,11 @@ class groq(ChatModel):system_message: str = DEFAULT_SYSTEM_MESSAGE,api_key: Optional[str] = None,base_url: Optional[str] = None,- stream: bool = False,):self.model = modelself.system_message = system_messageself.api_key = api_keyself.base_url = base_url- self.stream = stream@propertydef _require_api_key(self) -> str:@@ -516,24 +457,7 @@ class groq(ChatModel):+ messages)- if self.stream:- # Return generator for streaming- return self._stream_response(client, groq_messages, config)- else:- # Non-streaming response - return directly- response = client.chat.completions.create(- model=self.model,- messages=groq_messages,- max_tokens=config.max_tokens,- temperature=config.temperature,- top_p=config.top_p,- stop=None,- stream=False,- )-- choice = response.choices[0]- content = choice.message.content- return content or ""+ return self._stream_response(client, groq_messages, config)class bedrock(ChatModel):@@ -550,10 +474,6 @@ class bedrock(ChatModel):Dict with keys: "aws_access_key_id" and "aws_secret_access_key"If not provided, credentials will be retrieved from the environmentor the AWS configuration files.- stream: Whether to stream the response token-by-token as it's generated.- When True, the response appears progressively (like ChatGPT), creating- a more responsive user experience. When False, the complete response- is returned all at once. Defaults to False for backward compatibility."""def __init__(@@ -565,7 +485,6 @@ class bedrock(ChatModel):profile_name: Optional[str] = None,aws_access_key_id: Optional[str] = None,aws_secret_access_key: Optional[str] = None,- stream: bool = False,):if not model.startswith("bedrock/"):model = f"bedrock/{model}"@@ -575,7 +494,6 @@ class bedrock(ChatModel):self.profile_name = profile_nameself.aws_access_key_id = aws_access_key_idself.aws_secret_access_key = aws_secret_access_key- self.stream = streamdef _setup_credentials(self) -> None:# Use profile name if provided, otherwise use API key@@ -627,35 +545,10 @@ class bedrock(ChatModel):DependencyManager.litellm.require("bedrock chat model requires litellm. `pip install litellm`")- from litellm import completion as litellm_completion-self._setup_credentials()try:- if self.stream:- # Return generator for streaming- return self._stream_response(messages, config)- else:- # Non-streaming response - return directly- response = litellm_completion(- model=self.model,- messages=convert_to_openai_messages(- [- ChatMessage(- role="system", content=self.system_message- )- ]- + messages- ),- max_tokens=config.max_tokens,- temperature=config.temperature,- top_p=config.top_p,- frequency_penalty=config.frequency_penalty,- presence_penalty=config.presence_penalty,- stream=False,- )- return response.choices[0].message.content-+ return self._stream_response(messages, config)except Exception as e:# Handle common AWS exceptions with helpful messageserror_msg = str(e)diff --git a/tests/_ai/llm/test_impl.py b/tests/_ai/llm/test_impl.pyindex 8680b43538..73b9b5f8d4 100644--- a/tests/_ai/llm/test_impl.py+++ b/tests/_ai/llm/test_impl.py@@ -27,14 +27,16 @@ def mock_openai_client():mock_client = MagicMock()mock_openai_class.return_value = mock_client- # Setup the response structure- mock_response = MagicMock()+ # Setup the streaming response structure+ mock_chunk = MagicMock()mock_choice = MagicMock()- mock_message = MagicMock()- mock_message.content = "Test response"- mock_choice.message = mock_message- mock_response.choices = [mock_choice]- mock_client.chat.completions.create.return_value = mock_response+ mock_delta = MagicMock()+ mock_delta.content = "Test response"+ mock_choice.delta = mock_delta+ mock_chunk.choices = [mock_choice]++ # Return an iterable for streaming+ mock_client.chat.completions.create.return_value = [mock_chunk]yield mock_client, mock_openai_class@@ -97,14 +99,16 @@ def mock_azure_openai_client():mock_client = MagicMock()mock_azure_openai_class.return_value = mock_client- # Setup the response structure- mock_response = MagicMock()+ # Setup the streaming response structure+ mock_chunk = MagicMock()mock_choice = MagicMock()- mock_message = MagicMock()- mock_message.content = "Test response"- mock_choice.message = mock_message- mock_response.choices = [mock_choice]- mock_client.chat.completions.create.return_value = mock_response+ mock_delta = MagicMock()+ mock_delta.content = "Test response"+ mock_choice.delta = mock_delta+ mock_chunk.choices = [mock_choice]++ # Return an iterable for streaming+ mock_client.chat.completions.create.return_value = [mock_chunk]yield mock_client, mock_azure_openai_class@@ -258,7 +262,9 @@ class TestOpenAI:# Create model with API key to avoid _require_api_keymodel = openai("gpt-4", api_key="test-key")- result = model(test_messages, test_config)+ result_gen = model(test_messages, test_config)+ # Consume the generator to get the final result+ result = list(result_gen)[-1] if result_gen else ""# Verify resultassert result == "Test response"@@ -287,7 +293,7 @@ class TestOpenAI:assert call_args["top_p"] == pytest.approx(0.9)assert call_args["frequency_penalty"] == pytest.approx(0.5)assert call_args["presence_penalty"] == pytest.approx(0.5)- assert call_args["stream"] is False+ assert call_args["stream"] is Truedef test_call_with_base_url(self, mock_openai_client, test_messages, test_config@@ -300,7 +306,9 @@ class TestOpenAI:"gpt-4", api_key="test-key", base_url="https://example.com")- result = model(test_messages, test_config)+ result_gen = model(test_messages, test_config)+ # Consume the generator to get the final result+ result = list(result_gen)[-1] if result_gen else ""# Verify resultassert result == "Test response"@@ -323,7 +331,9 @@ class TestOpenAI:base_url="https://example.openai.azure.com/openai/deployments/gpt-4/chat/completions?api-version=2023-05-15",)- result = model(test_messages, test_config)+ result_gen = model(test_messages, test_config)+ # Consume the generator to get the final result+ result = list(result_gen)[-1] if result_gen else ""# Verify resultassert result == "Test response"@@ -346,15 +356,19 @@ class TestOpenAI:"""Test calling the openai class with an empty response."""mock_client, _ = mock_openai_client- # Modify the mock to return an empty content- mock_client.chat.completions.create.return_value.choices[- 0- ].message.content = None+ # For streaming, we need to mock a streaming response with no content+ # Create an empty chunk+ mock_chunk = MagicMock()+ mock_chunk.choices = []+ mock_client.chat.completions.create.return_value = [mock_chunk]# Create model with API keymodel = openai("gpt-4", api_key="test-key")- result = model(test_messages, test_config)+ result_gen = model(test_messages, test_config)+ # Consume the generator to get the final result+ result_list = list(result_gen)+ result = result_list[-1] if result_list else ""# Verify empty string is returned when content is Noneassert result == ""@@ -732,9 +746,11 @@ class TestGoogle:mock_require_api_key.return_value = "test-key"mock_client = MagicMock()mock_genai_client_class.return_value = mock_client- mock_response = MagicMock()- mock_response.text = "Test response"- mock_client.models.generate_content.return_value = mock_response++ # Setup streaming response+ mock_chunk = MagicMock()+ mock_chunk.text = "Test response"+ mock_client.models.generate_content_stream.return_value = [mock_chunk]model = google("gemini-2.5-flash-preview-05-20")# Patch the _require_api_key property to return the test key directly@@ -749,13 +765,15 @@ class TestGoogle:presence_penalty=0.5,)- result = model(messages, config)+ result_gen = model(messages, config)+ # Consume the generator to get the final result+ result = list(result_gen)[-1] if result_gen else ""assert result == "Test response"mock_genai_client_class.assert_called_once_with(api_key="test-key")- mock_client.models.generate_content.assert_called_once()- call_args = mock_client.models.generate_content.call_args[1]+ mock_client.models.generate_content_stream.assert_called_once()+ call_args = mock_client.models.generate_content_stream.call_args[1]assert call_args["model"] == "gemini-2.5-flash-preview-05-20"config_arg = call_args["config"]assert config_arg["system_instruction"] == DEFAULT_SYSTEM_MESSAGE@@ -832,12 +850,13 @@ class TestAnthropic:mock_require_api_key.return_value = "test-key"mock_client = MagicMock()mock_anthropic_class.return_value = mock_client- mock_response = MagicMock()- mock_content = MagicMock()- mock_content.type = "text"- mock_content.text = "Test response"- mock_response.content = [mock_content]- mock_client.messages.create.return_value = mock_response++ # Setup streaming response using context manager+ mock_stream = MagicMock()+ mock_stream.__enter__ = MagicMock(return_value=mock_stream)+ mock_stream.__exit__ = MagicMock(return_value=None)+ mock_stream.text_stream = ["Test response"]+ mock_client.messages.stream.return_value = mock_streammodel = anthropic("claude-3-opus-20240229")# Patch the _require_api_key property to return the test key directly@@ -850,43 +869,55 @@ class TestAnthropic:top_k=10,)- result = model(messages, config)+ result_gen = model(messages, config)+ # Consume the generator to get the final result+ result = list(result_gen)[-1] if result_gen else ""assert result == "Test response"mock_anthropic_class.assert_called_once_with(api_key="test-key", base_url=None)- mock_client.messages.create.assert_called_once()- call_args = mock_client.messages.create.call_args[1]+ mock_client.messages.stream.assert_called_once()+ call_args = mock_client.messages.stream.call_args[1]assert call_args["model"] == "claude-3-opus-20240229"assert call_args["system"] == DEFAULT_SYSTEM_MESSAGEassert call_args["max_tokens"] == 100assert call_args["temperature"] == 0.7assert call_args["top_p"] == 0.9assert call_args["top_k"] == 10- assert call_args["stream"] is False+ assert call_args["stream"] is True@patch.object(anthropic, "_require_api_key")@patch("anthropic.Anthropic")def test_call_tool_use(self, mock_anthropic_class: MagicMock, mock_require_api_key: MagicMock) -> None:- """Test calling the anthropic class with tool use response."""+ """Test calling the anthropic class with tool use response.++ Note: With streaming API, tool use may not be supported in the same way.+ This test is kept for backwards compatibility but may need revision.+ """mock_require_api_key.return_value = "test-key"mock_client = MagicMock()mock_anthropic_class.return_value = mock_client- mock_response = MagicMock()- mock_content = MagicMock()- mock_content.type = "tool_use"- mock_response.content = [mock_content]- mock_client.messages.create.return_value = mock_response++ # Setup streaming response with empty text (tool use case)+ mock_stream = MagicMock()+ mock_stream.__enter__ = MagicMock(return_value=mock_stream)+ mock_stream.__exit__ = MagicMock(return_value=None)+ mock_stream.text_stream = [] # No text for tool use+ mock_client.messages.stream.return_value = mock_streammodel = anthropic("claude-3-opus-20240229")messages = [ChatMessage(role="user", content="Test prompt")]config = ChatModelConfig()- result = model(messages, config)- assert result == [mock_content]+ result_gen = model(messages, config)+ # Consume the generator+ result_list = list(result_gen)+ # For empty text stream, expect empty result+ result = result_list[-1] if result_list else ""+ assert result == ""@patch.object(anthropic, "_require_api_key")@patch("anthropic.Anthropic")@@ -897,15 +928,22 @@ class TestAnthropic:mock_require_api_key.return_value = "test-key"mock_client = MagicMock()mock_anthropic_class.return_value = mock_client- mock_response = MagicMock()- mock_response.content = []- mock_client.messages.create.return_value = mock_response++ # Setup streaming response with no content+ mock_stream = MagicMock()+ mock_stream.__enter__ = MagicMock(return_value=mock_stream)+ mock_stream.__exit__ = MagicMock(return_value=None)+ mock_stream.text_stream = []+ mock_client.messages.stream.return_value = mock_streammodel = anthropic("claude-3-opus-20240229")messages = [ChatMessage(role="user", content="Test prompt")]config = ChatModelConfig()- result = model(messages, config)+ result_gen = model(messages, config)+ # Consume the generator+ result_list = list(result_gen)+ result = result_list[-1] if result_list else ""assert result == ""@patch.dict(os.environ, {"ANTHROPIC_API_KEY": "env-key"})