Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix(ai): fall back to non-streaming when a model rejects streaming (#11274)

alexcastgonz67f4795d7958
diff --git a/marimo/_ai/llm/_impl.py b/marimo/_ai/llm/_impl.py
index 724af4ee11..e99858373b 100644
--- a/marimo/_ai/llm/_impl.py
+++ b/marimo/_ai/llm/_impl.py
@@ -48,6 +48,36 @@ def _looks_like_streaming_error(e: Exception) -> bool:
)
+def _start_stream(
+ chunks: Generator[str, None, None],
+) -> Generator[str, None, None]:
+ """Return a generator that replays the first chunk, then yields the rest.
+
+ Starting it here raises a streaming error inside the caller's try block, so non-streaming models can fallback safely.
+ """
+ first = next(chunks, None)
+
+ def resume() -> Generator[str, None, None]:
+ if first is not None:
+ yield first
+ yield from chunks
+
+ return resume()
+
+
+def _stream_or_fallback(
+ stream: Callable[[], Generator[str, None, None]],
+ fallback: Callable[[], str],
+) -> Generator[str, None, None] | str:
+ """Stream the response, or call `fallback` if the model can't stream."""
+ try:
+ return _start_stream(stream())
+ except Exception as e:
+ if _looks_like_streaming_error(e):
+ return fallback()
+ raise
+
+
class simple(ChatModel):
"""
Convenience class for wrapping a ChatModel or callable to
@@ -303,17 +333,23 @@ class anthropic(ChatModel):
):
params["temperature"] = config.temperature
- # Try streaming first, fall back to non-streaming on error
- try:
- # Note: client.messages.stream() doesn't take a 'stream' parameter
- # It's already a streaming method
- return self._stream_response(client, params)
- except Exception as e:
- # Fall back to non-streaming mode if streaming fails
- if _looks_like_streaming_error(e):
- response = client.messages.create(**params)
- return response.content[0].text
- raise
+ def non_streaming() -> str:
+ response = client.messages.create(**params)
+ # The first block can be a thinking block, which has no text
+ return next(
+ (
+ block.text
+ for block in response.content
+ if block.type == "text"
+ ),
+ "",
+ )
+
+ # Note: client.messages.stream() doesn't take a 'stream' parameter
+ # It's already a streaming method
+ return _stream_or_fallback(
+ lambda: self._stream_response(client, params), non_streaming
+ )
class google(ChatModel):
@@ -407,21 +443,20 @@ class google(ChatModel):
"presence_penalty": config.presence_penalty,
}
- # Try streaming first, fall back to non-streaming on error
- try:
- return self._stream_response(
- client, google_messages, generation_config
+ def non_streaming() -> str:
+ response = client.models.generate_content(
+ model=self.model,
+ contents=google_messages,
+ config=generation_config, # type: ignore[arg-type]
)
- except Exception as e:
- # Fall back to non-streaming mode if streaming fails
- if _looks_like_streaming_error(e):
- response = client.models.generate_content(
- model=self.model,
- contents=google_messages,
- config=generation_config, # type: ignore[arg-type]
- )
- return response.text
- raise
+ return response.text or ""
+
+ return _stream_or_fallback(
+ lambda: self._stream_response(
+ client, google_messages, generation_config
+ ),
+ non_streaming,
+ )
class groq(ChatModel):
@@ -517,23 +552,22 @@ class groq(ChatModel):
+ messages
)
- # Try streaming first, fall back to non-streaming on error
- try:
- return self._stream_response(client, groq_messages, config)
- except Exception as e:
- # Fall back to non-streaming mode if streaming fails
- if _looks_like_streaming_error(e):
- response = client.chat.completions.create(
- model=self.model,
- messages=groq_messages,
- max_tokens=config.max_tokens,
- temperature=config.temperature,
- top_p=config.top_p,
- stop=None,
- stream=False,
- )
- return response.choices[0].message.content or ""
- raise
+ def non_streaming() -> str:
+ response = client.chat.completions.create(
+ model=self.model,
+ messages=groq_messages,
+ max_tokens=config.max_tokens,
+ temperature=config.temperature,
+ top_p=config.top_p,
+ stop=None,
+ stream=False,
+ )
+ return response.choices[0].message.content or ""
+
+ return _stream_or_fallback(
+ lambda: self._stream_response(client, groq_messages, config),
+ non_streaming,
+ )
class bedrock(ChatModel):
@@ -643,19 +677,21 @@ class bedrock(ChatModel):
client = self._create_client()
params = self._build_converse_params(messages, config)
+ def non_streaming() -> str:
+ response = client.converse(**params)
+ output = response.get("output", {})
+ message = output.get("message", {})
+ content = message.get("content", [])
+ # The first block can be reasoning, which has no text
+ return next(
+ (block["text"] for block in content if "text" in block),
+ "",
+ )
+
try:
- try:
- return self._stream_response(client, params)
- except Exception as stream_error:
- if _looks_like_streaming_error(stream_error):
- response = client.converse(**params)
- output = response.get("output", {})
- message = output.get("message", {})
- content = message.get("content", [])
- if content and "text" in content[0]:
- return content[0]["text"]
- return ""
- raise
+ return _stream_or_fallback(
+ lambda: self._stream_response(client, params), non_streaming
+ )
except Exception as e:
error_msg = str(e)
diff --git a/tests/_ai/llm/test_impl.py b/tests/_ai/llm/test_impl.py
index d9d8ed10e7..ba865a771b 100644
--- a/tests/_ai/llm/test_impl.py
+++ b/tests/_ai/llm/test_impl.py
@@ -1750,3 +1750,106 @@ def test_pydantic_ai_require() -> None:
mock_agent = MagicMock()
with pytest.raises(ModuleNotFoundError):
pydantic_ai(mock_agent)
+
+
+class TestStreamingFallback:
+ """A model that rejects streaming gets a non-streaming request instead.
+
+ The streaming request is made lazily inside a generator, so these tests
+ fake the SDKs and check the fallback end to end, without them installed.
+ """
+
+ STREAMING_ERROR = "This model does not support streaming"
+ messages = [ChatMessage(role="user", content="Test prompt")] # noqa: RUF012
+
+ def _streaming_error(self, *_args: Any, **_kwargs: Any) -> Any:
+ raise RuntimeError(self.STREAMING_ERROR)
+
+ def test_anthropic(self) -> None:
+ client = MagicMock()
+ client.messages.stream.side_effect = self._streaming_error
+ client.messages.create.return_value = MagicMock(
+ content=[
+ MagicMock(type="thinking", spec=["type", "thinking"]),
+ MagicMock(type="text", text="Fallback response"),
+ ]
+ )
+ fake_sdk = MagicMock(Anthropic=MagicMock(return_value=client))
+
+ with (
+ patch.dict("sys.modules", {"anthropic": fake_sdk}),
+ patch.object(DependencyManager.anthropic, "require"),
+ ):
+ model = anthropic("claude-x", api_key="test-key")
+ result = model(self.messages, ChatModelConfig())
+
+ assert result == "Fallback response"
+ client.messages.create.assert_called_once()
+
+ def test_google(self) -> None:
+ client = MagicMock()
+ client.models.generate_content_stream.side_effect = (
+ self._streaming_error
+ )
+ client.models.generate_content.return_value = MagicMock(
+ text="Fallback response"
+ )
+ genai = MagicMock(Client=MagicMock(return_value=client))
+ fake_sdk = MagicMock(genai=genai)
+
+ with (
+ patch.dict(
+ "sys.modules", {"google": fake_sdk, "google.genai": genai}
+ ),
+ patch.object(DependencyManager.google_ai, "require"),
+ ):
+ model = google("gemini-x", api_key="test-key")
+ result = model(self.messages, ChatModelConfig())
+
+ assert result == "Fallback response"
+ client.models.generate_content.assert_called_once()
+
+ def test_groq(self) -> None:
+ client = MagicMock()
+
+ def create(**kwargs: Any) -> Any:
+ if kwargs.get("stream"):
+ self._streaming_error()
+ message = MagicMock(content="Fallback response")
+ return MagicMock(choices=[MagicMock(message=message)])
+
+ client.chat.completions.create.side_effect = create
+ fake_sdk = MagicMock(Groq=MagicMock(return_value=client))
+
+ with (
+ patch.dict("sys.modules", {"groq": fake_sdk}),
+ patch.object(DependencyManager.groq, "require"),
+ ):
+ model = groq("llama-x", api_key="test-key")
+ result = model(self.messages, ChatModelConfig())
+
+ assert result == "Fallback response"
+ assert (
+ client.chat.completions.create.call_args.kwargs["stream"] is False
+ )
+
+ def test_bedrock(self) -> None:
+ client = MagicMock()
+ client.converse_stream.side_effect = self._streaming_error
+ client.converse.return_value = {
+ "output": {
+ "message": {
+ "content": [
+ {"reasoningContent": {"reasoningText": {"text": "…"}}},
+ {"text": "Fallback response"},
+ ]
+ }
+ }
+ }
+
+ model = bedrock("bedrock/anthropic.claude-x")
+ with patch.object(model, "_create_client", return_value=client):
+ result = model(self.messages, ChatModelConfig())
+
+ assert result == "Fallback response"
+ client.converse.assert_called_once()