A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix(ai): fall back to non-streaming when a model rejects streaming (#11274)
alexcastgonz67f4795d7958
diff --git a/marimo/_ai/llm/_impl.py b/marimo/_ai/llm/_impl.pyindex 724af4ee11..e99858373b 100644--- a/marimo/_ai/llm/_impl.py+++ b/marimo/_ai/llm/_impl.py@@ -48,6 +48,36 @@ def _looks_like_streaming_error(e: Exception) -> bool:)+def _start_stream(+ chunks: Generator[str, None, None],+) -> Generator[str, None, None]:+ """Return a generator that replays the first chunk, then yields the rest.++ Starting it here raises a streaming error inside the caller's try block, so non-streaming models can fallback safely.+ """+ first = next(chunks, None)++ def resume() -> Generator[str, None, None]:+ if first is not None:+ yield first+ yield from chunks++ return resume()+++def _stream_or_fallback(+ stream: Callable[[], Generator[str, None, None]],+ fallback: Callable[[], str],+) -> Generator[str, None, None] | str:+ """Stream the response, or call `fallback` if the model can't stream."""+ try:+ return _start_stream(stream())+ except Exception as e:+ if _looks_like_streaming_error(e):+ return fallback()+ raise++class simple(ChatModel):"""Convenience class for wrapping a ChatModel or callable to@@ -303,17 +333,23 @@ class anthropic(ChatModel):):params["temperature"] = config.temperature- # Try streaming first, fall back to non-streaming on error- try:- # Note: client.messages.stream() doesn't take a 'stream' parameter- # It's already a streaming method- return self._stream_response(client, params)- except Exception as e:- # Fall back to non-streaming mode if streaming fails- if _looks_like_streaming_error(e):- response = client.messages.create(**params)- return response.content[0].text- raise+ def non_streaming() -> str:+ response = client.messages.create(**params)+ # The first block can be a thinking block, which has no text+ return next(+ (+ block.text+ for block in response.content+ if block.type == "text"+ ),+ "",+ )++ # Note: client.messages.stream() doesn't take a 'stream' parameter+ # It's already a streaming method+ return _stream_or_fallback(+ lambda: self._stream_response(client, params), non_streaming+ )class google(ChatModel):@@ -407,21 +443,20 @@ class google(ChatModel):"presence_penalty": config.presence_penalty,}- # Try streaming first, fall back to non-streaming on error- try:- return self._stream_response(- client, google_messages, generation_config+ def non_streaming() -> str:+ response = client.models.generate_content(+ model=self.model,+ contents=google_messages,+ config=generation_config, # type: ignore[arg-type])- except Exception as e:- # Fall back to non-streaming mode if streaming fails- if _looks_like_streaming_error(e):- response = client.models.generate_content(- model=self.model,- contents=google_messages,- config=generation_config, # type: ignore[arg-type]- )- return response.text- raise+ return response.text or ""++ return _stream_or_fallback(+ lambda: self._stream_response(+ client, google_messages, generation_config+ ),+ non_streaming,+ )class groq(ChatModel):@@ -517,23 +552,22 @@ class groq(ChatModel):+ messages)- # Try streaming first, fall back to non-streaming on error- try:- return self._stream_response(client, groq_messages, config)- except Exception as e:- # Fall back to non-streaming mode if streaming fails- if _looks_like_streaming_error(e):- response = client.chat.completions.create(- model=self.model,- messages=groq_messages,- max_tokens=config.max_tokens,- temperature=config.temperature,- top_p=config.top_p,- stop=None,- stream=False,- )- return response.choices[0].message.content or ""- raise+ def non_streaming() -> str:+ response = client.chat.completions.create(+ model=self.model,+ messages=groq_messages,+ max_tokens=config.max_tokens,+ temperature=config.temperature,+ top_p=config.top_p,+ stop=None,+ stream=False,+ )+ return response.choices[0].message.content or ""++ return _stream_or_fallback(+ lambda: self._stream_response(client, groq_messages, config),+ non_streaming,+ )class bedrock(ChatModel):@@ -643,19 +677,21 @@ class bedrock(ChatModel):client = self._create_client()params = self._build_converse_params(messages, config)+ def non_streaming() -> str:+ response = client.converse(**params)+ output = response.get("output", {})+ message = output.get("message", {})+ content = message.get("content", [])+ # The first block can be reasoning, which has no text+ return next(+ (block["text"] for block in content if "text" in block),+ "",+ )+try:- try:- return self._stream_response(client, params)- except Exception as stream_error:- if _looks_like_streaming_error(stream_error):- response = client.converse(**params)- output = response.get("output", {})- message = output.get("message", {})- content = message.get("content", [])- if content and "text" in content[0]:- return content[0]["text"]- return ""- raise+ return _stream_or_fallback(+ lambda: self._stream_response(client, params), non_streaming+ )except Exception as e:error_msg = str(e)diff --git a/tests/_ai/llm/test_impl.py b/tests/_ai/llm/test_impl.pyindex d9d8ed10e7..ba865a771b 100644--- a/tests/_ai/llm/test_impl.py+++ b/tests/_ai/llm/test_impl.py@@ -1750,3 +1750,106 @@ def test_pydantic_ai_require() -> None:mock_agent = MagicMock()with pytest.raises(ModuleNotFoundError):pydantic_ai(mock_agent)+++class TestStreamingFallback:+ """A model that rejects streaming gets a non-streaming request instead.++ The streaming request is made lazily inside a generator, so these tests+ fake the SDKs and check the fallback end to end, without them installed.+ """++ STREAMING_ERROR = "This model does not support streaming"+ messages = [ChatMessage(role="user", content="Test prompt")] # noqa: RUF012++ def _streaming_error(self, *_args: Any, **_kwargs: Any) -> Any:+ raise RuntimeError(self.STREAMING_ERROR)++ def test_anthropic(self) -> None:+ client = MagicMock()+ client.messages.stream.side_effect = self._streaming_error+ client.messages.create.return_value = MagicMock(+ content=[+ MagicMock(type="thinking", spec=["type", "thinking"]),+ MagicMock(type="text", text="Fallback response"),+ ]+ )+ fake_sdk = MagicMock(Anthropic=MagicMock(return_value=client))++ with (+ patch.dict("sys.modules", {"anthropic": fake_sdk}),+ patch.object(DependencyManager.anthropic, "require"),+ ):+ model = anthropic("claude-x", api_key="test-key")+ result = model(self.messages, ChatModelConfig())++ assert result == "Fallback response"+ client.messages.create.assert_called_once()++ def test_google(self) -> None:+ client = MagicMock()+ client.models.generate_content_stream.side_effect = (+ self._streaming_error+ )+ client.models.generate_content.return_value = MagicMock(+ text="Fallback response"+ )+ genai = MagicMock(Client=MagicMock(return_value=client))+ fake_sdk = MagicMock(genai=genai)++ with (+ patch.dict(+ "sys.modules", {"google": fake_sdk, "google.genai": genai}+ ),+ patch.object(DependencyManager.google_ai, "require"),+ ):+ model = google("gemini-x", api_key="test-key")+ result = model(self.messages, ChatModelConfig())++ assert result == "Fallback response"+ client.models.generate_content.assert_called_once()++ def test_groq(self) -> None:+ client = MagicMock()++ def create(**kwargs: Any) -> Any:+ if kwargs.get("stream"):+ self._streaming_error()+ message = MagicMock(content="Fallback response")+ return MagicMock(choices=[MagicMock(message=message)])++ client.chat.completions.create.side_effect = create+ fake_sdk = MagicMock(Groq=MagicMock(return_value=client))++ with (+ patch.dict("sys.modules", {"groq": fake_sdk}),+ patch.object(DependencyManager.groq, "require"),+ ):+ model = groq("llama-x", api_key="test-key")+ result = model(self.messages, ChatModelConfig())++ assert result == "Fallback response"+ assert (+ client.chat.completions.create.call_args.kwargs["stream"] is False+ )++ def test_bedrock(self) -> None:+ client = MagicMock()+ client.converse_stream.side_effect = self._streaming_error+ client.converse.return_value = {+ "output": {+ "message": {+ "content": [+ {"reasoningContent": {"reasoningText": {"text": "…"}}},+ {"text": "Fallback response"},+ ]+ }+ }+ }++ model = bedrock("bedrock/anthropic.claude-x")+ with patch.object(model, "_create_client", return_value=client):+ result = model(self.messages, ChatModelConfig())++ assert result == "Fallback response"+ client.converse.assert_called_once()