Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

improvement: handle errors in stream responses (#6153)

Myles Scolnick77bf5a74fc1d
diff --git a/marimo/_ai/_convert.py b/marimo/_ai/_convert.py
index 568ec9d4cd..86771f4c0a 100644
--- a/marimo/_ai/_convert.py
+++ b/marimo/_ai/_convert.py
@@ -427,11 +427,13 @@ def convert_to_ai_sdk_messages(
"tool_call_end",
"tool_result",
"finish_reason",
+ "error",
],
) -> str:
"""
Format text events for the AI SDK stream protocol.
- See: https://ai-sdk.dev/docs/ai-sdk-ui/stream-protocol
+ This follows the data-stream v1 protocol
+ See: https://v4.ai-sdk.dev/docs/ai-sdk-ui/stream-protocol
"""
TEXT_PREFIX = "0:"
REASON_PREFIX = "g:"
@@ -441,6 +443,7 @@ def convert_to_ai_sdk_messages(
TOOL_CALL_PREFIX = "9:"
TOOL_RESULT_PREFIX = "a:"
FINISH_REASON_PREFIX = "d:"
+ ERROR_PREFIX = "3:"
# Text events
if content_type == "text" and isinstance(content_text, str):
@@ -472,6 +475,10 @@ def convert_to_ai_sdk_messages(
):
return f"{REASON_SIGNATURE_PREFIX}{json.dumps(content_text)}\n"
+ # Error events
+ elif content_type == "error" and isinstance(content_text, str):
+ return f"{ERROR_PREFIX}{json.dumps(content_text)}\n"
+
else:
# Default to text for unknown types
return f"{TEXT_PREFIX}{json.dumps(content_text)}\n"
diff --git a/marimo/_server/api/endpoints/ai.py b/marimo/_server/api/endpoints/ai.py
index 906feaa4c3..6a7ab395a2 100644
--- a/marimo/_server/api/endpoints/ai.py
+++ b/marimo/_server/api/endpoints/ai.py
@@ -13,6 +13,7 @@ from starlette.responses import (
)
from marimo import _loggers
+from marimo._ai._convert import convert_to_ai_sdk_messages
from marimo._ai._types import ChatMessage
from marimo._config.config import AiConfig, MarimoConfig
from marimo._server.ai.config import (
@@ -51,6 +52,8 @@ from marimo._server.models.models import (
from marimo._server.router import APIRouter
if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator
+
from starlette.requests import Request
@@ -60,6 +63,33 @@ LOGGER = _loggers.marimo_logger()
router = APIRouter()
+async def safe_stream_wrapper(
+ stream_generator: AsyncGenerator[str, None],
+ text_only: bool,
+) -> AsyncGenerator[str, None]:
+ """
+ Wraps a streaming generator to catch and handle errors gracefully.
+
+ Args:
+ stream_generator: The original streaming generator
+ text_only: Whether to return text only or the full AI SDK stream protocol format
+
+ Yields:
+ Stream chunks or error messages in AI SDK stream protocol format
+ """
+ try:
+ async for chunk in stream_generator:
+ yield chunk
+ except Exception as e:
+ LOGGER.error("Error in AI streaming response: %s", str(e))
+ # Send an error message using AI SDK stream protocol format
+ # Error Part format: 3:string\n
+ if text_only:
+ yield str(e)
+ else:
+ yield convert_to_ai_sdk_messages(str(e), "error")
+
+
def get_ai_config(config: MarimoConfig) -> AiConfig:
ai_config = config.get("ai", None)
LOGGER.debug(f"ai_config: {ai_config}")
@@ -127,12 +157,16 @@ async def ai_completion(
)
return StreamingResponse(
- content=without_wrapping_backticks(
- provider.as_stream_response(
- response, StreamOptions(text_only=True)
- )
+ content=safe_stream_wrapper(
+ without_wrapping_backticks(
+ provider.as_stream_response(
+ response, StreamOptions(text_only=True)
+ )
+ ),
+ text_only=True,
),
media_type="application/json",
+ headers={"x-vercel-ai-data-stream": "v1"},
)
@@ -183,10 +217,14 @@ async def ai_chat(
)
return StreamingResponse(
- content=provider.as_stream_response(
- response, StreamOptions(format_stream=True)
+ content=safe_stream_wrapper(
+ provider.as_stream_response(
+ response, StreamOptions(format_stream=True, text_only=False)
+ ),
+ text_only=False,
),
media_type="application/json",
+ headers={"x-vercel-ai-data-stream": "v1"},
)
@@ -236,13 +274,20 @@ async def ai_inline_completion(
provider_config.tools.clear()
provider = get_completion_provider(provider_config, model=model)
- response = await provider.stream_completion(
- messages=messages,
- system_prompt=system_prompt,
- max_tokens=INLINE_COMPLETION_MAX_TOKENS,
- )
+ try:
+ response = await provider.stream_completion(
+ messages=messages,
+ system_prompt=system_prompt,
+ max_tokens=INLINE_COMPLETION_MAX_TOKENS,
+ )
- content = await provider.collect_stream(response)
+ content = await provider.collect_stream(response)
+ except Exception as e:
+ LOGGER.error("Error in AI inline completion: %s", str(e))
+ raise HTTPException(
+ status_code=500, # Internal Server Error
+ detail=f"AI completion failed: {str(e)}",
+ ) from None
# Filter out `<|file_separator|>` which is sometimes returned FIM models
content = content.replace("<|file_separator|>", "")
@@ -280,17 +325,28 @@ async def invoke_tool(
body = await parse_request(request, cls=InvokeAiToolRequest)
- # Invoke the tool
- result = await get_tool_manager().invoke_tool(
- body.tool_name, body.arguments
- )
+ try:
+ # Invoke the tool
+ result = await get_tool_manager().invoke_tool(
+ body.tool_name, body.arguments
+ )
- # Create and return the response
- response = InvokeAiToolResponse(
- success=result.error is None,
- tool_name=result.tool_name,
- result=result.result,
- error=result.error,
- )
+ # Create and return the response
+ response = InvokeAiToolResponse(
+ success=result.error is None,
+ tool_name=result.tool_name,
+ result=result.result,
+ error=result.error,
+ )
- return JSONResponse(content=asdict(response))
+ return JSONResponse(content=asdict(response))
+ except Exception as e:
+ LOGGER.error("Error invoking AI tool %s: %s", body.tool_name, str(e))
+ # Return error response instead of letting it crash
+ error_response = InvokeAiToolResponse(
+ success=False,
+ tool_name=body.tool_name,
+ result=None,
+ error=f"Tool invocation failed: {str(e)}",
+ )
+ return JSONResponse(content=asdict(error_response))
diff --git a/tests/_ai/test_chat_convert.py b/tests/_ai/test_chat_convert.py
index ae3a476afa..13ba500abf 100644
--- a/tests/_ai/test_chat_convert.py
+++ b/tests/_ai/test_chat_convert.py
@@ -452,8 +452,13 @@ def test_convert_to_ai_sdk_messages():
expected = 'd:{"finishReason": "stop", "usage": {"promptTokens": 0, "completionTokens": 0}}\n'
assert result == expected
- # Test unknown type defaults to text
- result = convert_to_ai_sdk_messages("fallback", "unknown")
+ # Test error type
+ error_message = "Model not found"
+ result = convert_to_ai_sdk_messages(error_message, "error")
+ assert result == f"3:{json.dumps(error_message)}\n"
+
+ # Test unknown type defaults to text (using type ignore for testing)
+ result = convert_to_ai_sdk_messages("fallback", "unknown") # type: ignore
assert result == f"0:{json.dumps('fallback')}\n"