Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

migrate remaining providers to pydantic-ai (#7732)

Shahmir Varqha0c70ca8fd7f3
diff --git a/frontend/src/components/chat/chat-utils.ts b/frontend/src/components/chat/chat-utils.ts
index b08a5d5042..22cf7e9f7c 100644
--- a/frontend/src/components/chat/chat-utils.ts
+++ b/frontend/src/components/chat/chat-utils.ts
@@ -8,7 +8,6 @@ import type {
InvokeAiToolRequest,
InvokeAiToolResponse,
} from "@/core/network/types";
-import type { ChatMessage } from "@/plugins/impl/chat/types";
import { blobToString } from "@/utils/fileToBase64";
import { Logger } from "@/utils/Logger";
import { getAICompletionBodyWithAttachments } from "../editor/ai/completion-utils";
@@ -68,7 +67,6 @@ function stringifyTextParts(parts: UIMessage["parts"]): string {
export async function buildCompletionRequestBody(
messages: UIMessage[],
): Promise<{
- messages: ChatMessage[]; // Deprecated. TODO: Remove in the future
uiMessages: UIMessage[];
context?: (null | components["schemas"]["AiCompletionContext"]) | undefined;
includeOtherCode: string;
@@ -91,25 +89,8 @@ export async function buildCompletionRequestBody(
};
}
- function toChatMessage(message: UIMessage, isLast: boolean): ChatMessage {
- // Clone parts to avoid mutating the original message
- const parts = [...message.parts];
- if (isLast) {
- parts.push(...completionBody.attachments);
- }
- return {
- id: message.id,
- role: message.role,
- content: stringifyTextParts(message.parts), // This is no longer used in the backend
- parts,
- };
- }
-
return {
...completionBody.body,
- messages: messages.map((m, idx) =>
- toChatMessage(m, idx === messages.length - 1),
- ),
uiMessages: messages.map((m, idx) =>
addAttachmentsToMessage(m, idx === messages.length - 1),
),
diff --git a/frontend/src/components/editor/ai/completion-handlers.tsx b/frontend/src/components/editor/ai/completion-handlers.tsx
index ec79442968..a6b8651a02 100644
--- a/frontend/src/components/editor/ai/completion-handlers.tsx
+++ b/frontend/src/components/editor/ai/completion-handlers.tsx
@@ -124,7 +124,7 @@ export const AcceptCompletionButton: React.FC<{
size={size}
disabled={isLoading}
onClick={handleAcceptAndRun}
- className={`${baseClasses} rounded-l-none px-1.5 ${borderless && "border-0 border-l-1"} ${playButtonStyles}`}
+ className={`${baseClasses} rounded-l-none px-1.5 ${borderless && "border-0 border-l"} ${playButtonStyles}`}
>
<PlayIcon className="h-2.5 w-2.5" />
</Button>
diff --git a/frontend/src/plugins/impl/chat/types.ts b/frontend/src/plugins/impl/chat/types.ts
index eeb0c241d8..28392fd528 100644
--- a/frontend/src/plugins/impl/chat/types.ts
+++ b/frontend/src/plugins/impl/chat/types.ts
@@ -8,18 +8,6 @@ export interface ChatMessage extends UIMessage {
content: string | null; // Content is only added for backwards compatibility
}
-export interface SendMessageRequest {
- messages: ChatMessage[];
- config: {
- max_tokens: number | null;
- temperature: number | null;
- top_p: number | null;
- top_k: number | null;
- frequency_penalty: number | null;
- presence_penalty: number | null;
- };
-}
-
/**
* These are snake_case because they come from the backend,
* and are not modified when sent to the frontend.
@@ -32,3 +20,8 @@ export interface ChatConfig {
frequency_penalty: number | null;
presence_penalty: number | null;
}
+
+export interface SendMessageRequest {
+ messages: ChatMessage[];
+ config: ChatConfig;
+}
diff --git a/marimo/_ai/_convert.py b/marimo/_ai/_convert.py
index 6a3e842b62..832640f875 100644
--- a/marimo/_ai/_convert.py
+++ b/marimo/_ai/_convert.py
@@ -17,7 +17,6 @@ from marimo._ai._types import (
TextPart,
ToolInvocationPart,
)
-from marimo._server.ai.tools.types import ToolDefinition
if TYPE_CHECKING:
from anthropic.types import ( # type: ignore[import-not-found]
@@ -563,82 +562,3 @@ def convert_to_ai_sdk_messages(
if text_id is None:
text_id = f"text_{uuid.uuid4().hex}"
return f"data: {json.dumps({'type': 'text-delta', 'id': text_id, 'delta': str(content_text)})}\n\n"
-
-
-# Tool conversions
-def convert_to_openai_tools(
- tools: list[ToolDefinition],
-) -> list[dict[str, Any]]:
- return [
- {
- "type": "function",
- "function": {
- "name": tool.name,
- "description": tool.description,
- "parameters": tool.parameters,
- },
- }
- for tool in tools
- ]
-
-
-def convert_to_anthropic_tools(
- tools: list[ToolDefinition],
-) -> list[dict[str, Any]]:
- return [
- {
- "name": tool.name,
- "description": tool.description,
- "input_schema": tool.parameters,
- }
- for tool in tools
- ]
-
-
-def convert_to_google_tools(
- tools: list[ToolDefinition],
-) -> list[dict[str, Any]]:
- return [
- {
- "function_declarations": [
- {
- "name": tool.name,
- "description": tool.description,
- "parameters": _clean_google_parameters(
- {
- # Pydantic will raise validation errors if unknown keys are present
- # So we only include necessary keys
- "type": tool.parameters.get("type", "object"),
- "properties": tool.parameters.get(
- "properties", {}
- ),
- "required": tool.parameters.get("required", []),
- }
- ),
- }
- ]
- }
- for tool in tools
- ]
-
-
-def _clean_google_parameters(parameters: dict[str, Any]) -> dict[str, Any]:
- """
- Recursively remove unsupported keys from Google tool parameters.
- Google has strict parameter validation, so we should remove any keys that are not supported.
- """
- cleaned = {}
- for key, value in parameters.items():
- if key in ("additionalProperties"):
- continue
- if key == "properties" and isinstance(value, dict):
- # Recursively clean nested properties
- cleaned[key] = {
- k: _clean_google_parameters(v) if isinstance(v, dict) else v
- for k, v in value.items()
- }
- elif isinstance(value, dict):
- cleaned[key] = _clean_google_parameters(value)
- else:
- cleaned[key] = value
- return cleaned
diff --git a/marimo/_server/ai/providers.py b/marimo/_server/ai/providers.py
index 74a9f03d09..2436228a81 100644
--- a/marimo/_server/ai/providers.py
+++ b/marimo/_server/ai/providers.py
@@ -1,11 +1,8 @@
# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
-import json
import os
-import uuid
from abc import ABC, abstractmethod
-from collections.abc import AsyncGenerator, AsyncIterator
from dataclasses import dataclass
from typing import (
TYPE_CHECKING,
@@ -14,26 +11,19 @@ from typing import (
Literal,
Optional,
TypeVar,
- Union,
- cast,
+ get_args,
)
from urllib.parse import parse_qs, urlparse
from starlette.exceptions import HTTPException
from marimo import _loggers
-from marimo._ai._convert import (
- convert_to_ai_sdk_messages,
- convert_to_openai_messages,
- convert_to_openai_tools,
- extract_text,
-)
+from marimo._ai._convert import extract_text
from marimo._ai._pydantic_ai_utils import (
convert_to_pydantic_messages,
form_toolsets,
generate_id,
)
-from marimo._ai._types import ChatMessage
from marimo._dependencies.dependencies import Dependency, DependencyManager
from marimo._server.ai.config import AnyProviderConfig
from marimo._server.ai.ids import AiModelId
@@ -42,25 +32,18 @@ from marimo._server.ai.tools.types import ToolDefinition
from marimo._server.models.completion import UIMessage as ServerUIMessage
from marimo._utils.http import HTTPStatus
-TIMEOUT = 30
-# Long-thinking models can take a long time to complete, so we set a longer timeout
-LONG_THINKING_TIMEOUT = 120
-
if TYPE_CHECKING:
- # Used for Bedrock, unified interface for all models
+ from collections.abc import AsyncGenerator, AsyncIterator
+
from anthropic.types.beta import BetaThinkingConfigParam
- from openai import ( # type: ignore[import-not-found]
- AsyncOpenAI,
- AsyncStream as OpenAiStream,
- )
- from openai.types.chat import ( # type: ignore[import-not-found]
- ChatCompletionChunk,
- )
+ from openai import AsyncOpenAI
+ from openai.types.shared.reasoning_effort import ReasoningEffort
from pydantic_ai import Agent, DeferredToolRequests, FunctionToolset
from pydantic_ai.messages import ThinkingPart
from pydantic_ai.models import Model
from pydantic_ai.models.bedrock import BedrockConverseModel
from pydantic_ai.models.google import GoogleModel
+ from pydantic_ai.models.openai import OpenAIChatModel, OpenAIResponsesModel
from pydantic_ai.providers import Provider
from pydantic_ai.providers.anthropic import (
AnthropicProvider as PydanticAnthropic,
@@ -69,45 +52,13 @@ if TYPE_CHECKING:
BedrockProvider as PydanticBedrock,
)
from pydantic_ai.providers.google import GoogleProvider as PydanticGoogle
+ from pydantic_ai.providers.openai import OpenAIProvider as PydanticOpenAI
from pydantic_ai.ui.vercel_ai import VercelAIAdapter
from pydantic_ai.ui.vercel_ai.request_types import UIMessage, UIMessagePart
from pydantic_ai.ui.vercel_ai.response_types import BaseChunk
from starlette.responses import StreamingResponse
-ResponseT = TypeVar("ResponseT")
-StreamT = TypeVar("StreamT", bound=AsyncIterator[Any])
-FinishReason = Literal["tool_calls", "stop"]
-
-# Types for extract_content method return
-DictContent = tuple[
- dict[str, Any],
- Literal[
- "tool_call_start",
- "tool_call_end",
- "reasoning_signature",
- "tool_call_delta",
- ],
-]
-TextContent = tuple[str, Literal["text", "reasoning"]]
-ExtractedContent = Union[TextContent, DictContent]
-ExtractedContentList = list[ExtractedContent]
-
-# Types for format_stream method parameter
-FinishContent = tuple[FinishReason, Literal["finish_reason"]]
-# StreamContent
-StreamTextContent = tuple[str, Literal["text", "reasoning"]]
-StreamDictContent = tuple[
- dict[str, Any],
- Literal[
- "tool_call_start",
- "tool_call_end",
- "tool_call_delta",
- "reasoning_signature",
- ],
-]
-StreamContent = Union[StreamTextContent, StreamDictContent, FinishContent]
-
LOGGER = _loggers.marimo_logger()
@@ -135,6 +86,14 @@ class PydanticProvider(ABC, Generic[ProviderT]):
config: AnyProviderConfig,
deps: list[Dependency] | None = None,
):
+ """
+ Initialize a Pydantic provider.
+
+ Args:
+ model: The model name.
+ config: The provider config.
+ deps: The dependencies to require.
+ """
DependencyManager.require_many(
"for AI assistance", DependencyManager.pydantic_ai, *(deps or [])
)
@@ -320,352 +279,10 @@ class GoogleProvider(PydanticProvider["PydanticGoogle"]):
)
-class CompletionProvider(Generic[ResponseT, StreamT], ABC):
- """Base class for AI completion providers."""
-
- def __init__(self, model: str, config: AnyProviderConfig):
- self.model = model
- self.config = config
-
- @abstractmethod
- async def stream_completion(
- self,
- messages: list[ChatMessage],
- system_prompt: str,
- max_tokens: int,
- additional_tools: list[ToolDefinition],
- ) -> StreamT:
- """Create a completion stream."""
- pass
-
- @abstractmethod
- def extract_content(
- self, response: ResponseT, tool_call_ids: Optional[list[str]] = None
- ) -> Optional[ExtractedContentList]:
- """Extract content from a response chunk."""
- pass
-
- @abstractmethod
- def get_finish_reason(self, response: ResponseT) -> Optional[FinishReason]:
- """Get the stop reason for a response."""
- pass
-
- def format_stream(self, content: StreamContent) -> str:
- """Format a response into stream protocol string."""
- content_text, content_type = content
- if content_type in [
- "text",
- "text_start",
- "text_end",
- "reasoning",
- "reasoning_start",
- "reasoning_end",
- "reasoning_signature",
- "tool_call_start",
- "tool_call_delta",
- "tool_call_end",
- "finish_reason",
- ]:
- return convert_to_ai_sdk_messages(content_text, content_type)
- return ""
-
- async def collect_stream(self, response: StreamT) -> str:
- """Collect a stream into a single string."""
- result: list[str] = []
- async for chunk in self.as_stream_response(
- response, StreamOptions(text_only=True)
- ):
- result.append(chunk)
- return "".join(result)
-
- def _content_to_string(
- self, content_data: Union[str, dict[str, Any]]
- ) -> str:
- """Convert content data to string for buffer operations."""
- return (
- json.dumps(content_data)
- if isinstance(content_data, dict)
- else str(content_data)
- )
-
- def _create_stream_content(
- self, content_data: Union[str, dict[str, Any]], content_type: str
- ) -> StreamContent:
- """Create type-safe StreamContent tuple for format_stream method."""
- # String content types
- if isinstance(content_data, str):
- if content_type == "text":
- return (content_data, "text")
- elif content_type == "reasoning":
- return (content_data, "reasoning")
-
- # Dict content types
- if isinstance(content_data, dict):
- if content_type == "tool_call_start":
- return (content_data, "tool_call_start")
- elif content_type == "tool_call_end":
- return (content_data, "tool_call_end")
- elif content_type == "tool_call_delta":
- return (content_data, "tool_call_delta")
- elif content_type == "reasoning_signature":
- return (content_data, "reasoning_signature")
-
- # Fallback - convert to string content
- content_str = self._content_to_string(content_data)
- return (content_str, "text")
-
- def validate_tool_call_args(
- self, tool_call_args: str
- ) -> Optional[dict[str, Any]]:
- """Validate tool call arguments."""
- if not tool_call_args:
- return None
- try:
- result = (
- json.loads(tool_call_args)
- if isinstance(tool_call_args, str)
- else tool_call_args
- )
- return result if isinstance(result, dict) else None
- except Exception as e:
- LOGGER.error(
- f"Failed to parse tool call arguments: {tool_call_args} (error: {e})"
- )
- raise HTTPException(
- status_code=HTTPStatus.BAD_REQUEST,
- detail=f"Invalid tool call arguments: malformed JSON: {tool_call_args}",
- ) from e
-
- async def as_stream_response(
- self, response: StreamT, options: Optional[StreamOptions] = None
- ) -> AsyncGenerator[str, None]:
- """Convert a stream to an async generator of strings."""
- original_content = ""
- buffer = ""
- options = options or StreamOptions()
-
- # Tool info collected from the first chunk
- tool_calls: dict[str, ActiveToolCall] = {}
- tool_calls_order: list[str] = []
-
- # Finish reason collected from the last chunk
- finish_reason: Optional[FinishReason] = None
-
- # Text block tracking for start/delta/end pattern
- current_text_id: Optional[str] = None
- current_reasoning_id: Optional[str] = None
- has_text_started = False
- has_reasoning_started = False
-
- async for chunk in response:
- # Always check for finish reason first, before checking content
- # Some chunks (like RawMessageDeltaEvent) contain finish reasons but no extractable content
- # If we check content first, these chunks get skipped and finish reason is never detected
- finish_reason = self.get_finish_reason(chunk) or finish_reason
-
- content = self.extract_content(chunk, tool_calls_order)
- if not content:
- continue
-
- # Loop through all content chunks
- for content_data, content_type in content:
- if options.text_only and content_type != "text":
- continue
-
- # Handle text content with start/delta/end pattern
- if (
- content_type == "text"
- and isinstance(content_data, str)
- and options.format_stream
- ):
- if not has_text_started:
- # Emit text-start event
- current_text_id = f"text_{uuid.uuid4().hex}"
- yield convert_to_ai_sdk_messages(
- "", "text_start", current_text_id
- )
- has_text_started = True
-
- # Emit text-delta event with the actual content
- yield convert_to_ai_sdk_messages(
- content_data, "text", current_text_id
- )
- continue
-
- # Handle reasoning content with start/delta/end pattern
- elif (
- content_type == "reasoning"
- and isinstance(content_data, str)
- and options.format_stream
- ):
- if not has_reasoning_started:
- # Emit reasoning-start event
- current_reasoning_id = f"reasoning_{uuid.uuid4().hex}"
- yield convert_to_ai_sdk_messages(
- "", "reasoning_start", current_reasoning_id
- )
- has_reasoning_started = True
-
- # Emit reasoning-delta event with the actual content
- yield convert_to_ai_sdk_messages(
- content_data, "reasoning", current_reasoning_id
- )
- continue
-
- # Tool handling
- if content_type == "tool_call_start" and isinstance(
- content_data, dict
- ):
- tool_call_id: Optional[str] = content_data.get(
- "toolCallId", None
- )
- tool_call_name: Optional[str] = content_data.get(
- "toolName", None
- )
- # Sometimes GoogleProvider emits the args in the tool_call_start chunk
- tool_call_args: str = ""
- if content_data.get("args"):
- # don't yield args in tool_call_start chunk
- # it will throw an error in ai-sdk-ui
- tool_call_args = content_data.pop("args")
-
- if tool_call_id and tool_call_name:
- # Add new tool calls to the list for tracking
- tool_calls_order.append(tool_call_id)
- tool_calls[tool_call_id] = ActiveToolCall(
- tool_call_id=tool_call_id,
- tool_call_name=tool_call_name,
- tool_call_args=tool_call_args,
- )
-
- if content_type == "tool_call_delta" and isinstance(
- content_data, dict
- ):
- tool_call_delta_id = content_data.get("toolCallId", None)
- tool_call_delta: str = content_data.get(
- "inputTextDelta", ""
- )
-
- if not tool_call_delta_id:
- if not tool_call_delta_id:
- LOGGER.error(
- f"Tool call id not found for tool call delta: {content_data}"
- )
- continue
- tool_call = tool_calls.get(tool_call_delta_id, None)
- if not tool_call:
- continue
-
- # tool_call_args is built up incrementally from deltas.
- tool_call.tool_call_args += tool_call_delta
- # update tool_call_delta to ai-sdk-ui structure
- # based on https://ai-sdk.dev/docs/ai-sdk-ui/stream-protocol#tool-call-delta-part
- content_data = {
- "toolCallId": tool_call.tool_call_id,
- "inputTextDelta": tool_call.tool_call_args,
- }
-
- content_str = self._content_to_string(content_data)
-
- if options.format_stream:
- stream_content = self._create_stream_content(
- content_data, content_type
- )
- content_str = self.format_stream(stream_content)
-
- buffer += content_str
- original_content += content_str
-
- yield buffer
- buffer = ""
-
- # Emit text-end event if we started a text block
- if has_text_started and current_text_id and options.format_stream:
- yield convert_to_ai_sdk_messages("", "text_end", current_text_id)
-
- # Emit reasoning-end event if we started a reasoning block
- if (
- has_reasoning_started
- and current_reasoning_id
- and options.format_stream
- ):
- yield convert_to_ai_sdk_messages(
- "", "reasoning_end", current_reasoning_id
- )
-
- # Handle tool call end after the stream is complete
- if len(tool_calls_order) > 0 and not options.text_only:
- for tool_call_id in tool_calls_order:
- tool_call = tool_calls.get(tool_call_id, None)
- if not tool_call:
- continue
- content_data = {
- "toolCallId": tool_call_id,
- "toolName": tool_call.tool_call_name,
- "input": self.validate_tool_call_args(
- tool_call.tool_call_args
- ),
- }
- content_type = "tool_call_end"
- yield self.format_stream((content_data, content_type))
-
- # Add a final finish reason chunk
- if finish_reason and not options.text_only:
- finish_content: FinishContent = (finish_reason, "finish_reason")
- yield self.format_stream(finish_content)
- # reset finish reason for next stream
- finish_reason = None
-
- LOGGER.debug(f"Completion content: {original_content}")
-
-
-class OpenAIProvider(
- CompletionProvider[
- "ChatCompletionChunk", "OpenAiStream[ChatCompletionChunk]"
- ]
-):
- # Medium effort provides a balance between speed and accuracy
- # https://openai.com/index/openai-o3-mini/
- DEFAULT_REASONING_EFFORT = "medium"
-
- def _is_reasoning_model(self, model: str) -> bool:
- """
- Check if reasoning_effort should be added to the request.
- Only add for actual OpenAI reasoning models, not for OpenAI-compatible APIs.
-
- OpenAI-compatible APIs (identified by custom base_url) may not support
- the reasoning_effort parameter even if the model name suggests it's a
- reasoning model.
- """
- import re
-
- # Check for reasoning model patterns: o{digit} or gpt-5, with optional openai/ prefix
- reasoning_patterns = [
- r"^openai/o\d", # openai/o1, openai/o3, etc.
- r"^o\d", # o1, o3, etc.
- r"^openai/gpt-5", # openai/gpt-5*
- r"^gpt-5", # gpt-5*
- ]
-
- is_reasoning_model_name = any(
- re.match(pattern, model) for pattern in reasoning_patterns
- )
-
- if not is_reasoning_model_name:
- return False
-
- # If using a custom base_url that's not OpenAI, don't assume reasoning is supported
- if (
- self.config.base_url
- and "api.openai.com" not in self.config.base_url
- ):
- return False
-
- return True
-
- def get_client(self, config: AnyProviderConfig) -> AsyncOpenAI:
- DependencyManager.openai.require(why="for AI assistance with OpenAI")
+class OpenAIClientMixin:
+ """Mixin providing OpenAI client creation logic for OpenAI-based providers."""
+ def get_openai_client(self, config: AnyProviderConfig) -> AsyncOpenAI:
import ssl
from pathlib import Path
@@ -742,129 +359,78 @@ class OpenAIProvider(
project=project,
)
- async def stream_completion(
- self,
- messages: list[ChatMessage],
- system_prompt: str,
- max_tokens: int,
- additional_tools: list[ToolDefinition],
- ) -> OpenAiStream[ChatCompletionChunk]:
- client = self.get_client(self.config)
- tools = self.config.tools
- create_params = {
- "model": self.model,
- "messages": cast(
- Any,
- convert_to_openai_messages(
- self._maybe_convert_roles(
- [ChatMessage(role="system", content=system_prompt)]
- )
- + messages
- ),
- ),
- "stream": True,
- "timeout": LONG_THINKING_TIMEOUT
- if self._is_reasoning_model(self.model)
- else TIMEOUT,
- }
- if tools:
- all_tools = tools + additional_tools
- create_params["tools"] = convert_to_openai_tools(all_tools)
- if self._is_reasoning_model(self.model):
- create_params["reasoning_effort"] = self.DEFAULT_REASONING_EFFORT
- create_params["max_completion_tokens"] = max_tokens
- else:
- create_params["max_tokens"] = max_tokens
- return cast(
- "OpenAiStream[ChatCompletionChunk]",
- await client.chat.completions.create(**create_params),
+
+class OpenAIProvider(OpenAIClientMixin, PydanticProvider["PydanticOpenAI"]):
+ # Medium effort provides a balance between speed and accuracy
+ # https://openai.com/index/openai-o3-mini/
+ DEFAULT_REASONING_EFFORT: ReasoningEffort = "medium"
+ DEFAULT_REASONING_SUMMARY: Literal["detailed", "concise", "auto"] = "auto"
+
+ def create_provider(self, config: AnyProviderConfig) -> PydanticOpenAI:
+ from pydantic_ai.providers.openai import (
+ OpenAIProvider as PydanticOpenAI,
)
- def extract_content(
- self,
- response: ChatCompletionChunk,
- tool_call_ids: Optional[list[str]] = None,
- ) -> Optional[ExtractedContentList]:
- tool_call_ids = tool_call_ids or []
- if (
- hasattr(response, "choices")
- and response.choices
- and response.choices[0].delta
- ):
- delta = response.choices[0].delta
-
- # Text content
- content = delta.content
- if content:
- return [(content, "text")]
-
- # Tool call:
- if delta.tool_calls:
- tool_content: ExtractedContentList = []
- for tool_call in delta.tool_calls:
- tool_index = tool_call.index
-
- # Start of tool call
- # id is only present for the first tool call chunk
- if (
- tool_call.id
- and tool_call.function
- and tool_call.function.name
- ):
- tool_info = {
- "toolCallId": tool_call.id,
- "toolName": tool_call.function.name,
- }
- tool_content.append((tool_info, "tool_call_start"))
-
- # Delta of tool call
- # arguments is only present second chunk onwards
- if (
- tool_call.function
- and tool_call.function.arguments
- and tool_index < len(tool_call_ids)
- and tool_call_ids[tool_index]
- ):
- tool_delta = {
- "toolCallId": tool_call_ids[tool_index],
- "inputTextDelta": tool_call.function.arguments,
- }
- tool_content.append((tool_delta, "tool_call_delta"))
+ client = self.get_openai_client(config)
+ return PydanticOpenAI(openai_client=client)
- # return the tool content
- return tool_content
+ def create_model(self, max_tokens: int) -> OpenAIResponsesModel:
+ from pydantic_ai.models.openai import (
+ OpenAIResponsesModel,
+ OpenAIResponsesModelSettings,
+ )
- return None
+ is_reasoning_model = self._is_reasoning_model(self.model)
- def get_finish_reason(
- self, response: ChatCompletionChunk
- ) -> Optional[FinishReason]:
- if (
- hasattr(response, "choices")
- and response.choices
- and response.choices[0].finish_reason
- ):
- return (
- "tool_calls"
- if response.choices[0].finish_reason == "tool_calls"
- else "stop"
+ settings = (
+ OpenAIResponsesModelSettings(
+ max_tokens=max_tokens,
+ openai_reasoning_summary=self.DEFAULT_REASONING_SUMMARY,
+ openai_reasoning_effort=self.DEFAULT_REASONING_EFFORT,
)
- return None
+ if is_reasoning_model
+ else OpenAIResponsesModelSettings(max_tokens=max_tokens)
+ )
+ return OpenAIResponsesModel(
+ model_name=self.model,
+ provider=self.provider,
+ settings=settings,
+ )
- def _maybe_convert_roles(
- self, messages: list[ChatMessage]
- ) -> list[ChatMessage]:
- # https://community.openai.com/t/o1-models-do-not-support-system-role-in-chat-completion/953880/3
- if self.model.startswith("o1") or self.model.startswith("o3"):
+ def _is_reasoning_model(self, model: str) -> bool:
+ """
+ Check if reasoning_effort should be added to the request.
+ Only add for actual OpenAI reasoning models, not for OpenAI-compatible APIs.
- def update_role(message: ChatMessage) -> ChatMessage:
- if message.role == "system":
- return ChatMessage(role="user", content=message.content)
- return message
+ OpenAI-compatible APIs (identified by custom base_url) may not support
+ the reasoning_effort parameter even if the model name suggests it's a
+ reasoning model.
+ """
+ import re
- return [update_role(message) for message in messages]
+ # Check for reasoning model patterns: o{digit} or gpt-5, with optional openai/ prefix
+ reasoning_patterns = [
+ r"^openai/o\d", # openai/o1, openai/o3, etc.
+ r"^o\d", # o1, o3, etc.
+ r"^openai/gpt-5", # openai/gpt-5*
+ r"^gpt-5", # gpt-5*
+ ]
- return messages
+ is_reasoning_model_name = any(
+ re.match(pattern, model) for pattern in reasoning_patterns
+ )
+
+ if not is_reasoning_model_name:
+ return False
+
+ # If using a custom base_url that's not OpenAI, don't assume reasoning is supported
+ if (
+ self.config.base_url
+ and "api.openai.com" not in self.config.base_url
+ ):
+ return False
+
+ return True
class AzureOpenAIProvider(OpenAIProvider):
@@ -893,7 +459,7 @@ class AzureOpenAIProvider(OpenAIProvider):
endpoint = f"{parsed_url.scheme}://{parsed_url.hostname}"
return api_version, deployment_name, endpoint
- def get_client(self, config: AnyProviderConfig) -> AsyncOpenAI:
+ def get_openai_client(self, config: AnyProviderConfig) -> AsyncOpenAI:
from openai import AsyncAzureOpenAI
base_url = config.base_url or None
@@ -933,6 +499,202 @@ class AzureOpenAIProvider(OpenAIProvider):
)
+class CustomProvider(OpenAIClientMixin, PydanticProvider["Provider[Any]"]):
+ """Support for custom providers which may or may not be OpenAI-compatible.
+
+ Note:
+ We need to use the specific provider and model classes, because Pydantic AI has tuned them to send & return messages correctly.
+ We can also use `Agent("provider:model_name")` to avoid finding the provider ourselves. However, this does not let
+ us create custom providers. They rely on env vars to be set.
+ """
+
+ def __init__(
+ self,
+ model_id: AiModelId,
+ config: AnyProviderConfig,
+ deps: list[Dependency] | None = None,
+ ):
+ self._provider_name = model_id.provider
+ self._responses_compatible, self._chat_compatible = (
+ self._get_openai_compatible_providers()
+ )
+ super().__init__(model_id.model, config, deps)
+
+ def _get_openai_compatible_providers(self) -> tuple[set[str], set[str]]:
+ """Get the sets of OpenAI-compatible providers from pydantic_ai.
+
+ Returns:
+ Tuple of (responses_compatible, chat_compatible) provider names.
+ """
+ from pydantic_ai.models import (
+ OpenAIChatCompatibleProvider,
+ OpenAIResponsesCompatibleProvider,
+ )
+
+ # These are TypeAliasType objects, so we need .__value__ to get the Literal
+ responses_compatible = set(
+ get_args(OpenAIResponsesCompatibleProvider.__value__)
+ )
+ chat_compatible = set(get_args(OpenAIChatCompatibleProvider.__value__))
+ return responses_compatible, chat_compatible
+
+ def _is_openai_compatible(self) -> bool:
+ """Check if the provider uses an OpenAI-compatible API."""
+ provider = self._provider_name.lower()
+ return (
+ provider in self._responses_compatible
+ or provider in self._chat_compatible
+ )
+
+ def _supports_responses_api(self) -> bool:
+ """Check if the provider supports the OpenAI Responses API. We currently default to Pydantic's inferred model"""
+ return self._provider_name.lower() in self._responses_compatible
+
+ def create_provider(self, config: AnyProviderConfig) -> Provider[Any]:
+ """Create a provider based on the provider name.
+
+ 1. Try to infer the provider class from the name
+ 2. For OpenAI-compatible providers, pass openai_client (with SSL settings)
+ 3. For other providers, try to create the provider directly with the credentials
+ 4. Fall back to OpenAIProvider if nothing else works or not found
+
+ Reference: https://ai.pydantic.dev/models/openai/#openai-compatible-models
+ """
+ from pydantic_ai.providers import infer_provider_class
+ from pydantic_ai.providers.openai import (
+ OpenAIProvider as PydanticOpenAI,
+ )
+
+ # Try to infer the provider class
+ try:
+ provider_class = infer_provider_class(self._provider_name)
+ LOGGER.debug(f"Inferred provider class: {provider_class.__name__}")
+ except ValueError:
+ # Unknown provider, fall back to OpenAI-compatible
+ LOGGER.debug(
+ f"Unknown provider: {self._provider_name}. Falling back to OpenAIProvider."
+ )
+ client = self.get_openai_client(config)
+ return PydanticOpenAI(openai_client=client)
+
+ if self._is_openai_compatible():
+ client = self.get_openai_client(config)
+ try:
+ return provider_class(openai_client=client) # type: ignore[call-arg]
+ except TypeError:
+ LOGGER.warning(
+ f"Provider {provider_class.__name__} doesn't accept openai_client"
+ )
+
+ return self._create_custom_provider(provider_class, config)
+
+ def _create_custom_provider(
+ self, provider_class: type[Provider[Any]], config: AnyProviderConfig
+ ) -> Provider[Any]:
+ """
+ Create a custom provider based on the provider class. These providers are not OpenAI-compatible.
+ Import on-demand to avoid requiring the provider packages to be installed.
+ """
+
+ provider_name = provider_class.__name__
+ LOGGER.debug(f"Creating custom provider: {provider_name}")
+
+ if provider_name == "CerebrasProvider":
+ from pydantic_ai.providers.cerebras import CerebrasProvider
+
+ return CerebrasProvider(api_key=config.api_key)
+ if provider_name == "CohereProvider":
+ from pydantic_ai.providers.cohere import CohereProvider
+
+ return CohereProvider(api_key=config.api_key)
+ if provider_name == "GroqProvider":
+ from pydantic_ai.providers.groq import GroqProvider
+
+ return GroqProvider(
+ api_key=config.api_key, base_url=config.base_url
+ )
+ if provider_name == "HuggingFaceProvider":
+ from pydantic_ai.providers.huggingface import HuggingFaceProvider
+
+ if config.base_url:
+ return HuggingFaceProvider(
+ api_key=config.api_key, base_url=config.base_url
+ )
+ return HuggingFaceProvider(api_key=config.api_key)
+ if provider_name == "MistralProvider":
+ from pydantic_ai.providers.mistral import MistralProvider
+
+ return MistralProvider(api_key=config.api_key)
+
+ LOGGER.warning(
+ f"Unknown provider: {provider_name}. Create a GitHub issue to request support."
+ )
+ # Try returning provider directly (use env vars)
+ try:
+ return provider_class()
+ except Exception as e:
+ from pydantic_ai.providers.openai import (
+ OpenAIProvider as PydanticOpenAI,
+ )
+
+ LOGGER.warning(
+ f"Failed to create provider {provider_class.__name__}: {e}. "
+ f"Falling back to OpenAIProvider."
+ )
+ client = self.get_openai_client(config)
+ return PydanticOpenAI(openai_client=client)
+
+ def create_model(self, max_tokens: int) -> OpenAIChatModel:
+ """Default to OpenAIChatModel"""
+
+ from pydantic_ai.models.openai import (
+ OpenAIChatModel,
+ OpenAIChatModelSettings,
+ )
+
+ return OpenAIChatModel(
+ model_name=self.model,
+ provider=self.provider,
+ settings=OpenAIChatModelSettings(max_tokens=max_tokens),
+ )
+
+ def create_agent(
+ self,
+ max_tokens: int,
+ tools: list[ToolDefinition],
+ system_prompt: str,
+ ) -> Agent[None, DeferredToolRequests | str]:
+ """Create a Pydantic AI agent"""
+ from pydantic_ai import Agent, UserError
+ from pydantic_ai.models import infer_model
+ from pydantic_ai.settings import ModelSettings
+
+ try:
+ model = infer_model(
+ f"{self._provider_name}:{self.model}",
+ provider_factory=lambda _: self.provider,
+ )
+ except UserError:
+ LOGGER.warning(
+ f"Model {self.model} not found. Falling back to OpenAIChatModel."
+ )
+ model = self.create_model(max_tokens)
+ except Exception as e:
+ LOGGER.error(
+ f"Error creating model: {e}. Falling back to OpenAIChatModel."
+ )
+ model = self.create_model(max_tokens)
+
+ toolset, output_type = self._get_toolsets_and_output_type(tools)
+ return Agent(
+ model,
+ model_settings=ModelSettings(max_tokens=max_tokens),
+ toolsets=[toolset] if tools else None,
+ instructions=system_prompt,
+ output_type=output_type,
+ )
+
+
class AnthropicProvider(PydanticProvider["PydanticAnthropic"]):
# Temperature of 0.2 was recommended for coding and data science in these links:
# https://community.openai.com/t/cheat-sheet-mastering-temperature-and-top-p-in-chatgpt-api/172683
@@ -1132,7 +894,7 @@ class BedrockProvider(PydanticProvider["PydanticBedrock"]):
def get_completion_provider(
config: AnyProviderConfig, model: str
-) -> CompletionProvider[Any, Any] | PydanticProvider[Any]:
+) -> PydanticProvider[Any]:
model_id = AiModelId.from_model(model)
if model_id.provider == "anthropic":
@@ -1149,10 +911,12 @@ def get_completion_provider(
)
elif model_id.provider == "azure":
return AzureOpenAIProvider(model_id.model, config)
- elif model_id.provider == "openrouter":
- return OpenAIProvider(model_id.model, config)
+ elif model_id.provider == "openai":
+ return OpenAIProvider(
+ model_id.model, config, [DependencyManager.openai]
+ )
else:
- return OpenAIProvider(model_id.model, config)
+ return CustomProvider(model_id, config, [DependencyManager.openai])
async def merge_backticks(
diff --git a/marimo/_server/api/endpoints/ai.py b/marimo/_server/api/endpoints/ai.py
index a60a92d706..de36d84f79 100644
--- a/marimo/_server/api/endpoints/ai.py
+++ b/marimo/_server/api/endpoints/ai.py
@@ -14,7 +14,6 @@ from starlette.responses import (
from marimo import _loggers
from marimo._ai._convert import convert_to_ai_sdk_messages
from marimo._ai._pydantic_ai_utils import create_simple_prompt
-from marimo._ai._types import ChatMessage
from marimo._config.config import AiConfig, MarimoConfig
from marimo._server.ai.config import (
AnyProviderConfig,
@@ -33,7 +32,6 @@ from marimo._server.ai.prompts import (
get_refactor_or_insert_notebook_cell_system_prompt,
)
from marimo._server.ai.providers import (
- PydanticProvider,
StreamOptions,
get_completion_provider,
without_wrapping_backticks,
@@ -60,7 +58,6 @@ if TYPE_CHECKING:
from collections.abc import AsyncGenerator
from starlette.requests import Request
- from starlette.responses import ContentStream
# Taken from pydantic_ai.ui import SSE_CONTENT_TYPE
SSE_CONTENT_TYPE = "text/event-stream"
@@ -148,13 +145,12 @@ async def ai_completion(
ai_config = get_ai_config(config)
custom_rules = ai_config.get("rules", None)
- use_messages = len(body.messages) >= 1 # Deprecated
use_ui_messages = len(body.ui_messages) >= 1
system_prompt = get_refactor_or_insert_notebook_cell_system_prompt(
language=body.language,
is_insert=False,
- support_multiple_cells=use_messages or use_ui_messages,
+ support_multiple_cells=use_ui_messages,
custom_rules=custom_rules,
cell_code=body.code,
selected_text=body.selected_text,
@@ -169,65 +165,26 @@ async def ai_completion(
model=model,
)
- if isinstance(provider, PydanticProvider):
- # Currently, only useChat (use_messages=True) supports UI messages
- # So, we can stream back the UI messages here. Else, we stream back the text.
- if use_ui_messages:
- return await provider.stream_completion(
- messages=body.ui_messages,
- system_prompt=system_prompt,
- max_tokens=get_max_tokens(config),
- additional_tools=[],
- )
- response = provider.stream_text(
- user_prompt=prompt,
+ # Currently, only useChat (use_ui_messages=True) supports UI messages
+ # So, we can stream back the UI messages here. Else, we stream back the text.
+ if use_ui_messages:
+ return await provider.stream_completion(
messages=body.ui_messages,
system_prompt=system_prompt,
max_tokens=get_max_tokens(config),
additional_tools=[],
)
- safe_content = safe_stream_wrapper(response, text_only=False)
- content_without_wrapping = without_wrapping_backticks(safe_content)
- return StreamingResponse(
- content=content_without_wrapping,
- media_type="application/json",
- headers={"x-vercel-ai-data-stream": "v1"},
- )
-
- messages = (
- body.messages
- if use_messages
- else [ChatMessage(role="user", content=prompt)]
- )
-
- response = await provider.stream_completion(
- messages=messages,
+ response = provider.stream_text(
+ user_prompt=prompt,
+ messages=body.ui_messages,
system_prompt=system_prompt,
max_tokens=get_max_tokens(config),
additional_tools=[],
)
-
- # Pass back the entire SDK message if the frontend can handle it
- content: ContentStream
- if use_messages:
- content = safe_stream_wrapper(
- provider.as_stream_response(
- response, StreamOptions(format_stream=True, text_only=False)
- ),
- text_only=False,
- )
- else:
- content = safe_stream_wrapper(
- without_wrapping_backticks(
- provider.as_stream_response(
- response, StreamOptions(text_only=True)
- )
- ),
- text_only=True,
- )
-
+ safe_content = safe_stream_wrapper(response, text_only=False)
+ content_without_wrapping = without_wrapping_backticks(safe_content)
return StreamingResponse(
- content=content,
+ content=content_without_wrapping,
media_type="application/json",
headers={"x-vercel-ai-data-stream": "v1"},
)
@@ -264,7 +221,6 @@ async def ai_chat(
)
ai_config = get_ai_config(config)
custom_rules = ai_config.get("rules", None)
- messages = body.messages
# Get the system prompt
system_prompt = get_chat_system_prompt(
@@ -288,31 +244,12 @@ async def ai_chat(
format_stream=True, text_only=False, accept=accept
)
- if isinstance(provider, PydanticProvider):
- return await provider.stream_completion(
- messages=body.ui_messages,
- system_prompt=system_prompt,
- max_tokens=max_tokens,
- additional_tools=additional_tools,
- stream_options=stream_options,
- )
-
- response = await provider.stream_completion(
- messages=messages,
+ return await provider.stream_completion(
+ messages=body.ui_messages,
system_prompt=system_prompt,
max_tokens=max_tokens,
additional_tools=additional_tools,
- )
-
- return StreamingResponse(
- content=safe_stream_wrapper(
- provider.as_stream_response(
- response, StreamOptions(format_stream=True, text_only=False)
- ),
- text_only=False,
- ),
- media_type="application/json",
- headers={"x-vercel-ai-data-stream": "v1"},
+ stream_options=stream_options,
)
@@ -352,7 +289,6 @@ async def ai_inline_completion(
)
# Use FIM (Fill-In-Middle) format for inline completion
prompt = f"{FIM_PREFIX_TAG}{body.prefix}{FIM_SUFFIX_TAG}{body.suffix}{FIM_MIDDLE_TAG}"
- messages = [ChatMessage(role="user", content=prompt)]
system_prompt = get_inline_system_prompt(language=body.language)
# This is currently not configurable and smaller than the default
@@ -369,21 +305,12 @@ async def ai_inline_completion(
provider = get_completion_provider(provider_config, model=model)
try:
- if isinstance(provider, PydanticProvider):
- content = await provider.completion(
- messages=[create_simple_prompt(prompt)],
- system_prompt=system_prompt,
- max_tokens=INLINE_COMPLETION_MAX_TOKENS,
- additional_tools=[],
- )
- else:
- response = await provider.stream_completion(
- messages=messages,
- system_prompt=system_prompt,
- max_tokens=INLINE_COMPLETION_MAX_TOKENS,
- additional_tools=[],
- )
- content = await provider.collect_stream(response)
+ content = await provider.completion(
+ messages=[create_simple_prompt(prompt)],
+ system_prompt=system_prompt,
+ max_tokens=INLINE_COMPLETION_MAX_TOKENS,
+ additional_tools=[],
+ )
except Exception as e:
LOGGER.error("Error in AI inline completion: %s", str(e))
raise HTTPException(
diff --git a/marimo/_server/models/completion.py b/marimo/_server/models/completion.py
index 3bb2a84ef3..687b11f1c1 100644
--- a/marimo/_server/models/completion.py
+++ b/marimo/_server/models/completion.py
@@ -5,7 +5,6 @@ from typing import Any, Literal, Optional, Union
import msgspec
-from marimo._ai._types import ChatMessage
from marimo._server.ai.tools.types import ToolDefinition
@@ -36,6 +35,7 @@ class AiCompletionContext(msgspec.Struct, rename="camel"):
Language = Literal["python", "markdown", "sql"]
+
UIMessage = dict[str, Any]
@@ -48,7 +48,6 @@ class AiCompletionRequest(msgspec.Struct, rename="camel"):
prompt: str
include_other_code: str
code: str
- messages: list[ChatMessage] = [] # Deprecated
ui_messages: list[UIMessage] = []
selected_text: Optional[str] = None
context: Optional[AiCompletionContext] = None
@@ -69,7 +68,6 @@ class ChatRequest(msgspec.Struct, rename="camel"):
context: AiCompletionContext
include_other_code: str
- messages: list[ChatMessage] # Deprecated
ui_messages: list[UIMessage]
tools: Optional[list[ToolDefinition]] = None
model: Optional[str] = None
diff --git a/packages/openapi/api.yaml b/packages/openapi/api.yaml
index 06132789ae..267e6468fd 100644
--- a/packages/openapi/api.yaml
+++ b/packages/openapi/api.yaml
@@ -75,11 +75,6 @@ components:
- markdown
- python
- sql
- messages:
- default: []
- items:
- $ref: '#/components/schemas/ChatMessage'
- type: array
prompt:
type: string
selectedText:
@@ -544,10 +539,6 @@ components:
$ref: '#/components/schemas/AiCompletionContext'
includeOtherCode:
type: string
- messages:
- items:
- $ref: '#/components/schemas/ChatMessage'
- type: array
model:
anyOf:
- type: string
@@ -576,7 +567,6 @@ components:
required:
- context
- includeOtherCode
- - messages
- uiMessages
title: ChatRequest
type: object
diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.ts
index d8fd7f92e4..82c7c58987 100644
--- a/packages/openapi/src/api.ts
+++ b/packages/openapi/src/api.ts
@@ -2961,8 +2961,6 @@ export interface components {
* @enum {unknown}
*/
language?: "markdown" | "python" | "sql";
- /** @default [] */
- messages?: components["schemas"]["ChatMessage"][];
prompt: string;
/** @default null */
selectedText?: string | null;
@@ -3268,7 +3266,6 @@ export interface components {
ChatRequest: {
context: components["schemas"]["AiCompletionContext"];
includeOtherCode: string;
- messages: components["schemas"]["ChatMessage"][];
/** @default null */
model?: string | null;
/** @default null */
diff --git a/pixi.lock b/pixi.lock
index 4aa7786221..91190acafe 100644
--- a/pixi.lock
+++ b/pixi.lock
@@ -1133,7 +1133,7 @@ packages:
- pypi: ./
name: marimo
version: 0.19.2
- sha256: 90b3f019966fb4460ac05e6237db94ab2ca00f5a5f27aedaf5b41d4b4e109df2
+ sha256: a9b9623f9875a6ed5a6dc5adfce07b6ae1f89695dcc9e0a38ae8adf71e75efa9
requires_dist:
- click>=8.0,<9
- jedi>=0.18.0
@@ -1161,7 +1161,7 @@ packages:
- marimo[sql] ; extra == 'recommended'
- marimo[sandbox] ; extra == 'recommended'
- altair>=5.4.0 ; extra == 'recommended'
- - openai>=1.55.3 ; extra == 'recommended'
+ - pydantic-ai-slim[openai]>=1.39.0 ; extra == 'recommended'
- ruff ; extra == 'recommended'
- nbformat>=5.7.0 ; extra == 'recommended'
- python-lsp-server>=1.13.0 ; extra == 'lsp'
diff --git a/pyproject.toml b/pyproject.toml
index 119fcdc494..bf25d0812e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -97,11 +97,11 @@ sandbox = [
# in order to unlock all features in marimo
recommended = [
"marimo[sql]",
- "marimo[sandbox]", # For `marimo edit --sandbox DIRECTORY`
- "altair>=5.4.0", # Plotting in datasource viewer
- "openai>=1.55.3", # AI features
- "ruff", # Formatting
- "nbformat>=5.7.0", # Export as IPYNB
+ "marimo[sandbox]", # For `marimo edit --sandbox DIRECTORY`
+ "altair>=5.4.0", # Plotting in datasource viewer
+ "pydantic-ai-slim[openai]>=1.39.0", # AI features
+ "ruff", # Formatting
+ "nbformat>=5.7.0", # Export as IPYNB
]
lsp = [
@@ -129,7 +129,7 @@ dev = [
# For linting
"ruff>=0.13.2",
# For AI
- "openai>=1.55.3",
+ "pydantic-ai-slim[openai]>=1.39.0",
]
test = [
@@ -192,8 +192,7 @@ dependencies = [
"matplotlib>=3.8.0",
"sqlglot[rs]>=26.2.0",
"sqlalchemy>=2.0.40",
- "pydantic-ai-slim[google,anthropic,bedrock]>=1.39.0",
- "openai>=1.55.3",
+ "pydantic-ai-slim[google,anthropic,bedrock,openai]>=1.39.0",
"loro>=1.5.0",
"pandas-stubs>=1.5.3.230321",
"pyiceberg>=0.9.0",
@@ -272,8 +271,7 @@ extra-dependencies = [
"anywidget~=0.9.18",
"ipython~=8.12.3",
# testing gen ai
- "openai>=1.55.3",
- "pydantic-ai-slim[google,anthropic,bedrock]>=1.39.0",
+ "pydantic-ai-slim[google,anthropic,bedrock,openai]>=1.39.0",
# - google-auth uses cachetools, and cachetools<5.0.0 uses collections.MutableMapping (removed in Python 3.10)
"cachetools>=5.0.0",
"boto3>=1.38.46",
diff --git a/tests/_ai/test_chat_convert.py b/tests/_ai/test_chat_convert.py
index b0a2c56be1..ce4db3b7f4 100644
--- a/tests/_ai/test_chat_convert.py
+++ b/tests/_ai/test_chat_convert.py
@@ -9,15 +9,11 @@ import pytest
from marimo._ai._convert import (
convert_to_ai_sdk_messages,
convert_to_anthropic_messages,
- convert_to_anthropic_tools,
convert_to_google_messages,
- convert_to_google_tools,
convert_to_groq_messages,
convert_to_openai_messages,
- convert_to_openai_tools,
extract_text,
get_google_messages_from_parts,
- get_openai_messages_from_parts,
)
from marimo._ai._types import (
ChatMessage,
@@ -1025,7 +1021,7 @@ def test_from_chat_message_dict():
role="user",
content="I'm already a ChatMessage object",
attachments=None,
- parts=None,
+ parts=[],
)
result_existing = from_chat_message_dict(existing_message)
@@ -1049,86 +1045,6 @@ def sample_tools():
]
-def test_convert_to_openai_tools(sample_tools):
- result = convert_to_openai_tools(sample_tools)
- assert len(result) == 1
- assert result[0]["type"] == "function"
- assert result[0]["function"]["name"] == "test_tool"
- assert result[0]["function"]["description"] == "A test tool"
- assert result[0]["function"]["parameters"] == {
- "type": "object",
- "properties": {"x": {"type": "integer"}},
- }
-
-
-def test_convert_to_anthropic_tools(sample_tools):
- result = convert_to_anthropic_tools(sample_tools)
- assert len(result) == 1
- assert result[0]["name"] == "test_tool"
- assert result[0]["description"] == "A test tool"
- assert result[0]["input_schema"] == {
- "type": "object",
- "properties": {"x": {"type": "integer"}},
- }
-
-
-def test_convert_to_google_tools(sample_tools):
- # Add some additional parameters in tools, it should be ignored
- sample_tools[0].parameters["maxNumResults"] = 10
- result = convert_to_google_tools(sample_tools)
- assert result == [
- {
- "function_declarations": [
- {
- "name": "test_tool",
- "description": "A test tool",
- "parameters": {
- "type": "object",
- "properties": {"x": {"type": "integer"}},
- "required": [],
- },
- }
- ]
- }
- ]
-
-
-def test_convert_to_google_tools_with_unnecessary_parameters():
- tools = [
- ToolDefinition(
- name="test_tool",
- description="A test tool",
- parameters={
- "type": "object",
- "properties": {
- "x": {
- "type": "integer",
- "additionalProperties": True,
- } # additionalProperties is not supported by Google
- },
- },
- source="mcp",
- mode=["manual"],
- )
- ]
- result = convert_to_google_tools(tools)
- assert result == [
- {
- "function_declarations": [
- {
- "name": "test_tool",
- "description": "A test tool",
- "parameters": {
- "type": "object",
- "properties": {"x": {"type": "integer"}},
- "required": [],
- },
- }
- ]
- }
- ]
-
-
def test_convert_to_ai_sdk_messages():
# Test text type
text = "hello world"
@@ -1238,69 +1154,6 @@ def test_convert_to_ai_sdk_messages():
assert result == expected
-# Tests for helper functions that convert parts to provider-specific formats
-def test_get_openai_messages_from_parts_text_only():
- """Test converting TextPart to OpenAI format."""
- parts = [
- TextPart(type="text", text="Hello"),
- TextPart(type="text", text="World"),
- ]
-
- result = get_openai_messages_from_parts("user", parts)
-
- assert len(result) == 2
- assert result[0] == {"role": "user", "content": "Hello"}
- assert result[1] == {"role": "user", "content": "World"}
-
-
-def test_get_openai_messages_from_parts_with_tool_invocation():
- """Test converting ToolInvocationPart to OpenAI format."""
- parts = [
- TextPart(type="text", text="Let me check the weather"),
- ToolInvocationPart(
- type="tool-weather_tool",
- tool_call_id="call_123",
- state="output-available",
- input={"location": "New York"},
- output={"temperature": "72°F", "condition": "sunny"},
- ),
- ]
-
- result = get_openai_messages_from_parts("assistant", parts)
-
- assert len(result) == 3 # text message, assistant tool call, tool result
-
- # Check text message
- assert result[0] == {
- "role": "assistant",
- "content": "Let me check the weather",
- }
-
- # Check assistant tool call message
- assert result[1]["role"] == "assistant"
- assert result[1]["content"] is None
- assert len(result[1]["tool_calls"]) == 1
- tool_call = result[1]["tool_calls"][0]
- assert tool_call["id"] == "call_123"
- assert tool_call["type"] == "function"
- assert tool_call["function"]["name"] == "weather_tool"
- assert tool_call["function"]["arguments"] == str({"location": "New York"})
-
- # Check tool result message
- assert result[2]["role"] == "tool"
- assert result[2]["tool_call_id"] == "call_123"
- assert result[2]["name"] == "weather_tool"
- assert result[2]["content"] == str(
- {"temperature": "72°F", "condition": "sunny"}
- )
-
-
-def test_get_openai_messages_from_parts_empty():
- """Test converting empty parts list."""
- result = get_openai_messages_from_parts("user", [])
- assert result == []
-
-
def test_get_google_messages_from_parts_text_only():
"""Test converting TextPart to Google format."""
parts = [
diff --git a/tests/_server/ai/test_providers.py b/tests/_server/ai/test_providers.py
index 1da50f1c92..b7f1da584b 100644
--- a/tests/_server/ai/test_providers.py
+++ b/tests/_server/ai/test_providers.py
@@ -1,10 +1,7 @@
"""Tests for the LLM providers in marimo._server.ai.providers."""
-from unittest.mock import AsyncMock, MagicMock, patch
-
import pytest
-from marimo._ai._types import ChatMessage
from marimo._config.config import AiConfig
from marimo._dependencies.dependencies import Dependency, DependencyManager
from marimo._server.ai.config import AnyProviderConfig
@@ -12,6 +9,7 @@ from marimo._server.ai.providers import (
AnthropicProvider,
AzureOpenAIProvider,
BedrockProvider,
+ CustomProvider,
GoogleProvider,
OpenAIProvider,
get_completion_provider,
@@ -83,7 +81,7 @@ def test_anyprovider_for_model(model_name: str, provider_name: str) -> None:
id="bedrock",
),
pytest.param(
- "openrouter/gpt-4", OpenAIProvider, None, id="openrouter"
+ "openrouter/gpt-4", CustomProvider, None, id="openrouter"
),
],
)
@@ -109,74 +107,6 @@ def test_get_completion_provider(
assert isinstance(provider, provider_type)
- ("model_name", "expected_params"),
- [
- pytest.param(
- "o1-mini",
- {"max_completion_tokens": 1000, "reasoning_effort": "medium"},
- id="reasoning_model_o1_mini",
- ),
- pytest.param(
- "o1-preview",
- {"max_completion_tokens": 1000, "reasoning_effort": "medium"},
- id="reasoning_model_o1_preview",
- ),
- pytest.param(
- "gpt-4",
- {"max_tokens": 1000},
- id="non_reasoning_model_gpt4",
- ),
- pytest.param(
- "gpt-3.5-turbo",
- {"max_tokens": 1000},
- id="non_reasoning_model_gpt35",
- ),
- ],
-)
-@patch("openai.AsyncOpenAI")
-async def test_openai_provider_max_tokens_parameter(
- mock_openai_class, model_name: str, expected_params: dict
-) -> None:
- """Test that OpenAI provider uses correct token parameter for reasoning vs non-reasoning models."""
- # Setup mock
- mock_client = AsyncMock()
- mock_openai_class.return_value = mock_client
- mock_stream = AsyncMock()
- mock_client.chat.completions.create.return_value = mock_stream
-
- # Create provider
- config = AnyProviderConfig(
- api_key="test-key", base_url="https://api.openai.com/v1"
- )
- provider = OpenAIProvider(model_name, config)
-
- # Call stream_completion
- messages = [ChatMessage(role="user", content="test message")]
- await provider.stream_completion(messages, "system prompt", 1000, [])
-
- # Verify the correct parameters were passed
- mock_client.chat.completions.create.assert_called_once()
- call_kwargs = mock_client.chat.completions.create.call_args[1]
-
- # Check that the expected parameters are present
- for param_name, param_value in expected_params.items():
- assert param_name in call_kwargs, (
- f"Expected parameter {param_name} not found"
- )
- assert call_kwargs[param_name] == param_value
-
- # Ensure mutually exclusive parameters are not both present
- if "max_completion_tokens" in expected_params:
- assert "max_tokens" not in call_kwargs, (
- "max_tokens should not be present for reasoning models"
- )
- else:
- assert "max_completion_tokens" not in call_kwargs, (
- "max_completion_tokens should not be present for non-reasoning models"
- )
-
-
async def test_azure_openai_provider() -> None:
"""Test that Azure OpenAI provider uses correct parameters."""
config = AnyProviderConfig(
@@ -200,45 +130,6 @@ async def test_azure_openai_provider() -> None:
assert endpoint == "https://unknown_domain.openai"
-def test_openai_extract_content_multiple_tool_calls() -> None:
- """Test OpenAI extracts multiple tool calls correctly."""
- config = AnyProviderConfig(api_key="test-key", base_url="http://test")
- provider = OpenAIProvider("gpt-4", config)
-
- mock_response = MagicMock()
- mock_delta = MagicMock()
- mock_delta.content = None
-
- mock_tool_1 = MagicMock()
- mock_tool_1.index = 0
- mock_tool_1.id = "call_1"
- mock_tool_1.function = MagicMock()
- mock_tool_1.function.name = "get_weather"
- mock_tool_1.function.arguments = None
-
- mock_tool_2 = MagicMock()
- mock_tool_2.index = 1
- mock_tool_2.id = "call_2"
- mock_tool_2.function = MagicMock()
- mock_tool_2.function.name = "get_time"
- mock_tool_2.function.arguments = None
-
- mock_delta.tool_calls = [mock_tool_1, mock_tool_2]
- mock_choice = MagicMock()
- mock_choice.delta = mock_delta
- mock_response.choices = [mock_choice]
-
- result = provider.extract_content(mock_response, None)
- assert result is not None
- assert len(result) == 2
- tool_data_0, _ = result[0]
- tool_data_1, _ = result[1]
- assert isinstance(tool_data_0, dict)
- assert isinstance(tool_data_1, dict)
- assert tool_data_0["toolName"] == "get_weather"
- assert tool_data_1["toolName"] == "get_time"
-
-
@pytest.mark.skipif(
not DependencyManager.anthropic.has()
or not DependencyManager.pydantic_ai.has(),
@@ -391,97 +282,49 @@ def test_is_reasoning_model(
@pytest.mark.parametrize(
- ("model_name", "base_url", "expected_params"),
+ ("model_name", "expected"),
[
pytest.param(
- "o1-mini",
- "https://custom-openai-compatible.com/v1",
- {"max_tokens": 1000},
- id="reasoning_model_name_custom_api_no_reasoning",
+ "claude-opus-4-20250514",
+ True,
+ id="claude_opus_4",
),
pytest.param(
- "o1-preview",
- "https://litellm.proxy.com/api/v1",
- {"max_tokens": 1000},
- id="o1_preview_litellm_proxy_no_reasoning",
+ "claude-sonnet-4-20250514",
+ True,
+ id="claude_sonnet_4",
),
pytest.param(
- "o3-mini",
- "https://corporate-llm.internal/api",
- {"max_tokens": 1000},
- id="o3_mini_corporate_proxy_no_reasoning",
+ "claude-haiku-4-5-20250514",
+ True,
+ id="claude_haiku_4_5",
+ ),
+ pytest.param(
+ "claude-3-7-sonnet-20250219",
+ True,
+ id="claude_3_7_sonnet",
+ ),
+ pytest.param(
+ "claude-3-5-sonnet-20241022",
+ False,
+ id="claude_3_5_sonnet_not_thinking",
+ ),
+ pytest.param(
+ "claude-3-opus-20240229",
+ False,
+ id="claude_3_opus_not_thinking",
),
],
)
-@patch("openai.AsyncOpenAI")
-async def test_openai_compatible_api_no_reasoning_effort(
- mock_openai_class, model_name: str, base_url: str, expected_params: dict
+ not DependencyManager.anthropic.has()
+ or not DependencyManager.pydantic_ai.has(),
+ reason="anthropic or pydantic_ai not installed",
+)
+def test_anthropic_is_extended_thinking_model(
+ model_name: str, expected: bool
) -> None:
- """Test that OpenAI-compatible APIs don't get reasoning_effort even with reasoning model names."""
- # Setup mock
- mock_client = AsyncMock()
- mock_openai_class.return_value = mock_client
- mock_stream = AsyncMock()
- mock_client.chat.completions.create.return_value = mock_stream
-
- # Create provider with custom base_url (simulating OpenAI-compatible API)
- config = AnyProviderConfig(api_key="test-key", base_url=base_url)
- provider = OpenAIProvider(model_name, config)
-
- # Call stream_completion
- messages = [ChatMessage(role="user", content="test message")]
- await provider.stream_completion(messages, "system prompt", 1000, [])
-
- # Verify the correct parameters were passed
- mock_client.chat.completions.create.assert_called_once()
- call_kwargs = mock_client.chat.completions.create.call_args[1]
-
- # Check that reasoning_effort is NOT present
- assert "reasoning_effort" not in call_kwargs, (
- "reasoning_effort should not be present for OpenAI-compatible APIs"
- )
-
- # Check that the expected parameters are present
- for param_name, param_value in expected_params.items():
- assert param_name in call_kwargs, (
- f"Expected parameter {param_name} not found"
- )
- assert call_kwargs[param_name] == param_value
-
- # Ensure max_completion_tokens is not present when reasoning_effort is not used
- assert "max_completion_tokens" not in call_kwargs, (
- "max_completion_tokens should not be present when reasoning_effort is not used"
- )
-
-
-def test_openai_extract_content_tool_delta_out_of_bounds() -> None:
- """Test OpenAI handles tool call delta with out-of-bounds index gracefully.
-
- This reproduces the issue reported in #7330 where some OpenAI-compatible
- providers (like deepseek) may send tool call deltas before the tool call
- start, causing an IndexError.
- """
- config = AnyProviderConfig(api_key="test-key", base_url="http://test")
- provider = OpenAIProvider("gpt-4", config)
-
- mock_response = MagicMock()
- mock_delta = MagicMock()
- mock_delta.content = None
-
- # Create a tool call delta with index 0, but tool_call_ids is empty
- mock_tool_delta = MagicMock()
- mock_tool_delta.index = 0
- mock_tool_delta.id = None # No id for delta chunks
- mock_tool_delta.function = MagicMock()
- mock_tool_delta.function.name = None # No name for delta chunks
- mock_tool_delta.function.arguments = '{"location": "SF"}'
-
- mock_delta.tool_calls = [mock_tool_delta]
- mock_choice = MagicMock()
- mock_choice.delta = mock_delta
- mock_response.choices = [mock_choice]
-
- # Call with empty tool_call_ids - this should not crash
- result = provider.extract_content(mock_response, [])
- # Should return None or empty list since we can't process the delta
- assert result is None or result == []
+ """Test that is_extended_thinking_model correctly identifies thinking models."""
+ config = AnyProviderConfig(api_key="test-key", base_url=None)
+ provider = AnthropicProvider(model_name, config)
+ assert provider.is_extended_thinking_model(model_name) == expected
diff --git a/tests/_server/api/endpoints/test_ai.py b/tests/_server/api/endpoints/test_ai.py
index ce3093606a..b6aff2ce51 100644
--- a/tests/_server/api/endpoints/test_ai.py
+++ b/tests/_server/api/endpoints/test_ai.py
@@ -1,22 +1,19 @@
# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
-import unittest
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Optional
-from unittest.mock import AsyncMock, MagicMock, Mock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from marimo._dependencies.dependencies import DependencyManager
-from marimo._server.ai.config import AnyProviderConfig
from marimo._server.ai.prompts import (
FIM_MIDDLE_TAG,
FIM_PREFIX_TAG,
FIM_SUFFIX_TAG,
)
from marimo._server.ai.providers import (
- OpenAIProvider,
without_wrapping_backticks,
)
from marimo._server.ai.tools.types import ToolCallResult
@@ -92,11 +89,9 @@ def _create_messages(prompt: str) -> list[dict[str, Any]]:
class TestOpenAiEndpoints:
@staticmethod
@with_session(SESSION_ID)
- @patch("openai.OpenAI")
def test_completion_without_token(
- client: TestClient, openai_mock: Any
+ client: TestClient,
) -> None:
- del openai_mock
user_config_manager = get_session_config_manager(client)
with patch.object(
@@ -120,24 +115,17 @@ class TestOpenAiEndpoints:
@staticmethod
@with_session(SESSION_ID)
- @patch("openai.AsyncOpenAI")
+ @patch("marimo._server.ai.providers.OpenAIProvider.stream_text")
def test_completion_without_code(
- client: TestClient, openai_mock: Any
+ client: TestClient, mock_stream_text: Any
) -> None:
user_config_manager = get_session_config_manager(client)
- oaiclient = MagicMock()
- openai_mock.return_value = oaiclient
-
# Mock async stream
async def mock_stream():
- yield FakeChoices(
- choices=[Choice(delta=Delta(content="import pandas as pd"))]
- )
+ yield "import pandas as pd"
- oaiclient.chat.completions.create = AsyncMock(
- side_effect=lambda **kwargs: mock_stream() # noqa: ARG005
- )
+ mock_stream_text.return_value = mock_stream()
with patch.object(
user_config_manager,
@@ -153,36 +141,26 @@ class TestOpenAiEndpoints:
"code": "",
},
)
- assert response.status_code == 200, "nope"
+ assert response.status_code == 200, response.text
+ # Verify stream_text was called
+ mock_stream_text.assert_called_once()
# Assert the prompt it was called with
- prompt = oaiclient.chat.completions.create.call_args.kwargs[
- "messages"
- ][1]["content"][0]["text"]
- assert prompt == ("Help me create a dataframe")
- # Assert the model it was called with
- model = oaiclient.chat.completions.create.call_args.kwargs["model"]
- assert model == "some-openai-model"
+ call_kwargs = mock_stream_text.call_args.kwargs
+ assert call_kwargs["user_prompt"] == "Help me create a dataframe"
@staticmethod
@with_session(SESSION_ID)
- @patch("openai.AsyncOpenAI")
+ @patch("marimo._server.ai.providers.OpenAIProvider.stream_text")
def test_completion_with_code(
- client: TestClient, openai_mock: Any
+ client: TestClient, mock_stream_text: Any
) -> None:
user_config_manager = get_session_config_manager(client)
- oaiclient = MagicMock()
- openai_mock.return_value = oaiclient
-
# Mock async stream
async def mock_stream():
- yield FakeChoices(
- choices=[Choice(delta=Delta(content="import pandas as pd"))]
- )
+ yield "import pandas as pd"
- oaiclient.chat.completions.create = AsyncMock(
- side_effect=lambda **kwargs: mock_stream() # noqa: ARG005
- )
+ mock_stream_text.return_value = mock_stream()
with patch.object(
user_config_manager,
@@ -199,32 +177,25 @@ class TestOpenAiEndpoints:
},
)
assert response.status_code == 200, response.text
+ # Verify stream_text was called
+ mock_stream_text.assert_called_once()
# Assert the prompt it was called with
- prompt = oaiclient.chat.completions.create.call_args.kwargs[
- "messages"
- ][1]["content"][0]["text"]
- assert prompt == "Help me create a dataframe"
+ call_kwargs = mock_stream_text.call_args.kwargs
+ assert call_kwargs["user_prompt"] == "Help me create a dataframe"
@staticmethod
@with_session(SESSION_ID)
- @patch("openai.AsyncOpenAI")
+ @patch("marimo._server.ai.providers.OpenAIProvider.stream_text")
def test_completion_with_custom_model(
- client: TestClient, openai_mock: Any
+ client: TestClient, mock_stream_text: Any
) -> None:
user_config_manager = get_session_config_manager(client)
- oaiclient = MagicMock()
- openai_mock.return_value = oaiclient
-
# Mock async stream
async def mock_stream():
- yield FakeChoices(
- choices=[Choice(delta=Delta(content="import pandas as pd"))]
- )
+ yield "import pandas as pd"
- oaiclient.chat.completions.create = AsyncMock(
- side_effect=lambda **kwargs: mock_stream() # noqa: ARG005
- )
+ mock_stream_text.return_value = mock_stream()
with patch.object(
user_config_manager,
@@ -241,30 +212,22 @@ class TestOpenAiEndpoints:
},
)
assert response.status_code == 200, response.text
- # Assert the model it was called with
- model = oaiclient.chat.completions.create.call_args.kwargs["model"]
- assert model == "gpt-marimo"
+ # Verify stream_text was called
+ mock_stream_text.assert_called_once()
@staticmethod
@with_session(SESSION_ID)
- @patch("openai.AsyncOpenAI")
+ @patch("marimo._server.ai.providers.OpenAIProvider.stream_text")
def test_completion_with_custom_base_url(
- client: TestClient, openai_mock: Any
+ client: TestClient, mock_stream_text: Any
) -> None:
user_config_manager = get_session_config_manager(client)
- oaiclient = MagicMock()
- openai_mock.return_value = oaiclient
-
# Mock async stream
async def mock_stream():
- yield FakeChoices(
- choices=[Choice(delta=Delta(content="import pandas as pd"))]
- )
+ yield "import pandas as pd"
- oaiclient.chat.completions.create = AsyncMock(
- side_effect=lambda **kwargs: mock_stream() # noqa: ARG005
- )
+ mock_stream_text.return_value = mock_stream()
with patch.object(
user_config_manager,
@@ -281,31 +244,19 @@ class TestOpenAiEndpoints:
},
)
assert response.status_code == 200, response.text
- # Assert the base_url it was called with
- base_url = openai_mock.call_args.kwargs["base_url"]
- assert base_url == "https://my-openai-instance.com"
- # Assert the model it was called with
- model = oaiclient.chat.completions.create.call_args.kwargs["model"]
- assert model == "some-openai-model-with-base-url"
+ # Verify stream_text was called
+ mock_stream_text.assert_called_once()
@staticmethod
@with_session(SESSION_ID)
- @patch("openai.AsyncOpenAI")
- def test_inline_completion(client: TestClient, openai_mock: Any) -> None:
+ @patch("marimo._server.ai.providers.OpenAIProvider.completion")
+ def test_inline_completion(
+ client: TestClient, mock_completion: Any
+ ) -> None:
user_config_manager = get_session_config_manager(client)
- oaiclient = MagicMock()
- openai_mock.return_value = oaiclient
-
- # Mock async stream
- async def mock_stream():
- yield FakeChoices(
- choices=[Choice(delta=Delta(content="df = pd.DataFrame()"))]
- )
-
- oaiclient.chat.completions.create = AsyncMock(
- side_effect=lambda **kwargs: mock_stream() # noqa: ARG005
- )
+ # Mock completion to return a string
+ mock_completion.return_value = "df = pd.DataFrame()"
with patch.object(
user_config_manager, "get_config", return_value=_openai_config()
@@ -320,33 +271,24 @@ class TestOpenAiEndpoints:
},
)
assert response.status_code == 200, response.text
- # Assert the prompt it was called with
- prompt = oaiclient.chat.completions.create.call_args.kwargs[
- "messages"
- ][1]["content"][0]["text"]
- assert (
- prompt
- == f"{FIM_PREFIX_TAG}import pandas as pd\n{FIM_SUFFIX_TAG}\ndf.head(){FIM_MIDDLE_TAG}"
- )
- # Assert the system prompt for FIM models
- system_prompt = oaiclient.chat.completions.create.call_args.kwargs[
- "messages"
- ][0]["content"][0]["text"]
- assert (
- system_prompt
- == f"You are a python code completion assistant. Complete the missing code between the prefix and suffix while maintaining proper syntax, style, and functionality.Only output the code that goes after the {FIM_SUFFIX_TAG} part. Do not add any explanation or markdown."
+ # Verify completion was called
+ mock_completion.assert_called_once()
+ # Assert the messages contain FIM format
+ call_kwargs = mock_completion.call_args.kwargs
+ messages = call_kwargs["messages"]
+ assert len(messages) == 1
+ # Verify FIM format is used
+ assert messages[0].parts[0].text == (
+ f"{FIM_PREFIX_TAG}import pandas as pd\n"
+ f"{FIM_SUFFIX_TAG}\ndf.head()"
+ f"{FIM_MIDDLE_TAG}"
)
- # Assert the model it was called with
- model = oaiclient.chat.completions.create.call_args.kwargs["model"]
- assert model == "gpt-marimo-for-inline-completion"
@staticmethod
@with_session(SESSION_ID)
- @patch("openai.OpenAI")
def test_inline_completion_without_token(
- client: TestClient, openai_mock: Any
+ client: TestClient,
) -> None:
- del openai_mock
user_config_manager = get_session_config_manager(client)
with patch.object(
@@ -368,24 +310,14 @@ class TestOpenAiEndpoints:
@staticmethod
@with_session(SESSION_ID)
- @patch("openai.AsyncOpenAI")
+ @patch("marimo._server.ai.providers.OpenAIProvider.completion")
def test_inline_completion_different_language(
- client: TestClient, openai_mock: Any
+ client: TestClient, mock_completion: Any
) -> None:
user_config_manager = get_session_config_manager(client)
- oaiclient = MagicMock()
- openai_mock.return_value = oaiclient
-
- # Mock async stream
- async def mock_stream():
- yield FakeChoices(
- choices=[Choice(delta=Delta(content="SELECT 1;"))]
- )
-
- oaiclient.chat.completions.create = AsyncMock(
- side_effect=lambda **kwargs: mock_stream() # noqa: ARG005
- )
+ # Mock completion to return a string
+ mock_completion.return_value = "SELECT 1;"
with patch.object(
user_config_manager, "get_config", return_value=_openai_config()
@@ -400,17 +332,11 @@ class TestOpenAiEndpoints:
},
)
assert response.status_code == 200, response.text
- # Assert the system prompt for FIM models
- system_prompt = oaiclient.chat.completions.create.call_args.kwargs[
- "messages"
- ][0]["content"][0]["text"]
- assert (
- system_prompt
- == f"You are a sql code completion assistant. Complete the missing code between the prefix and suffix while maintaining proper syntax, style, and functionality.Only output the code that goes after the {FIM_SUFFIX_TAG} part. Do not add any explanation or markdown."
- )
- # Assert model
- model = oaiclient.chat.completions.create.call_args.kwargs["model"]
- assert model == "gpt-marimo-for-inline-completion"
+ # Verify completion was called
+ mock_completion.assert_called_once()
+ # Assert the system prompt mentions SQL
+ call_kwargs = mock_completion.call_args.kwargs
+ assert "sql" in call_kwargs["system_prompt"].lower()
@pytest.mark.skipif(
@@ -419,11 +345,9 @@ class TestOpenAiEndpoints:
class TestAnthropicAiEndpoints:
@staticmethod
@with_session(SESSION_ID)
- @patch("anthropic.Client")
def test_anthropic_completion_without_token(
- client: TestClient, anthropic_mock: Any
+ client: TestClient,
) -> None:
- del anthropic_mock
user_config_manager = get_session_config_manager(client)
with patch.object(
@@ -744,197 +668,83 @@ def _google_ai_config():
@with_session(SESSION_ID)
-def test_chat_without_code(client: TestClient) -> None:
+@patch("marimo._server.ai.providers.OpenAIProvider.stream_completion")
+def test_chat_without_code(
+ client: TestClient, mock_stream_completion: Any
+) -> None:
user_config_manager = get_session_config_manager(client)
- with patch("openai.AsyncOpenAI") as openai_mock:
- oaiclient = MagicMock()
- openai_mock.return_value = oaiclient
+ # Create a mock StreamingResponse
+ from starlette.responses import StreamingResponse
- # Mock async stream
- async def mock_stream():
- yield FakeChoices(
- choices=[
- Choice(delta=Delta(content="Hello, how can I help you?"))
- ]
- )
+ async def mock_stream():
+ yield b"Hello, how can I help you?"
- oaiclient.chat.completions.create = AsyncMock(
- side_effect=lambda **kwargs: mock_stream() # noqa: ARG005
- )
+ mock_response = StreamingResponse(
+ content=mock_stream(),
+ media_type="text/event-stream",
+ )
+ mock_stream_completion.return_value = mock_response
- with patch.object(
- user_config_manager, "get_config", return_value=_openai_config()
- ):
- response = client.post(
- "/api/ai/chat",
- headers=HEADERS,
- json={
- "messages": _create_messages("Hello"),
- "uiMessages": [],
- "model": "gpt-4-turbo",
- "variables": [],
- "includeOtherCode": "",
- "context": {},
- "id": "123",
- },
- )
- assert response.status_code == 200, response.text
- # Assert the prompt it was called with
- prompt = oaiclient.chat.completions.create.call_args.kwargs[
- "messages"
- ][1]["content"][0]["text"]
- assert prompt == "Hello"
+ with patch.object(
+ user_config_manager, "get_config", return_value=_openai_config()
+ ):
+ response = client.post(
+ "/api/ai/chat",
+ headers=HEADERS,
+ json={
+ "messages": _create_messages("Hello"),
+ "uiMessages": _create_messages("Hello"),
+ "model": "gpt-4-turbo",
+ "variables": [],
+ "includeOtherCode": "",
+ "context": {},
+ "id": "123",
+ },
+ )
+ assert response.status_code == 200, response.text
+ # Verify stream_completion was called
+ mock_stream_completion.assert_called_once()
@with_session(SESSION_ID)
-def test_chat_with_code(client: TestClient) -> None:
+@patch("marimo._server.ai.providers.OpenAIProvider.stream_completion")
+def test_chat_with_code(
+ client: TestClient, mock_stream_completion: Any
+) -> None:
user_config_manager = get_session_config_manager(client)
- with patch("openai.AsyncOpenAI") as openai_mock:
- oaiclient = MagicMock()
- openai_mock.return_value = oaiclient
+ # Create a mock StreamingResponse
+ from starlette.responses import StreamingResponse
- # Mock async stream
- async def mock_stream():
- yield FakeChoices(
- choices=[Choice(delta=Delta(content="import pandas as pd"))]
- )
+ async def mock_stream():
+ yield b"import pandas as pd"
- oaiclient.chat.completions.create = AsyncMock(
- side_effect=lambda **kwargs: mock_stream() # noqa: ARG005
- )
+ mock_response = StreamingResponse(
+ content=mock_stream(),
+ media_type="text/event-stream",
+ )
+ mock_stream_completion.return_value = mock_response
- with patch.object(
- user_config_manager, "get_config", return_value=_openai_config()
- ):
- response = client.post(
- "/api/ai/chat",
- headers=HEADERS,
- json={
- "messages": _create_messages("Help me create a dataframe"),
- "uiMessages": [],
- "model": "gpt-4-turbo",
- "variables": [],
- "includeOtherCode": "import pandas as pd",
- "context": {},
- "id": "123",
- },
- )
- assert response.status_code == 200, response.text
- # Assert the prompt it was called with
- prompt = oaiclient.chat.completions.create.call_args.kwargs[
- "messages"
- ][1]["content"][0]["text"]
- assert prompt == "Help me create a dataframe"
-
-
-class TestGetContent(unittest.TestCase):
- def test_extract_content_with_none_delta(self) -> None:
- config = AnyProviderConfig(base_url=None, api_key="test-key")
- provider = OpenAIProvider(model="gpt-4o", config=config)
- # Create a mock response with choices but delta is None
- mock_response = Mock()
- mock_response.choices = [Mock()]
- mock_response.choices[0].delta = None
-
- # Ensure text attribute doesn't exist to avoid fallback
- type(mock_response).text = property(lambda _: None)
-
- # Call get_content with the mock response
- result = provider.extract_content(mock_response)
-
- # Assert that the result is None
- assert result is None
-
- def test_extract_content_with_delta_content(self) -> None:
- # Create a mock response with choices and delta.content
- mock_response = Mock()
- mock_response.choices = [Mock()]
- mock_response.choices[0].delta = Mock()
- mock_response.choices[0].delta.content = "Test content"
-
- # Call get_content with the mock response
- config = AnyProviderConfig(base_url=None, api_key="test-key")
- provider = OpenAIProvider(model="gpt-4o", config=config)
- result = provider.extract_content(mock_response)
-
- # Assert that the result is not None and has expected content
- assert result is not None
- result_text, result_type = result[0]
- assert result_text == "Test content"
- assert result_type == "text"
-
-
-class TestGetFinishReason(unittest.TestCase):
- def test_get_finish_reason_with_no_choices(self) -> None:
- config = AnyProviderConfig(base_url=None, api_key="test-key")
- provider = OpenAIProvider(model="gpt-4o", config=config)
- # Create a mock response with no choices
- mock_response = Mock()
- mock_response.choices = []
-
- # Call get_finish_reason with the mock response
- result = provider.get_finish_reason(mock_response)
-
- # Assert that the result is None
- assert result is None
-
- def test_get_finish_reason_with_none_finish_reason(self) -> None:
- config = AnyProviderConfig(base_url=None, api_key="test-key")
- provider = OpenAIProvider(model="gpt-4o", config=config)
- # Create a mock response with choices but finish_reason is None
- mock_response = Mock()
- mock_response.choices = [Mock()]
- mock_response.choices[0].finish_reason = None
-
- # Call get_finish_reason with the mock response
- result = provider.get_finish_reason(mock_response)
-
- # Assert that the result is None
- assert result is None
-
- def test_get_finish_reason_with_tool_calls(self) -> None:
- config = AnyProviderConfig(base_url=None, api_key="test-key")
- provider = OpenAIProvider(model="gpt-4o", config=config)
- # Create a mock response with choices and finish_reason = "tool_calls"
- mock_response = Mock()
- mock_response.choices = [Mock()]
- mock_response.choices[0].finish_reason = "tool_calls"
-
- # Call get_finish_reason with the mock response
- result = provider.get_finish_reason(mock_response)
-
- # Assert that the result is "tool_calls"
- assert result == "tool_calls"
-
- def test_get_finish_reason_with_stop(self) -> None:
- config = AnyProviderConfig(base_url=None, api_key="test-key")
- provider = OpenAIProvider(model="gpt-4o", config=config)
- # Create a mock response with choices and finish_reason = "stop"
- mock_response = Mock()
- mock_response.choices = [Mock()]
- mock_response.choices[0].finish_reason = "stop"
-
- # Call get_finish_reason with the mock response
- result = provider.get_finish_reason(mock_response)
-
- # Assert that the result is "stop"
- assert result == "stop"
-
- def test_get_finish_reason_with_other_reason(self) -> None:
- config = AnyProviderConfig(base_url=None, api_key="test-key")
- provider = OpenAIProvider(model="gpt-4o", config=config)
- # Create a mock response with choices and finish_reason = "length"
- mock_response = Mock()
- mock_response.choices = [Mock()]
- mock_response.choices[0].finish_reason = "length"
-
- # Call get_finish_reason with the mock response
- result = provider.get_finish_reason(mock_response)
-
- # Assert that the result is "stop" (fallback for non-tool_calls reasons)
- assert result == "stop"
+ with patch.object(
+ user_config_manager, "get_config", return_value=_openai_config()
+ ):
+ response = client.post(
+ "/api/ai/chat",
+ headers=HEADERS,
+ json={
+ "messages": _create_messages("Help me create a dataframe"),
+ "uiMessages": _create_messages("Help me create a dataframe"),
+ "model": "gpt-4-turbo",
+ "variables": [],
+ "includeOtherCode": "import pandas as pd",
+ "context": {},
+ "id": "123",
+ },
+ )
+ assert response.status_code == 200, response.text
+ # Verify stream_completion was called
+ mock_stream_completion.assert_called_once()
@pytest.mark.parametrize(