A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
use unified thinking for pydantic-ai (#9477)
Shahmir Varqha6b8d3d12be10
diff --git a/marimo/_server/ai/providers.py b/marimo/_server/ai/providers.pyindex 712f8840d8..0c896a4618 100644--- a/marimo/_server/ai/providers.py+++ b/marimo/_server/ai/providers.py@@ -38,14 +38,16 @@ from marimo._utils.http import HTTPStatusif TYPE_CHECKING:from collections.abc import AsyncGenerator, AsyncIterator- from anthropic.types.beta import BetaThinkingConfigParamfrom openai import AsyncOpenAI- from openai.types.shared.reasoning_effort import ReasoningEffortfrom pydantic_ai import Agent, DeferredToolRequests, FunctionToolsetfrom pydantic_ai.models import Modelfrom pydantic_ai.models.bedrock import BedrockConverseModelfrom pydantic_ai.models.google import GoogleModel- from pydantic_ai.models.openai import OpenAIChatModel, OpenAIResponsesModel+ from pydantic_ai.models.openai import (+ OpenAIChatModel,+ OpenAIResponsesModel,+ OpenAIResponsesModelSettings,+ )from pydantic_ai.providers import Providerfrom pydantic_ai.providers.anthropic import (AnthropicProvider as PydanticAnthropic,@@ -55,6 +57,7 @@ if TYPE_CHECKING:)from pydantic_ai.providers.google import GoogleProvider as PydanticGooglefrom pydantic_ai.providers.openai import OpenAIProvider as PydanticOpenAI+ from pydantic_ai.settings import ModelSettings, ThinkingLevelfrom pydantic_ai.ui.vercel_ai.request_types import UIMessage, UIMessagePartfrom starlette.responses import StreamingResponse@@ -127,11 +130,31 @@ class PydanticProvider(ABC, Generic[ProviderT]):toolset, output_type = self._get_toolsets_and_output_type(tools)return Agent(model,+ model_settings=self._build_agent_settings(model),toolsets=[toolset] if tools else None,instructions=system_prompt,output_type=output_type,)+ def _build_agent_settings(self, model: Model) -> ModelSettings | None:+ """Settings applied at agent level on every request."""+ from pydantic_ai.settings import ModelSettings++ thinking = self._default_thinking(model)+ if thinking is None:+ return None+ if not (+ model.profile.supports_thinking+ or model.profile.thinking_always_enabled+ ):+ return None+ return ModelSettings(thinking=thinking)++ def _default_thinking(self, model: Model) -> ThinkingLevel | None:+ """Default unified thinking flag. Return None to skip."""+ del model+ return True+def convert_messages(self, messages: list[ServerUIMessage]) -> list[UIMessage]:@@ -277,11 +300,7 @@ class GoogleProvider(PydanticProvider["PydanticGoogle"]):return GoogleModel(model_name=self.model,provider=self.provider,- settings=GoogleModelSettings(- max_tokens=max_tokens,- # Works on non-thinking models too- google_thinking_config={"include_thoughts": True},- ),+ settings=GoogleModelSettings(max_tokens=max_tokens),)@@ -367,9 +386,9 @@ class OpenAIClientMixin:class OpenAIProvider(OpenAIClientMixin, PydanticProvider["PydanticOpenAI"]):- # Medium effort provides a balance between speed and accuracy# https://openai.com/index/openai-o3-mini/- DEFAULT_REASONING_EFFORT: ReasoningEffort = "medium"+ # 'auto' lets OpenAI decide between detailed/concise based on the prompt;+ # marimo wants reasoning summaries surfaced for display.DEFAULT_REASONING_SUMMARY: Literal["detailed", "concise", "auto"] = "auto"def create_provider(self, config: AnyProviderConfig) -> PydanticOpenAI:@@ -386,62 +405,41 @@ class OpenAIProvider(OpenAIClientMixin, PydanticProvider["PydanticOpenAI"]):OpenAIResponsesModelSettings,)- is_reasoning_model = self._is_reasoning_model(self.model)-- settings = (- OpenAIResponsesModelSettings(- max_tokens=max_tokens,- openai_reasoning_summary=self.DEFAULT_REASONING_SUMMARY,- openai_reasoning_effort=self.DEFAULT_REASONING_EFFORT,- )- if is_reasoning_model- else OpenAIResponsesModelSettings(max_tokens=max_tokens)- )return OpenAIResponsesModel(model_name=self.model,provider=self.provider,- settings=settings,+ settings=OpenAIResponsesModelSettings(max_tokens=max_tokens),)- def _is_reasoning_model(self, model: str) -> bool:- """- Check if reasoning_effort should be added to the request.- Only add for actual OpenAI reasoning models, not for OpenAI-compatible APIs.-- OpenAI-compatible APIs (identified by custom base_url) may not support- the reasoning_effort parameter even if the model name suggests it's a- reasoning model.- """- import re-- # Check for reasoning model patterns: o{digit} or gpt-5, with optional openai/ prefix- reasoning_patterns = [- r"^openai/o\d", # openai/o1, openai/o3, etc.- r"^o\d", # o1, o3, etc.- r"^openai/gpt-5", # openai/gpt-5*- r"^gpt-5", # gpt-5*- ]-- is_reasoning_model_name = any(- re.match(pattern, model) for pattern in reasoning_patterns- )-- if not is_reasoning_model_name:- return False-- # If using a custom base_url that's not OpenAI, don't assume reasoning is supported- return not (+ def _build_agent_settings(self, model: Model) -> ModelSettings | None:+ # `reasoning.summary` is only valid for OpenAI reasoning models (gpt-5+ # and the o-series).+ settings = super()._build_agent_settings(model)+ if settings is not None and "thinking" in settings:+ extra: OpenAIResponsesModelSettings = {+ "openai_reasoning_summary": self.DEFAULT_REASONING_SUMMARY,+ }+ settings.update(extra)+ return settings++ def _default_thinking(self, model: Model) -> ThinkingLevel | None:+ # OpenAI-compatible third-party endpoints (custom base_url) may not+ # accept `reasoning_effort` even when the model name looks like a+ # reasoning model. Suppress the unified thinking flag in that case.+ if (self.config.base_urland "api.openai.com" not in self.config.base_url- )+ ):+ return None+ return super()._default_thinking(model)class AzureOpenAIProvider(OpenAIProvider):- def _is_reasoning_model(self, model: str) -> bool:- # https://learn.microsoft.com/en-us/answers/questions/5519548/does-gpt-5-via-azure-support-reasoning-effort-and- # Only custom models support reasoning effort, we can expose this as a parameter in the future+ # Only custom Azure deployments support `reasoning_effort`, and we don't expose that config yet.+ # https://learn.microsoft.com/en-us/answers/questions/5519548/does-gpt-5-via-azure-support-reasoning-effort-and+ def _default_thinking(self, model: Model) -> ThinkingLevel | None:del model- return False+ return Nonedef _handle_azure_openai(self, base_url: str) -> tuple[str, str, str]:"""Handle Azure OpenAI.@@ -689,15 +687,25 @@ class CustomProvider(OpenAIClientMixin, PydanticProvider["Provider[Any]"]):)model = self.create_model(max_tokens)+ agent_settings = ModelSettings(max_tokens=max_tokens)+ agent_settings.update(self._build_agent_settings(model) or {})+toolset, output_type = self._get_toolsets_and_output_type(tools)return Agent(model,- model_settings=ModelSettings(max_tokens=max_tokens),+ model_settings=agent_settings,toolsets=[toolset] if tools else None,instructions=system_prompt,output_type=output_type,)+ def _default_thinking(self, model: Model) -> ThinkingLevel | None:+ # Custom OpenAI-compatible endpoints (Together, vLLM, LM Studio, ...)+ # often don't honor `reasoning_effort`+ if self._is_openai_compatible():+ return None+ return super()._default_thinking(model)+class AnthropicProvider(PydanticProvider["PydanticAnthropic"]):# Temperature of 0.2 was recommended for coding and data science in these links:@@ -705,18 +713,9 @@ class AnthropicProvider(PydanticProvider["PydanticAnthropic"]):# https://docs.anthropic.com/en/docs/test-and-evaluate/strengthen-guardrails/reduce-latency?utm_source=chatgpt.comDEFAULT_TEMPERATURE = 0.2- # Extended thinking defaults based on:+ # Extended thinking requires temperature of 1.# https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking- # Extended thinking requires temperature of 1DEFAULT_EXTENDED_THINKING_TEMPERATURE = 1- EXTENDED_THINKING_MODEL_PREFIXES = [- "claude-opus-4",- "claude-sonnet-4",- "claude-haiku-4-5",- "claude-3-7-sonnet",- ]- # 1024 tokens is the minimum budget for extended thinking- DEFAULT_EXTENDED_THINKING_BUDGET_TOKENS = 1024def create_provider(self, config: AnyProviderConfig) -> PydanticAnthropic:from pydantic_ai.providers.anthropic import (@@ -730,36 +729,33 @@ class AnthropicProvider(PydanticProvider["PydanticAnthropic"]):AnthropicModel,AnthropicModelSettings,)+ from pydantic_ai.profiles.anthropic import (+ AnthropicModelProfile,+ anthropic_model_profile,+ )- is_thinking_model = self.is_extended_thinking_model(self.model)- thinking_config: BetaThinkingConfigParam = {"type": "disabled"}- if is_thinking_model:- thinking_config = {- "type": "enabled",- "budget_tokens": self.DEFAULT_EXTENDED_THINKING_BUDGET_TOKENS,- }+ settings: AnthropicModelSettings = {"max_tokens": max_tokens}++ # Anthropic extended thinking requires temperature=1; non-thinking+ # models keep our default coding temperature. Some adaptive-only+ # models (Opus 4.7+) reject sampling settings entirely — skip+ # `temperature` for them so pydantic-ai doesn't drop it with a warning.+ profile = AnthropicModelProfile.from_profile(+ anthropic_model_profile(self.model)+ )+ if not getattr(+ profile, "anthropic_disallows_sampling_settings", False+ ):+ settings["temperature"] = (+ self.DEFAULT_EXTENDED_THINKING_TEMPERATURE+ if profile.supports_thinking+ else self.DEFAULT_TEMPERATURE+ )return AnthropicModel(model_name=self.model,provider=self.provider,- settings=AnthropicModelSettings(- max_tokens=max_tokens,- temperature=self.get_temperature(),- anthropic_thinking=thinking_config,- ),- )-- def is_extended_thinking_model(self, model: str) -> bool:- return any(- model.startswith(prefix)- for prefix in self.EXTENDED_THINKING_MODEL_PREFIXES- )-- def get_temperature(self) -> float:- return (- self.DEFAULT_EXTENDED_THINKING_TEMPERATURE- if self.is_extended_thinking_model(self.model)- else self.DEFAULT_TEMPERATURE+ settings=settings,)def convert_messages(@@ -825,10 +821,7 @@ class BedrockProvider(PydanticProvider["PydanticBedrock"]):return BedrockConverseModel(model_name=self.model,provider=self.provider,- settings=BedrockModelSettings(- max_tokens=max_tokens,- # TODO: Add reasoning support- ),+ settings=BedrockModelSettings(max_tokens=max_tokens),)diff --git a/packages/llm-info/data/models.yml b/packages/llm-info/data/models.ymlindex fa89a962fd..b893dc0858 100644--- a/packages/llm-info/data/models.yml+++ b/packages/llm-info/data/models.yml@@ -3,6 +3,13 @@# Anthropic+- name: Claude Opus 4.7+ model: claude-opus-4-7+ description: Latest Opus model, strongest for coding and long-running professional tasks+ providers: [anthropic]+ roles: [chat, edit]+ thinking: true+- name: Claude Sonnet 4.6model: claude-sonnet-4-6description: Most capable Sonnet-class model, with frontier performance across coding, agents, and professional workdiff --git a/pyproject.toml b/pyproject.tomlindex a17e236fab..071ded2cc1 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -140,7 +140,7 @@ dev = [# For linting"ruff>=0.15.9",# For AI- "pydantic-ai-slim[openai]>=1.71.0",+ "pydantic-ai-slim[openai]>=1.84.0",]test = [@@ -203,7 +203,7 @@ test-optional = ["anywidget~=0.9.21","ipython~=8.12.3",# testing gen ai- "pydantic-ai-slim[google,anthropic,bedrock,openai]>=1.71.0",+ "pydantic-ai-slim[google,anthropic,bedrock,openai]>=1.84.0",# - google-auth uses cachetools, and cachetools<5.0.0 uses collections.MutableMapping (removed in Python 3.10)"cachetools>=5.0.0","boto3>=1.38.46",@@ -240,7 +240,7 @@ typecheck = ["sqlalchemy>=2.0.40","obstore>=0.8.2","fsspec>=2026.2.0",- "pydantic-ai-slim[google,anthropic,bedrock,openai]>=1.71.0",+ "pydantic-ai-slim[google,anthropic,bedrock,openai]>=1.84.0","loro>=1.5.0","boto3-stubs>=1.38.46","pandas-stubs>=1.5.3.230321",diff --git a/tests/_server/ai/test_providers.py b/tests/_server/ai/test_providers.pyindex edb5e3ff63..337dbef9fb 100644--- a/tests/_server/ai/test_providers.py+++ b/tests/_server/ai/test_providers.py@@ -1,5 +1,6 @@"""Tests for the LLM providers in marimo._server.ai.providers."""+from typing import Anyfrom unittest.mock import AsyncMock, MagicMock, patchimport pytest@@ -172,154 +173,181 @@ def test_anthropic_process_part_text_file() -> None:@pytest.mark.parametrize(- ("model_name", "base_url", "expected"),+ ("provider_kind", "model_name", "base_url", "expected_thinking"),[+ # OpenAI: profile drives the decision via supports_thinking.+ pytest.param("openai", "o1-mini", None, True, id="openai_o1_mini"),pytest.param(- "o1-mini",- None,- True,- id="o1_mini_no_base_url",- ),- pytest.param(+ "openai","o1-preview",- None,- True,- id="o1_preview_no_base_url",- ),- pytest.param(- "o1",- None,- True,- id="o1_no_base_url",- ),- pytest.param(- "o1-2024-12-17","https://api.openai.com/v1",True,- id="o1_dated_openai_base_url",- ),- pytest.param(- "o3-mini",- None,- True,- id="o3_mini_no_base_url",- ),- pytest.param(- "gpt-5-turbo",- None,- True,- id="gpt5_turbo_no_base_url",- ),- pytest.param(- "gpt-5-preview",- None,- True,- id="gpt5_preview_no_base_url",- ),- pytest.param(- "openai/o1-mini",- None,- True,- id="openai_prefix_o1_mini_no_base_url",+ id="openai_o1_preview_official_url",),+ pytest.param("openai", "o3", None, True, id="openai_o3"),+ pytest.param("openai", "o3-mini", None, True, id="openai_o3_mini"),+ pytest.param("openai", "gpt-5", None, True, id="openai_gpt5"),pytest.param(- "openai/o1-preview",- None,- True,- id="openai_prefix_o1_preview_no_base_url",+ "openai", "gpt-4", None, False, id="openai_gpt4_no_thinking"),pytest.param(- "openai/gpt-5-turbo",- None,- True,- id="openai_prefix_gpt5_no_base_url",+ "openai", "gpt-4o", None, False, id="openai_gpt4o_no_thinking"),+ # Custom base URL (litellm/vLLM/Together/etc.) suppresses thinking+ # even when the model name looks like a reasoning model: third-party+ # endpoints often don't accept `reasoning_effort`.pytest.param(+ "openai","o1-mini","https://custom.api.com/v1",False,- id="o1_custom_base_url",+ id="openai_o1_custom_base_url",),pytest.param(- "o1-preview",+ "openai",+ "gpt-5","https://litellm.proxy.com/api/v1",False,- id="o1_litellm_proxy",- ),- pytest.param(- "gpt-4",- None,- False,- id="gpt4_no_base_url",- ),- pytest.param(- "gpt-4o",- None,- False,- id="gpt4o_no_base_url",- ),- pytest.param(- "gpt-4",- "https://custom.api.com/v1",- False,- id="gpt4_custom_base_url",+ id="openai_gpt5_litellm_proxy",),+ # Azure: thinking is always suppressed (only custom Azure deployments+ # support reasoning_effort, which we don't expose yet).pytest.param(- "olive-model",- None,+ "azure",+ "o1-mini",+ "https://my.openai.azure.com/openai/deployments/o1-mini?api-version=2024-12-01-preview",False,- id="model_starting_with_o_but_not_reasoning",+ id="azure_o1_mini",),pytest.param(- "openrouter/o1-mini",- None,+ "azure",+ "gpt-5",+ "https://my.openai.azure.com/openai/deployments/gpt-5?api-version=2024-12-01-preview",False,- id="openrouter_prefix_not_openai",+ id="azure_gpt5",),],)@pytest.mark.requires("pydantic_ai")-def test_is_reasoning_model(- model_name: str, base_url: str | None, expected: bool+def test_openai_default_thinking(+ provider_kind: str,+ model_name: str,+ base_url: str | None,+ expected_thinking: bool,) -> None:- """Test that _is_reasoning_model correctly identifies reasoning models."""+ """The base url heuristic + pydantic-ai's profile drive the on/off decision.++ `openai_reasoning_summary` rides on the same profile-driven path: it is+ set iff `thinking` is, so we never send it to non-reasoning models or to+ custom OpenAI-compatible endpoints that wouldn't accept it.+ """config = AnyProviderConfig(api_key="test-key", base_url=base_url)- provider = OpenAIProvider(model_name, config)- assert provider._is_reasoning_model(model_name) == expected+ provider: OpenAIProvider = (+ AzureOpenAIProvider(model_name, config)+ if provider_kind == "azure"+ else OpenAIProvider(model_name, config)+ )+ model = provider.create_model(max_tokens=512)+ settings = provider._build_agent_settings(model)++ has_thinking = settings is not None and settings.get("thinking") is True+ has_summary = (+ settings is not None and "openai_reasoning_summary" in settings+ )+ assert has_thinking == expected_thinking+ assert has_summary == expected_thinking@pytest.mark.parametrize(- ("model_name", "expected"),+ (+ "model_name",+ "expected_model_settings",+ "expected_agent_thinking",+ ),[pytest.param(- "claude-opus-4-20250514",+ "claude-opus-4-7",+ # Opus 4.7 disallows sampling settings, so no temperature.+ {"max_tokens": 1024},True,- id="claude_opus_4",+ id="opus_4_7_adaptive_no_sampling",),pytest.param(- "claude-sonnet-4-20250514",+ "claude-opus-4-6",+ {"max_tokens": 1024, "temperature": 1},True,- id="claude_sonnet_4",+ id="opus_4_6",),pytest.param(- "claude-haiku-4-5-20250514",+ "claude-sonnet-4-6",+ {"max_tokens": 1024, "temperature": 1},True,- id="claude_haiku_4_5",+ id="sonnet_4_6",+ ),+ pytest.param(+ "claude-opus-4-5-20251101",+ {"max_tokens": 1024, "temperature": 1},+ True,+ id="opus_4_5",),pytest.param("claude-3-7-sonnet-20250219",+ {"max_tokens": 1024, "temperature": 1},True,- id="claude_3_7_sonnet",+ id="sonnet_3_7",),+ # NOTE: pydantic-ai's profile reports `supports_thinking=True` for all+ # Anthropic models — even 3.5 — so by trusting it we end up enabling+ # thinking on 3.5 too. The Anthropic API will reject thinking for 3.5+ # at request time. We accept that trade-off in exchange for not+ # maintaining our own per-model gate; if pydantic-ai's profile gets+ # corrected upstream, behavior here will follow automatically.pytest.param("claude-3-5-sonnet-20241022",- False,- id="claude_3_5_sonnet_not_thinking",+ {"max_tokens": 1024, "temperature": 1},+ True,+ id="sonnet_3_5_trusts_profile",+ ),+ ],+)+ not DependencyManager.anthropic.has()+ or not DependencyManager.pydantic_ai.has(),+ reason="anthropic or pydantic_ai not installed",+)+def test_anthropic_settings_split(+ model_name: str,+ expected_model_settings: dict[str, Any],+ expected_agent_thinking: bool,+) -> None:+ """Verify the model-level settings (temperature) and agent-level thinking flag."""+ config = AnyProviderConfig(api_key="test-key", base_url=None)+ provider = AnthropicProvider(model_name, config)+ model = provider.create_model(max_tokens=1024)+ assert dict(model.settings) == expected_model_settings++ agent_settings = provider._build_agent_settings(model)+ actual_thinking = (+ agent_settings is not None and agent_settings.get("thinking") is True+ )+ assert actual_thinking == expected_agent_thinking+++ ("model_name", "expected_payload_kind"),+ [+ # Adaptive-only / adaptive-supported models route to {'type': 'adaptive'}.+ pytest.param("claude-opus-4-7", "adaptive", id="opus_4_7_adaptive"),+ pytest.param("claude-opus-4-6", "adaptive", id="opus_4_6_adaptive"),+ pytest.param(+ "claude-sonnet-4-6", "adaptive", id="sonnet_4_6_adaptive"),+ # Older models route to {'type': 'enabled', 'budget_tokens': N}.pytest.param(- "claude-3-opus-20240229",- False,- id="claude_3_opus_not_thinking",+ "claude-opus-4-5-20251101", "enabled", id="opus_4_5_manual"+ ),+ pytest.param(+ "claude-3-7-sonnet-20250219", "enabled", id="sonnet_3_7_manual"),],)@@ -328,13 +356,65 @@ def test_is_reasoning_model(or not DependencyManager.pydantic_ai.has(),reason="anthropic or pydantic_ai not installed",)-def test_anthropic_is_extended_thinking_model(- model_name: str, expected: bool+def test_anthropic_thinking_payload_translation(+ model_name: str, expected_payload_kind: str) -> None:- """Test that is_extended_thinking_model correctly identifies thinking models."""+ """End-to-end: per-model Anthropic API payload via pydantic-ai's profile.++ Opus 4.7 is the critical case here: it only accepts `{"type": "adaptive"}`+ and rejects `{"type": "enabled", "budget_tokens": ...}` with HTTP 400.+ """+ from pydantic_ai.models import ModelRequestParameters+ from pydantic_ai.models.anthropic import AnthropicModel+config = AnyProviderConfig(api_key="test-key", base_url=None)provider = AnthropicProvider(model_name, config)- assert provider.is_extended_thinking_model(model_name) == expected+ model = provider.create_model(max_tokens=1024)+ assert isinstance(model, AnthropicModel)++ # Combine the model-level settings (temperature) with the agent-level+ # thinking flag the way pydantic-ai does at request time.+ merged = dict(model.settings or {})+ merged.update(provider._build_agent_settings(model) or {})++ prepared_settings, prepared_params = model.prepare_request(+ merged, ModelRequestParameters()+ )+ payload = model._translate_thinking( # type: ignore[attr-defined]+ prepared_settings or {}, prepared_params+ )+ if expected_payload_kind == "adaptive":+ assert payload == {"type": "adaptive"}+ else:+ assert payload["type"] == "enabled"+ assert payload["budget_tokens"] > 0+++ ("model_name", "expected_thinking"),+ [+ pytest.param("gemini-3-pro-preview", True, id="gemini_3_pro"),+ pytest.param("gemini-2.5-pro", True, id="gemini_2_5_pro"),+ pytest.param(+ "gemini-2.0-flash", False, id="gemini_2_0_flash_not_thinking"+ ),+ ],+)+ not DependencyManager.google_ai.has()+ or not DependencyManager.pydantic_ai.has(),+ reason="google or pydantic_ai not installed",+)+def test_google_default_thinking(+ model_name: str, expected_thinking: bool+) -> None:+ """Google's profile correctly distinguishes thinking vs non-thinking models."""+ config = AnyProviderConfig(api_key="test-key", base_url=None)+ provider = GoogleProvider(model_name, config)+ model = provider.create_model(max_tokens=512)+ settings = provider._build_agent_settings(model)+ actual = settings is not None and settings.get("thinking") is True+ assert actual == expected_thinking@pytest.mark.requires("pydantic_ai")