Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
refactor: enhance model validation and provider inference in LLM class (#3976)
Lorenze Jay4d8eec96e843
diff --git a/lib/crewai/src/crewai/llm.py b/lib/crewai/src/crewai/llm.pyindex 00d609f41..cc8bfefcd 100644--- a/lib/crewai/src/crewai/llm.py+++ b/lib/crewai/src/crewai/llm.py@@ -407,45 +407,99 @@ class LLM(BaseLLM):return instance@classmethod- def _validate_model_in_constants(cls, model: str, provider: str) -> bool:- """Validate if a model name exists in the provider's constants.+ def _matches_provider_pattern(cls, model: str, provider: str) -> bool:+ """Check if a model name matches provider-specific patterns.++ This allows supporting models that aren't in the hardcoded constants list,+ including "latest" versions and new models that follow provider naming conventions.Args:- model: The model name to validate+ model: The model name to checkprovider: The provider to check against (canonical name)Returns:- True if the model exists in the provider's constants, False otherwise+ True if the model matches the provider's naming pattern, False otherwise"""+ model_lower = model.lower()+if provider == "openai":- return model in OPENAI_MODELS+ return any(+ model_lower.startswith(prefix)+ for prefix in ["gpt-", "o1", "o3", "o4", "whisper-"]+ )if provider == "anthropic" or provider == "claude":- return model in ANTHROPIC_MODELS+ return any(+ model_lower.startswith(prefix) for prefix in ["claude-", "anthropic."]+ )- if provider == "gemini":- return model in GEMINI_MODELS+ if provider == "gemini" or provider == "google":+ return any(+ model_lower.startswith(prefix)+ for prefix in ["gemini-", "gemma-", "learnlm-"]+ )if provider == "bedrock":- return model in BEDROCK_MODELS+ return "." in model_lower++ if provider == "azure":+ return any(+ model_lower.startswith(prefix)+ for prefix in ["gpt-", "gpt-35-", "o1", "o3", "o4", "azure-"]+ )++ return False++ @classmethod+ def _validate_model_in_constants(cls, model: str, provider: str) -> bool:+ """Validate if a model name exists in the provider's constants or matches provider patterns.++ This method first checks the hardcoded constants list for known models.+ If not found, it falls back to pattern matching to support new models,+ "latest" versions, and models that follow provider naming conventions.++ Args:+ model: The model name to validate+ provider: The provider to check against (canonical name)++ Returns:+ True if the model exists in constants or matches provider patterns, False otherwise+ """+ if provider == "openai" and model in OPENAI_MODELS:+ return True++ if (+ provider == "anthropic" or provider == "claude"+ ) and model in ANTHROPIC_MODELS:+ return True++ if (provider == "gemini" or provider == "google") and model in GEMINI_MODELS:+ return True++ if provider == "bedrock" and model in BEDROCK_MODELS:+ return Trueif provider == "azure":# azure does not provide a list of available models, determine a better way to handle thisreturn True- return False+ # Fallback to pattern matching for models not in constants+ return cls._matches_provider_pattern(model, provider)@classmethoddef _infer_provider_from_model(cls, model: str) -> str:"""Infer the provider from the model name.+ This method first checks the hardcoded constants list for known models.+ If not found, it uses pattern matching to infer the provider from model name patterns.+ This allows supporting new models and "latest" versions without hardcoding.+Args:model: The model name without provider prefixReturns:The inferred provider name, defaults to "openai""""-if model in OPENAI_MODELS:return "openai"@@ -1699,12 +1753,14 @@ class LLM(BaseLLM):max_tokens=self.max_tokens,presence_penalty=self.presence_penalty,frequency_penalty=self.frequency_penalty,- logit_bias=copy.deepcopy(self.logit_bias, memo)- if self.logit_bias- else None,- response_format=copy.deepcopy(self.response_format, memo)- if self.response_format- else None,+ logit_bias=(+ copy.deepcopy(self.logit_bias, memo) if self.logit_bias else None+ ),+ response_format=(+ copy.deepcopy(self.response_format, memo)+ if self.response_format+ else None+ ),seed=self.seed,logprobs=self.logprobs,top_logprobs=self.top_logprobs,diff --git a/lib/crewai/src/crewai/llms/constants.py b/lib/crewai/src/crewai/llms/constants.pyindex fc4656455..02a138297 100644--- a/lib/crewai/src/crewai/llms/constants.py+++ b/lib/crewai/src/crewai/llms/constants.py@@ -182,6 +182,8 @@ OPENAI_MODELS: list[OpenAIModels] = [AnthropicModels: TypeAlias = Literal[+ "claude-opus-4-5-20251101",+ "claude-opus-4-5","claude-3-7-sonnet-latest","claude-3-7-sonnet-20250219","claude-3-5-haiku-latest",@@ -208,6 +210,8 @@ AnthropicModels: TypeAlias = Literal["claude-3-haiku-20240307",]ANTHROPIC_MODELS: list[AnthropicModels] = [+ "claude-opus-4-5-20251101",+ "claude-opus-4-5","claude-3-7-sonnet-latest","claude-3-7-sonnet-20250219","claude-3-5-haiku-latest",@@ -452,6 +456,7 @@ BedrockModels: TypeAlias = Literal["anthropic.claude-3-sonnet-20240229-v1:0:28k","anthropic.claude-haiku-4-5-20251001-v1:0","anthropic.claude-instant-v1:2:100k",+ "anthropic.claude-opus-4-5-20251101-v1:0","anthropic.claude-opus-4-1-20250805-v1:0","anthropic.claude-opus-4-20250514-v1:0","anthropic.claude-sonnet-4-20250514-v1:0",@@ -524,6 +529,7 @@ BEDROCK_MODELS: list[BedrockModels] = ["anthropic.claude-3-sonnet-20240229-v1:0:28k","anthropic.claude-haiku-4-5-20251001-v1:0","anthropic.claude-instant-v1:2:100k",+ "anthropic.claude-opus-4-5-20251101-v1:0","anthropic.claude-opus-4-1-20250805-v1:0","anthropic.claude-opus-4-20250514-v1:0","anthropic.claude-sonnet-4-20250514-v1:0",diff --git a/lib/crewai/tests/test_llm.py b/lib/crewai/tests/test_llm.pyindex 50df854d4..977d40f2c 100644--- a/lib/crewai/tests/test_llm.py+++ b/lib/crewai/tests/test_llm.py@@ -243,7 +243,11 @@ def test_validate_call_params_not_supported():# Patch supports_response_schema to simulate an unsupported model.with patch("crewai.llm.supports_response_schema", return_value=False):- llm = LLM(model="gemini/gemini-1.5-pro", response_format=DummyResponse, is_litellm=True)+ llm = LLM(+ model="gemini/gemini-1.5-pro",+ response_format=DummyResponse,+ is_litellm=True,+ )with pytest.raises(ValueError) as excinfo:llm._validate_call_params()assert "does not support response_format" in str(excinfo.value)@@ -702,13 +706,16 @@ def test_ollama_does_not_modify_when_last_is_user(ollama_llm):assert formatted == original_messages+def test_native_provider_raises_error_when_supported_but_fails():"""Test that when a native provider is in SUPPORTED_NATIVE_PROVIDERS but fails to instantiate, we raise the error."""with patch("crewai.llm.SUPPORTED_NATIVE_PROVIDERS", ["openai"]):with patch("crewai.llm.LLM._get_native_provider") as mock_get_native:# Mock that provider exists but throws an error when instantiatedmock_provider = MagicMock()- mock_provider.side_effect = ValueError("Native provider initialization failed")+ mock_provider.side_effect = ValueError(+ "Native provider initialization failed"+ )mock_get_native.return_value = mock_providerwith pytest.raises(ImportError) as excinfo:@@ -751,16 +758,16 @@ def test_prefixed_models_with_valid_constants_use_native_sdk():def test_prefixed_models_with_invalid_constants_use_litellm():- """Test that models with native provider prefixes use LiteLLM when model is NOT in constants."""+ """Test that models with native provider prefixes use LiteLLM when model is NOT in constants and does NOT match patterns."""# Test openai/ prefix with non-OpenAI model (not in OPENAI_MODELS) → LiteLLMllm = LLM(model="openai/gemini-2.5-flash", is_litellm=False)assert llm.is_litellm is Trueassert llm.model == "openai/gemini-2.5-flash"- # Test openai/ prefix with unknown future model → LiteLLM- llm2 = LLM(model="openai/gpt-future-6", is_litellm=False)+ # Test openai/ prefix with model that doesn't match patterns (e.g. no gpt- prefix) → LiteLLM+ llm2 = LLM(model="openai/custom-finetune-model", is_litellm=False)assert llm2.is_litellm is True- assert llm2.model == "openai/gpt-future-6"+ assert llm2.model == "openai/custom-finetune-model"# Test anthropic/ prefix with non-Anthropic model → LiteLLMllm3 = LLM(model="anthropic/gpt-4o", is_litellm=False)@@ -768,6 +775,21 @@ def test_prefixed_models_with_invalid_constants_use_litellm():assert llm3.model == "anthropic/gpt-4o"+def test_prefixed_models_with_valid_patterns_use_native_sdk():+ """Test that models matching provider patterns use native SDK even if not in constants."""+ with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}):+ llm = LLM(model="openai/gpt-future-6", is_litellm=False)+ assert llm.is_litellm is False+ assert llm.provider == "openai"+ assert llm.model == "gpt-future-6"++ with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "test-key"}):+ llm2 = LLM(model="anthropic/claude-future-5", is_litellm=False)+ assert llm2.is_litellm is False+ assert llm2.provider == "anthropic"+ assert llm2.model == "claude-future-5"++def test_prefixed_models_with_non_native_providers_use_litellm():"""Test that models with non-native provider prefixes always use LiteLLM."""# Test groq/ prefix (not a native provider) → LiteLLM@@ -821,19 +843,36 @@ def test_validate_model_in_constants():"""Test the _validate_model_in_constants method."""# OpenAI modelsassert LLM._validate_model_in_constants("gpt-4o", "openai") is True- assert LLM._validate_model_in_constants("gpt-future-6", "openai") is False+ assert LLM._validate_model_in_constants("gpt-future-6", "openai") is True+ assert LLM._validate_model_in_constants("o1-latest", "openai") is True+ assert LLM._validate_model_in_constants("unknown-model", "openai") is False# Anthropic modelsassert LLM._validate_model_in_constants("claude-opus-4-0", "claude") is True- assert LLM._validate_model_in_constants("claude-future-5", "claude") is False+ assert LLM._validate_model_in_constants("claude-future-5", "claude") is True+ assert (+ LLM._validate_model_in_constants("claude-3-5-sonnet-latest", "claude") is True+ )+ assert LLM._validate_model_in_constants("unknown-model", "claude") is False# Gemini modelsassert LLM._validate_model_in_constants("gemini-2.5-pro", "gemini") is True- assert LLM._validate_model_in_constants("gemini-future", "gemini") is False+ assert LLM._validate_model_in_constants("gemini-future", "gemini") is True+ assert LLM._validate_model_in_constants("gemma-3-latest", "gemini") is True+ assert LLM._validate_model_in_constants("unknown-model", "gemini") is False# Azure modelsassert LLM._validate_model_in_constants("gpt-4o", "azure") is Trueassert LLM._validate_model_in_constants("gpt-35-turbo", "azure") is True# Bedrock models- assert LLM._validate_model_in_constants("anthropic.claude-opus-4-1-20250805-v1:0", "bedrock") is True+ assert (+ LLM._validate_model_in_constants(+ "anthropic.claude-opus-4-1-20250805-v1:0", "bedrock"+ )+ is True+ )+ assert (+ LLM._validate_model_in_constants("anthropic.claude-future-v1:0", "bedrock")+ is True+ )