Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

feat(llm): add provider input token count support (#8109)

Vidit Ostwalaaaa312cbac8
diff --git a/lib/crewai/src/crewai/llm.py b/lib/crewai/src/crewai/llm.py
index 17264cee3..834d94876 100644
--- a/lib/crewai/src/crewai/llm.py
+++ b/lib/crewai/src/crewai/llm.py
@@ -773,6 +773,45 @@ class LLM(BaseLLM):
return {k: v for k, v in params.items() if v is not None}
+ def get_input_tokens_count(
+ self,
+ messages: str | list[LLMMessage],
+ tools: list[dict[str, BaseTool]] | None = None,
+ response_model: type[BaseModel] | None = None,
+ ) -> int:
+ """Estimate request input tokens with LiteLLM's local tokenizer.
+
+ LiteLLM does not make a provider request for this calculation. Its
+ tokenizer uses the same formatted messages and tools as a completion.
+ """
+ if not _ensure_litellm() or litellm is None:
+ raise ImportError(
+ 'LiteLLM is required for token estimation; install it with `uv add "crewai[litellm]"`.'
+ )
+
+ params = self._prepare_completion_params(messages, tools)
+ if response_model is not None:
+ from instructor import Mode, handle_response_model
+
+ _, structured_output_params = handle_response_model(
+ response_model,
+ mode=Mode.TOOLS,
+ tools=params.get("tools"),
+ )
+ params["tools"] = structured_output_params["tools"]
+ params["tool_choice"] = structured_output_params["tool_choice"]
+ try:
+ return int(
+ litellm.token_counter(
+ model=params["model"],
+ messages=params["messages"],
+ tools=params.get("tools"),
+ tool_choice=params.get("tool_choice"),
+ )
+ )
+ except Exception:
+ return super().get_input_tokens_count(messages, tools, response_model)
+
def _handle_streaming_response(
self,
params: dict[str, Any],
@@ -2197,7 +2236,9 @@ class LLM(BaseLLM):
"""
return extract_choices_finish_reason_and_id(response_or_chunk)
- def _process_message_files(self, messages: list[LLMMessage]) -> list[LLMMessage]:
+ def _process_message_files(
+ self, messages: list[LLMMessage], *, api: str | None = None
+ ) -> list[LLMMessage]:
"""Process files attached to messages and format for provider.
For each message with a `files` field, formats the files into
diff --git a/lib/crewai/src/crewai/llms/base_llm.py b/lib/crewai/src/crewai/llms/base_llm.py
index cdc66962d..33a511070 100644
--- a/lib/crewai/src/crewai/llms/base_llm.py
+++ b/lib/crewai/src/crewai/llms/base_llm.py
@@ -86,6 +86,7 @@ class LLMCallBlockedError(ValueError):
DEFAULT_CONTEXT_WINDOW_SIZE: Final[int] = 4096
DEFAULT_SUPPORTS_STOP_WORDS: Final[bool] = True
_JSON_EXTRACTION_PATTERN: Final[re.Pattern[str]] = re.compile(r"\{.*}", re.DOTALL)
+logger = logging.getLogger(__name__)
_current_call_id: contextvars.ContextVar[str | None] = contextvars.ContextVar(
"_current_call_id", default=None
@@ -363,6 +364,43 @@ class BaseLLM(BaseModel, ABC):
return config
+ def get_input_tokens_count(
+ self,
+ messages: str | list[LLMMessage],
+ tools: list[dict[str, BaseTool]] | None = None,
+ response_model: type[BaseModel] | None = None,
+ ) -> int:
+ """Estimate the input-token count a request will consume.
+
+ Providers with native token-estimation support should override this
+ method. This fallback estimates one token per four text characters,
+ rounded up, so callers can safely preflight requests when a native
+ count endpoint is unavailable.
+
+ Args:
+ messages: Input messages for the LLM.
+ tools: Optional tool schemas included in the request.
+ response_model: Optional structured-output model included in the
+ request.
+
+ Returns:
+ A conservative estimate of the provider's input-token count.
+ """
+ logger.warning(
+ "Using an approximate input token count for model %r: assuming one "
+ "token per four characters because a native token count is unavailable.",
+ self.model,
+ )
+ from crewai.utilities.agent_utils import message_content_text
+
+ if isinstance(messages, str):
+ character_count = len(messages)
+ else:
+ character_count = sum(
+ len(message_content_text(message)) for message in messages
+ )
+ return (character_count + 3) // 4
+
@abstractmethod
def call(
self,
@@ -884,7 +922,9 @@ class BaseLLM(BaseModel, ABC):
return None
- def _format_messages(self, messages: str | list[LLMMessage]) -> list[LLMMessage]:
+ def _format_messages(
+ self, messages: str | list[LLMMessage], *, file_api: str | None = None
+ ) -> list[LLMMessage]:
"""Convert messages to standard format.
Args:
@@ -921,9 +961,11 @@ class BaseLLM(BaseModel, ABC):
}
cleaned.append(cast(_LLMMessage, copy))
- return self._process_message_files(cleaned)
+ return self._process_message_files(cleaned, api=file_api)
- def _process_message_files(self, messages: list[LLMMessage]) -> list[LLMMessage]:
+ def _process_message_files(
+ self, messages: list[LLMMessage], *, api: str | None = None
+ ) -> list[LLMMessage]:
"""Process files attached to messages and format for the provider.
For each message with a `files` field, formats the files into
@@ -948,7 +990,7 @@ class BaseLLM(BaseModel, ABC):
return messages
formatter = self._multimodal_formatter_name()
- api = getattr(self, "api", None)
+ api = api if api is not None else getattr(self, "api", None)
for msg in messages:
files = msg.get("files")
diff --git a/lib/crewai/src/crewai/llms/providers/anthropic/completion.py b/lib/crewai/src/crewai/llms/providers/anthropic/completion.py
index 2c757b5ae..a3d8d7833 100644
--- a/lib/crewai/src/crewai/llms/providers/anthropic/completion.py
+++ b/lib/crewai/src/crewai/llms/providers/anthropic/completion.py
@@ -140,7 +140,7 @@ def _is_pydantic_model_class(obj: Any) -> TypeGuard[type[BaseModel]]:
return isinstance(obj, type) and issubclass(obj, BaseModel)
-def _contains_file_id_reference(messages: list[dict[str, Any]]) -> bool:
+def _contains_file_id_reference(messages: list[LLMMessage]) -> bool:
"""Check if any message content contains a file_id reference.
Anthropic's Files API is in beta and requires a special header when
@@ -334,6 +334,89 @@ class AnthropicCompletion(BaseLLM):
self._async_client = self._build_async_client()
return self._async_client
+ def get_input_tokens_count(
+ self,
+ messages: str | list[LLMMessage],
+ tools: list[dict[str, Any]] | None = None,
+ response_model: type[BaseModel] | None = None,
+ ) -> int:
+ """Get request input tokens with Anthropic's Messages API.
+
+ The token-count request uses the same message and tool formatting as a
+ completion, while excluding generation-only parameters such as output
+ limits and sampling settings.
+ """
+ try:
+ formatted_messages, system_message = self._format_messages_for_anthropic(
+ messages
+ )
+ completion_params = self._prepare_completion_params(
+ formatted_messages, system_message, tools
+ )
+ count_params = {
+ key: completion_params[key]
+ for key in (
+ "model",
+ "messages",
+ "system",
+ "thinking",
+ "tool_choice",
+ "tools",
+ )
+ if key in completion_params
+ }
+ effective_response_model = response_model or self.response_format
+ uses_file_api = _contains_file_id_reference(formatted_messages)
+
+ def count_tokens(
+ *,
+ betas: list[str] | None = None,
+ output_format: dict[str, Any] | None = None,
+ ) -> Any:
+ if betas:
+ return self._get_sync_client().beta.messages.count_tokens(
+ **count_params,
+ betas=betas,
+ **({"output_format": output_format} if output_format else {}),
+ )
+ return self._get_sync_client().messages.count_tokens(**count_params)
+
+ if _is_pydantic_model_class(effective_response_model):
+ schema = transform_schema(effective_response_model.model_json_schema())
+ if _supports_native_structured_outputs(self.model):
+ betas = [ANTHROPIC_STRUCTURED_OUTPUTS_BETA]
+ if uses_file_api:
+ betas.append(ANTHROPIC_FILES_API_BETA)
+ response = count_tokens(
+ betas=betas,
+ output_format={"type": "json_schema", "schema": schema},
+ )
+ else:
+ count_params["tools"] = [
+ {
+ "name": "structured_output",
+ "description": "Output the structured response",
+ "input_schema": schema,
+ }
+ ]
+ count_params["tool_choice"] = {
+ "type": "tool",
+ "name": "structured_output",
+ }
+ if uses_file_api:
+ response = count_tokens(betas=[ANTHROPIC_FILES_API_BETA])
+ else:
+ response = count_tokens()
+ elif uses_file_api:
+ response = count_tokens(betas=[ANTHROPIC_FILES_API_BETA])
+ else:
+ response = count_tokens()
+ if response is None:
+ return super().get_input_tokens_count(messages, tools, response_model)
+ return int(response.input_tokens)
+ except Exception:
+ return super().get_input_tokens_count(messages, tools, response_model)
+
def to_config_dict(self) -> dict[str, Any]:
"""Extend base config with Anthropic-specific fields."""
config = super().to_config_dict()
diff --git a/lib/crewai/src/crewai/llms/providers/azure/completion.py b/lib/crewai/src/crewai/llms/providers/azure/completion.py
index 7b7f7d9f0..0941ae35b 100644
--- a/lib/crewai/src/crewai/llms/providers/azure/completion.py
+++ b/lib/crewai/src/crewai/llms/providers/azure/completion.py
@@ -247,6 +247,15 @@ class AzureCompletion(BaseLLM):
self._responses_delegate = OpenAICompletion(**delegate_kwargs)
+ def get_input_tokens_count(
+ self,
+ messages: str | list[LLMMessage],
+ tools: list[dict[str, Any]] | None = None,
+ response_model: type[BaseModel] | None = None,
+ ) -> int:
+ """Estimate input tokens until Azure offers a count endpoint."""
+ return super().get_input_tokens_count(messages, tools, response_model)
+
def _get_responses_base_url(self) -> str:
"""Construct the base URL for the Azure OpenAI Responses API.
diff --git a/lib/crewai/src/crewai/llms/providers/bedrock/completion.py b/lib/crewai/src/crewai/llms/providers/bedrock/completion.py
index 50845ed6c..a6fe8bfd2 100644
--- a/lib/crewai/src/crewai/llms/providers/bedrock/completion.py
+++ b/lib/crewai/src/crewai/llms/providers/bedrock/completion.py
@@ -336,6 +336,81 @@ class BedrockCompletion(BaseLLM):
using ``aiobotocore`` inside an exit stack."""
return self._async_client
+ def get_input_tokens_count(
+ self,
+ messages: str | list[LLMMessage],
+ tools: list[dict[Any, Any]] | None = None,
+ response_model: type[BaseModel] | None = None,
+ ) -> int:
+ """Get request input tokens with Bedrock's CountTokens API.
+
+ The CountTokens request accepts the Converse message, system, tool, and
+ additional-model fields that affect input tokenization. Generation-only
+ settings, such as output limits and sampling, are excluded.
+ """
+ formatted_messages, system_message = self._format_messages_for_converse(
+ messages
+ )
+ converse_input: dict[str, Any] = {"messages": formatted_messages}
+
+ if system_message:
+ converse_input["system"] = [{"text": system_message}]
+
+ if tools:
+ converse_input["toolConfig"] = {
+ "tools": self._format_tools_for_converse(tools)
+ }
+ elif self._messages_contain_tool_content(formatted_messages):
+ tools_from_history = self._extract_tools_from_message_history(
+ formatted_messages
+ )
+ if tools_from_history:
+ converse_input["toolConfig"] = {"tools": tools_from_history}
+
+ if self.additional_model_request_fields:
+ converse_input["additionalModelRequestFields"] = (
+ self.additional_model_request_fields
+ )
+
+ effective_response_model = response_model or self.response_format
+ if effective_response_model:
+ structured_tool: ConverseToolTypeDef = {
+ "toolSpec": {
+ "name": STRUCTURED_OUTPUT_TOOL_NAME,
+ "description": (
+ "Use this tool to provide your final structured response. "
+ "Call this tool when you have gathered all necessary information "
+ "and are ready to provide the final answer in the required format."
+ ),
+ "inputSchema": {
+ "json": generate_model_description(effective_response_model)
+ .get("json_schema", {})
+ .get("schema", {})
+ },
+ }
+ }
+ tool_config = converse_input.get("toolConfig")
+ existing_tools = list(tool_config.get("tools", [])) if tool_config else []
+ if not any(
+ tool.get("toolSpec", {}).get("name") == STRUCTURED_OUTPUT_TOOL_NAME
+ for tool in existing_tools
+ ):
+ existing_tools.append(structured_tool)
+ converse_input["toolConfig"] = {"tools": existing_tools}
+ if len(existing_tools) == 1:
+ converse_input["toolConfig"]["toolChoice"] = {
+ "tool": {"name": STRUCTURED_OUTPUT_TOOL_NAME}
+ }
+
+ try:
+ response = self._get_sync_client().count_tokens(
+ modelId=self.model_id,
+ input={"converse": converse_input},
+ )
+ except ClientError:
+ return super().get_input_tokens_count(messages, tools, response_model)
+ return int(response["inputTokens"])
+
def to_config_dict(self) -> dict[str, Any]:
"""Extend base config with Bedrock-specific fields."""
config = super().to_config_dict()
diff --git a/lib/crewai/src/crewai/llms/providers/gemini/completion.py b/lib/crewai/src/crewai/llms/providers/gemini/completion.py
index 1129d4112..6ab0039ee 100644
--- a/lib/crewai/src/crewai/llms/providers/gemini/completion.py
+++ b/lib/crewai/src/crewai/llms/providers/gemini/completion.py
@@ -143,6 +143,65 @@ class GeminiCompletion(BaseLLM):
"""Gemini uses a single client for both sync and async calls."""
return self._get_sync_client()
+ def get_input_tokens_count(
+ self,
+ messages: str | list[LLMMessage],
+ tools: list[dict[str, Any]] | None = None,
+ response_model: type[BaseModel] | None = None,
+ ) -> int:
+ """Get request input tokens with Gemini's Count Tokens API."""
+ formatted_content, system_instruction = self._format_messages_for_gemini(
+ messages
+ )
+ generation_config = self._prepare_generation_config(
+ system_instruction, tools, response_model or self.response_format
+ )
+ config_data = generation_config.model_dump(exclude_none=True)
+ client = self._get_sync_client()
+ if not getattr(client, "vertexai", False):
+ unsupported_context = [
+ name
+ for name, value in (
+ ("system instructions", config_data.get("system_instruction")),
+ ("tools", config_data.get("tools")),
+ ("structured output", response_model or self.response_format),
+ )
+ if value is not None
+ ]
+ if unsupported_context:
+ return super().get_input_tokens_count(messages, tools, response_model)
+ try:
+ response = client.models.count_tokens(
+ model=self.model,
+ contents=formatted_content,
+ )
+ except Exception:
+ return super().get_input_tokens_count(messages, tools, response_model)
+ return int(response.total_tokens)
+
+ count_config_params: dict[str, Any] = {
+ "system_instruction": config_data.pop("system_instruction", None),
+ "tools": config_data.pop("tools", None),
+ }
+ generation_config_data = {
+ key: value
+ for key, value in config_data.items()
+ if key in types.GenerationConfig.model_fields
+ }
+ count_config_params["generation_config"] = types.GenerationConfig(
+ **generation_config_data
+ )
+ count_config = types.CountTokensConfig(**count_config_params)
+ try:
+ response = client.models.count_tokens(
+ model=self.model,
+ contents=formatted_content,
+ config=count_config,
+ )
+ except Exception:
+ return super().get_input_tokens_count(messages, tools, response_model)
+ return int(response.total_tokens)
+
def to_config_dict(self) -> dict[str, Any]:
"""Extend base config with Gemini/Vertex-specific fields."""
config = super().to_config_dict()
diff --git a/lib/crewai/src/crewai/llms/providers/openai/completion.py b/lib/crewai/src/crewai/llms/providers/openai/completion.py
index 26d8fc7ab..b221f558a 100644
--- a/lib/crewai/src/crewai/llms/providers/openai/completion.py
+++ b/lib/crewai/src/crewai/llms/providers/openai/completion.py
@@ -578,6 +578,48 @@ class OpenAICompletion(BaseLLM):
return client_params
+ def get_input_tokens_count(
+ self,
+ messages: str | list[LLMMessage],
+ tools: list[dict[str, BaseTool]] | None = None,
+ response_model: type[BaseModel] | None = None,
+ ) -> int:
+ """Get request input tokens with OpenAI's Responses API.
+
+ The count endpoint uses the configured model but is independent of the
+ generation endpoint selected by :attr:`api`. This lets Chat Completions
+ users preflight an equivalent Responses payload without changing their
+ subsequent generation request.
+ """
+ if self._on_compatible_server():
+ return super().get_input_tokens_count(messages, tools, response_model)
+
+ formatted_messages = self._format_messages(messages, file_api="responses")
+ response_params = self._prepare_responses_params(
+ messages=formatted_messages,
+ tools=tools,
+ response_model=response_model,
+ )
+ count_params = {
+ key: response_params[key]
+ for key in (
+ "model",
+ "input",
+ "instructions",
+ "text",
+ "tools",
+ "previous_response_id",
+ )
+ if key in response_params
+ }
+ try:
+ response = self._get_sync_client().responses.input_tokens.count(
+ **count_params
+ )
+ except Exception:
+ return super().get_input_tokens_count(messages, tools, response_model)
+ return int(response.input_tokens)
+
def call(
self,
messages: str | list[LLMMessage],
@@ -3028,9 +3070,11 @@ class OpenAICompletion(BaseLLM):
return result
return {"total_tokens": 0}
- def _format_messages(self, messages: str | list[LLMMessage]) -> list[LLMMessage]:
+ def _format_messages(
+ self, messages: str | list[LLMMessage], *, file_api: str | None = None
+ ) -> list[LLMMessage]:
"""Format messages for OpenAI API."""
- base_formatted = super()._format_messages(messages)
+ base_formatted = super()._format_messages(messages, file_api=file_api)
formatted_messages: list[LLMMessage] = []
diff --git a/lib/crewai/src/crewai/llms/providers/snowflake/completion.py b/lib/crewai/src/crewai/llms/providers/snowflake/completion.py
index fa9151c0b..2e02d397e 100644
--- a/lib/crewai/src/crewai/llms/providers/snowflake/completion.py
+++ b/lib/crewai/src/crewai/llms/providers/snowflake/completion.py
@@ -4,7 +4,7 @@ import ast
import os
from typing import Any, Literal
-from pydantic import model_validator
+from pydantic import BaseModel, model_validator
from crewai.llms.providers.openai.completion import OpenAICompletion
from crewai.utilities.types import LLMMessage
@@ -131,8 +131,24 @@ class SnowflakeCompletion(OpenAICompletion):
"SNOWFLAKE_ACCOUNT, or pass account_url/base_url/account_identifier."
)
- def _format_messages(self, messages: str | list[LLMMessage]) -> list[LLMMessage]:
- formatted_messages = super()._format_messages(messages)
+ def get_input_tokens_count(
+ self,
+ messages: str | list[LLMMessage],
+ tools: list[dict[str, Any]] | None = None,
+ response_model: type[BaseModel] | None = None,
+ ) -> int:
+ """Estimate input tokens until Snowflake Cortex support is implemented.
+
+ Snowflake's OpenAI-compatible Cortex endpoint supports only Chat
+ Completions, while the inherited OpenAI implementation requires the
+ Responses input-token-count endpoint.
+ """
+ return super().get_input_tokens_count(messages, tools, response_model)
+
+ def _format_messages(
+ self, messages: str | list[LLMMessage], *, file_api: str | None = None
+ ) -> list[LLMMessage]:
+ formatted_messages = super()._format_messages(messages, file_api=file_api)
if self._is_claude_model():
formatted_messages = self._normalize_stringified_tool_calls(
formatted_messages
diff --git a/lib/crewai/tests/cassettes/llms/anthropic/test_anthropic_input_token_count_live.yaml b/lib/crewai/tests/cassettes/llms/anthropic/test_anthropic_input_token_count_live.yaml
new file mode 100644
index 000000000..f362f55eb
--- /dev/null
+++ b/lib/crewai/tests/cassettes/llms/anthropic/test_anthropic_input_token_count_live.yaml
@@ -0,0 +1,76 @@
+interactions:
+- request:
+ body: '{"messages":[{"role":"user","content":"Count these input tokens."}],"model":"claude-haiku-4-5"}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - ACCEPT-ENCODING-XXX
+ anthropic-version:
+ - '2023-06-01'
+ connection:
+ - keep-alive
+ content-length:
+ - '95'
+ content-type:
+ - application/json
+ host:
+ - api.anthropic.com
+ user-agent:
+ - X-USER-AGENT-XXX
+ x-stainless-arch:
+ - X-STAINLESS-ARCH-XXX
+ x-stainless-async:
+ - 'false'
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - X-STAINLESS-OS-XXX
+ x-stainless-package-version:
+ - 0.73.0
+ x-stainless-retry-count:
+ - '0'
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.14
+ x-stainless-timeout:
+ - NOT_GIVEN
+ method: POST
+ uri: https://api.anthropic.com/v1/messages/count_tokens
+ response:
+ body:
+ string: '{"input_tokens":12}'
+ headers:
+ anthropic-organization-id:
+ - ANTHROPIC-ORGANIZATION-ID-XXX
+ anthropic-workspace-id:
+ - ANTHROPIC-WORKSPACE-ID-XXX
+ cf-cache-status:
+ - DYNAMIC
+ cf-ray:
+ - CF-RAY-XXX
+ connection:
+ - keep-alive
+ content-security-policy:
+ - CSP-FILTERED
+ content-type:
+ - application/json
+ date:
+ - Tue, 13 Oct 2026 12:05:28 GMT
+ request-id:
+ - REQUEST-ID-XXX
+ server:
+ - cloudflare
+ server-timing:
+ - x-originResponse;dur=69
+ transfer-encoding:
+ - chunked
+ vary:
+ - Accept-Encoding
+ x-robots-tag:
+ - none
+ status:
+ code: 200
+ message: OK
+version: 1
diff --git a/lib/crewai/tests/cassettes/llms/google/test_gemini_developer_input_token_count.yaml b/lib/crewai/tests/cassettes/llms/google/test_gemini_developer_input_token_count.yaml
new file mode 100644
index 000000000..1468289e1
--- /dev/null
+++ b/lib/crewai/tests/cassettes/llms/google/test_gemini_developer_input_token_count.yaml
@@ -0,0 +1,56 @@
+interactions:
+- request:
+ body: '{"contents": [{"parts": [{"text": "CrewAI coordinates autonomous AI agents
+ to complete a shared task."}], "role": "user"}]}'
+ headers:
+ User-Agent:
+ - X-USER-AGENT-XXX
+ accept:
+ - '*/*'
+ accept-encoding:
+ - ACCEPT-ENCODING-XXX
+ connection:
+ - keep-alive
+ content-length:
+ - '123'
+ content-type:
+ - application/json
+ host:
+ - generativelanguage.googleapis.com
+ x-goog-api-client:
+ - google-genai-sdk/1.65.0 gl-python/3.13.14
+ x-goog-api-key:
+ - X-GOOG-API-KEY-XXX
+ method: POST
+ uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:countTokens
+ response:
+ body:
+ string: "{\n \"totalTokens\": 13,\n \"promptTokensDetails\": [\n {\n \"modality\":
+ \"TEXT\",\n \"tokenCount\": 13\n }\n ]\n}\n"
+ headers:
+ alt-svc:
+ - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
+ content-type:
+ - application/json; charset=UTF-8
+ date:
+ - Tue, 13 Oct 2026 17:55:09 GMT
+ server:
+ - scaffolding on HTTPServer2
+ server-timing:
+ - gfet4t7; dur=266
+ transfer-encoding:
+ - chunked
+ vary:
+ - Origin
+ - X-Origin
+ - Referer
+ x-content-type-options:
+ - X-CONTENT-TYPE-XXX
+ x-frame-options:
+ - X-FRAME-OPTIONS-XXX
+ x-xss-protection:
+ - X-XSS-PROTECTION-XXX
+ status:
+ code: 200
+ message: OK
+version: 1
diff --git a/lib/crewai/tests/cassettes/llms/openai/test_openai_input_token_count_live.yaml b/lib/crewai/tests/cassettes/llms/openai/test_openai_input_token_count_live.yaml
new file mode 100644
index 000000000..ad69b8d12
--- /dev/null
+++ b/lib/crewai/tests/cassettes/llms/openai/test_openai_input_token_count_live.yaml
@@ -0,0 +1,82 @@
+interactions:
+- request:
+ body: '{"input":[{"role":"user","content":"Count these input tokens."}],"model":"gpt-4o-mini"}'
+ headers:
+ User-Agent:
+ - X-USER-AGENT-XXX
+ accept:
+ - application/json
+ accept-encoding:
+ - ACCEPT-ENCODING-XXX
+ authorization:
+ - AUTHORIZATION-XXX
+ connection:
+ - keep-alive
+ content-length:
+ - '87'
+ content-type:
+ - application/json
+ host:
+ - api.openai.com
+ x-stainless-arch:
+ - X-STAINLESS-ARCH-XXX
+ x-stainless-async:
+ - 'false'
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - X-STAINLESS-OS-XXX
+ x-stainless-package-version:
+ - 2.41.0
+ x-stainless-read-timeout:
+ - X-STAINLESS-READ-TIMEOUT-XXX
+ x-stainless-retry-count:
+ - '0'
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.14
+ method: POST
+ uri: https://api.openai.com/v1/responses/input_tokens
+ response:
+ body:
+ string: "{\n \"object\": \"response.input_tokens\",\n \"input_tokens\": 12\n}"
+ headers:
+ access-control-expose-headers:
+ - ACCESS-CONTROL-XXX
+ alt-svc:
+ - h3=":443"; ma=86400
+ cf-cache-status:
+ - DYNAMIC
+ cf-ray:
+ - CF-RAY-XXX
+ connection:
+ - keep-alive
+ content-type:
+ - application/json
+ date:
+ - Tue, 13 Oct 2026 09:39:37 GMT
+ openai-organization:
+ - OPENAI-ORG-XXX
+ openai-processing-ms:
+ - '280'
+ openai-project:
+ - OPENAI-PROJECT-XXX
+ openai-version:
+ - '2020-10-01'
+ server:
+ - cloudflare
+ set-cookie:
+ - SET-COOKIE-XXX
+ strict-transport-security:
+ - STS-XXX
+ transfer-encoding:
+ - chunked
+ x-content-type-options:
+ - X-CONTENT-TYPE-XXX
+ x-request-id:
+ - X-REQUEST-ID-XXX
+ status:
+ code: 200
+ message: OK
+version: 1
diff --git a/lib/crewai/tests/llms/anthropic/test_anthropic.py b/lib/crewai/tests/llms/anthropic/test_anthropic.py
index b6b2d8b6f..059627ba9 100644
--- a/lib/crewai/tests/llms/anthropic/test_anthropic.py
+++ b/lib/crewai/tests/llms/anthropic/test_anthropic.py
@@ -648,6 +648,129 @@ def test_anthropic_token_usage_tracking():
assert usage["total_tokens"] == 75
+def test_anthropic_input_token_count_forwards_completion_payload():
+ """Counting formats system messages and tools like a completion request."""
+ llm = LLM(model="anthropic/claude-3-5-sonnet-20241022")
+ client = MagicMock()
+ client.messages.count_tokens.return_value = MagicMock(input_tokens=42)
+ messages = [
+ {"role": "system", "content": "Be concise."},
+ {"role": "user", "content": "What is the weather?"},
+ ]
+ tools = [
+ {
+ "name": "get_weather",
+ "description": "Get weather for a location.",
+ "parameters": {"type": "object", "properties": {}},
+ }
+ ]
+
+ with patch.object(llm, "_get_sync_client", return_value=client):
+ assert llm.get_input_tokens_count(messages, tools=tools) == 42
+
+ client.messages.count_tokens.assert_called_once_with(
+ model="claude-3-5-sonnet-20241022",
+ system="Be concise.",
+ messages=[{"role": "user", "content": "What is the weather?"}],
+ tools=[
+ {
+ "name": "get_weather",
+ "description": "Get weather for a location.",
+ "input_schema": {"type": "object", "properties": {}},
+ }
+ ],
+ )
+
+
+def test_anthropic_input_token_count_includes_files_api_beta():
+ """File-backed messages use the same beta header as generation."""
+ llm = LLM(model="anthropic/claude-3-5-sonnet-20241022")
+ client = MagicMock()
+ client.beta.messages.count_tokens.return_value = MagicMock(input_tokens=42)
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "document",
+ "source": {"type": "file", "file_id": "file_123"},
+ }
+ ],
+ }
+ ]
+
+ with patch.object(llm, "_get_sync_client", return_value=client):
+ assert llm.get_input_tokens_count(messages) == 42
+
+ client.beta.messages.count_tokens.assert_called_once_with(
+ model="claude-3-5-sonnet-20241022",
+ messages=messages,
+ betas=["files-api-2025-04-14"],
+ )
+
+
+def test_anthropic_input_token_count_uses_character_fallback():
+ """Count endpoint errors fall back to the character estimate."""
+ llm = LLM(model="anthropic/claude-3-5-sonnet-20241022")
+ client = MagicMock()
+ client.messages.count_tokens.side_effect = RuntimeError("count failed")
+
+ with patch.object(llm, "_get_sync_client", return_value=client):
+ assert llm.get_input_tokens_count("12345") == 2
+
+
+def test_anthropic_input_token_count_uses_files_beta_for_structured_output_fallback():
+ """Structured output fallback retains the Files API beta header."""
+ class Answer(BaseModel):
+ value: str
+
+ llm = LLM(model="anthropic/claude-3-5-sonnet-20241022")
+ client = MagicMock()
+ client.beta.messages.count_tokens.return_value = MagicMock(input_tokens=42)
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "document",
+ "source": {"type": "file", "file_id": "file_123"},
+ }
+ ],
+ }
+ ]
+
+ with patch.object(llm, "_get_sync_client", return_value=client):
+ assert llm.get_input_tokens_count(messages, response_model=Answer) == 42
+
+ client.beta.messages.count_tokens.assert_called_once_with(
+ model="claude-3-5-sonnet-20241022",
+ messages=messages,
+ tools=[
+ {
+ "name": "structured_output",
+ "description": "Output the structured response",
+ "input_schema": {
+ "additionalProperties": False,
+ "properties": {"value": {"title": "Value", "type": "string"}},
+ "required": ["value"],
+ "title": "Answer",
+ "type": "object",
+ },
+ }
+ ],
+ tool_choice={"type": "tool", "name": "structured_output"},
+ betas=["files-api-2025-04-14"],
+ )
+
+
[email protected](filter_headers=["authorization", "x-api-key"])
+def test_anthropic_input_token_count_live():
+ """Anthropic returns a positive input-token count for a real request."""
+ llm = LLM(model="anthropic/claude-haiku-4-5")
+
+ assert llm.get_input_tokens_count("Count these input tokens.") > 0
+
+
def test_anthropic_stop_sequences_sync():
"""Test that stop and stop_sequences attributes stay synchronized."""
llm = LLM(model="anthropic/claude-3-5-sonnet-20241022")
diff --git a/lib/crewai/tests/llms/azure/test_azure_responses.py b/lib/crewai/tests/llms/azure/test_azure_responses.py
index 4047f284d..82fa6da2b 100644
--- a/lib/crewai/tests/llms/azure/test_azure_responses.py
+++ b/lib/crewai/tests/llms/azure/test_azure_responses.py
@@ -16,6 +16,7 @@ class _FakeOpenAICompletion:
def __init__(self) -> None:
self.call = MagicMock(return_value="responses-result")
self.acall = AsyncMock(return_value="async-responses-result")
+ self.get_input_tokens_count = MagicMock(return_value=42)
self.last_response_id = "resp_abc123"
self.last_reasoning_items = [{"type": "reasoning"}]
self.reset_chain = MagicMock()
@@ -241,6 +242,12 @@ class TestAzureResponsesCall:
assert isinstance(result, str)
assert len(result) > 0
+ def test_input_token_count_uses_character_fallback_for_responses(self, mock_openai_completion):
+ _mock_cls, _instance = mock_openai_completion
+ comp = _create_azure_responses()
+
+ assert comp.get_input_tokens_count("12345") == 2
+
# Delegated property & method tests
diff --git a/lib/crewai/tests/llms/bedrock/test_bedrock.py b/lib/crewai/tests/llms/bedrock/test_bedrock.py
index f1e019f37..a492a298e 100644
--- a/lib/crewai/tests/llms/bedrock/test_bedrock.py
+++ b/lib/crewai/tests/llms/bedrock/test_bedrock.py
@@ -694,6 +694,75 @@ def test_bedrock_token_usage_tracking():
assert llm._token_usage['total_tokens'] == 75
+def test_bedrock_input_token_count_forwards_converse_payload(bedrock_mocks):
+ """Counting preserves the Converse system, message, and tool payload."""
+ _, client = bedrock_mocks
+ llm = LLM(model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0")
+ client.count_tokens.return_value = {"inputTokens": 42}
+ messages = [
+ {"role": "system", "content": "Be concise."},
+ {"role": "user", "content": "What is the weather?"},
+ ]
+ tools = [
+ {
+ "name": "get_weather",
+ "description": "Get weather for a location.",
+ "parameters": {"type": "object", "properties": {}},
+ }
+ ]
+
+ assert llm.get_input_tokens_count(messages, tools=tools) == 42
+
+ client.count_tokens.assert_called_once_with(
+ modelId="anthropic.claude-3-5-sonnet-20241022-v2:0",
+ input={
+ "converse": {
+ "system": [{"text": "Be concise."}],
+ "messages": [
+ {
+ "role": "user",
+ "content": [{"text": "What is the weather?"}],
+ }
+ ],
+ "toolConfig": {
+ "tools": [
+ {
+ "toolSpec": {
+ "name": "get_weather",
+ "description": "Get weather for a location.",
+ "inputSchema": {
+ "json": {"type": "object", "properties": {}}
+ },
+ }
+ }
+ ]
+ },
+ }
+ },
+ )
+
+
+def test_bedrock_input_token_count_uses_character_fallback_for_unsupported_models(
+ bedrock_mocks,
+):
+ """Runtime models without CountTokens report the Mantle requirement."""
+ from botocore.exceptions import ClientError
+
+ _, client = bedrock_mocks
+ llm = LLM(model="bedrock/us.anthropic.claude-opus-4-7-v1:0")
+ client.count_tokens.side_effect = ClientError(
+ {
+ "Error": {
+ "Code": "ValidationException",
+ "Message": "This model does not support CountTokens.",
+ }
+ },
+ "CountTokens",
+ )
+
+ assert llm.get_input_tokens_count("12345") == 2
+
+
def test_bedrock_tool_use_conversation_flow():
"""
Test that the Bedrock completion properly handles tool use conversation flow
diff --git a/lib/crewai/tests/llms/google/test_google.py b/lib/crewai/tests/llms/google/test_google.py
index af9095601..8a747f285 100644
--- a/lib/crewai/tests/llms/google/test_google.py
+++ b/lib/crewai/tests/llms/google/test_google.py
@@ -139,6 +139,106 @@ def test_gemini_completion_initialization_parameters():
assert llm.top_k == 40
+def test_gemini_input_token_count_forwards_generation_context_for_vertex():
+ """Vertex Count Tokens receives the same prompt context as completion."""
+ llm = LLM(model="gemini/gemini-2.0-flash-001", api_key="test-key")
+ client = MagicMock()
+ client.vertexai = True
+ client.models.count_tokens.return_value = MagicMock(total_tokens=42)
+ messages = [
+ {"role": "system", "content": "Be concise."},
+ {"role": "user", "content": "What is CrewAI?"},
+ ]
+ tools = [
+ {
+ "type": "function",
+ "function": {
+ "name": "search_docs",
+ "description": "Search the documentation.",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ }
+ ]
+
+ with patch.object(llm, "_get_sync_client", return_value=client):
+ assert llm.get_input_tokens_count(messages, tools=tools) == 42
+
+ count_kwargs = client.models.count_tokens.call_args.kwargs
+ assert count_kwargs["model"] == "gemini-2.0-flash-001"
+ assert count_kwargs["contents"][0].role == "user"
+ assert count_kwargs["config"].system_instruction.parts[0].text == "Be concise."
+ assert count_kwargs["config"].tools[0].function_declarations[0].name == "search_docs"
+
+
+def test_gemini_developer_token_count_uses_character_fallback_for_unrepresentable_context():
+ """Unsupported Developer API context falls back to the character estimate."""
+ llm = LLM(model="gemini/gemini-2.0-flash-001", api_key="test-key")
+ client = MagicMock()
+ client.vertexai = False
+
+ with patch.object(llm, "_get_sync_client", return_value=client):
+ assert llm.get_input_tokens_count(
+ [
+ {"role": "system", "content": "Be concise."},
+ {"role": "user", "content": "What is CrewAI?"},
+ ],
+ tools=[
+ {
+ "type": "function",
+ "function": {
+ "name": "search_docs",
+ "description": "Search the documentation.",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ }
+ ],
+ ) == 7
+
+
+def test_gemini_developer_token_count_forwards_supported_content():
+ """Developer API counting succeeds when the prompt has no unsupported context."""
+ llm = LLM(model="gemini/gemini-2.0-flash-001", api_key="test-key")
+ client = MagicMock()
+ client.vertexai = False
+ client.models.count_tokens.return_value = MagicMock(total_tokens=42)
+
+ with patch.object(llm, "_get_sync_client", return_value=client):
+ assert llm.get_input_tokens_count("What is CrewAI?") == 42
+
+ count_kwargs = client.models.count_tokens.call_args.kwargs
+ assert count_kwargs["model"] == "gemini-2.0-flash-001"
+ assert count_kwargs["contents"][0].role == "user"
+ assert count_kwargs["contents"][0].parts[0].text == "What is CrewAI?"
+
+
+def test_gemini_developer_input_token_count(monkeypatch):
+ """Gemini Developer API counts input tokens through its native endpoint."""
+ llm = LLM(model="gemini/gemini-2.5-flash")
+ models = llm._get_sync_client().models
+ native_count_tokens = models.count_tokens
+ calls = []
+ errors = []
+
+ def count_tokens(*args, **kwargs):
+ calls.append((args, kwargs))
+ try:
+ return native_count_tokens(*args, **kwargs)
+ except Exception as error:
+ errors.append(error)
+ raise
+
+ monkeypatch.setattr(models, "count_tokens", count_tokens)
+
+ token_count = llm.get_input_tokens_count(
+ "CrewAI coordinates autonomous AI agents to complete a shared task."
+ )
+
+ assert len(calls) == 1
+ assert not errors
+ assert token_count > 0
+
+
def test_gemini_started_event_surfaces_max_output_tokens():
from crewai.events.event_bus import crewai_event_bus
from crewai.events.types.llm_events import LLMCallStartedEvent
diff --git a/lib/crewai/tests/llms/openai/test_openai.py b/lib/crewai/tests/llms/openai/test_openai.py
index 9c2b653c9..1a7706f02 100644
--- a/lib/crewai/tests/llms/openai/test_openai.py
+++ b/lib/crewai/tests/llms/openai/test_openai.py
@@ -66,6 +66,20 @@ def test_custom_openai_flag_uses_native_openai_without_provider_prefix():
assert rebuilt.base_url == "https://gateway.example/v1"
+ "kwargs",
+ [
+ {"custom_openai": True, "base_url": "https://vllm.example/v1"},
+ {"base_url": "https://vllm.example/v1"},
+ ],
+)
+def test_openai_compatible_token_count_uses_character_fallback(kwargs):
+ """Compatible endpoints fall back when they lack Responses counting."""
+ llm = OpenAICompletion(model="custom-model", api_key="test-key", **kwargs)
+
+ assert llm.get_input_tokens_count("12345") == 2
+
+
def test_custom_openai_flag_requires_custom_base_url():
"""Avoid routing arbitrary custom model ids to api.openai.com by mistake."""
with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}, clear=True):
@@ -801,6 +815,140 @@ def test_openai_responses_api_default_is_completions():
assert llm.api == "completions"
+def test_openai_input_token_count_uses_responses_without_changing_generation_api():
+ """Counting uses Responses while generation keeps the configured API."""
+ llm = OpenAICompletion(model="gpt-4o", api="completions")
+ client = MagicMock()
+ client.responses.input_tokens.count.return_value = MagicMock(input_tokens=17)
+
+ with patch.object(llm, "_get_sync_client", return_value=client):
+ assert llm.get_input_tokens_count("Hello!") == 17
+
+ client.responses.input_tokens.count.assert_called_once_with(
+ model="gpt-4o",
+ input=[{"role": "user", "content": "Hello!"}],
+ )
+ assert llm.api == "completions"
+
+ with patch.object(
+ llm, "_call_completions", return_value="completion result"
+ ) as mock_completions:
+ assert llm.call("Hello!") == "completion result"
+
+ mock_completions.assert_called_once()
+
+
+def test_openai_input_token_count_uses_responses_file_format():
+ """File content is formatted for the Responses count endpoint."""
+ llm = OpenAICompletion(model="gpt-4o", api="completions")
+ client = MagicMock()
+ client.responses.input_tokens.count.return_value = MagicMock(input_tokens=17)
+
+ with (
+ patch.object(
+ llm,
+ "_format_messages",
+ return_value=[{"role": "user", "content": "Hello!"}],
+ ) as format_messages,
+ patch.object(llm, "_get_sync_client", return_value=client),
+ ):
+ assert llm.get_input_tokens_count("Hello!") == 17
+
+ format_messages.assert_called_once_with("Hello!", file_api="responses")
+
+
+ ("previous_response_id", "auto_chain", "last_response_id"),
+ [
+ ("resp_explicit", False, None),
+ (None, True, "resp_auto_chain"),
+ ],
+)
+def test_openai_input_token_count_forwards_previous_response_id(
+ previous_response_id, auto_chain, last_response_id
+):
+ """Counting preserves the conversation context selected for generation."""
+ llm = OpenAICompletion(
+ model="gpt-4o",
+ previous_response_id=previous_response_id,
+ auto_chain=auto_chain,
+ )
+ llm._last_response_id = last_response_id
+ client = MagicMock()
+ client.responses.input_tokens.count.return_value = MagicMock(input_tokens=17)
+
+ with patch.object(llm, "_get_sync_client", return_value=client):
+ assert llm.get_input_tokens_count("Hello!") == 17
+
+ assert client.responses.input_tokens.count.call_args.kwargs[
+ "previous_response_id"
+ ] == (previous_response_id or last_response_id)
+
+
+def test_openai_input_token_count_forwards_responses_payload():
+ """Messages, multimodal content, and tools use the Responses request shape."""
+ llm = OpenAICompletion(model="gpt-4o", api="responses")
+ client = MagicMock()
+ client.responses.input_tokens.count.return_value = MagicMock(input_tokens=42)
+ messages = [
+ {"role": "system", "content": "Be concise."},
+ {
+ "role": "user",
+ "content": [
+ {"type": "input_text", "text": "Describe this image."},
+ {
+ "type": "input_image",
+ "image_url": "https://example.com/image.png",
+ },
+ ],
+ },
+ ]
+ tools = [
+ {
+ "name": "get_weather",
+ "description": "Get weather for a location.",
+ "parameters": {"type": "object", "properties": {}},
+ }
+ ]
+
+ with patch.object(llm, "_get_sync_client", return_value=client):
+ assert llm.get_input_tokens_count(messages, tools=tools) == 42
+
+ client.responses.input_tokens.count.assert_called_once_with(
+ model="gpt-4o",
+ instructions="Be concise.",
+ input=[messages[1]],
+ tools=[
+ {
+ "type": "function",
+ "name": "get_weather",
+ "description": "Get weather for a location.",
+ "parameters": {"type": "object", "properties": {}},
+ }
+ ],
+ )
+
+
+def test_openai_input_token_count_falls_back_after_api_errors(caplog):
+ """Count-endpoint errors fall back to the character estimate."""
+ llm = OpenAICompletion(model="gpt-4o")
+ client = MagicMock()
+ client.responses.input_tokens.count.side_effect = RuntimeError("count failed")
+
+ with patch.object(llm, "_get_sync_client", return_value=client):
+ assert llm.get_input_tokens_count("Hello!") == 2
+
+ assert "assuming one token per four characters" in caplog.text
+
+
[email protected](record_mode="once")
+def test_openai_input_token_count_live():
+ """OpenAI returns a positive input-token count for a real request."""
+ llm = OpenAICompletion(model="gpt-4o-mini")
+
+ assert llm.get_input_tokens_count("Count these input tokens.") > 0
+
+
def test_openai_responses_api_prepare_params():
"""Test that Responses API params are prepared correctly."""
llm = OpenAICompletion(
diff --git a/lib/crewai/tests/llms/snowflake/test_snowflake.py b/lib/crewai/tests/llms/snowflake/test_snowflake.py
index 552088044..439c27ab8 100644
--- a/lib/crewai/tests/llms/snowflake/test_snowflake.py
+++ b/lib/crewai/tests/llms/snowflake/test_snowflake.py
@@ -98,6 +98,12 @@ class TestSnowflakeConfig:
with pytest.raises(ValueError, match="supports only the Chat Completions API"):
SnowflakeCompletion(model="openai-gpt-4.1", api="responses")
+ def test_input_token_count_uses_character_fallback(self, monkeypatch: pytest.MonkeyPatch):
+ _snowflake_env(monkeypatch)
+ llm = SnowflakeCompletion(model="openai-gpt-4.1")
+
+ assert llm.get_input_tokens_count("12345") == 2
+
class TestSnowflakeFactory:
def test_llm_creates_native_snowflake_provider(
diff --git a/lib/crewai/tests/test_custom_llm.py b/lib/crewai/tests/test_custom_llm.py
index af07cfd03..ae35cc8fe 100644
--- a/lib/crewai/tests/test_custom_llm.py
+++ b/lib/crewai/tests/test_custom_llm.py
@@ -140,6 +140,14 @@ def test_custom_llm_message_formatting():
assert result == "Test response"
+def test_base_llm_input_token_count_estimates_text_characters(caplog):
+ """Custom providers have a conservative fallback without native support."""
+ custom_llm = CustomLLM(model="test-model")
+
+ assert custom_llm.get_input_tokens_count("12345") == 2
+ assert "assuming one token per four characters" in caplog.text
+
+
class JWTAuthLLM(BaseLLM):
"""Custom LLM implementation with JWT authentication."""
diff --git a/lib/crewai/tests/test_llm.py b/lib/crewai/tests/test_llm.py
index c461334a6..50842c14a 100644
--- a/lib/crewai/tests/test_llm.py
+++ b/lib/crewai/tests/test_llm.py
@@ -810,6 +810,67 @@ def test_litellm_non_gpt5_sends_stop_in_params():
)
+def test_litellm_input_token_count_uses_formatted_messages_and_tools():
+ llm = LLM(model="openai/gpt-4o-mini", is_litellm=True)
+ tools = [
+ {
+ "type": "function",
+ "function": {
+ "name": "search_docs",
+ "description": "Search the documentation.",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ }
+ ]
+
+ with patch("crewai.llm.litellm.token_counter", return_value=42) as token_counter:
+ assert llm.get_input_tokens_count("What is CrewAI?", tools=tools) == 42
+
+ token_counter.assert_called_once_with(
+ model="openai/gpt-4o-mini",
+ messages=[{"role": "user", "content": "What is CrewAI?"}],
+ tools=tools,
+ tool_choice=None,
+ )
+
+
+def test_litellm_input_token_count_includes_structured_output_tool():
+ class Answer(BaseModel):
+ value: str
+
+ llm = LLM(model="openai/gpt-4o-mini", is_litellm=True)
+
+ with patch("crewai.llm.litellm.token_counter", return_value=42) as token_counter:
+ assert llm.get_input_tokens_count("What is CrewAI?", response_model=Answer) == 42
+
+ count_kwargs = token_counter.call_args.kwargs
+ assert count_kwargs["tools"] == [
+ {
+ "type": "function",
+ "function": {
+ "name": "Answer",
+ "description": "Correctly extracted `Answer` with all the required parameters with correct types",
+ "parameters": {
+ "properties": {"value": {"title": "Value", "type": "string"}},
+ "required": ["value"],
+ "type": "object",
+ },
+ },
+ }
+ ]
+ assert count_kwargs["tool_choice"] == {
+ "type": "function",
+ "function": {"name": "Answer"},
+ }
+
+
+def test_litellm_input_token_count_uses_character_fallback():
+ llm = LLM(model="openai/gpt-4o-mini", is_litellm=True)
+
+ with patch("crewai.llm.litellm.token_counter", side_effect=RuntimeError):
+ assert llm.get_input_tokens_count("12345") == 2
+
+
def test_litellm_retry_catches_litellm_unsupported_params_error(caplog):
"""
Test that the retry logic catches LiteLLM's UnsupportedParamsError format