The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
refactor(types): replace Any with proven types in 15 files (#47509)
Mateo Wang04eaa6b4a86d
diff --git a/litellm/a2a_protocol/litellm_completion_bridge/handler.py b/litellm/a2a_protocol/litellm_completion_bridge/handler.pyindex 0002a597d6a..6d3b09aed68 100644--- a/litellm/a2a_protocol/litellm_completion_bridge/handler.py+++ b/litellm/a2a_protocol/litellm_completion_bridge/handler.py@@ -80,7 +80,7 @@ class A2ACompletionBridgeHandler:@staticmethoddef _build_completion_params(params: dict[str, Any],- litellm_params: Mapping[str, Any],+ litellm_params: Mapping[str, object],api_base: str | None,agent_extra_headers: Mapping[str, str] | None,*,diff --git a/litellm/harness/handlers/deepagents_handler.py b/litellm/harness/handlers/deepagents_handler.pyindex 102f4a741cf..4a36b1a08d1 100644--- a/litellm/harness/handlers/deepagents_handler.py+++ b/litellm/harness/handlers/deepagents_handler.py@@ -183,7 +183,7 @@ class DeepAgentsHandler(BaseHarnessHandler):breakresume: dict[str, object] = {} # mutable-ok: Command(resume=) payload, filled per answered approvalfor interrupt in state.interrupts:- decisions: list[dict[str, Any]] = [] # mutable-ok: HITL decisions collected across awaited approvals+ decisions: list[dict[str, str]] = [] # mutable-ok: HITL decisions collected across awaited approvalsfor request in approval_requests(getattr(interrupt, "value", None)):approval = Approval(tool=normalized_tool_name(str(request.get("name") or "")),diff --git a/litellm/interactions/streaming_iterator.py b/litellm/interactions/streaming_iterator.pyindex a2468cd894f..f4ad57bf7b4 100644--- a/litellm/interactions/streaming_iterator.py+++ b/litellm/interactions/streaming_iterator.py@@ -11,6 +11,7 @@ from datetime import datetimefrom typing import Any, Finalimport httpx+from pydantic import ConfigDict, TypeAdapterfrom litellm._logging import verbose_loggerfrom litellm.constants import STREAM_SSE_DONE_STRING@@ -25,6 +26,10 @@ from litellm.types.interactions import ()from litellm.utils import CustomStreamWrapper+_INTERACTIONS_STREAMING_CHUNK_ADAPTER: Final = TypeAdapter(+ object, config=ConfigDict(strict=True, hide_input_in_errors=True)+)+class BaseInteractionsAPIStreamingIterator:"""@@ -83,7 +88,7 @@ class BaseInteractionsAPIStreamingIterator:try:# Parse the JSON chunk- parsed_chunk: Final = json.loads(stripped_chunk)+ parsed_chunk: Final = _INTERACTIONS_STREAMING_CHUNK_ADAPTER.validate_python(json.loads(stripped_chunk))# Format as InteractionsAPIStreamingResponseif isinstance(parsed_chunk, dict):diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.pyindex 0db8f5c33ed..6f08d6b0d61 100644--- a/litellm/llms/anthropic/common_utils.py+++ b/litellm/llms/anthropic/common_utils.py@@ -712,7 +712,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):optional_params: dict,model: str,param: str,- value: Any,+ value: object,drop_params: bool,output_key: str,) -> None:@@ -1788,6 +1788,21 @@ def strip_thinking_blocks_from_anthropic_messages_request_dict(data.pop("thinking", None)+_DROP_ANTHROPIC_MESSAGE: Final = object()+++def _strip_empty_content_blocks_from_message(message: object) -> object:+ if not isinstance(message, dict) or not isinstance(message.get("content"), list):+ return message+ content: Final = message["content"]+ filtered: Final = [+ block for block in content if not _is_empty_text_block(block) and not is_empty_unsigned_thinking_block(block)+ ]+ if len(filtered) == len(content):+ return message+ return {**message, "content": filtered} if filtered else _DROP_ANTHROPIC_MESSAGE++def strip_empty_content_blocks_from_anthropic_messages(messages: Sequence[object],) -> list[Any]:@@ -1822,18 +1837,11 @@ def strip_empty_content_blocks_from_anthropic_messages(The caller's list and its content blocks are never mutated; modifiedmessages are returned as shallow copies with a fresh content list."""- out: Final[list[Any]] = []- for m in messages:- if not isinstance(m, dict) or not isinstance(m.get("content"), list):- out.append(m)- continue- content = m["content"]- filtered = [b for b in content if not _is_empty_text_block(b) and not is_empty_unsigned_thinking_block(b)]- if len(filtered) == len(content):- out.append(m)- elif filtered:- out.append({**m, "content": filtered})- return out+ return [+ stripped_message+ for message in messages+ if (stripped_message := _strip_empty_content_blocks_from_message(message)) is not _DROP_ANTHROPIC_MESSAGE+ ]def _is_empty_text_block(block: object) -> bool:@@ -1921,18 +1929,15 @@ def sanitize_tool_use_ids_in_anthropic_messages(messages: Sequence[object]) -> land ``:`` — valid on the upstream provider but rejected by Anthropic whenthe session is switched to a native Anthropic deployment."""- out: Final[list[Any]] = []- for m in messages:- if not isinstance(m, dict) or not isinstance(m.get("content"), list):- out.append(m)- continue- content = m["content"]- new_content = [_sanitize_tool_use_id_content_block(b) for b in content]- if new_content == content:- out.append(m)- else:- out.append({**m, "content": new_content})- return out+ return [_sanitize_tool_use_ids_in_message(message) for message in messages]+++def _sanitize_tool_use_ids_in_message(message: object) -> object:+ if not isinstance(message, dict) or not isinstance(message.get("content"), list):+ return message+ content: Final = message["content"]+ new_content: Final = [_sanitize_tool_use_id_content_block(block) for block in content]+ return message if new_content == content else {**message, "content": new_content}class _ReplayedSearchQuery(LiteLLMBaseModel):diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.pyindex 0fafd4abd62..bd0f57c3ed9 100644--- a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py+++ b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py@@ -464,33 +464,32 @@ class LiteLLMAnthropicToResponsesAPIAdapter:return input_items+ @staticmethod+ def _translate_tool_to_responses_api(tool_dict: Mapping[str, object]) -> dict[str, object]:+ tool_type: Final = tool_dict.get("type", "")+ tool_name: Final = tool_dict.get("name", "")+ # web_search tool+ if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search":+ return {"type": "web_search_preview"}+ # Responses turns strict mode on when `strict` is omitted, silently rewriting+ # `required` to every property. Anthropic tools are non-strict unless asked.+ func_tool: Final[dict[str, object]] = {+ "type": "function",+ "name": tool_name,+ "strict": bool(tool_dict.get("strict")),+ }+ if "description" in tool_dict:+ func_tool["description"] = tool_dict["description"]+ if "input_schema" in tool_dict:+ func_tool["parameters"] = json_copy_or_original(tool_dict["input_schema"])+ return func_tool+def translate_tools_to_responses_api(self,tools: list[AllAnthropicToolsValues],) -> list[dict[str, object]]:"""Convert Anthropic tool definitions to Responses API function tools."""- result: Final[list[dict[str, object]]] = []- for tool in tools:- tool_dict = cast(dict[str, Any], tool)- tool_type = tool_dict.get("type", "")- tool_name = tool_dict.get("name", "")- # web_search tool- if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search":- result.append({"type": "web_search_preview"})- continue- # Responses turns strict mode on when `strict` is omitted, silently rewriting- # `required` to every property. Anthropic tools are non-strict unless asked.- func_tool: dict[str, object] = {- "type": "function",- "name": tool_name,- "strict": bool(tool_dict.get("strict")),- }- if "description" in tool_dict:- func_tool["description"] = tool_dict["description"]- if "input_schema" in tool_dict:- func_tool["parameters"] = json_copy_or_original(tool_dict["input_schema"])- result.append(func_tool)- return result+ return [self._translate_tool_to_responses_api(tool) for tool in tools]@staticmethoddef translate_tool_choice_to_responses_api(diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.pyindex ee2f6eb9f1a..5fe81a7f1b7 100644--- a/litellm/llms/base_llm/guardrail_translation/base_translation.py+++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py@@ -185,7 +185,7 @@ class BaseTranslation(ABC):request_data: dict | None = None,stream_transform_sink: StreamTransformSink | None = None,deliver_ended_stream_rewrites: bool = False,- ) -> Any:+ ) -> object:"""Process output streaming response with guardrails.diff --git a/litellm/llms/base_llm/managed_resources/base_managed_resource.py b/litellm/llms/base_llm/managed_resources/base_managed_resource.pyindex bb4cf42fa1e..d3b05716452 100644--- a/litellm/llms/base_llm/managed_resources/base_managed_resource.py+++ b/litellm/llms/base_llm/managed_resources/base_managed_resource.py@@ -28,7 +28,7 @@ if TYPE_CHECKING:from litellm.proxy.utils import PrismaClient as _PrismaClientfrom litellm.router import Router as _Router- Span = _Span | Any+ Span = _SpanInternalUsageCache = _InternalUsageCachePrismaClient = _PrismaClientRouter = _Router@@ -550,7 +550,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):limit: int | None = None,after: str | None = None,additional_filters: Mapping[str, object] | None = None,- ) -> dict[str, Any]:+ ) -> dict[str, object]:"""List resources created by a user.diff --git a/litellm/llms/chatgpt/chat/transformation.py b/litellm/llms/chatgpt/chat/transformation.pyindex 7c985b7dd5e..a1aac7f315b 100644--- a/litellm/llms/chatgpt/chat/transformation.py+++ b/litellm/llms/chatgpt/chat/transformation.py@@ -72,7 +72,7 @@ class ChatGPTConfig(OpenAIConfig):default_headers: Final = get_chatgpt_default_headers(api_key or "", account_id, session_id)return {**default_headers, **validated_headers}- def post_stream_processing(self, stream: Any) -> Any:+ def post_stream_processing(self, stream: Any) -> ChatGPTToolCallNormalizer:return ChatGPTToolCallNormalizer(stream)def map_openai_params(diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.pyindex 166a42adfc0..58a801eebe4 100644--- a/litellm/proxy/db/db_spend_update_writer.py+++ b/litellm/proxy/db/db_spend_update_writer.py@@ -13,7 +13,7 @@ import osimport randomimport timeimport traceback-from collections.abc import AsyncGenerator, Callable, Coroutine, Mapping, Sequence+from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable, Mapping, Sequencefrom contextlib import asynccontextmanagerfrom contextvars import ContextVarfrom datetime import datetime, timedelta, timezone@@ -21,7 +21,7 @@ from types import MappingProxyTypefrom typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, overloadfrom urllib.parse import quote, unquote-from pydantic import TypeAdapter+from pydantic import ConfigDict, TypeAdapterfrom typing_extensions import LiteralString, ReadOnly, TypedDictimport litellm@@ -102,6 +102,24 @@ else:RESPONSES_SESSION_CALL_TYPES: Final = frozenset({CallTypes.responses.value, CallTypes.aresponses.value})_SPEND_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object])+_JsonPayload: TypeAlias = Mapping[str, object] | Sequence[object] | str | int | float | bool | None+_JSON_VALUE_ADAPTER: Final[TypeAdapter[_JsonPayload]] = TypeAdapter(+ _JsonPayload, config=ConfigDict(strict=True, hide_input_in_errors=True)+)+++def _parsed_request_tags(request_tags: object) -> Iterable[object] | TypeError | None:+ if isinstance(request_tags, list):+ return request_tags+ if not isinstance(request_tags, str):+ return None+ parsed: Final = _JSON_VALUE_ADAPTER.validate_python(safe_json_loads(request_tags, default=[]))+ if not parsed:+ verbose_proxy_logger.debug("Failed to parse request_tags JSON: %s", request_tags)+ return None+ if isinstance(parsed, (Sequence, Mapping)):+ return parsed+ return TypeError(f"'{type(parsed).__name__}' object is not iterable")def _org_member_transaction_key(org_id: str, user_id: str) -> str:@@ -1441,16 +1459,11 @@ class DBSpendUpdateWriter:return# Parse tags from JSON string- tags: Sequence[object] = []- if isinstance(request_tags, str):- tags = safe_json_loads(request_tags, default=[])- if not tags:- verbose_proxy_logger.debug("Failed to parse request_tags JSON: %s", request_tags)- return- elif isinstance(request_tags, list):- tags = request_tags- else:+ tags: Final[Iterable[object] | TypeError | None] = _parsed_request_tags(request_tags)+ if tags is None:return+ if isinstance(tags, TypeError):+ raise tags# Update spend for each tagfor tag_name in tags:diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.pyindex 9d051eb90d6..da937bd0325 100644--- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py+++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py@@ -8,19 +8,39 @@ sensitive information like SSNs, credit cards, API keys, etc.import jsonimport osimport re-from collections.abc import Iterator+from collections.abc import Iterator, Sequencefrom enum import Enumfrom re import Patternfrom typing import Any, Final+from pydantic import ConfigDict, TypeAdapter, with_config+from typing_extensions import ReadOnly, TypedDict+from litellm.proxy.guardrails.content_filter_data import DATA_ROOTS, category_dirs-def _load_patterns_from_json() -> dict:+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))+class _PatternEntry(TypedDict):+ name: ReadOnly[str]+ display_name: ReadOnly[str]+ pattern: ReadOnly[str]+ category: ReadOnly[str]+ description: ReadOnly[str]+++@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))+class _PatternsFile(TypedDict):+ patterns: ReadOnly[Sequence[_PatternEntry]]+++_PATTERNS_FILE: Final = TypeAdapter(_PatternsFile)+++def _load_patterns_from_json() -> _PatternsFile:"""Load pattern definitions from patterns.json file"""json_path: Final = os.path.join(os.path.dirname(__file__), "patterns.json")with open(json_path, "r") as f:- return json.load(f)+ return _PATTERNS_FILE.validate_python(json.load(f))# Load patterns from JSONdiff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.pyindex 0595d23bd9a..517a05773c8 100644--- a/litellm/proxy/management_endpoints/team_endpoints.py+++ b/litellm/proxy/management_endpoints/team_endpoints.py@@ -1848,7 +1848,7 @@ async def new_team(members_with_roles = complete_team_data.members_with_rolescomplete_team_data.members_with_roles = []- complete_team_data_dict = complete_team_data.model_dump(exclude_none=True)+ complete_team_data_dict: dict[str, object] = complete_team_data.model_dump(exclude_none=True)# Serialize router_settings to JSON (matching key creation pattern)router_settings_value: Final = data.router_settings@@ -2563,8 +2563,8 @@ async def update_team(# Drop server-owned metadata keys from caller input so they can only# be written by the same code path that creates the underlying rows.- if isinstance(updated_kv.get("metadata"), dict):- TeamMemberBudgetHandler.strip_system_managed_metadata_keys(updated_kv["metadata"])+ if isinstance(metadata_update := updated_kv.get("metadata"), dict):+ TeamMemberBudgetHandler.strip_system_managed_metadata_keys(metadata_update)if "metadata" in updated_kv:stored_metadata: Final[Mapping[str, JsonValue] | None] = (diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.pyindex 31732f2dbc9..84278f4619d 100644--- a/litellm/proxy/spend_tracking/spend_management_endpoints.py+++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py@@ -16,7 +16,6 @@ from typing import (NamedTuple,Protocol,TypeAlias,- TypedDict,TypeVar,cast, # noqa: TID251 # custom-logger and cold-storage payloads are untyped JSON)@@ -24,7 +23,7 @@ from typing import (import fastapifrom fastapi import APIRouter, Depends, HTTPException, Request, Response, statusfrom pydantic import TypeAdapter-from typing_extensions import ReadOnly, assert_never+from typing_extensions import ReadOnly, TypedDict, assert_neverimport litellmfrom litellm._logging import verbose_proxy_logger@@ -2721,7 +2720,7 @@ async def ui_view_spend_logs(end_date_obj = parse_date(end_date)# Build where conditions- where_conditions: Final[dict[str, Any]] = {}+ where_conditions: Final[dict[str, object]] = {}if start_date_obj is not None and end_date_obj is not None:where_conditions["startTime"] = {"gte": start_date_obj.isoformat(), # Already in UTC, no need to add Z@@ -2783,18 +2782,14 @@ async def ui_view_spend_logs(if len(metadata_filters) == 1:where_conditions["metadata"] = metadata_filters[0]else:- where_conditions["AND"] = where_conditions.get("AND", []) + [- {"metadata": filter_cond} for filter_cond in metadata_filters- ]+ where_conditions["AND"] = [{"metadata": filter_cond} for filter_cond in metadata_filters]if end_user is not None:where_conditions["end_user"] = end_userif min_spend is not None or max_spend is not None:- where_conditions["spend"] = {}- if min_spend is not None:- where_conditions["spend"]["gte"] = min_spend- if max_spend is not None:- where_conditions["spend"]["lte"] = max_spend+ where_conditions["spend"] = {+ key: value for key, value in (("gte", min_spend), ("lte", max_spend)) if value is not None+ }# A request_id lookup drops the date window, so a non-admin could otherwise# reach any single row by id; require they own one of the matches, mirroring# the detail endpoint, and keep the general scoping below so a collidingdiff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.pyindex b01f53c8c6b..2699dab2e29 100644--- a/litellm/proxy/utils.py+++ b/litellm/proxy/utils.py@@ -502,7 +502,7 @@ class InternalUsageCache:key: str,local_only: bool = False,**kwargs: object,- ) -> Any:+ ) -> object:return self.dual_cache.get_cache(key=key,local_only=local_only,diff --git a/litellm/types/proxy/auth/user_api_key_auth.py b/litellm/types/proxy/auth/user_api_key_auth.pyindex c2f7c487b7b..2c1c75fbe04 100644--- a/litellm/types/proxy/auth/user_api_key_auth.py+++ b/litellm/types/proxy/auth/user_api_key_auth.py@@ -198,14 +198,14 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob"narrow the agent's access."),)- budget_reservation: dict[str, Any] | None = Field(default=None, exclude=True)+ budget_reservation: dict[str, object] | None = Field(default=None, exclude=True)team_budget_snapshot: TeamBudgetSnapshot | None = Field(default=None, exclude=True)user_budget_snapshot: UserBudgetSnapshot | None = Field(default=None, exclude=True)org_budget_snapshot: OrgBudgetSnapshot | None = Field(default=None, exclude=True)matched_model_access_groups: list[str] | None = Field(default=None, exclude=True)budget_throttle_pct: float | None = Field(default=None, exclude=True)- user: Any | None = None # Expanded user object when expand=user is used- created_by_user: Any | None = None # Expanded created_by user when expand=user is used+ user: object | None = None # Expanded user object when expand=user is used+ created_by_user: object | None = None # Expanded created_by user when expand=user is usedend_user_object_permission: LiteLLM_ObjectPermissionTable | None = None# Team object_permission preloaded in auth (e.g. get_team_object) to avoid# per-request object_permission fetches in downstream checks (vector stores, etc.)diff --git a/litellm/types/proxy/management_endpoints/scim_v2.py b/litellm/types/proxy/management_endpoints/scim_v2.pyindex 9b519f9af4f..fadb309e0cd 100644--- a/litellm/types/proxy/management_endpoints/scim_v2.py+++ b/litellm/types/proxy/management_endpoints/scim_v2.py@@ -21,6 +21,9 @@ SCIM_ROLES_METADATA_KEY: Final = "scim_roles"SCIM_MANAGED_TEAM_METADATA_KEY: Final = "scim_managed"SCIM_TEAM_DATA_METADATA_KEY: Final = "scim_data"+_SCIM_SERIALIZED_FIELDS_ADAPTER: Final = TypeAdapter(+ dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)+)class LiteLLM_UserScimMetadata(LiteLLMBaseModel):@@ -123,7 +126,7 @@ class SCIMUser(SCIMResource):@model_serializer(mode="wrap")def _omit_absent_optional_blocks(self, handler: SerializerFunctionWrapHandler) -> dict[str, object]:- dumped: Final = handler(self)+ dumped: Final = _SCIM_SERIALIZED_FIELDS_ADAPTER.validate_python(handler(self))if self.enterprise_user is None:dumped.pop(SCIM_ENTERPRISE_USER_SCHEMA, None)dumped.pop("enterprise_user", None)diff --git a/tests/unit/llms/chatgpt/chat/test_streaming_utils.py b/tests/unit/llms/chatgpt/chat/test_streaming_utils.pyindex 8d95a3954a0..f6c5f768c5d 100644--- a/tests/unit/llms/chatgpt/chat/test_streaming_utils.py+++ b/tests/unit/llms/chatgpt/chat/test_streaming_utils.py@@ -7,9 +7,13 @@ are normalized to match the OpenAI streaming spec:- Deduplication of "closing" chunks with repeated id/name"""+from pathlib import Path+from typing import Final+import pytestfrom litellm.llms.chatgpt.chat.streaming_utils import ChatGPTToolCallNormalizer+from litellm.llms.chatgpt.chat.transformation import ChatGPTConfigfrom litellm.types.utils import (ChatCompletionDeltaToolCall,Delta,@@ -201,3 +205,17 @@ class TestChatGPTToolCallNormalizer:normalizer = ChatGPTToolCallNormalizer(FakeStream())assert normalizer.custom_attr == "test_value"+++def test_chatgpt_config_streams_parallel_tool_calls_with_distinct_indices(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+):+ monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path))+ chunks: Final = (+ _make_chunk(tool_calls=[_make_tc(index=0, id="call_aaa", name="get_weather")]),+ _make_chunk(tool_calls=[_make_tc(index=0, id="call_bbb", name="get_time")]),+ )++ results: Final = list(ChatGPTConfig().post_stream_processing(iter(chunks)))++ assert [result.choices[0].delta.tool_calls[0].index for result in results] == [0, 1]