Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
fix: OpenLIT-LiteLLM Proxy instrumentation failure with Azure AI Foundry (#1050)
Ishan Jain111128fe87fe
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex c6217af4..5f73556c 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.37.5"+version = "1.37.6"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]license = "Apache-2.0"diff --git a/sdk/python/src/openlit/__helpers.py b/sdk/python/src/openlit/__helpers.pyindex 2f19f553..13cd3fc1 100644--- a/sdk/python/src/openlit/__helpers.py+++ b/sdk/python/src/openlit/__helpers.py@@ -3,6 +3,7 @@This module has functions to calculate model costs based on tokens and to fetch pricing information."""+import asyncioimport osimport jsonimport logging@@ -46,6 +47,9 @@ def response_as_dict(response):"""# pylint: disable=no-else-return+ if asyncio.iscoroutine(response):+ logger.warning("response_as_dict received an unawaited coroutine")+ return {}if isinstance(response, dict):return responseif hasattr(response, "model_dump"):diff --git a/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py b/sdk/python/src/openlit/instrumentation/ai21/async_ai21.pyindex 2ef8ac65..5f606a6e 100644--- a/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py+++ b/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py@@ -79,9 +79,9 @@ def async_chat(def __aiter__(self):return self- async def __getattr__(self, name):+ def __getattr__(self, name):"""Delegate attribute access to the wrapped object."""- return getattr(await self.__wrapped__, name)+ return getattr(self.__wrapped__, name)async def __anext__(self):try:diff --git a/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py b/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.pyindex 302a6329..bc396793 100644--- a/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py+++ b/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py@@ -83,9 +83,9 @@ def async_messages(def __aiter__(self):return self- async def __getattr__(self, name):+ def __getattr__(self, name):"""Delegate attribute access to the wrapped object."""- return getattr(await self.__wrapped__, name)+ return getattr(self.__wrapped__, name)async def __anext__(self):try:diff --git a/sdk/python/src/openlit/instrumentation/cohere/async_cohere.py b/sdk/python/src/openlit/instrumentation/cohere/async_cohere.pyindex 53ce83dc..0fe469c5 100644--- a/sdk/python/src/openlit/instrumentation/cohere/async_cohere.py+++ b/sdk/python/src/openlit/instrumentation/cohere/async_cohere.py@@ -162,9 +162,9 @@ def async_chat_stream(def __aiter__(self):return self- async def __getattr__(self, name):+ def __getattr__(self, name):"""Delegate attribute access to the wrapped object."""- return getattr(await self.__wrapped__, name)+ return getattr(self.__wrapped__, name)async def __anext__(self):try:diff --git a/sdk/python/src/openlit/instrumentation/google_ai_studio/async_google_ai_studio.py b/sdk/python/src/openlit/instrumentation/google_ai_studio/async_google_ai_studio.pyindex faa84068..93d0c5a8 100644--- a/sdk/python/src/openlit/instrumentation/google_ai_studio/async_google_ai_studio.py+++ b/sdk/python/src/openlit/instrumentation/google_ai_studio/async_google_ai_studio.py@@ -164,9 +164,9 @@ def async_generate_stream(def __aiter__(self):return self- async def __getattr__(self, name):+ def __getattr__(self, name):"""Delegate attribute access to the wrapped object."""- return getattr(await self.__wrapped__, name)+ return getattr(self.__wrapped__, name)async def __anext__(self):try:diff --git a/sdk/python/src/openlit/instrumentation/groq/async_groq.py b/sdk/python/src/openlit/instrumentation/groq/async_groq.pyindex d963f521..16d5221a 100644--- a/sdk/python/src/openlit/instrumentation/groq/async_groq.py+++ b/sdk/python/src/openlit/instrumentation/groq/async_groq.py@@ -80,9 +80,9 @@ def async_chat(def __aiter__(self):return self- async def __getattr__(self, name):+ def __getattr__(self, name):"""Delegate attribute access to the wrapped object."""- return getattr(await self.__wrapped__, name)+ return getattr(self.__wrapped__, name)async def __anext__(self):try:diff --git a/sdk/python/src/openlit/instrumentation/litellm/async_litellm.py b/sdk/python/src/openlit/instrumentation/litellm/async_litellm.pyindex 7265e6b9..3e2e6237 100644--- a/sdk/python/src/openlit/instrumentation/litellm/async_litellm.py+++ b/sdk/python/src/openlit/instrumentation/litellm/async_litellm.py@@ -6,11 +6,11 @@ import timefrom opentelemetry.trace import SpanKindfrom openlit.__helpers import (handle_exception,- set_server_address_and_port,record_completion_metrics,record_embedding_metrics,)from openlit.instrumentation.litellm.utils import (+ get_litellm_server_address,process_chunk,process_streaming_chat_response,process_chat_response,@@ -84,9 +84,9 @@ def acompletion(def __aiter__(self):return self- async def __getattr__(self, name):+ def __getattr__(self, name):"""Delegate attribute access to the wrapped object."""- return getattr(await self.__wrapped__, name)+ return getattr(self.__wrapped__, name)async def __anext__(self):try:@@ -122,9 +122,7 @@ def acompletion("""# Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)- server_address, server_port = set_server_address_and_port(- instance, "NOT_FOUND", "NOT_FOUND"- )+ server_address, server_port = get_litellm_server_address(instance, kwargs)request_model = kwargs.get("model", "openai/gpt-4o")span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"@@ -214,9 +212,7 @@ def aembedding("""Wraps the GenAI embedding function call."""- server_address, server_port = set_server_address_and_port(- instance, "NOT_FOUND", "NOT_FOUND"- )+ server_address, server_port = get_litellm_server_address(instance, kwargs)request_model = kwargs.get("model", "text-embedding-ada-002")span_name = (diff --git a/sdk/python/src/openlit/instrumentation/litellm/litellm.py b/sdk/python/src/openlit/instrumentation/litellm/litellm.pyindex 2513fc0d..e1372484 100644--- a/sdk/python/src/openlit/instrumentation/litellm/litellm.py+++ b/sdk/python/src/openlit/instrumentation/litellm/litellm.py@@ -6,11 +6,11 @@ import timefrom opentelemetry.trace import SpanKindfrom openlit.__helpers import (handle_exception,- set_server_address_and_port,record_completion_metrics,record_embedding_metrics,)from openlit.instrumentation.litellm.utils import (+ get_litellm_server_address,process_chunk,process_streaming_chat_response,process_chat_response,@@ -122,9 +122,7 @@ def completion("""# Check if streaming is enabled for the API callstreaming = kwargs.get("stream", False)- server_address, server_port = set_server_address_and_port(- instance, "NOT_FOUND", "NOT_FOUND"- )+ server_address, server_port = get_litellm_server_address(instance, kwargs)request_model = kwargs.get("model", "openai/gpt-4o")span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"@@ -214,9 +212,7 @@ def embedding("""Wraps the GenAI embedding function call."""- server_address, server_port = set_server_address_and_port(- instance, "NOT_FOUND", "NOT_FOUND"- )+ server_address, server_port = get_litellm_server_address(instance, kwargs)request_model = kwargs.get("model", "text-embedding-ada-002")span_name = (diff --git a/sdk/python/src/openlit/instrumentation/litellm/utils.py b/sdk/python/src/openlit/instrumentation/litellm/utils.pyindex b6db8b19..78b14bc6 100644--- a/sdk/python/src/openlit/instrumentation/litellm/utils.py+++ b/sdk/python/src/openlit/instrumentation/litellm/utils.py@@ -5,6 +5,7 @@ LiteLLM OpenTelemetry instrumentation utility functionsimport jsonimport loggingimport time+from urllib.parse import urlparsefrom opentelemetry.trace import Status, StatusCode@@ -23,6 +24,57 @@ from openlit.semcov import SemanticConventionlogger = logging.getLogger(__name__)+LITELLM_PROVIDER_HOSTS = {+ "openai": "api.openai.com",+ "azure": "azure.openai.com",+ "anthropic": "api.anthropic.com",+ "cohere": "api.cohere.com",+ "ollama": "localhost",+ "groq": "api.groq.com",+ "mistral": "api.mistral.ai",+ "together_ai": "api.together.xyz",+ "vertex_ai": "aiplatform.googleapis.com",+ "gemini": "generativelanguage.googleapis.com",+ "bedrock": "bedrock-runtime.amazonaws.com",+ "huggingface": "api-inference.huggingface.co",+ "deepseek": "api.deepseek.com",+ "fireworks_ai": "api.fireworks.ai",+ "replicate": "api.replicate.com",+ "ai21": "api.ai21.com",+ "perplexity": "api.perplexity.ai",+ "cerebras": "api.cerebras.ai",+ "xai": "api.x.ai",+ "sambanova": "api.sambanova.ai",+}+++def get_litellm_server_address(instance, kwargs):+ """+ Resolve server address and port for LiteLLM calls.++ LiteLLM uses module-level functions (not client instances), so the generic+ set_server_address_and_port helper cannot extract a base_url. Instead we+ check kwargs, module-level settings, and infer from the model prefix.+ """+ api_base = kwargs.get("api_base") or kwargs.get("base_url")+ if not api_base:+ api_base = getattr(instance, "api_base", None)++ if api_base and isinstance(api_base, str):+ if api_base.startswith(("http://", "https://")):+ url = urlparse(api_base)+ return (url.hostname or "unknown", url.port or 443)+ return (api_base, 443)++ model = kwargs.get("model", "")+ provider = model.split("/", 1)[0] if "/" in model else ""+ host = LITELLM_PROVIDER_HOSTS.get(provider)+ if host:+ port = 11434 if provider == "ollama" else 443+ return (host, port)++ return ("unknown", 443)+def format_content(messages):"""@@ -321,9 +373,20 @@ def process_chunk(scope, chunk):"arguments", "")+ scope._response_id = chunked.get("id") or scope._response_id+ scope._response_model = chunked.get("model") or scope._response_model+ try:+ scope._finish_reason = (+ chunked.get("choices", [])[0].get("finish_reason") or scope._finish_reason+ )+ except (IndexError, AttributeError, TypeError):+ pass+ scope._response_service_tier = (+ str(chunked.get("system_fingerprint", "")) or scope._response_service_tier+ )+if chunked.get("usage"):usage = chunked.get("usage", {})- # Handle token usage including reasoning tokens and cached tokensscope._input_tokens = usage.get("prompt_tokens", 0)scope._output_tokens = usage.get("completion_tokens", 0)prompt_tokens_details = usage.get(@@ -333,13 +396,6 @@ def process_chunk(scope, chunk):scope._cache_creation_input_tokens = usage.get("completion_tokens_details", {}).get("cached_tokens", 0)- scope._response_id = chunked.get("id")- scope._response_model = chunked.get("model")- finish_reason = chunked.get("choices", [{}])[0].get("finish_reason")- # Only update finish_reason if it's not None (preserve previous valid value)- if finish_reason is not None:- scope._finish_reason = finish_reason- scope._response_service_tier = str(chunked.get("system_fingerprint", ""))scope._end_time = time.time()diff --git a/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py b/sdk/python/src/openlit/instrumentation/mistral/async_mistral.pyindex 84bb21c0..eb2a6a78 100644--- a/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py+++ b/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py@@ -159,9 +159,9 @@ def async_stream(def __aiter__(self):return self- async def __getattr__(self, name):+ def __getattr__(self, name):"""Delegate attribute access to the wrapped object."""- return getattr(await self.__wrapped__, name)+ return getattr(self.__wrapped__, name)async def __anext__(self):try:diff --git a/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.pyindex f919b49b..e7bb846d 100644--- a/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py+++ b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py@@ -85,9 +85,9 @@ def async_chat(def __aiter__(self):return self- async def __getattr__(self, name):+ def __getattr__(self, name):"""Delegate attribute access to the wrapped object."""- return getattr(await self.__wrapped__, name)+ return getattr(self.__wrapped__, name)async def __anext__(self):try:@@ -260,9 +260,9 @@ def async_generate(def __aiter__(self):return self- async def __getattr__(self, name):+ def __getattr__(self, name):"""Delegate attribute access to the wrapped object."""- return getattr(await self.__wrapped__, name)+ return getattr(self.__wrapped__, name)async def __anext__(self):try:diff --git a/sdk/python/src/openlit/instrumentation/together/async_together.py b/sdk/python/src/openlit/instrumentation/together/async_together.pyindex 029ea28e..f5a62026 100644--- a/sdk/python/src/openlit/instrumentation/together/async_together.py+++ b/sdk/python/src/openlit/instrumentation/together/async_together.py@@ -77,9 +77,9 @@ def async_completion(def __aiter__(self):return self- async def __getattr__(self, name):+ def __getattr__(self, name):"""Delegate attribute access to the wrapped object."""- return getattr(await self.__wrapped__, name)+ return getattr(self.__wrapped__, name)async def __anext__(self):try:diff --git a/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py b/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.pyindex ccb2a7c4..f3bfad16 100644--- a/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py+++ b/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py@@ -81,9 +81,9 @@ def async_send_message(def __aiter__(self):return self- async def __getattr__(self, name):+ def __getattr__(self, name):"""Delegate attribute access to the wrapped object."""- return getattr(await self.__wrapped__, name)+ return getattr(self.__wrapped__, name)async def __anext__(self):try: