Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

fix: OpenLIT-LiteLLM Proxy instrumentation failure with Azure AI Foundry (#1050)

Ishan Jain111128fe87fe
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index c6217af4..5f73556c 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "openlit"
-version = "1.37.5"
+version = "1.37.6"
description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"
authors = ["OpenLIT"]
license = "Apache-2.0"
diff --git a/sdk/python/src/openlit/__helpers.py b/sdk/python/src/openlit/__helpers.py
index 2f19f553..13cd3fc1 100644
--- a/sdk/python/src/openlit/__helpers.py
+++ b/sdk/python/src/openlit/__helpers.py
@@ -3,6 +3,7 @@
This module has functions to calculate model costs based on tokens and to fetch pricing information.
"""
+import asyncio
import os
import json
import logging
@@ -46,6 +47,9 @@ def response_as_dict(response):
"""
# pylint: disable=no-else-return
+ if asyncio.iscoroutine(response):
+ logger.warning("response_as_dict received an unawaited coroutine")
+ return {}
if isinstance(response, dict):
return response
if hasattr(response, "model_dump"):
diff --git a/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py b/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py
index 2ef8ac65..5f606a6e 100644
--- a/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py
+++ b/sdk/python/src/openlit/instrumentation/ai21/async_ai21.py
@@ -79,9 +79,9 @@ def async_chat(
def __aiter__(self):
return self
- async def __getattr__(self, name):
+ def __getattr__(self, name):
"""Delegate attribute access to the wrapped object."""
- return getattr(await self.__wrapped__, name)
+ return getattr(self.__wrapped__, name)
async def __anext__(self):
try:
diff --git a/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py b/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py
index 302a6329..bc396793 100644
--- a/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py
+++ b/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py
@@ -83,9 +83,9 @@ def async_messages(
def __aiter__(self):
return self
- async def __getattr__(self, name):
+ def __getattr__(self, name):
"""Delegate attribute access to the wrapped object."""
- return getattr(await self.__wrapped__, name)
+ return getattr(self.__wrapped__, name)
async def __anext__(self):
try:
diff --git a/sdk/python/src/openlit/instrumentation/cohere/async_cohere.py b/sdk/python/src/openlit/instrumentation/cohere/async_cohere.py
index 53ce83dc..0fe469c5 100644
--- a/sdk/python/src/openlit/instrumentation/cohere/async_cohere.py
+++ b/sdk/python/src/openlit/instrumentation/cohere/async_cohere.py
@@ -162,9 +162,9 @@ def async_chat_stream(
def __aiter__(self):
return self
- async def __getattr__(self, name):
+ def __getattr__(self, name):
"""Delegate attribute access to the wrapped object."""
- return getattr(await self.__wrapped__, name)
+ return getattr(self.__wrapped__, name)
async def __anext__(self):
try:
diff --git a/sdk/python/src/openlit/instrumentation/google_ai_studio/async_google_ai_studio.py b/sdk/python/src/openlit/instrumentation/google_ai_studio/async_google_ai_studio.py
index faa84068..93d0c5a8 100644
--- a/sdk/python/src/openlit/instrumentation/google_ai_studio/async_google_ai_studio.py
+++ b/sdk/python/src/openlit/instrumentation/google_ai_studio/async_google_ai_studio.py
@@ -164,9 +164,9 @@ def async_generate_stream(
def __aiter__(self):
return self
- async def __getattr__(self, name):
+ def __getattr__(self, name):
"""Delegate attribute access to the wrapped object."""
- return getattr(await self.__wrapped__, name)
+ return getattr(self.__wrapped__, name)
async def __anext__(self):
try:
diff --git a/sdk/python/src/openlit/instrumentation/groq/async_groq.py b/sdk/python/src/openlit/instrumentation/groq/async_groq.py
index d963f521..16d5221a 100644
--- a/sdk/python/src/openlit/instrumentation/groq/async_groq.py
+++ b/sdk/python/src/openlit/instrumentation/groq/async_groq.py
@@ -80,9 +80,9 @@ def async_chat(
def __aiter__(self):
return self
- async def __getattr__(self, name):
+ def __getattr__(self, name):
"""Delegate attribute access to the wrapped object."""
- return getattr(await self.__wrapped__, name)
+ return getattr(self.__wrapped__, name)
async def __anext__(self):
try:
diff --git a/sdk/python/src/openlit/instrumentation/litellm/async_litellm.py b/sdk/python/src/openlit/instrumentation/litellm/async_litellm.py
index 7265e6b9..3e2e6237 100644
--- a/sdk/python/src/openlit/instrumentation/litellm/async_litellm.py
+++ b/sdk/python/src/openlit/instrumentation/litellm/async_litellm.py
@@ -6,11 +6,11 @@ import time
from opentelemetry.trace import SpanKind
from openlit.__helpers import (
handle_exception,
- set_server_address_and_port,
record_completion_metrics,
record_embedding_metrics,
)
from openlit.instrumentation.litellm.utils import (
+ get_litellm_server_address,
process_chunk,
process_streaming_chat_response,
process_chat_response,
@@ -84,9 +84,9 @@ def acompletion(
def __aiter__(self):
return self
- async def __getattr__(self, name):
+ def __getattr__(self, name):
"""Delegate attribute access to the wrapped object."""
- return getattr(await self.__wrapped__, name)
+ return getattr(self.__wrapped__, name)
async def __anext__(self):
try:
@@ -122,9 +122,7 @@ def acompletion(
"""
# Check if streaming is enabled for the API call
streaming = kwargs.get("stream", False)
- server_address, server_port = set_server_address_and_port(
- instance, "NOT_FOUND", "NOT_FOUND"
- )
+ server_address, server_port = get_litellm_server_address(instance, kwargs)
request_model = kwargs.get("model", "openai/gpt-4o")
span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"
@@ -214,9 +212,7 @@ def aembedding(
"""
Wraps the GenAI embedding function call.
"""
- server_address, server_port = set_server_address_and_port(
- instance, "NOT_FOUND", "NOT_FOUND"
- )
+ server_address, server_port = get_litellm_server_address(instance, kwargs)
request_model = kwargs.get("model", "text-embedding-ada-002")
span_name = (
diff --git a/sdk/python/src/openlit/instrumentation/litellm/litellm.py b/sdk/python/src/openlit/instrumentation/litellm/litellm.py
index 2513fc0d..e1372484 100644
--- a/sdk/python/src/openlit/instrumentation/litellm/litellm.py
+++ b/sdk/python/src/openlit/instrumentation/litellm/litellm.py
@@ -6,11 +6,11 @@ import time
from opentelemetry.trace import SpanKind
from openlit.__helpers import (
handle_exception,
- set_server_address_and_port,
record_completion_metrics,
record_embedding_metrics,
)
from openlit.instrumentation.litellm.utils import (
+ get_litellm_server_address,
process_chunk,
process_streaming_chat_response,
process_chat_response,
@@ -122,9 +122,7 @@ def completion(
"""
# Check if streaming is enabled for the API call
streaming = kwargs.get("stream", False)
- server_address, server_port = set_server_address_and_port(
- instance, "NOT_FOUND", "NOT_FOUND"
- )
+ server_address, server_port = get_litellm_server_address(instance, kwargs)
request_model = kwargs.get("model", "openai/gpt-4o")
span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT} {request_model}"
@@ -214,9 +212,7 @@ def embedding(
"""
Wraps the GenAI embedding function call.
"""
- server_address, server_port = set_server_address_and_port(
- instance, "NOT_FOUND", "NOT_FOUND"
- )
+ server_address, server_port = get_litellm_server_address(instance, kwargs)
request_model = kwargs.get("model", "text-embedding-ada-002")
span_name = (
diff --git a/sdk/python/src/openlit/instrumentation/litellm/utils.py b/sdk/python/src/openlit/instrumentation/litellm/utils.py
index b6db8b19..78b14bc6 100644
--- a/sdk/python/src/openlit/instrumentation/litellm/utils.py
+++ b/sdk/python/src/openlit/instrumentation/litellm/utils.py
@@ -5,6 +5,7 @@ LiteLLM OpenTelemetry instrumentation utility functions
import json
import logging
import time
+from urllib.parse import urlparse
from opentelemetry.trace import Status, StatusCode
@@ -23,6 +24,57 @@ from openlit.semcov import SemanticConvention
logger = logging.getLogger(__name__)
+LITELLM_PROVIDER_HOSTS = {
+ "openai": "api.openai.com",
+ "azure": "azure.openai.com",
+ "anthropic": "api.anthropic.com",
+ "cohere": "api.cohere.com",
+ "ollama": "localhost",
+ "groq": "api.groq.com",
+ "mistral": "api.mistral.ai",
+ "together_ai": "api.together.xyz",
+ "vertex_ai": "aiplatform.googleapis.com",
+ "gemini": "generativelanguage.googleapis.com",
+ "bedrock": "bedrock-runtime.amazonaws.com",
+ "huggingface": "api-inference.huggingface.co",
+ "deepseek": "api.deepseek.com",
+ "fireworks_ai": "api.fireworks.ai",
+ "replicate": "api.replicate.com",
+ "ai21": "api.ai21.com",
+ "perplexity": "api.perplexity.ai",
+ "cerebras": "api.cerebras.ai",
+ "xai": "api.x.ai",
+ "sambanova": "api.sambanova.ai",
+}
+
+
+def get_litellm_server_address(instance, kwargs):
+ """
+ Resolve server address and port for LiteLLM calls.
+
+ LiteLLM uses module-level functions (not client instances), so the generic
+ set_server_address_and_port helper cannot extract a base_url. Instead we
+ check kwargs, module-level settings, and infer from the model prefix.
+ """
+ api_base = kwargs.get("api_base") or kwargs.get("base_url")
+ if not api_base:
+ api_base = getattr(instance, "api_base", None)
+
+ if api_base and isinstance(api_base, str):
+ if api_base.startswith(("http://", "https://")):
+ url = urlparse(api_base)
+ return (url.hostname or "unknown", url.port or 443)
+ return (api_base, 443)
+
+ model = kwargs.get("model", "")
+ provider = model.split("/", 1)[0] if "/" in model else ""
+ host = LITELLM_PROVIDER_HOSTS.get(provider)
+ if host:
+ port = 11434 if provider == "ollama" else 443
+ return (host, port)
+
+ return ("unknown", 443)
+
def format_content(messages):
"""
@@ -321,9 +373,20 @@ def process_chunk(scope, chunk):
"arguments", ""
)
+ scope._response_id = chunked.get("id") or scope._response_id
+ scope._response_model = chunked.get("model") or scope._response_model
+ try:
+ scope._finish_reason = (
+ chunked.get("choices", [])[0].get("finish_reason") or scope._finish_reason
+ )
+ except (IndexError, AttributeError, TypeError):
+ pass
+ scope._response_service_tier = (
+ str(chunked.get("system_fingerprint", "")) or scope._response_service_tier
+ )
+
if chunked.get("usage"):
usage = chunked.get("usage", {})
- # Handle token usage including reasoning tokens and cached tokens
scope._input_tokens = usage.get("prompt_tokens", 0)
scope._output_tokens = usage.get("completion_tokens", 0)
prompt_tokens_details = usage.get(
@@ -333,13 +396,6 @@ def process_chunk(scope, chunk):
scope._cache_creation_input_tokens = usage.get(
"completion_tokens_details", {}
).get("cached_tokens", 0)
- scope._response_id = chunked.get("id")
- scope._response_model = chunked.get("model")
- finish_reason = chunked.get("choices", [{}])[0].get("finish_reason")
- # Only update finish_reason if it's not None (preserve previous valid value)
- if finish_reason is not None:
- scope._finish_reason = finish_reason
- scope._response_service_tier = str(chunked.get("system_fingerprint", ""))
scope._end_time = time.time()
diff --git a/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py b/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py
index 84bb21c0..eb2a6a78 100644
--- a/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py
+++ b/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py
@@ -159,9 +159,9 @@ def async_stream(
def __aiter__(self):
return self
- async def __getattr__(self, name):
+ def __getattr__(self, name):
"""Delegate attribute access to the wrapped object."""
- return getattr(await self.__wrapped__, name)
+ return getattr(self.__wrapped__, name)
async def __anext__(self):
try:
diff --git a/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py
index f919b49b..e7bb846d 100644
--- a/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py
+++ b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py
@@ -85,9 +85,9 @@ def async_chat(
def __aiter__(self):
return self
- async def __getattr__(self, name):
+ def __getattr__(self, name):
"""Delegate attribute access to the wrapped object."""
- return getattr(await self.__wrapped__, name)
+ return getattr(self.__wrapped__, name)
async def __anext__(self):
try:
@@ -260,9 +260,9 @@ def async_generate(
def __aiter__(self):
return self
- async def __getattr__(self, name):
+ def __getattr__(self, name):
"""Delegate attribute access to the wrapped object."""
- return getattr(await self.__wrapped__, name)
+ return getattr(self.__wrapped__, name)
async def __anext__(self):
try:
diff --git a/sdk/python/src/openlit/instrumentation/together/async_together.py b/sdk/python/src/openlit/instrumentation/together/async_together.py
index 029ea28e..f5a62026 100644
--- a/sdk/python/src/openlit/instrumentation/together/async_together.py
+++ b/sdk/python/src/openlit/instrumentation/together/async_together.py
@@ -77,9 +77,9 @@ def async_completion(
def __aiter__(self):
return self
- async def __getattr__(self, name):
+ def __getattr__(self, name):
"""Delegate attribute access to the wrapped object."""
- return getattr(await self.__wrapped__, name)
+ return getattr(self.__wrapped__, name)
async def __anext__(self):
try:
diff --git a/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py b/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py
index ccb2a7c4..f3bfad16 100644
--- a/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py
+++ b/sdk/python/src/openlit/instrumentation/vertexai/async_vertexai.py
@@ -81,9 +81,9 @@ def async_send_message(
def __aiter__(self):
return self
- async def __getattr__(self, name):
+ def __getattr__(self, name):
"""Delegate attribute access to the wrapped object."""
- return getattr(await self.__wrapped__, name)
+ return getattr(self.__wrapped__, name)
async def __anext__(self):
try: