Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Feat]: Add OpenTelemetry Auto Instrumentation for monitoring vLLM (#358)
patcher99deecd08495f
diff --git a/sdk/python/README.md b/sdk/python/README.mdindex 4787e9af..3a16d3c5 100644--- a/sdk/python/README.md+++ b/sdk/python/README.md@@ -41,6 +41,8 @@ This project adheres to the [Semantic Conventions](https://github.com/open-telem| [✅ Vertex AI](https://docs.openlit.io/latest/integrations/vertexai) | | | || [✅ Groq](https://docs.openlit.io/latest/integrations/groq) | | | || [✅ ElevenLabs](https://docs.openlit.io/latest/integrations/elevenlabs) | | | |+| [✅ vLLM](https://docs.openlit.io/latest/integrations/vllm) | | | |+## Supported Destinations- [✅ OpenTelemetry Collector](https://docs.openlit.io/latest/connections/otelcol)- [✅ Prometheus + Tempo](https://docs.openlit.io/latest/connections/prometheus-tempo)diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex f01a8b6e..e25038c4 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.17.0"+version = "1.18.0"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]repository = "https://github.com/openlit/openlit/tree/main/openlit/python"diff --git a/sdk/python/src/openlit/__init__.py b/sdk/python/src/openlit/__init__.pyindex 2c224f16..c51c04de 100644--- a/sdk/python/src/openlit/__init__.py+++ b/sdk/python/src/openlit/__init__.py@@ -32,6 +32,7 @@ from openlit.instrumentation.groq import GroqInstrumentorfrom openlit.instrumentation.ollama import OllamaInstrumentorfrom openlit.instrumentation.gpt4all import GPT4AllInstrumentorfrom openlit.instrumentation.elevenlabs import ElevenLabsInstrumentor+from openlit.instrumentation.vllm import VLLMInstrumentorfrom openlit.instrumentation.langchain import LangChainInstrumentorfrom openlit.instrumentation.llamaindex import LlamaIndexInstrumentorfrom openlit.instrumentation.haystack import HaystackInstrumentor@@ -194,6 +195,7 @@ def init(environment="default", application_name="default", tracer=None, otlp_en"ollama": "ollama","gpt4all": "gpt4all","elevenlabs": "elevenlabs",+ "vllm": "vllm","langchain": "langchain","llama_index": "llama_index","haystack": "haystack",@@ -270,6 +272,7 @@ def init(environment="default", application_name="default", tracer=None, otlp_en"ollama": OllamaInstrumentor(),"gpt4all": GPT4AllInstrumentor(),"elevenlabs": ElevenLabsInstrumentor(),+ "vllm": VLLMInstrumentor(),"langchain": LangChainInstrumentor(),"llama_index": LlamaIndexInstrumentor(),"haystack": HaystackInstrumentor(),diff --git a/sdk/python/src/openlit/instrumentation/vllm/__init__.py b/sdk/python/src/openlit/instrumentation/vllm/__init__.pynew file mode 100644index 00000000..f13468b8--- /dev/null+++ b/sdk/python/src/openlit/instrumentation/vllm/__init__.py@@ -0,0 +1,43 @@+# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code+"""Initializer of Auto Instrumentation of vLLM Functions"""++from typing import Collection+import importlib.metadata+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor+from wrapt import wrap_function_wrapper++from openlit.instrumentation.vllm.vllm import (+ generate+)++_instruments = ("vllm >= 0.5.4",)++class VLLMInstrumentor(BaseInstrumentor):+ """+ An instrumentor for vLLM's client library.+ """++ def instrumentation_dependencies(self) -> Collection[str]:+ return _instruments++ def _instrument(self, **kwargs):+ application_name = kwargs.get("application_name", "default_application")+ environment = kwargs.get("environment", "default_environment")+ tracer = kwargs.get("tracer")+ metrics = kwargs.get("metrics_dict")+ pricing_info = kwargs.get("pricing_info", {})+ trace_content = kwargs.get("trace_content", False)+ disable_metrics = kwargs.get("disable_metrics")+ version = importlib.metadata.version("vllm")++ # sync chat+ wrap_function_wrapper(+ "vllm",+ "LLM.generate",+ generate("vllm.generate", version, environment, application_name,+ tracer, pricing_info, trace_content, metrics, disable_metrics),+ )++ def _uninstrument(self, **kwargs):+ # Proper uninstrumentation logic to revert patched methods+ passdiff --git a/sdk/python/src/openlit/instrumentation/vllm/vllm.py b/sdk/python/src/openlit/instrumentation/vllm/vllm.pynew file mode 100644index 00000000..835c32be--- /dev/null+++ b/sdk/python/src/openlit/instrumentation/vllm/vllm.py@@ -0,0 +1,143 @@+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment+"""+Module for monitoring vLLM API calls.+"""++import logging+from opentelemetry.trace import SpanKind, Status, StatusCode+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME+from openlit.__helpers import handle_exception, general_tokens+from openlit.semcov import SemanticConvetion++# Initialize logger for logging potential issues and operations+logger = logging.getLogger(__name__)++def generate(gen_ai_endpoint, version, environment, application_name,+ tracer, pricing_info, trace_content, metrics, disable_metrics):+ """+ Generates a telemetry wrapper for generate to collect metrics.++ Args:+ gen_ai_endpoint: Endpoint identifier for logging and tracing.+ version: Version of the monitoring package.+ environment: Deployment environment (e.g., production, staging).+ application_name: Name of the application using the vLLM API.+ tracer: OpenTelemetry tracer for creating spans.+ pricing_info: Information used for calculating the cost of vLLM usage.+ trace_content: Flag indicating whether to trace the actual content.++ Returns:+ A function that wraps the generate method to add telemetry.+ """++ def wrapper(wrapped, instance, args, kwargs):+ """+ Wraps the 'generate' API call to add telemetry.++ This collects metrics such as execution time, cost, and token usage, and handles errors+ gracefully, adding details to the trace for observability.++ Args:+ wrapped: The original 'generate' method to be wrapped.+ instance: The instance of the class where the original method is defined.+ args: Positional arguments for the 'generate' method.+ kwargs: Keyword arguments for the 'generate' method.++ Returns:+ The response from the original 'generate' method.+ """++ # pylint: disable=line-too-long+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ response = wrapped(*args, **kwargs)++ try:+ model = instance.llm_engine.model_config.model or "facebook/opt-125m"+ # Set base span attribues+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_VLLM)+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,+ SemanticConvetion.GEN_AI_TYPE_CHAT)+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,+ gen_ai_endpoint)+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,+ environment)+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ model)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ False)+ input_tokens = 0+ output_tokens = 0+ cost = 0++ if trace_content:+ prompt_attributes = {}+ completion_attributes = {}++ for i, output in enumerate(response):+ prompt_attributes[f"{SemanticConvetion.GEN_AI_CONTENT_PROMPT}.{i}"] = output.prompt+ completion_attributes[f"{SemanticConvetion.GEN_AI_CONTENT_COMPLETION}.{i}"] = output.outputs[0].text+ input_tokens += general_tokens(output.prompt)+ output_tokens += general_tokens(output.outputs[0].text)++ # Add a single event for all prompts+ span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,+ attributes=prompt_attributes,+ )++ # Add a single event for all completions+ span.add_event(+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,+ attributes=completion_attributes,+ )++ total_tokens = input_tokens + output_tokens++ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ total_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)++ span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = {+ TELEMETRY_SDK_NAME:+ "openlit",+ SemanticConvetion.GEN_AI_APPLICATION_NAME:+ application_name,+ SemanticConvetion.GEN_AI_SYSTEM:+ SemanticConvetion.GEN_AI_SYSTEM_VLLM,+ SemanticConvetion.GEN_AI_ENVIRONMENT:+ environment,+ SemanticConvetion.GEN_AI_TYPE:+ SemanticConvetion.GEN_AI_TYPE_CHAT,+ SemanticConvetion.GEN_AI_REQUEST_MODEL:+ model+ }++ metrics["genai_requests"].add(1, attributes)+ metrics["genai_total_tokens"].add(total_tokens, attributes)+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)++ # Return original response+ return response++ except Exception as e:+ handle_exception(span, e)+ logger.error("Error in trace creation: %s", e)++ # Return original response+ return response++ return wrapperdiff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.pyindex 34682538..855cf2be 100644--- a/sdk/python/src/openlit/semcov/__init__.py+++ b/sdk/python/src/openlit/semcov/__init__.py@@ -101,6 +101,7 @@ class SemanticConvetion:GEN_AI_SYSTEM_OLLAMA = "ollama"GEN_AI_SYSTEM_GPT4ALL = "gpt4all"GEN_AI_SYSTEM_ELEVENLABS = "elevenlabs"+ GEN_AI_SYSTEM_VLLM = "vLLM"GEN_AI_SYSTEM_LANGCHAIN = "langchain"GEN_AI_SYSTEM_LLAMAINDEX = "llama_index"GEN_AI_SYSTEM_HAYSTACK = "haystack"