Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

[Feat]: Add OpenTelemetry Auto Instrumentation for monitoring vLLM (#358)

patcher99deecd08495f
diff --git a/sdk/python/README.md b/sdk/python/README.md
index 4787e9af..3a16d3c5 100644
--- a/sdk/python/README.md
+++ b/sdk/python/README.md
@@ -41,6 +41,8 @@ This project adheres to the [Semantic Conventions](https://github.com/open-telem
| [✅ Vertex AI](https://docs.openlit.io/latest/integrations/vertexai) | | | |
| [✅ Groq](https://docs.openlit.io/latest/integrations/groq) | | | |
| [✅ ElevenLabs](https://docs.openlit.io/latest/integrations/elevenlabs) | | | |
+| [✅ vLLM](https://docs.openlit.io/latest/integrations/vllm) | | | |
+
## Supported Destinations
- [✅ OpenTelemetry Collector](https://docs.openlit.io/latest/connections/otelcol)
- [✅ Prometheus + Tempo](https://docs.openlit.io/latest/connections/prometheus-tempo)
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index f01a8b6e..e25038c4 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "openlit"
-version = "1.17.0"
+version = "1.18.0"
description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications, facilitating the integration of observability into your GenAI-driven projects"
authors = ["OpenLIT"]
repository = "https://github.com/openlit/openlit/tree/main/openlit/python"
diff --git a/sdk/python/src/openlit/__init__.py b/sdk/python/src/openlit/__init__.py
index 2c224f16..c51c04de 100644
--- a/sdk/python/src/openlit/__init__.py
+++ b/sdk/python/src/openlit/__init__.py
@@ -32,6 +32,7 @@ from openlit.instrumentation.groq import GroqInstrumentor
from openlit.instrumentation.ollama import OllamaInstrumentor
from openlit.instrumentation.gpt4all import GPT4AllInstrumentor
from openlit.instrumentation.elevenlabs import ElevenLabsInstrumentor
+from openlit.instrumentation.vllm import VLLMInstrumentor
from openlit.instrumentation.langchain import LangChainInstrumentor
from openlit.instrumentation.llamaindex import LlamaIndexInstrumentor
from openlit.instrumentation.haystack import HaystackInstrumentor
@@ -194,6 +195,7 @@ def init(environment="default", application_name="default", tracer=None, otlp_en
"ollama": "ollama",
"gpt4all": "gpt4all",
"elevenlabs": "elevenlabs",
+ "vllm": "vllm",
"langchain": "langchain",
"llama_index": "llama_index",
"haystack": "haystack",
@@ -270,6 +272,7 @@ def init(environment="default", application_name="default", tracer=None, otlp_en
"ollama": OllamaInstrumentor(),
"gpt4all": GPT4AllInstrumentor(),
"elevenlabs": ElevenLabsInstrumentor(),
+ "vllm": VLLMInstrumentor(),
"langchain": LangChainInstrumentor(),
"llama_index": LlamaIndexInstrumentor(),
"haystack": HaystackInstrumentor(),
diff --git a/sdk/python/src/openlit/instrumentation/vllm/__init__.py b/sdk/python/src/openlit/instrumentation/vllm/__init__.py
new file mode 100644
index 00000000..f13468b8
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/vllm/__init__.py
@@ -0,0 +1,43 @@
+# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code
+"""Initializer of Auto Instrumentation of vLLM Functions"""
+
+from typing import Collection
+import importlib.metadata
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from wrapt import wrap_function_wrapper
+
+from openlit.instrumentation.vllm.vllm import (
+ generate
+)
+
+_instruments = ("vllm >= 0.5.4",)
+
+class VLLMInstrumentor(BaseInstrumentor):
+ """
+ An instrumentor for vLLM's client library.
+ """
+
+ def instrumentation_dependencies(self) -> Collection[str]:
+ return _instruments
+
+ def _instrument(self, **kwargs):
+ application_name = kwargs.get("application_name", "default_application")
+ environment = kwargs.get("environment", "default_environment")
+ tracer = kwargs.get("tracer")
+ metrics = kwargs.get("metrics_dict")
+ pricing_info = kwargs.get("pricing_info", {})
+ trace_content = kwargs.get("trace_content", False)
+ disable_metrics = kwargs.get("disable_metrics")
+ version = importlib.metadata.version("vllm")
+
+ # sync chat
+ wrap_function_wrapper(
+ "vllm",
+ "LLM.generate",
+ generate("vllm.generate", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ def _uninstrument(self, **kwargs):
+ # Proper uninstrumentation logic to revert patched methods
+ pass
diff --git a/sdk/python/src/openlit/instrumentation/vllm/vllm.py b/sdk/python/src/openlit/instrumentation/vllm/vllm.py
new file mode 100644
index 00000000..835c32be
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/vllm/vllm.py
@@ -0,0 +1,143 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment
+"""
+Module for monitoring vLLM API calls.
+"""
+
+import logging
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import handle_exception, general_tokens
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def generate(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for generate to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the vLLM API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of vLLM usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the generate method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'generate' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'generate' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'generate' method.
+ kwargs: Keyword arguments for the 'generate' method.
+
+ Returns:
+ The response from the original 'generate' method.
+ """
+
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ model = instance.llm_engine.model_config.model or "facebook/opt-125m"
+ # Set base span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_VLLM)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ input_tokens = 0
+ output_tokens = 0
+ cost = 0
+
+ if trace_content:
+ prompt_attributes = {}
+ completion_attributes = {}
+
+ for i, output in enumerate(response):
+ prompt_attributes[f"{SemanticConvetion.GEN_AI_CONTENT_PROMPT}.{i}"] = output.prompt
+ completion_attributes[f"{SemanticConvetion.GEN_AI_CONTENT_COMPLETION}.{i}"] = output.outputs[0].text
+ input_tokens += general_tokens(output.prompt)
+ output_tokens += general_tokens(output.outputs[0].text)
+
+ # Add a single event for all prompts
+ span.add_event(
+ name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
+ attributes=prompt_attributes,
+ )
+
+ # Add a single event for all completions
+ span.add_event(
+ name=SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT,
+ attributes=completion_attributes,
+ )
+
+ total_tokens = input_tokens + output_tokens
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ input_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ output_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_VLLM,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(output_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.py
index 34682538..855cf2be 100644
--- a/sdk/python/src/openlit/semcov/__init__.py
+++ b/sdk/python/src/openlit/semcov/__init__.py
@@ -101,6 +101,7 @@ class SemanticConvetion:
GEN_AI_SYSTEM_OLLAMA = "ollama"
GEN_AI_SYSTEM_GPT4ALL = "gpt4all"
GEN_AI_SYSTEM_ELEVENLABS = "elevenlabs"
+ GEN_AI_SYSTEM_VLLM = "vLLM"
GEN_AI_SYSTEM_LANGCHAIN = "langchain"
GEN_AI_SYSTEM_LLAMAINDEX = "llama_index"
GEN_AI_SYSTEM_HAYSTACK = "haystack"