Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

[Feat]: Update NVIDIA GPU Instrumentation and add AMD GPU Instrumentation (#455)

Rudi MK952f0c32c754
diff --git a/otel-gpu-collector/README.md b/otel-gpu-collector/README.md
index a02b55fb..1a33b3cc 100644
--- a/otel-gpu-collector/README.md
+++ b/otel-gpu-collector/README.md
@@ -23,7 +23,7 @@ OpenTelemetry GPU Collector is a lightweight, efficient COLLECTOR designed to co
- Collects detailed GPU performance metrics
- OpenTelemetry-native
- Lightweight and efficient
-- Supports NVIDIA GPUs
+- Supports NVIDIA and AMD GPUs
## πŸš€ Getting Started with GPU Monitoring
@@ -141,9 +141,8 @@ We are dedicated to continuously improving OpenTelemetry GPU Collector. Here's a
| Feature | Status |
|----------------------------------------------------------------------------------------------|---------------|
-| [OpenTelmetry metrics for NVIDIA GPUs using `gpustat`]() | βœ… Completed |
-| [OpenTelmetry metrics for AMD GPUs]() | πŸ”œ Coming Soon |
-| [OpenTelmetry metrics for NVIDIA GPUs using `nvidia-smi`]() | πŸ”œ Coming Soon |
+| [OpenTelmetry-native AMD GPU Monitoring]() | βœ… Completed |
+| [OpenTelmetry-native NVIDIA GPU Monitoring]() | βœ… Completed |
diff --git a/sdk/python/README.md b/sdk/python/README.md
index d9689c00..5114a6a9 100644
--- a/sdk/python/README.md
+++ b/sdk/python/README.md
@@ -34,8 +34,8 @@ This project proudly follows and maintains the [Semantic Conventions](https://gi
| LLMs | Vector DBs | Frameworks | GPUs |
|--------------------------------------------------------------------------|----------------------------------------------|----------------------------------------------|---------------|
-| [βœ… OpenAI](https://docs.openlit.io/latest/integrations/openai) | [βœ… ChromaDB](https://docs.openlit.io/latest/integrations/chromadb) | [βœ… Langchain](https://docs.openlit.io/latest/integrations/langchain) | [βœ… NVIDIA GPUs](https://docs.openlit.io/latest/integrations/nvidia-gpu) |
-| [βœ… Ollama](https://docs.openlit.io/latest/integrations/ollama) | [βœ… Pinecone](https://docs.openlit.io/latest/integrations/pinecone) | [βœ… LiteLLM](https://docs.openlit.io/latest/integrations/litellm) | |
+| [βœ… OpenAI](https://docs.openlit.io/latest/integrations/openai) | [βœ… ChromaDB](https://docs.openlit.io/latest/integrations/chromadb) | [βœ… Langchain](https://docs.openlit.io/latest/integrations/langchain) | [βœ… NVIDIA](https://docs.openlit.io/latest/integrations/nvidia-gpu) |
+| [βœ… Ollama](https://docs.openlit.io/latest/integrations/ollama) | [βœ… Pinecone](https://docs.openlit.io/latest/integrations/pinecone) | [βœ… LiteLLM](https://docs.openlit.io/latest/integrations/litellm) | [βœ… AMD](#) |
| [βœ… Anthropic](https://docs.openlit.io/latest/integrations/anthropic) | [βœ… Qdrant](https://docs.openlit.io/latest/integrations/qdrant) | [βœ… LlamaIndex](https://docs.openlit.io/latest/integrations/llama-index) | |
| [βœ… GPT4All](https://docs.openlit.io/latest/integrations/gpt4all) | [βœ… Milvus](https://docs.openlit.io/latest/integrations/milvus) | [βœ… Haystack](https://docs.openlit.io/latest/integrations/haystack) | |
| [βœ… Cohere](https://docs.openlit.io/latest/integrations/cohere) | | [βœ… EmbedChain](https://docs.openlit.io/latest/integrations/embedchain) | |
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index 73c76f17..0684df96 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "openlit"
-version = "1.27.1"
+version = "1.28.0"
description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"
authors = ["OpenLIT"]
repository = "https://github.com/openlit/openlit/tree/main/openlit/python"
@@ -14,7 +14,7 @@ requests = "^2.26.0"
schedule = "^1.2.2"
tiktoken = "^0.7.0"
pydantic = "^2.0.0"
-gpustat = "^1.1.1"
+xmltodict = "^0.13.0"
boto3 = "^1.34.0"
botocore = "^1.34.0"
opentelemetry-api = "^1.27.0"
diff --git a/sdk/python/src/openlit/__init__.py b/sdk/python/src/openlit/__init__.py
index c49c6421..8356933d 100644
--- a/sdk/python/src/openlit/__init__.py
+++ b/sdk/python/src/openlit/__init__.py
@@ -46,7 +46,7 @@ from openlit.instrumentation.pinecone import PineconeInstrumentor
from openlit.instrumentation.qdrant import QdrantInstrumentor
from openlit.instrumentation.milvus import MilvusInstrumentor
from openlit.instrumentation.transformers import TransformersInstrumentor
-from openlit.instrumentation.gpu import NvidiaGPUInstrumentor
+from openlit.instrumentation.gpu import GPUInstrumentor
import openlit.guard
import openlit.evals
@@ -313,7 +313,7 @@ def init(environment="default", application_name="default", tracer=None, otlp_en
disabled_instrumentors, module_name_map)
if not disable_metrics and collect_gpu_stats:
- NvidiaGPUInstrumentor().instrument(
+ GPUInstrumentor().instrument(
environment=config.environment,
application_name=config.application_name,
)
diff --git a/sdk/python/src/openlit/instrumentation/gpu/__init__.py b/sdk/python/src/openlit/instrumentation/gpu/__init__.py
index fc7698ae..aa133474 100644
--- a/sdk/python/src/openlit/instrumentation/gpu/__init__.py
+++ b/sdk/python/src/openlit/instrumentation/gpu/__init__.py
@@ -1,132 +1,208 @@
-# pylint: disable=useless-return, bad-staticmethod-argument, duplicate-code, import-outside-toplevel, broad-exception-caught, unused-argument
+# pylint: disable=useless-return, bad-staticmethod-argument, duplicate-code, import-outside-toplevel, broad-exception-caught, unused-argument, import-error, too-many-return-statements, superfluous-parens
"""Initializer of Auto Instrumentation of GPU Metrics"""
from typing import Collection, Iterable
import logging
from functools import partial
-
from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
from opentelemetry.metrics import get_meter, CallbackOptions, Observation
-
from openlit.semcov import SemanticConvetion
# Initialize logger for logging potential issues and operations
logger = logging.getLogger(__name__)
-class NvidiaGPUInstrumentor(BaseInstrumentor):
+class GPUInstrumentor(BaseInstrumentor):
"""
- An instrumentor for collecting NVIDIA GPU metrics.
+ An instrumentor for collecting GPU metrics.
"""
def instrumentation_dependencies(self) -> Collection[str]:
return []
def _instrument(self, **kwargs):
-
application_name = kwargs.get("application_name", "default")
environment = kwargs.get("environment", "default")
-
+ # pylint: disable=attribute-defined-outside-init
+ self.gpu_type = self._get_gpu_type()
meter = get_meter(
__name__,
"0.1.0",
schema_url="https://opentelemetry.io/schemas/1.11.0",
)
- def check_and_record(value):
- return value if value is not None else 0
-
- meter.create_observable_gauge(
- name=SemanticConvetion.GPU_UTILIZATION,
- callbacks=[partial(self._collect_metric, environment,
- application_name, check_and_record, "utilization")],
- description="GPU Utilization",
- )
- meter.create_observable_gauge(
- name=SemanticConvetion.GPU_UTILIZATION_ENC,
- callbacks=[partial(self._collect_metric, environment,
- application_name, check_and_record, "utilization_enc")],
- description="GPU Encoder Utilization",
- )
- meter.create_observable_gauge(
- name=SemanticConvetion.GPU_UTILIZATION_DEC,
- callbacks=[partial(self._collect_metric, environment,
- application_name, check_and_record, "utilization_dec")],
- description="GPU Decoder Utilization",
- )
- meter.create_observable_gauge(
- name=SemanticConvetion.GPU_TEMPERATURE,
- callbacks=[partial(self._collect_metric, environment,
- application_name, check_and_record, "temperature")],
- description="GPU Temperature",
- )
- meter.create_observable_gauge(
- name=SemanticConvetion.GPU_FAN_SPEED,
- callbacks=[partial(self._collect_metric, environment,
- application_name, check_and_record, "fan_speed")],
- description="GPU Fan Speed",
- )
- meter.create_observable_gauge(
- name=SemanticConvetion.GPU_MEMORY_AVAILABLE,
- callbacks=[partial(self._collect_metric, environment,
- application_name, check_and_record, "memory_available")],
- description="GPU Memory Available",
- )
- meter.create_observable_gauge(
- name=SemanticConvetion.GPU_MEMORY_TOTAL,
- callbacks=[partial(self._collect_metric, environment,
- application_name, check_and_record, "memory_total")],
- description="GPU Memory Total",
- )
- meter.create_observable_gauge(
- name=SemanticConvetion.GPU_MEMORY_USED,
- callbacks=[partial(self._collect_metric, environment,
- application_name, check_and_record, "memory_used")],
- description="GPU Memory Used",
- )
- meter.create_observable_gauge(
- name=SemanticConvetion.GPU_MEMORY_FREE,
- callbacks=[partial(self._collect_metric, environment,
- application_name, check_and_record, "memory_free")],
- description="GPU Memory Free",
- )
- meter.create_observable_gauge(
- name=SemanticConvetion.GPU_POWER_DRAW,
- callbacks=[partial(self._collect_metric, environment,
- application_name, check_and_record, "power_draw")],
- description="GPU Power Draw",
- )
- meter.create_observable_gauge(
- name=SemanticConvetion.GPU_POWER_LIMIT,
- callbacks=[partial(self._collect_metric, environment,
- application_name, check_and_record, "power_limit")],
- description="GPU Power Limit",
- )
+ if not self.gpu_type:
+ logger.error(
+ "OpenLIT GPU Instrumentation Error: No supported GPUs found."
+ "If this is a non-GPU host, set `collect_gpu_stats=False` to disable GPU stats."
+ )
+ return
+
+ metric_names = [
+ ("GPU_UTILIZATION", "utilization"),
+ ("GPU_UTILIZATION_ENC", "utilization_enc"),
+ ("GPU_UTILIZATION_DEC", "utilization_dec"),
+ ("GPU_TEMPERATURE", "temperature"),
+ ("GPU_FAN_SPEED", "fan_speed"),
+ ("GPU_MEMORY_AVAILABLE", "memory_available"),
+ ("GPU_MEMORY_TOTAL", "memory_total"),
+ ("GPU_MEMORY_USED", "memory_used"),
+ ("GPU_MEMORY_FREE", "memory_free"),
+ ("GPU_POWER_DRAW", "power_draw"),
+ ("GPU_POWER_LIMIT", "power_limit"),
+ ]
+
+ for semantic_name, internal_name in metric_names:
+ meter.create_observable_gauge(
+ name=getattr(SemanticConvetion, semantic_name),
+ callbacks=[partial(self._collect_metric,
+ environment, application_name, internal_name)],
+ description=f"GPU {internal_name.replace('_', ' ').title()}",
+ )
def _uninstrument(self, **kwargs):
# Proper uninstrumentation logic to revert patched methods
pass
+ def _get_gpu_type(self) -> str:
+ try:
+ import pynvml
+ pynvml.nvmlInit()
+ return "nvidia"
+ except Exception:
+ try:
+ import amdsmi
+ amdsmi.amdsmi_init()
+ return "amd"
+ except Exception:
+ return None
+
+
def _collect_metric(self, environment, application_name,
- check_and_record, metric_name,
+ metric_name,
options: CallbackOptions) -> Iterable[Observation]:
+ # pylint: disable=no-else-return
+ if self.gpu_type == "nvidia":
+ return self._collect_nvidia_metrics(environment, application_name, metric_name, options)
+ elif self.gpu_type == "amd":
+ return self._collect_amd_metrics(environment, application_name, metric_name, options)
+ return []
- import gpustat
-
+ def _collect_nvidia_metrics(self, environment, application_name,
+ metric_name,
+ options: CallbackOptions) -> Iterable[Observation]:
try:
- gpu_stats = gpustat.GPUStatCollection.new_query()
+ import pynvml
+ gpu_count = pynvml.nvmlDeviceGetCount()
+ mega_bytes = 1024 * 1024
+ gpu_index = 0
+ for gpu_index in range(gpu_count):
+ handle = pynvml.nvmlDeviceGetHandleByIndex(gpu_index)
+
+ def get_metric_value(handle, metric_name):
+ try:
+ # pylint: disable=no-else-return
+ if metric_name == "temperature":
+ return pynvml.nvmlDeviceGetTemperature(handle,
+ pynvml.NVML_TEMPERATURE_GPU)
+ elif metric_name == "utilization":
+ return pynvml.nvmlDeviceGetUtilizationRates(handle).gpu
+ elif metric_name == "utilization_enc":
+ return pynvml.nvmlDeviceGetEncoderUtilization(handle)[0]
+ elif metric_name == "utilization_dec":
+ return pynvml.nvmlDeviceGetDecoderUtilization(handle)[0]
+ elif metric_name == "fan_speed":
+ return 0
+ elif metric_name == "memory_available":
+ memory_info = pynvml.nvmlDeviceGetMemoryInfo(handle)
+ return (memory_info.free // mega_bytes) # Assuming reserved memory is 0
+ elif metric_name == "memory_total":
+ return (pynvml.nvmlDeviceGetMemoryInfo(handle).total // mega_bytes)
+ elif metric_name == "memory_used":
+ return (pynvml.nvmlDeviceGetMemoryInfo(handle).used // mega_bytes)
+ elif metric_name == "memory_free":
+ return (pynvml.nvmlDeviceGetMemoryInfo(handle).free // mega_bytes)
+ elif metric_name == "power_draw":
+ return (pynvml.nvmlDeviceGetPowerUsage(handle) // 1000.0)
+ elif metric_name == "power_limit":
+ return (pynvml.nvmlDeviceGetEnforcedPowerLimit(handle) // 1000.0)
+ except Exception as e:
+ # pylint: disable=cell-var-from-loop
+ logger.error("Error collecting metric %s for GPU %d: %s", metric_name,
+ gpu_index, e)
+ return 0
- for gpu in gpu_stats.gpus:
attributes = {
TELEMETRY_SDK_NAME: "openlit",
SemanticConvetion.GEN_AI_APPLICATION_NAME: application_name,
SemanticConvetion.GEN_AI_ENVIRONMENT: environment,
- SemanticConvetion.GPU_INDEX: gpu.index,
- SemanticConvetion.GPU_UUID: gpu.uuid,
- SemanticConvetion.GPU_NAME: gpu.name,
+ SemanticConvetion.GPU_INDEX: str(gpu_index),
+ SemanticConvetion.GPU_UUID: pynvml.nvmlDeviceGetUUID(handle).decode('utf-8'),
+ SemanticConvetion.GPU_NAME: pynvml.nvmlDeviceGetName(handle).decode('utf-8')
}
-
- yield Observation(check_and_record(getattr(gpu, metric_name, 0)), attributes)
+ yield Observation(get_metric_value(handle, metric_name), attributes)
except Exception as e:
logger.error("Error in GPU metrics collection: %s", e)
+
+ def _collect_amd_metrics(self, environment, application_name,
+ metric_name,
+ options: CallbackOptions) -> Iterable[Observation]:
+ try:
+ import amdsmi
+ # Get the number of AMD GPUs
+ devices = amdsmi.amdsmi_get_processor_handles()
+ mega_bytes = 1024 * 1024
+ for device_handle in devices:
+
+ def get_metric_value(device_handle, metric_name):
+ try:
+ # pylint: disable=no-else-return
+ if metric_name == "temperature":
+ # pylint: disable=line-too-long
+ return amdsmi.amdsmi_get_temp_metric(device_handle,
+ amdsmi.AmdSmiTemperatureType.EDGE,
+ amdsmi.AmdSmiTemperatureMetric.CURRENT)
+ elif metric_name == "utilization":
+ # pylint: disable=line-too-long
+ return amdsmi.amdsmi_get_utilization_count(device_handle,
+ amdsmi.AmdSmiUtilizationCounterType.COARSE_GRAIN_GFX_ACTIVITY)
+ elif metric_name in ["utilization_enc", "utilization_dec"]:
+ return 0 # Placeholder if unsupported
+ elif metric_name == "fan_speed":
+ return amdsmi.amdsmi_get_gpu_fan_speed(device_handle, 0)
+ elif metric_name == "memory_available":
+ return (amdsmi.amdsmi_get_gpu_memory_total(device_handle) // mega_bytes)
+ elif metric_name == "memory_total":
+ return (amdsmi.amdsmi_get_gpu_memory_total(device_handle) // mega_bytes)
+ elif metric_name == "memory_used":
+ return (amdsmi.amdsmi_get_gpu_memory_usage(device_handle) // mega_bytes)
+ elif metric_name == "memory_free":
+ total_mem = (amdsmi.amdsmi_get_gpu_memory_total(device_handle) // mega_bytes)
+ used_mem = (amdsmi.amdsmi_get_gpu_memory_usage(device_handle) // mega_bytes)
+ return (total_mem - used_mem)
+ elif metric_name == "power_draw":
+ # pylint: disable=line-too-long
+ return (amdsmi.amdsmi_get_power_info(device_handle)['average_socket_power'] // 1000.0)
+ elif metric_name == "power_limit":
+ # pylint: disable=line-too-long
+ return (amdsmi.amdsmi_get_power_info(device_handle)['power_limit'] // 1000.0)
+ except Exception as e:
+ logger.error("Error collecting metric %s for AMD GPU %d: %s", metric_name,
+ amdsmi.amdsmi_get_xgmi_info(device_handle)['index'], e)
+ return 0
+
+ attributes = {
+ TELEMETRY_SDK_NAME: "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME: application_name,
+ SemanticConvetion.GEN_AI_ENVIRONMENT: environment,
+ # pylint: disable=line-too-long
+ SemanticConvetion.GPU_INDEX: amdsmi.amdsmi_get_xgmi_info(device_handle)['index'],
+ # pylint: disable=line-too-long
+ SemanticConvetion.GPU_UUID: amdsmi.amdsmi_get_gpu_asic_info(device_handle)['market_name'],
+ SemanticConvetion.GPU_NAME: amdsmi.amdsmi_get_device_name(device_handle)
+ }
+ yield Observation(get_metric_value(device_handle, metric_name), attributes)
+
+ except Exception as e:
+ logger.error("Error in AMD GPU metrics collection: %s", e)
diff --git a/sdk/python/tests/requirements.txt b/sdk/python/tests/requirements.txt
index 9de965e8..2fd68417 100644
--- a/sdk/python/tests/requirements.txt
+++ b/sdk/python/tests/requirements.txt
@@ -31,7 +31,7 @@ langchainhub>=0.1.20
embedchain>=0.1.104
# GPU
-gpustat>=1.1.1
+xmltodict>=0.13.0
# Utilities
bs4
diff --git a/sdk/python/tests/test_chroma.py b/sdk/python/tests/test_chroma.py
index 861c92f2..774f8a4c 100644
--- a/sdk/python/tests/test_chroma.py
+++ b/sdk/python/tests/test_chroma.py
@@ -1,4 +1,4 @@
-# pylint: disable=duplicate-code, no-name-in-module
+# pylint: disable=duplicate-code, no-name-in-module, assignment-from-no-return
"""
This module contains tests for ChromaDB functionality using the ChromaDB Python library.