Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Feat]: Update NVIDIA GPU Instrumentation and add AMD GPU Instrumentation (#455)
Rudi MK952f0c32c754
diff --git a/otel-gpu-collector/README.md b/otel-gpu-collector/README.mdindex a02b55fb..1a33b3cc 100644--- a/otel-gpu-collector/README.md+++ b/otel-gpu-collector/README.md@@ -23,7 +23,7 @@ OpenTelemetry GPU Collector is a lightweight, efficient COLLECTOR designed to co- Collects detailed GPU performance metrics- OpenTelemetry-native- Lightweight and efficient-- Supports NVIDIA GPUs+- Supports NVIDIA and AMD GPUs## π Getting Started with GPU Monitoring@@ -141,9 +141,8 @@ We are dedicated to continuously improving OpenTelemetry GPU Collector. Here's a| Feature | Status ||----------------------------------------------------------------------------------------------|---------------|-| [OpenTelmetry metrics for NVIDIA GPUs using `gpustat`]() | β Completed |-| [OpenTelmetry metrics for AMD GPUs]() | π Coming Soon |-| [OpenTelmetry metrics for NVIDIA GPUs using `nvidia-smi`]() | π Coming Soon |+| [OpenTelmetry-native AMD GPU Monitoring]() | β Completed |+| [OpenTelmetry-native NVIDIA GPU Monitoring]() | β Completed |diff --git a/sdk/python/README.md b/sdk/python/README.mdindex d9689c00..5114a6a9 100644--- a/sdk/python/README.md+++ b/sdk/python/README.md@@ -34,8 +34,8 @@ This project proudly follows and maintains the [Semantic Conventions](https://gi| LLMs | Vector DBs | Frameworks | GPUs ||--------------------------------------------------------------------------|----------------------------------------------|----------------------------------------------|---------------|-| [β OpenAI](https://docs.openlit.io/latest/integrations/openai) | [β ChromaDB](https://docs.openlit.io/latest/integrations/chromadb) | [β Langchain](https://docs.openlit.io/latest/integrations/langchain) | [β NVIDIA GPUs](https://docs.openlit.io/latest/integrations/nvidia-gpu) |-| [β Ollama](https://docs.openlit.io/latest/integrations/ollama) | [β Pinecone](https://docs.openlit.io/latest/integrations/pinecone) | [β LiteLLM](https://docs.openlit.io/latest/integrations/litellm) | |+| [β OpenAI](https://docs.openlit.io/latest/integrations/openai) | [β ChromaDB](https://docs.openlit.io/latest/integrations/chromadb) | [β Langchain](https://docs.openlit.io/latest/integrations/langchain) | [β NVIDIA](https://docs.openlit.io/latest/integrations/nvidia-gpu) |+| [β Ollama](https://docs.openlit.io/latest/integrations/ollama) | [β Pinecone](https://docs.openlit.io/latest/integrations/pinecone) | [β LiteLLM](https://docs.openlit.io/latest/integrations/litellm) | [β AMD](#) || [β Anthropic](https://docs.openlit.io/latest/integrations/anthropic) | [β Qdrant](https://docs.openlit.io/latest/integrations/qdrant) | [β LlamaIndex](https://docs.openlit.io/latest/integrations/llama-index) | || [β GPT4All](https://docs.openlit.io/latest/integrations/gpt4all) | [β Milvus](https://docs.openlit.io/latest/integrations/milvus) | [β Haystack](https://docs.openlit.io/latest/integrations/haystack) | || [β Cohere](https://docs.openlit.io/latest/integrations/cohere) | | [β EmbedChain](https://docs.openlit.io/latest/integrations/embedchain) | |diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex 73c76f17..0684df96 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.27.1"+version = "1.28.0"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]repository = "https://github.com/openlit/openlit/tree/main/openlit/python"@@ -14,7 +14,7 @@ requests = "^2.26.0"schedule = "^1.2.2"tiktoken = "^0.7.0"pydantic = "^2.0.0"-gpustat = "^1.1.1"+xmltodict = "^0.13.0"boto3 = "^1.34.0"botocore = "^1.34.0"opentelemetry-api = "^1.27.0"diff --git a/sdk/python/src/openlit/__init__.py b/sdk/python/src/openlit/__init__.pyindex c49c6421..8356933d 100644--- a/sdk/python/src/openlit/__init__.py+++ b/sdk/python/src/openlit/__init__.py@@ -46,7 +46,7 @@ from openlit.instrumentation.pinecone import PineconeInstrumentorfrom openlit.instrumentation.qdrant import QdrantInstrumentorfrom openlit.instrumentation.milvus import MilvusInstrumentorfrom openlit.instrumentation.transformers import TransformersInstrumentor-from openlit.instrumentation.gpu import NvidiaGPUInstrumentor+from openlit.instrumentation.gpu import GPUInstrumentorimport openlit.guardimport openlit.evals@@ -313,7 +313,7 @@ def init(environment="default", application_name="default", tracer=None, otlp_endisabled_instrumentors, module_name_map)if not disable_metrics and collect_gpu_stats:- NvidiaGPUInstrumentor().instrument(+ GPUInstrumentor().instrument(environment=config.environment,application_name=config.application_name,)diff --git a/sdk/python/src/openlit/instrumentation/gpu/__init__.py b/sdk/python/src/openlit/instrumentation/gpu/__init__.pyindex fc7698ae..aa133474 100644--- a/sdk/python/src/openlit/instrumentation/gpu/__init__.py+++ b/sdk/python/src/openlit/instrumentation/gpu/__init__.py@@ -1,132 +1,208 @@-# pylint: disable=useless-return, bad-staticmethod-argument, duplicate-code, import-outside-toplevel, broad-exception-caught, unused-argument+# pylint: disable=useless-return, bad-staticmethod-argument, duplicate-code, import-outside-toplevel, broad-exception-caught, unused-argument, import-error, too-many-return-statements, superfluous-parens"""Initializer of Auto Instrumentation of GPU Metrics"""from typing import Collection, Iterableimport loggingfrom functools import partial-from opentelemetry.instrumentation.instrumentor import BaseInstrumentorfrom opentelemetry.sdk.resources import TELEMETRY_SDK_NAMEfrom opentelemetry.metrics import get_meter, CallbackOptions, Observation-from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-class NvidiaGPUInstrumentor(BaseInstrumentor):+class GPUInstrumentor(BaseInstrumentor):"""- An instrumentor for collecting NVIDIA GPU metrics.+ An instrumentor for collecting GPU metrics."""def instrumentation_dependencies(self) -> Collection[str]:return []def _instrument(self, **kwargs):-application_name = kwargs.get("application_name", "default")environment = kwargs.get("environment", "default")-+ # pylint: disable=attribute-defined-outside-init+ self.gpu_type = self._get_gpu_type()meter = get_meter(__name__,"0.1.0",schema_url="https://opentelemetry.io/schemas/1.11.0",)- def check_and_record(value):- return value if value is not None else 0-- meter.create_observable_gauge(- name=SemanticConvetion.GPU_UTILIZATION,- callbacks=[partial(self._collect_metric, environment,- application_name, check_and_record, "utilization")],- description="GPU Utilization",- )- meter.create_observable_gauge(- name=SemanticConvetion.GPU_UTILIZATION_ENC,- callbacks=[partial(self._collect_metric, environment,- application_name, check_and_record, "utilization_enc")],- description="GPU Encoder Utilization",- )- meter.create_observable_gauge(- name=SemanticConvetion.GPU_UTILIZATION_DEC,- callbacks=[partial(self._collect_metric, environment,- application_name, check_and_record, "utilization_dec")],- description="GPU Decoder Utilization",- )- meter.create_observable_gauge(- name=SemanticConvetion.GPU_TEMPERATURE,- callbacks=[partial(self._collect_metric, environment,- application_name, check_and_record, "temperature")],- description="GPU Temperature",- )- meter.create_observable_gauge(- name=SemanticConvetion.GPU_FAN_SPEED,- callbacks=[partial(self._collect_metric, environment,- application_name, check_and_record, "fan_speed")],- description="GPU Fan Speed",- )- meter.create_observable_gauge(- name=SemanticConvetion.GPU_MEMORY_AVAILABLE,- callbacks=[partial(self._collect_metric, environment,- application_name, check_and_record, "memory_available")],- description="GPU Memory Available",- )- meter.create_observable_gauge(- name=SemanticConvetion.GPU_MEMORY_TOTAL,- callbacks=[partial(self._collect_metric, environment,- application_name, check_and_record, "memory_total")],- description="GPU Memory Total",- )- meter.create_observable_gauge(- name=SemanticConvetion.GPU_MEMORY_USED,- callbacks=[partial(self._collect_metric, environment,- application_name, check_and_record, "memory_used")],- description="GPU Memory Used",- )- meter.create_observable_gauge(- name=SemanticConvetion.GPU_MEMORY_FREE,- callbacks=[partial(self._collect_metric, environment,- application_name, check_and_record, "memory_free")],- description="GPU Memory Free",- )- meter.create_observable_gauge(- name=SemanticConvetion.GPU_POWER_DRAW,- callbacks=[partial(self._collect_metric, environment,- application_name, check_and_record, "power_draw")],- description="GPU Power Draw",- )- meter.create_observable_gauge(- name=SemanticConvetion.GPU_POWER_LIMIT,- callbacks=[partial(self._collect_metric, environment,- application_name, check_and_record, "power_limit")],- description="GPU Power Limit",- )+ if not self.gpu_type:+ logger.error(+ "OpenLIT GPU Instrumentation Error: No supported GPUs found."+ "If this is a non-GPU host, set `collect_gpu_stats=False` to disable GPU stats."+ )+ return++ metric_names = [+ ("GPU_UTILIZATION", "utilization"),+ ("GPU_UTILIZATION_ENC", "utilization_enc"),+ ("GPU_UTILIZATION_DEC", "utilization_dec"),+ ("GPU_TEMPERATURE", "temperature"),+ ("GPU_FAN_SPEED", "fan_speed"),+ ("GPU_MEMORY_AVAILABLE", "memory_available"),+ ("GPU_MEMORY_TOTAL", "memory_total"),+ ("GPU_MEMORY_USED", "memory_used"),+ ("GPU_MEMORY_FREE", "memory_free"),+ ("GPU_POWER_DRAW", "power_draw"),+ ("GPU_POWER_LIMIT", "power_limit"),+ ]++ for semantic_name, internal_name in metric_names:+ meter.create_observable_gauge(+ name=getattr(SemanticConvetion, semantic_name),+ callbacks=[partial(self._collect_metric,+ environment, application_name, internal_name)],+ description=f"GPU {internal_name.replace('_', ' ').title()}",+ )def _uninstrument(self, **kwargs):# Proper uninstrumentation logic to revert patched methodspass+ def _get_gpu_type(self) -> str:+ try:+ import pynvml+ pynvml.nvmlInit()+ return "nvidia"+ except Exception:+ try:+ import amdsmi+ amdsmi.amdsmi_init()+ return "amd"+ except Exception:+ return None++def _collect_metric(self, environment, application_name,- check_and_record, metric_name,+ metric_name,options: CallbackOptions) -> Iterable[Observation]:+ # pylint: disable=no-else-return+ if self.gpu_type == "nvidia":+ return self._collect_nvidia_metrics(environment, application_name, metric_name, options)+ elif self.gpu_type == "amd":+ return self._collect_amd_metrics(environment, application_name, metric_name, options)+ return []- import gpustat-+ def _collect_nvidia_metrics(self, environment, application_name,+ metric_name,+ options: CallbackOptions) -> Iterable[Observation]:try:- gpu_stats = gpustat.GPUStatCollection.new_query()+ import pynvml+ gpu_count = pynvml.nvmlDeviceGetCount()+ mega_bytes = 1024 * 1024+ gpu_index = 0+ for gpu_index in range(gpu_count):+ handle = pynvml.nvmlDeviceGetHandleByIndex(gpu_index)++ def get_metric_value(handle, metric_name):+ try:+ # pylint: disable=no-else-return+ if metric_name == "temperature":+ return pynvml.nvmlDeviceGetTemperature(handle,+ pynvml.NVML_TEMPERATURE_GPU)+ elif metric_name == "utilization":+ return pynvml.nvmlDeviceGetUtilizationRates(handle).gpu+ elif metric_name == "utilization_enc":+ return pynvml.nvmlDeviceGetEncoderUtilization(handle)[0]+ elif metric_name == "utilization_dec":+ return pynvml.nvmlDeviceGetDecoderUtilization(handle)[0]+ elif metric_name == "fan_speed":+ return 0+ elif metric_name == "memory_available":+ memory_info = pynvml.nvmlDeviceGetMemoryInfo(handle)+ return (memory_info.free // mega_bytes) # Assuming reserved memory is 0+ elif metric_name == "memory_total":+ return (pynvml.nvmlDeviceGetMemoryInfo(handle).total // mega_bytes)+ elif metric_name == "memory_used":+ return (pynvml.nvmlDeviceGetMemoryInfo(handle).used // mega_bytes)+ elif metric_name == "memory_free":+ return (pynvml.nvmlDeviceGetMemoryInfo(handle).free // mega_bytes)+ elif metric_name == "power_draw":+ return (pynvml.nvmlDeviceGetPowerUsage(handle) // 1000.0)+ elif metric_name == "power_limit":+ return (pynvml.nvmlDeviceGetEnforcedPowerLimit(handle) // 1000.0)+ except Exception as e:+ # pylint: disable=cell-var-from-loop+ logger.error("Error collecting metric %s for GPU %d: %s", metric_name,+ gpu_index, e)+ return 0- for gpu in gpu_stats.gpus:attributes = {TELEMETRY_SDK_NAME: "openlit",SemanticConvetion.GEN_AI_APPLICATION_NAME: application_name,SemanticConvetion.GEN_AI_ENVIRONMENT: environment,- SemanticConvetion.GPU_INDEX: gpu.index,- SemanticConvetion.GPU_UUID: gpu.uuid,- SemanticConvetion.GPU_NAME: gpu.name,+ SemanticConvetion.GPU_INDEX: str(gpu_index),+ SemanticConvetion.GPU_UUID: pynvml.nvmlDeviceGetUUID(handle).decode('utf-8'),+ SemanticConvetion.GPU_NAME: pynvml.nvmlDeviceGetName(handle).decode('utf-8')}-- yield Observation(check_and_record(getattr(gpu, metric_name, 0)), attributes)+ yield Observation(get_metric_value(handle, metric_name), attributes)except Exception as e:logger.error("Error in GPU metrics collection: %s", e)++ def _collect_amd_metrics(self, environment, application_name,+ metric_name,+ options: CallbackOptions) -> Iterable[Observation]:+ try:+ import amdsmi+ # Get the number of AMD GPUs+ devices = amdsmi.amdsmi_get_processor_handles()+ mega_bytes = 1024 * 1024+ for device_handle in devices:++ def get_metric_value(device_handle, metric_name):+ try:+ # pylint: disable=no-else-return+ if metric_name == "temperature":+ # pylint: disable=line-too-long+ return amdsmi.amdsmi_get_temp_metric(device_handle,+ amdsmi.AmdSmiTemperatureType.EDGE,+ amdsmi.AmdSmiTemperatureMetric.CURRENT)+ elif metric_name == "utilization":+ # pylint: disable=line-too-long+ return amdsmi.amdsmi_get_utilization_count(device_handle,+ amdsmi.AmdSmiUtilizationCounterType.COARSE_GRAIN_GFX_ACTIVITY)+ elif metric_name in ["utilization_enc", "utilization_dec"]:+ return 0 # Placeholder if unsupported+ elif metric_name == "fan_speed":+ return amdsmi.amdsmi_get_gpu_fan_speed(device_handle, 0)+ elif metric_name == "memory_available":+ return (amdsmi.amdsmi_get_gpu_memory_total(device_handle) // mega_bytes)+ elif metric_name == "memory_total":+ return (amdsmi.amdsmi_get_gpu_memory_total(device_handle) // mega_bytes)+ elif metric_name == "memory_used":+ return (amdsmi.amdsmi_get_gpu_memory_usage(device_handle) // mega_bytes)+ elif metric_name == "memory_free":+ total_mem = (amdsmi.amdsmi_get_gpu_memory_total(device_handle) // mega_bytes)+ used_mem = (amdsmi.amdsmi_get_gpu_memory_usage(device_handle) // mega_bytes)+ return (total_mem - used_mem)+ elif metric_name == "power_draw":+ # pylint: disable=line-too-long+ return (amdsmi.amdsmi_get_power_info(device_handle)['average_socket_power'] // 1000.0)+ elif metric_name == "power_limit":+ # pylint: disable=line-too-long+ return (amdsmi.amdsmi_get_power_info(device_handle)['power_limit'] // 1000.0)+ except Exception as e:+ logger.error("Error collecting metric %s for AMD GPU %d: %s", metric_name,+ amdsmi.amdsmi_get_xgmi_info(device_handle)['index'], e)+ return 0++ attributes = {+ TELEMETRY_SDK_NAME: "openlit",+ SemanticConvetion.GEN_AI_APPLICATION_NAME: application_name,+ SemanticConvetion.GEN_AI_ENVIRONMENT: environment,+ # pylint: disable=line-too-long+ SemanticConvetion.GPU_INDEX: amdsmi.amdsmi_get_xgmi_info(device_handle)['index'],+ # pylint: disable=line-too-long+ SemanticConvetion.GPU_UUID: amdsmi.amdsmi_get_gpu_asic_info(device_handle)['market_name'],+ SemanticConvetion.GPU_NAME: amdsmi.amdsmi_get_device_name(device_handle)+ }+ yield Observation(get_metric_value(device_handle, metric_name), attributes)++ except Exception as e:+ logger.error("Error in AMD GPU metrics collection: %s", e)diff --git a/sdk/python/tests/requirements.txt b/sdk/python/tests/requirements.txtindex 9de965e8..2fd68417 100644--- a/sdk/python/tests/requirements.txt+++ b/sdk/python/tests/requirements.txt@@ -31,7 +31,7 @@ langchainhub>=0.1.20embedchain>=0.1.104# GPU-gpustat>=1.1.1+xmltodict>=0.13.0# Utilitiesbs4diff --git a/sdk/python/tests/test_chroma.py b/sdk/python/tests/test_chroma.pyindex 861c92f2..774f8a4c 100644--- a/sdk/python/tests/test_chroma.py+++ b/sdk/python/tests/test_chroma.py@@ -1,4 +1,4 @@-# pylint: disable=duplicate-code, no-name-in-module+# pylint: disable=duplicate-code, no-name-in-module, assignment-from-no-return"""This module contains tests for ChromaDB functionality using the ChromaDB Python library.