Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

fix(gpu): parse decimal NVIDIA temperatures (#8224)

Mike Bradley018bed0be697
diff --git a/ods/extensions/services/dashboard-api/gpu.py b/ods/extensions/services/dashboard-api/gpu.py
index ea6d27cae..82ec7603d 100644
--- a/ods/extensions/services/dashboard-api/gpu.py
+++ b/ods/extensions/services/dashboard-api/gpu.py
@@ -194,7 +194,7 @@ def get_gpu_info_nvidia() -> Optional[GPUInfo]:
mem_used = int(parts[1])
mem_total = int(parts[2])
util = int(parts[3]) if parts[3] not in na_values else 0
- temp = int(parts[4]) if parts[4] not in na_values else 0
+ temp = int(float(parts[4])) if parts[4] not in na_values else 0
gpus.append({
"name": parts[0],
"mem_used": mem_used,
@@ -264,7 +264,7 @@ def get_gpu_info_nvidia() -> Optional[GPUInfo]:
utilization_available=all(g["utilization_available"] for g in gpus),
temperature_available=any(g["temperature_available"] for g in gpus),
)
- except (ValueError, IndexError):
+ except (ValueError, OverflowError, IndexError):
pass
return None
@@ -685,14 +685,14 @@ def get_gpu_info_nvidia_detailed() -> Optional[list[IndividualGPU]]:
memory_total_mb=mem_total,
memory_percent=round(mem_used / mem_total * 100, 1) if mem_total > 0 else 0.0,
utilization_percent=int(parts[5]) if parts[5] not in na_values else 0,
- temperature_c=int(parts[6]) if parts[6] not in na_values else 0,
+ temperature_c=int(float(parts[6])) if parts[6] not in na_values else 0,
power_w=power_w,
assigned_services=uuid_service_map.get(uuid, []),
memory_usage_available=not unified,
utilization_available=parts[5] not in na_values,
temperature_available=parts[6] not in na_values,
))
- except (ValueError, IndexError):
+ except (ValueError, OverflowError, IndexError):
logger.warning("Skipping unparseable nvidia-smi row: %s", line)
return gpus or None
diff --git a/ods/extensions/services/dashboard-api/tests/test_gpu.py b/ods/extensions/services/dashboard-api/tests/test_gpu.py
index 73ed10980..09274fcb2 100644
--- a/ods/extensions/services/dashboard-api/tests/test_gpu.py
+++ b/ods/extensions/services/dashboard-api/tests/test_gpu.py
@@ -68,8 +68,9 @@ class TestGetGpuTierUnified:
class TestGetGpuInfoNvidia:
- def test_parses_valid_output(self, monkeypatch):
- csv = "NVIDIA GeForce RTX 4090, 2048, 24564, 35, 62, 285.5"
+ @pytest.mark.parametrize("temperature", ["62", "62.00", "62.60"])
+ def test_parses_valid_output(self, monkeypatch, temperature):
+ csv = f"NVIDIA GeForce RTX 4090, 2048, 24564, 35, {temperature}, 285.5"
monkeypatch.setattr("gpu.run_command", lambda cmd, **kw: (True, csv))
info = get_gpu_info_nvidia()
@@ -102,6 +103,13 @@ class TestGetGpuInfoNvidia:
assert info.utilization_available is False
assert info.temperature_available is False
+ @pytest.mark.parametrize("temperature", ["NaN", "inf", "-inf"])
+ def test_rejects_nonfinite_temperature(self, monkeypatch, temperature):
+ csv = f"NVIDIA GeForce RTX 4090, 2048, 24564, 35, {temperature}, 285.5"
+ monkeypatch.setattr("gpu.run_command", lambda cmd, **kw: (True, csv))
+
+ assert get_gpu_info_nvidia() is None
+
def test_returns_none_on_command_failure(self, monkeypatch):
monkeypatch.setattr("gpu.run_command", lambda cmd, **kw: (False, ""))
diff --git a/ods/extensions/services/dashboard-api/tests/test_gpu_detailed.py b/ods/extensions/services/dashboard-api/tests/test_gpu_detailed.py
index 21d439c8a..4281559aa 100644
--- a/ods/extensions/services/dashboard-api/tests/test_gpu_detailed.py
+++ b/ods/extensions/services/dashboard-api/tests/test_gpu_detailed.py
@@ -134,8 +134,9 @@ class TestDecodeGpuAssignment:
class TestGetGpuInfoNvidiaDetailed:
- def test_parses_single_gpu(self, monkeypatch):
- csv = "0, GPU-abc123, NVIDIA GeForce RTX 4090, 2048, 24564, 35, 62, 285.5"
+ @pytest.mark.parametrize("temperature", ["62", "62.00", "62.60"])
+ def test_parses_single_gpu(self, monkeypatch, temperature):
+ csv = f"0, GPU-abc123, NVIDIA GeForce RTX 4090, 2048, 24564, 35, {temperature}, 285.5"
monkeypatch.setattr("gpu.run_command", lambda cmd, **kw: (True, csv))
monkeypatch.delenv("GPU_ASSIGNMENT_JSON_B64", raising=False)
@@ -211,6 +212,21 @@ class TestGetGpuInfoNvidiaDetailed:
assert result is not None
assert result[0].power_w is None
+ @pytest.mark.parametrize("temperature", ["NaN", "inf", "-inf"])
+ def test_skips_nonfinite_temperature_row(self, monkeypatch, temperature):
+ csv = (
+ f"0, GPU-bad, RTX 4090, 2048, 24564, 35, {temperature}, 285.5\n"
+ "1, GPU-good, RTX 4090, 4096, 24564, 50, 70.25, 300.0"
+ )
+ monkeypatch.setattr("gpu.run_command", lambda cmd, **kw: (True, csv))
+ monkeypatch.delenv("GPU_ASSIGNMENT_JSON_B64", raising=False)
+
+ result = get_gpu_info_nvidia_detailed()
+ assert result is not None
+ assert [gpu.uuid for gpu in result] == ["GPU-good"]
+ assert result[0].memory_total_mb == 24564
+ assert result[0].temperature_c == 70
+
def test_returns_none_on_failure(self, monkeypatch):
monkeypatch.setattr("gpu.run_command", lambda cmd, **kw: (False, ""))
assert get_gpu_info_nvidia_detailed() is None