Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: cut polled-probe cost and release memory sooner (#14595)
Michael Han173153b07fe9
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex dfdc77bd4ee..4b27191125a 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -8153,6 +8153,11 @@ def _load_probe_memo_scope():# "Probed, no usable answer", kept apart from None = "not probed yet"._MISSING = object()+# Display reads only (/api/system would spawn the probe per poll); a GPU memory event or fresh_reads() skips it.+_VULKAN_INVENTORY_TTL_S = 30.0+_vulkan_inventory_lock = threading.Lock()+_vulkan_inventory_cache: dict = {}+# Discrete by construction: `_weights_in_host_memory` classifies them upstream. Vulkan# gets the shared-memory probe; SYCL/OpenCL/MUSA/CANN may be integrated, so they decline._SELF_EVIDENTLY_DISCRETE = frozenset({"cuda"})@@ -15742,10 +15747,38 @@ class LlamaCppBackend:caller labels it, unlike the fit which zeroes it). Same rows as``_run_vulkan_probe``; names fall back to ``Vulkan<i>``."""+ binary = binary or LlamaCppBackend._find_llama_server_binary()+ if not binary:+ return []+ from utils import gpu_memory_events+ from utils.hardware import gpu_query++ use_cache = gpu_query.display_cache_allowed() and not _load_probe_memo_armed()+ key = ("vulkan", binary)+ if use_cache:+ with _vulkan_inventory_lock:+ entry = _vulkan_inventory_cache.get(key)+ if (+ entry is not None+ and time.monotonic() - entry[0] < _VULKAN_INVENTORY_TTL_S+ and entry[1] == gpu_memory_events.generation()+ ):+ return [dict(row) for row in entry[2]]+ generation = gpu_memory_events.generation()rows = LlamaCppBackend._run_vulkan_probe(binary)for row in rows:if not row["name"]:row["name"] = f"Vulkan{row['index']}"+ # An empty answer may be a failed probe: never served, like gpu_query's failures.+ with _vulkan_inventory_lock:+ if rows:+ _vulkan_inventory_cache[key] = (+ time.monotonic(),+ generation,+ [dict(row) for row in rows],+ )+ else:+ _vulkan_inventory_cache.pop(key, None)return rows@staticmethoddiff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.pyindex ce08397b54a..d3b6c5d08a2 100644--- a/studio/backend/core/rag/embeddings.py+++ b/studio/backend/core/rag/embeddings.py@@ -808,6 +808,14 @@ def _release_st_model() -> bool:released = _model is not None_model = None_name = None+ if released:+ # Dropping the reference leaves its blocks reserved; gated, since before detection clear_gpu_cache() runs it.+ try:+ from utils import hardware+ if hardware.hardware.DEVICE is not None:+ hardware.clear_gpu_cache()+ except Exception as e: # noqa: BLE001 - freeing is best effort+ logger.debug("GPU cache clear after embedder release failed: %s", e)return releaseddiff --git a/studio/backend/main.py b/studio/backend/main.pyindex 99d5db827b9..24036b14314 100644--- a/studio/backend/main.py+++ b/studio/backend/main.py@@ -67,6 +67,61 @@ _SYSTEM_GPU_CACHE_TTL_SECONDS = 10.0_system_gpu_cache_lock = threading.Lock()_system_gpu_cache: Optional[tuple[float, tuple[dict[str, Any], dict[str, Any]]]] = None+_CPU_PERCENT_MIN_WINDOW_S = 1.0+_cpu_percent_lock = threading.Lock()+_cpu_percent_reading: Optional[tuple[float, Optional[float]]] = None+++def _prime_cpu_percent() -> None:+ global _cpu_percent_reading+ try:+ import psutil+ psutil.cpu_percent(interval = None)+ except Exception:+ return+ with _cpu_percent_lock:+ _cpu_percent_reading = (time.monotonic(), None)+++def _cpu_usage_percent() -> float:+ """cpu_percent(None) measures since anyone's last call: sample at most once per window."""+ global _cpu_percent_reading+ import psutil++ with _cpu_percent_lock:+ now = time.monotonic()+ reading = _cpu_percent_reading+ if (+ reading is not None+ and reading[1] is not None+ and now - reading[0] < _CPU_PERCENT_MIN_WINDOW_S+ ):+ return reading[1]+ value = psutil.cpu_percent(interval = None)+ _cpu_percent_reading = (now, value)+ return value+++_prime_cpu_percent()+++def _process_tree_rss_mb(process: Any) -> int:+ """RSS of this process and its children (llama-server, the training worker), best effort."""+ import psutil++ total = process.memory_info().rss+ try:+ children = process.children(recursive = True)+ except (psutil.NoSuchProcess, psutil.AccessDenied):+ children = []+ for child in children:+ try:+ total += child.memory_info().rss+ except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):+ continue+ return round(total / 1024**2)++# Windows AMD ROCm DLL injection: Python 3.8+ ignores PATH for extension modules, so register ROCm bin# dirs with os.add_dll_directory() to find amdhip64.dll etc. before any torch import.if sys.platform == "win32":@@ -2445,6 +2500,7 @@ def _get_cached_system_gpu_info(logger, *, refresh_memory: bool = False) -> tuple[dict[str, Any], dict[str, Any]]:"""Return training and inference GPU info with bounded live-probe churn."""+ import contextlibimport timefrom utils.hardware import (LIVE_GPU_TELEMETRY_KEYS,@@ -2453,6 +2509,7 @@ def _get_cached_system_gpu_info(get_cross_vendor_inference_gpu_info,get_visible_gpu_utilization,get_vulkan_inference_gpu_info,+ gpu_query,)global _system_gpu_cache@@ -2470,10 +2527,6 @@ def _get_cached_system_gpu_info(visibility_info = {"available": False, "devices": []}try:- import contextlib-- from utils.hardware import gpu_query-# Already behind a 10 s cache: no stale-while-revalidate on top.with (contextlib.nullcontext() if refresh_memory else gpu_query.display_reads(max_stale = 0)@@ -2574,7 +2627,10 @@ def _get_cached_system_gpu_info(gpu_info["gguf_gpu_ids_supported"] = bool(enriched_devices)inference_gpu_info = gpu_infoelse:- vulkan_info = get_vulkan_inference_gpu_info()+ with (+ contextlib.nullcontext() if refresh_memory else gpu_query.display_reads(max_stale = 0)+ ):+ vulkan_info = get_vulkan_inference_gpu_info()cross_vendor_info = (get_cross_vendor_inference_gpu_info() if vulkan_info is None else None)@@ -2849,7 +2905,15 @@ def get_system_info(process_used_mb = round(current_process.memory_info().rss / 1024**2)except Exception as e:logger.debug(f"Failed to get current process memory: {e}")+ current_process = Noneprocess_used_mb = 0+ try:+ process_tree_used_mb = (+ _process_tree_rss_mb(current_process) if current_process else process_used_mb+ )+ except Exception as e:+ logger.debug(f"Failed to get process tree memory: {e}")+ process_tree_used_mb = process_used_mbtry:boot_time = psutil.boot_time()@@ -2879,7 +2943,7 @@ def get_system_info("cpu": {"logical_count": psutil.cpu_count(logical = True),"physical_count": psutil.cpu_count(logical = False),- "usage_percent": psutil.cpu_percent(interval = None),+ "usage_percent": _cpu_usage_percent(),"frequency_mhz": cpu_freq_mhz,},"memory": {@@ -2887,6 +2951,7 @@ def get_system_info("available_gb": round(memory_available / 1024**3, 2),"percent_used": memory_percent,"process_used_mb": process_used_mb,+ "process_tree_used_mb": process_tree_used_mb,},"disk": {"total_gb": round(disk.total / 1e9, 2) if disk else 0,diff --git a/studio/backend/routes/data_recipe/seed.py b/studio/backend/routes/data_recipe/seed.pyindex 1331f8ee845..12930eba139 100644--- a/studio/backend/routes/data_recipe/seed.py+++ b/studio/backend/routes/data_recipe/seed.py@@ -1388,7 +1388,15 @@ async def upload_unstructured_file(original_filename = file.filename or "upload"# Before the read: a rejected 500 MB upload must not be pulled into memory first._require_unstructured_ext(original_filename)- content = await file.read()+ limit = max(0, min(UNSTRUCTURED_RECIPE_UPLOAD_MAX_BYTES, budget))+ content = await file.read(limit + 1)+ if len(content) > limit:+ # Only limit + 1 bytes were read: report the real size the full read used to.+ declared = getattr(file, "size", None)+ _require_within_budget(+ declared if isinstance(declared, int) and declared > limit else len(content),+ budget,+ )else:raise HTTPException(400, "No file was provided.")diff --git a/studio/backend/routes/training_history.py b/studio/backend/routes/training_history.pyindex 02e04a3eff8..809b9fe4a10 100644--- a/studio/backend/routes/training_history.py+++ b/studio/backend/routes/training_history.py@@ -4,6 +4,7 @@"""Training history API routes: browse, view, and delete past training runs."""import asyncio+import functoolsimport jsonimport shutilimport uuid@@ -278,8 +279,8 @@ async def list_training_runs(via_api_key: bool = Depends(authenticated_via_api_key),):"""List training runs, newest first."""- result = list_runs(limit = limit, offset = offset)- sharing_on = get_preview_sharing_enabled() and not no_credential+ result = await asyncio.to_thread(functools.partial(list_runs, limit = limit, offset = offset))+ sharing_on = await asyncio.to_thread(get_preview_sharing_enabled) and not no_credentialruns = await asyncio.to_thread(_summaries_from_rows,result["runs"],@@ -303,7 +304,7 @@ async def get_training_run_detail(via_api_key: bool = Depends(authenticated_via_api_key),):"""Get a single training run with full config and metrics."""- run = get_run(run_id)+ run = await asyncio.to_thread(get_run, run_id)if run is None:raise HTTPException(status_code = 404, detail = f"Run {run_id} not found")@@ -314,13 +315,10 @@ async def get_training_run_detail(logger.debug("Failed to parse config_json for run %s", run_id)config = {}- metrics_data = get_run_metrics(run_id)+ metrics_data = await asyncio.to_thread(get_run_metrics, run_id)+ sharing_on = await asyncio.to_thread(get_preview_sharing_enabled) and not no_credential- summary = await asyncio.to_thread(- _summary_from_row,- run,- get_preview_sharing_enabled() and not no_credential,- )+ summary = await asyncio.to_thread(_summary_from_row, run, sharing_on)# The same persisted path, in the summary as `model_name` and again inside `config_json`,# so the WHOLE response goes through.from hub.utils.host_paths import redact_host_pathsdiff --git a/studio/backend/routes/video.py b/studio/backend/routes/video.pyindex ec4ebce4d81..a956860f70e 100644--- a/studio/backend/routes/video.py+++ b/studio/backend/routes/video.py@@ -1840,7 +1840,7 @@ async def _media_fields(fields: dict) -> dict:async def _audio_upload_to_data_url(upload: UploadFile, name: str) -> str:- data = await upload.read()+ data = await upload.read(_VIDEO_REFERENCE_AUDIO_MAX_BYTES + 1)if not data:raise _openai_video_error(400, f"{name} is empty.", param = "reference_audios")if len(data) > _VIDEO_REFERENCE_AUDIO_MAX_BYTES:@@ -1941,7 +1941,7 @@ async def _reference_to_data_url(reference: Any, param: str = "input_reference")if reference is None:return Noneif isinstance(reference, UploadFile):- data = await reference.read()+ data = await reference.read(VIDEO_INPUT_REFERENCE_MAX_BYTES + 1)if not data:raise _openai_video_error(400, f"{param} is empty.", param = param)if len(data) > VIDEO_INPUT_REFERENCE_MAX_BYTES:diff --git a/studio/backend/tests/test_data_recipe_seed.py b/studio/backend/tests/test_data_recipe_seed.pyindex e3386d3c9c2..185d6ab5205 100644--- a/studio/backend/tests/test_data_recipe_seed.py+++ b/studio/backend/tests/test_data_recipe_seed.py@@ -29,8 +29,8 @@ class _FakeUpload:self.filename = filenameself._content = content- async def read(self) -> bytes:- return self._content+ async def read(self, size: int = -1) -> bytes:+ return self._content if size < 0 else self._content[:size]def _load_seed_route(@@ -347,6 +347,29 @@ def test_total_upload_quota_is_scoped_per_block(monkeypatch, tmp_path):assert other.status == "ok"+def test_an_oversized_upload_is_refused_after_reading_one_byte_past_the_cap(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ monkeypatch.setattr(seed_route, "UNSTRUCTURED_RECIPE_UPLOAD_MAX_BYTES", 32)+ upload = _FakeUpload("big.txt", b"x" * 4096)+ upload.size = 4096+ sizes: list[int] = []+ real_read = upload.read++ async def tracking_read(size: int = -1) -> bytes:+ sizes.append(size)+ return await real_read(size)++ upload.read = tracking_read++ with pytest.raises(seed_route.HTTPException) as exc:+ asyncio.run(seed_route.upload_unstructured_file(upload, "block"))++ assert sizes == [33]+ assert exc.value.status_code == 413+ assert "(4096 bytes)" in exc.value.detail+ assert _block_files(seed_route) == []++def test_an_oversized_native_drop_is_refused_before_it_is_read(monkeypatch, tmp_path):seed_route = _load_seed_route(monkeypatch, tmp_path)huge = tmp_path / "corpus.txt"diff --git a/studio/backend/tests/test_polled_probe_costs.py b/studio/backend/tests/test_polled_probe_costs.pynew file mode 100644index 00000000000..db95588f5fb--- /dev/null+++ b/studio/backend/tests/test_polled_probe_costs.py@@ -0,0 +1,348 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""What /api/system and the training poll spawn per request, and what they give back.++Display polls reuse recent Vulkan / amd-smi / ioreg answers; fit and refresh paths still read+fresh. Also: CPU percent sampling, process-tree memory, embedder release, upload read bounds,+and a broken torch import retried once per detection epoch.+"""++import asyncio+import builtins+import subprocess+import sys+import types++import pytest++from utils import gpu_memory_events+from utils.hardware import gpu_query+++def vulkan(monkeypatch):+ from core.inference import llama_cpp++ monkeypatch.setattr(llama_cpp, "_vulkan_inventory_cache", {})+ calls = []+ answer = {+ "rows": [{"index": 0, "free_mib": 1024, "total_mib": 2048, "is_igpu": False, "name": ""}]+ }++ def probe(binary = None):+ calls.append(binary)+ return [dict(row) for row in answer["rows"]]++ monkeypatch.setattr(llama_cpp.LlamaCppBackend, "_run_vulkan_probe", staticmethod(probe))+ return llama_cpp, calls, answer+++def test_vulkan_inventory_is_reused_by_display_polls(vulkan):+ llama_cpp, calls, _ = vulkan+ with gpu_query.display_reads(max_stale = 0):+ first = llama_cpp.LlamaCppBackend.vulkan_device_inventory("/bin/llama-server")+ first[0]["name"] = "mutated"+ second = llama_cpp.LlamaCppBackend.vulkan_device_inventory("/bin/llama-server")+ assert len(calls) == 1+ assert second[0]["name"] == "Vulkan0"+++def test_vulkan_inventory_reprobes_outside_display_and_after_memory_events(vulkan):+ llama_cpp, calls, _ = vulkan+ inventory = llama_cpp.LlamaCppBackend.vulkan_device_inventory+ inventory("/bin/llama-server")+ inventory("/bin/llama-server")+ assert len(calls) == 2 # refresh_memory path: no display_reads, always live+ with gpu_query.display_reads(max_stale = 0):+ inventory("/bin/llama-server") # served from the live read above+ assert len(calls) == 2+ gpu_memory_events.invalidate_gpu_memory("model loaded")+ inventory("/bin/llama-server")+ with gpu_query.fresh_reads():+ inventory("/bin/llama-server")+ assert len(calls) == 4+++def test_vulkan_inventory_never_serves_an_empty_answer_or_a_load_memo(vulkan, monkeypatch):+ llama_cpp, calls, answer = vulkan+ inventory = llama_cpp.LlamaCppBackend.vulkan_device_inventory+ answer["rows"] = []+ with gpu_query.display_reads(max_stale = 0):+ inventory("/bin/llama-server")+ inventory("/bin/llama-server")+ assert len(calls) == 2+ monkeypatch.setattr(llama_cpp, "_load_probe_memo_armed", lambda: True)+ with gpu_query.display_reads(max_stale = 0):+ inventory("/bin/llama-server")+ assert len(calls) == 3+++def test_system_route_reads_vulkan_inside_display_reads_unless_refreshing():+ from pathlib import Path++ src = (Path(__file__).resolve().parents[1] / "main.py").read_text(encoding = "utf-8")+ block = src[src.index("vulkan_info = get_vulkan_inference_gpu_info()") - 200 :]+ assert "nullcontext() if refresh_memory else gpu_query.display_reads" in block[:220]+++def amd(monkeypatch):+ from utils.hardware import amd as amd_mod++ monkeypatch.setattr(amd_mod, "_display_cache", {})+ # An empty mask (CPU-only CI) would make the primary GPU id None and skip amd-smi.+ for name in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):+ monkeypatch.delenv(name, raising = False)+ calls = []+ answer = {"data": [{"gpu": 0, "usage": {"gfx_activity": {"value": 50, "unit": "%"}}}]}++ def run(*args, **kwargs):+ calls.append(args)+ return answer["data"]++ monkeypatch.setattr(amd_mod, "_run_amd_smi", run)+ return amd_mod, calls, answer+++def test_amd_smi_metric_is_coalesced_for_display_polls(amd):+ amd_mod, calls, _ = amd+ with gpu_query.display_reads():+ amd_mod.get_visible_gpu_utilization([0])+ amd_mod.get_visible_gpu_utilization([0])+ amd_mod.get_primary_gpu_utilization()+ amd_mod.get_primary_gpu_utilization()+ assert calls == [("metric",), ("metric", "-g", "0")]+++def test_amd_smi_metric_stays_live_for_fit_and_after_failures(amd):+ amd_mod, calls, answer = amd+ amd_mod.get_visible_gpu_utilization([0])+ amd_mod.get_visible_gpu_utilization([0])+ amd_mod.get_gpu_vram_report()+ assert len(calls) == 3+ answer["data"] = None+ with gpu_query.display_reads():+ amd_mod.get_visible_gpu_utilization([0])+ amd_mod.get_visible_gpu_utilization([0])+ assert len(calls) == 5+++def test_cpu_percent_is_sampled_at_most_once_per_window(monkeypatch):+ psutil = pytest.importorskip("psutil")++ import main++ clock = [100.0]+ readings = iter([12.5, 40.0])+ calls = []++ def cpu_percent(interval = None):+ calls.append(interval)+ return next(readings)++ monkeypatch.setattr(psutil, "cpu_percent", cpu_percent)+ monkeypatch.setattr(main, "time", types.SimpleNamespace(monotonic = lambda: clock[0]))+ monkeypatch.setattr(main, "_cpu_percent_reading", (99.5, None))++ assert main._cpu_usage_percent() == 12.5+ clock[0] += 0.4+ assert main._cpu_usage_percent() == 12.5+ clock[0] += 1.0+ assert main._cpu_usage_percent() == 40.0+ assert calls == [None, None]+++def test_cpu_percent_is_primed_at_import():+ pytest.importorskip("psutil")+ import main+ assert main._cpu_percent_reading is not None+++def test_process_tree_memory_adds_children_and_skips_vanished_ones():+ psutil = pytest.importorskip("psutil")++ import main++ mib = 1024**2++ class Proc:+ def __init__(+ self,+ rss,+ exc = None,+ children = (),+ ):+ self._rss, self._exc, self._children = rss, exc, list(children)++ def memory_info(self):+ if self._exc:+ raise self._exc+ return types.SimpleNamespace(rss = self._rss)++ def children(self, recursive = False):+ assert recursive+ return self._children++ root = Proc(+ 100 * mib,+ children = [+ Proc(900 * mib),+ Proc(0, psutil.NoSuchProcess(1)),+ Proc(0, psutil.AccessDenied(2)),+ ],+ )+ assert main._process_tree_rss_mb(root) == 1000++ class Locked(Proc):+ def children(self, recursive = False):+ raise psutil.AccessDenied(3)++ assert main._process_tree_rss_mb(Locked(100 * mib)) == 100+++def test_mlx_visible_utilization_carries_apple_telemetry(monkeypatch):+ from utils.hardware import apple+ from utils.hardware import hardware as hw++ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.MLX)+ monkeypatch.setattr(+ hw,+ "get_gpu_memory_info",+ lambda: {+ "available": True,+ "allocated_gb": 4.0,+ "total_gb": 32.0,+ "free_gb": 20.0,+ "utilization_pct": 12.5,+ },+ )+ monkeypatch.setattr(hw, "_read_apple_gpu_stats", lambda: {"utilization_pct": 37})+ monkeypatch.setattr(apple, "read_gpu_temperature_c", lambda: 51.5)+ monkeypatch.setattr(apple, "read_gpu_power_w", lambda: 6.2)++ (device,) = hw.get_visible_gpu_utilization()["devices"]+ assert (device["gpu_utilization_pct"], device["temperature_c"], device["power_draw_w"]) == (+ 37,+ 51.5,+ 6.2,+ )+ assert device["vram_total_gb"] == 32.0 and device["vram_free_gb"] == 20.0++ def broken():+ raise OSError("no SMC")++ monkeypatch.setattr(apple, "read_gpu_temperature_c", broken)+ (device,) = hw.get_visible_gpu_utilization()["devices"]+ assert device["temperature_c"] is None and device["vram_used_gb"] == 4.0+++def test_agx_ioreg_is_read_once_per_display_refresh(monkeypatch):+ from utils.hardware import hardware as hw++ monkeypatch.setattr(hw, "_apple_gpu_stats_cache", None)+ calls = []+ block = '"PerformanceStatistics" = {"Device Utilization %"=21,"In use system memory"=1024}'++ def fake_run(cmd, **kwargs):+ calls.append(cmd)+ return types.SimpleNamespace(stdout = block.encode())++ monkeypatch.setattr(subprocess, "run", fake_run)+ # Visibility reads live and seeds the reading; the display read right after reuses it.+ assert hw._read_apple_gpu_stats() == {"utilization_pct": 21, "vram_used_bytes": 1024}+ with gpu_query.display_reads(max_stale = 0):+ assert hw._read_apple_gpu_stats()["utilization_pct"] == 21+ assert len(calls) == 1+ # Fit and placement never take the cached reading.+ hw._read_apple_gpu_stats()+ assert len(calls) == 2+++def test_releasing_the_st_embedder_frees_the_allocator_cache(monkeypatch):+ from core.rag import embeddings+ import utils.hardware as hardware++ cleared = []+ monkeypatch.setattr(hardware, "clear_gpu_cache", lambda: cleared.append(True))+ monkeypatch.setattr(embeddings, "_model", object())+ monkeypatch.setattr(embeddings, "_name", "m")++ # Before detection publishes, a release must not trigger it through clear_gpu_cache().+ monkeypatch.setattr(hardware.hardware, "DEVICE", None)+ assert embeddings._release_st_model() is True+ assert cleared == []++ monkeypatch.setattr(hardware.hardware, "DEVICE", hardware.DeviceType.CUDA)+ monkeypatch.setattr(embeddings, "_model", object())+ assert embeddings._release_st_model() is True+ assert cleared == [True]+ assert embeddings._release_st_model() is False+ assert cleared == [True]++ def boom():+ raise RuntimeError("CUDA error")++ monkeypatch.setattr(hardware, "clear_gpu_cache", boom)+ monkeypatch.setattr(embeddings, "_model", object())+ assert embeddings._release_st_model() is True+++class _Upload:+ def __init__(self, data: bytes, content_type: str):+ self._data, self.content_type, self.sizes = data, content_type, []++ async def read(self, size: int = -1) -> bytes:+ self.sizes.append(size)+ return self._data if size < 0 else self._data[:size]+++def test_video_reference_uploads_read_at_most_one_byte_past_the_cap(monkeypatch):+ import routes.video as video_routes++ monkeypatch.setattr(video_routes, "_VIDEO_REFERENCE_AUDIO_MAX_BYTES", 8)+ monkeypatch.setattr(video_routes, "VIDEO_INPUT_REFERENCE_MAX_BYTES", 16)++ audio = _Upload(b"x" * 1000, "audio/wav")+ with pytest.raises(Exception) as exc:+ asyncio.run(video_routes._audio_upload_to_data_url(audio, "reference_audios[0]"))+ assert audio.sizes == [9]+ assert "is too large; the limit is 24 MB." in str(getattr(exc.value, "detail", exc.value))++ image = _Upload(b"x" * 1000, "image/png")+ monkeypatch.setattr(video_routes, "UploadFile", _Upload)+ with pytest.raises(Exception) as exc:+ asyncio.run(video_routes._reference_to_data_url(image))+ assert image.sizes == [17]+ assert "is too large; the limit is 0 MB." in str(getattr(exc.value, "detail", exc.value))+++def test_a_broken_torch_import_is_retried_once_per_detection_epoch(monkeypatch):+ from utils import torch_warmup+ from utils.hardware import hardware as hw++ monkeypatch.setattr(hw, "TORCH_IMPORT_ERROR", hw.TORCH_IMPORT_ERROR)+ monkeypatch.setattr(hw, "_TORCH_IMPORT_FAILED_EPOCH", None)+ monkeypatch.setattr(torch_warmup, "purge_partial_import", lambda name: None)+ for name in [n for n in sys.modules if n == "torch" or n.startswith("torch.")]:+ monkeypatch.delitem(sys.modules, name)+ errors = []+ monkeypatch.setattr(hw.logger, "error", lambda *a, **k: errors.append(a))+ real_import = builtins.__import__+ attempts = []++ def fake_import(name, *args, **kwargs):+ if name == "torch":+ attempts.append(name)+ raise OSError("libcudart.so.12: cannot open shared object file")+ return real_import(name, *args, **kwargs)++ monkeypatch.setattr(builtins, "__import__", fake_import)+ assert hw._has_torch() is False+ assert hw._has_torch() is False+ assert len(attempts) == 1 and len(errors) == 1+ assert hw.TORCH_IMPORT_ERROR is not None++ monkeypatch.setattr(hw, "DETECTION_EPOCH", hw.DETECTION_EPOCH + 1)+ assert hw._has_torch() is False+ assert len(attempts) == 2diff --git a/studio/backend/tests/test_training_history_delete.py b/studio/backend/tests/test_training_history_delete.pyindex 89045bf9fa7..f00595b8898 100644--- a/studio/backend/tests/test_training_history_delete.py+++ b/studio/backend/tests/test_training_history_delete.py@@ -254,7 +254,8 @@ def test_list_training_runs_offloads_summary_batch(monkeypatch):)assert [summary.id for summary in response.runs] == ["b", "a", "c"]- assert offloads == [(training_history._summaries_from_rows, (rows, False))]+ assert offloads[-1] == (training_history._summaries_from_rows, (rows, False))+ assert len(offloads) == 3def _delete(@@ -788,3 +789,37 @@ def test_shared_guard_compares_canonical_paths(monkeypatch, tmp_path):)assert training_history._output_dir_shared(str(run_dir), "run-1") is True+++def test_training_run_detail_reads_the_db_off_the_event_loop(monkeypatch):+ offloaded: list[object] = []+ row = _run_row(id = "a", output_dir = "/tmp/a")++ async def record_offload(function, *args):+ offloaded.append(function)+ return function(*args)++ def get_run(run_id):+ return dict(row)++ def get_run_metrics(run_id):+ return {}++ def sharing():+ return False++ monkeypatch.setattr(training_history.asyncio, "to_thread", record_offload)+ monkeypatch.setattr(training_history, "get_run", get_run)+ monkeypatch.setattr(training_history, "get_run_metrics", get_run_metrics)+ monkeypatch.setattr(training_history, "get_preview_sharing_enabled", sharing)+ monkeypatch.setattr(+ training_history, "can_resume_run", lambda run, *, resource_cache = None: False+ )+ monkeypatch.setattr(training_history, "artifacts_present", lambda path: False)++ response = asyncio.run(+ training_history.get_training_run_detail("a", current_subject = "test-user")+ )++ assert response.run.id == "a"+ assert {get_run, get_run_metrics, sharing} <= set(offloaded)diff --git a/studio/backend/utils/hardware/amd.py b/studio/backend/utils/hardware/amd.pyindex c01fea84da7..1622f78298e 100644--- a/studio/backend/utils/hardware/amd.py+++ b/studio/backend/utils/hardware/amd.py@@ -3,6 +3,7 @@"""AMD GPU monitoring via amd-smi. Mirrors nvidia.py so hardware.py can swap backends based on IS_ROCM; all functions return the same dict shapes as their nvidia.py counterparts."""+import copyimport globimport jsonimport math@@ -132,6 +133,39 @@ def _run_amd_smi(return None+_display_lock = threading.Lock()+_display_cache: dict[tuple[str, ...], tuple[float, int, Any]] = {}+++def _run_amd_smi_for_display(*args: str) -> Optional[Any]:+ """``_run_amd_smi`` under gpu_query's display rules; the lock coalesces pollers, failures are never cached."""+ try:+ from utils import gpu_memory_events+ from utils.hardware import gpu_query++ use_cache = gpu_query.display_cache_allowed()+ ttl = gpu_query.ttl_for(gpu_query.DISPLAY)+ except Exception:+ use_cache = False+ if not use_cache:+ return _run_amd_smi(*args)+ with _display_lock:+ cached = _display_cache.get(args)+ if (+ cached is not None+ and time.monotonic() - cached[0] < ttl+ and cached[1] == gpu_memory_events.generation()+ ):+ return copy.deepcopy(cached[2])+ generation = gpu_memory_events.generation()+ data = _run_amd_smi(*args)+ if data is None:+ _display_cache.pop(args, None)+ else:+ _display_cache[args] = (time.monotonic(), generation, copy.deepcopy(data))+ return data++def _parse_numeric(value: Any) -> Optional[float]:"""Extract a numeric value from amd-smi output (str, int, float, or dict)."""if value is None:@@ -402,7 +436,7 @@ def get_primary_gpu_utilization() -> dict[str, Any]:gpu_idx = _first_visible_amd_gpu_id()if gpu_idx is None:return {"available": False}- data = _run_amd_smi("metric", "-g", gpu_idx)+ data = _run_amd_smi_for_display("metric", "-g", gpu_idx)if data is None:return {"available": False}@@ -436,7 +470,7 @@ def get_visible_gpu_utilization("index_kind": "unresolved",}- data = _run_amd_smi("metric")+ data = _run_amd_smi_for_display("metric")if data is None:return {"available": False,diff --git a/studio/backend/utils/hardware/apple.py b/studio/backend/utils/hardware/apple.pyindex 82c9b01b6d0..df3da61cfcc 100644--- a/studio/backend/utils/hardware/apple.py+++ b/studio/backend/utils/hardware/apple.py@@ -18,6 +18,7 @@ Public API (never raises; returns None when sensors are unavailable):import ctypesimport struct+import threadingimport timefrom typing import Iterable, Optional@@ -380,6 +381,8 @@ _smc: Optional[_SMCConnection] = None_smc_failed = False_energy: Optional[_IOReportEnergy] = None_energy_failed = False+# Two polling threads: gpu_power_w releases the previous CF sample, so unlocked it double-frees.+_sensor_lock = threading.Lock()def read_gpu_temperature_c() -> Optional[float]:@@ -388,9 +391,10 @@ def read_gpu_temperature_c() -> Optional[float]:if _smc_failed:return Nonetry:- if _smc is None:- _smc = _SMCConnection()- return _smc.gpu_temperature_c()+ with _sensor_lock:+ if _smc is None:+ _smc = _SMCConnection()+ return _smc.gpu_temperature_c()except Exception as e:_smc_failed = Truelogger.warning("Apple SMC GPU temperature unavailable: %s", e)@@ -406,9 +410,10 @@ def read_gpu_power_w() -> Optional[float]:if _energy_failed:return Nonetry:- if _energy is None:- _energy = _IOReportEnergy()- return _energy.gpu_power_w()+ with _sensor_lock:+ if _energy is None:+ _energy = _IOReportEnergy()+ return _energy.gpu_power_w()except Exception as e:_energy_failed = Truelogger.warning("Apple IOReport GPU power unavailable: %s", e)diff --git a/studio/backend/utils/hardware/gpu_query.py b/studio/backend/utils/hardware/gpu_query.pyindex 71374b79bde..32a1575c971 100644--- a/studio/backend/utils/hardware/gpu_query.py+++ b/studio/backend/utils/hardware/gpu_query.py@@ -122,6 +122,11 @@ def fresh_reads() -> Iterator[None]:_fresh_mode.reset(token)+def display_cache_allowed() -> bool:+ """Gate for Studio's other display-only CLI caches (amd-smi, Vulkan probe): same rules as nvidia-smi."""+ return enabled() and _display_mode.get() is not None and not _fresh_mode.get()++def classify(argv: Sequence[str]) -> str:args = list(argv[1:])if args and args[0] in _STATIC_SUBCOMMANDS:diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex a7d63467d93..e2cc9d543ec 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -179,19 +179,30 @@ def _running_under_rosetta() -> bool:# Set by _has_torch() when torch is installed but its import blew up (unresolved CUDA libs); the CPU fallback reports that as a detection failure, not as "no GPU".TORCH_IMPORT_ERROR: Optional[str] = None+# Detection epoch of the last broken import: retried (and logged) once per epoch, not on every poll.+_TORCH_IMPORT_FAILED_EPOCH: Optional[int] = Nonedef _has_torch() -> bool:"""True if PyTorch is importable. Any failure counts as "no torch": ensure_hardware_detected() re-runs while DEVICE is None, so an escaping OSError would make every request retry the import. The error is recorded, or the host is told to install the torch it already has."""- global TORCH_IMPORT_ERROR+ global TORCH_IMPORT_ERROR, _TORCH_IMPORT_FAILED_EPOCH+ if (+ TORCH_IMPORT_ERROR is not None+ and _TORCH_IMPORT_FAILED_EPOCH == DETECTION_EPOCH+ and "torch" not in sys.modules+ ):+ return Falsetry:import torch+TORCH_IMPORT_ERROR = None+ _TORCH_IMPORT_FAILED_EPOCH = Nonereturn Trueexcept Exception as exc:# ImportError does NOT mean "not installed": a wheel with unresolved native libs raises it from torch's own __init__. Only ModuleNotFoundError naming torch itself is absent; a failed submodule is a broken install.absent = isinstance(exc, ModuleNotFoundError) and exc.name == "torch"TORCH_IMPORT_ERROR = None if absent else repr(exc)+ _TORCH_IMPORT_FAILED_EPOCH = None if absent else DETECTION_EPOCHif TORCH_IMPORT_ERROR is not None:logger.error("torch is installed but failed to import: %r", exc)# A part-way failure leaves submodules cached under an evicted parent, so the next importer gets a torch missing pieces; purge_partial_import() clears that.@@ -3124,8 +3135,33 @@ def _smi_query(func_name: str, *args, **kwargs) -> Optional[Dict[str, Any]]:return None+_APPLE_GPU_STATS_TTL_S = 1.5+_apple_gpu_stats_lock = threading.Lock()+_apple_gpu_stats_cache: Optional[tuple[float, Dict[str, Any]]] = None++def _read_apple_gpu_stats() -> Dict[str, Any]:- """macOS IORegistry AGX live stats (utilization_pct, and vram_used_bytes when reported; system-wide), or {} on failure. No sudo needed."""+ """macOS IORegistry AGX live stats (utilization_pct, and vram_used_bytes when reported; system-wide), or {} on failure. No sudo needed. Display reads reuse one up to 1.5 s old (a refresh asks twice); fit and placement read fresh."""+ global _apple_gpu_stats_cache+ try:+ from utils.hardware import gpu_query+ serve_cached = gpu_query.display_cache_allowed()+ except Exception:+ serve_cached = False+ with _apple_gpu_stats_lock:+ cached = _apple_gpu_stats_cache+ if (+ serve_cached+ and cached is not None+ and time.monotonic() - cached[0] < _APPLE_GPU_STATS_TTL_S+ ):+ return dict(cached[1])+ stats = _read_apple_gpu_stats_uncached()+ _apple_gpu_stats_cache = (time.monotonic(), dict(stats)) if stats else None+ return stats+++def _read_apple_gpu_stats_uncached() -> Dict[str, Any]:try:result = subprocess.run(["ioreg", "-r", "-c", "AGXAccelerator"],@@ -5160,6 +5196,15 @@ def get_visible_gpu_utilization(*, no_context: bool = False) -> Dict[str, Any]:"devices": [],"index_kind": "relative",}+ gpu_utilization_pct = temperature_c = power_draw_w = None+ try:+ from . import apple++ gpu_utilization_pct = _read_apple_gpu_stats().get("utilization_pct")+ temperature_c = apple.read_gpu_temperature_c()+ power_draw_w = apple.read_gpu_power_w()+ except Exception as e:+ logger.debug("Apple GPU telemetry unavailable: %s", e)return {"available": True,"backend": _backend_label(device),@@ -5169,14 +5214,14 @@ def get_visible_gpu_utilization(*, no_context: bool = False) -> Dict[str, Any]:"index": 0,"index_kind": "relative","visible_ordinal": 0,- "gpu_utilization_pct": None,- "temperature_c": None,+ "gpu_utilization_pct": gpu_utilization_pct,+ "temperature_c": temperature_c,"vram_used_gb": round(mem.get("allocated_gb", 0), 2),"vram_total_gb": round(mem.get("total_gb", 0), 2),# Unified memory: free is not total - used, so publish it rather than let the caller subtract."vram_free_gb": round(mem.get("free_gb", 0), 2),"vram_utilization_pct": round(mem.get("utilization_pct", 0), 1),- "power_draw_w": None,+ "power_draw_w": power_draw_w,"power_limit_w": None,"power_utilization_pct": None,}