Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: report free memory on Mac as what is actually free (#10224)
Nilaye46b7231abff
diff --git a/studio/backend/main.py b/studio/backend/main.pyindex 4d261ae7dc6..f5f44f1eb10 100644--- a/studio/backend/main.py+++ b/studio/backend/main.py@@ -2035,10 +2035,14 @@ def _get_cached_system_gpu_info(logger) -> tuple[dict[str, Any], dict[str, Any]]enriched_dev = dict(dev)enriched_dev["vram_used_gb"] = used_vram+ # A producer that reports free wins: on Apple unified memory free is+ # not total - used, so recomputing it here would undo that answer.enriched_dev["vram_free_gb"] = (- round(total_vram - used_vram, 2)+ reported_free_vram+ if reported_free_vram is not None+ else round(total_vram - used_vram, 2)if total_vram and used_vram is not None- else reported_free_vram+ else None)enriched_dev["vram_utilization_pct"] = util.get("vram_utilization_pct", dev.get("vram_utilization_pct")diff --git a/studio/backend/tests/test_system_vulkan_gpu_info.py b/studio/backend/tests/test_system_vulkan_gpu_info.pyindex 7e9dc78d1fd..8ca7ed29087 100644--- a/studio/backend/tests/test_system_vulkan_gpu_info.py+++ b/studio/backend/tests/test_system_vulkan_gpu_info.py@@ -287,3 +287,105 @@ def test_vulkan_inference_gpu_uses_inventory_fallback_names(monkeypatch):info = get_vulkan_inference_gpu_info()assert info["devices"][0]["name"] == "Vulkan0"assert info["devices"][0]["memory_total_gb"] == 16.0+++def test_system_gpu_info_keeps_a_reported_free_over_total_minus_used(monkeypatch):+ """Apple unified memory reports free directly because it is not total - used.+ Recomputing it here put the overstated figure back on the Resources tab while+ /api/system/hardware served the honest one."""+ import utils.hardware as hardware++ monkeypatch.setattr(+ hardware,+ "get_backend_visible_gpu_info",+ lambda: {+ "available": True,+ "backend": "mlx",+ "index_kind": "relative",+ "devices": [+ {+ "index": 0,+ "index_kind": "relative",+ "visible_ordinal": 0,+ "name": "Apple Silicon (Apple M2)",+ "memory_total_gb": 16.0,+ }+ ],+ },+ )+ monkeypatch.setattr(+ hardware,+ "get_visible_gpu_utilization",+ lambda: {+ "available": True,+ "backend": "mlx",+ "index_kind": "relative",+ "parent_visible_gpu_ids": [0],+ "devices": [+ {+ "index": 0,+ "vram_total_gb": 16.0,+ "vram_used_gb": 1.2,+ "vram_free_gb": 6.0,+ "vram_utilization_pct": 7.5,+ }+ ],+ },+ )++ from core.inference.llama_cpp import LlamaCppBackend++ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda: False))+ monkeypatch.setattr(main, "_system_gpu_cache", None)++ gpu, _inference_gpu = main._get_cached_system_gpu_info(+ SimpleNamespace(debug = lambda *args: None)+ )++ assert gpu["devices"][0]["vram_free_gb"] == 6.0+++def test_system_gpu_info_still_derives_free_when_the_probe_reports_none(monkeypatch):+ """CUDA's utilization probe reports no free, so the subtraction has to stay."""+ import utils.hardware as hardware++ monkeypatch.setattr(+ hardware,+ "get_backend_visible_gpu_info",+ lambda: {+ "available": True,+ "backend": "cuda",+ "index_kind": "relative",+ "devices": [+ {+ "index": 0,+ "index_kind": "relative",+ "visible_ordinal": 0,+ "name": "NVIDIA GeForce RTX 4090",+ "memory_total_gb": 24.0,+ }+ ],+ },+ )+ monkeypatch.setattr(+ hardware,+ "get_visible_gpu_utilization",+ lambda: {+ "available": True,+ "backend": "cuda",+ "index_kind": "relative",+ "parent_visible_gpu_ids": [0],+ "devices": [{"index": 0, "vram_total_gb": 24.0, "vram_used_gb": 6.0}],+ },+ )++ from core.inference.llama_cpp import LlamaCppBackend++ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda: False))+ monkeypatch.setattr(main, "_system_gpu_cache", None)++ gpu, _inference_gpu = main._get_cached_system_gpu_info(+ SimpleNamespace(debug = lambda *args: None)+ )++ assert gpu["devices"][0]["vram_free_gb"] == 18.0diff --git a/studio/backend/tests/test_utils.py b/studio/backend/tests/test_utils.pyindex 8551a458668..595b5b5017f 100644--- a/studio/backend/tests/test_utils.py+++ b/studio/backend/tests/test_utils.py@@ -10,6 +10,7 @@ is imported at top level; tests needing torch/mlx internals skip when unavailablimport platformimport sysimport types+from contextlib import contextmanagerfrom unittest.mock import patch, MagicMockimport pytest@@ -235,6 +236,105 @@ class TestGetGpuMemoryInfo:assert 0 <= result["utilization_pct"] <= 100assert "device_name" in result+ @contextmanager+ def _mlx_machine(+ self,+ *,+ available_gb,+ recommended_gb,+ used_gb = 1.2,+ legacy_mlx = False,+ ):+ props = {+ "device_name": "Apple M2",+ "max_recommended_working_set_size": int(recommended_gb * (1024**3)),+ }+ fake_core = types.ModuleType("mlx.core")+ if legacy_mlx:+ # mlx below 0.30 only has mx.metal.device_info().+ fake_metal = types.ModuleType("mlx.core.metal")+ fake_metal.device_info = lambda: props+ fake_core.metal = fake_metal+ else:+ fake_core.device_info = lambda: props+ fake_pkg = types.ModuleType("mlx")+ fake_pkg.core = fake_core++ with (+ patch.dict(sys.modules, {"mlx": fake_pkg, "mlx.core": fake_core}),+ patch("utils.hardware.hardware.get_device", return_value = DeviceType.MLX),+ patch(+ "psutil.virtual_memory",+ return_value = types.SimpleNamespace(+ total = 16 * (1024**3),+ available = int(available_gb * (1024**3)),+ ),+ ),+ patch(+ "utils.hardware.hardware._read_apple_gpu_stats",+ return_value = {"vram_used_bytes": int(used_gb * (1024**3))},+ ),+ ):+ yield++ def _mlx_memory_info(self, **machine):+ with self._mlx_machine(**machine):+ return get_gpu_memory_info()++ def test_mlx_free_is_what_a_new_allocation_can_get(self):+ result = self._mlx_memory_info(available_gb = 6, recommended_gb = 11)++ assert result["available"] is True+ assert abs(result["total_gb"] - 16.0) < 0.01+ assert abs(result["free_gb"] - 6.0) < 0.01++ def test_mlx_free_is_bounded_by_the_metal_working_set(self):+ result = self._mlx_memory_info(available_gb = 15, recommended_gb = 11)++ assert abs(result["free_gb"] - 11.0) < 0.01++ def test_mlx_free_is_not_reduced_by_whole_device_gpu_use(self):+ """The working set is a per-process budget, and the AGX counter behind+ used_gb is whole-device and only the active subset, so charging one+ against the other would let another app's GPU work pick the training+ method."""+ busy = self._mlx_memory_info(available_gb = 6, recommended_gb = 11, used_gb = 8)+ idle = self._mlx_memory_info(available_gb = 6, recommended_gb = 11, used_gb = 0.4)++ assert abs(busy["free_gb"] - idle["free_gb"]) < 0.01+ assert abs(busy["free_gb"] - 6.0) < 0.01++ def test_mlx_free_reads_the_working_set_on_pre_0_30_mlx(self):+ """The stack gate accepts mlx >= 0.22.0, and mlx below 0.30 spells this+ mx.metal.device_info(). Reading only mx.device_info() left the cap+ unapplied on an M1 running mlx 0.29.3, which the gate calls usable."""+ legacy = self._mlx_memory_info(available_gb = 15, recommended_gb = 11, legacy_mlx = True)+ current = self._mlx_memory_info(available_gb = 15, recommended_gb = 11)++ assert abs(legacy["free_gb"] - 11.0) < 0.01+ assert abs(legacy["free_gb"] - current["free_gb"]) < 0.01+ assert legacy["device_name"] == current["device_name"]++ def test_mlx_free_survives_a_missing_working_set_size(self):+ result = self._mlx_memory_info(available_gb = 6, recommended_gb = 0)++ assert abs(result["free_gb"] - 6.0) < 0.01++ def test_mlx_utilization_device_publishes_the_same_free_as_the_summary(self):+ """The Resources tab reads the per-device figure, and /api/system falls+ back to total - used for any device that does not report free. On+ unified memory that fallback is the 14.8 GB overstatement the tests+ above reject, so this probe has to carry free itself."""+ from utils.hardware.hardware import get_visible_gpu_utilization++ with self._mlx_machine(available_gb = 6, recommended_gb = 11):+ summary_free = get_gpu_memory_info()["free_gb"]+ device = get_visible_gpu_utilization()["devices"][0]++ assert abs(device["vram_free_gb"] - summary_free) < 0.01+ assert abs(device["vram_free_gb"] - 6.0) < 0.01+ assert device["vram_total_gb"] - device["vram_used_gb"] > 14.0+# --- CUDA-specific mocked test ---@needs_torchdiff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex 15b93a81b54..44205a40bfc 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -2396,6 +2396,50 @@ def _free_in_torch_scope(total_bytes: int, used_gb: float) -> int:return min(total_bytes, max(0, total_bytes - round(used_gb * (1024**3))))+def _mlx_device_info(mx: Any) -> Dict[str, Any]:+ """Metal device properties across the MLX rename.++ ``mx.device_info()`` is the current spelling; mlx below 0.30 has only+ ``mx.metal.device_info()``, and the stack gate accepts mlx >= 0.22.0+ (utils.mlx_repair._MLX_MIN_VERSIONS), so reading just the new name leaves the+ working set cap silently unapplied on a stack Studio considers usable.+ """+ for probe in (+ getattr(mx, "device_info", None),+ getattr(getattr(mx, "metal", None), "device_info", None),+ ):+ if callable(probe):+ try:+ return probe() or {}+ except Exception:+ continue+ return {}+++def _apple_unified_free_bytes(available_bytes: int, device_info: Any) -> int:+ """What is available right now, bounded by the Metal working set.++ Total RAM minus GPU-only usage counts host RAM as free, which the+ training-method policy then reads as room it does not have.++ The cap is not reduced by the AGX "In use system memory" counter: that is+ whole-device and only the active subset (a real M4 Pro reports 4.70 GiB+ allocated against 0.61 GiB in use), while the working set is a per-process+ budget, which is how torch.mps applies it.++ A floor rather than a capacity: macOS reclaims compressed and file-backed+ pages, so MLX can still allocate past what psutil calls available. Reading+ low costs a QLoRA suggestion where LoRA would have fit; reading high costs+ an OOM partway through a run.+ """+ free = max(0, int(available_bytes or 0))+ try:+ recommended = int(device_info.get("max_recommended_working_set_size") or 0)+ except Exception:+ recommended = 0+ return min(free, recommended) if recommended > 0 else free++def _context_free_cuda_memory_info(idx: int,total_bytes: int,@@ -2646,16 +2690,15 @@ def get_gpu_memory_info() -> Dict[str, Any]:import psutil# Unified memory: total = system RAM, GPU used from IORegistry AGX.- total = psutil.virtual_memory().total+ memory = psutil.virtual_memory()+ total = memory.totalagx = _read_apple_gpu_stats()allocated = agx.get("vram_used_bytes", 0) if agx else 0- try:- info = mx.device_info()- # prefer machine(); processor() can return "i386" on native arm64.- gpu_name = info.get("device_name") or platform.machine() or "arm64"- except Exception:- gpu_name = platform.machine() or "arm64"+ info = _mlx_device_info(mx)+ # prefer machine(); processor() can return "i386" on native arm64.+ gpu_name = info.get("device_name") or platform.machine() or "arm64"+ free = _apple_unified_free_bytes(getattr(memory, "available", 0), info)return {"available": True,@@ -2665,7 +2708,7 @@ def get_gpu_memory_info() -> Dict[str, Any]:"total_gb": total / (1024**3),"allocated_gb": allocated / (1024**3),"reserved_gb": allocated / (1024**3),- "free_gb": (total - allocated) / (1024**3),+ "free_gb": free / (1024**3),"utilization_pct": (allocated / total) * 100 if total else 0,}except Exception as e:@@ -5313,6 +5356,9 @@ def get_visible_gpu_utilization() -> Dict[str, Any]:"temperature_c": None,"vram_used_gb": round(mem.get("allocated_gb", 0), 2),"vram_total_gb": round(mem.get("total_gb", 0), 2),+ # Unified memory: free is not total - used, so publish it+ # rather than let the caller subtract.+ "vram_free_gb": round(mem.get("free_gb", 0), 2),"vram_utilization_pct": round(mem.get("utilization_pct", 0), 1),"power_draw_w": None,"power_limit_w": None,