Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: pin the kept offload weights on first onload when host RAM allows (#11766)
Daniel Hanb0af0ea684dd
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 120e882e041..dcaaf55c57d 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -114,6 +114,7 @@ from .diffusion_memory import (reclaimable_snapshot_device_memory,reclaim_host_memory,reclaim_offload_host_memory,+ release_pinned_host_memory,refine_memory_plan_for_components,settled_snapshot_device_memory,snapshot_device_memory,@@ -7846,7 +7847,11 @@ class DiffusionBackend:self._cn_models.clear()self._state = Nonedel state- clear_gpu_cache()+ try:+ clear_gpu_cache()+ finally:+ # finally: a sticky CUDA fault must not keep the pinned chunks locked.+ release_pinned_host_memory()# Must follow clear_gpu_cache() (runs gc) so the freed staging buffers can be returned.reclaim_host_memory(logger = logger)diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex a510504e61c..4c0e2a806ac 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -159,7 +159,7 @@ def _wrap_cpu_offload_hooks(if not isinstance(hook, hook_cls) or getattr(hook, _KEEP_ATTR, False):continuetry:- _wrap_cpu_offload_hook(hook, module)+ _wrap_cpu_offload_hook(hook, module, logger)wrapped += 1except Exception as exc: # noqa: BLE001 - that module keeps the stock copyif logger is not None:@@ -171,7 +171,133 @@ def _wrap_cpu_offload_hooks(return wrapped-def _wrap_cpu_offload_hook(hook: Any, module: Any) -> None:+OFFLOAD_PIN_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_PIN"+# The host allocator rounds every block up to a power of two, so weights share chunks instead of one pin each.+_PIN_CHUNK_BYTES = 1 << 30+_PIN_ALIGN = 256+_PIN_RESERVE_MIN_BYTES = 4 << 30+_PIN_RESERVE_FRACTION = 0.15+++def _pin_mode() -> str:+ raw = (os.environ.get(OFFLOAD_PIN_ENV) or "").strip().lower()+ if raw in ("0", "off", "false", "no"):+ return "off"+ if raw in ("1", "on", "true", "yes", "force"):+ return "on"+ return "auto"+++def _pow2_ceil(n: int) -> int:+ return 1 << max(0, int(n) - 1).bit_length()+++def release_pinned_host_memory() -> None:+ _host_empty_cache()+++def _host_empty_cache() -> None:+ try:+ import torch+ empty = getattr(torch._C, "_host_emptyCache", None)+ if callable(empty) and torch.cuda.is_available():+ empty()+ except Exception: # noqa: BLE001+ pass+++def _pin_host_weights(+ module: Any,+ host: dict,+ logger: Any = None,+) -> int:+ """Pack kept host weights into page-locked chunks; returns bytes pinned. Contiguous only, so channels_last survives."""+ import torch++ mode = _pin_mode()+ if mode == "off" or not torch.cuda.is_available():+ return 0+ params = [+ (name, p)+ for name, p in module.named_parameters()+ if host.get(name) is not None+ and p.device.type == "cpu"+ and _keepable(p)+ and p.data.is_contiguous()+ and not p.data.is_pinned()+ ]+ sizes = [-(-p.data.nbytes // _PIN_ALIGN) * _PIN_ALIGN for _, p in params]+ if not sizes or sum(sizes) == 0:+ return 0+ # Lay out chunks first so the RAM gate counts chunk tails and last-chunk rounding, not just weight bytes.+ chunk = max(_PIN_CHUNK_BYTES, _pow2_ceil(max(sizes)))+ chunks: list = []+ slots: list = []+ off, left = 0, sum(sizes)+ for size in sizes:+ if not chunks or off + size > chunks[-1]:+ chunks.append(chunk if left >= chunk else _pow2_ceil(left))+ off = 0+ slots.append((len(chunks) - 1, off))+ off += size+ left -= size+ need = sum(chunks)+ if mode == "auto":+ try:+ import psutil+ vm = psutil.virtual_memory()+ except Exception: # noqa: BLE001 - no way to size it, so do not lock memory+ return 0+ # psutil reads the host; pinned pages are charged to an enforcing cgroup, so size from the container.+ available, total = vm.available, vm.total+ remainder = _cgroup_available_memory_mib()+ if remainder is not None:+ available = min(available, int(remainder) << 20)+ limit = _cgroup_memory_limit_mib()+ if limit is not None:+ total = min(total, int(limit) << 20)+ reserve = max(_PIN_RESERVE_MIN_BYTES, int(total * _PIN_RESERVE_FRACTION))+ if available - need < reserve:+ if logger is not None:+ logger.info(+ "diffusion.memory: not pinning %s (%.1f GiB): %.1f GiB of host RAM available",+ type(module).__name__,+ need / 2**30,+ available / 2**30,+ )+ return 0+ bufs: list = []+ placed: list = []+ try:+ for (name, p), (index, start) in zip(params, slots):+ if index == len(bufs):+ bufs.append(torch.empty(chunks[index], dtype = torch.uint8, pin_memory = True))+ view = bufs[index][start : start + p.data.nbytes].view(p.dtype).view(p.shape)+ view.copy_(p.data)+ placed.append((name, p, view))+ except Exception as exc: # noqa: BLE001 - e.g. a WSL pinned-memory cap: keep the pageable weights+ bufs.clear()+ placed.clear()+ view = None+ _host_empty_cache()+ if logger is not None:+ logger.info(+ "diffusion.memory: pinning %s failed (%s); weights stay pageable",+ type(module).__name__,+ exc,+ )+ return 0+ for name, p, view in placed:+ p.data = view+ host[name] = view+ return need+++def _wrap_cpu_offload_hook(+ hook: Any,+ module: Any,+ logger: Any = None,+) -> None:# Keyed by name: a move can hand the module new Parameter objects, and a stale key must miss, not alias.state = module.__dict__.get(_KEEP_ATTR)if state is None:@@ -228,6 +354,19 @@ def _wrap_cpu_offload_hook(hook: Any, module: Any) -> None:):host.pop(name, None)owner.pop(name, None)+ if onload and not state.get("pin_tried"):+ # Once per module, on its first onload, so loading pays nothing.+ state["pin_tried"] = True+ try:+ pinned = _pin_host_weights(mod, host, logger)+ except Exception: # noqa: BLE001 - pinning is an optimisation, never a failure+ pinned = 0+ if pinned and logger is not None:+ logger.info(+ "diffusion.memory: pinned %.1f GiB of %s host weights",+ pinned / 2**30,+ type(mod).__name__,+ )out = pre_forward(mod, *args, **kwargs)if onload:for name, p in mod.named_parameters():@@ -278,6 +417,7 @@ def reclaim_host_memory(logger: Any = None) -> bool:passreturn Falsereclaim()+ _host_empty_cache()return Trueexcept Exception as exc: # noqa: BLE001if logger is not None and not _host_memory_reclaim_warning_logged:diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex df78c1001bb..733a214e87b 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -82,6 +82,7 @@ from .diffusion_memory import (raise_on_unified_memory_shortfall,reclaim_host_memory,reclaim_offload_host_memory,+ release_pinned_host_memory,settled_snapshot_device_memory,)from .diffusion_torchao_patches import install_torchao_int_mm_patch@@ -6906,7 +6907,11 @@ class VideoBackend:getattr(getattr(state, "pipe", None), "_unsloth_cuda_graphs", ()) or ())del state- clear_gpu_cache()+ try:+ clear_gpu_cache()+ finally:+ # finally: a sticky CUDA fault must not keep the pinned chunks locked.+ release_pinned_host_memory()reclaim_host_memory(logger = logger)def unload(self, *, expected_account: Optional[str] = None) -> dict[str, Any]:diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.pyindex afe0ee09629..3c193525f4b 100644--- a/studio/backend/tests/test_diffusion_backend.py+++ b/studio/backend/tests/test_diffusion_backend.py@@ -11884,6 +11884,45 @@ def test_a_prequant_repo_missing_its_artifact_marks_the_plan_incomplete(monkeypaassert "prequant artifact missing" in str(failures[0])+def test_unload_drains_pinned_host_memory_after_the_pipeline_is_gone(+ fake_runtime, tmp_path, monkeypatch+):+ from core.inference import diffusion as diffusion_module++ backend = _loaded_backend(tmp_path)+ calls: list = []+ monkeypatch.setattr(+ diffusion_module, "clear_gpu_cache", lambda: calls.append(("clear", backend._state))+ )+ monkeypatch.setattr(+ diffusion_module,+ "release_pinned_host_memory",+ lambda: calls.append(("host", backend._state)),+ )+ backend.unload()+ assert calls == [("clear", None), ("host", None)]+++def test_unload_drains_pinned_host_memory_even_when_gpu_cleanup_raises(+ fake_runtime, tmp_path, monkeypatch+):+ from core.inference import diffusion as diffusion_module++ backend = _loaded_backend(tmp_path)+ drained: list = []++ def _sticky():+ raise RuntimeError("CUDA error: an illegal memory access was encountered")++ monkeypatch.setattr(diffusion_module, "clear_gpu_cache", _sticky)+ monkeypatch.setattr(+ diffusion_module, "release_pinned_host_memory", lambda: drained.append(True)+ )+ with pytest.raises(RuntimeError, match = "illegal memory access"):+ backend.unload()+ assert drained == [True]++def test_status_reports_cuda_graph_off_once_every_armed_step_ran_eager():backend = DiffusionBackend()handle = types.SimpleNamespace(diff --git a/studio/backend/tests/test_diffusion_offload_keep_cpu.py b/studio/backend/tests/test_diffusion_offload_keep_cpu.pyindex 57cc0e18454..af236c07451 100644--- a/studio/backend/tests/test_diffusion_offload_keep_cpu.py+++ b/studio/backend/tests/test_diffusion_offload_keep_cpu.py@@ -131,14 +131,18 @@ cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDAdef _render(pipe, x):- out = pipe.transformer(pipe.text_encoder(x)).detach().cpu()+ out = x+ for module in pipe.components.values():+ out = module(out)+ out = out.detach().cpu()pipe.transformer._hf_hook.init_hook(pipe.transformer)pipe.enable_model_cpu_offload()return out@cuda-def test_real_offload_keeps_host_storage_and_is_bit_identical():+def test_real_offload_keeps_host_storage_and_is_bit_identical(monkeypatch):+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "0")x = torch.randn(4, 8)stock = _pipe("cuda", "text_encoder", "transformer")stock.enable_model_cpu_offload()@@ -166,6 +170,60 @@ def test_a_weight_written_on_the_device_is_copied_back():assert torch.equal(pipe.transformer.weight.detach(), before + 1)+@cuda+def test_weights_are_pinned_on_first_onload_into_shared_chunks(monkeypatch):+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")+ x = torch.randn(4, 8)+ stock = _pipe("cuda", "text_encoder", "transformer")+ stock.enable_model_cpu_offload()+ ref = _render(stock, x)+ pipe = _pipe("cuda", "text_encoder", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ assert not pipe.transformer.weight.is_pinned() # nothing locked at load+ for _ in range(2):+ assert torch.equal(_render(pipe, x), ref)+ for module in pipe.components.values():+ assert all(p.is_pinned() for p in module.parameters())+ assert (+ module.weight.untyped_storage().data_ptr() == module.bias.untyped_storage().data_ptr()+ )+++@cuda+def test_pinning_is_off_by_switch_and_when_ram_is_short(monkeypatch):+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "0")+ pipe = _pipe("cuda", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ _render(pipe, torch.randn(4, 8))+ assert not pipe.transformer.weight.is_pinned()++ psutil = pytest.importorskip("psutil")+ monkeypatch.delenv(dm.OFFLOAD_PIN_ENV)+ short = types.SimpleNamespace(total = 16 << 30, available = 2 << 30)+ monkeypatch.setattr(psutil, "virtual_memory", lambda: short)+ pipe = _pipe("cuda", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ _render(pipe, torch.randn(4, 8))+ assert not pipe.transformer.weight.is_pinned()+++@cuda+def test_a_non_contiguous_weight_keeps_its_layout(monkeypatch):+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")+ pipe = _pipe("cuda", "transformer")+ lin = pipe.transformer+ lin.weight = torch.nn.Parameter(lin.weight.detach().t().contiguous().t()) # transposed strides+ assert not lin.weight.is_contiguous()+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ _render(pipe, torch.randn(4, 8))+ assert not lin.weight.is_pinned() and lin.bias.is_pinned()+ assert lin.weight.stride() == (1, 8)++@cudadef test_a_parameter_replaced_on_the_device_is_copied_back():pipe = _pipe("cuda", "transformer")@@ -185,6 +243,54 @@ def test_a_parameter_replaced_on_the_device_is_copied_back():assert torch.equal(pipe.transformer.weight.detach(), torch.full((8, 8), 3.0))+def _host_of(module):+ return {name: p.data for name, p in module.named_parameters()}+++@cuda+def test_the_ram_gate_counts_the_chunks_really_allocated(monkeypatch):+ psutil = pytest.importorskip("psutil")+ # Weight 256 B and bias 32 B (256 aligned) do not share a 384 B chunk: 384 + 256 B are allocated for 512.+ monkeypatch.setattr(dm, "_PIN_CHUNK_BYTES", 384)+ reserve = 4 << 30+ lin = torch.nn.Linear(8, 8)+ monkeypatch.setattr(+ psutil,+ "virtual_memory",+ lambda: types.SimpleNamespace(total = 16 << 30, available = reserve + 600),+ )+ assert dm._pin_host_weights(lin, _host_of(lin)) == 0+ assert not lin.weight.is_pinned()+ monkeypatch.setattr(+ psutil,+ "virtual_memory",+ lambda: types.SimpleNamespace(total = 16 << 30, available = reserve + 640),+ )+ assert dm._pin_host_weights(lin, _host_of(lin)) == 640+ assert lin.weight.is_pinned() and lin.bias.is_pinned()+++@cuda+def test_a_failed_pin_hands_the_partial_chunks_back(monkeypatch):+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")+ monkeypatch.setattr(dm, "_PIN_CHUNK_BYTES", 384)+ real_empty, calls, emptied = torch.empty, [], []++ def empty(*args, **kwargs):+ calls.append(args)+ if len(calls) == 2:+ raise RuntimeError("cudaHostAlloc failed")+ return real_empty(*args, **kwargs)++ lin = torch.nn.Linear(8, 8)+ host = _host_of(lin)+ monkeypatch.setattr(torch, "empty", empty)+ monkeypatch.setattr(dm, "_host_empty_cache", lambda: emptied.append(True))+ assert dm._pin_host_weights(lin, host) == 0+ assert len(calls) == 2 and emptied+ assert not lin.weight.is_pinned() and not lin.bias.is_pinned()++@cuda@pytest.mark.parametrize("how", ["parameter", "data"])def test_a_parameter_replaced_while_offloaded_is_not_restored_to_the_old_weight(how):@@ -220,3 +326,77 @@ def test_a_weight_reassigned_through_data_on_the_device_is_copied_back():pipe.transformer._hf_hook.init_hook(pipe.transformer)assert pipe.transformer.weight.device.type == "cpu"assert torch.equal(pipe.transformer.weight.detach(), torch.full((8, 8), 7.0))+++def _fake_cgroup(+ tmp_path,+ monkeypatch,+ limit,+ current,+ version = 2,+):+ import core.inference.llama_cpp as llama_cpp_module++ root = tmp_path / "cgroup"+ leaf = root / ("studio.slice" if version == 2 else "memory/studio.slice")+ leaf.mkdir(parents = True)+ names = (+ ("memory.max", "memory.current")+ if version == 2+ else ("memory.limit_in_bytes", "memory.usage_in_bytes")+ )+ if limit is not None:+ (leaf / names[0]).write_text(str(limit), encoding = "utf-8")+ (leaf / names[1]).write_text(str(current), encoding = "utf-8")+ proc = tmp_path / "self.cgroup"+ line = "0::/studio.slice" if version == 2 else "4:memory:/studio.slice"+ proc.write_text(line + "\n", encoding = "utf-8")+ monkeypatch.setattr(llama_cpp_module, "_CGROUP_ROOT", str(root))+ monkeypatch.setattr(llama_cpp_module, "_PROC_SELF_CGROUP", str(proc))+++ "version, limit, current, host_available, pinned",+ [+ # 64 GiB container on a 512 GiB host, 10 GiB left in it: the host reading would pin 16 GiB.+ (2, 64 << 30, 54 << 30, 400 << 30, False),+ (1, 64 << 30, 54 << 30, 400 << 30, False),+ # 40 GiB left: fits over a reserve of 15% of the container; 15% of the host would refuse it.+ (2, 64 << 30, 24 << 30, 400 << 30, True),+ (1, 64 << 30, 24 << 30, 400 << 30, True),+ # 24 GiB left: 8 GiB would remain, under the container's 9.6 GiB reserve.+ (2, 64 << 30, 40 << 30, 400 << 30, False),+ (2, 1 << 50, 0, 400 << 30, True),+ (2, 1 << 50, 0, 90 << 30, False),+ (2, None, 0, 400 << 30, True),+ (2, None, 0, 90 << 30, False),+ ],+)+def test_the_ram_gate_is_sized_from_the_container(+ tmp_path, monkeypatch, version, limit, current, host_available, pinned+):+ psutil = pytest.importorskip("psutil")+ _fake_cgroup(tmp_path, monkeypatch, limit, current, version)+ monkeypatch.setattr(+ psutil,+ "virtual_memory",+ lambda: types.SimpleNamespace(total = 512 << 30, available = host_available),+ )+ # 16 GiB (power of two, no rounding): the gate refuses before any allocation.+ data = types.SimpleNamespace(+ nbytes = 16 << 30, is_contiguous = lambda: True, is_pinned = lambda: False+ )+ weight = types.SimpleNamespace(device = torch.device("cpu"), data = data)+ module = types.SimpleNamespace(named_parameters = lambda: [("weight", weight)])+ allocated = []++ def empty(*args, **kwargs):+ allocated.append(args)+ raise RuntimeError("not allocating in a test")++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(dm, "_keepable", lambda p: True)+ monkeypatch.setattr(dm, "_host_empty_cache", lambda: None)+ monkeypatch.setattr(torch, "empty", empty)+ assert dm._pin_host_weights(module, {"weight": data}) == 0+ assert bool(allocated) is pinneddiff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.pyindex 0f6b82d9c56..1d644a25eb3 100644--- a/studio/backend/tests/test_video_backend.py+++ b/studio/backend/tests/test_video_backend.py@@ -9798,6 +9798,43 @@ def test_the_boundary_marker_waits_out_a_busy_capture_lock(fake_runtime, monkeypassert at_decode.get("phase") == "decode"+def test_teardown_drains_pinned_host_memory_after_the_pipeline_is_gone(+ fake_runtime, tmp_path, monkeypatch+):+ from core.inference import video as video_mod++ backend = VideoBackend()+ _load_gguf(backend, tmp_path)+ calls: list = []+ monkeypatch.setattr(+ video_mod, "clear_gpu_cache", lambda: calls.append(("clear", backend._state))+ )+ monkeypatch.setattr(+ video_mod, "release_pinned_host_memory", lambda: calls.append(("host", backend._state))+ )+ backend.unload()+ assert calls == [("clear", None), ("host", None)]+++def test_teardown_drains_pinned_host_memory_even_when_gpu_cleanup_raises(+ fake_runtime, tmp_path, monkeypatch+):+ from core.inference import video as video_mod++ backend = VideoBackend()+ _load_gguf(backend, tmp_path)+ drained: list = []++ def _sticky():+ raise RuntimeError("CUDA error: an illegal memory access was encountered")++ monkeypatch.setattr(video_mod, "clear_gpu_cache", _sticky)+ monkeypatch.setattr(video_mod, "release_pinned_host_memory", lambda: drained.append(True))+ with pytest.raises(RuntimeError, match = "illegal memory access"):+ backend.unload()+ assert drained == [True]++def test_video_status_reports_cuda_graph_off_once_every_armed_step_ran_eager(fake_runtime):backend = VideoBackend()backend.load_pipeline("Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline")