Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: pin the kept offload weights on first onload when host RAM allows (#11766)

Daniel Hanb0af0ea684dd
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 120e882e041..dcaaf55c57d 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -114,6 +114,7 @@ from .diffusion_memory import (
reclaimable_snapshot_device_memory,
reclaim_host_memory,
reclaim_offload_host_memory,
+ release_pinned_host_memory,
refine_memory_plan_for_components,
settled_snapshot_device_memory,
snapshot_device_memory,
@@ -7846,7 +7847,11 @@ class DiffusionBackend:
self._cn_models.clear()
self._state = None
del state
- clear_gpu_cache()
+ try:
+ clear_gpu_cache()
+ finally:
+ # finally: a sticky CUDA fault must not keep the pinned chunks locked.
+ release_pinned_host_memory()
# Must follow clear_gpu_cache() (runs gc) so the freed staging buffers can be returned.
reclaim_host_memory(logger = logger)
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index a510504e61c..4c0e2a806ac 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -159,7 +159,7 @@ def _wrap_cpu_offload_hooks(
if not isinstance(hook, hook_cls) or getattr(hook, _KEEP_ATTR, False):
continue
try:
- _wrap_cpu_offload_hook(hook, module)
+ _wrap_cpu_offload_hook(hook, module, logger)
wrapped += 1
except Exception as exc: # noqa: BLE001 - that module keeps the stock copy
if logger is not None:
@@ -171,7 +171,133 @@ def _wrap_cpu_offload_hooks(
return wrapped
-def _wrap_cpu_offload_hook(hook: Any, module: Any) -> None:
+OFFLOAD_PIN_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_PIN"
+# The host allocator rounds every block up to a power of two, so weights share chunks instead of one pin each.
+_PIN_CHUNK_BYTES = 1 << 30
+_PIN_ALIGN = 256
+_PIN_RESERVE_MIN_BYTES = 4 << 30
+_PIN_RESERVE_FRACTION = 0.15
+
+
+def _pin_mode() -> str:
+ raw = (os.environ.get(OFFLOAD_PIN_ENV) or "").strip().lower()
+ if raw in ("0", "off", "false", "no"):
+ return "off"
+ if raw in ("1", "on", "true", "yes", "force"):
+ return "on"
+ return "auto"
+
+
+def _pow2_ceil(n: int) -> int:
+ return 1 << max(0, int(n) - 1).bit_length()
+
+
+def release_pinned_host_memory() -> None:
+ _host_empty_cache()
+
+
+def _host_empty_cache() -> None:
+ try:
+ import torch
+ empty = getattr(torch._C, "_host_emptyCache", None)
+ if callable(empty) and torch.cuda.is_available():
+ empty()
+ except Exception: # noqa: BLE001
+ pass
+
+
+def _pin_host_weights(
+ module: Any,
+ host: dict,
+ logger: Any = None,
+) -> int:
+ """Pack kept host weights into page-locked chunks; returns bytes pinned. Contiguous only, so channels_last survives."""
+ import torch
+
+ mode = _pin_mode()
+ if mode == "off" or not torch.cuda.is_available():
+ return 0
+ params = [
+ (name, p)
+ for name, p in module.named_parameters()
+ if host.get(name) is not None
+ and p.device.type == "cpu"
+ and _keepable(p)
+ and p.data.is_contiguous()
+ and not p.data.is_pinned()
+ ]
+ sizes = [-(-p.data.nbytes // _PIN_ALIGN) * _PIN_ALIGN for _, p in params]
+ if not sizes or sum(sizes) == 0:
+ return 0
+ # Lay out chunks first so the RAM gate counts chunk tails and last-chunk rounding, not just weight bytes.
+ chunk = max(_PIN_CHUNK_BYTES, _pow2_ceil(max(sizes)))
+ chunks: list = []
+ slots: list = []
+ off, left = 0, sum(sizes)
+ for size in sizes:
+ if not chunks or off + size > chunks[-1]:
+ chunks.append(chunk if left >= chunk else _pow2_ceil(left))
+ off = 0
+ slots.append((len(chunks) - 1, off))
+ off += size
+ left -= size
+ need = sum(chunks)
+ if mode == "auto":
+ try:
+ import psutil
+ vm = psutil.virtual_memory()
+ except Exception: # noqa: BLE001 - no way to size it, so do not lock memory
+ return 0
+ # psutil reads the host; pinned pages are charged to an enforcing cgroup, so size from the container.
+ available, total = vm.available, vm.total
+ remainder = _cgroup_available_memory_mib()
+ if remainder is not None:
+ available = min(available, int(remainder) << 20)
+ limit = _cgroup_memory_limit_mib()
+ if limit is not None:
+ total = min(total, int(limit) << 20)
+ reserve = max(_PIN_RESERVE_MIN_BYTES, int(total * _PIN_RESERVE_FRACTION))
+ if available - need < reserve:
+ if logger is not None:
+ logger.info(
+ "diffusion.memory: not pinning %s (%.1f GiB): %.1f GiB of host RAM available",
+ type(module).__name__,
+ need / 2**30,
+ available / 2**30,
+ )
+ return 0
+ bufs: list = []
+ placed: list = []
+ try:
+ for (name, p), (index, start) in zip(params, slots):
+ if index == len(bufs):
+ bufs.append(torch.empty(chunks[index], dtype = torch.uint8, pin_memory = True))
+ view = bufs[index][start : start + p.data.nbytes].view(p.dtype).view(p.shape)
+ view.copy_(p.data)
+ placed.append((name, p, view))
+ except Exception as exc: # noqa: BLE001 - e.g. a WSL pinned-memory cap: keep the pageable weights
+ bufs.clear()
+ placed.clear()
+ view = None
+ _host_empty_cache()
+ if logger is not None:
+ logger.info(
+ "diffusion.memory: pinning %s failed (%s); weights stay pageable",
+ type(module).__name__,
+ exc,
+ )
+ return 0
+ for name, p, view in placed:
+ p.data = view
+ host[name] = view
+ return need
+
+
+def _wrap_cpu_offload_hook(
+ hook: Any,
+ module: Any,
+ logger: Any = None,
+) -> None:
# Keyed by name: a move can hand the module new Parameter objects, and a stale key must miss, not alias.
state = module.__dict__.get(_KEEP_ATTR)
if state is None:
@@ -228,6 +354,19 @@ def _wrap_cpu_offload_hook(hook: Any, module: Any) -> None:
):
host.pop(name, None)
owner.pop(name, None)
+ if onload and not state.get("pin_tried"):
+ # Once per module, on its first onload, so loading pays nothing.
+ state["pin_tried"] = True
+ try:
+ pinned = _pin_host_weights(mod, host, logger)
+ except Exception: # noqa: BLE001 - pinning is an optimisation, never a failure
+ pinned = 0
+ if pinned and logger is not None:
+ logger.info(
+ "diffusion.memory: pinned %.1f GiB of %s host weights",
+ pinned / 2**30,
+ type(mod).__name__,
+ )
out = pre_forward(mod, *args, **kwargs)
if onload:
for name, p in mod.named_parameters():
@@ -278,6 +417,7 @@ def reclaim_host_memory(logger: Any = None) -> bool:
pass
return False
reclaim()
+ _host_empty_cache()
return True
except Exception as exc: # noqa: BLE001
if logger is not None and not _host_memory_reclaim_warning_logged:
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index df78c1001bb..733a214e87b 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -82,6 +82,7 @@ from .diffusion_memory import (
raise_on_unified_memory_shortfall,
reclaim_host_memory,
reclaim_offload_host_memory,
+ release_pinned_host_memory,
settled_snapshot_device_memory,
)
from .diffusion_torchao_patches import install_torchao_int_mm_patch
@@ -6906,7 +6907,11 @@ class VideoBackend:
getattr(getattr(state, "pipe", None), "_unsloth_cuda_graphs", ()) or ()
)
del state
- clear_gpu_cache()
+ try:
+ clear_gpu_cache()
+ finally:
+ # finally: a sticky CUDA fault must not keep the pinned chunks locked.
+ release_pinned_host_memory()
reclaim_host_memory(logger = logger)
def unload(self, *, expected_account: Optional[str] = None) -> dict[str, Any]:
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index afe0ee09629..3c193525f4b 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -11884,6 +11884,45 @@ def test_a_prequant_repo_missing_its_artifact_marks_the_plan_incomplete(monkeypa
assert "prequant artifact missing" in str(failures[0])
+def test_unload_drains_pinned_host_memory_after_the_pipeline_is_gone(
+ fake_runtime, tmp_path, monkeypatch
+):
+ from core.inference import diffusion as diffusion_module
+
+ backend = _loaded_backend(tmp_path)
+ calls: list = []
+ monkeypatch.setattr(
+ diffusion_module, "clear_gpu_cache", lambda: calls.append(("clear", backend._state))
+ )
+ monkeypatch.setattr(
+ diffusion_module,
+ "release_pinned_host_memory",
+ lambda: calls.append(("host", backend._state)),
+ )
+ backend.unload()
+ assert calls == [("clear", None), ("host", None)]
+
+
+def test_unload_drains_pinned_host_memory_even_when_gpu_cleanup_raises(
+ fake_runtime, tmp_path, monkeypatch
+):
+ from core.inference import diffusion as diffusion_module
+
+ backend = _loaded_backend(tmp_path)
+ drained: list = []
+
+ def _sticky():
+ raise RuntimeError("CUDA error: an illegal memory access was encountered")
+
+ monkeypatch.setattr(diffusion_module, "clear_gpu_cache", _sticky)
+ monkeypatch.setattr(
+ diffusion_module, "release_pinned_host_memory", lambda: drained.append(True)
+ )
+ with pytest.raises(RuntimeError, match = "illegal memory access"):
+ backend.unload()
+ assert drained == [True]
+
+
def test_status_reports_cuda_graph_off_once_every_armed_step_ran_eager():
backend = DiffusionBackend()
handle = types.SimpleNamespace(
diff --git a/studio/backend/tests/test_diffusion_offload_keep_cpu.py b/studio/backend/tests/test_diffusion_offload_keep_cpu.py
index 57cc0e18454..af236c07451 100644
--- a/studio/backend/tests/test_diffusion_offload_keep_cpu.py
+++ b/studio/backend/tests/test_diffusion_offload_keep_cpu.py
@@ -131,14 +131,18 @@ cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA
def _render(pipe, x):
- out = pipe.transformer(pipe.text_encoder(x)).detach().cpu()
+ out = x
+ for module in pipe.components.values():
+ out = module(out)
+ out = out.detach().cpu()
pipe.transformer._hf_hook.init_hook(pipe.transformer)
pipe.enable_model_cpu_offload()
return out
@cuda
-def test_real_offload_keeps_host_storage_and_is_bit_identical():
+def test_real_offload_keeps_host_storage_and_is_bit_identical(monkeypatch):
+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "0")
x = torch.randn(4, 8)
stock = _pipe("cuda", "text_encoder", "transformer")
stock.enable_model_cpu_offload()
@@ -166,6 +170,60 @@ def test_a_weight_written_on_the_device_is_copied_back():
assert torch.equal(pipe.transformer.weight.detach(), before + 1)
+@cuda
+def test_weights_are_pinned_on_first_onload_into_shared_chunks(monkeypatch):
+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")
+ x = torch.randn(4, 8)
+ stock = _pipe("cuda", "text_encoder", "transformer")
+ stock.enable_model_cpu_offload()
+ ref = _render(stock, x)
+ pipe = _pipe("cuda", "text_encoder", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ assert not pipe.transformer.weight.is_pinned() # nothing locked at load
+ for _ in range(2):
+ assert torch.equal(_render(pipe, x), ref)
+ for module in pipe.components.values():
+ assert all(p.is_pinned() for p in module.parameters())
+ assert (
+ module.weight.untyped_storage().data_ptr() == module.bias.untyped_storage().data_ptr()
+ )
+
+
+@cuda
+def test_pinning_is_off_by_switch_and_when_ram_is_short(monkeypatch):
+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "0")
+ pipe = _pipe("cuda", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ _render(pipe, torch.randn(4, 8))
+ assert not pipe.transformer.weight.is_pinned()
+
+ psutil = pytest.importorskip("psutil")
+ monkeypatch.delenv(dm.OFFLOAD_PIN_ENV)
+ short = types.SimpleNamespace(total = 16 << 30, available = 2 << 30)
+ monkeypatch.setattr(psutil, "virtual_memory", lambda: short)
+ pipe = _pipe("cuda", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ _render(pipe, torch.randn(4, 8))
+ assert not pipe.transformer.weight.is_pinned()
+
+
+@cuda
+def test_a_non_contiguous_weight_keeps_its_layout(monkeypatch):
+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")
+ pipe = _pipe("cuda", "transformer")
+ lin = pipe.transformer
+ lin.weight = torch.nn.Parameter(lin.weight.detach().t().contiguous().t()) # transposed strides
+ assert not lin.weight.is_contiguous()
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ _render(pipe, torch.randn(4, 8))
+ assert not lin.weight.is_pinned() and lin.bias.is_pinned()
+ assert lin.weight.stride() == (1, 8)
+
+
@cuda
def test_a_parameter_replaced_on_the_device_is_copied_back():
pipe = _pipe("cuda", "transformer")
@@ -185,6 +243,54 @@ def test_a_parameter_replaced_on_the_device_is_copied_back():
assert torch.equal(pipe.transformer.weight.detach(), torch.full((8, 8), 3.0))
+def _host_of(module):
+ return {name: p.data for name, p in module.named_parameters()}
+
+
+@cuda
+def test_the_ram_gate_counts_the_chunks_really_allocated(monkeypatch):
+ psutil = pytest.importorskip("psutil")
+ # Weight 256 B and bias 32 B (256 aligned) do not share a 384 B chunk: 384 + 256 B are allocated for 512.
+ monkeypatch.setattr(dm, "_PIN_CHUNK_BYTES", 384)
+ reserve = 4 << 30
+ lin = torch.nn.Linear(8, 8)
+ monkeypatch.setattr(
+ psutil,
+ "virtual_memory",
+ lambda: types.SimpleNamespace(total = 16 << 30, available = reserve + 600),
+ )
+ assert dm._pin_host_weights(lin, _host_of(lin)) == 0
+ assert not lin.weight.is_pinned()
+ monkeypatch.setattr(
+ psutil,
+ "virtual_memory",
+ lambda: types.SimpleNamespace(total = 16 << 30, available = reserve + 640),
+ )
+ assert dm._pin_host_weights(lin, _host_of(lin)) == 640
+ assert lin.weight.is_pinned() and lin.bias.is_pinned()
+
+
+@cuda
+def test_a_failed_pin_hands_the_partial_chunks_back(monkeypatch):
+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")
+ monkeypatch.setattr(dm, "_PIN_CHUNK_BYTES", 384)
+ real_empty, calls, emptied = torch.empty, [], []
+
+ def empty(*args, **kwargs):
+ calls.append(args)
+ if len(calls) == 2:
+ raise RuntimeError("cudaHostAlloc failed")
+ return real_empty(*args, **kwargs)
+
+ lin = torch.nn.Linear(8, 8)
+ host = _host_of(lin)
+ monkeypatch.setattr(torch, "empty", empty)
+ monkeypatch.setattr(dm, "_host_empty_cache", lambda: emptied.append(True))
+ assert dm._pin_host_weights(lin, host) == 0
+ assert len(calls) == 2 and emptied
+ assert not lin.weight.is_pinned() and not lin.bias.is_pinned()
+
+
@cuda
@pytest.mark.parametrize("how", ["parameter", "data"])
def test_a_parameter_replaced_while_offloaded_is_not_restored_to_the_old_weight(how):
@@ -220,3 +326,77 @@ def test_a_weight_reassigned_through_data_on_the_device_is_copied_back():
pipe.transformer._hf_hook.init_hook(pipe.transformer)
assert pipe.transformer.weight.device.type == "cpu"
assert torch.equal(pipe.transformer.weight.detach(), torch.full((8, 8), 7.0))
+
+
+def _fake_cgroup(
+ tmp_path,
+ monkeypatch,
+ limit,
+ current,
+ version = 2,
+):
+ import core.inference.llama_cpp as llama_cpp_module
+
+ root = tmp_path / "cgroup"
+ leaf = root / ("studio.slice" if version == 2 else "memory/studio.slice")
+ leaf.mkdir(parents = True)
+ names = (
+ ("memory.max", "memory.current")
+ if version == 2
+ else ("memory.limit_in_bytes", "memory.usage_in_bytes")
+ )
+ if limit is not None:
+ (leaf / names[0]).write_text(str(limit), encoding = "utf-8")
+ (leaf / names[1]).write_text(str(current), encoding = "utf-8")
+ proc = tmp_path / "self.cgroup"
+ line = "0::/studio.slice" if version == 2 else "4:memory:/studio.slice"
+ proc.write_text(line + "\n", encoding = "utf-8")
+ monkeypatch.setattr(llama_cpp_module, "_CGROUP_ROOT", str(root))
+ monkeypatch.setattr(llama_cpp_module, "_PROC_SELF_CGROUP", str(proc))
+
+
+ "version, limit, current, host_available, pinned",
+ [
+ # 64 GiB container on a 512 GiB host, 10 GiB left in it: the host reading would pin 16 GiB.
+ (2, 64 << 30, 54 << 30, 400 << 30, False),
+ (1, 64 << 30, 54 << 30, 400 << 30, False),
+ # 40 GiB left: fits over a reserve of 15% of the container; 15% of the host would refuse it.
+ (2, 64 << 30, 24 << 30, 400 << 30, True),
+ (1, 64 << 30, 24 << 30, 400 << 30, True),
+ # 24 GiB left: 8 GiB would remain, under the container's 9.6 GiB reserve.
+ (2, 64 << 30, 40 << 30, 400 << 30, False),
+ (2, 1 << 50, 0, 400 << 30, True),
+ (2, 1 << 50, 0, 90 << 30, False),
+ (2, None, 0, 400 << 30, True),
+ (2, None, 0, 90 << 30, False),
+ ],
+)
+def test_the_ram_gate_is_sized_from_the_container(
+ tmp_path, monkeypatch, version, limit, current, host_available, pinned
+):
+ psutil = pytest.importorskip("psutil")
+ _fake_cgroup(tmp_path, monkeypatch, limit, current, version)
+ monkeypatch.setattr(
+ psutil,
+ "virtual_memory",
+ lambda: types.SimpleNamespace(total = 512 << 30, available = host_available),
+ )
+ # 16 GiB (power of two, no rounding): the gate refuses before any allocation.
+ data = types.SimpleNamespace(
+ nbytes = 16 << 30, is_contiguous = lambda: True, is_pinned = lambda: False
+ )
+ weight = types.SimpleNamespace(device = torch.device("cpu"), data = data)
+ module = types.SimpleNamespace(named_parameters = lambda: [("weight", weight)])
+ allocated = []
+
+ def empty(*args, **kwargs):
+ allocated.append(args)
+ raise RuntimeError("not allocating in a test")
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(dm, "_keepable", lambda p: True)
+ monkeypatch.setattr(dm, "_host_empty_cache", lambda: None)
+ monkeypatch.setattr(torch, "empty", empty)
+ assert dm._pin_host_weights(module, {"weight": data}) == 0
+ assert bool(allocated) is pinned
diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py
index 0f6b82d9c56..1d644a25eb3 100644
--- a/studio/backend/tests/test_video_backend.py
+++ b/studio/backend/tests/test_video_backend.py
@@ -9798,6 +9798,43 @@ def test_the_boundary_marker_waits_out_a_busy_capture_lock(fake_runtime, monkeyp
assert at_decode.get("phase") == "decode"
+def test_teardown_drains_pinned_host_memory_after_the_pipeline_is_gone(
+ fake_runtime, tmp_path, monkeypatch
+):
+ from core.inference import video as video_mod
+
+ backend = VideoBackend()
+ _load_gguf(backend, tmp_path)
+ calls: list = []
+ monkeypatch.setattr(
+ video_mod, "clear_gpu_cache", lambda: calls.append(("clear", backend._state))
+ )
+ monkeypatch.setattr(
+ video_mod, "release_pinned_host_memory", lambda: calls.append(("host", backend._state))
+ )
+ backend.unload()
+ assert calls == [("clear", None), ("host", None)]
+
+
+def test_teardown_drains_pinned_host_memory_even_when_gpu_cleanup_raises(
+ fake_runtime, tmp_path, monkeypatch
+):
+ from core.inference import video as video_mod
+
+ backend = VideoBackend()
+ _load_gguf(backend, tmp_path)
+ drained: list = []
+
+ def _sticky():
+ raise RuntimeError("CUDA error: an illegal memory access was encountered")
+
+ monkeypatch.setattr(video_mod, "clear_gpu_cache", _sticky)
+ monkeypatch.setattr(video_mod, "release_pinned_host_memory", lambda: drained.append(True))
+ with pytest.raises(RuntimeError, match = "illegal memory access"):
+ backend.unload()
+ assert drained == [True]
+
+
def test_video_status_reports_cuda_graph_off_once_every_armed_step_ran_eager(fake_runtime):
backend = VideoBackend()
backend.load_pipeline("Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline")