Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep GGUF weights and the last-run module on their host copy under whole-model offload (#12022)

Daniel Han75323d58df02
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index e714658e97e..7e66f32d259 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -130,6 +130,8 @@ def keep_cpu_weights_on_offload(pipe: Any, logger: Any = None) -> int:
if callable(enable) and not getattr(enable, _KEEP_ATTR, False):
def _enable(*args: Any, **kwargs: Any) -> Any:
+ # The re-enable after every call starts with pipe.to("cpu"), which would copy back the module run last.
+ _offload_through_kept_hooks(pipe, CpuOffload, logger)
out = enable(*args, **kwargs)
_wrap_cpu_offload_hooks(pipe, CpuOffload, logger)
return out
@@ -142,9 +144,43 @@ def keep_cpu_weights_on_offload(pipe: Any, logger: Any = None) -> int:
return _wrap_cpu_offload_hooks(pipe, CpuOffload, logger)
+def _offload_through_kept_hooks(
+ pipe: Any,
+ hook_cls: type,
+ logger: Any = None,
+) -> None:
+ components = getattr(pipe, "components", None) or {}
+ for module in components.values():
+ hook = getattr(module, "_hf_hook", None)
+ if not isinstance(hook, hook_cls) or not getattr(hook, _KEEP_ATTR, False):
+ continue
+ try:
+ onloaded = [*module.parameters(), *(b for _, b in _named_buffers(module))]
+ if any(t.device.type != "cpu" for t in onloaded):
+ hook.init_hook(module)
+ except Exception as exc: # noqa: BLE001 - the stock re-enable still moves it
+ if logger is not None:
+ logger.debug(
+ "diffusion.memory: kept offload of %s before re-enable failed: %s",
+ type(module).__name__,
+ exc,
+ )
+
+
+def _gguf_parameter_class() -> Optional[type]:
+ module = sys.modules.get("diffusers.quantizers.gguf.utils")
+ return getattr(module, "GGUFParameter", None) if module is not None else None
+
+
def _keepable(param: Any) -> bool:
import torch
- return type(param.data) is torch.Tensor
+
+ data = param.data
+ if type(data) is torch.Tensor:
+ return True
+ # GGUF weights are packed bytes; the quant type lives on the Parameter, which the swap never replaces.
+ gguf = _gguf_parameter_class()
+ return gguf is not None and type(param) is gguf and type(data) is gguf
def _wrap_cpu_offload_hooks(
@@ -408,7 +444,7 @@ def _wrap_cpu_offload_hook(
return out
def _pre_forward(mod: Any, *args: Any, **kwargs: Any) -> Any:
- # `host` gate: an all-subclass module (GGUF, torchao) never fills `version`, so would rescan every forward.
+ # `host` gate: an all-subclass module (torchao) never fills `version`, so would rescan every forward.
onload = not version and not buffer_version and bool(host or buffer_host)
if onload:
for name, p in mod.named_parameters():
diff --git a/studio/backend/tests/test_diffusion_offload_keep_cpu.py b/studio/backend/tests/test_diffusion_offload_keep_cpu.py
index 7bef7fecddd..7a71b410572 100644
--- a/studio/backend/tests/test_diffusion_offload_keep_cpu.py
+++ b/studio/backend/tests/test_diffusion_offload_keep_cpu.py
@@ -92,6 +92,42 @@ def test_subclass_weights_are_not_kept():
assert not dm._keepable(lin.weight)
+def _gguf_utils():
+ pytest.importorskip("gguf")
+ return pytest.importorskip("diffusers.quantizers.gguf.utils")
+
+
+def _gguf_linear(
+ utils,
+ out_features = 8,
+ in_features = 64,
+):
+ """A diffusers GGUFLinear holding a real Q8_0 weight: per 32-value block, an fp16 scale then 32 int8 codes."""
+ import numpy as np
+
+ rng = np.random.default_rng(0)
+ blocks = out_features * in_features // 32
+ scale = np.full((blocks, 1), 0.01, dtype = np.float16).view(np.uint8)
+ codes = rng.integers(-127, 128, size = (blocks, 32), dtype = np.int8).view(np.uint8)
+ raw = torch.from_numpy(np.concatenate([scale, codes], axis = 1).reshape(out_features, -1).copy())
+ lin = utils.GGUFLinear(in_features, out_features, bias = True, compute_dtype = torch.float32)
+ lin.weight = utils.GGUFParameter(raw, quant_type = utils.gguf.GGMLQuantizationType.Q8_0)
+ return lin
+
+
+def test_gguf_weights_are_kept_but_other_subclasses_are_not():
+ utils = _gguf_utils()
+ lin = _gguf_linear(utils)
+ assert type(lin.weight) is utils.GGUFParameter
+ assert dm._keepable(lin.weight) and dm._keepable(lin.bias)
+
+ class Sub(torch.Tensor):
+ pass
+
+ other = torch.nn.Parameter(torch.zeros(2).as_subclass(Sub))
+ assert not dm._keepable(other)
+
+
def test_an_all_subclass_module_is_not_rescanned_every_forward():
class Sub(torch.Tensor):
pass
@@ -402,6 +438,25 @@ def test_the_ram_gate_is_sized_from_the_container(
assert bool(allocated) is pinned
+def _gguf_pipe(device):
+ utils = _gguf_utils()
+ pipe = _pipe(device, "text_encoder", "transformer", "vae")
+ comps = pipe.components
+ comps["text_encoder"] = pipe.text_encoder = torch.nn.Linear(8, 64)
+ comps["transformer"] = pipe.transformer = _gguf_linear(utils)
+ return pipe, utils
+
+
+def _run(pipe, x):
+ """One call the way a diffusers pipeline makes it: the last module stays onloaded until the re-enable."""
+ out = x
+ for module in pipe.components.values():
+ out = module(out)
+ out = out.detach().cpu()
+ pipe.enable_model_cpu_offload()
+ return out
+
+
class _BufferLinear(torch.nn.Module):
"""A weight held as plain buffers, as the torchao-free int8 linear stores it."""
@@ -426,6 +481,46 @@ def _buffer_ptrs(module):
return [b.data_ptr() for b in module.buffers()]
+@cuda
[email protected]("pin", ["0", "1"])
+def test_a_gguf_transformer_keeps_its_host_bytes_and_quant_type(monkeypatch, pin):
+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, pin)
+ x = torch.randn(4, 8)
+ stock, _ = _gguf_pipe("cuda")
+ stock.enable_model_cpu_offload()
+ ref = _run(stock, x)
+ pipe, utils = _gguf_pipe("cuda")
+ raw = pipe.transformer.weight.detach().clone()
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ _run(pipe, x) # the first onload pins
+ # Held, so a stock copy can never land back at a freed address and pass for the kept tensor.
+ held = [p.data for p in pipe.transformer.parameters()]
+ host = [t.data_ptr() for t in held]
+ for _ in range(3):
+ assert torch.equal(_run(pipe, x), ref)
+ weight = pipe.transformer.weight
+ assert weight.device.type == "cpu" and _ptrs(pipe.transformer) == host
+ assert type(weight) is utils.GGUFParameter
+ assert weight.quant_type == utils.gguf.GGMLQuantizationType.Q8_0
+ assert torch.equal(weight.detach().view(torch.uint8), raw.view(torch.uint8))
+ assert weight.is_pinned() is (pin == "1")
+
+
+@cuda
+def test_the_module_left_onloaded_by_a_call_is_offloaded_through_its_kept_hook(monkeypatch):
+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "0")
+ pipe = _pipe("cuda", "text_encoder", "transformer", "vae")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ held = {n: [p.data for p in m.parameters()] for n, m in pipe.components.items()}
+ host = {n: [t.data_ptr() for t in ts] for n, ts in held.items()}
+ for _ in range(3):
+ _run(pipe, torch.randn(4, 8))
+ assert pipe.vae.weight.device.type == "cpu"
+ assert {n: _ptrs(m) for n, m in pipe.components.items()} == host
+
+
@cuda
@pytest.mark.parametrize("pin", ["0", "1"])
def test_buffer_weights_keep_their_host_storage_and_are_pinned(monkeypatch, pin):