Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep GGUF weights and the last-run module on their host copy under whole-model offload (#12022)
Daniel Han75323d58df02
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex e714658e97e..7e66f32d259 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -130,6 +130,8 @@ def keep_cpu_weights_on_offload(pipe: Any, logger: Any = None) -> int:if callable(enable) and not getattr(enable, _KEEP_ATTR, False):def _enable(*args: Any, **kwargs: Any) -> Any:+ # The re-enable after every call starts with pipe.to("cpu"), which would copy back the module run last.+ _offload_through_kept_hooks(pipe, CpuOffload, logger)out = enable(*args, **kwargs)_wrap_cpu_offload_hooks(pipe, CpuOffload, logger)return out@@ -142,9 +144,43 @@ def keep_cpu_weights_on_offload(pipe: Any, logger: Any = None) -> int:return _wrap_cpu_offload_hooks(pipe, CpuOffload, logger)+def _offload_through_kept_hooks(+ pipe: Any,+ hook_cls: type,+ logger: Any = None,+) -> None:+ components = getattr(pipe, "components", None) or {}+ for module in components.values():+ hook = getattr(module, "_hf_hook", None)+ if not isinstance(hook, hook_cls) or not getattr(hook, _KEEP_ATTR, False):+ continue+ try:+ onloaded = [*module.parameters(), *(b for _, b in _named_buffers(module))]+ if any(t.device.type != "cpu" for t in onloaded):+ hook.init_hook(module)+ except Exception as exc: # noqa: BLE001 - the stock re-enable still moves it+ if logger is not None:+ logger.debug(+ "diffusion.memory: kept offload of %s before re-enable failed: %s",+ type(module).__name__,+ exc,+ )+++def _gguf_parameter_class() -> Optional[type]:+ module = sys.modules.get("diffusers.quantizers.gguf.utils")+ return getattr(module, "GGUFParameter", None) if module is not None else None++def _keepable(param: Any) -> bool:import torch- return type(param.data) is torch.Tensor++ data = param.data+ if type(data) is torch.Tensor:+ return True+ # GGUF weights are packed bytes; the quant type lives on the Parameter, which the swap never replaces.+ gguf = _gguf_parameter_class()+ return gguf is not None and type(param) is gguf and type(data) is ggufdef _wrap_cpu_offload_hooks(@@ -408,7 +444,7 @@ def _wrap_cpu_offload_hook(return outdef _pre_forward(mod: Any, *args: Any, **kwargs: Any) -> Any:- # `host` gate: an all-subclass module (GGUF, torchao) never fills `version`, so would rescan every forward.+ # `host` gate: an all-subclass module (torchao) never fills `version`, so would rescan every forward.onload = not version and not buffer_version and bool(host or buffer_host)if onload:for name, p in mod.named_parameters():diff --git a/studio/backend/tests/test_diffusion_offload_keep_cpu.py b/studio/backend/tests/test_diffusion_offload_keep_cpu.pyindex 7bef7fecddd..7a71b410572 100644--- a/studio/backend/tests/test_diffusion_offload_keep_cpu.py+++ b/studio/backend/tests/test_diffusion_offload_keep_cpu.py@@ -92,6 +92,42 @@ def test_subclass_weights_are_not_kept():assert not dm._keepable(lin.weight)+def _gguf_utils():+ pytest.importorskip("gguf")+ return pytest.importorskip("diffusers.quantizers.gguf.utils")+++def _gguf_linear(+ utils,+ out_features = 8,+ in_features = 64,+):+ """A diffusers GGUFLinear holding a real Q8_0 weight: per 32-value block, an fp16 scale then 32 int8 codes."""+ import numpy as np++ rng = np.random.default_rng(0)+ blocks = out_features * in_features // 32+ scale = np.full((blocks, 1), 0.01, dtype = np.float16).view(np.uint8)+ codes = rng.integers(-127, 128, size = (blocks, 32), dtype = np.int8).view(np.uint8)+ raw = torch.from_numpy(np.concatenate([scale, codes], axis = 1).reshape(out_features, -1).copy())+ lin = utils.GGUFLinear(in_features, out_features, bias = True, compute_dtype = torch.float32)+ lin.weight = utils.GGUFParameter(raw, quant_type = utils.gguf.GGMLQuantizationType.Q8_0)+ return lin+++def test_gguf_weights_are_kept_but_other_subclasses_are_not():+ utils = _gguf_utils()+ lin = _gguf_linear(utils)+ assert type(lin.weight) is utils.GGUFParameter+ assert dm._keepable(lin.weight) and dm._keepable(lin.bias)++ class Sub(torch.Tensor):+ pass++ other = torch.nn.Parameter(torch.zeros(2).as_subclass(Sub))+ assert not dm._keepable(other)++def test_an_all_subclass_module_is_not_rescanned_every_forward():class Sub(torch.Tensor):pass@@ -402,6 +438,25 @@ def test_the_ram_gate_is_sized_from_the_container(assert bool(allocated) is pinned+def _gguf_pipe(device):+ utils = _gguf_utils()+ pipe = _pipe(device, "text_encoder", "transformer", "vae")+ comps = pipe.components+ comps["text_encoder"] = pipe.text_encoder = torch.nn.Linear(8, 64)+ comps["transformer"] = pipe.transformer = _gguf_linear(utils)+ return pipe, utils+++def _run(pipe, x):+ """One call the way a diffusers pipeline makes it: the last module stays onloaded until the re-enable."""+ out = x+ for module in pipe.components.values():+ out = module(out)+ out = out.detach().cpu()+ pipe.enable_model_cpu_offload()+ return out++class _BufferLinear(torch.nn.Module):"""A weight held as plain buffers, as the torchao-free int8 linear stores it."""@@ -426,6 +481,46 @@ def _buffer_ptrs(module):return [b.data_ptr() for b in module.buffers()]+@cuda[email protected]("pin", ["0", "1"])+def test_a_gguf_transformer_keeps_its_host_bytes_and_quant_type(monkeypatch, pin):+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, pin)+ x = torch.randn(4, 8)+ stock, _ = _gguf_pipe("cuda")+ stock.enable_model_cpu_offload()+ ref = _run(stock, x)+ pipe, utils = _gguf_pipe("cuda")+ raw = pipe.transformer.weight.detach().clone()+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ _run(pipe, x) # the first onload pins+ # Held, so a stock copy can never land back at a freed address and pass for the kept tensor.+ held = [p.data for p in pipe.transformer.parameters()]+ host = [t.data_ptr() for t in held]+ for _ in range(3):+ assert torch.equal(_run(pipe, x), ref)+ weight = pipe.transformer.weight+ assert weight.device.type == "cpu" and _ptrs(pipe.transformer) == host+ assert type(weight) is utils.GGUFParameter+ assert weight.quant_type == utils.gguf.GGMLQuantizationType.Q8_0+ assert torch.equal(weight.detach().view(torch.uint8), raw.view(torch.uint8))+ assert weight.is_pinned() is (pin == "1")+++@cuda+def test_the_module_left_onloaded_by_a_call_is_offloaded_through_its_kept_hook(monkeypatch):+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "0")+ pipe = _pipe("cuda", "text_encoder", "transformer", "vae")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ held = {n: [p.data for p in m.parameters()] for n, m in pipe.components.items()}+ host = {n: [t.data_ptr() for t in ts] for n, ts in held.items()}+ for _ in range(3):+ _run(pipe, torch.randn(4, 8))+ assert pipe.vae.weight.device.type == "cpu"+ assert {n: _ptrs(m) for n, m in pipe.components.items()} == host++@cuda@pytest.mark.parametrize("pin", ["0", "1"])def test_buffer_weights_keep_their_host_storage_and_are_pinned(monkeypatch, pin):