Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stream a torchao denoiser from an unpinned host copy instead of leaving it on the GPU (#12551)
oobaboogaa71db3c42680
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 1641a775d17..e5561317c44 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -3345,12 +3345,39 @@ def install_group_offload_hooks_eager() -> bool:return patched+def _install_group_offload_torchao_host_copy(go: Any) -> bool:+ """Give aliased torchao host copies a separate wrapper so ``swap_tensors`` cannot move them to CUDA."""+ group_cls = getattr(go, "ModuleGroup", None)+ original = getattr(group_cls, "_to_cpu", None)+ is_torchao = getattr(go, "_is_torchao_tensor", None)+ if (+ original is None+ or not callable(is_torchao)+ or getattr(original, "_unsloth_host_copy", False)+ ):+ return False++ def _to_cpu(tensor: Any, low_cpu_mem_usage: bool) -> Any:+ copy = original(tensor, low_cpu_mem_usage)+ if copy is not tensor or not is_torchao(tensor):+ return copy+ names, ctx = tensor.__tensor_flatten__()+ return type(tensor).__tensor_unflatten__(+ {name: getattr(tensor, name) for name in names}, ctx, tensor.size(), tensor.stride()+ )++ _to_cpu._unsloth_host_copy = True+ group_cls._to_cpu = staticmethod(_to_cpu)+ return True++def install_group_offload_buffer_restore() -> bool:- """diffusers stream group offload restores only parameters, leaving buffers (native int8 weights) on the GPU."""+ """Fix buffer restoration and torchao host-copy aliasing in diffusers stream group offload."""try:from diffusers.hooks import group_offloading as goexcept Exception: # noqa: BLE001 - no group offload in this diffusersreturn False+ _install_group_offload_torchao_host_copy(go)group_cls = getattr(go, "ModuleGroup", None)original = getattr(group_cls, "_offload_to_memory", None)if original is None or getattr(original, "_unsloth_buffer_restore", False):diff --git a/studio/backend/tests/test_diffusion_native_quant.py b/studio/backend/tests/test_diffusion_native_quant.pyindex 013f6de9010..d7d01c9754c 100644--- a/studio/backend/tests/test_diffusion_native_quant.py+++ b/studio/backend/tests/test_diffusion_native_quant.py@@ -1023,6 +1023,76 @@ def test_native_int8_under_real_stream_group_offload_leaves_nothing_resident():assert {b.device.type for b in model.buffers()} == {"cpu"}+def _torchao_int8_blocks(n = 3, features = 256):+ quant = pytest.importorskip("torchao.quantization")++ class Model(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.blocks = torch.nn.ModuleList(+ [torch.nn.Linear(features, features, dtype = torch.bfloat16) for _ in range(n)]+ )++ def forward(self, x):+ for block in self.blocks:+ x = block(x)+ return x++ model = Model()+ quant.quantize_(model, quant.Int8WeightOnlyConfig())+ return model.requires_grad_(False)+++def test_unpinned_torchao_host_copy_is_a_distinct_wrapper_over_the_same_data():+ go = pytest.importorskip("diffusers.hooks.group_offloading")+ from core.inference.diffusion_memory import install_group_offload_buffer_restore++ install_group_offload_buffer_restore()+ weight = _torchao_int8_blocks(n = 1).blocks[0].weight+ copy = go.ModuleGroup._to_cpu(weight, True)+ assert copy is not weight+ assert type(copy) is type(weight)+ assert copy.qdata.data_ptr() == weight.qdata.data_ptr()+ assert copy.scale.data_ptr() == weight.scale.data_ptr()+ plain = torch.zeros(4)+ assert go.ModuleGroup._to_cpu(plain, True).data_ptr() == plain.data_ptr()+++ not torch.cuda.is_available(), reason = "stream group offload needs a CUDA device"+)+def test_unpinned_torchao_stream_group_offload_leaves_nothing_resident():+ pytest.importorskip("diffusers.hooks.group_offloading")+ from diffusers.hooks import apply_group_offloading++ from core.inference.diffusion_memory import install_group_offload_buffer_restore++ install_group_offload_buffer_restore()+ model = _torchao_int8_blocks()+ resident = _torchao_int8_blocks()+ resident.load_state_dict(model.state_dict(), assign = True)+ resident.cuda()+ apply_group_offloading(+ model,+ onload_device = torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ use_stream = True,+ low_cpu_mem_usage = True,+ non_blocking = True,+ record_stream = True,+ )+ x = torch.randn(8, 256, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ expected = resident(x)+ for _ in range(2): # Reuse the host copies after offloading.+ out = model(x)+ torch.cuda.synchronize()+ assert {b.weight.qdata.device.type for b in model.blocks} == {"cpu"}+ assert torch.equal(out, expected)++class _GateSeen(Exception):pass