Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stream a torchao denoiser from an unpinned host copy instead of leaving it on the GPU (#12551)

oobaboogaa71db3c42680
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 1641a775d17..e5561317c44 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -3345,12 +3345,39 @@ def install_group_offload_hooks_eager() -> bool:
return patched
+def _install_group_offload_torchao_host_copy(go: Any) -> bool:
+ """Give aliased torchao host copies a separate wrapper so ``swap_tensors`` cannot move them to CUDA."""
+ group_cls = getattr(go, "ModuleGroup", None)
+ original = getattr(group_cls, "_to_cpu", None)
+ is_torchao = getattr(go, "_is_torchao_tensor", None)
+ if (
+ original is None
+ or not callable(is_torchao)
+ or getattr(original, "_unsloth_host_copy", False)
+ ):
+ return False
+
+ def _to_cpu(tensor: Any, low_cpu_mem_usage: bool) -> Any:
+ copy = original(tensor, low_cpu_mem_usage)
+ if copy is not tensor or not is_torchao(tensor):
+ return copy
+ names, ctx = tensor.__tensor_flatten__()
+ return type(tensor).__tensor_unflatten__(
+ {name: getattr(tensor, name) for name in names}, ctx, tensor.size(), tensor.stride()
+ )
+
+ _to_cpu._unsloth_host_copy = True
+ group_cls._to_cpu = staticmethod(_to_cpu)
+ return True
+
+
def install_group_offload_buffer_restore() -> bool:
- """diffusers stream group offload restores only parameters, leaving buffers (native int8 weights) on the GPU."""
+ """Fix buffer restoration and torchao host-copy aliasing in diffusers stream group offload."""
try:
from diffusers.hooks import group_offloading as go
except Exception: # noqa: BLE001 - no group offload in this diffusers
return False
+ _install_group_offload_torchao_host_copy(go)
group_cls = getattr(go, "ModuleGroup", None)
original = getattr(group_cls, "_offload_to_memory", None)
if original is None or getattr(original, "_unsloth_buffer_restore", False):
diff --git a/studio/backend/tests/test_diffusion_native_quant.py b/studio/backend/tests/test_diffusion_native_quant.py
index 013f6de9010..d7d01c9754c 100644
--- a/studio/backend/tests/test_diffusion_native_quant.py
+++ b/studio/backend/tests/test_diffusion_native_quant.py
@@ -1023,6 +1023,76 @@ def test_native_int8_under_real_stream_group_offload_leaves_nothing_resident():
assert {b.device.type for b in model.buffers()} == {"cpu"}
+def _torchao_int8_blocks(n = 3, features = 256):
+ quant = pytest.importorskip("torchao.quantization")
+
+ class Model(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.blocks = torch.nn.ModuleList(
+ [torch.nn.Linear(features, features, dtype = torch.bfloat16) for _ in range(n)]
+ )
+
+ def forward(self, x):
+ for block in self.blocks:
+ x = block(x)
+ return x
+
+ model = Model()
+ quant.quantize_(model, quant.Int8WeightOnlyConfig())
+ return model.requires_grad_(False)
+
+
+def test_unpinned_torchao_host_copy_is_a_distinct_wrapper_over_the_same_data():
+ go = pytest.importorskip("diffusers.hooks.group_offloading")
+ from core.inference.diffusion_memory import install_group_offload_buffer_restore
+
+ install_group_offload_buffer_restore()
+ weight = _torchao_int8_blocks(n = 1).blocks[0].weight
+ copy = go.ModuleGroup._to_cpu(weight, True)
+ assert copy is not weight
+ assert type(copy) is type(weight)
+ assert copy.qdata.data_ptr() == weight.qdata.data_ptr()
+ assert copy.scale.data_ptr() == weight.scale.data_ptr()
+ plain = torch.zeros(4)
+ assert go.ModuleGroup._to_cpu(plain, True).data_ptr() == plain.data_ptr()
+
+
+ not torch.cuda.is_available(), reason = "stream group offload needs a CUDA device"
+)
+def test_unpinned_torchao_stream_group_offload_leaves_nothing_resident():
+ pytest.importorskip("diffusers.hooks.group_offloading")
+ from diffusers.hooks import apply_group_offloading
+
+ from core.inference.diffusion_memory import install_group_offload_buffer_restore
+
+ install_group_offload_buffer_restore()
+ model = _torchao_int8_blocks()
+ resident = _torchao_int8_blocks()
+ resident.load_state_dict(model.state_dict(), assign = True)
+ resident.cuda()
+ apply_group_offloading(
+ model,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = True,
+ low_cpu_mem_usage = True,
+ non_blocking = True,
+ record_stream = True,
+ )
+ x = torch.randn(8, 256, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ expected = resident(x)
+ for _ in range(2): # Reuse the host copies after offloading.
+ out = model(x)
+ torch.cuda.synchronize()
+ assert {b.weight.qdata.device.type for b in model.blocks} == {"cpu"}
+ assert torch.equal(out, expected)
+
+
class _GateSeen(Exception):
pass