Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: onload torchao int8 / fp8 weights through the pinned chunks (#14832)

Daniel Hanea9db57c003c
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 7d3e0aab92f..f8e9ef48d43 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -401,9 +401,8 @@ def _pin_host_weights(
members: list = [[] for _ in bufs]
wrapped: dict = {}
for name, p, attr, view, index, start in placed:
- if attr is None:
- # Wrapped (torchao) inner tensors stay with the stock module.to: the chunk upload re-points p.data only.
- members[index].append((name, start, view))
+ # A torchao wrapper weight gets one entry per inner tensor, under the weight's name.
+ members[index].append((name, start, view))
if name.startswith("\0"):
name = name[1:]
_set_buffer(module, name, view)
@@ -497,13 +496,46 @@ def _upload_pinned_chunks(
and with the device cache emptied between components each of those was a cudaMalloc (and a cudaFree on the way
out): under host load that ran at 57-120 MiB/s against a 50 GiB/s link. Chunks are near-uniform, so the next
component reuses the previous one's cached chunks. Same bytes, same strides, so outputs are bit-identical.
- Weights no longer at their pinned host view (replaced, reassigned) are left for the stock copy. Returns bytes."""
+ Weights no longer at their pinned host view (replaced, reassigned) are left for the stock copy. A torchao
+ wrapper weight (int8 / fp8) is rebuilt over its inner tensors' device slices with ``__tensor_unflatten__`` and
+ swapped in, as the stock ``_to_copy`` + ``Module.to`` do. Returns bytes."""
import torch
params = dict(module.named_parameters())
buffers = dict(_named_buffers(module))
copied = 0
moved: list = []
+ wrapped: dict = {}
+ for name, p in params.items():
+ kept = host.get(name)
+ inner = _wrapper_inner(kept) if kept is not None else None
+ if (
+ inner is None
+ or owner.get(name) is not p
+ or p.device.type != "cpu"
+ or type(p) is not type(kept)
+ or _tensor_ptr(p.data) != _tensor_ptr(kept)
+ ):
+ continue
+ wrapped[name] = dict(inner)
+ # (name, inner attr) -> device slice once uploaded
+ inner_dev: dict = {}
+
+ def _inner_attr(name: str, view: Any) -> Optional[str]:
+ for attr, t in (wrapped.get(name) or {}).items():
+ if _same_view(t, view):
+ return attr
+ return None
+
+ for _buf, entries in layout:
+ for name, _start, view in entries:
+ attr = _inner_attr(name, view)
+ if attr is not None:
+ inner_dev[(name, attr)] = None
+ # Only a wrapper with every inner tensor in the layout is rebuilt; the rest stay for the stock module.to.
+ whole = {
+ name for name, inner in wrapped.items() if all((name, attr) in inner_dev for attr in inner)
+ }
try:
for buf, entries in layout:
live = []
@@ -515,20 +547,24 @@ def _upload_pinned_chunks(
if b is not None and buffer_host.get(leaf) is b and _same_view(b, view):
live.append((name, start, view))
continue
+ if name in wrapped:
+ if name in whole and _inner_attr(name, view) is not None:
+ live.append((name, start, view))
+ continue
p = params.get(name)
kept = host.get(name)
if (
p is not None
and owner.get(name) is p
and kept is not None
+ and _wrapper_inner(kept) is None
and _same_view(kept, view)
and _same_view(p.data, view)
):
live.append((name, start, view))
if not live:
continue
- # Pack the movable weights: replaced weights and torchao wrapper tensors (left to the stock module.to)
- # would otherwise leave holes that are allocated and uploaded for nothing.
+ # Pack live weights so skipped ones leave no allocated holes.
used, slots = 0, []
for name, start, view in sorted(live, key = lambda e: e[1]):
used = -(-used // _PIN_ALIGN) * _PIN_ALIGN
@@ -557,15 +593,28 @@ def _upload_pinned_chunks(
)
if name.startswith("\0"):
_set_buffer(module, name[1:], dev)
+ elif name in wrapped:
+ inner_dev[(name, _inner_attr(name, view))] = dev
+ continue
else:
params[name].data = dev
moved.append((name, view))
+ for name in sorted(whole):
+ p, kept = params[name], host[name]
+ names, ctx = kept.__tensor_flatten__()
+ rebuilt = type(kept).__tensor_unflatten__(
+ {attr: inner_dev[(name, attr)] for attr in names}, ctx, kept.shape, kept.stride()
+ )
+ torch.utils.swap_tensors(p, torch.nn.Parameter(rebuilt, requires_grad = p.requires_grad))
+ moved.append((name, kept))
except Exception:
# Put every weight back on its host view; the stock per-tensor copy then runs (and raises a real OOM itself).
for name, view in moved:
try:
if name.startswith("\0"):
_set_buffer(module, name[1:], view)
+ elif name in wrapped:
+ _repoint(params[name], view)
else:
params[name].data = view
except Exception: # noqa: BLE001
diff --git a/studio/backend/tests/test_diffusion_offload_fast_upload.py b/studio/backend/tests/test_diffusion_offload_fast_upload.py
index e719be70ade..59e28b1944b 100644
--- a/studio/backend/tests/test_diffusion_offload_fast_upload.py
+++ b/studio/backend/tests/test_diffusion_offload_fast_upload.py
@@ -333,3 +333,154 @@ def test_tensors_left_out_of_the_layout_are_not_uploaded_with_the_chunk(fake_pin
for name, _, view in kept:
assert params[name].data_ptr() != view.data_ptr()
assert torch.equal(params[name].detach(), expect[name])
+
+
+def _quantised(scheme, n = N_LAYERS):
+ """A Linear stack with torchao int8 / fp8 wrapper weights (Studio's quantised denoisers under low_vram)."""
+ quant = pytest.importorskip("torchao.quantization")
+ torch.manual_seed(0)
+ module = torch.nn.Sequential(
+ *(torch.nn.Linear(64, 64, bias = True, dtype = torch.bfloat16) for _ in range(n))
+ )
+ if scheme == "int8":
+ try:
+ config = quant.Int8DynamicActivationInt8WeightConfig(version = 2)
+ except TypeError:
+ config = quant.Int8DynamicActivationInt8WeightConfig()
+ else:
+ config = quant.Float8DynamicActivationFloat8WeightConfig(granularity = quant.PerRow())
+ try:
+ quant.quantize_(module, config)
+ except Exception as exc: # noqa: BLE001 - e.g. an fp8 build that needs a capable GPU
+ pytest.skip(f"torchao {scheme} quantize unavailable here: {exc}")
+ if dm._wrapper_inner(module[0].weight.data) is None:
+ pytest.skip("this torchao build stores the weights without a traceable wrapper")
+ return module
+
+
+def _inner_state(t):
+ return {attr: x.detach().clone() for attr, x in dm._wrapper_inner(t)}
+
+
[email protected]("scheme", ["int8", "fp8"])
+def test_wrapper_weights_ride_the_chunk_and_are_rebuilt_over_its_slices(fake_pin, scheme):
+ module = _quantised(scheme)
+ host = {n: p.data for n, p in module.named_parameters()}
+ owner = dict(module.named_parameters())
+ expect = {
+ n: _inner_state(p.data) for n, p in module.named_parameters() if dm._wrapper_inner(p.data)
+ }
+ layout: list = []
+ assert dm._pin_host_weights(module, host, layout = layout) > 0
+ listed = [(name, view) for _, entries in layout for name, _, view in entries]
+ for name in expect:
+ # one layout entry per inner tensor, each at the pinned inner tensor the host weight now reads
+ views = [view for n, view in listed if n == name]
+ assert sorted(v.data_ptr() for v in views) == sorted(
+ x.data_ptr() for _, x in dm._wrapper_inner(host[name])
+ )
+ params = dict(module.named_parameters())
+ ids = {n: id(p) for n, p in params.items()}
+ copied = dm._upload_pinned_chunks(module, layout, host, owner, {}, torch.device("cpu"))
+ assert copied >= sum(x.nbytes for n in expect for _, x in dm._wrapper_inner(host[n]))
+ chunk_ptrs = {buf.data_ptr() for buf, _ in layout}
+ for name, inner in expect.items():
+ p = dict(module.named_parameters())[name]
+ assert id(p) == ids[name] and type(p) is type(
+ host[name]
+ ) # swapped in place, same Parameter object
+ now = dict(dm._wrapper_inner(p.data))
+ assert set(now) == set(inner)
+ for attr, x in now.items():
+ assert torch.equal(x, inner[attr])
+ assert (
+ x.data_ptr() not in chunk_ptrs
+ and x.data_ptr() != dict(dm._wrapper_inner(host[name]))[attr].data_ptr()
+ )
+ assert p.shape == host[name].shape and p.dtype == host[name].dtype
+ storages = {
+ x.untyped_storage().data_ptr()
+ for n in expect
+ for _, x in dm._wrapper_inner(dict(module.named_parameters())[n].data)
+ }
+ assert len(storages) <= len(layout)
+
+
+def test_a_wrapper_with_an_inner_tensor_outside_the_layout_takes_the_stock_copy(fake_pin):
+ module = _quantised("int8", n = 2)
+ host = {n: p.data for n, p in module.named_parameters()}
+ owner = dict(module.named_parameters())
+ layout: list = []
+ assert dm._pin_host_weights(module, host, layout = layout) > 0
+ name = "0.weight"
+ drop = dm._wrapper_inner(host[name])[1][1] # its scale
+ trimmed = [
+ (buf, [e for e in entries if not (e[0] == name and e[2].data_ptr() == drop.data_ptr())])
+ for buf, entries in layout
+ ]
+ before = [x.data_ptr() for _, x in dm._wrapper_inner(module[0].weight.data)]
+ dm._upload_pinned_chunks(module, trimmed, host, owner, {}, torch.device("cpu"))
+ assert [
+ x.data_ptr() for _, x in dm._wrapper_inner(module[0].weight.data)
+ ] == before # left on the host
+ assert [x.data_ptr() for _, x in dm._wrapper_inner(module[1].weight.data)] != [
+ x.data_ptr() for _, x in dm._wrapper_inner(host["1.weight"])
+ ]
+
+
+def _quantised_pipe(scheme):
+ pipe = _pipe("cuda", "text_encoder", "transformer")
+ pipe.components["text_encoder"] = torch.nn.Linear(8, 64, dtype = torch.bfloat16)
+ pipe.components["transformer"] = _quantised(scheme)
+ pipe.text_encoder, pipe.transformer = (
+ pipe.components["text_encoder"],
+ pipe.components["transformer"],
+ )
+ return pipe
+
+
+@cuda
[email protected]("scheme", ["int8", "fp8"])
+def test_quantised_chunk_onload_is_bit_identical_and_skips_the_per_tensor_copy(monkeypatch, scheme):
+ # no_grad only: torchao weights cannot move under inference_mode, so Studio renders offloaded quant under no_grad.
+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")
+ monkeypatch.setattr(dm, "_PIN_CHUNK_BYTES", 4096)
+ if scheme == "fp8" and torch.cuda.get_device_capability() < (8, 9):
+ pytest.skip("fp8 matmul needs sm89+")
+ ctx = torch.no_grad
+ x = torch.randn(4, 8, dtype = torch.bfloat16)
+ stock = _quantised_pipe(scheme)
+ stock.enable_model_cpu_offload()
+ ref = _render(stock, x, ctx)
+ pipe = _quantised_pipe(scheme)
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ assert torch.equal(_render(pipe, x, ctx), ref) # the first render pins
+ module = pipe.transformer
+ layout = module.__dict__[dm._KEEP_ATTR]["pin_layout"]
+ assert layout
+ seen = []
+
+ def spy(mod, *args):
+ inner = [
+ x
+ for p in mod.parameters()
+ if dm._wrapper_inner(p.data)
+ for _, x in dm._wrapper_inner(p.data)
+ ]
+ assert inner and all(x.device.type == "cuda" for x in inner)
+ seen.append({x.untyped_storage().data_ptr() for x in inner})
+
+ module.register_forward_hook(spy)
+
+ allocs = _device_allocs()
+ for _ in range(3):
+ assert torch.equal(_render(pipe, x, ctx), ref)
+ # every inner tensor is a slice of one of the chunk allocations (per tensor: one storage each) ...
+ assert all(len(s) <= len(layout) for s in seen) and len(seen) == 3
+ # ... the chunks come back out of the caching allocator ...
+ assert _device_allocs() - allocs <= 2
+ # ... and every offload re-points the weights at their pinned host tensors
+ host = module.__dict__[dm._KEEP_ATTR]["host"]
+ for n, p in module.named_parameters():
+ assert p.device.type == "cpu" and dm._tensor_ptr(p.data) == dm._tensor_ptr(host[n])