Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: onload torchao int8 / fp8 weights through the pinned chunks (#14832)
Daniel Hanea9db57c003c
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 7d3e0aab92f..f8e9ef48d43 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -401,9 +401,8 @@ def _pin_host_weights(members: list = [[] for _ in bufs]wrapped: dict = {}for name, p, attr, view, index, start in placed:- if attr is None:- # Wrapped (torchao) inner tensors stay with the stock module.to: the chunk upload re-points p.data only.- members[index].append((name, start, view))+ # A torchao wrapper weight gets one entry per inner tensor, under the weight's name.+ members[index].append((name, start, view))if name.startswith("\0"):name = name[1:]_set_buffer(module, name, view)@@ -497,13 +496,46 @@ def _upload_pinned_chunks(and with the device cache emptied between components each of those was a cudaMalloc (and a cudaFree on the wayout): under host load that ran at 57-120 MiB/s against a 50 GiB/s link. Chunks are near-uniform, so the nextcomponent reuses the previous one's cached chunks. Same bytes, same strides, so outputs are bit-identical.- Weights no longer at their pinned host view (replaced, reassigned) are left for the stock copy. Returns bytes."""+ Weights no longer at their pinned host view (replaced, reassigned) are left for the stock copy. A torchao+ wrapper weight (int8 / fp8) is rebuilt over its inner tensors' device slices with ``__tensor_unflatten__`` and+ swapped in, as the stock ``_to_copy`` + ``Module.to`` do. Returns bytes."""import torchparams = dict(module.named_parameters())buffers = dict(_named_buffers(module))copied = 0moved: list = []+ wrapped: dict = {}+ for name, p in params.items():+ kept = host.get(name)+ inner = _wrapper_inner(kept) if kept is not None else None+ if (+ inner is None+ or owner.get(name) is not p+ or p.device.type != "cpu"+ or type(p) is not type(kept)+ or _tensor_ptr(p.data) != _tensor_ptr(kept)+ ):+ continue+ wrapped[name] = dict(inner)+ # (name, inner attr) -> device slice once uploaded+ inner_dev: dict = {}++ def _inner_attr(name: str, view: Any) -> Optional[str]:+ for attr, t in (wrapped.get(name) or {}).items():+ if _same_view(t, view):+ return attr+ return None++ for _buf, entries in layout:+ for name, _start, view in entries:+ attr = _inner_attr(name, view)+ if attr is not None:+ inner_dev[(name, attr)] = None+ # Only a wrapper with every inner tensor in the layout is rebuilt; the rest stay for the stock module.to.+ whole = {+ name for name, inner in wrapped.items() if all((name, attr) in inner_dev for attr in inner)+ }try:for buf, entries in layout:live = []@@ -515,20 +547,24 @@ def _upload_pinned_chunks(if b is not None and buffer_host.get(leaf) is b and _same_view(b, view):live.append((name, start, view))continue+ if name in wrapped:+ if name in whole and _inner_attr(name, view) is not None:+ live.append((name, start, view))+ continuep = params.get(name)kept = host.get(name)if (p is not Noneand owner.get(name) is pand kept is not None+ and _wrapper_inner(kept) is Noneand _same_view(kept, view)and _same_view(p.data, view)):live.append((name, start, view))if not live:continue- # Pack the movable weights: replaced weights and torchao wrapper tensors (left to the stock module.to)- # would otherwise leave holes that are allocated and uploaded for nothing.+ # Pack live weights so skipped ones leave no allocated holes.used, slots = 0, []for name, start, view in sorted(live, key = lambda e: e[1]):used = -(-used // _PIN_ALIGN) * _PIN_ALIGN@@ -557,15 +593,28 @@ def _upload_pinned_chunks()if name.startswith("\0"):_set_buffer(module, name[1:], dev)+ elif name in wrapped:+ inner_dev[(name, _inner_attr(name, view))] = dev+ continueelse:params[name].data = devmoved.append((name, view))+ for name in sorted(whole):+ p, kept = params[name], host[name]+ names, ctx = kept.__tensor_flatten__()+ rebuilt = type(kept).__tensor_unflatten__(+ {attr: inner_dev[(name, attr)] for attr in names}, ctx, kept.shape, kept.stride()+ )+ torch.utils.swap_tensors(p, torch.nn.Parameter(rebuilt, requires_grad = p.requires_grad))+ moved.append((name, kept))except Exception:# Put every weight back on its host view; the stock per-tensor copy then runs (and raises a real OOM itself).for name, view in moved:try:if name.startswith("\0"):_set_buffer(module, name[1:], view)+ elif name in wrapped:+ _repoint(params[name], view)else:params[name].data = viewexcept Exception: # noqa: BLE001diff --git a/studio/backend/tests/test_diffusion_offload_fast_upload.py b/studio/backend/tests/test_diffusion_offload_fast_upload.pyindex e719be70ade..59e28b1944b 100644--- a/studio/backend/tests/test_diffusion_offload_fast_upload.py+++ b/studio/backend/tests/test_diffusion_offload_fast_upload.py@@ -333,3 +333,154 @@ def test_tensors_left_out_of_the_layout_are_not_uploaded_with_the_chunk(fake_pinfor name, _, view in kept:assert params[name].data_ptr() != view.data_ptr()assert torch.equal(params[name].detach(), expect[name])+++def _quantised(scheme, n = N_LAYERS):+ """A Linear stack with torchao int8 / fp8 wrapper weights (Studio's quantised denoisers under low_vram)."""+ quant = pytest.importorskip("torchao.quantization")+ torch.manual_seed(0)+ module = torch.nn.Sequential(+ *(torch.nn.Linear(64, 64, bias = True, dtype = torch.bfloat16) for _ in range(n))+ )+ if scheme == "int8":+ try:+ config = quant.Int8DynamicActivationInt8WeightConfig(version = 2)+ except TypeError:+ config = quant.Int8DynamicActivationInt8WeightConfig()+ else:+ config = quant.Float8DynamicActivationFloat8WeightConfig(granularity = quant.PerRow())+ try:+ quant.quantize_(module, config)+ except Exception as exc: # noqa: BLE001 - e.g. an fp8 build that needs a capable GPU+ pytest.skip(f"torchao {scheme} quantize unavailable here: {exc}")+ if dm._wrapper_inner(module[0].weight.data) is None:+ pytest.skip("this torchao build stores the weights without a traceable wrapper")+ return module+++def _inner_state(t):+ return {attr: x.detach().clone() for attr, x in dm._wrapper_inner(t)}++[email protected]("scheme", ["int8", "fp8"])+def test_wrapper_weights_ride_the_chunk_and_are_rebuilt_over_its_slices(fake_pin, scheme):+ module = _quantised(scheme)+ host = {n: p.data for n, p in module.named_parameters()}+ owner = dict(module.named_parameters())+ expect = {+ n: _inner_state(p.data) for n, p in module.named_parameters() if dm._wrapper_inner(p.data)+ }+ layout: list = []+ assert dm._pin_host_weights(module, host, layout = layout) > 0+ listed = [(name, view) for _, entries in layout for name, _, view in entries]+ for name in expect:+ # one layout entry per inner tensor, each at the pinned inner tensor the host weight now reads+ views = [view for n, view in listed if n == name]+ assert sorted(v.data_ptr() for v in views) == sorted(+ x.data_ptr() for _, x in dm._wrapper_inner(host[name])+ )+ params = dict(module.named_parameters())+ ids = {n: id(p) for n, p in params.items()}+ copied = dm._upload_pinned_chunks(module, layout, host, owner, {}, torch.device("cpu"))+ assert copied >= sum(x.nbytes for n in expect for _, x in dm._wrapper_inner(host[n]))+ chunk_ptrs = {buf.data_ptr() for buf, _ in layout}+ for name, inner in expect.items():+ p = dict(module.named_parameters())[name]+ assert id(p) == ids[name] and type(p) is type(+ host[name]+ ) # swapped in place, same Parameter object+ now = dict(dm._wrapper_inner(p.data))+ assert set(now) == set(inner)+ for attr, x in now.items():+ assert torch.equal(x, inner[attr])+ assert (+ x.data_ptr() not in chunk_ptrs+ and x.data_ptr() != dict(dm._wrapper_inner(host[name]))[attr].data_ptr()+ )+ assert p.shape == host[name].shape and p.dtype == host[name].dtype+ storages = {+ x.untyped_storage().data_ptr()+ for n in expect+ for _, x in dm._wrapper_inner(dict(module.named_parameters())[n].data)+ }+ assert len(storages) <= len(layout)+++def test_a_wrapper_with_an_inner_tensor_outside_the_layout_takes_the_stock_copy(fake_pin):+ module = _quantised("int8", n = 2)+ host = {n: p.data for n, p in module.named_parameters()}+ owner = dict(module.named_parameters())+ layout: list = []+ assert dm._pin_host_weights(module, host, layout = layout) > 0+ name = "0.weight"+ drop = dm._wrapper_inner(host[name])[1][1] # its scale+ trimmed = [+ (buf, [e for e in entries if not (e[0] == name and e[2].data_ptr() == drop.data_ptr())])+ for buf, entries in layout+ ]+ before = [x.data_ptr() for _, x in dm._wrapper_inner(module[0].weight.data)]+ dm._upload_pinned_chunks(module, trimmed, host, owner, {}, torch.device("cpu"))+ assert [+ x.data_ptr() for _, x in dm._wrapper_inner(module[0].weight.data)+ ] == before # left on the host+ assert [x.data_ptr() for _, x in dm._wrapper_inner(module[1].weight.data)] != [+ x.data_ptr() for _, x in dm._wrapper_inner(host["1.weight"])+ ]+++def _quantised_pipe(scheme):+ pipe = _pipe("cuda", "text_encoder", "transformer")+ pipe.components["text_encoder"] = torch.nn.Linear(8, 64, dtype = torch.bfloat16)+ pipe.components["transformer"] = _quantised(scheme)+ pipe.text_encoder, pipe.transformer = (+ pipe.components["text_encoder"],+ pipe.components["transformer"],+ )+ return pipe+++@cuda[email protected]("scheme", ["int8", "fp8"])+def test_quantised_chunk_onload_is_bit_identical_and_skips_the_per_tensor_copy(monkeypatch, scheme):+ # no_grad only: torchao weights cannot move under inference_mode, so Studio renders offloaded quant under no_grad.+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")+ monkeypatch.setattr(dm, "_PIN_CHUNK_BYTES", 4096)+ if scheme == "fp8" and torch.cuda.get_device_capability() < (8, 9):+ pytest.skip("fp8 matmul needs sm89+")+ ctx = torch.no_grad+ x = torch.randn(4, 8, dtype = torch.bfloat16)+ stock = _quantised_pipe(scheme)+ stock.enable_model_cpu_offload()+ ref = _render(stock, x, ctx)+ pipe = _quantised_pipe(scheme)+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ assert torch.equal(_render(pipe, x, ctx), ref) # the first render pins+ module = pipe.transformer+ layout = module.__dict__[dm._KEEP_ATTR]["pin_layout"]+ assert layout+ seen = []++ def spy(mod, *args):+ inner = [+ x+ for p in mod.parameters()+ if dm._wrapper_inner(p.data)+ for _, x in dm._wrapper_inner(p.data)+ ]+ assert inner and all(x.device.type == "cuda" for x in inner)+ seen.append({x.untyped_storage().data_ptr() for x in inner})++ module.register_forward_hook(spy)++ allocs = _device_allocs()+ for _ in range(3):+ assert torch.equal(_render(pipe, x, ctx), ref)+ # every inner tensor is a slice of one of the chunk allocations (per tensor: one storage each) ...+ assert all(len(s) <= len(layout) for s in seen) and len(seen) == 3+ # ... the chunks come back out of the caching allocator ...+ assert _device_allocs() - allocs <= 2+ # ... and every offload re-points the weights at their pinned host tensors+ host = module.__dict__[dm._KEEP_ATTR]["host"]+ for n, p in module.named_parameters():+ assert p.device.type == "cpu" and dm._tensor_ptr(p.data) == dm._tensor_ptr(host[n])