Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: drop stale pinned host layout entries after a dtype change in low_vram offload (#14857)
Daniel Hand61d15748bea
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 98b9389762d..a137937f884 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -491,6 +491,37 @@ def _same_view(tensor: Any, view: Any) -> bool:)+def _prune_pin_layout(layout: list, host: dict, buffer_host: dict) -> None:+ """Drop, in place, the layout entries whose kept host weight is no longer their pinned view, and the chunks left+ with none: the layout's own references would otherwise keep those pinned chunks alive."""+ kept = []+ for buf, entries in layout:+ live = []+ for entry in entries:+ name, view = entry[0], entry[2]+ tensor = buffer_host.get(name[1:]) if name.startswith("\0") else host.get(name)+ if tensor is None:+ continue+ # A torchao wrapper weight has one entry per inner tensor (as ``_upload_pinned_chunks`` reads them).+ inner = None if name.startswith("\0") else _wrapper_inner(tensor)+ if inner is not None:+ if any(_same_view(t, view) for _, t in inner):+ live.append(entry)+ elif _same_view(tensor, view):+ live.append(entry)+ if live:+ kept.append((buf, live))+ layout[:] = kept+++def _prune_state_layout(state: dict) -> None:+ layout = state.get("pin_layout")+ if layout:+ _prune_pin_layout(layout, state.get("host") or {}, state.get("buffer_host") or {})+ if not layout:+ state.pop("pin_layout", None)++def _upload_pinned_chunks(module: Any, layout: list, host: dict, owner: dict, buffer_host: dict, device: Any) -> int:@@ -720,6 +751,8 @@ def _wrap_cpu_offload_hook(passout = init_hook(mod)_capture(mod)+ # a weight cast or replaced on the device comes back as a new host tensor: release its pinned chunk now+ _prune_state_layout(state)version.clear()buffer_version.clear()return out@@ -741,6 +774,8 @@ def _wrap_cpu_offload_hook(for name in list(buffer_host):if current.get(name) is not buffer_host[name]:buffer_host.pop(name, None)+ # changed while offloaded (a dtype cast, a replacement): its pinned chunk is not kept for nothing+ _prune_state_layout(state)if onload and not state.get("pin_tried"):# Once per module, on its first onload, so loading pays nothing.state["pin_tried"] = Truediff --git a/studio/backend/tests/test_diffusion_offload_fast_upload.py b/studio/backend/tests/test_diffusion_offload_fast_upload.pyindex 59e28b1944b..6aa2789e7f0 100644--- a/studio/backend/tests/test_diffusion_offload_fast_upload.py+++ b/studio/backend/tests/test_diffusion_offload_fast_upload.py@@ -335,6 +335,53 @@ def test_tensors_left_out_of_the_layout_are_not_uploaded_with_the_chunk(fake_pinassert torch.equal(params[name].detach(), expect[name])+def _layout_chunk_ptrs(module):+ layout = module.__dict__[dm._KEEP_ATTR].get("pin_layout") or []+ return {buf.data_ptr() for buf, _ in layout}++[email protected]("fast_upload", ["1", "0"])[email protected]("cast_on", ["host", "device"])+def test_a_dtype_change_releases_the_obsolete_pinned_chunks(+ fake_pin, monkeypatch, fast_upload, cast_on+):+ # The SDXL fp32 VAE decode casts an offloaded component and back: its kept weights are new host tensors, so the+ # chunks pinned for the old ones must not stay referenced by the layout (fast upload on or off).+ monkeypatch.setenv(dm.OFFLOAD_FAST_UPLOAD_ENV, fast_upload)+ pipe = _pipe("cpu", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ module = _onload(pipe)+ old = _layout_chunk_ptrs(module)+ assert old+ if cast_on == "device":+ module.half() # cast while onloaded: the offload brings back new host tensors+ module._hf_hook.init_hook(module)+ else:+ module._hf_hook.init_hook(module)+ module.half() # cast while offloaded: the next onload finds new host tensors+ _onload(pipe)+ assert not old & _layout_chunk_ptrs(module)+ state = module.__dict__[dm._KEEP_ATTR]+ live = {v.untyped_storage().data_ptr() for v in state["host"].values()}+ assert not old & live+++def test_a_replaced_weight_drops_out_of_the_layout_and_the_rest_stays(fake_pin):+ pipe = _pipe("cpu", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ module = _onload(pipe)+ module._hf_hook.init_hook(module)+ before = module.__dict__[dm._KEEP_ATTR]["pin_layout"]+ names = {name for _, entries in before for name, _, _ in entries}+ module[0].weight = torch.nn.Parameter(torch.zeros(16, 16))+ _onload(pipe)+ after = module.__dict__[dm._KEEP_ATTR]["pin_layout"]+ left = {name for _, entries in after for name, _, _ in entries}+ assert left == names - {"0.weight"}++def _quantised(scheme, n = N_LAYERS):"""A Linear stack with torchao int8 / fp8 wrapper weights (Studio's quantised denoisers under low_vram)."""quant = pytest.importorskip("torchao.quantization")@@ -428,6 +475,27 @@ def test_a_wrapper_with_an_inner_tensor_outside_the_layout_takes_the_stock_copy(][email protected]("scheme", ["int8", "fp8"])+def test_the_layout_prune_keeps_unchanged_wrapper_weights_and_drops_replaced_ones(fake_pin, scheme):+ # A wrapper weight's entries are its inner tensors: the prune must match those, not the wrapper itself.+ module = _quantised(scheme)+ host = {n: p.data for n, p in module.named_parameters()}+ layout: list = []+ assert dm._pin_host_weights(module, host, layout = layout) > 0+ host = {n: p.data for n, p in module.named_parameters()}+ names = [name for _, entries in layout for name, _, _ in entries]+ wrapped = {n for n in names if dm._wrapper_inner(host.get(n)) is not None}+ assert wrapped+ dm._prune_pin_layout(layout, host, {})+ assert [name for _, entries in layout for name, _, _ in entries] == names+ # a dequantised (dtype-changed) or re-quantised weight is a new host tensor: its entries go, the rest stay+ host["0.weight"] = host["0.weight"].dequantize()+ host["1.weight"] = _quantised(scheme, n = 2)[1].weight.data+ dm._prune_pin_layout(layout, host, {})+ left = [name for _, entries in layout for name, _, _ in entries]+ assert left == [n for n in names if n not in ("0.weight", "1.weight")]++def _quantised_pipe(scheme):pipe = _pipe("cuda", "text_encoder", "transformer")pipe.components["text_encoder"] = torch.nn.Linear(8, 64, dtype = torch.bfloat16)