Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: drop stale pinned host layout entries after a dtype change in low_vram offload (#14857)

Daniel Hand61d15748bea
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 98b9389762d..a137937f884 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -491,6 +491,37 @@ def _same_view(tensor: Any, view: Any) -> bool:
)
+def _prune_pin_layout(layout: list, host: dict, buffer_host: dict) -> None:
+ """Drop, in place, the layout entries whose kept host weight is no longer their pinned view, and the chunks left
+ with none: the layout's own references would otherwise keep those pinned chunks alive."""
+ kept = []
+ for buf, entries in layout:
+ live = []
+ for entry in entries:
+ name, view = entry[0], entry[2]
+ tensor = buffer_host.get(name[1:]) if name.startswith("\0") else host.get(name)
+ if tensor is None:
+ continue
+ # A torchao wrapper weight has one entry per inner tensor (as ``_upload_pinned_chunks`` reads them).
+ inner = None if name.startswith("\0") else _wrapper_inner(tensor)
+ if inner is not None:
+ if any(_same_view(t, view) for _, t in inner):
+ live.append(entry)
+ elif _same_view(tensor, view):
+ live.append(entry)
+ if live:
+ kept.append((buf, live))
+ layout[:] = kept
+
+
+def _prune_state_layout(state: dict) -> None:
+ layout = state.get("pin_layout")
+ if layout:
+ _prune_pin_layout(layout, state.get("host") or {}, state.get("buffer_host") or {})
+ if not layout:
+ state.pop("pin_layout", None)
+
+
def _upload_pinned_chunks(
module: Any, layout: list, host: dict, owner: dict, buffer_host: dict, device: Any
) -> int:
@@ -720,6 +751,8 @@ def _wrap_cpu_offload_hook(
pass
out = init_hook(mod)
_capture(mod)
+ # a weight cast or replaced on the device comes back as a new host tensor: release its pinned chunk now
+ _prune_state_layout(state)
version.clear()
buffer_version.clear()
return out
@@ -741,6 +774,8 @@ def _wrap_cpu_offload_hook(
for name in list(buffer_host):
if current.get(name) is not buffer_host[name]:
buffer_host.pop(name, None)
+ # changed while offloaded (a dtype cast, a replacement): its pinned chunk is not kept for nothing
+ _prune_state_layout(state)
if onload and not state.get("pin_tried"):
# Once per module, on its first onload, so loading pays nothing.
state["pin_tried"] = True
diff --git a/studio/backend/tests/test_diffusion_offload_fast_upload.py b/studio/backend/tests/test_diffusion_offload_fast_upload.py
index 59e28b1944b..6aa2789e7f0 100644
--- a/studio/backend/tests/test_diffusion_offload_fast_upload.py
+++ b/studio/backend/tests/test_diffusion_offload_fast_upload.py
@@ -335,6 +335,53 @@ def test_tensors_left_out_of_the_layout_are_not_uploaded_with_the_chunk(fake_pin
assert torch.equal(params[name].detach(), expect[name])
+def _layout_chunk_ptrs(module):
+ layout = module.__dict__[dm._KEEP_ATTR].get("pin_layout") or []
+ return {buf.data_ptr() for buf, _ in layout}
+
+
[email protected]("fast_upload", ["1", "0"])
[email protected]("cast_on", ["host", "device"])
+def test_a_dtype_change_releases_the_obsolete_pinned_chunks(
+ fake_pin, monkeypatch, fast_upload, cast_on
+):
+ # The SDXL fp32 VAE decode casts an offloaded component and back: its kept weights are new host tensors, so the
+ # chunks pinned for the old ones must not stay referenced by the layout (fast upload on or off).
+ monkeypatch.setenv(dm.OFFLOAD_FAST_UPLOAD_ENV, fast_upload)
+ pipe = _pipe("cpu", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ module = _onload(pipe)
+ old = _layout_chunk_ptrs(module)
+ assert old
+ if cast_on == "device":
+ module.half() # cast while onloaded: the offload brings back new host tensors
+ module._hf_hook.init_hook(module)
+ else:
+ module._hf_hook.init_hook(module)
+ module.half() # cast while offloaded: the next onload finds new host tensors
+ _onload(pipe)
+ assert not old & _layout_chunk_ptrs(module)
+ state = module.__dict__[dm._KEEP_ATTR]
+ live = {v.untyped_storage().data_ptr() for v in state["host"].values()}
+ assert not old & live
+
+
+def test_a_replaced_weight_drops_out_of_the_layout_and_the_rest_stays(fake_pin):
+ pipe = _pipe("cpu", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ module = _onload(pipe)
+ module._hf_hook.init_hook(module)
+ before = module.__dict__[dm._KEEP_ATTR]["pin_layout"]
+ names = {name for _, entries in before for name, _, _ in entries}
+ module[0].weight = torch.nn.Parameter(torch.zeros(16, 16))
+ _onload(pipe)
+ after = module.__dict__[dm._KEEP_ATTR]["pin_layout"]
+ left = {name for _, entries in after for name, _, _ in entries}
+ assert left == names - {"0.weight"}
+
+
def _quantised(scheme, n = N_LAYERS):
"""A Linear stack with torchao int8 / fp8 wrapper weights (Studio's quantised denoisers under low_vram)."""
quant = pytest.importorskip("torchao.quantization")
@@ -428,6 +475,27 @@ def test_a_wrapper_with_an_inner_tensor_outside_the_layout_takes_the_stock_copy(
]
[email protected]("scheme", ["int8", "fp8"])
+def test_the_layout_prune_keeps_unchanged_wrapper_weights_and_drops_replaced_ones(fake_pin, scheme):
+ # A wrapper weight's entries are its inner tensors: the prune must match those, not the wrapper itself.
+ module = _quantised(scheme)
+ host = {n: p.data for n, p in module.named_parameters()}
+ layout: list = []
+ assert dm._pin_host_weights(module, host, layout = layout) > 0
+ host = {n: p.data for n, p in module.named_parameters()}
+ names = [name for _, entries in layout for name, _, _ in entries]
+ wrapped = {n for n in names if dm._wrapper_inner(host.get(n)) is not None}
+ assert wrapped
+ dm._prune_pin_layout(layout, host, {})
+ assert [name for _, entries in layout for name, _, _ in entries] == names
+ # a dequantised (dtype-changed) or re-quantised weight is a new host tensor: its entries go, the rest stay
+ host["0.weight"] = host["0.weight"].dequantize()
+ host["1.weight"] = _quantised(scheme, n = 2)[1].weight.data
+ dm._prune_pin_layout(layout, host, {})
+ left = [name for _, entries in layout for name, _, _ in entries]
+ assert left == [n for n in names if n not in ("0.weight", "1.weight")]
+
+
def _quantised_pipe(scheme):
pipe = _pipe("cuda", "text_encoder", "transformer")
pipe.components["text_encoder"] = torch.nn.Linear(8, 64, dtype = torch.bfloat16)