Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio native cache write: read each weight from the live module at write time so offload can reclaim VRAM (#15461)

Daniel Hana2e639077a7e
diff --git a/studio/backend/core/inference/diffusion_native_cache.py b/studio/backend/core/inference/diffusion_native_cache.py
index 4603b800404..c1947b123d6 100644
--- a/studio/backend/core/inference/diffusion_native_cache.py
+++ b/studio/backend/core/inference/diffusion_native_cache.py
@@ -179,6 +179,65 @@ def _layer_facts(transformer: Any, spec: dict) -> Optional[dict]:
return layers or None
+def _moves_with_offload(tensor: Any) -> bool:
+ """Accelerator storage is what an offload release frees; a host tensor is a stable copy, kept as captured."""
+ return tensor.device.type != "cpu"
+
+
+def _wait_for_transfers(value: Any) -> None:
+ """Group offload onloads on its own stream with non_blocking copies: finish them before the writer reads."""
+ if value.device.type == "cuda":
+ from .diffusion_cuda_graph import synchronize_outside_capture
+
+ # A device sync inside another thread's CUDA graph capture would invalidate it.
+ synchronize_outside_capture(value.device)
+
+
+def _live_refs(transformer: Any, state: dict) -> dict:
+ """``state`` with module params / buffers swapped for live reads at write time, so a held state_dict does not
+ pin offloaded CUDA storage; hook-built tensors stay captured, a shape / dtype change fails the write."""
+ import torch
+
+ def resolve(name: str) -> Any:
+ prefix, _, attr = name.rpartition(".")
+ owner = transformer.get_submodule(prefix) if prefix else transformer
+ return getattr(owner, attr)
+
+ def getter(name: str, dtype: Any, shape: tuple) -> Any:
+ def get() -> Any:
+ value = resolve(name)
+ if value is None or value.dtype != dtype or tuple(value.shape) != shape:
+ raise RuntimeError(f"native cache write: {name} changed during the write")
+ # Detach before the sync: a later `param.data =` onload must not swap in a buffer still being copied.
+ value = value.detach()
+ _wait_for_transfers(value)
+ return value
+
+ return get
+
+ out: dict = {}
+ for name, tensor in state.items():
+ value = None
+ try:
+ if _moves_with_offload(tensor):
+ value = resolve(name)
+ # Same storage AND same view: a hook that hands back e.g. a transposed view keeps its captured tensor.
+ same = (
+ isinstance(value, torch.Tensor)
+ and value.dtype == tensor.dtype
+ and tuple(value.shape) == tuple(tensor.shape)
+ and value.data_ptr() == tensor.data_ptr()
+ and tuple(value.stride()) == tuple(tensor.stride())
+ )
+ except Exception: # noqa: BLE001 - not a module attribute (hook-built): keep the captured tensor
+ same = False
+ if same:
+ out[name] = _cache._FlatRef(getter(name, tensor.dtype, tuple(tensor.shape)), tensor)
+ else:
+ out[name] = tensor
+ return out
+
+
def save(
transformer: Any,
fam: Any,
@@ -256,6 +315,9 @@ def save(
f"native {component}: skipped ({name} is not a plain tensor)",
)
return None
+ name = tensor = (
+ None # the loop's last tensor would otherwise stay referenced through the write
+ )
nbytes = sum(_cache._tensor_bytes(t) for t in state.values())
budget = _cache.max_cache_bytes()
if budget is not None and nbytes > budget:
@@ -299,6 +361,7 @@ def save(
)
os.close(fd)
try:
+ state = _live_refs(transformer, state)
hasher = hashlib.sha256()
size = _cache.write_streaming_safetensors(
tmp, state, header, cancel_check = cancel_check, hasher = hasher
diff --git a/studio/backend/tests/test_native_cache_write_releases_offloaded_storage.py b/studio/backend/tests/test_native_cache_write_releases_offloaded_storage.py
new file mode 100644
index 00000000000..3a5bfdd8954
--- /dev/null
+++ b/studio/backend/tests/test_native_cache_write_releases_offloaded_storage.py
@@ -0,0 +1,268 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A native cache write in flight must not keep a module's old weight storage alive once the residency code moves
+the weights (group offload release swaps ``param.data`` back to host), and must still write the exact weights."""
+
+from __future__ import annotations
+
+import os
+import sys
+import threading
+import types
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+diffusers = pytest.importorskip("diffusers")
+safetensors_torch = pytest.importorskip("safetensors.torch")
+pytest.importorskip("accelerate")
+
+BACKEND = Path(__file__).resolve().parents[1]
+if str(BACKEND) not in sys.path:
+ sys.path.insert(0, str(BACKEND))
+REPO_ROOT = Path(__file__).resolve().parents[3]
+sys.path.insert(0, str(REPO_ROOT / "tests" / "_shared"))
+
+from real_accelerator import has_real_cuda # noqa: E402
+
+from core.inference import diffusion_native_cache as NC # noqa: E402
+from core.inference import diffusion_prequant_cache as C # noqa: E402
+from core.inference import diffusion_transformer_quant as tq # noqa: E402
+from core.inference.video_families import detect_video_family # noqa: E402
+
+FAMILY = "wan2.1-t2v-1.3b"
+CONFIG = dict(
+ patch_size = (1, 2, 2),
+ num_attention_heads = 2,
+ attention_head_dim = 256,
+ in_channels = 4,
+ out_channels = 4,
+ text_dim = 64,
+ freq_dim = 32,
+ ffn_dim = 1024,
+ num_layers = 2,
+ cross_attn_norm = True,
+ qk_norm = "rms_norm_across_heads",
+ eps = 1e-6,
+ rope_max_seq_len = 64,
+)
+
+
[email protected](autouse = True)
+def cache_root(tmp_path, monkeypatch):
+ root = tmp_path / "cache"
+ monkeypatch.setenv(C.ENV_DIR, str(root))
+ monkeypatch.delenv(C.ENV_MODE, raising = False)
+ monkeypatch.setattr(C, "_hub_source_stale", lambda shards, logger = None: False)
+ return root
+
+
+def base_dir(tmp_path):
+ root = tmp_path / "Wan2.1-T2V-1.3B-Diffusers"
+ torch.manual_seed(3)
+ diffusers.WanTransformer3DModel(**CONFIG).save_pretrained(
+ str(root / "transformer"), safe_serialization = True
+ )
+ return str(root)
+
+
+def _native(base_dir):
+ model = diffusers.WanTransformer3DModel.from_pretrained(
+ base_dir, subfolder = "transformer", torch_dtype = torch.bfloat16
+ )
+ view = types.SimpleNamespace(transformer = model)
+ assert (
+ tq._quantize_native(
+ view,
+ "int8",
+ family = FAMILY,
+ min_features = tq.DEFAULT_MIN_LINEAR_FEATURES,
+ act_int8 = True,
+ logger = None,
+ )
+ == "int8"
+ )
+ return model
+
+
+def _live_tensors(model):
+ out = [p for _n, p in model.named_parameters()]
+ out += [b for _n, b in model.named_buffers() if b is not None]
+ return out
+
+
+def _paused_save(monkeypatch, model, base_dir, during):
+ """Run ``NC.save`` on a thread, paused inside the file write; ``during(tensors)`` runs while it is paused."""
+ entered, resume = threading.Event(), threading.Event()
+ seen: dict = {}
+ original = C.write_streaming_safetensors
+
+ def hold(path, tensors, *a, **kw):
+ seen["tensors"] = tensors
+ entered.set()
+ assert resume.wait(60)
+ return original(path, tensors, *a, **kw)
+
+ monkeypatch.setattr(C, "write_streaming_safetensors", hold)
+ fam = detect_video_family("Wan-AI/Wan2.1-T2V-1.3B-Diffusers", override = FAMILY)
+ spec = NC.native_spec("int8", FAMILY, act_int8 = True)
+ result: dict = {}
+ thread = threading.Thread(
+ target = lambda: result.setdefault(
+ "path", NC.save(model, fam, spec, base_repo = base_dir, component = "transformer")
+ )
+ )
+ thread.start()
+ assert entered.wait(60), "the cache writer never reached the file write"
+ try:
+ during(seen.pop("tensors"))
+ finally:
+ resume.set()
+ thread.join(60)
+ assert not thread.is_alive()
+ return result.get("path")
+
+
+def _entry_tensors(path):
+ path = str(path)
+ if os.path.isdir(path):
+ path = os.path.join(path, C.FILE_NAME)
+ return safetensors_torch.load_file(path)
+
+
+def _as_if_on_the_accelerator(monkeypatch):
+ # CPU stands in for resident accelerator storage here.
+ monkeypatch.setattr(NC, "_moves_with_offload", lambda tensor: True)
+
+
+def test_writer_holds_no_old_storage_after_the_weights_move(monkeypatch, base_dir, cache_root):
+ _as_if_on_the_accelerator(monkeypatch)
+ model = _native(base_dir)
+ expected = {k: v.detach().clone() for k, v in model.state_dict().items()}
+ old_ptrs: set = set()
+ old_storage: list = []
+
+ def offload(tensors):
+ # what release_resident_groups does to a resident group: param.data swapped to another copy
+ for t in _live_tensors(model):
+ old_storage.append(t.data)
+ old_ptrs.add(t.data_ptr())
+ t.data = t.data.clone()
+ held = [v for v in tensors.values() if isinstance(v, torch.Tensor)]
+ stale = [v for v in held if v.data_ptr() in old_ptrs]
+ assert (
+ not stale
+ ), f"{len(stale)} tensors in the in-flight write still alias the moved storage"
+
+ path = _paused_save(monkeypatch, model, base_dir, offload)
+ assert path is not None
+ got = _entry_tensors(path)
+ assert set(got) == set(expected)
+ for k, v in expected.items():
+ assert got[k].dtype == v.dtype and torch.equal(got[k], v), k
+
+
+def test_a_weight_that_changes_shape_mid_write_leaves_no_entry(monkeypatch, base_dir, cache_root):
+ _as_if_on_the_accelerator(monkeypatch)
+ model = _native(base_dir)
+
+ def reshape(_tensors):
+ p = next(p for _n, p in model.named_parameters() if p.dim() == 2)
+ p.data = p.data[:1].clone()
+
+ assert _paused_save(monkeypatch, model, base_dir, reshape) is None
+ leftovers = [f for f in cache_root.rglob("*") if f.is_file() and C.FILE_NAME in f.name]
+ assert leftovers == [], leftovers
+
+
+def test_a_host_weight_is_written_as_captured_even_if_it_is_onloaded_mid_write(
+ monkeypatch, base_dir, cache_root
+):
+ """A host-resident weight keeps its captured copy, never an in-flight onload swapped in mid-write."""
+ model = _native(base_dir)
+ expected = {k: v.detach().clone() for k, v in model.state_dict().items()}
+
+ def onload(_tensors):
+ for t in _live_tensors(model):
+ if t.is_floating_point():
+ t.data = torch.full_like(t.data, 7)
+
+ path = _paused_save(monkeypatch, model, base_dir, onload)
+ assert path is not None
+ got = _entry_tensors(path)
+ for k, v in expected.items():
+ assert torch.equal(got[k], v), k
+
+
+def test_an_onload_right_after_the_sync_does_not_reach_the_written_bytes(
+ monkeypatch, base_dir, cache_root
+):
+ _as_if_on_the_accelerator(monkeypatch)
+ model = _native(base_dir)
+ expected = {k: v.detach().clone() for k, v in model.state_dict().items()}
+ by_ptr = {t.data_ptr(): t for t in _live_tensors(model) if t.is_floating_point()}
+
+ def sync_then_onload(value):
+ # a stream onload swapping in a buffer whose copy is still in flight, just after the writer's sync
+ owner = by_ptr.pop(value.data_ptr(), None)
+ if owner is not None:
+ owner.data = torch.full_like(owner.data, 7)
+
+ monkeypatch.setattr(NC, "_wait_for_transfers", sync_then_onload)
+ fam = detect_video_family("Wan-AI/Wan2.1-T2V-1.3B-Diffusers", override = FAMILY)
+ spec = NC.native_spec("int8", FAMILY, act_int8 = True)
+ path = NC.save(model, fam, spec, base_repo = base_dir, component = "transformer")
+ assert path is not None
+ got = _entry_tensors(path)
+ for k, v in expected.items():
+ assert torch.equal(got[k], v), k
+
+
[email protected](not has_real_cuda(), reason = "needs a CUDA GPU")
+def test_cuda_storage_is_reclaimed_while_the_write_is_in_flight(monkeypatch, base_dir):
+ baseline = torch.cuda.memory_allocated()
+ model = _native(base_dir).to("cuda")
+ expected = {k: v.detach().cpu() for k, v in model.state_dict().items()}
+ torch.cuda.synchronize()
+ seen: dict = {}
+
+ def offload(_tensors):
+ before = torch.cuda.memory_allocated()
+ for t in _live_tensors(model):
+ t.data = t.data.cpu()
+ _tensors = None # noqa: F841 - only the writer's own reference may remain
+ torch.cuda.synchronize()
+ seen["before"], seen["after"] = before, torch.cuda.memory_allocated()
+
+ path = _paused_save(monkeypatch, model, base_dir, offload)
+ assert seen["before"] > baseline
+ held = seen["after"] - baseline
+ assert held <= 0, f"{held} CUDA bytes still held by the in-flight write"
+ got = _entry_tensors(path)
+ for k, v in expected.items():
+ assert torch.equal(got[k], v), k
+
+
+def test_a_hook_built_view_of_a_weight_keeps_its_captured_tensor(monkeypatch, base_dir, cache_root):
+ _as_if_on_the_accelerator(monkeypatch)
+ model = _native(base_dir)
+ key = next(
+ k for k, v in model.state_dict().items() if v.dim() == 2 and v.shape[0] == v.shape[1] > 1
+ )
+
+ def hook(module, state, prefix, local_metadata):
+ state[key] = state[key].t() # same storage, same shape, different view
+ return state
+
+ model._register_state_dict_hook(hook)
+ want = {k: v.detach().clone() for k, v in model.state_dict().items()}
+ fam = detect_video_family("Wan-AI/Wan2.1-T2V-1.3B-Diffusers", override = FAMILY)
+ spec = NC.native_spec("int8", FAMILY, act_int8 = True)
+ path = NC.save(model, fam, spec, base_repo = base_dir, component = "transformer")
+ assert path is not None
+ got = _entry_tensors(path)
+ assert torch.equal(got[key], want[key])