Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: streamed torchao weights no longer crash after a compile (Cannot swap t1 ... weakref) (#17141)

Daniel Hanc9489258ae71
diff --git a/studio/backend/core/inference/diffusion_offload_prefetch.py b/studio/backend/core/inference/diffusion_offload_prefetch.py
index 96e4e6e3fc0..2d94e33d5b0 100644
--- a/studio/backend/core/inference/diffusion_offload_prefetch.py
+++ b/studio/backend/core/inference/diffusion_offload_prefetch.py
@@ -440,11 +440,9 @@ class GroupPrefetcher:
_copy_into(buffers[i], src)
_point_at(t, buffers[i], is_torchao(t), swap)
continue
- moved = _to_device(src, self.device)
- if is_torchao(t) and swap is not None:
- swap(t, moved)
- else:
- t.data = moved
+ # a wrapper already on the device takes the new inner tensors: swap_tensors would refuse a weight a
+ # compile traced (its fake-mode memo keeps a weakref to it after the compile)
+ _point_at(t, _to_device(src, self.device), is_torchao(t), swap)
if buffers is not None:
self.slot_owner[where] = id(group)
self.stats["slot_fills"] = self.stats.get("slot_fills", 0) + 1
diff --git a/studio/backend/tests/test_diffusion_offload_prefetch.py b/studio/backend/tests/test_diffusion_offload_prefetch.py
index d22db9ebd61..b1c35609bdd 100644
--- a/studio/backend/tests/test_diffusion_offload_prefetch.py
+++ b/studio/backend/tests/test_diffusion_offload_prefetch.py
@@ -691,3 +691,38 @@ def test_torchao_int8_weights_stream_bit_identical(version):
go = pytest.importorskip("diffusers.hooks.group_offloading")
# the torchao wrapper reports its onload device, so check the inner tensors
assert dm._placed_on(net.proj_in.weight, "cpu", go._is_torchao_tensor)
+
+
+def test_streamed_onload_of_a_compiled_wrapper_weight_does_not_swap(monkeypatch):
+ """A compile keeps a weakref to every wrapper-subclass weight it traced (its fake-mode memo outlives it), and
+ swap_tensors refuses such a tensor: FLUX.3 INT8 streamed at 6 GiB failed every render with 'Cannot swap t1 because
+ it has weakref associated with it'. A weight whose wrapper already reports the device needs no swap."""
+ import weakref
+
+ two = pytest.importorskip("torch.testing._internal.two_tensor")
+ pf, groups, gpu = _frozen_gpu_prefetcher(monkeypatch, pinned = True, groups_n = 1, tensors = 1)
+ lin = torch.nn.Linear(8, 8)
+ w = lin.weight.detach().clone()
+ lin.weight = torch.nn.Parameter(two.TwoTensor(w, w.clone()), requires_grad = False)
+ torch._dynamo.reset()
+ with torch.no_grad():
+ torch.compile(lin, backend = "eager")(torch.randn(2, 8))
+ assert weakref.getweakrefs(lin.weight)
+ host = two.TwoTensor(w + 1, w + 2)
+ monkeypatch.setattr(type(host), "is_pinned", lambda self: True, raising = False)
+ group = groups[0]
+ group.parameters = [lin.weight]
+ group.cpu_param_dict = {lin.weight: host}
+ monkeypatch.setattr(
+ op,
+ "_go",
+ lambda: types.SimpleNamespace(
+ _is_torchao_tensor = lambda t: isinstance(t, two.TwoTensor),
+ _swap_torchao_tensor = torch.utils.swap_tensors,
+ ),
+ )
+ param = lin.weight
+ pf._issue(group)
+ assert lin.weight is param
+ assert torch.equal(param.a, w + 1) and torch.equal(param.b, w + 2)
+ torch._dynamo.reset()