Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: streamed torchao weights no longer crash after a compile (Cannot swap t1 ... weakref) (#17141)
Daniel Hanc9489258ae71
diff --git a/studio/backend/core/inference/diffusion_offload_prefetch.py b/studio/backend/core/inference/diffusion_offload_prefetch.pyindex 96e4e6e3fc0..2d94e33d5b0 100644--- a/studio/backend/core/inference/diffusion_offload_prefetch.py+++ b/studio/backend/core/inference/diffusion_offload_prefetch.py@@ -440,11 +440,9 @@ class GroupPrefetcher:_copy_into(buffers[i], src)_point_at(t, buffers[i], is_torchao(t), swap)continue- moved = _to_device(src, self.device)- if is_torchao(t) and swap is not None:- swap(t, moved)- else:- t.data = moved+ # a wrapper already on the device takes the new inner tensors: swap_tensors would refuse a weight a+ # compile traced (its fake-mode memo keeps a weakref to it after the compile)+ _point_at(t, _to_device(src, self.device), is_torchao(t), swap)if buffers is not None:self.slot_owner[where] = id(group)self.stats["slot_fills"] = self.stats.get("slot_fills", 0) + 1diff --git a/studio/backend/tests/test_diffusion_offload_prefetch.py b/studio/backend/tests/test_diffusion_offload_prefetch.pyindex d22db9ebd61..b1c35609bdd 100644--- a/studio/backend/tests/test_diffusion_offload_prefetch.py+++ b/studio/backend/tests/test_diffusion_offload_prefetch.py@@ -691,3 +691,38 @@ def test_torchao_int8_weights_stream_bit_identical(version):go = pytest.importorskip("diffusers.hooks.group_offloading")# the torchao wrapper reports its onload device, so check the inner tensorsassert dm._placed_on(net.proj_in.weight, "cpu", go._is_torchao_tensor)+++def test_streamed_onload_of_a_compiled_wrapper_weight_does_not_swap(monkeypatch):+ """A compile keeps a weakref to every wrapper-subclass weight it traced (its fake-mode memo outlives it), and+ swap_tensors refuses such a tensor: FLUX.3 INT8 streamed at 6 GiB failed every render with 'Cannot swap t1 because+ it has weakref associated with it'. A weight whose wrapper already reports the device needs no swap."""+ import weakref++ two = pytest.importorskip("torch.testing._internal.two_tensor")+ pf, groups, gpu = _frozen_gpu_prefetcher(monkeypatch, pinned = True, groups_n = 1, tensors = 1)+ lin = torch.nn.Linear(8, 8)+ w = lin.weight.detach().clone()+ lin.weight = torch.nn.Parameter(two.TwoTensor(w, w.clone()), requires_grad = False)+ torch._dynamo.reset()+ with torch.no_grad():+ torch.compile(lin, backend = "eager")(torch.randn(2, 8))+ assert weakref.getweakrefs(lin.weight)+ host = two.TwoTensor(w + 1, w + 2)+ monkeypatch.setattr(type(host), "is_pinned", lambda self: True, raising = False)+ group = groups[0]+ group.parameters = [lin.weight]+ group.cpu_param_dict = {lin.weight: host}+ monkeypatch.setattr(+ op,+ "_go",+ lambda: types.SimpleNamespace(+ _is_torchao_tensor = lambda t: isinstance(t, two.TwoTensor),+ _swap_torchao_tensor = torch.utils.swap_tensors,+ ),+ )+ param = lin.weight+ pf._issue(group)+ assert lin.weight is param+ assert torch.equal(param.a, w + 1) and torch.equal(param.b, w + 2)+ torch._dynamo.reset()