Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: release a streamed encoder's safetensors mapping once its weights are pinned (#16914)
Daniel Han992185ae8e03
diff --git a/studio/backend/core/inference/diffusion_comfy_components.py b/studio/backend/core/inference/diffusion_comfy_components.pyindex 63844c1a898..749f6b8e8fe 100644--- a/studio/backend/core/inference/diffusion_comfy_components.py+++ b/studio/backend/core/inference/diffusion_comfy_components.py@@ -660,6 +660,10 @@ def build_text_encoder(for key, tensor in list(state.items()):if key.endswith(".bias") and tensor.is_floating_point():state[key] = tensor.to(dtype)+ from .prequant_safetensors import _unviewed++ # Views of the file mapping would keep it alive through the streaming pin's ``.data =`` re-point.+ state = {key: _unviewed(tensor) for key, tensor in state.items()}missing, unexpected = encoder.load_state_dict(state, strict = False, assign = True)bad_missing = [k for k in missing if k not in tied and not _is_dead(k, _DEAD_MISSING)]if bad_missing or unexpected:diff --git a/studio/backend/core/inference/prequant_safetensors.py b/studio/backend/core/inference/prequant_safetensors.pyindex ec15b0991b3..4e213d6dbef 100644--- a/studio/backend/core/inference/prequant_safetensors.py+++ b/studio/backend/core/inference/prequant_safetensors.py@@ -129,6 +129,20 @@ def plain_safetensors_supported() -> bool:return True+def _unviewed(tensor: Any) -> Any:+ """``tensor`` as a base-less alias of the same storage (no copy). safetensors hands back views of one whole-file+ mapping; a view keeps that base alive through ``.data =`` (which is how a streamed buffer is re-pointed at its+ pinned copy), so one buffer would keep the full file mapped for the life of the model."""+ if getattr(tensor, "_base", None) is None or type(tensor).__name__ != "Tensor":+ return tensor+ import torch++ out = torch.empty(0, dtype = tensor.dtype, device = tensor.device)+ return out.set_(+ tensor.untyped_storage(), tensor.storage_offset(), tensor.size(), tensor.stride()+ )++def load_plain_prequant_safetensors(path: str,*,@@ -169,9 +183,11 @@ def load_plain_prequant_safetensors(if json.loads(raw.get(name) or "null") != {"_type": "Tensor"}:raise ValueError(f"{path} requires a tensor-subclass reader for {name!r}")skip = frozenset(skip_names)- state_dict = {name: handle.get_tensor(name) for name in names if name not in skip}+ state_dict = {+ name: _unviewed(handle.get_tensor(name)) for name in names if name not in skip+ }state_dict.update(- (key[len(UNSLOTH_ROOT_PREFIX) :], handle.get_tensor(key)) for key in roots+ (key[len(UNSLOTH_ROOT_PREFIX) :], _unviewed(handle.get_tensor(key))) for key in roots)return {"format": str(fmt), "state_dict": state_dict, "metadata": metadata}diff --git a/studio/backend/tests/test_diffusion_comfy_components.py b/studio/backend/tests/test_diffusion_comfy_components.pyindex f9e07aca528..9af50b61e8c 100644--- a/studio/backend/tests/test_diffusion_comfy_components.py+++ b/studio/backend/tests/test_diffusion_comfy_components.py@@ -431,6 +431,45 @@ def test_qwen3_comfy_layout_int8_convrot_stays_int8(tmp_path):assert cos > 0.98+def test_int8_convrot_encoder_releases_the_file_once_its_weights_are_repointed(tmp_path):+ """Streaming re-points every host tensor at its pinned copy with ``.data =``; an int8 code buffer that was still a+ view of the file mapping kept the whole file mapped (and undeletable on Windows) for the model's life."""+ import gc+ import os+ import sys++ cfg, ref = _tiny_qwen3()+ group = 16+ out = {}+ for key, value in ref.state_dict().items():+ comfy_key = "model." + key+ if ".mlp." in key and key.endswith(".weight"):+ codes, scale = _convrot_int8(value, group)+ stem = comfy_key[: -len(".weight")]+ out[comfy_key] = codes+ out[stem + ".weight_scale"] = scale+ out[stem + ".comfy_quant"] = _quant_blob(+ {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": group}+ )+ else:+ out[comfy_key] = value.to(torch.bfloat16)+ path = _save(tmp_path / "qwen_3_4b_int8_convrot.safetensors", out)+ enc = C.build_text_encoder(+ path, encoder_cls = transformers.Qwen3Model, config = cfg, dtype = torch.bfloat16+ )+ expected = {k: v.clone() for k, v in enc.state_dict().items()}+ for tensor in list(enc.parameters()) + list(enc.buffers()):+ tensor.data = tensor.data.clone() # the pinned copy, as the group pinner re-points it+ gc.collect()+ if sys.platform.startswith("linux"):+ real = os.path.realpath(path)+ with open("/proc/self/maps", encoding = "utf-8") as fh:+ assert not any(line.rstrip().endswith(real) for line in fh)+ os.remove(path)+ for key, value in enc.state_dict().items():+ assert torch.equal(value, expected[key]), key++def test_qwen3_bf16_strip_model_prefix_and_dead_lm_head(tmp_path):cfg, ref = _tiny_qwen3()out = {"model." + k: v for k, v in ref.state_dict().items()}diff --git a/studio/backend/tests/test_prequant_safetensors.py b/studio/backend/tests/test_prequant_safetensors.pyindex 1eff12bff6d..14b067f4ade 100644--- a/studio/backend/tests/test_prequant_safetensors.py+++ b/studio/backend/tests/test_prequant_safetensors.py@@ -6,6 +6,7 @@from __future__ import annotationsimport json+import osimport sysimport pytest@@ -635,6 +636,40 @@ def test_plain_encoder_needs_no_torchao(plain_checkpoint):assert loaded["metadata"]["scheme"] == "fp8"+def _mapped(path):+ """Whether this process still maps ``path`` (Linux only; None elsewhere)."""+ if not sys.platform.startswith("linux"):+ return None+ real = os.path.realpath(path)+ with open("/proc/self/maps", encoding = "utf-8") as fh:+ return any(line.rstrip().endswith(real) for line in fh)+++def test_a_buffer_repointed_at_its_pinned_copy_releases_the_file_mapping(plain_checkpoint):+ """Streaming re-points each host weight at its pinned copy with ``.data =``. A view keeps its base through that,+ so a buffer loaded straight from the file kept the whole safetensors mapping alive for the model's life (8.7 GiB+ for Qwen-Image-2.1's int8 encoder; on Windows the open mapping also blocks deleting the file)."""+ import gc++ import torch++ path = plain_checkpoint(+ tensors = {"encoder.weight": torch.arange(4096 * 64, dtype = torch.float32).reshape(4096, 64)}+ )+ module = torch.nn.Module()+ module.register_buffer(+ "weight", ps.load_plain_prequant_safetensors(path)["state_dict"]["encoder.weight"]+ )+ expected = module.weight.clone()+ assert _mapped(path) in (True, None)+ # the pinned copy, as _GroupPinner / the pin arena re-point it+ module.weight.data = module.weight.clone()+ gc.collect()+ assert _mapped(path) in (False, None)+ os.remove(path)+ assert torch.equal(module.weight, expected)++@pytest.mark.parametrize("header",[