Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: release a streamed encoder's safetensors mapping once its weights are pinned (#16914)

Daniel Han992185ae8e03
diff --git a/studio/backend/core/inference/diffusion_comfy_components.py b/studio/backend/core/inference/diffusion_comfy_components.py
index 63844c1a898..749f6b8e8fe 100644
--- a/studio/backend/core/inference/diffusion_comfy_components.py
+++ b/studio/backend/core/inference/diffusion_comfy_components.py
@@ -660,6 +660,10 @@ def build_text_encoder(
for key, tensor in list(state.items()):
if key.endswith(".bias") and tensor.is_floating_point():
state[key] = tensor.to(dtype)
+ from .prequant_safetensors import _unviewed
+
+ # Views of the file mapping would keep it alive through the streaming pin's ``.data =`` re-point.
+ state = {key: _unviewed(tensor) for key, tensor in state.items()}
missing, unexpected = encoder.load_state_dict(state, strict = False, assign = True)
bad_missing = [k for k in missing if k not in tied and not _is_dead(k, _DEAD_MISSING)]
if bad_missing or unexpected:
diff --git a/studio/backend/core/inference/prequant_safetensors.py b/studio/backend/core/inference/prequant_safetensors.py
index ec15b0991b3..4e213d6dbef 100644
--- a/studio/backend/core/inference/prequant_safetensors.py
+++ b/studio/backend/core/inference/prequant_safetensors.py
@@ -129,6 +129,20 @@ def plain_safetensors_supported() -> bool:
return True
+def _unviewed(tensor: Any) -> Any:
+ """``tensor`` as a base-less alias of the same storage (no copy). safetensors hands back views of one whole-file
+ mapping; a view keeps that base alive through ``.data =`` (which is how a streamed buffer is re-pointed at its
+ pinned copy), so one buffer would keep the full file mapped for the life of the model."""
+ if getattr(tensor, "_base", None) is None or type(tensor).__name__ != "Tensor":
+ return tensor
+ import torch
+
+ out = torch.empty(0, dtype = tensor.dtype, device = tensor.device)
+ return out.set_(
+ tensor.untyped_storage(), tensor.storage_offset(), tensor.size(), tensor.stride()
+ )
+
+
def load_plain_prequant_safetensors(
path: str,
*,
@@ -169,9 +183,11 @@ def load_plain_prequant_safetensors(
if json.loads(raw.get(name) or "null") != {"_type": "Tensor"}:
raise ValueError(f"{path} requires a tensor-subclass reader for {name!r}")
skip = frozenset(skip_names)
- state_dict = {name: handle.get_tensor(name) for name in names if name not in skip}
+ state_dict = {
+ name: _unviewed(handle.get_tensor(name)) for name in names if name not in skip
+ }
state_dict.update(
- (key[len(UNSLOTH_ROOT_PREFIX) :], handle.get_tensor(key)) for key in roots
+ (key[len(UNSLOTH_ROOT_PREFIX) :], _unviewed(handle.get_tensor(key))) for key in roots
)
return {"format": str(fmt), "state_dict": state_dict, "metadata": metadata}
diff --git a/studio/backend/tests/test_diffusion_comfy_components.py b/studio/backend/tests/test_diffusion_comfy_components.py
index f9e07aca528..9af50b61e8c 100644
--- a/studio/backend/tests/test_diffusion_comfy_components.py
+++ b/studio/backend/tests/test_diffusion_comfy_components.py
@@ -431,6 +431,45 @@ def test_qwen3_comfy_layout_int8_convrot_stays_int8(tmp_path):
assert cos > 0.98
+def test_int8_convrot_encoder_releases_the_file_once_its_weights_are_repointed(tmp_path):
+ """Streaming re-points every host tensor at its pinned copy with ``.data =``; an int8 code buffer that was still a
+ view of the file mapping kept the whole file mapped (and undeletable on Windows) for the model's life."""
+ import gc
+ import os
+ import sys
+
+ cfg, ref = _tiny_qwen3()
+ group = 16
+ out = {}
+ for key, value in ref.state_dict().items():
+ comfy_key = "model." + key
+ if ".mlp." in key and key.endswith(".weight"):
+ codes, scale = _convrot_int8(value, group)
+ stem = comfy_key[: -len(".weight")]
+ out[comfy_key] = codes
+ out[stem + ".weight_scale"] = scale
+ out[stem + ".comfy_quant"] = _quant_blob(
+ {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": group}
+ )
+ else:
+ out[comfy_key] = value.to(torch.bfloat16)
+ path = _save(tmp_path / "qwen_3_4b_int8_convrot.safetensors", out)
+ enc = C.build_text_encoder(
+ path, encoder_cls = transformers.Qwen3Model, config = cfg, dtype = torch.bfloat16
+ )
+ expected = {k: v.clone() for k, v in enc.state_dict().items()}
+ for tensor in list(enc.parameters()) + list(enc.buffers()):
+ tensor.data = tensor.data.clone() # the pinned copy, as the group pinner re-points it
+ gc.collect()
+ if sys.platform.startswith("linux"):
+ real = os.path.realpath(path)
+ with open("/proc/self/maps", encoding = "utf-8") as fh:
+ assert not any(line.rstrip().endswith(real) for line in fh)
+ os.remove(path)
+ for key, value in enc.state_dict().items():
+ assert torch.equal(value, expected[key]), key
+
+
def test_qwen3_bf16_strip_model_prefix_and_dead_lm_head(tmp_path):
cfg, ref = _tiny_qwen3()
out = {"model." + k: v for k, v in ref.state_dict().items()}
diff --git a/studio/backend/tests/test_prequant_safetensors.py b/studio/backend/tests/test_prequant_safetensors.py
index 1eff12bff6d..14b067f4ade 100644
--- a/studio/backend/tests/test_prequant_safetensors.py
+++ b/studio/backend/tests/test_prequant_safetensors.py
@@ -6,6 +6,7 @@
from __future__ import annotations
import json
+import os
import sys
import pytest
@@ -635,6 +636,40 @@ def test_plain_encoder_needs_no_torchao(plain_checkpoint):
assert loaded["metadata"]["scheme"] == "fp8"
+def _mapped(path):
+ """Whether this process still maps ``path`` (Linux only; None elsewhere)."""
+ if not sys.platform.startswith("linux"):
+ return None
+ real = os.path.realpath(path)
+ with open("/proc/self/maps", encoding = "utf-8") as fh:
+ return any(line.rstrip().endswith(real) for line in fh)
+
+
+def test_a_buffer_repointed_at_its_pinned_copy_releases_the_file_mapping(plain_checkpoint):
+ """Streaming re-points each host weight at its pinned copy with ``.data =``. A view keeps its base through that,
+ so a buffer loaded straight from the file kept the whole safetensors mapping alive for the model's life (8.7 GiB
+ for Qwen-Image-2.1's int8 encoder; on Windows the open mapping also blocks deleting the file)."""
+ import gc
+
+ import torch
+
+ path = plain_checkpoint(
+ tensors = {"encoder.weight": torch.arange(4096 * 64, dtype = torch.float32).reshape(4096, 64)}
+ )
+ module = torch.nn.Module()
+ module.register_buffer(
+ "weight", ps.load_plain_prequant_safetensors(path)["state_dict"]["encoder.weight"]
+ )
+ expected = module.weight.clone()
+ assert _mapped(path) in (True, None)
+ # the pinned copy, as _GroupPinner / the pin arena re-point it
+ module.weight.data = module.weight.clone()
+ gc.collect()
+ assert _mapped(path) in (False, None)
+ os.remove(path)
+ assert torch.equal(module.weight, expected)
+
+
@pytest.mark.parametrize(
"header",
[