Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: reload a released text encoder from its own safetensors files, no snapshot write (#17184)

Daniel Hanf53fecac4a18
diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.py
index 11a06e46946..df1fb41508c 100644
--- a/studio/backend/core/inference/diffusion_te_prequant.py
+++ b/studio/backend/core/inference/diffusion_te_prequant.py
@@ -1218,6 +1218,7 @@ def load_prequant_text_encoder(
encoder_cls, config, state_dict, dtype = dtype, trim = trim
)
setattr(encoder, TE_PREQUANT_SCHEME_ATTR, "int8")
+ _tag_weight_files(encoder, path)
if logger is not None:
logger.info(
"diffusion.te_prequant: loaded %s int8 ConvRot weight-only checkpoint (%s)",
@@ -1264,6 +1265,7 @@ def load_prequant_text_encoder(
target.dtype = dtype
_cast_fp8(encoder, target)
setattr(encoder, TE_PREQUANT_SCHEME_ATTR, "fp8")
+ _tag_weight_files(encoder, path)
if logger is not None:
# The FILE's scheme, not the request's (an fp8 fallback used to log int8).
logger.info(
@@ -1738,6 +1740,13 @@ def _build_int8_convrot_encoder(
return encoder
+def _tag_weight_files(encoder: Any, path: Any) -> None:
+ """Record the source file so a release reloads matching tensors from it instead of snapshotting."""
+ from .diffusion_te_release import WEIGHT_FILES_ATTR
+ if str(path).endswith(".safetensors"):
+ setattr(encoder, WEIGHT_FILES_ATTR, (str(path),))
+
+
def _fill_tied_aliases(encoder: Any, state_dict: dict) -> None:
"""Give a key the file leaves out because the model ties it to another one the source of that tie.
diff --git a/studio/backend/core/inference/diffusion_te_release.py b/studio/backend/core/inference/diffusion_te_release.py
index e3a2d926b12..51daac4e02a 100644
--- a/studio/backend/core/inference/diffusion_te_release.py
+++ b/studio/backend/core/inference/diffusion_te_release.py
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Releases text encoders during denoise on unified memory (disk snapshot, bit-exact reload)."""
+"""Releases text encoders during denoise on unified memory (bit-exact reload from the encoder's own safetensors files,
+else from a disk snapshot)."""
from __future__ import annotations
@@ -15,6 +16,13 @@ from pathlib import Path
from typing import Any, Optional
RELEASE_ENV = "UNSLOTH_DIFFUSION_RELEASE_TEXT_ENCODER"
+# 0: snapshot every released tensor, as before, instead of reloading the ones the encoder's own files hold byte for byte.
+SOURCE_REUSE_ENV = "UNSLOTH_DIFFUSION_TE_RELEASE_FROM_SOURCE"
+# Set by a loader that knows the safetensors files an encoder was built from; else ``name_or_path`` when it is a folder.
+WEIGHT_FILES_ATTR = "_unsloth_weight_files"
+# A folder with more files than this is not one encoder's checkpoint.
+_MAX_SOURCE_FILES = 64
+_MAX_CANDIDATES = 4
TEXT_ENCODER_ATTRS = ("text_encoder", "text_encoder_2", "text_encoder_3", "text_encoder_4")
# Tensors below this stay resident: norms, biases and rotary tables are a rounding error next to the projections.
_MIN_RELEASE_BYTES = 1 << 20
@@ -33,6 +41,63 @@ def release_override() -> Optional[bool]:
return None
+def source_reuse_enabled() -> bool:
+ return (os.environ.get(SOURCE_REUSE_ENV) or "").strip().lower() not in (
+ "0",
+ "false",
+ "no",
+ "off",
+ )
+
+
+def _source_files(encoder: Any) -> list[Path]:
+ tagged = getattr(encoder, WEIGHT_FILES_ATTR, None)
+ if tagged:
+ files = [Path(str(f)) for f in tagged]
+ else:
+ files = []
+ for name in (
+ getattr(encoder, "name_or_path", None),
+ getattr(getattr(encoder, "config", None), "_name_or_path", None),
+ ):
+ try:
+ folder = Path(str(name)) if name else None
+ if folder is not None and folder.is_dir():
+ files = sorted(folder.glob("*.safetensors"))
+ break
+ except OSError:
+ continue
+ files = [f for f in files if f.suffix == ".safetensors" and f.is_file()]
+ return files if len(files) <= _MAX_SOURCE_FILES else []
+
+
+def _safetensors_dtype(name: str) -> Any:
+ import torch
+ return {
+ "F64": torch.float64,
+ "F32": torch.float32,
+ "F16": torch.float16,
+ "BF16": torch.bfloat16,
+ "I64": torch.int64,
+ "I32": torch.int32,
+ "I16": torch.int16,
+ "I8": torch.int8,
+ "U8": torch.uint8,
+ "BOOL": torch.bool,
+ "F8_E4M3": getattr(torch, "float8_e4m3fn", None),
+ "F8_E5M2": getattr(torch, "float8_e5m2", None),
+ }.get(str(name))
+
+
+def _same_bytes(a: Any, b: Any) -> bool:
+ import torch
+ if a.dtype != b.dtype or tuple(a.shape) != tuple(b.shape):
+ return False
+ return torch.equal(
+ a.contiguous().reshape(-1).view(torch.uint8), b.contiguous().reshape(-1).view(torch.uint8)
+ )
+
+
def release_wanted(plan: Any) -> tuple[bool, str]:
"""``(wanted, reason)``: auto engages only on unified memory when weights plus generation headroom exceed the
safe budget."""
@@ -124,9 +189,15 @@ class TextEncoderReleaser:
self._released = False
self._snapshot: Optional[Path] = None
self._shards: list[Path] = []
+ self._prepared = False
+ # Open handles keep a source file's bytes readable even if the path is replaced or deleted meanwhile.
+ self._source_handles: list[Any] = []
+ # key -> (index into _source_handles, tensor name in that file)
+ self._from_source: dict[str, tuple[int, str]] = {}
# key -> (owner module, leaf name, is_parameter)
self._slots: dict[str, tuple[Any, str, bool]] = {}
self._released_bytes = 0
+ self._bytes: dict[str, int] = {}
self._hooks: list[Any] = []
self.reloads = 0
self.releases = 0
@@ -175,10 +246,81 @@ class TextEncoderReleaser:
for owner, leaf, is_param in self._slots.values()
)
+ def _tensor_bytes(self, key: str) -> int:
+ return self._bytes.get(key, 0)
+
def _tensor(self, key: str) -> Any:
owner, leaf, is_param = self._slots[key]
return (owner._parameters if is_param else owner._buffers)[leaf]
+ def _match_sources(self) -> None:
+ """Map tensors whose bytes match the encoder's own files to them; the rest are snapshotted."""
+ import torch
+ from safetensors import safe_open
+
+ for attr, encoder in self._encoders:
+ keys = [
+ k for k in self._slots if k.startswith(attr + ".") and k not in self._from_source
+ ]
+ if not keys:
+ continue
+ by_suffix: dict[str, list[tuple[int, str]]] = {}
+ for path in _source_files(encoder):
+ try:
+ handle = safe_open(str(path), framework = "pt", device = "cpu")
+ names = list(handle.keys())
+ except Exception: # noqa: BLE001 - an unreadable file is just not a source
+ continue
+ self._source_handles.append(handle)
+ index = len(self._source_handles) - 1
+ for name in names:
+ parts = name.split(".")
+ for i in range(len(parts)):
+ by_suffix.setdefault(".".join(parts[i:]), []).append((index, name))
+ if not by_suffix:
+ continue
+ for key in keys:
+ parts = [p for p in key[len(attr) + 1 :].split(".") if p]
+ candidates: list[tuple[int, str]] = []
+ # Longest matching name suffix first: transformers may nest the checkpoint's names one level deeper.
+ for i in range(len(parts)):
+ candidates = by_suffix.get(".".join(parts[i:]), [])
+ if candidates:
+ break
+ tensor = self._tensor(key).detach()
+ for index, name in candidates[:_MAX_CANDIDATES]:
+ handle = self._source_handles[index]
+ try:
+ meta = handle.get_slice(name)
+ if _safetensors_dtype(meta.get_dtype()) != tensor.dtype or tuple(
+ meta.get_shape()
+ ) != tuple(tensor.shape):
+ continue
+ with torch.no_grad():
+ if _same_bytes(handle.get_tensor(name), tensor.to("cpu")):
+ self._from_source[key] = (index, name)
+ break
+ except Exception: # noqa: BLE001 - unreadable: snapshot this tensor instead
+ continue
+ used = sorted({index for index, _ in self._from_source.values()})
+ remap = {old: new for new, old in enumerate(used)}
+ self._source_handles = [self._source_handles[i] for i in used]
+ self._from_source = {k: (remap[i], name) for k, (i, name) in self._from_source.items()}
+
+ def _prepare(self) -> None:
+ if source_reuse_enabled():
+ try:
+ self._match_sources()
+ except Exception as exc: # noqa: BLE001 - reuse is an optimisation: snapshot everything
+ self._from_source = {}
+ self._source_handles = []
+ self._note(
+ "info", "diffusion.text_encoder_release: snapshotting every tensor: %s", exc
+ )
+ if any(key not in self._from_source for key in self._slots):
+ self._write_snapshot()
+ self._prepared = True
+
def _write_snapshot(self) -> None:
import torch
from safetensors.torch import save_file
@@ -186,7 +328,11 @@ class TextEncoderReleaser:
root = _snapshot_root()
root.mkdir(parents = True, exist_ok = True)
_sweep_stale(root)
- need = self.releasable_bytes
+ need = sum(
+ self._tensor(key).numel() * self._tensor(key).element_size()
+ for key in self._slots
+ if key not in self._from_source
+ )
free = shutil.disk_usage(root).free
if free < need + (1 << 30):
raise RuntimeError(
@@ -212,6 +358,8 @@ class TextEncoderReleaser:
with torch.no_grad():
for key in self._slots:
+ if key in self._from_source:
+ continue
tensor = self._tensor(key).detach()
batch[key] = tensor.to("cpu", copy = True).contiguous()
size += tensor.numel() * tensor.element_size()
@@ -235,17 +383,20 @@ class TextEncoderReleaser:
for key in self._slots:
if not _plain_tensor(self._tensor(key)):
raise RuntimeError(f"{key} is not a plain dense tensor")
- if self._snapshot is None:
- self._write_snapshot()
+ if not self._prepared:
+ self._prepare()
except Exception as exc: # noqa: BLE001 - releasing is an optimisation: stay resident
self._note("warning", "diffusion.text_encoder_release: kept resident: %s", exc)
self._slots = {}
+ self._from_source = {}
+ self._source_handles = []
return 0
released = 0
with torch.no_grad():
for key, (owner, leaf, is_param) in self._slots.items():
tensor = self._tensor(key)
- released += tensor.numel() * tensor.element_size()
+ self._bytes[key] = tensor.numel() * tensor.element_size()
+ released += self._bytes[key]
empty = torch.empty(0, dtype = tensor.dtype, device = tensor.device)
if is_param:
owner._parameters[leaf].data = empty
@@ -255,11 +406,17 @@ class TextEncoderReleaser:
self._released_bytes = released
self.releases += 1
_empty_device_cache()
+ from_source = sum(
+ self._tensor_bytes(key) for key in self._from_source if key in self._slots
+ )
self._note(
"info",
- "diffusion.text_encoder_release: released %.2f GB of text-encoder weights for the denoise (%.2f s)",
+ "diffusion.text_encoder_release: released %.2f GB of text-encoder weights for the denoise (%.2f s; "
+ "%.2f GB reload from the encoder's own files, %.2f GB from a snapshot)",
released / 1e9,
time.monotonic() - started,
+ from_source / 1e9,
+ (released - from_source) / 1e9,
)
return released
@@ -273,6 +430,14 @@ class TextEncoderReleaser:
return
started = time.monotonic()
with torch.no_grad():
+ for key, (index, name) in self._from_source.items():
+ owner, leaf, is_param = self._slots[key]
+ current = (owner._parameters if is_param else owner._buffers)[leaf]
+ restored = self._source_handles[index].get_tensor(name).to(current.device)
+ if is_param:
+ owner._parameters[leaf].data = restored
+ else:
+ owner._buffers[leaf] = restored
for shard in self._shards:
with safe_open(str(shard), framework = "pt", device = "cpu") as handle:
for key in handle.keys():
@@ -301,6 +466,8 @@ class TextEncoderReleaser:
except Exception: # noqa: BLE001
pass
self._hooks = []
+ self._from_source = {}
+ self._source_handles = []
self._finalizer()
def _note(self, level: str, msg: str, *args: Any) -> None:
diff --git a/studio/backend/tests/test_diffusion_flux3_te_prequant.py b/studio/backend/tests/test_diffusion_flux3_te_prequant.py
index 833c332f036..4c4dda6f7ef 100644
--- a/studio/backend/tests/test_diffusion_flux3_te_prequant.py
+++ b/studio/backend/tests/test_diffusion_flux3_te_prequant.py
@@ -420,3 +420,26 @@ def test_flux3_assembly_hands_the_staged_dir_to_the_prequant_encoder(tmp_path, m
"root": staged,
"te": "enc",
}
+
+
+def test_released_int8_convrot_encoder_reloads_from_its_own_file(mirror, tmp_path, monkeypatch):
+ """The hosted int8 file is the encoder byte for byte, so releasing it on unified memory writes no snapshot."""
+ from core.inference import diffusion_te_release as te_release
+
+ monkeypatch.delenv(te_release.SOURCE_REUSE_ENV, raising = False)
+ monkeypatch.setattr(te_release, "_MIN_RELEASE_BYTES", 0)
+ monkeypatch.setattr(te_release, "_snapshot_root", lambda: tmp_path / "snap")
+ encoder = _pipe_kwargs("flux.3-klein", "int8")["text_encoder"]
+ _base, repo, stem = VARIANTS["flux.3-klein"]
+ assert getattr(encoder, te_release.WEIGHT_FILES_ATTR) == (str(mirror / repo / _names(stem)[0]),)
+ ids = torch.randint(0, VOCAB, (1, 9))
+ before = _hidden(encoder, ids)
+ releaser = te_release.TextEncoderReleaser([("text_encoder", encoder)])
+ total = releaser.releasable_bytes
+ assert total > 0 and releaser.release() == total
+ unmatched = sorted(set(releaser._slots) - set(releaser._from_source))
+ assert unmatched == []
+ assert not any((tmp_path / "snap").rglob("*.safetensors"))
+ after = _hidden(encoder, ids)
+ assert all(torch.equal(a, b) for a, b in zip(before, after))
+ releaser.close()
diff --git a/studio/backend/tests/test_diffusion_te_release.py b/studio/backend/tests/test_diffusion_te_release.py
index e25ebc61669..741afaf8575 100644
--- a/studio/backend/tests/test_diffusion_te_release.py
+++ b/studio/backend/tests/test_diffusion_te_release.py
@@ -7,6 +7,7 @@ lifts, and releasing the encoders between prompts. The release tests run on what
from __future__ import annotations
+import os
import types
import pytest
@@ -331,6 +332,122 @@ def test_maybe_install_only_when_wanted(snapshot_root, monkeypatch):
releaser.close()
+def _save_source(
+ encoder,
+ path,
+ prefix = "",
+):
+ """The encoder's weights as a checkpoint file, each tied tensor once."""
+ from safetensors.torch import save_file
+
+ tensors = {}
+ seen = set()
+ for name, tensor in list(encoder.named_parameters()) + list(encoder.named_buffers()):
+ if id(tensor) in seen or name == "rotary":
+ continue
+ seen.add(id(tensor))
+ tensors[prefix + name] = tensor.detach().to("cpu").contiguous().clone()
+ save_file(tensors, str(path))
+ return path
+
+
+def _snapshot_bytes(root):
+ return sum(f.stat().st_size for f in root.rglob("*.safetensors")) if root.exists() else 0
+
+
+def test_release_reloads_unchanged_weights_from_the_source_file(
+ snapshot_root, tmp_path, monkeypatch
+):
+ monkeypatch.delenv(te_release.SOURCE_REUSE_ENV, raising = False)
+ device = _device()
+ pipe = _pipe(device)
+ source = _save_source(pipe.text_encoder, tmp_path / "te.safetensors")
+ setattr(pipe.text_encoder, te_release.WEIGHT_FILES_ATTR, (str(source),))
+ ids = torch.arange(8, device = device)
+ with torch.no_grad():
+ before = pipe.text_encoder(ids).cpu()
+ releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])
+ total = releaser.releasable_bytes
+ assert releaser.release() == total
+ assert len(releaser._from_source) == len(releaser._slots) == 3
+ assert _snapshot_bytes(snapshot_root) == 0
+ if not os.name == "nt":
+ # The open handle keeps the bytes even when the file is replaced or deleted meanwhile.
+ source.unlink()
+ with torch.no_grad():
+ after = pipe.text_encoder(ids).cpu()
+ assert releaser.reloads == 1
+ assert torch.equal(before, after)
+ assert pipe.text_encoder.head.weight is pipe.text_encoder.embed.weight
+ assert pipe.text_encoder.proj.weight.device.type == device
+ assert releaser.release() == total
+ with torch.no_grad():
+ assert torch.equal(before, pipe.text_encoder(ids).cpu())
+ releaser.close()
+
+
+def test_release_snapshots_only_the_tensors_that_differ_from_the_source(
+ snapshot_root, tmp_path, monkeypatch
+):
+ """A merged / cast tensor no longer matches its file: it alone goes to the snapshot, and the reload is exact."""
+ monkeypatch.delenv(te_release.SOURCE_REUSE_ENV, raising = False)
+ device = _device()
+ pipe = _pipe(device)
+ # The checkpoint nests one level shallower than the module, as transformers 5 remaps some encoders.
+ _save_source(pipe.text_encoder, tmp_path / "model.safetensors")
+ with torch.no_grad():
+ pipe.text_encoder.proj.weight.add_(1e-3)
+ wrapper = torch.nn.Module()
+ wrapper.inner = pipe.text_encoder
+ wrapper.name_or_path = str(tmp_path)
+ ids = torch.arange(8, device = device)
+ with torch.no_grad():
+ before = wrapper.inner(ids).cpu()
+ releaser = te_release.TextEncoderReleaser([("text_encoder", wrapper)])
+ total = releaser.releasable_bytes
+ assert releaser.release() == total
+ assert sorted(releaser._from_source) == [
+ "text_encoder.inner.codes",
+ "text_encoder.inner.embed.weight",
+ ]
+ proj = pipe.text_encoder.proj.weight
+ assert 0 < _snapshot_bytes(snapshot_root) < total
+ assert _snapshot_bytes(snapshot_root) >= 1024 * 1024 * proj.element_size()
+ with torch.no_grad():
+ after = wrapper.inner(ids).cpu()
+ assert torch.equal(before, after)
+ releaser.close()
+
+
+def test_release_source_reuse_kill_switch_snapshots_everything(
+ snapshot_root, tmp_path, monkeypatch
+):
+ monkeypatch.setenv(te_release.SOURCE_REUSE_ENV, "0")
+ pipe = _pipe(_device())
+ source = _save_source(pipe.text_encoder, tmp_path / "te.safetensors")
+ setattr(pipe.text_encoder, te_release.WEIGHT_FILES_ATTR, (str(source),))
+ releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])
+ total = releaser.releasable_bytes
+ assert releaser.release() == total
+ assert releaser._from_source == {}
+ assert _snapshot_bytes(snapshot_root) >= total
+ releaser.ensure_loaded()
+ releaser.close()
+
+
+def test_release_without_a_known_source_still_snapshots(snapshot_root, tmp_path, monkeypatch):
+ monkeypatch.delenv(te_release.SOURCE_REUSE_ENV, raising = False)
+ pipe = _pipe(_device())
+ pipe.text_encoder.name_or_path = "some/repo-id"
+ releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])
+ total = releaser.releasable_bytes
+ assert releaser.release() == total
+ assert releaser._from_source == {}
+ assert _snapshot_bytes(snapshot_root) >= total
+ releaser.ensure_loaded()
+ releaser.close()
+
+
@pytest.mark.allow_network
def test_tiny_qwen_image_21_pipeline_renders_the_same_with_release(snapshot_root, monkeypatch):
"""The real QwenImage21Pipeline, tiny: release at denoise entry, reload for a new prompt, same images."""