Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: reload a released text encoder from its own safetensors files, no snapshot write (#17184)
Daniel Hanf53fecac4a18
diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.pyindex 11a06e46946..df1fb41508c 100644--- a/studio/backend/core/inference/diffusion_te_prequant.py+++ b/studio/backend/core/inference/diffusion_te_prequant.py@@ -1218,6 +1218,7 @@ def load_prequant_text_encoder(encoder_cls, config, state_dict, dtype = dtype, trim = trim)setattr(encoder, TE_PREQUANT_SCHEME_ATTR, "int8")+ _tag_weight_files(encoder, path)if logger is not None:logger.info("diffusion.te_prequant: loaded %s int8 ConvRot weight-only checkpoint (%s)",@@ -1264,6 +1265,7 @@ def load_prequant_text_encoder(target.dtype = dtype_cast_fp8(encoder, target)setattr(encoder, TE_PREQUANT_SCHEME_ATTR, "fp8")+ _tag_weight_files(encoder, path)if logger is not None:# The FILE's scheme, not the request's (an fp8 fallback used to log int8).logger.info(@@ -1738,6 +1740,13 @@ def _build_int8_convrot_encoder(return encoder+def _tag_weight_files(encoder: Any, path: Any) -> None:+ """Record the source file so a release reloads matching tensors from it instead of snapshotting."""+ from .diffusion_te_release import WEIGHT_FILES_ATTR+ if str(path).endswith(".safetensors"):+ setattr(encoder, WEIGHT_FILES_ATTR, (str(path),))++def _fill_tied_aliases(encoder: Any, state_dict: dict) -> None:"""Give a key the file leaves out because the model ties it to another one the source of that tie.diff --git a/studio/backend/core/inference/diffusion_te_release.py b/studio/backend/core/inference/diffusion_te_release.pyindex e3a2d926b12..51daac4e02a 100644--- a/studio/backend/core/inference/diffusion_te_release.py+++ b/studio/backend/core/inference/diffusion_te_release.py@@ -1,7 +1,8 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""Releases text encoders during denoise on unified memory (disk snapshot, bit-exact reload)."""+"""Releases text encoders during denoise on unified memory (bit-exact reload from the encoder's own safetensors files,+else from a disk snapshot)."""from __future__ import annotations@@ -15,6 +16,13 @@ from pathlib import Pathfrom typing import Any, OptionalRELEASE_ENV = "UNSLOTH_DIFFUSION_RELEASE_TEXT_ENCODER"+# 0: snapshot every released tensor, as before, instead of reloading the ones the encoder's own files hold byte for byte.+SOURCE_REUSE_ENV = "UNSLOTH_DIFFUSION_TE_RELEASE_FROM_SOURCE"+# Set by a loader that knows the safetensors files an encoder was built from; else ``name_or_path`` when it is a folder.+WEIGHT_FILES_ATTR = "_unsloth_weight_files"+# A folder with more files than this is not one encoder's checkpoint.+_MAX_SOURCE_FILES = 64+_MAX_CANDIDATES = 4TEXT_ENCODER_ATTRS = ("text_encoder", "text_encoder_2", "text_encoder_3", "text_encoder_4")# Tensors below this stay resident: norms, biases and rotary tables are a rounding error next to the projections._MIN_RELEASE_BYTES = 1 << 20@@ -33,6 +41,63 @@ def release_override() -> Optional[bool]:return None+def source_reuse_enabled() -> bool:+ return (os.environ.get(SOURCE_REUSE_ENV) or "").strip().lower() not in (+ "0",+ "false",+ "no",+ "off",+ )+++def _source_files(encoder: Any) -> list[Path]:+ tagged = getattr(encoder, WEIGHT_FILES_ATTR, None)+ if tagged:+ files = [Path(str(f)) for f in tagged]+ else:+ files = []+ for name in (+ getattr(encoder, "name_or_path", None),+ getattr(getattr(encoder, "config", None), "_name_or_path", None),+ ):+ try:+ folder = Path(str(name)) if name else None+ if folder is not None and folder.is_dir():+ files = sorted(folder.glob("*.safetensors"))+ break+ except OSError:+ continue+ files = [f for f in files if f.suffix == ".safetensors" and f.is_file()]+ return files if len(files) <= _MAX_SOURCE_FILES else []+++def _safetensors_dtype(name: str) -> Any:+ import torch+ return {+ "F64": torch.float64,+ "F32": torch.float32,+ "F16": torch.float16,+ "BF16": torch.bfloat16,+ "I64": torch.int64,+ "I32": torch.int32,+ "I16": torch.int16,+ "I8": torch.int8,+ "U8": torch.uint8,+ "BOOL": torch.bool,+ "F8_E4M3": getattr(torch, "float8_e4m3fn", None),+ "F8_E5M2": getattr(torch, "float8_e5m2", None),+ }.get(str(name))+++def _same_bytes(a: Any, b: Any) -> bool:+ import torch+ if a.dtype != b.dtype or tuple(a.shape) != tuple(b.shape):+ return False+ return torch.equal(+ a.contiguous().reshape(-1).view(torch.uint8), b.contiguous().reshape(-1).view(torch.uint8)+ )++def release_wanted(plan: Any) -> tuple[bool, str]:"""``(wanted, reason)``: auto engages only on unified memory when weights plus generation headroom exceed thesafe budget."""@@ -124,9 +189,15 @@ class TextEncoderReleaser:self._released = Falseself._snapshot: Optional[Path] = Noneself._shards: list[Path] = []+ self._prepared = False+ # Open handles keep a source file's bytes readable even if the path is replaced or deleted meanwhile.+ self._source_handles: list[Any] = []+ # key -> (index into _source_handles, tensor name in that file)+ self._from_source: dict[str, tuple[int, str]] = {}# key -> (owner module, leaf name, is_parameter)self._slots: dict[str, tuple[Any, str, bool]] = {}self._released_bytes = 0+ self._bytes: dict[str, int] = {}self._hooks: list[Any] = []self.reloads = 0self.releases = 0@@ -175,10 +246,81 @@ class TextEncoderReleaser:for owner, leaf, is_param in self._slots.values())+ def _tensor_bytes(self, key: str) -> int:+ return self._bytes.get(key, 0)+def _tensor(self, key: str) -> Any:owner, leaf, is_param = self._slots[key]return (owner._parameters if is_param else owner._buffers)[leaf]+ def _match_sources(self) -> None:+ """Map tensors whose bytes match the encoder's own files to them; the rest are snapshotted."""+ import torch+ from safetensors import safe_open++ for attr, encoder in self._encoders:+ keys = [+ k for k in self._slots if k.startswith(attr + ".") and k not in self._from_source+ ]+ if not keys:+ continue+ by_suffix: dict[str, list[tuple[int, str]]] = {}+ for path in _source_files(encoder):+ try:+ handle = safe_open(str(path), framework = "pt", device = "cpu")+ names = list(handle.keys())+ except Exception: # noqa: BLE001 - an unreadable file is just not a source+ continue+ self._source_handles.append(handle)+ index = len(self._source_handles) - 1+ for name in names:+ parts = name.split(".")+ for i in range(len(parts)):+ by_suffix.setdefault(".".join(parts[i:]), []).append((index, name))+ if not by_suffix:+ continue+ for key in keys:+ parts = [p for p in key[len(attr) + 1 :].split(".") if p]+ candidates: list[tuple[int, str]] = []+ # Longest matching name suffix first: transformers may nest the checkpoint's names one level deeper.+ for i in range(len(parts)):+ candidates = by_suffix.get(".".join(parts[i:]), [])+ if candidates:+ break+ tensor = self._tensor(key).detach()+ for index, name in candidates[:_MAX_CANDIDATES]:+ handle = self._source_handles[index]+ try:+ meta = handle.get_slice(name)+ if _safetensors_dtype(meta.get_dtype()) != tensor.dtype or tuple(+ meta.get_shape()+ ) != tuple(tensor.shape):+ continue+ with torch.no_grad():+ if _same_bytes(handle.get_tensor(name), tensor.to("cpu")):+ self._from_source[key] = (index, name)+ break+ except Exception: # noqa: BLE001 - unreadable: snapshot this tensor instead+ continue+ used = sorted({index for index, _ in self._from_source.values()})+ remap = {old: new for new, old in enumerate(used)}+ self._source_handles = [self._source_handles[i] for i in used]+ self._from_source = {k: (remap[i], name) for k, (i, name) in self._from_source.items()}++ def _prepare(self) -> None:+ if source_reuse_enabled():+ try:+ self._match_sources()+ except Exception as exc: # noqa: BLE001 - reuse is an optimisation: snapshot everything+ self._from_source = {}+ self._source_handles = []+ self._note(+ "info", "diffusion.text_encoder_release: snapshotting every tensor: %s", exc+ )+ if any(key not in self._from_source for key in self._slots):+ self._write_snapshot()+ self._prepared = True+def _write_snapshot(self) -> None:import torchfrom safetensors.torch import save_file@@ -186,7 +328,11 @@ class TextEncoderReleaser:root = _snapshot_root()root.mkdir(parents = True, exist_ok = True)_sweep_stale(root)- need = self.releasable_bytes+ need = sum(+ self._tensor(key).numel() * self._tensor(key).element_size()+ for key in self._slots+ if key not in self._from_source+ )free = shutil.disk_usage(root).freeif free < need + (1 << 30):raise RuntimeError(@@ -212,6 +358,8 @@ class TextEncoderReleaser:with torch.no_grad():for key in self._slots:+ if key in self._from_source:+ continuetensor = self._tensor(key).detach()batch[key] = tensor.to("cpu", copy = True).contiguous()size += tensor.numel() * tensor.element_size()@@ -235,17 +383,20 @@ class TextEncoderReleaser:for key in self._slots:if not _plain_tensor(self._tensor(key)):raise RuntimeError(f"{key} is not a plain dense tensor")- if self._snapshot is None:- self._write_snapshot()+ if not self._prepared:+ self._prepare()except Exception as exc: # noqa: BLE001 - releasing is an optimisation: stay residentself._note("warning", "diffusion.text_encoder_release: kept resident: %s", exc)self._slots = {}+ self._from_source = {}+ self._source_handles = []return 0released = 0with torch.no_grad():for key, (owner, leaf, is_param) in self._slots.items():tensor = self._tensor(key)- released += tensor.numel() * tensor.element_size()+ self._bytes[key] = tensor.numel() * tensor.element_size()+ released += self._bytes[key]empty = torch.empty(0, dtype = tensor.dtype, device = tensor.device)if is_param:owner._parameters[leaf].data = empty@@ -255,11 +406,17 @@ class TextEncoderReleaser:self._released_bytes = releasedself.releases += 1_empty_device_cache()+ from_source = sum(+ self._tensor_bytes(key) for key in self._from_source if key in self._slots+ )self._note("info",- "diffusion.text_encoder_release: released %.2f GB of text-encoder weights for the denoise (%.2f s)",+ "diffusion.text_encoder_release: released %.2f GB of text-encoder weights for the denoise (%.2f s; "+ "%.2f GB reload from the encoder's own files, %.2f GB from a snapshot)",released / 1e9,time.monotonic() - started,+ from_source / 1e9,+ (released - from_source) / 1e9,)return released@@ -273,6 +430,14 @@ class TextEncoderReleaser:returnstarted = time.monotonic()with torch.no_grad():+ for key, (index, name) in self._from_source.items():+ owner, leaf, is_param = self._slots[key]+ current = (owner._parameters if is_param else owner._buffers)[leaf]+ restored = self._source_handles[index].get_tensor(name).to(current.device)+ if is_param:+ owner._parameters[leaf].data = restored+ else:+ owner._buffers[leaf] = restoredfor shard in self._shards:with safe_open(str(shard), framework = "pt", device = "cpu") as handle:for key in handle.keys():@@ -301,6 +466,8 @@ class TextEncoderReleaser:except Exception: # noqa: BLE001passself._hooks = []+ self._from_source = {}+ self._source_handles = []self._finalizer()def _note(self, level: str, msg: str, *args: Any) -> None:diff --git a/studio/backend/tests/test_diffusion_flux3_te_prequant.py b/studio/backend/tests/test_diffusion_flux3_te_prequant.pyindex 833c332f036..4c4dda6f7ef 100644--- a/studio/backend/tests/test_diffusion_flux3_te_prequant.py+++ b/studio/backend/tests/test_diffusion_flux3_te_prequant.py@@ -420,3 +420,26 @@ def test_flux3_assembly_hands_the_staged_dir_to_the_prequant_encoder(tmp_path, m"root": staged,"te": "enc",}+++def test_released_int8_convrot_encoder_reloads_from_its_own_file(mirror, tmp_path, monkeypatch):+ """The hosted int8 file is the encoder byte for byte, so releasing it on unified memory writes no snapshot."""+ from core.inference import diffusion_te_release as te_release++ monkeypatch.delenv(te_release.SOURCE_REUSE_ENV, raising = False)+ monkeypatch.setattr(te_release, "_MIN_RELEASE_BYTES", 0)+ monkeypatch.setattr(te_release, "_snapshot_root", lambda: tmp_path / "snap")+ encoder = _pipe_kwargs("flux.3-klein", "int8")["text_encoder"]+ _base, repo, stem = VARIANTS["flux.3-klein"]+ assert getattr(encoder, te_release.WEIGHT_FILES_ATTR) == (str(mirror / repo / _names(stem)[0]),)+ ids = torch.randint(0, VOCAB, (1, 9))+ before = _hidden(encoder, ids)+ releaser = te_release.TextEncoderReleaser([("text_encoder", encoder)])+ total = releaser.releasable_bytes+ assert total > 0 and releaser.release() == total+ unmatched = sorted(set(releaser._slots) - set(releaser._from_source))+ assert unmatched == []+ assert not any((tmp_path / "snap").rglob("*.safetensors"))+ after = _hidden(encoder, ids)+ assert all(torch.equal(a, b) for a, b in zip(before, after))+ releaser.close()diff --git a/studio/backend/tests/test_diffusion_te_release.py b/studio/backend/tests/test_diffusion_te_release.pyindex e25ebc61669..741afaf8575 100644--- a/studio/backend/tests/test_diffusion_te_release.py+++ b/studio/backend/tests/test_diffusion_te_release.py@@ -7,6 +7,7 @@ lifts, and releasing the encoders between prompts. The release tests run on whatfrom __future__ import annotations+import osimport typesimport pytest@@ -331,6 +332,122 @@ def test_maybe_install_only_when_wanted(snapshot_root, monkeypatch):releaser.close()+def _save_source(+ encoder,+ path,+ prefix = "",+):+ """The encoder's weights as a checkpoint file, each tied tensor once."""+ from safetensors.torch import save_file++ tensors = {}+ seen = set()+ for name, tensor in list(encoder.named_parameters()) + list(encoder.named_buffers()):+ if id(tensor) in seen or name == "rotary":+ continue+ seen.add(id(tensor))+ tensors[prefix + name] = tensor.detach().to("cpu").contiguous().clone()+ save_file(tensors, str(path))+ return path+++def _snapshot_bytes(root):+ return sum(f.stat().st_size for f in root.rglob("*.safetensors")) if root.exists() else 0+++def test_release_reloads_unchanged_weights_from_the_source_file(+ snapshot_root, tmp_path, monkeypatch+):+ monkeypatch.delenv(te_release.SOURCE_REUSE_ENV, raising = False)+ device = _device()+ pipe = _pipe(device)+ source = _save_source(pipe.text_encoder, tmp_path / "te.safetensors")+ setattr(pipe.text_encoder, te_release.WEIGHT_FILES_ATTR, (str(source),))+ ids = torch.arange(8, device = device)+ with torch.no_grad():+ before = pipe.text_encoder(ids).cpu()+ releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])+ total = releaser.releasable_bytes+ assert releaser.release() == total+ assert len(releaser._from_source) == len(releaser._slots) == 3+ assert _snapshot_bytes(snapshot_root) == 0+ if not os.name == "nt":+ # The open handle keeps the bytes even when the file is replaced or deleted meanwhile.+ source.unlink()+ with torch.no_grad():+ after = pipe.text_encoder(ids).cpu()+ assert releaser.reloads == 1+ assert torch.equal(before, after)+ assert pipe.text_encoder.head.weight is pipe.text_encoder.embed.weight+ assert pipe.text_encoder.proj.weight.device.type == device+ assert releaser.release() == total+ with torch.no_grad():+ assert torch.equal(before, pipe.text_encoder(ids).cpu())+ releaser.close()+++def test_release_snapshots_only_the_tensors_that_differ_from_the_source(+ snapshot_root, tmp_path, monkeypatch+):+ """A merged / cast tensor no longer matches its file: it alone goes to the snapshot, and the reload is exact."""+ monkeypatch.delenv(te_release.SOURCE_REUSE_ENV, raising = False)+ device = _device()+ pipe = _pipe(device)+ # The checkpoint nests one level shallower than the module, as transformers 5 remaps some encoders.+ _save_source(pipe.text_encoder, tmp_path / "model.safetensors")+ with torch.no_grad():+ pipe.text_encoder.proj.weight.add_(1e-3)+ wrapper = torch.nn.Module()+ wrapper.inner = pipe.text_encoder+ wrapper.name_or_path = str(tmp_path)+ ids = torch.arange(8, device = device)+ with torch.no_grad():+ before = wrapper.inner(ids).cpu()+ releaser = te_release.TextEncoderReleaser([("text_encoder", wrapper)])+ total = releaser.releasable_bytes+ assert releaser.release() == total+ assert sorted(releaser._from_source) == [+ "text_encoder.inner.codes",+ "text_encoder.inner.embed.weight",+ ]+ proj = pipe.text_encoder.proj.weight+ assert 0 < _snapshot_bytes(snapshot_root) < total+ assert _snapshot_bytes(snapshot_root) >= 1024 * 1024 * proj.element_size()+ with torch.no_grad():+ after = wrapper.inner(ids).cpu()+ assert torch.equal(before, after)+ releaser.close()+++def test_release_source_reuse_kill_switch_snapshots_everything(+ snapshot_root, tmp_path, monkeypatch+):+ monkeypatch.setenv(te_release.SOURCE_REUSE_ENV, "0")+ pipe = _pipe(_device())+ source = _save_source(pipe.text_encoder, tmp_path / "te.safetensors")+ setattr(pipe.text_encoder, te_release.WEIGHT_FILES_ATTR, (str(source),))+ releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])+ total = releaser.releasable_bytes+ assert releaser.release() == total+ assert releaser._from_source == {}+ assert _snapshot_bytes(snapshot_root) >= total+ releaser.ensure_loaded()+ releaser.close()+++def test_release_without_a_known_source_still_snapshots(snapshot_root, tmp_path, monkeypatch):+ monkeypatch.delenv(te_release.SOURCE_REUSE_ENV, raising = False)+ pipe = _pipe(_device())+ pipe.text_encoder.name_or_path = "some/repo-id"+ releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])+ total = releaser.releasable_bytes+ assert releaser.release() == total+ assert releaser._from_source == {}+ assert _snapshot_bytes(snapshot_root) >= total+ releaser.ensure_loaded()+ releaser.close()++@pytest.mark.allow_networkdef test_tiny_qwen_image_21_pipeline_renders_the_same_with_release(snapshot_root, monkeypatch):"""The real QwenImage21Pipeline, tiny: release at denoise entry, reload for a new prompt, same images."""