Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: int8 ConvRot Qwen-Image-2.1 text encoder on Apple Silicon, and release text encoders during denoise on unified memory (#13026)
Daniel Han97b416cbaae8
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 88d379525ed..1012805c9a2 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -210,6 +210,7 @@ from . import diffusion_compile_cache as compile_cachefrom .diffusion_compile_config import family_filters_reductionsfrom . import diffusion_cond_cache as cond_cachefrom . import diffusion_prompt_cache as prompt_cache+from . import diffusion_te_release as te_releasefrom . import diffusion_gguf_compile as gguf_compilefrom . import diffusion_bg_compile as bg_compilefrom . import diffusion_cuda_graph as cuda_graph@@ -1242,6 +1243,8 @@ class _LoadState:variant_hint: str = ""# Promoted tiers leave no room for a later ControlNet.calibrated_placement: bool = False+ # Unified memory only: frees the text encoders during each denoise and reloads them before they run again.+ te_releaser: Any = None@dataclass@@ -8085,6 +8088,7 @@ class DiffusionBackend:load_bg_compile = (bg_compile.arm(bg_module, logger = logger) if bg_module is not None else None)+ te_releaser = te_release.maybe_install(pipe, plan, logger = logger)state = _LoadState(pipe = pipe,family = fam,@@ -8128,6 +8132,7 @@ class DiffusionBackend:repo_id,base,),+ te_releaser = te_releaser,)# Serialize publication with cancellation.with self._load_cancel_lock:@@ -10798,6 +10803,9 @@ class DiffusionBackend:def _enter_denoise_phase(gen = gen) -> None:if gen.phase == "encode":gen.phase = "denoise"+ # The prompt is encoded: the encoders are idle until the next prompt, so free them.+ if state.te_releaser is not None:+ state.te_releaser.release()chunk_ticker[0] = _CompletedStepTicker(steps)@@ -11177,6 +11185,8 @@ class DiffusionBackend:cuda_graph.uninstall_all(state.cuda_graphs)uninstall_static_step_skip(state.pipe)prompt_cache.release(state.pipe)+ if state.te_releaser is not None:+ state.te_releaser.close()for aux in (*self._aux_pipes.values(), *self._cn_pipes.values()):prompt_cache.release(aux)gguf_compile.uninstall_all()diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.pyindex b2a938677fb..3d3b7aedc78 100644--- a/studio/backend/core/inference/diffusion_te_prequant.py+++ b/studio/backend/core/inference/diffusion_te_prequant.py@@ -26,7 +26,7 @@ None and the caller falls back to the dense download + cast. Inert with nothingfrom __future__ import annotationsimport sys-from dataclasses import dataclass+from dataclasses import dataclass, replacefrom typing import Any, Iterable, Optional# Reuse the DiT module's operator allowlist for local paths: one env var, one policy.@@ -129,6 +129,9 @@ TE_PREQUANT_COMPONENTS = ("text_encoder", "text_encoder_2", "text_encoder_3")# measured encoder rather than under-stating any: an under-estimate is the expensive direction, since it lets an# oversized load through to the OS killer.TE_PREQUANT_BUDGET_SCALE = 0.65+# The int8 ConvRot encoder alone (no fp8 fallback to cover): Qwen-Image-2.1's is 9,349,769,248 bytes against+# 17,534,339,488 for the dense encoder, 0.533, rounded up.+TE_INT8_CONVROT_BUDGET_SCALE = 0.56def te_prequant_budget_scale(@@ -156,7 +159,24 @@ def te_prequant_budget_scale()except Exception: # noqa: BLE001 -- an unresolvable pre-cast just means the dense encoderreturn 1.0- return TE_PREQUANT_BUDGET_SCALE if sources else 1.0+ if not sources:+ return 1.0+ if _int8_convrot_only(te_quant_mode, target):+ return TE_INT8_CONVROT_BUDGET_SCALE+ return TE_PREQUANT_BUDGET_SCALE+++def _int8_convrot_only(te_quant_mode: Optional[str], target: Any) -> bool:+ """Whether this pick can ONLY take the int8 ConvRot file (no fp8 fallback): Apple Silicon."""+ try:+ from .diffusion_precision import TE_QUANT_FP8, normalize_te_quant, te_quant_supported+ return (+ normalize_te_quant(te_quant_mode) == "int8"+ and int8_convrot_te_runs_on(target)+ and not te_quant_supported(target, TE_QUANT_FP8)+ )+ except Exception: # noqa: BLE001+ return False# Bases whose text-encoder weights are VERIFIED byte-identical (every shard LFS sha256 compared on 2026-07-18), so one@@ -226,6 +246,15 @@ class TePrequantSource:fallback_filenames: tuple = ()+def int8_convrot_te_runs_on(target: Any) -> bool:+ """MPS runs the int8 ConvRot encoder without fp8: plain tensors, a Hadamard matmul and ``F.linear``."""+ return getattr(target, "device", None) == "mps"+++def _is_fp8_te_name(name: Optional[str]) -> bool:+ return bool(name) and "fp8" in str(name).lower()++def te_prequant_repo_stem(repo_id: str, component: str, scheme: str) -> str:"""The extensionless checkpoint name for ``(component, scheme)`` in ``repo_id``: hosted reposare named <Model>-FP8 (or -INT8 / -quantized) and carry <Model>-<component>-<SCHEME> files,@@ -398,12 +427,17 @@ def te_prequant_sources(denied = getattr(precision, "_te_family_denied", None)if callable(denied) and denied(family, mode):return {}- # int8 dequantizes to bf16 per call (no torchao) and falls back to the fp8 file: it needs what fp8 needs.- if not te_quant_supported(target, TE_QUANT_FP8):+ # int8 falls back to the fp8 file, so it needs fp8; MPS runs the int8 file alone and drops the fp8 names.+ fp8_runs = te_quant_supported(target, TE_QUANT_FP8)+ if not fp8_runs and not (mode == "int8" and int8_convrot_te_runs_on(target)):return {}sources: dict[str, TePrequantSource] = {}for component in components:source = resolve_te_prequant_source(fam, component, mode)+ if source is not None and not fp8_runs:+ if source.kind == "repo" and _is_fp8_te_name(source.filename):+ continue+ source = replace(source, fallback_filenames = ())if source is not None:sources[component] = sourcereturn sources@@ -745,7 +779,7 @@ def _te_prequant_pipe_kwargs(The later ``quantize_text_encoders`` call re-applies the cast idempotently and keepsstatus reporting truthful."""try:- from .diffusion_precision import TE_QUANT_FP8, normalize_te_quant+ from .diffusion_precision import TE_QUANT_FP8, normalize_te_quant, te_quant_supportedfrom .diffusion_text_encoder_trim import family_trims_lm_headsources = te_prequant_sources_for_base(@@ -780,6 +814,7 @@ def _te_prequant_pipe_kwargs(and mode == "int8"and source.kind == "repo"and fp8_names+ and te_quant_supported(target, TE_QUANT_FP8)and _held_locally(source.location, source.filename, hf_token)):# The int8 file resolved but was refused: the plan dropped the dense shards, so take the fp8 names.diff --git a/studio/backend/core/inference/diffusion_te_release.py b/studio/backend/core/inference/diffusion_te_release.pynew file mode 100644index 00000000000..32139a81e06--- /dev/null+++ b/studio/backend/core/inference/diffusion_te_release.py@@ -0,0 +1,372 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Release the text encoders during denoise on unified memory, where offloading frees nothing.++The first release snapshots the encoders' large tensors to local disk; a forward pre-hook reloads them bit-exact before+an encoder runs again. ``UNSLOTH_DIFFUSION_RELEASE_TEXT_ENCODER=1`` / ``=0`` forces it on / off.+"""++from __future__ import annotations++import os+import shutil+import threading+import time+import uuid+import weakref+from pathlib import Path+from typing import Any, Optional++RELEASE_ENV = "UNSLOTH_DIFFUSION_RELEASE_TEXT_ENCODER"+TEXT_ENCODER_ATTRS = ("text_encoder", "text_encoder_2", "text_encoder_3", "text_encoder_4")+# Tensors below this stay resident: norms, biases and rotary tables are a rounding error next to the projections.+_MIN_RELEASE_BYTES = 1 << 20+# Snapshot shard size: bounds the host copy a write holds at once (unified memory has no second pool to spill to).+_SHARD_BYTES = 512 << 20+_SNAPSHOT_DIR = "text_encoder_release"+++def release_override() -> Optional[bool]:+ """``True`` / ``False`` when ``UNSLOTH_DIFFUSION_RELEASE_TEXT_ENCODER`` forces it, else None (auto)."""+ value = (os.environ.get(RELEASE_ENV) or "").strip().lower()+ if value in ("1", "true", "yes", "on"):+ return True+ if value in ("0", "false", "no", "off"):+ return False+ return None+++def release_wanted(plan: Any) -> tuple[bool, str]:+ """``(wanted, reason)``: auto engages only on unified memory when weights plus generation headroom exceed the+ safe budget."""+ forced = release_override()+ if forced is not None:+ return forced, f"{RELEASE_ENV}={'1' if forced else '0'}"+ try:+ memory = plan.device_memory+ if getattr(memory, "memory_kind", None) != "unified_memory":+ return False, "not unified memory"+ estimates = plan.estimates+ budget = estimates.get("safe_device_budget_mib")+ required = estimates.get("resident_required_mib")+ encoders = estimates.get("text_encoder_dense_mib")+ except Exception: # noqa: BLE001 - no plan, no release+ return False, "no memory plan"+ if budget is None or required is None or not encoders:+ return False, "budget or text-encoder size unknown"+ if int(required) <= int(budget):+ return (+ False,+ f"weights and generation headroom ({required} MiB) fit the {budget} MiB budget",+ )+ return True, (+ f"unified memory: weights and generation headroom ({required} MiB) exceed the {budget} MiB budget, "+ f"and the text encoders ({encoders} MiB) run once per prompt"+ )+++def _snapshot_root() -> Path:+ try:+ from utils.paths.storage_roots import cache_root+ return Path(cache_root()) / _SNAPSHOT_DIR+ except Exception: # noqa: BLE001 - no Studio home (tests, bare scripts): the system temp dir+ import tempfile+ return Path(tempfile.gettempdir()) / f"unsloth_{_SNAPSHOT_DIR}"+++def _pid_alive(pid: int) -> bool:+ try:+ os.kill(pid, 0)+ except ProcessLookupError:+ return False+ except Exception: # noqa: BLE001 - permission errors mean the process exists+ return True+ return True+++def _sweep_stale(root: Path) -> None:+ """Delete snapshots a crashed backend left behind (the directory name carries its pid)."""+ try:+ entries = list(root.iterdir())+ except OSError:+ return+ for entry in entries:+ pid_text = entry.name.split("-", 1)[0]+ if pid_text.isdigit() and int(pid_text) != os.getpid() and not _pid_alive(int(pid_text)):+ shutil.rmtree(entry, ignore_errors = True)+++def _offload_hooked(module: Any) -> bool:+ """accelerate / diffusers offload hooks move weights themselves; never release under them."""+ for sub in module.modules():+ if (+ getattr(sub, "_hf_hook", None) is not None+ or getattr(sub, "_diffusers_hook", None) is not None+ ):+ return True+ return False+++def _plain_tensor(tensor: Any) -> bool:+ import torch+ return type(tensor) in (torch.Tensor, torch.nn.Parameter) and tensor.device.type != "meta"+++class TextEncoderReleaser:+ """Frees a pipeline's text-encoder weights during denoise and restores them, bit for bit, before they run again."""++ def __init__(+ self,+ encoders: list[tuple[str, Any]],+ *,+ logger: Any = None,+ ) -> None:+ self._encoders = encoders+ self._logger = logger+ self._lock = threading.RLock()+ self._released = False+ self._snapshot: Optional[Path] = None+ self._shards: list[Path] = []+ # key -> (owner module, leaf name, is_parameter)+ self._slots: dict[str, tuple[Any, str, bool]] = {}+ self._released_bytes = 0+ self._hooks: list[Any] = []+ self.reloads = 0+ self.releases = 0+ self._collect()+ self._install_hooks()+ self._finalizer = weakref.finalize(self, _remove_dir, None)++ def _collect(self) -> None:+ seen: set[int] = set()+ for attr, encoder in self._encoders:+ for module_name, module in encoder.named_modules():+ for leaf, param in list(module._parameters.items()):+ if param is None or id(param) in seen:+ continue+ seen.add(id(param))+ if param.numel() * param.element_size() >= _MIN_RELEASE_BYTES:+ self._slots[f"{attr}.{module_name}.{leaf}"] = (module, leaf, True)+ for leaf, buf in list(module._buffers.items()):+ if buf is None or id(buf) in seen or leaf in module._non_persistent_buffers_set:+ continue+ seen.add(id(buf))+ if buf.numel() * buf.element_size() >= _MIN_RELEASE_BYTES:+ self._slots[f"{attr}.{module_name}.{leaf}"] = (module, leaf, False)++ def _install_hooks(self) -> None:+ owners = {id(owner): owner for owner, _, _ in self._slots.values()}+ for _, encoder in self._encoders:+ owners.setdefault(id(encoder), encoder)++ def _pre_hook(module: Any, args: Any) -> None:+ if self._released:+ self.ensure_loaded()++ for owner in owners.values():+ self._hooks.append(owner.register_forward_pre_hook(_pre_hook))++ @property+ def released(self) -> bool:+ return self._released++ @property+ def releasable_bytes(self) -> int:+ return sum(+ getattr(owner, "_parameters" if is_param else "_buffers")[leaf].numel()+ * getattr(owner, "_parameters" if is_param else "_buffers")[leaf].element_size()+ for owner, leaf, is_param in self._slots.values()+ )++ def _tensor(self, key: str) -> Any:+ owner, leaf, is_param = self._slots[key]+ return (owner._parameters if is_param else owner._buffers)[leaf]++ def _write_snapshot(self) -> None:+ import torch+ from safetensors.torch import save_file++ root = _snapshot_root()+ root.mkdir(parents = True, exist_ok = True)+ _sweep_stale(root)+ need = self.releasable_bytes+ free = shutil.disk_usage(root).free+ if free < need + (1 << 30):+ raise RuntimeError(+ f"not enough free disk for the text-encoder snapshot ({need / 1e9:.1f} GB needed, "+ f"{free / 1e9:.1f} GB free under {root})"+ )+ snapshot = root / f"{os.getpid()}-{uuid.uuid4().hex[:10]}"+ snapshot.mkdir()+ self._finalizer.detach()+ self._finalizer = weakref.finalize(self, _remove_dir, str(snapshot))+ shards: list[Path] = []+ batch: dict[str, Any] = {}+ size = 0++ def _flush() -> None:+ nonlocal batch, size+ if not batch:+ return+ path = snapshot / f"shard-{len(shards):05d}.safetensors"+ save_file(batch, str(path))+ shards.append(path)+ batch, size = {}, 0++ with torch.no_grad():+ for key in self._slots:+ tensor = self._tensor(key).detach()+ batch[key] = tensor.to("cpu", copy = True).contiguous()+ size += tensor.numel() * tensor.element_size()+ if size >= _SHARD_BYTES:+ _flush()+ _flush()+ self._snapshot = snapshot+ self._shards = shards++ def release(self) -> int:+ """Bytes freed; 0 when already released or refused."""+ import torch+ with self._lock:+ if self._released or not self._slots:+ return 0+ started = time.monotonic()+ try:+ for _, encoder in self._encoders:+ if _offload_hooked(encoder):+ raise RuntimeError("an offload hook manages these weights")+ for key in self._slots:+ if not _plain_tensor(self._tensor(key)):+ raise RuntimeError(f"{key} is not a plain dense tensor")+ if self._snapshot is None:+ self._write_snapshot()+ except Exception as exc: # noqa: BLE001 - releasing is an optimisation: stay resident+ self._note("warning", "diffusion.text_encoder_release: kept resident: %s", exc)+ self._slots = {}+ return 0+ released = 0+ with torch.no_grad():+ for key, (owner, leaf, is_param) in self._slots.items():+ tensor = self._tensor(key)+ released += tensor.numel() * tensor.element_size()+ empty = torch.empty(0, dtype = tensor.dtype, device = tensor.device)+ if is_param:+ owner._parameters[leaf].data = empty+ else:+ owner._buffers[leaf] = empty+ self._released = True+ self._released_bytes = released+ self.releases += 1+ _empty_device_cache()+ self._note(+ "info",+ "diffusion.text_encoder_release: released %.2f GB of text-encoder weights for the denoise (%.2f s)",+ released / 1e9,+ time.monotonic() - started,+ )+ return released++ def ensure_loaded(self) -> None:+ """Restore every released tensor from the snapshot onto its original device."""+ import torch+ from safetensors import safe_open++ with self._lock:+ if not self._released:+ return+ started = time.monotonic()+ with torch.no_grad():+ for shard in self._shards:+ with safe_open(str(shard), framework = "pt", device = "cpu") as handle:+ for key in handle.keys():+ owner, leaf, is_param = self._slots[key]+ current = (owner._parameters if is_param else owner._buffers)[leaf]+ restored = handle.get_tensor(key).to(current.device)+ if is_param:+ owner._parameters[leaf].data = restored+ else:+ owner._buffers[leaf] = restored+ self._released = False+ self.reloads += 1+ self._note(+ "info",+ "diffusion.text_encoder_release: reloaded %.2f GB of text-encoder weights in %.2f s",+ self._released_bytes / 1e9,+ time.monotonic() - started,+ )++ def close(self) -> None:+ """Drop the hooks and the snapshot; the weights stay as they are."""+ with self._lock:+ for hook in self._hooks:+ try:+ hook.remove()+ except Exception: # noqa: BLE001+ pass+ self._hooks = []+ self._finalizer()++ def _note(self, level: str, msg: str, *args: Any) -> None:+ if self._logger is not None:+ getattr(self._logger, level)(msg, *args)+++def _remove_dir(path: Optional[str]) -> None:+ if path:+ shutil.rmtree(path, ignore_errors = True)+++def _empty_device_cache() -> None:+ import torch+ for backend in ("mps", "cuda"):+ module = getattr(torch, backend, None)+ try:+ if module is not None and module.is_available():+ module.empty_cache()+ except Exception: # noqa: BLE001+ pass+++def maybe_install(+ pipe: Any,+ plan: Any,+ *,+ logger: Any = None,+) -> Optional[TextEncoderReleaser]:+ """A releaser for ``pipe`` when ``plan`` calls for one (see ``release_wanted``), else None. Never raises."""+ try:+ wanted, reason = release_wanted(plan)+ if not wanted:+ return None+ encoders = [+ (attr, getattr(pipe, attr))+ for attr in TEXT_ENCODER_ATTRS+ if _is_module(getattr(pipe, attr, None))+ ]+ if not encoders:+ return None+ releaser = TextEncoderReleaser(encoders, logger = logger)+ if not releaser._slots:+ releaser.close()+ return None+ if logger is not None:+ logger.info(+ "diffusion.text_encoder_release: on for %s (%.2f GB): %s",+ ", ".join(a for a, _ in encoders),+ releaser.releasable_bytes / 1e9,+ reason,+ )+ return releaser+ except Exception as exc: # noqa: BLE001 - an optimisation: the encoders stay resident+ if logger is not None:+ logger.warning("diffusion.text_encoder_release: not installed: %s", exc)+ return None+++def _is_module(value: Any) -> bool:+ try:+ import torch+ return isinstance(value, torch.nn.Module)+ except Exception: # noqa: BLE001+ return Falsediff --git a/studio/backend/tests/test_diffusion_te_release.py b/studio/backend/tests/test_diffusion_te_release.pynew file mode 100644index 00000000000..efa78a7b7ba--- /dev/null+++ b/studio/backend/tests/test_diffusion_te_release.py@@ -0,0 +1,349 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Text encoders on unified memory (#11792): the int8 ConvRot encoder on Apple Silicon, the load-time refusal it+lifts, and releasing the encoders between prompts. The release tests run on whatever accelerator the runner has+(MPS on Apple Silicon runners, CUDA, else CPU), never skipped for the device."""++from __future__ import annotations++import types++import pytest++torch = pytest.importorskip("torch")++from core.inference import diffusion_te_prequant as te_prequant+from core.inference import diffusion_te_release as te_release+from core.inference.diffusion_device import DiffusionDeviceTarget+from core.inference.diffusion_memory import (+ DeviceMemory,+ plan_diffusion_memory,+ unified_memory_shortfall_message,+)++GIB = 1024++# Qwen-Image-2.1 on a 48 GB M5 Max, from the report: macOS said 27 GB free.+Q21_DIT_Q4_K_M_MIB = 4_199_565_024 >> 20+Q21_TE_BF16_MIB = 17_534_339_488 >> 20+Q21_TE_INT8_CONVROT_MIB = 9_349_769_248 >> 20+Q21_VAE_MIB = 650+++def _target(device: str, dtype = None) -> DiffusionDeviceTarget:+ return DiffusionDeviceTarget(+ device = device,+ dtype = dtype if dtype is not None else torch.bfloat16,+ backend = device,+ vendor = None,+ supports_model_cpu_offload = device == "cuda",+ supports_default_torch_compile = device == "cuda",+ supports_pinned_transfer = device == "cuda",+ )+++def _q21_family():+ from core.inference.diffusion_families import detect_family++ fam = detect_family("Qwen/Qwen-Image-2.1")+ assert fam is not None and fam.name == "qwen-image-2.1"+ return fam+++def _plan(+ memory: DeviceMemory,+ weights: int,+ encoders: int,+ headroom: int = 8 * GIB,+):+ return plan_diffusion_memory(+ target = _target("mps" if memory.memory_kind == "unified_memory" else "cuda"),+ device_memory = memory,+ model_dense_mib = weights,+ runtime_headroom_mib = headroom,+ companion_dense_mib = encoders + Q21_VAE_MIB,+ text_encoder_dense_mib = encoders,+ )+++M5_MAX = DeviceMemory("mps", "mps", "unified_memory", free_mib = 27 * GIB, total_mib = 48 * GIB)+++def test_mps_takes_the_int8_convrot_encoder_without_the_fp8_fallback():+ sources = te_prequant.te_prequant_sources(+ _q21_family(), te_quant_mode = "int8", target = _target("mps")+ )+ source = sources["text_encoder"]+ assert source.filename == "Qwen-Image-2.1-text_encoder-INT8-ConvRot.safetensors"+ assert source.fallback_filenames == ()+++def test_mps_fp8_request_still_takes_no_precast_encoder():+ assert (+ te_prequant.te_prequant_sources(_q21_family(), te_quant_mode = "fp8", target = _target("mps"))+ == {}+ )+++def test_cpu_takes_no_precast_encoder():+ assert (+ te_prequant.te_prequant_sources(+ _q21_family(), te_quant_mode = "int8", target = _target("cpu", torch.float32)+ )+ == {}+ )+++def test_cuda_keeps_the_fp8_fallback(monkeypatch):+ from core.inference import diffusion_precision++ monkeypatch.setattr(diffusion_precision, "te_quant_supported", lambda target, mode: True)+ source = te_prequant.te_prequant_sources(+ _q21_family(), te_quant_mode = "int8", target = _target("cuda")+ )["text_encoder"]+ assert source.fallback_filenames, "the CUDA path must keep its fp8 fallback names"+ assert (+ te_prequant.te_prequant_budget_scale(+ _q21_family(), te_quant_mode = "int8", target = _target("cuda"), base = "Qwen/Qwen-Image-2.1"+ )+ == te_prequant.TE_PREQUANT_BUDGET_SCALE+ )+++def test_mps_budgets_the_int8_encoder_at_its_own_size():+ scale = te_prequant.te_prequant_budget_scale(+ _q21_family(), te_quant_mode = "int8", target = _target("mps"), base = "Qwen/Qwen-Image-2.1"+ )+ assert scale == te_prequant.TE_INT8_CONVROT_BUDGET_SCALE+ assert Q21_TE_BF16_MIB * scale >= Q21_TE_INT8_CONVROT_MIB+++def test_int8_convrot_linear_runs_on_this_accelerator():+ """The int8 ConvRot projection the Apple Silicon path now loads: plain tensors, so it runs on MPS too."""+ from core.inference.video_minimax_h3_te import _int8_convrot_linear_class++ device = _device()+ torch.manual_seed(0)+ dense = torch.nn.Linear(512, 256, bias = True)+ codes, scale = te_prequant.quantize_int8_convrot_weight(dense.weight.detach(), group_size = 256)+ layer = _int8_convrot_linear_class()(512, 256, bias = True, group_size = 256)+ layer.load_state_dict(+ {"weight": codes, "weight_scale": scale, "bias": dense.bias.detach().to(torch.bfloat16)},+ assign = True,+ )+ layer = layer.to(device)+ x = torch.randn(4, 512)+ with torch.no_grad():+ got = layer(x.to(device)).float().cpu()+ want = dense(x)+ rel = (got - want).norm() / want.norm()+ assert got.shape == (4, 256)+ assert rel < 0.02, float(rel)+++def test_m5_max_refuses_the_bf16_encoder():+ weights = Q21_DIT_Q4_K_M_MIB + Q21_TE_BF16_MIB + Q21_VAE_MIB+ assert unified_memory_shortfall_message(_plan(M5_MAX, weights, Q21_TE_BF16_MIB)) is not None+++def test_m5_max_accepts_the_int8_convrot_encoder():+ encoders = int(Q21_TE_BF16_MIB * te_prequant.TE_INT8_CONVROT_BUDGET_SCALE)+ weights = Q21_DIT_Q4_K_M_MIB + encoders + Q21_VAE_MIB+ assert unified_memory_shortfall_message(_plan(M5_MAX, weights, encoders)) is None+++def test_a_genuinely_oversized_load_is_still_refused():+ # The dense bf16 Qwen-Image-2.1 transformer with the int8 encoder does not fit 27 GB free either.+ encoders = int(Q21_TE_BF16_MIB * te_prequant.TE_INT8_CONVROT_BUDGET_SCALE)+ weights = (14_230_284_408 >> 20) + encoders + Q21_VAE_MIB+ assert unified_memory_shortfall_message(_plan(M5_MAX, weights, encoders)) is not None+++def test_discrete_vram_never_refuses_here():+ memory = DeviceMemory("cuda", "cuda", "discrete_vram", free_mib = 8 * GIB, total_mib = 24 * GIB)+ weights = Q21_DIT_Q4_K_M_MIB + Q21_TE_BF16_MIB + Q21_VAE_MIB+ assert unified_memory_shortfall_message(_plan(memory, weights, Q21_TE_BF16_MIB)) is None+++def test_release_engages_on_unified_memory_when_the_resident_set_does_not_fit(monkeypatch):+ monkeypatch.delenv(te_release.RELEASE_ENV, raising = False)+ encoders = int(Q21_TE_BF16_MIB * te_prequant.TE_INT8_CONVROT_BUDGET_SCALE)+ weights = Q21_DIT_Q4_K_M_MIB + encoders + Q21_VAE_MIB+ wanted, reason = te_release.release_wanted(_plan(M5_MAX, weights, encoders))+ assert wanted, reason+++def test_release_stays_off_when_everything_fits(monkeypatch):+ monkeypatch.delenv(te_release.RELEASE_ENV, raising = False)+ roomy = DeviceMemory("mps", "mps", "unified_memory", free_mib = 100 * GIB, total_mib = 128 * GIB)+ weights = Q21_DIT_Q4_K_M_MIB + Q21_TE_BF16_MIB + Q21_VAE_MIB+ assert te_release.release_wanted(_plan(roomy, weights, Q21_TE_BF16_MIB))[0] is False+++def test_release_stays_off_on_discrete_vram(monkeypatch):+ monkeypatch.delenv(te_release.RELEASE_ENV, raising = False)+ memory = DeviceMemory("cuda", "cuda", "discrete_vram", free_mib = 8 * GIB, total_mib = 24 * GIB)+ weights = Q21_DIT_Q4_K_M_MIB + Q21_TE_BF16_MIB + Q21_VAE_MIB+ assert te_release.release_wanted(_plan(memory, weights, Q21_TE_BF16_MIB))[0] is False++[email protected]("value, expected", [("1", True), ("0", False)])+def test_release_env_forces_either_way(monkeypatch, value, expected):+ monkeypatch.setenv(te_release.RELEASE_ENV, value)+ memory = DeviceMemory("cuda", "cuda", "discrete_vram", free_mib = 8 * GIB, total_mib = 24 * GIB)+ assert te_release.release_wanted(_plan(memory, 1, 1))[0] is expected+++def _device() -> str:+ if getattr(torch.backends, "mps", None) is not None and torch.backends.mps.is_available():+ return "mps"+ if torch.cuda.is_available():+ return "cuda"+ return "cpu"+++class _Encoder(torch.nn.Module):+ """A stand-in encoder: a tied embedding / head pair, int8 buffers like the ConvRot encoder, a small norm."""++ def __init__(self):+ super().__init__()+ self.embed = torch.nn.Embedding(512, 1024)+ self.proj = torch.nn.Linear(1024, 1024)+ self.register_buffer("codes", torch.randint(-127, 127, (1024, 1024), dtype = torch.int8))+ self.register_buffer("rotary", torch.arange(16, dtype = torch.float32), persistent = False)+ self.norm = torch.nn.LayerNorm(1024)+ self.head = torch.nn.Linear(1024, 512, bias = False)+ self.head.weight = self.embed.weight++ def forward(self, ids):+ h = self.norm(+ self.proj(self.embed(ids))+ + self.codes[: ids.shape[-1]].to(self.proj.weight.dtype).sum(-1, keepdim = True)+ )+ return self.head(h) + self.rotary.sum()+++def snapshot_root(tmp_path, monkeypatch):+ monkeypatch.setattr(te_release, "_snapshot_root", lambda: tmp_path / "snap")+ return tmp_path / "snap"+++def _pipe(device: str):+ torch.manual_seed(0)+ encoder = _Encoder().to(device)+ return types.SimpleNamespace(text_encoder = encoder, transformer = torch.nn.Linear(4, 4).to(device))+++def test_release_frees_and_reload_restores_bit_for_bit(snapshot_root):+ device = _device()+ pipe = _pipe(device)+ ids = torch.arange(8, device = device)+ with torch.no_grad():+ before = pipe.text_encoder(ids).cpu()+ releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])+ total = releaser.releasable_bytes+ assert total > 2 << 20+ assert releaser.release() == total+ assert releaser.released+ assert pipe.text_encoder.proj.weight.numel() == 0+ assert pipe.text_encoder.codes.numel() == 0+ # Device and dtype survive, so pipelines that read them before encoding still work.+ assert pipe.text_encoder.proj.weight.device.type == device+ assert pipe.text_encoder.proj.weight.dtype == torch.float32+ assert pipe.text_encoder.rotary.numel() == 16+ with torch.no_grad():+ after = pipe.text_encoder(ids).cpu()+ assert not releaser.released and releaser.reloads == 1+ assert torch.equal(before, after)+ # The tie survives: one tensor, released and restored once.+ assert pipe.text_encoder.head.weight is pipe.text_encoder.embed.weight+ # A second release reuses the snapshot; a submodule called directly also reloads.+ shards = list(snapshot_root.rglob("*.safetensors"))+ assert releaser.release() == total+ assert sorted(snapshot_root.rglob("*.safetensors")) == sorted(shards)+ with torch.no_grad():+ pipe.text_encoder.proj(torch.zeros(1, 1024, device = device))+ assert releaser.reloads == 2+ releaser.close()+ assert not any(snapshot_root.rglob("*.safetensors"))+++def test_release_refuses_offload_hooked_encoders(snapshot_root):+ pipe = _pipe(_device())+ pipe.text_encoder.proj._hf_hook = object()+ releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])+ assert releaser.release() == 0+ assert pipe.text_encoder.proj.weight.numel() > 0+++def test_release_refuses_when_the_disk_is_full(snapshot_root, monkeypatch):+ pipe = _pipe(_device())+ usage = types.SimpleNamespace(total = 1, used = 1, free = 1)+ monkeypatch.setattr(te_release.shutil, "disk_usage", lambda path: usage)+ releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])+ assert releaser.release() == 0+ assert pipe.text_encoder.proj.weight.numel() > 0+++def test_maybe_install_only_when_wanted(snapshot_root, monkeypatch):+ pipe = _pipe(_device())+ monkeypatch.setenv(te_release.RELEASE_ENV, "0")+ assert te_release.maybe_install(pipe, None) is None+ monkeypatch.setenv(te_release.RELEASE_ENV, "1")+ releaser = te_release.maybe_install(pipe, None)+ assert releaser is not None+ releaser.close()++[email protected]_network+def test_tiny_qwen_image_21_pipeline_renders_the_same_with_release(snapshot_root, monkeypatch):+ """The real QwenImage21Pipeline, tiny: release at denoise entry, reload for a new prompt, same images."""+ diffusers = pytest.importorskip("diffusers")+ pipeline_cls = getattr(diffusers, "QwenImage21Pipeline", None)+ if pipeline_cls is None:+ pytest.skip("this diffusers has no QwenImage21Pipeline")+ from huggingface_hub import snapshot_download++ try:+ path = snapshot_download("hf-internal-testing/tiny-qwenimage21-pipe")+ except Exception as exc: # noqa: BLE001+ pytest.skip(f"tiny pipeline not reachable: {exc}")+ from core.inference.media_decode_phase import denoise_phase++ device = _device()+ dtype = torch.float32++ def render(+ pipe,+ prompt,+ releaser = None,+ ):+ def on_denoise():+ if releaser is not None:+ releaser.release()++ with denoise_phase(pipe, on_denoise):+ out = pipe(+ prompt = prompt,+ height = 64,+ width = 64,+ num_inference_steps = 2,+ generator = torch.Generator("cpu").manual_seed(0),+ output_type = "pt",+ )+ return out.images.cpu()++ pipe = pipeline_cls.from_pretrained(path, torch_dtype = dtype).to(device)+ pipe.set_progress_bar_config(disable = True)+ reference = [render(pipe, "a red cube"), render(pipe, "a blue sphere")]++ releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])+ first = render(pipe, "a red cube", releaser)+ assert releaser.released and releaser.releases == 1+ second = render(pipe, "a blue sphere", releaser)+ assert releaser.reloads == 1 and releaser.releases == 2+ assert torch.equal(first, reference[0])+ assert torch.equal(second, reference[1])+ releaser.close()