Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: int8 ConvRot Qwen-Image-2.1 text encoder on Apple Silicon, and release text encoders during denoise on unified memory (#13026)

Daniel Han97b416cbaae8
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 88d379525ed..1012805c9a2 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -210,6 +210,7 @@ from . import diffusion_compile_cache as compile_cache
from .diffusion_compile_config import family_filters_reductions
from . import diffusion_cond_cache as cond_cache
from . import diffusion_prompt_cache as prompt_cache
+from . import diffusion_te_release as te_release
from . import diffusion_gguf_compile as gguf_compile
from . import diffusion_bg_compile as bg_compile
from . import diffusion_cuda_graph as cuda_graph
@@ -1242,6 +1243,8 @@ class _LoadState:
variant_hint: str = ""
# Promoted tiers leave no room for a later ControlNet.
calibrated_placement: bool = False
+ # Unified memory only: frees the text encoders during each denoise and reloads them before they run again.
+ te_releaser: Any = None
@dataclass
@@ -8085,6 +8088,7 @@ class DiffusionBackend:
load_bg_compile = (
bg_compile.arm(bg_module, logger = logger) if bg_module is not None else None
)
+ te_releaser = te_release.maybe_install(pipe, plan, logger = logger)
state = _LoadState(
pipe = pipe,
family = fam,
@@ -8128,6 +8132,7 @@ class DiffusionBackend:
repo_id,
base,
),
+ te_releaser = te_releaser,
)
# Serialize publication with cancellation.
with self._load_cancel_lock:
@@ -10798,6 +10803,9 @@ class DiffusionBackend:
def _enter_denoise_phase(gen = gen) -> None:
if gen.phase == "encode":
gen.phase = "denoise"
+ # The prompt is encoded: the encoders are idle until the next prompt, so free them.
+ if state.te_releaser is not None:
+ state.te_releaser.release()
chunk_ticker[0] = _CompletedStepTicker(steps)
@@ -11177,6 +11185,8 @@ class DiffusionBackend:
cuda_graph.uninstall_all(state.cuda_graphs)
uninstall_static_step_skip(state.pipe)
prompt_cache.release(state.pipe)
+ if state.te_releaser is not None:
+ state.te_releaser.close()
for aux in (*self._aux_pipes.values(), *self._cn_pipes.values()):
prompt_cache.release(aux)
gguf_compile.uninstall_all()
diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.py
index b2a938677fb..3d3b7aedc78 100644
--- a/studio/backend/core/inference/diffusion_te_prequant.py
+++ b/studio/backend/core/inference/diffusion_te_prequant.py
@@ -26,7 +26,7 @@ None and the caller falls back to the dense download + cast. Inert with nothing
from __future__ import annotations
import sys
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
from typing import Any, Iterable, Optional
# Reuse the DiT module's operator allowlist for local paths: one env var, one policy.
@@ -129,6 +129,9 @@ TE_PREQUANT_COMPONENTS = ("text_encoder", "text_encoder_2", "text_encoder_3")
# measured encoder rather than under-stating any: an under-estimate is the expensive direction, since it lets an
# oversized load through to the OS killer.
TE_PREQUANT_BUDGET_SCALE = 0.65
+# The int8 ConvRot encoder alone (no fp8 fallback to cover): Qwen-Image-2.1's is 9,349,769,248 bytes against
+# 17,534,339,488 for the dense encoder, 0.533, rounded up.
+TE_INT8_CONVROT_BUDGET_SCALE = 0.56
def te_prequant_budget_scale(
@@ -156,7 +159,24 @@ def te_prequant_budget_scale(
)
except Exception: # noqa: BLE001 -- an unresolvable pre-cast just means the dense encoder
return 1.0
- return TE_PREQUANT_BUDGET_SCALE if sources else 1.0
+ if not sources:
+ return 1.0
+ if _int8_convrot_only(te_quant_mode, target):
+ return TE_INT8_CONVROT_BUDGET_SCALE
+ return TE_PREQUANT_BUDGET_SCALE
+
+
+def _int8_convrot_only(te_quant_mode: Optional[str], target: Any) -> bool:
+ """Whether this pick can ONLY take the int8 ConvRot file (no fp8 fallback): Apple Silicon."""
+ try:
+ from .diffusion_precision import TE_QUANT_FP8, normalize_te_quant, te_quant_supported
+ return (
+ normalize_te_quant(te_quant_mode) == "int8"
+ and int8_convrot_te_runs_on(target)
+ and not te_quant_supported(target, TE_QUANT_FP8)
+ )
+ except Exception: # noqa: BLE001
+ return False
# Bases whose text-encoder weights are VERIFIED byte-identical (every shard LFS sha256 compared on 2026-07-18), so one
@@ -226,6 +246,15 @@ class TePrequantSource:
fallback_filenames: tuple = ()
+def int8_convrot_te_runs_on(target: Any) -> bool:
+ """MPS runs the int8 ConvRot encoder without fp8: plain tensors, a Hadamard matmul and ``F.linear``."""
+ return getattr(target, "device", None) == "mps"
+
+
+def _is_fp8_te_name(name: Optional[str]) -> bool:
+ return bool(name) and "fp8" in str(name).lower()
+
+
def te_prequant_repo_stem(repo_id: str, component: str, scheme: str) -> str:
"""The extensionless checkpoint name for ``(component, scheme)`` in ``repo_id``: hosted repos
are named <Model>-FP8 (or -INT8 / -quantized) and carry <Model>-<component>-<SCHEME> files,
@@ -398,12 +427,17 @@ def te_prequant_sources(
denied = getattr(precision, "_te_family_denied", None)
if callable(denied) and denied(family, mode):
return {}
- # int8 dequantizes to bf16 per call (no torchao) and falls back to the fp8 file: it needs what fp8 needs.
- if not te_quant_supported(target, TE_QUANT_FP8):
+ # int8 falls back to the fp8 file, so it needs fp8; MPS runs the int8 file alone and drops the fp8 names.
+ fp8_runs = te_quant_supported(target, TE_QUANT_FP8)
+ if not fp8_runs and not (mode == "int8" and int8_convrot_te_runs_on(target)):
return {}
sources: dict[str, TePrequantSource] = {}
for component in components:
source = resolve_te_prequant_source(fam, component, mode)
+ if source is not None and not fp8_runs:
+ if source.kind == "repo" and _is_fp8_te_name(source.filename):
+ continue
+ source = replace(source, fallback_filenames = ())
if source is not None:
sources[component] = source
return sources
@@ -745,7 +779,7 @@ def _te_prequant_pipe_kwargs(
The later ``quantize_text_encoders`` call re-applies the cast idempotently and keeps
status reporting truthful."""
try:
- from .diffusion_precision import TE_QUANT_FP8, normalize_te_quant
+ from .diffusion_precision import TE_QUANT_FP8, normalize_te_quant, te_quant_supported
from .diffusion_text_encoder_trim import family_trims_lm_head
sources = te_prequant_sources_for_base(
@@ -780,6 +814,7 @@ def _te_prequant_pipe_kwargs(
and mode == "int8"
and source.kind == "repo"
and fp8_names
+ and te_quant_supported(target, TE_QUANT_FP8)
and _held_locally(source.location, source.filename, hf_token)
):
# The int8 file resolved but was refused: the plan dropped the dense shards, so take the fp8 names.
diff --git a/studio/backend/core/inference/diffusion_te_release.py b/studio/backend/core/inference/diffusion_te_release.py
new file mode 100644
index 00000000000..32139a81e06
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_te_release.py
@@ -0,0 +1,372 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Release the text encoders during denoise on unified memory, where offloading frees nothing.
+
+The first release snapshots the encoders' large tensors to local disk; a forward pre-hook reloads them bit-exact before
+an encoder runs again. ``UNSLOTH_DIFFUSION_RELEASE_TEXT_ENCODER=1`` / ``=0`` forces it on / off.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import threading
+import time
+import uuid
+import weakref
+from pathlib import Path
+from typing import Any, Optional
+
+RELEASE_ENV = "UNSLOTH_DIFFUSION_RELEASE_TEXT_ENCODER"
+TEXT_ENCODER_ATTRS = ("text_encoder", "text_encoder_2", "text_encoder_3", "text_encoder_4")
+# Tensors below this stay resident: norms, biases and rotary tables are a rounding error next to the projections.
+_MIN_RELEASE_BYTES = 1 << 20
+# Snapshot shard size: bounds the host copy a write holds at once (unified memory has no second pool to spill to).
+_SHARD_BYTES = 512 << 20
+_SNAPSHOT_DIR = "text_encoder_release"
+
+
+def release_override() -> Optional[bool]:
+ """``True`` / ``False`` when ``UNSLOTH_DIFFUSION_RELEASE_TEXT_ENCODER`` forces it, else None (auto)."""
+ value = (os.environ.get(RELEASE_ENV) or "").strip().lower()
+ if value in ("1", "true", "yes", "on"):
+ return True
+ if value in ("0", "false", "no", "off"):
+ return False
+ return None
+
+
+def release_wanted(plan: Any) -> tuple[bool, str]:
+ """``(wanted, reason)``: auto engages only on unified memory when weights plus generation headroom exceed the
+ safe budget."""
+ forced = release_override()
+ if forced is not None:
+ return forced, f"{RELEASE_ENV}={'1' if forced else '0'}"
+ try:
+ memory = plan.device_memory
+ if getattr(memory, "memory_kind", None) != "unified_memory":
+ return False, "not unified memory"
+ estimates = plan.estimates
+ budget = estimates.get("safe_device_budget_mib")
+ required = estimates.get("resident_required_mib")
+ encoders = estimates.get("text_encoder_dense_mib")
+ except Exception: # noqa: BLE001 - no plan, no release
+ return False, "no memory plan"
+ if budget is None or required is None or not encoders:
+ return False, "budget or text-encoder size unknown"
+ if int(required) <= int(budget):
+ return (
+ False,
+ f"weights and generation headroom ({required} MiB) fit the {budget} MiB budget",
+ )
+ return True, (
+ f"unified memory: weights and generation headroom ({required} MiB) exceed the {budget} MiB budget, "
+ f"and the text encoders ({encoders} MiB) run once per prompt"
+ )
+
+
+def _snapshot_root() -> Path:
+ try:
+ from utils.paths.storage_roots import cache_root
+ return Path(cache_root()) / _SNAPSHOT_DIR
+ except Exception: # noqa: BLE001 - no Studio home (tests, bare scripts): the system temp dir
+ import tempfile
+ return Path(tempfile.gettempdir()) / f"unsloth_{_SNAPSHOT_DIR}"
+
+
+def _pid_alive(pid: int) -> bool:
+ try:
+ os.kill(pid, 0)
+ except ProcessLookupError:
+ return False
+ except Exception: # noqa: BLE001 - permission errors mean the process exists
+ return True
+ return True
+
+
+def _sweep_stale(root: Path) -> None:
+ """Delete snapshots a crashed backend left behind (the directory name carries its pid)."""
+ try:
+ entries = list(root.iterdir())
+ except OSError:
+ return
+ for entry in entries:
+ pid_text = entry.name.split("-", 1)[0]
+ if pid_text.isdigit() and int(pid_text) != os.getpid() and not _pid_alive(int(pid_text)):
+ shutil.rmtree(entry, ignore_errors = True)
+
+
+def _offload_hooked(module: Any) -> bool:
+ """accelerate / diffusers offload hooks move weights themselves; never release under them."""
+ for sub in module.modules():
+ if (
+ getattr(sub, "_hf_hook", None) is not None
+ or getattr(sub, "_diffusers_hook", None) is not None
+ ):
+ return True
+ return False
+
+
+def _plain_tensor(tensor: Any) -> bool:
+ import torch
+ return type(tensor) in (torch.Tensor, torch.nn.Parameter) and tensor.device.type != "meta"
+
+
+class TextEncoderReleaser:
+ """Frees a pipeline's text-encoder weights during denoise and restores them, bit for bit, before they run again."""
+
+ def __init__(
+ self,
+ encoders: list[tuple[str, Any]],
+ *,
+ logger: Any = None,
+ ) -> None:
+ self._encoders = encoders
+ self._logger = logger
+ self._lock = threading.RLock()
+ self._released = False
+ self._snapshot: Optional[Path] = None
+ self._shards: list[Path] = []
+ # key -> (owner module, leaf name, is_parameter)
+ self._slots: dict[str, tuple[Any, str, bool]] = {}
+ self._released_bytes = 0
+ self._hooks: list[Any] = []
+ self.reloads = 0
+ self.releases = 0
+ self._collect()
+ self._install_hooks()
+ self._finalizer = weakref.finalize(self, _remove_dir, None)
+
+ def _collect(self) -> None:
+ seen: set[int] = set()
+ for attr, encoder in self._encoders:
+ for module_name, module in encoder.named_modules():
+ for leaf, param in list(module._parameters.items()):
+ if param is None or id(param) in seen:
+ continue
+ seen.add(id(param))
+ if param.numel() * param.element_size() >= _MIN_RELEASE_BYTES:
+ self._slots[f"{attr}.{module_name}.{leaf}"] = (module, leaf, True)
+ for leaf, buf in list(module._buffers.items()):
+ if buf is None or id(buf) in seen or leaf in module._non_persistent_buffers_set:
+ continue
+ seen.add(id(buf))
+ if buf.numel() * buf.element_size() >= _MIN_RELEASE_BYTES:
+ self._slots[f"{attr}.{module_name}.{leaf}"] = (module, leaf, False)
+
+ def _install_hooks(self) -> None:
+ owners = {id(owner): owner for owner, _, _ in self._slots.values()}
+ for _, encoder in self._encoders:
+ owners.setdefault(id(encoder), encoder)
+
+ def _pre_hook(module: Any, args: Any) -> None:
+ if self._released:
+ self.ensure_loaded()
+
+ for owner in owners.values():
+ self._hooks.append(owner.register_forward_pre_hook(_pre_hook))
+
+ @property
+ def released(self) -> bool:
+ return self._released
+
+ @property
+ def releasable_bytes(self) -> int:
+ return sum(
+ getattr(owner, "_parameters" if is_param else "_buffers")[leaf].numel()
+ * getattr(owner, "_parameters" if is_param else "_buffers")[leaf].element_size()
+ for owner, leaf, is_param in self._slots.values()
+ )
+
+ def _tensor(self, key: str) -> Any:
+ owner, leaf, is_param = self._slots[key]
+ return (owner._parameters if is_param else owner._buffers)[leaf]
+
+ def _write_snapshot(self) -> None:
+ import torch
+ from safetensors.torch import save_file
+
+ root = _snapshot_root()
+ root.mkdir(parents = True, exist_ok = True)
+ _sweep_stale(root)
+ need = self.releasable_bytes
+ free = shutil.disk_usage(root).free
+ if free < need + (1 << 30):
+ raise RuntimeError(
+ f"not enough free disk for the text-encoder snapshot ({need / 1e9:.1f} GB needed, "
+ f"{free / 1e9:.1f} GB free under {root})"
+ )
+ snapshot = root / f"{os.getpid()}-{uuid.uuid4().hex[:10]}"
+ snapshot.mkdir()
+ self._finalizer.detach()
+ self._finalizer = weakref.finalize(self, _remove_dir, str(snapshot))
+ shards: list[Path] = []
+ batch: dict[str, Any] = {}
+ size = 0
+
+ def _flush() -> None:
+ nonlocal batch, size
+ if not batch:
+ return
+ path = snapshot / f"shard-{len(shards):05d}.safetensors"
+ save_file(batch, str(path))
+ shards.append(path)
+ batch, size = {}, 0
+
+ with torch.no_grad():
+ for key in self._slots:
+ tensor = self._tensor(key).detach()
+ batch[key] = tensor.to("cpu", copy = True).contiguous()
+ size += tensor.numel() * tensor.element_size()
+ if size >= _SHARD_BYTES:
+ _flush()
+ _flush()
+ self._snapshot = snapshot
+ self._shards = shards
+
+ def release(self) -> int:
+ """Bytes freed; 0 when already released or refused."""
+ import torch
+ with self._lock:
+ if self._released or not self._slots:
+ return 0
+ started = time.monotonic()
+ try:
+ for _, encoder in self._encoders:
+ if _offload_hooked(encoder):
+ raise RuntimeError("an offload hook manages these weights")
+ for key in self._slots:
+ if not _plain_tensor(self._tensor(key)):
+ raise RuntimeError(f"{key} is not a plain dense tensor")
+ if self._snapshot is None:
+ self._write_snapshot()
+ except Exception as exc: # noqa: BLE001 - releasing is an optimisation: stay resident
+ self._note("warning", "diffusion.text_encoder_release: kept resident: %s", exc)
+ self._slots = {}
+ return 0
+ released = 0
+ with torch.no_grad():
+ for key, (owner, leaf, is_param) in self._slots.items():
+ tensor = self._tensor(key)
+ released += tensor.numel() * tensor.element_size()
+ empty = torch.empty(0, dtype = tensor.dtype, device = tensor.device)
+ if is_param:
+ owner._parameters[leaf].data = empty
+ else:
+ owner._buffers[leaf] = empty
+ self._released = True
+ self._released_bytes = released
+ self.releases += 1
+ _empty_device_cache()
+ self._note(
+ "info",
+ "diffusion.text_encoder_release: released %.2f GB of text-encoder weights for the denoise (%.2f s)",
+ released / 1e9,
+ time.monotonic() - started,
+ )
+ return released
+
+ def ensure_loaded(self) -> None:
+ """Restore every released tensor from the snapshot onto its original device."""
+ import torch
+ from safetensors import safe_open
+
+ with self._lock:
+ if not self._released:
+ return
+ started = time.monotonic()
+ with torch.no_grad():
+ for shard in self._shards:
+ with safe_open(str(shard), framework = "pt", device = "cpu") as handle:
+ for key in handle.keys():
+ owner, leaf, is_param = self._slots[key]
+ current = (owner._parameters if is_param else owner._buffers)[leaf]
+ restored = handle.get_tensor(key).to(current.device)
+ if is_param:
+ owner._parameters[leaf].data = restored
+ else:
+ owner._buffers[leaf] = restored
+ self._released = False
+ self.reloads += 1
+ self._note(
+ "info",
+ "diffusion.text_encoder_release: reloaded %.2f GB of text-encoder weights in %.2f s",
+ self._released_bytes / 1e9,
+ time.monotonic() - started,
+ )
+
+ def close(self) -> None:
+ """Drop the hooks and the snapshot; the weights stay as they are."""
+ with self._lock:
+ for hook in self._hooks:
+ try:
+ hook.remove()
+ except Exception: # noqa: BLE001
+ pass
+ self._hooks = []
+ self._finalizer()
+
+ def _note(self, level: str, msg: str, *args: Any) -> None:
+ if self._logger is not None:
+ getattr(self._logger, level)(msg, *args)
+
+
+def _remove_dir(path: Optional[str]) -> None:
+ if path:
+ shutil.rmtree(path, ignore_errors = True)
+
+
+def _empty_device_cache() -> None:
+ import torch
+ for backend in ("mps", "cuda"):
+ module = getattr(torch, backend, None)
+ try:
+ if module is not None and module.is_available():
+ module.empty_cache()
+ except Exception: # noqa: BLE001
+ pass
+
+
+def maybe_install(
+ pipe: Any,
+ plan: Any,
+ *,
+ logger: Any = None,
+) -> Optional[TextEncoderReleaser]:
+ """A releaser for ``pipe`` when ``plan`` calls for one (see ``release_wanted``), else None. Never raises."""
+ try:
+ wanted, reason = release_wanted(plan)
+ if not wanted:
+ return None
+ encoders = [
+ (attr, getattr(pipe, attr))
+ for attr in TEXT_ENCODER_ATTRS
+ if _is_module(getattr(pipe, attr, None))
+ ]
+ if not encoders:
+ return None
+ releaser = TextEncoderReleaser(encoders, logger = logger)
+ if not releaser._slots:
+ releaser.close()
+ return None
+ if logger is not None:
+ logger.info(
+ "diffusion.text_encoder_release: on for %s (%.2f GB): %s",
+ ", ".join(a for a, _ in encoders),
+ releaser.releasable_bytes / 1e9,
+ reason,
+ )
+ return releaser
+ except Exception as exc: # noqa: BLE001 - an optimisation: the encoders stay resident
+ if logger is not None:
+ logger.warning("diffusion.text_encoder_release: not installed: %s", exc)
+ return None
+
+
+def _is_module(value: Any) -> bool:
+ try:
+ import torch
+ return isinstance(value, torch.nn.Module)
+ except Exception: # noqa: BLE001
+ return False
diff --git a/studio/backend/tests/test_diffusion_te_release.py b/studio/backend/tests/test_diffusion_te_release.py
new file mode 100644
index 00000000000..efa78a7b7ba
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_te_release.py
@@ -0,0 +1,349 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Text encoders on unified memory (#11792): the int8 ConvRot encoder on Apple Silicon, the load-time refusal it
+lifts, and releasing the encoders between prompts. The release tests run on whatever accelerator the runner has
+(MPS on Apple Silicon runners, CUDA, else CPU), never skipped for the device."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_te_prequant as te_prequant
+from core.inference import diffusion_te_release as te_release
+from core.inference.diffusion_device import DiffusionDeviceTarget
+from core.inference.diffusion_memory import (
+ DeviceMemory,
+ plan_diffusion_memory,
+ unified_memory_shortfall_message,
+)
+
+GIB = 1024
+
+# Qwen-Image-2.1 on a 48 GB M5 Max, from the report: macOS said 27 GB free.
+Q21_DIT_Q4_K_M_MIB = 4_199_565_024 >> 20
+Q21_TE_BF16_MIB = 17_534_339_488 >> 20
+Q21_TE_INT8_CONVROT_MIB = 9_349_769_248 >> 20
+Q21_VAE_MIB = 650
+
+
+def _target(device: str, dtype = None) -> DiffusionDeviceTarget:
+ return DiffusionDeviceTarget(
+ device = device,
+ dtype = dtype if dtype is not None else torch.bfloat16,
+ backend = device,
+ vendor = None,
+ supports_model_cpu_offload = device == "cuda",
+ supports_default_torch_compile = device == "cuda",
+ supports_pinned_transfer = device == "cuda",
+ )
+
+
+def _q21_family():
+ from core.inference.diffusion_families import detect_family
+
+ fam = detect_family("Qwen/Qwen-Image-2.1")
+ assert fam is not None and fam.name == "qwen-image-2.1"
+ return fam
+
+
+def _plan(
+ memory: DeviceMemory,
+ weights: int,
+ encoders: int,
+ headroom: int = 8 * GIB,
+):
+ return plan_diffusion_memory(
+ target = _target("mps" if memory.memory_kind == "unified_memory" else "cuda"),
+ device_memory = memory,
+ model_dense_mib = weights,
+ runtime_headroom_mib = headroom,
+ companion_dense_mib = encoders + Q21_VAE_MIB,
+ text_encoder_dense_mib = encoders,
+ )
+
+
+M5_MAX = DeviceMemory("mps", "mps", "unified_memory", free_mib = 27 * GIB, total_mib = 48 * GIB)
+
+
+def test_mps_takes_the_int8_convrot_encoder_without_the_fp8_fallback():
+ sources = te_prequant.te_prequant_sources(
+ _q21_family(), te_quant_mode = "int8", target = _target("mps")
+ )
+ source = sources["text_encoder"]
+ assert source.filename == "Qwen-Image-2.1-text_encoder-INT8-ConvRot.safetensors"
+ assert source.fallback_filenames == ()
+
+
+def test_mps_fp8_request_still_takes_no_precast_encoder():
+ assert (
+ te_prequant.te_prequant_sources(_q21_family(), te_quant_mode = "fp8", target = _target("mps"))
+ == {}
+ )
+
+
+def test_cpu_takes_no_precast_encoder():
+ assert (
+ te_prequant.te_prequant_sources(
+ _q21_family(), te_quant_mode = "int8", target = _target("cpu", torch.float32)
+ )
+ == {}
+ )
+
+
+def test_cuda_keeps_the_fp8_fallback(monkeypatch):
+ from core.inference import diffusion_precision
+
+ monkeypatch.setattr(diffusion_precision, "te_quant_supported", lambda target, mode: True)
+ source = te_prequant.te_prequant_sources(
+ _q21_family(), te_quant_mode = "int8", target = _target("cuda")
+ )["text_encoder"]
+ assert source.fallback_filenames, "the CUDA path must keep its fp8 fallback names"
+ assert (
+ te_prequant.te_prequant_budget_scale(
+ _q21_family(), te_quant_mode = "int8", target = _target("cuda"), base = "Qwen/Qwen-Image-2.1"
+ )
+ == te_prequant.TE_PREQUANT_BUDGET_SCALE
+ )
+
+
+def test_mps_budgets_the_int8_encoder_at_its_own_size():
+ scale = te_prequant.te_prequant_budget_scale(
+ _q21_family(), te_quant_mode = "int8", target = _target("mps"), base = "Qwen/Qwen-Image-2.1"
+ )
+ assert scale == te_prequant.TE_INT8_CONVROT_BUDGET_SCALE
+ assert Q21_TE_BF16_MIB * scale >= Q21_TE_INT8_CONVROT_MIB
+
+
+def test_int8_convrot_linear_runs_on_this_accelerator():
+ """The int8 ConvRot projection the Apple Silicon path now loads: plain tensors, so it runs on MPS too."""
+ from core.inference.video_minimax_h3_te import _int8_convrot_linear_class
+
+ device = _device()
+ torch.manual_seed(0)
+ dense = torch.nn.Linear(512, 256, bias = True)
+ codes, scale = te_prequant.quantize_int8_convrot_weight(dense.weight.detach(), group_size = 256)
+ layer = _int8_convrot_linear_class()(512, 256, bias = True, group_size = 256)
+ layer.load_state_dict(
+ {"weight": codes, "weight_scale": scale, "bias": dense.bias.detach().to(torch.bfloat16)},
+ assign = True,
+ )
+ layer = layer.to(device)
+ x = torch.randn(4, 512)
+ with torch.no_grad():
+ got = layer(x.to(device)).float().cpu()
+ want = dense(x)
+ rel = (got - want).norm() / want.norm()
+ assert got.shape == (4, 256)
+ assert rel < 0.02, float(rel)
+
+
+def test_m5_max_refuses_the_bf16_encoder():
+ weights = Q21_DIT_Q4_K_M_MIB + Q21_TE_BF16_MIB + Q21_VAE_MIB
+ assert unified_memory_shortfall_message(_plan(M5_MAX, weights, Q21_TE_BF16_MIB)) is not None
+
+
+def test_m5_max_accepts_the_int8_convrot_encoder():
+ encoders = int(Q21_TE_BF16_MIB * te_prequant.TE_INT8_CONVROT_BUDGET_SCALE)
+ weights = Q21_DIT_Q4_K_M_MIB + encoders + Q21_VAE_MIB
+ assert unified_memory_shortfall_message(_plan(M5_MAX, weights, encoders)) is None
+
+
+def test_a_genuinely_oversized_load_is_still_refused():
+ # The dense bf16 Qwen-Image-2.1 transformer with the int8 encoder does not fit 27 GB free either.
+ encoders = int(Q21_TE_BF16_MIB * te_prequant.TE_INT8_CONVROT_BUDGET_SCALE)
+ weights = (14_230_284_408 >> 20) + encoders + Q21_VAE_MIB
+ assert unified_memory_shortfall_message(_plan(M5_MAX, weights, encoders)) is not None
+
+
+def test_discrete_vram_never_refuses_here():
+ memory = DeviceMemory("cuda", "cuda", "discrete_vram", free_mib = 8 * GIB, total_mib = 24 * GIB)
+ weights = Q21_DIT_Q4_K_M_MIB + Q21_TE_BF16_MIB + Q21_VAE_MIB
+ assert unified_memory_shortfall_message(_plan(memory, weights, Q21_TE_BF16_MIB)) is None
+
+
+def test_release_engages_on_unified_memory_when_the_resident_set_does_not_fit(monkeypatch):
+ monkeypatch.delenv(te_release.RELEASE_ENV, raising = False)
+ encoders = int(Q21_TE_BF16_MIB * te_prequant.TE_INT8_CONVROT_BUDGET_SCALE)
+ weights = Q21_DIT_Q4_K_M_MIB + encoders + Q21_VAE_MIB
+ wanted, reason = te_release.release_wanted(_plan(M5_MAX, weights, encoders))
+ assert wanted, reason
+
+
+def test_release_stays_off_when_everything_fits(monkeypatch):
+ monkeypatch.delenv(te_release.RELEASE_ENV, raising = False)
+ roomy = DeviceMemory("mps", "mps", "unified_memory", free_mib = 100 * GIB, total_mib = 128 * GIB)
+ weights = Q21_DIT_Q4_K_M_MIB + Q21_TE_BF16_MIB + Q21_VAE_MIB
+ assert te_release.release_wanted(_plan(roomy, weights, Q21_TE_BF16_MIB))[0] is False
+
+
+def test_release_stays_off_on_discrete_vram(monkeypatch):
+ monkeypatch.delenv(te_release.RELEASE_ENV, raising = False)
+ memory = DeviceMemory("cuda", "cuda", "discrete_vram", free_mib = 8 * GIB, total_mib = 24 * GIB)
+ weights = Q21_DIT_Q4_K_M_MIB + Q21_TE_BF16_MIB + Q21_VAE_MIB
+ assert te_release.release_wanted(_plan(memory, weights, Q21_TE_BF16_MIB))[0] is False
+
+
[email protected]("value, expected", [("1", True), ("0", False)])
+def test_release_env_forces_either_way(monkeypatch, value, expected):
+ monkeypatch.setenv(te_release.RELEASE_ENV, value)
+ memory = DeviceMemory("cuda", "cuda", "discrete_vram", free_mib = 8 * GIB, total_mib = 24 * GIB)
+ assert te_release.release_wanted(_plan(memory, 1, 1))[0] is expected
+
+
+def _device() -> str:
+ if getattr(torch.backends, "mps", None) is not None and torch.backends.mps.is_available():
+ return "mps"
+ if torch.cuda.is_available():
+ return "cuda"
+ return "cpu"
+
+
+class _Encoder(torch.nn.Module):
+ """A stand-in encoder: a tied embedding / head pair, int8 buffers like the ConvRot encoder, a small norm."""
+
+ def __init__(self):
+ super().__init__()
+ self.embed = torch.nn.Embedding(512, 1024)
+ self.proj = torch.nn.Linear(1024, 1024)
+ self.register_buffer("codes", torch.randint(-127, 127, (1024, 1024), dtype = torch.int8))
+ self.register_buffer("rotary", torch.arange(16, dtype = torch.float32), persistent = False)
+ self.norm = torch.nn.LayerNorm(1024)
+ self.head = torch.nn.Linear(1024, 512, bias = False)
+ self.head.weight = self.embed.weight
+
+ def forward(self, ids):
+ h = self.norm(
+ self.proj(self.embed(ids))
+ + self.codes[: ids.shape[-1]].to(self.proj.weight.dtype).sum(-1, keepdim = True)
+ )
+ return self.head(h) + self.rotary.sum()
+
+
+def snapshot_root(tmp_path, monkeypatch):
+ monkeypatch.setattr(te_release, "_snapshot_root", lambda: tmp_path / "snap")
+ return tmp_path / "snap"
+
+
+def _pipe(device: str):
+ torch.manual_seed(0)
+ encoder = _Encoder().to(device)
+ return types.SimpleNamespace(text_encoder = encoder, transformer = torch.nn.Linear(4, 4).to(device))
+
+
+def test_release_frees_and_reload_restores_bit_for_bit(snapshot_root):
+ device = _device()
+ pipe = _pipe(device)
+ ids = torch.arange(8, device = device)
+ with torch.no_grad():
+ before = pipe.text_encoder(ids).cpu()
+ releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])
+ total = releaser.releasable_bytes
+ assert total > 2 << 20
+ assert releaser.release() == total
+ assert releaser.released
+ assert pipe.text_encoder.proj.weight.numel() == 0
+ assert pipe.text_encoder.codes.numel() == 0
+ # Device and dtype survive, so pipelines that read them before encoding still work.
+ assert pipe.text_encoder.proj.weight.device.type == device
+ assert pipe.text_encoder.proj.weight.dtype == torch.float32
+ assert pipe.text_encoder.rotary.numel() == 16
+ with torch.no_grad():
+ after = pipe.text_encoder(ids).cpu()
+ assert not releaser.released and releaser.reloads == 1
+ assert torch.equal(before, after)
+ # The tie survives: one tensor, released and restored once.
+ assert pipe.text_encoder.head.weight is pipe.text_encoder.embed.weight
+ # A second release reuses the snapshot; a submodule called directly also reloads.
+ shards = list(snapshot_root.rglob("*.safetensors"))
+ assert releaser.release() == total
+ assert sorted(snapshot_root.rglob("*.safetensors")) == sorted(shards)
+ with torch.no_grad():
+ pipe.text_encoder.proj(torch.zeros(1, 1024, device = device))
+ assert releaser.reloads == 2
+ releaser.close()
+ assert not any(snapshot_root.rglob("*.safetensors"))
+
+
+def test_release_refuses_offload_hooked_encoders(snapshot_root):
+ pipe = _pipe(_device())
+ pipe.text_encoder.proj._hf_hook = object()
+ releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])
+ assert releaser.release() == 0
+ assert pipe.text_encoder.proj.weight.numel() > 0
+
+
+def test_release_refuses_when_the_disk_is_full(snapshot_root, monkeypatch):
+ pipe = _pipe(_device())
+ usage = types.SimpleNamespace(total = 1, used = 1, free = 1)
+ monkeypatch.setattr(te_release.shutil, "disk_usage", lambda path: usage)
+ releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])
+ assert releaser.release() == 0
+ assert pipe.text_encoder.proj.weight.numel() > 0
+
+
+def test_maybe_install_only_when_wanted(snapshot_root, monkeypatch):
+ pipe = _pipe(_device())
+ monkeypatch.setenv(te_release.RELEASE_ENV, "0")
+ assert te_release.maybe_install(pipe, None) is None
+ monkeypatch.setenv(te_release.RELEASE_ENV, "1")
+ releaser = te_release.maybe_install(pipe, None)
+ assert releaser is not None
+ releaser.close()
+
+
+def test_tiny_qwen_image_21_pipeline_renders_the_same_with_release(snapshot_root, monkeypatch):
+ """The real QwenImage21Pipeline, tiny: release at denoise entry, reload for a new prompt, same images."""
+ diffusers = pytest.importorskip("diffusers")
+ pipeline_cls = getattr(diffusers, "QwenImage21Pipeline", None)
+ if pipeline_cls is None:
+ pytest.skip("this diffusers has no QwenImage21Pipeline")
+ from huggingface_hub import snapshot_download
+
+ try:
+ path = snapshot_download("hf-internal-testing/tiny-qwenimage21-pipe")
+ except Exception as exc: # noqa: BLE001
+ pytest.skip(f"tiny pipeline not reachable: {exc}")
+ from core.inference.media_decode_phase import denoise_phase
+
+ device = _device()
+ dtype = torch.float32
+
+ def render(
+ pipe,
+ prompt,
+ releaser = None,
+ ):
+ def on_denoise():
+ if releaser is not None:
+ releaser.release()
+
+ with denoise_phase(pipe, on_denoise):
+ out = pipe(
+ prompt = prompt,
+ height = 64,
+ width = 64,
+ num_inference_steps = 2,
+ generator = torch.Generator("cpu").manual_seed(0),
+ output_type = "pt",
+ )
+ return out.images.cpu()
+
+ pipe = pipeline_cls.from_pretrained(path, torch_dtype = dtype).to(device)
+ pipe.set_progress_bar_config(disable = True)
+ reference = [render(pipe, "a red cube"), render(pipe, "a blue sphere")]
+
+ releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])
+ first = render(pipe, "a red cube", releaser)
+ assert releaser.released and releaser.releases == 1
+ second = render(pipe, "a blue sphere", releaser)
+ assert releaser.reloads == 1 and releaser.releases == 2
+ assert torch.equal(first, reference[0])
+ assert torch.equal(second, reference[1])
+ releaser.close()