Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: int8 ConvRot text encoder for Qwen-Image-2.1, fp8 fallback (#12684)

Daniel Han9bf3dec61572
diff --git a/scripts/build_te_int8_convrot_checkpoint.py b/scripts/build_te_int8_convrot_checkpoint.py
new file mode 100644
index 00000000000..5b7086e9dda
--- /dev/null
+++ b/scripts/build_te_int8_convrot_checkpoint.py
@@ -0,0 +1,186 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Build an int8 ConvRot weight-only text-encoder checkpoint for the Unsloth TE prequant path.
+
+Every decoder projection of the dense encoder (q/k/v/o and gate/up/down in each
+``model.language_model.layers.N``) is rotated by the group-256 Hadamard and stored int8 with a
+float32 per-output-channel scale (``diffusion_te_prequant.quantize_int8_convrot_weight``); the
+vision tower, embedding table and norms are copied bf16, bit for bit; ``lm_head`` is left out
+(the pipelines that load this file read hidden states only). Each quantized Linear carries a
+``<name>.comfy_quant`` JSON blob naming the format, and the header is the plain-tensor container
+the TE prequant loader reads (``tensor_names`` + ``unsloth_format`` + ``unsloth_metadata``).
+
+Reads one source tensor at a time, so host RAM peaks at about the output size. Deterministic: the
+same source and group give a byte-identical file.
+
+ python scripts/build_te_int8_convrot_checkpoint.py \\
+ --base Qwen/Qwen-Image-2.1 --family qwen-image-2.1 \\
+ --out Qwen-Image-2.1-text_encoder-INT8-ConvRot.safetensors
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import sys
+from pathlib import Path
+
+BACKEND = Path(__file__).resolve().parent.parent / "studio" / "backend"
+DECODER_LINEAR = re.compile(
+ r"^model\.language_model\.layers\.\d+\.(self_attn\.[qkvo]_proj|mlp\.(gate|up|down)_proj)\.weight$"
+)
+
+
+_ST_DTYPES = {"float32": "F32", "bfloat16": "BF16", "float16": "F16", "int8": "I8", "uint8": "U8"}
+
+
+def _write_safetensors(path: Path, tensors: dict, metadata: dict) -> None:
+ """safetensors with a canonical header (sorted keys) so a rebuild is byte-identical; ``save_file`` orders the
+ metadata map differently per run. Widest dtype first, so every tensor starts at its own alignment."""
+ import torch
+
+ order = sorted(tensors, key = lambda n: (-tensors[n].element_size(), n))
+ header: dict = {"__metadata__": dict(sorted(metadata.items()))}
+ offset = 0
+ for name in order:
+ t = tensors[name]
+ size = t.numel() * t.element_size()
+ header[name] = {
+ "dtype": _ST_DTYPES[str(t.dtype).replace("torch.", "")],
+ "shape": list(t.shape),
+ "data_offsets": [offset, offset + size],
+ }
+ offset += size
+ blob = json.dumps(header, sort_keys = True, separators = (",", ":")).encode("utf-8")
+ blob += b" " * (-len(blob) % 8)
+ with open(path, "wb") as fh:
+ fh.write(len(blob).to_bytes(8, "little"))
+ fh.write(blob)
+ for name in order:
+ fh.write(tensors[name].contiguous().view(torch.uint8).numpy().tobytes())
+
+
+def _download_component(base: str, component: str, token) -> Path:
+ """The component folder of ``base``, data files only: the build reads JSON and safetensors (safe_open), never
+ anything executable, so nothing else is fetched."""
+ from huggingface_hub import snapshot_download
+
+ root = snapshot_download(
+ base,
+ allow_patterns = [f"{component}/*.json", f"{component}/*.safetensors"],
+ token = token,
+ )
+ return Path(root) / component
+
+
+def main(argv = None) -> int:
+ p = argparse.ArgumentParser()
+ p.add_argument("--base", required = True, help = "diffusers base repo carrying text_encoder/")
+ p.add_argument("--family", required = True)
+ p.add_argument("--component", default = "text_encoder")
+ p.add_argument("--src", default = None, help = "local dense encoder folder (skips the download)")
+ p.add_argument("--out", required = True, help = "output .safetensors path")
+ p.add_argument("--group", type = int, default = 256)
+ p.add_argument(
+ "--device", default = "cuda" if os.environ.get("CUDA_VISIBLE_DEVICES") != "" else "cpu"
+ )
+ p.add_argument("--hf-token", default = None)
+ args = p.parse_args(argv)
+
+ sys.path.insert(0, str(BACKEND))
+ import safetensors
+ import torch
+ import transformers
+ from safetensors import safe_open
+
+ from core.inference.diffusion_te_prequant import (
+ TE_PREQUANT_FORMAT_INT8_CONVROT,
+ quantize_int8_convrot_weight,
+ )
+ from core.inference.prequant_safetensors import UNSLOTH_FORMAT_KEY, UNSLOTH_METADATA_KEY
+
+ src = (
+ Path(args.src)
+ if args.src
+ else _download_component(args.base, args.component, args.hf_token)
+ )
+ config = json.loads((src / "config.json").read_text(encoding = "utf-8"))
+ index_path = src / "model.safetensors.index.json"
+ if index_path.is_file():
+ weight_map = json.loads(index_path.read_text(encoding = "utf-8"))["weight_map"]
+ else:
+ with safe_open(str(src / "model.safetensors"), framework = "pt") as handle:
+ weight_map = {name: "model.safetensors" for name in handle.keys()}
+ device = torch.device(args.device if torch.cuda.is_available() else "cpu")
+ quant = {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": args.group}
+ blob = torch.tensor(list(json.dumps(quant).encode("utf-8")), dtype = torch.uint8)
+ tensors: dict = {}
+ quantized = 0
+ for shard in sorted(set(weight_map.values())):
+ with safe_open(str(src / shard), framework = "pt", device = "cpu") as handle:
+ for name in sorted(n for n, s in weight_map.items() if s == shard):
+ if name == "lm_head.weight":
+ continue
+ tensor = handle.get_tensor(name)
+ if DECODER_LINEAR.match(name):
+ q, scale = quantize_int8_convrot_weight(tensor.to(device), args.group)
+ prefix = name[: -len(".weight")]
+ tensors[name] = q.cpu().contiguous()
+ tensors[prefix + ".weight_scale"] = scale.cpu().contiguous()
+ tensors[prefix + ".comfy_quant"] = blob.clone()
+ quantized += 1
+ else:
+ tensors[name] = tensor.to(torch.bfloat16).contiguous()
+ if not quantized:
+ raise SystemExit("no decoder projections matched; is this a Qwen3-VL text encoder?")
+ metadata = {
+ "base_model_id": args.base,
+ "family": args.family,
+ "scheme": "int8",
+ "component": args.component,
+ "te_class": (config.get("architectures") or [None])[0],
+ "torch_dtype": "bfloat16",
+ "quant": dict(
+ quant, weight_only = True, scale = "per_output_channel_absmax_over_127", linears = quantized
+ ),
+ "lm_head": "dropped",
+ "torch_version": torch.__version__,
+ "transformers_version": transformers.__version__,
+ "safetensors_version": safetensors.__version__,
+ }
+ names = sorted(tensors)
+ header = {
+ UNSLOTH_FORMAT_KEY: TE_PREQUANT_FORMAT_INT8_CONVROT,
+ UNSLOTH_METADATA_KEY: json.dumps(metadata, sort_keys = True),
+ "tensor_names": json.dumps(names),
+ }
+ header.update({name: json.dumps({"_type": "Tensor"}) for name in names})
+ out = Path(args.out)
+ out.parent.mkdir(parents = True, exist_ok = True)
+ tmp = out.with_name(out.name + ".tmp")
+ _write_safetensors(tmp, tensors, header)
+ os.replace(tmp, out)
+ digest = hashlib.sha256()
+ with open(out, "rb") as fh:
+ for chunk in iter(lambda: fh.read(1 << 24), b""):
+ digest.update(chunk)
+ print(
+ json.dumps(
+ {
+ "out": str(out),
+ "bytes": out.stat().st_size,
+ "sha256": digest.hexdigest(),
+ "quantized_linears": quantized,
+ "tensors": len(names),
+ }
+ )
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 3fd22f13b3d..6ee3f151860 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -243,7 +243,6 @@ from .diffusion_step_skip import (
)
from .diffusion_nvfp4_protect import protect_generation
from .diffusion_precision import (
- TE_QUANT_FP8,
effective_te_quant,
normalize_te_quant,
quantize_text_encoders,
@@ -3520,7 +3519,9 @@ class DiffusionBackend:
if local_files_only:
return {}
try:
- if normalize_te_quant(text_encoder_quant) != TE_QUANT_FP8:
+ from .diffusion_te_prequant import TE_PREQUANT_SCHEMES
+
+ if normalize_te_quant(text_encoder_quant) not in TE_PREQUANT_SCHEMES:
return {}
from huggingface_hub import HfApi
@@ -4451,7 +4452,13 @@ class DiffusionBackend:
}
)
+ from .diffusion_te_prequant import te_prequant_unmirrored
+
for repo, files in te_files.values():
+ # A mirrored encoder still drops the dense shards above but is never staged: the Hub may not hold it.
+ files = te_prequant_unmirrored(repo, files)
+ if not files:
+ continue
add_missing_entry(
repo,
[name for name, _size in files],
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.py
index 1e92fbcecf1..0b1307b450f 100644
--- a/studio/backend/core/inference/diffusion_families.py
+++ b/studio/backend/core/inference/diffusion_families.py
@@ -430,13 +430,14 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
("fp8", "Qwen-Image-2.1-FP8.safetensors"),
("int8", "Qwen-Image-2.1-INT8.safetensors"),
),
- # Qwen3-VL 8B, pre-cast. Independent of the DiT scheme, as on every other family.
+ # Qwen3-VL 8B pre-cast fp8, the int8 ConvRot encoder's fallback (diffusion_te_prequant.TE_INT8_CONVROT_FILES).
te_prequant_repos = (("fp8", "text_encoder", "unsloth/Qwen-Image-2.1-FP8"),),
# The encoder is the BIG component here, not the denoiser: Qwen3-VL-8B is 16.33 GiB dense against 6.76 GiB for
# the INT8 transformer, so a quantised denoiser alone still costs ~26 GB and the hosted pre-cast encoder is
# what makes the family fit a 24 GB card. Measured on this family at 1024/40 steps: 23.74 -> 16.16 GiB resident
# and 11.6 -> 1.5 s to load the encoder, with per-image render time unchanged.
- te_quant_auto = "fp8",
+ # int8 ConvRot weight-only, as ComfyUI's Qwen-Image-2.1 template (fp8 misspells rendered text).
+ te_quant_auto = "int8",
cfg_kwarg = "true_cfg_scale",
# 2.1 is UNIFIED: one pipeline, and QwenImage21Pipeline.__call__ takes ``image`` as condition
# images alongside the prompt, so this is the FLUX.2 shape rather than the Qwen-Image-Edit
diff --git a/studio/backend/core/inference/diffusion_precision.py b/studio/backend/core/inference/diffusion_precision.py
index 89611e9d4a6..61686b20a57 100644
--- a/studio/backend/core/inference/diffusion_precision.py
+++ b/studio/backend/core/inference/diffusion_precision.py
@@ -263,6 +263,15 @@ def quantize_text_encoders(
mode = normalize_te_quant(mode)
if mode is None:
return TEQuantOutcome(None)
+ # A hosted int8 ConvRot encoder is already quantized (plain tensors, offload-safe): report it, never re-cast it.
+ present = [a for a in _TEXT_ENCODER_ATTRS if getattr(pipe, a, None) is not None]
+ hosted_int8 = [a for a in present if _hosted_te_scheme(getattr(pipe, a)) == TE_QUANT_INT8]
+ if mode == TE_QUANT_INT8 and hosted_int8 and len(hosted_int8) == len(present):
+ return TEQuantOutcome(
+ TE_QUANT_INT8,
+ "hosted int8 ConvRot weight-only text encoder",
+ RESOLVED_APPLIED,
+ )
downgrade_reason = ""
skip: Optional[tuple[int, int]] = None
if mode == TE_QUANT_INT8:
@@ -274,6 +283,15 @@ def quantize_text_encoders(
f"int8 has no measured keep-bf16 schedule for family '{family}' "
"(it degrades large encoders without one), so fp8 was used instead"
)
+ if _family_hosts_te_int8(family):
+ from .diffusion_text_encoder_trim import KEEP_LM_HEAD_ENV, family_trims_lm_head
+ downgrade_reason = (
+ f"{KEEP_LM_HEAD_ENV} keeps lm_head, which the int8 ConvRot text encoder does not carry, "
+ "so fp8 was used instead"
+ if not family_trims_lm_head(family)
+ else "the hosted int8 ConvRot text encoder is not available (not published yet, offline, "
+ "or refused, see the server log), so fp8 was used instead"
+ )
# torchao modes produce subclasses that reject Module.to(), which an offload placement uses. Layerwise fp8 streams
# fine.
if offload_active and mode in (TE_QUANT_INT8, TE_QUANT_FP8_DYNAMIC, TE_QUANT_NVFP4):
@@ -313,6 +331,8 @@ def quantize_text_encoders(
encoder = getattr(pipe, attr, None)
if encoder is None:
continue
+ if attr in hosted_int8:
+ continue
try:
caster(encoder, target)
cast.append(attr)
@@ -341,6 +361,22 @@ def quantize_text_encoders(
return TEQuantOutcome(mode, "dense text encoder(s) quantised in place", RESOLVED_APPLIED)
+def _hosted_te_scheme(encoder: Any) -> Optional[str]:
+ """The scheme of the hosted artifact ``encoder`` was loaded from ("fp8" / "int8"), or None for a dense load."""
+ from .diffusion_te_prequant import TE_PREQUANT_SCHEME_ATTR
+
+ value = getattr(encoder, TE_PREQUANT_SCHEME_ATTR, None)
+ return value if isinstance(value, str) else None
+
+
+def _family_hosts_te_int8(family: Optional[str]) -> bool:
+ try:
+ from .diffusion_te_prequant import TE_INT8_CONVROT_FILES
+ return (family or "").strip().lower() in TE_INT8_CONVROT_FILES
+ except Exception: # noqa: BLE001
+ return False
+
+
def _te_exclude_tokens(encoder: Any) -> tuple[str, ...]:
"""fqn tokens whose Linears stay bf16 in a torchao TE quant: the VLM vision tower, the unused
lm_head, and the encoder's own fp32-kept modules (T5 ``wo``, which explodes in low precision)."""
diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.py
index 1f31b3eb0c5..7ba2a143845 100644
--- a/studio/backend/core/inference/diffusion_te_prequant.py
+++ b/studio/backend/core/inference/diffusion_te_prequant.py
@@ -39,8 +39,81 @@ from .diffusion_prequant import (
# torch.save dict layout tag; bump on an on-disk change so old/foreign artifacts are rejected.
TE_PREQUANT_FORMAT = "unsloth_prequant_text_encoder_state_dict_v1"
-# The one scheme hosted in v1 (see module docstring).
-TE_PREQUANT_SCHEMES = ("fp8",)
+# fp8: layerwise storage cast (families with te_prequant_repos). int8: ConvRot weight-only, TE_INT8_CONVROT_FILES only.
+TE_PREQUANT_SCHEMES = ("fp8", "int8")
+
+# Distinct from TE_PREQUANT_FORMAT so an fp8-only build refuses the file instead of loading int8 into bf16 Linears.
+TE_PREQUANT_FORMAT_INT8_CONVROT = "unsloth_prequant_text_encoder_int8_convrot_v1"
+
+# family -> {component: (repo_id, filename)}. ComfyUI's qwen3vl_8b_int8_convrot scheme: decoder projections int8 in the
+# group-256 Hadamard basis + fp32 per-row scale; vision tower, embedding, norms bf16; no lm_head. Built by
+# scripts/build_te_int8_convrot_checkpoint.py.
+TE_INT8_CONVROT_FILES: dict[str, dict[str, tuple[str, str]]] = {
+ "qwen-image-2.1": {
+ "text_encoder": (
+ "unsloth/Qwen-Image-2.1-FP8",
+ "Qwen-Image-2.1-text_encoder-INT8-ConvRot.safetensors",
+ ),
+ },
+}
+
+# The scheme the loaded FILE carried ("fp8" / "int8"), so quantize_text_encoders never re-casts an int8 encoder.
+TE_PREQUANT_SCHEME_ATTR = "_unsloth_te_prequant_scheme"
+
+# ``<root>/<owner>/<repo>/<filename>`` used before the Hub (os.pathsep-separated roots), as for the DiT pre-quants.
+TE_PREQUANT_MIRROR_ENV = "UNSLOTH_DIFFUSION_PREQUANT_MIRROR"
+
+
+def quantize_int8_convrot_weight(weight: Any, group_size: int = 256) -> tuple[Any, Any]:
+ """``(int8 weight, float32 [out, 1] scale)`` of a Linear weight in the ConvRot basis: ``W_rot = W @ blockdiag(H).T``
+ in float32, then symmetric per-output-channel absmax / 127 with round-half-to-even. The builder's quantizer, here so
+ the artifact and its tests share one definition; ``Int8ConvRotLinear`` is its inverse."""
+ import torch
+
+ from .diffusion_convrot import build_convrot_hadamard
+
+ out_features, in_features = weight.shape
+ if in_features % group_size:
+ raise ValueError(
+ f"in_features {in_features} is not divisible by the ConvRot group {group_size}"
+ )
+ h = build_convrot_hadamard(group_size, device = weight.device, dtype = torch.float32)
+ rotated = (
+ weight.float().reshape(out_features, in_features // group_size, group_size) @ h.T
+ ).reshape(out_features, in_features)
+ scale = (rotated.abs().amax(dim = 1, keepdim = True) / 127.0).clamp_min(1e-12)
+ return torch.round(rotated / scale).clamp(-127, 127).to(torch.int8), scale
+
+
+def family_te_int8_convrot(fam: Any, component: str) -> Optional[tuple[str, str]]:
+ """``(repo_id, filename)`` of the family's hosted int8 ConvRot encoder for ``component``, or None."""
+ name = str(getattr(fam, "name", fam) or "").strip().lower()
+ return TE_INT8_CONVROT_FILES.get(name, {}).get(component)
+
+
+def te_prequant_mirror_path(repo_id: Optional[str], name: Optional[str]) -> Optional[str]:
+ """``<mirror>/<repo_id>/<name>`` when a configured mirror holds that file, else None. Never raises."""
+ import os
+
+ raw = (os.environ.get(TE_PREQUANT_MIRROR_ENV) or "").strip()
+ if not raw or not repo_id or not name:
+ return None
+ parts = [*str(repo_id).split("/"), *str(name).split("/")]
+ if any(p in ("", ".", "..") or "\\" in p for p in parts):
+ return None
+ for root in raw.split(os.pathsep):
+ root = root.strip()
+ if not root:
+ continue
+ try:
+ base = os.path.realpath(os.path.expanduser(root))
+ candidate = os.path.realpath(os.path.join(base, *parts))
+ except Exception: # noqa: BLE001 - a bad root is simply not a mirror
+ continue
+ if candidate.startswith(base.rstrip(os.sep) + os.sep) and os.path.isfile(candidate):
+ return candidate
+ return None
+
# Components the pipeline-assembly injection covers (text_encoder_4 is family-assembled separately, see
# diffusion_hidream.py).
@@ -257,6 +330,27 @@ def resolve_te_prequant_source(
override = (path_override or "").strip()
if override:
return TePrequantSource(kind = "path", location = override, filename = None)
+ if scheme == "int8":
+ # int8 file first, then the fp8 names in the same repo, so a miss takes fp8 rather than the dense shards.
+ hosted = family_te_int8_convrot(fam, component)
+ if hosted is None:
+ return None
+ repo_id, filename = hosted
+ fp8 = resolve_te_prequant_source(fam, component, "fp8")
+ from .diffusion_text_encoder_trim import family_trims_lm_head
+
+ if not family_trims_lm_head(getattr(fam, "name", None)):
+ # The int8 file has no lm_head; with the head kept (UNSLOTH_TE_KEEP_LM_HEAD) only the fp8 one can serve.
+ return fp8
+ fallback = ()
+ if fp8 is not None and fp8.kind == "repo" and fp8.location.lower() == repo_id.lower():
+ fallback = te_candidate_filenames(fp8)
+ return TePrequantSource(
+ kind = "repo",
+ location = repo_id,
+ filename = filename,
+ fallback_filenames = tuple(n for n in fallback if n != filename),
+ )
repo_id = family_te_prequant_repo(fam, scheme, component)
if repo_id:
names = te_prequant_repo_filenames(repo_id, component, scheme)
@@ -296,7 +390,7 @@ def te_prequant_sources(
)
mode = normalize_te_quant(te_quant_mode)
- if mode != TE_QUANT_FP8:
+ if mode not in TE_PREQUANT_SCHEMES:
return {}
family = getattr(fam, "name", None)
# The per-family TE deny table ships on the video branch precision module (the image branch has no denials), so
@@ -304,7 +398,8 @@ def te_prequant_sources(
denied = getattr(precision, "_te_family_denied", None)
if callable(denied) and denied(family, mode):
return {}
- if not te_quant_supported(target, mode):
+ # int8 dequantizes to bf16 per call (no torchao) and falls back to the fp8 file: it needs what fp8 needs.
+ if not te_quant_supported(target, TE_QUANT_FP8):
return {}
sources: dict[str, TePrequantSource] = {}
for component in components:
@@ -406,6 +501,8 @@ def load_prequant_text_encoder(
)
return None
+ import os
+
from utils.hf_cache_settings import active_hf_hub_cache
cache_dir = active_hf_hub_cache()
@@ -444,7 +541,16 @@ def load_prequant_text_encoder(
ckpt = load_plain_prequant_safetensors(path, skip_names = skip)
else:
ckpt = torch.load(path, weights_only = True, map_location = "cpu")
- if not _validate_checkpoint(ckpt, scheme, component, base, logger):
+ # By format tag: an int8 request accepts its fp8 fallback; an fp8 request never accepts an int8 file.
+ file_scheme = (
+ "int8"
+ if isinstance(ckpt, dict) and ckpt.get("format") == TE_PREQUANT_FORMAT_INT8_CONVROT
+ else "fp8"
+ )
+ if file_scheme != scheme and not (scheme == "int8" and file_scheme == "fp8"):
+ _warn(logger, scheme, ValueError(f"checkpoint scheme {file_scheme!r} != {scheme!r}"))
+ return None
+ if not _validate_checkpoint(ckpt, file_scheme, component, base, logger):
return None
state_dict = ckpt["state_dict"]
te_class = (ckpt.get("metadata") or {}).get("te_class")
@@ -477,6 +583,20 @@ def load_prequant_text_encoder(
remap_rope_parameters(getattr(config, "text_config", config))
for key, value in (config_overrides or {}).items():
setattr(config, key, value)
+ if file_scheme == "int8":
+ encoder = _build_int8_convrot_encoder(
+ encoder_cls, config, state_dict, dtype = dtype, trim = trim
+ )
+ setattr(encoder, TE_PREQUANT_SCHEME_ATTR, "int8")
+ if logger is not None:
+ logger.info(
+ "diffusion.te_prequant: loaded %s int8 ConvRot weight-only checkpoint (%s)",
+ component,
+ path
+ if source.kind == "path"
+ else f"{source.location}/{os.path.basename(path)}",
+ )
+ return encoder
if trim and config_ties_lm_head(config):
# Tied: nothing to drop.
trim = False
@@ -517,6 +637,7 @@ def load_prequant_text_encoder(
target = _Target()
target.dtype = dtype
_cast_fp8(encoder, target)
+ setattr(encoder, TE_PREQUANT_SCHEME_ATTR, "fp8")
if logger is not None:
logger.info(
"diffusion.te_prequant: loaded %s %s checkpoint (%s)",
@@ -551,7 +672,7 @@ def te_prequant_pipe_kwargs(
The later ``quantize_text_encoders`` call re-applies the cast idempotently and keeps
status reporting truthful."""
try:
- from .diffusion_precision import TE_QUANT_FP8
+ from .diffusion_precision import TE_QUANT_FP8, normalize_te_quant
from .diffusion_text_encoder_trim import family_trims_lm_head
sources = te_prequant_sources_for_base(
@@ -560,10 +681,11 @@ def te_prequant_pipe_kwargs(
te_quant_mode = te_quant_mode,
target = target,
)
- # Non-empty only for the one hosted scheme (see te_prequant_sources' gate).
- mode = TE_QUANT_FP8
+ # Non-empty only for a hosted scheme (see te_prequant_sources' gate).
+ mode = normalize_te_quant(te_quant_mode) or TE_QUANT_FP8
injected: dict[str, Any] = {}
for component, source in sources.items():
+ trim = component == "text_encoder" and family_trims_lm_head(getattr(fam, "name", None))
encoder = load_prequant_text_encoder(
base,
component,
@@ -573,9 +695,35 @@ def te_prequant_pipe_kwargs(
scheme = mode,
logger = logger,
local_files_only = local_files_only,
- trim_lm_head = component == "text_encoder"
- and family_trims_lm_head(getattr(fam, "name", None)),
+ trim_lm_head = trim,
)
+ fp8_names = tuple(
+ n for n in te_candidate_filenames(source) if n != getattr(source, "filename", None)
+ )
+ if (
+ encoder is None
+ and mode == "int8"
+ and source.kind == "repo"
+ and fp8_names
+ and _held_locally(source.location, source.filename, hf_token)
+ ):
+ # The int8 file resolved but was refused: the plan dropped the dense shards, so take the fp8 names.
+ encoder = load_prequant_text_encoder(
+ base,
+ component,
+ TePrequantSource(
+ kind = "repo",
+ location = source.location,
+ filename = fp8_names[0],
+ fallback_filenames = fp8_names[1:],
+ ),
+ dtype = dtype,
+ hf_token = hf_token,
+ scheme = TE_QUANT_FP8,
+ logger = logger,
+ local_files_only = local_files_only,
+ trim_lm_head = trim,
+ )
if encoder is not None:
injected[component] = encoder
return injected
@@ -584,6 +732,113 @@ def te_prequant_pipe_kwargs(
return {}
+def _held_locally(repo_id: str, name: Optional[str], hf_token: Optional[str]) -> bool:
+ """Whether ``repo_id/name`` is on this machine (mirror or Hub cache), without touching the network."""
+ try:
+ if te_prequant_mirror_path(repo_id, name) is not None:
+ return True
+ from huggingface_hub import hf_hub_download
+
+ from utils.hf_cache_settings import active_hf_hub_cache
+
+ hf_hub_download(
+ repo_id = repo_id,
+ filename = name,
+ token = hf_token,
+ cache_dir = active_hf_hub_cache(),
+ local_files_only = True,
+ )
+ return True
+ except Exception: # noqa: BLE001 - not cached, or unanswerable
+ return False
+
+
+def _build_int8_convrot_encoder(
+ encoder_cls: Any, config: Any, state_dict: dict, *, dtype: Any, trim: bool
+) -> Any:
+ """Every Linear with a ``<name>.weight_scale`` becomes MiniMax-H3's ``Int8ConvRotLinear``, the rest loads dense in
+ ``dtype``. Plain tensors, so ``Module.to()`` and group offloading work. Raises on any mismatch."""
+ import torch
+ from accelerate import init_empty_weights
+
+ from .diffusion_text_encoder_trim import LM_HEAD_KEY, trim_text_encoder
+ from .video_minimax_h3_te import (
+ H3_TE_CONVROT_GROUP,
+ _int8_convrot_linear_class,
+ _meta_tensor_names,
+ _validate_comfy_quant,
+ )
+
+ if not trim:
+ # The file has no lm_head; a build that keeps one would load it random.
+ raise ValueError("the int8 ConvRot encoder carries no lm_head and needs the lm_head trim")
+ if LM_HEAD_KEY in state_dict:
+ raise ValueError("the int8 ConvRot encoder unexpectedly carries an lm_head")
+ scale_suffix, quant_suffix = ".weight_scale", ".comfy_quant"
+ quantized = sorted(k[: -len(scale_suffix)] for k in state_dict if k.endswith(scale_suffix))
+ if not quantized:
+ raise ValueError("no int8 ConvRot projections in this checkpoint")
+ # include_buffers=False: rotary inv_freq buffers are built for real; parameters arrive via assign=True.
+ with init_empty_weights(include_buffers = False):
+ encoder = encoder_cls(config)
+ trim_text_encoder(encoder)
+ linear_cls = _int8_convrot_linear_class()
+ for prefix in quantized:
+ blob = state_dict.pop(prefix + quant_suffix, None)
+ if blob is None:
+ raise ValueError(f"{prefix}: quantized weight with no {quant_suffix} metadata")
+ _validate_comfy_quant(blob, prefix)
+ parent_path, _, leaf = prefix.rpartition(".")
+ parent = encoder.get_submodule(parent_path)
+ existing = getattr(parent, leaf)
+ weight = state_dict[prefix + ".weight"]
+ if not isinstance(existing, torch.nn.Linear) or weight.dtype != torch.int8:
+ raise ValueError(f"{prefix}: not an int8 weight for a Linear")
+ if (existing.out_features, existing.in_features) != tuple(weight.shape):
+ raise ValueError(
+ f"{prefix}: checkpoint shape {tuple(weight.shape)} != model "
+ f"{(existing.out_features, existing.in_features)}"
+ )
+ setattr(
+ parent,
+ leaf,
+ linear_cls(
+ existing.in_features,
+ existing.out_features,
+ bias = (prefix + ".bias") in state_dict,
+ group_size = H3_TE_CONVROT_GROUP,
+ ),
+ )
+ if any(k.endswith(quant_suffix) for k in state_dict):
+ raise ValueError("quant metadata for a projection with no scale")
+ # Storage dtypes stay (int8 payload, float32 scales); every dense float tensor follows the compute dtype.
+ for key, tensor in list(state_dict.items()):
+ if (
+ tensor.is_floating_point()
+ and not key.endswith(scale_suffix)
+ and dtype is not None
+ and tensor.dtype != dtype
+ ):
+ state_dict[key] = tensor.to(dtype)
+ # A dense decoder projection would load as bf16 and outgrow the budget this path is priced at.
+ language_model = getattr(getattr(encoder, "model", None), "language_model", None)
+ layers = getattr(language_model, "layers", None)
+ if layers is None:
+ raise ValueError("encoder has no model.language_model.layers")
+ dense = [n for n, m in layers.named_modules() if isinstance(m, torch.nn.Linear)]
+ if dense:
+ raise ValueError(
+ f"{len(dense)} decoder projection(s) are not quantized in this artifact, e.g. {sorted(dense)[0]}"
+ )
+ encoder.load_state_dict(state_dict, strict = True, assign = True)
+ stranded = _meta_tensor_names(encoder)
+ if stranded:
+ raise ValueError(f"{len(stranded)} tensor(s) still on the meta device, e.g. {stranded[0]}")
+ encoder.requires_grad_(False)
+ encoder.eval()
+ return encoder
+
+
def _safetensors_te_class(path: str) -> Optional[str]:
try:
import json
@@ -636,6 +891,11 @@ def _resolve_checkpoint_path(
else (EntryNotFoundError,)
)
names = [n for n in te_candidate_filenames(source) if te_candidate_is_readable(n)]
+ # Mirror for every name before the Hub for any, so the order is the same with or without one.
+ for name in names:
+ mirrored = te_prequant_mirror_path(source.location, name)
+ if mirrored is not None:
+ return mirrored
last: Optional[Exception] = None
from .diffusion_prequant import _first_mirrored
@@ -675,10 +935,24 @@ def _resolve_checkpoint_path(
)
return path
except LocalEntryNotFoundError:
- # Online this is the Hub being unreachable, not a missing name: re-raise as itself
- # rather than blaming the next candidate for it.
+ # Online, the Hub is unreachable (not a missing name): take a later name already cached (fp8
+ # cached before the int8 file was published), else re-raise rather than blame the next one.
if not local_files_only:
- raise
+ unreachable = sys.exc_info()[1]
+ for cached_name in names[names.index(name) + 1 :]:
+ try:
+ # Both cache roots, as for any other name.
+ return _download_checkpoint_name(
+ source,
+ cached_name,
+ hf_token,
+ cache_dir,
+ propagate_missing = False,
+ local_files_only = True,
+ )
+ except (EntryNotFoundError, LocalEntryNotFoundError):
+ continue
+ raise unreachable
last = sys.exc_info()[1]
continue
except miss as exc:
@@ -698,7 +972,8 @@ def _validate_checkpoint(ckpt: Any, scheme: str, component: str, base: str, logg
``te_class`` presence is checked by the caller (it resolves the class); torch /
transformers versions are recorded by the builder for forensics but not enforced (the
fp8 storage cast is version-stable plain-tensor data)."""
- if not isinstance(ckpt, dict) or ckpt.get("format") != TE_PREQUANT_FORMAT:
+ expected_format = TE_PREQUANT_FORMAT_INT8_CONVROT if scheme == "int8" else TE_PREQUANT_FORMAT
+ if not isinstance(ckpt, dict) or ckpt.get("format") != expected_format:
_warn(logger, scheme, ValueError("unrecognised pre-cast text-encoder checkpoint format"))
return False
if "state_dict" not in ckpt:
@@ -734,6 +1009,13 @@ def _validate_checkpoint(ckpt: Any, scheme: str, component: str, base: str, logg
return True
+def te_prequant_unmirrored(
+ repo_id: Optional[str], files: list[tuple[str, int]]
+) -> list[tuple[str, int]]:
+ """``files`` minus those a configured mirror serves: the loader reads those in place, never from the Hub."""
+ return [(name, size) for name, size in files if te_prequant_mirror_path(repo_id, name) is None]
+
+
def te_prequant_hub_files(
sources: dict[str, "TePrequantSource"],
api: Any,
@@ -750,6 +1032,19 @@ def te_prequant_hub_files(
for component, source in sources.items():
if getattr(source, "kind", None) != "repo" or not getattr(source, "filename", None):
continue
+ mirrored = next(
+ (
+ (name, te_prequant_mirror_path(source.location, name))
+ for name in te_candidate_filenames(source)
+ if te_candidate_is_readable(name)
+ and te_prequant_mirror_path(source.location, name) is not None
+ ),
+ None,
+ )
+ if mirrored is not None:
+ import os
+ found[component] = [(mirrored[0], os.path.getsize(mirrored[1]))]
+ continue
try:
info = api.model_info(source.location, files_metadata = True)
except Exception as exc: # noqa: BLE001 -- unavailable pre-cast means the dense encoder
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 120b379d935..f9b1a9cf4a0 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -4904,8 +4904,13 @@ class VideoBackend:
base = base,
)
te_files = self._te_prequant_hub_files(te_sources, api)
+ from .diffusion_te_prequant import te_prequant_unmirrored
+
for component, files in te_files.items():
- total += add(te_sources[component].location, files)
+ location = te_sources[component].location
+ files = te_prequant_unmirrored(location, files)
+ if files:
+ total += add(location, files)
# The denoiser's replacement artifact, for the same reason: the base entry below drops the dense DiT shards
# only when this resolves, so it is staged in their place.
dq_repo, dq_files = self._denoiser_prequant_hub_files(
@@ -5228,7 +5233,13 @@ class VideoBackend:
readable = te_candidate_is_readable,
)
got = False
+ from .diffusion_te_prequant import te_prequant_mirror_path
+
for name in names:
+ # The loader reads a mirrored file in place (_resolve_checkpoint_path), as the plan assumes.
+ if te_prequant_mirror_path(source.location, name) is not None:
+ got = True
+ break
try:
hf_hub_download_with_xet_fallback(
source.location,
diff --git a/studio/backend/tests/test_diffusion_predownload_guard_platforms.py b/studio/backend/tests/test_diffusion_predownload_guard_platforms.py
index 6ea23523c03..fa687dd53cc 100644
--- a/studio/backend/tests/test_diffusion_predownload_guard_platforms.py
+++ b/studio/backend/tests/test_diffusion_predownload_guard_platforms.py
@@ -444,3 +444,29 @@ def test_the_default_plan_still_refuses_an_oversized_pipeline(monkeypatch):
plan = backend.download_plan("unsloth/FLUX.2-dev", model_kind = "pipeline")
assert plan["incompatible_reason"] is not None
assert "unified memory" in plan["incompatible_reason"]
+
+
[email protected]("mirrored", [False, True])
+def test_a_mirrored_pre_cast_encoder_is_never_staged_from_the_hub(monkeypatch, tmp_path, mirrored):
+ from core.inference import diffusion_te_prequant as te_prequant
+
+ repo, name = "unsloth/FLUX.2-dev-FP8", "te.safetensors"
+ backend = _plan_probe(monkeypatch, [])
+ monkeypatch.setattr(
+ DiffusionBackend,
+ "_te_prequant_plan_files",
+ lambda *_a, **_k: {"text_encoder": (repo, [(name, 7)])},
+ )
+ monkeypatch.setattr(
+ DiffusionBackend, "_hub_file_is_cached", staticmethod(lambda *_a, **_k: False)
+ )
+ monkeypatch.delenv(te_prequant.TE_PREQUANT_MIRROR_ENV, raising = False)
+ if mirrored:
+ (tmp_path / repo).mkdir(parents = True)
+ (tmp_path / repo / name).write_bytes(b"x" * 7)
+ monkeypatch.setenv(te_prequant.TE_PREQUANT_MIRROR_ENV, str(tmp_path))
+ plan = backend.download_plan("unsloth/FLUX.2-dev", model_kind = "pipeline", memory_verdict = False)
+ staged = [e for e in plan["entries"] if e["repo_id"] == repo]
+ assert bool(staged) is not mirrored
+ base = [e for e in plan["entries"] if e["repo_id"] == "unsloth/FLUX.2-dev"]
+ assert all(not f.startswith("text_encoder/") for e in base for f in e["files"])
diff --git a/studio/backend/tests/test_diffusion_te_int8_convrot.py b/studio/backend/tests/test_diffusion_te_int8_convrot.py
new file mode 100644
index 00000000000..b5cb2e94fcc
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_te_int8_convrot.py
@@ -0,0 +1,489 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Qwen-Image-2.1 defaults to the hosted int8 ConvRot weight-only text encoder and falls back to the fp8 one."""
+
+from __future__ import annotations
+
+import json
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("transformers.models.qwen3_vl")
+pytest.importorskip("accelerate")
+
+import core.inference.diffusion_te_prequant as tpq # noqa: E402
+import core.inference.prequant_safetensors as ps # noqa: E402
+from core.inference.diffusion_families import detect_family # noqa: E402
+from core.inference.diffusion_precision import ( # noqa: E402
+ RESOLVED_APPLIED,
+ RESOLVED_FELL_BACK,
+ effective_te_quant,
+ quantize_text_encoders,
+ resolve_te_quant_request,
+ te_quant_needs_resident_weights,
+)
+from core.inference.diffusion_prequant import ALLOW_LOCAL_PREQUANT_PATH_ENV # noqa: E402
+from core.inference.diffusion_text_encoder_trim import is_trimmed # noqa: E402
+
+GROUP = 256
+HIDDEN, VOCAB = 256, 64
+BASE = "Qwen/Qwen-Image-2.1"
+REPO, INT8_NAME = (
+ "unsloth/Qwen-Image-2.1-FP8",
+ "Qwen-Image-2.1-text_encoder-INT8-ConvRot.safetensors",
+)
+FP8_NAME = "Qwen-Image-2.1-text_encoder-FP8.safetensors"
+CUDA_BF16 = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)
+
+
+def get_family(name):
+ return detect_family("", override = name)
+
+
+def _config():
+ from transformers.models.qwen3_vl.configuration_qwen3_vl import (
+ Qwen3VLConfig,
+ Qwen3VLTextConfig,
+ Qwen3VLVisionConfig,
+ )
+
+ config = Qwen3VLConfig(
+ vision_config = Qwen3VLVisionConfig(
+ depth = 1,
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_heads = 4,
+ out_hidden_size = HIDDEN,
+ deepstack_visual_indexes = [],
+ ).to_dict(),
+ text_config = Qwen3VLTextConfig(
+ hidden_size = HIDDEN,
+ intermediate_size = 512,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 64,
+ vocab_size = VOCAB,
+ tie_word_embeddings = False,
+ rope_scaling = {
+ "rope_type": "default",
+ "mrope_section": [16, 8, 8],
+ "mrope_interleaved": True,
+ },
+ ).to_dict(),
+ tie_word_embeddings = False,
+ )
+ config._attn_implementation = "sdpa"
+ return config
+
+
+def _encoder(seed = 0):
+ from transformers import Qwen3VLForConditionalGeneration
+ torch.manual_seed(seed)
+ return Qwen3VLForConditionalGeneration(_config()).eval()
+
+
+def _hidden(encoder, ids):
+ lm = encoder.model.language_model
+ handle = lm.norm.register_forward_hook(lambda m, a, o: a[0])
+ try:
+ with torch.no_grad():
+ out = encoder(
+ input_ids = ids, attention_mask = torch.ones_like(ids), output_hidden_states = True
+ )
+ finally:
+ handle.remove()
+ return out.hidden_states
+
+
+def _container(tmp_path, name, state, fmt, scheme):
+ from safetensors.torch import save_file
+
+ meta = {
+ ps.UNSLOTH_FORMAT_KEY: fmt,
+ ps.UNSLOTH_METADATA_KEY: json.dumps(
+ {
+ "scheme": scheme,
+ "component": "text_encoder",
+ "base_model_id": BASE,
+ "te_class": "Qwen3VLForConditionalGeneration",
+ }
+ ),
+ "tensor_names": json.dumps(sorted(state)),
+ **{n: json.dumps({"_type": "Tensor"}) for n in state},
+ }
+ path = tmp_path / name
+ save_file({k: v.contiguous() for k, v in state.items()}, str(path), metadata = meta)
+ return path
+
+
+def _int8_state(encoder):
+ """The builder's layout: decoder projections int8 + scale + comfy_quant, lm_head dropped, the rest dense."""
+ import re
+
+ lin = re.compile(
+ r"^model\.language_model\.layers\.\d+\.(self_attn\.[qkvo]_proj|mlp\.(gate|up|down)_proj)\.weight$"
+ )
+ blob = torch.tensor(
+ list(
+ json.dumps(
+ {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": GROUP}
+ ).encode()
+ ),
+ dtype = torch.uint8,
+ )
+ state = {}
+ for k, v in encoder.state_dict().items():
+ if k == "lm_head.weight":
+ continue
+ if lin.match(k):
+ q, s = tpq.quantize_int8_convrot_weight(v, GROUP)
+ state[k] = q
+ state[k[:-7] + ".weight_scale"] = s
+ state[k[:-7] + ".comfy_quant"] = blob.clone()
+ else:
+ state[k] = v.detach().clone()
+ return state
+
+
+def _dequantized_reference(encoder, state):
+ """Dense encoder whose projections are the int8 weights mapped back: W = (q * s) @ blockdiag(H)."""
+ from core.inference.diffusion_convrot import build_convrot_hadamard
+
+ h = build_convrot_hadamard(GROUP, dtype = torch.float32)
+ sd = encoder.state_dict()
+ for k in list(sd):
+ if k + "_scale" in state:
+ q, s = state[k].float(), state[k + "_scale"]
+ o, i = q.shape
+ sd[k] = ((q * s).reshape(o, i // GROUP, GROUP) @ h).reshape(o, i)
+ encoder.load_state_dict(sd)
+ return encoder
+
+
+def int8_file(tmp_path, monkeypatch):
+ monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)
+ monkeypatch.setenv(ALLOW_LOCAL_PREQUANT_PATH_ENV, str(tmp_path))
+ import transformers
+
+ monkeypatch.setattr(transformers.AutoConfig, "from_pretrained", lambda *_a, **_k: _config())
+ encoder = _encoder(seed = 5)
+ state = _int8_state(encoder)
+ path = _container(tmp_path, INT8_NAME, state, tpq.TE_PREQUANT_FORMAT_INT8_CONVROT, "int8")
+ return encoder, state, path
+
+
+def _load(
+ path,
+ scheme = "int8",
+ trim = True,
+):
+ return tpq.load_prequant_text_encoder(
+ BASE,
+ "text_encoder",
+ tpq.TePrequantSource(kind = "path", location = str(path)),
+ dtype = torch.float32,
+ scheme = scheme,
+ local_files_only = True,
+ trim_lm_head = trim,
+ )
+
+
+def test_family_default_is_hosted_int8_and_needs_no_torchao():
+ fam = get_family("qwen-image-2.1")
+ mode, auto = resolve_te_quant_request(None, fam.te_quant_auto)
+ assert (mode, auto) == ("int8", True)
+ # No keep-bf16 schedule, so every pre-load gate sees the fp8 storage cast: no torchao, offload allowed.
+ assert effective_te_quant(mode, fam.name) == "fp8"
+ assert not te_quant_needs_resident_weights(effective_te_quant(mode, fam.name))
+
+
+def test_int8_source_prefers_the_convrot_file_then_the_fp8_one():
+ fam = get_family("qwen-image-2.1")
+ sources = tpq.te_prequant_sources_for_base(fam, BASE, te_quant_mode = "int8", target = CUDA_BF16)
+ src = sources["text_encoder"]
+ assert (src.kind, src.location) == ("repo", REPO)
+ assert tpq.te_candidate_filenames(src) == (
+ INT8_NAME,
+ FP8_NAME,
+ "Qwen-Image-2.1-text_encoder-FP8.pt",
+ )
+ # An explicit fp8 request never asks for the int8 file.
+ fp8 = tpq.te_prequant_sources(fam, te_quant_mode = "fp8", target = CUDA_BF16)["text_encoder"]
+ assert INT8_NAME not in tpq.te_candidate_filenames(fp8)
+ # Families without a hosted int8 encoder resolve nothing for int8, as before.
+ assert (
+ tpq.te_prequant_sources(get_family("qwen-image"), te_quant_mode = "int8", target = CUDA_BF16)
+ == {}
+ )
+
+
+def test_keeping_the_lm_head_resolves_the_fp8_file(monkeypatch):
+ from core.inference.diffusion_text_encoder_trim import KEEP_LM_HEAD_ENV
+
+ monkeypatch.setenv(KEEP_LM_HEAD_ENV, "1")
+ src = tpq.resolve_te_prequant_source(get_family("qwen-image-2.1"), "text_encoder", "int8")
+ assert src.filename == FP8_NAME
+
+
+def test_int8_file_loads_as_convrot_linears_and_matches_its_dequantized_weights(int8_file):
+ from core.inference.video_minimax_h3_te import _int8_convrot_linear_class
+
+ reference, state, path = int8_file
+ encoder = _load(path)
+ assert encoder is not None
+ assert getattr(encoder, tpq.TE_PREQUANT_SCHEME_ATTR) == "int8"
+ assert is_trimmed(encoder)
+ cls = _int8_convrot_linear_class()
+ layers = encoder.model.language_model.layers
+ assert sum(isinstance(m, cls) for m in layers.modules()) == 2 * 7
+ assert not any(isinstance(m, torch.nn.Linear) for m in layers.modules())
+ assert all(m.weight.dtype == torch.int8 for m in layers.modules() if isinstance(m, cls))
+ # Vision tower and embedding stay dense.
+ assert any(isinstance(m, torch.nn.Linear) for m in encoder.model.visual.modules())
+ # Plain tensors only, so Module.to() and group offloading move it like any module.
+ assert all(
+ type(t) in (torch.Tensor, torch.nn.Parameter)
+ for t in [*encoder.parameters(), *encoder.buffers()]
+ )
+ ids = torch.randint(0, VOCAB, (1, 9))
+ expected = _hidden(_dequantized_reference(_encoder(seed = 5), state), ids)
+ got = _hidden(encoder, ids)
+ for a, b in zip(expected, got):
+ torch.testing.assert_close(b, a, rtol = 1e-4, atol = 1e-4)
+ # and it is close to the dense bf16 encoder it was quantized from
+ dense = _hidden(reference, ids)[-1]
+ cos = torch.nn.functional.cosine_similarity(dense.flatten(), got[-1].flatten(), dim = 0)
+ assert cos > 0.99
+
+
+def test_int8_file_is_refused_for_an_fp8_request_and_without_the_trim(int8_file):
+ _, _, path = int8_file
+ assert _load(path, scheme = "fp8") is None
+ assert _load(path, trim = False) is None
+
+
+def test_int8_request_falls_back_to_the_fp8_file_when_the_hub_404s(tmp_path, monkeypatch):
+ from huggingface_hub.errors import EntryNotFoundError
+
+ monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)
+ monkeypatch.delenv(tpq.TE_PREQUANT_MIRROR_ENV, raising = False)
+ fp8 = tmp_path / FP8_NAME
+ fp8.write_bytes(b"x")
+ asked: list = []
+
+ def fake_download(repo_id, filename, **_k):
+ asked.append(filename)
+ if filename == INT8_NAME:
+ raise EntryNotFoundError(f"404 {filename}")
+ return str(fp8)
+
+ import huggingface_hub
+
+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_download)
+ src = tpq.resolve_te_prequant_source(get_family("qwen-image-2.1"), "text_encoder", "int8")
+ assert tpq._resolve_checkpoint_path(src, None, cache_dir = str(tmp_path)) == str(fp8)
+ assert asked == [INT8_NAME, FP8_NAME]
+
+
+def test_mirror_serves_the_int8_file_before_the_hub(tmp_path, monkeypatch):
+ mirror = tmp_path / "mirror"
+ target = mirror / REPO / INT8_NAME
+ target.parent.mkdir(parents = True)
+ target.write_bytes(b"x" * 7)
+ monkeypatch.setenv(tpq.TE_PREQUANT_MIRROR_ENV, str(mirror))
+ monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)
+ import huggingface_hub
+
+ monkeypatch.setattr(
+ huggingface_hub, "hf_hub_download", lambda *a, **k: pytest.fail("the Hub was asked")
+ )
+ src = tpq.resolve_te_prequant_source(get_family("qwen-image-2.1"), "text_encoder", "int8")
+ assert tpq._resolve_checkpoint_path(src, None, cache_dir = str(tmp_path)) == str(target.resolve())
+
+ class NoApi:
+ def model_info(self, *a, **k):
+ raise AssertionError("planning asked the Hub")
+
+ assert tpq.te_prequant_hub_files({"text_encoder": src}, NoApi()) == {
+ "text_encoder": [(INT8_NAME, 7)]
+ }
+ # An escaping name is never served.
+ assert tpq.te_prequant_mirror_path(REPO, "../x") is None
+
+
+def test_hub_plan_counts_the_file_the_repo_holds(monkeypatch):
+ monkeypatch.delenv(tpq.TE_PREQUANT_MIRROR_ENV, raising = False)
+ src = tpq.resolve_te_prequant_source(get_family("qwen-image-2.1"), "text_encoder", "int8")
+
+ def api(names):
+ sib = [types.SimpleNamespace(rfilename = n, size = s) for n, s in names]
+ return types.SimpleNamespace(model_info = lambda *a, **k: types.SimpleNamespace(siblings = sib))
+
+ assert tpq.te_prequant_hub_files({"te": src}, api([(FP8_NAME, 9)])) == {"te": [(FP8_NAME, 9)]}
+ assert tpq.te_prequant_hub_files({"te": src}, api([(FP8_NAME, 9), (INT8_NAME, 8)])) == {
+ "te": [(INT8_NAME, 8)]
+ }
+
+
+def _pipe(encoder):
+ return types.SimpleNamespace(text_encoder = encoder)
+
+
+def test_hosted_int8_encoder_is_reported_and_never_recast(int8_file, monkeypatch):
+ import core.inference.diffusion_precision as prec
+
+ _, _, path = int8_file
+ encoder = _load(path)
+ monkeypatch.setattr(
+ prec, "_cast_fp8", lambda *a, **k: pytest.fail("int8 encoder re-cast to fp8")
+ )
+ for offload in (False, True):
+ out = quantize_text_encoders(
+ _pipe(encoder), CUDA_BF16, mode = "int8", family = "qwen-image-2.1", offload_active = offload
+ )
+ assert (out.mode, out.status) == ("int8", RESOLVED_APPLIED)
+
+
+def test_fp8_fallback_encoder_reports_why_int8_did_not_engage(monkeypatch):
+ import core.inference.diffusion_precision as prec
+
+ encoder = torch.nn.Linear(2, 2)
+ setattr(encoder, tpq.TE_PREQUANT_SCHEME_ATTR, "fp8")
+ cast: list = []
+ monkeypatch.setattr(prec, "_cast_fp8", lambda enc, tgt: cast.append(enc))
+ monkeypatch.setattr(prec, "te_quant_supported", lambda *a: True)
+ out = quantize_text_encoders(_pipe(encoder), CUDA_BF16, mode = "int8", family = "qwen-image-2.1")
+ assert (out.mode, out.status) == ("fp8", RESOLVED_FELL_BACK)
+ assert "int8 ConvRot" in out.reason
+ assert cast == [encoder]
+
+
+def test_pipe_kwargs_loads_with_the_requested_scheme(monkeypatch):
+ seen: list = []
+ monkeypatch.setattr(
+ tpq,
+ "te_prequant_sources_for_base",
+ lambda *a, **k: {"text_encoder": tpq.TePrequantSource(kind = "repo", location = REPO)},
+ )
+ monkeypatch.setattr(
+ tpq, "load_prequant_text_encoder", lambda *a, **k: seen.append(k["scheme"]) or object()
+ )
+ for mode in ("int8", "fp8"):
+ tpq.te_prequant_pipe_kwargs(
+ get_family("qwen-image-2.1"), BASE, te_quant_mode = mode, target = CUDA_BF16, dtype = None
+ )
+ assert seen == ["int8", "fp8"]
+
+
+def test_an_unreachable_hub_still_takes_the_cached_fp8_file(tmp_path, monkeypatch):
+ """Online, int8 never cached, Hub down: the cached fp8 file loads instead of a dense pull."""
+ from huggingface_hub.errors import LocalEntryNotFoundError
+
+ monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)
+ monkeypatch.delenv(tpq.TE_PREQUANT_MIRROR_ENV, raising = False)
+ fp8 = tmp_path / FP8_NAME
+ fp8.write_bytes(b"x")
+ asked: list = []
+
+ def fake_download(
+ repo_id,
+ filename,
+ local_files_only = False,
+ **_k,
+ ):
+ asked.append((filename, local_files_only))
+ if filename == FP8_NAME and local_files_only:
+ return str(fp8)
+ raise LocalEntryNotFoundError("connection error")
+
+ import huggingface_hub
+
+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_download)
+ src = tpq.resolve_te_prequant_source(get_family("qwen-image-2.1"), "text_encoder", "int8")
+ assert tpq._resolve_checkpoint_path(src, None, cache_dir = str(tmp_path)) == str(fp8)
+ assert asked == [(INT8_NAME, False), (FP8_NAME, True)]
+ # Nothing cached at all: the outage itself surfaces.
+ monkeypatch.setattr(
+ huggingface_hub,
+ "hf_hub_download",
+ lambda *a, **k: (_ for _ in ()).throw(LocalEntryNotFoundError("connection error")),
+ )
+ with pytest.raises(LocalEntryNotFoundError):
+ tpq._resolve_checkpoint_path(src, None, cache_dir = str(tmp_path))
+
+
+def test_an_unreachable_hub_takes_the_fp8_file_from_the_other_cache_root(tmp_path, monkeypatch):
+ """Outage with the fp8 file cached only under the default root (a moved cache setting)."""
+ from huggingface_hub.errors import LocalEntryNotFoundError
+
+ monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)
+ monkeypatch.delenv(tpq.TE_PREQUANT_MIRROR_ENV, raising = False)
+ fp8 = tmp_path / "other" / FP8_NAME
+ fp8.parent.mkdir()
+ fp8.write_bytes(b"x")
+ import core.inference.diffusion_prequant as dpq
+
+ monkeypatch.setattr(
+ dpq,
+ "_cached_in_root",
+ lambda _src, root, name = None: str(fp8) if root is None and name == FP8_NAME else None,
+ )
+
+ def fake_download(
+ repo_id,
+ filename,
+ cache_dir = None,
+ local_files_only = False,
+ **_k,
+ ):
+ if filename == FP8_NAME and local_files_only and cache_dir is None:
+ return str(fp8)
+ raise LocalEntryNotFoundError("connection error")
+
+ import huggingface_hub
+
+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_download)
+ src = tpq.resolve_te_prequant_source(get_family("qwen-image-2.1"), "text_encoder", "int8")
+ assert tpq._resolve_checkpoint_path(src, None, cache_dir = str(tmp_path / "live")) == str(fp8)
+
+
+def test_an_int8_file_that_will_not_build_falls_back_to_the_fp8_names(monkeypatch):
+ calls: list = []
+ sentinel = object()
+ src = tpq.resolve_te_prequant_source(get_family("qwen-image-2.1"), "text_encoder", "int8")
+ monkeypatch.setattr(tpq, "te_prequant_sources_for_base", lambda *a, **k: {"text_encoder": src})
+
+ def fake_load(base, component, source, **k):
+ calls.append((source.filename, k["scheme"]))
+ return None if source.filename == INT8_NAME else sentinel
+
+ monkeypatch.setattr(tpq, "load_prequant_text_encoder", fake_load)
+ for held, expected in ((True, sentinel), (False, None)):
+ calls.clear()
+ monkeypatch.setattr(tpq, "_held_locally", lambda *a, held = held: held)
+ out = tpq.te_prequant_pipe_kwargs(
+ get_family("qwen-image-2.1"), BASE, te_quant_mode = "int8", target = CUDA_BF16, dtype = None
+ )
+ assert out.get("text_encoder") is expected
+ # Only a present-but-unbuildable int8 file earns a second (fp8) attempt; a 404 already fell through.
+ assert calls == (
+ [(INT8_NAME, "int8"), (FP8_NAME, "fp8")] if held else [(INT8_NAME, "int8")]
+ )
+
+
+def test_keep_lm_head_reason_names_the_switch(monkeypatch):
+ import core.inference.diffusion_precision as prec
+ from core.inference.diffusion_text_encoder_trim import KEEP_LM_HEAD_ENV
+
+ monkeypatch.setenv(KEEP_LM_HEAD_ENV, "1")
+ monkeypatch.setattr(prec, "_cast_fp8", lambda enc, tgt: None)
+ monkeypatch.setattr(prec, "te_quant_supported", lambda *a: True)
+ out = quantize_text_encoders(
+ _pipe(torch.nn.Linear(2, 2)), CUDA_BF16, mode = "int8", family = "qwen-image-2.1"
+ )
+ assert out.mode == "fp8" and KEEP_LM_HEAD_ENV in out.reason
diff --git a/studio/backend/tests/test_diffusion_te_prequant.py b/studio/backend/tests/test_diffusion_te_prequant.py
index aabe4e9cd9d..f01a221e413 100644
--- a/studio/backend/tests/test_diffusion_te_prequant.py
+++ b/studio/backend/tests/test_diffusion_te_prequant.py
@@ -985,13 +985,17 @@ def test_an_unreachable_hub_is_not_a_missing_filename(monkeypatch):
def unreachable(**kw):
asked.append(kw["filename"])
+ online.append(not kw.get("local_files_only"))
raise LocalEntryNotFoundError("Hub unreachable")
+ online: list = []
monkeypatch.setattr(huggingface_hub, "hf_hub_download", unreachable)
- # ONLINE: surfaces as itself, and the second candidate is never attempted.
- with pytest.raises(LocalEntryNotFoundError):
+ # ONLINE: surfaces as itself, and the second candidate is never fetched: only looked up in the cache, which
+ # costs no network attempt (a cached fallback is still usable during an outage).
+ with pytest.raises(LocalEntryNotFoundError, match = "Hub unreachable"):
tpq._resolve_checkpoint_path(src, None, cache_dir = "/tmp/x", local_files_only = False)
- assert asked == ["hosted-text_encoder-FP8.safetensors"], asked
+ assert asked == ["hosted-text_encoder-FP8.safetensors", "hosted-text_encoder-FP8.pt"], asked
+ assert online == [True, False], online
# OFFLINE: a cache miss is the only verdict there is, so the chain is walked.
asked.clear()
@@ -1048,15 +1052,17 @@ def test_every_family_default_scheme_is_one_we_actually_host_for_that_family():
default load, which is the opposite of why the field exists. Catches a family opting in
before its artifact is published, and a scheme/component pair that does not line up."""
from core.inference.diffusion_families import _FAMILIES
- from core.inference.diffusion_te_prequant import family_te_prequant_repo
+ from core.inference.diffusion_te_prequant import resolve_te_prequant_source
offenders = []
for fam in _FAMILIES:
scheme = getattr(fam, "te_quant_auto", None)
if scheme is None:
continue
+ # The resolver, not the fp8 table alone: a hosted int8 encoder (TE_INT8_CONVROT_FILES) lists the family's fp8
+ # file behind it, so the default never costs a dense download.
if not any(
- family_te_prequant_repo(fam, scheme, component)
+ resolve_te_prequant_source(fam, component, scheme)
for component in tpq.TE_PREQUANT_COMPONENTS
):
offenders.append(
@@ -1067,16 +1073,22 @@ def test_every_family_default_scheme_is_one_we_actually_host_for_that_family():
)
-def test_qwen_image_2_1_defaults_to_the_hosted_fp8_encoder():
+def test_qwen_image_2_1_defaults_to_a_hosted_encoder():
"""The family this was built for. Its encoder (Qwen3-VL-8B, 16.33 GiB dense) is bigger than
its INT8 denoiser (6.76 GiB), so leaving it dense is what made a quantised pick still cost
~26 GB. Named rather than covered only by the sweep above, because the whole change is
- pointless if this one row regresses."""
+ pointless if this one row regresses. The default is the int8 ConvRot encoder, with the fp8
+ one behind it in the same repo."""
from core.inference.diffusion_families import detect_family
from core.inference.diffusion_te_prequant import resolve_te_prequant_source
fam = detect_family("Qwen/Qwen-Image-2.1")
- assert fam.te_quant_auto == "fp8"
+ assert fam.te_quant_auto == "int8"
+ source = resolve_te_prequant_source(fam, "text_encoder", "int8")
+ assert source is not None and source.kind == "repo"
+ assert source.location == "unsloth/Qwen-Image-2.1-FP8"
+ assert source.filename == "Qwen-Image-2.1-text_encoder-INT8-ConvRot.safetensors"
+ assert source.fallback_filenames[0] == "Qwen-Image-2.1-text_encoder-FP8.safetensors"
source = resolve_te_prequant_source(fam, "text_encoder", "fp8")
assert source is not None and source.kind == "repo"
assert source.location == "unsloth/Qwen-Image-2.1-FP8"
diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py
index 52510253521..61a0d3ba59b 100644
--- a/studio/backend/tests/test_video_backend.py
+++ b/studio/backend/tests/test_video_backend.py
@@ -5228,6 +5228,27 @@ def test_fetch_te_prequant_only_reports_what_it_downloaded(monkeypatch):
assert backend._fetch_te_prequant({"text_encoder": local}, None) == ()
+def test_fetch_te_prequant_takes_a_mirrored_encoder_without_the_hub(monkeypatch, tmp_path):
+ from core.inference import diffusion_te_prequant as te_prequant
+
+ backend = VideoBackend()
+ source = types.SimpleNamespace(
+ kind = "repo", location = "unsloth/LTX-2-FP8", filename = "LTX-2-text_encoder-FP8.safetensors"
+ )
+ (tmp_path / "unsloth" / "LTX-2-FP8").mkdir(parents = True)
+ (tmp_path / "unsloth" / "LTX-2-FP8" / "LTX-2-text_encoder-FP8.safetensors").write_bytes(b"x")
+ monkeypatch.setenv(te_prequant.TE_PREQUANT_MIRROR_ENV, str(tmp_path))
+
+ def _no_hub(*_a, **_k):
+ raise AssertionError("the Hub was asked")
+
+ monkeypatch.setattr("utils.hf_xet_fallback.hf_hub_download_with_xet_fallback", _no_hub)
+ for offline in (False, True):
+ assert backend._fetch_te_prequant(
+ {"text_encoder": source}, None, local_files_only = offline
+ ) == ("text_encoder",)
+
+
def test_load_pipeline_tops_up_the_dense_encoder_when_injection_fails(fake_runtime, tmp_path):
# Injection is best-effort, but the pre-download already dropped the dense shards, so a failed injection must restore them rather than crash the load.
backend = VideoBackend()