Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: image families default to a hosted quantized text encoder (#14009)

Daniel Han8ef162f7057a
diff --git a/scripts/build_te_int8_convrot_checkpoint.py b/scripts/build_te_int8_convrot_checkpoint.py
index 4228fb3f253..70e5274b62f 100644
--- a/scripts/build_te_int8_convrot_checkpoint.py
+++ b/scripts/build_te_int8_convrot_checkpoint.py
@@ -4,8 +4,8 @@
"""Build an int8 ConvRot weight-only text-encoder checkpoint for the Unsloth TE prequant path.
Every decoder projection of the dense encoder (q/k/v/o and gate/up/down in each
-``model.language_model.layers.N``) is rotated by the group-256 Hadamard and stored int8 with a
-float32 per-output-channel scale (``diffusion_te_prequant.quantize_int8_convrot_weight``); the
+``model.language_model.layers.N``, or ``model.layers.N`` for a text-only Qwen3) is rotated by
+the group-256 Hadamard and stored int8 with a float32 per-output-channel scale (``diffusion_te_prequant.quantize_int8_convrot_weight``); the
vision tower, embedding table and norms are copied bf16, bit for bit; ``lm_head`` is left out
(the pipelines that load this file read hidden states only). Each quantized Linear carries a
``<name>.comfy_quant`` JSON blob naming the format, and the header is the plain-tensor container
@@ -17,6 +17,13 @@ same source and group give a byte-identical file.
python scripts/build_te_int8_convrot_checkpoint.py \\
--base Qwen/Qwen-Image-2.1 --family qwen-image-2.1 \\
--out Qwen-Image-2.1-text_encoder-INT8-ConvRot.safetensors
+
+``--load transformers`` builds any other encoder from the module the pipeline instantiates (``--te-class``, default
+the config's architecture): the dense encoder is loaded with ``from_pretrained`` and its runtime state dict is
+written, so the keys are the ones the loader's strict ``load_state_dict`` expects even where a checkpoint stores an
+older layout (Qwen2.5-VL's ``model.layers`` / ``visual``). Every Linear whose module name matches
+``diffusion_te_prequant.TE_INT8_CONVROT_LINEAR`` is quantized: decoder q/k/v/o + gate/up/down (Qwen, Llama, Mistral,
+Gemma; never a vision tower, which names its blocks differently) and T5's SelfAttention q/k/v/o + DenseReluDense.
"""
from __future__ import annotations
@@ -31,7 +38,7 @@ from pathlib import Path
BACKEND = Path(__file__).resolve().parent.parent / "studio" / "backend"
DECODER_LINEAR = re.compile(
- r"^model\.language_model\.layers\.\d+\.(self_attn\.[qkvo]_proj|mlp\.(gate|up|down)_proj)\.weight$"
+ r"^model\.(language_model\.)?layers\.\d+\.(self_attn\.[qkvo]_proj|mlp\.(gate|up|down)_proj)\.weight$"
)
# UMT5 / T5 encoder blocks (Wan's UMT5-XXL): self-attention q/k/v/o and the MLP, gated (wi_0 / wi_1) or not (wi), + wo.
T5_ENCODER_LINEAR = re.compile(
@@ -100,6 +107,19 @@ def main(argv = None) -> int:
"--device", default = "cuda" if os.environ.get("CUDA_VISIBLE_DEVICES") != "" else "cpu"
)
p.add_argument("--hf-token", default = None)
+ p.add_argument(
+ "--load",
+ choices = ("shards", "transformers", "ideogram4"),
+ default = "shards",
+ help = "ideogram4: build the encoder with Studio's Ideogram 4 loader (vendor fp8 + weight_scale "
+ "dequantized to a bf16 Qwen3VLModel), then quantize it like --load transformers",
+ )
+ p.add_argument(
+ "--te-class", default = None, help = "transformers class to load (--load transformers)"
+ )
+ p.add_argument(
+ "--base-model-id", default = None, help = "base_model_id to record (default: --base)"
+ )
p.add_argument(
"--keep-dense",
default = "",
@@ -107,6 +127,8 @@ def main(argv = None) -> int:
)
args = p.parse_args(argv)
keep_dense = tuple(t for t in (x.strip() for x in args.keep_dense.split(",")) if t)
+ if args.load in ("transformers", "ideogram4"):
+ return _main_transformers(args)
sys.path.insert(0, str(BACKEND))
import safetensors
@@ -161,9 +183,9 @@ def main(argv = None) -> int:
else:
tensors[name] = tensor.to(torch.bfloat16).contiguous()
if not quantized:
- raise SystemExit("no projections matched; is this a Qwen3-VL or UMT5 text encoder?")
+ raise SystemExit("no projections matched; is this a Qwen3 / Qwen3-VL or UMT5 text encoder?")
metadata = {
- "base_model_id": args.base,
+ "base_model_id": args.base_model_id or args.base,
"family": args.family,
"scheme": "int8",
"component": args.component,
@@ -212,5 +234,120 @@ def main(argv = None) -> int:
return 0
+def _main_transformers(args) -> int:
+ """``--load transformers``: quantize the runtime state dict of the encoder class the pipeline builds."""
+ sys.path.insert(0, str(BACKEND))
+ import safetensors
+ import torch
+ import transformers
+
+ from core.inference.diffusion_te_prequant import (
+ TE_INT8_CONVROT_LINEAR,
+ TE_PREQUANT_FORMAT_INT8_CONVROT,
+ quantize_int8_convrot_weight,
+ )
+ from core.inference.prequant_safetensors import UNSLOTH_FORMAT_KEY, UNSLOTH_METADATA_KEY
+
+ src = args.src or args.base
+ if args.load == "ideogram4":
+ if args.component != "text_encoder":
+ raise SystemExit("--load ideogram4 reads <base>/text_encoder only")
+ from core.inference.diffusion_ideogram4 import load_ideogram4_text_encoder
+
+ # The exact module the runtime builds (from_pretrained cannot read the vendor fp8 layout).
+ encoder = load_ideogram4_text_encoder(src, torch.bfloat16, hf_token = args.hf_token).eval()
+ te_class = type(encoder).__name__
+ else:
+ kwargs = {"token": args.hf_token, "torch_dtype": torch.bfloat16}
+ if not args.src:
+ kwargs["subfolder"] = args.component
+ config = transformers.AutoConfig.from_pretrained(
+ src, **{k: v for k, v in kwargs.items() if k != "torch_dtype"}
+ )
+ te_class = args.te_class or (getattr(config, "architectures", None) or [None])[0]
+ encoder = getattr(transformers, te_class).from_pretrained(src, **kwargs).eval()
+ device = torch.device(args.device if torch.cuda.is_available() else "cpu")
+ quant = {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": args.group}
+ blob = torch.tensor(list(json.dumps(quant).encode("utf-8")), dtype = torch.uint8)
+ keep_dense = tuple(t for t in (x.strip() for x in args.keep_dense.split(",")) if t)
+ linears = {
+ name
+ for name, module in encoder.named_modules()
+ if isinstance(module, torch.nn.Linear)
+ and TE_INT8_CONVROT_LINEAR.search(name)
+ and name.rsplit(".", 1)[-1] not in keep_dense
+ }
+ # The encoder's own fp32-kept modules (T5's ``wo``) are stored float32 when left dense, as the shard path does.
+ keep_fp32 = tuple(str(m) for m in (getattr(encoder, "_keep_in_fp32_modules", None) or ()))
+ tensors: dict = {}
+ for name, tensor in encoder.state_dict().items():
+ if name == "lm_head.weight":
+ continue
+ prefix = name[: -len(".weight")] if name.endswith(".weight") else None
+ if prefix in linears:
+ q, scale = quantize_int8_convrot_weight(tensor.to(device), args.group)
+ tensors[name] = q.cpu().contiguous()
+ tensors[prefix + ".weight_scale"] = scale.cpu().contiguous()
+ tensors[prefix + ".comfy_quant"] = blob.clone()
+ else:
+ if not tensor.is_floating_point():
+ dtype = tensor.dtype
+ elif any(t in name.split(".") for t in keep_fp32):
+ dtype = torch.float32
+ else:
+ dtype = torch.bfloat16
+ tensors[name] = tensor.detach().to("cpu", dtype)
+ tensors[name] = tensors[name].contiguous().clone()
+ if not linears:
+ raise SystemExit(f"no Linear in {te_class} matched TE_INT8_CONVROT_LINEAR")
+ metadata = {
+ "base_model_id": args.base_model_id or args.base,
+ "family": args.family,
+ "scheme": "int8",
+ "component": args.component,
+ "te_class": te_class,
+ "torch_dtype": "bfloat16",
+ "quant": dict(
+ quant,
+ weight_only = True,
+ scale = "per_output_channel_absmax_over_127",
+ linears = len(linears),
+ ),
+ "lm_head": "dropped",
+ "torch_version": torch.__version__,
+ "transformers_version": transformers.__version__,
+ "safetensors_version": safetensors.__version__,
+ }
+ names = sorted(tensors)
+ header = {
+ UNSLOTH_FORMAT_KEY: TE_PREQUANT_FORMAT_INT8_CONVROT,
+ UNSLOTH_METADATA_KEY: json.dumps(metadata, sort_keys = True),
+ "tensor_names": json.dumps(names),
+ }
+ header.update({name: json.dumps({"_type": "Tensor"}) for name in names})
+ out = Path(args.out)
+ out.parent.mkdir(parents = True, exist_ok = True)
+ tmp = out.with_name(out.name + ".tmp")
+ _write_safetensors(tmp, tensors, header)
+ os.replace(tmp, out)
+ digest = hashlib.sha256()
+ with open(out, "rb") as fh:
+ for chunk in iter(lambda: fh.read(1 << 24), b""):
+ digest.update(chunk)
+ print(
+ json.dumps(
+ {
+ "out": str(out),
+ "bytes": out.stat().st_size,
+ "sha256": digest.hexdigest(),
+ "te_class": te_class,
+ "quantized_linears": len(linears),
+ "tensors": len(names),
+ }
+ )
+ )
+ return 0
+
+
if __name__ == "__main__":
raise SystemExit(main())
diff --git a/scripts/build_te_prequant_checkpoint.py b/scripts/build_te_prequant_checkpoint.py
index 088bb31e86b..8cc014e01b3 100644
--- a/scripts/build_te_prequant_checkpoint.py
+++ b/scripts/build_te_prequant_checkpoint.py
@@ -14,6 +14,13 @@ touches storage dtypes only, no kernels.
python scripts/build_te_prequant_checkpoint.py \
--base Lightricks/LTX-2 --family ltx-2 --component text_encoder \
--out outputs/te_prequant/ltx2/text_encoder_fp8.pt
+
+Ideogram 4 ships its Qwen3-VL encoder as vendor fp8 + weight_scale, which ``from_pretrained`` cannot read;
+``--loader ideogram4`` builds it through Studio's own loader first:
+
+ python scripts/build_te_prequant_checkpoint.py \
+ --base unsloth/ideogram-4-fp8 --base-model-id ideogram-ai/ideogram-4-fp8 --family ideogram-4 \
+ --loader ideogram4 --out temp/te_default_build/ideogram-4-text_encoder-FP8.pt
"""
from __future__ import annotations
@@ -48,6 +55,19 @@ def main(argv = None) -> int:
p.add_argument("--out", required = True, help = "output .pt path for the checkpoint")
p.add_argument("--dtype", default = "bfloat16", choices = ["bfloat16"])
p.add_argument("--hf-token", default = None)
+ p.add_argument(
+ "--base-model-id",
+ default = None,
+ help = "base_model_id to record (default: --base); the upstream id when --base is an ungated mirror",
+ )
+ p.add_argument(
+ "--loader",
+ default = "auto",
+ choices = ["auto", "ideogram4"],
+ help = "how to load the dense encoder: 'auto' = transformers from_pretrained on the config's "
+ "architecture; 'ideogram4' = Studio's Ideogram 4 loader, which dequantizes the vendor "
+ "fp8 + per-channel weight_scale Qwen3-VL encoder to bf16 (from_pretrained cannot read it)",
+ )
args = p.parse_args(argv)
sys.path.insert(0, str(BACKEND))
@@ -68,20 +88,30 @@ def main(argv = None) -> int:
print(f"== build TE prequant ({family}/{args.component}/{args.scheme}) ==", flush = True)
print(f" loading dense encoder from {args.base} (subfolder={subfolder!r}) ...", flush = True)
t0 = time.time()
- config = transformers.AutoConfig.from_pretrained(args.base, **from_pretrained_kwargs)
- # Prefer the checkpoint's own architecture; AutoModel.from_config gives an unusable bare base class.
- arch = (getattr(config, "architectures", None) or [None])[0]
- if arch and hasattr(transformers, arch):
- encoder_cls_name = arch
- else:
- encoder = transformers.AutoModel.from_config(config)
+ if args.loader == "ideogram4":
+ if args.component != "text_encoder" or subfolder != "text_encoder":
+ p.error("--loader ideogram4 reads <base>/text_encoder only")
+ from core.inference.diffusion_ideogram4 import load_ideogram4_text_encoder
+
+ # The exact module the runtime builds (vendor fp8 dequantized to bf16, rope remapped), so the cast below
+ # matches dense-load-then-cast in Studio.
+ encoder = load_ideogram4_text_encoder(args.base, torch.bfloat16, hf_token = args.hf_token)
encoder_cls_name = type(encoder).__name__
- del encoder
- encoder = getattr(transformers, encoder_cls_name).from_pretrained(
- args.base,
- torch_dtype = torch.bfloat16,
- **from_pretrained_kwargs,
- )
+ else:
+ config = transformers.AutoConfig.from_pretrained(args.base, **from_pretrained_kwargs)
+ # Prefer the checkpoint's own architecture; AutoModel.from_config gives an unusable bare base class.
+ arch = (getattr(config, "architectures", None) or [None])[0]
+ if arch and hasattr(transformers, arch):
+ encoder_cls_name = arch
+ else:
+ encoder = transformers.AutoModel.from_config(config)
+ encoder_cls_name = type(encoder).__name__
+ del encoder
+ encoder = getattr(transformers, encoder_cls_name).from_pretrained(
+ args.base,
+ torch_dtype = torch.bfloat16,
+ **from_pretrained_kwargs,
+ )
print(f" casting in place (layerwise {args.scheme}) ...", flush = True)
class _Target:
@@ -94,7 +124,7 @@ def main(argv = None) -> int:
for k, v in encoder.state_dict().items()
}
metadata = {
- "base_model_id": args.base,
+ "base_model_id": args.base_model_id or args.base,
"family": family,
"scheme": args.scheme,
"component": args.component,
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 6ffef0a0d43..1bab04817ae 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -282,6 +282,7 @@ from .diffusion_step_skip import (
from .diffusion_nvfp4_protect import protect_generation
from .diffusion_precision import (
effective_te_quant,
+ preflight_te_quant,
normalize_te_quant,
quantize_text_encoders,
resolve_te_quant_request,
@@ -1218,9 +1219,14 @@ def _with_supplied_components(load: Callable[..., dict]) -> Callable[..., dict]:
kwargs["text_encoder_quant"] = _te_quant_for_supplied_encoders(
kwargs.get("text_encoder_quant")
)
+ from .diffusion_precision import te_quant_is_auto
+ from .diffusion_te_prequant import auto_te_request
+
token = set_active_component_overrides(overrides)
try:
- return load(self, repo_id, **kwargs)
+ # An unset / "auto" encoder scheme never fails the load: a hosted file that cannot load falls back to dense.
+ with auto_te_request(te_quant_is_auto(kwargs.get("text_encoder_quant"))):
+ return load(self, repo_id, **kwargs)
finally:
reset_active_component_overrides(token)
@@ -2904,6 +2910,9 @@ class DiffusionBackend:
gpu_ordinal: Optional[int] = None,
repo_id: Optional[str] = None,
base_repo: Optional[str] = None,
+ local_files_only: bool = False,
+ text_encoder_files: Optional[Sequence[str]] = None,
+ gguf_filename: Optional[str] = None,
) -> None:
"""Raise ``RuntimeError`` (the route's 409) when an EXPLICIT precision cannot run here.
@@ -2934,6 +2943,10 @@ class DiffusionBackend:
# different generation, which would refuse a scheme the selected card supports, or pass one it does not and
# fail only after eviction. Scoped, because the route calls this on a pooled thread and the probes below use
# argument-less CUDA calls.
+ if model_kind != "pipeline" and not (base_repo or "").strip() and fam is not None:
+ # The variant base the loader derives from the pick's name (a FLUX.2 Klein 9B checkpoint), network-free:
+ # judged on the family default, the int8 gate would check another variant's encoder file.
+ base_repo = named_variant_base(fam, repo_id, gguf_filename) or base_repo
with diffusion_device_scope(gpu_ordinal):
target = self._target_for_ordinal(fam, gpu_ordinal)
return self._assert_precision_for_target(
@@ -2946,8 +2959,54 @@ class DiffusionBackend:
cpu_offload = cpu_offload,
speed_mode = speed_mode,
base_repo = repo_id if model_kind == "pipeline" else base_repo,
+ local_files_only = local_files_only,
+ supplied_text_encoders = self._planned_supplied_text_encoders(
+ fam,
+ repo_id,
+ repo_id if model_kind == "pipeline" else base_repo,
+ text_encoder_files,
+ te_mode,
+ ),
)
+ def _planned_supplied_text_encoders(
+ self,
+ fam: Optional[DiffusionFamily],
+ repo_id: Optional[str],
+ base_repo: Optional[str],
+ text_encoder_files: Optional[Sequence[str]],
+ te_mode: Optional[str],
+ ) -> tuple[str, ...]:
+ """The encoder slots the supplied files fill, for the preflight: it runs before the load installs the
+ component overrides. Network-free (local files, a cached model index); () when that cannot tell, and the
+ load's own check then refuses a mixed build."""
+ if not text_encoder_files or fam is None or te_mode != "int8":
+ return ()
+ try:
+ from .diffusion_comfy_components import parse_component_file
+
+ refs = [
+ parse_component_file(
+ spec, model_path = repo_id, trusted_repo = _is_trusted_diffusion_repo
+ )
+ for spec in text_encoder_files
+ ]
+ if any(not ref.local_path for ref in refs):
+ return ()
+ overrides = self._plan_component_overrides(
+ fam,
+ repo_id or "",
+ base_repo or getattr(fam, "base_repo", None) or repo_id or "",
+ list(text_encoder_files),
+ None,
+ None,
+ local_files_only = True,
+ download = False,
+ )
+ return tuple(overrides.text_encoder_components) if overrides is not None else ()
+ except Exception: # noqa: BLE001 -- unplannable here: the load refuses it
+ return ()
+
def _assert_precision_for_target(
self,
fam: Optional[DiffusionFamily],
@@ -2960,6 +3019,8 @@ class DiffusionBackend:
cpu_offload: bool,
speed_mode: Optional[str] = None,
base_repo: Optional[str] = None,
+ local_files_only: bool = False,
+ supplied_text_encoders: tuple[str, ...] = (),
) -> None:
"""The body of ``assert_precision_available``, run with the selected card current."""
if pinned is not None and pinned != TQ_AUTO:
@@ -3056,13 +3117,19 @@ class DiffusionBackend:
# keep-bf16 schedule is rewritten to layerwise fp8 before support is consulted, and that path needs no
# torchao. Refusing on the raw int8 rejected loads the runtime would run and report as fell_back.
te_effective = effective_te_quant(te_mode, getattr(fam, "name", None))
+ if te_effective is not None and normalize_te_quant(te_mode) == "int8":
+ # int8 depends on whether the hosted int8 ConvRot file will really load for this base and device.
+ te_effective = preflight_te_quant(
+ te_mode,
+ fam,
+ base_repo,
+ target,
+ supplied = supplied_text_encoders or _active_supplied_text_encoders(),
+ local_files_only = local_files_only,
+ )
+ # preflight_te_quant judges the hosted int8 file itself (None when it will load, on Apple Silicon too), so a
+ # file that is not held under a cache-only load stays refused.
te_reason = None
- if te_mode == "int8" and te_effective != te_mode:
- from .diffusion_te_prequant import hosted_int8_convrot_te_runs_on
- if hosted_int8_convrot_te_runs_on(fam, target):
- # The family's hosted int8 ConvRot encoder (FLUX.3, Qwen-Image-2.1) is plain tensors and runs where the
- # runtime-cast rewrite to fp8 does not (Apple Silicon): judge the file, not the downgrade.
- te_effective = None
if te_quant_needs_resident_weights(te_effective) and not torchao_quantize_importable():
# The casters import torchao only after the pipeline is downloaded and built, so a broken or absent
# install failed through load-progress instead of this 409.
@@ -3734,6 +3801,23 @@ class DiffusionBackend:
planned_quant, auto = resolve_te_quant_request(
text_encoder_quant, getattr(fam, "te_quant_auto", None)
)
+ auto_scheme = planned_quant if auto else None
+ if auto_scheme is not None:
+ from .diffusion_te_prequant import (
+ te_prequant_sources_confirmed,
+ te_prequant_sources_for_base,
+ )
+ hosted = te_prequant_sources_for_base(
+ fam,
+ base,
+ te_quant_mode = planned_quant,
+ target = self._target_for_ordinal(fam, None),
+ )
+ if not hosted or not te_prequant_sources_confirmed(
+ hosted, local_files_only = local_files_only
+ ):
+ # Without its hosted file the default loads dense, and the ComfyUI copy is that encoder.
+ auto_scheme = None
def _fits(overrides: Any) -> None:
verify_override_fit(
@@ -3751,7 +3835,7 @@ class DiffusionBackend:
model_index = index,
family = fam.name,
fits = _fits,
- auto_scheme = planned_quant if auto else None,
+ auto_scheme = auto_scheme,
)
except Exception as exc: # noqa: BLE001 - the base repo's encoder is always the fallback
logger.info(
@@ -4051,6 +4135,9 @@ class DiffusionBackend:
gpu_ordinal = gpu_ordinal,
repo_id = repo_id,
base_repo = base_repo,
+ local_files_only = local_files_only,
+ text_encoder_files = text_encoder_files,
+ gguf_filename = gguf_filename,
)
with self._lock, self._load_cancel_lock:
@@ -6551,6 +6638,14 @@ class DiffusionBackend:
sources = {c: v for c, v in (sources or {}).items() if c not in skip_components}
if not sources:
return None
+ from .diffusion_precision import te_quant_picked_automatically
+ from .diffusion_te_prequant import te_prequant_sources_confirmed
+
+ if te_quant_picked_automatically(
+ text_encoder_quant
+ ) and not te_prequant_sources_confirmed(sources):
+ # An automatic pick without a confirmed file loads the dense encoder: no pre-cast saving to price.
+ return None
total = 0
uncached = 0
table_priced = 0
@@ -6615,14 +6710,27 @@ class DiffusionBackend:
table = family_bf16_components_gb(fam, base)
if table is None:
return None
- share = table_priced / max(1, len(sources))
+ # The table's encoder term holds HiDream's TE4 too, which is priced on its own above (cached or from
+ # its dense size): out of both the share and the hosted bytes.
+ te4_in_table = "text_encoder_4" in sources
+ share = table_priced / max(1, len(sources) - (1 if te4_in_table else 0))
# MPS only takes the int8 ConvRot file, smaller than the fp8 one.
scale = (
te_int8_convrot_budget_scale(fam)
if int8_convrot_te_runs_on(target)
else TE_PREQUANT_BUDGET_SCALE
)
- total += int(table[1] * (1000.0**3) * scale * share)
+ from .diffusion_te_prequant import _TE_DENSE_SIBLING_GB
+
+ # Only the hosted encoders: a sibling no file replaces (CLIP-L, ByT5) is priced by its own dense
+ # shards, not as part of the pre-cast file.
+ hosted_gb = max(
+ 0.0,
+ float(table[1])
+ - _TE_DENSE_SIBLING_GB.get(str(getattr(fam, "name", "") or ""), 0.0)
+ - (HIDREAM_LLAMA_BF16_BYTES / 1000.0**3 if te4_in_table else 0.0),
+ )
+ total += int(hosted_gb * (1000.0**3) * scale * share)
if total <= 0:
return None
return max(1, total // (1024 * 1024)), tuple(sources), uncached == 0
@@ -7860,7 +7968,21 @@ class DiffusionBackend:
if kind == "pipeline":
if fam.name == IDEOGRAM4_FAMILY_NAME:
# ideogram ships the same transformers-5.x Qwen stack as krea; assemble per-component too,
- # from the mirror for the same reason.
+ # from the mirror for the same reason. A hosted pre-cast fp8 encoder replaces the dense
+ # vendor-fp8 one (whose shards the download plan then leaves out), as on krea.
+ ideogram_te = te_prequant_pipe_kwargs(
+ fam,
+ fetch_base,
+ te_quant_mode = text_encoder_quant,
+ target = target,
+ dtype = dtype,
+ hf_token = hf_token,
+ logger = logger,
+ local_files_only = local_files_only,
+ # assembly reads fetch_base (never the staged dir), like krea
+ dense_source = fetch_base if local_files_only else None,
+ ).get("text_encoder")
+ self._raise_if_load_cancelled(_load_token)
pipe = load_ideogram4_pipeline(
fetch_base,
dtype,
@@ -7868,6 +7990,7 @@ class DiffusionBackend:
check_cancelled = lambda: self._raise_if_load_cancelled(
_load_token
),
+ text_encoder = ideogram_te,
)
else:
pipe_kwargs = {
@@ -10709,7 +10832,8 @@ class DiffusionBackend:
from .diffusion_te_prequant import int8_convrot_te_runs_on
# Never below scanned dense shards: a bad checkpoint falls back to opening them. Not on MPS, where
- # that failure raises instead (_te_prequant_pipe_kwargs, #11792).
+ # that failure raises instead (_te_prequant_pipe_kwargs, #11792). An automatic pick whose file is not
+ # confirmed never gets here: _precast_text_encoder_mib prices nothing for it, so the dense terms stand.
if not int8_convrot_te_runs_on(target):
precast_mib = max(int(precast_mib), scanned_te)
text_encoder_mib = max(0, int(text_encoder_mib or 0) - scanned_te) + int(
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.py
index acbdad30e20..4308b5f4adf 100644
--- a/studio/backend/core/inference/diffusion_families.py
+++ b/studio/backend/core/inference/diffusion_families.py
@@ -176,8 +176,14 @@ class DiffusionFamily:
# ``family_prequant_filename``.
prequant_filenames: tuple[tuple[str, ...], ...] = field(default_factory = tuple)
# Hosted PRE-CAST text-encoder checkpoints as (scheme, component, repo_id). Layerwise-fp8 only: the cast is
- # deterministic, so the artifact is bit-identical while skipping the dense TE download.
- te_prequant_repos: tuple[tuple[str, str, str], ...] = field(default_factory = tuple)
+ # deterministic, so the artifact is bit-identical while skipping the dense TE download. An optional fourth element
+ # names the component the hosted file was built as, when this family loads another family's bytes into a
+ # different slot (HiDream's text_encoder_3 is FLUX.1's text_encoder_2).
+ te_prequant_repos: tuple[tuple[str, ...], ...] = field(default_factory = tuple)
+ # The same rows for a VARIANT base whose encoder is a different model, as (base_repo, scheme, component, repo_id):
+ # FLUX.2 klein 9B carries Qwen3-8B where the family base carries Qwen3-4B. A load of that base (or one in its
+ # equivalence group) resolves these rows only; ``te_prequant_repos`` keeps serving the family base.
+ te_prequant_variant_repos: tuple[tuple[str, str, str, str], ...] = field(default_factory = tuple)
# The text-encoder scheme an UNSET request resolves to on this family, or None to keep the released bf16 encoder.
# Opt-in per family rather than implied by ``te_prequant_repos``, because hosting an artifact says the cast is
# reproducible, not that its conditioning is good enough to hand someone who asked for nothing. These encoders are
@@ -387,6 +393,10 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
# Pre-cast T5-XXL (9.52 -> 5.90 GB; CLIP-L stays dense). One artifact serves schnell/dev/Krea-dev (T5 shards
# are byte-identical).
te_prequant_repos = (("fp8", "text_encoder_2", "unsloth/FLUX.1-schnell-FP8"),),
+ # No te_quant_auto: a small encoder (T5-XXL, 4.8B) stays bf16 unless ComfyUI's template quantizes it, and the
+ # FLUX.1 templates load t5xxl_fp16. Measured against the bf16 encoder (bf16 DiT, schnell 4 steps, 24 prompts,
+ # bf16 A/A repeat bit-identical): the hosted int8 ConvRot T5 scores LPIPS-alex mean 0.055, SSIM 0.927, layerwise
+ # fp8 0.096 on 12; both stay one explicit pick away (T5 resident 8.87 -> 4.56 GiB).
aliases = ("flux1", "flux-1"),
# LoRA training targets FLUX.1-dev via the DiT trainer (QLoRA nf4); the dev repo is gated
trainable = True,
@@ -420,6 +430,24 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
("int8", "unsloth/FLUX.2-klein-4B-FP8"),
("fp8", "unsloth/FLUX.2-klein-4B-FP8"),
),
+ # Stock Qwen3-4B pre-cast fp8, the int8 ConvRot encoder's fallback (in
+ # diffusion_te_prequant.TE_INT8_CONVROT_FILES); the distilled and base 4B repos ship the same bytes.
+ te_prequant_repos = (("fp8", "text_encoder", "unsloth/FLUX.2-klein-4B-FP8"),),
+ # The 9B pair carries stock Qwen3-8B, a different model, so it has its own files (int8 ConvRot in
+ # diffusion_te_prequant.TE_INT8_CONVROT_VARIANT_FILES).
+ te_prequant_variant_repos = (
+ (
+ "black-forest-labs/FLUX.2-klein-9B",
+ "fp8",
+ "text_encoder",
+ "unsloth/FLUX.2-klein-9B-FP8",
+ ),
+ ),
+ # No te_quant_auto: an unset request keeps the bf16 encoder. ComfyUI loads qwen_3_4b bf16 for the 4B pair and
+ # qwen_3_8b_fp8mixed for the 9B pair; against the bf16 encoder (bf16 DiT) the hosted int8 ConvRot files score
+ # LPIPS-alex mean 0.078 (4B, 4 steps, 24 prompts), 0.103 (base 4B, 20 steps CFG 5) and 0.047 (9B), layerwise fp8
+ # 0.115 / 0.214 / 0.072. The encoder itself is close (int8 hidden-state cosine 0.9997 at the read layers), but
+ # the distilled 4B amplifies it past what a silent default should cost; the files stay for explicit picks.
aliases = ("flux2-klein",),
# A local ComfyUI file has no card tag, so without these a 9B pick got the 4B base and was refused.
named_variant_bases = (
@@ -578,6 +606,11 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
),
# Pre-cast Mistral-Small-24B conditioner (bf16 ~48 GB dense, ~24.7 GB pre-cast).
te_prequant_repos = (("fp8", "text_encoder", "unsloth/FLUX.2-dev-FP8"),),
+ # int8 ConvRot by default. ComfyUI's FLUX.2-dev templates load mistral_3_small_flux2_bf16 (2 of 3; the third
+ # fp8); against the bf16 encoder (bf16 DiT, 20 steps guidance 4, 24 prompts, bf16 A/A repeat bit-identical) the
+ # hosted int8 ConvRot file scores LPIPS-alex mean 0.057, SSIM 0.929, layerwise fp8 0.266 / SSIM 0.730 on 12. The
+ # Pixtral vision tower stays bf16. Resident 44.7 -> 22.8 GiB, load 9.3 -> 3.6 s.
+ te_quant_auto = "int8",
aliases = ("flux2-dev", "flux2dev"),
trainable = True,
train_base_repos = ("black-forest-labs/FLUX.2-dev",),
@@ -607,6 +640,12 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
uses_negative_prompt = False,
transformer_class = "FluxTransformer2DModel",
base_repo = "black-forest-labs/FLUX.1-Kontext-dev",
+ # FLUX.1's pre-cast T5-XXL: Kontext ships the same text_encoder_2 shards (sha256 compared 2026-10-16).
+ te_prequant_repos = (("fp8", "text_encoder_2", "unsloth/FLUX.1-schnell-FP8"),),
+ # fp8 by default, as ComfyUI's Kontext template (t5xxl_fp8_e4m3fn_scaled): against the bf16 encoder (bf16 DiT, 8
+ # edits, 20 steps, bf16 A/A repeat bit-identical) LPIPS-alex mean 0.034 / max 0.143, SSIM 0.945. The shared int8
+ # ConvRot T5 scores 0.021 and is one explicit pick away.
+ te_quant_auto = "fp8",
aliases = ("flux.1-kontext-dev", "flux1-kontext", "flux-kontext", "kontext"),
edit = True,
# Native: FLUX.1's VAE + CLIP-L / T5-XXL, the source as a reference image (sd.cpp docs/kontext.md).
@@ -641,6 +680,13 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
pipeline_class = "QwenImageEditPlusPipeline",
transformer_class = "QwenImageTransformer2DModel",
base_repo = "Qwen/Qwen-Image-Edit-2511",
+ # Qwen-Image's pre-cast Qwen2.5-VL-7B: every edit release (original, 2509, 2511) ships the same text_encoder/
+ # shards. The vision tower is cast too, and the edit path runs it.
+ te_prequant_repos = (("fp8", "text_encoder", "unsloth/Qwen-Image-FP8"),),
+ # int8 ConvRot by default. ComfyUI loads qwen_2.5_vl_7b_fp8_scaled; our layerwise fp8 (no scales) sits at the
+ # limit against the bf16 encoder (bf16 DiT, 8 edits, 40 steps: LPIPS-alex mean 0.060, max 0.330), the shared
+ # int8 ConvRot file at 0.017 / max 0.033, SSIM 0.970. The vision tower stays bf16 in it.
+ te_quant_auto = "int8",
cfg_kwarg = "true_cfg_scale",
aliases = (
"qwen-image-edit-2511",
@@ -685,6 +731,12 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
pipeline_class = "QwenImageLayeredPipeline",
transformer_class = "QwenImageTransformer2DModel",
base_repo = "Qwen/Qwen-Image-Layered",
+ # Qwen-Image's pre-cast Qwen2.5-VL-7B: the layered release ships the same text_encoder/ shards.
+ te_prequant_repos = (("fp8", "text_encoder", "unsloth/Qwen-Image-FP8"),),
+ # fp8 by default, as ComfyUI's layered templates (qwen_2.5_vl_7b_fp8_scaled): LPIPS-alex mean 0.0055 / max
+ # 0.034, SSIM 0.993 against the bf16 encoder (bf16 DiT, 8 inputs, 20 steps CFG 2.5, bf16 A/A repeat
+ # bit-identical).
+ te_quant_auto = "fp8",
cfg_kwarg = "true_cfg_scale",
aliases = ("qwen_image_layered", "qwenimagelayered"),
edit = True,
@@ -740,6 +792,11 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
),
# Pre-cast Qwen2.5-VL-7B (16.6 -> 8.8 GB). Always was independent of the DiT scheme rules.
te_prequant_repos = (("fp8", "text_encoder", "unsloth/Qwen-Image-FP8"),),
+ # int8 ConvRot by default. ComfyUI loads qwen_2.5_vl_7b_fp8_scaled, but our layerwise fp8 (no per-tensor scale)
+ # moves too much: against the bf16 encoder (bf16 DiT, 20 steps CFG 4, bf16 A/A repeat bit-identical) LPIPS-alex
+ # mean 0.138 / SSIM 0.844 on 12 prompts, int8 ConvRot 0.057 / SSIM 0.938 on 24 (one 0.47 is a cone moved to the
+ # side the prompt asks for). Text renders correctly in both. Resident 15.45 -> 8.36 GiB, load 7.3 -> 2.6 s.
+ te_quant_auto = "int8",
cfg_kwarg = "true_cfg_scale",
aliases = ("qwen_image", "qwenimage"),
# fp16 overflows to NaN latents (black images)
@@ -946,8 +1003,13 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
# must quantize its own dense weights.
prequant_excluded_bases = ("tongyi-mai/z-image",),
# Pre-cast Qwen3-4B TE (8.04 -> 4.41 GB), shared with the undistilled base (byte-identical encoder). NOT
- # shared with flux.2-klein-4B: klein TE retrained layer 35 MLP (up/down_proj maxdiff 0.86 vs this checkpoint).
+ # shared with flux.2-klein-4B: klein ships stock Qwen3-4B (sha256-identical to unsloth/Qwen3-4B), while this
+ # encoder is Z-Image's own variant of it (layer 35 MLP up/down_proj maxdiff 0.86 against klein's).
te_prequant_repos = (("fp8", "text_encoder", "unsloth/Z-Image-Turbo-FP8"),),
+ # No te_quant_auto: a small encoder (Qwen3-4B) stays bf16 unless ComfyUI's template quantizes it, and 3 of its
+ # 4 templates load qwen_3_4b bf16. Measured against the bf16 encoder (bf16 DiT, 8 steps, 12 prompts, bf16 A/A
+ # repeat bit-identical): int8 ConvRot LPIPS-alex mean 0.049, SSIM 0.934, layerwise fp8 0.105; both stay one
+ # explicit pick away (resident 7.49 -> 4.11 GiB).
aliases = ("zimage", "z_image"),
# LoRA training via the DiT trainer (bf16); defaults to the prequant nf4 repo for QLoRA
trainable = True,
@@ -987,6 +1049,10 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
# Pre-cast Qwen3-VL-4B TE (8.88 -> 4.83 GB); handed into load_krea2_pipeline directly (assembly never sees
# pipe_kwargs).
te_prequant_repos = (("fp8", "text_encoder", "unsloth/Krea-2-Turbo-FP8"),),
+ # int8 ConvRot by default. ComfyUI loads qwen3vl_4b_fp8_scaled; our layerwise fp8 scores LPIPS-alex mean 0.110
+ # against the bf16 encoder (bf16 DiT, Turbo 8 steps, 12 prompts, bf16 A/A repeat bit-identical), int8 ConvRot
+ # (the Comfy-Org/Krea-2 qwen3vl_4b_int8_convrot scheme) 0.047 / SSIM 0.934. Resident 8.27 -> 4.79 GiB.
+ te_quant_auto = "int8",
aliases = ("krea2",),
# no prequant repo yet, so nf4 quantizes on the fly
trainable = True,
@@ -1010,6 +1076,10 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
),
# pre-cast Gemma2-2B TE; the Hub stores it fp32 (10.46 GB), so the 3.20 GB artifact is a 3.3x download cut
te_prequant_repos = (("fp8", "text_encoder", "unsloth/Lumina-Image-2.0-FP8"),),
+ # No te_quant_auto: a small encoder (Gemma-2-2B) stays bf16 unless ComfyUI's template quantizes it, and ComfyUI
+ # ships gemma_2_2b_fp16 only. Measured against the bf16 encoder (bf16 DiT, 50 steps, 8 prompts, bf16 A/A repeat
+ # bit-identical): int8 ConvRot LPIPS-alex mean 0.015, SSIM 0.986, layerwise fp8 0.177; both stay one explicit
+ # pick away (resident 4.87 -> 2.99 GiB, download 10.46 GB fp32 -> 3.21 GB).
aliases = ("lumina-image-2.0", "lumina-image-2", "lumina2"),
fp16_incompatible = True,
),
@@ -1033,6 +1103,10 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
),
# the Qwen2.5-VL TE is byte-identical to Qwen-Image (verified sha256), so reuse that artifact: 16.58 -> 8.84 GB.
te_prequant_repos = (("fp8", "text_encoder", "unsloth/Qwen-Image-FP8"),),
+ # int8 ConvRot by default (Comfy-Org's HunyuanImage repo offers bf16 and fp8_scaled, no template): Qwen-Image's
+ # shared file scores LPIPS-alex mean 0.063, SSIM 0.917 against the bf16 encoder (bf16 DiT, 50 steps, 24 prompts,
+ # bf16 A/A repeat bit-identical); layerwise fp8 0.165. ByT5 stays bf16. Resident 15.45 -> 8.36 GiB.
+ te_quant_auto = "int8",
pipeline_class = "HunyuanImagePipeline",
transformer_class = "HunyuanImageTransformer2DModel",
base_repo = "hunyuanvideo-community/HunyuanImage-2.1-Diffusers",
@@ -1063,8 +1137,17 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
# refuses the Full artifact only after the plan has dropped their released shards.
prequant_excluded_bases = ("hidream-ai/hidream-i1-dev", "hidream-ai/hidream-i1-fast"),
# Pre-cast Llama-3.1-8B TE4 (16.1 -> 8.1 GB). The generic TE pass only covers text_encoder.._3, so TE4 engages
- # via hidream_te4_kwargs.
- te_prequant_repos = (("fp8", "text_encoder_4", "unsloth/HiDream-I1-Full-FP8"),),
+ # via hidream_te4_kwargs. text_encoder_3 is FLUX.1's T5-XXL byte for byte (sha256 compared 2026-10-16, Full and
+ # Dev), so it loads FLUX.1's hosted text_encoder_2 file: 9.52 -> 5.90 GB.
+ te_prequant_repos = (
+ ("fp8", "text_encoder_4", "unsloth/HiDream-I1-Full-FP8"),
+ ("fp8", "text_encoder_3", "unsloth/FLUX.1-schnell-FP8", "text_encoder_2"),
+ ),
+ # No te_quant_auto. ComfyUI loads t5xxl_fp8_e4m3fn_scaled + llama_3.1_8b_instruct_fp8_scaled; against the bf16
+ # encoders (bf16 DiT, Full 50 steps CFG 5, 12 prompts, bf16 A/A repeat bit-identical) the hosted int8 ConvRot T5
+ # + Llama score LPIPS-alex mean 0.103 / SSIM 0.890 and layerwise fp8 0.225 / SSIM 0.763: this DiT reads every
+ # Llama layer, and neither is close enough for a silent default. Both stay one explicit pick away (CLIPs stay
+ # bf16).
pipeline_class = "HiDreamImagePipeline",
transformer_class = "HiDreamImageTransformer2DModel",
base_repo = "HiDream-ai/HiDream-I1-Full",
@@ -1082,6 +1165,13 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
uses_negative_prompt = False,
transformer_class = "Ideogram4Transformer2DModel",
base_repo = "ideogram-ai/ideogram-4-fp8",
+ # Qwen3-VL-8B, built from the vendor's per-channel-scaled fp8 shards dequantized to bf16 (what a dense load
+ # runs): layerwise fp8 here, int8 ConvRot in diffusion_te_prequant.TE_INT8_CONVROT_FILES.
+ te_prequant_repos = (("fp8", "text_encoder", "unsloth/ideogram-4-FP8"),),
+ # No te_quant_auto. ComfyUI loads qwen3vl_8b_fp8_scaled; against the dequantized vendor encoder (bf16 DiT, 20
+ # steps CFG 7, 12 prompts, bf16 A/A repeat bit-identical) int8 ConvRot scores LPIPS-alex mean 0.081 and
+ # layerwise fp8 0.114, both past what a silent default should cost, although text survives and the encoder loads
+ # in 2.1 s instead of 153 s (8.7 instead of 15.2 GiB). The files stay for explicit picks.
aliases = ("ideogram4", "ideogram-v4", "ideogram"),
# Two DiTs assembled per-component, so no transformer-only single-file / GGUF load.
pipeline_only = True,
@@ -2720,7 +2810,8 @@ def sd_cpp_companion_only_repo_ids() -> frozenset[str]:
loadable.add(fam.deploy_base_repo)
loadable.update(repo for _scheme, repo in fam.prequant_repos)
loadable.update(repo for _base, _scheme, repo in fam.prequant_variant_repos)
- loadable.update(repo for _scheme, _component, repo in fam.te_prequant_repos)
+ loadable.update(row[2] for row in fam.te_prequant_repos)
+ loadable.update(row[3] for row in fam.te_prequant_variant_repos)
companions.update(repo for repo, _f, _k in _FLUX2_KLEIN_9B_SD_CPP_TEXT_ENCODERS)
return (
frozenset(r.strip().lower() for r in companions - loadable if r)
@@ -2737,7 +2828,8 @@ def prequant_only_repo_ids() -> frozenset[str]:
for fam in _FAMILIES:
hosted.update(repo for _scheme, repo in fam.prequant_repos)
hosted.update(repo for _base, _scheme, repo in fam.prequant_variant_repos)
- hosted.update(repo for _scheme, _component, repo in fam.te_prequant_repos)
+ hosted.update(row[2] for row in fam.te_prequant_repos)
+ hosted.update(row[3] for row in fam.te_prequant_variant_repos)
bases.add(fam.base_repo)
bases.update(fam.train_base_repos)
if fam.deploy_base_repo:
@@ -2771,13 +2863,11 @@ def prequant_repo_role(
if repo.strip().lower() == key:
bases.append(fam.base_repo)
schemes.append(scheme)
- te = sorted(
- {
- scheme
- for scheme, _component, repo in fam.te_prequant_repos
- if repo.strip().lower() == key
- }
- )
+ te = sorted({row[0] for row in fam.te_prequant_repos if row[2].strip().lower() == key})
+ te_variant = [row for row in fam.te_prequant_variant_repos if row[3].strip().lower() == key]
+ te = sorted({*te, *(row[1] for row in te_variant)})
+ if te_variant and not bases:
+ bases.extend(known.get(row[0].lower(), row[0]) for row in te_variant)
if te and not bases:
bases.append(fam.base_repo)
return tuple(dict.fromkeys(bases)), tuple(sorted(set(schemes))), tuple(te)
diff --git a/studio/backend/core/inference/diffusion_hidream.py b/studio/backend/core/inference/diffusion_hidream.py
index b188d836872..a81b6d647de 100644
--- a/studio/backend/core/inference/diffusion_hidream.py
+++ b/studio/backend/core/inference/diffusion_hidream.py
@@ -68,6 +68,7 @@ def hidream_te4_kwargs(
)
fp8_engages = False
+ hosted_int8 = False
if target is not None:
try:
from . import diffusion_precision as precision
@@ -76,22 +77,28 @@ def hidream_te4_kwargs(
normalize_te_quant,
te_quant_supported,
)
+ from .diffusion_te_prequant import int8_convrot_te_runs_on
mode = normalize_te_quant(te_quant_mode)
denied = getattr(precision, "_te_family_denied", None)
+ allowed = not (callable(denied) and denied(getattr(fam, "name", None), mode))
+ # fp8 (the hosted fp8 file, else a dense load and cast) needs fp8 support; int8 takes the hosted int8
+ # ConvRot file, which Apple Silicon runs as plain tensors where fp8 does not.
fp8_engages = (
- mode == TE_QUANT_FP8
- and te_quant_supported(target, mode)
- and not (callable(denied) and denied(getattr(fam, "name", None), mode))
+ mode in (TE_QUANT_FP8, "int8")
+ and te_quant_supported(target, TE_QUANT_FP8)
+ and allowed
)
+ hosted_int8 = mode == "int8" and allowed and int8_convrot_te_runs_on(target)
except Exception: # noqa: BLE001 -- quant probe failure keeps the dense bf16 path
- fp8_engages = False
+ fp8_engages = hosted_int8 = False
- if fp8_engages and fam is not None:
+ if (fp8_engages or hosted_int8) and fam is not None:
from .diffusion_te_prequant import (
load_prequant_text_encoder,
te_prequant_sources_for_base,
)
+
source = te_prequant_sources_for_base(
fam,
HIDREAM_LLAMA_REPO,
@@ -101,14 +108,17 @@ def hidream_te4_kwargs(
standalone_component_bases = {"text_encoder_4": HIDREAM_LLAMA_REPO},
).get("text_encoder_4")
if source is not None:
+ scheme = "int8" if normalize_te_quant(te_quant_mode) == "int8" else "fp8"
encoder = load_prequant_text_encoder(
HIDREAM_LLAMA_REPO,
"text_encoder_4",
source,
dtype = dtype,
hf_token = hf_token,
- scheme = "fp8",
+ scheme = scheme,
logger = logger,
+ # The int8 file carries no lm_head; the pipeline reads hidden states only.
+ trim_lm_head = scheme == "int8",
# The Llama TE4 lives in its own standalone repo (config at the root), and the pipeline needs hidden
# states/attentions from its forward.
config_subfolder = "",
@@ -120,6 +130,17 @@ def hidream_te4_kwargs(
)
if encoder is not None:
return {"text_encoder_4": encoder, "tokenizer_4": tokenizer_4}
+ if hosted_int8 and not fp8_engages:
+ from .diffusion_precision import te_quant_picked_automatically
+ if not te_quant_picked_automatically(te_quant_mode):
+ # Apple Silicon: the plan priced the hosted int8 TE4 and nothing else runs here, so the 16 GB dense
+ # Llama is not loaded in its place (as _te_prequant_pipe_kwargs refuses for the other encoders).
+ raise RuntimeError(
+ f"The pre-quantized text_encoder_4 for {HIDREAM_LLAMA_REPO} could not be loaded "
+ f"({'not available' if source is None else 'see the server log'}). The memory check sized "
+ "this load for it, so the larger dense encoder is not loaded in its place. Set Text encoder "
+ "precision to Dense (bf16) to load the dense encoder instead."
+ )
logger.info("diffusion.hidream: loading Llama TE4 from %s", HIDREAM_LLAMA_REPO)
text_encoder_4 = LlamaForCausalLM.from_pretrained(
@@ -141,6 +162,9 @@ def hidream_te4_kwargs(
cast_target = _Target()
cast_target.dtype = dtype
_cast_fp8(text_encoder_4, cast_target)
+ from .diffusion_precision import TE_RUNTIME_SCHEME_ATTR
+
+ setattr(text_encoder_4, TE_RUNTIME_SCHEME_ATTR, "fp8")
logger.info("diffusion.hidream: TE4 layerwise fp8 cast engaged")
except Exception as exc: # noqa: BLE001 -- best-effort like the generic TE pass
# A mid-pass failure can leave fp8 storage / upcast hooks behind, and a half-cast encoder cannot run dense,
diff --git a/studio/backend/core/inference/diffusion_ideogram4.py b/studio/backend/core/inference/diffusion_ideogram4.py
index 42bb907dd33..7cff95a4f35 100644
--- a/studio/backend/core/inference/diffusion_ideogram4.py
+++ b/studio/backend/core/inference/diffusion_ideogram4.py
@@ -458,8 +458,12 @@ def load_ideogram4_pipeline(
dtype,
hf_token: Optional[str] = None,
check_cancelled: Optional[Callable[[], None]] = None,
+ text_encoder: Any = None,
):
- """Assemble Ideogram4Pipeline from ``repo_id`` per-component (see module doc)."""
+ """Assemble Ideogram4Pipeline from ``repo_id`` per-component (see module doc).
+
+ ``text_encoder`` is a ready encoder (the hosted pre-cast fp8 one); the dense vendor-fp8 encoder is read and
+ dequantized only when it is None, so a pre-cast load never touches the ~9 GB of text_encoder shards."""
check_cancelled = check_cancelled or (lambda: None)
check_cancelled()
import diffusers
@@ -473,9 +477,10 @@ def load_ideogram4_pipeline(
if token:
model_kwargs["token"] = token
- text_encoder = load_ideogram4_text_encoder(
- repo_id, dtype, hf_token = token, check_cancelled = check_cancelled
- )
+ if text_encoder is None:
+ text_encoder = load_ideogram4_text_encoder(
+ repo_id, dtype, hf_token = token, check_cancelled = check_cancelled
+ )
check_cancelled()
tokenizer = load_krea2_tokenizer(repo_id, hf_token = token, check_cancelled = check_cancelled)
check_cancelled()
diff --git a/studio/backend/core/inference/diffusion_precision.py b/studio/backend/core/inference/diffusion_precision.py
index 38c75a3e676..00b7d28ea47 100644
--- a/studio/backend/core/inference/diffusion_precision.py
+++ b/studio/backend/core/inference/diffusion_precision.py
@@ -96,6 +96,26 @@ def te_quant_is_auto(value: Optional[str]) -> bool:
return not normalized or normalized == "auto"
+class AutoTeQuant(str):
+ """A scheme ``resolve_te_quant_request`` picked from the family default, not one the caller typed. Equal to and
+ hashed as the plain string, so every consumer that only needs the scheme is unchanged; the type lets the planners
+ that run outside a load (download plan, memory budget) tell an automatic pick from an explicit one."""
+
+ __slots__ = ()
+
+
+def te_quant_picked_automatically(mode: Any) -> bool:
+ """Whether ``mode`` is an automatic pick: an ``AutoTeQuant``, or any mode inside a load whose raw request was
+ unset / "auto" (``diffusion_te_prequant.auto_te_request``)."""
+ if isinstance(mode, AutoTeQuant):
+ return True
+ try:
+ from .diffusion_te_prequant import te_request_is_auto
+ return te_request_is_auto()
+ except Exception: # noqa: BLE001
+ return False
+
+
def resolve_te_quant_request(
value: Optional[str], auto_scheme: Optional[str]
) -> tuple[Optional[str], bool]:
@@ -115,7 +135,8 @@ def resolve_te_quant_request(
return None, False
# Validate the family's own field rather than trusting it: a typo here would otherwise reach
# quantize_text_encoders as an unknown mode on every default load of that family.
- return normalize_te_quant(auto_scheme), True
+ scheme = normalize_te_quant(auto_scheme)
+ return (AutoTeQuant(scheme) if scheme is not None else None), True
def effective_te_quant(mode: Optional[str], family: Optional[str]) -> Optional[str]:
@@ -128,11 +149,72 @@ def effective_te_quant(mode: Optional[str], family: Optional[str]) -> Optional[s
unsupported while fp8 still works.
"""
normalized = normalize_te_quant(mode)
- if normalized == TE_QUANT_INT8 and _TE_INT8_SKIP.get((family or "").lower()) is None:
+ if normalized == TE_QUANT_INT8 and (
+ _TE_INT8_SKIP.get((family or "").lower()) is None or _family_hosts_te_int8(family)
+ ):
+ # A family hosting an int8 ConvRot file loads it as plain tensors (no torchao), with its fp8 file behind it.
return TE_QUANT_FP8
return normalized
+def preflight_te_quant(
+ mode: Optional[str],
+ fam: Any,
+ base: Optional[str],
+ target: Any,
+ *,
+ supplied: Any = (),
+ local_files_only: Optional[bool] = None,
+) -> Optional[str]:
+ """The text-encoder mode a pre-load gate must check for an EXPLICIT request: None when an int8 request will
+ load the hosted int8 ConvRot file (plain tensors, no torchao, offload-safe, runs on MPS), else the mode the
+ runtime will attempt without it -- torchao int8 on a family with a keep-bf16 schedule and torchao present (with
+ its device and offload limits), fp8 otherwise -- so the gate refuses what the load would refuse."""
+ normalized = normalize_te_quant(mode)
+ family = getattr(fam, "name", None)
+ if normalized != TE_QUANT_INT8:
+ return effective_te_quant(normalized, family)
+ from .diffusion_te_prequant import (
+ TE_INT8_CONVROT_FILES,
+ explicit_int8_hosted_components,
+ explicit_int8_takes_hosted_file,
+ )
+
+ if explicit_int8_takes_hosted_file(
+ fam, base, target, supplied = supplied or (), local_files_only = local_files_only
+ ):
+ return None
+ takes = explicit_int8_hosted_components(
+ fam, base, target, supplied = supplied or (), local_files_only = local_files_only
+ )
+ missing = [c for c, ok in takes.items() if not ok]
+ supplied_hosted = [c for c in missing if c in set(supplied or ())]
+ if supplied_hosted and len(missing) < len(takes):
+ # A supplied dense encoder in one hosted slot beside a hosted int8 file in another: mixed precisions.
+ raise RuntimeError(
+ f"Text encoder precision int8 would load the hosted int8 ConvRot file for some {family} text encoders, "
+ f"but the supplied {', '.join(supplied_hosted)} is not int8, so the encoders would load at mixed "
+ "precisions. Choose fp8 or Dense (bf16), or supply every text encoder."
+ )
+ if missing and len(missing) < len(takes):
+ # Some encoders would load their hosted int8 file and the rest fall back (HiDream: T5 int8 beside an fp8
+ # Llama), a mixed build quantize_text_encoders refuses only after the download and the eviction. Refuse here.
+ files = TE_INT8_CONVROT_FILES.get((family or "").strip().lower(), {})
+ named = ", ".join(
+ f"{c} ({files[c][1]})" if c in files and len(files[c]) > 1 else c for c in missing
+ )
+ raise RuntimeError(
+ f"Text encoder precision int8 needs the hosted int8 ConvRot file for every {family} text encoder, but "
+ f"{named} is not available here while the others are, so the encoders would load at mixed precisions. "
+ "Choose fp8 for all of them, or Dense (bf16)."
+ )
+ if _family_hosts_te_int8(family):
+ if _TE_INT8_SKIP.get((family or "").lower()) is not None and torchao_quantize_importable():
+ return TE_QUANT_INT8
+ return TE_QUANT_FP8
+ return effective_te_quant(normalized, family)
+
+
def te_quant_needs_resident_weights(mode: Optional[str]) -> bool:
"""Whether ``mode`` is a torchao text-encoder cast, which CPU offload rules out.
@@ -238,6 +320,10 @@ class TEQuantOutcome(NamedTuple):
partial: bool = False
+# Set on an encoder cast in place outside this pass (HiDream's text_encoder_4), so the outcome can report it.
+TE_RUNTIME_SCHEME_ATTR = "_unsloth_te_runtime_scheme"
+
+
def quantize_text_encoders(
pipe: Any,
target: Any,
@@ -247,6 +333,39 @@ def quantize_text_encoders(
offload_active: bool = False,
logger: Any = None,
encoders: Optional[tuple[str, ...]] = None,
+) -> TEQuantOutcome:
+ """See ``_quantize_text_encoders``; also folds in HiDream's ``text_encoder_4``, which its family loader quantizes
+ (``hidream_te4_kwargs``), so a TE4 that ended up on another scheme is reported rather than hidden."""
+ outcome = _quantize_text_encoders(
+ pipe,
+ target,
+ mode = mode,
+ family = family,
+ offload_active = offload_active,
+ logger = logger,
+ encoders = encoders,
+ )
+ te4 = getattr(pipe, "text_encoder_4", None)
+ if te4 is None or outcome.mode is None:
+ return outcome
+ te4_scheme = _hosted_te_scheme(te4) or getattr(te4, TE_RUNTIME_SCHEME_ATTR, None)
+ if te4_scheme == outcome.mode:
+ return outcome
+ held = f"text_encoder_4 is {te4_scheme}" if te4_scheme else "text_encoder_4 stayed dense bf16"
+ reason = f"{outcome.reason}; {held}" if outcome.reason else held
+ # The fourth conditioner is not on the scheme the rest took: a mixed build, which an explicit request refuses.
+ return outcome._replace(reason = reason, status = RESOLVED_FELL_BACK, partial = True)
+
+
+def _quantize_text_encoders(
+ pipe: Any,
+ target: Any,
+ *,
+ mode: Optional[str],
+ family: Optional[str] = None,
+ offload_active: bool = False,
+ logger: Any = None,
+ encoders: Optional[tuple[str, ...]] = None,
) -> TEQuantOutcome:
"""Quantise each present text encoder in place with ``mode``. Returns a ``TEQuantOutcome``
carrying the mode applied (None when disabled, unsupported, or nothing was cast) plus WHY it
@@ -255,6 +374,7 @@ def quantize_text_encoders(
subclasses reject ``Module.to()``); layerwise ``fp8`` still engages. Best-effort: any failure
leaves the encoder dense. ``encoders`` limits the cast to those pipeline attributes (the rest stay as loaded),
e.g. HunyuanVideo-1.5's automatic fp8 targets its Qwen tower and leaves the ByT5 glyph encoder dense."""
+ picked_automatically = te_quant_picked_automatically(mode)
mode = normalize_te_quant(mode)
attrs = tuple(a for a in _TEXT_ENCODER_ATTRS if encoders is None or a in encoders)
if mode is None:
@@ -277,10 +397,60 @@ def quantize_text_encoders(
"hosted int8 ConvRot weight-only text encoder",
RESOLVED_APPLIED,
)
+ if (
+ mode == TE_QUANT_INT8
+ and hosted_int8
+ and _hosts_te_int8_for_all_but_small(family, hosted_int8, present)
+ ):
+ # FLUX.1's CLIP-L, HunyuanImage's ByT5: the hosted file covers the big encoder, and casting a sub-GB pooled /
+ # glyph encoder in place would be a runtime-only conversion the file does not carry. Leave them bf16.
+ dense = [a for a in present if a not in hosted_int8]
+ return TEQuantOutcome(
+ TE_QUANT_INT8,
+ f"hosted int8 ConvRot weight-only {', '.join(hosted_int8)}; {', '.join(dense)} kept bf16",
+ RESOLVED_APPLIED,
+ )
+ if picked_automatically and not any(
+ _hosted_te_scheme(getattr(pipe, a)) == mode for a in present
+ ):
+ # The automatic default is the measured hosted file and nothing else: a runtime cast (fp8, torchao int8) has a
+ # dense bf16 peak and is not what was measured, so without the file the released bf16 encoder stays.
+ label = "int8 ConvRot" if mode == TE_QUANT_INT8 else mode
+ why = "is not available (not published yet, offline, or refused, see the server log)"
+ if mode == TE_QUANT_INT8 and _family_hosts_te_int8(family):
+ from .diffusion_text_encoder_trim import KEEP_LM_HEAD_ENV, family_trims_lm_head
+ if not family_trims_lm_head(family):
+ why = f"carries no lm_head, which {KEEP_LM_HEAD_ENV} keeps"
+ _note(logger, f"automatic {label} text encoder not used; keeping the dense bf16 encoder")
+ return TEQuantOutcome(
+ None,
+ f"the hosted {label} text encoder {why}, so the automatic default kept the dense bf16 encoder",
+ RESOLVED_FELL_BACK,
+ )
+ if picked_automatically:
+ # The automatic default is the hosted file and nothing more: a sibling it does not replace (Kontext's CLIP-L)
+ # stays bf16 rather than taking a runtime cast nobody measured. The hosted encoders arrive with their hooks.
+ hosted = [a for a in present if _hosted_te_scheme(getattr(pipe, a)) == mode]
+ dense = [a for a in present if a not in hosted]
+ return TEQuantOutcome(
+ mode,
+ f"hosted {mode} {', '.join(hosted)}"
+ + (f"; {', '.join(dense)} kept bf16" if dense else ""),
+ RESOLVED_APPLIED,
+ )
downgrade_reason = ""
skip: Optional[tuple[int, int]] = None
if mode == TE_QUANT_INT8:
skip = _TE_INT8_SKIP.get((family or "").lower())
+ pre_cast = [a for a in present if _hosted_te_scheme(getattr(pipe, a)) is not None]
+ if (
+ skip is not None
+ and _family_hosts_te_int8(family)
+ and (pre_cast or not torchao_quantize_importable())
+ ):
+ # The hosted int8 file did not engage. The on-the-fly torchao int8 (the path before the file existed) is
+ # the fallback only over a dense encoder and with torchao present; otherwise fp8.
+ skip = None
if skip is None:
_note(logger, f"int8 has no keep-bf16 schedule for family '{family}'; using fp8")
mode = TE_QUANT_FP8
@@ -376,6 +546,21 @@ def _hosted_te_scheme(encoder: Any) -> Optional[str]:
return value if isinstance(value, str) else None
+def _hosts_te_int8_for_all_but_small(family: Optional[str], hosted: list, present: list) -> bool:
+ """Whether every encoder the family hosts an int8 ConvRot file for loaded from it, so the rest are the small
+ companions the family deliberately leaves dense."""
+ try:
+ from .diffusion_te_prequant import TE_INT8_CONVROT_FILES
+
+ # HiDream's text_encoder_4 is assembled outside the attributes this pass walks.
+ listed = set(TE_INT8_CONVROT_FILES.get((family or "").strip().lower(), {})) & set(
+ _TEXT_ENCODER_ATTRS
+ )
+ except Exception: # noqa: BLE001
+ return False
+ return bool(listed) and listed <= set(hosted) and set(hosted) < set(present)
+
+
def _family_hosts_te_int8(family: Optional[str]) -> bool:
try:
from .diffusion_te_prequant import TE_INT8_CONVROT_FILES
diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.py
index 7361202dd1c..566c065379b 100644
--- a/studio/backend/core/inference/diffusion_te_prequant.py
+++ b/studio/backend/core/inference/diffusion_te_prequant.py
@@ -25,7 +25,10 @@ None and the caller falls back to the dense download + cast. Inert with nothing
from __future__ import annotations
+import re
import sys
+from contextlib import contextmanager
+from contextvars import ContextVar
from dataclasses import dataclass, replace
from typing import Any, Iterable, Optional
@@ -48,13 +51,79 @@ TE_PREQUANT_FORMAT_INT8_CONVROT = "unsloth_prequant_text_encoder_int8_convrot_v1
# family -> {component: (repo_id, filename)}. ComfyUI's qwen3vl_8b_int8_convrot scheme: decoder projections int8 in the
# group-256 Hadamard basis + fp32 per-row scale; vision tower, embedding, norms bf16; no lm_head. Built by
# scripts/build_te_int8_convrot_checkpoint.py.
-TE_INT8_CONVROT_FILES: dict[str, dict[str, tuple[str, str]]] = {
+_T5_INT8 = ("unsloth/FLUX.1-schnell-FP8", "FLUX.1-schnell-text_encoder_2-INT8-ConvRot.safetensors")
+_QWEN25VL_INT8 = ("unsloth/Qwen-Image-FP8", "Qwen-Image-text_encoder-INT8-ConvRot.safetensors")
+
+TE_INT8_CONVROT_FILES: dict[str, dict[str, tuple[str, ...]]] = {
"qwen-image-2.1": {
"text_encoder": (
"unsloth/Qwen-Image-2.1-FP8",
"Qwen-Image-2.1-text_encoder-INT8-ConvRot.safetensors",
),
},
+ # Qwen3-4B (Qwen3ForCausalLM, ``model.layers``), stock weights: the same build as above on a text-only Qwen3. The
+ # distilled and base 4B repos ship identical bytes, so one file serves both.
+ "flux.2-klein": {
+ "text_encoder": (
+ "unsloth/FLUX.2-klein-4B-FP8",
+ "FLUX.2-klein-4B-text_encoder-INT8-ConvRot.safetensors",
+ ),
+ },
+ # The rest are built with ``--load transformers`` from the class the pipeline instantiates, so their keys are that
+ # class's runtime state dict. One file per distinct encoder; byte-identical encoders share it through
+ # ``_TE_EQUIVALENT_BASES``. T5-XXL (FLUX.1 schnell / dev / Krea / Kontext): all seven projections per block int8.
+ "flux.1": {"text_encoder_2": _T5_INT8},
+ "flux.1-kontext": {"text_encoder_2": _T5_INT8},
+ # Mistral-Small-3.2-24B (the Mistral3 conditional-generation class): the Pixtral vision tower and projector stay bf16.
+ "flux.2-dev": {
+ "text_encoder": (
+ "unsloth/FLUX.2-dev-FP8",
+ "FLUX.2-dev-text_encoder-INT8-ConvRot.safetensors",
+ ),
+ },
+ # Qwen2.5-VL-7B, shared by every Qwen-Image release and HunyuanImage-2.1; the vision tower stays bf16 (edit runs
+ # it).
+ "qwen-image": {"text_encoder": _QWEN25VL_INT8},
+ "qwen-image-edit": {"text_encoder": _QWEN25VL_INT8},
+ "qwen-image-layered": {"text_encoder": _QWEN25VL_INT8},
+ "hunyuanimage-2.1": {"text_encoder": _QWEN25VL_INT8},
+ # Z-Image's own Qwen3-4B, as the bare Qwen3Model the pipeline builds
+ "z-image": {
+ "text_encoder": (
+ "unsloth/Z-Image-Turbo-FP8",
+ "Z-Image-Turbo-text_encoder-INT8-ConvRot.safetensors",
+ ),
+ },
+ # Krea's Qwen3-VL-4B, as Qwen3VLModel
+ "krea-2": {
+ "text_encoder": (
+ "unsloth/Krea-2-Turbo-FP8",
+ "Krea-2-Turbo-text_encoder-INT8-ConvRot.safetensors",
+ ),
+ },
+ # HiDream-I1: text_encoder_3 is FLUX.1's T5-XXL (the third element names the component the file was built as), and
+ # text_encoder_4 the Llama-3.1-8B-Instruct mirror, assembled by diffusion_hidream.hidream_te4_kwargs.
+ "hidream-i1": {
+ "text_encoder_3": (*_T5_INT8, "text_encoder_2"),
+ "text_encoder_4": (
+ "unsloth/HiDream-I1-Full-FP8",
+ "HiDream-I1-Full-text_encoder_4-INT8-ConvRot.safetensors",
+ ),
+ },
+ # Ideogram 4's Qwen3-VL-8B (Qwen3VLModel), from the vendor fp8 shards dequantized the way the dense load does
+ "ideogram-4": {
+ "text_encoder": (
+ "unsloth/ideogram-4-FP8",
+ "ideogram-4-text_encoder-INT8-ConvRot.safetensors",
+ ),
+ },
+ # Gemma-2-2B (Gemma2Model), stored fp32 on the Hub
+ "lumina-2": {
+ "text_encoder": (
+ "unsloth/Lumina-Image-2.0-FP8",
+ "Lumina-Image-2.0-text_encoder-INT8-ConvRot.safetensors",
+ ),
+ },
# UMT5-XXL, one file for all four Wan bases (an equivalence group below): every self-attention and gated-FFN
# projection of encoder.block int8 in the group-256 Hadamard basis, the shared embedding and norms dense. Opt-in
# (text_encoder_quant=int8); the family default is the fp8 file beside it, as ComfyUI's Wan templates.
@@ -83,6 +152,53 @@ TE_INT8_CONVROT_FILES: dict[str, dict[str, tuple[str, str]]] = {
},
}
+# family -> lowercased variant base -> {component: (repo_id, filename)}, for a variant whose encoder is a different
+# model from the family base's (FLUX.2 klein 9B: Qwen3-8B, not Qwen3-4B). Looked up by the base the family view
+# carries (``te_prequant_sources_for_base`` swaps it in), so the family base keeps the table above.
+TE_INT8_CONVROT_VARIANT_FILES: dict[str, dict[str, dict[str, tuple[str, str]]]] = {
+ "flux.2-klein": {
+ "black-forest-labs/flux.2-klein-9b": {
+ "text_encoder": (
+ "unsloth/FLUX.2-klein-9B-FP8",
+ "FLUX.2-klein-9B-text_encoder-INT8-ConvRot.safetensors",
+ ),
+ },
+ },
+}
+
+# Module names of the Linears an int8 ConvRot file stores int8: decoder q/k/v/o + gate/up/down of a Qwen / Llama /
+# Mistral / Gemma stack (``...layers.N.``; vision towers name their blocks ``blocks`` / ``attention`` and stay bf16)
+# and T5's SelfAttention q/k/v/o + DenseReluDense wi_0/wi_1/wo. The builder quantizes exactly these and the loader
+# refuses a file that leaves one of them dense.
+TE_INT8_CONVROT_LINEAR = re.compile(
+ r"(?:^|\.)layers\.\d+\.(?:self_attn\.[qkvo]_proj|mlp\.(?:gate|up|down)_proj)$"
+ r"|(?:^|\.)block\.\d+\.layer\.\d+\.(?:SelfAttention\.[qkvo]|DenseReluDense\.(?:wi_0|wi_1|wo))$"
+)
+
+# True while a load runs whose text-encoder scheme was picked automatically (unset / "auto"). Nobody asked for that
+# scheme, so a hosted file that cannot be loaded never fails the load: the dense encoder is loaded instead (logged). An
+# explicit request keeps the strict refusal. Set by the image loader around each load (``auto_te_request``).
+_AUTO_TE_REQUEST: ContextVar[bool] = ContextVar("unsloth_auto_te_request", default = False)
+
+
+@contextmanager
+def auto_te_request(auto: bool):
+ token = _AUTO_TE_REQUEST.set(bool(auto))
+ try:
+ yield
+ finally:
+ _AUTO_TE_REQUEST.reset(token)
+
+
+def te_request_is_auto() -> bool:
+ return _AUTO_TE_REQUEST.get()
+
+
+def _picked_automatically(mode: Any) -> bool:
+ from .diffusion_precision import te_quant_picked_automatically
+ return te_quant_picked_automatically(mode)
+
+
# Families whose encoder class has no lm_head at all (UMT5EncoderModel), so the int8 file needs no lm_head trim and the
# UNSLOTH_TE_KEEP_LM_HEAD kill switch has nothing to keep.
_TE_INT8_HEADLESS_FAMILIES = frozenset(
@@ -117,9 +233,14 @@ def quantize_int8_convrot_weight(weight: Any, group_size: int = 256) -> tuple[An
return torch.round(rotated / scale).clamp(-127, 127).to(torch.int8), scale
-def family_te_int8_convrot(fam: Any, component: str) -> Optional[tuple[str, str]]:
- """``(repo_id, filename)`` of the family's hosted int8 ConvRot encoder for ``component``, or None."""
+def family_te_int8_convrot(fam: Any, component: str) -> Optional[tuple[str, ...]]:
+ """``(repo_id, filename)`` of the family's hosted int8 ConvRot encoder for ``component``, or None. A family view
+ carrying a variant base with its own encoder (``TE_INT8_CONVROT_VARIANT_FILES``) gets that variant's file only."""
name = str(getattr(fam, "name", fam) or "").strip().lower()
+ base = str(getattr(fam, "base_repo", "") or "").strip().lower()
+ variant = TE_INT8_CONVROT_VARIANT_FILES.get(name, {}).get(base)
+ if variant is not None:
+ return variant.get(component)
return TE_INT8_CONVROT_FILES.get(name, {}).get(component)
@@ -164,11 +285,15 @@ TE_PREQUANT_BUDGET_SCALE = 0.65
# The int8 ConvRot encoder alone (no fp8 fallback to cover): Qwen-Image-2.1's is 9,349,769,248 bytes against
# 17,534,339,488 for the dense encoder, 0.533, rounded up.
TE_INT8_CONVROT_BUDGET_SCALE = 0.56
-# Per family where the default under-states: FLUX.3's Qwen3-VL-4B file is 5,247,076,320 bytes against 8,875,719,344
-# dense, 0.591 (its bf16 embedding table is a larger share of a 4B encoder), rounded up.
+# Per family where the default under-states: an encoder whose bf16 embedding table is a large share of it keeps more
+# bytes dense. Lumina-2's Gemma-2-2B file is 3,207,382,688 bytes against the 5.23 GB bf16 equivalent of its fp32 Hub
+# copy (0.613), Krea-2's Qwen3-VL-4B 5,247,054,376 against 8.88 GB dense (0.591); both rounded up.
+# FLUX.3's Qwen3-VL-4B file is 5,247,076,320 bytes against 8,875,719,344 dense (0.591), rounded up the same way.
# Wan's UMT5 keeps its 256k-row embedding dense: its int8 file is 6,735,404,944 bytes against 11,361,851,208 dense,
# 0.593, rounded up the same way.
_TE_INT8_CONVROT_BUDGET_SCALES = {
+ "lumina-2": 0.62,
+ "krea-2": 0.60,
"flux.3-klein": 0.60,
"flux.3-klein-base": 0.60,
**{
@@ -219,9 +344,192 @@ def te_prequant_budget_scale(
return 1.0
if not sources:
return 1.0
- if _int8_convrot_only(te_quant_mode, target):
- return te_int8_convrot_budget_scale(fam)
- return TE_PREQUANT_BUDGET_SCALE
+ if _auto_pick(te_quant_mode) and not te_prequant_sources_confirmed(sources):
+ # An automatic default whose hosted file is not confirmed loads the DENSE encoder, so budget that.
+ return 1.0
+ scale = (
+ te_int8_convrot_budget_scale(fam)
+ if _int8_convrot_only(te_quant_mode, target)
+ else TE_PREQUANT_BUDGET_SCALE
+ )
+ return _with_dense_siblings(fam, base, te_quant_mode, scale)
+
+
+# GB of each family's bf16 text-encoder term (diffusion_auto_policy._FAMILY_BF16_GB) held by companions no hosted file
+# replaces: FLUX.1's CLIP-L, HunyuanImage-2.1's ByT5, HiDream's CLIP-L + CLIP-G. They stay bf16 under a hosted int8
+# file and under an automatic pick (an explicit fp8 still casts them, as before), so the scale must not shrink them.
+_TE_DENSE_SIBLING_GB = {
+ "flux.1": 0.25,
+ "flux.1-kontext": 0.25,
+ "hunyuanimage-2.1": 0.8,
+ "hidream-i1": 3.3,
+}
+
+
+def te_dense_sibling_gb(fam: Any, te_quant_mode: Any) -> float:
+ """GB of the family's text-encoder term that stays dense bf16 beside a hosted pre-cast file for this pick."""
+ try:
+ from .diffusion_precision import TE_QUANT_INT8, normalize_te_quant
+ if normalize_te_quant(te_quant_mode) != TE_QUANT_INT8 and not _auto_pick(te_quant_mode):
+ return 0.0
+ except Exception: # noqa: BLE001
+ return 0.0
+ return _TE_DENSE_SIBLING_GB.get(str(getattr(fam, "name", "") or "").strip().lower(), 0.0)
+
+
+def _with_dense_siblings(fam: Any, base: Optional[str], te_quant_mode: Any, scale: float) -> float:
+ """``scale`` applied to the hosted share of the encoder term only; the dense siblings keep their full size."""
+ sibling = te_dense_sibling_gb(fam, te_quant_mode)
+ if sibling <= 0.0:
+ return scale
+ try:
+ from .diffusion_auto_policy import family_bf16_components_gb
+ table = family_bf16_components_gb(fam, base)
+ total = float(table[1]) if table else 0.0
+ except Exception: # noqa: BLE001
+ total = 0.0
+ if total <= sibling:
+ return scale
+ return (sibling + (total - sibling) * scale) / total
+
+
+def explicit_int8_hosted_components(
+ fam: Any,
+ base: Optional[str],
+ target: Any,
+ *,
+ supplied: Iterable[str] = (),
+ local_files_only: Optional[bool] = None,
+) -> dict[str, bool]:
+ """``{component: takes its hosted int8 ConvRot file}`` for every encoder ``fam`` hosts one for that no supplied
+ encoder replaces (HiDream's text_encoder_4 included): the device runs the file, the lm_head trim is on (else the
+ resolver puts fp8 first), the base matches the file's, and the file is confirmed (on disk, or listed by the
+ Hub)."""
+ name = str(getattr(fam, "name", "") or "").strip().lower()
+ out: dict[str, bool] = {}
+ for component in TE_INT8_CONVROT_FILES.get(name, {}):
+ if component in set(supplied or ()):
+ # A supplied encoder replaces the hosted file in this slot, so the slot does not take int8.
+ out[component] = False
+ continue
+ try:
+ standalone: dict[str, str] = {}
+ if component == "text_encoder_4":
+ from .diffusion_hidream import HIDREAM_LLAMA_REPO
+ standalone = {"text_encoder_4": HIDREAM_LLAMA_REPO}
+ # TE4 is matched against its own standalone repo, the rest against the selected base.
+ component_base = standalone.get(component) or base or getattr(fam, "base_repo", "")
+ got = te_prequant_sources_for_base(
+ fam,
+ str(component_base or ""),
+ te_quant_mode = "int8",
+ target = target,
+ components = (component,),
+ **({"standalone_component_bases": standalone} if standalone else {}),
+ ).get(component)
+ out[component] = bool(
+ got is not None
+ and not (got.kind == "repo" and _is_fp8_te_name(got.filename))
+ and te_prequant_sources_confirmed(
+ {component: replace(got, fallback_filenames = ())},
+ local_files_only = local_files_only,
+ )
+ )
+ except Exception: # noqa: BLE001 - unanswerable: not granted
+ out[component] = False
+ return out
+
+
+def explicit_int8_takes_hosted_file(
+ fam: Any,
+ base: Optional[str],
+ target: Any,
+ *,
+ supplied: Iterable[str] = (),
+ local_files_only: Optional[bool] = None,
+) -> bool:
+ """Whether an explicit int8 request on ``fam`` / ``base`` will load the hosted int8 ConvRot file for EVERY
+ encoder the family hosts one for (see ``explicit_int8_hosted_components``), with none replaced by a supplied
+ encoder. Preflight, precision and the HiDream TE4 gate all ask this, so none of them grants the plain-tensor int8
+ path to a load that will not take it."""
+ name = str(getattr(fam, "name", "") or "").strip().lower()
+ if not TE_INT8_CONVROT_FILES.get(name):
+ return False
+ takes = explicit_int8_hosted_components(
+ fam, base, target, supplied = supplied, local_files_only = local_files_only
+ )
+ return len(takes) == len(TE_INT8_CONVROT_FILES[name]) and all(takes.values())
+
+
+def _auto_pick(te_quant_mode: Any) -> bool:
+ try:
+ from .diffusion_precision import te_quant_picked_automatically
+ return te_quant_picked_automatically(te_quant_mode)
+ except Exception: # noqa: BLE001
+ return False
+
+
+# How long a Hub-listing verdict is reused, so the several budget calls of one load ask the Hub once.
+_CONFIRMED_TTL_S = 300.0
+_confirmed_cache: dict[tuple, tuple[float, bool]] = {}
+
+
+def _hub_offline() -> bool:
+ try:
+ from huggingface_hub import constants
+ return bool(constants.HF_HUB_OFFLINE)
+ except Exception: # noqa: BLE001
+ return False
+
+
+def te_prequant_sources_confirmed(
+ sources: dict[str, "TePrequantSource"], *, local_files_only: Optional[bool] = None
+) -> bool:
+ """Whether every hosted pre-cast file in ``sources`` is CONFIRMED: on disk (path, mirror, Hub cache), or listed
+ by the Hub. Offline, or when the Hub cannot answer, only what is on disk counts. An automatic pick budgets the
+ dense encoder unless this holds, since that is what its load falls back to."""
+ import os
+ import time
+
+ pending: dict[str, "TePrequantSource"] = {}
+ for component, source in sources.items():
+ kind = getattr(source, "kind", None)
+ if kind == "path":
+ if not os.path.isfile(os.path.expanduser(str(source.location))):
+ return False
+ continue
+ if kind != "repo" or not getattr(source, "filename", None):
+ return False
+ names = [n for n in te_candidate_filenames(source) if te_candidate_is_readable(n)]
+ if not any(_held_locally(source.location, n, None) for n in names):
+ pending[component] = source
+ if not pending:
+ return True
+ if local_files_only is None:
+ # Inside a load the prequant layer records the load's own local_files_only.
+ try:
+ from .diffusion_prequant import _LOCAL_FILES_ONLY
+ local_files_only = bool(_LOCAL_FILES_ONLY.get())
+ except Exception: # noqa: BLE001
+ local_files_only = False
+ if local_files_only or _hub_offline():
+ # A load that may not fetch can only use what is on disk.
+ return False
+ key = tuple(
+ sorted((c, s.location, tuple(te_candidate_filenames(s))) for c, s in pending.items())
+ )
+ now = time.monotonic()
+ hit = _confirmed_cache.get(key)
+ if hit is not None and now - hit[0] < _CONFIRMED_TTL_S:
+ return hit[1]
+ try:
+ from huggingface_hub import HfApi
+ found = te_prequant_hub_files(pending, HfApi())
+ confirmed = all(c in found for c in pending)
+ except Exception: # noqa: BLE001 - an unanswerable Hub confirms nothing
+ confirmed = False
+ _confirmed_cache[key] = (now, confirmed)
+ return confirmed
def _int8_convrot_only(te_quant_mode: Optional[str], target: Any) -> bool:
@@ -248,6 +556,11 @@ _TE_EQUIVALENT_BASES: tuple[frozenset[str], ...] = (
{
"qwen/qwen-image",
"qwen/qwen-image-2512",
+ # The edit and layered releases republish the same four shards (sha256 compared 2026-10-16).
+ "qwen/qwen-image-edit",
+ "qwen/qwen-image-edit-2509",
+ "qwen/qwen-image-edit-2511",
+ "qwen/qwen-image-layered",
"hunyuanvideo-community/hunyuanimage-2.1-diffusers",
}
),
@@ -284,29 +597,49 @@ _TE_EQUIVALENT_BASES: tuple[frozenset[str], ...] = (
"black-forest-labs/flux-3-image-klein-base",
}
),
- # Qwen3-4B: identical sha256 across the Krea-2 pair (compared 2026-08-25)
+ # Qwen3-VL-4B (Krea's own weights, not FLUX.3's): identical sha256 across the Krea-2 pair (compared 2026-08-25)
frozenset(
{
"krea/krea-2-turbo",
"krea/krea-2-raw",
}
),
+ # T5-XXL: 2 shards, 9,524,648,584 bytes, identical sha256 across every FLUX.1 release (text_encoder_2; Kontext
+ # re-verified 2026-10-16) and HiDream-I1 Full / Dev / Fast, which ship the same bytes as text_encoder_3 (the
+ # family's te_prequant_repos row names the hosted file's component). The CLIP-L files differ between the two, which
+ # is safe only because no CLIP is hosted: a group vouches for the components that have hosted files.
frozenset(
{
"black-forest-labs/flux.1-schnell",
"black-forest-labs/flux.1-dev",
"black-forest-labs/flux.1-krea-dev",
+ "black-forest-labs/flux.1-kontext-dev",
"hidream-ai/hidream-i1-full",
+ "hidream-ai/hidream-i1-dev",
+ "hidream-ai/hidream-i1-fast",
}
),
- # T5-XXL (text_encoder_2): 2 shards, 9,524,648,584 bytes, identical sha256 across every FLUX.1 release; HiDream-I1
- # ships the same bytes as text_encoder_3 (cross-component mapping is not wired yet).
+ # Z-Image's own Qwen3-4B (not stock, unlike FLUX.2 klein 4B's): identical sha256 across Turbo and base
frozenset(
{
"tongyi-mai/z-image-turbo",
"tongyi-mai/z-image",
}
),
+ # Stock Qwen3-4B (= unsloth/Qwen3-4B): the distilled and base FLUX.2 klein 4B repos ship identical shards
+ frozenset(
+ {
+ "black-forest-labs/flux.2-klein-4b",
+ "black-forest-labs/flux.2-klein-base-4b",
+ }
+ ),
+ # Stock Qwen3-8B (= unsloth/Qwen3-8B): likewise for the 9B pair (compared 2026-10-16)
+ frozenset(
+ {
+ "black-forest-labs/flux.2-klein-9b",
+ "black-forest-labs/flux.2-klein-base-9b",
+ }
+ ),
)
@@ -335,6 +668,9 @@ class TePrequantSource:
# Names to try after ``filename``, in order, when the repo does not carry it. Only the "repo"
# kind uses this: a local path either exists or it does not.
fallback_filenames: tuple = ()
+ # The component the hosted FILE was built as, when another family loads the same bytes under a different slot
+ # (HiDream's text_encoder_3 is FLUX.1's text_encoder_2). None = the component it is loaded into.
+ component: Optional[str] = None
def int8_convrot_te_runs_on(target: Any) -> bool:
@@ -396,6 +732,34 @@ def te_prequant_repo_filenames(repo_id: str, component: str, scheme: str) -> tup
return (f"{stem}{SAFETENSORS_SUFFIX}", f"{stem}.pt")
+def te_prequant_companion_names(repo_id: str, families: Iterable[Any]) -> set[str]:
+ """Lowercased root file names a pre-cast text encoder can take in ``repo_id``: every ``te_prequant_repos`` row
+ (three fields, or four with the component the file was built as), every variant row, and the concrete int8
+ ConvRot files. Cache cleanup reads these as companions, not as an installed checkpoint."""
+ pairs: set[tuple[str, str]] = set()
+ for fam in families:
+ for row in getattr(fam, "te_prequant_repos", ()) or ():
+ if len(row) >= 3:
+ pairs.add((str(row[3] if len(row) > 3 else row[1]), str(row[0])))
+ for row in getattr(fam, "te_prequant_variant_repos", ()) or ():
+ if len(row) >= 4:
+ pairs.add((str(row[2]), str(row[1])))
+ pairs |= {(c, s) for c, _s in pairs for s in ("fp8", "int8")}
+ names = {n.lower() for c, s in pairs for n in te_prequant_repo_filenames(repo_id, c, s)}
+ key = str(repo_id or "").strip().lower()
+ entries = [e for comps in TE_INT8_CONVROT_FILES.values() for e in comps.values()]
+ entries += [
+ e
+ for bases in TE_INT8_CONVROT_VARIANT_FILES.values()
+ for comps in bases.values()
+ for e in comps.values()
+ ]
+ for entry in entries:
+ if len(entry) >= 2 and str(entry[0]).strip().lower() == key:
+ names.add(str(entry[1]).lower())
+ return names
+
+
def te_prequant_repo_filename(repo_id: str, component: str, scheme: str) -> str:
"""The preferred checkpoint filename for ``(component, scheme)`` in ``repo_id``."""
return te_prequant_repo_filenames(repo_id, component, scheme)[0]
@@ -439,17 +803,26 @@ def family_te_prequant_repo(fam: Any, scheme: str, component: str) -> Optional[s
Reads the family's ``te_prequant_repos`` (scheme, component, repo_id) triples; the field
is optional on both DiffusionFamily and VideoFamily, so one resolver serves both loaders.
"""
+ entry = _family_te_prequant_entry(fam, scheme, component)
+ return entry[0] if entry else None
+
+
+def _family_te_prequant_entry(fam: Any, scheme: str, component: str) -> Optional[tuple[str, str]]:
+ """``(repo_id, hosted_component)`` of the row for ``(scheme, component)``. A row may carry a fourth element naming
+ the component the hosted file was built as (another family's slot for the same bytes)."""
from .diffusion_nvfp4_flag import nvfp4_blocked
if nvfp4_blocked(scheme):
return None
for entry in getattr(fam, "te_prequant_repos", ()) or ():
try:
- entry_scheme, entry_component, repo_id = entry
+ entry_scheme, entry_component, repo_id, *rest = entry
except Exception: # noqa: BLE001 - a malformed entry must not break the load
continue
+ if len(rest) > 1 or not isinstance(repo_id, str):
+ continue
if entry_scheme == scheme and entry_component == component:
- return repo_id
+ return repo_id, (rest[0] if rest else component)
return None
@@ -474,7 +847,7 @@ def resolve_te_prequant_source(
hosted = family_te_int8_convrot(fam, component)
if hosted is None:
return None
- repo_id, filename = hosted
+ repo_id, filename, *hosted_component = hosted
fp8 = resolve_te_prequant_source(fam, component, "fp8")
from .diffusion_text_encoder_trim import family_trims_lm_head
@@ -483,26 +856,44 @@ def resolve_te_prequant_source(
# The int8 file has no lm_head; with the head kept (UNSLOTH_TE_KEEP_LM_HEAD) only the fp8 one can serve.
return fp8
fallback = ()
- if fp8 is not None and fp8.kind == "repo" and fp8.location.lower() == repo_id.lower():
+ if (
+ fp8 is not None
+ and fp8.kind == "repo"
+ and fp8.location.lower() == repo_id.lower()
+ and not _family_has_torchao_int8(fam)
+ ):
+ # A family with a keep-bf16 int8 schedule (Qwen-Image and its releases, FLUX.2-dev) keeps the on-the-fly
+ # torchao int8 as the fallback for a missing file, as before the file existed; fp8 comes after that.
fallback = te_candidate_filenames(fp8)
return TePrequantSource(
kind = "repo",
location = repo_id,
filename = filename,
fallback_filenames = tuple(n for n in fallback if n != filename),
+ component = hosted_component[0] if hosted_component else None,
)
- repo_id = family_te_prequant_repo(fam, scheme, component)
- if repo_id:
- names = te_prequant_repo_filenames(repo_id, component, scheme)
+ entry = _family_te_prequant_entry(fam, scheme, component)
+ if entry:
+ repo_id, hosted_component = entry
+ names = te_prequant_repo_filenames(repo_id, hosted_component, scheme)
return TePrequantSource(
kind = "repo",
location = repo_id,
filename = names[0],
fallback_filenames = names[1:],
+ component = hosted_component if hosted_component != component else None,
)
return None
+def _family_has_torchao_int8(fam: Any) -> bool:
+ try:
+ from .diffusion_precision import _TE_INT8_SKIP
+ return _TE_INT8_SKIP.get(str(getattr(fam, "name", "") or "").lower()) is not None
+ except Exception: # noqa: BLE001
+ return False
+
+
def te_prequant_sources(
fam: Any,
*,
@@ -529,6 +920,9 @@ def te_prequant_sources(
te_quant_supported,
)
+ from .diffusion_precision import te_quant_picked_automatically
+
+ auto = te_quant_picked_automatically(te_quant_mode)
mode = normalize_te_quant(te_quant_mode)
if mode not in TE_PREQUANT_SCHEMES:
return {}
@@ -545,6 +939,12 @@ def te_prequant_sources(
sources: dict[str, TePrequantSource] = {}
for component in components:
source = resolve_te_prequant_source(fam, component, mode)
+ if source is not None and auto and mode == "int8":
+ # An automatic int8 pick takes the int8 file or the dense encoder, never the fp8 file behind it, nor
+ # the fp8 file put first by a kept lm_head (UNSLOTH_TE_KEEP_LM_HEAD).
+ if source.kind == "repo" and _is_fp8_te_name(source.filename):
+ continue
+ source = replace(source, fallback_filenames = ())
if source is not None and not fp8_runs:
if source.kind == "repo" and _is_fp8_te_name(source.filename):
continue
@@ -572,7 +972,11 @@ def te_prequant_sources_for_base(
the artifact before checkpoint metadata could reject it. Components loaded from their own
standalone repos may map both sides of this comparison through
``standalone_component_bases``.
+
+ A variant base whose encoder is a different model (``te_prequant_variant_repos``) resolves
+ against that variant's rows and files only, never the family base's.
"""
+ fam = te_variant_family_view(fam, base)
sources = te_prequant_sources(
fam,
te_quant_mode = te_quant_mode,
@@ -590,6 +994,32 @@ def te_prequant_sources_for_base(
return compatible
+def te_variant_family_view(fam: Any, base: Optional[str]) -> Any:
+ """``fam`` as seen by a load of ``base``: unchanged unless ``base`` is (equivalent to) a variant listed in the
+ family's ``te_prequant_variant_repos``, in which case a copy whose ``base_repo`` is that variant and whose
+ ``te_prequant_repos`` are that variant's rows. ``family_te_int8_convrot`` keys its variant table on the same base,
+ so the copy can never resolve the family base's encoder for a variant that ships another model."""
+ rows = getattr(fam, "te_prequant_variant_repos", ()) or ()
+ if not rows or not base:
+ return fam
+ registered = str(getattr(fam, "base_repo", "") or "")
+ if registered and te_base_equivalent(registered, base):
+ return fam
+ for variant in dict.fromkeys(str(row[0]) for row in rows if row):
+ if te_base_equivalent(variant, base):
+ import dataclasses
+ import types
+
+ own = tuple(tuple(row[1:]) for row in rows if row and str(row[0]) == variant)
+ try:
+ return dataclasses.replace(fam, base_repo = variant, te_prequant_repos = own)
+ except Exception: # noqa: BLE001 - not a dataclass: no pre-cast encoder rather than the wrong one
+ return types.SimpleNamespace(
+ name = getattr(fam, "name", None), base_repo = variant, te_prequant_repos = own
+ )
+ return fam
+
+
# Weight files a dense encoder folder holds. Everything else (config.json, the shard index, tokenizer JSON) is kept
# when the pre-cast checkpoint replaces the weights: the pre-cast loader still meta-inits from the base repo component
# config.
@@ -700,7 +1130,9 @@ def load_prequant_text_encoder(
if file_scheme != scheme and not (scheme == "int8" and file_scheme == "fp8"):
_warn(logger, scheme, ValueError(f"checkpoint scheme {file_scheme!r} != {scheme!r}"))
return None
- if not _validate_checkpoint(ckpt, file_scheme, component, base, logger):
+ if not _validate_checkpoint(
+ ckpt, file_scheme, getattr(source, "component", None) or component, base, logger
+ ):
return None
state_dict = ckpt["state_dict"]
te_class = (ckpt.get("metadata") or {}).get("te_class")
@@ -910,12 +1342,13 @@ def _te_prequant_pipe_kwargs(
those shards out of the prefetch, so assembly would die on a missing shard instead."""
unavailable: list[str] = []
unbudgeted: list[str] = []
+ built_failed: dict[str, str] = {}
+ refusal: Optional[str] = None
failures: list = []
cause = ""
try:
from .diffusion_precision import TE_QUANT_FP8, normalize_te_quant, te_quant_supported
from .diffusion_text_encoder_trim import family_trims_lm_head
- from .media_locality import _hosted_component_cached
sources = te_prequant_sources_for_base(
fam,
@@ -950,6 +1383,7 @@ def _te_prequant_pipe_kwargs(
if (
encoder is None
and mode == "int8"
+ and not _picked_automatically(te_quant_mode)
and source.kind == "repo"
and fp8_names
and te_quant_supported(target, TE_QUANT_FP8)
@@ -959,9 +1393,8 @@ def _te_prequant_pipe_kwargs(
encoder = load_prequant_text_encoder(
base,
component,
- TePrequantSource(
- kind = "repo",
- location = source.location,
+ replace(
+ source,
filename = fp8_names[0],
fallback_filenames = fp8_names[1:],
),
@@ -974,20 +1407,75 @@ def _te_prequant_pipe_kwargs(
failures = failures,
config_source = config_source,
)
+ if (
+ encoder is None
+ and _picked_automatically(te_quant_mode)
+ and te_prequant_sources_confirmed(
+ {component: source}, local_files_only = local_files_only
+ )
+ ):
+ # The file is there but did not build: the plan priced it, not the dense encoder.
+ built_failed[component] = getattr(source, "filename", None) or str(source.location)
if encoder is not None:
injected[component] = encoder
elif int8_convrot_te_runs_on(target):
# The MPS plan priced this checkpoint, not the dense shards beside it.
unbudgeted.append(component)
cause = f" ({failures[-1]})" if failures else cause
- elif dense_source and not _hosted_component_cached(dense_source, component, "", ""):
+ elif (dense_source or component in built_failed) and not _dense_component_cached(
+ dense_source or base, component
+ ):
+ # Online too for a confirmed automatic file that did not build: the plan left the dense shards out and
+ # priced the file, so fetching the dense encoder now would overrun it.
unavailable.append(component)
cause = f" ({failures[-1]})" if failures else cause
if not unavailable and not unbudgeted:
+ if built_failed and logger is not None:
+ # Dense shards already cached off MPS: the cached-plan path priced max(file, scanned dense), so the
+ # dense encoder is inside the plan.
+ logger.warning(
+ "diffusion.te_prequant: automatic %s %s for %s is present but could not be loaded%s; loading the "
+ "cached dense encoder, which the memory plan covered",
+ mode,
+ ", ".join(built_failed.values()),
+ base,
+ cause,
+ )
+ return injected
+ uncovered = [c for c in unbudgeted + unavailable if c in built_failed]
+ if uncovered:
+ # A confirmed automatic file that fails to build: the memory plan was sized for it (te_prequant_budget_scale,
+ # _precast_text_encoder_mib), and the dense encoder is not covered (not cached, or MPS, where the plan never
+ # takes the dense shards). Loading it anyway would overrun the plan after the resident model was evicted.
+ names = ", ".join(f"{built_failed[c]} ({c})" for c in uncovered)
+ refusal = (
+ f"The hosted pre-quantized text encoder {names} for {base} is present but could not be "
+ f"loaded{cause}. This load was sized for it, so the larger dense encoder is not loaded in its "
+ "place. Set Text encoder precision to Dense (bf16) to load the dense encoder instead, or delete "
+ "the cached file so it is downloaded again."
+ )
+ elif _picked_automatically(te_quant_mode) and _fetch_dense_components(
+ dense_source or base,
+ unavailable + unbudgeted,
+ hf_token = hf_token,
+ local_files_only = local_files_only,
+ logger = logger,
+ ):
+ # Nobody asked for this scheme: the dense encoder loads in its place rather than failing the load.
+ if logger is not None:
+ logger.warning(
+ "diffusion.te_prequant: automatic %s %s for %s could not be loaded%s; loading the dense encoder",
+ mode,
+ ", ".join(unbudgeted + unavailable),
+ base,
+ cause,
+ )
return injected
except Exception as exc: # noqa: BLE001 - injection is an optimisation, never a blocker
_warn(logger, "pipe_kwargs", exc)
return {}
+ if refusal is not None:
+ raise RuntimeError(refusal)
if unbudgeted:
raise RuntimeError(
f"The pre-quantized {', '.join(unbudgeted)} for {base} could not be loaded{cause}. The "
@@ -1002,27 +1490,101 @@ def _te_prequant_pipe_kwargs(
)
-def _held_locally(repo_id: str, name: Optional[str], hf_token: Optional[str]) -> bool:
- """Whether ``repo_id/name`` is on this machine (mirror or Hub cache), without touching the network."""
+def _fetch_dense_components(
+ source: str,
+ components: list,
+ *,
+ hf_token: Optional[str],
+ local_files_only: bool,
+ logger: Any = None,
+) -> bool:
+ """Bring the dense shards of ``components`` the plan left out back into ``source`` (a Hub cache snapshot or repo
+ id) so assembly can load them. True when there is nothing to fetch or the fetch worked; False offline, for a local
+ folder that is not a cache snapshot, or on any failure."""
+ if not components:
+ return True
try:
- if te_prequant_mirror_path(repo_id, name) is not None:
+ if all(_dense_component_cached(source, c) for c in components):
return True
- from huggingface_hub import hf_hub_download
+ except Exception: # noqa: BLE001 - unknown: try the fetch
+ pass
+ if local_files_only:
+ return False
+ try:
+ import os
+
+ from huggingface_hub import snapshot_download
from utils.hf_cache_settings import active_hf_hub_cache
- hf_hub_download(
- repo_id = repo_id,
- filename = name,
+ repo_id, revision, cache_dir = source, None, active_hf_hub_cache()
+ if os.path.isdir(str(source)):
+ parts = os.path.normpath(str(source)).split(os.sep)
+ if len(parts) < 3 or parts[-2] != "snapshots" or not parts[-3].startswith("models--"):
+ return False
+ repo_id = parts[-3][len("models--") :].replace("--", "/")
+ # Into the SAME snapshot assembly opens: its cache root (the secondary one included) and its revision, so
+ # a staged snapshot gains the shards rather than a new one appearing elsewhere.
+ revision, cache_dir = parts[-1], os.sep.join(parts[:-3]) or os.sep
+ fetched = snapshot_download(
+ repo_id,
+ allow_patterns = [f"{c}/*" for c in components],
token = hf_token,
- cache_dir = active_hf_hub_cache(),
- local_files_only = True,
+ cache_dir = cache_dir,
+ revision = revision,
)
+ if revision is not None and os.path.realpath(str(fetched)) != os.path.realpath(str(source)):
+ return False
return True
+ except Exception as exc: # noqa: BLE001 - the strict refusal below still explains the load
+ _warn(logger, "dense_fallback_fetch", exc)
+ return False
+
+
+def _held_locally(repo_id: str, name: Optional[str], hf_token: Optional[str]) -> bool:
+ """Whether ``repo_id/name`` is on this machine, without touching the network: the operator's mirror, or either
+ cache root the loader reuses (the live one and huggingface_hub's import-time one, as _download_checkpoint_name)."""
+ try:
+ if te_prequant_mirror_path(repo_id, name) is not None:
+ return True
+ import types
+
+ from utils.hf_cache_settings import active_hf_hub_cache
+
+ from .diffusion_prequant import _cached_in_root
+
+ source = types.SimpleNamespace(kind = "repo", location = repo_id, filename = name)
+ return any(
+ _cached_in_root(source, root, name) is not None
+ for root in (active_hf_hub_cache(), None)
+ )
except Exception: # noqa: BLE001 - not cached, or unanswerable
return False
+def _dense_component_cached(source: str, component: str) -> bool:
+ """Whether ``component``'s dense shards are on disk for ``source`` (a folder, or a repo id in either cache root)."""
+ from .media_locality import _component_present, _hosted_component_cached
+
+ if _hosted_component_cached(source, component, "", ""):
+ return True
+ import os
+
+ if os.path.isdir(str(source)):
+ return False
+ try:
+ from pathlib import Path
+
+ from huggingface_hub import constants
+
+ repo_dir = Path(constants.HF_HUB_CACHE) / f"models--{str(source).replace('/', '--')}"
+ ref = (repo_dir / "refs" / "main").read_text(encoding = "utf-8").strip()
+ snapshot = repo_dir / "snapshots" / ref
+ return bool(ref) and snapshot.is_dir() and _component_present(snapshot / component)
+ except Exception: # noqa: BLE001 - not cached there
+ return False
+
+
def _build_int8_convrot_encoder(
encoder_cls: Any, config: Any, state_dict: dict, *, dtype: Any, trim: bool
) -> Any:
@@ -1058,7 +1620,8 @@ def _build_int8_convrot_encoder(
# carries no lm_head and a kept one would load random.
if getattr(encoder, "lm_head", None) is not None and not trim:
raise ValueError("the int8 ConvRot encoder carries no lm_head and needs the lm_head trim")
- trim_text_encoder(encoder)
+ if trim:
+ trim_text_encoder(encoder)
if (
getattr(encoder, "lm_head", None) is not None
and not is_trimmed(encoder)
@@ -1114,16 +1677,12 @@ def _build_int8_convrot_encoder(
and not (tensor.dtype == torch.float32 and _kept_fp32(key))
):
state_dict[key] = tensor.to(dtype)
- # A dense block projection would load as bf16 and outgrow the budget this path is priced at. The block stack is
- # model.language_model.layers on a decoder VLM, encoder.block on T5 / UMT5.
- language_model = getattr(getattr(encoder, "model", None), "language_model", None)
- layers = getattr(language_model, "layers", None)
- if layers is None:
- layers = getattr(getattr(encoder, "encoder", None), "block", None)
- if layers is None:
- raise ValueError("encoder has no model.language_model.layers or encoder.block")
+ # A dense block projection would load as bf16 and outgrow the budget this path is priced at (the encoder's own
+ # fp32-kept modules, UMT5's ``wo``, stay dense by design).
dense = [
- n for n, m in layers.named_modules() if isinstance(m, torch.nn.Linear) and not _kept_fp32(n)
+ n
+ for n, m in encoder.named_modules()
+ if isinstance(m, torch.nn.Linear) and TE_INT8_CONVROT_LINEAR.search(n) and not _kept_fp32(n)
]
if dense:
raise ValueError(
@@ -1131,8 +1690,9 @@ def _build_int8_convrot_encoder(
)
_fill_tied_aliases(encoder, state_dict)
encoder.load_state_dict(state_dict, strict = True, assign = True)
- # assign=True gives a tied pair separate tensors; re-tie a headless encoder (UMT5's encoder.embed_tokens ->
- # shared). One with a (trimmed) head is left as loaded: its head is a stand-in with no weight to tie.
+ # assign=True gives a tied pair separate tensors; re-tie a headless encoder (T5 / UMT5's encoder.embed_tokens ->
+ # shared), as the fp8 path does. One with a (trimmed) head is left as loaded: its head is a stand-in with no weight
+ # to tie.
tie = getattr(encoder, "tie_weights", None)
if callable(tie) and getattr(encoder, "lm_head", None) is None:
tie()
diff --git a/studio/backend/core/inference/diffusion_text_encoder_trim.py b/studio/backend/core/inference/diffusion_text_encoder_trim.py
index 9b741845d99..6b58487ff58 100644
--- a/studio/backend/core/inference/diffusion_text_encoder_trim.py
+++ b/studio/backend/core/inference/diffusion_text_encoder_trim.py
@@ -17,10 +17,45 @@ from typing import Any, Optional
KEEP_LM_HEAD_ENV = "UNSLOTH_TE_KEEP_LM_HEAD"
# By class name so this module never imports transformers.
-_HIDDEN_STATE_ONLY_CLASSES = frozenset({"Qwen3VLForConditionalGeneration"})
-
-# FLUX.3 Klein stacks hidden_states[4..32] (diffusers Flux3TextEncoderStep) and never reads logits.
-_HIDDEN_STATE_ONLY_FAMILIES = frozenset({"qwen-image-2.1", "flux.3-klein", "flux.3-klein-base"})
+_HIDDEN_STATE_ONLY_CLASSES = frozenset(
+ {
+ "Qwen3VLForConditionalGeneration",
+ "Qwen3ForCausalLM",
+ "Qwen2_5_VLForConditionalGeneration",
+ "Mistral3ForConditionalGeneration",
+ # HiDream's text_encoder_4: the pipeline stacks its hidden states and never reads logits
+ "LlamaForCausalLM",
+ }
+)
+
+# Every image pipeline below reads encoder hidden states and never logits (diffusers prompt encoders: FLUX.2 klein
+# stacks hidden_states (9, 18, 27), FLUX.2-dev (10, 20, 30), Qwen-Image and its edit / layered releases [-1],
+# HunyuanImage-2.1 [-3], Z-Image and Lumina-2 [-2], Krea-2 a config-driven stack, FLUX.1 T5's last state), which is
+# also what lets their int8 ConvRot files leave the head out. A tied head (klein 4B, Z-Image) is kept by the trim and
+# costs nothing; the headless classes (T5EncoderModel, Qwen3Model, Qwen3VLModel, Gemma2Model) have none to drop.
+_HIDDEN_STATE_ONLY_FAMILIES = frozenset(
+ {
+ "qwen-image-2.1",
+ "flux.2-klein",
+ "flux.2-dev",
+ "qwen-image",
+ "qwen-image-edit",
+ "qwen-image-layered",
+ "hunyuanimage-2.1",
+ "z-image",
+ "krea-2",
+ "lumina-2",
+ "flux.1",
+ "flux.1-kontext",
+ # text_encoder_4 (Llama-3.1-8B): all hidden states stacked, no logits
+ "hidream-i1",
+ # Qwen3VLModel, headless; a hidden-state stack (0, 3, ..., 35)
+ "ideogram-4",
+ # FLUX.3 Klein stacks hidden_states[4..32] (diffusers Flux3TextEncoderStep)
+ "flux.3-klein",
+ "flux.3-klein-base",
+ }
+)
LM_HEAD_KEY = "lm_head.weight"
diff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.py
index f0bc5e709b9..c9c6a0b14a6 100644
--- a/studio/backend/core/training/diffusion_train_common.py
+++ b/studio/backend/core/training/diffusion_train_common.py
@@ -194,10 +194,12 @@ def _component_only_repos() -> dict[str, tuple[str, str, str]]:
components.setdefault(repo, (fam.name, "transformer", str(fam.base_repo)))
else:
bases.add(repo)
- for _scheme, component, repo in getattr(fam, "te_prequant_repos", ()):
+ for row in getattr(fam, "te_prequant_repos", ()):
components.setdefault(
- str(repo).strip().lower(), (fam.name, str(component), str(fam.base_repo))
+ str(row[2]).strip().lower(), (fam.name, str(row[1]), str(fam.base_repo))
)
+ for row in getattr(fam, "te_prequant_variant_repos", ()):
+ components.setdefault(str(row[3]).strip().lower(), (fam.name, str(row[2]), str(row[0])))
return {repo: hit for repo, hit in components.items() if repo not in bases}
diff --git a/studio/backend/hub/utils/companion_assets.py b/studio/backend/hub/utils/companion_assets.py
index 0a87972c883..3cd72d13927 100644
--- a/studio/backend/hub/utils/companion_assets.py
+++ b/studio/backend/hub/utils/companion_assets.py
@@ -185,11 +185,8 @@ def _te_prequant_names(repo_id: str) -> set[str]:
"""Lowercased root file names the loader uses for a pre-cast text encoder hosted in *repo_id*."""
try:
from core.inference.diffusion_families import _FAMILIES
- from core.inference.diffusion_te_prequant import te_prequant_repo_filenames
-
- pairs = {(c, s) for fam in _FAMILIES for s, c, _r in fam.te_prequant_repos}
- pairs |= {(c, s) for c, _s in pairs for s in ("fp8", "int8")}
- return {n.lower() for c, s in pairs for n in te_prequant_repo_filenames(repo_id, c, s)}
+ from core.inference.diffusion_te_prequant import te_prequant_companion_names
+ return te_prequant_companion_names(repo_id, _FAMILIES)
except Exception as exc: # noqa: BLE001 -- no table means no exclusions, as before
logger.debug("te prequant names unavailable: %s", exc)
return set()
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 1b2ee886b3b..f4d1a12ba81 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -49759,6 +49759,8 @@ async def diffusion_download_plan(
gpu_ordinal = gpu_ordinal,
repo_id = request.model_path,
base_repo = request.base_repo,
+ text_encoder_files = request.supplied_text_encoder_files(),
+ gguf_filename = request.gguf_filename,
)
else:
_assert_native_precision_unset(
@@ -50087,6 +50089,9 @@ async def load_diffusion_model_gated(
gpu_ordinal = gpu_ordinal,
repo_id = request.model_path,
base_repo = request.base_repo,
+ local_files_only = not user_initiated,
+ text_encoder_files = request.supplied_text_encoder_files(),
+ gguf_filename = request.gguf_filename,
)
elif fam is not None and pending_name in (ENGINE_SD_CPP, ENGINE_MLX):
# The opt-in MLX engine shares the native contract: one GGUF, no dense-quant knobs.
@@ -50177,6 +50182,9 @@ async def load_diffusion_model_gated(
gpu_ordinal = gpu_ordinal,
repo_id = request.model_path,
base_repo = request.base_repo,
+ local_files_only = not user_initiated,
+ text_encoder_files = request.supplied_text_encoder_files(),
+ gguf_filename = request.gguf_filename,
)
if off_torch is not None:
if activated == ENGINE_SD_CPP:
diff --git a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
index 9f337e94cf4..9cde1a53043 100644
--- a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
+++ b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
@@ -818,7 +818,7 @@
},
"page": "image",
"preview_key": "qwen_image",
- "te_int8_effective": "int8"
+ "te_int8_effective": "fp8"
},
"flux.1": {
"auto": {
@@ -1082,7 +1082,7 @@
},
"page": "image",
"preview_key": "flux2",
- "te_int8_effective": "int8"
+ "te_int8_effective": "fp8"
},
"flux.2-klein": {
"auto": {
@@ -2422,7 +2422,7 @@
},
"page": "image",
"preview_key": "qwen_image",
- "te_int8_effective": "int8"
+ "te_int8_effective": "fp8"
},
"qwen-image-2.1": {
"auto": {
@@ -2616,7 +2616,7 @@
},
"page": "image",
"preview_key": "qwen_image",
- "te_int8_effective": "int8"
+ "te_int8_effective": "fp8"
},
"qwen-image-layered": {
"auto": {
@@ -2710,7 +2710,7 @@
},
"page": "image",
"preview_key": null,
- "te_int8_effective": "int8"
+ "te_int8_effective": "fp8"
},
"sdxl": {
"auto": {
diff --git a/studio/backend/tests/test_comfy_auto_text_encoders.py b/studio/backend/tests/test_comfy_auto_text_encoders.py
index faf52f299fe..7756d29d1a2 100644
--- a/studio/backend/tests/test_comfy_auto_text_encoders.py
+++ b/studio/backend/tests/test_comfy_auto_text_encoders.py
@@ -205,6 +205,36 @@ def test_backend_takes_the_comfy_encoder_for_a_local_dit(tmp_path, monkeypatch):
assert picked == [str(Path(te))]
[email protected]("confirmed, takes_comfy", [(False, True), (True, False)])
+def test_an_automatic_default_takes_the_comfy_encoder_until_its_hosted_file_is_confirmed(
+ tmp_path, monkeypatch, confirmed, takes_comfy
+):
+ from core.inference import diffusion, diffusion_te_prequant as tp
+
+ models = _comfy_tree(tmp_path)
+ te = _encoder_file(models / "text_encoders" / "qwen_2.5_vl_7b_fp8_scaled.safetensors")
+ base = _base(tmp_path)
+ fam = types.SimpleNamespace(
+ name = "qwen-image", single_file_is_pipeline = False, pipeline_only = False, te_quant_auto = "int8"
+ )
+ hosted = {"text_encoder": object()}
+ monkeypatch.setattr(tp, "te_prequant_sources_for_base", lambda *a, **k: hosted)
+ monkeypatch.setattr(tp, "te_prequant_sources_confirmed", lambda sources, **k: confirmed)
+ backend = diffusion.DiffusionBackend.__new__(diffusion.DiffusionBackend)
+ monkeypatch.setattr(backend, "_target_for_ordinal", lambda *a: None, raising = False)
+ picked = diffusion.DiffusionBackend._comfy_text_encoders(
+ backend,
+ fam,
+ "single_file",
+ str(models / "diffusion_models"),
+ "dit.safetensors",
+ base,
+ None,
+ local_files_only = True,
+ )
+ assert picked == ([str(Path(te))] if takes_comfy else [])
+
+
@pytest.mark.parametrize("env, kind", [("0", "single_file"), (None, "pipeline")])
def test_backend_opt_out_and_pipeline_loads_keep_the_base_repo(tmp_path, monkeypatch, env, kind):
picked, _ = _backend_pick(tmp_path, monkeypatch, env = env, kind = kind)
@@ -238,6 +268,8 @@ def test_download_plan_drops_the_base_encoder_a_comfy_file_replaces(tmp_path, mo
model_kind = "single_file",
allow_device_probe = False,
memory_verdict = False,
+ # z-image defaults to its hosted int8 ConvRot encoder, which a bf16 file never displaces; pin the dense one
+ text_encoder_quant = "none",
)
assert "text_encoder" in seen["skip_te_components"]
@@ -270,7 +302,9 @@ def test_the_mlx_plan_keeps_the_base_encoder_it_always_opens(tmp_path, monkeypat
model_kind = "gguf",
allow_device_probe = False,
)
- backend.download_plan(str(models / "diffusion_models"), memory_verdict = False, **pick)
+ backend.download_plan(
+ str(models / "diffusion_models"), memory_verdict = False, text_encoder_quant = "none", **pick
+ )
mb.MlxDiffusionBackend().download_plan(str(models / "diffusion_models"), **pick)
mb.MlxDiffusionBackend().download_plan(
str(models / "diffusion_models"), text_encoder_files = [te], **pick
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index 3de09f54c55..7938def0894 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -917,7 +917,7 @@ def fake_runtime(monkeypatch):
monkeypatch.setattr(
"core.inference.diffusion.load_ideogram4_pipeline",
- lambda repo_id, dtype, hf_token = None, check_cancelled = None: _FakePipe(),
+ lambda repo_id, dtype, hf_token = None, check_cancelled = None, text_encoder = None: _FakePipe(),
)
monkeypatch.setitem(sys.modules, "torch", torch)
@@ -3945,6 +3945,7 @@ def test_pipeline_kind_assembles_krea_and_ideogram_from_the_mirror(fake_runtime,
dtype,
hf_token = None,
check_cancelled = None,
+ text_encoder = None,
):
seen["ideogram"] = repo_id
return _FakePipe()
@@ -13933,21 +13934,31 @@ def test_precast_text_encoder_mib_prices_hidreams_standalone_fourth_encoder(monk
snap.mkdir(parents = True)
with open(snap / te_candidate_filenames(source)[0], "wb") as fh:
fh.truncate(7700 * 1024 * 1024)
+ # text_encoder_3 is FLUX.1's T5-XXL, served from FLUX.1's hosted text_encoder_2 file; cache it too.
+ t5 = te_prequant_sources(
+ fam, te_quant_mode = "fp8", target = target, components = ("text_encoder_3",)
+ )["text_encoder_3"]
+ t5_repo = live / ("models--" + t5.location.replace("/", "--"))
+ (t5_repo / "refs").mkdir(parents = True)
+ (t5_repo / "refs" / "main").write_text(rev)
+ (t5_repo / "snapshots" / rev).mkdir(parents = True)
+ with open(t5_repo / "snapshots" / rev / te_candidate_filenames(t5)[0], "wb") as fh:
+ fh.truncate(5000 * 1024 * 1024)
assert DiffusionBackend._precast_text_encoder_mib(
fam, "HiDream-ai/HiDream-I1-Full", target, "fp8"
- ) == (7700, ("text_encoder_4",), True)
+ ) == (12700, ("text_encoder_3", "text_encoder_4"), True)
+ (t5_repo / "snapshots" / rev / te_candidate_filenames(t5)[0]).unlink()
(snap / te_candidate_filenames(source)[0]).unlink()
from core.inference.diffusion_hidream import HIDREAM_LLAMA_BF16_BYTES
from core.inference.diffusion_te_prequant import TE_PREQUANT_BUDGET_SCALE
- assert DiffusionBackend._precast_text_encoder_mib(
+ mib, components, exact = DiffusionBackend._precast_text_encoder_mib(
fam, "HiDream-ai/HiDream-I1-Full", target, "fp8"
- ) == (
- int(HIDREAM_LLAMA_BF16_BYTES * TE_PREQUANT_BUDGET_SCALE) // (1024 * 1024),
- ("text_encoder_4",),
- False,
)
+ # Neither file is cached now: both are priced from the tables, the Llama from its own bf16 size.
+ assert (components, exact) == (("text_encoder_3", "text_encoder_4"), False)
+ assert mib > int(HIDREAM_LLAMA_BF16_BYTES * TE_PREQUANT_BUDGET_SCALE) // (1024 * 1024)
def test_precast_text_encoder_mib_prices_an_uncached_checkpoint_from_the_family_table(
@@ -14844,7 +14855,9 @@ def test_gguf_offload_swap_on_mps_keeps_the_gguf(fake_runtime, tmp_path, monkeyp
from core.inference.diffusion_transformer_quant import dense_transformer_supported
monkeypatch.setattr(dmod, "dense_transformer_supported", dense_transformer_supported)
- status = _load_gguf(backend, tmp_path)
+ # z-image defaults to its hosted int8 ConvRot encoder, which this offline fixture does not carry; MPS budgets
+ # for that file alone, so the encoder is not what is under test here.
+ status = _load_gguf(backend, tmp_path, text_encoder_quant = "none")
assert calls == []
assert status["transformer_quant"] is None
diff --git a/studio/backend/tests/test_diffusion_fast_load.py b/studio/backend/tests/test_diffusion_fast_load.py
index ee5668db393..ae2214dd9df 100644
--- a/studio/backend/tests/test_diffusion_fast_load.py
+++ b/studio/backend/tests/test_diffusion_fast_load.py
@@ -224,11 +224,13 @@ def test_only_hosted_precast_encoders_leave_the_prefetch(monkeypatch):
fam = detect_family("black-forest-labs/FLUX.1-schnell")
target = types.SimpleNamespace(device = "cuda", backend = "cuda", dtype = torch.bfloat16)
base = "black-forest-labs/FLUX.1-schnell"
- # A runtime cast loads the dense shards first, so they stay in the prefetch.
- for mode in (None, "off", "int8"):
+ # A dense load keeps the dense shards in the prefetch.
+ for mode in (None, "off"):
assert fl.te_precast_components(fam, base, mode, target) == frozenset()
- replaced = fl.te_precast_components(fam, base, "fp8", target)
- assert replaced and replaced <= {"text_encoder_2"}
+ # Both hosted schemes (fp8, and the int8 ConvRot T5) replace only the T5; CLIP-L stays dense.
+ for mode in ("fp8", "int8"):
+ replaced = fl.te_precast_components(fam, base, mode, target)
+ assert replaced and replaced <= {"text_encoder_2"}
assert fl.te_precast_components(object(), base, "fp8", target) == frozenset()
diff --git a/studio/backend/tests/test_diffusion_ideogram4_te_prequant.py b/studio/backend/tests/test_diffusion_ideogram4_te_prequant.py
new file mode 100644
index 00000000000..38dabbd254f
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_ideogram4_te_prequant.py
@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Ideogram 4 takes the hosted pre-cast fp8 text encoder: the per-component branch hands it to the pipeline and
+reads the dense vendor-fp8 encoder only when there is none. CPU only, torch / diffusers stubbed."""
+
+from __future__ import annotations
+
+import dataclasses
+import sys
+import types
+
+import pytest
+
+from core.inference import diffusion as dmod
+from core.inference import diffusion_families as families
+from core.inference import diffusion_ideogram4 as ideogram
+from core.inference.diffusion import DiffusionBackend
+from .test_diffusion_backend import ( # noqa: F401 -- fixtures
+ _FakePipe,
+ _unmeasured_torchao,
+ _write_pipeline,
+ fake_runtime,
+)
+
+_HOSTED = "unsloth/ideogram-4-FP8"
+
+
+def _with_hosted_te(monkeypatch):
+ """The rows the family table carries once the artifact is published."""
+ rows = []
+ for fam in families._FAMILIES:
+ if fam.name == families.IDEOGRAM4_FAMILY_NAME:
+ fam = dataclasses.replace(
+ fam,
+ te_prequant_repos = (("fp8", "text_encoder", _HOSTED),),
+ te_quant_auto = "fp8",
+ )
+ rows.append(fam)
+ monkeypatch.setattr(families, "_FAMILIES", tuple(rows))
+ return next(f for f in rows if f.name == families.IDEOGRAM4_FAMILY_NAME)
+
+
+def _stub_components(monkeypatch, calls, built):
+ def component(name):
+ calls.append(name)
+ return _FakePipe()
+
+ monkeypatch.setattr(dmod, "load_ideogram4_pipeline", ideogram.load_ideogram4_pipeline)
+ monkeypatch.setattr(ideogram, "_patch_create_causal_mask", lambda: None)
+ monkeypatch.setattr(
+ ideogram, "load_ideogram4_text_encoder", lambda *a, **k: component("dense_text_encoder")
+ )
+ monkeypatch.setattr(ideogram, "load_krea2_tokenizer", lambda *a, **k: component("tokenizer"))
+ monkeypatch.setattr(
+ ideogram,
+ "load_ideogram4_transformer",
+ lambda repo, subfolder, *a, **k: component(subfolder),
+ )
+ diffusers = sys.modules["diffusers"]
+
+ def pipeline(**kwargs):
+ built.update(kwargs)
+ return _FakePipe()
+
+ diffusers.Ideogram4Pipeline = pipeline
+ diffusers.AutoencoderKLFlux2 = types.SimpleNamespace(
+ from_pretrained = lambda *a, **k: component("vae")
+ )
+ diffusers.FlowMatchEulerDiscreteScheduler = types.SimpleNamespace(
+ from_pretrained = lambda *a, **k: component("scheduler")
+ )
+
+
[email protected]("precast", [True, False])
+def test_ideogram_branch_takes_precast_text_encoder_else_dense(
+ fake_runtime, tmp_path, monkeypatch, precast
+):
+ calls: list = []
+ built: dict = {}
+ asked: list = []
+ precast_te = object()
+ _stub_components(monkeypatch, calls, built)
+
+ def te_kwargs(fam, base, **kwargs):
+ asked.append((fam.name, kwargs.get("te_quant_mode")))
+ return {"text_encoder": precast_te} if precast else {}
+
+ monkeypatch.setattr(dmod, "te_prequant_pipe_kwargs", te_kwargs)
+ # The stubbed host cannot run the later runtime cast; this test is only about what assembly is handed.
+ from core.inference import diffusion_auto_policy
+
+ monkeypatch.setenv(diffusion_auto_policy._PRECISION_FALLBACK_ENV, "1")
+ _write_pipeline(tmp_path, weight = "pytorch_model.bin")
+ assert DiffusionBackend().load_pipeline(
+ str(tmp_path),
+ family_override = "ideogram-4",
+ local_files_only = True,
+ text_encoder_quant = "fp8",
+ )["loaded"]
+
+ assert asked == [("ideogram-4", "fp8")]
+ if precast:
+ assert built["text_encoder"] is precast_te
+ assert "dense_text_encoder" not in calls
+ else:
+ assert calls.count("dense_text_encoder") == 1
+ assert built["text_encoder"] is not precast_te and built["text_encoder"] is not None
+ assert {"transformer", "unconditional_transformer", "vae", "scheduler"} <= set(calls)
+
+
+def test_ideogram_hosted_te_resolves_for_the_upstream_and_mirror_base(monkeypatch):
+ """The plan / prefetch / injection all key off te_prequant_sources_for_base: the mirror the load fetches from must
+ match the registered upstream base, or the dense shards would be staged and the artifact never read."""
+ from core.inference import diffusion_precision, diffusion_te_prequant as tp
+
+ fam = _with_hosted_te(monkeypatch)
+ monkeypatch.setattr(diffusion_precision, "te_quant_supported", lambda target, mode: True)
+ for base in ("ideogram-ai/ideogram-4-fp8", "unsloth/ideogram-4-fp8"):
+ sources = tp.te_prequant_sources_for_base(fam, base, te_quant_mode = "fp8", target = object())
+ assert set(sources) == {"text_encoder"}
+ assert sources["text_encoder"].location == _HOSTED
+ assert sources["text_encoder"].filename == "ideogram-4-text_encoder-FP8.safetensors"
+ # Dense (none) never resolves a pre-cast source, so the dense shards stay in the plan.
+ assert (
+ tp.te_prequant_sources_for_base(
+ fam, "unsloth/ideogram-4-fp8", te_quant_mode = None, target = object()
+ )
+ == {}
+ )
diff --git a/studio/backend/tests/test_diffusion_image_te_default.py b/studio/backend/tests/test_diffusion_image_te_default.py
new file mode 100644
index 00000000000..dc23f9abf23
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_image_te_default.py
@@ -0,0 +1,1493 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Image families default to a hosted pre-quantized text encoder; ``none`` still pins the bf16 one.
+
+The scheme per family is the ComfyUI template's where ours holds up against the bf16 encoder, else the best hosted
+scheme that does (measured numbers in each family's comment). FLUX.2 klein, HiDream-I1 and Ideogram 4 keep bf16 by
+default (nothing hosted came close enough), SDXL's CLIPs are too small to bother.
+"""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+import core.inference.diffusion_te_prequant as tpq # noqa: E402
+from core.inference.diffusion_families import _FAMILIES # noqa: E402
+from core.inference.diffusion_precision import ( # noqa: E402
+ effective_te_quant,
+ resolve_te_quant_request,
+)
+
+CUDA_BF16 = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)
+
+_T5_INT8 = ("unsloth/FLUX.1-schnell-FP8", "FLUX.1-schnell-text_encoder_2-INT8-ConvRot.safetensors")
+_T5_FP8 = ("unsloth/FLUX.1-schnell-FP8", "FLUX.1-schnell-text_encoder_2-FP8.safetensors")
+_QWEN25VL_INT8 = ("unsloth/Qwen-Image-FP8", "Qwen-Image-text_encoder-INT8-ConvRot.safetensors")
+_QWEN25VL_FP8 = ("unsloth/Qwen-Image-FP8", "Qwen-Image-text_encoder-FP8.safetensors")
+
+# family -> (scheme an unset request resolves to, component, hosted (repo, filename), fp8 (repo, filename) behind it)
+EXPECTED = {
+ "flux.1-kontext": ("fp8", "text_encoder_2", _T5_FP8, None),
+ "flux.2-dev": (
+ "int8",
+ "text_encoder",
+ ("unsloth/FLUX.2-dev-FP8", "FLUX.2-dev-text_encoder-INT8-ConvRot.safetensors"),
+ None,
+ ),
+ # Families with a torchao keep-bf16 int8 schedule have no fp8 file behind the int8 one: a missing file takes the
+ # on-the-fly int8 cast, as before the file existed.
+ "qwen-image": ("int8", "text_encoder", _QWEN25VL_INT8, None),
+ "qwen-image-edit": ("int8", "text_encoder", _QWEN25VL_INT8, None),
+ "qwen-image-layered": ("fp8", "text_encoder", _QWEN25VL_FP8, None),
+ "hunyuanimage-2.1": ("int8", "text_encoder", _QWEN25VL_INT8, _QWEN25VL_FP8),
+ "krea-2": (
+ "int8",
+ "text_encoder",
+ ("unsloth/Krea-2-Turbo-FP8", "Krea-2-Turbo-text_encoder-INT8-ConvRot.safetensors"),
+ ("unsloth/Krea-2-Turbo-FP8", "Krea-2-Turbo-text_encoder-FP8.safetensors"),
+ ),
+}
+# Measured and deliberately left on the bf16 encoder by default: a small encoder (about 5B or less) stays bf16 unless
+# ComfyUI's template quantizes it (FLUX.1's T5, Z-Image, Lumina-2, klein), and HiDream / Ideogram 4 missed the bar.
+KEEP_BF16 = ("flux.1", "z-image", "lumina-2", "flux.2-klein", "hidream-i1", "ideogram-4", "sdxl")
+
+
+def _fam(name):
+ return next(f for f in _FAMILIES if f.name == name)
+
+
[email protected]("name", sorted(EXPECTED))
+def test_unset_request_resolves_to_the_hosted_scheme(name):
+ fam = _fam(name)
+ scheme, component, hosted, fp8 = EXPECTED[name]
+ for value in (None, "", "auto"):
+ assert resolve_te_quant_request(value, fam.te_quant_auto) == (scheme, True)
+ source = tpq.te_prequant_sources_for_base(
+ fam, fam.base_repo, te_quant_mode = scheme, target = CUDA_BF16
+ )[component]
+ assert (source.location, source.filename) == hosted
+ if fp8 is not None:
+ # The int8 file's fallback is the fp8 file in the same repo, never the dense shards.
+ assert source.fallback_filenames[0] == fp8[1]
+ # The pre-load gate asks about fp8 support only: neither hosted file needs torchao.
+ assert effective_te_quant(scheme, name) == "fp8"
+
+
[email protected]("name", sorted(EXPECTED))
[email protected]("value", ["none", "off", "None", "OFF"])
+def test_none_still_pins_the_bf16_encoder(name, value):
+ fam = _fam(name)
+ # Paired with the default: the same family resolves an unset request to a hosted scheme.
+ assert resolve_te_quant_request(None, fam.te_quant_auto)[0] is not None
+ assert resolve_te_quant_request(value, fam.te_quant_auto) == (None, False)
+ assert (
+ tpq.te_prequant_sources_for_base(fam, fam.base_repo, te_quant_mode = None, target = CUDA_BF16)
+ == {}
+ )
+
+
+def test_every_image_family_is_either_defaulted_or_measured_and_kept_bf16():
+ undefaulted = {f.name for f in _FAMILIES if f.te_quant_auto is None}
+ # FLUX.3 Klein and Qwen-Image-2.1 default to int8 ConvRot on their own; everything else is listed above.
+ assert undefaulted == set(KEEP_BF16)
+ for name in KEEP_BF16:
+ assert resolve_te_quant_request(None, _fam(name).te_quant_auto) == (None, False)
+
+
+ "name, base",
+ [
+ ("qwen-image", "Qwen/Qwen-Image-2512"),
+ ("qwen-image-edit", "Qwen/Qwen-Image-Edit"),
+ ("qwen-image-edit", "Qwen/Qwen-Image-Edit-2509"),
+ ("flux.1-kontext", "black-forest-labs/FLUX.1-Kontext-dev"),
+ ("krea-2", "krea/Krea-2-Raw"),
+ ],
+)
+def test_every_release_of_the_same_encoder_gets_the_default_file(name, base):
+ fam = _fam(name)
+ scheme, component, hosted, _fp8 = EXPECTED[name]
+ source = tpq.te_prequant_sources_for_base(fam, base, te_quant_mode = scheme, target = CUDA_BF16)[
+ component
+ ]
+ assert (source.location, source.filename) == hosted
+
+
+# ── an automatic default never fails a load ───────────────────────────────────
+
+MPS_BF16 = types.SimpleNamespace(device = "mps", dtype = torch.bfloat16, backend = "mps")
+
+
+def _missing_file(monkeypatch):
+ """The hosted file cannot be loaded (not uploaded yet, offline, refused) and the dense shards are on disk."""
+ import core.inference.media_locality as ml
+
+ monkeypatch.setattr(tpq, "load_prequant_text_encoder", lambda *a, **k: None)
+ monkeypatch.setattr(tpq, "_held_locally", lambda *a, **k: False)
+ monkeypatch.setattr(ml, "_hosted_component_cached", lambda *a, **k: True)
+
+
[email protected]("name", sorted(n for n, row in EXPECTED.items() if row[0] == "int8"))
+def test_auto_int8_with_the_file_missing_on_mps_loads_dense_without_an_error(name, monkeypatch):
+ _missing_file(monkeypatch)
+ fam = _fam(name)
+ with tpq.auto_te_request(True):
+ out = tpq.te_prequant_pipe_kwargs(
+ fam,
+ fam.base_repo,
+ te_quant_mode = fam.te_quant_auto,
+ target = MPS_BF16,
+ dtype = None,
+ dense_source = "/somewhere/staged",
+ )
+ assert not any(k.startswith("text_encoder") for k in out)
+
+
+def test_explicit_int8_with_the_file_missing_on_mps_is_still_refused(monkeypatch):
+ _missing_file(monkeypatch)
+ fam = _fam("qwen-image")
+ with tpq.auto_te_request(False), pytest.raises(RuntimeError, match = "could not be loaded"):
+ tpq.te_prequant_pipe_kwargs(
+ fam, fam.base_repo, te_quant_mode = "int8", target = MPS_BF16, dtype = None
+ )
+
+
+def test_auto_with_the_dense_shards_left_out_fetches_them_instead_of_failing(monkeypatch, tmp_path):
+ import core.inference.media_locality as ml
+
+ monkeypatch.setattr(tpq, "load_prequant_text_encoder", lambda *a, **k: None)
+ monkeypatch.setattr(tpq, "_held_locally", lambda *a, **k: False)
+ monkeypatch.setattr(ml, "_hosted_component_cached", lambda *a, **k: False)
+ fetched = []
+ import huggingface_hub
+
+ fam = _fam("krea-2")
+ snap = tmp_path / "models--krea--Krea-2-Turbo" / "snapshots" / "abc"
+ snap.mkdir(parents = True)
+ snap = str(snap)
+ monkeypatch.setattr(
+ huggingface_hub,
+ "snapshot_download",
+ lambda repo, **k: fetched.append((repo, k["allow_patterns"])) or snap,
+ )
+ with tpq.auto_te_request(True):
+ out = tpq.te_prequant_pipe_kwargs(
+ fam,
+ fam.base_repo,
+ te_quant_mode = "int8",
+ target = CUDA_BF16,
+ dtype = None,
+ dense_source = snap,
+ )
+ assert out == {} and fetched == [("krea/Krea-2-Turbo", ["text_encoder/*"])]
+ # offline: nothing to fall back to, so the load explains itself as before
+ with tpq.auto_te_request(True), pytest.raises(RuntimeError):
+ tpq.te_prequant_pipe_kwargs(
+ fam,
+ fam.base_repo,
+ te_quant_mode = "int8",
+ target = CUDA_BF16,
+ dtype = None,
+ dense_source = snap,
+ local_files_only = True,
+ )
+
+
[email protected]("device", ["cpu", "xpu", "cuda"])
+def test_auto_on_a_device_that_cannot_run_the_scheme_keeps_the_dense_encoder(device, monkeypatch):
+ import core.inference.diffusion_precision as prec
+
+ monkeypatch.setattr(prec, "te_quant_supported", lambda *a, **k: False)
+ target = types.SimpleNamespace(device = device, dtype = torch.bfloat16)
+ for name in sorted(EXPECTED):
+ fam = _fam(name)
+ # nothing pre-cast is planned or loaded ...
+ assert (
+ tpq.te_prequant_sources_for_base(
+ fam, fam.base_repo, te_quant_mode = fam.te_quant_auto, target = target
+ )
+ == {}
+ )
+ # ... and the in-place pass reports it unsupported instead of raising, leaving the encoder dense
+ pipe = types.SimpleNamespace(text_encoder = torch.nn.Linear(4, 4))
+ out = prec.quantize_text_encoders(pipe, target, mode = fam.te_quant_auto, family = name)
+ assert out.mode is None and pipe.text_encoder.weight.dtype == torch.float32
+
+
+def test_the_auto_flag_follows_the_raw_request():
+ from core.inference.diffusion import _with_supplied_components
+
+ seen = []
+
+ @_with_supplied_components
+ def load(self, repo_id, **kwargs):
+ seen.append(tpq.te_request_is_auto())
+ return {}
+
+ for value, auto in ((None, True), ("auto", True), ("int8", False), ("none", False)):
+ load(object(), "r", text_encoder_quant = value)
+ assert seen.pop() is auto
+ assert tpq.te_request_is_auto() is False
+
+
+# ── explicit int8 with the file missing: never worse than before the file existed ─
+
+
+ "name", ["qwen-image", "qwen-image-edit", "qwen-image-layered", "flux.2-dev"]
+)
+def test_explicit_int8_without_the_file_still_takes_the_torchao_int8_cast(name, monkeypatch):
+ import core.inference.diffusion_precision as prec
+
+ # no fp8 file behind the int8 one: a missing file means the dense encoder, then the old on-the-fly int8
+ src = tpq.resolve_te_prequant_source(_fam(name), "text_encoder", "int8")
+ assert src.filename.endswith("INT8-ConvRot.safetensors") and src.fallback_filenames == ()
+ monkeypatch.setattr(prec, "te_quant_supported", lambda *a, **k: True)
+ monkeypatch.setattr(prec, "torchao_quantize_importable", lambda: True)
+ calls = []
+ monkeypatch.setattr(prec, "_cast_int8_selective", lambda enc, tgt, f, l: calls.append((f, l)))
+ pipe = types.SimpleNamespace(text_encoder = torch.nn.Linear(4, 4))
+ out = prec.quantize_text_encoders(pipe, CUDA_BF16, mode = "int8", family = name)
+ assert out.mode == "int8" and calls == [prec._TE_INT8_SKIP[name]]
+ # without torchao it is the fp8 cast, never a refusal
+ monkeypatch.setattr(prec, "torchao_quantize_importable", lambda: False)
+ fp8 = []
+ monkeypatch.setattr(prec, "_cast_fp8", lambda enc, tgt: fp8.append(enc))
+ out = prec.quantize_text_encoders(pipe, CUDA_BF16, mode = "int8", family = name)
+ assert out.mode == "fp8" and fp8 == [pipe.text_encoder]
+
+
+# ── an automatic int8 default without its file is the dense encoder, never fp8 or torchao ─
+
+
+INT8_DEFAULTS = sorted(n for n, row in EXPECTED.items() if row[0] == "int8")
+
+
[email protected]("name", INT8_DEFAULTS)
+def test_the_automatic_int8_default_never_plans_the_fp8_file_behind_it(name):
+ fam = _fam(name)
+ mode, auto = resolve_te_quant_request(None, fam.te_quant_auto)
+ assert auto and mode == "int8"
+ component = EXPECTED[name][1]
+ # outside a load (download plan, memory budget): the resolved mode itself carries the automatic pick
+ source = tpq.te_prequant_sources_for_base(
+ fam, fam.base_repo, te_quant_mode = mode, target = CUDA_BF16
+ )[component]
+ assert "INT8" in source.filename and source.fallback_filenames == ()
+ # inside a load: the raw request does
+ with tpq.auto_te_request(True):
+ source = tpq.te_prequant_sources_for_base(
+ fam, fam.base_repo, te_quant_mode = "int8", target = CUDA_BF16
+ )[component]
+ assert source.fallback_filenames == ()
+
+
[email protected]("name", INT8_DEFAULTS)
+def test_the_automatic_int8_default_without_its_file_keeps_the_dense_encoder(name, monkeypatch):
+ import core.inference.diffusion_precision as prec
+
+ monkeypatch.setattr(prec, "te_quant_supported", lambda *a, **k: True)
+ monkeypatch.setattr(prec, "torchao_quantize_importable", lambda: True)
+ monkeypatch.setattr(
+ prec, "_cast_fp8", lambda *a, **k: pytest.fail("fp8 cast of an automatic int8 default")
+ )
+ monkeypatch.setattr(
+ prec,
+ "_cast_int8_selective",
+ lambda *a, **k: pytest.fail("torchao cast of an automatic default"),
+ )
+ mode, _ = resolve_te_quant_request(None, _fam(name).te_quant_auto)
+ pipe = types.SimpleNamespace(text_encoder = torch.nn.Linear(4, 4))
+ out = prec.quantize_text_encoders(pipe, CUDA_BF16, mode = mode, family = name)
+ assert out.mode is None and out.status == prec.RESOLVED_FELL_BACK
+ assert pipe.text_encoder.weight.dtype == torch.float32
+
+
+def test_an_automatic_int8_file_that_will_not_build_is_not_retried_as_fp8(monkeypatch):
+ calls = []
+
+ def fake_load(base, component, source, **kwargs):
+ calls.append(source.filename)
+ return None
+
+ monkeypatch.setattr(tpq, "load_prequant_text_encoder", fake_load)
+ monkeypatch.setattr(tpq, "_held_locally", lambda *a, **k: True)
+ import core.inference.media_locality as ml
+
+ monkeypatch.setattr(ml, "_hosted_component_cached", lambda *a, **k: True)
+ fam = _fam("krea-2")
+ with tpq.auto_te_request(True):
+ tpq.te_prequant_pipe_kwargs(
+ fam,
+ fam.base_repo,
+ te_quant_mode = "int8",
+ target = CUDA_BF16,
+ dtype = None,
+ dense_source = "/d",
+ )
+ assert calls == ["Krea-2-Turbo-text_encoder-INT8-ConvRot.safetensors"]
+ calls.clear()
+ # explicit int8 keeps the int8 -> fp8 chain
+ with tpq.auto_te_request(False):
+ tpq.te_prequant_pipe_kwargs(
+ fam,
+ fam.base_repo,
+ te_quant_mode = "int8",
+ target = CUDA_BF16,
+ dtype = None,
+ dense_source = "/d",
+ )
+ assert calls == [
+ "Krea-2-Turbo-text_encoder-INT8-ConvRot.safetensors",
+ "Krea-2-Turbo-text_encoder-FP8.safetensors",
+ ]
+
+
+# ---- memory budget: an automatic default budgets dense until its hosted file is confirmed ------------------------
+
+
+def _budget(
+ monkeypatch,
+ mode,
+ *,
+ held = False,
+ listed = (),
+ offline = False,
+ hub_raises = False,
+):
+ from core.inference.diffusion_precision import AutoTeQuant # noqa: F401
+
+ fam = _fam("qwen-image")
+ tpq._confirmed_cache.clear()
+ monkeypatch.setattr(tpq, "_held_locally", lambda repo, name, token: held)
+ monkeypatch.setattr(tpq, "_hub_offline", lambda: offline)
+ calls = []
+
+ class _Api:
+ def model_info(
+ self,
+ repo,
+ files_metadata = False,
+ ):
+ calls.append(repo)
+ if hub_raises:
+ raise OSError("Hub unreachable")
+ sib = [types.SimpleNamespace(rfilename = n, size = 9_000_000_000) for n in listed]
+ return types.SimpleNamespace(siblings = sib)
+
+ import huggingface_hub
+
+ monkeypatch.setattr(huggingface_hub, "HfApi", lambda *a, **k: _Api())
+ scale = tpq.te_prequant_budget_scale(
+ fam, te_quant_mode = mode, target = CUDA_BF16, base = fam.base_repo
+ )
+ return scale, calls
+
+
+def _auto_int8():
+ from core.inference.diffusion_precision import AutoTeQuant
+ return AutoTeQuant("int8")
+
+
+def test_automatic_default_budgets_dense_while_the_file_is_not_on_the_hub(monkeypatch):
+ scale, calls = _budget(monkeypatch, _auto_int8(), listed = ())
+ assert scale == 1.0
+ assert calls == [_QWEN25VL_INT8[0]]
+
+
+def test_automatic_default_budgets_dense_offline_without_the_file_cached(monkeypatch):
+ scale, calls = _budget(monkeypatch, _auto_int8(), offline = True, listed = (_QWEN25VL_INT8[1],))
+ assert scale == 1.0
+ assert calls == []
+
+
+def test_automatic_default_budgets_dense_when_the_hub_cannot_answer(monkeypatch):
+ scale, _ = _budget(monkeypatch, _auto_int8(), hub_raises = True)
+ assert scale == 1.0
+
+
+def test_automatic_default_budgets_int8_once_the_hub_lists_the_file(monkeypatch):
+ scale, _ = _budget(monkeypatch, _auto_int8(), listed = (_QWEN25VL_INT8[1],))
+ explicit, _ = _budget(monkeypatch, "int8")
+ assert scale == explicit < 1.0
+
+
+def test_automatic_default_budgets_int8_when_the_file_is_cached_without_asking_the_hub(monkeypatch):
+ scale, calls = _budget(monkeypatch, _auto_int8(), held = True, offline = True)
+ assert scale < 1.0
+ assert calls == []
+
+
+def test_automatic_default_inside_a_load_budgets_dense_without_the_file(monkeypatch):
+ with tpq.auto_te_request(True):
+ scale, _ = _budget(monkeypatch, "int8", listed = ())
+ assert scale == 1.0
+
+
+def test_explicit_request_budget_is_unchanged_by_the_confirmation(monkeypatch):
+ scale, calls = _budget(monkeypatch, "int8", listed = ())
+ assert scale < 1.0
+ assert calls == []
+
+
+# ---- review round 1 ---------------------------------------------------------------------------------------------
+
+
[email protected]("name", ["flux.1-kontext", "qwen-image-layered"])
+def test_the_automatic_fp8_default_without_its_file_keeps_the_dense_encoder(name, monkeypatch):
+ import core.inference.diffusion_precision as prec
+
+ monkeypatch.setattr(prec, "te_quant_supported", lambda *a, **k: True)
+ monkeypatch.setattr(
+ prec, "_cast_fp8", lambda *a, **k: pytest.fail("runtime fp8 cast of an automatic default")
+ )
+ mode, auto = resolve_te_quant_request(None, _fam(name).te_quant_auto)
+ assert auto and mode == "fp8"
+ pipe = types.SimpleNamespace(text_encoder = torch.nn.Linear(4, 4))
+ out = prec.quantize_text_encoders(pipe, CUDA_BF16, mode = mode, family = name)
+ assert out.mode is None and out.status == prec.RESOLVED_FELL_BACK
+ with tpq.auto_te_request(True):
+ out = prec.quantize_text_encoders(pipe, CUDA_BF16, mode = "fp8", family = name)
+ assert out.mode is None
+
+
+def test_the_automatic_fp8_default_with_its_file_is_applied(monkeypatch):
+ import core.inference.diffusion_precision as prec
+
+ monkeypatch.setattr(prec, "te_quant_supported", lambda *a, **k: True)
+ cast = []
+ monkeypatch.setattr(prec, "_cast_fp8", lambda enc, tgt: cast.append(enc))
+ encoder = torch.nn.Linear(4, 4)
+ setattr(encoder, tpq.TE_PREQUANT_SCHEME_ATTR, "fp8")
+ mode, _ = resolve_te_quant_request(None, _fam("flux.1-kontext").te_quant_auto)
+ out = prec.quantize_text_encoders(
+ types.SimpleNamespace(text_encoder = encoder), CUDA_BF16, mode = mode, family = "flux.1-kontext"
+ )
+ assert out.mode == "fp8"
+
+
+def test_an_explicit_fp8_request_still_casts_a_dense_encoder(monkeypatch):
+ import core.inference.diffusion_precision as prec
+
+ monkeypatch.setattr(prec, "te_quant_supported", lambda *a, **k: True)
+ cast = []
+ monkeypatch.setattr(prec, "_cast_fp8", lambda enc, tgt: cast.append(enc))
+ out = prec.quantize_text_encoders(
+ types.SimpleNamespace(text_encoder = torch.nn.Linear(4, 4)),
+ CUDA_BF16,
+ mode = "fp8",
+ family = "flux.1-kontext",
+ )
+ assert out.mode == "fp8" and len(cast) == 1
+
+
+def test_companion_cleanup_reads_four_field_rows_and_int8_convrot_files():
+ from hub.utils.companion_assets import _te_prequant_names
+
+ flux1 = _te_prequant_names("unsloth/FLUX.1-schnell-FP8")
+ assert "flux.1-schnell-text_encoder_2-fp8.safetensors" in flux1
+ assert "flux.1-schnell-text_encoder_2-int8-convrot.safetensors" in flux1
+ for repo, filename in (
+ _QWEN25VL_INT8,
+ ("unsloth/FLUX.2-klein-9B-FP8", "FLUX.2-klein-9B-text_encoder-INT8-ConvRot.safetensors"),
+ ("unsloth/HiDream-I1-Full-FP8", "HiDream-I1-Full-text_encoder_4-INT8-ConvRot.safetensors"),
+ ("unsloth/HiDream-I1-Full-FP8", "HiDream-I1-Full-text_encoder_4-FP8.safetensors"),
+ ):
+ assert filename.lower() in _te_prequant_names(repo), filename
+ # names stay per repo: another repo's ConvRot file is not a companion here
+ assert "qwen-image-text_encoder-int8-convrot.safetensors" not in flux1
+
+
+def test_a_repo_holding_only_an_int8_convrot_encoder_is_not_an_installed_checkpoint(monkeypatch):
+ import hub.utils.companion_assets as ca
+
+ repo = types.SimpleNamespace(repo_id = _QWEN25VL_INT8[0])
+ monkeypatch.setattr(ca, "_snapshot_relative_names", lambda r: [_QWEN25VL_INT8[1]])
+ assert ca.repo_holds_denoiser(repo) is False
+
+
+def _hidream_pipe(
+ te3_scheme,
+ te4_scheme = None,
+ te4_runtime = None,
+):
+ from core.inference.diffusion_precision import TE_RUNTIME_SCHEME_ATTR
+
+ te3, te4 = torch.nn.Linear(4, 4), torch.nn.Linear(4, 4)
+ setattr(te3, tpq.TE_PREQUANT_SCHEME_ATTR, te3_scheme)
+ if te4_scheme:
+ setattr(te4, tpq.TE_PREQUANT_SCHEME_ATTR, te4_scheme)
+ if te4_runtime:
+ setattr(te4, TE_RUNTIME_SCHEME_ATTR, te4_runtime)
+ return types.SimpleNamespace(text_encoder_3 = te3, text_encoder_4 = te4)
+
+
+ "te4_scheme, te4_runtime, held",
+ [
+ ("fp8", None, "text_encoder_4 is fp8"),
+ (None, "fp8", "text_encoder_4 is fp8"),
+ (None, None, "dense bf16"),
+ ],
+)
+def test_hidream_reports_a_fourth_encoder_off_the_int8_scheme(
+ monkeypatch, te4_scheme, te4_runtime, held
+):
+ import core.inference.diffusion_precision as prec
+
+ monkeypatch.setattr(prec, "te_quant_supported", lambda *a, **k: True)
+ out = prec.quantize_text_encoders(
+ _hidream_pipe("int8", te4_scheme, te4_runtime), CUDA_BF16, mode = "int8", family = "hidream-i1"
+ )
+ assert out.mode == "int8" and out.status == prec.RESOLVED_FELL_BACK and held in out.reason
+
+
+def test_hidream_int8_on_both_encoders_is_applied(monkeypatch):
+ import core.inference.diffusion_precision as prec
+
+ monkeypatch.setattr(prec, "te_quant_supported", lambda *a, **k: True)
+ out = prec.quantize_text_encoders(
+ _hidream_pipe("int8", "int8"), CUDA_BF16, mode = "int8", family = "hidream-i1"
+ )
+ assert out.mode == "int8" and out.status == prec.RESOLVED_APPLIED
+
+
[email protected]("confirmed, priced", [(False, False), (True, True)])
+def test_mps_automatic_int8_prices_the_file_only_once_it_is_confirmed(
+ monkeypatch, confirmed, priced
+):
+ from core.inference import diffusion_prequant as pq
+ from core.inference.diffusion import DiffusionBackend
+
+ monkeypatch.setattr(pq, "prefer_cached_pickle_twins", lambda repo, names, **k: names)
+ monkeypatch.setattr(pq, "first_cached_as_resolved", lambda *a, **k: None)
+ monkeypatch.setattr(tpq, "te_prequant_sources_confirmed", lambda sources, **k: confirmed)
+ fam = _fam("flux.2-dev")
+ mode, _ = resolve_te_quant_request(None, fam.te_quant_auto)
+ got = DiffusionBackend._precast_text_encoder_mib(fam, fam.base_repo, MPS_BF16, mode)
+ # None leaves the scanned / table dense encoder as the plan's encoder term
+ assert (got is not None) is priced
+ # an explicit request keeps pricing the file it either loads or refuses on
+ assert (
+ DiffusionBackend._precast_text_encoder_mib(fam, fam.base_repo, MPS_BF16, "int8") is not None
+ )
+
+
+# ---- review round 2 ---------------------------------------------------------------------------------------------
+
+
+def test_dense_fallback_fills_the_staged_snapshot_in_its_own_cache_root(monkeypatch, tmp_path):
+ import huggingface_hub
+
+ import core.inference.media_locality as ml
+
+ rev = "a" * 40
+ snap = tmp_path / "secondary" / "models--Qwen--Qwen-Image" / "snapshots" / rev
+ snap.mkdir(parents = True)
+ monkeypatch.setattr(ml, "_hosted_component_cached", lambda *a, **k: False)
+ calls = []
+
+ def fake_download(repo_id, **kwargs):
+ calls.append((repo_id, kwargs))
+ return str(snap)
+
+ monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_download)
+ assert tpq._fetch_dense_components(
+ str(snap), ["text_encoder"], hf_token = None, local_files_only = False
+ )
+ repo_id, kwargs = calls[0]
+ assert repo_id == "Qwen/Qwen-Image"
+ assert kwargs["cache_dir"] == str(tmp_path / "secondary")
+ assert kwargs["revision"] == rev
+ assert kwargs["allow_patterns"] == ["text_encoder/*"]
+
+
+def test_dense_fallback_into_another_snapshot_is_not_reported_as_filled(monkeypatch, tmp_path):
+ import huggingface_hub
+
+ import core.inference.media_locality as ml
+
+ snap = tmp_path / "models--Qwen--Qwen-Image" / "snapshots" / ("b" * 40)
+ snap.mkdir(parents = True)
+ monkeypatch.setattr(ml, "_hosted_component_cached", lambda *a, **k: False)
+ monkeypatch.setattr(
+ huggingface_hub, "snapshot_download", lambda *a, **k: str(tmp_path / "elsewhere")
+ )
+ assert not tpq._fetch_dense_components(
+ str(snap), ["text_encoder"], hf_token = None, local_files_only = False
+ )
+
+
[email protected]("name", INT8_DEFAULTS)
+def test_a_kept_lm_head_keeps_the_automatic_int8_default_dense(name, monkeypatch):
+ import core.inference.diffusion_precision as prec
+ from core.inference.diffusion_text_encoder_trim import KEEP_LM_HEAD_ENV
+
+ monkeypatch.setenv(KEEP_LM_HEAD_ENV, "1")
+ fam = _fam(name)
+ mode, _ = resolve_te_quant_request(None, fam.te_quant_auto)
+ assert (
+ tpq.te_prequant_sources_for_base(fam, fam.base_repo, te_quant_mode = mode, target = CUDA_BF16)
+ == {}
+ )
+ with tpq.auto_te_request(True):
+ assert (
+ tpq.te_prequant_sources_for_base(
+ fam, fam.base_repo, te_quant_mode = "int8", target = CUDA_BF16
+ )
+ == {}
+ )
+ # an explicit int8 still takes the fp8 file, which carries the head
+ explicit = tpq.te_prequant_sources_for_base(
+ fam, fam.base_repo, te_quant_mode = "int8", target = CUDA_BF16
+ )
+ assert all("FP8" in s.filename for s in explicit.values()) and explicit
+ monkeypatch.setattr(prec, "te_quant_supported", lambda *a, **k: True)
+ out = prec.quantize_text_encoders(
+ types.SimpleNamespace(text_encoder = torch.nn.Linear(4, 4)), CUDA_BF16, mode = mode, family = name
+ )
+ assert out.mode is None and KEEP_LM_HEAD_ENV in out.reason
+
+
+MPS_TARGET = types.SimpleNamespace(
+ device = "mps", dtype = torch.bfloat16, backend = "mps", supports_model_cpu_offload = False
+)
+
+
+def _preflight(fam, te_mode):
+ from core.inference.diffusion import DiffusionBackend
+ DiffusionBackend()._assert_precision_for_target(
+ fam,
+ MPS_TARGET,
+ model_kind = "pipeline",
+ pinned = None,
+ te_mode = te_mode,
+ memory_mode = None,
+ cpu_offload = False,
+ base_repo = fam.base_repo,
+ )
+
+
[email protected]("name", ["qwen-image", "flux.2-dev", "krea-2", "z-image"])
+def test_mps_preflight_accepts_explicit_int8_on_a_family_hosting_the_int8_file(name, monkeypatch):
+ monkeypatch.setattr(tpq, "te_prequant_sources_confirmed", lambda sources, **k: True)
+ _preflight(_fam(name), "int8")
+
+
+def test_mps_preflight_still_refuses_explicit_fp8():
+ with pytest.raises(RuntimeError):
+ _preflight(_fam("qwen-image"), "fp8")
+
+
+# ---- review round 3: one answer to "will this explicit int8 load the hosted int8 ConvRot file here" --------------
+
+_DEVICES = {
+ "cuda": CUDA_BF16,
+ "mps": MPS_TARGET,
+ "cpu": types.SimpleNamespace(
+ device = "cpu", dtype = torch.bfloat16, backend = "cpu", supports_model_cpu_offload = False
+ ),
+}
+_HOSTING = sorted(n for n in tpq.TE_INT8_CONVROT_FILES if any(f.name == n for f in _FAMILIES))
+
+
+def _table_env(monkeypatch, device, present, keep_head):
+ import core.inference.diffusion_precision as prec
+ from core.inference.diffusion_text_encoder_trim import KEEP_LM_HEAD_ENV
+
+ if keep_head:
+ monkeypatch.setenv(KEEP_LM_HEAD_ENV, "1")
+ else:
+ monkeypatch.delenv(KEEP_LM_HEAD_ENV, raising = False)
+ import core.inference.diffusion as D
+
+ # fp8 runs on the CUDA card only; MPS and CPU do not. diffusion.py imports the helper by name, so patch both.
+ supported = lambda target, mode: getattr(target, "device", None) == "cuda"
+ monkeypatch.setattr(prec, "te_quant_supported", supported)
+ monkeypatch.setattr(D, "te_quant_supported", supported)
+ monkeypatch.setattr(tpq, "te_prequant_sources_confirmed", lambda sources, **k: present)
+ return _DEVICES[device]
+
+
[email protected]("keep_head", [False, True])
[email protected]("present", [True, False])
[email protected]("device", sorted(_DEVICES))
[email protected]("name", _HOSTING)
+def test_explicit_int8_takes_the_hosted_file_table(name, device, present, keep_head, monkeypatch):
+ target = _table_env(monkeypatch, device, present, keep_head)
+ fam = _fam(name)
+ got = tpq.explicit_int8_takes_hosted_file(fam, fam.base_repo, target)
+ assert got is (device in ("cuda", "mps") and present and not keep_head)
+ import core.inference.diffusion_precision as prec
+
+ checked = prec.preflight_te_quant("int8", fam, fam.base_repo, target)
+ if got:
+ assert checked is None
+ else:
+ assert checked in ("int8", "fp8")
+ # a supplied encoder for a hosted component replaces the file
+ comp = next(iter(tpq.TE_INT8_CONVROT_FILES[name]))
+ assert (
+ tpq.explicit_int8_takes_hosted_file(fam, fam.base_repo, target, supplied = (comp,)) is False
+ )
+
+
+def test_explicit_int8_on_a_custom_base_does_not_take_the_hosted_file(monkeypatch):
+ target = _table_env(monkeypatch, "mps", True, False)
+ assert (
+ tpq.explicit_int8_takes_hosted_file(_fam("qwen-image"), "someone/custom-qwen-image", target)
+ is False
+ )
+
+
[email protected]("present", [True, False])
+def test_mps_preflight_refuses_explicit_int8_unless_the_file_will_load(present, monkeypatch):
+ target = _table_env(monkeypatch, "mps", present, False)
+ from core.inference.diffusion import DiffusionBackend
+
+ fam = _fam("krea-2")
+
+ def run():
+ DiffusionBackend()._assert_precision_for_target(
+ fam,
+ target,
+ model_kind = "pipeline",
+ pinned = None,
+ te_mode = "int8",
+ memory_mode = None,
+ cpu_offload = False,
+ base_repo = fam.base_repo,
+ )
+
+ if present:
+ run()
+ else:
+ with pytest.raises(RuntimeError):
+ run()
+
+
+def test_mps_preflight_refuses_explicit_int8_with_a_kept_lm_head(monkeypatch):
+ target = _table_env(monkeypatch, "mps", True, True)
+ with pytest.raises(RuntimeError):
+ _preflight(_fam("qwen-image"), "int8")
+
+
[email protected]("present", [True, False])
+def test_explicit_int8_under_offload_checks_torchao_when_the_file_will_not_load(
+ present, monkeypatch
+):
+ import core.inference.diffusion as D
+ import core.inference.diffusion_precision as prec
+
+ target = _table_env(monkeypatch, "cuda", present, False)
+ monkeypatch.setattr(prec, "torchao_quantize_importable", lambda: True)
+ monkeypatch.setattr(D, "torchao_quantize_importable", lambda: True)
+ fam = _fam("qwen-image")
+ assert prec.preflight_te_quant("int8", fam, fam.base_repo, target) == (
+ None if present else "int8"
+ )
+
+ def run():
+ D.DiffusionBackend()._assert_precision_for_target(
+ fam,
+ target,
+ model_kind = "pipeline",
+ pinned = None,
+ te_mode = "int8",
+ memory_mode = "balanced",
+ cpu_offload = True,
+ base_repo = fam.base_repo,
+ )
+
+ if present:
+ run()
+ else:
+ with pytest.raises(RuntimeError, match = "offload"):
+ run()
+
+
+def test_hidream_loads_the_hosted_int8_te4_on_mps(monkeypatch):
+ import core.inference.diffusion_hidream as dh
+ import core.inference.diffusion_precision as prec
+ import transformers
+
+ monkeypatch.delenv("UNSLOTH_TE_KEEP_LM_HEAD", raising = False)
+ monkeypatch.setattr(prec, "te_quant_supported", lambda target, mode: False)
+ monkeypatch.setattr(
+ transformers.AutoTokenizer, "from_pretrained", staticmethod(lambda *a, **k: "tok4")
+ )
+ monkeypatch.setattr(
+ transformers.LlamaForCausalLM,
+ "from_pretrained",
+ staticmethod(lambda *a, **k: pytest.fail("dense TE4")),
+ )
+ loaded = []
+ sentinel = object()
+
+ def fake_load(base, component, source, **kwargs):
+ loaded.append((source.filename, kwargs.get("scheme")))
+ return sentinel
+
+ monkeypatch.setattr(tpq, "load_prequant_text_encoder", fake_load)
+ out = dh.hidream_te4_kwargs(
+ None, None, fam = _fam("hidream-i1"), te_quant_mode = "int8", target = MPS_TARGET
+ )
+ assert out["text_encoder_4"] is sentinel
+ assert loaded == [("HiDream-I1-Full-text_encoder_4-INT8-ConvRot.safetensors", "int8")]
+
+
+def _hidream_mps(monkeypatch, *, load):
+ import core.inference.diffusion_precision as prec
+ import transformers
+
+ monkeypatch.delenv("UNSLOTH_TE_KEEP_LM_HEAD", raising = False)
+ monkeypatch.setattr(prec, "te_quant_supported", lambda target, mode: False)
+ monkeypatch.setattr(prec, "_cast_fp8", lambda *a, **k: pytest.fail("fp8 cast on MPS"))
+ monkeypatch.setattr(
+ transformers.AutoTokenizer, "from_pretrained", staticmethod(lambda *a, **k: "tok4")
+ )
+ dense = torch.nn.Linear(2, 2)
+ monkeypatch.setattr(
+ transformers.LlamaForCausalLM, "from_pretrained", staticmethod(lambda *a, **k: dense)
+ )
+ monkeypatch.setattr(tpq, "load_prequant_text_encoder", lambda *a, **k: load)
+ return dense
+
+
+def test_hidream_explicit_int8_on_mps_refuses_when_the_te4_file_fails(monkeypatch):
+ import core.inference.diffusion_hidream as dh
+ _hidream_mps(monkeypatch, load = None)
+ with pytest.raises(RuntimeError, match = "text_encoder_4"):
+ dh.hidream_te4_kwargs(
+ None, None, fam = _fam("hidream-i1"), te_quant_mode = "int8", target = MPS_TARGET
+ )
+
+
+def test_hidream_none_on_mps_still_loads_the_dense_te4(monkeypatch):
+ import core.inference.diffusion_hidream as dh
+
+ dense = _hidream_mps(monkeypatch, load = None)
+ out = dh.hidream_te4_kwargs(
+ None, None, fam = _fam("hidream-i1"), te_quant_mode = None, target = MPS_TARGET
+ )
+ assert out["text_encoder_4"] is dense
+
+
+# ---- review round 4: both cache roots --------------------------------------------------------------------------
+
+
+def _cache_file(
+ root,
+ repo,
+ name,
+ rev = "c" * 40,
+):
+ repo_dir = root / ("models--" + repo.replace("/", "--"))
+ (repo_dir / "refs").mkdir(parents = True, exist_ok = True)
+ (repo_dir / "refs" / "main").write_text(rev)
+ snap = repo_dir / "snapshots" / rev
+ snap.mkdir(parents = True, exist_ok = True)
+ (snap / name).parent.mkdir(parents = True, exist_ok = True)
+ (snap / name).write_bytes(b"x")
+ return snap
+
+
+def _two_roots(monkeypatch, tmp_path):
+ from huggingface_hub import constants
+
+ import utils.hf_cache_settings as hcs
+
+ live, other = tmp_path / "live", tmp_path / "import-time"
+ live.mkdir()
+ other.mkdir()
+ monkeypatch.setattr(hcs, "active_hf_hub_cache", lambda: str(live))
+ monkeypatch.setattr(constants, "HF_HUB_CACHE", str(other))
+ import core.inference.diffusion as D
+
+ monkeypatch.setattr(D, "hub_cache_dir", lambda: str(live))
+ monkeypatch.delenv(tpq.TE_PREQUANT_MIRROR_ENV, raising = False)
+ return live, other
+
+
[email protected]("where", ["live", "import-time"])
+def test_a_hosted_encoder_in_either_cache_root_is_confirmed_offline(where, monkeypatch, tmp_path):
+ live, other = _two_roots(monkeypatch, tmp_path)
+ _cache_file(live if where == "live" else other, *_QWEN25VL_INT8)
+ monkeypatch.setattr(tpq, "_hub_offline", lambda: True)
+ tpq._confirmed_cache.clear()
+ assert tpq._held_locally(*_QWEN25VL_INT8, None)
+ fam = _fam("qwen-image")
+ sources = tpq.te_prequant_sources_for_base(
+ fam, fam.base_repo, te_quant_mode = "int8", target = CUDA_BF16
+ )
+ assert tpq.te_prequant_sources_confirmed(sources)
+
+
+def test_a_hosted_encoder_in_neither_root_is_not_confirmed_offline(monkeypatch, tmp_path):
+ _two_roots(monkeypatch, tmp_path)
+ monkeypatch.setattr(tpq, "_hub_offline", lambda: True)
+ assert not tpq._held_locally(*_QWEN25VL_INT8, None)
+
+
+def test_dense_shards_in_the_import_time_root_count_as_cached(monkeypatch, tmp_path):
+ live, other = _two_roots(monkeypatch, tmp_path)
+ import core.inference.media_locality as ml
+
+ monkeypatch.setattr(ml, "_hosted_component_cached", lambda *a, **k: False)
+ _cache_file(other, "Qwen/Qwen-Image", "text_encoder/model.safetensors")
+ _cache_file(other, "Qwen/Qwen-Image", "text_encoder/config.json")
+ assert tpq._dense_component_cached("Qwen/Qwen-Image", "text_encoder")
+ assert not tpq._dense_component_cached("Qwen/Qwen-Image", "vae")
+
+
+# ---- review round 5 ---------------------------------------------------------------------------------------------
+
+
[email protected]("te4_scheme, te4_runtime", [("fp8", None), (None, "fp8"), (None, None)])
+def test_hidream_fourth_encoder_off_the_scheme_is_partial(monkeypatch, te4_scheme, te4_runtime):
+ import core.inference.diffusion_precision as prec
+
+ monkeypatch.setattr(prec, "te_quant_supported", lambda *a, **k: True)
+ out = prec.quantize_text_encoders(
+ _hidream_pipe("int8", te4_scheme, te4_runtime), CUDA_BF16, mode = "int8", family = "hidream-i1"
+ )
+ assert out.mode == "int8" and out.status == prec.RESOLVED_FELL_BACK and out.partial is True
+
+
+def _failing_build(monkeypatch, *, confirmed, dense_cached, fetched):
+ import huggingface_hub
+
+ import core.inference.media_locality as ml
+
+ monkeypatch.setattr(tpq, "load_prequant_text_encoder", lambda *a, **k: None)
+ monkeypatch.setattr(tpq, "te_prequant_sources_confirmed", lambda sources, **k: confirmed)
+ monkeypatch.setattr(tpq, "_dense_component_cached", lambda *a, **k: dense_cached)
+ monkeypatch.setattr(ml, "_hosted_component_cached", lambda *a, **k: dense_cached)
+ monkeypatch.setattr(
+ huggingface_hub, "snapshot_download", lambda repo, **k: fetched.append(repo) or repo
+ )
+
+
[email protected]("target", [CUDA_BF16, MPS_TARGET], ids = ["cuda", "mps"])
+def test_a_confirmed_automatic_file_that_fails_to_build_refuses_when_dense_is_not_covered(
+ target, monkeypatch
+):
+ fetched = []
+ _failing_build(monkeypatch, confirmed = True, dense_cached = False, fetched = fetched)
+ fam = _fam("krea-2")
+ mode, _ = resolve_te_quant_request(None, fam.te_quant_auto)
+ with pytest.raises(
+ RuntimeError, match = "Krea-2-Turbo-text_encoder-INT8-ConvRot.safetensors.*Dense \\(bf16\\)"
+ ):
+ tpq.te_prequant_pipe_kwargs(
+ fam,
+ fam.base_repo,
+ te_quant_mode = mode,
+ target = target,
+ dtype = None,
+ dense_source = fam.base_repo,
+ )
+ assert fetched == []
+
+
+def test_an_online_load_whose_confirmed_file_fails_to_build_still_refuses_the_dense_fetch(
+ monkeypatch,
+):
+ # Online, assembly passes no dense_source (it may still fetch), but the plan priced the file and left the dense
+ # shards out, so the fallback would download and materialize an encoder the plan never covered.
+ fetched = []
+ _failing_build(monkeypatch, confirmed = True, dense_cached = False, fetched = fetched)
+ fam = _fam("krea-2")
+ mode, _ = resolve_te_quant_request(None, fam.te_quant_auto)
+ with pytest.raises(RuntimeError, match = "Dense \\(bf16\\)"):
+ tpq.te_prequant_pipe_kwargs(
+ fam,
+ fam.base_repo,
+ te_quant_mode = mode,
+ target = CUDA_BF16,
+ dtype = None,
+ dense_source = None,
+ )
+ assert fetched == []
+
+
+def test_a_confirmed_automatic_file_that_fails_to_build_loads_cached_dense_the_plan_covered(
+ monkeypatch,
+):
+ fetched = []
+ _failing_build(monkeypatch, confirmed = True, dense_cached = True, fetched = fetched)
+ fam = _fam("krea-2")
+ mode, _ = resolve_te_quant_request(None, fam.te_quant_auto)
+ out = tpq.te_prequant_pipe_kwargs(
+ fam,
+ fam.base_repo,
+ te_quant_mode = mode,
+ target = CUDA_BF16,
+ dtype = None,
+ dense_source = fam.base_repo,
+ )
+ assert out == {} and fetched == []
+
+
[email protected]("target", [CUDA_BF16, MPS_TARGET], ids = ["cuda", "mps"])
+def test_an_unconfirmed_automatic_file_still_falls_back_to_dense(target, monkeypatch):
+ # the plan budgeted dense for an unconfirmed file, so the dense fetch is inside it
+ fetched = []
+ _failing_build(monkeypatch, confirmed = False, dense_cached = False, fetched = fetched)
+ fam = _fam("krea-2")
+ mode, _ = resolve_te_quant_request(None, fam.te_quant_auto)
+ out = tpq.te_prequant_pipe_kwargs(
+ fam,
+ fam.base_repo,
+ te_quant_mode = mode,
+ target = target,
+ dtype = None,
+ dense_source = fam.base_repo,
+ )
+ assert out == {} and fetched == [fam.base_repo]
+
+
+# te_prequant_budget_scale on the merge base (origin/main 6583ae4de8) for every family with a hosted encoder,
+# request (unset resolved through that tree's own te_quant_auto, fp8, int8) and device; computed by
+# running te_prequant_budget_scale there with fp8 support on CUDA only.
+_ORIGIN_MAIN_BUDGET_SCALE = {
+ "flux.1|fp8|cuda": 0.65,
+ "flux.1|fp8|mps": 1.0,
+ "flux.1|int8|cuda": 1.0,
+ "flux.1|int8|mps": 1.0,
+ "flux.1|unset|cuda": 1.0,
+ "flux.1|unset|mps": 1.0,
+ "flux.2-dev|fp8|cuda": 0.65,
+ "flux.2-dev|fp8|mps": 1.0,
+ "flux.2-dev|int8|cuda": 1.0,
+ "flux.2-dev|int8|mps": 1.0,
+ "flux.2-dev|unset|cuda": 1.0,
+ "flux.2-dev|unset|mps": 1.0,
+ "hidream-i1|fp8|cuda": 1.0,
+ "hidream-i1|fp8|mps": 1.0,
+ "hidream-i1|int8|cuda": 1.0,
+ "hidream-i1|int8|mps": 1.0,
+ "hidream-i1|unset|cuda": 1.0,
+ "hidream-i1|unset|mps": 1.0,
+ "hunyuanimage-2.1|fp8|cuda": 0.65,
+ "hunyuanimage-2.1|fp8|mps": 1.0,
+ "hunyuanimage-2.1|int8|cuda": 1.0,
+ "hunyuanimage-2.1|int8|mps": 1.0,
+ "hunyuanimage-2.1|unset|cuda": 1.0,
+ "hunyuanimage-2.1|unset|mps": 1.0,
+ "krea-2|fp8|cuda": 0.65,
+ "krea-2|fp8|mps": 1.0,
+ "krea-2|int8|cuda": 1.0,
+ "krea-2|int8|mps": 1.0,
+ "krea-2|unset|cuda": 1.0,
+ "krea-2|unset|mps": 1.0,
+ "lumina-2|fp8|cuda": 0.65,
+ "lumina-2|fp8|mps": 1.0,
+ "lumina-2|int8|cuda": 1.0,
+ "lumina-2|int8|mps": 1.0,
+ "lumina-2|unset|cuda": 1.0,
+ "lumina-2|unset|mps": 1.0,
+ "qwen-image-2.1|fp8|cuda": 0.65,
+ "qwen-image-2.1|fp8|mps": 1.0,
+ "qwen-image-2.1|int8|cuda": 0.65,
+ "qwen-image-2.1|int8|mps": 0.56,
+ "qwen-image-2.1|unset|cuda": 0.65,
+ "qwen-image-2.1|unset|mps": 0.56,
+ "qwen-image|fp8|cuda": 0.65,
+ "qwen-image|fp8|mps": 1.0,
+ "qwen-image|int8|cuda": 1.0,
+ "qwen-image|int8|mps": 1.0,
+ "qwen-image|unset|cuda": 1.0,
+ "qwen-image|unset|mps": 1.0,
+ "z-image|fp8|cuda": 0.65,
+ "z-image|fp8|mps": 1.0,
+ "z-image|int8|cuda": 1.0,
+ "z-image|int8|mps": 1.0,
+ "z-image|unset|cuda": 1.0,
+ "z-image|unset|mps": 1.0,
+}
+
+
[email protected]("key", sorted(_ORIGIN_MAIN_BUDGET_SCALE))
+def test_no_pick_budgets_above_origin_main_with_the_file_confirmed(key, monkeypatch):
+ import core.inference.diffusion_precision as prec
+
+ name, req, device = key.split("|")
+ monkeypatch.delenv("UNSLOTH_TE_KEEP_LM_HEAD", raising = False)
+ monkeypatch.setattr(
+ prec, "te_quant_supported", lambda target, mode: getattr(target, "device", None) == "cuda"
+ )
+ monkeypatch.setattr(tpq, "te_prequant_sources_confirmed", lambda sources, **k: True)
+ fam = _fam(name)
+ mode, _ = resolve_te_quant_request(None if req == "unset" else req, fam.te_quant_auto)
+ target = CUDA_BF16 if device == "cuda" else MPS_TARGET
+ scale = tpq.te_prequant_budget_scale(fam, te_quant_mode = mode, target = target, base = fam.base_repo)
+ assert scale <= _ORIGIN_MAIN_BUDGET_SCALE[key] + 1e-9
+
+
+# ---- review round 6 ---------------------------------------------------------------------------------------------
+
+
+def _kontext_pipe():
+ clip, t5 = torch.nn.Linear(4, 4), torch.nn.Linear(4, 4)
+ setattr(t5, tpq.TE_PREQUANT_SCHEME_ATTR, "fp8")
+ return types.SimpleNamespace(text_encoder = clip, text_encoder_2 = t5)
+
+
+def test_the_automatic_fp8_default_leaves_an_unhosted_sibling_bf16(monkeypatch):
+ import core.inference.diffusion_precision as prec
+
+ monkeypatch.setattr(prec, "te_quant_supported", lambda *a, **k: True)
+ monkeypatch.setattr(
+ prec, "_cast_fp8", lambda *a, **k: pytest.fail("runtime fp8 cast under an automatic pick")
+ )
+ mode, auto = resolve_te_quant_request(None, _fam("flux.1-kontext").te_quant_auto)
+ assert auto and mode == "fp8"
+ out = prec.quantize_text_encoders(
+ _kontext_pipe(), CUDA_BF16, mode = mode, family = "flux.1-kontext"
+ )
+ assert (
+ out.mode == "fp8"
+ and out.status == prec.RESOLVED_APPLIED
+ and "text_encoder kept bf16" in out.reason
+ )
+
+
+def test_an_explicit_fp8_still_casts_the_sibling_as_on_origin_main(monkeypatch):
+ import core.inference.diffusion_precision as prec
+
+ monkeypatch.setattr(prec, "te_quant_supported", lambda *a, **k: True)
+ cast = []
+ monkeypatch.setattr(prec, "_cast_fp8", lambda enc, tgt: cast.append(enc))
+ pipe = _kontext_pipe()
+ out = prec.quantize_text_encoders(pipe, CUDA_BF16, mode = "fp8", family = "flux.1-kontext")
+ assert out.mode == "fp8" and pipe.text_encoder in cast
+
+
[email protected]("device", ["cuda", "mps"])
+ "name, sibling", [("hunyuanimage-2.1", 0.8), ("flux.1", 0.25), ("hidream-i1", 3.3)]
+)
+def test_int8_budget_scales_only_the_hosted_encoder(name, sibling, device, monkeypatch):
+ from core.inference.diffusion_auto_policy import family_bf16_components_gb
+
+ target = _table_env(monkeypatch, device, True, False)
+ fam = _fam(name)
+ total = family_bf16_components_gb(fam, fam.base_repo)[1]
+ file_scale = (
+ tpq.te_int8_convrot_budget_scale(fam) if device == "mps" else tpq.TE_PREQUANT_BUDGET_SCALE
+ )
+ scale = tpq.te_prequant_budget_scale(
+ fam, te_quant_mode = "int8", target = target, base = fam.base_repo
+ )
+ assert scale * total == pytest.approx(sibling + (total - sibling) * file_scale)
+
+
+def test_the_uncached_precast_price_leaves_out_the_dense_sibling(monkeypatch):
+ from core.inference import diffusion_prequant as pq
+ from core.inference.diffusion import DiffusionBackend
+ from core.inference.diffusion_auto_policy import family_bf16_components_gb
+
+ _table_env(monkeypatch, "mps", True, False)
+ monkeypatch.setattr(pq, "prefer_cached_pickle_twins", lambda repo, names, **k: names)
+ monkeypatch.setattr(pq, "first_cached_as_resolved", lambda *a, **k: None)
+ fam = _fam("hunyuanimage-2.1")
+ mib, _components, _exact = DiffusionBackend._precast_text_encoder_mib(
+ fam, fam.base_repo, MPS_TARGET, "int8"
+ )
+ hosted = family_bf16_components_gb(fam, fam.base_repo)[1] - 0.8
+ assert mib == int(hosted * 1000**3 * tpq.te_int8_convrot_budget_scale(fam)) // (1024 * 1024)
+
+
+# ---- review round 7 ---------------------------------------------------------------------------------------------
+
+
+def test_hidream_is_the_only_family_with_more_than_one_hosted_int8_encoder():
+ assert [n for n, comps in tpq.TE_INT8_CONVROT_FILES.items() if len(comps) > 1] == ["hidream-i1"]
+
+
+def _hidream_partial(monkeypatch, device, te4_present):
+ target = _table_env(monkeypatch, device, True, False)
+ # TE3's T5 file is confirmed; TE4's Llama file only when te4_present
+ monkeypatch.setattr(
+ tpq,
+ "te_prequant_sources_confirmed",
+ lambda sources, **k: te4_present or "text_encoder_4" not in sources,
+ )
+ return target
+
+
[email protected]("device", ["cuda", "mps"])
+def test_explicit_hidream_int8_with_one_hosted_file_missing_refuses_in_preflight(
+ device, monkeypatch
+):
+ target = _hidream_partial(monkeypatch, device, te4_present = False)
+ fam = _fam("hidream-i1")
+ from core.inference.diffusion import DiffusionBackend
+
+ with pytest.raises(
+ RuntimeError,
+ match = "text_encoder_4 \\(HiDream-I1-Full-text_encoder_4-INT8-ConvRot.safetensors\\).*fp8.*Dense \\(bf16\\)",
+ ):
+ DiffusionBackend()._assert_precision_for_target(
+ fam,
+ target,
+ model_kind = "pipeline",
+ pinned = None,
+ te_mode = "int8",
+ memory_mode = None,
+ cpu_offload = False,
+ base_repo = fam.base_repo,
+ )
+
+
+def test_explicit_hidream_int8_with_both_files_passes_preflight(monkeypatch):
+ target = _hidream_partial(monkeypatch, "mps", te4_present = True)
+ fam = _fam("hidream-i1")
+ import core.inference.diffusion_precision as prec
+
+ assert prec.preflight_te_quant("int8", fam, fam.base_repo, target) is None
+
+
+def test_explicit_hidream_int8_with_neither_file_is_not_a_partial_refusal(monkeypatch):
+ import core.inference.diffusion_precision as prec
+
+ target = _table_env(monkeypatch, "cuda", False, False)
+ fam = _fam("hidream-i1")
+ assert prec.preflight_te_quant("int8", fam, fam.base_repo, target) == "fp8"
+
+
+# ---- review round 8 ---------------------------------------------------------------------------------------------
+
+
+def _hub_lists_everything(monkeypatch):
+ import huggingface_hub
+
+ calls = []
+
+ class _Api:
+ def model_info(
+ self,
+ repo,
+ files_metadata = False,
+ ):
+ calls.append(repo)
+ names = [e[1] for comps in tpq.TE_INT8_CONVROT_FILES.values() for e in comps.values()]
+ return types.SimpleNamespace(
+ siblings = [types.SimpleNamespace(rfilename = n, size = 1) for n in names]
+ )
+
+ monkeypatch.setattr(huggingface_hub, "HfApi", lambda *a, **k: _Api())
+ monkeypatch.setattr(tpq, "_held_locally", lambda *a, **k: False)
+ monkeypatch.setattr(tpq, "_hub_offline", lambda: False)
+ tpq._confirmed_cache.clear()
+ return calls
+
+
+def test_local_files_only_confirms_only_locally_held_files(monkeypatch):
+ calls = _hub_lists_everything(monkeypatch)
+ fam = _fam("qwen-image")
+ sources = tpq.te_prequant_sources_for_base(
+ fam, fam.base_repo, te_quant_mode = "int8", target = CUDA_BF16
+ )
+ assert not tpq.te_prequant_sources_confirmed(sources, local_files_only = True)
+ assert calls == []
+ assert tpq.te_prequant_sources_confirmed(sources, local_files_only = False)
+
+
+def test_local_files_only_is_read_from_the_load_when_not_passed(monkeypatch):
+ from core.inference.diffusion_prequant import _LOCAL_FILES_ONLY
+
+ calls = _hub_lists_everything(monkeypatch)
+ fam = _fam("qwen-image")
+ sources = tpq.te_prequant_sources_for_base(
+ fam, fam.base_repo, te_quant_mode = "int8", target = CUDA_BF16
+ )
+ token = _LOCAL_FILES_ONLY.set(True)
+ try:
+ assert not tpq.te_prequant_sources_confirmed(sources)
+ finally:
+ _LOCAL_FILES_ONLY.reset(token)
+ assert calls == []
+
+
+def test_local_only_preflight_refuses_mps_int8_for_a_file_that_is_not_held(monkeypatch):
+ _hub_lists_everything(monkeypatch)
+ import core.inference.diffusion as D
+ import core.inference.diffusion_precision as prec
+
+ monkeypatch.setattr(
+ prec, "te_quant_supported", lambda target, mode: getattr(target, "device", None) == "cuda"
+ )
+ monkeypatch.setattr(
+ D, "te_quant_supported", lambda target, mode: getattr(target, "device", None) == "cuda"
+ )
+ fam = _fam("krea-2")
+
+ def run(local):
+ D.DiffusionBackend()._assert_precision_for_target(
+ fam,
+ MPS_TARGET,
+ model_kind = "pipeline",
+ pinned = None,
+ te_mode = "int8",
+ memory_mode = None,
+ cpu_offload = False,
+ base_repo = fam.base_repo,
+ local_files_only = local,
+ )
+
+ run(False)
+ with pytest.raises(RuntimeError):
+ run(True)
+
+
+def test_begin_load_and_the_gated_route_pass_local_files_only_to_the_preflight():
+ import inspect
+
+ import core.inference.diffusion as D
+ import routes.inference as R
+
+ src = inspect.getsource(D.DiffusionBackend.begin_load)
+ call = src.split("self.assert_precision_available(", 1)[1].split("with self._lock", 1)[0]
+ assert "local_files_only = local_files_only," in call
+ gated = inspect.getsource(R.load_diffusion_model_gated)
+ calls = gated.split("backend.assert_precision_available,")[1:]
+ assert calls and all(
+ "local_files_only = not user_initiated" in c.split("\n )", 1)[0] for c in calls
+ )
+
+
+def test_explicit_hidream_int8_with_a_supplied_te3_refuses_in_preflight(monkeypatch):
+ import core.inference.diffusion_precision as prec
+
+ target = _table_env(monkeypatch, "cuda", True, False)
+ fam = _fam("hidream-i1")
+ assert tpq.explicit_int8_hosted_components(
+ fam, fam.base_repo, target, supplied = ("text_encoder_3",)
+ ) == {
+ "text_encoder_3": False,
+ "text_encoder_4": True,
+ }
+ with pytest.raises(
+ RuntimeError,
+ match = "supplied text_encoder_3 is not int8.*fp8 or Dense \\(bf16\\).*supply every",
+ ):
+ prec.preflight_te_quant("int8", fam, fam.base_repo, target, supplied = ("text_encoder_3",))
+ # supplying every hosted slot is not a mix
+ assert (
+ prec.preflight_te_quant(
+ "int8", fam, fam.base_repo, target, supplied = ("text_encoder_3", "text_encoder_4")
+ )
+ == "fp8"
+ )
+
+
+def test_the_load_preflight_refuses_a_mixed_hidream_int8_build_before_the_load(monkeypatch):
+ import core.inference.diffusion as D
+
+ target = _table_env(monkeypatch, "cuda", True, False)
+ fam = _fam("hidream-i1")
+ backend = D.DiffusionBackend()
+ monkeypatch.setattr(D, "precision_fallback_allowed", lambda: False, raising = False)
+ monkeypatch.setattr(backend, "_target_for_ordinal", lambda *_a, **_k: target)
+ seen = []
+
+ def planned(_fam, _repo, _base, files, te_mode):
+ seen.append((list(files or ()), te_mode))
+ return ("text_encoder_3",)
+
+ monkeypatch.setattr(backend, "_planned_supplied_text_encoders", planned)
+ # The component overrides are installed only inside the load, so the preflight plans the slots itself.
+ with pytest.raises(RuntimeError, match = "supplied text_encoder_3 is not int8"):
+ backend.assert_precision_available(
+ fam,
+ model_kind = "pipeline",
+ text_encoder_quant = "int8",
+ repo_id = fam.base_repo,
+ text_encoder_files = ["/models/t5xxl.safetensors"],
+ )
+ assert seen == [(["/models/t5xxl.safetensors"], "int8")]
+
+
+def test_supplied_slots_are_planned_only_from_local_files():
+ import core.inference.diffusion as D
+
+ fam = _fam("hidream-i1")
+ backend = D.DiffusionBackend()
+ assert (
+ backend._planned_supplied_text_encoders(fam, fam.base_repo, fam.base_repo, None, "int8")
+ == ()
+ )
+ # A Hub file would need its header fetched: left to the load's own check.
+ assert (
+ backend._planned_supplied_text_encoders(
+ fam, fam.base_repo, fam.base_repo, ["unsloth/x/t5.safetensors"], "int8"
+ )
+ == ()
+ )
+
+
+def test_both_route_preflights_hand_over_the_supplied_files():
+ import inspect
+
+ import routes.inference as R
+
+ src = inspect.getsource(R)
+ calls = src.split("backend.assert_precision_available,")[1:]
+ assert len(calls) == 3
+ assert all(
+ "text_encoder_files = request.supplied_text_encoder_files()," in c.split(")\n", 1)[0]
+ for c in calls
+ )
+
+
+def test_the_int8_preflight_judges_the_variant_base_a_klein_9b_pick_names(monkeypatch):
+ import core.inference.diffusion as D
+
+ fam = _fam("flux.2-klein")
+ backend = D.DiffusionBackend()
+ monkeypatch.setattr(D, "precision_fallback_allowed", lambda: False, raising = False)
+ monkeypatch.setattr(backend, "_target_for_ordinal", lambda *_a, **_k: CUDA_BF16)
+ seen = []
+ monkeypatch.setattr(
+ D, "preflight_te_quant", lambda mode, f, base, target, **k: seen.append(base) or None
+ )
+ backend.assert_precision_available(
+ fam,
+ model_kind = "gguf",
+ text_encoder_quant = "int8",
+ repo_id = "unsloth/FLUX.2-klein-9B-GGUF",
+ gguf_filename = "flux-2-klein-9b-Q4_K_M.gguf",
+ )
+ # The loader derives the 9B companion base from the pick's name, not the family's 4B default.
+ assert seen == ["black-forest-labs/FLUX.2-klein-9B"]
+
+
+def test_hidream_table_share_leaves_out_the_separately_priced_te4(monkeypatch):
+ from core.inference import diffusion_prequant as pq
+ from core.inference.diffusion import DiffusionBackend
+ from core.inference.diffusion_auto_policy import family_bf16_components_gb
+ from core.inference.diffusion_hidream import HIDREAM_LLAMA_BF16_BYTES
+
+ _table_env(monkeypatch, "cuda", True, False)
+ monkeypatch.setattr(pq, "prefer_cached_pickle_twins", lambda repo, names, **k: names)
+ monkeypatch.setattr(pq, "first_cached_as_resolved", lambda *a, **k: None)
+ fam = _fam("hidream-i1")
+ got = DiffusionBackend._precast_text_encoder_mib(fam, fam.base_repo, CUDA_BF16, "fp8")
+ assert got is not None
+ mib, components, _exact = got
+ assert set(components) == {"text_encoder_3", "text_encoder_4"}
+ t5_gb = (
+ family_bf16_components_gb(fam, fam.base_repo)[1] - 3.3 - HIDREAM_LLAMA_BF16_BYTES / 1000**3
+ )
+ expected = int(HIDREAM_LLAMA_BF16_BYTES * tpq.TE_PREQUANT_BUDGET_SCALE) + int(
+ t5_gb * 1000**3 * tpq.TE_PREQUANT_BUDGET_SCALE
+ )
+ # TE4 once, on its own dense size; the table share is T5's alone.
+ assert mib == max(1, expected // (1024 * 1024))
diff --git a/studio/backend/tests/test_diffusion_precision.py b/studio/backend/tests/test_diffusion_precision.py
index ed1756533a0..ac048b124e6 100644
--- a/studio/backend/tests/test_diffusion_precision.py
+++ b/studio/backend/tests/test_diffusion_precision.py
@@ -274,14 +274,32 @@ def test_quantize_int8_uses_family_keep_bf16_schedule(monkeypatch):
monkeypatch.setattr(
dp, "_cast_int8_selective", lambda enc, tgt, first, last: calls.append((enc, first, last))
)
+ # A family with a schedule and no hosted int8 ConvRot file (qwen-image now hosts one, so it never takes this path).
+ monkeypatch.setitem(dp._TE_INT8_SKIP, "scheduled-family", (6, 6))
te = object()
pipe = types.SimpleNamespace(text_encoder = te)
- outcome = quantize_text_encoders(pipe, _target(), mode = "int8", family = "qwen-image")
+ outcome = quantize_text_encoders(pipe, _target(), mode = "int8", family = "scheduled-family")
assert outcome.mode == TE_QUANT_INT8
assert outcome.status == "applied"
assert calls == [(te, 6, 6)]
+def test_int8_never_takes_the_torchao_cast_over_a_pre_cast_encoder(monkeypatch):
+ # qwen-image has a keep-bf16 schedule AND a hosted int8 ConvRot file: an int8 request whose encoder came from a
+ # pre-cast fp8 file is the (idempotent) fp8 cast, never torchao on top of fp8 storage.
+ _stub_torch(monkeypatch, cc = (10, 0))
+ monkeypatch.setattr(dp, "_cast_int8_selective", lambda *a: pytest.fail("torchao int8 cast"))
+ fp8_calls: list = []
+ monkeypatch.setattr(dp, "_cast_fp8", lambda enc, tgt: fp8_calls.append(enc))
+ te = types.SimpleNamespace(_unsloth_te_prequant_scheme = "fp8")
+ outcome = quantize_text_encoders(
+ types.SimpleNamespace(text_encoder = te), _target(), mode = "int8", family = "qwen-image"
+ )
+ assert (outcome.mode, outcome.status) == (TE_QUANT_FP8, "fell_back")
+ assert "int8 ConvRot text encoder is not available" in outcome.reason
+ assert fp8_calls == [te]
+
+
def test_quantize_int8_unknown_family_falls_back_to_fp8(monkeypatch):
# A family without an int8 keep-bf16 schedule falls back to layerwise fp8 (logged), never silent full int8 that would degrade the encoder.
_stub_torch(monkeypatch, cc = (10, 0))
@@ -572,9 +590,10 @@ def test_int8_without_a_schedule_reports_fp8_as_the_effective_mode():
# host whose torchao cannot do int8 while fp8 still works, every unscheduled family died.
assert effective_te_quant(TE_QUANT_INT8, "z-image-turbo") == TE_QUANT_FP8
assert effective_te_quant(TE_QUANT_INT8, None) == TE_QUANT_FP8
- # A family WITH a schedule really does run int8, so the gate must keep asking about int8.
- assert effective_te_quant(TE_QUANT_INT8, "qwen-image") == TE_QUANT_INT8
- assert effective_te_quant(TE_QUANT_INT8, "Flux.2-Dev") == TE_QUANT_INT8
+ # A family hosting an int8 ConvRot file loads it as plain tensors (fp8 behind it), so the gate asks about fp8 even
+ # where a keep-bf16 schedule exists.
+ assert effective_te_quant(TE_QUANT_INT8, "qwen-image") == TE_QUANT_FP8
+ assert effective_te_quant(TE_QUANT_INT8, "Flux.2-Dev") == TE_QUANT_FP8
# Every other mode is its own effective mode, and absent stays absent.
assert effective_te_quant(TE_QUANT_FP8_DYNAMIC, "z-image-turbo") == TE_QUANT_FP8_DYNAMIC
assert effective_te_quant(None, "qwen-image") is None
diff --git a/studio/backend/tests/test_diffusion_routes.py b/studio/backend/tests/test_diffusion_routes.py
index baacf0b21c9..6c0f4f9374b 100644
--- a/studio/backend/tests/test_diffusion_routes.py
+++ b/studio/backend/tests/test_diffusion_routes.py
@@ -2456,10 +2456,11 @@ def test_an_offloading_memory_request_refuses_a_torchao_text_encoder(monkeypatch
# Support and the torchao install are not what is under test here.
monkeypatch.setattr(diffusion_module, "te_quant_supported", lambda target, m: True)
monkeypatch.setattr(diffusion_module, "torchao_quantize_importable", lambda: True)
+ _scheduled_family_without_a_hosted_int8_file(monkeypatch)
with pytest.raises(RuntimeError) as excinfo:
backend.assert_precision_available(
# A family WITH an int8 schedule, so the int8 case is not downgraded to fp8 first.
- types.SimpleNamespace(name = "qwen-image"),
+ types.SimpleNamespace(name = "scheduled-family"),
model_kind = "gguf",
text_encoder_quant = mode,
memory_mode = "low_vram",
@@ -2489,6 +2490,13 @@ def test_layerwise_fp8_survives_an_offloading_memory_request(monkeypatch):
)
+def _scheduled_family_without_a_hosted_int8_file(monkeypatch):
+ """A family whose int8 is the torchao keep-bf16 cast. Every image family with a schedule (qwen-image, its edit and
+ layered releases, FLUX.2-dev) now hosts an int8 ConvRot file, which loads as plain tensors and needs no torchao."""
+ import core.inference.diffusion_precision as precision
+ monkeypatch.setitem(precision._TE_INT8_SKIP, "scheduled-family", (6, 6))
+
+
@pytest.mark.parametrize("mode", ["int8", "fp8_dynamic", "nvfp4"])
def test_a_broken_torchao_refuses_a_torchao_text_encoder_before_the_download(monkeypatch, mode):
"""The casters import torchao only after the pipeline has been downloaded and built, so an
@@ -2505,9 +2513,10 @@ def test_a_broken_torchao_refuses_a_torchao_text_encoder_before_the_download(mon
)
monkeypatch.setattr(diffusion_module, "te_quant_supported", lambda target, m: True)
monkeypatch.setattr(diffusion_module, "torchao_quantize_importable", lambda: False)
+ _scheduled_family_without_a_hosted_int8_file(monkeypatch)
with pytest.raises(RuntimeError) as excinfo:
backend.assert_precision_available(
- types.SimpleNamespace(name = "qwen-image"),
+ types.SimpleNamespace(name = "scheduled-family"),
model_kind = "gguf",
text_encoder_quant = mode,
)
diff --git a/studio/backend/tests/test_diffusion_te_int8_convrot.py b/studio/backend/tests/test_diffusion_te_int8_convrot.py
index a9d2e9abab5..26d3bc585c0 100644
--- a/studio/backend/tests/test_diffusion_te_int8_convrot.py
+++ b/studio/backend/tests/test_diffusion_te_int8_convrot.py
@@ -216,10 +216,7 @@ def test_int8_source_prefers_the_convrot_file_then_the_fp8_one():
fp8 = tpq.te_prequant_sources(fam, te_quant_mode = "fp8", target = CUDA_BF16)["text_encoder"]
assert INT8_NAME not in tpq.te_candidate_filenames(fp8)
# Families without a hosted int8 encoder resolve nothing for int8, as before.
- assert (
- tpq.te_prequant_sources(get_family("qwen-image"), te_quant_mode = "int8", target = CUDA_BF16)
- == {}
- )
+ assert tpq.te_prequant_sources(get_family("sdxl"), te_quant_mode = "int8", target = CUDA_BF16) == {}
def test_keeping_the_lm_head_resolves_the_fp8_file(monkeypatch):
@@ -510,11 +507,38 @@ def _te_gate_mps(monkeypatch, family, te_mode):
@pytest.mark.parametrize("family", ["flux.3-klein", "flux.3-klein-base", "qwen-image-2.1"])
def test_preflight_admits_the_hosted_int8_encoder_on_mps(monkeypatch, family):
# effective_te_quant rewrites int8 to fp8 on a family with no keep-bf16 schedule, and MPS has no fp8, but these
- # families host a plain-tensor int8 ConvRot file that int8_convrot_te_runs_on enables there.
+ # families host a plain-tensor int8 ConvRot file that int8_convrot_te_runs_on enables there. The preflight
+ # confirms the file is published (the Hub listing, stubbed here) before it admits the load.
+ import types
+
+ import huggingface_hub
+
+ import core.inference.diffusion_te_prequant as tpq
+
+ names = [e[1] for comps in tpq.TE_INT8_CONVROT_FILES.values() for e in comps.values()]
+
+ class _Api:
+ def model_info(
+ self,
+ repo,
+ files_metadata = False,
+ ):
+ return types.SimpleNamespace(
+ siblings = [types.SimpleNamespace(rfilename = n, size = 1) for n in names]
+ )
+
+ monkeypatch.setattr(huggingface_hub, "HfApi", lambda *a, **k: _Api())
+ monkeypatch.setattr(tpq, "_held_locally", lambda *a, **k: False)
+ monkeypatch.setattr(tpq, "_hub_offline", lambda: False)
+ tpq._confirmed_cache.clear()
_te_gate_mps(monkeypatch, family, "int8")
def test_preflight_still_refuses_int8_on_mps_without_a_hosted_file(monkeypatch):
+ import core.inference.diffusion_te_prequant as tpq
+
+ # Every family with a hosted encoder now has an int8 file too, so take one away.
+ monkeypatch.delitem(tpq.TE_INT8_CONVROT_FILES, "flux.2-klein")
with pytest.raises(RuntimeError, match = "text_encoder_quant"):
_te_gate_mps(monkeypatch, "flux.2-klein", "int8")
with pytest.raises(RuntimeError, match = "text_encoder_quant"):
diff --git a/studio/backend/tests/test_diffusion_te_prequant.py b/studio/backend/tests/test_diffusion_te_prequant.py
index abf84b57c8d..c175a36077a 100644
--- a/studio/backend/tests/test_diffusion_te_prequant.py
+++ b/studio/backend/tests/test_diffusion_te_prequant.py
@@ -553,9 +553,11 @@ def test_hosted_te_prequant_entries():
"LTX-2-text_encoder-FP8.safetensors",
"LTX-2-text_encoder-FP8.pt",
)
- # HiDream's heavyweight is TE4 (Llama-3.1-8B), engaged via hidream_te4_kwargs since the generic pass only covers text_encoder.._3.
+ # HiDream's heavyweight is TE4 (Llama-3.1-8B), engaged via hidream_te4_kwargs since the generic pass only covers
+ # text_encoder.._3; its text_encoder_3 is FLUX.1's T5-XXL, loaded from FLUX.1's text_encoder_2 file.
assert detect_family("HiDream-ai/HiDream-I1-Full").te_prequant_repos == (
("fp8", "text_encoder_4", "unsloth/HiDream-I1-Full-FP8"),
+ ("fp8", "text_encoder_3", "unsloth/FLUX.1-schnell-FP8", "text_encoder_2"),
)
assert te_prequant_repo_filenames("unsloth/HiDream-I1-Full-FP8", "text_encoder_4", "fp8") == (
"HiDream-I1-Full-text_encoder_4-FP8.safetensors",
@@ -581,8 +583,11 @@ def test_hosted_te_prequant_entries():
assert detect_family("hunyuanvideo-community/HunyuanImage-2.1-Diffusers").te_prequant_repos == (
("fp8", "text_encoder", "unsloth/Qwen-Image-FP8"),
)
- # flux.2-klein-4B hosts NO TE entry: its Qwen3-4B retrained layer 35's MLP, so the z-image artifact must not serve it (maxdiff 0.86).
- assert detect_family("black-forest-labs/FLUX.2-klein-4B").te_prequant_repos == ()
+ # flux.2-klein-4B hosts its OWN entry: it ships stock Qwen3-4B while Z-Image's encoder is a variant of it (layer 35
+ # MLP maxdiff 0.86), so the z-image artifact must not serve it.
+ assert detect_family("black-forest-labs/FLUX.2-klein-4B").te_prequant_repos == (
+ ("fp8", "text_encoder", "unsloth/FLUX.2-klein-4B-FP8"),
+ )
def _hidream_transformers_stub(monkeypatch, recorder):
diff --git a/studio/backend/tests/test_diffusion_te_shared_files.py b/studio/backend/tests/test_diffusion_te_shared_files.py
new file mode 100644
index 00000000000..9b7a84afa49
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_te_shared_files.py
@@ -0,0 +1,478 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Hosted pre-quantized text encoders shared across image families.
+
+Equivalence groups let byte-identical encoders share one hosted file; FLUX.2 klein 9B resolves its own (Qwen3-8B)
+files rather than the family base's Qwen3-4B ones; HiDream's T5 loads FLUX.1's file under another component name; the
+int8 ConvRot loader takes text-only Qwen3 and T5 encoders; a missing hosted file falls back to the dense encoder.
+"""
+
+from __future__ import annotations
+
+import json
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("accelerate")
+
+import core.inference.diffusion_te_prequant as tpq # noqa: E402
+import core.inference.prequant_safetensors as ps # noqa: E402
+from core.inference.diffusion_families import _FAMILIES, detect_family # noqa: E402
+from core.inference.diffusion_prequant import ALLOW_LOCAL_PREQUANT_PATH_ENV # noqa: E402
+
+CUDA_BF16 = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)
+GROUP = 256
+
+
+def _fam(name):
+ return next(f for f in _FAMILIES if f.name == name)
+
+
+ "name, base, component, repo, filename",
+ [
+ # every Qwen2.5-VL-7B release shares Qwen-Image's file
+ ("qwen-image-edit", "Qwen/Qwen-Image-Edit", "text_encoder", "unsloth/Qwen-Image-FP8", None),
+ (
+ "qwen-image-edit",
+ "Qwen/Qwen-Image-Edit-2509",
+ "text_encoder",
+ "unsloth/Qwen-Image-FP8",
+ None,
+ ),
+ (
+ "qwen-image-edit",
+ "Qwen/Qwen-Image-Edit-2511",
+ "text_encoder",
+ "unsloth/Qwen-Image-FP8",
+ None,
+ ),
+ (
+ "qwen-image-layered",
+ "Qwen/Qwen-Image-Layered",
+ "text_encoder",
+ "unsloth/Qwen-Image-FP8",
+ None,
+ ),
+ # every FLUX.1 release shares schnell's T5
+ (
+ "flux.1-kontext",
+ "black-forest-labs/FLUX.1-Kontext-dev",
+ "text_encoder_2",
+ "unsloth/FLUX.1-schnell-FP8",
+ None,
+ ),
+ # HiDream's text_encoder_3 is the same T5, under another slot, for every distillation
+ (
+ "hidream-i1",
+ "HiDream-ai/HiDream-I1-Dev",
+ "text_encoder_3",
+ "unsloth/FLUX.1-schnell-FP8",
+ "FLUX.1-schnell-text_encoder_2-FP8.safetensors",
+ ),
+ (
+ "hidream-i1",
+ "HiDream-ai/HiDream-I1-Fast",
+ "text_encoder_3",
+ "unsloth/FLUX.1-schnell-FP8",
+ "FLUX.1-schnell-text_encoder_2-FP8.safetensors",
+ ),
+ # klein: the base and distilled repos of each size share their encoder; the sizes never share
+ (
+ "flux.2-klein",
+ "black-forest-labs/FLUX.2-klein-base-4B",
+ "text_encoder",
+ "unsloth/FLUX.2-klein-4B-FP8",
+ "FLUX.2-klein-4B-text_encoder-INT8-ConvRot.safetensors",
+ ),
+ (
+ "flux.2-klein",
+ "black-forest-labs/FLUX.2-klein-9B",
+ "text_encoder",
+ "unsloth/FLUX.2-klein-9B-FP8",
+ "FLUX.2-klein-9B-text_encoder-INT8-ConvRot.safetensors",
+ ),
+ (
+ "flux.2-klein",
+ "black-forest-labs/FLUX.2-klein-base-9B",
+ "text_encoder",
+ "unsloth/FLUX.2-klein-9B-FP8",
+ "FLUX.2-klein-9B-text_encoder-INT8-ConvRot.safetensors",
+ ),
+ # the ungated mirror of a gated base is the same encoder
+ (
+ "flux.2-klein",
+ "unsloth/FLUX.2-klein-9B",
+ "text_encoder",
+ "unsloth/FLUX.2-klein-9B-FP8",
+ "FLUX.2-klein-9B-text_encoder-INT8-ConvRot.safetensors",
+ ),
+ ],
+)
+def test_identical_encoders_resolve_the_shared_file_for_their_base(
+ name, base, component, repo, filename
+):
+ fam = _fam(name)
+ sources = tpq.te_prequant_sources_for_base(
+ fam,
+ base,
+ te_quant_mode = "int8" if filename and "INT8" in filename else "fp8",
+ target = CUDA_BF16,
+ )
+ assert component in sources, sources
+ assert sources[component].location == repo
+ if filename is not None:
+ assert sources[component].filename == filename
+
+
+def test_klein_9b_int8_falls_back_to_its_own_fp8_file_never_the_4b_one():
+ fam = _fam("flux.2-klein")
+ nine = tpq.te_prequant_sources_for_base(
+ fam, "black-forest-labs/FLUX.2-klein-9B", te_quant_mode = "int8", target = CUDA_BF16
+ )["text_encoder"]
+ assert nine.fallback_filenames[0] == "FLUX.2-klein-9B-text_encoder-FP8.safetensors"
+ four = tpq.te_prequant_sources_for_base(
+ fam, "black-forest-labs/FLUX.2-klein-4B", te_quant_mode = "int8", target = CUDA_BF16
+ )["text_encoder"]
+ assert four.location == "unsloth/FLUX.2-klein-4B-FP8"
+ assert four.fallback_filenames[0] == "FLUX.2-klein-4B-text_encoder-FP8.safetensors"
+ # explicit fp8 on the 9B: its own file
+ fp8 = tpq.te_prequant_sources_for_base(
+ fam, "black-forest-labs/FLUX.2-klein-base-9B", te_quant_mode = "fp8", target = CUDA_BF16
+ )["text_encoder"]
+ assert (fp8.location, fp8.filename) == (
+ "unsloth/FLUX.2-klein-9B-FP8",
+ "FLUX.2-klein-9B-text_encoder-FP8.safetensors",
+ )
+
+
+ "name, base",
+ [
+ # Z-Image's Qwen3-4B is its own variant, not klein's stock one
+ ("flux.2-klein", "Tongyi-MAI/Z-Image-Turbo"),
+ ("z-image", "black-forest-labs/FLUX.2-klein-4B"),
+ # Qwen-Image-2.1 is Qwen3-VL-8B
+ ("qwen-image-edit", "Qwen/Qwen-Image-2.1"),
+ # an unknown finetune keeps its own dense encoder
+ ("flux.2-klein", "someone/FLUX.2-klein-9B-finetune"),
+ ("qwen-image-edit", "someone/Qwen-Image-Edit-finetune"),
+ ],
+)
+def test_a_different_encoder_never_takes_the_hosted_file(name, base):
+ fam = _fam(name)
+ for scheme in ("fp8", "int8"):
+ assert (
+ tpq.te_prequant_sources_for_base(fam, base, te_quant_mode = scheme, target = CUDA_BF16)
+ == {}
+ )
+
+
+def test_equivalence_groups_hold_the_byte_identical_encoders():
+ eq = tpq.te_base_equivalent
+ for edit in ("Qwen/Qwen-Image-Edit", "Qwen/Qwen-Image-Edit-2509", "Qwen/Qwen-Image-Edit-2511"):
+ assert eq("Qwen/Qwen-Image", edit)
+ assert eq("Qwen/Qwen-Image", "Qwen/Qwen-Image-Layered")
+ assert eq("black-forest-labs/FLUX.1-schnell", "black-forest-labs/FLUX.1-Kontext-dev")
+ assert eq("black-forest-labs/FLUX.1-schnell", "HiDream-ai/HiDream-I1-Dev")
+ assert eq("black-forest-labs/FLUX.2-klein-4B", "black-forest-labs/FLUX.2-klein-base-4B")
+ assert eq("black-forest-labs/FLUX.2-klein-9B", "black-forest-labs/FLUX.2-klein-base-9B")
+ assert not eq("black-forest-labs/FLUX.2-klein-4B", "black-forest-labs/FLUX.2-klein-9B")
+ assert not eq("black-forest-labs/FLUX.2-klein-4B", "Tongyi-MAI/Z-Image-Turbo")
+ assert not eq("Qwen/Qwen-Image-Edit-2511", "Qwen/Qwen-Image-2.1")
+
+
+def test_a_missing_hosted_file_falls_back_to_the_dense_encoder(monkeypatch):
+ """A default nobody asked for must never fail the load: no file -> no injected encoder, and the caller assembles
+ the dense one (dense weights were kept because nothing resolved on the Hub)."""
+ calls = []
+
+ def fake_load(base, component, source, **kwargs):
+ calls.append((component, source.filename, kwargs["scheme"]))
+ return None
+
+ monkeypatch.setattr(tpq, "load_prequant_text_encoder", fake_load)
+ monkeypatch.setattr(tpq, "_held_locally", lambda *a, **k: False)
+ for name, base in (
+ ("qwen-image-edit", "Qwen/Qwen-Image-Edit-2511"),
+ ("flux.2-klein", "black-forest-labs/FLUX.2-klein-9B"),
+ ("flux.1-kontext", "black-forest-labs/FLUX.1-Kontext-dev"),
+ ):
+ fam = _fam(name)
+ calls.clear()
+ out = tpq.te_prequant_pipe_kwargs(
+ fam, base, te_quant_mode = "int8", target = CUDA_BF16, dtype = None
+ )
+ assert {k: v for k, v in out.items() if k.startswith("text_encoder")} == {}
+ assert calls, name
+
+
+def test_klein_reads_hidden_states_only_so_its_qwen3_head_is_trimmed():
+ from core.inference.diffusion_text_encoder_trim import class_trims_lm_head, family_trims_lm_head
+
+ assert family_trims_lm_head("flux.2-klein")
+ assert class_trims_lm_head("Qwen3ForCausalLM")
+ # SDXL's CLIPs have no head and the family is not opted in
+ assert not family_trims_lm_head("sdxl")
+
+
+# ── loading: HiDream's T5 from FLUX.1's text_encoder_2 file ──────────────────
+
+
+def test_hosted_component_name_is_what_the_checkpoint_is_validated_against():
+ src = tpq.resolve_te_prequant_source(_fam("hidream-i1"), "text_encoder_3", "fp8")
+ assert src.component == "text_encoder_2"
+ ckpt = {
+ "format": tpq.TE_PREQUANT_FORMAT,
+ "state_dict": {},
+ "metadata": {
+ "scheme": "fp8",
+ "component": "text_encoder_2",
+ "base_model_id": "black-forest-labs/FLUX.1-schnell",
+ },
+ }
+ assert tpq._validate_checkpoint(ckpt, "fp8", src.component, "HiDream-ai/HiDream-I1-Full", None)
+ # without the row's component the same file is refused for text_encoder_3
+ assert not tpq._validate_checkpoint(
+ ckpt, "fp8", "text_encoder_3", "HiDream-ai/HiDream-I1-Full", None
+ )
+ # and the T5 row never leaks into a slot it does not name
+ assert tpq.resolve_te_prequant_source(_fam("hidream-i1"), "text_encoder_2", "fp8") is None
+ # FLUX.1's own row is unchanged (no hosted-component override)
+ assert tpq.resolve_te_prequant_source(_fam("flux.1"), "text_encoder_2", "fp8").component is None
+
+
+def test_a_four_element_row_is_read_everywhere_a_row_is_unpacked():
+ from core.inference.diffusion_families import (
+ prequant_only_repo_ids,
+ sd_cpp_companion_only_repo_ids,
+ prequant_repo_role,
+ )
+
+ assert "unsloth/flux.2-klein-9b-fp8" in prequant_only_repo_ids()
+ assert "unsloth/flux.2-klein-9b-fp8" not in sd_cpp_companion_only_repo_ids()
+ bases, _schemes, te = prequant_repo_role(_fam("flux.2-klein"), "unsloth/FLUX.2-klein-9B-FP8")
+ assert te == ("fp8",)
+ assert bases == ("black-forest-labs/FLUX.2-klein-9B",)
+
+
+# ── loading: a text-only Qwen3 int8 ConvRot file (klein) ──────────────────────
+
+
+def _qwen3_config(tied):
+ from transformers import Qwen3Config
+
+ config = Qwen3Config(
+ hidden_size = 256,
+ intermediate_size = 512,
+ num_hidden_layers = 2,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ head_dim = 128,
+ vocab_size = 64,
+ tie_word_embeddings = tied,
+ )
+ config._attn_implementation = "sdpa"
+ return config
+
+
+def _int8_file(tmp_path, encoder, config):
+ """scripts/build_te_int8_convrot_checkpoint.py's layout, built with its own regex."""
+ import importlib.util
+ from pathlib import Path
+
+ from safetensors.torch import save_file
+
+ script = Path(__file__).resolve().parents[3] / "scripts" / "build_te_int8_convrot_checkpoint.py"
+ spec = importlib.util.spec_from_file_location("build_te_int8", script)
+ builder = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(builder)
+ blob = torch.tensor(
+ list(
+ json.dumps(
+ {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": GROUP}
+ ).encode()
+ ),
+ dtype = torch.uint8,
+ )
+ state = {}
+ for k, v in encoder.state_dict().items():
+ if k == "lm_head.weight":
+ continue
+ if builder.DECODER_LINEAR.match(k):
+ q, s = tpq.quantize_int8_convrot_weight(v, GROUP)
+ state[k], state[k[:-7] + ".weight_scale"] = q, s
+ state[k[:-7] + ".comfy_quant"] = blob.clone()
+ else:
+ state[k] = v.detach().clone()
+ assert sum(k.endswith(".weight_scale") for k in state) == 7 * config.num_hidden_layers
+ meta = {
+ ps.UNSLOTH_FORMAT_KEY: tpq.TE_PREQUANT_FORMAT_INT8_CONVROT,
+ ps.UNSLOTH_METADATA_KEY: json.dumps(
+ {
+ "scheme": "int8",
+ "component": "text_encoder",
+ "base_model_id": "black-forest-labs/FLUX.2-klein-4B",
+ "te_class": "Qwen3ForCausalLM",
+ }
+ ),
+ "tensor_names": json.dumps(sorted(state)),
+ **{n: json.dumps({"_type": "Tensor"}) for n in state},
+ }
+ path = tmp_path / "klein-int8.safetensors"
+ save_file({k: v.contiguous() for k, v in state.items()}, str(path), metadata = meta)
+ return path
+
+
[email protected]("tied", [True, False], ids = ["qwen3-4b-tied", "qwen3-8b-untied"])
+def test_text_only_qwen3_int8_convrot_file_loads_and_tracks_the_dense_hidden_states(
+ tmp_path, monkeypatch, tied
+):
+ import transformers
+ from transformers import Qwen3ForCausalLM
+
+ from core.inference.diffusion_text_encoder_trim import is_trimmed
+
+ config = _qwen3_config(tied)
+ torch.manual_seed(3)
+ dense = Qwen3ForCausalLM(config).eval()
+ path = _int8_file(tmp_path, dense, config)
+ monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)
+ monkeypatch.setenv(ALLOW_LOCAL_PREQUANT_PATH_ENV, str(tmp_path))
+ monkeypatch.setattr(
+ transformers.AutoConfig, "from_pretrained", lambda *_a, **_k: _qwen3_config(tied)
+ )
+ failures: list = []
+ encoder = tpq.load_prequant_text_encoder(
+ "black-forest-labs/FLUX.2-klein-4B",
+ "text_encoder",
+ tpq.TePrequantSource(kind = "path", location = str(path)),
+ dtype = torch.float32,
+ scheme = "int8",
+ local_files_only = True,
+ trim_lm_head = True,
+ failures = failures,
+ )
+ assert encoder is not None, failures
+ assert getattr(encoder, tpq.TE_PREQUANT_SCHEME_ATTR) == "int8"
+ assert is_trimmed(encoder)
+ assert not any(isinstance(m, torch.nn.Linear) for m in encoder.model.layers.modules())
+ ids = torch.tensor([[1, 5, 9, 13, 17, 21]])
+ with torch.no_grad():
+ ref = dense(input_ids = ids, output_hidden_states = True).hidden_states
+ got = encoder(input_ids = ids, output_hidden_states = True).hidden_states
+ for a, b in zip(ref[1:], got[1:]):
+ cos = torch.nn.functional.cosine_similarity(a.flatten(1), b.flatten(1)).min()
+ assert cos > 0.99
+
+
+def test_klein_int8_file_has_the_fp8_file_behind_it():
+ fam = detect_family("black-forest-labs/FLUX.2-klein-4B")
+ source = tpq.resolve_te_prequant_source(fam, "text_encoder", "int8")
+ assert source.filename == "FLUX.2-klein-4B-text_encoder-INT8-ConvRot.safetensors"
+ assert source.fallback_filenames[0] == "FLUX.2-klein-4B-text_encoder-FP8.safetensors"
+
+
+# ── loading: a T5 int8 ConvRot file (FLUX.1, HiDream text_encoder_3) ──────────
+
+
+def test_t5_int8_convrot_file_loads_with_its_tied_embedding_and_tracks_dense(tmp_path, monkeypatch):
+ """T5EncoderModel has no head and ties encoder.embed_tokens to shared; every projection, wo included, is int8."""
+ import transformers
+ from safetensors.torch import save_file
+ from transformers import T5Config, T5EncoderModel
+
+ def config():
+ c = T5Config(
+ d_model = 256,
+ d_ff = 512,
+ d_kv = 64,
+ num_heads = 4,
+ num_layers = 2,
+ vocab_size = 64,
+ feed_forward_proj = "gated-gelu",
+ )
+ c._attn_implementation = "eager"
+ return c
+
+ torch.manual_seed(1)
+ dense = T5EncoderModel(config()).eval()
+ blob = torch.tensor(
+ list(
+ json.dumps(
+ {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": GROUP}
+ ).encode()
+ ),
+ dtype = torch.uint8,
+ )
+ linears = {
+ n
+ for n, m in dense.named_modules()
+ if isinstance(m, torch.nn.Linear) and tpq.TE_INT8_CONVROT_LINEAR.search(n)
+ }
+ assert len(linears) == 2 * 7 # q, k, v, o, wi_0, wi_1, wo per block
+ state = {}
+ for k, v in dense.state_dict().items():
+ prefix = k[:-7] if k.endswith(".weight") else None
+ if prefix in linears:
+ q, s = tpq.quantize_int8_convrot_weight(v, GROUP)
+ state[k], state[prefix + ".weight_scale"], state[prefix + ".comfy_quant"] = (
+ q,
+ s,
+ blob.clone(),
+ )
+ else:
+ state[k] = v.detach().clone()
+ meta = {
+ ps.UNSLOTH_FORMAT_KEY: tpq.TE_PREQUANT_FORMAT_INT8_CONVROT,
+ ps.UNSLOTH_METADATA_KEY: json.dumps(
+ {
+ "scheme": "int8",
+ "component": "text_encoder_2",
+ "base_model_id": "black-forest-labs/FLUX.1-schnell",
+ "te_class": "T5EncoderModel",
+ }
+ ),
+ "tensor_names": json.dumps(sorted(state)),
+ **{n: json.dumps({"_type": "Tensor"}) for n in state},
+ }
+ path = tmp_path / "t5-int8.safetensors"
+ save_file({k: v.contiguous() for k, v in state.items()}, str(path), metadata = meta)
+ monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)
+ monkeypatch.setenv(ALLOW_LOCAL_PREQUANT_PATH_ENV, str(tmp_path))
+ monkeypatch.setattr(transformers.AutoConfig, "from_pretrained", lambda *_a, **_k: config())
+ failures: list = []
+ # Loaded into HiDream's text_encoder_3 slot from a file built as FLUX.1's text_encoder_2.
+ encoder = tpq.load_prequant_text_encoder(
+ "HiDream-ai/HiDream-I1-Full",
+ "text_encoder_3",
+ tpq.TePrequantSource(kind = "path", location = str(path), component = "text_encoder_2"),
+ dtype = torch.float32,
+ scheme = "int8",
+ local_files_only = True,
+ failures = failures,
+ )
+ assert encoder is not None, failures
+ assert encoder.encoder.embed_tokens.weight is encoder.shared.weight
+ ids = torch.tensor([[3, 9, 27, 5, 1]])
+ with torch.no_grad():
+ ref = dense(input_ids = ids).last_hidden_state
+ got = encoder(input_ids = ids).last_hidden_state
+ assert torch.nn.functional.cosine_similarity(ref.flatten(1), got.flatten(1)).min() > 0.99
+
+
+def test_int8_leaves_the_small_companion_encoders_bf16(monkeypatch):
+ """FLUX.1's CLIP-L next to its hosted int8 T5: never cast in place (a runtime-only conversion), reported applied."""
+ import core.inference.diffusion_precision as prec
+
+ monkeypatch.setattr(prec, "te_quant_supported", lambda *a, **k: True)
+ monkeypatch.setattr(prec, "_cast_fp8", lambda *a, **k: pytest.fail("CLIP cast in place"))
+ t5 = torch.nn.Linear(4, 4)
+ setattr(t5, tpq.TE_PREQUANT_SCHEME_ATTR, "int8")
+ pipe = types.SimpleNamespace(text_encoder = torch.nn.Linear(4, 4), text_encoder_2 = t5)
+ out = prec.quantize_text_encoders(pipe, CUDA_BF16, mode = "int8", family = "flux.1")
+ assert (out.mode, out.status) == ("int8", prec.RESOLVED_APPLIED)
+ assert "text_encoder kept bf16" in out.reason
diff --git a/studio/backend/tests/test_diffusion_text_encoder_trim.py b/studio/backend/tests/test_diffusion_text_encoder_trim.py
index 5e85d820bfe..3568313f015 100644
--- a/studio/backend/tests/test_diffusion_text_encoder_trim.py
+++ b/studio/backend/tests/test_diffusion_text_encoder_trim.py
@@ -107,7 +107,7 @@ def test_trim_keeps_every_hidden_state_bit_identical_and_drops_the_head():
def test_trim_is_idempotent_and_gated():
encoder = _encoder()
- assert trim_text_encoder(encoder, family = "qwen-image")["lm_head"] == "kept"
+ assert trim_text_encoder(encoder, family = "sdxl")["lm_head"] == "kept"
assert not is_trimmed(encoder)
assert trim_text_encoder(encoder, family = "qwen-image-2.1")["lm_head"] == "dropped"
assert trim_text_encoder(encoder, family = "qwen-image-2.1") == {
@@ -236,7 +236,7 @@ def test_pipe_kwargs_asks_the_loader_to_trim_only_for_hidden_state_families(monk
"load_prequant_text_encoder",
lambda *a, **k: seen.append(k.get("trim_lm_head")) or object(),
)
- for name in ("qwen-image-2.1", "qwen-image"):
+ for name in ("qwen-image-2.1", "sdxl"):
tpq.te_prequant_pipe_kwargs(
types.SimpleNamespace(name = name),
"base",
diff --git a/studio/backend/tests/test_image_fetch_ids_unsloth.py b/studio/backend/tests/test_image_fetch_ids_unsloth.py
index c1991d9f1da..5973ab2fa02 100644
--- a/studio/backend/tests/test_image_fetch_ids_unsloth.py
+++ b/studio/backend/tests/test_image_fetch_ids_unsloth.py
@@ -137,7 +137,7 @@ def _family_fetch_ids() -> list[tuple[str, str, str, str]]:
for base, _scheme, repo in fam.prequant_variant_repos:
add(fam, "prequant_variant_base", base)
add(fam, "prequant_variant", repo)
- for _scheme, _component, repo in fam.te_prequant_repos:
+ for _scheme, _component, repo, *_built_as in fam.te_prequant_repos:
add(fam, "te_prequant", repo)
for repo in fam.train_base_repos:
add(fam, "train_base", repo)
diff --git a/studio/backend/tests/test_video_te_quant_default.py b/studio/backend/tests/test_video_te_quant_default.py
index 2581393ae44..2c2e65b0699 100644
--- a/studio/backend/tests/test_video_te_quant_default.py
+++ b/studio/backend/tests/test_video_te_quant_default.py
@@ -436,6 +436,39 @@ def test_the_builder_quantizes_a_non_gated_t5_feed_forward():
assert not pattern.match("encoder.block.3.layer.1.DenseReluDense.wix.weight")
+
+def test_the_transformers_builder_honours_keep_dense(tmp_path):
+ import importlib.util
+ from pathlib import Path
+
+ import torch
+ import transformers
+ from safetensors import safe_open
+
+ path = Path(__file__).resolve().parents[3] / "scripts" / "build_te_int8_convrot_checkpoint.py"
+ spec = importlib.util.spec_from_file_location("build_te_int8_convrot_checkpoint", path)
+ builder = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(builder)
+ config = transformers.T5Config(
+ vocab_size = 32, d_model = 64, d_kv = 16, d_ff = 128, num_layers = 1, num_heads = 4,
+ feed_forward_proj = "gated-gelu",
+ )
+ src = tmp_path / "te"
+ transformers.T5EncoderModel(config).save_pretrained(src)
+ out = tmp_path / "te-int8.safetensors"
+ builder.main([
+ "--base", "someone/t5", "--family", "flux.1", "--src", str(src), "--out", str(out),
+ "--load", "transformers", "--te-class", "T5EncoderModel", "--keep-dense", "wo",
+ "--group", "64", "--device", "cpu",
+ ])
+ with safe_open(str(out), framework = "pt") as handle:
+ keys = set(handle.keys())
+ wo = handle.get_tensor("encoder.block.0.layer.1.DenseReluDense.wo.weight")
+ assert "encoder.block.0.layer.1.DenseReluDense.wi_0.weight_scale" in keys
+ assert "encoder.block.0.layer.1.DenseReluDense.wo.weight_scale" not in keys
+ # T5 keeps ``wo`` in float32, and a dense one is stored that way.
+ assert wo.dtype == torch.float32
+
def test_a_wan_load_claims_the_hosted_encoder_repo_while_it_fetches():
# The hosted file lives in a third repo; without the claim the delete-cached guard admits deleting it mid-load.
import inspect