Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: load the ComfyUI-format twin of a hosted image prequant, and run ComfyUI fp8 on the fp8 path (#12820)

Daniel Hanb4aa7b1d2e2e
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index b578b8fcd0f..db9886be6cd 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -275,6 +275,7 @@ from .diffusion_denoiser_prequant import (
prequant_artifact_label,
)
from .diffusion_comfy_quant import (
+ comfy_fp8_backend,
comfy_int8_backend,
comfy_torchao_quantized,
load_comfy_quant_transformer,
@@ -6734,7 +6735,8 @@ class DiffusionBackend:
_install_gguf_prefix_strip(transformer_cls, logger)
_install_gguf_dim_restore(logger)
if comfy_scan is not None:
- # int8 codes and scales go to the int8 runtime unchanged where it runs; the rest dequantize.
+ # int8 / fp8 codes go to Studio's runtime unchanged where it runs; the rest dequantize.
+ _comfy_offload = not plan_keeps_transformer_resident(plan)
transformer = load_comfy_quant_transformer(
transformer_cls,
single_file_path,
@@ -6744,10 +6746,17 @@ class DiffusionBackend:
target,
fam.name,
base,
- offload = not plan_keeps_transformer_resident(plan),
+ offload = _comfy_offload,
+ ),
+ fp8_backend = comfy_fp8_backend(
+ target,
+ fam.name,
+ base,
+ offload = _comfy_offload,
),
family = fam.name,
target = target,
+ fast_accum = transformer_quant_fast_accum,
logger = logger,
)
comfy_compile = comfy_torchao_quantized(transformer)
@@ -7766,6 +7775,27 @@ class DiffusionBackend:
except Exception: # noqa: BLE001 - status only
return None
+ @staticmethod
+ def _comfy_single_file_resident_mib(
+ single_file_path: Optional[str], fam: Any, target: Any, base: Optional[str]
+ ) -> Optional[int]:
+ """``comfy_resident_mib`` for a ComfyUI-quantized single file under a resident plan, else None."""
+ try:
+ from .diffusion_comfy_quant import comfy_resident_mib, scan_comfy_quant
+
+ scan = scan_comfy_quant(single_file_path)
+ if scan is None or scan.problems:
+ return None
+ name = getattr(fam, "name", None)
+ return comfy_resident_mib(
+ single_file_path,
+ scan,
+ keep_int8 = comfy_int8_backend(target, name, base) is not None,
+ keep_fp8 = comfy_fp8_backend(target, name, base) is not None,
+ )
+ except Exception: # noqa: BLE001 - a planning aid: the file-size estimate stands
+ return None
+
@staticmethod
def _gguf_offload_prequant_placement(
replanned: Any,
@@ -7905,6 +7935,7 @@ class DiffusionBackend:
cache_dir = hub_cache_dir(),
logger = logger,
placement_device = None if seed_device == device else seed_device,
+ family = fam.name,
)
check_cancelled()
if transformer is None:
@@ -8848,6 +8879,13 @@ class DiffusionBackend:
transformer_resident = estimate_safetensors_dense_mib(
file_size_mib(single_file_path), fp8_upcast = fp8_upcast
)
+ if not getattr(fam, "single_file_is_pipeline", False):
+ # Priced from the header: layers a resident runtime keeps at stored size, dequantized ones at 2x.
+ _comfy_mib = self._comfy_single_file_resident_mib(
+ single_file_path, fam, target, base
+ )
+ if _comfy_mib is not None:
+ transformer_resident = _comfy_mib
else:
transformer_resident = estimate_gguf_resident_mib(file_size_mib(single_file_path))
# Companions (VAE + text encoders) load near on-disk size; sum the base-repo cache, or a LOCAL base's
diff --git a/studio/backend/core/inference/diffusion_comfy_quant.py b/studio/backend/core/inference/diffusion_comfy_quant.py
index dbced0aa206..990929afa1e 100644
--- a/studio/backend/core/inference/diffusion_comfy_quant.py
+++ b/studio/backend/core/inference/diffusion_comfy_quant.py
@@ -3,38 +3,20 @@
"""Load a ComfyUI-format quantized single-file DiT, or refuse it by name.
-ComfyUI's quantized checkpoints keep the quantized codes under the layer's own ``.weight`` key and put
-the dequant parameters beside it: ``.weight_scale`` (and for some formats ``.input_scale``,
-``.weight_scale_2``, ...). The format of each layer is declared in one of three ways:
-
- per layer ``<layer>.comfy_quant``: a uint8 tensor holding JSON, ``{"format": "int8_tensorwise",
- "convrot": true, "convrot_groupsize": 256}``
- whole file the safetensors header's ``_quantization_metadata``: ``{"layers": {<layer>: {...}}}``
- legacy a ``scaled_fp8`` marker tensor plus ``<layer>.scale_weight`` / ``.scale_input``
-
-A stock ``from_single_file`` knows none of this: it casts the int8 / fp8 codes to the compute dtype
-as if they were weights and reports the scales as unused keys, so the load succeeds and every
-quantized layer is off by its scale (pure noise for int8). This module reads the declaration from the
-header, before a weight byte is loaded, and either
-
-- maps ``int8_tensorwise`` layers (optionally ConvRot-rotated) into Studio's own int8 runtime with the
- codes and per-row scales unchanged: the same ``Int8Tensor`` and ``ConvRotLinear`` a hosted
- INT8 / INT8-ConvRot checkpoint rebuilds into,
-- or dequantizes them (``codes * scale``, then the ConvRot rotation undone) where that runtime is not
- available or the layer is one Studio keeps in bf16, and dequantizes fp8 layers
- (``float8_e4m3fn`` / ``float8_e5m2``, scalar or per-row scale) on load,
-- and refuses anything else (nvfp4, mxfp8, the 4/6-bit int8 packings, an unknown or missing format,
- scales with no declaration) with an error naming the format, never a silent mis-load.
-
-The row bookkeeping is generic: the family's own single-file converter runs once, with every int8
-weight replaced by a float64 tensor whose rows are tagged ``(source key, row)``. Whatever splits,
-concatenations and renames the converter applies, the tags that come out say which source rows each
-diffusers weight is made of, and the codes and scales are gathered by row from the file. A converter
-that splices columns of different layers, changes the column count, casts or transforms values
-leaves tags that do not decode, and the load is refused. (The tags are constant along a row, so a
-pure reordering of input columns would go unseen; no diffusers converter reorders a Linear's inputs.)
-
-Reading the declaration is torch-free.
+ComfyUI keeps quantized codes under each layer's ``.weight`` with ``.weight_scale`` (some formats also
+``.input_scale``, ...) beside it, the format declared per layer (``<layer>.comfy_quant`` JSON tensor), in
+the header's ``_quantization_metadata`` table, or by the legacy ``scaled_fp8`` marker + ``.scale_weight``.
+Stock ``from_single_file`` casts the codes as weights and drops the scales: the load succeeds and renders noise.
+
+From the header alone: int8_tensorwise (optionally ConvRot) maps into Studio's int8 runtime with codes and
+per-row scales unchanged; float8_e4m3fn into Studio's per-row fp8 runtime (scalar scale repeated per row);
+where neither runs, or for layers Studio keeps in bf16 and float8_e5m2, layers dequantize. Any other
+format is refused by name, never mis-loaded.
+
+Row mapping: the family's converter runs once on float64 row tags ``(source key, row)``; the tags that
+come out say which source rows each diffusers weight holds. A converter that splices columns, changes
+the column count or transforms values leaves undecodable tags and is refused. (Tags are constant along a
+row: a pure input-column reorder would go unseen; no diffusers converter does one.)
"""
from __future__ import annotations
@@ -49,8 +31,12 @@ COMFY_QUANT_SUFFIX = ".comfy_quant"
QUANT_METADATA_KEY = "_quantization_metadata"
LEGACY_SCALED_FP8_KEY = "scaled_fp8"
COMFY_INT8_ENV = "UNSLOTH_DIFFUSION_COMFY_INT8"
+COMFY_FP8_ENV = "UNSLOTH_DIFFUSION_COMFY_FP8"
+# Header metadata our own converter writes (base model, scheme); ComfyUI ignores it.
+CONVERSION_RECORD_KEY = "unsloth_comfy_conversion"
INT8_TENSORWISE = "int8_tensorwise"
+FP8_E4M3 = "float8_e4m3fn"
FP8_FORMATS = {"float8_e4m3fn": "F8_E4M3", "float8_e5m2": "F8_E5M2"}
# Every companion tensor a ComfyUI quantized layer can carry. A suffix outside what a supported format
@@ -75,6 +61,8 @@ _LEGACY_RENAME = {".scale_weight": ".weight_scale", ".scale_input": ".input_scal
# Row tag: key index * 2**24 + row, exact in float64 for any real checkpoint.
_TAG = float(1 << 24)
+# Tag width of the fast pass: enough columns to see a splice (first / middle / last differ), none of the real width.
+_NARROW_TAG_COLUMNS = 4
_MAX_HEADER_BYTES = 256 * 1024 * 1024
@@ -248,6 +236,67 @@ def scan_comfy_quant(path: Optional[str]) -> Optional[ComfyQuantScan]:
return scan
+_DTYPE_BYTES = {
+ "F64": 8,
+ "F32": 4,
+ "F16": 2,
+ "BF16": 2,
+ "F8_E4M3": 1,
+ "F8_E5M2": 1,
+ "I8": 1,
+ "U8": 1,
+ "I16": 2,
+ "I32": 4,
+ "I64": 8,
+ "BOOL": 1,
+}
+
+
+def comfy_resident_mib(
+ path: Optional[str],
+ scan: Optional[ComfyQuantScan] = None,
+ *,
+ keep_int8: bool,
+ keep_fp8: bool,
+ compute_bytes: int = 2,
+) -> Optional[int]:
+ """What the loader leaves resident for a ComfyUI-quantized file, priced from its header: a quantized
+ weight a runtime keeps costs its stored bytes, one that is dequantized costs ``numel * compute_bytes``
+ (2x an int8 / fp8 file), every other floating tensor is cast to the compute dtype. The file name says
+ nothing reliable here (an fp8 file Studio runs natively is not upcast; an int8 one with no runtime is).
+ None when the header cannot be read. Torch-free."""
+ try:
+ scan = scan if scan is not None else scan_comfy_quant(path)
+ if scan is None:
+ return None
+ header, _base = _read_header(str(path))
+ except Exception: # noqa: BLE001 -- unknown size: the caller keeps its own estimate
+ return None
+ header.pop("__metadata__", None)
+ kept = {
+ name + ".weight": (layer.format == INT8_TENSORWISE and keep_int8)
+ or (layer.format == FP8_E4M3 and keep_fp8)
+ for name, layer in scan.layers.items()
+ }
+ total = 0
+ for key, entry in header.items():
+ shape = entry.get("shape") or ()
+ numel = 1
+ for d in shape:
+ numel *= int(d)
+ dtype = str(entry.get("dtype", ""))
+ stored = numel * _DTYPE_BYTES.get(dtype, 2)
+ if key in kept:
+ total += stored if kept[key] else numel * compute_bytes
+ elif key.endswith(COMFY_QUANT_SUFFIX) or any(key.endswith(s) for s in _SCALE_SUFFIXES):
+ total += stored
+ elif dtype.startswith(("F", "BF")):
+ total += numel * compute_bytes
+ else:
+ total += stored
+ return -(-total // (1024 * 1024)) if total > 0 else None
+
+
def _is_power_of_four(size: Any) -> bool:
from .diffusion_convrot import is_power_of_four
return is_power_of_four(size)
@@ -328,18 +377,93 @@ def _int8_tensor(name: str, codes: Any, scale: Any, dtype: Any) -> Optional[Any]
return None if rebuilt is None else rebuilt[name]
+def _fp8_tensor(
+ name: str,
+ codes: Any,
+ scale: Any,
+ dtype: Any,
+ fast_accum: Optional[bool] = None,
+) -> Optional[Any]:
+ """The torchao fp8 weight a hosted FP8 checkpoint rebuilds into (per-row ``Float8Tensor``, dynamic
+ per-row fp8 activations with the same 1e-12 floor, the plain-torch kernel), built by the same decoder.
+ A per-tensor scale is the per-row layout with every row's scale equal, so it is repeated, not changed."""
+ import torch
+
+ from .diffusion_transformer_quant import _resolve_fast_accum
+ from .prequant_native import native_unflatten
+
+ rows, cols = codes.shape
+ dtype_name = str(dtype).replace("torch.", "")
+ kernel = {"_type": "KernelPreference", "_data": "TORCH"}
+ entry = {
+ "_type": "Float8Tensor",
+ "_data": {
+ "block_size": [1, cols],
+ "mm_config": {
+ "_type": "Float8MMConfig",
+ "_data": {
+ "emulate": False,
+ "use_fast_accum": bool(_resolve_fast_accum(fast_accum)),
+ "pad_inner_dim": False,
+ },
+ },
+ "act_quant_kwargs": {
+ "_type": "QuantizeTensorToFloat8Kwargs",
+ "_data": {
+ "float8_dtype": {"_type": "torch.dtype", "_data": FP8_E4M3},
+ "granularity": {"_type": "PerRow", "_data": {"dim": -1}},
+ "mm_config": None,
+ "hp_value_lb": 1e-12,
+ "hp_value_ub": None,
+ "kernel_preference": kernel,
+ },
+ },
+ "kernel_preference": kernel,
+ "dtype": {"_type": "torch.dtype", "_data": dtype_name},
+ },
+ "_tensor_data_names": ["qdata", "scale"],
+ }
+ stem = name.rsplit(".", 1)[0]
+ tensors = {
+ f"{stem}._weight_qdata": codes.contiguous(),
+ f"{stem}._weight_scale": scale.to(torch.float32)
+ .reshape(-1, 1)
+ .expand(rows, 1)
+ .contiguous(),
+ }
+ raw = {"tensor_names": json.dumps([name]), name: json.dumps(entry)}
+ rebuilt = native_unflatten(tensors, raw)
+ return None if rebuilt is None else rebuilt[name]
+
+
def _mapping(transformer_cls: Any) -> tuple[Any, Any]:
from diffusers.loaders import single_file_model as sfm
name = sfm._get_single_file_loadable_mapping_class(transformer_cls)
entry = sfm.SINGLE_FILE_LOADABLE_CLASSES.get(name or "")
+ if not entry:
+ # The single-file branch registers Studio's converters for classes diffusers lacks (Qwen-Image-2.1) just
+ # before its own call; the hosted prequant route reaches here without passing through it.
+ try:
+ from .diffusion import _register_unregistered_single_file_classes
+ _register_unregistered_single_file_classes()
+ except Exception: # noqa: BLE001 -- nothing more to register: refused below
+ pass
+ name = sfm._get_single_file_loadable_mapping_class(transformer_cls)
+ entry = sfm.SINGLE_FILE_LOADABLE_CLASSES.get(name or "")
if not entry:
raise ValueError(f"{transformer_cls.__name__} has no single-file converter")
return entry["checkpoint_mapping_fn"], sfm
-def _decode_rows(name: str, tagged: Any, sources: list) -> list:
- """``[(source index, first row, n rows)]`` for a converted int8 weight, or raise."""
+def _decode_rows(
+ name: str,
+ tagged: Any,
+ sources: list,
+ width: Optional[int] = None,
+) -> list:
+ """``[(source index, first row, n rows)]`` for a converted int8 weight, or raise. ``width``: the tag width
+ the weights were given (None: their real column count)."""
import torch
if tagged.dtype != torch.float64 or tagged.dim() != 2 or tagged.numel() == 0:
@@ -348,19 +472,22 @@ def _decode_rows(name: str, tagged: Any, sources: list) -> list:
for j in (tagged.shape[1] // 2, tagged.shape[1] - 1):
if not torch.equal(col, tagged[:, j]):
raise ValueError(f"{name}: the converter mixed columns of an int8 weight")
- if not torch.equal(col, col.floor()) or bool((col < 0).any()):
- raise ValueError(f"{name}: the converter transformed int8 weight values")
- ids = (col / _TAG).floor().long()
- rows = (col - ids.double() * _TAG).long()
+ # Pure Python on one list: per-weight tensor ops each wake the intra-op pool and cost more than the decode.
segments: list = []
- for i, r in zip(ids.tolist(), rows.tolist()):
+ for value in col.tolist():
+ if value < 0 or value != int(value):
+ raise ValueError(f"{name}: the converter transformed int8 weight values")
+ i, r = divmod(int(value), int(_TAG))
if i >= len(sources) or r >= sources[i][1].shape[0]:
raise ValueError(f"{name}: an int8 row does not decode to a source row")
if segments and segments[-1][0] == i and segments[-1][1] + segments[-1][2] == r:
segments[-1][2] += 1
else:
segments.append([i, r, 1])
- if any(tagged.shape[1] != sources[i][1].shape[1] for i, _r, _n in segments):
+ if any(
+ tagged.shape[1] != (sources[i][1].shape[1] if width is None else width)
+ for i, _r, _n in segments
+ ):
raise ValueError(f"{name}: the converter changed an int8 weight's column count")
return segments
@@ -402,6 +529,147 @@ def comfy_int8_backend(
return None
+def comfy_fp8_backend(
+ target: Any,
+ family: Optional[str],
+ base_repo: Optional[str] = None,
+ *,
+ offload: bool = False,
+) -> Optional[str]:
+ """Which of Studio's fp8 runtimes takes the ``float8_e4m3fn`` layers, by the rule its own fp8 quant
+ follows: ``"torchao"`` (the per-row ``Float8Tensor`` and ``_scaled_mm``) on a resident plan whose GPU
+ passes Studio's fp8 probe, ``"native"`` (the torchao-free weight-only twin) where Studio's own fp8
+ quant runs natively (ROCm, the stubbed torchao), None (dequantize to bf16) where neither runs: an
+ older GPU, CPU, MPS, or an offloaded plan on the torchao path (the model offload hooks cannot move a
+ ``Float8Tensor``: "Attempted to set the storage of a tensor on device cuda:0 to a storage on ... cpu").
+ ``UNSLOTH_DIFFUSION_COMFY_FP8=0`` always dequantizes."""
+ if (os.environ.get(COMFY_FP8_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
+ return None
+ try:
+ import torch
+
+ from .diffusion_transformer_quant import (
+ TQ_FP8,
+ native_quant_scheme,
+ select_transformer_quant_scheme,
+ )
+
+ if native_quant_scheme(target, TQ_FP8, family = family, offload = offload) == TQ_FP8:
+ return "native"
+ if (
+ not offload
+ and getattr(target, "dtype", None) is torch.bfloat16
+ and select_transformer_quant_scheme(target, TQ_FP8, family = family, base_repo = base_repo)
+ == TQ_FP8
+ ):
+ return "torchao"
+ except Exception: # noqa: BLE001 -- an unanswerable probe takes the exact dense path
+ pass
+ return None
+
+
+def comfy_conversion_record(path: Optional[str]) -> dict:
+ """The ``unsloth_comfy_conversion`` header record our converter writes (base model, scheme, family),
+ or {} for any other file. Torch-free, never raises."""
+ try:
+ header, _base = _read_header(str(path))
+ raw = (header.get("__metadata__") or {}).get(CONVERSION_RECORD_KEY)
+ record = json.loads(raw) if raw else {}
+ return record if isinstance(record, dict) else {}
+ except Exception: # noqa: BLE001 -- no record is not an error
+ return {}
+
+
+# The ComfyUI layer format each Studio prequant scheme is published in.
+PREQUANT_SCHEME_FORMATS = {"int8": (INT8_TENSORWISE,), "fp8": (FP8_E4M3,)}
+
+
+def comfy_prequant_scheme(scan: Optional[ComfyQuantScan]) -> Optional[str]:
+ """The Studio prequant scheme a ComfyUI file's quantized layers all share, else None."""
+ if scan is None or scan.problems or not scan.layers:
+ return None
+ formats = {layer.format for layer in scan.layers.values()}
+ for scheme, allowed in PREQUANT_SCHEME_FORMATS.items():
+ if formats <= set(allowed):
+ return scheme
+ return None
+
+
+def load_comfy_prequant(
+ transformer_cls: Any,
+ path: str,
+ *,
+ scheme: str,
+ base: str,
+ family: Optional[str],
+ dtype: Any,
+ hf_token: Optional[str] = None,
+ cache_dir: Optional[str] = None,
+ local_files_only: bool = False,
+ fast_accum: Optional[bool] = None,
+ min_features: Optional[int] = None,
+ config_subfolder: str = "transformer",
+ logger: Any = None,
+) -> Any:
+ """A hosted (or local) ComfyUI-format prequant as the transformer Studio's own ``scheme`` checkpoint
+ rebuilds into, on the CPU: int8 layers as ``Int8Tensor`` under ConvRot-rotating Linears, fp8 layers as
+ per-row ``Float8Tensor``. Placement, small-M padding and the rest stay with the caller, exactly as for
+ Studio's own checkpoints. Raises ``ValueError`` for a file it must refuse: a format Studio cannot run,
+ layers of another scheme, or a recorded base model that is not ``base``."""
+ name = os.path.basename(str(path))
+ scan = scan_comfy_quant(path)
+ if scan is None:
+ raise ValueError(f"{name} is not a ComfyUI-quantized checkpoint")
+ problem = comfy_quant_error(scan, name)
+ if problem:
+ raise ValueError(problem)
+ found = comfy_prequant_scheme(scan)
+ if found != scheme:
+ raise ValueError(
+ f"{name} holds {', '.join(f'{k} x{v}' for k, v in sorted(scan.counts().items()))}, "
+ f"not a {scheme} checkpoint"
+ )
+ if not family:
+ raise ValueError(f"{name}: a ComfyUI checkpoint needs the model family to pick its layers")
+ record = comfy_conversion_record(path)
+ recorded_base = record.get("base_model_id")
+ if recorded_base and base:
+ from .diffusion_prequant import _same_base_model
+ if not _same_base_model(str(recorded_base), str(base)):
+ raise ValueError(f"{name} was converted from {recorded_base}, not {base}")
+ kwargs = {
+ "config": base,
+ "subfolder": config_subfolder,
+ "torch_dtype": dtype,
+ "cache_dir": cache_dir,
+ "local_files_only": local_files_only,
+ }
+ if hf_token:
+ kwargs["token"] = hf_token
+ import torch
+
+ if scheme == "fp8" and dtype is not torch.bfloat16:
+ raise ValueError(f"{name}: Studio's fp8 runtime needs a bfloat16 pipeline, not {dtype}")
+ model = load_comfy_quant_transformer(
+ transformer_cls,
+ path,
+ scan,
+ kwargs,
+ int8_backend = "torchao" if scheme == "int8" else None,
+ fp8_backend = "torchao" if scheme == "fp8" else None,
+ family = family,
+ fast_accum = fast_accum,
+ min_features = min_features,
+ finalize = False,
+ logger = logger,
+ )
+ kept = (getattr(model, "_unsloth_comfy_quant", None) or {}).get(scheme, 0)
+ if not kept:
+ # never a dense model under a prequant label: the caller falls back to its own quantize path
+ raise ValueError(f"{name}: no layer could be rebuilt as a {scheme} weight on this install")
+ return model
+
+
def load_comfy_quant_transformer(
transformer_cls: Any,
path: str,
@@ -409,8 +677,12 @@ def load_comfy_quant_transformer(
sf_kwargs: dict,
*,
int8_backend: Optional[str],
+ fp8_backend: Optional[str] = None,
family: Optional[str] = None,
target: Any = None,
+ fast_accum: Optional[bool] = None,
+ min_features: Optional[int] = None,
+ finalize: bool = True,
logger: Any = None,
) -> Any:
"""Build ``transformer_cls`` from the ComfyUI-quantized ``path``.
@@ -419,16 +691,33 @@ def load_comfy_quant_transformer(
``subfolder``, ``torch_dtype``, ``token``, ``cache_dir``, ``local_files_only``). With an
``int8_backend`` (``comfy_int8_backend``) the int8 layers Studio's own int8 filter selects keep
their codes and scales, as torchao ``Int8Tensor`` weights under ConvRot-rotating Linears
- (``"torchao"``) or as native int8 twins (``"native"``); everything else is dequantized to the
- compute dtype. Raises ``ValueError`` for a checkpoint it must refuse."""
+ (``"torchao"``) or as native int8 twins (``"native"``); with an ``fp8_backend``
+ (``comfy_fp8_backend``) the ``float8_e4m3fn`` layers Studio's own fp8 filter selects do the same,
+ as per-row ``Float8Tensor`` weights or native fp8 twins. Everything else is dequantized to the
+ compute dtype. ``finalize`` applies the small-M padding here (the single-file path); the hosted
+ prequant loader passes False and pads after placement, as for its own checkpoints. Raises
+ ``ValueError`` for a checkpoint it must refuse."""
import torch
from safetensors.torch import load_file
+ from .diffusion_transformer_quant import (
+ DEFAULT_MIN_LINEAR_FEATURES,
+ TQ_FP8,
+ TQ_INT8,
+ divisible_for_scheme,
+ exclude_tokens_for_scheme,
+ make_filter_fn,
+ native_int8_act,
+ )
+
problem = comfy_quant_error(scan, os.path.basename(path))
if problem:
raise ValueError(problem)
kwargs = dict(sf_kwargs)
dtype = kwargs.pop("torch_dtype", None) or kwargs.pop("dtype", None) or torch.bfloat16
+ if dtype is not torch.bfloat16:
+ # Studio's fp8 runtime asserts bf16 weights; an fp16 / fp32 pipeline dequantizes them instead.
+ fp8_backend = None
state = load_file(str(path))
state.pop(LEGACY_SCALED_FP8_KEY, None)
for key in [k for k in state if k.endswith(COMFY_QUANT_SUFFIX)]:
@@ -448,16 +737,18 @@ def load_comfy_quant_transformer(
def _dtype_for(key: str) -> Any:
return torch.float32 if fp16_keep and any(m in key.split(".") for m in keep_fp32) else dtype
- int8_sources: list = [] # (layer, codes, scale)
+ backends = {INT8_TENSORWISE: int8_backend, FP8_E4M3: fp8_backend}
+ sources: list = [] # (layer, codes, scale) kept for a runtime
dequantized = 0
for layer in scan.layers.values():
codes = state.pop(layer.name + ".weight")
scale = state.pop(layer.name + ".weight_scale")
+ # Activations are quantized per row at runtime, as for Studio's own fp8; input_scale is unused.
state.pop(layer.name + ".input_scale", None)
if layer.format in FP8_FORMATS and codes.dtype == torch.uint8:
codes = codes.view(getattr(torch, layer.format))
- if (layer.format == INT8_TENSORWISE and int8_backend) or fp16_keep:
- int8_sources.append((layer, codes, scale))
+ if backends.get(layer.format) or fp16_keep:
+ sources.append((layer, codes, scale))
state[layer.name + ".weight"] = None # placeholder, tagged below
else:
state[layer.name + ".weight"] = _dequant(codes, scale, layer.group, dtype)
@@ -467,10 +758,6 @@ def load_comfy_quant_transformer(
if fp16_keep or value is None or not value.is_floating_point() or value.dtype == dtype:
continue
state[key] = value.to(dtype)
- for index, (layer, codes, _scale) in enumerate(int8_sources):
- rows = torch.arange(codes.shape[0], dtype = torch.float64) + index * _TAG
- state[layer.name + ".weight"] = rows.view(-1, 1).expand(codes.shape)
-
from accelerate import init_empty_weights
mapping_fn, sfm = _mapping(transformer_cls)
@@ -493,79 +780,101 @@ def load_comfy_quant_transformer(
with init_empty_weights():
model = transformer_cls.from_config(config)
wanted = model.state_dict()
- if sfm._should_convert_state_dict_to_diffusers(wanted, state):
- converted = mapping_fn(
- config = config,
- checkpoint = dict(state),
- **sfm._get_mapping_function_kwargs(mapping_fn, **kwargs),
- )
- else:
- converted = dict(state)
- del state
- rotations: dict = {}
- native: dict = {}
- built = 0
- for name in [
- k for k, v in converted.items() if torch.is_tensor(v) and v.dtype == torch.float64
- ]:
- segments = _decode_rows(name, converted[name], int8_sources)
- sources = {(int8_sources[i][0].format, int8_sources[i][0].group) for i, _r, _n in segments}
- if len(sources) != 1:
- raise ValueError(f"{name}: rows from layers of different formats or ConvRot groups")
- fmt, group = sources.pop()
- codes = torch.cat([int8_sources[i][1][r : r + n] for i, r, n in segments])
- scale = torch.cat(
- [
- int8_sources[i][2].reshape(-1, 1).expand(int8_sources[i][1].shape[0], 1)[r : r + n]
- for i, r, n in segments
- ]
- )
- converted[name] = (codes, scale, group, fmt)
+ def _convert_tagged(width: Optional[int]) -> dict:
+ # float64 row tags, ``width`` columns wide (None: real width); converted tags name each output's source rows.
+ for index, (layer, codes, _scale) in enumerate(sources):
+ rows = torch.arange(codes.shape[0], dtype = torch.float64) + index * _TAG
+ cols = codes.shape[1] if width is None else width
+ state[layer.name + ".weight"] = rows.view(-1, 1).expand(codes.shape[0], cols)
+ if sfm._should_convert_state_dict_to_diffusers(wanted, state):
+ out = mapping_fn(
+ config = config,
+ checkpoint = dict(state),
+ **sfm._get_mapping_function_kwargs(mapping_fn, **kwargs),
+ )
+ else:
+ out = dict(state)
+ for name in [k for k, v in out.items() if torch.is_tensor(v) and v.dtype == torch.float64]:
+ segments = _decode_rows(name, out[name], sources, width = width)
+ kinds = {(sources[i][0].format, sources[i][0].group) for i, _r, _n in segments}
+ if len(kinds) != 1:
+ raise ValueError(f"{name}: rows from layers of different formats or ConvRot groups")
+ fmt, group = kinds.pop()
+ parts = [sources[i][1][r : r + n] for i, r, n in segments]
+ # one segment (a split or renamed layer) is a row slice of a contiguous tensor: no copy
+ codes = parts[0] if len(parts) == 1 else torch.cat(parts)
+ scale = torch.cat(
+ [
+ sources[i][2].reshape(-1, 1).expand(sources[i][1].shape[0], 1)[r : r + n]
+ for i, r, n in segments
+ ]
+ )
+ if name in wanted and tuple(wanted[name].shape) != tuple(codes.shape):
+ raise ValueError(
+ f"{name}: rebuilt as {tuple(codes.shape)}, the model expects {tuple(wanted[name].shape)}"
+ )
+ out[name] = (codes, scale, group, fmt)
+ return out
+
+ try:
+ # Narrow tags first (full width costs tens of seconds on a large DiT); anything unproven reruns at full width.
+ converted = _convert_tagged(_NARROW_TAG_COLUMNS if sources else None)
+ except Exception: # noqa: BLE001 -- the full-width pass raises the real refusal
+ converted = _convert_tagged(None)
+ del state
if fp16_keep:
for key, value in list(converted.items()):
if torch.is_tensor(value) and value.is_floating_point():
converted[key] = value.to(_dtype_for(key))
- from .diffusion_transformer_quant import (
- DEFAULT_MIN_LINEAR_FEATURES,
- TQ_INT8,
- exclude_tokens_for_scheme,
- make_filter_fn,
- native_int8_act,
- )
-
- filter_fn = make_filter_fn(
- DEFAULT_MIN_LINEAR_FEATURES,
- exclude_name_tokens = exclude_tokens_for_scheme(TQ_INT8, family),
- )
+ min_features = DEFAULT_MIN_LINEAR_FEATURES if min_features is None else int(min_features)
+ schemes = {INT8_TENSORWISE: TQ_INT8, FP8_E4M3: TQ_FP8}
+ filters = {
+ fmt: make_filter_fn(
+ min_features,
+ exclude_name_tokens = exclude_tokens_for_scheme(scheme, family),
+ require_divisible = divisible_for_scheme(scheme),
+ )
+ for fmt, scheme in schemes.items()
+ }
modules = dict(model.named_modules())
+ rotations: dict = {}
+ native: dict = {}
+ built = {TQ_INT8: 0, TQ_FP8: 0}
for name, value in list(converted.items()):
if not isinstance(value, tuple):
continue
codes, scale, group, fmt = value
+ scheme = schemes.get(fmt)
+ backend = backends.get(fmt)
fqn = name[: -len(".weight")] if name.endswith(".weight") else name
module = modules.get(fqn)
weight = None
- if (
- int8_backend
- and fmt == INT8_TENSORWISE
+ keep = (
+ backend is not None
and module is not None
and name.endswith(".weight")
- and filter_fn(module, fqn)
- ):
- if int8_backend == "native":
- native[fqn] = (codes, scale, group)
+ and filters[fmt](module, fqn)
+ # Studio's fp8 quant leaves the Linears a DiT keeps in fp32 (require_bf16) at full precision.
+ and not (scheme == TQ_FP8 and any(m in fqn.split(".") for m in keep_fp32))
+ )
+ if keep:
+ if backend == "native":
+ native[fqn] = (codes, scale, group, scheme)
del converted[name]
- built += 1
+ built[scheme] += 1
continue
- weight = _int8_tensor(name, codes, scale, dtype)
+ if scheme == TQ_INT8:
+ weight = _int8_tensor(name, codes, scale, dtype)
+ else:
+ weight = _fp8_tensor(name, codes, scale, dtype, fast_accum)
if weight is None:
converted[name] = _dequant(codes, scale, group, _dtype_for(name))
dequantized += 1
continue
converted[name] = weight
- built += 1
+ built[scheme] += 1
if group:
rotations.setdefault(group, []).append(fqn)
if len(rotations) > 1:
@@ -576,13 +885,18 @@ def load_comfy_quant_transformer(
cls = native_linear_class()
act_int8 = native_int8_act(target) if target is not None else False
- for fqn, (codes, scale, group) in native.items():
+ for fqn, (codes, scale, group, scheme) in native.items():
parent_name, _, leaf = fqn.rpartition(".")
parent = model.get_submodule(parent_name) if parent_name else model
# the Linear supplies only shapes, compute dtype and bias (assigned from the checkpoint below)
linear = getattr(parent, leaf).to(dtype)
layer = cls(
- linear, TQ_INT8, act_int8 = act_int8, rot_group = group, codes = codes, scale = scale
+ linear,
+ scheme,
+ act_int8 = act_int8 and scheme == TQ_INT8,
+ rot_group = group,
+ codes = codes,
+ scale = scale,
)
setattr(parent, leaf, layer)
converted[fqn + ".weight_q"] = layer.weight_q
@@ -635,32 +949,42 @@ def load_comfy_quant_transformer(
warm_rotation_cache(model, device, dtype)
except Exception: # noqa: BLE001 -- only saves one recompile
pass
- if built:
- if not native:
- from .diffusion_transformer_quant import apply_small_m_padding
- apply_small_m_padding(model, TQ_INT8, family, logger = logger)
+ native_schemes = {v[3] for v in native.values()}
+ on_torchao = {s for s, count in built.items() if count and s not in native_schemes}
+ if finalize:
+ from .diffusion_transformer_quant import apply_small_m_padding, apply_zero_row_guard
+ for scheme in sorted(on_torchao):
+ apply_small_m_padding(model, scheme, family, logger = logger)
+ apply_zero_row_guard(model, scheme, family, logger = logger)
+ if any(built.values()):
try:
- model._unsloth_runtime_quant = TQ_INT8
+ # the scheme the runtime treats this transformer as: the one most of its layers run
+ model._unsloth_runtime_quant = max(built, key = lambda s: built[s])
except Exception: # noqa: BLE001 -- marker is best-effort
pass
model.eval()
+ convrot = len(rotated) + sum(1 for v in native.values() if v[2])
if logger is not None:
logger.info(
"diffusion.comfy_quant: %s loaded from a ComfyUI checkpoint (%s): %d int8 layers kept as "
- "int8 (%s, %d ConvRot), %d layers dequantized to %s",
+ "int8 (%s, %d ConvRot), %d fp8 layers kept as fp8 (%s), %d layers dequantized to %s",
transformer_cls.__name__,
", ".join(f"{k} x{v}" for k, v in sorted(scan.counts().items())),
- built,
- f"{int8_backend} runtime" if built else "no int8 runtime",
- len(rotated) + sum(1 for v in native.values() if v[2]),
+ built[TQ_INT8],
+ f"{int8_backend} runtime" if built[TQ_INT8] else "no int8 runtime",
+ convrot,
+ built[TQ_FP8],
+ f"{fp8_backend} runtime" if built[TQ_FP8] else "no fp8 runtime",
dequantized,
str(dtype).replace("torch.", ""),
)
try:
model._unsloth_comfy_quant = {
- "backend": int8_backend if built else None,
- "int8": built,
- "convrot": len(rotated) + sum(1 for v in native.values() if v[2]),
+ "backend": int8_backend if built[TQ_INT8] else None,
+ "int8": built[TQ_INT8],
+ "convrot": convrot,
+ "fp8_backend": fp8_backend if built[TQ_FP8] else None,
+ "fp8": built[TQ_FP8],
"dequantized": dequantized,
}
except Exception: # noqa: BLE001 -- diagnostic marker only
diff --git a/studio/backend/core/inference/diffusion_denoiser_prequant.py b/studio/backend/core/inference/diffusion_denoiser_prequant.py
index 9866ba2fe01..cbda6b57ce3 100644
--- a/studio/backend/core/inference/diffusion_denoiser_prequant.py
+++ b/studio/backend/core/inference/diffusion_denoiser_prequant.py
@@ -136,6 +136,7 @@ def denoiser_prequant_pipe_kwargs(
local_files_only = local_files_only,
logger = logger,
placement_device = placement_device,
+ family = getattr(fam, "name", None),
)
if module is None:
if logger is not None:
diff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.py
index 3cbb478b1cd..bbedbc9f122 100644
--- a/studio/backend/core/inference/diffusion_native_quant.py
+++ b/studio/backend/core/inference/diffusion_native_quant.py
@@ -60,9 +60,10 @@ def native_linear_class():
codes: Any = None,
scale: Any = None,
):
- """``codes`` / ``scale``: int8 codes [out, in] and per-row scales made elsewhere, stored as given
- (``linear`` then supplies only the shapes, dtype and bias). With ``rot_group`` they are already
- ConvRot-rotated, so the rotation is installed whether or not activations go int8."""
+ """``codes`` / ``scale``: int8 (or, for the fp8 scheme, float8_e4m3fn) codes [out, in] and per-row
+ scales made elsewhere, stored as given (``linear`` then supplies only the shapes, dtype and bias).
+ With ``rot_group`` int8 codes are already ConvRot-rotated, so the rotation is installed whether or
+ not activations go int8."""
super().__init__()
if scheme not in _QMAX:
raise ValueError(f"unsupported native scheme {scheme!r}")
@@ -77,8 +78,13 @@ def native_linear_class():
else 0
)
if codes is not None:
- if scheme != NATIVE_INT8 or codes.dtype != torch.int8:
- raise ValueError("pre-quantized native codes must be int8")
+ wanted = torch.int8 if scheme == NATIVE_INT8 else torch.float8_e4m3fn
+ if codes.dtype != wanted:
+ raise ValueError(
+ f"pre-quantized native {scheme} codes must be {str(wanted).replace('torch.', '')}"
+ )
+ if rot_group and scheme != NATIVE_INT8:
+ raise ValueError("a ConvRot rotation needs int8 codes")
self.rot_group = int(rot_group or 0)
if self.rot_group and self.in_features % self.rot_group:
raise ValueError(
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index 72ef5bd291a..cc852114d0f 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -500,6 +500,78 @@ PREFER_SAFETENSORS_ENV = "UNSLOTH_PREQUANT_PREFER_SAFETENSORS"
_PICKLE_SUFFIXES = (".pt", ".pth")
_logged_twin_choices: set = set()
+# ``<stem>-ComfyUI.safetensors``: the twin of a hosted artifact both ComfyUI and Studio load; older builds never ask for it.
+COMFY_PREQUANT_TAG = "-ComfyUI"
+COMFY_PREQUANT_SUFFIX = COMFY_PREQUANT_TAG + ".safetensors"
+# 0 drops the ComfyUI-format names from the chain (back to Studio's own containers only).
+COMFY_PREQUANT_ENV = "UNSLOTH_DIFFUSION_PREQUANT_COMFY"
+_ARTIFACT_SUFFIXES = (".safetensors",) + _PICKLE_SUFFIXES
+
+
+def is_comfy_prequant_filename(name: Optional[str]) -> bool:
+ """Whether ``name`` is the ComfyUI-format spelling of a hosted artifact."""
+ return bool(name) and str(name).endswith(COMFY_PREQUANT_SUFFIX)
+
+
+def _artifact_stem(name: Optional[str]) -> Optional[str]:
+ """``<stem>`` of a hosted ``<stem>.safetensors`` / ``.pt`` / ``.pth`` / ``-ComfyUI.safetensors`` artifact;
+ None for the legacy ``transformer_<scheme>.pt`` names, which have no ComfyUI twin."""
+ if not name or "/" in str(name):
+ return None
+ name = str(name)
+ if name.startswith("transformer_"):
+ return None
+ if name.endswith(COMFY_PREQUANT_SUFFIX):
+ return name[: -len(COMFY_PREQUANT_SUFFIX)] or None
+ for suffix in _ARTIFACT_SUFFIXES:
+ if name.endswith(suffix):
+ return name[: -len(suffix)] or None
+ return None
+
+
+def comfy_prequant_filename(name: Optional[str]) -> Optional[str]:
+ """The ComfyUI-format twin of the hosted artifact ``name`` (``<stem>-ComfyUI.safetensors``), else None."""
+ stem = _artifact_stem(name)
+ if stem is None or is_comfy_prequant_filename(name):
+ return None
+ return stem + COMFY_PREQUANT_SUFFIX
+
+
+def comfy_prequant_enabled() -> bool:
+ import os
+ return (os.environ.get(COMFY_PREQUANT_ENV) or "").strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+# Twins that hold Studio's weights bit for bit (int8) lead the chain. ComfyUI fp8 has one scale per tensor, a coarser
+# second rounding of Studio's per-row fp8, so an fp8 twin trails Studio's own fp8 artifact.
+COMFY_TWIN_LEADS = frozenset({"int8"})
+
+
+def with_comfy_twins(names: Sequence[str], *, lead: bool = True) -> list:
+ """``names`` with each artifact's ComfyUI-format twin added. ``lead``: right ahead of the artifact's first name,
+ so the one file ComfyUI also loads is the default download and the containers older builds request stay behind
+ it as the fallback when the repo does not host the twin; otherwise right behind the artifact's last name.
+ Order-preserving, no duplicates."""
+ names = [n for n in names if n]
+ out: list = []
+ for index, name in enumerate(names):
+ twin = comfy_prequant_filename(name)
+ stem = _artifact_stem(name)
+ if twin and twin not in names and twin not in out:
+ if lead:
+ out.append(twin)
+ elif not any(_artifact_stem(n) == stem for n in names[index + 1 :]):
+ out.extend(n for n in (name, twin) if n not in out)
+ continue
+ if name not in out:
+ out.append(name)
+ return out
+
def _hub_name_cached(
repo_id: Optional[str], name: Optional[str], root: Optional[str]
@@ -603,8 +675,6 @@ def prefer_cached_pickle_twins(
out = [n for n in names if n]
if not repo_id or len(out) < 2:
return out
- if (os.environ.get(PREFER_SAFETENSORS_ENV) or "").strip().lower() in ("1", "true", "yes", "on"):
- return out
try:
# only the roots the caller's download will reuse, else it re-fetches the pickle it was promised
roots = tuple(roots) if roots is not None else _twin_cache_roots(cache_dir)
@@ -618,7 +688,49 @@ def prefer_cached_pickle_twins(
except Exception: # noqa: BLE001 - an unanswerable question is a no
return False
- for st in [n for n in out if n.lower().endswith(".safetensors")]:
+ # A cached container of an artifact beats an uncached one of the SAME artifact: never download a second copy.
+ for comfy in [n for n in out if is_comfy_prequant_filename(n)]:
+ stem = _artifact_stem(comfy)
+ twins = [n for n in out if n != comfy and _artifact_stem(n) == stem]
+ if not twins:
+ continue
+ hit = next((t for t in twins if _ok(t) and _cached(t)), None)
+ if _cached(comfy):
+ if hit is None and _ok(comfy) and out.index(comfy) > out.index(twins[0]):
+ out.remove(comfy)
+ out.insert(out.index(twins[0]), comfy)
+ continue
+ if hit is None:
+ continue
+ out.remove(comfy)
+ out.insert(out.index(twins[-1]) + 1, comfy)
+ key = (repo_id, hit, comfy)
+ if log and key not in _logged_twin_choices:
+ _logged_twin_choices.add(key)
+ sink = logger
+ if sink is None:
+ import logging
+ sink = logging.getLogger(__name__)
+ sink.info(
+ "diffusion.prequant_cached_twin: %s: using the cached %s; its ComfyUI-format twin %s is "
+ "not downloaded",
+ repo_id,
+ hit,
+ comfy,
+ )
+ if (os.environ.get(PREFER_SAFETENSORS_ENV) or "").strip().lower() in (
+ "1",
+ "true",
+ "yes",
+ "on",
+ ):
+ return out
+
+ for st in [
+ n
+ for n in out
+ if n.lower().endswith(".safetensors") and not is_comfy_prequant_filename(n)
+ ]:
stem = st[: -len(".safetensors")]
twin = next(
(
@@ -920,6 +1032,8 @@ def resolve_prequant_source(
for name in declared + derived:
if name and name not in names:
names.append(name)
+ if comfy_prequant_enabled() and _comfy_prequant_family(fam):
+ names = with_comfy_twins(names, lead = scheme in COMFY_TWIN_LEADS)
return PrequantSource(
kind = "repo",
location = repo_id,
@@ -930,6 +1044,16 @@ def resolve_prequant_source(
return None
+def _comfy_prequant_family(fam: Any) -> bool:
+ """Whether ``fam``'s hosted artifacts may resolve to a ComfyUI-format twin: the image families, whose
+ denoiser loads through ``load_prequantized_transformer`` as one whole module."""
+ try:
+ from .diffusion_families import DiffusionFamily
+ return isinstance(fam, DiffusionFamily)
+ except Exception: # noqa: BLE001 - no registry: keep the chain as it was
+ return False
+
+
_LOCAL_PREQUANT_SCHEME: dict[tuple[str, int, int], Optional[str]] = {}
@@ -978,6 +1102,12 @@ def local_prequant_scheme(path: str) -> Optional[str]:
scheme = str(recorded) if recorded else None
except Exception: # noqa: BLE001 -- a checkpoint we cannot parse is "unknown", never a match
scheme = None
+ if scheme is None and is_safetensors_checkpoint(real):
+ try:
+ from .diffusion_comfy_quant import comfy_prequant_scheme, scan_comfy_quant
+ scheme = comfy_prequant_scheme(scan_comfy_quant(real))
+ except Exception: # noqa: BLE001 -- unreadable is "unknown"
+ scheme = None
_LOCAL_PREQUANT_SCHEME[key] = scheme
return scheme
@@ -1396,6 +1526,16 @@ def usable_prequant_source(
return None
if local_prequant_scheme(src.location) != scheme:
return None
+ if not _comfy_prequant_family(fam):
+ # only the image denoiser loader reads a ComfyUI-format file; anywhere else it would be refused at load
+ try:
+ from .diffusion_comfy_quant import scan_comfy_quant
+
+ from os.path import expanduser
+ if scan_comfy_quant(expanduser(src.location)) is not None:
+ return None
+ except Exception: # noqa: BLE001 -- unreadable: the scheme check above already decided
+ pass
return src
@@ -1564,9 +1704,13 @@ def load_prequantized_transformer(
local_files_only: bool = False,
logger: Any = None,
placement_device: Optional[str] = None,
+ family: Optional[str] = None,
) -> Optional[Any]:
"""Load the pre-quantized transformer described by ``source`` onto ``device``.
+ ``family`` (the Studio family name) is what a ComfyUI-format artifact needs to pick the layers Studio's
+ own ``scheme`` quantizes; Studio's own checkpoints record it themselves.
+
``placement_device`` (default ``device``) is where the module is materialised; ``device`` selects kernels.
``cache_dir`` is the live Hub cache root, as every other loader call pins it: unset, a fetch
@@ -1608,93 +1752,128 @@ def load_prequantized_transformer(
if path is None:
return None
- # A safetensors artifact, or a torch.save pickle deserialized under the constructor ALLOWLIST above and never
- # as a free-running one. First-party hosting is no reason to execute whatever bytes arrive: the artifact is
- # mutable, fetched over the network, and reached by loads that never asked for one (auto resolves an unset
- # precision to a hosted checkpoint), so a mutated file must fail to load rather than run. Both containers
- # hand back the same dict, so every check below applies to them equally.
- ckpt = _read_prequant_for(path, placement_device or device, logger)
- if not _validate_checkpoint(
- ckpt,
- scheme,
- base,
- logger,
- min_features = min_features,
- fast_accum = fast_accum,
- component = component,
- ):
- return None
- state_dict = ckpt["state_dict"]
- # The only check reading what the artifact HOLDS: corruption after build passes the rest.
- if not _verify_packed_fingerprint(
- state_dict, ckpt.get("metadata") or {}, logger = logger, path = path
- ):
- return None
- _repair_legacy_checkpoint(ckpt, scheme, logger)
- _pin_kernel_preference(state_dict, logger)
-
- # Read from the root that actually supplied the checkpoint: after a mid-session cache change the pinned root
- # may be gone or read-only, and load_config's raise is swallowed below into a None return, silently dropping a
- # prequant whose checkpoint is cached and already loaded.
- config = _load_transformer_config(
- transformer_cls,
- base,
- hf_token,
- cache_dir,
- path,
- config_subfolder,
- local_files_only = local_files_only,
- )
- from accelerate import init_empty_weights
-
- metadata = ckpt.get("metadata") or {}
- with init_empty_weights():
- transformer = transformer_cls.from_config(config)
- if prepare_model is not None:
- prepare_model(transformer, metadata)
- transformer.load_state_dict(state_dict, strict = True, assign = True)
- if _has_meta_tensors(transformer):
- # Non-persistent buffers (built in __init__, absent from the state dict) stay on meta. Rebuild on CPU so
- # they hold real values, then re-assign the quantized weights; dense bf16 never reaches the GPU.
- transformer = transformer_cls.from_config(config)
- # The retry REPLACES the module, so the hook has to run again: skipping it here would load the same state
- # dict into a differently shaped model, and this branch is the one families with non-persistent buffers
- # always take -- the mismatch would be the norm, not the corner case, and strict=True would surface it as
- # a bare key error.
+ # A ComfyUI-format file rebuilds into the same torchao weights as Studio's own checkpoint of this scheme.
+ from .diffusion_comfy_quant import load_comfy_prequant, scan_comfy_quant
+
+ comfy_scan = scan_comfy_quant(path)
+ comfy_rotated = False
+ if comfy_scan is not None:
+ if prepare_model is not None or (component and component != DEFAULT_PREQUANT_COMPONENT):
+ raise ValueError(
+ "a ComfyUI-format checkpoint is only read as a whole denoiser, not as the "
+ f"{component or 'reshaped'} component"
+ )
+ transformer = load_comfy_prequant(
+ transformer_cls,
+ path,
+ scheme = scheme,
+ base = base,
+ family = family,
+ dtype = dtype,
+ hf_token = hf_token,
+ cache_dir = cache_dir,
+ local_files_only = local_files_only,
+ fast_accum = fast_accum,
+ min_features = min_features,
+ config_subfolder = config_subfolder,
+ logger = logger,
+ )
+ from .diffusion_convrot import declares_rotation, is_rotated_linear, warm_rotation_cache
+
+ comfy_rotated = any(is_rotated_linear(m) for m in transformer.modules())
+ metadata = {
+ "family": family,
+ "torch_dtype": str(dtype).replace("torch.", ""),
+ "comfy_format": True,
+ }
+ else:
+ # A safetensors artifact, or a torch.save pickle deserialized under the constructor ALLOWLIST above and never
+ # as a free-running one. First-party hosting is no reason to execute whatever bytes arrive: the artifact is
+ # mutable, fetched over the network, and reached by loads that never asked for one (auto resolves an unset
+ # precision to a hosted checkpoint), so a mutated file must fail to load rather than run. Both containers
+ # hand back the same dict, so every check below applies to them equally.
+ ckpt = _read_prequant_for(path, placement_device or device, logger)
+ if not _validate_checkpoint(
+ ckpt,
+ scheme,
+ base,
+ logger,
+ min_features = min_features,
+ fast_accum = fast_accum,
+ component = component,
+ ):
+ return None
+ state_dict = ckpt["state_dict"]
+ # The only check reading what the artifact HOLDS: corruption after build passes the rest.
+ if not _verify_packed_fingerprint(
+ state_dict, ckpt.get("metadata") or {}, logger = logger, path = path
+ ):
+ return None
+ _repair_legacy_checkpoint(ckpt, scheme, logger)
+ _pin_kernel_preference(state_dict, logger)
+
+ # Read from the root that actually supplied the checkpoint: after a mid-session cache change the pinned root
+ # may be gone or read-only, and load_config's raise is swallowed below into a None return, silently dropping a
+ # prequant whose checkpoint is cached and already loaded.
+ config = _load_transformer_config(
+ transformer_cls,
+ base,
+ hf_token,
+ cache_dir,
+ path,
+ config_subfolder,
+ local_files_only = local_files_only,
+ )
+ from accelerate import init_empty_weights
+
+ metadata = ckpt.get("metadata") or {}
+ with init_empty_weights():
+ transformer = transformer_cls.from_config(config)
if prepare_model is not None:
prepare_model(transformer, metadata)
transformer.load_state_dict(state_dict, strict = True, assign = True)
+ if _has_meta_tensors(transformer):
+ # Non-persistent buffers (built in __init__, absent from the state dict) stay on meta. Rebuild on CPU so
+ # they hold real values, then re-assign the quantized weights; dense bf16 never reaches the GPU.
+ transformer = transformer_cls.from_config(config)
+ # The retry REPLACES the module, so the hook has to run again: skipping it here would load the same state
+ # dict into a differently shaped model, and this branch is the one families with non-persistent buffers
+ # always take -- the mismatch would be the norm, not the corner case, and strict=True would surface it as
+ # a bare key error.
+ if prepare_model is not None:
+ prepare_model(transformer, metadata)
+ transformer.load_state_dict(state_dict, strict = True, assign = True)
+
+ # The ONLINE half of an activation rotation, applied here rather than in a family's ``prepare_model`` hook so
+ # that no route can load a rotated checkpoint without it: the offline half is already baked into the weights
+ # that were just assigned, and a rotated weight met by an unrotated activation renders plausible garbage with
+ # nothing to catch. A no-op for every artifact that declares no rotation, and a RAISE (caught below into the
+ # dense fallback) for one this build cannot honour exactly. After load_state_dict because the meta retry above
+ # rebuilds the module; before apply_small_m_padding because padding reparents the Linears and the recorded
+ # fqns name the unwrapped tree.
+ from .diffusion_convrot import (
+ apply_activation_rotation,
+ declares_rotation,
+ warm_rotation_cache,
+ )
- # The ONLINE half of an activation rotation, applied here rather than in a family's ``prepare_model`` hook so
- # that no route can load a rotated checkpoint without it: the offline half is already baked into the weights
- # that were just assigned, and a rotated weight met by an unrotated activation renders plausible garbage with
- # nothing to catch. A no-op for every artifact that declares no rotation, and a RAISE (caught below into the
- # dense fallback) for one this build cannot honour exactly. After load_state_dict because the meta retry above
- # rebuilds the module; before apply_small_m_padding because padding reparents the Linears and the recorded
- # fqns name the unwrapped tree.
- from .diffusion_convrot import (
- apply_activation_rotation,
- declares_rotation,
- warm_rotation_cache,
- )
-
- apply_activation_rotation(transformer, metadata, logger = logger)
+ apply_activation_rotation(transformer, metadata, logger = logger)
- if scheme == "nvfp4":
- from .diffusion_nvfp4_linear import convert_nvfp4_backend
- from .diffusion_nvfp4_ops import select_nvfp4_backend
- convert_nvfp4_backend(
- transformer, metadata, select_nvfp4_backend(device), logger = logger
- )
- # assign=True shares the tensors: a live reference doubles the peak on unified memory.
- del state_dict
- del ckpt
+ if scheme == "nvfp4":
+ from .diffusion_nvfp4_linear import convert_nvfp4_backend
+ from .diffusion_nvfp4_ops import select_nvfp4_backend
+ convert_nvfp4_backend(
+ transformer, metadata, select_nvfp4_backend(device), logger = logger
+ )
+ # assign=True shares the tensors: a live reference doubles the peak on unified memory.
+ del state_dict
+ del ckpt
from .diffusion_fast_load import fast_upload
with fast_upload([transformer], placement_device or device, logger = logger):
transformer = transformer.to(placement_device or device)
- if declares_rotation(metadata):
+ if declares_rotation(metadata) or comfy_rotated:
try:
import torch
diff --git a/studio/backend/tests/test_diffusion_comfy_quant.py b/studio/backend/tests/test_diffusion_comfy_quant.py
index 65459ea36ea..e4e26333b5d 100644
--- a/studio/backend/tests/test_diffusion_comfy_quant.py
+++ b/studio/backend/tests/test_diffusion_comfy_quant.py
@@ -335,6 +335,8 @@ def test_dequant_path_reproduces_the_dense_weights(comfy_file):
"backend": None,
"int8": 0,
"convrot": 0,
+ "fp8_backend": None,
+ "fp8": 0,
"dequantized": 6,
}
assert not any(is_rotated_linear(m) for m in model.modules())
@@ -360,6 +362,8 @@ def test_int8_runtime_keeps_codes_and_scales_unchanged(comfy_file):
"backend": "torchao",
"int8": 8,
"convrot": 8,
+ "fp8_backend": None,
+ "fp8": 0,
"dequantized": 2,
}
assert model._unsloth_runtime_quant == "int8"
@@ -386,6 +390,8 @@ def test_native_backend_keeps_codes_and_computes_the_dense_product(comfy_file):
"backend": "native",
"int8": 8,
"convrot": 8,
+ "fp8_backend": None,
+ "fp8": 0,
"dequantized": 2,
}
block = model.blocks[1]
@@ -484,6 +490,332 @@ def test_the_loader_refuses_what_the_scan_refuses(tmp_path):
)
+# ------------------------------------------------------------------------------------- fp8 runtime
+def _fp8_file(
+ tmp_path,
+ *,
+ fmt = "float8_e4m3fn",
+ per_row = False,
+ record = None,
+):
+ """A ComfyUI fp8 file of ``_Tiny`` (fused qkv, per-tensor or per-row ``weight_scale``) and its dense source."""
+ torch.manual_seed(2)
+ dense = _Tiny()
+ tensors = {}
+ fp8 = getattr(torch, fmt)
+ top = torch.finfo(fp8).max
+ for b, block in enumerate(dense.blocks):
+ layers = {
+ f"blocks.{b}.qkv": (
+ torch.cat([block.to_q.weight, block.to_k.weight, block.to_v.weight]).detach(),
+ torch.cat([block.to_q.bias, block.to_k.bias, block.to_v.bias]),
+ ),
+ f"blocks.{b}.out": (block.to_out.weight.detach(), block.to_out.bias),
+ f"blocks.{b}.adaLN_modulation.0": (
+ block.adaLN_modulation[0].weight.detach(),
+ block.adaLN_modulation[0].bias,
+ ),
+ }
+ for name, (weight, bias) in layers.items():
+ w = weight.float()
+ scale = (w.abs().amax(dim = 1, keepdim = True) if per_row else w.abs().max()) / top
+ tensors[f"{name}.weight"] = (w / scale).to(fp8)
+ tensors[f"{name}.weight_scale"] = scale.float()
+ tensors[f"{name}.input_scale"] = torch.tensor(1.0)
+ tensors[f"{name}.bias"] = bias.detach().clone()
+ tensors[f"{name}.comfy_quant"] = _conf(format = fmt, full_precision_matrix_mult = True)
+ tensors["norm.weight"] = dense.norm.weight.detach().clone()
+ tensors["norm.bias"] = dense.norm.bias.detach().clone()
+ meta = {cq.CONVERSION_RECORD_KEY: json.dumps(record)} if record else None
+ return _save(tmp_path / f"tiny_{fmt}.safetensors", tensors, metadata = meta), dense, tensors
+
+
+def _load_bf16(path, **kwargs):
+ kwargs.setdefault("int8_backend", None)
+ return cq.load_comfy_quant_transformer(
+ _Tiny,
+ path,
+ cq.refuse_comfy_quant(path),
+ {"torch_dtype": torch.bfloat16, "config": "base/repo", "subfolder": "transformer"},
+ family = "z-image",
+ **kwargs,
+ )
+
+
[email protected]("per_row", [False, True])
+def test_fp8_runtime_keeps_codes_and_repeats_a_tensor_scale_per_row(tmp_path, monkeypatch, per_row):
+ pytest.importorskip("torchao")
+ monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
+ path, _dense, tensors = _fp8_file(tmp_path, per_row = per_row)
+ model = _load_bf16(path, fp8_backend = "torchao")
+ assert model._unsloth_comfy_quant == {
+ "backend": None,
+ "int8": 0,
+ "convrot": 0,
+ "fp8_backend": "torchao",
+ "fp8": 10,
+ "dequantized": 0,
+ }
+ assert model._unsloth_runtime_quant == "fp8"
+ for b, block in enumerate(model.blocks):
+ codes = tensors[f"blocks.{b}.qkv.weight"]
+ scale = tensors[f"blocks.{b}.qkv.weight_scale"].reshape(-1, 1).expand(3 * DIM, 1)
+ for i, leaf in enumerate(("to_q", "to_k", "to_v")):
+ weight = getattr(block, leaf).weight
+ assert type(weight).__name__ == "Float8Tensor"
+ assert torch.equal(
+ weight.qdata.view(torch.uint8), codes[i * DIM : (i + 1) * DIM].view(torch.uint8)
+ )
+ assert torch.equal(weight.scale.float().reshape(-1, 1), scale[i * DIM : (i + 1) * DIM])
+ # the same per-row dynamic-activation layout as Studio's own FP8 checkpoints
+ assert weight.act_quant_kwargs.granularity.__class__.__name__ == "PerRow"
+ assert type(block.adaLN_modulation[0].weight).__name__ == "Float8Tensor"
+ assert not any(is_rotated_linear(m) for m in model.modules())
+
+
+def test_fp8_native_backend_keeps_codes_and_computes_the_dense_product(tmp_path, monkeypatch):
+ monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
+ path, dense, tensors = _fp8_file(tmp_path)
+ model = _load_bf16(path, fp8_backend = "native")
+ assert model._unsloth_comfy_quant["fp8_backend"] == "native"
+ assert model._unsloth_comfy_quant["fp8"] == 10
+ linear = model.blocks[1].to_k
+ assert type(linear).__name__ == "NativeWeightOnlyLinear" and linear.scheme == "fp8"
+ codes = tensors["blocks.1.qkv.weight"][DIM : 2 * DIM]
+ assert torch.equal(linear.weight_q, codes.view(torch.uint8))
+ x = torch.randn(4, DIM, dtype = torch.bfloat16)
+ want = torch.nn.functional.linear(
+ x.float(), dense.blocks[1].to_k.weight, dense.blocks[1].to_k.bias
+ )
+ assert _cos(linear(x), want) > 0.999
+
+
+def test_fp8_without_a_runtime_dequantizes(tmp_path, monkeypatch):
+ monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
+ path, _dense, tensors = _fp8_file(tmp_path)
+ model = _load_bf16(path, fp8_backend = None)
+ assert model._unsloth_comfy_quant["fp8"] == 0
+ assert model._unsloth_comfy_quant["dequantized"] == 6
+ w = tensors["blocks.0.out.weight"].float() * tensors["blocks.0.out.weight_scale"]
+ assert torch.equal(model.blocks[0].to_out.weight, w.to(torch.bfloat16))
+
+
+def test_e5m2_and_non_bf16_pipelines_dequantize(tmp_path, monkeypatch):
+ monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
+ path, _dense, _tensors = _fp8_file(tmp_path, fmt = "float8_e5m2")
+ assert _load_bf16(path, fp8_backend = "torchao")._unsloth_comfy_quant["fp8"] == 0
+ path, _dense, _tensors = _fp8_file(tmp_path)
+ model = _load(path, fp8_backend = "torchao") # float32 pipeline
+ assert model._unsloth_comfy_quant["fp8"] == 0
+ assert model._unsloth_comfy_quant["dequantized"] == 6
+
+
+ "native, supported, offload, dtype, env, want",
+ [
+ (None, "fp8", False, "bfloat16", None, "torchao"),
+ (None, "fp8", True, "bfloat16", None, None), # offloaded torchao fp8: dequantize, as before
+ (None, None, False, "bfloat16", None, None), # no fp8 GEMM (older GPU, CPU, MPS)
+ ("fp8", None, False, "bfloat16", None, "native"), # ROCm / stubbed torchao
+ (None, "fp8", False, "float16", None, None),
+ (None, "fp8", False, "bfloat16", "0", None), # kill switch
+ ],
+)
+def test_fp8_backend_follows_studios_own_fp8_rule(
+ monkeypatch, native, supported, offload, dtype, env, want
+):
+ import core.inference.diffusion_transformer_quant as tq
+
+ monkeypatch.setattr(tq, "native_quant_scheme", lambda *a, **k: native)
+ monkeypatch.setattr(tq, "select_transformer_quant_scheme", lambda *a, **k: supported)
+ if env is None:
+ monkeypatch.delenv(cq.COMFY_FP8_ENV, raising = False)
+ else:
+ monkeypatch.setenv(cq.COMFY_FP8_ENV, env)
+ target = types.SimpleNamespace(device = "cuda", dtype = getattr(torch, dtype))
+ assert cq.comfy_fp8_backend(target, "z-image", offload = offload) == want
+
+
+# ------------------------------------------------------------------------------------- hosted prequant route
+def test_comfy_prequant_scheme_is_read_from_the_header(tmp_path, comfy_file):
+ path, _, _ = comfy_file
+ assert cq.comfy_prequant_scheme(cq.scan_comfy_quant(path)) == "int8"
+ fp8_path, _, _ = _fp8_file(tmp_path)
+ assert cq.comfy_prequant_scheme(cq.scan_comfy_quant(fp8_path)) == "fp8"
+ from core.inference.diffusion_prequant import local_prequant_scheme
+
+ assert local_prequant_scheme(fp8_path) == "fp8"
+ assert local_prequant_scheme(path) == "int8"
+
+
+def test_comfy_prequant_refuses_another_scheme_or_base(tmp_path, monkeypatch):
+ monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
+ path, _, _ = _fp8_file(tmp_path, record = {"base_model_id": "org/other-model", "scheme": "fp8"})
+ kw = dict(base = "base/repo", family = "z-image", dtype = torch.bfloat16)
+ with pytest.raises(ValueError, match = "not a int8 checkpoint"):
+ cq.load_comfy_prequant(_Tiny, path, scheme = "int8", **kw)
+ with pytest.raises(ValueError, match = "converted from org/other-model"):
+ cq.load_comfy_prequant(_Tiny, path, scheme = "fp8", **kw)
+ with pytest.raises(ValueError, match = "needs a bfloat16 pipeline"):
+ cq.load_comfy_prequant(
+ _Tiny, path, scheme = "fp8", **dict(kw, dtype = torch.float32, base = "org/other-model")
+ )
+
+
+def test_hosted_loader_rebuilds_a_comfy_file_into_studios_int8_weights(
+ comfy_file, monkeypatch, tmp_path
+):
+ """``load_prequantized_transformer`` on a ComfyUI file: Int8Tensor weights under ConvRot, the loaded file
+ recorded, the same placement tail as Studio's own checkpoints."""
+ pytest.importorskip("torchao")
+ import core.inference.diffusion_prequant as pq
+
+ path, _dense, tensors = comfy_file
+ monkeypatch.setenv(pq.ALLOW_LOCAL_PREQUANT_PATH_ENV, str(tmp_path))
+ source = pq.PrequantSource(kind = "path", location = path)
+ model = pq.load_prequantized_transformer(
+ _Tiny,
+ "base/repo",
+ source,
+ device = "cpu",
+ dtype = torch.float32,
+ scheme = "int8",
+ family = "z-image",
+ )
+ assert model is not None, pq.last_prequant_failure()
+ assert model._unsloth_prequant_path == path
+ assert model._unsloth_runtime_quant == "int8"
+ weight = model.blocks[0].to_out.weight
+ assert type(getattr(weight, "data", weight)).__name__ == "Int8Tensor"
+ assert torch.equal(weight.qdata, tensors["blocks.0.out.weight"])
+ assert is_rotated_linear(model.blocks[0].to_out)
+ # Studio's int8 filter keeps the modulation projection dense, as its own INT8 checkpoints do
+ assert type(model.blocks[0].adaLN_modulation[0].weight).__name__ in ("Parameter", "Tensor")
+ # asked for the wrong scheme: refused, with the reason recorded, so the caller quantizes dense instead
+ assert (
+ pq.load_prequantized_transformer(
+ _Tiny,
+ "base/repo",
+ source,
+ device = "cpu",
+ dtype = torch.float32,
+ scheme = "fp8",
+ family = "z-image",
+ )
+ is None
+ )
+ assert "not a fp8 checkpoint" in (pq.last_prequant_failure() or "")
+
+
+def test_resident_size_is_priced_from_the_header_not_the_name(tmp_path, comfy_file):
+ """Kept int8 / fp8 layers cost their stored bytes, dequantized ones twice that; the file name is not read."""
+ int8_path, _, tensors = comfy_file
+ scan = cq.scan_comfy_quant(int8_path)
+ quant = sum(tensors[f"{n}.weight"].numel() for n in scan.layers)
+ plain = sum(
+ v.numel()
+ for k, v in tensors.items()
+ if v.is_floating_point() and k.endswith((".bias", "norm.weight", "norm.bias"))
+ )
+ kept = cq.comfy_resident_mib(int8_path, scan, keep_int8 = True, keep_fp8 = True)
+ dequant = cq.comfy_resident_mib(int8_path, scan, keep_int8 = False, keep_fp8 = True)
+ assert dequant - kept == pytest.approx(quant / 2**20, abs = 1)
+ assert kept >= (quant + 2 * plain) / 2**20
+ fp8_path, _, _ = _fp8_file(tmp_path)
+ fp8_scan = cq.scan_comfy_quant(fp8_path)
+ assert cq.comfy_resident_mib(
+ fp8_path, fp8_scan, keep_int8 = False, keep_fp8 = True
+ ) < cq.comfy_resident_mib(fp8_path, fp8_scan, keep_int8 = False, keep_fp8 = False)
+ assert (
+ cq.comfy_resident_mib(str(tmp_path / "missing.safetensors"), keep_int8 = True, keep_fp8 = True)
+ is None
+ )
+
+
+def test_the_row_map_is_traced_on_narrow_tags_and_falls_back_to_full_width(comfy_file, monkeypatch):
+ """The fast pass hands the converter 4-column tags; a converter that needs the real width (here: one that
+ reshapes by it) gets a second, full-width pass and the same result; one that drops columns is refused."""
+ pytest.importorskip("torchao")
+ path, _, tensors = comfy_file
+ widths = []
+
+ def _spy(
+ checkpoint = None,
+ config = None,
+ **kw,
+ ):
+ widths.append(checkpoint["blocks.0.qkv.weight"].shape[1])
+ return _convert(checkpoint = checkpoint, config = config, **kw)
+
+ monkeypatch.setattr(cq, "_mapping", lambda cls: (_spy, _sfm()))
+ fast = _load(path, int8_backend = "torchao")
+ assert widths == [cq._NARROW_TAG_COLUMNS]
+
+ def _needs_width(
+ checkpoint = None,
+ config = None,
+ **kw,
+ ):
+ w = checkpoint["blocks.0.qkv.weight"]
+ checkpoint["blocks.0.qkv.weight"] = w.reshape(-1, DIM) # only valid at the real width
+ return _convert(checkpoint = checkpoint, config = config, **kw)
+
+ monkeypatch.setattr(cq, "_mapping", lambda cls: (_needs_width, _sfm()))
+ full = _load(path, int8_backend = "torchao")
+ for a, b in zip(fast.state_dict().values(), full.state_dict().values()):
+ a, b = getattr(a, "qdata", a), getattr(b, "qdata", b)
+ assert torch.equal(a, b)
+
+ def _drops_columns(
+ checkpoint = None,
+ config = None,
+ **kw,
+ ):
+ out = _convert(checkpoint = checkpoint, config = config, **kw)
+ out["blocks.0.to_out.weight"] = out["blocks.0.to_out.weight"].chunk(2, dim = 1)[0]
+ return out
+
+ monkeypatch.setattr(cq, "_mapping", lambda cls: (_drops_columns, _sfm()))
+ with pytest.raises(ValueError, match = "column count"):
+ _load(path, int8_backend = "torchao")
+
+
+def test_the_hosted_route_registers_studios_single_file_converters(monkeypatch):
+ """Qwen-Image-2.1's converter is Studio's own, registered by the single-file branch right before its call. The
+ hosted prequant route never passes through there, so the row map must register it itself (it used to fail with
+ "has no single-file converter" and fall back to quantizing the dense weights)."""
+ diffusers = pytest.importorskip("diffusers")
+ cls = getattr(diffusers, "QwenImage21Transformer2DModel", None)
+ if cls is None:
+ pytest.skip("this diffusers has no QwenImage21Transformer2DModel")
+ from diffusers.loaders import single_file_model as sfm
+
+ monkeypatch.setattr(
+ sfm,
+ "SINGLE_FILE_LOADABLE_CLASSES",
+ {k: v for k, v in sfm.SINGLE_FILE_LOADABLE_CLASSES.items() if k != cls.__name__},
+ )
+ mapping_fn, _ = cq._mapping(cls)
+ assert callable(mapping_fn)
+
+
+def test_a_local_comfy_override_is_usable_only_for_an_image_family(tmp_path, monkeypatch):
+ """A local ComfyUI-format override reads its scheme from the header for an image family; a video family's loader
+ cannot read the layout, so planning must not count on it there."""
+ import core.inference.diffusion_prequant as pq
+ from core.inference.diffusion_families import detect_family
+ from core.inference.video_families import _FAMILIES as VIDEO_FAMILIES
+
+ path, _, _ = _fp8_file(tmp_path)
+ monkeypatch.setenv(pq.ALLOW_LOCAL_PREQUANT_PATH_ENV, str(tmp_path))
+ monkeypatch.setattr(pq, "restricted_prequant_load_supported", lambda *a, **k: True)
+ image = detect_family("Tongyi-MAI/Z-Image-Turbo")
+ assert pq.usable_prequant_source(image, "fp8", path_override = path) is not None
+ assert pq.usable_prequant_source(image, "int8", path_override = path) is None
+ video = next(f for f in VIDEO_FAMILIES if any(s == "fp8" for s, _r in (f.prequant_repos or ())))
+ assert pq.usable_prequant_source(video, "fp8", path_override = path) is None
+
+
def test_only_torchao_comfy_loads_ask_for_compile(comfy_file):
path, _, _ = comfy_file
assert not cq.comfy_torchao_quantized(_load(path))
diff --git a/studio/backend/tests/test_diffusion_int8_runtime_convrot.py b/studio/backend/tests/test_diffusion_int8_runtime_convrot.py
index 6f960d3956d..84afa1c224d 100644
--- a/studio/backend/tests/test_diffusion_int8_runtime_convrot.py
+++ b/studio/backend/tests/test_diffusion_int8_runtime_convrot.py
@@ -14,6 +14,14 @@ import core.inference.diffusion_transformer_quant as tq
from core.inference.diffusion_convrot import CONVROT_ATTR, is_rotated_linear
+def candidate_filenames_of(source):
+ """Studio's own containers in resolver order; the ComfyUI-format twins have their own tests."""
+ from core.inference.diffusion_prequant import candidate_filenames_of as names_of
+ from core.inference.diffusion_prequant import is_comfy_prequant_filename
+
+ return tuple(n for n in names_of(source) if not is_comfy_prequant_filename(n))
+
+
@pytest.fixture(autouse = True)
def _convrot_opted_in(monkeypatch):
monkeypatch.setenv(tq.INT8_CONVROT_ENV, "1")
@@ -108,7 +116,7 @@ def test_convrot_spec_only_for_int8_on_declared_families():
def test_int8_artifact_names_the_rotated_build_first_and_keeps_the_plain_one():
from core.inference.diffusion_families import detect_family
- from core.inference.diffusion_prequant import candidate_filenames_of, resolve_prequant_source
+ from core.inference.diffusion_prequant import resolve_prequant_source
fam = detect_family("Qwen/Qwen-Image-2.1", override = "qwen-image-2.1")
names = candidate_filenames_of(resolve_prequant_source(fam, "int8"))
@@ -125,7 +133,7 @@ def test_int8_artifact_names_the_rotated_build_first_and_keeps_the_plain_one():
def test_without_the_opt_in_int8_is_plain_everywhere(monkeypatch):
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
from core.inference.diffusion_families import detect_family
- from core.inference.diffusion_prequant import candidate_filenames_of, resolve_prequant_source
+ from core.inference.diffusion_prequant import resolve_prequant_source
fam = detect_family("Qwen/Qwen-Image-2.1", override = "qwen-image-2.1")
names = candidate_filenames_of(resolve_prequant_source(fam, "int8"))
@@ -304,7 +312,9 @@ def test_unreachable_hub_still_loads_the_plain_artifact_already_cached(monkeypat
detect_family("Qwen/Qwen-Image-2.1", override = "qwen-image-2.1"), "int8"
)
assert pq._resolve_checkpoint_path(source, None, None) == str(plain)
- assert asked == ["Qwen-Image-2.1-INT8-ConvRot.safetensors"]
+ # only the first name is asked (the ComfyUI-format twin of the rotated build); the unreachable Hub then
+ # answers from the cache
+ assert asked == ["Qwen-Image-2.1-INT8-ConvRot-ComfyUI.safetensors"]
plain.unlink()
with pytest.raises(LocalEntryNotFoundError):
pq._resolve_checkpoint_path(source, None, None)
@@ -446,7 +456,7 @@ def test_zimage_int8_resolves_the_rotated_artifact_first_unless_killed(
monkeypatch, env, rotated_first
):
from core.inference.diffusion_families import detect_family
- from core.inference.diffusion_prequant import candidate_filenames_of, resolve_prequant_source
+ from core.inference.diffusion_prequant import resolve_prequant_source
if env is None:
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
@@ -503,7 +513,7 @@ def test_zimage_rotated_artifact_is_only_named_in_its_own_repo(monkeypatch):
import dataclasses
from core.inference.diffusion_families import detect_family
- from core.inference.diffusion_prequant import candidate_filenames_of, resolve_prequant_source
+ from core.inference.diffusion_prequant import resolve_prequant_source
monkeypatch.delenv(tq.INT8_CONVROT_ENV)
fam = dataclasses.replace(
diff --git a/studio/backend/tests/test_diffusion_more_families.py b/studio/backend/tests/test_diffusion_more_families.py
index 286049f50d5..64270515022 100644
--- a/studio/backend/tests/test_diffusion_more_families.py
+++ b/studio/backend/tests/test_diffusion_more_families.py
@@ -435,7 +435,14 @@ def test_qwen_image_2512_prequant_filenames_match_its_repo():
assert source is not None
assert source.location == "unsloth/Qwen-Image-2512-FP8"
names = list(candidate_filenames_of(source))
- assert names[0] == safetensors_name, names
+ # the int8 ComfyUI twin (Studio's int8 codes bit for bit) leads; Studio's own containers follow unchanged
+ own = [n for n in names if not n.endswith("-ComfyUI.safetensors")]
+ assert own[0] == safetensors_name, names
+ assert names[0] == (
+ safetensors_name.replace(".safetensors", "-ComfyUI.safetensors")
+ if scheme == "int8"
+ else safetensors_name
+ ), names
assert pickle_name in names[1:], names
# And the legacy repo-agnostic spelling stays last, for a repo predating the model-named one.
assert names[-1] == f"transformer_{scheme}.pt", names
diff --git a/studio/backend/tests/test_diffusion_prequant.py b/studio/backend/tests/test_diffusion_prequant.py
index 10f362605a0..62121b990e4 100644
--- a/studio/backend/tests/test_diffusion_prequant.py
+++ b/studio/backend/tests/test_diffusion_prequant.py
@@ -69,9 +69,10 @@ def test_resolve_family_repo_by_scheme():
fam = _fam(prequant_repos = (("fp8", "org/hosted-fp8"), ("int8", "org/hosted-int8")))
src = resolve_prequant_source(fam, "int8")
assert src.kind == "repo" and src.location == "org/hosted-int8"
- # Model-name convention first (repo scheme suffix stripped), safetensors ahead of the pickle,
- # legacy name last.
+ # Model-name convention first (repo scheme suffix stripped), its ComfyUI-format twin ahead of it,
+ # safetensors ahead of the pickle, legacy name last.
assert src.candidate_filenames == (
+ "hosted-INT8-ComfyUI.safetensors",
"hosted-INT8.safetensors",
"hosted-INT8.pt",
"transformer_int8.pt",
@@ -110,7 +111,8 @@ def test_resolve_variant_base_picks_variant_repo():
)
src = resolve_prequant_source(fam, "int8", base_repo = "Org/Model-DEV")
assert src.kind == "repo" and src.location == "org/dev-fp8"
- assert src.filename == "dev-INT8.safetensors"
+ assert src.filename == "dev-INT8-ComfyUI.safetensors"
+ assert src.fallback_filenames[0] == "dev-INT8.safetensors"
def test_resolve_variant_base_falls_back_to_default():
@@ -151,8 +153,10 @@ def test_resolve_prefers_a_family_declared_filename():
fam = _fam(prequant_repos = (("int8", "unsloth/Model-FP8"),))
fam = dataclasses.replace(fam, prequant_filenames = (("int8", "Model-INT8-ConvRot.pt"),))
src = resolve_prequant_source(fam, "int8")
- assert src.filename == "Model-INT8-ConvRot.pt"
+ assert src.filename == "Model-INT8-ConvRot-ComfyUI.safetensors"
assert src.fallback_filenames == (
+ "Model-INT8-ConvRot.pt",
+ "Model-INT8-ComfyUI.safetensors",
"Model-INT8.safetensors",
"Model-INT8.pt",
"transformer_int8.pt",
@@ -164,6 +168,7 @@ def test_resolve_prefers_a_family_declared_filename():
assert other.candidate_filenames == (
"Model-FP8.safetensors",
"Model-FP8.pt",
+ "Model-FP8-ComfyUI.safetensors",
"transformer_fp8.pt",
)
@@ -2896,3 +2901,125 @@ def test_a_full_fingerprint_pass_is_remembered_per_unchanged_file(monkeypatch, t
assert pq._verify_packed_fingerprint(expected, meta)
assert pq._verify_packed_fingerprint(expected, meta)
assert len(calls) == 5
+
+
+# ---- ComfyUI-format twins in the hosted chain ----
+
+
+def _own_names(names):
+ from core.inference.diffusion_prequant import is_comfy_prequant_filename
+ return tuple(n for n in names if not is_comfy_prequant_filename(n))
+
+
+ "repo, family, scheme, env, own",
+ [
+ ("Tongyi-MAI/Z-Image-Turbo", None, "int8", {}, "Z-Image-Turbo-INT8-ConvRot.safetensors"),
+ ("Tongyi-MAI/Z-Image-Turbo", None, "fp8", {}, "Z-Image-Turbo-FP8.safetensors"),
+ (
+ "Qwen/Qwen-Image-2.1",
+ "qwen-image-2.1",
+ "int8",
+ {"UNSLOTH_DIFFUSION_INT8_CONVROT": "1"},
+ "Qwen-Image-2.1-INT8-ConvRot.safetensors",
+ ),
+ ("Qwen/Qwen-Image-2.1", "qwen-image-2.1", "int8", {}, "Qwen-Image-2.1-INT8.safetensors"),
+ ("Qwen/Qwen-Image-2.1", "qwen-image-2.1", "fp8", {}, "Qwen-Image-2.1-FP8.safetensors"),
+ ("black-forest-labs/FLUX.2-klein-4B", None, "fp8", {}, "FLUX.2-klein-4B-FP8.safetensors"),
+ ],
+)
+def test_image_families_resolve_the_comfy_twin_and_keep_every_old_name(
+ monkeypatch, repo, family, scheme, env, own
+):
+ """New builds also ask for ``<stem>-ComfyUI.safetensors``: AHEAD of the artifact for int8 (the twin holds the
+ same codes and scales), BEHIND it for fp8 (ComfyUI's per-tensor fp8 is a coarser rounding). Older builds never
+ ask for it, and the chain without it is exactly what the kill switch (and an older build) resolves."""
+ from core.inference.diffusion_families import detect_family
+ from core.inference.diffusion_prequant import COMFY_PREQUANT_ENV, comfy_prequant_filename
+
+ for k, v in env.items():
+ monkeypatch.setenv(k, v)
+ fam = detect_family(repo, override = family) if family else detect_family(repo)
+ names = resolve_prequant_source(fam, scheme).candidate_filenames
+ twin = comfy_prequant_filename(own)
+ monkeypatch.setenv(COMFY_PREQUANT_ENV, "0")
+ old = resolve_prequant_source(fam, scheme).candidate_filenames
+ assert _own_names(names) == old
+ assert old[0] == own
+ if scheme == "int8":
+ assert names[0] == twin
+ # every artifact gets its twin right ahead of its first container
+ for name in old:
+ other = comfy_prequant_filename(name)
+ if other is not None and not name.endswith(".pt"):
+ assert names.index(other) == names.index(name) - 1
+ else:
+ stems = [n for n in names if n.startswith(own[: -len(".safetensors")] + ".")]
+ assert names.index(twin) == names.index(stems[-1]) + 1
+
+
+def test_comfy_twin_names():
+ from core.inference.diffusion_prequant import (
+ comfy_prequant_filename,
+ is_comfy_prequant_filename,
+ with_comfy_twins,
+ )
+
+ assert (
+ comfy_prequant_filename("Z-Image-Turbo-FP8.safetensors")
+ == "Z-Image-Turbo-FP8-ComfyUI.safetensors"
+ )
+ assert (
+ comfy_prequant_filename("Z-Image-Turbo-INT8.pt") == "Z-Image-Turbo-INT8-ComfyUI.safetensors"
+ )
+ # the legacy names, nested paths and the twin itself have none
+ assert comfy_prequant_filename("transformer_int8.pt") is None
+ assert comfy_prequant_filename("text_encoders/x.safetensors") is None
+ assert comfy_prequant_filename("A-FP8-ComfyUI.safetensors") is None
+ assert is_comfy_prequant_filename("A-FP8-ComfyUI.safetensors")
+ assert not is_comfy_prequant_filename("A-FP8.safetensors")
+ assert with_comfy_twins(["A-INT8.safetensors", "A-INT8.pt", "transformer_int8.pt"]) == [
+ "A-INT8-ComfyUI.safetensors",
+ "A-INT8.safetensors",
+ "A-INT8.pt",
+ "transformer_int8.pt",
+ ]
+ assert with_comfy_twins(
+ ["A-FP8.safetensors", "A-FP8.pt", "transformer_fp8.pt"], lead = False
+ ) == [
+ "A-FP8.safetensors",
+ "A-FP8.pt",
+ "A-FP8-ComfyUI.safetensors",
+ "transformer_fp8.pt",
+ ]
+ # a pickle-only chain still gets the twin ahead of it; a declared twin is not duplicated
+ assert with_comfy_twins(["A-INT8.pt"]) == ["A-INT8-ComfyUI.safetensors", "A-INT8.pt"]
+ assert with_comfy_twins(["A-INT8-ComfyUI.safetensors", "A-INT8.safetensors"]) == [
+ "A-INT8-ComfyUI.safetensors",
+ "A-INT8.safetensors",
+ ]
+
+
+def test_video_families_get_no_comfy_twin():
+ """Only the image denoiser loader reads the ComfyUI layout; video chains are unchanged."""
+ from core.inference.diffusion_prequant import is_comfy_prequant_filename
+ from core.inference.video_families import _FAMILIES as VIDEO_FAMILIES
+
+ checked = 0
+ for fam in VIDEO_FAMILIES:
+ for scheme, _repo in getattr(fam, "prequant_repos", ()) or ():
+ src = resolve_prequant_source(fam, scheme)
+ if src is None:
+ continue
+ checked += 1
+ assert not any(is_comfy_prequant_filename(n) for n in src.candidate_filenames)
+ assert checked
+
+
+def test_task_specific_artifacts_get_no_twin():
+ fam = dataclasses.replace(
+ _fam(prequant_repos = (("int8", "unsloth/Model-FP8"),)),
+ prequant_filenames = (("int8", "keyframe", "Model-KF-INT8.safetensors"),),
+ )
+ src = resolve_prequant_source(fam, "int8", task = "keyframe")
+ assert src.candidate_filenames == ("Model-KF-INT8.safetensors",)
diff --git a/studio/backend/tests/test_prequant_prefer_cached_pickle.py b/studio/backend/tests/test_prequant_prefer_cached_pickle.py
index d17df14d708..806a957fccc 100644
--- a/studio/backend/tests/test_prequant_prefer_cached_pickle.py
+++ b/studio/backend/tests/test_prequant_prefer_cached_pickle.py
@@ -667,3 +667,126 @@ def test_a_local_files_only_load_probes_like_an_offline_one(hub, monkeypatch):
assert _load(local_files_only = True) is True and seen["inner"] is True
assert _load(local_files_only = False) is False
assert pq.prequant_checkpoint_cached(_convrot_dit()) is False
+
+
+# ---- ComfyUI-format twin: the default download, never a re-download of a cached artifact ----
+
+COMFY = "Model-INT8-ComfyUI.safetensors"
+
+
+def _comfy_dit():
+ return _dit(COMFY, "Model-INT8.safetensors", "Model-INT8.pt", "transformer_int8.pt")
+
+
+def test_new_user_downloads_the_comfy_twin(hub):
+ assert _resolve(_comfy_dit()) == hub.cached[(REPO, COMFY)]
+ assert hub.fetched == [COMFY]
+
+
+def test_cached_own_safetensors_is_kept_over_the_comfy_twin(hub, caplog):
+ hub.cache("Model-INT8.safetensors")
+ caplog.set_level("INFO")
+ assert _resolve(_comfy_dit()) == hub.cached[(REPO, "Model-INT8.safetensors")]
+ assert COMFY not in hub.fetched
+ assert "its ComfyUI-format twin" in caplog.text
+
+
+def test_cached_own_pickle_is_kept_over_both_twins(hub):
+ hub.cache("Model-INT8.pt")
+ assert pq.prefer_cached_pickle_twins(REPO, _comfy_dit().candidate_filenames) == [
+ "Model-INT8.pt",
+ "Model-INT8.safetensors",
+ COMFY,
+ "transformer_int8.pt",
+ ]
+ assert _resolve(_comfy_dit()) == hub.cached[(REPO, "Model-INT8.pt")]
+ assert hub.fetched == ["Model-INT8.pt"]
+
+
+def test_cached_own_pickle_offline_never_asks_for_the_twin(hub):
+ hub.cache("Model-INT8.pt")
+ assert _resolve(_comfy_dit(), local_files_only = True) == hub.cached[(REPO, "Model-INT8.pt")]
+ assert hub.downloads == [("Model-INT8.pt", True)]
+
+
+def test_cached_comfy_twin_stays_first(hub):
+ hub.cache(COMFY)
+ hub.cache("Model-INT8.pt")
+ assert _resolve(_comfy_dit()) == hub.cached[(REPO, COMFY)]
+ assert hub.fetched == [COMFY]
+
+
+def test_repo_without_the_twin_falls_back_to_the_old_names(hub):
+ """A repo that does not host the twin yet (or an older publish): one 404, then today's artifact."""
+ hub.hosted = {"Model-INT8.safetensors", "Model-INT8.pt"}
+ assert _resolve(_comfy_dit()) == hub.cached[(REPO, "Model-INT8.safetensors")]
+ assert hub.fetched == [COMFY, "Model-INT8.safetensors"]
+
+
+def test_a_cached_other_artifact_does_not_demote_the_twin(hub):
+ """INT8 cached does not stand in for the INT8-ConvRot twin (different weights); only its own twin moves."""
+ hub.cache("Model-INT8.safetensors")
+ names = (
+ "Model-INT8-ConvRot-ComfyUI.safetensors",
+ "Model-INT8-ConvRot.safetensors",
+ COMFY,
+ "Model-INT8.safetensors",
+ )
+ assert pq.prefer_cached_pickle_twins(REPO, names) == [
+ "Model-INT8-ConvRot-ComfyUI.safetensors",
+ "Model-INT8-ConvRot.safetensors",
+ "Model-INT8.safetensors",
+ COMFY,
+ ]
+
+
+def test_cache_probe_reads_a_cached_own_file_as_free_online(hub):
+ """The planner's no-download answer: the cached own file, not 'must fetch the twin'."""
+ hub.cache("Model-INT8.safetensors")
+ src = _comfy_dit()
+ assert pq.cached_checkpoint_path(src) == hub.cached[(REPO, "Model-INT8.safetensors")]
+ assert pq.cached_checkpoint_path(src) == _resolve(src)
+
+
+def test_dit_planner_stages_the_twin_only_when_nothing_is_cached(hub, monkeypatch):
+ from core.inference.diffusion import DiffusionBackend
+
+ api = _Api([COMFY, "Model-INT8.safetensors", "Model-INT8.pt"])
+ monkeypatch.setattr(huggingface_hub, "HfApi", lambda token = None: api)
+ assert DiffusionBackend._prequant_source_hub_entry(_comfy_dit(), None, scheme = "int8") == (
+ REPO,
+ COMFY,
+ 100,
+ )
+ hub.cache("Model-INT8.safetensors")
+ assert DiffusionBackend._prequant_source_hub_entry(_comfy_dit(), None, scheme = "int8") == (
+ REPO,
+ "Model-INT8.safetensors",
+ 101,
+ )
+ # a repo listing without the twin stages today's artifact
+ hub.cached.clear()
+ api.names = ["Model-INT8.safetensors", "Model-INT8.pt"]
+ assert DiffusionBackend._prequant_source_hub_entry(_comfy_dit(), None, scheme = "int8") == (
+ REPO,
+ "Model-INT8.safetensors",
+ 100,
+ )
+
+
+def test_a_cached_trailing_fp8_twin_is_used_before_an_uncached_own_file(hub):
+ """fp8 twins trail Studio's own artifact; one that is already cached still wins over downloading the other."""
+ fp8 = _dit(
+ "Model-FP8.safetensors",
+ "Model-FP8.pt",
+ "Model-FP8-ComfyUI.safetensors",
+ "transformer_fp8.pt",
+ )
+ hub.cache("Model-FP8-ComfyUI.safetensors")
+ assert _resolve(fp8) == hub.cached[(REPO, "Model-FP8-ComfyUI.safetensors")]
+ assert hub.fetched == ["Model-FP8-ComfyUI.safetensors"]
+ # nothing cached: Studio's own per-row fp8 is the download
+ hub.cached.clear()
+ hub.downloads.clear()
+ assert _resolve(fp8) == hub.cached[(REPO, "Model-FP8.safetensors")]
+ assert hub.fetched == ["Model-FP8.safetensors"]
diff --git a/studio/backend/tests/test_prequant_safetensors.py b/studio/backend/tests/test_prequant_safetensors.py
index 8e0737ce2d6..81e342d49fd 100644
--- a/studio/backend/tests/test_prequant_safetensors.py
+++ b/studio/backend/tests/test_prequant_safetensors.py
@@ -150,6 +150,7 @@ def test_the_derived_chain_puts_safetensors_first_and_keeps_the_pickles(monkeypa
assert src.candidate_filenames == (
"Model-FP8.safetensors",
"Model-FP8.pt",
+ "Model-FP8-ComfyUI.safetensors",
"transformer_fp8.pt",
)