Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: keep fp8 single-file weights stored in fp8 where widening would offload or crowd host RAM (#14549)

Daniel Han4ff4c6689a21
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index d556059032c..5ec8875f948 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -382,9 +382,14 @@ from .diffusion_comfy_block import (
comfy_nvfp4_runtime_possible,
)
from .diffusion_comfy_quant import (
+ header_has_fp8_weights,
comfy_fp8_backend,
+ comfy_fp8_storage,
comfy_int8_backend,
comfy_torchao_quantized,
+ fp8_storage_wanted,
+ fp8_stored_layers,
+ pipe_fp8_stored_layers,
load_comfy_quant_transformer,
refuse_comfy_quant,
)
@@ -2884,11 +2889,14 @@ def _denoiser_hooked(pipe: Any) -> bool:
def _torchao_render_needs_no_grad(state: Any) -> bool:
- """inference_mode rejects moving torchao weights between devices, so offloaded quant renders use no_grad."""
+ """inference_mode rejects moving torchao weights between devices, so offloaded quant renders use no_grad.
+ Stored fp8 codes too: moved under inference_mode they become unversionable, re-copied to host every offload."""
if getattr(state, "offload_policy", OFFLOAD_NONE) == OFFLOAD_NONE:
return False
return bool(
- getattr(state, "transformer_quant", None) or getattr(state, "text_encoder_quant", None)
+ getattr(state, "transformer_quant", None)
+ or getattr(state, "text_encoder_quant", None)
+ or pipe_fp8_stored_layers(getattr(state, "pipe", None))
)
@@ -3122,6 +3130,28 @@ def _prequant_bakes_auto_accumulate(source: Any) -> bool:
return False
+def _single_file_fp8_upcast(single_file_path: Any, fam: Any) -> bool:
+ """Whether a single-file denoiser stores fp8 weights a widened load upcasts (~2x resident). Read from the header
+ (the basename only when it is unreadable); SDXL single-file pipelines are already bf16."""
+ if not single_file_path or getattr(fam, "single_file_is_pipeline", False):
+ return False
+ stored = header_has_fp8_weights(str(single_file_path))
+ return "fp8" in Path(single_file_path).name.lower() if stored is None else stored
+
+
+def _comfy_keep_fp32_pricing(fam: Any, target: Any) -> dict:
+ """``comfy_resident_mib`` kwargs for the DiT's ``_keep_in_fp32_modules``: never stored, fp32 on an fp16 load."""
+ import torch
+
+ from .diffusion_comfy_quant import keep_fp32_modules_for
+
+ keep = keep_fp32_modules_for(fam)
+ if not keep:
+ return {}
+ fp16 = getattr(target, "dtype", None) is torch.float16
+ return {"keep_fp32": keep, "keep_fp32_bytes": 4 if fp16 else None}
+
+
class DiffusionBackend:
"""Holds at most one loaded diffusers pipeline. All mutations are serialised."""
@@ -8751,7 +8781,7 @@ class DiffusionBackend:
_install_gguf_dim_restore(logger)
if comfy_scan is not None:
# int8 / fp8 codes go to Studio's runtime unchanged where it runs; the rest dequantize.
- _comfy_offload = not plan_keeps_transformer_resident(plan)
+ _comfy_offload = self._comfy_offload_for(plan, single_file_path)
transformer = load_comfy_quant_transformer(
transformer_cls,
single_file_path,
@@ -8781,6 +8811,9 @@ class DiffusionBackend:
target = target,
fast_accum = transformer_quant_fast_accum,
logger = logger,
+ fp8_storage = self._fp8_storage_for(
+ single_file_path, target, loras
+ ),
)
comfy_compile = comfy_torchao_quantized(transformer)
else:
@@ -8791,7 +8824,13 @@ class DiffusionBackend:
== "Krea2Transformer2DModel"
):
transformer = load_original_layout_transformer(
- transformer_cls, single_file_path, sf_kwargs, logger
+ transformer_cls,
+ single_file_path,
+ sf_kwargs,
+ logger,
+ fp8_storage = self._fp8_storage_for(
+ single_file_path, target, loras
+ ),
)
else:
# A safetensors single-file (fp8) carries its own dtype: no GGUF dequant config.
@@ -10079,6 +10118,18 @@ class DiffusionBackend:
except Exception: # noqa: BLE001 - an install hint only: the load decides on its own
return False
+ def _comfy_offload_for(self, plan: Any, single_file_path: Any) -> bool:
+ """Probe the ComfyUI backends as offloaded: the plan offloads, or it was priced with the offloaded backends."""
+ priced = getattr(self, "_comfy_priced_offload", None) or {}
+ return not plan_keeps_transformer_resident(plan) or bool(priced.get(str(single_file_path)))
+
+ def _fp8_storage_for(self, single_file_path: Any, target: Any, loras: Any) -> bool:
+ """The planner's ``fp8_storage_wanted`` pick for this file, re-checked against device / LoRA / env."""
+ picks = getattr(self, "_fp8_storage_picks", None) or {}
+ return bool(picks.get(str(single_file_path))) and comfy_fp8_storage(
+ target, lora = _has_active_lora(loras)
+ )
+
@staticmethod
def _comfy_single_file_resident_mib(
single_file_path: Optional[str],
@@ -10087,8 +10138,11 @@ class DiffusionBackend:
base: Optional[str],
*,
lora: bool = False,
+ fp8_storage: bool = False,
+ offload: bool = False,
) -> Optional[int]:
- """``comfy_resident_mib`` for a ComfyUI-quantized single file under a resident plan, else None."""
+ """``comfy_resident_mib`` for a ComfyUI-quantized single file, else None. ``offload`` prices the backends an
+ offloaded load keeps (the loader probes them the same way)."""
try:
from .diffusion_comfy_quant import comfy_resident_mib, scan_comfy_quant
from .diffusion_transformer_quant import (
@@ -10106,8 +10160,10 @@ class DiffusionBackend:
return comfy_resident_mib(
single_file_path,
scan,
- keep_int8 = comfy_int8_backend(target, name, base) is not None,
- keep_fp8 = comfy_fp8_backend(target, name, base) is not None,
+ keep_int8 = comfy_int8_backend(target, name, base, offload = offload) is not None,
+ keep_fp8 = comfy_fp8_backend(target, name, base, offload = offload) is not None,
+ fp8_stored = fp8_storage and comfy_fp8_storage(target, lora = lora),
+ **_comfy_keep_fp32_pricing(fam, target),
# the loader's runtime filter: layers it skips are priced dequantized
min_features = DEFAULT_MIN_LINEAR_FEATURES,
fp8_divisible = divisible_for_scheme(TQ_FP8),
@@ -11194,6 +11250,11 @@ class DiffusionBackend:
pre-cast checkpoint's size, which base-repo scans never see.
"""
# Settled (max-over-reads) on cuda: a transient foreign allocation would make an empty card look full
+ single_file_fp8_stored_mib: Optional[int] = None
+ single_file_resident_at: Any = (
+ None # (fp8_storage, offload) -> MiB, set for a single-file DiT
+ )
+ single_file_stored_at: Any = None
device_memory = (
device_memory_override
if device_memory_override is not None
@@ -11312,21 +11373,46 @@ class DiffusionBackend:
# Planning the dense-quant candidate: the auto-policy estimate replaces the file-size derivation.
transformer_resident = transformer_resident_override_mib
elif kind == "single_file":
- # An fp8 checkpoint upcasts to bf16 on load (~2x resident); detect from the basename. SDXL single-file
- # is already bf16.
- fp8_upcast = not getattr(fam, "single_file_is_pipeline", False) and (
- "fp8" in Path(single_file_path).name.lower() if single_file_path else False
- )
- transformer_resident = estimate_safetensors_dense_mib(
- file_size_mib(single_file_path), fp8_upcast = fp8_upcast
- )
- if not getattr(fam, "single_file_is_pipeline", False):
- # Priced from the header: layers a resident runtime keeps at stored size, dequantized ones at 2x.
- _comfy_mib = self._comfy_single_file_resident_mib(
- single_file_path, fam, target, base, lora = lora
+
+ def _single_file_resident(
+ fp8_storage: bool, offload: bool = False
+ ) -> Optional[int]:
+ fp8_upcast = _single_file_fp8_upcast(single_file_path, fam)
+ if fp8_upcast and fp8_storage and fam.name == KREA2_FAMILY_NAME:
+ # storage keeps only OCP fp8 codes; FNUZ ones still widen
+ fp8_upcast = bool(
+ header_has_fp8_weights(str(single_file_path), fnuz_only = True)
+ )
+ mib = estimate_safetensors_dense_mib(
+ file_size_mib(single_file_path), fp8_upcast = fp8_upcast
)
- if _comfy_mib is not None:
- transformer_resident = _comfy_mib
+ if not getattr(fam, "single_file_is_pipeline", False):
+ # Priced from the header: layers a runtime keeps at stored size, dequantized ones at 2x.
+ _comfy_mib = self._comfy_single_file_resident_mib(
+ single_file_path,
+ fam,
+ target,
+ base,
+ lora = lora,
+ fp8_storage = fp8_storage,
+ offload = offload,
+ )
+ if _comfy_mib is not None:
+ mib = _comfy_mib
+ return mib
+
+ def _stored_mib(widened: Optional[int], offload: bool = False) -> Optional[int]:
+ if not (comfy_fp8_storage(target, lora = lora) and single_file_path):
+ return None
+ stored = _single_file_resident(True, offload)
+ if stored is not None and widened is not None and stored < widened:
+ return stored
+ return None
+
+ transformer_resident = _single_file_resident(False)
+ single_file_fp8_stored_mib = _stored_mib(transformer_resident)
+ single_file_resident_at = _single_file_resident
+ single_file_stored_at = _stored_mib
elif getattr(fam, "single_file_is_pipeline", False):
# Only the denoiser's linears stay packed; the text encoders, VAE and the rest load dense.
transformer_resident = whole_pipeline_gguf_resident_mib(
@@ -11424,17 +11510,62 @@ class DiffusionBackend:
# the 1024x1024 default for PLACEMENT. generate() re-checks the real resolution against the free budget before
# it samples (raise_on_image_activation_shortfall).
runtime_headroom = estimate_image_runtime_mib(width = None, height = None, family = variant_hint)
- return plan_diffusion_memory(
- target = target,
- device_memory = device_memory,
- model_dense_mib = model_dense_mib,
- companion_dense_mib = companion_mib,
- text_encoder_dense_mib = text_encoder_mib,
- runtime_headroom_mib = runtime_headroom,
- requested_mode = memory_mode,
- explicit_offload = cpu_offload,
- calibrated_activation = _calibrated_activation(fam, target),
- )
+
+ def _plan(model_mib: Optional[int]) -> Any:
+ return plan_diffusion_memory(
+ target = target,
+ device_memory = device_memory,
+ model_dense_mib = model_mib,
+ companion_dense_mib = companion_mib,
+ text_encoder_dense_mib = text_encoder_mib,
+ runtime_headroom_mib = runtime_headroom,
+ requested_mode = memory_mode,
+ explicit_offload = cpu_offload,
+ calibrated_activation = _calibrated_activation(fam, target),
+ )
+
+ plan = _plan(model_dense_mib)
+ if kind == "single_file" and single_file_path:
+ self.__dict__.setdefault("_comfy_priced_offload", {})[str(single_file_path)] = False
+ if (
+ single_file_resident_at is not None
+ and model_dense_mib is not None
+ and transformer_resident is not None
+ and not plan_keeps_transformer_resident(plan)
+ ):
+ # Offloaded: the loader probes the backends with offload=True (torchao fp8 does not survive the hooks), so
+ # price what that load builds, widened and fp8-stored, before choosing storage.
+ offloaded = single_file_resident_at(False, True)
+ if offloaded is not None and offloaded != transformer_resident:
+ # Smaller (the int8 / fp8 twin only offload runs): the load keeps those backends even if the new plan
+ # fits resident, else it dequantizes past the size it was planned at.
+ self.__dict__.setdefault("_comfy_priced_offload", {})[str(single_file_path)] = int(
+ offloaded
+ ) < int(transformer_resident)
+ model_dense_mib = model_dense_mib - int(transformer_resident) + int(offloaded)
+ transformer_resident = offloaded
+ single_file_fp8_stored_mib = single_file_stored_at(offloaded, True)
+ plan = _plan(model_dense_mib)
+ if single_file_fp8_stored_mib is not None and model_dense_mib is not None:
+ # Re-plan at the fp8-stored size: a widened load that had to offload may now stay resident.
+ from .diffusion_memory import _available_system_memory_mib
+
+ store = fp8_storage_wanted(
+ target,
+ lora = lora,
+ offload = not plan_keeps_transformer_resident(plan),
+ widened_mib = transformer_resident,
+ host_available_mib = _available_system_memory_mib(),
+ )
+ picks = self.__dict__.setdefault("_fp8_storage_picks", {})
+ picks[str(single_file_path)] = store
+ if store:
+ plan = _plan(
+ model_dense_mib - int(transformer_resident) + single_file_fp8_stored_mib
+ )
+ elif kind == "single_file" and single_file_path:
+ self.__dict__.setdefault("_fp8_storage_picks", {})[str(single_file_path)] = False
+ return plan
@staticmethod
def _from_pipe_no_recast(pipe: Any, pipe_cls: Any, **extra: Any) -> Any:
@@ -11732,6 +11863,12 @@ class DiffusionBackend:
"quantized Linears, which cannot carry adapters. Load the file with the LoRA selected (Unsloth then "
"dequantizes those layers), or set UNSLOTH_DIFFUSION_COMFY_NVFP4=0 / UNSLOTH_DIFFUSION_COMFY_MXFP8=0."
)
+ if fp8_stored_layers(getattr(pipe, "transformer", None)):
+ raise ValueError(
+ "LoRA is not available on this load: the file's fp8 layers are stored as fp8 and widened per layer "
+ "(this GPU has no fp8 GEMM), which cannot carry adapters. Load the file with the LoRA selected "
+ "(Unsloth then widens those layers at load), or set UNSLOTH_DIFFUSION_FP8_STORAGE=0."
+ )
if not diffusion_lora.supports_lora(
engine = "diffusers",
family = getattr(state.family, "name", None),
@@ -13449,8 +13586,10 @@ class DiffusionBackend:
transformer_quant = state.transformer_quant,
)
compiled = "compiled" in (getattr(state, "speed_optims", ()) or ())
- lora_load_ok = not _small_host_int8(state.pipe) and not comfy_block_runtime_layers(
- getattr(state.pipe, "transformer", None)
+ lora_load_ok = (
+ not _small_host_int8(state.pipe)
+ and not comfy_block_runtime_layers(getattr(state.pipe, "transformer", None))
+ and not fp8_stored_layers(getattr(state.pipe, "transformer", None))
)
return {
"loaded": True,
diff --git a/studio/backend/core/inference/diffusion_comfy_quant.py b/studio/backend/core/inference/diffusion_comfy_quant.py
index d03335e98fe..9e20379479b 100644
--- a/studio/backend/core/inference/diffusion_comfy_quant.py
+++ b/studio/backend/core/inference/diffusion_comfy_quant.py
@@ -36,6 +36,10 @@ QUANT_METADATA_KEY = "_quantization_metadata"
LEGACY_SCALED_FP8_KEY = "scaled_fp8"
COMFY_INT8_ENV = "UNSLOTH_DIFFUSION_COMFY_INT8"
COMFY_FP8_ENV = "UNSLOTH_DIFFUSION_COMFY_FP8"
+# Store fp8 layers no fp8 GEMM takes as fp8, widened per layer (ComfyUI manual_cast). Unset: only when widening
+# would offload or crowd host RAM (per-layer widen costs eager steps); ``1``: always; ``0``: never.
+FP8_STORAGE_ENV = "UNSLOTH_DIFFUSION_FP8_STORAGE"
+FP8_STORAGE_HOST_RESERVE_MIB = 4096
# Header metadata our own converter writes (base model, scheme); ComfyUI ignores it.
CONVERSION_RECORD_KEY = "unsloth_comfy_conversion"
@@ -133,6 +137,26 @@ def _layer_conf(raw: Any) -> tuple[Optional[str], bool, int]:
return (fmt if isinstance(fmt, str) else None), convrot, group
+# ROCm-style float8 tags: every loader widens them (fp8 storage keeps only the OCP formats above).
+FP8_FNUZ_TAGS = ("F8_E4M3FNUZ", "F8_E5M2FNUZ")
+
+
+def header_has_fp8_weights(path: Optional[str], *, fnuz_only: bool = False) -> Optional[bool]:
+ """Whether a safetensors file stores any fp8 ``.weight`` (plain, scaled or FNUZ; ``fnuz_only``: FNUZ only), or
+ None if its header is unreadable."""
+ if not path or not os.path.isfile(path):
+ return None
+ try:
+ header, _base = _read_header(str(path))
+ except Exception: # noqa: BLE001
+ return None
+ tags = FP8_FNUZ_TAGS if fnuz_only else (*FP8_FORMATS.values(), *FP8_FNUZ_TAGS)
+ return any(
+ key.endswith(".weight") and isinstance(entry, dict) and entry.get("dtype") in tags
+ for key, entry in header.items()
+ )
+
+
def scan_comfy_quant(path: Optional[str]) -> Optional[ComfyQuantScan]:
"""The ComfyUI quantization a safetensors single file declares, or None for a plain checkpoint.
@@ -269,12 +293,46 @@ _DTYPE_BYTES = {
}
+# Original-layout names of modules a DiT keeps in fp32 (Wan's ``time_embedding.N`` loads as ``time_embedder``).
+_KEEP_FP32_ORIGINAL_ALIASES = {"time_embedder": ("time_embedding",)}
+
+
+def keeps_fp32(name: str, keep_fp32: Any = ()) -> bool:
+ """Whether header key ``name`` names a ``_keep_in_fp32_modules`` module, under its diffusers or original name."""
+ keep = tuple(keep_fp32 or ())
+ tokens = str(name).split(".")
+ return any(t in tokens for m in keep for t in (m, *_KEEP_FP32_ORIGINAL_ALIASES.get(m, ())))
+
+
+def keep_fp32_modules_for(fam: Any) -> tuple:
+ """The family transformer's ``_keep_in_fp32_modules`` (the loader refuses fp8 storage for them), else ()."""
+ try:
+ import diffusers
+ cls = getattr(diffusers, getattr(fam, "transformer_class", None) or "", None)
+ keep = getattr(cls, "_keep_in_fp32_modules", None) or ()
+ except Exception: # noqa: BLE001 -- unknown class: nothing excluded
+ return ()
+ return (keep,) if isinstance(keep, str) else tuple(keep)
+
+
+def fp8_stored_by_header(
+ fmt: str,
+ shape: Any,
+ name: str = "",
+ keep_fp32: Any = (),
+) -> bool:
+ """Header rule for fp8 layers kept as codes, shared by the host RAM estimate and the planner; the loader can only
+ widen more (charged live)."""
+ return fmt in FP8_FORMATS and len(shape or ()) == 2 and not keeps_fp32(name, keep_fp32)
+
+
def comfy_resident_mib(
path: Optional[str],
scan: Optional[ComfyQuantScan] = None,
*,
keep_int8: bool,
keep_fp8: bool,
+ fp8_stored: bool = False,
keep_nvfp4: bool = False,
keep_mxfp8: bool = False,
compute_bytes: int = 2,
@@ -284,6 +342,8 @@ def comfy_resident_mib(
fp8_divisible: int = 0,
block_divisible: Optional[dict] = None,
key_map: Any = None,
+ keep_fp32: Any = (),
+ keep_fp32_bytes: Optional[int] = None,
) -> Optional[int]:
"""What the loader leaves resident for a ComfyUI-quantized file, priced from its header: a quantized
weight a runtime keeps costs its stored bytes, one that is dequantized costs ``numel * compute_bytes``
@@ -293,7 +353,10 @@ def comfy_resident_mib(
an int8 layer whose name holds one of ``exclude_tokens`` is priced dequantized, as Studio's int8 filter
leaves it, and so is a layer the runtime filter skips (in / out features under ``min_features``, or fp8 features
not multiples of ``fp8_divisible``). ``key_map`` names layers as the loader's filter sees them (an original-layout
- file). None when the header cannot be read. Torch-free."""
+ file). ``keep_fp32``: the transformer's ``_keep_in_fp32_modules``; those layers are never fp8-stored and, widened,
+ cost ``keep_fp32_bytes`` per element (4 on an fp16 load, default ``compute_bytes``). None when the header cannot
+ be read. Torch-free."""
+ fp32_bytes = compute_bytes if keep_fp32_bytes is None else int(keep_fp32_bytes)
try:
scan = scan if scan is not None else scan_comfy_quant(path)
if scan is None:
@@ -311,14 +374,22 @@ def comfy_resident_mib(
return False
return not divisible or not (out_f % divisible or in_f % divisible)
- def _excluded(key: str, shape: Any) -> bool:
+ def _names(key: str, shape: Any) -> list:
names = [key]
if key_map is not None:
try:
names = [k for k, _rows in key_map(key, tuple(shape or ()))] or names
except ValueError:
pass
- return any(t in n for n in names for t in exclude_tokens)
+ return names
+
+ def _excluded(key: str, shape: Any) -> bool:
+ return any(t in n for n in _names(key, shape) for t in exclude_tokens)
+
+ def _bytes(key: str, shape: Any) -> int:
+ if keep_fp32 and any(keeps_fp32(n, keep_fp32) for n in _names(key, shape)):
+ return fp32_bytes
+ return compute_bytes
def _logical(fmt: str, shape: Any) -> Any:
if fmt == NVFP4 and len(shape or ()) == 2:
@@ -335,6 +406,13 @@ def comfy_resident_mib(
and _fits(shape, 0)
) or (
(layer.format == FP8_E4M3 and keep_fp8 and _fits(shape, fp8_divisible))
+ or (
+ fp8_stored
+ and all(
+ fp8_stored_by_header(layer.format, shape, n, keep_fp32)
+ for n in _names(name + ".weight", shape)
+ )
+ )
or (
{NVFP4: keep_nvfp4, MXFP8: keep_mxfp8}.get(layer.format, False)
# input smoothing has no runtime Linear: the loader dequantizes such a layer
@@ -356,11 +434,13 @@ def comfy_resident_mib(
dtype = str(entry.get("dtype", ""))
stored = numel * _DTYPE_BYTES.get(dtype, 2)
if key in kept:
- total += stored if kept[key] else numel * compute_bytes * (2 if key in packed else 1)
+ total += (
+ stored if kept[key] else numel * _bytes(key, shape) * (2 if key in packed else 1)
+ )
elif key.endswith(COMFY_QUANT_SUFFIX) or any(key.endswith(s) for s in _SCALE_SUFFIXES):
total += stored
elif dtype.startswith(("F", "BF")):
- total += numel * compute_bytes
+ total += numel * _bytes(key, shape)
else:
total += stored
return -(-total // (1024 * 1024)) if total > 0 else None
@@ -829,7 +909,7 @@ def comfy_fp8_backend(
quant runs natively (ROCm, the stubbed torchao), None (dequantize to bf16) where neither runs: an
older GPU, CPU, MPS, or an offloaded plan on the torchao path (the model offload hooks cannot move a
``Float8Tensor``: "Attempted to set the storage of a tensor on device cuda:0 to a storage on ... cpu").
- ``UNSLOTH_DIFFUSION_COMFY_FP8=0`` always dequantizes."""
+ ``UNSLOTH_DIFFUSION_COMFY_FP8=0`` never takes an fp8 GEMM."""
if (os.environ.get(COMFY_FP8_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
return None
try:
@@ -855,6 +935,64 @@ def comfy_fp8_backend(
return None
+FP8_STORED_ATTR = "_unsloth_fp8_stored"
+
+
+def fp8_stored_layers(model: Any) -> int:
+ """Count of fp8-stored Linears (they cannot carry a LoRA)."""
+ try:
+ return int(getattr(model, FP8_STORED_ATTR, 0) or 0)
+ except (TypeError, ValueError):
+ return 0
+
+
+def pipe_fp8_stored_layers(pipe: Any) -> int:
+ return sum(
+ fp8_stored_layers(getattr(pipe, attr, None)) for attr in ("transformer", "transformer_2")
+ )
+
+
+def fp8_storage_disabled() -> bool:
+ return (os.environ.get(FP8_STORAGE_ENV) or "").strip().lower() in ("0", "off", "false", "no")
+
+
+def fp8_storage_forced() -> bool:
+ return (os.environ.get(FP8_STORAGE_ENV) or "").strip().lower() in ("1", "on", "true", "yes")
+
+
+def fp8_storage_wanted(
+ target: Any = None,
+ *,
+ lora: bool = False,
+ offload: bool = False,
+ widened_mib: Optional[int] = None,
+ host_available_mib: Optional[int] = None,
+) -> bool:
+ """Store fp8 only when widening would offload or leave under the host reserve: widening is otherwise faster
+ (measured +45% s/step eager for storage on Krea 2 / B200, parity compiled)."""
+ if not comfy_fp8_storage(target, lora = lora):
+ return False
+ if fp8_storage_forced() or offload:
+ return True
+ if widened_mib is None or host_available_mib is None:
+ return False
+ return int(widened_mib) + FP8_STORAGE_HOST_RESERVE_MIB > int(host_available_mib)
+
+
+def comfy_fp8_storage(target: Any = None, *, lora: bool = False) -> bool:
+ """Whether fp8 storage can run: same weights as the load-time dequant, half the bytes. Off for a LoRA (needs
+ dense Linears), MPS and torch without float8."""
+ if lora or fp8_storage_disabled():
+ return False
+ if str(getattr(target, "device", "") or "").split(":")[0] == "mps":
+ return False
+ try:
+ import torch
+ return hasattr(torch, "float8_e4m3fn") and hasattr(torch, "float8_e5m2")
+ except Exception: # noqa: BLE001 -- no torch: nothing to store
+ return False
+
+
def comfy_conversion_record(path: Optional[str]) -> dict:
"""The ``unsloth_comfy_conversion`` header record our converter writes (base model, scheme, family),
or {} for any other file. Torch-free, never raises."""
@@ -979,6 +1117,7 @@ def load_comfy_quant_transformer(
key_map: Any = None,
prepare_model: Any = None,
keep_dtype: Any = None,
+ fp8_storage: Optional[bool] = None,
) -> Any:
"""Build ``transformer_cls`` from the ComfyUI-quantized ``path``.
@@ -991,7 +1130,8 @@ def load_comfy_quant_transformer(
as per-row ``Float8Tensor`` weights or native fp8 twins; with an ``nvfp4_backend`` / ``mxfp8_backend``
(``diffusion_comfy_block.comfy_block_backend``) the nvfp4 / mxfp8 layers the matching Studio filter selects
keep their codes on Studio's FlashInfer NVFP4 Linear / the ``torch._scaled_mm`` MXFP8 Linear. Everything
- else is dequantized to the compute dtype. ``finalize`` applies the small-M padding here (the single-file path); the hosted
+ else is dequantized to the compute dtype, except fp8 Linears under ``fp8_storage`` (None = env opt-in
+ only), which widen per layer in the forward. ``finalize`` applies the small-M padding here (the single-file path); the hosted
prequant loader passes False and pads after placement, as for its own checkpoints.
``keep_key(key)`` limits the read to the DiT's keys of a file that bundles other components;
@@ -1037,6 +1177,8 @@ def load_comfy_quant_transformer(
# Studio's fp8 runtime asserts bf16 weights; an fp16 / fp32 pipeline dequantizes them instead.
fp8_backend = None
mxfp8_backend = None
+ if fp8_storage is None:
+ fp8_storage = fp8_storage_forced() and comfy_fp8_storage(target)
if keep_key is None:
state = load_file(str(path))
else:
@@ -1086,15 +1228,38 @@ def load_comfy_quant_transformer(
# Mirrors the source loop below: a layer is kept as codes only with a runtime for its format (fp16_keep
# merely defers the dtype choice; the layer is still expanded) and, for a block format, no key map.
# nvfp4 headers describe packed uint8 (K / 2 columns); the dense weight is N x K.
+ header_shapes = {}
+ if fp8_storage:
+ try:
+ _hdr, _ = _read_header(str(path))
+ header_shapes = {
+ k: (v or {}).get("shape") for k, v in _hdr.items() if k != "__metadata__"
+ }
+ except Exception: # noqa: BLE001 -- the estimate then prices them widened (conservative)
+ header_shapes = {}
+
+ def _stored(layer: Any) -> bool:
+ shape = header_shapes.get(layer.name + ".weight")
+ return (
+ bool(fp8_storage)
+ and shape is not None
+ and fp8_stored_by_header(layer.format, shape, layer.name, keep_fp32)
+ )
+
_widened = {
layer.name + ".weight": (
(2 if layer.format == NVFP4 else 1)
- if not backends.get(layer.format)
+ if not (backends.get(layer.format) or _stored(layer))
or (layer.format in BLOCK_SIZES and key_map is not None)
else 0
)
for layer in scan.layers.values()
}
+ _stored_scale_bytes = sum(
+ 4 * int(header_shapes[layer.name + ".weight"][0])
+ for layer in scan.layers.values()
+ if _stored(layer) and not backends.get(layer.format)
+ )
# The header estimate refuses early; the budget it activates is charged by every allocating helper
# below, so a layer the estimate cannot see (runtime filters, nibble swaps) still cannot OOM the host.
budget = HostRamBudget(
@@ -1108,6 +1273,7 @@ def load_comfy_quant_transformer(
dequantized = _widened.get,
keep_key = keep_key,
)
+ + _stored_scale_bytes
)
budget.activate()
sources: list = [] # (layer, codes, scale, block extras or None) kept for a runtime
@@ -1133,9 +1299,9 @@ def load_comfy_quant_transformer(
elif layer.format in FP8_FORMATS and codes.dtype == torch.uint8:
codes = codes.view(getattr(torch, layer.format))
# A key-mapped (original-layout) family moves rows whole, which the tiled block scales do not survive.
- if (backends.get(layer.format) or fp16_keep) and not (
- extra is not None and key_map is not None
- ):
+ if (
+ backends.get(layer.format) or fp16_keep or (fp8_storage and layer.format in FP8_FORMATS)
+ ) and not (extra is not None and key_map is not None):
sources.append((layer, codes, scale, extra))
state[layer.name + ".weight"] = None # placeholder, tagged below
elif extra is not None:
@@ -1278,6 +1444,7 @@ def load_comfy_quant_transformer(
modules = dict(model.named_modules())
rotations: dict = {}
native: dict = {}
+ stored: dict = {} # fqn -> (codes, scale)
block_native: dict = {} # fqn -> (fmt, codes, scale, tensor_scale, input_scale, bias)
built = {TQ_INT8: 0, TQ_FP8: 0, TQ_NVFP4: 0, TQ_MXFP8: 0}
for name, value in list(converted.items()):
@@ -1331,6 +1498,20 @@ def load_comfy_quant_transformer(
weight = _int8_tensor(name, codes, scale, dtype)
else:
weight = _fp8_tensor(name, codes, scale, dtype, fast_accum)
+ if (
+ weight is None
+ and fp8_storage
+ and fmt in FP8_FORMATS
+ and not group
+ and isinstance(module, torch.nn.Linear)
+ and name.endswith(".weight")
+ and _dtype_for(name) == dtype
+ and not any(m in fqn.split(".") for m in keep_fp32)
+ ):
+ # forward widen must match ``_dequant``
+ stored[fqn] = (codes, scale.reshape(-1, 1).expand(codes.shape[0], 1))
+ del converted[name]
+ continue
if weight is None:
converted[name] = _dequant(codes, scale, group, _dtype_for(name))
dequantized += 1
@@ -1363,6 +1544,16 @@ def load_comfy_quant_transformer(
setattr(parent, leaf, layer)
converted[fqn + ".weight_q"] = layer.weight_q
converted[fqn + ".weight_scale"] = layer.weight_scale
+ for fqn, (codes, scale) in stored.items():
+ parent_name, _, leaf = fqn.rpartition(".")
+ parent = model.get_submodule(parent_name) if parent_name else model
+ charge_host_ram(
+ 4 * int(codes.shape[0])
+ ) # the fp32 row scale; the codes stay as read (a view)
+ layer = cls(getattr(parent, leaf).to(dtype), TQ_FP8, codes = codes, scale = scale)
+ setattr(parent, leaf, layer)
+ converted[fqn + ".weight_q"] = layer.weight_q
+ converted[fqn + ".weight_scale"] = layer.weight_scale
for fqn, (fmt, codes, scale, tensor_scale, input_scale, bias) in block_native.items():
parent_name, _, leaf = fqn.rpartition(".")
parent = model.get_submodule(parent_name) if parent_name else model
@@ -1462,7 +1653,8 @@ def load_comfy_quant_transformer(
logger.info(
"diffusion.comfy_quant: %s loaded from a ComfyUI checkpoint (%s): %d int8 layers kept as "
"int8 (%s, %d ConvRot), %d fp8 layers kept as fp8 (%s), %d nvfp4 layers kept as nvfp4 (%s), "
- "%d mxfp8 layers kept as mxfp8 (%s), %d layers dequantized to %s",
+ "%d mxfp8 layers kept as mxfp8 (%s), %d fp8 layers stored as fp8 and widened per layer to %s, "
+ "%d layers dequantized to %s",
transformer_cls.__name__,
", ".join(f"{k} x{v}" for k, v in sorted(scan.counts().items())),
built[TQ_INT8],
@@ -1474,6 +1666,8 @@ def load_comfy_quant_transformer(
f"{nvfp4_backend} runtime" if built[TQ_NVFP4] else "no nvfp4 runtime",
built[TQ_MXFP8],
f"{mxfp8_backend} runtime" if built[TQ_MXFP8] else "no mxfp8 runtime",
+ len(stored),
+ str(dtype).replace("torch.", ""),
dequantized,
str(dtype).replace("torch.", ""),
)
@@ -1486,6 +1680,9 @@ def load_comfy_quant_transformer(
"fp8": built[TQ_FP8],
"dequantized": dequantized,
}
+ if stored:
+ info["fp8_stored"] = len(stored)
+ setattr(model, FP8_STORED_ATTR, len(stored))
for fmt, backend in ((NVFP4, nvfp4_backend), (MXFP8, mxfp8_backend)):
if scan.counts().get(fmt):
info[f"{fmt}_backend"] = backend if built[schemes[fmt]] else None
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.py
index 5298b404b1c..1f12fab98b9 100644
--- a/studio/backend/core/inference/diffusion_compile_cache.py
+++ b/studio/backend/core/inference/diffusion_compile_cache.py
@@ -402,6 +402,10 @@ def _weight_signature(module: Any, weight: Any) -> str:
parts.append(f"rot{rot}")
if getattr(module, "act_int8", False):
parts.append("act_int8")
+ fmt = getattr(module, "fp8_format", None)
+ if isinstance(fmt, str) and fmt != "float8_e4m3fn":
+ # Stored E5M2 codes decode through a different integer graph; E4M3 keeps the key it always had.
+ parts.append(fmt)
fwd = module.__dict__.get("forward") if hasattr(module, "__dict__") else None
if fwd is not None:
func = getattr(fwd, "__func__", fwd)
diff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.py
index b1e656485c4..6708c6919b0 100644
--- a/studio/backend/core/inference/diffusion_native_quant.py
+++ b/studio/backend/core/inference/diffusion_native_quant.py
@@ -31,6 +31,31 @@ _INT8PACK_PROBED: dict = {}
# torch._int_mm needs M > 16 and K, N multiples of 8.
_INT_MM_MIN_ROWS = 17
_INT_MM_CHUNK_ELEMS = 1 << 25
+# fp8 codes a native fp8 layer may hold; e5m2 only from a checkpoint (the quantise path writes e4m3fn).
+NATIVE_FP8_FORMATS = ("float8_e4m3fn", "float8_e5m2")
+
+
+def fp8_bits_to_float32(bits: Any, fmt: str = "float8_e4m3fn") -> Any:
+ """uint8 fp8 codes to float32 by integer ops, bit-exact; Triton has no e4m3fn convert below sm_89."""
+ import torch
+
+ b = bits.to(torch.int32)
+ if fmt == "float8_e5m2":
+ exp_bits, man_bits, bias = 5, 2, 15
+ else:
+ exp_bits, man_bits, bias = 4, 3, 7
+ e = (b >> man_bits) & ((1 << exp_bits) - 1)
+ m = b & ((1 << man_bits) - 1)
+ normal = (((e + (127 - bias)) << 23) | (m << (23 - man_bits))).view(torch.float32)
+ # subnormal: m * 2**(1 - bias - man_bits), exact in float32
+ sub = m.to(torch.float32) * (2.0 ** (1 - bias - man_bits))
+ mag = torch.where(e == 0, sub, normal)
+ if fmt == "float8_e5m2":
+ top = e == (1 << exp_bits) - 1
+ mag = torch.where(top, torch.where(m == 0, float("inf"), float("nan")), mag)
+ else:
+ mag = torch.where((e == 15) & (m == 7), float("nan"), mag)
+ return torch.where((b & 0x80) != 0, -mag, mag)
def int8_act_requested() -> bool:
@@ -259,8 +284,15 @@ def native_linear_class():
if self.act_int8 and rot_group and self.in_features % int(rot_group) == 0
else 0
)
+ self.fp8_format = "float8_e4m3fn"
if codes is not None:
wanted = torch.int8 if scheme == NATIVE_INT8 else torch.float8_e4m3fn
+ if (
+ scheme == NATIVE_FP8
+ and str(codes.dtype).replace("torch.", "") in NATIVE_FP8_FORMATS
+ ):
+ wanted = codes.dtype
+ self.fp8_format = str(codes.dtype).replace("torch.", "")
if codes.dtype != wanted:
raise ValueError(
f"pre-quantized native {scheme} codes must be {str(wanted).replace('torch.', '')}"
@@ -313,13 +345,14 @@ def native_linear_class():
self._int8pack_scales: dict = {}
def _stored_weight(self, dtype: Any) -> Any:
- wq = (
- self.weight_q.view(torch.float8_e4m3fn)
- if self.scheme == NATIVE_FP8
- else self.weight_q
- )
scale = self.weight_scale.view(torch.float32)
- return (wq.to(torch.float32) * scale[:, None]).to(dtype)
+ if self.scheme != NATIVE_FP8:
+ wide = self.weight_q.to(torch.float32)
+ elif torch.compiler.is_compiling():
+ wide = fp8_bits_to_float32(self.weight_q, self.fp8_format)
+ else:
+ wide = self.weight_q.view(getattr(torch, self.fp8_format)).to(torch.float32)
+ return (wide * scale[:, None]).to(dtype)
def _rotate(self, x: Any) -> Any:
g = self.rot_group
@@ -465,6 +498,7 @@ def native_linear_class():
return (
f"in_features={self.in_features}, out_features={self.out_features}, "
f"scheme={self.scheme}, act_int8={self.act_int8}, rot_group={self.rot_group}, bias={self.bias is not None}"
+ + (f", codes={self.fp8_format}" if self.scheme == NATIVE_FP8 else "")
)
return NativeWeightOnlyLinear
diff --git a/studio/backend/core/inference/diffusion_single_file_converters.py b/studio/backend/core/inference/diffusion_single_file_converters.py
index 6fd86c1b1ab..84521321467 100644
--- a/studio/backend/core/inference/diffusion_single_file_converters.py
+++ b/studio/backend/core/inference/diffusion_single_file_converters.py
@@ -411,7 +411,12 @@ def converted_host_bytes(
numel *= int(dim)
except Exception: # noqa: BLE001
continue
- verdict = dequantized(key) if dequantized is not None else None
+ if dequantized is None:
+ verdict = None
+ elif getattr(dequantized, "wants_meta", False):
+ verdict = dequantized(key, meta)
+ else:
+ verdict = dequantized(key)
if verdict is not None:
total += numel * int(verdict) * _ST_ITEMSIZE[target]
continue
@@ -574,10 +579,13 @@ def load_original_layout_transformer(
path: str,
sf_kwargs: dict,
logger: Any = None,
+ *,
+ fp8_storage: bool = False,
) -> Any:
"""Load a plain original-layout safetensors file for a class without ``from_single_file``
(Krea 2 on diffusers 0.40). ``sf_kwargs`` = the ``from_single_file`` kwargs. Cast like
- ``from_pretrained`` (``_keep_in_fp32_modules`` stay fp32); strict on keys and shapes."""
+ ``from_pretrained`` (``_keep_in_fp32_modules`` stay fp32); strict on keys and shapes. ``fp8_storage``
+ keeps fp8 Linears as fp8 with unit scales."""
import torch
from diffusers.loaders import single_file_model as sfm
@@ -606,12 +614,67 @@ def load_original_layout_transformer(
budget = HostRamBudget(
path, f"widening its weights to {str(dtype).replace('torch.', '')}", logger
)
- budget.check(converted_host_bytes(path, dtype, keep_fp32 = keep))
+ budget.check(
+ converted_host_bytes(
+ path,
+ dtype,
+ keep_fp32 = keep,
+ dequantized = fp8_stored_estimate(keep) if fp8_storage else None,
+ )
+ + (_fp8_stored_scale_bytes(path, keep) if fp8_storage else 0)
+ )
with budget:
- return _load_original_layout(transformer_cls, path, config, mapping_fn, keep, dtype, logger)
+ return _load_original_layout(
+ transformer_cls, path, config, mapping_fn, keep, dtype, logger, fp8_storage
+ )
+
+
+def fp8_stored_estimate(keep: Any = ()) -> Any:
+ """``dequantized`` for fp8 storage: 2-D fp8 ``.weight`` outside ``keep`` stays mmapped (0); later widens are
+ charged live."""
+ keep = tuple(keep or ())
+
+ def verdict(key: str, meta: Optional[dict] = None) -> Any:
+ meta = meta or {}
+ if (
+ str(meta.get("dtype")) in ("F8_E4M3", "F8_E5M2")
+ and len(meta.get("shape") or ()) == 2
+ and key.endswith(".weight")
+ and not any(m in key.split(".") for m in keep)
+ ):
+ return 0
+ return None
+
+ verdict.wants_meta = True
+ return verdict
+
+
+def _fp8_stored_scale_bytes(path: str, keep: Any = ()) -> int:
+ from .diffusion_comfy_quant import _read_header
+
+ try:
+ header, _base = _read_header(str(path))
+ except Exception: # noqa: BLE001
+ return 0
+ header.pop("__metadata__", None)
+ verdict = fp8_stored_estimate(keep)
+ return sum(
+ 4 * int((meta.get("shape") or (0,))[0])
+ for key, meta in header.items()
+ if verdict(key, meta) == 0
+ )
-def _load_original_layout(transformer_cls, path, config, mapping_fn, keep, dtype, logger) -> Any:
+def _load_original_layout(
+ transformer_cls,
+ path,
+ config,
+ mapping_fn,
+ keep,
+ dtype,
+ logger,
+ fp8_storage = False,
+) -> Any:
import torch
from accelerate import init_empty_weights
from safetensors.torch import load_file
@@ -631,7 +694,10 @@ def _load_original_layout(transformer_cls, path, config, mapping_fn, keep, dtype
f"{name} does not match {transformer_cls.__name__}: {len(missing)} missing "
f"(e.g. {missing[:1]}), {len(unexpected)} unused (e.g. {unexpected[:1]})"
)
- for key, value in converted.items():
+ fp8_dtypes = {getattr(torch, n) for n in ("float8_e4m3fn", "float8_e5m2") if hasattr(torch, n)}
+ modules = dict(model.named_modules()) if fp8_storage else {}
+ stored: list = []
+ for key, value in list(converted.items()):
want = torch.float32 if any(m in key.split(".") for m in keep) else dtype
if not value.is_floating_point():
continue
@@ -639,14 +705,45 @@ def _load_original_layout(transformer_cls, path, config, mapping_fn, keep, dtype
raise ValueError(
f"{key}: {tuple(value.shape)} in the file, {tuple(wanted[key].shape)} in the model"
)
+ fqn = key[: -len(".weight")]
+ if (
+ value.dtype in fp8_dtypes
+ and want == dtype
+ and key.endswith(".weight")
+ and isinstance(modules.get(fqn), torch.nn.Linear)
+ ):
+ stored.append(fqn)
+ continue
if value.dtype != want:
charge_cast(value, want)
converted[key] = value.to(want)
+ if stored:
+ from .diffusion_native_quant import NATIVE_FP8, native_linear_class
+ cls = native_linear_class()
+ for fqn in stored:
+ parent_name, _, leaf = fqn.rpartition(".")
+ parent = model.get_submodule(parent_name) if parent_name else model
+ codes = converted.pop(fqn + ".weight")
+ charge_host_ram(codes.shape[0] * 4) # the unit scale; the codes stay as read
+ layer = cls(
+ getattr(parent, leaf).to(dtype),
+ NATIVE_FP8,
+ codes = codes,
+ scale = torch.ones(codes.shape[0], 1, dtype = torch.float32),
+ )
+ setattr(parent, leaf, layer)
+ converted[fqn + ".weight_q"] = layer.weight_q
+ converted[fqn + ".weight_scale"] = layer.weight_scale
model.load_state_dict(converted, strict = True, assign = True)
+ if stored:
+ from .diffusion_comfy_quant import FP8_STORED_ATTR
+ setattr(model, FP8_STORED_ATTR, len(stored))
model.eval()
if logger is not None:
logger.info(
- "diffusion.single_file: %s loaded from an original-layout file",
+ "diffusion.single_file: %s loaded from an original-layout file (%d fp8 layers stored as fp8, "
+ "widened per layer)",
transformer_cls.__name__,
+ len(stored),
)
return model
diff --git a/studio/backend/core/inference/gpu_residency.py b/studio/backend/core/inference/gpu_residency.py
index 5b50e99d83e..b77a0f84123 100644
--- a/studio/backend/core/inference/gpu_residency.py
+++ b/studio/backend/core/inference/gpu_residency.py
@@ -206,6 +206,7 @@ def _image_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int,
from core.inference.diffusion import (
DiffusionBackend as DB,
_image_variant_hint,
+ _single_file_fp8_upcast,
resolve_model_kind,
)
from core.inference.diffusion_auto_policy import resident_bytes_from_declared
@@ -261,8 +262,9 @@ def _image_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int,
return None
whole = getattr(fam, "single_file_is_pipeline", False)
if kind == "single_file":
- fp8 = not whole and "fp8" in str(name).lower()
- weights = estimate_safetensors_dense_mib(file_size_mib(path), fp8_upcast = fp8)
+ weights = estimate_safetensors_dense_mib(
+ file_size_mib(path), fp8_upcast = _single_file_fp8_upcast(path, fam)
+ )
elif whole:
weights = whole_pipeline_gguf_resident_mib(path) or estimate_gguf_resident_mib(
file_size_mib(path)
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 6b4c6a7a693..67964b1874f 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -63,6 +63,8 @@ from .diffusion_flow_shift import apply_comfy_flow_shift
from .diffusion_comfy_quant import (
comfy_fp8_backend,
comfy_int8_backend,
+ fp8_storage_wanted,
+ pipe_fp8_stored_layers,
comfy_resident_mib,
load_comfy_quant_transformer,
refuse_comfy_quant,
@@ -208,6 +210,7 @@ from .diffusion_transformer_quant import (
transformer_is_quantised,
)
from .diffusion import (
+ _comfy_keep_fp32_pricing,
_family_override_resolved,
_companion_split_expected,
_layer_split_expected,
@@ -481,20 +484,60 @@ def _video_comfy_backends(
bf16_plan: Any = None,
*,
keep: bool = True,
+ priced_offload: bool = False,
+ widened_mib: Optional[int] = None,
) -> dict[str, Optional[str]]:
"""``load_comfy_quant_transformer`` backends for a ComfyUI-quantized video DiT: the ones Studio's own int8 / fp8
quant would run on this card and plan (int8: torchao resident, the torchao-free twin under offload; fp8: torchao
- resident only). ``keep=False`` (the user asked for the bf16 DiT) dequantizes everything."""
+ resident only; else fp8 storage under offload). ``priced_offload``: the plan was sized with the offloaded
+ backends (``_video_offload_reprice``), so a load that now fits resident keeps them. ``widened_mib``: the planned
+ DiT price; a resident load stores fp8 when widening it would leave under the host reserve, as an image does.
+ ``keep=False`` (the user asked for the bf16 DiT) dequantizes everything."""
if not keep:
- return {"int8_backend": None, "fp8_backend": None}
- offload = not _video_comfy_resident(plan, bf16_plan)
+ return {"int8_backend": None, "fp8_backend": None, "fp8_storage": False}
+ offload = priced_offload or not _video_comfy_resident(plan, bf16_plan)
name = getattr(fam, "name", None)
return {
"int8_backend": comfy_int8_backend(target, name, base, offload = offload),
"fp8_backend": comfy_fp8_backend(target, name, base, offload = offload),
+ "fp8_storage": _video_fp8_storage(fam, target, offload, widened_mib = widened_mib),
}
+def _video_offload_reprice(
+ fam: Any, transformer_mib: Optional[int], offload_mib: Optional[int]
+) -> tuple[Optional[int], bool]:
+ """``(transformer_mib, priced_offload)`` once an offloaded plan re-prices the DiT with the offloaded backends.
+ Larger: re-plan at it. Smaller (fp8 stored, the int8 twin where only it runs): re-plan at it too, and the load must
+ then keep those backends even if the new plan fits resident, else it widens past the size it was planned at.
+ H3 keeps its resident torchao path, so only a larger price moves it."""
+ if offload_mib is None or transformer_mib is None or offload_mib == transformer_mib:
+ return transformer_mib, False
+ if offload_mib < transformer_mib and getattr(fam, "name", None) == "minimax-h3":
+ return transformer_mib, False
+ return offload_mib, offload_mib < transformer_mib
+
+
+def _video_fp8_storage(
+ fam: Any,
+ target: Any,
+ offload: bool,
+ *,
+ widened_mib: Optional[int] = None,
+) -> bool:
+ """``fp8_storage_wanted`` for a video DiT, with the host RAM the load would widen into when ``widened_mib`` is
+ known. H3 is excluded: its pin / stream / residency path is built on torchao weights."""
+ if getattr(fam, "name", None) == "minimax-h3":
+ return False
+ available = None
+ if widened_mib is not None and not offload:
+ from .diffusion_small_host import host_ram_mib
+ available = host_ram_mib()[1]
+ return fp8_storage_wanted(
+ target, offload = offload, widened_mib = widened_mib, host_available_mib = available
+ )
+
+
def _video_comfy_resident_mib(
fam: Any,
base: str,
@@ -510,7 +553,7 @@ def _video_comfy_resident_mib(
int8 runtime runs here, fp8 layers as fp8 when the resident fp8 path does, the rest at bf16 (a dequantized fp8
file is twice its size on disk). ``offload`` prices what an offloaded load keeps instead."""
name = getattr(fam, "name", None)
- keep_int8 = keep_fp8 = False
+ keep_int8 = keep_fp8 = fp8_stored = False
exclude: tuple = ()
filters: dict = {}
if keep:
@@ -530,15 +573,18 @@ def _video_comfy_resident_mib(
# Only a resident runtime keeps codes in a resident plan; an offloaded plan is re-priced.
keep_int8 = bool(comfy_int8_backend(target, name, base, offload = offload))
keep_fp8 = comfy_fp8_backend(target, name, base, offload = offload) is not None
+ fp8_stored = _video_fp8_storage(fam, target, offload)
return comfy_resident_mib(
str(path),
scan,
keep_int8 = keep_int8,
keep_fp8 = keep_fp8,
+ fp8_stored = fp8_stored,
keep_key = keep_key,
exclude_tokens = exclude,
key_map = _video_comfy_key_map(fam),
**filters,
+ **_comfy_keep_fp32_pricing(fam, target),
)
@@ -3021,9 +3067,12 @@ def _video_seed_stays_resident(
def _render_under_no_grad(state: Any) -> bool:
- """Render under no_grad: torchao weight subclasses fail an offload hook's in-place swap under inference_mode."""
+ """Render under no_grad: torchao weight subclasses fail an offload hook's in-place swap under inference_mode.
+ Stored fp8 codes too: moved under inference_mode they become unversionable, re-copied to host every offload."""
if not getattr(state, "transformer_quant", None):
- return False
+ return getattr(state, "offload_policy", OFFLOAD_NONE) != OFFLOAD_NONE and bool(
+ pipe_fp8_stored_layers(getattr(state, "pipe", None))
+ )
if getattr(state, "denoiser_streamed", False):
return True
family = getattr(state, "family", None)
@@ -8392,6 +8441,7 @@ class VideoBackend:
denoiser_seed_scheme = None
denoiser_seed_gb = None
plan, bf16_plan, quant_replanned = _plan_for_te_scale(te_scale, log = False)
+ comfy_priced_offload = False
if comfy_scan is not None and not _video_comfy_resident(plan, bf16_plan):
# Offloaded, fp8 layers dequantize on NVIDIA: price the DiT the load will actually build.
offload_mib = _video_comfy_resident_mib(
@@ -8421,9 +8471,14 @@ class VideoBackend:
)
if offload_mib is not None and transformer_mib is not None:
offload_mib = int(offload_mib * dtype_scale)
- if offload_mib > transformer_mib:
- transformer_mib = offload_mib
+ repriced, comfy_priced_offload = _video_offload_reprice(
+ fam, transformer_mib, offload_mib
+ )
+ if repriced != transformer_mib:
+ transformer_mib = repriced
plan, bf16_plan, quant_replanned = _plan_for_te_scale(te_scale, log = False)
+ # The DiT as planned: a resident load that would widen it into under the host reserve stores fp8 instead.
+ comfy_widened_mib = transformer_mib if comfy_scan is not None else None
settled_te_scale = te_scale
# On unified memory the plan's 'none' policy is a placement, not a fit: there is no offload tier left to fall
@@ -8657,7 +8712,14 @@ class VideoBackend:
cache_dir = hub_cache_dir(),
local_files_only = local_files_only,
**_video_comfy_backends(
- fam, base, target, plan, bf16_plan, keep = comfy_keep
+ fam,
+ base,
+ target,
+ plan,
+ bf16_plan,
+ keep = comfy_keep,
+ priced_offload = comfy_priced_offload,
+ widened_mib = comfy_widened_mib,
),
family = fam.name,
target = target,
@@ -8773,7 +8835,14 @@ class VideoBackend:
cache_dir = hub_cache_dir(),
local_files_only = local_files_only,
**_video_comfy_backends(
- fam, base, target, plan, bf16_plan, keep = comfy_keep
+ fam,
+ base,
+ target,
+ plan,
+ bf16_plan,
+ keep = comfy_keep,
+ priced_offload = comfy_priced_offload,
+ widened_mib = comfy_widened_mib,
),
family = fam.name,
target = target,
@@ -8823,7 +8892,14 @@ class VideoBackend:
scan,
kwargs,
**_video_comfy_backends(
- fam, base, target, plan, bf16_plan, keep = comfy_keep
+ fam,
+ base,
+ target,
+ plan,
+ bf16_plan,
+ keep = comfy_keep,
+ priced_offload = comfy_priced_offload,
+ widened_mib = comfy_widened_mib,
),
family = fam.name,
target = target,
diff --git a/studio/backend/core/inference/video_ltx2.py b/studio/backend/core/inference/video_ltx2.py
index 69989fc8d85..1223194bbdb 100644
--- a/studio/backend/core/inference/video_ltx2.py
+++ b/studio/backend/core/inference/video_ltx2.py
@@ -1248,6 +1248,7 @@ def load_ltx23_comfy_transformer(
local_files_only: bool = False,
int8_backend: Optional[str] = None,
fp8_backend: Optional[str] = None,
+ fp8_storage: Optional[bool] = None,
family: Optional[str] = None,
target: Any = None,
logger: Any = None,
@@ -1275,6 +1276,7 @@ def load_ltx23_comfy_transformer(
},
int8_backend = int8_backend,
fp8_backend = fp8_backend,
+ fp8_storage = fp8_storage,
family = family,
target = target,
logger = logger,
diff --git a/studio/backend/core/inference/video_ltx25.py b/studio/backend/core/inference/video_ltx25.py
index 07ada73b195..9946f17ae88 100644
--- a/studio/backend/core/inference/video_ltx25.py
+++ b/studio/backend/core/inference/video_ltx25.py
@@ -413,6 +413,7 @@ def load_ltx25_comfy_transformer(
local_files_only: bool = False,
int8_backend: Optional[str] = None,
fp8_backend: Optional[str] = None,
+ fp8_storage: Optional[bool] = None,
family: Optional[str] = None,
target: Any = None,
logger: Any = None,
@@ -438,6 +439,7 @@ def load_ltx25_comfy_transformer(
},
int8_backend = int8_backend,
fp8_backend = fp8_backend,
+ fp8_storage = fp8_storage,
family = family,
target = target,
logger = logger,
diff --git a/studio/backend/core/inference/video_minimax_h3_comfy.py b/studio/backend/core/inference/video_minimax_h3_comfy.py
index d30c132aef1..53147d0742a 100644
--- a/studio/backend/core/inference/video_minimax_h3_comfy.py
+++ b/studio/backend/core/inference/video_minimax_h3_comfy.py
@@ -188,6 +188,8 @@ def load_h3_comfy_transformer(
fp8_backend: Optional[str],
target: Any = None,
logger: Any = None,
+ # H3's pin / stream / residency path is built on torchao weights: dequantize what torchao does not take
+ fp8_storage: Optional[bool] = False,
) -> Any:
"""The H3 denoiser of a ComfyUI-quantized file, int8 / fp8 codes kept, curve adaLN installed."""
from .diffusion_comfy_quant import load_comfy_quant_transformer
@@ -210,6 +212,7 @@ def load_h3_comfy_transformer(
},
int8_backend = int8_backend,
fp8_backend = fp8_backend,
+ fp8_storage = fp8_storage,
family = "minimax-h3",
target = target,
logger = logger,
diff --git a/studio/backend/tests/test_diffusion_comfy_quant.py b/studio/backend/tests/test_diffusion_comfy_quant.py
index eaaa4fceb97..4a618202833 100644
--- a/studio/backend/tests/test_diffusion_comfy_quant.py
+++ b/studio/backend/tests/test_diffusion_comfy_quant.py
@@ -13,6 +13,7 @@ ConvRot rotation is installed on exactly those, and that what cannot be mapped i
from __future__ import annotations
import json
+import sys
import types
import pytest
@@ -443,9 +444,14 @@ def test_fp8_layers_dequantize_with_their_scale(tmp_path, monkeypatch):
tensors.update({f"{layer}.weight": codes, f"{layer}.scale_weight": scale.reshape(1)})
tensors["scaled_fp8"] = torch.zeros(2, dtype = torch.float8_e4m3fn)
path = _save(tmp_path / "legacy_fp8_scaled.safetensors", tensors)
- model = _load(path)
+ model = _load(path, fp8_storage = False)
assert torch.equal(model.blocks[0].to_out.weight, codes.float() * scale)
assert model._unsloth_comfy_quant["dequantized"] == 1
+ stored = _load(path, fp8_storage = True)
+ assert stored._unsloth_comfy_quant["dequantized"] == 0
+ assert stored._unsloth_comfy_quant["fp8_stored"] == 1
+ assert torch.equal(stored.blocks[0].to_out.weight, codes.float() * scale)
+ assert torch.equal(stored.blocks[0].to_out.weight_q, codes.view(torch.uint8))
def test_a_converter_that_mixes_columns_is_refused(comfy_file, monkeypatch):
@@ -591,7 +597,7 @@ def test_fp8_native_backend_keeps_codes_and_computes_the_dense_product(tmp_path,
def test_fp8_without_a_runtime_dequantizes(tmp_path, monkeypatch):
monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
path, _dense, tensors = _fp8_file(tmp_path)
- model = _load_bf16(path, fp8_backend = None)
+ model = _load_bf16(path, fp8_backend = None, fp8_storage = False)
assert model._unsloth_comfy_quant["fp8"] == 0
assert model._unsloth_comfy_quant["dequantized"] == 6
w = tensors["blocks.0.out.weight"].float() * tensors["blocks.0.out.weight_scale"]
@@ -601,13 +607,127 @@ def test_fp8_without_a_runtime_dequantizes(tmp_path, monkeypatch):
def test_e5m2_and_non_bf16_pipelines_dequantize(tmp_path, monkeypatch):
monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
path, _dense, _tensors = _fp8_file(tmp_path, fmt = "float8_e5m2")
- assert _load_bf16(path, fp8_backend = "torchao")._unsloth_comfy_quant["fp8"] == 0
+ assert (
+ _load_bf16(path, fp8_backend = "torchao", fp8_storage = False)._unsloth_comfy_quant["fp8"] == 0
+ )
path, _dense, _tensors = _fp8_file(tmp_path)
- model = _load(path, fp8_backend = "torchao") # float32 pipeline
+ model = _load(path, fp8_backend = "torchao", fp8_storage = False) # float32 pipeline
assert model._unsloth_comfy_quant["fp8"] == 0
assert model._unsloth_comfy_quant["dequantized"] == 6
+def _linear_bytes(model) -> int:
+ """Bytes the model's Linear-shaped weights hold (dense ``weight`` or stored ``weight_q`` + ``weight_scale``)."""
+ total = 0
+ for name, tensor in model.state_dict().items():
+ if name.endswith((".weight", ".weight_q", ".weight_scale")) and not name.startswith(
+ "norm."
+ ):
+ total += tensor.numel() * tensor.element_size()
+ return total
+
+
+ "fmt, dtype",
+ [
+ ("float8_e4m3fn", torch.bfloat16),
+ ("float8_e4m3fn", torch.float16),
+ ("float8_e4m3fn", torch.float32),
+ ("float8_e5m2", torch.bfloat16),
+ ],
+)
[email protected]("per_row", [False, True])
+def test_fp8_without_a_runtime_stays_fp8_and_matches_the_dequant_exactly(
+ tmp_path, monkeypatch, fmt, dtype, per_row
+):
+ """No fp8 GEMM: codes stay 1 byte per weight and outputs are bit-identical to the load-time dequant."""
+ monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
+ path, _dense, tensors = _fp8_file(tmp_path, fmt = fmt, per_row = per_row)
+
+ def load(**kwargs):
+ return cq.load_comfy_quant_transformer(
+ _Tiny,
+ path,
+ cq.refuse_comfy_quant(path),
+ {"torch_dtype": dtype, "config": "base/repo", "subfolder": "transformer"},
+ family = "z-image",
+ int8_backend = None,
+ fp8_backend = None,
+ **kwargs,
+ )
+
+ widened = load(fp8_storage = False)
+ stored = load(fp8_storage = True)
+ assert widened._unsloth_comfy_quant["dequantized"] == 6
+ assert stored._unsloth_comfy_quant["dequantized"] == 0
+ assert stored._unsloth_comfy_quant["fp8_stored"] == 10
+ assert stored._unsloth_comfy_quant["fp8"] == 0 # storage is not an fp8 runtime
+ assert getattr(stored, "_unsloth_runtime_quant", None) is None
+ for b in range(2):
+ for leaf in ("to_q", "to_k", "to_v", "to_out"):
+ layer = getattr(stored.blocks[b], leaf)
+ assert layer.weight_q.dtype is torch.uint8 and layer.fp8_format == fmt
+ assert layer.weight_scale.view(torch.float32).shape == (DIM,)
+ assert torch.equal(layer.weight, getattr(widened.blocks[b], leaf).weight)
+ # half the bytes of a 16-bit dequant (a quarter of fp32), plus a 4-byte scale per row
+ rows = 2 * (4 * DIM + 2 * DIM)
+ assert _linear_bytes(stored) == rows * DIM + rows * 4
+ assert _linear_bytes(widened) == rows * DIM * torch.finfo(dtype).bits // 8
+ x = torch.randn(3, 7, DIM).to(dtype)
+ for b in range(2):
+ for leaf in ("to_q", "to_out"):
+ got = getattr(stored.blocks[b], leaf)(x)
+ want = getattr(widened.blocks[b], leaf)(x)
+ assert got.dtype is dtype and torch.equal(got, want)
+ got = stored.blocks[b].adaLN_modulation(x)
+ assert torch.equal(got, widened.blocks[b].adaLN_modulation(x))
+ # no plan to judge by (None): widen, unless the env opts in; the kill switch beats an explicit request
+ monkeypatch.delenv(cq.FP8_STORAGE_ENV, raising = False)
+ assert load()._unsloth_comfy_quant["dequantized"] == 6
+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "1")
+ assert load()._unsloth_comfy_quant["fp8_stored"] == 10
+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "0")
+ assert cq.fp8_storage_wanted(None, offload = True) is False
+
+
+def test_fp8_storage_follows_the_device_lora_and_kill_switch(monkeypatch):
+ monkeypatch.delenv(cq.FP8_STORAGE_ENV, raising = False)
+ cuda = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)
+ assert cq.comfy_fp8_storage(cuda) is True
+ assert cq.comfy_fp8_storage(types.SimpleNamespace(device = "cpu")) is True
+ assert cq.comfy_fp8_storage(None) is True
+ # MPS has no float8 dtypes; a LoRA needs dense Linears to attach to
+ assert cq.comfy_fp8_storage(types.SimpleNamespace(device = "mps")) is False
+ assert cq.comfy_fp8_storage(cuda, lora = True) is False
+ for off in ("0", "off", "false"):
+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, off)
+ assert cq.comfy_fp8_storage(cuda) is False
+
+
+def test_fp8_storage_keeps_fp32_modules_and_the_torchao_layers_apart(tmp_path, monkeypatch):
+ """Only layers the fp8 GEMM runtime skips are stored; fp32-kept Linears still widen to fp32."""
+ monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
+ path, _dense, _tensors = _fp8_file(tmp_path)
+
+ class _Keep(_Tiny):
+ _keep_in_fp32_modules = ["adaLN_modulation"]
+
+ model = cq.load_comfy_quant_transformer(
+ _Keep,
+ path,
+ cq.refuse_comfy_quant(path),
+ {"torch_dtype": torch.float16, "config": "base/repo", "subfolder": "transformer"},
+ family = "z-image",
+ int8_backend = None,
+ fp8_backend = None,
+ fp8_storage = True,
+ )
+ assert model._unsloth_comfy_quant["fp8_stored"] == 8
+ assert model._unsloth_comfy_quant["dequantized"] == 2
+ assert type(model.blocks[0].adaLN_modulation[0]) is nn.Linear
+ assert model.blocks[0].adaLN_modulation[0].weight.dtype is torch.float32
+
+
@pytest.mark.parametrize(
"native, supported, offload, dtype, env, want",
[
@@ -843,3 +963,389 @@ def test_buffers_built_in_init_follow_the_compute_dtype(comfy_file):
)
assert model.freqs.dtype == torch.bfloat16
assert model.blocks[0].to_q.weight_q.dtype == torch.int8
+
+
+def test_a_lora_on_stored_fp8_layers_is_refused_with_the_way_out(tmp_path, monkeypatch):
+ """Stored fp8 Linears are not nn.Linear: a generation-time LoRA is refused by name."""
+ import threading
+
+ from core.inference.diffusion import DiffusionBackend
+
+ monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
+ path, _dense, _tensors = _fp8_file(tmp_path)
+ model = _load_bf16(path, fp8_backend = None, fp8_storage = True)
+ assert cq.fp8_stored_layers(model) == 10
+ assert cq.fp8_stored_layers(_load_bf16(path, fp8_backend = None, fp8_storage = False)) == 0
+ assert cq.fp8_stored_layers(None) == 0
+ pipe = types.SimpleNamespace(transformer = model, _unsloth_loras = ())
+ state = types.SimpleNamespace(pipe = pipe, transformer_quant = None)
+ backend = DiffusionBackend.__new__(DiffusionBackend)
+ with pytest.raises(ValueError, match = "stored as fp8"):
+ DiffusionBackend._apply_loras(backend, state, [("some/lora", 1.0)], threading.Event())
+ DiffusionBackend._apply_loras(backend, state, [], threading.Event())
+
+
+def test_fp8_storage_default_rule_is_offload_or_host_ram(monkeypatch):
+ """Unset env stores only when widening would offload or crowd host RAM; ``1`` always; MPS and LoRA never."""
+ monkeypatch.delenv(cq.FP8_STORAGE_ENV, raising = False)
+ cuda = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)
+ want = cq.fp8_storage_wanted
+ assert want(cuda) is False
+ assert want(cuda, offload = True) is True
+ reserve = cq.FP8_STORAGE_HOST_RESERVE_MIB
+ assert want(cuda, widened_mib = 24000, host_available_mib = 24000 + reserve + 1) is False
+ assert want(cuda, widened_mib = 24000, host_available_mib = 24000 + reserve - 1) is True
+ assert want(cuda, offload = True, lora = True) is False
+ assert want(types.SimpleNamespace(device = "mps"), offload = True) is False
+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "1")
+ assert want(cuda) is True
+
+
+def test_fp8_upcast_is_read_from_the_header_not_the_basename(tmp_path):
+ """A plain fp8 file named without 'fp8' still widens ~2x (and so can be stored); a bf16 one named 'fp8' does not."""
+ from safetensors.torch import save_file
+
+ from core.inference import diffusion as d
+
+ fam = types.SimpleNamespace(single_file_is_pipeline = False)
+ plain = tmp_path / "krea2.safetensors"
+ save_file({"blocks.0.weight": torch.zeros(4, 4, dtype = torch.float8_e4m3fn)}, str(plain))
+ dense = tmp_path / "krea2_fp8.safetensors"
+ save_file({"blocks.0.weight": torch.zeros(4, 4, dtype = torch.bfloat16)}, str(dense))
+ assert cq.header_has_fp8_weights(str(plain)) is True
+ assert cq.header_has_fp8_weights(str(dense)) is False
+ assert d._single_file_fp8_upcast(str(plain), fam) is True
+ assert d._single_file_fp8_upcast(str(dense), fam) is False
+ broken = tmp_path / "broken_fp8.safetensors"
+ broken.write_bytes(b"\x00" * 4)
+ assert cq.header_has_fp8_weights(str(broken)) is None
+ assert d._single_file_fp8_upcast(str(broken), fam) is True
+ assert (
+ d._single_file_fp8_upcast(str(plain), types.SimpleNamespace(single_file_is_pipeline = True))
+ is False
+ )
+
+
+def test_fnuz_fp8_headers_are_priced_widened_and_never_stored(tmp_path):
+ """FNUZ fp8 is seen by the header predicate and always priced widened."""
+ from safetensors.torch import save_file
+
+ from core.inference import diffusion as d
+
+ fam = types.SimpleNamespace(single_file_is_pipeline = False)
+ fnuz = tmp_path / "krea2.safetensors"
+ save_file({"blocks.0.weight": torch.zeros(4, 4, dtype = torch.float8_e4m3fnuz)}, str(fnuz))
+ ocp = tmp_path / "krea2_ocp.safetensors"
+ save_file({"blocks.0.weight": torch.zeros(4, 4, dtype = torch.float8_e4m3fn)}, str(ocp))
+ assert cq.header_has_fp8_weights(str(fnuz)) is True
+ assert d._single_file_fp8_upcast(str(fnuz), fam) is True
+ assert cq.header_has_fp8_weights(str(fnuz), fnuz_only = True) is True
+ assert cq.header_has_fp8_weights(str(ocp), fnuz_only = True) is False
+
+
+def _plan_harness(
+ monkeypatch,
+ tmp_path,
+ *,
+ fam,
+ resident_at,
+ fits_mib,
+ comfy_mib = None,
+):
+ """Run ``_plan_memory`` for a single file with a fake planner (resident iff the DiT fits ``fits_mib``)."""
+ from core.inference import diffusion as d
+ from core.inference import diffusion_memory as dm
+
+ path = tmp_path / "dit.safetensors"
+ if not path.exists():
+ path.write_bytes(b"\x00" * 16)
+ seen: list = []
+
+ def plan(**kwargs):
+ mib = kwargs["model_dense_mib"]
+ seen.append(mib)
+ return types.SimpleNamespace(resident = mib is not None and mib <= fits_mib)
+
+ monkeypatch.setattr(d, "plan_diffusion_memory", plan)
+ monkeypatch.setattr(d, "plan_keeps_transformer_resident", lambda p: p.resident)
+ monkeypatch.setattr(d, "comfy_fp8_storage", lambda target, lora = False: True)
+ monkeypatch.setattr(dm, "_available_system_memory_mib", lambda: 10**7)
+ monkeypatch.setattr(
+ d.DiffusionBackend, "_companion_cache_bytes", staticmethod(lambda *a, **k: 0)
+ )
+ monkeypatch.setattr(
+ d.DiffusionBackend, "_text_encoder_cache_bytes", staticmethod(lambda *a, **k: 0)
+ )
+ monkeypatch.setattr(d.DiffusionBackend, "_supplied_component_mib", lambda self, *a, **k: None)
+ if comfy_mib is not None:
+ monkeypatch.setattr(
+ d.DiffusionBackend, "_comfy_single_file_resident_mib", staticmethod(comfy_mib)
+ )
+ else:
+ monkeypatch.setattr(
+ d.DiffusionBackend,
+ "_comfy_single_file_resident_mib",
+ staticmethod(lambda *a, **k: None),
+ )
+ monkeypatch.setattr(d, "file_size_mib", lambda p: 1000)
+ monkeypatch.setattr(
+ d,
+ "estimate_safetensors_dense_mib",
+ lambda size, fp8_upcast = False: size * (2 if fp8_upcast else 1),
+ )
+ from core.inference.diffusion_device import resolve_diffusion_device_target
+
+ backend = d.DiffusionBackend()
+ final = backend._plan_memory(
+ resolve_diffusion_device_target(),
+ str(resident_at or path),
+ "base/repo",
+ fam,
+ None,
+ False,
+ kind = "single_file",
+ )
+ return backend, final, seen, str(resident_at or path)
+
+
+def test_offloaded_plan_prices_fp8_with_the_offloaded_backend_and_picks_storage(
+ monkeypatch, tmp_path
+):
+ """Offload loses torchao fp8: the plan re-prices offloaded and picks storage."""
+ calls = []
+
+ def comfy_mib(
+ path,
+ fam,
+ target,
+ base,
+ *,
+ lora = False,
+ fp8_storage = False,
+ offload = False,
+ ):
+ calls.append((fp8_storage, offload))
+ if not offload:
+ return 1000 # torchao keeps the codes either way
+ return 1000 if fp8_storage else 2000
+
+ fam = types.SimpleNamespace(name = "flux.1", single_file_is_pipeline = False)
+ # fits_mib below 1000: the plan offloads at every size
+ backend, final, seen, path = _plan_harness(
+ monkeypatch, tmp_path, fam = fam, resident_at = None, fits_mib = 500, comfy_mib = comfy_mib
+ )
+ assert (False, True) in calls and (True, True) in calls
+ assert backend._fp8_storage_picks[path] is True
+ assert seen[-1] == 1000 and 2000 in seen
+ assert final.resident is False
+
+
+def test_offloaded_plan_without_storage_is_priced_widened(monkeypatch, tmp_path):
+ """No storage possible (LoRA / kill switch): the offloaded plan carries the widened size the loader builds."""
+ from core.inference import diffusion as d
+
+ def comfy_mib(
+ path,
+ fam,
+ target,
+ base,
+ *,
+ lora = False,
+ fp8_storage = False,
+ offload = False,
+ ):
+ return 2000 if offload and not fp8_storage else 1000
+
+ real_harness_storage = []
+ monkeypatch.setattr(
+ d.DiffusionBackend,
+ "_fp8_storage_off",
+ True,
+ raising = False,
+ )
+ fam = types.SimpleNamespace(name = "flux.1", single_file_is_pipeline = False)
+ orig_setattr = monkeypatch.setattr
+
+ def setattr_no_storage(target, name, value, *a, **k):
+ if target is d and name == "comfy_fp8_storage":
+ value = lambda target, lora = False: False # noqa: E731
+ real_harness_storage.append(name)
+ return orig_setattr(target, name, value, *a, **k)
+
+ monkeypatch.setattr = setattr_no_storage
+ try:
+ backend, final, seen, path = _plan_harness(
+ monkeypatch, tmp_path, fam = fam, resident_at = None, fits_mib = 500, comfy_mib = comfy_mib
+ )
+ finally:
+ monkeypatch.setattr = orig_setattr
+ assert seen[-1] == 2000
+ assert backend._fp8_storage_picks[path] is False
+
+
+def test_a_resident_plan_keeps_the_resident_backend_price(monkeypatch, tmp_path):
+ """Resident torchao fp8: no offload re-price, no storage."""
+ calls = []
+
+ def comfy_mib(
+ path,
+ fam,
+ target,
+ base,
+ *,
+ lora = False,
+ fp8_storage = False,
+ offload = False,
+ ):
+ calls.append(offload)
+ return 1000 if not offload else 2000
+
+ fam = types.SimpleNamespace(name = "flux.1", single_file_is_pipeline = False)
+ backend, final, seen, path = _plan_harness(
+ monkeypatch, tmp_path, fam = fam, resident_at = None, fits_mib = 5000, comfy_mib = comfy_mib
+ )
+ assert True not in calls and seen == [1000]
+ assert backend._fp8_storage_picks[path] is False and final.resident is True
+
+
+def test_fnuz_krea2_file_is_not_planned_at_the_stored_size(monkeypatch, tmp_path):
+ """A Krea 2 FNUZ file widens under storage too, so the planner records no storage pick for it."""
+ from safetensors.torch import save_file
+
+ from core.inference.diffusion_krea2 import KREA2_FAMILY_NAME
+
+ fam = types.SimpleNamespace(name = KREA2_FAMILY_NAME, single_file_is_pipeline = False)
+ for dtype, picked in ((torch.float8_e4m3fnuz, False), (torch.float8_e4m3fn, True)):
+ path = tmp_path / f"krea2_{str(dtype).split('_')[-1]}.safetensors"
+ save_file({"blocks.0.weight": torch.zeros(4, 4, dtype = dtype)}, str(path))
+ backend, final, seen, p = _plan_harness(
+ monkeypatch, tmp_path, fam = fam, resident_at = path, fits_mib = 500
+ )
+ assert backend._fp8_storage_picks[p] is picked, dtype
+ assert seen[-1] == (1000 if picked else 2000)
+
+
+def test_a_plan_priced_with_the_offload_only_backend_keeps_it_when_resident(monkeypatch, tmp_path):
+ """Int8 priced via the offload twin: the re-plan fits resident but the loader must still probe offloaded."""
+
+ def comfy_mib(
+ path,
+ fam,
+ target,
+ base,
+ *,
+ lora = False,
+ fp8_storage = False,
+ offload = False,
+ ):
+ return 1000 if offload else 2000
+
+ fam = types.SimpleNamespace(name = "flux.1", single_file_is_pipeline = False)
+ backend, final, seen, path = _plan_harness(
+ monkeypatch, tmp_path, fam = fam, resident_at = None, fits_mib = 1500, comfy_mib = comfy_mib
+ )
+ assert seen == [2000, 1000] and final.resident is True
+ assert backend._comfy_offload_for(final, path) is True
+ # Not priced offloaded: a resident plan probes resident.
+ backend._comfy_priced_offload[path] = False
+ assert backend._comfy_offload_for(final, path) is False
+
+
+def test_a_plan_repriced_larger_under_offload_does_not_force_the_offload_probe(
+ monkeypatch, tmp_path
+):
+ def comfy_mib(
+ path,
+ fam,
+ target,
+ base,
+ *,
+ lora = False,
+ fp8_storage = False,
+ offload = False,
+ ):
+ return 2000 if offload else 1000
+
+ fam = types.SimpleNamespace(name = "flux.1", single_file_is_pipeline = False)
+ backend, final, seen, path = _plan_harness(
+ monkeypatch, tmp_path, fam = fam, resident_at = None, fits_mib = 500, comfy_mib = comfy_mib
+ )
+ assert backend._comfy_priced_offload[path] is False
+ assert backend._comfy_offload_for(types.SimpleNamespace(resident = True), path) is False
+
+
+def test_fp8_stored_pricing_widens_keep_fp32_layers_as_the_loader_does(tmp_path):
+ """``_keep_in_fp32_modules`` linears are never priced at one byte, under either name."""
+ n = 1024
+ names = ("blocks.0.ffn", "condition_embedder.time_embedder.linear_1", "time_embedding.0")
+ tensors, layers = {}, {}
+ for name in names:
+ tensors[f"{name}.weight"] = torch.zeros(n, n, dtype = torch.float8_e4m3fn)
+ tensors[f"{name}.weight_scale"] = torch.tensor(1.0)
+ layers[name] = {"format": "float8_e4m3fn"}
+ path = _save(
+ tmp_path / "wan.safetensors",
+ tensors,
+ {cq.QUANT_METADATA_KEY: json.dumps({"layers": layers})},
+ )
+ scan = cq.scan_comfy_quant(path)
+ kw = dict(keep_int8 = False, keep_fp8 = False, fp8_stored = True)
+ one = n * n / 2**20 # MiB per layer per byte
+ plain = cq.comfy_resident_mib(path, scan, **kw)
+ assert plain == pytest.approx(3 * one, abs = 1)
+ keep = ("time_embedder", "scale_shift_table")
+ fp16 = cq.comfy_resident_mib(path, scan, **kw, keep_fp32 = keep, keep_fp32_bytes = 4)
+ assert fp16 == pytest.approx(one + 2 * 4 * one, abs = 1)
+ bf16 = cq.comfy_resident_mib(path, scan, **kw, keep_fp32 = keep)
+ assert bf16 == pytest.approx(one + 2 * 2 * one, abs = 1)
+ for name in names[1:]:
+ assert not cq.fp8_stored_by_header("float8_e4m3fn", (n, n), name, keep)
+ assert cq.fp8_stored_by_header("float8_e4m3fn", (n, n), names[0], keep)
+
+
+def _fake_diffusers(monkeypatch):
+ # CPU CI may run without diffusers (or with a stub): pin the two classes these tests read.
+ class WanTransformer3DModel:
+ _keep_in_fp32_modules = ["time_embedder", "scale_shift_table", "norm1", "norm2", "norm3"]
+
+ class FluxTransformer2DModel:
+ pass
+
+ fake = types.SimpleNamespace(
+ WanTransformer3DModel = WanTransformer3DModel, FluxTransformer2DModel = FluxTransformer2DModel
+ )
+ monkeypatch.setitem(sys.modules, "diffusers", fake)
+
+
+def test_real_wan_transformer_keeps_its_time_embedder_in_fp32():
+ try:
+ import diffusers
+ cls = getattr(diffusers, "WanTransformer3DModel", None)
+ except Exception: # noqa: BLE001 -- no usable diffusers here
+ cls = None
+ if cls is None:
+ pytest.skip("this diffusers has no WanTransformer3DModel")
+ wan = types.SimpleNamespace(transformer_class = "WanTransformer3DModel")
+ assert "time_embedder" in cq.keep_fp32_modules_for(wan)
+
+
+def test_keep_fp32_modules_come_from_the_family_transformer(monkeypatch):
+ _fake_diffusers(monkeypatch)
+ wan = types.SimpleNamespace(transformer_class = "WanTransformer3DModel")
+ assert "time_embedder" in cq.keep_fp32_modules_for(wan)
+ assert cq.keep_fp32_modules_for(types.SimpleNamespace(transformer_class = "NoSuchModel")) == ()
+ assert cq.keep_fp32_modules_for(types.SimpleNamespace()) == ()
+
+
+def test_planner_prices_keep_fp32_layers_at_fp32_only_on_fp16(monkeypatch):
+ from core.inference import diffusion as d
+
+ _fake_diffusers(monkeypatch)
+
+ wan = types.SimpleNamespace(transformer_class = "WanTransformer3DModel")
+ fp16 = d._comfy_keep_fp32_pricing(wan, types.SimpleNamespace(dtype = torch.float16))
+ bf16 = d._comfy_keep_fp32_pricing(wan, types.SimpleNamespace(dtype = torch.bfloat16))
+ assert "time_embedder" in fp16["keep_fp32"] and fp16["keep_fp32_bytes"] == 4
+ assert bf16["keep_fp32_bytes"] is None
+ flux = types.SimpleNamespace(transformer_class = "FluxTransformer2DModel")
+ assert d._comfy_keep_fp32_pricing(flux, types.SimpleNamespace(dtype = torch.float16)) == {}
diff --git a/studio/backend/tests/test_diffusion_compile_cache_variant.py b/studio/backend/tests/test_diffusion_compile_cache_variant.py
index 3ec4ca2ae7d..2cb278c74e3 100644
--- a/studio/backend/tests/test_diffusion_compile_cache_variant.py
+++ b/studio/backend/tests/test_diffusion_compile_cache_variant.py
@@ -258,3 +258,24 @@ def test_per_row_and_per_tensor_scales_key_apart():
per_tensor = cc._tensor_signature(FakeFp8([64, 256], (1,)))
assert per_row != per_tensor
assert "block_size=1,full" in per_row and "block_size=full,full" in per_tensor
+
+
+def _stored_fp8(fmt):
+ from core.inference.diffusion_native_quant import NATIVE_FP8, native_linear_class
+
+ m = _model()
+ lin = m[0]
+ fp8 = getattr(torch, fmt)
+ w = lin.weight.detach().float()
+ scale = w.abs().amax(dim = 1, keepdim = True) / torch.finfo(fp8).max
+ m[0] = native_linear_class()(lin, NATIVE_FP8, codes = (w / scale).to(fp8), scale = scale)
+ return m
+
+
+def test_stored_fp8_formats_key_apart_and_each_keys_stable():
+ """E4M3 and E5M2 codes decode through different integer graphs, so they never share a bundle."""
+ e4, e5 = _stored_fp8("float8_e4m3fn"), _stored_fp8("float8_e5m2")
+ assert e4[0].fp8_format == "float8_e4m3fn" and e5[0].fp8_format == "float8_e5m2"
+ assert _key(e4, "fp8") != _key(e5, "fp8")
+ assert _key(e4, "fp8") == _key(_stored_fp8("float8_e4m3fn"), "fp8")
+ assert _key(e5, "fp8") == _key(_stored_fp8("float8_e5m2"), "fp8")
diff --git a/studio/backend/tests/test_diffusion_native_quant.py b/studio/backend/tests/test_diffusion_native_quant.py
index 2812df5a681..57502a4dda8 100644
--- a/studio/backend/tests/test_diffusion_native_quant.py
+++ b/studio/backend/tests/test_diffusion_native_quant.py
@@ -1211,3 +1211,43 @@ def test_a_rotated_layer_passes_an_empty_input_through():
)
out = layer(torch.zeros(2, 0, 256, dtype = torch.bfloat16))
assert out.shape == (2, 0, 64)
+
+
[email protected]("fmt", ["float8_e4m3fn", "float8_e5m2"])
+def test_fp8_integer_decode_equals_the_cast_bit_for_bit(fmt):
+ """Integer decode equals the eager cast on all 256 codes."""
+ from core.inference.diffusion_native_quant import fp8_bits_to_float32
+
+ bits = torch.arange(256, dtype = torch.int32).to(torch.uint8)
+ want = bits.view(getattr(torch, fmt)).to(torch.float32)
+ got = fp8_bits_to_float32(bits, fmt)
+ nan = want.isnan()
+ assert torch.equal(got.isnan(), nan)
+ assert torch.equal(got[~nan].view(torch.int32), want[~nan].view(torch.int32))
+
+
[email protected]("fmt", ["float8_e4m3fn", "float8_e5m2"])
+def test_fp8_codes_layer_matches_the_dequant_formula_eager_and_compiled(fmt):
+ """A native fp8 layer built from checkpoint codes + per-row scale: its forward equals F.linear on
+ ``(codes.float() * scale).to(dtype)`` (the load-time dequant) in eager, and the compiled forward (integer
+ decode) matches eager exactly with no graph break."""
+ from core.inference.diffusion_native_quant import NATIVE_FP8, native_linear_class
+
+ torch.manual_seed(0)
+ fp8 = getattr(torch, fmt)
+ base = torch.nn.Linear(64, 48).to(torch.bfloat16)
+ w = torch.randn(48, 64)
+ scale = w.abs().amax(dim = 1, keepdim = True) / torch.finfo(fp8).max
+ codes = (w / scale).to(fp8)
+ layer = native_linear_class()(base, NATIVE_FP8, codes = codes, scale = scale)
+ assert layer.weight_q.dtype is torch.uint8 and layer.fp8_format == fmt
+ dense = (codes.to(torch.float32) * scale).to(torch.bfloat16)
+ x = torch.randn(3, 64, dtype = torch.bfloat16)
+ want = torch.nn.functional.linear(x, dense, base.bias)
+ assert torch.equal(layer.weight, dense)
+ assert torch.equal(layer(x), want)
+ torch._dynamo.reset()
+ explained = torch._dynamo.explain(layer)(x)
+ assert explained.graph_break_count == 0
+ compiled = torch.compile(layer, fullgraph = True)
+ assert torch.equal(compiled(x), layer(x))
diff --git a/studio/backend/tests/test_diffusion_offload_precision.py b/studio/backend/tests/test_diffusion_offload_precision.py
index bc55b7328e8..196f1308abd 100644
--- a/studio/backend/tests/test_diffusion_offload_precision.py
+++ b/studio/backend/tests/test_diffusion_offload_precision.py
@@ -309,6 +309,26 @@ def test_render_grad_mode(policy, transformer_quant, text_encoder_quant, no_grad
assert dmod._torchao_render_needs_no_grad(state) is no_grad
+ "policy, no_grad",
+ [(OFFLOAD_NONE, False), (OFFLOAD_MODEL, True), (OFFLOAD_GROUP, True)],
+)
+def test_offloaded_stored_fp8_renders_under_no_grad(policy, no_grad):
+ """Stored fp8 codes are plain buffers: moved under inference_mode they are inference tensors the keep-host
+ offload cannot version, so each offload would copy 12 GB back to fresh host memory (Krea 2)."""
+ transformer = types.SimpleNamespace(_unsloth_fp8_stored = 256)
+ state = types.SimpleNamespace(
+ offload_policy = policy,
+ transformer_quant = None,
+ text_encoder_quant = None,
+ pipe = types.SimpleNamespace(transformer = transformer),
+ )
+ assert dmod._torchao_render_needs_no_grad(state) is no_grad
+ from core.inference import video as vid
+
+ assert vid._render_under_no_grad(state) is no_grad
+
+
def test_render_uses_the_widened_switch():
import inspect
diff --git a/studio/backend/tests/test_diffusion_single_file_host_ram.py b/studio/backend/tests/test_diffusion_single_file_host_ram.py
index c3ce4d19c23..8da0aa8de08 100644
--- a/studio/backend/tests/test_diffusion_single_file_host_ram.py
+++ b/studio/backend/tests/test_diffusion_single_file_host_ram.py
@@ -218,3 +218,52 @@ def test_kill_switch_charges_nothing(tmp_path, monkeypatch, krea2, kind):
fp8_backend = None,
)
assert charged == []
+
+
[email protected]("kind", ["fp8", "fp8_scaled"])
+def test_a_file_that_fits_only_stored_loads_with_fp8_storage_and_is_refused_widened(
+ tmp_path, monkeypatch, krea2, kind
+):
+ """fp8 storage keeps the codes memory-mapped, so the up-front estimate prices them at their stored bytes (plus
+ the fp32 row scales): free RAM between the stored and the widened need loads with storage, and is refused when
+ UNSLOTH_DIFFUSION_FP8_STORAGE=0 makes the load widen."""
+ path, _ = _write(tmp_path, kind)
+ keep = krea2._keep_in_fp32_modules
+ # Price in bytes, no reserve: the tiny model's needs are a few KB apart.
+ monkeypatch.setattr(conv, "_MIB", 1)
+ monkeypatch.setattr(small_host, "host_reserve_mib", lambda total: 0)
+ if kind == "fp8":
+ widened = conv.converted_host_bytes(str(path), torch.bfloat16, keep_fp32 = keep)
+ stored = conv.converted_host_bytes(
+ str(path), torch.bfloat16, keep_fp32 = keep, dequantized = conv.fp8_stored_estimate(keep)
+ ) + conv._fp8_stored_scale_bytes(str(path), keep)
+ else:
+ scan = cq.refuse_comfy_quant(str(path))
+ widened = conv.converted_host_bytes(
+ str(path), torch.bfloat16, dequantized = {n + ".weight": 1 for n in scan.layers}.get
+ )
+ stored = conv.converted_host_bytes(
+ str(path), torch.bfloat16, dequantized = {n + ".weight": 0 for n in scan.layers}.get
+ )
+ assert 0 < stored < widened
+ _host(monkeypatch, available_mib = (stored + widened) // 2)
+
+ def load(**kw):
+ if kind == "fp8":
+ return conv.load_original_layout_transformer(krea2, str(path), dict(_KWARGS), **kw)
+ return cq.load_comfy_quant_transformer(
+ krea2,
+ str(path),
+ cq.refuse_comfy_quant(str(path)),
+ dict(_KWARGS),
+ int8_backend = None,
+ fp8_backend = None,
+ **kw,
+ )
+
+ model = load(fp8_storage = True)
+ assert model._unsloth_fp8_stored > 0
+ assert model.transformer_blocks[0].attn.to_q.weight_q.dtype == torch.uint8
+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "0")
+ with pytest.raises(RuntimeError, match = "Not enough system RAM"):
+ load(fp8_storage = cq.comfy_fp8_storage(None))
diff --git a/studio/backend/tests/test_diffusion_single_file_krea2_hunyuanimage.py b/studio/backend/tests/test_diffusion_single_file_krea2_hunyuanimage.py
index ed938c8cfac..c503ff7492c 100644
--- a/studio/backend/tests/test_diffusion_single_file_krea2_hunyuanimage.py
+++ b/studio/backend/tests/test_diffusion_single_file_krea2_hunyuanimage.py
@@ -468,6 +468,49 @@ def test_comfy_int8_krea2_file_loads_with_codes_exact(tmp_path, monkeypatch):
assert loaded._unsloth_comfy_quant["dequantized"] == len(expected)
[email protected]("fmt", ["float8_e4m3fn", "float8_e5m2"])
+def test_plain_fp8_krea2_file_keeps_its_linears_fp8_and_widens_exactly(tmp_path, monkeypatch, fmt):
+ """A plain (unscaled) fp8 file: with fp8_storage every Linear weight stays 1 byte per element and the forward
+ rebuilds exactly the bf16 weight the load-time cast held (fp8 values are exact in bf16), so outputs match."""
+ cls = _class("Krea2Transformer2DModel")
+ model = _tiny("Krea2Transformer2DModel", KREA2_CFG)
+ fp8 = getattr(torch, fmt)
+ tensors = {}
+ for k, v in krea2_original(model.state_dict()).items():
+ tensors[k] = (v.to(fp8) if v.dim() == 2 else v.to(torch.bfloat16)).contiguous()
+ path = tmp_path / "krea2_tiny_fp8.safetensors"
+ safetensors_torch.save_file(tensors, str(path))
+ monkeypatch.setattr(cls, "load_config", classmethod(lambda c, *a, **k: dict(KREA2_CFG)))
+ kwargs = {
+ "torch_dtype": torch.bfloat16,
+ "config": "krea/Krea-2-Turbo",
+ "subfolder": "transformer",
+ }
+ widened = conv.load_original_layout_transformer(cls, str(path), kwargs)
+ stored = conv.load_original_layout_transformer(cls, str(path), kwargs, fp8_storage = True)
+ linears = [n for n, m in widened.named_modules() if isinstance(m, torch.nn.Linear)]
+ assert linears
+ keep = cls._keep_in_fp32_modules or []
+ count = 0
+ for name in linears:
+ dense = widened.get_submodule(name)
+ layer = stored.get_submodule(name)
+ if dense.weight.dtype is not torch.bfloat16 or any(m in name.split(".") for m in keep):
+ assert type(layer) is torch.nn.Linear, name
+ continue
+ count += 1
+ assert layer.weight_q.dtype is torch.uint8 and layer.fp8_format == fmt, name
+ assert torch.equal(layer.weight, dense.weight), name
+ x = torch.randn(2, 5, layer.in_features, dtype = torch.bfloat16)
+ assert torch.equal(layer(x), dense(x)), name
+ assert count
+ assert stored._unsloth_fp8_stored == count
+ assert not getattr(widened, "_unsloth_fp8_stored", 0)
+ wide_bytes = sum(t.numel() * t.element_size() for t in widened.state_dict().values())
+ stored_bytes = sum(t.numel() * t.element_size() for t in stored.state_dict().values())
+ assert stored_bytes < 0.6 * wide_bytes
+
+
@pytest.mark.parametrize("prefix", ["", "model.model.", "diffusion_model."])
def test_plain_krea2_file_loads_like_the_base_repo(tmp_path, monkeypatch, prefix):
"""A plain bf16 ComfyUI file through Studio's loader (used while diffusers gives Krea 2 no
diff --git a/studio/backend/tests/test_gpu_residency_coexistence.py b/studio/backend/tests/test_gpu_residency_coexistence.py
index dde3fef1e42..1250333cc82 100644
--- a/studio/backend/tests/test_gpu_residency_coexistence.py
+++ b/studio/backend/tests/test_gpu_residency_coexistence.py
@@ -686,3 +686,48 @@ class TestKeptSlotReport(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
+
+
+def test_single_file_image_need_reads_fp8_from_the_header(tmp_path, monkeypatch):
+ """The residency estimate prices a single file like the load planner: a plain fp8 file named without 'fp8'
+ widens ~2x, a bf16 file named 'fp8' does not."""
+ import types
+
+ import torch
+ from safetensors.torch import save_file
+
+ import core.inference.diffusion as dmod
+ import core.inference.diffusion_engine_router as router
+ import core.inference.diffusion_families as dfam
+ import core.inference.gpu_residency as gr
+ from core.inference.sd_cpp_engine import ENGINE_DIFFUSERS
+
+ fam = types.SimpleNamespace(name = "krea-2", single_file_is_pipeline = False)
+ monkeypatch.setattr(dfam, "detect_family_for_pick", lambda *a, **k: fam)
+ monkeypatch.setattr(dfam, "resolve_base_repo", lambda f, b: b)
+ monkeypatch.setattr(dfam, "prefer_ungated_mirror", lambda r: r)
+ monkeypatch.setattr(router, "predict_engine", lambda *a, **k: ENGINE_DIFFUSERS)
+ monkeypatch.setattr(dmod, "_image_variant_hint", lambda *a, **k: None)
+ monkeypatch.setattr(
+ dmod.DiffusionBackend, "_companion_cache_bytes", staticmethod(lambda *a, **k: 1 << 20)
+ )
+
+ def need(path):
+ monkeypatch.setattr(gr, "_cached_pick", lambda repo, name: str(path))
+ request = types.SimpleNamespace(
+ model_path = "org/krea",
+ gguf_filename = path.name,
+ text_encoder_file = None,
+ vae_file = None,
+ model_kind = "single_file",
+ family_override = None,
+ base_repo = "org/base",
+ )
+ return gr._image_need(request, 0)[0]
+
+ plain = tmp_path / "krea2.safetensors"
+ save_file({"blocks.0.weight": torch.zeros(2048, 2048, dtype = torch.float8_e4m3fn)}, str(plain))
+ dense = tmp_path / "krea2_fp8.safetensors"
+ save_file({"blocks.0.weight": torch.zeros(1024, 2048, dtype = torch.bfloat16)}, str(dense))
+ # Both files are 4 MiB on disk: only the one that stores fp8 widens on load.
+ assert need(plain) > need(dense)
diff --git a/studio/backend/tests/test_video_comfy_quant.py b/studio/backend/tests/test_video_comfy_quant.py
index 68028862659..c3687c2711e 100644
--- a/studio/backend/tests/test_video_comfy_quant.py
+++ b/studio/backend/tests/test_video_comfy_quant.py
@@ -413,16 +413,50 @@ def test_video_backends_follow_studio_rules(monkeypatch):
assert vid._video_comfy_backends(fam, "b", None, _plan(True), _plan(True)) == {
"int8_backend": "torchao",
"fp8_backend": "torchao",
+ "fp8_storage": False,
}
- # either plan offloading the DiT means the hooks move it: int8 takes the torchao-free twin, fp8 dequantizes
+ # either plan offloading the DiT means the hooks move it: int8 takes the torchao-free twin, fp8 has no GEMM
+ # and stays stored as fp8, widened per layer
assert vid._video_comfy_backends(fam, "b", None, _plan(True), _plan(False)) == {
"int8_backend": "native",
"fp8_backend": None,
+ "fp8_storage": True,
}
assert vid._video_comfy_backends(fam, "b", None, _plan(True), keep = False) == {
"int8_backend": None,
"fp8_backend": None,
+ "fp8_storage": False,
}
+ h3 = types.SimpleNamespace(name = "minimax-h3")
+ assert (
+ vid._video_comfy_backends(h3, "b", None, _plan(True), _plan(False))["fp8_storage"] is False
+ )
+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "0")
+ assert (
+ vid._video_comfy_backends(fam, "b", None, _plan(True), _plan(False))["fp8_storage"] is False
+ )
+
+
+def test_a_smaller_offloaded_price_replans_and_keeps_the_offloaded_backends(monkeypatch):
+ """fp8 stored under offload prices below the widened resident estimate: re-plan at it, and a plan that now fits
+ resident keeps fp8 storage (else the load widens past the size it was planned at)."""
+ fam = types.SimpleNamespace(name = "wan2.2-ti2v-5b")
+ assert vid._video_offload_reprice(fam, 2000, 1000) == (1000, True)
+ assert vid._video_offload_reprice(fam, 1000, 2000) == (2000, False)
+ assert vid._video_offload_reprice(fam, 1000, 1000) == (1000, False)
+ assert vid._video_offload_reprice(fam, None, 1000) == (None, False)
+ h3 = types.SimpleNamespace(name = "minimax-h3")
+ assert vid._video_offload_reprice(h3, 2000, 1000) == (2000, False)
+ assert vid._video_offload_reprice(h3, 1000, 2000) == (2000, False)
+ monkeypatch.setattr(vid, "plan_keeps_transformer_resident", lambda p: p.resident)
+ monkeypatch.setattr(
+ vid, "comfy_int8_backend", lambda t, f, b, *, offload = False: "native" if offload else None
+ )
+ monkeypatch.setattr(vid, "comfy_fp8_backend", lambda t, f, b, *, offload = False: None)
+ resident = vid._video_comfy_backends(fam, "b", None, _plan(True), _plan(True))
+ assert resident["fp8_storage"] is False and resident["int8_backend"] is None
+ kept = vid._video_comfy_backends(fam, "b", None, _plan(True), _plan(True), priced_offload = True)
+ assert kept == {"int8_backend": "native", "fp8_backend": None, "fp8_storage": True}
def test_video_plan_size_prices_only_what_runs_quantized(monkeypatch, tmp_path):
@@ -444,6 +478,12 @@ def test_video_plan_size_prices_only_what_runs_quantized(monkeypatch, tmp_path):
monkeypatch.setattr(vid, "comfy_fp8_backend", lambda *a, **k: None)
assert vid._video_comfy_resident_mib(fam, "b", None, path, scan) == 3
assert vid._video_comfy_resident_mib(fam, "b", None, path, scan, keep = False) == 3
+ # no fp8 GEMM under offload: the codes stay stored as fp8 (widened per layer), priced at their stored size
+ assert vid._video_comfy_resident_mib(fam, "b", None, path, scan, offload = True) == 2
+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "1")
+ assert vid._video_comfy_resident_mib(fam, "b", None, path, scan) == 2
+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "0")
+ assert vid._video_comfy_resident_mib(fam, "b", None, path, scan, offload = True) == 3
assert n == 2**20
@@ -679,6 +719,8 @@ def test_offloaded_plan_size_prices_fp8_that_only_runs_resident(monkeypatch, tmp
vid, "comfy_fp8_backend", lambda *a, offload = False, **k: None if offload else "torchao"
)
assert vid._video_comfy_resident_mib(fam, "b", None, path, scan) == 2
+ assert vid._video_comfy_resident_mib(fam, "b", None, path, scan, offload = True) == 2
+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "0")
assert vid._video_comfy_resident_mib(fam, "b", None, path, scan, offload = True) == 3
@@ -973,3 +1015,96 @@ def test_a_full_width_tag_retry_that_does_not_fit_is_refused_before_converting(
int8_backend = "native",
)
assert len(calls) == 1 # refused before the full-width conversion ran
+
+
+def test_a_resident_video_dit_that_fits_only_stored_gets_fp8_storage(tmp_path, monkeypatch):
+ """A resident ComfyUI fp8 video DiT whose widened form would leave under the host reserve stores fp8, as an
+ image single file does: free RAM between the stored and the widened need loads, and is refused when
+ UNSLOTH_DIFFUSION_FP8_STORAGE=0 makes the load widen."""
+ import core.inference.diffusion_single_file_converters as conv
+ import core.inference.diffusion_small_host as small_host
+
+ torch.manual_seed(4)
+ dense = _H3Like()
+ tensors = {"head.weight": dense.head.weight.detach(), "head.bias": dense.head.bias.detach()}
+ quantized = {
+ "attn.to_q": dense.attn.to_q.weight,
+ "attn.to_k": dense.attn.to_k.weight,
+ "attn.to_v": dense.attn.to_v.weight,
+ "ff.proj": dense.ff.proj.weight,
+ }
+ for name, w in quantized.items():
+ w = w.detach().float()
+ scale = w.abs().max() / torch.finfo(torch.float8_e4m3fn).max
+ tensors[f"{name}.weight"] = (w / scale).to(torch.float8_e4m3fn)
+ tensors[f"{name}.weight_scale"] = scale.reshape(1)
+ tensors[f"{name}.comfy_quant"] = _conf(format = "float8_e4m3fn")
+ path = _save(tmp_path / "wan_fp8.safetensors", tensors)
+ monkeypatch.setattr(
+ cq,
+ "_mapping",
+ lambda cls: (
+ lambda checkpoint = None, **_kw: checkpoint,
+ types.SimpleNamespace(
+ _should_convert_state_dict_to_diffusers = lambda a, b: False,
+ _get_mapping_function_kwargs = lambda fn, **kw: {},
+ ),
+ ),
+ )
+ monkeypatch.setattr(vid, "plan_keeps_transformer_resident", lambda p: p.resident)
+ monkeypatch.setattr(vid, "comfy_int8_backend", lambda *a, **k: None)
+ monkeypatch.setattr(vid, "comfy_fp8_backend", lambda *a, **k: None)
+ monkeypatch.delenv(cq.FP8_STORAGE_ENV, raising = False)
+ # Price in bytes, no reserves: the tiny model's needs are a few hundred KB apart.
+ monkeypatch.setattr(conv, "HOST_RAM_CHECK_MIN_MIB", 0)
+ monkeypatch.delenv(small_host.HOST_RAM_CHECK_ENV, raising = False)
+ monkeypatch.setattr(conv, "_MIB", 1)
+ monkeypatch.setattr(small_host, "host_reserve_mib", lambda total: 0)
+ monkeypatch.setattr(cq, "FP8_STORAGE_HOST_RESERVE_MIB", 0)
+ keys = [n + ".weight" for n in quantized]
+ widened = conv.converted_host_bytes(
+ path, torch.bfloat16, dequantized = dict.fromkeys(keys, 1).get
+ )
+ stored = conv.converted_host_bytes(
+ path, torch.bfloat16, dequantized = dict.fromkeys(keys, 0).get
+ ) + 4 * sum(int(tensors[k].shape[0]) for k in keys)
+ assert 0 < stored < widened
+ available = (stored + widened) // 2
+ monkeypatch.setattr(small_host, "host_ram_mib", lambda: (1 << 40, available))
+ fam = types.SimpleNamespace(name = "wan2.2-ti2v-5b")
+
+ def backends(**kw):
+ return vid._video_comfy_backends(fam, "b", None, _plan(True), _plan(True), **kw)
+
+ def load(**kw):
+ return cq.load_comfy_quant_transformer(
+ _H3Like,
+ path,
+ cq.refuse_comfy_quant(path),
+ {"torch_dtype": torch.bfloat16, "config": "base/repo"},
+ family = fam.name,
+ **kw,
+ )
+
+ picked = backends(widened_mib = widened)
+ assert picked == {"int8_backend": None, "fp8_backend": None, "fp8_storage": True}
+ model = load(**picked)
+ assert model._unsloth_comfy_quant["fp8_stored"] == 4
+ assert model.attn.to_q.weight_q.dtype == torch.uint8
+ # room to widen: the faster widened load; H3 keeps its torchao-or-dequantize rule
+ monkeypatch.setattr(small_host, "host_ram_mib", lambda: (1 << 40, 4 * widened))
+ assert backends(widened_mib = widened)["fp8_storage"] is False
+ monkeypatch.setattr(small_host, "host_ram_mib", lambda: (1 << 40, available))
+ h3 = types.SimpleNamespace(name = "minimax-h3")
+ assert (
+ vid._video_comfy_backends(h3, "b", None, _plan(True), _plan(True), widened_mib = widened)[
+ "fp8_storage"
+ ]
+ is False
+ )
+ # the kill switch widens, and the host RAM check refuses that load
+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "0")
+ widened_pick = backends(widened_mib = widened)
+ assert widened_pick["fp8_storage"] is False
+ with pytest.raises(RuntimeError, match = "Not enough system RAM"):
+ load(**widened_pick)