Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: keep fp8 single-file weights stored in fp8 where widening would offload or crowd host RAM (#14549)
Daniel Han4ff4c6689a21
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex d556059032c..5ec8875f948 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -382,9 +382,14 @@ from .diffusion_comfy_block import (comfy_nvfp4_runtime_possible,)from .diffusion_comfy_quant import (+ header_has_fp8_weights,comfy_fp8_backend,+ comfy_fp8_storage,comfy_int8_backend,comfy_torchao_quantized,+ fp8_storage_wanted,+ fp8_stored_layers,+ pipe_fp8_stored_layers,load_comfy_quant_transformer,refuse_comfy_quant,)@@ -2884,11 +2889,14 @@ def _denoiser_hooked(pipe: Any) -> bool:def _torchao_render_needs_no_grad(state: Any) -> bool:- """inference_mode rejects moving torchao weights between devices, so offloaded quant renders use no_grad."""+ """inference_mode rejects moving torchao weights between devices, so offloaded quant renders use no_grad.+ Stored fp8 codes too: moved under inference_mode they become unversionable, re-copied to host every offload."""if getattr(state, "offload_policy", OFFLOAD_NONE) == OFFLOAD_NONE:return Falsereturn bool(- getattr(state, "transformer_quant", None) or getattr(state, "text_encoder_quant", None)+ getattr(state, "transformer_quant", None)+ or getattr(state, "text_encoder_quant", None)+ or pipe_fp8_stored_layers(getattr(state, "pipe", None)))@@ -3122,6 +3130,28 @@ def _prequant_bakes_auto_accumulate(source: Any) -> bool:return False+def _single_file_fp8_upcast(single_file_path: Any, fam: Any) -> bool:+ """Whether a single-file denoiser stores fp8 weights a widened load upcasts (~2x resident). Read from the header+ (the basename only when it is unreadable); SDXL single-file pipelines are already bf16."""+ if not single_file_path or getattr(fam, "single_file_is_pipeline", False):+ return False+ stored = header_has_fp8_weights(str(single_file_path))+ return "fp8" in Path(single_file_path).name.lower() if stored is None else stored+++def _comfy_keep_fp32_pricing(fam: Any, target: Any) -> dict:+ """``comfy_resident_mib`` kwargs for the DiT's ``_keep_in_fp32_modules``: never stored, fp32 on an fp16 load."""+ import torch++ from .diffusion_comfy_quant import keep_fp32_modules_for++ keep = keep_fp32_modules_for(fam)+ if not keep:+ return {}+ fp16 = getattr(target, "dtype", None) is torch.float16+ return {"keep_fp32": keep, "keep_fp32_bytes": 4 if fp16 else None}++class DiffusionBackend:"""Holds at most one loaded diffusers pipeline. All mutations are serialised."""@@ -8751,7 +8781,7 @@ class DiffusionBackend:_install_gguf_dim_restore(logger)if comfy_scan is not None:# int8 / fp8 codes go to Studio's runtime unchanged where it runs; the rest dequantize.- _comfy_offload = not plan_keeps_transformer_resident(plan)+ _comfy_offload = self._comfy_offload_for(plan, single_file_path)transformer = load_comfy_quant_transformer(transformer_cls,single_file_path,@@ -8781,6 +8811,9 @@ class DiffusionBackend:target = target,fast_accum = transformer_quant_fast_accum,logger = logger,+ fp8_storage = self._fp8_storage_for(+ single_file_path, target, loras+ ),)comfy_compile = comfy_torchao_quantized(transformer)else:@@ -8791,7 +8824,13 @@ class DiffusionBackend:== "Krea2Transformer2DModel"):transformer = load_original_layout_transformer(- transformer_cls, single_file_path, sf_kwargs, logger+ transformer_cls,+ single_file_path,+ sf_kwargs,+ logger,+ fp8_storage = self._fp8_storage_for(+ single_file_path, target, loras+ ),)else:# A safetensors single-file (fp8) carries its own dtype: no GGUF dequant config.@@ -10079,6 +10118,18 @@ class DiffusionBackend:except Exception: # noqa: BLE001 - an install hint only: the load decides on its ownreturn False+ def _comfy_offload_for(self, plan: Any, single_file_path: Any) -> bool:+ """Probe the ComfyUI backends as offloaded: the plan offloads, or it was priced with the offloaded backends."""+ priced = getattr(self, "_comfy_priced_offload", None) or {}+ return not plan_keeps_transformer_resident(plan) or bool(priced.get(str(single_file_path)))++ def _fp8_storage_for(self, single_file_path: Any, target: Any, loras: Any) -> bool:+ """The planner's ``fp8_storage_wanted`` pick for this file, re-checked against device / LoRA / env."""+ picks = getattr(self, "_fp8_storage_picks", None) or {}+ return bool(picks.get(str(single_file_path))) and comfy_fp8_storage(+ target, lora = _has_active_lora(loras)+ )+@staticmethoddef _comfy_single_file_resident_mib(single_file_path: Optional[str],@@ -10087,8 +10138,11 @@ class DiffusionBackend:base: Optional[str],*,lora: bool = False,+ fp8_storage: bool = False,+ offload: bool = False,) -> Optional[int]:- """``comfy_resident_mib`` for a ComfyUI-quantized single file under a resident plan, else None."""+ """``comfy_resident_mib`` for a ComfyUI-quantized single file, else None. ``offload`` prices the backends an+ offloaded load keeps (the loader probes them the same way)."""try:from .diffusion_comfy_quant import comfy_resident_mib, scan_comfy_quantfrom .diffusion_transformer_quant import (@@ -10106,8 +10160,10 @@ class DiffusionBackend:return comfy_resident_mib(single_file_path,scan,- keep_int8 = comfy_int8_backend(target, name, base) is not None,- keep_fp8 = comfy_fp8_backend(target, name, base) is not None,+ keep_int8 = comfy_int8_backend(target, name, base, offload = offload) is not None,+ keep_fp8 = comfy_fp8_backend(target, name, base, offload = offload) is not None,+ fp8_stored = fp8_storage and comfy_fp8_storage(target, lora = lora),+ **_comfy_keep_fp32_pricing(fam, target),# the loader's runtime filter: layers it skips are priced dequantizedmin_features = DEFAULT_MIN_LINEAR_FEATURES,fp8_divisible = divisible_for_scheme(TQ_FP8),@@ -11194,6 +11250,11 @@ class DiffusionBackend:pre-cast checkpoint's size, which base-repo scans never see."""# Settled (max-over-reads) on cuda: a transient foreign allocation would make an empty card look full+ single_file_fp8_stored_mib: Optional[int] = None+ single_file_resident_at: Any = (+ None # (fp8_storage, offload) -> MiB, set for a single-file DiT+ )+ single_file_stored_at: Any = Nonedevice_memory = (device_memory_overrideif device_memory_override is not None@@ -11312,21 +11373,46 @@ class DiffusionBackend:# Planning the dense-quant candidate: the auto-policy estimate replaces the file-size derivation.transformer_resident = transformer_resident_override_mibelif kind == "single_file":- # An fp8 checkpoint upcasts to bf16 on load (~2x resident); detect from the basename. SDXL single-file- # is already bf16.- fp8_upcast = not getattr(fam, "single_file_is_pipeline", False) and (- "fp8" in Path(single_file_path).name.lower() if single_file_path else False- )- transformer_resident = estimate_safetensors_dense_mib(- file_size_mib(single_file_path), fp8_upcast = fp8_upcast- )- if not getattr(fam, "single_file_is_pipeline", False):- # Priced from the header: layers a resident runtime keeps at stored size, dequantized ones at 2x.- _comfy_mib = self._comfy_single_file_resident_mib(- single_file_path, fam, target, base, lora = lora++ def _single_file_resident(+ fp8_storage: bool, offload: bool = False+ ) -> Optional[int]:+ fp8_upcast = _single_file_fp8_upcast(single_file_path, fam)+ if fp8_upcast and fp8_storage and fam.name == KREA2_FAMILY_NAME:+ # storage keeps only OCP fp8 codes; FNUZ ones still widen+ fp8_upcast = bool(+ header_has_fp8_weights(str(single_file_path), fnuz_only = True)+ )+ mib = estimate_safetensors_dense_mib(+ file_size_mib(single_file_path), fp8_upcast = fp8_upcast)- if _comfy_mib is not None:- transformer_resident = _comfy_mib+ if not getattr(fam, "single_file_is_pipeline", False):+ # Priced from the header: layers a runtime keeps at stored size, dequantized ones at 2x.+ _comfy_mib = self._comfy_single_file_resident_mib(+ single_file_path,+ fam,+ target,+ base,+ lora = lora,+ fp8_storage = fp8_storage,+ offload = offload,+ )+ if _comfy_mib is not None:+ mib = _comfy_mib+ return mib++ def _stored_mib(widened: Optional[int], offload: bool = False) -> Optional[int]:+ if not (comfy_fp8_storage(target, lora = lora) and single_file_path):+ return None+ stored = _single_file_resident(True, offload)+ if stored is not None and widened is not None and stored < widened:+ return stored+ return None++ transformer_resident = _single_file_resident(False)+ single_file_fp8_stored_mib = _stored_mib(transformer_resident)+ single_file_resident_at = _single_file_resident+ single_file_stored_at = _stored_mibelif getattr(fam, "single_file_is_pipeline", False):# Only the denoiser's linears stay packed; the text encoders, VAE and the rest load dense.transformer_resident = whole_pipeline_gguf_resident_mib(@@ -11424,17 +11510,62 @@ class DiffusionBackend:# the 1024x1024 default for PLACEMENT. generate() re-checks the real resolution against the free budget before# it samples (raise_on_image_activation_shortfall).runtime_headroom = estimate_image_runtime_mib(width = None, height = None, family = variant_hint)- return plan_diffusion_memory(- target = target,- device_memory = device_memory,- model_dense_mib = model_dense_mib,- companion_dense_mib = companion_mib,- text_encoder_dense_mib = text_encoder_mib,- runtime_headroom_mib = runtime_headroom,- requested_mode = memory_mode,- explicit_offload = cpu_offload,- calibrated_activation = _calibrated_activation(fam, target),- )++ def _plan(model_mib: Optional[int]) -> Any:+ return plan_diffusion_memory(+ target = target,+ device_memory = device_memory,+ model_dense_mib = model_mib,+ companion_dense_mib = companion_mib,+ text_encoder_dense_mib = text_encoder_mib,+ runtime_headroom_mib = runtime_headroom,+ requested_mode = memory_mode,+ explicit_offload = cpu_offload,+ calibrated_activation = _calibrated_activation(fam, target),+ )++ plan = _plan(model_dense_mib)+ if kind == "single_file" and single_file_path:+ self.__dict__.setdefault("_comfy_priced_offload", {})[str(single_file_path)] = False+ if (+ single_file_resident_at is not None+ and model_dense_mib is not None+ and transformer_resident is not None+ and not plan_keeps_transformer_resident(plan)+ ):+ # Offloaded: the loader probes the backends with offload=True (torchao fp8 does not survive the hooks), so+ # price what that load builds, widened and fp8-stored, before choosing storage.+ offloaded = single_file_resident_at(False, True)+ if offloaded is not None and offloaded != transformer_resident:+ # Smaller (the int8 / fp8 twin only offload runs): the load keeps those backends even if the new plan+ # fits resident, else it dequantizes past the size it was planned at.+ self.__dict__.setdefault("_comfy_priced_offload", {})[str(single_file_path)] = int(+ offloaded+ ) < int(transformer_resident)+ model_dense_mib = model_dense_mib - int(transformer_resident) + int(offloaded)+ transformer_resident = offloaded+ single_file_fp8_stored_mib = single_file_stored_at(offloaded, True)+ plan = _plan(model_dense_mib)+ if single_file_fp8_stored_mib is not None and model_dense_mib is not None:+ # Re-plan at the fp8-stored size: a widened load that had to offload may now stay resident.+ from .diffusion_memory import _available_system_memory_mib++ store = fp8_storage_wanted(+ target,+ lora = lora,+ offload = not plan_keeps_transformer_resident(plan),+ widened_mib = transformer_resident,+ host_available_mib = _available_system_memory_mib(),+ )+ picks = self.__dict__.setdefault("_fp8_storage_picks", {})+ picks[str(single_file_path)] = store+ if store:+ plan = _plan(+ model_dense_mib - int(transformer_resident) + single_file_fp8_stored_mib+ )+ elif kind == "single_file" and single_file_path:+ self.__dict__.setdefault("_fp8_storage_picks", {})[str(single_file_path)] = False+ return plan@staticmethoddef _from_pipe_no_recast(pipe: Any, pipe_cls: Any, **extra: Any) -> Any:@@ -11732,6 +11863,12 @@ class DiffusionBackend:"quantized Linears, which cannot carry adapters. Load the file with the LoRA selected (Unsloth then ""dequantizes those layers), or set UNSLOTH_DIFFUSION_COMFY_NVFP4=0 / UNSLOTH_DIFFUSION_COMFY_MXFP8=0.")+ if fp8_stored_layers(getattr(pipe, "transformer", None)):+ raise ValueError(+ "LoRA is not available on this load: the file's fp8 layers are stored as fp8 and widened per layer "+ "(this GPU has no fp8 GEMM), which cannot carry adapters. Load the file with the LoRA selected "+ "(Unsloth then widens those layers at load), or set UNSLOTH_DIFFUSION_FP8_STORAGE=0."+ )if not diffusion_lora.supports_lora(engine = "diffusers",family = getattr(state.family, "name", None),@@ -13449,8 +13586,10 @@ class DiffusionBackend:transformer_quant = state.transformer_quant,)compiled = "compiled" in (getattr(state, "speed_optims", ()) or ())- lora_load_ok = not _small_host_int8(state.pipe) and not comfy_block_runtime_layers(- getattr(state.pipe, "transformer", None)+ lora_load_ok = (+ not _small_host_int8(state.pipe)+ and not comfy_block_runtime_layers(getattr(state.pipe, "transformer", None))+ and not fp8_stored_layers(getattr(state.pipe, "transformer", None)))return {"loaded": True,diff --git a/studio/backend/core/inference/diffusion_comfy_quant.py b/studio/backend/core/inference/diffusion_comfy_quant.pyindex d03335e98fe..9e20379479b 100644--- a/studio/backend/core/inference/diffusion_comfy_quant.py+++ b/studio/backend/core/inference/diffusion_comfy_quant.py@@ -36,6 +36,10 @@ QUANT_METADATA_KEY = "_quantization_metadata"LEGACY_SCALED_FP8_KEY = "scaled_fp8"COMFY_INT8_ENV = "UNSLOTH_DIFFUSION_COMFY_INT8"COMFY_FP8_ENV = "UNSLOTH_DIFFUSION_COMFY_FP8"+# Store fp8 layers no fp8 GEMM takes as fp8, widened per layer (ComfyUI manual_cast). Unset: only when widening+# would offload or crowd host RAM (per-layer widen costs eager steps); ``1``: always; ``0``: never.+FP8_STORAGE_ENV = "UNSLOTH_DIFFUSION_FP8_STORAGE"+FP8_STORAGE_HOST_RESERVE_MIB = 4096# Header metadata our own converter writes (base model, scheme); ComfyUI ignores it.CONVERSION_RECORD_KEY = "unsloth_comfy_conversion"@@ -133,6 +137,26 @@ def _layer_conf(raw: Any) -> tuple[Optional[str], bool, int]:return (fmt if isinstance(fmt, str) else None), convrot, group+# ROCm-style float8 tags: every loader widens them (fp8 storage keeps only the OCP formats above).+FP8_FNUZ_TAGS = ("F8_E4M3FNUZ", "F8_E5M2FNUZ")+++def header_has_fp8_weights(path: Optional[str], *, fnuz_only: bool = False) -> Optional[bool]:+ """Whether a safetensors file stores any fp8 ``.weight`` (plain, scaled or FNUZ; ``fnuz_only``: FNUZ only), or+ None if its header is unreadable."""+ if not path or not os.path.isfile(path):+ return None+ try:+ header, _base = _read_header(str(path))+ except Exception: # noqa: BLE001+ return None+ tags = FP8_FNUZ_TAGS if fnuz_only else (*FP8_FORMATS.values(), *FP8_FNUZ_TAGS)+ return any(+ key.endswith(".weight") and isinstance(entry, dict) and entry.get("dtype") in tags+ for key, entry in header.items()+ )++def scan_comfy_quant(path: Optional[str]) -> Optional[ComfyQuantScan]:"""The ComfyUI quantization a safetensors single file declares, or None for a plain checkpoint.@@ -269,12 +293,46 @@ _DTYPE_BYTES = {}+# Original-layout names of modules a DiT keeps in fp32 (Wan's ``time_embedding.N`` loads as ``time_embedder``).+_KEEP_FP32_ORIGINAL_ALIASES = {"time_embedder": ("time_embedding",)}+++def keeps_fp32(name: str, keep_fp32: Any = ()) -> bool:+ """Whether header key ``name`` names a ``_keep_in_fp32_modules`` module, under its diffusers or original name."""+ keep = tuple(keep_fp32 or ())+ tokens = str(name).split(".")+ return any(t in tokens for m in keep for t in (m, *_KEEP_FP32_ORIGINAL_ALIASES.get(m, ())))+++def keep_fp32_modules_for(fam: Any) -> tuple:+ """The family transformer's ``_keep_in_fp32_modules`` (the loader refuses fp8 storage for them), else ()."""+ try:+ import diffusers+ cls = getattr(diffusers, getattr(fam, "transformer_class", None) or "", None)+ keep = getattr(cls, "_keep_in_fp32_modules", None) or ()+ except Exception: # noqa: BLE001 -- unknown class: nothing excluded+ return ()+ return (keep,) if isinstance(keep, str) else tuple(keep)+++def fp8_stored_by_header(+ fmt: str,+ shape: Any,+ name: str = "",+ keep_fp32: Any = (),+) -> bool:+ """Header rule for fp8 layers kept as codes, shared by the host RAM estimate and the planner; the loader can only+ widen more (charged live)."""+ return fmt in FP8_FORMATS and len(shape or ()) == 2 and not keeps_fp32(name, keep_fp32)++def comfy_resident_mib(path: Optional[str],scan: Optional[ComfyQuantScan] = None,*,keep_int8: bool,keep_fp8: bool,+ fp8_stored: bool = False,keep_nvfp4: bool = False,keep_mxfp8: bool = False,compute_bytes: int = 2,@@ -284,6 +342,8 @@ def comfy_resident_mib(fp8_divisible: int = 0,block_divisible: Optional[dict] = None,key_map: Any = None,+ keep_fp32: Any = (),+ keep_fp32_bytes: Optional[int] = None,) -> Optional[int]:"""What the loader leaves resident for a ComfyUI-quantized file, priced from its header: a quantizedweight a runtime keeps costs its stored bytes, one that is dequantized costs ``numel * compute_bytes``@@ -293,7 +353,10 @@ def comfy_resident_mib(an int8 layer whose name holds one of ``exclude_tokens`` is priced dequantized, as Studio's int8 filterleaves it, and so is a layer the runtime filter skips (in / out features under ``min_features``, or fp8 featuresnot multiples of ``fp8_divisible``). ``key_map`` names layers as the loader's filter sees them (an original-layout- file). None when the header cannot be read. Torch-free."""+ file). ``keep_fp32``: the transformer's ``_keep_in_fp32_modules``; those layers are never fp8-stored and, widened,+ cost ``keep_fp32_bytes`` per element (4 on an fp16 load, default ``compute_bytes``). None when the header cannot+ be read. Torch-free."""+ fp32_bytes = compute_bytes if keep_fp32_bytes is None else int(keep_fp32_bytes)try:scan = scan if scan is not None else scan_comfy_quant(path)if scan is None:@@ -311,14 +374,22 @@ def comfy_resident_mib(return Falsereturn not divisible or not (out_f % divisible or in_f % divisible)- def _excluded(key: str, shape: Any) -> bool:+ def _names(key: str, shape: Any) -> list:names = [key]if key_map is not None:try:names = [k for k, _rows in key_map(key, tuple(shape or ()))] or namesexcept ValueError:pass- return any(t in n for n in names for t in exclude_tokens)+ return names++ def _excluded(key: str, shape: Any) -> bool:+ return any(t in n for n in _names(key, shape) for t in exclude_tokens)++ def _bytes(key: str, shape: Any) -> int:+ if keep_fp32 and any(keeps_fp32(n, keep_fp32) for n in _names(key, shape)):+ return fp32_bytes+ return compute_bytesdef _logical(fmt: str, shape: Any) -> Any:if fmt == NVFP4 and len(shape or ()) == 2:@@ -335,6 +406,13 @@ def comfy_resident_mib(and _fits(shape, 0)) or ((layer.format == FP8_E4M3 and keep_fp8 and _fits(shape, fp8_divisible))+ or (+ fp8_stored+ and all(+ fp8_stored_by_header(layer.format, shape, n, keep_fp32)+ for n in _names(name + ".weight", shape)+ )+ )or ({NVFP4: keep_nvfp4, MXFP8: keep_mxfp8}.get(layer.format, False)# input smoothing has no runtime Linear: the loader dequantizes such a layer@@ -356,11 +434,13 @@ def comfy_resident_mib(dtype = str(entry.get("dtype", ""))stored = numel * _DTYPE_BYTES.get(dtype, 2)if key in kept:- total += stored if kept[key] else numel * compute_bytes * (2 if key in packed else 1)+ total += (+ stored if kept[key] else numel * _bytes(key, shape) * (2 if key in packed else 1)+ )elif key.endswith(COMFY_QUANT_SUFFIX) or any(key.endswith(s) for s in _SCALE_SUFFIXES):total += storedelif dtype.startswith(("F", "BF")):- total += numel * compute_bytes+ total += numel * _bytes(key, shape)else:total += storedreturn -(-total // (1024 * 1024)) if total > 0 else None@@ -829,7 +909,7 @@ def comfy_fp8_backend(quant runs natively (ROCm, the stubbed torchao), None (dequantize to bf16) where neither runs: anolder GPU, CPU, MPS, or an offloaded plan on the torchao path (the model offload hooks cannot move a``Float8Tensor``: "Attempted to set the storage of a tensor on device cuda:0 to a storage on ... cpu").- ``UNSLOTH_DIFFUSION_COMFY_FP8=0`` always dequantizes."""+ ``UNSLOTH_DIFFUSION_COMFY_FP8=0`` never takes an fp8 GEMM."""if (os.environ.get(COMFY_FP8_ENV) or "").strip().lower() in ("0", "off", "false", "no"):return Nonetry:@@ -855,6 +935,64 @@ def comfy_fp8_backend(return None+FP8_STORED_ATTR = "_unsloth_fp8_stored"+++def fp8_stored_layers(model: Any) -> int:+ """Count of fp8-stored Linears (they cannot carry a LoRA)."""+ try:+ return int(getattr(model, FP8_STORED_ATTR, 0) or 0)+ except (TypeError, ValueError):+ return 0+++def pipe_fp8_stored_layers(pipe: Any) -> int:+ return sum(+ fp8_stored_layers(getattr(pipe, attr, None)) for attr in ("transformer", "transformer_2")+ )+++def fp8_storage_disabled() -> bool:+ return (os.environ.get(FP8_STORAGE_ENV) or "").strip().lower() in ("0", "off", "false", "no")+++def fp8_storage_forced() -> bool:+ return (os.environ.get(FP8_STORAGE_ENV) or "").strip().lower() in ("1", "on", "true", "yes")+++def fp8_storage_wanted(+ target: Any = None,+ *,+ lora: bool = False,+ offload: bool = False,+ widened_mib: Optional[int] = None,+ host_available_mib: Optional[int] = None,+) -> bool:+ """Store fp8 only when widening would offload or leave under the host reserve: widening is otherwise faster+ (measured +45% s/step eager for storage on Krea 2 / B200, parity compiled)."""+ if not comfy_fp8_storage(target, lora = lora):+ return False+ if fp8_storage_forced() or offload:+ return True+ if widened_mib is None or host_available_mib is None:+ return False+ return int(widened_mib) + FP8_STORAGE_HOST_RESERVE_MIB > int(host_available_mib)+++def comfy_fp8_storage(target: Any = None, *, lora: bool = False) -> bool:+ """Whether fp8 storage can run: same weights as the load-time dequant, half the bytes. Off for a LoRA (needs+ dense Linears), MPS and torch without float8."""+ if lora or fp8_storage_disabled():+ return False+ if str(getattr(target, "device", "") or "").split(":")[0] == "mps":+ return False+ try:+ import torch+ return hasattr(torch, "float8_e4m3fn") and hasattr(torch, "float8_e5m2")+ except Exception: # noqa: BLE001 -- no torch: nothing to store+ return False++def comfy_conversion_record(path: Optional[str]) -> dict:"""The ``unsloth_comfy_conversion`` header record our converter writes (base model, scheme, family),or {} for any other file. Torch-free, never raises."""@@ -979,6 +1117,7 @@ def load_comfy_quant_transformer(key_map: Any = None,prepare_model: Any = None,keep_dtype: Any = None,+ fp8_storage: Optional[bool] = None,) -> Any:"""Build ``transformer_cls`` from the ComfyUI-quantized ``path``.@@ -991,7 +1130,8 @@ def load_comfy_quant_transformer(as per-row ``Float8Tensor`` weights or native fp8 twins; with an ``nvfp4_backend`` / ``mxfp8_backend``(``diffusion_comfy_block.comfy_block_backend``) the nvfp4 / mxfp8 layers the matching Studio filter selectskeep their codes on Studio's FlashInfer NVFP4 Linear / the ``torch._scaled_mm`` MXFP8 Linear. Everything- else is dequantized to the compute dtype. ``finalize`` applies the small-M padding here (the single-file path); the hosted+ else is dequantized to the compute dtype, except fp8 Linears under ``fp8_storage`` (None = env opt-in+ only), which widen per layer in the forward. ``finalize`` applies the small-M padding here (the single-file path); the hostedprequant loader passes False and pads after placement, as for its own checkpoints.``keep_key(key)`` limits the read to the DiT's keys of a file that bundles other components;@@ -1037,6 +1177,8 @@ def load_comfy_quant_transformer(# Studio's fp8 runtime asserts bf16 weights; an fp16 / fp32 pipeline dequantizes them instead.fp8_backend = Nonemxfp8_backend = None+ if fp8_storage is None:+ fp8_storage = fp8_storage_forced() and comfy_fp8_storage(target)if keep_key is None:state = load_file(str(path))else:@@ -1086,15 +1228,38 @@ def load_comfy_quant_transformer(# Mirrors the source loop below: a layer is kept as codes only with a runtime for its format (fp16_keep# merely defers the dtype choice; the layer is still expanded) and, for a block format, no key map.# nvfp4 headers describe packed uint8 (K / 2 columns); the dense weight is N x K.+ header_shapes = {}+ if fp8_storage:+ try:+ _hdr, _ = _read_header(str(path))+ header_shapes = {+ k: (v or {}).get("shape") for k, v in _hdr.items() if k != "__metadata__"+ }+ except Exception: # noqa: BLE001 -- the estimate then prices them widened (conservative)+ header_shapes = {}++ def _stored(layer: Any) -> bool:+ shape = header_shapes.get(layer.name + ".weight")+ return (+ bool(fp8_storage)+ and shape is not None+ and fp8_stored_by_header(layer.format, shape, layer.name, keep_fp32)+ )+_widened = {layer.name + ".weight": ((2 if layer.format == NVFP4 else 1)- if not backends.get(layer.format)+ if not (backends.get(layer.format) or _stored(layer))or (layer.format in BLOCK_SIZES and key_map is not None)else 0)for layer in scan.layers.values()}+ _stored_scale_bytes = sum(+ 4 * int(header_shapes[layer.name + ".weight"][0])+ for layer in scan.layers.values()+ if _stored(layer) and not backends.get(layer.format)+ )# The header estimate refuses early; the budget it activates is charged by every allocating helper# below, so a layer the estimate cannot see (runtime filters, nibble swaps) still cannot OOM the host.budget = HostRamBudget(@@ -1108,6 +1273,7 @@ def load_comfy_quant_transformer(dequantized = _widened.get,keep_key = keep_key,)+ + _stored_scale_bytes)budget.activate()sources: list = [] # (layer, codes, scale, block extras or None) kept for a runtime@@ -1133,9 +1299,9 @@ def load_comfy_quant_transformer(elif layer.format in FP8_FORMATS and codes.dtype == torch.uint8:codes = codes.view(getattr(torch, layer.format))# A key-mapped (original-layout) family moves rows whole, which the tiled block scales do not survive.- if (backends.get(layer.format) or fp16_keep) and not (- extra is not None and key_map is not None- ):+ if (+ backends.get(layer.format) or fp16_keep or (fp8_storage and layer.format in FP8_FORMATS)+ ) and not (extra is not None and key_map is not None):sources.append((layer, codes, scale, extra))state[layer.name + ".weight"] = None # placeholder, tagged belowelif extra is not None:@@ -1278,6 +1444,7 @@ def load_comfy_quant_transformer(modules = dict(model.named_modules())rotations: dict = {}native: dict = {}+ stored: dict = {} # fqn -> (codes, scale)block_native: dict = {} # fqn -> (fmt, codes, scale, tensor_scale, input_scale, bias)built = {TQ_INT8: 0, TQ_FP8: 0, TQ_NVFP4: 0, TQ_MXFP8: 0}for name, value in list(converted.items()):@@ -1331,6 +1498,20 @@ def load_comfy_quant_transformer(weight = _int8_tensor(name, codes, scale, dtype)else:weight = _fp8_tensor(name, codes, scale, dtype, fast_accum)+ if (+ weight is None+ and fp8_storage+ and fmt in FP8_FORMATS+ and not group+ and isinstance(module, torch.nn.Linear)+ and name.endswith(".weight")+ and _dtype_for(name) == dtype+ and not any(m in fqn.split(".") for m in keep_fp32)+ ):+ # forward widen must match ``_dequant``+ stored[fqn] = (codes, scale.reshape(-1, 1).expand(codes.shape[0], 1))+ del converted[name]+ continueif weight is None:converted[name] = _dequant(codes, scale, group, _dtype_for(name))dequantized += 1@@ -1363,6 +1544,16 @@ def load_comfy_quant_transformer(setattr(parent, leaf, layer)converted[fqn + ".weight_q"] = layer.weight_qconverted[fqn + ".weight_scale"] = layer.weight_scale+ for fqn, (codes, scale) in stored.items():+ parent_name, _, leaf = fqn.rpartition(".")+ parent = model.get_submodule(parent_name) if parent_name else model+ charge_host_ram(+ 4 * int(codes.shape[0])+ ) # the fp32 row scale; the codes stay as read (a view)+ layer = cls(getattr(parent, leaf).to(dtype), TQ_FP8, codes = codes, scale = scale)+ setattr(parent, leaf, layer)+ converted[fqn + ".weight_q"] = layer.weight_q+ converted[fqn + ".weight_scale"] = layer.weight_scalefor fqn, (fmt, codes, scale, tensor_scale, input_scale, bias) in block_native.items():parent_name, _, leaf = fqn.rpartition(".")parent = model.get_submodule(parent_name) if parent_name else model@@ -1462,7 +1653,8 @@ def load_comfy_quant_transformer(logger.info("diffusion.comfy_quant: %s loaded from a ComfyUI checkpoint (%s): %d int8 layers kept as ""int8 (%s, %d ConvRot), %d fp8 layers kept as fp8 (%s), %d nvfp4 layers kept as nvfp4 (%s), "- "%d mxfp8 layers kept as mxfp8 (%s), %d layers dequantized to %s",+ "%d mxfp8 layers kept as mxfp8 (%s), %d fp8 layers stored as fp8 and widened per layer to %s, "+ "%d layers dequantized to %s",transformer_cls.__name__,", ".join(f"{k} x{v}" for k, v in sorted(scan.counts().items())),built[TQ_INT8],@@ -1474,6 +1666,8 @@ def load_comfy_quant_transformer(f"{nvfp4_backend} runtime" if built[TQ_NVFP4] else "no nvfp4 runtime",built[TQ_MXFP8],f"{mxfp8_backend} runtime" if built[TQ_MXFP8] else "no mxfp8 runtime",+ len(stored),+ str(dtype).replace("torch.", ""),dequantized,str(dtype).replace("torch.", ""),)@@ -1486,6 +1680,9 @@ def load_comfy_quant_transformer("fp8": built[TQ_FP8],"dequantized": dequantized,}+ if stored:+ info["fp8_stored"] = len(stored)+ setattr(model, FP8_STORED_ATTR, len(stored))for fmt, backend in ((NVFP4, nvfp4_backend), (MXFP8, mxfp8_backend)):if scan.counts().get(fmt):info[f"{fmt}_backend"] = backend if built[schemes[fmt]] else Nonediff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.pyindex 5298b404b1c..1f12fab98b9 100644--- a/studio/backend/core/inference/diffusion_compile_cache.py+++ b/studio/backend/core/inference/diffusion_compile_cache.py@@ -402,6 +402,10 @@ def _weight_signature(module: Any, weight: Any) -> str:parts.append(f"rot{rot}")if getattr(module, "act_int8", False):parts.append("act_int8")+ fmt = getattr(module, "fp8_format", None)+ if isinstance(fmt, str) and fmt != "float8_e4m3fn":+ # Stored E5M2 codes decode through a different integer graph; E4M3 keeps the key it always had.+ parts.append(fmt)fwd = module.__dict__.get("forward") if hasattr(module, "__dict__") else Noneif fwd is not None:func = getattr(fwd, "__func__", fwd)diff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.pyindex b1e656485c4..6708c6919b0 100644--- a/studio/backend/core/inference/diffusion_native_quant.py+++ b/studio/backend/core/inference/diffusion_native_quant.py@@ -31,6 +31,31 @@ _INT8PACK_PROBED: dict = {}# torch._int_mm needs M > 16 and K, N multiples of 8._INT_MM_MIN_ROWS = 17_INT_MM_CHUNK_ELEMS = 1 << 25+# fp8 codes a native fp8 layer may hold; e5m2 only from a checkpoint (the quantise path writes e4m3fn).+NATIVE_FP8_FORMATS = ("float8_e4m3fn", "float8_e5m2")+++def fp8_bits_to_float32(bits: Any, fmt: str = "float8_e4m3fn") -> Any:+ """uint8 fp8 codes to float32 by integer ops, bit-exact; Triton has no e4m3fn convert below sm_89."""+ import torch++ b = bits.to(torch.int32)+ if fmt == "float8_e5m2":+ exp_bits, man_bits, bias = 5, 2, 15+ else:+ exp_bits, man_bits, bias = 4, 3, 7+ e = (b >> man_bits) & ((1 << exp_bits) - 1)+ m = b & ((1 << man_bits) - 1)+ normal = (((e + (127 - bias)) << 23) | (m << (23 - man_bits))).view(torch.float32)+ # subnormal: m * 2**(1 - bias - man_bits), exact in float32+ sub = m.to(torch.float32) * (2.0 ** (1 - bias - man_bits))+ mag = torch.where(e == 0, sub, normal)+ if fmt == "float8_e5m2":+ top = e == (1 << exp_bits) - 1+ mag = torch.where(top, torch.where(m == 0, float("inf"), float("nan")), mag)+ else:+ mag = torch.where((e == 15) & (m == 7), float("nan"), mag)+ return torch.where((b & 0x80) != 0, -mag, mag)def int8_act_requested() -> bool:@@ -259,8 +284,15 @@ def native_linear_class():if self.act_int8 and rot_group and self.in_features % int(rot_group) == 0else 0)+ self.fp8_format = "float8_e4m3fn"if codes is not None:wanted = torch.int8 if scheme == NATIVE_INT8 else torch.float8_e4m3fn+ if (+ scheme == NATIVE_FP8+ and str(codes.dtype).replace("torch.", "") in NATIVE_FP8_FORMATS+ ):+ wanted = codes.dtype+ self.fp8_format = str(codes.dtype).replace("torch.", "")if codes.dtype != wanted:raise ValueError(f"pre-quantized native {scheme} codes must be {str(wanted).replace('torch.', '')}"@@ -313,13 +345,14 @@ def native_linear_class():self._int8pack_scales: dict = {}def _stored_weight(self, dtype: Any) -> Any:- wq = (- self.weight_q.view(torch.float8_e4m3fn)- if self.scheme == NATIVE_FP8- else self.weight_q- )scale = self.weight_scale.view(torch.float32)- return (wq.to(torch.float32) * scale[:, None]).to(dtype)+ if self.scheme != NATIVE_FP8:+ wide = self.weight_q.to(torch.float32)+ elif torch.compiler.is_compiling():+ wide = fp8_bits_to_float32(self.weight_q, self.fp8_format)+ else:+ wide = self.weight_q.view(getattr(torch, self.fp8_format)).to(torch.float32)+ return (wide * scale[:, None]).to(dtype)def _rotate(self, x: Any) -> Any:g = self.rot_group@@ -465,6 +498,7 @@ def native_linear_class():return (f"in_features={self.in_features}, out_features={self.out_features}, "f"scheme={self.scheme}, act_int8={self.act_int8}, rot_group={self.rot_group}, bias={self.bias is not None}"+ + (f", codes={self.fp8_format}" if self.scheme == NATIVE_FP8 else ""))return NativeWeightOnlyLineardiff --git a/studio/backend/core/inference/diffusion_single_file_converters.py b/studio/backend/core/inference/diffusion_single_file_converters.pyindex 6fd86c1b1ab..84521321467 100644--- a/studio/backend/core/inference/diffusion_single_file_converters.py+++ b/studio/backend/core/inference/diffusion_single_file_converters.py@@ -411,7 +411,12 @@ def converted_host_bytes(numel *= int(dim)except Exception: # noqa: BLE001continue- verdict = dequantized(key) if dequantized is not None else None+ if dequantized is None:+ verdict = None+ elif getattr(dequantized, "wants_meta", False):+ verdict = dequantized(key, meta)+ else:+ verdict = dequantized(key)if verdict is not None:total += numel * int(verdict) * _ST_ITEMSIZE[target]continue@@ -574,10 +579,13 @@ def load_original_layout_transformer(path: str,sf_kwargs: dict,logger: Any = None,+ *,+ fp8_storage: bool = False,) -> Any:"""Load a plain original-layout safetensors file for a class without ``from_single_file``(Krea 2 on diffusers 0.40). ``sf_kwargs`` = the ``from_single_file`` kwargs. Cast like- ``from_pretrained`` (``_keep_in_fp32_modules`` stay fp32); strict on keys and shapes."""+ ``from_pretrained`` (``_keep_in_fp32_modules`` stay fp32); strict on keys and shapes. ``fp8_storage``+ keeps fp8 Linears as fp8 with unit scales."""import torchfrom diffusers.loaders import single_file_model as sfm@@ -606,12 +614,67 @@ def load_original_layout_transformer(budget = HostRamBudget(path, f"widening its weights to {str(dtype).replace('torch.', '')}", logger)- budget.check(converted_host_bytes(path, dtype, keep_fp32 = keep))+ budget.check(+ converted_host_bytes(+ path,+ dtype,+ keep_fp32 = keep,+ dequantized = fp8_stored_estimate(keep) if fp8_storage else None,+ )+ + (_fp8_stored_scale_bytes(path, keep) if fp8_storage else 0)+ )with budget:- return _load_original_layout(transformer_cls, path, config, mapping_fn, keep, dtype, logger)+ return _load_original_layout(+ transformer_cls, path, config, mapping_fn, keep, dtype, logger, fp8_storage+ )+++def fp8_stored_estimate(keep: Any = ()) -> Any:+ """``dequantized`` for fp8 storage: 2-D fp8 ``.weight`` outside ``keep`` stays mmapped (0); later widens are+ charged live."""+ keep = tuple(keep or ())++ def verdict(key: str, meta: Optional[dict] = None) -> Any:+ meta = meta or {}+ if (+ str(meta.get("dtype")) in ("F8_E4M3", "F8_E5M2")+ and len(meta.get("shape") or ()) == 2+ and key.endswith(".weight")+ and not any(m in key.split(".") for m in keep)+ ):+ return 0+ return None++ verdict.wants_meta = True+ return verdict+++def _fp8_stored_scale_bytes(path: str, keep: Any = ()) -> int:+ from .diffusion_comfy_quant import _read_header++ try:+ header, _base = _read_header(str(path))+ except Exception: # noqa: BLE001+ return 0+ header.pop("__metadata__", None)+ verdict = fp8_stored_estimate(keep)+ return sum(+ 4 * int((meta.get("shape") or (0,))[0])+ for key, meta in header.items()+ if verdict(key, meta) == 0+ )-def _load_original_layout(transformer_cls, path, config, mapping_fn, keep, dtype, logger) -> Any:+def _load_original_layout(+ transformer_cls,+ path,+ config,+ mapping_fn,+ keep,+ dtype,+ logger,+ fp8_storage = False,+) -> Any:import torchfrom accelerate import init_empty_weightsfrom safetensors.torch import load_file@@ -631,7 +694,10 @@ def _load_original_layout(transformer_cls, path, config, mapping_fn, keep, dtypef"{name} does not match {transformer_cls.__name__}: {len(missing)} missing "f"(e.g. {missing[:1]}), {len(unexpected)} unused (e.g. {unexpected[:1]})")- for key, value in converted.items():+ fp8_dtypes = {getattr(torch, n) for n in ("float8_e4m3fn", "float8_e5m2") if hasattr(torch, n)}+ modules = dict(model.named_modules()) if fp8_storage else {}+ stored: list = []+ for key, value in list(converted.items()):want = torch.float32 if any(m in key.split(".") for m in keep) else dtypeif not value.is_floating_point():continue@@ -639,14 +705,45 @@ def _load_original_layout(transformer_cls, path, config, mapping_fn, keep, dtyperaise ValueError(f"{key}: {tuple(value.shape)} in the file, {tuple(wanted[key].shape)} in the model")+ fqn = key[: -len(".weight")]+ if (+ value.dtype in fp8_dtypes+ and want == dtype+ and key.endswith(".weight")+ and isinstance(modules.get(fqn), torch.nn.Linear)+ ):+ stored.append(fqn)+ continueif value.dtype != want:charge_cast(value, want)converted[key] = value.to(want)+ if stored:+ from .diffusion_native_quant import NATIVE_FP8, native_linear_class+ cls = native_linear_class()+ for fqn in stored:+ parent_name, _, leaf = fqn.rpartition(".")+ parent = model.get_submodule(parent_name) if parent_name else model+ codes = converted.pop(fqn + ".weight")+ charge_host_ram(codes.shape[0] * 4) # the unit scale; the codes stay as read+ layer = cls(+ getattr(parent, leaf).to(dtype),+ NATIVE_FP8,+ codes = codes,+ scale = torch.ones(codes.shape[0], 1, dtype = torch.float32),+ )+ setattr(parent, leaf, layer)+ converted[fqn + ".weight_q"] = layer.weight_q+ converted[fqn + ".weight_scale"] = layer.weight_scalemodel.load_state_dict(converted, strict = True, assign = True)+ if stored:+ from .diffusion_comfy_quant import FP8_STORED_ATTR+ setattr(model, FP8_STORED_ATTR, len(stored))model.eval()if logger is not None:logger.info(- "diffusion.single_file: %s loaded from an original-layout file",+ "diffusion.single_file: %s loaded from an original-layout file (%d fp8 layers stored as fp8, "+ "widened per layer)",transformer_cls.__name__,+ len(stored),)return modeldiff --git a/studio/backend/core/inference/gpu_residency.py b/studio/backend/core/inference/gpu_residency.pyindex 5b50e99d83e..b77a0f84123 100644--- a/studio/backend/core/inference/gpu_residency.py+++ b/studio/backend/core/inference/gpu_residency.py@@ -206,6 +206,7 @@ def _image_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int,from core.inference.diffusion import (DiffusionBackend as DB,_image_variant_hint,+ _single_file_fp8_upcast,resolve_model_kind,)from core.inference.diffusion_auto_policy import resident_bytes_from_declared@@ -261,8 +262,9 @@ def _image_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int,return Nonewhole = getattr(fam, "single_file_is_pipeline", False)if kind == "single_file":- fp8 = not whole and "fp8" in str(name).lower()- weights = estimate_safetensors_dense_mib(file_size_mib(path), fp8_upcast = fp8)+ weights = estimate_safetensors_dense_mib(+ file_size_mib(path), fp8_upcast = _single_file_fp8_upcast(path, fam)+ )elif whole:weights = whole_pipeline_gguf_resident_mib(path) or estimate_gguf_resident_mib(file_size_mib(path)diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 6b4c6a7a693..67964b1874f 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -63,6 +63,8 @@ from .diffusion_flow_shift import apply_comfy_flow_shiftfrom .diffusion_comfy_quant import (comfy_fp8_backend,comfy_int8_backend,+ fp8_storage_wanted,+ pipe_fp8_stored_layers,comfy_resident_mib,load_comfy_quant_transformer,refuse_comfy_quant,@@ -208,6 +210,7 @@ from .diffusion_transformer_quant import (transformer_is_quantised,)from .diffusion import (+ _comfy_keep_fp32_pricing,_family_override_resolved,_companion_split_expected,_layer_split_expected,@@ -481,20 +484,60 @@ def _video_comfy_backends(bf16_plan: Any = None,*,keep: bool = True,+ priced_offload: bool = False,+ widened_mib: Optional[int] = None,) -> dict[str, Optional[str]]:"""``load_comfy_quant_transformer`` backends for a ComfyUI-quantized video DiT: the ones Studio's own int8 / fp8quant would run on this card and plan (int8: torchao resident, the torchao-free twin under offload; fp8: torchao- resident only). ``keep=False`` (the user asked for the bf16 DiT) dequantizes everything."""+ resident only; else fp8 storage under offload). ``priced_offload``: the plan was sized with the offloaded+ backends (``_video_offload_reprice``), so a load that now fits resident keeps them. ``widened_mib``: the planned+ DiT price; a resident load stores fp8 when widening it would leave under the host reserve, as an image does.+ ``keep=False`` (the user asked for the bf16 DiT) dequantizes everything."""if not keep:- return {"int8_backend": None, "fp8_backend": None}- offload = not _video_comfy_resident(plan, bf16_plan)+ return {"int8_backend": None, "fp8_backend": None, "fp8_storage": False}+ offload = priced_offload or not _video_comfy_resident(plan, bf16_plan)name = getattr(fam, "name", None)return {"int8_backend": comfy_int8_backend(target, name, base, offload = offload),"fp8_backend": comfy_fp8_backend(target, name, base, offload = offload),+ "fp8_storage": _video_fp8_storage(fam, target, offload, widened_mib = widened_mib),}+def _video_offload_reprice(+ fam: Any, transformer_mib: Optional[int], offload_mib: Optional[int]+) -> tuple[Optional[int], bool]:+ """``(transformer_mib, priced_offload)`` once an offloaded plan re-prices the DiT with the offloaded backends.+ Larger: re-plan at it. Smaller (fp8 stored, the int8 twin where only it runs): re-plan at it too, and the load must+ then keep those backends even if the new plan fits resident, else it widens past the size it was planned at.+ H3 keeps its resident torchao path, so only a larger price moves it."""+ if offload_mib is None or transformer_mib is None or offload_mib == transformer_mib:+ return transformer_mib, False+ if offload_mib < transformer_mib and getattr(fam, "name", None) == "minimax-h3":+ return transformer_mib, False+ return offload_mib, offload_mib < transformer_mib+++def _video_fp8_storage(+ fam: Any,+ target: Any,+ offload: bool,+ *,+ widened_mib: Optional[int] = None,+) -> bool:+ """``fp8_storage_wanted`` for a video DiT, with the host RAM the load would widen into when ``widened_mib`` is+ known. H3 is excluded: its pin / stream / residency path is built on torchao weights."""+ if getattr(fam, "name", None) == "minimax-h3":+ return False+ available = None+ if widened_mib is not None and not offload:+ from .diffusion_small_host import host_ram_mib+ available = host_ram_mib()[1]+ return fp8_storage_wanted(+ target, offload = offload, widened_mib = widened_mib, host_available_mib = available+ )++def _video_comfy_resident_mib(fam: Any,base: str,@@ -510,7 +553,7 @@ def _video_comfy_resident_mib(int8 runtime runs here, fp8 layers as fp8 when the resident fp8 path does, the rest at bf16 (a dequantized fp8file is twice its size on disk). ``offload`` prices what an offloaded load keeps instead."""name = getattr(fam, "name", None)- keep_int8 = keep_fp8 = False+ keep_int8 = keep_fp8 = fp8_stored = Falseexclude: tuple = ()filters: dict = {}if keep:@@ -530,15 +573,18 @@ def _video_comfy_resident_mib(# Only a resident runtime keeps codes in a resident plan; an offloaded plan is re-priced.keep_int8 = bool(comfy_int8_backend(target, name, base, offload = offload))keep_fp8 = comfy_fp8_backend(target, name, base, offload = offload) is not None+ fp8_stored = _video_fp8_storage(fam, target, offload)return comfy_resident_mib(str(path),scan,keep_int8 = keep_int8,keep_fp8 = keep_fp8,+ fp8_stored = fp8_stored,keep_key = keep_key,exclude_tokens = exclude,key_map = _video_comfy_key_map(fam),**filters,+ **_comfy_keep_fp32_pricing(fam, target),)@@ -3021,9 +3067,12 @@ def _video_seed_stays_resident(def _render_under_no_grad(state: Any) -> bool:- """Render under no_grad: torchao weight subclasses fail an offload hook's in-place swap under inference_mode."""+ """Render under no_grad: torchao weight subclasses fail an offload hook's in-place swap under inference_mode.+ Stored fp8 codes too: moved under inference_mode they become unversionable, re-copied to host every offload."""if not getattr(state, "transformer_quant", None):- return False+ return getattr(state, "offload_policy", OFFLOAD_NONE) != OFFLOAD_NONE and bool(+ pipe_fp8_stored_layers(getattr(state, "pipe", None))+ )if getattr(state, "denoiser_streamed", False):return Truefamily = getattr(state, "family", None)@@ -8392,6 +8441,7 @@ class VideoBackend:denoiser_seed_scheme = Nonedenoiser_seed_gb = Noneplan, bf16_plan, quant_replanned = _plan_for_te_scale(te_scale, log = False)+ comfy_priced_offload = Falseif comfy_scan is not None and not _video_comfy_resident(plan, bf16_plan):# Offloaded, fp8 layers dequantize on NVIDIA: price the DiT the load will actually build.offload_mib = _video_comfy_resident_mib(@@ -8421,9 +8471,14 @@ class VideoBackend:)if offload_mib is not None and transformer_mib is not None:offload_mib = int(offload_mib * dtype_scale)- if offload_mib > transformer_mib:- transformer_mib = offload_mib+ repriced, comfy_priced_offload = _video_offload_reprice(+ fam, transformer_mib, offload_mib+ )+ if repriced != transformer_mib:+ transformer_mib = repricedplan, bf16_plan, quant_replanned = _plan_for_te_scale(te_scale, log = False)+ # The DiT as planned: a resident load that would widen it into under the host reserve stores fp8 instead.+ comfy_widened_mib = transformer_mib if comfy_scan is not None else Nonesettled_te_scale = te_scale# On unified memory the plan's 'none' policy is a placement, not a fit: there is no offload tier left to fall@@ -8657,7 +8712,14 @@ class VideoBackend:cache_dir = hub_cache_dir(),local_files_only = local_files_only,**_video_comfy_backends(- fam, base, target, plan, bf16_plan, keep = comfy_keep+ fam,+ base,+ target,+ plan,+ bf16_plan,+ keep = comfy_keep,+ priced_offload = comfy_priced_offload,+ widened_mib = comfy_widened_mib,),family = fam.name,target = target,@@ -8773,7 +8835,14 @@ class VideoBackend:cache_dir = hub_cache_dir(),local_files_only = local_files_only,**_video_comfy_backends(- fam, base, target, plan, bf16_plan, keep = comfy_keep+ fam,+ base,+ target,+ plan,+ bf16_plan,+ keep = comfy_keep,+ priced_offload = comfy_priced_offload,+ widened_mib = comfy_widened_mib,),family = fam.name,target = target,@@ -8823,7 +8892,14 @@ class VideoBackend:scan,kwargs,**_video_comfy_backends(- fam, base, target, plan, bf16_plan, keep = comfy_keep+ fam,+ base,+ target,+ plan,+ bf16_plan,+ keep = comfy_keep,+ priced_offload = comfy_priced_offload,+ widened_mib = comfy_widened_mib,),family = fam.name,target = target,diff --git a/studio/backend/core/inference/video_ltx2.py b/studio/backend/core/inference/video_ltx2.pyindex 69989fc8d85..1223194bbdb 100644--- a/studio/backend/core/inference/video_ltx2.py+++ b/studio/backend/core/inference/video_ltx2.py@@ -1248,6 +1248,7 @@ def load_ltx23_comfy_transformer(local_files_only: bool = False,int8_backend: Optional[str] = None,fp8_backend: Optional[str] = None,+ fp8_storage: Optional[bool] = None,family: Optional[str] = None,target: Any = None,logger: Any = None,@@ -1275,6 +1276,7 @@ def load_ltx23_comfy_transformer(},int8_backend = int8_backend,fp8_backend = fp8_backend,+ fp8_storage = fp8_storage,family = family,target = target,logger = logger,diff --git a/studio/backend/core/inference/video_ltx25.py b/studio/backend/core/inference/video_ltx25.pyindex 07ada73b195..9946f17ae88 100644--- a/studio/backend/core/inference/video_ltx25.py+++ b/studio/backend/core/inference/video_ltx25.py@@ -413,6 +413,7 @@ def load_ltx25_comfy_transformer(local_files_only: bool = False,int8_backend: Optional[str] = None,fp8_backend: Optional[str] = None,+ fp8_storage: Optional[bool] = None,family: Optional[str] = None,target: Any = None,logger: Any = None,@@ -438,6 +439,7 @@ def load_ltx25_comfy_transformer(},int8_backend = int8_backend,fp8_backend = fp8_backend,+ fp8_storage = fp8_storage,family = family,target = target,logger = logger,diff --git a/studio/backend/core/inference/video_minimax_h3_comfy.py b/studio/backend/core/inference/video_minimax_h3_comfy.pyindex d30c132aef1..53147d0742a 100644--- a/studio/backend/core/inference/video_minimax_h3_comfy.py+++ b/studio/backend/core/inference/video_minimax_h3_comfy.py@@ -188,6 +188,8 @@ def load_h3_comfy_transformer(fp8_backend: Optional[str],target: Any = None,logger: Any = None,+ # H3's pin / stream / residency path is built on torchao weights: dequantize what torchao does not take+ fp8_storage: Optional[bool] = False,) -> Any:"""The H3 denoiser of a ComfyUI-quantized file, int8 / fp8 codes kept, curve adaLN installed."""from .diffusion_comfy_quant import load_comfy_quant_transformer@@ -210,6 +212,7 @@ def load_h3_comfy_transformer(},int8_backend = int8_backend,fp8_backend = fp8_backend,+ fp8_storage = fp8_storage,family = "minimax-h3",target = target,logger = logger,diff --git a/studio/backend/tests/test_diffusion_comfy_quant.py b/studio/backend/tests/test_diffusion_comfy_quant.pyindex eaaa4fceb97..4a618202833 100644--- a/studio/backend/tests/test_diffusion_comfy_quant.py+++ b/studio/backend/tests/test_diffusion_comfy_quant.py@@ -13,6 +13,7 @@ ConvRot rotation is installed on exactly those, and that what cannot be mapped ifrom __future__ import annotationsimport json+import sysimport typesimport pytest@@ -443,9 +444,14 @@ def test_fp8_layers_dequantize_with_their_scale(tmp_path, monkeypatch):tensors.update({f"{layer}.weight": codes, f"{layer}.scale_weight": scale.reshape(1)})tensors["scaled_fp8"] = torch.zeros(2, dtype = torch.float8_e4m3fn)path = _save(tmp_path / "legacy_fp8_scaled.safetensors", tensors)- model = _load(path)+ model = _load(path, fp8_storage = False)assert torch.equal(model.blocks[0].to_out.weight, codes.float() * scale)assert model._unsloth_comfy_quant["dequantized"] == 1+ stored = _load(path, fp8_storage = True)+ assert stored._unsloth_comfy_quant["dequantized"] == 0+ assert stored._unsloth_comfy_quant["fp8_stored"] == 1+ assert torch.equal(stored.blocks[0].to_out.weight, codes.float() * scale)+ assert torch.equal(stored.blocks[0].to_out.weight_q, codes.view(torch.uint8))def test_a_converter_that_mixes_columns_is_refused(comfy_file, monkeypatch):@@ -591,7 +597,7 @@ def test_fp8_native_backend_keeps_codes_and_computes_the_dense_product(tmp_path,def test_fp8_without_a_runtime_dequantizes(tmp_path, monkeypatch):monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))path, _dense, tensors = _fp8_file(tmp_path)- model = _load_bf16(path, fp8_backend = None)+ model = _load_bf16(path, fp8_backend = None, fp8_storage = False)assert model._unsloth_comfy_quant["fp8"] == 0assert model._unsloth_comfy_quant["dequantized"] == 6w = tensors["blocks.0.out.weight"].float() * tensors["blocks.0.out.weight_scale"]@@ -601,13 +607,127 @@ def test_fp8_without_a_runtime_dequantizes(tmp_path, monkeypatch):def test_e5m2_and_non_bf16_pipelines_dequantize(tmp_path, monkeypatch):monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))path, _dense, _tensors = _fp8_file(tmp_path, fmt = "float8_e5m2")- assert _load_bf16(path, fp8_backend = "torchao")._unsloth_comfy_quant["fp8"] == 0+ assert (+ _load_bf16(path, fp8_backend = "torchao", fp8_storage = False)._unsloth_comfy_quant["fp8"] == 0+ )path, _dense, _tensors = _fp8_file(tmp_path)- model = _load(path, fp8_backend = "torchao") # float32 pipeline+ model = _load(path, fp8_backend = "torchao", fp8_storage = False) # float32 pipelineassert model._unsloth_comfy_quant["fp8"] == 0assert model._unsloth_comfy_quant["dequantized"] == 6+def _linear_bytes(model) -> int:+ """Bytes the model's Linear-shaped weights hold (dense ``weight`` or stored ``weight_q`` + ``weight_scale``)."""+ total = 0+ for name, tensor in model.state_dict().items():+ if name.endswith((".weight", ".weight_q", ".weight_scale")) and not name.startswith(+ "norm."+ ):+ total += tensor.numel() * tensor.element_size()+ return total+++ "fmt, dtype",+ [+ ("float8_e4m3fn", torch.bfloat16),+ ("float8_e4m3fn", torch.float16),+ ("float8_e4m3fn", torch.float32),+ ("float8_e5m2", torch.bfloat16),+ ],+)[email protected]("per_row", [False, True])+def test_fp8_without_a_runtime_stays_fp8_and_matches_the_dequant_exactly(+ tmp_path, monkeypatch, fmt, dtype, per_row+):+ """No fp8 GEMM: codes stay 1 byte per weight and outputs are bit-identical to the load-time dequant."""+ monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))+ path, _dense, tensors = _fp8_file(tmp_path, fmt = fmt, per_row = per_row)++ def load(**kwargs):+ return cq.load_comfy_quant_transformer(+ _Tiny,+ path,+ cq.refuse_comfy_quant(path),+ {"torch_dtype": dtype, "config": "base/repo", "subfolder": "transformer"},+ family = "z-image",+ int8_backend = None,+ fp8_backend = None,+ **kwargs,+ )++ widened = load(fp8_storage = False)+ stored = load(fp8_storage = True)+ assert widened._unsloth_comfy_quant["dequantized"] == 6+ assert stored._unsloth_comfy_quant["dequantized"] == 0+ assert stored._unsloth_comfy_quant["fp8_stored"] == 10+ assert stored._unsloth_comfy_quant["fp8"] == 0 # storage is not an fp8 runtime+ assert getattr(stored, "_unsloth_runtime_quant", None) is None+ for b in range(2):+ for leaf in ("to_q", "to_k", "to_v", "to_out"):+ layer = getattr(stored.blocks[b], leaf)+ assert layer.weight_q.dtype is torch.uint8 and layer.fp8_format == fmt+ assert layer.weight_scale.view(torch.float32).shape == (DIM,)+ assert torch.equal(layer.weight, getattr(widened.blocks[b], leaf).weight)+ # half the bytes of a 16-bit dequant (a quarter of fp32), plus a 4-byte scale per row+ rows = 2 * (4 * DIM + 2 * DIM)+ assert _linear_bytes(stored) == rows * DIM + rows * 4+ assert _linear_bytes(widened) == rows * DIM * torch.finfo(dtype).bits // 8+ x = torch.randn(3, 7, DIM).to(dtype)+ for b in range(2):+ for leaf in ("to_q", "to_out"):+ got = getattr(stored.blocks[b], leaf)(x)+ want = getattr(widened.blocks[b], leaf)(x)+ assert got.dtype is dtype and torch.equal(got, want)+ got = stored.blocks[b].adaLN_modulation(x)+ assert torch.equal(got, widened.blocks[b].adaLN_modulation(x))+ # no plan to judge by (None): widen, unless the env opts in; the kill switch beats an explicit request+ monkeypatch.delenv(cq.FP8_STORAGE_ENV, raising = False)+ assert load()._unsloth_comfy_quant["dequantized"] == 6+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "1")+ assert load()._unsloth_comfy_quant["fp8_stored"] == 10+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "0")+ assert cq.fp8_storage_wanted(None, offload = True) is False+++def test_fp8_storage_follows_the_device_lora_and_kill_switch(monkeypatch):+ monkeypatch.delenv(cq.FP8_STORAGE_ENV, raising = False)+ cuda = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)+ assert cq.comfy_fp8_storage(cuda) is True+ assert cq.comfy_fp8_storage(types.SimpleNamespace(device = "cpu")) is True+ assert cq.comfy_fp8_storage(None) is True+ # MPS has no float8 dtypes; a LoRA needs dense Linears to attach to+ assert cq.comfy_fp8_storage(types.SimpleNamespace(device = "mps")) is False+ assert cq.comfy_fp8_storage(cuda, lora = True) is False+ for off in ("0", "off", "false"):+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, off)+ assert cq.comfy_fp8_storage(cuda) is False+++def test_fp8_storage_keeps_fp32_modules_and_the_torchao_layers_apart(tmp_path, monkeypatch):+ """Only layers the fp8 GEMM runtime skips are stored; fp32-kept Linears still widen to fp32."""+ monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))+ path, _dense, _tensors = _fp8_file(tmp_path)++ class _Keep(_Tiny):+ _keep_in_fp32_modules = ["adaLN_modulation"]++ model = cq.load_comfy_quant_transformer(+ _Keep,+ path,+ cq.refuse_comfy_quant(path),+ {"torch_dtype": torch.float16, "config": "base/repo", "subfolder": "transformer"},+ family = "z-image",+ int8_backend = None,+ fp8_backend = None,+ fp8_storage = True,+ )+ assert model._unsloth_comfy_quant["fp8_stored"] == 8+ assert model._unsloth_comfy_quant["dequantized"] == 2+ assert type(model.blocks[0].adaLN_modulation[0]) is nn.Linear+ assert model.blocks[0].adaLN_modulation[0].weight.dtype is torch.float32++@pytest.mark.parametrize("native, supported, offload, dtype, env, want",[@@ -843,3 +963,389 @@ def test_buffers_built_in_init_follow_the_compute_dtype(comfy_file):)assert model.freqs.dtype == torch.bfloat16assert model.blocks[0].to_q.weight_q.dtype == torch.int8+++def test_a_lora_on_stored_fp8_layers_is_refused_with_the_way_out(tmp_path, monkeypatch):+ """Stored fp8 Linears are not nn.Linear: a generation-time LoRA is refused by name."""+ import threading++ from core.inference.diffusion import DiffusionBackend++ monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))+ path, _dense, _tensors = _fp8_file(tmp_path)+ model = _load_bf16(path, fp8_backend = None, fp8_storage = True)+ assert cq.fp8_stored_layers(model) == 10+ assert cq.fp8_stored_layers(_load_bf16(path, fp8_backend = None, fp8_storage = False)) == 0+ assert cq.fp8_stored_layers(None) == 0+ pipe = types.SimpleNamespace(transformer = model, _unsloth_loras = ())+ state = types.SimpleNamespace(pipe = pipe, transformer_quant = None)+ backend = DiffusionBackend.__new__(DiffusionBackend)+ with pytest.raises(ValueError, match = "stored as fp8"):+ DiffusionBackend._apply_loras(backend, state, [("some/lora", 1.0)], threading.Event())+ DiffusionBackend._apply_loras(backend, state, [], threading.Event())+++def test_fp8_storage_default_rule_is_offload_or_host_ram(monkeypatch):+ """Unset env stores only when widening would offload or crowd host RAM; ``1`` always; MPS and LoRA never."""+ monkeypatch.delenv(cq.FP8_STORAGE_ENV, raising = False)+ cuda = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)+ want = cq.fp8_storage_wanted+ assert want(cuda) is False+ assert want(cuda, offload = True) is True+ reserve = cq.FP8_STORAGE_HOST_RESERVE_MIB+ assert want(cuda, widened_mib = 24000, host_available_mib = 24000 + reserve + 1) is False+ assert want(cuda, widened_mib = 24000, host_available_mib = 24000 + reserve - 1) is True+ assert want(cuda, offload = True, lora = True) is False+ assert want(types.SimpleNamespace(device = "mps"), offload = True) is False+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "1")+ assert want(cuda) is True+++def test_fp8_upcast_is_read_from_the_header_not_the_basename(tmp_path):+ """A plain fp8 file named without 'fp8' still widens ~2x (and so can be stored); a bf16 one named 'fp8' does not."""+ from safetensors.torch import save_file++ from core.inference import diffusion as d++ fam = types.SimpleNamespace(single_file_is_pipeline = False)+ plain = tmp_path / "krea2.safetensors"+ save_file({"blocks.0.weight": torch.zeros(4, 4, dtype = torch.float8_e4m3fn)}, str(plain))+ dense = tmp_path / "krea2_fp8.safetensors"+ save_file({"blocks.0.weight": torch.zeros(4, 4, dtype = torch.bfloat16)}, str(dense))+ assert cq.header_has_fp8_weights(str(plain)) is True+ assert cq.header_has_fp8_weights(str(dense)) is False+ assert d._single_file_fp8_upcast(str(plain), fam) is True+ assert d._single_file_fp8_upcast(str(dense), fam) is False+ broken = tmp_path / "broken_fp8.safetensors"+ broken.write_bytes(b"\x00" * 4)+ assert cq.header_has_fp8_weights(str(broken)) is None+ assert d._single_file_fp8_upcast(str(broken), fam) is True+ assert (+ d._single_file_fp8_upcast(str(plain), types.SimpleNamespace(single_file_is_pipeline = True))+ is False+ )+++def test_fnuz_fp8_headers_are_priced_widened_and_never_stored(tmp_path):+ """FNUZ fp8 is seen by the header predicate and always priced widened."""+ from safetensors.torch import save_file++ from core.inference import diffusion as d++ fam = types.SimpleNamespace(single_file_is_pipeline = False)+ fnuz = tmp_path / "krea2.safetensors"+ save_file({"blocks.0.weight": torch.zeros(4, 4, dtype = torch.float8_e4m3fnuz)}, str(fnuz))+ ocp = tmp_path / "krea2_ocp.safetensors"+ save_file({"blocks.0.weight": torch.zeros(4, 4, dtype = torch.float8_e4m3fn)}, str(ocp))+ assert cq.header_has_fp8_weights(str(fnuz)) is True+ assert d._single_file_fp8_upcast(str(fnuz), fam) is True+ assert cq.header_has_fp8_weights(str(fnuz), fnuz_only = True) is True+ assert cq.header_has_fp8_weights(str(ocp), fnuz_only = True) is False+++def _plan_harness(+ monkeypatch,+ tmp_path,+ *,+ fam,+ resident_at,+ fits_mib,+ comfy_mib = None,+):+ """Run ``_plan_memory`` for a single file with a fake planner (resident iff the DiT fits ``fits_mib``)."""+ from core.inference import diffusion as d+ from core.inference import diffusion_memory as dm++ path = tmp_path / "dit.safetensors"+ if not path.exists():+ path.write_bytes(b"\x00" * 16)+ seen: list = []++ def plan(**kwargs):+ mib = kwargs["model_dense_mib"]+ seen.append(mib)+ return types.SimpleNamespace(resident = mib is not None and mib <= fits_mib)++ monkeypatch.setattr(d, "plan_diffusion_memory", plan)+ monkeypatch.setattr(d, "plan_keeps_transformer_resident", lambda p: p.resident)+ monkeypatch.setattr(d, "comfy_fp8_storage", lambda target, lora = False: True)+ monkeypatch.setattr(dm, "_available_system_memory_mib", lambda: 10**7)+ monkeypatch.setattr(+ d.DiffusionBackend, "_companion_cache_bytes", staticmethod(lambda *a, **k: 0)+ )+ monkeypatch.setattr(+ d.DiffusionBackend, "_text_encoder_cache_bytes", staticmethod(lambda *a, **k: 0)+ )+ monkeypatch.setattr(d.DiffusionBackend, "_supplied_component_mib", lambda self, *a, **k: None)+ if comfy_mib is not None:+ monkeypatch.setattr(+ d.DiffusionBackend, "_comfy_single_file_resident_mib", staticmethod(comfy_mib)+ )+ else:+ monkeypatch.setattr(+ d.DiffusionBackend,+ "_comfy_single_file_resident_mib",+ staticmethod(lambda *a, **k: None),+ )+ monkeypatch.setattr(d, "file_size_mib", lambda p: 1000)+ monkeypatch.setattr(+ d,+ "estimate_safetensors_dense_mib",+ lambda size, fp8_upcast = False: size * (2 if fp8_upcast else 1),+ )+ from core.inference.diffusion_device import resolve_diffusion_device_target++ backend = d.DiffusionBackend()+ final = backend._plan_memory(+ resolve_diffusion_device_target(),+ str(resident_at or path),+ "base/repo",+ fam,+ None,+ False,+ kind = "single_file",+ )+ return backend, final, seen, str(resident_at or path)+++def test_offloaded_plan_prices_fp8_with_the_offloaded_backend_and_picks_storage(+ monkeypatch, tmp_path+):+ """Offload loses torchao fp8: the plan re-prices offloaded and picks storage."""+ calls = []++ def comfy_mib(+ path,+ fam,+ target,+ base,+ *,+ lora = False,+ fp8_storage = False,+ offload = False,+ ):+ calls.append((fp8_storage, offload))+ if not offload:+ return 1000 # torchao keeps the codes either way+ return 1000 if fp8_storage else 2000++ fam = types.SimpleNamespace(name = "flux.1", single_file_is_pipeline = False)+ # fits_mib below 1000: the plan offloads at every size+ backend, final, seen, path = _plan_harness(+ monkeypatch, tmp_path, fam = fam, resident_at = None, fits_mib = 500, comfy_mib = comfy_mib+ )+ assert (False, True) in calls and (True, True) in calls+ assert backend._fp8_storage_picks[path] is True+ assert seen[-1] == 1000 and 2000 in seen+ assert final.resident is False+++def test_offloaded_plan_without_storage_is_priced_widened(monkeypatch, tmp_path):+ """No storage possible (LoRA / kill switch): the offloaded plan carries the widened size the loader builds."""+ from core.inference import diffusion as d++ def comfy_mib(+ path,+ fam,+ target,+ base,+ *,+ lora = False,+ fp8_storage = False,+ offload = False,+ ):+ return 2000 if offload and not fp8_storage else 1000++ real_harness_storage = []+ monkeypatch.setattr(+ d.DiffusionBackend,+ "_fp8_storage_off",+ True,+ raising = False,+ )+ fam = types.SimpleNamespace(name = "flux.1", single_file_is_pipeline = False)+ orig_setattr = monkeypatch.setattr++ def setattr_no_storage(target, name, value, *a, **k):+ if target is d and name == "comfy_fp8_storage":+ value = lambda target, lora = False: False # noqa: E731+ real_harness_storage.append(name)+ return orig_setattr(target, name, value, *a, **k)++ monkeypatch.setattr = setattr_no_storage+ try:+ backend, final, seen, path = _plan_harness(+ monkeypatch, tmp_path, fam = fam, resident_at = None, fits_mib = 500, comfy_mib = comfy_mib+ )+ finally:+ monkeypatch.setattr = orig_setattr+ assert seen[-1] == 2000+ assert backend._fp8_storage_picks[path] is False+++def test_a_resident_plan_keeps_the_resident_backend_price(monkeypatch, tmp_path):+ """Resident torchao fp8: no offload re-price, no storage."""+ calls = []++ def comfy_mib(+ path,+ fam,+ target,+ base,+ *,+ lora = False,+ fp8_storage = False,+ offload = False,+ ):+ calls.append(offload)+ return 1000 if not offload else 2000++ fam = types.SimpleNamespace(name = "flux.1", single_file_is_pipeline = False)+ backend, final, seen, path = _plan_harness(+ monkeypatch, tmp_path, fam = fam, resident_at = None, fits_mib = 5000, comfy_mib = comfy_mib+ )+ assert True not in calls and seen == [1000]+ assert backend._fp8_storage_picks[path] is False and final.resident is True+++def test_fnuz_krea2_file_is_not_planned_at_the_stored_size(monkeypatch, tmp_path):+ """A Krea 2 FNUZ file widens under storage too, so the planner records no storage pick for it."""+ from safetensors.torch import save_file++ from core.inference.diffusion_krea2 import KREA2_FAMILY_NAME++ fam = types.SimpleNamespace(name = KREA2_FAMILY_NAME, single_file_is_pipeline = False)+ for dtype, picked in ((torch.float8_e4m3fnuz, False), (torch.float8_e4m3fn, True)):+ path = tmp_path / f"krea2_{str(dtype).split('_')[-1]}.safetensors"+ save_file({"blocks.0.weight": torch.zeros(4, 4, dtype = dtype)}, str(path))+ backend, final, seen, p = _plan_harness(+ monkeypatch, tmp_path, fam = fam, resident_at = path, fits_mib = 500+ )+ assert backend._fp8_storage_picks[p] is picked, dtype+ assert seen[-1] == (1000 if picked else 2000)+++def test_a_plan_priced_with_the_offload_only_backend_keeps_it_when_resident(monkeypatch, tmp_path):+ """Int8 priced via the offload twin: the re-plan fits resident but the loader must still probe offloaded."""++ def comfy_mib(+ path,+ fam,+ target,+ base,+ *,+ lora = False,+ fp8_storage = False,+ offload = False,+ ):+ return 1000 if offload else 2000++ fam = types.SimpleNamespace(name = "flux.1", single_file_is_pipeline = False)+ backend, final, seen, path = _plan_harness(+ monkeypatch, tmp_path, fam = fam, resident_at = None, fits_mib = 1500, comfy_mib = comfy_mib+ )+ assert seen == [2000, 1000] and final.resident is True+ assert backend._comfy_offload_for(final, path) is True+ # Not priced offloaded: a resident plan probes resident.+ backend._comfy_priced_offload[path] = False+ assert backend._comfy_offload_for(final, path) is False+++def test_a_plan_repriced_larger_under_offload_does_not_force_the_offload_probe(+ monkeypatch, tmp_path+):+ def comfy_mib(+ path,+ fam,+ target,+ base,+ *,+ lora = False,+ fp8_storage = False,+ offload = False,+ ):+ return 2000 if offload else 1000++ fam = types.SimpleNamespace(name = "flux.1", single_file_is_pipeline = False)+ backend, final, seen, path = _plan_harness(+ monkeypatch, tmp_path, fam = fam, resident_at = None, fits_mib = 500, comfy_mib = comfy_mib+ )+ assert backend._comfy_priced_offload[path] is False+ assert backend._comfy_offload_for(types.SimpleNamespace(resident = True), path) is False+++def test_fp8_stored_pricing_widens_keep_fp32_layers_as_the_loader_does(tmp_path):+ """``_keep_in_fp32_modules`` linears are never priced at one byte, under either name."""+ n = 1024+ names = ("blocks.0.ffn", "condition_embedder.time_embedder.linear_1", "time_embedding.0")+ tensors, layers = {}, {}+ for name in names:+ tensors[f"{name}.weight"] = torch.zeros(n, n, dtype = torch.float8_e4m3fn)+ tensors[f"{name}.weight_scale"] = torch.tensor(1.0)+ layers[name] = {"format": "float8_e4m3fn"}+ path = _save(+ tmp_path / "wan.safetensors",+ tensors,+ {cq.QUANT_METADATA_KEY: json.dumps({"layers": layers})},+ )+ scan = cq.scan_comfy_quant(path)+ kw = dict(keep_int8 = False, keep_fp8 = False, fp8_stored = True)+ one = n * n / 2**20 # MiB per layer per byte+ plain = cq.comfy_resident_mib(path, scan, **kw)+ assert plain == pytest.approx(3 * one, abs = 1)+ keep = ("time_embedder", "scale_shift_table")+ fp16 = cq.comfy_resident_mib(path, scan, **kw, keep_fp32 = keep, keep_fp32_bytes = 4)+ assert fp16 == pytest.approx(one + 2 * 4 * one, abs = 1)+ bf16 = cq.comfy_resident_mib(path, scan, **kw, keep_fp32 = keep)+ assert bf16 == pytest.approx(one + 2 * 2 * one, abs = 1)+ for name in names[1:]:+ assert not cq.fp8_stored_by_header("float8_e4m3fn", (n, n), name, keep)+ assert cq.fp8_stored_by_header("float8_e4m3fn", (n, n), names[0], keep)+++def _fake_diffusers(monkeypatch):+ # CPU CI may run without diffusers (or with a stub): pin the two classes these tests read.+ class WanTransformer3DModel:+ _keep_in_fp32_modules = ["time_embedder", "scale_shift_table", "norm1", "norm2", "norm3"]++ class FluxTransformer2DModel:+ pass++ fake = types.SimpleNamespace(+ WanTransformer3DModel = WanTransformer3DModel, FluxTransformer2DModel = FluxTransformer2DModel+ )+ monkeypatch.setitem(sys.modules, "diffusers", fake)+++def test_real_wan_transformer_keeps_its_time_embedder_in_fp32():+ try:+ import diffusers+ cls = getattr(diffusers, "WanTransformer3DModel", None)+ except Exception: # noqa: BLE001 -- no usable diffusers here+ cls = None+ if cls is None:+ pytest.skip("this diffusers has no WanTransformer3DModel")+ wan = types.SimpleNamespace(transformer_class = "WanTransformer3DModel")+ assert "time_embedder" in cq.keep_fp32_modules_for(wan)+++def test_keep_fp32_modules_come_from_the_family_transformer(monkeypatch):+ _fake_diffusers(monkeypatch)+ wan = types.SimpleNamespace(transformer_class = "WanTransformer3DModel")+ assert "time_embedder" in cq.keep_fp32_modules_for(wan)+ assert cq.keep_fp32_modules_for(types.SimpleNamespace(transformer_class = "NoSuchModel")) == ()+ assert cq.keep_fp32_modules_for(types.SimpleNamespace()) == ()+++def test_planner_prices_keep_fp32_layers_at_fp32_only_on_fp16(monkeypatch):+ from core.inference import diffusion as d++ _fake_diffusers(monkeypatch)++ wan = types.SimpleNamespace(transformer_class = "WanTransformer3DModel")+ fp16 = d._comfy_keep_fp32_pricing(wan, types.SimpleNamespace(dtype = torch.float16))+ bf16 = d._comfy_keep_fp32_pricing(wan, types.SimpleNamespace(dtype = torch.bfloat16))+ assert "time_embedder" in fp16["keep_fp32"] and fp16["keep_fp32_bytes"] == 4+ assert bf16["keep_fp32_bytes"] is None+ flux = types.SimpleNamespace(transformer_class = "FluxTransformer2DModel")+ assert d._comfy_keep_fp32_pricing(flux, types.SimpleNamespace(dtype = torch.float16)) == {}diff --git a/studio/backend/tests/test_diffusion_compile_cache_variant.py b/studio/backend/tests/test_diffusion_compile_cache_variant.pyindex 3ec4ca2ae7d..2cb278c74e3 100644--- a/studio/backend/tests/test_diffusion_compile_cache_variant.py+++ b/studio/backend/tests/test_diffusion_compile_cache_variant.py@@ -258,3 +258,24 @@ def test_per_row_and_per_tensor_scales_key_apart():per_tensor = cc._tensor_signature(FakeFp8([64, 256], (1,)))assert per_row != per_tensorassert "block_size=1,full" in per_row and "block_size=full,full" in per_tensor+++def _stored_fp8(fmt):+ from core.inference.diffusion_native_quant import NATIVE_FP8, native_linear_class++ m = _model()+ lin = m[0]+ fp8 = getattr(torch, fmt)+ w = lin.weight.detach().float()+ scale = w.abs().amax(dim = 1, keepdim = True) / torch.finfo(fp8).max+ m[0] = native_linear_class()(lin, NATIVE_FP8, codes = (w / scale).to(fp8), scale = scale)+ return m+++def test_stored_fp8_formats_key_apart_and_each_keys_stable():+ """E4M3 and E5M2 codes decode through different integer graphs, so they never share a bundle."""+ e4, e5 = _stored_fp8("float8_e4m3fn"), _stored_fp8("float8_e5m2")+ assert e4[0].fp8_format == "float8_e4m3fn" and e5[0].fp8_format == "float8_e5m2"+ assert _key(e4, "fp8") != _key(e5, "fp8")+ assert _key(e4, "fp8") == _key(_stored_fp8("float8_e4m3fn"), "fp8")+ assert _key(e5, "fp8") == _key(_stored_fp8("float8_e5m2"), "fp8")diff --git a/studio/backend/tests/test_diffusion_native_quant.py b/studio/backend/tests/test_diffusion_native_quant.pyindex 2812df5a681..57502a4dda8 100644--- a/studio/backend/tests/test_diffusion_native_quant.py+++ b/studio/backend/tests/test_diffusion_native_quant.py@@ -1211,3 +1211,43 @@ def test_a_rotated_layer_passes_an_empty_input_through():)out = layer(torch.zeros(2, 0, 256, dtype = torch.bfloat16))assert out.shape == (2, 0, 64)++[email protected]("fmt", ["float8_e4m3fn", "float8_e5m2"])+def test_fp8_integer_decode_equals_the_cast_bit_for_bit(fmt):+ """Integer decode equals the eager cast on all 256 codes."""+ from core.inference.diffusion_native_quant import fp8_bits_to_float32++ bits = torch.arange(256, dtype = torch.int32).to(torch.uint8)+ want = bits.view(getattr(torch, fmt)).to(torch.float32)+ got = fp8_bits_to_float32(bits, fmt)+ nan = want.isnan()+ assert torch.equal(got.isnan(), nan)+ assert torch.equal(got[~nan].view(torch.int32), want[~nan].view(torch.int32))++[email protected]("fmt", ["float8_e4m3fn", "float8_e5m2"])+def test_fp8_codes_layer_matches_the_dequant_formula_eager_and_compiled(fmt):+ """A native fp8 layer built from checkpoint codes + per-row scale: its forward equals F.linear on+ ``(codes.float() * scale).to(dtype)`` (the load-time dequant) in eager, and the compiled forward (integer+ decode) matches eager exactly with no graph break."""+ from core.inference.diffusion_native_quant import NATIVE_FP8, native_linear_class++ torch.manual_seed(0)+ fp8 = getattr(torch, fmt)+ base = torch.nn.Linear(64, 48).to(torch.bfloat16)+ w = torch.randn(48, 64)+ scale = w.abs().amax(dim = 1, keepdim = True) / torch.finfo(fp8).max+ codes = (w / scale).to(fp8)+ layer = native_linear_class()(base, NATIVE_FP8, codes = codes, scale = scale)+ assert layer.weight_q.dtype is torch.uint8 and layer.fp8_format == fmt+ dense = (codes.to(torch.float32) * scale).to(torch.bfloat16)+ x = torch.randn(3, 64, dtype = torch.bfloat16)+ want = torch.nn.functional.linear(x, dense, base.bias)+ assert torch.equal(layer.weight, dense)+ assert torch.equal(layer(x), want)+ torch._dynamo.reset()+ explained = torch._dynamo.explain(layer)(x)+ assert explained.graph_break_count == 0+ compiled = torch.compile(layer, fullgraph = True)+ assert torch.equal(compiled(x), layer(x))diff --git a/studio/backend/tests/test_diffusion_offload_precision.py b/studio/backend/tests/test_diffusion_offload_precision.pyindex bc55b7328e8..196f1308abd 100644--- a/studio/backend/tests/test_diffusion_offload_precision.py+++ b/studio/backend/tests/test_diffusion_offload_precision.py@@ -309,6 +309,26 @@ def test_render_grad_mode(policy, transformer_quant, text_encoder_quant, no_gradassert dmod._torchao_render_needs_no_grad(state) is no_grad+ "policy, no_grad",+ [(OFFLOAD_NONE, False), (OFFLOAD_MODEL, True), (OFFLOAD_GROUP, True)],+)+def test_offloaded_stored_fp8_renders_under_no_grad(policy, no_grad):+ """Stored fp8 codes are plain buffers: moved under inference_mode they are inference tensors the keep-host+ offload cannot version, so each offload would copy 12 GB back to fresh host memory (Krea 2)."""+ transformer = types.SimpleNamespace(_unsloth_fp8_stored = 256)+ state = types.SimpleNamespace(+ offload_policy = policy,+ transformer_quant = None,+ text_encoder_quant = None,+ pipe = types.SimpleNamespace(transformer = transformer),+ )+ assert dmod._torchao_render_needs_no_grad(state) is no_grad+ from core.inference import video as vid++ assert vid._render_under_no_grad(state) is no_grad++def test_render_uses_the_widened_switch():import inspectdiff --git a/studio/backend/tests/test_diffusion_single_file_host_ram.py b/studio/backend/tests/test_diffusion_single_file_host_ram.pyindex c3ce4d19c23..8da0aa8de08 100644--- a/studio/backend/tests/test_diffusion_single_file_host_ram.py+++ b/studio/backend/tests/test_diffusion_single_file_host_ram.py@@ -218,3 +218,52 @@ def test_kill_switch_charges_nothing(tmp_path, monkeypatch, krea2, kind):fp8_backend = None,)assert charged == []++[email protected]("kind", ["fp8", "fp8_scaled"])+def test_a_file_that_fits_only_stored_loads_with_fp8_storage_and_is_refused_widened(+ tmp_path, monkeypatch, krea2, kind+):+ """fp8 storage keeps the codes memory-mapped, so the up-front estimate prices them at their stored bytes (plus+ the fp32 row scales): free RAM between the stored and the widened need loads with storage, and is refused when+ UNSLOTH_DIFFUSION_FP8_STORAGE=0 makes the load widen."""+ path, _ = _write(tmp_path, kind)+ keep = krea2._keep_in_fp32_modules+ # Price in bytes, no reserve: the tiny model's needs are a few KB apart.+ monkeypatch.setattr(conv, "_MIB", 1)+ monkeypatch.setattr(small_host, "host_reserve_mib", lambda total: 0)+ if kind == "fp8":+ widened = conv.converted_host_bytes(str(path), torch.bfloat16, keep_fp32 = keep)+ stored = conv.converted_host_bytes(+ str(path), torch.bfloat16, keep_fp32 = keep, dequantized = conv.fp8_stored_estimate(keep)+ ) + conv._fp8_stored_scale_bytes(str(path), keep)+ else:+ scan = cq.refuse_comfy_quant(str(path))+ widened = conv.converted_host_bytes(+ str(path), torch.bfloat16, dequantized = {n + ".weight": 1 for n in scan.layers}.get+ )+ stored = conv.converted_host_bytes(+ str(path), torch.bfloat16, dequantized = {n + ".weight": 0 for n in scan.layers}.get+ )+ assert 0 < stored < widened+ _host(monkeypatch, available_mib = (stored + widened) // 2)++ def load(**kw):+ if kind == "fp8":+ return conv.load_original_layout_transformer(krea2, str(path), dict(_KWARGS), **kw)+ return cq.load_comfy_quant_transformer(+ krea2,+ str(path),+ cq.refuse_comfy_quant(str(path)),+ dict(_KWARGS),+ int8_backend = None,+ fp8_backend = None,+ **kw,+ )++ model = load(fp8_storage = True)+ assert model._unsloth_fp8_stored > 0+ assert model.transformer_blocks[0].attn.to_q.weight_q.dtype == torch.uint8+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "0")+ with pytest.raises(RuntimeError, match = "Not enough system RAM"):+ load(fp8_storage = cq.comfy_fp8_storage(None))diff --git a/studio/backend/tests/test_diffusion_single_file_krea2_hunyuanimage.py b/studio/backend/tests/test_diffusion_single_file_krea2_hunyuanimage.pyindex ed938c8cfac..c503ff7492c 100644--- a/studio/backend/tests/test_diffusion_single_file_krea2_hunyuanimage.py+++ b/studio/backend/tests/test_diffusion_single_file_krea2_hunyuanimage.py@@ -468,6 +468,49 @@ def test_comfy_int8_krea2_file_loads_with_codes_exact(tmp_path, monkeypatch):assert loaded._unsloth_comfy_quant["dequantized"] == len(expected)[email protected]("fmt", ["float8_e4m3fn", "float8_e5m2"])+def test_plain_fp8_krea2_file_keeps_its_linears_fp8_and_widens_exactly(tmp_path, monkeypatch, fmt):+ """A plain (unscaled) fp8 file: with fp8_storage every Linear weight stays 1 byte per element and the forward+ rebuilds exactly the bf16 weight the load-time cast held (fp8 values are exact in bf16), so outputs match."""+ cls = _class("Krea2Transformer2DModel")+ model = _tiny("Krea2Transformer2DModel", KREA2_CFG)+ fp8 = getattr(torch, fmt)+ tensors = {}+ for k, v in krea2_original(model.state_dict()).items():+ tensors[k] = (v.to(fp8) if v.dim() == 2 else v.to(torch.bfloat16)).contiguous()+ path = tmp_path / "krea2_tiny_fp8.safetensors"+ safetensors_torch.save_file(tensors, str(path))+ monkeypatch.setattr(cls, "load_config", classmethod(lambda c, *a, **k: dict(KREA2_CFG)))+ kwargs = {+ "torch_dtype": torch.bfloat16,+ "config": "krea/Krea-2-Turbo",+ "subfolder": "transformer",+ }+ widened = conv.load_original_layout_transformer(cls, str(path), kwargs)+ stored = conv.load_original_layout_transformer(cls, str(path), kwargs, fp8_storage = True)+ linears = [n for n, m in widened.named_modules() if isinstance(m, torch.nn.Linear)]+ assert linears+ keep = cls._keep_in_fp32_modules or []+ count = 0+ for name in linears:+ dense = widened.get_submodule(name)+ layer = stored.get_submodule(name)+ if dense.weight.dtype is not torch.bfloat16 or any(m in name.split(".") for m in keep):+ assert type(layer) is torch.nn.Linear, name+ continue+ count += 1+ assert layer.weight_q.dtype is torch.uint8 and layer.fp8_format == fmt, name+ assert torch.equal(layer.weight, dense.weight), name+ x = torch.randn(2, 5, layer.in_features, dtype = torch.bfloat16)+ assert torch.equal(layer(x), dense(x)), name+ assert count+ assert stored._unsloth_fp8_stored == count+ assert not getattr(widened, "_unsloth_fp8_stored", 0)+ wide_bytes = sum(t.numel() * t.element_size() for t in widened.state_dict().values())+ stored_bytes = sum(t.numel() * t.element_size() for t in stored.state_dict().values())+ assert stored_bytes < 0.6 * wide_bytes++@pytest.mark.parametrize("prefix", ["", "model.model.", "diffusion_model."])def test_plain_krea2_file_loads_like_the_base_repo(tmp_path, monkeypatch, prefix):"""A plain bf16 ComfyUI file through Studio's loader (used while diffusers gives Krea 2 nodiff --git a/studio/backend/tests/test_gpu_residency_coexistence.py b/studio/backend/tests/test_gpu_residency_coexistence.pyindex dde3fef1e42..1250333cc82 100644--- a/studio/backend/tests/test_gpu_residency_coexistence.py+++ b/studio/backend/tests/test_gpu_residency_coexistence.py@@ -686,3 +686,48 @@ class TestKeptSlotReport(unittest.TestCase):if __name__ == "__main__":unittest.main()+++def test_single_file_image_need_reads_fp8_from_the_header(tmp_path, monkeypatch):+ """The residency estimate prices a single file like the load planner: a plain fp8 file named without 'fp8'+ widens ~2x, a bf16 file named 'fp8' does not."""+ import types++ import torch+ from safetensors.torch import save_file++ import core.inference.diffusion as dmod+ import core.inference.diffusion_engine_router as router+ import core.inference.diffusion_families as dfam+ import core.inference.gpu_residency as gr+ from core.inference.sd_cpp_engine import ENGINE_DIFFUSERS++ fam = types.SimpleNamespace(name = "krea-2", single_file_is_pipeline = False)+ monkeypatch.setattr(dfam, "detect_family_for_pick", lambda *a, **k: fam)+ monkeypatch.setattr(dfam, "resolve_base_repo", lambda f, b: b)+ monkeypatch.setattr(dfam, "prefer_ungated_mirror", lambda r: r)+ monkeypatch.setattr(router, "predict_engine", lambda *a, **k: ENGINE_DIFFUSERS)+ monkeypatch.setattr(dmod, "_image_variant_hint", lambda *a, **k: None)+ monkeypatch.setattr(+ dmod.DiffusionBackend, "_companion_cache_bytes", staticmethod(lambda *a, **k: 1 << 20)+ )++ def need(path):+ monkeypatch.setattr(gr, "_cached_pick", lambda repo, name: str(path))+ request = types.SimpleNamespace(+ model_path = "org/krea",+ gguf_filename = path.name,+ text_encoder_file = None,+ vae_file = None,+ model_kind = "single_file",+ family_override = None,+ base_repo = "org/base",+ )+ return gr._image_need(request, 0)[0]++ plain = tmp_path / "krea2.safetensors"+ save_file({"blocks.0.weight": torch.zeros(2048, 2048, dtype = torch.float8_e4m3fn)}, str(plain))+ dense = tmp_path / "krea2_fp8.safetensors"+ save_file({"blocks.0.weight": torch.zeros(1024, 2048, dtype = torch.bfloat16)}, str(dense))+ # Both files are 4 MiB on disk: only the one that stores fp8 widens on load.+ assert need(plain) > need(dense)diff --git a/studio/backend/tests/test_video_comfy_quant.py b/studio/backend/tests/test_video_comfy_quant.pyindex 68028862659..c3687c2711e 100644--- a/studio/backend/tests/test_video_comfy_quant.py+++ b/studio/backend/tests/test_video_comfy_quant.py@@ -413,16 +413,50 @@ def test_video_backends_follow_studio_rules(monkeypatch):assert vid._video_comfy_backends(fam, "b", None, _plan(True), _plan(True)) == {"int8_backend": "torchao","fp8_backend": "torchao",+ "fp8_storage": False,}- # either plan offloading the DiT means the hooks move it: int8 takes the torchao-free twin, fp8 dequantizes+ # either plan offloading the DiT means the hooks move it: int8 takes the torchao-free twin, fp8 has no GEMM+ # and stays stored as fp8, widened per layerassert vid._video_comfy_backends(fam, "b", None, _plan(True), _plan(False)) == {"int8_backend": "native","fp8_backend": None,+ "fp8_storage": True,}assert vid._video_comfy_backends(fam, "b", None, _plan(True), keep = False) == {"int8_backend": None,"fp8_backend": None,+ "fp8_storage": False,}+ h3 = types.SimpleNamespace(name = "minimax-h3")+ assert (+ vid._video_comfy_backends(h3, "b", None, _plan(True), _plan(False))["fp8_storage"] is False+ )+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "0")+ assert (+ vid._video_comfy_backends(fam, "b", None, _plan(True), _plan(False))["fp8_storage"] is False+ )+++def test_a_smaller_offloaded_price_replans_and_keeps_the_offloaded_backends(monkeypatch):+ """fp8 stored under offload prices below the widened resident estimate: re-plan at it, and a plan that now fits+ resident keeps fp8 storage (else the load widens past the size it was planned at)."""+ fam = types.SimpleNamespace(name = "wan2.2-ti2v-5b")+ assert vid._video_offload_reprice(fam, 2000, 1000) == (1000, True)+ assert vid._video_offload_reprice(fam, 1000, 2000) == (2000, False)+ assert vid._video_offload_reprice(fam, 1000, 1000) == (1000, False)+ assert vid._video_offload_reprice(fam, None, 1000) == (None, False)+ h3 = types.SimpleNamespace(name = "minimax-h3")+ assert vid._video_offload_reprice(h3, 2000, 1000) == (2000, False)+ assert vid._video_offload_reprice(h3, 1000, 2000) == (2000, False)+ monkeypatch.setattr(vid, "plan_keeps_transformer_resident", lambda p: p.resident)+ monkeypatch.setattr(+ vid, "comfy_int8_backend", lambda t, f, b, *, offload = False: "native" if offload else None+ )+ monkeypatch.setattr(vid, "comfy_fp8_backend", lambda t, f, b, *, offload = False: None)+ resident = vid._video_comfy_backends(fam, "b", None, _plan(True), _plan(True))+ assert resident["fp8_storage"] is False and resident["int8_backend"] is None+ kept = vid._video_comfy_backends(fam, "b", None, _plan(True), _plan(True), priced_offload = True)+ assert kept == {"int8_backend": "native", "fp8_backend": None, "fp8_storage": True}def test_video_plan_size_prices_only_what_runs_quantized(monkeypatch, tmp_path):@@ -444,6 +478,12 @@ def test_video_plan_size_prices_only_what_runs_quantized(monkeypatch, tmp_path):monkeypatch.setattr(vid, "comfy_fp8_backend", lambda *a, **k: None)assert vid._video_comfy_resident_mib(fam, "b", None, path, scan) == 3assert vid._video_comfy_resident_mib(fam, "b", None, path, scan, keep = False) == 3+ # no fp8 GEMM under offload: the codes stay stored as fp8 (widened per layer), priced at their stored size+ assert vid._video_comfy_resident_mib(fam, "b", None, path, scan, offload = True) == 2+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "1")+ assert vid._video_comfy_resident_mib(fam, "b", None, path, scan) == 2+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "0")+ assert vid._video_comfy_resident_mib(fam, "b", None, path, scan, offload = True) == 3assert n == 2**20@@ -679,6 +719,8 @@ def test_offloaded_plan_size_prices_fp8_that_only_runs_resident(monkeypatch, tmpvid, "comfy_fp8_backend", lambda *a, offload = False, **k: None if offload else "torchao")assert vid._video_comfy_resident_mib(fam, "b", None, path, scan) == 2+ assert vid._video_comfy_resident_mib(fam, "b", None, path, scan, offload = True) == 2+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "0")assert vid._video_comfy_resident_mib(fam, "b", None, path, scan, offload = True) == 3@@ -973,3 +1015,96 @@ def test_a_full_width_tag_retry_that_does_not_fit_is_refused_before_converting(int8_backend = "native",)assert len(calls) == 1 # refused before the full-width conversion ran+++def test_a_resident_video_dit_that_fits_only_stored_gets_fp8_storage(tmp_path, monkeypatch):+ """A resident ComfyUI fp8 video DiT whose widened form would leave under the host reserve stores fp8, as an+ image single file does: free RAM between the stored and the widened need loads, and is refused when+ UNSLOTH_DIFFUSION_FP8_STORAGE=0 makes the load widen."""+ import core.inference.diffusion_single_file_converters as conv+ import core.inference.diffusion_small_host as small_host++ torch.manual_seed(4)+ dense = _H3Like()+ tensors = {"head.weight": dense.head.weight.detach(), "head.bias": dense.head.bias.detach()}+ quantized = {+ "attn.to_q": dense.attn.to_q.weight,+ "attn.to_k": dense.attn.to_k.weight,+ "attn.to_v": dense.attn.to_v.weight,+ "ff.proj": dense.ff.proj.weight,+ }+ for name, w in quantized.items():+ w = w.detach().float()+ scale = w.abs().max() / torch.finfo(torch.float8_e4m3fn).max+ tensors[f"{name}.weight"] = (w / scale).to(torch.float8_e4m3fn)+ tensors[f"{name}.weight_scale"] = scale.reshape(1)+ tensors[f"{name}.comfy_quant"] = _conf(format = "float8_e4m3fn")+ path = _save(tmp_path / "wan_fp8.safetensors", tensors)+ monkeypatch.setattr(+ cq,+ "_mapping",+ lambda cls: (+ lambda checkpoint = None, **_kw: checkpoint,+ types.SimpleNamespace(+ _should_convert_state_dict_to_diffusers = lambda a, b: False,+ _get_mapping_function_kwargs = lambda fn, **kw: {},+ ),+ ),+ )+ monkeypatch.setattr(vid, "plan_keeps_transformer_resident", lambda p: p.resident)+ monkeypatch.setattr(vid, "comfy_int8_backend", lambda *a, **k: None)+ monkeypatch.setattr(vid, "comfy_fp8_backend", lambda *a, **k: None)+ monkeypatch.delenv(cq.FP8_STORAGE_ENV, raising = False)+ # Price in bytes, no reserves: the tiny model's needs are a few hundred KB apart.+ monkeypatch.setattr(conv, "HOST_RAM_CHECK_MIN_MIB", 0)+ monkeypatch.delenv(small_host.HOST_RAM_CHECK_ENV, raising = False)+ monkeypatch.setattr(conv, "_MIB", 1)+ monkeypatch.setattr(small_host, "host_reserve_mib", lambda total: 0)+ monkeypatch.setattr(cq, "FP8_STORAGE_HOST_RESERVE_MIB", 0)+ keys = [n + ".weight" for n in quantized]+ widened = conv.converted_host_bytes(+ path, torch.bfloat16, dequantized = dict.fromkeys(keys, 1).get+ )+ stored = conv.converted_host_bytes(+ path, torch.bfloat16, dequantized = dict.fromkeys(keys, 0).get+ ) + 4 * sum(int(tensors[k].shape[0]) for k in keys)+ assert 0 < stored < widened+ available = (stored + widened) // 2+ monkeypatch.setattr(small_host, "host_ram_mib", lambda: (1 << 40, available))+ fam = types.SimpleNamespace(name = "wan2.2-ti2v-5b")++ def backends(**kw):+ return vid._video_comfy_backends(fam, "b", None, _plan(True), _plan(True), **kw)++ def load(**kw):+ return cq.load_comfy_quant_transformer(+ _H3Like,+ path,+ cq.refuse_comfy_quant(path),+ {"torch_dtype": torch.bfloat16, "config": "base/repo"},+ family = fam.name,+ **kw,+ )++ picked = backends(widened_mib = widened)+ assert picked == {"int8_backend": None, "fp8_backend": None, "fp8_storage": True}+ model = load(**picked)+ assert model._unsloth_comfy_quant["fp8_stored"] == 4+ assert model.attn.to_q.weight_q.dtype == torch.uint8+ # room to widen: the faster widened load; H3 keeps its torchao-or-dequantize rule+ monkeypatch.setattr(small_host, "host_ram_mib", lambda: (1 << 40, 4 * widened))+ assert backends(widened_mib = widened)["fp8_storage"] is False+ monkeypatch.setattr(small_host, "host_ram_mib", lambda: (1 << 40, available))+ h3 = types.SimpleNamespace(name = "minimax-h3")+ assert (+ vid._video_comfy_backends(h3, "b", None, _plan(True), _plan(True), widened_mib = widened)[+ "fp8_storage"+ ]+ is False+ )+ # the kill switch widens, and the host RAM check refuses that load+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "0")+ widened_pick = backends(widened_mib = widened)+ assert widened_pick["fp8_storage"] is False+ with pytest.raises(RuntimeError, match = "Not enough system RAM"):+ load(**widened_pick)