Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio MiniMax-H3: run int8 / fp8 denoisers where torchao's runtime does not (Ampere fp8, ROCm, MPS) (#17064)
Daniel Han1dcfcf27bd08
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 9e910c94be9..b0405f93acd 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -583,10 +583,11 @@ def _video_fp8_storage(widen_chosen: bool = False,) -> bool:"""``fp8_storage_wanted`` for a video DiT, with the host RAM the load would widen into when ``widened_mib`` is- known. H3 is excluded: its pin / stream / residency path is built on torchao weights. ``widen_chosen``: answer- ``fp8_widen_chosen`` instead (the rule had the choice and widened)."""+ known. H3 stores fp8 only where no fp8 runtime runs (``h3_comfy_backends``): widened it would be the 40 GB+ bfloat16 model. ``widen_chosen``: answer ``fp8_widen_chosen`` instead (the rule had the choice and widened)."""if getattr(fam, "name", None) == "minimax-h3":- return False+ from .video_minimax_h3_native_quant import h3_fp8_storage+ return False if widen_chosen else h3_fp8_storage(target)available = Noneif widened_mib is not None and not offload:from .diffusion_small_host import host_ram_mib@@ -783,6 +784,15 @@ def _ltx23_prequant_serves_on_card(return True+def _h3_hosted_fp8_native(fam: Any, target: Any, scheme: Optional[str]) -> bool:+ """An explicit fp8 H3 denoiser on a card without torchao's fp8 GEMM: the hosted codes run stored (1 byte)."""+ if getattr(fam, "name", None) != "minimax-h3" or scheme != TQ_FP8:+ return False+ from .video_minimax_h3_native_quant import h3_hosted_fp8_native++ return h3_hosted_fp8_native(target)++def _assert_video_precision_for_target(fam: Any,target: Any,@@ -852,16 +862,13 @@ def _assert_video_precision_for_target(f"'{normalize_memory_mode(memory_mode)}' memory places the DiT under CPU ""offload, and torchao quantised tensors cannot be moved by the offload hooks")- elif (- select_transformer_quant_scheme(- target,- pinned,- family = getattr(fam, "name", None),- # Pre-eviction: an OOM in the smoke probe is the resident model, not the scheme.- unproven_ok = True,- )- is None- ):+ elif select_transformer_quant_scheme(+ target,+ pinned,+ family = getattr(fam, "name", None),+ # Pre-eviction: an OOM in the smoke probe is the resident model, not the scheme.+ unproven_ok = True,+ ) is None and not _h3_hosted_fp8_native(fam, target, pinned):# None means a deny-listed family, unusable torchao, or no GPU kernels; each needs a different fix.reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)if reason is not None:@@ -11374,24 +11381,21 @@ class VideoBackend:if comfy_scan is not None:from .video_minimax_h3_comfy import load_h3_comfy_transformer- # torchao only: H3's pin / stream / residency path is built on torchao weights, like the hosted ones.+ from .video_minimax_h3_native_quant import h3_comfy_backends++ # torchao where it runs, else the torchao-free twin (fp8 stored at 1 byte where no fp8 GEMM runs).dense_pin = _h3_precision_pinned_dense(transformer_quant_requested)- h3_int8 = h3_fp8 = None+ h3_backends = {"int8_backend": None, "fp8_backend": None, "fp8_storage": False}if not dense_pin:- h3_int8 = (- "torchao"- if comfy_int8_backend(umem_target, fam.name, base) == "torchao"- else None- )- h3_fp8 = (- "torchao"- if comfy_fp8_backend(umem_target, fam.name, base) == "torchao"- else None- )+ h3_backends = h3_comfy_backends(umem_target, base)if (comfy_scheme is not Noneand not dense_pin- and (h3_int8 if comfy_scheme == TQ_INT8 else h3_fp8) is None+ and not (+ h3_backends["int8_backend"]+ if comfy_scheme == TQ_INT8+ else h3_backends["fp8_backend"] or h3_backends["fp8_storage"]+ )):raise RuntimeError(f"{Path(comfy_checkpoint).name} needs Unsloth's {comfy_scheme} runtime, which this "@@ -11408,14 +11412,17 @@ class VideoBackend:hf_token = hf_token,cache_dir = hub_cache_dir(),local_files_only = local_files_only,- int8_backend = h3_int8,- fp8_backend = h3_fp8,target = umem_target,logger = logger,+ **h3_backends,)comfy_info = getattr(transformer, "_unsloth_comfy_quant", None) or {}engaged = (- TQ_INT8 if comfy_info.get("int8") else TQ_FP8 if comfy_info.get("fp8") else None+ TQ_INT8+ if comfy_info.get("int8")+ else TQ_FP8+ if comfy_info.get("fp8") or comfy_info.get("fp8_stored")+ else None)if engaged is None and comfy_scheme is not None and not dense_pin:del transformer@@ -11431,6 +11438,7 @@ class VideoBackend:transformer_quant_reason = (f"ComfyUI-quantized denoiser ({Path(comfy_checkpoint).name}): "f"{comfy_info.get('int8', 0)} int8 / {comfy_info.get('fp8', 0)} fp8 layers kept, "+ f"{comfy_info.get('fp8_stored', 0)} fp8 stored, "f"{comfy_info.get('dequantized', 0)} dequantized"if engaged is not Noneelse f"ComfyUI denoiser ({Path(comfy_checkpoint).name}) at bfloat16"@@ -11485,6 +11493,9 @@ class VideoBackend:+ "; loaded the released bfloat16 denoiser instead")else:+ if _h3_hosted_fp8_native(fam, umem_target, scheme):+ from .video_minimax_h3_native_quant import h3_hosted_fp8_to_native+ h3_hosted_fp8_to_native(transformer, logger = logger)# Must seed before load_components, which skips set attributes (avoids the 66.3 GB fetch).pipe.update_components(**{denoiser_component: transformer})transformer_quant_engaged = schemediff --git a/studio/backend/core/inference/video_minimax_h3_comfy.py b/studio/backend/core/inference/video_minimax_h3_comfy.pyindex c3e76656954..12f03324eec 100644--- a/studio/backend/core/inference/video_minimax_h3_comfy.py+++ b/studio/backend/core/inference/video_minimax_h3_comfy.py@@ -188,7 +188,7 @@ def load_h3_comfy_transformer(fp8_backend: Optional[str],target: Any = None,logger: Any = None,- # H3's pin / stream / residency path is built on torchao weights: dequantize what torchao does not take+ # fp8 layers no fp8 runtime takes stay 1-byte codes widened per layer (``h3_comfy_backends``); False dequantizesfp8_storage: Optional[bool] = False,) -> Any:"""The H3 denoiser of a ComfyUI-quantized file, int8 / fp8 codes kept, curve adaLN installed."""@@ -198,7 +198,10 @@ def load_h3_comfy_transformer(metadata = h3_comfy_curve_metadata(path)# Header only, before any tensor is read: a PDD trunk without its head bank is refused here.heads = h3_pdd_heads_in_file(path)- return load_comfy_quant_transformer(+ prepare, fp32_buffers = _keep_fp32_init_buffers(+ transformer_cls, _h3_prepare_model(metadata, heads, logger = logger)+ )+ model = load_comfy_quant_transformer(transformer_cls,path,scan,@@ -217,6 +220,44 @@ def load_h3_comfy_transformer(target = target,logger = logger,key_map = h3_comfy_key_map,- prepare_model = _h3_prepare_model(metadata, heads, logger = logger),+ prepare_model = prepare,keep_dtype = functools.partial(h3_comfy_keep_dtype, pruned = metadata is not None),)+ _restore_fp32_init_buffers(model, fp32_buffers)+ return model+++def _keep_fp32_init_buffers(transformer_cls: Any, prepare: Any) -> tuple:+ """``prepare_model`` that also snapshots the float32 ``__init__`` buffers of ``_keep_in_fp32_modules``: the loader+ casts unloaded buffers to the compute dtype, and a bf16 rope ``inv_freq`` drifts the clip off the hosted one's."""+ keep = getattr(transformer_cls, "_keep_in_fp32_modules", None) or []+ if isinstance(keep, str):+ keep = [keep]+ saved: dict = {}++ def hook(model: Any) -> None:+ import torch++ if prepare is not None:+ prepare(model)+ saved.clear()+ for name, module in model.named_modules():+ if not name or not any(k in name.split(".") for k in keep):+ continue+ for buffer_name, buffer in module._buffers.items():+ if (+ buffer is not None+ and buffer.dtype == torch.float32+ and buffer.device.type != "meta"+ ):+ saved[(name, buffer_name)] = buffer.detach().clone()++ return hook, saved+++def _restore_fp32_init_buffers(model: Any, saved: dict) -> None:+ for (name, buffer_name), value in saved.items():+ module = model.get_submodule(name)+ current = module._buffers.get(buffer_name)+ if current is not None and current.dtype != value.dtype and current.shape == value.shape:+ module._buffers[buffer_name] = value.to(current.device)diff --git a/studio/backend/core/inference/video_minimax_h3_native_quant.py b/studio/backend/core/inference/video_minimax_h3_native_quant.pynew file mode 100644index 00000000000..d47a69779f3--- /dev/null+++ b/studio/backend/core/inference/video_minimax_h3_native_quant.py@@ -0,0 +1,151 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""MiniMax-H3 int8 / fp8 denoisers where torchao's runtime does not run (Ampere fp8, ROCm, MPS, stubbed torchao):+the file's codes go to the 1-byte ``NativeWeightOnlyLinear`` twin by the image path's rules instead of widening to+the 40 GB bf16 model. ``UNSLOTH_H3_NATIVE_QUANT=0`` restores the torchao-only refusal."""++from __future__ import annotations++import os+from typing import Any, Optional++H3_NATIVE_QUANT_ENV = "UNSLOTH_H3_NATIVE_QUANT"+H3_FAMILY = "minimax-h3"+# CPU never loads H3 quantized.+_STORAGE_DEVICES = ("cuda", "mps")+++def h3_native_quant_enabled() -> bool:+ raw = (os.environ.get(H3_NATIVE_QUANT_ENV) or "").strip().lower()+ return raw not in ("0", "off", "false", "no")+++def _storage_device(target: Any) -> bool:+ return getattr(target, "device", None) in _STORAGE_DEVICES+++def h3_comfy_backends(target: Any, base_repo: Optional[str] = None) -> dict:+ """``load_comfy_quant_transformer`` backends for an H3 file: torchao, else the native twin, else (fp8) storage.+ None / False = no runtime: the load refuses rather than dequantize to 40 GB bf16."""+ from .diffusion_comfy_quant import comfy_fp8_backend, comfy_fp8_storage, comfy_int8_backend++ int8 = comfy_int8_backend(target, H3_FAMILY, base_repo)+ fp8 = comfy_fp8_backend(target, H3_FAMILY, base_repo)+ if not h3_native_quant_enabled():+ return {+ "int8_backend": "torchao" if int8 == "torchao" else None,+ "fp8_backend": "torchao" if fp8 == "torchao" else None,+ "fp8_storage": False,+ }+ return {+ "int8_backend": int8,+ "fp8_backend": fp8,+ "fp8_storage": fp8 is None and _storage_device(target) and comfy_fp8_storage(target),+ }+++def h3_fp8_storage(target: Any, base_repo: Optional[str] = None) -> bool:+ """The planner's view of ``h3_comfy_backends``: stored fp8 is priced at 1 byte."""+ try:+ return bool(h3_comfy_backends(target, base_repo)["fp8_storage"])+ except Exception: # noqa: BLE001 -- unanswerable: priced widened, as before+ return False+++def h3_hosted_fp8_native(target: Any, base_repo: Optional[str] = None) -> bool:+ """Hosted fp8 checkpoint as stored fp8: a CUDA card where torchao's fp8 runtime does not run (below sm_89)."""+ if not h3_native_quant_enabled() or getattr(target, "device", None) != "cuda":+ return False+ try:+ import torch++ from .diffusion_comfy_quant import comfy_fp8_storage+ from .diffusion_transformer_quant import TQ_FP8, select_transformer_quant_scheme++ if getattr(target, "dtype", None) is not torch.bfloat16 or not comfy_fp8_storage(target):+ return False+ return (+ select_transformer_quant_scheme(target, TQ_FP8, family = H3_FAMILY, base_repo = base_repo)+ != TQ_FP8+ )+ except Exception: # noqa: BLE001 -- unanswerable: the torchao rule decides, as before+ return False+++def _fp8_parts(weight: Any) -> Optional[tuple]:+ """``(codes, per-row float32 scale)`` of a per-row / per-tensor ``Float8Tensor``, else None."""+ import torch++ if type(weight).__name__ != "Float8Tensor":+ return None+ codes = getattr(weight, "qdata", None)+ scale = getattr(weight, "scale", None)+ if codes is None or scale is None or codes.dim() != 2:+ return None+ if codes.dtype not in (torch.float8_e4m3fn, torch.float8_e5m2):+ return None+ rows = int(codes.shape[0])+ if scale.numel() == 1:+ scale = scale.reshape(1).expand(rows)+ elif scale.numel() == rows:+ scale = scale.reshape(rows)+ else:+ return None # block-wise scales+ return codes, scale.to(torch.float32).contiguous()+++def h3_hosted_fp8_to_native(model: Any, logger: Any = None) -> int:+ """Rebuild every ``Float8Tensor`` Linear as a stored-fp8 twin (same codes and per-row scales). Raises+ ``ValueError`` on a layout the twin cannot hold: it could not run here."""+ import torch++ from .diffusion_comfy_quant import FP8_STORED_ATTR+ from .diffusion_native_quant import NATIVE_FP8, native_linear_class++ cls = native_linear_class()+ built = 0+ for name, module in list(model.named_modules()):+ if not isinstance(module, torch.nn.Linear):+ continue+ weight = module.weight+ if type(weight).__name__ != "Float8Tensor":+ continue+ parts = _fp8_parts(weight)+ if parts is None:+ raise ValueError(f"{name}: a Float8Tensor layout the stored-fp8 runtime cannot hold")+ codes, scale = parts+ dtype = getattr(weight, "dtype", None) or torch.bfloat16+ stand_in = torch.nn.Linear(+ int(module.in_features),+ int(module.out_features),+ bias = False,+ device = "meta",+ dtype = dtype,+ )+ stand_in.bias = module.bias+ layer = cls(stand_in, NATIVE_FP8, codes = codes, scale = scale)+ parent_name, _, leaf = name.rpartition(".")+ parent = model.get_submodule(parent_name) if parent_name else model+ setattr(parent, leaf, layer)+ built += 1+ left = [+ n+ for n, m in model.named_modules()+ if type(getattr(m, "weight", None)).__name__ == "Float8Tensor"+ ]+ if left:+ raise ValueError(f"{left[0]}: Float8Tensor weights the stored-fp8 runtime did not take")+ if built:+ try:+ setattr(model, FP8_STORED_ATTR, built)+ model._unsloth_runtime_quant_info = {"scheme": "fp8", "native": True, "storage": True}+ except Exception: # noqa: BLE001 -- markers are best-effort+ pass+ if logger is not None:+ logger.info(+ "video.h3_native_quant: %d fp8 layers of the hosted checkpoint stored as fp8 and widened per "+ "layer (this card has no fp8 GEMM)",+ built,+ )+ return builtdiff --git a/studio/backend/tests/test_h3_native_quant.py b/studio/backend/tests/test_h3_native_quant.pynew file mode 100644index 00000000000..caeb3d741e1--- /dev/null+++ b/studio/backend/tests/test_h3_native_quant.py@@ -0,0 +1,442 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""MiniMax-H3 int8 / fp8 files where torchao does not run: routing per host, the 1-byte twins, hosted fp8."""++from __future__ import annotations++import json+import types++import pytest++torch = pytest.importorskip("torch")+safetensors_torch = pytest.importorskip("safetensors.torch")+diffusers = pytest.importorskip("diffusers")+if not hasattr(diffusers, "MiniMaxH3Transformer3DModel"):+ pytest.skip("diffusers has no MiniMaxH3Transformer3DModel", allow_module_level = True)++from core.inference import diffusion_comfy_quant as cq # noqa: E402+from core.inference import diffusion_transformer_quant as dtq # noqa: E402+from core.inference import video as vid # noqa: E402+from core.inference import video_minimax_h3_native_quant as nq # noqa: E402+from core.inference.diffusion_native_quant import native_linear_class # noqa: E402+from core.inference.video_minimax_h3_comfy import _RENAMES, load_h3_comfy_transformer # noqa: E402++# Wide enough that Studio's int8 filter (512 features) keeps the block Linears.+TINY = dict(+ num_attention_heads = 4,+ attention_head_dim = 128,+ hidden_size = 512,+ num_layers = 2,+ num_refiner_layers = 1,+ ffn_dim = 1024,+ in_channels = 24,+ audio_in_channels = 32,+ text_dim = 16,+ freq_dim = 32,+ time_embed_hidden_dim = 32,+ time_embed_dim = 16,+ rope_freq_dim = 2,+)+FP8_NAME = "minimax_h3_fl2va_pruned_fp8_scaled.safetensors"+INT8_NAME = "minimax_h3_fl2va_pruned_int8.safetensors"+H3 = types.SimpleNamespace(name = "minimax-h3")++[email protected](autouse = True)+def _switch_default(monkeypatch):+ monkeypatch.delenv(nq.H3_NATIVE_QUANT_ENV, raising = False)+ monkeypatch.delenv(cq.FP8_STORAGE_ENV, raising = False)+ monkeypatch.delenv(cq.COMFY_FP8_ENV, raising = False)+ monkeypatch.delenv(cq.COMFY_INT8_ENV, raising = False)+++def _target(device = "cuda"):+ return types.SimpleNamespace(device = device, dtype = torch.bfloat16)+++def _host(monkeypatch, kind):+ """Studio's host predicates as ``kind`` answers them."""+ rocm = kind == "rocm"+ monkeypatch.setattr(dtq, "torch_is_rocm", lambda: rocm)+ monkeypatch.setattr(dtq, "is_stubbed", lambda name: False)+ runs = {+ "sm89": {dtq.TQ_INT8, dtq.TQ_FP8},+ "sm80": {dtq.TQ_INT8}, # no fp8 GEMM below sm_89+ "rocm": set(),+ "mps": set(),+ }[kind]++ def select(+ target,+ requested,+ family = None,+ **_k,+ ):+ scheme = dtq.normalize_transformer_quant(requested)+ return scheme if scheme in runs else None++ monkeypatch.setattr(dtq, "select_transformer_quant_scheme", select)+ return _target("mps" if kind == "mps" else "cuda")+++# ── routing per host ──+++ "kind,int8,fp8,storage",+ [+ ("sm89", "torchao", "torchao", False), # unchanged: torchao for both+ ("sm80", "torchao", None, True), # A100 / RTX 3090: fp8 stored at 1 byte+ ("rocm", "native", "native", False), # the torchao-free twin for both+ ("mps", "native", None, True), # Metal int8; fp8 codes stored (no fp8 cast on MPS)+ ],+)+def test_h3_backends_per_host(monkeypatch, kind, int8, fp8, storage):+ target = _host(monkeypatch, kind)+ got = nq.h3_comfy_backends(target)+ assert got == {"int8_backend": int8, "fp8_backend": fp8, "fp8_storage": storage}+ assert vid._video_fp8_storage(H3, target, False) is storage+ assert vid._video_fp8_storage(H3, target, True) is storage+ assert vid._video_fp8_storage(H3, target, False, widen_chosen = True) is False++[email protected]("kind", ["sm89", "sm80", "rocm", "mps"])+def test_kill_switch_restores_torchao_only(monkeypatch, kind):+ target = _host(monkeypatch, kind)+ monkeypatch.setenv(nq.H3_NATIVE_QUANT_ENV, "0")+ got = nq.h3_comfy_backends(target)+ want = "torchao" if kind in ("sm89", "sm80") else None+ assert got["int8_backend"] == want+ assert got["fp8_backend"] == ("torchao" if kind == "sm89" else None)+ assert got["fp8_storage"] is False+ assert vid._video_fp8_storage(H3, target, False) is False+++def test_storage_off_switch_and_cpu_never_store(monkeypatch):+ target = _host(monkeypatch, "sm80")+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "0")+ assert nq.h3_comfy_backends(target)["fp8_storage"] is False+ monkeypatch.delenv(cq.FP8_STORAGE_ENV)+ assert nq.h3_comfy_backends(_target("cpu"))["fp8_storage"] is False+ assert nq.h3_fp8_storage(None) is False++[email protected]("kind,native", [("sm89", False), ("sm80", True), ("rocm", False)])+def test_hosted_fp8_native_only_where_torchao_fp8_does_not_run_on_cuda(monkeypatch, kind, native):+ target = _host(monkeypatch, kind)+ # ROCm keeps torchao's own answer: this rule is for CUDA cards without an fp8 GEMM.+ if kind == "rocm":+ monkeypatch.setattr(dtq, "select_transformer_quant_scheme", lambda *a, **k: dtq.TQ_FP8)+ assert nq.h3_hosted_fp8_native(target) is native+ assert vid._h3_hosted_fp8_native(H3, target, "fp8") is native+ assert vid._h3_hosted_fp8_native(H3, target, "int8") is False+ assert vid._h3_hosted_fp8_native(types.SimpleNamespace(name = "ltx-2"), target, "fp8") is False+ monkeypatch.setenv(nq.H3_NATIVE_QUANT_ENV, "0")+ assert nq.h3_hosted_fp8_native(target) is False+++# ── a tiny H3 single file through the real loader ──+++def _tiny(seed = 0):+ torch.manual_seed(seed)+ model = diffusers.MiniMaxH3Transformer3DModel(**TINY)+ with torch.no_grad():+ for p in model.parameters():+ p.copy_(torch.randn_like(p) * 0.05)+ return model.eval()+++def _comfy_key(name):+ if name.startswith("transformer_blocks."):+ name = "blocks." + name[len("transformer_blocks.") :]+ elif name.startswith("token_refiner.refiner_blocks."):+ name = "token_refiner.blocks." + name[len("token_refiner.refiner_blocks.") :]+ for old, new in reversed(_RENAMES):+ if new in name and (not new.startswith(("proj_", "audio_proj")) or name.startswith(new)):+ name = name.replace(new, old, 1)+ break+ return name+++def _quant(w, fmt):+ w = w.float()+ if fmt == "int8_tensorwise":+ scale = w.abs().amax().clamp_min(1e-8) / 127.0+ return torch.round(w / scale).clamp(-127, 127).to(torch.int8), scale.reshape(())+ scale = w.abs().amax().clamp_min(1e-8) / 448.0+ return (w / scale).to(torch.float8_e4m3fn), scale.reshape(())+++def _write(tmp_path, model, name, fmt):+ """An original-layout H3 file (fused qkv, [gate; value] fc1), block Linears as ``fmt`` codes."""+ sd = {k: v.detach().clone() for k, v in model.state_dict().items()}+ out = {}+ for key in list(sd):+ if ".attn.to_q.weight" in key:+ stem = key[: -len("to_q.weight")]+ fused = torch.cat([sd.pop(stem + p + ".weight") for p in ("to_q", "to_k", "to_v")])+ out[_comfy_key(stem) + "qkv_proj.weight"] = fused+ for key, value in sd.items():+ if key.endswith(".ff.net.0.proj.weight"):+ value_half, gate = value.chunk(2)+ out[_comfy_key(key.replace(".ff.net.0.proj.weight", ".mlp.fc1.weight"))] = torch.cat(+ [gate, value_half]+ )+ continue+ out[_comfy_key(key)] = value+ quantized = [+ k+ for k in out+ if k.startswith(("blocks.", "token_refiner.blocks."))+ and k.endswith(("qkv_proj.weight", "out_proj.weight", "fc1.weight", "fc2.weight"))+ ]+ for key in quantized:+ codes, scale = _quant(out[key], fmt)+ out[key] = codes+ out[key[: -len(".weight")] + ".weight_scale"] = scale+ out[key[: -len(".weight")] + ".comfy_quant"] = torch.tensor(+ list(json.dumps({"format": fmt}).encode()), dtype = torch.uint8+ )+ path = tmp_path / name+ safetensors_torch.save_file({k: v.contiguous() for k, v in out.items()}, str(path))+ cfg = tmp_path / "base" / "transformer"+ cfg.mkdir(parents = True, exist_ok = True)+ (cfg / "config.json").write_text(+ json.dumps({"_class_name": "MiniMaxH3Transformer3DModel", **TINY})+ )+ return str(path), str(tmp_path / "base"), len(quantized)+++def _load(+ path,+ base,+ target,+ dtype = torch.float32,+ **backends,+):+ return load_h3_comfy_transformer(+ diffusers.MiniMaxH3Transformer3DModel,+ path,+ cq.refuse_comfy_quant(path),+ config = base,+ subfolder = "transformer",+ dtype = dtype,+ hf_token = None,+ cache_dir = None,+ local_files_only = True,+ target = target,+ **backends,+ )+++def _inputs(+ seed = 0,+ n_text = 4,+ n_video = 6,+ n_audio = 3,+):+ g = torch.Generator().manual_seed(seed)+ s = n_text + n_video + n_audio+ text = torch.arange(n_text)+ video = torch.arange(n_text, n_text + n_video)+ audio = torch.arange(n_text + n_video, s)+ tags = torch.zeros(s, dtype = torch.long)+ tags[text], tags[audio] = 1, 2+ row_t = torch.full((s,), 0.6)+ row_t[audio] = 0.4+ uniq, inv = torch.unique(row_t, sorted = True, return_inverse = True)+ pos = torch.stack(+ [torch.arange(s), torch.zeros(s, dtype = torch.long), torch.arange(s) % 3], 1+ ).float()+ return dict(+ hidden_states = torch.randn(1, n_video, 96, generator = g),+ audio_hidden_states = torch.randn(1, n_audio, 32, generator = g),+ encoder_hidden_states = torch.randn(1, n_text, 16, generator = g),+ timestep = uniq,+ timestep_indices = inv,+ token_tags = tags,+ position_ids = pos,+ video_indices = video,+ audio_indices = audio,+ text_indices = text,+ return_dict = False,+ )+++def _forward(model):+ with torch.no_grad():+ out = model(**_inputs())+ return torch.cat([t.float().reshape(-1) for t in out if torch.is_tensor(t)])+++def _twins(model):+ cls = native_linear_class()+ return [m for m in model.modules() if isinstance(m, cls)]+++def test_fp8_file_on_ampere_loads_stored_fp8_and_matches_the_dequantized_model(+ tmp_path, monkeypatch+):+ path, base, n_quant = _write(tmp_path, _tiny(), FP8_NAME, "float8_e4m3fn")+ target = _host(monkeypatch, "sm80")+ backends = nq.h3_comfy_backends(target)+ model = _load(path, base, target, **backends)+ info = model._unsloth_comfy_quant+ # qkv splits into three layers+ assert info["fp8_stored"] == n_quant + 2 * TINY["num_layers"] + 2 * TINY["num_refiner_layers"]+ assert info["dequantized"] == 0+ twins = _twins(model)+ assert len(twins) == info["fp8_stored"]+ assert all(t.weight_q.element_size() == 1 and t.scheme == "fp8" for t in twins)+ reference = _load(path, base, target, int8_backend = None, fp8_backend = None, fp8_storage = False)+ assert not _twins(reference)+ torch.testing.assert_close(_forward(model), _forward(reference), rtol = 0, atol = 0)+++def test_fp8_stored_bytes_are_half_the_widened_bf16_bytes(tmp_path, monkeypatch):+ path, base, _ = _write(tmp_path, _tiny(), FP8_NAME, "float8_e4m3fn")+ target = _host(monkeypatch, "sm80")+ stored = _load(path, base, target, dtype = torch.bfloat16, **nq.h3_comfy_backends(target))+ widened = _load(path, base, target, dtype = torch.bfloat16, int8_backend = None, fp8_backend = None)++ def block_bytes(model):+ total = 0+ for name, t in list(model.named_parameters()) + list(model.named_buffers()):+ if name.startswith("transformer_blocks."):+ total += t.numel() * t.element_size()+ return total++ assert block_bytes(stored) < 0.55 * block_bytes(widened)+ scan = cq.refuse_comfy_quant(path)+ priced = cq.comfy_resident_mib(path, scan, keep_int8 = False, keep_fp8 = False, fp8_stored = True)+ widened_mib = cq.comfy_resident_mib(+ path, scan, keep_int8 = False, keep_fp8 = False, fp8_stored = False+ )+ assert priced is not None and widened_mib is not None and priced <= widened_mib+++def test_a_single_file_keeps_the_fp32_rope_the_hosted_checkpoint_keeps(tmp_path, monkeypatch):+ path, base, _ = _write(tmp_path, _tiny(), FP8_NAME, "float8_e4m3fn")+ target = _host(monkeypatch, "sm80")+ model = _load(path, base, target, dtype = torch.bfloat16, **nq.h3_comfy_backends(target))+ fresh = diffusers.MiniMaxH3Transformer3DModel(**TINY)+ assert model.rope.inv_freq.dtype == torch.float32+ assert torch.equal(model.rope.inv_freq, fresh.rope.inv_freq)++[email protected]("kind", ["rocm", "mps"])+def test_int8_file_on_native_hosts_keeps_int8_codes(tmp_path, monkeypatch, kind):+ path, base, _ = _write(tmp_path, _tiny(), INT8_NAME, "int8_tensorwise")+ target = _host(monkeypatch, kind)+ backends = nq.h3_comfy_backends(target)+ assert backends["int8_backend"] == "native"+ model = _load(path, base, target, **backends)+ info = model._unsloth_comfy_quant+ assert info["int8"] > 0 and info["backend"] == "native"+ twins = _twins(model)+ assert twins and all(t.weight_q.dtype == torch.int8 for t in twins)+ reference = _load(path, base, target, int8_backend = None, fp8_backend = None)+ torch.testing.assert_close(_forward(model), _forward(reference), rtol = 1e-5, atol = 1e-5)+++def test_fp8_file_on_rocm_keeps_fp8_codes(tmp_path, monkeypatch):+ path, base, _ = _write(tmp_path, _tiny(), FP8_NAME, "float8_e4m3fn")+ target = _host(monkeypatch, "rocm")+ # Studio's fp8 runtimes assert bf16 weights (an fp32 load dequantizes them)+ model = _load(path, base, target, dtype = torch.bfloat16, **nq.h3_comfy_backends(target))+ info = model._unsloth_comfy_quant+ assert info["fp8"] > 0 and info["fp8_backend"] == "native" and info["dequantized"] == 0+ reference = _load(path, base, target, dtype = torch.bfloat16, int8_backend = None, fp8_backend = None)+ torch.testing.assert_close(_forward(model), _forward(reference), rtol = 0, atol = 0)+++# ── the hosted torchao fp8 checkpoint ──+++def test_hosted_float8tensor_layers_become_stored_fp8_with_identical_output():+ quant = pytest.importorskip("torchao.quantization")+ model = _tiny().to(torch.bfloat16)+ for name, module in model.named_modules():+ if (+ isinstance(module, torch.nn.Linear)+ and name.startswith("transformer_blocks.")+ and module.in_features >= 512+ ):+ weight = quant.Float8Tensor.from_hp(+ module.weight.detach(), float8_dtype = torch.float8_e4m3fn, granularity = quant.PerRow()+ )+ module.weight = torch.nn.Parameter(weight, requires_grad = False)+ fp8 = {+ n: m.weight+ for n, m in model.named_modules()+ if type(getattr(m, "weight", None)).__name__ == "Float8Tensor"+ }+ if not fp8:+ pytest.skip("torchao produced no Float8Tensor weights")+ reference = {n: w.dequantize().to(torch.bfloat16) for n, w in fp8.items()}+ built = nq.h3_hosted_fp8_to_native(model)+ assert built == len(fp8)+ modules = dict(model.named_modules())+ for name, want in reference.items():+ layer = modules[name]+ assert layer.weight_q.element_size() == 1+ torch.testing.assert_close(layer.weight, want, rtol = 0, atol = 0)+ assert getattr(model, cq.FP8_STORED_ATTR) == built+ _forward(model) # runs with no torchao fp8 GEMM+++def test_hosted_conversion_refuses_a_layout_it_cannot_hold():+ class Float8Tensor: # a block-scaled stand-in: neither per-row nor per-tensor+ qdata = torch.zeros(8, 16).to(torch.float8_e4m3fn)+ scale = torch.ones(8, 2)+ dtype = torch.bfloat16++ lin = torch.nn.Linear(16, 8)+ lin._parameters["weight"] = Float8Tensor()+ with pytest.raises(ValueError, match = "cannot hold"):+ nq.h3_hosted_fp8_to_native(torch.nn.Sequential(lin))+++# ── on this host's real GPU (CUDA / ROCm / MPS): the backends it picks, run there ──+++def _gpu_target():+ from core.inference.diffusion_device import resolve_diffusion_device_target++ target = resolve_diffusion_device_target()+ if getattr(target, "device", None) not in ("cuda", "mps"):+ pytest.skip("no GPU here")+ return target++[email protected]("fmt,name", [("float8_e4m3fn", FP8_NAME), ("int8_tensorwise", INT8_NAME)])+def test_the_host_backends_run_on_its_gpu_like_the_dequantized_model(tmp_path, fmt, name):+ target = _gpu_target()+ device = target.torch_device+ path, base, _ = _write(tmp_path, _tiny(), name, fmt)+ backends = nq.h3_comfy_backends(target)+ scheme_runs = (+ backends["int8_backend"]+ if fmt.startswith("int8")+ else (backends["fp8_backend"] or backends["fp8_storage"])+ )+ if not scheme_runs:+ pytest.skip(f"{fmt} has no runtime on this host: {backends}")+ model = _load(path, base, target, dtype = torch.bfloat16, **backends).to(device)+ info = model._unsloth_comfy_quant+ kept = info["int8"] if fmt.startswith("int8") else info["fp8"] + info.get("fp8_stored", 0)+ assert kept > 0 and info["dequantized"] == 0, info+ reference = _load(+ path, base, target, dtype = torch.bfloat16, int8_backend = None, fp8_backend = None+ ).to(device)+ kw = {k: (v.to(device) if torch.is_tensor(v) else v) for k, v in _inputs().items()}+ with torch.no_grad():+ got = torch.cat([t.float().reshape(-1) for t in model(**kw) if torch.is_tensor(t)])+ want = torch.cat([t.float().reshape(-1) for t in reference(**kw) if torch.is_tensor(t)])+ assert torch.isfinite(got).all()+ # W8A8 int8 adds activation rounding+ rel = (got - want).norm() / want.norm().clamp_min(1e-12)+ assert rel < (5e-2 if fmt.startswith("int8") else 1e-2), (backends, float(rel))