Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio MiniMax-H3: run int8 / fp8 denoisers where torchao's runtime does not (Ampere fp8, ROCm, MPS) (#17064)

Daniel Han1dcfcf27bd08
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 9e910c94be9..b0405f93acd 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -583,10 +583,11 @@ def _video_fp8_storage(
widen_chosen: bool = False,
) -> bool:
"""``fp8_storage_wanted`` for a video DiT, with the host RAM the load would widen into when ``widened_mib`` is
- known. H3 is excluded: its pin / stream / residency path is built on torchao weights. ``widen_chosen``: answer
- ``fp8_widen_chosen`` instead (the rule had the choice and widened)."""
+ known. H3 stores fp8 only where no fp8 runtime runs (``h3_comfy_backends``): widened it would be the 40 GB
+ bfloat16 model. ``widen_chosen``: answer ``fp8_widen_chosen`` instead (the rule had the choice and widened)."""
if getattr(fam, "name", None) == "minimax-h3":
- return False
+ from .video_minimax_h3_native_quant import h3_fp8_storage
+ return False if widen_chosen else h3_fp8_storage(target)
available = None
if widened_mib is not None and not offload:
from .diffusion_small_host import host_ram_mib
@@ -783,6 +784,15 @@ def _ltx23_prequant_serves_on_card(
return True
+def _h3_hosted_fp8_native(fam: Any, target: Any, scheme: Optional[str]) -> bool:
+ """An explicit fp8 H3 denoiser on a card without torchao's fp8 GEMM: the hosted codes run stored (1 byte)."""
+ if getattr(fam, "name", None) != "minimax-h3" or scheme != TQ_FP8:
+ return False
+ from .video_minimax_h3_native_quant import h3_hosted_fp8_native
+
+ return h3_hosted_fp8_native(target)
+
+
def _assert_video_precision_for_target(
fam: Any,
target: Any,
@@ -852,16 +862,13 @@ def _assert_video_precision_for_target(
f"'{normalize_memory_mode(memory_mode)}' memory places the DiT under CPU "
"offload, and torchao quantised tensors cannot be moved by the offload hooks"
)
- elif (
- select_transformer_quant_scheme(
- target,
- pinned,
- family = getattr(fam, "name", None),
- # Pre-eviction: an OOM in the smoke probe is the resident model, not the scheme.
- unproven_ok = True,
- )
- is None
- ):
+ elif select_transformer_quant_scheme(
+ target,
+ pinned,
+ family = getattr(fam, "name", None),
+ # Pre-eviction: an OOM in the smoke probe is the resident model, not the scheme.
+ unproven_ok = True,
+ ) is None and not _h3_hosted_fp8_native(fam, target, pinned):
# None means a deny-listed family, unusable torchao, or no GPU kernels; each needs a different fix.
reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
if reason is not None:
@@ -11374,24 +11381,21 @@ class VideoBackend:
if comfy_scan is not None:
from .video_minimax_h3_comfy import load_h3_comfy_transformer
- # torchao only: H3's pin / stream / residency path is built on torchao weights, like the hosted ones.
+ from .video_minimax_h3_native_quant import h3_comfy_backends
+
+ # torchao where it runs, else the torchao-free twin (fp8 stored at 1 byte where no fp8 GEMM runs).
dense_pin = _h3_precision_pinned_dense(transformer_quant_requested)
- h3_int8 = h3_fp8 = None
+ h3_backends = {"int8_backend": None, "fp8_backend": None, "fp8_storage": False}
if not dense_pin:
- h3_int8 = (
- "torchao"
- if comfy_int8_backend(umem_target, fam.name, base) == "torchao"
- else None
- )
- h3_fp8 = (
- "torchao"
- if comfy_fp8_backend(umem_target, fam.name, base) == "torchao"
- else None
- )
+ h3_backends = h3_comfy_backends(umem_target, base)
if (
comfy_scheme is not None
and not dense_pin
- and (h3_int8 if comfy_scheme == TQ_INT8 else h3_fp8) is None
+ and not (
+ h3_backends["int8_backend"]
+ if comfy_scheme == TQ_INT8
+ else h3_backends["fp8_backend"] or h3_backends["fp8_storage"]
+ )
):
raise RuntimeError(
f"{Path(comfy_checkpoint).name} needs Unsloth's {comfy_scheme} runtime, which this "
@@ -11408,14 +11412,17 @@ class VideoBackend:
hf_token = hf_token,
cache_dir = hub_cache_dir(),
local_files_only = local_files_only,
- int8_backend = h3_int8,
- fp8_backend = h3_fp8,
target = umem_target,
logger = logger,
+ **h3_backends,
)
comfy_info = getattr(transformer, "_unsloth_comfy_quant", None) or {}
engaged = (
- TQ_INT8 if comfy_info.get("int8") else TQ_FP8 if comfy_info.get("fp8") else None
+ TQ_INT8
+ if comfy_info.get("int8")
+ else TQ_FP8
+ if comfy_info.get("fp8") or comfy_info.get("fp8_stored")
+ else None
)
if engaged is None and comfy_scheme is not None and not dense_pin:
del transformer
@@ -11431,6 +11438,7 @@ class VideoBackend:
transformer_quant_reason = (
f"ComfyUI-quantized denoiser ({Path(comfy_checkpoint).name}): "
f"{comfy_info.get('int8', 0)} int8 / {comfy_info.get('fp8', 0)} fp8 layers kept, "
+ f"{comfy_info.get('fp8_stored', 0)} fp8 stored, "
f"{comfy_info.get('dequantized', 0)} dequantized"
if engaged is not None
else f"ComfyUI denoiser ({Path(comfy_checkpoint).name}) at bfloat16"
@@ -11485,6 +11493,9 @@ class VideoBackend:
+ "; loaded the released bfloat16 denoiser instead"
)
else:
+ if _h3_hosted_fp8_native(fam, umem_target, scheme):
+ from .video_minimax_h3_native_quant import h3_hosted_fp8_to_native
+ h3_hosted_fp8_to_native(transformer, logger = logger)
# Must seed before load_components, which skips set attributes (avoids the 66.3 GB fetch).
pipe.update_components(**{denoiser_component: transformer})
transformer_quant_engaged = scheme
diff --git a/studio/backend/core/inference/video_minimax_h3_comfy.py b/studio/backend/core/inference/video_minimax_h3_comfy.py
index c3e76656954..12f03324eec 100644
--- a/studio/backend/core/inference/video_minimax_h3_comfy.py
+++ b/studio/backend/core/inference/video_minimax_h3_comfy.py
@@ -188,7 +188,7 @@ def load_h3_comfy_transformer(
fp8_backend: Optional[str],
target: Any = None,
logger: Any = None,
- # H3's pin / stream / residency path is built on torchao weights: dequantize what torchao does not take
+ # fp8 layers no fp8 runtime takes stay 1-byte codes widened per layer (``h3_comfy_backends``); False dequantizes
fp8_storage: Optional[bool] = False,
) -> Any:
"""The H3 denoiser of a ComfyUI-quantized file, int8 / fp8 codes kept, curve adaLN installed."""
@@ -198,7 +198,10 @@ def load_h3_comfy_transformer(
metadata = h3_comfy_curve_metadata(path)
# Header only, before any tensor is read: a PDD trunk without its head bank is refused here.
heads = h3_pdd_heads_in_file(path)
- return load_comfy_quant_transformer(
+ prepare, fp32_buffers = _keep_fp32_init_buffers(
+ transformer_cls, _h3_prepare_model(metadata, heads, logger = logger)
+ )
+ model = load_comfy_quant_transformer(
transformer_cls,
path,
scan,
@@ -217,6 +220,44 @@ def load_h3_comfy_transformer(
target = target,
logger = logger,
key_map = h3_comfy_key_map,
- prepare_model = _h3_prepare_model(metadata, heads, logger = logger),
+ prepare_model = prepare,
keep_dtype = functools.partial(h3_comfy_keep_dtype, pruned = metadata is not None),
)
+ _restore_fp32_init_buffers(model, fp32_buffers)
+ return model
+
+
+def _keep_fp32_init_buffers(transformer_cls: Any, prepare: Any) -> tuple:
+ """``prepare_model`` that also snapshots the float32 ``__init__`` buffers of ``_keep_in_fp32_modules``: the loader
+ casts unloaded buffers to the compute dtype, and a bf16 rope ``inv_freq`` drifts the clip off the hosted one's."""
+ keep = getattr(transformer_cls, "_keep_in_fp32_modules", None) or []
+ if isinstance(keep, str):
+ keep = [keep]
+ saved: dict = {}
+
+ def hook(model: Any) -> None:
+ import torch
+
+ if prepare is not None:
+ prepare(model)
+ saved.clear()
+ for name, module in model.named_modules():
+ if not name or not any(k in name.split(".") for k in keep):
+ continue
+ for buffer_name, buffer in module._buffers.items():
+ if (
+ buffer is not None
+ and buffer.dtype == torch.float32
+ and buffer.device.type != "meta"
+ ):
+ saved[(name, buffer_name)] = buffer.detach().clone()
+
+ return hook, saved
+
+
+def _restore_fp32_init_buffers(model: Any, saved: dict) -> None:
+ for (name, buffer_name), value in saved.items():
+ module = model.get_submodule(name)
+ current = module._buffers.get(buffer_name)
+ if current is not None and current.dtype != value.dtype and current.shape == value.shape:
+ module._buffers[buffer_name] = value.to(current.device)
diff --git a/studio/backend/core/inference/video_minimax_h3_native_quant.py b/studio/backend/core/inference/video_minimax_h3_native_quant.py
new file mode 100644
index 00000000000..d47a69779f3
--- /dev/null
+++ b/studio/backend/core/inference/video_minimax_h3_native_quant.py
@@ -0,0 +1,151 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MiniMax-H3 int8 / fp8 denoisers where torchao's runtime does not run (Ampere fp8, ROCm, MPS, stubbed torchao):
+the file's codes go to the 1-byte ``NativeWeightOnlyLinear`` twin by the image path's rules instead of widening to
+the 40 GB bf16 model. ``UNSLOTH_H3_NATIVE_QUANT=0`` restores the torchao-only refusal."""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Optional
+
+H3_NATIVE_QUANT_ENV = "UNSLOTH_H3_NATIVE_QUANT"
+H3_FAMILY = "minimax-h3"
+# CPU never loads H3 quantized.
+_STORAGE_DEVICES = ("cuda", "mps")
+
+
+def h3_native_quant_enabled() -> bool:
+ raw = (os.environ.get(H3_NATIVE_QUANT_ENV) or "").strip().lower()
+ return raw not in ("0", "off", "false", "no")
+
+
+def _storage_device(target: Any) -> bool:
+ return getattr(target, "device", None) in _STORAGE_DEVICES
+
+
+def h3_comfy_backends(target: Any, base_repo: Optional[str] = None) -> dict:
+ """``load_comfy_quant_transformer`` backends for an H3 file: torchao, else the native twin, else (fp8) storage.
+ None / False = no runtime: the load refuses rather than dequantize to 40 GB bf16."""
+ from .diffusion_comfy_quant import comfy_fp8_backend, comfy_fp8_storage, comfy_int8_backend
+
+ int8 = comfy_int8_backend(target, H3_FAMILY, base_repo)
+ fp8 = comfy_fp8_backend(target, H3_FAMILY, base_repo)
+ if not h3_native_quant_enabled():
+ return {
+ "int8_backend": "torchao" if int8 == "torchao" else None,
+ "fp8_backend": "torchao" if fp8 == "torchao" else None,
+ "fp8_storage": False,
+ }
+ return {
+ "int8_backend": int8,
+ "fp8_backend": fp8,
+ "fp8_storage": fp8 is None and _storage_device(target) and comfy_fp8_storage(target),
+ }
+
+
+def h3_fp8_storage(target: Any, base_repo: Optional[str] = None) -> bool:
+ """The planner's view of ``h3_comfy_backends``: stored fp8 is priced at 1 byte."""
+ try:
+ return bool(h3_comfy_backends(target, base_repo)["fp8_storage"])
+ except Exception: # noqa: BLE001 -- unanswerable: priced widened, as before
+ return False
+
+
+def h3_hosted_fp8_native(target: Any, base_repo: Optional[str] = None) -> bool:
+ """Hosted fp8 checkpoint as stored fp8: a CUDA card where torchao's fp8 runtime does not run (below sm_89)."""
+ if not h3_native_quant_enabled() or getattr(target, "device", None) != "cuda":
+ return False
+ try:
+ import torch
+
+ from .diffusion_comfy_quant import comfy_fp8_storage
+ from .diffusion_transformer_quant import TQ_FP8, select_transformer_quant_scheme
+
+ if getattr(target, "dtype", None) is not torch.bfloat16 or not comfy_fp8_storage(target):
+ return False
+ return (
+ select_transformer_quant_scheme(target, TQ_FP8, family = H3_FAMILY, base_repo = base_repo)
+ != TQ_FP8
+ )
+ except Exception: # noqa: BLE001 -- unanswerable: the torchao rule decides, as before
+ return False
+
+
+def _fp8_parts(weight: Any) -> Optional[tuple]:
+ """``(codes, per-row float32 scale)`` of a per-row / per-tensor ``Float8Tensor``, else None."""
+ import torch
+
+ if type(weight).__name__ != "Float8Tensor":
+ return None
+ codes = getattr(weight, "qdata", None)
+ scale = getattr(weight, "scale", None)
+ if codes is None or scale is None or codes.dim() != 2:
+ return None
+ if codes.dtype not in (torch.float8_e4m3fn, torch.float8_e5m2):
+ return None
+ rows = int(codes.shape[0])
+ if scale.numel() == 1:
+ scale = scale.reshape(1).expand(rows)
+ elif scale.numel() == rows:
+ scale = scale.reshape(rows)
+ else:
+ return None # block-wise scales
+ return codes, scale.to(torch.float32).contiguous()
+
+
+def h3_hosted_fp8_to_native(model: Any, logger: Any = None) -> int:
+ """Rebuild every ``Float8Tensor`` Linear as a stored-fp8 twin (same codes and per-row scales). Raises
+ ``ValueError`` on a layout the twin cannot hold: it could not run here."""
+ import torch
+
+ from .diffusion_comfy_quant import FP8_STORED_ATTR
+ from .diffusion_native_quant import NATIVE_FP8, native_linear_class
+
+ cls = native_linear_class()
+ built = 0
+ for name, module in list(model.named_modules()):
+ if not isinstance(module, torch.nn.Linear):
+ continue
+ weight = module.weight
+ if type(weight).__name__ != "Float8Tensor":
+ continue
+ parts = _fp8_parts(weight)
+ if parts is None:
+ raise ValueError(f"{name}: a Float8Tensor layout the stored-fp8 runtime cannot hold")
+ codes, scale = parts
+ dtype = getattr(weight, "dtype", None) or torch.bfloat16
+ stand_in = torch.nn.Linear(
+ int(module.in_features),
+ int(module.out_features),
+ bias = False,
+ device = "meta",
+ dtype = dtype,
+ )
+ stand_in.bias = module.bias
+ layer = cls(stand_in, NATIVE_FP8, codes = codes, scale = scale)
+ parent_name, _, leaf = name.rpartition(".")
+ parent = model.get_submodule(parent_name) if parent_name else model
+ setattr(parent, leaf, layer)
+ built += 1
+ left = [
+ n
+ for n, m in model.named_modules()
+ if type(getattr(m, "weight", None)).__name__ == "Float8Tensor"
+ ]
+ if left:
+ raise ValueError(f"{left[0]}: Float8Tensor weights the stored-fp8 runtime did not take")
+ if built:
+ try:
+ setattr(model, FP8_STORED_ATTR, built)
+ model._unsloth_runtime_quant_info = {"scheme": "fp8", "native": True, "storage": True}
+ except Exception: # noqa: BLE001 -- markers are best-effort
+ pass
+ if logger is not None:
+ logger.info(
+ "video.h3_native_quant: %d fp8 layers of the hosted checkpoint stored as fp8 and widened per "
+ "layer (this card has no fp8 GEMM)",
+ built,
+ )
+ return built
diff --git a/studio/backend/tests/test_h3_native_quant.py b/studio/backend/tests/test_h3_native_quant.py
new file mode 100644
index 00000000000..caeb3d741e1
--- /dev/null
+++ b/studio/backend/tests/test_h3_native_quant.py
@@ -0,0 +1,442 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MiniMax-H3 int8 / fp8 files where torchao does not run: routing per host, the 1-byte twins, hosted fp8."""
+
+from __future__ import annotations
+
+import json
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+safetensors_torch = pytest.importorskip("safetensors.torch")
+diffusers = pytest.importorskip("diffusers")
+if not hasattr(diffusers, "MiniMaxH3Transformer3DModel"):
+ pytest.skip("diffusers has no MiniMaxH3Transformer3DModel", allow_module_level = True)
+
+from core.inference import diffusion_comfy_quant as cq # noqa: E402
+from core.inference import diffusion_transformer_quant as dtq # noqa: E402
+from core.inference import video as vid # noqa: E402
+from core.inference import video_minimax_h3_native_quant as nq # noqa: E402
+from core.inference.diffusion_native_quant import native_linear_class # noqa: E402
+from core.inference.video_minimax_h3_comfy import _RENAMES, load_h3_comfy_transformer # noqa: E402
+
+# Wide enough that Studio's int8 filter (512 features) keeps the block Linears.
+TINY = dict(
+ num_attention_heads = 4,
+ attention_head_dim = 128,
+ hidden_size = 512,
+ num_layers = 2,
+ num_refiner_layers = 1,
+ ffn_dim = 1024,
+ in_channels = 24,
+ audio_in_channels = 32,
+ text_dim = 16,
+ freq_dim = 32,
+ time_embed_hidden_dim = 32,
+ time_embed_dim = 16,
+ rope_freq_dim = 2,
+)
+FP8_NAME = "minimax_h3_fl2va_pruned_fp8_scaled.safetensors"
+INT8_NAME = "minimax_h3_fl2va_pruned_int8.safetensors"
+H3 = types.SimpleNamespace(name = "minimax-h3")
+
+
[email protected](autouse = True)
+def _switch_default(monkeypatch):
+ monkeypatch.delenv(nq.H3_NATIVE_QUANT_ENV, raising = False)
+ monkeypatch.delenv(cq.FP8_STORAGE_ENV, raising = False)
+ monkeypatch.delenv(cq.COMFY_FP8_ENV, raising = False)
+ monkeypatch.delenv(cq.COMFY_INT8_ENV, raising = False)
+
+
+def _target(device = "cuda"):
+ return types.SimpleNamespace(device = device, dtype = torch.bfloat16)
+
+
+def _host(monkeypatch, kind):
+ """Studio's host predicates as ``kind`` answers them."""
+ rocm = kind == "rocm"
+ monkeypatch.setattr(dtq, "torch_is_rocm", lambda: rocm)
+ monkeypatch.setattr(dtq, "is_stubbed", lambda name: False)
+ runs = {
+ "sm89": {dtq.TQ_INT8, dtq.TQ_FP8},
+ "sm80": {dtq.TQ_INT8}, # no fp8 GEMM below sm_89
+ "rocm": set(),
+ "mps": set(),
+ }[kind]
+
+ def select(
+ target,
+ requested,
+ family = None,
+ **_k,
+ ):
+ scheme = dtq.normalize_transformer_quant(requested)
+ return scheme if scheme in runs else None
+
+ monkeypatch.setattr(dtq, "select_transformer_quant_scheme", select)
+ return _target("mps" if kind == "mps" else "cuda")
+
+
+# ── routing per host ──
+
+
+ "kind,int8,fp8,storage",
+ [
+ ("sm89", "torchao", "torchao", False), # unchanged: torchao for both
+ ("sm80", "torchao", None, True), # A100 / RTX 3090: fp8 stored at 1 byte
+ ("rocm", "native", "native", False), # the torchao-free twin for both
+ ("mps", "native", None, True), # Metal int8; fp8 codes stored (no fp8 cast on MPS)
+ ],
+)
+def test_h3_backends_per_host(monkeypatch, kind, int8, fp8, storage):
+ target = _host(monkeypatch, kind)
+ got = nq.h3_comfy_backends(target)
+ assert got == {"int8_backend": int8, "fp8_backend": fp8, "fp8_storage": storage}
+ assert vid._video_fp8_storage(H3, target, False) is storage
+ assert vid._video_fp8_storage(H3, target, True) is storage
+ assert vid._video_fp8_storage(H3, target, False, widen_chosen = True) is False
+
+
[email protected]("kind", ["sm89", "sm80", "rocm", "mps"])
+def test_kill_switch_restores_torchao_only(monkeypatch, kind):
+ target = _host(monkeypatch, kind)
+ monkeypatch.setenv(nq.H3_NATIVE_QUANT_ENV, "0")
+ got = nq.h3_comfy_backends(target)
+ want = "torchao" if kind in ("sm89", "sm80") else None
+ assert got["int8_backend"] == want
+ assert got["fp8_backend"] == ("torchao" if kind == "sm89" else None)
+ assert got["fp8_storage"] is False
+ assert vid._video_fp8_storage(H3, target, False) is False
+
+
+def test_storage_off_switch_and_cpu_never_store(monkeypatch):
+ target = _host(monkeypatch, "sm80")
+ monkeypatch.setenv(cq.FP8_STORAGE_ENV, "0")
+ assert nq.h3_comfy_backends(target)["fp8_storage"] is False
+ monkeypatch.delenv(cq.FP8_STORAGE_ENV)
+ assert nq.h3_comfy_backends(_target("cpu"))["fp8_storage"] is False
+ assert nq.h3_fp8_storage(None) is False
+
+
[email protected]("kind,native", [("sm89", False), ("sm80", True), ("rocm", False)])
+def test_hosted_fp8_native_only_where_torchao_fp8_does_not_run_on_cuda(monkeypatch, kind, native):
+ target = _host(monkeypatch, kind)
+ # ROCm keeps torchao's own answer: this rule is for CUDA cards without an fp8 GEMM.
+ if kind == "rocm":
+ monkeypatch.setattr(dtq, "select_transformer_quant_scheme", lambda *a, **k: dtq.TQ_FP8)
+ assert nq.h3_hosted_fp8_native(target) is native
+ assert vid._h3_hosted_fp8_native(H3, target, "fp8") is native
+ assert vid._h3_hosted_fp8_native(H3, target, "int8") is False
+ assert vid._h3_hosted_fp8_native(types.SimpleNamespace(name = "ltx-2"), target, "fp8") is False
+ monkeypatch.setenv(nq.H3_NATIVE_QUANT_ENV, "0")
+ assert nq.h3_hosted_fp8_native(target) is False
+
+
+# ── a tiny H3 single file through the real loader ──
+
+
+def _tiny(seed = 0):
+ torch.manual_seed(seed)
+ model = diffusers.MiniMaxH3Transformer3DModel(**TINY)
+ with torch.no_grad():
+ for p in model.parameters():
+ p.copy_(torch.randn_like(p) * 0.05)
+ return model.eval()
+
+
+def _comfy_key(name):
+ if name.startswith("transformer_blocks."):
+ name = "blocks." + name[len("transformer_blocks.") :]
+ elif name.startswith("token_refiner.refiner_blocks."):
+ name = "token_refiner.blocks." + name[len("token_refiner.refiner_blocks.") :]
+ for old, new in reversed(_RENAMES):
+ if new in name and (not new.startswith(("proj_", "audio_proj")) or name.startswith(new)):
+ name = name.replace(new, old, 1)
+ break
+ return name
+
+
+def _quant(w, fmt):
+ w = w.float()
+ if fmt == "int8_tensorwise":
+ scale = w.abs().amax().clamp_min(1e-8) / 127.0
+ return torch.round(w / scale).clamp(-127, 127).to(torch.int8), scale.reshape(())
+ scale = w.abs().amax().clamp_min(1e-8) / 448.0
+ return (w / scale).to(torch.float8_e4m3fn), scale.reshape(())
+
+
+def _write(tmp_path, model, name, fmt):
+ """An original-layout H3 file (fused qkv, [gate; value] fc1), block Linears as ``fmt`` codes."""
+ sd = {k: v.detach().clone() for k, v in model.state_dict().items()}
+ out = {}
+ for key in list(sd):
+ if ".attn.to_q.weight" in key:
+ stem = key[: -len("to_q.weight")]
+ fused = torch.cat([sd.pop(stem + p + ".weight") for p in ("to_q", "to_k", "to_v")])
+ out[_comfy_key(stem) + "qkv_proj.weight"] = fused
+ for key, value in sd.items():
+ if key.endswith(".ff.net.0.proj.weight"):
+ value_half, gate = value.chunk(2)
+ out[_comfy_key(key.replace(".ff.net.0.proj.weight", ".mlp.fc1.weight"))] = torch.cat(
+ [gate, value_half]
+ )
+ continue
+ out[_comfy_key(key)] = value
+ quantized = [
+ k
+ for k in out
+ if k.startswith(("blocks.", "token_refiner.blocks."))
+ and k.endswith(("qkv_proj.weight", "out_proj.weight", "fc1.weight", "fc2.weight"))
+ ]
+ for key in quantized:
+ codes, scale = _quant(out[key], fmt)
+ out[key] = codes
+ out[key[: -len(".weight")] + ".weight_scale"] = scale
+ out[key[: -len(".weight")] + ".comfy_quant"] = torch.tensor(
+ list(json.dumps({"format": fmt}).encode()), dtype = torch.uint8
+ )
+ path = tmp_path / name
+ safetensors_torch.save_file({k: v.contiguous() for k, v in out.items()}, str(path))
+ cfg = tmp_path / "base" / "transformer"
+ cfg.mkdir(parents = True, exist_ok = True)
+ (cfg / "config.json").write_text(
+ json.dumps({"_class_name": "MiniMaxH3Transformer3DModel", **TINY})
+ )
+ return str(path), str(tmp_path / "base"), len(quantized)
+
+
+def _load(
+ path,
+ base,
+ target,
+ dtype = torch.float32,
+ **backends,
+):
+ return load_h3_comfy_transformer(
+ diffusers.MiniMaxH3Transformer3DModel,
+ path,
+ cq.refuse_comfy_quant(path),
+ config = base,
+ subfolder = "transformer",
+ dtype = dtype,
+ hf_token = None,
+ cache_dir = None,
+ local_files_only = True,
+ target = target,
+ **backends,
+ )
+
+
+def _inputs(
+ seed = 0,
+ n_text = 4,
+ n_video = 6,
+ n_audio = 3,
+):
+ g = torch.Generator().manual_seed(seed)
+ s = n_text + n_video + n_audio
+ text = torch.arange(n_text)
+ video = torch.arange(n_text, n_text + n_video)
+ audio = torch.arange(n_text + n_video, s)
+ tags = torch.zeros(s, dtype = torch.long)
+ tags[text], tags[audio] = 1, 2
+ row_t = torch.full((s,), 0.6)
+ row_t[audio] = 0.4
+ uniq, inv = torch.unique(row_t, sorted = True, return_inverse = True)
+ pos = torch.stack(
+ [torch.arange(s), torch.zeros(s, dtype = torch.long), torch.arange(s) % 3], 1
+ ).float()
+ return dict(
+ hidden_states = torch.randn(1, n_video, 96, generator = g),
+ audio_hidden_states = torch.randn(1, n_audio, 32, generator = g),
+ encoder_hidden_states = torch.randn(1, n_text, 16, generator = g),
+ timestep = uniq,
+ timestep_indices = inv,
+ token_tags = tags,
+ position_ids = pos,
+ video_indices = video,
+ audio_indices = audio,
+ text_indices = text,
+ return_dict = False,
+ )
+
+
+def _forward(model):
+ with torch.no_grad():
+ out = model(**_inputs())
+ return torch.cat([t.float().reshape(-1) for t in out if torch.is_tensor(t)])
+
+
+def _twins(model):
+ cls = native_linear_class()
+ return [m for m in model.modules() if isinstance(m, cls)]
+
+
+def test_fp8_file_on_ampere_loads_stored_fp8_and_matches_the_dequantized_model(
+ tmp_path, monkeypatch
+):
+ path, base, n_quant = _write(tmp_path, _tiny(), FP8_NAME, "float8_e4m3fn")
+ target = _host(monkeypatch, "sm80")
+ backends = nq.h3_comfy_backends(target)
+ model = _load(path, base, target, **backends)
+ info = model._unsloth_comfy_quant
+ # qkv splits into three layers
+ assert info["fp8_stored"] == n_quant + 2 * TINY["num_layers"] + 2 * TINY["num_refiner_layers"]
+ assert info["dequantized"] == 0
+ twins = _twins(model)
+ assert len(twins) == info["fp8_stored"]
+ assert all(t.weight_q.element_size() == 1 and t.scheme == "fp8" for t in twins)
+ reference = _load(path, base, target, int8_backend = None, fp8_backend = None, fp8_storage = False)
+ assert not _twins(reference)
+ torch.testing.assert_close(_forward(model), _forward(reference), rtol = 0, atol = 0)
+
+
+def test_fp8_stored_bytes_are_half_the_widened_bf16_bytes(tmp_path, monkeypatch):
+ path, base, _ = _write(tmp_path, _tiny(), FP8_NAME, "float8_e4m3fn")
+ target = _host(monkeypatch, "sm80")
+ stored = _load(path, base, target, dtype = torch.bfloat16, **nq.h3_comfy_backends(target))
+ widened = _load(path, base, target, dtype = torch.bfloat16, int8_backend = None, fp8_backend = None)
+
+ def block_bytes(model):
+ total = 0
+ for name, t in list(model.named_parameters()) + list(model.named_buffers()):
+ if name.startswith("transformer_blocks."):
+ total += t.numel() * t.element_size()
+ return total
+
+ assert block_bytes(stored) < 0.55 * block_bytes(widened)
+ scan = cq.refuse_comfy_quant(path)
+ priced = cq.comfy_resident_mib(path, scan, keep_int8 = False, keep_fp8 = False, fp8_stored = True)
+ widened_mib = cq.comfy_resident_mib(
+ path, scan, keep_int8 = False, keep_fp8 = False, fp8_stored = False
+ )
+ assert priced is not None and widened_mib is not None and priced <= widened_mib
+
+
+def test_a_single_file_keeps_the_fp32_rope_the_hosted_checkpoint_keeps(tmp_path, monkeypatch):
+ path, base, _ = _write(tmp_path, _tiny(), FP8_NAME, "float8_e4m3fn")
+ target = _host(monkeypatch, "sm80")
+ model = _load(path, base, target, dtype = torch.bfloat16, **nq.h3_comfy_backends(target))
+ fresh = diffusers.MiniMaxH3Transformer3DModel(**TINY)
+ assert model.rope.inv_freq.dtype == torch.float32
+ assert torch.equal(model.rope.inv_freq, fresh.rope.inv_freq)
+
+
[email protected]("kind", ["rocm", "mps"])
+def test_int8_file_on_native_hosts_keeps_int8_codes(tmp_path, monkeypatch, kind):
+ path, base, _ = _write(tmp_path, _tiny(), INT8_NAME, "int8_tensorwise")
+ target = _host(monkeypatch, kind)
+ backends = nq.h3_comfy_backends(target)
+ assert backends["int8_backend"] == "native"
+ model = _load(path, base, target, **backends)
+ info = model._unsloth_comfy_quant
+ assert info["int8"] > 0 and info["backend"] == "native"
+ twins = _twins(model)
+ assert twins and all(t.weight_q.dtype == torch.int8 for t in twins)
+ reference = _load(path, base, target, int8_backend = None, fp8_backend = None)
+ torch.testing.assert_close(_forward(model), _forward(reference), rtol = 1e-5, atol = 1e-5)
+
+
+def test_fp8_file_on_rocm_keeps_fp8_codes(tmp_path, monkeypatch):
+ path, base, _ = _write(tmp_path, _tiny(), FP8_NAME, "float8_e4m3fn")
+ target = _host(monkeypatch, "rocm")
+ # Studio's fp8 runtimes assert bf16 weights (an fp32 load dequantizes them)
+ model = _load(path, base, target, dtype = torch.bfloat16, **nq.h3_comfy_backends(target))
+ info = model._unsloth_comfy_quant
+ assert info["fp8"] > 0 and info["fp8_backend"] == "native" and info["dequantized"] == 0
+ reference = _load(path, base, target, dtype = torch.bfloat16, int8_backend = None, fp8_backend = None)
+ torch.testing.assert_close(_forward(model), _forward(reference), rtol = 0, atol = 0)
+
+
+# ── the hosted torchao fp8 checkpoint ──
+
+
+def test_hosted_float8tensor_layers_become_stored_fp8_with_identical_output():
+ quant = pytest.importorskip("torchao.quantization")
+ model = _tiny().to(torch.bfloat16)
+ for name, module in model.named_modules():
+ if (
+ isinstance(module, torch.nn.Linear)
+ and name.startswith("transformer_blocks.")
+ and module.in_features >= 512
+ ):
+ weight = quant.Float8Tensor.from_hp(
+ module.weight.detach(), float8_dtype = torch.float8_e4m3fn, granularity = quant.PerRow()
+ )
+ module.weight = torch.nn.Parameter(weight, requires_grad = False)
+ fp8 = {
+ n: m.weight
+ for n, m in model.named_modules()
+ if type(getattr(m, "weight", None)).__name__ == "Float8Tensor"
+ }
+ if not fp8:
+ pytest.skip("torchao produced no Float8Tensor weights")
+ reference = {n: w.dequantize().to(torch.bfloat16) for n, w in fp8.items()}
+ built = nq.h3_hosted_fp8_to_native(model)
+ assert built == len(fp8)
+ modules = dict(model.named_modules())
+ for name, want in reference.items():
+ layer = modules[name]
+ assert layer.weight_q.element_size() == 1
+ torch.testing.assert_close(layer.weight, want, rtol = 0, atol = 0)
+ assert getattr(model, cq.FP8_STORED_ATTR) == built
+ _forward(model) # runs with no torchao fp8 GEMM
+
+
+def test_hosted_conversion_refuses_a_layout_it_cannot_hold():
+ class Float8Tensor: # a block-scaled stand-in: neither per-row nor per-tensor
+ qdata = torch.zeros(8, 16).to(torch.float8_e4m3fn)
+ scale = torch.ones(8, 2)
+ dtype = torch.bfloat16
+
+ lin = torch.nn.Linear(16, 8)
+ lin._parameters["weight"] = Float8Tensor()
+ with pytest.raises(ValueError, match = "cannot hold"):
+ nq.h3_hosted_fp8_to_native(torch.nn.Sequential(lin))
+
+
+# ── on this host's real GPU (CUDA / ROCm / MPS): the backends it picks, run there ──
+
+
+def _gpu_target():
+ from core.inference.diffusion_device import resolve_diffusion_device_target
+
+ target = resolve_diffusion_device_target()
+ if getattr(target, "device", None) not in ("cuda", "mps"):
+ pytest.skip("no GPU here")
+ return target
+
+
[email protected]("fmt,name", [("float8_e4m3fn", FP8_NAME), ("int8_tensorwise", INT8_NAME)])
+def test_the_host_backends_run_on_its_gpu_like_the_dequantized_model(tmp_path, fmt, name):
+ target = _gpu_target()
+ device = target.torch_device
+ path, base, _ = _write(tmp_path, _tiny(), name, fmt)
+ backends = nq.h3_comfy_backends(target)
+ scheme_runs = (
+ backends["int8_backend"]
+ if fmt.startswith("int8")
+ else (backends["fp8_backend"] or backends["fp8_storage"])
+ )
+ if not scheme_runs:
+ pytest.skip(f"{fmt} has no runtime on this host: {backends}")
+ model = _load(path, base, target, dtype = torch.bfloat16, **backends).to(device)
+ info = model._unsloth_comfy_quant
+ kept = info["int8"] if fmt.startswith("int8") else info["fp8"] + info.get("fp8_stored", 0)
+ assert kept > 0 and info["dequantized"] == 0, info
+ reference = _load(
+ path, base, target, dtype = torch.bfloat16, int8_backend = None, fp8_backend = None
+ ).to(device)
+ kw = {k: (v.to(device) if torch.is_tensor(v) else v) for k, v in _inputs().items()}
+ with torch.no_grad():
+ got = torch.cat([t.float().reshape(-1) for t in model(**kw) if torch.is_tensor(t)])
+ want = torch.cat([t.float().reshape(-1) for t in reference(**kw) if torch.is_tensor(t)])
+ assert torch.isfinite(got).all()
+ # W8A8 int8 adds activation rounding
+ rel = (got - want).norm() / want.norm().clamp_min(1e-12)
+ assert rel < (5e-2 if fmt.startswith("int8") else 1e-2), (backends, float(rel))