Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio MiniMax-H3: opt-in 4-bit GGUF text encoder for hosts that cannot keep int8 resident (#17479)

Daniel Han568e79e2f9f0
diff --git a/studio/backend/core/inference/diffusion_weight_only_quant.py b/studio/backend/core/inference/diffusion_weight_only_quant.py
new file mode 100644
index 00000000000..98d03338c9f
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_weight_only_quant.py
@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Weight-only GGUF layers for text encoders: blocks stay resident, dequantized per call. Imports torch lazily."""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+
+def gguf_tensor(reader_tensor: Any) -> tuple[Any, Optional[int]]:
+ """``(tensor, quant_type or None)``: quantized as a raw ``[rows, bytes]`` uint8 copy, floats reshaped ``[rows, cols]``."""
+ import gguf
+ import numpy as np
+ import torch
+
+ qtype = reader_tensor.tensor_type
+ data = reader_tensor.data
+ shape = tuple(int(x) for x in reversed(list(reader_tensor.shape)))
+ if qtype == gguf.GGMLQuantizationType.BF16:
+ raw = torch.from_numpy(np.ascontiguousarray(data).view(np.int16).copy()).view(
+ torch.bfloat16
+ )
+ return raw.reshape(shape), None
+ if qtype in (gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16):
+ return torch.from_numpy(np.ascontiguousarray(data).copy()).reshape(shape), None
+ return torch.from_numpy(np.ascontiguousarray(data).copy()), int(qtype)
+
+
+def gguf_weight_only_linear(
+ raw: Any, quant_type: int, *, in_features: int, out_features: int, compute_dtype: Any
+) -> Any:
+ """A ``GGUFLinear`` over ``raw``; raises unless it holds exactly ``out x in`` weights."""
+ import gguf
+ import torch
+ from diffusers.quantizers.gguf.utils import GGUFLinear, GGUFParameter
+
+ block, type_size = gguf.GGML_QUANT_SIZES[gguf.GGMLQuantizationType(quant_type)]
+ if in_features % block or tuple(raw.shape) != (out_features, in_features // block * type_size):
+ raise ValueError(
+ f"GGUF {gguf.GGMLQuantizationType(quant_type).name} payload {tuple(raw.shape)} does not hold a "
+ f"{out_features}x{in_features} weight"
+ )
+ layer = GGUFLinear(
+ in_features, out_features, bias = False, compute_dtype = compute_dtype, device = "meta"
+ )
+ layer.weight = GGUFParameter(raw, quant_type = gguf.GGMLQuantizationType(quant_type))
+ layer.device = None
+ layer.requires_grad_(False)
+ return layer
+
+
+def is_dense_linear(module: Any) -> bool:
+ import torch
+ return (
+ isinstance(module, torch.nn.Linear) and getattr(module.weight, "quant_type", None) is None
+ )
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index d490cfc965b..dfa1f89ae20 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -327,9 +327,9 @@ from .video_minimax_h3 import (
h3_transformer_task,
)
from .video_minimax_h3_te import (
- H3_TE_QUANT_DEFAULT,
- H3_TE_QUANT_REPO,
+ h3_te_auto_scheme,
h3_te_quant_scheme,
+ h3_te_quant_source,
h3_te_resident_gb,
)
from .video_encode import encode_x264
@@ -6411,7 +6411,9 @@ class VideoBackend:
# under the assembly, while the base snapshot that no longer carries a dense encoder is still coming
# down). Same registration the native H3 path makes for its companions.
if h3_te_scheme:
- self._loading.asset_repos = self._loading.asset_repos + (H3_TE_QUANT_REPO,)
+ self._loading.asset_repos = self._loading.asset_repos + (
+ h3_te_quant_source(h3_te_scheme),
+ )
# The same for a hosted pre-cast encoder (unsloth/Wan2.2-TI2V-5B-FP8 for every Wan base).
te_repos = tuple(
src.location
@@ -7539,10 +7541,10 @@ class VideoBackend:
if _h3_precision_pinned_dense(text_encoder_quant):
return None
if _h3_precision_unset(text_encoder_quant):
+ auto = h3_te_auto_scheme()
return (
- H3_TE_QUANT_DEFAULT
- if h3_te_quant_scheme(H3_TE_QUANT_DEFAULT) is not None
- and _h3_auto_precision_ok(target)
+ auto
+ if h3_te_quant_scheme(auto) is not None and _h3_auto_precision_ok(target)
else None
)
return h3_te_quant_scheme(normalize_te_quant(text_encoder_quant))
@@ -7928,10 +7930,9 @@ class VideoBackend:
filename = h3_te_quant_filename(scheme)
if filename is None:
return None, []
- # Cached check uses the fetch repo; size comes from the mirror, as the repack may be gone.
repo = h3_te_quant_source(scheme)
try:
- info = api.model_info(H3_TE_QUANT_REPO, files_metadata = True)
+ info = api.model_info(repo, files_metadata = True)
except Exception: # noqa: BLE001 -- an unavailable artifact keeps the dense encoder
return None, []
files = [
@@ -12124,7 +12125,7 @@ class VideoBackend:
pipe.update_components(text_encoder = text_encoder)
text_encoder_quant_engaged = te_scheme
text_encoder_quant_reason = (
- f"hosted quantized {te_scheme} conditioner ({H3_TE_QUANT_REPO})"
+ f"hosted quantized {te_scheme} conditioner ({h3_te_quant_source(te_scheme)})"
)
logger.info(
"video.text_encoder_quant: %s quantized conditioner seeded for %s",
diff --git a/studio/backend/core/inference/video_minimax_h3_te.py b/studio/backend/core/inference/video_minimax_h3_te.py
index 3e977fe1a40..6dab889c26e 100644
--- a/studio/backend/core/inference/video_minimax_h3_te.py
+++ b/studio/backend/core/inference/video_minimax_h3_te.py
@@ -20,25 +20,39 @@ from .diffusion_convrot import ( # noqa: F401 (re-export: callers and tests na
# Not H3_COMPONENT_REPO: the VAEs moved to the GGUF mirror, the conditioner did not.
H3_TE_QUANT_REPO = "unsloth/MiniMax-H3-FP8"
-# Hosted quantized conditioners, by ``text_encoder_quant`` scheme. nvfp4
-# (``qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors``, 14.61 GiB) is deliberately absent: it is an AWQ NVFP4 layout with
-# two levels of scale and a per-tensor global scale, a different loader and a different kernel, and shipping it needs
-# its own numerical verification. The table is the one place to add it.
+H3_TE_INT8 = "int8"
+# Hosted Q4_K_M GGUF: 18.2 GB vs 27.2, but 5.8% hidden_states[50] error vs bf16 against int8's 1.6%.
+H3_TE_Q4_K_M = "q4_k_m"
H3_TE_QUANT_FILES: dict[str, str] = {
# Built from MiniMaxAI text_encoder/ with quantize_convrot_int8_weight; reads the official path
# only.
- "int8": "text_encoders/official/qwen3vl_32b_minimax_h3_int8_convrot.safetensors",
+ H3_TE_INT8: "text_encoders/official/qwen3vl_32b_minimax_h3_int8_convrot.safetensors",
+ H3_TE_Q4_K_M: "qwen3vl_32b_minimax_h3-Q4_K_M.gguf",
}
+H3_TE_QUANT_SOURCES: dict[str, str] = {
+ H3_TE_INT8: H3_TE_QUANT_REPO,
+ H3_TE_Q4_K_M: "unsloth/MiniMax-H3-GGUF",
+}
+# Never a request value: an explicit "nvfp4" keeps meaning the W4A4 runtime cast (refused for H3).
+H3_TE_AUTO_ONLY_SCHEMES = frozenset({H3_TE_Q4_K_M})
# Default int8: 27 GB over 50 layers vs 66.7 GB dense 64-layer; transformers has no early exit.
# Not an INT8 GEMM: ConvRot dequantizes then F.linear; the win is bytes moved, not math.
-H3_TE_QUANT_DEFAULT = "int8"
+H3_TE_QUANT_DEFAULT = H3_TE_INT8
-# From Hub safetensors headers; the load is storage-faithful, so file size == resident size.
H3_TE_QUANT_RESIDENT_GB: dict[str, float] = {
- "int8": 27.2,
+ H3_TE_INT8: 27.2,
+ H3_TE_Q4_K_M: 18.3,
}
+H3_TE_QUANT_AUTO_ENV = "UNSLOTH_H3_TE_QUANT_AUTO"
+# Conditioner + denoiser + VAEs + headroom; below it int8 is re-read every prompt (L4: 47-53 s vs 19.7 s).
+H3_TE_INT8_HOST_SET_GB = 27.2 + 20.3 + 11.1 + 5.9
+H3_TE_LOW_RAM_SCHEME = H3_TE_Q4_K_M
+# Off until the q4_k_m accuracy trade (LPIPS 0.08-0.30 vs int8) is accepted.
+H3_TE_LOW_RAM_AUTO = False
+
+
# Must equal MiniMaxH3ModularPipeline.text_encoder_layer.
H3_TE_READ_LAYER = 50
@@ -61,16 +75,48 @@ def h3_te_quant_scheme(mode: Optional[str]) -> Optional[str]:
return normalized if normalized in H3_TE_QUANT_FILES else None
+def h3_te_auto_scheme(
+ *, host_capacity_gb: Optional[float] = None, denoiser_gb: Optional[float] = None
+) -> str:
+ """Scheme for an unset request. Uses total (cgroup-capped) RAM, not free RAM, so plan, staging and assembly agree."""
+ import os
+
+ forced = h3_te_quant_scheme(os.environ.get(H3_TE_QUANT_AUTO_ENV))
+ if forced is not None:
+ return forced
+ if not H3_TE_LOW_RAM_AUTO:
+ return H3_TE_INT8
+ try:
+ if host_capacity_gb is None:
+ from utils.host_memory import host_ram_capacity_mib
+
+ mib = host_ram_capacity_mib()
+ if mib is None:
+ return H3_TE_INT8
+ host_capacity_gb = mib * (1 << 20) / 1e9
+ need = H3_TE_INT8_HOST_SET_GB
+ if denoiser_gb is not None:
+ need = need - 20.3 + float(denoiser_gb)
+ if float(host_capacity_gb) >= need:
+ return H3_TE_INT8
+ # Without gguf the 4-bit load falls back to the 66.7 GB dense encoder.
+ import importlib.util
+
+ if H3_TE_LOW_RAM_SCHEME == H3_TE_Q4_K_M and importlib.util.find_spec("gguf") is None:
+ return H3_TE_INT8
+ return H3_TE_LOW_RAM_SCHEME
+ except Exception: # noqa: BLE001 -- no reading, no change
+ return H3_TE_INT8
+
+
def h3_te_quant_filename(scheme: Optional[str]) -> Optional[str]:
"""The hosted checkpoint's path inside ``H3_TE_QUANT_REPO`` for ``scheme``, or None."""
return H3_TE_QUANT_FILES.get(scheme or "")
def h3_te_quant_source(scheme: Optional[str]) -> str:
- """The repo to fetch the hosted quantized conditioner for ``scheme`` from: always ``H3_TE_QUANT_REPO``. A video
- load never reads from the Comfy-Org repack, not even bytes an older install cached under it."""
- del scheme
- return H3_TE_QUANT_REPO
+ """Unsloth repo for ``scheme``; never a third-party repack, even one an older install cached."""
+ return H3_TE_QUANT_SOURCES.get(h3_te_quant_scheme(scheme) or "", H3_TE_QUANT_REPO)
def h3_te_resident_gb(scheme: Optional[str], *, bf16_gb: float) -> float:
@@ -273,60 +319,68 @@ def load_h3_quantized_text_encoder(
language_model.norm = torch.nn.Identity()
encoder.lm_head = torch.nn.Identity()
- int8_linear_cls = _int8_convrot_linear_class()
- with safe_open(path, framework = "pt", device = "cpu") as handle:
- names = set(handle.keys())
- quantized = {
- name[: -len(_SCALE_SUFFIX)] for name in names if name.endswith(_SCALE_SUFFIX)
- }
- # Swap before loading so the state dict lands on int8 buffers, not bf16 Linears.
- for prefix in sorted(quantized):
- quant_key = prefix + _QUANT_SUFFIX
- if quant_key not in names:
- raise ValueError(f"{prefix}: quantized weight with no {_QUANT_SUFFIX} metadata")
- _validate_comfy_quant(handle.get_tensor(quant_key), prefix)
- target = h3_te_remap_key(prefix)
- parent_path, _, leaf = target.rpartition(".")
- parent = encoder.get_submodule(parent_path)
- existing = getattr(parent, leaf)
- slice_ = handle.get_slice(prefix + ".weight")
- out_features, in_features = slice_.get_shape()
- if (existing.out_features, existing.in_features) != (out_features, in_features):
- raise ValueError(
- f"{target}: checkpoint shape {(out_features, in_features)} != model "
- f"{(existing.out_features, existing.in_features)}"
+ gguf_weights: set = set()
+ if filename.endswith(".gguf"):
+ state_dict, quantized, gguf_weights = _h3_gguf_state(encoder, path, dtype)
+ else:
+ int8_linear_cls = _int8_convrot_linear_class()
+ with safe_open(path, framework = "pt", device = "cpu") as handle:
+ names = set(handle.keys())
+ quantized = {
+ name[: -len(_SCALE_SUFFIX)] for name in names if name.endswith(_SCALE_SUFFIX)
+ }
+ # Swap before loading so the state dict lands on int8 buffers, not bf16 Linears.
+ for prefix in sorted(quantized):
+ quant_key = prefix + _QUANT_SUFFIX
+ if quant_key not in names:
+ raise ValueError(
+ f"{prefix}: quantized weight with no {_QUANT_SUFFIX} metadata"
+ )
+ _validate_comfy_quant(handle.get_tensor(quant_key), prefix)
+ target = h3_te_remap_key(prefix)
+ parent_path, _, leaf = target.rpartition(".")
+ parent = encoder.get_submodule(parent_path)
+ existing = getattr(parent, leaf)
+ slice_ = handle.get_slice(prefix + ".weight")
+ out_features, in_features = slice_.get_shape()
+ if (existing.out_features, existing.in_features) != (out_features, in_features):
+ raise ValueError(
+ f"{target}: checkpoint shape {(out_features, in_features)} != model "
+ f"{(existing.out_features, existing.in_features)}"
+ )
+ setattr(
+ parent,
+ leaf,
+ int8_linear_cls(
+ in_features,
+ out_features,
+ bias = (prefix + ".bias") in names,
+ group_size = H3_TE_CONVROT_GROUP,
+ ),
)
- setattr(
- parent,
- leaf,
- int8_linear_cls(
- in_features,
- out_features,
- bias = (prefix + ".bias") in names,
- group_size = H3_TE_CONVROT_GROUP,
- ),
- )
- # INT8 payload and fp32 scales keep their dtypes; dense parts follow the compute dtype.
- state_dict = {}
- for name in names:
- if name.endswith(_QUANT_SUFFIX):
- continue
- tensor = handle.get_tensor(name)
- if (
- not name.endswith(_SCALE_SUFFIX)
- and tensor.is_floating_point()
- and dtype is not None
- ):
- tensor = tensor.to(dtype)
- state_dict[h3_te_remap_key(name)] = tensor
+ # INT8 payload and fp32 scales keep their dtypes; dense parts follow the compute dtype.
+ state_dict = {}
+ for name in names:
+ if name.endswith(_QUANT_SUFFIX):
+ continue
+ tensor = handle.get_tensor(name)
+ if (
+ not name.endswith(_SCALE_SUFFIX)
+ and tensor.is_floating_point()
+ and dtype is not None
+ ):
+ tensor = tensor.to(dtype)
+ state_dict[h3_te_remap_key(name)] = tensor
# strict=True does not prove quantization: a dense re-upload loads as bf16 Linear and breaks
# the VRAM preflight, so no plain Linear may survive in the decoder stack.
+ from .diffusion_weight_only_quant import is_dense_linear
+
dense = [
name
for name, module in language_model.layers.named_modules()
- if isinstance(module, torch.nn.Linear)
+ if is_dense_linear(module)
]
if dense:
raise ValueError(
@@ -334,7 +388,13 @@ def load_h3_quantized_text_encoder(
f"e.g. {', '.join(sorted(dense)[:3])}; it is not the {scheme} checkpoint this "
f"path budgets for"
)
- encoder.load_state_dict(state_dict, strict = True, assign = True)
+ # GGUF projections were swapped in whole, so only their weights may be missing.
+ missing, unexpected = encoder.load_state_dict(state_dict, strict = False, assign = True)
+ if unexpected or set(missing) - gguf_weights:
+ raise ValueError(
+ f"the {scheme} checkpoint does not match the conditioner: missing "
+ f"{sorted(set(missing) - gguf_weights)[:3]}, unexpected {sorted(unexpected)[:3]}"
+ )
# A dense CPU rebuild is the 51 GB allocation this path avoids, so leftover meta is a refusal.
stranded = _meta_tensor_names(encoder)
if stranded:
@@ -346,7 +406,7 @@ def load_h3_quantized_text_encoder(
if logger is not None:
logger.info(
"video.h3_te_quant: loaded the hosted %s conditioner (%s, %s), "
- "%d ConvRot INT8 projections over %d decoder layers",
+ "%d quantized layers over %d decoder layers",
scheme,
source_repo,
filename,
@@ -365,6 +425,69 @@ def load_h3_quantized_text_encoder(
return None
+def _model_shape(module: Any, name: str) -> Optional[tuple]:
+ owner_path, _, leaf = name.rpartition(".")
+ try:
+ owner = module.get_submodule(owner_path) if owner_path else module
+ except AttributeError:
+ return None
+ tensor = owner._parameters.get(leaf)
+ if tensor is None:
+ tensor = owner._buffers.get(leaf)
+ return None if tensor is None else tuple(tensor.shape)
+
+
+def _h3_gguf_state(encoder: Any, path: str, dtype: Any) -> tuple[dict, set, set]:
+ """``(dense state dict, quantized layer names, swapped weight keys)``; quantized projections become ``GGUFLinear``."""
+ import math
+
+ import gguf
+ import torch
+
+ from .diffusion_weight_only_quant import gguf_tensor, gguf_weight_only_linear
+
+ reader = gguf.GGUFReader(path)
+ state: dict = {}
+ quantized: set = set()
+ swapped: set = set()
+ for t in reader.tensors:
+ target = h3_te_remap_key(t.name)
+ tensor, qtype = gguf_tensor(t)
+ if qtype is None:
+ if tensor.is_floating_point() and dtype is not None:
+ tensor = tensor.to(dtype)
+ # GGUF caps tensors at 4-D: the 5-D patch conv is stored with its first two dims merged.
+ want = _model_shape(encoder, target)
+ if want is not None and tuple(tensor.shape) != want:
+ if tensor.numel() != math.prod(want) or len(tensor.shape) > len(want):
+ raise ValueError(
+ f"{t.name}: shape {tuple(tensor.shape)} cannot hold the model's {want}"
+ )
+ tensor = tensor.reshape(want)
+ state[target] = tensor
+ continue
+ if not target.endswith(".weight"):
+ raise ValueError(f"{t.name}: a block-quantized tensor that is not a weight")
+ parent_path, _, leaf = target[: -len(".weight")].rpartition(".")
+ existing = getattr(encoder.get_submodule(parent_path), leaf)
+ if type(existing) is not torch.nn.Linear or existing.bias is not None:
+ raise ValueError(f"{target}: a quantized weight on a {type(existing).__name__}")
+ setattr(
+ encoder.get_submodule(parent_path),
+ leaf,
+ gguf_weight_only_linear(
+ tensor,
+ qtype,
+ in_features = existing.in_features,
+ out_features = existing.out_features,
+ compute_dtype = dtype or torch.bfloat16,
+ ),
+ )
+ quantized.add(target[: -len(".weight")])
+ swapped.add(target)
+ return state, quantized, swapped
+
+
def _meta_tensor_names(module: Any) -> list[str]:
"""Names of every parameter and buffer still on the meta device."""
from itertools import chain
diff --git a/studio/backend/hub/utils/companion_assets.py b/studio/backend/hub/utils/companion_assets.py
index 8edd3eda5c9..02a558672fa 100644
--- a/studio/backend/hub/utils/companion_assets.py
+++ b/studio/backend/hub/utils/companion_assets.py
@@ -248,12 +248,18 @@ def _hosted_component_table() -> dict[str, dict[str, set[tuple[str, str, str]]]]
logger.debug("Hosted text-encoder table unavailable: %s", exc)
try:
- from core.inference.video_minimax_h3_te import H3_TE_QUANT_FILES, H3_TE_QUANT_REPO
+ from core.inference.video_minimax_h3_te import (
+ H3_TE_QUANT_FILES,
+ H3_TE_QUANT_REPO,
+ h3_te_quant_source,
+ )
# Old released builds kept the identical conditioner outside official/; both locations are this asset.
+ # The GGUF conditioner ships from the GGUF mirror, already covered elsewhere.
names = [
n
- for name in H3_TE_QUANT_FILES.values()
+ for scheme, name in H3_TE_QUANT_FILES.items()
+ if h3_te_quant_source(scheme) == H3_TE_QUANT_REPO
for n in (name, name.replace("/official/", "/"))
]
for owner in by_name.get("minimax-h3", ()):
diff --git a/studio/backend/tests/test_h3_te_4bit.py b/studio/backend/tests/test_h3_te_4bit.py
new file mode 100644
index 00000000000..1fdc40fde67
--- /dev/null
+++ b/studio/backend/tests/test_h3_te_4bit.py
@@ -0,0 +1,297 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""H3 4-bit GGUF and int8 conditioner loads and the host-RAM pick, on a tiny CPU model."""
+
+from __future__ import annotations
+
+import json
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import video_minimax_h3_te as te # noqa: E402
+
+
+_LAYERS = te.H3_TE_READ_LAYER + 2
+
+
+def _tiny_config(hidden: int = 32):
+ transformers = pytest.importorskip("transformers")
+ if not hasattr(transformers, "Qwen3VLForConditionalGeneration"):
+ pytest.skip("transformers without Qwen3-VL")
+ from transformers import Qwen3VLConfig
+
+ return Qwen3VLConfig(
+ text_config = {
+ "hidden_size": hidden,
+ "intermediate_size": 2 * hidden,
+ "num_hidden_layers": _LAYERS,
+ "num_attention_heads": 2,
+ "num_key_value_heads": 1,
+ "head_dim": hidden // 2,
+ "vocab_size": 96,
+ "max_position_embeddings": 128,
+ "rope_scaling": {
+ "rope_type": "default",
+ "mrope_section": [2, 3, 3],
+ "mrope_interleaved": True,
+ },
+ },
+ vision_config = {
+ "depth": 1,
+ "hidden_size": 16,
+ "intermediate_size": 32,
+ "num_heads": 2,
+ "out_hidden_size": hidden,
+ "patch_size": 4,
+ "num_position_embeddings": 16,
+ "deepstack_visual_indexes": [0],
+ },
+ )
+
+
+def _layout_key_name(key: str) -> str:
+ if key.startswith("model.visual."):
+ return key[len("model.") :]
+ if key.startswith("model.language_model."):
+ return "model." + key[len("model.language_model.") :]
+ return key
+
+
+def _blob(meta: dict):
+ return torch.tensor(list(json.dumps(meta).encode()), dtype = torch.uint8)
+
+
+def _write_file(tmp_path):
+ import transformers
+ from safetensors.torch import save_file
+
+ from core.inference.diffusion_convrot import quantize_convrot_int8_weight
+
+ torch.manual_seed(2)
+ # ConvRot rotates groups of 256 input columns, so the int8 layout needs 256-wide projections.
+ config = _tiny_config(256)
+ (tmp_path / "text_encoder").mkdir()
+ config.save_pretrained(tmp_path / "text_encoder")
+ ref = transformers.Qwen3VLForConditionalGeneration(config).to(torch.bfloat16).eval()
+ out = {}
+ for key, tensor in ref.state_dict().items():
+ if key.startswith("lm_head") or key.endswith("language_model.norm.weight"):
+ continue
+ if ".layers." in key and "language_model" in key:
+ if int(key.split("layers.")[1].split(".")[0]) >= te.H3_TE_READ_LAYER:
+ continue
+ name = _layout_key_name(key)
+ stem = name[: -len(".weight")]
+ if "language_model.layers." in key and key.endswith("_proj.weight"):
+ codes, scale = quantize_convrot_int8_weight(tensor.float(), te.H3_TE_CONVROT_GROUP)
+ out[name] = codes
+ out[stem + ".weight_scale"] = scale.reshape(-1, 1).float()
+ out[stem + ".comfy_quant"] = _blob(
+ {
+ "format": "int8_tensorwise",
+ "convrot": True,
+ "convrot_groupsize": te.H3_TE_CONVROT_GROUP,
+ }
+ )
+ else:
+ out[name] = tensor.contiguous()
+ path = tmp_path / "int8.safetensors"
+ save_file(out, str(path))
+ return path, ref
+
+
+def _load(monkeypatch, tmp_path, scheme, path):
+ import sys
+ monkeypatch.setitem(
+ sys.modules,
+ "utils.hf_xet_fallback",
+ types.SimpleNamespace(hf_hub_download_with_xet_fallback = lambda *a, **k: str(path)),
+ )
+ return te.load_h3_quantized_text_encoder(
+ "MiniMaxAI/MiniMax-H3",
+ scheme,
+ dtype = torch.bfloat16,
+ local_base = str(tmp_path),
+ local_files_only = True,
+ )
+
+
+def _hidden(model, ids):
+ with torch.inference_mode():
+ return (
+ model.model(input_ids = ids, use_cache = False, output_hidden_states = True)
+ .hidden_states[te.H3_TE_READ_LAYER]
+ .float()
+ )
+
+
+def test_the_int8_file_still_loads_through_the_same_loader(monkeypatch, tmp_path):
+ path, ref = _write_file(tmp_path)
+ enc = _load(monkeypatch, tmp_path, "int8", path)
+ assert enc is not None
+ cls = te._int8_convrot_linear_class()
+ assert sum(isinstance(m, cls) for m in enc.modules()) == 7 * te.H3_TE_READ_LAYER
+ ids = torch.randint(0, 96, (1, 9))
+ got, want = _hidden(enc, ids), _hidden(ref, ids)
+ assert float((got - want).norm() / want.norm()) < 0.05
+
+
+def test_a_file_under_the_wrong_scheme_is_refused(monkeypatch, tmp_path):
+ path8, _ = _write_file(tmp_path)
+ assert _load(monkeypatch, tmp_path, "q4_k_m", path8) is None
+ other = tmp_path / "g"
+ other.mkdir()
+ pathg, _, _ = _write_gguf(other)
+ assert _load(monkeypatch, other, "int8", pathg) is None
+
+
+def _gguf_reader(monkeypatch, present: bool):
+ import importlib.util
+ real = importlib.util.find_spec
+ monkeypatch.setattr(
+ importlib.util,
+ "find_spec",
+ lambda name, *a: (object() if present else None) if name == "gguf" else real(name, *a),
+ )
+
+
+def test_an_unset_request_keeps_int8_on_every_host_while_the_switch_is_off(monkeypatch):
+ monkeypatch.delenv(te.H3_TE_QUANT_AUTO_ENV, raising = False)
+ _gguf_reader(monkeypatch, True)
+ assert te.H3_TE_LOW_RAM_AUTO is False
+ for gb in (10.0, 53.0, 200.0):
+ assert te.h3_te_auto_scheme(host_capacity_gb = gb) == "int8"
+
+
+def test_the_env_opts_in_to_q4_k_m(monkeypatch):
+ monkeypatch.setenv(te.H3_TE_QUANT_AUTO_ENV, "Q4-K-M")
+ assert te.h3_te_auto_scheme(host_capacity_gb = 500.0) == "q4_k_m"
+ monkeypatch.setenv(te.H3_TE_QUANT_AUTO_ENV, "int8")
+ assert te.h3_te_auto_scheme(host_capacity_gb = 10.0) == "int8"
+ monkeypatch.setenv(te.H3_TE_QUANT_AUTO_ENV, "bogus")
+ assert te.h3_te_auto_scheme(host_capacity_gb = 10.0) == "int8"
+
+
+def test_the_switch_when_on_follows_host_ram(monkeypatch):
+ monkeypatch.delenv(te.H3_TE_QUANT_AUTO_ENV, raising = False)
+ monkeypatch.setattr(te, "H3_TE_LOW_RAM_AUTO", True)
+ _gguf_reader(monkeypatch, True)
+ assert te.h3_te_auto_scheme(host_capacity_gb = 200.0) == "int8"
+ assert te.h3_te_auto_scheme(host_capacity_gb = te.H3_TE_INT8_HOST_SET_GB + 0.1) == "int8"
+ assert te.h3_te_auto_scheme(host_capacity_gb = 53.0) == te.H3_TE_LOW_RAM_SCHEME
+ assert te.h3_te_auto_scheme(host_capacity_gb = 66.0, denoiser_gb = 30.0) == te.H3_TE_LOW_RAM_SCHEME
+ assert te.H3_TE_LOW_RAM_SCHEME in te.H3_TE_AUTO_ONLY_SCHEMES
+ # no gguf module: int8 stays
+ _gguf_reader(monkeypatch, False)
+ assert te.h3_te_auto_scheme(host_capacity_gb = 53.0) == "int8"
+
+
+def test_the_pick_reaches_the_video_resolver(monkeypatch):
+ from core.inference.video import VideoBackend
+
+ fam = types.SimpleNamespace(
+ name = "minimax-h3", modular_workflow = "fl2va", base_repo = "MiniMaxAI/MiniMax-H3"
+ )
+ target = types.SimpleNamespace(
+ device = "cuda", dtype = torch.bfloat16, supports_default_torch_compile = True
+ )
+ monkeypatch.delenv(te.H3_TE_QUANT_AUTO_ENV, raising = False)
+ assert VideoBackend._h3_te_quant_scheme(fam, None, "MiniMaxAI/MiniMax-H3", target) == "int8"
+ monkeypatch.setenv(te.H3_TE_QUANT_AUTO_ENV, "q4_k_m")
+ assert VideoBackend._h3_te_quant_scheme(fam, None, "MiniMaxAI/MiniMax-H3", target) == "q4_k_m"
+ assert VideoBackend._h3_te_quant_scheme(fam, "int8", "MiniMaxAI/MiniMax-H3", target) == "int8"
+ assert VideoBackend._h3_te_quant_scheme(fam, "nvfp4", "MiniMaxAI/MiniMax-H3", target) is None
+
+
+def test_the_4bit_file_is_not_a_request_value_and_is_sized():
+ from core.inference.diffusion_precision import normalize_te_quant
+
+ with pytest.raises(ValueError):
+ normalize_te_quant(te.H3_TE_Q4_K_M)
+ assert set(te.H3_TE_QUANT_FILES) == {"int8", "q4_k_m"}
+ assert te.h3_te_resident_gb("q4_k_m", bf16_gb = 66.7) < te.h3_te_resident_gb("int8", bf16_gb = 66.7)
+
+
+def _write_gguf(tmp_path):
+ """Tiny GGUF in the hosted naming; Q8_0 since gguf-py cannot write Q4_K, which takes the same path."""
+ gguf = pytest.importorskip("gguf")
+ import numpy as np
+ import transformers
+
+ torch.manual_seed(3)
+ config = _tiny_config(32)
+ (tmp_path / "text_encoder").mkdir()
+ config.save_pretrained(tmp_path / "text_encoder")
+ ref = transformers.Qwen3VLForConditionalGeneration(config).to(torch.bfloat16).eval()
+ writer = gguf.GGUFWriter(str(tmp_path / "te.gguf"), "qwen3vl")
+ quantized = 0
+ for key, tensor in ref.state_dict().items():
+ if key.startswith("lm_head") or key.endswith("language_model.norm.weight"):
+ continue
+ if ".layers." in key and "language_model" in key:
+ if int(key.split("layers.")[1].split(".")[0]) >= te.H3_TE_READ_LAYER:
+ continue
+ name = _layout_key_name(key)
+ if "language_model.layers." in key and key.endswith("_proj.weight"):
+ q = gguf.quants.quantize(tensor.float().numpy(), gguf.GGMLQuantizationType.Q8_0)
+ writer.add_tensor(name, q, raw_dtype = gguf.GGMLQuantizationType.Q8_0)
+ dense = gguf.quants.dequantize(q, gguf.GGMLQuantizationType.Q8_0)
+ with torch.no_grad():
+ ref.get_parameter(key).copy_(
+ torch.from_numpy(np.ascontiguousarray(dense)).to(torch.bfloat16)
+ )
+ quantized += 1
+ elif key.endswith("embed_tokens.weight"):
+ raw = tensor.contiguous().view(torch.int16).numpy().view(np.uint16)
+ writer.add_tensor(
+ name, raw, raw_shape = tuple(tensor.shape), raw_dtype = gguf.GGMLQuantizationType.BF16
+ )
+ else:
+ arr = tensor.float()
+ if (
+ arr.dim() > 4
+ ): # GGUF holds 4 dimensions: the hosted file merges the patch conv's first two
+ arr = arr.reshape(-1, *arr.shape[2:])
+ writer.add_tensor(name, arr.numpy())
+ writer.write_header_to_file()
+ writer.write_kv_data_to_file()
+ writer.write_tensors_to_file()
+ writer.close()
+ return tmp_path / "te.gguf", ref, quantized
+
+
+def test_the_gguf_conditioner_loads_as_gguf_linears(monkeypatch, tmp_path):
+ pytest.importorskip(
+ "diffusers.quantizers.gguf.utils"
+ ) # the GGUF projections are diffusers' GGUFLinear
+ path, ref, quantized = _write_gguf(tmp_path)
+ enc = _load(monkeypatch, tmp_path, "q4_k_m", path)
+ assert enc is not None
+ from diffusers.quantizers.gguf.utils import GGUFLinear
+
+ from core.inference.diffusion_weight_only_quant import is_dense_linear
+
+ swapped = [m for m in enc.modules() if isinstance(m, GGUFLinear)]
+ assert len(swapped) == quantized == 7 * te.H3_TE_READ_LAYER
+ assert all(m.weight.dtype == torch.uint8 for m in swapped)
+ assert not [n for n, m in enc.model.language_model.layers.named_modules() if is_dense_linear(m)]
+ assert enc.get_input_embeddings().weight.dtype == torch.bfloat16
+ ids = torch.randint(0, 96, (2, 6))
+ got, want = _hidden(enc, ids), _hidden(ref, ids)
+ assert float((got - want).norm() / want.norm()) < 0.02
+
+
+def test_the_gguf_conditioner_comes_from_the_gguf_mirror():
+ assert te.h3_te_quant_source("q4_k_m") == "unsloth/MiniMax-H3-GGUF"
+ assert te.h3_te_quant_filename("q4_k_m") == "qwen3vl_32b_minimax_h3-Q4_K_M.gguf"
+ from core.inference.video_minimax_h3 import H3_GGUF_REPO, H3_QWEN_Q4
+
+ assert (te.h3_te_quant_source("q4_k_m"), te.h3_te_quant_filename("q4_k_m")) == (
+ H3_GGUF_REPO,
+ H3_QWEN_Q4,
+ )
+ assert "q4_k_m" in te.H3_TE_AUTO_ONLY_SCHEMES
diff --git a/studio/backend/tests/test_video_h3_te_quant.py b/studio/backend/tests/test_video_h3_te_quant.py
index 9c2283a555e..45444d62afb 100644
--- a/studio/backend/tests/test_video_h3_te_quant.py
+++ b/studio/backend/tests/test_video_h3_te_quant.py
@@ -86,7 +86,8 @@ def test_the_hosted_filename_comes_from_an_unsloth_repo():
def test_every_hosted_scheme_is_a_valid_text_encoder_quant_request():
from core.inference.diffusion_precision import normalize_te_quant
- for scheme in H3_TE_QUANT_FILES:
+ from core.inference.video_minimax_h3_te import H3_TE_AUTO_ONLY_SCHEMES
+ for scheme in set(H3_TE_QUANT_FILES) - H3_TE_AUTO_ONLY_SCHEMES:
assert normalize_te_quant(scheme) == scheme