Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: MiniMax-H3-Image text to image and reference edits (#17453)

Daniel Han1f68ed5ea316
diff --git a/scripts/risky_loaders_baseline.json b/scripts/risky_loaders_baseline.json
index f861a5592be..dafd344f0fe 100644
--- a/scripts/risky_loaders_baseline.json
+++ b/scripts/risky_loaders_baseline.json
@@ -10,14 +10,6 @@
"unsloth-cli.py"
],
"entries": [
- {
- "file": "unsloth/models/float8_training.py",
- "rule": "dynamic-import",
- "sink": "spec_from_file_location",
- "digest": "78b864cf6acb1555",
- "count": 1,
- "reason": "Standalone import fallback loads only unsloth/kernels/fp8_lora.py beside this shipped module, from __file__; no model metadata or caller input selects the module name or path."
- },
{
"file": "scripts/prequant_probe.py",
"rule": "unsafe-deserialize",
@@ -98,6 +90,14 @@
"count": 1,
"reason": "Imports a name from the constant _TORCHAO_INTMM_MODULES tuple (torchao's safe_int_mm homes); never a model, request or download name."
},
+ {
+ "file": "studio/backend/core/inference/diffusion_minimax_h3_image.py",
+ "rule": "dynamic-import",
+ "sink": "spec_from_file_location",
+ "digest": "78b864cf6acb1555",
+ "count": 1,
+ "reason": "Loads only the two vendored diffusers model files shipped in studio/backend/vendor/diffusers_minimax_h3_image, by a path built from __file__ and a literal module name; no model metadata or caller input selects the module name or path."
+ },
{
"file": "studio/backend/core/inference/diffusion_prequant.py",
"rule": "dynamic-import",
@@ -666,6 +666,14 @@
"count": 1,
"reason": "Fallback when this file is loaded on its own: loads unsloth/precision/__init__.py from the sibling directory of the shipped package, a fixed path, never a user-supplied one."
},
+ {
+ "file": "unsloth/models/float8_training.py",
+ "rule": "dynamic-import",
+ "sink": "spec_from_file_location",
+ "digest": "78b864cf6acb1555",
+ "count": 1,
+ "reason": "Standalone import fallback loads only unsloth/kernels/fp8_lora.py beside this shipped module, from __file__; no model metadata or caller input selects the module name or path."
+ },
{
"file": "unsloth/models/fsdp2_save.py",
"rule": "unsafe-deserialize",
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index eaff93400d3..f65abb51e16 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -165,7 +165,7 @@ from .diffusion_hidream import (
hidream_te4_kwargs,
)
from .diffusion_krea2 import KREA2_FAMILY_NAME, load_krea2_pipeline
-from .diffusion_families import MOSS_IMAGE_FAMILY_NAME
+from .diffusion_families import MINIMAX_H3_IMAGE_FAMILY_NAME, MOSS_IMAGE_FAMILY_NAME
from .diffusion_anima import (
ANIMA_COMPONENT_FOLDERS,
ANIMA_COMPONENTS,
@@ -2036,6 +2036,9 @@ def _family_default_ids(fam: Any) -> tuple:
return (fam.name,)
if is_anima_family(fam):
return (fam.base_repo,)
+ if getattr(fam, "name", None) == MINIMAX_H3_IMAGE_FAMILY_NAME:
+ # Its 6 steps / guidance 1 are fixed by the checkpoint (PDD heads), whatever the folder is called.
+ return (fam.name,)
return ()
@@ -2164,6 +2167,13 @@ def _family_model_class(diffusers: Any, name: str) -> Any:
if name in ("MOSSImagePipeline", "MOSSImageTransformer", "MOSSImageScheduler"):
from .diffusion_moss_image import in_tree_class
return in_tree_class(name)
+ if name in (
+ "MiniMaxH3ImagePipeline",
+ "MiniMaxH3ImageTransformer2DModel",
+ "AutoencoderKLMiniMaxH3Image",
+ ):
+ from .diffusion_minimax_h3_image import in_tree_class as h3_image_class
+ return h3_image_class(name)
return getattr(diffusers, name)
@@ -5490,6 +5500,11 @@ class DiffusionBackend:
)
if text_encoder_files or vae_file:
self._validate_component_files(fam, kind, repo_id, text_encoder_files, vae_file)
+ if getattr(fam, "name", None) == MINIMAX_H3_IMAGE_FAMILY_NAME:
+ from .diffusion_minimax_h3_image import h3_image_device_refusal
+ refusal = h3_image_device_refusal(resolve_diffusion_device_target().device)
+ if refusal:
+ raise ValueError(refusal)
return fam
def preflight_base_access(
@@ -5753,6 +5768,13 @@ class DiffusionBackend:
kwargs.get("gguf_filename"),
)
kwargs["base_repo"] = base
+ if getattr(fam, "name", None) == MINIMAX_H3_IMAGE_FAMILY_NAME:
+ # Its loader fetches only the files it reads, so the generic estimate / prefetch is skipped.
+ self.load_pipeline(**kwargs)
+ with self._load_cancel_lock:
+ if self._load_token == token:
+ self._loading = None
+ return
discovered = False
if not kwargs.get("text_encoder_files") and not kwargs.get("vae_file"):
kwargs["text_encoder_files"] = (
@@ -8559,6 +8581,24 @@ class DiffusionBackend:
reset_compile_cache_tag()
except Exception as exc: # noqa: BLE001 - keying only
logger.debug("nvfp4 cache tag reset skipped: %r", exc)
+ if getattr(fam, "name", None) == MINIMAX_H3_IMAGE_FAMILY_NAME:
+ # Native layout: the in-tree loader streams, converts, quantizes and places it, so the generic
+ # plan (which would materialise the bf16 DiT) never runs.
+ return self._load_h3_image_locked(
+ repo_id = repo_id,
+ display_repo_id = display_repo_id,
+ base = base,
+ fetch_base = _base_local_dir or fetch_base,
+ fam = fam,
+ target = target,
+ hf_token = hf_token,
+ local_files_only = local_files_only,
+ transformer_quant = transformer_quant,
+ # the raw request: "none" (keep bf16) and unset must stay apart
+ text_encoder_quant = text_encoder_quant_requested,
+ text_encoder_quant_explicit = bool(text_encoder_quant_requested),
+ load_token = _load_token,
+ )
# Single-file kinds resolve a checkpoint path; the pipeline kind has none.
single_file_path = (
@@ -11736,6 +11776,97 @@ class DiffusionBackend:
trim_host_heap_after_load(logger)
return self.status()
+ def _load_h3_image_locked(
+ self,
+ *,
+ repo_id: str,
+ display_repo_id: Optional[str],
+ base: str,
+ fetch_base: str,
+ fam: Any,
+ target: Any,
+ hf_token: Optional[str],
+ local_files_only: bool,
+ transformer_quant: Optional[str],
+ text_encoder_quant: Optional[str],
+ load_token: Optional[int],
+ text_encoder_quant_explicit: bool = False,
+ ) -> dict[str, Any]:
+ """MiniMax-H3-Image load under the transition slot and state lock (the previous model is already evicted)."""
+ from .diffusion_minimax_h3_image import (
+ PLACE_RESIDENT,
+ PLACE_SEQUENTIAL,
+ load_minimax_h3_image_pipeline,
+ )
+
+ # Unset = Studio's hosted int8 encoder (the bf16 one is 66.7 GB); only an explicit dense ask keeps bf16.
+ requested_te = str(text_encoder_quant or "").strip().lower()
+ te_scheme = None if requested_te in ("none", "off", "bf16", "dense", "bfloat16") else "int8"
+ pipe = load_minimax_h3_image_pipeline(
+ fetch_base,
+ dtype = target.dtype,
+ device = target.device,
+ transformer_quant = transformer_quant,
+ text_encoder_quant = te_scheme,
+ hf_token = hf_token,
+ cache_dir = hub_cache_dir(),
+ local_files_only = local_files_only,
+ check_cancelled = lambda: self._raise_if_load_cancelled(load_token),
+ )
+ report = pipe._unsloth_h3_report
+ plan = report.plan
+ policy = (
+ OFFLOAD_NONE
+ if plan.placement == PLACE_RESIDENT
+ else OFFLOAD_MODEL
+ if plan.placement == PLACE_SEQUENTIAL
+ else OFFLOAD_STREAMING
+ )
+ resolved = {
+ "transformer_quant": {
+ "value": report.scheme or "bf16",
+ "source": "explicit" if transformer_quant not in (None, "", "auto") else "auto",
+ "reason": report.scheme_reason,
+ },
+ "text_encoder_quant": {
+ "value": report.te_scheme or "bf16",
+ "source": "explicit" if text_encoder_quant_explicit else "auto",
+ "reason": "Studio's hosted int8 ConvRot Qwen3-VL"
+ if report.te_scheme == "int8"
+ else "released bf16",
+ },
+ "memory_mode": {
+ "value": plan.placement,
+ "source": "auto",
+ "reason": "; ".join(plan.reasons),
+ },
+ }
+ state = _LoadState(
+ pipe = pipe,
+ family = fam,
+ repo_id = repo_id,
+ display_repo_id = display_repo_id,
+ base_repo = base,
+ device = target.device,
+ gpu_ordinal = target.ordinal,
+ placed_ordinal = placed_cuda_ordinal(target),
+ dtype = str(target.dtype).replace("torch.", ""),
+ kind = "pipeline",
+ cpu_offload = policy != OFFLOAD_NONE,
+ offload_policy = policy,
+ memory_mode = plan.placement,
+ text_encoder_quant = report.te_scheme,
+ transformer_quant = report.scheme,
+ hf_token = hf_token,
+ resolved = resolved,
+ )
+ with self._load_cancel_lock:
+ self._raise_if_load_cancelled(load_token)
+ self._state = state
+ commit_load_budget_fraction("image")
+ logger.info("diffusion.loaded: repo=%s family=%s %s", repo_id, fam.name, report.as_dict())
+ return self.status()
+
@staticmethod
def _gguf_route_artifact(
pipe: Any,
@@ -16075,6 +16206,11 @@ class DiffusionBackend:
cuda_graph.uninstall_all(state.cuda_graphs)
uninstall_static_step_skip(state.pipe)
prompt_cache.release(state.pipe)
+ if getattr(state.family, "name", None) == MINIMAX_H3_IMAGE_FAMILY_NAME:
+ try:
+ state.pipe.release()
+ except Exception as exc: # noqa: BLE001 - teardown is best effort; the pipe is dropped below anyway
+ logger.debug("minimax_h3_image release skipped: %r", exc)
if state.te_releaser is not None:
state.te_releaser.close()
for aux in (*self._aux_pipes.values(), *self._cn_pipes.values()):
diff --git a/studio/backend/core/inference/diffusion_auto_policy.py b/studio/backend/core/inference/diffusion_auto_policy.py
index 301f2c1b3f9..1bb231d1f5d 100644
--- a/studio/backend/core/inference/diffusion_auto_policy.py
+++ b/studio/backend/core/inference/diffusion_auto_policy.py
@@ -56,6 +56,8 @@ _FAMILY_BF16_GB: dict[str, tuple[float, float, float]] = {
# FLUX.3 Image Klein: 5.55B DiT (11.1 GB bf16) + Qwen3-VL-4B (8.9 GB) + the FLUX.2 VAE (stored fp32, 0.17 GB
# bf16). From the black-forest-labs/flux-3-image-klein component folders; the base shares the architecture.
"flux.3-klein": (11.1, 8.9, 0.2),
+ # MiniMax-H3-Image: its loader plans placement itself; this row only sizes the hosted pre-quantized DiT.
+ "minimax-h3-image": (66.2, 66.7, 2.3),
"flux.3-klein-base": (11.1, 8.9, 0.2),
"flux.2-dev": (64.5, 48.0, 0.4),
"qwen-image": (40.9, 16.6, 0.3),
diff --git a/studio/backend/core/inference/diffusion_cfg_batch.py b/studio/backend/core/inference/diffusion_cfg_batch.py
index e6c60c9e098..c72a664ceae 100644
--- a/studio/backend/core/inference/diffusion_cfg_batch.py
+++ b/studio/backend/core/inference/diffusion_cfg_batch.py
@@ -54,6 +54,8 @@ NOT_BATCHABLE: dict[str, str] = {
"MOSSImageTransformer": "the negative branch skips block 22 (uncond_skip_layers): per-branch compute",
"Cosmos3OmniTransformer": "each branch packs its own variable-length token sequence",
"MiniMaxH3Transformer3DModel": "one denoiser call per step (no guidance pair to batch)",
+ "MiniMaxH3ImageTransformer2DModel": "guidance-distilled: one denoiser call per step, guidance_scale must be 1 "
+ "(no guidance pair to batch)",
"ChromaTransformer2DModel": "not measured batched: each branch passes its own T5 attention mask, and Chroma "
"runs eager",
"CosmosTransformer3DModel": "Anima's modular guider runs each branch as its own guider batch "
diff --git a/studio/backend/core/inference/diffusion_dynamic_text.py b/studio/backend/core/inference/diffusion_dynamic_text.py
index 49623aeb966..30a3d97c3ef 100644
--- a/studio/backend/core/inference/diffusion_dynamic_text.py
+++ b/studio/backend/core/inference/diffusion_dynamic_text.py
@@ -135,6 +135,8 @@ PROMPT_LENGTH_POLICY: dict[str, Any] = {
"Ideogram4Transformer2DModel": FIXED_LENGTH, # token ids left-padded into max_sequence_length 2048
"UNet2DConditionModel": FIXED_LENGTH, # CLIP padded to 77
"MiniMaxH3Transformer3DModel": _MINIMAX_H3_SOURCES,
+ # The same block stack and packed sequence without the audio rows: the prompt sets the packed length.
+ "MiniMaxH3ImageTransformer2DModel": _MINIMAX_H3_SOURCES,
"MOSSImageTransformer": _MOSS_IMAGE_SOURCES,
"LTX2VideoTransformer3DModel": FIXED_LENGTH, # Gemma left-padded to max_sequence_length 1024
"WanTransformer3DModel": FIXED_LENGTH, # UMT5 padded to max_sequence_length 512
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.py
index 527e0048160..f852108e564 100644
--- a/studio/backend/core/inference/diffusion_families.py
+++ b/studio/backend/core/inference/diffusion_families.py
@@ -441,6 +441,7 @@ _KREA2_SD_CPP_ARCH_MARKER = "Krea2"
# Keyed by architecture, not per variant: the base repo is read from the HF base_model tag at load time, so one entry
# covers Turbo/full, schnell/dev.
MOSS_IMAGE_FAMILY_NAME = "moss-image"
+MINIMAX_H3_IMAGE_FAMILY_NAME = "minimax-h3-image"
ANIMA_FAMILY_NAME = "anima"
# MOSS-Image keep-list: text_embedder, timestep MLP, AdaLN stay bf16 (LPIPS 0.058 vs 0.077 generic).
_MOSS_IMAGE_QUANT_KEEP: tuple[str, ...] = ("text_embedder", "time_embedding", "modulation")
@@ -1482,6 +1483,43 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
sd_cpp_arch_marker = "MOSS-Image",
sd_cpp_scheduler = "simple",
),
+ # MiniMax-H3-Image: 33B H3 DiT without audio plus per-step PDD heads, conditioned by Qwen3-VL-32B. diffusers has
+ # no class yet, so diffusion_minimax_h3_image loads, converts, quantizes and places it itself.
+ DiffusionFamily(
+ name = MINIMAX_H3_IMAGE_FAMILY_NAME,
+ label = "MiniMax-H3-Image",
+ pipeline_class = "MiniMaxH3ImagePipeline",
+ transformer_class = "MiniMaxH3ImageTransformer2DModel",
+ base_repo = "MiniMaxAI/MiniMax-H3-Image",
+ aliases = ("minimax_h3_image", "minimaxh3image", "h3-image", "h3_image"),
+ uses_negative_prompt = False,
+ uses_guidance = False,
+ # Text-to-image that also takes 1-16 reference images (<Picture N>); edits go through the same call.
+ reference = True,
+ reference_edit = True,
+ max_condition_images = 16,
+ dimension_multiple = 32,
+ max_output_side = 6272,
+ max_output_pixels = 20 * 1024 * 1024,
+ prompt_hint = (
+ "MiniMax-H3-Image follows long, complete descriptions of the whole image. For edits, refer to the inputs"
+ " as <Picture 1>, <Picture 2> and say what to keep and what to change."
+ ),
+ # The in-tree loader reads the native shards; no single-file / GGUF denoiser load.
+ pipeline_only = True,
+ gguf_loadable = False,
+ supports_torch_compile = False,
+ # No te_quant_auto: the in-tree loader picks the hosted int8 Qwen3-VL itself when the encoder is unset.
+ # Hosted pre-quantized DiTs: the loader checks the file exists and quantizes on the fly otherwise.
+ prequant_repos = (
+ ("int8", "unsloth/MiniMax-H3-Image-FP8"),
+ ("fp8", "unsloth/MiniMax-H3-Image-FP8"),
+ ),
+ prequant_filenames = (
+ ("int8", "MiniMax-H3-Image-INT8-ConvRot.safetensors"),
+ ("fp8", "MiniMax-H3-Image-FP8.safetensors"),
+ ),
+ ),
# SDXL is the one U-Net family: the denoiser is ``pipe.unet`` and a single-file ``.safetensors`` is the WHOLE
# pipeline. img2img / inpaint / ControlNet are the standard SDXL pipelines. No GGUF path.
DiffusionFamily(
@@ -2255,6 +2293,10 @@ _GENERATION_DEFAULTS: tuple[tuple[str, int, float], ...] = (
("hidream", 50, 5.0),
# Ideogram 4: an explicit 48 steps at 7 still runs the card's tapered schedule.
("ideogram", 20, 7.0),
+ ("minimax-h3-image", 6, 1.0),
+ ("minimax_h3_image", 6, 1.0),
+ ("minimaxh3image", 6, 1.0),
+ ("h3-image", 6, 1.0),
# MOSS-Image: the SFT and Base t2i templates both run 25 steps at CFG 2.5.
("moss-image", 25, 2.5),
("moss_image", 25, 2.5),
@@ -2791,6 +2833,8 @@ _PIPELINE_PROBE_FILES: dict[str, str] = {
# In-tree pipelines: (diffusers VAE module, transformers floor, family label).
_IN_TREE_PIPELINES: dict[str, tuple[str, str, str]] = {
"MOSSImagePipeline": ("models/autoencoders/autoencoder_kl_flux2.py", "5.3.0", "MOSS-Image"),
+ # Transformer and VAE vendored (vendor/diffusers_minimax_h3_image) when diffusers lacks them; Qwen3-VL needs 4.57.
+ "MiniMaxH3ImagePipeline": ("models/attention_dispatch.py", "4.57.0", "MiniMax-H3-Image"),
}
@@ -2815,11 +2859,27 @@ def in_tree_pipeline_refusal(pipeline_class: str, family_name: str) -> Optional[
if installed is None:
return f"'{family_name}' ({label}) needs transformers >= {min_transformers}, which is not installed."
if not _installed_at_least(installed, min_transformers):
+ encoder = (
+ "Qwen3.5 text encoder (transformers.Qwen3_5Model)"
+ if pipeline_class == "MOSSImagePipeline"
+ else "Qwen3-VL text encoder (transformers.Qwen3VLForConditionalGeneration)"
+ )
return (
- f"'{family_name}' ({label}) needs transformers >= {min_transformers} for its Qwen3.5 text encoder "
- f"(transformers.Qwen3_5Model), but this environment has transformers {installed}. Update Unsloth Studio "
+ f"'{family_name}' ({label}) needs transformers >= {min_transformers} for its {encoder}, but this "
+ f"environment has transformers {installed}. Update Unsloth Studio "
f"(or run: pip install -U 'transformers>={min_transformers}') and restart it."
)
+ if pipeline_class == "MiniMaxH3ImagePipeline":
+ import sys
+
+ if sys.version_info < (3, 10):
+ return f"'{family_name}' ({label}) needs Python 3.10 or newer. Update Unsloth Studio and restart it."
+ if _diffusers_package_has_file(vae_file) is False:
+ return (
+ f"'{family_name}' ({label}) needs a diffusers with the attention dispatcher (0.35 or newer). "
+ f"{DIFFUSERS_UPDATE_REMEDY}"
+ )
+ return None
if _diffusers_package_has_file(vae_file) is False:
return (
f"'{family_name}' ({label}) needs a diffusers with the FLUX.2 VAE (AutoencoderKLFlux2). "
diff --git a/studio/backend/core/inference/diffusion_minimax_h3_image.py b/studio/backend/core/inference/diffusion_minimax_h3_image.py
new file mode 100644
index 00000000000..50d20d74700
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_minimax_h3_image.py
@@ -0,0 +1,2625 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+#
+# The sampling recipe below (canvas rule, packed [text | references | image] layout, rotary grid, reference noising,
+# velocity-form Euler step, presentation of edit references, decode) is a port of the MiniMax-H3-Image Modular
+# Diffusers blocks of huggingface/diffusers PR 15230 (Apache-2.0, commit in vendor/diffusers_minimax_h3_image_manifest.json),
+# and the native-checkpoint key mapping follows that PR's scripts/convert_minimax_h3_to_diffusers.py.
+
+"""MiniMax-H3-Image (33B DiT + Qwen3-VL-32B conditioner) in Studio's image backend.
+
+Runs on today's pinned diffusers: the transformer and VAE classes come from diffusers when it ships them, else from
+the vendored PR copy. The released checkpoint is in MiniMax's native layout, so the DiT and VAE are converted while
+they stream off the safetensors shards, never as a second full copy. The 6-step PDD grid is fixed, so every block's
+AdaLN projection (13B of the 33B parameters) collapses to a lookup table over the 7 timesteps a render can ask for
+(6 steps + the 0.999 reference level), computed from the dense weights at load: exact to the dense bf16 math and
+20B parameters left to quantize."""
+
+from __future__ import annotations
+
+import functools
+import gc
+import importlib.util
+import inspect
+import json
+import math
+import os
+import sys
+import time
+import types
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable, Optional, Sequence
+
+from loggers import get_logger
+
+logger = get_logger(__name__)
+
+H3_IMAGE_FAMILY_NAME = "minimax-h3-image"
+H3_IMAGE_BASE_REPO = "MiniMaxAI/MiniMax-H3-Image"
+# The image release conditions on MiniMax-H3's text encoder without carrying it (model_index.json names this repo).
+H3_IMAGE_TEXT_ENCODER_REPO = "MiniMaxAI/MiniMax-H3"
+H3_IMAGE_PIPELINE_CLASS = "MiniMaxH3ImagePipeline"
+H3_IMAGE_TRANSFORMER_CLASS = "MiniMaxH3ImageTransformer2DModel"
+H3_IMAGE_NATIVE_TRANSFORMER_CLASS = "MiniMaxH3DiTModel"
+H3_IMAGE_VAE_CLASS = "AutoencoderKLMiniMaxH3Image"
+H3_IMAGE_NATIVE_VAE_CLASS = "MiniMaxH3ImageVAE"
+
+# Model card / pdd_config.json: distilled to 6 steps with one output head each, no CFG, no negative prompt.
+H3_IMAGE_STEPS = 6
+H3_IMAGE_GUIDANCE = 1.0
+H3_IMAGE_SIGMAS: tuple[float, ...] = (
+ 1.0,
+ 0.9675236940383911,
+ 0.8875485062599182,
+ 0.75,
+ 0.5327739119529724,
+ 0.23200887441635132,
+ 0.0,
+)
+# Smaller requests are generated on a canvas of this area at the same aspect and resized back (LANCZOS).
+H3_IMAGE_MIN_GENERATION_AREA = 2048 * 2048
+H3_IMAGE_MIN_GENERATION_AREA_RANGE = (1024, 4096 * 4096)
+H3_IMAGE_MAX_SEQUENCE_LENGTH = 4096
+H3_IMAGE_TEXT_ENCODER_LAYER = 50
+H3_IMAGE_MAX_REFERENCES = 16
+H3_IMAGE_CANVAS_MULTIPLE = 32
+H3_IMAGE_VAE_SCALE = 16
+H3_IMAGE_PIXEL_MEAN = (0.485, 0.456, 0.406)
+H3_IMAGE_PIXEL_STD = (0.229, 0.224, 0.225)
+H3_IMAGE_REFERENCE_NOISE_AUG = 0.999
+H3_IMAGE_REFERENCE_SHORT_EDGE = (1024, 2048)
+H3_IMAGE_VISION_SHORT_EDGE = (1024, 1024)
+# Per-row modality tags index the AdaLN table: a checkpoint contract.
+H3_IMAGE_IMAGE_TAG = 0
+H3_IMAGE_TEXT_TAG = 1
+H3_IMAGE_MODALITY_NUM = 3
+# Model card "Sizes": the recommended ~16 MP canvases (width, height); 4 MP and up is the working range.
+H3_IMAGE_16MP_SIZES: tuple[tuple[int, int], ...] = (
+ (4096, 4096),
+ (4736, 3552),
+ (3552, 4736),
+ (5024, 3360),
+ (3360, 5024),
+ (5472, 3072),
+ (3072, 5472),
+ (6272, 2688),
+)
+H3_IMAGE_PRESETS: tuple[tuple[int, int], ...] = (
+ (2048, 2048),
+ (2368, 1760),
+ (1760, 2368),
+ (2496, 1664),
+ (1664, 2496),
+ (2720, 1536),
+ (1536, 2720),
+) + H3_IMAGE_16MP_SIZES
+
+H3_IMAGE_DIT_BF16_GB = 66.3
+H3_IMAGE_DIT_TABLE_BF16_GB = 40.3
+H3_IMAGE_DIT_TABLE_INT8_GB = 21.9
+H3_IMAGE_VAE_GB = 2.34
+H3_IMAGE_TE_BF16_GB = 66.7
+H3_IMAGE_TE_INT8_GB = 27.2
+
+PDD_HEADS_FILE = "pdd_fused_heads.safetensors"
+PDD_CONFIG_FILE = "pdd_config.json"
+DIFFUSERS_TRANSFORMER_SUBFOLDER = "transformer_diffusers"
+DIFFUSERS_VAE_SUBFOLDER = "vae_diffusers"
+
+# "0" keeps the dense AdaLN projections (diagnostic: the table is the default and exact to the dense math).
+ADALN_TABLE_ENV = "UNSLOTH_H3_IMAGE_ADALN_TABLE"
+# Force a placement for tests and A/B runs: resident | sequential | stream.
+PLACEMENT_ENV = "UNSLOTH_H3_IMAGE_PLACEMENT"
+VRAM_BUDGET_ENV = "UNSLOTH_H3_IMAGE_VRAM_GB"
+HOST_BUDGET_ENV = "UNSLOTH_H3_IMAGE_HOST_GB"
+
+QUANT_SCHEMES = ("int8", "fp8")
+# ConvRot group: the one the H3 video denoiser and its hosted encoder use (divides 5376, 7168 and 14336).
+CONVROT_GROUP = 256
+# The block linears the on-the-fly quantizer converts. The text-only token refiner and context embedder run at
+# M = prompt tokens (can be under the int8 GEMM floor) and stay bf16 (0.8B parameters).
+QUANT_LINEAR_SUFFIXES = (
+ "attn.to_q",
+ "attn.to_k",
+ "attn.to_v",
+ "attn.to_out.0",
+ "ff.net.0.proj",
+ "ff.net.2",
+)
+
+
+def _env_off(name: str) -> bool:
+ return (os.environ.get(name) or "").strip().lower() in ("0", "off", "false", "no")
+
+
+_VENDOR_DIR = Path(__file__).resolve().parents[2] / "vendor" / "diffusers_minimax_h3_image"
+_VENDOR_PACKAGE = "unsloth_studio_vendor_minimax_h3_image"
+
+
+def _load_vendored(module: str) -> Any:
+ """The vendored module by file path, registered under a private name (never on sys.path)."""
+ name = f"{_VENDOR_PACKAGE}.{module}"
+ loaded = sys.modules.get(name)
+ if loaded is not None:
+ return loaded
+ path = _VENDOR_DIR / f"{module}.py"
+ spec = importlib.util.spec_from_file_location(name, path)
+ if spec is None or spec.loader is None:
+ raise ImportError(f"vendored MiniMax-H3-Image source missing: {path}")
+ mod = importlib.util.module_from_spec(spec)
+ sys.modules[name] = mod
+ try:
+ spec.loader.exec_module(mod)
+ except BaseException:
+ sys.modules.pop(name, None)
+ raise
+ return mod
+
+
+def _diffusers_class(name: str) -> Any:
+ try:
+ import diffusers
+ except Exception: # noqa: BLE001 - no diffusers at all: the caller reports it
+ return None
+ try:
+ cls = getattr(diffusers, name, None)
+ except Exception: # noqa: BLE001 - lazy-import failure inside diffusers
+ return None
+ return (
+ cls if cls is not None and hasattr(cls, "from_config") and hasattr(cls, "forward") else None
+ )
+
+
[email protected]_cache(maxsize = None)
+def transformer_class() -> Any:
+ """``MiniMaxH3ImageTransformer2DModel``: diffusers' when installed, else the vendored copy."""
+ return _diffusers_class(H3_IMAGE_TRANSFORMER_CLASS) or getattr(
+ _load_vendored("transformer_minimax_h3_image"), H3_IMAGE_TRANSFORMER_CLASS
+ )
+
+
[email protected]_cache(maxsize = None)
+def vae_class() -> Any:
+ """``AutoencoderKLMiniMaxH3Image``: diffusers' when installed, else the vendored copy."""
+ return _diffusers_class(H3_IMAGE_VAE_CLASS) or getattr(
+ _load_vendored("autoencoder_kl_minimax_h3_image"), H3_IMAGE_VAE_CLASS
+ )
+
+
+def classes_source() -> str:
+ """``"diffusers"`` when the installed diffusers ships both classes, else ``"vendored"``."""
+ both = _diffusers_class(H3_IMAGE_TRANSFORMER_CLASS) and _diffusers_class(H3_IMAGE_VAE_CLASS)
+ return "diffusers" if both else "vendored"
+
+
+def in_tree_class(name: str) -> Any:
+ """Resolve a class name the family table or a model_index names to Studio's in-tree class."""
+ if name in (H3_IMAGE_PIPELINE_CLASS,):
+ return MiniMaxH3ImagePipeline
+ if name in (H3_IMAGE_TRANSFORMER_CLASS, H3_IMAGE_NATIVE_TRANSFORMER_CLASS):
+ return transformer_class()
+ if name in (H3_IMAGE_VAE_CLASS, H3_IMAGE_NATIVE_VAE_CLASS):
+ return vae_class()
+ if name == "MiniMaxH3ImageScheduler":
+ return MiniMaxH3ImageScheduler
+ raise KeyError(name)
+
+
+def transformer_config_from_native(native: dict, num_pdd_heads: int = H3_IMAGE_STEPS) -> dict:
+ """``MiniMaxH3ImageTransformer2DModel`` kwargs from the release's ``transformer/config.json``."""
+ if native.get("audio_enabled"):
+ raise ValueError(
+ "MiniMax-H3-Image transformer config has audio_enabled; this is the video DiT."
+ )
+ patch = native.get("patch_size", [1, 2, 2])
+ return {
+ "num_attention_heads": int(native.get("num_attention_heads", 56)),
+ "attention_head_dim": int(native.get("attention_head_dim", 128)),
+ "hidden_size": int(native.get("hidden_size", 5376)),
+ "num_layers": int(native.get("num_layers", 50)),
+ "num_refiner_layers": int(native.get("token_refiner_num_layers", 2)),
+ "ffn_dim": int(native.get("ffn_hidden_size", 14336)),
+ "in_channels": int(native.get("latents_dim", 24)),
+ "patch_size": tuple(int(p) for p in patch),
+ "text_dim": int(native.get("text_dim", 5120)),
+ "freq_dim": int(native.get("timestep_input_dim", 256)),
+ "time_embed_hidden_dim": int(native.get("time_embed_hidden_size", 5376)),
+ "time_embed_dim": int(native.get("time_embed_dim", 2688)),
+ "rope_freq_dim": int(native.get("rope_inv_freq_len", 16)),
+ "rope_theta": float(native.get("rope_theta", 10000.0)),
+ "norm_eps": float(native.get("norm_eps", 1e-5)),
+ "qk_norm_eps": float(native.get("qk_norm_eps", 1e-5)),
+ "final_norm_eps": float(native.get("final_norm_eps", 1e-5)),
+ "num_pdd_heads": int(num_pdd_heads),
+ }
+
+
+def vae_config_from_native(native: dict) -> dict:
+ """``AutoencoderKLMiniMaxH3Image`` kwargs from the release's ``vae/config.json`` (converter checks included)."""
+ if native.get("vit_decoder_2d") is not True:
+ raise ValueError(
+ "MiniMax-H3-Image VAE config is not the 2D ViT decoder (vit_decoder_2d is not true)."
+ )
+ kwargs = dict(native.get("vit_decoder_kwargs") or {})
+ expected = {
+ "vae_encoder_tiling": 1,
+ "vae_decoder_tiling": 1,
+ "vae_tile_size": 256,
+ "vae_tile_overlap_min": 64,
+ }
+ bad = {k: native.get(k) for k, v in expected.items() if native.get(k, v) != v}
+ if bad:
+ raise ValueError(f"MiniMax-H3-Image VAE tile geometry differs from the released one: {bad}")
+ geometry = {
+ k: list(native[k])
+ for k in ("block_out_channels", "spatial_downsample_factors", "temporal_downsample_factors")
+ if k in native
+ }
+ if "layers_per_block" in native:
+ geometry["layers_per_block"] = int(native["layers_per_block"])
+ return {
+ **geometry,
+ "latent_channels": int(native.get("latent_channels", 24)),
+ "decoder_num_layers": int(kwargs.get("num_layers", 24)),
+ "decoder_num_attention_heads": int(kwargs.get("heads", 16)),
+ "decoder_attention_head_dim": int(kwargs.get("dim_head", 64)),
+ "decoder_rope_theta": float(kwargs.get("rope_theta", 100.0)),
+ "decoder_rope_dim_ratio": float(kwargs.get("rope_dim_ratio", 0.5)),
+ "latents_mean": tuple(float(v) for v in native["latents_mean"]),
+ "latents_std": tuple(float(v) for v in native["latents_std"]),
+ }
+
+
+def reorder_interleaved_qkv(weight: Any, heads: int, head_dim: int) -> Any:
+ """Raw shards store fused QKV per head ``[h0: q k v, h1: q k v, ...]``; returns ``[q_all; k_all; v_all]``."""
+ import torch
+
+ if weight.shape[0] != heads * 3 * head_dim:
+ raise ValueError(
+ f"fused qkv weight has {weight.shape[0]} rows, expected {heads} heads * 3 * {head_dim}"
+ )
+ grouped = weight.reshape(heads, 3 * head_dim, *weight.shape[1:])
+ q, k, v = grouped.split(head_dim, dim = 1)
+ return torch.cat([t.reshape(heads * head_dim, *weight.shape[1:]) for t in (q, k, v)], dim = 0)
+
+
+_TRANSFORMER_RENAMES = (
+ ("time_embedder.proj_in.", "time_embedder.linear_1."),
+ ("time_embedder.proj_out.", "time_embedder.linear_2."),
+ ("video_patch_proj.", "proj_in."),
+ ("condition_proj.", "context_embedder."),
+ ("final_layer.norm.", "norm_out.norm."),
+ ("final_layer.adaln_proj.linear.", "norm_out.linear."),
+ (".attn.q_norm.", ".attn.norm_q."),
+ (".attn.k_norm.", ".attn.norm_k."),
+ (".attn.out_proj.", ".attn.to_out.0."),
+ (".mlp.fc2.", ".ff.net.2."),
+)
+_TRANSFORMER_DROPPED = ("rope.inv_freq",)
+
+
+def native_transformer_key(key: str) -> Optional[str]:
+ """The diffusers name of one native DiT key (before the qkv / fc1 tensor transforms), None when dropped."""
+ if key in _TRANSFORMER_DROPPED or key.startswith("final_layer.video_out."):
+ # The per-step PDD heads replace the release's own output layer.
+ return None
+ if key.startswith("audio_") or key.startswith("final_layer.audio_out."):
+ return None
+ name = key
+ if name.startswith("token_refiner.blocks."):
+ name = "token_refiner.refiner_blocks." + name[len("token_refiner.blocks.") :]
+ elif name.startswith("blocks."):
+ name = "transformer_blocks." + name[len("blocks.") :]
+ for old, new in _TRANSFORMER_RENAMES:
+ name = name.replace(old, new)
+ return name
+
+
+def convert_native_transformer_tensor(key: str, tensor: Any, heads: int, head_dim: int) -> list:
+ """``[(diffusers key, tensor), ...]`` for one native DiT key/tensor (qkv split, SwiGLU half swap)."""
+ import torch
+
+ name = native_transformer_key(key)
+ if name is None:
+ return []
+ if name.endswith(".attn.qkv_proj.weight"):
+ fused = reorder_interleaved_qkv(tensor, heads, head_dim)
+ q, k, v = fused.split(heads * head_dim, dim = 0)
+ prefix = name[: -len("qkv_proj.weight")]
+ return [
+ (f"{prefix}to_q.weight", q.contiguous()),
+ (f"{prefix}to_k.weight", k.contiguous()),
+ (f"{prefix}to_v.weight", v.contiguous()),
+ ]
+ if name.endswith(".mlp.fc1.weight"):
+ # Reference fc2(silu(gate) * value) from [gate; value]; diffusers SwiGLU reads [value; gate].
+ gate, value = tensor.chunk(2, dim = 0)
+ return [
+ (
+ name.replace(".mlp.fc1.weight", ".ff.net.0.proj.weight"),
+ torch.cat([value, gate]).contiguous(),
+ )
+ ]
+ return [(name, tensor)]
+
+
+def split_pdd_heads(heads: dict, num_steps: int) -> dict:
+ """``video_out.{weight,bias}`` stacked ``(steps, ...)`` -> ``pdd_proj_out.{i}.{weight,bias}``."""
+ if set(heads) != {"video_out.weight", "video_out.bias"}:
+ raise KeyError(
+ f"{PDD_HEADS_FILE} must hold exactly video_out.weight and video_out.bias, got {sorted(heads)}"
+ )
+ out = {}
+ for key, tensor in heads.items():
+ if tensor.shape[0] != num_steps:
+ raise ValueError(
+ f"{PDD_HEADS_FILE} {key} has {tensor.shape[0]} heads, pdd_config says {num_steps}"
+ )
+ suffix = key[len("video_out.") :]
+ for step in range(num_steps):
+ out[f"pdd_proj_out.{step}.{suffix}"] = tensor[step].contiguous()
+ return out
+
+
+def native_vae_key(key: str) -> Optional[str]:
+ """Rename one native image-VAE key onto the diffusers module path (no tensor transform); None when dropped."""
+ if key == "decoder.mask_token":
+ return None
+ name = key
+ if name.startswith("encoder.down."):
+ level, rest = name[len("encoder.down.") :].split(".", 1)
+ rest = rest.replace("block.", "resnets.", 1).replace("nin_shortcut.", "conv_shortcut.", 1)
+ rest = rest.replace("downsample.", "downsamplers.0.", 1)
+ name = f"encoder.down_blocks.{level}.{rest}"
+ name = name.replace("decoder.x_embedder.", "decoder.proj_in.")
+ name = name.replace(".attn.to_out.", ".attn.to_out.0.")
+ name = name.replace(".ff.w1.", ".ff.net.0.proj.")
+ name = name.replace(".ff.w2.", ".ff.net.2.")
+ return name
+
+
+def convert_native_vae_tensor(key: str, tensor: Any, heads: int, head_dim: int) -> list:
+ import torch
+
+ if key == "decoder.mask_token":
+ return []
+ if ".attn.to_qkv." in key:
+ fused = reorder_interleaved_qkv(tensor, heads, head_dim)
+ q, k, v = fused.split(heads * head_dim, dim = 0)
+ prefix, suffix = key.split(".attn.to_qkv.")
+ return [
+ (f"{prefix}.attn.to_q.{suffix}", q.contiguous()),
+ (f"{prefix}.attn.to_k.{suffix}", k.contiguous()),
+ (f"{prefix}.attn.to_v.{suffix}", v.contiguous()),
+ ]
+ name = native_vae_key(key)
+ if ".ff.w1." in key:
+ gate, up = tensor.chunk(2, dim = 0)
+ return [(name, torch.cat([up, gate]).contiguous())]
+ return [(name, tensor)]
+
+
+def grid_timesteps(sigmas: Sequence[float] = H3_IMAGE_SIGMAS) -> list[float]:
+ """Every timestep a render can ask the DiT for: the float32 ``1 - sigma`` of each step, then the reference
+ level. Float32 values, as the pipeline builds them."""
+ import torch
+
+ steps = torch.tensor([1.0 - s for s in sigmas[:-1]], dtype = torch.float32).tolist()
+ ref = float(torch.tensor(H3_IMAGE_REFERENCE_NOISE_AUG, dtype = torch.float32))
+ out = []
+ for t in steps + [ref]:
+ if t not in out:
+ out.append(t)
+ return out
+
+
+def adaln_table_enabled() -> bool:
+ return not _env_off(ADALN_TABLE_ENV)
+
+
+def _grid_embedder_class() -> Any:
+ import torch
+ from torch import nn
+
+ class H3ImageGridTimestepEmbedder(nn.Module):
+ """Maps each timestep to a one-hot row over the fixed grid; the table Linears then select exact rows."""
+
+ def __init__(self, grid: Sequence[float]) -> None:
+ super().__init__()
+ self.register_buffer(
+ "grid", torch.tensor(list(grid), dtype = torch.float32), persistent = False
+ )
+
+ def forward(self, timestep: Any) -> Any:
+ t = timestep.to(torch.float32).reshape(-1, 1)
+ grid = self.grid.to(t.device)
+ dist = (t - grid[None, :]).abs()
+ best, index = dist.min(dim = 1)
+ # A timestep off the grid has no table row: refuse rather than return a neighbour's modulation.
+ if not torch.compiler.is_compiling() and bool((best > 1e-6).any()):
+ raise ValueError(
+ "MiniMax-H3-Image samples on its fixed 6-step grid (the AdaLN table holds only those timesteps); "
+ f"got timesteps {timestep.tolist()}"
+ )
+ return torch.nn.functional.one_hot(index, grid.numel()).to(torch.float32)
+
+ return H3ImageGridTimestepEmbedder
+
+
+def _raw_timestep_module() -> Any:
+ from torch import nn
+ class H3ImageRawTimestep(nn.Module):
+ def forward(self, timestep: Any) -> Any:
+ return timestep
+
+ return H3ImageRawTimestep()
+
+
+def _table_modulation_forward(self: Any, temb: Any) -> tuple:
+ """One-hot rows in, the dense projection's rows out: no SiLU (already folded into the table)."""
+ out = self.linear(temb.to(self.linear.weight.dtype))
+ out = out.view(-1, 6 * self.hidden_size)
+ return out.chunk(6, dim = -1)
+
+
+def _table_norm_out_forward(self: Any, hidden_states: Any, temb: Any, timestep_indices: Any) -> Any:
+ shift, scale = self.linear(temb.to(self.linear.weight.dtype)).chunk(2, dim = -1)
+ hidden_states = self.norm(hidden_states)
+ return hidden_states * (1.0 + scale.index_select(0, timestep_indices)) + shift.index_select(
+ 0, timestep_indices
+ )
+
+
+def dense_adaln_rows(
+ temb: Any,
+ weight: Any,
+ bias: Any,
+ dtype: Any,
+ device: Any = None,
+) -> Any:
+ """``linear(silu(temb).to(dtype))`` as the dense module computes it, rows per grid timestep; computed on
+ ``device`` (the card the render runs on) and returned on the CPU."""
+ import torch
+ import torch.nn.functional as F
+
+ dev = torch.device(device) if device is not None else weight.device
+ x = F.silu(temb.to(dev, torch.float32)).to(dtype)
+ rows = F.linear(x, weight.to(dev, dtype), None if bias is None else bias.to(dev, dtype))
+ return rows.to("cpu")
+
+
+def table_linear(rows: Any) -> Any:
+ """A ``Linear(K -> out)`` whose one-hot input selects ``rows[k]`` exactly (weight = rows.T, no bias)."""
+ import torch
+ from torch import nn
+
+ k, out_features = rows.shape
+ linear = nn.Linear(k, out_features, bias = False, device = "meta")
+ linear.weight = nn.Parameter(rows.t().contiguous(), requires_grad = False)
+ return linear
+
+
+def install_grid_time_embedding(transformer: Any, grid: Sequence[float]) -> None:
+ """Swap the timestep sinusoid + MLP for the one-hot grid embedder (the AdaLN Linears must already be tables)."""
+ transformer.time_proj = _raw_timestep_module()
+ transformer.time_embedder = _grid_embedder_class()(grid)
+ transformer._unsloth_adaln_table = tuple(float(t) for t in grid)
+
+
+def convert_block_adaln_to_table(block: Any, rows: Any) -> None:
+ block.adaln_proj.linear = table_linear(rows)
+ block.adaln_proj.forward = types.MethodType(_table_modulation_forward, block.adaln_proj)
+
+
+def convert_norm_out_to_table(norm_out: Any, rows: Any) -> None:
+ norm_out.linear = table_linear(rows)
+ norm_out.forward = types.MethodType(_table_norm_out_forward, norm_out)
+
+
+def grid_temb(transformer_dense_time: Any, grid: Sequence[float]) -> Any:
+ """``time_embedder(time_proj(t))`` for every grid timestep, float32, on CPU."""
+ import torch
+
+ time_proj, time_embedder = transformer_dense_time
+ t = torch.tensor(list(grid), dtype = torch.float32)
+ with torch.no_grad():
+ return time_embedder(time_proj(t).to(torch.float32)).to(torch.float32)
+
+
+def convert_dense_transformer_to_table(
+ transformer: Any, grid: Optional[Sequence[float]] = None
+) -> int:
+ """In place on an already-loaded dense model (tests, small builds): every AdaLN projection -> its grid table."""
+ import torch
+
+ grid = list(grid or grid_timesteps())
+ temb = grid_temb((transformer.time_proj, transformer.time_embedder), grid)
+ converted = 0
+ with torch.no_grad():
+ for block in transformer.transformer_blocks:
+ lin = block.adaln_proj.linear
+ rows = dense_adaln_rows(temb, lin.weight, lin.bias, lin.weight.dtype)
+ convert_block_adaln_to_table(block, rows)
+ converted += 1
+ lin = transformer.norm_out.linear
+ convert_norm_out_to_table(
+ transformer.norm_out, dense_adaln_rows(temb, lin.weight, lin.bias, lin.weight.dtype)
+ )
+ install_grid_time_embedding(transformer, grid)
+ return converted
+
+
+def canvas_size(
+ height: int,
+ width: int,
+ min_generation_area: int = H3_IMAGE_MIN_GENERATION_AREA,
+) -> tuple[int, int]:
+ """The ``(height, width)`` canvas a ``height x width`` request is generated on: scaled up to
+ ``min_generation_area`` at the same aspect when smaller, both sides rounded to multiples of 32."""
+ if height <= 0 or width <= 0:
+ raise ValueError(f"height and width must be positive, got {height}x{width}")
+ lo, hi = H3_IMAGE_MIN_GENERATION_AREA_RANGE
+ if not lo <= int(min_generation_area) <= hi:
+ raise ValueError(
+ f"min_generation_area must be within [{lo}, {hi}], got {min_generation_area}"
+ )
+ m = H3_IMAGE_CANVAS_MULTIPLE
+ scale = max(1.0, math.sqrt(min_generation_area / (width * height)))
+ return (max(m, round(height * scale / m) * m), max(m, round(width * scale / m) * m))
+
+
+def bound_short_edge(
+ width: int,
+ height: int,
+ lo: int,
+ hi: int,
+ multiple: int = H3_IMAGE_CANVAS_MULTIPLE,
+) -> tuple:
+ """Reference resize rule: keep a short edge in ``[lo, hi]``, else scale to the nearer bound (multiples of 32)."""
+ short = min(width, height)
+ if lo <= short <= hi:
+ return width, height
+ scale = (lo if short < lo else hi) / short
+ return (
+ max(multiple, round(width * scale / multiple) * multiple),
+ max(multiple, round(height * scale / multiple) * multiple),
+ )
+
+
+def patchify(latents: Any, patch: tuple) -> Any:
+ """``(B, C, T, H, W)`` -> rows ``(B * patches, C * prod(patch))``, frame-major then row-major."""
+ pt, ph, pw = patch
+ b, c, t, h, w = latents.shape
+ if t % pt or h % ph or w % pw:
+ raise ValueError(f"latents {tuple(latents.shape)} not divisible by patch {patch}")
+ x = latents.reshape(b, c, t // pt, pt, h // ph, ph, w // pw, pw)
+ x = x.permute(0, 2, 4, 6, 1, 3, 5, 7)
+ return x.reshape(-1, c * pt * ph * pw).contiguous()
+
+
+def unpatchify(
+ rows: Any, latent_height: int, latent_width: int, channels: int, patch: tuple
+) -> Any:
+ _, ph, pw = patch
+ x = rows.reshape(-1, latent_height // ph, latent_width // pw, channels, ph, pw)
+ x = x.permute(0, 3, 1, 4, 2, 5)
+ return x.reshape(-1, channels, latent_height, latent_width).contiguous()
+
+
+def _spatial_grid(dim: int, patch: int, sqrt_area: float) -> Any:
+ import numpy as np
+ import torch
+
+ ratio = dim / sqrt_area
+ left = (1.0 - ratio) / 2.0
+ grid = np.linspace(left, left + ratio, dim // patch, endpoint = False) * 32
+ return torch.from_numpy(grid).to(torch.float64)
+
+
+def _frame_grid(latent_height: int, latent_width: int, ph: int, pw: int) -> Any:
+ import numpy as np
+ import torch
+
+ sqrt_area = np.sqrt(latent_height * latent_width)
+ hg = _spatial_grid(latent_height, ph, sqrt_area)
+ wg = _spatial_grid(latent_width, pw, sqrt_area)
+ grids = torch.meshgrid(hg, wg, indexing = "ij")
+ return torch.stack([g.reshape(-1) for g in grids], dim = -1)
+
+
+def build_packed_layout(
+ text_token_tags: Any,
+ reference_grids: Sequence[tuple[int, int]],
+ latent_height: int,
+ latent_width: int,
+ patch: tuple = (1, 2, 2),
+) -> tuple:
+ """``[text | references | generated image]``: (position_ids f64 (S,3), token_tags, image_indices, text_indices,
+ number of reference rows)."""
+ import torch
+
+ _, ph, pw = patch
+ n_text = int(text_token_tags.shape[0])
+ n_ref = sum((h // ph) * (w // pw) for h, w in reference_grids)
+ n_target = (latent_height // ph) * (latent_width // pw)
+ seq = n_text + n_ref + n_target
+ pos = torch.zeros(seq, 3, dtype = torch.float64)
+ pos[:n_text, 0] = torch.arange(n_text, dtype = torch.float64)
+ cursor = n_text
+ clock = float(n_text)
+ for h, w in reference_grids:
+ rows = slice(cursor, cursor + (h // ph) * (w // pw))
+ cursor = rows.stop
+ pos[rows, 0] = clock
+ pos[rows, 1:] = _frame_grid(h, w, ph, pw)
+ clock += 1.0
+ pos[cursor:, 0] = clock
+ pos[cursor:, 1:] = _frame_grid(latent_height, latent_width, ph, pw)
+ text_indices = torch.arange(n_text)
+ image_indices = torch.arange(n_text, seq)
+ tags = torch.empty(seq, dtype = torch.long)
+ tags[text_indices] = text_token_tags.to(torch.long)
+ tags[image_indices] = H3_IMAGE_IMAGE_TAG
+ return pos, tags, image_indices, text_indices, n_ref
+
+
+def row_timestep_plan(timesteps: Any, seq_len: int, reference_rows: Any) -> list:
+ """Per step: (distinct timesteps, per-row index); reference rows stay at 0.999 for every step."""
+ import torch
+
+ plan = []
+ for t in timesteps:
+ row = torch.full((seq_len,), float(t), dtype = torch.float32)
+ if reference_rows is not None and reference_rows.numel():
+ row[reference_rows] = H3_IMAGE_REFERENCE_NOISE_AUG
+ unique, index = torch.unique(row, sorted = True, return_inverse = True)
+ plan.append((unique, index))
+ return plan
+
+
+def _scheduler_bases() -> tuple:
+ try:
+ from diffusers.configuration_utils import ConfigMixin, register_to_config
+ from diffusers.schedulers.scheduling_utils import SchedulerMixin
+ return SchedulerMixin, ConfigMixin, register_to_config
+ except Exception: # noqa: BLE001 - diffusers missing: distinct empty bases (object twice is a TypeError)
+
+ class _NoSchedulerMixin:
+ pass
+
+ class _NoConfigMixin:
+ pass
+
+ return _NoSchedulerMixin, _NoConfigMixin, (lambda f: f)
+
+
+_SchedulerMixin, _ConfigMixin, _register_to_config = _scheduler_bases()
+
+
+class MiniMaxH3ImageScheduler(_SchedulerMixin, _ConfigMixin):
+ """The release's sampler: explicit sigma grid taken verbatim, ``t = 1 - sigma``, Euler in velocity form
+ ``x_next = x + (sigma - sigma_next) * v`` in float32 (MiniMaxH3Scheduler with ``euler_update_form="velocity"``,
+ ``shift_provided_sigmas=False``)."""
+
+ config_name = "scheduler_config.json"
+ order = 1
+
+ @_register_to_config
+ def __init__(self, sample_sigmas: Optional[list] = None) -> None:
+ self.sigmas = None
+ self.timesteps = None
+ self.num_inference_steps = None
+ self._step_index = None
+
+ def set_timesteps(
+ self,
+ sigmas: Optional[Sequence[float]] = None,
+ device: Any = None,
+ **_: Any,
+ ) -> None:
+ import torch
+
+ base = list(
+ sigmas if sigmas is not None else (self.config.sample_sigmas or H3_IMAGE_SIGMAS)
+ )
+ # The released sampler rounds every sigma through its float32 timestep 1 - sigma and back.
+ timesteps = torch.tensor([1.0 - s for s in base[:-1]], dtype = torch.float32)
+ sig = torch.cat([1.0 - timesteps, torch.zeros(1)])
+ if sig.numel() < 2 or not bool((sig[1:] < sig[:-1]).all()):
+ raise ValueError("sigmas must be strictly decreasing and end at 0.0")
+ self.sigmas = sig.to(device)
+ self.timesteps = (1.0 - sig[:-1]).to(device)
+ self.num_inference_steps = int(self.timesteps.numel())
+ self._step_index = 0
+
+ def step(
+ self,
+ model_output: Any,
+ timestep: Any,
+ sample: Any,
+ return_dict: bool = False,
+ **_: Any,
+ ) -> tuple:
+ import torch
+
+ i = self._step_index
+ sigma = self.sigmas[i].to(device = sample.device, dtype = torch.float32)
+ sigma_next = self.sigmas[i + 1].to(device = sample.device, dtype = torch.float32)
+ prev = sample.to(torch.float32) + (sigma - sigma_next) * model_output.to(torch.float32)
+ self._step_index = i + 1
+ return (prev.to(sample.dtype),)
+
+ @staticmethod
+ def scale_noise(sample: Any, timestep: Any, noise: Any) -> Any:
+ """``x_t = t * x0 + (1 - t) * noise``; a tensor ``t`` mixes in float32 as the release does."""
+ import torch
+
+ if not isinstance(timestep, torch.Tensor):
+ return timestep * sample + (1.0 - timestep) * noise
+ t = timestep.to(device = sample.device, dtype = sample.dtype)
+ while t.ndim < sample.ndim:
+ t = t.unsqueeze(-1)
+ return t * sample + (1.0 - t) * noise
+
+
+PLACE_RESIDENT = "resident"
+PLACE_DIT_RESIDENT = "dit_resident"
+PLACE_SEQUENTIAL = "sequential"
+PLACE_STREAM = "stream"
+PLACEMENTS = (PLACE_RESIDENT, PLACE_DIT_RESIDENT, PLACE_SEQUENTIAL, PLACE_STREAM)
+
+
+@dataclass
+class H3ImagePlan:
+ """What the memory planner decided; recorded on the pipe and in the load's resolved controls."""
+
+ placement: str
+ dit_gb: float
+ te_gb: float
+ vae_gb: float
+ vram_gb: Optional[float]
+ host_gb: Optional[float]
+ dit_resident_blocks: Optional[int] = None
+ te_resident_layers: Optional[int] = None
+ # False: the encoder is loaded for each encode and freed after (host RAM cannot hold it beside the DiT)
+ te_kept_on_host: bool = True
+ reasons: list = field(default_factory = list)
+
+ def as_dict(self) -> dict:
+ return {
+ "placement": self.placement,
+ "dit_gb": round(self.dit_gb, 2),
+ "te_gb": round(self.te_gb, 2),
+ "vae_gb": round(self.vae_gb, 2),
+ "vram_gb": None if self.vram_gb is None else round(self.vram_gb, 2),
+ "host_gb": None if self.host_gb is None else round(self.host_gb, 2),
+ "dit_resident_blocks": self.dit_resident_blocks,
+ "te_resident_layers": self.te_resident_layers,
+ "te_kept_on_host": self.te_kept_on_host,
+ "reasons": list(self.reasons),
+ }
+
+
+def activation_gb(
+ width: int,
+ height: int,
+ references: int = 0,
+) -> float:
+ """Peak denoise + decode working set at a canvas (bf16 DiT, fp32 tiled VAE); generous by design."""
+ mp = max(width * height, H3_IMAGE_MIN_GENERATION_AREA) / 1e6
+ return 1.5 + 0.35 * mp * (1 + references)
+
+
+def dit_gb_for(scheme: Optional[str], table: bool = True) -> float:
+ dense_adaln_gb = 0.0 if table else H3_IMAGE_DIT_BF16_GB - H3_IMAGE_DIT_TABLE_BF16_GB
+ if scheme in QUANT_SCHEMES:
+ # The AdaLN projections are never quantized: without the table they stay dense bf16.
+ return H3_IMAGE_DIT_TABLE_INT8_GB + dense_adaln_gb
+ return H3_IMAGE_DIT_TABLE_BF16_GB + dense_adaln_gb
+
+
+def te_gb_for(scheme: Optional[str]) -> float:
+ return H3_IMAGE_TE_INT8_GB if scheme == "int8" else H3_IMAGE_TE_BF16_GB
+
+
+# Device-side pieces that are not in the streamed lists: the DiT's refiner, embedders and heads (bf16 refiner,
+# 0.8B), the encoder's token embedding + vision tower (1.56 + ~1.2 GB).
+DIT_TOP_GB = 2.0
+TE_TOP_GB = 3.0
+
+
+def plan_placement(
+ *,
+ vram_gb: Optional[float],
+ host_gb: Optional[float],
+ dit_gb: float,
+ te_gb: float,
+ vae_gb: float = H3_IMAGE_VAE_GB,
+ width: int = 2048,
+ height: int = 2048,
+ references: int = 0,
+ num_blocks: int = 50,
+ te_layers: int = 50,
+ forced: Optional[str] = None,
+) -> H3ImagePlan:
+ """Pure: VRAM and host budgets (GB, None = unknown/CPU) -> placement, best first:
+
+ - resident: DiT + encoder + VAE + working set fit;
+ - dit_resident: the DiT and VAE stay, the encoder's top stays and as many of its layers as fit, the rest stream
+ through once per prompt (A100 40 GB with both at int8);
+ - sequential: one big component on the card at a time (a DiT larger than the encoder room);
+ - stream: DiT blocks stream past the ones that fit, the encoder streams every layer (L4 24 GB).
+
+ Host: the encoder is kept only when it fits beside the DiT, else it loads per prompt and is freed after."""
+ reasons = []
+ act = activation_gb(width, height, references)
+ margin = 1.5
+ plan = H3ImagePlan(
+ placement = PLACE_RESIDENT,
+ dit_gb = dit_gb,
+ te_gb = te_gb,
+ vae_gb = vae_gb,
+ vram_gb = vram_gb,
+ host_gb = host_gb,
+ )
+ if vram_gb is None:
+ plan.placement = PLACE_RESIDENT
+ reasons.append("no accelerator budget: everything on the compute device")
+ plan.reasons = reasons
+ return plan
+ layer_gb = max(te_gb - TE_TOP_GB, 0.1) / max(1, te_layers)
+ block_gb = max(dit_gb - DIT_TOP_GB, 0.1) / max(1, num_blocks)
+ if forced in PLACEMENTS:
+ choice = forced
+ reasons.append(f"placement forced to {forced} by {PLACEMENT_ENV}")
+ elif dit_gb + te_gb + vae_gb + act + margin <= vram_gb:
+ choice = PLACE_RESIDENT
+ reasons.append(
+ f"DiT + encoder + VAE ({dit_gb + te_gb + vae_gb:.1f} GB) fit {vram_gb:.1f} GB"
+ )
+ elif dit_gb + vae_gb + act + margin + TE_TOP_GB + layer_gb <= vram_gb:
+ choice = PLACE_DIT_RESIDENT
+ reasons.append(
+ f"DiT + VAE ({dit_gb + vae_gb:.1f} GB) stay on {vram_gb:.1f} GB; the encoder streams"
+ )
+ elif max(dit_gb, te_gb) + vae_gb + act + margin <= vram_gb:
+ choice = PLACE_SEQUENTIAL
+ reasons.append(
+ f"one component at a time: largest {max(dit_gb, te_gb):.1f} GB + working set fits {vram_gb:.1f} GB"
+ )
+ else:
+ choice = PLACE_STREAM
+ reasons.append(f"DiT {dit_gb:.1f} GB + working set exceeds {vram_gb:.1f} GB: blocks stream")
+ plan.placement = choice
+ if choice == PLACE_DIT_RESIDENT:
+ spare = vram_gb - dit_gb - vae_gb - act - margin - TE_TOP_GB - layer_gb
+ plan.te_resident_layers = max(0, min(te_layers, int(spare / layer_gb)))
+ reasons.append(f"{plan.te_resident_layers}/{te_layers} encoder layers stay on the device")
+ elif choice == PLACE_STREAM:
+ spare = vram_gb - act - margin - vae_gb - DIT_TOP_GB - TE_TOP_GB - block_gb
+ plan.dit_resident_blocks = max(0, min(num_blocks, int(spare / block_gb)))
+ # The resident blocks hold their room through the encode, so every encoder layer streams.
+ plan.te_resident_layers = 0
+ reasons.append(
+ f"{plan.dit_resident_blocks}/{num_blocks} DiT blocks stay on the device; encoder streams"
+ )
+ if host_gb is not None:
+ dit_host = {
+ PLACE_RESIDENT: 0.0,
+ PLACE_DIT_RESIDENT: 0.0,
+ PLACE_SEQUENTIAL: dit_gb + vae_gb,
+ PLACE_STREAM: block_gb * (num_blocks - int(plan.dit_resident_blocks or 0)),
+ }[choice]
+ need = dit_host + te_gb + 4.0
+ plan.te_kept_on_host = host_gb >= need
+ # Loading per prompt only frees the encoder between prompts: each load still holds it whole beside the
+ # host-side DiT blocks, so a host below that peak would be OOM-killed on the first prompt.
+ peak = dit_host + te_gb + 1.0
+ if host_gb < peak:
+ raise RuntimeError(
+ f"MiniMax-H3-Image needs about {peak:.0f} GB of free system memory on this GPU "
+ f"({choice} placement: {dit_host:.1f} GB of DiT blocks on the host plus the {te_gb:.1f} GB "
+ f"text encoder while it loads), and {host_gb:.1f} GB is available. Close other applications or "
+ f"use a machine with more memory."
+ )
+ if not plan.te_kept_on_host:
+ reasons.append(
+ f"host {host_gb:.1f} GB < {need:.1f} GB for the host-side DiT + encoder: the encoder loads per prompt"
+ )
+ plan.reasons = reasons
+ return plan
+
+
+def replan_for_encoder(
+ plan: H3ImagePlan,
+ te_gb: float,
+ plan_args: dict,
+ slack_gb: float = 1.0,
+) -> H3ImagePlan:
+ """The plan again with the encoder's real size when it is larger than the one budgeted (a bf16 fallback for the
+ int8 encoder, a local text_encoder/ folder); the plan unchanged otherwise. Raises when no tier fits."""
+ if te_gb <= plan.te_gb + slack_gb:
+ return plan
+ new = plan_placement(te_gb = te_gb, **plan_args)
+ vram = plan_args.get("vram_gb")
+ if vram is not None and new.placement == PLACE_STREAM and not plan_args.get("forced"):
+ floor = activation_gb(plan_args.get("width", 2048), plan_args.get("height", 2048), 0) + 1.5
+ floor += H3_IMAGE_VAE_GB + DIT_TOP_GB + TE_TOP_GB
+ if vram < floor:
+ raise RuntimeError(
+ f"MiniMax-H3-Image: the text encoder loaded is {te_gb:.1f} GB (the bf16 one, not Studio's 27.2 GB "
+ f"int8 encoder) and no placement fits {vram:.1f} GB of GPU memory with it. Load it with the int8 "
+ f"text encoder."
+ )
+ new.reasons = [
+ f"encoder is {te_gb:.1f} GB, not the {plan.te_gb:.1f} GB planned: planned again"
+ ] + list(new.reasons)
+ logger.warning("diffusion.minimax_h3_image: %s", new.reasons[0])
+ return new
+
+
+class ModuleListStreamer:
+ """Streams the non-resident members of a ModuleList through the device: a forward pre-hook swaps each tensor for
+ a device copy, the post-hook swaps the host original back (no device-to-host copy). Works for torchao
+ tensor-subclass weights, whose ``.to(device)`` returns a device subclass tensor."""
+
+ def __init__(self, modules: Any, device: Any, resident: int) -> None:
+ import torch
+
+ self.device = torch.device(device)
+ self.handles = []
+ self.streamed = 0
+ self._host: dict = {}
+ for index, module in enumerate(modules):
+ if index < resident:
+ module.to(self.device)
+ continue
+ module.to("cpu")
+ self._host[id(module)] = [
+ (owner, kind, name, tensor)
+ for owner in module.modules()
+ for kind, table in (("p", owner._parameters), ("b", owner._buffers))
+ for name, tensor in table.items()
+ if tensor is not None
+ ]
+ self.handles.append(module.register_forward_pre_hook(self._onload))
+ self.handles.append(module.register_forward_hook(self._offload))
+ self.streamed += 1
+
+ def _onload(self, module: Any, args: Any) -> None:
+ import torch
+ for owner, kind, name, tensor in self._host[id(module)]:
+ moved = tensor.to(self.device, non_blocking = True)
+ if kind == "p":
+ owner._parameters[name] = torch.nn.Parameter(moved, requires_grad = False)
+ else:
+ owner._buffers[name] = moved
+
+ def _offload(self, module: Any, args: Any, output: Any) -> None:
+ for owner, kind, name, tensor in self._host[id(module)]:
+ (owner._parameters if kind == "p" else owner._buffers)[name] = tensor
+
+ def remove(self) -> None:
+ for handle in self.handles:
+ handle.remove()
+ self.handles = []
+
+
+def torch_no_grad(fn: Callable) -> Callable:
+ @functools.wraps(fn)
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
+ import torch
+ with torch.no_grad():
+ return fn(*args, **kwargs)
+
+ return wrapper
+
+
+def _pipeline_base() -> Any:
+ try:
+ from diffusers import DiffusionPipeline
+ except (
+ ImportError
+ ): # diffusers missing: the module still imports (planner, family tables); loading raises
+ return object
+ return DiffusionPipeline
+
+
+def _image_output(images: list) -> Any:
+ from diffusers.pipelines.pipeline_utils import ImagePipelineOutput
+ return ImagePipelineOutput(images = images)
+
+
+class MiniMaxH3ImagePipeline(_pipeline_base()):
+ """Text-to-image and reference editing (1-16 ``<Picture N>`` references) for MiniMax-H3-Image."""
+
+ model_cpu_offload_seq = "text_encoder->transformer->vae"
+ _callback_tensor_inputs = ["latents"]
+ _optional_components = ["processor"]
+
+ def __init__(
+ self,
+ transformer: Any,
+ vae: Any,
+ text_encoder: Any,
+ tokenizer: Any,
+ scheduler: Any,
+ processor: Any = None,
+ ) -> None:
+ super().__init__()
+ self.register_modules(
+ transformer = transformer,
+ vae = vae,
+ text_encoder = text_encoder,
+ tokenizer = tokenizer,
+ scheduler = scheduler,
+ processor = processor,
+ )
+ self._interrupt = False
+ self._num_timesteps = H3_IMAGE_STEPS
+ self._unsloth_h3_plan: Optional[H3ImagePlan] = None
+ self._unsloth_h3_device: Any = None
+ self._unsloth_h3_te_loader: Optional[Callable[[], Any]] = None
+ self._unsloth_h3_streamers: list = []
+ self._unsloth_h3_last_stats: dict = {}
+
+ @property
+ def interrupt(self) -> bool:
+ return bool(self._interrupt)
+
+ def _device(self) -> Any:
+ import torch
+ if self._unsloth_h3_device is not None:
+ return torch.device(self._unsloth_h3_device)
+ try:
+ return self._execution_device
+ except Exception: # noqa: BLE001 - no registered module on a device
+ return torch.device("cpu")
+
+ def _placement(self) -> str:
+ plan = self._unsloth_h3_plan
+ return plan.placement if plan is not None else PLACE_RESIDENT
+
+ def _enter(self, name: str) -> Any:
+ """Put component ``name`` where its phase runs; returns it."""
+ module = getattr(self, name, None)
+ if module is None and name == "text_encoder" and self._unsloth_h3_te_loader is not None:
+ module = self._unsloth_h3_te_loader()
+ self.text_encoder = module
+ if module is None:
+ return None
+ if self._placement() == PLACE_SEQUENTIAL:
+ module.to(self._device())
+ return module
+
+ def _leave(self, name: str) -> None:
+ module = getattr(self, name, None)
+ if module is None:
+ return
+ placement = self._placement()
+ if placement == PLACE_SEQUENTIAL:
+ module.to("cpu")
+ _empty_cache()
+ if name == "text_encoder" and self._unsloth_h3_te_loader is not None:
+ self.text_encoder = None
+ for streamer in list(self._unsloth_h3_streamers):
+ if getattr(streamer, "_unsloth_role", None) == "text_encoder":
+ streamer.remove()
+ streamer._host.clear()
+ self._unsloth_h3_streamers.remove(streamer)
+ free_module_storage(module)
+ del module
+ gc.collect()
+ _empty_cache()
+
+ def release(self) -> None:
+ """Drop every component and streaming hook so an unload frees the card and the host at once (the hooks hold
+ the host copies of the streamed blocks)."""
+ for streamer in list(self._unsloth_h3_streamers):
+ streamer.remove()
+ streamer._host.clear()
+ self._unsloth_h3_streamers.clear()
+ self._unsloth_h3_te_loader = None
+ owned = getattr(self, "_unsloth_h3_owned", ())
+ for name in ("transformer", "text_encoder", "vae"):
+ module = getattr(self, name, None)
+ if module is not None:
+ if name in owned:
+ free_module_storage(module)
+ setattr(self, name, None)
+ gc.collect()
+ _empty_cache()
+
+ def _processor(self) -> Any:
+ if self.processor is not None:
+ return self.processor
+ raise ValueError(
+ "MiniMax-H3-Image editing needs the Qwen3-VL processor (processor/ of the repo)."
+ )
+
+ def _encode(self, prompt: str, vision_images: Optional[list]) -> tuple:
+ """(prompt_embeds (1, n, 5120), per-row text tags) for one presentation."""
+ import torch
+
+ tokenizer = self.tokenizer
+ max_len = H3_IMAGE_MAX_SEQUENCE_LENGTH
+ token_ids: list[int] = []
+ tags: list[int] = []
+ vision_inputs: dict = {}
+ if vision_images:
+ processor = self._processor()
+ vision = processor.image_processor(images = vision_images, return_tensors = "pt")
+ merge = int(processor.image_processor.merge_size) ** 2
+ start = tokenizer.convert_tokens_to_ids("<|vision_start|>")
+ pad = tokenizer.convert_tokens_to_ids("<|image_pad|>")
+ end = tokenizer.convert_tokens_to_ids("<|vision_end|>")
+ for index, grid in enumerate(vision["image_grid_thw"]):
+ label = tokenizer(f"Picture {index + 1}: ", add_special_tokens = False)["input_ids"][
+ :max_len
+ ]
+ block = [start] + [pad] * (int(grid.prod()) // merge) + [end]
+ token_ids += label + block
+ tags += [H3_IMAGE_TEXT_TAG] * len(label) + [H3_IMAGE_IMAGE_TAG] * len(block)
+ vision_inputs = {
+ "pixel_values": vision["pixel_values"],
+ "image_grid_thw": vision["image_grid_thw"],
+ }
+ prompt_ids = tokenizer(prompt, add_special_tokens = False)["input_ids"][:max_len]
+ token_ids += prompt_ids
+ tags += [H3_IMAGE_TEXT_TAG] * len(prompt_ids)
+ if not token_ids:
+ raise ValueError("MiniMax-H3-Image needs a non-empty prompt.")
+ encoder = self._enter("text_encoder")
+ try:
+ embeds = qwen3vl_hidden_state(
+ encoder,
+ self.processor,
+ token_ids,
+ vision_inputs,
+ layer = H3_IMAGE_TEXT_ENCODER_LAYER,
+ device = self._device(),
+ )
+ finally:
+ self._leave("text_encoder")
+ return embeds, torch.tensor(tags, dtype = torch.long)
+
+ def _encode_references(self, images: list, generator: Any) -> list:
+ import numpy as np
+ import torch
+
+ device = self._device()
+ vae = self._enter("vae")
+ try:
+ vae_dtype = _module_dtype(vae)
+ mean = torch.tensor(H3_IMAGE_PIXEL_MEAN, device = device).view(1, -1, 1, 1)
+ std = torch.tensor(H3_IMAGE_PIXEL_STD, device = device).view(1, -1, 1, 1)
+ lmean = torch.tensor(vae.config.latents_mean, device = device).view(1, -1, 1, 1)
+ lstd = torch.tensor(vae.config.latents_std, device = device).view(1, -1, 1, 1)
+ out = []
+ r = H3_IMAGE_VAE_SCALE
+ for image in images:
+ pixels = torch.from_numpy(np.array(image)).to(device).permute(2, 0, 1)[None]
+ pixels = pixels[..., : pixels.shape[-2] // r * r, : pixels.shape[-1] // r * r]
+ pixels = (pixels.to(torch.float32).div(255.0) - mean) / std
+ posterior = vae.encode(pixels.to(vae_dtype), return_dict = False)[0]
+ latents = posterior.sample(generator = generator).to(torch.float16).float()
+ latents = latents[..., : latents.shape[-2] // 2 * 2, : latents.shape[-1] // 2 * 2]
+ out.append((latents - lmean) / lstd)
+ return out
+ finally:
+ self._leave("vae")
+
+ @staticmethod
+ def prepare_references(images: Any) -> tuple[list, list]:
+ """(VAE copies, encoder copies) of the references: upright, RGB, short edge bounded per consumer."""
+ from PIL import Image, ImageOps
+
+ if images is None:
+ return [], []
+ items = images if isinstance(images, (list, tuple)) else [images]
+ if len(items) > H3_IMAGE_MAX_REFERENCES:
+ raise ValueError(
+ f"MiniMax-H3-Image takes at most {H3_IMAGE_MAX_REFERENCES} reference images, got {len(items)}."
+ )
+ prepared = []
+ for image in items:
+ if not isinstance(image, Image.Image):
+ raise ValueError("MiniMax-H3-Image references must be PIL images.")
+ image = ImageOps.exif_transpose(image).convert("RGB")
+ if min(image.size) < H3_IMAGE_CANVAS_MULTIPLE:
+ raise ValueError(
+ f"A reference image needs at least {H3_IMAGE_CANVAS_MULTIPLE} pixels on each side, got {image.size}."
+ )
+ prepared.append(image)
+
+ def resize(image: Any, lo: int, hi: int) -> Any:
+ size = bound_short_edge(*image.size, lo, hi)
+ return image if image.size == size else image.resize(size, resample = Image.LANCZOS)
+
+ vae_images = [resize(i, *H3_IMAGE_REFERENCE_SHORT_EDGE) for i in prepared]
+ vision_images = [resize(i, *H3_IMAGE_VISION_SHORT_EDGE) for i in prepared]
+ return vae_images, vision_images
+
+ def __call__(
+ self,
+ prompt: Any = None,
+ image: Any = None,
+ height: Optional[int] = None,
+ width: Optional[int] = None,
+ num_inference_steps: int = H3_IMAGE_STEPS,
+ guidance_scale: float = H3_IMAGE_GUIDANCE,
+ negative_prompt: Any = None,
+ num_images_per_prompt: int = 1,
+ generator: Any = None,
+ latents: Any = None,
+ output_type: str = "pil",
+ return_dict: bool = True,
+ callback_on_step_end: Optional[Callable] = None,
+ callback_on_step_end_tensor_inputs: Optional[list] = None,
+ min_generation_area: int = H3_IMAGE_MIN_GENERATION_AREA,
+ max_sequence_length: int = H3_IMAGE_MAX_SEQUENCE_LENGTH,
+ ) -> Any:
+ import torch
+
+ if num_inference_steps not in (None, H3_IMAGE_STEPS):
+ raise ValueError(
+ f"MiniMax-H3-Image is distilled to {H3_IMAGE_STEPS} steps with one output head each; "
+ f"got num_inference_steps={num_inference_steps}."
+ )
+ if guidance_scale is not None and float(guidance_scale) != H3_IMAGE_GUIDANCE:
+ raise ValueError(
+ "MiniMax-H3-Image is guidance-distilled: guidance_scale must be 1 (no CFG)."
+ )
+ if negative_prompt:
+ logger.info(
+ "diffusion.minimax_h3_image: negative prompt ignored (the model has no unconditional branch)"
+ )
+ prompts = [prompt] if isinstance(prompt, str) else list(prompt or [])
+ if not prompts or any(not isinstance(p, str) or not p for p in prompts):
+ raise ValueError("MiniMax-H3-Image needs a non-empty prompt.")
+ jobs = [p for p in prompts for _ in range(max(1, int(num_images_per_prompt or 1)))]
+ generators = generator if isinstance(generator, list) else [generator] * len(jobs)
+ if len(generators) != len(jobs):
+ raise ValueError(f"got {len(generators)} generators for {len(jobs)} images")
+ height = int(height or 2048)
+ width = int(width or 2048)
+ vae_refs, vision_refs = self.prepare_references(image)
+ self._interrupt = False
+ images = []
+ for index, (text, gen) in enumerate(zip(jobs, generators)):
+ pixels = self._render_one(
+ text,
+ vae_refs,
+ vision_refs,
+ height,
+ width,
+ gen,
+ latents if index == 0 else None,
+ min_generation_area,
+ callback_on_step_end,
+ callback_on_step_end_tensor_inputs or ["latents"],
+ output_type,
+ )
+ if pixels is None:
+ break
+ images.extend(pixels)
+ if output_type == "pt" and images and isinstance(images[0], torch.Tensor):
+ images = torch.stack(images)
+ if not return_dict:
+ return (images,)
+ return _image_output(images)
+
+ @torch_no_grad
+ def _render_one(
+ self,
+ prompt: str,
+ vae_refs: list,
+ vision_refs: list,
+ height: int,
+ width: int,
+ generator: Any,
+ latents: Any,
+ min_generation_area: int,
+ callback: Optional[Callable],
+ callback_inputs: list,
+ output_type: str,
+ ) -> Optional[list]:
+ import torch
+
+ stats: dict = {"canvas": None}
+ t0 = time.perf_counter()
+ device = self._device()
+ canvas_h, canvas_w = canvas_size(height, width, min_generation_area)
+ stats["canvas"] = (canvas_w, canvas_h)
+ embeds, text_tags = self._encode(prompt, vision_refs)
+ stats["encode_s"] = time.perf_counter() - t0
+ patch = tuple(self.transformer.config.patch_size)
+ channels = int(self.vae.config.latent_channels) if self.vae is not None else 24
+ lh, lw = canvas_h // H3_IMAGE_VAE_SCALE, canvas_w // H3_IMAGE_VAE_SCALE
+ shape = (1, channels, 1, lh, lw)
+ if latents is None:
+ from diffusers.utils.torch_utils import randn_tensor
+ noise = randn_tensor(shape, generator = generator, device = device, dtype = torch.float32)
+ else:
+ noise = latents.to(device, torch.float32).reshape(shape)
+ target_rows = patchify(noise, patch)
+ reference_latents = self._encode_references(vae_refs, generator) if vae_refs else []
+ grids = [tuple(r.shape[-2:]) for r in reference_latents]
+ pos, tags, image_idx, text_idx, n_ref = build_packed_layout(text_tags, grids, lh, lw, patch)
+ if reference_latents:
+ from diffusers.utils.torch_utils import randn_tensor
+
+ cond = []
+ for ref in reference_latents:
+ ref = ref.to(device, torch.float32)
+ ref_noise = randn_tensor(
+ ref.shape, generator = generator, device = device, dtype = torch.float32
+ )
+ noised = self.scheduler.scale_noise(
+ ref, torch.tensor(H3_IMAGE_REFERENCE_NOISE_AUG), ref_noise
+ )
+ cond.append(patchify(noised[:, :, None], patch))
+ cond = torch.cat(cond)
+ if cond.shape[0] != n_ref:
+ raise ValueError(
+ f"layout reserved {n_ref} reference rows, references packed {cond.shape[0]}"
+ )
+ rows = torch.cat([cond, target_rows])
+ else:
+ rows = target_rows
+ self.scheduler.set_timesteps(device = device)
+ timesteps = self.scheduler.timesteps
+ self._num_timesteps = len(timesteps)
+ seq = int(image_idx.numel() + text_idx.numel())
+ plan = row_timestep_plan(timesteps.cpu(), seq, image_idx[:n_ref])
+ pos_dtype = torch.float64 if device.type in ("cpu", "cuda") else torch.float32
+ fields = {
+ "position_ids": pos.to(device, pos_dtype),
+ "token_tags": tags.to(device),
+ "image_indices": image_idx.to(device),
+ "text_indices": text_idx.to(device),
+ }
+ transformer = self._enter("transformer")
+ t1 = time.perf_counter()
+ try:
+ dtype = _module_dtype(transformer, skip_fp32 = True)
+ prompt_embeds = embeds.to(device = device, dtype = dtype)
+ for i, t in enumerate(timesteps):
+ if self.interrupt:
+ return None
+ unique, index = plan[i]
+ velocity = transformer(
+ hidden_states = rows[None],
+ encoder_hidden_states = prompt_embeds,
+ timestep = unique.to(device),
+ timestep_indices = index.to(device),
+ pdd_step_index = i,
+ return_dict = False,
+ **fields,
+ )[0]
+ rows[n_ref:] = self.scheduler.step(velocity[0, n_ref:].float(), t, rows[n_ref:])[0]
+ if callback is not None:
+ view = unpatchify(rows[n_ref:], lh, lw, channels, patch)
+ payload = {"latents": view} if "latents" in callback_inputs else {}
+ callback(self, i, t, payload)
+ finally:
+ self._leave("transformer")
+ if device.type == "cuda":
+ torch.cuda.synchronize(device)
+ stats["denoise_s"] = time.perf_counter() - t1
+ t2 = time.perf_counter()
+ final = unpatchify(rows[n_ref:], lh, lw, channels, patch)
+ if output_type == "latent":
+ self._unsloth_h3_last_stats = stats
+ return [final[0]]
+ images = self._decode(final, height, width, output_type)
+ stats["decode_s"] = time.perf_counter() - t2
+ stats["total_s"] = time.perf_counter() - t0
+ self._unsloth_h3_last_stats = stats
+ return images
+
+ def _decode(self, latents: Any, height: int, width: int, output_type: str) -> list:
+ import numpy as np
+ import torch
+ from PIL import Image
+
+ device = self._device()
+ vae = self._enter("vae")
+ try:
+ lmean = torch.tensor(vae.config.latents_mean, device = device).view(1, -1, 1, 1)
+ lstd = torch.tensor(vae.config.latents_std, device = device).view(1, -1, 1, 1)
+ z = latents.to(device, torch.float32) * lstd + lmean
+ image = vae.decode(z.to(_module_dtype(vae)), return_dict = False)[0]
+ finally:
+ self._leave("vae")
+ mean = torch.tensor(H3_IMAGE_PIXEL_MEAN, device = image.device).view(1, -1, 1, 1)
+ std = torch.tensor(H3_IMAGE_PIXEL_STD, device = image.device).view(1, -1, 1, 1)
+ image = (image.float() * std + mean).clamp(0, 1)
+ # The release truncates to 8 bits, then resizes the 8-bit image (LANCZOS).
+ u8 = (image * 255).clamp(0, 255).to(torch.uint8).permute(0, 2, 3, 1).cpu().numpy()
+ out = []
+ for frame in u8:
+ pil = Image.fromarray(frame)
+ if pil.size != (width, height):
+ pil = pil.resize((width, height), resample = Image.LANCZOS)
+ out.append(pil)
+ if output_type == "pil":
+ return out
+ arrays = [np.asarray(p).astype(np.float32) / 255.0 for p in out]
+ if output_type == "np":
+ return arrays
+ return [torch.from_numpy(a).permute(2, 0, 1) for a in arrays]
+
+
+def _module_dtype(module: Any, skip_fp32: bool = False) -> Any:
+ import torch
+ for name, p in module.named_parameters():
+ if not p.is_floating_point():
+ continue
+ if (
+ skip_fp32
+ and p.dtype == torch.float32
+ and any(k in name for k in ("proj_in", "time_embedder", "pdd_proj_out", "rope"))
+ ):
+ continue
+ return p.dtype
+ return torch.float32
+
+
+# "1" lets the loader run on any device (the CPU tests, diagnostics); Studio refuses everything but CUDA / ROCm.
+ALLOW_ANY_DEVICE_ENV = "UNSLOTH_H3_IMAGE_ALLOW_ANY_DEVICE"
+
+
+def h3_image_device_refusal(device: Any) -> Optional[str]:
+ """The load refusal for a device this model cannot run on, else None. The 33B DiT and the 33B conditioner are
+ streamed and quantized through CUDA (NVIDIA, or AMD through ROCm); Apple Silicon and the CPU have no route yet."""
+ kind = str(getattr(device, "type", device) or "").split(":")[0].strip().lower()
+ if kind == "cuda" or os.environ.get(ALLOW_ANY_DEVICE_ENV, "").strip() == "1":
+ return None
+ where = {"mps": "Apple Silicon", "cpu": "the CPU", "": "this machine"}.get(
+ kind, f"this {kind} device"
+ )
+ return (
+ f"MiniMax-H3-Image needs an NVIDIA or AMD GPU and cannot run on {where} yet: its 33B image model and 33B "
+ f"text encoder are streamed and quantized through CUDA or ROCm. Load it on a machine with an NVIDIA or AMD "
+ f"GPU."
+ )
+
+
+def _free_tensor_storage(tensor: Any) -> bool:
+ """Swap ``tensor`` in place for an empty one, so its storage goes even while some other object still holds the
+ tensor (plain tensors through ``.data``, torchao subclasses through ``swap_tensors``)."""
+ import torch
+
+ empty = torch.empty(0, dtype = tensor.dtype)
+ try:
+ if type(tensor) in (torch.Tensor, torch.nn.Parameter) and type(tensor.data) is torch.Tensor:
+ tensor.data = empty
+ return True
+ replacement = (
+ torch.nn.Parameter(empty, requires_grad = False)
+ if isinstance(tensor, torch.nn.Parameter) or type(tensor) is not torch.Tensor
+ else empty
+ )
+ torch.utils.swap_tensors(tensor, replacement)
+ return True
+ except Exception: # noqa: BLE001 - teardown is best effort; dropping the module still frees an unshared tensor
+ return False
+
+
+def free_module_storage(module: Any) -> int:
+ """Empty every parameter and buffer of a component this pipeline owns, in place: an unload must not depend on
+ every outside reference (a retained frame, a cached closure) having let go of the module first."""
+ freed = 0
+ for sub in module.modules():
+ for table in (sub._parameters, sub._buffers):
+ for tensor in list(table.values()):
+ if tensor is not None and tensor.numel():
+ freed += int(_free_tensor_storage(tensor))
+ return freed
+
+
+def _empty_cache() -> None:
+ try:
+ import torch
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ except Exception: # noqa: BLE001 - best effort
+ pass
+
+
+def qwen3vl_hidden_state(
+ text_encoder: Any,
+ processor: Any,
+ token_ids: list,
+ vision_inputs: dict,
+ *,
+ layer: int,
+ device: Any,
+) -> Any:
+ """``hidden_states[layer]`` of the Qwen3-VL conditioner for one tokenized presentation. Calls
+ ``text_encoder.model`` (the head is never needed) and builds ``mm_token_type_ids`` only where the installed
+ transformers takes them (5.x); 4.57 derives the vision layout from the pad tokens."""
+ import torch
+
+ config = getattr(text_encoder.config, "text_config", text_encoder.config)
+ if int(config.num_hidden_layers) <= layer:
+ raise ValueError(
+ f"MiniMax-H3 conditions on hidden_states[{layer}], which needs more than {layer} decoder layers; "
+ f"the encoder has {config.num_hidden_layers}."
+ )
+ model = text_encoder.model
+ hook = getattr(text_encoder, "_hf_hook", None)
+ if hook is not None and hasattr(hook, "pre_forward"):
+ hook.pre_forward(text_encoder)
+ input_ids = torch.tensor([token_ids], dtype = torch.long, device = device)
+ kwargs: dict = {
+ "input_ids": input_ids,
+ "attention_mask": torch.ones_like(input_ids),
+ "use_cache": False,
+ "output_hidden_states": False,
+ }
+ params = inspect.signature(model.forward).parameters
+ if (
+ "mm_token_type_ids" in params
+ and processor is not None
+ and hasattr(processor, "create_mm_token_type_ids")
+ ):
+ kwargs["mm_token_type_ids"] = torch.tensor(
+ processor.create_mm_token_type_ids([token_ids]), dtype = torch.long, device = device
+ )
+ te_dtype = _module_dtype(text_encoder)
+ for name, value in (vision_inputs or {}).items():
+ kwargs[name] = value.to(device, te_dtype) if name.startswith("pixel_") else value.to(device)
+ # hidden_states[layer] is the output of decoder layer ``layer - 1``: captured by a hook instead of keeping every
+ # layer's states (~10 GB with 16 references), and the layers past it are cut for the call.
+ language_model = getattr(model, "language_model", None)
+ layers = getattr(language_model, "layers", None)
+ if layers is None or len(layers) < layer:
+ kwargs["output_hidden_states"] = True
+ return model(**kwargs).hidden_states[layer]
+ captured: dict = {}
+
+ def _capture(_module: Any, _args: Any, output: Any) -> None:
+ captured["h"] = output[0] if isinstance(output, (tuple, list)) else output
+ raise _ReadLayerReached()
+
+ handle = layers[layer - 1].register_forward_hook(_capture)
+ try:
+ model(**kwargs)
+ except _ReadLayerReached:
+ pass
+ finally:
+ handle.remove()
+ if "h" not in captured:
+ raise RuntimeError(f"the conditioner never ran decoder layer {layer - 1}")
+ return captured["h"]
+
+
+class _ReadLayerReached(Exception):
+ """Stops the conditioner once the read layer has run: nothing after it is used."""
+
+
+# Metadata a hosted table-form checkpoint declares (the prequant_repos extension point), mirroring video's curve form.
+ADALN_FORM_KEY = "adaln_form"
+ADALN_GRID_FORM = "grid_table"
+ADALN_GRID_KEY = "adaln_grid"
+
+
+def _hub_files(
+ repo_id: str,
+ patterns: Sequence[str],
+ *,
+ hf_token: Optional[str],
+ cache_dir: Optional[str],
+ local_files_only: bool,
+) -> Path:
+ """Local root holding ``patterns`` of ``repo_id`` (a local folder is used as is)."""
+ if os.path.isdir(str(repo_id)):
+ return Path(repo_id)
+ from huggingface_hub import snapshot_download
+
+ from .diffusion_families import pinned_revision_kwargs
+
+ root = snapshot_download(
+ repo_id,
+ allow_patterns = list(patterns),
+ token = hf_token,
+ cache_dir = cache_dir,
+ local_files_only = local_files_only,
+ max_workers = 8,
+ **pinned_revision_kwargs(repo_id),
+ )
+ return Path(root)
+
+
+def _read_json(path: Path) -> dict:
+ with open(path, "r", encoding = "utf-8") as f:
+ return json.load(f)
+
+
+def native_shard_map(transformer_dir: Path) -> dict[str, Path]:
+ """``{native key: shard path}`` from the index, else from every shard's header."""
+ index = transformer_dir / "model.safetensors.index.json"
+ if index.is_file():
+ weight_map = _read_json(index).get("weight_map") or {}
+ return {k: transformer_dir / v for k, v in weight_map.items()}
+ from safetensors import safe_open
+
+ out: dict[str, Path] = {}
+ for shard in sorted(transformer_dir.glob("*.safetensors")):
+ with safe_open(str(shard), framework = "pt", device = "cpu") as f:
+ for key in f.keys():
+ out[key] = shard
+ if not out:
+ raise FileNotFoundError(f"no transformer safetensors under {transformer_dir}")
+ return out
+
+
+class _ShardReader:
+ """Keeps one mmap'd ``safe_open`` per shard; ``get`` materialises a single tensor."""
+
+ def __init__(self, shard_map: dict[str, Path]) -> None:
+ self.map = shard_map
+ self._open: dict[Path, Any] = {}
+ self.bytes_read = 0
+
+ def get(self, key: str) -> Any:
+ from safetensors import safe_open
+
+ path = self.map[key]
+ handle = self._open.get(path)
+ if handle is None:
+ handle = safe_open(str(path), framework = "pt", device = "cpu")
+ self._open[path] = handle
+ tensor = handle.get_tensor(key)
+ self.bytes_read += tensor.numel() * tensor.element_size()
+ return tensor
+
+ def close(self) -> None:
+ self._open.clear()
+
+
+def _assign(module: Any, state: dict, *, where: str) -> None:
+ missing, unexpected = module.load_state_dict(state, strict = False, assign = True)
+ if unexpected:
+ raise KeyError(f"{where}: unexpected keys {sorted(unexpected)[:5]}")
+ stranded = [
+ n for n, p in list(module.named_parameters()) + list(module.named_buffers()) if p.is_meta
+ ]
+ if stranded:
+ raise KeyError(f"{where}: no checkpoint tensor for {sorted(stranded)[:5]}")
+
+
+def _cast(tensor: Any, key: str, dtype: Any) -> Any:
+ """Mixed precision as released: the fp32 modules stay fp32, the block stack follows ``dtype``."""
+ import torch
+
+ if not tensor.is_floating_point():
+ return tensor
+ if key.startswith(("proj_in.", "time_embedder.", "pdd_proj_out.")):
+ return tensor.to(torch.float32)
+ return tensor.to(dtype)
+
+
+def quantize_block_(block: Any, scheme: str, *, device: Any) -> int:
+ """Quantize the six big Linears of one block in place on ``device`` (ConvRot-rotated for int8)."""
+ from torch import nn
+ from torchao.quantization import quantize_
+
+ from .diffusion_transformer_quant import _make_quant_config
+
+ names = [
+ n
+ for n, m in block.named_modules()
+ if isinstance(m, nn.Linear) and n in QUANT_LINEAR_SUFFIXES
+ ]
+ if not names:
+ return 0
+ block.to(device)
+ if scheme == "int8":
+ from .diffusion_convrot import rotate_linears_
+
+ modules = dict(block.named_modules())
+ rotatable = [n for n in names if modules[n].in_features % CONVROT_GROUP == 0]
+ rotate_linears_(block, rotatable, CONVROT_GROUP)
+ wanted = set(names)
+ quantize_(block, _make_quant_config(scheme), filter_fn = lambda module, fqn: fqn in wanted)
+ wrap_row_multiple(block, names)
+ return len(names)
+
+
+# cuBLASLt int8 GEMMs (torch._int_mm) refuse some row counts with CUBLAS_STATUS_NOT_SUPPORTED, and torchao then
+# silently falls back to a much slower fp32 matmul. A packed sequence rarely lands on a supported count; multiples
+# of 32 always ran and also suit the fp8 row-wise GEMM.
+ROW_MULTIPLE = 32
+
+
+def _row_pad_class() -> Any:
+ from torch import nn
+ class H3ImageRowPad(nn.Module):
+ """Pads a quantized Linear's rows to a multiple of ``ROW_MULTIPLE`` by repeating row 0, then drops them. Exact:
+ every row's activation scale is its own (per-row dynamic quantization), so the kept rows are unchanged."""
+
+ def __init__(self, inner: Any) -> None:
+ super().__init__()
+ self.inner = inner
+
+ def forward(self, x: Any) -> Any:
+ import torch
+
+ lead = x.shape[:-1]
+ flat = x.reshape(-1, x.shape[-1])
+ m = flat.shape[0]
+ pad = (-m) % ROW_MULTIPLE
+ if pad and m:
+ flat = torch.cat([flat, flat[:1].expand(pad, -1)], dim = 0)
+ out = self.inner(flat)[:m]
+ else:
+ out = self.inner(flat)
+ return out.reshape(*lead, out.shape[-1])
+
+ def __getattr__(self, name: str) -> Any:
+ try:
+ return super().__getattr__(name)
+ except AttributeError:
+ if name == "inner":
+ raise
+ return getattr(self._modules["inner"], name)
+
+ return H3ImageRowPad
+
+
+_ROW_PAD_CLASS: Any = None
+
+
+def wrap_row_multiple(module: Any, names: Sequence[str]) -> int:
+ """Wrap each named quantized Linear (per-row activation scales only) in the row pad; returns the count."""
+ global _ROW_PAD_CLASS
+ from .diffusion_quant_pad import activation_granularity_is_per_row, is_quantized_linear
+
+ if _ROW_PAD_CLASS is None:
+ _ROW_PAD_CLASS = _row_pad_class()
+ wrapped = 0
+ for fqn in names:
+ parent_name, _, leaf = fqn.rpartition(".")
+ parent = module.get_submodule(parent_name) if parent_name else module
+ linear = getattr(parent, leaf)
+ if not is_quantized_linear(linear) or activation_granularity_is_per_row(linear) is not True:
+ continue
+ setattr(parent, leaf, _ROW_PAD_CLASS(linear))
+ wrapped += 1
+ return wrapped
+
+
+def resolve_dit_scheme(
+ requested: Optional[str],
+ *,
+ device: Any,
+ vram_gb: Optional[float],
+ width: int = 2048,
+ height: int = 2048,
+) -> tuple[Optional[str], str]:
+ """(scheme or None for bf16, reason). Auto: bf16 when the table-form bf16 DiT fits the card beside its working
+ set, else int8. fp8 needs SM 8.9+ (falls back to int8 elsewhere)."""
+ import torch
+
+ req = (requested or "auto").strip().lower()
+ if req in ("bf16", "none", "off", "dense"):
+ return None, "bf16 requested"
+ dev = torch.device(device)
+ if dev.type != "cuda":
+ if req in QUANT_SCHEMES:
+ return None, f"{req} needs a CUDA device; {dev.type} runs bf16"
+ return None, f"{dev.type}: bf16"
+ try:
+ import torchao # noqa: F401
+ except Exception: # noqa: BLE001 - torchao absent
+ return None, "torchao unavailable: bf16"
+ if req == "fp8":
+ cap = torch.cuda.get_device_capability(dev)
+ if tuple(cap) < (8, 9):
+ return (
+ "int8",
+ f"fp8 needs compute capability 8.9+ (this card is {cap[0]}.{cap[1]}): int8 instead",
+ )
+ return "fp8", "fp8 requested"
+ if req == "int8":
+ return "int8", "int8 requested"
+ need = H3_IMAGE_DIT_TABLE_BF16_GB + activation_gb(width, height) + 1.5
+ if vram_gb is not None and vram_gb >= need:
+ return None, f"auto: bf16 DiT ({H3_IMAGE_DIT_TABLE_BF16_GB} GB) fits {vram_gb:.1f} GB"
+ return (
+ "int8",
+ f"auto: bf16 DiT would not fit {vram_gb if vram_gb is not None else 0:.1f} GB, int8 ConvRot",
+ )
+
+
+def device_budgets(device: Any) -> tuple[Optional[float], Optional[float]]:
+ """(free VRAM GB on ``device`` or None, available host GB or None); env overrides win."""
+ import torch
+
+ vram = None
+ dev = torch.device(device)
+ if os.environ.get(VRAM_BUDGET_ENV):
+ vram = float(os.environ[VRAM_BUDGET_ENV])
+ elif dev.type == "cuda" and torch.cuda.is_available():
+ free, _total = torch.cuda.mem_get_info(dev)
+ vram = free / 1e9
+ host = None
+ if os.environ.get(HOST_BUDGET_ENV):
+ host = float(os.environ[HOST_BUDGET_ENV])
+ else:
+ try:
+ import psutil
+ host = psutil.virtual_memory().available / 1e9
+ except Exception: # noqa: BLE001 - psutil missing
+ host = None
+ return vram, host
+
+
+def build_transformer_skeleton(config: dict, *, table_grid: Optional[Sequence[float]]) -> Any:
+ """Meta-device model (buffers real), with the AdaLN table structure installed when ``table_grid`` is set."""
+ from accelerate import init_empty_weights
+
+ cls = transformer_class()
+ with init_empty_weights(include_buffers = False):
+ model = cls.from_config(config)
+ if table_grid is not None:
+ install_grid_time_embedding(model, table_grid)
+ return model
+
+
+def h3_image_prepare_prequant_model(logger_: Any = None) -> Callable:
+ """``prepare_model`` for ``load_prequantized_transformer``: reshapes the skeleton to a hosted checkpoint's AdaLN
+ form before ``load_state_dict``: the H3 video curve form (``adaln_form=curve``) or this module's table form
+ (``adaln_form=grid_table`` + its grid)."""
+
+ def _prepare(transformer: Any, metadata: Optional[dict]) -> None:
+ import torch
+
+ from .video_minimax_h3_adaln import apply_h3_adaln_curve, is_curve_checkpoint
+
+ if is_curve_checkpoint(metadata):
+ # The hosted H3-Image files use the H3 video curve form: the raw timestep indexes the curve, so the
+ # pipeline's float32 timesteps (0.999 reference rows included) drive it unchanged.
+ apply_h3_adaln_curve(transformer, metadata, logger = logger_)
+ return
+ if not isinstance(metadata, dict) or metadata.get(ADALN_FORM_KEY) != ADALN_GRID_FORM:
+ return
+ grid = metadata.get(ADALN_GRID_KEY)
+ if isinstance(grid, str):
+ grid = json.loads(grid)
+ grid = [float(t) for t in grid]
+ k = len(grid)
+ dtype = torch.bfloat16
+ for block in transformer.transformer_blocks:
+ out = block.adaln_proj.linear.out_features
+ convert_block_adaln_to_table(block, torch.empty(k, out, dtype = dtype))
+ out = transformer.norm_out.linear.out_features
+ convert_norm_out_to_table(transformer.norm_out, torch.empty(k, out, dtype = dtype))
+ install_grid_time_embedding(transformer, grid)
+ if logger_ is not None:
+ logger_.info(
+ "diffusion.minimax_h3_image: hosted checkpoint is table form over %d timesteps", k
+ )
+
+ return _prepare
+
+
[email protected]_cache(maxsize = None)
+def native_config_transformer_class() -> Any:
+ """The transformer class whose ``from_config`` also takes the release's native ``transformer/config.json``, for
+ loaders that read the base repo's config (``diffusion_prequant.load_prequantized_transformer``)."""
+ base = transformer_class()
+
+ class MiniMaxH3ImageNativeConfigTransformer(base):
+ @classmethod
+ def from_config(
+ cls,
+ config: Any = None,
+ **kwargs: Any,
+ ) -> Any:
+ cfg = dict(config or {})
+ if (
+ cfg.get("_class_name") == H3_IMAGE_NATIVE_TRANSFORMER_CLASS
+ or "ffn_hidden_size" in cfg
+ ):
+ cfg = transformer_config_from_native(cfg)
+ return base.from_config(cfg, **kwargs)
+
+ MiniMaxH3ImageNativeConfigTransformer.__name__ = base.__name__
+ return MiniMaxH3ImageNativeConfigTransformer
+
+
+# The hosted files also quantize the text-conditioning path (context_embedder and the two token_refiner blocks), which
+# the on-the-fly path keeps bf16: per-row dynamic activation quantization of the prompt embeddings moves the
+# conditioning every block reads. "1" keeps them quantized (A/B only).
+QUANT_CONDITIONING_ENV = "UNSLOTH_H3_IMAGE_QUANT_CONDITIONING"
+CONDITIONING_PREFIXES = ("context_embedder", "token_refiner.")
+
+
+def densify_conditioning_path(transformer: Any, dtype: Any = None) -> int:
+ """Replace the quantized weights of the text-conditioning Linears with their dequantized copies (bf16), so the
+ prompt embeddings are never activation-quantized. Rotated (ConvRot) Linears keep rotating the activation: the
+ dequantized weight is the rotated one, so the product is unchanged. Returns the count."""
+ import torch
+
+ from .diffusion_quant_pad import is_quantized_linear
+
+ if os.environ.get(QUANT_CONDITIONING_ENV, "").strip() == "1":
+ return 0
+ dtype = dtype or torch.bfloat16
+ count = 0
+ for name, module in transformer.named_modules():
+ if not name.startswith(CONDITIONING_PREFIXES) or not isinstance(module, torch.nn.Linear):
+ continue
+ if not is_quantized_linear(module):
+ continue
+ weight = module.weight
+ dense = weight.dequantize() if hasattr(weight, "dequantize") else None
+ if dense is None:
+ continue
+ module.weight = torch.nn.Parameter(dense.to(dtype), requires_grad = False)
+ count += 1
+ return count
+
+
+def hosted_prequant_available(
+ repo: str,
+ filename: Optional[str],
+ *,
+ hf_token: Optional[str],
+ cache_dir: Optional[str],
+ local_files_only: bool,
+) -> bool:
+ """Whether the hosted file exists: the local cache first, then (online) one Hub lookup per process. A failed
+ lookup reads as absent, so the load quantizes on the fly instead of failing."""
+ if not filename:
+ return False
+ try:
+ from huggingface_hub import try_to_load_from_cache
+ if isinstance(try_to_load_from_cache(repo, filename, cache_dir = cache_dir), str):
+ return True
+ except Exception: # noqa: BLE001 - a cache probe only
+ pass
+ if local_files_only:
+ return False
+ return _hub_file_exists(repo, filename, hf_token)
+
+
[email protected]_cache(maxsize = 32)
+def _hub_file_exists(repo: str, filename: str, hf_token: Optional[str]) -> bool:
+ try:
+ from huggingface_hub import file_exists
+ return bool(file_exists(repo, filename, token = hf_token))
+ except Exception: # noqa: BLE001 - offline / gated / not yet created: quantize on the fly
+ return False
+
+
+def load_hosted_prequant_transformer(
+ scheme: str,
+ *,
+ base: str,
+ device: Any,
+ dtype: Any,
+ hf_token: Optional[str],
+ cache_dir: Optional[str],
+ local_files_only: bool,
+ keep_on_device: bool,
+) -> Optional[Any]:
+ """A hosted pre-quantized DiT for ``scheme`` when the family declares one (``prequant_repos`` /
+ ``prequant_filenames``, the H3 video mechanism), else None and the caller quantizes on the fly.
+
+ The checkpoint is Studio's prequant container over the diffusers-layout model INCLUDING ``pdd_proj_out.*``, with a
+ curve-form or table-form AdaLN that ``h3_image_prepare_prequant_model`` installs on the skeleton before the state
+ dict is assigned. Nothing is fetched unless ``hosted_prequant_available`` finds the file."""
+ try:
+ from .diffusion_families import (
+ detect_family,
+ family_prequant_filename,
+ family_prequant_repo,
+ )
+
+ fam = detect_family(base) or detect_family("", override = H3_IMAGE_FAMILY_NAME)
+ if fam is None:
+ return None
+ repo = family_prequant_repo(fam, scheme, base)
+ if not repo:
+ return None
+ from .diffusion_prequant import PrequantSource, load_prequantized_transformer
+
+ filename = family_prequant_filename(fam, scheme)
+ if not hosted_prequant_available(
+ repo,
+ filename,
+ hf_token = hf_token,
+ cache_dir = cache_dir,
+ local_files_only = local_files_only,
+ ):
+ logger.info(
+ "diffusion.minimax_h3_image: no hosted %s DiT at %s/%s yet; quantizing the release on the fly",
+ scheme,
+ repo,
+ filename,
+ )
+ return None
+ source = PrequantSource(
+ kind = "repo",
+ location = repo,
+ filename = filename,
+ declared_filenames = (filename,) if filename else (),
+ )
+ return load_prequantized_transformer(
+ native_config_transformer_class(),
+ base,
+ source,
+ device = str(device),
+ dtype = dtype,
+ hf_token = hf_token,
+ scheme = scheme,
+ cache_dir = cache_dir,
+ prepare_model = h3_image_prepare_prequant_model(logger),
+ local_files_only = local_files_only,
+ logger = logger,
+ placement_device = str(device) if keep_on_device else "cpu",
+ family = H3_IMAGE_FAMILY_NAME,
+ )
+ except Exception as exc: # noqa: BLE001 - a hosted artifact is an optimisation: quantize on the fly instead
+ logger.warning(
+ "diffusion.minimax_h3_image: hosted %s DiT unusable (%s); quantizing on the fly",
+ scheme,
+ exc,
+ )
+ return None
+
+
+@dataclass
+class H3ImageLoadReport:
+ scheme: Optional[str]
+ scheme_reason: str
+ te_scheme: Optional[str]
+ plan: H3ImagePlan
+ classes: str
+ layout: str
+ table: bool
+ load_s: float = 0.0
+ quantized_linears: int = 0
+ bytes_read: int = 0
+
+ def as_dict(self) -> dict:
+ return {
+ "transformer_quant": self.scheme or "bf16",
+ "transformer_quant_reason": self.scheme_reason,
+ "text_encoder_quant": self.te_scheme or "bf16",
+ "classes": self.classes,
+ "layout": self.layout,
+ "adaln_table": self.table,
+ "load_s": round(self.load_s, 1),
+ "quantized_linears": self.quantized_linears,
+ "bytes_read": self.bytes_read,
+ **{f"plan_{k}": v for k, v in self.plan.as_dict().items()},
+ }
+
+
+def load_h3_image_transformer(
+ root: Path,
+ *,
+ dtype: Any,
+ scheme: Optional[str],
+ device: Any,
+ keep_on_device: bool,
+ table: bool = True,
+ check_cancelled: Optional[Callable[[], None]] = None,
+) -> tuple[Any, dict]:
+ """The DiT from the native ``transformer/`` + ``pdd_fused_heads.safetensors`` (or a converted
+ ``transformer_diffusers/``), one block at a time: convert, table the AdaLN, quantize on ``device``, then park on
+ the host unless ``keep_on_device``. Host peak = the finished blocks + one dense block."""
+ import torch
+ from safetensors import safe_open
+
+ check = check_cancelled or (lambda: None)
+ converted_dir = root / DIFFUSERS_TRANSFORMER_SUBFOLDER
+ native = not (converted_dir / "config.json").is_file()
+ tdir = root / ("transformer" if native else DIFFUSERS_TRANSFORMER_SUBFOLDER)
+ raw_config = _read_json(tdir / "config.json")
+ if native:
+ pdd = _read_json(root / PDD_CONFIG_FILE) if (root / PDD_CONFIG_FILE).is_file() else {}
+ config = transformer_config_from_native(
+ raw_config, int(pdd.get("num_inference_steps", H3_IMAGE_STEPS))
+ )
+ else:
+ config = {k: v for k, v in raw_config.items() if not k.startswith("_")}
+ heads, head_dim = int(config["num_attention_heads"]), int(config["attention_head_dim"])
+ grid = grid_timesteps() if table else None
+ model = build_transformer_skeleton(config, table_grid = None)
+ shard_map = native_shard_map(tdir)
+ reader = _ShardReader(shard_map)
+ target = torch.device(device)
+ park = torch.device("cpu") if not keep_on_device else target
+ stats = {"quantized_linears": 0, "layout": "native" if native else "diffusers"}
+
+ def converted(prefix_native: str) -> dict:
+ out = {}
+ for key in [k for k in shard_map if k.startswith(prefix_native)]:
+ tensor = reader.get(key)
+ pairs = (
+ convert_native_transformer_tensor(key, tensor, heads, head_dim)
+ if native
+ else [(key, tensor)]
+ )
+ for name, t in pairs:
+ out[name] = _cast(t, name, dtype)
+ return out
+
+ try:
+ top: dict = {}
+ block_prefix = "blocks." if native else "transformer_blocks."
+ for key in shard_map:
+ if key.startswith(block_prefix):
+ continue
+ tensor = reader.get(key)
+ pairs = (
+ convert_native_transformer_tensor(key, tensor, heads, head_dim)
+ if native
+ else [(key, tensor)]
+ )
+ for name, t in pairs:
+ top[name] = _cast(t, name, dtype)
+ if native:
+ heads_path = root / PDD_HEADS_FILE
+ index = root / "model_index.json"
+ if index.is_file():
+ name = (
+ (_read_json(index).get("_minimax_h3") or {}).get("pdd_fused_heads")
+ ) or PDD_HEADS_FILE
+ heads_path = root / name
+ with safe_open(str(heads_path), framework = "pt", device = "cpu") as f:
+ pdd = {k: f.get_tensor(k) for k in f.keys()}
+ for name, t in split_pdd_heads(pdd, int(config["num_pdd_heads"])).items():
+ top[name] = t.to(torch.float32)
+ blocks = model.transformer_blocks
+ model.transformer_blocks = torch.nn.ModuleList()
+ _assign(model, top, where = "transformer (top level)")
+ model.transformer_blocks = blocks
+ temb = grid_temb((model.time_proj, model.time_embedder), grid) if grid is not None else None
+ if temb is not None:
+ lin = model.norm_out.linear
+ rows = dense_adaln_rows(temb, lin.weight, lin.bias, dtype, target)
+ convert_norm_out_to_table(model.norm_out, rows)
+ check()
+ for index, block in enumerate(model.transformer_blocks):
+ native_prefix = f"blocks.{index}." if native else f"transformer_blocks.{index}."
+ state = converted(native_prefix)
+ local = {k[len(f"transformer_blocks.{index}.") :]: v for k, v in state.items()}
+ if temb is not None:
+ weight = local.pop("adaln_proj.linear.weight")
+ bias = local.pop("adaln_proj.linear.bias", None)
+ rows = dense_adaln_rows(temb, weight, bias, dtype, target)
+ del weight, bias
+ convert_block_adaln_to_table(block, rows)
+ _assign(block, local, where = f"transformer_blocks.{index}")
+ del state, local
+ if scheme in QUANT_SCHEMES:
+ stats["quantized_linears"] += quantize_block_(block, scheme, device = target)
+ block.to(park)
+ check()
+ if grid is not None:
+ install_grid_time_embedding(model, grid)
+ model.eval()
+ model.requires_grad_(False)
+ if keep_on_device:
+ model.to(target)
+ else:
+ for name, child in model.named_children():
+ if name != "transformer_blocks":
+ child.to(target if keep_on_device else "cpu")
+ stats["bytes_read"] = reader.bytes_read
+ finally:
+ reader.close()
+ return model, stats
+
+
+def load_h3_image_vae(root: Path, *, dtype: Any = None) -> Any:
+ """The image VAE from the native ``vae/`` (or a converted ``vae_diffusers/``), fp32 by default."""
+ import torch
+ from accelerate import init_empty_weights
+ from safetensors import safe_open
+
+ converted_dir = root / DIFFUSERS_VAE_SUBFOLDER
+ native = not (converted_dir / "config.json").is_file()
+ vdir = root / ("vae" if native else DIFFUSERS_VAE_SUBFOLDER)
+ raw = _read_json(vdir / "config.json")
+ config = (
+ vae_config_from_native(raw)
+ if native
+ else {k: v for k, v in raw.items() if not k.startswith("_")}
+ )
+ cls = vae_class()
+ with init_empty_weights(include_buffers = False):
+ vae = cls.from_config(config)
+ heads = int(config.get("decoder_num_attention_heads", 16))
+ head_dim = int(config.get("decoder_attention_head_dim", 64))
+ state = {}
+ files = sorted(vdir.glob("*.safetensors"))
+ if not files:
+ raise FileNotFoundError(f"no VAE safetensors under {vdir}")
+ for path in files:
+ with safe_open(str(path), framework = "pt", device = "cpu") as f:
+ for key in f.keys():
+ tensor = f.get_tensor(key)
+ pairs = (
+ convert_native_vae_tensor(key, tensor, heads, head_dim)
+ if native
+ else [(key, tensor)]
+ )
+ for name, t in pairs:
+ state[name] = t.to(dtype or torch.float32) if t.is_floating_point() else t
+ _assign(vae, state, where = "vae")
+ vae.eval()
+ vae.requires_grad_(False)
+ return vae
+
+
+def load_h3_image_text_encoder(
+ scheme: Optional[str],
+ *,
+ dtype: Any,
+ text_encoder_repo: str,
+ hf_token: Optional[str],
+ cache_dir: Optional[str],
+ local_files_only: bool,
+ fallback_check: Optional[Callable[[], None]] = None,
+) -> tuple[Any, Optional[str]]:
+ """(encoder on the host, scheme engaged). int8 = Studio's hosted ConvRot conditioner (the video family's file),
+ bf16 = the released encoder; a broken hosted file falls back to bf16 (66.7 GB). ``fallback_check`` runs before
+ that fallback allocates anything and raises when the plan cannot hold it."""
+ if scheme == "int8":
+ from .video_minimax_h3_te import load_h3_quantized_text_encoder
+
+ # __wrapped__: this backend owns the encoder's lifetime (the low-host tier frees it after each prompt).
+ loader = getattr(
+ load_h3_quantized_text_encoder, "__wrapped__", load_h3_quantized_text_encoder
+ )
+ encoder = loader(
+ text_encoder_repo,
+ "int8",
+ dtype = dtype,
+ hf_token = hf_token,
+ cache_dir = cache_dir,
+ local_files_only = local_files_only,
+ logger = logger,
+ )
+ if encoder is not None:
+ return encoder, "int8"
+ if fallback_check is not None:
+ fallback_check()
+ logger.warning(
+ "diffusion.minimax_h3_image: hosted int8 encoder unusable, loading the bf16 encoder"
+ )
+ import transformers
+
+ from .diffusion_families import pinned_revision_kwargs
+
+ encoder = transformers.Qwen3VLForConditionalGeneration.from_pretrained(
+ text_encoder_repo,
+ subfolder = "text_encoder",
+ torch_dtype = dtype,
+ token = hf_token,
+ cache_dir = cache_dir,
+ local_files_only = local_files_only,
+ low_cpu_mem_usage = True,
+ **pinned_revision_kwargs(text_encoder_repo),
+ )
+ encoder.eval()
+ return encoder, None
+
+
+def _place_streamed_dit(
+ transformer: Any, vae: Any, device: Any, plan: H3ImagePlan
+) -> ModuleListStreamer:
+ vae.to(device)
+ for name, child in transformer.named_children():
+ if name != "transformer_blocks":
+ child.to(device)
+ resident = int(plan.dit_resident_blocks or 0)
+ streamer = ModuleListStreamer(transformer.transformer_blocks, device, resident)
+ streamer._unsloth_role = "transformer"
+ streamer._unsloth_resident = resident
+ return streamer
+
+
+def _place_text_encoder(pipe: Any, encoder: Any, plan: H3ImagePlan, device: Any) -> Any:
+ if encoder is None:
+ return None
+ if plan.placement == PLACE_RESIDENT:
+ encoder.to(device)
+ elif plan.placement in (PLACE_STREAM, PLACE_DIT_RESIDENT):
+ layers = encoder.model.language_model.layers
+ for name, child in encoder.model.named_children():
+ if name != "language_model":
+ child.to(device)
+ for name, child in encoder.model.language_model.named_children():
+ if name != "layers":
+ child.to(device)
+ streamer = ModuleListStreamer(layers, device, int(plan.te_resident_layers or 0))
+ streamer._unsloth_role = "text_encoder"
+ pipe._unsloth_h3_streamers.append(streamer)
+ return encoder
+
+
+def load_minimax_h3_image_pipeline(
+ repo_id: str = H3_IMAGE_BASE_REPO,
+ *,
+ dtype: Any = None,
+ device: Any = "cuda",
+ transformer_quant: Optional[str] = None,
+ text_encoder_quant: Optional[str] = "int8",
+ text_encoder_repo: Optional[str] = None,
+ hf_token: Optional[str] = None,
+ cache_dir: Optional[str] = None,
+ local_files_only: bool = False,
+ check_cancelled: Optional[Callable[[], None]] = None,
+ text_encoder: Any = None,
+ transformer: Any = None,
+ plan_width: int = 2048,
+ plan_height: int = 2048,
+) -> Any:
+ """A ready ``MiniMaxH3ImagePipeline`` for ``repo_id`` (Hub id or local folder) placed for ``device``.
+
+ ``transformer_quant``: None/"auto" (bf16 when it fits, else int8), "int8", "fp8", "bf16". ``text_encoder_quant``:
+ "int8" (Studio's hosted ConvRot conditioner) or None (bf16). ``text_encoder`` / ``transformer`` hand in prebuilt
+ components (tests, a hosted prequant DiT)."""
+ import torch
+
+ t0 = time.perf_counter()
+ # What release() may empty in place: never a component the caller handed in.
+ owned = (
+ {"vae"}
+ | ({"transformer"} if transformer is None else set())
+ | ({"text_encoder"} if text_encoder is None else set())
+ )
+ check = check_cancelled or (lambda: None)
+ dtype = dtype or torch.bfloat16
+ device = torch.device(device)
+ vram, host = device_budgets(device)
+ patterns = [
+ "model_index.json",
+ PDD_CONFIG_FILE,
+ "transformer/config.json",
+ "vae/*",
+ "tokenizer/*",
+ "processor/*",
+ f"{DIFFUSERS_TRANSFORMER_SUBFOLDER}/config.json",
+ f"{DIFFUSERS_VAE_SUBFOLDER}/*",
+ ]
+ hub = {"hf_token": hf_token, "cache_dir": cache_dir, "local_files_only": local_files_only}
+ root = _hub_files(repo_id, patterns, **hub)
+ check()
+ index = _read_json(root / "model_index.json") if (root / "model_index.json").is_file() else {}
+ te_entry = index.get("text_encoder")
+ te_repo = text_encoder_repo
+ if (
+ te_repo is None
+ and isinstance(te_entry, list)
+ and len(te_entry) > 2
+ and isinstance(te_entry[2], dict)
+ ):
+ te_repo = te_entry[2].get("pretrained_model_name_or_path")
+ te_repo = te_repo or H3_IMAGE_TEXT_ENCODER_REPO
+ scheme, scheme_reason = resolve_dit_scheme(
+ transformer_quant, device = device, vram_gb = vram, width = plan_width, height = plan_height
+ )
+ te_scheme = "int8" if (text_encoder_quant or "").strip().lower() == "int8" else None
+ table = adaln_table_enabled()
+ plan_args = {
+ "vram_gb": vram if device.type == "cuda" else None,
+ "host_gb": host,
+ "dit_gb": dit_gb_for(scheme, table),
+ "width": plan_width,
+ "height": plan_height,
+ "forced": (os.environ.get(PLACEMENT_ENV) or "").strip().lower() or None,
+ }
+ plan = plan_placement(
+ te_gb = te_gb_for(te_scheme) if text_encoder is None else _module_gb(text_encoder),
+ **plan_args,
+ )
+ logger.info(
+ "diffusion.minimax_h3_image: dit=%s (%s) te=%s placement=%s reasons=%s",
+ scheme or "bf16",
+ scheme_reason,
+ te_scheme or "bf16",
+ plan.placement,
+ "; ".join(plan.reasons),
+ )
+ stats: dict = {"layout": "supplied", "quantized_linears": 0, "bytes_read": 0}
+ if transformer is None and scheme in QUANT_SCHEMES and not os.path.isdir(str(repo_id)):
+ transformer = load_hosted_prequant_transformer(
+ scheme,
+ base = repo_id,
+ device = device,
+ dtype = dtype,
+ hf_token = hf_token,
+ cache_dir = cache_dir,
+ local_files_only = local_files_only,
+ keep_on_device = plan.placement in (PLACE_RESIDENT, PLACE_DIT_RESIDENT),
+ )
+ if transformer is not None:
+ for block in transformer.transformer_blocks:
+ wrap_row_multiple(block, QUANT_LINEAR_SUFFIXES)
+ densified = densify_conditioning_path(transformer, dtype)
+ logger.info(
+ "diffusion.minimax_h3_image: hosted DiT, text-conditioning path kept dense (%d Linears)",
+ densified,
+ )
+ stats = {
+ "layout": "hosted_prequant",
+ "quantized_linears": 0,
+ "bytes_read": 0,
+ "dense_conditioning_linears": densified,
+ }
+ if transformer is None:
+ root = _hub_files(
+ repo_id,
+ [PDD_HEADS_FILE, "transformer/*", f"{DIFFUSERS_TRANSFORMER_SUBFOLDER}/*"],
+ **hub,
+ )
+ transformer, stats = load_h3_image_transformer(
+ root,
+ dtype = dtype,
+ scheme = scheme,
+ device = device,
+ keep_on_device = plan.placement in (PLACE_RESIDENT, PLACE_DIT_RESIDENT),
+ table = table,
+ check_cancelled = check,
+ )
+ check()
+ vae = load_h3_image_vae(root)
+ check()
+ early_streamer = None
+ if plan.placement == PLACE_STREAM:
+ # Place the streamed tier before any encoder is materialised: the host-memory plan counts only the DiT
+ # blocks left on the host, not the whole DiT + VAE the loaders park there.
+ early_streamer = _place_streamed_dit(transformer, vae, device, plan)
+ from transformers import AutoTokenizer
+
+ tokenizer = AutoTokenizer.from_pretrained(str(root / "tokenizer"))
+ processor = None
+ if (root / "processor").is_dir():
+ try:
+ from transformers import AutoProcessor
+ processor = AutoProcessor.from_pretrained(str(root / "processor"))
+ except Exception as exc: # noqa: BLE001 - t2i still works; editing reports the missing processor
+ logger.warning(
+ "diffusion.minimax_h3_image: processor unavailable, editing disabled (%s)", exc
+ )
+ te_kwargs = {
+ "dtype": dtype,
+ "text_encoder_repo": te_repo,
+ "hf_token": hf_token,
+ "cache_dir": cache_dir,
+ "local_files_only": local_files_only,
+ }
+ engaged_te = te_scheme if text_encoder is None else "supplied"
+
+ def _encoder_fits(te_gb: float, what: str) -> None:
+ again = replan_for_encoder(plan, te_gb, plan_args)
+ cpu_short = (
+ plan.vram_gb is None
+ and plan.host_gb is not None
+ and plan.host_gb < plan.dit_gb + te_gb + 1.0
+ )
+ if cpu_short or not again.te_kept_on_host:
+ raise RuntimeError(
+ f"MiniMax-H3-Image: {what} ({te_gb:.1f} GB) does not fit this machine's "
+ f"{plan.host_gb:.1f} GB of free system memory beside the model."
+ )
+
+ local_te = root / "text_encoder"
+ if text_encoder is None and (local_te / "config.json").is_file():
+ local_gb = sum(f.stat().st_size for f in local_te.glob("*.safetensors")) / 1e9
+ if local_gb > plan.te_gb + 1.0:
+ _encoder_fits(local_gb, "the text encoder in this model folder")
+ import transformers
+
+ text_encoder = transformers.Qwen3VLForConditionalGeneration.from_pretrained(
+ str(root / "text_encoder"), torch_dtype = dtype, low_cpu_mem_usage = True
+ ).eval()
+ engaged_te = None
+ if text_encoder is None and plan.te_kept_on_host:
+
+ def _bf16_fits() -> None:
+ _encoder_fits(
+ H3_IMAGE_TE_BF16_GB,
+ "Studio's hosted int8 text encoder could not be loaded (check the network or the Hugging Face "
+ "cache), and the bf16 encoder it would fall back to",
+ )
+
+ text_encoder, engaged_te = load_h3_image_text_encoder(
+ te_scheme, fallback_check = _bf16_fits, **te_kwargs
+ )
+ check()
+ if text_encoder is not None:
+ plan = replan_for_encoder(plan, _module_gb(text_encoder), plan_args)
+ pipe = MiniMaxH3ImagePipeline(
+ transformer = transformer,
+ vae = vae,
+ text_encoder = text_encoder,
+ tokenizer = tokenizer,
+ scheduler = MiniMaxH3ImageScheduler(sample_sigmas = list(H3_IMAGE_SIGMAS)),
+ processor = processor,
+ )
+ pipe._unsloth_h3_plan = plan
+ pipe._unsloth_h3_device = str(device)
+ pipe._unsloth_h3_owned = frozenset(owned)
+ if text_encoder is None:
+
+ def _refuse_bf16() -> None:
+ raise RuntimeError(
+ "MiniMax-H3-Image: Studio's hosted int8 text encoder could not be loaded, and the 66.7 GB bf16 "
+ "encoder it would fall back to does not fit this machine's memory plan (it loads the encoder "
+ "for each prompt because host memory is short). Check the network or the Hugging Face cache "
+ "and load the model again."
+ )
+
+ def _te_loader() -> Any:
+ encoder, engaged = load_h3_image_text_encoder(
+ te_scheme, fallback_check = _refuse_bf16, **te_kwargs
+ )
+ if te_scheme is not None and engaged != te_scheme:
+ # Placed under this plan, the bf16 fallback would overrun the budget the plan made for the int8 one.
+ free_module_storage(encoder)
+ raise RuntimeError(
+ "MiniMax-H3-Image: Studio's hosted int8 text encoder could not be loaded, and the 66.7 GB bf16 "
+ "encoder it would fall back to does not fit this machine's memory plan (it loads the encoder "
+ "for each prompt because host memory is short). Check the network or the Hugging Face cache "
+ "and load the model again."
+ )
+ return _place_text_encoder(pipe, encoder, plan, device)
+
+ pipe._unsloth_h3_te_loader = _te_loader
+ else:
+ _place_text_encoder(pipe, text_encoder, plan, device)
+ if plan.placement != PLACE_SEQUENTIAL:
+ vae.to(device)
+ if plan.placement in (PLACE_RESIDENT, PLACE_DIT_RESIDENT):
+ transformer.to(device)
+ elif plan.placement == PLACE_SEQUENTIAL:
+ transformer.to("cpu")
+ elif plan.placement == PLACE_STREAM:
+ resident = int(plan.dit_resident_blocks or 0)
+ if early_streamer is None or early_streamer._unsloth_resident != resident:
+ if early_streamer is not None:
+ early_streamer.remove()
+ early_streamer = _place_streamed_dit(transformer, vae, device, plan)
+ pipe._unsloth_h3_streamers.append(early_streamer)
+ early_streamer = None
+ if early_streamer is not None:
+ early_streamer.remove()
+ report = H3ImageLoadReport(
+ scheme = scheme,
+ scheme_reason = scheme_reason,
+ te_scheme = engaged_te,
+ plan = plan,
+ classes = classes_source(),
+ layout = stats.get("layout", "native"),
+ table = table and stats.get("layout") not in ("supplied", "hosted_prequant"),
+ load_s = time.perf_counter() - t0,
+ quantized_linears = int(stats.get("quantized_linears", 0)),
+ bytes_read = int(stats.get("bytes_read", 0)),
+ )
+ pipe._unsloth_h3_report = report
+ try:
+ transformer._unsloth_runtime_quant = scheme
+ except Exception: # noqa: BLE001 - marker only
+ pass
+ logger.info("diffusion.minimax_h3_image: loaded %s", report.as_dict())
+ return pipe
+
+
+def _module_gb(module: Any) -> float:
+ total = 0
+ for t in list(module.parameters()) + list(module.buffers()):
+ total += t.numel() * t.element_size()
+ return total / 1e9
diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.py
index 91055ac719f..13bde5d2ae6 100644
--- a/studio/backend/core/inference/video_families.py
+++ b/studio/backend/core/inference/video_families.py
@@ -1101,7 +1101,17 @@ _FAMILIES: tuple[VideoFamily, ...] = (
# named "fastwan", "i2v" or "fun" is not such a checkpoint), and "fun" only right after the Wan2.1 token, where every Fun
# release puts it.
_WAN21_TOKEN = r"(?<![a-z0-9])wan-?2-?1(?![0-9])"
+_H3_IMAGE_NOT_VIDEO = (
+ "MiniMax-H3-Image is a text-to-image and image-editing model, not a video model. Load it on the Images page "
+ "instead."
+)
_UNSUPPORTED_VIDEO_CHECKPOINTS: tuple[tuple[str, str], ...] = (
+ (
+ # MiniMax-H3-Image carries the "minimax-h3" token of the video family but is an image model (no audio, PDD
+ # heads): it loads on the Images page, never as a video pick.
+ r"(?:^|[/\\])(?=[^/\\]*minimax[-_ .]?h3[-_ .]?image)[^/\\]*[/\\]*$",
+ _H3_IMAGE_NOT_VIDEO,
+ ),
(
r"(?:^|[/\\])(?=[^/\\]*(?<![a-z0-9])fastwan(?![a-z]))[^/\\]*[/\\]*$",
"This is a FastVideo FastWan checkpoint. FastWan is distilled for FastVideo's own few-step "
@@ -1163,6 +1173,8 @@ def unsupported_video_checkpoint_reason(
for pattern, message in _UNSUPPORTED_VIDEO_CHECKPOINTS:
if re.search(pattern, needle):
return message
+ if any(is_minimax_h3_image_folder(str(identifier)) for identifier in identifiers if identifier):
+ return _H3_IMAGE_NOT_VIDEO
return None
@@ -1213,6 +1225,54 @@ def _detect_video_family_memo() -> dict:
return memo
+# What a MiniMax-H3-Image folder declares about itself (the image partition of the H3 release: no audio, PDD heads).
+_H3_IMAGE_CLASSES = frozenset(
+ (
+ "MiniMaxH3ImagePipeline",
+ "MiniMaxH3ImageModularPipeline",
+ "MiniMaxH3ImageBlocks",
+ "MiniMaxH3ImageTransformer2DModel",
+ )
+)
+
+
+def is_minimax_h3_image_folder(path: Optional[str]) -> bool:
+ """True for a local folder whose own metadata says MiniMax-H3-Image, whatever the folder is called: its
+ model_index / modular index (class, blocks, or the release's ``_minimax_h3.partition == "image"``) or its
+ diffusers transformer config. Such a folder is an Images-page model even though it shares the video family's
+ "minimax-h3" / "h3" tokens."""
+ import json
+
+ root = (path or "").strip()
+ if not root or not os.path.isdir(root):
+ return False
+ for name in (
+ "model_index.json",
+ "modular_model_index.json",
+ os.path.join("transformer_diffusers", "config.json"),
+ os.path.join("transformer", "config.json"),
+ ):
+ try:
+ target = os.path.join(root, name)
+ if not os.path.isfile(target) or os.path.getsize(target) > 1_000_000:
+ continue
+ with open(target, encoding = "utf-8-sig") as handle:
+ payload = json.load(handle)
+ except (OSError, ValueError, RecursionError):
+ continue
+ if not isinstance(payload, dict):
+ continue
+ if (
+ payload.get("_class_name") in _H3_IMAGE_CLASSES
+ or payload.get("_blocks_class_name") in _H3_IMAGE_CLASSES
+ ):
+ return True
+ h3 = payload.get("_minimax_h3")
+ if isinstance(h3, dict) and str(h3.get("partition", "")).strip().lower() == "image":
+ return True
+ return False
+
+
def _detect_video_family_by_name(repo_id: str) -> Optional[VideoFamily]:
# I2V / VACE / FastWan names can carry a T2V token (Wan2.1_T2V_14B_FusionX_VACE): no T2V pipeline over those weights.
if unsupported_video_checkpoint_reason(repo_id):
diff --git a/studio/backend/tests/_minimax_h3_image_tiny.py b/studio/backend/tests/_minimax_h3_image_tiny.py
new file mode 100644
index 00000000000..076334891cc
--- /dev/null
+++ b/studio/backend/tests/_minimax_h3_image_tiny.py
@@ -0,0 +1,343 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A tiny random MiniMax-H3-Image release in the NATIVE layout (MiniMaxH3DiTModel keys, per-head interleaved fused
+qkv, [gate; value] fc1, stacked PDD heads, native VAE keys), built offline from a random diffusers-layout model, plus
+a tiny Qwen3-VL conditioner (51 layers: the DiT reads hidden_states[50]) and an offline tokenizer / processor."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+TINY_TRANSFORMER_NATIVE = {
+ "_class_name": "MiniMaxH3DiTModel",
+ "hidden_size": 64,
+ "num_layers": 2,
+ "token_refiner_num_layers": 2,
+ "num_attention_heads": 2,
+ "attention_head_dim": 32,
+ "ffn_hidden_size": 128,
+ "latents_dim": 24,
+ "audio_latents_dim": 32,
+ "patch_size": [1, 2, 2],
+ "text_dim": 48,
+ "timestep_input_dim": 16,
+ "time_embed_hidden_size": 64,
+ "time_embed_dim": 32,
+ "rope_inv_freq_len": 4,
+ "norm_eps": 1e-05,
+ "qk_norm_eps": 1e-05,
+ "final_norm_eps": 1e-05,
+ "audio_enabled": False,
+}
+
+TINY_VAE_NATIVE = {
+ "_class_name": "MiniMaxH3ImageVAE",
+ "latent_channels": 24,
+ "latents_mean": [0.1 * (i % 5) for i in range(24)],
+ "latents_std": [1.0 + 0.05 * i for i in range(24)],
+ "vae_decoder_tiling": 1,
+ "vae_encoder_tiling": 1,
+ "vae_tile_overlap_min": 64,
+ "vae_tile_size": 256,
+ "vit_decoder_2d": True,
+ "vit_decoder_kwargs": {
+ "dim_head": 32,
+ "heads": 2,
+ "num_layers": 2,
+ "rope_dim_ratio": 0.5,
+ "rope_theta": 100.0,
+ },
+ # Not in the release (it uses the class defaults): shrinks the CNN encoder for CPU tests, 16x like the release.
+ "block_out_channels": [32, 32, 32, 32],
+ "layers_per_block": 1,
+ "spatial_downsample_factors": [2, 2, 2, 2],
+ "temporal_downsample_factors": [1, 2, 2, 1],
+}
+
+SPECIAL = ["<|endoftext|>", "<|vision_start|>", "<|vision_end|>", "<|image_pad|>", "<|video_pad|>"]
+
+
+def interleave_qkv(q: Any, k: Any, v: Any, heads: int, head_dim: int) -> Any:
+ """Inverse of the converter's reorder: [q_all; k_all; v_all] -> per-head [q k v]."""
+ import torch
+
+ parts = [t.reshape(heads, head_dim, *t.shape[1:]) for t in (q, k, v)]
+ return torch.cat(parts, dim = 1).reshape(heads * 3 * head_dim, *q.shape[1:])
+
+
+def to_native_transformer(state: dict, heads: int, head_dim: int) -> dict:
+ """diffusers-layout DiT state -> the release's native keys (the inverse of the PR converter)."""
+ import torch
+
+ out = {}
+ done = set()
+ renames = [
+ ("time_embedder.linear_1.", "time_embedder.proj_in."),
+ ("time_embedder.linear_2.", "time_embedder.proj_out."),
+ ("proj_in.", "video_patch_proj."),
+ ("context_embedder.", "condition_proj."),
+ ("norm_out.norm.", "final_layer.norm."),
+ ("norm_out.linear.", "final_layer.adaln_proj.linear."),
+ (".attn.norm_q.", ".attn.q_norm."),
+ (".attn.norm_k.", ".attn.k_norm."),
+ (".attn.to_out.0.", ".attn.out_proj."),
+ (".ff.net.2.", ".mlp.fc2."),
+ ]
+ for key, tensor in state.items():
+ if key in done or key.startswith("pdd_proj_out."):
+ continue
+ name = key
+ if name.startswith("token_refiner.refiner_blocks."):
+ name = "token_refiner.blocks." + name[len("token_refiner.refiner_blocks.") :]
+ elif name.startswith("transformer_blocks."):
+ name = "blocks." + name[len("transformer_blocks.") :]
+ if key.endswith(".attn.to_q.weight"):
+ prefix = key[: -len("to_q.weight")]
+ q, k, v = (
+ state[prefix + "to_q.weight"],
+ state[prefix + "to_k.weight"],
+ state[prefix + "to_v.weight"],
+ )
+ done.update({prefix + "to_k.weight", prefix + "to_v.weight"})
+ native_prefix = name[: -len("to_q.weight")]
+ out[native_prefix + "qkv_proj.weight"] = interleave_qkv(q, k, v, heads, head_dim)
+ continue
+ if key.endswith((".attn.to_k.weight", ".attn.to_v.weight")):
+ continue
+ if key.endswith(".ff.net.0.proj.weight"):
+ value, gate = tensor.chunk(2, dim = 0)
+ out[name.replace(".ff.net.0.proj.weight", ".mlp.fc1.weight")] = torch.cat(
+ [gate, value]
+ ).contiguous()
+ continue
+ for old, new in renames:
+ if name.startswith(old) or old.startswith("."):
+ name = name.replace(old, new) if old.startswith(".") else new + name[len(old) :]
+ out[name] = tensor.contiguous()
+ # The release ships its rope buffer and a base output head that the PDD heads replace.
+ return out
+
+
+def to_native_vae(state: dict, heads: int, head_dim: int) -> dict:
+ import torch
+
+ out = {}
+ for key, tensor in state.items():
+ if ".attn.to_k." in key or ".attn.to_v." in key:
+ continue
+ if ".attn.to_q." in key:
+ prefix, suffix = key.split(".attn.to_q.")
+ q = tensor
+ k = state[f"{prefix}.attn.to_k.{suffix}"]
+ v = state[f"{prefix}.attn.to_v.{suffix}"]
+ out[f"{prefix}.attn.to_qkv.{suffix}"] = interleave_qkv(q, k, v, heads, head_dim)
+ continue
+ name = key
+ if name.startswith("encoder.down_blocks."):
+ level, rest = name[len("encoder.down_blocks.") :].split(".", 1)
+ rest = rest.replace("resnets.", "block.", 1).replace(
+ "conv_shortcut.", "nin_shortcut.", 1
+ )
+ rest = rest.replace("downsamplers.0.", "downsample.", 1)
+ name = f"encoder.down.{level}.{rest}"
+ name = name.replace("decoder.proj_in.", "decoder.x_embedder.")
+ name = name.replace(".attn.to_out.0.", ".attn.to_out.")
+ if ".ff.net.0.proj." in name:
+ up, gate = tensor.chunk(2, dim = 0)
+ tensor = torch.cat([gate, up]).contiguous()
+ name = name.replace(".ff.net.0.proj.", ".ff.w1.")
+ name = name.replace(".ff.net.2.", ".ff.w2.")
+ out[name] = tensor.contiguous()
+ return out
+
+
+def randomize(
+ module: Any,
+ seed: int,
+ std: float = 0.05,
+) -> None:
+ import torch
+ g = torch.Generator().manual_seed(seed)
+ with torch.no_grad():
+ for p in module.parameters():
+ p.copy_(torch.randn(p.shape, generator = g, dtype = torch.float32).to(p.dtype) * std)
+
+
+def make_tokenizer_and_processor(out: Path) -> tuple[Any, Any]:
+ from tokenizers import Tokenizer, models, pre_tokenizers
+ from transformers import PreTrainedTokenizerFast
+
+ words = [chr(c) for c in range(32, 127)] + ["Picture", ":", "the", "a", "cat", "<", ">"]
+ vocab = {w: i for i, w in enumerate(dict.fromkeys(SPECIAL + words + ["[UNK]"]))}
+ tok = Tokenizer(models.WordLevel(vocab = vocab, unk_token = "[UNK]"))
+ tok.pre_tokenizer = pre_tokenizers.Split(pattern = " ", behavior = "isolated")
+ fast = PreTrainedTokenizerFast(tokenizer_object = tok, unk_token = "[UNK]")
+ fast.add_special_tokens({"additional_special_tokens": SPECIAL})
+ (out / "tokenizer").mkdir(parents = True, exist_ok = True)
+ fast.save_pretrained(str(out / "tokenizer"))
+ processor = None
+ try:
+ from transformers import Qwen3VLProcessor
+ from transformers.models.qwen2_vl.image_processing_qwen2_vl import Qwen2VLImageProcessor
+
+ image_processor = Qwen2VLImageProcessor(
+ patch_size = 16,
+ temporal_patch_size = 2,
+ merge_size = 2,
+ min_pixels = 32 * 32,
+ max_pixels = 2048 * 2048,
+ )
+ try:
+ from transformers.models.qwen3_vl.video_processing_qwen3_vl import Qwen3VLVideoProcessor
+ video = Qwen3VLVideoProcessor()
+ except Exception: # noqa: BLE001 - older transformers
+ video = None
+ kwargs = {"image_processor": image_processor, "tokenizer": fast}
+ if video is not None:
+ kwargs["video_processor"] = video
+ processor = Qwen3VLProcessor(**kwargs)
+ (out / "processor").mkdir(parents = True, exist_ok = True)
+ processor.save_pretrained(str(out / "processor"))
+ except Exception: # noqa: BLE001 - editing tests skip without it
+ processor = None
+ return fast, processor
+
+
+def make_text_encoder(
+ tokenizer: Any,
+ text_dim: int = 48,
+ layers: int = 51,
+ seed: int = 3,
+) -> Any:
+ import torch
+ from transformers import Qwen3VLConfig, Qwen3VLForConditionalGeneration
+
+ ids = {t: tokenizer.convert_tokens_to_ids(t) for t in SPECIAL}
+ config = Qwen3VLConfig(
+ text_config = {
+ "hidden_size": text_dim,
+ "intermediate_size": 64,
+ "num_hidden_layers": layers,
+ "num_attention_heads": 2,
+ "num_key_value_heads": 1,
+ "head_dim": 24,
+ "vocab_size": len(tokenizer),
+ "max_position_embeddings": 8192,
+ "rope_scaling": {
+ "rope_type": "default",
+ "mrope_section": [4, 4, 4],
+ "mrope_interleaved": True,
+ },
+ },
+ vision_config = {
+ "depth": 2,
+ "hidden_size": 32,
+ "intermediate_size": 64,
+ "num_heads": 2,
+ "out_hidden_size": text_dim,
+ "patch_size": 16,
+ "spatial_merge_size": 2,
+ "temporal_patch_size": 2,
+ "num_position_embeddings": 64,
+ "deepstack_visual_indexes": [0],
+ },
+ image_token_id = ids["<|image_pad|>"],
+ video_token_id = ids["<|video_pad|>"],
+ vision_start_token_id = ids["<|vision_start|>"],
+ vision_end_token_id = ids["<|vision_end|>"],
+ )
+ torch.manual_seed(seed)
+ model = Qwen3VLForConditionalGeneration(config).eval()
+ randomize(model, seed, std = 0.05)
+ return model
+
+
+def make_tiny_release(out: Path, seed: int = 0) -> dict:
+ """Writes the release layout under ``out``; returns the diffusers-layout reference modules."""
+ import torch
+ from safetensors.torch import save_file
+
+ from core.inference import diffusion_minimax_h3_image as h3i
+
+ out.mkdir(parents = True, exist_ok = True)
+ cfg = h3i.transformer_config_from_native(TINY_TRANSFORMER_NATIVE)
+ torch.manual_seed(seed)
+ dense = h3i.transformer_class()(**cfg).eval()
+ randomize(dense, seed)
+ # Release precision: bf16 block stack, fp32 embedders / heads.
+ state = {}
+ for k, v in dense.state_dict().items():
+ if k.startswith("rope."):
+ continue
+ state[k] = (
+ v.to(torch.float32)
+ if k.startswith(("proj_in.", "time_embedder.", "pdd_proj_out."))
+ else v.to(torch.bfloat16)
+ )
+ dense.load_state_dict(state, strict = False, assign = True)
+ native = to_native_transformer(state, cfg["num_attention_heads"], cfg["attention_head_dim"])
+ native["rope.inv_freq"] = (
+ dense.rope.inv_freq.clone() if hasattr(dense.rope, "inv_freq") else torch.zeros(4)
+ )
+ hidden, patch_dim = cfg["hidden_size"], 24 * 4
+ native["final_layer.video_out.weight"] = torch.zeros(patch_dim, hidden)
+ native["final_layer.video_out.bias"] = torch.zeros(patch_dim)
+ tdir = out / "transformer"
+ tdir.mkdir(exist_ok = True)
+ keys = sorted(native)
+ half = len(keys) // 2
+ weight_map = {}
+ for i, chunk in enumerate((keys[:half], keys[half:])):
+ name = f"model-0000{i + 1}-of-00002.safetensors"
+ save_file({k: native[k].contiguous() for k in chunk}, str(tdir / name))
+ weight_map.update({k: name for k in chunk})
+ (tdir / "model.safetensors.index.json").write_text(json.dumps({"weight_map": weight_map}))
+ (tdir / "config.json").write_text(json.dumps(TINY_TRANSFORMER_NATIVE))
+ heads = cfg["num_pdd_heads"]
+ save_file(
+ {
+ "video_out.weight": torch.stack(
+ [state[f"pdd_proj_out.{i}.weight"] for i in range(heads)]
+ ).contiguous(),
+ "video_out.bias": torch.stack(
+ [state[f"pdd_proj_out.{i}.bias"] for i in range(heads)]
+ ).contiguous(),
+ },
+ str(out / h3i.PDD_HEADS_FILE),
+ )
+ (out / h3i.PDD_CONFIG_FILE).write_text(
+ json.dumps({"num_inference_steps": 6, "sigmas": list(h3i.H3_IMAGE_SIGMAS)})
+ )
+ (out / "model_index.json").write_text(
+ json.dumps(
+ {
+ "_class_name": "MiniMaxH3ImagePipeline",
+ "_minimax_h3": {"pdd_fused_heads": h3i.PDD_HEADS_FILE, "tasks": ["t2i", "edit"]},
+ "transformer": ["diffusers", "MiniMaxH3DiTModel"],
+ "vae": ["diffusers", "MiniMaxH3ImageVAE"],
+ "text_encoder": [
+ "transformers",
+ "Qwen3VLForConditionalGeneration",
+ {
+ "pretrained_model_name_or_path": "MiniMaxAI/MiniMax-H3",
+ "subfolder": "text_encoder",
+ },
+ ],
+ }
+ )
+ )
+ vcfg = h3i.vae_config_from_native(TINY_VAE_NATIVE)
+ vae = h3i.vae_class()(**vcfg).eval()
+ randomize(vae, seed + 1, std = 0.02)
+ vdir = out / "vae"
+ vdir.mkdir(exist_ok = True)
+ save_file(
+ {k: v.contiguous() for k, v in to_native_vae(vae.state_dict(), 2, 32).items()},
+ str(vdir / "diffusion_pytorch_model.safetensors"),
+ )
+ (vdir / "config.json").write_text(json.dumps(TINY_VAE_NATIVE))
+ tokenizer, processor = make_tokenizer_and_processor(out)
+ return {"transformer": dense, "vae": vae, "tokenizer": tokenizer, "processor": processor}
diff --git a/studio/backend/tests/data/minimax_h3_image/convert_minimax_h3_to_diffusers.py.txt b/studio/backend/tests/data/minimax_h3_image/convert_minimax_h3_to_diffusers.py.txt
new file mode 100644
index 00000000000..1eab993099f
--- /dev/null
+++ b/studio/backend/tests/data/minimax_h3_image/convert_minimax_h3_to_diffusers.py.txt
@@ -0,0 +1,1230 @@
+"""Convert an original MiniMax-H3 checkpoint into the diffusers layout.
+
+The transformer checkpoint is streamed shard by shard, so peak memory stays close to a single shard (~4.9 GiB) and
+never approaches the 62 GiB of the full 33B DiT.
+
+Every source key maps onto a diffusers module by renaming alone, except for three things:
+
+* the fused QKV projection is split into `to_q` / `to_k` / `to_v`. Raw checkpoint shards store QKV per-head
+ interleaved; the shard streamer first reorders them into the reference's in-memory `[q_all; k_all; v_all]` layout
+ (`reorder_interleaved_qkv`, mirroring the reference's load-time transform), then `convert_transformer_key` splits
+ contiguous thirds (`split_fused_qkv`),
+* the gated FFN's fused `mlp.fc1` becomes `ff.net.0.proj` and its two halves are swapped, because diffusers'
+ [`SwiGLU`] reads `[value; gate]` where the reference stores `[gate; value]` (the same transform the video VAE needs,
+ see `convert_video_vae_key`); `mlp.fc2` becomes `ff.net.2`,
+* `rope.inv_freq` is dropped: it is a pure function of `rope_theta` and `rope_freq_dim`, the port recomputes it in a
+ non-persistent buffer, and the recomputed value is bitwise equal to the shipped one for both released variants.
+
+There are no transposes anywhere.
+
+The FL2VA and Ref2VA variants differ only in the transformer weights, so the variant is selected by pointing
+`--checkpoint_path` at the corresponding folder. Both land in one repository, which carries a single
+`modular_model_index.json`: MiniMax-H3 is integrated as Modular Diffusers blocks only, so no `model_index.json` is
+written.
+
+`--version image` converts a MiniMax-H3-Image release instead: the same transformer without the audio projections,
+with the per-step output heads of `pdd_fused_heads.safetensors` folded in as `pdd_proj_out.{i}` in place of the base
+head, the image VAE, a scheduler config and a `modular_model_index.json`. The release keeps its original `transformer/`
+and `vae/` at the repository root, so the converted ones are written to `transformer_diffusers/` and `vae_diffusers/`
+and can be published next to them. The release conditions on the MiniMax-H3 text encoder without carrying it, so
+`--text_encoder_path` names a local copy that is copied in: the converted repository is self-contained.
+
+Usage:
+
+```bash
+# Validate the key mapping without any weights present.
+python scripts/convert_minimax_h3_to_diffusers.py \
+ --checkpoint_path /path/to/MiniMax-H3/FL2VA --output_path /tmp/h3-diffusers --dry_run
+
+# Convert, and point the component loading specs at the Hub id the result is published under.
+python scripts/convert_minimax_h3_to_diffusers.py \
+ --checkpoint_path /path/to/MiniMax-H3/FL2VA --output_path /tmp/h3-diffusers \
+ --modular_repo_id MiniMaxAI/MiniMax-H3
+
+# Convert a MiniMax-H3-Image release.
+python scripts/convert_minimax_h3_to_diffusers.py --version image \
+ --checkpoint_path /path/to/MiniMax-H3-Image --output_path /tmp/h3-image-diffusers \
+ --text_encoder_path /path/to/MiniMax-H3/text_encoder --modular_repo_id MiniMaxAI/MiniMax-H3-Image
+```
+"""
+
+import argparse
+import glob
+import json
+import math
+import os
+import shutil
+import struct
+from typing import Any
+
+import torch
+from safetensors import safe_open
+from safetensors.torch import save_file
+
+from diffusers.models.autoencoders.autoencoder_kl_minimax_h3_audio import AutoencoderKLMiniMaxH3Audio
+from diffusers.utils.constants import SAFE_WEIGHTS_INDEX_NAME
+
+
+# `MiniMaxH3Transformer3DModel` argument names. The original config uses the sglang-native names listed in the
+# comments; everything else in the original config (`adaln_out_features`, `final_adaln_out_features`) is derived.
+MINIMAX_H3_TRANSFORMER_CONFIG = {
+ "num_attention_heads": 56,
+ "attention_head_dim": 128,
+ "hidden_size": 5376,
+ "num_layers": 50,
+ "num_refiner_layers": 2, # token_refiner_num_layers
+ "ffn_dim": 14336, # ffn_hidden_size
+ "in_channels": 24, # latents_dim
+ "audio_in_channels": 32, # audio_latents_dim
+ "patch_size": [1, 2, 2],
+ "text_dim": 5120,
+ "freq_dim": 256, # timestep_input_dim
+ "time_embed_hidden_dim": 5376, # time_embed_hidden_size
+ "time_embed_dim": 2688,
+ "rope_freq_dim": 16, # rope_inv_freq_len
+ "rope_theta": 10000.0,
+ "norm_eps": 1e-05,
+ "qk_norm_eps": 1e-05,
+ "final_norm_eps": 1e-05,
+}
+
+# A tiny configuration with the checkpoint-tied dimensions left intact, for building fixtures.
+MINIMAX_H3_TEST_TRANSFORMER_CONFIG = {
+ **MINIMAX_H3_TRANSFORMER_CONFIG,
+ "num_attention_heads": 2,
+ "attention_head_dim": 32,
+ "hidden_size": 64,
+ "num_layers": 2,
+ "num_refiner_layers": 2,
+ "ffn_dim": 128,
+ "text_dim": 48,
+ "freq_dim": 16,
+ "time_embed_hidden_dim": 64,
+ "time_embed_dim": 32,
+ "rope_freq_dim": 4,
+}
+
+# `MiniMaxH3ImageTransformer2DModel` argument names. MiniMax-H3-Image is the same DiT without the audio input
+# projection and output head (`audio_enabled: false`), plus one output head per step of its 6-step Parallel Decoding
+# Distillation (PDD) schedule (`pdd_fused_heads.safetensors`).
+MINIMAX_H3_IMAGE_TRANSFORMER_CONFIG = {
+ **{name: value for name, value in MINIMAX_H3_TRANSFORMER_CONFIG.items() if name != "audio_in_channels"},
+ "num_pdd_heads": 6, # pdd_config.json num_inference_steps
+}
+
+# MiniMax-H3 ships a mixed-precision checkpoint. These *original* keys are float32; everything else is bfloat16 —
+# including the AdaLN projections.
+MINIMAX_H3_FP32_SOURCE_PREFIXES = (
+ "video_patch_proj.",
+ "audio_patch_proj.",
+ "time_embedder.",
+ "final_layer.video_out.",
+ "final_layer.audio_out.",
+)
+
+# `rope.inv_freq` is `1 / rope_theta ** (arange(0, 2 * rope_freq_dim, 2) / (2 * rope_freq_dim))`, which
+# `MiniMaxH3RotaryPosEmbed` recomputes into a non-persistent buffer. The recomputed tensor is bitwise equal to the
+# shipped one in both released variants, so the key is not carried into the diffusers checkpoint.
+MINIMAX_H3_TRANSFORMER_DROPPED_KEYS = ("rope.inv_freq",)
+
+
+def reorder_interleaved_qkv(weight: torch.Tensor, num_attention_heads: int, attention_head_dim: int) -> torch.Tensor:
+ """Reorder a *raw-checkpoint* per-head-interleaved fused QKV weight into `[q_all; k_all; v_all]`.
+
+ The original checkpoint shards store rows as `[head0: q(head_dim) k(head_dim) v(head_dim), head1: q, k, v, ...]`.
+ The reference implementation applies exactly this reorder at load time (`_reorder_grouped_qkv_to_qkv` with one head
+ per query group), so `[q_all; k_all; v_all]` is the reference's in-memory / state-dict layout. There is no
+ transpose.
+ """
+ expected_rows = num_attention_heads * 3 * attention_head_dim
+ if weight.shape[0] != expected_rows:
+ raise ValueError(
+ f"fused qkv weight has {weight.shape[0]} rows, expected "
+ f"{expected_rows} = {num_attention_heads} heads * 3 * {attention_head_dim}."
+ )
+ grouped = weight.reshape(num_attention_heads, 3 * attention_head_dim, *weight.shape[1:])
+ query, key, value = grouped.split(attention_head_dim, dim=1)
+ return torch.cat(
+ [
+ tensor.reshape(num_attention_heads * attention_head_dim, *weight.shape[1:])
+ for tensor in (query, key, value)
+ ],
+ dim=0,
+ )
+
+
+def split_fused_qkv(
+ weight: torch.Tensor, num_attention_heads: int, attention_head_dim: int
+) -> tuple[torch.Tensor, ...]:
+ """Split a fused `[q_all; k_all; v_all]` QKV weight into separate `to_q` / `to_k` / `to_v` weights.
+
+ The input is the *reference model* layout — what `MiniMaxH3DiTModel.state_dict()` holds after the reference's
+ load-time reorder — i.e. the three logical projection matrices stacked contiguously, NOT the raw checkpoint's
+ per-head interleave (see `reorder_interleaved_qkv`, which the shard streamer applies first).
+ """
+ inner_dim = num_attention_heads * attention_head_dim
+ if weight.shape[0] != 3 * inner_dim:
+ raise ValueError(
+ f"fused qkv weight has {weight.shape[0]} rows, expected "
+ f"{3 * inner_dim} = 3 * {num_attention_heads} heads * {attention_head_dim}."
+ )
+ query, key, value = weight.split(inner_dim, dim=0)
+ return tuple(tensor.contiguous() for tensor in (query, key, value))
+
+
+def get_transformer_key_plan(config: dict[str, Any]) -> dict[str, list[tuple[str, list[int]]]]:
+ """Map every original transformer key to the diffusers key(s) it produces, with the resulting shapes.
+
+ The plan is derived from the config alone, so it can be printed and checked without any weights present.
+ """
+ hidden_size = config["hidden_size"]
+ heads = config["num_attention_heads"]
+ head_dim = config["attention_head_dim"]
+ inner_dim = heads * head_dim
+ ffn_dim = config["ffn_dim"]
+ time_embed_dim = config["time_embed_dim"]
+ video_patch_dim = (
+ config["in_channels"] * config["patch_size"][0] * config["patch_size"][1] * config["patch_size"][2]
+ )
+
+ plan: dict[str, list[tuple[str, list[int]]]] = {
+ "video_patch_proj.weight": [("proj_in.weight", [hidden_size, video_patch_dim])],
+ "video_patch_proj.bias": [("proj_in.bias", [hidden_size])],
+ "condition_proj.weight": [("context_embedder.weight", [hidden_size, config["text_dim"]])],
+ "condition_proj.bias": [("context_embedder.bias", [hidden_size])],
+ # `Timesteps` + `TimestepEmbedding` reproduce the reference sinusoid and MLP exactly, so the timestep MLP is
+ # renamed onto `TimestepEmbedding`'s `linear_1` / `linear_2`.
+ "time_embedder.proj_in.weight": [
+ ("time_embedder.linear_1.weight", [config["time_embed_hidden_dim"], config["freq_dim"]])
+ ],
+ "time_embedder.proj_in.bias": [("time_embedder.linear_1.bias", [config["time_embed_hidden_dim"]])],
+ "time_embedder.proj_out.weight": [
+ ("time_embedder.linear_2.weight", [time_embed_dim, config["time_embed_hidden_dim"]])
+ ],
+ "time_embedder.proj_out.bias": [("time_embedder.linear_2.bias", [time_embed_dim])],
+ "token_refiner.final_norm.weight": [("token_refiner.final_norm.weight", [hidden_size])],
+ "final_layer.norm.weight": [("norm_out.norm.weight", [hidden_size])],
+ "final_layer.adaln_proj.linear.weight": [("norm_out.linear.weight", [2 * hidden_size, time_embed_dim])],
+ "final_layer.adaln_proj.linear.bias": [("norm_out.linear.bias", [2 * hidden_size])],
+ "final_layer.video_out.weight": [("proj_out.weight", [video_patch_dim, hidden_size])],
+ "final_layer.video_out.bias": [("proj_out.bias", [video_patch_dim])],
+ }
+ if config.get("audio_in_channels") is not None:
+ plan.update(
+ {
+ "audio_patch_proj.weight": [("audio_proj_in.weight", [hidden_size, config["audio_in_channels"]])],
+ "audio_patch_proj.bias": [("audio_proj_in.bias", [hidden_size])],
+ "final_layer.audio_out.weight": [
+ ("audio_proj_out.weight", [config["audio_in_channels"], hidden_size])
+ ],
+ "final_layer.audio_out.bias": [("audio_proj_out.bias", [config["audio_in_channels"]])],
+ }
+ )
+ for key in MINIMAX_H3_TRANSFORMER_DROPPED_KEYS:
+ plan[key] = []
+ if config.get("num_pdd_heads") is not None:
+ # A PDD release ships its per-step heads outside `transformer/` (see `convert_pdd_heads`) and runs every step
+ # with one of them, so the checkpoint's own `video_out` head is never used.
+ plan["final_layer.video_out.weight"] = []
+ plan["final_layer.video_out.bias"] = []
+
+ block_specs = [
+ ("blocks", "transformer_blocks", config["num_layers"], True),
+ ("token_refiner.blocks", "token_refiner.refiner_blocks", config["num_refiner_layers"], False),
+ ]
+ for source_prefix, target_prefix, num_layers, has_adaln in block_specs:
+ for i in range(num_layers):
+ source = f"{source_prefix}.{i}"
+ target = f"{target_prefix}.{i}"
+ plan[f"{source}.norm1.weight"] = [(f"{target}.norm1.weight", [hidden_size])]
+ plan[f"{source}.norm2.weight"] = [(f"{target}.norm2.weight", [hidden_size])]
+ plan[f"{source}.attn.qkv_proj.weight"] = [
+ (f"{target}.attn.to_q.weight", [inner_dim, hidden_size]),
+ (f"{target}.attn.to_k.weight", [inner_dim, hidden_size]),
+ (f"{target}.attn.to_v.weight", [inner_dim, hidden_size]),
+ ]
+ plan[f"{source}.attn.q_norm.weight"] = [(f"{target}.attn.norm_q.weight", [head_dim])]
+ plan[f"{source}.attn.k_norm.weight"] = [(f"{target}.attn.norm_k.weight", [head_dim])]
+ plan[f"{source}.attn.out_proj.weight"] = [(f"{target}.attn.to_out.0.weight", [hidden_size, inner_dim])]
+ # `fc1` stays fused, as diffusers' `SwiGLU` also fuses its two projections, but the halves are swapped
+ # from `[gate; value]` to `[value; gate]` (see `convert_transformer_key`).
+ plan[f"{source}.mlp.fc1.weight"] = [(f"{target}.ff.net.0.proj.weight", [2 * ffn_dim, hidden_size])]
+ plan[f"{source}.mlp.fc2.weight"] = [(f"{target}.ff.net.2.weight", [hidden_size, ffn_dim])]
+ if has_adaln:
+ plan[f"{source}.adaln_proj.linear.weight"] = [
+ (f"{target}.adaln_proj.linear.weight", [6 * 3 * hidden_size, time_embed_dim])
+ ]
+ plan[f"{source}.adaln_proj.linear.bias"] = [
+ (f"{target}.adaln_proj.linear.bias", [6 * 3 * hidden_size])
+ ]
+
+ return plan
+
+
+def convert_transformer_key(
+ source_key: str, tensor: torch.Tensor, config: dict[str, Any]
+) -> list[tuple[str, torch.Tensor]]:
+ """Convert one original key/tensor pair into the diffusers key/tensor pair(s) it maps to."""
+ if source_key in MINIMAX_H3_TRANSFORMER_DROPPED_KEYS:
+ return []
+ if config.get("num_pdd_heads") is not None and source_key.startswith("final_layer.video_out."):
+ # Replaced by the per-step PDD heads (see `get_transformer_key_plan`).
+ return []
+
+ target_key = source_key
+ if target_key.startswith("token_refiner.blocks."):
+ target_key = target_key.replace("token_refiner.blocks.", "token_refiner.refiner_blocks.", 1)
+ elif target_key.startswith("blocks."):
+ target_key = target_key.replace("blocks.", "transformer_blocks.", 1)
+ target_key = target_key.replace("time_embedder.proj_in.", "time_embedder.linear_1.")
+ target_key = target_key.replace("time_embedder.proj_out.", "time_embedder.linear_2.")
+ target_key = target_key.replace("video_patch_proj.", "proj_in.")
+ target_key = target_key.replace("audio_patch_proj.", "audio_proj_in.")
+ target_key = target_key.replace("condition_proj.", "context_embedder.")
+ target_key = target_key.replace("final_layer.norm.", "norm_out.norm.")
+ target_key = target_key.replace("final_layer.adaln_proj.linear.", "norm_out.linear.")
+ target_key = target_key.replace("final_layer.video_out.", "proj_out.")
+ target_key = target_key.replace("final_layer.audio_out.", "audio_proj_out.")
+ target_key = target_key.replace(".attn.q_norm.", ".attn.norm_q.")
+ target_key = target_key.replace(".attn.k_norm.", ".attn.norm_k.")
+ target_key = target_key.replace(".attn.out_proj.", ".attn.to_out.0.")
+
+ if target_key.endswith(".attn.qkv_proj.weight"):
+ # `convert_transformer_key` consumes tensors in the reference model's state-dict layout, where the fused QKV
+ # rows are already `[q_all; k_all; v_all]`. Raw checkpoint shards are per-head interleaved instead; the shard
+ # streamer (`convert_transformer`) normalizes them with `reorder_interleaved_qkv` before calling this.
+ query, key, value = split_fused_qkv(tensor, config["num_attention_heads"], config["attention_head_dim"])
+ prefix = target_key.removesuffix("qkv_proj.weight")
+ return [(f"{prefix}to_q.weight", query), (f"{prefix}to_k.weight", key), (f"{prefix}to_v.weight", value)]
+
+ if target_key.endswith(".mlp.fc1.weight"):
+ # The reference computes `fc2(silu(gate) * value)` from a fused `[gate; value]`; diffusers' `SwiGLU` computes
+ # `value * silu(gate)` from a fused `[value; gate]`, so the two halves swap places. Identical transform to the
+ # video VAE's `ff.w1` (see `convert_video_vae_key`).
+ gate, value = tensor.chunk(2, dim=0)
+ target_key = target_key.replace(".mlp.fc1.weight", ".ff.net.0.proj.weight")
+ return [(target_key, torch.cat([value, gate], dim=0).contiguous())]
+
+ target_key = target_key.replace(".mlp.fc2.", ".ff.net.2.")
+ return [(target_key, tensor)]
+
+
+#
+# ---------------------------------------------------------------------------------------------------------------
+# Video VAE
+# ---------------------------------------------------------------------------------------------------------------
+#
+
+# `AutoencoderKLMiniMaxH3` argument names. Field-for-field equal to `video_vae/source/config.json`, with the original
+# names in the comments. The keys that only ever take one value in the release (`use_3d_conv`, `use_vit_decoder`,
+# `causal_encoder`, `causal_decoder`, `use_t_isolated_gn`, `space_up` / `time_up`, `zq_ch_*`, `num_res_blocks_decoder`,
+# `shift_factor` / `scaling_factor`) are baked into the port instead of being config knobs.
+MINIMAX_H3_VIDEO_VAE_CONFIG = {
+ "in_channels": 3,
+ "out_channels": 3, # out_ch
+ "latent_channels": 24, # z_channels == embed_dim
+ "block_out_channels": [128, 256, 256, 512, 512, 1024], # ch * ch_mult
+ "layers_per_block": 2, # num_res_blocks
+ "spatial_downsample_factors": [2, 2, 2, 2, 1, 1], # space_down
+ "temporal_downsample_factors": [1, 2, 2, 1, 1, 1], # time_down
+ "norm_num_groups": 32,
+ "norm_eps": 1e-06,
+ "spatial_padding_mode": "reflect", # padding_mode
+ "decoder_num_layers": 36, # vit_decoder_kwargs.num_layers
+ "decoder_num_attention_heads": 32, # vit_decoder_kwargs.heads
+ "decoder_attention_head_dim": 64, # vit_decoder_kwargs.dim_head
+ "decoder_num_register_tokens": 4, # ViT3DDecoder default
+ "decoder_ffn_mult": 4, # FeedForward default
+ "decoder_rope_theta": 100.0, # vit_decoder_kwargs.rope_theta
+ "decoder_rope_dim_ratio": 0.75, # vit_decoder_kwargs.rope_dim_ratio
+ "decoder_norm_eps": 1e-05, # ViT3DDecoder eps
+ "clip_length": 17, # video_vae/config.json vae_clip_length
+ "token_drop": 3, # video_vae/config.json vae_token_drop
+}
+
+# A tiny configuration with the checkpoint-tied dimensions (`latent_channels`, the temporal geometry and the rotary
+# ratio) left intact, for building fixtures and for the CPU parity check.
+MINIMAX_H3_TEST_VIDEO_VAE_CONFIG = {
+ **MINIMAX_H3_VIDEO_VAE_CONFIG,
+ "block_out_channels": [32, 64],
+ "layers_per_block": 1,
+ "spatial_downsample_factors": [2, 2],
+ "temporal_downsample_factors": [2, 2],
+ "decoder_num_layers": 4,
+ "decoder_num_attention_heads": 4,
+ "decoder_attention_head_dim": 32,
+}
+
+# `decoder.mask_token` is an all-zero buffer belonging to the masked-autoencoding training objective; the released
+# decoder never reads it, so the port does not carry the module and the conversion drops the key.
+MINIMAX_H3_VIDEO_VAE_DROPPED_KEYS = ("decoder.mask_token",)
+
+
+def convert_video_vae_key(
+ source_key: str, tensor: torch.Tensor, config: dict[str, Any]
+) -> list[tuple[str, torch.Tensor]]:
+ """Convert one original video-VAE key/tensor pair into the diffusers key/tensor pair(s) it maps to.
+
+ `quant_conv` / `post_quant_conv`, the encoder's `conv_in` / `norm_out` / `conv_out` and the ViT decoder's
+ `register_tokens` / `norm_out` / `proj_out` / `norm{1,2}` / `scale{1,2}` are pure pass-throughs. What moves:
+
+ * the encoder's CNN levels are renamed from the original CompVis spelling onto the diffusers autoencoder idiom:
+ `down.{i}.block.{j}` -> `down_blocks.{i}.resnets.{j}`, `nin_shortcut` -> `conv_shortcut`, and
+ `down.{i}.downsample` -> `down_blocks.{i}.downsamplers.0`,
+ * the ViT decoder's `x_embedder` becomes `proj_in`, the counterpart of the `proj_out` it already ships,
+ * the fused per-head-interleaved `attn.to_qkv` is split into `attn.to_q` / `to_k` / `to_v`,
+ * `attn.to_out` becomes `attn.to_out.0` (diffusers wraps the output projection in an `nn.ModuleList`),
+ * the gated FFN's `w1` / `w2` become `ff.net.0.proj` / `ff.net.2`, and the two halves of `w1` are swapped because
+ diffusers' `SwiGLU` reads `[up; gate]` where the reference stores `[gate; up]`.
+ """
+ if source_key in MINIMAX_H3_VIDEO_VAE_DROPPED_KEYS:
+ return []
+
+ if ".attn.to_qkv." in source_key:
+ # Same per-head interleave as the DiT: `[head0: q k v, head1: q k v, ...]`.
+ reordered = reorder_interleaved_qkv(
+ tensor, config["decoder_num_attention_heads"], config["decoder_attention_head_dim"]
+ )
+ query, key, value = split_fused_qkv(
+ reordered, config["decoder_num_attention_heads"], config["decoder_attention_head_dim"]
+ )
+ prefix, suffix = source_key.split(".attn.to_qkv.")
+ return [
+ (f"{prefix}.attn.to_q.{suffix}", query),
+ (f"{prefix}.attn.to_k.{suffix}", key),
+ (f"{prefix}.attn.to_v.{suffix}", value),
+ ]
+
+ target_key = _rename_video_vae_key(source_key)
+
+ if ".ff.w1." in source_key:
+ gate, up = tensor.chunk(2, dim=0)
+ return [(target_key, torch.cat([up, gate], dim=0).contiguous())]
+
+ return [(target_key, tensor)]
+
+
+def _rename_video_vae_key(source_key: str) -> str:
+ """Rename one original video-VAE key onto its diffusers module path (no tensor transform)."""
+ target_key = source_key
+ if target_key.startswith("encoder.down."):
+ level, rest = target_key.removeprefix("encoder.down.").split(".", 1)
+ rest = rest.replace("block.", "resnets.", 1).replace("nin_shortcut.", "conv_shortcut.", 1)
+ rest = rest.replace("downsample.", "downsamplers.0.", 1)
+ target_key = f"encoder.down_blocks.{level}.{rest}"
+ target_key = target_key.replace("decoder.x_embedder.", "decoder.proj_in.")
+ target_key = target_key.replace(".attn.to_out.", ".attn.to_out.0.")
+ target_key = target_key.replace(".ff.w1.", ".ff.net.0.proj.")
+ target_key = target_key.replace(".ff.w2.", ".ff.net.2.")
+ return target_key
+
+
+def get_video_vae_key_plan(config: dict[str, Any]) -> dict[str, list[str]]:
+ """Map every original video-VAE key to the diffusers key(s) it produces, derived from the config alone."""
+ block_out_channels = config["block_out_channels"]
+ block_in_channels = [block_out_channels[0]] + block_out_channels[:-1]
+ plan: dict[str, list[str]] = {}
+
+ def renamed(*keys: str) -> None:
+ """Register keys whose diffusers name is `_rename_video_vae_key(key)` and whose tensor is unchanged."""
+ for key in keys:
+ plan[key] = [_rename_video_vae_key(key)]
+
+ renamed("quant_conv.weight", "quant_conv.bias", "post_quant_conv.weight", "post_quant_conv.bias")
+ renamed("encoder.conv_in.weight", "encoder.conv_in.bias")
+ for level, (in_channels, out_channels) in enumerate(zip(block_in_channels, block_out_channels)):
+ for i in range(config["layers_per_block"]):
+ prefix = f"encoder.down.{level}.block.{i}"
+ for name in ("norm1", "conv1", "norm2", "conv2"):
+ renamed(f"{prefix}.{name}.weight", f"{prefix}.{name}.bias")
+ if (in_channels if i == 0 else out_channels) != out_channels:
+ renamed(f"{prefix}.nin_shortcut.weight", f"{prefix}.nin_shortcut.bias")
+ if config["spatial_downsample_factors"][level] * config["temporal_downsample_factors"][level] > 1:
+ renamed(f"encoder.down.{level}.downsample.conv.weight", f"encoder.down.{level}.downsample.conv.bias")
+ renamed("encoder.norm_out.weight", "encoder.norm_out.bias", "encoder.conv_out.weight", "encoder.conv_out.bias")
+
+ renamed("decoder.x_embedder.weight", "decoder.x_embedder.bias", "decoder.register_tokens")
+ renamed("decoder.norm_out.weight", "decoder.norm_out.bias", "decoder.proj_out.weight", "decoder.proj_out.bias")
+ for i in range(config["decoder_num_layers"]):
+ prefix = f"decoder.transformer_blocks.{i}"
+ renamed(f"{prefix}.norm1.weight", f"{prefix}.norm2.weight", f"{prefix}.scale1", f"{prefix}.scale2")
+ for suffix in ("weight", "bias"):
+ plan[f"{prefix}.attn.to_qkv.{suffix}"] = [
+ f"{prefix}.attn.to_q.{suffix}",
+ f"{prefix}.attn.to_k.{suffix}",
+ f"{prefix}.attn.to_v.{suffix}",
+ ]
+ plan[f"{prefix}.attn.to_out.{suffix}"] = [f"{prefix}.attn.to_out.0.{suffix}"]
+ plan[f"{prefix}.ff.w1.{suffix}"] = [f"{prefix}.ff.net.0.proj.{suffix}"]
+ plan[f"{prefix}.ff.w2.{suffix}"] = [f"{prefix}.ff.net.2.{suffix}"]
+ for key in MINIMAX_H3_VIDEO_VAE_DROPPED_KEYS:
+ plan[key] = []
+ return plan
+
+
+def convert_video_vae(
+ checkpoint_path: str, output_path: str, config: dict[str, Any], diffusers_version: str, max_shard_size: int
+) -> None:
+ """Convert the video VAE and emit its config.
+
+ The original weights live one level deeper than the rest of the checkpoint
+ (`video_vae/source/model.safetensors`, resolved by a hook in the reference); the diffusers layout flattens that to
+ `vae/`. `latents_mean` / `latents_std` and the tiling geometry come from `video_vae/config.json` — MiniMax-H3
+ normalizes latents per channel instead of with a `scaling_factor`.
+ """
+ source_dir = os.path.join(checkpoint_path, "video_vae")
+ with open(os.path.join(source_dir, "config.json")) as f:
+ wrapper_config = json.load(f)
+ for key in ("latents_mean", "latents_std"):
+ if key not in wrapper_config:
+ raise KeyError(f"{source_dir}/config.json does not carry `{key}`.")
+
+ weights_path = os.path.join(source_dir, wrapper_config["source_path"], wrapper_config["source_safetensors_path"])
+ plan = get_video_vae_key_plan(config)
+
+ os.makedirs(output_path, exist_ok=True)
+ weight_map: dict[str, str] = {}
+ total_size = 0
+ written: list[str] = []
+ buffer: dict[str, torch.Tensor] = {}
+ buffer_size = 0
+ seen_source_keys: set[str] = set()
+
+ def flush() -> None:
+ nonlocal buffer, buffer_size
+ if not buffer:
+ return
+ path = os.path.join(output_path, f".tmp-shard-{len(written):05d}.safetensors")
+ save_file(buffer, path, metadata={"format": "pt"})
+ for key in buffer:
+ weight_map[key] = path
+ written.append(path)
+ buffer = {}
+ buffer_size = 0
+
+ # `safe_open` memory-maps the file, so only the tensor being read is materialized.
+ with safe_open(weights_path, framework="pt", device="cpu") as f:
+ for source_key in f.keys():
+ if source_key not in plan:
+ raise KeyError(f"Unexpected key in {weights_path}: {source_key}")
+ seen_source_keys.add(source_key)
+ for target_key, tensor in convert_video_vae_key(source_key, f.get_tensor(source_key), config):
+ if tensor.dtype != torch.float32:
+ raise ValueError(f"{source_key}: expected torch.float32, got {tensor.dtype}.")
+ buffer[target_key] = tensor
+ buffer_size += tensor.numel() * tensor.element_size()
+ total_size += tensor.numel() * tensor.element_size()
+ if buffer_size >= max_shard_size:
+ flush()
+ flush()
+
+ missing = sorted(set(plan) - seen_source_keys)
+ if missing:
+ raise KeyError(f"{len(missing)} planned key(s) missing from {weights_path}, e.g. {missing[:5]}.")
+
+ renames = {
+ path: os.path.join(output_path, f"diffusion_pytorch_model-{i + 1:05d}-of-{len(written):05d}.safetensors")
+ for i, path in enumerate(written)
+ }
+ for old, new in renames.items():
+ os.rename(old, new)
+ index = {
+ "metadata": {"total_size": total_size},
+ "weight_map": {key: os.path.basename(renames[path]) for key, path in weight_map.items()},
+ }
+ with open(os.path.join(output_path, SAFE_WEIGHTS_INDEX_NAME), "w") as f:
+ json.dump(index, f, indent=2, sort_keys=True)
+
+ with open(os.path.join(output_path, "config.json"), "w") as f:
+ json.dump(
+ {
+ "_class_name": "AutoencoderKLMiniMaxH3",
+ "_diffusers_version": diffusers_version,
+ **config,
+ "latents_mean": wrapper_config["latents_mean"],
+ "latents_std": wrapper_config["latents_std"],
+ },
+ f,
+ indent=2,
+ )
+
+ print(
+ f"video_vae: {len(seen_source_keys)} original keys -> {len(weight_map)} diffusers keys in "
+ f"{len(written)} shard(s), {total_size / 1024**3:.2f} GiB "
+ f"(latents_mean/latents_std: {len(wrapper_config['latents_mean'])}/"
+ f"{len(wrapper_config['latents_std'])} channels; tiling {wrapper_config['vae_tile_size']}px / "
+ f"{wrapper_config['vae_tile_overlap_min']}px min overlap)."
+ )
+
+
+# `AutoencoderKLMiniMaxH3Image` argument names. The encoder is the video VAE's; the decoder comes from
+# `vit_decoder_kwargs` in the release's `vae/config.json` (names in the comments), which `convert_image_vae` checks.
+MINIMAX_H3_IMAGE_VAE_CONFIG = {
+ **{key: MINIMAX_H3_VIDEO_VAE_CONFIG[key] for key in MINIMAX_H3_VIDEO_VAE_CONFIG if not key.startswith("decoder_")},
+ "decoder_num_layers": 24, # vit_decoder_kwargs.num_layers
+ "decoder_num_attention_heads": 16, # vit_decoder_kwargs.heads
+ "decoder_attention_head_dim": 64, # vit_decoder_kwargs.dim_head
+ "decoder_num_register_tokens": 4, # ViT3DDecoder default
+ "decoder_ffn_mult": 4, # FeedForward default
+ "decoder_rope_theta": 100.0, # vit_decoder_kwargs.rope_theta
+ "decoder_rope_dim_ratio": 0.5, # vit_decoder_kwargs.rope_dim_ratio
+ "decoder_norm_eps": 1e-05, # ViT3DDecoder eps
+}
+# The video temporal chunking does not apply to a single frame.
+del MINIMAX_H3_IMAGE_VAE_CONFIG["clip_length"], MINIMAX_H3_IMAGE_VAE_CONFIG["token_drop"]
+
+
+def convert_image_vae(checkpoint_path: str, output_path: str, config: dict[str, Any], diffusers_version: str) -> None:
+ """Convert the MiniMax-H3-Image VAE (`vae/`) and emit its config.
+
+ The keys are the video VAE's, so they convert with `convert_video_vae_key`; only `post_quant_conv` is a 2D
+ convolution here, which `AutoencoderKLMiniMaxH3Image` matches.
+ """
+ source_dir = os.path.join(checkpoint_path, "vae")
+ with open(os.path.join(source_dir, "config.json")) as f:
+ source_config = json.load(f)
+ decoder_kwargs = source_config["vit_decoder_kwargs"]
+ expected = {
+ "vit_decoder_2d": (source_config.get("vit_decoder_2d"), True),
+ "num_layers": (decoder_kwargs["num_layers"], config["decoder_num_layers"]),
+ "heads": (decoder_kwargs["heads"], config["decoder_num_attention_heads"]),
+ "dim_head": (decoder_kwargs["dim_head"], config["decoder_attention_head_dim"]),
+ "rope_theta": (decoder_kwargs["rope_theta"], config["decoder_rope_theta"]),
+ "rope_dim_ratio": (decoder_kwargs["rope_dim_ratio"], config["decoder_rope_dim_ratio"]),
+ "latent_channels": (source_config["latent_channels"], config["latent_channels"]),
+ # The tile geometry `AutoencoderKLMiniMaxH3Image` enables by default.
+ "vae_encoder_tiling": (source_config["vae_encoder_tiling"], 1),
+ "vae_decoder_tiling": (source_config["vae_decoder_tiling"], 1),
+ "vae_tile_size": (source_config["vae_tile_size"], 256),
+ "vae_tile_overlap_min": (source_config["vae_tile_overlap_min"], 64),
+ }
+ mismatches = {key: values for key, values in expected.items() if values[0] != values[1]}
+ if mismatches:
+ raise ValueError(
+ f"{source_dir}/config.json disagrees with the image VAE config (source, expected): {mismatches}"
+ )
+
+ weights_path = os.path.join(source_dir, "diffusion_pytorch_model.safetensors")
+ plan = get_video_vae_key_plan(config)
+ state_dict: dict[str, torch.Tensor] = {}
+ with safe_open(weights_path, framework="pt", device="cpu") as f:
+ source_keys = set(f.keys())
+ unexpected = sorted(source_keys - set(plan))
+ missing = sorted(set(plan) - source_keys)
+ if unexpected or missing:
+ raise KeyError(f"{weights_path}: unexpected keys {unexpected[:5]}, missing keys {missing[:5]}.")
+ for source_key in sorted(source_keys):
+ for target_key, tensor in convert_video_vae_key(source_key, f.get_tensor(source_key), config):
+ if tensor.dtype != torch.float32:
+ raise ValueError(f"{source_key}: expected torch.float32, got {tensor.dtype}.")
+ state_dict[target_key] = tensor
+
+ os.makedirs(output_path, exist_ok=True)
+ save_file(state_dict, os.path.join(output_path, "diffusion_pytorch_model.safetensors"), metadata={"format": "pt"})
+ with open(os.path.join(output_path, "config.json"), "w") as f:
+ json.dump(
+ {
+ "_class_name": "AutoencoderKLMiniMaxH3Image",
+ "_diffusers_version": diffusers_version,
+ **config,
+ "latents_mean": source_config["latents_mean"],
+ "latents_std": source_config["latents_std"],
+ },
+ f,
+ indent=2,
+ )
+ total_size = sum(tensor.numel() * tensor.element_size() for tensor in state_dict.values())
+ print(
+ f"vae: {len(source_keys)} original keys -> {len(state_dict)} diffusers keys, {total_size / 1024**3:.2f} GiB."
+ )
+
+
+# Not present in `audio_vae/metadata.json`: the reference implementation hardcodes these in its DAC audio VAE and its
+# attention projection, keyed off the sample rate.
+MINIMAX_H3_AUDIO_VAE_FIXED_CONFIG = {
+ "num_attention_heads": 8,
+ "resblock_kernel_sizes": [3, 7, 11],
+ "resblock_dilation_sizes": [[1, 3, 5], [1, 3, 5], [1, 3, 5]],
+}
+
+
+def get_audio_vae_config(checkpoint_path: str) -> dict[str, Any]:
+ """Build the `AutoencoderKLMiniMaxH3Audio` config from the original audio-VAE metadata.
+
+ `audio_vae/metadata.json` carries the constructor kwargs the checkpoint was built with, and
+ `audio_vae/config.json` carries the per-channel `latents_mean` / `latents_std` MiniMax-H3 normalizes with. The two
+ are cross-checked here because they duplicate the latent width and sample rate.
+ """
+ source_dir = os.path.join(checkpoint_path, "audio_vae")
+ with open(os.path.join(source_dir, "metadata.json")) as f:
+ kwargs = json.load(f)["metadata"]["kwargs"]
+ with open(os.path.join(source_dir, "config.json")) as f:
+ wrapper_config = json.load(f)
+
+ if kwargs["decoder_type"] != "bigvgan":
+ raise ValueError(f"Only the BigVGAN decoder is supported, got {kwargs['decoder_type']!r}.")
+ if not kwargs["attn_proj"]:
+ raise ValueError("The audio VAE is expected to carry the causal-attention latent projection.")
+ latent_channels = kwargs["vae_latent_channels"]
+ if wrapper_config["latent_channels"] != latent_channels:
+ raise ValueError(
+ f"latent width disagreement: metadata.json says {latent_channels}, "
+ f"config.json says {wrapper_config['latent_channels']}."
+ )
+ if wrapper_config["sample_rate"] != kwargs["sample_rate"]:
+ raise ValueError(
+ f"sample rate disagreement: metadata.json says {kwargs['sample_rate']}, "
+ f"config.json says {wrapper_config['sample_rate']}."
+ )
+ for key in ("latents_mean", "latents_std"):
+ if len(wrapper_config[key]) != latent_channels:
+ raise KeyError(f"{source_dir}/config.json `{key}` does not have {latent_channels} entries.")
+
+ return {
+ "encoder_dim": kwargs["encoder_dim"],
+ "encoder_rates": kwargs["encoder_rates"],
+ "latent_dim": kwargs["latent_dim"],
+ "latent_channels": latent_channels,
+ "decoder_dim": kwargs["decoder_dim"],
+ "decoder_rates": kwargs["decoder_rates"],
+ # The reference's two hardcoded BigVGAN tables (16 kHz and 32 kHz) both pair rate `u` with kernel
+ # `2u` for even `u` and `2u - 1` for odd `u`, i.e. [5, 5, 2, ...] -> [9, 9, 4, ...].
+ "decoder_kernel_sizes": [2 * rate - (rate % 2) for rate in kwargs["decoder_rates"]],
+ **MINIMAX_H3_AUDIO_VAE_FIXED_CONFIG,
+ # Renamed from the original `sample_rate` to the diffusers audio convention.
+ "sampling_rate": kwargs["sample_rate"],
+ "latents_mean": wrapper_config["latents_mean"],
+ "latents_std": wrapper_config["latents_std"],
+ }
+
+
+def convert_audio_vae(checkpoint_path: str, output_path: str, diffusers_version: str) -> None:
+ """Convert the audio VAE and emit its config.
+
+ The mapping is an identity: `AutoencoderKLMiniMaxH3Audio` reproduces the original module tree name for name,
+ including `torch.nn.utils.weight_norm`'s `weight_g` / `weight_v` spelling and the Kaiser-window `filter` buffers of
+ the anti-aliased activations. The keys are therefore only *validated* against a freshly built model, not renamed.
+ """
+ config = get_audio_vae_config(checkpoint_path)
+ expected_keys = set(AutoencoderKLMiniMaxH3Audio(**config).state_dict())
+
+ weights_path = os.path.join(checkpoint_path, "audio_vae", "model.safetensors")
+ state_dict: dict[str, torch.Tensor] = {}
+ total_size = 0
+ with safe_open(weights_path, framework="pt", device="cpu") as f:
+ for key in f.keys():
+ if key not in expected_keys:
+ raise KeyError(f"Unexpected key in {weights_path}: {key}")
+ tensor = f.get_tensor(key)
+ if tensor.dtype != torch.float32:
+ raise ValueError(f"{key}: expected torch.float32, got {tensor.dtype}.")
+ state_dict[key] = tensor
+ total_size += tensor.numel() * tensor.element_size()
+
+ missing = sorted(expected_keys - set(state_dict))
+ if missing:
+ raise KeyError(f"{len(missing)} key(s) missing from {weights_path}, e.g. {missing[:5]}.")
+
+ os.makedirs(output_path, exist_ok=True)
+ save_file(
+ state_dict,
+ os.path.join(output_path, "diffusion_pytorch_model.safetensors"),
+ metadata={"format": "pt"},
+ )
+ with open(os.path.join(output_path, "config.json"), "w") as f:
+ json.dump(
+ {"_class_name": "AutoencoderKLMiniMaxH3Audio", "_diffusers_version": diffusers_version, **config},
+ f,
+ indent=2,
+ )
+
+ print(
+ f"audio_vae: {len(state_dict)} keys carried over unchanged, {total_size / 1024**3:.2f} GiB "
+ f"({config['sampling_rate'] // math.prod(config['encoder_rates'])} latents/s, "
+ f"latents_mean/latents_std: {len(config['latents_mean'])}/{len(config['latents_std'])} channels)."
+ )
+
+
+def write_scheduler_configs(checkpoint_path: str, output_path: str, diffusers_version: str) -> None:
+ """Emit the two `MiniMaxH3Scheduler` configs, one per modality.
+
+ The source `model_index.json` leaves `scheduler` null and instead carries the schedule constants in its
+ `_minimax_h3.sigma_shift_scales` block. The sigma shift is the only per-modality difference, so it becomes two
+ scheduler folders holding the same class at different `shift` values.
+ """
+ with open(os.path.join(checkpoint_path, "model_index.json")) as f:
+ shift_scales = json.load(f)["_minimax_h3"]["sigma_shift_scales"]
+
+ for folder, modality in (("scheduler", "video"), ("audio_scheduler", "audio")):
+ folder_path = os.path.join(output_path, folder)
+ os.makedirs(folder_path, exist_ok=True)
+ with open(os.path.join(folder_path, "scheduler_config.json"), "w") as f:
+ json.dump(
+ {
+ "_class_name": "MiniMaxH3Scheduler",
+ "_diffusers_version": diffusers_version,
+ "shift": float(shift_scales[modality]),
+ },
+ f,
+ indent=2,
+ )
+ print(f"scheduler: shift={shift_scales['video']} (video), audio_scheduler: shift={shift_scales['audio']} (audio).")
+
+
+def read_safetensors_header(path: str) -> dict[str, Any]:
+ """Read the metadata header of a safetensors file without touching the tensor payload."""
+ with open(path, "rb") as f:
+ header_size = struct.unpack("<Q", f.read(8))[0]
+ header = json.loads(f.read(header_size))
+ header.pop("__metadata__", None)
+ return header
+
+
+def dry_run(checkpoint_path: str, config: dict[str, Any]) -> None:
+ plan = get_transformer_key_plan(config)
+
+ transformer_dir = os.path.join(checkpoint_path, "transformer")
+ shards = sorted(glob.glob(os.path.join(transformer_dir, "*.safetensors")))
+ header: dict[str, Any] = {}
+ for shard in shards:
+ header.update(read_safetensors_header(shard))
+ if shards:
+ print(f"Read headers of {len(shards)} shard(s) in {transformer_dir}: {len(header)} keys present.\n")
+ else:
+ print(f"No shards found under {transformer_dir}; validating the plan against the config only.\n")
+
+ print(f"{'original key':<48} {'->':^4} {'diffusers key':<52} {'shape':<24} dtype")
+ print("-" * 150)
+ num_target_keys = 0
+ shape_mismatches: list[str] = []
+ for source_key, targets in plan.items():
+ present = source_key in header
+ if not targets:
+ print(f"{source_key:<48} {'-x':^4} {'(dropped, recomputed by the port)':<52}")
+ continue
+ for index, (target_key, shape) in enumerate(targets):
+ num_target_keys += 1
+ expected_dtype = "F32" if source_key.startswith(MINIMAX_H3_FP32_SOURCE_PREFIXES) else "BF16"
+ if present:
+ actual_dtype = header[source_key]["dtype"]
+ actual_shape = header[source_key]["shape"]
+ if index == 0 and len(targets) == 1 and actual_shape != shape:
+ shape_mismatches.append(f"{source_key}: header {actual_shape} != planned {shape}")
+ if actual_dtype != expected_dtype:
+ shape_mismatches.append(f"{source_key}: header dtype {actual_dtype} != expected {expected_dtype}")
+ marker = "->"
+ else:
+ marker = "->?"
+ left = source_key if index == 0 else ""
+ print(f"{left:<48} {marker:^4} {target_key:<52} {str(shape):<24} {expected_dtype}")
+
+ missing = [key for key in plan if key not in header]
+ unexpected = [key for key in header if key not in plan]
+
+ print("\n" + "=" * 150)
+ print(f"planned original keys : {len(plan)}")
+ print(f"planned diffusers keys: {num_target_keys}")
+ print(
+ f"dropped original keys : {len(MINIMAX_H3_TRANSFORMER_DROPPED_KEYS)} {list(MINIMAX_H3_TRANSFORMER_DROPPED_KEYS)}"
+ )
+ print(f"fp32 diffusers keys : {sum(1 for key in plan if key.startswith(MINIMAX_H3_FP32_SOURCE_PREFIXES))}")
+ if shards:
+ print(f"keys present in shards: {len(header)}")
+ print(f"planned but absent : {len(missing)}" + (" (shards still downloading?)" if missing else ""))
+ print(f"present but unplanned : {len(unexpected)}")
+ if unexpected:
+ print(f" {unexpected}")
+ print(f"header disagreements : {len(shape_mismatches)}")
+ for line in shape_mismatches:
+ print(f" {line}")
+ total_bytes = sum(
+ (4 if source_key.startswith(MINIMAX_H3_FP32_SOURCE_PREFIXES) else 2) * torch.Size(shape).numel()
+ for source_key, targets in plan.items()
+ for _, shape in targets
+ )
+ print(f"total output bytes : {total_bytes} ({total_bytes / 1024**3:.2f} GiB)")
+
+
+def convert_pdd_heads(checkpoint_path: str, config: dict[str, Any]) -> dict[str, torch.Tensor]:
+ """Split the stacked PDD heads of a MiniMax-H3-Image release into one `pdd_proj_out.{i}` head per step.
+
+ The release keeps them outside `transformer/`, in the file `_minimax_h3.pdd_fused_heads` of `model_index.json`
+ names: `video_out.weight` of shape `(num_steps, out, in)` and `video_out.bias` of shape `(num_steps, out)`, float32,
+ with `pdd_config.json` listing the step count and the sigma grid they were distilled on.
+ """
+ with open(os.path.join(checkpoint_path, "model_index.json")) as f:
+ heads_path = os.path.join(checkpoint_path, json.load(f)["_minimax_h3"]["pdd_fused_heads"])
+ with open(os.path.join(checkpoint_path, "pdd_config.json")) as f:
+ num_steps = json.load(f)["num_inference_steps"]
+ if num_steps != config["num_pdd_heads"]:
+ raise ValueError(f"pdd_config.json has {num_steps} steps, the config {config['num_pdd_heads']} heads.")
+
+ with safe_open(heads_path, framework="pt", device="cpu") as f:
+ heads = {key: f.get_tensor(key) for key in f.keys()}
+ if set(heads) != {"video_out.weight", "video_out.bias"}:
+ raise KeyError(f"{heads_path} must hold exactly `video_out.weight` and `video_out.bias`, got {sorted(heads)}.")
+
+ hidden_size = config["hidden_size"]
+ video_patch_dim = config["in_channels"] * math.prod(config["patch_size"])
+ expected_shapes = {
+ "video_out.weight": (num_steps, video_patch_dim, hidden_size),
+ "video_out.bias": (num_steps, video_patch_dim),
+ }
+ converted = {}
+ for key, tensor in heads.items():
+ if tuple(tensor.shape) != expected_shapes[key] or tensor.dtype != torch.float32:
+ raise ValueError(
+ f"{key}: expected float32 {expected_shapes[key]}, got {tensor.dtype} {tuple(tensor.shape)}."
+ )
+ suffix = key.removeprefix("video_out.")
+ for step in range(num_steps):
+ converted[f"pdd_proj_out.{step}.{suffix}"] = tensor[step].contiguous()
+ return converted
+
+
+def convert_transformer(checkpoint_path: str, output_path: str, config: dict[str, Any], max_shard_size: int) -> None:
+ plan = get_transformer_key_plan(config)
+ transformer_dir = os.path.join(checkpoint_path, "transformer")
+ shards = sorted(glob.glob(os.path.join(transformer_dir, "*.safetensors")))
+ if not shards:
+ raise FileNotFoundError(f"No `*.safetensors` shards found under {transformer_dir}.")
+
+ os.makedirs(output_path, exist_ok=True)
+ weight_map: dict[str, str] = {}
+ total_size = 0
+ written: list[str] = []
+ buffer: dict[str, torch.Tensor] = {}
+ buffer_size = 0
+ seen_source_keys: set[str] = set()
+
+ def flush() -> None:
+ nonlocal buffer, buffer_size
+ if not buffer:
+ return
+ path = os.path.join(output_path, f".tmp-shard-{len(written):05d}.safetensors")
+ save_file(buffer, path, metadata={"format": "pt"})
+ for key in buffer:
+ weight_map[key] = path
+ written.append(path)
+ buffer = {}
+ buffer_size = 0
+
+ for shard in shards:
+ # `safe_open` memory-maps the file, so only the tensor being read is materialized.
+ with safe_open(shard, framework="pt", device="cpu") as f:
+ for source_key in f.keys():
+ if source_key not in plan:
+ raise KeyError(f"Unexpected key in {os.path.basename(shard)}: {source_key}")
+ seen_source_keys.add(source_key)
+ source_tensor = f.get_tensor(source_key)
+ if source_key.endswith(".attn.qkv_proj.weight"):
+ # Raw shards store fused QKV per-head interleaved; normalize to the reference's
+ # `[q_all; k_all; v_all]` layout (the same transform the reference applies at load time) so
+ # `convert_transformer_key` sees its state-dict-layout contract. The composition is bit-identical
+ # to de-interleaving the raw tensor directly.
+ source_tensor = reorder_interleaved_qkv(
+ source_tensor, config["num_attention_heads"], config["attention_head_dim"]
+ )
+ for target_key, tensor in convert_transformer_key(source_key, source_tensor, config):
+ expected_dtype = (
+ torch.float32 if source_key.startswith(MINIMAX_H3_FP32_SOURCE_PREFIXES) else torch.bfloat16
+ )
+ if tensor.dtype != expected_dtype:
+ raise ValueError(f"{source_key}: expected {expected_dtype}, got {tensor.dtype}.")
+ buffer[target_key] = tensor
+ buffer_size += tensor.numel() * tensor.element_size()
+ total_size += tensor.numel() * tensor.element_size()
+ if buffer_size >= max_shard_size:
+ flush()
+ if config.get("num_pdd_heads") is not None:
+ for target_key, tensor in convert_pdd_heads(checkpoint_path, config).items():
+ buffer[target_key] = tensor
+ buffer_size += tensor.numel() * tensor.element_size()
+ total_size += tensor.numel() * tensor.element_size()
+ flush()
+
+ missing = sorted(set(plan) - seen_source_keys)
+ if missing:
+ raise KeyError(f"{len(missing)} planned key(s) missing from the checkpoint, e.g. {missing[:5]}.")
+
+ # The shard count is only known once every source shard has been streamed, so the files are written under
+ # provisional names and renamed here.
+ renames = {
+ path: os.path.join(output_path, f"diffusion_pytorch_model-{i + 1:05d}-of-{len(written):05d}.safetensors")
+ for i, path in enumerate(written)
+ }
+ for old, new in renames.items():
+ os.rename(old, new)
+ index = {
+ "metadata": {"total_size": total_size},
+ "weight_map": {key: os.path.basename(renames[path]) for key, path in weight_map.items()},
+ }
+ with open(os.path.join(output_path, SAFE_WEIGHTS_INDEX_NAME), "w") as f:
+ json.dump(index, f, indent=2, sort_keys=True)
+
+ print(
+ f"transformer: {len(seen_source_keys)} original keys -> {len(weight_map)} diffusers keys "
+ f"in {len(written)} shard(s), {total_size / 1024**3:.2f} GiB."
+ )
+
+
+def write_transformer_config(
+ output_path: str, config: dict[str, Any], diffusers_version: str, class_name: str = "MiniMaxH3Transformer3DModel"
+) -> None:
+ os.makedirs(output_path, exist_ok=True)
+ with open(os.path.join(output_path, "config.json"), "w") as f:
+ json.dump(
+ {"_class_name": class_name, "_diffusers_version": diffusers_version, **config},
+ f,
+ indent=2,
+ )
+
+
+# The components a MiniMax-H3 repository holds, and the class each one loads as. `video_processor` is absent: the
+# blocks create it from config rather than loading it.
+MINIMAX_H3_COMPONENTS = {
+ # The source names a checkpoint-local wrapper class (`MiniMaxH3Qwen3VLHFEncoder`); the conditioner is the
+ # released Qwen3-VL, read at its 50th decoder layer with its language-model head unused.
+ "text_encoder": ["transformers", "Qwen3VLForConditionalGeneration"],
+ "tokenizer": ["transformers", "Qwen2TokenizerFast"],
+ "processor": ["transformers", "Qwen3VLProcessor"],
+ # Renamed to the diffusers audio/video VAE convention (see `LTX2Pipeline`).
+ "vae": ["diffusers", "AutoencoderKLMiniMaxH3"],
+ "audio_vae": ["diffusers", "AutoencoderKLMiniMaxH3Audio"],
+ "transformer": ["diffusers", "MiniMaxH3Transformer3DModel"],
+ # One repository holds both checkpoint partitions: `transformer/` serves `MiniMaxH3Blocks` (`t2va` / `fl2va`) and
+ # `transformer_ref/` serves `MiniMaxH3Ref2VABlocks`, while every other component is shared and converted once.
+ "transformer_ref": ["diffusers", "MiniMaxH3Transformer3DModel"],
+ # The source leaves `scheduler` null. MiniMax-H3 samples with Euler at eta=0 over shifted flow-matching sigmas, at
+ # a different shift per modality, so it needs two scheduler entries (see `write_scheduler_configs`).
+ "scheduler": ["diffusers", "MiniMaxH3Scheduler"],
+ "audio_scheduler": ["diffusers", "MiniMaxH3Scheduler"],
+}
+
+
+def write_model_index(output_path: str, repo_id: str, diffusers_version: str) -> None:
+ """Emit `modular_model_index.json`, the only index a MiniMax-H3 repository carries.
+
+ MiniMax-H3 is integrated as Modular Diffusers blocks only, so there is no `model_index.json`: a modular repository
+ declares one entry per component with its full loading spec rather than just its class, and a blockset then fetches
+ exactly the subfolders it declares. That is what lets one repository hold both transformer partitions, and the
+ original checkpoint folders next to the converted ones, without either half pulling the rest down.
+
+ `_class_name` and `_blocks_class_name` name the `t2va` / `fl2va` half, which is what
+ `ModularPipeline.from_pretrained` resolves to. The `ref2va` half reads the very same file through
+ `MiniMaxH3Ref2VABlocks().init_pipeline(repo_id)`.
+
+ The component map is the static one above, so this needs no source checkpoint: an index can be regenerated for a
+ repository that is already published.
+ """
+ modular_index = {
+ "_class_name": "MiniMaxH3ModularPipeline",
+ "_diffusers_version": diffusers_version,
+ "_blocks_class_name": "MiniMaxH3Blocks",
+ }
+ for name, (library, class_name) in MINIMAX_H3_COMPONENTS.items():
+ modular_index[name] = [
+ library,
+ class_name,
+ {
+ "type_hint": [library, class_name],
+ "pretrained_model_name_or_path": repo_id,
+ "subfolder": name,
+ "variant": None,
+ "revision": None,
+ },
+ ]
+ os.makedirs(output_path, exist_ok=True)
+ with open(os.path.join(output_path, "modular_model_index.json"), "w") as f:
+ json.dump(modular_index, f, indent=2)
+ print(f"modular_model_index.json: {len(MINIMAX_H3_COMPONENTS)} components load from {repo_id}.")
+
+
+# The components a MiniMax-H3-Image repository names. The release does not carry its text encoder (its
+# `model_index.json` points at `text_encoder/` of the MiniMax-H3 repository); the converted repository holds a copy.
+MINIMAX_H3_IMAGE_COMPONENTS = {
+ "text_encoder": ["transformers", "Qwen3VLForConditionalGeneration"],
+ "tokenizer": ["transformers", "Qwen2TokenizerFast"],
+ "processor": ["transformers", "Qwen3VLProcessor"],
+ "vae": ["diffusers", "AutoencoderKLMiniMaxH3Image"],
+ "transformer": ["diffusers", "MiniMaxH3ImageTransformer2DModel"],
+ "scheduler": ["diffusers", "MiniMaxH3Scheduler"],
+}
+
+# The release already has `transformer/` and `vae/` folders in the original format, so the converted components live
+# in folders of their own. Every other component uses its name as its folder.
+MINIMAX_H3_IMAGE_SUBFOLDERS = {"transformer": "transformer_diffusers", "vae": "vae_diffusers"}
+
+
+def write_image_scheduler_config(output_path: str, diffusers_version: str) -> None:
+ """Emit the `MiniMaxH3Scheduler` config of a MiniMax-H3-Image release.
+
+ The release samples on the explicit grid of its `pdd_config.json`, which the blocks pass to `set_timesteps`
+ verbatim (`shift_provided_sigmas=False`), and evaluates the Euler update in the velocity form its heads were distilled with.
+ """
+ folder_path = os.path.join(output_path, "scheduler")
+ os.makedirs(folder_path, exist_ok=True)
+ with open(os.path.join(folder_path, "scheduler_config.json"), "w") as f:
+ json.dump(
+ {
+ "_class_name": "MiniMaxH3Scheduler",
+ "_diffusers_version": diffusers_version,
+ "shift_provided_sigmas": False,
+ "euler_update_form": "velocity",
+ },
+ f,
+ indent=2,
+ )
+ print("scheduler: explicit sigmas taken verbatim, Euler update in the velocity form.")
+
+
+def write_image_model_index(checkpoint_path: str, output_path: str, repo_id: str, diffusers_version: str) -> None:
+ """Emit the `modular_model_index.json` of a MiniMax-H3-Image repository.
+
+ Besides the components it carries `sample_sigmas`, the grid of the release's `pdd_config.json` that the per-step
+ output heads were distilled on. The transformer and the VAE load from the folders of
+ `MINIMAX_H3_IMAGE_SUBFOLDERS`.
+ """
+ with open(os.path.join(checkpoint_path, "pdd_config.json")) as f:
+ pdd_config = json.load(f)
+ sigmas = pdd_config["sigmas"]
+ if len(sigmas) != pdd_config["num_inference_steps"] + 1 or sigmas[0] != 1.0 or sigmas[-1] != 0.0:
+ raise ValueError(f"pdd_config.json must list `num_inference_steps + 1` sigmas from 1.0 to 0.0, got {sigmas}.")
+
+ modular_index = {
+ "_class_name": "MiniMaxH3ImageModularPipeline",
+ "_diffusers_version": diffusers_version,
+ "_blocks_class_name": "MiniMaxH3ImageBlocks",
+ "sample_sigmas": sigmas,
+ }
+ for name, (library, class_name) in MINIMAX_H3_IMAGE_COMPONENTS.items():
+ modular_index[name] = [
+ library,
+ class_name,
+ {
+ "type_hint": [library, class_name],
+ "pretrained_model_name_or_path": repo_id,
+ "subfolder": MINIMAX_H3_IMAGE_SUBFOLDERS.get(name, name),
+ "variant": None,
+ "revision": None,
+ },
+ ]
+ os.makedirs(output_path, exist_ok=True)
+ with open(os.path.join(output_path, "modular_model_index.json"), "w") as f:
+ json.dump(modular_index, f, indent=2)
+ print(
+ f"modular_model_index.json: {len(MINIMAX_H3_IMAGE_COMPONENTS)} components load from {repo_id}; "
+ f"{len(sigmas) - 1} sampling steps."
+ )
+
+
+def get_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--checkpoint_path",
+ type=str,
+ required=True,
+ help="Local path to an original MiniMax-H3 variant folder (the one holding `transformer/`, `audio_vae/`, ...).",
+ )
+ parser.add_argument("--output_path", type=str, required=True, help="Where the diffusers checkpoint is written.")
+ parser.add_argument(
+ "--modular_repo_id",
+ type=str,
+ default=None,
+ help=(
+ "Repository the component entries of `modular_model_index.json` point at. Defaults to `--output_path`, so "
+ "pass the Hub id the checkpoint is published under. Every entry carries its own loading spec, so a "
+ "blockset fetches exactly the subfolders it declares out of that repository."
+ ),
+ )
+ parser.add_argument(
+ "--text_encoder_path",
+ type=str,
+ default=None,
+ help=(
+ "`--version image` only: local path to the `text_encoder/` folder of a MiniMax-H3 checkpoint, which the "
+ "image release conditions on without carrying it. It is copied into the output as is."
+ ),
+ )
+ parser.add_argument(
+ "--version",
+ type=str,
+ default="h3",
+ choices=["h3", "image", "test"],
+ help=(
+ "`image` converts a MiniMax-H3-Image release (transformer and VAE); `test` emits the tiny config used for "
+ "fixtures."
+ ),
+ )
+ parser.add_argument(
+ "--max_shard_size",
+ type=int,
+ default=5 * 1024**3,
+ help="Maximum size of an output safetensors shard, in bytes.",
+ )
+ parser.add_argument(
+ "--dry_run",
+ action="store_true",
+ help="Print the full planned key mapping (and cross-check any shard headers already present) without writing.",
+ )
+ return parser.parse_args()
+
+
+def main(args):
+ from diffusers import __version__ as diffusers_version
+
+ config = {
+ "h3": MINIMAX_H3_TRANSFORMER_CONFIG,
+ "image": MINIMAX_H3_IMAGE_TRANSFORMER_CONFIG,
+ "test": MINIMAX_H3_TEST_TRANSFORMER_CONFIG,
+ }[args.version]
+
+ if args.dry_run:
+ dry_run(args.checkpoint_path, config)
+ return
+
+ if args.version == "image":
+ if args.text_encoder_path is None:
+ raise ValueError(
+ "`--version image` needs `--text_encoder_path`, the `text_encoder/` folder of MiniMax-H3."
+ )
+ transformer_path = os.path.join(args.output_path, MINIMAX_H3_IMAGE_SUBFOLDERS["transformer"])
+ convert_transformer(args.checkpoint_path, transformer_path, config, args.max_shard_size)
+ write_transformer_config(transformer_path, config, diffusers_version, "MiniMaxH3ImageTransformer2DModel")
+ convert_image_vae(
+ args.checkpoint_path,
+ os.path.join(args.output_path, MINIMAX_H3_IMAGE_SUBFOLDERS["vae"]),
+ MINIMAX_H3_IMAGE_VAE_CONFIG,
+ diffusers_version,
+ )
+ # The tokenizer and the processor are already in the `transformers` layout.
+ for name in ("tokenizer", "processor"):
+ shutil.copytree(
+ os.path.join(args.checkpoint_path, name), os.path.join(args.output_path, name), dirs_exist_ok=True
+ )
+ # A plain copy keeps the files identical to MiniMax-H3's, which the Hub deduplicates.
+ shutil.copytree(args.text_encoder_path, os.path.join(args.output_path, "text_encoder"), dirs_exist_ok=True)
+ write_image_scheduler_config(args.output_path, diffusers_version)
+ write_image_model_index(
+ args.checkpoint_path, args.output_path, args.modular_repo_id or args.output_path, diffusers_version
+ )
+ return
+
+ transformer_path = os.path.join(args.output_path, "transformer")
+ convert_transformer(args.checkpoint_path, transformer_path, config, args.max_shard_size)
+ write_transformer_config(transformer_path, config, diffusers_version)
+ video_vae_config = MINIMAX_H3_TEST_VIDEO_VAE_CONFIG if args.version == "test" else MINIMAX_H3_VIDEO_VAE_CONFIG
+ convert_video_vae(
+ args.checkpoint_path,
+ os.path.join(args.output_path, "vae"),
+ video_vae_config,
+ diffusers_version,
+ args.max_shard_size,
+ )
+ convert_audio_vae(args.checkpoint_path, os.path.join(args.output_path, "audio_vae"), diffusers_version)
+ write_scheduler_configs(args.checkpoint_path, args.output_path, diffusers_version)
+ write_model_index(args.output_path, args.modular_repo_id or args.output_path, diffusers_version)
+
+
+if __name__ == "__main__":
+ main(get_args())
diff --git a/studio/backend/tests/data/minimax_h3_image/transformer_config.json b/studio/backend/tests/data/minimax_h3_image/transformer_config.json
new file mode 100644
index 00000000000..ef7bd84de2a
--- /dev/null
+++ b/studio/backend/tests/data/minimax_h3_image/transformer_config.json
@@ -0,0 +1,28 @@
+{
+ "_class_name": "MiniMaxH3DiTModel",
+ "_diffusers_version": "0.32.2",
+ "hidden_size": 5376,
+ "num_layers": 50,
+ "token_refiner_num_layers": 2,
+ "num_attention_heads": 56,
+ "attention_head_dim": 128,
+ "ffn_hidden_size": 14336,
+ "latents_dim": 24,
+ "audio_latents_dim": 32,
+ "patch_size": [
+ 1,
+ 2,
+ 2
+ ],
+ "text_dim": 5120,
+ "timestep_input_dim": 256,
+ "time_embed_hidden_size": 5376,
+ "time_embed_dim": 2688,
+ "adaln_out_features": 96768,
+ "final_adaln_out_features": 10752,
+ "rope_inv_freq_len": 16,
+ "norm_eps": 1e-05,
+ "qk_norm_eps": 1e-05,
+ "final_norm_eps": 1e-05,
+ "audio_enabled": false
+}
diff --git a/studio/backend/tests/fixtures/diffusion_autoopt_parity.json b/studio/backend/tests/fixtures/diffusion_autoopt_parity.json
index cf00ead304a..eff9876e02b 100644
--- a/studio/backend/tests/fixtures/diffusion_autoopt_parity.json
+++ b/studio/backend/tests/fixtures/diffusion_autoopt_parity.json
@@ -1,5 +1,5 @@
{
- "meta": {"base": "origin/main 5f38afbc2d (before diffusion_autoopt)", "dtypes": ["bfloat16", "float16"], "families": ["image|anima", "image|chroma", "image|flux.1", "image|flux.1-kontext", "image|flux.2-dev", "image|flux.2-klein", "image|flux.3-klein", "image|flux.3-klein-base", "image|hidream-i1", "image|hunyuanimage-2.1", "image|ideogram-4", "image|krea-2", "image|lumina-2", "image|moss-image", "image|qwen-image", "image|qwen-image-2.1", "image|qwen-image-edit", "image|qwen-image-layered", "image|sdxl", "image|z-image", "video|cosmos3-edge", "video|cosmos3-nano", "video|hunyuanvideo-1.5", "video|hunyuanvideo-1.5-720p", "video|ltx-2", "video|ltx-2.5", "video|minimax-h3", "video|wan2.1-t2v-1.3b", "video|wan2.1-t2v-14b", "video|wan2.2-t2v-a14b", "video|wan2.2-ti2v-5b"], "speeds": ["off", "eager", "default", "max"]},
+ "meta": {"base": "origin/main 5f38afbc2d (before diffusion_autoopt)", "dtypes": ["bfloat16", "float16"], "families": ["image|anima", "image|chroma", "image|flux.1", "image|flux.1-kontext", "image|flux.2-dev", "image|flux.2-klein", "image|flux.3-klein", "image|flux.3-klein-base", "image|hidream-i1", "image|hunyuanimage-2.1", "image|ideogram-4", "image|krea-2", "image|lumina-2", "image|minimax-h3-image", "image|moss-image", "image|qwen-image", "image|qwen-image-2.1", "image|qwen-image-edit", "image|qwen-image-layered", "image|sdxl", "image|z-image", "video|cosmos3-edge", "video|cosmos3-nano", "video|hunyuanvideo-1.5", "video|hunyuanvideo-1.5-720p", "video|ltx-2", "video|ltx-2.5", "video|minimax-h3", "video|wan2.1-t2v-1.3b", "video|wan2.1-t2v-14b", "video|wan2.2-t2v-a14b", "video|wan2.2-ti2v-5b"], "speeds": ["off", "eager", "default", "max"]},
"cases": {
"image|flux.1-kontext|default|bfloat16|offload=0|gguf=0": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["compiled","cuda_graph","fp16_accum","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"eligible","graph_after_placement":false,"modules":{"transformer":{"compile_calls":[{"fullgraph":true,"dynamic":true}],"static":false,"auto_dynamic":false,"int8_gemm":false}}},
"image|flux.1-kontext|default|bfloat16|offload=0|gguf=1": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["compiled_dequant","cuda_graph","fp16_accum","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"eligible","graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
@@ -721,6 +721,38 @@
"video|wan2.2-ti2v-5b|off|bfloat16|offload=1|gguf=0": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{}},
"video|wan2.2-ti2v-5b|off|float16|offload=0|gguf=0": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{}},
"video|wan2.2-ti2v-5b|off|float16|offload=1|gguf=0": {"applied":[[]],"graph_after_placement":false,"graph_reason":null,"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"modules":{"transformer":{"auto_dynamic":null,"compile_calls":[],"int8_gemm":false,"static":false}},"patches":{}},
+ "image|minimax-h3-image|default|bfloat16|offload=0|gguf=0": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cuda_graph","cudnn_benchmark","fp16_accum","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"eligible","graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|default|bfloat16|offload=0|gguf=1": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cuda_graph","cudnn_benchmark","fp16_accum","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"eligible","graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|default|bfloat16|offload=1|gguf=0": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cudnn_benchmark","fp16_accum","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"armed after placement","graph_after_placement":true,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|default|bfloat16|offload=1|gguf=1": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cudnn_benchmark","fp16_accum","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"armed after placement","graph_after_placement":true,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|default|float16|offload=0|gguf=0": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cuda_graph","cudnn_benchmark","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"eligible","graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|default|float16|offload=0|gguf=1": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cuda_graph","cudnn_benchmark","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"eligible","graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|default|float16|offload=1|gguf=0": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cudnn_benchmark","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"armed after placement","graph_after_placement":true,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|default|float16|offload=1|gguf=1": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cudnn_benchmark","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"armed after placement","graph_after_placement":true,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|eager|bfloat16|offload=0|gguf=0": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cudnn_benchmark","fp16_accum"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":null,"graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|eager|bfloat16|offload=0|gguf=1": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cudnn_benchmark","fp16_accum"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":null,"graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|eager|bfloat16|offload=1|gguf=0": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cudnn_benchmark","fp16_accum"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":null,"graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|eager|bfloat16|offload=1|gguf=1": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cudnn_benchmark","fp16_accum"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":null,"graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|eager|float16|offload=0|gguf=0": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cudnn_benchmark"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":null,"graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|eager|float16|offload=0|gguf=1": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cudnn_benchmark"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":null,"graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|eager|float16|offload=1|gguf=0": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cudnn_benchmark"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":null,"graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|eager|float16|offload=1|gguf=1": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cudnn_benchmark"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":null,"graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|max|bfloat16|offload=0|gguf=0": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cuda_graph","cudnn_benchmark","fp16_accum","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"eligible","graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|max|bfloat16|offload=0|gguf=1": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cuda_graph","cudnn_benchmark","fp16_accum","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"eligible","graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|max|bfloat16|offload=1|gguf=0": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cudnn_benchmark","fp16_accum","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"armed after placement","graph_after_placement":true,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|max|bfloat16|offload=1|gguf=1": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cudnn_benchmark","fp16_accum","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"armed after placement","graph_after_placement":true,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|max|float16|offload=0|gguf=0": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cuda_graph","cudnn_benchmark","fp16_accum","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"eligible","graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|max|float16|offload=0|gguf=1": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cuda_graph","cudnn_benchmark","fp16_accum","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"eligible","graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|max|float16|offload=1|gguf=0": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cudnn_benchmark","fp16_accum","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"armed after placement","graph_after_placement":true,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|max|float16|offload=1|gguf=1": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["cudnn_benchmark","fp16_accum","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"armed after placement","graph_after_placement":true,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|off|bfloat16|offload=0|gguf=0": {"patches":{"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[[]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":null,"graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|off|bfloat16|offload=0|gguf=1": {"patches":{"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[[]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":null,"graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|off|bfloat16|offload=1|gguf=0": {"patches":{"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[[]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":null,"graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|off|bfloat16|offload=1|gguf=1": {"patches":{"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[[]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":null,"graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|off|float16|offload=0|gguf=0": {"patches":{"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[[]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":null,"graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|off|float16|offload=0|gguf=1": {"patches":{"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[[]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":null,"graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|off|float16|offload=1|gguf=0": {"patches":{"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[[]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":null,"graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
+ "image|minimax-h3-image|off|float16|offload=1|gguf=1": {"patches":{"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[[]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":null,"graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
"image|moss-image|default|bfloat16|offload=0|gguf=0": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["compiled","cuda_graph","fp16_accum","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"eligible","graph_after_placement":false,"modules":{"transformer":{"compile_calls":[{"fullgraph":true,"dynamic":true}],"static":false,"auto_dynamic":false,"int8_gemm":false}}},
"image|moss-image|default|bfloat16|offload=0|gguf=1": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["compiled_dequant","cuda_graph","fp16_accum","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"eligible","graph_after_placement":false,"modules":{"transformer":{"compile_calls":[],"static":false,"auto_dynamic":null,"int8_gemm":false}}},
"image|moss-image|default|bfloat16|offload=1|gguf=0": {"patches":{"arch_fusions":true,"eager_fusions":true,"flux2_rope":false,"flux3_fp16_attn":false,"flux3_rope":false,"fp16_eager_rmsnorm":false,"qwenimage21_fast_step":false,"rocm_fused":false},"applied":[["compiled","fp16_accum","tf32"]],"keys":[["channels_last","compiled","compiled_dequant","compiled_vae_decode","cuda_graph","cudnn_benchmark","fp16_accum","fused_qkv","int8_gemm","rocm_query_chunks","tf32","vae_fp16_decode","vae_fused","vae_single_frame"]],"graph_reason":"armed after placement","graph_after_placement":true,"modules":{"transformer":{"compile_calls":[{"fullgraph":false,"dynamic":true}],"static":false,"auto_dynamic":false,"int8_gemm":false}}},
diff --git a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
index 070f935d5f3..83c50129a9b 100644
--- a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
+++ b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
@@ -76,6 +76,9 @@
"steps": 30
}
},
+ "minimax-h3-image": {
+ "MiniMaxAI/MiniMax-H3-Image": null
+ },
"moss-image": {
"OpenMOSS-Team/MOSS-Image-Base": null,
"OpenMOSS-Team/MOSS-Image-SFT": null
@@ -247,6 +250,10 @@
1.0
]
},
+ "minimax-h3-image": [
+ 6,
+ 1.0
+ ],
"moss-image": [
25,
2.5
@@ -471,6 +478,12 @@
"filter_reduction_configs_archs": [],
"vae_fused_encoder": true
},
+ "minimax-h3-image": {
+ "cudnn_benchmark": true,
+ "filter_reduction_configs": false,
+ "filter_reduction_configs_archs": [],
+ "vae_fused_encoder": true
+ },
"moss-image": {
"cudnn_benchmark": false,
"filter_reduction_configs": false,
@@ -3134,6 +3147,106 @@
"preview_key": null,
"te_int8_effective": "fp8"
},
+ "minimax-h3-image": {
+ "auto": {
+ "convrot_filename": null,
+ "convrot_repo": null,
+ "convrot_spec": [
+ 0,
+ []
+ ],
+ "convrot_variant_repos": [],
+ "exclude": [],
+ "pad": [],
+ "runtime_quant": [],
+ "streamed_weight_only": [],
+ "zero_row": []
+ },
+ "auto_bf16_when_resident": null,
+ "bf16": {
+ "convrot_filename": null,
+ "convrot_repo": null,
+ "convrot_spec": [
+ 0,
+ []
+ ],
+ "convrot_variant_repos": [],
+ "exclude": [],
+ "pad": [],
+ "runtime_quant": [],
+ "streamed_weight_only": [],
+ "zero_row": []
+ },
+ "convrot_default_on": false,
+ "fp8": {
+ "convrot_filename": null,
+ "convrot_repo": null,
+ "convrot_spec": [
+ 0,
+ []
+ ],
+ "convrot_variant_repos": [],
+ "exclude": [],
+ "pad": [],
+ "runtime_quant": [],
+ "streamed_weight_only": [],
+ "zero_row": []
+ },
+ "int8": {
+ "convrot_filename": null,
+ "convrot_repo": null,
+ "convrot_spec": [
+ 0,
+ []
+ ],
+ "convrot_variant_repos": [],
+ "exclude": [
+ "norm",
+ "_mod",
+ "modulation",
+ "timestep_embed",
+ "guidance_embed",
+ "time_text_embed",
+ "pooled",
+ "time_embed"
+ ],
+ "pad": [],
+ "runtime_quant": [],
+ "streamed_weight_only": [],
+ "zero_row": []
+ },
+ "mxfp8": {
+ "convrot_filename": null,
+ "convrot_repo": null,
+ "convrot_spec": [
+ 0,
+ []
+ ],
+ "convrot_variant_repos": [],
+ "exclude": [],
+ "pad": [],
+ "runtime_quant": [],
+ "streamed_weight_only": [],
+ "zero_row": []
+ },
+ "nvfp4": {
+ "convrot_filename": null,
+ "convrot_repo": null,
+ "convrot_spec": [
+ 0,
+ []
+ ],
+ "convrot_variant_repos": [],
+ "exclude": [],
+ "pad": [],
+ "runtime_quant": [],
+ "streamed_weight_only": [],
+ "zero_row": []
+ },
+ "page": "image",
+ "preview_key": null,
+ "te_int8_effective": "fp8"
+ },
"moss-image": {
"auto": {
"convrot_filename": null,
@@ -4881,6 +4994,7 @@
"krea-2": "Krea 2",
"ltx-2": "LTX-2",
"minimax-h3": "MiniMax-H3",
+ "minimax-h3-image": "MiniMax-H3-Image",
"moss-image": "MOSS-Image",
"qwen-image": "Qwen-Image",
"sdxl": "SDXL",
@@ -4916,6 +5030,7 @@
"krea/krea-2-raw",
"krea/krea-2-turbo",
"lodestones/chroma1-hd",
+ "minimaxai/minimax-h3-image",
"openmoss-team/moss-image-base",
"openmoss-team/moss-image-sft",
"qwen/qwen-image",
diff --git a/studio/backend/tests/fixtures/vae_tile_configs.json b/studio/backend/tests/fixtures/vae_tile_configs.json
index 4f97dbde697..a4cc4d5319a 100644
--- a/studio/backend/tests/fixtures/vae_tile_configs.json
+++ b/studio/backend/tests/fixtures/vae_tile_configs.json
@@ -1034,6 +1034,106 @@
"use_quant_conv": true
}
},
+ "minimax-h3-image": {
+ "kind": "image",
+ "repo": "MiniMaxAI/MiniMax-H3-Image",
+ "revision": "fbde899fffb5c5225ee8dd7027680113746af42e",
+ "config": {
+ "_class_name": "AutoencoderKLMiniMaxH3Image",
+ "_diffusers_version": "0.42.0.dev0",
+ "in_channels": 3,
+ "out_channels": 3,
+ "latent_channels": 24,
+ "block_out_channels": [
+ 128,
+ 256,
+ 256,
+ 512,
+ 512,
+ 1024
+ ],
+ "layers_per_block": 2,
+ "spatial_downsample_factors": [
+ 2,
+ 2,
+ 2,
+ 2,
+ 1,
+ 1
+ ],
+ "temporal_downsample_factors": [
+ 1,
+ 2,
+ 2,
+ 1,
+ 1,
+ 1
+ ],
+ "norm_num_groups": 32,
+ "norm_eps": 1e-06,
+ "spatial_padding_mode": "reflect",
+ "decoder_num_layers": 24,
+ "decoder_num_attention_heads": 16,
+ "decoder_attention_head_dim": 64,
+ "decoder_num_register_tokens": 4,
+ "decoder_ffn_mult": 4,
+ "decoder_rope_theta": 100.0,
+ "decoder_rope_dim_ratio": 0.5,
+ "decoder_norm_eps": 1e-05,
+ "latents_mean": [
+ 0.858090341091156,
+ -0.9606591463088989,
+ 1.0661640167236328,
+ -0.5090325474739075,
+ -0.2727581858634949,
+ -1.3675414323806763,
+ -0.2553254961967468,
+ -0.26907554268836975,
+ -0.5376840829849243,
+ -0.0464097298681736,
+ 0.6657370328903198,
+ 0.19690127670764923,
+ -0.5460608005523682,
+ -0.4035342037677765,
+ -0.23683024942874908,
+ 0.25928452610969543,
+ -0.30133944749832153,
+ 0.211341992020607,
+ -1.1206848621368408,
+ 0.3581933379173279,
+ -0.04225143790245056,
+ 0.2604829967021942,
+ 0.22864092886447906,
+ 0.7056031823158264
+ ],
+ "latents_std": [
+ 1.2223774194717407,
+ 1.2767263650894165,
+ 1.6831774711608887,
+ 1.7549455165863037,
+ 1.5636216402053833,
+ 2.194143533706665,
+ 0.9653137922286987,
+ 1.0569885969161987,
+ 0.841948926448822,
+ 0.7729952931404114,
+ 1.8955937623977661,
+ 0.946841835975647,
+ 0.7996809482574463,
+ 0.44988900423049927,
+ 0.7197399735450745,
+ 0.6936293244361877,
+ 2.961095094680786,
+ 2.7694199085235596,
+ 3.0496184825897217,
+ 2.1088054180145264,
+ 3.276226282119751,
+ 3.1627357006073,
+ 2.2816812992095947,
+ 2.6127843856811523
+ ]
+ }
+ },
"sdxl": {
"kind": "image",
"repo": "stabilityai/stable-diffusion-xl-base-1.0",
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index e690132f502..86291a7e5f2 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -281,6 +281,9 @@ def test_every_third_party_bf16_pipeline_the_catalog_offers_is_mirrored():
for repo in offered
if not repo.lower().startswith("unsloth/")
and "hunyuan" not in repo.lower()
+ # MiniMax-H3-Image's in-tree loader fetches only the files it reads from the official base (its shards and
+ # Studio's hosted int8 encoder), never a whole-repo snapshot, so it has no mirror.
+ and repo.lower() != "minimaxai/minimax-h3-image"
and repo.lower() not in mirrored
)
assert not missing, f"catalog offers these vendor bases with no unsloth mirror: {missing}"
diff --git a/studio/backend/tests/test_diffusion_control_capabilities.py b/studio/backend/tests/test_diffusion_control_capabilities.py
index 12641471c55..0c6499f6561 100644
--- a/studio/backend/tests/test_diffusion_control_capabilities.py
+++ b/studio/backend/tests/test_diffusion_control_capabilities.py
@@ -42,9 +42,13 @@ def test_flux3_thresholds_are_one():
assert diffusion._negative_prompt_guidance_threshold(by_name["flux.3-klein-base"]) == 1.0
-def test_only_the_distilled_flux3_graph_ignores_guidance_as_a_family():
- # Flux3DistilledModularPipeline has no guider, so diffusion_flux3._set_guidance drops the value.
- assert [f.name for f in _FAMILIES if not f.uses_guidance] == ["flux.3-klein"]
+def test_only_the_guidance_distilled_graphs_ignore_guidance_as_a_family():
+ # Flux3DistilledModularPipeline has no guider, so diffusion_flux3._set_guidance drops the value; MiniMax-H3-Image
+ # is guidance-distilled to 6 PDD steps and only takes guidance 1.
+ assert [f.name for f in _FAMILIES if not f.uses_guidance] == [
+ "flux.3-klein",
+ "minimax-h3-image",
+ ]
@pytest.mark.parametrize(
diff --git a/studio/backend/tests/test_diffusion_image_te_default.py b/studio/backend/tests/test_diffusion_image_te_default.py
index 86f4ea9631e..c9c22ada3da 100644
--- a/studio/backend/tests/test_diffusion_image_te_default.py
+++ b/studio/backend/tests/test_diffusion_image_te_default.py
@@ -65,6 +65,9 @@ KEEP_BF16 = (
"chroma",
"anima",
)
+# Families whose in-tree loader picks the encoder itself, outside te_quant_auto: MiniMax-H3-Image's loader defaults
+# its Qwen3-VL-32B conditioner to Studio's hosted int8 ConvRot file (text_encoder_quant = "int8").
+IN_TREE_TE_DEFAULT = ("minimax-h3-image",)
def _fam(name):
@@ -104,9 +107,15 @@ def test_none_still_pins_the_bf16_encoder(name, value):
def test_every_image_family_is_either_defaulted_or_measured_and_kept_bf16():
undefaulted = {f.name for f in _FAMILIES if f.te_quant_auto is None}
# FLUX.3 Klein and Qwen-Image-2.1 default to int8 ConvRot on their own; everything else is listed above.
- assert undefaulted == set(KEEP_BF16)
+ assert undefaulted == set(KEEP_BF16) | set(IN_TREE_TE_DEFAULT)
for name in KEEP_BF16:
assert resolve_te_quant_request(None, _fam(name).te_quant_auto) == (None, False)
+ import inspect
+
+ from core.inference import diffusion_minimax_h3_image as h3i
+
+ default = inspect.signature(h3i.load_minimax_h3_image_pipeline).parameters["text_encoder_quant"]
+ assert default.default == "int8"
@pytest.mark.parametrize(
diff --git a/studio/backend/tests/test_diffusion_minimax_h3_image.py b/studio/backend/tests/test_diffusion_minimax_h3_image.py
new file mode 100644
index 00000000000..400cb56a1bc
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_minimax_h3_image.py
@@ -0,0 +1,1141 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MiniMax-H3-Image on CPU with a tiny random release in the native layout: key conversion (round trip and the PR
+converter's own functions), the fixed-grid AdaLN table against the dense model, packing of text + reference rows,
+the canvas rule, the sampler, VAE encode/decode, the planner tiers, routing / family defaults and the backend load +
+generate path."""
+
+from __future__ import annotations
+
+import hashlib
+import importlib
+import importlib.machinery
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+_BACKEND = Path(__file__).resolve().parents[1]
+for p in (_BACKEND, Path(__file__).resolve().parent):
+ if str(p) not in sys.path:
+ sys.path.insert(0, str(p))
+
+CONVERTER = (
+ Path(__file__).with_name("data") / "minimax_h3_image" / "convert_minimax_h3_to_diffusers.py.txt"
+)
+
+
+def _stack():
+ torch = pytest.importorskip("torch")
+ pytest.importorskip("diffusers")
+ pytest.importorskip("transformers")
+ pytest.importorskip("accelerate")
+ try:
+ from diffusers import DiffusionPipeline # noqa: F401
+ except Exception as exc: # noqa: BLE001 - a broken diffusers / hub pair
+ pytest.skip(f"diffusers pipelines unavailable: {exc}")
+ return torch
+
+
[email protected](scope = "module")
+def h3i():
+ _stack()
+ from core.inference import diffusion_minimax_h3_image
+ return diffusion_minimax_h3_image
+
+
[email protected](scope = "module")
+def release(h3i, tmp_path_factory):
+ import _minimax_h3_image_tiny as tiny
+
+ root = tmp_path_factory.mktemp("h3img_release")
+ ref = tiny.make_tiny_release(root)
+ te = tiny.make_text_encoder(ref["tokenizer"])
+ return root, ref, te
+
+
[email protected](scope = "module")
+def pipe(h3i, release):
+ root, _ref, te = release
+ return h3i.load_minimax_h3_image_pipeline(str(root), device = "cpu", text_encoder = te)
+
+
+def test_vendored_files_match_the_manifest():
+ manifest = json.loads(
+ (_BACKEND / "vendor" / "diffusers_minimax_h3_image_manifest.json").read_text(
+ encoding = "utf-8"
+ )
+ )
+ assert manifest["license"] == "Apache-2.0"
+ assert len(manifest["upstream_commit"]) == 40
+ for rel, digest in manifest["files"].items():
+ data = (_BACKEND / "vendor" / rel).read_bytes()
+ assert hashlib.sha256(data).hexdigest() == digest, rel
+ for rel in manifest["files"]:
+ if rel.endswith(".py") and rel not in manifest["guard_files"]:
+ text = (_BACKEND / "vendor" / rel).read_text(encoding = "utf-8")
+ assert "Licensed under the Apache License, Version 2.0" in text
+ assert "\nfrom ." not in text
+
+
+def test_vendored_directory_is_not_importable_as_a_package(h3i, monkeypatch):
+ # native_tls appends vendor/ to sys.path; the guard keeps the copy reachable only through _load_vendored().
+ monkeypatch.syspath_prepend(str(_BACKEND / "vendor"))
+ monkeypatch.delitem(sys.modules, "diffusers_minimax_h3_image", raising = False)
+ with pytest.raises(ImportError, match = "_load_vendored"):
+ importlib.import_module("diffusers_minimax_h3_image.transformer_minimax_h3_image")
+ mod = h3i._load_vendored("transformer_minimax_h3_image")
+ assert mod.__name__ == "unsloth_studio_vendor_minimax_h3_image.transformer_minimax_h3_image"
+
+
+def test_classes_resolve_to_diffusers_or_the_vendored_copy(h3i):
+ t, v = h3i.transformer_class(), h3i.vae_class()
+ assert t.__name__ == "MiniMaxH3ImageTransformer2DModel"
+ assert v.__name__ == "AutoencoderKLMiniMaxH3Image"
+ assert h3i.classes_source() in ("diffusers", "vendored")
+ assert h3i.in_tree_class("MiniMaxH3DiTModel") is t
+ assert h3i.in_tree_class("MiniMaxH3ImagePipeline") is h3i.MiniMaxH3ImagePipeline
+
+
+def test_native_release_loads_bitwise(h3i, release):
+ import torch
+
+ root, ref, _te = release
+ model, stats = h3i.load_h3_image_transformer(
+ root, dtype = torch.bfloat16, scheme = None, device = "cpu", keep_on_device = True, table = False
+ )
+ assert stats["layout"] == "native"
+ want = {k: v for k, v in ref["transformer"].state_dict().items() if not k.startswith("rope.")}
+ got = {k: v for k, v in model.state_dict().items() if not k.startswith("rope.")}
+ assert set(got) == set(want)
+ for k in want:
+ assert got[k].dtype == want[k].dtype, k
+ assert torch.equal(got[k], want[k]), k
+ vae = h3i.load_h3_image_vae(root)
+ vwant = ref["vae"].state_dict()
+ vgot = vae.state_dict()
+ assert set(vgot) == set(vwant)
+ assert all(torch.equal(vgot[k], vwant[k]) for k in vwant)
+
+
+def _converter():
+ loader = importlib.machinery.SourceFileLoader("h3_converter_reference", str(CONVERTER))
+ spec = importlib.util.spec_from_loader("h3_converter_reference", loader)
+ module = importlib.util.module_from_spec(spec)
+ try:
+ loader.exec_module(module)
+ except Exception as exc: # noqa: BLE001 - it imports diffusers internals of its own era
+ pytest.skip(f"PR converter not importable here: {exc}")
+ return module
+
+
+def test_key_conversion_matches_the_pr_converter(h3i):
+ import torch
+
+ conv = _converter()
+ cfg = {
+ **conv.MINIMAX_H3_IMAGE_TRANSFORMER_CONFIG,
+ "num_attention_heads": 2,
+ "attention_head_dim": 8,
+ }
+ g = torch.Generator().manual_seed(0)
+ keys = [
+ ("blocks.3.attn.qkv_proj.weight", (2 * 3 * 8, 16)),
+ ("blocks.3.mlp.fc1.weight", (32, 16)),
+ ("blocks.3.mlp.fc2.weight", (16, 16)),
+ ("blocks.3.attn.out_proj.weight", (16, 16)),
+ ("blocks.3.attn.q_norm.weight", (8,)),
+ ("token_refiner.blocks.1.attn.qkv_proj.weight", (2 * 3 * 8, 16)),
+ ("final_layer.adaln_proj.linear.weight", (32, 8)),
+ ("time_embedder.proj_in.weight", (8, 4)),
+ ("condition_proj.bias", (16,)),
+ ("video_patch_proj.weight", (16, 96)),
+ ]
+ for key, shape in keys:
+ tensor = torch.randn(shape, generator = g)
+ source = tensor
+ if key.endswith(".attn.qkv_proj.weight"):
+ source = conv.reorder_interleaved_qkv(tensor, 2, 8)
+ want = dict(conv.convert_transformer_key(key, source, cfg))
+ got = dict(h3i.convert_native_transformer_tensor(key, tensor, 2, 8))
+ assert set(got) == set(want), key
+ assert all(torch.equal(got[k], want[k]) for k in want), key
+ for dropped in ("rope.inv_freq", "final_layer.video_out.weight"):
+ assert h3i.convert_native_transformer_tensor(dropped, torch.zeros(4), 2, 8) == []
+ vcfg = {"decoder_num_attention_heads": 2, "decoder_attention_head_dim": 8}
+ for key, shape in (
+ ("decoder.transformer_blocks.0.attn.to_qkv.weight", (48, 16)),
+ ("decoder.transformer_blocks.0.ff.w1.weight", (64, 16)),
+ ("encoder.down.2.block.1.nin_shortcut.weight", (4, 4, 1, 1, 1)),
+ ("encoder.down.1.downsample.conv.bias", (4,)),
+ ("decoder.x_embedder.weight", (16, 24)),
+ ):
+ tensor = torch.randn(shape, generator = g)
+ want = dict(conv.convert_video_vae_key(key, tensor, vcfg))
+ got = dict(h3i.convert_native_vae_tensor(key, tensor, 2, 8))
+ assert set(got) == set(want), key
+ assert all(torch.equal(got[k], want[k]) for k in want), key
+
+
+def test_pdd_heads_split_and_count_is_checked(h3i):
+ import torch
+
+ heads = {"video_out.weight": torch.randn(6, 4, 3), "video_out.bias": torch.randn(6, 4)}
+ out = h3i.split_pdd_heads(heads, 6)
+ assert sorted(out) == sorted(
+ f"pdd_proj_out.{i}.{s}" for i in range(6) for s in ("weight", "bias")
+ )
+ assert torch.equal(out["pdd_proj_out.5.weight"], heads["video_out.weight"][5])
+ with pytest.raises(ValueError):
+ h3i.split_pdd_heads(heads, 7)
+ with pytest.raises(KeyError):
+ h3i.split_pdd_heads({"video_out.weight": heads["video_out.weight"]}, 6)
+
+
+def test_native_config_maps_and_refuses_the_video_dit(h3i):
+ real = json.loads(
+ (
+ Path(__file__).with_name("data") / "minimax_h3_image" / "transformer_config.json"
+ ).read_text(encoding = "utf-8")
+ )
+ cfg = h3i.transformer_config_from_native(real)
+ assert cfg["num_layers"] == 50 and cfg["hidden_size"] == 5376 and cfg["ffn_dim"] == 14336
+ assert cfg["in_channels"] == 24 and cfg["text_dim"] == 5120 and cfg["num_pdd_heads"] == 6
+ assert cfg["freq_dim"] == 256 and cfg["rope_freq_dim"] == 16 and cfg["num_refiner_layers"] == 2
+ with pytest.raises(ValueError):
+ h3i.transformer_config_from_native({**real, "audio_enabled": True})
+
+
+def _forward_inputs(
+ h3i,
+ timesteps,
+ n_ref = 0,
+):
+ import torch
+
+ g = torch.Generator().manual_seed(5)
+ n_text, lh, lw = 7, 8, 6
+ tags = torch.full((n_text,), h3i.H3_IMAGE_TEXT_TAG)
+ grids = [(4, 4)] * n_ref
+ pos, tok, img, txt, nref = h3i.build_packed_layout(tags, grids, lh, lw)
+ rows = torch.randn(img.numel(), 96, generator = g)
+ text = torch.randn(1, n_text, 48, generator = g).to(torch.bfloat16)
+ plan = h3i.row_timestep_plan(
+ torch.tensor(timesteps), int(img.numel() + txt.numel()), img[:nref]
+ )
+ return rows, text, pos, tok, img, txt, plan
+
+
[email protected]("n_ref", [0, 2])
+def test_table_adaln_matches_the_dense_projection(h3i, release, n_ref):
+ import copy
+
+ import torch
+
+ _root, ref, _te = release
+ dense = ref["transformer"]
+ table = copy.deepcopy(dense)
+ assert h3i.convert_dense_transformer_to_table(table) == len(dense.transformer_blocks)
+ steps = h3i.grid_timesteps()[:6]
+ rows, text, pos, tok, img, txt, plan = _forward_inputs(h3i, steps, n_ref)
+ for i in (0, 3, 5):
+ unique, index = plan[i]
+ kw = dict(
+ hidden_states = rows[None],
+ encoder_hidden_states = text,
+ timestep = unique,
+ timestep_indices = index,
+ token_tags = tok,
+ position_ids = pos,
+ image_indices = img,
+ text_indices = txt,
+ pdd_step_index = i,
+ return_dict = False,
+ )
+ with torch.no_grad():
+ want = dense(**kw)[0]
+ got = table(**kw)[0]
+ assert torch.allclose(got, want, atol = 2e-2, rtol = 0), (i, (got - want).abs().max())
+ assert (got - want).abs().mean() < 2e-3
+
+
+def test_table_rows_are_exactly_the_dense_rows(h3i, release):
+ import torch
+
+ _root, ref, _te = release
+ dense = ref["transformer"]
+ grid = h3i.grid_timesteps()
+ temb = h3i.grid_temb((dense.time_proj, dense.time_embedder), grid)
+ lin = dense.transformer_blocks[0].adaln_proj.linear
+ rows = h3i.dense_adaln_rows(temb, lin.weight, lin.bias, lin.weight.dtype)
+ linear = h3i.table_linear(rows)
+ onehot = torch.nn.functional.one_hot(torch.arange(len(grid)), len(grid)).to(torch.bfloat16)
+ assert torch.equal(linear(onehot), rows)
+
+
+def test_grid_has_seven_timesteps_and_refuses_off_grid(h3i):
+ import torch
+
+ grid = h3i.grid_timesteps()
+ assert len(grid) == 7 and grid[0] == 0.0 and abs(grid[-1] - 0.999) < 1e-6
+ embedder = h3i._grid_embedder_class()(grid)
+ assert embedder(torch.tensor(grid)).argmax(dim = 1).tolist() == list(range(7))
+ with pytest.raises(ValueError, match = "fixed 6-step grid"):
+ embedder(torch.tensor([0.5]))
+
+
+def test_canvas_rule(h3i):
+ assert h3i.canvas_size(2048, 2048) == (2048, 2048)
+ assert h3i.canvas_size(1024, 1024) == (2048, 2048)
+ assert h3i.canvas_size(1024, 1024, 1024) == (1024, 1024)
+ h, w = h3i.canvas_size(768, 1344)
+ assert (
+ h % 32 == 0
+ and w % 32 == 0
+ and h * w >= 0.98 * 2048 * 2048
+ and abs(w / h - 1344 / 768) < 0.03
+ )
+ assert h3i.canvas_size(4096, 4096) == (4096, 4096)
+ with pytest.raises(ValueError):
+ h3i.canvas_size(512, 512, 512)
+ for w_, h_ in h3i.H3_IMAGE_PRESETS:
+ assert w_ % 32 == 0 and h_ % 32 == 0
+ for w_, h_ in h3i.H3_IMAGE_16MP_SIZES:
+ assert 15.5e6 < w_ * h_ < 17.5e6
+
+
+def test_reference_short_edge_bounds(h3i):
+ assert h3i.bound_short_edge(1500, 1200, 1024, 2048) == (1500, 1200)
+ assert h3i.bound_short_edge(600, 400, 1024, 2048) == (1536, 1024)
+ assert h3i.bound_short_edge(5000, 3000, 1024, 2048) == (3424, 2048)
+ assert h3i.bound_short_edge(2000, 1500, 1024, 1024) == (1376, 1024)
+
+
+def test_packed_layout_text_references_target(h3i):
+ import torch
+
+ tags = torch.tensor([1, 1, 0, 0, 0, 1, 1])
+ pos, tok, img, txt, nref = h3i.build_packed_layout(tags, [(4, 6), (8, 2)], 8, 8)
+ assert nref == 2 * 3 + 4 * 1
+ assert txt.tolist() == list(range(7)) and img[0] == 7 and img[-1] == pos.shape[0] - 1
+ assert tok[:7].tolist() == tags.tolist() and bool((tok[7:] == h3i.H3_IMAGE_IMAGE_TAG).all())
+ assert pos.dtype == torch.float64
+ assert pos[:7, 0].tolist() == list(range(7))
+ assert set(pos[7:13, 0].tolist()) == {7.0} and set(pos[13:17, 0].tolist()) == {8.0}
+ assert set(pos[17:, 0].tolist()) == {9.0} and pos.shape[0] == 7 + 10 + 16
+ plan = h3i.row_timestep_plan(torch.tensor([0.0, 0.5]), pos.shape[0], img[:nref])
+ unique, index = plan[1]
+ assert unique.tolist() == pytest.approx([0.5, 0.999])
+ assert bool((index[img[:nref]] == 1).all()) and bool((index[txt] == 0).all())
+
+
+def test_patchify_roundtrip(h3i):
+ import torch
+
+ x = torch.randn(1, 24, 1, 8, 6)
+ rows = h3i.patchify(x, (1, 2, 2))
+ assert rows.shape == (12, 96)
+ assert torch.equal(h3i.unpatchify(rows, 8, 6, 24, (1, 2, 2)), x[:, :, 0])
+
+
+def test_scheduler_is_the_velocity_euler_on_the_fixed_grid(h3i):
+ import torch
+
+ s = h3i.MiniMaxH3ImageScheduler(sample_sigmas = list(h3i.H3_IMAGE_SIGMAS))
+ s.set_timesteps()
+ assert s.timesteps.numel() == 6 and float(s.sigmas[-1]) == 0.0
+ x, v = torch.randn(4, 3), torch.randn(4, 3)
+ out = s.step(v, s.timesteps[0], x)[0]
+ assert torch.equal(out, x + (s.sigmas[0] - s.sigmas[1]) * v)
+ noisy = s.scale_noise(torch.ones(2), torch.tensor(0.999), torch.zeros(2))
+ assert torch.allclose(noisy, torch.full((2,), 0.999))
+
+
+def test_vae_encode_decode_shapes(h3i, release):
+ import torch
+
+ _root, ref, _te = release
+ vae = ref["vae"]
+ assert vae.spatial_compression_ratio == 16
+ x = torch.randn(1, 3, 64, 96)
+ post = vae.encode(x, return_dict = False)[0]
+ z = post.sample(generator = torch.Generator().manual_seed(0))
+ assert z.shape[1] == 24 and z.shape[-2:] == (4, 6)
+ y = vae.decode(z.squeeze(2) if z.ndim == 5 else z, return_dict = False)[0]
+ assert y.shape == (1, 3, 64, 96)
+
+
+def test_t2i_renders_at_the_requested_size_and_is_seeded(h3i, pipe):
+ import numpy as np
+ import torch
+
+ a = pipe(
+ prompt = "a cat",
+ height = 192,
+ width = 256,
+ generator = torch.Generator().manual_seed(0),
+ min_generation_area = 1024,
+ )
+ b = pipe(
+ prompt = "a cat",
+ height = 192,
+ width = 256,
+ generator = torch.Generator().manual_seed(0),
+ min_generation_area = 1024,
+ )
+ c = pipe(
+ prompt = "a cat",
+ height = 192,
+ width = 256,
+ generator = torch.Generator().manual_seed(1),
+ min_generation_area = 1024,
+ )
+ assert a.images[0].size == (256, 192)
+ assert np.array_equal(np.asarray(a.images[0]), np.asarray(b.images[0]))
+ assert not np.array_equal(np.asarray(a.images[0]), np.asarray(c.images[0]))
+
+
+def test_small_request_renders_on_the_4mp_canvas_and_resizes_back(h3i, pipe):
+ import torch
+
+ img = pipe(
+ prompt = "a cat", height = 64, width = 96, generator = torch.Generator().manual_seed(0)
+ ).images[0]
+ assert img.size == (96, 64)
+ w, h = pipe._unsloth_h3_last_stats["canvas"]
+ assert w * h >= 0.97 * 2048 * 2048 and w % 32 == 0 and h % 32 == 0
+
+
+def test_edit_packs_labels_vision_tokens_and_reference_rows(h3i, pipe, monkeypatch):
+ import numpy as np
+ import torch
+ from PIL import Image
+
+ if pipe.processor is None:
+ pytest.skip("no Qwen3-VL processor in this transformers")
+ seen = {}
+ real = h3i.qwen3vl_hidden_state
+
+ def spy(encoder, processor, token_ids, vision_inputs, **kw):
+ seen["ids"] = list(token_ids)
+ seen["grid"] = vision_inputs.get("image_grid_thw")
+ return real(encoder, processor, token_ids, vision_inputs, **kw)
+
+ monkeypatch.setattr(h3i, "qwen3vl_hidden_state", spy)
+ ref = Image.fromarray((np.random.RandomState(0).rand(40, 60, 3) * 255).astype("uint8"))
+ out = pipe(
+ prompt = "<Picture 1> a cat",
+ image = [ref],
+ height = 64,
+ width = 64,
+ generator = torch.Generator().manual_seed(0),
+ min_generation_area = 1024,
+ )
+ assert out.images[0].size == (64, 64)
+ tok = pipe.tokenizer
+ pad = tok.convert_tokens_to_ids("<|image_pad|>")
+ merge = pipe.processor.image_processor.merge_size**2
+ assert seen["ids"].count(pad) == int(seen["grid"][0].prod()) // merge
+ label = tok("Picture 1: ", add_special_tokens = False)["input_ids"]
+ assert seen["ids"][: len(label)] == label
+
+
+def test_edit_refuses_more_than_sixteen_references(h3i, pipe):
+ from PIL import Image
+ img = Image.new("RGB", (64, 64))
+ with pytest.raises(ValueError, match = "at most 16"):
+ pipe.prepare_references([img] * 17)
+
+
+def test_step_and_guidance_contract(h3i, pipe):
+ with pytest.raises(ValueError, match = "6 steps"):
+ pipe(prompt = "a cat", num_inference_steps = 20, height = 64, width = 64)
+ with pytest.raises(ValueError, match = "guidance"):
+ pipe(prompt = "a cat", guidance_scale = 3.5, height = 64, width = 64)
+ with pytest.raises(ValueError, match = "prompt"):
+ pipe(prompt = "", height = 64, width = 64)
+
+
+def test_step_callback_sees_bchw_latents_and_interrupt_stops(h3i, pipe):
+ import torch
+
+ calls = []
+
+ def cb(p, i, t, kw):
+ calls.append((i, tuple(kw["latents"].shape)))
+ if i == 1:
+ p._interrupt = True
+ return kw
+
+ out = pipe(
+ prompt = "a cat",
+ height = 64,
+ width = 64,
+ generator = torch.Generator().manual_seed(0),
+ min_generation_area = 1024,
+ callback_on_step_end = cb,
+ )
+ assert calls == [(0, (1, 24, 4, 4)), (1, (1, 24, 4, 4))]
+ assert out.images == []
+
+
+def test_int8_quantizes_only_the_block_linears(h3i, release):
+ import torch
+
+ pytest.importorskip("torchao")
+ root, _ref, _te = release
+ model, stats = h3i.load_h3_image_transformer(
+ root, dtype = torch.bfloat16, scheme = "int8", device = "cpu", keep_on_device = True
+ )
+ assert stats["quantized_linears"] == 6 * len(model.transformer_blocks)
+ block = model.transformer_blocks[0]
+ assert (
+ type(block.attn.to_q.weight).__name__ != "Parameter"
+ or not isinstance(block.attn.to_q.weight.data, torch.Tensor)
+ or type(block.attn.to_q.weight.data).__name__ != "Tensor"
+ )
+ assert type(block.adaln_proj.linear.weight.data).__name__ == "Tensor"
+ assert type(model.token_refiner.refiner_blocks[0].attn.to_q.weight.data).__name__ == "Tensor"
+
+
+ "vram, host, te, placement, te_kept",
+ [
+ (95.0, 177.0, 27.2, "resident", True), # G4: everything resident
+ (39.5, 83.5, 27.2, "dit_resident", True), # A100 40 GB: DiT resident, encoder streams
+ (22.0, 50.0, 27.2, "stream", True), # L4: blocks stream, the host holds the rest
+ (22.0, 39.0, 27.2, "stream", False), # small host: the encoder loads per prompt
+ (79.0, 167.0, 66.7, "dit_resident", True), # A100 80 GB with the bf16 encoder
+ ],
+)
+def test_planner_tiers(h3i, vram, host, te, placement, te_kept):
+ plan = h3i.plan_placement(
+ vram_gb = vram, host_gb = host, dit_gb = h3i.H3_IMAGE_DIT_TABLE_INT8_GB, te_gb = te
+ )
+ assert plan.placement == placement, plan.reasons
+ assert plan.te_kept_on_host is te_kept
+ if placement == "stream":
+ assert 10 <= plan.dit_resident_blocks < 50 and plan.te_resident_layers == 0
+ if placement == "dit_resident":
+ assert 0 <= plan.te_resident_layers <= 50
+
+
[email protected]("vram, host", [(22.0, 30.0), (23.5, 32.0), (11.0, 40.0)])
+def test_a_host_below_the_per_prompt_encoder_peak_is_refused_at_plan_time(h3i, vram, host):
+ # Per-prompt loading still holds the whole encoder beside the host-side DiT blocks while it loads.
+ with pytest.raises(RuntimeError, match = "free system memory"):
+ h3i.plan_placement(
+ vram_gb = vram, host_gb = host, dit_gb = h3i.H3_IMAGE_DIT_TABLE_INT8_GB, te_gb = 27.2
+ )
+
+
+def test_a_quantized_dit_without_the_adaln_table_budgets_the_dense_adaln(h3i):
+ dense = h3i.H3_IMAGE_DIT_BF16_GB - h3i.H3_IMAGE_DIT_TABLE_BF16_GB
+ assert h3i.dit_gb_for("int8") == h3i.H3_IMAGE_DIT_TABLE_INT8_GB
+ assert h3i.dit_gb_for("int8", table = False) == pytest.approx(
+ h3i.H3_IMAGE_DIT_TABLE_INT8_GB + dense
+ )
+ assert h3i.dit_gb_for(None, table = False) == pytest.approx(h3i.H3_IMAGE_DIT_BF16_GB)
+ # A100 40 GB: the table-form int8 DiT stays resident, the dense-AdaLN one (~48 GB) has to stream.
+ args = dict(vram_gb = 39.5, host_gb = 83.5, te_gb = 27.2)
+ assert h3i.plan_placement(dit_gb = h3i.dit_gb_for("int8"), **args).placement == "dit_resident"
+ assert (
+ h3i.plan_placement(dit_gb = h3i.dit_gb_for("int8", table = False), **args).placement == "stream"
+ )
+
+
+def test_auto_scheme_is_bf16_only_when_it_fits(h3i, monkeypatch):
+ import torch
+
+ assert h3i.resolve_dit_scheme(None, device = "cpu", vram_gb = None)[0] is None
+ assert h3i.resolve_dit_scheme("int8", device = "cpu", vram_gb = None)[0] is None
+ if torch.cuda.is_available():
+ assert h3i.resolve_dit_scheme(None, device = "cuda", vram_gb = 39.5)[0] == "int8"
+ assert h3i.resolve_dit_scheme(None, device = "cuda", vram_gb = 95.0)[0] is None
+
+
+def test_streamer_runs_blocks_from_the_host(h3i):
+ import torch
+
+ blocks = torch.nn.ModuleList([torch.nn.Linear(4, 4) for _ in range(3)])
+ x = torch.randn(2, 4)
+ want = blocks[2](blocks[1](blocks[0](x)))
+ streamer = h3i.ModuleListStreamer(blocks, "cpu", resident = 1)
+ assert streamer.streamed == 2
+ got = blocks[2](blocks[1](blocks[0](x)))
+ assert torch.equal(got, want)
+ streamer.remove()
+
+
+def test_family_routing_and_defaults():
+ from core.inference import diffusion_families as df
+ from core.inference import video_families as vf
+
+ fam = df.detect_family("MiniMaxAI/MiniMax-H3-Image")
+ assert fam is not None and fam.name == "minimax-h3-image"
+ assert df.detect_family_by_pipeline_class("MiniMaxH3ImagePipeline") is fam
+ assert df.default_generation_params("MiniMaxAI/MiniMax-H3-Image") == (6, 1.0)
+ assert fam.reference and fam.reference_edit and fam.max_condition_images == 16
+ assert not fam.uses_guidance and not fam.uses_negative_prompt
+ assert fam.te_quant_auto is None and fam.dimension_multiple == 32
+ assert df.in_tree_pipeline("MiniMaxH3ImagePipeline")
+ # The video registry shares the "minimax-h3" token: it must not claim the image model.
+ assert vf.detect_video_family("MiniMaxAI/MiniMax-H3-Image") is None
+ assert "Images page" in (
+ vf.unsupported_video_checkpoint_reason("MiniMaxAI/MiniMax-H3-Image") or ""
+ )
+ assert vf.detect_video_family("MiniMaxAI/MiniMax-H3").name == "minimax-h3"
+
+
+def test_backend_loads_and_generates(h3i, release, tmp_path, monkeypatch):
+ import base64
+ import io
+
+ import torch
+ from PIL import Image
+
+ from core.inference import diffusion_attention
+ from core.inference.diffusion import DiffusionBackend
+
+ # A real CPU load memoizes the SDPA probe process-wide; later tests run against a faked torch.
+ monkeypatch.setattr(diffusion_attention, "_SDPA_PROBE_CACHE", {})
+ monkeypatch.setenv(h3i.ALLOW_ANY_DEVICE_ENV, "1")
+
+ root, _ref, te = release
+ te.save_pretrained(str(root / "text_encoder"))
+ backend = DiffusionBackend()
+ try:
+ status = backend.load_pipeline(str(root), model_kind = "pipeline")
+ assert backend._state is not None and backend._state.family.name == "minimax-h3-image"
+ assert backend._state.pipe._unsloth_h3_report.layout == "native"
+ out = backend.generate(prompt = "a cat", width = 256, height = 256, steps = 6, guidance = 1.0, seed = 3)
+ assert out["images"], out
+ raw = out["images"][0]
+ data = raw.get("b64") if isinstance(raw, dict) else raw
+ if isinstance(data, str):
+ img = Image.open(io.BytesIO(base64.b64decode(data.split(",")[-1])))
+ assert img.size == (256, 256)
+ assert status
+ finally:
+ backend.unload()
+
+
[email protected]("layers", [51, 53])
+def test_read_layer_hook_equals_hidden_states_50(h3i, release, layers):
+ import torch
+
+ import _minimax_h3_image_tiny as tiny
+
+ _root, ref, te = release
+ encoder = te if layers == 51 else tiny.make_text_encoder(ref["tokenizer"], layers = layers)
+ ids = ref["tokenizer"]("a cat", add_special_tokens = False)["input_ids"]
+ with torch.no_grad():
+ got = h3i.qwen3vl_hidden_state(encoder, None, ids, {}, layer = 50, device = torch.device("cpu"))
+ x = torch.tensor([ids])
+ want = encoder.model(
+ input_ids = x, attention_mask = torch.ones_like(x), output_hidden_states = True
+ ).hidden_states[50]
+ assert torch.equal(got, want)
+
+
+def test_release_drops_components_and_hooks(h3i, release):
+ import torch
+
+ root, _ref, te = release
+ import copy
+
+ supplied = copy.deepcopy(te)
+ p = h3i.load_minimax_h3_image_pipeline(str(root), device = "cpu", text_encoder = supplied)
+ assert p._unsloth_h3_owned == {"transformer", "vae"}
+ held_dit, held_vae = p.transformer, p.vae # an outside reference that outlives the unload
+ blocks = p.transformer.transformer_blocks
+ p._unsloth_h3_streamers.append(h3i.ModuleListStreamer(blocks, "cpu", resident = 0))
+ assert any(b._forward_pre_hooks for b in blocks)
+ p.release()
+ assert p.transformer is None and p.text_encoder is None and p.vae is None
+ assert not any(b._forward_pre_hooks for b in blocks)
+ assert torch.is_tensor(next(blocks[0].parameters()))
+ # Owned components are emptied in place even while held; a supplied encoder is the caller's and stays intact.
+ assert all(t.numel() == 0 for t in [*held_dit.parameters(), *held_vae.parameters()])
+ assert all(
+ a.numel() and torch.equal(a, b) for a, b in zip(supplied.parameters(), te.parameters())
+ )
+
+
+def test_freeing_storage_reaches_quantized_weights(h3i):
+ import gc
+ import weakref
+
+ import torch
+
+ pytest.importorskip("torchao")
+ from torchao.quantization import quantize_
+
+ from core.inference.diffusion_transformer_quant import _make_quant_config
+
+ block = torch.nn.Sequential(torch.nn.Linear(256, 64)).to(torch.bfloat16)
+ quantize_(block, _make_quant_config("int8"))
+ weight = block[0].weight
+ inner = [weakref.ref(getattr(weight, n)) for n in weight.__tensor_flatten__()[0]]
+ assert h3i.free_module_storage(block) == 2
+ gc.collect()
+ assert weight.numel() == 0 and block[0].bias.numel() == 0
+ assert all(ref() is None for ref in inner)
+
+
+def test_int8_linears_pad_rows_to_the_supported_multiple_exactly(h3i):
+ import torch
+
+ pytest.importorskip("torchao")
+ from torchao.quantization import quantize_
+
+ from core.inference.diffusion_transformer_quant import _make_quant_config
+
+ torch.manual_seed(0)
+ block = torch.nn.Sequential(torch.nn.Linear(256, 64, bias = False)).to(torch.bfloat16)
+ quantize_(block, _make_quant_config("int8"))
+ x = torch.randn(1, 37, 256, dtype = torch.bfloat16)
+ with torch.no_grad():
+ want = block[0](x)
+ assert h3i.wrap_row_multiple(block, ["0"]) == 1
+ seen = []
+ block[0].inner.register_forward_pre_hook(lambda m, a: seen.append(a[0].shape[0]))
+ with torch.no_grad():
+ got = block[0](x)
+ assert h3i.ROW_MULTIPLE == 32
+ assert seen == [64] and got.shape == want.shape
+ assert torch.equal(got, want)
+
+
+def test_the_family_declares_the_hosted_dits(h3i):
+ from core.inference.diffusion_families import (
+ detect_family,
+ family_prequant_filename,
+ family_prequant_repo,
+ )
+
+ fam = detect_family("MiniMaxAI/MiniMax-H3-Image")
+ assert fam.name == "minimax-h3-image"
+ for scheme, filename in (
+ ("int8", "MiniMax-H3-Image-INT8-ConvRot.safetensors"),
+ ("fp8", "MiniMax-H3-Image-FP8.safetensors"),
+ ):
+ assert (
+ family_prequant_repo(fam, scheme, "MiniMaxAI/MiniMax-H3-Image")
+ == "unsloth/MiniMax-H3-Image-FP8"
+ )
+ assert family_prequant_filename(fam, scheme) == filename
+
+
+def test_an_unpublished_hosted_dit_falls_back_without_fetching(h3i, monkeypatch):
+ import huggingface_hub
+
+ from core.inference import diffusion_prequant
+
+ calls = []
+ monkeypatch.setattr(huggingface_hub, "try_to_load_from_cache", lambda *a, **k: None)
+ monkeypatch.setattr(huggingface_hub, "file_exists", lambda *a, **k: calls.append(a) or False)
+ monkeypatch.setattr(
+ diffusion_prequant, "load_prequantized_transformer", lambda *a, **k: pytest.fail("fetched")
+ )
+ h3i._hub_file_exists.cache_clear()
+ kwargs = dict(
+ base = "MiniMaxAI/MiniMax-H3-Image",
+ device = "cpu",
+ dtype = None,
+ hf_token = None,
+ cache_dir = None,
+ keep_on_device = False,
+ )
+ assert h3i.load_hosted_prequant_transformer("int8", local_files_only = False, **kwargs) is None
+ assert calls == [("unsloth/MiniMax-H3-Image-FP8", "MiniMax-H3-Image-INT8-ConvRot.safetensors")]
+ # Offline: the cache alone decides, never the Hub.
+ assert h3i.load_hosted_prequant_transformer("fp8", local_files_only = True, **kwargs) is None
+ assert len(calls) == 1
+ h3i._hub_file_exists.cache_clear()
+
+
+def test_a_published_hosted_dit_loads_through_the_curve_hook(h3i, monkeypatch):
+ import huggingface_hub
+
+ from core.inference import diffusion_prequant
+
+ seen = {}
+
+ def _load(cls, base, source, **kwargs):
+ seen.update(cls = cls, base = base, source = source, **kwargs)
+ return "model"
+
+ monkeypatch.setattr(huggingface_hub, "try_to_load_from_cache", lambda *a, **k: "/cache/file")
+ monkeypatch.setattr(diffusion_prequant, "load_prequantized_transformer", _load)
+ got = h3i.load_hosted_prequant_transformer(
+ "int8",
+ base = "MiniMaxAI/MiniMax-H3-Image",
+ device = "cpu",
+ dtype = None,
+ hf_token = None,
+ cache_dir = None,
+ local_files_only = True,
+ keep_on_device = False,
+ )
+ assert got == "model"
+ assert seen["source"].location == "unsloth/MiniMax-H3-Image-FP8"
+ assert seen["source"].filename == "MiniMax-H3-Image-INT8-ConvRot.safetensors"
+ assert seen["family"] == "minimax-h3-image" and seen["scheme"] == "int8"
+
+
+def test_the_prepare_hook_installs_the_curve_form(h3i):
+ import torch
+
+ from _minimax_h3_image_tiny import TINY_TRANSFORMER_NATIVE
+
+ cfg = h3i.transformer_config_from_native(TINY_TRANSFORMER_NATIVE)
+ model = h3i.transformer_class()(**cfg).eval()
+ prepare = h3i.h3_image_prepare_prequant_model()
+ prepare(model, {"adaln_form": "curve", "curve_dim": "8", "curve_grid": "17"})
+ assert model.transformer_blocks[0].adaln_proj.linear.in_features == 8
+ assert model.norm_out.linear.in_features == 8
+ assert tuple(model.time_embedder.table.shape) == (17, 8)
+ # The pipeline's raw float32 timesteps (0.999 reference rows included) drive the curve directly.
+ t = torch.tensor([0.25, 0.999], dtype = torch.float32)
+ assert torch.equal(model.time_proj(t), t)
+ dense = h3i.transformer_class()(**cfg).eval()
+ prepare(dense, {})
+ assert dense.transformer_blocks[0].adaln_proj.linear.in_features != 8
+
+
+ "device, where",
+ [("mps", "Apple Silicon"), ("cpu", "the CPU"), ("xpu", "this xpu device")],
+)
+def test_only_cuda_and_rocm_hosts_may_load(h3i, monkeypatch, device, where):
+ monkeypatch.delenv(h3i.ALLOW_ANY_DEVICE_ENV, raising = False)
+ assert h3i.h3_image_device_refusal("cuda") is None
+ assert h3i.h3_image_device_refusal("cuda:1") is None
+ refusal = h3i.h3_image_device_refusal(device)
+ assert refusal.startswith("MiniMax-H3-Image needs an NVIDIA or AMD GPU") and where in refusal
+
+
+def test_the_backend_refuses_a_cpu_host_before_loading(h3i, release, monkeypatch):
+ from core.inference.diffusion import DiffusionBackend
+
+ monkeypatch.delenv(h3i.ALLOW_ANY_DEVICE_ENV, raising = False)
+ root, _ref, te = release
+ if not (root / "text_encoder" / "config.json").is_file():
+ te.save_pretrained(
+ str(root / "text_encoder")
+ ) # a complete local pipeline, so only the device can refuse
+ backend = DiffusionBackend()
+ with pytest.raises(ValueError, match = "needs an NVIDIA or AMD GPU"):
+ backend.validate_load_request(str(root), model_kind = "pipeline")
+ assert backend._state is None
+
+
+ "repo_id",
+ [
+ "MiniMaxAI/MiniMax-H3-Image",
+ "unsloth/MiniMax-H3-Image-FP8",
+ "minimaxai/minimax_h3_image",
+ "/models/MiniMax-H3-Image",
+ ],
+)
+def test_h3_image_ids_route_to_the_image_family_not_the_video_one(repo_id):
+ from core.inference import diffusion_families as df
+ from core.inference import video_families as vf
+
+ assert vf.detect_video_family(repo_id) is None
+ assert "Images page" in (vf.unsupported_video_checkpoint_reason(repo_id) or "")
+ assert df.detect_family(repo_id).name == "minimax-h3-image"
+
+
+ "files",
+ [
+ {"model_index.json": {"_class_name": "MiniMaxH3ImagePipeline"}},
+ {"model_index.json": {"_class_name": "Other", "_minimax_h3": {"partition": "image"}}},
+ {
+ "modular_model_index.json": {
+ "_class_name": "X",
+ "_blocks_class_name": "MiniMaxH3ImageBlocks",
+ }
+ },
+ {"transformer_diffusers/config.json": {"_class_name": "MiniMaxH3ImageTransformer2DModel"}},
+ ],
+)
+def test_a_folder_that_says_h3_image_is_never_a_video_pick(tmp_path, files):
+ import json
+
+ from core.inference import diffusion_families as df
+ from core.inference import video_families as vf
+
+ # Named only "h3", the video family's alias: the folder's own metadata decides.
+ root = tmp_path / "h3"
+ for name, payload in files.items():
+ (root / name).parent.mkdir(parents = True, exist_ok = True)
+ (root / name).write_text(json.dumps(payload))
+ assert vf.is_minimax_h3_image_folder(str(root))
+ assert vf.detect_video_family(str(root)) is None
+ assert "Images page" in (vf.unsupported_video_checkpoint_reason(str(root)) or "")
+ if (
+ "model_index.json" in files
+ and files["model_index.json"]["_class_name"] == "MiniMaxH3ImagePipeline"
+ ):
+ assert df.detect_family_for_pick(str(root)).name == "minimax-h3-image"
+
+
+def test_the_h3_video_release_and_its_folders_stay_video(tmp_path):
+ import json
+
+ from core.inference import video_families as vf
+
+ for repo_id in ("MiniMaxAI/MiniMax-H3", "unsloth/MiniMax-H3-FP8", "unsloth/MiniMax-H3-GGUF"):
+ assert vf.detect_video_family(repo_id).name == "minimax-h3"
+ root = tmp_path / "MiniMax-H3"
+ root.mkdir()
+ (root / "modular_model_index.json").write_text(
+ json.dumps({"_class_name": "MiniMaxH3ModularPipeline"})
+ )
+ assert not vf.is_minimax_h3_image_folder(str(root))
+ assert vf.detect_video_family(str(root)).name == "minimax-h3"
+
+
+def test_the_hosted_files_conditioning_path_runs_dense(h3i, monkeypatch):
+ import torch
+
+ pytest.importorskip("torchao")
+ from torchao.quantization import quantize_
+
+ from core.inference.diffusion_quant_pad import is_quantized_linear
+ from core.inference.diffusion_transformer_quant import _make_quant_config
+
+ torch.manual_seed(0)
+ model = torch.nn.Module()
+ model.context_embedder = torch.nn.Linear(256, 64)
+ model.token_refiner = torch.nn.Module()
+ model.token_refiner.refiner_blocks = torch.nn.ModuleList([torch.nn.Linear(256, 256)])
+ model.transformer_blocks = torch.nn.ModuleList([torch.nn.Linear(256, 256)])
+ model.to(torch.bfloat16)
+ quantize_(model, _make_quant_config("int8"))
+ want = model.context_embedder.weight.dequantize().to(torch.bfloat16)
+ monkeypatch.setenv(h3i.QUANT_CONDITIONING_ENV, "1")
+ assert h3i.densify_conditioning_path(model) == 0
+ monkeypatch.delenv(h3i.QUANT_CONDITIONING_ENV)
+ assert h3i.densify_conditioning_path(model) == 2
+ assert not is_quantized_linear(model.context_embedder)
+ assert not is_quantized_linear(model.token_refiner.refiner_blocks[0])
+ assert torch.equal(model.context_embedder.weight, want)
+ assert is_quantized_linear(model.transformer_blocks[0])
+
+
+def _release_copy(root, dst, with_encoder):
+ import shutil
+ shutil.copytree(
+ root, dst, ignore = None if with_encoder else shutil.ignore_patterns("text_encoder")
+ )
+ return dst
+
+
+def _bf16_sized(h3i, monkeypatch, encoder_cls):
+ """``_module_gb`` reports the 66.7 GB bf16 encoder for the tiny one; the planner calls are recorded."""
+ real_gb, real_plan = h3i._module_gb, h3i.plan_placement
+ sized = []
+ monkeypatch.setattr(
+ h3i,
+ "_module_gb",
+ lambda m: h3i.H3_IMAGE_TE_BF16_GB if isinstance(m, encoder_cls) else real_gb(m),
+ )
+ monkeypatch.setattr(
+ h3i, "plan_placement", lambda **kw: sized.append(kw["te_gb"]) or real_plan(**kw)
+ )
+ return sized
+
+
+def test_the_plan_is_redone_when_the_int8_encoder_falls_back_to_bf16(
+ h3i, release, tmp_path, monkeypatch
+):
+ import copy
+
+ root, _ref, te = release
+ rel = _release_copy(root, tmp_path / "rel", with_encoder = False)
+ monkeypatch.setattr(
+ h3i, "load_h3_image_text_encoder", lambda scheme, **kw: (copy.deepcopy(te), None)
+ )
+ sized = _bf16_sized(h3i, monkeypatch, type(te))
+ pipe = h3i.load_minimax_h3_image_pipeline(str(rel), device = "cpu", text_encoder_quant = "int8")
+ assert sized == [h3i.H3_IMAGE_TE_INT8_GB, h3i.H3_IMAGE_TE_BF16_GB]
+ assert pipe._unsloth_h3_plan.te_gb == h3i.H3_IMAGE_TE_BF16_GB
+ assert "planned again" in pipe._unsloth_h3_plan.reasons[0]
+ pipe.release()
+
+
+def _no_hosted_int8(monkeypatch):
+ """The hosted int8 encoder is unusable; a bf16 ``from_pretrained`` would be recorded (never allocated)."""
+ import transformers
+ from core.inference import video_minimax_h3_te
+
+ monkeypatch.setattr(video_minimax_h3_te, "load_h3_quantized_text_encoder", lambda *a, **k: None)
+ calls = []
+
+ def _record(*a, **k):
+ calls.append(a)
+ raise AssertionError("the bf16 encoder was materialised")
+
+ monkeypatch.setattr(transformers.Qwen3VLForConditionalGeneration, "from_pretrained", _record)
+ return calls
+
+
+def test_the_bf16_fallback_is_refused_before_it_allocates_when_the_host_cannot_hold_it(
+ h3i, release, tmp_path, monkeypatch
+):
+ root, _ref, _te = release
+ rel = _release_copy(root, tmp_path / "rel", with_encoder = False)
+ calls = _no_hosted_int8(monkeypatch)
+ # Room for the 27.2 GB int8 encoder, not for the 66.7 GB bf16 one.
+ monkeypatch.setenv(h3i.HOST_BUDGET_ENV, "40")
+ with pytest.raises(RuntimeError, match = "free system memory"):
+ h3i.load_minimax_h3_image_pipeline(str(rel), device = "cpu", text_encoder_quant = "int8")
+ assert calls == []
+
+
+def test_a_local_dense_encoder_is_refused_before_it_allocates_when_the_host_cannot_hold_it(
+ h3i, release, tmp_path, monkeypatch
+):
+ import transformers
+
+ root, _ref, _te = release
+ rel = _release_copy(root, tmp_path / "rel", with_encoder = False)
+ folder = rel / "text_encoder"
+ folder.mkdir(exist_ok = True)
+ (folder / "config.json").write_text("{}")
+ with open(folder / "model.safetensors", "wb") as f: # sparse: 66 GB on paper, no disk used
+ f.truncate(66 * 10**9)
+ calls = []
+ monkeypatch.setattr(
+ transformers.Qwen3VLForConditionalGeneration,
+ "from_pretrained",
+ lambda *a, **k: calls.append(a),
+ )
+ monkeypatch.setenv(h3i.HOST_BUDGET_ENV, "40")
+ with pytest.raises(RuntimeError, match = "free system memory"):
+ h3i.load_minimax_h3_image_pipeline(str(rel), device = "cpu", text_encoder_quant = "int8")
+ assert calls == []
+
+
+def test_the_per_prompt_encoder_refuses_the_bf16_fallback_before_it_allocates(h3i, monkeypatch):
+ calls = _no_hosted_int8(monkeypatch)
+
+ def refuse():
+ raise RuntimeError("refused")
+
+ with pytest.raises(RuntimeError, match = "refused"):
+ h3i.load_h3_image_text_encoder(
+ "int8",
+ dtype = None,
+ text_encoder_repo = "x/y",
+ hf_token = None,
+ cache_dir = None,
+ local_files_only = True,
+ fallback_check = refuse,
+ )
+ assert calls == []
+
+
+def test_a_streamed_dit_is_placed_before_the_encoder_is_materialised(
+ h3i, release, tmp_path, monkeypatch
+):
+ import copy
+
+ root, _ref, te = release
+ rel = _release_copy(root, tmp_path / "rel", with_encoder = False)
+ real_plan = h3i.plan_placement
+
+ def stream(**kw):
+ plan = real_plan(**kw)
+ plan.placement, plan.dit_resident_blocks = h3i.PLACE_STREAM, 1
+ return plan
+
+ seen = []
+
+ def load_te(scheme, **kw):
+ blocks = pipe_blocks[0]
+ seen.append([bool(b._forward_pre_hooks) for b in blocks])
+ return copy.deepcopy(te), "int8"
+
+ pipe_blocks = []
+ real_place = h3i._place_streamed_dit
+
+ def place(transformer, *a, **k):
+ pipe_blocks.append(transformer.transformer_blocks)
+ return real_place(transformer, *a, **k)
+
+ monkeypatch.setattr(h3i, "plan_placement", stream)
+ monkeypatch.setattr(h3i, "_place_streamed_dit", place)
+ monkeypatch.setattr(h3i, "load_h3_image_text_encoder", load_te)
+ pipe = h3i.load_minimax_h3_image_pipeline(str(rel), device = "cpu", text_encoder_quant = "int8")
+ assert len(pipe_blocks) == 1 and seen and seen[0][0] is False and all(seen[0][1:])
+ assert [getattr(x, "_unsloth_role", None) for x in pipe._unsloth_h3_streamers].count(
+ "transformer"
+ ) == 1
+ pipe.release()
+
+
+def test_the_plan_is_redone_for_a_local_text_encoder_folder(h3i, release, tmp_path, monkeypatch):
+ import transformers
+
+ root, _ref, te = release
+ rel = _release_copy(root, tmp_path / "rel", with_encoder = False)
+ te.save_pretrained(str(rel / "text_encoder"))
+ sized = _bf16_sized(h3i, monkeypatch, transformers.Qwen3VLForConditionalGeneration)
+ pipe = h3i.load_minimax_h3_image_pipeline(str(rel), device = "cpu", text_encoder_quant = "int8")
+ assert sized == [h3i.H3_IMAGE_TE_INT8_GB, h3i.H3_IMAGE_TE_BF16_GB]
+ assert pipe._unsloth_h3_plan.te_gb == h3i.H3_IMAGE_TE_BF16_GB
+ pipe.release()
+
+
+def test_replanning_moves_to_a_tier_that_fits_and_refuses_when_none_does(h3i):
+ args = {
+ "vram_gb": 95.0,
+ "host_gb": 177.0,
+ "dit_gb": 21.9,
+ "width": 2048,
+ "height": 2048,
+ "forced": None,
+ }
+ plan = h3i.plan_placement(te_gb = h3i.H3_IMAGE_TE_INT8_GB, **args)
+ assert plan.placement == h3i.PLACE_RESIDENT
+ # The whole bf16 encoder no longer fits beside the DiT: the DiT stays, the encoder streams.
+ again = h3i.replan_for_encoder(plan, h3i.H3_IMAGE_TE_BF16_GB, args)
+ assert again.placement == h3i.PLACE_DIT_RESIDENT and again.te_gb == h3i.H3_IMAGE_TE_BF16_GB
+ assert h3i.replan_for_encoder(plan, h3i.H3_IMAGE_TE_INT8_GB + 0.5, args) is plan
+ small = dict(args, vram_gb = 6.0)
+ with pytest.raises(RuntimeError, match = "no placement fits"):
+ h3i.replan_for_encoder(
+ h3i.plan_placement(te_gb = 27.2, **small), h3i.H3_IMAGE_TE_BF16_GB, small
+ )
+
+
+def test_the_per_prompt_encoder_refuses_a_bf16_fallback(h3i, release, tmp_path, monkeypatch):
+ import copy
+
+ root, _ref, te = release
+ rel = _release_copy(root, tmp_path / "rel", with_encoder = False)
+ real_plan = h3i.plan_placement
+
+ def low_host(**kw):
+ plan = real_plan(**kw)
+ plan.te_kept_on_host = False # the low-host tier: the encoder loads for each prompt
+ return plan
+
+ monkeypatch.setattr(h3i, "plan_placement", low_host)
+ pipe = h3i.load_minimax_h3_image_pipeline(str(rel), device = "cpu", text_encoder_quant = "int8")
+ assert pipe.text_encoder is None and pipe._unsloth_h3_te_loader is not None
+ monkeypatch.setattr(
+ h3i, "load_h3_image_text_encoder", lambda scheme, **kw: (copy.deepcopy(te), None)
+ )
+ with pytest.raises(RuntimeError, match = "bf16 encoder it would fall back to"):
+ pipe._unsloth_h3_te_loader()
+ pipe.release()
diff --git a/studio/backend/tests/test_diffusion_prompt_hint.py b/studio/backend/tests/test_diffusion_prompt_hint.py
index 9ddc58a1fe1..8ebf2e1d8d3 100644
--- a/studio/backend/tests/test_diffusion_prompt_hint.py
+++ b/studio/backend/tests/test_diffusion_prompt_hint.py
@@ -33,7 +33,14 @@ from test_diffusion_backend import ( # noqa: E402,F401
from test_mlx_diffusion_backend import _load as _load_mlx, env # noqa: E402,F401
from test_sd_cpp_backend import _loaded_backend as _loaded_sd_cpp_backend # noqa: E402
-_TIPPED = {"flux.1-kontext", "qwen-image-edit", "qwen-image-2.1", "ideogram-4", "moss-image"}
+_TIPPED = {
+ "flux.1-kontext",
+ "qwen-image-edit",
+ "qwen-image-2.1",
+ "ideogram-4",
+ "moss-image",
+ "minimax-h3-image",
+}
_HINT = "Describe the scene first."
diff --git a/studio/backend/tests/test_diffusion_prompt_length_policy.py b/studio/backend/tests/test_diffusion_prompt_length_policy.py
index 5fafd5a56ad..1555db3e1d4 100644
--- a/studio/backend/tests/test_diffusion_prompt_length_policy.py
+++ b/studio/backend/tests/test_diffusion_prompt_length_policy.py
@@ -20,6 +20,7 @@ VARIABLE = (
"QwenImage21Transformer2DModel",
"ZImageTransformer2DModel",
"MiniMaxH3Transformer3DModel",
+ "MiniMaxH3ImageTransformer2DModel",
"Cosmos3OmniTransformer",
"MOSSImageTransformer",
)
diff --git a/studio/backend/tests/test_diffusion_shift_override.py b/studio/backend/tests/test_diffusion_shift_override.py
index ed8e395a27a..bb8c656f987 100644
--- a/studio/backend/tests/test_diffusion_shift_override.py
+++ b/studio/backend/tests/test_diffusion_shift_override.py
@@ -60,6 +60,8 @@ EXPECTED_NODES = {
"hunyuanimage-2.1": SHIFT_NODE_SD3,
"hidream-i1": SHIFT_NODE_SD3,
"moss-image": SHIFT_NODE_SD3,
+ # Fixed PDD sigma grid (one output head per step): nothing to shift.
+ "minimax-h3-image": None,
"chroma": SHIFT_NODE_SD3, # the template's ModelSamplingAuraFlow is the same time_snr_shift
# Not offered: Ideogram 4 runs its own mu/std preset, SDXL is not flow matching, Anima keeps its shift 3.
"anima": None,
diff --git a/studio/backend/tests/test_image_fetch_ids_unsloth.py b/studio/backend/tests/test_image_fetch_ids_unsloth.py
index 53b78ecab5f..4fe7f15296b 100644
--- a/studio/backend/tests/test_image_fetch_ids_unsloth.py
+++ b/studio/backend/tests/test_image_fetch_ids_unsloth.py
@@ -38,6 +38,8 @@ _LICENCE_BLOCKED: dict[str, str] = {
_UNMIRRORED: dict[str, str] = {
"lodestones/chroma1-hd": "Chroma",
"circlestone-labs/anima-base-v1.0-diffusers": "Anima",
+ # Native-layout release read by Studio's own loader; no unsloth mirror of it exists yet.
+ "minimaxai/minimax-h3-image": "MiniMax-H3-Image",
}
# Third-party DENOISER artifacts the catalog still offers. Not a companion fetch and not licence-blocked: there is
diff --git a/studio/backend/tests/test_vae_tile_geometry.py b/studio/backend/tests/test_vae_tile_geometry.py
index 35b8cd0270d..e24f7f8ef4c 100644
--- a/studio/backend/tests/test_vae_tile_geometry.py
+++ b/studio/backend/tests/test_vae_tile_geometry.py
@@ -86,6 +86,13 @@ SMALLER_GEOMETRY_OK: dict[str, Floor] = {
"AutoencoderKLMiniMaxH3@16x": Floor(
16, 4, 16, "decoder works at its 256 px tile; untiled is out of distribution"
),
+ # The release's own geometry (256 px tiles, 64 px minimum overlap), which the loader enforces.
+ "AutoencoderKLMiniMaxH3Image@16x": Floor(
+ 16,
+ 4,
+ 16,
+ "seam audit on 16 decodes at 2048: tile ends match plain rows, starts 1.11x, no visible line",
+ ),
"AutoencoderKLHunyuanVideo15@16x": Floor(
16, 4, 4, "video seam audit: PSNR >= 50.6 dB, no line at the boundaries"
),
@@ -222,6 +229,11 @@ def _vae_class(name: str):
cls = getattr(importlib.import_module(mod), name, None)
if cls is not None:
break
+ if cls is None:
+ # MiniMax-H3-Image's VAE: diffusers' class when installed, else Studio's vendored copy.
+ h3i = importlib.import_module("core.inference.diffusion_minimax_h3_image")
+ if name == h3i.H3_IMAGE_VAE_CLASS:
+ cls = h3i.vae_class()
return cls
diff --git a/studio/backend/tests/test_vendored_truststore.py b/studio/backend/tests/test_vendored_truststore.py
index 306fcf8ca4b..0274e0510ac 100644
--- a/studio/backend/tests/test_vendored_truststore.py
+++ b/studio/backend/tests/test_vendored_truststore.py
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Vendored truststore and laya stay byte-identical to upstream except manifest-recorded patches."""
+"""Vendored truststore, laya and the diffusers MiniMax-H3-Image models stay byte-identical to upstream except manifest-recorded patches."""
from __future__ import annotations
@@ -19,11 +19,12 @@ _MANIFESTS = {
for path in sorted(_VENDOR.glob("*_manifest.json"))
}
_MANIFEST = _MANIFESTS["truststore"]
-_PACKAGES = {"truststore", "laya"}
+_PACKAGES = {"truststore", "laya", "diffusers_minimax_h3_image"}
_SIDECARS = {
"LICENSE",
"LICENSE.laya",
+ "LICENSE.diffusers",
"README.md",
*(f"{name}_manifest.json" for name in _PACKAGES),
}
@@ -149,7 +150,7 @@ def test_vendored_laya_version_is_the_one_recorded():
assert f'__version__ = "{_MANIFESTS["laya"]["version"]}"' in init
[email protected]("relative", ["LICENSE", "LICENSE.laya", "README.md"])
[email protected]("relative", ["LICENSE", "LICENSE.laya", "LICENSE.diffusers", "README.md"])
def test_provenance_files_are_present(relative):
"""MIT and Apache-2.0 both require the licence to travel with the copy."""
assert (_VENDOR / relative).read_text(encoding = "utf-8").strip()
diff --git a/studio/backend/vendor/LICENSE.diffusers b/studio/backend/vendor/LICENSE.diffusers
new file mode 100644
index 00000000000..038e32f6445
--- /dev/null
+++ b/studio/backend/vendor/LICENSE.diffusers
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, Any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/studio/backend/vendor/README.md b/studio/backend/vendor/README.md
index 824484f4f25..85af26a3588 100644
--- a/studio/backend/vendor/README.md
+++ b/studio/backend/vendor/README.md
@@ -92,3 +92,43 @@ Replace `laya/` with the new wheel's `laya/` directory, copy its licence to `LIC
`version`, `wheel`, `wheel_sha256` and the per-file hashes in `laya_manifest.json`. Then check every
laya internal `laya_runtime.py` reaches into, and run `SYSTEMONE_TEST_LAYA=<snapshot> pytest
tests/test_systemone.py`, which compares the fast path against `laya.Agent.predict`.
+
+## diffusers MiniMax-H3-Image models (Apache-2.0)
+
+- Upstream: https://github.com/huggingface/diffusers/pull/15230, merged into diffusers `main`
+- Commit: `875373ac7f5856071e345a3e394c74efccc6b9e6` (the merge; recorded in `diffusers_minimax_h3_image_manifest.json`). Upstream's
+ two files are byte-identical at the PR head `529403c21a36d03551f09bfd4d432cb64ab6d450` they were first vendored from.
+- Files: `diffusers_minimax_h3_image/transformer_minimax_h3_image.py` (`MiniMaxH3ImageTransformer2DModel`) and
+ `diffusers_minimax_h3_image/autoencoder_kl_minimax_h3_image.py` (`AutoencoderKLMiniMaxH3Image`), from
+ `src/diffusers/models/` of that commit, with their Apache-2.0 headers.
+- Licence: `LICENSE.diffusers` beside this file, copied unmodified from the diffusers repository at that commit.
+
+`core/inference/diffusion_minimax_h3_image.py` uses them only when the installed diffusers does not ship the two
+classes (feature-detected by name), so MiniMax-H3-Image runs on the pinned diffusers today and moves to diffusers'
+own classes as soon as a release carries them. The pipeline itself (canvas rule, packed layout, sampler, decode) is
+ported into that module, not vendored: the PR ships it as Modular Diffusers blocks.
+
+### Local patch
+
+Relative imports (`from ...configuration_utils`, `from ..attention`, `from .vae`) are rewritten onto the installed
+`diffusers` package, and `PackedColwiseParallel` (only read by diffusers' tensor-parallel hooks, which diffusers
+before 0.43 does not have) falls back to a stub when it cannot be imported. Both files also gain `from __future__ import annotations` after the
+licence header: upstream annotates with PEP 604 unions (`int | None`), which Python 3.9 (the backend's floor) cannot
+evaluate at import. diffusers' `register_to_config` and `ConfigMixin` read parameter names and defaults through
+`inspect.signature`, never the annotations, so the string annotations change nothing at runtime. Nothing else
+differs; the upstream hashes are under `patches` in the manifest.
+
+### How it is imported
+
+Only through `diffusion_minimax_h3_image._load_vendored()`, which loads each file by path under a private module
+name (`unsloth_studio_vendor_minimax_h3_image.*`), never through `sys.path`. `utils/native_tls.py` appends this
+directory to `sys.path` for truststore, so `diffusers_minimax_h3_image/__init__.py` (Unsloth's, not upstream's, listed
+under `guard_files` in the manifest) raises `ImportError`: the directory is never importable as a top-level package,
+which would load the same classes a second time under another name. `_load_vendored()` never runs it.
+
+### Updating
+
+When diffusers releases the classes, the vendored copy stops being used without a change here; drop it once the
+pinned diffusers carries them. To move to a newer PR commit, copy the two files from it, re-apply the import and
+`__future__` patch, and update the hashes and `upstream_commit` in `diffusers_minimax_h3_image_manifest.json`;
+`tests/test_vendored_truststore.py` and `tests/test_diffusion_minimax_h3_image.py` check the manifest.
diff --git a/studio/backend/vendor/diffusers_minimax_h3_image/__init__.py b/studio/backend/vendor/diffusers_minimax_h3_image/__init__.py
new file mode 100644
index 00000000000..ee436ce53fd
--- /dev/null
+++ b/studio/backend/vendor/diffusers_minimax_h3_image/__init__.py
@@ -0,0 +1,7 @@
+# Unsloth guard, not upstream source. The vendor directory is appended to sys.path for truststore, which would make
+# this directory importable as `diffusers_minimax_h3_image`. Its modules must only be loaded by file path through
+# core.inference.diffusion_minimax_h3_image._load_vendored(), so a package import fails loudly instead.
+raise ImportError(
+ "diffusers_minimax_h3_image is vendored source loaded by "
+ "core.inference.diffusion_minimax_h3_image._load_vendored(), not an importable package"
+)
diff --git a/studio/backend/vendor/diffusers_minimax_h3_image/autoencoder_kl_minimax_h3_image.py b/studio/backend/vendor/diffusers_minimax_h3_image/autoencoder_kl_minimax_h3_image.py
new file mode 100644
index 00000000000..1888241027a
--- /dev/null
+++ b/studio/backend/vendor/diffusers_minimax_h3_image/autoencoder_kl_minimax_h3_image.py
@@ -0,0 +1,840 @@
+# Copyright 2026 The MiniMax and HuggingFace Teams. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations # Unsloth patch: PEP 604 annotations must not evaluate on Python 3.9.
+
+import math
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+from diffusers.utils import logging
+from diffusers.utils.accelerate_utils import apply_forward_hook
+from diffusers.models.attention import AttentionMixin, AttentionModuleMixin, FeedForward
+from diffusers.models.attention_dispatch import dispatch_attention_fn
+from diffusers.models.modeling_outputs import AutoencoderKLOutput
+from diffusers.models.modeling_utils import ModelMixin, get_parameter_dtype
+from diffusers.models.autoencoders.vae import AutoencoderMixin, DecoderOutput, DiagonalGaussianDistribution
+
+
+logger = logging.get_logger(__name__) # pylint: disable=invalid-name
+
+
+# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoCausalConv3d with MiniMaxH3Video->MiniMaxH3Image
+class MiniMaxH3ImageCausalConv3d(nn.Conv3d):
+ r"""
+ 3D convolution used throughout the MiniMax-H3 video encoder.
+
+ Spatial padding is symmetric and uses `spatial_padding_mode` (`"reflect"` in the released checkpoint); temporal
+ padding is causal, i.e. `kernel_size_t - 1` zero frames are prepended and nothing is appended.
+ """
+
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ kernel_size: int | tuple[int, int, int],
+ stride: int | tuple[int, int, int] = 1,
+ spatial_padding: int = 0,
+ temporal_padding: int = 0,
+ spatial_padding_mode: str = "reflect",
+ ) -> None:
+ super().__init__(in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=0)
+ self.spatial_padding = spatial_padding
+ self.temporal_padding = temporal_padding
+ self.spatial_padding_mode = spatial_padding_mode
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ if self.spatial_padding > 0:
+ padding = self.spatial_padding
+ hidden_states = F.pad(
+ hidden_states, (padding, padding, padding, padding, 0, 0), mode=self.spatial_padding_mode
+ )
+ if self.temporal_padding > 0:
+ hidden_states = F.pad(hidden_states, (0, 0, 0, 0, self.temporal_padding, 0), mode="constant")
+ return F.conv3d(hidden_states, self.weight, self.bias, stride=self.stride, padding=0, dilation=self.dilation)
+
+
+# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoGroupNorm with MiniMaxH3Video->MiniMaxH3Image
+class MiniMaxH3ImageGroupNorm(nn.GroupNorm):
+ r"""
+ Group normalization applied to each latent frame in isolation (`use_t_isolated_gn` in the original config): the
+ temporal axis is folded into the batch axis so statistics never mix across frames.
+ """
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ batch_size, num_channels, num_frames, height, width = hidden_states.shape
+ hidden_states = hidden_states.permute(0, 2, 1, 3, 4).contiguous()
+ hidden_states = hidden_states.view(batch_size * num_frames, num_channels, 1, height, width)
+ hidden_states = super().forward(hidden_states)
+ hidden_states = hidden_states.view(batch_size, num_frames, num_channels, height, width)
+ return hidden_states.permute(0, 2, 1, 3, 4).contiguous()
+
+
+# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoResnetBlock3d with MiniMaxH3Video->MiniMaxH3Image
+class MiniMaxH3ImageResnetBlock3d(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ norm_num_groups: int = 32,
+ norm_eps: float = 1e-6,
+ spatial_padding_mode: str = "reflect",
+ ) -> None:
+ super().__init__()
+ self.in_channels = in_channels
+ self.out_channels = out_channels
+
+ self.norm1 = MiniMaxH3ImageGroupNorm(norm_num_groups, in_channels, eps=norm_eps, affine=True)
+ self.conv1 = MiniMaxH3ImageCausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=3,
+ spatial_padding=1,
+ temporal_padding=2,
+ spatial_padding_mode=spatial_padding_mode,
+ )
+ self.norm2 = MiniMaxH3ImageGroupNorm(norm_num_groups, out_channels, eps=norm_eps, affine=True)
+ self.conv2 = MiniMaxH3ImageCausalConv3d(
+ out_channels,
+ out_channels,
+ kernel_size=3,
+ spatial_padding=1,
+ temporal_padding=2,
+ spatial_padding_mode=spatial_padding_mode,
+ )
+ self.conv_shortcut = None
+ if in_channels != out_channels:
+ self.conv_shortcut = MiniMaxH3ImageCausalConv3d(in_channels, out_channels, kernel_size=1)
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ residual = hidden_states
+ hidden_states = F.silu(self.norm1(hidden_states))
+ hidden_states = self.conv1(hidden_states)
+ hidden_states = F.silu(self.norm2(hidden_states))
+ hidden_states = self.conv2(hidden_states)
+ if self.conv_shortcut is not None:
+ residual = self.conv_shortcut(residual)
+ return residual + hidden_states
+
+
+# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoDownsample3d with MiniMaxH3Video->MiniMaxH3Image
+class MiniMaxH3ImageDownsample3d(nn.Module):
+ r"""
+ Strided 3x3x3 downsampling convolution. A spatial stride of 2 is preceded by an asymmetric bottom/right pad of 1
+ (the convolution itself carries no spatial padding), so the output is exactly `ceil(size / 2)`.
+ """
+
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ temporal_stride: int = 1,
+ spatial_stride: int = 2,
+ spatial_padding_mode: str = "reflect",
+ ) -> None:
+ super().__init__()
+ self.spatial_stride = spatial_stride
+ self.spatial_padding_mode = spatial_padding_mode
+ self.conv = MiniMaxH3ImageCausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=3,
+ stride=(temporal_stride, spatial_stride, spatial_stride),
+ spatial_padding=0,
+ temporal_padding=2,
+ spatial_padding_mode=spatial_padding_mode,
+ )
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ if self.spatial_stride == 2:
+ hidden_states = F.pad(hidden_states, (0, 1, 0, 1, 0, 0), mode=self.spatial_padding_mode)
+ return self.conv(hidden_states)
+
+
+# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoDownBlock3d with MiniMaxH3Video->MiniMaxH3Image
+class MiniMaxH3ImageDownBlock3d(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ num_layers: int,
+ temporal_downsample_factor: int,
+ spatial_downsample_factor: int,
+ norm_num_groups: int = 32,
+ norm_eps: float = 1e-6,
+ spatial_padding_mode: str = "reflect",
+ ) -> None:
+ super().__init__()
+ self.resnets = nn.ModuleList(
+ [
+ MiniMaxH3ImageResnetBlock3d(
+ in_channels=in_channels if i == 0 else out_channels,
+ out_channels=out_channels,
+ norm_num_groups=norm_num_groups,
+ norm_eps=norm_eps,
+ spatial_padding_mode=spatial_padding_mode,
+ )
+ for i in range(num_layers)
+ ]
+ )
+ self.downsamplers = None
+ if temporal_downsample_factor * spatial_downsample_factor > 1:
+ self.downsamplers = nn.ModuleList(
+ [
+ MiniMaxH3ImageDownsample3d(
+ out_channels,
+ out_channels,
+ temporal_stride=temporal_downsample_factor,
+ spatial_stride=spatial_downsample_factor,
+ spatial_padding_mode=spatial_padding_mode,
+ )
+ ]
+ )
+
+ self.gradient_checkpointing = False
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ for resnet in self.resnets:
+ if torch.is_grad_enabled() and self.gradient_checkpointing:
+ hidden_states = self._gradient_checkpointing_func(resnet, hidden_states)
+ else:
+ hidden_states = resnet(hidden_states)
+ if self.downsamplers is not None:
+ for downsampler in self.downsamplers:
+ hidden_states = downsampler(hidden_states)
+ return hidden_states
+
+
+# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoEncoder3d with MiniMaxH3Video->MiniMaxH3Image
+class MiniMaxH3ImageEncoder3d(nn.Module):
+ r"""
+ Causal 3D CNN encoder. `block_out_channels` gives the channel count of every level; the per-level
+ `spatial_downsample_factors` / `temporal_downsample_factors` multiply out to the total compression ratios.
+ """
+
+ def __init__(
+ self,
+ in_channels: int = 3,
+ out_channels: int = 48,
+ block_out_channels: tuple[int, ...] = (128, 256, 256, 512, 512, 1024),
+ layers_per_block: int = 2,
+ spatial_downsample_factors: tuple[int, ...] = (2, 2, 2, 2, 1, 1),
+ temporal_downsample_factors: tuple[int, ...] = (1, 2, 2, 1, 1, 1),
+ norm_num_groups: int = 32,
+ norm_eps: float = 1e-6,
+ spatial_padding_mode: str = "reflect",
+ ) -> None:
+ super().__init__()
+
+ self.conv_in = MiniMaxH3ImageCausalConv3d(
+ in_channels,
+ block_out_channels[0],
+ kernel_size=3,
+ spatial_padding=1,
+ temporal_padding=2,
+ spatial_padding_mode=spatial_padding_mode,
+ )
+
+ block_in_channels = (block_out_channels[0],) + tuple(block_out_channels[:-1])
+ self.down_blocks = nn.ModuleList(
+ [
+ MiniMaxH3ImageDownBlock3d(
+ in_channels=block_in_channels[i],
+ out_channels=block_out_channels[i],
+ num_layers=layers_per_block,
+ temporal_downsample_factor=temporal_downsample_factors[i],
+ spatial_downsample_factor=spatial_downsample_factors[i],
+ norm_num_groups=norm_num_groups,
+ norm_eps=norm_eps,
+ spatial_padding_mode=spatial_padding_mode,
+ )
+ for i in range(len(block_out_channels))
+ ]
+ )
+
+ self.norm_out = MiniMaxH3ImageGroupNorm(norm_num_groups, block_out_channels[-1], eps=norm_eps, affine=True)
+ self.conv_out = MiniMaxH3ImageCausalConv3d(
+ block_out_channels[-1],
+ out_channels,
+ kernel_size=3,
+ spatial_padding=1,
+ temporal_padding=2,
+ spatial_padding_mode=spatial_padding_mode,
+ )
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ hidden_states = self.conv_in(hidden_states)
+ for down_block in self.down_blocks:
+ hidden_states = down_block(hidden_states)
+ hidden_states = F.silu(self.norm_out(hidden_states))
+ return self.conv_out(hidden_states)
+
+
+# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoRotaryPosEmbed with MiniMaxH3Video->MiniMaxH3Image
+class MiniMaxH3ImageRotaryPosEmbed(nn.Module):
+ r"""
+ 3-axis rotary embedding for the ViT decoder. Coordinates are length-normalized to `[-1, 1)` per axis and scaled by
+ `2 * pi`, and the resulting `(t, h, w)` angles are concatenated and then duplicated, so the first `rope_dim_ratio *
+ attention_head_dim` channels of every head are rotated.
+ """
+
+ def __init__(self, dim: int, theta: float = 100.0, num_axes: int = 3) -> None:
+ super().__init__()
+ if dim % (2 * num_axes) != 0:
+ raise ValueError(f"`dim` {dim} must be divisible by `2 * num_axes` {2 * num_axes}.")
+ inv_freq = 1.0 / theta ** torch.arange(0, 1, 2 * num_axes / dim, dtype=torch.float32)
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+ def forward(self, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+ angles = 2.0 * math.pi * position_ids[:, :, :, None] * self.inv_freq[None, None, None, :]
+ angles = angles.flatten(2, 3).tile(2).unsqueeze(2)
+ return angles.cos(), angles.sin()
+
+
+# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoAttnProcessor with MiniMaxH3Video->MiniMaxH3Image
+class MiniMaxH3ImageAttnProcessor:
+ _attention_backend = None
+ _parallel_config = None
+
+ def __call__(
+ self,
+ attn: "MiniMaxH3ImageAttention",
+ hidden_states: torch.Tensor,
+ rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None,
+ ) -> torch.Tensor:
+ query = attn.to_q(hidden_states).unflatten(2, (attn.heads, -1))
+ key = attn.to_k(hidden_states).unflatten(2, (attn.heads, -1))
+ value = attn.to_v(hidden_states).unflatten(2, (attn.heads, -1))
+
+ # The reference normalizes Q/K in float32 regardless of the compute dtype.
+ query = attn.norm_q(query.float()).to(query.dtype)
+ key = attn.norm_k(key.float()).to(key.dtype)
+
+ if rotary_emb is not None:
+ cos, sin = rotary_emb
+ cos = cos.to(query.dtype)
+ sin = sin.to(query.dtype)
+ rotary_dim = cos.shape[-1]
+ query_rotary, query_pass = query[..., :rotary_dim], query[..., rotary_dim:]
+ key_rotary, key_pass = key[..., :rotary_dim], key[..., rotary_dim:]
+ query_first, query_second = query_rotary.chunk(2, dim=-1)
+ key_first, key_second = key_rotary.chunk(2, dim=-1)
+ query_rotated = torch.cat([-query_second, query_first], dim=-1)
+ key_rotated = torch.cat([-key_second, key_first], dim=-1)
+ query = torch.cat([query_rotary * cos + query_rotated * sin, query_pass], dim=-1)
+ key = torch.cat([key_rotary * cos + key_rotated * sin, key_pass], dim=-1)
+
+ hidden_states = dispatch_attention_fn(
+ query,
+ key,
+ value,
+ attn_mask=None,
+ backend=self._attention_backend,
+ parallel_config=self._parallel_config,
+ )
+ hidden_states = hidden_states.flatten(2, 3)
+ return attn.to_out[0](hidden_states)
+
+
+# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoAttention with MiniMaxH3Video->MiniMaxH3Image
+class MiniMaxH3ImageAttention(nn.Module, AttentionModuleMixin):
+ _default_processor_cls = MiniMaxH3ImageAttnProcessor
+ _available_processors = [MiniMaxH3ImageAttnProcessor]
+
+ def __init__(self, dim: int, heads: int, dim_head: int, eps: float = 1e-5, bias: bool = True) -> None:
+ super().__init__()
+ self.heads = heads
+ self.dim_head = dim_head
+ self.use_bias = bias
+ inner_dim = heads * dim_head
+
+ self.norm_q = nn.RMSNorm(dim_head, eps=eps, elementwise_affine=False)
+ self.norm_k = nn.RMSNorm(dim_head, eps=eps, elementwise_affine=False)
+ self.to_q = nn.Linear(dim, inner_dim, bias=bias)
+ self.to_k = nn.Linear(dim, inner_dim, bias=bias)
+ self.to_v = nn.Linear(dim, inner_dim, bias=bias)
+ self.to_out = nn.ModuleList([nn.Linear(inner_dim, dim, bias=bias), nn.Dropout(0.0)])
+
+ self.set_processor(MiniMaxH3ImageAttnProcessor())
+
+ def forward(
+ self, hidden_states: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None
+ ) -> torch.Tensor:
+ return self.processor(self, hidden_states, rotary_emb)
+
+
+# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoTransformerBlock with MiniMaxH3Video->MiniMaxH3Image
+class MiniMaxH3ImageTransformerBlock(nn.Module):
+ def __init__(
+ self,
+ dim: int,
+ heads: int,
+ dim_head: int,
+ ffn_mult: int = 4,
+ eps: float = 1e-5,
+ bias: bool = True,
+ ) -> None:
+ super().__init__()
+ self.norm1 = nn.RMSNorm(dim, eps=eps, elementwise_affine=True)
+ self.attn = MiniMaxH3ImageAttention(dim=dim, heads=heads, dim_head=dim_head, eps=eps, bias=bias)
+ self.scale1 = nn.Parameter(torch.zeros(dim))
+ self.norm2 = nn.RMSNorm(dim, eps=eps, elementwise_affine=True)
+ self.ff = FeedForward(dim, mult=ffn_mult, activation_fn="swiglu", bias=bias)
+ self.scale2 = nn.Parameter(torch.zeros(dim))
+
+ def forward(
+ self, hidden_states: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None
+ ) -> torch.Tensor:
+ # The reference normalizes in float32 regardless of the compute dtype. The residual stream is float32 too
+ # because the anchors are, so the normed activation follows the projections instead.
+ compute_dtype = get_parameter_dtype(self.attn.to_q)
+ norm_hidden_states = self.norm1(hidden_states.float()).to(compute_dtype)
+ hidden_states = hidden_states + self.attn(norm_hidden_states, rotary_emb) * self.scale1
+ norm_hidden_states = self.norm2(hidden_states.float()).to(compute_dtype)
+ hidden_states = hidden_states + self.ff(norm_hidden_states) * self.scale2
+ return hidden_states
+
+
+class MiniMaxH3ImageViTDecoder2d(nn.Module):
+ r"""
+ Non-causal 2D ViT decoder. Every latent pixel becomes one token; `num_register_tokens` learned register tokens plus
+ a single all-zero token are appended (all at position `0`), attended over with full self-attention, and dropped
+ again before the patch projection expands each token into a `patch_size x patch_size` pixel block. The rotary
+ embedding carries `(h, w)` coordinates only.
+ """
+
+ def __init__(
+ self,
+ in_channels: int = 24,
+ out_channels: int = 3,
+ patch_size: int = 16,
+ num_layers: int = 24,
+ num_attention_heads: int = 16,
+ attention_head_dim: int = 64,
+ num_register_tokens: int = 4,
+ ffn_mult: int = 4,
+ rope_theta: float = 100.0,
+ rope_dim_ratio: float = 0.5,
+ norm_eps: float = 1e-5,
+ ) -> None:
+ super().__init__()
+ dim = num_attention_heads * attention_head_dim
+ self.patch_size = patch_size
+ self.out_channels = out_channels
+ self.num_register_tokens = num_register_tokens
+
+ self.rope = MiniMaxH3ImageRotaryPosEmbed(
+ int(attention_head_dim * rope_dim_ratio), theta=rope_theta, num_axes=2
+ )
+ self.proj_in = nn.Linear(in_channels, dim)
+ self.register_tokens = nn.Parameter(torch.zeros(1, num_register_tokens, dim))
+ self.transformer_blocks = nn.ModuleList(
+ [
+ MiniMaxH3ImageTransformerBlock(
+ dim=dim,
+ heads=num_attention_heads,
+ dim_head=attention_head_dim,
+ ffn_mult=ffn_mult,
+ eps=norm_eps,
+ )
+ for _ in range(num_layers)
+ ]
+ )
+ self.norm_out = nn.LayerNorm(dim, elementwise_affine=True, eps=norm_eps)
+ self.proj_out = nn.Linear(dim, out_channels * patch_size * patch_size)
+
+ self.gradient_checkpointing = False
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ batch_size, num_channels, height, width = hidden_states.shape
+
+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch_size, height * width, num_channels)
+ hidden_states = self.proj_in(hidden_states)
+ num_patches = hidden_states.shape[1]
+
+ # `proj_in` is pinned to float32 and `register_tokens` is not, so align rather than rely on promotion.
+ register_tokens = self.register_tokens.expand(batch_size, -1, -1).to(hidden_states.dtype)
+ cls_token = torch.zeros_like(hidden_states[:, :1, :])
+ hidden_states = torch.cat([hidden_states, register_tokens, cls_token], dim=1)
+
+ grids = [
+ 2.0 * (torch.arange(0.5, size, dtype=torch.float32, device=hidden_states.device) / size) - 1.0
+ for size in (height, width)
+ ]
+ position_ids = torch.stack(torch.meshgrid(*grids, indexing="ij"), dim=-1).flatten(0, 1)
+ position_ids = position_ids.unsqueeze(0).expand(batch_size, -1, -1)
+ suffix_ids = position_ids.new_zeros((batch_size, self.num_register_tokens + 1, 2))
+ position_ids = torch.cat([position_ids, suffix_ids], dim=1)
+ rotary_emb = self.rope(position_ids)
+
+ for block in self.transformer_blocks:
+ if torch.is_grad_enabled() and self.gradient_checkpointing:
+ hidden_states = self._gradient_checkpointing_func(block, hidden_states, rotary_emb)
+ else:
+ hidden_states = block(hidden_states, rotary_emb)
+
+ # `norm_out` is pinned to float32 and `proj_out` is not.
+ hidden_states = self.norm_out(hidden_states).to(get_parameter_dtype(self.proj_out))
+ hidden_states = self.proj_out(hidden_states)
+ hidden_states = hidden_states[:, :num_patches, :]
+
+ patch_size = self.patch_size
+ hidden_states = hidden_states.view(batch_size, height, width, self.out_channels, patch_size, patch_size)
+ hidden_states = hidden_states.permute(0, 3, 1, 4, 2, 5).contiguous()
+ return hidden_states.reshape(batch_size, self.out_channels, height * patch_size, width * patch_size)
+
+
+class AutoencoderKLMiniMaxH3Image(ModelMixin, ConfigMixin, AttentionMixin, AutoencoderMixin):
+ r"""
+ A VAE model with a causal 3D CNN encoder and a non-causal 2D ViT decoder, used in
+ [MiniMax-H3-Image](https://huggingface.co/MiniMaxAI). The encoder is the MiniMax-H3 video encoder run over a single
+ frame; the decoder is an image-only ViT.
+
+ This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented
+ for all models (such as downloading or saving).
+
+ Latents are normalized with per-channel `latents_mean` / `latents_std` rather than a `scaling_factor`; a pipeline
+ encodes with `(latent - latents_mean) / latents_std` and decodes with `latent * latents_std + latents_mean`.
+
+ The pixel convention is ImageNet-normalized RGB over a `[0, 1]` base range, not the usual `[-1, 1]`: `encode`
+ expects `(pixel - imagenet_mean) / imagenet_std` and `decode` returns values in that same space, so a pipeline has
+ to apply `sample * imagenet_std + imagenet_mean` (mean `(0.485, 0.456, 0.406)`, std `(0.229, 0.224, 0.225)`) and
+ clamp to `[0, 1]` before postprocessing.
+
+ Unlike most autoencoders in the library, spatial tiling is **on by default**: MiniMax-H3-Image was released with
+ tiling enabled for both encoding and decoding, and the released images are the blended-tile ones, so disabling
+ tiling changes the output. Use `enable_tiling` to change the tile geometry, `disable_tiling` to turn it off.
+ """
+
+ _supports_gradient_checkpointing = True
+ _no_split_modules = ["MiniMaxH3ImageResnetBlock3d", "MiniMaxH3ImageTransformerBlock"]
+ _repeated_blocks = ["MiniMaxH3ImageTransformerBlock"]
+ _skip_layerwise_casting_patterns = ["norm"]
+ # The released checkpoint is float32; only the cast-sensitive modules are pinned so `torch_dtype` still reaches
+ # the decoder block stack.
+ _keep_in_fp32_modules = [
+ "encoder",
+ "quant_conv",
+ "norm1",
+ "norm2",
+ "norm_out",
+ "scale1",
+ "scale2",
+ ]
+
+ @register_to_config
+ def __init__(
+ self,
+ in_channels: int = 3,
+ out_channels: int = 3,
+ latent_channels: int = 24,
+ block_out_channels: tuple[int, ...] = (128, 256, 256, 512, 512, 1024),
+ layers_per_block: int = 2,
+ spatial_downsample_factors: tuple[int, ...] = (2, 2, 2, 2, 1, 1),
+ temporal_downsample_factors: tuple[int, ...] = (1, 2, 2, 1, 1, 1),
+ norm_num_groups: int = 32,
+ norm_eps: float = 1e-6,
+ spatial_padding_mode: str = "reflect",
+ decoder_num_layers: int = 24,
+ decoder_num_attention_heads: int = 16,
+ decoder_attention_head_dim: int = 64,
+ decoder_num_register_tokens: int = 4,
+ decoder_ffn_mult: int = 4,
+ decoder_rope_theta: float = 100.0,
+ decoder_rope_dim_ratio: float = 0.5,
+ decoder_norm_eps: float = 1e-5,
+ latents_mean: tuple[float, ...] = (0.0,) * 24,
+ latents_std: tuple[float, ...] = (1.0,) * 24,
+ ) -> None:
+ super().__init__()
+
+ self.spatial_compression_ratio = math.prod(spatial_downsample_factors)
+
+ self.encoder = MiniMaxH3ImageEncoder3d(
+ in_channels=in_channels,
+ out_channels=2 * latent_channels,
+ block_out_channels=block_out_channels,
+ layers_per_block=layers_per_block,
+ spatial_downsample_factors=spatial_downsample_factors,
+ temporal_downsample_factors=temporal_downsample_factors,
+ norm_num_groups=norm_num_groups,
+ norm_eps=norm_eps,
+ spatial_padding_mode=spatial_padding_mode,
+ )
+ self.quant_conv = nn.Conv3d(2 * latent_channels, 2 * latent_channels, kernel_size=1)
+ self.post_quant_conv = nn.Conv2d(latent_channels, latent_channels, kernel_size=1)
+ self.decoder = MiniMaxH3ImageViTDecoder2d(
+ in_channels=latent_channels,
+ out_channels=out_channels,
+ patch_size=self.spatial_compression_ratio,
+ num_layers=decoder_num_layers,
+ num_attention_heads=decoder_num_attention_heads,
+ attention_head_dim=decoder_attention_head_dim,
+ num_register_tokens=decoder_num_register_tokens,
+ ffn_mult=decoder_ffn_mult,
+ rope_theta=decoder_rope_theta,
+ rope_dim_ratio=decoder_rope_dim_ratio,
+ norm_eps=decoder_norm_eps,
+ )
+
+ # When decoding a batch of latents at a time, one can save memory by slicing across the batch dimension to
+ # perform decoding of a single latent at a time.
+ self.use_slicing = False
+
+ # Large images are encoded and decoded in overlapping tiles that are linearly blended back together.
+ # MiniMax-H3-Image ships with tiling enabled.
+ self.use_tiling = True
+
+ # The tile size in pixel space, and the minimum overlap between two neighbouring tiles. The actual overlaps are
+ # widened (in multiples of `spatial_compression_ratio`) so that the tiles cover the image exactly.
+ self.tile_sample_min_height = 256
+ self.tile_sample_min_width = 256
+ self.tile_sample_min_overlap_height = 64
+ self.tile_sample_min_overlap_width = 64
+
+ # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3.enable_tiling
+ def enable_tiling(
+ self,
+ tile_sample_min_height: int | None = None,
+ tile_sample_min_width: int | None = None,
+ tile_sample_min_overlap_height: int | None = None,
+ tile_sample_min_overlap_width: int | None = None,
+ ) -> None:
+ r"""
+ Enable tiled VAE encoding/decoding. When this option is enabled, the VAE splits the frames into tiles, encodes
+ or decodes each tile separately and linearly blends the overlaps back together. This lowers the memory
+ requirement and allows processing larger frames.
+
+ Args:
+ tile_sample_min_height (`int`, *optional*):
+ The tile height in pixel space. Frames taller than this are split along the height dimension.
+ tile_sample_min_width (`int`, *optional*):
+ The tile width in pixel space. Frames wider than this are split along the width dimension.
+ tile_sample_min_overlap_height (`int`, *optional*):
+ The minimum overlap, in pixels, between two consecutive vertical tiles.
+ tile_sample_min_overlap_width (`int`, *optional*):
+ The minimum overlap, in pixels, between two consecutive horizontal tiles.
+ """
+ self.use_tiling = True
+ self.tile_sample_min_height = tile_sample_min_height or self.tile_sample_min_height
+ self.tile_sample_min_width = tile_sample_min_width or self.tile_sample_min_width
+ self.tile_sample_min_overlap_height = tile_sample_min_overlap_height or self.tile_sample_min_overlap_height
+ self.tile_sample_min_overlap_width = tile_sample_min_overlap_width or self.tile_sample_min_overlap_width
+
+ # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3._split_tiles
+ def _split_tiles(self, length: int, tile_size: int, min_overlap: int) -> tuple[list[int], list[int], list[int]]:
+ r"""
+ Lay `tile_size`-wide tiles over `length` pixels. The number of tiles is the smallest one whose union can cover
+ `length` while keeping every overlap at least `min_overlap`; the slack is then distributed round-robin over the
+ overlaps in whole `spatial_compression_ratio` steps so that every tile boundary stays latent-aligned.
+ """
+ if tile_size >= length:
+ return [0], [length], []
+
+ num_tiles = math.ceil(length / tile_size)
+ while tile_size * num_tiles - min_overlap * (num_tiles - 1) - length < 0:
+ num_tiles += 1
+
+ overlaps = [min_overlap] * (num_tiles - 1)
+ remaining = tile_size * num_tiles - sum(overlaps) - length
+ for i in range(remaining // self.spatial_compression_ratio):
+ overlaps[i % (num_tiles - 1)] += self.spatial_compression_ratio
+
+ tile_start_indices = [0]
+ for i in range(num_tiles - 1):
+ tile_start_indices.append(tile_start_indices[-1] + tile_size - overlaps[i])
+ return tile_start_indices, [tile_size] * num_tiles, overlaps
+
+ # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3._blend
+ def _blend(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int, dim: int) -> torch.Tensor:
+ blend_extent = min(a.shape[dim], b.shape[dim], blend_extent)
+ positions = torch.arange(blend_extent, device=b.device, dtype=b.dtype)
+ shape = [1] * a.ndim
+ shape[dim] = blend_extent
+ weight_a = (1 - positions / blend_extent).view(shape)
+ weight_b = (positions / blend_extent).view(shape)
+
+ slice_a = [slice(None)] * a.ndim
+ slice_a[dim] = slice(-blend_extent, None)
+ slice_b = [slice(None)] * b.ndim
+ slice_b[dim] = slice(0, blend_extent)
+ blended = a[tuple(slice_a)] * weight_a + b[tuple(slice_b)] * weight_b
+
+ if blend_extent == b.shape[dim]:
+ return blended
+ slice_rest = [slice(None)] * b.ndim
+ slice_rest[dim] = slice(blend_extent, None)
+ return torch.cat([blended, b[tuple(slice_rest)]], dim=dim)
+
+ # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3._stitch_tiles
+ def _stitch_tiles(
+ self,
+ tiles: list[list[torch.Tensor]],
+ height_overlaps: list[int],
+ width_overlaps: list[int],
+ ) -> torch.Tensor:
+ result_rows = []
+ for i, row in enumerate(tiles):
+ result_row = []
+ for j, tile in enumerate(row):
+ if i > 0:
+ tile = self._blend(tiles[i - 1][j], tile, height_overlaps[i - 1], dim=-2)
+ if j > 0:
+ tile = self._blend(row[j - 1], tile, width_overlaps[j - 1], dim=-1)
+ if i < len(tiles) - 1:
+ tile = tile[..., : -height_overlaps[i], :]
+ if j < len(row) - 1:
+ tile = tile[..., :, : -width_overlaps[j]]
+ result_row.append(tile)
+ result_rows.append(torch.cat(result_row, dim=-1))
+ return torch.cat(result_rows, dim=-2)
+
+ def _encode(self, x: torch.Tensor) -> torch.Tensor:
+ # The encoder is a causal video encoder; an image is a single frame, which maps to a single latent frame.
+ if not self.use_tiling:
+ return self.quant_conv(self.encoder(x[:, :, None]))[:, :, -1]
+
+ height, width = x.shape[-2], x.shape[-1]
+ y_indices, y_lengths, y_overlaps = self._split_tiles(
+ height, self.tile_sample_min_height, self.tile_sample_min_overlap_height
+ )
+ x_indices, x_lengths, x_overlaps = self._split_tiles(
+ width, self.tile_sample_min_width, self.tile_sample_min_overlap_width
+ )
+
+ rows = []
+ for i_pos, i_len in zip(y_indices, y_lengths):
+ row = []
+ for j_pos, j_len in zip(x_indices, x_lengths):
+ tile = x[..., i_pos : i_pos + i_len, j_pos : j_pos + j_len]
+ row.append(self.quant_conv(self.encoder(tile[:, :, None]))[:, :, -1])
+ rows.append(row)
+
+ latent_y_overlaps = [overlap // self.spatial_compression_ratio for overlap in y_overlaps]
+ latent_x_overlaps = [overlap // self.spatial_compression_ratio for overlap in x_overlaps]
+ return self._stitch_tiles(rows, latent_y_overlaps, latent_x_overlaps)
+
+ def _decode(self, z: torch.Tensor) -> torch.Tensor:
+ if not self.use_tiling:
+ return self.decoder(self.post_quant_conv(z))
+
+ # Tiles are laid out in pixel space and then mapped back onto the latent grid.
+ height = z.shape[-2] * self.spatial_compression_ratio
+ width = z.shape[-1] * self.spatial_compression_ratio
+ y_indices, y_lengths, y_overlaps = self._split_tiles(
+ height, self.tile_sample_min_height, self.tile_sample_min_overlap_height
+ )
+ x_indices, x_lengths, x_overlaps = self._split_tiles(
+ width, self.tile_sample_min_width, self.tile_sample_min_overlap_width
+ )
+
+ ratio = self.spatial_compression_ratio
+ rows = []
+ for i_pos, i_len in zip(y_indices, y_lengths):
+ row = []
+ for j_pos, j_len in zip(x_indices, x_lengths):
+ tile = z[
+ ...,
+ i_pos // ratio : i_pos // ratio + i_len // ratio,
+ j_pos // ratio : j_pos // ratio + j_len // ratio,
+ ]
+ row.append(self.decoder(self.post_quant_conv(tile)))
+ rows.append(row)
+
+ return self._stitch_tiles(rows, y_overlaps, x_overlaps)
+
+ @apply_forward_hook
+ def encode(self, x: torch.Tensor, return_dict: bool = True) -> AutoencoderKLOutput | tuple[torch.Tensor]:
+ r"""
+ Encode a batch of images into latents.
+
+ Args:
+ x (`torch.Tensor`):
+ Input batch of images, shape `(batch_size, in_channels, height, width)`.
+ return_dict (`bool`, *optional*, defaults to `True`):
+ Whether to return a [`~models.autoencoders.autoencoder_kl.AutoencoderKLOutput`] instead of a plain
+ tuple.
+
+ Returns:
+ The latent distribution of the encoded images, over latents of shape `(batch_size, latent_channels, height
+ // spatial_compression_ratio, width // spatial_compression_ratio)`. Note that MiniMax-H3-Image normalizes
+ the sampled latents with `latents_mean` / `latents_std` afterwards.
+ """
+ # The encoder is pinned to float32 by `_keep_in_fp32_modules`; align lower-precision pixels with the weights.
+ x = x.to(get_parameter_dtype(self.encoder))
+ if self.use_slicing and x.shape[0] > 1:
+ moments = torch.cat([self._encode(x_slice) for x_slice in x.split(1)])
+ else:
+ moments = self._encode(x)
+ posterior = DiagonalGaussianDistribution(moments)
+ if not return_dict:
+ return (posterior,)
+ return AutoencoderKLOutput(latent_dist=posterior)
+
+ @apply_forward_hook
+ def decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | tuple[torch.Tensor]:
+ r"""
+ Decode a batch of latents.
+
+ Args:
+ z (`torch.Tensor`):
+ Input batch of latents, shape `(batch_size, latent_channels, height, width)`.
+ return_dict (`bool`, *optional*, defaults to `True`):
+ Whether to return a [`~models.autoencoders.vae.DecoderOutput`] instead of a plain tuple.
+
+ Returns:
+ [`~models.autoencoders.vae.DecoderOutput`] or `tuple`:
+ The decoded images, shape `(batch_size, out_channels, height * spatial_compression_ratio, width *
+ spatial_compression_ratio)`.
+ """
+ # The decoder is mixed precision, so align with the first module that consumes the latents.
+ z = z.to(get_parameter_dtype(self.post_quant_conv))
+ if self.use_slicing and z.shape[0] > 1:
+ decoded = torch.cat([self._decode(z_slice) for z_slice in z.split(1)])
+ else:
+ decoded = self._decode(z)
+ if not return_dict:
+ return (decoded,)
+ return DecoderOutput(sample=decoded)
+
+ def forward(
+ self,
+ sample: torch.Tensor,
+ sample_posterior: bool = False,
+ generator: torch.Generator | None = None,
+ return_dict: bool = True,
+ ) -> DecoderOutput | tuple[torch.Tensor]:
+ r"""
+ Encode then decode a batch of images.
+
+ Args:
+ sample (`torch.Tensor`):
+ Input batch of images, shape `(batch_size, in_channels, height, width)`.
+ sample_posterior (`bool`, *optional*, defaults to `False`):
+ Whether to sample the posterior instead of taking its mode.
+ generator (`torch.Generator`, *optional*):
+ Generator used when `sample_posterior=True`.
+ return_dict (`bool`, *optional*, defaults to `True`):
+ Whether to return a [`~models.autoencoders.vae.DecoderOutput`] instead of a plain tuple.
+
+ Returns:
+ [`~models.autoencoders.vae.DecoderOutput`] or `tuple`:
+ The round-tripped images, shape `(batch_size, out_channels, height, width)`.
+ """
+ posterior = self.encode(sample).latent_dist
+ z = posterior.sample(generator=generator) if sample_posterior else posterior.mode()
+ return self.decode(z, return_dict=return_dict)
diff --git a/studio/backend/vendor/diffusers_minimax_h3_image/transformer_minimax_h3_image.py b/studio/backend/vendor/diffusers_minimax_h3_image/transformer_minimax_h3_image.py
new file mode 100644
index 00000000000..0a267bdb69b
--- /dev/null
+++ b/studio/backend/vendor/diffusers_minimax_h3_image/transformer_minimax_h3_image.py
@@ -0,0 +1,641 @@
+# Copyright 2026 The MiniMax and HuggingFace Teams. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations # Unsloth patch: PEP 604 annotations must not evaluate on Python 3.9.
+
+import torch
+import torch.nn as nn
+
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+try: # Unsloth patch: diffusers < 0.43 has no tensor-parallel hooks; the plan is only read by them.
+ from diffusers.hooks.tensor_parallel import PackedColwiseParallel
+except ImportError: # pragma: no cover - depends on the installed diffusers
+ def PackedColwiseParallel(*args, **kwargs):
+ return None
+from diffusers.utils import logging
+from diffusers.models._modeling_parallel import ContextParallelInput, ContextParallelOutput
+from diffusers.models.attention import AttentionMixin, AttentionModuleMixin, FeedForward
+from diffusers.models.attention_dispatch import dispatch_attention_fn
+from diffusers.models.embeddings import TimestepEmbedding, Timesteps
+from diffusers.models.modeling_outputs import Transformer2DModelOutput
+from diffusers.models.modeling_utils import ModelMixin, get_parameter_dtype
+
+
+logger = logging.get_logger(__name__) # pylint: disable=invalid-name
+
+
+# The AdaLN projections of MiniMax-H3 keep one set of modulation parameters per (timestep, modality) pair: 0 = video,
+# 1 = text, 2 = audio. MiniMax-H3-Image keeps the three-way table, with image rows tagged as video and no audio rows.
+MINIMAX_H3_MODALITY_NUM = 3
+
+
+# Copied from diffusers.models.transformers.transformer_minimax_h3._apply_rotary_emb
+def _apply_rotary_emb(hidden_states: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
+ r"""
+ Rotate the leading `rotary_dim` channels of every head and pass the remaining channels through unchanged.
+ `hidden_states` is `(batch_size, seq_len, num_heads, head_dim)` and `cos`/`sin` are `(seq_len, rotary_dim)`.
+ """
+ rotary_dim = cos.shape[-1]
+ hidden_states_rotary = hidden_states[..., :rotary_dim]
+ hidden_states_pass = hidden_states[..., rotary_dim:]
+
+ cos = cos.to(hidden_states.dtype)[None, :, None, :]
+ sin = sin.to(hidden_states.dtype)[None, :, None, :]
+ x1, x2 = hidden_states_rotary.chunk(2, dim=-1)
+ hidden_states_rotated = torch.cat((-x2, x1), dim=-1)
+ hidden_states_rotary = hidden_states_rotary * cos + hidden_states_rotated * sin
+ return torch.cat((hidden_states_rotary, hidden_states_pass), dim=-1).contiguous()
+
+
+# Copied from diffusers.models.transformers.transformer_minimax_h3.MiniMaxH3RotaryPosEmbed with MiniMaxH3->MiniMaxH3Image
+class MiniMaxH3ImageRotaryPosEmbed(nn.Module):
+ r"""
+ 3-axis rotary embedding over the `(t, h, w)` coordinates of the packed sequence.
+
+ A single `inv_freq` buffer of `rope_freq_dim` frequencies is shared by the three axes. Each axis contributes
+ `rope_freq_dim` angles, the three blocks are concatenated to `3 * rope_freq_dim` and then concatenated with
+ themselves so that the `rotate_half` convention rotates `2 * 3 * rope_freq_dim` of the `head_dim` channels.
+ """
+
+ def __init__(self, rope_freq_dim: int = 16, rope_theta: float = 10000.0):
+ super().__init__()
+ self.rope_freq_dim = rope_freq_dim
+ inv_freq = 1.0 / (
+ rope_theta ** (torch.arange(0, 2 * rope_freq_dim, 2, dtype=torch.float32) / (2 * rope_freq_dim))
+ )
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+ def forward(self, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+ # position_ids: (seq_len, 3) -> cos/sin: (seq_len, 2 * 3 * rope_freq_dim)
+ position_ids = position_ids.to(torch.float32)
+ freqs = position_ids.unsqueeze(-1) * self.inv_freq.view(1, 1, -1) # (seq_len, 3, rope_freq_dim)
+ freqs_t, freqs_h, freqs_w = freqs.unbind(dim=1)
+ freqs = torch.cat((freqs_t, freqs_h, freqs_w), dim=-1)
+ freqs = torch.cat((freqs, freqs), dim=-1)
+ return freqs.cos(), freqs.sin()
+
+
+# Copied from diffusers.models.transformers.transformer_minimax_h3.MiniMaxH3AdaLayerNormModulation with MiniMaxH3->MiniMaxH3Image
+class MiniMaxH3ImageAdaLayerNormModulation(nn.Module):
+ r"""
+ Projects the shared timestep embedding into the six per-(timestep, modality) modulation parameters of one
+ transformer block.
+
+ `(num_timesteps, time_embed_dim)` -> six tensors of shape `(num_timesteps * MINIMAX_H3_MODALITY_NUM, hidden_size)`,
+ in the diffusers `shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp` order. The row layout of the
+ returned tensors is `[t0_mod0, t0_mod1, t0_mod2, t1_mod0, ...]`, which is what `timestep_indices *
+ MINIMAX_H3_MODALITY_NUM + token_tags` addresses.
+
+ A single projection is shared by `norm1` and `norm2` and by the three modalities, so it cannot be folded into
+ either norm the way [`~models.normalization.AdaLayerNormZero`] does. It is therefore a block-level module of its
+ own, named after the checkpoint's `adaln_proj`, with the modulation projection under the `linear` name diffusers
+ uses inside every AdaLN module.
+ """
+
+ def __init__(self, time_embed_dim: int, hidden_size: int):
+ super().__init__()
+ self.hidden_size = hidden_size
+ self.linear = nn.Linear(time_embed_dim, 6 * hidden_size * MINIMAX_H3_MODALITY_NUM, bias=True)
+
+ def forward(self, temb: torch.Tensor) -> tuple[torch.Tensor, ...]:
+ # The activation runs at `temb`'s own precision — float32, since `time_embedder` is a float32 module in this
+ # mixed-precision checkpoint — and only its result is cast down to the bfloat16 projection. Every block reads
+ # the same `temb`, so a rounding applied before the activation biases every block's modulation parameters
+ # identically at every sampling step, which accumulates coherently over the denoising trajectory.
+ temb = self.linear(nn.functional.silu(temb).to(get_parameter_dtype(self.linear)))
+ temb = temb.view(-1, 6 * self.hidden_size)
+ return temb.chunk(6, dim=-1)
+
+
+# Copied from diffusers.models.transformers.transformer_minimax_h3.MiniMaxH3AdaLayerNormOut with MiniMaxH3->MiniMaxH3Image
+class MiniMaxH3ImageAdaLayerNormOut(nn.Module):
+ r"""
+ Final norm of the packed sequence, shift/scale modulated per row.
+
+ Same module layout and checkpoint keys as [`~models.normalization.AdaLayerNormContinuous`] (`norm` plus a `linear`
+ projecting the conditioning embedding to `2 * hidden_size`), with two MiniMax-H3 specifics: the modulation table
+ holds one row per *timestep* and is addressed per row of the packed sequence rather than per batch item, and the
+ two halves of the projection are `shift` then `scale`, the order `LTX2Transformer3DModel` and
+ `WanTransformer3DModel` also use in their output layers.
+ """
+
+ def __init__(self, hidden_size: int, time_embed_dim: int, eps: float):
+ super().__init__()
+ self.norm = nn.RMSNorm(hidden_size, eps=eps)
+ self.linear = nn.Linear(time_embed_dim, 2 * hidden_size, bias=True)
+
+ def forward(self, hidden_states: torch.Tensor, temb: torch.Tensor, timestep_indices: torch.Tensor) -> torch.Tensor:
+ # As in `MiniMaxH3ImageAdaLayerNormModulation`: activate at `temb`'s precision, cast to the projection's dtype after.
+ shift, scale = self.linear(nn.functional.silu(temb).to(get_parameter_dtype(self.linear))).chunk(2, dim=-1)
+ # The modulation itself stays at the block stack's precision; `forward` casts to the output heads' dtype.
+ hidden_states = self.norm(hidden_states)
+ return hidden_states * (1.0 + scale.index_select(0, timestep_indices)) + shift.index_select(
+ 0, timestep_indices
+ )
+
+
+# Copied from diffusers.models.transformers.transformer_minimax_h3.MiniMaxH3AttnProcessor with MiniMaxH3->MiniMaxH3Image
+class MiniMaxH3ImageAttnProcessor:
+ r"""
+ Full self-attention over one packed sequence. There is no cross-attention anywhere in MiniMax-H3.
+ """
+
+ _attention_backend = None
+ _parallel_config = None
+
+ def __call__(
+ self,
+ attn: "MiniMaxH3ImageAttention",
+ hidden_states: torch.Tensor,
+ rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None,
+ attention_mask: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ if attn.fused_projections:
+ query, key, value = attn.to_qkv(hidden_states).chunk(3, dim=-1)
+ else:
+ query = attn.to_q(hidden_states)
+ key = attn.to_k(hidden_states)
+ value = attn.to_v(hidden_states)
+
+ # Reshape by a fixed `head_dim` and let `-1` absorb the head count. Under tensor parallelism each rank
+ # holds a column-sharded slice (`attn.heads // tp_degree` heads); this keeps the processor TP-agnostic.
+ query = query.unflatten(-1, (-1, attn.head_dim))
+ key = key.unflatten(-1, (-1, attn.head_dim))
+ value = value.unflatten(-1, (-1, attn.head_dim))
+
+ query = attn.norm_q(query)
+ key = attn.norm_k(key)
+
+ if rotary_emb is not None:
+ query = _apply_rotary_emb(query, *rotary_emb)
+ key = _apply_rotary_emb(key, *rotary_emb)
+
+ # MiniMax-H3 packs one request into a single attention document, so the model passes no mask and every
+ # attention backend stays available; `attention_mask` is here because it is the processor signature every
+ # other one in diffusers has, and a custom processor may need it.
+ hidden_states = dispatch_attention_fn(
+ query,
+ key,
+ value,
+ attn_mask=attention_mask,
+ dropout_p=0.0,
+ is_causal=False,
+ backend=self._attention_backend,
+ parallel_config=self._parallel_config,
+ )
+ hidden_states = hidden_states.flatten(2, 3).type_as(query)
+ hidden_states = attn.to_out[0](hidden_states)
+ hidden_states = attn.to_out[1](hidden_states)
+ return hidden_states
+
+
+# Copied from diffusers.models.transformers.transformer_minimax_h3.MiniMaxH3Attention with MiniMaxH3->MiniMaxH3Image
+class MiniMaxH3ImageAttention(nn.Module, AttentionModuleMixin):
+ _default_processor_cls = MiniMaxH3ImageAttnProcessor
+ _available_processors = [MiniMaxH3ImageAttnProcessor]
+
+ def __init__(
+ self,
+ hidden_size: int,
+ heads: int,
+ dim_head: int,
+ qk_norm_eps: float = 1e-5,
+ processor=None,
+ ):
+ super().__init__()
+ self.heads = heads
+ self.head_dim = dim_head
+ self.inner_dim = heads * dim_head
+ self.use_bias = False
+
+ self.to_q = nn.Linear(hidden_size, self.inner_dim, bias=False)
+ self.to_k = nn.Linear(hidden_size, self.inner_dim, bias=False)
+ self.to_v = nn.Linear(hidden_size, self.inner_dim, bias=False)
+ self.norm_q = nn.RMSNorm(dim_head, eps=qk_norm_eps)
+ self.norm_k = nn.RMSNorm(dim_head, eps=qk_norm_eps)
+ self.to_out = nn.ModuleList([nn.Linear(self.inner_dim, hidden_size, bias=False), nn.Dropout(0.0)])
+
+ if processor is None:
+ processor = self._default_processor_cls()
+ self.set_processor(processor)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None,
+ attention_mask: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ return self.processor(self, hidden_states, rotary_emb, attention_mask)
+
+
+# Copied from diffusers.models.transformers.transformer_minimax_h3.MiniMaxH3TokenRefinerBlock with MiniMaxH3->MiniMaxH3Image
+class MiniMaxH3ImageTokenRefinerBlock(nn.Module):
+ r"""
+ Plain pre-norm transformer block used to refine the projected text stream. No AdaLN and no rotary embedding.
+ """
+
+ def __init__(
+ self,
+ hidden_size: int,
+ num_attention_heads: int,
+ attention_head_dim: int,
+ ffn_dim: int,
+ norm_eps: float,
+ qk_norm_eps: float,
+ ):
+ super().__init__()
+ self.norm1 = nn.RMSNorm(hidden_size, eps=norm_eps)
+ self.attn = MiniMaxH3ImageAttention(
+ hidden_size=hidden_size,
+ heads=num_attention_heads,
+ dim_head=attention_head_dim,
+ qk_norm_eps=qk_norm_eps,
+ )
+ self.norm2 = nn.RMSNorm(hidden_size, eps=norm_eps)
+ self.ff = FeedForward(hidden_size, inner_dim=ffn_dim, activation_fn="swiglu", bias=False)
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ hidden_states = hidden_states + self.attn(self.norm1(hidden_states))
+ hidden_states = hidden_states + self.ff(self.norm2(hidden_states))
+ return hidden_states
+
+
+# Copied from diffusers.models.transformers.transformer_minimax_h3.MiniMaxH3TokenRefiner with MiniMaxH3->MiniMaxH3Image
+class MiniMaxH3ImageTokenRefiner(nn.Module):
+ def __init__(
+ self,
+ hidden_size: int,
+ num_attention_heads: int,
+ attention_head_dim: int,
+ ffn_dim: int,
+ num_layers: int,
+ norm_eps: float,
+ qk_norm_eps: float,
+ final_norm_eps: float,
+ ):
+ super().__init__()
+ self.refiner_blocks = nn.ModuleList(
+ [
+ MiniMaxH3ImageTokenRefinerBlock(
+ hidden_size=hidden_size,
+ num_attention_heads=num_attention_heads,
+ attention_head_dim=attention_head_dim,
+ ffn_dim=ffn_dim,
+ norm_eps=norm_eps,
+ qk_norm_eps=qk_norm_eps,
+ )
+ for _ in range(num_layers)
+ ]
+ )
+ self.final_norm = nn.RMSNorm(hidden_size, eps=final_norm_eps)
+ self.gradient_checkpointing = False
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ for block in self.refiner_blocks:
+ if torch.is_grad_enabled() and self.gradient_checkpointing:
+ hidden_states = self._gradient_checkpointing_func(block, hidden_states)
+ else:
+ hidden_states = block(hidden_states)
+ return self.final_norm(hidden_states)
+
+
+# Copied from diffusers.models.transformers.transformer_minimax_h3.MiniMaxH3TransformerBlock with MiniMaxH3->MiniMaxH3Image
+class MiniMaxH3ImageTransformerBlock(nn.Module):
+ r"""
+ MiniMax-H3 block: pre-norm self-attention and feed-forward, each modulated by AdaLN parameters selected per row of
+ the packed sequence from the `(timestep, modality)` table.
+ """
+
+ def __init__(
+ self,
+ hidden_size: int,
+ num_attention_heads: int,
+ attention_head_dim: int,
+ ffn_dim: int,
+ time_embed_dim: int,
+ norm_eps: float,
+ qk_norm_eps: float,
+ ):
+ super().__init__()
+ self.norm1 = nn.RMSNorm(hidden_size, eps=norm_eps)
+ self.attn = MiniMaxH3ImageAttention(
+ hidden_size=hidden_size,
+ heads=num_attention_heads,
+ dim_head=attention_head_dim,
+ qk_norm_eps=qk_norm_eps,
+ )
+ self.norm2 = nn.RMSNorm(hidden_size, eps=norm_eps)
+ self.ff = FeedForward(hidden_size, inner_dim=ffn_dim, activation_fn="swiglu", bias=False)
+ self.adaln_proj = MiniMaxH3ImageAdaLayerNormModulation(time_embed_dim=time_embed_dim, hidden_size=hidden_size)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ temb: torch.Tensor,
+ adaln_indices: torch.Tensor,
+ rotary_emb: tuple[torch.Tensor, torch.Tensor],
+ attention_mask: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaln_proj(temb)
+
+ residual = hidden_states
+ norm_hidden_states = self.norm1(hidden_states)
+ norm_hidden_states = norm_hidden_states * (
+ 1.0 + scale_msa.index_select(0, adaln_indices)
+ ) + shift_msa.index_select(0, adaln_indices)
+ attn_output = self.attn(norm_hidden_states, rotary_emb, attention_mask)
+ hidden_states = residual + gate_msa.index_select(0, adaln_indices) * attn_output
+
+ residual = hidden_states
+ norm_hidden_states = self.norm2(hidden_states)
+ norm_hidden_states = norm_hidden_states * (
+ 1.0 + scale_mlp.index_select(0, adaln_indices)
+ ) + shift_mlp.index_select(0, adaln_indices)
+ ff_output = self.ff(norm_hidden_states)
+ hidden_states = residual + gate_mlp.index_select(0, adaln_indices) * ff_output
+
+ return hidden_states
+
+
+class MiniMaxH3ImageTransformer2DModel(ModelMixin, ConfigMixin, AttentionMixin):
+ r"""
+ A Transformer model for image generation and editing, introduced in MiniMax-H3-Image.
+
+ It is the MiniMax-H3 transformer without its audio stream: a single stack of blocks runs over **one packed 1-D
+ sequence** that holds the text condition, the rows of the reference images and the rows of the generated image.
+ Attention is full self-attention over that sequence; there is no cross-attention. The checkpoint is distilled to a
+ fixed number of sampling steps with Parallel Decoding Distillation (PDD), which keeps one output head per step, so
+ `forward` takes the index of the step it is running.
+
+ The caller is responsible for building the packed layout: patchifying the latents, ordering the rows, and producing
+ the `(t, h, w)` position grid, the per-row modality tags and the per-row timestep indices. The sequence carries no
+ padding, so attention runs unmasked over one document and every attention backend stays available.
+
+ The batch axis is a pure replication axis: the structural arguments (`timestep`, `timestep_indices`, `token_tags`,
+ `position_ids` and the two index tensors) describe one packed layout that every batch item shares.
+
+ Args:
+ num_attention_heads (`int`, defaults to `56`):
+ The number of heads to use for multi-head attention.
+ attention_head_dim (`int`, defaults to `128`):
+ The number of channels in each attention head. Note that `num_attention_heads * attention_head_dim` is
+ *larger* than `hidden_size`.
+ hidden_size (`int`, defaults to `5376`):
+ The number of channels of the packed sequence (the residual stream).
+ num_layers (`int`, defaults to `50`):
+ The number of transformer blocks.
+ num_refiner_layers (`int`, defaults to `2`):
+ The number of token refiner blocks applied to the projected text stream.
+ ffn_dim (`int`, defaults to `14336`):
+ The inner dimension of the SwiGLU feed-forward layers.
+ in_channels (`int`, defaults to `24`):
+ The number of channels of the image latents.
+ patch_size (`tuple[int, int, int]`, defaults to `(1, 2, 2)`):
+ The `(t, h, w)` patch used to pack the latents into rows.
+ text_dim (`int`, defaults to `5120`):
+ The number of channels of the text conditioning produced by the text encoder.
+ freq_dim (`int`, defaults to `256`):
+ The dimension of the sinusoidal timestep embedding. Timesteps are consumed unscaled in `[0, 1]`.
+ time_embed_hidden_dim (`int`, defaults to `5376`):
+ The inner dimension of the timestep MLP.
+ time_embed_dim (`int`, defaults to `2688`):
+ The output dimension of the timestep MLP, i.e. the input of every AdaLN projection.
+ rope_freq_dim (`int`, defaults to `16`):
+ The number of rotary frequencies per axis. The `(t, h, w)` axes share one `inv_freq` buffer of this length
+ and `2 * 3 * rope_freq_dim` of the `attention_head_dim` channels are rotated.
+ rope_theta (`float`, defaults to `10000.0`):
+ The base of the rotary frequency schedule the `rope.inv_freq` buffer is computed from.
+ norm_eps (`float`, defaults to `1e-5`):
+ Epsilon of the pre-attention and pre-feed-forward norms.
+ qk_norm_eps (`float`, defaults to `1e-5`):
+ Epsilon of the per-head query/key norms.
+ final_norm_eps (`float`, defaults to `1e-5`):
+ Epsilon of the token refiner output norm and of `norm_out`.
+ num_pdd_heads (`int`, defaults to `6`):
+ The number of sampling steps the checkpoint was distilled to, each with its own output head.
+ """
+
+ _supports_gradient_checkpointing = True
+ _no_split_modules = [
+ "MiniMaxH3ImageTransformerBlock",
+ "MiniMaxH3ImageTokenRefinerBlock",
+ "MiniMaxH3ImageAdaLayerNormOut",
+ ]
+ _repeated_blocks = ["MiniMaxH3ImageTransformerBlock", "MiniMaxH3ImageTokenRefinerBlock"]
+ _skip_layerwise_casting_patterns = ["norm"]
+ # The checkpoint is mixed precision, as MiniMax-H3's: the input patch projection, the timestep MLP and the output
+ # heads are float32 while everything else is bfloat16. The `rope.inv_freq` buffer is computed and kept float32.
+ _keep_in_fp32_modules = ["proj_in", "time_embedder", "pdd_proj_out", "rope"]
+ # The tensor-parallel plan of `MiniMaxH3Transformer3DModel`: the block stacks are the same.
+ _tp_plan = {
+ "transformer_blocks.*.attn.to_q": "colwise",
+ "transformer_blocks.*.attn.to_k": "colwise",
+ "transformer_blocks.*.attn.to_v": "colwise",
+ "transformer_blocks.*.attn.to_out.0": "rowwise",
+ "transformer_blocks.*.ff.net.0.proj": PackedColwiseParallel([1, 1]),
+ "transformer_blocks.*.ff.net.2": "rowwise",
+ "transformer_blocks.*.adaln_proj.linear": "rowwise_split_input",
+ "token_refiner.refiner_blocks.*.attn.to_q": "colwise",
+ "token_refiner.refiner_blocks.*.attn.to_k": "colwise",
+ "token_refiner.refiner_blocks.*.attn.to_v": "colwise",
+ "token_refiner.refiner_blocks.*.attn.to_out.0": "rowwise",
+ "token_refiner.refiner_blocks.*.ff.net.0.proj": PackedColwiseParallel([1, 1]),
+ "token_refiner.refiner_blocks.*.ff.net.2": "rowwise",
+ }
+ # The context-parallel plan of `MiniMaxH3Transformer3DModel`, gathering the sequence back at the per-step output
+ # heads: the packed sequence is split once it is built, at the first block, together with everything indexed per
+ # row of it.
+ _cp_plan = {
+ "rope": {
+ 0: ContextParallelInput(split_dim=0, expected_dims=2, split_output=True),
+ 1: ContextParallelInput(split_dim=0, expected_dims=2, split_output=True),
+ },
+ "transformer_blocks.0": {
+ "hidden_states": ContextParallelInput(split_dim=1, expected_dims=3, split_output=False),
+ },
+ "transformer_blocks.*": {
+ "adaln_indices": ContextParallelInput(split_dim=0, expected_dims=1, split_output=False),
+ },
+ "norm_out": {
+ "timestep_indices": ContextParallelInput(split_dim=0, expected_dims=1, split_output=False),
+ },
+ "pdd_proj_out.*": ContextParallelOutput(gather_dim=1, expected_dims=3),
+ }
+
+ @register_to_config
+ def __init__(
+ self,
+ num_attention_heads: int = 56,
+ attention_head_dim: int = 128,
+ hidden_size: int = 5376,
+ num_layers: int = 50,
+ num_refiner_layers: int = 2,
+ ffn_dim: int = 14336,
+ in_channels: int = 24,
+ patch_size: tuple[int, int, int] = (1, 2, 2),
+ text_dim: int = 5120,
+ freq_dim: int = 256,
+ time_embed_hidden_dim: int = 5376,
+ time_embed_dim: int = 2688,
+ rope_freq_dim: int = 16,
+ rope_theta: float = 10000.0,
+ norm_eps: float = 1e-5,
+ qk_norm_eps: float = 1e-5,
+ final_norm_eps: float = 1e-5,
+ num_pdd_heads: int = 6,
+ ) -> None:
+ super().__init__()
+
+ patch_dim = in_channels * patch_size[0] * patch_size[1] * patch_size[2]
+
+ # 1. Input projections
+ self.proj_in = nn.Linear(patch_dim, hidden_size, bias=True)
+ self.context_embedder = nn.Linear(text_dim, hidden_size, bias=True)
+
+ # 2. Timestep embedding, shared by every AdaLN projection
+ self.time_proj = Timesteps(num_channels=freq_dim, flip_sin_to_cos=True, downscale_freq_shift=0)
+ self.time_embedder = TimestepEmbedding(
+ in_channels=freq_dim, time_embed_dim=time_embed_hidden_dim, out_dim=time_embed_dim
+ )
+
+ # 3. Rotary embedding over the packed (t, h, w) grid
+ self.rope = MiniMaxH3ImageRotaryPosEmbed(rope_freq_dim=rope_freq_dim, rope_theta=rope_theta)
+
+ # 4. Text stream refiner
+ self.token_refiner = MiniMaxH3ImageTokenRefiner(
+ hidden_size=hidden_size,
+ num_attention_heads=num_attention_heads,
+ attention_head_dim=attention_head_dim,
+ ffn_dim=ffn_dim,
+ num_layers=num_refiner_layers,
+ norm_eps=norm_eps,
+ qk_norm_eps=qk_norm_eps,
+ final_norm_eps=final_norm_eps,
+ )
+
+ # 5. The block stack
+ self.transformer_blocks = nn.ModuleList(
+ [
+ MiniMaxH3ImageTransformerBlock(
+ hidden_size=hidden_size,
+ num_attention_heads=num_attention_heads,
+ attention_head_dim=attention_head_dim,
+ ffn_dim=ffn_dim,
+ time_embed_dim=time_embed_dim,
+ norm_eps=norm_eps,
+ qk_norm_eps=qk_norm_eps,
+ )
+ for _ in range(num_layers)
+ ]
+ )
+
+ # 6. Shared output norm and one output head per sampling step: step `i` projects with head `i`.
+ self.norm_out = MiniMaxH3ImageAdaLayerNormOut(
+ hidden_size=hidden_size, time_embed_dim=time_embed_dim, eps=final_norm_eps
+ )
+ self.pdd_proj_out = nn.ModuleList([nn.Linear(hidden_size, patch_dim, bias=True) for _ in range(num_pdd_heads)])
+
+ self.gradient_checkpointing = False
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ encoder_hidden_states: torch.Tensor,
+ timestep: torch.Tensor,
+ timestep_indices: torch.Tensor,
+ token_tags: torch.Tensor,
+ position_ids: torch.Tensor,
+ image_indices: torch.Tensor,
+ text_indices: torch.Tensor,
+ pdd_step_index: int,
+ return_dict: bool = True,
+ ) -> Transformer2DModelOutput | tuple[torch.Tensor]:
+ r"""
+ Args:
+ hidden_states (`torch.Tensor` of shape `(batch_size, num_image_tokens, in_channels * prod(patch_size))`):
+ Patchified latent rows — the rows of the reference images and the rows of the generated image — ordered
+ as they appear in the packed sequence, i.e. matching `image_indices`.
+ encoder_hidden_states (`torch.Tensor` of shape `(batch_size, num_text_tokens, text_dim)`):
+ Text conditioning, ordered to match `text_indices`.
+ timestep (`torch.Tensor` of shape `(num_timesteps,)`):
+ The *distinct* timestep values present in the packed sequence, in `[0, 1]` and unscaled. One forward
+ serves rows at different noise levels (generated rows, reference rows).
+ timestep_indices (`torch.Tensor` of shape `(seq_len,)`):
+ For every row of the packed sequence, the index of its timestep in `timestep`.
+ token_tags (`torch.Tensor` of shape `(seq_len,)`):
+ For every row of the packed sequence, its modality: `0` image, `1` text.
+ position_ids (`torch.Tensor` of shape `(seq_len, 3)`):
+ The `(t, h, w)` rotary coordinates of every row of the packed sequence.
+ image_indices (`torch.Tensor` of shape `(num_image_tokens,)`):
+ Positions of the image rows in the packed sequence.
+ text_indices (`torch.Tensor` of shape `(num_text_tokens,)`):
+ Positions of the text rows in the packed sequence.
+ pdd_step_index (`int`):
+ The sampling step, which selects the output head.
+ return_dict (`bool`, defaults to `True`):
+ Whether to return a [`~models.modeling_outputs.Transformer2DModelOutput`] instead of a plain tuple.
+
+ Returns:
+ [`~models.modeling_outputs.Transformer2DModelOutput`] or `tuple`:
+ The velocity of every image row, of shape `(batch_size, num_image_tokens, in_channels *
+ prod(patch_size))`, in the row order of `image_indices`.
+ """
+ if position_ids.ndim != 2 or position_ids.shape[-1] != 3:
+ raise ValueError(f"`position_ids` must be a `(seq_len, 3)` tensor, got {list(position_ids.shape)}.")
+ sequence_length = position_ids.shape[0]
+ if token_tags.shape != (sequence_length,) or timestep_indices.shape != (sequence_length,):
+ raise ValueError(
+ "`token_tags` and `timestep_indices` must both be `(seq_len,)` tensors matching `position_ids`, got "
+ f"{list(token_tags.shape)} and {list(timestep_indices.shape)} for seq_len={sequence_length}."
+ )
+
+ rotary_emb = self.rope(position_ids)
+
+ # 1. Project each modality and scatter the rows into the packed sequence buffer. The checkpoint is
+ # mixed-precision (the patch projection is float32 while `context_embedder` and the block stack are bfloat16 —
+ # see `_keep_in_fp32_modules`), so every input is aligned with its projection's parameter dtype. The text
+ # stream sets the dtype of the packed sequence.
+ image_embeds = self.proj_in(hidden_states.to(get_parameter_dtype(self.proj_in)))
+ text_embeds = self.context_embedder(encoder_hidden_states.to(get_parameter_dtype(self.context_embedder)))
+ text_embeds = self.token_refiner(text_embeds)
+
+ hidden_states = text_embeds.new_zeros((text_embeds.shape[0], sequence_length, text_embeds.shape[-1]))
+ hidden_states = hidden_states.index_copy(1, text_indices, text_embeds)
+ hidden_states = hidden_states.index_copy(1, image_indices, image_embeds.to(text_embeds.dtype))
+
+ # 2. One timestep embedding per distinct noise level. `temb` stays at the time embedder's precision: each
+ # AdaLN module applies its own activation to it and casts to its projection's dtype afterwards.
+ temb = self.time_proj(timestep)
+ temb = self.time_embedder(temb.to(get_parameter_dtype(self.time_embedder)))
+
+ # 3. Row -> AdaLN table row.
+ adaln_indices = timestep_indices * MINIMAX_H3_MODALITY_NUM + token_tags
+
+ for block in self.transformer_blocks:
+ if torch.is_grad_enabled() and self.gradient_checkpointing:
+ hidden_states = self._gradient_checkpointing_func(
+ block, hidden_states, temb, adaln_indices, rotary_emb
+ )
+ else:
+ hidden_states = block(hidden_states, temb, adaln_indices, rotary_emb)
+
+ # 4. The head of this step runs over every row, then the image rows are selected. The heads are float32 while
+ # the block stack runs in the requested `torch_dtype`; align the activation with their parameter dtype.
+ proj_out = self.pdd_proj_out[pdd_step_index]
+ hidden_states = self.norm_out(hidden_states, temb, timestep_indices).to(get_parameter_dtype(proj_out))
+ output = proj_out(hidden_states).index_select(1, image_indices)
+
+ if not return_dict:
+ return (output,)
+ return Transformer2DModelOutput(sample=output)
diff --git a/studio/backend/vendor/diffusers_minimax_h3_image_manifest.json b/studio/backend/vendor/diffusers_minimax_h3_image_manifest.json
new file mode 100644
index 00000000000..b6117df9ec9
--- /dev/null
+++ b/studio/backend/vendor/diffusers_minimax_h3_image_manifest.json
@@ -0,0 +1,29 @@
+{
+ "files": {
+ "LICENSE.diffusers": "f9e2070c247517b1ddf65f7b11b393484a18da91a958fd18a97bd0f241c3125c",
+ "diffusers_minimax_h3_image/__init__.py": "4e11808f514c975a1be8417561c5915bee093eb7749b726cf9d9ded1259b5087",
+ "diffusers_minimax_h3_image/autoencoder_kl_minimax_h3_image.py": "657aa796864d65ddfb3d25cb7102f132b31290e927ec362cba899d4f4bdf5d37",
+ "diffusers_minimax_h3_image/transformer_minimax_h3_image.py": "9f50ea2714c7dbec63f02c29729579cce6c4c362f5f5d2260325607469630560"
+ },
+ "guard_files": {
+ "diffusers_minimax_h3_image/__init__.py": "Unsloth guard (not upstream): raises ImportError so the directory is never importable as a top-level package"
+ },
+ "license": "Apache-2.0",
+ "package": "diffusers_minimax_h3_image",
+ "patches": {
+ "diffusers_minimax_h3_image/autoencoder_kl_minimax_h3_image.py": {
+ "reason": "relative imports rewritten onto the installed diffusers; `from __future__ import annotations` added so the PEP 604 annotations are not evaluated on Python 3.9",
+ "upstream_path": "src/diffusers/models/autoencoders/autoencoder_kl_minimax_h3_image.py",
+ "upstream_sha256": "82da645ee07b0c0443aacd64f160c14c3fe928d61d64023e0102ecbe99557ce1"
+ },
+ "diffusers_minimax_h3_image/transformer_minimax_h3_image.py": {
+ "reason": "relative imports rewritten onto the installed diffusers; PackedColwiseParallel optional (diffusers < 0.43); `from __future__ import annotations` added so the PEP 604 annotations are not evaluated on Python 3.9",
+ "upstream_path": "src/diffusers/models/transformers/transformer_minimax_h3_image.py",
+ "upstream_sha256": "0928960f80447c5e4db43d61a361ccc4c785021207325aabe65416f8f6f94e62"
+ }
+ },
+ "pr_head_commit": "529403c21a36d03551f09bfd4d432cb64ab6d450",
+ "source": "https://github.com/huggingface/diffusers/pull/15230 (merged)",
+ "upstream_branch": "huggingface:main",
+ "upstream_commit": "875373ac7f5856071e345a3e394c74efccc6b9e6"
+}
diff --git a/studio/frontend/src/features/hub/lib/model-capabilities.ts b/studio/frontend/src/features/hub/lib/model-capabilities.ts
index 4f1d9255f4f..0c41e9fca38 100644
--- a/studio/frontend/src/features/hub/lib/model-capabilities.ts
+++ b/studio/frontend/src/features/hub/lib/model-capabilities.ts
@@ -45,9 +45,10 @@ const VIDEO_GEN_TAGS = new Set([
"image-to-video",
"image-text-to-video",
]);
+// MiniMax-H3-Image is the image partition of the H3 release (the Images page), not a video model.
// Cosmos3 counts only as its Nano and Edge generators, not the Reasoner / Policy / Sim / Transfer / FP8 repos.
const VIDEO_GEN_ID_RE =
- /(?:^|[-_.])wan2|hunyuan[-_]?video|\bltx(?:[-_]?video|v|[-_]?\d)|minimax[-_]?h\d|cogvideo|mochi|skyreels|[-_]t2v\b|[-_]i2v\b|(?:^|[-_.])cosmos3[-_.](?:nano|edge)(?![a-z0-9]|[-_.](?:reasoner|policy|sim|transfer|fp8)(?![a-z0-9]))/;
+ /(?:^|[-_.])wan2|hunyuan[-_]?video|\bltx(?:[-_]?video|v|[-_]?\d)|minimax[-_]?h\d(?![-_.]?image)|cogvideo|mochi|skyreels|[-_]t2v\b|[-_]i2v\b|(?:^|[-_.])cosmos3[-_.](?:nano|edge)(?![a-z0-9]|[-_.](?:reasoner|policy|sim|transfer|fp8)(?![a-z0-9]))/;
const NEITHER_GEN_TASKS = new Set([
"video-to-video",
diff --git a/studio/frontend/src/features/images/edit-conditioning.ts b/studio/frontend/src/features/images/edit-conditioning.ts
index b9cb1e63eeb..23cb7e7725a 100644
--- a/studio/frontend/src/features/images/edit-conditioning.ts
+++ b/studio/frontend/src/features/images/edit-conditioning.ts
@@ -83,6 +83,81 @@ export function presetsWithin(limits: SizeLimits) {
);
}
+export const MINIMAX_H3_IMAGE_FAMILY = "minimax-h3-image";
+
+/** MiniMax-H3-Image's own sizes (its model card): the 4 MP canvas per aspect, then the 16 MP table. It renders
+ * on at least 4 MP and resizes back, so a smaller request costs the 4 MP time. Mirrors the backend's
+ * H3_IMAGE_PRESETS. */
+export const MINIMAX_H3_IMAGE_PRESETS: ReadonlyArray<{
+ label: string;
+ width: number;
+ height: number;
+}> = [
+ { label: "1:1 4 MP", width: 2048, height: 2048 },
+ { label: "4:3 4 MP", width: 2368, height: 1760 },
+ { label: "3:4 4 MP", width: 1760, height: 2368 },
+ { label: "3:2 4 MP", width: 2496, height: 1664 },
+ { label: "2:3 4 MP", width: 1664, height: 2496 },
+ { label: "16:9 4 MP", width: 2720, height: 1536 },
+ { label: "9:16 4 MP", width: 1536, height: 2720 },
+ { label: "1:1 16 MP", width: 4096, height: 4096 },
+ { label: "4:3 16 MP", width: 4736, height: 3552 },
+ { label: "3:4 16 MP", width: 3552, height: 4736 },
+ { label: "3:2 16 MP", width: 5024, height: 3360 },
+ { label: "2:3 16 MP", width: 3360, height: 5024 },
+ { label: "16:9 16 MP", width: 5472, height: 3072 },
+ { label: "9:16 16 MP", width: 3072, height: 5472 },
+ { label: "7:3 16 MP", width: 6272, height: 2688 },
+];
+
+function isMinimaxH3Image(family: string | null | undefined): boolean {
+ return (family ?? "").trim().toLowerCase() === MINIMAX_H3_IMAGE_FAMILY;
+}
+
+export function familyPresetsWithin(
+ family: string | null | undefined,
+ limits: SizeLimits,
+) {
+ if (!isMinimaxH3Image(family)) return presetsWithin(limits);
+ return MINIMAX_H3_IMAGE_PRESETS.filter(
+ (p) =>
+ Math.max(p.width, p.height) <= limits.maxSide &&
+ p.width * p.height <= limits.maxPixels &&
+ p.width % limits.multiple === 0 &&
+ p.height % limits.multiple === 0,
+ );
+}
+
+export function familyPresetsCopy(family: string | null | undefined): {
+ label: string;
+ hint: string;
+} {
+ if (isMinimaxH3Image(family)) {
+ return {
+ label: "Model sizes",
+ hint: "MiniMax-H3-Image's native sizes. It always renders at least 4 MP and resizes back to the size you ask for; the 16 MP sizes take about four times the memory and time.",
+ };
+ }
+ return {
+ label: "2K presets",
+ hint: "The model's native 2K sizes. They take several times the memory and time of a 1 megapixel image.",
+ };
+}
+
+/** What the inputs are called: MiniMax-H3-Image prompts refer to them as <Picture N>. */
+export function inputImageWord(
+ family: string | null | undefined,
+): "Picture" | "Image" {
+ return isMinimaxH3Image(family) ? "Picture" : "Image";
+}
+
+export function inputImageReference(
+ family: string | null | undefined,
+ n: number,
+): string {
+ return isMinimaxH3Image(family) ? `<Picture ${n}>` : `Image ${n}`;
+}
+
export const TRANSPARENCY_PREFIX = "This is an RGBA image with transparency.";
export const TRANSPARENCY_SUFFIX =
"The image has alpha channel and the background is transparent.";
diff --git a/studio/frontend/src/features/images/image-generation-defaults.ts b/studio/frontend/src/features/images/image-generation-defaults.ts
index 0ad871d3bf0..538e6d044ee 100644
--- a/studio/frontend/src/features/images/image-generation-defaults.ts
+++ b/studio/frontend/src/features/images/image-generation-defaults.ts
@@ -56,6 +56,8 @@ const MODEL_DEFAULTS: Array<{
{ match: "qwen-image", steps: 20, guidance: 4 },
{ match: "z-image", steps: 25, guidance: 3 },
{ match: "ideogram", steps: 20, guidance: 7 },
+ // MiniMax-H3-Image: 6 distilled steps, one output head each, no CFG (the backend refuses anything else).
+ { match: "minimax-h3-image", steps: 6, guidance: 1 },
{ match: "moss-image", steps: 25, guidance: 2.5 },
{ match: "moss_image", steps: 25, guidance: 2.5 },
{ match: "mossimage", steps: 25, guidance: 2.5 },
@@ -69,6 +71,15 @@ const MODEL_DEFAULTS: Array<{
{ match: "sdxl", steps: 25, guidance: 7 },
];
+// Families whose step count is part of the checkpoint (one distilled output head per step): the backend refuses any
+// other count, so the slider is locked to it.
+const FIXED_STEPS: Readonly<Record<string, number>> = { "minimax-h3-image": 6 };
+
+/** The step count the loaded family requires, or null when the user may choose. */
+export function fixedStepsFor(family: string | null | undefined): number | null {
+ return FIXED_STEPS[(family ?? "").trim().toLowerCase()] ?? null;
+}
+
// The Steps slider's usual top, and the request's (DiffusionGenerateRequest.steps is 1-100).
export const STEPS_SLIDER_MAX = 50;
export const STEPS_REQUEST_MAX = 100;
diff --git a/studio/frontend/src/features/images/images-page.tsx b/studio/frontend/src/features/images/images-page.tsx
index 65e8e64a42b..7c412e2b6ce 100644
--- a/studio/frontend/src/features/images/images-page.tsx
+++ b/studio/frontend/src/features/images/images-page.tsx
@@ -240,6 +240,7 @@ import {
DEFAULT_GEN,
defaultsFor,
defaultsKeyFor,
+ fixedStepsFor,
loadedRecipeFor,
residentDefaultsKey,
residentRecipeFor,
@@ -263,8 +264,10 @@ import {
TRANSPARENCY_CHECKER,
additionalImageNumber,
conditionedRequestFields,
+ familyPresetsCopy,
+ familyPresetsWithin,
+ inputImageWord,
maxAdditionalImages,
- presetsWithin,
resolveEditSize,
restoreInputsNote,
seedReferenceResolution,
@@ -2410,6 +2413,10 @@ export function ImagesPage({
const negativeCapable = !external && status?.supports_negative_prompt !== false;
// A model that reads no guidance here runs at its default, which the request and recipe then carry.
const guidanceFixed = !external && status?.loaded === true && status.supports_guidance === false;
+ const fixedSteps = !external && status?.loaded === true ? fixedStepsFor(status.family) : null;
+ useEffect(() => {
+ if (fixedSteps !== null && steps !== fixedSteps) setSteps(fixedSteps);
+ }, [fixedSteps, steps]);
const effectiveGuidance = guidanceFixed
? (status?.generation_defaults?.guidance ?? DEFAULT_GEN.guidance)
: guidance;
@@ -5087,8 +5094,15 @@ export function ImagesPage({
resolveEditSize(editSizing, sourceDims, matchResolution, { width, height }, sizeLimits),
[editSizing, sourceDims, matchResolution, width, height, sizeLimits],
);
- const officialPresets = useMemo(() => presetsWithin(sizeLimits), [sizeLimits]);
+ const loadedFamily = status?.family ?? null;
+ const officialPresets = useMemo(
+ () => familyPresetsWithin(loadedFamily, sizeLimits),
+ [loadedFamily, sizeLimits],
+ );
+ const presetsCopy = familyPresetsCopy(loadedFamily);
const showOfficialPresets = sizeLimits.maxSide > MAX_OUTPUT_DEFAULT && officialPresets.length > 0;
+ // "Picture" where the family's prompts name the inputs <Picture N> (MiniMax-H3-Image), else "Image".
+ const imageWord = inputImageWord(loadedFamily);
const unifiedEditActive = workflow === "edit" && unifiedEdit;
const onLocalizedLayer = useCallback((dataUrl: string | null) => setLocalizedLayer(dataUrl), []);
const onLocalizedColors = useCallback((names: string[]) => setLocalizedColors(names), []);
@@ -5096,15 +5110,15 @@ export function ImagesPage({
const renderAdditionalImages = (numberOf: (i: number) => number, hint: string) => (
<>
{referenceImages.map((img, i) => (
- <Field key={i} label={`Image ${numberOf(i)}`} hint={hint}>
+ <Field key={i} label={`${imageWord} ${numberOf(i)}`} hint={hint}>
<div className="space-y-1.5">
<ImageDropzone
value={img}
onChange={(v) =>
setReferenceImages((prev) => prev.map((p, j) => (j === i ? (v ?? "") : p)))
}
- removeLabel={`Remove image ${numberOf(i)}`}
- cropSubject={`Image ${numberOf(i)}`}
+ removeLabel={`Remove ${imageWord.toLowerCase()} ${numberOf(i)}`}
+ cropSubject={`${imageWord} ${numberOf(i)}`}
/>
<Button
type="button"
@@ -5114,7 +5128,7 @@ export function ImagesPage({
onClick={() => setReferenceImages((prev) => prev.filter((_, j) => j !== i))}
>
<HugeiconsIcon icon={TrashIcon} className="size-3.5" />
- Remove image {numberOf(i)}
+ Remove {imageWord.toLowerCase()} {numberOf(i)}
</Button>
</div>
</Field>
@@ -5134,7 +5148,7 @@ export function ImagesPage({
disabled={!initImage}
onClick={() => setReferenceImages((prev) => [...prev, ""])}
>
- Add image
+ Add {imageWord.toLowerCase()}
</Button>
)}
</>
@@ -5224,10 +5238,7 @@ export function ImagesPage({
</div>
</Field>
{showOfficialPresets && (
- <Field
- label="2K presets"
- hint="The model's native 2K sizes. They take several times the memory and time of a 1 megapixel image."
- >
+ <Field label={presetsCopy.label} hint={presetsCopy.hint}>
<Select
value=""
onValueChange={(v) => {
@@ -5240,8 +5251,8 @@ export function ImagesPage({
setPortrait(m.portrait);
}}
>
- <SelectTrigger aria-label="2K presets">
- <SelectValue placeholder="Choose a 2K size" />
+ <SelectTrigger aria-label={presetsCopy.label}>
+ <SelectValue placeholder="Choose a size" />
</SelectTrigger>
<SelectContent>
{officialPresets.map((p) => (
@@ -6602,10 +6613,10 @@ export function ImagesPage({
{workflow === "reference" && (
<>
<Field
- label={unifiedEdit ? "Image 1" : "Reference image"}
+ label={unifiedEdit ? `${imageWord} 1` : "Reference image"}
hint="A reference the model draws on (subject, style, or composition) while generating a NEW image from your prompt at the size below. Unlike Transform, it is not a redraw of this image, so there is no strength."
>
- <ImageDropzone value={initImage} onChange={handleInitChange} cropSubject={unifiedEdit ? "Image 1" : "Reference image"} />
+ <ImageDropzone value={initImage} onChange={handleInitChange} cropSubject={unifiedEdit ? `${imageWord} 1` : "Reference image"} />
</Field>
{renderAdditionalImages(
(i) => i + 2,
@@ -6677,10 +6688,14 @@ export function ImagesPage({
</Field>
) : (
<Field
- label="Image 1 (source)"
- hint="The image to edit. Describe the change in the instruction below; add more images to combine them, and refer to them as Image 2, Image 3 and so on."
+ label={`${imageWord} 1 (source)`}
+ hint={
+ imageWord === "Picture"
+ ? "The image to edit. Describe the change in the instruction below; add up to 15 more pictures and refer to them as <Picture 1>, <Picture 2> and so on, saying what to keep and what to change."
+ : "The image to edit. Describe the change in the instruction below; add more images to combine them, and refer to them as Image 2, Image 3 and so on."
+ }
>
- <ImageDropzone value={initImage} onChange={handleInitChange} cropSubject="Image 1" />
+ <ImageDropzone value={initImage} onChange={handleInitChange} cropSubject={`${imageWord} 1`} />
</Field>
)}
{conditioning?.localized_edit_modes?.length ? (
@@ -7072,12 +7087,17 @@ export function ImagesPage({
<div className="pt-2">
<SliderField
label="Steps"
- hint="Number of denoising steps. Start with the selected model's default; more steps take longer and may not improve quality."
- value={steps}
+ hint={
+ fixedSteps !== null
+ ? `The loaded model is distilled to exactly ${fixedSteps} steps, one output head per step, so the count is fixed.`
+ : "Number of denoising steps. Start with the selected model's default; more steps take longer and may not improve quality."
+ }
+ value={fixedSteps ?? steps}
min={1}
max={stepsSliderMax(pendingModelDefaults?.steps ?? residentSteps, steps)}
step={1}
onChange={setSteps}
+ disabled={fixedSteps !== null}
/>
</div>
<SliderField
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/family-override.ts b/studio/frontend/src/features/model-picker/components/model-selector/family-override.ts
index 15f628f986b..c83c0c1dafd 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/family-override.ts
+++ b/studio/frontend/src/features/model-picker/components/model-selector/family-override.ts
@@ -23,6 +23,7 @@ const LABELS: Readonly<Record<string, string>> = {
"hidream-i1": "HiDream-I1",
"ideogram-4": "Ideogram 4",
"moss-image": "MOSS-Image",
+ "minimax-h3-image": "MiniMax-H3-Image",
chroma: "Chroma",
anima: "Anima",
sdxl: "SDXL",
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-capabilities.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-capabilities.ts
index 21a264fc5f1..da8bba68ed3 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/model-capabilities.ts
+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-capabilities.ts
@@ -67,11 +67,11 @@ const AUDIO_NAME_RE = new RegExp(
// boundary, not END, since stems run into versions ("flux1") but not "fluxion".
const FAMILY_END = "(?![a-z])";
const IMAGE_GEN_NAME_RE = new RegExp(
- `${SEP}(?:flux|sdxl|sd3|stable[-_]?diffusion|z[-_]?image|qwen[-_]?image|hidream|ideogram|lumina|hunyuanimage|krea|kolors|playground|pixart)${FAMILY_END}`,
+ `${SEP}(?:flux|sdxl|sd3|stable[-_]?diffusion|z[-_]?image|qwen[-_]?image|hidream|ideogram|lumina|hunyuanimage|krea|kolors|playground|pixart|minimax[-_]?h\\d[-_.]?image)${FAMILY_END}`,
"i",
);
const VIDEO_GEN_NAME_RE = new RegExp(
- `${SEP}(?:wan\\d|ltx|hunyuanvideo|minimax[-_]?h\\d|cogvideox?|mochi|animatediff|svd|zeroscope)${FAMILY_END}`,
+ `${SEP}(?:wan\\d|ltx|hunyuanvideo|minimax[-_]?h\\d(?![-_.]?image)|cogvideox?|mochi|animatediff|svd|zeroscope)${FAMILY_END}`,
"i",
);
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
index 899b4e011dd..0240ac0230b 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
@@ -289,6 +289,22 @@ export const IMAGE_CATALOG: CatalogGroup[] = [
gguf("unsloth/flux-3-image-klein-base-GGUF"),
],
},
+ {
+ // MiniMax-H3-Image: text to image and edits with 1-16 <Picture N> references. Size is the released bf16 total
+ // (the backend quantizes at load); tiers are measured offloaded runs. CUDA / ROCm only.
+ canonicalId: "MiniMaxAI/MiniMax-H3-Image",
+ displayName: "MiniMax-H3-Image",
+ description: "Text-to-image and image editing with up to 16 pictures",
+ scope: "image",
+ artifacts: [
+ bf16Pipeline("MiniMaxAI/MiniMax-H3-Image", 126, {
+ offloadFitTiers: [
+ { gpuGb: 22, systemRamGb: 48, requiresDenseQuant: true },
+ { gpuGb: 39, systemRamGb: 64, requiresDenseQuant: true },
+ ],
+ }),
+ ],
+ },
{
// MOSS-Image SFT: the bf16 row keeps the OpenMOSS id but fetches from unsloth/MOSS-Image-SFT-FP8 (upstream
// folders mirrored byte-identical). GGUFs are sd.cpp only, VAE and tokenizer from the same repo.
diff --git a/studio/frontend/tests/minimax-h3-image-ui.test.ts b/studio/frontend/tests/minimax-h3-image-ui.test.ts
new file mode 100644
index 00000000000..6032a507c33
--- /dev/null
+++ b/studio/frontend/tests/minimax-h3-image-ui.test.ts
@@ -0,0 +1,205 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ MINIMAX_H3_IMAGE_PRESETS,
+ familyPresetsCopy,
+ familyPresetsWithin,
+ inputImageReference,
+ inputImageWord,
+ maxAdditionalImages,
+ presetsWithin,
+} from "../src/features/images/edit-conditioning.ts";
+import {
+ defaultsFor,
+ defaultsKeyFor,
+ fixedStepsFor,
+} from "../src/features/images/image-generation-defaults.ts";
+import { sizeLimitsFrom } from "../src/features/images/image-size.ts";
+import { familyOverrideOptions } from "../src/features/model-picker/components/model-selector/family-override.ts";
+import {
+ classifyHost,
+ curatedArtifactIsOfferable,
+} from "../src/features/model-picker/components/model-selector/host-artifact-policy.ts";
+import { detectCapabilities } from "../src/features/model-picker/components/model-selector/model-capabilities.ts";
+import {
+ IMAGE_CATALOG,
+ VIDEO_CATALOG,
+ artifactForRepoId,
+ catalogToModelOptions,
+ curatedArtifactFit,
+} from "../src/features/model-picker/components/model-selector/model-catalog.ts";
+
+import { readSrc, registerBundlerResolver } from "./helpers/kit.ts";
+
+registerBundlerResolver();
+
+const { generationKinds, isVideoFamilyName } = await import(
+ "../src/features/hub/lib/model-capabilities.ts"
+);
+
+const REPO = "MiniMaxAI/MiniMax-H3-Image";
+
+// What the backend reports for a loaded MiniMax-H3-Image (diffusion_conditioning for the family).
+const H3_IMAGE_CONDITIONING = {
+ max_condition_images: 16,
+ alpha: false,
+ dimension_multiple: 32,
+ max_output_side: 6272,
+ max_output_pixels: 20 * 1024 * 1024,
+ reference_resolutions: [],
+ unified_edit: true,
+ multi_image_edit: false,
+ localized_edit_modes: [],
+};
+
+test("MiniMax-H3-Image is an image model, while MiniMax-H3 stays a video model", () => {
+ for (const id of [
+ REPO,
+ "unsloth/MiniMax-H3-Image-FP8",
+ "/models/minimax_h3_image",
+ ]) {
+ const caps = detectCapabilities({ id });
+ assert.equal(caps.imageGen, true, id);
+ assert.equal(caps.videoGen, false, id);
+ assert.equal(isVideoFamilyName(id), false, id);
+ assert.deepEqual(
+ generationKinds([], "text-to-image", id),
+ { image: true, video: false },
+ id,
+ );
+ }
+ for (const id of [
+ "MiniMaxAI/MiniMax-H3",
+ "unsloth/MiniMax-H3-GGUF",
+ "Comfy-Org/MiniMax-H3",
+ ]) {
+ assert.equal(detectCapabilities({ id }).videoGen, true, id);
+ assert.equal(isVideoFamilyName(id), true, id);
+ }
+});
+
+test("the family selector names MiniMax-H3-Image", () => {
+ const options = familyOverrideOptions(["minimax-h3", "minimax-h3-image"]);
+ assert.deepEqual(options.slice(1), [
+ ["minimax-h3", "MiniMax-H3"],
+ ["minimax-h3-image", "MiniMax-H3-Image"],
+ ]);
+});
+
+test("the image catalog lists MiniMax-H3-Image as a pipeline, not the video catalog", () => {
+ const hit = artifactForRepoId(REPO, IMAGE_CATALOG);
+ assert.ok(hit);
+ assert.equal(hit.group.scope, "image");
+ assert.equal(hit.artifact.loadKind, "pipeline");
+ assert.equal(artifactForRepoId(REPO, VIDEO_CATALOG), null);
+});
+
+test("it fits the measured L4 and A100 40GB tiers only where a dense quant runs", () => {
+ const dense = ["int8"];
+ const fit = (
+ gpuGb: number,
+ systemRamGb: number,
+ denseQuantSchemes: string[],
+ ) =>
+ curatedArtifactFit(REPO, IMAGE_CATALOG, {
+ gpuGb,
+ systemRamGb,
+ denseQuantSchemes,
+ })?.fits;
+ assert.equal(fit(22, 52, dense), true);
+ assert.equal(fit(39.5, 83, dense), true);
+ assert.equal(fit(22, 52, []), false);
+ assert.equal(fit(16, 64, dense), false);
+ assert.equal(fit(22, 32, dense), false);
+});
+
+test("Apple Silicon and CPU hosts keep the row but it never reads as fitting (the backend refuses the load)", () => {
+ // A Mac must not open the picker with a family missing, so the row stays and the load answers with the backend's
+ // "needs an NVIDIA or AMD GPU" refusal (diffusion_minimax_h3_image.h3_image_device_refusal).
+ const mac = classifyHost({
+ deviceType: "mac",
+ deviceBackend: "mlx",
+ budgetKnown: true,
+ });
+ const cpu = classifyHost({ deviceBackend: "cpu", budgetKnown: true });
+ for (const host of [mac, cpu]) {
+ assert.equal(curatedArtifactIsOfferable(REPO, host), true);
+ assert.equal(
+ catalogToModelOptions(IMAGE_CATALOG, host).some((o) => o.id === REPO),
+ true,
+ );
+ }
+ // Unified memory with no dense quant (MPS / CPU): no tier is met.
+ assert.equal(
+ curatedArtifactFit(REPO, IMAGE_CATALOG, {
+ gpuGb: 128,
+ systemRamGb: 128,
+ denseQuantSchemes: [],
+ })?.fits,
+ false,
+ );
+ for (const deviceBackend of ["cuda", "rocm"]) {
+ const host = classifyHost({
+ deviceBackend,
+ budgetKnown: true,
+ denseQuantSupported: true,
+ });
+ assert.equal(
+ catalogToModelOptions(IMAGE_CATALOG, host).some((o) => o.id === REPO),
+ true,
+ deviceBackend,
+ );
+ }
+});
+
+test("the page opens on 6 steps at guidance 1, and the step count is locked", () => {
+ assert.deepEqual(defaultsFor(REPO), { steps: 6, guidance: 1 });
+ assert.deepEqual(defaultsFor("/models/minimax_h3_image"), {
+ steps: 6,
+ guidance: 1,
+ });
+ assert.deepEqual(
+ defaultsFor(defaultsKeyFor("/models/opaque", "minimax-h3-image")),
+ { steps: 6, guidance: 1 },
+ );
+ assert.equal(fixedStepsFor("minimax-h3-image"), 6);
+ assert.equal(fixedStepsFor("flux.3-klein"), null);
+ assert.equal(fixedStepsFor(null), null);
+ const page = readSrc("features/images/images-page.tsx");
+ assert.match(page, /value=\{fixedSteps \?\? steps\}/);
+ assert.match(page, /disabled=\{fixedSteps !== null\}/);
+});
+
+test("its size presets are the 4 MP and 16 MP tables, all inside the backend's limits", () => {
+ const limits = sizeLimitsFrom(H3_IMAGE_CONDITIONING);
+ const presets = familyPresetsWithin("minimax-h3-image", limits);
+ assert.deepEqual(presets, MINIMAX_H3_IMAGE_PRESETS);
+ assert.equal(presets.length, 15);
+ for (const p of presets) {
+ assert.equal(p.width % 32, 0, p.label);
+ assert.equal(p.height % 32, 0, p.label);
+ assert.ok(p.width * p.height >= 2048 * 2048 * 0.98, p.label);
+ }
+ assert.equal(presets.filter((p) => p.label.endsWith("16 MP")).length, 8);
+ assert.match(familyPresetsCopy("minimax-h3-image").hint, /at least 4 MP/);
+ assert.deepEqual(
+ familyPresetsWithin("flux.3-klein", limits),
+ presetsWithin(limits),
+ );
+ assert.equal(familyPresetsCopy("flux.3-klein").label, "2K presets");
+});
+
+test("edits take up to 16 pictures, named the way the prompt refers to them", () => {
+ assert.equal(maxAdditionalImages(H3_IMAGE_CONDITIONING, null), 15);
+ assert.equal(inputImageWord("minimax-h3-image"), "Picture");
+ assert.equal(inputImageReference("minimax-h3-image", 2), "<Picture 2>");
+ assert.equal(inputImageWord("qwen-image-edit"), "Image");
+ const page = readSrc("features/images/images-page.tsx");
+ // The inputs reuse the page's crop editor (ImageDropzone's cropSubject) under the family's word.
+ assert.match(page, /cropSubject=\{`\$\{imageWord\} \$\{numberOf\(i\)\}`\}/);
+ assert.match(page, /label=\{`\$\{imageWord\} 1 \(source\)`\}/);
+});