Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: run MiniMax-H3 PDD checkpoints with their 32-head output bank, refuse the ones that cannot run (#14410)

Daniel Han6901dd3bf32c
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 47301289c84..adac540845e 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -437,6 +437,15 @@ def _video_comfy_key_map(fam: Any) -> Any:
return None
+def _h3_pdd_default_steps(fam: Any, pipe: Any, default_steps: int) -> int:
+ """ComfyUI's 8 steps for a loaded MiniMax-H3 PDD head bank whatever the file is called, else ``default_steps``."""
+ if getattr(fam, "name", None) != "minimax-h3" or pipe is None:
+ return default_steps
+ from .video_minimax_h3_pdd import H3_PDD_DEFAULT_STEPS, pipe_h3_pdd_heads
+
+ return H3_PDD_DEFAULT_STEPS if pipe_h3_pdd_heads(pipe) else default_steps
+
+
def _h3_single_file_task(
fam: Any, kind: str, gguf_filename: Optional[str], h3_task: Optional[str]
) -> Optional[str]:
@@ -11522,6 +11531,7 @@ class VideoBackend:
fallback = (fam.default_steps, fam.default_guidance),
family = fam.name,
)
+ default_steps = _h3_pdd_default_steps(fam, state.pipe, default_steps)
steps = int(steps or default_steps)
guidance = float(default_guidance if guidance is None else guidance)
# A guidance-free family (supports_cfg=False, MiniMax-H3) forwards no CFG control to either engine: the
@@ -11921,6 +11931,18 @@ class VideoBackend:
setter = getattr(getattr(pipe, component, None), "set_shift", None)
if callable(setter) and value is not None:
setter(float(value))
+ if fam.name == "minimax-h3":
+ from .video_minimax_h3_pdd import (
+ bind_h3_pdd_render,
+ h3_pdd_num_inference_steps,
+ )
+
+ # A PDD head bank reads each step's (sigma, sigma_next) off this render's video schedule. The
+ # H3 scheduler's grid holds the terminal 0, so `steps` model evaluations (ComfyUI's count, which
+ # the bank's 8-step partition is distilled for) is a grid of steps + 1.
+ if bind_h3_pdd_render(pipe, shift, audio_shift):
+ kwargs["num_inference_steps"] = h3_pdd_num_inference_steps(steps)
+ request_shape["pdd"] = True
# What the clip records; LTX-2.5's two-stage render reports its stage-2 steps in the progress below.
recorded_steps = steps
@@ -13257,6 +13279,7 @@ class VideoBackend:
fallback = (fam.default_steps, fam.default_guidance),
family = fam.name,
)
+ default_steps = _h3_pdd_default_steps(fam, getattr(state, "pipe", None), default_steps)
return {
"loaded": True,
"repo_id": state.repo_id,
diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.py
index e98b556cd3b..f6da110806d 100644
--- a/studio/backend/core/inference/video_families.py
+++ b/studio/backend/core/inference/video_families.py
@@ -390,6 +390,23 @@ _LTX_VIDEO_09_REFUSAL = (
"run. Load LTX-2 or LTX-2.3 instead (for example unsloth/LTX-2.3-GGUF)."
)
# Last path segment of each identifier only: an owner or parent folder named i2v does not make a checkpoint I2V.
+# MiniMax-H3 PDD files Studio cannot run. A "without_heads" bake keeps the single original output heads and needs the
+# external PDD head bank applied per step by a custom node; loaded as a plain denoiser it renders wrong output with no
+# error. The custom-node LoRA files hold a trunk LoRA plus that bank, not a denoiser. A "with_heads" bake carries its
+# 32-head bank in the file and runs natively (video_minimax_h3_pdd). An unnamed without-heads file is caught at load
+# from its header (pdd_acc_baked metadata with single heads).
+_H3_PDD_WITHOUT_HEADS_RE = r"(?<![a-z0-9])pdd(?![a-z])[^/\\|]*without[-_ ]?heads"
+H3_PDD_WITHOUT_HEADS_REFUSAL = (
+ "This MiniMax-H3 PDD checkpoint has no head bank: it keeps the single original output heads and needs the "
+ "external PDD head bank applied per step by a custom node, which Studio does not have, so it would render "
+ "wrong output. Load the matching '..._pdd_acc_8step_baked_with_heads_...' checkpoint instead, which carries "
+ "the 32 video and audio heads in the file."
+)
+_H3_PDD_BANK_RE = r"(?<![a-z0-9])pdd(?![a-z])[^/\\|]*(?:lora|_comfyui\.safetensors$)"
+_H3_PDD_BANK_REFUSAL = (
+ "This is a MiniMax-H3 PDD acceleration LoRA with an external head bank for a companion custom node, not a "
+ "denoiser Studio can load. Load a '..._pdd_acc_8step_baked_with_heads_...' checkpoint instead."
+)
_HV15_I2V_RE = r"(?<![a-z0-9])i2v(?![a-z0-9])(?=[^/\\|]*[/\\]*\s*(?:$|\|))"
_HV15_I2V_REFUSAL = (
"This is a HunyuanVideo-1.5 image-to-video checkpoint. It needs its own image-conditioned "
@@ -558,6 +575,10 @@ _FAMILIES: tuple[VideoFamily, ...] = (
default_audio_flow_shift = 3.0,
supports_keyframes = True,
supports_references = True,
+ unsupported_checkpoints = (
+ (_H3_PDD_WITHOUT_HEADS_RE, H3_PDD_WITHOUT_HEADS_REFUSAL),
+ (_H3_PDD_BANK_RE, _H3_PDD_BANK_REFUSAL),
+ ),
supports_cfg = False,
),
# LTX-2.5 (diffusers >= 0.40): the 22B LTX-2.3 DiT without the video FFN bias plus a keyframe position embedding,
@@ -1609,6 +1630,9 @@ def validate_video_reference_conditioning(
# Default (steps, guidance) per checkpoint variant, matched by substring (picked id then base repo), most specific
# first.
_VIDEO_GENERATION_DEFAULTS: tuple[tuple[str, int, float], ...] = (
+ # MiniMax-H3 PDD baked checkpoints: ComfyUI's recipe is 8 Euler steps on the simple schedule at CFG 1 (shifts 12 / 3
+ # are the family defaults). Its head bank was distilled for that partition.
+ ("pdd", 8, 1.0),
("distilled", 8, 1.0),
# Wan2.1 14B FusionX merge (CausVid / AccVideo distilled): its card asks for CFG 1 and 8-10 steps.
("fusionx", 8, 1.0),
@@ -1629,8 +1653,10 @@ _VIDEO_GENERATION_DEFAULTS: tuple[tuple[str, int, float], ...] = (
# The Wan and FusionX keys count only in the model / file name, so an owner or folder spelling one cannot outrank the family the
# checkpoint itself names.
_NAME_SEGMENT_DEFAULT_KEYS = frozenset(
- k for k, _s, _g in _VIDEO_GENERATION_DEFAULTS if k.startswith(("wan", "fusionx"))
+ k for k, _s, _g in _VIDEO_GENERATION_DEFAULTS if k.startswith(("wan", "fusionx", "pdd"))
)
+# A PDD recipe on another family's checkpoint (a Wan file under /models/pdd/) would silently run 8 steps at CFG 1.
+_FAMILY_SCOPED_DEFAULT_KEYS = {"pdd": "minimax-h3"}
def default_video_generation_params(
@@ -1646,7 +1672,7 @@ def default_video_generation_params(
if family == LTX25_FAMILY:
# The generic table's "ltx" key is the LTX-2 dev recipe, and 2.5's Diffusers pack names no variant at all.
return LTX25_DEV_DEFAULTS if ltx25_is_dev(*identifiers) else LTX25_DISTILLED_DEFAULTS
- variant = video_generation_variant(*identifiers)
+ variant = video_generation_variant(*identifiers, family = family)
for key, steps, guidance in _VIDEO_GENERATION_DEFAULTS:
if key == variant:
return steps, guidance
@@ -1673,13 +1699,19 @@ def ltx25_is_dev(*identifiers: Optional[str]) -> bool:
return False
-def video_generation_variant(*identifiers: Optional[str]) -> Optional[str]:
+def video_generation_variant(
+ *identifiers: Optional[str], family: Optional[str] = None
+) -> Optional[str]:
"""The ``_VIDEO_GENERATION_DEFAULTS`` key the first identifier naming one matches, or None; the precedence
- ``default_video_generation_params`` uses, so every per-variant decision agrees with the defaults."""
+ ``default_video_generation_params`` uses, so every per-variant decision agrees with the defaults. Family-scoped
+ keys (``_FAMILY_SCOPED_DEFAULT_KEYS``) match only for their own family."""
for identifier in identifiers:
whole = (identifier or "").lower()
name = whole.replace("\\", "/").rstrip("/").rsplit("/", 1)[-1]
for key, _steps, _guidance in _VIDEO_GENERATION_DEFAULTS:
+ scope = _FAMILY_SCOPED_DEFAULT_KEYS.get(key)
+ if scope is not None and family != scope:
+ continue
needle = name if key in _NAME_SEGMENT_DEFAULT_KEYS else whole
# Match the key as a name segment: reject a preceding ASCII letter so "swan-video" does not false-match
# "wan".
diff --git a/studio/backend/core/inference/video_minimax_h3_comfy.py b/studio/backend/core/inference/video_minimax_h3_comfy.py
index f111f0c657b..d30c132aef1 100644
--- a/studio/backend/core/inference/video_minimax_h3_comfy.py
+++ b/studio/backend/core/inference/video_minimax_h3_comfy.py
@@ -136,18 +136,43 @@ def h3_comfy_curve_metadata(path: str) -> Optional[dict]:
def comfy_layout(path: str) -> dict:
"""``diffusion_comfy_quant.original_layout`` hooks for MiniMax-H3: key map, float32 tensors and, for a pruned
file, the curve adaLN installed on the freshly built model."""
- from .video_minimax_h3_adaln import apply_h3_adaln_curve
+ from .video_minimax_h3_pdd import h3_pdd_heads_in_file
metadata = h3_comfy_curve_metadata(path)
+ heads = h3_pdd_heads_in_file(path)
return {
"key_map": h3_comfy_key_map,
"keep_dtype": functools.partial(h3_comfy_keep_dtype, pruned = metadata is not None),
- "prepare_model": (lambda model: apply_h3_adaln_curve(model, metadata))
- if metadata is not None
- else None,
+ "prepare_model": _h3_prepare_model(metadata, heads),
}
+def _h3_prepare_model(
+ metadata: Optional[dict],
+ heads: int,
+ logger: Any = None,
+) -> Any:
+ """The ``prepare_model`` hook for one H3 file: curve adaLN for a pruned file, the stacked PDD head bank for a
+ ``with_heads`` one, None when the stock model already matches."""
+ if metadata is None and heads <= 1:
+ return None
+
+ def prepare(model: Any) -> None:
+ if metadata is not None:
+ from .video_minimax_h3_adaln import apply_h3_adaln_curve
+ if logger is not None:
+ apply_h3_adaln_curve(model, metadata, logger = logger)
+ else:
+ apply_h3_adaln_curve(model, metadata)
+ if heads > 1:
+ from .video_minimax_h3_pdd import install_h3_pdd_heads
+ install_h3_pdd_heads(model, heads)
+ if logger is not None:
+ logger.info("video.h3_pdd: %d-head PDD output bank per stream installed", heads)
+
+ return prepare
+
+
def load_h3_comfy_transformer(
transformer_cls: Any,
path: str,
@@ -166,9 +191,11 @@ def load_h3_comfy_transformer(
) -> Any:
"""The H3 denoiser of a ComfyUI-quantized file, int8 / fp8 codes kept, curve adaLN installed."""
from .diffusion_comfy_quant import load_comfy_quant_transformer
- from .video_minimax_h3_adaln import apply_h3_adaln_curve
+ from .video_minimax_h3_pdd import h3_pdd_heads_in_file
metadata = h3_comfy_curve_metadata(path)
+ # Header only, before any tensor is read: a PDD trunk without its head bank is refused here.
+ heads = h3_pdd_heads_in_file(path)
return load_comfy_quant_transformer(
transformer_cls,
path,
@@ -187,10 +214,6 @@ def load_h3_comfy_transformer(
target = target,
logger = logger,
key_map = h3_comfy_key_map,
- prepare_model = (
- (lambda model: apply_h3_adaln_curve(model, metadata, logger = logger))
- if metadata is not None
- else None
- ),
+ prepare_model = _h3_prepare_model(metadata, heads, logger = logger),
keep_dtype = functools.partial(h3_comfy_keep_dtype, pruned = metadata is not None),
)
diff --git a/studio/backend/core/inference/video_minimax_h3_pdd.py b/studio/backend/core/inference/video_minimax_h3_pdd.py
new file mode 100644
index 00000000000..5a9ebf72bcd
--- /dev/null
+++ b/studio/backend/core/inference/video_minimax_h3_pdd.py
@@ -0,0 +1,369 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MiniMax-H3 PDD (parallel decoding distillation) checkpoints: the stacked output-head bank.
+
+A PDD ``with_heads`` file stores ``pdd_steps`` (32) video and audio output heads stacked along the rows of
+``final_layer.video_out`` / ``final_layer.audio_out`` (``[32 * 96, 5376]`` and ``[32 * 32, 5376]``): head 0 is a full
+head and every later head is an offset from it. One sampler step from ``sigma`` to ``sigma_next`` covers a contiguous
+run of fine intervals on the unshifted grid, and consumes head 0 plus the dt-weighted mean of the offsets it spans.
+This matches ComfyUI's PDD head selection (``FinalLayer.forward`` / ``_pdd_head``) term for term.
+
+Graph safety: the ``(sigma, sigma_next)`` pair is a device tensor on each bank (``pdd_state``), rewritten in place by
+a forward pre-hook on the denoiser before every call, and the head selection is pure tensor math on it. Nothing about
+the step is a Python constant, so a CUDA graph replays it with new sigmas and torch.compile never specializes on it.
+
+A ``without_heads`` bake keeps the plain single heads and needs an external head bank applied by a custom node, which
+Studio does not have, so it is refused rather than run as a plain trunk (that renders wrong output with no error).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from typing import Any, Optional
+
+# ComfyUI's recipe for the baked 8-step PDD checkpoints: Euler, simple schedule, 8 steps, CFG 1, shifts 12 / 3.
+H3_PDD_DEFAULT_STEPS = 8
+H3_PDD_GUIDANCE = 1.0
+
+VIDEO_HEAD_KEY = "final_layer.video_out.weight"
+AUDIO_HEAD_KEY = "final_layer.audio_out.weight"
+_BANKS_ATTR = "_unsloth_h3_pdd_banks"
+_SCHEDULER_ATTR = "_unsloth_h3_pdd_scheduler"
+_HOOK_ATTR = "_unsloth_h3_pdd_hook"
+HEADS_ATTR = "_unsloth_h3_pdd_heads"
+
+_PDD_TOKEN = re.compile(r"(?<![a-z0-9])pdd(?![a-z])")
+
+
+def _name(filename: Optional[str]) -> str:
+ return os.path.basename(str(filename or "")).lower()
+
+
+def h3_pdd_name(filename: Optional[str]) -> bool:
+ """The file name says PDD (``..._pdd_acc_8step_...``)."""
+ return bool(_PDD_TOKEN.search(_name(filename)))
+
+
+def _without_heads_message(name: str) -> str:
+ from .video_families import H3_PDD_WITHOUT_HEADS_REFUSAL
+ return f"'{name}': {H3_PDD_WITHOUT_HEADS_REFUSAL}"
+
+
+def _metadata_pdd_steps(metadata: dict) -> Optional[int]:
+ raw = metadata.get("config")
+ if not raw:
+ return None
+ try:
+ value = (json.loads(raw).get("transformer") or {}).get("pdd_steps")
+ except (TypeError, ValueError, AttributeError):
+ return None
+ return int(value) if value is not None else None
+
+
+def _in_dim(header: dict, key: str, default: int) -> int:
+ shape = (header.get(key) or {}).get("shape") or ()
+ return int(shape[1]) if len(shape) == 2 else default
+
+
+def h3_pdd_heads_from_header(
+ header: dict,
+ *,
+ video_dim: Optional[int] = None,
+ audio_dim: Optional[int] = None,
+ filename: Optional[str] = None,
+) -> int:
+ """Heads per stream of a ComfyUI-layout H3 file (1 for a plain denoiser). Raises ``ValueError`` for a PDD file
+ Studio cannot run correctly: a trunk without its bank, or a bank whose shapes disagree."""
+ name = os.path.basename(str(filename or "")) or "this checkpoint"
+ metadata = header.get("__metadata__") or {}
+ video = header.get(VIDEO_HEAD_KEY) or {}
+ audio = header.get(AUDIO_HEAD_KEY) or {}
+ if not video or not audio:
+ return 1
+ video_dim = int(video_dim or _in_dim(header, "video_patch_proj.weight", 96))
+ audio_dim = int(audio_dim or _in_dim(header, "audio_patch_proj.weight", 32))
+ rows_v = int((video.get("shape") or [0])[0])
+ rows_a = int((audio.get("shape") or [0])[0])
+ if rows_v % video_dim or rows_a % audio_dim or rows_v // video_dim != rows_a // audio_dim:
+ raise ValueError(
+ f"'{name}': output heads of {rows_v} video / {rows_a} audio rows are not one stacked bank of "
+ f"{video_dim} / {audio_dim}-row heads."
+ )
+ heads = rows_v // video_dim
+ declared = _metadata_pdd_steps(metadata)
+ if heads == 1:
+ baked = str(metadata.get("pdd_acc_baked", "")).strip().lower() == "true"
+ if baked or (declared or 0) > 1 or h3_pdd_name(filename):
+ raise ValueError(_without_heads_message(name))
+ elif declared is not None and declared > 1 and declared != heads:
+ raise ValueError(
+ f"'{name}' declares {declared} PDD heads but stores {heads}: the file is inconsistent."
+ )
+ return heads
+
+
+def h3_pdd_heads_in_file(
+ path: str,
+ *,
+ video_dim: Optional[int] = None,
+ audio_dim: Optional[int] = None,
+ filename: Optional[str] = None,
+) -> int:
+ from .diffusion_comfy_quant import _read_header
+ header, _ = _read_header(path)
+ return h3_pdd_heads_from_header(
+ header, video_dim = video_dim, audio_dim = audio_dim, filename = filename or path
+ )
+
+
+# ── the head-bank math (shared by every bank and by the tests' oracle comparisons) ──
+
+
+def pdd_head_weights(state: Any, heads: int) -> Any:
+ """Per-head blend weights ``(heads,)`` float64 for one step, from ``state = [sigma, sigma_next, select_shift,
+ stream_shift]`` (float64). Head 0 gets 0 here: it is always added whole.
+
+ ComfyUI: ``start, stop = round((1 - unshift(sigma)) * n), round((1 - unshift(sigma_next)) * n)`` with the unshift
+ in float32 (the sampler's sigma dtype), ``start = min(start, n - 1)``, ``stop = max(stop, start + 1)``, then
+ ``dt`` of the stream's shifted ``n``-interval grid over ``[start, stop)`` normalised to sum 1, applied to heads
+ ``max(start, 1) .. stop - 1``. Same here, as masks instead of slices so no index ever reaches the host."""
+ import torch
+
+ sig = state[0:2].to(torch.float32)
+ sel = state[2]
+ # time_shift_sigma(s, shift, 1.0): the Python-float shift and (1 - shift) become float32 operands.
+ base = sig / (sel.to(torch.float32) + sig * (1.0 - sel).to(torch.float32))
+ pos = torch.round((1.0 - base).to(torch.float64) * heads)
+ start = pos[0].clamp(min = 0, max = heads - 1)
+ stop = torch.maximum(pos[1], start + 1)
+ shift = state[3]
+ grid = torch.linspace(1.0, 0.0, heads + 1, dtype = torch.float64, device = state.device)
+ dt = (1.0 - shift * grid / (1.0 + (shift - 1.0) * grid)).diff()
+ k = torch.arange(heads, dtype = torch.float64, device = state.device)
+ inside = (k >= start) & (k < stop)
+ dt = torch.where(inside, dt, torch.zeros_like(dt))
+ w = dt / dt.sum()
+ return torch.where(k >= 1, w, torch.zeros_like(w))
+
+
+def _bank_class():
+ import torch
+ from torch import nn
+
+ class H3PDDHeadBank(nn.Module):
+ """``heads`` stacked output heads (``weight [heads * out, in]``, the checkpoint's own layout and key), blended
+ per step from ``pdd_state``. Exposes ``weight`` / ``in_features`` / ``out_features`` like the Linear it
+ replaces (the denoiser casts its activation to ``proj_out.weight.dtype``)."""
+
+ def __init__(
+ self,
+ in_features: int,
+ out_features: int,
+ heads: int,
+ *,
+ dtype: Any = None,
+ ):
+ super().__init__()
+ self.in_features = int(in_features)
+ self.out_features = int(out_features)
+ self.heads = int(heads)
+ dtype = dtype or torch.float32
+ self.weight = nn.Parameter(
+ torch.empty(self.heads * self.out_features, self.in_features, dtype = dtype)
+ )
+ self.bias = nn.Parameter(torch.empty(self.heads * self.out_features, dtype = dtype))
+ # [sigma, sigma_next, select (video) shift, this stream's shift], float64. A plain tensor attribute, not a
+ # buffer: a module-wide .to(bfloat16) must not round the sigmas, and offload must not move it.
+ self.pdd_state = torch.tensor([1.0, 0.0, 12.0, 12.0], dtype = torch.float64)
+
+ def extra_repr(self) -> str:
+ return f"in_features={self.in_features}, out_features={self.out_features}, heads={self.heads}"
+
+ def _own_state(self, device: Any = None) -> Any:
+ """``pdd_state`` as a normal (non-inference) tensor on ``device``. The first render moves it to the
+ GPU inside the pipeline's inference_mode; a tensor made there is an inference tensor, and the next
+ render's set_shifts (outside inference_mode) could not write it."""
+ state = self.pdd_state
+ if state.is_inference() or (device is not None and state.device != device):
+ with torch.inference_mode(False):
+ state = state.to(device or state.device, copy = True)
+ self.pdd_state = state
+ return state
+
+ def set_state(self, pair: Any) -> None:
+ self._own_state(pair.device)[0:2].copy_(pair)
+
+ def set_shifts(self, select_shift: float, stream_shift: float) -> None:
+ state = self._own_state()
+ state[2] = float(select_shift)
+ state[3] = float(stream_shift)
+
+ def forward(self, hidden_states: Any) -> Any:
+ state = self.pdd_state
+ if state.device != hidden_states.device:
+ state = state.to(hidden_states.device)
+ w = pdd_head_weights(state, self.heads).to(hidden_states.dtype)
+ weight = self.weight.to(hidden_states.dtype)
+ bias = self.bias.to(hidden_states.dtype)
+ rows = weight.view(self.heads, self.out_features, self.in_features)
+ brows = bias.view(self.heads, self.out_features)
+ return nn.functional.linear(
+ hidden_states,
+ rows[0] + torch.einsum("n,noi->oi", w, rows),
+ brows[0] + torch.einsum("n,no->o", w, brows),
+ )
+
+ return H3PDDHeadBank
+
+
+_BANK_CLASS = None
+
+
+def head_bank_class():
+ global _BANK_CLASS
+ if _BANK_CLASS is None:
+ _BANK_CLASS = _bank_class()
+ return _BANK_CLASS
+
+
+# ── the per-step sigma pair ──
+
+_SIGMA_ARGS = ("timestep", "timestep_indices", "video_indices")
+# Their MiniMaxH3Transformer3DModel.forward positions.
+_SIGMA_POSITIONS = (3, 4, 7)
+
+
+def h3_pdd_sigma_pair(
+ timestep: Any, timestep_indices: Any, video_indices: Any, sample_sigmas: Any
+) -> Any:
+ """``[sigma, sigma_next]`` float64 on the device, without a host sync. ``sigma`` is the generated video rows'
+ sigma (conditioning rows sit at a larger t, so it is the largest video-row sigma), clamped like ComfyUI's; the
+ next one is the schedule entry after the nearest, the last entry for the last step (sigma 0)."""
+ import torch
+
+ rows = timestep_indices.index_select(0, video_indices.reshape(-1).to(timestep_indices.device))
+ t = timestep.reshape(-1).index_select(0, rows).to(torch.float32).min()
+ sigma = (1.0 - t).clamp(min = 1e-6)
+ sigmas = sample_sigmas.reshape(-1).to(device = sigma.device, dtype = torch.float32)
+ i = (sigmas - sigma).abs().argmin()
+ nxt = sigmas.index_select(0, (i + 1).clamp(max = sigmas.shape[0] - 1).reshape(1))[0]
+ return torch.stack([sigma, nxt]).to(torch.float64)
+
+
+class _Schedule:
+ """Holds the scheduler whose ``sigmas`` drive the bank (bound per render)."""
+
+ __slots__ = ("scheduler",)
+
+ def __init__(self) -> None:
+ self.scheduler = None
+
+
+def _pre_hook(module: Any, args: tuple, kwargs: dict) -> None:
+ banks = module.__dict__.get(_BANKS_ATTR) or ()
+ if not banks:
+ return None
+ values = []
+ for name, position in zip(_SIGMA_ARGS, _SIGMA_POSITIONS):
+ value = kwargs.get(name)
+ if value is None and len(args) > position:
+ value = args[position]
+ values.append(value)
+ holder = module.__dict__.get(_SCHEDULER_ATTR)
+ scheduler = getattr(holder, "scheduler", None)
+ sigmas = getattr(scheduler, "sigmas", None)
+ if sigmas is None or any(v is None for v in values):
+ raise ValueError(
+ "MiniMax-H3 PDD heads need the sampler's sigma schedule: bind the render's scheduler with "
+ "bind_h3_pdd_schedule before calling the denoiser."
+ )
+ pair = h3_pdd_sigma_pair(*values, sigmas)
+ for bank in banks:
+ bank.set_state(pair)
+ return None
+
+
+def install_h3_pdd_heads(model: Any, heads: int) -> bool:
+ """Swap a freshly built (meta) H3 denoiser's ``proj_out`` / ``audio_proj_out`` for ``heads``-head banks and hook
+ the per-step sigma pair. No-op (False) for ``heads <= 1``."""
+ if int(heads) <= 1:
+ return False
+ import torch
+
+ bank_cls = head_bank_class()
+ banks = []
+ for attr in ("proj_out", "audio_proj_out"):
+ linear = getattr(model, attr)
+ bank = bank_cls(linear.in_features, linear.out_features, heads, dtype = torch.float32)
+ setattr(model, attr, bank)
+ banks.append(bank)
+ model.__dict__[_BANKS_ATTR] = tuple(banks)
+ model.__dict__[_SCHEDULER_ATTR] = _Schedule()
+ model.__dict__[HEADS_ATTR] = int(heads)
+ model.__dict__[_HOOK_ATTR] = model.register_forward_pre_hook(_pre_hook, with_kwargs = True)
+ return True
+
+
+def h3_pdd_heads(module: Any) -> int:
+ """Heads per stream of an installed bank, 0 otherwise."""
+ try:
+ return int((getattr(module, "__dict__", None) or {}).get(HEADS_ATTR) or 0)
+ except (TypeError, ValueError):
+ return 0
+
+
+def bind_h3_pdd_schedule(
+ module: Any, scheduler: Any, video_shift: Optional[float], audio_shift: Optional[float]
+) -> bool:
+ """Point one PDD denoiser at the scheduler whose sigmas it reads, and set the per-stream shifts (head selection
+ runs on the video shift; each stream's dt grid on its own). False for a plain denoiser."""
+ if not h3_pdd_heads(module):
+ return False
+ video_shift = 12.0 if video_shift is None else float(video_shift)
+ audio_shift = 3.0 if audio_shift is None else float(audio_shift)
+ module.__dict__[_SCHEDULER_ATTR].scheduler = scheduler
+ video_bank, audio_bank = module.__dict__[_BANKS_ATTR]
+ video_bank.set_shifts(video_shift, video_shift)
+ audio_bank.set_shifts(video_shift, audio_shift)
+ return True
+
+
+H3_DENOISER_COMPONENTS = ("transformer", "transformer_ref")
+
+
+def _components(pipe: Any) -> list:
+ out = []
+ for name in H3_DENOISER_COMPONENTS:
+ try:
+ module = getattr(pipe, name, None)
+ except Exception: # noqa: BLE001 - a modular pipeline may raise for an unloaded component
+ module = None
+ if module is not None:
+ out.append(module)
+ return out
+
+
+def pipe_h3_pdd_heads(pipe: Any) -> int:
+ return max((h3_pdd_heads(m) for m in _components(pipe)), default = 0)
+
+
+def bind_h3_pdd_render(
+ pipe: Any, video_shift: Optional[float], audio_shift: Optional[float]
+) -> int:
+ """Bind every PDD denoiser of an H3 pipeline to its video scheduler for this render; returns the head count
+ (0: no PDD denoiser, nothing changed)."""
+ scheduler = getattr(pipe, "scheduler", None)
+ heads = 0
+ for module in _components(pipe):
+ if bind_h3_pdd_schedule(module, scheduler, video_shift, audio_shift):
+ heads = max(heads, h3_pdd_heads(module))
+ return heads
+
+
+def h3_pdd_num_inference_steps(steps: int) -> int:
+ """The MiniMax-H3 scheduler's grid size for ``steps`` model evaluations: its grid holds the terminal 0
+ (``linspace(1, 0, n)``), so ComfyUI's 8-step simple schedule (``1, 7/8, ..., 1/8, 0`` before the shift) is 9."""
+ return int(steps) + 1
diff --git a/studio/backend/tests/test_video_minimax_h3_pdd.py b/studio/backend/tests/test_video_minimax_h3_pdd.py
new file mode 100644
index 00000000000..322e0b84af0
--- /dev/null
+++ b/studio/backend/tests/test_video_minimax_h3_pdd.py
@@ -0,0 +1,823 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MiniMax-H3 PDD checkpoints: detection, the without-heads refusal, and the stacked head bank against a direct port of
+ComfyUI's ``FinalLayer`` PDD math and a float64 oracle, per step (the last one included), through the real loader,
+under torch.compile, a CUDA graph replaying changing sigmas, and diffusers group offload.
+
+The model is a tiny random ``MiniMaxH3Transformer3DModel`` (no checkpoint, no network); the CUDA cases skip on a host
+without a GPU."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+torch = pytest.importorskip("torch")
+safetensors_torch = pytest.importorskip("safetensors.torch")
+
+from core.inference import video_minimax_h3_pdd as pdd # noqa: E402
+
+HEADS = 32
+VIDEO_DIM = 96
+AUDIO_DIM = 32
+SHIFT_V, SHIFT_A = 12.0, 3.0
+
+
+# ── a direct port of ComfyUI's MiniMax-H3 PDD head math (comfy/ldm/minimax_h3/model.py) ──
+
+
+def comfy_time_shift_sigma(sigma, from_shift, to_shift):
+ base = sigma / (from_shift + sigma * (1.0 - from_shift))
+ return to_shift * base / (1.0 + (to_shift - 1.0) * base)
+
+
+def comfy_span(sigma, sample_sigmas, n, shift_v):
+ i = int((sample_sigmas - sigma).abs().argmin())
+ sigma_next = sample_sigmas[min(i + 1, sample_sigmas.shape[0] - 1)]
+ start, stop = (
+ round(float(1.0 - comfy_time_shift_sigma(s, shift_v, 1.0)) * n) for s in (sigma, sigma_next)
+ )
+ start = min(start, n - 1)
+ stop = max(stop, start + 1)
+ return start, stop
+
+
+def comfy_pdd_head(weight, bias, h, n, start, stop, flow_shift):
+ grid = torch.linspace(1.0, 0.0, n + 1, dtype = torch.float64)
+ dt = (1.0 - flow_shift * grid / (1.0 + (flow_shift - 1.0) * grid)).diff()[start:stop]
+ w = (dt / dt.sum()).to(h)
+ rows = weight.to(h).reshape(n, -1, weight.shape[1])
+ brows = bias.to(h).reshape(n, -1)
+ first = max(start, 1)
+ return torch.nn.functional.linear(
+ h,
+ rows[0] + torch.einsum("n,noi->oi", w[first - start :], rows[first:stop]),
+ brows[0] + torch.einsum("n,no->o", w[first - start :], brows[first:stop]),
+ )
+
+
+def oracle_head(weight, bias, h, n, start, stop, flow_shift):
+ """The same blend entirely in float64 (weights, activations and the matmul)."""
+ grid = torch.linspace(1.0, 0.0, n + 1, dtype = torch.float64)
+ dt = (1.0 - flow_shift * grid / (1.0 + (flow_shift - 1.0) * grid)).diff()[start:stop]
+ w = dt / dt.sum()
+ rows = weight.double().reshape(n, -1, weight.shape[1])
+ brows = bias.double().reshape(n, -1)
+ W, B = rows[0].clone(), brows[0].clone()
+ for j, k in enumerate(range(start, stop)):
+ if k >= 1:
+ W += w[j] * rows[k]
+ B += w[j] * brows[k]
+ return h.double() @ W.T + B
+
+
+def comfy_simple_sigmas(steps, shift = SHIFT_V):
+ """ComfyUI's ``simple`` scheduler over ModelSamplingDiscreteFlow (1000 timesteps, multiplier 1000)."""
+ t = torch.arange(1, 1001, dtype = torch.float32) / 1000.0
+ table = shift * t / (1 + (shift - 1) * t)
+ ss = len(table) / steps
+ return torch.tensor([float(table[-(1 + int(x * ss))]) for x in range(steps)] + [0.0])
+
+
+def h3_sigmas(steps, shift = SHIFT_V):
+ """MiniMaxH3Scheduler.set_timesteps(steps + 1): the grid Studio drives a PDD bank with."""
+ base = torch.linspace(1.0, 0.0, pdd.h3_pdd_num_inference_steps(steps), dtype = torch.float32)
+ return torch.unique_consecutive(shift * base / (1 + (shift - 1) * base))
+
+
+def _bank(
+ in_features = 24,
+ out_features = VIDEO_DIM,
+ heads = HEADS,
+ seed = 0,
+):
+ torch.manual_seed(seed)
+ bank = pdd.head_bank_class()(in_features, out_features, heads)
+ with torch.no_grad():
+ bank.weight.normal_(0, 0.05)
+ bank.bias.normal_(0, 0.05)
+ return bank
+
+
+def _state(
+ sigma,
+ sigma_next,
+ select = SHIFT_V,
+ stream = SHIFT_V,
+ device = "cpu",
+):
+ return torch.tensor(
+ [float(sigma), float(sigma_next), select, stream], dtype = torch.float64, device = device
+ )
+
+
+# ── detection and refusal ──
+
+
+def _header(
+ video_rows,
+ audio_rows,
+ metadata = None,
+):
+ header = {
+ "video_patch_proj.weight": {"dtype": "F32", "shape": [5376, VIDEO_DIM]},
+ "audio_patch_proj.weight": {"dtype": "F32", "shape": [5376, AUDIO_DIM]},
+ pdd.VIDEO_HEAD_KEY: {"dtype": "F32", "shape": [video_rows, 5376]},
+ pdd.AUDIO_HEAD_KEY: {"dtype": "F32", "shape": [audio_rows, 5376]},
+ }
+ if metadata is not None:
+ header["__metadata__"] = metadata
+ return header
+
+
+WITH = "minimax_h3_ref2va_pruned_pdd_acc_8step_baked_with_heads_int8_convrot.safetensors"
+WITHOUT = "minimax_h3_ref2va_pdd_acc_8step_baked_without_heads_int8_convrot.safetensors"
+# The without-heads file's former name: nothing in it says "without heads".
+WITHOUT_OLD = "minimax_h3_ref2va_pdd_acc_8step_baked_int8_convrot.safetensors"
+
+
+def test_a_with_heads_bank_is_detected_from_its_shapes_and_metadata():
+ meta = {"config": json.dumps({"transformer": {"pdd_steps": 32}}), "pdd_acc_baked": "true"}
+ assert pdd.h3_pdd_heads_from_header(_header(3072, 1024, meta), filename = WITH) == 32
+ # shapes alone are enough
+ assert pdd.h3_pdd_heads_from_header(_header(3072, 1024), filename = "opaque.safetensors") == 32
+ assert (
+ pdd.h3_pdd_heads_from_header(
+ _header(VIDEO_DIM, AUDIO_DIM), filename = "minimax_h3_fl2va_int8_convrot.safetensors"
+ )
+ == 1
+ )
+
+
+ "header,name",
+ [
+ (_header(VIDEO_DIM, AUDIO_DIM, {"pdd_acc_baked": "true"}), WITHOUT_OLD),
+ (_header(VIDEO_DIM, AUDIO_DIM, {"pdd_acc_baked": "true"}), "opaque.safetensors"),
+ (_header(VIDEO_DIM, AUDIO_DIM), WITHOUT),
+ (
+ _header(
+ VIDEO_DIM, AUDIO_DIM, {"config": json.dumps({"transformer": {"pdd_steps": 32}})}
+ ),
+ "x.safetensors",
+ ),
+ ],
+)
+def test_a_pdd_trunk_without_its_head_bank_is_refused_at_load(header, name):
+ with pytest.raises(ValueError, match = "no head bank"):
+ pdd.h3_pdd_heads_from_header(header, filename = name)
+
+
+def test_an_inconsistent_bank_is_refused():
+ with pytest.raises(ValueError, match = "not one stacked bank"):
+ pdd.h3_pdd_heads_from_header(_header(3072, 512), filename = WITH)
+ with pytest.raises(ValueError, match = "declares 16"):
+ pdd.h3_pdd_heads_from_header(
+ _header(3072, 1024, {"config": json.dumps({"transformer": {"pdd_steps": 16}})}),
+ filename = WITH,
+ )
+
+
+ "name,match",
+ [
+ (WITHOUT, "no head bank"),
+ ("minimax_h3_fl2va_pdd_acc_8step_comfyui.safetensors", "acceleration LoRA"),
+ ("minimax_h3_ref2va_pdd_acc_8step_comfyui.safetensors", "acceleration LoRA"),
+ ],
+)
+def test_the_family_refuses_pdd_files_it_cannot_run_by_name(name, match):
+ from core.inference.video import VideoBackend
+ with pytest.raises(ValueError, match = match):
+ VideoBackend().validate_load_request(
+ "aptech0081/MiniMax-H3-PDD-ComfyUI",
+ gguf_filename = name,
+ model_kind = "single_file",
+ family_override = "minimax-h3",
+ )
+
+
+def test_the_with_heads_file_passes_the_name_refusals():
+ from core.inference.video import VideoBackend
+ try:
+ VideoBackend().validate_load_request(
+ "aptech0081/MiniMax-H3-PDD-ComfyUI",
+ gguf_filename = WITH,
+ model_kind = "single_file",
+ family_override = "minimax-h3",
+ )
+ except (
+ ValueError,
+ ImportError,
+ ) as exc: # later probes may refuse on this host; never a PDD refusal
+ assert "PDD" not in str(exc) and "single .safetensors checkpoint" not in str(exc)
+
+
+def test_pdd_files_default_to_comfyuis_8_step_recipe():
+ from core.inference.video_families import default_video_generation_params, detect_video_family
+
+ fam = detect_video_family("MiniMaxAI/MiniMax-H3")
+ assert default_video_generation_params(
+ None, WITH, "aptech0081/MiniMax-H3-PDD-ComfyUI", fallback = (30, 1.0), family = fam.name
+ ) == (8, 1.0)
+ assert default_video_generation_params(
+ None,
+ "minimax_h3_fl2va_int8_convrot.safetensors",
+ "MiniMaxAI/MiniMax-H3",
+ fallback = (30, 1.0),
+ family = fam.name,
+ ) == (30, 1.0)
+ assert (fam.default_flow_shift, fam.default_audio_flow_shift) == (SHIFT_V, SHIFT_A)
+
+
+def test_the_default_grid_is_comfyuis_simple_schedule():
+ for steps in (4, 8):
+ ours, comfy = h3_sigmas(steps), comfy_simple_sigmas(steps)
+ assert ours.shape == comfy.shape == (steps + 1,)
+ assert torch.allclose(ours, comfy, atol = 1e-6, rtol = 0)
+ sched = pytest.importorskip("diffusers").MiniMaxH3Scheduler(shift = SHIFT_V)
+ sched.set_timesteps(pdd.h3_pdd_num_inference_steps(8))
+ assert torch.equal(sched.sigmas.cpu(), h3_sigmas(8))
+ assert len(sched.timesteps) == 8 # 8 model evaluations, as in ComfyUI
+
+
+# ── head selection and the blend, per step ──
+
+
[email protected]("steps", [8, 4, 6, 20])
+def test_head_selection_matches_comfyui_at_every_step(steps):
+ sigmas = h3_sigmas(steps)
+ for i in range(steps):
+ sigma = sigmas[i]
+ start, stop = comfy_span(sigma, sigmas, HEADS, SHIFT_V)
+ state = _state(sigma, sigmas[i + 1])
+ w = pdd.pdd_head_weights(state, HEADS)
+ picked = [k for k in range(HEADS) if w[k] != 0]
+ assert picked == [k for k in range(max(start, 1), stop)], (steps, i)
+ if steps == 8:
+ assert (start, stop) == (4 * i, 4 * i + 4)
+ # the last step lands on sigma_next = 0 and spans to the end of the bank
+ last = comfy_span(sigmas[-2], sigmas, HEADS, SHIFT_V)
+ assert float(sigmas[-1]) == 0.0 and last[1] == HEADS
+
+
[email protected]("steps", [8, 4])
+def test_the_blend_matches_comfyui_and_the_float64_oracle_per_step(steps):
+ sigmas = h3_sigmas(steps)
+ for out_dim, stream_shift, seed in ((VIDEO_DIM, SHIFT_V, 1), (AUDIO_DIM, SHIFT_A, 2)):
+ bank = _bank(out_features = out_dim, seed = seed)
+ h = torch.randn(7, 24, generator = torch.Generator().manual_seed(seed))
+ for i in range(steps):
+ bank.pdd_state = _state(sigmas[i], sigmas[i + 1], stream = stream_shift)
+ start, stop = comfy_span(sigmas[i], sigmas, HEADS, SHIFT_V)
+ with torch.no_grad():
+ ours = bank(h)
+ comfy = comfy_pdd_head(bank.weight, bank.bias, h, HEADS, start, stop, stream_shift)
+ oracle = oracle_head(bank.weight, bank.bias, h, HEADS, start, stop, stream_shift)
+ assert (ours.double() - oracle).abs().max() < 1e-6
+ assert (comfy.double() - oracle).abs().max() < 1e-6
+ assert (ours - comfy).abs().max() < 1e-6
+
+
+def test_sigma_pair_comes_off_the_device_schedule_like_comfyui():
+ sigmas = h3_sigmas(8)
+ timestep = (1.0 - sigmas[:-1]).float()
+ for i in range(8):
+ # three conditioning video rows pinned near clean, then the generated rows at this step
+ uniq = torch.unique(torch.tensor([float(timestep[i]), 0.999, 1.0]))
+ rows_t = torch.tensor([0.999] * 3 + [float(timestep[i])] * 5 + [1.0] * 2)
+ indices = torch.searchsorted(uniq, rows_t)
+ pair = pdd.h3_pdd_sigma_pair(uniq, indices, torch.arange(8), sigmas)
+ assert pair.dtype == torch.float64
+ assert abs(float(pair[0]) - float(sigmas[i])) < 1e-6
+ assert float(pair[1]) == float(sigmas[i + 1])
+ assert float(pdd.h3_pdd_sigma_pair(uniq, indices, torch.arange(8), sigmas)[1]) == 0.0
+
+
+# ── the real denoiser: tiny random MiniMaxH3Transformer3DModel ──
+
+TINY = dict(
+ num_attention_heads = 2,
+ attention_head_dim = 16,
+ hidden_size = 32,
+ num_layers = 2,
+ num_refiner_layers = 1,
+ ffn_dim = 64,
+ in_channels = 24,
+ audio_in_channels = 32,
+ text_dim = 16,
+ freq_dim = 32,
+ time_embed_hidden_dim = 32,
+ time_embed_dim = 16,
+ rope_freq_dim = 2,
+)
+
+
+def _diffusers():
+ return pytest.importorskip("diffusers")
+
+
+def _tiny(
+ heads = HEADS,
+ seed = 0,
+ device = "cpu",
+):
+ diffusers = _diffusers()
+ torch.manual_seed(seed)
+ model = diffusers.MiniMaxH3Transformer3DModel(**TINY)
+ if heads > 1:
+ dense = {
+ "proj_out": model.proj_out,
+ "audio_proj_out": model.audio_proj_out,
+ }
+ assert pdd.install_h3_pdd_heads(model, heads)
+ with torch.no_grad():
+ for name, lin in dense.items():
+ bank = getattr(model, name)
+ bank.weight.copy_(torch.randn_like(bank.weight) * 0.05)
+ bank.bias.copy_(torch.randn_like(bank.bias) * 0.05)
+ bank.weight[: lin.out_features].copy_(lin.weight)
+ bank.bias[: lin.out_features].copy_(lin.bias)
+ return model.to(device).eval()
+
+
+class _Sched:
+ def __init__(self, sigmas):
+ self.sigmas = sigmas
+
+
+def _inputs(
+ device,
+ t_video,
+ n_text = 4,
+ n_cond = 2,
+ n_video = 6,
+ n_audio = 3,
+ seed = 0,
+):
+ g = torch.Generator().manual_seed(seed)
+ s = n_text + n_cond + n_video + n_audio
+ text = torch.arange(n_text)
+ video = torch.arange(n_text, n_text + n_cond + n_video)
+ audio = torch.arange(n_text + n_cond + n_video, s)
+ tags = torch.zeros(s, dtype = torch.long)
+ tags[text], tags[audio] = 1, 2
+ row_t = torch.full((s,), float(t_video))
+ row_t[video[:n_cond]] = max(float(t_video), 0.999)
+ row_t[audio] = 1.0 - pdd_shift_sigma(1.0 - float(t_video))
+ uniq, inv = torch.unique(row_t, sorted = True, return_inverse = True)
+ pos = torch.stack(
+ [torch.arange(s), torch.zeros(s, dtype = torch.long), torch.arange(s) % 3], 1
+ ).float()
+ return dict(
+ hidden_states = torch.randn(1, n_cond + n_video, 96, generator = g).to(device),
+ audio_hidden_states = torch.randn(1, n_audio, 32, generator = g).to(device),
+ encoder_hidden_states = torch.randn(1, n_text, 16, generator = g).to(device),
+ timestep = uniq.to(device),
+ timestep_indices = inv.to(device),
+ token_tags = tags.to(device),
+ position_ids = pos.to(device),
+ video_indices = video.to(device),
+ audio_indices = audio.to(device),
+ text_indices = text.to(device),
+ return_dict = False,
+ )
+
+
+def pdd_shift_sigma(sigma_v):
+ return float(comfy_time_shift_sigma(torch.tensor(sigma_v), SHIFT_V, SHIFT_A))
+
+
+def _run_and_capture(model, kw):
+ seen = {}
+
+ def grab(name):
+ def hook(_m, args, out):
+ seen[name] = (args[0].detach().clone(), out.detach().clone())
+
+ return hook
+
+ handles = [
+ model.proj_out.register_forward_hook(grab("video")),
+ model.audio_proj_out.register_forward_hook(grab("audio")),
+ ]
+ try:
+ with torch.no_grad():
+ out = model(**kw)
+ finally:
+ for handle in handles:
+ handle.remove()
+ return out, seen
+
+
+def _check_step(model, seen, sigmas, i):
+ start, stop = comfy_span(sigmas[i], sigmas, HEADS, SHIFT_V)
+ worst = 0.0
+ for name, bank, shift in (
+ ("video", model.proj_out, SHIFT_V),
+ ("audio", model.audio_proj_out, SHIFT_A),
+ ):
+ h, out = seen[name]
+ w, b = bank.weight.detach().cpu(), bank.bias.detach().cpu()
+ oracle = oracle_head(w, b, h.cpu(), HEADS, start, stop, shift)
+ comfy = comfy_pdd_head(w, b, h.cpu().float(), HEADS, start, stop, shift)
+ worst = max(worst, float((out.cpu().double() - oracle).abs().max()))
+ assert (out.cpu().double() - oracle).abs().max() < 1e-5, (name, i)
+ assert (out.cpu() - comfy).abs().max() < 1e-5, (name, i)
+ return worst
+
+
+def _bound(model, steps = 8):
+ sigmas = h3_sigmas(steps)
+ pdd.bind_h3_pdd_schedule(model, _Sched(sigmas), SHIFT_V, SHIFT_A)
+ return sigmas
+
+
+def test_the_denoiser_blends_the_heads_comfyui_picks_at_every_step():
+ model = _tiny()
+ sigmas = _bound(model)
+ for i in range(8):
+ kw = _inputs("cpu", 1.0 - float(sigmas[i]), seed = i)
+ _out, seen = _run_and_capture(model, kw)
+ state = model.proj_out.pdd_state
+ assert abs(float(state[0]) - float(sigmas[i])) < 1e-6 and float(state[1]) == float(
+ sigmas[i + 1]
+ )
+ _check_step(model, seen, sigmas, i)
+
+
+def test_an_unbound_pdd_denoiser_refuses_to_run():
+ model = _tiny()
+ with pytest.raises(ValueError, match = "sigma schedule"):
+ with torch.no_grad():
+ model(**_inputs("cpu", 0.5))
+
+
+def test_a_plain_denoiser_is_untouched():
+ model = _tiny(heads = 1)
+ assert isinstance(model.proj_out, torch.nn.Linear) and pdd.h3_pdd_heads(model) == 0
+ assert not pdd.bind_h3_pdd_schedule(model, _Sched(h3_sigmas(8)), SHIFT_V, SHIFT_A)
+
+
+class _Pipe:
+ def __init__(self, transformer, scheduler):
+ self.transformer = transformer
+ self.scheduler = scheduler
+
+
+def test_the_render_binding_finds_the_partition_and_sizes_the_grid():
+ model = _tiny()
+ pipe = _Pipe(model, _Sched(h3_sigmas(8)))
+ assert pdd.pipe_h3_pdd_heads(pipe) == HEADS
+ assert pdd.bind_h3_pdd_render(pipe, SHIFT_V, SHIFT_A) == HEADS
+ assert model.audio_proj_out.pdd_state[2:].tolist() == [SHIFT_V, SHIFT_A]
+ assert pdd.h3_pdd_num_inference_steps(8) == 9
+ assert pdd.bind_h3_pdd_render(_Pipe(_tiny(heads = 1), _Sched(None)), SHIFT_V, SHIFT_A) == 0
+
+
+# ── the real loader: a ComfyUI-layout file built from the tiny model ──
+
+
+def _comfy_key(name):
+ from core.inference.video_minimax_h3_comfy import _RENAMES
+
+ if name.startswith("transformer_blocks."):
+ name = "blocks." + name[len("transformer_blocks.") :]
+ elif name.startswith("token_refiner.refiner_blocks."):
+ name = "token_refiner.blocks." + name[len("token_refiner.refiner_blocks.") :]
+ for old, new in reversed(_RENAMES):
+ if new in name and (not new.startswith(("proj_", "audio_proj")) or name.startswith(new)):
+ name = name.replace(new, old, 1)
+ break
+ return name
+
+
+def _write_comfy_file(
+ tmp_path,
+ model,
+ name,
+ metadata = None,
+ int8_layer = True,
+):
+ sd = {k: v.detach().clone() for k, v in model.state_dict().items()}
+ out = {}
+ for key in list(sd):
+ if ".attn.to_q.weight" in key:
+ stem = key[: -len("to_q.weight")]
+ fused = torch.cat([sd.pop(stem + p + ".weight") for p in ("to_q", "to_k", "to_v")])
+ out[_comfy_key(stem).replace("transformer_blocks.", "blocks.") + "qkv_proj.weight"] = (
+ fused
+ )
+ for key, value in sd.items():
+ if key.endswith(".ff.net.0.proj.weight"):
+ value_half, gate = value.chunk(2)
+ out[_comfy_key(key.replace(".ff.net.0.proj.weight", ".mlp.fc1.weight"))] = torch.cat(
+ [gate, value_half]
+ )
+ continue
+ out[_comfy_key(key)] = value
+ if int8_layer:
+ key = "blocks.0.attn.out_proj.weight"
+ w = out[key].float()
+ scale = w.abs().amax().clamp_min(1e-8) / 127.0
+ out[key] = torch.round(w / scale).clamp(-127, 127).to(torch.int8)
+ out[key.replace(".weight", ".weight_scale")] = scale.reshape(())
+ out["blocks.0.attn.out_proj.comfy_quant"] = torch.tensor(
+ list(json.dumps({"format": "int8_tensorwise"}).encode()), dtype = torch.uint8
+ )
+ path = tmp_path / name
+ safetensors_torch.save_file(
+ {k: v.contiguous() for k, v in out.items()}, str(path), metadata = metadata
+ )
+ cfg = tmp_path / "base" / "transformer"
+ cfg.mkdir(parents = True, exist_ok = True)
+ (cfg / "config.json").write_text(
+ json.dumps({"_class_name": "MiniMaxH3Transformer3DModel", **TINY})
+ )
+ return str(path), str(tmp_path / "base")
+
+
+def _load(path, base):
+ from core.inference import diffusion_comfy_quant as cq
+ from core.inference.video_minimax_h3_comfy import load_h3_comfy_transformer
+ return load_h3_comfy_transformer(
+ _diffusers().MiniMaxH3Transformer3DModel,
+ path,
+ cq.refuse_comfy_quant(path),
+ config = base,
+ subfolder = "transformer",
+ dtype = torch.float32,
+ hf_token = None,
+ cache_dir = None,
+ local_files_only = True,
+ int8_backend = None,
+ fp8_backend = None,
+ )
+
+
+def test_a_with_heads_file_loads_through_the_comfy_loader_with_its_bank(tmp_path):
+ src = _tiny()
+ meta = {"config": json.dumps({"transformer": {"pdd_steps": HEADS}}), "pdd_acc_baked": "true"}
+ path, base = _write_comfy_file(tmp_path, src, WITH, metadata = meta)
+ model = _load(path, base).eval()
+ assert pdd.h3_pdd_heads(model) == HEADS
+ assert tuple(model.proj_out.weight.shape) == (HEADS * VIDEO_DIM, TINY["hidden_size"])
+ assert model.proj_out.weight.dtype == torch.float32
+ assert torch.equal(model.proj_out.weight, src.proj_out.weight)
+ assert torch.equal(model.audio_proj_out.bias, src.audio_proj_out.bias)
+ sigmas = _bound(model)
+ for i in (0, 7):
+ _out, seen = _run_and_capture(model, _inputs("cpu", 1.0 - float(sigmas[i])))
+ _check_step(model, seen, sigmas, i)
+
+
+def test_a_without_heads_bake_is_refused_by_the_comfy_loader(tmp_path):
+ src = _tiny(heads = 1)
+ for name, meta in (
+ (WITHOUT_OLD, {"pdd_acc_baked": "true"}),
+ (WITHOUT, None),
+ ):
+ path, base = _write_comfy_file(tmp_path, src, name, metadata = meta)
+ with pytest.raises(ValueError, match = "no head bank"):
+ _load(path, base)
+ plain, base = _write_comfy_file(tmp_path, src, "minimax_h3_fl2va_int8_convrot.safetensors")
+ assert isinstance(_load(plain, base).proj_out, torch.nn.Linear)
+
+
+# ── torch.compile and CUDA graphs ──
+
+
+def test_the_bank_compiles_with_no_graph_break_and_no_recompile_across_steps():
+ import torch._dynamo as dynamo
+
+ dynamo.reset()
+ bank = _bank()
+ h = torch.randn(5, 24)
+ sigmas = h3_sigmas(8)
+ explain = dynamo.explain(bank)(h)
+ assert explain.graph_break_count == 0 and explain.graph_count == 1
+ dynamo.reset()
+ counts = dynamo.utils.counters
+ counts.clear()
+ compiled = torch.compile(bank, fullgraph = True, backend = "eager")
+ with torch.no_grad():
+ for i in range(8):
+ bank.set_state(_state(sigmas[i], sigmas[i + 1])[:2])
+ ref = comfy_pdd_head(
+ bank.weight,
+ bank.bias,
+ h,
+ HEADS,
+ *comfy_span(sigmas[i], sigmas, HEADS, SHIFT_V),
+ SHIFT_V,
+ )
+ assert (compiled(h) - ref).abs().max() < 1e-6, i
+ assert counts["stats"]["unique_graphs"] == 1, dict(counts["stats"])
+ assert not counts["recompiles"] if "recompiles" in counts else True
+
+
+def _cuda():
+ if not torch.cuda.is_available():
+ pytest.skip("needs CUDA")
+
+
+def test_the_bank_compiles_with_inductor_on_cuda_and_tracks_every_step():
+ _cuda()
+ import torch._dynamo as dynamo
+
+ dynamo.reset()
+ dynamo.utils.counters.clear()
+ bank = _bank().cuda()
+ h = torch.randn(5, 24, device = "cuda")
+ sigmas = h3_sigmas(8)
+ compiled = torch.compile(bank, fullgraph = True)
+ with torch.no_grad():
+ for i in range(8):
+ bank.set_state(_state(sigmas[i], sigmas[i + 1], device = "cuda")[:2])
+ ref = comfy_pdd_head(
+ bank.weight.cpu(),
+ bank.bias.cpu(),
+ h.cpu(),
+ HEADS,
+ *comfy_span(sigmas[i], sigmas, HEADS, SHIFT_V),
+ SHIFT_V,
+ )
+ assert (compiled(h).cpu() - ref).abs().max() < 1e-5, i
+ assert dynamo.utils.counters["stats"]["unique_graphs"] == 1
+
+
+def test_a_cuda_graph_of_the_denoiser_replays_with_changing_sigmas(monkeypatch):
+ _cuda()
+ from core.inference import diffusion_cuda_graph as cg
+
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_CUDA_GRAPHS", "1")
+ model = _tiny(device = "cuda")
+ sigmas = _bound(model)
+ eager = []
+ with torch.no_grad():
+ for i in range(8):
+ eager.append(model(**_inputs("cuda", 1.0 - float(sigmas[i]), seed = 0)))
+ handle = cg.GraphedForward(model).install().enable()
+ try:
+ replayed = []
+ with torch.inference_mode():
+ for _round in range(2):
+ for i in range(8):
+ out = model(**_inputs("cuda", 1.0 - float(sigmas[i]), seed = 0))
+ replayed.append([o.clone() for o in out])
+ stats = dict(handle.stats)
+ finally:
+ handle.uninstall()
+ # one recording per distinct input shape (step 0's audio t equals its video t, so its timestep table is shorter);
+ # every other call is a replay with that step's sigmas
+ shapes = {tuple(_inputs("cuda", 1.0 - float(sigmas[i]))["timestep"].shape) for i in range(8)}
+ assert stats["captures"] == len(shapes), stats
+ assert stats["replays"] >= 16 - len(shapes) * 2, stats
+ assert stats["fallbacks"] == 0 and stats["eager_calls"] == 0, stats
+ for k, out in enumerate(replayed):
+ ref = eager[k % 8]
+ assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1]), k
+ # the steps really differ: a graph that baked the first step's heads would repeat it
+ assert not torch.equal(eager[0][0], eager[7][0])
+
+
+def test_the_denoiser_under_group_offload_matches_resident():
+ _cuda()
+ resident = _tiny(device = "cuda")
+ sigmas = _bound(resident)
+ ref = []
+ with torch.no_grad():
+ for i in range(8):
+ ref.append(resident(**_inputs("cuda", 1.0 - float(sigmas[i]))))
+ for kind in ("leaf_level", "block_level"):
+ model = _tiny()
+ _bound(model)
+ # block_level as Studio's streamed H3 denoiser applies it (stream, record_stream, non_blocking)
+ kwargs = (
+ {"num_blocks_per_group": 1, "record_stream": True, "non_blocking": True}
+ if kind == "block_level"
+ else {}
+ )
+ model.enable_group_offload(
+ torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = kind,
+ use_stream = True,
+ **kwargs,
+ )
+ with torch.no_grad():
+ for i in range(8):
+ out = model(**_inputs("cuda", 1.0 - float(sigmas[i])))
+ assert (out[0] - ref[i][0]).abs().max() < 1e-5, (kind, i)
+ assert (out[1] - ref[i][1]).abs().max() < 1e-5, (kind, i)
+
+
+def test_the_whole_pdd_denoiser_forward_compiles_fullgraph_without_recompiles():
+ """The pre-hook runs eager in ``Module.__call__`` (Studio compiles the forward / its repeated blocks, never the
+ call), so the compiled forward reads the step's sigmas as a tensor: one graph for every step."""
+ import torch._dynamo as dynamo
+
+ model = _tiny()
+ eager = _tiny()
+ sigmas = _bound(model)
+ _bound(eager)
+ kw = _inputs("cpu", 1.0 - float(sigmas[3]))
+ pdd._pre_hook(model, (), kw)
+ dynamo.reset()
+ explain = dynamo.explain(model.forward)(**kw)
+ assert explain.graph_break_count == 0 and explain.graph_count == 1
+ dynamo.reset()
+ dynamo.utils.counters.clear()
+ model.__dict__["forward"] = torch.compile(model.forward, fullgraph = True, backend = "eager")
+ with torch.no_grad():
+ # steps 1..7 share one input shape (step 0's timestep table is shorter, a different shape by design)
+ for i in range(1, 8):
+ kw = _inputs("cpu", 1.0 - float(sigmas[i]), seed = i)
+ out, ref = model(**kw), eager(**kw)
+ assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1]), i
+ assert float(model.proj_out.pdd_state[1]) == float(sigmas[i + 1]), i
+ assert dynamo.utils.counters["stats"]["unique_graphs"] == 1, dict(
+ dynamo.utils.counters["stats"]
+ )
+
+
+def test_the_denoiser_under_model_cpu_offload_matches_resident():
+ _cuda()
+ accelerate = pytest.importorskip("accelerate")
+ resident = _tiny(device = "cuda")
+ sigmas = _bound(resident)
+ model = _tiny()
+ _bound(model)
+ model, hook = accelerate.cpu_offload_with_hook(model, execution_device = torch.device("cuda"))
+ with torch.no_grad():
+ for i in range(8):
+ kw = _inputs("cuda", 1.0 - float(sigmas[i]))
+ out, ref = model(**kw), resident(**kw)
+ assert (out[0] - ref[0]).abs().max() < 1e-5 and (out[1] - ref[1]).abs().max() < 1e-5, i
+ hook.offload()
+ assert model.proj_out.weight.device.type == "cpu"
+
+
+def test_a_bound_render_reads_the_schedule_the_pipeline_sets_afterwards():
+ """Studio binds the pipeline's scheduler before the call; the modular loop's set_timesteps runs inside it."""
+ diffusers = _diffusers()
+ model = _tiny()
+ scheduler = diffusers.MiniMaxH3Scheduler(shift = 1.0)
+ pipe = _Pipe(model, scheduler)
+ scheduler.set_shift(SHIFT_V)
+ assert pdd.bind_h3_pdd_render(pipe, SHIFT_V, SHIFT_A) == HEADS
+ scheduler.set_timesteps(pdd.h3_pdd_num_inference_steps(pdd.H3_PDD_DEFAULT_STEPS))
+ spans = []
+ latents = torch.randn(8, 96)
+ for i, t in enumerate(scheduler.timesteps):
+ kw = _inputs("cpu", float(t), seed = i)
+ out, seen = _run_and_capture(model, kw)
+ _check_step(model, seen, scheduler.sigmas, i)
+ spans.append(comfy_span(scheduler.sigmas[i], scheduler.sigmas, HEADS, SHIFT_V))
+ latents = scheduler.step(out[0][0].float(), t, latents, return_dict = False)[0]
+ assert spans == [(4 * i, 4 * i + 4) for i in range(8)]
+
+
+def test_a_second_render_can_rebind_a_bank_its_first_render_moved_in_inference_mode():
+ """The pipeline runs under inference_mode, so the first render's device move made pdd_state an inference tensor
+ and the second render's set_shifts (outside it) raised. A pdd_state left as an inference tensor is replaced too."""
+ bank = _bank()
+ with torch.inference_mode():
+ bank.pdd_state = bank.pdd_state.clone()
+ assert bank.pdd_state.is_inference()
+ bank.set_shifts(SHIFT_V, SHIFT_A)
+ assert not bank.pdd_state.is_inference()
+ assert bank.pdd_state.tolist()[2:] == [SHIFT_V, SHIFT_A]
+ with torch.inference_mode():
+ bank.set_state(_state(0.7, 0.6)[:2])
+ assert not bank.pdd_state.is_inference()
+ assert bank.pdd_state.tolist()[:2] == pytest.approx([0.7, 0.6])
+ if torch.cuda.is_available():
+ bank = _bank()
+ with torch.inference_mode():
+ bank.set_state(_state(0.9, 0.8, device = "cuda")[:2])
+ assert not bank.pdd_state.is_inference() and bank.pdd_state.is_cuda
+ bank.set_shifts(SHIFT_V, SHIFT_A)
+
+
+def test_pdd_defaults_apply_only_to_minimax_h3():
+ """A Wan or Hunyuan checkpoint under a path naming "pdd" keeps its family defaults."""
+ from core.inference.video_families import default_video_generation_params
+
+ h3 = "/models/minimax_h3_ref2va_pruned_pdd_acc_8step_baked_with_heads_int8_convrot.safetensors"
+ assert default_video_generation_params(h3, fallback = (30, 5.0), family = "minimax-h3") == (8, 1.0)
+ wan = "/models/pdd/wan2.2-ti2v-5b.safetensors"
+ assert default_video_generation_params(wan, fallback = (50, 5.0), family = "wan2.2") == (20, 5.0)
+ other = "/models/pdd/hunyuan.safetensors"
+ assert default_video_generation_params(
+ other, fallback = (50, 6.0), family = "hunyuanvideo-1.5"
+ ) == (50, 6.0)
+ assert default_video_generation_params(other, fallback = (50, 6.0)) == (50, 6.0)
+ # A stock H3 checkpoint in a folder named pdd keeps the stock recipe: only the file name decides.
+ stock = "/models/pdd/minimax_h3_fl2va_int8_convrot.safetensors"
+ assert default_video_generation_params(stock, fallback = (30, 1.0), family = "minimax-h3") == (
+ 30,
+ 1.0,
+ )