Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: run MiniMax-H3 PDD checkpoints with their 32-head output bank, refuse the ones that cannot run (#14410)
Daniel Han6901dd3bf32c
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 47301289c84..adac540845e 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -437,6 +437,15 @@ def _video_comfy_key_map(fam: Any) -> Any:return None+def _h3_pdd_default_steps(fam: Any, pipe: Any, default_steps: int) -> int:+ """ComfyUI's 8 steps for a loaded MiniMax-H3 PDD head bank whatever the file is called, else ``default_steps``."""+ if getattr(fam, "name", None) != "minimax-h3" or pipe is None:+ return default_steps+ from .video_minimax_h3_pdd import H3_PDD_DEFAULT_STEPS, pipe_h3_pdd_heads++ return H3_PDD_DEFAULT_STEPS if pipe_h3_pdd_heads(pipe) else default_steps++def _h3_single_file_task(fam: Any, kind: str, gguf_filename: Optional[str], h3_task: Optional[str]) -> Optional[str]:@@ -11522,6 +11531,7 @@ class VideoBackend:fallback = (fam.default_steps, fam.default_guidance),family = fam.name,)+ default_steps = _h3_pdd_default_steps(fam, state.pipe, default_steps)steps = int(steps or default_steps)guidance = float(default_guidance if guidance is None else guidance)# A guidance-free family (supports_cfg=False, MiniMax-H3) forwards no CFG control to either engine: the@@ -11921,6 +11931,18 @@ class VideoBackend:setter = getattr(getattr(pipe, component, None), "set_shift", None)if callable(setter) and value is not None:setter(float(value))+ if fam.name == "minimax-h3":+ from .video_minimax_h3_pdd import (+ bind_h3_pdd_render,+ h3_pdd_num_inference_steps,+ )++ # A PDD head bank reads each step's (sigma, sigma_next) off this render's video schedule. The+ # H3 scheduler's grid holds the terminal 0, so `steps` model evaluations (ComfyUI's count, which+ # the bank's 8-step partition is distilled for) is a grid of steps + 1.+ if bind_h3_pdd_render(pipe, shift, audio_shift):+ kwargs["num_inference_steps"] = h3_pdd_num_inference_steps(steps)+ request_shape["pdd"] = True# What the clip records; LTX-2.5's two-stage render reports its stage-2 steps in the progress below.recorded_steps = steps@@ -13257,6 +13279,7 @@ class VideoBackend:fallback = (fam.default_steps, fam.default_guidance),family = fam.name,)+ default_steps = _h3_pdd_default_steps(fam, getattr(state, "pipe", None), default_steps)return {"loaded": True,"repo_id": state.repo_id,diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.pyindex e98b556cd3b..f6da110806d 100644--- a/studio/backend/core/inference/video_families.py+++ b/studio/backend/core/inference/video_families.py@@ -390,6 +390,23 @@ _LTX_VIDEO_09_REFUSAL = ("run. Load LTX-2 or LTX-2.3 instead (for example unsloth/LTX-2.3-GGUF).")# Last path segment of each identifier only: an owner or parent folder named i2v does not make a checkpoint I2V.+# MiniMax-H3 PDD files Studio cannot run. A "without_heads" bake keeps the single original output heads and needs the+# external PDD head bank applied per step by a custom node; loaded as a plain denoiser it renders wrong output with no+# error. The custom-node LoRA files hold a trunk LoRA plus that bank, not a denoiser. A "with_heads" bake carries its+# 32-head bank in the file and runs natively (video_minimax_h3_pdd). An unnamed without-heads file is caught at load+# from its header (pdd_acc_baked metadata with single heads).+_H3_PDD_WITHOUT_HEADS_RE = r"(?<![a-z0-9])pdd(?![a-z])[^/\\|]*without[-_ ]?heads"+H3_PDD_WITHOUT_HEADS_REFUSAL = (+ "This MiniMax-H3 PDD checkpoint has no head bank: it keeps the single original output heads and needs the "+ "external PDD head bank applied per step by a custom node, which Studio does not have, so it would render "+ "wrong output. Load the matching '..._pdd_acc_8step_baked_with_heads_...' checkpoint instead, which carries "+ "the 32 video and audio heads in the file."+)+_H3_PDD_BANK_RE = r"(?<![a-z0-9])pdd(?![a-z])[^/\\|]*(?:lora|_comfyui\.safetensors$)"+_H3_PDD_BANK_REFUSAL = (+ "This is a MiniMax-H3 PDD acceleration LoRA with an external head bank for a companion custom node, not a "+ "denoiser Studio can load. Load a '..._pdd_acc_8step_baked_with_heads_...' checkpoint instead."+)_HV15_I2V_RE = r"(?<![a-z0-9])i2v(?![a-z0-9])(?=[^/\\|]*[/\\]*\s*(?:$|\|))"_HV15_I2V_REFUSAL = ("This is a HunyuanVideo-1.5 image-to-video checkpoint. It needs its own image-conditioned "@@ -558,6 +575,10 @@ _FAMILIES: tuple[VideoFamily, ...] = (default_audio_flow_shift = 3.0,supports_keyframes = True,supports_references = True,+ unsupported_checkpoints = (+ (_H3_PDD_WITHOUT_HEADS_RE, H3_PDD_WITHOUT_HEADS_REFUSAL),+ (_H3_PDD_BANK_RE, _H3_PDD_BANK_REFUSAL),+ ),supports_cfg = False,),# LTX-2.5 (diffusers >= 0.40): the 22B LTX-2.3 DiT without the video FFN bias plus a keyframe position embedding,@@ -1609,6 +1630,9 @@ def validate_video_reference_conditioning(# Default (steps, guidance) per checkpoint variant, matched by substring (picked id then base repo), most specific# first._VIDEO_GENERATION_DEFAULTS: tuple[tuple[str, int, float], ...] = (+ # MiniMax-H3 PDD baked checkpoints: ComfyUI's recipe is 8 Euler steps on the simple schedule at CFG 1 (shifts 12 / 3+ # are the family defaults). Its head bank was distilled for that partition.+ ("pdd", 8, 1.0),("distilled", 8, 1.0),# Wan2.1 14B FusionX merge (CausVid / AccVideo distilled): its card asks for CFG 1 and 8-10 steps.("fusionx", 8, 1.0),@@ -1629,8 +1653,10 @@ _VIDEO_GENERATION_DEFAULTS: tuple[tuple[str, int, float], ...] = (# The Wan and FusionX keys count only in the model / file name, so an owner or folder spelling one cannot outrank the family the# checkpoint itself names._NAME_SEGMENT_DEFAULT_KEYS = frozenset(- k for k, _s, _g in _VIDEO_GENERATION_DEFAULTS if k.startswith(("wan", "fusionx"))+ k for k, _s, _g in _VIDEO_GENERATION_DEFAULTS if k.startswith(("wan", "fusionx", "pdd")))+# A PDD recipe on another family's checkpoint (a Wan file under /models/pdd/) would silently run 8 steps at CFG 1.+_FAMILY_SCOPED_DEFAULT_KEYS = {"pdd": "minimax-h3"}def default_video_generation_params(@@ -1646,7 +1672,7 @@ def default_video_generation_params(if family == LTX25_FAMILY:# The generic table's "ltx" key is the LTX-2 dev recipe, and 2.5's Diffusers pack names no variant at all.return LTX25_DEV_DEFAULTS if ltx25_is_dev(*identifiers) else LTX25_DISTILLED_DEFAULTS- variant = video_generation_variant(*identifiers)+ variant = video_generation_variant(*identifiers, family = family)for key, steps, guidance in _VIDEO_GENERATION_DEFAULTS:if key == variant:return steps, guidance@@ -1673,13 +1699,19 @@ def ltx25_is_dev(*identifiers: Optional[str]) -> bool:return False-def video_generation_variant(*identifiers: Optional[str]) -> Optional[str]:+def video_generation_variant(+ *identifiers: Optional[str], family: Optional[str] = None+) -> Optional[str]:"""The ``_VIDEO_GENERATION_DEFAULTS`` key the first identifier naming one matches, or None; the precedence- ``default_video_generation_params`` uses, so every per-variant decision agrees with the defaults."""+ ``default_video_generation_params`` uses, so every per-variant decision agrees with the defaults. Family-scoped+ keys (``_FAMILY_SCOPED_DEFAULT_KEYS``) match only for their own family."""for identifier in identifiers:whole = (identifier or "").lower()name = whole.replace("\\", "/").rstrip("/").rsplit("/", 1)[-1]for key, _steps, _guidance in _VIDEO_GENERATION_DEFAULTS:+ scope = _FAMILY_SCOPED_DEFAULT_KEYS.get(key)+ if scope is not None and family != scope:+ continueneedle = name if key in _NAME_SEGMENT_DEFAULT_KEYS else whole# Match the key as a name segment: reject a preceding ASCII letter so "swan-video" does not false-match# "wan".diff --git a/studio/backend/core/inference/video_minimax_h3_comfy.py b/studio/backend/core/inference/video_minimax_h3_comfy.pyindex f111f0c657b..d30c132aef1 100644--- a/studio/backend/core/inference/video_minimax_h3_comfy.py+++ b/studio/backend/core/inference/video_minimax_h3_comfy.py@@ -136,18 +136,43 @@ def h3_comfy_curve_metadata(path: str) -> Optional[dict]:def comfy_layout(path: str) -> dict:"""``diffusion_comfy_quant.original_layout`` hooks for MiniMax-H3: key map, float32 tensors and, for a prunedfile, the curve adaLN installed on the freshly built model."""- from .video_minimax_h3_adaln import apply_h3_adaln_curve+ from .video_minimax_h3_pdd import h3_pdd_heads_in_filemetadata = h3_comfy_curve_metadata(path)+ heads = h3_pdd_heads_in_file(path)return {"key_map": h3_comfy_key_map,"keep_dtype": functools.partial(h3_comfy_keep_dtype, pruned = metadata is not None),- "prepare_model": (lambda model: apply_h3_adaln_curve(model, metadata))- if metadata is not None- else None,+ "prepare_model": _h3_prepare_model(metadata, heads),}+def _h3_prepare_model(+ metadata: Optional[dict],+ heads: int,+ logger: Any = None,+) -> Any:+ """The ``prepare_model`` hook for one H3 file: curve adaLN for a pruned file, the stacked PDD head bank for a+ ``with_heads`` one, None when the stock model already matches."""+ if metadata is None and heads <= 1:+ return None++ def prepare(model: Any) -> None:+ if metadata is not None:+ from .video_minimax_h3_adaln import apply_h3_adaln_curve+ if logger is not None:+ apply_h3_adaln_curve(model, metadata, logger = logger)+ else:+ apply_h3_adaln_curve(model, metadata)+ if heads > 1:+ from .video_minimax_h3_pdd import install_h3_pdd_heads+ install_h3_pdd_heads(model, heads)+ if logger is not None:+ logger.info("video.h3_pdd: %d-head PDD output bank per stream installed", heads)++ return prepare++def load_h3_comfy_transformer(transformer_cls: Any,path: str,@@ -166,9 +191,11 @@ def load_h3_comfy_transformer() -> Any:"""The H3 denoiser of a ComfyUI-quantized file, int8 / fp8 codes kept, curve adaLN installed."""from .diffusion_comfy_quant import load_comfy_quant_transformer- from .video_minimax_h3_adaln import apply_h3_adaln_curve+ from .video_minimax_h3_pdd import h3_pdd_heads_in_filemetadata = h3_comfy_curve_metadata(path)+ # Header only, before any tensor is read: a PDD trunk without its head bank is refused here.+ heads = h3_pdd_heads_in_file(path)return load_comfy_quant_transformer(transformer_cls,path,@@ -187,10 +214,6 @@ def load_h3_comfy_transformer(target = target,logger = logger,key_map = h3_comfy_key_map,- prepare_model = (- (lambda model: apply_h3_adaln_curve(model, metadata, logger = logger))- if metadata is not None- else None- ),+ prepare_model = _h3_prepare_model(metadata, heads, logger = logger),keep_dtype = functools.partial(h3_comfy_keep_dtype, pruned = metadata is not None),)diff --git a/studio/backend/core/inference/video_minimax_h3_pdd.py b/studio/backend/core/inference/video_minimax_h3_pdd.pynew file mode 100644index 00000000000..5a9ebf72bcd--- /dev/null+++ b/studio/backend/core/inference/video_minimax_h3_pdd.py@@ -0,0 +1,369 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""MiniMax-H3 PDD (parallel decoding distillation) checkpoints: the stacked output-head bank.++A PDD ``with_heads`` file stores ``pdd_steps`` (32) video and audio output heads stacked along the rows of+``final_layer.video_out`` / ``final_layer.audio_out`` (``[32 * 96, 5376]`` and ``[32 * 32, 5376]``): head 0 is a full+head and every later head is an offset from it. One sampler step from ``sigma`` to ``sigma_next`` covers a contiguous+run of fine intervals on the unshifted grid, and consumes head 0 plus the dt-weighted mean of the offsets it spans.+This matches ComfyUI's PDD head selection (``FinalLayer.forward`` / ``_pdd_head``) term for term.++Graph safety: the ``(sigma, sigma_next)`` pair is a device tensor on each bank (``pdd_state``), rewritten in place by+a forward pre-hook on the denoiser before every call, and the head selection is pure tensor math on it. Nothing about+the step is a Python constant, so a CUDA graph replays it with new sigmas and torch.compile never specializes on it.++A ``without_heads`` bake keeps the plain single heads and needs an external head bank applied by a custom node, which+Studio does not have, so it is refused rather than run as a plain trunk (that renders wrong output with no error).+"""++from __future__ import annotations++import json+import os+import re+from typing import Any, Optional++# ComfyUI's recipe for the baked 8-step PDD checkpoints: Euler, simple schedule, 8 steps, CFG 1, shifts 12 / 3.+H3_PDD_DEFAULT_STEPS = 8+H3_PDD_GUIDANCE = 1.0++VIDEO_HEAD_KEY = "final_layer.video_out.weight"+AUDIO_HEAD_KEY = "final_layer.audio_out.weight"+_BANKS_ATTR = "_unsloth_h3_pdd_banks"+_SCHEDULER_ATTR = "_unsloth_h3_pdd_scheduler"+_HOOK_ATTR = "_unsloth_h3_pdd_hook"+HEADS_ATTR = "_unsloth_h3_pdd_heads"++_PDD_TOKEN = re.compile(r"(?<![a-z0-9])pdd(?![a-z])")+++def _name(filename: Optional[str]) -> str:+ return os.path.basename(str(filename or "")).lower()+++def h3_pdd_name(filename: Optional[str]) -> bool:+ """The file name says PDD (``..._pdd_acc_8step_...``)."""+ return bool(_PDD_TOKEN.search(_name(filename)))+++def _without_heads_message(name: str) -> str:+ from .video_families import H3_PDD_WITHOUT_HEADS_REFUSAL+ return f"'{name}': {H3_PDD_WITHOUT_HEADS_REFUSAL}"+++def _metadata_pdd_steps(metadata: dict) -> Optional[int]:+ raw = metadata.get("config")+ if not raw:+ return None+ try:+ value = (json.loads(raw).get("transformer") or {}).get("pdd_steps")+ except (TypeError, ValueError, AttributeError):+ return None+ return int(value) if value is not None else None+++def _in_dim(header: dict, key: str, default: int) -> int:+ shape = (header.get(key) or {}).get("shape") or ()+ return int(shape[1]) if len(shape) == 2 else default+++def h3_pdd_heads_from_header(+ header: dict,+ *,+ video_dim: Optional[int] = None,+ audio_dim: Optional[int] = None,+ filename: Optional[str] = None,+) -> int:+ """Heads per stream of a ComfyUI-layout H3 file (1 for a plain denoiser). Raises ``ValueError`` for a PDD file+ Studio cannot run correctly: a trunk without its bank, or a bank whose shapes disagree."""+ name = os.path.basename(str(filename or "")) or "this checkpoint"+ metadata = header.get("__metadata__") or {}+ video = header.get(VIDEO_HEAD_KEY) or {}+ audio = header.get(AUDIO_HEAD_KEY) or {}+ if not video or not audio:+ return 1+ video_dim = int(video_dim or _in_dim(header, "video_patch_proj.weight", 96))+ audio_dim = int(audio_dim or _in_dim(header, "audio_patch_proj.weight", 32))+ rows_v = int((video.get("shape") or [0])[0])+ rows_a = int((audio.get("shape") or [0])[0])+ if rows_v % video_dim or rows_a % audio_dim or rows_v // video_dim != rows_a // audio_dim:+ raise ValueError(+ f"'{name}': output heads of {rows_v} video / {rows_a} audio rows are not one stacked bank of "+ f"{video_dim} / {audio_dim}-row heads."+ )+ heads = rows_v // video_dim+ declared = _metadata_pdd_steps(metadata)+ if heads == 1:+ baked = str(metadata.get("pdd_acc_baked", "")).strip().lower() == "true"+ if baked or (declared or 0) > 1 or h3_pdd_name(filename):+ raise ValueError(_without_heads_message(name))+ elif declared is not None and declared > 1 and declared != heads:+ raise ValueError(+ f"'{name}' declares {declared} PDD heads but stores {heads}: the file is inconsistent."+ )+ return heads+++def h3_pdd_heads_in_file(+ path: str,+ *,+ video_dim: Optional[int] = None,+ audio_dim: Optional[int] = None,+ filename: Optional[str] = None,+) -> int:+ from .diffusion_comfy_quant import _read_header+ header, _ = _read_header(path)+ return h3_pdd_heads_from_header(+ header, video_dim = video_dim, audio_dim = audio_dim, filename = filename or path+ )+++# ── the head-bank math (shared by every bank and by the tests' oracle comparisons) ──+++def pdd_head_weights(state: Any, heads: int) -> Any:+ """Per-head blend weights ``(heads,)`` float64 for one step, from ``state = [sigma, sigma_next, select_shift,+ stream_shift]`` (float64). Head 0 gets 0 here: it is always added whole.++ ComfyUI: ``start, stop = round((1 - unshift(sigma)) * n), round((1 - unshift(sigma_next)) * n)`` with the unshift+ in float32 (the sampler's sigma dtype), ``start = min(start, n - 1)``, ``stop = max(stop, start + 1)``, then+ ``dt`` of the stream's shifted ``n``-interval grid over ``[start, stop)`` normalised to sum 1, applied to heads+ ``max(start, 1) .. stop - 1``. Same here, as masks instead of slices so no index ever reaches the host."""+ import torch++ sig = state[0:2].to(torch.float32)+ sel = state[2]+ # time_shift_sigma(s, shift, 1.0): the Python-float shift and (1 - shift) become float32 operands.+ base = sig / (sel.to(torch.float32) + sig * (1.0 - sel).to(torch.float32))+ pos = torch.round((1.0 - base).to(torch.float64) * heads)+ start = pos[0].clamp(min = 0, max = heads - 1)+ stop = torch.maximum(pos[1], start + 1)+ shift = state[3]+ grid = torch.linspace(1.0, 0.0, heads + 1, dtype = torch.float64, device = state.device)+ dt = (1.0 - shift * grid / (1.0 + (shift - 1.0) * grid)).diff()+ k = torch.arange(heads, dtype = torch.float64, device = state.device)+ inside = (k >= start) & (k < stop)+ dt = torch.where(inside, dt, torch.zeros_like(dt))+ w = dt / dt.sum()+ return torch.where(k >= 1, w, torch.zeros_like(w))+++def _bank_class():+ import torch+ from torch import nn++ class H3PDDHeadBank(nn.Module):+ """``heads`` stacked output heads (``weight [heads * out, in]``, the checkpoint's own layout and key), blended+ per step from ``pdd_state``. Exposes ``weight`` / ``in_features`` / ``out_features`` like the Linear it+ replaces (the denoiser casts its activation to ``proj_out.weight.dtype``)."""++ def __init__(+ self,+ in_features: int,+ out_features: int,+ heads: int,+ *,+ dtype: Any = None,+ ):+ super().__init__()+ self.in_features = int(in_features)+ self.out_features = int(out_features)+ self.heads = int(heads)+ dtype = dtype or torch.float32+ self.weight = nn.Parameter(+ torch.empty(self.heads * self.out_features, self.in_features, dtype = dtype)+ )+ self.bias = nn.Parameter(torch.empty(self.heads * self.out_features, dtype = dtype))+ # [sigma, sigma_next, select (video) shift, this stream's shift], float64. A plain tensor attribute, not a+ # buffer: a module-wide .to(bfloat16) must not round the sigmas, and offload must not move it.+ self.pdd_state = torch.tensor([1.0, 0.0, 12.0, 12.0], dtype = torch.float64)++ def extra_repr(self) -> str:+ return f"in_features={self.in_features}, out_features={self.out_features}, heads={self.heads}"++ def _own_state(self, device: Any = None) -> Any:+ """``pdd_state`` as a normal (non-inference) tensor on ``device``. The first render moves it to the+ GPU inside the pipeline's inference_mode; a tensor made there is an inference tensor, and the next+ render's set_shifts (outside inference_mode) could not write it."""+ state = self.pdd_state+ if state.is_inference() or (device is not None and state.device != device):+ with torch.inference_mode(False):+ state = state.to(device or state.device, copy = True)+ self.pdd_state = state+ return state++ def set_state(self, pair: Any) -> None:+ self._own_state(pair.device)[0:2].copy_(pair)++ def set_shifts(self, select_shift: float, stream_shift: float) -> None:+ state = self._own_state()+ state[2] = float(select_shift)+ state[3] = float(stream_shift)++ def forward(self, hidden_states: Any) -> Any:+ state = self.pdd_state+ if state.device != hidden_states.device:+ state = state.to(hidden_states.device)+ w = pdd_head_weights(state, self.heads).to(hidden_states.dtype)+ weight = self.weight.to(hidden_states.dtype)+ bias = self.bias.to(hidden_states.dtype)+ rows = weight.view(self.heads, self.out_features, self.in_features)+ brows = bias.view(self.heads, self.out_features)+ return nn.functional.linear(+ hidden_states,+ rows[0] + torch.einsum("n,noi->oi", w, rows),+ brows[0] + torch.einsum("n,no->o", w, brows),+ )++ return H3PDDHeadBank+++_BANK_CLASS = None+++def head_bank_class():+ global _BANK_CLASS+ if _BANK_CLASS is None:+ _BANK_CLASS = _bank_class()+ return _BANK_CLASS+++# ── the per-step sigma pair ──++_SIGMA_ARGS = ("timestep", "timestep_indices", "video_indices")+# Their MiniMaxH3Transformer3DModel.forward positions.+_SIGMA_POSITIONS = (3, 4, 7)+++def h3_pdd_sigma_pair(+ timestep: Any, timestep_indices: Any, video_indices: Any, sample_sigmas: Any+) -> Any:+ """``[sigma, sigma_next]`` float64 on the device, without a host sync. ``sigma`` is the generated video rows'+ sigma (conditioning rows sit at a larger t, so it is the largest video-row sigma), clamped like ComfyUI's; the+ next one is the schedule entry after the nearest, the last entry for the last step (sigma 0)."""+ import torch++ rows = timestep_indices.index_select(0, video_indices.reshape(-1).to(timestep_indices.device))+ t = timestep.reshape(-1).index_select(0, rows).to(torch.float32).min()+ sigma = (1.0 - t).clamp(min = 1e-6)+ sigmas = sample_sigmas.reshape(-1).to(device = sigma.device, dtype = torch.float32)+ i = (sigmas - sigma).abs().argmin()+ nxt = sigmas.index_select(0, (i + 1).clamp(max = sigmas.shape[0] - 1).reshape(1))[0]+ return torch.stack([sigma, nxt]).to(torch.float64)+++class _Schedule:+ """Holds the scheduler whose ``sigmas`` drive the bank (bound per render)."""++ __slots__ = ("scheduler",)++ def __init__(self) -> None:+ self.scheduler = None+++def _pre_hook(module: Any, args: tuple, kwargs: dict) -> None:+ banks = module.__dict__.get(_BANKS_ATTR) or ()+ if not banks:+ return None+ values = []+ for name, position in zip(_SIGMA_ARGS, _SIGMA_POSITIONS):+ value = kwargs.get(name)+ if value is None and len(args) > position:+ value = args[position]+ values.append(value)+ holder = module.__dict__.get(_SCHEDULER_ATTR)+ scheduler = getattr(holder, "scheduler", None)+ sigmas = getattr(scheduler, "sigmas", None)+ if sigmas is None or any(v is None for v in values):+ raise ValueError(+ "MiniMax-H3 PDD heads need the sampler's sigma schedule: bind the render's scheduler with "+ "bind_h3_pdd_schedule before calling the denoiser."+ )+ pair = h3_pdd_sigma_pair(*values, sigmas)+ for bank in banks:+ bank.set_state(pair)+ return None+++def install_h3_pdd_heads(model: Any, heads: int) -> bool:+ """Swap a freshly built (meta) H3 denoiser's ``proj_out`` / ``audio_proj_out`` for ``heads``-head banks and hook+ the per-step sigma pair. No-op (False) for ``heads <= 1``."""+ if int(heads) <= 1:+ return False+ import torch++ bank_cls = head_bank_class()+ banks = []+ for attr in ("proj_out", "audio_proj_out"):+ linear = getattr(model, attr)+ bank = bank_cls(linear.in_features, linear.out_features, heads, dtype = torch.float32)+ setattr(model, attr, bank)+ banks.append(bank)+ model.__dict__[_BANKS_ATTR] = tuple(banks)+ model.__dict__[_SCHEDULER_ATTR] = _Schedule()+ model.__dict__[HEADS_ATTR] = int(heads)+ model.__dict__[_HOOK_ATTR] = model.register_forward_pre_hook(_pre_hook, with_kwargs = True)+ return True+++def h3_pdd_heads(module: Any) -> int:+ """Heads per stream of an installed bank, 0 otherwise."""+ try:+ return int((getattr(module, "__dict__", None) or {}).get(HEADS_ATTR) or 0)+ except (TypeError, ValueError):+ return 0+++def bind_h3_pdd_schedule(+ module: Any, scheduler: Any, video_shift: Optional[float], audio_shift: Optional[float]+) -> bool:+ """Point one PDD denoiser at the scheduler whose sigmas it reads, and set the per-stream shifts (head selection+ runs on the video shift; each stream's dt grid on its own). False for a plain denoiser."""+ if not h3_pdd_heads(module):+ return False+ video_shift = 12.0 if video_shift is None else float(video_shift)+ audio_shift = 3.0 if audio_shift is None else float(audio_shift)+ module.__dict__[_SCHEDULER_ATTR].scheduler = scheduler+ video_bank, audio_bank = module.__dict__[_BANKS_ATTR]+ video_bank.set_shifts(video_shift, video_shift)+ audio_bank.set_shifts(video_shift, audio_shift)+ return True+++H3_DENOISER_COMPONENTS = ("transformer", "transformer_ref")+++def _components(pipe: Any) -> list:+ out = []+ for name in H3_DENOISER_COMPONENTS:+ try:+ module = getattr(pipe, name, None)+ except Exception: # noqa: BLE001 - a modular pipeline may raise for an unloaded component+ module = None+ if module is not None:+ out.append(module)+ return out+++def pipe_h3_pdd_heads(pipe: Any) -> int:+ return max((h3_pdd_heads(m) for m in _components(pipe)), default = 0)+++def bind_h3_pdd_render(+ pipe: Any, video_shift: Optional[float], audio_shift: Optional[float]+) -> int:+ """Bind every PDD denoiser of an H3 pipeline to its video scheduler for this render; returns the head count+ (0: no PDD denoiser, nothing changed)."""+ scheduler = getattr(pipe, "scheduler", None)+ heads = 0+ for module in _components(pipe):+ if bind_h3_pdd_schedule(module, scheduler, video_shift, audio_shift):+ heads = max(heads, h3_pdd_heads(module))+ return heads+++def h3_pdd_num_inference_steps(steps: int) -> int:+ """The MiniMax-H3 scheduler's grid size for ``steps`` model evaluations: its grid holds the terminal 0+ (``linspace(1, 0, n)``), so ComfyUI's 8-step simple schedule (``1, 7/8, ..., 1/8, 0`` before the shift) is 9."""+ return int(steps) + 1diff --git a/studio/backend/tests/test_video_minimax_h3_pdd.py b/studio/backend/tests/test_video_minimax_h3_pdd.pynew file mode 100644index 00000000000..322e0b84af0--- /dev/null+++ b/studio/backend/tests/test_video_minimax_h3_pdd.py@@ -0,0 +1,823 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""MiniMax-H3 PDD checkpoints: detection, the without-heads refusal, and the stacked head bank against a direct port of+ComfyUI's ``FinalLayer`` PDD math and a float64 oracle, per step (the last one included), through the real loader,+under torch.compile, a CUDA graph replaying changing sigmas, and diffusers group offload.++The model is a tiny random ``MiniMaxH3Transformer3DModel`` (no checkpoint, no network); the CUDA cases skip on a host+without a GPU."""++from __future__ import annotations++import json++import pytest++torch = pytest.importorskip("torch")+safetensors_torch = pytest.importorskip("safetensors.torch")++from core.inference import video_minimax_h3_pdd as pdd # noqa: E402++HEADS = 32+VIDEO_DIM = 96+AUDIO_DIM = 32+SHIFT_V, SHIFT_A = 12.0, 3.0+++# ── a direct port of ComfyUI's MiniMax-H3 PDD head math (comfy/ldm/minimax_h3/model.py) ──+++def comfy_time_shift_sigma(sigma, from_shift, to_shift):+ base = sigma / (from_shift + sigma * (1.0 - from_shift))+ return to_shift * base / (1.0 + (to_shift - 1.0) * base)+++def comfy_span(sigma, sample_sigmas, n, shift_v):+ i = int((sample_sigmas - sigma).abs().argmin())+ sigma_next = sample_sigmas[min(i + 1, sample_sigmas.shape[0] - 1)]+ start, stop = (+ round(float(1.0 - comfy_time_shift_sigma(s, shift_v, 1.0)) * n) for s in (sigma, sigma_next)+ )+ start = min(start, n - 1)+ stop = max(stop, start + 1)+ return start, stop+++def comfy_pdd_head(weight, bias, h, n, start, stop, flow_shift):+ grid = torch.linspace(1.0, 0.0, n + 1, dtype = torch.float64)+ dt = (1.0 - flow_shift * grid / (1.0 + (flow_shift - 1.0) * grid)).diff()[start:stop]+ w = (dt / dt.sum()).to(h)+ rows = weight.to(h).reshape(n, -1, weight.shape[1])+ brows = bias.to(h).reshape(n, -1)+ first = max(start, 1)+ return torch.nn.functional.linear(+ h,+ rows[0] + torch.einsum("n,noi->oi", w[first - start :], rows[first:stop]),+ brows[0] + torch.einsum("n,no->o", w[first - start :], brows[first:stop]),+ )+++def oracle_head(weight, bias, h, n, start, stop, flow_shift):+ """The same blend entirely in float64 (weights, activations and the matmul)."""+ grid = torch.linspace(1.0, 0.0, n + 1, dtype = torch.float64)+ dt = (1.0 - flow_shift * grid / (1.0 + (flow_shift - 1.0) * grid)).diff()[start:stop]+ w = dt / dt.sum()+ rows = weight.double().reshape(n, -1, weight.shape[1])+ brows = bias.double().reshape(n, -1)+ W, B = rows[0].clone(), brows[0].clone()+ for j, k in enumerate(range(start, stop)):+ if k >= 1:+ W += w[j] * rows[k]+ B += w[j] * brows[k]+ return h.double() @ W.T + B+++def comfy_simple_sigmas(steps, shift = SHIFT_V):+ """ComfyUI's ``simple`` scheduler over ModelSamplingDiscreteFlow (1000 timesteps, multiplier 1000)."""+ t = torch.arange(1, 1001, dtype = torch.float32) / 1000.0+ table = shift * t / (1 + (shift - 1) * t)+ ss = len(table) / steps+ return torch.tensor([float(table[-(1 + int(x * ss))]) for x in range(steps)] + [0.0])+++def h3_sigmas(steps, shift = SHIFT_V):+ """MiniMaxH3Scheduler.set_timesteps(steps + 1): the grid Studio drives a PDD bank with."""+ base = torch.linspace(1.0, 0.0, pdd.h3_pdd_num_inference_steps(steps), dtype = torch.float32)+ return torch.unique_consecutive(shift * base / (1 + (shift - 1) * base))+++def _bank(+ in_features = 24,+ out_features = VIDEO_DIM,+ heads = HEADS,+ seed = 0,+):+ torch.manual_seed(seed)+ bank = pdd.head_bank_class()(in_features, out_features, heads)+ with torch.no_grad():+ bank.weight.normal_(0, 0.05)+ bank.bias.normal_(0, 0.05)+ return bank+++def _state(+ sigma,+ sigma_next,+ select = SHIFT_V,+ stream = SHIFT_V,+ device = "cpu",+):+ return torch.tensor(+ [float(sigma), float(sigma_next), select, stream], dtype = torch.float64, device = device+ )+++# ── detection and refusal ──+++def _header(+ video_rows,+ audio_rows,+ metadata = None,+):+ header = {+ "video_patch_proj.weight": {"dtype": "F32", "shape": [5376, VIDEO_DIM]},+ "audio_patch_proj.weight": {"dtype": "F32", "shape": [5376, AUDIO_DIM]},+ pdd.VIDEO_HEAD_KEY: {"dtype": "F32", "shape": [video_rows, 5376]},+ pdd.AUDIO_HEAD_KEY: {"dtype": "F32", "shape": [audio_rows, 5376]},+ }+ if metadata is not None:+ header["__metadata__"] = metadata+ return header+++WITH = "minimax_h3_ref2va_pruned_pdd_acc_8step_baked_with_heads_int8_convrot.safetensors"+WITHOUT = "minimax_h3_ref2va_pdd_acc_8step_baked_without_heads_int8_convrot.safetensors"+# The without-heads file's former name: nothing in it says "without heads".+WITHOUT_OLD = "minimax_h3_ref2va_pdd_acc_8step_baked_int8_convrot.safetensors"+++def test_a_with_heads_bank_is_detected_from_its_shapes_and_metadata():+ meta = {"config": json.dumps({"transformer": {"pdd_steps": 32}}), "pdd_acc_baked": "true"}+ assert pdd.h3_pdd_heads_from_header(_header(3072, 1024, meta), filename = WITH) == 32+ # shapes alone are enough+ assert pdd.h3_pdd_heads_from_header(_header(3072, 1024), filename = "opaque.safetensors") == 32+ assert (+ pdd.h3_pdd_heads_from_header(+ _header(VIDEO_DIM, AUDIO_DIM), filename = "minimax_h3_fl2va_int8_convrot.safetensors"+ )+ == 1+ )+++ "header,name",+ [+ (_header(VIDEO_DIM, AUDIO_DIM, {"pdd_acc_baked": "true"}), WITHOUT_OLD),+ (_header(VIDEO_DIM, AUDIO_DIM, {"pdd_acc_baked": "true"}), "opaque.safetensors"),+ (_header(VIDEO_DIM, AUDIO_DIM), WITHOUT),+ (+ _header(+ VIDEO_DIM, AUDIO_DIM, {"config": json.dumps({"transformer": {"pdd_steps": 32}})}+ ),+ "x.safetensors",+ ),+ ],+)+def test_a_pdd_trunk_without_its_head_bank_is_refused_at_load(header, name):+ with pytest.raises(ValueError, match = "no head bank"):+ pdd.h3_pdd_heads_from_header(header, filename = name)+++def test_an_inconsistent_bank_is_refused():+ with pytest.raises(ValueError, match = "not one stacked bank"):+ pdd.h3_pdd_heads_from_header(_header(3072, 512), filename = WITH)+ with pytest.raises(ValueError, match = "declares 16"):+ pdd.h3_pdd_heads_from_header(+ _header(3072, 1024, {"config": json.dumps({"transformer": {"pdd_steps": 16}})}),+ filename = WITH,+ )+++ "name,match",+ [+ (WITHOUT, "no head bank"),+ ("minimax_h3_fl2va_pdd_acc_8step_comfyui.safetensors", "acceleration LoRA"),+ ("minimax_h3_ref2va_pdd_acc_8step_comfyui.safetensors", "acceleration LoRA"),+ ],+)+def test_the_family_refuses_pdd_files_it_cannot_run_by_name(name, match):+ from core.inference.video import VideoBackend+ with pytest.raises(ValueError, match = match):+ VideoBackend().validate_load_request(+ "aptech0081/MiniMax-H3-PDD-ComfyUI",+ gguf_filename = name,+ model_kind = "single_file",+ family_override = "minimax-h3",+ )+++def test_the_with_heads_file_passes_the_name_refusals():+ from core.inference.video import VideoBackend+ try:+ VideoBackend().validate_load_request(+ "aptech0081/MiniMax-H3-PDD-ComfyUI",+ gguf_filename = WITH,+ model_kind = "single_file",+ family_override = "minimax-h3",+ )+ except (+ ValueError,+ ImportError,+ ) as exc: # later probes may refuse on this host; never a PDD refusal+ assert "PDD" not in str(exc) and "single .safetensors checkpoint" not in str(exc)+++def test_pdd_files_default_to_comfyuis_8_step_recipe():+ from core.inference.video_families import default_video_generation_params, detect_video_family++ fam = detect_video_family("MiniMaxAI/MiniMax-H3")+ assert default_video_generation_params(+ None, WITH, "aptech0081/MiniMax-H3-PDD-ComfyUI", fallback = (30, 1.0), family = fam.name+ ) == (8, 1.0)+ assert default_video_generation_params(+ None,+ "minimax_h3_fl2va_int8_convrot.safetensors",+ "MiniMaxAI/MiniMax-H3",+ fallback = (30, 1.0),+ family = fam.name,+ ) == (30, 1.0)+ assert (fam.default_flow_shift, fam.default_audio_flow_shift) == (SHIFT_V, SHIFT_A)+++def test_the_default_grid_is_comfyuis_simple_schedule():+ for steps in (4, 8):+ ours, comfy = h3_sigmas(steps), comfy_simple_sigmas(steps)+ assert ours.shape == comfy.shape == (steps + 1,)+ assert torch.allclose(ours, comfy, atol = 1e-6, rtol = 0)+ sched = pytest.importorskip("diffusers").MiniMaxH3Scheduler(shift = SHIFT_V)+ sched.set_timesteps(pdd.h3_pdd_num_inference_steps(8))+ assert torch.equal(sched.sigmas.cpu(), h3_sigmas(8))+ assert len(sched.timesteps) == 8 # 8 model evaluations, as in ComfyUI+++# ── head selection and the blend, per step ──++[email protected]("steps", [8, 4, 6, 20])+def test_head_selection_matches_comfyui_at_every_step(steps):+ sigmas = h3_sigmas(steps)+ for i in range(steps):+ sigma = sigmas[i]+ start, stop = comfy_span(sigma, sigmas, HEADS, SHIFT_V)+ state = _state(sigma, sigmas[i + 1])+ w = pdd.pdd_head_weights(state, HEADS)+ picked = [k for k in range(HEADS) if w[k] != 0]+ assert picked == [k for k in range(max(start, 1), stop)], (steps, i)+ if steps == 8:+ assert (start, stop) == (4 * i, 4 * i + 4)+ # the last step lands on sigma_next = 0 and spans to the end of the bank+ last = comfy_span(sigmas[-2], sigmas, HEADS, SHIFT_V)+ assert float(sigmas[-1]) == 0.0 and last[1] == HEADS++[email protected]("steps", [8, 4])+def test_the_blend_matches_comfyui_and_the_float64_oracle_per_step(steps):+ sigmas = h3_sigmas(steps)+ for out_dim, stream_shift, seed in ((VIDEO_DIM, SHIFT_V, 1), (AUDIO_DIM, SHIFT_A, 2)):+ bank = _bank(out_features = out_dim, seed = seed)+ h = torch.randn(7, 24, generator = torch.Generator().manual_seed(seed))+ for i in range(steps):+ bank.pdd_state = _state(sigmas[i], sigmas[i + 1], stream = stream_shift)+ start, stop = comfy_span(sigmas[i], sigmas, HEADS, SHIFT_V)+ with torch.no_grad():+ ours = bank(h)+ comfy = comfy_pdd_head(bank.weight, bank.bias, h, HEADS, start, stop, stream_shift)+ oracle = oracle_head(bank.weight, bank.bias, h, HEADS, start, stop, stream_shift)+ assert (ours.double() - oracle).abs().max() < 1e-6+ assert (comfy.double() - oracle).abs().max() < 1e-6+ assert (ours - comfy).abs().max() < 1e-6+++def test_sigma_pair_comes_off_the_device_schedule_like_comfyui():+ sigmas = h3_sigmas(8)+ timestep = (1.0 - sigmas[:-1]).float()+ for i in range(8):+ # three conditioning video rows pinned near clean, then the generated rows at this step+ uniq = torch.unique(torch.tensor([float(timestep[i]), 0.999, 1.0]))+ rows_t = torch.tensor([0.999] * 3 + [float(timestep[i])] * 5 + [1.0] * 2)+ indices = torch.searchsorted(uniq, rows_t)+ pair = pdd.h3_pdd_sigma_pair(uniq, indices, torch.arange(8), sigmas)+ assert pair.dtype == torch.float64+ assert abs(float(pair[0]) - float(sigmas[i])) < 1e-6+ assert float(pair[1]) == float(sigmas[i + 1])+ assert float(pdd.h3_pdd_sigma_pair(uniq, indices, torch.arange(8), sigmas)[1]) == 0.0+++# ── the real denoiser: tiny random MiniMaxH3Transformer3DModel ──++TINY = dict(+ num_attention_heads = 2,+ attention_head_dim = 16,+ hidden_size = 32,+ num_layers = 2,+ num_refiner_layers = 1,+ ffn_dim = 64,+ in_channels = 24,+ audio_in_channels = 32,+ text_dim = 16,+ freq_dim = 32,+ time_embed_hidden_dim = 32,+ time_embed_dim = 16,+ rope_freq_dim = 2,+)+++def _diffusers():+ return pytest.importorskip("diffusers")+++def _tiny(+ heads = HEADS,+ seed = 0,+ device = "cpu",+):+ diffusers = _diffusers()+ torch.manual_seed(seed)+ model = diffusers.MiniMaxH3Transformer3DModel(**TINY)+ if heads > 1:+ dense = {+ "proj_out": model.proj_out,+ "audio_proj_out": model.audio_proj_out,+ }+ assert pdd.install_h3_pdd_heads(model, heads)+ with torch.no_grad():+ for name, lin in dense.items():+ bank = getattr(model, name)+ bank.weight.copy_(torch.randn_like(bank.weight) * 0.05)+ bank.bias.copy_(torch.randn_like(bank.bias) * 0.05)+ bank.weight[: lin.out_features].copy_(lin.weight)+ bank.bias[: lin.out_features].copy_(lin.bias)+ return model.to(device).eval()+++class _Sched:+ def __init__(self, sigmas):+ self.sigmas = sigmas+++def _inputs(+ device,+ t_video,+ n_text = 4,+ n_cond = 2,+ n_video = 6,+ n_audio = 3,+ seed = 0,+):+ g = torch.Generator().manual_seed(seed)+ s = n_text + n_cond + n_video + n_audio+ text = torch.arange(n_text)+ video = torch.arange(n_text, n_text + n_cond + n_video)+ audio = torch.arange(n_text + n_cond + n_video, s)+ tags = torch.zeros(s, dtype = torch.long)+ tags[text], tags[audio] = 1, 2+ row_t = torch.full((s,), float(t_video))+ row_t[video[:n_cond]] = max(float(t_video), 0.999)+ row_t[audio] = 1.0 - pdd_shift_sigma(1.0 - float(t_video))+ uniq, inv = torch.unique(row_t, sorted = True, return_inverse = True)+ pos = torch.stack(+ [torch.arange(s), torch.zeros(s, dtype = torch.long), torch.arange(s) % 3], 1+ ).float()+ return dict(+ hidden_states = torch.randn(1, n_cond + n_video, 96, generator = g).to(device),+ audio_hidden_states = torch.randn(1, n_audio, 32, generator = g).to(device),+ encoder_hidden_states = torch.randn(1, n_text, 16, generator = g).to(device),+ timestep = uniq.to(device),+ timestep_indices = inv.to(device),+ token_tags = tags.to(device),+ position_ids = pos.to(device),+ video_indices = video.to(device),+ audio_indices = audio.to(device),+ text_indices = text.to(device),+ return_dict = False,+ )+++def pdd_shift_sigma(sigma_v):+ return float(comfy_time_shift_sigma(torch.tensor(sigma_v), SHIFT_V, SHIFT_A))+++def _run_and_capture(model, kw):+ seen = {}++ def grab(name):+ def hook(_m, args, out):+ seen[name] = (args[0].detach().clone(), out.detach().clone())++ return hook++ handles = [+ model.proj_out.register_forward_hook(grab("video")),+ model.audio_proj_out.register_forward_hook(grab("audio")),+ ]+ try:+ with torch.no_grad():+ out = model(**kw)+ finally:+ for handle in handles:+ handle.remove()+ return out, seen+++def _check_step(model, seen, sigmas, i):+ start, stop = comfy_span(sigmas[i], sigmas, HEADS, SHIFT_V)+ worst = 0.0+ for name, bank, shift in (+ ("video", model.proj_out, SHIFT_V),+ ("audio", model.audio_proj_out, SHIFT_A),+ ):+ h, out = seen[name]+ w, b = bank.weight.detach().cpu(), bank.bias.detach().cpu()+ oracle = oracle_head(w, b, h.cpu(), HEADS, start, stop, shift)+ comfy = comfy_pdd_head(w, b, h.cpu().float(), HEADS, start, stop, shift)+ worst = max(worst, float((out.cpu().double() - oracle).abs().max()))+ assert (out.cpu().double() - oracle).abs().max() < 1e-5, (name, i)+ assert (out.cpu() - comfy).abs().max() < 1e-5, (name, i)+ return worst+++def _bound(model, steps = 8):+ sigmas = h3_sigmas(steps)+ pdd.bind_h3_pdd_schedule(model, _Sched(sigmas), SHIFT_V, SHIFT_A)+ return sigmas+++def test_the_denoiser_blends_the_heads_comfyui_picks_at_every_step():+ model = _tiny()+ sigmas = _bound(model)+ for i in range(8):+ kw = _inputs("cpu", 1.0 - float(sigmas[i]), seed = i)+ _out, seen = _run_and_capture(model, kw)+ state = model.proj_out.pdd_state+ assert abs(float(state[0]) - float(sigmas[i])) < 1e-6 and float(state[1]) == float(+ sigmas[i + 1]+ )+ _check_step(model, seen, sigmas, i)+++def test_an_unbound_pdd_denoiser_refuses_to_run():+ model = _tiny()+ with pytest.raises(ValueError, match = "sigma schedule"):+ with torch.no_grad():+ model(**_inputs("cpu", 0.5))+++def test_a_plain_denoiser_is_untouched():+ model = _tiny(heads = 1)+ assert isinstance(model.proj_out, torch.nn.Linear) and pdd.h3_pdd_heads(model) == 0+ assert not pdd.bind_h3_pdd_schedule(model, _Sched(h3_sigmas(8)), SHIFT_V, SHIFT_A)+++class _Pipe:+ def __init__(self, transformer, scheduler):+ self.transformer = transformer+ self.scheduler = scheduler+++def test_the_render_binding_finds_the_partition_and_sizes_the_grid():+ model = _tiny()+ pipe = _Pipe(model, _Sched(h3_sigmas(8)))+ assert pdd.pipe_h3_pdd_heads(pipe) == HEADS+ assert pdd.bind_h3_pdd_render(pipe, SHIFT_V, SHIFT_A) == HEADS+ assert model.audio_proj_out.pdd_state[2:].tolist() == [SHIFT_V, SHIFT_A]+ assert pdd.h3_pdd_num_inference_steps(8) == 9+ assert pdd.bind_h3_pdd_render(_Pipe(_tiny(heads = 1), _Sched(None)), SHIFT_V, SHIFT_A) == 0+++# ── the real loader: a ComfyUI-layout file built from the tiny model ──+++def _comfy_key(name):+ from core.inference.video_minimax_h3_comfy import _RENAMES++ if name.startswith("transformer_blocks."):+ name = "blocks." + name[len("transformer_blocks.") :]+ elif name.startswith("token_refiner.refiner_blocks."):+ name = "token_refiner.blocks." + name[len("token_refiner.refiner_blocks.") :]+ for old, new in reversed(_RENAMES):+ if new in name and (not new.startswith(("proj_", "audio_proj")) or name.startswith(new)):+ name = name.replace(new, old, 1)+ break+ return name+++def _write_comfy_file(+ tmp_path,+ model,+ name,+ metadata = None,+ int8_layer = True,+):+ sd = {k: v.detach().clone() for k, v in model.state_dict().items()}+ out = {}+ for key in list(sd):+ if ".attn.to_q.weight" in key:+ stem = key[: -len("to_q.weight")]+ fused = torch.cat([sd.pop(stem + p + ".weight") for p in ("to_q", "to_k", "to_v")])+ out[_comfy_key(stem).replace("transformer_blocks.", "blocks.") + "qkv_proj.weight"] = (+ fused+ )+ for key, value in sd.items():+ if key.endswith(".ff.net.0.proj.weight"):+ value_half, gate = value.chunk(2)+ out[_comfy_key(key.replace(".ff.net.0.proj.weight", ".mlp.fc1.weight"))] = torch.cat(+ [gate, value_half]+ )+ continue+ out[_comfy_key(key)] = value+ if int8_layer:+ key = "blocks.0.attn.out_proj.weight"+ w = out[key].float()+ scale = w.abs().amax().clamp_min(1e-8) / 127.0+ out[key] = torch.round(w / scale).clamp(-127, 127).to(torch.int8)+ out[key.replace(".weight", ".weight_scale")] = scale.reshape(())+ out["blocks.0.attn.out_proj.comfy_quant"] = torch.tensor(+ list(json.dumps({"format": "int8_tensorwise"}).encode()), dtype = torch.uint8+ )+ path = tmp_path / name+ safetensors_torch.save_file(+ {k: v.contiguous() for k, v in out.items()}, str(path), metadata = metadata+ )+ cfg = tmp_path / "base" / "transformer"+ cfg.mkdir(parents = True, exist_ok = True)+ (cfg / "config.json").write_text(+ json.dumps({"_class_name": "MiniMaxH3Transformer3DModel", **TINY})+ )+ return str(path), str(tmp_path / "base")+++def _load(path, base):+ from core.inference import diffusion_comfy_quant as cq+ from core.inference.video_minimax_h3_comfy import load_h3_comfy_transformer+ return load_h3_comfy_transformer(+ _diffusers().MiniMaxH3Transformer3DModel,+ path,+ cq.refuse_comfy_quant(path),+ config = base,+ subfolder = "transformer",+ dtype = torch.float32,+ hf_token = None,+ cache_dir = None,+ local_files_only = True,+ int8_backend = None,+ fp8_backend = None,+ )+++def test_a_with_heads_file_loads_through_the_comfy_loader_with_its_bank(tmp_path):+ src = _tiny()+ meta = {"config": json.dumps({"transformer": {"pdd_steps": HEADS}}), "pdd_acc_baked": "true"}+ path, base = _write_comfy_file(tmp_path, src, WITH, metadata = meta)+ model = _load(path, base).eval()+ assert pdd.h3_pdd_heads(model) == HEADS+ assert tuple(model.proj_out.weight.shape) == (HEADS * VIDEO_DIM, TINY["hidden_size"])+ assert model.proj_out.weight.dtype == torch.float32+ assert torch.equal(model.proj_out.weight, src.proj_out.weight)+ assert torch.equal(model.audio_proj_out.bias, src.audio_proj_out.bias)+ sigmas = _bound(model)+ for i in (0, 7):+ _out, seen = _run_and_capture(model, _inputs("cpu", 1.0 - float(sigmas[i])))+ _check_step(model, seen, sigmas, i)+++def test_a_without_heads_bake_is_refused_by_the_comfy_loader(tmp_path):+ src = _tiny(heads = 1)+ for name, meta in (+ (WITHOUT_OLD, {"pdd_acc_baked": "true"}),+ (WITHOUT, None),+ ):+ path, base = _write_comfy_file(tmp_path, src, name, metadata = meta)+ with pytest.raises(ValueError, match = "no head bank"):+ _load(path, base)+ plain, base = _write_comfy_file(tmp_path, src, "minimax_h3_fl2va_int8_convrot.safetensors")+ assert isinstance(_load(plain, base).proj_out, torch.nn.Linear)+++# ── torch.compile and CUDA graphs ──+++def test_the_bank_compiles_with_no_graph_break_and_no_recompile_across_steps():+ import torch._dynamo as dynamo++ dynamo.reset()+ bank = _bank()+ h = torch.randn(5, 24)+ sigmas = h3_sigmas(8)+ explain = dynamo.explain(bank)(h)+ assert explain.graph_break_count == 0 and explain.graph_count == 1+ dynamo.reset()+ counts = dynamo.utils.counters+ counts.clear()+ compiled = torch.compile(bank, fullgraph = True, backend = "eager")+ with torch.no_grad():+ for i in range(8):+ bank.set_state(_state(sigmas[i], sigmas[i + 1])[:2])+ ref = comfy_pdd_head(+ bank.weight,+ bank.bias,+ h,+ HEADS,+ *comfy_span(sigmas[i], sigmas, HEADS, SHIFT_V),+ SHIFT_V,+ )+ assert (compiled(h) - ref).abs().max() < 1e-6, i+ assert counts["stats"]["unique_graphs"] == 1, dict(counts["stats"])+ assert not counts["recompiles"] if "recompiles" in counts else True+++def _cuda():+ if not torch.cuda.is_available():+ pytest.skip("needs CUDA")+++def test_the_bank_compiles_with_inductor_on_cuda_and_tracks_every_step():+ _cuda()+ import torch._dynamo as dynamo++ dynamo.reset()+ dynamo.utils.counters.clear()+ bank = _bank().cuda()+ h = torch.randn(5, 24, device = "cuda")+ sigmas = h3_sigmas(8)+ compiled = torch.compile(bank, fullgraph = True)+ with torch.no_grad():+ for i in range(8):+ bank.set_state(_state(sigmas[i], sigmas[i + 1], device = "cuda")[:2])+ ref = comfy_pdd_head(+ bank.weight.cpu(),+ bank.bias.cpu(),+ h.cpu(),+ HEADS,+ *comfy_span(sigmas[i], sigmas, HEADS, SHIFT_V),+ SHIFT_V,+ )+ assert (compiled(h).cpu() - ref).abs().max() < 1e-5, i+ assert dynamo.utils.counters["stats"]["unique_graphs"] == 1+++def test_a_cuda_graph_of_the_denoiser_replays_with_changing_sigmas(monkeypatch):+ _cuda()+ from core.inference import diffusion_cuda_graph as cg++ monkeypatch.setenv("UNSLOTH_DIFFUSION_CUDA_GRAPHS", "1")+ model = _tiny(device = "cuda")+ sigmas = _bound(model)+ eager = []+ with torch.no_grad():+ for i in range(8):+ eager.append(model(**_inputs("cuda", 1.0 - float(sigmas[i]), seed = 0)))+ handle = cg.GraphedForward(model).install().enable()+ try:+ replayed = []+ with torch.inference_mode():+ for _round in range(2):+ for i in range(8):+ out = model(**_inputs("cuda", 1.0 - float(sigmas[i]), seed = 0))+ replayed.append([o.clone() for o in out])+ stats = dict(handle.stats)+ finally:+ handle.uninstall()+ # one recording per distinct input shape (step 0's audio t equals its video t, so its timestep table is shorter);+ # every other call is a replay with that step's sigmas+ shapes = {tuple(_inputs("cuda", 1.0 - float(sigmas[i]))["timestep"].shape) for i in range(8)}+ assert stats["captures"] == len(shapes), stats+ assert stats["replays"] >= 16 - len(shapes) * 2, stats+ assert stats["fallbacks"] == 0 and stats["eager_calls"] == 0, stats+ for k, out in enumerate(replayed):+ ref = eager[k % 8]+ assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1]), k+ # the steps really differ: a graph that baked the first step's heads would repeat it+ assert not torch.equal(eager[0][0], eager[7][0])+++def test_the_denoiser_under_group_offload_matches_resident():+ _cuda()+ resident = _tiny(device = "cuda")+ sigmas = _bound(resident)+ ref = []+ with torch.no_grad():+ for i in range(8):+ ref.append(resident(**_inputs("cuda", 1.0 - float(sigmas[i]))))+ for kind in ("leaf_level", "block_level"):+ model = _tiny()+ _bound(model)+ # block_level as Studio's streamed H3 denoiser applies it (stream, record_stream, non_blocking)+ kwargs = (+ {"num_blocks_per_group": 1, "record_stream": True, "non_blocking": True}+ if kind == "block_level"+ else {}+ )+ model.enable_group_offload(+ torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = kind,+ use_stream = True,+ **kwargs,+ )+ with torch.no_grad():+ for i in range(8):+ out = model(**_inputs("cuda", 1.0 - float(sigmas[i])))+ assert (out[0] - ref[i][0]).abs().max() < 1e-5, (kind, i)+ assert (out[1] - ref[i][1]).abs().max() < 1e-5, (kind, i)+++def test_the_whole_pdd_denoiser_forward_compiles_fullgraph_without_recompiles():+ """The pre-hook runs eager in ``Module.__call__`` (Studio compiles the forward / its repeated blocks, never the+ call), so the compiled forward reads the step's sigmas as a tensor: one graph for every step."""+ import torch._dynamo as dynamo++ model = _tiny()+ eager = _tiny()+ sigmas = _bound(model)+ _bound(eager)+ kw = _inputs("cpu", 1.0 - float(sigmas[3]))+ pdd._pre_hook(model, (), kw)+ dynamo.reset()+ explain = dynamo.explain(model.forward)(**kw)+ assert explain.graph_break_count == 0 and explain.graph_count == 1+ dynamo.reset()+ dynamo.utils.counters.clear()+ model.__dict__["forward"] = torch.compile(model.forward, fullgraph = True, backend = "eager")+ with torch.no_grad():+ # steps 1..7 share one input shape (step 0's timestep table is shorter, a different shape by design)+ for i in range(1, 8):+ kw = _inputs("cpu", 1.0 - float(sigmas[i]), seed = i)+ out, ref = model(**kw), eager(**kw)+ assert torch.equal(out[0], ref[0]) and torch.equal(out[1], ref[1]), i+ assert float(model.proj_out.pdd_state[1]) == float(sigmas[i + 1]), i+ assert dynamo.utils.counters["stats"]["unique_graphs"] == 1, dict(+ dynamo.utils.counters["stats"]+ )+++def test_the_denoiser_under_model_cpu_offload_matches_resident():+ _cuda()+ accelerate = pytest.importorskip("accelerate")+ resident = _tiny(device = "cuda")+ sigmas = _bound(resident)+ model = _tiny()+ _bound(model)+ model, hook = accelerate.cpu_offload_with_hook(model, execution_device = torch.device("cuda"))+ with torch.no_grad():+ for i in range(8):+ kw = _inputs("cuda", 1.0 - float(sigmas[i]))+ out, ref = model(**kw), resident(**kw)+ assert (out[0] - ref[0]).abs().max() < 1e-5 and (out[1] - ref[1]).abs().max() < 1e-5, i+ hook.offload()+ assert model.proj_out.weight.device.type == "cpu"+++def test_a_bound_render_reads_the_schedule_the_pipeline_sets_afterwards():+ """Studio binds the pipeline's scheduler before the call; the modular loop's set_timesteps runs inside it."""+ diffusers = _diffusers()+ model = _tiny()+ scheduler = diffusers.MiniMaxH3Scheduler(shift = 1.0)+ pipe = _Pipe(model, scheduler)+ scheduler.set_shift(SHIFT_V)+ assert pdd.bind_h3_pdd_render(pipe, SHIFT_V, SHIFT_A) == HEADS+ scheduler.set_timesteps(pdd.h3_pdd_num_inference_steps(pdd.H3_PDD_DEFAULT_STEPS))+ spans = []+ latents = torch.randn(8, 96)+ for i, t in enumerate(scheduler.timesteps):+ kw = _inputs("cpu", float(t), seed = i)+ out, seen = _run_and_capture(model, kw)+ _check_step(model, seen, scheduler.sigmas, i)+ spans.append(comfy_span(scheduler.sigmas[i], scheduler.sigmas, HEADS, SHIFT_V))+ latents = scheduler.step(out[0][0].float(), t, latents, return_dict = False)[0]+ assert spans == [(4 * i, 4 * i + 4) for i in range(8)]+++def test_a_second_render_can_rebind_a_bank_its_first_render_moved_in_inference_mode():+ """The pipeline runs under inference_mode, so the first render's device move made pdd_state an inference tensor+ and the second render's set_shifts (outside it) raised. A pdd_state left as an inference tensor is replaced too."""+ bank = _bank()+ with torch.inference_mode():+ bank.pdd_state = bank.pdd_state.clone()+ assert bank.pdd_state.is_inference()+ bank.set_shifts(SHIFT_V, SHIFT_A)+ assert not bank.pdd_state.is_inference()+ assert bank.pdd_state.tolist()[2:] == [SHIFT_V, SHIFT_A]+ with torch.inference_mode():+ bank.set_state(_state(0.7, 0.6)[:2])+ assert not bank.pdd_state.is_inference()+ assert bank.pdd_state.tolist()[:2] == pytest.approx([0.7, 0.6])+ if torch.cuda.is_available():+ bank = _bank()+ with torch.inference_mode():+ bank.set_state(_state(0.9, 0.8, device = "cuda")[:2])+ assert not bank.pdd_state.is_inference() and bank.pdd_state.is_cuda+ bank.set_shifts(SHIFT_V, SHIFT_A)+++def test_pdd_defaults_apply_only_to_minimax_h3():+ """A Wan or Hunyuan checkpoint under a path naming "pdd" keeps its family defaults."""+ from core.inference.video_families import default_video_generation_params++ h3 = "/models/minimax_h3_ref2va_pruned_pdd_acc_8step_baked_with_heads_int8_convrot.safetensors"+ assert default_video_generation_params(h3, fallback = (30, 5.0), family = "minimax-h3") == (8, 1.0)+ wan = "/models/pdd/wan2.2-ti2v-5b.safetensors"+ assert default_video_generation_params(wan, fallback = (50, 5.0), family = "wan2.2") == (20, 5.0)+ other = "/models/pdd/hunyuan.safetensors"+ assert default_video_generation_params(+ other, fallback = (50, 6.0), family = "hunyuanvideo-1.5"+ ) == (50, 6.0)+ assert default_video_generation_params(other, fallback = (50, 6.0)) == (50, 6.0)+ # A stock H3 checkpoint in a folder named pdd keeps the stock recipe: only the file name decides.+ stock = "/models/pdd/minimax_h3_fl2va_int8_convrot.safetensors"+ assert default_video_generation_params(stock, fallback = (30, 1.0), family = "minimax-h3") == (+ 30,+ 1.0,+ )