Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Local text-to-video inference backend (diffusers).
A deliberate sibling of ``DiffusionBackend`` rather than a mode of it: video
pipelines take frame/fps arguments, return frame stacks (plus synchronized audio
for LTX-2) instead of PIL images, and persist MP4s -- none of the image module's
img2img/inpaint/ControlNet/LoRA surface applies. The concurrency skeleton
(load token, per-generation cancel event, split status/generate locks) is copied
from the image backend so the two cannot diverge in lifecycle behaviour, and the
hardware/optimisation layers are IMPORTED from the image stack unchanged:
device/dtype resolution, memory planning + offload tiers, attention backends,
speed profiles (regional torch.compile), and FBCache step caching all operate on
``pipe.transformer`` generically.
Video-specific behaviour lives here:
- the runtime headroom estimate is frames-aware (``estimate_video_runtime_mib``):
the VAE decode of a whole clip is the memory peak, not the denoise;
- VAE tiling is always enabled (decode of 100+ frames at 720p-class resolutions
spikes far beyond the image case, and tiling's quality cost is negligible);
- generation snaps num_frames to the family's temporal lattice (k * step + 1)
and width/height to its required multiple BEFORE latents are allocated;
- the result is encoded to MP4 (H.264) via diffusers' PyAV-backed exporter,
muxing the audio track for families that produce one.
Loads are gated to trusted repos exactly like the image backend: unsloth/*, the
family's official base repos, or a local path the user explicitly picked.
"""
from __future__ import annotations
import contextlib
import dataclasses
import functools
import inspect
import os
import re
import tempfile
import threading
from utils.account_context import account_thread, current_account_id
import time
import types
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Callable, NamedTuple, Optional, Sequence
from hub.utils.hf_errors import modelscope_missing
from loggers import get_logger
from utils.gpu_memory_events import invalidates_gpu_memory as _invalidates_gpu_memory
from .diffusion_attention import (
SDPA_MATH_ONLY_MESSAGE,
apply_attention_backend,
attention_status_reason,
install_hunyuan_attention_trim,
sdpa_math_only,
_dit_head_dims,
begin_attention_run,
report_attention_run,
select_attention_backend,
)
from .diffusion_flow_shift import apply_comfy_flow_shift, apply_scheduler_overrides
from .diffusion_init_buffers import install_single_file_fp32_buffers
from .diffusion_component_sizes import video_family_has_no_text_encoder
from . import hub_reachability
from .diffusion_comfy_quant import (
comfy_fp8_backend,
comfy_int8_backend,
fp8_storage_wanted,
fp8_widen_chosen,
pipe_fp8_stored_layers,
comfy_resident_mib,
load_comfy_quant_transformer,
refuse_comfy_quant,
)
from .diffusion_single_file_trust import assert_safetensors_file, single_file_load_allowed
from .diffusers_safetensors import component_safetensors_kwargs, pipeline_safetensors_kwargs
from .diffusion_prequant import scoped_local_files_only
from .video_moe_pair import moe_expert_of, moe_partner_filename, moe_pick_pairs
from .diffusion_cache import (
FBCACHE_MIN_STEPS,
TC_AUTO,
TC_FBCACHE,
TC_STATIC,
apply_step_cache,
auto_static_skip_entry,
auto_static_skip_plan,
auto_step_cache_allowed,
skip_tier,
cache_breaks_graph,
maybe_toggle_step_cache,
normalize_transformer_cache,
resolve_auto_step_cache,
step_cache_supported,
)
from .diffusion_step_skip import (
auto_static_settings,
install_static_step_skip,
mark_step_end,
reset_static_step_skip,
static_skip_is_auto,
static_skip_stats,
static_skip_view,
uninstall_static_step_skip,
)
from .diffusion_device import (
apply_cpu_render_threads,
DiffusionDeviceTarget,
apply_diffusion_device_ordinal,
diffusion_device_scope,
force_float32_rope,
install_rocm_vae_bf16_decode,
install_vae_sync,
pin_cuda_ordinal,
placed_cuda_ordinal,
auto_cuda_ordinal_for_load,
resolve_diffusion_device_target,
resolve_selected_cuda_ordinal,
resolve_selected_cuda_ordinals,
)
from .diffusion_cfg_batch import cfg_batch_status, end_cfg_batch_render, install_cfg_batch
from .diffusion_multigpu import (
AUTOMATIC_ORDINALS,
end_cfg_render,
install_cfg_parallel,
multigpu_mode,
is_split as multigpu_is_split,
placement_status as multigpu_placement_status,
refine_plan_for_multigpu,
set_cfg_parallel_fallback,
uninstall_cfg_parallel,
with_load_selection,
)
from .diffusion_memory import (
DeviceMemory,
commit_load_budget_fraction,
release_load_budget_fraction,
vram_budget_scope,
carry_load_budget,
DEFAULT_BASE_OVERHEAD_MIB,
DEFAULT_GROUP_BLOCKS,
MEMORY_MODE_AUTO,
MEMORY_MODE_FAST,
OFFLOAD_GROUP,
OFFLOAD_MODEL,
OFFLOAD_NONE,
OFFLOAD_STREAMING,
apply_memory_plan,
estimate_gguf_resident_mib,
estimate_safetensors_dense_mib,
estimate_video_runtime_mib,
file_size_mib,
mps_plan_tight,
normalize_memory_mode,
plan_diffusion_memory,
plan_keeps_transformer_resident,
prequant_seed_device,
raise_on_unified_memory_shortfall,
scoped_allow_oversized_load,
unified_memory_refusal_code,
reclaim_host_memory,
reclaim_offload_host_memory,
refine_memory_plan_for_components,
applied_plan,
defer_late_expert_pins,
finish_background_pins,
release_pinned_host_memory,
request_background_pins,
request_fast_pins,
safetensors_prefix_mib,
settled_snapshot_device_memory,
start_background_pins,
stop_background_pins,
)
from .diffusion_torchao_patches import install_torchao_int_mm_patch
from .media_decode_phase import decode_phase as _decode_phase
from .media_decode_phase import denoise_phase as _denoise_phase
from . import diffusion_bg_compile
from . import diffusion_render_thread as render_thread
from .diffusion_fp16_guard import fp16_promotes_to_fp32
from .diffusion_speed import (
SPEED_DEFAULT,
SPEED_EAGER,
SPEED_MAX,
SPEED_OFF,
apply_speed_optims,
arm_graphs_after_placement,
engage_pinned_denoisers,
rocm_probe_load_scoped,
release_rocm_tunableop as _release_rocm_tunableop,
resolve_speed_mode,
restore_backend_flags,
settle_compile_fallback,
snapshot_backend_flags,
)
from .diffusion_auto_policy import (
_QUANT_STEADY_FACTOR,
RESOLVED_APPLIED,
RESOLVED_FELL_BACK,
RESOLVED_UNSUPPORTED,
build_resolved_record,
precision_fallback_allowed,
precision_refusal_message,
)
from .diffusion_nvfp4_flag import nvfp4_diffusion_enabled
from .diffusion_nvfp4_route import with_route_scope
from .diffusion_nvfp4_install import nvfp4_backend_fields as _nvfp4_backend_fields
from .diffusion_transformer_quant import (
TQ_AUTO,
TQ_FP8,
TQ_INT8,
TQ_NVFP4,
dense_transformer_supported,
dense_transformer_unsupported_reason,
explain_unusable_scheme,
native_int8_act,
native_offload_host,
native_quant_host,
native_quant_scheme,
NATIVE_OFFLOAD_SCHEMES,
NATIVE_QUANT_SCHEMES,
normalize_transformer_quant,
quantize_transformer,
mark_source_precision,
explicit_scheme_cached_ok,
select_transformer_quant_scheme,
stored_denoiser_precision,
transformer_is_quantised,
)
from .diffusion import (
_comfy_keep_fp32_pricing,
_family_override_resolved,
_companion_split_expected,
_layer_split_expected,
_memory_request_forces_offload,
_native_quant_needs_compile,
_restore_step_cache_after_layer_split,
_step_cache_dropped_by_layer_split,
)
from .diffusion_native_quant import (
native_quant_on_device_enabled,
native_quant_reason,
quantise_host_linears_on,
)
from .diffusion_batched import is_oom_error
from .diffusion_block_graph import compile_pipe_below_offload_hooks
from .diffusion_offload_prefetch import (
cross_forward_render,
cross_forward_step,
end_cross_forward,
)
from .diffusion_precision import (
effective_te_quant,
normalize_te_quant,
quantize_text_encoders,
resolve_te_quant_request,
te_quant_needs_resident_weights,
te_quant_supported,
te_quant_unsupported_reason,
torchao_quantize_importable,
)
from .video_families import (
VIDEO_CANCELLED_MSG,
VIDEO_GENERATION_BUSY_MSG,
VIDEO_MODEL_CHANGED_MSG,
VIDEO_NOT_LOADED_MSG,
COSMOS3_FAMILIES,
LTX_FAMILIES,
assert_local_pipeline_family,
assert_ltx_pick_supported,
assert_supported_ltx_version,
ltx_family_by_name,
local_cosmos3_pipeline_family,
local_ltx_pipeline_family,
pipeline_available_video_families,
COSMOS3_CONTINUATION_FRAMES,
DEFAULT_VIDEO_CONTINUATION_KEEP,
VideoFamily,
default_video_generation_params,
detect_video_family,
resolve_video_base_repo,
snap_num_frames,
snap_video_size,
supported_video_family_names,
validate_video_continuation,
validate_video_flow_controls,
validate_video_keyframe_conditioning,
validate_video_reference_conditioning,
validate_video_fps,
validate_video_request_shape,
video_custom_size_bounds,
video_family_prequant_available,
video_family_prequant_repo,
video_family_prequant_resident_gb,
video_family_prequant_schemes,
first_frame_canvas,
video_checkpoint_refusal,
video_family_accepts_first_frame,
video_family_accepts_last_frame,
video_family_uses_i2v_pipeline,
video_i2v_call_kwargs,
)
from .video_frames import (
NON_FINITE_FRAMES_MSG,
NonFiniteFramesError,
finite_video_decode,
nonfinite_check_enabled,
require_finite_frames,
uint8_video_frames,
)
from .video_minimax_h3 import (
H3_ANCHOR_FIRST,
H3_ANCHOR_LAST,
H3_CANVAS_MAX_PIXELS,
H3_CANVAS_SHORT_EDGE,
H3_REF_IMAGE_SOURCE_MAX_PIXELS,
H3_REF_IMAGE_SOURCE_MAX_SIDE,
H3_REF_SIZE_MATCH,
H3_TASK_REFERENCES,
fit_h3_keyframe,
fit_h3_reference_image,
h3_canvas_for_aspect,
h3_conditioning_mode,
h3_denoiser_component,
h3_transformer_resident_gb,
h3_transformer_task,
)
from .video_minimax_h3_te import (
h3_te_auto_scheme,
h3_te_quant_scheme,
h3_te_quant_source,
h3_te_resident_gb,
)
from .video_encode import encode_x264
from .video_nvenc import encode_nvenc, nvenc_gpu
from .video_stream_encode import (
claim_stream,
offer_stream,
open_h3_stream_encode,
open_stream_encode,
)
from utils.hardware import clear_gpu_cache
from core.inference.inference_gate import holds_inference, pauses_background_embedding
# Shared with the image backend so both pin every loader call to the same live cache root
from core.inference.diffusion import hub_cache_dir
logger = get_logger(__name__)
# Asked for here as well as in the image backend, so the video path never depends on that import order.
install_torchao_int_mm_patch()
# Kinds: gguf (GGUF DiT + base repo), single_file (safetensors DiT), pipeline (full diffusers repo).
_MODEL_KINDS = frozenset({"gguf", "single_file", "pipeline"})
# Official non-unsloth bases allowed to load as full (non-GGUF) artifacts: safetensors-only, no remote code, exact
# lowercased match. Derived from the registry rows (base_repo, variant / training bases, FamilyProfile.trusted_repos).
from .video_families import _FAMILIES as _VIDEO_REGISTRY # noqa: E402
from .family_profile import trusted_repo_ids as _trusted_repo_ids # noqa: E402
_TRUSTED_NON_GGUF_VIDEO_REPOS = _trusted_repo_ids(_VIDEO_REGISTRY)
def _gguf_quant_token(filename: Optional[str]) -> Optional[str]:
"""The quant a GGUF filename names (``Q4_K_M``, ``BF16``), as the status reports it in ``gguf_variant``."""
if not filename:
return None
from hub.utils.gguf import extract_quant_token
return extract_quant_token(filename)
def resolve_video_model_kind(gguf_filename: Optional[str], model_kind: Optional[str]) -> str:
"""Classify a load request; explicit model_kind wins, else the filename decides."""
if model_kind:
kind = model_kind.strip().lower()
if kind not in _MODEL_KINDS:
raise ValueError(
f"Unknown model_kind '{model_kind}'. Expected one of {sorted(_MODEL_KINDS)}."
)
return kind
if not gguf_filename:
return "pipeline"
return "gguf" if gguf_filename.strip().lower().endswith(".gguf") else "single_file"
def auto_te_fetch_failure(
fam: Any, scheme: Optional[str], failures: Optional[dict]
) -> Optional[str]:
"""Why an automatic text-encoder ``scheme`` cannot engage: its family's hosted pre-cast encoder for that scheme
could not be fetched (``failures`` from ``_fetch_te_prequant``). None when it was fetched or never asked for."""
if not scheme or not failures:
return None
for entry in getattr(fam, "te_prequant_repos", None) or ():
try:
entry_scheme, component = entry[0], entry[1]
except Exception: # noqa: BLE001
continue
if str(entry_scheme).lower() == str(scheme).lower() and component in failures:
return str(failures[component])
return None
def auto_te_outcome_log(scheme: str, family: str, failure: Optional[str], injected: bool) -> tuple:
if failure and not injected:
return (
"video.text_encoder_quant: the automatic '%s' for %s is not available (hosted pre-cast encoder "
"unreachable: %s); the dense bf16 text encoder loads",
scheme,
family,
failure,
)
return (
"video.text_encoder_quant: '%s' selected automatically for %s (unset request)",
scheme,
family,
)
def video_te_quant_request(
fam: Any,
text_encoder_quant: Optional[str],
*,
base: Optional[str] = None,
speed_mode: Optional[str] = None,
) -> tuple[Optional[str], bool]:
"""``(request, auto_selected)``: an unset / "auto" encoder request on a family with ``te_quant_auto`` becomes that
scheme; anything else comes back as given, raw, because MiniMax-H3 and LTX-2.5 read the raw tri-state themselves.
``auto_selected`` keeps the pick from being treated as a request the caller made: nobody asked for it, so a host
that cannot run it (fp8 below cc 8.9, a hosted file that is not there) gets the dense encoder, never a refusal.
Two limits on the automatic pick, never on an explicit scheme: Speed="off" is bit-exact, so it keeps the released
encoder (LTX-2.5's raw unset included, which otherwise takes the hosted int8 encoder); and only an exactly identified official
``base`` takes the hosted file, so a custom pack that merely shares the folder name keeps its own weights."""
unset = text_encoder_quant is None or str(text_encoder_quant).strip().lower() in ("", "auto")
auto_scheme = getattr(fam, "te_quant_auto", None)
speed_off = speed_mode is not None and str(speed_mode).strip().lower() == SPEED_OFF
if unset and speed_off and (auto_scheme or getattr(fam, "name", None) == "ltx-2.5"):
return "none", False
if not auto_scheme:
return text_encoder_quant, False
mode, auto = resolve_te_quant_request(text_encoder_quant, auto_scheme)
if not auto:
return text_encoder_quant, False
if base is not None and not _te_base_is_official(fam, base):
return None, False
return mode, True
def _te_base_is_official(fam: Any, base: str) -> bool:
"""The load's base is the family's registered base (or a verified equivalent), by exact id after mirror and HF
cache canonicalization: no final-folder-name match."""
from .diffusion_families import canonical_base
from .diffusion_te_prequant import _TE_EQUIVALENT_BASES
from .model_ids import hf_cache_repo_id
registered = canonical_base(str(getattr(fam, "base_repo", "") or "")).lower()
loading = canonical_base(hf_cache_repo_id(str(base)) or str(base)).lower()
if not registered or not loading:
return False
return loading == registered or any(
loading in group and registered in group for group in _TE_EQUIVALENT_BASES
)
def assert_video_precision_available(
fam: Any,
*,
model_kind: str,
transformer_quant: Optional[str] = None,
text_encoder_quant: Optional[str] = None,
memory_mode: Optional[str] = None,
gpu_ordinal: Optional[int] = None,
checkpoint_filename: Optional[str] = None,
checkpoint_repo: Optional[str] = None,
) -> None:
"""Must run in the route before the GPU is taken, because begin_load evicts chat first."""
from .diffusion_nvfp4_flag import refuse_disabled_nvfp4
refuse_disabled_nvfp4(
transformer_quant = transformer_quant, text_encoder_quant = text_encoder_quant
)
if precision_fallback_allowed():
return
pinned = normalize_transformer_quant(transformer_quant)
te_mode = normalize_te_quant(text_encoder_quant)
if pinned is None and te_mode is None:
return
# Scoped to the load's card: the probes use argument-less CUDA calls on a pooled thread.
with diffusion_device_scope(gpu_ordinal):
target = (
resolve_diffusion_device_target()
if gpu_ordinal is None
else resolve_diffusion_device_target(ordinal = gpu_ordinal)
)
_assert_video_precision_for_target(
fam,
target,
model_kind = model_kind,
transformer_quant = transformer_quant,
text_encoder_quant = text_encoder_quant,
memory_mode = memory_mode,
checkpoint_filename = checkpoint_filename,
checkpoint_repo = checkpoint_repo,
)
def _video_comfy_key_map(fam: Any) -> Any:
"""The original-layout key map for a family diffusers has no single-file converter for, else None."""
if getattr(fam, "transformer_class", None) == "HunyuanVideo15Transformer3DModel":
from .video_hv15_comfy import hv15_comfy_key_map
return hv15_comfy_key_map
return None
def _h3_pdd_default_steps(fam: Any, pipe: Any, default_steps: int) -> int:
"""ComfyUI's 8 steps for a loaded MiniMax-H3 PDD head bank whatever the file is called, else ``default_steps``."""
if getattr(fam, "name", None) != "minimax-h3" or pipe is None:
return default_steps
from .video_minimax_h3_pdd import H3_PDD_DEFAULT_STEPS, pipe_h3_pdd_heads
return H3_PDD_DEFAULT_STEPS if pipe_h3_pdd_heads(pipe) else default_steps
def _h3_single_file_task(
fam: Any, kind: str, gguf_filename: Optional[str], h3_task: Optional[str]
) -> Optional[str]:
"""The partition an H3 ComfyUI denoiser file serves (its name), unless the request set one."""
if not (getattr(fam, "modular_workflow", None) and kind == "single_file"):
return h3_task
from .video_minimax_h3_comfy import h3_comfy_task
return h3_task or h3_comfy_task(gguf_filename)
def _video_comfy_resident(plan: Any, bf16_plan: Any = None) -> bool:
"""A single-file load never re-plans for a runtime quant, so the DiT is resident only if every plan keeps it."""
return all(plan_keeps_transformer_resident(p) for p in (plan, bf16_plan) if p is not None)
def _video_comfy_backends(
fam: Any,
base: str,
target: Any,
plan: Any,
bf16_plan: Any = None,
*,
keep: bool = True,
priced_offload: bool = False,
widened_mib: Optional[int] = None,
) -> dict[str, Optional[str]]:
"""``load_comfy_quant_transformer`` backends for a ComfyUI-quantized video DiT: the ones Studio's own int8 / fp8
quant would run on this card and plan (int8: torchao resident, the torchao-free twin under offload; fp8: torchao
resident only; else fp8 storage under offload or a tight MPS plan). ``priced_offload``: the plan was sized with the
offloaded backends (``_video_offload_reprice``), so a load that now fits resident keeps them. ``widened_mib``: the
planned DiT price; a resident load stores fp8 when widening it would leave under the host reserve, as an image does.
``keep=False`` (the user asked for the bf16 DiT) dequantizes everything. nvfp4 / mxfp8 layers take the image
path's native runtimes where their flag, device and family allow; without one, fp8 storage keeps them packed."""
if not keep:
return {
"int8_backend": None,
"fp8_backend": None,
"nvfp4_backend": None,
"mxfp8_backend": None,
"fp8_storage": False,
"fp8_widen_chosen": False,
}
offload = priced_offload or not _video_comfy_resident(plan, bf16_plan)
name = getattr(fam, "name", None)
store_pays = offload or mps_plan_tight(plan)
storage = _video_fp8_storage(fam, target, store_pays, widened_mib = widened_mib)
return {
"int8_backend": comfy_int8_backend(target, name, base, offload = offload),
"fp8_backend": comfy_fp8_backend(target, name, base, offload = offload),
**_video_block_backends(target, name),
"fp8_storage": storage,
"fp8_widen_chosen": not storage
and _video_fp8_storage(fam, target, store_pays, widened_mib = widened_mib, widen_chosen = True),
}
def _video_block_backends(target: Any, family: Optional[str]) -> dict:
"""``comfy_block_backend`` per block format (None = no native runtime here)."""
from .diffusion_comfy_block import MXFP8, NVFP4, comfy_block_backend
dtype = getattr(target, "dtype", None)
return {
f"{fmt}_backend": comfy_block_backend(fmt, target, family, dtype = dtype)[0]
for fmt in (NVFP4, MXFP8)
}
def _video_comfy_reprice(plan: Any, bf16_plan: Any = None) -> bool:
"""Whether a ComfyUI DiT is re-priced with the offloaded backends: an offloaded plan (fp8 layers dequantize on
NVIDIA), or an MPS plan past its resident margin (MPS never offloads; its offloaded backends are its resident ones
plus fp8 storage)."""
return not _video_comfy_resident(plan, bf16_plan) or mps_plan_tight(plan)
def _video_offload_reprice(
fam: Any, transformer_mib: Optional[int], offload_mib: Optional[int]
) -> tuple[Optional[int], bool]:
"""``(transformer_mib, priced_offload)`` once an offloaded (or tight MPS) plan re-prices the DiT with the offloaded
backends. Larger: re-plan at it. Smaller (fp8 stored, the int8 twin where only it runs): re-plan at it too, and
the load must then keep those backends even if the new plan fits resident, else it widens past the size it was
planned at. H3 keeps its resident torchao path, so only a larger price moves it."""
if offload_mib is None or transformer_mib is None or offload_mib == transformer_mib:
return transformer_mib, False
if offload_mib < transformer_mib and getattr(fam, "name", None) == "minimax-h3":
return transformer_mib, False
return offload_mib, offload_mib < transformer_mib
def _video_fp8_storage(
fam: Any,
target: Any,
offload: bool,
*,
widened_mib: Optional[int] = None,
widen_chosen: bool = False,
) -> bool:
"""``fp8_storage_wanted`` for a video DiT, with the host RAM the load would widen into when ``widened_mib`` is
known. H3 stores fp8 only where no fp8 runtime runs (``h3_comfy_backends``): widened it would be the 40 GB
bfloat16 model. ``widen_chosen``: answer ``fp8_widen_chosen`` instead (the rule had the choice and widened)."""
if getattr(fam, "name", None) == "minimax-h3":
from .video_minimax_h3_native_quant import h3_fp8_storage
return False if widen_chosen else h3_fp8_storage(target)
available = None
if widened_mib is not None and not offload:
from .diffusion_small_host import host_ram_mib
available = host_ram_mib()[1]
rule = fp8_widen_chosen if widen_chosen else fp8_storage_wanted
return rule(target, offload = offload, widened_mib = widened_mib, host_available_mib = available)
def _video_comfy_resident_mib(
fam: Any,
base: str,
target: Any,
path: Any,
scan: Any,
*,
keep: bool = True,
keep_key: Any = None,
offload: bool = False,
) -> Optional[int]:
"""Planning size of a ComfyUI-quantized DiT, before the plan exists: int8 layers are priced as int8 when either
int8 runtime runs here, fp8 layers as fp8 when the resident fp8 path does, the rest at bf16 (a dequantized fp8
file is twice its size on disk). ``offload`` prices what an offloaded load keeps instead."""
name = getattr(fam, "name", None)
keep_int8 = keep_fp8 = fp8_stored = False
exclude: tuple = ()
filters: dict = {}
keep_block: dict = {}
if keep:
try:
from .diffusion_transformer_quant import (
DEFAULT_MIN_LINEAR_FEATURES,
TQ_MXFP8,
TQ_NVFP4,
divisible_for_scheme,
exclude_tokens_for_scheme,
)
exclude = exclude_tokens_for_scheme(TQ_INT8, name)
filters = {
"min_features": DEFAULT_MIN_LINEAR_FEATURES,
"fp8_divisible": divisible_for_scheme(TQ_FP8),
"block_divisible": {
"nvfp4": divisible_for_scheme(TQ_NVFP4),
"mxfp8": divisible_for_scheme(TQ_MXFP8),
},
}
except Exception: # noqa: BLE001 -- the estimate then keeps every int8 layer
exclude = ()
# Only a resident runtime keeps codes in a resident plan; an offloaded plan is re-priced.
keep_int8 = bool(comfy_int8_backend(target, name, base, offload = offload))
keep_fp8 = comfy_fp8_backend(target, name, base, offload = offload) is not None
fp8_stored = _video_fp8_storage(fam, target, offload)
blocks = _video_block_backends(target, name)
keep_block = {
"keep_nvfp4": blocks["nvfp4_backend"] is not None,
"keep_mxfp8": blocks["mxfp8_backend"] is not None,
}
return comfy_resident_mib(
str(path),
scan,
keep_int8 = keep_int8,
keep_fp8 = keep_fp8,
fp8_stored = fp8_stored,
**keep_block,
target = target,
keep_key = keep_key,
exclude_tokens = exclude,
key_map = _video_comfy_key_map(fam),
**filters,
**_comfy_keep_fp32_pricing(fam, target),
)
def _ltx23_prequant_pick(
fam: Any,
model_kind: str,
checkpoint_filename: Optional[str],
pinned: Optional[str],
checkpoint_repo: Optional[str] = None,
) -> bool:
"""An explicit fp8 on the OFFICIAL bf16 LTX-2.3 distilled single file (``checkpoint_repo`` is the pick's repo id or
local path): served by the hosted pre-quantized DiT. A same-named file of other provenance keeps its own DiT."""
if (
getattr(fam, "name", None) != "ltx-2"
or model_kind != "single_file"
or not checkpoint_filename
):
return False
if pinned != TQ_FP8:
return False
from .video_ltx2 import ltx23_prequant_eligible
return ltx23_prequant_eligible(checkpoint_filename, checkpoint_repo)
def _decodes_untiled_when_fits(family: str, offload_policy: str, device: str, speed: str) -> bool:
"""Untiled when it fits (free memory read at decode time); not under sequential offload, nor on SPEED_OFF
(bit-identical) except Cosmos3, whose reference pipeline decodes untiled (stock tiles seam at 480 / 832 px).
On a CPU host "fits" is usable host RAM: the stock tiles recompute their overlaps (Wan2.2 at 896x512: 15 tiles
covering 1.6x the frame's pixels), and the decode is the slowest stage there."""
if device not in ("cuda", "cpu") or offload_policy not in (
OFFLOAD_NONE,
OFFLOAD_MODEL,
OFFLOAD_GROUP,
OFFLOAD_STREAMING,
):
return False
return speed != SPEED_OFF or family in COSMOS3_FAMILIES
def _memory_mode_resolved(memory_mode: Any, plan: Any, offload_policy: str) -> tuple:
"""A ``fast`` that had to offload is a fallback valued at the engaged policy ("FAST -> GROUP", not "FAST -> FAST")."""
if plan.requested_mode == MEMORY_MODE_FAST and offload_policy != OFFLOAD_NONE:
return (memory_mode, offload_policy, "; ".join(plan.reasons), RESOLVED_FELL_BACK)
return (
memory_mode,
plan.requested_mode,
f"planned '{plan.offload_policy}' offload ({_video_plan_label(plan)}) from the family size table",
)
def _ltx23_prequant_serves(
fam: Any,
model_kind: str,
checkpoint_filename: Optional[str],
pinned: Optional[str],
*,
target: Any,
memory_mode: Optional[str],
checkpoint_repo: Optional[str] = None,
probe: bool = True,
) -> bool:
"""Skips ~19 GB hosted DiT on bf16-fallback loads (UNSLOTH_DIFFUSION_ALLOW_PRECISION_FALLBACK)."""
return (
_ltx23_prequant_pick(fam, model_kind, checkpoint_filename, pinned, checkpoint_repo)
and not _memory_request_forces_offload(memory_mode, False)
and bool(dense_transformer_supported(target))
and (
_ltx23_prequant_scheme_supported(fam, target, pinned)
if probe
# Training owns the GPU: cached verdict only, never the smoke probe.
else explicit_scheme_cached_ok(target, pinned, family = getattr(fam, "name", None))
)
)
def _ltx23_prequant_scheme_supported(fam: Any, target: Any, pinned: Optional[str]) -> bool:
"""A card without fp8 kernels (Ampere, int8 only) must not stage, price or seed the hosted fp8 DiT."""
try:
return (
select_transformer_quant_scheme(
target, pinned, family = getattr(fam, "name", None), unproven_ok = True
)
== pinned
)
except Exception: # noqa: BLE001 -- unanswerable reads as unsupported, which keeps the refusal
return False
def _ltx23_prequant_serves_on_card(
fam: Any,
model_kind: str,
checkpoint_filename: Optional[str],
transformer_quant: Optional[str],
*,
memory_mode: Optional[str],
gpu_ordinal: Optional[int],
checkpoint_repo: Optional[str] = None,
probe: bool = True,
hash_source: bool = True,
) -> bool:
"""``_ltx23_prequant_serves`` before the load, asked of the card it will use. An unanswerable probe keeps the pick.
``hash_source=False`` (planning) never hashes a local source file; see ``ltx23_identity_without_hashing``."""
if not hash_source:
from .video_ltx2 import ltx23_identity_without_hashing
with ltx23_identity_without_hashing():
return _ltx23_prequant_serves_on_card(
fam,
model_kind,
checkpoint_filename,
transformer_quant,
memory_mode = memory_mode,
gpu_ordinal = gpu_ordinal,
checkpoint_repo = checkpoint_repo,
probe = probe,
)
pinned = normalize_transformer_quant(transformer_quant)
if not _ltx23_prequant_pick(fam, model_kind, checkpoint_filename, pinned, checkpoint_repo):
return False
try:
# Scoped, not pinned: a pooled worker thread must not keep this request's card.
with diffusion_device_scope(gpu_ordinal):
target = (
resolve_diffusion_device_target()
if gpu_ordinal is None
else resolve_diffusion_device_target(ordinal = gpu_ordinal)
)
return _ltx23_prequant_serves(
fam,
model_kind,
checkpoint_filename,
pinned,
target = target,
memory_mode = memory_mode,
checkpoint_repo = checkpoint_repo,
probe = probe,
)
except Exception: # noqa: BLE001
return True
def _h3_hosted_fp8_native(fam: Any, target: Any, scheme: Optional[str]) -> bool:
"""An explicit fp8 H3 denoiser on a card without torchao's fp8 GEMM: the hosted codes run stored (1 byte)."""
if getattr(fam, "name", None) != "minimax-h3" or scheme != TQ_FP8:
return False
from .video_minimax_h3_native_quant import h3_hosted_fp8_native
return h3_hosted_fp8_native(target)
def _assert_video_precision_for_target(
fam: Any,
target: Any,
*,
model_kind: str,
transformer_quant: Optional[str] = None,
text_encoder_quant: Optional[str] = None,
memory_mode: Optional[str] = None,
checkpoint_filename: Optional[str] = None,
checkpoint_repo: Optional[str] = None,
) -> None:
"""The body of ``assert_video_precision_available``, run with the selected card current."""
pinned = normalize_transformer_quant(transformer_quant)
te_mode = normalize_te_quant(text_encoder_quant)
# Modular H3 uses ComponentsManager offload and pins its prequant denoiser, so balanced/low_vram load fine.
forces_offload = not getattr(fam, "modular_workflow", None) and _memory_request_forces_offload(
memory_mode, False
)
if pinned is not None and pinned != TQ_AUTO:
reason = None
if _ltx23_prequant_pick(fam, model_kind, checkpoint_filename, pinned, checkpoint_repo):
if not dense_transformer_supported(target):
reason = dense_transformer_unsupported_reason(target)
elif forces_offload:
reason = (
f"'{normalize_memory_mode(memory_mode)}' memory places the DiT under CPU "
"offload, and torchao quantised tensors cannot be moved by the offload hooks"
)
elif not _ltx23_prequant_scheme_supported(fam, target, pinned):
# Check the hosted DiT's scheme now, else it fails only after the eviction and the 19 GB pull.
reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
elif model_kind != "pipeline":
reason = (
f"the dense DiT quant applies to full-pipeline loads only, and this is a "
f"'{model_kind}' load, which runs the precision its checkpoint carries"
)
from .video_ltx2 import LTX23_PREQUANT_BASE, LTX23_PREQUANT_SOURCE_FILES
if (
pinned == TQ_FP8
and getattr(fam, "name", None) == "ltx-2"
and Path(str(checkpoint_filename or "")).name.lower() in LTX23_PREQUANT_SOURCE_FILES
):
reason += (
f"; the hosted fp8 DiT replaces only the official {LTX23_PREQUANT_BASE} file, and this "
"one could not be verified as it"
)
elif (
pinned in NATIVE_QUANT_SCHEMES
and not getattr(fam, "modular_workflow", None)
and native_quant_host(target)
):
# The modular workflow seeds torchao checkpoints, so it stays off the native path.
if native_quant_scheme(target, pinned, family = getattr(fam, "name", None)) is None:
reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
elif not dense_transformer_supported(target):
reason = dense_transformer_unsupported_reason(target)
elif forces_offload and pinned in NATIVE_OFFLOAD_SCHEMES and native_offload_host(target):
# forces_offload is already False for the modular workflow.
if (
native_quant_scheme(target, pinned, family = getattr(fam, "name", None), offload = True)
is None
):
reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
elif forces_offload:
# Offload hooks use Module.to(), which torchao tensors do not survive; refuse before eviction.
reason = (
f"'{normalize_memory_mode(memory_mode)}' memory places the DiT under CPU "
"offload, and torchao quantised tensors cannot be moved by the offload hooks"
)
elif select_transformer_quant_scheme(
target,
pinned,
family = getattr(fam, "name", None),
# Pre-eviction: an OOM in the smoke probe is the resident model, not the scheme.
unproven_ok = True,
) is None and not _h3_hosted_fp8_native(fam, target, pinned):
# None means a deny-listed family, unusable torchao, or no GPU kernels; each needs a different fix.
reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
if reason is not None:
raise RuntimeError(
precision_refusal_message(
"transformer_quant", pinned, reason, off_label = "Off to run the DiT at bf16"
)
)
# Gate the mode actually attempted, not the raw int8 request; a hosted conditioner skips this
# path.
if (
model_kind in ("pipeline", "single_file")
and getattr(fam, "modular_workflow", None)
and h3_te_quant_scheme(te_mode) is not None
):
return
if getattr(fam, "name", None) == "ltx-2.5":
return
# MPS runs a family's hosted int8 ConvRot encoder without fp8 (the loader's own rule), so an explicit int8 must not
# be rewritten to fp8 below and then refused for the fp8 support MPS lacks.
from .diffusion_te_prequant import (
explicit_int8_unavailable_reason,
family_te_int8_convrot,
int8_convrot_te_runs_on,
)
if (
te_mode == "int8"
and model_kind in ("pipeline", "single_file")
and family_te_int8_convrot(fam, "text_encoder", include_pending = True) is not None
and int8_convrot_te_runs_on(target)
):
# An unpublished file otherwise failed inside the load, after the eviction.
missing = explicit_int8_unavailable_reason(fam, getattr(fam, "base_repo", None), target)
if missing:
raise RuntimeError(
precision_refusal_message(
"text_encoder_quant",
te_mode,
f"{missing}, and this host runs int8 text encoders only from that file "
f"({te_quant_unsupported_reason('fp8')})",
off_label = "leave it unset to keep the dense bf16 encoder",
auto_available = False,
)
)
return
te_effective = effective_te_quant(te_mode, getattr(fam, "name", None))
te_reason = None
if te_quant_needs_resident_weights(te_effective) and not torchao_quantize_importable():
te_reason = (
"torchao is not importable on this install, and these encoder modes are torchao "
"quantisations"
)
elif te_effective is not None and not te_quant_supported(target, te_effective):
te_reason = te_quant_unsupported_reason(te_effective)
elif te_quant_needs_resident_weights(te_effective) and forces_offload:
# Encoder: offload rejects these after the resident model is gone; layerwise fp8 is a dtype cast.
te_reason = (
f"'{normalize_memory_mode(memory_mode)}' memory places the text encoder under CPU "
"offload, and torchao quantised tensors cannot be moved by the offload hooks"
)
if te_reason is not None:
raise RuntimeError(
precision_refusal_message(
"text_encoder_quant",
te_mode,
te_reason,
off_label = "leave it unset to keep the dense bf16 encoder",
auto_available = False,
)
)
def _is_trusted_video_repo(repo_id: str) -> bool:
"""Whether a NON-GGUF load may deserialise this repo (see the image twin)."""
try:
if Path(repo_id).expanduser().exists():
return True
except OSError:
pass
rid = repo_id.strip().lower()
return rid.startswith("unsloth/") or rid in _TRUSTED_NON_GGUF_VIDEO_REPOS
def _cached_checkpoint_file(repo_id: str, filename: str) -> Optional[Path]:
"""``filename`` in a local ``repo_id`` dir, else its cached hub blob (no network; active, legacy and default cache
roots), else None."""
path = Path(repo_id).expanduser() / filename
if path.is_file():
return path
# A mirrored repo is fetched into the MIRROR's cache key, an older install's copy sits under the upstream's.
for alias in _video_repo_aliases(repo_id) or (repo_id,):
hit = _cached_hub_checkpoint_file(alias, filename)
if hit is not None:
return hit
return None
def _cached_hub_checkpoint_file(repo_id: str, filename: str) -> Optional[Path]:
"""``filename``'s cached hub blob for exactly ``repo_id`` (active, legacy and default cache roots), else None."""
from huggingface_hub import try_to_load_from_cache
cached = try_to_load_from_cache(repo_id, filename)
if not isinstance(cached, str):
from hub.utils.paths import hf_default_cache_dir, legacy_hf_cache_dir
for root_fn in (legacy_hf_cache_dir, hf_default_cache_dir):
try:
cached = try_to_load_from_cache(repo_id, filename, cache_dir = str(root_fn()))
except Exception: # noqa: BLE001 -- a bad/absent root just falls through
cached = None
if isinstance(cached, str):
break
return Path(cached) if isinstance(cached, str) else None
def _picked_gguf_arch(repo_id: str, gguf_filename: str) -> Optional[str]:
"""Reads the arch, not the name, so a renamed GGUF resolves the same family the picker offered."""
try:
path = _cached_checkpoint_file(repo_id, gguf_filename)
if path is None:
return None
from utils.models.gguf_metadata import read_gguf_architecture
return read_gguf_architecture(str(path))
except Exception: # noqa: BLE001 -- a header read glitch just falls through to name detection
return None
# Enough for a 15-second reference video after base64 decoding.
_MAX_REFERENCE_MEDIA_BYTES = 96 * 1024 * 1024
def _decode_b64_media(data: Optional[str]) -> bytes:
"""Decode a base64 media payload, optionally wrapped in a data URL."""
import base64
import binascii
raw = (data or "").strip()
if not raw:
raise ValueError("A reference was sent empty.")
if raw.startswith("data:"):
_, _, raw = raw.partition(",")
try:
blob = base64.b64decode(raw, validate = False)
except (binascii.Error, ValueError) as exc:
raise ValueError(f"Invalid base64 media data: {exc}") from exc
if not blob:
raise ValueError("A reference decoded to no data.")
if len(blob) > _MAX_REFERENCE_MEDIA_BYTES:
raise ValueError(
f"A reference is too large ({len(blob) / 1e6:.0f} MB); the limit is "
f"{_MAX_REFERENCE_MEDIA_BYTES / 1e6:.0f} MB."
)
return blob
VIDEO_CONTINUATION_CONDITIONING = "v2v"
class _VideoGenerationCancelled(Exception):
"""Unwinds a denoise loop that has no cooperative interrupt (no step callback);
generate() maps it to the VIDEO_CANCELLED_MSG sentinel the routes 409 on."""
@contextlib.contextmanager
def _scheduler_step_progress(
pipe: Any,
on_step: Any,
on_step_done: Any = None,
):
"""Wraps scheduler.step, always restored even on error: on_step cancels before it, on_step_done after."""
scheduler = pipe.scheduler
original = scheduler.step
count = {"n": 0}
def _step(*args: Any, **kwargs: Any) -> Any:
count["n"] += 1
on_step(count["n"])
result = original(*args, **kwargs)
if on_step_done is not None:
on_step_done(count["n"])
return result
scheduler.step = _step
try:
yield
finally:
scheduler.step = original
def _record_device_event() -> Any:
"""Returns a CUDA or MPS event or None; never synchronises, and never records inside a graph capture."""
try:
import torch
if torch.cuda.is_available():
if torch.cuda.is_current_stream_capturing():
return None
event = torch.cuda.Event()
elif torch.backends.mps.is_available():
event = torch.mps.Event()
else:
return None
event.record()
return event
except Exception: # noqa: BLE001 -- progress reporting must never fail a render
return None
class _CompletedStepTicker:
"""The denoise step the GPU has FINISHED, not the one the CPU has enqueued.
With the denoiser's forward replayed from a captured CUDA graph the Python loop enqueues every
step and returns long before the GPU has run them: on MiniMax-H3 at 960x544x124 over 30 steps
the 29 host-side ticks land in 0.9 s and the GPU is still 26 s from done. A counter driven by
``scheduler.step`` or ``callback_on_step_end`` therefore runs the bar to the end of the denoise
while no denoising has happened, and the user watches an apparent freeze at the far right of a
bar that claims a second remains.
So mark each step's place in the stream with a ``torch.cuda.Event`` and report the last one
that has completed. ``query()`` does not block and does not serialise, which is the whole
point: a ``synchronize()`` per step would give the same honest number and hand back the
CUDA-graph win that the video speed path exists for.
MPS runs ahead without any graph (torch >= 2.13), so it gets ``torch.mps.Event`` markers too.
With no event (CPU, where work is synchronous) the enqueued count is the progress.
"""
def __init__(self, total: int) -> None:
self.total = max(0, int(total))
self._lock = threading.Lock()
self._events: list = []
self._event_backed = False
self._enqueued = 0
self._done = 0
self._scanned = 0
self._boundary: Any = None
self._boundary_marked = False
@property
def event_backed(self) -> bool:
"""Whether any step is being tracked by a real GPU event."""
with self._lock:
return self._event_backed
def mark_boundary(self) -> bool:
"""Records the denoise end in the stream; True means wait on boundary_reached(), else it is
reached."""
event = _record_device_event()
with self._lock:
self._boundary = event
self._boundary_marked = True
return event is not None
@property
def boundary_marked(self) -> bool:
"""Whether the end of the denoise has been marked yet."""
with self._lock:
return self._boundary_marked
def boundary_reached(self) -> bool:
"""An unreadable marker counts as reached, so it cannot strand the bar in the denoise phase."""
with self._lock:
if not self._boundary_marked:
return False
event = self._boundary
if event is None:
return True
try:
return bool(event.query())
except Exception: # noqa: BLE001 -- an unqueryable marker counts as reached
return True
def record(self, enqueued: int) -> None:
"""Note that step ``enqueued`` (1-based) has been submitted to the device."""
enqueued = int(enqueued)
with self._lock:
if enqueued <= self._enqueued:
return
self._enqueued = enqueued
if len(self._events) >= self.total:
return
event = _record_device_event()
if event is None:
return
with self._lock:
# The step travels with the event, so an unmarked step never shifts later ones.
self._events.append((enqueued, event))
self._event_backed = True
def caught_up(self, lead: int) -> bool:
"""Whether the device has finished every step but the last ``lead`` enqueued; never blocks."""
with self._lock:
if self._enqueued >= self.total or len(self._events) <= lead:
return True
event = self._events[-1 - lead][1]
try:
return bool(event.query())
except Exception: # noqa: BLE001 -- an unqueryable marker counts as finished
return True
def completed(self) -> int:
"""Steps the GPU has finished. Never decreases, never exceeds what was enqueued, never
blocks, and never raises."""
with self._lock:
if not self._event_backed:
return self._enqueued
pending = self._events[self._scanned :]
done, scanned = self._done, self._scanned
for enqueued, event in pending:
try:
if not event.query():
break
except Exception: # noqa: BLE001 -- an unqueryable event just stops the scan
break
done = max(done, enqueued)
scanned += 1
with self._lock:
self._done = max(self._done, done)
self._scanned = max(self._scanned, scanned)
return self._done
@contextlib.contextmanager
def _hold_off_cuda_graph_capture():
"""Holds off graph capture while the caller queries events; with no graph layer no capture can start."""
try:
from . import diffusion_cuda_graph
guard = diffusion_cuda_graph.hold_off_capture
except Exception: # noqa: BLE001 -- no graph layer means no capture to avoid
yield True
return
with guard() as clear:
yield clear
def _hold_for_device(
ticker: Any,
cancel: Any,
lead: int = 1,
) -> None:
"""Keeps the host at most ``lead`` steps ahead of the device: an enqueued step cannot be cancelled."""
while not cancel.is_set():
with _hold_off_cuda_graph_capture() as clear:
if not clear or ticker.caught_up(lead):
return
cancel.wait(0.002)
@contextlib.contextmanager
def _completed_step_poller(pump: Any, poll_seconds: float = 0.1):
"""The pump runs under the capture hold-off, since a query inside a capture invalidates the graph."""
stop = threading.Event()
def _run() -> None:
while not stop.is_set():
stop.wait(poll_seconds)
try:
with _hold_off_cuda_graph_capture() as clear:
if not clear:
continue
pump()
except Exception: # noqa: BLE001 -- progress must never fail a render
pass
thread = threading.Thread(target = _run, name = "video-denoise-progress", daemon = True)
thread.start()
try:
yield
finally:
stop.set()
thread.join(timeout = 2.0)
_BOUNDARY_MARK_ATTEMPTS = 20
_BOUNDARY_MARK_RETRY_SECONDS = 0.02
def _assert_pick_is_not_speech(
repo_id: str,
gguf_filename: Optional[str],
hf_token: Optional[str] = None,
allow_network: bool = True,
) -> None:
"""The shared speech refusal, imported lazily so this module keeps its import cost."""
from .diffusion_compat import assert_pick_is_not_speech
assert_pick_is_not_speech(repo_id, gguf_filename, hf_token, allow_network)
def _refuse_non_dit_pick(repo_id: str, gguf_filename: Optional[str], page: str) -> None:
from .diffusion_content import assert_local_pick_is_dit
assert_local_pick_is_dit(repo_id, gguf_filename, page)
def _i2v_pipeline_available(name: Optional[str]) -> bool:
from .video_i2v import i2v_pipeline_available
return i2v_pipeline_available(name)
def _i2v_state_call_kwargs(fam: VideoFamily, state: Any) -> dict[str, Any]:
return video_i2v_call_kwargs(
fam,
state.variant_id,
state.gguf_filename,
state.display_repo_id,
state.repo_id,
state.base_repo,
)
def _bg_i2v_render(first_pil: Any, i2v_active: bool) -> bool:
"""An image-conditioned render for the background compile, however the family conditions (a swapped i2v pipeline,
or a pipeline its own render derives): its denoiser inputs (per-token timesteps) are never a text-to-video sample,
so it neither seeds the compile nor counts as an eager text render."""
return bool(i2v_active) or first_pil is not None
def _settle_bg_compile(bg: Any, *, i2v: bool) -> None:
if not i2v:
bg.note_eager_generation()
bg.kick()
elif not bg.kick():
# An image-to-video render records nothing; without this an i2v-only session never leaves eager.
bg.abandon("an image-to-video render records no compile sample")
# Shared with the image backend's first compiled render.
_float_seed_dits = diffusion_bg_compile.float_seed_dits
_COMPILE_PREWARM_LOCK = threading.Lock()
_COMPILE_PREWARM_STARTED = False
def _start_compile_prewarm(logger: Any = None) -> bool:
"""Once per process, on a side thread: inductor's compile workers, Triton's and inductor's keys (1.5 s of Triton
install hashing landed in a returning LTX-2.5 first clip on an L4). Never raises; False when already started."""
global _COMPILE_PREWARM_STARTED
if not diffusion_bg_compile.prewarm_enabled():
return False
with _COMPILE_PREWARM_LOCK:
if _COMPILE_PREWARM_STARTED:
return False
_COMPILE_PREWARM_STARTED = True
def run() -> None:
for warm in (
diffusion_bg_compile.warm_compile_pool,
diffusion_bg_compile.warm_triton_key,
diffusion_bg_compile.warm_inductor_key,
):
try:
warm(logger)
except Exception: # noqa: BLE001 - the first compile does it inline as before
pass
try:
threading.Thread(target = run, name = "unsloth-video-compile-prewarm", daemon = True).start()
except Exception: # noqa: BLE001
return False
return True
_seed_before_first_forward = diffusion_bg_compile.seed_before_first_forward
def _video_float_seed_dir(
pipe: Any, *, family: Any, dtype: Any, quant: Any, attention_backend: Any
) -> tuple[list, Optional[Path]]:
"""(DiTs, per-key seed dir keyed like the image compile cache), or ([], None). Video has no compile-cache ctx."""
from . import diffusion_bg_compile
if not diffusion_bg_compile.float_seeds_enabled():
return [], None
dits = _float_seed_dits(pipe)
if not dits:
return [], None
try:
from . import diffusion_compile_cache as cc
model_fp = cc.model_fingerprint(
family = family,
transformer = dits[0],
dtype = str(dtype).replace("torch.", ""),
quant = quant,
attention_backend = attention_backend,
compile_kwargs = dict(dits[0]._unsloth_regional_compile_kwargs),
)
model_fp["video_dits"] = [type(d).__name__ for d in dits]
key = cc.cache_key(cc.environment_fingerprint(), model_fp)
return dits, cc.cache_root() / "video_float_seeds" / key
except Exception: # noqa: BLE001 - no seeds, main's compile
return [], None
def assert_first_frame_pipeline_available(fam: VideoFamily) -> None:
if video_family_uses_i2v_pipeline(fam) and not _i2v_pipeline_available(fam.i2v_pipeline_class):
raise ValueError(
f"{fam.name} needs diffusers' {fam.i2v_pipeline_class} for a first frame, which this "
"install does not have. Update Unsloth to get a newer diffusers."
)
def _detect_load_family(
repo_id: str,
gguf_filename: Optional[str],
family_override: Optional[str],
display_repo_id: Optional[str] = None,
model_kind: Optional[str] = None,
) -> Optional[VideoFamily]:
# An explicit pipeline load never opens the filename, so it must not vouch for the repo.
ltx_file = None if str(model_kind or "").strip().lower() == "pipeline" else gguf_filename
fam, by_content = _resolve_load_family_with_source(
repo_id, gguf_filename, family_override, display_repo_id, ltx_file = ltx_file
)
if fam is not None and fam.name == "ltx-2.5" and not ltx_file:
from .video_ltx25 import LTX25_SINGLE_FILE_REPO, is_ltx25_single_file_base
# Mirror-folded: the carrier (unsloth/LTX-2.5-GGUF) has no model_index.json either. Only a FILE pick keeps
# the unfolded check (ltx25_default_base), so a GGUF from the carrier keeps the Diffusers base.
if is_ltx25_single_file_base(repo_id):
# No model_index.json there: a pipeline load would stage ~29 GB of companions and then fail.
raise ValueError(
f"'{repo_id}' ships LTX-2.5 as single files, not a Diffusers pipeline. Pick one of its DiT files "
f"(diffusion_models/), or load {LTX25_SINGLE_FILE_REPO}-Diffusers."
)
if fam is not None and fam.name in LTX_FAMILIES:
# A local diffusers folder's model_index.json outranks its name, as a file's header does.
by_content = by_content or (not ltx_file and local_ltx_pipeline_family(repo_id) is not None)
# After resolution: an LTX family never loads the other release's weights (LTX-2.5 never reaches the 2.3
# assembly), and a release after 2.5 is refused. A local diffusers folder answers by its model_index.json.
if not ltx_file:
assert_local_pipeline_family(repo_id, fam.name)
if by_content:
# The header decided the family; only an unknown release named by the pick is still refused.
assert_supported_ltx_version(repo_id, ltx_file, display_repo_id)
else:
assert_ltx_pick_supported(fam.name, repo_id, ltx_file, display_repo_id)
return fam
def _resolve_load_family(
repo_id: str,
gguf_filename: Optional[str],
family_override: Optional[str],
display_repo_id: Optional[str] = None,
) -> Optional[VideoFamily]:
"""Family detection shared by validate_load_request and the load worker: the
repo id first, then the picked filename -- a local directory or generically
named repo often carries the family token only in the checkpoint filename,
and the worker must resolve the same family the validator accepted."""
return _resolve_load_family_with_source(
repo_id, gguf_filename, family_override, display_repo_id, ltx_file = gguf_filename
)[0]
def _assert_local_ltx_header_matches(
repo_id: str, gguf_filename: Optional[str], family: str
) -> None:
"""A forced LTX family still never loads the other release's weights: a local file's header names its release."""
from .diffusion_content import inspect_checkpoint, local_pick_file
from .video_families import LTX25_FAMILY, wrong_ltx_family_message
path = local_pick_file(repo_id, gguf_filename)
if not path:
return
found = inspect_checkpoint(path).family
if found in LTX_FAMILIES and found != family:
version = "2.5" if found == LTX25_FAMILY else "2.3"
raise ValueError(
wrong_ltx_family_message(
version, family, f"'{os.path.basename(path)}' (by its weights)"
)
)
def _ltx25_dev_load(
fam: Any,
kind: str,
gguf_filename: Optional[str],
repo_id: Optional[str],
display_repo_id: Optional[str] = None,
) -> bool:
"""Whether an LTX-2.5 load is the dev DiT (single stage, no latent upsampler). A pipeline load opens no single file,
so its repo names the variant (the official pack is distilled); a single file decides by its own name first."""
if getattr(fam, "name", None) != "ltx-2.5":
return False
from .video_families import ltx25_is_dev
if kind == "pipeline":
return ltx25_is_dev(display_repo_id, repo_id)
return ltx25_is_dev(gguf_filename, repo_id)
def _stop_ltx25_encoder(client: Any) -> None:
if client is None:
return
try:
client.close()
except Exception as exc: # noqa: BLE001 -- cleanup is best-effort
logger.warning("video.ltx25_te: could not stop a pending encoder worker: %s", exc)
def _resolve_load_family_with_source(
repo_id: str,
gguf_filename: Optional[str],
family_override: Optional[str],
display_repo_id: Optional[str] = None,
*,
ltx_file: Optional[str] = None,
) -> tuple[Optional[VideoFamily], bool]:
"""``_resolve_load_family`` plus whether a local file's header decided the family."""
if family_override:
fam = detect_video_family(repo_id, family_override)
if fam is not None and fam.name in LTX_FAMILIES:
# ltx_file, not the raw filename: an explicit pipeline load never opens a child file, so its folder answers
# by its model_index.json instead.
_assert_local_ltx_header_matches(repo_id, ltx_file, fam.name)
return fam, False
fam = (
(display_repo_id and detect_video_family(display_repo_id))
or detect_video_family(repo_id)
or (gguf_filename and detect_video_family(f"{repo_id}/{gguf_filename}"))
or None
)
if fam is None and not gguf_filename:
local_cosmos3 = local_cosmos3_pipeline_family(repo_id)
if local_cosmos3:
fam = detect_video_family("", override = local_cosmos3)
if fam is None and gguf_filename:
# A renamed GGUF has no family token, so resolve via general.architecture.
arch = _picked_gguf_arch(repo_id, gguf_filename)
if arch:
fam = detect_video_family(repo_id, override = arch)
if fam is not None and fam.name in COSMOS3_FAMILIES and not gguf_filename:
by_configs = local_cosmos3_pipeline_family(repo_id)
if by_configs and by_configs != fam.name:
fam = detect_video_family("", override = by_configs) or fam
if fam is not None and fam.name in LTX_FAMILIES:
# Both LTX families share the "ltx" tokens, so the release decides: a local diffusers folder by its
# model_index.json, else the picked file's own version token, else the display / repo id.
named = (
local_ltx_pipeline_family(repo_id) if not ltx_file else None
) or ltx_family_by_name(repo_id, ltx_file, display_repo_id)
if named is not None and named != fam.name:
fam = detect_video_family("", override = named) or fam
from .diffusion_content import SAME_ARCH_VARIANTS, local_pick_file, resolve_family_with_content
# Same layout, different model: the picked file's own name outranks its repo or folder (a Wan2.1 14B GGUF in a
# Wan2.2 A14B repo, or the reverse), local or remote.
if fam is not None and gguf_filename:
named = detect_video_family(os.path.basename(gguf_filename))
if (
named is not None
and named.name != fam.name
and (
named.name in SAME_ARCH_VARIANTS.get(fam.name, ())
or fam.name in SAME_ARCH_VARIANTS.get(named.name, ())
# A remote pick has no header pass, so a file naming another Wan model picks the base config.
or (named.name.startswith("wan") and fam.name.startswith("wan"))
)
):
fam = named
# A local file's header outranks its name (the name still picks 480p vs 720p).
path = local_pick_file(repo_id, gguf_filename)
if path:
file_fam = detect_video_family(os.path.basename(path))
name, by_content = resolve_family_with_content(
fam.name if fam else None,
path,
"video",
file_family = file_fam.name if file_fam else None,
)
if name is None:
return None, by_content
if fam is None or fam.name != name:
fam = detect_video_family("", override = name)
return fam, by_content
return fam, False
def _ensure_mp4_encoder_available() -> None:
"""Fail a load fast when PyAV is missing: the export otherwise dies AFTER a
multi-minute denoise, which is the worst possible time to learn about it."""
try:
import av # noqa: F401
except Exception as exc: # noqa: BLE001 -- any import failure means no encoder
raise ValueError(
"Video generation needs the 'av' package (PyAV) to encode MP4s, and this install "
"is missing it. Update Unsloth to restore it (in the desktop app: Settings, Check for "
"updates; from a terminal: unsloth studio update). On a plain pip install: "
"pip install av"
) from exc
def _video_denoiser_hooked(pipe: Any) -> bool:
"""Whether an offload hook moves one of ``pipe``'s denoisers."""
for name in ("transformer", "transformer_2"):
module = getattr(pipe, name, None)
if module is None:
continue
if getattr(module, "_hf_hook", None) is not None:
return True
registry = getattr(module, "_diffusers_hook", None)
hooks = getattr(registry, "hooks", None) or {}
if any("offload" in str(key) for key in hooks):
return True
return False
@dataclass(frozen = True)
class _VideoLoadState:
"""Everything about the currently-loaded video pipeline, swapped as one unit."""
pipe: Any
family: VideoFamily
repo_id: str
base_repo: str
device: str
dtype: str
kind: str
display_repo_id: Optional[str] = None
engine: str = "diffusers"
# Committed with the pipeline, so a load in flight never moves the resident model's card.
gpu_ordinal: Optional[int] = None
# Actual card, automatic loads included; only for re-pinning a worker thread.
placed_ordinal: Optional[int] = None
gguf_filename: Optional[str] = None
# The repo the load actually read ``base_repo`` from (the unsloth mirror on a fresh install): the learned-peak key
# names it. None: no load decision (the key falls back to the mirror when the upstream is not cached).
fetch_base: Optional[str] = None
# Content-detected checkpoint variant identifier (LTX-2.3 "ltx-2.3-22b-distilled" / "-dev"), read before the names.
variant_id: Optional[str] = None
# Resident MiniMax-H3 denoiser partition, if any.
h3_task: Optional[str] = None
offload_policy: str = "none"
vae_tiling: bool = True
memory_mode: str = "auto"
speed_mode: str = SPEED_OFF
speed_optims: tuple = ()
backend_flags: Optional[dict] = None
attention_backend: Optional[str] = None
# Only fbcache breaks the graph (cache_breaks_graph); static keeps fullgraph and CUDA-graph eligibility.
transformer_cache: Optional[str] = None
# Auto only: generate() toggles it across FBCACHE_MIN_STEPS; explicit never toggles.
cache_auto: bool = False
# Inputs the generation-time toggle re-applies (quantised threshold + override).
cache_quant_active: bool = False
cache_threshold: Optional[float] = None
# "int8"|"fp8"|"nvfp4"|"mxfp8" or None. Pipeline-kind only.
transformer_quant: Optional[str] = None
# "fp8"|"fp8_dynamic"|"int8"|"nvfp4" or None.
text_encoder_quant: Optional[str] = None
# MiniMax-H3: prequant denoiser pinned resident, so the memory floor is a sum, not a max.
denoiser_pinned: bool = False
# MiniMax-H3: streamed denoiser, so the VRAM floor is the largest single phase, not a sum.
denoiser_streamed: bool = False
# MiB the offload tier holds on the device at once; None when nothing is offloaded.
vram_floor_mib: Optional[int] = None
# MiB still on the device during the VAE decode (the streamed DiT has offloaded by then).
vram_decode_floor_mib: Optional[int] = None
vram_split_cards: Optional[tuple] = None
# MiniMax-H3: the streamed denoiser also holds a full pinned host copy, which the host floor counts twice.
denoiser_host_copy: bool = False
# MiniMax-H3: conditioner streamed leaf by leaf (VRAM floor counts its streamed footprint).
te_streamed: bool = False
# MiniMax-H3: video_minimax_h3_residency.H3Residency of a streamed denoiser, re-fitted per request.
h3_residency: Any = None
resolved: Optional[dict] = None
bg_compile: Any = None
@dataclass(frozen = True)
class _VideoResolvedInputs:
"""CPU-side request inputs resolved against one exact resident pipeline state."""
state: _VideoLoadState
first_frame: Any
last_frame: Any
width: int
height: int
conditioning: str
references: Any
flow_shift: Optional[float]
audio_flow_shift: Optional[float]
continuation: Any = None
@dataclass
class _VideoLoadingState:
repo_id: str
base_repo: str
expected_bytes: Optional[int] = None
error: Optional[str] = None
error_code: Optional[str] = None
# Companion repos this load is ALSO pulling from, beyond repo_id / base_repo (the native H3 GGUF and component
# repos). Mirrors the image backend's _SdLoading.asset_repos.
asset_repos: tuple[str, ...] = ()
# Fetch ROLE ("checkpoint" / "base") -> the one plan this load settled at estimate time: the repo that role
# fetches from, the revision it is pinned to and every file with its blob key and size. Written only by the worker
# that owns this state (under the load-token check), replaced whole by a re-estimate, never patched by a fetch site.
# The fetch sites read their repo + revision from it, and load_progress reads nothing else.
fetch_plan: dict[str, "_PlannedFetch"] = field(default_factory = dict)
class _PlannedFetch(NamedTuple):
"""One fetch role's plan: the repo it reads, the revision it is pinned to (the listing commit when that repo IS the
listing; None for an upstream kept for its cache, which resolves its own ref) and ``(name, blob key, size)`` per
file. The blob key (LFS sha256, else git blob id) is what huggingface_hub names the cache blob, so progress reads a
file's bytes by content: revision-proof, root-proof, and blind to any other blob in the repo."""
repo: str
revision: Optional[str]
files: tuple
# The encoder components the plan left out (hosted pre-cast checkpoints it expected to find); _run_load re-plans
# the base when the actual fetches skip a different set.
skip_te: tuple = ()
def _sibling_etag(sibling: Any) -> Optional[str]:
"""The key huggingface_hub names a sibling's cache blob by: the LFS sha256, else the git blob id."""
lfs = getattr(sibling, "lfs", None)
sha = getattr(lfs, "sha256", None) or (lfs.get("sha256") if isinstance(lfs, dict) else None)
return sha or getattr(sibling, "blob_id", None) or None
def _fetched_pins(plan: dict[str, "_PlannedFetch"]) -> dict[str, str]:
"""The commit the checkpoint was already fetched under, keyed by its repo, for a re-plan to list that repo at."""
entry = plan.get(_FETCH_CHECKPOINT)
return {entry.repo: entry.revision} if entry is not None and entry.revision else {}
def _keep_fetched_checkpoint(
replanned: dict[str, "_PlannedFetch"], previous: dict[str, "_PlannedFetch"]
) -> dict[str, "_PlannedFetch"]:
"""A re-plan made after the checkpoint was fetched keeps the checkpoint entry it was fetched under."""
out = dict(replanned)
if _FETCH_CHECKPOINT in previous:
out[_FETCH_CHECKPOINT] = previous[_FETCH_CHECKPOINT]
return out
class _OneCommitListings:
"""``model_info`` per physical repo, ONCE per plan: two roles reading one repo (a carrier's own checkpoint and the
base it mirrors) are sized and pinned from the same listing, so they can never land on two commits. ``pins`` maps
a lowercased repo to the commit a role was already fetched under (a re-plan after the checkpoint fetch), and that
repo is then listed at exactly that commit."""
def __init__(
self,
api: Any,
pins: Optional[dict[str, str]] = None,
):
self._api = api
self._pins = {k.lower(): v for k, v in (pins or {}).items() if k and v}
self._infos: dict[str, Any] = {}
def info(self, repo: str) -> Any:
key = repo.strip().lower()
if key not in self._infos:
pinned = self._pins.get(key)
self._infos[key] = self._api.model_info(
repo, files_metadata = True, **({"revision": pinned} if pinned else {})
)
return self._infos[key]
def _plan_total(plan: dict[str, "_PlannedFetch"]) -> Optional[int]:
"""The bytes a plan sums to (what ``expected_bytes`` reports), None for an empty one."""
# Counted once per blob in a repo, exactly as load_progress counts them: two names with identical bytes share one
# cache blob, so summing both would leave the bar short of 100% forever.
seen: set[tuple[str, str]] = set()
total = 0
for entry in plan.values():
for name, key, size in entry.files:
ident = ((entry.repo or "").strip().lower(), key or name)
if ident not in seen:
seen.add(ident)
total += int(size or 0)
return total or None
def _cached_snapshot_plan(repo: str, names: list) -> Optional[tuple[str, tuple]]:
"""``(commit, ((name, blob key, size), ...))`` of the one cached revision of ``repo`` that holds every name, in the
live root or the import-time root (the roots the fetches resolve through), else None. The blob key is the cache
blob the snapshot entry points at (None where the cache keeps plain copies, e.g. no symlinks)."""
try:
from huggingface_hub import constants
from .diffusion_families import _cached_revisions
roots = [Path(hub_cache_dir())]
if Path(constants.HF_HUB_CACHE) != roots[0]:
roots.append(Path(constants.HF_HUB_CACHE))
for root in roots:
for rev in _cached_revisions(root, repo):
if not all((rev / name).is_file() for name in names):
continue
files = []
for name in names:
entry = rev / name
blob = Path(os.path.realpath(entry))
key = blob.name if blob.parent.name == "blobs" else None
files.append((name, key, int(blob.stat().st_size)))
return rev.name, tuple(files)
except Exception: # noqa: BLE001 -- unreadable cache: fall back to the listing's metadata
return None
return None
def _plan_fetch(
logical: str,
listing: str,
info: Any,
names: Any,
hf_token: Optional[str],
*,
checkpoint: bool = False,
) -> "_PlannedFetch":
"""The plan for one role: its fetch repo settled on the exact names, pinned to the listing commit when the fetch
reads the listing repo, and each file's blob key + size from the same listing."""
wanted = list(dict.fromkeys(names))
repo = _video_fetch_repo(logical, hf_token, files = wanted, checkpoint = checkpoint) or logical
if repo.lower() != (listing or "").lower():
# An upstream kept for its complete cache: the plan describes THAT snapshot (its commit, its blobs), not the
# mirror listing, which may already be a newer revision of a file.
cached = _cached_snapshot_plan(repo, wanted)
if cached is not None:
return _PlannedFetch(repo, cached[0], cached[1])
meta = {s.rfilename: s for s in (getattr(info, "siblings", None) or [])}
files = tuple(
(
name,
_sibling_etag(meta[name]) if name in meta else None,
int(getattr(meta.get(name), "size", 0) or 0),
)
for name in wanted
)
revision = getattr(info, "sha", None) if repo.lower() == (listing or "").lower() else None
return _PlannedFetch(repo, revision, files)
_FETCH_CHECKPOINT = "checkpoint"
_FETCH_BASE = "base"
def _physical_card_name(ordinal: Optional[int]) -> "tuple[Optional[str], Optional[int]]":
from .sd_cpp_backend import physical_card_name
return physical_card_name(ordinal)
def _release_h3_server_slot(slot: Any, reason: str) -> str:
"""Release an H3 sd-server slot without waiting on a render: "stopped" (memory freed now), "pending" (mid-render
or starting; stops at the render's end, nothing freed yet) or "none" (no server)."""
if slot is None:
return "none"
try:
if not slot.release(reason):
return "none"
return "pending" if slot.is_alive() else "stopped"
except Exception as exc: # noqa: BLE001 - a release failure leaves the reading as it was
logger.warning("video.h3_native_server_release_failed: %s", exc)
return "none"
def _h3_offloaded_render_flags(
runtime: Any,
render_flags: "list[str]",
*,
render_resident: bool,
memory_mode: Optional[str],
reuse_live_server: bool = True,
compute_bytes: Optional[int] = None,
) -> list[str]:
"""The flags an H3 native render runs with once Auto chose resident or offloaded. Every offloaded render is
re-fitted on live RAM, swap and GTT readings (file reads only) before a process starts, so pressure since the load
is caught and memory freed since lifts a refusal; Auto may move the encoder, an explicit mode is only refused. ``reuse_live_server``: a live offloaded sd-server the render will reuse already
holds its weights and keeps its own flags; pass False before launching any new process (the one-shot sd-cli),
which releases that server and fits afresh."""
from .video_minimax_h3 import (
H3_NATIVE_RENDER_RUNNING_HINT,
h3_native_offloaded_render_flags,
)
from .video_minimax_h3 import h3_native_fit_check_enabled
inputs = getattr(runtime, "fit_inputs", None)
if not h3_native_fit_check_enabled():
inputs = None # UNSLOTH_H3_NATIVE_FIT_CHECK=0: no reading, no release, the committed flags as they are
if render_resident or "--offload-to-cpu" not in render_flags or inputs is None:
if inputs is not None:
# A server with other flags is replaced by get() anyway: free it so its pinned RAM is not counted.
slot = getattr(runtime, "server_slot", None)
live = slot.alive_signature() if slot is not None else None
hint = ""
if live is not None and tuple(live[2]) != tuple(render_flags):
if _release_h3_server_slot(slot, "a resident render needs its memory") == "pending":
hint = H3_NATIVE_RENDER_RUNNING_HINT
_h3_check_render_overhead(hint)
return h3_native_offloaded_render_flags(
render_flags,
render_resident = render_resident,
offloaded_flags = getattr(runtime, "offloaded_flags", None),
offload_refusal = getattr(runtime, "offload_refusal", None),
)
slot = getattr(runtime, "server_slot", None)
live = slot.alive_signature() if slot is not None else None
if reuse_live_server and live is not None and "--offload-to-cpu" in live[2]:
# A reading now would count the server's own weights; check only the per-render host RAM.
_h3_check_render_overhead()
return list(live[2])
released = "none"
if live is not None:
# Free any server before the reading (a resident one is replaced by slot.get() anyway).
released = _release_h3_server_slot(slot, "an offloaded render needs its memory")
return _h3_launch_flags(
runtime,
render_flags,
memory_mode,
hint = H3_NATIVE_RENDER_RUNNING_HINT if released == "pending" else "",
compute_bytes = compute_bytes,
)
def _h3_check_render_overhead(hint: str = "") -> None:
from .video_minimax_h3 import H3NativeMemoryRefusal, h3_native_render_overhead_refusal
refusal = h3_native_render_overhead_refusal()
if refusal:
raise H3NativeMemoryRefusal(refusal + hint)
def _h3_launch_flags(
runtime: Any,
render_flags: "list[str]",
memory_mode: Optional[str],
*,
hint: str = "",
compute_bytes: Optional[int] = None,
) -> list[str]:
"""The live fit for one process launch, with no server-slot calls (it also runs under the slot's lock, from
``H3NativeServerSlot.get``'s ``pre_spawn``). Auto may move the encoder; an explicit mode runs exactly its flags
or is refused. A resident render or a runtime without fit inputs passes through. ``compute_bytes`` is the
render's denoiser compute buffer."""
from .sd_cpp_args import device_backend_flags
from .video_minimax_h3 import (
H3NativeMemoryRefusal,
h3_native_live_fit,
h3_native_with_max_vram,
)
from .video_minimax_h3 import h3_native_fit_check_enabled
inputs = getattr(runtime, "fit_inputs", None)
if inputs is None or not h3_native_fit_check_enabled():
return list(render_flags)
if "--offload-to-cpu" not in render_flags:
# A resident start: its weights go to VRAM (judged by the resident choice), its per-render RAM is checked.
_h3_check_render_overhead(hint)
return list(render_flags)
mode = memory_mode or "auto"
fit = h3_native_live_fit(inputs, mode, compute_bytes)
if fit.refusal:
raise H3NativeMemoryRefusal(fit.refusal + hint)
if fit.note:
logger.info("video.h3_native_memory_fit (launch): %s", fit.note)
if mode != "auto":
# The placement is the mode's; only a UMA device's --max-vram reserve is applied.
return h3_native_with_max_vram(render_flags, fit.max_vram)
flags = tuple(fit.flags)
return list(flags + tuple(device_backend_flags(inputs.device_name, list(flags))))
def _h3_budgeted_launch_flags(
runtime: Any,
flags: "Sequence[str]",
*,
cap: Any = None,
) -> list[str]:
"""Swap ``--max-vram -1`` for a budget from free VRAM read now, never above ``cap``; anything unknown keeps -1."""
from .sd_cpp_args import (
explicit_vram_budget_enabled,
graph_cut_budget_gib,
max_vram_is_placeholder,
max_vram_value,
with_max_vram_budget,
)
out = list(flags)
# Never replace a UMA pinned-weight reserve (-17.05): it would hand sd.cpp the memory those weights take.
if not max_vram_is_placeholder(max_vram_value(out)):
return out
if not explicit_vram_budget_enabled():
return _h3_spare_user_reserve(runtime, out, read_free = True)
probe = getattr(runtime, "vram_free_probe", None)
try:
free = probe() if probe is not None else None
except Exception: # noqa: BLE001 -- an unreadable card keeps sd.cpp's own reading
free = None
# Too little free now: "-1" is already below this reading, while an earlier retry cap could exceed it.
known_small = False
if isinstance(free, dict):
budget: Any = {}
for name, value in free.items():
gib = graph_cut_budget_gib(value)
limit = cap.get(name) if isinstance(cap, dict) else cap
if gib is None:
known_small = known_small or value is not None
budget = None
break
budget[name] = min(gib, float(limit)) if limit is not None else gib
else:
budget = graph_cut_budget_gib(free)
known_small = budget is None and free is not None
if budget is not None and cap is not None and not isinstance(cap, dict):
budget = min(budget, float(cap))
if not budget:
if cap is not None and not known_small:
out = with_max_vram_budget(out, cap)
else:
# "-1" is sd.cpp's RAW free reading: under a lowered VRAM Budget it must spare the reserve too, and a
out = _h3_spare_user_reserve(runtime, out, free = free if known_small else None)
logger.info(
"video.h3_native_vram_budget: free VRAM unreadable or too small (%s); %s",
free,
f"--max-vram {max_vram_value(out)}",
)
return out
out = with_max_vram_budget(out, budget)
logger.info(
"video.h3_native_vram_budget: --max-vram %s (free %s at launch%s)",
max_vram_value(out),
free,
f", capped by retry budget {cap}" if cap is not None else "",
)
return out
class _H3UnpinnedMultiCardReserve(int):
"""The smallest card's VRAM Budget reserve on an unpinned multi-card H3 load (``_h3_card_reserve_bytes`` with ``single_card_only``)."""
def _h3_spare_user_reserve(
runtime: Any,
flags: "list[str]",
*,
free: Any = None,
read_free: bool = False,
) -> list[str]:
"""``flags`` with the ``--max-vram -1`` placeholder kept within a lowered VRAM Budget (unchanged at the default, off CUDA, or when the reserve is unread)."""
from .sd_cpp_args import (
GRAPH_CUT_VRAM_FLAGS,
graph_cut_budget_gib,
max_vram_is_placeholder,
max_vram_value,
with_max_vram_budget,
)
from .video_minimax_h3 import H3NativeMemoryRefusal
if not max_vram_is_placeholder(max_vram_value(flags)):
return list(flags)
probe = getattr(runtime, "vram_reserve_probe", None)
try:
reserve = probe() if probe is not None else None
except Exception: # noqa: BLE001 -- an unreadable reserve keeps sd.cpp's own reading
reserve = None
if not reserve or int(reserve) <= 0:
return list(flags)
multi_card = isinstance(reserve, _H3UnpinnedMultiCardReserve)
if read_free:
free_probe = getattr(runtime, "vram_free_probe", None)
try:
free = free_probe() if free_probe is not None else None
except Exception: # noqa: BLE001 -- an unreadable card keeps the single-card spare
free = None
if not multi_card:
if isinstance(free, dict):
small = next(
(v is not None for v in free.values() if graph_cut_budget_gib(v) is None), False
)
else:
small = free is not None and graph_cut_budget_gib(free) is None
free = free if small else None
def _refuse(shown_bytes: Optional[int]) -> H3NativeMemoryRefusal:
shown = (
"an unreadable amount of"
if shown_bytes is None
else (f"about {max(0.0, shown_bytes / float(1 << 30)):.1f} GiB of")
)
return H3NativeMemoryRefusal(
f"MiniMax-H3 has {shown} VRAM left inside the VRAM Budget on the render card, "
"too little for the graph-cut render. Free VRAM on that card, raise the VRAM Budget, or lower the "
"resolution or duration."
)
sd_spare_bytes = int(-float(GRAPH_CUT_VRAM_FLAGS[1]) * (1 << 30))
if multi_card:
readings = list(free.values()) if isinstance(free, dict) else [free]
if not readings or any(value is None for value in readings):
raise _refuse(None)
smallest = min(int(value) for value in readings)
# Rounded DOWN to 0.01 GiB, so the cap never exceeds the tightest card's budget-adjusted free less 1 GiB.
cap = max(0, (smallest - sd_spare_bytes) * 100 // (1 << 30)) / 100.0
if cap <= 0:
raise _refuse(smallest)
out = with_max_vram_budget(flags, cap)
logger.info(
"video.h3_native_vram_budget: --max-vram %s (unpinned multi-card: smallest budget-adjusted free %s "
"less sd.cpp's 1 GiB spare)",
max_vram_value(out),
smallest,
)
return out
# Rounded UP to 0.01 GiB, so the spare never undercuts the reserve; the refusal then checks THIS spare.
spare_hundredths = int(-float(GRAPH_CUT_VRAM_FLAGS[1]) * 100) + -(
-int(reserve) * 100 // (1 << 30)
)
spare_bytes = -(-spare_hundredths * (1 << 30) // 100)
if isinstance(free, dict):
known = [int(v) for v in free.values() if v is not None]
raise _refuse(min(known) if known else None)
if free is not None and int(free) + int(reserve) <= spare_bytes:
raise _refuse(int(free))
out = list(flags)
index = out.index("--max-vram") + 1
out[index] = f"-{spare_hundredths // 100}.{spare_hundredths % 100:02d}"
logger.info(
"video.h3_native_vram_budget: --max-vram %s (sd.cpp's free reading less the VRAM Budget reserve %s)",
out[index],
reserve,
)
return out
def _h3_restart_for_retry(slot: Any, reason: str) -> bool:
"""Stop for a retry unless a release is pending; under the slot lock so a racing release is not lost."""
restart = getattr(slot, "restart_for_retry", None)
if restart is not None:
return bool(restart(reason))
if getattr(slot, "release_pending", None) is not None:
return False
slot.stop(reason)
return True
def _refuse_h3_native_nan_render(path: Any) -> None:
"""Raise ``NonFiniteFramesError`` for an sd.cpp H3 clip that is a NaN render (``h3_native_clip_is_nan_render``):
sd.cpp writes it and exits 0. Off with UNSLOTH_VIDEO_NONFINITE_CHECK=0; a clip the probe cannot read is kept."""
if not nonfinite_check_enabled():
return
from .video_minimax_h3 import h3_native_clip_is_nan_render
try:
broken = h3_native_clip_is_nan_render(Path(path))
except Exception as exc: # noqa: BLE001 - the probe never fails a render on its own
logger.debug("video.h3_native_nonfinite_probe_failed: %s", exc)
return
if broken:
logger.warning(
"video.h3_native_nonfinite: sd.cpp returned a clip whose every frame is flat black / white and whose "
"soundtrack is one full-scale level, the output of a NaN render; refusing to save it"
)
raise NonFiniteFramesError(NON_FINITE_FRAMES_MSG)
def _h3_retry_vram_cap(launched: "Optional[Sequence[str]]", error: str) -> Any:
from .sd_cpp_args import (
GRAPH_CUT_RETRY_SCALE,
explicit_max_vram_gib,
is_device_oom_text,
scaled_vram_budget,
)
if not launched or not is_device_oom_text(error):
return None
budget = explicit_max_vram_gib(launched)
if budget is None:
return None
return scaled_vram_budget(budget, GRAPH_CUT_RETRY_SCALE)
def _h3_cuda_cc_fallback(
cuda_cc: "Optional[tuple[int, int]]",
*,
binary: Optional[str],
native_device: Optional[str],
listed_accelerator: Optional[bool],
accelerator_names: "Sequence[Any]",
device_name: Optional[str],
) -> "Optional[tuple[int, int]]":
"""torch's compute capability when ``--list-devices`` was unreadable (else sm80+ fell to exact kernels: 29.3 vs
~5 s/step). Unpinned reads visible ordinal 0, sd.cpp's default, never the lowest across a mixed host."""
from .sd_cpp_backend import (
_accelerator_class_of,
sd_cpp_device_listing_known,
torch_cuda_compute_capability,
)
if cuda_cc is not None or native_device == "cpu" or listed_accelerator is None:
return cuda_cc
if sd_cpp_device_listing_known(binary):
return cuda_cc
# Only the install record says what the build targets; an unowned SD_CLI_PATH build may be Vulkan / CPU.
if not any(
isinstance(name, str) and _accelerator_class_of(name) == "cuda"
for name in accelerator_names
):
return cuda_cc
ordinal: Optional[int] = 0
if device_name:
head = device_name.rstrip("0123456789")
if head.upper() != "CUDA" or not device_name[len(head) :]:
return cuda_cc
ordinal = int(device_name[len(head) :])
fallback = torch_cuda_compute_capability(ordinal)
logger.warning(
"video.h3_cuda_cc_fallback: sd-cli --list-devices was unreadable; compute capability %s from torch "
"(device %s)",
fallback,
device_name or "CUDA0, sd.cpp's default",
)
return fallback
def _pairs(flags: "Sequence[str]") -> "list[tuple[str, str]]":
items = list(flags or ())
return [(items[i], items[i + 1]) for i in range(len(items) - 1) if items[i].startswith("--")]
def _h3_one_shot_with_vram_retry(
runtime: Any,
run: "Callable[[list[str]], Any]",
render_flags: "Sequence[str]",
cancel: threading.Event,
) -> Any:
launch = _h3_budgeted_launch_flags(runtime, render_flags)
try:
return run(launch)
except RuntimeError as exc:
if cancel.is_set():
raise
cap = _h3_retry_vram_cap(launch, str(exc))
if cap is None:
raise
logger.warning(
"video.h3_native_vram_retry: device out of memory under --max-vram %s (sd-cli); retrying once with %s",
dict(_pairs(launch)).get("--max-vram"),
cap,
)
return run(_h3_budgeted_launch_flags(runtime, render_flags, cap = cap))
def _torch_pci_slot(ordinal: Optional[int]) -> Optional[str]:
"""``DDDD:BB:DD`` of the torch-visible card at *ordinal* (no function digit: torch does not report it)."""
if ordinal is None:
return None
try:
import torch
props = torch.cuda.get_device_properties(int(ordinal))
return f"{int(props.pci_domain_id):04x}:{int(props.pci_bus_id):02x}:{int(props.pci_device_id):02x}"
except Exception: # noqa: BLE001 - unknown means every amdgpu card is a candidate
return None
def _sd_cli_identity(binary: Optional[str]) -> Optional[tuple[int, int]]:
"""Size and mtime_ns of the sd.cpp binary, with no subprocess; None counts as a change."""
if not binary:
return None
try:
stat = Path(binary).stat()
except OSError:
return None
return (stat.st_size, stat.st_mtime_ns)
def _note_sd_cpp_accelerator_failure(
binary: Optional[str],
output: str,
*,
card: Optional[str] = None,
) -> None:
try:
from .sd_cpp_backend import note_accelerator_failure_from_output
note_accelerator_failure_from_output(binary, output, source = "video", card = card)
except Exception as exc: # noqa: BLE001
logger.debug("could not record the sd.cpp accelerator failure: %s", exc)
# "Not read yet", distinct from None, the real answer for a binary the installer does not own.
_UNREAD_ACCELERATOR = object()
# _close_pending_ltx25_encoder's default: stop the pending worker whichever load started it.
_ANY_LOAD = object()
def _h3_te_canonical(repo_id: Optional[str]) -> str:
"""A repo id normalised for an EXACT identity compare: mirrors folded onto the id they copy,
then trimmed and lowercased. Deliberately no tail-segment tolerance -- ``someone/MiniMax-H3``
is a different repo with different weights."""
from .diffusion_families import canonical_base
return canonical_base(repo_id).strip().lower()
def _video_fetch_repo(
repo_id: Optional[str],
hf_token: Optional[str] = None,
*,
files: Optional[Sequence[str]] = None,
checkpoint: bool = False,
) -> Optional[str]:
"""The repo a video download should hit for ``repo_id``: its byte-identical unsloth mirror when one exists
(``diffusion_families._VIDEO_MIRROR_PAIRS``), else ``repo_id`` unchanged.
FETCH sites only, exactly like the image side's ``prefer_ungated_mirror`` calls: the upstream id stays the
logical identity (picker, status(), prequant / te-prequant base checks, size tables), and every table keyed on
it normalises a mirror back through ``canonical_base``. Same opt-outs too: a local path, an upstream already
satisfying the load from cache (``files`` sharpens that to the names about to be fetched), and
``UNSLOTH_DIFFUSION_NO_MIRROR``. Pure: table lookup, env read, local stat."""
if not repo_id:
return repo_id
# ``checkpoint``: the user-selected file itself. The opt-out folds a mirror back onto its upstream only for BASE
# components; an unsloth carrier repo's own root FP8 / GGUF checkpoints do not exist upstream, so a pick stays in
# the repo it was selected from.
no_mirror = bool(os.environ.get("UNSLOTH_DIFFUSION_NO_MIRROR", "").strip())
if checkpoint and no_mirror:
return repo_id
from .diffusion_families import (
_is_local_path,
_upstream_is_cached,
mirror_repo,
prefer_ungated_mirror,
)
mirror = None if no_mirror or _is_local_path(repo_id) else mirror_repo(repo_id)
if mirror:
if files:
# Every video fetch passes ``reuse_other_cache_root``, so an upstream complete under EITHER root is reused
# rather than re-staged under the mirror (a cache-folder change leaves it in the import-time root).
if _upstream_whole_in_one_root(repo_id, tuple(files)):
return repo_id
else:
# No names to check (assembly, an offline load, progress): the any-weight default would keep a partial
# upstream over a complete mirror. Pick the side whose cached snapshot is more complete; upstream on a tie.
# A GATED upstream wins only when its snapshot is whole: anything missing would be fetched from the vendor
# repo, a 401 for a user without its licence, while the mirror serves the same bytes to everyone. A load
# that cannot download keeps the cached upstream unless the mirror is cached too (it may read only the
# upstream's cached part, e.g. a GGUF pick that skips the transformer).
(up, up_whole), mir = (
_video_cached_completeness_detail(repo_id),
_video_cached_completeness(mirror),
)
if up >= 0 or mir >= 0:
if (
not up_whole
and _is_gated_video_upstream(repo_id)
and (mir >= 0 or _can_download())
):
return mirror
return repo_id if up >= mir else mirror
return prefer_ungated_mirror(repo_id, hf_token, files = files)
def _is_gated_video_upstream(repo_id: str) -> bool:
from .diffusion_families import _GATED_UPSTREAMS
return (repo_id or "").strip().lower() in _GATED_UPSTREAMS
def _can_download() -> bool:
from .diffusion_prequant import _LOCAL_FILES_ONLY, hub_offline
return not hub_offline() and not _LOCAL_FILES_ONLY.get()
def _video_index_repo(base: str, *, local_files_only: bool = False) -> str:
"""The repo to read ``base``'s index from: the one its base pull will stage. The plan keeps a cached upstream only
when it already holds the files the pull needs, so the upstream answers only when its snapshot holds every
component (its ``text_encoder`` aside: an H3 load may skip those shards); else the mirror, unless the load cannot
download and only the upstream is cached."""
from .diffusion_families import _is_local_path, mirror_repo
if os.environ.get("UNSLOTH_DIFFUSION_NO_MIRROR", "").strip() or _is_local_path(base):
return base
mirror = mirror_repo(base)
if not mirror:
return base
up, up_whole = _video_cached_completeness_detail(base, skip = ("text_encoder",))
if up_whole:
return base
if (
up >= 0
and (local_files_only or not _can_download())
and _video_cached_completeness(mirror) < 0
):
return base
return mirror
def _video_offline_pinned_snapshot(
base: Optional[str], fetch_repo: Optional[str] = None
) -> Optional[tuple[str, str]]:
"""``(repo, commit)`` of the one cached pipeline snapshot of ``base``'s mirror pair (``fetch_repo`` first) when
that repo has no ``refs/main``: a fetch pinned to a commit writes only ``snapshots/<commit>``, which an unpinned
offline lookup cannot resolve. None when a repo has ``refs/main`` (it resolves as is), when no or several
snapshots hold a pipeline index (ambiguous), or for a local path."""
from .diffusion_families import _is_local_path, mirror_repo
repos: list[str] = []
for repo in (fetch_repo, base, mirror_repo(base)):
if repo and not _is_local_path(repo) and repo.lower() not in [r.lower() for r in repos]:
repos.append(repo)
root = Path(hub_cache_dir())
found: list[tuple[str, str]] = []
for repo in repos:
folder = root / ("models--" + repo.replace("/", "--"))
if not folder.is_dir():
continue
if (folder / "refs" / "main").is_file():
return None
snaps = folder / "snapshots"
if not snaps.is_dir():
continue
holding = [
d.name
for d in snaps.iterdir()
if d.is_dir()
and any(
(d / name).is_file() for name in ("model_index.json", "modular_model_index.json")
)
]
if len(holding) > 1:
return None
if holding:
found.append((repo, holding[0]))
return found[0] if len(found) == 1 else None
def _video_offline_cache_miss(exc: BaseException) -> bool:
return isinstance(exc, OSError) or bool(getattr(exc, "cache_miss", False))
def _video_cached_while_hub_unreachable(
base_local: Optional[str],
base_repo: Optional[str],
checkpoint_repo: Optional[str],
checkpoint_file: Optional[str],
*,
revision: Optional[str] = None,
family: Any = None,
) -> bool:
if base_local or not base_repo or not hub_reachability.hub_unreachable():
return False
try:
if Path(base_repo).expanduser().exists():
return False
import json
from huggingface_hub import try_to_load_from_cache
from core.inference.diffusion import _cached_while_hub_unreachable
from .diffusion_families import _local_pipeline_component_is_complete, canonical_base
from .diffusion_offline_source import offline_snapshot_source
manifest = None
manifests = (
("modular_model_index.json", "model_index.json")
if getattr(family, "modular_workflow", None)
else ("model_index.json", "modular_model_index.json")
)
for name in manifests:
hit = try_to_load_from_cache(
base_repo, name, cache_dir = hub_cache_dir(), revision = revision
)
if isinstance(hit, str):
manifest = Path(hit)
break
if manifest is None:
return False
metadata_types = (
("tokenizer",)
if getattr(family, "name", None) == "ltx-2.5"
else ("tokenizer", "processor")
)
for name, spec in json.loads(manifest.read_text(encoding = "utf-8")).items():
if not isinstance(spec, (list, tuple)) or len(spec) < 2:
continue
library, cls = spec[:2]
if (
library != "transformers"
or not isinstance(cls, str)
or not any(kind in cls.lower() for kind in metadata_types)
):
continue
source = (
spec[2]
if manifest.name == "modular_model_index.json"
and len(spec) > 2
and isinstance(spec[2], dict)
else {}
)
source_repo = source.get("pretrained_model_name_or_path") or source.get("repo")
repo = source_repo or base_repo
same_base = canonical_base(repo) == canonical_base(base_repo)
repo = base_repo if same_base else repo
subfolder = source.get("subfolder", "" if source_repo else name) or ""
root = offline_snapshot_source(
repo,
subfolder,
local_files_only = True,
cache_dir = hub_cache_dir(),
revision = source.get("revision") or (revision if same_base else None),
)
if not _local_pipeline_component_is_complete(
Path(root) / subfolder, library, cls, False
):
return False
if checkpoint_file and checkpoint_repo and not Path(checkpoint_repo).expanduser().exists():
return _cached_while_hub_unreachable(checkpoint_repo, checkpoint_file)
return True
except Exception: # noqa: BLE001 - any doubt keeps the online load
return False
def _video_offline_base_choice(
base: str,
folders: Optional[Sequence[str]] = None,
skip: Sequence[str] = (),
) -> Optional[str]:
"""Offline (no plan to read the choice from): the side of ``base``'s mirror pair whose cached snapshot holds more of
the ``folders`` this load reads (every index component but ``skip`` when None), upstream on a tie; None when
neither is cached or there is no mirror."""
from .diffusion_families import _is_local_path, mirror_repo
if os.environ.get("UNSLOTH_DIFFUSION_NO_MIRROR", "").strip() or _is_local_path(base):
return None
mirror = mirror_repo(base)
if not mirror:
return None
up = _video_cached_completeness(base, folders, skip)
mir = _video_cached_completeness(mirror, folders, skip)
if up < 0 and mir < 0:
return None
return base if up >= mir else mirror
def _upstream_whole_in_one_root(repo_id: str, files: tuple) -> bool:
"""Whether ONE cache root (the live one, or the import-time root the fetches fall back to) holds every name in one
revision: the only upstream cache assembly can open as a single snapshot. Names split across the two roots are not
a hit, so the load takes the mirror instead of a set that assembly sees half of (a 401 on a gated upstream)."""
from .diffusion_families import _root_holds_upstream
roots = [Path(hub_cache_dir())]
try:
from huggingface_hub import constants
if Path(constants.HF_HUB_CACHE) != roots[0]:
roots.append(Path(constants.HF_HUB_CACHE))
except Exception: # noqa: BLE001 -- the live root alone
pass
return any(_root_holds_upstream(root, repo_id, files) for root in roots)
def _h3_mirrored_component_sources(
pipe: Any,
base: Optional[str],
fetch_repo: Optional[str],
*,
local_files_only: bool,
cache_dir: Optional[str] = None,
staged_dir: Optional[str] = None,
) -> dict[str, str]:
"""``{component: source}`` sending every modular spec that names ``base`` (either side of its mirror pair) to
``fetch_repo``, the repo the base was staged from, as ``load_components(pretrained_model_name_or_path = ...)``.
``staged_dir``, a snapshot the plan reused from another cache root, takes those specs instead: load_components
searches only the live root and would fetch the ~145 GB of components again.
Offline, tokenizer / processor specs resolve to that repo's cached snapshot folder (as
``offline_component_sources`` does). Empty when nothing is redirected: a local base, no mirror in play, or a
spec naming a different repo (a derivative's own encoder keeps its source)."""
from .diffusion_families import _is_local_path
if not base or not fetch_repo or _is_local_path(fetch_repo):
return {}
specs = getattr(pipe, "_component_specs", None)
if not isinstance(specs, dict):
return {}
from .diffusion_offline_source import _is_tokenizer_or_processor, offline_snapshot_source
want = _h3_te_canonical(base)
out: dict[str, str] = {}
for name, spec in specs.items():
source = getattr(spec, "pretrained_model_name_or_path", None)
if not isinstance(source, str) or not source or (source == fetch_repo and not staged_dir):
continue
if _h3_te_canonical(source) != want:
continue
if staged_dir:
out[name] = staged_dir
continue
target = fetch_repo
if _is_tokenizer_or_processor(getattr(spec, "type_hint", None)):
target = offline_snapshot_source(
fetch_repo,
getattr(spec, "subfolder", None),
local_files_only = local_files_only,
cache_dir = cache_dir,
)
out[name] = target
return out
def _is_under_dir(path: str, root: Optional[str]) -> bool:
try:
return bool(root) and Path(path).resolve().is_relative_to(Path(str(root)).resolve())
except (OSError, ValueError):
return False
def _video_cached_completeness(
repo_id: str,
folders: Optional[Sequence[str]] = None,
skip: Sequence[str] = (),
) -> int:
"""How complete ``repo_id``'s cached diffusers snapshot is in the live root (the one ``from_pretrained`` reads):
the number of ``model_index.json`` (or H3's ``modular_model_index.json``) components whose folder holds a file and
whose shard sets are fully present, best revision. -1 without a cached index. Local stat only, never raises."""
return _video_cached_completeness_detail(repo_id, folders, skip)[0]
_SHARD_NAME = re.compile(r"^(?P<stem>.+)-(?P<part>\d+)-of-(?P<total>\d+)(?P<ext>\.[^.]+)$")
def _shards_complete(folder: Path, names: Sequence[str]) -> bool:
"""Whether every ``<stem>-NNNNN-of-MMMMM<ext>`` set among ``names`` has all M parts in ``folder``: an interrupted
pull can leave one shard and no index, which the index check alone cannot see."""
for name in names:
m = _SHARD_NAME.match(name)
if not m:
continue
width, total = len(m["part"]), int(m["total"])
for part in range(1, total + 1):
if not (folder / f"{m['stem']}-{part:0{width}d}-of-{m['total']}{m['ext']}").is_file():
return False
return True
def _video_cached_completeness_detail(
repo_id: str,
folders: Optional[Sequence[str]] = None,
skip: Sequence[str] = (),
) -> tuple[int, bool]:
"""``(best score, whether some cached revision holds EVERY component it lists)``; ``(-1, False)`` when nothing is
cached. The score is ``_video_cached_completeness``."""
try:
import json
from .diffusion_families import _WEIGHT_SUFFIXES, _cached_revisions
best, whole = -1, False
for rev in _cached_revisions(Path(hub_cache_dir()), repo_id):
index = next(
(
rev / n
for n in ("model_index.json", "modular_model_index.json")
if (rev / n).is_file()
),
None,
)
if index is None:
continue
components = []
for key, value in json.loads(index.read_text(encoding = "utf-8")).items():
if key.startswith("_") or not isinstance(value, (list, tuple)) or not value:
continue
if value[0] in (None, "null"):
continue
spec = value[2] if len(value) == 3 and isinstance(value[2], dict) else {}
components.append(spec.get("subfolder") or key)
if folders is not None:
# Only what this load reads from the base (LTX-2.3: its scheduler, tokenizer and encoder).
components = [c for c in components if c in folders]
# A single-file / GGUF pick brings its own denoiser: the base's transformer folders are not read.
components = [c for c in components if not any(c.startswith(p) for p in skip)]
score = 0
for key in components:
folder = rev / key
if not folder.is_dir() or not any(p.is_file() for p in folder.iterdir()):
continue
complete = True
# A model component (it has a config.json) needs real weights, not just the configs an interrupted
# pull leaves; scheduler / tokenizer folders carry no weights and count on any file.
weights = [
p.name
for p in folder.iterdir()
if p.is_file() and p.suffix.lower() in _WEIGHT_SUFFIXES
]
if (folder / "config.json").is_file() and not weights:
complete = False
elif not _shards_complete(folder, weights):
complete = False
for shard_index in folder.glob("*.index.json"):
weight_map = (
json.loads(shard_index.read_text(encoding = "utf-8")).get("weight_map") or {}
)
if not all((folder / name).is_file() for name in set(weight_map.values())):
complete = False
break
score += complete
best = max(best, score)
whole = whole or score == len(components)
return best, whole
except Exception: # noqa: BLE001 -- an unreadable cache scores as absent
return -1, False
def _video_listing_repo(repo_id: str) -> str:
"""The repo to read ``repo_id``'s file LISTING / sizes from: its mirror whenever one exists, cached upstream or
not, since the two listings are identical and only the mirror keeps the metadata call inside unsloth/*. Honours
``UNSLOTH_DIFFUSION_NO_MIRROR`` and never rewrites a local path."""
from .diffusion_families import _is_local_path, mirror_repo
if (
not repo_id
or os.environ.get("UNSLOTH_DIFFUSION_NO_MIRROR", "").strip()
or _is_local_path(repo_id)
):
return repo_id
return mirror_repo(repo_id) or repo_id
def _video_repo_aliases(repo_id: Optional[str]) -> tuple[str, ...]:
"""``repo_id`` plus the other side of its mirror pair, for the guards that must protect whichever id a load is
actually reading from (the delete-cached guard, the progress cache scan)."""
if not repo_id:
return ()
from .diffusion_families import canonical_base, mirror_repo
upstream = canonical_base(repo_id)
return tuple(dict.fromkeys(r for r in (repo_id, upstream, mirror_repo(upstream)) if r))
def _video_fetch_key(repo_id: Optional[str]) -> str:
"""A mirror pair's key: the upstream id, lowercased (load_progress dedupes its no-plan fallback on it)."""
from .diffusion_families import canonical_base
return canonical_base(repo_id).strip().lower()
# MiniMax-H3 modular precision tri-state. unset/"auto" -> whatever the backend decided;
# ── MiniMax-H3 modular precision tri-state ──────────────────────────────────── The modular workflow builds each
# component through its own from_pretrained, so nothing dense can be quantised in place: a hosted checkpoint is the only
# way to run a component small, which makes "unset" a real choice rather than a no-op. unset / "auto" -> whatever the
# backend decided for that component "none" / "off" -> the released bfloat16 component, pinned a scheme -> that scheme,
# or a recorded fallback The two components decided it differently on measured quality (see each site): the CONDITIONER
# defaults to the hosted quantized artifact, which preserves the sample; the DENOISER keeps the released weights,
# because both hosted checkpoints re-roll it. These helpers are the one place the tri-state is read, so the plan, the
# estimate, the pull and the load cannot disagree.
# ── MiniMax-H3 modular precision tri-state ────────────────────────────────────
def _h3_precision_unset(value: Optional[str]) -> bool:
"""True when the caller left this precision to the backend."""
return value is None or str(value).strip().lower() in ("", "auto")
def _h3_precision_pinned_dense(value: Optional[str]) -> bool:
"""True when the caller explicitly asked for the released bfloat16 component."""
return value is not None and str(value).strip().lower() in ("none", "off")
def _h3_auto_precision_target(target: Any = None) -> Any:
"""The device target the auto choice is made against. Never raises."""
if target is not None:
return target
try:
return resolve_diffusion_device_target()
except Exception: # noqa: BLE001 -- an unanswerable probe keeps the dense components
return None
def _h3_auto_precision_ok(target: Any = None) -> bool:
"""Unset H3 precision uses hosted parts only on CUDA with bfloat16; MPS cannot reach that loader."""
resolved = _h3_auto_precision_target(target)
if resolved is None:
return False
try:
import torch
return (
getattr(resolved, "device", None) == "cuda"
and getattr(resolved, "dtype", None) is torch.bfloat16
)
except Exception: # noqa: BLE001 -- no torch means no diffusers path to optimise
return False
def h3_streamed_int8_supported(target: Any = None) -> bool:
"""Whether this host can run H3's hosted INT8 denoiser streamed: the auto gate (CUDA bf16, so
sm_80+ on NVIDIA), torchao-aware group offload, and a loader that can read the checkpoint
(else auto keeps bf16 and the tier's load is refused). Never raises."""
if not _h3_auto_precision_ok(target):
return False
try:
from .diffusion_prequant import (
restricted_prequant_load_supported,
torchao_group_offload_supported,
)
return bool(torchao_group_offload_supported()) and bool(
restricted_prequant_load_supported(H3_AUTO_FALLBACK_SCHEME)
)
except Exception: # noqa: BLE001 -- an unanswerable probe hides the tier
return False
def _h3_others_bytes(
fam: Any,
components: Any,
*,
te_scheme: Optional[str],
dtype: Any,
rotating: bool,
te_streamed: bool = False,
) -> int:
"""Bytes beside the denoiser. ``rotating``: conditioner and VAEs rotate (the 40 GB reserve evicts
one before placing the other), so only the larger counts; otherwise they add up. ``te_streamed``: the
conditioner counts at its streamed encode footprint."""
import torch
scale = 2.0 if dtype is torch.float32 else 1.0
if te_streamed:
from .video_minimax_h3_te import H3_TE_STREAMED_GB
text_encoder_gb = H3_TE_STREAMED_GB
else:
text_encoder_gb = h3_te_resident_gb(te_scheme, bf16_gb = components[1])
headroom_bytes = (
estimate_video_runtime_mib(
width = fam.resolution_presets[0][0],
height = fam.resolution_presets[0][1],
num_frames = fam.default_num_frames,
)
* 1024
* 1024
)
beside = max(text_encoder_gb, components[2]) if rotating else text_encoder_gb + components[2]
return int(beside * scale * 1000.0**3) + headroom_bytes
def _h3_dense_denoiser_resident_bytes(
fam: Any,
*,
denoiser: Any,
te_scheme: Optional[str],
dtype: Any,
rotating: bool = False,
te_streamed: bool = False,
) -> Optional[tuple[int, int]]:
"""Denoiser size is measured from the built module, not the family table's released dense figure."""
components = getattr(fam, "bf16_components_gb", None)
if not components or denoiser is None:
return None
try:
from itertools import chain
from .diffusion_prequant import tensor_payload_bytes
denoiser_bytes = sum(
tensor_payload_bytes(t)
for t in chain(denoiser.parameters(), denoiser.buffers())
if not getattr(t, "is_meta", False)
)
if denoiser_bytes <= 0:
return None
others = _h3_others_bytes(
fam,
components,
te_scheme = te_scheme,
dtype = dtype,
rotating = rotating,
te_streamed = te_streamed,
)
return int(denoiser_bytes), others
except Exception: # noqa: BLE001 -- an unanswerable estimate keeps the rotation as it is
return None
def _h3_planned_denoiser_bytes(
fam: Any,
*,
te_scheme: Optional[str],
dtype: Any,
rotating: bool = False,
) -> Optional[tuple[int, int]]:
"""Predicted from the table, not a built module, so the seeding decision is made before any fetch."""
components = getattr(fam, "bf16_components_gb", None)
if not components:
return None
try:
import torch
scale = 2.0 if dtype is torch.float32 else 1.0
denoiser_bytes = int(components[0] * scale * 1000.0**3)
others = _h3_others_bytes(
fam, components, te_scheme = te_scheme, dtype = dtype, rotating = rotating
)
return denoiser_bytes, others
except Exception: # noqa: BLE001 -- an unanswerable estimate keeps the released denoiser
return None
def _user_reserve_bytes(device: Any, total_bytes: Optional[int]) -> int:
"""What the user's VRAM Budget keeps free on ``device`` (total ``total_bytes``), in bytes."""
from .diffusion_memory import user_budget_reserve_bytes
return user_budget_reserve_bytes(device, total_bytes)
def _planned_free_less_user_reserve_mib(memory: Any) -> Optional[int]:
"""The load-time placement guard's free MiB."""
free = getattr(memory, "free_mib", None)
if free is None:
return None
from .diffusion_memory import _device_user_budget_reserve_mib
return int(free) - _device_user_budget_reserve_mib(
memory, getattr(memory, "total_mib", None) or free
)
def _request_available_bytes(device_obj: Any, module: Any) -> int:
"""A request guard's available VRAM on ``device_obj``."""
from utils.hardware import trusted_mem_get_info
free_bytes, total_bytes = trusted_mem_get_info(device_obj, module = module)
reserved_bytes = module.memory_reserved(device_obj) if hasattr(module, "memory_reserved") else 0
return int(free_bytes) + int(reserved_bytes) - _user_reserve_bytes(device_obj, total_bytes)
def _h3_host_copy_follows(device: Any) -> bool:
"""True on a discrete GPU (pinning allowed): H3's prequant denoiser is copied off its host load."""
if str(device or "").strip().lower() in ("", "cpu", "meta"):
return False
try:
from .host_pinning import host_pinning_allowed
return bool(host_pinning_allowed(device))
except Exception: # noqa: BLE001
return False
def _h3_free_device_bytes(device: str) -> Optional[int]:
"""Live free VRAM, or None when it cannot be read. None means "cannot tell", never zero."""
free = _h3_signed_free_device_bytes(device)
return None if free is None else max(0, free)
def _h3_signed_free_device_bytes(device: str) -> Optional[int]:
""":func:`_h3_free_device_bytes` before the clamp at 0."""
if device != "cuda":
return None
try:
from utils.hardware import trusted_mem_get_info
from .diffusion_memory import _user_budget_reserve_mib
# Windows ROCm over-reports free (#8403); pinning the denoiser on that reading is how a card that is already
free, total = trusted_mem_get_info()
return int(free) - (_user_budget_reserve_mib(int(total) >> 20) << 20)
except Exception: # noqa: BLE001 -- an unreadable card decides nothing
return None
def _h3_card_memory_bytes(
device: Optional[str],
ordinal: Optional[int],
*,
allow_context: bool = True,
) -> tuple[Optional[int], Optional[int]]:
"""(free, total) VRAM of the H3 render card, or (None, None). No pin = ordinal 0. nvidia-smi / ROCm KFD + sysfs
first (no CUDA context); torch only when none answers and ``allow_context``, as the residency and pressure reads
always did. The pre-spawn budget passes False: it never pays for a context. Free is less the user's VRAM Budget
reserve (raw at the default), so the sd.cpp resident-vs-offload pick, its ``--max-vram`` probe and the
server-pressure check all keep that share free."""
if device != "cuda":
return None, None
found = _h3_read_card_memory_bytes(ordinal, no_context = True)
if found == (None, None) and allow_context:
found = _h3_read_card_memory_bytes(ordinal, no_context = False)
return found
def _h3_read_card_memory_bytes(
ordinal: Optional[int], *, no_context: bool
) -> tuple[Optional[int], Optional[int]]:
try:
from utils.hardware import get_visible_gpu_utilization, gpu_query
with gpu_query.fresh_reads():
devices = get_visible_gpu_utilization(no_context = no_context).get("devices", [])
want = 0 if ordinal is None else int(ordinal)
for entry in devices:
if entry.get("visible_ordinal") != want:
continue
total = entry.get("vram_total_gb")
used = entry.get("vram_used_gb")
if total is None or used is None:
return None, None
total_bytes = int(float(total) * 1024**3)
free_bytes = int((float(total) - float(used)) * 1024**3)
return max(0, free_bytes - _user_reserve_bytes("cuda", total_bytes)), total_bytes
except Exception: # noqa: BLE001 -- an unreadable card decides nothing
return None, None
return None, None
def _h3_card_free_bytes(
device: Optional[str],
ordinal: Optional[int],
*,
single_card_only: bool = False,
allow_context: bool = True,
) -> Optional[int]:
"""Free VRAM of the H3 render card, or None."""
if single_card_only and _h3_visible_card_count(device) != 1:
readings = _h3_unpinned_multi_card_readings(device)
if not readings or any(free is None for free, _total in readings):
return None
return min(free for free, _total in readings)
return _h3_card_memory_bytes(device, ordinal, allow_context = allow_context)[0]
def _h3_unpinned_multi_card_readings(
device: Optional[str],
) -> Optional[list[tuple[Optional[int], int]]]:
"""(budget-adjusted free, total) bytes of every visible card for an unpinned multi-card H3 render under a lowered VRAM Budget, or None at the default / off CUDA / unread."""
if device != "cuda":
return None
from .diffusion_memory import user_budget_reserve_share
if user_budget_reserve_share() <= 0:
return None
try:
from utils.hardware import get_visible_gpu_utilization, gpu_query
with gpu_query.fresh_reads():
devices = get_visible_gpu_utilization(no_context = True).get("devices", [])
out: list[tuple[Optional[int], int]] = []
for entry in devices:
total = entry.get("vram_total_gb")
if total is None:
return None
total_bytes = int(float(total) * 1024**3)
used = entry.get("vram_used_gb")
free = (
None
if used is None
else max(
0,
int((float(total) - float(used)) * 1024**3)
- _user_reserve_bytes("cuda", total_bytes),
)
)
out.append((free, total_bytes))
except Exception: # noqa: BLE001 -- an unreadable card decides nothing
return None
return out or None
def _h3_card_reserve_bytes(
device: Optional[str],
ordinal: Optional[int],
*,
single_card_only: bool = False,
) -> Optional[int]:
"""Bytes the user's VRAM Budget keeps free on the H3 render card."""
if device != "cuda":
return 0
from .diffusion_memory import user_budget_reserve_share
if user_budget_reserve_share() <= 0:
return 0
try:
from utils.hardware import get_visible_gpu_utilization, gpu_query
with gpu_query.fresh_reads():
devices = get_visible_gpu_utilization(no_context = True).get("devices", [])
want = 0 if ordinal is None else int(ordinal)
totals = [
int(float(entry["vram_total_gb"]) * 1024**3)
for entry in devices
if entry.get("vram_total_gb") is not None
and ((single_card_only and len(devices) != 1) or entry.get("visible_ordinal") == want)
]
except Exception: # noqa: BLE001 -- an unreadable card decides nothing
return None
if not totals:
return None
smallest = min(_user_reserve_bytes("cuda", total) for total in totals)
if single_card_only and len(devices) != 1:
return _H3UnpinnedMultiCardReserve(smallest)
return smallest
def _h3_visible_card_count(device: Optional[str]) -> Optional[int]:
if device != "cuda":
return None
try:
from utils.hardware import get_visible_gpu_utilization, gpu_query
with gpu_query.fresh_reads():
return len(get_visible_gpu_utilization(no_context = True).get("devices", []))
except Exception: # noqa: BLE001 -- unknown count reads as not single
return None
def _h3_native_server_pressure(device: Optional[str], ordinal: Optional[int]) -> Optional[str]:
"""Why the resident H3 sd-server should not outlive a render (``h3_native_server_pressure``), or None."""
from .diffusion_memory import _available_system_memory_mib, _host_ram_capacity_mib
from .video_minimax_h3 import h3_native_server_pressure
vram_free, vram_total = _h3_card_memory_bytes(device, ordinal)
try:
# Available RAM is cgroup-capped, so the reserve must be a share of the same limit.
host_total_mib = _host_ram_capacity_mib()
host_avail_mib = _available_system_memory_mib()
except Exception: # noqa: BLE001 -- unknown host memory decides nothing
host_total_mib = host_avail_mib = None
return h3_native_server_pressure(
vram_free = vram_free,
vram_total = vram_total,
host_available = None if host_avail_mib is None else int(host_avail_mib) << 20,
host_total = None if host_total_mib is None else int(host_total_mib) << 20,
)
def _h3_device_capacity_bytes(device: str) -> Optional[int]:
"""TOTAL VRAM on the card, or None when it cannot be read.
The reading for a decision made before the download, where the free one lies: the resident
pipeline is not torn down until ``load_pipeline`` runs, so a free reading taken while the plan
is being built describes the OLD model's occupancy, not this load's. Capacity cannot -- it is
an upper bound on any later free reading, so "does not fit in the whole card" still holds when
the loader asks again against live free memory. Less the user's VRAM Budget reserve, which
``_h3_free_device_bytes`` charges at placement, so planning and placement cannot disagree."""
if device != "cuda":
return None
try:
import torch
from .diffusion_memory import _user_budget_reserve_mib
total = int(torch.cuda.mem_get_info()[1])
return max(0, total - (_user_budget_reserve_mib(total >> 20) << 20))
except Exception: # noqa: BLE001 -- an unreadable card decides nothing
return None
def _h3_dense_denoiser_fits(sizes: Optional[tuple[int, int]], free_bytes: Optional[int]) -> bool:
"""Pin the released denoiser only if free memory covers it plus co-resident work; a wrong pin is OOM."""
if sizes is None or free_bytes is None:
return False
denoiser_bytes, others_bytes = sizes
return int(free_bytes) >= int(denoiser_bytes) + int(others_bytes)
def _h3_dense_pin_plan(
rotating_sizes: Optional[tuple[int, int]],
streamed_sizes: Optional[tuple[int, int]],
free_bytes: Optional[int],
) -> Optional[str]:
"""``"pin"`` (fits beside a rotating conditioner), ``"pin_stream_te"`` (only beside a streamed one) or None."""
if _h3_dense_denoiser_fits(rotating_sizes, free_bytes):
return "pin"
if _h3_dense_denoiser_fits(streamed_sizes, free_bytes):
return "pin_stream_te"
return None
def _h3_dense_demote(
pipe: Any,
denoiser: Any,
*,
logger: Any = None,
) -> bool:
"""Move a streamed-conditioner-pinned denoiser back into the CPU-offload rotation; False = unchanged."""
try:
manager = getattr(pipe, "_components_manager", None)
hooks = list(getattr(manager, "model_hooks", None) or ())
if any(getattr(hook, "model", None) is denoiser for hook in hooks):
return True
device = getattr(manager, "_auto_offload_device", None)
name = next(
(k for k, v in (getattr(manager, "components", None) or {}).items() if v is denoiser),
None,
)
if denoiser is None or not hooks or device is None or name is None:
return False
from .diffusion_memory import _available_system_memory_mib
from .diffusion_prequant import tensor_payload_bytes
need_mib = sum(tensor_payload_bytes(t) for t in denoiser.parameters()) >> 20
available_mib = _available_system_memory_mib()
if available_mib is None or available_mib < need_mib + max(4096, need_mib // 10):
return False
from diffusers.modular_pipelines.components_manager import custom_offload_with_hook
hook = custom_offload_with_hook(
name, denoiser, device, offload_strategy = getattr(manager, "_offload_strategy", None)
)
for other in hooks:
if other.hook.execution_device == hook.hook.execution_device:
hook.add_other_hook(other)
other.add_other_hook(hook)
manager.model_hooks = hooks + [hook]
hook.offload()
clear_gpu_cache()
if logger is not None:
logger.info(
"video.h3_denoiser_placement: this request needs the rotation's VRAM floor, so the bf16 denoiser "
"rejoins the CPU-offload rotation until the next load"
)
return True
except Exception as exc: # noqa: BLE001 -- the request guard refuses as before
if logger is not None:
logger.warning("video.h3_denoiser_placement: could not rejoin the rotation: %s", exc)
return False
def _h3_group_offload_available() -> bool:
try:
from diffusers.hooks import apply_group_offloading # noqa: F401
except Exception: # noqa: BLE001 -- no group offloading: the conditioner cannot stream
return False
return True
def _h3_dense_unpin(
manager: Any,
denoiser: Any,
device: Any,
*,
logger: Any = None,
) -> None:
"""Return a pinned released denoiser to the CPU-offload rotation (re-hooks every component)."""
try:
denoiser.to("cpu")
manager.disable_auto_cpu_offload()
manager.enable_auto_cpu_offload(device = device, memory_reserve_margin = "40GB")
except Exception as exc: # noqa: BLE001 -- the load continues with what placement it has
logger.warning(
"video.h3_denoiser_placement: could not return the denoiser to the rotation: %s", exc
)
def _h3_residency_budget_bytes(
residency: Any,
device: str,
*,
width: int,
height: int,
frames: int,
te_streamed: bool,
te_scheme: Optional[str],
fragmentation: bool = True,
top_gb: float = 0.0,
) -> int:
"""Device bytes the resident set may occupy for this request."""
try:
from .diffusion_memory import cuda_reusable_cache_bytes
from .video_minimax_h3 import H3_TEXT_ENCODER_BF16_GB
from .video_minimax_h3_residency import h3_phase_need_gb
from .video_minimax_h3_te import H3_TE_STREAMED_GB
# Signed: a card already inside the budget reserve owes that deficit before cache and resident groups count.
free = _h3_signed_free_device_bytes(device)
if free is None:
return 0
cached = cuda_reusable_cache_bytes(device)
# The conditioner's device share is demotable like a resident block (and it yields first).
held = getattr(residency, "device_bytes", None)
available = (
int(free) + cached + int(held() if callable(held) else residency.resident_bytes())
)
te_gb = (
H3_TE_STREAMED_GB
if te_streamed
else h3_te_resident_gb(te_scheme, bf16_gb = H3_TEXT_ENCODER_BF16_GB)
)
need = h3_phase_need_gb(
width,
height,
frames,
te_streamed_gb = te_gb,
fragmentation = fragmentation,
top_gb = top_gb,
)
return available - int(need * 1e9)
except Exception: # noqa: BLE001
return 0
def _h3_resident_pin_deferral(
fam: Any, denoiser: Any, device: str, *, te_streamed: bool, te_scheme: Optional[str]
) -> Optional[int]:
"""Denoiser bytes if the initial fit is predicted to hold it whole (so pins can be deferred), else None."""
try:
from .video_minimax_h3_residency import (
h3_defer_resident_pin_enabled,
h3_dit_resident_enabled,
h3_resident_unpin_enabled,
)
if denoiser is None or not str(device).startswith("cuda"):
return None
if not (
h3_defer_resident_pin_enabled()
and h3_dit_resident_enabled()
and h3_resident_unpin_enabled()
):
return None
from itertools import chain
from .diffusion_prequant import tensor_payload_bytes
total = sum(
tensor_payload_bytes(t) for t in chain(denoiser.parameters(), denoiser.buffers())
)
big_w, big_h = max(fam.resolution_presets, key = lambda wh: wh[0] * wh[1])
budget = _h3_residency_budget_bytes(
types.SimpleNamespace(resident_bytes = lambda: 0),
device,
width = big_w,
height = big_h,
frames = fam.default_num_frames,
te_streamed = te_streamed,
te_scheme = te_scheme,
)
return int(total) if 0 < total <= budget else None
except Exception: # noqa: BLE001 -- unpredictable: pin up front
return None
def _h3_placement_tier(
*,
quantised: bool,
whole_set_sizes: Optional[tuple[int, int]],
pinned_sizes: Optional[tuple[int, int]],
free_bytes: Optional[int],
speed_off: bool,
) -> str:
"""H3 denoiser placement, fastest first: ``resident``, ``pinned``, ``stream`` (quantised only) or ``rotation``.
A quantised denoiser never rotates (torchao does not survive the mid-block move); an unreadable card keeps it pinned."""
if not speed_off and _h3_dense_denoiser_fits(whole_set_sizes, free_bytes):
return "resident"
if quantised:
if pinned_sizes is None or free_bytes is None:
return "pinned"
return "pinned" if _h3_dense_denoiser_fits(pinned_sizes, free_bytes) else "stream"
if not speed_off and _h3_dense_denoiser_fits(pinned_sizes, free_bytes):
return "pinned"
return "rotation"
# int8 over fp8: same size and speed, closer to released (SSIM 0.49 vs 0.43).
H3_AUTO_FALLBACK_SCHEME = "int8"
_H3_STREAM_MODES = ("stream", "stream_lazy", "sync")
def _h3_auto_denoiser_scheme(
fam: Any,
*,
target: Any,
dtype: Any,
device: str,
te_scheme: Optional[str],
task: Optional[str],
base_repo: Optional[str],
speed_mode: Optional[str] = None,
free_reader: Any = None,
on_unreadable: Any = None,
) -> Optional[str]:
"""Hosted int8 for UNSET transformer_quant; None keeps bf16 or speed_mode off. Re-rolls the sample."""
if speed_mode is not None and str(speed_mode).strip().lower() == SPEED_OFF:
return None
if not _h3_auto_precision_ok(target):
return None
# Auto substitution only for the exact canonical base; a derivative has different weights.
canonical = getattr(fam, "base_repo", None)
if not base_repo or not canonical:
return None
if _h3_te_canonical(base_repo) != _h3_te_canonical(canonical):
return None
if not video_family_prequant_available(
fam, H3_AUTO_FALLBACK_SCHEME, task = task, base_repo = base_repo
):
# Per (scheme, partition): the other partition's checkpoint would generate the wrong thing.
return None
from .diffusion_prequant import (
restricted_prequant_load_supported,
torchao_group_offload_supported,
)
if not torchao_group_offload_supported():
# Without streaming, a torchao denoiser can only be pinned beside the larger rotating component.
sizes = _h3_planned_denoiser_bytes(fam, te_scheme = te_scheme, dtype = dtype, rotating = True)
free_bytes = (free_reader or _h3_free_device_bytes)(device)
hosted_bytes = int(h3_transformer_resident_gb(H3_AUTO_FALLBACK_SCHEME) * 1000.0**3)
if sizes is None or not _h3_dense_denoiser_fits((hosted_bytes, sizes[1]), free_bytes):
return None
if not restricted_prequant_load_supported(H3_AUTO_FALLBACK_SCHEME):
# Before the download plan, so never pick an unloadable artifact; asked last for on_unreadable.
if on_unreadable is not None:
on_unreadable()
return None
return H3_AUTO_FALLBACK_SCHEME
def _checkpoint_files(gguf_filename: Optional[str], kind: str) -> tuple[str, ...]:
"""The checkpoint file(s) a gguf / single_file pick reads: the pick, plus its partner expert when the name
pairs one (only a dual-expert family gets past validation with such a name, and a single-DiT one never asks)."""
if not gguf_filename:
return ()
partner = moe_partner_filename(gguf_filename) if kind != "pipeline" else None
return (gguf_filename, partner) if partner else (gguf_filename,)
def _planned_denoiser_components(fam: Any, kind: str) -> tuple[str, ...]:
"""The denoiser component(s) a CONVENTIONAL seeded load covers, or ``()``."""
try:
if kind != "pipeline" or getattr(fam, "modular_workflow", None):
return ()
from .video_denoiser_prequant import denoiser_components
return denoiser_components(fam)
except Exception: # noqa: BLE001 -- an unanswerable probe keeps the dense shards
return ()
def _video_auto_denoiser_scheme(
fam: Any,
*,
target: Any,
requested: Optional[str],
base_repo: Optional[str],
speed_mode: Optional[str] = None,
hf_token: Optional[str] = None,
) -> Optional[str]:
"""Seed scheme: a hosted checkpoint, else (auto only) the local cached resident build; None = dense shards."""
scheme = _video_hosted_denoiser_scheme(
fam,
target = target,
requested = requested,
base_repo = base_repo,
speed_mode = speed_mode,
hf_token = hf_token,
)
if scheme is not None:
return scheme
return _video_cached_resident_scheme(
fam,
target = target,
requested = requested,
base_repo = base_repo,
speed_mode = speed_mode,
hf_token = hf_token,
)
def _video_hosted_denoiser_scheme(
fam: Any,
*,
target: Any,
requested: Optional[str],
base_repo: Optional[str],
speed_mode: Optional[str] = None,
hf_token: Optional[str] = None,
) -> Optional[str]:
"""The scheme a CONVENTIONAL video load would seed from a hosted checkpoint, or None. ``hf_token``
reaches the hosted header probe, so a gated artifact is judged, not read as unknown."""
# Only an AUTO precision becomes "off" under Speed="off"; an EXPLICIT scheme is honored, so it keeps its seed.
auto = requested is None or str(requested).strip().lower() in ("", "auto")
if auto and speed_mode is not None and str(speed_mode).strip().lower() == SPEED_OFF:
return None
try:
if getattr(fam, "modular_workflow", None):
return None
# Registry first: a scheme with no hosted row for THIS base never seeds, so it must not spawn probes.
from .video_denoiser_prequant import denoiser_prequant_sources
hosted = tuple(
s
for s in video_family_prequant_schemes(fam)
if denoiser_prequant_sources(fam, s, base_repo, hf_token = hf_token) is not None
)
if not auto and normalize_transformer_quant(requested) not in hosted:
# An EXPLICIT unhosted scheme may seed from the local cache; never under auto, so the cache can't change
# auto's pick.
from .diffusion_prequant_cache import CACHEABLE_SCHEMES
wanted = normalize_transformer_quant(requested)
if (
wanted in CACHEABLE_SCHEMES
and denoiser_prequant_sources(fam, wanted, base_repo, hf_token = hf_token) is not None
):
hosted = hosted + (wanted,)
if not hosted or (not auto and normalize_transformer_quant(requested) not in hosted):
return None
scheme = select_transformer_quant_scheme(
target,
requested,
family = getattr(fam, "name", None),
base_repo = base_repo,
has_prequant = lambda candidate: (
denoiser_prequant_sources(fam, candidate, base_repo, hf_token = hf_token) is not None
),
)
if scheme is None or scheme == TQ_AUTO:
return None
# EVERY component or none: partial coverage drops shards the dense fallback then has to open.
if denoiser_prequant_sources(fam, scheme, base_repo, hf_token = hf_token) is None:
return None
from .diffusion_prequant import restricted_prequant_load_supported
if not restricted_prequant_load_supported(scheme):
return None
return scheme
except Exception: # noqa: BLE001 -- an unanswerable probe keeps the dense denoiser
return None
def _save_runtime_quant_to_cache(
fam: Any,
views: Any,
*,
base: Optional[str],
source: Optional[str],
cancel_check: Any = None,
defer: bool = False,
) -> Any:
"""Cache a runtime-quantized single denoiser; MoE is skipped (no lookup slot for the second expert).
``defer``: return a ``PendingWrite`` to start after publish, or None. Never raises."""
try:
from .video_denoiser_prequant import denoiser_components
views = list(views or ())
if len(denoiser_components(fam)) != 1 or len(views) != 1:
logger.info("diffusion.prequant_cache: skipped (multi-denoiser video family)")
return None
from .diffusion_prequant_cache import save_runtime_quant
return save_runtime_quant(
getattr(views[0], "transformer", None),
fam,
base_repo = base,
source_base = source,
component = "transformer",
logger = logger,
cancel_check = cancel_check,
**({"defer": True, "host_snapshot": True} if defer else {}),
)
except Exception as exc: # noqa: BLE001 - a cache write never fails a load
logger.warning("diffusion.prequant_cache: skipped (%s)", exc)
return None
# "0": no local prequant cache for AUTO resident torchao DiTs (explicit schemes unaffected).
VIDEO_RESIDENT_QUANT_CACHE_ENV = "UNSLOTH_VIDEO_RESIDENT_QUANT_CACHE"
_RESIDENT_QUANT_WRITE_ATTR = "_unsloth_resident_quant_write"
def _resident_quant_cache_enabled() -> bool:
raw = (os.environ.get(VIDEO_RESIDENT_QUANT_CACHE_ENV) or "").strip().lower()
if raw in ("0", "off", "false", "no", "none"):
return False
try:
from .diffusion_prequant_cache import enabled
return bool(enabled())
except Exception: # noqa: BLE001
return False
def _is_prequant_cache_source(source: Any) -> bool:
if getattr(source, "kind", None) != "path":
return False
try:
from .diffusion_prequant_cache import path_in_cache
return bool(path_in_cache(getattr(source, "location", None)))
except Exception: # noqa: BLE001
return False
def _video_cached_resident_scheme(
fam: Any,
*,
target: Any,
requested: Optional[str],
base_repo: Optional[str],
speed_mode: Optional[str] = None,
hf_token: Optional[str] = None,
) -> Optional[str]:
"""AUTO's own resident pick if the local cache holds exactly that build, else None. Only that scheme is
served, so an explicit int8 entry cannot turn later auto loads into int8. Never raises."""
try:
# Unset is auto: the Studio page omits the field.
auto = requested is None or str(requested).strip().lower() in ("", "auto")
if not auto:
return None
if speed_mode is not None and str(speed_mode).strip().lower() == SPEED_OFF:
return None
if getattr(fam, "modular_workflow", None) or not _resident_quant_cache_enabled():
return None
from .video_denoiser_prequant import denoiser_components, denoiser_prequant_sources
if len(denoiser_components(fam)) != 1:
return None
import torch
# Entries are bf16 builds only (the save refuses any other dtype).
if getattr(target, "dtype", None) is not torch.bfloat16:
return None
from .diffusion_prequant import restricted_prequant_load_supported
from .diffusion_prequant_cache import CACHEABLE_SCHEMES
cached = {}
for scheme in CACHEABLE_SCHEMES:
sources = denoiser_prequant_sources(fam, scheme, base_repo, hf_token = hf_token)
if sources and all(_is_prequant_cache_source(src) for src in sources.values()):
cached[scheme] = sources
if not cached:
return None
pick = select_transformer_quant_scheme(target, TQ_AUTO, family = getattr(fam, "name", None))
if pick not in cached or not restricted_prequant_load_supported(pick):
return None
return pick
except Exception: # noqa: BLE001 -- unanswerable: the dense shards load and quantise as before
return None
def _discard_resident_quant_write(owner: Any, token: Any = None) -> None:
slot = getattr(owner, "_resident_quant_write", None)
if slot is None or (token is not None and slot[0] != token):
return
owner._resident_quant_write = None
try:
slot[1].discard()
except Exception: # noqa: BLE001 - best effort
pass
def _take_resident_quant_write(owner: Any, token: Any) -> Any:
"""This load's pending resident write, now owned by the caller (None if another load replaced it)."""
slot = getattr(owner, "_resident_quant_write", None)
if slot is None or slot[0] != token:
return None
owner._resident_quant_write = None
return slot[1]
def _start_resident_quant_write(owner: Any, pipe: Any, token: Any) -> None:
_start_taken_resident_write(pipe, _take_resident_quant_write(owner, token))
def _start_taken_resident_write(pipe: Any, write: Any) -> None:
if write is None:
return
try:
setattr(pipe, _RESIDENT_QUANT_WRITE_ATTR, write)
write.start()
except Exception as exc: # noqa: BLE001 - a cache write never fails a load
logger.warning("diffusion.prequant_cache: resident background write skipped (%s)", exc)
try:
write.discard()
except Exception: # noqa: BLE001
pass
# "0": a first load's cache writes start as soon as it is published, beside its first render.
VIDEO_CACHE_WRITES_AFTER_RENDER_ENV = "UNSLOTH_VIDEO_CACHE_WRITES_AFTER_RENDER"
# Seconds a load with no render may sit before its held writes start anyway.
VIDEO_CACHE_WRITE_IDLE_ENV = "UNSLOTH_VIDEO_CACHE_WRITE_IDLE_S"
_VIDEO_CACHE_WRITE_IDLE_DEFAULT_S = 120.0
_HELD_CACHE_WRITES_ATTR = "_unsloth_held_cache_writes"
def _cache_writes_after_render() -> bool:
raw = (os.environ.get(VIDEO_CACHE_WRITES_AFTER_RENDER_ENV) or "").strip().lower()
return raw not in ("0", "off", "false", "no")
def _cache_write_idle_s() -> float:
raw = (os.environ.get(VIDEO_CACHE_WRITE_IDLE_ENV) or "").strip()
try:
value = float(raw) if raw else _VIDEO_CACHE_WRITE_IDLE_DEFAULT_S
except ValueError:
value = _VIDEO_CACHE_WRITE_IDLE_DEFAULT_S
return max(0.0, value)
def _start_cache_writes(
pipe: Any, cast_writes: Any, native_writes: Any, resident_write: Any
) -> None:
if cast_writes is not None:
pipe._unsloth_cast_writes = cast_writes
cast_writes.start()
if native_writes is not None:
setattr(pipe, _NATIVE_WRITES_ATTR, native_writes)
native_writes.start()
_start_taken_resident_write(pipe, resident_write)
class _HeldCacheWrites:
"""A first load's cache writes, held until its first render ends or ``idle_s`` idle: beside that render they
contend with its compile and denoise for CPU, disk and host RAM."""
def __init__(self, pipe: Any, writes: tuple, busy: Callable[[], bool], idle_s: float):
import weakref
self._pipe = weakref.ref(pipe)
self._writes: Optional[tuple] = writes
self._busy = busy
self._lock = threading.Lock()
self._timer: Optional[threading.Timer] = None
if idle_s > 0:
self._timer = threading.Timer(idle_s, self._on_idle)
self._timer.daemon = True
self._timer.name = "video-cache-write-idle"
def arm(self) -> None:
if self._timer is not None:
self._timer.start()
def _take(self) -> Optional[tuple]:
# Caller holds self._lock.
writes, self._writes = self._writes, None
timer, self._timer = self._timer, None
if timer is not None:
timer.cancel()
return writes
def _on_idle(self) -> None:
# A render in flight releases them when it ends.
try:
if self._busy():
return
except Exception: # noqa: BLE001 - unreadable: start them
pass
self.release("the backend sat idle after the load")
def release(self, why: str) -> None:
# Started and registered on the pipe under the lock: a concurrent discard either cancels them first or runs
# after, so the teardown's finish calls see every started writer.
with self._lock:
writes = self._take()
pipe = self._pipe()
if writes is None or pipe is None:
return
logger.info("diffusion.prequant_cache: first-load cache writes start now (%s)", why)
try:
_start_cache_writes(pipe, *writes)
except Exception as exc: # noqa: BLE001 - a cache write never fails a render
logger.warning("diffusion.prequant_cache: held cache writes skipped (%s)", exc)
def discard(self) -> None:
with self._lock:
writes = self._take()
for write in writes or ():
if write is None:
continue
try:
write.discard() if hasattr(write, "discard") else write.finish(0.0)
except Exception: # noqa: BLE001 - teardown is best effort
pass
def _release_held_cache_writes(pipe: Any, why: str) -> None:
held = getattr(pipe, _HELD_CACHE_WRITES_ATTR, None) if pipe is not None else None
if held is not None:
held.release(why)
def _discard_held_cache_writes(pipe: Any) -> None:
held = getattr(pipe, _HELD_CACHE_WRITES_ATTR, None) if pipe is not None else None
if held is None:
return
try:
setattr(pipe, _HELD_CACHE_WRITES_ATTR, None)
except Exception: # noqa: BLE001
pass
held.discard()
def _finish_resident_quant_write(pipe: Any) -> None:
"""Teardown: let a running write finish (bounded), then cancel."""
write = getattr(pipe, _RESIDENT_QUANT_WRITE_ATTR, None)
if write is None:
return
try:
from .diffusion_prequant_cache import FINISH_TIMEOUT_SECONDS
if not write.wait(FINISH_TIMEOUT_SECONDS):
write.cancel()
write.wait(30.0)
setattr(pipe, _RESIDENT_QUANT_WRITE_ATTR, None)
except Exception: # noqa: BLE001 - teardown is best effort
pass
def _read_cast_cache(
pipe_kwargs: dict,
*,
source: Any,
revision: Optional[str],
base: Optional[str],
cancel_check: Any = None,
device: Any = None,
) -> list:
"""Inject cached float16 / bfloat16 casts into ``pipe_kwargs``; returns the unserved candidates. Never raises.
``device``: a miss whose dense conversion would not fit host RAM is first staged there (``video_small_host``)."""
try:
from .diffusion_cast_cache import candidates, read_all
cands = candidates(
source,
torch_dtype = pipe_kwargs.get("torch_dtype"),
cache_dir = None
if source and os.path.isdir(str(source))
else pipe_kwargs.get("cache_dir"),
revision = revision,
skip = tuple(pipe_kwargs),
base_repo = base,
logger = logger,
)
if not cands:
return []
if device is not None:
from .video_small_host import converted_others_mib, stage_misses
others = converted_others_mib(
source,
cands,
torch_dtype = pipe_kwargs.get("torch_dtype"),
cache_dir = None
if source and os.path.isdir(str(source))
else pipe_kwargs.get("cache_dir"),
revision = revision,
skip = tuple(pipe_kwargs),
)
stage_misses(cands, device, logger = logger, cancel_check = cancel_check, others_mib = others)
hits = read_all(cands, logger = logger, cancel_check = cancel_check)
pipe_kwargs.update(hits)
if hits:
logger.info(
"video.cast_cache: read %s from the local cast cache",
", ".join(sorted(hits)),
)
return [c for c in cands if c.component not in hits]
except Exception as exc: # noqa: BLE001 - a cache problem is a miss
if cancel_check is not None:
cancel_check()
logger.warning("video.cast_cache: skipped (%s)", exc)
return []
def _finish_cast_writes(pipe: Any) -> None:
writes = getattr(pipe, "_unsloth_cast_writes", None)
if writes is None:
return
try:
from .diffusion_cast_cache import finish
finish(writes)
pipe._unsloth_cast_writes = None
except Exception: # noqa: BLE001 - teardown is best effort
pass
def _video_native_cache_spec(
fam: Any,
*,
target: Any,
transformer_quant: Optional[str],
plan: Any,
kind: str,
source_dir: Optional[str],
) -> Optional[dict]:
"""Cache spec of the native build an OFFLOADED CUDA load's runtime quant makes, else None. Never raises.
Must use the same inputs as the quant step (``native_scheme`` / ``native_kwargs``) so a hit replays that build."""
try:
if kind != "pipeline" or getattr(fam, "modular_workflow", None):
return None
if getattr(target, "device", None) != "cuda" or plan is None:
return None
if plan_keeps_transformer_resident(plan):
return None
requested = normalize_transformer_quant(transformer_quant)
if requested is None:
return None
pinned = None if requested == TQ_AUTO else requested
scheme = native_quant_scheme(
target,
_auto_offload_scheme(transformer_quant) if pinned is None else pinned,
family = fam.name,
offload = True,
)
if scheme is None:
return None
if stored_denoiser_precision(source_dir) is not None:
return None # the quant step declines a widened checkpoint
from .diffusion_native_cache import native_spec
return native_spec(
scheme,
fam.name,
act_int8 = scheme == TQ_INT8 and native_int8_act(target),
)
except Exception: # noqa: BLE001 -- unanswerable: no cache, the dense build runs
return None
def _native_spec_for(scheme: Optional[str], fam: Any, act_int8: Any) -> Optional[dict]:
try:
from .diffusion_native_cache import native_spec
return native_spec(scheme, getattr(fam, "name", None), act_int8 = bool(act_int8))
except Exception: # noqa: BLE001
return None
def _save_native_quant_to_cache(
fam: Any,
views: Any,
spec: dict,
*,
base: Optional[str],
source: Optional[str],
cancel_check: Any = None,
) -> None:
"""Cache every expert of a native runtime build, one entry per component. Never raises."""
try:
from .diffusion_native_cache import save
from .video_denoiser_prequant import denoiser_components
components = denoiser_components(fam)
views = list(views or ())
if len(components) != len(views):
logger.info("diffusion.prequant_cache: native skipped (denoiser views != components)")
return
for component, view in zip(components, views):
save(
getattr(view, "transformer", None),
fam,
spec,
base_repo = base,
source_base = source,
component = component,
logger = logger,
cancel_check = cancel_check,
)
except Exception as exc: # noqa: BLE001 - a cache write never fails a load
logger.warning("diffusion.prequant_cache: native skipped (%s)", exc)
# "0": the offloaded native build is written to the local prequant cache inside the load, as before.
VIDEO_NATIVE_CACHE_BACKGROUND_ENV = "UNSLOTH_VIDEO_NATIVE_CACHE_BACKGROUND"
_NATIVE_WRITES_ATTR = "_unsloth_native_writes"
def _native_cache_background() -> bool:
raw = (os.environ.get(VIDEO_NATIVE_CACHE_BACKGROUND_ENV) or "").strip().lower()
if raw in ("0", "off", "false", "no"):
return False
try:
from .diffusion_prequant_cache import async_enabled
return bool(async_enabled())
except Exception: # noqa: BLE001
return False
class _NativeCacheWrites:
"""Offloaded native build's cache writes, run after the load is published.
Waits for the background pins so the writer reads the pinned copies, not pageable tensors it would keep alive."""
def __init__(
self,
pipe: Any,
fam: Any,
views: Any,
spec: dict,
*,
base: Optional[str],
source: Optional[str],
):
import threading
self._pipe = pipe
self._args = (fam, list(views or ()), spec, base, source)
self._lock = threading.Lock()
self._closed = False
self._writes: list = []
self._thread: Optional[Any] = None
def start(self) -> None:
import threading
self._thread = threading.Thread(
target = self._run, name = "video-native-cache-write", daemon = True
)
self._thread.start()
def _run(self) -> None:
try:
finish_background_pins(self._pipe)
fam, views, spec, base, source = self._args
from .diffusion_native_cache import save_deferred
from .video_denoiser_prequant import denoiser_components
components = denoiser_components(fam)
if len(components) != len(views):
logger.info(
"diffusion.prequant_cache: native skipped (denoiser views != components)"
)
return
logger.info(
"diffusion.prequant_cache: writing the native %s build of %s in the background, after the load",
spec.get("scheme"),
", ".join(components),
)
for component, view in zip(components, views):
with self._lock:
if self._closed:
return
write = save_deferred(
getattr(view, "transformer", None),
fam,
base_repo = base,
source_base = source,
component = component,
logger = logger,
)
if write is None or not hasattr(write, "start"):
continue
self._writes.append(write)
write.start()
write.wait()
except Exception as exc: # noqa: BLE001 - a cache write never surfaces
logger.warning("diffusion.prequant_cache: native background write skipped (%s)", exc)
finally:
self._args = None
self._pipe = None
def finish(self, timeout: Optional[float] = None) -> None:
"""Teardown: let a running write finish (bounded), then cancel; a write not yet begun never begins."""
try:
from .diffusion_prequant_cache import FINISH_TIMEOUT_SECONDS
timeout = FINISH_TIMEOUT_SECONDS if timeout is None else timeout
except Exception: # noqa: BLE001
timeout = 600.0 if timeout is None else timeout
with self._lock:
self._closed = True
writes = list(self._writes)
for write in writes:
try:
if not write.wait(timeout):
write.cancel()
write.wait(30.0)
except Exception: # noqa: BLE001 - teardown is best effort
pass
if self._thread is not None:
self._thread.join(30.0)
def _finish_native_writes(pipe: Any) -> None:
writes = getattr(pipe, _NATIVE_WRITES_ATTR, None)
if writes is None:
return
try:
writes.finish()
setattr(pipe, _NATIVE_WRITES_ATTR, None)
except Exception: # noqa: BLE001 - teardown is best effort
pass
def _read_native_quant_from_cache(
fam: Any,
spec: Optional[dict],
*,
base: Optional[str],
source: Optional[str],
config_kwargs: dict,
cancel_check: Any = None,
) -> dict[str, Any]:
"""``{component: module}`` for EVERY denoiser from the native cache, or ``{}`` (all or nothing). Never raises."""
if spec is None:
return {}
try:
from utils.torch_warmup import close_dynamo_import_window
close_dynamo_import_window(logger) # a no-op once the load closed it above
except Exception as exc: # noqa: BLE001 - optimisation only
logger.debug("dynamo pre-import skipped: %r", exc)
try:
import diffusers
from .diffusion_native_cache import load, lookup
from .video_denoiser_prequant import denoiser_components
transformer_cls = getattr(diffusers, str(getattr(fam, "transformer_class", "")), None)
if transformer_cls is None:
return {}
components = denoiser_components(fam)
paths = {}
for component in components:
path = lookup(
fam, spec, base_repo = base, source_base = source, component = component, logger = logger
)
if path is None:
return {}
paths[component] = path
out: dict[str, Any] = {}
for component, path in paths.items():
module = load(
path,
transformer_cls,
str(source or base),
component = component,
spec = spec,
config_kwargs = config_kwargs,
logger = logger,
cancel_check = cancel_check,
)
if module is None:
return {}
out[component] = module
logger.info(
"video.transformer_quant: read %d native %s denoiser(s) for %s from the local prequant "
"cache; the dense denoiser shards are not loaded",
len(out),
spec.get("scheme"),
getattr(fam, "name", None),
)
return out
except Exception as exc: # noqa: BLE001 - a cache problem is a miss
logger.warning("diffusion.prequant_cache: native read skipped (%s)", exc)
return {}
def _read_native_quant_from_hosted(
fam: Any,
spec: Optional[dict],
*,
base: Optional[str],
source: Optional[str],
config_kwargs: dict,
hf_token: Optional[str] = None,
local_files_only: bool = False,
) -> dict[str, Any]:
"""``{component: module}`` for EVERY denoiser read from the hosted int8 file as native Linears, or ``{}`` (all or
nothing; the dense shards then load as before). Never raises."""
if spec is None:
return {}
try:
from utils.torch_warmup import close_dynamo_import_window
close_dynamo_import_window(logger) # a no-op once the load closed it above
except Exception as exc: # noqa: BLE001 - optimisation only
logger.debug("dynamo pre-import skipped: %r", exc)
try:
import diffusers
from .diffusion import hub_cache_dir
from .diffusion_native_hosted import load_native_from_hosted
from .diffusion_prequant import _resolve_checkpoint_path
from .video_denoiser_prequant import denoiser_prequant_sources
transformer_cls = getattr(diffusers, str(getattr(fam, "transformer_class", "")), None)
sources = denoiser_prequant_sources(fam, spec.get("scheme"), base, hf_token = hf_token)
if transformer_cls is None or not sources:
return {}
out: dict[str, Any] = {}
for component, src in sources.items():
path = _resolve_checkpoint_path(
src,
hf_token,
hub_cache_dir(),
local_files_only = local_files_only,
scheme = spec.get("scheme"),
logger = logger,
)
if path is None:
return {}
module = load_native_from_hosted(
path,
transformer_cls,
str(source or base),
spec = spec,
family = getattr(fam, "name", None),
component = component,
config_kwargs = config_kwargs,
base = base or getattr(fam, "base_repo", None),
logger = logger,
)
if module is None:
return {}
out[component] = module
return out
except Exception as exc: # noqa: BLE001 - a hosted problem keeps the dense build
logger.warning("video.native_hosted: hosted read skipped (%s)", exc)
return {}
def _video_native_stream_build(
fam: Any,
spec: Optional[dict],
*,
target: Any,
dtype: Any,
base: Optional[str],
source: Optional[str],
config_kwargs: dict,
skip: tuple = (),
cancel_check: Any = None,
) -> dict[str, Any]:
"""``{component: module}`` for EVERY denoiser built per shard into the native cache when the dense load would
not fit host RAM (``video_native_stream``), or ``{}``. Raises only a cancel or an out-of-memory."""
if spec is None:
return {}
try:
from utils.torch_warmup import close_dynamo_import_window
close_dynamo_import_window(logger) # a no-op once the load closed it above
except Exception as exc: # noqa: BLE001 - optimisation only
logger.debug("dynamo pre-import skipped: %r", exc)
try:
import diffusers
from .video_native_stream import build_into_cache
transformer_cls = getattr(diffusers, str(getattr(fam, "transformer_class", "")), None)
except Exception as exc: # noqa: BLE001 - the dense load still runs
logger.warning("video.native_stream: skipped (%s)", exc)
return {}
return build_into_cache(
fam,
spec,
transformer_cls,
target = target,
dtype = dtype,
base = base,
source = source,
config_kwargs = config_kwargs,
skip = skip,
# The device the runtime quant would build each host Linear on, so the codes match it bit for bit.
host_quant_device = _host_quant_device(target),
logger = logger,
cancel_check = cancel_check,
)
# The planner DECIDED against the seed (not "never ran"); a load must not re-take it after teardown.
DENOISER_SEED_DECLINED = "__declined__"
def _target_ordinal(target: Any) -> Optional[int]:
"""Card index of a device target (unindexed CUDA = current); None on a single-device backend."""
ordinal = getattr(target, "ordinal", None)
if ordinal is not None or getattr(target, "device", None) != "cuda":
return ordinal
try:
import torch
return int(torch.cuda.current_device())
except Exception: # noqa: BLE001 -- unanswerable: count every card, the old reading
return None
def _pipeline_device_mib(pipe: Any, ordinal: Optional[int] = None) -> int:
"""Accelerator MiB the pipe holds on ``ordinal`` (or every card), counting each storage once."""
if pipe is None:
return 0
seen: set[int] = set()
total = 0
def _add(t: Any) -> None:
nonlocal total
flatten = getattr(t, "__tensor_flatten__", None)
if callable(flatten) and type(t).__name__ not in ("Tensor", "Parameter"):
try:
for name in flatten()[0]:
_add(getattr(t, name))
return
except Exception: # noqa: BLE001 -- fall through to the plain reading
pass
try:
device = getattr(t, "device", None)
if getattr(device, "type", "cpu") in ("cpu", "meta"):
return
if ordinal is not None and (getattr(device, "index", None) or 0) != ordinal:
return
storage = t.untyped_storage()
if storage.data_ptr() in seen:
return
seen.add(storage.data_ptr())
total += int(storage.nbytes())
except Exception: # noqa: BLE001 -- an unreadable tensor adds nothing
pass
components = getattr(pipe, "components", None)
modules = components.values() if isinstance(components, dict) else ()
for module in modules:
for fn in ("parameters", "buffers"):
try:
for t in getattr(module, fn)():
_add(getattr(t, "data", t))
except Exception: # noqa: BLE001 -- not a torch module
break
return total // (1024 * 1024)
def _auto_seed_yields_to_resident_bf16(
scheme: Optional[str],
requested: Optional[str],
fam: Any = None,
) -> bool:
"""Whether an AUTO seed of ``scheme`` is bound by "auto keeps a resident bf16 DiT".
nvfp4 is exempt: it reaches auto only through a gated family preference (``_FAMILY_AUTO_PREFER``) that is
measured to be worth taking on a card that holds bf16. So is a scheme in the family's
``auto_prequant_when_resident``. An explicit scheme is a request, honored as asked."""
if scheme is None or scheme == DENOISER_SEED_DECLINED or scheme == TQ_NVFP4:
return False
profile = getattr(fam, "profile", None)
if scheme in (getattr(profile, "auto_prequant_when_resident", None) or ()):
return False
return requested is None or normalize_transformer_quant(requested) == TQ_AUTO
def _auto_keeps_resident_bf16(plan: Any, target: Any) -> bool:
"""The bf16 plan keeps the DiT resident on a measured budget, so auto runs bf16 (int8 fails the LPIPS bar there)."""
if plan is None or not plan_keeps_transformer_resident(plan):
return False
estimates = getattr(plan, "estimates", None) or {}
# An unmeasured budget also plans "none" without proving a fit.
return (
estimates.get("safe_device_budget_mib") is not None
and estimates.get("resident_required_mib") is not None
and bool(dense_transformer_supported(target))
)
def _video_seed_stays_resident(
fam: Any,
*,
target: Any,
scheme: Optional[str],
memory_mode: Optional[str],
text_encoder_quant: Optional[str],
base_repo: Optional[str],
reclaimable_mib: int = 0,
hf_token: Optional[str] = None,
ltx25_dev: bool = False,
device_memory_reader: Optional[Callable[[], Any]] = None,
plan_out: Optional[list] = None,
) -> bool:
"""True when an artifact-sized plan keeps the denoiser resident (text encoders may stream).
``scheme=None`` sizes the released bf16 DiT instead, and then also requires a measured budget (see
``_auto_keeps_resident_bf16``). ``device_memory_reader`` lets one planner call settle the card once
for both checks; it must return the raw settled snapshot (reclaimable is applied here). ``plan_out`` (a
list) receives the plan judged, so a caller can place an offloaded seed on it without a second settle."""
planned = _video_seed_plan(
fam,
target = target,
scheme = scheme,
memory_mode = memory_mode,
text_encoder_quant = text_encoder_quant,
base_repo = base_repo,
reclaimable_mib = reclaimable_mib,
hf_token = hf_token,
ltx25_dev = ltx25_dev,
device_memory_reader = device_memory_reader,
)
if planned is None:
return scheme is not None
if plan_out is not None:
plan_out.append(planned)
if scheme is None:
return _auto_keeps_resident_bf16(planned, target)
return plan_keeps_transformer_resident(planned)
def _video_seed_plan(
fam: Any,
*,
target: Any,
scheme: Optional[str],
memory_mode: Optional[str],
text_encoder_quant: Optional[str],
base_repo: Optional[str],
reclaimable_mib: int = 0,
hf_token: Optional[str] = None,
ltx25_dev: bool = False,
device_memory_reader: Optional[Callable[[], Any]] = None,
) -> Any:
"""The artifact-sized plan ``_video_seed_stays_resident`` judges, or None (no component sizes, or a
scheme with no steady size: both answered by the caller)."""
from .diffusion_component_sizes import video_components_gb
components = video_components_gb(fam, base_repo, allow_hub = True, token = hf_token, dev = ltx25_dev)
if not components:
return None
import torch
dtype = getattr(target, "dtype", None)
if scheme is None:
denoiser_gb = components[0] * (
2.0 if getattr(target, "device", None) != "cpu" and dtype is torch.float32 else 1.0
)
else:
measured = video_family_prequant_resident_gb(fam, scheme)
factor = _QUANT_STEADY_FACTOR.get(scheme)
if measured:
denoiser_gb = float(measured)
elif factor is not None:
denoiser_gb = components[0] * factor
else:
return None
from .diffusion_te_prequant import te_prequant_budget_scale
dtype_scale = (
2.0 if getattr(target, "device", None) != "cpu" and dtype is torch.float32 else 1.0
)
te_scale = te_prequant_budget_scale(
fam,
te_quant_mode = text_encoder_quant,
target = target,
base = base_repo,
)
vae_scale = 1.0 if getattr(fam, "vae_force_fp32", False) else dtype_scale
mib_per_gb = 1000.0**3 / (1024.0 * 1024.0)
text_encoder_gb = components[1] * te_scale * dtype_scale
companions_gb = text_encoder_gb + components[2] * vae_scale
if getattr(fam, "name", None) == "ltx-2.5":
from .video_ltx25 import LTX25_WORKER_RESIDENT_GB
companions_gb += LTX25_WORKER_RESIDENT_GB
model_dense_mib = int((denoiser_gb + companions_gb) * mib_per_gb)
device_memory = (
settled_snapshot_device_memory(target)
if device_memory_reader is None
else device_memory_reader()
)
if reclaimable_mib > 0 and device_memory.free_mib is not None:
free = device_memory.free_mib + int(reclaimable_mib)
if device_memory.total_mib is not None:
free = min(free, device_memory.total_mib)
device_memory = dataclasses.replace(device_memory, free_mib = free)
planned = plan_diffusion_memory(
target = target,
device_memory = device_memory,
model_dense_mib = model_dense_mib,
runtime_headroom_mib = estimate_video_runtime_mib(
width = fam.resolution_presets[0][0],
height = fam.resolution_presets[0][1],
num_frames = fam.default_num_frames,
),
companion_dense_mib = int(companions_gb * mib_per_gb),
text_encoder_dense_mib = int(text_encoder_gb * mib_per_gb),
requested_mode = normalize_memory_mode(memory_mode),
)
return planned
# 0 restores the refusal: an explicit fp8 pipeline load whose plan moves the DiT keeps the dense shards and is declined.
VIDEO_FP8_STREAM_ENV = "UNSLOTH_VIDEO_FP8_STREAM"
# Schemes whose hosted torchao denoiser streams under group offload; int8 has its torchao-free twin there instead.
_VIDEO_TORCHAO_STREAM_SCHEMES = (TQ_FP8,)
def video_fp8_stream_enabled() -> bool:
raw = (os.environ.get(VIDEO_FP8_STREAM_ENV) or "").strip().lower()
return raw not in ("0", "false", "no", "off")
def _video_torchao_seed_placement(fam: Any, scheme: Optional[str], target: Any, plan: Any) -> Any:
"""The streaming placement a hosted torchao ``scheme`` denoiser takes under an offloaded ``plan``, else None.
The rule the image path's seeded and single-file fp8 loads follow (``torchao_offload_plan``): group or
streaming offload keep their placement, a whole-module plan becomes the streaming plan, sequential offload
and a torchao, diffusers or pin budget that cannot stream it return None. Only fp8 on a card whose fp8
GEMM passes Studio's probe (sm_89+), on a conventional pipeline load; int8 keeps its torchao-free twin."""
if not video_fp8_stream_enabled() or scheme not in _VIDEO_TORCHAO_STREAM_SCHEMES:
return None
if plan is None or plan_keeps_transformer_resident(plan):
return None
if getattr(fam, "modular_workflow", None):
return None
try:
import torch
from .diffusion_memory import torchao_offload_plan, torchao_streaming_plan
if (
getattr(target, "device", None) != "cuda"
or getattr(target, "dtype", None) is not torch.bfloat16
):
return None
if (
select_transformer_quant_scheme(
target, scheme, family = getattr(fam, "name", None), unproven_ok = True
)
!= scheme
):
return None
placed = torchao_offload_plan(plan, scheme)
if placed is not None and getattr(placed, "offload_policy", None) == OFFLOAD_MODEL:
# Whole-module hooks move a Float8Tensor by storage swap, which it does not survive: stream instead.
from .diffusion_memory import _torchao_stream_pinnable, torchao_scheme_streams
placed = (
torchao_streaming_plan(placed)
if torchao_scheme_streams(scheme) and _torchao_stream_pinnable(placed)
else None
)
except Exception: # noqa: BLE001 -- an unanswerable probe keeps today's decline
return None
if placed is None or getattr(placed, "offload_policy", None) not in (
OFFLOAD_GROUP,
OFFLOAD_STREAMING,
):
return None
return placed
def _h3_carry_by_scheduler(fam: Any, state: Any, has_step_callback: bool) -> bool:
"""H3 modular has no step callback: its scheduler.step wrapper drives a streamed denoiser's cross-forward carry."""
from .video_minimax_h3_residency import h3_stream_carry_enabled
return (
not has_step_callback
and getattr(fam, "name", None) == "minimax-h3"
and bool(getattr(state, "denoiser_streamed", False))
and h3_stream_carry_enabled()
)
def _scheduler_step_carry(pipe: Any, steps: int, on_done: Any, cancelled: Any) -> Any:
"""``on_done`` after each scheduler.step, then the cross-forward step (forward k's step is index k - 1)."""
def done(enqueued: int) -> None:
on_done(enqueued)
if not cancelled():
cross_forward_step(pipe, enqueued - 1, total = steps)
return done
def _carried_call(pipe: Any, kwargs: dict, has_step_callback: bool) -> Any:
"""Armed on the render thread (the interleave check is per thread); needs the step callback to stop before decode."""
with cross_forward_render(pipe, enabled = has_step_callback):
return pipe(**kwargs)
def _render_under_no_grad(state: Any) -> bool:
"""Render under no_grad: torchao weight subclasses fail an offload hook's in-place swap under inference_mode.
Stored fp8 codes too: moved under inference_mode they become unversionable, re-copied to host every offload."""
if not getattr(state, "transformer_quant", None):
return getattr(state, "offload_policy", OFFLOAD_NONE) != OFFLOAD_NONE and bool(
pipe_fp8_stored_layers(getattr(state, "pipe", None))
)
if getattr(state, "denoiser_streamed", False):
return True
family = getattr(state, "family", None)
return not getattr(family, "modular_workflow", None) and (
getattr(state, "offload_policy", OFFLOAD_NONE) != OFFLOAD_NONE
)
# Staging headroom: base overhead plus one linear's bf16, rotated and int8 copies, as a multiple of the largest.
_STAGE_LINEAR_WORKSPACE = 3
def _stage_units(module: Any) -> list:
import torch
units: list = []
for child in module.children():
if isinstance(child, torch.nn.ModuleList):
units.extend(child)
else:
units.append(child)
return units
def _module_dense_bytes(module: Any) -> int:
from itertools import chain
return sum(
int(t.numel()) * int(t.element_size()) for t in chain(module.parameters(), module.buffers())
)
def _stage_denoiser_for_quant(
transformer: Any,
target: Any,
*,
logger: Any = None,
) -> list:
"""Stage a CPU DiT onto ``target``'s CUDA device block by block as far as it fits; quantisers work per linear,
so a partial fit gives the same result. Returns the moved modules; never raises."""
try:
import torch
if (
transformer is None
or getattr(target, "device", None) != "cuda"
or not torch.cuda.is_available()
):
return []
first = next(transformer.parameters(), None)
if first is None or first.device.type != "cpu":
return []
device = torch.device(getattr(target, "torch_device", None) or "cuda")
from utils.hardware import trusted_mem_get_info
free_bytes, total_bytes = trusted_mem_get_info(device, module = torch.cuda)
largest_linear = max(
(
_module_dense_bytes(m)
for m in transformer.modules()
if isinstance(m, torch.nn.Linear)
),
default = 0,
)
budget = (
int(free_bytes)
- _user_reserve_bytes(device, total_bytes)
- DEFAULT_BASE_OVERHEAD_MIB * 1024 * 1024
- _STAGE_LINEAR_WORKSPACE * largest_linear
)
staged: list = []
moved_bytes = 0
try:
for unit in _stage_units(transformer):
size = _module_dense_bytes(unit)
if size == 0 or moved_bytes + size > budget:
continue
unit.to(device)
staged.append(unit)
moved_bytes += size
except Exception:
# The DiT started on the host: put every moved unit, and a half-moved one, back before the CPU fallback.
transformer.to("cpu")
clear_gpu_cache()
raise
if staged and logger is not None:
logger.info(
"video.transformer_quant: quantising on %s with %.1f of %.1f GB of the DiT staged there",
device,
moved_bytes / 1e9,
_module_dense_bytes(transformer) / 1e9,
)
return staged
except Exception as exc: # noqa: BLE001 -- quantising on the host is slower, never wrong
if logger is not None:
logger.warning("video.transformer_quant: staging on the card failed (%s)", exc)
return []
def _host_quant_device(target: Any) -> Any:
try:
import torch
if (
getattr(target, "device", None) != "cuda"
or not torch.cuda.is_available()
or not native_quant_on_device_enabled()
):
return None
return torch.device(getattr(target, "torch_device", None) or "cuda")
except Exception: # noqa: BLE001 -- the host build is slower, never wrong
return None
def _unstage_modules(modules: list) -> None:
for module in modules:
module.to("cpu")
clear_gpu_cache()
def _auto_offload_scheme(requested: Optional[str]) -> Optional[str]:
"""auto takes int8 for an offloaded DiT: its torchao-free build rides every offload hook."""
scheme = normalize_transformer_quant(requested)
return TQ_INT8 if scheme == TQ_AUTO else scheme
# "0" disables reading the hosted native file when the load's own plan offloads a seeded denoiser.
OFFLOAD_HOSTED_READ_ENV = "UNSLOTH_VIDEO_OFFLOAD_HOSTED_READ"
def _offload_hosted_read_enabled() -> bool:
return (os.environ.get(OFFLOAD_HOSTED_READ_ENV) or "").strip().lower() not in (
"0",
"off",
"false",
"no",
)
def _video_native_hosted_scheme(fam: Any, scheme: Optional[str], target: Any) -> Optional[str]:
"""``int8`` when an offloaded load of ``scheme`` builds the W8A8 ConvRot native DiT that the family's hosted int8
checkpoint already holds (``diffusion_native_hosted``), else None. The load re-checks the file; never raises."""
try:
from .diffusion_native_hosted import native_hosted_family, native_hosted_pickle
from .diffusion_native_quant import int8_rotation_group
if scheme != TQ_INT8 or not native_hosted_family(fam):
return None
if getattr(target, "device", None) != "cuda":
return None
# A pickle unpickles through torchao: not on ROCm or a stubbed torchao (native_offload_host's set).
if native_hosted_pickle(fam) and not native_offload_host(target):
return None
if native_quant_scheme(target, TQ_INT8, family = fam.name, offload = True) != TQ_INT8:
return None
if not native_int8_act(target):
return None
# A ConvRot file only matches a rotated runtime build; a pickle is read unrotated, so ConvRot off keeps it.
if not native_hosted_pickle(fam) and int(int8_rotation_group() or 0) <= 0:
return None
return TQ_INT8
except Exception: # noqa: BLE001 -- unanswerable: the dense shards and runtime build, as before
return None
# Denoise-side base of estimate_video_runtime_mib; the rest scales with the decoded clip, held only by the VAE.
_VIDEO_DENOISE_ACTIVATION_MIB = 4096
def _video_denoise_phase_headroom_mib(plan: Any) -> int:
"""What the denoise phase holds beside the DiT under whole-module offload (the same pricing as below)."""
try:
runtime = int(((getattr(plan, "estimates", None) or {}).get("runtime_headroom_mib")) or 0)
except (TypeError, ValueError):
runtime = 0
return min(runtime, _VIDEO_DENOISE_ACTIVATION_MIB) + DEFAULT_BASE_OVERHEAD_MIB
def _video_prefer_whole_module(
plan: Any,
*,
denoiser_mib: Optional[int],
text_encoder_mib: Optional[int],
vae_mib: Optional[int],
runtime_mib: int,
resident_mib: int = 0,
) -> Any:
"""Swap a streamed-DiT plan for whole-module offload when every component fits whole, one at a time.
Wan2.2-5B at 16 GiB: 0.14-0.28 s per forward whole-module vs 0.28-0.33 s int8 streamed, so streaming only wins
very short calls. ``denoiser_mib`` is one expert of a dual-DiT family. ``resident_mib`` stays on the card in every
phase (LTX-2.5's encoder worker context), excluded from ``vae_mib``. An explicit memory mode is left as asked."""
if (
getattr(plan, "offload_policy", None) != OFFLOAD_GROUP
or not bool(getattr(plan, "stream_transformer", True))
or getattr(plan, "requested_mode", None) != MEMORY_MODE_AUTO
or denoiser_mib is None
or text_encoder_mib is None
or vae_mib is None
):
return plan
budget = (getattr(plan, "estimates", None) or {}).get("safe_device_budget_mib")
if budget is None:
return plan
resident = max(0, int(resident_mib))
phases = (
int(text_encoder_mib) + resident + DEFAULT_BASE_OVERHEAD_MIB,
int(denoiser_mib)
+ min(int(runtime_mib), _VIDEO_DENOISE_ACTIVATION_MIB)
+ resident
+ DEFAULT_BASE_OVERHEAD_MIB,
# the decode holds only the decoded-clip share; the denoise base is already in the DiT phase
int(vae_mib)
+ max(0, int(runtime_mib) - _VIDEO_DENOISE_ACTIVATION_MIB)
+ resident
+ DEFAULT_BASE_OVERHEAD_MIB,
)
if max(phases) > int(budget):
return plan
return dataclasses.replace(
plan,
estimates = {**(plan.estimates or {}), "whole_module_fits_mib": max(phases)},
offload_policy = OFFLOAD_MODEL,
stream_text_encoders = False,
stream_transformer = True,
vae_tiling = True,
vae_slicing = True,
reasons = tuple(plan.reasons)
+ (
f"every component fits the card whole (largest phase {max(phases)} MiB of {int(budget)}): "
"whole-module offload pages the DiT once per call instead of streaming it every step",
),
)
def _video_plan_rank(plan: Any) -> int:
"""Video speed rank, fastest first (0..6); see the tier comments in the tests."""
policy = getattr(plan, "offload_policy", OFFLOAD_NONE)
if policy == OFFLOAD_NONE:
return 0
if policy == OFFLOAD_GROUP:
if not bool(getattr(plan, "stream_transformer", True)):
return 1
return 4 if bool(getattr(plan, "stream_text_encoders", False)) else 3
if policy == OFFLOAD_MODEL:
fits = (getattr(plan, "estimates", None) or {}).get("whole_module_fits_mib") is not None
return 2 if fits else 5
return 6
def _video_plan_label(plan: Any) -> str:
return (
"every component resident",
"denoiser resident, text encoders streamed",
"whole-module CPU offload, each component fits whole",
"denoiser streamed block by block, companions resident",
"denoiser and text encoders streamed",
"whole-module CPU offload",
"denoiser blocks and text-encoder layers streamed",
)[_video_plan_rank(plan)]
_VIDEO_DENOISER_ATTRS = ("transformer", "transformer_2", "unconditional_transformer")
def _video_streamed_peak_bytes(module: Any, offload_type: str, *, prefetch: bool, size: Any) -> int:
"""Bytes a group-offload hook holds on the device at once for ``module`` (diffusers' own grouping)."""
from itertools import chain
import torch
try:
from diffusers.hooks._common import _GO_LC_SUPPORTED_PYTORCH_LAYERS as leaf_types
except Exception: # noqa: BLE001 -- same layer set diffusers streams leaf by leaf
nn = torch.nn
leaf_types = (
nn.Conv1d,
nn.Conv2d,
nn.Conv3d,
nn.ConvTranspose1d,
nn.ConvTranspose2d,
nn.ConvTranspose3d,
nn.Linear,
nn.Embedding,
)
counted: set[int] = set()
def _own(mod: Any, recurse: bool) -> int:
# a tied weight (shared embedding) moves once, so count it for its first group only
total = 0
for t in chain(mod.parameters(recurse = recurse), mod.buffers(recurse = recurse)):
if id(t) not in counted:
counted.add(id(t))
total += size(t)
return total
if offload_type == "block_level":
windows: list[int] = []
grouped: set[int] = set()
for child in module.children():
if isinstance(child, (torch.nn.ModuleList, torch.nn.Sequential)):
blocks = list(child)
for i in range(0, len(blocks), DEFAULT_GROUP_BLOCKS):
windows.append(sum(_own(b, True) for b in blocks[i : i + DEFAULT_GROUP_BLOCKS]))
grouped |= {id(t) for t in chain(child.parameters(), child.buffers())}
# the unmatched group (embeddings, heads, loose params) stays onloaded for the whole forward
unmatched = sum(
size(t)
for t in {id(t): t for t in chain(module.parameters(), module.buffers())}.values()
if id(t) not in grouped and id(t) not in counted
)
units, fixed = windows, unmatched
else:
# (size, enclosing ancestors' loose bytes, ids of this group and its ancestors) per onload group
groups: list[tuple[int, int, frozenset]] = []
visited: set[int] = set()
def _walk(mod: Any, enclosing: int, chain: frozenset) -> None:
if id(mod) in visited:
return
visited.add(id(mod))
if isinstance(mod, leaf_types):
groups.append((_own(mod, True), enclosing, chain | {id(mod)}))
elif next(mod.children(), None) is not None:
# A parent's loose params stay onloaded under every descendant; only the enclosing chain stacks.
loose = _own(mod, False)
here = chain | {id(mod)}
groups.append((loose, enclosing, here))
for child in mod.children():
_walk(child, enclosing + loose, here)
else:
# loose params of a childless non-leaf (a norm) form their own group
groups.append((_own(mod, False), enclosing, chain | {id(mod)}))
_walk(module, 0, frozenset())
ranked = sorted(range(len(groups)), key = lambda i: groups[i][0], reverse = True)
peak = 0
for size, enclosing, here in groups:
# a copy stream prefetches the next group while this one computes: any group not already held
other = 0
if prefetch:
for i in ranked:
if not (groups[i][2] <= here):
other = groups[i][0]
break
peak = max(peak, enclosing + size + other)
return peak
units = sorted(units, reverse = True)
# a copy stream prefetches the next group while the current one computes
return fixed + sum(units[: 2 if prefetch else 1])
def _video_group_hooked(module: Any) -> bool:
"""Whether group offloading actually hooked ``module`` (leaf level hooks its leaves). Unreadable counts as hooked."""
try:
for sub in module.modules():
hooks = getattr(getattr(sub, "_diffusers_hook", None), "hooks", None) or {}
if any("group_offloading" in str(key) for key in hooks):
return True
except Exception: # noqa: BLE001 -- trust the plan
return True
return False
def _video_offload_vram_floor_mib(
pipe: Any,
plan: Any,
*,
applied: bool = False,
phase: str = "peak",
) -> Optional[int]:
"""MiB of weights co-resident on the device under ``plan``'s tier, from the loaded modules; None if not offloaded.
``applied`` reads which modules the hooks actually stream: an encoder that refused leaf offload stays resident.
``phase="decode"`` counts only what stays on the device during the VAE decode, after the DiT has offloaded."""
policy = getattr(plan, "offload_policy", OFFLOAD_NONE)
if policy == OFFLOAD_NONE:
return None
try:
from itertools import chain
import torch
from .diffusion_prequant import tensor_payload_bytes
components = getattr(pipe, "components", None)
if not isinstance(components, dict):
return None
sizes: dict[str, int] = {}
modules: dict[str, Any] = {}
seen: set[int] = set()
for name, module in components.items():
if not isinstance(module, torch.nn.Module):
continue
total = 0
for tensor in chain(module.parameters(), module.buffers()):
if id(tensor) in seen or getattr(tensor, "is_meta", False):
continue
seen.add(id(tensor))
total += tensor_payload_bytes(tensor)
sizes[str(name)] = total
modules[str(name)] = module
if not sizes:
return None
denoisers = {n for n in sizes if n in _VIDEO_DENOISER_ATTRS}
encoders = {n for n in sizes if n.startswith("text_encoder")}
if policy == OFFLOAD_MODEL:
if phase == "decode":
# whole-module offload onloads one component at a time; the decode runs the decoders alone
decoders = [
size for name, size in sizes.items() if name not in denoisers | encoders
]
return int(max(decoders, default = 0) // (1024 * 1024))
return int(max(sizes.values()) // (1024 * 1024))
if policy == OFFLOAD_GROUP:
streamed = (
set() if not bool(getattr(plan, "stream_transformer", True)) else set(denoisers)
)
if bool(getattr(plan, "stream_text_encoders", False)):
streamed |= encoders
elif policy == OFFLOAD_STREAMING:
from .diffusion_memory import run_once_companions
encoders |= {n for n in run_once_companions(pipe) if n in sizes}
streamed = denoisers | encoders
else:
return None
if applied:
# From the hooks alone: a refusing encoder stays resident, and the fallback may hook a DiT the plan kept.
streamed = {name for name in denoisers | encoders if _video_group_hooked(modules[name])}
held = sum(size for name, size in sizes.items() if name not in streamed)
if phase == "decode":
# streamed groups have all offloaded by the decode; only the residents remain
return int(held // (1024 * 1024))
backend = getattr(getattr(plan, "device_memory", None), "backend", None)
prefetch = backend not in ("mps", "cpu")
def _size(tensor: Any) -> int:
return 0 if getattr(tensor, "is_meta", False) else tensor_payload_bytes(tensor)
# streamed modules run one after another, so only the largest onloaded unit joins the residents
held += max(
(
_video_streamed_peak_bytes(
modules[name],
"block_level" if name in denoisers else "leaf_level",
prefetch = prefetch,
size = _size,
)
for name in streamed
),
default = 0,
)
return int(held // (1024 * 1024))
except Exception: # noqa: BLE001 -- an unreadable pipeline keeps today's unchecked behaviour
return None
# Text encoder runs in a sidecar worker outside ``pipe.components``: a split cannot move it.
_VIDEO_SIDECAR_ENCODER_FAMILIES = frozenset({"ltx-2.5"})
# "0" keeps LTX-2.5 on one card under a multi-GPU selection.
LTX25_MULTI_GPU_ENV = "UNSLOTH_LTX25_MULTI_GPU"
def _video_sidecar_multigpu(fam: Any) -> bool:
"""Sidecar-encoder family may use two cards: lever on, multi-card selection active, CUDA (not ROCm)."""
if getattr(fam, "name", None) not in _VIDEO_SIDECAR_ENCODER_FAMILIES:
return False
if (os.environ.get(LTX25_MULTI_GPU_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
return False
from .diffusion_multigpu import _rocm_torch, current_selection
return current_selection() is not None and not _rocm_torch()
def _video_sidecar_worker_card(fam: Any, plan: Any) -> Optional[int]:
secondary = getattr(plan, "secondary_ordinal", None)
if secondary is None or not _video_sidecar_multigpu(fam):
return None
return int(secondary)
def _video_single_card_for_sidecar(
fam: Any,
plan: Any,
logger: Any = None,
) -> Any:
"""Single-card plan for a sidecar-encoder split, unless the partner can also hold the worker's context."""
if getattr(fam, "name", None) not in _VIDEO_SIDECAR_ENCODER_FAMILIES:
return plan
fallback = getattr(plan, "fallback_plan", None)
secondary = getattr(plan, "secondary_ordinal", None)
if secondary is None or fallback is None:
return plan
why = f"the {fam.name} text encoder runs in a sidecar a split cannot move"
if _video_sidecar_multigpu(fam):
from .diffusion_multigpu import secondary_resident_limit_mib
from .video_ltx25 import LTX25_WORKER_RESIDENT_GB
need = (getattr(plan, "estimates", None) or {}).get("multigpu_secondary_required_mib")
limit = secondary_resident_limit_mib(int(secondary), getattr(plan, "requested_mode", None))
context = int(LTX25_WORKER_RESIDENT_GB * 1024)
if need is not None and limit is not None and int(need) + context <= int(limit):
if logger is not None:
logger.info(
"video.multigpu: the %s text encoder worker runs on cuda:%s with the split (%d MiB of %d MiB)",
fam.name,
secondary,
int(need) + context,
int(limit),
)
return plan
why = f"cuda:{secondary} cannot also hold the {fam.name} text encoder worker"
if logger is not None:
logger.info("video.multigpu: one card (%s)", why)
return fallback
def _video_sidecar_card_reserve_mib(pipe: Any, plan: Any = None) -> Optional[dict[int, int]]:
"""``{ordinal: MiB}`` peak the sidecar encoder holds, unseen by ``_loaded_component_mib``; None = unpriceable.
On the primary a host-kept encoder reuses the denoise headroom (encode runs first). Waits for the worker's load
report and raises its load error here (the message is read once)."""
client = getattr(pipe, "_unsloth_ltx25_encoder", None)
config = getattr(client, "config", None) or {}
device = str(config.get("device") or "")
if client is None or not device.startswith("cuda"):
return None
info = client.wait_ready(getattr(client, "cancel_event", None))
try:
import torch
from .video_ltx25 import LTX25_WORKER_RESIDENT_GB
from .video_ltx25_te_worker import ENCODE_MARGIN_MIB
ordinal = torch.device(device).index
ordinal = int(ordinal if ordinal is not None else torch.cuda.current_device())
weights = int(info.get("weights_mib") or 0)
if weights <= 0:
return None
if info.get("resident"):
need = weights
else:
stream = info.get("stream_need_mib")
need = int(stream) if stream else weights + ENCODE_MARGIN_MIB
from .diffusion_multigpu import current_selection
selection = current_selection()
if selection is not None and int(selection.primary) == ordinal:
estimates = getattr(plan, "estimates", None) or {}
flat = (
estimates.get("calibrated_headroom_mib")
or estimates.get("calibrated_split_headroom_mib")
or estimates.get("runtime_headroom_mib")
)
headroom = int(estimates.get("measured_runtime_headroom_mib") or flat or 0)
need = max(0, need - headroom)
return {ordinal: int(need + LTX25_WORKER_RESIDENT_GB * 1024)}
except Exception as exc: # noqa: BLE001 -- unpriced: one card, the worker's load error surfaces later
logger.info("video.multigpu: the sidecar encoder could not be priced (%s)", exc)
return None
def _video_refine_for_multigpu(
fam: Any,
pipe: Any,
plan: Any,
logger: Any = None,
) -> Any:
"""``refine_plan_for_multigpu``; sidecar families only under ``_video_sidecar_multigpu``, charged for the worker."""
if getattr(fam, "name", None) not in _VIDEO_SIDECAR_ENCODER_FAMILIES:
return refine_plan_for_multigpu(pipe, plan, logger)
if not _video_sidecar_multigpu(fam):
return plan
if (
getattr(plan, "secondary_ordinal", None) is not None
or getattr(plan, "layer_split", None) is not None
):
return refine_plan_for_multigpu(pipe, plan, logger, card_reserve_mib = {})
from .diffusion_layer_split_plan import layer_split_mode
if (
getattr(plan, "offload_policy", None) == "none"
and multigpu_mode() != "force"
and layer_split_mode() != "force"
):
# resident: nothing is priced, so do not wait for the worker
return refine_plan_for_multigpu(pipe, plan, logger, card_reserve_mib = {})
reserve = _video_sidecar_card_reserve_mib(pipe, plan)
if reserve is None:
return plan
return refine_plan_for_multigpu(pipe, plan, logger, card_reserve_mib = reserve)
def _video_split_card_floors(pipe: Any) -> Optional[tuple]:
"""``((ordinal, resident MiB, runs the decode), ...)`` for a pipeline split across cards, else None. A split plan
reports no offload policy, so the offload floor is None and the request guard would be skipped; this records what
each card holds so the guard can run per card."""
from .diffusion_layer_split_plan import layer_split_card_floors
layered = layer_split_card_floors(pipe)
if layered is not None:
return layered
status = multigpu_placement_status(pipe) if multigpu_is_split(pipe) else None
placement = (status or {}).get("placement") or {}
if not placement:
return None
try:
from itertools import chain
import torch
from .diffusion_prequant import tensor_payload_bytes
components = getattr(pipe, "components", None) or {}
cards: dict[int, list] = {}
seen: set[int] = set()
for name, where in placement.items():
device = torch.device(where)
module = components.get(name)
if (
device.type != "cuda"
or device.index is None
or not isinstance(module, torch.nn.Module)
):
continue
card = cards.setdefault(int(device.index), [0, False])
for tensor in chain(module.parameters(), module.buffers()):
if id(tensor) in seen or getattr(tensor, "is_meta", False):
continue
seen.add(id(tensor))
card[0] += tensor_payload_bytes(tensor)
card[1] = card[1] or str(name) == "vae"
if len(cards) < 2:
return None
home = placement.get("transformer")
home = torch.device(home).index if home else None
denoisers = {
torch.device(where).index
for name, where in placement.items()
if str(name).startswith("transformer") and torch.device(where).type == "cuda"
}
return tuple(
(o, int(b // (1024 * 1024)), bool(d), o == home, o in denoisers)
for o, (b, d) in sorted(cards.items())
)
except Exception: # noqa: BLE001 -- an unreadable placement keeps the unchecked behaviour
return None
def video_split_shortfall_message(
cards: tuple,
*,
family: str,
available_mib: Callable[[int], int],
width: Optional[int] = None,
height: Optional[int] = None,
frames: Optional[int] = None,
planned_shape: Optional[tuple] = None,
) -> Optional[str]:
"""The offload guard run on every card of a split: each card's resident weights plus the base headroom, the
decoded-clip share on the card that runs the VAE decode, and on the denoiser's card, when the decode ran
elsewhere, the larger of the denoise activations and the decoded frames (fp32) copied back. A layer split's
block-only card (a sixth field: the headroom its plan reserved at ``planned_shape`` = (width, height, frames))
is charged that headroom, scaled by how many more tokens this request carries, plus its fixed reserve (a seventh
field, unscaled). The first refusal, else None."""
for card in cards:
ordinal, floor_mib, decodes = card[:3]
home = len(card) > 3 and bool(card[3])
denoises = bool(card[4]) if len(card) > 4 else home
shaped = bool(width and height and frames)
block_headroom = card[5] if len(card) > 5 else None
block_reserve = int(card[6] or 0) if len(card) > 6 else 0
if block_headroom is not None and not home and not decodes:
scale = 1.0
if shaped and planned_shape and all(planned_shape):
planned = int(planned_shape[0]) * int(planned_shape[1]) * int(planned_shape[2])
scale = max(1.0, int(width) * int(height) * int(frames) / float(planned))
headroom = int(int(block_headroom) * scale) + block_reserve
required = int(floor_mib) + headroom
available = int(available_mib(int(ordinal)))
if available < required:
shape = f" for {width}x{height} at {frames} frames" if shaped else ""
return (
f"{family} needs about {required / 1024:.1f} GiB of available VRAM{shape} under this load's "
f"placement (split across GPUs, cuda:{int(ordinal)}: {int(floor_mib) / 1024:.1f} GiB of model "
f"blocks plus {headroom / 1024:.1f} GiB for their activations); {available / 1024:.1f} GiB is "
"available. Free VRAM held by other processes, generate a shorter or smaller clip, or set the "
"DiT precision lower."
)
continue
frames_mib = (
-(-int(width) * int(height) * int(frames) * 3 * 4 // (1 << 20)) if shaped else 0
)
if shaped and denoises:
phase_mib = max(
frames_mib if home and not decodes else 0, _VIDEO_DENOISE_ACTIVATION_MIB
)
shortfall = video_offload_shortfall_message(
family = family,
floor_mib = int(floor_mib) + phase_mib,
available_mib = int(available_mib(int(ordinal))),
placement = f"split across GPUs, cuda:{int(ordinal)}",
)
if shortfall is not None:
return shortfall
elif shaped and home and not decodes:
floor_mib = int(floor_mib) + frames_mib
shortfall = video_offload_shortfall_message(
family = family,
floor_mib = int(floor_mib),
available_mib = int(available_mib(int(ordinal))),
placement = f"split across GPUs, cuda:{int(ordinal)}",
width = width if decodes else None,
height = height if decodes else None,
frames = frames if decodes else None,
decode_floor_mib = int(floor_mib) if decodes else None,
)
if shortfall is not None:
return shortfall
return None
def video_offload_shortfall_message(
*,
family: str,
floor_mib: int,
available_mib: int,
placement: str,
width: Optional[int] = None,
height: Optional[int] = None,
frames: Optional[int] = None,
decode_floor_mib: Optional[int] = None,
measured_extra_mib: Optional[int] = None,
) -> Optional[str]:
"""Refusal when co-resident weights plus the activation headroom exceed the device, else None.
``measured_extra_mib``: a recorded peak (above the request's start) of a completed request at least this large
under the same load; it replaces the estimated headroom when it asks for less (margin and slack as the Wan
residency fit uses).
Denoise and VAE decode are separate phases: the decoded-clip share is held only beside the weights still on the
device during the decode (``decode_floor_mib``, the full floor when unknown), never beside the streamed DiT."""
headroom = DEFAULT_BASE_OVERHEAD_MIB
full_floor_mib = int(floor_mib)
required = int(floor_mib) + headroom
if width and height and frames:
# decoded-clip share of the runtime estimate; its fixed 4 GiB denoise base over-refuses measured cells
clip_mib = max(
0,
estimate_video_runtime_mib(width = width, height = height, num_frames = frames)
- _VIDEO_DENOISE_ACTIVATION_MIB,
)
decode_floor = int(
floor_mib if decode_floor_mib is None else min(decode_floor_mib, floor_mib)
)
if decode_floor + headroom + clip_mib > required:
floor_mib, headroom = decode_floor, headroom + clip_mib
required = decode_floor + headroom
if measured_extra_mib is not None and measured_extra_mib > 0:
from .video_stream_residency import MEASURED_PEAK_MARGIN, MEASURED_SLACK_MIB
measured_headroom = int(measured_extra_mib * MEASURED_PEAK_MARGIN) + MEASURED_SLACK_MIB
measured_floor = full_floor_mib
if measured_floor + measured_headroom < required:
floor_mib, headroom = measured_floor, measured_headroom
required = measured_floor + measured_headroom
if int(available_mib) >= required:
return None
shape = f" for {width}x{height} at {frames} frames" if width and height and frames else ""
return (
f"{family} needs about {required / 1024:.1f} GiB of available VRAM{shape} under this load's "
f"placement ({placement}: {floor_mib / 1024:.1f} GiB of weights on the GPU at once plus "
f"{headroom / 1024:.1f} GiB for activations); {available_mib / 1024:.1f} GiB is "
"available. Free VRAM held by other processes, set the text encoder or DiT precision lower, "
"or pick a smaller model."
)
def _progress(phase: Optional[str], **extra: Any) -> dict[str, Any]:
return {"phase": phase, **extra}
# dual-DiT (Wan2.2-A14B MoE) helpers.
# ── dual-DiT (Wan2.2-A14B MoE) helpers ──────────────────────────────────────── The optimisation helpers and the
# quantiser all act on ``pipe.transformer``. Wan2.2-A14B is a dual-expert MoE (transformer = high-noise, transformer_2 =
# low-noise), so present the second DiT AS ``pipe.transformer`` via a thin proxy.
# ── dual-DiT (Wan2.2-A14B MoE) helpers ────────────────────────────────────────
def _calibration_revision(state: Any) -> Optional[str]:
"""The exact weights this pipeline holds (cache snapshot commit / local fingerprint, plus the companion base for a
GGUF / single-file load), read once per pipeline."""
pipe = getattr(state, "pipe", None)
cached = getattr(pipe, "_unsloth_calibration_revision", None)
if cached is not None:
return cached or None
from .diffusion import _peak_model_identity
# the image peak identity's rule: snapshot commit, or a local folder's manifests plus its weight files' size and
# mtime, plus the companion base for a GGUF / single-file load; "unknown" persists nothing
revision = _peak_model_identity(
getattr(state, "repo_id", None),
getattr(state, "gguf_filename", None),
getattr(state, "kind", None),
getattr(state, "base_repo", None),
getattr(state, "fetch_base", None),
)
try:
pipe._unsloth_calibration_revision = revision or ""
except Exception: # noqa: BLE001
pass
return revision
def _transformer_names(pipe: Any, fam: VideoFamily) -> tuple[str, ...]:
"""Attribute names of the denoiser(s) on ``pipe`` to optimise. Just
("transformer",) for a single-DiT family; also "transformer_2" for an MoE family
whose second expert is actually present (a checkpoint may ship only the first)."""
names = ["transformer"]
if fam.is_moe and getattr(pipe, "transformer_2", None) is not None:
names.append("transformer_2")
return tuple(names)
def _video_transformer_quant_backend(state: Any) -> Optional[str]:
"""Which NVFP4 kernel path the loaded denoiser(s) run, or None. Never raises."""
if getattr(state, "transformer_quant", None) != "nvfp4":
return None
try:
pipe = getattr(state, "pipe", None)
if pipe is None:
return None
from .diffusion_nvfp4_linear import is_nvfp4_flashinfer_linear
for name in _transformer_names(pipe, state.family):
denoiser = getattr(pipe, name, None)
if denoiser is None:
continue
declared = getattr(denoiser, "_unsloth_nvfp4_backend", None)
if declared:
return str(declared)
for module in denoiser.modules():
if is_nvfp4_flashinfer_linear(module):
return "flashinfer"
return "torchao"
except Exception: # noqa: BLE001 -- a poll must not fail on a probe
return None
class _SecondDiTView:
"""A thin proxy that makes ``pipe.transformer_2`` look like ``pipe.transformer`` to a
helper that hardcodes ``getattr(pipe, "transformer")``, while every other attribute
(vae, components, __call__, ...) reads through to the real pipe unchanged.
This lets the existing single-DiT helpers optimise the second expert without a fork:
``apply_speed_optims(_SecondDiTView(pipe), ...)`` compiles / caches / sets attention on
``transformer_2``. Only ever wrapped around an MoE pipe (guarded by fam.is_moe)."""
def __init__(self, pipe: Any) -> None:
object.__setattr__(self, "_pipe", pipe)
@property
def transformer(self) -> Any:
return self._pipe.transformer_2
def __getattr__(self, name: str) -> Any:
return getattr(object.__getattribute__(self, "_pipe"), name)
def __setattr__(self, name: str, value: Any) -> None:
# Writes land on the real pipe or helper reassignments vanish; transformer also mirrors to expert 2.
pipe = object.__getattribute__(self, "_pipe")
setattr(pipe, "transformer_2" if name == "transformer" else name, value)
class _NamedDiTView:
"""``_SecondDiTView`` for an arbitrary attribute: presents ``pipe.<attr>`` as
``pipe.transformer`` to the helpers that hardcode that name, reading everything else through.
MiniMax-H3 keeps one denoiser per workflow partition, so a reference load's DiT is
``transformer_ref``. ``_denoiser_dits`` / ``_attention_dits`` enumerate only ``transformer``,
``transformer_2`` and ``unconditional_transformer``, so without this the partition that load
actually denoises with gets neither the selected attention backend nor the regional compile,
while the resolved record still reports the profile that was asked for -- exactly the
over-reporting those two helpers' docstrings exist to rule out.
"""
def __init__(self, pipe: Any, attr: str) -> None:
object.__setattr__(self, "_pipe", pipe)
object.__setattr__(self, "_attr", attr)
@property
def transformer(self) -> Any:
return getattr(self._pipe, object.__getattribute__(self, "_attr"), None)
def __getattr__(self, name: str) -> Any:
return getattr(object.__getattribute__(self, "_pipe"), name)
def __setattr__(self, name: str, value: Any) -> None:
pipe = object.__getattribute__(self, "_pipe")
attr = object.__getattribute__(self, "_attr")
setattr(pipe, attr if name == "transformer" else name, value)
def _denoiser_view(pipe: Any, component: str) -> Any:
"""``pipe`` itself for the usual ``transformer``; a view onto ``component`` otherwise."""
return pipe if component == "transformer" else _NamedDiTView(pipe, component)
def _skip_pipe(state: Any, pipe: Any = None) -> Any:
"""What the static step skip helpers read ``transformer`` from: the pipe, or for MiniMax-H3 a view onto the
partition this load denoises with (a reference load's DiT is ``transformer_ref``)."""
pipe = getattr(state, "pipe", None) if pipe is None else pipe
fam = getattr(state, "family", None)
if pipe is None or not getattr(fam, "modular_workflow", None):
return pipe
return _denoiser_view(pipe, h3_denoiser_component(getattr(state, "h3_task", None)))
def _is_static_cache_request(value: Optional[str]) -> bool:
try:
return normalize_transformer_cache(value) == TC_STATIC
except ValueError:
return False
def _views_for(pipe: Any, fam: VideoFamily) -> tuple[Any, ...]:
"""Pipe views for the transformer helpers, so a dual-DiT MoE's transformer_2 is covered too."""
if fam.is_moe and getattr(pipe, "transformer_2", None) is not None:
return (pipe, _SecondDiTView(pipe))
return (pipe,)
def _log_failed_generation(request_shape: Optional[dict[str, Any]], exc: BaseException) -> None:
"""Logs the request shape on failure; cancels are not reported. Must never replace the real error."""
if request_shape is None:
return
try:
if str(exc) in (VIDEO_CANCELLED_MSG, VIDEO_NOT_LOADED_MSG):
return
if type(exc).__name__ == "_VideoGenerationCancelled":
return
# ValueError is client input feedback: no generate_failed record, no ERROR log.
if isinstance(exc, ValueError):
return
logger.error(
"video.generate_failed_request: %s",
" ".join(f"{k}={v}" for k, v in request_shape.items()),
)
# Math SDPA OOM (#8225) is quadratic in tokens; only diagnose when native SDPA actually ran.
if (
_is_out_of_memory(exc)
and request_shape.get("attention_backend") is None
and sdpa_math_only(
_probe_target(request_shape), request_shape.get("attention_head_dims")
)
):
logger.error("video.generate_failed_request: %s", SDPA_MATH_ONLY_MESSAGE)
except Exception: # noqa: BLE001 - diagnostics never mask the real failure
pass
VIDEO_GENERATION_FAILED_MSG = "Video generation failed."
_NATIVE_CRASH_NEEDLES = (
"sd-cli exited",
"process exited",
"ggml_abort",
"signal",
"connection lost",
"worker died",
)
_FAILURE_REASON_MAX_CHARS = 120
_PATH_START = r"(?:file://)?(?:~[\\/]|/|[A-Za-z]:[\\/]|\\\\)"
# Absolute paths (quoted ones may hold spaces); URLs are left for redact_log_text.
_FAILURE_PATH_RE = re.compile(
r"(?P<url>\b(?:https?|wss?|ftp)://[^\s\"'`,;]+)"
rf"|(?P<q>['\"`]){_PATH_START}[^\n]*?(?P=q)"
rf"|(?<![\w/~]){_PATH_START}[^\s\"'`,;]+"
)
def _failure_reason(exc: BaseException) -> str:
"""Exception class and first message line, without credentials or filesystem paths."""
from utils.log_redaction import redact_log_text
lines = str(exc).strip().splitlines()
text = redact_log_text(lines[0] if lines else "")
text = _FAILURE_PATH_RE.sub(lambda m: m["url"] or f"{m['q'] or ''}<path>{m['q'] or ''}", text)
text = " ".join(text.split())
reason = f"{type(exc).__name__}: {text}" if text else type(exc).__name__
if len(reason) > _FAILURE_REASON_MAX_CHARS:
cut = reason[: _FAILURE_REASON_MAX_CHARS - 3]
reason = (cut.rsplit(" ", 1)[0] if " " in cut else cut).rstrip(" ,;:") + "..."
return reason
def video_failure_detail(exc: BaseException, request_shape: Optional[dict[str, Any]] = None) -> str:
"""A user-facing reason for a failed generation: fixed text for known classes, else the
redacted exception class and first line."""
try:
from .video_minimax_h3 import H3NativeMemoryRefusal
if isinstance(exc, H3NativeMemoryRefusal):
return f"{VIDEO_GENERATION_FAILED_MSG} {exc}"
if isinstance(exc, NonFiniteFramesError):
return f"{VIDEO_GENERATION_FAILED_MSG} {NON_FINITE_FRAMES_MSG}"
native_memory = _native_memory_failure(exc)
if native_memory:
return f"{VIDEO_GENERATION_FAILED_MSG} {native_memory}"
if _is_out_of_memory(exc):
if (
request_shape is not None
and request_shape.get("attention_backend") is None
and sdpa_math_only(
_probe_target(request_shape), request_shape.get("attention_head_dims")
)
):
return (
f"{VIDEO_GENERATION_FAILED_MSG} Out of memory: this device has no fused attention "
"kernel. Lower the resolution or frame count."
)
return (
f"{VIDEO_GENERATION_FAILED_MSG} Out of memory. Try a smaller resolution, fewer "
"frames or fewer steps."
)
text = str(exc).lower()
if any(needle in text for needle in _NATIVE_CRASH_NEEDLES):
return f"{VIDEO_GENERATION_FAILED_MSG} The renderer crashed. See Settings > Logs."
return f"{VIDEO_GENERATION_FAILED_MSG} {_failure_reason(exc)}"
except Exception: # noqa: BLE001 -- diagnostics never mask the real failure
pass
return VIDEO_GENERATION_FAILED_MSG
# How a Vulkan (radv) device reports running out of memory.
_NATIVE_DEVICE_LOST_NEEDLES = ("device lost", "errordevicelost", "error_device_lost")
NATIVE_MEMORY_ADVICE = (
"Set Memory mode to Low VRAM, pick a smaller resolution (864x480 is the smallest preset), or load a "
"smaller quant."
)
def _native_memory_failure(exc: BaseException) -> Optional[str]:
"""The advice for a native (sd.cpp) renderer that ran out of GPU or GPU-mapped memory, else None."""
text = str(exc)
lowered = text.lower()
if not any(n in lowered for n in ("sd-cli exited", "sd-server", "stable-diffusion.cpp")):
return None
from .sd_cpp_backend import output_shows_capacity_failure
if output_shows_capacity_failure(text):
return f"The native renderer ran out of GPU or system memory. {NATIVE_MEMORY_ADVICE}"
if any(n in lowered for n in _NATIVE_DEVICE_LOST_NEEDLES):
return (
"The GPU driver reset the device during the render, which usually means it ran out of GPU or "
f"GPU-mapped system memory. {NATIVE_MEMORY_ADVICE}"
)
return None
def _is_out_of_memory(exc: BaseException) -> bool:
"""Whether ``exc`` is an allocator failure, by name and text rather than by class: torch
raises ``torch.OutOfMemoryError`` on CUDA and a plain RuntimeError on some backends."""
if any(cls.__name__ in ("OutOfMemoryError", "OutOfMemoryError_") for cls in type(exc).__mro__):
return True
text = str(exc).lower()
return "out of memory" in text or "tried to allocate" in text
def _probe_target(request_shape: dict[str, Any]) -> Any:
"""Carries the run's recorded dtype; the default fp16 probe would answer for a precision never used."""
dtype = None
recorded = request_shape.get("dtype")
if recorded:
try:
import torch
candidate = getattr(torch, str(recorded).replace("torch.", ""), None)
if isinstance(candidate, torch.dtype):
dtype = candidate
except Exception: # noqa: BLE001 - fall back to the probe's own default
dtype = None
return types.SimpleNamespace(device = request_shape.get("device"), dtype = dtype)
@functools.cache
def _video_family_capabilities(
*, detect_device: bool = True
) -> tuple[tuple[str, ...], tuple[str, ...]]:
# Cache provisional and measured catalogs separately: idle status must not import torch,
# and on Metal the actual device probe allocates and syncs, so later polls reuse its answer.
device = resolve_diffusion_device_target().device if detect_device else None
available = pipeline_available_video_families(device = device)
return (
tuple(fam.name for fam in available),
tuple(fam.name for fam in available if fam.modular_workflow),
)
def _return_direct_loaded_modules(
pipe: Any,
plan: Any,
logger: Any = None,
) -> None:
"""Move what the LTX-2.3 direct load put on the card back to the host when a refined ``plan`` offloads it."""
try:
import torch
if not plan_keeps_transformer_resident(plan):
moved = ["pipeline"]
pipe.to("cpu")
elif getattr(plan, "offload_policy", OFFLOAD_NONE) != OFFLOAD_NONE and bool(
getattr(plan, "stream_text_encoders", False)
):
moved = []
for name, comp in getattr(pipe, "components", {}).items():
if not name.startswith("text_encoder") or not isinstance(comp, torch.nn.Module):
continue
if any(p.device.type != "cpu" for p in comp.parameters()):
comp.to("cpu")
moved.append(name)
else:
return
if moved and logger is not None:
logger.info(
"video.ltx23_direct_load: plan %s offloads %s; moved back to the host",
_video_plan_label(plan),
", ".join(moved),
)
except Exception as exc: # noqa: BLE001 - placement below still applies the plan
if logger is not None:
logger.warning(
"video.ltx23_direct_load: could not return modules to the host (%s)", exc
)
class VideoBackend:
"""One loaded video pipeline; loads swap it atomically (same model as images)."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._generate_lock = threading.Lock()
self._state: Optional[_VideoLoadState] = None
self._loading: Optional[_VideoLoadingState] = None
self._load_token = 0
self._cancel_event = threading.Event()
self._active_generate_cancel: Optional[threading.Event] = None
# (load token, client) of an LTX-2.5 Gemma 4 encoder worker whose load has not committed yet; a failed load stops
# it.
self._ltx25_pending_encoder: Optional[tuple[Optional[int], Any]] = None
self._float_seeds: Optional[tuple[Any, Any]] = None
# How many unloads / superseding loads are waiting on _generate_lock to free this pipeline. A generation queued
# behind the active one holds no cancel event yet, so without this fence it could win the lock after an eject
# and denoise a whole new clip against a pipeline being freed. A count, so concurrent teardowns each own their
# own release.
self._teardown_waiters = 0
# Generation progress, written by the step callback / phase transitions.
self._gen: dict[str, Any] = {"active": False}
self._last_request_shape: Optional[dict[str, Any]] = None
# True from begin_generate() to the worker's terminal state; a second generate is refused meanwhile.
self._generate_job_active = False
# Identifies the job owning the flag; compared by identity.
self._generate_job_token: Optional[object] = None
self._generate_job_account: Optional[str] = None
# The OpenAI /v1/videos job id this run was started under, or None for a Studio-page run
self._gen_video_id: Optional[str] = None
def _device_target(self, ordinal: Optional[int] = None) -> DiffusionDeviceTarget:
"""Pins the device per worker thread: torch.cuda.set_device is thread-local, not set once at
load."""
if ordinal is None:
return resolve_diffusion_device_target()
target = resolve_diffusion_device_target(ordinal = ordinal)
apply_diffusion_device_ordinal(target)
return target
def _inline_float_seeds(self, state: Any) -> Any:
"""This load's ``InlineFloatSeeds`` until its first compiled clip kept its names, else None."""
held = self._float_seeds
if held is not None and held[0] is state:
return None if held[1] is None or held[1].done() else held[1]
seeds = None
try:
if getattr(state, "bg_compile", None) is None:
from . import diffusion_bg_compile
dits, seed_dir = _video_float_seed_dir(
state.pipe,
family = getattr(state.family, "name", None),
dtype = state.dtype,
quant = state.transformer_quant,
attention_backend = state.attention_backend,
)
if seed_dir is not None:
seeds = diffusion_bg_compile.InlineFloatSeeds(dits, seed_dir, logger = logger)
except Exception: # noqa: BLE001 - no seeds, main's compile
seeds = None
self._float_seeds = (state, seeds)
return seeds
def _state_device_target(self, state: _VideoLoadState) -> DiffusionDeviceTarget:
"""The resident pipeline's target, pinned onto the calling thread. Every worker that
touches the loaded pipeline goes through this: the weights are on ``state.gpu_ordinal``
and ``state.device`` is un-indexed, so an unpinned thread resolves to its own card."""
target = self._device_target(state.gpu_ordinal)
# And an automatic load back onto its own card, in case this thread is a shared one a previous pinned load left
# elsewhere. The image path reaches this through a pooled executor; here it costs one no-op call to keep the two
# engines identical. Skipped when the pin above already set that card.
if state.gpu_ordinal is None or state.placed_ordinal != state.gpu_ordinal:
pin_cuda_ordinal(state.placed_ordinal)
return target
# ── validation ───────────────────────────────────────────────────────────
def validate_load_request(
self,
repo_id: str,
*,
gguf_filename: Optional[str] = None,
base_repo: Optional[str] = None,
family_override: Optional[str] = None,
display_repo_id: Optional[str] = None,
model_kind: Optional[str] = None,
transformer_quant: Optional[str] = None,
text_encoder_quant: Optional[str] = None,
h3_task: Optional[str] = None,
) -> VideoFamily:
"""Cheap, network-free validation shared by the route and the load path."""
kind = resolve_video_model_kind(gguf_filename, model_kind)
# A -GGUF repo without a quant filename would fail in from_pretrained after eviction.
if kind == "pipeline" and repo_id.strip().lower().rstrip("/").endswith("-gguf"):
raise ValueError(
f"'{repo_id}' is a GGUF repo: pick one of its .gguf files "
"(gguf_filename) instead of loading it as a diffusers pipeline."
)
_refuse_non_dit_pick(repo_id, gguf_filename, "video")
fam = _detect_load_family(
repo_id, gguf_filename, family_override, display_repo_id, model_kind
)
# Every pick name is judged, not just an unresolved one: a T2V repo id can hold an I2V / VACE file.
# A family override names the family for an opaque pick; it cannot make refused weights runnable.
from .video_families import unsupported_video_checkpoint_reason
reason = unsupported_video_checkpoint_reason(
display_repo_id,
repo_id,
gguf_filename,
wan21_checkpoint = (gguf_filename or repo_id)
if fam and fam.name.startswith("wan2.1")
else None,
)
if reason:
raise ValueError(reason)
if fam is None:
raise ValueError(
f"'{repo_id}' is not a supported text-to-video model. Supported families: "
f"{', '.join(supported_video_family_names())}. If this is a variant of one "
f"of them, pass family_override with that family name."
)
# Network-free, so refused before any eviction.
refusal = video_checkpoint_refusal(fam, repo_id, display_repo_id, gguf_filename)
if refusal:
raise ValueError(refusal)
if fam.name in COSMOS3_FAMILIES:
from .video_cosmos3 import load_request_refusal as cosmos3_load_request_refusal
refusal = cosmos3_load_request_refusal(fam.name, kind, text_encoder_quant)
if refusal:
raise ValueError(refusal)
# ── modular-workflow refusals, before anything heavier. Deliberately the FIRST thing after the family resolves:
# everything below reaches into diffusers (assert_pipeline_class_available, the transformer_class probe), so a
# refusal placed after them would be unreachable on any install whose diffusers cannot even be imported -- and
# these two are exactly the picks that cost the most to discover late.
# ── modular-workflow refusals, before anything heavier.
from .video_minimax_h3_comfy import is_h3_comfy_name
if fam.modular_workflow and kind == "single_file" and is_h3_comfy_name(gguf_filename):
from .video_minimax_h3_comfy import h3_comfy_task
file_task = h3_comfy_task(gguf_filename)
if h3_task and str(h3_task).strip().lower() != file_task:
raise ValueError(
f"'{Path(str(gguf_filename)).name}' is the {file_task} partition, but the load asked for "
f"{h3_task}. Pick the matching checkpoint."
)
if fam.modular_workflow and kind == "single_file" and not is_h3_comfy_name(gguf_filename):
# A modular workflow has no single-file assembly; refuse before the download and the
# teardown.
gguf_hint = (
f", or a .gguf checkpoint from '{fam.gguf_repo}' for a quantized single-file load"
if fam.gguf_repo
else ""
)
raise ValueError(
f"'{fam.name}' cannot load from a single .safetensors checkpoint: it is assembled "
f"by its Modular Diffusers workflow, which builds every component from a repo. "
f"Load the diffusers pipeline repo '{fam.base_repo}' for the full bfloat16 "
f"model{gguf_hint}, or a ComfyUI-quantized MiniMax-H3 denoiser "
f"(minimax_h3_*_int8_convrot / *_fp8_scaled .safetensors) as its denoiser."
)
if fam.modular_workflow and kind in ("pipeline", "single_file"):
# Refuse on Metal up front: torch.mps lacks mem_get_info, which auto CPU offload needs.
if resolve_diffusion_device_target().device == "mps":
gguf_hint = (
f" Load a .gguf checkpoint from '{fam.gguf_repo}' instead, which runs on the "
f"native engine."
if fam.gguf_repo
else ""
)
raise ValueError(
f"'{fam.name}' cannot run on Apple Silicon: its Modular Diffusers workflow "
f"places components through the Diffusers auto CPU offload, which needs a "
f"torch device exposing mem_get_info, and Metal (MPS) does not have it."
f"{gguf_hint}"
)
if fam.modular_workflow and kind == "pipeline":
# Same normaliser the load uses, so a malformed value raises the identical message it would below and only a
# real scheme reaches the availability check.
requested_scheme = normalize_transformer_quant(transformer_quant)
# A pipeline's modular base IS repo_id, so validation judges the variant the load will ask for.
quant_base = repo_id
# The "auto" choice is the backend's own: never refused, it stays on released bfloat16 components.
if requested_scheme is not None and requested_scheme != TQ_AUTO:
# Per (scheme, task): partitions share shapes, so a mismatched checkpoint would silently load.
if not video_family_prequant_available(
fam, requested_scheme, task = h3_task, base_repo = quant_base
):
available = video_family_prequant_schemes(fam, task = h3_task)
hint = (
f"Use one of: {', '.join(available)}."
if available
else "Leave transformer_quant unset for this model."
)
# Name the partition asked for, so "fp8 is unavailable" is not read as family-wide.
task_label = f" {h3_task}" if h3_task else ""
raise ValueError(
f"transformer_quant '{requested_scheme}' is unavailable for "
f"'{fam.name}'{task_label}: its Modular Diffusers workflow builds the "
f"denoiser itself, so only schemes with a hosted pre-quantized checkpoint "
f"can be applied and the dense weights cannot be quantized in place. "
f"{hint}"
)
if kind == "pipeline":
from .diffusion_families import local_pipeline_components_are_complete
root = Path(repo_id).expanduser()
index = "modular_model_index.json" if fam.modular_workflow else "model_index.json"
if root.exists() and not (
root.is_dir() and local_pipeline_components_are_complete(root, index)
):
raise ValueError(
f"Local pipeline path is not a diffusers directory (no valid {index}): {repo_id}"
)
from .video_minimax_h3 import is_h3_native, validate_h3_transformer_filename
if is_h3_native(fam, kind):
validate_h3_transformer_filename(gguf_filename or "")
# The GGUF filename and explicit task must name the same partition.
picked = h3_transformer_task(gguf_filename or "")
if h3_task and h3_task != picked:
raise ValueError(
f"'{Path(gguf_filename or '').name}' is the {picked} partition, but the "
f"load asked for {h3_task}. Pick the matching checkpoint."
)
else:
# assert_pipeline_class_available closes the dynamo window before its own import diffusers.
from .diffusion_families import assert_pipeline_class_available
assert_pipeline_class_available(fam.pipeline_class, fam.name)
if fam.modular_workflow:
import diffusers
if not hasattr(diffusers, fam.transformer_class):
raise ValueError(
"MiniMax-H3 needs the Diffusers revision bundled with this Unsloth "
"version. Reinstall Unsloth dependencies and retry."
)
# A lone .safetensors file is trusted per file, from any repo (diffusion_single_file_trust).
if kind != "gguf" and not single_file_load_allowed(
_is_trusted_video_repo(repo_id), kind, gguf_filename
):
raise ValueError(
f"Non-GGUF video loads from '{repo_id}' are limited to a single .safetensors "
f"checkpoint; full pipelines and other weight formats load only from unsloth/* "
f"repos, the official family base repos, and local paths."
)
# After the trust gate, so an untrusted repo is refused as untrusted.
if fam.name in COSMOS3_FAMILIES and kind == "pipeline":
from .video_cosmos3 import guardrail_refusal
refusal = guardrail_refusal(repo_id)
if refusal:
raise ValueError(refusal)
# Companions load with from_pretrained, so a base repo is held to the non-GGUF bar: a GGUF pick must not smuggle
# in a remote base.
if base_repo and (base_repo or "").strip() and not _is_trusted_video_repo(base_repo):
raise ValueError(
f"base_repo is limited to unsloth/* repos, the official family base "
f"repos, and local paths; '{base_repo}' is neither."
)
if base_repo and base_repo.strip():
# The companion supplies the text encoder / VAE, so it must be the picked release's own base.
assert_ltx_pick_supported(fam.name, base_repo)
assert_local_pipeline_family(base_repo, fam.name)
# A local base_repo loads as a full pipeline (needs model_index.json); reject a non-pipeline one here, before
# the load.
from core.inference.diffusion import _assert_local_base_is_pipeline
excluded = (
((fam.denoiser_attr, "transformer_2") if fam.is_moe else (fam.denoiser_attr,))
if kind in ("gguf", "single_file")
else ()
)
# A local copy of the split LTX-2.5 release has no model_index.json by design: its companion files beside the
# DiTs are the base (assemble_ltx25_single_file_pipeline).
split_ltx25_base = False
if fam.name == "ltx-2.5" and kind in ("gguf", "single_file") and base_repo:
from .video_ltx25 import is_ltx25_single_file_base
split_ltx25_base = is_ltx25_single_file_base(base_repo)
if not split_ltx25_base:
_assert_local_base_is_pipeline(base_repo, excluded_components = excluded)
if kind in ("gguf", "single_file") and not gguf_filename:
raise ValueError("A gguf/single_file load needs the checkpoint filename.")
local_file = Path(repo_id).expanduser().is_file()
if (
kind in ("gguf", "single_file")
and fam.is_moe
and not moe_pick_pairs(fam, repo_id if local_file else gguf_filename)
):
raise ValueError(
f"'{fam.name}' is a dual-expert model: a single {kind} file covers only "
f"one of its two transformers. Pick one expert of a high_noise / low_noise pair "
f"(Unsloth loads its partner from the same folder), or load the diffusers "
f"pipeline repo ('{fam.base_repo}')."
)
# A missing local checkpoint must fail HERE, before the route evicts a resident model.
if kind in ("gguf", "single_file"):
# Fail a kind/extension mismatch before the GPU handoff: gguf needs .gguf, single_file needs .safetensors.
is_gguf_name = (gguf_filename or "").lower().endswith(".gguf")
if kind == "gguf" and not is_gguf_name:
raise ValueError("a 'gguf' load requires a .gguf checkpoint name.")
if kind == "single_file" and is_gguf_name:
raise ValueError("a .gguf checkpoint needs model_kind 'gguf', not 'single_file'.")
if kind == "single_file" and not (gguf_filename or "").lower().endswith(".safetensors"):
raise ValueError(
f"'{gguf_filename}' is not a loadable single-file checkpoint "
f"(expected a .safetensors name; use a .gguf name for a GGUF load)."
)
root = Path(repo_id).expanduser()
# Path-shaped picks (./ ../ ~ backslash, absolute) fail here, before the handoff.
path_shaped = (
repo_id.startswith(("/", "\\", "~", ".")) or "\\" in repo_id or root.is_absolute()
)
if root.is_dir():
from .diffusion_families import resolve_local_gguf_child
try:
resolve_local_gguf_child(root, gguf_filename or "")
if fam.is_moe:
resolve_local_gguf_child(root, moe_partner_filename(gguf_filename) or "")
except Exception as exc: # noqa: BLE001 -- surface as client input error
raise ValueError(str(exc)) from exc
elif root.is_file():
if fam.is_moe:
partner = root.with_name(moe_partner_filename(root.name) or root.name)
if not partner.is_file():
raise ValueError(
f"'{root.name}' is one expert of a dual-expert pair, but its partner "
f"'{partner.name}' is not in the same folder."
)
# The loader hands a local FILE straight through (ignoring gguf_filename), so the file's own suffix must
# match the kind.
suffix = root.suffix.lower()
if kind == "gguf" and suffix != ".gguf":
raise ValueError(
f"Local checkpoint '{repo_id}' is not a .gguf file; a 'gguf' load "
f"needs a .gguf checkpoint."
)
if kind == "single_file" and suffix != ".safetensors":
raise ValueError(
f"Local checkpoint '{repo_id}' is not a .safetensors file; a "
f"'single_file' load needs a .safetensors checkpoint."
)
elif path_shaped:
raise ValueError(f"Local model path '{repo_id}' does not exist.")
# Reject malformed transformer_quant before the handoff; pipeline kind only, like the image backend.
normalize_transformer_quant(transformer_quant)
# Reject a malformed text_encoder_quant the same way. Every kind: an unsupported scheme is a bad request
# regardless of whether this family has a hosted quantized encoder for it.
if fam.name == "ltx-2.5":
from .video_ltx25 import ltx25_te_quant_scheme
ltx25_te_quant_scheme(text_encoder_quant)
_ensure_mp4_encoder_available()
return fam
# ── background load + progress ───────────────────────────────────────────
@pauses_background_embedding
@vram_budget_scope("video", load = True)
def begin_load(
self,
repo_id: str,
*,
display_repo_id: Optional[str] = None,
local_files_only: bool = False,
gguf_filename: Optional[str] = None,
base_repo: Optional[str] = None,
family_override: Optional[str] = None,
hf_token: Optional[str] = None,
memory_mode: Optional[str] = None,
speed_mode: Optional[str] = None,
attention_backend: Optional[str] = None,
transformer_cache: Optional[str] = None,
transformer_cache_threshold: Optional[float] = None,
transformer_quant: Optional[str] = None,
text_encoder_quant: Optional[str] = None,
model_kind: Optional[str] = None,
h3_task: Optional[str] = None,
gpu_ids: Optional[list[int]] = None,
# The ordinal the ROUTE already ranked, so the preflight and the load agree on one card
gpu_ordinal: Optional[int] = None,
allow_oversized_load: bool = False,
nvfp4_route: Any = None,
) -> dict[str, Any]:
"""Validate, then run the (slow) load on a daemon thread. Returns at once."""
hf_token = (hf_token.strip() if isinstance(hf_token, str) else hf_token) or None
from .diffusion_nvfp4_route import ROUTE_UNCHANGED, plan_nvfp4_route
if nvfp4_route is ROUTE_UNCHANGED:
nvfp4_route = None
elif nvfp4_route is None:
nvfp4_route = plan_nvfp4_route(
media = "video",
model_path = repo_id,
gguf_filename = gguf_filename,
model_kind = model_kind,
base_repo = base_repo,
family_override = family_override,
transformer_quant = transformer_quant,
text_encoder_quant = text_encoder_quant,
gpu_ordinal = gpu_ordinal,
gpu_ids = gpu_ids,
)
if nvfp4_route is not None:
updates = nvfp4_route.updates
repo_id = updates.get("model_path", repo_id)
display_repo_id = updates.get("display_repo_id", display_repo_id)
gguf_filename = updates.get("gguf_filename", gguf_filename)
model_kind = updates.get("model_kind", model_kind)
base_repo = updates.get("base_repo", base_repo)
family_override = updates.get("family_override", family_override)
transformer_quant = updates.get("transformer_quant", transformer_quant)
text_encoder_quant = updates.get("text_encoder_quant", text_encoder_quant)
# Resolved ONCE, here, and carried to the worker: outside it so a bad pick is the route's 400 rather than a load
# that dies tens of GB later, and only once so free VRAM cannot re-rank the choice after the weights land. Gated
# on the resolved backend, since XPU / MPS / CPU ignore physical ids and would otherwise 400 a selection the
# contract drops. Re-ranked only when the caller did not already do it: free VRAM moves between the route's
# preflight and here, so resolving twice can approve a scheme against one card and place the weights on another.
auto_ordinal = gpu_ordinal is None and not gpu_ids
if gpu_ordinal is None and gpu_ids:
gpu_ordinal = (
resolve_selected_cuda_ordinal(gpu_ids)
if resolve_diffusion_device_target().device == "cuda"
else None
)
gpu_ordinals = (
resolve_selected_cuda_ordinals(gpu_ids)
if gpu_ids
and len(set(gpu_ids)) > 1
and multigpu_mode() != "off"
and resolve_diffusion_device_target().device == "cuda"
else AUTOMATIC_ORDINALS
if not gpu_ids
else []
)
fam = self.validate_load_request(
repo_id,
gguf_filename = gguf_filename,
base_repo = base_repo,
family_override = family_override,
display_repo_id = display_repo_id,
model_kind = model_kind,
transformer_quant = transformer_quant,
text_encoder_quant = text_encoder_quant,
h3_task = h3_task,
)
if auto_ordinal:
# Automatic, after validation: one GPU, Apple, CPU and ROCm skip every probe here.
gpu_ordinal = auto_cuda_ordinal_for_load()
# Refuse an EXPLICIT precision this host can never honor BEFORE the load starts, so the route answers 409 with
# the reason instead of evicting the resident model and pulling tens of GB first. Footprint-dependent declines
# still surface via load-progress. The ROUTE makes the same call earlier still (see
# assert_video_precision_available): this one runs inside acquire_for, which has already evicted chat by the
# time it fires.
assert_video_precision_available(
fam,
model_kind = resolve_video_model_kind(gguf_filename, model_kind),
transformer_quant = transformer_quant,
text_encoder_quant = text_encoder_quant,
memory_mode = memory_mode,
gpu_ordinal = gpu_ordinal,
checkpoint_filename = gguf_filename,
checkpoint_repo = repo_id,
)
# Publish the companion claim with _loading, or a delete in the gap removes a needed repo.
from .video_minimax_h3 import (
H3_COMPONENT_REPO,
H3_GGUF_REPO,
is_h3_native,
)
h3_native = is_h3_native(fam, resolve_video_model_kind(gguf_filename, model_kind))
# Both ids: an install may hold the components under either the mirror or the repack it was mirrored from, and
# whichever one this load reads must be protected.
claimed_assets = (H3_GGUF_REPO, H3_COMPONENT_REPO) if h3_native else ()
# Claim the hosted LTX-2.3 FP8 DiT repo with _loading.
if _ltx23_prequant_serves_on_card(
fam,
resolve_video_model_kind(gguf_filename, model_kind),
gguf_filename,
transformer_quant,
memory_mode = memory_mode,
gpu_ordinal = gpu_ordinal,
checkpoint_repo = repo_id,
):
from .video_ltx2 import LTX23_PREQUANT_BASE
claimed_assets = claimed_assets + self._denoiser_prequant_repo_ids(
fam, TQ_FP8, LTX23_PREQUANT_BASE
)
with self._lock:
if self._loading is not None and self._loading.error is None:
raise RuntimeError("A video load is already in progress.")
self._load_token += 1
token = self._load_token
# A new event per load: clear() would un-cancel the event the previous worker still watches.
cancel_event = threading.Event()
self._cancel_event = cancel_event
self._loading = _VideoLoadingState(
repo_id = repo_id,
# A split LTX-2.5 single-file pick reads its companions from its own repo, not the pack.
base_repo = resolve_video_base_repo(fam, base_repo, repo_id)
if fam.name == "ltx-2.5"
and resolve_video_model_kind(gguf_filename, model_kind) != "pipeline"
else fam.base_repo,
asset_repos = claimed_assets,
)
account_thread(
target = carry_load_budget(self._run_load),
kwargs = dict(
repo_id = repo_id,
display_repo_id = display_repo_id,
local_files_only = local_files_only,
gguf_filename = gguf_filename,
base_repo = base_repo,
family_override = family_override,
hf_token = hf_token,
memory_mode = memory_mode,
speed_mode = speed_mode,
attention_backend = attention_backend,
transformer_cache = transformer_cache,
transformer_cache_threshold = transformer_cache_threshold,
transformer_quant = transformer_quant,
text_encoder_quant = text_encoder_quant,
model_kind = model_kind,
h3_task = h3_task,
gpu_ordinal = gpu_ordinal,
allow_oversized_load = bool(allow_oversized_load),
_load_token = token,
_cancel_event = cancel_event,
_gpu_ordinals = gpu_ordinals,
_nvfp4_route = nvfp4_route,
),
daemon = True,
).start()
return self.status()
# Outermost, so the resolved record reports the NVFP4 ask it replaced.
@with_route_scope
@scoped_allow_oversized_load
# The multi-GPU selection (begin_load's ``_gpu_ordinals``) is visible to this load's planner and placement only.
@with_load_selection
@scoped_local_files_only
@vram_budget_scope("video", load = True)
def _run_load(self, **kwargs: Any) -> None:
token = kwargs.get("_load_token")
# This load's own event: a later load replaces self._cancel_event rather than clearing it.
cancel_event = kwargs.pop("_cancel_event", None) or self._cancel_event
# API loads must download nothing, metadata probes included; read, not popped (load_pipeline).
local_files_only = bool(kwargs.get("local_files_only"))
# Before any download: unsloth_zoo enters the dynamo cycle from the inductor side, racing the warm.
try:
from utils.torch_warmup import gate_torch_stack_import
gate_torch_stack_import("video load", logger)
except Exception as exc: # noqa: BLE001 - a safety net, never a new failure
logger.debug("dynamo import gate skipped: %r", exc)
try:
fam = _detect_load_family(
kwargs["repo_id"],
kwargs.get("gguf_filename"),
kwargs.get("family_override"),
kwargs.get("display_repo_id"),
kwargs.get("model_kind"),
)
# Here, not in validate_load_request, which is network-free by contract.
_assert_pick_is_not_speech(
kwargs["repo_id"],
kwargs.get("gguf_filename"),
kwargs.get("hf_token"),
allow_network = not local_files_only,
)
kind = resolve_video_model_kind(kwargs.get("gguf_filename"), kwargs.get("model_kind"))
ltx25_dev = _ltx25_dev_load(
fam,
kind,
kwargs.get("gguf_filename"),
kwargs["repo_id"],
kwargs.get("display_repo_id"),
)
from .video_minimax_h3 import is_h3_native
if is_h3_native(fam, kind):
# local_files_only is a named parameter there, so bind it here instead of letting kwargs swallow it.
self._run_load_h3_native(
fam = fam,
token = token,
cancel_event = cancel_event,
**kwargs,
)
with self._lock:
if self._load_token == token:
self._loading = None
return
base = (
kwargs["repo_id"]
if kind == "pipeline"
else resolve_video_base_repo(fam, kwargs.get("base_repo"), kwargs.get("repo_id"))
)
kwargs["base_repo"] = base
# Before the plan: it stages the partition the file serves (transformer_ref/ for ref2va).
kwargs["h3_task"] = _h3_single_file_task(
fam, kind, kwargs.get("gguf_filename"), kwargs.get("h3_task")
)
# An fp8 encoder request loads a hosted pre-cast checkpoint, so neither the estimate nor the pull includes
# those dense shards. Resolved through the family default the loader applies, so an unset request on a
# family whose default is a hosted encoder does not stage the dense one. kwargs keep the RAW request.
te_planned, _ = video_te_quant_request(
fam,
kwargs.get("text_encoder_quant"),
base = base,
speed_mode = kwargs.get("speed_mode"),
)
te_sources = self._te_prequant_sources(
fam,
te_planned,
kwargs.get("gpu_ordinal"),
base = base,
)
# Verify the hosted denoiser here, or keep dense shards: it drops 66 GB and is never re-
# checked.
h3_auto_denoiser = (
self._h3_planned_auto_denoiser_scheme(
fam,
base = base,
transformer_quant = kwargs.get("transformer_quant"),
text_encoder_quant = kwargs.get("text_encoder_quant"),
speed_mode = kwargs.get("speed_mode"),
h3_task = kwargs.get("h3_task"),
gpu_ordinal = kwargs.get("gpu_ordinal"),
)
if kind != "single_file"
else None
)
native_hosted_out: list = []
video_auto_denoiser = self._video_planned_auto_denoiser_scheme(
fam,
base = base,
kind = kind,
transformer_quant = kwargs.get("transformer_quant"),
speed_mode = kwargs.get("speed_mode"),
memory_mode = kwargs.get("memory_mode"),
text_encoder_quant = te_planned,
gpu_ordinal = kwargs.get("gpu_ordinal"),
hf_token = kwargs.get("hf_token"),
ltx25_dev = ltx25_dev,
native_hosted = native_hosted_out,
)
video_seed_declined = video_auto_denoiser == DENOISER_SEED_DECLINED
if video_seed_declined:
video_auto_denoiser = None
conventional_denoiser = kind == "pipeline" and not getattr(
fam, "modular_workflow", None
)
requested_denoiser = None if conventional_denoiser else kwargs.get("transformer_quant")
skip_transformer_weights = self._denoiser_prequant_verified(
fam,
h3_auto_denoiser or video_auto_denoiser or requested_denoiser,
base,
kwargs.get("h3_task"),
kwargs.get("hf_token"),
local_files_only = local_files_only,
kind = kind,
)
# Offloaded int8 read from the hosted file as native Linears: its dense shards are not staged either.
native_hosted_planned = None
if (
video_seed_declined
and native_hosted_out
and not skip_transformer_weights
and self._denoiser_prequant_verified(
fam,
native_hosted_out[0],
base,
kwargs.get("h3_task"),
kwargs.get("hf_token"),
local_files_only = local_files_only,
kind = kind,
)
):
skip_transformer_weights = True
native_hosted_planned = native_hosted_out[0]
kwargs["_video_native_hosted_planned"] = native_hosted_planned
# The scheme whose hosted artifact replaces the dense shards (claimed, then fetched under the cancel event).
planned_denoiser = (
h3_auto_denoiser
or video_auto_denoiser
or requested_denoiser
or native_hosted_planned
)
skip_transformer_components = (
_planned_denoiser_components(fam, kind) if skip_transformer_weights else ()
)
kwargs["_h3_auto_denoiser_planned"] = (
h3_auto_denoiser if h3_auto_denoiser and skip_transformer_weights else None
)
kwargs["_video_auto_denoiser_planned"] = (
DENOISER_SEED_DECLINED
if video_seed_declined
else (
video_auto_denoiser
if video_auto_denoiser and skip_transformer_weights
else None
)
)
kwargs["_denoiser_prequant_skipped"] = skip_transformer_components
h3_te_scheme = self._h3_te_quant_scheme_verified(
fam,
kwargs.get("text_encoder_quant"),
base,
kwargs.get("hf_token"),
local_files_only = local_files_only,
)
expected = self._estimate_download_bytes(
kwargs["repo_id"],
kwargs.get("gguf_filename"),
base,
kwargs.get("hf_token"),
kind,
te_sources = te_sources,
skip_transformer_weights = skip_transformer_weights,
skip_transformer_components = skip_transformer_components,
h3_task = kwargs.get("h3_task"),
h3_te_scheme = h3_te_scheme,
local_files_only = local_files_only,
ltx25_dev = ltx25_dev,
planned_files = (planned_files := {}),
)
with self._lock:
if self._load_token == token and self._loading is not None:
self._loading.base_repo = base
if hasattr(self._loading, "fetch_plan"):
self._loading.fetch_plan = planned_files
# The conditioner artifact comes from a THIRD repo, neither repo_id nor base_repo, so without this
# the delete-cached guard would let it be deleted out from under the in-flight fetch (or out from
# under the assembly, while the base snapshot that no longer carries a dense encoder is still coming
# down). Same registration the native H3 path makes for its companions.
if h3_te_scheme:
self._loading.asset_repos = self._loading.asset_repos + (
h3_te_quant_source(h3_te_scheme),
)
# The same for a hosted pre-cast encoder (unsloth/Wan2.2-TI2V-5B-FP8 for every Wan base).
te_repos = tuple(
src.location
for src in (te_sources or {}).values()
if getattr(src, "kind", None) == "repo"
)
if te_repos:
self._loading.asset_repos = tuple(
dict.fromkeys(self._loading.asset_repos + te_repos)
)
# Dense shards dropped: a delete of the seeded repo mid-fetch strands the load.
if skip_transformer_weights:
claimed = self._denoiser_prequant_repo_ids(
fam,
planned_denoiser,
base,
kwargs.get("h3_task"),
)
self._loading.asset_repos = tuple(
dict.fromkeys(self._loading.asset_repos + claimed)
)
self._loading.expected_bytes = expected
# Outside the lock so an unload can preempt the multi-GB pull.
checkpoint_local: Optional[Path] = None
if kwargs.get("gguf_filename") and not Path(kwargs["repo_id"]).expanduser().exists():
from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback
partner = moe_partner_filename(kwargs["gguf_filename"])
fetch_partner = bool(
getattr(fam, "is_moe", False) and kind != "pipeline" and partner
)
# One choice for the pick and its partner expert, settled on both names, so they come from one repo and
# progress reads that repo for both.
# The estimate's plan when it made one (the repo + revision progress reads); else settled here.
checkpoint_plan = planned_files.get(_FETCH_CHECKPOINT)
checkpoint_repo = (
checkpoint_plan.repo
if checkpoint_plan is not None
else _video_fetch_repo(
kwargs["repo_id"],
kwargs.get("hf_token"),
files = (kwargs["gguf_filename"],) + ((partner,) if fetch_partner else ()),
checkpoint = True,
)
)
checkpoint_revision = (
checkpoint_plan.revision if checkpoint_plan is not None else None
)
checkpoint_local = Path(
hf_hub_download_with_xet_fallback(
checkpoint_repo,
kwargs["gguf_filename"],
kwargs.get("hf_token"),
revision = checkpoint_revision,
cancel_event = cancel_event,
# The plan counts files cached under either root, so the load must resolve through both or re-pull.
reuse_other_cache_root = True,
# An API-initiated load uses the cached checkpoint or fails; it never pulls the multi-GB file.
local_files_only = local_files_only,
gguf_header_delta = True,
)
)
if fetch_partner:
hf_hub_download_with_xet_fallback(
checkpoint_repo,
partner,
kwargs.get("hf_token"),
revision = checkpoint_revision,
cancel_event = cancel_event,
reuse_other_cache_root = True,
local_files_only = local_files_only,
)
# An LTX-2.3 checkpoint supplies the VAEs/vocoder/connectors, so the base pull shrinks to scheduler + TE +
# tokenizer; recompute the estimate.
ltx23 = False
if fam is not None and fam.name == "ltx-2" and kind != "pipeline":
from .video_ltx2 import is_ltx23_checkpoint
probe = checkpoint_local
if probe is None:
# Local repos: a bare file or dir child, found by load_pipeline's resolver; else keeps the wide pull.
root = Path(kwargs["repo_id"]).expanduser()
if root.is_file():
probe = root
elif root.is_dir():
try:
probe = self._resolve_checkpoint_path(
kwargs["repo_id"],
kwargs.get("gguf_filename"),
kwargs.get("hf_token"),
local_files_only = local_files_only,
)
except Exception: # noqa: BLE001 -- surfaced by load_pipeline
probe = None
ltx23 = probe is not None and is_ltx23_checkpoint(probe)
if ltx23:
expected = self._estimate_download_bytes(
kwargs["repo_id"],
kwargs.get("gguf_filename"),
base,
kwargs.get("hf_token"),
kind,
ltx23 = True,
te_sources = te_sources,
skip_transformer_weights = skip_transformer_weights,
skip_transformer_components = skip_transformer_components,
h3_task = kwargs.get("h3_task"),
h3_te_scheme = h3_te_scheme,
local_files_only = local_files_only,
ltx25_dev = ltx25_dev,
planned_files = (replanned := {}),
pin_revisions = _fetched_pins(planned_files),
)
planned_files = _keep_fetched_checkpoint(replanned, planned_files)
# The total follows the plan actually kept: the retained checkpoint entry, not a re-listed one.
expected = _plan_total(planned_files) if expected else expected
with self._lock:
if self._load_token == token and self._loading is not None:
self._loading.expected_bytes = expected
if hasattr(self._loading, "fetch_plan"):
self._loading.fetch_plan = planned_files
# Only a pre-cast checkpoint actually on disk earns the dense skip below.
te_failed: dict = {}
te_skipped = self._fetch_te_prequant(
te_sources,
kwargs.get("hf_token"),
cancel_event = cancel_event,
local_files_only = local_files_only,
failures = te_failed,
)
kwargs["_te_prequant_skipped"] = te_skipped
kwargs["_te_prequant_failed"] = te_failed
# H3 conditioner: its artifact must be on disk before the base pull may skip the dense encoder.
h3_te_skipped = self._fetch_h3_te_quant(
h3_te_scheme,
kwargs.get("hf_token"),
cancel_event = cancel_event,
local_files_only = local_files_only,
)
# Fetch the hosted DiT here, under this load's cancel event, and claim it against a mid-load delete.
if _ltx23_prequant_serves_on_card(
fam,
kind,
kwargs.get("gguf_filename"),
kwargs.get("transformer_quant"),
memory_mode = kwargs.get("memory_mode"),
gpu_ordinal = kwargs.get("gpu_ordinal"),
checkpoint_repo = kwargs["repo_id"],
):
from .video_ltx2 import LTX23_PREQUANT_BASE
ltx23_sources = self._denoiser_prequant_source_list(
fam, TQ_FP8, LTX23_PREQUANT_BASE, hf_token = kwargs.get("hf_token")
)
with self._lock:
if self._load_token == token and self._loading is not None:
self._loading.asset_repos = tuple(
dict.fromkeys(
self._loading.asset_repos
+ tuple(src.location for src in ltx23_sources)
)
)
self._fetch_denoiser_prequant(
ltx23_sources,
kwargs.get("hf_token"),
cancel_event = cancel_event,
local_files_only = local_files_only,
scheme = TQ_FP8,
)
# The denoiser artifact too: the injection that would fetch it has no cancel event.
if skip_transformer_weights:
self._fetch_denoiser_prequant(
self._denoiser_prequant_source_list(
fam,
planned_denoiser,
base,
kwargs.get("h3_task"),
hf_token = kwargs.get("hf_token"),
),
kwargs.get("hf_token"),
cancel_event = cancel_event,
local_files_only = local_files_only,
scheme = planned_denoiser,
)
base_plan = planned_files.get(_FETCH_BASE)
actual_te_skip = tuple(te_skipped) + tuple(h3_te_skipped)
if base_plan is not None and set(base_plan.skip_te) != set(actual_te_skip):
# An encoder fetch fell back (or found more than planned): the base file set changed, so its repo /
# revision choice is settled again on the new names, and progress follows.
expected = self._estimate_download_bytes(
kwargs["repo_id"],
kwargs.get("gguf_filename"),
base,
kwargs.get("hf_token"),
kind,
ltx23 = ltx23,
te_sources = te_sources,
skip_transformer_weights = skip_transformer_weights,
skip_transformer_components = skip_transformer_components,
h3_task = kwargs.get("h3_task"),
h3_te_scheme = h3_te_scheme,
local_files_only = local_files_only,
ltx25_dev = ltx25_dev,
planned_files = (replanned := {}),
skip_te_override = actual_te_skip,
pin_revisions = _fetched_pins(planned_files),
)
planned_files = _keep_fetched_checkpoint(replanned, planned_files)
# The total follows the plan actually kept: the retained checkpoint entry, not a re-listed newer one.
expected = _plan_total(planned_files) if expected else expected
with self._lock:
if self._load_token == token and self._loading is not None:
self._loading.expected_bytes = expected
if hasattr(self._loading, "fetch_plan"):
self._loading.fetch_plan = planned_files
base_plan = planned_files.get(_FETCH_BASE)
checkpoint_plan = planned_files.get(_FETCH_CHECKPOINT)
if checkpoint_plan is not None:
# Assembly opens the very file the prefetch pinned, never a re-resolved newer one.
kwargs["_checkpoint_fetch"] = (checkpoint_plan.repo, checkpoint_plan.revision)
base_fetch_choice: dict[str, str] = {}
base_local = self._predownload_base(
base,
kwargs.get("hf_token"),
kind,
fetch_choice = base_fetch_choice,
fetch_repo = base_plan.repo if base_plan is not None else None,
revision = base_plan.revision if base_plan is not None else None,
ltx23 = ltx23,
skip_te_components = te_skipped + h3_te_skipped,
skip_transformer_weights = skip_transformer_weights,
skip_transformer_components = skip_transformer_components,
# fl2va stages transformer/, ref2va stages transformer_ref/ (66.28 GB).
h3_task = kwargs.get("h3_task"),
cancel_event = cancel_event,
local_files_only = local_files_only,
ltx25_dev = ltx25_dev,
)
# The 2.3 assembly pulls per component from the hub id (its snapshot lacks the base VAEs).
kwargs["_base_local_dir"] = None if ltx23 else base_local
# The repo the exact-name check staged the base into: re-deciding at assembly without the names would flip a
# partially cached upstream back in (LTX-2.3 resolves every component through it).
# The predownload's choice; when it left early (no scoped pull), the plan it was handed still stands.
kwargs["_base_fetch_repo"] = base_fetch_choice.get("repo") or (
base_plan.repo if base_plan is not None else None
)
if kwargs["_base_fetch_repo"] is None and kind != "pipeline":
# No plan (a cache-only load): assembly's nameless choice counts every index component, and an upstream
# holding the transformer folders this pick replaces (or 2.3's unused VAEs) would outscore a mirror
# holding what the load reads.
kwargs["_base_fetch_repo"] = _video_offline_base_choice(
base,
tuple(p.rstrip("/") for p in VideoBackend._LTX23_BASE_PREFIXES)
if ltx23
else None,
skip = ("transformer",),
)
if (
kwargs["_base_fetch_repo"] is None
and kind == "pipeline"
and hub_reachability.hub_unreachable()
):
kwargs["_base_fetch_repo"] = _video_fetch_repo(base, kwargs.get("hf_token"))
kwargs["_base_fetch_revision"] = (
base_fetch_choice.get("revision")
if base_fetch_choice
else (base_plan.revision if base_plan is not None else None)
)
if (
kwargs["_base_fetch_revision"] is None
and base_local is None
and not local_files_only
and kind == "pipeline"
and hub_reachability.hub_unreachable()
):
# A plan-pinned fetch writes snapshots/<commit> and no refs/main, so with no plan (the Hub is down)
# neither the cached check nor from_pretrained could find the snapshot by "main". Pin the one cached
# snapshot instead; the checks below still refuse it when it is incomplete.
pinned_snapshot = _video_offline_pinned_snapshot(
base, kwargs.get("_base_fetch_repo")
)
if pinned_snapshot is not None:
kwargs["_base_fetch_repo"], kwargs["_base_fetch_revision"] = pinned_snapshot
logger.info(
"video.hub_unreachable: %s has one cached snapshot (%s) and no refs/main; pinning it",
pinned_snapshot[0],
pinned_snapshot[1][:12],
)
from .safetensors_pread import pread_safetensors
from .diffusion_families import pinned_repo_revision
checkpoint_cached_repo = (
str(checkpoint_local) if checkpoint_local is not None else kwargs["repo_id"]
)
forced_offline = not local_files_only and _video_cached_while_hub_unreachable(
base_local,
kwargs.get("_base_fetch_repo") or base,
checkpoint_cached_repo if kind != "pipeline" else None,
kwargs.get("gguf_filename") if kind != "pipeline" else None,
revision = kwargs.get("_base_fetch_revision"),
family = fam,
)
def _load(offline: bool) -> None:
if forced_offline:
kwargs["local_files_only"] = offline
with pread_safetensors():
with pinned_repo_revision(
kwargs.get("_base_fetch_repo"), kwargs.get("_base_fetch_revision")
):
self.load_pipeline(**kwargs)
if forced_offline:
logger.info(
"video.hub_unreachable: %s is cached and the Hub did not connect moments ago; loading "
"from the cache",
kwargs["repo_id"],
)
retry_online = False
try:
_load(True if forced_offline else local_files_only)
except Exception as exc:
if (
not forced_offline
or not _video_offline_cache_miss(exc)
or self._load_token != token
or (cancel_event is not None and cancel_event.is_set())
):
raise
logger.info(
"video.hub_unreachable: %s is not fully cached (%s); retrying online",
kwargs["repo_id"],
str(exc),
)
retry_online = True
if retry_online:
# Outside the except so the failed attempt's traceback frames are released before retrying online.
self._close_pending_ltx25_encoder(token)
self._rollback_precommit_globals(token)
_discard_resident_quant_write(self, token)
import gc
gc.collect()
try:
clear_gpu_cache()
except Exception: # noqa: BLE001 -- cleanup is best-effort
pass
_load(False)
with self._lock:
if self._load_token == token:
self._loading = None
except Exception as exc: # noqa: BLE001 -- surfaced via load_progress
# A failed load never commits state; roll back speed globals here, token-scoped to this load.
self._rollback_precommit_globals(token)
_discard_resident_quant_write(self, token)
# Token-scoped: a newer LTX-2.5 load may already hold the slot with its own worker.
self._close_pending_ltx25_encoder(token)
if self._load_token != token:
return
logger.error("video.load_failed: %s", exc)
if self._state is not None:
from .gpu_arbiter import VIDEO, restore_media_need, restore_owner_account
from hub.services.models.account_access import restore_resident_metadata
restore_owner_account(VIDEO)
restore_media_need(VIDEO)
restore_resident_metadata(VIDEO)
# NVFP4 caches pin the denoiser, but a failed replacement keeps the old model using them.
if self._state is None:
try:
from .diffusion_nvfp4_linear import reset_nvfp4_state
reset_nvfp4_state()
except Exception: # noqa: BLE001 -- cleanup is best-effort
pass
try:
clear_gpu_cache()
except Exception: # noqa: BLE001 -- cleanup is best-effort
pass
from utils.native_path_leases import redact_native_paths
with self._lock:
if self._load_token == token and self._loading is not None:
self._loading.error = modelscope_missing(exc) or redact_native_paths(str(exc))
self._loading.error_code = unified_memory_refusal_code(exc)
@vram_budget_scope("video", load = True)
def _run_load_h3_native(
self,
*,
fam: VideoFamily,
token: Optional[int],
cancel_event: threading.Event,
repo_id: str,
display_repo_id: Optional[str] = None,
gguf_filename: Optional[str] = None,
hf_token: Optional[str] = None,
memory_mode: Optional[str] = None,
family_override: Optional[str] = None,
gpu_ordinal: Optional[int] = None,
# NAMED so a static step-skip ask can be recorded as declined: sd.cpp runs every step itself.
transformer_cache: Optional[str] = None,
speed_mode: Optional[str] = None,
# Named, not swallowed by **_, or an offline load would fetch the bundle and sd-cli.
local_files_only: bool = False,
**_: Any,
) -> None:
"""Downloads and commits the four-file sd.cpp H3 runtime; with local_files_only, cache only, no fetch."""
from huggingface_hub import HfApi
from . import sd_cpp_args as sd_cpp_args_mod
from .sd_cpp_args import (
GRAPH_CUT_STREAM_FLAGS,
GRAPH_CUT_VRAM_FLAGS,
SdCppModelFiles,
device_backend_flags,
offload_flags,
)
from .diffusion_engine_router import _install_accelerator_for
from .sd_cpp_backend import (
_accelerator_fingerprint,
_install_allowed,
_installed_accelerator_of,
ACCELERATOR_FALLBACK_CLEAR_HINT,
accelerator_fallback_reason,
accelerator_probe_failure_is_decisive,
accelerator_verdict_keeps_gpu,
usable_or_recorded_failure,
ensure_h3_sd_cpp_binary,
fallback_accelerator_for,
note_accelerator_runtime_failure,
preferred_accelerator,
selected_card_identity,
sd_cpp_accelerator_device_verdict,
sd_cpp_device_name_for_ordinal,
sd_cpp_graph_cut_options,
sd_cpp_listed_device,
sd_cpp_supports_sage_attn,
sd_cpp_vulkan_device_is_uma,
)
from .sd_cpp_engine import SdCppEngine
from .video_minimax_h3 import (
H3_COMPONENT_REPO,
H3_GGUF_REPO,
h3_component_metadata_repo,
h3_download_error,
h3_offline_miss_error,
h3_text_encoder_filename,
)
from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback
filename = gguf_filename or ""
qwen_filename = h3_text_encoder_filename(filename)
# Claim companions before the slow preflight; a later claim cannot revoke an admitted delete.
with self._lock:
if self._load_token == token and self._loading is not None:
self._loading.base_repo = fam.base_repo
# All ids: the two VAEs resolve independently (repack vs mirror).
self._loading.asset_repos = (
H3_GGUF_REPO,
H3_COMPONENT_REPO,
)
if cancel_event.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
target = self._device_target(gpu_ordinal)
# Installing downloads sd-cli, so offline loads may only use an existing build.
allow_install = _install_allowed() and not local_files_only
# Read back per card, as it is recorded, so one card's failure does not divert another.
selected_card = selected_card_identity(gpu_ordinal)
requested_accelerator = _install_accelerator_for(target.backend)
accelerator = preferred_accelerator(requested_accelerator, selected_card)
if accelerator != requested_accelerator:
logger.warning(
"video.sd_cpp_accelerator_recorded_fallback: using the %s stable-diffusion.cpp build, "
"not %s, because this host recorded the %s build as unable to run; %s",
accelerator,
requested_accelerator,
requested_accelerator,
ACCELERATOR_FALLBACK_CLEAR_HINT,
)
binary = usable_or_recorded_failure(
ensure_h3_sd_cpp_binary(
allow_install = allow_install,
accelerator = accelerator,
),
accelerator,
selected_card,
)
native_device = target.device
# sd.cpp runs a paravirtual Mac GPU on the CPU backend: CPU policy, and no VIDEO claim held.
if sd_cpp_args_mod.forces_cpu_backend(native_device):
native_device = "cpu"
# What the accelerator decision below was made on, or None when it was never asked (a CPU or MPS target never
# consults it). Re-checked under the reader claim, so a replacement that arrives mid-load cannot silently change
# the answer this device choice rests on.
listed_accelerator: Optional[bool] = None
decided_accelerator: Any = _UNREAD_ACCELERATOR
if target.backend not in ("cpu", "mps"):
# Probe under the claim: it spawns sd-cli, and an install could overwrite the running binary.
from .sd_cpp_backend import _tree_reader as _claim_tree
with _claim_tree(binary, cancel_event, VIDEO_CANCELLED_MSG):
# Raw: None (could not be asked) is how a ROCm build without its runtime answers.
accelerator_verdict = sd_cpp_accelerator_device_verdict(binary) if binary else False
decided_accelerator = _installed_accelerator_of(binary)
# An ensure can keep another class's build; its answer is not evidence about this one.
accelerator_probe_ran = bool(binary) and decided_accelerator == accelerator
listed_accelerator = accelerator_verdict_keeps_gpu(accelerator_verdict)
if not accelerator_verdict:
fallback = fallback_accelerator_for(accelerator)
# The CPU rung is the branch below, which also releases the GPU placement.
if fallback and fallback != "cpu":
# Before the fallback ensure replaces the tree and its bundle tag.
failed_fingerprint = _accelerator_fingerprint(binary)
fallback_binary = usable_or_recorded_failure(
ensure_h3_sd_cpp_binary(allow_install = allow_install, accelerator = fallback),
fallback,
selected_card,
)
fallback_verdict: Optional[bool] = None
fallback_class: Optional[str] = None
if fallback_binary:
with _claim_tree(fallback_binary, cancel_event, VIDEO_CANCELLED_MSG):
fallback_verdict = sd_cpp_accelerator_device_verdict(fallback_binary)
fallback_class = _installed_accelerator_of(fallback_binary)
# The ensure may return the ROCm build itself; a second yes would pin an uninstalled rung.
if fallback_verdict and fallback_class == fallback:
logger.warning(
"video.sd_cpp_accelerator_fallback: the %s stable-diffusion.cpp build "
"does not run on this host (%s), using the %s build instead; %s",
accelerator,
accelerator_fallback_reason(accelerator),
fallback,
ACCELERATOR_FALLBACK_CLEAR_HINT,
)
# Proof only when our own build answered; a timeout is not proof.
note_accelerator_runtime_failure(
accelerator,
proven = (
accelerator_probe_ran
and accelerator_verdict is not None
and (
accelerator_verdict is not False
or accelerator_probe_failure_is_decisive(accelerator)
)
),
fingerprint = failed_fingerprint,
card = selected_card,
)
binary = fallback_binary
accelerator = fallback
decided_accelerator = fallback_class
listed_accelerator = True
elif fallback_binary:
# The binary now in the tree, with its own reading; `and` so it can only lower it.
binary = fallback_binary
decided_accelerator = fallback_class
listed_accelerator = listed_accelerator and accelerator_verdict_keeps_gpu(
fallback_verdict
)
accelerator = fallback
if target.backend not in ("cpu", "mps") and not listed_accelerator:
# No upstream Linux CUDA archive: fall back to CPU. Test what the binary offers, not truthiness,
# since later loads find the CPU binary already installed.
binary = usable_or_recorded_failure(
ensure_h3_sd_cpp_binary(allow_install = allow_install, accelerator = "cpu"),
"cpu",
selected_card,
)
native_device = "cpu"
decided_accelerator = _installed_accelerator_of(binary)
# Baseline is the decision, not a fresh probe: an install could swap in a GPU build meanwhile.
listed_accelerator = False
# Refuse an empty preflight before downloading the bundle; the re-vet guards mid-download swaps.
if not binary:
raise RuntimeError(
"stable-diffusion.cpp could not be installed or started for MiniMax-H3."
)
# The class too: two builds can both enumerate a GPU, so the boolean re-vet cannot tell them apart.
if decided_accelerator is _UNREAD_ACCELERATOR:
decided_accelerator = _installed_accelerator_of(binary)
if cancel_event.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
requests = self._h3_native_requests(repo_id, filename, qwen_filename)
total = 0
try:
# Skipped wholesale offline: model_info is a Hub call, and the number it produces is the size of a download
# that is not going to happen -- every file below either resolves from the cache or fails the load. total
# stays 0, which load_progress already reads as "no estimate" and reports as a bare phase.
api = (
hub_reachability.guarded(HfApi(token = hf_token or None))
if not local_files_only
else None
)
for repo, wanted in requests:
if api is None:
break
if Path(repo).expanduser().exists():
continue
info = api.model_info(h3_component_metadata_repo(repo), files_metadata = True)
total += sum(
int(s.size or 0) for s in (info.siblings or []) if s.rfilename == wanted
)
except Exception: # noqa: BLE001 -- progress estimate is optional
total = 0
with self._lock:
if self._load_token == token and self._loading is not None:
self._loading.expected_bytes = total or None
resolved: list[Path] = []
for repo, wanted in requests:
if cancel_event.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
root = Path(repo).expanduser()
if root.is_file():
local = root
elif root.is_dir():
from .diffusion_families import resolve_local_gguf_child
local = resolve_local_gguf_child(root, wanted)
else:
try:
local = Path(
hf_hub_download_with_xet_fallback(
repo,
wanted,
hf_token,
cancel_event = cancel_event,
reuse_other_cache_root = True,
local_files_only = local_files_only,
gguf_header_delta = True,
)
)
except Exception as exc: # noqa: BLE001 -- re-raised below, narrowed by name
# Cache-only miss: huggingface_hub's LocalEntryNotFoundError does not say how to recover.
if local_files_only and VideoBackend._te_fetch_miss(exc, local_files_only = True):
raise h3_offline_miss_error(repo, wanted) from exc
raise h3_download_error(repo, wanted, exc) from exc
resolved.append(local)
engine = SdCppEngine(binary)
from .sd_cpp_backend import (
_tree_reader,
sd_cpp_accelerator_device_verdict,
sd_cpp_binary_vets_for_h3,
sd_cpp_device_named,
)
from .sd_cpp_engine import is_managed_binary
with _tree_reader(binary, cancel_event, VIDEO_CANCELLED_MSG):
if not binary or engine.version() is None:
raise RuntimeError(
"stable-diffusion.cpp could not be installed or started for MiniMax-H3."
)
# Re-vet every binary (identity and capability): the window is the whole download.
# --ref-video alone is not enough: unrelated tools expose it.
if not sd_cpp_binary_vets_for_h3(binary):
raise RuntimeError(
"The stable-diffusion.cpp binary changed while this model was loading, and the "
"one now at that path is not an sd.cpp build with MiniMax-H3 support. Try the "
"load again."
)
# Re-check the accelerator decision for any owner; verdict form, since the list helper folds
# "unknown" into True. Skip when CPU/MPS never asked (avoids --list-devices per load).
current_accelerator = _installed_accelerator_of(binary)
if (
decided_accelerator is not None
and current_accelerator is not None
and current_accelerator != decided_accelerator
):
raise RuntimeError(
"The stable-diffusion.cpp binary changed while this model was loading, and the "
"one now at that path was built for a different accelerator. Try the load "
"again."
)
fresh_accelerator = (
sd_cpp_accelerator_device_verdict(binary)
if listed_accelerator is not None and binary
else None
)
if (
listed_accelerator is not None
and fresh_accelerator is not None
and fresh_accelerator != listed_accelerator
):
raise RuntimeError(
"The stable-diffusion.cpp binary changed while this model was loading, and the "
"one now at that path was built for a different accelerator. Try the load "
"again."
)
binary_identity = _sd_cli_identity(binary)
# probe under the claim; the CPU fallback has no card to select.
graph_cut_options = (
sd_cpp_graph_cut_options(binary) if native_device != "cpu" else frozenset()
)
# Lossy (INT8 QK^T), so only on an explicit speed_mode="max".
h3_sage = (
native_device != "cpu"
and str(speed_mode or "").strip().lower() == "max"
and os.environ.get("UNSLOTH_H3_SAGE_ATTN", "").strip().lower()
not in ("0", "false", "no", "off")
and sd_cpp_supports_sage_attn(binary)
)
# The CPU fallback runs on no card, so drop the ordinal.
native_ordinal = None if native_device == "cpu" else gpu_ordinal
# Under the claim like every other probe here; None on the CPU fallback
native_device_name = (
None
if native_device == "cpu"
else sd_cpp_device_name_for_ordinal(binary, native_ordinal)
)
if native_device_name is None and native_ordinal is not None:
# `Vulkan0` is not a physical index; pin by card name instead.
selected_name, selected_position = _physical_card_name(native_ordinal)
native_device_name = sd_cpp_device_named(
binary, selected_name, position = selected_position
)
# Same number the fork's dispatch reads, off the re-check's listing (no extra subprocess).
from .sd_cpp_backend import sd_cpp_cuda_compute_capability
native_cuda_cc = (
sd_cpp_cuda_compute_capability(binary, native_device_name, probe = False)
if native_device != "cpu" and listed_accelerator is not None
else None
)
native_cuda_cc = _h3_cuda_cc_fallback(
native_cuda_cc,
binary = binary,
native_device = native_device,
listed_accelerator = listed_accelerator,
accelerator_names = (decided_accelerator,),
device_name = native_device_name,
)
# What the executable itself renders on: an unowned build (SD_CLI_PATH) has no install record.
native_listed_class, native_listed_description = (
sd_cpp_listed_device(binary, native_device_name, probe = False)
if native_device != "cpu"
else (None, None)
)
requested_mode = normalize_memory_mode(memory_mode) or "auto"
policy = {
"auto": "none" if native_device == "cpu" else "group",
"fast": "none",
"balanced": "group",
"low_vram": "model",
}[requested_mode]
# NOT --vae-on-cpu: H3 audio VAE conv1d aborts on CPU (ggml F16 im2col assert, exit 134).
# Keep dropped even after the fork fix: under --offload-to-cpu it saves no VRAM and is 4.8x slower.
native_offload = tuple(
offload_flags(policy, vae_tiling = False, diffusion_fa = True, vae_on_cpu = False)
)
# H3's 20.5 GB DiT and 17 GB encoder OOM in auto and fast; --max-vram segments them, and streaming needs CPU offload.
if "--max-vram" in graph_cut_options:
native_offload += GRAPH_CUT_VRAM_FLAGS
if "--offload-to-cpu" in native_offload and "--stream-layers" in graph_cut_options:
native_offload += GRAPH_CUT_STREAM_FLAGS
native_env: tuple[tuple[str, str], ...] = ()
if h3_sage:
native_offload += ("--sage-attn",)
if native_device != "cpu":
from .video_minimax_h3 import h3_quant_cublas_env
native_env += h3_quant_cublas_env(native_cuda_cc, sage = h3_sage)
from .sd_cpp_cudnn import CudnnAttention, plan_cudnn_attention
try:
native_cudnn = (
plan_cudnn_attention(
binary,
native_cuda_cc,
allow_install = allow_install,
cancel_event = cancel_event,
)
if native_device != "cpu"
else CudnnAttention()
)
except Exception as exc: # noqa: BLE001 - an optional speedup never fails a load
logger.warning("video.h3_cudnn_plan_failed (ggml kernels): %s", exc)
native_cudnn = CudnnAttention()
native_env += native_cudnn.env
if cancel_event.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
from .video_minimax_h3 import H3_QUANT_CUBLAS_ENV, H3_QUANT_CUBLAS_MIN_CC
# The fork ignores the env below sm80, so a user value there is not the route.
h3_quant_cublas = (
native_device != "cpu"
and (native_cuda_cc is None or tuple(native_cuda_cc) >= H3_QUANT_CUBLAS_MIN_CC)
and dict(native_env)
.get(H3_QUANT_CUBLAS_ENV, os.environ.get(H3_QUANT_CUBLAS_ENV, ""))
.strip()
not in ("", "0")
)
# Before the pin, so a CPU-moved text encoder is pinned there; Auto commits nothing (a render may go resident).
native_fit_note: Optional[str] = None
native_fit_flags: Optional[tuple[str, ...]] = None
native_fit_refusal: Optional[str] = None
native_fit_inputs: Any = None
if native_device != "cpu":
from .video_minimax_h3 import (
H3_NATIVE_RENDER_RUNNING_HINT,
H3NativeFitInputs,
h3_native_compute_bytes,
h3_native_live_fit,
h3_native_render_flags,
h3_native_resident_bytes,
vulkan_device_is_amd,
)
def _size(path: Any) -> int:
try:
return os.path.getsize(path) if path else 0
except OSError:
return 0
# The build's own listing first: the install record names only builds Studio installed.
fit_accelerator = (
native_listed_class
or (decided_accelerator if isinstance(decided_accelerator, str) else None)
or accelerator
)
check_gtt = fit_accelerator == "vulkan" and vulkan_device_is_amd(
native_listed_description
)
# Everything a re-fit needs, resolved now (the PCI reading included) so later readings are file reads.
native_fit_inputs = H3NativeFitInputs(
base_flags = tuple(native_offload),
accelerator = fit_accelerator,
check_gtt = check_gtt,
# The torch card is the Vulkan device only when the pin resolved; otherwise sd.cpp picks its own,
# so no slot is named and the tightest amdgpu card is read.
gtt_pci_slot = _torch_pci_slot(native_ordinal)
if check_gtt and native_device_name
else None,
dit_bytes = _size(resolved[0]),
te_bytes = _size(resolved[1]),
vae_bytes = _size(resolved[2]) + _size(resolved[3]),
device_name = native_device_name,
uma = check_gtt
and sd_cpp_vulkan_device_is_uma(binary, native_device_name, probe = False) is True,
)
# The smallest clip's estimate: a load is refused only when even that is not expected to fit.
fam_presets = [p for p in (fam.resolution_presets or ()) if len(p) == 2]
small_w, small_h = (
min(fam_presets, key = lambda p: p[0] * p[1]) if fam_presets else (864, 480)
)
small_frames = fam.min_num_frames or fam.default_num_frames or 1
small_compute = h3_native_compute_bytes(small_w, small_h, small_frames)
fit = h3_native_live_fit(native_fit_inputs, requested_mode, small_compute)
release_pending = False
if fit.refusal or fit.note:
# Free the replaced model's idle sd-server (it respawns on demand) and re-measure.
released = self._release_native_server(
"a new MiniMax-H3 load is measuring host memory"
)
if released == "stopped":
fit = h3_native_live_fit(native_fit_inputs, requested_mode, small_compute)
release_pending = released == "pending"
if fit.refusal and release_pending:
fit = dataclasses.replace(fit, refusal = fit.refusal + H3_NATIVE_RENDER_RUNNING_HINT)
# An outgoing diffusers model's memory is freed only at commit, so the pre-launch live fit decides, not this.
with self._lock:
outgoing = self._state
outgoing_in_process = (
outgoing is not None and getattr(outgoing, "engine", None) != "sd_cpp"
)
if fit.refusal and outgoing_in_process:
logger.warning(
"video.h3_native_memory_fit: deferred to the first render, after the outgoing in-process "
"model is unloaded: %s",
fit.refusal,
)
if requested_mode == "auto" and (fit.refusal or fit.note):
# Auto decides resident vs offloaded at render time; refuse only if even the smallest clip cannot fit.
resident_need = h3_native_resident_bytes(
sum(_size(p) for p in resolved[:4]), small_w, small_h, small_frames
)
_flags, could_be_resident = h3_native_render_flags(
native_offload,
memory_mode = requested_mode,
free_bytes = _h3_card_free_bytes(native_device, native_ordinal),
need_bytes = resident_need,
)
if fit.refusal and not could_be_resident and not outgoing_in_process:
raise RuntimeError(fit.refusal)
native_fit_refusal = fit.refusal
if fit.note:
native_fit_flags = tuple(fit.flags)
native_fit_note = f"{fit.note} (applied when a render is offloaded)"
else:
if fit.refusal and not outgoing_in_process:
raise RuntimeError(fit.refusal)
# An explicit mode's flags are never altered: the fit only refuses them.
native_fit_note = fit.note
if native_fit_note:
logger.warning("video.h3_native_memory_fit: %s", native_fit_note)
# After the policy, so the pin can see which modules it left on the CPU; without it sd.cpp uses ordinal 0
# whatever was selected.
native_offload += tuple(device_backend_flags(native_device_name, list(native_offload)))
if native_fit_flags is not None:
native_fit_flags += tuple(
device_backend_flags(native_device_name, list(native_fit_flags))
)
from .video_minimax_h3 import (
H3NativeServerSlot,
MiniMaxH3NativeRuntime,
h3_sibling_server_binary,
)
native_files = SdCppModelFiles(
diffusion_model = str(resolved[0]),
llm = str(resolved[1]),
vae = str(resolved[2]),
audio_vae = str(resolved[3]),
)
# Nothing spawns here: the slot starts the server on the first render; no sibling sd-server keeps one-shot.
server_binary = h3_sibling_server_binary(getattr(engine, "binary", None))
runtime = MiniMaxH3NativeRuntime(
engine = engine,
# Pinned at load: taking it at generation would compare a replacement against itself.
binary_identity = binary_identity,
selected_card = selected_card,
files = native_files,
offload_flags = native_offload,
offloaded_flags = native_fit_flags,
offload_refusal = native_fit_refusal,
fit_inputs = native_fit_inputs,
vram_free_probe = (
functools.partial(
_h3_card_free_bytes,
native_device,
native_ordinal if native_device_name else None,
single_card_only = not native_device_name,
allow_context = False,
)
if native_device != "cpu"
else None
),
vram_reserve_probe = (
functools.partial(
_h3_card_reserve_bytes,
native_device,
native_ordinal if native_device_name else None,
single_card_only = not native_device_name,
)
if native_device != "cpu"
else None
),
env = native_env,
cudnn = native_cudnn,
server_slot = (
H3NativeServerSlot(
server_binary,
native_files,
native_offload,
pressure_probe = functools.partial(
_h3_native_server_pressure, native_device, native_ordinal
),
)
if server_binary
else None
),
)
with self._lock:
if token is not None and token != self._load_token:
raise RuntimeError("Video load was cancelled or superseded.")
if self._active_generate_cancel is not None:
self._active_generate_cancel.set()
self._teardown_waiters += 1
with self._generate_lock:
with self._lock:
try:
if token is not None and token != self._load_token:
raise RuntimeError("Video load was cancelled or superseded.")
self._teardown_state_locked()
self._state = _VideoLoadState(
pipe = runtime,
family = fam,
repo_id = repo_id,
display_repo_id = display_repo_id,
base_repo = fam.base_repo,
device = native_device,
gpu_ordinal = native_ordinal,
# sd.cpp is not torch, so no thread-local device is restored; the --backend pin is what it honours.
placed_ordinal = native_ordinal,
dtype = Path(filename).stem.split("-")[-1],
kind = "gguf",
engine = "sd_cpp",
h3_task = h3_transformer_task(filename),
gguf_filename = filename,
offload_policy = policy,
vae_tiling = False,
memory_mode = requested_mode,
attention_backend = "sage" if h3_sage else "flash",
speed_mode = SPEED_MAX if h3_sage else SPEED_OFF,
resolved = build_resolved_record(
{
"family_override": _family_override_resolved(family_override, fam),
"memory_mode": (
memory_mode,
policy,
f"native model offload; {native_fit_note}"
if native_fit_note
else "native model offload",
),
"speed_mode": (
speed_mode,
SPEED_MAX if h3_sage else SPEED_OFF,
{
(True, True): "sd.cpp SageAttention + BF16 cuBLAS",
(True, False): "sd.cpp SageAttention",
(False, True): "sd.cpp BF16 cuBLAS matmuls",
(False, False): "sd.cpp exact kernels",
}[(bool(h3_sage), bool(h3_quant_cublas))],
),
"attention_backend": (
None,
"sage" if h3_sage else "flash",
"sd.cpp SageAttention (speed_mode=max)"
if h3_sage
else "sd.cpp diffusion flash attention",
),
**(
{
"transformer_cache": (
transformer_cache,
"off",
"static step skip is not supported by the native sd.cpp runtime",
RESOLVED_UNSUPPORTED,
)
}
if _is_static_cache_request(transformer_cache)
else {}
),
}
),
)
commit_load_budget_fraction("video")
finally:
self._teardown_waiters -= 1
if native_device == "cpu":
# Committed to the CPU build, so drop the stale VIDEO claim; release_if keeps the token check atomic.
from .gpu_arbiter import VIDEO, release_if
release_if(VIDEO, lambda: token is None or token == self._load_token)
def _close_pending_ltx25_encoder(self, token: Any = _ANY_LOAD) -> None:
"""Stop an LTX-2.5 encoder worker whose load never committed (failed, cancelled or superseded).
With ``token``, only the worker that load started: a newer load's worker is left alone."""
with self._lock:
pending = self._ltx25_pending_encoder
if pending is None or (token is not _ANY_LOAD and pending[0] != token):
return
self._ltx25_pending_encoder = None
_stop_ltx25_encoder(pending[1])
def _rollback_precommit_globals(self, token: Optional[int]) -> None:
"""Restores speed globals after a pre-commit failure, unless a newer load now owns the snapshot slot."""
stored = getattr(self, "_precommit_globals", None)
if stored is None:
return
stored_token, flags = stored
if token is not None and stored_token is not None and stored_token != token:
return
self._precommit_globals = None
restore_backend_flags(flags)
from . import diffusion_gguf_compile
from .diffusion_autoopt.patches import GROUP_VIDEO_FUSED, uninstall_group
diffusion_gguf_compile.uninstall_all()
uninstall_group(GROUP_VIDEO_FUSED)
# LTX-2.3 gets DiT/connectors/VAEs/vocoder from the checkpoint + extras, so only the 2.0 base scheduler / text
# encoder / tokenizer are pulled.
_LTX23_BASE_PREFIXES = ("scheduler/", "text_encoder/", "tokenizer/")
@staticmethod
def _te_prequant_sources(
fam: Any,
text_encoder_quant: Optional[str],
gpu_ordinal: Optional[int] = None,
base: Optional[str] = None,
) -> dict[str, Any]:
"""``{component: source}`` for the text encoders this load will take PRE-CAST from a
hosted checkpoint instead of the base repo's dense weights (``{}`` when none)."""
if getattr(fam, "name", None) == "ltx-2.5":
# The sidecar worker loads LTX-2.5's hosted int8 ConvRot Gemma 4 encoder; resolving it here is what lets
# the plan, the estimate and the pre-fetch stage that file and drop the dense text_encoder/ shards.
from .video_ltx25 import ltx25_te_prequant_sources
return ltx25_te_prequant_sources(
text_encoder_quant, base or str(getattr(fam, "base_repo", "") or "")
)
from .diffusion_te_prequant import te_prequant_sources_for_base
return te_prequant_sources_for_base(
fam,
base or str(getattr(fam, "base_repo", "") or ""),
te_quant_mode = text_encoder_quant,
# The selected card, not the default, decides fp8 encoder pre-cast hosting, and vice versa.
target = (
resolve_diffusion_device_target()
if gpu_ordinal is None
else resolve_diffusion_device_target(ordinal = gpu_ordinal)
),
)
@staticmethod
def _h3_te_quant_scheme(
fam: Any,
text_encoder_quant: Optional[str],
base: Optional[str] = None,
target: Any = None,
) -> Optional[str]:
"""Hosted quantized conditioner for a modular family, only against its own base; else None."""
try:
if not getattr(fam, "modular_workflow", None):
return None
from .diffusion_te_prequant import te_base_equivalent
canonical = getattr(fam, "base_repo", None)
if not canonical or not base or not te_base_equivalent(canonical, base):
return None
if _h3_precision_pinned_dense(text_encoder_quant):
return None
if _h3_precision_unset(text_encoder_quant):
auto = h3_te_auto_scheme()
return (
auto
if h3_te_quant_scheme(auto) is not None and _h3_auto_precision_ok(target)
else None
)
return h3_te_quant_scheme(normalize_te_quant(text_encoder_quant))
except Exception: # noqa: BLE001 -- an unanswerable probe keeps the dense encoder
return None
@staticmethod
def _h3_te_base_index_source(
base: Optional[str],
hf_token: Optional[str],
local_files_only: bool = False,
) -> Optional[str]:
"""Reads base's modular index before the pull, so the dense encoder skip matches the seed's
choice."""
if not base:
return None
try:
import json
root = Path(base).expanduser()
if root.is_dir():
path = root / "modular_model_index.json"
else:
from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback
path = Path(
hf_hub_download_with_xet_fallback(
# The repo the base pull reads: a fresh install fetches the index from the mirror, never the
# vendor id, and an upstream cache holding only an older index does not win it.
_video_index_repo(base, local_files_only = local_files_only),
"modular_model_index.json",
hf_token,
cache_dir = hub_cache_dir(),
reuse_other_cache_root = True,
local_files_only = local_files_only,
)
)
with open(path, "r", encoding = "utf-8") as handle:
entry = json.load(handle).get("text_encoder")
# ["library", "ClassName", {spec}]; "repo" is the spec field's deprecated spelling.
spec = entry[2] if isinstance(entry, (list, tuple)) and len(entry) == 3 else None
source = (spec or {}).get("pretrained_model_name_or_path") or (spec or {}).get("repo")
return source if isinstance(source, str) and source else None
except Exception: # noqa: BLE001 -- unanswerable keeps the dense shards
return None
def _denoiser_prequant_verified(
self,
fam: Any,
transformer_quant: Optional[str],
base: Optional[str],
h3_task: Optional[str],
hf_token: Optional[str],
local_files_only: bool = False,
kind: str = "pipeline",
) -> bool:
"""``_denoiser_prequant_covered`` plus the Hub check, for the decisions that COMMIT.
The pure probe answers from the registry, which is right where it must not raise or touch
the network, but a renamed / gated task artifact still reads covered there. Unanswerable
keeps the dense shards; ``local_files_only`` answers from the cache."""
if not self._denoiser_prequant_covered(
fam, transformer_quant, base, h3_task, kind = kind, hf_token = hf_token
):
return False
from .diffusion_prequant import restricted_prequant_load_supported
if not restricted_prequant_load_supported(transformer_quant):
# This decision commits (drops 66 GB) and also runs for explicit requests.
logger.info(
"video.denoiser_prequant: this install cannot deserialize a pre-quantized "
"checkpoint, so the %s request keeps its dense denoiser shards",
transformer_quant,
)
return False
if local_files_only:
repo = self._denoiser_prequant_cached_repo(
fam, transformer_quant, base, h3_task, online = False, hf_token = hf_token
)
else:
try:
from huggingface_hub import HfApi
repo, _files = self._denoiser_prequant_hub_files(
fam, transformer_quant, base, HfApi(token = hf_token), h3_task
)
except Exception: # noqa: BLE001 -- an unanswerable probe keeps the dense shards
return False
if repo is None:
logger.info(
"video.denoiser_prequant: no hosted %s checkpoint resolved for %s %s; keeping its "
"dense denoiser shards",
transformer_quant,
getattr(fam, "name", None),
h3_task or getattr(fam, "modular_workflow", None),
)
return repo is not None
@staticmethod
def _install_flashinfer_for_seed(
wanted: bool,
seed_scheme: Optional[str],
device: Any,
*,
local_files_only: bool = False,
) -> Optional[tuple[bool, str]]:
"""FlashInfer install for a conventional load, only when the settled plan kept the NVFP4 seed."""
if not wanted or seed_scheme != "nvfp4":
return None
from .diffusion_nvfp4_install import ensure_flashinfer_for_nvfp4
return ensure_flashinfer_for_nvfp4(device, logger = logger, local_files_only = local_files_only)
def _nvfp4_denoiser_checkpoint_will_load(
self,
fam: Any,
base: Optional[str],
h3_task: Optional[str],
hf_token: Optional[str],
*,
local_files_only: bool = False,
kind: str = "pipeline",
planned: Optional[str] = None,
) -> bool:
"""Whether an NVFP4 video load opens hosted pre-quantised denoisers (the only FlashInfer path); unanswerable -> False."""
if not nvfp4_diffusion_enabled():
return False
try:
modular = bool(getattr(fam, "modular_workflow", None))
if planned == DENOISER_SEED_DECLINED and not modular:
return False
task = (h3_task or getattr(fam, "modular_workflow", None)) if modular else h3_task
if not self._denoiser_prequant_covered(
fam, "nvfp4", base, task, kind = kind, hf_token = hf_token
):
return False
from .diffusion_prequant import restricted_prequant_load_supported
if not restricted_prequant_load_supported("nvfp4"):
return False
if (
self._denoiser_prequant_cached_repo(
fam,
"nvfp4",
base,
task,
online = False if local_files_only else None,
hf_token = hf_token,
)
is not None
):
return True
if local_files_only:
return False
from huggingface_hub import HfApi
repo, _files = self._denoiser_prequant_hub_files(
fam, "nvfp4", base, HfApi(token = hf_token or None), task
)
return repo is not None
except Exception as exc: # noqa: BLE001 -- a refused or unreachable listing is no checkpoint
logger.info(
"video.nvfp4_install: no reachable NVFP4 checkpoint for %s, so the load runs on "
"torchao and FlashInfer is not installed (%s)",
base,
type(exc).__name__,
)
return False
def _h3_planned_auto_denoiser_scheme(
self,
fam: Any,
*,
base: Optional[str],
transformer_quant: Optional[str],
text_encoder_quant: Optional[str],
speed_mode: Optional[str],
h3_task: Optional[str],
gpu_ordinal: Optional[int] = None,
) -> Optional[str]:
"""Auto denoiser fallback decided before any download, against card capacity rather than
free memory."""
if not _h3_precision_unset(transformer_quant):
return None
try:
if not getattr(fam, "modular_workflow", None):
return None
import torch
# Scoped, not pinned (pooled to_thread worker); covers the whole sizing call, which reads the device.
with diffusion_device_scope(gpu_ordinal):
target = (
resolve_diffusion_device_target()
if gpu_ordinal is None
else resolve_diffusion_device_target(ordinal = gpu_ordinal)
)
dtype = target.dtype
if dtype is torch.float16 and fp16_promotes_to_fp32(fam):
dtype = torch.float32
return _h3_auto_denoiser_scheme(
fam,
target = target,
dtype = dtype,
device = target.device,
te_scheme = self._h3_te_quant_scheme(fam, text_encoder_quant, base, target),
task = h3_task or getattr(fam, "modular_workflow", None),
base_repo = base,
speed_mode = speed_mode,
free_reader = _h3_device_capacity_bytes,
)
except Exception: # noqa: BLE001 -- an unanswerable probe keeps the dense shards
return None
def _video_planned_auto_denoiser_scheme(
self,
fam: Any,
*,
base: Optional[str],
kind: str,
transformer_quant: Optional[str],
speed_mode: Optional[str],
memory_mode: Optional[str] = None,
text_encoder_quant: Optional[str] = None,
gpu_ordinal: Optional[int] = None,
hf_token: Optional[str] = None,
ltx25_dev: bool = False,
native_hosted: Optional[list] = None,
) -> Optional[str]:
"""The pre-download auto denoiser scheme, None, or ``DENOISER_SEED_DECLINED`` if the plan would offload.
On a decline, ``native_hosted`` (a list) receives the scheme when the offloaded native build can be read from
the hosted checkpoint instead (``_video_native_hosted_scheme``)."""
try:
if kind != "pipeline" or getattr(fam, "modular_workflow", None):
return None
import torch
# SCOPED, not pinned: this pooled to_thread worker must not keep this request's card.
with diffusion_device_scope(gpu_ordinal):
target = (
resolve_diffusion_device_target()
if gpu_ordinal is None
else resolve_diffusion_device_target(ordinal = gpu_ordinal)
)
if target.dtype is torch.float16 and fp16_promotes_to_fp32(fam):
# The loader promotes fp16 to float32 here, so read the dtype the load will.
return None
scheme = _video_auto_denoiser_scheme(
fam,
target = target,
requested = transformer_quant,
base_repo = base,
speed_mode = speed_mode,
hf_token = hf_token,
)
if scheme is None:
return None
resident = self._state
reclaimable = _pipeline_device_mib(
getattr(resident, "pipe", None),
ordinal = _target_ordinal(target),
)
settled: list[Any] = []
def _settled_once() -> Any:
# Both checks plan on one settled read: each settle can sleep up to 2 s.
if not settled:
settled.append(settled_snapshot_device_memory(target))
return settled[0]
if _auto_seed_yields_to_resident_bf16(
scheme, transformer_quant, fam
) and _video_seed_stays_resident(
fam,
target = target,
scheme = None,
memory_mode = memory_mode,
text_encoder_quant = text_encoder_quant,
base_repo = base,
reclaimable_mib = reclaimable,
hf_token = hf_token,
ltx25_dev = ltx25_dev,
device_memory_reader = _settled_once,
):
# auto keeps a resident bf16 DiT, so the dense shards are what this load runs.
return None
seed_plans: list = []
if not _video_seed_stays_resident(
fam,
target = target,
scheme = scheme,
memory_mode = memory_mode,
text_encoder_quant = text_encoder_quant,
base_repo = base,
reclaimable_mib = reclaimable,
hf_token = hf_token,
ltx25_dev = ltx25_dev,
device_memory_reader = _settled_once,
plan_out = seed_plans,
):
seed_plan = seed_plans[0] if seed_plans else None
streamed = _video_torchao_seed_placement(fam, scheme, target, seed_plan)
if streamed is not None:
logger.info(
"video.denoiser_prequant: an artifact-sized plan for %s offloads on this "
"card ('%s'), and its torchao weights stream there, so the hosted "
"checkpoint is seeded and streamed ('%s')",
scheme,
_video_plan_label(seed_plan),
_video_plan_label(streamed),
)
return scheme
hosted = _video_native_hosted_scheme(fam, scheme, target)
if native_hosted is not None and hosted is not None:
native_hosted.append(hosted)
logger.info(
"video.denoiser_prequant: an artifact-sized plan for %s still offloads on "
"this card, so the hosted %s checkpoint is read as offload-safe native "
"Linears instead of quantising the dense denoiser shards",
scheme,
hosted,
)
return DENOISER_SEED_DECLINED
logger.info(
"video.denoiser_prequant: an artifact-sized plan for %s still offloads on "
"this card, and offload moves the DiT, so the dense denoiser shards are "
"kept",
scheme,
)
return DENOISER_SEED_DECLINED
return scheme
except Exception: # noqa: BLE001 -- an unanswerable probe keeps the dense shards
return None
def _h3_te_quant_scheme_verified(
self,
fam: Any,
text_encoder_quant: Optional[str],
base: Optional[str],
hf_token: Optional[str],
local_files_only: bool = False,
) -> Optional[str]:
"""_h3_te_quant_scheme plus an exact index read for COMMIT decisions, so staging and seed agree."""
scheme = self._h3_te_quant_scheme(fam, text_encoder_quant, base)
if scheme is None:
return None
source = self._h3_te_base_index_source(base, hf_token, local_files_only = local_files_only)
if source is None or _h3_te_canonical(source) != _h3_te_canonical(
getattr(fam, "base_repo", None)
):
logger.info(
"video.h3_te_quant: %s names %s as its conditioner, not %s; keeping its dense "
"text_encoder shards",
base,
source or "an unreadable index",
getattr(fam, "base_repo", None),
)
return None
return scheme
@staticmethod
def _h3_te_index_source(pipe: Any) -> Optional[str]:
"""Repo the pipeline's own modular index names for text_encoder; None means refuse, never
substitute."""
try:
spec = pipe.get_component_spec("text_encoder")
except Exception: # noqa: BLE001 -- no spec, no substitution
return None
source = getattr(spec, "pretrained_model_name_or_path", None) or getattr(spec, "repo", None)
return source if isinstance(source, str) and source else None
@staticmethod
def _h3_te_quant_hub_files(
scheme: Optional[str], api: Any
) -> tuple[Optional[str], list[tuple[str, int]]]:
"""``(repo_id, [(rfilename, size)])`` for the hosted quantized conditioner, or ``(None, [])``.
The conditioner mirror of ``_denoiser_prequant_hub_files``, and it earns its keep the same
way: the base entry drops the dense ``text_encoder/`` shards whenever this resolves, so
without it the artifact that replaces them is in no entry at all and the disk preflight
passes on a volume that cannot hold the 27 GB it is about to pull."""
api = hub_reachability.always_asked(api) # picks the conditioner: bounded, never skipped
from .video_minimax_h3_te import h3_te_quant_filename, h3_te_quant_source
filename = h3_te_quant_filename(scheme)
if filename is None:
return None, []
repo = h3_te_quant_source(scheme)
try:
info = api.model_info(repo, files_metadata = True)
except Exception: # noqa: BLE001 -- an unavailable artifact keeps the dense encoder
return None, []
files = [
(s.rfilename, int(getattr(s, "size", 0) or 0))
for s in (info.siblings or [])
if s.rfilename == filename
]
return (repo, files) if files else (None, [])
def _fetch_h3_te_quant(
self,
scheme: Optional[str],
hf_token: Optional[str],
*,
cancel_event: Optional[threading.Event] = None,
local_files_only: bool = False,
) -> tuple[str, ...]:
"""Pre-fetch the hosted conditioner; only a file already on disk may drop the dense shards."""
from .video_minimax_h3_te import h3_te_quant_filename, h3_te_quant_source
filename = h3_te_quant_filename(scheme)
if filename is None:
return ()
cancel = cancel_event if cancel_event is not None else self._cancel_event
from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback
repo = h3_te_quant_source(scheme)
try:
hf_hub_download_with_xet_fallback(
repo,
filename,
hf_token,
cancel_event = cancel,
cache_dir = hub_cache_dir(),
reuse_other_cache_root = True,
local_files_only = local_files_only,
)
except Exception as exc: # noqa: BLE001 -- no artifact just means the dense encoder
if cancel.is_set():
raise
logger.warning("video.h3_te_quant_fetch_failed: %s/%s: %s", repo, filename, exc)
return ()
return ("text_encoder",)
def _fetch_denoiser_prequant(
self,
sources: list[Any],
hf_token: Optional[str],
*,
cancel_event: Optional[threading.Event] = None,
local_files_only: bool = False,
scheme: Optional[str] = None,
) -> None:
"""Pre-fetch the hosted denoiser checkpoint(s) under the load's cancel event; best effort except cancellation."""
cancel = cancel_event if cancel_event is not None else self._cancel_event
from core.inference.diffusion_prequant import (
candidate_filenames_of,
prefer_cached_pickle_twins,
restricted_prequant_load_supported,
)
from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback
for source in sources:
if getattr(source, "kind", None) != "repo":
continue
names = list(dict.fromkeys(candidate_filenames_of(source)))
# the resolver's filter: readable names, unless that leaves none
names = [n for n in names if restricted_prequant_load_supported(scheme, n)] or names
names = prefer_cached_pickle_twins(
source.location,
names,
readable = lambda n: restricted_prequant_load_supported(scheme, n),
cache_dir = hub_cache_dir(),
)
for index, name in enumerate(names):
try:
hf_hub_download_with_xet_fallback(
source.location,
name,
hf_token,
cancel_event = cancel,
cache_dir = hub_cache_dir(),
reuse_other_cache_root = True,
local_files_only = local_files_only,
)
break
except Exception as exc: # noqa: BLE001 -- the injection re-resolves and falls back
if cancel.is_set():
raise
if index == len(names) - 1:
logger.warning(
"video.denoiser_prequant_fetch_failed: %s/%s: %s",
source.location,
name,
exc,
)
@staticmethod
def _denoiser_prequant_covered(
fam: Any,
transformer_quant: Optional[str],
base: Optional[str],
h3_task: Optional[str] = None,
*,
kind: str = "pipeline",
hf_token: Optional[str] = None,
) -> bool:
"""True when hosted pre-quantized checkpoints replace ALL the family's dense DiT shards; never raises.
Registry plus the hosted header probe (cache-only offline); ``hf_token`` lets it read a gated header."""
try:
scheme = normalize_transformer_quant(transformer_quant)
# An unresolved "auto" must NOT drop dense shards; the backend settles it first.
if scheme is None or scheme == TQ_AUTO:
return False
if getattr(fam, "modular_workflow", None):
if not video_family_prequant_available(fam, scheme, task = h3_task, base_repo = base):
return False
# Covered only by a checkpoint the loader will accept: a stale header keeps the dense shards.
from .diffusion_prequant import resolve_prequant_source
from .video_denoiser_prequant import hosted_denoiser_refused
source = resolve_prequant_source(fam, scheme, base_repo = base, task = h3_task)
return source is None or not hosted_denoiser_refused(
source, scheme, base, hf_token = hf_token
)
if kind != "pipeline":
return False
from .video_denoiser_prequant import denoiser_prequant_sources
return denoiser_prequant_sources(fam, scheme, base, hf_token = hf_token) is not None
except Exception: # noqa: BLE001 -- an unanswerable probe keeps the dense shards
return False
@staticmethod
def _denoiser_prequant_source_list(
fam: Any,
transformer_quant: Optional[str],
base: Optional[str],
h3_task: Optional[str] = None,
*,
hf_token: Optional[str] = None,
) -> list[Any]:
"""Every ``PrequantSource`` a seeded load would open, or ``[]``; never raises. A hosted source the
loader would refuse from its header is dropped; ``hf_token`` lets that probe read a gated repo."""
scheme = (transformer_quant or "").strip().lower()
if scheme in ("", "auto", "off", "none"):
return []
try:
from .diffusion_prequant import resolve_prequant_source
from .video_denoiser_prequant import denoiser_prequant_sources, hosted_denoiser_refused
if getattr(fam, "modular_workflow", None):
source = resolve_prequant_source(fam, scheme, base_repo = base, task = h3_task)
# The modular loader checks no component; a header it would refuse is never staged.
if source is None or hosted_denoiser_refused(
source, scheme, base, hf_token = hf_token
):
return []
return [source]
resolved = denoiser_prequant_sources(fam, scheme, base, hf_token = hf_token)
return list(resolved.values()) if resolved else []
except Exception as exc: # noqa: BLE001 -- a bad registry entry must not sink the plan
logger.warning("video.denoiser_prequant_unresolved: %s", exc)
return []
@staticmethod
def _denoiser_prequant_repo_ids(
fam: Any,
transformer_quant: Optional[str],
base: Optional[str],
h3_task: Optional[str] = None,
) -> tuple[str, ...]:
"""Hosted repo id(s) a seeded denoiser is fetched from, for the in-flight delete guard."""
return tuple(
dict.fromkeys(
src.location
for src in VideoBackend._denoiser_prequant_source_list(
fam, transformer_quant, base, h3_task
)
if getattr(src, "kind", None) == "repo" and getattr(src, "location", None)
)
)
@staticmethod
def _denoiser_prequant_hub_files(
fam: Any,
transformer_quant: Optional[str],
base: Optional[str],
api: Any,
h3_task: Optional[str] = None,
) -> tuple[Optional[str], list[tuple[str, int]]]:
"""``(repo_id, [(rfilename, size)])`` for every hosted denoiser artifact, or ``(None, [])``, so preflight
counts the checkpoint that replaces the dropped dense shards. The header probe asks with ``api``'s own
token, so a gated artifact is judged by the same credentials that would stage it."""
api = hub_reachability.always_asked(
api
) # picks the denoiser source: bounded, never skipped
token = getattr(api, "token", None)
sources = VideoBackend._denoiser_prequant_source_list(
fam,
transformer_quant,
base,
h3_task,
hf_token = token if isinstance(token, str) and token else None,
)
if not sources or any(getattr(src, "kind", None) != "repo" for src in sources):
return None, []
locations = {src.location for src in sources}
if len(locations) != 1:
logger.warning("video.denoiser_prequant_split_repos: %s", sorted(locations))
return None, []
location = sources[0].location
from core.inference.diffusion_nvfp4_flag import nvfp4_repo_blocked
if nvfp4_repo_blocked(location):
return None, []
try:
info = api.model_info(location, files_metadata = True)
except Exception as exc: # noqa: BLE001 -- unavailable prequant means the dense DiT
logger.warning("video.denoiser_prequant_unavailable: %s: %s", location, exc)
return None, []
from core.inference.diffusion_prequant import (
candidate_filenames_of,
prefer_cached_pickle_twins,
restricted_prequant_load_supported,
)
by_name = {s.rfilename: int(s.size or 0) for s in (info.siblings or [])}
files: list[tuple[str, int]] = []
for src in sources:
names = list(candidate_filenames_of(src))
# the resolver's filter: readable names, unless that leaves none
names = [
n for n in names if restricted_prequant_load_supported(transformer_quant, n)
] or names
wanted = prefer_cached_pickle_twins(
src.location,
names,
readable = lambda n: restricted_prequant_load_supported(transformer_quant, n),
cache_dir = hub_cache_dir(),
)
found = next((n for n in wanted if n in by_name), None)
if found is None:
return None, []
files.append((found, by_name[found]))
return location, files
@staticmethod
def _denoiser_prequant_cached_repo(
fam: Any,
transformer_quant: Optional[str],
base: Optional[str],
h3_task: Optional[str] = None,
online: Optional[bool] = None,
hf_token: Optional[str] = None,
) -> Optional[str]:
"""The hosted pre-quantized denoiser repo when its checkpoint is ALREADY cached, else None.
Offline twin of ``_denoiser_prequant_hub_files``; online semantics per ``first_cached_as_resolved``."""
sources = VideoBackend._denoiser_prequant_source_list(
fam, transformer_quant, base, h3_task, hf_token = hf_token
)
# A local override is on disk by definition; only a hosted checkpoint has a cache to probe
if not sources or any(getattr(src, "kind", None) != "repo" for src in sources):
return None
from core.inference.diffusion import DiffusionBackend
from core.inference.diffusion_prequant import (
candidate_filenames_of,
first_cached_as_resolved,
prefer_cached_pickle_twins,
restricted_prequant_load_supported,
)
cached: list[str] = []
for src in sources:
# only names the loader can open: a cached unreadable file must not drop the dense shards
ordered = prefer_cached_pickle_twins(
src.location,
[
n
for n in candidate_filenames_of(src)
if restricted_prequant_load_supported(transformer_quant, n)
],
readable = lambda n: restricted_prequant_load_supported(transformer_quant, n),
cache_dir = hub_cache_dir(),
log = False,
)
hit = first_cached_as_resolved(
src.location,
ordered,
is_cached = lambda n, repo = src.location: DiffusionBackend._hub_file_is_cached(
repo, n
),
online = online,
cache_dir = hub_cache_dir(),
)
if hit is not None:
cached.append(src.location)
if len(cached) == len(sources):
return cached[0]
# Not logged here: the caller reports a miss as "keeping its dense denoiser shards"; no double log.
return None
@staticmethod
def _te_prequant_hub_files(
sources: dict[str, Any], api: Any
) -> dict[str, list[tuple[str, int]]]:
"""Pre-cast checkpoints that resolve on the Hub; delegated so plan and resolver name one
artifact."""
from .diffusion_te_prequant import te_prequant_hub_files
return te_prequant_hub_files(sources, api, logger)
@staticmethod
def _te_fetch_miss(exc: BaseException, *, local_files_only: bool) -> bool:
"""Whether exc means the name is absent; online, a LocalEntryNotFoundError can mean Hub
unreachable."""
try:
from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError
except Exception: # noqa: BLE001 - an unknown hub layout keeps today's behaviour
return True
if isinstance(exc, LocalEntryNotFoundError):
return bool(local_files_only)
if isinstance(exc, EntryNotFoundError):
return True
return VideoBackend._te_fetch_miss_by_name(exc, local_files_only = local_files_only)
# Whole leading token match so LocalEntryNotFoundError is never read as the remote miss.
_REMOTE_MISS_NAMES = frozenset({"EntryNotFoundError", "RemoteEntryNotFoundError"})
_LOCAL_MISS_NAMES = frozenset({"LocalEntryNotFoundError"})
@staticmethod
def _te_fetch_miss_by_name(exc: BaseException, *, local_files_only: bool) -> bool:
"""Child-process 404 arrives as RuntimeError; match only a message that begins with the class name."""
if not isinstance(exc, RuntimeError):
return False
message = str(exc)
name = message.split(":", 1)[0].strip() if ":" in message else ""
if name in VideoBackend._LOCAL_MISS_NAMES:
return bool(local_files_only)
return name in VideoBackend._REMOTE_MISS_NAMES
@staticmethod
def _base_download_files(
info: Any,
kind: str,
*,
ltx23: bool = False,
skip_te_components: tuple[str, ...] = (),
skip_transformer_weights: bool = False,
skip_transformer_components: tuple[str, ...] = (),
h3_task: Optional[str] = None,
ltx25_shards: Optional[dict[str, frozenset[str]]] = None,
ltx25_dev: bool = False,
) -> list[tuple[str, int]]:
"""Base-repo files a load reads; skips root singles, duplicate TE shards, and replaced dense weights."""
from .diffusion_te_prequant import is_prequant_covered_weight
from .video_minimax_h3 import H3_TASK_REFERENCES
from .video_ltx25 import (
is_ltx25_pack,
is_ltx25_single_file_listing,
ltx25_pack_files,
ltx25_single_file_companions,
)
siblings = list(info.siblings or [])
ltx25_pack = is_ltx25_pack(s.rfilename for s in siblings)
# LTX-2.5 and Cosmos3 also ship a modular_model_index.json; neither is H3's layout.
from .video_cosmos3 import NONE_COMPONENTS as COSMOS3_NONE_COMPONENTS, is_cosmos3_listing
cosmos3 = is_cosmos3_listing(s.rfilename for s in siblings)
# Only None-passed components: diffusers' hub-id load demands every other listed weight.
cosmos3_unused = tuple(f"{c}/" for c in COSMOS3_NONE_COMPONENTS) if cosmos3 else ()
is_h3_modular = (
not ltx25_pack
and not cosmos3
and any(s.rfilename == "modular_model_index.json" for s in siblings)
)
h3_is_references = is_h3_modular and (h3_task or "").strip().lower() == H3_TASK_REFERENCES
h3_denoiser_prefix = "transformer_ref/" if h3_is_references else "transformer/"
# Must be the partition this load opens: a reference load stages only transformer_ref/.
h3_skip_component = h3_denoiser_prefix.rstrip("/")
skip_components = (
(tuple(skip_transformer_components) or (h3_skip_component,))
if skip_transformer_weights
else ()
)
h3_prefixes = (
"audio_scheduler/",
"audio_vae/",
"processor/",
"scheduler/",
"text_encoder/",
"tokenizer/",
h3_denoiser_prefix,
"vae/",
)
files: list[tuple[str, int]] = []
for sibling in siblings:
name, size = sibling.rfilename, sibling.size or 0
if is_h3_modular and not (
name in ("model_index.json", "modular_model_index.json")
or name.startswith(h3_prefixes)
):
continue
# chat_template.jinja is needed at generation time.
if not name.endswith((".safetensors", ".json", ".model", ".txt", ".jinja")):
continue
if "/" not in name and name.endswith(".safetensors"):
continue
if (
kind != "pipeline"
# An H3 single file (a ComfyUI denoiser) replaces the partition it serves, transformer_ref/ included.
and name.startswith(("transformer/", "transformer_2/", h3_denoiser_prefix))
# transformer/config.json is the exception: from_single_file(config = <repo id>, subfolder =
# "transformer") reads it off the Hub, so a load that promised to download nothing needs it staged.
# Without it the locality gate cleared the pick and the fetch happened after eviction.
and not name.endswith("/config.json")
):
continue
# is_prequant_covered_weight is component-agnostic, so the encoder helper serves the denoiser too.
if skip_components and is_prequant_covered_weight(name, skip_components):
continue
if name.startswith("text_encoder/diffusion_pytorch_model"):
continue
if cosmos3_unused and name.startswith(cosmos3_unused):
continue
if ltx23 and "/" in name and not name.startswith(VideoBackend._LTX23_BASE_PREFIXES):
continue
if skip_te_components and is_prequant_covered_weight(name, skip_te_components):
continue
files.append((name, int(size)))
if is_ltx25_single_file_listing(s.rfilename for s in siblings):
# The split single-file release as a base: only the companion files beside the picked DiT.
return ltx25_single_file_companions(files, dev = ltx25_dev)
if ltx25_pack:
# Unused components and the duplicate shard sets the LTX-2.5 pack ships (see ltx25_pack_files).
files = ltx25_pack_files(files, ltx25_shards, dev = ltx25_dev)
return files
@staticmethod
def _ltx25_plan_shards(
info: Any, base: str, hf_token: Optional[str]
) -> Optional[dict[str, frozenset[str]]]:
"""For an LTX-2.5 pack, the shard set each component's ``*.index.json`` names, so a plan stages the copy the load
opens (the pack ships two transformer sets); None for any other repo, or when an index cannot be read."""
from .video_ltx25 import _PACK_SKIP_PREFIXES, is_ltx25_pack, ltx25_shards_from_indexes
names = [s.rfilename for s in (info.siblings or [])]
if not is_ltx25_pack(names):
return None
import json
from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback
indexes: dict[str, Any] = {}
try:
for name in names:
if (
"/" in name
and name.endswith(".safetensors.index.json")
and not name.startswith(_PACK_SKIP_PREFIXES)
):
# The listing's commit: the shard choice must describe the revision the plan pins.
sha = getattr(info, "sha", None)
local = hf_hub_download_with_xet_fallback(
base,
name,
hf_token,
**({"revision": sha} if sha else {}),
reuse_other_cache_root = True,
)
indexes[name] = json.loads(Path(local).read_text(encoding = "utf-8"))
except Exception as exc: # noqa: BLE001 -- the plan falls back to ltx25_pack_files' listing-only choice
logger.debug("video.ltx25_plan_shards: %s", exc)
return None
return ltx25_shards_from_indexes(indexes)
def _estimate_download_bytes(
self,
repo_id: str,
gguf_filename: Optional[str],
base: str,
hf_token: Optional[str],
kind: str,
ltx23: bool = False,
te_sources: Optional[dict[str, Any]] = None,
skip_transformer_weights: bool = False,
skip_transformer_components: tuple[str, ...] = (),
h3_task: Optional[str] = None,
h3_te_scheme: Optional[str] = None,
local_files_only: bool = False,
ltx25_dev: bool = False,
planned_files: Optional[dict[str, "_PlannedFetch"]] = None,
skip_te_override: Optional[tuple[str, ...]] = None,
pin_revisions: Optional[dict[str, str]] = None,
) -> Optional[int]:
"""Bytes this load pulls, minus replaced dense shards; None offline, where no download happens."""
if local_files_only:
return None
try:
from huggingface_hub import HfApi
total = 0
api = hub_reachability.guarded(HfApi(token = hf_token or None))
if skip_te_override is not None:
# The encoder fetches already ran: plan around what they actually staged.
skip_te_components = tuple(skip_te_override)
else:
skip_te_components = tuple(self._te_prequant_hub_files(te_sources or {}, api))
if self._h3_te_quant_hub_files(h3_te_scheme, api)[1]:
skip_te_components = skip_te_components + ("text_encoder",)
planned: dict[str, _PlannedFetch] = {}
listings = _OneCommitListings(api, pin_revisions)
if gguf_filename and not Path(repo_id).expanduser().exists():
listing = _video_listing_repo(repo_id)
info = listings.info(listing)
wanted = _checkpoint_files(gguf_filename, kind)
names = set()
for sibling in info.siblings or []:
if sibling.rfilename in wanted and sibling.size:
total += int(sibling.size)
names.add(sibling.rfilename)
planned[_FETCH_CHECKPOINT] = _plan_fetch(
repo_id, listing, info, names, hf_token, checkpoint = True
)
if base and not Path(base).expanduser().exists():
listing = _video_listing_repo(base)
info = listings.info(listing)
base_files = self._base_download_files(
info,
kind,
ltx23 = ltx23,
skip_te_components = skip_te_components,
skip_transformer_weights = skip_transformer_weights,
skip_transformer_components = skip_transformer_components,
h3_task = h3_task,
ltx25_dev = ltx25_dev,
ltx25_shards = self._ltx25_plan_shards(info, _video_listing_repo(base), hf_token),
)
total += sum(size for _, size in base_files)
planned[_FETCH_BASE] = _plan_fetch(
base, listing, info, [name for name, _ in base_files], hf_token
)._replace(skip_te = tuple(skip_te_components))
if planned_files is not None:
planned_files.update(planned)
# The plan's own total (one count per blob, as progress reads it); the raw sum only without a plan.
return _plan_total(planned) if planned else (total or None)
except Exception: # noqa: BLE001 -- progress totals are best-effort only
return None
@vram_budget_scope("video", load = True)
def download_plan(
self,
repo_id: str,
*,
gguf_filename: Optional[str] = None,
base_repo: Optional[str] = None,
family_override: Optional[str] = None,
display_repo_id: Optional[str] = None,
model_kind: Optional[str] = None,
hf_token: Optional[str] = None,
transformer_quant: Optional[str] = None,
text_encoder_quant: Optional[str] = None,
h3_task: Optional[str] = None,
allow_device_probe: bool = True,
**load_kwargs: Any,
) -> dict[str, Any]:
"""Files this pick needs; must read text_encoder_quant, transformer_quant, h3_task, not swallow them."""
from huggingface_hub import HfApi
fam = _detect_load_family(
repo_id, gguf_filename, family_override, display_repo_id, model_kind
)
# _detect_load_family resolves from the REPO id first, so a mixed repo answers its media family for every file
# in it, a csm quant included. Refuse before the plan stages a byte.
_assert_pick_is_not_speech(repo_id, gguf_filename, hf_token)
kind = resolve_video_model_kind(gguf_filename, model_kind)
from .video_minimax_h3 import is_h3_native
if is_h3_native(fam, kind):
return self._h3_native_download_plan(repo_id, gguf_filename or "", hf_token = hf_token)
base = repo_id if kind == "pipeline" else resolve_video_base_repo(fam, base_repo, repo_id)
h3_task = _h3_single_file_task(fam, kind, gguf_filename, h3_task)
# A dev LTX-2.5 load is single stage and never opens the latent upsampler.
ltx25_dev = _ltx25_dev_load(fam, kind, gguf_filename, repo_id, display_repo_id)
# The family default the loader applies to an unset encoder request, so the plan stages the hosted encoder
# instead of the dense shards the load would not open. H3 keeps reading the raw request below.
te_planned, _ = video_te_quant_request(
fam, text_encoder_quant, base = base, speed_mode = load_kwargs.get("speed_mode")
)
# The same resolution the load makes, so an H3 auto pick the loader will serve from a hosted checkpoint is not
# staged dense here first: that is 66.3 GB the panel downloads and the load never opens, and the replacement
# would then arrive outside this plan.
transformer_quant = (
self._h3_planned_auto_denoiser_scheme(
fam,
base = base,
transformer_quant = transformer_quant,
text_encoder_quant = text_encoder_quant,
speed_mode = load_kwargs.get("speed_mode"),
h3_task = h3_task,
# Sizes the file set from device capacity, so it has to read the selected card.
gpu_ordinal = load_kwargs.get("gpu_ordinal"),
)
if kind != "single_file"
else None
) or transformer_quant
native_hosted_out: list = []
video_planned = self._video_planned_auto_denoiser_scheme(
fam,
base = base,
kind = kind,
transformer_quant = transformer_quant,
speed_mode = load_kwargs.get("speed_mode"),
memory_mode = load_kwargs.get("memory_mode"),
text_encoder_quant = te_planned,
gpu_ordinal = load_kwargs.get("gpu_ordinal"),
hf_token = hf_token,
ltx25_dev = ltx25_dev,
native_hosted = native_hosted_out,
)
if video_planned == DENOISER_SEED_DECLINED:
# An offloaded int8 load reads the hosted file as native Linears: plan that file, not the dense shards.
video_planned = native_hosted_out[0] if native_hosted_out else None
if kind == "pipeline" and not getattr(fam, "modular_workflow", None):
transformer_quant = video_planned
else:
transformer_quant = video_planned or transformer_quant
# Only the header tells 2.3 from 2.0 and it is not on disk yet, so guess by name.
ltx23 = self._pick_looks_like_ltx23(fam, repo_id, gguf_filename, kind)
# Keyed by repo: two entries would collide on the job key.
entries: dict[str, dict[str, Any]] = {}
total = 0
checkpoint_bytes = 0
planned: dict[str, set[str]] = {}
# Per-repo: where each file resolves (live root, other root, nowhere), so staging is decided once.
located: dict[str, dict[str, tuple[Optional[str], int]]] = {}
labels: dict[str, dict[str, Any]] = {}
def add(
repo: str,
files: list[tuple[str, int]],
gguf: Optional[str] = None,
revision: Optional[str] = None,
checkpoint: bool = False,
) -> int:
"""Counts required files and where the loader finds each; which to stage is decided per
repo later."""
from core.inference.diffusion import DiffusionBackend
def _where(name: str, size: int) -> Optional[str]:
live = hub_cache_dir()
if DiffusionBackend._hub_file_is_cached(repo, name, revision, size, roots = (live,)):
return "live"
if not DiffusionBackend._hub_file_is_cached(
repo, name, revision, size, roots = (None,)
):
return None
# reuse_other_cache_root switches roots only when the live lookup finds nothing; test presence only.
if DiffusionBackend._hub_file_is_cached(repo, name, revision, None, roots = (live,)):
return None
return "other"
seen = planned.setdefault(repo, set())
where = located.setdefault(repo, {})
label = labels.setdefault(repo, {"gguf": None, "checkpoint": False})
if gguf:
label["gguf"] = gguf
# Only ever raised: a repo is keyed once, so a later companion add must not unflag it.
label["checkpoint"] = label["checkpoint"] or checkpoint
required = 0
for name, size in files:
if name in seen:
continue
seen.add(name)
required += int(size)
where[name] = (_where(name, int(size)), int(size))
return required
try:
api = hub_reachability.guarded(HfApi(token = hf_token or None))
listings = _OneCommitListings(api)
if gguf_filename and not Path(repo_id).expanduser().exists():
listing = _video_listing_repo(repo_id)
info = listings.info(listing)
wanted_checkpoints = (
_checkpoint_files(gguf_filename, kind) if fam.is_moe else (gguf_filename,)
)
sizes = [
(s.rfilename, int(s.size or 0))
for s in (info.siblings or [])
if s.rfilename in wanted_checkpoints
]
checkpoint_bytes = sum(size for _name, size in sizes)
# Staged under the repo the load will FETCH from (the download manager pulls the entry's repo); the
# mirror's commit only describes the mirror, so an upstream kept for its complete cache gets no pin.
fetch_repo = (
_video_fetch_repo(
repo_id, hf_token, files = [n for n, _ in sizes], checkpoint = True
)
or repo_id
)
total += add(
fetch_repo,
sizes,
gguf = gguf_filename,
revision = getattr(info, "sha", None) if fetch_repo == listing else None,
checkpoint = True,
)
if ltx23:
# 2.3 assembly reads these companions at load; unlisted, they'd be pulled outside disk preflight.
from .video_ltx2 import ltx23_extras_files, LTX23_EXTRAS_REPO
# A cached checkpoint's weights name its variant, as at load; else its name decides.
try:
variant_source = _cached_checkpoint_file(repo_id, gguf_filename)
except Exception: # noqa: BLE001 -- the name decides
variant_source = None
wanted = set(ltx23_extras_files(variant_source or gguf_filename))
extras_info = (
info
if LTX23_EXTRAS_REPO == repo_id
else api.model_info(LTX23_EXTRAS_REPO, files_metadata = True)
)
total += add(
LTX23_EXTRAS_REPO,
[
(s.rfilename, int(s.size or 0))
for s in (extras_info.siblings or [])
if s.rfilename in wanted
],
revision = getattr(extras_info, "sha", None),
)
# Pre-cast encoders first: only a checkpoint that really resolves earns the right to drop the dense shards
te_sources = self._te_prequant_sources(
fam,
te_planned,
load_kwargs.get("gpu_ordinal"),
base = base,
)
te_files = self._te_prequant_hub_files(te_sources, api)
from .diffusion_te_prequant import te_prequant_unmirrored
for component, files in te_files.items():
location = te_sources[component].location
files = te_prequant_unmirrored(location, files)
if files:
total += add(location, files)
# Denoiser replacement artifact: the base entry drops dense DiT shards only if this resolves.
dq_repo, dq_files = self._denoiser_prequant_hub_files(
fam, transformer_quant, base, api, h3_task
)
if dq_repo:
total += add(dq_repo, dq_files)
elif _ltx23_prequant_serves_on_card(
fam,
kind,
gguf_filename,
transformer_quant,
memory_mode = load_kwargs.get("memory_mode"),
gpu_ordinal = load_kwargs.get("gpu_ordinal"),
checkpoint_repo = repo_id,
probe = allow_device_probe,
hash_source = False,
):
from .video_ltx2 import LTX23_PREQUANT_BASE
lq_repo, lq_files = self._denoiser_prequant_hub_files(
fam, TQ_FP8, LTX23_PREQUANT_BASE, api
)
if lq_repo:
total += add(lq_repo, lq_files)
# Verified: adds 27 GB and removes the dense encoder, so a wrong match is wrong both ways.
h3_te_repo, h3_te_files = self._h3_te_quant_hub_files(
self._h3_te_quant_scheme_verified(fam, text_encoder_quant, base, hf_token), api
)
if h3_te_repo:
total += add(h3_te_repo, h3_te_files)
if base and not Path(base).expanduser().exists():
listing = _video_listing_repo(base)
info = listings.info(listing)
base_files = self._base_download_files(
info,
kind,
ltx23 = ltx23,
skip_te_components = (
tuple(te_files) + (("text_encoder",) if h3_te_repo else ())
),
# Gated on the artifact RESOLVING, exactly like the two encoder skips above, not on the registry
# alone: the registry says a checkpoint should exist, dq_repo says one does. When the
# task-specific file is absent or its repo cannot be read, dropping the dense shards leaves a
# plan with NEITHER denoiser, and the bf16 fallback the loader documents has nothing to fall
# back to offline. The registry probe still runs, because it is the one that refuses a
# non-modular family and an auto request.
skip_transformer_weights = (
dq_repo is not None
and self._denoiser_prequant_covered(
fam, transformer_quant, base, h3_task, kind = kind, hf_token = hf_token
)
),
skip_transformer_components = _planned_denoiser_components(fam, kind),
h3_task = h3_task,
ltx25_dev = ltx25_dev,
# The pack's indexes pick the transformer / connectors copy, as _predownload_base does.
ltx25_shards = self._ltx25_plan_shards(info, listing, hf_token),
)
# The same fetch-repo decision _predownload_base makes from the same names, so the staged entry is the
# repo the load then reads (its mirror, unless the upstream cache already holds every file).
fetch_base = (
_video_fetch_repo(base, hf_token, files = [n for n, _ in base_files]) or base
)
base_revision = getattr(info, "sha", None) if fetch_base == listing else None
if fetch_base.lower() != (listing or "").lower():
# An upstream kept for its complete cache: probe THAT snapshot (its commit, its sizes), as
# _plan_fetch does for the load, not the mirror head, whose sizes may be a newer revision.
cached = _cached_snapshot_plan(fetch_base, [n for n, _ in base_files])
if cached is not None:
base_revision = cached[0]
base_files = [(name, size) for name, _key, size in cached[1]]
total += add(
fetch_base,
base_files,
revision = base_revision,
# A pipeline pick has no single file: the base IS the selected model (base = repo_id above). Under a
# GGUF pick the base is a companion, so it stays unflagged.
checkpoint = kind == "pipeline",
)
except Exception as exc: # noqa: BLE001 -- an unavailable plan falls back to the inline pull
logger.warning("video.download_plan_failed: %s", exc)
# Flagged: zero means unknown, not nothing to fetch (media auto-switch must not download).
return {
"entries": [],
"total_bytes": 0,
"required_bytes": 0,
"checkpoint_bytes": 0,
"plan_failed": True,
}
for repo, where in located.items():
# Files straddling both roots can't be one snapshot (re-pulled inline); missing counts as live.
split = {w or "live" for w, _size in where.values()} == {"live", "other"}
staged = [
(name, size)
for name, (w, size) in where.items()
if w is None or (split and w != "live")
]
if not staged:
continue
missing_names = [name for name, _size in staged]
# Keep the scoped claim stable so a second pick adopts the running job instead of 409ing.
names = list(where)
gguf = labels[repo]["gguf"]
entries[repo] = {
"repo_id": repo,
"files": names,
"bytes": sum(size for _name, size in staged),
"gguf_filename": gguf,
# Describes the entry, not the pick: a 2.3 repo with a cached checkpoint stages companions only.
"checkpoint": labels[repo]["checkpoint"]
and (gguf is None or gguf in missing_names),
}
return {
"entries": list(entries.values()),
# Cache state changes what is left to fetch, never what the load needs on disk.
"total_bytes": sum(entry["bytes"] for entry in entries.values()),
"required_bytes": total,
"checkpoint_bytes": int(checkpoint_bytes),
}
@staticmethod
def _h3_bundled_repo(repo_id: str, filename: str, hub_repo: str) -> str:
"""A local bundle that holds filename, else the Hub repo; the bundle carries its own Qwen
and VAEs."""
root = Path(repo_id).expanduser()
if root.is_dir():
from .diffusion_families import resolve_local_gguf_child
try:
resolve_local_gguf_child(root, filename)
except Exception: # noqa: BLE001 -- not in the local clone, so the Hub copy it is
return hub_repo
return repo_id
return hub_repo
@staticmethod
def _h3_text_encoder_repo(repo_id: str, qwen_filename: str) -> str:
"""Which repo the H3 native text encoder comes from, preferring a local bundle."""
from .video_minimax_h3 import H3_GGUF_REPO
return VideoBackend._h3_bundled_repo(repo_id, qwen_filename, H3_GGUF_REPO)
@staticmethod
def _h3_native_requests(
repo_id: str, filename: str, qwen_filename: str
) -> tuple[tuple[str, str], ...]:
"""(repo, filename) for the four native H3 components, decided per file so plan and load agree."""
from .video_minimax_h3 import H3_AUDIO_VAE, H3_VIDEO_VAE, h3_component_source
return (
(repo_id, filename),
(VideoBackend._h3_text_encoder_repo(repo_id, qwen_filename), qwen_filename),
(
VideoBackend._h3_bundled_repo(
repo_id, H3_VIDEO_VAE, h3_component_source(H3_VIDEO_VAE)
),
H3_VIDEO_VAE,
),
(
VideoBackend._h3_bundled_repo(
repo_id, H3_AUDIO_VAE, h3_component_source(H3_AUDIO_VAE)
),
H3_AUDIO_VAE,
),
)
@staticmethod
def _h3_native_download_plan(
repo_id: str, gguf_filename: str, *, hf_token: Optional[str]
) -> dict[str, Any]:
from huggingface_hub import HfApi
from .video_minimax_h3 import (
h3_component_metadata_repo,
h3_text_encoder_filename,
validate_h3_transformer_filename,
)
from core.inference.diffusion import DiffusionBackend
validate_h3_transformer_filename(gguf_filename)
qwen_filename = h3_text_encoder_filename(gguf_filename)
# Same builder as the load, so the plan cannot promise one repo's files and fetch from another.
wanted = VideoBackend._h3_native_requests(repo_id, gguf_filename, qwen_filename)
grouped: dict[str, dict[str, Any]] = {}
missing_files: dict[str, set[str]] = {}
total = 0
required_total = 0
checkpoint_bytes = 0
try:
api = hub_reachability.guarded(HfApi(token = hf_token or None))
for repo, filename in wanted:
if Path(repo).expanduser().exists():
continue
# Sizes from our mirror: the repack may be gone, and one failing model_info fails the whole plan.
meta_repo = h3_component_metadata_repo(repo)
info = api.model_info(meta_repo, files_metadata = True)
match = next((s for s in (info.siblings or []) if s.rfilename == filename), None)
if match is None:
raise ValueError(
f"Required MiniMax-H3 component is missing: {meta_repo}/{filename}"
)
size = int(match.size or 0)
required_total += size
if repo == repo_id and filename == gguf_filename:
checkpoint_bytes = size
entry = grouped.setdefault(
repo,
{
"repo_id": repo,
"files": [],
"bytes": 0,
"gguf_filename": None,
"checkpoint": False,
},
)
if filename not in entry["files"]:
entry["files"].append(filename)
# The mirror's sha says nothing about the repack's cached snapshot.
revision = getattr(info, "sha", None) if meta_repo == repo else None
if not DiffusionBackend._hub_file_is_loadable(repo, filename, revision, size):
missing_files.setdefault(repo, set()).add(filename)
entry["bytes"] += size
total += size
if filename == gguf_filename:
entry["gguf_filename"] = filename
except Exception as exc: # noqa: BLE001 -- inline loading remains the fallback
logger.warning("video.h3_native_download_plan_failed: %s", exc)
# Flagged: zero means unknown, not nothing to fetch.
return {
"entries": [],
"total_bytes": 0,
"required_bytes": 0,
"checkpoint_bytes": 0,
"plan_failed": True,
}
entries = []
for repo, entry in grouped.items():
missing = missing_files.get(repo, set())
if not missing:
continue
entry["checkpoint"] = entry["gguf_filename"] in missing
entries.append(entry)
return {
"entries": entries,
"total_bytes": total,
"required_bytes": required_total,
"checkpoint_bytes": checkpoint_bytes,
}
@staticmethod
def _pick_looks_like_ltx23(
fam: Any, repo_id: str, gguf_filename: Optional[str], kind: str
) -> bool:
"""Name-only check (header unreadable at plan time); under-guessing only defers to load-time
pull."""
if kind == "pipeline" or fam is None or getattr(fam, "name", None) != "ltx-2":
return False
from .video_ltx2 import LTX23_EXTRAS_REPO
if repo_id.strip().lower() == LTX23_EXTRAS_REPO.lower():
return True
text = f"{repo_id} {gguf_filename or ''}".lower()
return "2.3" in text or "2_3" in text or "23b" in text
def _fetch_te_prequant(
self,
sources: dict[str, Any],
hf_token: Optional[str],
*,
cancel_event: Optional[threading.Event] = None,
local_files_only: bool = False,
failures: Optional[dict] = None,
) -> tuple[str, ...]:
"""Fetch pre-cast encoders first, so the base pull can skip only dense weights that are on disk. ``failures``
collects ``{component: reason}`` for an encoder none of whose files could be fetched."""
cancel = cancel_event if cancel_event is not None else self._cancel_event
from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback
fetched: list[str] = []
for component, source in sources.items():
last_error: Optional[BaseException] = None
# Local path overrides are validated at injection; they never justify skipping the dense download.
if getattr(source, "kind", None) != "repo" or not getattr(source, "filename", None):
continue
# Try every candidate: the preferred safetensors name is absent from most repos.
from .diffusion_te_prequant import te_candidate_filenames, te_candidate_is_readable
from .diffusion_prequant import prefer_cached_pickle_twins
names = prefer_cached_pickle_twins(
source.location,
[n for n in te_candidate_filenames(source) if te_candidate_is_readable(n)],
readable = te_candidate_is_readable,
)
got = False
from .diffusion_te_prequant import te_prequant_mirror_path
for name in names:
# The loader reads a mirrored file in place (_resolve_checkpoint_path), as the plan assumes.
if te_prequant_mirror_path(source.location, name) is not None:
got = True
break
try:
hf_hub_download_with_xet_fallback(
source.location,
name,
hf_token,
cancel_event = cancel,
reuse_other_cache_root = True,
local_files_only = local_files_only,
)
except Exception as exc: # noqa: BLE001 -- no pre-cast file means the dense encoder
if cancel.is_set():
raise
last_error = exc
logger.warning(
"video.te_prequant_fetch_failed: %s/%s: %s",
source.location,
name,
exc,
)
# Advance only on a missing name; repo-level errors would report a legacy artifact as fetched.
if not VideoBackend._te_fetch_miss(exc, local_files_only = local_files_only):
break
continue
got = True
break
if not got:
if failures is not None and last_error is not None:
failures[component] = (
f"{source.location}: {type(last_error).__name__}: "
f"{str(last_error).splitlines()[0][:120] if str(last_error) else ''}"
)
continue
fetched.append(component)
return tuple(fetched)
def _top_up_dense_base(
self,
base: str,
hf_token: Optional[str],
kind: str,
*,
base_local_dir: Optional[str],
fetch_repo: Optional[str],
revision: Optional[str],
local_files_only: bool,
ltx25_dev: bool = False,
) -> Optional[str]:
"""Restore the dense weights a failed injection left out: the same scoped pull with nothing skipped, from the
plan's repo at the plan's commit, so the topped-up snapshot is the one the rest of the base was staged into.
An offline load never had a scoped snapshot to top up (the prefetch returns None), so this is a no-op there --
but the flag is carried so the top-up can never become the one call that reaches the Hub."""
return (
self._predownload_base(
base,
hf_token,
kind,
ltx23 = False,
local_files_only = local_files_only,
ltx25_dev = ltx25_dev,
fetch_repo = fetch_repo,
revision = revision,
)
or base_local_dir
)
def _predownload_base(
self,
base: str,
hf_token: Optional[str],
kind: str,
*,
ltx23: bool = False,
skip_te_components: tuple[str, ...] = (),
skip_transformer_weights: bool = False,
skip_transformer_components: tuple[str, ...] = (),
h3_task: Optional[str] = None,
cancel_event: Optional[threading.Event] = None,
local_files_only: bool = False,
ltx25_dev: bool = False,
fetch_choice: Optional[dict[str, str]] = None,
fetch_repo: Optional[str] = None,
revision: Optional[str] = None,
) -> Optional[str]:
"""Pull exactly the base-repo files the load needs; return the local snapshot dir.
``fetch_repo`` / ``revision``: the load's settled plan (``_PlannedFetch``) for the base. Given, every file comes
from that repo at that commit, so the bytes land exactly where load_progress reads them; absent (a top-up, a
direct call), the repo is settled here on the exact names, unpinned, as before.
``fetch_choice`` (when given) receives ``{"repo": <id>}``, the side of the mirror pair the exact-name check
settled on, so the assembly that follows reads the same repo instead of re-deciding without the names.
A bare ``from_pretrained(repo_id)`` snapshot of Lightricks/LTX-2 downloads the
whole 314 GB repo (root packaged checkpoints plus a second 50 GB text-encoder
shard set) when ~93 GB is used. Downloading the scoped file list ourselves is
also cancellable per file, and handing the local dir to from_pretrained skips
diffusers' own expected-files sweep. None -> caller keeps the hub id (local
path, non-diffusers layout, or any metadata failure: from_pretrained then
resolves the repo exactly as before).
``local_files_only`` skips the whole prefetch and returns None. The scoped file list comes
from ``model_info``, which is a Hub call, and there is nothing to scope: the load below
passes ``local_files_only=True`` to ``from_pretrained``, which resolves the cached snapshot
itself and fetches nothing. Enumerating the cache here instead would only reconstruct what
diffusers is about to do offline anyway, and getting it wrong would hand it an incomplete
directory. None is the value this already returns whenever the metadata is unavailable, so
the caller path is the tested one."""
cancel = cancel_event if cancel_event is not None else self._cancel_event
if local_files_only:
return None
try:
if not base or Path(base).expanduser().exists():
return None
from huggingface_hub import HfApi
# Listed from the mirror (byte identical, so the same names and sizes), then the fetch repo is settled
# against the EXACT names: an upstream snapshot that already holds every one of them keeps being used.
listing = _video_listing_repo(base)
info = hub_reachability.model_info(
HfApi(token = hf_token or None),
listing,
files_metadata = True,
# The plan's commit describes the listing only when the plan reads the listing repo; an upstream kept for
# its cache is pinned to its own snapshot's commit, which the mirror does not have.
**(
{"revision": revision}
if revision and (fetch_repo or "").lower() == listing.lower()
else {}
),
)
files = self._base_download_files(
info,
kind,
ltx23 = ltx23,
skip_te_components = skip_te_components,
skip_transformer_weights = skip_transformer_weights,
skip_transformer_components = skip_transformer_components,
h3_task = h3_task,
ltx25_dev = ltx25_dev,
)
from .video_ltx25 import (
is_ltx25_pack,
is_ltx25_single_file_listing,
ltx25_shards_from_indexes,
)
single_file_base = is_ltx25_single_file_listing(
sib.rfilename for sib in (info.siblings or [])
)
if not single_file_base and not any(name == "model_index.json" for name, _ in files):
return None
from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback
if is_ltx25_pack(name for name, _ in files) or is_ltx25_pack(
sib.rfilename for sib in (info.siblings or [])
):
import json
# Read the shard indexes first so only the indexed copy of each duplicated weight set is fetched.
indexes: dict[str, Any] = {}
for name, _ in files:
if name.endswith(".safetensors.index.json"):
if cancel.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
local_index = hf_hub_download_with_xet_fallback(
fetch_repo or _video_fetch_repo(base, hf_token, files = (name,)) or base,
name,
hf_token,
revision = revision,
cancel_event = cancel,
reuse_other_cache_root = True,
)
indexes[name] = json.loads(Path(local_index).read_text(encoding = "utf-8"))
files = self._base_download_files(
info,
kind,
ltx23 = ltx23,
skip_te_components = skip_te_components,
skip_transformer_weights = skip_transformer_weights,
skip_transformer_components = skip_transformer_components,
h3_task = h3_task,
ltx25_dev = ltx25_dev,
ltx25_shards = ltx25_shards_from_indexes(indexes),
)
snapshot_root: Optional[str] = None
roots: set[str] = set()
base = (
fetch_repo
or _video_fetch_repo(base, hf_token, files = [name for name, _ in files])
or base
)
if fetch_choice is not None:
fetch_choice["repo"] = base
if revision:
fetch_choice["revision"] = revision
for name, _ in files:
# A cached file returns without consulting the event.
if cancel.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
local = Path(
hf_hub_download_with_xet_fallback(
base,
name,
hf_token,
revision = revision,
cancel_event = cancel,
reuse_other_cache_root = True,
)
)
# Root = resolved path minus relative parts; not resolve(), which follows the link into blobs/.
try:
root = str(local.parents[len(Path(name).parts) - 1])
except (IndexError, ValueError, OSError):
root = ""
roots.add(root)
if name == "model_index.json" or single_file_base:
snapshot_root = root or None
# Snapshot only if it holds the whole set; per-file reuse_other_cache_root can leave it partial.
if snapshot_root is not None and roots != {snapshot_root}:
return None
return snapshot_root
except Exception as exc: # noqa: BLE001 -- fall back to from_pretrained's own pull
if cancel.is_set():
raise
logger.warning("video.predownload_fallback: %s", exc)
return None
def _cache_bytes(self, repo_id: Optional[str]) -> int:
"""Bytes of ``repo_id`` currently in the HF blob cache (progress polling without a plan).
Walks the repo's cache directory directly instead of ``scan_cache_dir``:
the scanner skips in-flight ``*.incomplete`` blobs, so during a multi-GB
shard pull the counter would freeze at the last completed blob for minutes
while the disk keeps filling (the bar sat stuck mid-download)."""
if not repo_id:
return 0
try:
import os
folder = Path(hub_cache_dir()) / ("models--" + repo_id.strip().replace("/", "--"))
if not folder.is_dir():
return 0
total = 0
for root, _dirs, names in os.walk(folder):
for name in names:
try:
path = os.path.join(root, name)
# Snapshot entries are symlinks into blobs/; skip them so a blob is not counted twice.
if not os.path.islink(path):
total += os.path.getsize(path)
except OSError:
continue
return int(total)
except Exception: # noqa: BLE001 -- cache scan is best-effort
return 0
@staticmethod
def _planned_bytes(plan: dict[str, "_PlannedFetch"]) -> int:
"""Bytes of the load's planned files present in the cache, read by blob key from the plan's repo.
A blob is named by its content key, so this counts exactly the planned file whichever revision it lands under
(a superseded snapshot's same-named file has another key), never a carrier's other checkpoints or an unrelated
partial, and the in-flight ``<key>.incomplete`` while it downloads. Both cache roots are read, as every fetch
passes ``reuse_other_cache_root``. A file with no key falls back to its entry in the pinned revision (else
``refs/main``). Each file is capped at its listed size and counted once."""
import os
roots = [Path(hub_cache_dir())]
try:
from huggingface_hub import constants
if Path(constants.HF_HUB_CACHE) != roots[0]:
roots.append(Path(constants.HF_HUB_CACHE))
except Exception: # noqa: BLE001 -- the live root alone
pass
total = 0
seen: set[tuple[str, str]] = set()
for entry in plan.values():
repo = (entry.repo or "").strip()
if not repo:
continue
for name, etag, size in entry.files:
key = (repo.lower(), etag or name)
if key in seen:
continue
seen.add(key)
best = 0
for root in roots:
folder = root / ("models--" + repo.replace("/", "--"))
try:
if etag:
candidates = [
folder / "blobs" / etag,
folder / "blobs" / f"{etag}.incomplete",
]
else:
ref = folder / "refs" / "main"
rev = entry.revision or (
ref.read_text(encoding = "utf-8").strip() if ref.is_file() else None
)
candidates = [folder / "snapshots" / rev / name] if rev else []
for path in candidates:
if path.is_file():
best = max(best, os.path.getsize(path))
except OSError:
continue
total += min(best, size) if size else best
return int(total)
def load_progress(self) -> dict[str, Any]:
"""Phase + downloaded/total bytes for the in-flight load (cache-scan based)."""
loading = self._loading
if loading is not None and loading.error:
return _progress("error", error = loading.error, error_code = loading.error_code)
if loading is None:
return _progress("ready" if self._state is not None else None)
# The load's own plan, settled once at estimate time by the worker that owns this state: the repo + revision
# every fetch site reads, and each file's blob key and size. Nothing else is consulted.
plan = dict(getattr(loading, "fetch_plan", None) or {})
if plan:
downloaded = self._planned_bytes(plan)
else:
# No plan (offline, a failed estimate, or a state built outside _run_load): the whole cache of each id's
# default fetch side.
reading: dict[str, str] = {}
for rid, is_checkpoint in ((loading.repo_id, True), (loading.base_repo, False)):
if rid:
reading.setdefault(
_video_fetch_key(rid),
_video_fetch_repo(rid, checkpoint = is_checkpoint) or rid,
)
downloaded = sum(self._cache_bytes(rid) for rid in dict.fromkeys(reading.values()))
expected = loading.expected_bytes
phase = "downloading"
if not expected:
# No size estimate: downloaded counts cache contents, not this load; report phase alone.
return _progress(phase)
if downloaded >= expected:
phase = "finalizing"
# The cache scan counts every blob, so clamp to the scoped estimate.
downloaded = expected
return _progress(
phase,
downloaded_bytes = int(downloaded),
expected_bytes = int(expected) if expected else None,
)
def loading_repo_ids(self) -> tuple[str, ...]:
"""In-flight load repos, which the delete-cached guard must refuse while a download or
assembly runs."""
with self._lock:
loading = self._loading
if loading is None or loading.error is not None:
return ()
ids = (loading.repo_id, loading.base_repo, *loading.asset_repos)
# Both sides of a mirror pair: the fetch reads the mirror while status() names the upstream.
return tuple(dict.fromkeys(alias for r in ids if r for alias in _video_repo_aliases(r)))
def loaded_repo_ids(self) -> tuple[str, ...]:
"""Companion repos re-read each generation; status() omits them, so delete-cached must refuse them."""
with self._lock:
state = self._state
if state is None or state.engine != "sd_cpp":
return ()
repo_id = state.repo_id
from .video_minimax_h3 import (
H3_COMPONENT_REPO,
H3_GGUF_REPO,
)
return (repo_id, H3_GGUF_REPO, H3_COMPONENT_REPO)
# ── the load itself ──────────────────────────────────────────────────────
def _raise_if_video_load_superseded(self, token: Optional[int]) -> None:
if token is not None and token != self._load_token:
raise RuntimeError("Video load was cancelled or superseded.")
@_invalidates_gpu_memory("video load")
@scoped_allow_oversized_load
@rocm_probe_load_scoped
@scoped_local_files_only
@vram_budget_scope("video", load = True)
def load_pipeline(
self,
repo_id: str,
*,
display_repo_id: Optional[str] = None,
local_files_only: bool = False,
gguf_filename: Optional[str] = None,
base_repo: Optional[str] = None,
family_override: Optional[str] = None,
hf_token: Optional[str] = None,
memory_mode: Optional[str] = None,
speed_mode: Optional[str] = None,
attention_backend: Optional[str] = None,
transformer_cache: Optional[str] = None,
transformer_cache_threshold: Optional[float] = None,
transformer_quant: Optional[str] = None,
text_encoder_quant: Optional[str] = None,
model_kind: Optional[str] = None,
h3_task: Optional[str] = None,
gpu_ordinal: Optional[int] = None,
# Read by ``scoped_allow_oversized_load``.
allow_oversized_load: bool = False,
_load_token: Optional[int] = None,
_base_local_dir: Optional[str] = None,
_base_fetch_repo: Optional[str] = None,
_base_fetch_revision: Optional[str] = None,
_checkpoint_fetch: Optional[tuple[str, Optional[str]]] = None,
_te_prequant_skipped: tuple[str, ...] = (),
_te_prequant_failed: Optional[dict] = None,
_h3_auto_denoiser_planned: Optional[str] = None,
_video_auto_denoiser_planned: Optional[str] = None,
_video_native_hosted_planned: Optional[str] = None,
_denoiser_prequant_skipped: tuple[str, ...] = (),
) -> dict[str, Any]:
fam = self.validate_load_request(
repo_id,
gguf_filename = gguf_filename,
base_repo = base_repo,
family_override = family_override,
display_repo_id = display_repo_id,
model_kind = model_kind,
transformer_quant = transformer_quant,
text_encoder_quant = text_encoder_quant,
h3_task = h3_task,
)
install_single_file_fp32_buffers(logger)
kind = resolve_video_model_kind(gguf_filename, model_kind)
from .video_minimax_h3 import is_h3_native
if is_h3_native(fam, kind):
self._run_load_h3_native(
fam = fam,
token = _load_token,
cancel_event = threading.Event(),
repo_id = repo_id,
display_repo_id = display_repo_id,
gguf_filename = gguf_filename,
hf_token = hf_token,
memory_mode = memory_mode,
family_override = family_override,
# Carried: load_pipeline is also called directly, and an offline load must not fetch.
local_files_only = local_files_only,
transformer_cache = transformer_cache,
speed_mode = speed_mode,
)
return self.status()
# After the H3 native return (no diffusers), before anything importing diffusers (a dynamo importer).
try:
from utils.torch_warmup import close_dynamo_import_window
close_dynamo_import_window(logger)
except Exception as exc: # noqa: BLE001 - optimisation only
logger.debug("dynamo pre-import skipped: %r", exc)
import diffusers
import torch
# diffusers hard-codes _tqdm_active=True, honours no env var; setup_logging cannot silence its bar.
try:
from loggers.config import quiet_third_party_progress_bars
quiet_third_party_progress_bars()
except Exception: # noqa: BLE001 - never let log tidying stop a load
pass
base = repo_id if kind == "pipeline" else resolve_video_base_repo(fam, base_repo, repo_id)
# Where the base's components are FETCHED from when no staged snapshot dir covers them: its unsloth mirror.
# ``base`` stays the upstream id every table below is keyed on.
# The predownload's exact-name choice when it made one, so assembly reads the repo it just staged.
fetch_base = _base_fetch_repo or _video_fetch_repo(base, hf_token) or base
with self._lock:
if _load_token is not None and _load_token != self._load_token:
raise RuntimeError("Video load was cancelled or superseded.")
# Signal a generation from the PREVIOUS model (the token check above bailed a superseded worker)
if self._active_generate_cancel is not None:
self._active_generate_cancel.set()
# Raise the fence before the barrier: a queued generation holds no cancel event.
self._teardown_waiters += 1
# Barrier: wait for the signalled generation to exit before teardown, or two models coexist in VRAM.
with self._generate_lock:
with self._lock:
try:
# The barrier wait can outlive this load; recheck before touching shared state.
if _load_token is not None and _load_token != self._load_token:
raise RuntimeError("Video load was cancelled or superseded.")
self._teardown_state_locked()
finally:
# Released here so a raising teardown cannot leave the fence up forever.
self._teardown_waiters -= 1
target = self._device_target(gpu_ordinal)
device = target.device
# Text-encoder tri-state, resolved once here so the memory plan, the pre-cast injection and the dense cast below
# see one decision: unset / "auto" takes the family's te_quant_auto (the hosted encoder ComfyUI's template
# matches), "none" / "off" keep the released one, an explicit scheme pins itself. An automatic pick this card
# cannot run is dropped rather than refused.
text_encoder_quant_requested = text_encoder_quant
text_encoder_quant, text_encoder_quant_auto = video_te_quant_request(
fam,
text_encoder_quant,
base = repo_id if kind == "pipeline" else base,
speed_mode = speed_mode,
)
if fam.name == "ltx-2.5":
# Settle auto once so the memory plan and the encoder worker read one decision.
from .video_ltx25 import ltx25_resolve_te_request
text_encoder_quant = ltx25_resolve_te_request(
text_encoder_quant,
repo_id if kind == "pipeline" else base,
_te_prequant_failed,
)
# An automatic pick whose hosted pre-cast encoder could not be fetched loads the dense encoder: say so.
te_auto_unavailable = auto_te_fetch_failure(
fam, text_encoder_quant if text_encoder_quant_auto else None, _te_prequant_failed
)
if text_encoder_quant_auto:
from . import diffusion_precision as _precision
if not _precision.te_quant_supported(target, text_encoder_quant):
text_encoder_quant = None
elif not te_auto_unavailable:
logger.info(*auto_te_outcome_log(text_encoder_quant, fam.name, None, True))
# A failed prefetch is retried by the load below: its outcome is logged after that.
te_auto_retry = (
text_encoder_quant if text_encoder_quant_auto and te_auto_unavailable else None
)
# FlashInfer only serves hosted pre-quantised denoisers. Modular H3 installs below; a conventional load installs
# after its live-memory plan keeps the seed (a capacity-settled seed can still be dropped).
_nvfp4_install_wanted = (
kind == "pipeline"
and nvfp4_diffusion_enabled()
and "nvfp4"
in (
normalize_transformer_quant(transformer_quant),
_video_auto_denoiser_planned,
)
and (
_video_auto_denoiser_planned == "nvfp4"
or self._nvfp4_denoiser_checkpoint_will_load(
fam,
base,
h3_task,
hf_token,
local_files_only = local_files_only,
kind = kind,
planned = _video_auto_denoiser_planned,
)
)
)
# Bound only at the commit past the token check: a superseded load may return from the install late.
_nvfp4_install_outcome: Optional[tuple[bool, str]] = None
# Video DiTs are bf16-native: fp16 promotes to float32 unless the family declares an fp16_guard.
dtype = target.dtype
if dtype is torch.float16 and fp16_promotes_to_fp32(fam):
dtype = torch.float32
logger.warning(
"video.dtype_promoted: family=%s float16 -> float32 (fp16-incompatible)", fam.name
)
elif fam.fp16_incompatible and dtype is torch.float16:
logger.info(
"video.fp16_guard: family=%s recipe=%s (float16 kept)", fam.name, fam.fp16_guard
)
# Size tables below are bf16 (2-byte), so scale dense estimates when the promotion lands fp32 on an accelerator
dtype_scale = 2.0 if device != "cpu" and dtype is torch.float32 else 1.0
if fam.modular_workflow:
if _nvfp4_install_wanted:
from .diffusion_nvfp4_install import ensure_flashinfer_for_nvfp4
_nvfp4_install_outcome = ensure_flashinfer_for_nvfp4(
device, logger = logger, local_files_only = local_files_only
)
comfy_checkpoint = None
if kind == "single_file":
comfy_checkpoint = str(
self._resolve_checkpoint_path(
repo_id,
gguf_filename,
hf_token,
local_files_only = local_files_only,
fetch = _checkpoint_fetch,
)
)
if not _is_trusted_video_repo(repo_id):
assert_safetensors_file(comfy_checkpoint)
return self._load_h3_modular_pipeline(
diffusers = diffusers,
torch = torch,
local_files_only = local_files_only,
fam = fam,
target = target,
comfy_checkpoint = comfy_checkpoint,
repo_id = repo_id,
gguf_filename = gguf_filename,
display_repo_id = display_repo_id,
base = base,
kind = kind,
dtype = dtype,
device = device,
hf_token = hf_token,
memory_mode = memory_mode,
family_override = family_override,
transformer_quant = transformer_quant,
text_encoder_quant = text_encoder_quant,
# The speed layer is below this dispatch, which the modular branch never reaches.
speed_mode = speed_mode,
attention_backend = attention_backend,
h3_task = h3_task,
_load_token = _load_token,
_base_local_dir = _base_local_dir,
_base_fetch_repo = _base_fetch_repo,
_base_fetch_revision = _base_fetch_revision,
# Settled before the pull when the pull acted on it; None when it did not.
_h3_auto_denoiser_planned = _h3_auto_denoiser_planned,
_nvfp4_install_outcome = _nvfp4_install_outcome,
transformer_cache = transformer_cache,
)
transformer_quant_requested = transformer_quant
# Unset/auto: hardware ladder; none/off: dense bf16; a scheme pins it. Pipeline kind only.
if transformer_quant is None or str(transformer_quant).strip().lower() in (
"",
"auto",
):
# Explicit Speed=off must stay dense bf16 (bit-exact); auto-quant would engage int8/fp8 and compile.
speed_off = speed_mode is not None and str(speed_mode).strip().lower() == SPEED_OFF
transformer_quant = "off" if speed_off else TQ_AUTO
# Settled memory, so the previous pipeline's cached-but-free buffers do not read as used.
device_memory = settled_snapshot_device_memory(target)
from .diffusion_component_sizes import video_components_gb
components = video_components_gb(
fam,
# GGUF / single-file: companions come from the resolved base, not the family default
repo_id if kind == "pipeline" else base,
allow_hub = not local_files_only,
token = hf_token,
dev = _ltx25_dev_load(fam, kind, gguf_filename, repo_id, display_repo_id),
)
mib_per_gb = 1000.0**3 / (1024.0 * 1024.0)
# A hosted fp8 encoder makes the bf16 entry overstate by ~11 GB; VAE is never quantised.
from .diffusion_te_prequant import te_prequant_budget_scale
te_scale = te_prequant_budget_scale(
fam,
te_quant_mode = text_encoder_quant,
target = target,
base = repo_id if kind == "pipeline" else base,
)
transformer_mib: Optional[int] = None
comfy_scan = None
# False: the user pinned the bf16 DiT, so quantized layers dequantize.
comfy_keep = (
transformer_quant is None or normalize_transformer_quant(transformer_quant) is not None
)
partner_path: Optional[Path] = None
partner_scan = None
# LTX-2.3 variant read from the weights; outranks the file name in recipe lookups (None: the name decides).
variant_id: Optional[str] = None
ltx_keys = None
if kind != "pipeline":
checkpoint_path = self._resolve_checkpoint_path(
repo_id,
gguf_filename,
hf_token,
local_files_only = local_files_only,
fetch = _checkpoint_fetch,
)
# Admitted per file from an untrusted repo: prove each file is a safetensors container before any probe.
untrusted_single_file = kind == "single_file" and not _is_trusted_video_repo(repo_id)
if untrusted_single_file:
assert_safetensors_file(checkpoint_path)
if fam.name == "ltx-2":
from .video_ltx2 import ltx23_variant_identifier
variant_id = ltx23_variant_identifier(checkpoint_path)
if variant_id is not None:
logger.info(
"video.ltx23_variant: %s (from the checkpoint's weights)", variant_id
)
if fam.name == "ltx-2":
# The file also bundles VAE / audio VAE / vocoder, priced as companions; connectors stay in.
from .video_ltx2 import ltx23_is_dit_or_connector_key as ltx_keys
if fam.name == "ltx-2.5":
# Only the DiT: the pack's shared connectors are priced as companions.
from .video_ltx2 import ltx23_is_dit_key as ltx_keys
def _price_checkpoint(path: Any) -> tuple[Optional[int], Any]:
"""Resident MiB of one checkpoint file's DiT and its ComfyUI quant scan (None: a plain file)."""
scan = None
mib: Optional[int] = None
if untrusted_single_file:
assert_safetensors_file(path)
size_mib = file_size_mib(str(path))
if kind == "single_file":
scan = refuse_comfy_quant(str(path))
if scan is None and _video_comfy_key_map(fam) is not None:
# No diffusers converter: an unquantized original-layout file goes through the key map too.
from .diffusion_comfy_quant import ComfyQuantScan
scan = ComfyQuantScan()
if kind == "gguf":
mib = estimate_gguf_resident_mib(size_mib)
elif scan is not None:
mib = _video_comfy_resident_mib(
fam,
base,
target,
path,
scan,
keep = comfy_keep,
keep_key = ltx_keys,
)
if mib is None:
mib = estimate_safetensors_dense_mib(size_mib, fp8_upcast = True)
if mib is not None:
mib = int(mib * dtype_scale)
else:
mib = estimate_safetensors_dense_mib(size_mib)
if fam.name == "ltx-2" and mib is not None:
# The file also bundles VAE / audio VAE / vocoder, already priced as companions. The connectors
# under this prefix stay in: the table undercounts them (~2.9 of 6 GB), and dropping them plans
# below the peak.
dit_mib = safetensors_prefix_mib(str(path), "model.diffusion_model.")
if dit_mib is not None:
mib = min(mib, dit_mib)
if mib is not None:
mib = int(mib * dtype_scale)
return mib, scan
transformer_mib, comfy_scan = _price_checkpoint(checkpoint_path)
if getattr(fam, "is_moe", False):
# Both experts stay loaded, so price the pair; the planner splits the total per expert.
partner_path = self._resolve_moe_partner_path(
repo_id,
gguf_filename,
hf_token,
local_files_only = local_files_only,
fetch = _checkpoint_fetch,
)
partner_mib, partner_scan = _price_checkpoint(partner_path)
if (comfy_scan is None) != (partner_scan is None):
raise ValueError(
f"'{Path(str(checkpoint_path)).name}' and '{partner_path.name}' are not the same kind "
f"of checkpoint (one ComfyUI-quantized, one not); pick a matching pair."
)
transformer_mib = (
transformer_mib + partner_mib
if transformer_mib is not None and partner_mib is not None
else None
)
if moe_expert_of(Path(str(checkpoint_path)).name) == "low":
# The high-noise expert is transformer (the early steps), whichever of the two was picked.
checkpoint_path, partner_path = partner_path, checkpoint_path
comfy_scan, partner_scan = partner_scan, comfy_scan
# Price the plan at the hosted fp8 DiT: the bf16 file size would plan an offload and skip the seed.
ltx23_prequant_pick = (
transformer_mib is not None and comfy_scan is None
) and _ltx23_prequant_serves(
fam,
kind,
gguf_filename,
normalize_transformer_quant(transformer_quant),
target = target,
memory_mode = memory_mode,
checkpoint_repo = repo_id,
)
ltx23_dense_transformer_mib = transformer_mib
if ltx23_prequant_pick:
# The hosted artifact size, not 0.55 x file: the file also carries companions (over-stated ~6 GB).
from .video_ltx2 import LTX23_PREQUANT_RESIDENT_GB
transformer_mib = int(LTX23_PREQUANT_RESIDENT_GB * mib_per_gb)
runtime_mib = estimate_video_runtime_mib(
width = fam.resolution_presets[0][0],
height = fam.resolution_presets[0][1],
num_frames = fam.default_num_frames,
)
def _plan_for_te_scale(
scale: float,
*,
log: bool,
denoiser_gb: Optional[float] = None,
) -> tuple[Any, Any, bool]:
"""``(plan, bf16_plan, quant_replanned)``; ``denoiser_gb`` prices a SEEDED DiT (peak == steady)."""
text_encoder_gb = components[1] * scale if components is not None else 0.0
# vae_force_fp32 VAE is already counted at fp32; scaling it double-counts (2.8 GB on Wan TI2V-5B).
vae_gb = components[2] if components is not None else 0.0
worker_gb = 0.0
if components is not None and fam.name == "ltx-2.5":
from .video_ltx25 import LTX25_WORKER_RESIDENT_GB
# The encoder worker is a second process on the card: its CUDA context and allocator slack stay
# resident on every tier, even when its weights are off the card. The latent upsampler a dev (single
# stage) load never attaches is already out of components[2] (video_components_gb(dev=...)).
worker_gb = LTX25_WORKER_RESIDENT_GB
vae_gb += worker_gb
vae_scale = 1.0 if getattr(fam, "vae_force_fp32", False) else dtype_scale
scaled_te_gb = text_encoder_gb * dtype_scale
scaled_vae_gb = vae_gb * vae_scale
worker_mib = int(worker_gb * vae_scale * mib_per_gb)
scaled_companions_gb = scaled_te_gb + scaled_vae_gb if components is not None else 0.0
if log and scale != 1.0 and components is not None:
logger.info(
"video.te_prequant_budget: budgeting the pre-cast %s text encoder at %.2f of "
"bf16 (%.1f GB instead of %.1f GB)",
text_encoder_quant,
scale,
text_encoder_gb,
components[1],
)
# Without the text-encoder split the planner can only pick all-resident or whole-module offload.
companion_mib = (
int(scaled_companions_gb * mib_per_gb) if components is not None else None
)
text_encoder_mib = int(scaled_te_gb * mib_per_gb) if components is not None else None
if kind == "pipeline":
transformer_gb = (
components[0] * dtype_scale if denoiser_gb is None else float(denoiser_gb)
)
model_dense_mib = (
int((transformer_gb + scaled_companions_gb) * mib_per_gb)
if components is not None
else None
)
else:
# Budget all weights: companions stay resident, else auto picks OFFLOAD_NONE and OOMs.
model_dense_mib = (
transformer_mib + (companion_mib or 0) if transformer_mib is not None else None
)
experts = 2 if getattr(fam, "is_moe", False) else 1
vae_mib = int(scaled_vae_gb * mib_per_gb) if components is not None else None
dit_mib = (
(int(transformer_gb * mib_per_gb) if components is not None else None)
if kind == "pipeline"
else transformer_mib
)
planned = _video_prefer_whole_module(
plan_diffusion_memory(
target = target,
device_memory = device_memory,
model_dense_mib = model_dense_mib,
runtime_headroom_mib = runtime_mib,
companion_dense_mib = companion_mib,
text_encoder_dense_mib = text_encoder_mib,
requested_mode = normalize_memory_mode(memory_mode),
),
denoiser_mib = None if dit_mib is None else dit_mib // experts,
text_encoder_mib = text_encoder_mib,
vae_mib = None if vae_mib is None else vae_mib - worker_mib,
runtime_mib = runtime_mib,
resident_mib = worker_mib,
)
# Re-plan with the scheme factor: the bf16-table plan can offload a DiT the quantised one would not.
dense_plan = planned
replanned_for_quant = False
requested_scheme = normalize_transformer_quant(transformer_quant)
# auto keeps a resident bf16 DiT: quantising costs accuracy for little speed.
worth_replanning = (
not plan_keeps_transformer_resident(planned)
if requested_scheme == TQ_AUTO
else planned.offload_policy != OFFLOAD_NONE
)
# Not for a unified-memory shortfall: runtime quant still peaks at the dense bf16 DiT.
if (
kind == "pipeline"
and denoiser_gb is None
and worth_replanning
and requested_scheme is not None
and (
dense_transformer_supported(target)
or native_quant_scheme(target, transformer_quant, family = fam.name) is not None
or native_quant_scheme(
target,
_auto_offload_scheme(transformer_quant),
family = fam.name,
offload = True,
)
is not None
)
and components is not None
):
scheme_preview = (
native_quant_scheme(target, transformer_quant, family = fam.name)
or (
select_transformer_quant_scheme(target, transformer_quant, family = fam.name)
if dense_transformer_supported(target)
else None
)
or native_quant_scheme(
target,
_auto_offload_scheme(transformer_quant),
family = fam.name,
offload = True,
)
)
factor = _QUANT_STEADY_FACTOR.get(scheme_preview) if scheme_preview else None
if factor is not None:
# Companions priced as the split below (dtype-scaled), else the DiT share drifts on fp16-to-fp32.
quant_mib = int((components[0] * factor + scaled_companions_gb) * mib_per_gb)
replanned = _video_prefer_whole_module(
plan_diffusion_memory(
target = target,
device_memory = device_memory,
model_dense_mib = quant_mib,
runtime_headroom_mib = runtime_mib,
companion_dense_mib = companion_mib,
text_encoder_dense_mib = text_encoder_mib,
requested_mode = normalize_memory_mode(memory_mode),
),
denoiser_mib = int(components[0] * factor * mib_per_gb) // experts,
text_encoder_mib = text_encoder_mib,
vae_mib = None if vae_mib is None else vae_mib - worker_mib,
runtime_mib = runtime_mib,
resident_mib = worker_mib,
)
if _video_plan_rank(replanned) < _video_plan_rank(planned):
if log:
logger.info(
"video.transformer_quant: %s (%d MiB steady) places the DiT as '%s' "
"instead of the bf16 plan's '%s'",
scheme_preview,
quant_mib,
_video_plan_label(replanned),
_video_plan_label(planned),
)
planned = replanned
replanned_for_quant = True
# a sidecar-encoder family (LTX-2.5) keeps one card: a split cannot move its encoder
return (
_video_single_card_for_sidecar(fam, planned, logger if log else None),
_video_single_card_for_sidecar(fam, dense_plan),
replanned_for_quant,
)
denoiser_seed_scheme: Optional[str] = None
if kind == "pipeline" and _video_auto_denoiser_planned != DENOISER_SEED_DECLINED:
denoiser_seed_scheme = _video_auto_denoiser_planned or _video_auto_denoiser_scheme(
fam,
target = target,
requested = transformer_quant,
base_repo = base,
speed_mode = speed_mode,
hf_token = hf_token,
)
denoiser_seed_sources: dict[str, Any] = {}
if denoiser_seed_scheme is not None:
from .video_denoiser_prequant import denoiser_prequant_sources
denoiser_seed_sources = (
denoiser_prequant_sources(fam, denoiser_seed_scheme, base, hf_token = hf_token) or {}
)
if not denoiser_seed_sources:
denoiser_seed_scheme = None
denoiser_seed_gb: Optional[float] = None
if denoiser_seed_scheme is not None and components is not None:
measured = video_family_prequant_resident_gb(fam, denoiser_seed_scheme)
factor = _QUANT_STEADY_FACTOR.get(denoiser_seed_scheme)
if measured:
denoiser_seed_gb = float(measured)
elif factor is not None:
denoiser_seed_gb = components[0] * factor
plan, bf16_plan, quant_replanned = _plan_for_te_scale(
te_scale, log = True, denoiser_gb = denoiser_seed_gb
)
seed_streamed = (
_video_torchao_seed_placement(fam, denoiser_seed_scheme, target, plan)
if denoiser_seed_scheme is not None
else None
)
if seed_streamed is not None:
logger.info(
"video.denoiser_prequant: the %s denoiser is offloaded ('%s') on this card, and its "
"torchao weights stream there, so the hosted checkpoint is seeded as '%s'",
denoiser_seed_scheme,
_video_plan_label(plan),
_video_plan_label(seed_streamed),
)
plan = seed_streamed
elif denoiser_seed_scheme is not None and not plan_keeps_transformer_resident(plan):
logger.info(
"video.denoiser_prequant: the %s denoiser would have to be offloaded ('%s') on this "
"card, so the hosted checkpoint is not seeded",
denoiser_seed_scheme,
_video_plan_label(plan),
)
if _video_native_hosted_planned is None and _offload_hosted_read_enabled():
# The probe planned a resident seed; read the hosted native build rather than quantising dense shards.
_video_native_hosted_planned = _video_native_hosted_scheme(
fam, denoiser_seed_scheme, target
)
if _video_native_hosted_planned is not None:
logger.info(
"video.denoiser_prequant: the hosted %s checkpoint is read as offload-safe native Linears",
_video_native_hosted_planned,
)
denoiser_seed_scheme = None
denoiser_seed_gb = None
plan, bf16_plan, quant_replanned = _plan_for_te_scale(te_scale, log = False)
if _auto_seed_yields_to_resident_bf16(
denoiser_seed_scheme, transformer_quant, fam
) and _auto_keeps_resident_bf16(_plan_for_te_scale(te_scale, log = False)[1], target):
# Auto keeps a resident bf16 DiT; seeding would change auto's precision wherever bf16 fits.
logger.info(
"video.denoiser_prequant: auto keeps the bf16 DiT (it fits resident), so the hosted "
"%s checkpoint is not seeded",
denoiser_seed_scheme,
)
denoiser_seed_scheme = None
denoiser_seed_gb = None
plan, bf16_plan, quant_replanned = _plan_for_te_scale(te_scale, log = False)
comfy_priced_offload = False
if comfy_scan is not None and _video_comfy_reprice(plan, bf16_plan):
# Price the DiT the load will actually build.
offload_mib = _video_comfy_resident_mib(
fam,
base,
target,
checkpoint_path,
comfy_scan,
keep = comfy_keep,
keep_key = ltx_keys,
offload = True,
)
if offload_mib is not None and partner_path is not None and partner_scan is not None:
# transformer_mib prices the pair: reprice both experts.
partner_offload_mib = _video_comfy_resident_mib(
fam,
base,
target,
partner_path,
partner_scan,
keep = comfy_keep,
keep_key = ltx_keys,
offload = True,
)
offload_mib = (
offload_mib + partner_offload_mib if partner_offload_mib is not None else None
)
if offload_mib is not None and transformer_mib is not None:
offload_mib = int(offload_mib * dtype_scale)
repriced, comfy_priced_offload = _video_offload_reprice(
fam, transformer_mib, offload_mib
)
if repriced != transformer_mib:
transformer_mib = repriced
plan, bf16_plan, quant_replanned = _plan_for_te_scale(te_scale, log = False)
# The DiT as planned: a resident load that would widen it into under the host reserve stores fp8 instead.
comfy_widened_mib = transformer_mib if comfy_scan is not None else None
settled_te_scale = te_scale
# Unified memory has no offload: oversize is an OS kill; refuse after eviction, before weights load.
raise_on_unified_memory_shortfall(plan, family = getattr(fam, "name", None), logger = logger)
# ── LTX-2.5: its Gemma 4 encoder loads in a sidecar worker, overlapping the DiT load below.
ltx25_client = None
ltx25_dev = False
if fam.name == "ltx-2.5":
from .video_families import ltx25_is_dev
from .video_ltx25 import (
encoder_resident_for_plan,
ltx25_te_quant_scheme,
start_text_encoder,
)
ltx25_dev = _ltx25_dev_load(fam, kind, gguf_filename, repo_id, display_repo_id)
# A stale load's pending worker goes only while this load is current; a superseded load that resumes here
# must not stop the newer load's worker.
with self._lock:
if _load_token is not None and _load_token != self._load_token:
raise RuntimeError("Video load was cancelled or superseded.")
stale_encoder, self._ltx25_pending_encoder = self._ltx25_pending_encoder, None
if stale_encoder is not None:
_stop_ltx25_encoder(stale_encoder[1])
worker_card = _video_sidecar_worker_card(fam, plan)
if worker_card is not None:
from .diffusion_multigpu import secondary_safe_budget_mib
worker_device = f"cuda:{worker_card}"
worker_budget = secondary_safe_budget_mib(worker_card)
else:
worker_device = str(target.torch_device)
worker_budget = (getattr(plan, "estimates", None) or {}).get(
"safe_device_budget_mib"
)
ltx25_client = start_text_encoder(
repo = _base_local_dir or fetch_base,
# The plan's commit when the worker reads the hub id (a staged dir already is that commit).
revision = None if _base_local_dir else _base_fetch_revision,
hf_token = hf_token,
cache_dir = hub_cache_dir(),
local_files_only = local_files_only,
device = worker_device,
dtype = str(dtype).replace("torch.", ""),
resident = encoder_resident_for_plan(plan),
budget_mib = worker_budget,
te_quant = ltx25_te_quant_scheme(text_encoder_quant),
base_repo = base,
bind_device = _video_sidecar_multigpu(fam),
)
# Installed only while this load is current: a load superseded during the start must not overwrite the
# newer load's worker (whose token-scoped cleanup could then never find it).
with self._lock:
superseded = _load_token is not None and _load_token != self._load_token
if not superseded:
self._ltx25_pending_encoder = (_load_token, ltx25_client)
if superseded:
_stop_ltx25_encoder(ltx25_client)
raise RuntimeError("Video load was cancelled or superseded.")
# ── build the pipeline.
from .ltx2_import_compat import ensure_ltx2_pipelines_importable, is_ltx2_pipeline_class
if is_ltx2_pipeline_class(fam.pipeline_class):
ensure_ltx2_pipelines_importable(logger)
pipeline_cls = getattr(diffusers, fam.pipeline_class)
# cache_dir pins every loader call to the live cache root, so a mid-session change cannot split one model across
# roots.
def _pinned_base_kwargs() -> dict[str, Any]:
"""A hub-id base read takes the plan's commit; the staged dir already is that commit."""
return (
{"revision": _base_fetch_revision}
if _base_fetch_revision and not _base_local_dir
else {}
)
def _safetensors_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:
"""use_safetensors=True when the base folder ships safetensors for every component this load reads."""
return pipeline_safetensors_kwargs(
_base_local_dir or fetch_base,
passed = {**_pinned_base_kwargs(), **kwargs},
cache_dir = kwargs.get("cache_dir"),
pipeline_cls = pipeline_cls,
)
pipe_kwargs: dict[str, Any] = {
"torch_dtype": dtype,
"cache_dir": hub_cache_dir(),
"local_files_only": local_files_only,
}
if getattr(fam, "vae_force_fp32", False):
# VAE must stay fp32 (banding, black frames): scalar torch_dtype casts it to bf16; set a default
pipe_kwargs["torch_dtype"] = {"vae": torch.float32, "default": dtype}
if hf_token:
pipe_kwargs["token"] = hf_token
# Pre-cast fp8 encoder skips dense TE download (~50 GB for LTX Gemma3-27B); re-cast is idempotent.
from .diffusion_te_prequant import te_prequant_pipe_kwargs
te_injected = te_prequant_pipe_kwargs(
fam,
# the encoder config is read from here; the base checks fold a mirror back onto its upstream
fetch_base,
te_quant_mode = text_encoder_quant,
target = target,
dtype = dtype,
hf_token = hf_token,
logger = logger,
local_files_only = local_files_only,
)
pipe_kwargs.update(te_injected)
if te_auto_retry:
logger.info(
*auto_te_outcome_log(
te_auto_retry, fam.name, te_auto_unavailable, bool(te_injected)
)
)
if text_encoder_quant_auto and not te_injected:
# Nobody asked for a runtime cast: without its hosted file the automatic pick keeps the released encoder.
text_encoder_quant = None
# The fp8-sized budget is valid only once the pre-cast encoder is in hand. Injection is best-effort (an
# unreachable / corrupt / base-mismatched checkpoint falls back to the dense encoder), and a dense encoder under
# an fp8 budget under-states the resident requirement by ~11 GB for ltx-2, which is the direction that OOMs.
# Placement is applied post-assembly (apply_memory_plan below), so re-plan at bf16 here, exactly as the
# dense-quant fallback below restores bf16_plan.
if te_scale != 1.0 and not te_injected and fam.name != "ltx-2.5":
logger.warning(
"video.te_prequant_budget: no pre-cast encoder was injected; re-planning "
"memory at the dense bf16 text-encoder size"
)
settled_te_scale = 1.0
plan, bf16_plan, quant_replanned = _plan_for_te_scale(
settled_te_scale, log = False, denoiser_gb = denoiser_seed_gb
)
seed_streamed = (
_video_torchao_seed_placement(fam, denoiser_seed_scheme, target, plan)
if seed_streamed is not None
else None
)
if seed_streamed is not None:
plan = seed_streamed
elif denoiser_seed_scheme is not None and not plan_keeps_transformer_resident(plan):
denoiser_seed_scheme = None
denoiser_seed_gb = None
plan, bf16_plan, quant_replanned = _plan_for_te_scale(settled_te_scale, log = False)
_nvfp4_install_outcome = self._install_flashinfer_for_seed(
_nvfp4_install_wanted, denoiser_seed_scheme, device, local_files_only = local_files_only
)
denoiser_injected: dict[str, Any] = {}
ltx23_direct_loaded = False
if denoiser_seed_scheme is not None:
from .video_denoiser_prequant import denoiser_prequant_pipe_kwargs
denoiser_injected = denoiser_prequant_pipe_kwargs(
fam,
fetch_base,
scheme = denoiser_seed_scheme,
dtype = dtype,
device = device,
# a streamed seed is built on the host: on the card first it OOMs a 16 GiB budget
placement_device = (
prequant_seed_device(plan, device, denoiser_seed_scheme)
if seed_streamed is not None
else None
),
hf_token = hf_token,
target = target,
local_files_only = local_files_only,
cache_dir = hub_cache_dir(),
logger = logger,
)
pipe_kwargs.update(denoiser_injected)
if not denoiser_injected:
logger.warning(
"video.denoiser_prequant: no pre-quantized denoiser was seeded; re-planning "
"memory at the dense bf16 DiT size"
)
denoiser_seed_scheme = None
denoiser_seed_gb = None
plan, bf16_plan, quant_replanned = _plan_for_te_scale(settled_te_scale, log = False)
raise_on_unified_memory_shortfall(
plan, family = getattr(fam, "name", None), logger = logger
)
# Offloaded: no hosted checkpoint matches the native build; replay this machine's own (all experts or none).
native_cache_spec = (
_video_native_cache_spec(
fam,
target = target,
transformer_quant = transformer_quant,
plan = plan,
kind = kind,
source_dir = _base_local_dir or repo_id,
)
if kind == "pipeline" and not denoiser_injected
else None
)
native_config_kwargs: dict[str, Any] = (
{}
if _base_local_dir
else {
"cache_dir": hub_cache_dir(),
"local_files_only": local_files_only,
**({"token": hf_token} if hf_token else {}),
**_pinned_base_kwargs(),
}
)
native_cache_hits = _read_native_quant_from_cache(
fam,
native_cache_spec,
base = base,
source = _base_local_dir or fetch_base,
config_kwargs = native_config_kwargs,
cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),
)
self._raise_if_video_load_superseded(_load_token)
native_from_hosted = False
if (
not native_cache_hits
and native_cache_spec is not None
and _video_native_hosted_planned is not None
and _video_native_hosted_planned == native_cache_spec.get("scheme")
):
# The planner staged the hosted int8 file instead of the dense shards: read it as the native build.
native_cache_hits = _read_native_quant_from_hosted(
fam,
native_cache_spec,
base = fetch_base,
source = _base_local_dir or fetch_base,
config_kwargs = native_config_kwargs,
hf_token = hf_token,
local_files_only = local_files_only,
)
native_from_hosted = bool(native_cache_hits)
self._raise_if_video_load_superseded(_load_token)
if not native_cache_hits and native_cache_spec is not None:
# A miss whose dense load would not fit host RAM: build the same native experts per shard into the cache.
native_cache_hits = _video_native_stream_build(
fam,
native_cache_spec,
target = target,
dtype = dtype,
base = base,
source = _base_local_dir or fetch_base,
config_kwargs = native_config_kwargs,
skip = tuple(pipe_kwargs),
cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),
)
self._raise_if_video_load_superseded(_load_token)
pipe_kwargs.update(native_cache_hits)
# Injection is best-effort (a missing checkpoint falls back to the dense encoder), but the scoped pre-download
# already dropped those shards and from_pretrained cannot re-fetch from a local snapshot dir, so top them up
# here. ltx23 never reaches this: it gets the hub id.
missing_dense = [
c
for c in tuple(_te_prequant_skipped or ()) + tuple(_denoiser_prequant_skipped or ())
if c not in pipe_kwargs
# LTX-2.5's skipped encoder is the worker's: it loads the hosted file, else re-reads the Hub itself.
and not (ltx25_client is not None and c == "text_encoder")
]
if missing_dense and _base_local_dir:
logger.warning(
"video.te_prequant: %s not injected; restoring the dense weights the "
"pre-download skipped",
", ".join(missing_dense),
)
_base_local_dir = self._top_up_dense_base(
base,
hf_token,
kind,
base_local_dir = _base_local_dir,
fetch_repo = _base_fetch_repo,
revision = _base_fetch_revision,
local_files_only = local_files_only,
ltx25_dev = ltx25_dev,
)
if ltx25_client is not None:
from .video_ltx25 import LTX25_NONE_COMPONENTS
# The encoder runs in the worker; the enhancer, processor and duration head are never used.
pipe_kwargs.update({name: None for name in LTX25_NONE_COMPONENTS})
if kind == "pipeline" and fam.name in COSMOS3_FAMILIES:
from .video_cosmos3 import pipeline_kwargs as cosmos3_pipeline_kwargs
pipe_kwargs.update(cosmos3_pipeline_kwargs(str(repo_id)))
cast_writes = None
native_writes = None
if kind == "pipeline":
cast_missed = _read_cast_cache(
pipe_kwargs,
source = _base_local_dir or fetch_base,
revision = None if _base_local_dir else _base_fetch_revision,
base = base,
cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),
device = target.torch_device if target.device == "cuda" else None,
)
# The pre-downloaded snapshot dir keeps from_pretrained off the hub; hub id when pre-download was skipped.
if fam.name in COSMOS3_FAMILIES:
from .video_cosmos3 import guardrail_access_refusal, guardrail_hub_token
hub_token_scope = guardrail_hub_token(hf_token)
else:
hub_token_scope = contextlib.nullcontext()
try:
with hub_token_scope:
pipe = pipeline_cls.from_pretrained(
_base_local_dir or fetch_base,
**_pinned_base_kwargs(),
**pipe_kwargs,
**_safetensors_kwargs(pipe_kwargs),
)
except Exception as exc:
refusal = (
guardrail_access_refusal(str(repo_id), exc)
if fam.name in COSMOS3_FAMILIES
else None
)
if refusal is None:
raise
logger.warning("video.cosmos3: guardrail download refused: %s", exc)
raise ValueError(refusal) from exc
if cast_missed:
from .diffusion_cast_cache import defer_writes
# Names only: the write reads the live tensors once the load is published.
cast_writes = defer_writes(pipe, cast_missed, logger = logger)
if cast_writes is not None:
from .diffusion_memory import note_cast_cache_writes
# the pin plan below keeps its default price while this first load writes its cast cache
note_cast_cache_writes(pipe, cast_writes)
else:
transformer_cls = getattr(diffusers, fam.transformer_class)
# checkpoint_path was resolved (and downloaded) by the memory-planning branch above.
sf_kwargs: dict[str, Any] = {
"torch_dtype": dtype,
"config": fetch_base,
# The plan's commit for that config (absent for an unpinned load).
**({"revision": _base_fetch_revision} if _base_fetch_revision else {}),
"subfolder": "transformer",
"token": hf_token,
"cache_dir": hub_cache_dir(),
# diffusers resolves base via load_config(), so pass the flag or it fetches after eviction.
"local_files_only": local_files_only,
}
if kind == "gguf":
from .diffusion_gguf_exact import install_exact_dequant
from .diffusion_gguf_lowbit import install_lowbit_iq
install_lowbit_iq(logger)
install_exact_dequant(logger)
sf_kwargs["quantization_config"] = diffusers.GGUFQuantizationConfig(
compute_dtype = dtype
)
from .video_ltx2 import (
assert_not_ltx25_checkpoint,
direct_load_device,
is_ltx23_checkpoint,
load_ltx23_pipeline,
)
if fam.name == "ltx-2":
assert_not_ltx25_checkpoint(checkpoint_path)
if fam.name == "ltx-2" and is_ltx23_checkpoint(checkpoint_path):
# Explicit fp8 takes the hosted DiT, resident only: offload hooks' Module.to() rejects torchao tensors.
ltx23_override = None
ltx23_scheme = normalize_transformer_quant(transformer_quant)
seeded = None
if comfy_scan is not None:
# The DiT goes in as the override; the assembly reads the rest from the same file (or extras).
from .video_ltx2 import load_ltx23_comfy_transformer
ltx23_override = load_ltx23_comfy_transformer(
checkpoint_path,
comfy_scan,
base_repo = fetch_base,
torch_dtype = dtype,
hf_token = hf_token,
cache_dir = hub_cache_dir(),
local_files_only = local_files_only,
**_video_comfy_backends(
fam,
base,
target,
plan,
bf16_plan,
keep = comfy_keep,
priced_offload = comfy_priced_offload,
widened_mib = comfy_widened_mib,
),
family = fam.name,
target = target,
logger = logger,
)
elif ltx23_prequant_pick and plan_keeps_transformer_resident(plan):
from .video_ltx2 import load_ltx23_prequant_transformer
seeded = load_ltx23_prequant_transformer(
fam,
ltx23_scheme,
checkpoint_path,
config_repo = fetch_base,
device = device,
dtype = dtype,
hf_token = hf_token,
cache_dir = hub_cache_dir(),
local_files_only = local_files_only,
logger = logger,
)
if seeded is not None:
ltx23_override, ltx23_source = seeded
denoiser_injected = {"transformer": ltx23_override}
denoiser_seed_scheme = ltx23_scheme
denoiser_seed_sources = {"transformer": ltx23_source}
elif ltx23_prequant_pick:
# Decline before the dense assembly loads the 44 GB bf16 DiT only to refuse it.
ltx23_reason = (
f"this GPU's memory plan ({_video_plan_label(plan)}) offloads the DiT even at fp8 "
"size, and the hosted fp8 DiT is kept resident only (the text encoders may stream "
"around it)"
if not plan_keeps_transformer_resident(plan)
else "the hosted fp8 DiT (unsloth/LTX-2.3-FP8) could not be loaded"
+ (" from the local cache" if local_files_only else "")
)
if not precision_fallback_allowed():
clear_gpu_cache()
raise RuntimeError(
precision_refusal_message(
"transformer_quant",
ltx23_scheme,
ltx23_reason,
off_label = "Off to run the DiT at bf16",
)
)
logger.warning("video.ltx23_prequant: %s; loading the bf16 DiT", ltx23_reason)
transformer_mib = ltx23_dense_transformer_mib
plan, bf16_plan, quant_replanned = _plan_for_te_scale(
settled_te_scale, log = False
)
raise_on_unified_memory_shortfall(
plan, family = getattr(fam, "name", None), logger = logger
)
# Judged on the bf16 plan too: a single-file load never engages the runtime quant.
ltx23_device = None
ltx23_te_device = None
if kind != "gguf" and all(
plan_keeps_transformer_resident(p) for p in (plan, bf16_plan) if p is not None
):
ltx23_device = direct_load_device(target.torch_device)
if all(
getattr(p, "offload_policy", None) == OFFLOAD_NONE
for p in (plan, bf16_plan)
if p is not None
):
ltx23_te_device = ltx23_device
ltx23_direct_loaded = ltx23_device is not None
# 2.3 needs the full assembly: new config flags, key renames the stock converter lacks.
pipe = load_ltx23_pipeline(
checkpoint_path,
base_repo = fetch_base,
revision = _base_fetch_revision,
torch_dtype = dtype,
is_gguf = kind == "gguf",
hf_token = hf_token,
# 2.3 assembly never sees pipe_kwargs: pass the pre-cast encoder to it explicitly.
text_encoder = pipe_kwargs.get("text_encoder"),
# The 2.3 path skips pipe_kwargs and has no staged snapshot, so each component resolves the hub id.
local_files_only = local_files_only,
transformer_override = ltx23_override,
device = ltx23_device,
text_encoder_device = ltx23_te_device,
)
elif fam.name == "ltx-2.5":
# The split 2.5 DiT (bf16, int8-convrot or GGUF); connectors, VAEs, vocoder and upsampler come from the
# pack, the encoder from the worker.
from .video_ltx25 import (
is_ltx25_single_file_base,
load_ltx25_comfy_transformer,
load_ltx25_transformer,
)
if comfy_scan is not None and is_ltx25_single_file_base(
_base_local_dir or fetch_base
):
# The int8-convrot files quantize the text connectors too, and the pack-free assembly reads them from
# the DiT file: refuse before the DiT is read.
raise ValueError(
"This LTX-2.5 file stores its text connectors quantized, which the single-file assembly does "
"not dequantize. Load it on the Lightricks/LTX-2.5-Diffusers base (base_repo), or pick the "
"bf16 DiT or a GGUF quant."
)
if comfy_scan is not None:
ltx25_transformer = load_ltx25_comfy_transformer(
checkpoint_path,
comfy_scan,
base_repo = _base_local_dir or fetch_base,
torch_dtype = dtype,
hf_token = hf_token,
cache_dir = hub_cache_dir(),
local_files_only = local_files_only,
**_video_comfy_backends(
fam,
base,
target,
plan,
bf16_plan,
keep = comfy_keep,
priced_offload = comfy_priced_offload,
widened_mib = comfy_widened_mib,
),
family = fam.name,
target = target,
logger = logger,
)
else:
ltx25_transformer = load_ltx25_transformer(
checkpoint_path,
base_repo = _base_local_dir or fetch_base,
torch_dtype = dtype,
is_gguf = kind == "gguf",
hf_token = hf_token,
cache_dir = hub_cache_dir(),
local_files_only = local_files_only,
)
from .video_ltx25 import (
assemble_ltx25_single_file_pipeline,
is_ltx25_single_file_base,
)
if is_ltx25_single_file_base(_base_local_dir or fetch_base):
# The split single-file release: every component from the files beside the DiT, no pack.
pipe = assemble_ltx25_single_file_pipeline(
ltx25_transformer,
checkpoint_path,
base = _base_local_dir or fetch_base,
torch_dtype = dtype,
hf_token = hf_token,
local_files_only = local_files_only,
)
else:
pipe = pipeline_cls.from_pretrained(
_base_local_dir or fetch_base,
**_pinned_base_kwargs(),
**{**pipe_kwargs, "transformer": ltx25_transformer},
**_safetensors_kwargs({**pipe_kwargs, "transformer": ltx25_transformer}),
)
else:
def _load_expert(path: Any, scan: Any, subfolder: str) -> Any:
kwargs = dict(sf_kwargs, subfolder = subfolder)
if scan is not None:
# int8 codes go to Studio's int8 runtime (torchao resident, the torchao-free twin under
# offload) and fp8 codes to its fp8 GEMM when resident, by the rules Studio's own quant follows.
return load_comfy_quant_transformer(
transformer_cls,
str(path),
scan,
kwargs,
**_video_comfy_backends(
fam,
base,
target,
plan,
bf16_plan,
keep = comfy_keep,
priced_offload = comfy_priced_offload,
widened_mib = comfy_widened_mib,
),
family = fam.name,
target = target,
logger = logger,
key_map = _video_comfy_key_map(fam),
)
return transformer_cls.from_single_file(str(path), **kwargs)
transformer = _load_expert(checkpoint_path, comfy_scan, "transformer")
experts: dict[str, Any] = {"transformer": transformer}
if partner_path is not None:
experts["transformer_2"] = _load_expert(
partner_path, partner_scan, "transformer_2"
)
logger.info(
"video.moe_pair: transformer=%s transformer_2=%s",
Path(str(checkpoint_path)).name,
Path(str(partner_path)).name,
)
pipe = pipeline_cls.from_pretrained(
_base_local_dir or fetch_base,
**_pinned_base_kwargs(),
**{**pipe_kwargs, **experts},
**_safetensors_kwargs({**pipe_kwargs, **experts}),
)
# Dtype dict already loads the Wan VAE in float32; this backstops any path bypassing it (e.g. vae=).
if getattr(fam, "vae_force_fp32", False):
vae = getattr(pipe, "vae", None)
if vae is not None and getattr(vae, "dtype", None) is not torch.float32:
vae.to(torch.float32)
if ltx25_client is not None:
from .video_ltx25 import attach_to_pipeline
attach_to_pipeline(
pipe,
ltx25_client,
dev = ltx25_dev,
base_repo = _base_local_dir or fetch_base,
torch_dtype = dtype,
hf_token = hf_token,
cache_dir = hub_cache_dir(),
local_files_only = local_files_only,
)
if _load_token is not None and _load_token != self._load_token:
del pipe
clear_gpu_cache()
raise RuntimeError("Video load was cancelled or superseded.")
# Dual-DiT MoE: sites cover BOTH experts (``views`` = pipe, _SecondDiTView(pipe)); single: (pipe,).
views = _views_for(pipe, fam)
# Before the transformer quant, the first mutator; until the state commit a failure restores them itself.
backend_flags = snapshot_backend_flags()
self._precommit_globals = (_load_token, backend_flags)
# Quant must precede compile (eager torchao is ~30x slower).
transformer_quant_engaged: Optional[str] = None
quant_skipped_for_offload = False
# An explicit scheme is a contract: a decline must stop the load unless the escape hatch is set.
transformer_quant_pinned = normalize_transformer_quant(transformer_quant)
if transformer_quant_pinned == TQ_AUTO:
transformer_quant_pinned = None
# Why the quant did not engage, in the caller's terms; threaded into `resolved`.
transformer_quant_decline: Optional[str] = None
transformer_quant_decline_status = RESOLVED_FELL_BACK
transformer_quant_source: Optional[str] = None
# Text encoders streaming around a resident DiT do not move its weights, so torchao stays valid.
video_offload = not plan_keeps_transformer_resident(plan)
comfy_info = (
getattr(getattr(pipe, "transformer", None), "_unsloth_comfy_quant", None)
if comfy_scan is not None
else None
) or {}
partner_info = (
getattr(getattr(pipe, "transformer_2", None), "_unsloth_comfy_quant", None)
if partner_scan is not None
else None
)
if partner_info:
comfy_info = dict(comfy_info)
for key in ("int8", "fp8", "dequantized"):
comfy_info[key] = int(comfy_info.get(key) or 0) + int(partner_info.get(key) or 0)
comfy_scheme = (
TQ_INT8 if comfy_info.get("int8") else TQ_FP8 if comfy_info.get("fp8") else None
)
native_scheme = (
native_quant_scheme(
target,
_auto_offload_scheme(transformer_quant)
if video_offload and transformer_quant_pinned is None
else transformer_quant_pinned,
family = fam.name,
offload = video_offload,
)
# A seeded denoiser is a torchao checkpoint; the torchao-free path must not claim it.
if kind == "pipeline" and not denoiser_injected
else TQ_INT8
if comfy_scheme == TQ_INT8 and comfy_info.get("backend") == "native"
else None
)
native_kwargs = (
{
"offload": video_offload,
"act_int8": native_scheme == TQ_INT8 and native_int8_act(target),
}
if native_scheme is not None
else {}
)
native_cached = (
bool(native_cache_hits)
and native_cache_spec is not None
and video_offload
and native_scheme == native_cache_spec.get("scheme")
and bool(native_kwargs.get("act_int8")) == bool(native_cache_spec.get("act_int8"))
and _video_native_cache_spec(
fam,
target = target,
transformer_quant = transformer_quant,
plan = plan,
kind = kind,
source_dir = _base_local_dir or repo_id,
)
== native_cache_spec
)
if native_cache_hits and not native_cached:
# Not the build this load makes after all: put the dense experts back and let the quant below run.
logger.warning(
"video.transformer_quant: the cached native denoiser(s) do not match this load's "
"build; loading the dense denoiser shards instead"
)
dense_cls = getattr(diffusers, fam.transformer_class)
for component in native_cache_hits:
pipe.register_modules(
**{
component: dense_cls.from_pretrained(
_base_local_dir or fetch_base,
subfolder = component,
torch_dtype = dtype,
cache_dir = hub_cache_dir(),
local_files_only = local_files_only,
**({"token": hf_token} if hf_token else {}),
**_pinned_base_kwargs(),
**component_safetensors_kwargs(
_base_local_dir or fetch_base,
component,
cache_dir = hub_cache_dir(),
model_cls = dense_cls,
local_files_only = local_files_only,
revision = _pinned_base_kwargs().get("revision"),
),
)
}
)
native_cache_hits = {}
views = _views_for(pipe, fam)
# Auto quantises a video DiT only to keep it resident: with bf16 resident, int8 fails the default LPIPS bar.
if (
kind == "pipeline"
and normalize_transformer_quant(transformer_quant) == TQ_AUTO
and not quant_replanned
and not denoiser_injected
and _auto_keeps_resident_bf16(plan, target)
):
logger.info("video.transformer_quant: auto keeps the bf16 DiT (it fits resident)")
transformer_quant = "off"
transformer_quant_decline = (
"auto: the bf16 DiT fits resident, where int8 costs accuracy for little or no speed"
)
if (
transformer_quant_pinned is not None
and kind != "pipeline"
and not denoiser_injected
and transformer_quant_pinned != comfy_scheme
):
transformer_quant_decline = (
f"the dense DiT quant applies to full-pipeline loads only, and this is a "
f"'{kind}' load, which runs the precision its checkpoint carries"
)
transformer_quant_decline_status = RESOLVED_UNSUPPORTED
elif (
transformer_quant_pinned is not None
and native_scheme is None
and not dense_transformer_supported(target)
):
# Ask the helper: on ROCm and the Windows torchao stub it knows a truer reason.
transformer_quant_decline = dense_transformer_unsupported_reason(target)
transformer_quant_decline_status = RESOLVED_UNSUPPORTED
if comfy_scheme is not None:
transformer_quant_engaged = comfy_scheme
transformer_quant_source = (
f"ComfyUI-quantized checkpoint ({Path(str(checkpoint_path)).name}"
f"{' + ' + Path(str(partner_path)).name if partner_path is not None else ''}): "
f"{comfy_info.get('int8', 0)} int8 / {comfy_info.get('fp8', 0)} fp8 layers kept "
f"({comfy_info.get('backend') or comfy_info.get('fp8_backend')} runtime), "
f"{comfy_info.get('dequantized', 0)} dequantized"
)
logger.info("video.transformer_quant: %s", transformer_quant_source)
elif (
denoiser_injected
and denoiser_seed_sources
and all(_is_prequant_cache_source(src) for src in denoiser_seed_sources.values())
):
transformer_quant_engaged = denoiser_seed_scheme
transformer_quant_source = (
f"runtime {denoiser_seed_scheme} build read back from this machine's prequant cache of "
"the same build"
)
logger.info(
"video.transformer_quant: read the resident %s denoiser for %s from the local prequant "
"cache (%s); the dense denoiser shards are not loaded or quantised",
denoiser_seed_scheme,
fam.name,
", ".join(
sorted(
{
Path(str(src.location)).parent.name
for src in denoiser_seed_sources.values()
}
)
),
)
elif denoiser_injected:
transformer_quant_engaged = denoiser_seed_scheme
transformer_quant_source = (
f"hosted pre-quantized {denoiser_seed_scheme} denoiser checkpoint "
f"({', '.join(sorted({src.location for src in denoiser_seed_sources.values()}))})"
)
logger.info(
"video.transformer_quant: %s pre-quantized denoiser(s) seeded for %s; skipping "
"the runtime quantise",
denoiser_seed_scheme,
fam.name,
)
elif native_cached and native_from_hosted:
transformer_quant_engaged = native_scheme
transformer_quant_source = (
native_quant_reason(getattr(pipe, "transformer", None), native_scheme)
+ "; codes and scales read from the hosted pre-quantized checkpoint"
)
logger.info(
"video.transformer_quant: %s native denoiser(s) read from the hosted checkpoint for "
"%s; skipping the dense shards and the runtime quantise",
native_scheme,
fam.name,
)
elif native_cached:
transformer_quant_engaged = native_scheme
transformer_quant_source = (
native_quant_reason(getattr(pipe, "transformer", None), native_scheme)
+ "; read back from this machine's prequant cache of the same build"
)
logger.info(
"video.transformer_quant: %s native denoiser(s) read from the local prequant cache "
"for %s; skipping the runtime quantise",
native_scheme,
fam.name,
)
elif (
kind == "pipeline"
and normalize_transformer_quant(transformer_quant) is not None
and dense_transformer_supported(target)
and video_offload
and native_scheme is None
):
# torchao subclasses do not survive offload hooks under inference_mode; int8 has a torchao-free build, others skip.
logger.info(
"video.transformer_quant: skipped (the memory plan offloads the DiT: %s; only "
"int8 runs torchao-free there)",
_video_plan_label(plan),
)
quant_skipped_for_offload = True
transformer_quant_decline = (
f"this GPU's memory plan moves the DiT ({_video_plan_label(plan)}), and "
f"'{normalize_transformer_quant(transformer_quant)}' is built as torchao weights here, "
"which Unsloth keeps on a resident DiT only; int8 runs torchao-free under offload. Pick "
"int8, or a resident memory mode (Fast) on a card that holds the quantised DiT"
)
elif (
kind == "pipeline"
and normalize_transformer_quant(transformer_quant) is not None
and (dense_transformer_supported(target) or native_scheme is not None)
and (source_precision := stored_denoiser_precision(_base_local_dir or repo_id))
is not None
):
# from_pretrained widened a narrow checkpoint to bf16; requantising compounds the loss.
for view in views:
mark_source_precision(view, source_precision)
transformer_quant_decline = (
f"its published weights are {source_precision} and were widened to bf16 on load, so "
"quantising them again would compound that loss"
)
transformer_quant_decline_status = RESOLVED_UNSUPPORTED
logger.info("video.transformer_quant: skipped (%s)", transformer_quant_decline)
elif (
kind == "pipeline"
and normalize_transformer_quant(transformer_quant) is not None
and (dense_transformer_supported(target) or native_scheme is not None)
):
engaged = []
# An offloaded DiT goes back to the host anyway: build every linear on the card, one at a time.
host_quant_device = (
_host_quant_device(target) if video_offload and native_scheme else None
)
for view in views:
# Quantise on the card: on the host int8 packing took 59 s (Wan2.2-5B) to 154 s (LTX-2) of the load.
staged = (
[]
if host_quant_device is not None
else _stage_denoiser_for_quant(
getattr(view, "transformer", None), target, logger = logger
)
)
scheme = None
try:
with quantise_host_linears_on(host_quant_device):
scheme = quantize_transformer(
view,
target,
mode = native_scheme or transformer_quant,
family = fam.name,
logger = logger,
**native_kwargs,
)
finally:
# A declined quant can roll back to the bf16 offload plan, which must not start with dense blocks staged.
if staged and (video_offload or scheme is None):
_unstage_modules(staged)
if scheme is not None:
engaged.append(scheme)
# All experts or none: the first is mutated in place, so a second-expert failure cannot fall back to dense.
if engaged and len(engaged) < len(views):
del pipe
clear_gpu_cache()
raise RuntimeError(
f"transformer_quant={engaged[0]} engaged on only "
f"{len(engaged)}/{len(views)} experts; retry without quant."
)
# A part-converted view is unusable even when the precision fallback is allowed.
dirty = (
[]
if engaged
else [i for i, view in enumerate(views) if transformer_is_quantised(view)]
)
if dirty:
del pipe
clear_gpu_cache()
raise RuntimeError(
f"transformer_quant='{transformer_quant}' converted part of the denoiser and then "
"failed, leaving it neither dense nor usable. Reload with Precision set to Off to "
"run the checkpoint as-is."
)
if engaged:
transformer_quant_engaged = engaged[0]
if (
native_cache_spec is not None
and native_scheme == engaged[0]
and native_cache_spec
== _native_spec_for(native_scheme, fam, native_kwargs.get("act_int8"))
):
# The native build is read back under auto too: the cache replays the pick, never makes it.
if _native_cache_background():
native_writes = _NativeCacheWrites(
pipe,
fam,
views,
native_cache_spec,
base = base,
source = _base_local_dir or fetch_base,
)
else:
_save_native_quant_to_cache(
fam,
views,
native_cache_spec,
base = base,
source = _base_local_dir or fetch_base,
cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),
)
self._raise_if_video_load_superseded(_load_token)
elif transformer_quant_pinned is not None:
_save_runtime_quant_to_cache(
fam,
views,
base = base,
source = _base_local_dir or fetch_base,
cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),
)
self._raise_if_video_load_superseded(_load_token)
# Auto resident build: write after publish so the first render never waits on disk.
elif (
not video_offload
and native_scheme is None
and normalize_transformer_quant(transformer_quant) == TQ_AUTO
and _resident_quant_cache_enabled()
):
_discard_resident_quant_write(self)
pending = _save_runtime_quant_to_cache(
fam,
views,
base = base,
source = _base_local_dir or fetch_base,
cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),
defer = True,
)
if hasattr(pending, "start"):
self._resident_quant_write = (_load_token, pending)
self._raise_if_video_load_superseded(_load_token)
elif transformer_quant_pinned is not None:
transformer_quant_decline = (
f"'{transformer_quant_pinned}' did not engage on family '{fam.name}' with this "
"GPU (the scheme is unsupported here, or the family's measured deny list "
"rules it out); see the server log"
)
transformer_quant_decline_status = RESOLVED_UNSUPPORTED
# The quant-sized plan is valid only when quant engaged; a dense fallback keeps bf16 placement
if quant_replanned and transformer_quant_engaged is None:
plan = bf16_plan
# Fail closed on declined explicit precision; a different engaged scheme only gets a badge report.
if (
transformer_quant_engaged is None
and transformer_quant_pinned is not None
and not precision_fallback_allowed()
):
del pipe
clear_gpu_cache()
raise RuntimeError(
precision_refusal_message(
"transformer_quant",
transformer_quant_pinned,
transformer_quant_decline
or "the quantised DiT build did not engage on this host",
off_label = "Off to run the DiT at bf16",
)
)
# dense text-encoder quant (opt-in): the companion encoder is often the largest resident, so quantise it in
# place before placement, for every kind. Best-effort.
if ltx25_client is not None:
from .video_ltx25 import ltx25_te_outcome
text_encoder_outcome = ltx25_te_outcome(ltx25_client)
else:
from .diffusion_te_truncate import truncate_text_encoders
truncate_text_encoders(pipe, logger = logger)
text_encoder_outcome = quantize_text_encoders(
pipe,
target,
mode = text_encoder_quant,
family = fam.name,
offload_active = plan.offload_policy != "none",
logger = logger,
encoders = (getattr(fam, "te_quant_auto_encoders", None) or None)
if text_encoder_quant_auto
else None,
)
text_encoder_quant_engaged = text_encoder_outcome.mode
if (
(text_encoder_quant_engaged is None or text_encoder_outcome.partial)
and normalize_te_quant(text_encoder_quant) is not None
and not text_encoder_quant_auto
and not precision_fallback_allowed()
):
del pipe
clear_gpu_cache()
raise RuntimeError(
precision_refusal_message(
"text_encoder_quant",
normalize_te_quant(text_encoder_quant) or "",
text_encoder_outcome.reason or "no text encoder could be cast",
off_label = "leave it unset to keep the dense bf16 encoder",
auto_available = False,
)
)
# Order: step cache first (compile keys fullgraph off it), then attention, speed, placement.
effective_speed = resolve_speed_mode(
speed_mode, is_gguf = kind == "gguf", dense_default = SPEED_DEFAULT
)
video_refine = lambda p, pl, lg: _video_refine_for_multigpu(fam, p, pl, lg) # noqa: E731
split_expected = _layer_split_expected(pipe, plan, video_refine)
# A torchao (or torchao-free resident) int8 DiT must be compiled: eager is ~30x slower.
quant_forces_compile = transformer_quant_engaged is not None and (
native_scheme is None
or _native_quant_needs_compile(
target,
native_scheme,
split_expected
or plan.offload_policy == "none"
or _companion_split_expected(pipe, plan, video_refine),
)
)
if quant_forces_compile and effective_speed == SPEED_OFF:
logger.info(
"video.transformer_quant: forcing speed_mode=default "
"(quantized transformer must be compiled; eager is ~30x slower)"
)
effective_speed = SPEED_DEFAULT
# Before the step cache and apply_memory_plan: their hooks capture the block forward when they attach. The
# registered video patches (diffusion_autoopt.patches; the fused fp16 Wan block on a speed tier, uninstalled
# on `off`) run through the same orchestrator as the image backend's.
from . import diffusion_autoopt as autoopt
load_audit = autoopt.new_audit(backend = "video", family = fam)
wan_fused_engaged = bool(
autoopt.install_patches(
autoopt.STAGE_LOAD,
pipe = pipe,
target = target,
family = fam,
dtype = dtype,
device = getattr(target, "device", "cuda"),
speed_mode = effective_speed,
backend = "video",
audit = load_audit,
logger = logger,
).get("wan_fused")
)
# "off"/"fbcache" pinned; unset/"auto" only on max, by FBCACHE_MIN_STEPS. Run per expert.
cache_request = normalize_transformer_cache(transformer_cache)
cache_auto = transformer_cache is None or cache_request == TC_AUTO
cache_auto_live = cache_auto and auto_step_cache_allowed(effective_speed)
# GGUF and torchao-quantised DiTs need the higher threshold to trigger over quant noise
cache_quant_active = kind == "gguf" or transformer_quant_engaged is not None
default_cache_steps: Optional[int] = None
static_plan: Optional[dict] = None
if cache_auto:
default_cache_steps, _ = default_video_generation_params(
variant_id,
gguf_filename,
repo_id,
base,
fallback = (fam.default_steps, fam.default_guidance),
family = fam.name,
)
static_plan = auto_static_skip_plan(
(repo_id, base), skip_tier(speed_mode, effective_speed), default_cache_steps
)
cache_request = resolve_auto_step_cache(
effective_speed, default_cache_steps, static_plan = static_plan
)
cache_engaged = None
static_decline: Optional[str] = None
if cache_request == TC_STATIC and static_plan:
# Auto: only where it was measured, which never lists a two-expert or joint audio-video family.
cache_engaged = install_static_step_skip(
pipe, settings = auto_static_settings(static_plan, logger = logger), logger = logger
)
if cache_engaged is None:
static_plan = None
cache_request = resolve_auto_step_cache(effective_speed, default_cache_steps)
if cache_request == TC_STATIC and not static_plan:
# One denoiser only: a dual-expert MoE hands part of the trajectory to transformer_2, unseen by the history.
if len(views) > 1 or getattr(pipe, "transformer_2", None) is not None:
static_decline = (
"static step skip needs a single denoiser; this family runs two experts "
"(transformer_2), so it runs uncached"
)
logger.warning("video.step_skip: %s", static_decline)
elif getattr(fam, "has_audio", False):
# Engaged on LTX-2's joint (video, audio) denoiser it would report static while every call computed.
static_decline = (
"static step skip reuses one noise prediction per call; this family's denoiser "
"returns joint video and audio predictions, so it runs uncached"
)
logger.warning("video.step_skip: %s", static_decline)
else:
cache_engaged = install_static_step_skip(pipe, logger = logger)
if cache_engaged is None:
static_decline = (
"static step skip is unavailable for this pipeline; it runs uncached"
)
elif cache_request != TC_STATIC:
for view in views:
engaged = apply_step_cache(
view,
mode = cache_request,
threshold = transformer_cache_threshold,
# Quantized residuals are larger: engaged quant and GGUF need the higher FBCache threshold.
quant_active = cache_quant_active,
logger = logger,
)
if view is pipe:
cache_engaged = engaged
cache_graph_break = cache_breaks_graph(cache_engaged)
# Arm only where FBCache can engage: a live toggle drops fullgraph and retries every generation.
cache_may_toggle = (
cache_auto_live
and cache_engaged != TC_STATIC
and (
cache_engaged is not None
or (cache_request is None and step_cache_supported(pipe, logger = logger))
)
)
if cache_auto:
if cache_engaged == TC_STATIC:
cache_reason = (
f"auto: static step skip (every {static_plan['every']}) for this model at "
f"{static_plan['min_steps']}+ steps; set UNSLOTH_DIFFUSION_AUTO_STEP_SKIP=0 to turn it off"
)
elif not cache_auto_live:
# Only name the max tier where it would help: SDXL / LTX-2 never cache on any tier.
cache_reason = (
"auto: step caching engages on the max speed tier only"
if step_cache_supported(pipe, logger = logger)
else "auto: model does not support step caching"
)
elif cache_engaged:
cache_reason = (
f"auto: {default_cache_steps}-step default schedule reaches "
f"{FBCACHE_MIN_STEPS}; re-checked per generation"
)
elif cache_request is not None or not cache_may_toggle:
cache_reason = "auto: model does not support step caching"
else:
cache_reason = (
f"auto: {default_cache_steps}-step default schedule is below "
f"{FBCACHE_MIN_STEPS}; re-checked per generation"
)
else:
cache_reason = static_decline or "requested"
attention_engaged = None
# Hunyuan-1.5 only, once per pipe; before apply_attention_backend so kernel pins; off on SPEED_OFF.
attention_trim_engaged = (
install_hunyuan_attention_trim(pipe, fam, logger = logger)
if effective_speed != SPEED_OFF
else False
)
# Flag read when pipelines first build frequency tables; needed before generation, not placement.
force_float32_rope(pipe, target, logger = logger)
# HunyuanVideo-1.5 VAE only: same mask values, so bit-identical, but held off on SPEED_OFF like the trim above.
hv15_mask_engaged = False
if effective_speed != SPEED_OFF:
from .video_hv15_vae import install_vectorised_causal_mask
hv15_mask_engaged = install_vectorised_causal_mask(pipe, logger = logger) > 0
speed_optims: tuple = ()
speed_offload = plan.offload_policy != "none" and not split_expected
for view in views:
# Helpers act on ``view.transformer``, one call per view; is_gguf: kind==gguf and no quant engaged.
gguf_transformer = kind == "gguf" and transformer_quant_engaged is None
engaged = apply_attention_backend(
view,
select_attention_backend(
target, attention_backend, speed_active = effective_speed != SPEED_OFF
),
logger = logger,
# Probe at the post-promotion dtype: fused SDPA is half-only, fp32 falls to math backend.
target = types.SimpleNamespace(device = target.device, dtype = dtype),
)
# The optimizer's speed layer (diffusion_autoopt), shared with the image backend. Its family hooks are
# what used to be inline here: LTX static compile (token counts depend only on (w, h, frames): one
# recompile per shape, ~1.3x faster) before the primitive, the LTX STG compile adapter and the
# cudnn.benchmark opt-out after it.
applied = autoopt.run_speed_layer(
view,
target,
family = fam,
backend = "video",
speed_mode = effective_speed,
is_gguf = gguf_transformer,
offload_active = speed_offload,
audit = load_audit,
logger = logger,
apply = lambda family, log, view = view: apply_speed_optims(
view,
target,
is_gguf = gguf_transformer,
family = family,
speed_mode = effective_speed,
# An auto cache that could still engage also drops fullgraph (FBCache under a fullgraph-compiled
# DiT crashes)
cache_active = cache_graph_break or cache_may_toggle,
offload_active = speed_offload,
cuda_graph_default = False,
logger = log,
),
)
if view is pipe:
attention_engaged = engaged
speed_optims = tuple(k for k, v in applied.items() if v)
if attention_trim_engaged:
speed_optims += ("hunyuan_attn_trim",)
if hv15_mask_engaged:
speed_optims += ("hv15_vae_vector_mask",)
if wan_fused_engaged:
speed_optims += ("wan_fused_adaln",)
if "compiled" in speed_optims and getattr(target, "device", None) == "cuda":
# The first clip compiles inline: start the workers and hash the keys while the load finishes.
_start_compile_prewarm(logger)
with self._generate_lock:
# A cancelled load must not place weights on a GPU the arbiter may have reassigned.
if _load_token is not None and _load_token != self._load_token:
del pipe
clear_gpu_cache()
raise RuntimeError("Video load was cancelled or superseded.")
# A component larger than the card OOMs at its first whole-module onload; fall back to leaf streaming.
# priced as the denoise phase (_video_prefer_whole_module): the decoded-clip share is the VAE phase's
refined = refine_memory_plan_for_components(
pipe, plan, headroom_mib = _video_denoise_phase_headroom_mib(plan)
)
if refined is not plan:
logger.info("video.memory: %s", refined.reasons[-1])
plan = refined
plan = _video_refine_for_multigpu(fam, pipe, plan, logger)
vram_floor_mib = _video_offload_vram_floor_mib(pipe, plan)
free_mib = _planned_free_less_user_reserve_mib(getattr(plan, "device_memory", None))
if vram_floor_mib is not None and free_mib is not None:
shortfall = video_offload_shortfall_message(
family = fam.name,
floor_mib = vram_floor_mib,
available_mib = int(free_mib),
placement = _video_plan_label(plan),
)
if shortfall is not None:
del pipe
clear_gpu_cache()
raise RuntimeError(shortfall)
if ltx23_direct_loaded:
_return_direct_loaded_modules(pipe, plan, logger)
# the streamed modules pin after the load returns, overlapping the first encode and compile
request_background_pins(pipe)
request_fast_pins(pipe)
offload_policy, vae_tiling = apply_memory_plan(
pipe,
plan,
device = device,
# A bare "cuda" sends CPU-offload hooks to ordinal 0.
placement_device = target.torch_device,
logger = logger,
)
# a transient OOM during setup may have re-planned to lower residency
planned = plan
plan = applied_plan(pipe, plan)
if cache_engaged == TC_FBCACHE and _step_cache_dropped_by_layer_split(pipe):
cache_engaged = _restore_step_cache_after_layer_split(
pipe,
views,
threshold = transformer_cache_threshold,
quant_active = cache_quant_active,
logger = logger,
)
if cache_engaged is None and getattr(plan, "layer_split", None) is not None:
cache_may_toggle = False
cache_reason = "off: the step cache declined the denoiser split across cards"
if (
getattr(planned, "secondary_ordinal", None) is not None
or getattr(planned, "layer_split", None) is not None
) and plan is not planned:
vram_floor_mib = _video_offload_vram_floor_mib(pipe, plan)
ltx25_upsampler = getattr(pipe, "_unsloth_ltx25_upsampler", None)
if ltx25_upsampler is not None and offload_policy == OFFLOAD_NONE:
# ~1 GB; resident with the rest, else render() moves it in for the upsample only.
ltx25_upsampler.to(target.torch_device)
if (
offload_policy == plan.offload_policy
and vram_floor_mib is not None
and free_mib is not None
):
# An encoder that refused leaf offload is kept resident under the same policy: re-check what landed.
applied_floor = _video_offload_vram_floor_mib(pipe, plan, applied = True)
raised = applied_floor is not None and applied_floor > vram_floor_mib
if applied_floor is not None:
# record what landed even when lower (the fallback may stream a DiT the plan kept resident)
vram_floor_mib = applied_floor
if raised:
shortfall = video_offload_shortfall_message(
family = fam.name,
floor_mib = applied_floor,
available_mib = int(free_mib),
placement = _video_plan_label(plan),
)
if shortfall is not None:
del pipe
clear_gpu_cache()
raise RuntimeError(shortfall)
# apply_memory_plan covers every DiT; a second pass would duplicate-hook.
if not vae_tiling:
try:
pipe.vae.enable_tiling()
vae_tiling = True
except Exception as exc: # noqa: BLE001 -- tiling is an optimisation only
logger.warning("video.vae_tiling_failed: %s", exc)
# LTX-2's stock tiles (16 latents, a 2-latent blend) leave seam lines: tiles sized to free VRAM, wide overlaps.
if vae_tiling:
try:
from .video_ltx2_vae_tiles import install as install_ltx2_vae_tiles
if install_ltx2_vae_tiles(getattr(pipe, "vae", None), logger):
speed_optims += ("vae_wide_tiles",)
except Exception as exc: # noqa: BLE001 - keep the stock tiled decode
logger.warning("video.vae_wide_tiles: not installed: %s", exc)
if vae_tiling and fam.name in COSMOS3_FAMILIES:
from .video_cosmos3 import install_untiled_encode
install_untiled_encode(pipe, logger = logger)
if _decodes_untiled_when_fits(fam.name, offload_policy, target.device, effective_speed):
from .video_vae_untiled import install_untiled_decode
if install_untiled_decode(pipe, fam.name, logger = logger):
speed_optims += ("vae_untiled_when_fits",)
# Wan's decode also grows within a single tile, which tiling alone cannot bound.
install_vae_sync(pipe, target, logger = logger)
if _video_denoiser_hooked(pipe):
# Below the offload hooks (as the image backend), whether the speed layer planned the offload or a split
# fell back to a hooked placement: compiled through them, each block splits at the hook's
# torch.compiler.disable'd pre / post forward.
compile_pipe_below_offload_hooks(pipe, logger)
# A family that opts in to graphs arms them against the placement that landed (offload hooks included).
graph_applied = arm_graphs_after_placement(
pipe, {"cuda_graph": "cuda_graph" in speed_optims}, logger
)
if graph_applied.get("cuda_graph") and "cuda_graph" not in speed_optims:
speed_optims += ("cuda_graph",)
elif not graph_applied.get("cuda_graph"):
speed_optims = tuple(o for o in speed_optims if o != "cuda_graph")
if getattr(plan, "layer_split", None) is not None:
split_applied = {k: True for k in speed_optims}
engage_pinned_denoisers(pipe, split_applied, logger)
from . import diffusion_cuda_graph
diffusion_cuda_graph.arm_block_graphs(
pipe,
split_applied,
target = target,
family = fam,
hooked = _video_denoiser_hooked(pipe),
cache_engaged = bool(cache_graph_break),
speed_mode = effective_speed,
family_default = False,
logger = logger,
)
for key in ("int8_gemm", "cuda_graph"):
if split_applied.get(key) and key not in speed_optims:
speed_optims += (key,)
elif not split_applied.get(key):
speed_optims = tuple(o for o in speed_optims if o != key)
# Last, so the bf16 entry wraps whatever decode path the steps above installed. Not on SPEED_OFF.
if getattr(fam, "vae_force_fp32", False) and effective_speed != SPEED_OFF:
vae_bf16_mode = install_rocm_vae_bf16_decode(pipe, target, logger = logger)
if vae_bf16_mode:
speed_optims += ("vae_bf16_decode", f"vae_bf16_{vae_bf16_mode}")
# Offload-first retry on a decode OOM, installed last: its decode scope is the outermost decode layer
# (one recovery session per logical decode) and every inner wrapper that falls back on an OOM (wide
# tiles, untiled-when-fits) frees the card through it first (diffusion_vae_oom.free_first).
try:
from .diffusion_vae_oom import install as install_vae_oom_retry
install_vae_oom_retry(getattr(pipe, "vae", None), logger, pipe = pipe)
except Exception as exc: # noqa: BLE001 - a decode OOM then fails as before
logger.warning("video.vae_oom: not installed: %s", exc)
resolved = build_resolved_record(
{
"family_override": _family_override_resolved(family_override, fam),
"memory_mode": _memory_mode_resolved(memory_mode, plan, offload_policy),
"speed_mode": (
speed_mode,
effective_speed,
"quantized transformer requires compile"
if quant_forces_compile
else "clip denoises amortise the one-time compile within a single run"
if speed_mode is None
else "requested",
),
"attention_backend": (
attention_backend,
attention_engaged or "native",
attention_status_reason(attention_backend, attention_engaged),
),
"transformer_cache": (
None if cache_auto else transformer_cache,
cache_engaged or "off",
cache_reason,
RESOLVED_UNSUPPORTED if static_decline else None,
),
"cuda_graph": (
None,
"on" if "cuda_graph" in speed_optims else "off",
str(
getattr(pipe, "_unsloth_cuda_graph_reason", None)
or "speed tier does not capture"
),
),
"transformer_quant": (
transformer_quant_requested,
transformer_quant_engaged or "off",
# torchao schemes cut load time and memory ~2x, but per-step GEMMs are at best bf16 parity.
transformer_quant_source
or native_quant_reason(
getattr(pipe, "transformer", None), transformer_quant_engaged
)
if transformer_quant_engaged is not None
and (transformer_quant_source or native_scheme is not None)
else "DiT(s) quantised (halves resident weights; hosted checkpoints cut "
"load time; per-step speed is roughly bf16 parity)"
if transformer_quant_engaged is not None
else (
transformer_quant_decline
or (
"skipped: offload moves the DiT, unsupported for torchao "
"tensors; pin a resident memory mode to combine them"
if quant_skipped_for_offload
# Name the GGUF's quant: a BF16 GGUF is dense, so "quantized" would be wrong.
else (
"not engaged (GGUF DiT ("
+ (_gguf_quant_token(gguf_filename) or "GGUF")
+ ") loaded; the file's own weights are used as is)"
)
if kind == "gguf"
else "not engaged (dense bf16 DiT loaded)"
)
),
# Honored when a quant engaged or the ask was "off"; only with the escape hatch for pinned asks.
RESOLVED_APPLIED
if transformer_quant_engaged is not None or transformer_quant_pinned is None
else transformer_quant_decline_status,
),
"text_encoder_quant": (
# The RAW request: an automatic pick must not read as a scheme the caller pinned.
text_encoder_quant_requested,
text_encoder_quant_engaged or "off",
(
f"{text_encoder_outcome.reason or 'engaged'} (selected automatically: the family default)"
if text_encoder_quant_auto and text_encoder_quant_engaged is not None
else (
"not engaged: the automatic hosted pre-cast encoder is unreachable "
f"({te_auto_unavailable}); dense bf16 text encoder loaded"
)
if text_encoder_quant_auto and te_auto_unavailable
else text_encoder_outcome.reason
)
or (
"dense text encoder quantised in place"
if text_encoder_quant_engaged is not None
else "not applicable (no separate text encoder)"
if video_family_has_no_text_encoder(fam)
else "not engaged (dense bf16 text encoder loaded)"
),
text_encoder_outcome.status,
),
}
)
# The optimizer's per-step audit, beside the speed_mode entry (same record as the image backend).
autoopt.record_load_steps(
load_audit,
attention = attention_engaged,
step_cache = cache_engaged,
step_cache_reason = cache_reason,
compile_cache = False,
compile_cache_reason = autoopt.VIDEO_COMPILE_CACHE_DECLINE,
)
autoopt.finalize(load_audit, speed_optims, resolved)
apply_comfy_flow_shift(pipe, getattr(fam, "comfy_flow_shift", None), logger)
apply_scheduler_overrides(pipe, getattr(fam, "scheduler_overrides", ()), logger)
if ltx25_client is not None:
# The worker loaded the Gemma 4 encoder while the DiT loaded; a failure there fails this load.
ltx25_client.wait_ready(self._cancel_event if _load_token is not None else None)
from . import diffusion_prompt_cache
diffusion_prompt_cache.install(
pipe,
identity = {
"family": fam.name,
"repo": str(repo_id),
"base": str(base),
"dtype": str(dtype),
"te_quant": str(text_encoder_quant_engaged),
},
logger = logger,
)
install_cfg_parallel(pipe, offload_policy = offload_policy, logger = logger)
install_cfg_batch(
pipe,
offload_policy = offload_policy,
speed_mode = speed_mode,
transformer_quant = transformer_quant_engaged,
logger = logger,
)
# a render-time replica failure hands the denoiser to CFG batching from the next render
set_cfg_parallel_fallback(
pipe,
# called with the pipeline: capturing it here would keep a rolled-back load alive
functools.partial(
install_cfg_batch,
offload_policy = offload_policy,
speed_mode = speed_mode,
transformer_quant = transformer_quant_engaged,
logger = logger,
),
)
# Opt-in (UNSLOTH_DIFFUSION_BG_COMPILE=1): off by default so the same seed twice repeats.
from . import diffusion_bg_compile as bg_compile
bg_module = (
None
if not bg_compile.load_time_enabled()
else bg_compile.select_module(
pipe,
speed_optims = speed_optims,
default_tier = effective_speed == SPEED_DEFAULT,
quantized = transformer_quant_engaged is not None,
gguf = kind == "gguf",
step_cache = bool(cache_engaged) or bool(cache_may_toggle),
device = getattr(target, "device", device),
backend = getattr(target, "backend", None),
denoiser_hooked = offload_policy != "none" and _video_denoiser_hooked(pipe),
)
)
load_bg_compile = (
bg_compile.arm(
bg_module,
logger = logger,
seed_dir = _video_float_seed_dir(
pipe,
family = fam.name,
dtype = dtype,
quant = transformer_quant_engaged,
attention_backend = attention_engaged,
)[1],
)
if bg_module is not None
else None
)
with self._lock:
if _load_token is not None and _load_token != self._load_token:
if load_bg_compile is not None:
load_bg_compile.close()
load_bg_compile = None
del pipe
clear_gpu_cache()
raise RuntimeError("Video load was cancelled or superseded.")
from .diffusion_nvfp4_install import record_install_reason
record_install_reason(self, *(_nvfp4_install_outcome or (True, None)), device)
self._state = _VideoLoadState(
pipe = pipe,
family = fam,
repo_id = repo_id,
display_repo_id = display_repo_id,
base_repo = base,
fetch_base = fetch_base,
device = device,
gpu_ordinal = target.ordinal,
placed_ordinal = placed_cuda_ordinal(target),
dtype = str(dtype).replace("torch.", ""),
kind = kind,
gguf_filename = gguf_filename,
variant_id = variant_id,
offload_policy = offload_policy,
vae_tiling = vae_tiling,
memory_mode = plan.requested_mode,
speed_mode = effective_speed,
speed_optims = speed_optims,
backend_flags = backend_flags,
attention_backend = attention_engaged,
transformer_cache = cache_engaged,
cache_auto = cache_may_toggle,
cache_quant_active = cache_quant_active,
cache_threshold = transformer_cache_threshold,
transformer_quant = transformer_quant_engaged,
text_encoder_quant = text_encoder_quant_engaged,
vram_floor_mib = (
vram_floor_mib
if offload_policy == plan.offload_policy
else _video_offload_vram_floor_mib(
pipe, dataclasses.replace(plan, offload_policy = offload_policy)
)
),
vram_split_cards = (
_video_split_card_floors(pipe) if offload_policy == OFFLOAD_NONE else None
),
vram_decode_floor_mib = _video_offload_vram_floor_mib(
pipe,
plan
if offload_policy == plan.offload_policy
else dataclasses.replace(plan, offload_policy = offload_policy),
applied = True,
phase = "decode",
),
resolved = resolved,
bg_compile = load_bg_compile,
)
commit_load_budget_fraction("video")
self._precommit_globals = None
pending_encoder = self._ltx25_pending_encoder
if (
ltx25_client is not None
and pending_encoder is not None
and pending_encoder[1] is ltx25_client
):
self._ltx25_pending_encoder = None
logger.info(
"video.loaded: %s (%s, %s, offload=%s, speed=%s, quant=%s)",
repo_id,
fam.name,
kind,
offload_policy,
effective_speed,
transformer_quant_engaged or "off",
)
if fam.is_moe:
defer_late_expert_pins(pipe, (getattr(pipe, "transformer_2", None),), logger)
start_background_pins(pipe)
held = (cast_writes, native_writes, _take_resident_quant_write(self, _load_token))
if any(w is not None for w in held) and _cache_writes_after_render():
holder = _HeldCacheWrites(
pipe, held, busy = self._generate_lock.locked, idle_s = _cache_write_idle_s()
)
setattr(pipe, _HELD_CACHE_WRITES_ATTR, holder)
holder.arm()
logger.info(
"diffusion.prequant_cache: this first load's cache writes wait for its first render"
)
else:
_start_cache_writes(pipe, *held)
return self.status()
@staticmethod
def _raise_on_modular_unified_shortfall(
fam: VideoFamily,
*,
target: Any,
dtype: Any,
device: str,
memory_mode: Optional[str],
scheme: Optional[str],
te_scheme: Optional[str] = None,
device_memory: Optional[DeviceMemory] = None,
) -> None:
"""Refuse H3 modular loads that cannot fit unified memory; best-effort, skipped if no table/device."""
components = getattr(fam, "bf16_components_gb", None)
if not components:
return
import torch
# Same rule as the conventional path: the table is bf16, so an fp32 promotion on an accelerator doubles it.
dtype_scale = 2.0 if device != "cpu" and dtype is torch.float32 else 1.0
transformer_gb, text_encoder_gb, vae_gb = components
# Use measured ~20.3 GB, not 0.55 x 66.3: H3 hosted denoisers are pruned; else fitting loads refuse.
measured = getattr(fam, "prequant_resident_gb", None) if scheme else None
factor = _QUANT_STEADY_FACTOR.get(scheme) if scheme else None
if measured:
transformer_gb = float(measured)
elif factor is not None:
transformer_gb *= factor
# The hosted conditioner stays INT8 in memory; price the same artifact the loader seeds.
text_encoder_gb = h3_te_resident_gb(te_scheme, bf16_gb = text_encoder_gb)
mib_per_gb = 1000.0**3 / (1024.0 * 1024.0)
plan = plan_diffusion_memory(
target = target,
device_memory = (
device_memory
if device_memory is not None
else settled_snapshot_device_memory(target)
),
model_dense_mib = int(
(transformer_gb + text_encoder_gb + vae_gb) * dtype_scale * mib_per_gb
),
runtime_headroom_mib = estimate_video_runtime_mib(
width = fam.resolution_presets[0][0],
height = fam.resolution_presets[0][1],
num_frames = fam.default_num_frames,
),
requested_mode = normalize_memory_mode(memory_mode),
)
raise_on_unified_memory_shortfall(plan, family = getattr(fam, "name", None), logger = logger)
def _load_h3_modular_pipeline(
self,
*,
diffusers: Any,
torch: Any,
fam: VideoFamily,
repo_id: str,
display_repo_id: Optional[str] = None,
base: str,
kind: str,
dtype: Any,
device: str,
hf_token: Optional[str],
memory_mode: Optional[str],
family_override: Optional[str] = None,
transformer_quant: Optional[str] = None,
text_encoder_quant: Optional[str] = None,
speed_mode: Optional[str] = None,
attention_backend: Optional[str] = None,
h3_task: Optional[str] = None,
target: Any = None,
_load_token: Optional[int] = None,
_base_local_dir: Optional[str] = None,
_base_fetch_repo: Optional[str] = None,
_base_fetch_revision: Optional[str] = None,
_h3_auto_denoiser_planned: Optional[str] = None,
_nvfp4_install_outcome: Optional[tuple[bool, str]] = None,
local_files_only: bool = False,
transformer_cache: Optional[str] = None,
comfy_checkpoint: Optional[str] = None,
gguf_filename: Optional[str] = None,
) -> dict[str, Any]:
"""Load MiniMax-H3 through its official Modular Diffusers workflow.
``comfy_checkpoint``: a ComfyUI-quantized denoiser file (int8_convrot / fp8_scaled) seeded in place of the
released or hosted one, its codes kept on Studio's int8 / fp8 runtime.
``transformer_quant`` and ``text_encoder_quant`` are RAW requests, read as the tri-state
``_h3_precision_unset`` / ``_h3_precision_pinned_dense`` describe. Unset takes the hosted
quantized CONDITIONER and the released bfloat16 DENOISER; ``"none"`` pins the released
component either way; an explicit scheme is honored or falls back with the reason
recorded."""
# Component configs and the dense weights are FETCHED from the base's unsloth mirror; ``base`` stays the
# upstream id the prequant / te-prequant tables are keyed on.
fetch_base = _base_fetch_repo or _video_fetch_repo(base, hf_token) or base
# Defence in depth: validate_load_request now refuses a single_file modular pick outright, so this is
# unreachable from the routes. It stays because this method is also reachable directly, and by the time it runs
# the resident pipeline has already been torn down.
if kind != "pipeline" and not (kind == "single_file" and comfy_checkpoint):
raise ValueError("MiniMax-H3 Diffusers loading requires the pipeline artifact.")
comfy_scan = None
if comfy_checkpoint:
from .video_minimax_h3_comfy import h3_comfy_task
comfy_scan = refuse_comfy_quant(comfy_checkpoint)
if comfy_scan is None:
from .diffusion_comfy_quant import ComfyQuantScan
comfy_scan = ComfyQuantScan()
file_task = h3_comfy_task(comfy_checkpoint)
if h3_task and h3_task != file_task:
raise ValueError(
f"'{Path(comfy_checkpoint).name}' is the {file_task} partition, but the load asked for "
f"{h3_task}. Pick the matching checkpoint."
)
h3_task = file_task
umem_target = target if target is not None else self._device_target()
# Kept before the tri-state rewrite; an unset request must not read back as a pin.
transformer_quant_requested = transformer_quant
text_encoder_quant_requested = text_encoder_quant
transformer_quant_is_auto = _h3_precision_unset(transformer_quant)
text_encoder_quant_is_auto = _h3_precision_unset(text_encoder_quant)
# Load one denoiser partition. fl2va also covers text-only generation.
workflow = h3_task or fam.modular_workflow
# Seeding the wrong attribute fetches the dense 66.28 GB partition.
denoiser_component = h3_denoiser_component(workflow)
manager = diffusers.ComponentsManager()
load_kwargs: dict[str, Any] = {
"components_manager": manager,
"cache_dir": hub_cache_dir(),
"local_files_only": local_files_only,
}
if hf_token:
load_kwargs["token"] = hf_token
pipe = diffusers.ModularPipeline.from_pretrained(
_base_local_dir or fetch_base,
**(
{"revision": _base_fetch_revision}
if _base_fetch_revision and not _base_local_dir
else {}
),
**load_kwargs,
)
transformer_quant_engaged: Optional[str] = None
transformer_quant_reason = "released bfloat16 components"
# Auto picks hosted int8 where it exists; memory decides placement, never precision.
scheme = (
None if transformer_quant_is_auto else normalize_transformer_quant(transformer_quant)
)
if scheme == TQ_AUTO: # defence in depth; _h3_precision_unset already caught "auto"
scheme = None
auto_fallback_scheme = None
# A ComfyUI denoiser file IS the precision choice: no hosted checkpoint is picked around it.
comfy_scheme = None
if comfy_scan is not None:
counts = comfy_scan.counts()
comfy_scheme = (
TQ_INT8 if any(k.startswith("int8") for k in counts) else TQ_FP8 if counts else None
)
if _h3_precision_pinned_dense(transformer_quant):
comfy_scheme = None
scheme = None
if transformer_quant_is_auto and comfy_scan is None:
# Skip if the plan already acted, since the dense shards are gone and cannot be re-
# decided safely.
unreadable_blocked: list = []
auto_fallback_scheme = _h3_auto_denoiser_planned or _h3_auto_denoiser_scheme(
fam,
target = umem_target,
dtype = dtype,
device = device,
# Conditioner precision this load WILL engage (predicted as below), since it must fit beside it.
te_scheme = self._h3_te_quant_scheme(fam, text_encoder_quant, base, umem_target),
task = workflow,
base_repo = base,
speed_mode = speed_mode,
on_unreadable = lambda: unreadable_blocked.append(True),
)
if auto_fallback_scheme is None and unreadable_blocked:
from .diffusion_prequant import prequant_unreadable_reason
unreadable = prequant_unreadable_reason(
fam, H3_AUTO_FALLBACK_SCHEME, base_repo = base, task = workflow
) or (
f"the hosted {H3_AUTO_FALLBACK_SCHEME} checkpoint cannot be read by this install"
)
transformer_quant_reason = f"released bfloat16 components ({unreadable})"
logger.warning("video.transformer_quant: %s", transformer_quant_reason)
if auto_fallback_scheme is not None:
scheme = auto_fallback_scheme
logger.info(
"video.transformer_quant: auto is taking the hosted %s denoiser checkpoint "
"(ask for transformer_quant='none' to keep the released weights)",
auto_fallback_scheme,
)
# Per (scheme, partition): partitions look alike once loaded; a miss goes dense, never the wrong one.
if scheme is not None and not video_family_prequant_available(
fam, scheme, task = workflow, base_repo = base
):
transformer_quant_reason = (
f"no hosted pre-quantized {scheme} checkpoint for {fam.name} {workflow}"
)
scheme = None
te_scheme = self._h3_te_quant_scheme(fam, text_encoder_quant, base, umem_target)
te_index_source = self._h3_te_index_source(pipe) if te_scheme is not None else None
te_index_declined = False
if te_scheme is not None:
if te_index_source is None or _h3_te_canonical(te_index_source) != _h3_te_canonical(
fam.base_repo
):
te_scheme = None
te_index_declined = True
# Keep the pre-seed free pool: re-reading after seeding double-counts hosted components.
preseed_memory = settled_snapshot_device_memory(umem_target)
self._raise_on_modular_unified_shortfall(
fam,
target = umem_target,
dtype = dtype,
device = device,
memory_mode = memory_mode,
scheme = comfy_scheme or scheme,
te_scheme = te_scheme,
device_memory = preseed_memory,
)
if comfy_scan is not None:
from .video_minimax_h3_comfy import load_h3_comfy_transformer
from .video_minimax_h3_native_quant import h3_comfy_backends
# torchao where it runs, else the torchao-free twin (fp8 stored at 1 byte where no fp8 GEMM runs).
dense_pin = _h3_precision_pinned_dense(transformer_quant_requested)
h3_backends = {"int8_backend": None, "fp8_backend": None, "fp8_storage": False}
if not dense_pin:
h3_backends = h3_comfy_backends(umem_target, base)
if (
comfy_scheme is not None
and not dense_pin
and not (
h3_backends["int8_backend"]
if comfy_scheme == TQ_INT8
else h3_backends["fp8_backend"] or h3_backends["fp8_storage"]
)
):
raise RuntimeError(
f"{Path(comfy_checkpoint).name} needs Unsloth's {comfy_scheme} runtime, which this "
f"device does not run; dequantized, the denoiser would be the 40 GB bfloat16 model. "
f"Load '{fam.base_repo}' instead."
)
transformer = load_h3_comfy_transformer(
getattr(diffusers, fam.transformer_class),
comfy_checkpoint,
comfy_scan,
config = _base_local_dir or fetch_base,
subfolder = denoiser_component,
dtype = dtype,
hf_token = hf_token,
cache_dir = hub_cache_dir(),
local_files_only = local_files_only,
target = umem_target,
logger = logger,
**h3_backends,
)
comfy_info = getattr(transformer, "_unsloth_comfy_quant", None) or {}
engaged = (
TQ_INT8
if comfy_info.get("int8")
else TQ_FP8
if comfy_info.get("fp8") or comfy_info.get("fp8_stored")
else None
)
if engaged is None and comfy_scheme is not None and not dense_pin:
del transformer
raise RuntimeError(
f"{Path(comfy_checkpoint).name} needs Unsloth's {comfy_scheme} runtime, which this "
f"device does not run; dequantized, the denoiser would be the 40 GB bfloat16 model. "
f"Load '{fam.base_repo}' instead."
)
transformer.requires_grad_(False)
# Seeded before load_components, so the released denoiser is never fetched (as for a hosted one).
pipe.update_components(**{denoiser_component: transformer})
transformer_quant_engaged = engaged
transformer_quant_reason = (
f"ComfyUI-quantized denoiser ({Path(comfy_checkpoint).name}): "
f"{comfy_info.get('int8', 0)} int8 / {comfy_info.get('fp8', 0)} fp8 layers kept, "
f"{comfy_info.get('fp8_stored', 0)} fp8 stored, "
f"{comfy_info.get('dequantized', 0)} dequantized"
if engaged is not None
else f"ComfyUI denoiser ({Path(comfy_checkpoint).name}) at bfloat16"
)
logger.info("video.transformer_quant: %s", transformer_quant_reason)
if scheme is not None:
from .diffusion_prequant import load_prequantized_transformer, resolve_prequant_source
from .diffusion_transformer_quant import DEFAULT_MIN_LINEAR_FEATURES
from .video_minimax_h3_adaln import h3_prepare_prequant_model
source = resolve_prequant_source(fam, scheme, base_repo = base, task = workflow)
if source is None:
transformer_quant_reason = (
f"no hosted pre-quantized {scheme} checkpoint for {fam.name}"
)
else:
transformer = load_prequantized_transformer(
getattr(diffusers, fam.transformer_class),
fetch_base,
source,
# enable_auto_cpu_offload owns placement; pre-placing a 66 GB denoiser OOMs.
device = "cpu",
dtype = dtype,
hf_token = hf_token,
scheme = scheme,
# Same Linear filter as runtime quant, so prequant and runtime-quant are the same model.
min_features = DEFAULT_MIN_LINEAR_FEATURES,
# Auto is re-decided here against live memory, so this fetch may be unplanned; stay offline.
local_files_only = local_files_only,
# Pin live cache root: unset, fetches use huggingface_hub import-time path; moves re-download GBs.
cache_dir = hub_cache_dir(),
# Hosted H3 denoisers carry a pruned curve-form adaLN; reshape or the strict load fails.
prepare_model = h3_prepare_prequant_model(logger),
# Config from this partition: reference loads stage transformer_ref/; else offline loads hit Hub.
config_subfolder = denoiser_component,
logger = logger,
host_copy_follows = _h3_host_copy_follows(device),
)
if transformer is None:
# Best-effort: missing, mismatched or unreadable checkpoints fall back to dense, not after teardown.
from .diffusion_prequant import (
last_prequant_failure,
prequant_unreadable_reason,
)
why = (
prequant_unreadable_reason(fam, scheme, base_repo = base, task = workflow)
or last_prequant_failure()
)
transformer_quant_reason = (
f"hosted pre-quantized {scheme} checkpoint unusable"
+ (f" ({why})" if why else "")
+ "; loaded the released bfloat16 denoiser instead"
)
else:
if _h3_hosted_fp8_native(fam, umem_target, scheme):
from .video_minimax_h3_native_quant import h3_hosted_fp8_to_native
h3_hosted_fp8_to_native(transformer, logger = logger)
# Must seed before load_components, which skips set attributes (avoids the 66.3 GB fetch).
pipe.update_components(**{denoiser_component: transformer})
transformer_quant_engaged = scheme
transformer_quant_reason = (
f"hosted pre-quantized {scheme} denoiser checkpoint ({source.location})"
if auto_fallback_scheme is None
else (
f"auto: hosted pre-quantized {scheme} denoiser checkpoint "
f"({source.location}); transformer_quant='none' keeps the released weights"
)
)
logger.info(
"video.transformer_quant: %s pre-quantized denoiser seeded for %s",
scheme,
fam.name,
)
# Seed the hosted conditioner: a runtime cast peaks at bf16 and would download 62 GB dense.
text_encoder_quant_engaged: Optional[str] = None
text_encoder_quant_reason = "released bfloat16 components"
text_encoder_quant = (
None if text_encoder_quant_is_auto else normalize_te_quant(text_encoder_quant)
)
if text_encoder_quant_is_auto and te_scheme is not None:
text_encoder_quant_reason = f"auto: hosted quantized {te_scheme} conditioner"
if text_encoder_quant is not None and te_scheme is None:
# Valid request, no hosted artifact for this workflow or conditioner: say which; keep the request.
if te_index_declined:
text_encoder_quant_reason = (
f"this pipeline's conditioner comes from "
f"{te_index_source or 'a source that could not be read'}, not {fam.base_repo}, "
f"so the hosted quantized {text_encoder_quant} conditioner is not its "
f"quantization; loaded this pipeline's own bfloat16 encoder instead"
)
elif h3_te_quant_scheme(text_encoder_quant) is not None:
text_encoder_quant_reason = (
f"the hosted quantized {text_encoder_quant} conditioner is cut from "
f"{fam.base_repo}, not {base}; loaded this pipeline's own bfloat16 encoder "
f"instead"
)
else:
text_encoder_quant_reason = (
f"no hosted quantized {text_encoder_quant} conditioner for {fam.name}; "
f"loaded the released bfloat16 encoder instead"
)
elif text_encoder_quant_is_auto and te_scheme is None:
# An UNSET request never fails a load, but must still say why the fast default did not engage.
text_encoder_quant_reason = (
f"auto declined: this pipeline's conditioner comes from "
f"{te_index_source or 'a source that could not be read'}, not {fam.base_repo}"
if te_index_declined
else "auto: no hosted quantized conditioner applies here; loaded the released "
"bfloat16 encoder"
)
if te_scheme is not None:
from .video_minimax_h3_te import load_h3_quantized_text_encoder
text_encoder = load_h3_quantized_text_encoder(
fetch_base,
te_scheme,
dtype = dtype,
hf_token = hf_token,
cache_dir = hub_cache_dir(),
# Scoped pre-download keeps text_encoder/config.json; read it from the staged snapshot, not the hub.
local_base = _base_local_dir,
# The staging fetch accepted either cache root; this lookup searches only the live one.
local_files_only = local_files_only,
logger = logger,
)
if text_encoder is None:
# Best-effort like the denoiser: missing, unreadable or mismatched artifacts stay dense, not fail.
text_encoder_quant_reason = (
f"hosted quantized {te_scheme} conditioner unusable; "
f"loaded the released bfloat16 encoder instead"
)
else:
pipe.update_components(text_encoder = text_encoder)
text_encoder_quant_engaged = te_scheme
text_encoder_quant_reason = (
f"hosted quantized {te_scheme} conditioner ({h3_te_quant_source(te_scheme)})"
)
logger.info(
"video.text_encoder_quant: %s quantized conditioner seeded for %s",
te_scheme,
fam.name,
)
# Encoder precision engaging none fails before load_components; escape: precision_fallback_allowed()
if (
text_encoder_quant is not None
and text_encoder_quant_engaged is None
and not precision_fallback_allowed()
):
raise RuntimeError(
precision_refusal_message(
"text_encoder_quant",
text_encoder_quant,
text_encoder_quant_reason,
off_label = "leave it unset to keep the dense bf16 encoder",
auto_available = False,
)
)
# Pass the token again or gated components load anonymously; workflow= narrows names, not blocks.
if (scheme is not None and transformer_quant_engaged is None) or (
te_scheme is not None and text_encoder_quant_engaged is None
):
# Either hosted component can fall back to dense; re-check the components that actually landed.
self._raise_on_modular_unified_shortfall(
fam,
target = umem_target,
dtype = dtype,
device = device,
memory_mode = memory_mode,
scheme = transformer_quant_engaged,
te_scheme = text_encoder_quant_engaged,
device_memory = preseed_memory,
)
# Without cache_dir components resolve against the import-time HF_HUB_CACHE.
from .diffusion_offline_source import offline_component_sources
offline_sources = offline_component_sources(
pipe, local_files_only = local_files_only, cache_dir = hub_cache_dir()
)
# The index is byte identical, so every spec still names the upstream id; load_components would read those
# from the vendor repo under a different cache key than the mirror the base was staged into.
mirrored_sources = _h3_mirrored_component_sources(
pipe,
base,
fetch_base,
local_files_only = local_files_only,
cache_dir = hub_cache_dir(),
staged_dir = _base_local_dir
if _base_local_dir and not _is_under_dir(_base_local_dir, hub_cache_dir())
else None,
)
offline_sources = {**offline_sources, **mirrored_sources}
# The redirected components read the plan's commit of the staged repo (a snapshot folder needs none).
component_revisions = {
name: _base_fetch_revision
for name, source in mirrored_sources.items()
if _base_fetch_revision and source == fetch_base
}
pipe.load_components(
workflow = workflow,
dtype = dtype,
cache_dir = hub_cache_dir(),
local_files_only = local_files_only,
**({"token": hf_token} if hf_token else {}),
**({"pretrained_model_name_or_path": offline_sources} if offline_sources else {}),
**({"revision": component_revisions} if component_revisions else {}),
)
# Before any offload hook wraps the denoiser's forward (the hooks then wrap this one).
from .video_minimax_h3_forward import install_h3_forward_on_pipe
install_h3_forward_on_pipe(pipe, logger = logger)
# Pre-cast the decoder: fp32 VAE under fp16 autocast keeps both copies; encoder stays for keyframes.
from .video_minimax_h3 import trim_h3_video_vae
try:
trimmed = trim_h3_video_vae(getattr(pipe, "vae", None), workflow = workflow)
except Exception as exc: # noqa: BLE001 -- a saving is not worth failing a load over
logger.warning("video.h3_vae_trim failed, keeping the full VAE: %s", exc)
else:
if trimmed["encoder_freed"] or trimmed["decoder_freed"]:
logger.info(
"video.h3_vae_trim: freed %.2f GB (encoder %.2f, decoder pre-cast %.2f)",
(trimmed["encoder_freed"] + trimmed["decoder_freed"]) / 1_000_000_000,
trimmed["encoder_freed"] / 1_000_000_000,
trimmed["decoder_freed"] / 1_000_000_000,
)
offload_policy = "none"
denoiser_pinned = False
dense_compile_held = False
denoiser_streamed: Optional[str] = None
denoiser_single_host_copy = False
te_streamed: Optional[str] = None
block_pinner = None
residency = None
# Token fence before placement, or a superseded worker loads 66.3 GB onto another load's GPU.
if _load_token is not None and _load_token != self._load_token:
del pipe
clear_gpu_cache()
raise RuntimeError("Video load was cancelled or superseded.")
# Before placement: the dense pin must know about speed "off" to decline it.
effective_speed = resolve_speed_mode(speed_mode, is_gguf = False, dense_default = SPEED_DEFAULT)
h3_vae_speed = effective_speed
if effective_speed == SPEED_MAX:
# max-autotune: 2.41 vs 2.45 s/step but 110 vs 48 s first render (~45 renders to break even).
logger.info(
"video.speed_mode: MiniMax-H3 runs the 'default' regional profile under max "
"(max-autotune's first-render cost is not repaid by its 2% faster step)"
)
effective_speed = SPEED_DEFAULT
if device != "cpu":
# Pin and size the partition this workflow opened; reference runs denoise from transformer_ref.
denoiser = getattr(pipe, denoiser_component, None)
# Asked before installing offload; the size already includes every other component.
whole_set_sizes = _h3_dense_denoiser_resident_bytes(
fam, denoiser = denoiser, te_scheme = text_encoder_quant_engaged, dtype = dtype
)
free_before_placement = _h3_free_device_bytes(device)
# Headroom is for the default frame count; speed "off" keeps the rotation.
whole_set_resident = effective_speed != SPEED_OFF and _h3_dense_denoiser_fits(
whole_set_sizes, free_before_placement
)
if whole_set_resident:
# If it fits, skip enable_auto_cpu_offload (host RAM); pipe.to is safe at load, not mid-block.
pipe.to(device)
offload_policy = "none"
# Resident and hookless, which is the condition the regional compile needs.
denoiser_pinned = denoiser is not None
logger.info(
"video.h3_placement: the whole component set fits on this device (%.1f GB "
"against %.1f GB free), so every component stays resident and the CPU-offload "
"rotation is not installed",
sum(whole_set_sizes) / 1e9,
(free_before_placement or 0) / 1e9,
)
else:
manager.enable_auto_cpu_offload(
# Measured H3 activations exceed the official 12 GB example; 40 GB keeps big GPUs from OOMing.
device = device,
memory_reserve_margin = "40GB",
)
offload_policy = "model"
if transformer_quant_engaged:
# A torchao denoiser does not survive pre_forward moves: pin if it fits, else stream.
from .diffusion_prequant import (
pin_prequantized_module,
stream_prequantized_module,
)
pinned_sizes = _h3_dense_denoiser_resident_bytes(
fam,
denoiser = denoiser,
te_scheme = text_encoder_quant_engaged,
dtype = dtype,
rotating = True,
)
stream = (
denoiser is not None
and _h3_placement_tier(
quantised = True,
whole_set_sizes = None,
pinned_sizes = pinned_sizes,
free_bytes = free_before_placement,
speed_off = False,
)
== "stream"
)
if text_encoder_quant_engaged is not None:
# Stream the conditioner, not rotate it; denoiser sizing assumes rotation; None keeps rotation.
from .video_minimax_h3_te import h3_te_pin_yields, stream_h3_text_encoder
te_module = getattr(pipe, "text_encoder", None)
te_streamed = stream_h3_text_encoder(
manager,
te_module,
device,
pin = h3_te_pin_yields(
te_module, denoiser if stream else None, device, logger = logger
),
logger = logger,
)
pin_deferred_bytes = (
_h3_resident_pin_deferral(
fam,
denoiser,
device,
te_streamed = bool(te_streamed),
te_scheme = text_encoder_quant_engaged,
)
if stream
else None
)
placed = (
stream_prequantized_module(
manager,
denoiser,
device,
logger = logger,
**({"defer_pin": True} if pin_deferred_bytes is not None else {}),
)
if stream
else None
)
if stream and placed not in _H3_STREAM_MODES:
# Pinning is the tier the planner already ruled out, so it would OOM or be refused later.
raise RuntimeError(
f"MiniMax-H3's {transformer_quant_engaged} denoiser has to stream block by block "
f"on this GPU ({free_before_placement / 1e9:.1f} GB free, "
f"{sum(pinned_sizes) / 1e9:.1f} GB needed pinned), but group offloading is not "
"available in this install. Free GPU memory or pick the GGUF version."
)
if placed in _H3_STREAM_MODES:
# offload_policy stays "model": the post-generation host reclaim keys on it.
denoiser_streamed = placed
denoiser_single_host_copy = (
getattr(denoiser, "_unsloth_pin_arena_bytes", None) is not None
)
logger.info(
"video.h3_placement: %.1f GB free is under the %.1f GB a pinned %s "
"denoiser plus the larger rotating component need, so it streams "
"block by block",
free_before_placement / 1e9,
sum(pinned_sizes) / 1e9,
transformer_quant_engaged,
)
# Residency sized for the largest preset at the default length; re-fitted per request.
from .video_minimax_h3_residency import (
H3Residency,
h3_dit_resident_enabled,
)
pin_deferred = False
pin_deferred_budget = 0
if denoiser_streamed in ("stream", "stream_lazy"):
from .video_minimax_h3_residency import pin_streamed_top_level_group
try:
pin_streamed_top_level_group(denoiser, logger = logger)
except Exception as exc: # noqa: BLE001 -- a speed-up only
logger.warning("video.h3_top_group: %s", exc)
block_pinner = None
if (
pin_deferred_bytes is not None
and denoiser_streamed == "stream_lazy"
):
from .video_minimax_h3_residency import defer_resident_block_pins
try:
residency = H3Residency(denoiser, device, logger = logger)
big_w, big_h = max(
fam.resolution_presets, key = lambda wh: wh[0] * wh[1]
)
pin_deferred_budget = _h3_residency_budget_bytes(
residency,
device,
width = big_w,
height = big_h,
frames = fam.default_num_frames,
te_streamed = bool(te_streamed),
te_scheme = text_encoder_quant_engaged,
)
pin_deferred = defer_resident_block_pins(
residency, pin_deferred_budget, logger = logger
)
except Exception as exc: # noqa: BLE001 -- pin as the lazy path does
logger.warning("video.h3_block_pin: %s", exc)
pin_deferred = False
if pin_deferred:
# every block is promoted and released by the fit below
denoiser_streamed = "stream"
denoiser_single_host_copy = True
else:
residency = None
if denoiser_streamed == "stream_lazy":
from .video_minimax_h3_residency import (
defer_streamed_block_pins,
pin_streamed_block_groups,
)
try:
# the 9-12 s slab copy, off the load path
block_pinner = defer_streamed_block_pins(
pipe, denoiser, device, logger = logger
)
except Exception as exc: # noqa: BLE001 -- pin up front instead
logger.warning("video.h3_block_pin: %s", exc)
if block_pinner is not None:
denoiser_streamed = "stream"
denoiser_single_host_copy = True
else:
try:
done, total = pin_streamed_block_groups(
denoiser, logger = logger
)
except Exception as exc: # noqa: BLE001 -- a speed-up only
done, total = 0, 0
logger.warning("video.h3_block_pin: %s", exc)
if total and done == total:
# every group now holds one pinned copy: the prefetch below engages
denoiser_streamed = "stream"
denoiser_single_host_copy = True
# after the top-level pin (it becomes one of the prefetched groups), before the residency fit
from .video_minimax_h3_residency import install_h3_stream_prefetch
covered = 0
try:
covered = install_h3_stream_prefetch(
denoiser, device, logger = logger
)
except Exception as exc: # noqa: BLE001 -- a speed-up only
logger.warning("video.h3_stream_prefetch: %s", exc)
if block_pinner is not None and not covered:
# Only the prefetcher's onload is sure to wait for a group's pin: finish it now.
block_pinner.join()
if h3_dit_resident_enabled() and denoiser_streamed in (
"stream",
"stream_lazy",
):
try:
if not pin_deferred:
residency = H3Residency(denoiser, device, logger = logger)
if residency.usable:
from .video_minimax_h3_te import yield_h3_te_resident
_te_for_yield = getattr(pipe, "text_encoder", None)
residency.host_yield = (
lambda nbytes, _te = _te_for_yield: yield_h3_te_resident(
_te, nbytes, logger = logger
)
)
from .video_minimax_h3_te import top_up_h3_te_resident
residency.host_top_up = (
lambda _te = _te_for_yield: top_up_h3_te_resident(
_te, logger = logger
)
)
from .video_minimax_h3_te import h3_te_device_share
residency.te_share = h3_te_device_share(
_te_for_yield, device, te_streamed, logger = logger
)
big_w, big_h = max(
fam.resolution_presets, key = lambda wh: wh[0] * wh[1]
)
residency.fit(
pin_deferred_budget
if pin_deferred
else _h3_residency_budget_bytes(
residency,
device,
width = big_w,
height = big_h,
frames = fam.default_num_frames,
te_streamed = bool(te_streamed),
te_scheme = text_encoder_quant_engaged,
),
initial = True,
)
else:
residency = None
except Exception as exc: # noqa: BLE001 -- residency is a speed-up, stay streamed
logger.warning(
"video.h3_residency: staying fully streamed: %s", exc
)
if residency is not None:
# A partial fit left resident groups no controller would ever demote.
from .video_minimax_h3_residency import release_all
release_all(residency, logger = logger)
residency = None
if pin_deferred:
from .video_minimax_h3_residency import pin_deferred_block_groups
try:
pin_deferred_block_groups(denoiser, logger = logger)
except Exception as exc: # noqa: BLE001 -- they pin on each onload
logger.warning("video.h3_block_pin: %s", exc)
else:
denoiser_pinned = pin_prequantized_module(
manager, denoiser, device, logger = logger
)
elif denoiser is not None and effective_speed != SPEED_OFF:
# Pin the released bf16 denoiser for speed (moving modules cannot be compiled); speed=off skips it.
sizes = _h3_dense_denoiser_resident_bytes(
fam,
denoiser = denoiser,
te_scheme = text_encoder_quant_engaged,
dtype = dtype,
rotating = True,
)
free_bytes = None
if sizes is not None:
try:
free_bytes = _h3_free_device_bytes(device)
except Exception: # noqa: BLE001 -- unreadable free memory keeps the rotation
free_bytes = None
if sizes is not None and free_bytes is not None:
from .video_minimax_h3_te import h3_te_stream_enabled
streamed_sizes = (
_h3_dense_denoiser_resident_bytes(
fam,
denoiser = denoiser,
te_scheme = text_encoder_quant_engaged,
dtype = dtype,
rotating = True,
te_streamed = True,
)
if h3_te_stream_enabled() and _h3_group_offload_available()
else None
)
pin_plan = _h3_dense_pin_plan(sizes, streamed_sizes, free_bytes)
from .diffusion_prequant import pin_prequantized_module
if pin_plan == "pin_stream_te":
from .video_minimax_h3_te import stream_h3_text_encoder
# Denoiser first: pinning the conditioner first evicts its cached pages (66 GB re-read).
denoiser_pinned = pin_prequantized_module(
manager,
denoiser,
device,
logger = logger,
label = "released bfloat16 denoiser",
)
te_streamed = stream_h3_text_encoder(
manager,
getattr(pipe, "text_encoder", None),
device,
logger = logger,
)
if te_streamed:
sizes = streamed_sizes
else:
# Sized for the stream: beside a rotating conditioner it OOMs at the encode.
_h3_dense_unpin(manager, denoiser, device, logger = logger)
denoiser_pinned = False
pin_plan = None
elif pin_plan == "pin":
denoiser_pinned = pin_prequantized_module(
manager,
denoiser,
device,
logger = logger,
label = "released bfloat16 denoiser",
)
denoiser_bytes, others_bytes = sizes
if pin_plan is not None:
if te_streamed:
# Compile is not bit-identical here (G4: LPIPS 0.115 over 20 steps): stay eager.
dense_compile_held = bool(denoiser_pinned)
logger.info(
"video.h3_denoiser_placement: %.1f GB free holds the denoiser beside the "
"streamed conditioner (%.1f GB), not beside a rotating one, so it stays "
"resident",
free_bytes / 1e9,
(denoiser_bytes + others_bytes) / 1e9,
)
else:
logger.info(
"video.h3_denoiser_placement: %.1f GB free is under the %.1f GB the "
"denoiser plus the other components need, so it stays in the offload "
"rotation (and is not compiled)",
free_bytes / 1e9,
(denoiser_bytes + others_bytes) / 1e9,
)
# Speed layer applied here after offload (compile needs denoiser_pinned); not in load_pipeline.
if effective_speed in (SPEED_DEFAULT, SPEED_MAX) and dense_compile_held:
logger.info(
"video.speed_mode: the released bf16 denoiser is resident beside the streamed "
"conditioner; the regional compile is held off so the output matches the rotation "
"it replaces; keeping the lossless optimisations"
)
effective_speed = SPEED_EAGER
elif effective_speed in (SPEED_DEFAULT, SPEED_MAX) and not (
denoiser_pinned or denoiser_streamed
):
# Compile only a resident or streamed denoiser; in the rotation the graph fights the onload hooks.
logger.info(
"video.speed_mode: MiniMax-H3 denoiser is in the CPU-offload rotation, so the "
"regional compile is held off (measured slower than eager there); keeping the "
"lossless optimisations"
)
effective_speed = SPEED_EAGER
backend_flags = snapshot_backend_flags()
# Until the state commit, a failure has to restore these process-wide flags itself.
self._precommit_globals = (_load_token, backend_flags)
attention_engaged = None
speed_optims: tuple = ()
from . import diffusion_autoopt as autoopt
h3_audit = autoopt.new_audit(backend = "video", family = fam)
# Both helpers reach the denoiser through ``pipe.transformer``, and a reference load's DiT is
# ``transformer_ref``. Handed the bare pipe they would enumerate no denoiser at all, so the partition this run
# denoises with would keep native attention and stay uncompiled while the resolved record below still reported
# the requested profile -- and the pin above is what removes the eager downgrade that used to hide it.
speed_view = _denoiser_view(pipe, denoiser_component)
try:
from .video_minimax_h3_vae import reserve_h3_fast_decoder
reserve_h3_fast_decoder(
getattr(pipe, "vae", None), speed_mode = h3_vae_speed, workflow = workflow
)
except Exception: # noqa: BLE001 -- optimisation only, never fail a load
pass
h3_attn_levers: tuple = ()
try:
from .video_minimax_h3_attn import (
fuse_h3_qkv_,
h3_attention_backend,
install_strided_attention,
)
h3_denoiser = getattr(pipe, denoiser_component, None)
# Resident only: a streamed denoiser's offload hooks are keyed to the three projections.
if (
effective_speed != SPEED_OFF
and denoiser_pinned
and not denoiser_streamed
and transformer_quant_engaged == "int8"
and h3_denoiser is not None
and fuse_h3_qkv_(h3_denoiser, logger = logger)
):
h3_attn_levers += ("h3_fused_qkv",)
except Exception as exc: # noqa: BLE001 -- optimisation only, the stock projections stay
logger.warning("video.h3_attn: QKV fusion skipped: %s", exc)
try:
h3_attn_backend = select_attention_backend(
umem_target, attention_backend, speed_active = effective_speed != SPEED_OFF
)
try:
h3_attn_backend = h3_attention_backend(
h3_attn_backend,
requested = attention_backend,
target = types.SimpleNamespace(device = device, dtype = dtype),
)
except Exception: # noqa: BLE001 -- keep the generic pick
pass
attention_engaged = apply_attention_backend(
speed_view,
h3_attn_backend,
logger = logger,
# The probe behind this must see the dtype the pipeline actually RUNS in.
target = types.SimpleNamespace(device = device, dtype = dtype),
)
# speed off stays bit-identical: the fused q/k norm + RoPE only matches stock within a rounding step
if effective_speed != SPEED_OFF and attention_engaged in (
"_native_cudnn",
"_native_flash",
):
try:
if install_strided_attention(
getattr(pipe, denoiser_component, None), logger = logger
):
h3_attn_levers += ("h3_strided_attn",)
except Exception as exc: # noqa: BLE001 -- stock processor stays
logger.warning("video.h3_attn: strided attention skipped: %s", exc)
# FlashInfer SM120 H3 block kernels (opt-in, UNSLOTH_H3_FLASHINFER=1): resident fp8 / nvfp4 blocks, before
# the regional compile traces them
if effective_speed != SPEED_OFF:
try:
from .video_minimax_h3_flashinfer import install_h3_flashinfer
if install_h3_flashinfer(
getattr(pipe, denoiser_component, None),
resident = bool(denoiser_pinned) and not denoiser_streamed,
logger = logger,
).get("blocks"):
h3_attn_levers += ("h3_flashinfer",)
except Exception as exc: # noqa: BLE001 -- optimisation only, the stock blocks stay
logger.warning("video.h3_flashinfer: skipped: %s", exc)
h3_speed_target = types.SimpleNamespace(
device = device,
dtype = dtype,
# ROCm reports device "cuda"; the graph arm refuses it by backend, so keep the field.
backend = getattr(umem_target, "backend", "cuda"),
supports_default_torch_compile = getattr(
umem_target, "supports_default_torch_compile", False
),
)
# Same orchestrator as the image backend and the other video families (diffusion_autoopt).
applied = autoopt.run_speed_layer(
speed_view,
h3_speed_target,
family = fam,
backend = "video",
speed_mode = effective_speed,
offload_active = offload_policy != "none",
audit = h3_audit,
logger = logger,
apply = lambda family, log: apply_speed_optims(
speed_view,
types.SimpleNamespace(
device = device,
dtype = dtype,
# ROCm reports device "cuda"; the graph arm refuses it by backend, so keep the field.
backend = getattr(umem_target, "backend", "cuda"),
supports_default_torch_compile = getattr(
umem_target, "supports_default_torch_compile", False
),
),
is_gguf = False,
family = family,
speed_mode = effective_speed,
cache_active = False,
# The conditioner and the VAEs stay in the rotation even when the denoiser is pinned, so the
# onload hooks are live and fullgraph has to drop.
offload_active = offload_policy != "none",
# a denoiser pinned under the others' offload rotation never moves, so the fused int8 GEMM can
# engage
denoiser_offloaded = bool(denoiser_streamed) or not denoiser_pinned,
cuda_graph_default = False,
logger = log,
),
)
speed_optims = tuple(k for k, v in applied.items() if v) + h3_attn_levers
if denoiser_streamed and "compiled" in speed_optims:
# before the first forward traces the blocks below their hooks
try:
from .diffusion_int8_gemm import install as install_int8_gemm
if install_int8_gemm(
getattr(pipe, denoiser_component, None), logger, device = device
):
speed_optims += ("int8_gemm",)
except Exception as exc: # noqa: BLE001 -- optimisation only, the stock GEMM stays
logger.warning("video.h3_int8_gemm: streamed install skipped: %s", exc)
from .video_minimax_h3_residency import compile_blocks_below_offload_hooks
compile_blocks_below_offload_hooks(
getattr(pipe, denoiser_component, None), logger = logger
)
if "int8_gemm" in speed_optims:
# before the first forward traces the blocks
try:
from .video_minimax_h3_ffn import install as install_h3_ffn
if install_h3_ffn(
getattr(pipe, denoiser_component, None), device = device, logger = logger
):
speed_optims += ("h3_fused_swiglu",)
except Exception as exc: # noqa: BLE001 -- optimisation only, the stock MLP stays
logger.warning("video.h3_ffn: skipped: %s", exc)
if "h3_strided_attn" in speed_optims:
from .video_minimax_h3_attn import shared_qkv_count
shared_qkv = shared_qkv_count(getattr(pipe, denoiser_component, None))
if shared_qkv:
speed_optims += ("h3_shared_qkv_quant",)
logger.info(
"video.h3_attn: Q / K / V share one ConvRot rotation + act quant on %d "
"attention modules",
shared_qkv,
)
except Exception as exc: # noqa: BLE001 -- optimisation only, never fail a load
logger.warning("video.h3_speed_optims failed, continuing unoptimised: %s", exc)
# H3 declines graphs; this only acts when a family override forces them.
graph_applied = arm_graphs_after_placement(
speed_view, {"cuda_graph": "cuda_graph" in speed_optims}, logger
)
if graph_applied.get("cuda_graph") and "cuda_graph" not in speed_optims:
speed_optims += ("cuda_graph",)
elif not graph_applied.get("cuda_graph"):
speed_optims = tuple(o for o in speed_optims if o != "cuda_graph")
if speed_view is not pipe:
for attr in ("_unsloth_cuda_graph_reason", "_unsloth_cuda_graphs"):
if hasattr(speed_view, attr):
try:
setattr(pipe, attr, getattr(speed_view, attr))
except Exception: # noqa: BLE001
pass
# nothing here compiles, so it follows the REQUESTED tier, not the denoiser's eager downgrade above
try:
from .video_minimax_h3_vae import apply_h3_vae_speedups
vae_levers = apply_h3_vae_speedups(
getattr(pipe, "vae", None),
speed_mode = h3_vae_speed,
workflow = workflow,
logger = logger,
)
speed_optims = speed_optims + tuple(f"h3_vae_{name}" for name in vae_levers)
except Exception as exc: # noqa: BLE001 -- optimisation only, never fail a load
logger.warning("video.h3_vae_fast failed, keeping the stock VAE: %s", exc)
try:
from .video_minimax_h3_ref_encode import install_h3_chunked_reference_encode
install_h3_chunked_reference_encode(logger)
except Exception as exc: # noqa: BLE001 -- memory only, never fail a load
logger.warning("video.h3_ref_encode: keeping the stock reference encode: %s", exc)
# Opt-in Sol-H3-VAE (UNSLOTH_H3_SOL_VAE=1): the video stream's _decode only, the stock levers above stay as
# its fallback; the audio VAE is untouched.
try:
from .video_minimax_h3_sol_vae import h3_sol_vae_requested, install_h3_sol_vae
if h3_sol_vae_requested() and install_h3_sol_vae(
getattr(pipe, "vae", None),
device = device,
dtype = dtype,
hf_token = hf_token,
local_files_only = local_files_only,
logger = logger,
):
speed_optims = speed_optims + ("h3_sol_vae",)
except Exception as exc: # noqa: BLE001 -- optional decoder, never fail a load
logger.warning("video.h3_sol_vae: not engaged, keeping the stock decoder: %s", exc)
# the H3 VAE speedups rewrap vae.decode: the OOM retry's scope (when installed) goes back on top
try:
from .diffusion_vae_oom import ensure_outermost
ensure_outermost(getattr(pipe, "vae", None), logger)
except Exception as exc: # noqa: BLE001 - a decode OOM then fails as before
logger.warning("video.vae_oom: scope not restored after the H3 VAE speedups: %s", exc)
if "cudnn_benchmark" in speed_optims:
try:
from .video_minimax_h3_vae import install_audio_vae_without_cudnn_benchmark
if install_audio_vae_without_cudnn_benchmark(getattr(pipe, "audio_vae", None)):
logger.info("video.h3_audio_vae: cudnn.benchmark held off for the audio VAE")
except Exception as exc: # noqa: BLE001 -- optimisation only, never fail a load
logger.warning("video.h3_audio_vae: keeping the stock audio VAE: %s", exc)
# Modular loop opens no FBCache context; one call per step, so the call counter is the step.
h3_cache_engaged: Optional[str] = None
h3_cache_reason = "not supported by this modular workflow"
try:
h3_cache_request = normalize_transformer_cache(transformer_cache)
except ValueError:
h3_cache_request = str(
transformer_cache
) # the routes refuse it first; a direct call runs uncached
h3_cache_auto = transformer_cache is None or h3_cache_request == TC_AUTO
h3_static_plan = (
auto_static_skip_plan((repo_id, base), h3_vae_speed, fam.default_steps)
if h3_cache_auto
else None
)
if h3_static_plan or h3_cache_request == TC_STATIC:
h3_cache_engaged = install_static_step_skip(
_denoiser_view(pipe, denoiser_component),
settings = (
auto_static_settings(h3_static_plan, logger = logger) if h3_static_plan else None
),
logger = logger,
)
if h3_cache_engaged is None:
h3_cache_reason = (
"static step skip is unavailable for this pipeline; it runs uncached"
)
elif h3_static_plan:
h3_cache_reason = (
f"auto: static step skip (every {h3_static_plan['every']}) for this model at "
f"{h3_static_plan['min_steps']}+ steps; set UNSLOTH_DIFFUSION_AUTO_STEP_SKIP=0 to turn it off"
)
else:
h3_cache_reason = "requested"
elif h3_cache_auto:
h3_cache_reason = (
"auto: static step skip engages on the max speed tier only"
if auto_static_skip_entry(repo_id, base)
else "auto: not enabled for this model"
)
elif h3_cache_request is not None:
h3_cache_reason = (
"only static step skip is supported by this modular workflow; running uncached"
)
else:
h3_cache_reason = "requested"
if offload_policy != "none" and (te_streamed or denoiser_streamed):
# Only the VAEs rotate now, evicting each other every render. Installed last: the levers above replace weights.
from .video_minimax_h3_residency import install_pinned_swap
for vae_name in ("vae", "audio_vae"):
try:
install_pinned_swap(
getattr(pipe, vae_name, None), logger = logger, label = vae_name
)
except Exception as exc: # noqa: BLE001 -- stock moves
logger.warning("video.h3_pinned_swap: %s: %s", vae_name, exc)
from .video_minimax_h3_residency import install_h3_rotation_park
install_h3_rotation_park(pipe, park = bool(denoiser_streamed), logger = logger)
if te_streamed == "stream_lazy":
# Last host copy, so available RAM is final.
from .video_minimax_h3_te import h3_te_resident_enabled, keep_h3_te_resident
if h3_te_resident_enabled():
if block_pinner is not None:
block_pinner.join()
keep_h3_te_resident(getattr(pipe, "text_encoder", None), logger = logger)
resolved = build_resolved_record(
{
"family_override": _family_override_resolved(family_override, fam),
"memory_mode": (
memory_mode,
offload_policy,
"MiniMax-H3 ComponentsManager auto CPU offload"
+ ("; the pre-quantized denoiser stays resident" if denoiser_pinned else "")
+ (
"; the pre-quantized denoiser streams block by block (group offload)"
if denoiser_streamed
else ""
)
+ (
"; the conditioner streams leaf by leaf (group offload)"
if te_streamed
else ""
),
),
"speed_mode": (
speed_mode,
effective_speed,
(
"regional compile over the streamed pre-quantized denoiser"
if denoiser_streamed
else "regional compile over the resident pre-quantized denoiser"
)
if "compiled" in speed_optims
else (
"lossless optimisations only; the bf16 denoiser is resident beside the "
"streamed conditioner (compile held off: output unchanged)"
if dense_compile_held
else "lossless optimisations only; the denoiser is in the CPU-offload rotation"
),
),
"attention_backend": (
attention_backend,
attention_engaged or "native",
attention_status_reason(attention_backend, attention_engaged),
),
"transformer_cache": (
(
transformer_cache,
"off",
h3_cache_reason,
RESOLVED_UNSUPPORTED,
)
if h3_cache_engaged is None and h3_cache_request not in (None, TC_AUTO)
else (
None if h3_cache_auto else transformer_cache,
h3_cache_engaged or "off",
h3_cache_reason,
)
),
"cuda_graph": (
None,
"on" if "cuda_graph" in speed_optims else "off",
"denoiser step captured per input shape, replayed bit-identically"
if "cuda_graph" in speed_optims
else str(
getattr(pipe, "_unsloth_cuda_graph_reason", None)
or "speed tier does not capture"
),
),
"transformer_quant": (
transformer_quant_requested,
transformer_quant_engaged or "off",
transformer_quant_reason,
),
# Record the REQUEST, not None: a declined or failed one reads as a fallback, not a bare "off".
"text_encoder_quant": (
text_encoder_quant_requested,
text_encoder_quant_engaged or "off",
text_encoder_quant_reason,
),
}
)
autoopt.record_load_steps(
h3_audit,
attention = attention_engaged,
step_cache = h3_cache_engaged,
step_cache_reason = h3_cache_reason,
compile_cache = False,
compile_cache_reason = autoopt.VIDEO_COMPILE_CACHE_DECLINE,
)
autoopt.finalize(h3_audit, speed_optims, resolved)
apply_comfy_flow_shift(pipe, getattr(fam, "comfy_flow_shift", None), logger)
from . import diffusion_prompt_cache
from .video_minimax_h3_te import install_h3_te_read_capture
try:
install_h3_te_read_capture(
pipe.text_encoder,
layer = getattr(pipe, "text_encoder_layer", 50),
logger = logger,
)
except Exception: # noqa: BLE001 - a lazy component that is not loaded
pass
diffusion_prompt_cache.install(
pipe,
identity = {
"family": fam.name,
"repo": str(repo_id),
"base": str(base),
"dtype": str(dtype),
"workflow": str(workflow),
"te_quant": str(text_encoder_quant_engaged),
},
logger = logger,
)
with self._lock:
if _load_token is not None and _load_token != self._load_token:
stop_background_pins(pipe)
del pipe
clear_gpu_cache()
raise RuntimeError("Video load was cancelled or superseded.")
from .diffusion_nvfp4_install import record_install_reason
record_install_reason(self, *(_nvfp4_install_outcome or (True, None)), device)
self._state = _VideoLoadState(
pipe = pipe,
family = fam,
repo_id = repo_id,
display_repo_id = display_repo_id,
base_repo = base,
fetch_base = fetch_base,
device = device,
gpu_ordinal = umem_target.ordinal,
placed_ordinal = placed_cuda_ordinal(umem_target),
dtype = str(dtype).replace("torch.", ""),
kind = kind,
gguf_filename = gguf_filename if comfy_checkpoint else None,
engine = "diffusers",
h3_task = workflow,
offload_policy = offload_policy,
vae_tiling = True,
memory_mode = normalize_memory_mode(memory_mode),
speed_mode = effective_speed,
speed_optims = speed_optims,
backend_flags = backend_flags,
attention_backend = attention_engaged,
# Report the engaged scheme, not the requested one; a dense fallback must not claim a quant it lacks.
transformer_quant = transformer_quant_engaged,
# Preflight sizes from this; a dense fallback recorded as int8 understates by 39 GB.
text_encoder_quant = text_encoder_quant_engaged,
transformer_cache = h3_cache_engaged,
denoiser_pinned = denoiser_pinned,
denoiser_streamed = bool(denoiser_streamed),
# A slab-arena pin released the pageable source, so the pinned copy is the only one.
denoiser_host_copy = denoiser_streamed == "stream" and not denoiser_single_host_copy,
te_streamed = bool(te_streamed),
h3_residency = residency,
resolved = resolved,
)
commit_load_budget_fraction("video")
self._precommit_globals = None
logger.info(
"video.loaded: %s (%s, modular diffusers, speed=%s%s, transformer_quant=%s, "
"text_encoder_quant=%s)",
repo_id,
fam.name,
effective_speed,
f" [{', '.join(speed_optims)}]" if speed_optims else "",
transformer_quant_engaged or "off",
text_encoder_quant_engaged or "off",
)
return self.status()
@staticmethod
def _resolve_checkpoint_path(
repo_id: str,
gguf_filename: Optional[str],
hf_token: Optional[str],
local_files_only: bool = False,
fetch: Optional[tuple[str, Optional[str]]] = None,
) -> Path:
"""The local checkpoint file for a gguf/single_file load (downloads if hub).
``fetch``: the load plan's ``(repo, revision)`` for the checkpoint, so assembly opens the exact file the
prefetch pinned; absent, the repo is settled here and ``main`` is followed.
``local_files_only`` resolves it from the cache instead: an API-initiated load has already
been told the checkpoint is staged, and this call runs a second time inside load_pipeline
(after _run_load's own fetch), where a cache miss must fail rather than pull the file."""
from .diffusion_families import resolve_local_gguf_child
root = Path(repo_id).expanduser()
if root.is_dir():
return resolve_local_gguf_child(root, gguf_filename or "")
if root.is_file():
return root
from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback
fetch_repo, revision = fetch or (
_video_fetch_repo(repo_id, hf_token, files = (gguf_filename or "",), checkpoint = True),
None,
)
return Path(
hf_hub_download_with_xet_fallback(
fetch_repo,
gguf_filename or "",
hf_token,
revision = revision,
reuse_other_cache_root = True,
local_files_only = local_files_only,
gguf_header_delta = True,
)
)
@staticmethod
def _resolve_moe_partner_path(
repo_id: str,
gguf_filename: Optional[str],
hf_token: Optional[str],
local_files_only: bool = False,
fetch: Optional[tuple[str, Optional[str]]] = None,
) -> Path:
"""The OTHER expert's file of a dual-expert pick (same folder / repo, paired by name; downloads if hub)."""
root = Path(repo_id).expanduser()
if root.is_file():
partner = moe_partner_filename(root.name)
if partner is None:
raise ValueError(f"'{root.name}' does not name a high_noise / low_noise expert.")
path = root.with_name(partner)
if not path.is_file():
raise FileNotFoundError(f"The partner expert '{path}' does not exist.")
return path
partner = moe_partner_filename(gguf_filename)
if partner is None:
raise ValueError(f"'{gguf_filename}' does not name a high_noise / low_noise expert.")
return VideoBackend._resolve_checkpoint_path(
repo_id, partner, hf_token, local_files_only = local_files_only, fetch = fetch
)
# ── generation ───────────────────────────────────────────────────────────
def loaded_family(self) -> Optional[VideoFamily]:
"""Resident pipeline's family, read via getattr so a state without one degrades instead of
raising."""
with self._lock:
state = self._state
return getattr(state, "family", None) if state is not None else None
def generation_snapshot(self) -> tuple[dict[str, Any], Optional[object]]:
"""Return the route-facing generation fields and their exact resident-state token."""
from hub.utils.gguf import extract_quant_token
with self._lock:
state = self._state
if state is None:
return {"loaded": False, "repo_id": None, "defaults": None}, None
fam = state.family
status = {
"loaded": True,
"repo_id": getattr(state, "repo_id", None),
"dtype": getattr(state, "dtype", None),
"model_kind": getattr(state, "kind", None),
"gguf_variant": (
extract_quant_token(state.gguf_filename)
if getattr(state, "kind", None) == "gguf"
and getattr(state, "gguf_filename", None)
else None
),
"h3_task": getattr(state, "h3_task", None),
"defaults": _family_shape_defaults(fam),
}
return status, state
@staticmethod
def _reset_step_cache(pipe: Any) -> None:
"""Clear FBCache residuals on the resident DiT(s) so the next clip never reuses stale state."""
for name in ("transformer", "transformer_2"):
module = getattr(pipe, name, None)
reset = getattr(module, "_reset_stateful_cache", None) or getattr(
module, "reset_stateful_hooks", None
)
if callable(reset):
try:
reset()
except Exception: # noqa: BLE001 -- reset is best-effort, never fail a generation
pass
@vram_budget_scope("video")
def begin_generate(
self,
*,
prompt: str,
negative_prompt: Optional[str] = None,
width: Optional[int] = None,
height: Optional[int] = None,
num_frames: Optional[int] = None,
duration_s: Optional[float] = None,
fps: Optional[int] = None,
steps: Optional[int] = None,
guidance: Optional[float] = None,
guidance_2: Optional[float] = None,
seed: Optional[int] = None,
input_reference: Optional[str] = None,
first_frame: Optional[str] = None,
last_frame: Optional[str] = None,
reference_images: Optional[list[str]] = None,
reference_videos: Optional[list[dict]] = None,
reference_audios: Optional[list[str]] = None,
reference_image_size: Optional[str] = None,
flow_shift: Optional[float] = None,
audio_flow_shift: Optional[float] = None,
continuation_video: Optional[str] = None,
continuation_keep: Optional[str] = None,
video_id: Optional[str] = None,
decoded_reference_audios: Optional[Any] = None,
decoded_reference_images: Optional[Any] = None,
expected_state: Optional[object] = None,
live_preview: Optional[bool] = None,
before_commit: Optional[Callable[[], None]] = None,
) -> dict[str, int]:
"""Returns at once (proxy caps ~100 s). VIDEO_NOT_LOADED_MSG and VIDEO_GENERATION_BUSY_MSG map to 409.
``before_commit`` runs under the lock just before the reservation; raising aborts it."""
if input_reference is not None and (first_frame is not None or reference_images):
raise ValueError(
"input_reference cannot be combined with explicit conditioning inputs."
)
cancel = threading.Event()
from core.inference.generation_ticket import bind_generation_cancel
# A chat tools call that is stopped cancels this job too.
bind_generation_cancel(cancel)
job_token = object() # this reservation's identity; only its own worker may finalise it
while True:
with self._lock:
state = self._state
if expected_state is not None and state is not expected_state:
raise RuntimeError(VIDEO_MODEL_CHANGED_MSG)
if state is None:
raise RuntimeError(VIDEO_NOT_LOADED_MSG)
if self._generate_job_active:
raise RuntimeError(VIDEO_GENERATION_BUSY_MSG)
resolved_first_frame = first_frame
resolved_reference_images = reference_images
if input_reference is not None:
if state.h3_task == H3_TASK_REFERENCES:
resolved_reference_images = [input_reference]
else:
resolved_first_frame = input_reference
image_kwargs = (
{"decoded_images": decoded_reference_images.pool()}
if decoded_reference_images is not None
else {}
)
first_pil, last_pil, canvas_w, canvas_h, conditioning = self._resolve_keyframes(
state.family,
state.h3_task,
resolved_first_frame,
last_frame,
width,
height,
**image_kwargs,
)
references = self._resolve_references(
state.family,
state.h3_task,
state.engine,
resolved_reference_images,
reference_videos,
reference_audios,
reference_image_size,
canvas_w,
canvas_h,
decoded_audios = decoded_reference_audios,
**image_kwargs,
)
shift, audio_shift = self._resolve_flow_shifts(
state.family, state.engine, flow_shift, audio_flow_shift
)
run_num_frames = num_frames
if duration_s is not None:
run_fps = int(fps or state.family.default_fps or 24)
step = max(1, int(state.family.frame_step))
offset = max(1, int(state.family.frame_offset))
wanted = max(offset, int(round(float(duration_s) * run_fps)))
k = max(1, int(round((wanted - offset) / step)))
run_num_frames = k * step + offset
if references:
conditioning = h3_conditioning_mode(has_references = True)
continuation, canvas_w, canvas_h = self._resolve_continuation(
state.family,
continuation_video,
continuation_keep,
requested_size = (width, height),
canvas = (canvas_w, canvas_h),
fps = fps,
num_frames = run_num_frames,
has_keyframes = bool(resolved_first_frame or last_frame),
has_references = bool(references),
)
if continuation is not None:
conditioning = VIDEO_CONTINUATION_CONDITIONING
resolved_inputs = _VideoResolvedInputs(
state = state,
first_frame = first_pil,
last_frame = last_pil,
width = canvas_w,
height = canvas_h,
conditioning = conditioning,
references = references,
flow_shift = shift,
audio_flow_shift = audio_shift,
continuation = continuation,
)
with self._lock:
if self._state is not state:
continue
if self._generate_job_active:
raise RuntimeError(VIDEO_GENERATION_BUSY_MSG)
validate_video_request_shape(
state.family,
width = width,
height = height,
num_frames = run_num_frames,
fps = fps,
)
if before_commit is not None:
before_commit()
self._generate_job_active = True
self._generate_job_token = job_token
self._generate_job_account = current_account_id()
self._active_generate_cancel = cancel
self._gen_video_id = video_id
self._gen = {
"active": True,
"phase": "queued",
"step": 0,
"total": 0,
"eta_seconds": None,
}
self._last_request_shape = None
break
worker = account_thread(
# Not in kwargs: those are also replayed as a valid generate() call.
target = functools.partial(self._run_generate, job_token = job_token, video_id = video_id),
kwargs = dict(
prompt = prompt,
negative_prompt = negative_prompt,
width = width,
height = height,
num_frames = run_num_frames,
fps = fps,
steps = steps,
guidance = guidance,
guidance_2 = guidance_2,
seed = seed,
_resolved_inputs = resolved_inputs,
cancel_event = cancel,
live_preview = live_preview,
),
daemon = True,
)
try:
worker.start()
except RuntimeError:
# No thread started, so release the reservation; RuntimeError only, a signal may leave a live worker.
self._finish_generate_job(
job_token = job_token,
cancel_event = cancel,
error = "Video generation could not start.",
)
raise
# Read from the committed state: a load in between could swap the family.
state = resolved_inputs.state
fam = getattr(state, "family", None)
return {
"width": resolved_inputs.width,
"height": resolved_inputs.height,
"num_frames": (
run_num_frames
if run_num_frames is not None
else getattr(fam, "default_num_frames", None)
),
"fps": fps if fps is not None else getattr(fam, "default_fps", None),
"model": getattr(state, "repo_id", None),
}
def generate_job_account(self) -> Optional[str]:
return self._generate_job_account
def static_skip_view(self) -> tuple:
"""(owner, ``transformer_cache_stats``) from one read; the owner is never part of status()."""
state = self._state
if state is None or state.transformer_cache != TC_STATIC:
return None, None
return static_skip_view(_skip_pipe(state))
def _run_generate(
self,
*,
cancel_event: threading.Event,
job_token: Optional[object] = None,
video_id: Optional[str] = None,
**gen_kwargs: Any,
) -> None:
"""Reservation never outlives its thread after BaseException; begin_generate looks it up by name."""
try:
self._run_generate_body(
cancel_event = cancel_event, job_token = job_token, video_id = video_id, **gen_kwargs
)
finally:
if job_token is not None:
self._finish_generate_job(
job_token = job_token,
cancel_event = cancel_event,
error = "Video generation failed.",
)
def _run_generate_body(
self,
*,
cancel_event: threading.Event,
job_token: Optional[object] = None,
video_id: Optional[str] = None,
**gen_kwargs: Any,
) -> None:
"""Worker for begin_generate; unexpected errors are logged in full, the client gets
video_failure_detail's redacted reason."""
from . import video_gallery
def _record_outcome(error: Optional[str] = None) -> None:
if video_id is None:
return
try:
video_gallery.record_job_outcome(
video_id, completed_at = int(time.time()), error = error
)
except Exception as exc: # noqa: BLE001 -- job persistence must not strand the backend busy
logger.warning("video.persist_job_outcome_failed: %s", exc)
try:
result = self.generate(cancel_event = cancel_event, **gen_kwargs)
except ValueError as exc:
_record_outcome(str(exc))
self._finish_generate_job(
job_token = job_token, cancel_event = cancel_event, error = str(exc)
)
return
except RuntimeError as exc:
msg = str(exc)
if msg not in (VIDEO_NOT_LOADED_MSG, VIDEO_CANCELLED_MSG):
logger.error("video.generate_failed: %s", exc, exc_info = True)
msg = video_failure_detail(exc, self._last_request_shape)
_record_outcome(msg)
self._finish_generate_job(job_token = job_token, cancel_event = cancel_event, error = msg)
return
except Exception as exc: # noqa: BLE001 -- worker thread: never propagate
logger.error("video.generate_failed: %s", exc, exc_info = True)
detail = video_failure_detail(exc, self._last_request_shape)
_record_outcome(detail)
self._finish_generate_job(
job_token = job_token,
cancel_event = cancel_event,
error = detail,
)
return
# Persist the clip with its full recipe as the JSON sidecar the gallery reads back.
created_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
try:
record = video_gallery.save(
result["mp4_bytes"],
{
"prompt": gen_kwargs["prompt"],
# From the result: guidance-distilled families consume no negative prompt.
"negative_prompt": result.get("negative_prompt"),
"width": result["width"],
"height": result["height"],
"num_frames": result["num_frames"],
"fps": result["fps"],
"duration_s": result["duration_s"],
"steps": result["steps"],
"guidance": result["guidance"],
"guidance_2": gen_kwargs.get("guidance_2"),
"seed": result["seed"],
"has_audio": result["has_audio"],
"conditioning": result["conditioning"],
"flow_shift": result.get("flow_shift"),
"audio_flow_shift": result.get("audio_flow_shift"),
"model": result["repo_id"],
# Engaged values, so recipes keep precision after unload; old sidecars read absent keys as null.
"model_kind": result.get("model_kind"),
"gguf_filename": result.get("gguf_filename"),
"transformer_quant": result.get("transformer_quant"),
"text_encoder_quant": result.get("text_encoder_quant"),
"memory_mode": result.get("memory_mode"),
"offload_policy": result.get("offload_policy"),
"created_at": created_at,
},
**({"video_id": video_id} if video_id is not None else {}),
)
except Exception as exc: # noqa: BLE001 -- disk failure must reach the poller
logger.error("video.persist_failed: %s", exc)
_record_outcome("Failed to save the generated video.")
self._finish_generate_job(
job_token = job_token,
cancel_event = cancel_event,
error = "Failed to save the generated video.",
)
return
_record_outcome()
self._finish_generate_job(
job_token = job_token,
cancel_event = cancel_event,
video = record,
total = result["steps"],
)
def _finish_generate_job(
self,
*,
job_token: Optional[object] = None,
cancel_event: Optional[threading.Event] = None,
video: Optional[dict] = None,
error: Optional[str] = None,
total: int = 0,
) -> None:
"""Publish a job's terminal state once, keyed on its token, as the busy flag may be a
successor's."""
with self._lock:
if self._generate_job_token is not job_token:
return
self._generate_job_token = None
self._generate_job_active = False
if cancel_event is not None and self._active_generate_cancel is cancel_event:
# Covers a worker failing before generate()'s finally; identity check spares a direct generate().
self._active_generate_cancel = None
if error is not None:
self._gen = {
"active": False,
"phase": "failed",
"error": error,
"step": 0,
"total": 0,
"eta_seconds": None,
}
logger.info(
"video_generation_progress",
phase = "failed",
percent = 0,
step = 0,
total_steps = 0,
error = error,
)
else:
self._gen = {
"active": False,
"phase": "completed",
"video": video,
"step": total,
"total": total,
"eta_seconds": None,
}
logger.info(
"video_generation_progress",
phase = "completed",
percent = 100,
step = total,
total_steps = total,
video_id = (video or {}).get("id"),
)
@holds_inference
@vram_budget_scope("video")
def generate(
self,
*,
prompt: str,
negative_prompt: Optional[str] = None,
width: Optional[int] = None,
height: Optional[int] = None,
num_frames: Optional[int] = None,
fps: Optional[int] = None,
steps: Optional[int] = None,
guidance: Optional[float] = None,
guidance_2: Optional[float] = None,
seed: Optional[int] = None,
first_frame: Optional[str] = None,
last_frame: Optional[str] = None,
reference_images: Optional[list[str]] = None,
reference_videos: Optional[list[dict]] = None,
reference_audios: Optional[list[str]] = None,
reference_image_size: Optional[str] = None,
flow_shift: Optional[float] = None,
audio_flow_shift: Optional[float] = None,
continuation_video: Optional[str] = None,
continuation_keep: Optional[str] = None,
cancel_event: Optional[threading.Event] = None,
_resolved_inputs: Optional[_VideoResolvedInputs] = None,
# None = on unless UNSLOTH_DIFFUSION_PREVIEW=0.
live_preview: Optional[bool] = None,
) -> dict[str, Any]:
# begin_generate passes its already-registered event; a direct call makes its own.
cancel = cancel_event if cancel_event is not None else threading.Event()
with self._generate_lock:
# Never render beside a background compile; wait outside the state lock so status polls keep answering.
pending_bg = getattr(self._state, "bg_compile", None)
if pending_bg is not None and pending_bg.compiling():
logger.info("video.bg_compile: render waits for the background compile to finish")
waited = pending_bg.wait(cancel)
logger.info(
"video.bg_compile: render waited %.1f s for the background compile", waited
)
with self._lock:
# Python locks are not FIFO; refuse rather than denoise against a pipeline being torn down.
if self._teardown_waiters:
raise RuntimeError(VIDEO_CANCELLED_MSG)
state = self._state
if state is None:
raise RuntimeError(VIDEO_NOT_LOADED_MSG)
if _resolved_inputs is not None and _resolved_inputs.state is not state:
raise RuntimeError(VIDEO_CANCELLED_MSG)
self._active_generate_cancel = cancel
begin_attention_run(state)
apply_cpu_render_threads(state.device)
# Bound below, once the request is resolved. None means the failure beat the resolution, and there is
# nothing truthful to report.
request_shape: Optional[dict[str, Any]] = None
# Compile still pending: this render runs eager and records the denoiser inputs for it.
from . import diffusion_bg_compile as bg_compile
bg = getattr(state, "bg_compile", None)
bg_eager = bg is not None and bg.pending()
bg_token = bg_compile._FORCE_EAGER.set(True) if bg_eager else None
float_seeds = None if bg_eager else self._inline_float_seeds(state)
# Closed in the finally so the pipeline is text-to-video again on return, cancel or failure.
i2v_swap = contextlib.ExitStack()
# Encodes Wan chunks while the VAE decodes; closed in the finally on cancel or failure.
stream_session = None
try:
# First: on a fresh thread, un-indexed device calls resolve to the default card.
self._state_device_target(state)
# A render racing the post-load pinner ran every later render slower; the wait is usually over.
waited = finish_background_pins(
getattr(state, "pipe", None), cancel, include_late = False
)
if cancel.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
if waited >= 1.0:
logger.info(
"video.generate: waited %.1f s for the host weights to finish pinning",
waited,
)
fam = state.family
if _resolved_inputs is None:
requested_size = (width, height)
first_pil, last_pil, width, height, conditioning = self._resolve_keyframes(
fam, state.h3_task, first_frame, last_frame, width, height
)
references = self._resolve_references(
fam,
state.h3_task,
state.engine,
reference_images,
reference_videos,
reference_audios,
reference_image_size,
width,
height,
)
if references:
conditioning = h3_conditioning_mode(has_references = True)
continuation, width, height = self._resolve_continuation(
fam,
continuation_video,
continuation_keep,
requested_size = requested_size,
canvas = (width, height),
fps = fps,
num_frames = num_frames,
has_keyframes = bool(first_frame or last_frame),
has_references = bool(references),
)
if continuation is not None:
conditioning = VIDEO_CONTINUATION_CONDITIONING
shift, audio_shift = self._resolve_flow_shifts(
fam, state.engine, flow_shift, audio_flow_shift
)
else:
first_pil = _resolved_inputs.first_frame
last_pil = _resolved_inputs.last_frame
width = _resolved_inputs.width
height = _resolved_inputs.height
conditioning = _resolved_inputs.conditioning
references = _resolved_inputs.references
continuation = _resolved_inputs.continuation
shift = _resolved_inputs.flow_shift
audio_shift = _resolved_inputs.audio_flow_shift
frames = snap_num_frames(fam, num_frames or fam.default_num_frames)
validate_video_fps(fam, fps)
out_fps = (
fam.default_fps if fam.name == "minimax-h3" else int(fps or fam.default_fps)
)
# The same identifiers, in the same order, as the status defaults the UI shows.
default_steps, default_guidance = default_video_generation_params(
state.variant_id,
state.gguf_filename,
state.display_repo_id,
state.repo_id,
state.base_repo,
fallback = (fam.default_steps, fam.default_guidance),
family = fam.name,
)
default_steps = _h3_pdd_default_steps(fam, state.pipe, default_steps)
steps = int(steps or default_steps)
guidance = float(default_guidance if guidance is None else guidance)
# Guidance-free families ignore CFG and negative prompt on both engines; record the family
# default (fam.default_guidance, not the path-derived one).
if not fam.supports_cfg:
guidance = float(fam.default_guidance)
negative_prompt = None
if state.engine == "sd_cpp":
return self._generate_h3_native(
state = state,
prompt = prompt,
width = width,
height = height,
frames = frames,
fps = out_fps,
steps = steps,
guidance = guidance,
seed = seed,
first_frame = first_pil,
last_frame = last_pil,
references = references,
conditioning = conditioning,
flow_shift = shift,
cancel = cancel,
)
import torch
if state.engine == "diffusers" and fam.modular_workflow and state.device != "cpu":
from .video_minimax_h3 import (
H3_TEXT_ENCODER_BF16_GB,
estimate_h3_diffusers_vram_gb,
)
device_obj = torch.device(state.device)
device_module = getattr(torch, device_obj.type, None)
if device_module is not None and hasattr(device_module, "mem_get_info"):
# Windows ROCm's free half is an over-report and this is a hard refusal, so cap it against the
# allocator (#8403).
available_vram_gb = (
_request_available_bytes(device_obj, device_module) / 1_000_000_000
)
# Size the floor from the components this load ACTUALLY holds, not from the released bfloat16
# pair. Both fields are the ENGAGED schemes (a declined or failed request is recorded as None at
# load), so a dense fallback keeps the dense floor.
required_vram_gb = estimate_h3_diffusers_vram_gb(
width,
height,
frames,
text_encoder_gb = h3_te_resident_gb(
state.text_encoder_quant, bf16_gb = H3_TEXT_ENCODER_BF16_GB
),
transformer_gb = h3_transformer_resident_gb(state.transformer_quant),
transformer_pinned = bool(getattr(state, "denoiser_pinned", False)),
transformer_streamed = bool(getattr(state, "denoiser_streamed", False)),
text_encoder_streamed = bool(getattr(state, "te_streamed", False)),
)
residency = getattr(state, "h3_residency", None)
if residency is not None and getattr(state, "denoiser_streamed", False):
# Resident blocks are demotable, so they count as available. The floor omits the
# fragmentation slack (a fully streamed render fits as before); the resident set keeps it.
fit_kwargs = dict(
width = width,
height = height,
frames = frames,
te_streamed = bool(getattr(state, "te_streamed", False)),
te_scheme = state.text_encoder_quant,
)
floor_room = _h3_residency_budget_bytes(
residency,
state.device,
fragmentation = False,
top_gb = residency.top_bytes / 1_000_000_000,
**fit_kwargs,
)
available_vram_gb += residency.resident_bytes() / 1_000_000_000
required_vram_gb = available_vram_gb - floor_room / 1_000_000_000
residency.fit(
max(
0,
_h3_residency_budget_bytes(
residency, state.device, **fit_kwargs
),
)
)
if (
available_vram_gb + 0.25 < required_vram_gb
and getattr(state, "denoiser_pinned", False)
and getattr(state, "te_streamed", False)
and state.transformer_quant is None
and state.offload_policy != "none"
):
# Pin was sized for the default clip; a longer one the rotation fits must not be refused.
rotation_gb = estimate_h3_diffusers_vram_gb(
width,
height,
frames,
text_encoder_gb = h3_te_resident_gb(
state.text_encoder_quant, bf16_gb = H3_TEXT_ENCODER_BF16_GB
),
text_encoder_streamed = True,
)
if available_vram_gb + 0.25 >= rotation_gb and _h3_dense_demote(
state.pipe,
getattr(state.pipe, h3_denoiser_component(state.h3_task), None),
logger = logger,
):
required_vram_gb = rotation_gb
if available_vram_gb + 0.25 < required_vram_gb:
raise RuntimeError(
f"MiniMax-H3 needs about {required_vram_gb:.1f} GB available "
f"VRAM for {width}x{height} at {frames} frames; "
f"{max(0.0, available_vram_gb):.1f} GB is available. Lower the resolution "
"or duration, or load the GGUF artifact."
)
from .video_minimax_h3 import h3_host_ram_shortfall
# Same engaged components as the VRAM floor; released-pair sizing would refuse quantized loads.
shortfall = h3_host_ram_shortfall(
max(0.0, available_vram_gb),
text_encoder_gb = h3_te_resident_gb(
state.text_encoder_quant, bf16_gb = H3_TEXT_ENCODER_BF16_GB
),
transformer_gb = h3_transformer_resident_gb(state.transformer_quant),
transformer_streamed = bool(getattr(state, "denoiser_host_copy", False)),
text_encoder_streamed = bool(getattr(state, "te_streamed", False)),
)
if shortfall is not None:
raise RuntimeError(shortfall)
elif (
state.engine == "diffusers"
and not fam.modular_workflow
and getattr(state, "vram_split_cards", None)
and hasattr(torch, "cuda")
and hasattr(torch.cuda, "mem_get_info")
):
def _available_mib(ordinal: int) -> int:
available = _request_available_bytes(
torch.device("cuda", ordinal), torch.cuda
)
return int(available // (1024 * 1024))
shortfall = video_split_shortfall_message(
state.vram_split_cards,
family = fam.name,
available_mib = _available_mib,
width = width,
height = height,
frames = frames,
planned_shape = (
fam.resolution_presets[0][0],
fam.resolution_presets[0][1],
fam.default_num_frames,
)
if getattr(fam, "resolution_presets", None)
else None,
)
if shortfall is not None:
raise RuntimeError(shortfall)
elif (
state.engine == "diffusers"
and not fam.modular_workflow
and getattr(state, "vram_floor_mib", None) is not None
and str(state.device).startswith("cuda")
and hasattr(torch, "cuda")
and hasattr(torch.cuda, "mem_get_info")
):
# Otherwise a card that cannot hold the tier's weights is admitted and dies at the first onload.
ordinal = getattr(state, "placed_ordinal", None)
device_obj = (
torch.device("cuda", int(ordinal))
if ordinal is not None
else torch.device(state.device)
)
from . import video_stream_residency
residency_fit = video_stream_residency.applies(
fam.name,
is_moe = bool(getattr(fam, "is_moe", False)),
offload_policy = state.offload_policy,
device = state.device,
)
# Key on the cache mode this request runs; corrected after the runner-aware toggle.
if state.cache_auto:
from .diffusion_cache import auto_step_cache_mode
request_cache_mode = auto_step_cache_mode(steps)
else:
request_cache_mode = getattr(state, "transformer_cache", None)
calibration = video_stream_residency.calibration_key(
fam.name,
getattr(state, "repo_id", None),
state.offload_policy,
device_obj,
# the exact weights: a mutable repo updated and reloaded must not reuse old peaks
revision = _calibration_revision(state),
kind = getattr(state, "kind", None),
gguf_filename = getattr(state, "gguf_filename", None),
variant_id = getattr(state, "variant_id", None),
h3_task = getattr(state, "h3_task", None),
dtype = getattr(state, "dtype", None),
attention_backend = getattr(state, "attention_backend", None),
speed_mode = getattr(state, "speed_mode", None),
speed_optims = video_stream_residency.live_speed_optims(
state.pipe, getattr(state, "speed_optims", ())
),
transformer_cache = request_cache_mode,
vae_tiling = getattr(state, "vae_tiling", None),
memory_mode = getattr(state, "memory_mode", None),
transformer_quant = getattr(state, "transformer_quant", None),
text_encoder_quant = getattr(state, "text_encoder_quant", None),
fps = out_fps if getattr(fam, "has_audio", False) else None,
conditioning = conditioning,
first_frame = first_pil is not None,
last_frame = last_pil is not None,
references = len(references or ()),
cfg = bool(getattr(fam, "supports_cfg", True)) and float(guidance) > 1.0,
)
# Tag BEFORE the residency fit: after a reload the fit reads process-wide peaks through this key.
video_stream_residency.arm_request(
state.pipe,
key = calibration,
width = width,
height = height,
frames = frames,
device = device_obj,
rearm = False,
)
if residency_fit:
# Resident groups are allocated, so the reserved term below still counts them as available.
video_stream_residency.fit_for_request(
state.pipe,
device = device_obj,
floor_mib = state.vram_floor_mib,
width = width,
height = height,
frames = frames,
logger = logger,
)
else:
video_stream_residency.arm_request(
state.pipe,
key = calibration,
width = width,
height = height,
frames = frames,
device = device_obj,
)
available_bytes = _request_available_bytes(device_obj, torch.cuda)
shortfall = video_offload_shortfall_message(
family = fam.name,
floor_mib = int(state.vram_floor_mib),
available_mib = int(available_bytes // (1024 * 1024)),
placement = f"'{state.offload_policy}' offload",
width = width,
height = height,
frames = frames,
decode_floor_mib = getattr(state, "vram_decode_floor_mib", None),
measured_extra_mib = video_stream_residency.measured_extra_mib(
calibration, int(width) * int(height) * int(frames)
),
)
if shortfall is not None:
raise RuntimeError(shortfall)
# diffusers reproduces a Metal seed only with a CPU generator.
generator_device = (
"cpu" if fam.modular_workflow or str(state.device) == "mps" else state.device
)
generator = torch.Generator(device = generator_device)
if seed is None:
seed = int(generator.seed()) % (2**53)
generator = generator.manual_seed(int(seed))
# Snapshot the resolved request for failure logs; no prompt text, only its length.
request_shape = {
"family": fam.name,
"repo_id": state.display_repo_id or state.repo_id,
"gguf": state.gguf_filename,
"width": width,
"height": height,
"frames": frames,
"fps": out_fps,
"steps": steps,
"guidance": guidance,
"guidance_2": guidance_2,
"seed": int(seed),
"prompt_chars": len(prompt or ""),
"negative_prompt_chars": len(negative_prompt or ""),
"device": state.device,
"dtype": state.dtype,
"offload": state.offload_policy,
"attention_backend": state.attention_backend,
"attention_head_dims": sorted(_dit_head_dims(state.pipe)),
}
self._last_request_shape = request_shape
pipe = state.pipe
i2v_extra: dict[str, Any] = {}
i2v_active = first_pil is not None and video_family_uses_i2v_pipeline(fam)
if i2v_active:
from .video_i2v import first_frame_pipeline
pipe = i2v_swap.enter_context(
first_frame_pipeline(
pipe,
fam.i2v_pipeline_class,
config_overrides = dict(getattr(fam, "i2v_pipeline_config", ()) or ()),
)
)
i2v_extra = _i2v_state_call_kwargs(fam, state)
request_shape["pipeline"] = type(pipe).__name__
if bg_eager:
# LTX-2 i2v passes a per-token timestep (t2v: per sample), so it must not seed the compile.
i2v_swap.enter_context(bg_compile.no_record())
if fam.name in COSMOS3_FAMILIES:
from .video_cosmos3 import seam_safe_tile_strides
from .video_i2v import offload_vae_after_encode
# Every Cosmos3 call (t2v too) encodes a clip first; nothing else offloads the VAE after it.
i2v_swap.enter_context(offload_vae_after_encode(pipe))
i2v_swap.enter_context(seam_safe_tile_strides(pipe, height, width))
call_params = inspect.signature(pipe.__call__).parameters
kwargs: dict[str, Any] = {
"prompt": prompt,
"num_inference_steps": steps,
"width": width,
"height": height,
"num_frames": frames,
"generator": generator,
}
# The 2.3 distilled DiT uses a fixed 8-step sigma curve; other step counts keep scheduler spacing.
sigma_ctx: Any = contextlib.nullcontext()
if fam.name == "ltx-2" and "sigmas" in call_params:
from .video_ltx2 import (
LTX23_DISTILLED_SIGMAS,
ltx2_distilled_ids,
ltx23_verbatim_sigmas,
)
if steps == len(LTX23_DISTILLED_SIGMAS) and ltx2_distilled_ids(
state.variant_id, state.gguf_filename, state.repo_id, state.base_repo
):
kwargs["sigmas"] = list(LTX23_DISTILLED_SIGMAS)
sigma_ctx = ltx23_verbatim_sigmas(pipe)
if fam.name == "ltx-2":
from .video_ltx2 import (
ensure_recompile_limit,
ltx2_distilled_guidance_kwargs,
ltx2_distilled_ids,
)
# Distilled DiT is sampled unguided; newer diffusers defaults would add STG + modality passes.
if ltx2_distilled_ids(
state.variant_id, state.gguf_filename, state.repo_id, state.base_repo
):
kwargs.update(ltx2_distilled_guidance_kwargs(call_params, guidance))
# The render thread copies this thread's context, so raise the recompile limit here.
ensure_recompile_limit()
if not fam.supports_cfg:
pass
elif fam.guidance_via_guider:
# HunyuanVideo-1.5: __call__ has no guidance kwarg; CFG scale is a guider attribute set per request.
pipe.guider.guidance_scale = float(guidance)
else:
kwargs[fam.cfg_kwarg] = guidance
if negative_prompt and "negative_prompt" in call_params:
kwargs["negative_prompt"] = negative_prompt
# LTX-2 takes frame_rate (shapes audio length); others fix their rate, fps only at export
if "frame_rate" in call_params:
kwargs["frame_rate"] = float(out_fps)
if fam.name in COSMOS3_FAMILIES and "fps" in call_params:
kwargs["fps"] = float(out_fps)
for key, value in getattr(fam, "call_kwargs", ()) or ():
if key in call_params:
kwargs[key] = value
# Dual-DiT MoE: pass the low-noise expert guidance only when the family declares it AND the signature
# accepts it (WanPipeline raises with boundary_ratio=None)
if fam.cfg2_kwarg and fam.cfg2_kwarg in call_params and guidance_2 is not None:
kwargs[fam.cfg2_kwarg] = float(guidance_2)
if fam.modular_workflow:
kwargs["output"] = ["videos", "audio", "sampling_rate"]
if bg_eager and not i2v_active and _bg_i2v_render(first_pil, i2v_active):
# Same as the i2v swap above, for a family that conditions without one (its own render).
i2v_swap.enter_context(bg_compile.no_record())
# Auto-blocks distinguish omitted conditioning from empty conditioning.
# LTX-2.5 conditions through its own render (LTX2ConditionPipeline), not an image kwarg.
if first_pil is not None and fam.name != "ltx-2.5":
kwargs["image"] = first_pil
for key, value in i2v_extra.items():
if key in call_params:
kwargs[key] = value
if last_pil is not None and video_family_accepts_last_frame(fam):
kwargs["last_image"] = last_pil
if references:
from .video_minimax_h3 import h3_diffusers_references
kwargs["references"] = h3_diffusers_references(references)
if continuation is not None:
kwargs["video"], kwargs["condition_video_keep"] = continuation
request_shape["continuation"] = continuation[1]
# Set scheduler shifts every run so prior requests cannot leak state.
for component, value in (
("scheduler", shift),
("audio_scheduler", audio_shift),
):
setter = getattr(getattr(pipe, component, None), "set_shift", None)
if callable(setter) and value is not None:
setter(float(value))
if fam.name == "minimax-h3":
from .video_minimax_h3 import h3_num_inference_steps
from .video_minimax_h3_pdd import (
bind_h3_pdd_render,
h3_pdd_num_inference_steps,
)
# H3 grid includes the terminal 0: `steps` evaluations need steps + 1 points.
kwargs["num_inference_steps"] = h3_num_inference_steps(steps)
# PDD bank is distilled for ComfyUI's count, so it keeps steps + 1 even under the kill switch.
if bind_h3_pdd_render(pipe, shift, audio_shift):
kwargs["num_inference_steps"] = h3_pdd_num_inference_steps(steps)
request_shape["pdd"] = True
# What the clip records; LTX-2.5's two-stage render reports its stage-2 steps in the progress below.
recorded_steps = steps
ltx25_render_call = None
if fam.name == "ltx-2.5":
from .video_ltx2 import ensure_recompile_limit as ltx25_recompile_limit
from .video_ltx25 import render as ltx25_render, stage_steps
ltx25_dev = bool(getattr(pipe, "_unsloth_ltx25_dev", False))
steps = sum(stage_steps(steps, dev = ltx25_dev))
ltx25_recompile_limit()
def ltx25_render_call() -> Any:
return ltx25_render(
pipe,
kwargs,
dev = ltx25_dev,
image = first_pil,
upsampler = getattr(pipe, "_unsloth_ltx25_upsampler", None),
on_step = kwargs.get("callback_on_step_end"),
cancel = cancel,
# Stage 1 denoises at half size: the live preview unpacks its tokens with that grid.
on_stage = lambda w, h: (
previewer.set_render_size(h, w) if previewer is not None else None
),
)
started = time.monotonic()
self._gen = {
"active": True,
"phase": "encode",
"step": 0,
"total": steps,
"started": started,
"eta_seconds": None,
"error": None,
}
job_gen = self._gen
def _publish_preview(url: str, seq: int) -> None:
# A late frame must not land on a successor job's record.
if self._gen is job_gen:
job_gen.update(preview = url, preview_seq = seq)
from .diffusion_preview import LatentPreviewer, scheduler_step_preview
# Started just before the try below, whose finally finish()es it: its worker thread polls until then.
previewer = None
def _enter_denoise() -> None:
if float_seeds is not None:
# Before the first (compiling) denoiser forward.
float_seeds.seed()
if self._gen.get("phase") == "encode":
self._gen.update(phase = "denoise")
ticker = _CompletedStepTicker(steps)
def _report(done: int) -> None:
"""Publish a step the GPU has actually finished. Monotonic, and silent once the
denoise is over so a late poll cannot walk the bar back under a later phase."""
done = max(0, min(int(done), steps))
if done > 0:
_enter_denoise()
if self._gen.get("phase") != "denoise":
return
if done <= int(self._gen.get("step") or 0):
return
elapsed = time.monotonic() - started
self._gen.update(
step = done,
eta_seconds = (elapsed / max(1, done)) * max(0, steps - done),
)
def _tick(enqueued: int) -> None:
"""Mark submitted steps with a CUDA/MPS event; skip while another thread captures (events unsafe then)."""
_enter_denoise()
with _hold_off_cuda_graph_capture() as clear:
if not clear:
return
ticker.record(enqueued)
_report(ticker.completed())
_hold_for_device(ticker, cancel)
def _finish_denoise() -> None:
"""Finish the bar once the denoise is over: H3 makes 29 scheduler.step calls for
30 steps."""
self._gen.update(step = steps, eta_seconds = None)
def _enter_decode_phase() -> None:
_finish_denoise()
self._gen.update(phase = "decode", eta_seconds = None)
def _on_decode() -> None:
"""Mark the decode boundary in the stream, retrying under contention; flip when
the GPU reaches it."""
if float_seeds is not None:
# Before the VAE decode's compile adds its own names.
float_seeds.keep()
marked = False
for _ in range(_BOUNDARY_MARK_ATTEMPTS):
with _hold_off_cuda_graph_capture() as clear:
if clear:
marked = ticker.mark_boundary()
break
time.sleep(_BOUNDARY_MARK_RETRY_SECONDS)
if not marked:
_enter_decode_phase()
if carry_by_scheduler:
logger.info(
"video.h3_stream_carry: %d carried copies read across forwards so far",
end_cross_forward(pipe),
)
def _pump() -> None:
"""One poll, inside the capture hold-off. Advances the step from the GPU, and
takes the denoise to complete only once the GPU has reached the marked end."""
if self._gen.get("phase") not in ("encode", "denoise"):
return
if ticker.boundary_marked and ticker.boundary_reached():
_enter_decode_phase()
return
_report(ticker.completed())
static_skip = state.transformer_cache == TC_STATIC
def _on_step(p, step_index, timestep, callback_kwargs):
if static_skip:
mark_step_end(_skip_pipe(state, pipe))
# Called after scheduler.step, so the step's latent update is already submitted.
if not cancel.is_set():
_tick(step_index + 1)
if previewer is not None:
previewer.on_step(
callback_kwargs.get("latents"), getattr(p, "scheduler", None)
)
if cancel.is_set():
p._interrupt = True
cross_forward_step(p, step_index, timestep = timestep)
return callback_kwargs
def _on_scheduler_step_cancel(done: int) -> None:
# Runs before the step; no cooperative _interrupt, so cancel by exception.
if cancel.is_set():
raise _VideoGenerationCancelled()
from .diffusion_nvfp4_protect import protect_generation
protect_ctx = protect_generation(pipe, steps, logger = logger)
has_step_callback = "callback_on_step_end" in call_params
if has_step_callback:
kwargs["callback_on_step_end"] = _on_step
carry_by_scheduler = _h3_carry_by_scheduler(fam, state, has_step_callback)
@contextlib.contextmanager
def progress_ctx():
"""Everything that reports on a denoise, entered as one unit around ``pipe()``."""
with contextlib.ExitStack() as stack:
if not has_step_callback:
# HunyuanVideo-1.5 and H3 modular have no step callback:
# wrap scheduler.step for progress + cancel.
stack.enter_context(
_scheduler_step_progress(
pipe,
_on_scheduler_step_cancel,
_scheduler_step_carry(pipe, steps, _tick, cancel.is_set)
if carry_by_scheduler
else _tick,
)
)
stack.enter_context(scheduler_step_preview(pipe, previewer))
stack.enter_context(_denoise_phase(pipe, _enter_denoise))
if fam.name == "minimax-h3":
from .video_minimax_h3_forward import h3_render_scope
# H3's text stream runs once inside this render, dropped after it.
stack.enter_context(h3_render_scope(pipe, logger))
if float_seeds is not None:
stack.enter_context(_seed_before_first_forward(float_seeds))
# No video family has a denoise-to-decode callback, so the decoder itself reports the decode phase.
stack.enter_context(_decode_phase(pipe, _on_decode))
# NaN / inf pixels are refused at the decode, before a clamp or PIL / uint8 cast hides them.
stack.enter_context(finite_video_decode(pipe))
# The decoded clip becomes uint8 on the GPU, bit-identical to the np / pil export (video_frames).
stack.enter_context(uint8_video_frames(pipe))
if stream_session is not None:
stack.enter_context(stream_session.hooks())
stack.enter_context(_completed_step_poller(_pump))
yield
# Re-check an AUTO cache decision against the ACTUAL step count; explicit choices never toggle
if state.cache_auto:
# Judged against the runner (an I2V class swap shares the denoiser): FBCache hooks need it to
# open cache_context, which every registered I2V pipeline does today (diffusers 0.41).
from .diffusion_autoopt.runtime import auto_cache_steps
cache_steps, runner_reason = auto_cache_steps(state.pipe, pipe, steps)
toggled = state.transformer_cache
for view in _views_for(pipe, fam):
toggled = maybe_toggle_step_cache(
view,
steps = cache_steps,
quant_active = state.cache_quant_active,
threshold = state.cache_threshold,
logger = logger,
)
from . import video_stream_residency
# the peak this request records belongs to the cache mode that actually runs
video_stream_residency.retag(state.pipe, "transformer_cache", toggled)
if toggled != state.transformer_cache:
# _VideoLoadState is frozen; record the pipe-level toggle so status() is truthful.
object.__setattr__(state, "transformer_cache", toggled)
entry = (state.resolved or {}).get("transformer_cache")
if isinstance(entry, dict):
entry["value"] = toggled or "off"
entry["reason"] = (
f"auto: {runner_reason}"
if runner_reason
else f"auto: {steps}-step generation "
+ ("reaches" if toggled else "is below")
+ f" {FBCACHE_MIN_STEPS}"
)
# A step cache keys residuals on cond / uncond context, invisible to a graph key: graphs armed with the cache off run eager.
graph_handles = tuple(getattr(state.pipe, "_unsloth_cuda_graphs", ()) or ())
if graph_handles:
from . import diffusion_cuda_graph
diffusion_cuda_graph.set_bypass(
graph_handles, cache_breaks_graph(state.transformer_cache)
)
if static_skip:
# Auto skip was measured on text-to-video only.
conditioned = bool(first_pil is not None or last_pil is not None or references)
auto_unmeasured = conditioned and static_skip_is_auto(_skip_pipe(state, pipe))
# Without a step callback (HunyuanVideo-1.5) steps count per CFG branch from cache_context names.
reset_static_step_skip(
_skip_pipe(state, pipe),
steps,
step_signal = has_step_callback,
owner = current_account_id(),
compute_all = auto_unmeasured,
)
elif state.transformer_cache:
self._reset_step_cache(pipe)
# torchao weights fail a hooked weight move under inference_mode (see _render_under_no_grad).
grad_ctx = (
torch.no_grad()
if state.transformer_quant and getattr(state, "denoiser_streamed", False)
else torch.inference_mode()
if not _render_under_no_grad(state)
else torch.no_grad()
)
stream_session = open_stream_encode(
pipe,
out_fps,
engage = not fam.modular_workflow
and not fam.has_audio
and ltx25_render_call is None,
logger = logger,
) or open_h3_stream_encode(
pipe,
out_fps,
engage = fam.modular_workflow and ltx25_render_call is None,
logger = logger,
)
previewer = LatentPreviewer.create(
family = fam.name,
requested = live_preview,
height = height,
width = width,
device = state.device,
publish = _publish_preview,
total_steps = steps,
)
render_call = (
ltx25_render_call
if ltx25_render_call is not None
else (
lambda: _carried_call(pipe, kwargs, has_step_callback or carry_by_scheduler)
)
)
def _render_streamed_after_oom() -> Any:
try:
return render_call()
except Exception as exc: # noqa: BLE001
if static_skip or cancel.is_set() or not is_oom_error(exc):
raise
from . import video_stream_residency
if not video_stream_residency.may_stream_after_oom(state.pipe):
raise
from . import diffusion_cuda_graph, video_stream_residency
# outside the handler so the failed attempt's tensors are freed; graph pools escape empty_cache()
diffusion_cuda_graph.reset_all(
getattr(state.pipe, "_unsloth_cuda_graphs", ()) or ()
)
video_stream_residency.stream_after_oom(state.pipe, logger = logger)
from .diffusion_nvfp4_protect import restart_protect_generation
restart_protect_generation(pipe)
if state.transformer_cache:
self._reset_step_cache(pipe)
seeded = kwargs.get("generator")
if isinstance(seeded, torch.Generator):
seeded.manual_seed(int(seed))
return render_call()
try:
with grad_ctx, protect_ctx, progress_ctx(), sigma_ctx:
output = render_thread.run("video", _render_streamed_after_oom)
except _VideoGenerationCancelled:
# Unwinding by exception skips maybe_free_model_hooks(); offloaded modules would stay on GPU.
free_hooks = getattr(pipe, "maybe_free_model_hooks", None)
if callable(free_hooks):
try:
free_hooks()
except Exception: # noqa: BLE001 -- cleanup is best-effort
pass
raise RuntimeError(VIDEO_CANCELLED_MSG) from None
finally:
# A guarded block that failed to compile runs eager; status must stop reporting it compiled.
if previewer is not None:
previewer.finish()
settle_compile_fallback(state, pipe, logger)
if static_skip:
logger.debug(
"video.step_skip: %s", static_skip_stats(_skip_pipe(state, pipe))
)
reset_static_step_skip(_skip_pipe(state, pipe), None)
if fam.modular_workflow:
from .video_minimax_h3_vae import settle_h3_vae_fallback
settle_h3_vae_fallback(state, pipe)
from .video_minimax_h3_residency import park_h3_rotation
# The next request's VRAM floor reads free memory before its first forward evicts these.
park_h3_rotation(pipe)
if cancel.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
# Pipeline returned, so all steps ran; also the only place a latent-only render can complete.
_finish_denoise()
self._gen.update(phase = "export", eta_seconds = None)
if fam.modular_workflow:
video_frames = output["videos"][0]
audio = output.get("audio")
audio_track = audio[0] if audio is not None else None
audio_sample_rate = output.get("sampling_rate")
elif fam.name in COSMOS3_FAMILIES:
video_frames = (
output.video
) # Cosmos3OmniPipelineOutput: one clip, already unbatched
audio_track = None
audio_sample_rate = None
else:
video_frames = output.frames[0]
audio = getattr(output, "audio", None)
audio_track = audio[0] if fam.has_audio and audio is not None else None
audio_sample_rate = None
# Encoded during the decode when its frames (and soundtrack) are exactly these (_encode_mp4 claims it).
with offer_stream(stream_session):
if audio_sample_rate is None:
mp4_bytes = self._encode_mp4(
video_frames, out_fps, audio_track, pipe if fam.has_audio else None
)
else:
mp4_bytes = self._encode_mp4(
video_frames,
out_fps,
audio_track,
pipe if fam.has_audio else None,
audio_sample_rate = int(audio_sample_rate),
)
# A cancel during the blocking export/mux must still discard the clip; re-check before it is persisted.
if cancel.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
# The status badge reports the attention routes this run took (report only, keys are unchanged).
report_attention_run(state)
if len(video_frames) and not fam.modular_workflow:
from . import video_stream_residency
# after decode and export: this peak sizes the next request of its size
video_stream_residency.record_request_peak(pipe, logger = logger)
duration_s = len(video_frames) / float(out_fps) if out_fps else 0.0
self._gen = {"active": False}
# Deregister under cancel_generate's lock before the trim, or a late Stop is answered true wrongly.
with self._lock:
if cancel.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
if self._active_generate_cancel is cancel:
self._active_generate_cancel = None
if bg_eager:
_settle_bg_compile(bg, i2v = _bg_i2v_render(first_pil, i2v_active))
reclaim_offload_host_memory(state.offload_policy, logger = logger)
return {
"mp4_bytes": mp4_bytes,
"seed": int(seed),
"repo_id": state.display_repo_id or state.repo_id,
"width": width,
"height": height,
"num_frames": len(video_frames),
"fps": out_fps,
"duration_s": duration_s,
"has_audio": bool(audio_track is not None),
"conditioning": conditioning,
"steps": recorded_steps,
"guidance": guidance,
# Records the effective negative prompt, not the caller's; distilled families normalise it away.
"negative_prompt": negative_prompt,
"flow_shift": shift,
"audio_flow_shift": audio_shift,
# From the engaged state, never the request.
"model_kind": state.kind,
"gguf_filename": state.gguf_filename,
"transformer_quant": state.transformer_quant,
"text_encoder_quant": state.text_encoder_quant,
"memory_mode": state.memory_mode,
"offload_policy": state.offload_policy,
}
except Exception as exc:
self._gen = {"active": False}
if is_oom_error(exc):
# Drop graphs on any CUDA OOM: live graphs pin pool memory empty_cache() cannot reclaim.
try:
from . import diffusion_cuda_graph
diffusion_cuda_graph.reset_all(
getattr(getattr(state, "pipe", None), "_unsloth_cuda_graphs", ()) or ()
)
except Exception: # noqa: BLE001 -- cleanup must never mask the real failure
pass
_log_failed_generation(request_shape, exc)
raise
finally:
if stream_session is not None:
stream_session.close()
end_cfg_render(getattr(state, "pipe", None))
end_cfg_batch_render(getattr(state, "pipe", None))
try:
from . import video_stream_residency
video_stream_residency.discard_request(getattr(state, "pipe", None))
finally:
i2v_swap.close()
if bg_token is not None:
bg_compile._FORCE_EAGER.reset(bg_token)
with self._lock:
if self._active_generate_cancel is cancel:
self._active_generate_cancel = None
# The pinned clip and stream chunks (~3x the clip) otherwise stay in torch's host cache until unload.
output = video_frames = None
release_pinned_host_memory()
_release_held_cache_writes(getattr(state, "pipe", None), "the first render ended")
@staticmethod
def _resolve_keyframes(
fam: VideoFamily,
h3_task: Optional[str],
first_frame: Optional[str],
last_frame: Optional[str],
width: Optional[int],
height: Optional[int],
*,
decoded_images: Optional[list] = None,
) -> tuple[Any, Any, int, int, str]:
"""Decode keyframes onto one shared canvas; omitting both dimensions matches the source aspect."""
validate_video_keyframe_conditioning(
fam,
h3_task,
has_keyframes = bool(first_frame or last_frame),
has_last_frame = bool(last_frame),
)
if fam.name == "ltx-2.5":
return VideoBackend._resolve_ltx25_first_frame(
fam, first_frame, last_frame, width, height, decoded_images = decoded_images
)
first = last = None
if first_frame or last_frame:
from .video_minimax_h3 import take_decoded_image
limits = dict(max_side = 4096, max_pixels = None)
first = (
take_decoded_image(decoded_images, first_frame, **limits) if first_frame else None
)
last = take_decoded_image(decoded_images, last_frame, **limits) if last_frame else None
# i2v pipelines centre-crop onto the nearest-aspect preset, as the reference workflow does.
i2v_pipeline = video_family_uses_i2v_pipeline(fam)
if first is not None:
assert_first_frame_pipeline_available(fam)
if (width is None or height is None) and first is not None and i2v_pipeline:
width, height = first_frame_canvas(fam, *first.size)
elif (width is None or height is None) and (first is not None or last is not None):
width, height = h3_canvas_for_aspect(*(first if first is not None else last).size)
else:
# A custom family may declare no presets; fall back to 768x512.
default_w, default_h = (
fam.resolution_presets[0] if fam.resolution_presets else (768, 512)
)
width, height = snap_video_size(
fam,
width or default_w,
height or default_h,
)
# Fit once so both engines condition on identical pixels, unless the pipeline fits the frame itself.
if first is not None and fam.i2v_fits_first_frame:
anchor = H3_ANCHOR_LAST if i2v_pipeline else H3_ANCHOR_FIRST
first = fit_h3_keyframe(first, width, height, anchor = anchor)
if last is not None:
last = fit_h3_keyframe(last, width, height, anchor = H3_ANCHOR_LAST)
conditioning = h3_conditioning_mode(has_first = first is not None, has_last = last is not None)
return first, last, width, height, conditioning
@staticmethod
def _resolve_continuation(
fam: VideoFamily,
video: Optional[str],
keep: Optional[str],
*,
requested_size: tuple[Optional[int], Optional[int]],
canvas: tuple[int, int],
fps: Optional[int],
num_frames: Optional[int],
has_keyframes: bool,
has_references: bool,
) -> tuple[Any, int, int]:
"""Omitting both dimensions takes the preset nearest the clip's aspect, as a first frame does."""
validate_video_continuation(
fam,
has_video = bool(video),
keep = keep,
has_keyframes = has_keyframes,
has_references = has_references,
num_frames = (
snap_num_frames(fam, num_frames or fam.default_num_frames) if video else None
),
)
if not video:
return None, *canvas
from .video_cosmos3 import decode_continuation_clip, fit_continuation_frames
validate_video_fps(fam, fps)
keep = keep or DEFAULT_VIDEO_CONTINUATION_KEEP
clip = decode_continuation_clip(
_decode_b64_media(video),
keep = keep,
fps = int(fps or fam.default_fps),
count = COSMOS3_CONTINUATION_FRAMES,
)
width, height = canvas
if requested_size == (None, None):
width, height = first_frame_canvas(fam, *clip[0].size)
return (fit_continuation_frames(clip, width, height), keep), width, height
@staticmethod
def _resolve_ltx25_first_frame(
fam: VideoFamily,
first_frame: Optional[str],
last_frame: Optional[str],
width: Optional[int],
height: Optional[int],
*,
decoded_images: Optional[list] = None,
) -> tuple[Any, Any, int, int, str]:
"""LTX-2.5 image-to-video: a first frame only, on a "match source" canvas when no size is given."""
from .video_ltx25 import ltx25_canvas_for_aspect, prepare_ltx25_first_frame
if last_frame:
raise ValueError(
"LTX-2.5 animates from a start frame only; it takes no end frame. Remove the end frame."
)
first = None
if first_frame:
from .video_minimax_h3 import take_decoded_image
first = take_decoded_image(decoded_images, first_frame, max_side = 4096, max_pixels = None)
if (width is None or height is None) and first is not None:
width, height = ltx25_canvas_for_aspect(*first.size)
else:
default_w, default_h = fam.resolution_presets[0]
width, height = snap_video_size(fam, width or default_w, height or default_h)
if first is not None:
# Fitted to the canvas later, by LTX2ConditionPipeline after its CRF round trip, as the reference does.
first = prepare_ltx25_first_frame(first)
return first, None, width, height, h3_conditioning_mode(has_first = first is not None)
@staticmethod
def _resolve_flow_shifts(
fam: VideoFamily,
engine: str,
flow_shift: Optional[float],
audio_flow_shift: Optional[float],
) -> tuple[Optional[float], Optional[float]]:
"""Sigma shifts default to released values; sd.cpp rejects non-default audio shifts it
cannot apply."""
validate_video_flow_controls(fam, flow_shift, audio_flow_shift, engine = engine)
shift = flow_shift if flow_shift is not None else fam.default_flow_shift
audio_shift = (
audio_flow_shift if audio_flow_shift is not None else fam.default_audio_flow_shift
)
return shift, audio_shift
@staticmethod
def _resolve_references(
fam: VideoFamily,
h3_task: Optional[str],
engine: str,
reference_images: Optional[list[str]],
reference_videos: Optional[list[dict]],
reference_audios: Optional[list[str]],
reference_image_size: Optional[str],
width: int,
height: int,
*,
decoded_audios: Optional[Any] = None,
decoded_images: Optional[list] = None,
) -> Any:
"""Sizing is settled here so both engines' min(1, ...) scales are no-ops; native refuses max."""
from .video_minimax_h3 import (
MiniMaxH3References,
decode_h3_reference_audio,
decode_h3_reference_video,
validate_h3_reference_trim,
)
images = list(reference_images or [])
videos = list(reference_videos or [])
audios = list(reference_audios or [])
validate_video_reference_conditioning(
fam,
h3_task,
has_references = bool(images or videos or audios),
reference_image_size = reference_image_size,
engine = engine,
)
if not (images or videos or audios):
return MiniMaxH3References()
policy = (reference_image_size or H3_REF_SIZE_MATCH).strip().lower()
from .video_minimax_h3 import take_decoded_image
fitted_images = tuple(
fit_h3_reference_image(
take_decoded_image(
decoded_images,
item,
max_side = H3_REF_IMAGE_SOURCE_MAX_SIDE,
max_pixels = H3_REF_IMAGE_SOURCE_MAX_PIXELS,
),
width = width,
height = height,
policy = policy,
)
for item in images
)
decoded_videos = []
for item in videos:
blob = _decode_b64_media(item.get("video") if isinstance(item, dict) else item)
trim_start = item.get("trim_start_seconds") if isinstance(item, dict) else None
trim_end = item.get("trim_end_seconds") if isinstance(item, dict) else None
override = item.get("audio") if isinstance(item, dict) else None
frames, waveform, sample_rate = decode_h3_reference_video(
blob,
trim_start_seconds = trim_start,
trim_end_seconds = trim_end,
decode_audio = not bool(override),
)
if override:
# A replacement soundtrack has its own timeline; only the embedded track shares the video's.
clip = validate_h3_reference_trim(trim_start, trim_end)
waveform, sample_rate = decode_h3_reference_audio(
_decode_b64_media(override),
trim_start_seconds = None if clip is None else 0.0,
trim_end_seconds = None if clip is None else clip[1] - clip[0],
)
decoded_videos.append((frames, waveform, sample_rate))
if engine == "sd_cpp":
soundtrack_gap = False
for index, (_, waveform, _) in enumerate(decoded_videos, start = 1):
if waveform is None:
soundtrack_gap = True
elif soundtrack_gap:
raise ValueError(
"stable-diffusion.cpp can pair reference-video soundtracks only from "
f"Video 1 onward without gaps, but Video {index} has audio after a "
"silent earlier video. Reorder the videos so ones with soundtracks "
"come first, or add a replacement soundtrack to each earlier video."
)
# The route's readability check may have decoded these exact inputs already.
reused_audios = decoded_audios.for_inputs(audios) if decoded_audios is not None else None
if reused_audios is None:
reused_audios = tuple(
decode_h3_reference_audio(_decode_b64_media(item)) for item in audios
)
return MiniMaxH3References(
images = fitted_images,
videos = tuple(decoded_videos),
audios = reused_audios,
)
def _h3_native_server_render(
self,
runtime: Any,
params: Any,
*,
output_path: Path,
on_log: Callable[[str], None],
cancel: threading.Event,
flags: Optional[list[str]] = None,
env: Optional[dict[str, str]] = None,
memory_mode: Optional[str] = None,
) -> Optional[Path]:
"""Render one MiniMax-H3 clip on the runtime's resident sd-server, spawned with this render's sd-cli ``flags`` /
``env``; a server that has to be started is fitted on live readings first (``memory_mode``).
Returns the written container, or None for the one-shot sd-cli: no slot, kill switch, reference video / audio,
or a server that cannot start, lacks vid_gen, or died (the runtime then stays one-shot). Every fallback first
stops the server so the two never hold the model at once. A job the live server fails is raised after stopping
it. Called inside the generation's managed-tree reader claim."""
import base64
from .sd_cpp_args import (
build_vid_gen_request,
h3_reference_media_keys,
h3_server_refs_enabled,
sd_cli_frame_files,
)
from .sd_cpp_engine import SdCppCancelled
from .sd_cpp_server import SdCppServerUnsupported
from .video_minimax_h3 import H3NativeMemoryRefusal
from .video_minimax_h3 import h3_native_compute_bytes, h3_native_server_enabled
slot = getattr(runtime, "server_slot", None)
if slot is None or slot.disabled_reason is not None:
return None
if not h3_native_server_enabled():
slot.stop("UNSLOTH_H3_NATIVE_SERVER=0")
return None
ref_media = h3_reference_media_keys(params)
if ref_media and (not h3_server_refs_enabled() or getattr(slot, "refs_unsupported", False)):
# The sd-cli loads the whole model again; a resident copy beside it may not fit.
slot.release("reference video / audio render runs on sd-cli")
return None
def b64_file(path: Optional[str]) -> Optional[str]:
if not path:
return None
return base64.b64encode(Path(path).read_bytes()).decode("ascii")
payload = build_vid_gen_request(
params,
images_b64 = {
"init_image": b64_file(params.init_img),
"end_image": b64_file(params.end_img),
},
ref_images_b64 = [b64_file(p) for p in params.ref_images] or None,
ref_videos_b64 = [
[b64_file(f) for f in sd_cli_frame_files(d)] for d in params.ref_videos
],
ref_video_audios_b64 = [b64_file(p) for p in params.ref_video_audios],
ref_audios_b64 = [b64_file(p) for p in params.ref_audios],
)
render_flags = list(runtime.offload_flags) if flags is None else list(flags)
render_env = dict(runtime.env) if env is None else dict(env)
from .sd_cpp_args import is_device_oom_text
slot.begin_render()
try:
retry_cap: Any = None
while True:
try:
server = slot.get(
render_flags,
render_env,
cancel_event = cancel,
pre_spawn = functools.partial(
_h3_launch_flags,
runtime,
render_flags,
memory_mode,
compute_bytes = h3_native_compute_bytes(
params.width, params.height, params.num_frames
),
),
launch_transform = functools.partial(
_h3_budgeted_launch_flags, runtime, cap = retry_cap
),
)
except (SdCppCancelled, H3NativeMemoryRefusal):
raise
except Exception as exc: # noqa: BLE001 - any start failure leaves the one-shot path
if cancel.is_set():
raise SdCppCancelled("sd-server start was cancelled.") from exc
if retry_cap is not None and is_device_oom_text(str(exc)):
slot.stop("render failed")
raise
# A start-time device OOM gets one smaller retry instead of disabling the server.
cap = _h3_retry_vram_cap(getattr(slot, "attempted_flags", None), str(exc))
if cap is not None and _h3_restart_for_retry(
slot, "server start out of memory"
):
logger.warning(
"video.h3_native_vram_retry: server start out of device memory; retrying once with %s",
cap,
)
retry_cap = cap
continue
logger.warning("video.h3_native_server_start_failed (using sd-cli): %s", exc)
slot.disable(f"start failed: {exc}")
return None
if ref_media and not server.vid_gen_accepts(ref_media):
# An older sd-server would drop the fields; end_render stops it before sd-cli loads.
slot.refs_unsupported = True
slot.release("this sd-server does not take reference video / audio")
return None
try:
data = server.vid_gen(payload, on_step = on_log, cancel_event = cancel)
except SdCppServerUnsupported as exc:
logger.warning("video.h3_native_server_unsupported (using sd-cli): %s", exc)
slot.disable(str(exc))
return None
except SdCppCancelled:
raise
except RuntimeError as exc:
if cancel.is_set():
raise SdCppCancelled("sd-server generation was cancelled.") from exc
cap = (
_h3_retry_vram_cap(getattr(slot, "launched_flags", None), str(exc))
if retry_cap is None
else None
)
if cap is not None and _h3_restart_for_retry(
slot, "device out of memory; retrying with a smaller VRAM budget"
):
logger.warning(
"video.h3_native_vram_retry: device out of memory under --max-vram %s; "
"retrying once with %s: %s",
dict(_pairs(getattr(slot, "launched_flags", ()))).get("--max-vram"),
cap,
str(exc).splitlines()[0] if str(exc) else exc,
)
retry_cap = cap
continue
if retry_cap is not None and is_device_oom_text(str(exc)):
slot.stop("render failed")
raise
if not server.is_alive():
logger.warning(
"video.h3_native_server_died (retrying once on sd-cli, staying one-shot): %s",
exc,
)
slot.disable(f"server died: {exc}")
return None
slot.stop("render failed")
raise
Path(output_path).write_bytes(data)
return Path(output_path)
finally:
try:
slot.end_render()
except Exception as exc: # noqa: BLE001 - the render's own outcome wins
logger.warning("video.h3_native_server_release_failed: %s", exc)
def _generate_h3_native(
self,
*,
state: _VideoLoadState,
prompt: str,
width: int,
height: int,
frames: int,
fps: int,
steps: int,
guidance: float,
seed: Optional[int],
# Canvas-fitted keyframes, and the task name they add up to (see _resolve_keyframes).
first_frame: Any,
last_frame: Any,
references: Any,
conditioning: str,
flow_shift: Optional[float],
cancel: threading.Event,
) -> dict[str, Any]:
import random
import re
from .sd_cpp_args import SdCppVideoGenParams
from .sd_cpp_engine import SdCppCancelled
from .video_minimax_h3 import (
h3_native_compute_bytes,
h3_native_render_flags,
h3_native_resident_bytes,
inspect_video,
stage_h3_references,
transcode_video_to_mp4,
)
runtime = state.pipe
render_flags = list(runtime.offload_flags)
render_resident = False
if "--offload-to-cpu" in render_flags and state.memory_mode in (None, "auto"):
try:
files = runtime.files
file_bytes = sum(
os.path.getsize(p)
for p in (files.diffusion_model, files.llm, files.vae, files.audio_vae)
if p
)
need = h3_native_resident_bytes(file_bytes, width, height, frames)
except Exception: # noqa: BLE001 -- unknown size keeps the committed offload
need = None
free = None
if need:
# A live resident server's weights count as free; a larger clip's activations must still fit the rest.
slot = getattr(runtime, "server_slot", None)
live = slot.alive_signature() if slot is not None else None
resident_flags, _ = h3_native_render_flags(
runtime.offload_flags,
memory_mode = state.memory_mode,
free_bytes = need,
need_bytes = need,
)
if (
live is not None
and live[2] == tuple(resident_flags)
and "--offload-to-cpu" not in live[2]
):
card_free = _h3_card_free_bytes(state.device, state.gpu_ordinal)
free = None if card_free is None else card_free + file_bytes
else:
free = _h3_card_free_bytes(state.device, state.gpu_ordinal)
render_flags, render_resident = h3_native_render_flags(
runtime.offload_flags,
memory_mode = state.memory_mode,
free_bytes = free,
need_bytes = need,
)
# The GTT fit the load prepared for an offloaded Auto render; a resident one never needed it.
compute_bytes = h3_native_compute_bytes(width, height, frames)
render_flags = _h3_offloaded_render_flags(
runtime,
render_flags,
render_resident = render_resident,
memory_mode = state.memory_mode,
compute_bytes = compute_bytes,
)
if seed is None:
seed = random.SystemRandom().randrange(0, 2**53)
started = time.monotonic()
self._gen = {
"active": True,
"phase": "denoise",
"step": 0,
"total": steps,
"started": started,
"eta_seconds": None,
"error": None,
}
# sd.cpp reuses an unlabeled bar for load, denoise and VAE; count only bars in the denoise window.
denoise_open = re.compile(r"generate_video\s+\d+x\d+x\d+")
denoise_close = re.compile(r"sampling completed", re.IGNORECASE)
step_bar = re.compile(r"\|\s*(\d+)\s*/\s*(\d+)\s*-\s*[\d.]+\s*(?:it/s|s/it)")
denoising = False
cudnn = getattr(runtime, "cudnn", None)
if cudnn is not None:
cudnn.begin_render()
def on_log(line: str) -> None:
nonlocal denoising
if cudnn is not None:
cudnn.feed(line)
if denoise_open.search(line):
denoising = True
return
if denoise_close.search(line):
denoising = False
# sd.cpp still decodes both latent streams and writes video; don't show a finished denoise bar.
self._gen.update(phase = "decode", eta_seconds = None)
return
match = step_bar.search(line) if denoising else None
if match is None:
return
done, total = int(match.group(1)), int(match.group(2))
elapsed = time.monotonic() - started
self._gen.update(
step = done,
total = total,
eta_seconds = (elapsed / max(1, done)) * max(0, total - done),
)
tmp = tempfile.NamedTemporaryFile(suffix = ".webm", delete = False)
tmp.close()
output_path = Path(tmp.name)
# Stage in-memory conditioning in sd-cli's temporary disk format.
with tempfile.TemporaryDirectory(prefix = "unsloth-h3-keyframes-") as scratch:
def stage(image: Any, name: str) -> Optional[str]:
if image is None:
return None
path = Path(scratch) / f"{name}.png"
image.save(path, format = "PNG", compress_level = 1)
return str(path)
init_img = stage(first_frame, "first")
end_img = stage(last_frame, "last")
staged = stage_h3_references(references, Path(scratch))
# Take the reader claim, or an install sweep unlinks the CLI this runtime resolved at load.
from .sd_cpp_backend import _tree_reader
binary = getattr(runtime.engine, "binary", None)
# Compare against the identity recorded at load; an install may replace the binary in place.
binary_identity = getattr(runtime, "binary_identity", None)
try:
try:
with _tree_reader(binary, cancel, VIDEO_CANCELLED_MSG):
current_identity = _sd_cli_identity(binary)
# Unreadable now, or not the vetted build; with no recorded identity, only readability is checked.
if binary and (
current_identity is None
or (binary_identity is not None and current_identity != binary_identity)
):
# The engine is committed at load and cannot be re-resolved mid-generation.
raise RuntimeError(
"The stable-diffusion.cpp binary this model was loaded with was "
"replaced by an install, so it is no longer the build this model "
"was checked against. Reload the model and try again."
)
video_params = SdCppVideoGenParams(
prompt = prompt,
width = width,
height = height,
num_frames = frames,
fps = fps,
steps = steps,
cfg_scale = 1.0,
seed = int(seed),
init_img = init_img,
end_img = end_img,
ref_images = staged.images,
ref_videos = staged.videos,
ref_video_audios = staged.video_audios,
ref_audios = staged.audios,
flow_shift = flow_shift,
)
generated = self._h3_native_server_render(
runtime,
video_params,
output_path = output_path,
on_log = on_log,
cancel = cancel,
flags = render_flags,
env = dict(runtime.env),
memory_mode = state.memory_mode,
)
if generated is None:
# A new process (reference video / audio, no server, or a server that could not serve):
# fit it on live readings, never on flags a live server was shortcut to.
render_flags = _h3_offloaded_render_flags(
runtime,
render_flags,
render_resident = render_resident,
memory_mode = state.memory_mode,
reuse_live_server = False,
compute_bytes = compute_bytes,
)
from .sd_cpp_backend import _explicit_threads
generated = _h3_one_shot_with_vram_retry(
runtime,
lambda launch_flags: runtime.engine.generate_video(
runtime.files,
video_params,
output_path = str(output_path),
offload = launch_flags,
# sd.cpp's default undercounts M5 Pro / Max and ignores env / cgroup quota
threads = _explicit_threads(),
env = dict(runtime.env) or None,
on_log = on_log,
cancel_event = cancel,
),
render_flags,
cancel,
)
except SdCppCancelled:
raise RuntimeError(VIDEO_CANCELLED_MSG) from None
except RuntimeError as exc:
# #9278: the build starts, then dies in hipBLAS mid-render.
if not cancel.is_set() and VIDEO_CANCELLED_MSG not in str(exc):
_note_sd_cpp_accelerator_failure(
binary, str(exc), card = getattr(runtime, "selected_card", None)
)
raise
if cancel.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
if cudnn is not None:
cudnn.end_render()
self._gen.update(phase = "export", eta_seconds = None)
_refuse_h3_native_nan_render(generated)
actual_width, actual_height, actual_frames, has_audio = inspect_video(generated)
mp4_bytes = transcode_video_to_mp4(generated, fps = fps)
if cancel.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
self._gen = {"active": False}
return {
"mp4_bytes": mp4_bytes,
"seed": int(seed),
"repo_id": state.display_repo_id or state.repo_id,
"width": actual_width,
"height": actual_height,
"num_frames": actual_frames,
"fps": fps,
"duration_s": actual_frames / float(fps),
"has_audio": has_audio,
"conditioning": conditioning,
"steps": steps,
"guidance": guidance,
# sd-cli vid_gen has no negative-prompt input.
"negative_prompt": None,
"flow_shift": flow_shift,
# sd.cpp pins the audio schedule, so the recipe records what it actually ran.
"audio_flow_shift": state.family.default_audio_flow_shift,
# Built from the ENGAGED state, not the request; without these six, native GGUF recipes save blank.
"model_kind": state.kind,
"gguf_filename": state.gguf_filename,
"transformer_quant": state.transformer_quant,
"text_encoder_quant": state.text_encoder_quant,
"memory_mode": state.memory_mode,
"offload_policy": "none" if render_resident else state.offload_policy,
}
finally:
output_path.unlink(missing_ok = True)
@staticmethod
def _encode_mp4(
video_frames,
fps: int,
audio,
pipe,
*,
audio_sample_rate: Optional[int] = None,
) -> bytes:
"""Encode frames (+ optional audio) to H.264 MP4 bytes via diffusers' PyAV
exporter. A temp file bridges the exporter's path-based API; the bytes are
what the gallery persists."""
if audio is None:
streamed = claim_stream(video_frames, fps)
if streamed is not None:
return streamed
elif pipe is not None and audio_sample_rate:
# MiniMax-H3: encoded during the decode with this soundtrack's stream declared up front
streamed = claim_stream(video_frames, fps, audio, int(audio_sample_rate))
if streamed is not None:
return streamed
require_finite_frames(video_frames)
from diffusers.utils.export_utils import encode_video
tmp = tempfile.NamedTemporaryFile(suffix = ".mp4", delete = False)
tmp.close()
try:
encode_kwargs: dict[str, Any] = {}
if audio is not None and pipe is not None:
encode_kwargs["audio"] = audio
sample_rate = audio_sample_rate or getattr(
getattr(getattr(pipe, "vocoder", None), "config", None),
"output_sampling_rate",
None,
)
if sample_rate:
encode_kwargs["audio_sample_rate"] = int(sample_rate)
gpu = nvenc_gpu(logger = logger)
if gpu is None or not encode_nvenc(
video_frames,
fps,
tmp.name,
gpu,
encode_kwargs.get("audio"),
encode_kwargs.get("audio_sample_rate"),
):
if not encode_x264(
video_frames,
fps,
tmp.name,
encode_kwargs.get("audio"),
encode_kwargs.get("audio_sample_rate"),
):
encode_video(video_frames, fps, tmp.name, **encode_kwargs)
return Path(tmp.name).read_bytes()
finally:
try:
os.unlink(tmp.name)
except OSError:
pass
def generate_progress(self, expected_account: Optional[str] = None) -> Optional[dict[str, Any]]:
"""Progress under one lock; None if ``expected_account`` no longer owns the job."""
with self._lock:
if expected_account is not None and self._generate_job_account != expected_account:
return None
gen = dict(self._gen)
# generate() swaps in a bare active=False dict before the terminal one; report active across that.
if self._generate_job_active:
gen["active"] = True
if self._gen_video_id is not None:
gen["video_id"] = self._gen_video_id
gen.setdefault("active", False)
# Mirror image endpoint names (total_steps / fraction) beside native "total" for parity.
total = int(gen.get("total") or 0)
step = int(gen.get("step") or 0)
gen["total_steps"] = total
gen["fraction"] = min(1.0, step / total) if total > 0 else 0.0
return gen
def forget_terminal_video(self, video_id: Optional[str] = None) -> bool:
"""Drop a deleted clip's terminal record, else the UI resurrects a ghost card; live jobs are
kept."""
with self._lock:
gen = self._gen
if self._generate_job_active or not isinstance(gen, dict):
return False
if gen.get("phase") != "completed":
return False
held = str((gen.get("video") or {}).get("id") or "")
if video_id is not None and held != str(video_id):
return False
self._gen = {"active": False}
return True
def cancel_generate(
self,
expected_video_id: Optional[str] = None,
expected_account: Optional[str] = None,
) -> bool:
"""Stop the in-flight generation; expected_* are rechecked under begin_generate's lock,
so a cancel of a finished job cannot hit its successor."""
with self._lock:
if expected_video_id is not None and self._gen_video_id != expected_video_id:
return False
if expected_account is not None and self._generate_job_account != expected_account:
return False
cancel = self._active_generate_cancel
if cancel is None:
return False
cancel.set()
return True
# ── teardown + status ────────────────────────────────────────────────────
def _release_native_server(self, reason: str) -> str:
"""Stop the committed model's idle H3 sd-server, keeping the model itself loaded (the slot respawns on its
next render). "stopped" when its memory is freed now, "pending" when it is mid-render (or starting) and will
stop at the render's end, "none" when there was no server. Never waits on a render."""
with self._lock:
state = self._state
return _release_h3_server_slot(
getattr(getattr(state, "pipe", None), "server_slot", None), reason
)
def _teardown_state_locked(self) -> None:
"""Free the committed state. The caller holds _generate_lock (no generation
in flight) AND _lock, so the whole teardown is one atomic step: a generation
cannot observe a half-torn-down backend."""
release_load_budget_fraction("video")
state, self._state = self._state, None
self._float_seeds = None
if state is not None:
server_slot = getattr(getattr(state, "pipe", None), "server_slot", None)
if server_slot is not None:
try:
server_slot.stop("unload")
except Exception as exc: # noqa: BLE001 - teardown is best effort
logger.warning("video.h3_native_server_stop_failed: %s", exc)
# An in-flight background compile still runs the denoiser the teardown frees.
if getattr(state, "bg_compile", None) is not None:
try:
state.bg_compile.close()
except Exception: # noqa: BLE001 -- teardown is best effort
pass
restore_backend_flags(state.backend_flags)
_release_rocm_tunableop(getattr(state, "pipe", None))
# Restore stock GGUF kernels so a later speed=off load is bit-identical.
from . import diffusion_gguf_compile
from . import diffusion_cuda_graph
from .diffusion_autoopt.patches import GROUP_VIDEO_FUSED, uninstall_group
diffusion_gguf_compile.uninstall_all()
# The fused Wan block forward is class-level: a later bf16 / speed=off load gets the stock one back.
uninstall_group(GROUP_VIDEO_FUSED)
from . import diffusion_prompt_cache
diffusion_prompt_cache.release(getattr(state, "pipe", None))
if getattr(getattr(state, "family", None), "name", None) == "ltx-2.5":
from .video_ltx25 import close_encoder
close_encoder(getattr(state, "pipe", None))
# A pinner still copying would hold its chunks past release_pinned_host_memory().
stop_background_pins(getattr(state, "pipe", None))
# Cast writes still read the components the teardown frees.
_discard_held_cache_writes(getattr(state, "pipe", None))
_finish_cast_writes(getattr(state, "pipe", None))
_finish_native_writes(getattr(state, "pipe", None))
_finish_resident_quant_write(getattr(state, "pipe", None))
uninstall_cfg_parallel(getattr(state, "pipe", None))
# Before clear_gpu_cache(), or the graph pool stays reserved.
diffusion_cuda_graph.uninstall_all(
getattr(getattr(state, "pipe", None), "_unsloth_cuda_graphs", ()) or ()
)
uninstall_static_step_skip(_skip_pipe(state))
try:
from .diffusion_nvfp4_linear import reset_nvfp4_state
reset_nvfp4_state()
except Exception: # noqa: BLE001 - teardown is best effort
pass
del state
try:
clear_gpu_cache()
finally:
# finally: a sticky CUDA fault must not keep the pinned chunks locked.
release_pinned_host_memory()
reclaim_host_memory(logger = logger)
@_invalidates_gpu_memory("video unload")
def unload(
self,
*,
expected_account: Optional[str] = None,
only_if: Optional[Callable[[], bool]] = None,
) -> Optional[dict[str, Any]]:
with self._lock:
# Rechecked under the load worker's lock, so a load that ended since costs nothing.
if only_if is not None and not only_if():
return None
if expected_account is not None:
from .gpu_arbiter import VIDEO, GpuBusyForAnotherAccountError
from hub.services.models.account_access import require_resident_control
if (
self._active_generate_cancel is not None
and self._generate_job_account != expected_account
):
raise GpuBusyForAnotherAccountError(VIDEO, 1)
require_resident_control(
VIDEO, self._state.repo_id if self._state is not None else None
)
self._load_token += 1
self._cancel_event.set()
self._loading = None
if self._active_generate_cancel is not None:
self._active_generate_cancel.set()
# Queued generations hold no cancel event, so the signal cannot reach them.
self._teardown_waiters += 1
# A cancelled LTX-2.5 load's encoder worker: the load thread may sit in from_pretrained for minutes, so stop
# the worker now rather than when that thread unwinds.
pending_encoder = getattr(self, "_ltx25_pending_encoder", None)
self._ltx25_pending_encoder = None
if pending_encoder is not None:
_stop_ltx25_encoder(pending_encoder[1])
# Barrier: wait for the signalled generation to exit before freeing the pipeline, else we report the VRAM free
# while the clip still holds it.
with self._generate_lock:
with self._lock:
try:
self._teardown_state_locked()
finally:
# clear_gpu_cache() raises on a sticky CUDA fault; an undrained fence refuses all later generations.
self._teardown_waiters -= 1
logger.info("video.unloaded")
return self.status()
def status(self) -> dict[str, Any]:
from utils.hardware import hardware as hw
state = self._state
supported, modular = _video_family_capabilities(
detect_device = state is not None
or (hw.DEVICE is not None and hw.DETECTION_COMPLETE.is_set())
)
if state is None:
return {
"loaded": False,
"repo_id": None,
"display_repo_id": None,
"family": None,
"supported_families": list(supported),
"modular_families": list(modular),
"base_repo": None,
"device": None,
"dtype": None,
"model_kind": None,
"engine": None,
"gguf_variant": None,
"offload_policy": None,
"vae_tiling": False,
"memory_mode": None,
"speed_mode": None,
"speed_optims": [],
"attention_backend": None,
"sd_cpp_cudnn_attention": None,
"sd_cpp_cudnn_reason": None,
"transformer_cache": None,
"transformer_cache_stats": None,
"transformer_quant": None,
"transformer_quant_backend": None,
"transformer_quant_backend_reason": None,
"text_encoder_quant": None,
"has_audio": False,
"supports_cfg": True,
"supports_keyframes": False,
"supports_first_frame": False,
"supports_last_frame": False,
"supports_references": False,
"supports_video_continuation": False,
"h3_task": None,
"defaults": None,
"resolved": None,
}
from hub.utils.gguf import extract_quant_token
from . import diffusion_cuda_graph
resolved, speed_optims = diffusion_cuda_graph.live_status(
state.resolved,
state.speed_optims,
getattr(getattr(state, "pipe", None), "_unsloth_cuda_graphs", ()) or (),
)
fam = state.family
default_steps, default_guidance = default_video_generation_params(
state.variant_id,
state.gguf_filename,
state.display_repo_id,
state.repo_id,
state.base_repo,
fallback = (fam.default_steps, fam.default_guidance),
family = fam.name,
)
default_steps = _h3_pdd_default_steps(fam, getattr(state, "pipe", None), default_steps)
return {
"loaded": True,
"repo_id": state.repo_id,
"display_repo_id": state.display_repo_id,
"family": fam.name,
"supported_families": list(supported),
"modular_families": list(modular),
"base_repo": state.base_repo,
"device": state.device,
"dtype": state.dtype,
"model_kind": state.kind,
"engine": state.engine,
# As on the image runtime: dtype is compute precision, which labelled a Q4_K_M BF16.
"gguf_variant": (
extract_quant_token(state.gguf_filename)
if state.kind == "gguf" and state.gguf_filename
else None
),
"offload_policy": state.offload_policy,
"vae_tiling": state.vae_tiling,
"memory_mode": state.memory_mode,
"speed_mode": state.speed_mode,
"speed_optims": speed_optims,
"bg_compile": state.bg_compile.describe()
if getattr(state, "bg_compile", None) is not None
else None,
"attention_backend": state.attention_backend,
**_sd_cpp_cudnn_status(state),
"transformer_cache": state.transformer_cache,
"transformer_cache_stats": (
static_skip_stats(_skip_pipe(state))
if state.transformer_cache == TC_STATIC
else None
),
"transformer_quant": state.transformer_quant,
**_nvfp4_backend_fields(_video_transformer_quant_backend(state), owner = self),
"text_encoder_quant": state.text_encoder_quant,
"has_audio": fam.has_audio,
"prompt_hint": fam.prompt_hint,
"supports_cfg": fam.supports_cfg,
# Expose only conditioning supported by the resident partition.
"supports_keyframes": fam.supports_keyframes and state.h3_task != H3_TASK_REFERENCES,
# supports_keyframes above stays MiniMax-H3's contract for older frontends.
"supports_first_frame": video_family_accepts_first_frame(fam)
and state.h3_task != H3_TASK_REFERENCES
and (
not video_family_uses_i2v_pipeline(fam)
or _i2v_pipeline_available(fam.i2v_pipeline_class)
),
"supports_last_frame": video_family_accepts_last_frame(fam)
and state.h3_task != H3_TASK_REFERENCES,
"supports_references": fam.supports_references and state.h3_task == H3_TASK_REFERENCES,
"supports_video_continuation": fam.supports_video_continuation,
"h3_task": state.h3_task,
"defaults": {
"steps": default_steps,
"guidance": default_guidance,
"num_frames": fam.default_num_frames,
"fps": fam.default_fps,
"fps_range": list(fam.fps_range) if fam.fps_range else None,
"fps_choices": list(fam.fps_choices),
"min_num_frames": max(int(fam.frame_offset), int(fam.min_num_frames)),
"max_num_frames": fam.max_num_frames,
"frame_step": fam.frame_step,
"frame_offset": fam.frame_offset,
"duration_presets": list(fam.duration_presets),
"resolution_multiple": fam.resolution_multiple,
"resolution_presets": [list(p) for p in fam.resolution_presets],
**_custom_size_defaults(fam),
# Let the frontend preview the backend-owned "match source" rule.
"canvas_short_edge": _keyframe_canvas(fam)[0],
"canvas_max_pixels": _keyframe_canvas(fam)[1],
"canvas_from_presets": video_family_uses_i2v_pipeline(fam),
"flow_shift": fam.default_flow_shift,
"audio_flow_shift": fam.default_audio_flow_shift,
# sd.cpp has no flag for the audio shift, so the control is only real on Diffusers.
"supports_audio_flow_shift": fam.default_audio_flow_shift is not None
and state.engine != "sd_cpp",
},
"multi_gpu": multigpu_placement_status(getattr(state, "pipe", None)),
"cfg_batch": cfg_batch_status(getattr(state, "pipe", None)),
"graph_engagement": _graph_engagement(getattr(state, "pipe", None)),
"resolved": resolved,
}
def _custom_size_defaults(fam: Any) -> dict[str, Any]:
bounds = video_custom_size_bounds(fam)
if bounds is None:
return {}
return {
"custom_size_min_pixels": bounds[0],
"custom_size_max_pixels": bounds[1],
"custom_size_max_aspect": bounds[2],
}
def _keyframe_canvas(fam: Any) -> tuple[Optional[int], Optional[int]]:
"""(short edge, max pixels) of the family's "match source" keyframe canvas, or (None, None)."""
if not getattr(fam, "supports_keyframes", False):
return None, None
if getattr(fam, "name", None) == "ltx-2.5":
from .video_ltx25 import LTX25_CANVAS_MAX_PIXELS, LTX25_CANVAS_SHORT_EDGE
return LTX25_CANVAS_SHORT_EDGE, LTX25_CANVAS_MAX_PIXELS
return H3_CANVAS_SHORT_EDGE, H3_CANVAS_MAX_PIXELS
def _sd_cpp_cudnn_status(state: Any) -> dict[str, Any]:
"""Whether the native runtime's sd.cpp children run attention on cuDNN; nulls for every other runtime."""
cudnn = getattr(getattr(state, "pipe", None), "cudnn", None)
if cudnn is None or state.engine != "sd_cpp":
return {"sd_cpp_cudnn_attention": None, "sd_cpp_cudnn_reason": None}
return cudnn.status_fields()
_backend: Optional[VideoBackend] = None
_backend_lock = threading.Lock()
def _graph_engagement(pipe: Any) -> Optional[dict]:
"""``diffusion_cuda_graph.engagement`` for the loaded pipe; None when unreadable (never fails a status poll)."""
try:
from . import diffusion_cuda_graph
return diffusion_cuda_graph.engagement(
tuple(getattr(pipe, "_unsloth_cuda_graphs", ()) or ())
)
except Exception: # noqa: BLE001
return None
def get_video_backend() -> VideoBackend:
global _backend
with _backend_lock:
if _backend is None:
_backend = VideoBackend()
return _backend
def _family_shape_defaults(fam) -> dict:
return {
"num_frames": fam.default_num_frames,
"fps": fam.default_fps,
"frame_step": fam.frame_step,
"frame_offset": fam.frame_offset,
"resolution_presets": [list(p) for p in fam.resolution_presets],
}
def generation_in_flight() -> bool:
"""Read the background-job marker without constructing or locking the backend."""
backend = _backend
return backend is not None and bool(backend._generate_job_active)
def retire_load_for_account(account_id: str) -> bool:
"""Tear down an in-flight video load ``account_id`` started, without constructing the backend."""
from hub.services.models.account_access import retire_media_load
return retire_media_load("video", account_id, _backend)
def generation_account_in_flight() -> Optional[str]:
"""The account whose video job is running, read without constructing the backend."""
backend = _backend
if backend is None or not backend._generate_job_active:
return None
return backend._generate_job_account