Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""MiniMax-H3 clip datasets: frames must be 17*n+5, with no still-image path; torch-free geometry."""
from __future__ import annotations
import json
import math
import os
from pathlib import Path
from typing import Any, Callable, Optional
from utils.paths.path_utils import drop_appledouble_metadata
# Mirrored from diffusers.modular_pipelines.minimax_h3 because the trainer cannot import it; asserted against the live components so a move is caught.
H3_FPS = 24
# Video VAE chunks hold 17 pixel frames and keep 5 latent frames each, plus a 2-frame head.
H3_FRAMES_PER_CHUNK = 17
H3_LATENTS_PER_CHUNK = 5
H3_SPATIAL_COMPRESSION = 16
# The transformer's (t, h, w) patch. Only the spatial half is > 1.
H3_PATCH_T, H3_PATCH_H, H3_PATCH_W = 1, 2, 2
# Canvas must be whole 2x2 patches after 16x VAE compression, so the multiple is 16 * patch width.
H3_CANVAS_MULTIPLE = H3_SPATIAL_COMPRESSION * H3_PATCH_W
H3_AUDIO_SAMPLING_RATE = 32000
H3_AUDIO_LATENTS_PER_SECOND = 40
H3_AUDIO_CHANNELS = 2
# ~52 ms of a 5.17 s window: covers a container's short tail, refuses mostly-silent audio.
_MAX_AUDIO_PAD_FRACTION = 0.01
# About -80 dBFS: below any real noise floor, above codec rounding dust on digital silence.
_SILENT_AUDIO_PEAK = 1e-4
H3_AUDIO_LATENT_CHANNELS = 32
H3_VIDEO_LATENT_CHANNELS = 24
# Per-row modality tags, which index the transformer's AdaLN table.
H3_VIDEO_TAG, H3_TEXT_TAG, H3_AUDIO_TAG = 0, 1, 2
# The canvas the released checkpoint generates on.
H3_CANVAS_SHORT_EDGE = 768
H3_CANVAS_MAX_PIXELS = 768 * 1344
H3_MIN_ASPECT_RATIO = 1 / 4
H3_MAX_ASPECT_RATIO = 4
# ImageNet, the video VAE's pixel convention.
H3_PIXEL_MEAN = (0.485, 0.456, 0.406)
H3_PIXEL_STD = (0.229, 0.224, 0.225)
# ONE VAE chunk, far below the 5 s floor: attention is quadratic, short native-canvas clips keep spatial stats; the 22-frame rotary grid is a prefix.
H3_TRAIN_NUM_FRAMES = H3_FRAMES_PER_CHUNK + H3_LATENTS_PER_CHUNK
_VIDEO_EXTS = {".mp4", ".mov", ".mkv", ".webm", ".m4v", ".avi"}
_CAPTION_EXTS = (".txt", ".caption")
def h3_align_num_frames(num_frames: int) -> int:
"""Snap a frame count UP to the next ``17 * n + 5`` the video VAE can encode."""
if num_frames < 1:
raise ValueError(f"num_frames must be positive, got {num_frames}.")
while num_frames % H3_FRAMES_PER_CHUNK != H3_LATENTS_PER_CHUNK:
num_frames += 1
return num_frames
def h3_video_latent_frames(num_frames: int) -> int:
"""Latent frames the video VAE produces for an aligned frame count: ``5 * n + 2``."""
if num_frames % H3_FRAMES_PER_CHUNK != H3_LATENTS_PER_CHUNK:
raise ValueError(
f"num_frames must be of the form {H3_FRAMES_PER_CHUNK} * n + {H3_LATENTS_PER_CHUNK}, "
f"got {num_frames}."
)
return (num_frames - H3_LATENTS_PER_CHUNK) // H3_FRAMES_PER_CHUNK * H3_LATENTS_PER_CHUNK + 2
def h3_audio_latent_count(num_frames: int) -> int:
"""Audio latents (per channel) covering ``num_frames`` frames at 24 fps / 40 latents per s."""
return int(round(num_frames / H3_FPS * H3_AUDIO_LATENTS_PER_SECOND))
def h3_audio_sample_count(num_frames: int) -> int:
"""Latents times hop samples: the audio VAE's latent count then matches the reserved rows exactly."""
hop = H3_AUDIO_SAMPLING_RATE // H3_AUDIO_LATENTS_PER_SECOND
return h3_audio_latent_count(num_frames) * hop
def h3_rows_per_latent_frame(latent_height: int, latent_width: int) -> int:
return (latent_height // H3_PATCH_H) * (latent_width // H3_PATCH_W)
def h3_packed_sequence_length(
num_text_tokens: int, num_frames: int, height: int, width: int
) -> int:
"""Whole packed sequence for the t2va layout, the figure attention cost is quadratic in."""
latent_frames = h3_video_latent_frames(num_frames)
rows_per_frame = h3_rows_per_latent_frame(
height // H3_SPATIAL_COMPRESSION, width // H3_SPATIAL_COMPRESSION
)
audio_rows = h3_audio_latent_count(num_frames) * H3_AUDIO_CHANNELS
return num_text_tokens + audio_rows + latent_frames * rows_per_frame
def h3_train_canvas(
aspect_width: float,
aspect_height: float,
short_edge: int = H3_CANVAS_SHORT_EDGE,
max_pixels: Optional[int] = None,
) -> tuple[int, int]:
"""Mirrors resolve_canvas_size without diffusers, returning (width, height) where the pipeline returns (height, width); area cap scales with resolution."""
if aspect_width <= 0 or aspect_height <= 0:
raise ValueError(f"The aspect ratio must be positive, got {aspect_width}:{aspect_height}.")
ratio = aspect_width / aspect_height
if not H3_MIN_ASPECT_RATIO <= ratio <= H3_MAX_ASPECT_RATIO:
raise ValueError(
f"MiniMax-H3 was trained on aspect ratios from 1:4 to 4:1; this clip is "
f"{aspect_width:g}x{aspect_height:g} ({ratio:.2f}:1). Crop it first."
)
if max_pixels is None:
# The released cap, rescaled to the requested short edge: (1344/768) * short_edge^2.
max_pixels = int(H3_CANVAS_MAX_PIXELS * (short_edge / H3_CANVAS_SHORT_EDGE) ** 2)
if ratio >= 1.0:
width, height = short_edge * ratio, float(short_edge)
else:
width, height = float(short_edge), short_edge / ratio
area = width * height
if area > max_pixels:
scale = math.sqrt(max_pixels / area)
width, height = width * scale, height * scale
def snap(value: float) -> int:
return max(H3_CANVAS_MULTIPLE, round(value / H3_CANVAS_MULTIPLE) * H3_CANVAS_MULTIPLE)
return snap(width), snap(height)
def discover_clip_caption_pairs(
data_dir: str | os.PathLike[str],
*,
instance_prompt: Optional[str] = None,
caption_column: str = "text",
) -> list[tuple[str, str]]:
"""Caption rules match discover_image_caption_pairs; an empty sidecar suppresses metadata captions."""
root = Path(data_dir).expanduser()
if not root.is_dir():
raise FileNotFoundError(f"data_dir is not a directory: {data_dir}")
clips = drop_appledouble_metadata(
sorted(p for p in root.iterdir() if p.is_file() and p.suffix.lower() in _VIDEO_EXTS)
)
meta_caption: dict[str, str] = {}
for meta_name in ("metadata.jsonl", "captions.jsonl"):
meta_path = root / meta_name
if not meta_path.is_file():
continue
try:
meta_lines = meta_path.read_text(encoding = "utf-8").split("\n")
except (OSError, UnicodeError):
continue
for line in meta_lines:
line = line.strip()
if not line:
continue
try:
row = json.loads(line)
except (json.JSONDecodeError, TypeError):
continue
if not isinstance(row, dict):
continue
key = row.get("file_name") or row.get("video") or row.get("image") or row.get("file")
value = row.get(caption_column)
if key and value is not None:
meta_caption[str(key)] = str(value)
pairs: list[tuple[str, str]] = []
for clip in clips:
caption: Optional[str] = None
sidecar_present = False
for ext in _CAPTION_EXTS:
sidecar = clip.with_suffix(ext)
if sidecar.is_file():
sidecar_present = True
try:
caption = sidecar.read_text(encoding = "utf-8").strip()
except (OSError, UnicodeError):
caption = ""
break
if not sidecar_present:
caption = meta_caption.get(clip.name) or meta_caption.get(
clip.relative_to(root).as_posix()
)
if not caption and instance_prompt:
caption = instance_prompt
if caption:
pairs.append((str(clip), caption))
if not pairs:
raise ValueError(
"No captioned video clips found. MiniMax-H3 trains from clips with sound, not "
"stills: provide .mp4 / .mov / .mkv / .webm files plus a metadata.jsonl / "
"captions.jsonl, per-clip .txt captions, or an instance prompt."
)
return pairs
def decode_clip(
path: str | os.PathLike[str],
*,
num_frames: int,
width: int,
height: int,
on_note: Optional[Callable[[str], None]] = None,
) -> tuple[Any, Any]:
"""Decode to (frames, waveform) at 24 fps; only the first num_frames train. No audio track is refused: silence would teach the model to stop sounding."""
import av
import numpy as np
from PIL import Image
target_samples = h3_audio_sample_count(num_frames)
with av.open(str(path)) as container:
if not container.streams.video:
raise ValueError(f"{Path(path).name} carries no video track.")
if not container.streams.audio:
raise ValueError(
f"{Path(path).name} carries no audio track. MiniMax-H3 denoises video and audio "
f"in one packed sequence, so its training clips must have sound."
)
stream = container.streams.video[0]
source_fps = float(stream.average_rate or stream.guessed_rate or H3_FPS) or float(H3_FPS)
# Best effort: an unknown duration means no note, never a failed decode.
source_duration_s = 0.0
try:
if stream.duration is not None and stream.time_base is not None:
source_duration_s = float(stream.duration * stream.time_base)
elif getattr(container, "duration", None):
source_duration_s = float(container.duration) / 1_000_000.0
except Exception: # noqa: BLE001 -- a note is not worth failing a decode over
source_duration_s = 0.0
frames: list[Any] = []
next_target = 0
for source_index, frame in enumerate(container.decode(video = 0)):
if len(frames) >= num_frames:
break
if int(next_target * source_fps / H3_FPS) > source_index:
continue
image = frame.to_image().convert("RGB")
# Before the crop: the canvas is in display orientation.
image = apply_display_rotation(image, display_rotation_degrees(frame, stream), Image)
image = _cover_resize(image, width, height, Image)
while (
int(next_target * source_fps / H3_FPS) <= source_index and len(frames) < num_frames
):
frames.append(np.asarray(image, dtype = "uint8"))
next_target += 1
if len(frames) < num_frames:
raise ValueError(
f"{Path(path).name} decoded to {len(frames)} frames at {H3_FPS} fps, but a training "
f"clip needs {num_frames} ({num_frames / H3_FPS:.2f}s). Use longer clips."
)
if on_note is not None and source_duration_s > (num_frames / H3_FPS) * 1.05:
on_note(
f"{Path(path).name} is {source_duration_s:.1f}s; MiniMax-H3 trains its first "
f"{num_frames / H3_FPS:.2f}s and its caption is paired with that. Trim the clip to "
f"the part the caption describes."
)
waveform = _decode_clip_audio(path, target_samples, av, np)
return np.stack(frames), waveform
def display_rotation_degrees(frame: Any, stream: Any) -> int:
"""PyAV ignores the display matrix, so portrait phone clips cache sideways; returns 0 when unset or PyAV is too old, and never raises."""
import math
import struct
matrix = None
try:
from av.sidedata.sidedata import Type
entry = frame.side_data.get(Type.DISPLAYMATRIX)
if entry is not None:
raw = bytes(entry)
if len(raw) >= 36:
# Native byte order: the matrix is an in-memory int32[9], not a serialised field.
matrix = struct.unpack("=9i", raw[:36])
except Exception: # noqa: BLE001 -- no side-data API, or no matrix on this frame
matrix = None
if matrix is None:
# Legacy MOV/MP4 tag, still what older files carry.
try:
tag = (stream.metadata or {}).get("rotate")
return int(float(tag)) % 360 if tag is not None else 0
except Exception: # noqa: BLE001 -- an unparsable tag is not a decode failure
return 0
try:
conv = lambda v: v / (1 << 16) # noqa: E731
scale_x = math.hypot(conv(matrix[0]), conv(matrix[3]))
scale_y = math.hypot(conv(matrix[1]), conv(matrix[4]))
if not scale_x or not scale_y:
return 0
degrees = -math.degrees(math.atan2(conv(matrix[1]) / scale_y, conv(matrix[0]) / scale_x))
except Exception: # noqa: BLE001 -- a degenerate matrix means "no rotation", not a failure
return 0
theta = int(-round(degrees)) % 360
# Only the four square turns; anything else cannot be applied without resampling, and no camera writes one.
return theta if theta in (90, 180, 270) else 0
def apply_display_rotation(image: Any, theta: int, Image: Any) -> Any:
"""Rotate a decoded frame into display orientation. Verified against ffmpeg's autorotate."""
if theta == 90:
return image.transpose(Image.ROTATE_270)
if theta == 180:
return image.transpose(Image.ROTATE_180)
if theta == 270:
return image.transpose(Image.ROTATE_90)
return image
def _cover_resize(image: Any, width: int, height: int, Image: Any) -> Any:
"""Center cover-crop to the canvas aspect ratio, then resize to it. Never stretches."""
source_w, source_h = image.size
scale = max(width / source_w, height / source_h)
crop_w = min(source_w, math.ceil(width / scale))
crop_h = min(source_h, math.ceil(height / scale))
left = (source_w - crop_w) // 2
top = (source_h - crop_h) // 2
return image.resize(
(width, height), Image.LANCZOS, box = (left, top, left + crop_w, top + crop_h)
)
def _decode_clip_audio(path: Any, target_samples: int, av: Any, np: Any) -> Any:
"""Pads a short audio tail only up to _MAX_AUDIO_PAD_FRACTION, so the target is not mostly silence."""
resampler = av.AudioResampler(format = "flt", layout = "stereo", rate = H3_AUDIO_SAMPLING_RATE)
chunks = []
have = 0
with av.open(str(path)) as container:
# Only the training window: decoding the rest wastes time and can fail on unused damage.
for frame in container.decode(audio = 0):
for resampled in resampler.resample(frame):
block = resampled.to_ndarray().reshape(-1, H3_AUDIO_CHANNELS)
chunks.append(block)
have += block.shape[0]
if have >= target_samples:
break
if have < target_samples:
# Only when the stream ran out: the resampler's held-back tail is what the pad allowance measures.
for resampled in resampler.resample(None):
chunks.append(resampled.to_ndarray().reshape(-1, H3_AUDIO_CHANNELS))
if not chunks:
raise ValueError(f"{Path(path).name} decoded to no audio samples.")
samples = np.concatenate(chunks, axis = 0).astype("float32")[:target_samples]
if samples.shape[0] < target_samples:
missing = target_samples - samples.shape[0]
if missing > _MAX_AUDIO_PAD_FRACTION * target_samples:
have_s = samples.shape[0] / H3_AUDIO_SAMPLING_RATE
want_s = target_samples / H3_AUDIO_SAMPLING_RATE
raise ValueError(
f"{Path(path).name} carries {have_s:.2f}s of audio for a {want_s:.2f}s clip. "
f"MiniMax-H3 denoises video and audio together, so padding the rest with "
f"silence would train the adapter to stop generating sound. Use a clip whose "
f"soundtrack runs its full length."
)
samples = np.pad(samples, ((0, missing), (0, 0)))
# A muted track passes every check above and comes back all zeros.
if float(np.max(np.abs(samples))) <= _SILENT_AUDIO_PEAK:
raise ValueError(
f"{Path(path).name} has a soundtrack that is silent all the way through. "
f"MiniMax-H3 denoises video and audio together, so training on it would teach the "
f"adapter to stop generating sound. Use a clip whose soundtrack has audio in it, or "
f"take this one out of the dataset."
)
return np.ascontiguousarray(samples.T)