Free, open-source AI video generator for Mac. Run LTX-2.5 and MiniMax H3 (Hailuo) locally on Apple Silicon: text-to-video and image-to-video with sound, lip-sync, character LoRA training, songs from lyrics (YuE2) and image editing (Qwen-Image-Edit). One-click install via Pinokio. Offline: no cloud, no API key, no NVIDIA GPU.
Python ai-filmmakingai-videoai-video-generatorapple-siliconhailuoimage-to-videolip-synclocal-ailora-trainingltx-2ltx-2-5ltx-videomacosminimax-h3mlxmusic-generationoffline-aipinokiotext-to-videovideo-generation
"""A2V Distilled Pipeline — audio-to-video with the distilled model.
Mirrors the upstream ``DistilledPipeline`` approach: two-stage generation
(half-res → upscale → full-res refine) with the distilled transformer
and no CFG guidance. Handles audio conditioning from an input audio file.
Requires only the distilled checkpoint (``transformer-distilled.safetensors``
or ``transformer.safetensors``) — no dev model needed. Q4 compatible.
"""
from __future__ import annotations
from pathlib import Path
import tempfile
import mlx.core as mx
from ltx_core_mlx.components.patchifiers import compute_video_latent_shape
from ltx_core_mlx.conditioning.types.latent_cond import LatentState
from ltx_core_mlx.model.audio_vae import encode_audio
from ltx_core_mlx.model.transformer.model import X0Model
from ltx_core_mlx.utils.audio import load_audio
from ltx_core_mlx.utils.memory import aggressive_cleanup
from ltx_core_mlx.utils.positions import (
compute_audio_positions,
compute_audio_token_count,
compute_video_positions,
)
from ltx_pipelines_mlx.a2vid_two_stage import A2VidPipelineTwoStage
from ltx_pipelines_mlx.scheduler import DISTILLED_SIGMAS, STAGE_2_SIGMAS
from ltx_pipelines_mlx.utils.helpers import create_noised_state
from ltx_pipelines_mlx.utils.progress import phase
from ltx_pipelines_mlx.utils.samplers import denoise_loop
_materialize = getattr(mx, "eval")
class A2VidDistilledPipeline(A2VidPipelineTwoStage):
"""Audio-to-video two-stage pipeline using the distilled model.
Uses the distilled transformer directly — no dev model, no CFG,
no LoRA fusion. Two-stage: half-res distilled -> upscale -> full-res
refine. Q4 compatible.
Args:
model_dir: Path to model weights or HuggingFace repo ID. Must
contain the distilled checkpoint (``transformer.safetensors``
or ``transformer-distilled.safetensors``).
gemma_model_id: Gemma model for text encoding.
low_memory: Aggressive memory management.
low_ram_streaming: Stream transformer blocks from disk.
"""
def load(self) -> None:
"""Load distilled DiT + upsampler (skip VAE encoder, skip decoders).
VAE encoder is loaded on demand by ``image_conditioner`` callbacks
during I2V conditioning and upscale steps — pre-loading it here
would waste ~800 MB peak memory for no benefit.
"""
if self._loaded:
return
if self.dit is None:
transformer_path = self.model_dir / "transformer.safetensors"
if not transformer_path.exists():
transformer_path = self.model_dir / "transformer-distilled.safetensors"
self.dit = self._load_transformer_with_optional_streaming(transformer_path)
if self.upsampler is None:
self._load_upsampler()
self._loaded = True
def generate_and_save(
self,
prompt: str,
output_path: str,
audio_path: str | Path | None = None,
height: int = 480,
width: int = 704,
num_frames: int = 97,
*,
frame_rate: float,
seed: int = 42,
stage1_steps: int | None = None,
stage2_steps: int | None = None,
image: str | None = None,
images=None,
audio_start_time: float = 0.0,
audio_max_duration: float | None = None,
audio_conditioning_scale: float = 1.0,
) -> str:
"""Generate audio-to-video using the distilled pipeline.
Two-stage: half-res distilled -> upscale -> full-res refine.
Audio conditioning from the input file (frozen in stage 1,
denoised in stage 2). Original audio muxed into output for
maximum fidelity.
Args:
prompt: Text prompt.
output_path: Path to output video file.
audio_path: Path to input audio file (required).
height: Video height.
width: Video width.
num_frames: Number of frames.
frame_rate: Frame rate.
seed: Random seed.
stage1_steps: Stage 1 steps (default: full DISTILLED_SIGMAS = 8).
stage2_steps: Stage 2 steps (default: full STAGE_2_SIGMAS = 3).
image: Optional reference image for I2V conditioning (first frame).
audio_start_time: Start time in seconds for audio.
audio_max_duration: Max audio duration.
audio_conditioning_scale: Amplify audio tokens before DiT (default: 1.0).
Higher values increase audio adhesion at the cost of visual flexibility.
Returns:
Path to the output video file.
"""
if self.low_memory:
mx.set_cache_limit(0)
if audio_path is None:
raise ValueError("audio_path is required for A2VidDistilledPipeline")
if audio_max_duration is None:
audio_max_duration = num_frames / frame_rate
# --- Encode audio ---
self._load_audio_encoder()
assert self.audio_encoder is not None
assert self.audio_processor is not None
audio_data = load_audio(
audio_path,
target_sample_rate=16000,
start_time=audio_start_time,
max_duration=audio_max_duration,
)
if audio_data is None:
raise ValueError(f"No audio found in {audio_path}")
audio_latent = encode_audio(
audio_data.waveform,
audio_data.sample_rate,
self.audio_encoder,
self.audio_processor,
)
audio_T = compute_audio_token_count(num_frames, frame_rate)
audio_latent = audio_latent[:, :, :audio_T, :]
audio_tokens, _ = self.audio_patchifier.patchify(audio_latent)
if audio_conditioning_scale != 1.0:
audio_tokens = audio_tokens * audio_conditioning_scale
mx.synchronize()
audio_T_actual = audio_tokens.shape[1]
if self.low_memory:
self.audio_conditioner.free()
# --- Text encoding (positive only, no CFG) ---
self._load_text_encoder()
with phase("Encoding prompt", verbose=self.verbose):
video_embeds, audio_embeds = self._encode_text(prompt)
_materialize(video_embeds, audio_embeds)
if self.low_memory:
self.prompt_encoder.free()
aggressive_cleanup()
# --- Load distilled DiT + upsampler (VAE encoder on demand) ---
self.load()
assert self.dit is not None
assert self.upsampler is not None
# --- Stage 1: half resolution ---
half_h, half_w = height // 2, width // 2
F, H_half, W_half = compute_video_latent_shape(num_frames, half_h, half_w)
video_shape = (1, F * H_half * W_half, 128)
video_positions_1 = compute_video_positions(F, H_half, W_half, frame_rate=frame_rate)
audio_positions = compute_audio_positions(audio_T_actual)
# I2V conditioning at half resolution
from ltx_pipelines_mlx.utils._orchestration import combined_image_conditionings
from ltx_pipelines_mlx.utils.args import ImageConditioningInput
enc_h_half = H_half * 32
enc_w_half = W_half * 32
resolved_images = list(images) if images else []
if image is not None and not resolved_images:
resolved_images = [ImageConditioningInput(path=image, frame_idx=0, strength=1.0)]
conditionings_1: list = []
if resolved_images:
def _encode_combined(encoder):
conds = combined_image_conditionings(
resolved_images,
enc_h=enc_h_half,
enc_w=enc_w_half,
spatial_dims=(F, H_half, W_half),
video_encoder=encoder,
frame_rate=frame_rate,
)
mx.synchronize()
return conds
conditionings_1 = self.image_conditioner(_encode_combined, free_after=self.low_memory)
if self.low_memory:
aggressive_cleanup()
# Video: noisy state. Audio: frozen from input (denoise_mask=0)
video_state_1 = create_noised_state(
base_shape=video_shape,
conditionings=conditionings_1,
spatial_dims=(F, H_half, W_half),
positions=video_positions_1,
seed=seed,
sigma=1.0,
initial_latent=None,
legacy_scalar_blend=True,
)
audio_state_1 = LatentState(
latent=audio_tokens,
clean_latent=audio_tokens,
denoise_mask=mx.zeros((1, audio_tokens.shape[1], 1), dtype=mx.bfloat16),
positions=audio_positions,
)
sigmas_1 = DISTILLED_SIGMAS[: stage1_steps + 1] if stage1_steps else DISTILLED_SIGMAS
x0_model = X0Model(self.dit)
self._pre_denoise_flush(video_state_1, audio_state_1)
output_1 = denoise_loop(
model=x0_model,
video_state=video_state_1,
audio_state=audio_state_1,
video_text_embeds=video_embeds,
audio_text_embeds=audio_embeds,
sigmas=sigmas_1,
)
if self.low_memory:
aggressive_cleanup()
# --- Upscale (denorm/upsample/renorm) ---
gen_tokens_1 = output_1.video_latent[:, : F * H_half * W_half, :]
video_half = self.video_patchifier.unpatchify(gen_tokens_1, (F, H_half, W_half))
H_full = H_half * 2
W_full = W_half * 2
def _upscale_and_optionally_encode(encoder):
v_mlx = video_half.transpose(0, 2, 3, 4, 1)
v_denorm = encoder.denormalize_latent(v_mlx).transpose(0, 4, 1, 2, 3)
v_up = self.upsampler(v_denorm)
v_up_renorm = encoder.normalize_latent(v_up.transpose(0, 2, 3, 4, 1)).transpose(0, 4, 1, 2, 3)
mx.synchronize()
conds: list = []
if resolved_images:
enc_h_full = H_full * 32
enc_w_full = W_full * 32
conds = combined_image_conditionings(
resolved_images,
enc_h=enc_h_full,
enc_w=enc_w_full,
spatial_dims=(F, H_full, W_full),
video_encoder=encoder,
frame_rate=frame_rate,
)
return v_up_renorm, conds
video_upscaled, conditionings_2 = self.image_conditioner(
_upscale_and_optionally_encode, free_after=self.low_memory
)
if self.low_memory:
self.upsampler = None
aggressive_cleanup()
# --- Stage 2: full resolution refine (no LoRA swap, already distilled) ---
video_tokens, _ = self.video_patchifier.patchify(video_upscaled)
sigmas_2 = STAGE_2_SIGMAS[: stage2_steps + 1] if stage2_steps else STAGE_2_SIGMAS
start_sigma = sigmas_2[0]
video_positions_2 = compute_video_positions(F, H_full, W_full, frame_rate=frame_rate)
video_state_2 = create_noised_state(
base_shape=video_tokens.shape,
conditionings=conditionings_2,
spatial_dims=(F, H_full, W_full),
positions=video_positions_2,
seed=seed + 2,
sigma=start_sigma,
initial_latent=video_tokens,
legacy_scalar_blend=True,
)
audio_state_2 = create_noised_state(
base_shape=audio_tokens.shape,
conditionings=[],
spatial_dims=(F, H_full, W_full),
positions=audio_positions,
seed=seed + 2,
sigma=start_sigma,
initial_latent=audio_tokens,
)
self._pre_denoise_flush(video_state_2, audio_state_2)
output_2 = denoise_loop(
model=x0_model,
video_state=video_state_2,
audio_state=audio_state_2,
video_text_embeds=video_embeds,
audio_text_embeds=audio_embeds,
sigmas=sigmas_2,
)
if self.low_memory:
aggressive_cleanup()
gen_tokens_2 = output_2.video_latent[:, : F * H_full * W_full, :]
video_latent = self.video_patchifier.unpatchify(gen_tokens_2, (F, H_full, W_full))
# --- Decode and save (mux original audio for max fidelity) ---
# Free DiT before loading decoders — saves ~11 GB peak.
self.dit = None
self._loaded = False
aggressive_cleanup()
self._load_decoders()
video_duration = num_frames / frame_rate
audio_data_48k = load_audio(
audio_path,
target_sample_rate=48000,
start_time=audio_start_time,
max_duration=video_duration,
)
if audio_data_48k is not None:
max_samples = int(video_duration * 48000)
waveform_48k = audio_data_48k.waveform[:, :, :max_samples]
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as _tmp:
temp_audio = _tmp.name
self._save_waveform(waveform_48k, temp_audio, sample_rate=48000)
else:
temp_audio = None
self.video_decoder_block.decode_and_stream(
video_latent,
output_path,
frame_rate=frame_rate,
audio_path=temp_audio,
)
if temp_audio is not None:
Path(temp_audio).unlink(missing_ok=True)
# Free all remaining Metal buffers so 24GB systems recover fully.
# Without this, VAE decoder (~1 GB), audio decoder + vocoder
# (~500 MB), and upsampler (~200 MB) stay pinned in Metal memory
# until process exit.
self.video_decoder_block.free()
self.audio_decoder_block.free()
self.upsampler = None
self.dit = None
self._loaded = False
aggressive_cleanup()
return output_path
__all__ = ["A2VidDistilledPipeline"]