Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

9.5 KB
Raw Blame
"""MPS H3 decoding through the shared bounded readers and RGB writer."""
import gc
import json
from pathlib import Path
import time


def weight_cache_limits(allocator_bytes):
    """Bound optional retained weights by the actual admitted Metal capacity."""
    if type(allocator_bytes) is not int or allocator_bytes <= 0:
        raise ValueError('Decoder cache requires a positive admitted byte limit')
    # At 6 GiB the native decoder evicted 17 of 23 preloaded layers and loaded
    # 562 layers, versus 216 with the active-layer schedule. The 8 GiB retained
    # path loaded only 63 and was faster. Keep the 5 GiB cache plus 3 GiB of
    # workspace together; partial preloading below that point causes churn.
    retained = 0 if allocator_bytes < 8 * 2**30 else min(5 * 2**30, allocator_bytes - 3 * 2**30)
    return dict(resident_bytes=retained,
                working_reserve_bytes=2 * 2**30)


def tile_group_size(allocator_bytes, *, cache_weights=True):
    """Reuse streamed layers when the admitted allowance cannot retain a set."""
    limits = weight_cache_limits(allocator_bytes)
    if type(cache_weights) is not bool:
        raise ValueError('Decoder weight caching must be explicitly enabled or disabled')
    # Four simultaneous tile states were measured at 4 GiB. Smaller admitted
    # allowances keep one tile's activations instead of assuming those fit.
    return 4 if cache_weights and allocator_bytes >= 4 * 2**30 and not limits['resident_bytes'] else 0


def trim_decoded(frames, audio, continuation=None, *, hide_tail_frames=0, sample_rate=32000,
                 audio_lead_in_callback=None):
    """The shared seam clocks, applied before saving either decoded artifact."""
    from .continuation import validate_hide_tail, capture_audio_lead_in, trim_output
    validate_hide_tail(hide_tail_frames)
    if continuation is not None and audio_lead_in_callback is not None:
        audio_lead_in_callback(capture_audio_lead_in(audio, continuation, sample_rate))
    if hide_tail_frames:
        kept_frames = len(frames) - hide_tail_frames
        frames = frames[:kept_frames]
        audio = audio[:, :round(kept_frames * 40 / 24) * 800]
    return trim_output(frames, audio, continuation)


def decode_to_file(latents, audio_latents, output, *, base, artifacts_dir,
                   budget_bytes=4 * 2**30, reserve_bytes=2 * 2**30, cache_weights=True, output_size=None,
                   continuation=None, hide_tail_frames=0, audio_lead_in_callback=None):
    from .continuation import validate_hide_tail
    validate_hide_tail(hide_tail_frames)
    if type(cache_weights) is not bool:
        raise ValueError('Decoder weight caching must be explicitly enabled or disabled')
    from .macos_vdn import activate
    activate()
    import torch
    import numpy as np
    from diffusers import AutoencoderKLMiniMaxH3
    from diffusers.utils.export_utils import encode_video
    from src.inference.render import PIXEL_MEAN, PIXEL_STD, FPS
    from .backends import get_backend
    from .vae_weights import skeleton, load_parameters, load_audio_vae
    from .decode_stream import render_rgb
    from .system import system_memory
    backend = get_backend('mps')
    policy = backend.configure_budget(budget_bytes, reserve_bytes=reserve_bytes)
    cache_options = (dict(weight_cache_limits(policy['effective_allocator_limit_bytes']),
                          linear_dtype=torch.float16, preload=True) if cache_weights else {})
    tile_group = tile_group_size(policy['effective_allocator_limit_bytes'], cache_weights=cache_weights)
    output, artifacts_dir, base = Path(output), Path(artifacts_dir), Path(base)
    artifacts_dir.mkdir(parents=True, exist_ok=True)
    started = time.monotonic()
    vae = video_decoder = audio_vae = frames = mapping = None
    handles, specs = [], []
    completed = False
    def phase(text):
        print(json.dumps(dict(event='decode_phase', phase=text)), flush=True)
    def guard(module, inputs):
        if system_memory()['available_bytes'] < 2**30:
            raise MemoryError('MPS decoding reached the 1 GiB physical RAM emergency floor')
    def finite_video(module, inputs, output):
        if not bool(output.isfinite().all()):
            raise ValueError('MPS video decoder produced nonfinite pixels')
    try:
        with torch.no_grad():
            phase('Loading video VAE')
            vae = skeleton(AutoencoderKLMiniMaxH3, base / 'vae')
            vae.encoder = vae.quant_conv = None
            paths = tuple(sorted((base / 'vae').glob('*.safetensors')))
            load_parameters(vae, paths, device_for_name=lambda name:
                'meta' if name.startswith('decoder.transformer_blocks.') else 'mps',
                linear_fp16=False, mapped=False)
            # Constructor-computed RoPE buffers are not checkpoint parameters.
            for name, value in list(vae.named_buffers()):
                parent, _, leaf = name.rpartition('.')
                setattr(vae.get_submodule(parent) if parent else vae, leaf, value.to('mps'))
            vae.eval().requires_grad_(False)
            from .backends.mps_vae import install_mlx_attention
            mlx_attention = install_mlx_attention(vae)
            handles.append(vae.decoder.register_forward_hook(finite_video))
            for index, layer in enumerate(vae.decoder.transformer_blocks):
                specs.append((layer, paths, f'decoder.transformer_blocks.{index}.', ()))
                handles.append(layer.register_forward_pre_hook(guard))
            mean = torch.tensor(vae.config.latents_mean, device='mps').view(1, -1, 1, 1, 1)
            std = torch.tensor(vae.config.latents_std, device='mps').view(1, -1, 1, 1, 1)
            phase('Decoding the video')
            def progress(done, total, elapsed):
                print(json.dumps(dict(event='decode_progress', done=done, total=total,
                                      elapsed_seconds=elapsed)), flush=True)
            if tile_group:
                from .macos_decode_tiles import TileDecoder
                residency = video_decoder = TileDecoder(vae, backend, specs, group_tiles=tile_group)
            else:
                residency = backend.make_offloader(specs, linears={}, **cache_options)
                video_decoder = vae
            with residency, \
                    torch.autocast(device_type='mps', dtype=torch.float16, cache_enabled=False):
                frames, mapping, timings = render_rgb(video_decoder, latents.to('mps') * std + mean,
                    PIXEL_MEAN, PIXEL_STD, artifacts_dir / 'rgb.npy', progress=progress)
                video_residency = residency.stats()
            for handle in handles:
                handle.remove()
            handles.clear()
            specs.clear()
            residency.specs.clear()
            del mean, std
            layer = value = None
            vae = video_decoder = None
            gc.collect()
            backend.empty_cache()
            phase('Loading and decoding audio')
            audio_vae, audio_loading = load_audio_vae(base, device='mps')
            mean = torch.tensor(audio_vae.config.latents_mean, device='mps').view(1, -1, 1)
            std = torch.tensor(audio_vae.config.latents_std, device='mps').view(1, -1, 1)
            audio = audio_vae.decode(audio_latents.to('mps') * std + mean, return_dict=False)[0]
            audio = audio.float().permute(1, 0, 2)[0].cpu()
            backend.synchronize()
            if not bool(audio.isfinite().all()):
                raise ValueError('MPS audio decoder produced nonfinite samples')
            rate = audio_vae.config.sampling_rate
            frames, audio = trim_decoded(frames, audio, continuation, hide_tail_frames=hide_tail_frames,
                sample_rate=rate, audio_lead_in_callback=audio_lead_in_callback)
            if continuation is not None or hide_tail_frames:
                np.save(artifacts_dir / 'rgb-new.npy', frames, allow_pickle=False)
            np.save(artifacts_dir / 'audio.npy', audio.numpy(), allow_pickle=False)
            audio_vae = None
            del mean, std
            gc.collect()
            backend.empty_cache()
            phase('Saving MP4 and audio')
            output.parent.mkdir(parents=True, exist_ok=True)
            temporary = output.with_name(output.stem + '.partial' + output.suffix)
            if output_size is not None:
                # The model rendered a larger size; only the picture is scaled.
                from .output_fit import fit_frames
                encode_video(fit_frames(frames, *output_size), fps=FPS, output_path=str(temporary), audio=audio,
                             audio_sample_rate=rate)
            else:
                encode_video(frames, fps=FPS, output_path=str(temporary), audio=audio, audio_sample_rate=rate)
            temporary.replace(output)
            completed = True
            return dict(device_backend='mps', output=str(output), frames=len(frames),
                audio_sample_rate=rate, audio_shape=list(audio.shape), elapsed_seconds=time.monotonic()-started,
                policy=policy, video_residency=video_residency, audio_loading=audio_loading, timings=timings,
                precision='FP32 checkpoint; FP16 video autocast; FP32 audio decoder',
                mlx_attention_modules=mlx_attention)
    finally:
        for handle in handles:
            handle.remove()
        specs.clear()
        vae = video_decoder = audio_vae = frames = None
        if mapping is not None:
            mapping._mmap.close()
        if completed and (continuation is not None or hide_tail_frames):
            (artifacts_dir / 'rgb-new.npy').replace(artifacts_dir / 'rgb.npy')
        gc.collect()
        backend.empty_cache()