Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

16.5 KB
Raw Blame
"""Bound the pending Metal work and the live tensors of the original FP32 input-video encoder."""
import json
import time
from contextlib import contextmanager


def _lean_conv_ok(conv):
    return (getattr(conv, 'spatial_padding', None) == 1 and getattr(conv, 'temporal_padding', None) in (0, 2)
            and getattr(conv, 'spatial_padding_mode', None) == 'reflect' and conv.groups == 1
            and tuple(conv.padding) == (0, 0, 0) and conv.padding_mode == 'zeros')


def _is_resnet(module):
    names = ('norm1', 'conv1', 'norm2', 'conv2', 'conv_shortcut')
    return (type(module).__name__ == 'MiniMaxH3VideoResnetBlock3d' and all(hasattr(module, n) for n in names)
            and _lean_conv_ok(module.conv1) and _lean_conv_ok(module.conv2)
            and type(module.norm1).__name__ == 'MiniMaxH3VideoGroupNorm'
            and type(module.norm2).__name__ == 'MiniMaxH3VideoGroupNorm')


class _Hold:
    """The only reference to a tile-sized activation between two encoder stages.

    A caller's own variable or argument would keep the previous activation alive
    through the whole stage; the stage takes it out of here and drops it.
    """
    __slots__ = ('value',)

    def __init__(self, value):
        self.value = value

    def take(self):
        value, self.value = self.value, None
        return value


def _norm_silu(norm, box):
    """silu(norm(x)) as a (B, T, C, H, W) tensor viewed back as (B, C, T, H, W).

    `box` is a one-element list holding x; its copy is released as soon as the
    folded input exists, unless the caller keeps another reference.

    The pinned group norm folds the frame axis into the batch; the same call is
    made on the same values, only the intermediate copies are released early and
    the elementwise SiLU runs in place.
    """
    import torch
    x = box.pop()
    batch, channels, frames, height, width = x.shape
    folded = x.permute(0, 2, 1, 3, 4).contiguous().view(batch * frames, channels, 1, height, width)
    del x
    normed = torch.nn.GroupNorm.forward(norm, folded)
    del folded
    normed = normed.view(batch, frames, channels, height, width)
    return torch.nn.functional.silu(normed, inplace=True).permute(0, 2, 1, 3, 4)


def _padded(source, temporal):
    """`source` with a 1-pixel reflect border and `temporal` leading zero frames in one buffer.

    Same values as reflect pad followed by a constant temporal pad, written
    without the intermediate spatially padded copy.
    """
    import torch
    batch, channels, frames, height, width = source.shape
    out = torch.empty((batch, channels, frames + temporal, height + 2, width + 2),
                      dtype=source.dtype, device=source.device)
    if temporal:
        out[:, :, :temporal] = 0
    body = out[:, :, temporal:]
    body[:, :, :, 1:-1, 1:-1] = source
    body[:, :, :, 0, 1:-1] = source[:, :, :, 1, :]
    body[:, :, :, -1, 1:-1] = source[:, :, :, -2, :]
    body[:, :, :, :, 0] = body[:, :, :, :, 2]
    body[:, :, :, :, -1] = body[:, :, :, :, -3]
    return out


def _downsample_ok(module):
    conv = getattr(module, 'conv', None)
    return (type(module).__name__ == 'MiniMaxH3VideoDownsample3d' and conv is not None
            and getattr(module, 'spatial_stride', None) == 2 and getattr(module, 'spatial_padding_mode', None) == 'reflect'
            and getattr(conv, 'spatial_padding', None) == 0 and getattr(conv, 'temporal_padding', None) == 2
            and conv.groups == 1 and tuple(conv.padding) == (0, 0, 0) and conv.padding_mode == 'zeros')


def _lean_downsample(module, fence):
    """Reflect pad of one bottom/right pixel and the causal temporal zero pad in one buffer."""
    import torch
    def forward(hidden_states):
        if isinstance(hidden_states, _Hold):
            hidden_states = hidden_states.take()
        batch, channels, frames, height, width = hidden_states.shape
        padded = torch.empty((batch, channels, frames + 2, height + 1, width + 1),
                             dtype=hidden_states.dtype, device=hidden_states.device)
        padded[:, :, :2] = 0
        body = padded[:, :, 2:]
        body[:, :, :, :-1, :-1] = hidden_states
        body[:, :, :, -1, :-1] = hidden_states[:, :, :, -2, :]
        del hidden_states
        body[:, :, :, :, -1] = body[:, :, :, :, -3]
        hidden = torch.nn.Conv3d.forward(module.conv, padded)
        del padded, body
        fence()
        return hidden
    return forward


def _lean_resnet(block, fence, stats):
    """The original resnet forward with at most three activation-sized tensors alive.

    Every convolution still receives a contiguous, identically padded tensor and
    the same weights; the residual sum is the same two operands. With
    `stats['residual_offload']` the block input waits in host memory while the
    block runs (an exact copy), which keeps one more tile-sized tensor off the
    device at the cost of a copy there and back.
    """
    import torch
    def forward(hidden_states):
        if isinstance(hidden_states, _Hold):
            hidden_states = hidden_states.take()
        device = hidden_states.device
        offload = bool(stats.get('residual_offload')) and block.conv_shortcut is None
        residual = hidden_states.to('cpu', copy=True) if offload else hidden_states
        box = [hidden_states]
        del hidden_states
        activated = _norm_silu(block.norm1, box)
        padded = _padded(activated, block.conv1.temporal_padding)
        del activated
        hidden = torch.nn.Conv3d.forward(block.conv1, padded)
        del padded
        fence()
        box = [hidden]
        del hidden
        activated = _norm_silu(block.norm2, box)
        padded = _padded(activated, block.conv2.temporal_padding)
        del activated
        hidden = torch.nn.Conv3d.forward(block.conv2, padded)
        del padded
        fence()
        if block.conv_shortcut is not None:
            residual = block.conv_shortcut(residual)
        return hidden.add_(residual.to(device) if offload else residual)
    return forward


def _lean_encoder_ok(model):
    encoder = getattr(model, 'encoder', None)
    names = ('conv_in', 'down_blocks', 'norm_out', 'conv_out')
    if encoder is None or not all(hasattr(encoder, n) for n in names) or getattr(model, 'quant_conv', None) is None:
        return False
    blocks = list(encoder.down_blocks)
    return bool(blocks) and all(hasattr(b, 'resnets') and hasattr(b, 'downsamplers') and len(b.resnets)
                                and all(_is_resnet(r) for r in b.resnets)
                and all(_downsample_ok(d) for d in (b.downsamplers or ())) for b in blocks)


def _lean_block(block):
    def forward(held):
        for resnet in block.resnets:
            held.value = resnet(held)
        if block.downsamplers is not None:
            for downsampler in block.downsamplers:
                held.value = downsampler(held)
        return held
    return forward


def _lean_encoder(encoder, torch):
    def forward(hidden_states):
        held = _Hold(encoder.conv_in(hidden_states))
        for block in encoder.down_blocks:
            block(held)
        hidden_states = held.take()
        hidden_states = torch.nn.functional.silu(encoder.norm_out(hidden_states))
        return encoder.conv_out(hidden_states)
    return forward


def _residency(model, target, torch):
    """Hooks that hold each encoder stage's weights on `device` only while it runs.

    The weights are the same FP32 values read from the checkpoint; only the
    stage being computed is resident on the device, so the large late-stage
    weights are not alive while the early stages hold the largest activations.
    `target` holds the device the stages upload to and whether a CPU rescue runs.
    Returns (hooks, host copies still on the device, uploaded byte counter).
    """
    encoder = model.encoder
    units = [encoder.conv_in, *encoder.down_blocks, encoder.norm_out, encoder.conv_out, model.quant_conv]
    covered = {id(p) for unit in units for p in unit.parameters()}
    if any(id(p) not in covered for p in model.parameters()):
        return None
    host, uploaded, handles = {}, [0], []
    def upload(unit, inputs):
        for parameter in unit.parameters():
            # A failed run leaves its stage here with the device copy in place;
            # the host copy kept from the first upload stays the source.
            host.setdefault(parameter, parameter.data)
            parameter.data = host[parameter].to(target['device'])
            if not target['rescue']:
                uploaded[0] += parameter.numel() * parameter.element_size()
    def release(unit, inputs, output):
        for parameter in unit.parameters():
            parameter.data = host.pop(parameter)
    for unit in units:
        handles.append(unit.register_forward_pre_hook(upload))
        handles.append(unit.register_forward_hook(release))
    return handles, host, uploaded


CPU_TILE_LIMIT = 6


def _finite(tensor):
    return bool(tensor.isfinite().all())


def _guarded_tiles(encoder, patched, target, device, staged, stats, torch):
    """Check every tile the encoder returns and recompute a non-finite one.

    A reference clip takes many minutes to encode and the pinned H3 pipeline
    only notices a non-finite latent after the last tile. Each tile output is
    checked here instead. A tile with non-finite values is recomputed from the
    same input: once more on the device (a transient fault), then with the
    original operators and the same weights, then in FP32 on the CPU for a
    bounded number of tiles. The first level that succeeds stays in use for the
    following tiles, except the CPU, which is only ever a per-tile rescue. A
    tile that no level makes finite stops the encode at once with a clear error.
    Tile numbers and the CPU limit count over every reference encoded in the
    same `bounded_encoder` context. Only the encoder's own output is checked;
    the layers after it (quant_conv, sampling, stitching) are not.
    """
    inner = encoder.forward
    levels = ['device'] + (['plain'] if stats.get('lean_resnet_blocks') else []) + (['cpu'] if staged else [])
    stats.update(nonfinite_tiles=0, nonfinite_recoveries={})
    state = {'start': 0, 'tile': 0}
    def sweep():
        torch.mps.synchronize()
        torch.mps.empty_cache()
    def run(level, hidden_states):
        if level == 'device':
            return inner(hidden_states)
        saved = {m: m.__dict__.pop('forward') for m in patched if 'forward' in m.__dict__}
        try:
            if level == 'plain':
                return encoder(hidden_states)
            target.update(device='cpu', rescue=True)
            try:
                return encoder(hidden_states.to('cpu')).to(device)
            finally:
                target.update(device=device, rescue=False)
        finally:
            encoder.__dict__.pop('forward', None)
            for module, forward in saved.items():
                module.__dict__['forward'] = forward
    def forward(hidden_states):
        state['tile'] += 1
        output = inner(hidden_states) if state['start'] == 0 else run(levels[state['start']], hidden_states)
        if _finite(output):
            return output
        del output
        stats['nonfinite_tiles'] += 1
        first = stats.setdefault('nonfinite_first_tile', dict(index=state['tile'], shape=list(hidden_states.shape),
                                                              level=levels[state['start']]))
        if first['index'] == state['tile']:
            print(json.dumps(dict(event='encoder_nonfinite_tile', **first)), flush=True)
        attempts = [(0, 'device-retry')] if state['start'] == 0 else []
        attempts += [(index, level) for index, level in enumerate(levels) if index > state['start']]
        tried = []
        for index, level in attempts:
            if level == 'cpu' and stats.get('cpu_tiles', 0) >= CPU_TILE_LIMIT:
                break
            tried.append(level)
            sweep()
            started = time.monotonic()
            if level == 'cpu':
                stats['cpu_tiles'] = stats.get('cpu_tiles', 0) + 1
                print(json.dumps(dict(event='encoder_cpu_rescue', phase='start', tile=state['tile'],
                                      rescued=stats['cpu_tiles'])), flush=True)
            output = run(levels[index], hidden_states)
            if level == 'cpu':
                print(json.dumps(dict(event='encoder_cpu_rescue', phase='end', tile=state['tile'],
                                      seconds=round(time.monotonic() - started, 1))), flush=True)
            if _finite(output):
                stats['nonfinite_recoveries'][level] = stats['nonfinite_recoveries'].get(level, 0) + 1
                if level == 'plain':
                    state['start'] = 1
                return output
            del output
        raise RuntimeError('The reference video encoder returned non-finite values for tile %d since the reference '
                           'encoder was prepared (%s) at every level tried (%s); the encode was stopped instead of finishing '
                           'the remaining tiles.'
                           % (state['tile'], 'x'.join(str(n) for n in hidden_states.shape), ', '.join(tried) or 'none'))
    return forward


@contextmanager
def bounded_encoder(model, device=None):
    """Finish each convolution before submitting the next encoder operation.

    MPS can retain large temporary allocations for pending 3D convolutions even
    after their Python tensors are gone. Fencing the original operators bounds
    that lifetime without changing weights, precision, tiles or posterior noise.
    The residual blocks additionally build each padded convolution input in one
    buffer, run the SiLU in place, add the residual in place and let each stage
    release the previous activation: the same values, with fewer tile-sized
    tensors alive at the peak. With `device`, the model's weights stay on the
    host (the checkpoint mapping) and each stage uploads its own for its run.
    Hooks and instance forwards belong to this VAE instance and are always
    removed.
    """
    import torch
    if torch.is_grad_enabled():
        raise RuntimeError('Bounded MPS input encoding requires inference')
    modules = [module for module in model.modules() if isinstance(module, torch.nn.Conv3d)]
    if not modules:
        raise ValueError('Expected the H3 input encoder convolution modules')
    handles, patched, host, staged, uploaded = [], [], {}, None, [0]
    target = dict(device=device, rescue=False)
    stats = dict(policy='Complete original FP32 convolutions and release unused MPS allocations',
                 convolution_completions=0, convolution_modules=len(modules), lean_resnet_blocks=0,
                 staged_weight_bytes=0)
    def fence():
        torch.mps.synchronize()
        torch.mps.empty_cache()
        stats['convolution_completions'] += 1
    def complete(module, inputs, output):
        fence()
    try:
        for module in modules:
            handles.append(module.register_forward_hook(complete))
        if _lean_encoder_ok(model):
            for block in model.encoder.down_blocks:
                for resnet in block.resnets:
                    resnet.forward = _lean_resnet(resnet, fence, stats)
                    patched.append(resnet)
                for downsampler in block.downsamplers or ():
                    downsampler.forward = _lean_downsample(downsampler, fence)
                    patched.append(downsampler)
                block.forward = _lean_block(block)
                patched.append(block)
            model.encoder.forward = _lean_encoder(model.encoder, torch)
            patched.append(model.encoder)
            stats['lean_resnet_blocks'] = sum(len(b.resnets) for b in model.encoder.down_blocks)
            if device is not None:
                staged = _residency(model, target, torch)
                if staged is not None:
                    hooks, host, uploaded = staged
                    handles.extend(hooks)
        if device is not None and staged is None:
            # Not the pinned structure: whole model on the device, fences only.
            model.to(device)
        encoder = getattr(model, 'encoder', None)
        if device is not None and isinstance(encoder, torch.nn.Module):
            if encoder not in patched:
                patched.append(encoder)
            encoder.forward = _guarded_tiles(encoder, patched, target, device, staged is not None, stats, torch)
        yield stats
    finally:
        if staged is not None:
            stats['staged_weight_bytes'] = uploaded[0]
        for handle in handles:
            handle.remove()
        for parameter, data in host.items():
            parameter.data = data
        for module in patched:
            module.__dict__.pop('forward', None)