Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

5.0 KB
Raw Blame
"""Exact independent LBH three-step clocks and pass-local modulation tables."""
from contextlib import contextmanager
import json
from pathlib import Path
import time

COMMUNITY = 'community-sigma3-v1'
VIDEO_SIGMAS = (0.9035, 0.6316, 0.3158, 0.)


def schedulers(device):
    import torch
    from diffusers import MiniMaxH3Scheduler
    # Keep the leading 1.0 used by the verified community workflow. It is
    # initialization only: the three executed calls start at index one.
    sigma = torch.tensor((1.,) + VIDEO_SIGMAS, device=device, dtype=torch.float32)
    raw = sigma / (12. + (1. - 12.) * sigma)
    audio_sigma = 3. * raw / (1. + 2. * raw)
    video, audio = MiniMaxH3Scheduler(shift=12.), MiniMaxH3Scheduler(shift=3.)
    video.set_timesteps(device=device, sigmas=sigma)
    audio.set_timesteps(device=device, sigmas=audio_sigma)
    return video, audio


def timesteps(device, task):
    from .adaln import modality_timesteps
    video, audio = schedulers(device)
    return modality_timesteps(video.timesteps[1:], audio.timesteps[1:], task)


@contextmanager
def modulation(engine, schedule):
    """Swap only small constants; always restore the resident first-pass state."""
    if schedule is None:
        yield None
        return
    if schedule != COMMUNITY:
        raise ValueError('Unsupported refinement schedule')
    import torch
    from .adaln import CachedModulation, ScheduleCursor, TableCache
    from .adaln_assets import restore_projections
    model = getattr(engine, 'transformer', None) or engine.model
    backend = getattr(engine, 'device_backend', None) or engine.backend
    device = backend.device
    previous_cursor = engine.cursor
    # Uncached modulation evaluates the same exact clocks directly.
    if previous_cursor is None:
        yield dict(schedule=schedule, table_cache_hits=0)
        return
    manifest = json.loads((Path(engine.cache) / 'manifest.json').read_text(encoding='utf-8'))
    task = getattr(engine, 'adaln_task', engine.task)
    times = timesteps(device, task)
    cursor = ScheduleCursor(times)
    fetch = getattr(engine, 'fetch_tables', True)
    cache = TableCache(engine.cache, manifest['source_id'], 3, task=task,
        manifest=manifest, timesteps=times, channels=model.config.hidden_size, device='cpu', fetch=fetch)
    originals = []
    started, hits = time.perf_counter(), 0
    try:
        with torch.no_grad():
            embeddings = None
            for index, block in enumerate(model.transformer_blocks):
                # Preserve the original tensors without keeping two schedules
                # resident on the GPU. Offloaders are outside this boundary.
                originals.append((block, block.adaln_proj.cpu()))
                values = cache.load(index, 3)
                if values is None:
                    from diffusers.models.transformers.transformer_minimax_h3 import MiniMaxH3AdaLayerNormModulation
                    if embeddings is None:
                        embeddings = [model.time_embedder(model.time_proj(t).to(
                            model.time_embedder.linear_1.weight.dtype)) for t in times]
                    path = Path(engine.cache) / f'adaln/{index:02d}.safetensors'
                    if not path.is_file():
                        restore_projections(engine.cache, manifest)
                    prefix = f'transformer_blocks.{index}.adaln_proj.'
                    from .runtime import _load_safetensors
                    state = {k.removeprefix(prefix): v.to(device) for k, v in _load_safetensors(path).items()}
                    with torch.device('meta'):
                        projection = MiniMaxH3AdaLayerNormModulation(state['linear.weight'].shape[1], model.config.hidden_size)
                    projection.load_state_dict(state, strict=True, assign=True)
                    values = [projection(e) for e in embeddings]
                    cache.save(index, CachedModulation(values, cursor), producer_device=str(device))
                    del projection, state
                else:
                    hits += 1
                block.adaln_proj = CachedModulation(values, cursor).to(device)
                del values
            del embeddings
        if not fetch and cache.placeholders:
            engine.table_placeholders.add(cache.root.name)
        engine.schedule_hook.remove()
        engine.cursor = cursor
        engine.schedule_hook = model.register_forward_pre_hook(cursor.before, with_kwargs=True)
        yield dict(schedule=schedule, table_cache_hits=hits, seconds=time.perf_counter()-started,
                   table_identity=cache.identity)
    finally:
        # Clear all new tables before restoration, bounding GPU storage by the
        # larger of the two schedules rather than their combined size.
        for block, _ in originals:
            block.adaln_proj.cpu()
        for block, original in originals:
            block.adaln_proj = original.to(device)
        engine.schedule_hook.remove()
        engine.cursor = previous_cursor
        engine.schedule_hook = model.register_forward_pre_hook(previous_cursor.before, with_kwargs=True)