Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
"""Decode with original tiles and FP32 normalization; optional Linear compute cache."""
import gc
import json
from pathlib import Path
import time
import torch
from .offload import LayerOffloader, pin_layer_weights
from .vae_tiles import TileDecoder, compile_blocks
DECODE_LABELS = {
'vae_load': 'Loading video VAE',
'video_decode': 'Decoding the video',
'video_postprocess': 'Processing video colors',
'rgb_save': 'Saving decoded video',
'audio_load_decode': 'Loading and decoding audio',
'audio_save': 'Saving decoded audio',
'mp4_save': 'Saving MP4 and audio',
}
@torch.no_grad()
def decode_to_file(latents, audio_latents, out_path, *, base, offload=False, prefetch=True, tile_group=False,
preload=False, pin_weights=False, artifacts_dir=None, phase_callback=None,
stream_output=False, stream_weights=False, resident_blocks=0, model_cache=None,
linear_compute_cache=False, continuation=None, audio_lead_in_callback=None, output_size=None,
hide_tail_frames=0, ram_budget_bytes=None, original_audio=None):
from .continuation import validate_hide_tail
validate_hide_tail(hide_tail_frames)
def phase(name):
if phase_callback is not None:
phase_callback(name)
print(json.dumps({'event': 'decode_phase', 'phase': DECODE_LABELS[name]}), flush=True)
phase('vae_load')
from diffusers import AutoencoderKLMiniMaxH3
from diffusers.utils.export_utils import encode_video
from src.inference.render import PIXEL_MEAN, PIXEL_STD, FPS
started = time.perf_counter()
vae = None
weight_source = None
compute_cache_metrics = None
if type(linear_compute_cache) is not bool:
raise ValueError('linear_compute_cache must be a boolean')
if type(resident_blocks) is not int or resident_blocks < 0:
raise ValueError('VAE resident block count is outside this decoder')
if resident_blocks and not offload:
raise ValueError('Partial VAE residency requires decoder offloading')
if pin_weights and not offload:
raise ValueError('Pinned VAE weights require layer offloading')
if stream_weights and (not offload or pin_weights or preload):
raise ValueError('Streamed VAE weights require offloading without host preload or whole-model pinning')
if model_cache is not None and not offload:
from .resident_models import key, file_identity
vae_key = key([file_identity(path) for path in sorted((Path(base)/'vae').iterdir()) if path.is_file()])
if linear_compute_cache:
vae_key = key([vae_key, 'decoder-fp16-linear-v1'])
vae = model_cache.take('video_vae', vae_key)
vae_cache_hit = vae is not None
if vae is None:
# Decode runs after sampling has consumed most of the RAM budget.
# from_pretrained would hold the whole 9.7 GiB VAE on the CPU (Windows:
# whole 4.71 GiB shard reads); read only decoder tensors, one at a time,
# straight to their device. Without the Linear cache all stay FP32.
from .vae_weights import load_video_decoder
layers = AutoencoderKLMiniMaxH3.load_config(
str(Path(base)/'vae'), local_files_only=True)['decoder_num_layers']
count = resident_blocks if offload else layers
if stream_weights:
from .streamed_weights import SafetensorLayers
room = None
if ram_budget_bytes is not None:
# Converted layers are private memory: keep them inside the request's own
# working-memory budget, so they can neither trip an explicit budget nor
# lift the peak that later plans take as demonstrated.
def room():
import os
from .ram import ProcessMemory
from .system import memory_sample
used = memory_sample(ProcessMemory().sample(os.getpid()))
return ram_budget_bytes - used if type(used) is int and used >= 0 else None
weight_source = SafetensorLayers((Path(base)/'vae').glob('*.safetensors'),
[f'decoder.transformer_blocks.{index}.' for index in range(count, layers)],
intermediate_dtype=torch.float32 if linear_compute_cache else None,
keep_converted=True, converted_room=room,
converted_store=getattr(model_cache, 'converted_store', None))
vae, compute_cache_metrics = load_video_decoder(base, resident_blocks=count, weight_source=weight_source,
linear_fp16=linear_compute_cache)
vae.eval().requires_grad_(False)
if type(resident_blocks) is not int or not 0 <= resident_blocks <= len(vae.decoder.transformer_blocks):
raise ValueError('VAE resident block count is outside this decoder')
vae.encoder = None
vae.quant_conv = None
vae.post_quant_conv.to('cuda')
if offload:
for name, child in vae.decoder.named_children():
if name != 'transformer_blocks':
child.to('cuda')
vae.decoder.register_tokens.data = vae.decoder.register_tokens.data.to('cuda')
offloaded_blocks = list(vae.decoder.transformer_blocks)[resident_blocks:]
pinned_bytes = pin_layer_weights(offloaded_blocks) if pin_weights else 0
if pin_weights:
# The text-to-video decoder never calls these modules. Releasing
# their mappings also drops references to shards shared with the
# now-pinned decoder weights.
vae.encoder = None
vae.quant_conv = None
gc.collect()
tile_decoder = TileDecoder(vae.decoder, prefetch=prefetch, weight_source=weight_source,
resident_blocks=resident_blocks) if tile_group else None
if tile_decoder is not None:
tile_decoder.install(vae)
offloader = tile_decoder.offloader if tile_decoder is not None else LayerOffloader(offloaded_blocks, prefetch=prefetch, weight_source=weight_source)
else:
pinned_bytes = 0
vae.decoder.to('cuda')
offloader = None
tile_decoder = None
# Release unused encoder views of the checkpoint shards. Otherwise
# they keep CPU mappings alive after all decoder weights move to GPU.
vae.encoder = None
vae.quant_conv = None
gc.collect()
if model_cache is not None:
model_cache.put('video_vae', vae_key, vae)
preload_seconds = 0.
if offload and preload:
# Touch the immutable files sequentially after allocating transfer
# buffers, avoiding small random page faults in the offload thread.
read_started = time.perf_counter()
buffer = bytearray(8 * 1024 * 1024)
for path in sorted((Path(base) / 'vae').glob('*.safetensors')):
with path.open('rb', buffering=0) as stream:
while stream.readinto(buffer):
pass
del buffer
preload_seconds = time.perf_counter() - read_started
compiled_blocks = compile_blocks(vae.decoder)
mean = torch.tensor(vae.config.latents_mean, device='cuda').view(1, -1, 1, 1, 1)
std = torch.tensor(vae.config.latents_std, device='cuda').view(1, -1, 1, 1, 1)
torch.cuda.synchronize()
load_seconds = time.perf_counter() - started
decode_started = time.perf_counter()
phase('video_decode')
offload_stats = None
rgb_mapping = None
streamed_timings = {}
try:
# Autocast follows the original renderer. The opt-in Linear cache stores
# its existing FP16 compute values once; all other weights stay FP32.
with torch.autocast(device_type='cuda', dtype=torch.float16, cache_enabled=not offload):
if stream_output:
from .decode_stream import render_rgb
directory = Path(artifacts_dir) if artifacts_dir else Path(out_path).with_suffix('.artifacts')
def decoded(done, total, elapsed):
print(json.dumps(dict(event='decode_progress', done=done, total=total,
elapsed_seconds=elapsed)), flush=True)
frames, rgb_mapping, streamed_timings = render_rgb(vae, latents * std + mean,
PIXEL_MEAN, PIXEL_STD, directory/'rgb.npy', progress=decoded)
else:
video = vae.decode(latents * std + mean, return_dict=False)[0]
video = video.cpu()
if offloader is not None:
offload_stats = offloader.stats()
finally:
if offloader is not None:
offloader.close()
if tile_decoder is not None:
tile_decoder.close()
try:
del vae, offloader, tile_decoder, mean, std
if offload:
del offloaded_blocks
gc.collect()
torch.cuda.empty_cache()
video_decode_seconds = streamed_timings.get('video_decode_seconds', time.perf_counter() - decode_started)
# Bound postprocessing memory by frame chunks; use the same FP32 operations
# and rounding before converting to uint8. Do this on CPU after decoding.
if not stream_output:
phase('video_postprocess')
frames = torch.empty((video.shape[2], video.shape[3], video.shape[4], 3), dtype=torch.uint8)
pixel_mean = torch.tensor(PIXEL_MEAN).view(1, -1, 1, 1, 1)
pixel_std = torch.tensor(PIXEL_STD).view(1, -1, 1, 1, 1)
for start in range(0, video.shape[2], 8):
chunk = (video[:, :, start:start + 8].float() * pixel_std + pixel_mean).clamp_(0, 1)
frames[start:start + chunk.shape[2]] = (chunk[0].permute(1, 2, 3, 0) * 255).round().to(torch.uint8)
del video
if hide_tail_frames:
kept_frames = len(frames) - hide_tail_frames
frames = frames[:kept_frames]
if continuation is not None:
from .continuation import OVERLAP_FRAMES
frames = frames[OVERLAP_FRAMES:]
if original_audio is not None and 'segment' not in original_audio:
from .original_audio import verify
verify(original_audio)
count = original_audio['end_sample'] - original_audio['start_sample']
rate = original_audio['sample_rate']
frames = frames[:(count * FPS + rate - 1) // rate]
artifact_seconds = streamed_timings.get('decoded_artifact_save_seconds', 0.)
if artifacts_dir:
import numpy as np
phase('rgb_save')
tick = time.perf_counter()
artifacts_dir = Path(artifacts_dir)
artifacts_dir.mkdir(parents=True, exist_ok=True)
if not stream_output or continuation is not None or hide_tail_frames or original_audio is not None:
# A streamed mapping still contains overlap/hidden frames. Replace its
# retained artifact after the mapping closes, below.
rgb_path = artifacts_dir / ('rgb-new.npy' if stream_output else 'rgb.npy')
np.save(rgb_path, frames.numpy() if hasattr(frames, 'numpy') else frames, allow_pickle=False)
artifact_seconds += time.perf_counter() - tick
elif stream_output and hide_tail_frames:
# render_rgb also retains a mapping when no explicit artifacts directory was requested.
import numpy as np
np.save(directory / 'rgb-new.npy', frames.numpy() if hasattr(frames, 'numpy') else frames,
allow_pickle=False)
original_receipt = None
if original_audio is not None:
# The clip used as is is the sound; the model's audio is not decoded.
audio, rate, audio_seconds, audio_cache_hit, audio_level = None, original_audio['sample_rate'], 0., False, {}
else:
audio_start = time.perf_counter()
phase('audio_load_decode')
audio_vae = None
if model_cache is not None:
from .resident_models import key, file_identity
audio_key = key([file_identity(path) for path in sorted((Path(base)/'audio_vae').iterdir()) if path.is_file()])
audio_vae = model_cache.take('audio_vae', audio_key)
model_cache.make_room('audio_vae', 1*2**30, model=audio_vae, ram_need=256*2**20)
audio_cache_hit = audio_vae is not None
if audio_vae is None:
from .vae_weights import load_audio_vae
audio_vae, _ = load_audio_vae(base)
if model_cache is not None:
model_cache.put('audio_vae', audio_key, audio_vae)
audio_vae.eval().requires_grad_(False)
audio_mean = torch.tensor(audio_vae.config.latents_mean, device='cuda').view(1, -1, 1)
audio_std = torch.tensor(audio_vae.config.latents_std, device='cuda').view(1, -1, 1)
print(json.dumps({'event': 'decode_phase', 'phase': 'Decoding audio'}), flush=True)
audio = audio_vae.decode(audio_latents * audio_std + audio_mean, return_dict=False)[0]
audio = audio.float().permute(1, 0, 2)[0].cpu()
rate = audio_vae.config.sampling_rate
if continuation is not None:
if audio_lead_in_callback is not None:
from .continuation import capture_audio_lead_in
audio_lead_in_callback(capture_audio_lead_in(audio, continuation, rate))
if hide_tail_frames:
audio = audio[:, :round(kept_frames * 40 / 24) * 800]
if continuation is not None:
audio = audio[:, continuation['audio_latents'] * 800:]
audio_level = {'audio_rms': float(audio.square().mean().sqrt()) if audio.numel() else 0.,
'audio_peak': float(audio.abs().max()) if audio.numel() else 0.,
'audio_samples': int(audio.shape[-1])}
del audio_vae, audio_mean, audio_std
gc.collect()
torch.cuda.empty_cache()
audio_seconds = time.perf_counter() - audio_start
if artifacts_dir:
import wave
phase('audio_save')
tick = time.perf_counter()
np.save(artifacts_dir / 'audio.npy', audio.numpy(), allow_pickle=False)
# Preserve exact float audio above; WAV is an independently playable copy.
pcm = (audio.clamp(-1, 1).t().contiguous().numpy() * 32767).round().astype('<i2')
with wave.open(str(artifacts_dir / 'audio.wav'), 'wb') as wav:
wav.setnchannels(audio.shape[0])
wav.setsampwidth(2)
wav.setframerate(rate)
wav.writeframes(pcm.tobytes())
artifact_seconds += time.perf_counter() - tick
destination = Path(out_path)
destination.parent.mkdir(parents=True, exist_ok=True)
temporary = destination.with_name(destination.stem + '.partial' + destination.suffix)
encode_started = time.perf_counter()
phase('mp4_save')
picture = frames
if output_size is not None:
# The model rendered a larger size; only the picture is scaled.
from .output_fit import fit_frames
picture = fit_frames(frames, *output_size)
if original_audio is not None:
from .original_audio import mux
# Encode picture only; audio never enters the model decoder or AAC encoder.
try:
encode_video(picture, fps=FPS, output_path=str(temporary))
if 'segment' in original_audio:
# Keep the complete new picture grid for the ordinary
# join. Standalone/editor block playback still has its
# exact source sound; the final join ignores this track.
info = dict(original_audio, start_sample=original_audio['segment']['start_sample'],
end_sample=original_audio['segment']['end_sample'], full_source=False)
original_receipt = mux(temporary, destination, info, trim_video=False)
original_receipt['segment'] = original_audio['segment']
else:
original_receipt = mux(temporary, destination, original_audio)
if artifacts_dir:
from .monitoring import save
save(artifacts_dir / 'original-audio.json', original_receipt)
finally:
temporary.unlink(missing_ok=True)
else:
encode_video(picture, fps=FPS, output_path=str(temporary), audio=audio, audio_sample_rate=rate)
temporary.replace(destination)
frame_count = len(frames)
finally:
if rgb_mapping is not None:
del frames
rgb_mapping._mmap.close()
retained = Path(artifacts_dir) if artifacts_dir else directory
if (continuation is not None or hide_tail_frames or original_audio is not None) and (retained / 'rgb-new.npy').is_file():
(retained / 'rgb-new.npy').replace(retained / 'rgb.npy')
return {**({'original_audio': original_receipt} if original_receipt is not None else {}),
'vae_load_seconds': load_seconds, 'video_decode_seconds': video_decode_seconds,
'vae_linear_compute_cache': linear_compute_cache, 'vae_compute_cache_preparation': compute_cache_metrics,
'resident_vae_cache_hit': vae_cache_hit, 'resident_audio_vae_cache_hit': audio_cache_hit,
'vae_preload_seconds': preload_seconds,
**audio_level,
'audio_load_decode_seconds': audio_seconds, 'encode_seconds': time.perf_counter() - encode_started,
'decoded_artifact_save_seconds': artifact_seconds,
'streamed_video_output': stream_output, 'streamed_vae_weights': stream_weights,
'vae_resident_blocks': resident_blocks,
'vae_compiled_blocks': compiled_blocks,
'video_postprocess_seconds': streamed_timings.get('video_postprocess_seconds'),
'vae_pinned_weight_bytes': pinned_bytes,
'decode_save_seconds': time.perf_counter() - started,
'vae_offload': offload_stats, 'frames': frame_count, 'output': str(destination)}