Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
"""A continuation context encoded from the ending of a finished video.
A video made by an earlier version, or one whose file was replaced after it
was made, has no stored context. Its last 22 frames and the sound under them
are read back and encoded instead. The video encoder works on independent
17-frame clips, so those frames (17 + 5) fall on the same clip grid as the
last 7 latent frames of a generation: nothing before them has to be encoded.
The sound encoder looks back, so it is given up to 15 s and only its last
latents are kept.
The record describes the ending as a first block of 39 frames. Only the sound
clock reads that length: the kept sound ends exactly where the picture ends.
"""
import math
from pathlib import Path
from .continuation import (LATENT_FRAMES, MISSING, OVERLAP_FRAMES, SIZE_MISMATCH, audio_origin, validate_frames,
validate_path)
NOMINAL_FRAMES = 39
AUDIO_TICKS = round(NOMINAL_FRAMES * 40 / 24) - audio_origin(NOMINAL_FRAMES - OVERLAP_FRAMES)
AUDIO_RATE = 32000
TICK_SAMPLES = 800
AUDIO_WINDOW_TICKS = 600 # 15 s, the longest sound H3 generates
FPS = 24
TOO_SHORT = 'The previous video is too short to continue; it needs to be at least 1.7 s long.' # NOMINAL_FRAMES is 1.625 s at 24 fps; the message rounds up so a 1.7 s video always fits
UNREADABLE = 'The previous video cannot be read.'
LOW_RATE = "The previous video's frame rate is too low to continue; it needs at least 10 frames per second."
def _fit(image, width, height):
from PIL import Image
return image if image.size == (width, height) else image.resize((width, height), Image.Resampling.LANCZOS)
def _picture(frame):
"""The frame as an RGB image.
The decoder library's own conversion comes out about 1/255 darker than the picture that was saved,
which would show as a step where the next part begins. An 8-bit 4:2:0 frame of the usual range is
converted with the exact matrix instead.
"""
import numpy as np
from PIL import Image
if frame.format.name != 'yuv420p' or getattr(frame.color_range, 'value', frame.color_range) not in (0, 1):
return frame.to_image().convert('RGB')
y, cb, cr = frame.reformat(format='yuv444p', interpolation='BICUBIC').to_ndarray().astype(np.float32)
kr, kb = (.2126, .0722) if getattr(frame.colorspace, 'value', frame.colorspace) == 1 else (.299, .114)
y, cb, cr = (y - 16) * (255 / 219), (cb - 128) * (255 / 224), (cr - 128) * (255 / 224)
red, blue = y + 2 * (1 - kr) * cr, y + 2 * (1 - kb) * cb
green = (y - kr * red - kb * blue) / (1 - kr - kb)
return Image.fromarray(np.clip(np.stack([red, green, blue], -1) + .5, 0, 255).astype(np.uint8))
def _rate(stream):
rate = float(stream.average_rate or stream.guessed_rate or 0)
if 0 < rate < 10:
raise ValueError(LOW_RATE)
if not 10 <= rate <= 121:
raise ValueError(UNREADABLE)
return rate
def _admit(count, rate, width, height, canvas):
"""The length of the video as 24 fps frames, when its ending can be continued on `canvas`."""
frames = count if abs(rate - FPS) <= .01 else round(count * FPS / rate)
if frames < NOMINAL_FRAMES:
raise ValueError(TOO_SHORT)
# The same shape within one 32-pixel step of the canvas; another size of the same shape is scaled.
if min(width, height) < 1 or abs(width * canvas['height'] - height * canvas['width']) > 32 * max(width, height):
raise ValueError(SIZE_MISMATCH)
return frames
def check(video_path, canvas):
"""Whether the ending of this video can be encoded for `canvas`; a ValueError says why not.
Reads the file's packets and decodes nothing, so a caller can ask before it queues a run.
"""
import av
try:
with av.open(str(validate_path(video_path))) as container:
if not container.streams.video:
raise ValueError(UNREADABLE)
stream = container.streams.video[0]
count = sum(1 for packet in container.demux(stream) if packet.size)
return _admit(count, _rate(stream), stream.width or 0, stream.height or 0, canvas)
except (av.FFmpegError, OSError) as error:
raise ValueError(UNREADABLE) from error
def pending(video_path, canvas):
"""A continuation request whose context is encoded from the video when the run starts."""
validate_frames(canvas['frames'])
check(video_path, canvas)
origin = NOMINAL_FRAMES - OVERLAP_FRAMES
return dict(tail=dict(video=str(Path(video_path).resolve())), overlap_frames=OVERLAP_FRAMES,
new_frames=canvas['frames'] - OVERLAP_FRAMES, frame_origin=origin,
audio_origin=audio_origin(origin), segment=2, audio_latents=AUDIO_TICKS)
def read(video_path, canvas, first, directory):
"""The last 22 frames at both pass sizes and the sound under them, as files of this run.
`first` is the first pass's width and height. A video at another frame rate gives the
pictures a 24 fps copy of it would end with. Runs without a GPU.
"""
import av
import numpy as np
from collections import deque
directory = Path(directory)
directory.mkdir(parents=True, exist_ok=True)
try:
with av.open(str(video_path)) as container:
if not container.streams.video:
raise ValueError(UNREADABLE)
rate = _rate(container.streams.video[0])
kept, count = deque(maxlen=math.ceil(OVERLAP_FRAMES * rate / FPS) + 2), 0
for frame in container.decode(video=0):
kept.append(frame)
count += 1
if not count:
raise ValueError(UNREADABLE)
frames = _admit(count, rate, kept[-1].width, kept[-1].height, canvas)
oldest = count - len(kept)
if abs(rate - FPS) <= .01:
picked = range(count - OVERLAP_FRAMES, count)
else:
picked = [min(count - 1, max(oldest, round((frames - OVERLAP_FRAMES + index) * rate / FPS)))
for index in range(OVERLAP_FRAMES)]
images = [_picture(kept[index - oldest]) for index in picked]
del kept
sound = _sound(video_path, count / rate)
except (av.FFmpegError, OSError) as error:
raise ValueError(UNREADABLE) from error
width, height = images[0].size
result = {'frames': frames, 'source': {'width': width, 'height': height, 'fps': rate}}
for name, size in (('hi', canvas), ('lo', first)):
target = directory / ('continuation-tail-%s.npy' % name)
np.save(target, np.stack([np.asarray(_fit(image, size['width'], size['height'])) for image in images]),
allow_pickle=False)
result[name] = str(target)
target = directory / 'continuation-tail-audio.npy'
np.save(target, sound, allow_pickle=False)
result['audio'] = str(target)
return result
def _sound(video_path, seconds):
"""Up to 15 s of 32 kHz stereo ending where the picture ends; silence where the file has none."""
import av
import numpy as np
end = round(seconds * AUDIO_RATE)
window = min(AUDIO_WINDOW_TICKS, end // TICK_SAMPLES) * TICK_SAMPLES
begin = end - window
data = np.zeros((2, window), dtype=np.float32)
with av.open(str(video_path)) as container:
if not container.streams.audio:
return data
stream = container.streams.audio[0]
mono = stream.layout.name == 'mono' or stream.channels == 1
# A mono track has its channel repeated, as reference sound is; a stereo resampler would lower it by 3 dB.
resampler = av.AudioResampler(format='fltp', layout='mono' if mono else 'stereo', rate=AUDIO_RATE)
position = 0
def place(item):
nonlocal position
block = item.to_ndarray()
start, stop = max(position, begin), min(position + block.shape[1], end)
if stop > start:
data[:, start - begin:stop - begin] = block[:, start - position:stop - position]
position += block.shape[1]
for frame in container.decode(audio=0):
for item in resampler.resample(frame):
place(item)
if position >= end:
break
else:
for item in resampler.resample(None):
place(item)
if not np.isfinite(data).all():
raise ValueError(UNREADABLE)
return data
def encode(tail, base, *, device='cuda', before_upload=None, posterior='mode'):
"""Encode the files of `read` into the three tensors of a continuation record."""
import gc
import numpy as np
import torch
from .media_encoding import encode_visual
from .vae_weights import load_audio_vae, load_video_encoder
empty = torch.mps.empty_cache if torch.device(device).type == 'mps' else (
torch.cuda.empty_cache if torch.device(device).type == 'cuda' else lambda: None)
result = {}
vae, _ = load_video_encoder(base, before_upload=None if before_upload is None else
lambda model: before_upload(model, False), device=device)
for name in ('hi', 'lo'):
pixels = np.load(tail[name], mmap_mode='r', allow_pickle=False)
latent = encode_visual(vae, pixels, device=device, posterior=posterior)
if latent.shape[2] != LATENT_FRAMES:
raise ValueError('Unexpected continuation latent frames')
result['video_' + name] = latent.numpy()
del pixels, latent
del vae
gc.collect()
empty()
vae, _ = load_audio_vae(base, before_upload=None if before_upload is None else
lambda model: before_upload(model, True), device=device)
if vae.config.sampling_rate != AUDIO_RATE:
raise ValueError('Continuation sound requires the pinned 32 kHz H3 VAE')
mean = torch.tensor(vae.config.latents_mean).view(1, 1, -1)
std = torch.tensor(vae.config.latents_std).view(1, 1, -1)
audio = torch.from_numpy(np.load(tail['audio'], allow_pickle=False)).to(device)
latent = vae.encode(audio[:, None], return_dict=False)[0].mode().float().cpu().transpose(1, 2)
latent = ((latent - mean) / std).transpose(1, 2)
if latent.shape[2] < AUDIO_TICKS:
raise ValueError('Unexpected continuation sound latents')
result['audio'] = latent[:, :, -AUDIO_TICKS:].contiguous().numpy()
del vae, audio, latent
gc.collect()
empty()
return result
def metadata(identity, canvas, first):
"""The record's description: a first block whose last 22 frames are the file's."""
return dict(identity=identity,
geometry=dict(width=canvas['width'], height=canvas['height'], frames=NOMINAL_FRAMES),
first_canvas=dict(width=first['width'], height=first['height']),
chain=dict(segment=1, frame_origin=0, audio_origin=0), context='encoded')
def freeze(request, canvas, first, directory, base):
"""Keep the ending `pending` asked for as files of this run; the worker encodes them."""
from .continuation_store import video_id
directory = Path(directory)
video = request['tail']['video']
size = lambda value: {key: value[key] for key in ('width', 'height')}
tail = read(video, canvas, first, directory)
return dict(request, tail=dict(tail, video=video, canvas=size(canvas), first=size(first), base=str(base),
source_id=video_id(video),
snapshot=str((directory / 'continuation-encoded.safetensors').resolve())))
def complete(request, canvas, manifest, sampling_plan=None, *, resident=None, device='cuda'):
"""The ordinary snapshot request for a frozen ending, and how long encoding it took.
The first attempt of a run encodes and writes the record beside its other files; a retry reads it.
"""
import hashlib
import time
from .continuation import identity, validate_snapshot
from .continuation_store import write
tail = request['tail']
path = Path(tail['snapshot'])
if sampling_plan is not None and any(sampling_plan['first'][key] != tail['first'][key] for key in ('width', 'height')):
raise ValueError(MISSING)
seconds = None
if not path.is_file():
started = time.perf_counter()
data = encode(tail, tail['base'], device=device, before_upload=None if resident is None else
lambda model, audio: resident.input_vae_room(model, tail['canvas'], audio=audio))
write(path, dict(data, metadata=dict(metadata(identity(manifest), tail['canvas'], tail['first']),
source_id=tail['source_id'])))
path.chmod(0o400)
seconds = time.perf_counter() - started
result = dict({key: value for key, value in request.items() if key != 'tail'}, snapshot=str(path),
snapshot_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), source_id=tail['source_id'])
return validate_snapshot(result, canvas, manifest, sampling_plan), dict(seconds=seconds, reused=seconds is None)