Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
Reuse reference picture and clip latents when only the prompt changes
Bowen Xuefe5a71998bba
diff --git a/freevideo_engine/encode_worker.py b/freevideo_engine/encode_worker.pyindex 7df26d0..6704639 100644--- a/freevideo_engine/encode_worker.py+++ b/freevideo_engine/encode_worker.py@@ -645,7 +645,8 @@ def _encode(args, request, resident, trace):add_vdn()from .media_encoding import encode_latentstick = time.perf_counter()- info = encode_latents(normalized, value, request['base'], request['geometry'], resident=resident)+ info = encode_latents(normalized, value, request['base'], request['geometry'], resident=resident,+ cache_root=request.get('reference_latent_cache'))keyframe_metrics.update(media_vae_encode_seconds=time.perf_counter() - tick, conditioning_info=info)task = info['task']elif early_reference is not None:diff --git a/freevideo_engine/generate.py b/freevideo_engine/generate.pyindex d4fbaa1..b4d0bc5 100644--- a/freevideo_engine/generate.py+++ b/freevideo_engine/generate.py@@ -752,7 +752,8 @@ def _run(args):'metrics': str(destination.with_suffix('.encoding.json')),'diagnostic_output': str(destination)}if media:- encoding.update(media=media, geometry=canvas, base=profile['engine'].get('base'))+ encoding.update(media=media, geometry=canvas, base=profile['engine'].get('base'),+ reference_latent_cache=str(input_cache.root.parent.parent) if input_cache else None)# Lets the encoder child read the video model while it computes, on a slow drive.from .model_read_ahead import request_fieldsif getattr(args, 'cache', None):diff --git a/freevideo_engine/media_encoding.py b/freevideo_engine/media_encoding.pyindex 31da96f..5014737 100644--- a/freevideo_engine/media_encoding.py+++ b/freevideo_engine/media_encoding.py@@ -217,7 +217,67 @@ def reference_options(normalized, options, step):return {'minimax_ref_items': items} if items else {}-def encode_latents(normalized, value, base, canvas, *, resident=None, device='cuda'):+def _latent_cache(root, base, device):+ """Per-picture latent receipts; another encoder, recipe or device never matches."""+ import torch+ from .input_cache import InputCache, digest+ directory = Path(base) / 'vae'+ files = sorted(directory.glob('*.safetensors'))+ if not files:+ return None+ kind = torch.device(device).type+ try:+ from importlib.metadata import version+ diffusers = version('diffusers')+ except Exception:+ diffusers = None+ from . import __version__+ identity = {'recipe': 'reference-latent-v1', 'engine': __version__, 'diffusers': diffusers,+ 'torch': torch.__version__, 'device': kind,+ 'gpu': torch.cuda.get_device_name() if kind == 'cuda' else None,+ 'config': digest(directory / 'config.json') if (directory / 'config.json').exists() else None,+ 'weights': [[f.name, f.stat().st_size, f.stat().st_mtime_ns] for f in files]}+ return InputCache(identity, root=root, kind='reference-latent')+++def _pixels_key(mapped):+ import hashlib+ import numpy as np+ value = hashlib.sha256(json.dumps([list(mapped.shape), str(mapped.dtype)]).encode())+ for begin in range(0, len(mapped), 8):+ value.update(np.ascontiguousarray(mapped[begin:begin + 8]).reshape(-1).view('u1').data)+ return value.hexdigest()+++def _cached_latent(cache, key, directory):+ import torch+ target = Path(directory) / ('reference-latent-' + key[:16] + '.hit.pt')+ try:+ if cache.get(key, target) is None:+ return None+ latent = torch.load(target, map_location='cpu', weights_only=True)+ if isinstance(latent, torch.Tensor) and latent.is_floating_point() and bool(torch.isfinite(latent).all()):+ return latent+ except (OSError, ValueError, RuntimeError, KeyError, TypeError, EOFError):+ pass+ finally:+ target.unlink(missing_ok=True)+ return None+++def _remember_latent(cache, key, latent, directory):+ import torch+ path = Path(directory) / ('reference-latent-' + key[:16] + '.save.pt')+ try:+ torch.save(latent, path)+ cache.put(key, path, {'success': True})+ except (OSError, ValueError, RuntimeError):+ pass+ finally:+ path.unlink(missing_ok=True)+++def encode_latents(normalized, value, base, canvas, *, resident=None, device='cuda', cache_root=None):import gcimport numpy as npimport torch@@ -228,7 +288,25 @@ def encode_latents(normalized, value, base, canvas, *, resident=None, device='curaise ValueError('Media encoding requires a CUDA or MPS device')empty_cache = torch.mps.empty_cache if torch.device(device).type == 'mps' else torch.cuda.empty_cacherefs = [{key: row[key] for key in ('kind', 'early_steps') if key in row} for row in normalized]- if any(row.get('pixels') for row in normalized):+ keys, cache, hits = {}, None, set()+ if cache_root and any(row.get('pixels') for row in normalized):+ # The same picture or clip with another prompt reuses its latent and may skip loading the encoder.+ try:+ cache = _latent_cache(cache_root, base, device)+ for index, (row, ref) in enumerate(zip(normalized, refs)):+ if cache is None or not row.get('pixels'):+ continue+ mapped = np.load(row['pixels'], mmap_mode='r', allow_pickle=False)+ keys[index] = _pixels_key(mapped[:row['frames']])+ del mapped+ latent = _cached_latent(cache, keys[index], Path(row['pixels']).parent)+ if latent is not None:+ ref['latent'] = latent+ hits.add(index)+ except (OSError, ValueError, RuntimeError, KeyError, TypeError):+ cache = None+ print(json.dumps({'event': 'reference_latent_cache', 'hits': len(hits), 'misses': len(keys) - len(hits)}), flush=True)+ if any(row.get('pixels') for index, row in enumerate(normalized) if index not in hits):print(json.dumps({'event': 'media_encode_phase', 'phase': 'Encoding image/video references'}), flush=True)# Conditioning never decodes: the 9.0 GiB decoder in the shared shards# is not read. The independent output decoder is loaded after sampling.@@ -239,8 +317,8 @@ def encode_latents(normalized, value, base, canvas, *, resident=None, device='cufrom .backends.mps_vae_encode import bounded_encoderlifetime = bounded_encoder(vae)with lifetime as memory:- for row, ref in zip(normalized, refs):- if row.get('pixels'):+ for index, (row, ref) in enumerate(zip(normalized, refs)):+ if row.get('pixels') and index not in hits:mapped = np.load(row['pixels'], mmap_mode='r', allow_pickle=False)if row['frames'] == 1:pixels = torch.from_numpy(np.array(mapped[:1])).to(device).permute(3, 0, 1, 2)[None]@@ -249,6 +327,8 @@ def encode_latents(normalized, value, base, canvas, *, resident=None, device='cuelse:ref['latent'] = encode_visual(vae, mapped[:row['frames']], device=device)del mapped+ if cache is not None and index in keys:+ _remember_latent(cache, keys[index], ref['latent'], Path(row['pixels']).parent)from .streamed_weights import release_file_pagesrelease_file_pages([Path(row['pixels'])])if memory is not None: