Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Reuse reference picture and clip latents when only the prompt changes

Bowen Xuefe5a71998bba
diff --git a/freevideo_engine/encode_worker.py b/freevideo_engine/encode_worker.py
index 7df26d0..6704639 100644
--- a/freevideo_engine/encode_worker.py
+++ b/freevideo_engine/encode_worker.py
@@ -645,7 +645,8 @@ def _encode(args, request, resident, trace):
add_vdn()
from .media_encoding import encode_latents
tick = time.perf_counter()
- info = encode_latents(normalized, value, request['base'], request['geometry'], resident=resident)
+ info = encode_latents(normalized, value, request['base'], request['geometry'], resident=resident,
+ cache_root=request.get('reference_latent_cache'))
keyframe_metrics.update(media_vae_encode_seconds=time.perf_counter() - tick, conditioning_info=info)
task = info['task']
elif early_reference is not None:
diff --git a/freevideo_engine/generate.py b/freevideo_engine/generate.py
index d4fbaa1..b4d0bc5 100644
--- a/freevideo_engine/generate.py
+++ b/freevideo_engine/generate.py
@@ -752,7 +752,8 @@ def _run(args):
'metrics': str(destination.with_suffix('.encoding.json')),
'diagnostic_output': str(destination)}
if media:
- encoding.update(media=media, geometry=canvas, base=profile['engine'].get('base'))
+ encoding.update(media=media, geometry=canvas, base=profile['engine'].get('base'),
+ reference_latent_cache=str(input_cache.root.parent.parent) if input_cache else None)
# Lets the encoder child read the video model while it computes, on a slow drive.
from .model_read_ahead import request_fields
if getattr(args, 'cache', None):
diff --git a/freevideo_engine/media_encoding.py b/freevideo_engine/media_encoding.py
index 31da96f..5014737 100644
--- a/freevideo_engine/media_encoding.py
+++ b/freevideo_engine/media_encoding.py
@@ -217,7 +217,67 @@ def reference_options(normalized, options, step):
return {'minimax_ref_items': items} if items else {}
-def encode_latents(normalized, value, base, canvas, *, resident=None, device='cuda'):
+def _latent_cache(root, base, device):
+ """Per-picture latent receipts; another encoder, recipe or device never matches."""
+ import torch
+ from .input_cache import InputCache, digest
+ directory = Path(base) / 'vae'
+ files = sorted(directory.glob('*.safetensors'))
+ if not files:
+ return None
+ kind = torch.device(device).type
+ try:
+ from importlib.metadata import version
+ diffusers = version('diffusers')
+ except Exception:
+ diffusers = None
+ from . import __version__
+ identity = {'recipe': 'reference-latent-v1', 'engine': __version__, 'diffusers': diffusers,
+ 'torch': torch.__version__, 'device': kind,
+ 'gpu': torch.cuda.get_device_name() if kind == 'cuda' else None,
+ 'config': digest(directory / 'config.json') if (directory / 'config.json').exists() else None,
+ 'weights': [[f.name, f.stat().st_size, f.stat().st_mtime_ns] for f in files]}
+ return InputCache(identity, root=root, kind='reference-latent')
+
+
+def _pixels_key(mapped):
+ import hashlib
+ import numpy as np
+ value = hashlib.sha256(json.dumps([list(mapped.shape), str(mapped.dtype)]).encode())
+ for begin in range(0, len(mapped), 8):
+ value.update(np.ascontiguousarray(mapped[begin:begin + 8]).reshape(-1).view('u1').data)
+ return value.hexdigest()
+
+
+def _cached_latent(cache, key, directory):
+ import torch
+ target = Path(directory) / ('reference-latent-' + key[:16] + '.hit.pt')
+ try:
+ if cache.get(key, target) is None:
+ return None
+ latent = torch.load(target, map_location='cpu', weights_only=True)
+ if isinstance(latent, torch.Tensor) and latent.is_floating_point() and bool(torch.isfinite(latent).all()):
+ return latent
+ except (OSError, ValueError, RuntimeError, KeyError, TypeError, EOFError):
+ pass
+ finally:
+ target.unlink(missing_ok=True)
+ return None
+
+
+def _remember_latent(cache, key, latent, directory):
+ import torch
+ path = Path(directory) / ('reference-latent-' + key[:16] + '.save.pt')
+ try:
+ torch.save(latent, path)
+ cache.put(key, path, {'success': True})
+ except (OSError, ValueError, RuntimeError):
+ pass
+ finally:
+ path.unlink(missing_ok=True)
+
+
+def encode_latents(normalized, value, base, canvas, *, resident=None, device='cuda', cache_root=None):
import gc
import numpy as np
import torch
@@ -228,7 +288,25 @@ def encode_latents(normalized, value, base, canvas, *, resident=None, device='cu
raise ValueError('Media encoding requires a CUDA or MPS device')
empty_cache = torch.mps.empty_cache if torch.device(device).type == 'mps' else torch.cuda.empty_cache
refs = [{key: row[key] for key in ('kind', 'early_steps') if key in row} for row in normalized]
- if any(row.get('pixels') for row in normalized):
+ keys, cache, hits = {}, None, set()
+ if cache_root and any(row.get('pixels') for row in normalized):
+ # The same picture or clip with another prompt reuses its latent and may skip loading the encoder.
+ try:
+ cache = _latent_cache(cache_root, base, device)
+ for index, (row, ref) in enumerate(zip(normalized, refs)):
+ if cache is None or not row.get('pixels'):
+ continue
+ mapped = np.load(row['pixels'], mmap_mode='r', allow_pickle=False)
+ keys[index] = _pixels_key(mapped[:row['frames']])
+ del mapped
+ latent = _cached_latent(cache, keys[index], Path(row['pixels']).parent)
+ if latent is not None:
+ ref['latent'] = latent
+ hits.add(index)
+ except (OSError, ValueError, RuntimeError, KeyError, TypeError):
+ cache = None
+ print(json.dumps({'event': 'reference_latent_cache', 'hits': len(hits), 'misses': len(keys) - len(hits)}), flush=True)
+ if any(row.get('pixels') for index, row in enumerate(normalized) if index not in hits):
print(json.dumps({'event': 'media_encode_phase', 'phase': 'Encoding image/video references'}), flush=True)
# Conditioning never decodes: the 9.0 GiB decoder in the shared shards
# is not read. The independent output decoder is loaded after sampling.
@@ -239,8 +317,8 @@ def encode_latents(normalized, value, base, canvas, *, resident=None, device='cu
from .backends.mps_vae_encode import bounded_encoder
lifetime = bounded_encoder(vae)
with lifetime as memory:
- for row, ref in zip(normalized, refs):
- if row.get('pixels'):
+ for index, (row, ref) in enumerate(zip(normalized, refs)):
+ if row.get('pixels') and index not in hits:
mapped = np.load(row['pixels'], mmap_mode='r', allow_pickle=False)
if row['frames'] == 1:
pixels = torch.from_numpy(np.array(mapped[:1])).to(device).permute(3, 0, 1, 2)[None]
@@ -249,6 +327,8 @@ def encode_latents(normalized, value, base, canvas, *, resident=None, device='cu
else:
ref['latent'] = encode_visual(vae, mapped[:row['frames']], device=device)
del mapped
+ if cache is not None and index in keys:
+ _remember_latent(cache, keys[index], ref['latent'], Path(row['pixels']).parent)
from .streamed_weights import release_file_pages
release_file_pages([Path(row['pixels'])])
if memory is not None: