Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Keep the decoder's converted streamed layers in RAM between clips when memory allows

Bowen Xue84044039f9c4
diff --git a/freevideo_engine/decode.py b/freevideo_engine/decode.py
index 7ed5a3e..3f2ca80 100644
--- a/freevideo_engine/decode.py
+++ b/freevideo_engine/decode.py
@@ -72,7 +72,8 @@ def decode_to_file(latents, audio_latents, out_path, *, base, offload=False, pre
from .streamed_weights import SafetensorLayers
weight_source = SafetensorLayers((Path(base)/'vae').glob('*.safetensors'),
[f'decoder.transformer_blocks.{index}.' for index in range(count, layers)],
- intermediate_dtype=torch.float32 if linear_compute_cache else None)
+ intermediate_dtype=torch.float32 if linear_compute_cache else None,
+ keep_converted=True)
vae, compute_cache_metrics = load_video_decoder(base, resident_blocks=count, weight_source=weight_source,
linear_fp16=linear_compute_cache)
vae.eval().requires_grad_(False)
diff --git a/freevideo_engine/offload.py b/freevideo_engine/offload.py
index 085db76..335ab91 100644
--- a/freevideo_engine/offload.py
+++ b/freevideo_engine/offload.py
@@ -905,6 +905,11 @@ class LayerOffloader:
'host_view_hits': getattr(self.weight_source, 'host_view_hits', 0) - self.host_view_hits_start,
'host_view_hit_bytes': getattr(self.weight_source, 'host_view_hit_bytes', 0) - self.host_view_hit_bytes_start,
'host_view_released': getattr(self.weight_source, 'host_view_released', 0),
+ # Converted layers kept in RAM for the clips still to come.
+ 'converted_layer_count': len(getattr(self.weight_source, 'converted', None) or {}),
+ 'converted_bytes': getattr(self.weight_source, 'converted_bytes', 0),
+ 'converted_hits': getattr(self.weight_source, 'converted_hits', 0),
+ 'converted_refused': getattr(self.weight_source, 'converted_refused', 0),
# Direct reads kept in the file cache between steps and those passing through.
'cache_kept_layer_count': len(getattr(self.weight_source, 'cache_kept', None) or {}),
'cache_kept_bytes': getattr(self.weight_source, 'cache_kept_bytes', 0),
@@ -950,6 +955,9 @@ class LayerOffloader:
self.cached.clear()
self.cache_candidates.clear()
self.slots.clear()
+ release = getattr(self.weight_source, 'release_converted', None)
+ if release is not None:
+ release()
self.closed = True
def __enter__(self):
diff --git a/freevideo_engine/streamed_weights.py b/freevideo_engine/streamed_weights.py
index af23e44..36e172f 100644
--- a/freevideo_engine/streamed_weights.py
+++ b/freevideo_engine/streamed_weights.py
@@ -383,6 +383,43 @@ def _low_page_priority(enabled):
swap(previous)
+# Converted decoder layers are private copies and charge commit, but they stand
+# in for the clean file pages the streamed reads would otherwise keep cycling
+# through the standby list, so the floor left to the rest of the system is
+# smaller than for views. Admission is re-checked at every layer.
+CONVERTED_RESERVE = int(1.5 * 2**30)
+
+
+def _cgroup_headroom(mount='/sys/fs/cgroup', membership='/proc/self/cgroup'):
+ """Room left under this process's cgroup v2 memory limit, or None without one.
+
+ Private copies count against the limit while clean file pages do not, so
+ only anonymous, shared and kernel memory is taken as used.
+ """
+ try:
+ root = Path(mount)
+ for line in Path(membership).read_text(encoding='utf-8').splitlines():
+ if line.startswith('0::'):
+ root = root / line[3:].lstrip('/')
+ limit = (root / 'memory.max').read_text(encoding='utf-8').strip()
+ if limit == 'max':
+ return None
+ stat = dict(row.split() for row in (root / 'memory.stat').read_text(encoding='utf-8').splitlines())
+ return int(limit) - sum(int(stat.get(name, 0)) for name in ('anon', 'shmem', 'kernel'))
+ except (OSError, ValueError):
+ return None
+
+
+def converted_fits(size, reserve=CONVERTED_RESERVE):
+ """Whether one more converted layer may stay in RAM as a private copy."""
+ from .system import system_memory
+ available = system_memory()['available_bytes']
+ headroom = _cgroup_headroom()
+ if headroom is not None:
+ available = min(available, headroom)
+ return available - size >= reserve
+
+
def host_view_fits(size, reserve=HOST_VIEW_RESERVE):
"""Whether one more layer may stay in RAM as a read-only file view.
@@ -399,11 +436,20 @@ def host_view_fits(size, reserve=HOST_VIEW_RESERVE):
class SafetensorLayers:
def __init__(self, paths, prefixes, *, intermediate_dtype=None, direct_read=None, host_views=False,
- host_view_reserve=HOST_VIEW_RESERVE):
+ host_view_reserve=HOST_VIEW_RESERVE, keep_converted=False):
from .system import windows
self.direct_read = windows() if direct_read is None else direct_read
self.prefixes = tuple(prefixes)
self.intermediate_dtype = intermediate_dtype
+ # Layers converted from their checkpoint dtype (the decoder's FP16
+ # Linear values) kept for the layers' next uses. A tile decode streams
+ # the same layers once per clip; converting reads the whole FP32 layer
+ # each time, from disk once the file cache cannot hold them.
+ self.keep_converted = keep_converted and intermediate_dtype is not None
+ self.converted = {}
+ self.converted_bytes = 0
+ self.converted_hits = 0
+ self.converted_refused = 0
self.files = {}
self.handle_files = {}
self.keys = {}
@@ -701,6 +747,25 @@ class SafetensorLayers:
else torch.empty(shape, dtype=dtype))
return tensors
+ def _keep_converted(self, index, tensors, shared):
+ """Keep this layer's converted tensors if RAM has room, as private copies."""
+ size = sum(value.numel() * value.element_size() for value in tensors.values())
+ with self.cache_lock:
+ if index in self.converted:
+ return
+ if not converted_fits(size):
+ self.converted_refused += 1
+ return
+ # A tensor that needed no conversion may still map the checkpoint.
+ self.converted[index] = {name: value.clone() if name in shared else value
+ for name, value in tensors.items()}
+ self.converted_bytes += size
+
+ def release_converted(self):
+ with self.cache_lock:
+ self.converted.clear()
+ self.converted_bytes = 0
+
def adopt(self, index):
"""Keep a layer that loading just read as views, so its first pass reads no disk.
@@ -812,8 +877,16 @@ class SafetensorLayers:
@contextmanager
def layer(self, index, layout):
from contextlib import ExitStack
+ with self.cache_lock:
+ kept = self.converted.get(index)
+ if kept is not None:
+ self.converted_hits += 1
+ if kept is not None:
+ yield dict(kept)
+ return
prefix = self.prefixes[index]
tensors = {}
+ shared = set()
paths = {self.keys[prefix + name] for _, name, *_ in layout}
for path in paths:
self.check(path)
@@ -831,6 +904,10 @@ class SafetensorLayers:
if self.intermediate_dtype is not None:
tensor = tensor.to(dtype=self.intermediate_dtype)
tensors[name] = tensor.to(dtype=dtype)
+ if tensors[name] is tensor:
+ shared.add(name)
+ if self.keep_converted:
+ self._keep_converted(index, tensors, shared)
yield tensors
finally:
tensors.clear()