Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
"""Read the encoder's streamed layers a few layers ahead of the forward pass.
When live RAM cannot hold the 15.7 GB checkpoint, neither whole-file read-ahead
runs and every layer the device does not keep is copied from the read-only
mapping through page faults, one small read at a time. A 16 GB Windows RTX 4060
Ti spent 127.6 s computing a prompt that takes 1-3 s on the device, at 0.11 GB/s,
while the same drive read the video model's weights at 0.24-0.72 GB/s in large
blocks a minute later.
The forward pass visits the layers in order and touches each one once, so the
next layers' bytes are known. This reader keeps a small number of layers in
front of the pass: it reads their byte ranges in large blocks (the faults then
find the pages in the OS cache) and gives the pages of finished layers back, so
the encoder never holds more than a few hundred MB beyond what it is using.
It keeps no private copy, CUDA allocation or pinned memory.
"""
import ctypes
import json
import mmap
import os
import re
import struct
import threading
import time
from pathlib import Path
from .encoder_prewarm import CHUNK
from .hardware import cgroup_memory
from .system import system_memory
LEAD_LAYERS = 3
# The window holds a few layers and gives finished ones back, so it needs no
# room beyond what the pass would take for itself; only a nearly empty machine
# (live RAM below this) pauses it.
PRESSURE_FLOOR_BYTES = 32 * 2**20
# How long one layer may wait for room before the reader stands down for good.
STALL_LIMIT_SECONDS = 20.
LAYER_NAME = re.compile(r'(?:^|\.)layers\.(\d+)\.')
EMBED_NAME = re.compile(r'(?:^|\.)embed_tokens\.')
def available_memory():
"""Live RAM the window may count on: clean file cache is returned on demand, so the cgroup's
reclaimable headroom is used, not the raw remainder under the cap (which cache always fills)."""
available = system_memory()['available_bytes']
group = cgroup_memory()
reclaimable = group.get('reclaimable_available_bytes')
if reclaimable is not None:
available = min(available, reclaimable)
return available, group.get('complete', True)
def layer_ranges(path):
"""File byte ranges of the language model's embedding (key -1) and each layer, merged and sorted."""
with open(path, 'rb') as stream:
size = os.fstat(stream.fileno()).st_size
prefix = stream.read(8)
if len(prefix) != 8:
raise ValueError('Incomplete safetensors header')
header_size = struct.unpack('<Q', prefix)[0]
if not 2 <= header_size <= min(100_000_000, size - 8):
raise ValueError('Invalid safetensors header size')
header = json.loads(stream.read(header_size))
base = 8 + header_size
found = {}
for name, info in header.items():
if name == '__metadata__' or name.startswith('visual.') or '.visual.' in name:
continue
match = LAYER_NAME.search(name)
key = int(match.group(1)) if match else -1 if EMBED_NAME.search(name) else None
if key is None:
continue
begin, end = info['data_offsets']
if end > begin:
found.setdefault(key, []).append((base + begin, base + end))
merged = {}
for key, spans in found.items():
spans.sort()
out = [list(spans[0])]
for begin, end in spans[1:]:
if begin <= out[-1][1]:
out[-1][1] = max(out[-1][1], end)
else:
out.append([begin, end])
merged[key] = [tuple(span) for span in out]
return merged
def _mapping_base(path):
"""Address of the checkpoint mapping's first byte, from the loader's own record, or None."""
from .encoder_checkpoint import MAPPINGS, READ_ONLY_RANGES
live = [reference() for reference in MAPPINGS.get(os.path.realpath(path), [])]
live = [mapping for mapping in live if mapping is not None and not mapping.closed]
if len(live) != 1:
return None, None
for reference, begin, end in READ_ONLY_RANGES:
if reference() is live[0]:
return live[0], begin
return live[0], None
class LayerWindow:
def __init__(self, path, modules, *, lead=LEAD_LAYERS, release=True):
"""`modules` maps a layer key (-1 embedding, else layer index) to the module whose forward touches it."""
self.path = Path(path)
self.modules = modules
self.lead = max(1, int(lead))
self.release = release
self.cond = threading.Condition()
self.cancelled = False
self.thread = None
self.handles = []
self.plan = [] # [(key, ranges, bytes)] in execution order
self.reached = -1 # plan position the pass has entered
self.next = 0 # plan position the reader will read next
self.released = 0 # plan positions already given back, this sweep
self.closed = None
self.mapping = None
self.address = None
self.result = dict(state='not-started', layers=0, read_bytes=0, released_bytes=0, stalls=0,
sweeps=0, elapsed_seconds=0., lead_layers=self.lead,
scope='Streamed encoder layers; reclaimable OS cache, no GPU residency')
def start(self):
if self.thread is not None:
return self
ranges = layer_ranges(self.path)
keys = [key for key in sorted(ranges) if key in self.modules]
self.plan = [(key, ranges[key], sum(end - begin for begin, end in ranges[key])) for key in keys]
if not self.plan:
self.result.update(state='skipped', reason='No streamed layers')
return self
self.stamp = self.path.stat()
self.mapping, self.address = _mapping_base(self.path)
position = {key: index for index, (key, _, _) in enumerate(self.plan)}
for key, module in self.modules.items():
if key in position:
self.handles.append(module.register_forward_pre_hook(self._entered(position[key])))
self.result.update(state='reading', layers=len(self.plan),
planned_bytes=sum(size for _, _, size in self.plan),
reason='The forward pass is read a few layers ahead of its page faults')
self.thread = threading.Thread(target=self._run, name='FreeVideo encoder window', daemon=True)
self.thread.start()
return self
def _entered(self, position):
def hook(module, args, kwargs=None):
with self.cond:
if position <= self.reached:
# The pass started again (a retry): read from here once more.
self.result['sweeps'] += 1
self.next = position
self.released = position
self.reached = position
self.cond.notify_all()
return hook
def _run(self):
started = time.monotonic()
scratch = bytearray(CHUNK)
try:
with self.path.open('rb', buffering=0) as file:
while True:
with self.cond:
while not self.cancelled and (self.next >= len(self.plan)
or self.next > self.reached + self.lead):
self.cond.wait(0.1)
self._give_back()
if self.cancelled:
self.result['state'] = 'stopped'
self.result['reason'] = 'The encoder finished'
return
position = self.next
key, spans, size = self.plan[position]
if not self._room(size):
self.result.update(state='stopped', reason='Live RAM stayed too low to read ahead')
return
for begin, end in spans:
file.seek(begin)
remaining = end - begin
while remaining:
if self.cancelled:
break
count = file.readinto(memoryview(scratch)[:min(CHUNK, remaining)])
if not count:
raise ValueError('Encoder checkpoint truncated during read window')
remaining -= count
self.result['read_bytes'] += count
with self.cond:
if self.next == position:
self.next = position + 1
self._give_back()
except Exception as error:
# Speculative I/O never replaces the loader's own reads and validation.
self.result.update(state='failed', reason=type(error).__name__ + ': ' + str(error))
finally:
self.result['elapsed_seconds'] = time.monotonic() - started
def _room(self, size):
waited = time.monotonic()
while not self.cancelled:
available, complete = available_memory()
if not complete or available >= PRESSURE_FLOOR_BYTES:
return True
self.result['stalls'] += 1
if time.monotonic() - waited > STALL_LIMIT_SECONDS:
return False
time.sleep(0.02)
return True
def _give_back(self):
"""Release the pages of layers the pass has finished with (called holding the lock)."""
if not self.release:
return
while self.released < self.reached - 1 and self.released < len(self.plan):
for begin, end in self.plan[self.released][1]:
self.result['released_bytes'] += release_range(self.mapping, self.address, begin, end)
self.released += 1
def close(self):
"""Stop reading and remove the hooks; return what was done. Later calls return the same result."""
if self.closed is None:
with self.cond:
self.cancelled = True
self.cond.notify_all()
for handle in self.handles:
handle.remove()
self.handles.clear()
if self.thread is not None:
self.thread.join(timeout=2.)
if self.result['state'] == 'reading':
self.result['state'] = 'stopped'
self.closed = dict(self.result, reader_still_stopping=bool(self.thread and self.thread.is_alive()))
return self.closed
def release_range(mapping, address, begin, end):
"""Give the pages of [begin, end) of the checkpoint back; clean pages of a read-only file refault on use.
Linux/macOS advise the mapping away. Windows removes the pages from the
working set, which is where mapped pages stay until the system trims them.
Returns the bytes advised, 0 when nothing could be done."""
page = mmap.PAGESIZE
first = -(-begin // page) * page
last = end // page * page
if last <= first:
return 0
try:
if os.name == 'nt':
if address is None:
return 0
ctypes.windll.kernel32.VirtualUnlock(ctypes.c_void_p(address + first), ctypes.c_size_t(last - first))
return last - first
if mapping is None or not hasattr(mapping, 'madvise') or not hasattr(mmap, 'MADV_DONTNEED'):
return 0
mapping.madvise(mmap.MADV_DONTNEED, first, last - first)
return last - first
except (OSError, ValueError, AttributeError):
return 0
def layer_modules(model, owns_mapped):
"""{key: module} for the language model's embedding and layers whose weights still sit in the mapping."""
from .encoder_lowmem import language_model
llama = language_model(model)
found = {}
embed = getattr(llama, 'embed_tokens', None)
if embed is not None and any(owns_mapped(p) for p in embed.parameters()):
found[-1] = embed
for index, layer in enumerate(llama.layers):
if any(owns_mapped(p) for p in layer.parameters()):
found[index] = layer
return found