Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Read prepared weights and the VAE decoder on several threads (#36)

Bowen Xue5360ddbc0c65
diff --git a/freevideo_engine/runtime.py b/freevideo_engine/runtime.py
index 7ae533a..f1ec4b0 100644
--- a/freevideo_engine/runtime.py
+++ b/freevideo_engine/runtime.py
@@ -26,8 +26,19 @@ def _load_safetensors(path):
reserves system commit for the whole shard. During model preparation we
only need the tensors, so read them through the platform-aware streamed
opener and close the handle before returning the dictionary. Linux keeps
- its mmap path; Windows uses safetensors' bounded ``pread`` backend.
+ its mmap path. Windows reads each shard on a few threads without a
+ mapping (``tensor_io.ParallelReads``), twice as fast as safetensors'
+ ``pread`` backend on the RTX 5060 Ti machine.
"""
+ from .system import windows
+ if windows():
+ from .tensor_io import ParallelReads
+ reads = ParallelReads(path)
+ try:
+ if not reads.skipped:
+ return reads.tensors(sorted(reads.entries, key=lambda name: reads.entries[name][2]))
+ finally:
+ reads.close()
from .streamed_weights import _open_safetensors
with _open_safetensors(path, framework='pt', device='cpu') as handle:
return {name: handle.get_tensor(name) for name in handle.keys()}
diff --git a/freevideo_engine/tensor_io.py b/freevideo_engine/tensor_io.py
index 0d4bbc5..df006d3 100644
--- a/freevideo_engine/tensor_io.py
+++ b/freevideo_engine/tensor_io.py
@@ -1,4 +1,142 @@
"""Bounded CPU tensor reads for model preparation, without persistent file maps."""
+import json
+import os
+from pathlib import Path
+import threading
+
+_DTYPES = {'BOOL': 'bool', 'U8': 'uint8', 'I8': 'int8', 'I16': 'int16', 'I32': 'int32', 'I64': 'int64',
+ 'F16': 'float16', 'BF16': 'bfloat16', 'F32': 'float32', 'F64': 'float64',
+ 'F8_E4M3': 'float8_e4m3fn', 'F8_E5M2': 'float8_e5m2'}
+_PIECE = 8 << 20
+
+
+def read_threads():
+ """Readers per shard. FREEVIDEO_READ_THREADS=1 reads each tensor in one call."""
+ value = os.environ.get('FREEVIDEO_READ_THREADS', '')
+ return max(1, int(value)) if value.isdigit() else 4
+
+
+class ParallelReads:
+ """Whole tensors of one safetensors file, read in pieces on a few threads.
+
+ On the Windows RTX 5060 Ti machine, safetensors' pread backend read the
+ prepared blocks at 1.04 GB/s, and at 1.05 GB/s with two files at once;
+ the disk delivers 2.5 GB/s. FileIO.readinto releases the GIL, and with
+ one handle per thread these readers reached 2.1 GB/s. Each tensor owns
+ exactly its bytes, as with pread, and every handle closes with the
+ reader. Entries of a dtype listed nowhere here are left to safetensors.
+ """
+
+ def __init__(self, path, threads=None):
+ self.path = Path(path)
+ size = self.path.stat().st_size
+ with self.path.open('rb', buffering=0) as stream:
+ length = int.from_bytes(stream.read(8), 'little')
+ if not 2 <= length <= min(100_000_000, size - 8):
+ raise ValueError('Invalid safetensors header size: ' + str(self.path))
+ header = json.loads(stream.read(length))
+ header.pop('__metadata__', None)
+ self.base = 8 + length
+ self.entries, self.skipped = {}, []
+ import torch
+ for name, entry in header.items():
+ start, end = entry['data_offsets']
+ if not 0 <= start <= end <= size - self.base:
+ raise ValueError('Invalid safetensors entry %s in %s' % (name, self.path))
+ dtype = _DTYPES.get(entry['dtype'])
+ if dtype is None:
+ self.skipped.append(name)
+ continue
+ count = 1
+ for extent in entry['shape']:
+ count *= extent
+ if count * getattr(torch, dtype).itemsize != end - start:
+ raise ValueError('Invalid safetensors entry %s in %s' % (name, self.path))
+ self.entries[name] = (dtype, tuple(entry['shape']), start, end)
+ self.threads = read_threads() if threads is None else threads
+ self.pool = None
+ self.local = threading.local()
+ self.lock = threading.Lock()
+ self.streams = []
+
+ def _stream(self):
+ stream = getattr(self.local, 'stream', None)
+ if stream is None:
+ stream = open(self.path, 'rb', buffering=0)
+ with self.lock:
+ self.streams.append(stream)
+ self.local.stream = stream
+ return stream
+
+ def _read(self, piece):
+ position, view = piece
+ stream = self._stream()
+ stream.seek(position)
+ done = 0
+ while done < len(view):
+ count = stream.readinto(view[done:])
+ if not count:
+ raise ValueError('Truncated safetensors payload: ' + str(self.path))
+ done += count
+
+ def tensors(self, names):
+ """Read several whole tensors; all their pieces share the thread pool."""
+ import torch
+ results, pieces = {}, []
+ for name in names:
+ dtype, shape, start, end = self.entries[name]
+ raw = torch.empty(end - start, dtype=torch.uint8)
+ view = memoryview(raw.numpy())
+ pieces += [(self.base + start + offset, view[offset:offset + _PIECE])
+ for offset in range(0, end - start, _PIECE)]
+ results[name] = raw.view(getattr(torch, dtype)).reshape(shape)
+ if self.threads > 1 and len(pieces) > 1:
+ if self.pool is None:
+ from concurrent.futures import ThreadPoolExecutor
+ self.pool = ThreadPoolExecutor(self.threads, thread_name_prefix='freevideo-read')
+ list(self.pool.map(self._read, pieces))
+ else:
+ for piece in pieces:
+ self._read(piece)
+ return results
+
+ def tensor(self, name):
+ return self.tensors([name])[name]
+
+ def close(self):
+ if self.pool is not None:
+ self.pool.shutdown()
+ self.pool = None
+ with self.lock:
+ streams, self.streams = self.streams, []
+ for stream in streams:
+ stream.close()
+
+
+class _Tensors:
+ """A pread handle whose whole-tensor reads use ParallelReads."""
+
+ def __init__(self, handle, path):
+ self._handle = handle
+ self._reads = ParallelReads(path)
+
+ def __enter__(self):
+ self._handle.__enter__()
+ return self
+
+ def __exit__(self, *exc):
+ try:
+ return self._handle.__exit__(*exc)
+ finally:
+ self._reads.close()
+
+ def get_tensor(self, name):
+ if name not in self._reads.entries:
+ return self._handle.get_tensor(name)
+ return self._reads.tensor(name)
+
+ def __getattr__(self, name):
+ return getattr(self._handle, name)
def open_tensors(path, *, framework='pt'):
@@ -8,7 +146,8 @@ def open_tensors(path, *, framework='pt'):
# pread owns only the requested tensor's bytes, including BF16/FP8 tensors.
# Do not silently fall back to mmap on an older manually installed runtime.
try:
- return safe_open(path, framework=framework, device='cpu', backend='pread')
+ handle = safe_open(path, framework=framework, device='cpu', backend='pread')
except TypeError as error:
raise RuntimeError('Bounded model preparation requires safetensors >= 0.8.0. '
'Rerun setup to update the managed environment.') from error
+ return _Tensors(handle, path) if framework == 'pt' else handle