Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
Read prepared weights and the VAE decoder on several threads (#36)
Bowen Xue5360ddbc0c65
diff --git a/freevideo_engine/runtime.py b/freevideo_engine/runtime.pyindex 7ae533a..f1ec4b0 100644--- a/freevideo_engine/runtime.py+++ b/freevideo_engine/runtime.py@@ -26,8 +26,19 @@ def _load_safetensors(path):reserves system commit for the whole shard. During model preparation weonly need the tensors, so read them through the platform-aware streamedopener and close the handle before returning the dictionary. Linux keeps- its mmap path; Windows uses safetensors' bounded ``pread`` backend.+ its mmap path. Windows reads each shard on a few threads without a+ mapping (``tensor_io.ParallelReads``), twice as fast as safetensors'+ ``pread`` backend on the RTX 5060 Ti machine."""+ from .system import windows+ if windows():+ from .tensor_io import ParallelReads+ reads = ParallelReads(path)+ try:+ if not reads.skipped:+ return reads.tensors(sorted(reads.entries, key=lambda name: reads.entries[name][2]))+ finally:+ reads.close()from .streamed_weights import _open_safetensorswith _open_safetensors(path, framework='pt', device='cpu') as handle:return {name: handle.get_tensor(name) for name in handle.keys()}diff --git a/freevideo_engine/tensor_io.py b/freevideo_engine/tensor_io.pyindex 0d4bbc5..df006d3 100644--- a/freevideo_engine/tensor_io.py+++ b/freevideo_engine/tensor_io.py@@ -1,4 +1,142 @@"""Bounded CPU tensor reads for model preparation, without persistent file maps."""+import json+import os+from pathlib import Path+import threading++_DTYPES = {'BOOL': 'bool', 'U8': 'uint8', 'I8': 'int8', 'I16': 'int16', 'I32': 'int32', 'I64': 'int64',+ 'F16': 'float16', 'BF16': 'bfloat16', 'F32': 'float32', 'F64': 'float64',+ 'F8_E4M3': 'float8_e4m3fn', 'F8_E5M2': 'float8_e5m2'}+_PIECE = 8 << 20+++def read_threads():+ """Readers per shard. FREEVIDEO_READ_THREADS=1 reads each tensor in one call."""+ value = os.environ.get('FREEVIDEO_READ_THREADS', '')+ return max(1, int(value)) if value.isdigit() else 4+++class ParallelReads:+ """Whole tensors of one safetensors file, read in pieces on a few threads.++ On the Windows RTX 5060 Ti machine, safetensors' pread backend read the+ prepared blocks at 1.04 GB/s, and at 1.05 GB/s with two files at once;+ the disk delivers 2.5 GB/s. FileIO.readinto releases the GIL, and with+ one handle per thread these readers reached 2.1 GB/s. Each tensor owns+ exactly its bytes, as with pread, and every handle closes with the+ reader. Entries of a dtype listed nowhere here are left to safetensors.+ """++ def __init__(self, path, threads=None):+ self.path = Path(path)+ size = self.path.stat().st_size+ with self.path.open('rb', buffering=0) as stream:+ length = int.from_bytes(stream.read(8), 'little')+ if not 2 <= length <= min(100_000_000, size - 8):+ raise ValueError('Invalid safetensors header size: ' + str(self.path))+ header = json.loads(stream.read(length))+ header.pop('__metadata__', None)+ self.base = 8 + length+ self.entries, self.skipped = {}, []+ import torch+ for name, entry in header.items():+ start, end = entry['data_offsets']+ if not 0 <= start <= end <= size - self.base:+ raise ValueError('Invalid safetensors entry %s in %s' % (name, self.path))+ dtype = _DTYPES.get(entry['dtype'])+ if dtype is None:+ self.skipped.append(name)+ continue+ count = 1+ for extent in entry['shape']:+ count *= extent+ if count * getattr(torch, dtype).itemsize != end - start:+ raise ValueError('Invalid safetensors entry %s in %s' % (name, self.path))+ self.entries[name] = (dtype, tuple(entry['shape']), start, end)+ self.threads = read_threads() if threads is None else threads+ self.pool = None+ self.local = threading.local()+ self.lock = threading.Lock()+ self.streams = []++ def _stream(self):+ stream = getattr(self.local, 'stream', None)+ if stream is None:+ stream = open(self.path, 'rb', buffering=0)+ with self.lock:+ self.streams.append(stream)+ self.local.stream = stream+ return stream++ def _read(self, piece):+ position, view = piece+ stream = self._stream()+ stream.seek(position)+ done = 0+ while done < len(view):+ count = stream.readinto(view[done:])+ if not count:+ raise ValueError('Truncated safetensors payload: ' + str(self.path))+ done += count++ def tensors(self, names):+ """Read several whole tensors; all their pieces share the thread pool."""+ import torch+ results, pieces = {}, []+ for name in names:+ dtype, shape, start, end = self.entries[name]+ raw = torch.empty(end - start, dtype=torch.uint8)+ view = memoryview(raw.numpy())+ pieces += [(self.base + start + offset, view[offset:offset + _PIECE])+ for offset in range(0, end - start, _PIECE)]+ results[name] = raw.view(getattr(torch, dtype)).reshape(shape)+ if self.threads > 1 and len(pieces) > 1:+ if self.pool is None:+ from concurrent.futures import ThreadPoolExecutor+ self.pool = ThreadPoolExecutor(self.threads, thread_name_prefix='freevideo-read')+ list(self.pool.map(self._read, pieces))+ else:+ for piece in pieces:+ self._read(piece)+ return results++ def tensor(self, name):+ return self.tensors([name])[name]++ def close(self):+ if self.pool is not None:+ self.pool.shutdown()+ self.pool = None+ with self.lock:+ streams, self.streams = self.streams, []+ for stream in streams:+ stream.close()+++class _Tensors:+ """A pread handle whose whole-tensor reads use ParallelReads."""++ def __init__(self, handle, path):+ self._handle = handle+ self._reads = ParallelReads(path)++ def __enter__(self):+ self._handle.__enter__()+ return self++ def __exit__(self, *exc):+ try:+ return self._handle.__exit__(*exc)+ finally:+ self._reads.close()++ def get_tensor(self, name):+ if name not in self._reads.entries:+ return self._handle.get_tensor(name)+ return self._reads.tensor(name)++ def __getattr__(self, name):+ return getattr(self._handle, name)def open_tensors(path, *, framework='pt'):@@ -8,7 +146,8 @@ def open_tensors(path, *, framework='pt'):# pread owns only the requested tensor's bytes, including BF16/FP8 tensors.# Do not silently fall back to mmap on an older manually installed runtime.try:- return safe_open(path, framework=framework, device='cpu', backend='pread')+ handle = safe_open(path, framework=framework, device='cpu', backend='pread')except TypeError as error:raise RuntimeError('Bounded model preparation requires safetensors >= 0.8.0. ''Rerun setup to update the managed environment.') from error+ return _Tensors(handle, path) if framework == 'pt' else handle