Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
Image model loading shows its progress on the bar
Bowen Xue2fdf5d43a908
diff --git a/freevideo_engine/image_gen/image_runner.py b/freevideo_engine/image_gen/image_runner.pyindex 593a0db..563f296 100644--- a/freevideo_engine/image_gen/image_runner.py+++ b/freevideo_engine/image_gen/image_runner.py@@ -8,6 +8,7 @@ table is looked up on the CPU, the VAE and the vision tower visit the cardonly for the stage that needs them."""import time+import mathfrom pathlib import Pathimport torch@@ -19,6 +20,36 @@ GiB = 2**30_DIT_FILE = 'diffusion_models/qwen_image_2.1_turbo_int8_convrot.safetensors'_TE_FILE = 'text_encoders/qwen3vl_8b_int8_convrot.safetensors'_VAE_FILE = 'vae/qwen_image_2.1_vae_bf16.safetensors'+# Rounded mean stage times from the local cold/hot constructor comparison.+# See benchmarks/image_load_progress_045afix/summary.txt (small-model fallback).+_LOAD_SHARES = dict(dit=.42, vae=.03, text=.18, pin=.37)+++class _LoadProgress:+ def __init__(self, callback):+ self.callback = callback+ self.fraction = self.emitted = 0.+ self.callback('load', 0.)+ self.last_time = time.monotonic()++ def report(self, stage, done, total):+ offset = 0.+ for name, share in _LOAD_SHARES.items():+ if name == stage:+ fraction = offset + share * (done / total if total else 0.)+ break+ offset += share+ self.fraction = max(self.fraction, min(fraction, math.nextafter(1., 0.)))+ now = time.monotonic()+ if self.fraction > self.emitted and now - self.last_time >= .25:+ self.callback('load', self.fraction)+ self.emitted, self.last_time = self.fraction, now++ def complete(self, stage):+ self.report(stage, 1, 1)++ def finish(self):+ self.callback('load', 1.)def _non_block_children(module, skip):@@ -86,24 +117,27 @@ class ImageRunner:nested = self.pack_dir / directory / namereturn nested if nested.exists() else self.pack_dir / name- progress = progress or (lambda stage, fraction: None)- progress('load', 0.)+ loading = _LoadProgress(progress or (lambda stage, fraction: None))started = time.monotonic()- self.transformer = load_dit(self.pack_dir / _DIT_FILE, device='cpu')+ self.transformer = load_dit(self.pack_dir / _DIT_FILE, device='cpu',+ progress=lambda done, total: loading.report('dit', done, total))self.load_seconds['dit'] = time.monotonic() - started- progress('load', 1 / 3)+ loading.complete('dit')started = time.monotonic()- self.vae = load_vae(self.pack_dir / _VAE_FILE, device='cpu')+ self.vae = load_vae(self.pack_dir / _VAE_FILE, device='cpu',+ progress=lambda done, total: loading.report('vae', done, total))self.load_seconds['vae'] = time.monotonic() - started- progress('load', 2 / 3)+ loading.complete('vae')started = time.monotonic()config_dir = self.pack_dir / 'text_encoder'processor_dir = self.pack_dir / 'processor'- text = load_text_encoder(self.pack_dir / _TE_FILE, config_path=config_dir / 'config.json', device='cpu')+ text = load_text_encoder(self.pack_dir / _TE_FILE, config_path=config_dir / 'config.json', device='cpu',+ progress=lambda done, total: loading.report('text', done, total))from transformers import Qwen3VLProcessorprocessor = Qwen3VLProcessor.from_pretrained(str(processor_dir), local_files_only=True)self.load_seconds['text'] = time.monotonic() - startedself.text_model = text+ loading.complete('text')# Parts that always sit on the card: everything of the DiT except its blocks, and the# encoder's final norm and rotary table. The 1.2 GB vocabulary stays on the host and is@@ -119,10 +153,23 @@ class ImageRunner:self._embed_hooks = self._lookup_on_host(language.embed_tokens)started = time.monotonic()+ from ..offload import cpu_weights+ stacks = [self.transformer.transformer_blocks, language.layers]+ stack_bytes = [sum(value.numel() * value.element_size() for layer in layers+ for _, value in cpu_weights(layer)) for layers in stacks]+ pin_total = sum(stack_bytes)++ def pin_progress(index):+ offset = sum(stack_bytes[:index])+ return lambda done, total: loading.report(+ 'pin', offset + stack_bytes[index] * (done / total if total else 0.), pin_total)+with torch.no_grad(): # LayerOffloader refuses to be built with autograd on- self.dit_stack = StreamedStack(self.transformer.transformer_blocks, name='dit', device=self.device)- self.te_stack = StreamedStack(language.layers, name='text_encoder', device=self.device)+ self.dit_stack = StreamedStack(stacks[0], name='dit', device=self.device, progress=pin_progress(0))+ loading.report('pin', stack_bytes[0], pin_total)+ self.te_stack = StreamedStack(stacks[1], name='text_encoder', device=self.device, progress=pin_progress(1))self.load_seconds['pin'] = time.monotonic() - started+ loading.complete('pin')self.shape = image_policy.ImageShape.measure(list(self.transformer.transformer_blocks), list(language.layers),@@ -133,7 +180,7 @@ class ImageRunner:from .qwen_pipeline import load_sample_sigmasself.generator.sample_sigmas = load_sample_sigmas(self.pack_dir / _DIT_FILE)self.last_plan = None- progress('load', 1.)+ loading.finish()def _lookup_on_host(self, embedding):device = self.devicediff --git a/freevideo_engine/image_gen/image_stream.py b/freevideo_engine/image_gen/image_stream.pyindex 7b783a5..ee8db9b 100644--- a/freevideo_engine/image_gen/image_stream.py+++ b/freevideo_engine/image_gen/image_stream.py@@ -16,7 +16,8 @@ from ..offload import LayerOffloader, pin_layer_weightsclass StreamedStack:"""``layers`` hold frozen CPU weights; they are moved to the card on use."""- def __init__(self, layers, *, name, device='cuda', pin=True, max_pin_bytes=None, prefetch=True):+ def __init__(self, layers, *, name, device='cuda', pin=True, max_pin_bytes=None, prefetch=True,+ progress=None):self.name = nameself.layers = list(layers)if not self.layers:@@ -28,7 +29,8 @@ class StreamedStack:# Pinning is an optimization: a host that refuses it costs transfer# speed, not the request (pin_layer_weights keeps the unpinned layers# on their ordinary storage and the offloader stages them).- self.pinned_bytes = pin_layer_weights(self.layers, max_bytes=max_pin_bytes) if pin else 0+ pin_options = {'progress': progress} if progress is not None else {}+ self.pinned_bytes = pin_layer_weights(self.layers, max_bytes=max_pin_bytes, **pin_options) if pin else 0self.offloader = LayerOffloader(self.layers, device=device, prefetch=prefetch, host_prefetch=False)self.layer_bytes = max(sum(count * parameter.element_size()for parameter, _, _, _, count, _ in entries)diff --git a/freevideo_engine/image_gen/load_progress.py b/freevideo_engine/image_gen/load_progress.pynew file mode 100644index 0000000..f33b8bf--- /dev/null+++ b/freevideo_engine/image_gen/load_progress.py@@ -0,0 +1,27 @@+"""Byte accounting for completed safetensor reads, without opening tensor data."""+from math import prod+++_DTYPE_BYTES = {'BOOL': 1, 'U8': 1, 'I8': 1, 'I16': 2, 'I32': 4, 'I64': 8,+ 'F16': 2, 'BF16': 2, 'F32': 4, 'F64': 8,+ 'F8_E4M3': 1, 'F8_E5M2': 1}+++class TensorProgress:+ def __init__(self, checkpoint, callback, keys=None):+ self.callback = callback+ self.sizes = {}+ for key in checkpoint.keys() if keys is None else keys:+ value = checkpoint.get_slice(key)+ self.sizes[key] = prod(value.get_shape()) * _DTYPE_BYTES[value.get_dtype()]+ self.total = sum(self.sizes.values())+ self.done = self.last = 0++ def tensors(self, *keys):+ self.advance(sum(self.sizes[key] for key in keys))++ def advance(self, size):+ self.done += size+ if self.done > self.last and ((self.done - self.last) * 50 >= self.total or self.done == self.total):+ self.last = self.done+ self.callback(self.done, self.total)diff --git a/freevideo_engine/image_gen/qwen_te_weights.py b/freevideo_engine/image_gen/qwen_te_weights.pyindex abfb066..86d6299 100644--- a/freevideo_engine/image_gen/qwen_te_weights.py+++ b/freevideo_engine/image_gen/qwen_te_weights.py@@ -14,6 +14,7 @@ from pathlib import Pathimport torchfrom safetensors import safe_open+from .load_progress import TensorProgress_CONVROT = {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 256}@@ -41,11 +42,13 @@ def _row_scales(st, name, rows):return scales-def _dequantize_vocabulary(st, name, shape, dtype, chunk_rows):+def _dequantize_vocabulary(st, name, shape, dtype, chunk_rows, reads=None):weight = st.get_slice(name + ".weight")if list(weight.get_shape()) != list(shape) or weight.get_dtype() != "I8" or shape[1] % 256:raise ValueError(f"Expected group-256 int8 vocabulary for {name}")scales = _row_scales(st, name, shape[0])+ if reads is not None:+ reads.tensors(name + ".weight_scale")# Exactly int8_ops.hadamard16: the regular symmetric, orthogonal H256,# rather than the Sylvester Hadamard convention used by some quantizers.h4 = torch.tensor([[1, 1, 1, -1], [1, 1, -1, 1], [1, -1, 1, 1], [-1, 1, 1, 1]], dtype=torch.float32)@@ -55,6 +58,8 @@ def _dequantize_vocabulary(st, name, shape, dtype, chunk_rows):end = min(start + chunk_rows, shape[0])rotated = (weight[start:end].float() * scales[start:end, None]).reshape(-1, shape[1] // 256, 16, 16)full[start:end] = (h16 @ rotated @ h16).reshape(-1, shape[1]).to(dtype)+ if reads is not None:+ reads.advance((end - start) * shape[1])return full@@ -92,7 +97,7 @@ def _assign_tensor(model, name, value, dtype):module._buffers[leaf] = value-def _install_text_weights(model, st, *, dtype, chunk_rows):+def _install_text_weights(model, st, *, dtype, chunk_rows, reads=None):keys = set(st.keys())handled = set()for key in sorted(k for k in keys if k.endswith(".comfy_quant")):@@ -100,6 +105,8 @@ def _install_text_weights(model, st, *, dtype, chunk_rows):if name.startswith("model.visual.") or name == "lm_head":continue_quant_metadata(st, name)+ if reads is not None:+ reads.tensors(key)full_name = _text_name(model, name)module = model.get_submodule(full_name)shape = module.weight.shape if isinstance(module, (torch.nn.Linear, torch.nn.Embedding)) else ()@@ -107,7 +114,7 @@ def _install_text_weights(model, st, *, dtype, chunk_rows):raise ValueError(f"Quantized target is not Linear/Embedding: {name}")if isinstance(module, torch.nn.Embedding):module.weight = torch.nn.Parameter(- _dequantize_vocabulary(st, name, shape, dtype, chunk_rows), requires_grad=False+ _dequantize_vocabulary(st, name, shape, dtype, chunk_rows, reads), requires_grad=False)else:from ..int8_ops import Int8Linear@@ -126,6 +133,8 @@ def _install_text_weights(model, st, *, dtype, chunk_rows):replacement.input_dtype = dtypeparent, _, child = full_name.rpartition(".")setattr(model.get_submodule(parent), child, replacement)+ if reads is not None:+ reads.tensors(name + ".weight", name + ".weight_scale")handled.update({key, name + ".weight", name + ".weight_scale"})for key in sorted(keys - handled):@@ -141,13 +150,15 @@ def _install_text_weights(model, st, *, dtype, chunk_rows):if leaf == "bias" and hasattr(module, "weight_int8"):full_name += ".original"_assign_tensor(model, full_name + "." + leaf, st.get_tensor(key), dtype)+ if reads is not None:+ reads.tensors(key)missing = [name for name, value in model.state_dict().items() if value.is_meta]if missing:raise ValueError("Missing text encoder tensors: " + ", ".join(missing))def load_text_encoder(path, *, config=None, config_path=None, device="cpu", dtype=torch.bfloat16,- chunk_rows=8192):+ chunk_rows=8192, progress=None):"""Return a Qwen3VLForConditionalGeneration with int8 language projections.``config`` accepts a transformers Qwen3VLConfig or its dictionary. Otherwise@@ -156,6 +167,8 @@ def load_text_encoder(path, *, config=None, config_path=None, device="cpu", dtyp``load_vision(device='cpu')`` method installs only BF16 visual tensors andreturns that module, allowing the caller's placement policy to own moves.CPU supports loading/inspection; Int8Linear execution requires CUDA.+ ``progress(done_bytes, total_bytes)`` counts language source bytes, including+ vocabulary chunks, and excludes the deferred vision tensors and logits head."""from accelerate import init_empty_weightsfrom transformers import Qwen3VLConfig, Qwen3VLForConditionalGeneration@@ -181,7 +194,9 @@ def load_text_encoder(path, *, config=None, config_path=None, device="cpu", dtypmodel.model.visual = Nonemodel.lm_head = torch.nn.Identity()with safe_open(str(path), framework="pt", device="cpu") as st:- _install_text_weights(model, st, dtype=dtype, chunk_rows=chunk_rows)+ reads = (TensorProgress(st, progress, (key for key in st.keys()+ if not key.startswith(("model.visual.", "lm_head.")))) if progress is not None else None)+ _install_text_weights(model, st, dtype=dtype, chunk_rows=chunk_rows, reads=reads)model.eval().requires_grad_(False).to(device=device)model_ref = weakref.ref(model)diff --git a/freevideo_engine/image_gen/qwen_weights.py b/freevideo_engine/image_gen/qwen_weights.pyindex 36ce06b..662d12c 100644--- a/freevideo_engine/image_gen/qwen_weights.py+++ b/freevideo_engine/image_gen/qwen_weights.py@@ -13,6 +13,7 @@ from torch import nnfrom ..streamed_weights import _open_safetensorsfrom .qwen_dit import QwenImageDiTfrom .qwen_vae import QwenImageVAE, comfy_vae_key_to_diffusers, vae_key_map+from .load_progress import TensorProgressCONVROT_METADATA = {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 256}@@ -91,14 +92,16 @@ def _quant_targets(model, name):return targets, modules-def load_dit(path, device="cpu", *, config=None):+def load_dit(path, device="cpu", *, config=None, progress=None):"""Load a Comfy Turbo int8_convrot DiT; CPU supports storage, CUDA execution.The optional config override is for small models/tests. Production fileswithout JSON config metadata use the official 32-block Qwen-Image 2.1 config.An unknown marker, tensor layout, missing key, or extra key is an error.+ ``progress(done_bytes, total_bytes)`` reports completed source tensor batches."""with _open_safetensors(path, framework="pt", device="cpu") as checkpoint:+ reads = TensorProgress(checkpoint, progress) if progress is not None else Nonewith torch.device("meta"):model = QwenImageDiT(**_config(checkpoint.metadata() or {}, QwenImageDiT, config))keys = _keys(checkpoint, "model.diffusion_model.")@@ -120,6 +123,8 @@ def load_dit(path, device="cpu", *, config=None):raise ValueError(f"Invalid Comfy quantization JSON: {marker_key}") from excif marker != CONVROT_METADATA or type(marker.get("convrot")) is not bool or type(marker.get("convrot_groupsize")) is not int:raise ValueError(f"Unsupported Comfy quantization format for {name}: {marker!r}")+ if reads is not None:+ reads.tensors(marker_key)targets, modules = _quant_targets(model, name)rows = sum(module.out_features for module in modules)width = modules[0].in_features@@ -144,6 +149,8 @@ def load_dit(path, device="cpu", *, config=None):raise ValueError(f"Invalid Comfy attention hint: {key}") from excif hint != COMFY_ATTENTION_HINT:raise ValueError(f"Unsupported Comfy attention format for {key}: {hint!r}")+ if reads is not None:+ reads.tensors(keys[key])consumed.add(key)ordinary = set(expected) - replacedmissing, extra = ordinary - set(keys), set(keys) - consumed - ordinary@@ -174,23 +181,29 @@ def load_dit(path, device="cpu", *, config=None):_assign(model, target + ".weight_scale", scales[offset:offset + rows], device, torch.float32)offset += rowsdel weight, scales+ if reads is not None:+ reads.tensors(keys[name + ".weight"], keys[name + ".weight_scale"])for key in sorted(ordinary):# A quantized Linear's original bias (if a future supported config has# one) lives under original.bias in the engine wrapper.parent, _, leaf = key.rpartition(".")target = key if parent not in specs or leaf != "bias" else parent + ".original.bias"_assign(model, target, checkpoint.get_tensor(keys[key]), device, torch.bfloat16)+ if reads is not None:+ reads.tensors(keys[key])model.to(device=device)return model.eval().requires_grad_(False)-def load_vae(path, device="cpu", *, config=None):+def load_vae(path, device="cpu", *, config=None, progress=None):"""Load the Comfy BF16 VAE, accepting only explicit one-to-one key mappings.Single-frame Comfy Conv3d weights may retain a size-one temporal axis. It issqueezed without numerical conversion; any other shape is rejected.+ ``progress(done_bytes, total_bytes)`` counts source bytes before squeezing."""with _open_safetensors(path, framework="pt", device="cpu") as checkpoint:+ reads = TensorProgress(checkpoint, progress) if progress is not None else Nonewith torch.device("meta"):model = QwenImageVAE(**_config(checkpoint.metadata() or {}, QwenImageVAE, config))table = vae_key_map(model)@@ -220,4 +233,6 @@ def load_vae(path, device="cpu", *, config=None):tensor = tensor.squeeze(2)_assign(model, target, tensor, device, torch.bfloat16)del tensor+ if reads is not None:+ reads.tensors(key)return model.eval().requires_grad_(False)diff --git a/freevideo_engine/image_gen/service.py b/freevideo_engine/image_gen/service.pyindex c3727d7..9dae7ed 100644--- a/freevideo_engine/image_gen/service.py+++ b/freevideo_engine/image_gen/service.py@@ -389,6 +389,8 @@ class Service:if (stage in ('load', 'text', 'encode', 'denoise', 'decode') and type(fraction) in (int, float)and math.isfinite(fraction) and 0 <= fraction <= 1):with self.mutex:+ if self.state.get('stage') == stage and type(self.state.get('fraction')) in (int, float):+ fraction = max(fraction, self.state['fraction'])self.state.update(phase='cancelling' if self.stop.is_set() else 'progress',stage=stage, fraction=fraction)elif phase in ('result', 'error'):diff --git a/freevideo_engine/offload.py b/freevideo_engine/offload.pyindex d7d4e3f..19e4353 100644--- a/freevideo_engine/offload.py+++ b/freevideo_engine/offload.py@@ -99,7 +99,7 @@ def live_pin_capacity(*, headroom_bytes=None, nonlocal_reserve_bytes=None, commi@torch.no_grad()def pin_layer_weights(layers, max_bytes=None, *, headroom_bytes=None, nonlocal_reserve_bytes=None,- commit_reserve_bytes=0):+ commit_reserve_bytes=0, progress=None):"""Pack immutable CPU parameters into persistent pinned planes, one per dtype.This optional mode trades pageable/mapped storage for locked host residency.@@ -174,6 +174,8 @@ def pin_layer_weights(layers, max_bytes=None, *, headroom_bytes=None, nonlocal_roffset += parameter.numel()parameter.data = viewpinned += sum(p.numel() * p.element_size() for p in parameters)+ if progress is not None:+ progress(pinned, required)return pinned