Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Image model loading shows its progress on the bar

Bowen Xue2fdf5d43a908
diff --git a/freevideo_engine/image_gen/image_runner.py b/freevideo_engine/image_gen/image_runner.py
index 593a0db..563f296 100644
--- a/freevideo_engine/image_gen/image_runner.py
+++ b/freevideo_engine/image_gen/image_runner.py
@@ -8,6 +8,7 @@ table is looked up on the CPU, the VAE and the vision tower visit the card
only for the stage that needs them.
"""
import time
+import math
from pathlib import Path
import torch
@@ -19,6 +20,36 @@ GiB = 2**30
_DIT_FILE = 'diffusion_models/qwen_image_2.1_turbo_int8_convrot.safetensors'
_TE_FILE = 'text_encoders/qwen3vl_8b_int8_convrot.safetensors'
_VAE_FILE = 'vae/qwen_image_2.1_vae_bf16.safetensors'
+# Rounded mean stage times from the local cold/hot constructor comparison.
+# See benchmarks/image_load_progress_045afix/summary.txt (small-model fallback).
+_LOAD_SHARES = dict(dit=.42, vae=.03, text=.18, pin=.37)
+
+
+class _LoadProgress:
+ def __init__(self, callback):
+ self.callback = callback
+ self.fraction = self.emitted = 0.
+ self.callback('load', 0.)
+ self.last_time = time.monotonic()
+
+ def report(self, stage, done, total):
+ offset = 0.
+ for name, share in _LOAD_SHARES.items():
+ if name == stage:
+ fraction = offset + share * (done / total if total else 0.)
+ break
+ offset += share
+ self.fraction = max(self.fraction, min(fraction, math.nextafter(1., 0.)))
+ now = time.monotonic()
+ if self.fraction > self.emitted and now - self.last_time >= .25:
+ self.callback('load', self.fraction)
+ self.emitted, self.last_time = self.fraction, now
+
+ def complete(self, stage):
+ self.report(stage, 1, 1)
+
+ def finish(self):
+ self.callback('load', 1.)
def _non_block_children(module, skip):
@@ -86,24 +117,27 @@ class ImageRunner:
nested = self.pack_dir / directory / name
return nested if nested.exists() else self.pack_dir / name
- progress = progress or (lambda stage, fraction: None)
- progress('load', 0.)
+ loading = _LoadProgress(progress or (lambda stage, fraction: None))
started = time.monotonic()
- self.transformer = load_dit(self.pack_dir / _DIT_FILE, device='cpu')
+ self.transformer = load_dit(self.pack_dir / _DIT_FILE, device='cpu',
+ progress=lambda done, total: loading.report('dit', done, total))
self.load_seconds['dit'] = time.monotonic() - started
- progress('load', 1 / 3)
+ loading.complete('dit')
started = time.monotonic()
- self.vae = load_vae(self.pack_dir / _VAE_FILE, device='cpu')
+ self.vae = load_vae(self.pack_dir / _VAE_FILE, device='cpu',
+ progress=lambda done, total: loading.report('vae', done, total))
self.load_seconds['vae'] = time.monotonic() - started
- progress('load', 2 / 3)
+ loading.complete('vae')
started = time.monotonic()
config_dir = self.pack_dir / 'text_encoder'
processor_dir = self.pack_dir / 'processor'
- text = load_text_encoder(self.pack_dir / _TE_FILE, config_path=config_dir / 'config.json', device='cpu')
+ text = load_text_encoder(self.pack_dir / _TE_FILE, config_path=config_dir / 'config.json', device='cpu',
+ progress=lambda done, total: loading.report('text', done, total))
from transformers import Qwen3VLProcessor
processor = Qwen3VLProcessor.from_pretrained(str(processor_dir), local_files_only=True)
self.load_seconds['text'] = time.monotonic() - started
self.text_model = text
+ loading.complete('text')
# Parts that always sit on the card: everything of the DiT except its blocks, and the
# encoder's final norm and rotary table. The 1.2 GB vocabulary stays on the host and is
@@ -119,10 +153,23 @@ class ImageRunner:
self._embed_hooks = self._lookup_on_host(language.embed_tokens)
started = time.monotonic()
+ from ..offload import cpu_weights
+ stacks = [self.transformer.transformer_blocks, language.layers]
+ stack_bytes = [sum(value.numel() * value.element_size() for layer in layers
+ for _, value in cpu_weights(layer)) for layers in stacks]
+ pin_total = sum(stack_bytes)
+
+ def pin_progress(index):
+ offset = sum(stack_bytes[:index])
+ return lambda done, total: loading.report(
+ 'pin', offset + stack_bytes[index] * (done / total if total else 0.), pin_total)
+
with torch.no_grad(): # LayerOffloader refuses to be built with autograd on
- self.dit_stack = StreamedStack(self.transformer.transformer_blocks, name='dit', device=self.device)
- self.te_stack = StreamedStack(language.layers, name='text_encoder', device=self.device)
+ self.dit_stack = StreamedStack(stacks[0], name='dit', device=self.device, progress=pin_progress(0))
+ loading.report('pin', stack_bytes[0], pin_total)
+ self.te_stack = StreamedStack(stacks[1], name='text_encoder', device=self.device, progress=pin_progress(1))
self.load_seconds['pin'] = time.monotonic() - started
+ loading.complete('pin')
self.shape = image_policy.ImageShape.measure(
list(self.transformer.transformer_blocks), list(language.layers),
@@ -133,7 +180,7 @@ class ImageRunner:
from .qwen_pipeline import load_sample_sigmas
self.generator.sample_sigmas = load_sample_sigmas(self.pack_dir / _DIT_FILE)
self.last_plan = None
- progress('load', 1.)
+ loading.finish()
def _lookup_on_host(self, embedding):
device = self.device
diff --git a/freevideo_engine/image_gen/image_stream.py b/freevideo_engine/image_gen/image_stream.py
index 7b783a5..ee8db9b 100644
--- a/freevideo_engine/image_gen/image_stream.py
+++ b/freevideo_engine/image_gen/image_stream.py
@@ -16,7 +16,8 @@ from ..offload import LayerOffloader, pin_layer_weights
class StreamedStack:
"""``layers`` hold frozen CPU weights; they are moved to the card on use."""
- def __init__(self, layers, *, name, device='cuda', pin=True, max_pin_bytes=None, prefetch=True):
+ def __init__(self, layers, *, name, device='cuda', pin=True, max_pin_bytes=None, prefetch=True,
+ progress=None):
self.name = name
self.layers = list(layers)
if not self.layers:
@@ -28,7 +29,8 @@ class StreamedStack:
# Pinning is an optimization: a host that refuses it costs transfer
# speed, not the request (pin_layer_weights keeps the unpinned layers
# on their ordinary storage and the offloader stages them).
- self.pinned_bytes = pin_layer_weights(self.layers, max_bytes=max_pin_bytes) if pin else 0
+ pin_options = {'progress': progress} if progress is not None else {}
+ self.pinned_bytes = pin_layer_weights(self.layers, max_bytes=max_pin_bytes, **pin_options) if pin else 0
self.offloader = LayerOffloader(self.layers, device=device, prefetch=prefetch, host_prefetch=False)
self.layer_bytes = max(sum(count * parameter.element_size()
for parameter, _, _, _, count, _ in entries)
diff --git a/freevideo_engine/image_gen/load_progress.py b/freevideo_engine/image_gen/load_progress.py
new file mode 100644
index 0000000..f33b8bf
--- /dev/null
+++ b/freevideo_engine/image_gen/load_progress.py
@@ -0,0 +1,27 @@
+"""Byte accounting for completed safetensor reads, without opening tensor data."""
+from math import prod
+
+
+_DTYPE_BYTES = {'BOOL': 1, 'U8': 1, 'I8': 1, 'I16': 2, 'I32': 4, 'I64': 8,
+ 'F16': 2, 'BF16': 2, 'F32': 4, 'F64': 8,
+ 'F8_E4M3': 1, 'F8_E5M2': 1}
+
+
+class TensorProgress:
+ def __init__(self, checkpoint, callback, keys=None):
+ self.callback = callback
+ self.sizes = {}
+ for key in checkpoint.keys() if keys is None else keys:
+ value = checkpoint.get_slice(key)
+ self.sizes[key] = prod(value.get_shape()) * _DTYPE_BYTES[value.get_dtype()]
+ self.total = sum(self.sizes.values())
+ self.done = self.last = 0
+
+ def tensors(self, *keys):
+ self.advance(sum(self.sizes[key] for key in keys))
+
+ def advance(self, size):
+ self.done += size
+ if self.done > self.last and ((self.done - self.last) * 50 >= self.total or self.done == self.total):
+ self.last = self.done
+ self.callback(self.done, self.total)
diff --git a/freevideo_engine/image_gen/qwen_te_weights.py b/freevideo_engine/image_gen/qwen_te_weights.py
index abfb066..86d6299 100644
--- a/freevideo_engine/image_gen/qwen_te_weights.py
+++ b/freevideo_engine/image_gen/qwen_te_weights.py
@@ -14,6 +14,7 @@ from pathlib import Path
import torch
from safetensors import safe_open
+from .load_progress import TensorProgress
_CONVROT = {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 256}
@@ -41,11 +42,13 @@ def _row_scales(st, name, rows):
return scales
-def _dequantize_vocabulary(st, name, shape, dtype, chunk_rows):
+def _dequantize_vocabulary(st, name, shape, dtype, chunk_rows, reads=None):
weight = st.get_slice(name + ".weight")
if list(weight.get_shape()) != list(shape) or weight.get_dtype() != "I8" or shape[1] % 256:
raise ValueError(f"Expected group-256 int8 vocabulary for {name}")
scales = _row_scales(st, name, shape[0])
+ if reads is not None:
+ reads.tensors(name + ".weight_scale")
# Exactly int8_ops.hadamard16: the regular symmetric, orthogonal H256,
# rather than the Sylvester Hadamard convention used by some quantizers.
h4 = torch.tensor([[1, 1, 1, -1], [1, 1, -1, 1], [1, -1, 1, 1], [-1, 1, 1, 1]], dtype=torch.float32)
@@ -55,6 +58,8 @@ def _dequantize_vocabulary(st, name, shape, dtype, chunk_rows):
end = min(start + chunk_rows, shape[0])
rotated = (weight[start:end].float() * scales[start:end, None]).reshape(-1, shape[1] // 256, 16, 16)
full[start:end] = (h16 @ rotated @ h16).reshape(-1, shape[1]).to(dtype)
+ if reads is not None:
+ reads.advance((end - start) * shape[1])
return full
@@ -92,7 +97,7 @@ def _assign_tensor(model, name, value, dtype):
module._buffers[leaf] = value
-def _install_text_weights(model, st, *, dtype, chunk_rows):
+def _install_text_weights(model, st, *, dtype, chunk_rows, reads=None):
keys = set(st.keys())
handled = set()
for key in sorted(k for k in keys if k.endswith(".comfy_quant")):
@@ -100,6 +105,8 @@ def _install_text_weights(model, st, *, dtype, chunk_rows):
if name.startswith("model.visual.") or name == "lm_head":
continue
_quant_metadata(st, name)
+ if reads is not None:
+ reads.tensors(key)
full_name = _text_name(model, name)
module = model.get_submodule(full_name)
shape = module.weight.shape if isinstance(module, (torch.nn.Linear, torch.nn.Embedding)) else ()
@@ -107,7 +114,7 @@ def _install_text_weights(model, st, *, dtype, chunk_rows):
raise ValueError(f"Quantized target is not Linear/Embedding: {name}")
if isinstance(module, torch.nn.Embedding):
module.weight = torch.nn.Parameter(
- _dequantize_vocabulary(st, name, shape, dtype, chunk_rows), requires_grad=False
+ _dequantize_vocabulary(st, name, shape, dtype, chunk_rows, reads), requires_grad=False
)
else:
from ..int8_ops import Int8Linear
@@ -126,6 +133,8 @@ def _install_text_weights(model, st, *, dtype, chunk_rows):
replacement.input_dtype = dtype
parent, _, child = full_name.rpartition(".")
setattr(model.get_submodule(parent), child, replacement)
+ if reads is not None:
+ reads.tensors(name + ".weight", name + ".weight_scale")
handled.update({key, name + ".weight", name + ".weight_scale"})
for key in sorted(keys - handled):
@@ -141,13 +150,15 @@ def _install_text_weights(model, st, *, dtype, chunk_rows):
if leaf == "bias" and hasattr(module, "weight_int8"):
full_name += ".original"
_assign_tensor(model, full_name + "." + leaf, st.get_tensor(key), dtype)
+ if reads is not None:
+ reads.tensors(key)
missing = [name for name, value in model.state_dict().items() if value.is_meta]
if missing:
raise ValueError("Missing text encoder tensors: " + ", ".join(missing))
def load_text_encoder(path, *, config=None, config_path=None, device="cpu", dtype=torch.bfloat16,
- chunk_rows=8192):
+ chunk_rows=8192, progress=None):
"""Return a Qwen3VLForConditionalGeneration with int8 language projections.
``config`` accepts a transformers Qwen3VLConfig or its dictionary. Otherwise
@@ -156,6 +167,8 @@ def load_text_encoder(path, *, config=None, config_path=None, device="cpu", dtyp
``load_vision(device='cpu')`` method installs only BF16 visual tensors and
returns that module, allowing the caller's placement policy to own moves.
CPU supports loading/inspection; Int8Linear execution requires CUDA.
+ ``progress(done_bytes, total_bytes)`` counts language source bytes, including
+ vocabulary chunks, and excludes the deferred vision tensors and logits head.
"""
from accelerate import init_empty_weights
from transformers import Qwen3VLConfig, Qwen3VLForConditionalGeneration
@@ -181,7 +194,9 @@ def load_text_encoder(path, *, config=None, config_path=None, device="cpu", dtyp
model.model.visual = None
model.lm_head = torch.nn.Identity()
with safe_open(str(path), framework="pt", device="cpu") as st:
- _install_text_weights(model, st, dtype=dtype, chunk_rows=chunk_rows)
+ reads = (TensorProgress(st, progress, (key for key in st.keys()
+ if not key.startswith(("model.visual.", "lm_head.")))) if progress is not None else None)
+ _install_text_weights(model, st, dtype=dtype, chunk_rows=chunk_rows, reads=reads)
model.eval().requires_grad_(False).to(device=device)
model_ref = weakref.ref(model)
diff --git a/freevideo_engine/image_gen/qwen_weights.py b/freevideo_engine/image_gen/qwen_weights.py
index 36ce06b..662d12c 100644
--- a/freevideo_engine/image_gen/qwen_weights.py
+++ b/freevideo_engine/image_gen/qwen_weights.py
@@ -13,6 +13,7 @@ from torch import nn
from ..streamed_weights import _open_safetensors
from .qwen_dit import QwenImageDiT
from .qwen_vae import QwenImageVAE, comfy_vae_key_to_diffusers, vae_key_map
+from .load_progress import TensorProgress
CONVROT_METADATA = {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 256}
@@ -91,14 +92,16 @@ def _quant_targets(model, name):
return targets, modules
-def load_dit(path, device="cpu", *, config=None):
+def load_dit(path, device="cpu", *, config=None, progress=None):
"""Load a Comfy Turbo int8_convrot DiT; CPU supports storage, CUDA execution.
The optional config override is for small models/tests. Production files
without JSON config metadata use the official 32-block Qwen-Image 2.1 config.
An unknown marker, tensor layout, missing key, or extra key is an error.
+ ``progress(done_bytes, total_bytes)`` reports completed source tensor batches.
"""
with _open_safetensors(path, framework="pt", device="cpu") as checkpoint:
+ reads = TensorProgress(checkpoint, progress) if progress is not None else None
with torch.device("meta"):
model = QwenImageDiT(**_config(checkpoint.metadata() or {}, QwenImageDiT, config))
keys = _keys(checkpoint, "model.diffusion_model.")
@@ -120,6 +123,8 @@ def load_dit(path, device="cpu", *, config=None):
raise ValueError(f"Invalid Comfy quantization JSON: {marker_key}") from exc
if marker != CONVROT_METADATA or type(marker.get("convrot")) is not bool or type(marker.get("convrot_groupsize")) is not int:
raise ValueError(f"Unsupported Comfy quantization format for {name}: {marker!r}")
+ if reads is not None:
+ reads.tensors(marker_key)
targets, modules = _quant_targets(model, name)
rows = sum(module.out_features for module in modules)
width = modules[0].in_features
@@ -144,6 +149,8 @@ def load_dit(path, device="cpu", *, config=None):
raise ValueError(f"Invalid Comfy attention hint: {key}") from exc
if hint != COMFY_ATTENTION_HINT:
raise ValueError(f"Unsupported Comfy attention format for {key}: {hint!r}")
+ if reads is not None:
+ reads.tensors(keys[key])
consumed.add(key)
ordinary = set(expected) - replaced
missing, extra = ordinary - set(keys), set(keys) - consumed - ordinary
@@ -174,23 +181,29 @@ def load_dit(path, device="cpu", *, config=None):
_assign(model, target + ".weight_scale", scales[offset:offset + rows], device, torch.float32)
offset += rows
del weight, scales
+ if reads is not None:
+ reads.tensors(keys[name + ".weight"], keys[name + ".weight_scale"])
for key in sorted(ordinary):
# A quantized Linear's original bias (if a future supported config has
# one) lives under original.bias in the engine wrapper.
parent, _, leaf = key.rpartition(".")
target = key if parent not in specs or leaf != "bias" else parent + ".original.bias"
_assign(model, target, checkpoint.get_tensor(keys[key]), device, torch.bfloat16)
+ if reads is not None:
+ reads.tensors(keys[key])
model.to(device=device)
return model.eval().requires_grad_(False)
-def load_vae(path, device="cpu", *, config=None):
+def load_vae(path, device="cpu", *, config=None, progress=None):
"""Load the Comfy BF16 VAE, accepting only explicit one-to-one key mappings.
Single-frame Comfy Conv3d weights may retain a size-one temporal axis. It is
squeezed without numerical conversion; any other shape is rejected.
+ ``progress(done_bytes, total_bytes)`` counts source bytes before squeezing.
"""
with _open_safetensors(path, framework="pt", device="cpu") as checkpoint:
+ reads = TensorProgress(checkpoint, progress) if progress is not None else None
with torch.device("meta"):
model = QwenImageVAE(**_config(checkpoint.metadata() or {}, QwenImageVAE, config))
table = vae_key_map(model)
@@ -220,4 +233,6 @@ def load_vae(path, device="cpu", *, config=None):
tensor = tensor.squeeze(2)
_assign(model, target, tensor, device, torch.bfloat16)
del tensor
+ if reads is not None:
+ reads.tensors(key)
return model.eval().requires_grad_(False)
diff --git a/freevideo_engine/image_gen/service.py b/freevideo_engine/image_gen/service.py
index c3727d7..9dae7ed 100644
--- a/freevideo_engine/image_gen/service.py
+++ b/freevideo_engine/image_gen/service.py
@@ -389,6 +389,8 @@ class Service:
if (stage in ('load', 'text', 'encode', 'denoise', 'decode') and type(fraction) in (int, float)
and math.isfinite(fraction) and 0 <= fraction <= 1):
with self.mutex:
+ if self.state.get('stage') == stage and type(self.state.get('fraction')) in (int, float):
+ fraction = max(fraction, self.state['fraction'])
self.state.update(phase='cancelling' if self.stop.is_set() else 'progress',
stage=stage, fraction=fraction)
elif phase in ('result', 'error'):
diff --git a/freevideo_engine/offload.py b/freevideo_engine/offload.py
index d7d4e3f..19e4353 100644
--- a/freevideo_engine/offload.py
+++ b/freevideo_engine/offload.py
@@ -99,7 +99,7 @@ def live_pin_capacity(*, headroom_bytes=None, nonlocal_reserve_bytes=None, commi
@torch.no_grad()
def pin_layer_weights(layers, max_bytes=None, *, headroom_bytes=None, nonlocal_reserve_bytes=None,
- commit_reserve_bytes=0):
+ commit_reserve_bytes=0, progress=None):
"""Pack immutable CPU parameters into persistent pinned planes, one per dtype.
This optional mode trades pageable/mapped storage for locked host residency.
@@ -174,6 +174,8 @@ def pin_layer_weights(layers, max_bytes=None, *, headroom_bytes=None, nonlocal_r
offset += parameter.numel()
parameter.data = view
pinned += sum(p.numel() * p.element_size() for p in parameters)
+ if progress is not None:
+ progress(pinned, required)
return pinned