Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Image generation: a card that holds the whole model keeps no host copy of the weights

Bowen Xue196e700a2cb1
diff --git a/freevideo_engine/image_gen/image_runner.py b/freevideo_engine/image_gen/image_runner.py
index d288cf2..2c66df2 100644
--- a/freevideo_engine/image_gen/image_runner.py
+++ b/freevideo_engine/image_gen/image_runner.py
@@ -14,7 +14,7 @@ from pathlib import Path
import torch
from . import image_policy
-from .image_stream import StreamedStack
+from .image_stream import ResidentStack, StreamedStack
GiB = 2**30
_DIT_FILE = 'diffusion_models/qwen_image_2.1_turbo_int8_convrot.safetensors'
@@ -111,8 +111,13 @@ class ImageRunner:
``gpu_budget_bytes`` caps what a request may plan against (None reads the
card: free memory plus what this process already holds). Residency and VAE
tiling are chosen per request by ``image_policy.plan_image``.
+
+ ``full_card`` is set when the whole model sits on the card with no host copy; the first request
+ that no longer plans everything resident puts the weights back on the host and streams them.
"""
+ full_card = False
+
def __init__(self, pack_dir, *, device='cuda', gpu_budget_bytes=None, log=None, progress=None):
from .qwen_pipeline import QwenImageGenerator
from .qwen_te_weights import load_text_encoder
@@ -130,7 +135,9 @@ class ImageRunner:
loading = _LoadProgress(progress or (lambda stage, fraction: None))
started = time.monotonic()
- self.transformer = load_dit(self.pack_dir / _DIT_FILE, device='cpu',
+ # A card that holds every weight plus a request's working memory keeps no host copy.
+ self.full_card = self._fits_whole_model()
+ self.transformer = load_dit(self.pack_dir / _DIT_FILE, device=self.device if self.full_card else 'cpu',
progress=lambda done, total: loading.report('dit', done, total))
self.load_seconds['dit'] = time.monotonic() - started
loading.complete('dit')
@@ -167,7 +174,8 @@ class ImageRunner:
from ..offload import cpu_weights
stacks = [self.transformer.transformer_blocks, language.layers]
stack_bytes = [sum(value.numel() * value.element_size() for layer in layers
- for _, value in cpu_weights(layer)) for layers in stacks]
+ for _, value in (list(layer.named_parameters()) + list(layer.named_buffers())
+ if self.full_card else cpu_weights(layer))) for layers in stacks]
pin_total = sum(stack_bytes)
def pin_progress(index):
@@ -175,16 +183,25 @@ class ImageRunner:
return lambda done, total: loading.report(
'pin', offset + stack_bytes[index] * (done / total if total else 0.), pin_total)
- with torch.no_grad(): # LayerOffloader refuses to be built with autograd on
- self.dit_stack = StreamedStack(stacks[0], name='dit', device=self.device, progress=pin_progress(0))
- loading.report('pin', stack_bytes[0], pin_total)
- self.te_stack = StreamedStack(stacks[1], name='text_encoder', device=self.device, progress=pin_progress(1))
+ if self.full_card:
+ # Move the encoder layers over one at a time so the host copy is freed as it goes.
+ for layer in stacks[1]:
+ layer.to(self.device)
+ self.dit_stack = ResidentStack(stacks[0], name='dit', device=self.device)
+ self.te_stack = ResidentStack(stacks[1], name='text_encoder', device=self.device)
+ else:
+ with torch.no_grad(): # LayerOffloader refuses to be built with autograd on
+ self.dit_stack = StreamedStack(stacks[0], name='dit', device=self.device,
+ progress=pin_progress(0))
+ loading.report('pin', stack_bytes[0], pin_total)
+ self.te_stack = StreamedStack(stacks[1], name='text_encoder', device=self.device,
+ progress=pin_progress(1))
self.load_seconds['pin'] = time.monotonic() - started
loading.complete('pin')
self.shape = image_policy.ImageShape.measure(
list(self.transformer.transformer_blocks), list(language.layers),
- total_weight_bytes=self.dit_stack.host_bytes + self.te_stack.host_bytes)
+ total_weight_bytes=sum(stack_bytes))
self.generator = QwenImageGenerator(
None, placement=_Placement(self), device=self.device, transformer=self.transformer, vae=self.vae,
text_encoder=text, processor=processor, sample_sigmas=None, vae_tile=True)
@@ -204,6 +221,24 @@ class ImageRunner:
return [embedding.register_forward_pre_hook(to_host), embedding.register_forward_hook(to_card)]
+ def _fits_whole_model(self):
+ """True when a 1024x1024 request plans every DiT block and encoder layer on the card."""
+ try:
+ plan = image_policy.plan_image(gpu_budget_bytes=self.budget(), width=1024, height=1024)
+ except Exception:
+ return False
+ shape = image_policy.ImageShape()
+ return plan.feasible and plan.dit_resident_layers == shape.dit_blocks == plan.dit_kept_layers \
+ and plan.te_resident_layers == shape.te_layers
+
+ def _to_streamed(self):
+ """A request no longer fits the whole model on the card: put the host copy back and stream."""
+ with torch.no_grad():
+ self.dit_stack = self.dit_stack.to_streamed()
+ self.te_stack = self.te_stack.to_streamed()
+ self.full_card = False
+ torch.cuda.empty_cache()
+
def budget(self):
if self.gpu_budget_bytes is not None:
return int(self.gpu_budget_bytes)
@@ -228,6 +263,8 @@ class ImageRunner:
memory = system_memory()
free = memory.get('physical_available_bytes') or memory['available_bytes']
host_available = free + self.dit_stack.host_bytes + self.te_stack.host_bytes
+ if self.full_card:
+ host_available = None
except Exception:
host_available = None
return image_policy.plan_image(
@@ -237,6 +274,9 @@ class ImageRunner:
def generate(self, prompt, *, width, height, seed=None, reference_images=None, edit_mask=None,
progress=None, cancel=None):
plan = self.plan(width, height, reference_images)
+ if self.full_card and not self._all_on_card(plan):
+ self._to_streamed()
+ plan = self.plan(width, height, reference_images)
self.last_plan = plan
if not plan.feasible:
raise MemoryError('; '.join(plan.reasons) or 'the image model does not fit this machine')
@@ -248,6 +288,10 @@ class ImageRunner:
reference_images=reference_images, edit_mask=edit_mask,
progress=progress, cancel=cancel)
+ def _all_on_card(self, plan):
+ return (plan.feasible and _kept(plan) == len(self.dit_stack.layers)
+ and plan.te_resident_layers == len(self.te_stack.layers))
+
def at_boundary(self):
"""False when a failure left a weights streamer inside a layer pass; the next request cannot use it."""
return self.dit_stack.offloader.at_boundary and self.te_stack.offloader.at_boundary
diff --git a/freevideo_engine/image_gen/image_stream.py b/freevideo_engine/image_gen/image_stream.py
index ee8db9b..b08b6c8 100644
--- a/freevideo_engine/image_gen/image_stream.py
+++ b/freevideo_engine/image_gen/image_stream.py
@@ -61,3 +61,47 @@ class StreamedStack:
def close(self):
self.offloader.close()
+
+
+class ResidentStack:
+ """Every layer lives on the card and nothing is kept on the host.
+
+ Stands in for ``StreamedStack`` while the planned residency is the whole
+ stack, so the host holds no second copy of the weights. ``to_streamed``
+ brings the layers back for a request whose budget no longer fits them all.
+ """
+
+ def __init__(self, layers, *, name, device='cuda'):
+ self.name = name
+ self.layers = list(layers)
+ if not self.layers:
+ raise ValueError('ResidentStack needs at least one layer')
+ self.device = device
+ self.host_bytes = 0
+ self.layer_bytes = max(sum(tensor.numel() * tensor.element_size()
+ for tensor in list(layer.parameters()) + list(layer.buffers()))
+ for layer in self.layers)
+ self.resident = len(self.layers)
+
+ at_boundary = True
+ cpu = () # no host weights
+
+ @property
+ def offloader(self):
+ return self
+
+ def set_resident(self, count):
+ return self.resident
+
+ def stats(self):
+ return dict(name=self.name, layers=len(self.layers), layer_bytes=self.layer_bytes, pinned_layers=0,
+ pinned_bytes=0, resident_layers=self.resident, cached_bytes=0, host_copy=False)
+
+ def to_streamed(self, **options):
+ """Copy the layers back to the host and return a ``StreamedStack`` over them."""
+ for layer in self.layers:
+ layer.to('cpu')
+ return StreamedStack(self.layers, name=self.name, device=self.device, **options)
+
+ def close(self):
+ pass