Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
Image generation: a card that holds the whole model keeps no host copy of the weights
Bowen Xue196e700a2cb1
diff --git a/freevideo_engine/image_gen/image_runner.py b/freevideo_engine/image_gen/image_runner.pyindex d288cf2..2c66df2 100644--- a/freevideo_engine/image_gen/image_runner.py+++ b/freevideo_engine/image_gen/image_runner.py@@ -14,7 +14,7 @@ from pathlib import Pathimport torchfrom . import image_policy-from .image_stream import StreamedStack+from .image_stream import ResidentStack, StreamedStackGiB = 2**30_DIT_FILE = 'diffusion_models/qwen_image_2.1_turbo_int8_convrot.safetensors'@@ -111,8 +111,13 @@ class ImageRunner:``gpu_budget_bytes`` caps what a request may plan against (None reads thecard: free memory plus what this process already holds). Residency and VAEtiling are chosen per request by ``image_policy.plan_image``.++ ``full_card`` is set when the whole model sits on the card with no host copy; the first request+ that no longer plans everything resident puts the weights back on the host and streams them."""+ full_card = False+def __init__(self, pack_dir, *, device='cuda', gpu_budget_bytes=None, log=None, progress=None):from .qwen_pipeline import QwenImageGeneratorfrom .qwen_te_weights import load_text_encoder@@ -130,7 +135,9 @@ class ImageRunner:loading = _LoadProgress(progress or (lambda stage, fraction: None))started = time.monotonic()- self.transformer = load_dit(self.pack_dir / _DIT_FILE, device='cpu',+ # A card that holds every weight plus a request's working memory keeps no host copy.+ self.full_card = self._fits_whole_model()+ self.transformer = load_dit(self.pack_dir / _DIT_FILE, device=self.device if self.full_card else 'cpu',progress=lambda done, total: loading.report('dit', done, total))self.load_seconds['dit'] = time.monotonic() - startedloading.complete('dit')@@ -167,7 +174,8 @@ class ImageRunner:from ..offload import cpu_weightsstacks = [self.transformer.transformer_blocks, language.layers]stack_bytes = [sum(value.numel() * value.element_size() for layer in layers- for _, value in cpu_weights(layer)) for layers in stacks]+ for _, value in (list(layer.named_parameters()) + list(layer.named_buffers())+ if self.full_card else cpu_weights(layer))) for layers in stacks]pin_total = sum(stack_bytes)def pin_progress(index):@@ -175,16 +183,25 @@ class ImageRunner:return lambda done, total: loading.report('pin', offset + stack_bytes[index] * (done / total if total else 0.), pin_total)- with torch.no_grad(): # LayerOffloader refuses to be built with autograd on- self.dit_stack = StreamedStack(stacks[0], name='dit', device=self.device, progress=pin_progress(0))- loading.report('pin', stack_bytes[0], pin_total)- self.te_stack = StreamedStack(stacks[1], name='text_encoder', device=self.device, progress=pin_progress(1))+ if self.full_card:+ # Move the encoder layers over one at a time so the host copy is freed as it goes.+ for layer in stacks[1]:+ layer.to(self.device)+ self.dit_stack = ResidentStack(stacks[0], name='dit', device=self.device)+ self.te_stack = ResidentStack(stacks[1], name='text_encoder', device=self.device)+ else:+ with torch.no_grad(): # LayerOffloader refuses to be built with autograd on+ self.dit_stack = StreamedStack(stacks[0], name='dit', device=self.device,+ progress=pin_progress(0))+ loading.report('pin', stack_bytes[0], pin_total)+ self.te_stack = StreamedStack(stacks[1], name='text_encoder', device=self.device,+ progress=pin_progress(1))self.load_seconds['pin'] = time.monotonic() - startedloading.complete('pin')self.shape = image_policy.ImageShape.measure(list(self.transformer.transformer_blocks), list(language.layers),- total_weight_bytes=self.dit_stack.host_bytes + self.te_stack.host_bytes)+ total_weight_bytes=sum(stack_bytes))self.generator = QwenImageGenerator(None, placement=_Placement(self), device=self.device, transformer=self.transformer, vae=self.vae,text_encoder=text, processor=processor, sample_sigmas=None, vae_tile=True)@@ -204,6 +221,24 @@ class ImageRunner:return [embedding.register_forward_pre_hook(to_host), embedding.register_forward_hook(to_card)]+ def _fits_whole_model(self):+ """True when a 1024x1024 request plans every DiT block and encoder layer on the card."""+ try:+ plan = image_policy.plan_image(gpu_budget_bytes=self.budget(), width=1024, height=1024)+ except Exception:+ return False+ shape = image_policy.ImageShape()+ return plan.feasible and plan.dit_resident_layers == shape.dit_blocks == plan.dit_kept_layers \+ and plan.te_resident_layers == shape.te_layers++ def _to_streamed(self):+ """A request no longer fits the whole model on the card: put the host copy back and stream."""+ with torch.no_grad():+ self.dit_stack = self.dit_stack.to_streamed()+ self.te_stack = self.te_stack.to_streamed()+ self.full_card = False+ torch.cuda.empty_cache()+def budget(self):if self.gpu_budget_bytes is not None:return int(self.gpu_budget_bytes)@@ -228,6 +263,8 @@ class ImageRunner:memory = system_memory()free = memory.get('physical_available_bytes') or memory['available_bytes']host_available = free + self.dit_stack.host_bytes + self.te_stack.host_bytes+ if self.full_card:+ host_available = Noneexcept Exception:host_available = Nonereturn image_policy.plan_image(@@ -237,6 +274,9 @@ class ImageRunner:def generate(self, prompt, *, width, height, seed=None, reference_images=None, edit_mask=None,progress=None, cancel=None):plan = self.plan(width, height, reference_images)+ if self.full_card and not self._all_on_card(plan):+ self._to_streamed()+ plan = self.plan(width, height, reference_images)self.last_plan = planif not plan.feasible:raise MemoryError('; '.join(plan.reasons) or 'the image model does not fit this machine')@@ -248,6 +288,10 @@ class ImageRunner:reference_images=reference_images, edit_mask=edit_mask,progress=progress, cancel=cancel)+ def _all_on_card(self, plan):+ return (plan.feasible and _kept(plan) == len(self.dit_stack.layers)+ and plan.te_resident_layers == len(self.te_stack.layers))+def at_boundary(self):"""False when a failure left a weights streamer inside a layer pass; the next request cannot use it."""return self.dit_stack.offloader.at_boundary and self.te_stack.offloader.at_boundarydiff --git a/freevideo_engine/image_gen/image_stream.py b/freevideo_engine/image_gen/image_stream.pyindex ee8db9b..b08b6c8 100644--- a/freevideo_engine/image_gen/image_stream.py+++ b/freevideo_engine/image_gen/image_stream.py@@ -61,3 +61,47 @@ class StreamedStack:def close(self):self.offloader.close()+++class ResidentStack:+ """Every layer lives on the card and nothing is kept on the host.++ Stands in for ``StreamedStack`` while the planned residency is the whole+ stack, so the host holds no second copy of the weights. ``to_streamed``+ brings the layers back for a request whose budget no longer fits them all.+ """++ def __init__(self, layers, *, name, device='cuda'):+ self.name = name+ self.layers = list(layers)+ if not self.layers:+ raise ValueError('ResidentStack needs at least one layer')+ self.device = device+ self.host_bytes = 0+ self.layer_bytes = max(sum(tensor.numel() * tensor.element_size()+ for tensor in list(layer.parameters()) + list(layer.buffers()))+ for layer in self.layers)+ self.resident = len(self.layers)++ at_boundary = True+ cpu = () # no host weights++ @property+ def offloader(self):+ return self++ def set_resident(self, count):+ return self.resident++ def stats(self):+ return dict(name=self.name, layers=len(self.layers), layer_bytes=self.layer_bytes, pinned_layers=0,+ pinned_bytes=0, resident_layers=self.resident, cached_bytes=0, host_copy=False)++ def to_streamed(self, **options):+ """Copy the layers back to the host and return a ``StreamedStack`` over them."""+ for layer in self.layers:+ layer.to('cpu')+ return StreamedStack(self.layers, name=self.name, device=self.device, **options)++ def close(self):+ pass