Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
Prompt enhancement on 6 GB cards: keep int8 layers on the GPU and stream the rest from host memory
Bowen Xue088c00b75c02
diff --git a/freevideo_engine/prompt_vlm/int8.py b/freevideo_engine/prompt_vlm/int8.pyindex 4da67f8..0ba3205 100644--- a/freevideo_engine/prompt_vlm/int8.py+++ b/freevideo_engine/prompt_vlm/int8.py@@ -38,6 +38,16 @@ class Int8Linear(nn.Module):bias = source.biasself.register_buffer('bias', None if bias is None else bias.detach().to(device))+ def offload(self):+ """Keep the int8 rows in (pinned, when possible) host memory; a Streamer lends them to the GPU."""+ host = self.weight_int8.cpu()+ try:+ host = host.pin_memory()+ except RuntimeError:+ pass # Pageable memory copies slower but works.+ self.host = host+ self.weight_int8 = None+def forward(self, x):shape = x.shaperows = x.reshape(-1, self.in_features)@@ -121,3 +131,87 @@ def install(layer, device):source = getattr(parent, name)setattr(parent, name, Int8Linear(source, device))del source+++def layer_bytes(stored):+ """{layer index: GPU bytes of its int8 projections and their scales}."""+ import re+ projection = re.compile(r'model\.language_model\.layers\.(\d+)\.(self_attn\.[qkvo]_proj|mlp\.(gate|up|down)_proj)\.weight$')+ rows = {}+ for name, row in stored.items():+ match = projection.match(name)+ if match:+ index = int(match.group(1))+ rows[index] = rows.get(index, 0) + row['bytes'] // 2 + 4 * row['shape'][0]+ return rows+++PREFETCH = 2 # Layers copied ahead of the one running.+++def resident_layers(weights, stored, room):+ """How many leading layers stay on the GPU as int8 when the rest are streamed+ from host memory one layer at a time; None when even the fixed part and the+ staging buffers do not fit `room`. Returns (layers, host_bytes)."""+ sizes = layer_bytes(stored)+ count = len(sizes)+ if not count:+ return None+ projections = sum(row['bytes'] for name, row in stored.items() if name.startswith('model.language_model.layers.')+ and name.endswith('_proj.weight'))+ fixed = weights - projections+ largest = max(sizes.values())+ free = room - fixed - (PREFETCH + 1) * largest+ kept, used = 0, 0+ while kept < count and used + sizes[kept] <= free:+ used += sizes[kept]+ kept += 1+ if free < 0:+ return None+ return kept, sum(sizes[i] for i in range(kept, count))+++class Streamer:+ """Lends host-held int8 rows to the GPU one layer ahead of use.++ The copies run on their own stream into pinned memory; a layer's+ forward waits only for its own copy and drops the GPU rows afterwards.+ """++ def __init__(self, torch, layers):+ self.torch = torch+ self.layers = layers+ self.linears = [[getattr(parent, name) for parent, name in projections(layer)] for layer in layers]+ self.stream = torch.cuda.Stream()+ self.pending = {}+ for position, layer in enumerate(layers):+ layer.register_forward_pre_hook(lambda module, args, kwargs, p=position: self.before(p), with_kwargs=True)+ layer.register_forward_hook(lambda module, args, output, p=position: self.after(p))++ def fetch(self, position):+ if position in self.pending:+ return+ torch = self.torch+ with torch.cuda.stream(self.stream):+ tensors = [linear.host.to('cuda', non_blocking=True) for linear in self.linears[position]]+ event = torch.cuda.Event()+ event.record(self.stream)+ self.pending[position] = (event, tensors)++ def before(self, position):+ torch = self.torch+ self.fetch(position)+ event, tensors = self.pending.pop(position)+ current = torch.cuda.current_stream()+ current.wait_event(event)+ for linear, tensor in zip(self.linears[position], tensors):+ tensor.record_stream(current)+ linear.weight_int8 = tensor+ count = len(self.layers)+ for ahead in range(1, PREFETCH + 1):+ self.fetch((position + ahead) % count) # The next token starts at the first streamed layer.+ return None++ def after(self, position):+ for linear in self.linears[position]:+ linear.weight_int8 = Nonediff --git a/freevideo_engine/prompt_vlm/load.py b/freevideo_engine/prompt_vlm/load.pyindex 8d299d0..9119c96 100644--- a/freevideo_engine/prompt_vlm/load.py+++ b/freevideo_engine/prompt_vlm/load.py@@ -36,8 +36,11 @@ def stored(folder):return rows-def resident(folder, torch, device, precision):- """Returns the model on `device`, with int8 projections if precision == 'int8'."""+def resident(folder, torch, device, precision, keep=None):+ """Returns the model on `device`, with int8 projections if precision == 'int8'.++ With `keep`, layers from that index on hold their int8 rows in host memory+ and are streamed to the GPU as they run."""from accelerate import init_empty_weightsfrom torch import nnfrom transformers import AutoConfig, GenerationConfig, Qwen3VLForConditionalGeneration@@ -89,6 +92,9 @@ def resident(folder, torch, device, precision):merge(layer)if precision == 'int8':int8.install(layer, device)+ if keep is not None and layer.self_attn.layer_idx >= keep:+ for parent, name in int8.projections(layer):+ getattr(parent, name).offload()core.prepare(model, each_layer)finally:for reader in readers.values():@@ -96,5 +102,7 @@ def resident(folder, torch, device, precision):if tied:model.lm_head.weight = model.model.language_model.embed_tokens.weightmodel = model.to(device)+ if keep is not None:+ model.streamer = int8.Streamer(torch, list(model.model.language_model.layers)[keep:])torch.cuda.empty_cache() # Merge and quantize temporaries; KV space comes next.return modeldiff --git a/freevideo_engine/prompt_vlm/service.py b/freevideo_engine/prompt_vlm/service.pyindex e512ec4..1a8acbf 100644--- a/freevideo_engine/prompt_vlm/service.py+++ b/freevideo_engine/prompt_vlm/service.py@@ -229,11 +229,12 @@ class Service:"""Available RAM the model usually needs on this computer, for the download card.The worker decides from free VRAM when it loads: 8 GB and larger NVIDIA- cards hold the whole model. Asks the driver once, without CUDA; anything+ cards keep their layers on the GPU; 6 GB cards stream from RAM the layers+ that do not fit, which needs more of it. Asks the driver once, without CUDA; anythingunknown states the larger figure."""if self.ram is None:- from .worker import MAPPED_RAM, RESIDENT_RAM+ from .worker import MAPPED_RAM, RESIDENT_RAM, STREAMED_RAMself.ram = MAPPED_RAMif machine.get('device_backend', 'cuda') == 'cuda':from .. import processes@@ -246,6 +247,8 @@ class Service:total = totals.get(machine.get('gpu_uuid')) or (len(totals) == 1 and next(iter(totals.values())))if total and total >= 7.5 * 1024:self.ram = RESIDENT_RAM+ elif total and total >= 5.5 * 1024:+ self.ram = STREAMED_RAMexcept (OSError, RuntimeError, ValueError, subprocess.SubprocessError):passreturn self.ramdiff --git a/freevideo_engine/prompt_vlm/worker.py b/freevideo_engine/prompt_vlm/worker.pyindex 3459536..3a2b816 100644--- a/freevideo_engine/prompt_vlm/worker.py+++ b/freevideo_engine/prompt_vlm/worker.py@@ -150,7 +150,7 @@ def failure(error, mode='rewrite'):Tokenizers may quote only a fragment of either draft: replacing the fullprompt in an exception is not sufficient to make that exception private."""- allowed = {'ram_space', 'image_size', 'context_length', 'invalid_output'}+ allowed = {'ram_space', 'image_size', 'context_length', 'invalid_output', 'out_of_memory'}message = str(error)fallback = 'ask_failed' if mode == 'ask' else 'rewrite_failed'code = message if message in allowed else 'out_of_memory' if 'out of memory' in message.lower() else fallback@@ -205,6 +205,7 @@ def tokenize(processor, messages, max_pixels=512 * 32 * 32):# CUDA context alone take about 2.5 GB of commit). Other placements map and# merge the whole model in RAM; theirs is checked after the imports.RESIDENT_RAM = 4 * 2**30+STREAMED_RAM = 6 * 2**30 # download-card figure for 6 GB cards: resident part plus the streamed layers' host copyMAPPED_RAM = 11 * 2**30@@ -263,6 +264,8 @@ def load_model(folder, torch, stats, checkpoint):shapes = stored(folder)weights = sum(row['bytes'] for row in shapes.values())resident = None+ keep = None+ host_bytes = 0weight_budget = 0if torch.cuda.is_available():from .. import gpu_budget@@ -277,7 +280,19 @@ def load_model(folder, torch, stats, checkpoint):kv = kv_bytes(shapes, reserved_tokens(stats))stats.update(gpu_allocator_budget_bytes=free, gpu_weight_budget_bytes=weight_budget, gpu_kv_reserve_bytes=kv)resident = placement(weights, int8.saved_bytes(shapes), kv, free)- stats['ram_required_bytes'] = RESIDENT_RAM if resident else MAPPED_RAM+ if not resident:+ # Too big even as int8: keep what fits on the GPU and stream the other+ # layers' int8 rows from host memory, a layer ahead of use.+ plan = int8.resident_layers(weights, shapes, free - kv - WORKING_VRAM)+ if plan:+ resident, (keep, host_bytes) = 'int8', plan+ stats.update(streamed_layers=len(int8.layer_bytes(shapes)) - keep if keep is not None else 0,+ streamed_host_bytes=host_bytes)+ if not resident and weight_budget <= 2 * 2**30:+ # Not even the streaming tier fits: the model would run on the CPU at+ # about 0.2 tokens/s and hit the request deadline, so say so at once.+ raise ValueError('out_of_memory')+ stats['ram_required_bytes'] = (RESIDENT_RAM + host_bytes) if resident else MAPPED_RAMavailable = stats['available_memory_at_start_bytes' if resident else 'available_memory_before_load_bytes']if available < stats['ram_required_bytes']:raise ValueError('ram_space')@@ -285,7 +300,7 @@ def load_model(folder, torch, stats, checkpoint):loading = time.monotonic()if resident:from .load import resident as load_resident- model = load_resident(folder, torch, 'cuda:0', resident)+ model = load_resident(folder, torch, 'cuda:0', resident, keep)mapping, device = {'': 0}, 'cuda:0'stats.update(weights=resident, kv_positions=kv_positions(reserved_tokens(stats)))else: