Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Prompt enhancement on 6 GB cards: keep int8 layers on the GPU and stream the rest from host memory

Bowen Xue088c00b75c02
diff --git a/freevideo_engine/prompt_vlm/int8.py b/freevideo_engine/prompt_vlm/int8.py
index 4da67f8..0ba3205 100644
--- a/freevideo_engine/prompt_vlm/int8.py
+++ b/freevideo_engine/prompt_vlm/int8.py
@@ -38,6 +38,16 @@ class Int8Linear(nn.Module):
bias = source.bias
self.register_buffer('bias', None if bias is None else bias.detach().to(device))
+ def offload(self):
+ """Keep the int8 rows in (pinned, when possible) host memory; a Streamer lends them to the GPU."""
+ host = self.weight_int8.cpu()
+ try:
+ host = host.pin_memory()
+ except RuntimeError:
+ pass # Pageable memory copies slower but works.
+ self.host = host
+ self.weight_int8 = None
+
def forward(self, x):
shape = x.shape
rows = x.reshape(-1, self.in_features)
@@ -121,3 +131,87 @@ def install(layer, device):
source = getattr(parent, name)
setattr(parent, name, Int8Linear(source, device))
del source
+
+
+def layer_bytes(stored):
+ """{layer index: GPU bytes of its int8 projections and their scales}."""
+ import re
+ projection = re.compile(r'model\.language_model\.layers\.(\d+)\.(self_attn\.[qkvo]_proj|mlp\.(gate|up|down)_proj)\.weight$')
+ rows = {}
+ for name, row in stored.items():
+ match = projection.match(name)
+ if match:
+ index = int(match.group(1))
+ rows[index] = rows.get(index, 0) + row['bytes'] // 2 + 4 * row['shape'][0]
+ return rows
+
+
+PREFETCH = 2 # Layers copied ahead of the one running.
+
+
+def resident_layers(weights, stored, room):
+ """How many leading layers stay on the GPU as int8 when the rest are streamed
+ from host memory one layer at a time; None when even the fixed part and the
+ staging buffers do not fit `room`. Returns (layers, host_bytes)."""
+ sizes = layer_bytes(stored)
+ count = len(sizes)
+ if not count:
+ return None
+ projections = sum(row['bytes'] for name, row in stored.items() if name.startswith('model.language_model.layers.')
+ and name.endswith('_proj.weight'))
+ fixed = weights - projections
+ largest = max(sizes.values())
+ free = room - fixed - (PREFETCH + 1) * largest
+ kept, used = 0, 0
+ while kept < count and used + sizes[kept] <= free:
+ used += sizes[kept]
+ kept += 1
+ if free < 0:
+ return None
+ return kept, sum(sizes[i] for i in range(kept, count))
+
+
+class Streamer:
+ """Lends host-held int8 rows to the GPU one layer ahead of use.
+
+ The copies run on their own stream into pinned memory; a layer's
+ forward waits only for its own copy and drops the GPU rows afterwards.
+ """
+
+ def __init__(self, torch, layers):
+ self.torch = torch
+ self.layers = layers
+ self.linears = [[getattr(parent, name) for parent, name in projections(layer)] for layer in layers]
+ self.stream = torch.cuda.Stream()
+ self.pending = {}
+ for position, layer in enumerate(layers):
+ layer.register_forward_pre_hook(lambda module, args, kwargs, p=position: self.before(p), with_kwargs=True)
+ layer.register_forward_hook(lambda module, args, output, p=position: self.after(p))
+
+ def fetch(self, position):
+ if position in self.pending:
+ return
+ torch = self.torch
+ with torch.cuda.stream(self.stream):
+ tensors = [linear.host.to('cuda', non_blocking=True) for linear in self.linears[position]]
+ event = torch.cuda.Event()
+ event.record(self.stream)
+ self.pending[position] = (event, tensors)
+
+ def before(self, position):
+ torch = self.torch
+ self.fetch(position)
+ event, tensors = self.pending.pop(position)
+ current = torch.cuda.current_stream()
+ current.wait_event(event)
+ for linear, tensor in zip(self.linears[position], tensors):
+ tensor.record_stream(current)
+ linear.weight_int8 = tensor
+ count = len(self.layers)
+ for ahead in range(1, PREFETCH + 1):
+ self.fetch((position + ahead) % count) # The next token starts at the first streamed layer.
+ return None
+
+ def after(self, position):
+ for linear in self.linears[position]:
+ linear.weight_int8 = None
diff --git a/freevideo_engine/prompt_vlm/load.py b/freevideo_engine/prompt_vlm/load.py
index 8d299d0..9119c96 100644
--- a/freevideo_engine/prompt_vlm/load.py
+++ b/freevideo_engine/prompt_vlm/load.py
@@ -36,8 +36,11 @@ def stored(folder):
return rows
-def resident(folder, torch, device, precision):
- """Returns the model on `device`, with int8 projections if precision == 'int8'."""
+def resident(folder, torch, device, precision, keep=None):
+ """Returns the model on `device`, with int8 projections if precision == 'int8'.
+
+ With `keep`, layers from that index on hold their int8 rows in host memory
+ and are streamed to the GPU as they run."""
from accelerate import init_empty_weights
from torch import nn
from transformers import AutoConfig, GenerationConfig, Qwen3VLForConditionalGeneration
@@ -89,6 +92,9 @@ def resident(folder, torch, device, precision):
merge(layer)
if precision == 'int8':
int8.install(layer, device)
+ if keep is not None and layer.self_attn.layer_idx >= keep:
+ for parent, name in int8.projections(layer):
+ getattr(parent, name).offload()
core.prepare(model, each_layer)
finally:
for reader in readers.values():
@@ -96,5 +102,7 @@ def resident(folder, torch, device, precision):
if tied:
model.lm_head.weight = model.model.language_model.embed_tokens.weight
model = model.to(device)
+ if keep is not None:
+ model.streamer = int8.Streamer(torch, list(model.model.language_model.layers)[keep:])
torch.cuda.empty_cache() # Merge and quantize temporaries; KV space comes next.
return model
diff --git a/freevideo_engine/prompt_vlm/service.py b/freevideo_engine/prompt_vlm/service.py
index e512ec4..1a8acbf 100644
--- a/freevideo_engine/prompt_vlm/service.py
+++ b/freevideo_engine/prompt_vlm/service.py
@@ -229,11 +229,12 @@ class Service:
"""Available RAM the model usually needs on this computer, for the download card.
The worker decides from free VRAM when it loads: 8 GB and larger NVIDIA
- cards hold the whole model. Asks the driver once, without CUDA; anything
+ cards keep their layers on the GPU; 6 GB cards stream from RAM the layers
+ that do not fit, which needs more of it. Asks the driver once, without CUDA; anything
unknown states the larger figure.
"""
if self.ram is None:
- from .worker import MAPPED_RAM, RESIDENT_RAM
+ from .worker import MAPPED_RAM, RESIDENT_RAM, STREAMED_RAM
self.ram = MAPPED_RAM
if machine.get('device_backend', 'cuda') == 'cuda':
from .. import processes
@@ -246,6 +247,8 @@ class Service:
total = totals.get(machine.get('gpu_uuid')) or (len(totals) == 1 and next(iter(totals.values())))
if total and total >= 7.5 * 1024:
self.ram = RESIDENT_RAM
+ elif total and total >= 5.5 * 1024:
+ self.ram = STREAMED_RAM
except (OSError, RuntimeError, ValueError, subprocess.SubprocessError):
pass
return self.ram
diff --git a/freevideo_engine/prompt_vlm/worker.py b/freevideo_engine/prompt_vlm/worker.py
index 3459536..3a2b816 100644
--- a/freevideo_engine/prompt_vlm/worker.py
+++ b/freevideo_engine/prompt_vlm/worker.py
@@ -150,7 +150,7 @@ def failure(error, mode='rewrite'):
Tokenizers may quote only a fragment of either draft: replacing the full
prompt in an exception is not sufficient to make that exception private.
"""
- allowed = {'ram_space', 'image_size', 'context_length', 'invalid_output'}
+ allowed = {'ram_space', 'image_size', 'context_length', 'invalid_output', 'out_of_memory'}
message = str(error)
fallback = 'ask_failed' if mode == 'ask' else 'rewrite_failed'
code = message if message in allowed else 'out_of_memory' if 'out of memory' in message.lower() else fallback
@@ -205,6 +205,7 @@ def tokenize(processor, messages, max_pixels=512 * 32 * 32):
# CUDA context alone take about 2.5 GB of commit). Other placements map and
# merge the whole model in RAM; theirs is checked after the imports.
RESIDENT_RAM = 4 * 2**30
+STREAMED_RAM = 6 * 2**30 # download-card figure for 6 GB cards: resident part plus the streamed layers' host copy
MAPPED_RAM = 11 * 2**30
@@ -263,6 +264,8 @@ def load_model(folder, torch, stats, checkpoint):
shapes = stored(folder)
weights = sum(row['bytes'] for row in shapes.values())
resident = None
+ keep = None
+ host_bytes = 0
weight_budget = 0
if torch.cuda.is_available():
from .. import gpu_budget
@@ -277,7 +280,19 @@ def load_model(folder, torch, stats, checkpoint):
kv = kv_bytes(shapes, reserved_tokens(stats))
stats.update(gpu_allocator_budget_bytes=free, gpu_weight_budget_bytes=weight_budget, gpu_kv_reserve_bytes=kv)
resident = placement(weights, int8.saved_bytes(shapes), kv, free)
- stats['ram_required_bytes'] = RESIDENT_RAM if resident else MAPPED_RAM
+ if not resident:
+ # Too big even as int8: keep what fits on the GPU and stream the other
+ # layers' int8 rows from host memory, a layer ahead of use.
+ plan = int8.resident_layers(weights, shapes, free - kv - WORKING_VRAM)
+ if plan:
+ resident, (keep, host_bytes) = 'int8', plan
+ stats.update(streamed_layers=len(int8.layer_bytes(shapes)) - keep if keep is not None else 0,
+ streamed_host_bytes=host_bytes)
+ if not resident and weight_budget <= 2 * 2**30:
+ # Not even the streaming tier fits: the model would run on the CPU at
+ # about 0.2 tokens/s and hit the request deadline, so say so at once.
+ raise ValueError('out_of_memory')
+ stats['ram_required_bytes'] = (RESIDENT_RAM + host_bytes) if resident else MAPPED_RAM
available = stats['available_memory_at_start_bytes' if resident else 'available_memory_before_load_bytes']
if available < stats['ram_required_bytes']:
raise ValueError('ram_space')
@@ -285,7 +300,7 @@ def load_model(folder, torch, stats, checkpoint):
loading = time.monotonic()
if resident:
from .load import resident as load_resident
- model = load_resident(folder, torch, 'cuda:0', resident)
+ model = load_resident(folder, torch, 'cuda:0', resident, keep)
mapping, device = {'': 0}, 'cuda:0'
stats.update(weights=resident, kv_positions=kv_positions(reserved_tokens(stats)))
else: