Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
Hold one stream-sized buffer at the staged residual boundaries
Bowen Xueed2dd58208a5
diff --git a/freevideo_engine/residual.py b/freevideo_engine/residual.pyindex 64b2ac5..29f645f 100644--- a/freevideo_engine/residual.py+++ b/freevideo_engine/residual.py@@ -7,6 +7,12 @@ Copies retain dtype and shape; normalization and residual kernels are unchanged.import timeimport torch+# Rows handled per step of the tiled sublayer boundaries below. A bounded+# block holds the residual stream on the card only between the host copies, so+# a whole-stream normalization output or restored copy next to the stream+# itself doubles the largest activation. Tiles keep one stream-sized buffer.+ROW_TILE = 8192+class ResidualState:def __init__(self, tensor):@@ -113,6 +119,57 @@ class ResidualState:self.copy_wall_seconds += time.perf_counter()-tickreturn tensor+ def tiled(self):+ """True when the stream has more rows than one tile."""+ return self.shape[-2] > ROW_TILE++ def store_through(self, tensor, transform):+ """Stage `tensor` on the host, then overwrite its rows with `transform`.++ `transform(rows, start, stop)` maps a tile of rows to its replacement of+ the same shape. The staged copy of each tile is queued before that tile+ is overwritten on the same stream, so the host receives the original+ rows and the device buffer ends up holding the transformed ones, with+ no second stream-sized buffer. Row-wise kernels give every row the same+ bytes whatever the tile boundaries (row_kernels).+ """+ self._check(tensor)+ if self.tensor is not None:+ raise RuntimeError('Consume the device residual before staging it')+ tick = time.perf_counter()+ if self.host is None:+ self.host = self._allocate()+ for start in range(0, self.shape[-2], ROW_TILE):+ stop = min(start + ROW_TILE, self.shape[-2])+ rows = tensor[..., start:stop, :]+ self.host[..., start:stop, :].copy_(rows, non_blocking=self.pinned)+ rows.copy_(transform(rows, start, stop))+ self.stored = True+ self.copies['to_host'] += 1+ self.copy_wall_seconds += time.perf_counter()-tick+ return tensor++ def restore_into(self, branch, combine):+ """Combine the staged stream with `branch` tile by tile, in place.++ `combine(residual_rows, start, stop, branch_rows)` writes into+ `branch_rows` (a view of `branch`); the staged rows reach the device one+ tile at a time instead of as a second stream-sized copy.+ """+ if self.closed or not self.stored:+ raise RuntimeError('No complete host residual is available')+ if tuple(branch.shape) != self.shape or branch.dtype != self.dtype or branch.device != self.device:+ raise ValueError('Residual shape, dtype or device changed')+ tick = time.perf_counter()+ for start in range(0, self.shape[-2], ROW_TILE):+ stop = min(start + ROW_TILE, self.shape[-2])+ rows = self.host[..., start:stop, :].to(self.device, copy=True, non_blocking=self.pinned)+ combine(rows, start, stop, branch[..., start:stop, :])+ del rows+ self.copies['to_device'] += 1+ self.copy_wall_seconds += time.perf_counter()-tick+ return branch+def replace(self, tensor):self._check(tensor)if self.tensor is not None:@@ -139,17 +196,37 @@ def residual_forward(pre, post):raise RuntimeError('Residual offload requires inference without autograd')if not isinstance(state, ResidualState):raise TypeError('Residual offload needs the owning packed-input container')+ tiled = state.tiled()hidden = state.take()shift_a, scale_a, gate_a, shift_f, scale_f, gate_f = self.adaln_proj(temb)- normalized = pre(hidden, self.norm1.weight, self.norm1.eps, scale_a, shift_a, adaln_indices)- state.store(hidden)- del hidden+ if tiled:+ # The normalized rows replace the stream rows once they are staged.+ normalized = state.store_through(hidden, lambda rows, start, stop: pre(+ rows, self.norm1.weight, self.norm1.eps, scale_a, shift_a, adaln_indices[start:stop]))+ del hidden+ else:+ normalized = pre(hidden, self.norm1.weight, self.norm1.eps, scale_a, shift_a, adaln_indices)+ state.store(hidden)+ del hiddenif state.collect_after_store:state.collect_after_store = Falseimport gcgc.collect()branch = self.attn(normalized, rotary_emb, attention_mask)del normalized+ if tiled:+ hidden = state.restore_into(branch, lambda residual, start, stop, rows: post(+ residual, gate_a, adaln_indices[start:stop], rows))+ del branch+ normalized = state.store_through(hidden, lambda rows, start, stop: pre(+ rows, self.norm2.weight, self.norm2.eps, scale_f, shift_f, adaln_indices[start:stop]))+ del hidden+ branch = self.ff(normalized)+ del normalized+ state.replace(state.restore_into(branch, lambda residual, start, stop, rows: post(+ residual, gate_f, adaln_indices[start:stop], rows)))+ del branch+ return stateresidual = state.restore()hidden = post(residual, gate_a, adaln_indices, branch)del residual, branch