Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Hold one stream-sized buffer at the staged residual boundaries

Bowen Xueed2dd58208a5
diff --git a/freevideo_engine/residual.py b/freevideo_engine/residual.py
index 64b2ac5..29f645f 100644
--- a/freevideo_engine/residual.py
+++ b/freevideo_engine/residual.py
@@ -7,6 +7,12 @@ Copies retain dtype and shape; normalization and residual kernels are unchanged.
import time
import torch
+# Rows handled per step of the tiled sublayer boundaries below. A bounded
+# block holds the residual stream on the card only between the host copies, so
+# a whole-stream normalization output or restored copy next to the stream
+# itself doubles the largest activation. Tiles keep one stream-sized buffer.
+ROW_TILE = 8192
+
class ResidualState:
def __init__(self, tensor):
@@ -113,6 +119,57 @@ class ResidualState:
self.copy_wall_seconds += time.perf_counter()-tick
return tensor
+ def tiled(self):
+ """True when the stream has more rows than one tile."""
+ return self.shape[-2] > ROW_TILE
+
+ def store_through(self, tensor, transform):
+ """Stage `tensor` on the host, then overwrite its rows with `transform`.
+
+ `transform(rows, start, stop)` maps a tile of rows to its replacement of
+ the same shape. The staged copy of each tile is queued before that tile
+ is overwritten on the same stream, so the host receives the original
+ rows and the device buffer ends up holding the transformed ones, with
+ no second stream-sized buffer. Row-wise kernels give every row the same
+ bytes whatever the tile boundaries (row_kernels).
+ """
+ self._check(tensor)
+ if self.tensor is not None:
+ raise RuntimeError('Consume the device residual before staging it')
+ tick = time.perf_counter()
+ if self.host is None:
+ self.host = self._allocate()
+ for start in range(0, self.shape[-2], ROW_TILE):
+ stop = min(start + ROW_TILE, self.shape[-2])
+ rows = tensor[..., start:stop, :]
+ self.host[..., start:stop, :].copy_(rows, non_blocking=self.pinned)
+ rows.copy_(transform(rows, start, stop))
+ self.stored = True
+ self.copies['to_host'] += 1
+ self.copy_wall_seconds += time.perf_counter()-tick
+ return tensor
+
+ def restore_into(self, branch, combine):
+ """Combine the staged stream with `branch` tile by tile, in place.
+
+ `combine(residual_rows, start, stop, branch_rows)` writes into
+ `branch_rows` (a view of `branch`); the staged rows reach the device one
+ tile at a time instead of as a second stream-sized copy.
+ """
+ if self.closed or not self.stored:
+ raise RuntimeError('No complete host residual is available')
+ if tuple(branch.shape) != self.shape or branch.dtype != self.dtype or branch.device != self.device:
+ raise ValueError('Residual shape, dtype or device changed')
+ tick = time.perf_counter()
+ for start in range(0, self.shape[-2], ROW_TILE):
+ stop = min(start + ROW_TILE, self.shape[-2])
+ rows = self.host[..., start:stop, :].to(self.device, copy=True, non_blocking=self.pinned)
+ combine(rows, start, stop, branch[..., start:stop, :])
+ del rows
+ self.copies['to_device'] += 1
+ self.copy_wall_seconds += time.perf_counter()-tick
+ return branch
+
def replace(self, tensor):
self._check(tensor)
if self.tensor is not None:
@@ -139,17 +196,37 @@ def residual_forward(pre, post):
raise RuntimeError('Residual offload requires inference without autograd')
if not isinstance(state, ResidualState):
raise TypeError('Residual offload needs the owning packed-input container')
+ tiled = state.tiled()
hidden = state.take()
shift_a, scale_a, gate_a, shift_f, scale_f, gate_f = self.adaln_proj(temb)
- normalized = pre(hidden, self.norm1.weight, self.norm1.eps, scale_a, shift_a, adaln_indices)
- state.store(hidden)
- del hidden
+ if tiled:
+ # The normalized rows replace the stream rows once they are staged.
+ normalized = state.store_through(hidden, lambda rows, start, stop: pre(
+ rows, self.norm1.weight, self.norm1.eps, scale_a, shift_a, adaln_indices[start:stop]))
+ del hidden
+ else:
+ normalized = pre(hidden, self.norm1.weight, self.norm1.eps, scale_a, shift_a, adaln_indices)
+ state.store(hidden)
+ del hidden
if state.collect_after_store:
state.collect_after_store = False
import gc
gc.collect()
branch = self.attn(normalized, rotary_emb, attention_mask)
del normalized
+ if tiled:
+ hidden = state.restore_into(branch, lambda residual, start, stop, rows: post(
+ residual, gate_a, adaln_indices[start:stop], rows))
+ del branch
+ normalized = state.store_through(hidden, lambda rows, start, stop: pre(
+ rows, self.norm2.weight, self.norm2.eps, scale_f, shift_f, adaln_indices[start:stop]))
+ del hidden
+ branch = self.ff(normalized)
+ del normalized
+ state.replace(state.restore_into(branch, lambda residual, start, stop, rows: post(
+ residual, gate_f, adaln_indices[start:stop], rows)))
+ del branch
+ return state
residual = state.restore()
hidden = post(residual, gate_a, adaln_indices, branch)
del residual, branch