Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

8.8 KB
Raw Blame
"""Run the H3 text encoder's language model in token blocks for very long inputs.

The native forward keeps FP32 activations, builds one dense T x T causal mask
and runs every projection and MLP over the whole sequence at once. Measured on
the 32B encoder, a 15 s reference video (15.7k tokens) needed 9.5 GiB beyond
the weights and two videos plus three images (34k tokens) 22 GiB, more than a
16 GB card has. The same math fits in a fraction of that:

- attention runs per block of queries, each against all earlier keys with the
  causal limit as a lower-right bias, so no mask is ever materialized; the
  memory-efficient kernel takes FP32 but not grouped K/V heads, so each K/V
  head is broadcast (a view, no copy) to its group of query heads;
- the position-wise MLP runs per block of tokens;
- DeepStack features are dropped after the layers that use them.

Only the language-model path the H3 encoder uses is replaced. Calls with KV
caches, attention masks or intermediate layer lists go to the native code.
"""
from contextlib import contextmanager
import types

import torch
import torch.nn.functional as F


def grouped_causal_attention(q, k, v, start):
    """Queries at positions start..start+len(q) over keys 0..start+len(q), causal, grouped K/V heads."""
    from torch.nn.attention.bias import causal_lower_right
    batch, heads, rows, width = q.shape
    groups = k.shape[1]
    per_group = heads // groups
    end = start + rows
    bias = causal_lower_right(rows, end)
    output = torch.empty((batch, heads, rows, width), device=q.device, dtype=q.dtype)
    for group in range(groups):
        first = group * per_group
        keys = k[:, group:group + 1, :end].expand(batch, per_group, end, width)
        values = v[:, group:group + 1, :end].expand(batch, per_group, end, width)
        output[:, first:first + per_group] = F.scaled_dot_product_attention(
            q[:, first:first + per_group], keys, values, attn_mask=bias)
    return output


def rope(x, freqs_cis):
    """The native apply_rope, for one tensor (queries or keys) at a time."""
    cos, sin, nsin = freqs_cis[0], freqs_cis[1], freqs_cis[2]
    embedded = x * cos
    half = embedded.shape[-1] // 2
    embedded[..., :half].addcmul_(x[..., half:], nsin)
    embedded[..., half:].addcmul_(x[..., :half], sin)
    return embedded.to(x.dtype)


def chunked_attention(attention, chunk):
    original = attention.forward

    def forward(self, hidden_states, attention_mask=None, freqs_cis=None, optimized_attention=None,
                past_key_value=None, sliding_window=None):
        batch, length, _ = hidden_states.shape
        if optimized_attention is not None:  # a native caller: its mask and caches apply
            return original(hidden_states, attention_mask=attention_mask, freqs_cis=freqs_cis,
                            optimized_attention=optimized_attention, past_key_value=past_key_value,
                            sliding_window=sliding_window)
        if (attention_mask is not None or past_key_value is not None or sliding_window is not None
                or freqs_cis is None or any(t.shape[-2] != length for t in freqs_cis[:3])):
            raise ValueError('Block-wise encoder attention supports causal attention without caches only')
        if self.merged_qkv:
            fused = self.qkv_proj(hidden_states)
            projected = dict(zip(('q', 'k', 'v'), fused.split((self.inner_size, self.kv_size, self.kv_size), dim=-1)))
        else:
            projected = {}
        keys = (projected.get('k') if projected else self.k_proj(hidden_states))
        values = (projected.get('v') if projected else self.v_proj(hidden_states))
        keys = keys.view(batch, length, self.num_kv_heads, self.head_dim).transpose(1, 2)
        values = values.view(batch, length, self.num_kv_heads, self.head_dim).transpose(1, 2)
        if self.k_norm is not None:
            keys = self.k_norm(keys)
        keys = rope(keys, freqs_cis)
        output = None
        for start in range(0, length, chunk):
            end = min(length, start + chunk)
            queries = (projected['q'][:, start:end] if projected else self.q_proj(hidden_states[:, start:end]))
            queries = queries.reshape(batch, end - start, self.num_heads, self.head_dim)
            queries = queries.transpose(1, 2)
            if self.q_norm is not None:
                queries = self.q_norm(queries)
            queries = rope(queries, tuple(t[..., start:end, :] for t in freqs_cis[:3]))
            value = grouped_causal_attention(queries, keys, values, start)
            del queries
            value = self.o_proj(value.transpose(1, 2).reshape(batch, end - start, self.num_heads * self.head_dim))
            if output is None:
                output = torch.empty((batch, length, value.shape[-1]), device=value.device, dtype=value.dtype)
            output[:, start:end] = value
            del value
        return output, None
    return types.MethodType(forward, attention)


def chunked_mlp(mlp, chunk):
    original = mlp.forward

    def forward(self, x):
        if x.shape[1] <= chunk:
            return original(x)
        output = torch.empty_like(x)
        for start in range(0, x.shape[1], chunk):
            output[:, start:start + chunk] = original(x[:, start:start + chunk])
        return output
    return types.MethodType(forward, mlp)


def language_model(model):
    """The Llama2_ module inside the native H3 text encoder."""
    for _, module in model.named_modules():
        if type(module).__name__ == 'Llama2_' and hasattr(module, 'layers'):
            return module
    raise ValueError('The native H3 text encoder has no Llama language model')


def chunked_forward(llama, original):
    def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None,
                final_layer_norm_intermediate=True, dtype=None, position_ids=None, embeds_info=[],
                past_key_values=None, input_ids=None, deepstack_embeds=None, visual_pos_masks=None):
        if (attention_mask is not None or past_key_values is not None
                or isinstance(intermediate_output, (list, str))):
            return original(x, attention_mask=attention_mask, embeds=embeds, num_tokens=num_tokens,
                            intermediate_output=intermediate_output,
                            final_layer_norm_intermediate=final_layer_norm_intermediate, dtype=dtype,
                            position_ids=position_ids, embeds_info=embeds_info, past_key_values=past_key_values,
                            input_ids=input_ids, deepstack_embeds=deepstack_embeds, visual_pos_masks=visual_pos_masks)
        x = embeds if embeds is not None else self.embed_tokens(x, out_dtype=dtype)
        length = x.shape[1]
        if position_ids is None:
            position_ids = torch.arange(length, device=x.device).unsqueeze(0)
        freqs_cis = self.compute_freqs_cis(position_ids, x.device)
        if intermediate_output is not None and intermediate_output < 0:
            intermediate_output = len(self.layers) + intermediate_output
        deepstack = list(deepstack_embeds) if deepstack_embeds is not None else None
        deepstack_embeds = None
        intermediate = None
        for index, layer in enumerate(self.layers):
            # The block-wise attention installed on each layer needs no mask.
            x, _ = layer(x=x, attention_mask=None, freqs_cis=freqs_cis, optimized_attention=None,
                         past_key_value=None)
            # DeepStack: per-layer visual features at image positions (Qwen3-VL), as the native forward.
            if deepstack is not None and index < len(deepstack):
                x[visual_pos_masks] = x[visual_pos_masks] + deepstack[index].to(x)
                deepstack[index] = None  # used by this layer only
            if index == intermediate_output:
                intermediate = x.clone()
        if self.norm is not None:
            x = self.norm(x)
        if intermediate is not None and final_layer_norm_intermediate and self.norm is not None:
            intermediate = self.norm(intermediate)
        return x, intermediate
    return types.MethodType(forward, llama)


@contextmanager
def low_memory(model, chunk):
    """Within the block, the encoder's language model runs attention and MLPs in `chunk`-token blocks."""
    llama = language_model(model)
    patched = []

    def patch(module, replacement):
        patched.append((module, module.__dict__.get('forward')))
        module.forward = replacement

    try:
        patch(llama, chunked_forward(llama, llama.forward))
        for layer in llama.layers:
            patch(layer.self_attn, chunked_attention(layer.self_attn, chunk))
            patch(layer.mlp, chunked_mlp(layer.mlp, chunk))
        yield
    finally:
        for module, previous in reversed(patched):
            if previous is None:
                del module.forward  # back to the class's own forward
            else:
                module.forward = previous