Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

20.4 KB
Raw Blame
"""Bound H3 activations with head groups; optional locally validated overlap.

Reuses upstream's head-sliced linear branch (originally used by Ulysses).
No heads, windows, text states or scan frames are dropped. Sliced GEMMs can
change floating-point reduction order and must be reported as an ablation.
"""
from contextlib import nullcontext
import gc
import types
import torch
import torch.nn.functional as F
from .lora_online import shared_down


# Sequence lengths whose first attention call has run. That call compiles, and
# the compile leaves each head group's frame, with its Q/K/V, in a reference
# cycle: under Windows the next group then ran beside a dead group's Q/K/V.
# The first call of each length collects after every group.
COLLECTED_LENGTHS = set()

# Rows per output-projection tile on the GPU tail. Int8 results do not depend on
# the tiling; 1024-row tiles cost about 2% of a second-pass step in launches.
TAIL_ROWS = 8192


class RowPlanes:
    """A (rows, width) plane kept as two row blocks: the leading rows in a lent
    buffer, the remainder in a new one. Values and row order are unchanged."""
    def __init__(self, lead, rest):
        self.lead, self.rest, self.split = lead, rest, len(lead)

    def write(self, columns, value):
        self.lead[:, columns].copy_(value[:self.split])
        self.rest[:, columns].copy_(value[self.split:])

    def blocks(self, size):
        for offset, plane in ((0, self.lead), (self.split, self.rest)):
            for row in range(0, len(plane), size):
                yield slice(offset + row, offset + min(row + size, len(plane))), plane[row:row + size]


def project_rows(planes, out, project, size=TAIL_ROWS):
    """Project RowPlanes into out in ascending row tiles.

    out may share storage with planes.lead when the plane is at least as wide
    as out: tile [a, b) then writes over the lead's rows below b * out / width,
    which this tile or an earlier one has already read.
    """
    for rows, part in planes.blocks(size):
        out[rows] = project(part)


def release_readouts(policy):
    """Return a previous canvas's page-locked attention readouts before allocating new ones.

    The first pass's readouts stayed referenced while the refine pass allocated
    its larger ones, and the host allocator kept the freed blocks. Windows
    page-locks both against a non-local budget of about half of physical RAM:
    a 16 GiB machine failed cudaHostAlloc at the refine pass's first layer.
    """
    held = [getattr(policy, name, None) for name in ('group_readouts', 'host_outputs')]
    for name in ('group_readouts', 'host_outputs', 'host_output_key', 'host_copies'):
        if hasattr(policy, name):
            delattr(policy, name)
    if any(value is not None for value in held):
        del held
        gc.collect()
        # Copies into blocks the host allocator is about to free can still be
        # in flight at a pass's first layer (residual staging, the previous
        # pass's last tiles). Returning them then broke the next copy on
        # Windows with cudaErrorInvalidValue; wait for the device first.
        if torch.cuda.is_initialized():
            torch.cuda.synchronize()
        from .torch_compat import empty_host_cache
        empty_host_cache(torch)


def install_head_chunks(transformer, policy, chunk, cpu_outputs=False, projection_chunk=1024, grouped_outputs=False,
                        parallelism=1):
    if chunk < 1 or projection_chunk < 1:
        raise ValueError('Head and attention-output chunks must be positive')
    if type(parallelism) is not int or parallelism not in (1, 2, 4):
        raise ValueError('Head parallelism must be 1, 2 or 4')
    if parallelism > 1 and (cpu_outputs or grouped_outputs):
        raise ValueError('Parallel head groups require GPU attention outputs')
    from src.models.hybrid_transform import iter_hybrids
    from src.models.softmax_attention.kernels import _qk_prep
    from src.models.ops.fp8_linear import Fp8Linear, quantize_activation
    from .fp8_ops import ColumnScale, project_with_scale, row_scale, sliced_projection
    from .int8_ops import Int8Linear, matmul as int8_matmul

    def plain_int8(modules):
        # Exact only for int8 rows; FP8 GEMMs and LoRA products keep their original form.
        return all(isinstance(m, Int8Linear) and not hasattr(m, '_freevideo_lora') for m in modules)

    def row_tail(attn):
        return plain_int8((attn.orig.to_out[0], attn.to_out_linear))

    def reuses_input(attn, x, projections):
        # A block that owns its normalized input lends its storage to the softmax
        # output plane once every head group's inputs are taken from it.
        return (parallelism == 1 and getattr(attn, '_freevideo_owns_input', False) and x.is_contiguous()
                and row_tail(attn) and plain_int8(projections)
                and attn.linear_attention.output_gate.down is not None)

    def forward(attn, x, rotary):
        layout = attn.layout
        if layout is None or not attn.inference_mode:
            raise ValueError('Head chunks require an explicit packed layout and inference kernels')
        bounds = attn._bounds(layout)
        if all(lo <= 0 and hi >= layout.num_frames - 1 for lo, hi in bounds):
            raise ValueError('This profile is for the VDN hybrid window layout')
        orig, branch = attn.orig, attn.linear_attention
        projections = (orig.to_q, orig.to_k, orig.to_v)
        quantized = quantize_activation(x) if all(isinstance(p, Fp8Linear) for p in projections) else None
        video = slice(layout.video_start, layout.video_end)
        text = slice(*layout.text_range)
        frame_mean = x[video].view(layout.num_frames, layout.tokens_per_frame, -1).mean(1, dtype=torch.float32)
        gate_hidden = x[video] if branch.output_gate.down is None else branch.output_gate.down(x[video])
        soft_shape = (len(x), attn.num_heads * attn.head_dim)
        taken = {}
        linear_shape = (layout.num_frames * layout.tokens_per_frame, attn.num_heads * branch.head_dim)
        soft_scale = ColumnScale(len(x), x.device) if cpu_outputs and isinstance(orig.to_out[0], Fp8Linear) else None
        linear_scale = ColumnScale(linear_shape[0], x.device) if cpu_outputs and isinstance(attn.to_out_linear, Fp8Linear) else None
        output_width = orig.to_out[0].weight_fp8.shape[0] if isinstance(orig.to_out[0], Fp8Linear) else orig.to_out[0].out_features
        if grouped_outputs:
            from .activation_staging import HeadReadouts
            widths = [min(chunk, attn.num_heads - start) * attn.head_dim for start in range(0, attn.num_heads, chunk)]
            stores = getattr(policy, 'group_readouts', None)
            if stores is None or any(not store.supports(rows, widths, x.dtype, projection_chunk)
                                     for store, rows in zip(stores, (len(x), linear_shape[0]))):
                stores = None  # the old tuple must be unreferenced when it is returned
                release_readouts(policy)
                policy.group_readouts = tuple(HeadReadouts(rows, widths, x.dtype, projection_chunk)
                                             for rows in (len(x), linear_shape[0]))
            soft, linear = policy.group_readouts
        elif cpu_outputs:
            from .activation_staging import LockedCopies
            key = (soft_shape, linear_shape, x.dtype)
            if getattr(policy, 'host_output_key', None) != key:
                release_readouts(policy)
                policy.host_outputs = (torch.empty(soft_shape, dtype=x.dtype, pin_memory=True),
                                       torch.empty(linear_shape, dtype=x.dtype, pin_memory=True))
                policy.host_output_key = key
                policy.host_copies = LockedCopies()
            soft, linear = policy.host_outputs
        elif reuses_input(attn, x, projections):
            # Measured second-pass peak: the head groups, with x, both output planes,
            # x's int8 rows and one group's work alive at once. Everything a group
            # reads from x is taken first (int8 rows, softmax gates, betas, with
            # the same shapes as before), then x's storage holds the leading rows
            # of the softmax plane: one sequence buffer less at the peak.
            taken['rows'] = projections[0].quantized(x)
            Int8Linear._last.clear()
            gate_x = None
            if attn.enable_softmax_gate:
                gate_x = x if attn.softmax_gate.down is None else attn.softmax_gate.down(x)
            for start in range(0, attn.num_heads, chunk):
                heads = slice(start, min(start + chunk, attn.num_heads))
                gate = None
                if gate_x is not None:
                    gate = torch.sigmoid(F.linear(gate_x, attn.softmax_gate.up.weight[heads],
                                                  attn.softmax_gate.up.bias[heads]))
                taken[start] = (gate, torch.sigmoid(F.linear(x[video], branch.beta_proj.weight[heads])),
                                torch.sigmoid(F.linear(x[text], branch.beta_proj.weight[heads]))
                                if attn.enable_text_state else None)
            del gate_x
            lead = x.numel() // soft_shape[1]
            soft = RowPlanes(x.view(-1)[:lead * soft_shape[1]].view(lead, soft_shape[1]),
                             x.new_empty((len(x) - lead, soft_shape[1])))
            linear = x.new_empty(linear_shape)
        else:
            soft, linear = x.new_empty(soft_shape), x.new_empty(linear_shape)
        def group(start):
            end = min(start + chunk, attn.num_heads)
            heads = slice(start, end)
            channels = slice(start * attn.head_dim, end * attn.head_dim)
            if taken:
                # Int8Linear.project on x's int8 rows, which x no longer holds.
                raw = tuple(int8_matmul(taken['rows'], proj.weight_int8[channels],
                                        proj.weight_scale.reshape(-1)[channels],
                                        None if proj.bias is None else proj.bias[channels], out_dtype=x.dtype)
                            .view(len(x), end - start, attn.head_dim) for proj in projections)
                softmax_gate, beta, text_beta = taken.pop(start)
            else:
                raw = tuple(sliced_projection(proj, x, channels, quantized)
                            .view(len(x), end - start, attn.head_dim) for proj in projections)
            q = _qk_prep(raw[0], orig.norm_q.weight, orig.norm_q.eps, *rotary)
            k = _qk_prep(raw[1], orig.norm_k.weight, orig.norm_k.eps, *rotary)
            attended = policy(q, k, raw[2], layout, bounds, attn.head_dim ** -.5, attn.anchor_frames)
            del q, k
            if attn.enable_softmax_gate:
                if taken:
                    gate = softmax_gate
                else:
                    gate_module = attn.softmax_gate
                    gate_x = x if gate_module.down is None else gate_module.down(x)
                    gate = torch.sigmoid(F.linear(gate_x, gate_module.up.weight[heads], gate_module.up.bias[heads]))
                attended.mul_(gate[..., None])
            if soft_scale is not None:
                soft_scale.update(attended.flatten(1))
            if grouped_outputs:
                soft.store(start // chunk, attended.flatten(1))
            elif taken:
                soft.write(channels, attended.flatten(1))
            elif cpu_outputs:
                policy.host_copies.to_host(soft[:, channels], attended.flatten(1))
            else:
                soft[:, channels].copy_(attended.flatten(1))
            del attended
            if not taken:
                beta = torch.sigmoid(F.linear(x[video], branch.beta_proj.weight[heads]))
                text_beta = torch.sigmoid(F.linear(x[text], branch.beta_proj.weight[heads])) if attn.enable_text_state else None
            linear_channels = slice(start * branch.head_dim, end * branch.head_dim)
            gate = torch.sigmoid(F.linear(gate_hidden, branch.output_gate.up.weight[linear_channels],
                                         branch.output_gate.up.bias[linear_channels]))
            gate = gate.view(-1, end - start, branch.head_dim)
            readout = branch(None, layout.num_frames, layout.tokens_per_frame, bounds,
                             qkv_raw=tuple(t[video] for t in raw), frame_size=layout.frame_size,
                             skip_ends=attn.anchor_frames == 'both',
                             text_qkv_raw=tuple(t[text] for t in raw) if attn.enable_text_state else None,
                             inference=True, heads=heads, beta=beta, gate=gate,
                             frame_mean=frame_mean, text_beta=text_beta)
            if linear_scale is not None:
                linear_scale.update(readout)
            if grouped_outputs:
                linear.store(start // chunk, readout.contiguous())
            elif cpu_outputs:
                policy.host_copies.to_host(linear[:, linear_channels], readout)
            else:
                linear[:, linear_channels].copy_(readout)
            del raw, readout, gate, beta, text_beta

        # Keep the exact projection/head/window shapes. Merely widening the
        # head chunk changes GEMM rounding; independent streams can overlap
        # these original groups without changing their arithmetic partition.
        # First execute this shape on the calling stream: upstream attention
        # and scan caches lazily create shared device indices/compiled kernels.
        warm_key = (tuple(x.shape), x.dtype, str(x.device), chunk, layout, tuple(bounds),
                    attn.anchor_frames, attn.enable_text_state, policy.window_batch,
                    policy.backend, policy.window_varlen)
        concurrent = parallelism > 1 and getattr(attn, '_freevideo_parallel_warm', None) == warm_key
        parent = torch.cuda.current_stream(x.device) if concurrent else None
        streams = []
        if concurrent:
            policy.prepare(layout, bounds, x.device, attn.anchor_frames, batches=True)
            from src.models.linear_attention.scan import _gather_indices
            _gather_indices(bounds, layout.num_frames, x.device)
            stream_key = (str(x.device), parallelism)
            if getattr(policy, 'head_stream_key', None) != stream_key:
                policy.head_streams = [torch.cuda.Stream(device=x.device) for _ in range(parallelism)]
                policy.head_stream_key = stream_key
            streams = policy.head_streams
            # Inputs and disjoint output planes belong to the parent allocator.
            # record_stream also protects their lifetime if a group raises OOM.
            shared = [x, frame_mean, gate_hidden, soft, linear, *rotary]
            if quantized is not None:
                shared.extend(quantized)
            for stream in streams:
                stream.wait_stream(parent)
                for tensor in shared:
                    tensor.record_stream(stream)
        try:
            fresh = not streams and len(x) not in COLLECTED_LENGTHS
            with shared_down() if not streams else nullcontext():
                for index, start in enumerate(range(0, attn.num_heads, chunk)):
                    with torch.cuda.stream(streams[index % len(streams)]) if streams else nullcontext():
                        group(start)
                    if fresh:
                        gc.collect()
            COLLECTED_LENGTHS.add(len(x))
        finally:
            # Output projection, offload cleanup and the next block cannot
            # race unfinished groups, including after a recoverable OOM.
            for stream in streams:
                parent.wait_stream(stream)
        if parallelism > 1:
            attn._freevideo_parallel_warm = warm_key
            counter = 'parallel_head_calls' if concurrent else 'parallel_head_warmups'
            setattr(policy, counter, getattr(policy, counter, 0) + 1)
        del quantized
        soft_scale = soft_scale.finish() if soft_scale is not None else None
        linear_scale = linear_scale.finish() if linear_scale is not None else None
        if grouped_outputs:
            torch.cuda.current_stream(x.device).synchronize()
            out = x.new_empty((len(x), output_width))
            for rows, part in soft.project(lambda part: part, out, len(x)):
                scale = row_scale(soft_scale, rows) if soft_scale is not None else None
                out[rows] = orig.to_out[1](project_with_scale(orig.to_out[0], part, scale))
            for rows, part in linear.project(lambda part: part, out, linear_shape[0]):
                scale = row_scale(linear_scale, rows) if linear_scale is not None else None
                out[video][rows].add_(project_with_scale(attn.to_out_linear, part, scale))
            return out
        if cpu_outputs:
            # All D2H writes must finish before the pinned outputs are read by H2D.
            torch.cuda.current_stream(x.device).synchronize()
            # Project bounded row slices directly from the host readout planes.
            # Copying whole planes back would retain >2 GB of temporary buffers.
            out = x.new_empty((len(x), output_width))
            for row in range(0, len(x), projection_chunk):
                rows = slice(row, row + projection_chunk)
                part = policy.host_copies.to_device(soft[rows], x.device)
                scale = row_scale(soft_scale, rows) if soft_scale is not None else None
                out[rows] = orig.to_out[1](project_with_scale(orig.to_out[0], part, scale))
                del part
            for row in range(0, len(linear), projection_chunk):
                rows = slice(row, row + projection_chunk)
                part = policy.host_copies.to_device(linear[rows], x.device)
                scale = row_scale(linear_scale, rows) if linear_scale is not None else None
                out[video][rows].add_(project_with_scale(attn.to_out_linear, part, scale))
                del part
            return out
        if row_tail(attn):
            # Int8 projections quantize each row and accumulate exactly in int32, so
            # row slices give the bits of one full-height GEMM. The full-height tail
            # allocated a whole output, the quantized soft plane and a whole
            # linear-branch product beside soft and linear (1.45 GiB more through
            # the residual add and FF at 1344x768x243). A block that owns its
            # normalized input, dead once every head group ran, gets the output
            # written back into it.
            if isinstance(soft, RowPlanes):
                taken.clear()  # x's int8 rows
                # The softmax plane's leading rows occupy x's storage. Output rows
                # [a, b) overwrite the bytes of softmax rows below b * out / width,
                # all projected by this tile or an earlier one, so the output goes
                # back into x instead of a new sequence buffer. Without expandable
                # segments (Windows) that buffer was the allocation the freed
                # attention temporaries could not hold: 1344x768x362 reserved
                # 7.05 GiB for 5.74 allocated, 6.06 with the output in x.
                inplace = (x.shape[1] == output_width and x.is_contiguous()
                           and soft.lead.shape[1] >= output_width)
                out = x if inplace else x.new_empty((len(x), output_width))
                project_rows(soft, out, lambda part: orig.to_out[1](orig.to_out[0](part)))
            else:
                owned = getattr(attn, '_freevideo_owns_input', False) and x.shape[1] == output_width and x.is_contiguous()
                out = x if owned else x.new_empty((len(x), output_width))
                for row in range(0, len(x), TAIL_ROWS):
                    rows = slice(row, row + TAIL_ROWS)
                    out[rows] = orig.to_out[1](orig.to_out[0](soft[rows]))
            del soft
            target = out[video]
            for row in range(0, len(linear), TAIL_ROWS):
                rows = slice(row, row + TAIL_ROWS)
                target[rows] += attn.to_out_linear(linear[rows])
            del linear, target
            # The shared quantization entry keeps its source alive: here the last
            # linear tile, a view that would pin all of `linear` until the FF.
            Int8Linear._last.clear()
            return out
        out = orig.to_out[1](orig.to_out[0](soft))
        del soft
        out[video] += attn.to_out_linear(linear)
        return out

    for attn in iter_hybrids(transformer):
        if attn.head_dim != attn.linear_attention.head_dim:
            raise ValueError('The shared-QKV profile requires matching branch head dimensions')
        attn._hybrid_forward = types.MethodType(forward, attn)