Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

11.8 KB
Raw Blame
"""Shape-selected projection and token slices for the native Mac encoder.

No allocator queries or precision changes. The 3 GiB pool's explicit tensor
ledger reserves <1 GiB for surrounding H3 state, 256 MiB for MLP activations,
128 MiB for a projection (including its assembled output), and 256 MiB for
attention. These stages do not all overlap. The remaining space is headroom
for Metal; it is not a promise about undocumented kernel/driver allocations.
"""
import json
import math
import time
import types

FULL_LINEAR_MAX_BYTES = 128 * 2**20
LINEAR_WORKING_BYTES = 128 * 2**20
MLP_WORKING_BYTES = 256 * 2**20
# The limits above were sized for the 3 GiB trial pool. A larger live pool
# scales them by the same ratio; a smaller one keeps them as they are.
REFERENCE_POOL_BYTES = 3 * 2**30
# Output rows of one slice never drop below this, however large the assembled
# output is: one row per slice means one decode, fence and cache release each.
MIN_SLICE_ROWS = 256


def pool_scale(pool_bytes):
    if pool_bytes is None:
        return 1
    if type(pool_bytes) is not int or pool_bytes < 1:
        raise ValueError('Expected a positive encoder pool size')
    return max(1, pool_bytes // REFERENCE_POOL_BYTES)


def linear_plan(tokens, inputs, outputs, itemsize=4, *, packed_columns=None, packed_rows=None, pool=1):
    """Bound full output + decoded slab + packed/scales + slab result + bias.

    Packed geometry can include padding on the input axis. A result whose full
    output alone exceeds the budget still slices at least MIN_SLICE_ROWS output
    rows (or all outputs, if fewer) and reports the over-budget estimate in
    slice_working_bytes; minimum_slice_bytes is the theoretical one-row bound.
    """
    if min(tokens, inputs, outputs, itemsize) < 1:
        raise ValueError('Expected positive linear dimensions')
    columns = inputs if packed_columns is None else packed_columns
    if columns < inputs:
        raise ValueError('Packed columns cannot be smaller than the input')
    output = tokens * outputs * itemsize
    weight = (outputs if packed_rows is None else packed_rows) * columns * itemsize
    # NVFP4 upload: 1/2 byte per value + FP32 scale per 16 values.
    upload = 0 if packed_columns is None else columns // 2 + columns // 16 * 4
    per_row = columns * itemsize + upload + (tokens + 1) * itemsize
    scale = 4 if packed_columns is not None else 0
    floor = min(outputs, MIN_SLICE_ROWS)
    rows = min(outputs, max(floor, (LINEAR_WORKING_BYTES * pool - output - scale) // per_row))
    full = weight + output <= FULL_LINEAR_MAX_BYTES * pool
    return dict(tokens=tokens, inputs=inputs, outputs=outputs, itemsize=itemsize,
                decoded_weight_bytes=weight, output_bytes=output,
                full_weight_output_bytes=weight + output,
                mode='full' if full else 'sliced', output_rows=outputs if full else rows,
                slice_working_bytes=None if full else output + rows * per_row + scale,
                minimum_slice_bytes=output + per_row + scale)


def mlp_plan(tokens, hidden, intermediate, itemsize=4, pool=1):
    """Three simultaneous intermediate arrays plus two hidden-sized row arrays.

    The full assembled MLP output stays live across chunks. Linear workspace is
    counted separately, conservatively counting some output storage twice.
    """
    if min(tokens, hidden, intermediate, itemsize) < 1:
        raise ValueError('Expected positive MLP dimensions')
    output = tokens * hidden * itemsize
    per_row = (3 * intermediate + 2 * hidden) * itemsize
    budget = MLP_WORKING_BYTES * pool
    full = tokens * per_row <= budget
    rows = min(tokens, max(1, (budget - output) // per_row))
    return dict(tokens=tokens, hidden=hidden, intermediate=intermediate, itemsize=itemsize,
                mode='full' if full else 'sliced', token_rows=tokens if full else rows,
                full_activation_bytes=tokens * per_row,
                slice_activation_bytes=None if full else output + rows * per_row,
                minimum_slice_bytes=output + per_row)


def sliced_linear(input, outputs, rows, load_weight, bias=None, *, synchronize=lambda: None,
                  release=lambda: None):
    """Each output dot product uses ALL input channels; never sum partial dots."""
    import torch
    if torch.is_grad_enabled():
        raise RuntimeError('Bounded encoder linear requires inference without gradients')
    if rows < 1:
        raise ValueError('Expected positive output row slice')
    output = input.new_empty((*input.shape[:-1], outputs))
    for start in range(0, outputs, rows):
        stop = min(outputs, start + rows)
        weight = part = local_bias = None
        try:
            weight = load_weight(start, stop)
            if bias is not None:
                local_bias = bias[start:stop].to(device=input.device, dtype=input.dtype)
            part = torch.nn.functional.linear(input, weight, local_bias)
            output[..., start:stop].copy_(part)
            synchronize()
        finally:
            del weight, part, local_bias
            release()
    return output


def sliced_mlp(input, rows, original, *, synchronize=lambda: None, release=lambda: None):
    """Keep the native MLP/AWQ expression and dtype for each token row slice."""
    import torch
    if torch.is_grad_enabled():
        raise RuntimeError('Bounded encoder MLP requires inference without gradients')
    if input.ndim != 3 or input.shape[0] != 1 or rows < 1:
        raise ValueError('Expected batch-one H3 MLP input and positive token rows')
    output = torch.empty_like(input)
    for start in range(0, input.shape[1], rows):
        part = None
        try:
            part = original(input[:, start:start + rows])
            output[:, start:start + rows].copy_(part)
            synchronize()
        finally:
            del part
            release()
    return output


class _Restore:
    """Also restore an active override if execution aborts before its post hook."""
    def __init__(self, module, attribute):
        self.module, self.attribute = module, attribute
        self.original = module.__dict__.get(attribute)

    def remove(self):
        if self.original is None:
            self.module.__dict__.pop(self.attribute, None)
        else:
            setattr(self.module, self.attribute, self.original)


def install(linears, layers, *, decoder, quantized_type, backend, guard, release, reader_stats,
            pool_bytes=None):
    """Select before the native forward looks up its projection/MLP callable.

    Below the thresholds do not assign anything: the fourth-version weight
    reader and vendor MLP function objects remain intact. MLP selection occurs
    at the parent layer because Module reads its own forward before pre-hooks.
    """
    pool = pool_scale(pool_bytes)
    stats = dict(pool_scale=pool, linear_sliced_calls=0, linear_slices=0, mlp_sliced_calls=0, mlp_slices=0,
                 plans=[], full_linear_max_bytes=FULL_LINEAR_MAX_BYTES * pool,
                 linear_working_bytes=LINEAR_WORKING_BYTES * pool,
                 mlp_working_bytes=MLP_WORKING_BYTES * pool)
    handles, seen = [], set()

    def record(kind, selected):
        key = (kind, tuple(selected.values()))
        if key not in seen:
            seen.add(key)
            stats['plans'].append(dict(kind=kind, **selected))
            print(json.dumps(dict(event='encoder_' + kind + '_slices', **selected)), flush=True)

    def install_linear(module):
        restore = _Restore(module, 'forward_comfy_cast_weights')
        active = False

        def before(module, args, kwargs):
            nonlocal active
            x = args[0] if args else kwargs['input']
            quantized = isinstance(module.weight, quantized_type)
            columns = module.weight._qdata.shape[1] * 2 if quantized else None
            selected = linear_plan(math.prod(x.shape[:-1]), module.in_features,
                                   module.out_features, x.element_size(), packed_columns=columns,
                                   packed_rows=module.weight._qdata.shape[0] if quantized else None,
                                   pool=pool)
            record('linear', selected)
            if selected['mode'] == 'full':
                return None

            def linear(module, input, *args, **kwargs):
                import torch
                if input.device.type != 'mps' or input.dtype != torch.float32:
                    raise ValueError('Native H3 encoder must execute FP32 operations on MPS')
                guard()
                source = module.weight
                if quantized and module.quant_format != 'nvfp4':
                    raise NotImplementedError('Unsupported MPS encoder projection storage: ' + module.quant_format)

                def load(start, stop):
                    tick = time.monotonic()
                    weight = (decoder(source, input.dtype, input.device, row_range=(start, stop))
                              if quantized else source[start:stop].to(device=input.device, dtype=input.dtype))
                    reader_stats['weight_decode_seconds'] += time.monotonic() - tick
                    reader_stats['largest_weight_bytes'] = max(reader_stats['largest_weight_bytes'],
                        weight.untyped_storage().nbytes())
                    stats['linear_slices'] += 1
                    return weight

                def cleanup():
                    backend.empty_cache()
                    release()

                stats['linear_sliced_calls'] += 1
                result = sliced_linear(input, module.out_features, selected['output_rows'], load,
                                       module.bias, synchronize=backend.synchronize, release=cleanup)
                reader_stats['linear_calls'] += 1
                return result

            module.forward_comfy_cast_weights = types.MethodType(linear, module)
            active = True

        def after(module, args, kwargs, output):
            nonlocal active
            if active:
                restore.remove()
                active = False

        handles.append(restore)
        handles.append(module.register_forward_pre_hook(before, with_kwargs=True))
        handles.append(module.register_forward_hook(after, with_kwargs=True, always_call=True))

    def install_mlp(layer):
        mlp = layer.mlp
        if getattr(mlp, 'merged_mlp', False):
            raise ValueError('Expected native H3 separate gate/up projections')
        original = mlp.forward
        restore = _Restore(mlp, 'forward')
        active = False

        def before(module, args, kwargs):
            nonlocal active
            x = args[0] if args else kwargs['x']
            selected = mlp_plan(math.prod(x.shape[:-1]), mlp.down_proj.out_features,
                                mlp.down_proj.in_features, x.element_size(), pool)
            record('mlp', selected)
            if selected['mode'] == 'full':
                return None

            def forward(module, input):
                stats['mlp_sliced_calls'] += 1
                stats['mlp_slices'] += (input.shape[1] + selected['token_rows'] - 1) // selected['token_rows']
                return sliced_mlp(input, selected['token_rows'], original,
                                  synchronize=backend.synchronize, release=backend.empty_cache)

            mlp.forward = types.MethodType(forward, mlp)
            active = True

        def after(module, args, kwargs, output):
            nonlocal active
            if active:
                restore.remove()
                active = False

        handles.append(restore)
        handles.append(layer.register_forward_pre_hook(before, with_kwargs=True))
        handles.append(layer.register_forward_hook(after, with_kwargs=True, always_call=True))

    try:
        for module in linears:
            install_linear(module)
        for layer in layers:
            install_mlp(layer)
    except BaseException:
        for handle in handles:
            handle.remove()
        raise
    return stats, handles