Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

4.4 KB
Raw Blame
"""Contiguous D2H head groups with double-buffered CPU projection tiles."""
import collections
import torch

# Every refused locked-page copy of this process, residual stream and head
# readouts alike, so a report shows whether the fallback fired, which copy was
# refused and what CUDA said. The newest entries are kept; the count is exact.
_REFUSED = collections.deque(maxlen=32)
_refused_total = 0


def record_refusal(source, copy, error):
    global _refused_total
    lines = str(error).strip().splitlines()
    _REFUSED.append(dict(source=source, copy=copy, error=(lines[0] if lines else '')[:200]))
    _refused_total += 1


def refusals_mark():
    return _refused_total


def refusals_since(mark):
    """Refusals recorded after `mark`, as plain dicts for the sampling metrics."""
    count = min(_refused_total - mark, len(_REFUSED))
    return [dict(row) for row in list(_REFUSED)[len(_REFUSED) - count:]] if count > 0 else []


class LockedCopies:
    """Asynchronous copies through page-locked buffers that continue in pageable memory.

    Windows can refuse a queued copy with "invalid argument" when the system
    takes locked pages back. The queued work is finished first, so a sticky
    error is raised as itself rather than hidden by the retry. The copy then
    goes through a pageable temporary, which holds the same bytes, and the
    remaining copies of these buffers skip the locked path. A failed device
    allocation is not a refusal: it is raised for the caller to recover from,
    and the buffers stay locked.
    """

    def __init__(self):
        self.locked = True
        self.fallbacks = 0

    def _leave(self, device, copy, error):
        record_refusal('readout', copy, error)
        if device.type == 'cuda':
            torch.cuda.synchronize(device)
        self.locked = False
        self.fallbacks += 1

    def to_host(self, destination, source):
        if self.locked:
            try:
                destination.copy_(source, non_blocking=True)
                return
            except torch.cuda.OutOfMemoryError:
                raise
            except RuntimeError as error:
                self._leave(source.device, 'to_host', error)
        destination.copy_(source.to('cpu'))

    def to_device(self, source, device):
        if self.locked:
            try:
                return source.to(device, non_blocking=True)
            except torch.cuda.OutOfMemoryError:
                raise
            except RuntimeError as error:
                self._leave(torch.device(device), 'to_device', error)
        return source.clone().to(device)


class HeadReadouts:
    def __init__(self, rows, head_widths, dtype, tile_rows):
        self.capacity = (rows + 1023) // 1024 * 1024
        self.widths = tuple(head_widths)
        self.dtype = dtype
        self.tile_rows = tile_rows
        self.groups = [torch.empty((self.capacity, width), dtype=dtype, pin_memory=True) for width in head_widths]
        self.tiles = [torch.empty((tile_rows, sum(head_widths)), dtype=dtype, pin_memory=True) for _ in range(2)]
        self.events = [None, None]
        self.copies = LockedCopies()

    def supports(self, rows, head_widths, dtype, tile_rows):
        return rows <= self.capacity and tuple(head_widths) == self.widths and dtype == self.dtype and tile_rows == self.tile_rows

    def store(self, group, values):
        if values.shape[1] != self.widths[group] or not values.is_contiguous():
            raise ValueError('Head readouts must be contiguous [rows, head channels]')
        self.copies.to_host(self.groups[group][:len(values)], values)

    def project(self, module, output, rows):
        # The caller synchronizes once after all D2H groups have been written.
        stream = torch.cuda.current_stream(output.device)
        for index, start in enumerate(range(0, rows, self.tile_rows)):
            slot = index % 2
            if self.events[slot] is not None:
                self.events[slot].synchronize()
            count = min(self.tile_rows, rows - start)
            tile = self.tiles[slot][:count]
            offset = 0
            for group, width in zip(self.groups, self.widths):
                tile[:, offset:offset + width].copy_(group[start:start + count])
                offset += width
            device_values = self.copies.to_device(tile, output.device)
            done = torch.cuda.Event()
            done.record(stream)
            self.events[slot] = done
            yield slice(start, start + count), module(device_values)