Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

12.6 KB
Raw Blame
"""React inside a sampling step when Windows cuts this process's video-memory budget.

Windows moves a process's video memory to system memory while the process
stays over the budget the driver gives it, and every layer then waits for that
memory to come back. The budget was only read between steps, so a cut that
arrived inside a step slowed the whole step several times over: one step took
ten times as long as its neighbours.

Between layers this reads the budget and how long the layer just finished
took. When the pass is over its budget and held up, it first releases what the
pass does not need (idle models), then halves the attention head group for the
layers that follow. Between steps the planned group returns, once per pass,
when every read of a whole step left room for what the pass held with it.

While the budget covers the pass nothing changes: the hooks read two counters
and a clock, no tensor is written and the installed forwards stay as they
were, so outputs are bit-identical. A pass that was narrowed computes its
remaining layers with the narrower group, which rounds differently: the same
seed then gives a slightly different video, as it does on a card whose plan
takes the narrower group from the start.
"""
from contextlib import ExitStack, contextmanager
import gc
import json
import time

MiB, GiB = 1 << 20, 1 << 30
# Readings within this of the budget are not over it: the driver's counters
# and the allocator's pool move by a few MiB between two reads.
OVER_BYTES = 32 * MiB
# A layer is held up when it took this many times as long as the same layer of
# the previous step, and at least this much longer. Layers read from a slow
# drive are slow in every step and never count.
SLOW_FACTOR, SLOW_SECONDS = 3., 5.
# The first step of a pass has no earlier step to compare with. There a layer
# counts when the pass is over its budget by this much and the layer took at
# least SLOW_SECONDS: a layer of a fitting pass takes a fraction of that.
CUT_BYTES = GiB
# Time spent in held-up layers before anything is done, and again before the
# next measure: a budget that dips for a few seconds comes back by itself.
SUSTAIN_SECONDS = 15.
# The planned group returns when the budget stayed this far above what the
# pass used with it, at every read of one whole step.
RESTORE_BYTES = 256 * MiB
# Narrower groups than this were measured several times slower with the
# attention outputs in host memory, and save little.
MIN_HEAD = 4


def emit_event(event):
    print(json.dumps(event), flush=True)


class LayerWatch:
    """One pass's reaction to a budget cut. `source` supplies the readings:

    local()   -> {'budget_bytes', 'usage_bytes'} for this process, or None
    release() -> bytes freed by dropping idle models (may be 0)
    limit()   -> the allocator's current ceiling in bytes, or None
    """
    def __init__(self, engine, source, *, clock=time.monotonic, emit=emit_event, pass_cache=None):
        self.engine, self.source, self.clock, self.emit = engine, source, clock, emit
        self.pass_cache = pass_cache   # the pass's cache record, shared with the sampler
        self.planned = engine.config.get('head_chunk')
        self.stack = None          # open while a narrower group is installed
        self.previous = {}         # layer -> seconds it took in the last completed step
        self.current = {}          # layer -> seconds in the running step
        self.entered = None        # (layer, time) of the layer being dispatched
        self.pressed = 0.          # seconds of held-up layers since the last measure
        self.released = False
        self.exhausted = False
        self.wide_usage = 0        # most this process held with the planned group
        self.wide_peak = 0         # allocator peaks with the planned group, kept across the reset
        self.wide_cache = 0        # largest cache the pass held before its first narrowing
        self.boundary = None       # (lowest budget read, a reading failed) of the step just ended
        self.low_budget = None     # lowest budget read in the running step
        self.unread_step = False   # a reading of the running step failed
        self.pass_allocated = 0    # allocator peaks of the whole pass, kept across every reset
        self.pass_reserved = 0
        self.reset_workspace = False
        self.restored = False      # the planned group came back once in this pass
        self.step = 1
        self.receipt = dict(planned_head_chunk=self.planned, polls=0, unread=0, over_polls=0,
                            held_up_layers=0, actions=[])

    def read(self):
        try:
            sample = self.source.local()
        except (OSError, RuntimeError, AttributeError, ValueError):
            sample = None
        if (not isinstance(sample, dict) or type(sample.get('budget_bytes')) is not int
                or type(sample.get('usage_bytes')) is not int or sample['budget_bytes'] <= 0):
            self.receipt['unread'] += 1
            return None
        self.receipt['polls'] += 1
        return sample

    def enter(self, index):
        """Before layer `index` (1-based) is dispatched."""
        now = self.clock()
        took = None
        if self.entered is not None and self.entered[0] == index - 1:
            took = now - self.entered[1]
            self.current[index - 1] = took
        self.entered = (index, now)
        sample = self.read()
        if sample is None:
            self.pressed = 0.
            self.unread_step = True
            return
        budget, usage = sample['budget_bytes'], sample['usage_bytes']
        self.low_budget = budget if self.low_budget is None else min(self.low_budget, budget)
        if self.stack is None:
            self.wide_usage = max(self.wide_usage, usage)
        over = usage - budget
        if over >= OVER_BYTES:
            self.receipt['over_polls'] += 1
        if over < OVER_BYTES or took is None:
            self.pressed = 0.
            return
        before = self.previous.get(index - 1)
        if before is not None:
            held_up = took >= max(SLOW_FACTOR * before, before + SLOW_SECONDS)
        else:
            held_up = over >= CUT_BYTES and took >= SLOW_SECONDS
        if not held_up:
            self.pressed = 0.
            return
        self.receipt['held_up_layers'] += 1
        self.pressed += took
        if self.pressed >= SUSTAIN_SECONDS:
            self.pressed = 0.
            self.act(index, sample, took)

    def record(self, action, index, sample, **fields):
        row = dict(action=action, step=self.step, layer=index,
                   budget_bytes=sample['budget_bytes'] if sample else None,
                   usage_bytes=sample['usage_bytes'] if sample else None, **fields)
        if len(self.receipt['actions']) < 32:
            self.receipt['actions'].append(row)
        self.emit(dict(event='step_pressure', **row))

    def act(self, index, sample, took):
        if not self.released:
            self.released = True
            try:
                freed = int(self.source.release() or 0)
            except (OSError, RuntimeError, AttributeError, ValueError):
                freed = 0
            if freed > 0:
                self.record('release_idle', index, sample, freed_bytes=freed, layer_seconds=took)
                return
        self.narrow(index, sample, took)

    def narrow(self, index, sample, took):
        config = self.engine.config
        head = config.get('head_chunk')
        if (not config.get('inference_kernels') or type(head) is not int or head // 2 < MIN_HEAD):
            if not self.exhausted:
                self.exhausted = True
                self.record('nothing_left', index, sample, head_chunk=head, layer_seconds=took)
            return
        from .pass_configuration import configuration
        from . import head_chunk
        backend = self.engine.device_backend
        reserved, allocated = backend.max_memory_reserved(), backend.max_memory_allocated()
        if self.stack is None:
            self.stack = ExitStack()
            self.wide_peak = max(self.wide_peak, reserved)
            # The cache of later steps is not part of that peak: remember what
            # the peak held, so the planned group's own workspace is not read
            # as the peak minus a cache that grew beside the narrower group.
            held = (self.pass_cache or {}).get('peak_cache_bytes', 0)
            self.wide_cache = max(self.wide_cache, min(held, reserved))
        # Every reset below forgets what the group before it reached.
        self.pass_reserved = max(self.pass_reserved, reserved)
        self.pass_allocated = max(self.pass_allocated, allocated)
        self.stack.enter_context(configuration(self.engine, dict(head_chunk=head // 2)))
        # The narrower group's first call compiles: collect after each of its
        # groups again, or the frames that compile leaves behind stay beside
        # the next group exactly when this pass must hold less.
        head_chunk.COLLECTED_LENGTHS.clear()
        gc.collect()
        backend.empty_cache()
        # The next step boundary asks for the budget to cover the pass as it now runs.
        backend.reset_peak_memory_stats()
        self.reset_workspace = True
        # Times of the wider group no longer say what a layer should take.
        self.previous, self.current = {}, {}
        self.record('narrow', index, sample, head_chunk_from=head, head_chunk_to=head // 2,
                    layer_seconds=took)

    def restore(self, sample=None):
        stack, self.stack = self.stack, None
        if stack is not None:
            self.restored = True
            stack.close()
            self.previous, self.current = {}, {}
            self.record('restore', 0, sample, head_chunk_to=self.engine.config.get('head_chunk'))

    def step_done(self, pass_cache):
        """At a step boundary, before the budget refresh reads the pass's workspace."""
        self.previous, self.current, self.entered, self.pressed = self.current, {}, None, 0.
        self.step += 1
        self.pass_cache = pass_cache
        if self.reset_workspace:
            # Measured again from the allocator's peak since the group narrowed.
            self.reset_workspace = False
            pass_cache['workspace_peak_bytes'] = 0
        low, self.low_budget = self.low_budget, None
        # A step with a failed reading did not show room at every read.
        unread, self.unread_step = self.unread_step, False
        self.boundary = (low, unread)

    def restore_if_room(self, limit):
        """After the boundary's budget refresh: the planned group returns when the
        step's readings and the refreshed ceiling `limit` both leave room for it.

        Returns the planned group's workspace in bytes when it came back, else
        None. The caller then holds the pass cache to what that workspace leaves.
        """
        boundary, self.boundary = self.boundary, None
        if boundary is None:
            return None
        low, unread = boundary
        # Once per pass: a budget that comes back and is cut again would
        # otherwise switch groups at every boundary, recompiling each time.
        if self.stack is None or low is None or unread or self.restored:
            return None
        if low < self.wide_usage + RESTORE_BYTES:
            return None
        # The ceiling the refresh just set: a cut after the last layer's read
        # is already in it.
        if type(limit) is not int:
            try:
                limit = self.source.limit()
            except (OSError, RuntimeError, AttributeError, ValueError):
                limit = None
        if type(limit) is not int or limit < self.wide_peak:
            return None
        self.restore(dict(budget_bytes=low, usage_bytes=self.wide_usage))
        return max(0, self.wide_peak - self.wide_cache)

    @contextmanager
    def layers(self, blocks):
        handles = []
        try:
            for index, block in enumerate(blocks, 1):
                def entered(module, inputs, index=index):
                    self.enter(index)
                handles.append(block.register_forward_pre_hook(entered))
            yield self
        finally:
            for handle in handles:
                handle.remove()
            stack, self.stack = self.stack, None
            self.receipt['narrowed_at_end'] = stack is not None
            self.receipt['head_chunk_at_end'] = self.engine.config.get('head_chunk')
            if stack is not None:
                stack.close()

    def peaks(self, allocated, reserved):
        """The pass's allocator peaks, including the part before a narrowing reset them."""
        return max(allocated, self.pass_allocated), max(reserved, self.pass_reserved)

    def result(self):
        return dict(self.receipt, wide_peak_reserved_bytes=self.wide_peak or None)