Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

51.1 KB
Raw Blame
"""Sequential module offloading with bounded pinned-host and CUDA buffers."""
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from contextlib import contextmanager, nullcontext
import json
import os
import re
import threading
import time
import weakref

import torch


def cpu_weights(layer):
    """Frozen parameters and CPU buffers (including official FP8 weights/scales).

    Precomputed modulation tables already resident on CUDA are left in place.
    Only immutable buffers may be included in an offloaded layer.
    """
    for name, value in layer.named_parameters():
        if value.device.type != 'cpu' or value.requires_grad:
            raise ValueError('Offloaded parameters must be frozen CPU tensors')
        yield name, value
    for name, value in layer.named_buffers():
        if value.device.type == 'cpu':
            if value.requires_grad:
                raise ValueError('Offloaded buffers must be frozen')
            yield name, value


def _pin_reservation(sizes):
    """Charge the host allocator's configured rounding, not only tensor bytes."""
    settings = os.environ.get('PYTORCH_ALLOC_CONF', os.environ.get('PYTORCH_CUDA_ALLOC_CONF', ''))
    threshold = float('inf')
    for option in ('pinned_max_round_threshold_mb', 'pinned_max_cached_size_mb'):
        match = re.search(r'(?:^|,)\s*' + option + r':\s*(\d+)(?:,|$)', settings)
        if match:
            threshold = min(threshold, int(match[1]) * 2**20)
    return sum((1 << (size - 1).bit_length()) if 0 < size <= threshold else size for size in sizes)


def live_pin_capacity(*, headroom_bytes=None, nonlocal_reserve_bytes=None, commit_reserve_bytes=0):
    """Bytes that page-locking may take now, beyond the working headroom."""
    # Charge free physical memory only. Effective availability credits this
    # tree's reclaimable mapped pages, which is right for measuring pressure
    # but wrong for deciding how much to lock: locked pages cannot be
    # reclaimed, so space that only exists after reclaim cannot back them.
    # Crediting them raised the cap to about 15 GiB and cudaHostAlloc failed
    # with cudaErrorMemoryAllocation during load. On Windows, pinned pages
    # also consume system commit, so physical headroom alone can be wildly
    # optimistic when the pagefile is nearly full. Cap against both counters;
    # Linux reports no separate commit budget here and remains unchanged.
    #
    # The planned budget came from availability measured before text encoding
    # ran, and a plan made at 22.70 GiB free pinned 10.88 GiB and left 1.73 GiB,
    # under the emergency floor, because encoding had taken the host to
    # 2.77 GiB. Cap against live availability so whole groups drop out and keep
    # their mapped storage instead of failing the request.
    from .system import system_memory, HOST_WEIGHT_HEADROOM
    if headroom_bytes is None:
        headroom_bytes = HOST_WEIGHT_HEADROOM
    if type(headroom_bytes) is not int or headroom_bytes < 2 * 2**30:
        raise ValueError('Pinning must preserve at least 2 GiB of host working space')
    memory = system_memory()
    available = memory.get('physical_available_bytes') or memory['available_bytes']
    commit_available = memory.get('commit_available_bytes')
    if commit_available is not None:
        # WDDM charges this process's GPU allocations to system commit as they
        # are made, so allocator room the request has not reserved yet cannot
        # back locked pages either. Pinning against commit alone left an 8 GiB
        # card's request with under 1 GiB of commit once sampling reserved its
        # workspace, and live pressure then switched off host read-ahead and
        # the step cache for the rest of the request. The caller passes that
        # unreserved allocator room; physical RAM is not charged for it.
        available = min(available, commit_available - commit_reserve_bytes)
    cap = available - headroom_bytes
    from .system import windows
    if windows():
        # Free RAM does not bound locked pages on Windows: the WDDM non-local
        # budget does. A 3.5 GiB working allowance on a 27.98 GiB machine pinned
        # past it on the second request, and every retry failed the same way.
        # The reserve covers locked buffers allocated after the weights; the
        # caller adds its request's CPU attention readouts to the default.
        from .windows_gpu_memory import nonlocal_pin_capacity, NONLOCAL_PIN_RESERVE
        reserve = NONLOCAL_PIN_RESERVE if nonlocal_reserve_bytes is None else nonlocal_reserve_bytes
        capacity = nonlocal_pin_capacity(reserve)
        if capacity is None and memory.get('total_bytes'):
            # Without a DXGI reading, assume the Windows default of half of RAM.
            try:
                pinned = torch.cuda.memory.host_memory_stats()['allocated_bytes.current']
            except (RuntimeError, KeyError, AttributeError):
                pinned = 0
            capacity = max(0, memory['total_bytes'] // 2 - pinned - reserve)
        if capacity is not None:
            cap = min(cap, capacity)
    return cap


@torch.no_grad()
def pin_layer_weights(layers, max_bytes=None, *, headroom_bytes=None, nonlocal_reserve_bytes=None,
                      commit_reserve_bytes=0, progress=None):
    """Pack immutable CPU parameters into persistent pinned planes, one per dtype.

    This optional mode trades pageable/mapped storage for locked host residency.
    Rebinding preserves parameter values and permits direct asynchronous H2D.
    """
    if type(commit_reserve_bytes) is not int or commit_reserve_bytes < 0:
        raise ValueError('Commit reserve must be a nonnegative byte count')
    layers = list(layers)
    layer_groups = []
    for layer in layers:
        by_dtype = {}
        for name, parameter in cpu_weights(layer):
            if not isinstance(parameter, torch.nn.Parameter) and parameter._base is not None:
                # safetensors buffers are views. Reassigning their .data alone
                # keeps the old view base (and mapped FP8 file) alive alongside
                # the pinned copy. Detach the registered buffer first; this
                # shares the bytes without retaining view metadata.
                parent, _, leaf = name.rpartition('.')
                owner = layer.get_submodule(parent) if parent else layer
                parameter = parameter.detach()
                setattr(owner, leaf, parameter)
            by_dtype.setdefault(parameter.dtype, []).append(parameter)
        layer_groups.append(list(by_dtype.items()))

    cap = live_pin_capacity(headroom_bytes=headroom_bytes, nonlocal_reserve_bytes=nonlocal_reserve_bytes,
                            commit_reserve_bytes=commit_reserve_bytes)
    max_bytes = cap if max_bytes is None else min(max_bytes, cap)

    groups, sizes, reserved = [], [], 0
    for entries in layer_groups:
        current_sizes = [sum(p.numel() * p.element_size() for p in parameters) for _, parameters in entries]
        current_reserve = _pin_reservation(current_sizes)
        if reserved + current_reserve > max_bytes:
            continue
        groups.extend(entries)
        sizes.extend(current_sizes)
        reserved += current_reserve
    required = sum(sizes)
    # Charge the container's non-reclaimable working set, not clean file cache:
    # mmap-backed weights are replaced group by group while pinning, and their
    # clean pages can be reclaimed. Leave a separate 2 GiB process allowance.
    from pathlib import Path
    cgroup = Path('/sys/fs/cgroup')
    if (cgroup / 'memory.max').is_file():
        limit = (cgroup / 'memory.max').read_text(encoding='utf-8').strip()
        if limit != 'max':
            stat = {key: int(value) for key, value in (line.split() for line in (cgroup / 'memory.stat').read_text(encoding='utf-8').splitlines())}
            clean_file = max(0, stat.get('file', 0) - sum(stat.get(k, 0) for k in ('shmem', 'file_dirty', 'file_writeback', 'unevictable')))
            available_cgroup = int(limit) - int((cgroup / 'memory.current').read_text(encoding='utf-8')) + clean_file
            if available_cgroup < reserved + 2 * 2**30:
                raise MemoryError('Pinned weights exceed cgroup working-set allowance; select a smaller pinned budget')
    pinned = 0
    for index, (dtype, parameters) in enumerate(groups):
        try:
            plane = torch.empty(sum(p.numel() for p in parameters), dtype=dtype, pin_memory=True)
        except (RuntimeError, MemoryError) as error:
            # Pinning is an optimization, so a host allocator that stops short
            # must cost transfers, not the request. Locked host memory has a
            # platform ceiling independent of free RAM: this machine refused
            # every attempt past 12.60 GiB with 25.1, 10.9 and 9.9 GiB free,
            # while the plan asked for 15.05 GiB because the budget scales with
            # available RAM. Whatever is not pinned keeps its mapped storage.
            print(json.dumps({'event': 'host_pinning_truncated', 'pinned_bytes': pinned,
                              'planned_bytes': required, 'groups_pinned': index,
                              'groups_planned': len(groups),
                              'reason': str(error).splitlines()[0][:200]}), flush=True)
            return pinned
        offset = 0
        for parameter in parameters:
            view = plane[offset:offset + parameter.numel()].view(parameter.shape)
            view.copy_(parameter)
            offset += parameter.numel()
            parameter.data = view
        pinned += sum(p.numel() * p.element_size() for p in parameters)
        if progress is not None:
            progress(pinned, required)
    return pinned


@dataclass
class Slot:
    host: dict
    device: dict
    copy_done: object = None
    compute_done: object = None


@dataclass
class _HostRead:
    """One layer being prepared ahead in a staging block."""

    slot: Slot
    future: object
    from_drive: bool  # Not held in RAM as a view when it was queued.
    lane: str = None  # 'drive' or 'ram' once the two are read ahead separately.


# Live RAM that must stay free beside the one prepared block host read-ahead
# holds. It used to borrow the 8 GiB weight-cache allowance, which no 16 GiB
# machine reaches while sampling: a 16 GiB RTX 3060 Laptop reread all 21.6 GB
# of weights every step (about 80 s) with reads and compute never overlapping.
# 3 GiB still kept 32 GiB machines busy with other applications (2-3.5 GiB
# available while sampling) from overlapping. The read-ahead adds one block of
# locked pages, and the reads happen either way. With 41 GiB held by another
# process on a 56 GiB RTX 3090 machine (about 2 GiB available while sampling),
# 1344x768x124 went from 15.5-15.9 to 10.7-10.9 s per first-pass step and
# 29.8-30.0 to 26.0-26.1 per second-pass step at 1 GiB, twice each (327-332 s
# to 283-294 s for the request). The runtime check in _queue_host returns the
# extra block under live pressure; read-ahead then shares the staging buffer.
HOST_PREFETCH_HEADROOM = 1 * 2**30


# A read from the drive that keeps the inference thread waiting this long means
# the drive, not the GPU, sets the pace of those layers. From then on the next
# layer that comes from the drive is read into the extra block as soon as that
# block is free, past the layers RAM already holds, which keep going through the
# staging buffer. Until a read is that slow, read-ahead stays one layer deep on
# alternating blocks, as before.
HOST_FILE_READ_WAIT = 0.2


def _host_prefetch_fits(extra_bytes):
    """Speculative staging must fit both live physical RAM and Windows commit."""
    from .system import system_memory
    try:
        memory = system_memory()
        available = memory.get('physical_available_bytes')
        if available is None:
            available = memory['available_bytes']
        commit = memory.get('commit_available_bytes')
        if commit is not None:
            available = min(available, commit)
        return available >= HOST_PREFETCH_HEADROOM + extra_bytes
    except (OSError, ValueError, TypeError, KeyError):
        return False


@dataclass
class _StreamedLayout:
    """Metadata for reusing an unloaded layer, without owning any weight storage."""

    source: object
    index: int
    entries: tuple

    def shapes(self, source, index, weights):
        if self.source() is not source or source is None or self.index != index:
            raise ValueError('Unloaded streamed layers require their original checkpoint source and order')
        if len(weights) != len(self.entries):
            raise ValueError('Unloaded streamed layer tensors changed; reload the model')
        shapes = {}
        for (name, parameter), (reference, saved_name, dtype, count, shape) in zip(weights, self.entries):
            if (reference() is not parameter or name != saved_name or parameter.dtype != dtype
                    or parameter.numel() != 0 or tuple(parameter.shape) != (0,)):
                raise ValueError('Unloaded streamed layer tensors changed; reload the model')
            shapes[name] = count, shape
        return shapes


@torch.no_grad()
def initialize_streamed_layer(layer, source, index):
    """Bind an unmaterialized layer to a checkpoint without reading its weights."""
    tensors = ([(name, value, True) for name, value in layer.named_parameters()]
               + [(name, value, False) for name, value in layer.named_buffers()])
    if source is None or any(not value.is_meta for _, value, _ in tensors):
        raise ValueError('Streamed initialization requires a checkpoint and only meta tensors')
    entries = []
    for name, value, parameter in tensors:
        count, shape, dtype = value.numel(), tuple(value.shape), value.dtype
        empty = torch.empty(0, dtype=dtype, device='cpu')
        if parameter:
            empty = torch.nn.Parameter(empty, requires_grad=False)
        parent, _, leaf = name.rpartition('.')
        owner = layer.get_submodule(parent) if parent else layer
        setattr(owner, leaf, empty)
        entries.append((weakref.ref(empty), name, dtype, count, shape))
    layer._freevideo_streamed_layout = _StreamedLayout(weakref.ref(source), index, tuple(entries))


@torch.no_grad()
def unload_streamed_layer(layer, source, index):
    """Keep a newly loaded layer's layout without retaining its checkpoint bytes.

    Safetensors buffers are tensor views, including with the pread reader.
    Emptying their .data does not clear _base: it would retain the entire CPU
    tensor until sampling constructs an offloader. Detach registered buffers
    first, then record the new identities used to restore the exact layout.
    """
    entries = []
    for name, parameter in cpu_weights(layer):
        if not isinstance(parameter, torch.nn.Parameter) and parameter._base is not None:
            parent, _, leaf = name.rpartition('.')
            owner = layer.get_submodule(parent) if parent else layer
            parameter = parameter.detach()
            setattr(owner, leaf, parameter)
        entries.append((weakref.ref(parameter), name, parameter.dtype,
                        parameter.numel(), tuple(parameter.shape)))
        parameter.data = torch.empty(0, dtype=parameter.dtype)
    layer._freevideo_streamed_layout = _StreamedLayout(weakref.ref(source), index, tuple(entries))


@torch.no_grad()
def prepare_streamed_layer(layer, source, index, *, pin_budget_bytes, headroom_bytes, nonlocal_reserve_bytes=None,
                           commit_reserve_bytes=0):
    """Retain one affordable pinned layer or release it before loading the next.

    Return logical pinned bytes and the charged allocator reservation. A partial
    host-allocation failure keeps no half-pinned layer: the streaming offloader
    would discard it anyway, and it must not consume the next layer's budget.
    """
    sizes = {}
    for _, value in cpu_weights(layer):
        sizes[value.dtype] = sizes.get(value.dtype, 0) + value.numel() * value.element_size()
    reserved = _pin_reservation(sizes.values())
    pinned = 0
    if 0 < reserved <= pin_budget_bytes:
        pinned = pin_layer_weights([layer], max_bytes=pin_budget_bytes, headroom_bytes=headroom_bytes,
                                   nonlocal_reserve_bytes=nonlocal_reserve_bytes,
                                   commit_reserve_bytes=commit_reserve_bytes)
        if pinned == sum(sizes.values()) and all(value.is_pinned() for _, value in cpu_weights(layer)):
            return pinned, reserved
    unload_streamed_layer(layer, source, index)
    if pinned:
        from .torch_compat import empty_host_cache
        empty_host_cache(torch)
    return 0, 0


def loading_retains(source, index, *, pin_budget_bytes, **pin_options):
    """Whether preparing a streamed layer may keep what loading reads: a pin or a view.

    The checks of prepare_streamed_layer (planned budget, then the live cap of
    pin_layer_weights, with pin_options) and of SafetensorLayers.adopt, made on
    checkpoint header sizes, which equal the bound tensors, before the layer is
    read. pin_layer_weights measures with the layer already read into private
    memory, so the live cap here is charged its size as well.
    """
    sizes = source.layer_bytes(index)
    size = sum(sizes.values())
    reservation = _pin_reservation(sizes.values())
    if 0 < reservation <= pin_budget_bytes and reservation <= live_pin_capacity(**pin_options) - size:
        return True
    from .streamed_weights import host_view_fits
    return source.host_views is not None and host_view_fits(size, source.host_view_reserve)


class LayerOffloader:
    """Keep CPU weights and stream a sequential layer list through one or two slots.

    CUDA events protect both host-buffer reuse and device-buffer overwrite. No
    parameter is changed numerically; hooks rebind its storage only while it runs.
    This requires inference without autograd and one request at a time.

    Streamed layers retain zero-storage placeholders after close. Reuse them with
    the same checkpoint source and layer order; only in-memory sources are restored.
    """

    def __init__(self, layers, device='cuda', prefetch=True, manage_hooks=True, weight_source=None,
                 host_prefetch=None):
        if torch.is_grad_enabled():
            raise RuntimeError('LayerOffloader requires gradients disabled')
        self.layers = list(layers)
        self.device = torch.device(device)
        self.prefetch = prefetch
        self.weight_source = weight_source
        self.cpu = []
        self.layout = []
        sizes = {}
        for index, layer in enumerate(self.layers):
            weights, layout, offsets = {}, [], {}
            parameters = list(cpu_weights(layer))
            saved = getattr(layer, '_freevideo_streamed_layout', None)
            shapes = saved.shapes(weight_source, index, parameters) if saved is not None else {}
            for name, parameter in parameters:
                if weight_source is not None and not isinstance(parameter, torch.nn.Parameter) and parameter._base is not None:
                    parent, _, leaf = name.rpartition('.')
                    owner = layer.get_submodule(parent) if parent else layer
                    parameter = parameter.detach()
                    setattr(owner, leaf, parameter)
                dtype = parameter.dtype
                offset = offsets.get(dtype, 0)
                count, shape = shapes.get(name, (parameter.numel(), parameter.shape))
                weights[name] = parameter.detach()
                layout.append((parameter, name, dtype, offset, count, shape))
                offsets[dtype] = offset + count
            for dtype, count in offsets.items():
                sizes[dtype] = max(sizes.get(dtype, 0), count)
            self.cpu.append(weights)
            self.layout.append(layout)
            del parameters
        self.pinned_layers = [all(value.is_pinned() for value in weights.values()) for weights in self.cpu]
        if weight_source is not None:
            # Keep module objects and their metadata, but no full-checkpoint
            # tensor views. The pre-hook restores exact shapes before a layer
            # executes. Offloaded placeholders own zero bytes between calls.
            for index, (layer, pinned, weights, layout) in enumerate(zip(
                    self.layers, self.pinned_layers, self.cpu, self.layout)):
                if pinned:
                    continue  # Keep a bounded pinned subset; reopen only the rest.
                layer._freevideo_streamed_layout = _StreamedLayout(weakref.ref(weight_source), index, tuple(
                    (weakref.ref(parameter), name, dtype, count, shape)
                    for parameter, name, dtype, _, count, shape in layout))
                for parameter, name, dtype, *_ in layout:
                    empty = torch.empty(0, dtype=dtype)
                    parameter.data = empty
                    weights[name] = empty
        self.pinned_sources = all(self.pinned_layers)
        # Staging buffers are an optimization over pageable copies, so a host
        # allocator that refuses them must cost transfer speed, not the
        # request. Weight pinning had already taken 14.51 GiB of locked memory
        # and left nothing here, failing load with cudaErrorMemoryAllocation.
        self.pinned_staging = True

        def staging(count, dtype):
            if self.pinned_sources:
                return None
            try:
                return torch.empty(count, dtype=dtype, pin_memory=True)
            except (RuntimeError, MemoryError):
                self.pinned_staging = False
                return torch.empty(count, dtype=dtype)

        self.slots = [Slot(
            host={} if self.pinned_sources else {dtype: staging(count, dtype) for dtype, count in sizes.items()},
            device={dtype: torch.empty(count, dtype=dtype, device=self.device) for dtype, count in sizes.items()},
        ) for _ in range(2 if prefetch else 1)]
        self.copy_stream = torch.cuda.Stream(device=self.device)
        self.executor = ThreadPoolExecutor(max_workers=1) if prefetch else None
        self.pending = {}
        self.transfers = []
        self.host_buffer_wait_seconds = 0.
        self.prefetch_wait_seconds = 0.
        self.direct_read_layers = 0
        self.direct_read_bytes = 0
        self.source_layers = 0
        self.reader_layers = 0
        # The weight source outlives this pass; report only this pass's view hits.
        self.host_view_hits_start = getattr(weight_source, 'host_view_hits', 0)
        self.host_view_hit_bytes_start = getattr(weight_source, 'host_view_hit_bytes', 0)
        self.direct_reads_start = dict(getattr(weight_source, 'read_stats', None) or {})
        self.hooks = []
        self.expected = 0
        self.closed = False
        self.cached = {}
        self.cache_candidates = set()
        self.cache_budget_bytes = 0
        self.cache_hits = 0
        self.read_ahead_before_cache = None
        self.read_ahead = (weight_source.read_ahead(self.layout, self.pinned_layers)
                           if weight_source is not None else None)
        # A second CUDA slot is not affordable on every card. With enough RAM,
        # prepare one future layer on the CPU while the current layer computes;
        # upload it only at the next normal boundary through the SAME GPU slot.
        # Keep the existing full prefetch path when two device slots are funded.
        from .system import windows
        requested = windows() if host_prefetch is None else host_prefetch
        self.host_prefetch = False
        # Memory pressure no longer switches read-ahead off (see the shared
        # reason below); reports keep this field for earlier versions.
        self.host_prefetch_disabled_reason = None
        self.host_prefetch_shared_reason = None
        self.host_slots = []
        self.host_executor = None
        self.host_pending = {}
        self.host_cursor = 0
        self.host_prefetch_reads = 0
        self.host_prefetch_wait_seconds = 0.
        self.host_memory_checked = 0.
        # Reads of drive-backed layers past the next streamed layer: their own
        # thread, and the extra block while it is not being handed back.
        self.host_file_ahead = bool(getattr(weight_source, 'reads_delay_compute', False))
        self.host_file_executor = None
        self.host_file_ahead_reads = 0
        self.host_extra_returning = False
        self.host_counters = threading.Lock()
        if (requested and not prefetch and sum(not p for p in self.pinned_layers) > 1
                and getattr(weight_source, 'direct_read', False)
                and getattr(weight_source, 'intermediate_dtype', None) is None
                and self.pinned_staging):
            # Without room for a second block of locked pages, read ahead into
            # the staging buffer every streamed layer already passes through:
            # the next read starts once the current layer's upload completes,
            # while that layer computes. Reading on the inference thread instead
            # waits until it has queued the current layer, and with a full
            # launch queue the GPU then idles through most of the read. With 18
            # of 50 layers reread from the file every step, each read about as
            # long as a layer's compute, 1344x768x362 steps went from 41.1 to
            # 32.0 s at head groups of 2 and from 34.5 to 27.6 s at 4 on an RTX
            # PRO 6000 (latents identical); the second block saves 0.4 s more.
            self.host_slots = [Slot(host=self.slots[0].host, device={})]
            extra = _pin_reservation([plane.numel() * plane.element_size()
                                      for plane in self.slots[0].host.values()])
            if _host_prefetch_fits(extra):
                host = {}
                try:
                    for dtype, count in sizes.items():
                        host[dtype] = torch.empty(count, dtype=dtype, pin_memory=True)
                except (RuntimeError, MemoryError):
                    host.clear()
                    self.host_prefetch_shared_reason = 'host_allocation_refused'
                    from .torch_compat import empty_host_cache
                    empty_host_cache(torch)
                else:
                    self.host_slots.append(Slot(host=host, device={}))
            else:
                self.host_prefetch_shared_reason = 'host_headroom'
            self.host_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='freevideo-host-stage')
            self.host_prefetch = True
        for index, layer in enumerate(self.layers if manage_hooks else []):
            self.hooks.append(layer.register_forward_pre_hook(
                lambda module, args, index=index: self._before(index)))
            self.hooks.append(layer.register_forward_hook(
                lambda module, args, result, index=index: self._after(index), always_call=True))

    @property
    def at_boundary(self):
        """No layer pass is open, so the next pass can start; False after a failure inside one."""
        return not (self.expected or self.pending or self.host_pending)

    def end_pass_early(self):
        """Finish a successful partial traversal after its last forward hook.

        Indices belong to this offloaded subset, not the transformer's blocks.
        Block zero may be resident, so an already idle traversal is valid too.
        Keep admitted weight caches and live reader pools for the next step.
        """
        if self.closed:
            raise RuntimeError('Cannot finish a closed offloader')
        # _stage may still be submitting copies or filling an admitted cache.
        # Join producers before waiting on the final events they recorded.
        for future in self.pending.values():
            future.result()
        for read in self.host_pending.values():
            if not read.future.cancel():
                read.future.result()
        # A skipped prefetch has copy_done but no new compute_done. Likewise,
        # the first block's compute may still read its slot after Python returns.
        # Never reset/reuse either plane until both operations have completed.
        for slot in [*self.slots, *self.cached.values(), *self.host_slots]:
            for event in (slot.copy_done, slot.compute_done):
                if event is not None:
                    event.synchronize()
        if self.read_ahead is not None:
            self.read_ahead.end_pass_early()
        self.pending.clear()
        self.host_pending.clear()
        self.host_cursor = 0
        self.expected = 0

    def cache_between_steps(self, max_bytes, *, max_layers=None, allow_growth=True):
        """Retain affordable device planes until this sampling pass closes.

        Admit only at a completed step boundary, after measuring real workspace.
        Prefer disk-backed layers: caching them also removes repeated host reads.
        Loading remains incremental through the existing staging buffers.
        With no later reuse, retain only already-filled caches: filling a new
        plane on the final forward just spends memory before discarding it.
        """
        if self.expected or self.pending or self.host_pending or self.closed:
            raise RuntimeError('Pass cache admission requires an idle step boundary')
        remaining = self.cache_budget_bytes = max(0, int(max_bytes))
        candidates = set()
        # Disk reads cost most, then copies from RAM-resident views, then pinned H2D.
        views = getattr(self.weight_source, 'host_views', None) or {}
        for index in sorted(range(len(self.layers)), key=lambda i: (self.pinned_layers[i], i in views)):
            if not allow_growth and index not in self.cached:
                continue
            if max_layers is not None and len(candidates) >= max_layers:
                break
            size = sum(count * parameter.element_size() for parameter, _, _, _, count, _ in self.layout[index])
            if 0 < size <= remaining:
                candidates.add(index)
                remaining -= size
        for index in set(self.cached) - candidates:
            slot = self.cached.pop(index)
            for event in (slot.compute_done, slot.copy_done):
                if event is not None:
                    event.synchronize()
            slot.device.clear()
        changed = candidates != self.cache_candidates
        self.cache_candidates = candidates
        if changed and self.read_ahead is not None:
            self.read_ahead.close()
            self.read_ahead_before_cache = self.read_ahead.stats()
            self.read_ahead = self.weight_source.read_ahead(self.layout,
                [pinned or index in self.cache_candidates for index, pinned in enumerate(self.pinned_layers)])

    def cached_bytes(self):
        return sum(t.numel() * t.element_size() for slot in self.cached.values() for t in slot.device.values())

    @contextmanager
    def layer(self, index):
        """Keep one layer resident while a caller evaluates several independent tiles."""
        if self.hooks:
            raise RuntimeError('Explicit layer contexts require manage_hooks=False')
        self._before(index)
        try:
            yield self.layers[index]
        finally:
            self._after(index)

    def _fill_host(self, index, slot):
        # CPU writes must wait for the previous H2D read of this host plane.
        # They need not wait for GPU computation using the separate device plane.
        with torch.cuda.device(self.device):
            waited = 0.
            if slot.copy_done is not None:
                tick = time.monotonic()
                slot.copy_done.synchronize()
                waited = time.monotonic() - tick
            started = time.monotonic()
            if self.read_ahead is not None:
                self.read_ahead.before(index)
            direct = self.weight_source.stage_into(index, self.layout[index], slot.host)
            counts = {}
            if not direct:
                self.reader_layers += 1
                with self.weight_source.layer(index, self.layout[index]) as weights:
                    for _, name, dtype, offset, count, _ in self.layout[index]:
                        slot.host[dtype][offset:offset + count].copy_(weights[name].reshape(-1))
            read = 0
            for _, _, dtype, offset, count, _ in self.layout[index]:
                counts[dtype] = offset + count
                if direct:
                    read += count * slot.host[dtype].element_size()
            # The two read-ahead threads and the inference thread all fill.
            with self.host_counters:
                self.host_buffer_wait_seconds += waited
                self.direct_read_bytes += read
                self.direct_read_layers += int(bool(direct))
            if self.read_ahead is not None:
                self.read_ahead.consumed(index)
            return slot, counts, time.monotonic() - started

    def _stop_host_prefetch(self):
        for read in self.host_pending.values():
            read.future.cancel()
        for executor in (self.host_executor, self.host_file_executor):
            if executor is not None:
                executor.shutdown(wait=True, cancel_futures=True)
        self.host_executor = self.host_file_executor = None
        for slot in self.host_slots:
            if slot.copy_done is not None:
                slot.copy_done.synchronize()
        self.host_pending.clear()
        self.host_slots.clear()
        self.host_prefetch = False

    def _free_host_slot(self):
        """A staging block no queued read is filling, in the order blocks alternate."""
        for _ in self.host_slots:
            slot = self.host_slots[self.host_cursor % len(self.host_slots)]
            self.host_cursor += 1
            if all(slot is not read.slot for read in self.host_pending.values()):
                return slot
        raise RuntimeError('Every staging block is held by a queued read')

    def _host_views(self):
        return getattr(self.weight_source, 'host_views', None) or {}

    def _submit_host(self, index, slot, lane=None):
        executor = self.host_executor
        if lane == 'drive':
            if self.host_file_executor is None:
                self.host_file_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='freevideo-host-read')
            executor = self.host_file_executor
        self.host_pending[index] = _HostRead(slot, executor.submit(self._fill_host, index, slot),
                                             index not in self._host_views(), lane)
        self.host_prefetch_reads += 1

    def _queue_host(self, start):
        if not self.host_prefetch:
            return
        now = time.monotonic()
        if len(self.host_slots) > 1 and now - self.host_memory_checked >= 1.:
            self.host_memory_checked = now
            if not _host_prefetch_fits(0):
                # Return only the extra block of locked pages. Reading ahead
                # through the shared staging buffer holds no more memory than
                # reading on demand does.
                self.host_prefetch_shared_reason = 'live_host_pressure'
                self.host_extra_returning = True
        if self.host_extra_returning and all(read.slot is not self.host_slots[-1] for read in self.host_pending.values()):
            # A read already queued into the block is used first.
            self.host_extra_returning = False
            extra = self.host_slots.pop()
            if extra.copy_done is not None:
                extra.copy_done.synchronize()
            del extra
            self.host_cursor = 0
            from .torch_compat import empty_host_cache
            empty_host_cache(torch)
        ahead = [index for index in range(start, len(self.layers))
                 if not self.pinned_layers[index] and index not in self.cached]
        if not ahead:
            return
        if not (self.host_file_ahead and len(self.host_slots) > 1 and not self.host_extra_returning
                and self.read_ahead is None):
            if not self.host_pending:
                self._submit_host(ahead[0], self._free_host_slot())
            return
        # Layers RAM holds are copied in a fraction of a layer's compute, so
        # the staging buffer alone keeps them one layer ahead. The extra block
        # goes to the drive: the next layer that has to be read from it starts
        # as soon as the block's last upload is done, however many RAM-backed
        # layers run in between, and on a thread of its own so that those
        # layers do not queue behind the read. One drive read runs at a time.
        # With 10 of 50 layers read from the drive every step, spread through
        # the model by loading, and each read taking 3.3 times a second-pass
        # layer's compute, 1344x768x243 went from 26.4-27.6 to 18.2-20.1 s per
        # second-pass step (18.2 s without the reads) and from 16.4 to 12.5 s
        # per first-pass step, where the ten reads alone take 12 s, on an RTX
        # PRO 6000 with identical output. One layer ahead, each of those reads
        # overlapped a single layer's compute and the GPU waited for the rest.
        views = self._host_views()
        shared, extra = self.host_slots[0], self.host_slots[-1]
        near = ahead[0]
        target = next((index for index in ahead if index not in views), None)
        if (target is not None and target not in self.host_pending
                and not any(read.lane == 'drive' for read in self.host_pending.values())):
            if target == near and not self.host_pending:
                self._submit_host(target, self._free_host_slot(), 'drive')
            elif all(read.slot is not extra for read in self.host_pending.values()):
                self._submit_host(target, extra, 'drive')
                self.host_file_ahead_reads += int(target != near)
        if (near in views and near not in self.host_pending
                and all(read.slot is not shared and read.lane == 'drive' for read in self.host_pending.values())):
            self._submit_host(near, shared, 'ram')

    def _stage(self, index):
        slot = self.slots[index % len(self.slots)]
        with torch.cuda.device(self.device):
            if index in self.cached:
                if self.read_ahead is not None:
                    self.read_ahead.consumed(index)
                self.cache_hits += 1
                cached = self.cached[index]
                return cached, cached.copy_done
            # A previous H2D transfer must finish before the CPU writes this slot.
            pinned = self.pinned_layers[index]
            if not pinned and self.weight_source is not None:
                self.source_layers += 1
            host_slot = slot
            staged = self.host_prefetch and not pinned
            if slot.copy_done is not None and not pinned and not staged:
                wait_started = time.monotonic()
                slot.copy_done.synchronize()
                self.host_buffer_wait_seconds += time.monotonic() - wait_started
            started = time.monotonic()
            if self.read_ahead is not None and not staged:
                self.read_ahead.before(index)
            counts = {}
            direct = (not staged and self.weight_source is not None and not pinned
                      and self.weight_source.stage_into(index, self.layout[index], slot.host))
            if staged:
                if index in self.host_pending:
                    read = self.host_pending.pop(index)
                    tick = time.monotonic()
                    host_slot, counts, host_seconds = read.future.result()
                    waited = time.monotonic() - tick
                    self.host_prefetch_wait_seconds += waited
                    if read.from_drive and waited > HOST_FILE_READ_WAIT and not self.host_file_ahead:
                        self.host_file_ahead = True
                        if hasattr(self.weight_source, 'reads_delay_compute'):
                            # The next pass of this request starts with it on.
                            self.weight_source.reads_delay_compute = True
                else:
                    host_slot, counts, host_seconds = self._fill_host(index, self._free_host_slot())
            elif direct:
                self.direct_read_layers += 1
                for _, _, dtype, offset, count, _ in self.layout[index]:
                    counts[dtype] = offset + count
                    self.direct_read_bytes += count * slot.host[dtype].element_size()
            else:
                if self.weight_source is not None and not pinned:
                    self.reader_layers += 1
                source = (self.weight_source.layer(index, self.layout[index]) if self.weight_source is not None and not pinned
                          else nullcontext(self.cpu[index]))
                with source as weights:
                    for _, name, dtype, offset, count, _ in self.layout[index]:
                        if not pinned:
                            slot.host[dtype][offset:offset + count].copy_(weights[name].reshape(-1))
                        counts[dtype] = offset + count
            if self.read_ahead is not None and not staged:
                self.read_ahead.consumed(index)
            if not staged:
                host_seconds = time.monotonic() - started
            begin, end = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
            with torch.cuda.stream(self.copy_stream):
                destination = slot
                if index in self.cache_candidates:
                    # Allocate on the copying stream so reused allocator blocks
                    # cannot race outstanding work from another CUDA stream.
                    planes = {}
                    try:
                        for dtype, count in counts.items():
                            planes[dtype] = torch.empty(count, dtype=dtype, device=self.device)
                    except torch.cuda.OutOfMemoryError:
                        planes.clear()
                        self.cache_candidates.discard(index)
                    else:
                        destination = Slot(host={}, device=planes)
                        self.cached[index] = destination
                # Forward kernels may still be reading the device slot from two
                # layers ago even though Python already left that forward call.
                if destination.compute_done is not None:
                    self.copy_stream.wait_event(destination.compute_done)
                begin.record()
                if pinned:
                    for _, name, dtype, offset, count, _ in self.layout[index]:
                        destination.device[dtype][offset:offset + count].copy_(self.cpu[index][name].reshape(-1), non_blocking=True)
                else:
                    for dtype, count in counts.items():
                        destination.device[dtype][:count].copy_(host_slot.host[dtype][:count], non_blocking=True)
                end.record()
            slot.copy_done = end
            host_slot.copy_done = end
            destination.copy_done = end
            self.transfers.append((index, host_seconds, begin, end,
                                   sum(count * slot.device[dtype].element_size() for dtype, count in counts.items())))
            return destination, end

    def _before(self, index):
        if index != self.expected:
            raise RuntimeError(f'Expected layer {self.expected}, got {index}; concurrent/reordered execution is unsupported')
        if index in self.pending:
            wait_started = time.monotonic()
            slot, done = self.pending.pop(index).result()
            self.prefetch_wait_seconds += time.monotonic() - wait_started
        else:
            slot, done = self._stage(index)
        torch.cuda.current_stream(self.device).wait_event(done)
        for parameter, _, dtype, offset, count, shape in self.layout[index]:
            parameter.data = slot.device[dtype][offset:offset + count].view(shape)
        if self.prefetch and index + 1 < len(self.layers):
            self.pending[index + 1] = self.executor.submit(self._stage, index + 1)
        self._queue_host(index + 1)

    def _after(self, index):
        slot = self.cached.get(index, self.slots[index % len(self.slots)])
        done = torch.cuda.Event()
        done.record(torch.cuda.current_stream(self.device))
        slot.compute_done = done
        for parameter, name, *_ in self.layout[index]:
            parameter.data = self.cpu[index][name]
        self.expected = (index + 1) % len(self.layers)

    def drive_reads(self):
        """Layers this pass has read from the weight files so far; None if unknown.

        Pinned groups, cached device copies and adopted views are in memory.
        A direct-read source counts each layer it reads from its files; a
        layer staged through its tensor reader is one more. A source that
        keeps no such count has every staged layer taken as a read. Pageable
        tensors without a source may be mapped file pages, which the system
        reads again unseen.
        """
        if self.weight_source is None:
            return 0 if self.pinned_sources else None
        counted = getattr(self.weight_source, 'read_stats', None)
        if counted is None:
            return self.source_layers
        return self.reader_layers + sum(counted[name] - self.direct_reads_start.get(name, 0)
                                        for name in ('kept_layers', 'passed_layers'))

    def drive_layers(self):
        """Offloaded layers neither pinned nor held as a RAM view; None if unknown.

        Each use of such a layer reads its file unless a device copy is
        cached, and a cached copy can be released between steps.
        """
        if self.weight_source is None:
            return 0 if self.pinned_sources else None
        views = getattr(self.weight_source, 'host_views', None) or {}
        return sum(not pinned and index not in views for index, pinned in enumerate(self.pinned_layers))

    def stats(self):
        self.copy_stream.synchronize()
        return {'prefetch': self.prefetch, 'slots': len(self.slots),
                'streamed_checkpoint_layers': self.weight_source is not None,
                'streamed_layer_count': sum(not pinned for pinned in self.pinned_layers) if self.weight_source is not None else 0,
                'pinned_model_sources': self.pinned_sources,
                'pinned_layer_count': sum(self.pinned_layers),
                'pinned_layer_indices': [index for index, pinned in enumerate(self.pinned_layers) if pinned],
                'cuda_buffer_bytes': sum(t.numel() * t.element_size() for s in self.slots for t in s.device.values()),
                'pass_cache_budget_bytes': self.cache_budget_bytes,
                'pass_cache_bytes': sum(t.numel() * t.element_size() for s in self.cached.values() for t in s.device.values()),
                'pass_cache_layer_count': len(self.cached),
                'pass_cache_layer_indices': sorted(self.cached), 'pass_cache_hits': self.cache_hits,
                'pinned_buffer_bytes': sum(t.numel() * t.element_size()
                    for s in [*self.slots, *self.host_slots[1:]] for t in s.host.values() if t.is_pinned()),
                'pageable_buffer_bytes': sum(t.numel() * t.element_size() for s in self.slots for t in s.host.values() if not t.is_pinned()),
                'transfers': len(self.transfers),
                'h2d_bytes': sum(row[4] for row in self.transfers),
                'host_stage_seconds': sum(row[1] for row in self.transfers),
                'host_buffer_wait_seconds': self.host_buffer_wait_seconds,
                'prefetch_wait_seconds': self.prefetch_wait_seconds,
                'h2d_seconds': sum(row[2].elapsed_time(row[3]) for row in self.transfers) / 1000,
                'direct_read_layers': self.direct_read_layers,
                'direct_read_bytes': self.direct_read_bytes,
                'host_view_layers': len(getattr(self.weight_source, 'host_views', None) or {}),
                'host_view_bytes': getattr(self.weight_source, 'host_view_bytes', 0),
                'host_view_hits': getattr(self.weight_source, 'host_view_hits', 0) - self.host_view_hits_start,
                'host_view_hit_bytes': getattr(self.weight_source, 'host_view_hit_bytes', 0) - self.host_view_hit_bytes_start,
                'host_view_released': getattr(self.weight_source, 'host_view_released', 0),
                # Converted layers kept in RAM for the clips still to come.
                'converted_layer_count': len(getattr(self.weight_source, 'converted', None) or {}),
                'converted_bytes': getattr(self.weight_source, 'converted_bytes', 0),
                'converted_hits': getattr(self.weight_source, 'converted_hits', 0),
                'converted_refused': getattr(self.weight_source, 'converted_refused', 0),
                # Direct reads kept in the file cache between steps and those passing through.
                'cache_kept_layer_count': len(getattr(self.weight_source, 'cache_kept', None) or {}),
                'cache_kept_bytes': getattr(self.weight_source, 'cache_kept_bytes', 0),
                **{'direct_' + name: value - self.direct_reads_start.get(name, 0)
                   for name, value in (getattr(self.weight_source, 'read_stats', None) or {}).items()},
                'host_prefetch': self.host_prefetch,
                'host_prefetch_reads': self.host_prefetch_reads,
                'host_prefetch_wait_seconds': self.host_prefetch_wait_seconds,
                'host_prefetch_buffer_bytes': sum(t.numel() * t.element_size()
                    for slot in self.host_slots[1:] for t in slot.host.values()),
                # Drive-backed layers read ahead past RAM-backed ones.
                'host_prefetch_file_ahead': self.host_file_ahead,
                'host_prefetch_file_ahead_reads': self.host_file_ahead_reads,
                'host_prefetch_disabled_reason': self.host_prefetch_disabled_reason,
                'host_prefetch_shared_reason': self.host_prefetch_shared_reason,
                'disk_read_ahead': self.read_ahead.stats() if self.read_ahead is not None else {'enabled': False},
                'disk_read_ahead_before_cache': self.read_ahead_before_cache,
                'timing_note': 'Host staging, buffer waits and H2D may overlap compute. Prefetch wait can include '
                               'buffer waiting and host staging; do not add these counters to wall time.'}

    def close(self):
        if self.closed:
            return
        if self.executor is not None:
            self.executor.shutdown(wait=True)
        self._stop_host_prefetch()
        if self.read_ahead is not None:
            self.read_ahead.close()
        self.copy_stream.synchronize()
        for slot in [*self.slots, *self.cached.values()]:
            if slot.compute_done is not None:
                slot.compute_done.synchronize()
        for handle in self.hooks:
            handle.remove()
        for index, entries in enumerate(self.layout):
            # Streamed CPU entries are zero-storage placeholders. Reopening all
            # checkpoint mappings here defeats streaming and can exhaust Windows
            # commit after the final step, before the caller can save its latents.
            # Retained layout metadata reconstructs the next request's slots.
            for parameter, name, *_ in entries:
                parameter.data = self.cpu[index][name]
        self.pending.clear()
        self.cached.clear()
        self.cache_candidates.clear()
        self.slots.clear()
        release = getattr(self.weight_source, 'release_converted', None)
        if release is not None:
            release()
        self.closed = True

    def __enter__(self):
        return self

    def __exit__(self, *_):
        self.close()