Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

42.1 KB
Raw Blame
"""Reopen immutable safetensor layer views only while filling an offload slot."""
from contextlib import contextmanager
import functools
from pathlib import Path
import threading
import time


_TORCH_DTYPES = {'BOOL': 'bool', 'U8': 'uint8', 'I8': 'int8', 'I16': 'int16',
                 'I32': 'int32', 'I64': 'int64', 'F16': 'float16', 'BF16': 'bfloat16',
                 'F32': 'float32', 'F64': 'float64', 'F8_E4M3': 'float8_e4m3fn',
                 'F8_E5M2': 'float8_e5m2'}


_IDENTITY_FIELDS = ('st_dev', 'st_ino', 'st_size', 'st_mtime_ns', 'st_ctime_ns')


def _file_identity(value):
    return tuple(getattr(value, name) for name in _IDENTITY_FIELDS)


class CheckpointChangedError(ValueError):
    def __init__(self, path, source, expected, observed):
        self.checkpoint_change = dict(source=source, fields=[name for name, before, after
            in zip(_IDENTITY_FIELDS, expected, observed) if before != after])
        super().__init__('Checkpoint changed during streamed inference: ' + str(path))


def _open_safetensors(path, *, framework, device='cpu'):
    """Open a streamed shard without reserving the whole file on Windows.

    The mmap backend is excellent on Linux, where clean mapped pages are
    reclaimable and do not consume commit.  Windows charges the section views
    against the system commit even when their pages are clean.  Reopening a
    multi-hundred-MiB shard for every transformer block can therefore consume
    the pagefile while the process's working set stays small.  safetensors
    0.8's ``pread`` backend reads only the requested tensor bytes and avoids
    that commit reservation.  Runtime dependencies pin this version, so do not
    silently fall back to mmap on Windows.
    """
    from safetensors import safe_open
    from .system import windows
    kwargs = {'framework': framework, 'device': device}
    if windows():
        kwargs['backend'] = 'pread'
    return safe_open(str(path), **kwargs)


def interleaved_layer_order(count):
    """Spread any affordable prefix of pinned layers across the computation.

    A long cached prefix fills the bounded read-ahead window and then leaves
    the disk idle. Interleaving retained layers gives the reader useful work
    throughout a step without increasing its buffers or changing layer order.
    Longest-gap bisection also spreads partial allocations when live RAM or
    the platform's pinning limit is lower than the plan.
    """
    import heapq
    if type(count) is not int or count < 0:
        raise ValueError('Layer count must be a non-negative integer')
    pending = [(-count, 0, count)] if count else []
    order = []
    while pending:
        _, start, end = heapq.heappop(pending)
        middle = (start + end) // 2
        order.append(middle)
        for lo, hi in ((start, middle), (middle + 1, end)):
            if lo < hi:
                heapq.heappush(pending, (lo - hi, lo, hi))
    return order


def spread_layer_order(count):
    """Layers ordered so that any first part and any last part lie spread out.

    Loading keeps streamed layers in RAM in this order until RAM runs out, so
    the layers left on the drive are apart from each other wherever it stops,
    and each can be read while the layers between them compute. Kept in index
    order they were the last layers of the model, one after another, and the
    drive stood idle through the rest of every step. Reversing the bits of
    the index gives the order: 0, then the middle, then the quarters.
    """
    if type(count) is not int or count < 0:
        raise ValueError('Layer count must be a non-negative integer')
    bits = max(1, (count - 1).bit_length())
    return sorted(range(count), key=lambda index: int(format(index, f'0{bits}b')[::-1], 2))


def release_file_pages(paths):
    """Advise away consumed clean pages; never remove or rewrite a weight file."""
    import os
    advised, errors = 0, 0
    if not hasattr(os, 'posix_fadvise'):
        return advised, errors
    for path in paths:
        try:
            with path.open('rb', buffering=0) as file:
                os.posix_fadvise(file.fileno(), 0, 0, os.POSIX_FADV_DONTNEED)
                advised += path.stat().st_size
        except OSError:
            # Advisory cache management is optional. The normal tensor reader
            # still validates the immutable file before and after every use.
            errors += 1
    return advised, errors


def read_ahead_budget():
    """Bound clean file-cache lookahead by live RAM, without a pinned copy."""
    from .system import system_memory, windows
    from .hardware import cgroup_memory
    memory = system_memory()
    available = memory['available_bytes']
    # Windows reports physical and commit headroom separately.  File-cache
    # read-ahead is deliberately pageable, but it still must not run when the
    # pagefile/commit is already the tighter bound; otherwise a speculative
    # read can recreate the same pressure that streamed pread was introduced
    # to avoid.
    # Commit is a meaningful second capacity on Windows: section views and
    # pageable staging can consume the pagefile even when physical RAM looks
    # available.  Linux's MemAvailable already accounts for reclaimable file
    # cache; do not import the Windows commit ceiling into its read-ahead
    # budget.
    if windows():
        commit = memory.get('commit_available_bytes')
        if type(commit) is int and commit >= 0:
            available = min(available, commit)
    group = cgroup_memory()
    if not group.get('complete'):
        return 0
    limited = group.get('reclaimable_available_bytes')
    if limited is not None:
        available = min(available, limited)
    # Current-layer views, a small read buffer and changing application usage
    # also need space. These are clean reclaimable pages, not retained tensors.
    return max(0, min(1536 * 2**20, available - 512 * 2**20))


class LayerReadAhead:
    """One cancellable reader warms a byte-bounded window of immutable files.

    Only file cache retains the data. The reader owns one 4 MiB scratch buffer,
    never a tensor, a pinned allocation or a GPU slot. Submission advances as
    the inference thread consumes layers; it cannot race through the model.
    """
    def __init__(self, plans, check, layer_count, budget=read_ahead_budget):
        from concurrent.futures import ThreadPoolExecutor
        from threading import Event, Lock
        self.plans = sorted((index, tuple(paths), sum(path.stat().st_size for path in paths))
                            for index, paths in plans.items())
        self.check = check
        self.paths_by_index = {index: paths for index, paths, _ in self.plans}
        self.layer_count = layer_count
        self.budget = budget
        self.cursor = 0
        self.pending = {}
        self.pending_bytes = 0
        self.stop = Event()
        self.lock = Lock()
        self.executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='freevideo-read-ahead')
        self.closed = False
        self.failed = None
        self.last_read_finished = None
        self.counters = dict(read_bytes=0, read_seconds=0., wait_seconds=0.,
                             reader_idle_seconds=0.,
                             files_read=0, peak_window_bytes=0, skipped_layers=0,
                             discard_advised_bytes=0, discard_errors=0)
        self._fill()

    def _read(self, paths):
        import time
        started = time.monotonic()
        with self.lock:
            if self.last_read_finished is not None:
                self.counters['reader_idle_seconds'] += max(0., started - self.last_read_finished)
        scratch = bytearray(4 * 2**20)
        read_bytes = 0
        files = 0
        try:
            for path in paths:
                if self.stop.is_set():
                    break
                self.check(path)
                complete = False
                with path.open('rb', buffering=0) as file:
                    while not self.stop.is_set():
                        count = file.readinto(scratch)
                        if not count:
                            complete = True
                            break
                        read_bytes += count
                self.check(path)
                files += int(complete)
        finally:
            with self.lock:
                self.counters['read_bytes'] += read_bytes
                self.counters['read_seconds'] += time.monotonic() - started
                self.counters['files_read'] += files
                self.last_read_finished = time.monotonic()

    def _fill(self):
        if self.closed or self.failed:
            return
        maximum = max(0, int(self.budget()))
        while self.cursor < len(self.plans):
            index, paths, size = self.plans[self.cursor]
            if size > maximum:
                self.cursor += 1
                self.counters['skipped_layers'] += 1
                continue
            if self.pending_bytes + size > maximum:
                break
            self.pending[index] = (size, self.executor.submit(self._read, paths))
            self.pending_bytes += size
            self.counters['peak_window_bytes'] = max(self.counters['peak_window_bytes'], self.pending_bytes)
            self.cursor += 1

    def before(self, index):
        import time
        self._fill()
        item = self.pending.get(index)
        if item is None:
            return
        started = time.monotonic()
        try:
            item[1].result()
        except Exception as error:
            # A failed optimization must not replace the real tensor reader's
            # validation/error. Stop speculative reads; the normal path retries
            # this immutable file and still checks its identity and contents.
            self.failed = self.failed or type(error).__name__ + ': ' + str(error)
            self.stop.set()
            for _, future in self.pending.values():
                future.cancel()
        finally:
            self.counters['wait_seconds'] += time.monotonic() - started

    def consumed(self, index):
        item = self.pending.pop(index, None)
        if item is not None:
            self.pending_bytes -= item[0]
            if not self.failed:
                # mmap copies mark consumed pages active. Under a tight cgroup
                # Linux otherwise evicts the unread speculative pages first,
                # so both threads end up fetching the same weights from disk.
                advised, errors = release_file_pages(self.paths_by_index[index])
                self.counters['discard_advised_bytes'] += advised
                self.counters['discard_errors'] += errors
        if index == self.layer_count - 1:
            if self.pending:
                raise RuntimeError('Read-ahead layers were consumed out of order')
            self.cursor = 0
        self._fill()

    def end_pass_early(self):
        """Cancel the unread window and rewind without reading the skipped layers."""
        self.stop.set()
        for _, future in self.pending.values():
            future.cancel()
        for _, future in self.pending.values():
            if not future.cancelled():
                try:
                    future.result()
                except Exception as error:
                    self.failed = self.failed or type(error).__name__ + ': ' + str(error)
        self.pending.clear()
        self.pending_bytes = 0
        self.cursor = 0
        if not self.failed and not self.closed:
            self.stop.clear()
        # before() will refill on demand; cached steps need no speculative I/O.

    def stats(self):
        with self.lock:
            return dict(self.counters, enabled=True, failure=self.failed,
                        scratch_buffer_bytes=4 * 2**20,
                        read_scope='Logical CPU reads, including file-cache hits; not physical disk I/O',
                        scope='Reclaimable file cache; one reader, no extra pinned RAM or GPU slot')

    def close(self):
        if self.closed:
            return
        self.closed = True
        self.stop.set()
        for _, future in self.pending.values():
            future.cancel()
        self.executor.shutdown(wait=True, cancel_futures=True)
        self.pending.clear()
        self.pending_bytes = 0


_READ_PIECE = 8 << 20
_reader = None
_reader_lock = threading.Lock()


def _reader_pool():
    """Threads shared by every streamed source's direct reads."""
    global _reader
    with _reader_lock:
        if _reader is None:
            from concurrent.futures import ThreadPoolExecutor
            from .tensor_io import read_threads
            _reader = ThreadPoolExecutor(read_threads(), thread_name_prefix='freevideo-stream-read')
        return _reader


# Streamed layers kept in RAM as read-only file views must leave this much
# physical memory free. A 27.98 GiB Windows machine started sampling at
# 7.31 GiB available and ran the refine pass at 6.98 GiB: 0.35 GiB of later
# growth, plus a 2 GiB floor for the rest of the system.
HOST_VIEW_RESERVE = int(2.5 * 2**30)
# Below this much available memory, views are handed back one layer per use,
# so a request that grows (a larger refine pass, readouts, another program)
# trades them for disk reads instead of pushing Windows toward its floor.
HOST_VIEW_SHED = int(1.5 * 2**30)
# Direct reads of one step can exceed the memory left for the file cache. The
# cache then holds the layers read last and the next step starts with the
# first ones, so every layer comes from the drive again. Layers that fit beside
# one passing layer and this slack are kept: read at normal page priority. The
# rest are read at Windows' lowest page priority, whose pages the standby list
# gives up first, to this request's next read or to any other program.
CACHE_KEEP_SLACK = 512 * 2**20
_MEMORY_PRIORITY_VERY_LOW = 1


@functools.lru_cache(maxsize=None)
def _thread_page_priority():
    """Swap the calling thread's page priority on Windows: a function or None.

    The function sets the given priority and returns the previous one, or
    returns None and changes nothing when Windows refuses. It is None on other
    systems, on Windows versions without SetThreadInformation and with
    FREEVIDEO_PAGE_PRIORITY=off, where every read keeps normal priority.
    """
    import os
    from .system import windows
    if not windows() or os.environ.get('FREEVIDEO_PAGE_PRIORITY', 'auto').lower() in ('off', '0', 'false'):
        return None
    import ctypes
    from ctypes import wintypes
    try:
        kernel32 = ctypes.WinDLL('kernel32', use_last_error=True)
        current, get, set_ = (kernel32.GetCurrentThread, kernel32.GetThreadInformation,
                              kernel32.SetThreadInformation)
    except (AttributeError, OSError):
        return None
    current.restype = wintypes.HANDLE
    current.argtypes = ()
    for function in (get, set_):
        # THREAD_INFORMATION_CLASS ThreadMemoryPriority = 0, MEMORY_PRIORITY_INFORMATION.
        function.argtypes = (wintypes.HANDLE, ctypes.c_int, ctypes.POINTER(wintypes.ULONG), wintypes.DWORD)
        function.restype = wintypes.BOOL

    def swap(priority):
        # A hint must never fail a read: any refusal leaves the priority as it was.
        try:
            info = wintypes.ULONG()
            if not get(current(), 0, ctypes.byref(info), ctypes.sizeof(info)):
                return None
            previous = info.value
            info.value = priority
            return previous if set_(current(), 0, ctypes.byref(info), ctypes.sizeof(info)) else None
        except (ctypes.ArgumentError, OSError, ValueError):
            return None

    # Setting this thread's own priority again proves the calls work here.
    previous = swap(_MEMORY_PRIORITY_VERY_LOW)
    if previous is None:
        return None
    swap(previous)
    return swap


@contextmanager
def _low_page_priority(enabled):
    """Read at the lowest page priority inside this block, on this thread only."""
    swap = _thread_page_priority() if enabled else None
    previous = swap(_MEMORY_PRIORITY_VERY_LOW) if swap is not None else None
    try:
        yield
    finally:
        if previous is not None:
            swap(previous)


# Converted decoder layers are private copies and charge commit, but they stand
# in for the clean file pages the streamed reads would otherwise keep cycling
# through the standby list, so the floor left to the rest of the system is
# smaller than for views. Admission is re-checked at every layer.
CONVERTED_RESERVE = int(1.5 * 2**30)


def converted_fits(size, reserve=CONVERTED_RESERVE, room=None):
    """Whether one more converted layer may stay in RAM as a private copy.

    ``room`` is what the request's own RAM budget still allows, when it has one.
    """
    from .hardware import cgroup_memory
    from .system import system_memory
    if room is not None and room < size:
        return False
    available = system_memory()['available_bytes']
    headroom = cgroup_memory()['reclaimable_available_bytes']
    if headroom is not None:
        available = min(available, headroom)
    return available - size >= reserve


def host_view_fits(size, reserve=HOST_VIEW_RESERVE):
    """Whether one more layer may stay in RAM as a read-only file view.

    Read-only views charge no commit (a 412 MiB block mapped and touched moved
    Windows commit by 1 MiB; a private copy moved it by 415 MiB), and Windows
    can drop their clean pages without writing the pagefile. Only physical
    memory bounds them.
    """
    from .system import system_memory
    memory = system_memory()
    available = memory.get('physical_available_bytes') or memory['available_bytes']
    return available - size >= reserve


class ConvertedStore:
    """Converted decoder layers that outlive one decode, for a resident process.

    Entries belong to one checkpoint identity; a different one empties the store.
    """

    def __init__(self):
        self.identity = None
        self.layers = {}
        self.lock = threading.Lock()

    def bind(self, identity):
        with self.lock:
            if identity != self.identity:
                self.layers = {}
                self.identity = identity
            return self.layers

    def size(self):
        with self.lock:
            return sum(value.numel() * value.element_size()
                       for tensors in list(self.layers.values()) for value in tensors.values())

    def clear(self):
        with self.lock:
            self.layers.clear()
            self.identity = None


class SafetensorLayers:
    def __init__(self, paths, prefixes, *, intermediate_dtype=None, direct_read=None, host_views=False,
                 host_view_reserve=HOST_VIEW_RESERVE, keep_converted=False, converted_room=None,
                 converted_store=None):
        from .system import windows
        self.converted_store = None
        self.direct_read = windows() if direct_read is None else direct_read
        self.prefixes = tuple(prefixes)
        self.intermediate_dtype = intermediate_dtype
        # Layers converted from their checkpoint dtype (the decoder's FP16
        # Linear values) kept for the layers' next uses. A tile decode streams
        # the same layers once per clip; converting reads the whole FP32 layer
        # each time, from disk once the file cache cannot hold them.
        self.keep_converted = keep_converted and intermediate_dtype is not None
        self.converted_room = converted_room
        self.converted = {}
        self.converted_bytes = 0
        self.converted_hits = 0
        self.converted_refused = 0
        self.files = {}
        self.handle_files = {}
        self.keys = {}
        self.byte_ranges = {}
        # Layers that pinning could not hold but RAM can: read-only views of
        # their checkpoint bytes, adopted right after the first direct read.
        self.host_views = {} if host_views and self.direct_read else None
        self.host_view_reserve = host_view_reserve
        self.host_view_bytes = 0
        self.host_view_hits = 0
        self.host_view_hit_bytes = 0
        self.host_view_released = 0
        # Direct-read layers kept in the file cache between steps (index: bytes),
        # and the direct reads of kept and passing layers.
        self.cache_kept = {}
        self.cache_kept_bytes = 0
        self.cache_lock = threading.Lock()
        self.read_stats = dict(kept_layers=0, kept_bytes=0, kept_seconds=0.,
                               passed_layers=0, passed_bytes=0, passed_seconds=0.)
        # A read from the drive has kept the inference thread waiting; the
        # offloader then reads drive-backed layers further ahead (offload.py).
        self.reads_delay_compute = False
        for path in sorted(set(Path(p).resolve() for p in paths)):
            self.files[path] = self.identity(path)
            with _open_safetensors(path, framework='np') as handle:
                # safe_open validates the file layout first. Retain only its
                # header offsets, never tensor storage or a file mapping.
                if self.direct_read:
                    import json
                    import os
                    import struct
                    with path.open('rb') as stream:
                        # Windows stat().ctime can be creation time while
                        # fstat().ctime is NTFS ChangeTime. Keep each API's
                        # baseline instead of comparing these different clocks.
                        opened = _file_identity(os.fstat(stream.fileno()))
                        if opened[:4] != self.files[path][:4]:
                            raise CheckpointChangedError(path, 'handle', self.files[path][:4], opened[:4])
                        self.handle_files[path] = opened
                        length = struct.unpack('<Q', stream.read(8))[0]
                        if not 2 <= length <= min(100_000_000, self.files[path][2] - 8):
                            raise ValueError('Invalid safetensors header size')
                        header = json.loads(stream.read(length))
                    self.check(path)
                for key in handle.keys():
                    if key in self.keys:
                        raise ValueError('Duplicate checkpoint tensor: ' + key)
                    self.keys[key] = path
                    if self.direct_read:
                        spec = header[key]
                        tensor = handle.get_slice(key)
                        if spec['shape'] != tensor.get_shape() or spec['dtype'] != tensor.get_dtype():
                            raise ValueError('Checkpoint header changed: ' + key)
                        start, end = spec['data_offsets']
                        self.byte_ranges[key] = (8 + length + start, end - start,
                                                spec['dtype'], tuple(spec['shape']))
            self.check(path)
        if converted_store is not None and self.keep_converted:
            # A resident process keeps the layers for its next request; they are
            # served only while the checkpoint files, layers and dtype are unchanged.
            self.converted = converted_store.bind(
                (tuple(sorted((str(path), identity) for path, identity in self.files.items())),
                 self.prefixes, str(self.intermediate_dtype)))
            self.converted_bytes = converted_store.size()
            self.converted_store = converted_store

    @staticmethod
    def identity(path):
        return _file_identity(path.stat())

    def check(self, path):
        observed = self.identity(path)
        if observed != self.files[path]:
            raise CheckpointChangedError(path, 'path', self.files[path], observed)

    def stage_into(self, index, layout, planes):
        """Read matching tensor bytes directly into the existing CPU slot.

        Windows pread otherwise creates a whole layer of temporary tensors
        before copying it into this slot. FileIO.readinto uses the destination
        storage itself, so neither that allocation nor its CPU copy is needed.
        Dtype conversion (including VAE intermediate rounding) keeps the
        established tensor reader. Validate the entire plan before any write.
        """
        if not self.direct_read or self.intermediate_dtype is not None:
            return False
        import math
        import sys
        import torch
        if sys.byteorder != 'little':
            return False
        prefix = self.prefixes[index]
        reads = {}
        for _, name, dtype, offset, count, shape in layout:
            key = prefix + name
            start, size, stored, stored_shape = self.byte_ranges[key]
            if stored_shape != tuple(shape) or count != math.prod(shape):
                raise ValueError('Checkpoint layer shape changed: ' + key)
            if getattr(torch, _TORCH_DTYPES.get(stored, ''), None) != dtype:
                return False
            plane = planes[dtype]
            if (plane.device.type != 'cpu' or plane.dtype != dtype or not plane.is_contiguous()
                    or offset < 0 or count < 0 or offset + count > plane.numel()
                    or size != count * plane.element_size()):
                raise ValueError('Invalid streamed destination: ' + key)
            reads.setdefault(self.keys[key], []).append((start, size, dtype, offset, name))
        for path in reads:
            self.check(path)
        entry = self.host_views.get(index) if self.host_views is not None else None
        if entry is not None and any(name not in entry[0] or entry[0][name].numel() != size
                                     for entries in reads.values() for _, size, _, _, name in entries):
            # Views are kept by tensor name. One that cannot serve this layout
            # is dropped rather than copied to a guessed plane offset.
            self._release(index)
            entry = None
        if entry is not None:
            views, _, size = entry
            # torch copies release the GIL, so the inference thread keeps launching.
            try:
                for entries in reads.values():
                    for _, length, dtype, offset, name in entries:
                        begin = offset * planes[dtype].element_size()
                        planes[dtype].detach().view(torch.uint8)[begin:begin + length].copy_(views[name])
            finally:
                for path in reads:
                    self.check(path)
            with self.cache_lock:
                self.host_view_hits += 1
                self.host_view_hit_bytes += size
            del entry, views
            if not host_view_fits(0, HOST_VIEW_SHED):
                self._release(index)
            return True
        # numpy exposes a writable byte view of CPU/pinned storage without
        # copying, including BF16/FP8 planes that numpy cannot represent directly.
        buffers = {dtype: memoryview(plane.detach().view(torch.uint8).numpy()).cast('B')
                   for dtype, plane in planes.items()}
        try:
            pieces = []
            for path, entries in reads.items():
                for start, size, dtype, offset, _ in sorted(entries, key=lambda row: row[0]):
                    begin = offset * planes[dtype].element_size()
                    target = buffers[dtype][begin:begin + size]
                    pieces += [(path, start + at, target[at:at + _READ_PIECE])
                               for at in range(0, size, _READ_PIECE)]
            size = sum(row[1] for entries in reads.values() for row in entries)
            # A layer about to become a view stays in this process, at normal priority.
            keep = self._keeps_cached(index, size) or (
                self.host_views is not None and host_view_fits(size, self.host_view_reserve))
            started = time.monotonic()
            self._read_pieces(pieces, low_page_priority=not keep)
            kind = 'kept' if keep else 'passed'
            with self.cache_lock:
                self.read_stats[kind + '_layers'] += 1
                self.read_stats[kind + '_bytes'] += size
                self.read_stats[kind + '_seconds'] += time.monotonic() - started
            if self.host_views is not None:
                if host_view_fits(size, self.host_view_reserve):
                    self._adopt(index, reads, size)
                    if index in self.host_views:
                        self._forget_cached(index)
        finally:
            for path in reads:
                self.check(path)
        return True

    def _keeps_cached(self, index, size):
        """Whether this layer's direct read stays in the file cache for the next step.

        A layer is kept while the kept layers, one passing layer and
        CACHE_KEEP_SLACK fit in available memory, which counts cached pages.
        A kept layer is given up only once the kept layers and the slack no
        longer fit, one layer of hysteresis. Without page priority every read
        is kept, as before.
        """
        if _thread_page_priority() is None:
            return True
        from .system import system_memory
        memory = system_memory()
        available = memory.get('physical_available_bytes') or memory['available_bytes']
        with self.cache_lock:
            if index in self.cache_kept:
                if self.cache_kept_bytes + CACHE_KEEP_SLACK <= available:
                    return True
                self.cache_kept_bytes -= self.cache_kept.pop(index)
                return False
            if self.cache_kept_bytes + 2 * size + CACHE_KEEP_SLACK <= available:
                self.cache_kept[index] = size
                self.cache_kept_bytes += size
                return True
            return False

    def _forget_cached(self, index):
        with self.cache_lock:
            if index in self.cache_kept:
                self.cache_kept_bytes -= self.cache_kept.pop(index)

    def _read_pieces(self, pieces, low_page_priority=False):
        """Read (path, position, destination) pieces on a few handles at once.

        One handle reading a streamed layer tensor by tensor leaves most of a
        fast drive idle: 28 int8 layers (12.1 GB) read from an NVMe drive with
        a cold cache at 2.66 GB/s that way and at 3.4-4.4 GB/s as 8 MiB pieces
        on four handles. With 34 such layers read every step, 960x544 steps
        took 4.4 s instead of 6.0-6.2 s, bit-identical (Linux, RTX PRO 6000).
        Loading already reads its blocks like this (tensor_io.ParallelReads).
        Each piece fills exactly its bytes of the slot, and every handle is
        checked against the file it opened. FREEVIDEO_READ_THREADS=1 reads on
        the calling thread alone. With low_page_priority every reader reads at
        the lowest page priority and restores its own afterwards.
        """
        import os
        from contextlib import ExitStack
        from .tensor_io import read_threads
        threads = min(read_threads(), len(pieces))
        # A reader still running when this call leaves would write into a slot
        # its caller may reuse. After a failure here a reader writes nothing
        # more, and the call returns only once none is reading.
        state = dict(stop=False, active=0)
        idle = threading.Condition()

        def read(group):
            with idle:
                if state['stop']:
                    return
                state['active'] += 1
            try:
                streams = {}
                with ExitStack() as opened, _low_page_priority(low_page_priority):
                    for path, position, target in group:
                        if state['stop']:
                            return
                        stream = streams.get(path)
                        if stream is None:
                            stream = streams[path] = opened.enter_context(path.open('rb', buffering=0))
                            observed = _file_identity(os.fstat(stream.fileno()))
                            if observed != self.handle_files[path]:
                                raise CheckpointChangedError(path, 'handle', self.handle_files[path], observed)
                        stream.seek(position)
                        done = 0
                        while done < len(target):
                            count = stream.readinto(target[done:])
                            if not count:
                                raise ValueError('Incomplete streamed checkpoint: ' + str(path))
                            done += count
            finally:
                with idle:
                    state['active'] -= 1
                    idle.notify_all()

        if threads < 2:
            read(pieces)
            return
        # Neighbouring pieces go to different handles, so together the
        # readers still move through each file from front to back.
        from concurrent.futures import wait
        futures = []
        try:
            for first in range(threads):
                futures.append(_reader_pool().submit(read, pieces[first::threads]))
            wait(futures)
        except BaseException:
            with idle:
                state['stop'] = True
                idle.wait_for(lambda: not state['active'])
            raise
        for future in futures:
            future.result()

    def layer_bytes(self, index):
        """Checkpoint bytes of one layer by stored dtype, from the headers alone."""
        prefix = self.prefixes[index]
        sizes = {}
        for key, (_, size, stored, _) in self.byte_ranges.items():
            if key.startswith(prefix):
                sizes[stored] = sizes.get(stored, 0) + size
        return sizes

    def placeholders(self, index, path):
        """Tensors with one file's stored dtypes and shapes for a layer, read from nothing.

        Model loading binds a streamed layer only to record its layout, then
        drops the values. When the layer will be neither pinned nor kept as a
        view, reading it first is wasted: the first step reads it again. With
        the RAM of a 16 GB machine, 31 of 45 offloaded blocks (13.4 GB) were
        such layers; on the RTX 5060 Ti test machine loading took 7.1 s instead
        of 11.3 s, with identical output, and slower drives save far more.
        Each placeholder is one element expanded to the checkpoint shape.
        Return None for a dtype torch lacks.
        """
        import torch
        prefix = self.prefixes[index]
        path = Path(path).resolve()
        tensors = {}
        for key, owner in self.keys.items():
            if owner != path or not key.startswith(prefix):
                continue
            _, _, stored, shape = self.byte_ranges[key]
            dtype = getattr(torch, _TORCH_DTYPES.get(stored, ''), None)
            if dtype is None:
                return None
            tensors[key] = (torch.empty((1,) * len(shape), dtype=dtype).expand(shape) if all(shape)
                            else torch.empty(shape, dtype=dtype))
        return tensors

    def _keep_converted(self, index, tensors, shared):
        """Keep this layer's converted tensors if RAM has room, as private copies."""
        size = sum(value.numel() * value.element_size() for value in tensors.values())
        with self.cache_lock:
            if index in self.converted:
                return
            room = self.converted_room() if self.converted_room is not None else None
            if (self.converted_room is not None and room is None) or not converted_fits(size, room=room):
                self.converted_refused += 1
                return
            # A tensor that needed no conversion may still map the checkpoint.
            self.converted[index] = {name: value.clone() if name in shared else value
                                     for name, value in tensors.items()}
            self.converted_bytes += size

    def release_converted(self):
        if self.converted_store is not None:
            return  # owned by the resident process, which releases it under memory pressure
        with self.cache_lock:
            self.converted.clear()
            self.converted_bytes = 0

    def adopt(self, index):
        """Keep a layer that loading just read as views, so its first pass reads no disk.

        Pinned groups admitted after it consume the RAM that cached these
        pages: a 768p request after a 20 s one read its 16 GiB of unpinned
        layers again during the first step. Admission and shedding are the
        same as for a layer adopted after its first direct read.
        """
        if self.host_views is None or index in self.host_views:
            return False
        prefix = self.prefixes[index]
        reads = {}
        for key, path in self.keys.items():
            if key.startswith(prefix):
                start, size, _, _ = self.byte_ranges[key]
                reads.setdefault(path, []).append((start, size, None, None, key[len(prefix):]))
        size = sum(row[1] for entries in reads.values() for row in entries)
        if not reads or not host_view_fits(size, self.host_view_reserve):
            return False
        for path in reads:
            self.check(path)
        self._adopt(index, reads, size)
        return index in self.host_views

    def _adopt(self, index, reads, size):
        """Keep the layer just read resident as read-only views of its checkpoint."""
        import mmap
        import os
        import warnings
        import torch
        views, mappings = {}, []
        for path, entries in reads.items():
            with path.open('rb', buffering=0) as stream:
                observed = _file_identity(os.fstat(stream.fileno()))
                if observed != self.handle_files[path]:
                    raise CheckpointChangedError(path, 'handle', self.handle_files[path], observed)
                try:
                    mapping = mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ)
                except OSError:
                    # Address space or mapping refused: this layer keeps its direct reads.
                    for opened in mappings:
                        opened.close()
                    return
            mappings.append(mapping)
            with warnings.catch_warnings():
                # Read-only checkpoint bytes; these tensors are only ever copy sources.
                warnings.simplefilter('ignore', UserWarning)
                data = torch.frombuffer(mapping, dtype=torch.uint8)
            for start, length, _, _, name in entries:
                views[name] = data[start:start + length]
        # The direct read just left these pages in the file cache. Touch every
        # page now, so they join this process's working set before the next
        # streamed reads recycle the standby list.
        for source in views.values():
            source[::mmap.PAGESIZE].sum()
        with self.cache_lock:
            self.host_views[index] = (views, mappings, size)
            self.host_view_bytes += size

    def _release(self, index):
        with self.cache_lock:
            views, mappings, size = self.host_views.pop(index)
            self.host_view_bytes -= size
            self.host_view_released += 1
        del views
        for mapping in mappings:
            try:
                mapping.close()
            except BufferError:
                pass  # A tensor view is still referenced; the mapping closes when it is collected.

    def release_host_views(self):
        for index in list(self.host_views or ()):
            self._release(index)

    def read_ahead(self, layouts, pinned):
        from .system import windows
        # Windows uses the commit-safe safetensors ``pread`` backend.  A
        # second file-cache reader would warm pageable copies while WDDM/Commit
        # is already the tight resource and can make the next transfer slower.
        # Linux keeps this bounded reader because clean file-cache pages are
        # reclaimable and MemAvailable accounts for them.
        if windows():
            return None
        plans = {}
        for index, layout in enumerate(layouts):
            if pinned[index]:
                continue
            prefix = self.prefixes[index]
            paths = {self.keys[prefix + name] for _, name, *_ in layout}
            # Prepared transformer groups each own a file. Do not pull an
            # entire multi-layer VAE/refiner shard into RAM to warm one layer.
            if any(not key.startswith(prefix) for key, path in self.keys.items() if path in paths):
                continue
            plans[index] = sorted(paths)
        if not plans:
            return None
        # Pinned copies and zero-sized streamed placeholders already replaced
        # these mapped sources. Their old active cache pages must not crowd out
        # the next layers. No inference tensor or file content is discarded.
        for path in self.files:
            self.check(path)
        advised, errors = release_file_pages(self.files)
        reader = LayerReadAhead(plans, self.check, len(layouts))
        reader.counters['discard_advised_bytes'] += advised
        reader.counters['discard_errors'] += errors
        return reader

    @contextmanager
    def layer(self, index, layout):
        from contextlib import ExitStack
        with self.cache_lock:
            kept = self.converted.get(index)
            if kept is not None:
                self.converted_hits += 1
        if kept is not None:
            yield dict(kept)
            return
        prefix = self.prefixes[index]
        tensors = {}
        shared = set()
        paths = {self.keys[prefix + name] for _, name, *_ in layout}
        for path in paths:
            self.check(path)
        try:
            with ExitStack() as stack:
                opened = {path: stack.enter_context(_open_safetensors(path, framework='pt', device='cpu'))
                          for path in paths}
                for _, name, dtype, _, count, shape in layout:
                    key = prefix + name
                    tensor = opened[self.keys[key]].get_tensor(key)
                    if tuple(tensor.shape) != tuple(shape) or tensor.numel() != count:
                        raise ValueError('Checkpoint layer shape changed: ' + key)
                    # Prepared VAE Linear values follow the original loader's
                    # FP32 rounding before FP16 autocast, even for FP64 shards.
                    if self.intermediate_dtype is not None:
                        tensor = tensor.to(dtype=self.intermediate_dtype)
                    tensors[name] = tensor.to(dtype=dtype)
                    if tensors[name] is tensor:
                        shared.add(name)
                if self.keep_converted:
                    self._keep_converted(index, tensors, shared)
                yield tensors
        finally:
            tensors.clear()
            for path in paths:
                self.check(path)