Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

71.2 KB
Raw Blame
"""VDN T2VA/FL2VA engine with file-backed weights and bounded layer offloading."""
import gc
import hashlib
import json
import os
import sys
from pathlib import Path
import time
import types

import torch

from .adaln import CachedModulation, TableCache, ScheduleCursor, schedule_embeddings, schedule_timesteps
from .attention import WindowAttention
from .backends import get_backend
from .weights import skeleton
from .paths import base_path, checkpoint_path
from .processes import pin_attention_priority
from .torch_compat import empty_host_cache


def _load_safetensors(path):
    """Materialize a shard without retaining a file mapping.

    ``safetensors.torch.load_file`` uses a memory map by default.  That is
    useful for Linux page-cache reuse, but on Windows the section view also
    reserves system commit for the whole shard.  During model preparation we
    only need the tensors, so read them through the platform-aware streamed
    opener and close the handle before returning the dictionary.  Linux keeps
    its mmap path. Windows reads each shard on a few threads without a
    mapping (``tensor_io.ParallelReads``), twice as fast as safetensors'
    ``pread`` backend on the RTX 5060 Ti machine.
    """
    from .system import windows
    if windows():
        from .tensor_io import ParallelReads
        reads = ParallelReads(path)
        try:
            if not reads.skipped:
                return reads.tensors(sorted(reads.entries, key=lambda name: reads.entries[name][2]))
        finally:
            reads.close()
    from .streamed_weights import _open_safetensors
    with _open_safetensors(path, framework='pt', device='cpu') as handle:
        return {name: handle.get_tensor(name) for name in handle.keys()}


def refined_text_memo_capacity(canvas):
    """Stage prompts of an early-reference canvas plus the top-level prompt and one spare."""
    return len(((canvas or {}).get('early_reference') or {}).get('conditions') or []) + 2


class Engine:
    @torch.no_grad()
    def __init__(self, cache, *, base=None, checkpoint=None,
                 attention='dense', prefetch=True, adaln_cache=False,
                 inference_kernels=False, query_chunk=0, ff_chunk=0, head_chunk=0,
                 offload_refiner=False, attention_cpu_outputs=False, preload_host=False,
                 pin_host_weights=False, adaln_disk_cache=False, projection_chunk=0,
                 grouped_attention_outputs=False, fp8_ff_recompute=False,
                 resident_blocks=0, pin_host_gb=0., cache_refined_text=False, steps=8,
                 window_batch=1, linear_compute='native-fp8', input_cache_dir=None,
                 fp8_gemm='auto', window_varlen=False, varlen_smooth_k=True,
                 task='t2va', stream_weights=False, canvas=None, head_parallelism=1,
                 residual_offload=False, device_backend=None, gpu_allocation_bytes=None,
                 fetch_tables=True, adaln_task=None, step_cache=None):
        from .step_cache import requested
        requested(step_cache)  # Validate the effective selection before loading weights.
        self.device_backend = device_backend if device_backend is not None else get_backend()
        if gpu_allocation_bytes is not None and (type(gpu_allocation_bytes) is not int or gpu_allocation_bytes < 0):
            raise ValueError('GPU allocation limit must be a nonnegative byte count')
        self.gpu_allocation_bytes = gpu_allocation_bytes
        # Compiling ahead of a request never fetches a sampling table: the
        # directories of tables it compiled with zeros (adaln.TableCache).
        self.fetch_tables = fetch_tables
        self.table_placeholders = set()
        if self.device_backend.capabilities.name != 'cuda':
            raise NotImplementedError('Engine model execution currently requires the CUDA backend')
        from .media_request import TASKS
        if task not in TASKS:
            raise ValueError('Unsupported Engine task: ' + str(task))
        self.task = task
        if adaln_task not in (None, task, 'ref2va_audio') and not (task == 'ref2va' and adaln_task == 'ref2va_av'):
            raise ValueError('Unsupported continuation modulation schedule')
        self.adaln_task = adaln_task or task
        if type(residual_offload) is not bool:
            raise ValueError('Residual offload must be boolean')
        if residual_offload and (not projection_chunk or not inference_kernels):
            raise ValueError('Residual offload requires bounded packing and fused inference blocks')
        self.canvas = dict(canvas) if canvas is not None else None
        from .system import weight_cache_headroom, residual_host_headroom
        self.weight_cache_headroom_bytes = weight_cache_headroom(
            streamed=stream_weights, cpu_outputs=attention_cpu_outputs, canvas=self.canvas)
        if residual_offload:
            # The new pageable residual must not compete with retained weights.
            # A reference packed residual is ~0.75 GiB; reserve 1 GiB and scale
            # with video/reference rows. The live RAM guard remains in charge.
            self.weight_cache_headroom_bytes += residual_host_headroom(self.canvas)
        base = base_path() if base is None else Path(base)
        checkpoint = checkpoint_path() if checkpoint is None else Path(checkpoint)
        if linear_compute not in ('native-fp8', 'bf16-weight-only', 'int8'):
            raise ValueError('Unknown linear compute policy')
        if min(query_chunk, ff_chunk, head_chunk, projection_chunk) < 0:
            raise ValueError('Chunk sizes cannot be negative; zero disables chunking')
        if window_batch < 1:
            raise ValueError('Window batch must be positive')
        if type(head_parallelism) is not int or head_parallelism not in (1, 2, 4):
            raise ValueError('Head parallelism must be 1, 2 or 4')
        if head_parallelism > 1 and (resident_blocks != 50 or attention_cpu_outputs or grouped_attention_outputs or not head_chunk):
            raise ValueError('Parallel head groups require all 50 blocks resident and GPU head outputs')
        if attention_cpu_outputs and not head_chunk:
            raise ValueError('CPU attention outputs require head chunking')
        if grouped_attention_outputs and not attention_cpu_outputs:
            raise ValueError('Grouped readouts require CPU attention outputs')
        if pin_host_weights and preload_host:
            raise ValueError('Pinned weights already include host preloading; select one storage mode')
        if pin_host_gb < 0 or resident_blocks < 0:
            raise ValueError('Residency and host cache budgets cannot be negative')
        if pin_host_gb and (pin_host_weights or preload_host):
            raise ValueError('Bounded pinning is a separate storage mode')
        if stream_weights and (pin_host_weights or preload_host):
            raise ValueError('Streamed weights allow a bounded pinned subset, but not host preload or whole-model pinning')
        if cache_refined_text and (not offload_refiner or not projection_chunk):
            raise ValueError('Text refinement caching requires refiner offloading and streamed packing')
        self.base = Path(base)
        self.steps = steps
        self.prefetch = prefetch
        self.cache = Path(cache)
        self.input_cache_dir = input_cache_dir
        self.closed = False
        # (module, '_parameters' or '_buffers', name) moved to host by park().
        self.parked = None
        self.offloader = None
        self.stream_weight_source = None
        self.stream_refiner_count = 0
        self.cache_refined_text = cache_refined_text
        self.refiner_released = False
        self.refined_text_memo = {}
        self.refined_text_memo_capacity = refined_text_memo_capacity(self.canvas)
        started = time.perf_counter()
        manifest = json.loads((self.cache / 'manifest.json').read_text(encoding='utf-8'))
        online_lora = manifest.get('online_lora', {})
        self.online_lora_root = online_lora.get('blocks', {}).get('root', {}).get('file')
        if online_lora:
            if not head_chunk or not inference_kernels:
                raise ValueError('Online LoRA requires bounded attention; select the automatic generation profile')
            ff_chunk = ff_chunk or 2048
        lora_paths = [self.cache / spec['file'] for spec in online_lora.get('blocks', {}).values()]
        from .adaln_assets import SLIM_FORMATS, validate_catalog, restore_projections
        portable_adaln = manifest.get('format') in SLIM_FORMATS
        if portable_adaln:
            # In the slim format these constants are model parameters, not a
            # disposable optimization cache. The effective mode is reported.
            adaln_cache = True
        for group in manifest['groups']:
            if (self.cache / group['file']).stat().st_size != group['bytes']:
                raise ValueError('Incomplete prepared weights')
        self.attention = WindowAttention(attention, query_chunk, window_batch,
                                         window_varlen=window_varlen,
                                         varlen_smooth_k=varlen_smooth_k, device_backend=self.device_backend)
        model = skeleton(self.base, Path(checkpoint))
        validate_catalog(manifest, len(model.transformer_blocks))
        # Locked buffers allocated after the weights must also fit the Windows
        # non-local budget: CPU attention readouts grow with this request's rows.
        from .windows_gpu_memory import NONLOCAL_PIN_RESERVE, nonlocal_readout_bytes, nonlocal_residual_bytes
        readouts = (nonlocal_readout_bytes(self.canvas, model.config.num_attention_heads * model.config.attention_head_dim)
                    if attention_cpu_outputs else 0)
        self.nonlocal_reserve_bytes = NONLOCAL_PIN_RESERVE + readouts + (
            nonlocal_residual_bytes(residual_host_headroom(self.canvas)) if residual_offload else 0)
        # RAM views fill during the first pass; leave what the refine pass will
        # still allocate. A 20 s request that let views take that room drove
        # physical availability to 0.01 GiB when its full-size readouts arrived.
        from .streamed_weights import HOST_VIEW_RESERVE
        self.host_view_reserve_bytes = HOST_VIEW_RESERVE + readouts + (
            residual_host_headroom(self.canvas) if residual_offload else 0)
        if resident_blocks > len(model.transformer_blocks):
            raise ValueError('More resident blocks requested than the model contains')
        # Build the streamed source before binding transformer blocks.  The
        # previous order loaded every non-resident block into CPU tensors and
        # only converted them to zero-storage placeholders when the first
        # LayerOffloader was created at sampling time.  On Windows that made
        # model preparation retain the whole checkpoint's CPU working set (and
        # its pagefile commit) even though stream_weights was selected. On
        # Windows, select the bounded pinned subset one block at a time; a
        # load-all pass would consume the free RAM used to admit that cache.
        # Discard unpinned blocks immediately and restore them on demand.
        streamed_source = None
        from .system import windows
        incremental_pinning = stream_weights and bool(pin_host_gb) and windows()
        prepared_pins = dict(logical=0, reserved=0, seconds=0., views=0, unread=0, unread_bytes=0)
        streamed_refiner_count = (len(model.token_refiner.refiner_blocks)
                                  if stream_weights and offload_refiner and not cache_refined_text else 0)
        if stream_weights and not adaln_cache:
            # SafetensorLayers indexes every streamed shard up front.  Slim
            # caches may restore their AdaLN projections lazily, so ensure
            # those files exist before constructing that index.
            if any(not (self.cache / f'adaln/{index:02d}.safetensors').is_file()
                   for index in range(resident_blocks, len(model.transformer_blocks))):
                restore_projections(self.cache, manifest)
        if stream_weights and (not pin_host_gb or incremental_pinning):
            from .streamed_weights import SafetensorLayers
            streamed_prefixes = ([f'token_refiner.refiner_blocks.{index}.'
                                  for index in range(streamed_refiner_count)]
                                 + [f'transformer_blocks.{index}.'
                                    for index in range(resident_blocks, len(model.transformer_blocks))])
            streamed_paths = ([self.cache / 'root.safetensors'] if streamed_refiner_count else [])
            streamed_paths += [self.cache / f'blocks/{index:02d}.safetensors'
                               for index in range(resident_blocks, len(model.transformer_blocks))]
            if not adaln_cache:
                streamed_paths += [self.cache / f'adaln/{index:02d}.safetensors'
                                   for index in range(resident_blocks, len(model.transformer_blocks))]
            if streamed_prefixes:
                streamed_source = SafetensorLayers(streamed_paths + lora_paths, streamed_prefixes, host_views=True,
                                                   host_view_reserve=self.host_view_reserve_bytes)
                self.stream_weight_source = streamed_source
                # Only source-backed placeholders may remain empty between
                # refinement requests; pinned groups keep their own storage.
                self.stream_refiner_count = streamed_refiner_count
        def unread_weights(index):
            """Layout-only tensors for a streamed block that loading cannot keep.

            A block is read here only for a pinned group or a view. Otherwise
            its values are dropped, and where RAM is that short the file cache
            cannot hold them until the first step either.
            """
            if streamed_source is None or not streamed_source.direct_read or index < resident_blocks:
                return None
            from .offload import loading_retains
            layer = streamed_refiner_count + index - resident_blocks
            budget = max(0, int(pin_host_gb * 1e9) - prepared_pins['reserved']) if incremental_pinning else 0
            if loading_retains(streamed_source, layer, pin_budget_bytes=budget,
                               headroom_bytes=self.weight_cache_headroom_bytes,
                               nonlocal_reserve_bytes=self.nonlocal_reserve_bytes,
                               commit_reserve_bytes=self.gpu_commit_reserve()):
                return None
            tensors = streamed_source.placeholders(layer, self.cache / f'blocks/{index:02d}.safetensors')
            if tensors is not None:
                prepared_pins['unread'] += 1
                prepared_pins['unread_bytes'] += sum(streamed_source.layer_bytes(layer).values())
            return tensors
        def prepare_streamed(block, index, unread=False):
            logical = 0
            if unread:
                # Placeholder values: never pin or view them. The first direct
                # read in sampling adopts a view if RAM allows by then.
                self.device_backend.unload_streamed_layer(block, streamed_source, index)
                return
            if incremental_pinning:
                tick = time.perf_counter()
                logical, reserved = self.device_backend.prepare_streamed_layer(block, streamed_source, index,
                    pin_budget_bytes=max(0, int(pin_host_gb * 1e9) - prepared_pins['reserved']),
                    headroom_bytes=self.weight_cache_headroom_bytes,
                    nonlocal_reserve_bytes=self.nonlocal_reserve_bytes,
                    commit_reserve_bytes=self.gpu_commit_reserve())
                prepared_pins['logical'] += logical
                prepared_pins['reserved'] += reserved
                prepared_pins['seconds'] += time.perf_counter() - tick
            else:
                self.device_backend.unload_streamed_layer(block, streamed_source, index)
            if not logical:
                # Its bytes are still in the file cache; later pins would evict them.
                prepared_pins['views'] += int(streamed_source.adopt(index))
        precision = manifest.get('precision', 'bf16')
        actual_fp8_gemm = self.device_backend.prepare_linears(model, manifest, linear_compute, fp8_gemm)
        root_weights = _load_safetensors(self.cache / 'root.safetensors')
        root_weights = {name: value if offload_refiner and name.startswith('token_refiner.refiner_blocks.') else value.to(self.device_backend.device)
                        for name, value in root_weights.items()}
        result = model.load_state_dict(root_weights, strict=False, assign=True)
        if result.unexpected_keys or any(not key.startswith('transformer_blocks.') for key in result.missing_keys):
            raise ValueError('Unexpected root weight keys')
        del root_weights
        if online_lora:
            from .lora_online import attach_block
            attach_block(model, 'root', self.cache, manifest, _load_safetensors,
                         device='cpu' if offload_refiner else self.device_backend.device)
        if streamed_source is not None and streamed_refiner_count:
            # Refiner weights live in root.safetensors and otherwise remain a
            # second large CPU allocation until LayerOffloader is constructed.
            # Apply the same metadata-only representation used for transformer
            # blocks immediately after binding the root state.
            for index, block in enumerate(model.token_refiner.refiner_blocks):
                prepare_streamed(block, index)
        model.rope.to(self.device_backend.device)
        self.cursor = None
        self.schedule_hook = None
        embeddings = None
        if adaln_cache:
            self.cursor = ScheduleCursor(schedule_timesteps(steps, task=self.adaln_task))
        table_cache = (TableCache(self.cache, manifest['source_id'], steps, task=self.adaln_task, manifest=manifest,
                                 timesteps=self.cursor.timesteps, channels=model.config.hidden_size, device='cpu',
                                 fetch=fetch_tables)
                       if adaln_cache and (adaln_disk_cache or portable_adaln) else None)
        table_cache_hits = 0
        adaln_prepare_seconds = 0.
        self.resident_weight_bytes = 0
        block_bind_seconds = 0.
        upload_submit_seconds = 0.
        block_order = list(range(len(model.transformer_blocks)))
        from .model_load_progress import PreparedBlocks
        block_progress = PreparedBlocks(len(block_order))
        block_progress.report()
        if streamed_source is not None and streamed_source.direct_read:
            # Pins and views are granted in the order blocks are prepared, and
            # the blocks RAM cannot keep are read from the drive every step.
            # Prepared in index order those were the last blocks of the model.
            from .streamed_weights import spread_layer_order
            block_order[resident_blocks:] = [resident_blocks + index for index in
                                             spread_layer_order(len(block_order) - resident_blocks)]
        for prepared, index in enumerate(block_order, 1):
            block = model.transformer_blocks[index]
            prefix = f'transformer_blocks.{index}.'
            adaln_started = time.perf_counter()
            table = table_cache.load(index, steps) if table_cache is not None else None
            cache_hit = table is not None
            if cache_hit:
                block.adaln_proj = CachedModulation(table, self.cursor).to(self.device_backend.device, non_blocking=True)
                table_cache_hits += 1
                del table
            else:
                if not (self.cache / f'adaln/{index:02d}.safetensors').is_file():
                    restore_projections(self.cache, manifest)
                adaln = {name.removeprefix(prefix): value for name, value in
                         _load_safetensors(self.cache / f'adaln/{index:02d}.safetensors').items()}
            if adaln_cache and not cache_hit:
                if embeddings is None:
                    _, embeddings = schedule_embeddings(model, steps, task=self.adaln_task)
                block.load_state_dict({name: value.to(self.device_backend.device) for name, value in adaln.items()}, strict=False, assign=True)
                table = [block.adaln_proj(embedding) for embedding in embeddings]
                block.adaln_proj = CachedModulation(table, self.cursor)
                if table_cache is not None:
                    table_cache.save(index, block.adaln_proj)
                del table, adaln
            adaln_prepare_seconds += time.perf_counter() - adaln_started
            bind_started = time.perf_counter()
            weights = unread_weights(index)
            unread = weights is not None
            if not unread:
                weights = _load_safetensors(self.cache / f'blocks/{index:02d}.safetensors')
            weights = {name.removeprefix(prefix): value for name, value in weights.items()}
            if not adaln_cache:
                weights.update(adaln)
            result = block.load_state_dict(weights, strict=False, assign=True)
            if result.unexpected_keys or any(not key.startswith('adaln_proj.step_') for key in result.missing_keys):
                raise ValueError(f'Incomplete block {index}: {result}')
            del weights
            if online_lora:
                from .lora_online import attach_block
                attach_block(block, index, self.cache, manifest, _load_safetensors)
            if not adaln_cache:
                # ``weights.update(adaln)`` leaves the source dictionary alive
                # until the next iteration. Drop it before replacing this
                # block's storage with a streamed placeholder.
                del adaln
            block_bind_seconds += time.perf_counter() - bind_started
            if index < resident_blocks:
                # Bind one immutable block, enqueue its copies, then prepare
                # the next. Do not retain all 50 CPU mappings before starting
                # the upload. Default-stream ordering protects sampling and
                # the table copies without another GPU weight buffer.
                from .offload import cpu_weights
                self.resident_weight_bytes += sum(value.numel() * value.element_size() for _, value in cpu_weights(block))
                upload_started = time.perf_counter()
                block.to(self.device_backend.device, non_blocking=True)
                upload_submit_seconds += time.perf_counter() - upload_started
            elif streamed_source is not None:
                # Keep only the bounded pinned subset or metadata. Immutable
                # unpinned bytes are reopened by SafetensorLayers on demand,
                # rather than accumulating until the first sampling step.
                prepare_streamed(block, streamed_refiner_count + index - resident_blocks, unread)
            block_progress.complete(adaln_cache=adaln_cache, adaln_reused_blocks=table_cache_hits,
                resident_uploaded_blocks=min(prepared, resident_blocks),
                phase='Load cached blocks and upload resident weights', seconds=time.perf_counter() - started)
        if adaln_cache:
            del embeddings
            self.schedule_hook = model.register_forward_pre_hook(self.cursor.before, with_kwargs=True)
        if any(p.is_meta for p in list(model.parameters()) + list(model.buffers())):
            raise RuntimeError('Some transformer parameters are still on meta')
        model.eval().requires_grad_(False)
        from src.models.hybrid_transform import set_inference_mode, set_softmax_backend
        # Engine supplies every window call, including the official cuDNN/FA4
        # pair. Avoid upstream auto importing an unselected optional FA4 package.
        set_softmax_backend(model, 'flex')
        if inference_kernels:
            from .row_kernels import install as install_row_kernels
            from .linear_kernels import install as install_linear_kernels
            install_row_kernels()
            install_linear_kernels()
        set_inference_mode(model, inference_kernels)
        self.attention.install(model)
        if head_chunk:
            if self.attention is None or not inference_kernels:
                raise ValueError('Head chunks require a window policy and inference kernels')
            from .head_chunk import install_head_chunks
            install_head_chunks(model, self.attention, head_chunk, cpu_outputs=attention_cpu_outputs,
                                projection_chunk=projection_chunk or 1024, grouped_outputs=grouped_attention_outputs,
                                parallelism=head_parallelism)
        if ff_chunk:
            for block in model.transformer_blocks:
                if precision == 'fp8' and linear_compute == 'native-fp8':
                    if not inference_kernels:
                        raise ValueError('FP8 FF chunks require the official inference kernels')
                    self.device_backend.install_chunked_ff(block.ff, ff_chunk, recompute=fp8_ff_recompute)
                    continue
                from .blocks import chunked_ff
                original = block.ff.forward
                block.ff._freevideo_unchunked_forward = original
                block.ff.forward = types.MethodType(chunked_ff(original, ff_chunk), block.ff)
        if projection_chunk:
            from .packing import install_streamed_forward
            install_streamed_forward(model, projection_chunk, residual_offload=residual_offload)
            if inference_kernels:
                from .blocks import install_bounded_blocks
                install_bounded_blocks(model, residual_offload=residual_offload)
        self.transformer = model
        print(json.dumps({'event': 'model_load_phase', 'phase': 'Preparing offloaded weight storage'}), flush=True)
        self.offload_layers = (list(model.token_refiner.refiner_blocks) if offload_refiner and not cache_refined_text else []) + list(model.transformer_blocks)[resident_blocks:]
        if stream_weights and self.stream_weight_source is None:
            from .streamed_weights import SafetensorLayers
            prefixes = ([f'token_refiner.refiner_blocks.{index}.' for index in range(len(model.token_refiner.refiner_blocks))]
                        if offload_refiner and not cache_refined_text else [])
            paths = [self.cache / 'root.safetensors'] if prefixes else []
            prefixes += [f'transformer_blocks.{index}.' for index in range(resident_blocks, len(model.transformer_blocks))]
            paths += [self.cache / f'blocks/{index:02d}.safetensors' for index in range(resident_blocks, len(model.transformer_blocks))]
            if not adaln_cache:
                paths += [self.cache / f'adaln/{index:02d}.safetensors' for index in range(resident_blocks, len(model.transformer_blocks))]
            self.stream_weight_source = SafetensorLayers(paths + lora_paths, prefixes, host_views=True,
                                                         host_view_reserve=self.host_view_reserve_bytes)
        self.host_preload_seconds = 0.
        self.pinned_model_bytes = prepared_pins['logical']
        self.pinned_host_allocated_bytes = 0
        pin_seconds = prepared_pins['seconds']
        pin_order = 'sequential'
        if pin_host_weights or pin_host_gb:
            host_headroom = self.weight_cache_headroom_bytes
            if not incremental_pinning:
                pin_start = time.perf_counter()
                pin_layers = self.offload_layers
                if stream_weights and attention_cpu_outputs and cache_refined_text and not windows():
                    from .streamed_weights import interleaved_layer_order
                    pin_layers = [self.offload_layers[index] for index in interleaved_layer_order(len(self.offload_layers))]
                    pin_order = 'interleaved'
                self.pinned_model_bytes = self.device_backend.pin_layer_weights(pin_layers, max_bytes=int(pin_host_gb * 1e9) if pin_host_gb else None,
                                                           headroom_bytes=host_headroom,
                                                           nonlocal_reserve_bytes=self.nonlocal_reserve_bytes,
                                                           commit_reserve_bytes=self.gpu_commit_reserve())
                pin_seconds = time.perf_counter() - pin_start
            self.pinned_host_allocated_bytes = self.device_backend.host_memory_stats()['allocated_bytes.current']
            print(json.dumps({'event': 'host_weights_pinned', 'seconds': pin_seconds,
                              'logical_bytes': self.pinned_model_bytes,
                              'working_headroom_bytes': host_headroom,
                              'nonlocal_reserve_bytes': self.nonlocal_reserve_bytes,
                              'gpu_commit_reserve_bytes': self.gpu_commit_reserve(),
                              'pin_order': pin_order,
                              'host_allocator_bytes': self.pinned_host_allocated_bytes}), flush=True)
        if preload_host:
            preload_start = time.perf_counter()
            buffer = bytearray(8 * 1024 * 1024)
            for group in manifest['groups']:
                if group['group'].startswith('adaln/') and adaln_cache:
                    continue
                with (self.cache / group['file']).open('rb', buffering=0) as stream:
                    while stream.readinto(buffer):
                        pass
            del buffer
            self.host_preload_seconds = time.perf_counter() - preload_start
            print(json.dumps({'event': 'host_weights_read', 'seconds': self.host_preload_seconds}), flush=True)
        self.device_backend.synchronize()
        self.device_backend.empty_cache()
        self.load_seconds = time.perf_counter() - started
        self.load_breakdown = {'adaln_prepare_seconds': adaln_prepare_seconds, 'pin_weights_seconds': pin_seconds,
                               'block_bind_seconds': block_bind_seconds,
                               'resident_upload_submit_seconds': upload_submit_seconds,
                               'resident_upload_overlaps_block_preparation': True,
                               'weight_cache_headroom_bytes': self.weight_cache_headroom_bytes,
                               'nonlocal_reserve_bytes': self.nonlocal_reserve_bytes,
                               'host_view_reserve_bytes': self.host_view_reserve_bytes,
                               'weight_cache_pin_order': pin_order,
                               'host_pinning_during_load': incremental_pinning,
                               'host_views_adopted_during_load': prepared_pins['views'],
                               'streamed_blocks_unread_during_load': prepared_pins['unread'],
                               'streamed_bytes_unread_during_load': prepared_pins['unread_bytes'],
                               'host_preload_seconds': self.host_preload_seconds,
                               'remaining_load_seconds': self.load_seconds - adaln_prepare_seconds - pin_seconds - self.host_preload_seconds,
                               'scope': 'Host wall time within model load; AdaLN includes table reads or computation and writes.'}
        if table_cache is not None and table_cache.placeholders:
            self.table_placeholders.add(table_cache.root.name)
        fa4_module = sys.modules.get('flash_attn.cute.flash_fwd')
        fa4_path = Path(fa4_module.__file__).resolve() if fa4_module is not None else None
        self.config = {'device_backend': self.device_backend.capabilities.name, 'task': task, 'attention': attention, 'prefetch': prefetch, 'adaln_cache': adaln_cache,
                       'adaln_mode': ('portable-model-asset' if table_cache is not None and table_cache.asset else
                                      'optional-model-asset' if table_cache is not None and table_cache.optional_loaded else
                                      'local-precompute' if adaln_cache else 'original-projections'),
                       'adaln_optional_blocks': len(table_cache.optional_loaded) if table_cache is not None else 0,
                       'adaln_downloaded_files': len(table_cache.optional_downloaded) if table_cache is not None else 0,
                       'adaln_downloaded_bytes': table_cache.optional_download_bytes if table_cache is not None else 0,
                       'adaln_table_identity': table_cache.identity if table_cache is not None else None,
                       'adaln_table_producer': table_cache.producer if table_cache is not None else None,
                       'fa4_dependency': ({'file': str(fa4_path), 'sha256': hashlib.sha256(fa4_path.read_bytes()).hexdigest()}
                                          if fa4_path is not None else None),
                       'precision': precision, 'fp8_linears': len(manifest.get('linears', {})),
                       'linear_compute': linear_compute,
                       'online_lora': online_lora.get('summary'),
                       'fp8_gemm': actual_fp8_gemm,
                       'fp8_scale_granularity': manifest.get('scale_granularity'),
                       'fp8_ff_recompute': fp8_ff_recompute,
                       'resident_blocks': resident_blocks, 'resident_weight_bytes': self.resident_weight_bytes,
                       'stream_weights': stream_weights,
                       'cache_refined_text': cache_refined_text,
                       'pin_host_gb': pin_host_gb,
                       'inference_kernels': inference_kernels, 'query_chunk': query_chunk,
                       'window_batch': window_batch,
                       'ff_chunk': ff_chunk, 'head_chunk': head_chunk, 'head_parallelism': head_parallelism,
                       'offload_refiner': offload_refiner,
                       'attention_cpu_outputs': attention_cpu_outputs,
                       'grouped_attention_outputs': grouped_attention_outputs,
                       'preload_host': preload_host,
                       'pin_host_weights': pin_host_weights,
                       'pinned_model_bytes': self.pinned_model_bytes,
                       'pinned_host_allocated_bytes': self.pinned_host_allocated_bytes,
                       'allocator_config': os.environ.get('PYTORCH_ALLOC_CONF', os.environ.get('PYTORCH_CUDA_ALLOC_CONF', 'default')),
                       'adaln_disk_cache': adaln_disk_cache, 'adaln_table_cache_hits': table_cache_hits,
                       'projection_chunk': projection_chunk,
                       'reuse_block_outputs': bool(projection_chunk and inference_kernels),
                       'residual_offload': residual_offload,
                       'window_varlen': window_varlen,
                       'varlen_smooth_k': varlen_smooth_k if window_varlen else None,
                       'steps': steps, 'source_id': manifest['source_id']}
        if step_cache is not None:
            self.config['step_cache'] = step_cache
        if self.adaln_task != self.task:
            self.config['adaln_task'] = self.adaln_task

    def gpu_commit_reserve(self):
        """Allocator room this request may still reserve, which WDDM will charge to commit."""
        from .system import windows
        if self.gpu_allocation_bytes is None or not windows():
            return 0
        return max(0, self.gpu_allocation_bytes - torch.cuda.memory_reserved())

    @torch.no_grad()
    def sample(self, *args, compute_options=None, **kwargs):
        from .pass_configuration import configuration
        from .refine_schedule import modulation
        with configuration(self, compute_options), modulation(self, kwargs.get('refine_schedule')) as prepared:
            result = self._sample(*args, **kwargs)
            if prepared is not None:
                result[2]['refinement_schedule_preparation'] = prepared
            return result

    @torch.no_grad()
    def _sample(self, conditioning, seed, frames=243, width=1344, height=768, *, final_step_callback=None,
               step_callback=None, sampling_complete_callback=None, initial_latents=None, refine_steps=None, refine_schedule=None,
               allow_smaller_canvas=False, progress_offset=0, progress_total=None,
               pass_cache_budget_bytes=None, gpu_reserve_bytes=0, pass_resident_blocks=None,
               budget_refresh=None, continuation=None):
        from src.inference.render import load_prompt, generate_latents
        from .geometry import geometry, sampler_for_canvas
        from . import frame_preview
        refining = initial_latents is not None
        if continuation is not None:
            from .continuation import validate_route
            # Audio references are admitted below, after inspecting the encoded modalities.
            if continuation.get('video_only'):
                validate_route(task=self.task, anchor=bool(continuation.get('anchor')), original_audio=True)
            elif self.task not in ('ref2va_audio', 'ref2va_av'):
                validate_route(task=self.task, anchor=bool(continuation.get('anchor')))
            if tuple(continuation['video'].shape[-2:]) != (height // 16, width // 16):
                from .continuation import SIZE_MISMATCH
                raise ValueError(SIZE_MISMATCH)
        if refining != (refine_steps is not None):
            raise ValueError('Refinement requires both initial latents and a tail step count')
        if refining:
            from .refine import generate_latents, validate_tail
            if refine_schedule is None:
                validate_tail(self.steps, refine_steps)
        active_steps = refine_steps if refining else self.steps
        canvas = geometry(width, height, frames=frames)
        if self.canvas is not None and any(canvas[key] != self.canvas[key] for key in ('width', 'height', 'frames')):
            smaller = (allow_smaller_canvas and not refining and frames == self.canvas['frames']
                       and width <= self.canvas['width'] and height <= self.canvas['height'])
            if not smaller:
                raise ValueError('This engine was loaded for a different request geometry; reload its memory placement')
        if self.canvas is not None:
            for name in ('reference_video_tokens', 'reference_audio_tokens'):
                if name in self.canvas:
                    canvas[name] = self.canvas[name]
        frames = canvas['frames']
        if self.task.startswith('ref2va') and not refining:
            from .reference_sampler import generate_latents
        generate_latents = sampler_for_canvas(generate_latents, width, height)
        if refining:
            from functools import partial
            generate_latents = partial(generate_latents, initial_latents=initial_latents, refine_steps=refine_steps, refine_schedule=refine_schedule)
        if self.closed:
            raise RuntimeError('Engine is closed')
        from .sampling_progress import SamplingProgress
        progress = SamplingProgress(active_steps, len(self.transformer.transformer_blocks),
                                    offset=progress_offset, total=progress_total)
        progress.start()
        continuation_offset = continuation_audio_offset = 0
        early_receipt = None
        condition_schedule = None
        sampling_kwargs = {}
        if self.task == 't2va' and (self.canvas or {}).get('early_reference'):
            # A zero-step proxy may leave an ordinary text-only request. Keep
            # that sampler and its exact RNG path, but still report the cutoff.
            early_receipt = dict(self.canvas['early_reference'], steps=[dict(step=i,
                condition='base', condition_start_step=0, reference_video_rows=0, reference_audio_rows=0,
                text_rows=self.canvas['early_reference']['conditions'][0]['text_tokens']) for i in range(active_steps)])
        if self.task.startswith('ref2va'):
            from .media_conditioning import describe
            value = torch.load(conditioning, map_location='cpu', weights_only=True)
            info = describe(value, width, height, frames)
            if info['task'] != self.task:
                raise ValueError('Reference modalities changed the precomputed schedule')
            if 'early_reference' in info:
                from .media_conditioning import condition_at
                early_receipt = dict(info['early_reference'], steps=[])
                if value['early_reference']['conditions']:
                    sampling_kwargs['timestep_task'] = self.task
                if refining:
                    value = condition_at(value)
                    row = info['early_reference']['conditions'][-1]
                    early_receipt['steps'] = [dict(step=i, condition='base', condition_start_step=row['start_step'],
                        reference_video_rows=row['reference_video_tokens'], reference_audio_rows=row['reference_audio_tokens'],
                        text_rows=row['text_tokens']) for i in range(active_steps)]
                else:
                    condition_schedule = [dict(stage) for stage in value['early_reference']['conditions']]
                    sampling_kwargs.update(condition_schedule=condition_schedule, condition_trace=early_receipt['steps'])
            conditions = value.get('references') or None
            if continuation is not None:
                from .continuation import UNSUPPORTED_MEDIA, encoded_references
                from .fixed_track import split
                # A clip used as is fills the part's own sound; only the other rows are references.
                references, fixed = split(conditions or [], frames, self.device_backend.device)
                if (fixed is None) == bool(continuation.get('video_only')):
                    raise ValueError(UNSUPPORTED_MEDIA)
                del fixed
                if not continuation.get('video_only') or references:
                    if encoded_references(references, self.task, bool(continuation.get('anchor'))):
                        if continuation.get('video_only'):
                            raise ValueError(UNSUPPORTED_MEDIA)
                        validate_route(task=self.task, sound=True, anchor=bool(continuation.get('anchor')),
                                       media=dict(references=[dict(kind=row['kind']) for row in references]))
                # Ref2VA packs independent-size references before generated
                # video rows in both passes. Their latents do not need the
                # keyframe resize used by first/last-frame conditioning.
                continuation_offset = info['reference_video_tokens']
                continuation_audio_offset = info['reference_audio_tokens']
            prompt, tags = value['prompt_embeds'].to(self.device_backend.device, torch.bfloat16), value['text_token_tags']
        else:
            prompt, tags, conditions = load_prompt(str(conditioning), self.device_backend.device)
            if continuation is not None and conditions:
                from .continuation import UNSUPPORTED_MEDIA
                raise ValueError(UNSUPPORTED_MEDIA)
            from .keyframes import validate_conditioning
            source_canvas = self.canvas if allow_smaller_canvas and self.canvas is not None else canvas
            validate_conditioning(prompt, tags, conditions, self.task,
                                  source_canvas['width'], source_canvas['height'])
            if conditions and (width, height) != (source_canvas['width'], source_canvas['height']):
                from .keyframes import first_pass_conditions
                conditions = first_pass_conditions(conditions, source_canvas, canvas)
                validate_conditioning(prompt, tags, conditions, self.task, width, height)
            if conditions:
                canvas['reference_video_tokens'] = len(conditions[0]) * (width // 32) * (height // 32)
        if self.cursor is not None:
            self.cursor.reset(self.steps - refine_steps if refining and refine_schedule is None else 0)
        # DecoderReadAhead is CPU/file-cache only, so start it while the last
        # two denoising steps are still running. Starting at the final step
        # left slow SSDs with too little overlap to hide a cold VAE load;
        # beginning earlier is still bounded by DecoderReadAhead's live RAM
        # check and never allocates CUDA or pinned memory. Keep the trigger
        # timing in the sample receipt: it lets the report distinguish a
        # reader that had enough sampling time to finish from one that was
        # merely started before decode. The one-step case keeps the immediate
        # callback below.
        prefetch_step = max(1, active_steps - 2)
        prefetch_timing = dict(trigger_step=None, steps_remaining=None,
                               estimated_remaining_seconds=None,
                               trigger_elapsed_seconds=None,
                               drive_reads_last_step=None, drive_layers=None)
        # Layers the step just completed read from the weight files, and
        # offloaded layers that still depend on those files; both unknown
        # until this pass completes a step. Sampling leaves the drive to the
        # reader only when both are zero: a layer served by a cached device
        # copy goes back to its file once that copy is released.
        self.step_drive_reads = self.drive_layers = None
        self.drive_idle = False
        drive_reads = [0]

        def trigger_prefetch():
            if final_step_callback is None or prefetch_timing['trigger_step'] is not None:
                return
            completed = len(step_seconds)
            durations = list(step_seconds)
            average = (sum(durations) / len(durations)) if durations else None
            remaining = max(0, active_steps - completed)
            prefetch_timing.update(
                trigger_step=completed,
                steps_remaining=remaining,
                estimated_remaining_seconds=(average * remaining
                                             if average is not None else None),
                trigger_elapsed_seconds=time.perf_counter() - started,
                drive_reads_last_step=self.step_drive_reads, drive_layers=self.drive_layers)
            final_step_callback()

        class StepTimes(list):
            def append(self, value):
                if prefix is not None:
                    prefix.restore_video()
                super().append(value)
                if watch is not None:
                    watch.step_done(pass_cache)
                count = getattr(offloader, 'drive_reads', None)
                total = count() if count is not None else None
                if total is None:
                    owner.step_drive_reads = None
                else:
                    owner.step_drive_reads, drive_reads[0] = total - drive_reads[0], total
                cached = offloader.cached_bytes() if budget_refresh is not None else 0
                peak = owner.device_backend.max_memory_reserved()
                if budget_refresh is not None:
                    # Cache tensors are idle here; no forward hook or transfer
                    # owns them. Drop optional residency before lowering a cap.
                    # CUDA's peak is cumulative across this pass. Remember the
                    # largest cache too, so releasing a cache does not turn its
                    # historical peak into fictional activation workspace.
                    cache_peak = max(pass_cache.get('peak_cache_bytes', 0), cached)
                    pass_cache['peak_cache_bytes'] = cache_peak
                    base_peak = max(pass_cache.get('workspace_peak_bytes', 0), peak - cache_peak)
                    pass_cache['workspace_peak_bytes'] = base_peak
                    live_budget = budget_refresh('sample', lambda limit: offloader.cache_between_steps(
                        min(cached, max(0, limit - base_peak - 512 * 2**20)), allow_growth=False),
                        workspace=base_peak)
                    if watch is not None and type(live_budget) is int:
                        # After the refresh: a cut since the last layer's read
                        # is in `live_budget`. The planned group's workspace
                        # replaces the narrower one's, and the cache this pass
                        # grew beside the narrower group gives back what that
                        # workspace needs.
                        wide = watch.restore_if_room(live_budget)
                        if wide is not None:
                            base_peak = max(base_peak, wide)
                            pass_cache['workspace_peak_bytes'] = base_peak
                            offloader.cache_between_steps(
                                min(offloader.cached_bytes(), max(0, live_budget - base_peak - 512 * 2**20)),
                                allow_growth=False)
                    if len(self) < active_steps:
                        from .two_pass import pass_cache_allowance
                        from .system import system_memory
                        cached = offloader.cached_bytes()
                        free, _ = owner.device_backend.memory_info()
                        commit = system_memory().get('commit_available_bytes')
                        from .two_pass import reusable_cache_bytes
                        held = reusable_cache_bytes(cached, owner.device_backend.memory_reserved(), base_peak)
                        # Other applications' memory counts once the device is
                        # full: growing into it made Windows cut this process's
                        # budget below its own workspace. Unread: no extra bound.
                        room = getattr(budget_refresh, 'device_headroom', None)
                        headroom = room() if callable(room) else None
                        if type(headroom) is not int:
                            headroom = None
                        allowance = pass_cache_allowance(live_budget, base_peak, free + held,
                            gpu_reserve_bytes, None if commit is None else commit + held,
                            device_free=None if headroom is None else headroom + held)
                        pass_cache.update(gpu_budget_bytes=live_budget, measured_peak_reserved_bytes=peak,
                            live_free_bytes=free, commit_available_bytes=commit, admitted_bytes=allowance,
                            device_headroom_bytes=headroom)
                        # New planes fill during the next step and only save a
                        # transfer on a subsequent step. The two-step refine
                        # pass therefore cannot benefit from a late fill.
                        offloader.cache_between_steps(allowance, allow_growth=len(self) + 1 < active_steps)
                elif (len(self) == 1 and pass_cache_budget_bytes is not None and allow_smaller_canvas
                        and owner.canvas is not None and canvas['video_tokens'] < owner.canvas['video_tokens']):
                    from .two_pass import pass_cache_allowance
                    from .system import system_memory
                    free, _ = owner.device_backend.memory_info()
                    peak = owner.device_backend.max_memory_reserved()
                    commit = system_memory().get('commit_available_bytes')
                    allowance = pass_cache_allowance(pass_cache_budget_bytes, peak, free, gpu_reserve_bytes, commit)
                    pass_cache.update(measured_peak_reserved_bytes=peak, live_free_bytes=free,
                                      commit_available_bytes=commit, admitted_bytes=allowance)
                    target = (max(0, pass_resident_blocks - owner.config['resident_blocks'])
                              if pass_resident_blocks is not None else None)
                    offloader.cache_between_steps(allowance, max_layers=target,
                        allow_growth=len(self) + 1 < active_steps)
                if owner.config.get('residual_offload'):
                    residual_steps.append(dict(owner.transformer._freevideo_residual_stats))
                # After the cache changes above: what the remaining steps need.
                waiting = getattr(offloader, 'drive_layers', None)
                owner.drive_layers = waiting() if waiting is not None else None
                owner.drive_idle = owner.step_drive_reads == 0 and owner.drive_layers == 0
                progress.complete(value)
                if step_callback is not None:
                    step_callback(value)
                if active_steps > 1 and len(self) == prefetch_step:
                    trigger_prefetch()
                elif active_steps > 1 and 1 < len(self) < prefetch_step and owner.drive_idle:
                    # A drive too slow to read the decoder in the last two steps
                    # starts sooner. The first step carries compilation, so the
                    # remaining time comes from the steps after it.
                    due = getattr(final_step_callback, 'due', None)
                    if due is not None and prefetch_timing['trigger_step'] is None:
                        steady = list(self)[1:]
                        if due(sum(steady) / len(steady) * (active_steps - len(self))):
                            trigger_prefetch()

        owner = self
        step_seconds = StepTimes()
        pass_cache = dict(gpu_budget_bytes=pass_cache_budget_bytes, admitted_bytes=0,
                          requested_resident_blocks=pass_resident_blocks)
        residual_steps = []
        from .activation_staging import refusals_mark, refusals_since
        refusals_before = refusals_mark()
        attention_before = dict(self.attention.backend_calls) if self.attention is not None else None
        gemm_before = None
        if self.config.get('fp8_gemm'):
            from .fp8_gemm import execution_counts
            gemm_before = execution_counts()
        parallel_before = {name: getattr(self.attention, name, 0)
                           for name in ('parallel_head_calls', 'parallel_head_warmups')}
        self.device_backend.reset_peak_memory_stats()
        self.device_backend.synchronize()
        started = time.perf_counter()
        refinement = None
        checkpoint_seconds = 0.
        # A budget cut inside a step (Windows): see step_pressure. The watch is
        # the innermost context, so a narrowed group is put back before the
        # offloader and the progress hooks leave.
        from contextlib import nullcontext
        from .continuation import sampling
        planned, watch = self.config, None
        source = getattr(budget_refresh, 'pressure', None)
        if source is not None:
            from .step_pressure import LayerWatch
            watch = LayerWatch(self, source, pass_cache=pass_cache)
        try:
            if self.cache_refined_text:
                if condition_schedule:
                    # An engine serves several requests; size the memo for this
                    # request's stage prompts plus the top-level prompt.
                    self.refined_text_memo_capacity = max(self.refined_text_memo_capacity, len(condition_schedule) + 2)
                    for stage in condition_schedule:
                        # Prepare the final subset too, even when K >= steps:
                        # refinement always uses it. Keep host refiner weights
                        # until every distinct prompt has been prepared.
                        if stage['start_step'] >= self.steps and stage is not condition_schedule[-1]:
                            continue
                        self._prepare_refined_text(stage['value']['prompt_embeds'].to(self.device_backend.device, torch.bfloat16),
                                                   release_weights=False)
                        # Share the memo's CPU storage; another device-to-host
                        # copy here would double every stage's retained text.
                        stage['refined_text'] = next(reversed(self.refined_text_memo.values()))
                refinement = self._prepare_refined_text(prompt)
            if active_steps == 1:
                trigger_prefetch()
            with progress.layers(self.transformer.transformer_blocks), self.device_backend.make_offloader(
                    self.offload_layers, prefetch=self.prefetch, weight_source=self.stream_weight_source) as offloader, sampling(
                    self.transformer, continuation, seed, self.device_backend.device,
                    **({'video_offset': continuation_offset} if continuation_offset else {}),
                    **({'audio_offset': continuation_audio_offset} if continuation_audio_offset else {})) as prefix, \
                    (watch.layers(self.transformer.transformer_blocks) if watch is not None else nullcontext()), \
                    frame_preview.observing(latent_frames=canvas.get('latent_frames'), frame_h=height // 32,
                                            frame_w=width // 32, step_offset=progress_offset,
                                            total=progress_total if progress_total is not None else active_steps,
                                            seconds=canvas.get('seconds')):
                from .step_cache import begin_pass, end_pass
                if not refining:
                    disabled = ('residual_offload' if self.config.get('residual_offload') and self.config.get('step_cache') == 'fbcache' else
                                'streamed_forward_unavailable' if not self.config.get('projection_chunk') else None)
                    begin_pass(self.transformer, active_steps, self.config.get('step_cache'),
                               offloader=offloader if self.offload_layers else None, disabled_reason=disabled)
                try:
                    latents, audio = generate_latents(self.transformer, prompt, tags, frames,
                                                    self.steps, seed, self.device_backend.device, step_seconds=step_seconds,
                                                    conditions=conditions, **sampling_kwargs)
                finally:
                    step_cache = end_pass(self.transformer)
                if prefix is not None:
                    latents, audio = prefix.finish(latents, audio)
                stats = offloader.stats()
                # StepTimes.append closes over the live offloader. Retaining
                # that list subclass in a receipt also retains its pinned
                # weights through VAE loading. Reports own plain numbers only.
                sampled = {'sample_seconds': time.perf_counter() - started, 'step_seconds': list(step_seconds),
                           'offload': stats, 'residual_offload_steps': residual_steps,
                           'locked_copy_refusals': refusals_since(refusals_before),
                           'pass_cache_admission': pass_cache,
                           'compute_configuration': {key: planned.get(key) for key in
                               ('head_chunk', 'window_batch', 'ff_chunk', 'projection_chunk', 'prefetch',
                                'attention_cpu_outputs', 'grouped_attention_outputs', 'residual_offload',
                                'fp8_ff_recompute', 'resident_blocks', 'stream_weights', 'pin_host_gb')},
                           'decoder_prefetch': dict(prefetch_timing),
                           'geometry': canvas, 'conditioning_shape': list(prompt.shape),
                           'attention_backend_calls': ({key: count - attention_before[key]
                                for key, count in self.attention.backend_calls.items()} if attention_before is not None else None),
                           'fp8_kernel_calls': ({key: count - gemm_before[key]
                                for key, count in execution_counts().items()} if gemm_before is not None else None),
                           'head_execution': dict(requested_parallelism=self.config.get('head_parallelism', 1),
                               **{name: getattr(self.attention, name, 0) - count
                                  for name, count in parallel_before.items()}),
                           'text_refinement': refinement, 'step_cache': step_cache,
                           'torch_peak_allocated_bytes': self.device_backend.max_memory_allocated(),
                           'torch_peak_reserved_bytes': self.device_backend.max_memory_reserved()}
                if watch is not None:
                    sampled['step_pressure'] = watch.result()
                    sampled['torch_peak_allocated_bytes'], sampled['torch_peak_reserved_bytes'] = watch.peaks(
                        sampled['torch_peak_allocated_bytes'], sampled['torch_peak_reserved_bytes'])
                if refining:
                    sampled['refinement'] = dict(base_steps=self.steps, steps=refine_steps,
                        start_index=1 if refine_schedule else self.steps - refine_steps, restart_seed=seed,
                        schedule=refine_schedule or 'original-tail',
                        audio_policy='first_pass_preserved_with_audio_clock_conditioning')
                if early_receipt is not None:
                    sampled['early_reference'] = early_receipt
                # The final latents must be durable before exiting the weight
                # offloader: its cleanup can fail after all NFE have completed.
                if sampling_complete_callback is not None:
                    tick = time.perf_counter()
                    sampling_complete_callback(latents, audio, sampled)
                    checkpoint_seconds = time.perf_counter() - tick
        finally:
            self.transformer._freevideo_refined_text = None
            if condition_schedule and self.cache_refined_text and not self.stream_refiner_count:
                # Also release a partially prepared batch when refinement fails.
                self.transformer.token_refiner.refiner_blocks.to('meta')
                self.refiner_released = True
        self.device_backend.synchronize()
        elapsed = time.perf_counter() - started - checkpoint_seconds
        self.device_backend.empty_cache()
        allocated, reserved = self.device_backend.max_memory_allocated(), self.device_backend.max_memory_reserved()
        if watch is not None:
            sampled['step_pressure'] = watch.result()
            allocated, reserved = watch.peaks(allocated, reserved)
        sampled.update(sample_seconds=elapsed, torch_peak_allocated_bytes=allocated, torch_peak_reserved_bytes=reserved)
        return latents, audio, sampled

    @torch.no_grad()
    def _prepare_refined_text(self, prompt, *, release_weights=True):
        """The upstream text refiner has no timestep or video input.

        Keep a bounded LRU of CPU results for the stage and top-level prompts.
        A batch retains host weights until its last prompt, then discards them.
        Sampling clears the active device tensor even when preparation fails.
        """
        from .fp8 import input_dtype
        started = time.perf_counter()
        refiner = self.transformer.token_refiner.refiner_blocks
        if prompt.ndim not in (2, 3) or (prompt.ndim == 3 and prompt.shape[0] != 1):
            raise ValueError('Expected one text prompt for refinement')
        cpu = prompt.detach().to('cpu').contiguous()
        cache_key = str(cpu.dtype) + str(list(cpu.shape)) + hashlib.sha256(cpu.view(torch.uint8).numpy().tobytes()).hexdigest()
        del cpu
        # Pass-local tiling (pass_configuration) rewires only the video blocks
        # and this engine's weights are fixed, so the second pass of a request
        # reuses the first pass's tensor instead of reloading the refiner
        # weights to compute the same values again.
        memo = (cache_key, tuple(self.config.get(k) for k in ('attention', 'linear_compute', 'fp8_gemm', 'inference_kernels')))
        def remember(text):
            self.refined_text_memo[memo] = text.detach().cpu()
            while len(self.refined_text_memo) > self.refined_text_memo_capacity:
                del self.refined_text_memo[next(iter(self.refined_text_memo))]

        def release():
            if release_weights:
                if not self.stream_refiner_count:
                    refiner.to('meta')
                self.refiner_released = True

        if memo in self.refined_text_memo:
            cpu_text = self.refined_text_memo.pop(memo)
            self.refined_text_memo[memo] = cpu_text
            text = cpu_text.to(self.device_backend.device)
            self.transformer._freevideo_refined_text = text
            release()
            return dict(seconds=time.perf_counter() - started, shape=list(text.shape), cache_hit=False,
                        previous_pass=True, refiner_weights_released=self.refiner_released)
        cache, note = None, None
        if self.input_cache_dir:
            from .input_cache import InputCache, digest
            from .storage import fingerprint
            try:
                implementation = Path(sys.modules[type(self.transformer).__module__].__file__)
                identity = dict(manifest=digest(self.cache / 'manifest.json'), weights=fingerprint(self.cache / 'root.safetensors'),
                    engine=digest(Path(__file__)), upstream=digest(implementation), torch=str(torch.__version__),
                    arithmetic={k: self.config[k] for k in ('attention', 'linear_compute', 'fp8_gemm', 'inference_kernels', 'head_chunk', 'projection_chunk')},
                    **self.device_backend.arithmetic_identity())
                cache = InputCache(identity, root=self.input_cache_dir, kind='refined-text')
                found = cache.lookup(cache_key)
                if found:
                    path, entry = found
                    text = torch.load(path, map_location=self.device_backend.device, weights_only=True)
                    expected_shape = [1, prompt.shape[-2], self.transformer.context_embedder.out_features]
                    if (isinstance(text, torch.Tensor) and list(text.shape) == expected_shape
                            and str(text.dtype) == entry['metrics'].get('dtype') and bool(torch.isfinite(text).all())):
                        release()
                        self.transformer._freevideo_refined_text = text
                        remember(text)
                        return dict(seconds=time.perf_counter() - started, shape=list(text.shape), cache_hit=True,
                                    refiner_weights_released=self.refiner_released, cache_receipt=str(cache.receipt(cache_key)))
                    del text
            except (OSError, ValueError, KeyError, RuntimeError) as error:
                note = str(error)
        if self.refiner_released and not self.stream_refiner_count:
            prefix = 'token_refiner.refiner_blocks.'
            values = {name.removeprefix(prefix): tensor for name, tensor in
                      _load_safetensors(self.cache / 'root.safetensors').items() if name.startswith(prefix)}
            if self.online_lora_root:
                values.update({name.removeprefix(prefix): tensor for name, tensor in
                               _load_safetensors(self.cache / self.online_lora_root).items()})
            refiner.load_state_dict(values, strict=True, assign=True)
            del values
            self.refiner_released = False
        # load_prompt returns [tokens, width]; generate_latents adds the batch
        # dimension before the original transformer call.
        pin_attention_priority()
        prompt = prompt.unsqueeze(0) if prompt.ndim == 2 else prompt
        text = self.transformer.context_embedder(prompt.to(input_dtype(self.transformer.context_embedder)))
        with self.device_backend.make_offloader(refiner, prefetch=False,
                            weight_source=self.stream_weight_source if self.stream_refiner_count else None) as loader:
            text = self.transformer.token_refiner(text)
            stats = loader.stats()
        del loader
        self.device_backend.synchronize()
        release()
        self.transformer._freevideo_refined_text = text
        remember(text)
        saved = False
        if cache is not None:
            try:
                import uuid
                cache.root.mkdir(parents=True, exist_ok=True)
                temporary = cache.root / ('refined-' + uuid.uuid4().hex + '.partial')
                torch.save(self.refined_text_memo[memo], temporary)
                saved = cache.put(cache_key, temporary, dict(success=True, shape=list(text.shape), dtype=str(text.dtype)), move=True)
                note = cache.note or note
            except (OSError, ValueError) as error:
                note = str(error)
        if release_weights:
            gc.collect()
            self.device_backend.empty_cache()
            empty_host_cache(torch)
        return {'seconds': time.perf_counter() - started, 'shape': list(text.shape),
                'refiner_weights_released': self.refiner_released, 'offload': stats, 'cache_hit': False, 'cache_saved': saved, 'cache_note': note}

    def device_tensors(self):
        """The registered tensors this engine keeps on the GPU, or None.

        None when two of them share storage or one is a view of a larger
        storage: a per-tensor copy would duplicate that storage on its way back.
        """
        found, storages = [], set()
        device = torch.device(self.device_backend.device).type
        for module in self.transformer.modules():
            for kind in ('_parameters', '_buffers'):
                for name, tensor in getattr(module, kind).items():
                    if tensor is None or tensor.device.type != device:
                        continue
                    storage = tensor.untyped_storage()
                    if storage.data_ptr() in storages or storage.nbytes() != tensor.numel() * tensor.element_size():
                        return None
                    storages.add(storage.data_ptr())
                    found.append((module, kind, name))
        return found

    @torch.no_grad()
    def park(self):
        """Move the weights this engine keeps on the GPU to ordinary host memory.

        Streamed, pinned and mapped weights stay where they are, so the next
        request with the same plan copies only these back instead of reading,
        binding and pinning every block again. Copies are exact, so the output
        is unchanged. Returns the bytes moved, or None when this engine cannot
        be parked; the caller then releases it as before.
        """
        if self.closed or self.parked is not None:
            return None
        found = self.device_tensors()
        if found is None:
            return None
        moved = 0
        for module, kind, name in found:
            tensor = getattr(module, kind)[name]
            moved += tensor.numel() * tensor.element_size()
            # .data keeps every reference to this tensor object valid, as
            # Module.to does for parameters.
            tensor.data = tensor.data.to('cpu')
        self.parked = found
        gc.collect()
        self.device_backend.empty_cache()
        return moved

    @torch.no_grad()
    def unpark(self):
        """Copy parked weights back to the GPU; returns the bytes moved."""
        if self.parked is None:
            return 0
        moved = 0
        for module, kind, name in self.parked:
            tensor = getattr(module, kind)[name]
            moved += tensor.numel() * tensor.element_size()
            tensor.data = tensor.data.to(self.device_backend.device)
        self.parked = None
        self.device_backend.synchronize()
        return moved

    def close(self):
        if self.closed:
            return
        self.parked = None
        if self.schedule_hook is not None:
            self.schedule_hook.remove()
        self.transformer = None
        self.refined_text_memo.clear()
        self.offload_layers = []
        if self.stream_weight_source is not None:
            self.stream_weight_source.release_host_views()
        self.stream_weight_source = None
        self.attention = None
        self.closed = True
        gc.collect()
        self.device_backend.empty_cache()
        empty_host_cache(torch)