Skip to content

Looking Ahead: Modeling Future Evolution in Autoregressive Video Generation

29.8 KB
Raw Blame
"""Ordinary causal TI2V and shared training-preview pipeline."""

import logging
import math

import torch
import torch.distributed as dist
from tqdm import tqdm

from src.utils.wan_wrapper import WanDiffusionWrapper, WanTextEncoder, WanVAEWrapper
from src.wan.utils.fm_solvers import (
    FlowDPMSolverMultistepScheduler,
    get_sampling_sigmas,
    retrieve_timesteps,
)
from src.wan.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler

logger = logging.getLogger(__name__)


def validate_condition_only_inference_inputs(
    *, enabled, initial_latent, generated_frames, num_frame_per_block, consumer_name
):
    if initial_latent is None or initial_latent.ndim != 5 or initial_latent.shape[1] < 1:
        raise ValueError(f"{consumer_name}: context must have shape [B,T,C,H,W].")
    if enabled and (initial_latent.shape[1] - 1) % num_frame_per_block:
        raise ValueError(f"{consumer_name}: condition-only context must satisfy T=1+M*F.")
    if generated_frames < 0 or generated_frames % num_frame_per_block:
        raise ValueError(f"{consumer_name}: generated tail must be chunk-aligned.")


class CausalDiffusionInferencePipeline(torch.nn.Module):
    def __init__(self, args, device, generator=None, text_encoder=None, vae=None):
        super().__init__()
        self.args = args
        self.condition_only_first_frame = args.condition_only_first_frame
        self.generator = (
            generator
            if generator is not None
            else WanDiffusionWrapper(
                model_name=args.model_name,
                latent_shape=args.image_or_video_shape,
                load_diffusion_pretrained=args.load_diffusion_pretrained,
                initialization_seed=int(args.seed),
                **args.model_kwargs,
            ).to(device)
        )
        self.text_encoder = (
            text_encoder if text_encoder is not None else WanTextEncoder(args.model_name).to(device)
        )
        self.vae = vae if vae is not None else WanVAEWrapper(args.model_name).to(device)
        self.num_train_timesteps = 1000
        self.sampling_steps = args.sampling_steps
        self.sample_solver = args.sample_solver
        self.shift = args.timestep_shift
        model = self.generator.model
        self.num_heads = model.config.num_heads
        self.head_dim = model.config.dim // self.num_heads
        self.text_len = model.config.text_len
        self.num_transformer_blocks = model.config.num_layers
        self.frame_seq_length = self.generator.frame_seq_len
        self.kv_cache_pos = self.kv_cache_neg = None
        self.crossattn_cache_pos = self.crossattn_cache_neg = None
        self.num_frame_per_block = args.num_frame_per_block
        self.independent_first_frame = args.independent_first_frame
        model.num_frame_per_block = self.num_frame_per_block
        model.independent_first_frame = self.independent_first_frame
        self.local_attn_size = model.local_attn_size
        logger.info("KV inference with %d frames per block", self.num_frame_per_block)

    def _iter_crossattn_cache_lists(self):
        yield self.crossattn_cache_pos
        yield self.crossattn_cache_neg

    @staticmethod
    def _reset_crossattn_cache_list(cache_list):
        for block_cache in cache_list:
            block_cache["is_init"] = False

    def inference(
        self,
        noise: torch.Tensor,
        text_prompts: list[str],
        initial_latent: torch.Tensor | None = None,
        start_frame_index: int | None = 0,
        return_latents: bool = False,
        return_video: bool = True,
    ) -> torch.Tensor:
        """Generate a causal video window on GPU.

        Args:
            noise: Initial noise for generated latent frames,
                ``[B, T_noise, C, H, W]``. It excludes ``initial_latent``.
            text_prompts: List of ``B`` prompts.
            initial_latent: Required context latents
                ``[B, T_init, C, H, W]``. ``T_init=1`` is I2V; larger values
                are video extension/context.
            start_frame_index: Absolute latent-frame index of the first frame
                in this window. Used only for RoPE/KV positions; cache-local
                indices still begin at zero.
            return_latents: If true, return ``(video, output_latents)``.
            return_video: If false, skip VAE decode and return latents only.

        Returns:
            Pixel video ``[B, T_pixel, 3, H_pixel, W_pixel]`` in ``[0, 1]``;
            latents ``[B, T_init + T_noise, C, H, W]`` when requested; or
            latents alone when ``return_video=False``.

        Compatibility:
            Text plus supplied C0 is required. The generated tail must contain
            a whole number of chunks.
        """
        validate_condition_only_inference_inputs(
            enabled=bool(getattr(self, "condition_only_first_frame", False)),
            initial_latent=initial_latent,
            generated_frames=int(noise.shape[1]),
            num_frame_per_block=self.num_frame_per_block,
            consumer_name="CausalDiffusionInferencePipeline.inference",
        )
        batch_size, num_frames, num_channels, height, width = noise.shape
        assert num_frames % self.num_frame_per_block == 0
        num_blocks = num_frames // self.num_frame_per_block
        num_input_frames = initial_latent.shape[1]
        num_output_frames = num_frames + num_input_frames
        conditional_dict = self.text_encoder(text_prompts=text_prompts)
        unconditional_dict = self.text_encoder(
            text_prompts=[self.args.negative_prompt] * len(text_prompts)
        )
        output = torch.zeros(
            [batch_size, num_output_frames, num_channels, height, width],
            device=noise.device,
            dtype=noise.dtype,
        )
        if self.kv_cache_pos is None:
            self._initialize_kv_cache(batch_size=batch_size, dtype=noise.dtype, device=noise.device)
            self._initialize_crossattn_cache(
                batch_size=batch_size, dtype=noise.dtype, device=noise.device
            )
        else:
            for cache_list in self._iter_crossattn_cache_lists():
                self._reset_crossattn_cache_list(cache_list)
            for block_index in range(len(self.kv_cache_pos)):
                self.kv_cache_pos[block_index]["global_end_index"] = torch.tensor(
                    [0], dtype=torch.long, device=noise.device
                )
                self.kv_cache_pos[block_index]["local_end_index"] = torch.tensor(
                    [0], dtype=torch.long, device=noise.device
                )
                self.kv_cache_neg[block_index]["global_end_index"] = torch.tensor(
                    [0], dtype=torch.long, device=noise.device
                )
                self.kv_cache_neg[block_index]["local_end_index"] = torch.tensor(
                    [0], dtype=torch.long, device=noise.device
                )
        current_start_frame = start_frame_index
        cache_start_frame = 0
        timestep = torch.ones([batch_size, 1], device=noise.device, dtype=torch.int64) * 0
        if self.independent_first_frame:
            assert (num_input_frames - 1) % self.num_frame_per_block == 0
            num_input_blocks = (num_input_frames - 1) // self.num_frame_per_block
            output[:, :1].copy_(initial_latent[:, :1].detach())
            self.generator(
                noisy_image_or_video=initial_latent[:, :1],
                conditional_dict=conditional_dict,
                timestep=timestep * 0,
                kv_cache=self.kv_cache_pos,
                crossattn_cache=self.crossattn_cache_pos,
                current_start=current_start_frame * self.frame_seq_length,
                cache_start=cache_start_frame * self.frame_seq_length,
            )
            self.generator(
                noisy_image_or_video=initial_latent[:, :1],
                conditional_dict=unconditional_dict,
                timestep=timestep * 0,
                kv_cache=self.kv_cache_neg,
                crossattn_cache=self.crossattn_cache_neg,
                current_start=current_start_frame * self.frame_seq_length,
                cache_start=cache_start_frame * self.frame_seq_length,
            )
            current_start_frame += 1
            cache_start_frame += 1
        else:
            assert num_input_frames % self.num_frame_per_block == 0
            num_input_blocks = num_input_frames // self.num_frame_per_block
        for _ in range(num_input_blocks):
            current_ref_latents = initial_latent[
                :, cache_start_frame : cache_start_frame + self.num_frame_per_block
            ]
            output[:, cache_start_frame : cache_start_frame + self.num_frame_per_block].copy_(
                current_ref_latents.detach()
            )
            self.generator(
                noisy_image_or_video=current_ref_latents,
                conditional_dict=conditional_dict,
                timestep=timestep * 0,
                kv_cache=self.kv_cache_pos,
                crossattn_cache=self.crossattn_cache_pos,
                current_start=current_start_frame * self.frame_seq_length,
                cache_start=cache_start_frame * self.frame_seq_length,
            )
            self.generator(
                noisy_image_or_video=current_ref_latents,
                conditional_dict=unconditional_dict,
                timestep=timestep * 0,
                kv_cache=self.kv_cache_neg,
                crossattn_cache=self.crossattn_cache_neg,
                current_start=current_start_frame * self.frame_seq_length,
                cache_start=cache_start_frame * self.frame_seq_length,
            )
            current_start_frame += self.num_frame_per_block
            cache_start_frame += self.num_frame_per_block
        all_num_frames = [self.num_frame_per_block] * num_blocks
        with tqdm(
            total=sum(all_num_frames), disable=dist.is_initialized() and dist.get_rank() != 0
        ) as pbar:
            pbar.set_description("Processing latent frames")
            for current_num_frames in all_num_frames:
                noisy_input = noise[
                    :,
                    cache_start_frame - num_input_frames : cache_start_frame
                    - num_input_frames
                    + current_num_frames,
                ]
                latents = noisy_input
                sample_scheduler = self._initialize_sample_scheduler(noise)
                for _, t in enumerate(tqdm(sample_scheduler.timesteps, disable=True)):
                    latent_model_input = latents
                    timestep = t * torch.ones(
                        [batch_size, current_num_frames], device=noise.device, dtype=torch.float32
                    )
                    flow_pred_cond, _ = self.generator(
                        noisy_image_or_video=latent_model_input,
                        conditional_dict=conditional_dict,
                        timestep=timestep,
                        kv_cache=self.kv_cache_pos,
                        crossattn_cache=self.crossattn_cache_pos,
                        current_start=current_start_frame * self.frame_seq_length,
                        cache_start=cache_start_frame * self.frame_seq_length,
                    )
                    flow_pred_uncond, _ = self.generator(
                        noisy_image_or_video=latent_model_input,
                        conditional_dict=unconditional_dict,
                        timestep=timestep,
                        kv_cache=self.kv_cache_neg,
                        crossattn_cache=self.crossattn_cache_neg,
                        current_start=current_start_frame * self.frame_seq_length,
                        cache_start=cache_start_frame * self.frame_seq_length,
                    )
                    flow_pred = flow_pred_uncond + self.args.guidance_scale * (
                        flow_pred_cond - flow_pred_uncond
                    )
                    temp_x0 = sample_scheduler.step(flow_pred, t, latents, return_dict=False)[0]
                    latents = temp_x0
                logger.debug(
                    f"kv_cache local/global_end_index: {self.kv_cache_pos[0]['local_end_index'].item()}/{self.kv_cache_pos[0]['global_end_index'].item()}"
                )
                output[:, cache_start_frame : cache_start_frame + current_num_frames].copy_(
                    latents.detach()
                )
                self.generator(
                    noisy_image_or_video=latents,
                    conditional_dict=conditional_dict,
                    timestep=timestep * 0,
                    kv_cache=self.kv_cache_pos,
                    crossattn_cache=self.crossattn_cache_pos,
                    current_start=current_start_frame * self.frame_seq_length,
                    cache_start=cache_start_frame * self.frame_seq_length,
                )
                self.generator(
                    noisy_image_or_video=latents,
                    conditional_dict=unconditional_dict,
                    timestep=timestep * 0,
                    kv_cache=self.kv_cache_neg,
                    crossattn_cache=self.crossattn_cache_neg,
                    current_start=current_start_frame * self.frame_seq_length,
                    cache_start=cache_start_frame * self.frame_seq_length,
                )
                current_start_frame += current_num_frames
                cache_start_frame += current_num_frames
                pbar.update(current_num_frames)
                pbar.set_description("Processing latent frames")
        if return_video:
            # The next call re-creates the KV caches. Freeing them first leaves their
            # memory to the VAE decode (about 12 GiB for 5B at 1024x576).
            self.reset_cache()
            video = self.vae.decode_to_pixel(output)
            video = (video * 0.5 + 0.5).clamp(0, 1)
            if return_latents:
                return (video, output)
            else:
                return video
        else:
            return output

    def inference_long(
        self,
        noise: torch.Tensor,
        text_prompts: list[str],
        initial_latent: torch.Tensor | None = None,
        start_frame_index: int | None = 0,
        return_latents: bool = False,
        return_video: bool = True,
        vae_decode_chunk: int = 32,
    ) -> torch.Tensor:
        """Generate a causal window with CPU latent buffering and chunked VAE decode.

        Args are the same as :meth:`inference`, plus:
            vae_decode_chunk: Number of latent frames per VAE decode call.

        Returns:
            Same return shapes as :meth:`inference`.

        Compatibility:
            The output latent buffer lives on CPU pinned memory to reduce GPU
            peak usage. VAE decoding uses ``use_cache=True`` and therefore
            currently requires batch size ``1`` in ``WanVAEWrapper``.
        """
        validate_condition_only_inference_inputs(
            enabled=bool(getattr(self, "condition_only_first_frame", False)),
            initial_latent=initial_latent,
            generated_frames=int(noise.shape[1]),
            num_frame_per_block=self.num_frame_per_block,
            consumer_name="CausalDiffusionInferencePipeline.inference_long",
        )
        batch_size, num_frames, num_channels, height, width = noise.shape
        assert num_frames % self.num_frame_per_block == 0
        num_blocks = num_frames // self.num_frame_per_block
        num_input_frames = initial_latent.shape[1]
        num_output_frames = num_frames + num_input_frames
        conditional_dict = self.text_encoder(text_prompts=text_prompts)
        unconditional_dict = self.text_encoder(
            text_prompts=[self.args.negative_prompt] * len(text_prompts)
        )
        output = torch.zeros(
            [batch_size, num_output_frames, num_channels, height, width],
            device="cpu",
            dtype=noise.dtype,
            pin_memory=torch.cuda.is_available(),
        )
        if self.kv_cache_pos is None:
            self._initialize_kv_cache(batch_size=batch_size, dtype=noise.dtype, device=noise.device)
            self._initialize_crossattn_cache(
                batch_size=batch_size, dtype=noise.dtype, device=noise.device
            )
        else:
            for cache_list in self._iter_crossattn_cache_lists():
                self._reset_crossattn_cache_list(cache_list)
            for block_index in range(len(self.kv_cache_pos)):
                self.kv_cache_pos[block_index]["global_end_index"] = torch.tensor(
                    [0], dtype=torch.long, device=noise.device
                )
                self.kv_cache_pos[block_index]["local_end_index"] = torch.tensor(
                    [0], dtype=torch.long, device=noise.device
                )
                self.kv_cache_neg[block_index]["global_end_index"] = torch.tensor(
                    [0], dtype=torch.long, device=noise.device
                )
                self.kv_cache_neg[block_index]["local_end_index"] = torch.tensor(
                    [0], dtype=torch.long, device=noise.device
                )
        # CPU copy/cat below must not read pending asynchronous D2H transfers.
        # Keep D2H blocking; H2D decode transfers can remain asynchronous.
        current_start_frame = start_frame_index
        cache_start_frame = 0
        timestep = torch.ones([batch_size, 1], device=noise.device, dtype=torch.int64) * 0
        if self.independent_first_frame:
            assert (num_input_frames - 1) % self.num_frame_per_block == 0
            num_input_blocks = (num_input_frames - 1) // self.num_frame_per_block
            output[:, :1].copy_(initial_latent[:, :1].detach().to("cpu"))
            self.generator(
                noisy_image_or_video=initial_latent[:, :1],
                conditional_dict=conditional_dict,
                timestep=timestep * 0,
                kv_cache=self.kv_cache_pos,
                crossattn_cache=self.crossattn_cache_pos,
                current_start=current_start_frame * self.frame_seq_length,
                cache_start=cache_start_frame * self.frame_seq_length,
            )
            self.generator(
                noisy_image_or_video=initial_latent[:, :1],
                conditional_dict=unconditional_dict,
                timestep=timestep * 0,
                kv_cache=self.kv_cache_neg,
                crossattn_cache=self.crossattn_cache_neg,
                current_start=current_start_frame * self.frame_seq_length,
                cache_start=cache_start_frame * self.frame_seq_length,
            )
            current_start_frame += 1
            cache_start_frame += 1
        else:
            assert num_input_frames % self.num_frame_per_block == 0
            num_input_blocks = num_input_frames // self.num_frame_per_block
        for _ in range(num_input_blocks):
            current_ref_latents = initial_latent[
                :, cache_start_frame : cache_start_frame + self.num_frame_per_block
            ]
            output[:, cache_start_frame : cache_start_frame + self.num_frame_per_block].copy_(
                current_ref_latents.detach().to("cpu")
            )
            self.generator(
                noisy_image_or_video=current_ref_latents,
                conditional_dict=conditional_dict,
                timestep=timestep * 0,
                kv_cache=self.kv_cache_pos,
                crossattn_cache=self.crossattn_cache_pos,
                current_start=current_start_frame * self.frame_seq_length,
                cache_start=cache_start_frame * self.frame_seq_length,
            )
            self.generator(
                noisy_image_or_video=current_ref_latents,
                conditional_dict=unconditional_dict,
                timestep=timestep * 0,
                kv_cache=self.kv_cache_neg,
                crossattn_cache=self.crossattn_cache_neg,
                current_start=current_start_frame * self.frame_seq_length,
                cache_start=cache_start_frame * self.frame_seq_length,
            )
            current_start_frame += self.num_frame_per_block
            cache_start_frame += self.num_frame_per_block
        all_num_frames = [self.num_frame_per_block] * num_blocks
        with tqdm(
            total=sum(all_num_frames), disable=dist.is_initialized() and dist.get_rank() != 0
        ) as pbar:
            pbar.set_description("Processing latent frames")
            for current_num_frames in all_num_frames:
                noisy_input = noise[
                    :,
                    cache_start_frame - num_input_frames : cache_start_frame
                    - num_input_frames
                    + current_num_frames,
                ]
                latents = noisy_input
                sample_scheduler = self._initialize_sample_scheduler(noise)
                for _, t in enumerate(tqdm(sample_scheduler.timesteps, disable=True)):
                    latent_model_input = latents
                    timestep = t * torch.ones(
                        [batch_size, current_num_frames], device=noise.device, dtype=torch.float32
                    )
                    flow_pred_cond, _ = self.generator(
                        noisy_image_or_video=latent_model_input,
                        conditional_dict=conditional_dict,
                        timestep=timestep,
                        kv_cache=self.kv_cache_pos,
                        crossattn_cache=self.crossattn_cache_pos,
                        current_start=current_start_frame * self.frame_seq_length,
                        cache_start=cache_start_frame * self.frame_seq_length,
                    )
                    flow_pred_uncond, _ = self.generator(
                        noisy_image_or_video=latent_model_input,
                        conditional_dict=unconditional_dict,
                        timestep=timestep,
                        kv_cache=self.kv_cache_neg,
                        crossattn_cache=self.crossattn_cache_neg,
                        current_start=current_start_frame * self.frame_seq_length,
                        cache_start=cache_start_frame * self.frame_seq_length,
                    )
                    flow_pred = flow_pred_uncond + self.args.guidance_scale * (
                        flow_pred_cond - flow_pred_uncond
                    )
                    temp_x0 = sample_scheduler.step(flow_pred, t, latents, return_dict=False)[0]
                    latents = temp_x0
                logger.debug(
                    f"kv_cache local/global_end_index: {self.kv_cache_pos[0]['local_end_index'].item()}/{self.kv_cache_pos[0]['global_end_index'].item()}"
                )
                output[:, cache_start_frame : cache_start_frame + current_num_frames].copy_(
                    latents.detach().to("cpu")
                )
                self.generator(
                    noisy_image_or_video=latents,
                    conditional_dict=conditional_dict,
                    timestep=timestep * 0,
                    kv_cache=self.kv_cache_pos,
                    crossattn_cache=self.crossattn_cache_pos,
                    current_start=current_start_frame * self.frame_seq_length,
                    cache_start=cache_start_frame * self.frame_seq_length,
                )
                self.generator(
                    noisy_image_or_video=latents,
                    conditional_dict=unconditional_dict,
                    timestep=timestep * 0,
                    kv_cache=self.kv_cache_neg,
                    crossattn_cache=self.crossattn_cache_neg,
                    current_start=current_start_frame * self.frame_seq_length,
                    cache_start=cache_start_frame * self.frame_seq_length,
                )
                current_start_frame += current_num_frames
                cache_start_frame += current_num_frames
                pbar.set_description("Processing latent frames")
                pbar.update(current_num_frames)
                del latents, noisy_input, latent_model_input
                del flow_pred, flow_pred_cond, flow_pred_uncond, temp_x0
                del timestep, sample_scheduler
                if torch.cuda.is_available():
                    torch.cuda.empty_cache()
        if return_video:
            vae_device = noise.device
            chunk = max(1, int(vae_decode_chunk))
            total_lat = output.shape[1]
            pixel_chunks = []
            try:
                for s in range(0, total_lat, chunk):
                    e = min(s + chunk, total_lat)
                    lat_chunk = output[:, s:e].to(vae_device, non_blocking=True)
                    pix_chunk = self.vae.decode_to_pixel(lat_chunk, use_cache=True)
                    pixel_chunks.append(pix_chunk.detach().to("cpu"))
                    del lat_chunk, pix_chunk
                    if torch.cuda.is_available():
                        torch.cuda.empty_cache()
            finally:
                self.vae.model.clear_cache()
            video = torch.cat(pixel_chunks, dim=1)
            video = (video * 0.5 + 0.5).clamp(0, 1)
            if return_latents:
                return (video, output)
            else:
                return video
        else:
            return output

    def _initialize_kv_cache(self, batch_size, dtype, device):
        """Initialize per-GPU self-attention KV caches.

        Args:
            batch_size: ``B``.
            dtype: Cache tensor dtype, usually matching the latent noise dtype.
            device: CUDA device.

        Cache shape:
            For every transformer block, stores ``k`` and ``v`` tensors with
            shape ``[B, local_attn_size * frame_seq_len, num_heads, head_dim]``
            plus scalar ``global_end_index`` / ``local_end_index`` counters.
        """
        kv_cache_pos = []
        kv_cache_neg = []
        kv_cache_size = self.local_attn_size * self.frame_seq_length
        for _ in range(self.num_transformer_blocks):
            kv_cache_pos.append(
                {
                    "k": torch.zeros(
                        [batch_size, kv_cache_size, self.num_heads, self.head_dim],
                        dtype=dtype,
                        device=device,
                    ),
                    "v": torch.zeros(
                        [batch_size, kv_cache_size, self.num_heads, self.head_dim],
                        dtype=dtype,
                        device=device,
                    ),
                    "global_end_index": torch.tensor([0], dtype=torch.long, device=device),
                    "local_end_index": torch.tensor([0], dtype=torch.long, device=device),
                }
            )
            kv_cache_neg.append(
                {
                    "k": torch.zeros(
                        [batch_size, kv_cache_size, self.num_heads, self.head_dim],
                        dtype=dtype,
                        device=device,
                    ),
                    "v": torch.zeros(
                        [batch_size, kv_cache_size, self.num_heads, self.head_dim],
                        dtype=dtype,
                        device=device,
                    ),
                    "global_end_index": torch.tensor([0], dtype=torch.long, device=device),
                    "local_end_index": torch.tensor([0], dtype=torch.long, device=device),
                }
            )
        self.kv_cache_pos = kv_cache_pos
        self.kv_cache_neg = kv_cache_neg

    def _initialize_crossattn_cache(self, batch_size, dtype, device):
        """Initialize per-GPU text cross-attention caches.

        Shape per transformer block is
        ``[B, text_len, num_heads, head_dim]`` for both ``k`` and ``v``. The
        ``is_init`` flag lets the Wan block reuse text K/V across timesteps.
        """

        def make_cache_list():
            cache_list = []
            for _ in range(self.num_transformer_blocks):
                cache_list.append(
                    {
                        "k": torch.zeros(
                            [batch_size, self.text_len, self.num_heads, self.head_dim],
                            dtype=dtype,
                            device=device,
                        ),
                        "v": torch.zeros(
                            [batch_size, self.text_len, self.num_heads, self.head_dim],
                            dtype=dtype,
                            device=device,
                        ),
                        "is_init": False,
                    }
                )
            return cache_list

        crossattn_cache_pos = make_cache_list()
        crossattn_cache_neg = make_cache_list()
        self.crossattn_cache_pos = crossattn_cache_pos
        self.crossattn_cache_neg = crossattn_cache_neg

    def _initialize_sample_scheduler(self, noise, sampling_steps=-1, shift=None):
        """Create the per-block sampler on ``noise.device``."""
        if sampling_steps == -1:
            sampling_steps = self.sampling_steps
        if isinstance(shift, bool):
            raise ValueError("Scheduler shift must be finite and positive.")
        effective_shift = self.shift if shift is None else float(shift)
        if effective_shift <= 0 or not math.isfinite(effective_shift):
            raise ValueError("Scheduler shift must be finite and positive.")
        if self.sample_solver == "unipc":
            sample_scheduler = FlowUniPCMultistepScheduler(shift=1)
            sample_scheduler.set_timesteps(
                sampling_steps, device=noise.device, shift=effective_shift
            )
            self.timesteps = sample_scheduler.timesteps
        elif self.sample_solver == "dpm++":
            sample_scheduler = FlowDPMSolverMultistepScheduler(shift=1)
            sampling_sigmas = get_sampling_sigmas(sampling_steps, effective_shift)
            self.timesteps, _ = retrieve_timesteps(
                sample_scheduler, device=noise.device, sigmas=sampling_sigmas
            )
        else:
            raise NotImplementedError("Unsupported solver.")
        return sample_scheduler

    def reset_cache(self):
        """Drop KV/cross-attention caches and release cached CUDA blocks."""
        self.kv_cache_pos = None
        self.kv_cache_neg = None
        self.crossattn_cache_pos = None
        self.crossattn_cache_neg = None
        torch.cuda.empty_cache()