Skip to content

Looking Ahead: Modeling Future Evolution in Autoregressive Video Generation

18.1 KB
Raw Blame
"""FCP flow objective with independent future timesteps."""

import logging
import os
from dataclasses import dataclass

import torch

from src.model.fm import FlowMatching
from src.utils.predictive_multi_chunk_config import (
    build_predictive_chunk_layout,
    configure_predictive_multi_chunk_from_config,
    read_predictive_multi_chunk_config,
)
from src.utils.predictive_training_checkpoint import prepare_predictive_checkpoint
from src.utils.timestep_bucket_logging import (
    build_chunkwise_loss_observations,
    build_predictive_aux_chunkwise_loss_observations,
)
from src.utils.wan_model_family import validate_wan_checkpoint_family


@dataclass
class PredictiveLossComponents:
    total_loss: torch.Tensor
    main_loss: torch.Tensor
    aux_loss: torch.Tensor
    log_dict: dict


class PredictiveFlowMatching(FlowMatching):
    _predictive_consumer_name = "FCP"
    _aux_future_timestep_stream_id = 0xA17F_71A5_7E9D_51E1

    def __init__(self, args, device):
        self._independent_aux_scheduler = None
        self._loaded_live_checkpoint_has_predictive_aux = None
        super().__init__(args, device)
        # Validate the physical interval against the effective scheduler eagerly.
        self._build_aux_future_timestep_candidate_indices()

    def _after_generator_initialized(self, args):
        cfg = read_predictive_multi_chunk_config(args, self.generator.model)
        self.predictive_multi_chunk_config = cfg
        self.generator.model.num_frame_per_block = args.num_frame_per_block
        self.generator.model.independent_first_frame = args.independent_first_frame
        configure_predictive_multi_chunk_from_config(self.generator.model, cfg)
        self._predictive_checkpoint_shapes = {
            k: tuple(v.shape) for k, v in self.generator.state_dict().items()
        }

    def _prepare_checkpoint(self, state, *, ema=False):
        validate_wan_checkpoint_family(
            state,
            model_name=self.args.model_name,
            consumer_name="FCP EMA" if ema else "FCP training",
        )
        return prepare_predictive_checkpoint(
            state,
            self.predictive_multi_chunk_config,
            self._predictive_checkpoint_shapes,
            label="FCP EMA" if ema else "FCP training",
            ema=ema,
        )

    def load_generator_checkpoint_state_dict(self, state_dict):
        state, predictive = self._prepare_checkpoint(state_dict)
        self.generator.load_state_dict(
            state, strict=False
        )  # All exceptions checked before mutation.
        self._loaded_live_checkpoint_has_predictive_aux = predictive

    def load_generator_ema_checkpoint_state_dict(self, ema, state_dict, generator):
        state, predictive = self._prepare_checkpoint(state_dict, ema=True)
        live = self._loaded_live_checkpoint_has_predictive_aux
        if live is not None and live != predictive:
            raise RuntimeError("FCP live/EMA checkpoint identity conflict")
        ema.load_full_state_dict(
            state,
            generator,
            allow_missing_aux=True,
            expected_shapes=self._predictive_checkpoint_shapes,
        )

    def _runtime_predictive_chunk_layout(self, num_frame: int):
        """Validate runtime frames against the parser-normalized chunk layout."""
        cfg = self.predictive_multi_chunk_config
        layout = build_predictive_chunk_layout(
            num_frames=num_frame,
            block_size=cfg["num_frame_per_block"],
            independent_first_frame=cfg.get("independent_first_frame", False),
            consumer_name=self._predictive_consumer_name,
        )
        if layout.num_regular_chunks != cfg["num_chunks"]:
            raise ValueError(
                f"{self._predictive_consumer_name} runtime/config chunk mismatch: "
                f"runtime N={layout.num_regular_chunks}, configured N={cfg['num_chunks']}."
            )
        return layout

    def _get_aux_future_scheduler(self):
        """Return the effective scheduler for independent predictive targets."""
        config = getattr(self, "predictive_multi_chunk_config", {})
        aux_shift = config.get("aux_future_timestep_shift")
        if aux_shift is None:
            return self.scheduler
        if getattr(self, "_independent_aux_scheduler", None) is None:
            self._independent_aux_scheduler = self.scheduler.clone_with_shift(aux_shift)
            logging.info(
                "Predictive independent aux scheduler enabled: main_shift=%s, aux_shift=%s",
                self.scheduler.shift,
                aux_shift,
            )
        return self._independent_aux_scheduler

    @staticmethod
    def _mix_aux_future_timestep_seed(*components: int) -> int:
        """Mix position components into one deterministic 64-bit seed."""
        mask = (1 << 64) - 1
        value = 0x9E37_79B9_7F4A_7C15
        for component in components:
            value ^= (
                (int(component) & mask)
                + 0x9E37_79B9_7F4A_7C15
                + ((value << 6) & mask)
                + (value >> 2)
            )
            value &= mask
        value = (value ^ (value >> 30)) * 0xBF58_476D_1CE4_E5B9 & mask
        value = (value ^ (value >> 27)) * 0x94D0_49BB_1331_11EB & mask
        return int(value ^ (value >> 31))

    @staticmethod
    def _generator_device(device) -> torch.device:
        if isinstance(device, int):
            return torch.device("cuda", device)
        return torch.device(device)

    def _sample_aux_future_timestep_indices(
        self,
        *,
        batch_size: int,
        valid_chunks: int,
        block_size: int,
        future_offset: int,
        global_step: int,
        global_rank: int,
        training_seed: int,
        first_regular_chunk_index: int = 0,
        candidate_indices: torch.Tensor | None = None,
    ) -> torch.Tensor:
        """Sample chunk-shared aux timestep indices with isolated per-position seeds."""
        device = self._generator_device(self.device)
        if candidate_indices is not None:
            if candidate_indices.ndim != 1 or candidate_indices.numel() == 0:
                raise ValueError("Aux future timestep candidate indices must be non-empty and 1D.")
            candidate_indices = candidate_indices.to(device=device, dtype=torch.long)
        indices = torch.empty(
            batch_size,
            valid_chunks,
            block_size,
            device=device,
            dtype=torch.long,
        )
        for batch_slot in range(batch_size):
            for retained_position in range(valid_chunks):
                source_chunk_index = retained_position
                global_source_chunk_index = first_regular_chunk_index + source_chunk_index
                seed_components = [
                    training_seed,
                    global_rank,
                    global_step,
                    batch_slot,
                    global_source_chunk_index,
                    future_offset,
                ]
                seed_components.append(self._aux_future_timestep_stream_id)
                seed = self._mix_aux_future_timestep_seed(*seed_components)
                generator = torch.Generator(device=device).manual_seed(seed)
                if candidate_indices is None:
                    timestep_index = torch.randint(
                        0,
                        self._get_aux_future_scheduler().num_train_timesteps,
                        (1,),
                        device=device,
                        dtype=torch.long,
                        generator=generator,
                    )
                else:
                    ordinal = torch.randint(
                        0,
                        candidate_indices.numel(),
                        (1,),
                        device=device,
                        dtype=torch.long,
                        generator=generator,
                    )
                    timestep_index = candidate_indices[ordinal]
                indices[batch_slot, retained_position, :] = timestep_index
        return indices

    def _build_aux_future_timestep_candidate_indices_for_bounds(
        self,
        bounds: list[float],
        *,
        field_name: str,
    ) -> torch.Tensor:
        """Return scheduler indices inside one configured closed physical-t interval."""
        lower, upper = bounds
        timesteps = self._get_aux_future_scheduler().timesteps
        if timesteps.ndim != 1 or timesteps.numel() == 0:
            raise ValueError(
                f"{field_name} requires a non-empty one-dimensional scheduler.timesteps table."
            )

        candidate_indices = torch.nonzero(
            (timesteps >= lower) & (timesteps <= upper),
            as_tuple=False,
        ).flatten()
        if candidate_indices.numel() == 0:
            table_min = float(timesteps.min().item())
            table_max = float(timesteps.max().item())
            raise ValueError(
                f"{field_name} "
                f"[{lower}, {upper}] selects no scheduler timestep; runtime "
                f"scheduler.timesteps spans [{table_min}, {table_max}] across "
                f"{timesteps.numel()} entries."
            )
        return candidate_indices.to(device=self._generator_device(self.device))

    def _build_aux_future_timestep_candidate_indices(self) -> torch.Tensor:
        """Return scheduler indices inside the legacy global physical-t interval."""
        return self._build_aux_future_timestep_candidate_indices_for_bounds(
            self.predictive_multi_chunk_config["aux_future_timestep_range"],
            field_name="future_prediction.aux_future_timestep_range",
        )

    def _build_independent_aux_future_inputs(
        self,
        *,
        clean_latent: torch.Tensor,
        noise: torch.Tensor,
        batch_size: int,
        num_frame: int,
        global_step: int,
        global_rank: int,
    ) -> tuple[dict[int, torch.Tensor], dict[int, torch.Tensor], dict[int, torch.Tensor]]:
        """Build offset-keyed aux noisy chunks, targets and timesteps."""
        cfg = self.predictive_multi_chunk_config
        block_size = cfg["num_frame_per_block"]
        if clean_latent.shape[1] != num_frame or noise.shape[1] != num_frame:
            raise ValueError(
                "Independent predictive aux inputs must match the runtime temporal length: "
                f"clean={clean_latent.shape[1]}, noise={noise.shape[1]}, expected={num_frame}."
            )
        layout = self._runtime_predictive_chunk_layout(num_frame)
        num_chunks = layout.num_regular_chunks
        training_seed = int(getattr(self.args, "seed", 0))
        regular_start = layout.regular_frame_start
        clean_chunks = clean_latent[:, regular_start:].unflatten(1, (num_chunks, block_size))
        noise_chunks = noise[:, regular_start:].unflatten(1, (num_chunks, block_size))
        candidate_indices = self._build_aux_future_timestep_candidate_indices()

        aux_noisy_chunks_by_offset = {}
        aux_training_target_by_offset = {}
        aux_timestep_by_offset = {}
        for offset in cfg["future_offsets"]:
            valid_chunks = layout.valid_pair_count(offset)
            if valid_chunks < 1:
                raise ValueError(
                    f"{self._predictive_consumer_name} has no retained regular predictive "
                    f"pair for offset {offset}: T={num_frame}, F={block_size}, N={num_chunks}, "
                    f"independent_first_frame={layout.independent_first_frame}, "
                    f"valid_pair_count="
                    f"{valid_chunks}."
                )
            timestep_indices = self._sample_aux_future_timestep_indices(
                batch_size=batch_size,
                valid_chunks=valid_chunks,
                block_size=block_size,
                future_offset=int(offset),
                global_step=global_step,
                global_rank=global_rank,
                training_seed=training_seed,
                first_regular_chunk_index=layout.first_regular_chunk_index,
                candidate_indices=candidate_indices,
            )
            aux_timestep = (
                self._get_aux_future_scheduler()
                .timesteps[timestep_indices]
                .to(
                    dtype=self.dtype,
                    device=self.device,
                )
            )
            clean_future = clean_chunks[:, offset : offset + valid_chunks]
            noise_future = noise_chunks[:, offset : offset + valid_chunks]
            flat_clean = clean_future.reshape(
                batch_size * valid_chunks * block_size,
                *clean_latent.shape[2:],
            )
            flat_noise = noise_future.reshape_as(flat_clean)
            aux_noisy = (
                self._get_aux_future_scheduler()
                .add_noise(
                    flat_clean,
                    flat_noise,
                    aux_timestep.reshape(-1),
                )
                .reshape(batch_size, valid_chunks, block_size, *clean_latent.shape[2:])
            )
            aux_target = self._get_aux_future_scheduler().training_target(
                clean_future,
                noise_future,
                aux_timestep,
            )
            aux_noisy_chunks_by_offset[int(offset)] = aux_noisy
            aux_training_target_by_offset[int(offset)] = aux_target
            aux_timestep_by_offset[int(offset)] = aux_timestep
        return aux_noisy_chunks_by_offset, aux_training_target_by_offset, aux_timestep_by_offset

    def _auxiliary_losses(self, aux_outputs, targets, timesteps):
        cfg = self.predictive_multi_chunk_config
        scheduler = self._get_aux_future_scheduler()
        total = next(iter(targets.values())).new_zeros(())
        log, losses, times = {}, {}, {}
        for offset, weight in zip(cfg["future_offsets"], cfg["offset_loss_weights"], strict=True):
            expected = cfg["num_chunks"] - offset
            if aux_outputs[offset]["valid_chunks"] != expected:
                raise ValueError(f"FCP valid pair mismatch for offset {offset}")
            target, timestep = targets[offset], timesteps[offset]
            source_losses = []
            for source in cfg["source_context_streams"]:
                pred = aux_outputs[offset]["flow_pred_by_source"][source]
                if pred.shape != target.shape:
                    raise ValueError(f"FCP prediction/target shape mismatch for {source}/{offset}")
                frame_loss = (pred.float() - target.float()).square().mean((3, 4, 5))
                frame_loss = frame_loss * scheduler.training_weight(timestep.flatten()).reshape(
                    *timestep.shape
                )
                chunk_loss, chunk_time = build_predictive_aux_chunkwise_loss_observations(
                    frame_loss, timestep
                )
                losses[(source, offset)], times[(source, offset)] = chunk_loss, chunk_time
                source_loss = frame_loss.mean()
                source_losses.append(source_loss)
                log[f"pred_{source}_{offset}_loss"] = source_loss.detach()
            offset_loss = torch.stack(source_losses).mean()
            total = total + weight * offset_loss
            log[f"pred_{offset}_loss"] = offset_loss.detach()
            log[f"valid_{offset}"] = float(expected)
        log["_predictive_aux_chunk_loss"] = losses
        log["_predictive_aux_chunk_timestep"] = times
        return total, log

    def generator_loss_components(
        self,
        image_or_video_shape,
        conditional_dict,
        unconditional_dict,
        clean_latent,
        initial_latent=None,
        global_step=None,
        global_rank=None,
    ):
        noise = torch.randn_like(clean_latent)
        batch, frames = image_or_video_shape[:2]
        index = self._get_timestep(0, 1000, batch, frames, self.num_frame_per_block)
        canonical = self.scheduler.timesteps[index]
        timestep = canonical.to(dtype=self.dtype)
        noisy = self.scheduler.add_noise(
            clean_latent.flatten(0, 1), noise.flatten(0, 1), timestep.flatten()
        ).unflatten(0, (batch, frames))
        target = self.scheduler.training_target(clean_latent, noise, timestep)
        aux_noisy, aux_target, aux_timestep = self._build_independent_aux_future_inputs(
            clean_latent=clean_latent,
            noise=noise,
            batch_size=batch,
            num_frame=frames,
            global_step=0 if global_step is None else int(global_step),
            global_rank=0 if global_rank is None else int(global_rank),
        )
        flow, x0, outputs = self.generator(
            noisy_image_or_video=noisy,
            conditional_dict=conditional_dict,
            timestep=timestep,
            clean_x=clean_latent,
            return_predictive_aux=True,
            predictive_aux_noisy_chunks_by_offset=aux_noisy,
            predictive_aux_timestep_chunks_by_offset=aux_timestep,
        )
        frame_loss = self._flow_matching_frame_loss(flow, target, timestep, batch, frames)
        main, metrics = self._reduce_main_frame_loss(frame_loss)
        chunk_loss, chunk_time = build_chunkwise_loss_observations(
            frame_loss,
            canonical,
            num_frame_per_block=self.num_frame_per_block,
            independent_first_frame=self.independent_first_frame,
            exclude_first_frame=self.condition_only_first_frame,
        )
        aux, aux_metrics = self._auxiliary_losses(outputs, aux_target, aux_timestep)
        total = main + aux
        log = {
            "x0": clean_latent.detach(),
            "x0_pred": x0.detach(),
            "_generator_loss": main.detach(),
            "pred_total_loss": total.detach(),
            "_generator_chunk_loss": chunk_loss,
            "_generator_chunk_timestep": chunk_time,
            **metrics,
            **aux_metrics,
        }
        if os.environ.get("DEBUG") == "1":
            log.update(
                {
                    f"debug_aux_timestep_mean_{offset}": value.detach().float().mean()
                    for offset, value in aux_timestep.items()
                }
            )
        return PredictiveLossComponents(total, main, aux, log)

    def generator_loss(self, *args, **kwargs):
        components = self.generator_loss_components(*args, **kwargs)
        return components.total_loss, components.log_dict