Skip to content

Looking Ahead: Modeling Future Evolution in Autoregressive Video Generation

3.8 KB
Raw Blame
"""Video context sampled at model FPS: an explicit prefix or an automatic tail."""

import logging
import math
from pathlib import Path

import torch
from torch.utils.data import Dataset
from torchcodec.decoders import VideoDecoder
from torchvision.transforms import functional as TF

from src.dataset.image_preprocessing import resize_center_crop
from src.utils.video_prefix import automatic_prefix_geometry

logger = logging.getLogger(__name__)


class SingleVideoDataset(Dataset):
    """Decode only the selected context interval for causal VAE encoding."""

    def __init__(
        self,
        video_path,
        prompt,
        *,
        height,
        width,
        fps,
        condition_frames,
        auto_context_config=None,
    ):
        path = Path(video_path)
        if not path.is_file():
            raise FileNotFoundError(f"video_path: video file not found: {path}")
        decoder = VideoDecoder(str(path), device="cpu", num_ffmpeg_threads=1)
        start = decoder.metadata.begin_stream_seconds
        automatic = condition_frames is None
        self.context_latents = None
        if automatic:
            if auto_context_config is None:
                raise ValueError("Automatic video context requires model/output geometry")
            # Use the actual final frame PTS, including variable-rate videos. Anchoring
            # to stream end minus 1/FPS can silently omit the last source frame.
            last_pts = decoder.get_frame_at(len(decoder) - 1).pts_seconds
            available = math.floor((last_pts - start) * fps + 1e-7) + 1
            self.context_latents, condition_frames = automatic_prefix_geometry(
                auto_context_config, available
            )
            times = [
                max(start, last_pts - (condition_frames - 1 - index) / fps)
                for index in range(condition_frames)
            ]
        else:
            times = [start + index / fps for index in range(condition_frames)]
            if times[-1] >= decoder.metadata.end_stream_seconds:
                raise ValueError(
                    "video_path: video is too short for the requested context at model FPS"
                )
        batch = decoder.get_frames_played_at(times)
        frames = batch.data
        images = []
        geometry = None
        for frame in frames:
            image, geometry = resize_center_crop(TF.to_pil_image(frame), height, width)
            images.append(TF.normalize(TF.to_tensor(image), [0.5], [0.5]))
        self.video = torch.stack(images, dim=1)  # [C,T,H,W], selected context only.
        self.prompt = prompt
        self.report = {
            "source_path": str(path),
            "source_fps": decoder.metadata.average_fps,
            "source_size": {"width": frames.shape[-1], "height": frames.shape[-2]},
            "input_size": {"width": width, "height": height},
            "fps": fps,
            "condition_frames": condition_frames,
            "context_selection": "auto_tail" if automatic else "explicit_prefix",
            "sample_times_seconds": times,
            "source_pts_seconds": batch.pts_seconds.tolist(),
            "preprocessing": (
                "sample trailing frames at model FPS; RGB resize and center crop"
                if automatic
                else "sample leading frames at model FPS; RGB resize and center crop"
            ),
            **geometry,
        }
        if automatic:
            self.report.update(
                context_latents=self.context_latents,
                available_frames_at_target_fps=available,
                source_last_frame_pts_seconds=last_pts,
            )
        logger.info("Video context preprocessing: %s", self.report)

    def __len__(self):
        return 1

    def __getitem__(self, index):
        if index != 0:
            raise IndexError(index)
        return {"video": self.video, "prompts": self.prompt, "idx": 0}