Skip to content

Looking Ahead: Modeling Future Evolution in Autoregressive Video Generation

11.6 KB
Raw Blame
"""Text with image/video-prefix sampling and distributed existence voting."""

import json
import logging
import os
import random
from pathlib import Path

import numpy as np
import torch
import torch.distributed as dist
from PIL import Image
from torch.utils.data import DataLoader
from torchcodec.encoders import VideoEncoder
from torchvision import transforms

from src.dataset.image_preprocessing import ResizeCenterCrop
from src.dataset.single_image import SingleImageDataset, read_prompt_file
from src.dataset.single_video import SingleVideoDataset
from src.dataset.text import TextImagePairDataset
from src.pipeline.causal_diffusion_inference import CausalDiffusionInferencePipeline
from src.utils.inference_checkpoint import load_generator_checkpoint
from src.utils.memory import DynamicSwapInstaller, get_cuda_free_memory_gb
from src.utils.train_dataset import build_train_dataset
from src.utils.video_prefix import output_geometry, select_prefix
from src.utils.wan_model_family import get_wan_model_family

logger = logging.getLogger(__name__)


def output_exists_on_all_ranks(path, device):
    exists = Path(path).exists()
    if dist.is_initialized() and dist.get_world_size() > 1:
        flag = torch.tensor([int(exists)], dtype=torch.int32, device=device)
        dist.all_reduce(flag, op=dist.ReduceOp.MIN)
        exists = bool(flag.item())
    return exists


@torch.no_grad()
def run_inference(config):
    single_dataset = None
    family = get_wan_model_family(config.model_name)
    if config.get("image_path") is not None:
        h, w = config.image_or_video_shape[-2:]
        # Read one prompt/image and report changes before loading GPU models.
        prompt = (
            read_prompt_file(config.prompt_file)
            if config.get("prompt_file") is not None
            else config.prompt
        )
        single_dataset = SingleImageDataset(
            config.image_path,
            prompt,
            height=h * family.vae_spatial_stride,
            width=w * family.vae_spatial_stride,
        )
    v2v = config.get("v2v", False)
    # Config geometry was validated by the resolver; only sample bounds remain below.
    context_count = config.v2v_context_latents if v2v else 1
    automatic_context = v2v and context_count is None
    condition_frames = (
        None if automatic_context else 1 + family.vae_temporal_stride * (context_count - 1)
    )
    if config.get("video_path") is not None:
        h, w = config.image_or_video_shape[-2:]
        prompt = (
            read_prompt_file(config.prompt_file)
            if config.prompt_file is not None
            else config.prompt
        )
        single_dataset = SingleVideoDataset(
            config.video_path,
            prompt,
            height=h * family.vae_spatial_stride,
            width=w * family.vae_spatial_stride,
            fps=config.get("fps", family.default_fps),
            condition_frames=condition_frames,
            auto_context_config=config if automatic_context else None,
        )
        if automatic_context:
            context_count = single_dataset.context_latents
            condition_frames = single_dataset.report["condition_frames"]
        single_dataset.report["context_latents"] = context_count
    distributed = "LOCAL_RANK" in os.environ
    local_rank = int(os.environ.get("LOCAL_RANK", 0))
    torch.cuda.set_device(local_rank)
    device = torch.device("cuda", local_rank)
    if distributed:
        dist.init_process_group(backend="nccl", device_id=device)
    rank = dist.get_rank() if dist.is_initialized() else 0
    world_size = dist.get_world_size() if dist.is_initialized() else 1
    random.seed(config.seed)
    np.random.seed(config.seed)
    torch.manual_seed(config.seed)
    torch.cuda.manual_seed_all(config.seed)
    # Construct on CPU, assign strict ordinary weights, then transfer once.
    pipeline = CausalDiffusionInferencePipeline(config, device="cpu")
    loaded_checkpoint = load_generator_checkpoint(pipeline.generator, config)
    pipeline.to(dtype=torch.bfloat16)
    free_memory = get_cuda_free_memory_gb(device)
    if free_memory < 40:
        DynamicSwapInstaller.install_model(pipeline.text_encoder, device=device)
    else:
        pipeline.text_encoder.to(device)
    pipeline.generator.to(device)
    pipeline.vae.to(device)
    pixel_frames, latent_frames = output_geometry(config)
    height, width = pipeline.generator.frame_shape[-2:]
    _, channels, latent_height, latent_width = pipeline.generator.latent_shape
    limit = int(config.test_sample_limit)
    per_rank_limit = -1 if limit < 0 else (limit + world_size - 1) // world_size
    if config.use_train_data:
        dataset = build_train_dataset(config, max_samples=per_rank_limit)
    else:
        if dist.is_initialized():
            raise ValueError("External-image TI2V requires direct non-distributed Python.")
        transform = transforms.Compose(
            [
                ResizeCenterCrop(height, width),
                transforms.ToTensor(),
                transforms.Normalize([0.5], [0.5]),
            ]
        )
        if single_dataset is not None:
            dataset = single_dataset
            per_rank_limit = 0 if limit == 0 else 1
        else:
            dataset = TextImagePairDataset(
                config.data_path, transform=transform, eval_first_n=limit
            )
            per_rank_limit = len(dataset)
    loader = DataLoader(dataset, batch_size=1, num_workers=0, drop_last=False)
    output = Path(config.output_path)
    output.mkdir(parents=True, exist_ok=True)
    if dist.is_initialized():
        dist.barrier()
    completed = 0
    for index, batch in enumerate(loader):
        if per_rank_limit >= 0 and completed >= per_rank_limit:
            break
        prompt = batch["prompts"][0]
        if config.use_train_data and world_size > 1:
            stem = f"r{rank:02d}-{index:05d}"
        else:
            global_index = int(batch["idx"][0]) if "idx" in batch else index
            stem = f"{global_index:05d}"
        sample_stem = f"{stem}-{config.seed}"
        if v2v:
            sample_stem += f"-v2v-k={context_count}"
        video_path = output / f"{sample_stem}.mp4"
        if output_exists_on_all_ranks(video_path, device):
            completed += 1
            logger.info("Skipped existing video %s (%d/%s)", video_path, completed, per_rank_limit)
            continue
        if single_dataset is not None:
            report_path = output / f"{sample_stem}.input.json"
            report_path.write_text(
                json.dumps(single_dataset.report, indent=2, ensure_ascii=False) + "\n",
                encoding="utf-8",
            )
            logger.info("Image preprocessing report: %s", report_path)
        image_for_save = None
        if config.use_train_data:
            clean = batch["clean_latent"]
            if clean.ndim != 5 or tuple(clean.shape[2:]) != (channels, latent_height, latent_width):
                raise ValueError(
                    f"TI2V latent input incompatible with model geometry: {tuple(clean.shape)}"
                )
            prefix = select_prefix(clean, context_count) if v2v else clean[:, :1].contiguous()
            initial = prefix.to(device=device, dtype=torch.bfloat16)
        else:
            image = (batch["video"] if v2v else batch["image"].unsqueeze(2)).to(
                device=device, dtype=torch.bfloat16
            )
            initial = pipeline.vae.encode_to_latent(image).to(dtype=torch.bfloat16)
            if initial.shape[1] != context_count:
                raise ValueError("video_path: encoded prefix length differs from requested context")
            image_for_save = image[0, :, 0].float().cpu()
        noise_generator = torch.Generator(device=device).manual_seed(int(config.seed) % (2**63))
        noise = torch.randn(
            (1, latent_frames - context_count, channels, latent_height, latent_width),
            device=device,
            dtype=torch.bfloat16,
            generator=noise_generator,
        )
        logger.info(
            "Generating %s; context=%s, total latent frames=%d",
            video_path,
            tuple(initial.shape),
            latent_frames,
        )
        long_path = latent_frames >= config.long_inference_threshold
        logger.info(
            "Sampling path: %s; context_latents=%d; generated_latents=%d",
            "long" if long_path else "short",
            context_count,
            latent_frames - context_count,
        )
        if long_path:
            video = pipeline.inference_long(
                noise, [prompt], initial_latent=initial, vae_decode_chunk=config.vae_decode_chunk
            )
        else:
            video = pipeline.inference(noise, [prompt], initial_latent=initial)
        pixels = (video[0, :pixel_frames].clamp(0, 1) * 255).byte().cpu()
        VideoEncoder(pixels, frame_rate=config.get("fps", pipeline.generator.default_fps)).to_file(
            str(video_path)
        )
        if v2v:
            fps = config.get("fps", pipeline.generator.default_fps)
            for folder, interval in (
                ("conditions", slice(0, condition_frames)),
                ("continuations", slice(condition_frames, pixel_frames)),
            ):
                destination = output / folder / f"{sample_stem}.mp4"
                destination.parent.mkdir(parents=True, exist_ok=True)
                VideoEncoder(pixels[interval].contiguous(), frame_rate=fps).to_file(
                    str(destination)
                )
            manifest_dir = output / "manifests"
            manifest_dir.mkdir(exist_ok=True)
            manifest = {
                "mode": "v2v",
                "context_latents": context_count,
                "requested_context_latents": config.v2v_context_latents,
                "context_selection": (
                    single_dataset.report["context_selection"]
                    if single_dataset is not None
                    else "latent_prefix"
                ),
                "condition_frames": condition_frames,
                "output_frames": pixel_frames,
                "continuation_frames": pixel_frames - condition_frames,
                "total_latents": latent_frames,
                "fps": fps,
                "seed": config.seed,
                "model_name": config.model_name,
                "checkpoint_path": config.checkpoint_path,
                "loaded_checkpoint_path": loaded_checkpoint,
                "use_ema": config.use_ema,
                "source_id": config.get("inference_train_data_source_id"),
                "video_path": config.get("video_path"),
                "sample_metadata": {
                    key: value[0] for key, value in batch.get("metadata", {}).items()
                },
                "prompt": prompt,
                "condition_interval": [0, condition_frames],
                "continuation_interval": [condition_frames, pixel_frames],
            }
            (manifest_dir / f"{sample_stem}.json").write_text(
                json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
            )
        (output / f"{stem}.txt").write_text(prompt + "\n", encoding="utf-8")
        if image_for_save is None:
            image_for_save = (
                pipeline.vae.decode_to_pixel(initial[:, :1], use_cache=False)[0, 0].float().cpu()
            )
            pipeline.vae.model.clear_cache()
        png = ((image_for_save + 1) / 2).clamp(0, 1).mul(255).byte().permute(1, 2, 0).numpy()
        Image.fromarray(png).save(output / f"{sample_stem}.png")
        pipeline.vae.model.clear_cache()
        completed += 1
        logger.info("Completed %s (%d/%s)", video_path, completed, per_rank_limit)
    if dist.is_initialized():
        dist.barrier()
        dist.destroy_process_group()