Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

add image encoding support

Shen-Chenhuiee58ec16b4f0
diff --git a/configs/vae_magvit_v2/train/17x128x128.py b/configs/vae_magvit_v2/train/17x128x128.py
index ba08169..bad6a74 100644
--- a/configs/vae_magvit_v2/train/17x128x128.py
+++ b/configs/vae_magvit_v2/train/17x128x128.py
@@ -32,6 +32,8 @@ model = dict(
kl_embed_dim = 64,
custom_conv_padding = None,
activation_fn = 'swish',
+ image_size = image_size,
+ separate_first_frame_encoding = False,
)
# Others
diff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.py
index 5619e4b..cd64e09 100644
--- a/opensora/models/vae/vae_3d_v2.py
+++ b/opensora/models/vae/vae_3d_v2.py
@@ -8,7 +8,7 @@ from numpy import typing as nptyping
from opensora.models.vae import model_utils
from opensora.registry import MODELS
from opensora.utils.ckpt_utils import load_checkpoint
-from einops import rearrange
+from einops import rearrange, pack, unpack
import torch.nn.functional as F
"""Encoder and Decoder stuctures with 3D CNNs."""
@@ -33,6 +33,16 @@ def divisible_by(num, den):
def is_odd(n):
return not divisible_by(n, 2)
+def pad_at_dim(t, pad, dim = -1, value = 0.):
+ dims_from_right = (- dim - 1) if dim < 0 else (t.ndim - dim - 1)
+ zeros = ((0, 0) * dims_from_right)
+ return F.pad(t, (*zeros, *pad), value = value)
+
+def SameConv2d(dim_in, dim_out, kernel_size):
+ kernel_size = cast_tuple(kernel_size, 2)
+ padding = [k // 2 for k in kernel_size]
+ return nn.Conv2d(dim_in, dim_out, kernel_size = kernel_size, padding = padding)
+
class CausalConv3d(nn.Module):
def __init__(
self,
@@ -172,7 +182,8 @@ class Encoder(nn.Module):
device=device,
)
- self.conv1 = self.conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)
+ # NOTE: moved to VAE for separate first frame processing
+ # self.conv1 = self.conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)
# ResBlocks and conv downsample
self.block_res_blocks = []
@@ -210,7 +221,10 @@ class Encoder(nn.Module):
def forward(self, x):
# dtype, device = x.dtype, x.device
- x = self.conv1(x)
+
+ # NOTE: moved to VAE for separate first frame processing
+ # x = self.conv1(x)
+
for i in range(self.num_blocks):
for j in range(self.num_res_blocks):
x = self.block_res_blocks[i][j](x)
@@ -323,7 +337,9 @@ class Decoder(nn.Module):
)
self.norm1 = nn.GroupNorm(self.num_groups, prev_filters, device=device, dtype=dtype)
- self.conv2 = self.conv_fn(prev_filters, self.output_dim, kernel_size=(3, 3, 3))
+
+ # NOTE: moved to VAE for separate first frame processing
+ # self.conv2 = self.conv_fn(prev_filters, self.output_dim, kernel_size=(3, 3, 3))
def forward(
@@ -348,7 +364,8 @@ class Decoder(nn.Module):
x = self.norm1(x)
x = self.activate(x)
- x = self.conv2(x)
+ # NOTE: moved to VAE for separate first frame processing
+ # x = self.conv2(x)
return x
@@ -358,8 +375,10 @@ class VAE_3D_V2(nn.Module):
def __init__(
self,
latent_embed_dim = 256,
- filters = 64,
+ filters = 128,
num_res_blocks = 2,
+ image_size = (128, 128),
+ separate_first_frame_encoding = False,
channel_multipliers = (1, 2, 2, 4),
temporal_downsample = (True, True, False),
num_groups = 32, # for nn.GroupNorm
@@ -382,8 +401,25 @@ class VAE_3D_V2(nn.Module):
dtype = torch.float16
else:
raise NotImplementedError(f'dtype: {dtype}')
+
+
+ # ==== Model Params ====
+ self.image_size = cast_tuple(image_size, 2)
+ self.time_downsample_factor = 2**sum(temporal_downsample)
+ self.time_padding = self.time_downsample_factor - 1
+ self.separate_first_frame_encoding = separate_first_frame_encoding
- # Model Initialization
+ # ==== Model Initialization ====
+
+ # encoder & decoder first and last conv layer
+ # SCH: NOTE: following MAGVIT, conv in bias=False in encoder first conv
+ self.conv_in = CausalConv3d(in_out_channels, filters, kernel_size=(3, 3, 3), bias=False, dtype=dtype, device=device)
+ self.conv_in_first_frame = nn.Identity()
+ self.conv_out_first_frame = nn.Identity()
+ if separate_first_frame_encoding:
+ self.conv_in_first_frame = SameConv2d(in_out_channels, filters, (3,3))
+ self.conv_out_first_frame = SameConv2d(filters, in_out_channels, (3,3))
+ self.conv_out = CausalConv3d(filters, in_out_channels, (3,3), dtype=dtype, device=device)
self.encoder = Encoder(
@@ -415,8 +451,8 @@ class VAE_3D_V2(nn.Module):
dtype=dtype,
)
- self.quant_conv = nn.Conv3d(latent_embed_dim, 2*kl_embed_dim, 1)
- self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1)
+ self.quant_conv = nn.Conv3d(latent_embed_dim, 2*kl_embed_dim, 1, device=device, dtype=dtype)
+ self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1, device=device, dtype=dtype)
image_down = 2 ** len(temporal_downsample)
t_down = 2 ** len([x for x in temporal_downsample if x == True])
@@ -430,34 +466,99 @@ class VAE_3D_V2(nn.Module):
def encode(
self,
- x,
+ video,
+ video_contains_first_frame = True,
):
- encoded_feature = self.encoder(x)
- moments = self.quant_conv(encoded_feature).to(x.dtype)
+ encode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame
+
+ # whether to pad video or not
+ if video_contains_first_frame:
+ video_len = video.shape[2]
+ video = pad_at_dim(video, (self.time_padding, 0), value = 0., dim = 2)
+ video_packed_shape = [torch.Size([self.time_padding]), torch.Size([]), torch.Size([video_len - 1])]
+
+ # NOTE: moved encoder conv1 here for separate first frame encoding
+ if encode_first_frame_separately:
+ pad, first_frame, video = unpack(video, video_packed_shape, 'b c * h w')
+ first_frame = self.conv_in_first_frame(first_frame)
+ video = self.conv_in(video)
+ if encode_first_frame_separately:
+ video, _ = pack([first_frame, video], 'b c * h w')
+ video = pad_at_dim(video, (self.time_padding, 0), dim = 2)
+
+ encoded_feature = self.encoder(video)
+
+ moments = self.quant_conv(encoded_feature).to(video.dtype)
posterior = model_utils.DiagonalGaussianDistribution(moments)
return posterior
def decode(
self,
z,
+ video_contains_first_frame = True,
):
- dtype = z.dtype
- z = self.post_quant_conv(z).to(dtype)
+ # dtype = z.dtype
+ decode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame
+
+ z = self.post_quant_conv(z)
dec = self.decoder(z)
- return dec
+
+ # SCH: moved decoder last conv layer here for separate first frame decoding
+ if decode_first_frame_separately:
+ left_pad, dec_ff, dec = dec[:, :, :self.time_padding], dec[:, :, self.time_padding], dec[:, :, (self.time_padding + 1):]
+ out = self.conv_out(dec)
+ outff = self.conv_out_first_frame(dec_ff)
+ video, _ = pack([outff, out], 'b c * h w')
+ else:
+ video = self.conv_out(dec)
+ # if video were padded, remove padding
+ if video_contains_first_frame:
+ video = video[:, :, self.time_padding:]
+
+ return video
def forward(
self,
input,
sample_posterior=True,
+ video_contains_first_frame = True,
):
- posterior = self.encode(input)
+ assert input.ndim in {4, 5}, f"received input of {input.ndim} dimensions" # either image or video
+ assert input.shape[-2:] == self.image_size, f"received input size {input.shape[-2:]}, but config image size is {self.image_size}"
+
+ is_image = input.ndim == 4
+
+ if is_image:
+ video = rearrange(input, 'b c ... -> b c 1 ...')
+ video_contains_first_frame = True
+ else:
+ video = input
+
+ batch, channels, frames = video.shape[:3]
+ assert divisible_by(frames - int(video_contains_first_frame), self.time_downsample_factor), f'number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}'
+
+ posterior = self.encode(
+ video,
+ video_contains_first_frame = video_contains_first_frame,
+ )
+
if sample_posterior:
z = posterior.sample()
else:
z = posterior.mode()
- dec = self.decode(z)
- return dec, posterior
+
+ recon_video = self.decode(
+ z,
+ video_contains_first_frame = video_contains_first_frame
+ )
+
+ recon_loss = F.mse_loss(video, recon_video)
+
+ # TODO: add perceptual loss
+
+ # TODO: add adversarial loss
+
+ return recon_loss, recon_video
@MODELS.register_module("VAE_MAGVIT_V2")
diff --git a/scripts/train-vae-v2.py b/scripts/train-vae-v2.py
new file mode 100644
index 0000000..18046c9
--- /dev/null
+++ b/scripts/train-vae-v2.py
@@ -0,0 +1,271 @@
+from copy import deepcopy
+
+import colossalai
+import torch
+import torch.nn as nn
+from torch.optim import Optimizer
+from torch.optim.lr_scheduler import _LRScheduler
+import torch.distributed as dist
+import wandb
+from colossalai.booster import Booster
+from colossalai.booster.plugin import LowLevelZeroPlugin
+from colossalai.cluster import DistCoordinator
+from colossalai.nn.optimizer import HybridAdam
+from colossalai.utils import get_current_device
+from tqdm import tqdm
+import os
+
+from opensora.acceleration.checkpoint import set_grad_checkpoint
+from opensora.acceleration.parallel_states import (
+ get_data_parallel_group,
+ set_data_parallel_group,
+ set_sequence_parallel_group,
+)
+from opensora.acceleration.plugin import ZeroSeqParallelPlugin
+from opensora.datasets import DatasetFromCSV, get_transforms_image, get_transforms_video, prepare_dataloader
+from opensora.registry import MODELS, SCHEDULERS, build_module
+from opensora.utils.ckpt_utils import create_logger, load_json, save_json, load, model_sharding, record_model_param_shape, save
+from opensora.utils.config_utils import (
+ create_experiment_workspace,
+ create_tensorboard_writer,
+ parse_configs,
+ save_training_config,
+)
+from opensora.utils.misc import all_reduce_mean, format_numel_str, get_model_numel, requires_grad, to_torch_dtype
+from opensora.utils.train_utils import update_ema, MaskGenerator
+from opensora.models.vae.model_utils import VEA3DLoss
+
+
+def main():
+ # ======================================================
+ # 1. args & cfg
+ # ======================================================
+ cfg = parse_configs(training=True)
+ print(cfg)
+ exp_name, exp_dir = create_experiment_workspace(cfg)
+ save_training_config(cfg._cfg_dict, exp_dir)
+
+ # ======================================================
+ # 2. runtime variables & colossalai launch
+ # ======================================================
+ assert torch.cuda.is_available(), "Training currently requires at least one GPU."
+ assert cfg.dtype in ["fp16", "bf16"], f"Unknown mixed precision {cfg.dtype}"
+
+ # 2.1. colossalai init distributed training
+ colossalai.launch_from_torch({})
+ coordinator = DistCoordinator()
+ device = get_current_device()
+ dtype = to_torch_dtype(cfg.dtype)
+
+ # 2.2. init logger, tensorboard & wandb
+ if not coordinator.is_master():
+ logger = create_logger(None)
+ else:
+ logger = create_logger(exp_dir)
+ logger.info(f"Experiment directory created at {exp_dir}")
+
+ writer = create_tensorboard_writer(exp_dir)
+ if cfg.wandb:
+ wandb.init(project="opensora-vae", name=exp_name, config=cfg._cfg_dict)
+
+ # 2.3. initialize ColossalAI booster
+ if cfg.plugin == "zero2":
+ plugin = LowLevelZeroPlugin(
+ stage=2,
+ precision=cfg.dtype,
+ initial_scale=2**16,
+ max_norm=cfg.grad_clip,
+ )
+ set_data_parallel_group(dist.group.WORLD)
+ elif cfg.plugin == "zero2-seq":
+ plugin = ZeroSeqParallelPlugin(
+ sp_size=cfg.sp_size,
+ stage=2,
+ precision=cfg.dtype,
+ initial_scale=2**16,
+ max_norm=cfg.grad_clip,
+ )
+ set_sequence_parallel_group(plugin.sp_group)
+ set_data_parallel_group(plugin.dp_group)
+ else:
+ raise ValueError(f"Unknown plugin {cfg.plugin}")
+ booster = Booster(plugin=plugin)
+
+ # ======================================================
+ # 3. build dataset and dataloader
+ # ======================================================
+ dataset = DatasetFromCSV(
+ cfg.data_path,
+ # TODO: change transforms
+ transform=(
+ get_transforms_video(cfg.image_size[0])
+ if not cfg.use_image_transform
+ else get_transforms_image(cfg.image_size[0])
+ ),
+ num_frames=cfg.num_frames,
+ frame_interval=cfg.frame_interval,
+ root=cfg.root,
+ )
+
+ # TODO: use plugin's prepare dataloader
+ # a batch contains:
+ # {
+ # "video": torch.Tensor, # [B, C, T, H, W],
+ # "text": List[str],
+ # }
+ dataloader = prepare_dataloader(
+ dataset,
+ batch_size=cfg.batch_size,
+ num_workers=cfg.num_workers,
+ shuffle=True,
+ drop_last=True,
+ pin_memory=True,
+ process_group=get_data_parallel_group(),
+ )
+ logger.info(f"Dataset contains {len(dataset):,} videos ({cfg.data_path})")
+
+ total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size
+ logger.info(f"Total batch size: {total_batch_size}")
+
+ # ======================================================
+ # 4. build model
+ # ======================================================
+ # 4.1. build model
+ vae = build_module(cfg.model, MODELS, device=device)
+ vae_numel, vae_numel_trainable = get_model_numel(vae)
+ logger.info(
+ f"Trainable vae params: {format_numel_str(vae_numel_trainable)}, Total model params: {format_numel_str(vae_numel)}"
+ )
+
+
+ # 4.3. move to device
+ vae = vae.to(device, dtype)
+
+ # 4.5. setup optimizer
+ optimizer = HybridAdam(
+ filter(lambda p: p.requires_grad, vae.parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True
+ )
+ lr_scheduler = None
+
+ # 4.6. prepare for training
+ if cfg.grad_checkpoint:
+ set_grad_checkpoint(vae)
+ vae.train()
+
+
+ # =======================================================
+ # 5. boost model for distributed training with colossalai
+ # =======================================================
+ torch.set_default_dtype(dtype)
+ vae, optimizer, _, dataloader, lr_scheduler = booster.boost(
+ model=vae, optimizer=optimizer, lr_scheduler=lr_scheduler, dataloader=dataloader
+ )
+ torch.set_default_dtype(torch.float)
+ num_steps_per_epoch = len(dataloader)
+ logger.info("Boost vae for distributed training")
+
+ # =======================================================
+ # 6. training loop
+ # =======================================================
+ start_epoch = start_step = log_step = sampler_start_idx = 0
+ running_loss = 0.0
+
+
+ # 6.1. resume training
+ if cfg.load is not None:
+ logger.info("Loading checkpoint")
+ booster.load_model(vae, os.path.join(cfg.load, "model"))
+ booster.load_optimizer(optimizer, os.path.join(cfg.load, "optimizer"))
+ if lr_scheduler is not None:
+ booster.load_lr_scheduler(lr_scheduler, os.path.join(cfg.load, "lr_scheduler"))
+ running_states = load_json(os.path.join(cfg.load, "running_states.json"))
+ dist.barrier()
+ start_epoch, start_step, sampler_start_idx = running_states["epoch"], running_states["step"], running_states["sample_start_index"]
+ logger.info(f"Loaded checkpoint {cfg.load} at epoch {start_epoch} step {start_step}")
+ logger.info(f"Training for {cfg.epochs} epochs with {num_steps_per_epoch} steps per epoch")
+
+ dataloader.sampler.set_start_index(sampler_start_idx)
+
+ # # define loss function
+ # loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)
+
+
+ # 6.2. training loop
+ for epoch in range(start_epoch, cfg.epochs):
+ dataloader.sampler.set_epoch(epoch)
+ dataloader_iter = iter(dataloader)
+ logger.info(f"Beginning epoch {epoch}...")
+
+ with tqdm(
+ range(start_step, num_steps_per_epoch),
+ desc=f"Epoch {epoch}",
+ disable=not coordinator.is_master(),
+ total=num_steps_per_epoch,
+ initial=start_step,
+ ) as pbar:
+ for step in pbar:
+ batch = next(dataloader_iter)
+ x = batch["video"].to(device, dtype) # [B, C, T, H, W]
+
+ # loss = vae.get_loss(x)
+ loss, reconstructions = vae(x)
+ # loss = loss_function(x, reconstructions, posterior)
+
+ # Backward & update
+ booster.backward(loss=loss, optimizer=optimizer)
+ optimizer.step()
+ optimizer.zero_grad()
+
+ # Log loss values:
+ all_reduce_mean(loss)
+ running_loss += loss.item()
+ global_step = epoch * num_steps_per_epoch + step
+ log_step += 1
+
+ # Log to tensorboard
+ if coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:
+ avg_loss = running_loss / log_step
+ pbar.set_postfix({"loss": avg_loss, "step": step, "global_step": global_step})
+ running_loss = 0
+ log_step = 0
+ writer.add_scalar("loss", loss.item(), global_step)
+ if cfg.wandb:
+ wandb.log(
+ {
+ "iter": global_step,
+ "num_samples": global_step * total_batch_size,
+ "epoch": epoch,
+ "loss": loss.item(),
+ "avg_loss": avg_loss,
+ },
+ step=global_step,
+ )
+
+ # Save checkpoint
+ if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:
+ save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")
+ os.makedirs(os.path.join(save_dir, "model"), exist_ok=True)
+ # TODO: save in model?
+ booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)
+ booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)
+ if lr_scheduler is not None:
+ booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))
+ running_states = {
+ "epoch": epoch,
+ "step": step+1,
+ "global_step": global_step+1,
+ "sample_start_index": (step+1) * cfg.batch_size,
+ }
+ if coordinator.is_master():
+ save_json(running_states, os.path.join(save_dir, "running_states.json"))
+ dist.barrier()
+ logger.info(
+ f"Saved checkpoint at epoch {epoch} step {step + 1} global_step {global_step + 1} to {exp_dir}"
+ )
+
+ # the continue epochs are not resumed, so we need to reset the sampler start index and start step
+ dataloader.sampler.set_start_index(0)
+ start_step = 0
+
+if __name__ == "__main__":
+ main()