Open-Sora: Democratizing Efficient Video Production for All
Python
add image encoding support
Shen-Chenhuiee58ec16b4f0
diff --git a/configs/vae_magvit_v2/train/17x128x128.py b/configs/vae_magvit_v2/train/17x128x128.pyindex ba08169..bad6a74 100644--- a/configs/vae_magvit_v2/train/17x128x128.py+++ b/configs/vae_magvit_v2/train/17x128x128.py@@ -32,6 +32,8 @@ model = dict(kl_embed_dim = 64,custom_conv_padding = None,activation_fn = 'swish',+ image_size = image_size,+ separate_first_frame_encoding = False,)# Othersdiff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.pyindex 5619e4b..cd64e09 100644--- a/opensora/models/vae/vae_3d_v2.py+++ b/opensora/models/vae/vae_3d_v2.py@@ -8,7 +8,7 @@ from numpy import typing as nptypingfrom opensora.models.vae import model_utilsfrom opensora.registry import MODELSfrom opensora.utils.ckpt_utils import load_checkpoint-from einops import rearrange+from einops import rearrange, pack, unpackimport torch.nn.functional as F"""Encoder and Decoder stuctures with 3D CNNs."""@@ -33,6 +33,16 @@ def divisible_by(num, den):def is_odd(n):return not divisible_by(n, 2)+def pad_at_dim(t, pad, dim = -1, value = 0.):+ dims_from_right = (- dim - 1) if dim < 0 else (t.ndim - dim - 1)+ zeros = ((0, 0) * dims_from_right)+ return F.pad(t, (*zeros, *pad), value = value)++def SameConv2d(dim_in, dim_out, kernel_size):+ kernel_size = cast_tuple(kernel_size, 2)+ padding = [k // 2 for k in kernel_size]+ return nn.Conv2d(dim_in, dim_out, kernel_size = kernel_size, padding = padding)+class CausalConv3d(nn.Module):def __init__(self,@@ -172,7 +182,8 @@ class Encoder(nn.Module):device=device,)- self.conv1 = self.conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)+ # NOTE: moved to VAE for separate first frame processing+ # self.conv1 = self.conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)# ResBlocks and conv downsampleself.block_res_blocks = []@@ -210,7 +221,10 @@ class Encoder(nn.Module):def forward(self, x):# dtype, device = x.dtype, x.device- x = self.conv1(x)++ # NOTE: moved to VAE for separate first frame processing+ # x = self.conv1(x)+for i in range(self.num_blocks):for j in range(self.num_res_blocks):x = self.block_res_blocks[i][j](x)@@ -323,7 +337,9 @@ class Decoder(nn.Module):)self.norm1 = nn.GroupNorm(self.num_groups, prev_filters, device=device, dtype=dtype)- self.conv2 = self.conv_fn(prev_filters, self.output_dim, kernel_size=(3, 3, 3))++ # NOTE: moved to VAE for separate first frame processing+ # self.conv2 = self.conv_fn(prev_filters, self.output_dim, kernel_size=(3, 3, 3))def forward(@@ -348,7 +364,8 @@ class Decoder(nn.Module):x = self.norm1(x)x = self.activate(x)- x = self.conv2(x)+ # NOTE: moved to VAE for separate first frame processing+ # x = self.conv2(x)return x@@ -358,8 +375,10 @@ class VAE_3D_V2(nn.Module):def __init__(self,latent_embed_dim = 256,- filters = 64,+ filters = 128,num_res_blocks = 2,+ image_size = (128, 128),+ separate_first_frame_encoding = False,channel_multipliers = (1, 2, 2, 4),temporal_downsample = (True, True, False),num_groups = 32, # for nn.GroupNorm@@ -382,8 +401,25 @@ class VAE_3D_V2(nn.Module):dtype = torch.float16else:raise NotImplementedError(f'dtype: {dtype}')+++ # ==== Model Params ====+ self.image_size = cast_tuple(image_size, 2)+ self.time_downsample_factor = 2**sum(temporal_downsample)+ self.time_padding = self.time_downsample_factor - 1+ self.separate_first_frame_encoding = separate_first_frame_encoding- # Model Initialization+ # ==== Model Initialization ====++ # encoder & decoder first and last conv layer+ # SCH: NOTE: following MAGVIT, conv in bias=False in encoder first conv+ self.conv_in = CausalConv3d(in_out_channels, filters, kernel_size=(3, 3, 3), bias=False, dtype=dtype, device=device)+ self.conv_in_first_frame = nn.Identity()+ self.conv_out_first_frame = nn.Identity()+ if separate_first_frame_encoding:+ self.conv_in_first_frame = SameConv2d(in_out_channels, filters, (3,3))+ self.conv_out_first_frame = SameConv2d(filters, in_out_channels, (3,3))+ self.conv_out = CausalConv3d(filters, in_out_channels, (3,3), dtype=dtype, device=device)self.encoder = Encoder(@@ -415,8 +451,8 @@ class VAE_3D_V2(nn.Module):dtype=dtype,)- self.quant_conv = nn.Conv3d(latent_embed_dim, 2*kl_embed_dim, 1)- self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1)+ self.quant_conv = nn.Conv3d(latent_embed_dim, 2*kl_embed_dim, 1, device=device, dtype=dtype)+ self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1, device=device, dtype=dtype)image_down = 2 ** len(temporal_downsample)t_down = 2 ** len([x for x in temporal_downsample if x == True])@@ -430,34 +466,99 @@ class VAE_3D_V2(nn.Module):def encode(self,- x,+ video,+ video_contains_first_frame = True,):- encoded_feature = self.encoder(x)- moments = self.quant_conv(encoded_feature).to(x.dtype)+ encode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame++ # whether to pad video or not+ if video_contains_first_frame:+ video_len = video.shape[2]+ video = pad_at_dim(video, (self.time_padding, 0), value = 0., dim = 2)+ video_packed_shape = [torch.Size([self.time_padding]), torch.Size([]), torch.Size([video_len - 1])]++ # NOTE: moved encoder conv1 here for separate first frame encoding+ if encode_first_frame_separately:+ pad, first_frame, video = unpack(video, video_packed_shape, 'b c * h w')+ first_frame = self.conv_in_first_frame(first_frame)+ video = self.conv_in(video)+ if encode_first_frame_separately:+ video, _ = pack([first_frame, video], 'b c * h w')+ video = pad_at_dim(video, (self.time_padding, 0), dim = 2)++ encoded_feature = self.encoder(video)++ moments = self.quant_conv(encoded_feature).to(video.dtype)posterior = model_utils.DiagonalGaussianDistribution(moments)return posteriordef decode(self,z,+ video_contains_first_frame = True,):- dtype = z.dtype- z = self.post_quant_conv(z).to(dtype)+ # dtype = z.dtype+ decode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame++ z = self.post_quant_conv(z)dec = self.decoder(z)- return dec++ # SCH: moved decoder last conv layer here for separate first frame decoding+ if decode_first_frame_separately:+ left_pad, dec_ff, dec = dec[:, :, :self.time_padding], dec[:, :, self.time_padding], dec[:, :, (self.time_padding + 1):]+ out = self.conv_out(dec)+ outff = self.conv_out_first_frame(dec_ff)+ video, _ = pack([outff, out], 'b c * h w')+ else:+ video = self.conv_out(dec)+ # if video were padded, remove padding+ if video_contains_first_frame:+ video = video[:, :, self.time_padding:]++ return videodef forward(self,input,sample_posterior=True,+ video_contains_first_frame = True,):- posterior = self.encode(input)+ assert input.ndim in {4, 5}, f"received input of {input.ndim} dimensions" # either image or video+ assert input.shape[-2:] == self.image_size, f"received input size {input.shape[-2:]}, but config image size is {self.image_size}"++ is_image = input.ndim == 4++ if is_image:+ video = rearrange(input, 'b c ... -> b c 1 ...')+ video_contains_first_frame = True+ else:+ video = input++ batch, channels, frames = video.shape[:3]+ assert divisible_by(frames - int(video_contains_first_frame), self.time_downsample_factor), f'number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}'++ posterior = self.encode(+ video,+ video_contains_first_frame = video_contains_first_frame,+ )+if sample_posterior:z = posterior.sample()else:z = posterior.mode()- dec = self.decode(z)- return dec, posterior++ recon_video = self.decode(+ z,+ video_contains_first_frame = video_contains_first_frame+ )++ recon_loss = F.mse_loss(video, recon_video)++ # TODO: add perceptual loss++ # TODO: add adversarial loss++ return recon_loss, recon_video@MODELS.register_module("VAE_MAGVIT_V2")diff --git a/scripts/train-vae-v2.py b/scripts/train-vae-v2.pynew file mode 100644index 0000000..18046c9--- /dev/null+++ b/scripts/train-vae-v2.py@@ -0,0 +1,271 @@+from copy import deepcopy++import colossalai+import torch+import torch.nn as nn+from torch.optim import Optimizer+from torch.optim.lr_scheduler import _LRScheduler+import torch.distributed as dist+import wandb+from colossalai.booster import Booster+from colossalai.booster.plugin import LowLevelZeroPlugin+from colossalai.cluster import DistCoordinator+from colossalai.nn.optimizer import HybridAdam+from colossalai.utils import get_current_device+from tqdm import tqdm+import os++from opensora.acceleration.checkpoint import set_grad_checkpoint+from opensora.acceleration.parallel_states import (+ get_data_parallel_group,+ set_data_parallel_group,+ set_sequence_parallel_group,+)+from opensora.acceleration.plugin import ZeroSeqParallelPlugin+from opensora.datasets import DatasetFromCSV, get_transforms_image, get_transforms_video, prepare_dataloader+from opensora.registry import MODELS, SCHEDULERS, build_module+from opensora.utils.ckpt_utils import create_logger, load_json, save_json, load, model_sharding, record_model_param_shape, save+from opensora.utils.config_utils import (+ create_experiment_workspace,+ create_tensorboard_writer,+ parse_configs,+ save_training_config,+)+from opensora.utils.misc import all_reduce_mean, format_numel_str, get_model_numel, requires_grad, to_torch_dtype+from opensora.utils.train_utils import update_ema, MaskGenerator+from opensora.models.vae.model_utils import VEA3DLoss+++def main():+ # ======================================================+ # 1. args & cfg+ # ======================================================+ cfg = parse_configs(training=True)+ print(cfg)+ exp_name, exp_dir = create_experiment_workspace(cfg)+ save_training_config(cfg._cfg_dict, exp_dir)++ # ======================================================+ # 2. runtime variables & colossalai launch+ # ======================================================+ assert torch.cuda.is_available(), "Training currently requires at least one GPU."+ assert cfg.dtype in ["fp16", "bf16"], f"Unknown mixed precision {cfg.dtype}"++ # 2.1. colossalai init distributed training+ colossalai.launch_from_torch({})+ coordinator = DistCoordinator()+ device = get_current_device()+ dtype = to_torch_dtype(cfg.dtype)++ # 2.2. init logger, tensorboard & wandb+ if not coordinator.is_master():+ logger = create_logger(None)+ else:+ logger = create_logger(exp_dir)+ logger.info(f"Experiment directory created at {exp_dir}")++ writer = create_tensorboard_writer(exp_dir)+ if cfg.wandb:+ wandb.init(project="opensora-vae", name=exp_name, config=cfg._cfg_dict)++ # 2.3. initialize ColossalAI booster+ if cfg.plugin == "zero2":+ plugin = LowLevelZeroPlugin(+ stage=2,+ precision=cfg.dtype,+ initial_scale=2**16,+ max_norm=cfg.grad_clip,+ )+ set_data_parallel_group(dist.group.WORLD)+ elif cfg.plugin == "zero2-seq":+ plugin = ZeroSeqParallelPlugin(+ sp_size=cfg.sp_size,+ stage=2,+ precision=cfg.dtype,+ initial_scale=2**16,+ max_norm=cfg.grad_clip,+ )+ set_sequence_parallel_group(plugin.sp_group)+ set_data_parallel_group(plugin.dp_group)+ else:+ raise ValueError(f"Unknown plugin {cfg.plugin}")+ booster = Booster(plugin=plugin)++ # ======================================================+ # 3. build dataset and dataloader+ # ======================================================+ dataset = DatasetFromCSV(+ cfg.data_path,+ # TODO: change transforms+ transform=(+ get_transforms_video(cfg.image_size[0])+ if not cfg.use_image_transform+ else get_transforms_image(cfg.image_size[0])+ ),+ num_frames=cfg.num_frames,+ frame_interval=cfg.frame_interval,+ root=cfg.root,+ )++ # TODO: use plugin's prepare dataloader+ # a batch contains:+ # {+ # "video": torch.Tensor, # [B, C, T, H, W],+ # "text": List[str],+ # }+ dataloader = prepare_dataloader(+ dataset,+ batch_size=cfg.batch_size,+ num_workers=cfg.num_workers,+ shuffle=True,+ drop_last=True,+ pin_memory=True,+ process_group=get_data_parallel_group(),+ )+ logger.info(f"Dataset contains {len(dataset):,} videos ({cfg.data_path})")++ total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size+ logger.info(f"Total batch size: {total_batch_size}")++ # ======================================================+ # 4. build model+ # ======================================================+ # 4.1. build model+ vae = build_module(cfg.model, MODELS, device=device)+ vae_numel, vae_numel_trainable = get_model_numel(vae)+ logger.info(+ f"Trainable vae params: {format_numel_str(vae_numel_trainable)}, Total model params: {format_numel_str(vae_numel)}"+ )+++ # 4.3. move to device+ vae = vae.to(device, dtype)++ # 4.5. setup optimizer+ optimizer = HybridAdam(+ filter(lambda p: p.requires_grad, vae.parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True+ )+ lr_scheduler = None++ # 4.6. prepare for training+ if cfg.grad_checkpoint:+ set_grad_checkpoint(vae)+ vae.train()+++ # =======================================================+ # 5. boost model for distributed training with colossalai+ # =======================================================+ torch.set_default_dtype(dtype)+ vae, optimizer, _, dataloader, lr_scheduler = booster.boost(+ model=vae, optimizer=optimizer, lr_scheduler=lr_scheduler, dataloader=dataloader+ )+ torch.set_default_dtype(torch.float)+ num_steps_per_epoch = len(dataloader)+ logger.info("Boost vae for distributed training")++ # =======================================================+ # 6. training loop+ # =======================================================+ start_epoch = start_step = log_step = sampler_start_idx = 0+ running_loss = 0.0+++ # 6.1. resume training+ if cfg.load is not None:+ logger.info("Loading checkpoint")+ booster.load_model(vae, os.path.join(cfg.load, "model"))+ booster.load_optimizer(optimizer, os.path.join(cfg.load, "optimizer"))+ if lr_scheduler is not None:+ booster.load_lr_scheduler(lr_scheduler, os.path.join(cfg.load, "lr_scheduler"))+ running_states = load_json(os.path.join(cfg.load, "running_states.json"))+ dist.barrier()+ start_epoch, start_step, sampler_start_idx = running_states["epoch"], running_states["step"], running_states["sample_start_index"]+ logger.info(f"Loaded checkpoint {cfg.load} at epoch {start_epoch} step {start_step}")+ logger.info(f"Training for {cfg.epochs} epochs with {num_steps_per_epoch} steps per epoch")++ dataloader.sampler.set_start_index(sampler_start_idx)++ # # define loss function+ # loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)+++ # 6.2. training loop+ for epoch in range(start_epoch, cfg.epochs):+ dataloader.sampler.set_epoch(epoch)+ dataloader_iter = iter(dataloader)+ logger.info(f"Beginning epoch {epoch}...")++ with tqdm(+ range(start_step, num_steps_per_epoch),+ desc=f"Epoch {epoch}",+ disable=not coordinator.is_master(),+ total=num_steps_per_epoch,+ initial=start_step,+ ) as pbar:+ for step in pbar:+ batch = next(dataloader_iter)+ x = batch["video"].to(device, dtype) # [B, C, T, H, W]++ # loss = vae.get_loss(x)+ loss, reconstructions = vae(x)+ # loss = loss_function(x, reconstructions, posterior)++ # Backward & update+ booster.backward(loss=loss, optimizer=optimizer)+ optimizer.step()+ optimizer.zero_grad()++ # Log loss values:+ all_reduce_mean(loss)+ running_loss += loss.item()+ global_step = epoch * num_steps_per_epoch + step+ log_step += 1++ # Log to tensorboard+ if coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:+ avg_loss = running_loss / log_step+ pbar.set_postfix({"loss": avg_loss, "step": step, "global_step": global_step})+ running_loss = 0+ log_step = 0+ writer.add_scalar("loss", loss.item(), global_step)+ if cfg.wandb:+ wandb.log(+ {+ "iter": global_step,+ "num_samples": global_step * total_batch_size,+ "epoch": epoch,+ "loss": loss.item(),+ "avg_loss": avg_loss,+ },+ step=global_step,+ )++ # Save checkpoint+ if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:+ save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")+ os.makedirs(os.path.join(save_dir, "model"), exist_ok=True)+ # TODO: save in model?+ booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)+ booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)+ if lr_scheduler is not None:+ booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))+ running_states = {+ "epoch": epoch,+ "step": step+1,+ "global_step": global_step+1,+ "sample_start_index": (step+1) * cfg.batch_size,+ }+ if coordinator.is_master():+ save_json(running_states, os.path.join(save_dir, "running_states.json"))+ dist.barrier()+ logger.info(+ f"Saved checkpoint at epoch {epoch} step {step + 1} global_step {global_step + 1} to {exp_dir}"+ )++ # the continue epochs are not resumed, so we need to reset the sampler start index and start step+ dataloader.sampler.set_start_index(0)+ start_step = 0++if __name__ == "__main__":+ main()