Open-Sora: Democratizing Efficient Video Production for All
Python
inference v2
Shen-Chenhuibecff9d86fa0
diff --git a/configs/vae_3d/inference/16x256x256.py b/configs/vae_3d/inference/16x256x256.pyindex 5cda9bd..89591d3 100644--- a/configs/vae_3d/inference/16x256x256.py+++ b/configs/vae_3d/inference/16x256x256.py@@ -53,4 +53,6 @@ load = Nonebatch_size = 8grad_clip = 1.0-lr = 1e-4\ No newline at end of file+lr = 1e-4++calc_loss = False\ No newline at end of filediff --git a/configs/vae_magvit_v2/inference/16x128x128.py b/configs/vae_magvit_v2/inference/16x128x128.pynew file mode 100644index 0000000..4ed6d2d--- /dev/null+++ b/configs/vae_magvit_v2/inference/16x128x128.py@@ -0,0 +1,81 @@+num_frames = 16+frame_interval = 3+image_size = (128, 128)++# Define dataset+root = None+data_path = "CSV_PATH"+use_image_transform = False+num_workers = 4+video_contains_first_frame = False++# Define acceleration+dtype = "bf16"+grad_checkpoint = True+plugin = "zero2"+sp_size = 1+++# Define model++model = dict(+ type="VAE_MAGVIT_V2",+ in_out_channels = 3,+ latent_embed_dim = 256,+ filters = 128,+ num_res_blocks = 4,+ channel_multipliers = (1, 2, 2, 4),+ temporal_downsample = (False, True, True),+ num_groups = 32, # for nn.GroupNorm+ kl_embed_dim = 64,+ activation_fn = 'swish',+ separate_first_frame_encoding = False,+ custom_conv_padding = None+)+++discriminator = dict(+ type="DISCRIMINATOR_3D",+ image_size = image_size,+ num_frames = num_frames,+ in_channels = 3,+ filters = 128,+ channel_multipliers = (2,4,4,4,4) # (2,4,4,4) for 64x64 resolution+)+++# loss weights+logvar_init=0.0+kl_loss_weight = 0.000001+perceptual_loss_weight = 0.1 # use vgg is not None and more than 0+discriminator_factor = 1.0 # for discriminator adversarial loss+# discriminator_loss_weight = 0.5 # for generator adversarial loss+generator_factor = 0.5 # for generator adversarial loss+lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight+discriminator_loss_type="non-saturating"+generator_loss_type="non-saturating"+discriminator_start = 50000 # 50000 NOTE: change to correct val, debug use -1 for now+gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use+ema_decay = 0.999 # ema decay factor for generator+++# Others+seed = 42+outputs = "outputs/samples_v2"+wandb = False++# Training+''' NOTE:+magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128+==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],+3-6 epochs for pexel, from pexel observation its correct+'''++epochs = 10+log_every = 1+ckpt_every = 1000+load = None++batch_size = 4+lr = 1e-4+grad_clip = 1.0diff --git a/scripts/inference-debug.py b/scripts/inference-debug.pydeleted file mode 100644index 18183cf..0000000--- a/scripts/inference-debug.py+++ /dev/null@@ -1,151 +0,0 @@-import os--import colossalai-import torch-import torch.distributed as dist-from colossalai.cluster import DistCoordinator-from mmengine.runner import set_random_seed--from opensora.acceleration.parallel_states import set_sequence_parallel_group-from opensora.datasets import save_sample-from opensora.registry import MODELS, SCHEDULERS, build_module-from opensora.utils.config_utils import parse_configs-from opensora.utils.misc import to_torch_dtype-from opensora.datasets import DatasetFromCSV, get_transforms_image, get_transforms_video, prepare_dataloader-from opensora.acceleration.parallel_states import (- get_data_parallel_group,- set_data_parallel_group,- set_sequence_parallel_group,-)-from tqdm import tqdm-from opensora.models.vae.model_utils import VEA3DLoss--# DEBUG-from colossalai.utils import get_current_device---def main():- # ======================================================- # 1. cfg and init distributed env- # ======================================================- cfg = parse_configs(training=False)- print(cfg)-- # init distributed- colossalai.launch_from_torch({})- coordinator = DistCoordinator()-- # if coordinator.world_size > 1:- # set_sequence_parallel_group(dist.group.WORLD)- # enable_sequence_parallelism = True- # else:- # enable_sequence_parallelism = False-- # ======================================================- # 2. runtime variables- # ======================================================- torch.set_grad_enabled(False)- # torch.backends.cuda.matmul.allow_tf32 = True- # torch.backends.cudnn.allow_tf32 = True- # device = "cuda" if torch.cuda.is_available() else "cpu"- device = get_current_device()- dtype = to_torch_dtype(cfg.dtype)- # set_random_seed(seed=cfg.seed) # Issue is this line !!!!!!!----- # ======================================================- # 3. build dataset and dataloader- # ======================================================- dataset = DatasetFromCSV(- cfg.data_path,- # TODO: change transforms- transform=(- get_transforms_video(cfg.image_size[0])- if not cfg.use_image_transform- else get_transforms_image(cfg.image_size[0])- ),- num_frames=cfg.num_frames,- frame_interval=cfg.frame_interval,- root=cfg.root,- )-- dataloader = prepare_dataloader(- dataset,- batch_size=cfg.batch_size,- num_workers=cfg.num_workers,- shuffle=True,- drop_last=True,- pin_memory=True,- process_group=get_data_parallel_group(),- )- print(f"Dataset contains {len(dataset):,} videos ({cfg.data_path})")-- total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size- print(f"Total batch size: {total_batch_size}")-- # ======================================================- # 4. build model & load weights- # ======================================================- # 3.1. build model- # input_size = (cfg.num_frames, *cfg.image_size)- vae = build_module(cfg.model, MODELS, device=device)- # latent_size = vae.get_latent_size(input_size)-- # 3.2. move to device & eval- vae = vae.to(device, dtype).eval()--- # 3.4. support for multi-resolution- model_args = dict()- if cfg.multi_resolution:- image_size = cfg.image_size- hw = torch.tensor([image_size], device=device, dtype=dtype).repeat(cfg.batch_size, 1)- ar = torch.tensor([[image_size[0] / image_size[1]]], device=device, dtype=dtype).repeat(cfg.batch_size, 1)- model_args["data_info"] = dict(ar=ar, hw=hw)-- # ======================================================- # 4. inference- # ======================================================- save_dir = cfg.save_dir- os.makedirs(save_dir, exist_ok=True)--- # 4.1. batch generation-- # define loss function- loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)- running_loss = 0.0- loss_steps = 0--- total_steps = len(dataloader)- dataloader_iter = iter(dataloader)-- with tqdm(- range(total_steps),- # desc=f"Avg Loss: {running_loss}",- disable=not coordinator.is_master(),- total=total_steps,- initial=0,- ) as pbar:- for step in pbar:- batch = next(dataloader_iter)- x = batch["video"].to(device, dtype) # [B, C, T, H, W]- reconstructions, posterior = vae(x)- loss = loss_function(x, reconstructions, posterior)- loss_steps += 1- running_loss = loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)-- if coordinator.is_master():- for idx, sample in enumerate(reconstructions):- pos = step * cfg.batch_size + idx- save_path = os.path.join(save_dir, f"sample_{pos}")- save_sample(sample, fps=cfg.fps, save_path=save_path)-- print("test loss:", running_loss)---if __name__ == "__main__":- main()diff --git a/scripts/inference-vae-v2.py b/scripts/inference-vae-v2.pynew file mode 100644index 0000000..47e4159--- /dev/null+++ b/scripts/inference-vae-v2.py@@ -0,0 +1,240 @@+import os++import colossalai+import torch+import torch.distributed as dist+from colossalai.cluster import DistCoordinator+from mmengine.runner import set_random_seed++from opensora.acceleration.parallel_states import set_sequence_parallel_group+from opensora.datasets import save_sample+from opensora.registry import MODELS, SCHEDULERS, build_module+from opensora.utils.config_utils import parse_configs+from opensora.utils.misc import to_torch_dtype+from opensora.datasets import DatasetFromCSV, get_transforms_image, get_transforms_video, prepare_dataloader+from opensora.acceleration.parallel_states import (+ get_data_parallel_group,+ set_data_parallel_group,+ set_sequence_parallel_group,+)+from tqdm import tqdm+from opensora.models.vae.vae_3d_v2 import VEALoss, DiscriminatorLoss, AdversarialLoss, pad_at_dim++from einops import rearrange+from colossalai.utils import get_current_device+++def main():+ # ======================================================+ # 1. cfg and init distributed env+ # ======================================================+ cfg = parse_configs(training=False)+ print(cfg)++ # init distributed+ colossalai.launch_from_torch({})+ coordinator = DistCoordinator()++ # if coordinator.world_size > 1:+ # set_sequence_parallel_group(dist.group.WORLD)+ # enable_sequence_parallelism = True+ # else:+ # enable_sequence_parallelism = False++ # ======================================================+ # 2. runtime variables+ # ======================================================+ torch.set_grad_enabled(False)+ device = get_current_device()+ dtype = to_torch_dtype(cfg.dtype)+++++ # ======================================================+ # 3. build dataset and dataloader+ # ======================================================+ dataset = DatasetFromCSV(+ cfg.data_path,+ transform=(+ get_transforms_video(cfg.image_size[0])+ if not cfg.use_image_transform+ else get_transforms_image(cfg.image_size[0])+ ),+ num_frames=cfg.num_frames,+ frame_interval=cfg.frame_interval,+ root=cfg.root,+ )++ dataloader = prepare_dataloader(+ dataset,+ batch_size=cfg.batch_size,+ num_workers=cfg.num_workers,+ shuffle=True,+ drop_last=True,+ pin_memory=True,+ process_group=get_data_parallel_group(),+ )+ print(f"Dataset contains {len(dataset):,} videos ({cfg.data_path})")++ total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size+ print(f"Total batch size: {total_batch_size}")++ # ======================================================+ # 4. build model & load weights+ # ======================================================+ # 3.1. build model+ vae = build_module(cfg.model, MODELS, device=device)+ discriminator = build_module(cfg.discriminator, MODELS, device=device)++ # 3.2. move to device & eval+ vae = vae.to(device, dtype).eval()+ discriminator = discriminator.to(device, dtype).eval()++ # 3.4. support for multi-resolution+ model_args = dict()+ if cfg.multi_resolution:+ image_size = cfg.image_size+ hw = torch.tensor([image_size], device=device, dtype=dtype).repeat(cfg.batch_size, 1)+ ar = torch.tensor([[image_size[0] / image_size[1]]], device=device, dtype=dtype).repeat(cfg.batch_size, 1)+ model_args["data_info"] = dict(ar=ar, hw=hw)++ # ======================================================+ # 4. inference+ # ======================================================+ save_dir = cfg.save_dir+ os.makedirs(save_dir, exist_ok=True)++++ # 4.1. batch generation++ # define loss function+ if cfg.calc_loss:+ vae_loss_fn = VEALoss(+ logvar_init=cfg.logvar_init,+ perceptual_loss_weight = cfg.perceptual_loss_weight,+ kl_loss_weight = cfg.kl_loss_weight,+ device=device,+ dtype=dtype,+ )++ adversarial_loss_fn = AdversarialLoss(+ discriminator_factor = cfg.discriminator_factor,+ discriminator_start = cfg.discriminator_start,+ generator_factor = cfg.generator_factor,+ generator_loss_type = cfg.generator_loss_type,+ )++ disc_loss_fn = DiscriminatorLoss(+ discriminator_factor = cfg.discriminator_factor,+ discriminator_start = cfg.discriminator_start,+ discriminator_loss_type = cfg.discriminator_loss_type,+ lecam_loss_weight = cfg.lecam_loss_weight,+ gradient_penalty_loss_weight = cfg.gradient_penalty_loss_weight,+ )+++ running_loss = 0.0+ running_disc_loss = 0.0+ loss_steps = 0++ disc_time_downsample_factor = 2 ** len(cfg.discriminator.channel_multipliers)+ disc_time_padding = disc_time_downsample_factor - cfg.num_frames % disc_time_downsample_factor+ video_contains_first_frame = cfg.video_contains_first_frame++ lecam_ema_real = torch.tensor(0.0)+ lecam_ema_fake = torch.tensor(0.0)++ total_steps = len(dataloader)+ dataloader_iter = iter(dataloader)++ with tqdm(+ range(total_steps),+ # desc=f"Avg Loss: {running_loss}",+ disable=not coordinator.is_master(),+ total=total_steps,+ initial=0,+ ) as pbar:+ for step in pbar:+ batch = next(dataloader_iter)+ x = batch["video"].to(device, dtype) # [B, C, T, H, W]++ is_image = x.ndim == 4+ if is_image:+ video = rearrange(x, 'b c ... -> b c 1 ...')+ video_contains_first_frame = True+ else:+ video = x++ recon_video, posterior = vae(+ video,+ video_contains_first_frame = video_contains_first_frame+ )++ if cfg.calc_loss:+ # ====== Calc Loss ======+ # simple nll loss+ nll_loss, weighted_nll_loss, weighted_kl_loss = vae_loss_fn(+ video,+ recon_video,+ posterior,+ split = "eval"+ )++ fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)+ fake_logits = discriminator(fake_video.contiguous()) # TODO: take out contiguous?+ adversarial_loss = adversarial_loss_fn(+ fake_logits,+ nll_loss,+ vae.module.get_last_layer(),+ cfg.discriminator_start+1, # Hack to use discriminator+ is_training = vae.training,+ )++ vae_loss = weighted_nll_loss + weighted_kl_loss + adversarial_loss++ # ====== Discriminator Loss ======+ real_video = pad_at_dim(video, (disc_time_padding, 0), value = 0., dim = 2)+ fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)++ if cfg.gradient_penalty_loss_weight is not None and cfg.gradient_penalty_loss_weight > 0.0:+ real_video = real_video.requires_grad_()+ real_logits = discriminator(real_video.contiguous()) # SCH: not detached for now for gradient_penalty calculation+ else:+ real_logits = discriminator(real_video.contiguous().detach())++ fake_logits = discriminator(fake_video.contiguous().detach())+ disc_loss = disc_loss_fn(+ real_logits,+ fake_logits,+ cfg.discriminator_start+1, # Hack to use discriminator+ lecam_ema_real = lecam_ema_real,+ lecam_ema_fake = lecam_ema_fake,+ real_video = real_video if cfg.gradient_penalty_loss_weight is not None else None,+ )++ if cfg.ema_decay is not None:+ # SCH: TODO: is this written properly like this for moving average? e.g. distributed training etc.+ lecam_ema_real = lecam_ema_real * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(real_logits.clone().detach())+ lecam_ema_fake = lecam_ema_fake * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(fake_logits.clone().detach())+++ loss_steps += 1+ running_disc_loss = disc_loss.item()/loss_steps + running_loss * ((loss_steps - 1) / loss_steps)+ running_loss = vae_loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)++ if coordinator.is_master():+ for idx, sample in enumerate(recon_video):+ pos = step * cfg.batch_size + idx+ save_path = os.path.join(save_dir, f"sample_{pos}")+ save_sample(sample, fps=cfg.fps, save_path=save_path)++ if cfg.calc_loss:+ print("test vae loss:", running_loss)+ print("test disc loss:", running_disc_loss)++++if __name__ == "__main__":+ main()diff --git a/scripts/train-vae-v2.py b/scripts/train-vae-v2.pyindex 3963ea5..4a0116a 100644--- a/scripts/train-vae-v2.py+++ b/scripts/train-vae-v2.py@@ -353,10 +353,8 @@ def main():# ====== Discriminator Loss ======if global_step > cfg.discriminator_start:-# if video_contains_first_frame:# Since we don't have enough T frames, pad anyways-real_video = pad_at_dim(video, (disc_time_padding, 0), value = 0., dim = 2)fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)