Open-Sora: Democratizing Efficient Video Production for All
Python
clean up code
Shen-Chenhui4d6967166301
diff --git a/configs/vae_magvit_v2/inference/1x256x256_pixabay.py b/configs/vae/inference/1x256x256.pysimilarity index 100%rename from configs/vae_magvit_v2/inference/1x256x256_pixabay.pyrename to configs/vae/inference/1x256x256.pydiff --git a/configs/vae_magvit_v2/train/17x256x256_pixabay.py b/configs/vae/train/17x256x256.pysimilarity index 100%rename from configs/vae_magvit_v2/train/17x256x256_pixabay.pyrename to configs/vae/train/17x256x256.pydiff --git a/configs/vae_magvit_v2/train/1x256x256_pixabay.py b/configs/vae/train/1x256x256.pysimilarity index 100%rename from configs/vae_magvit_v2/train/1x256x256_pixabay.pyrename to configs/vae/train/1x256x256.pydiff --git a/configs/vae_3d/inference/16x256x256.py b/configs/vae_3d/inference/16x256x256.pydeleted file mode 100644index 5cda9bd..0000000--- a/configs/vae_3d/inference/16x256x256.py+++ /dev/null@@ -1,56 +0,0 @@-num_frames = 16-frame_interval = 3-fps = 24 // 3-image_size = (256, 256)-is_vae = True--# Define dataset-root = None-data_path = "CSV_PATH"-use_image_transform = False-num_workers = 4--# Define acceleration-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1--# Define Loss-kl_weight = 0.000001-perceptual_weight = 0.1 # TODO: need to change this to 0.1 !!! according to MAGVIT paper--# Define model--model = dict(- type="VAE_3D_B",- from_pretrained="PRETRAINED_MODEL", # TODO: double check how this loads the pretrained model- in_out_channels = 3,- latent_embed_dim = 256,- filters = 64,- num_res_blocks = 2,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (True, True, False),- num_groups = 32, # for nn.GroupNorm- conv_downsample = False,- upsample = "nearest+conv", # options: "deconv", "nearest+conv"- kl_embed_dim = 64,- custom_conv_padding = None,- activation_fn = 'swish',-)---# Others-seed = 42-save_dir = "outputs/samples"-wandb = False---epochs = 10-log_every = 1-ckpt_every = 1000-load = None--batch_size = 8-grad_clip = 1.0-lr = 1e-4\ No newline at end of filediff --git a/configs/vae_3d/train/16x256x256.py b/configs/vae_3d/train/16x256x256.pydeleted file mode 100644index ab6e1ce..0000000--- a/configs/vae_3d/train/16x256x256.py+++ /dev/null@@ -1,58 +0,0 @@-num_frames = 16-frame_interval = 3-image_size = (256, 256)--# Define dataset-root = None-data_path = "CSV_PATH"-use_image_transform = False-num_workers = 4--# Define acceleration-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1--# Define Loss-kl_weight = 0.000001-perceptual_weight = 0.1 # TODO: need to change this to 0.1 !!! according to MAGVIT paper--# Define model--model = dict(- type="VAE_3D_B",- in_out_channels = 3,- latent_embed_dim = 4,- filters = 64,- num_res_blocks = 2,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (True, True, False),- num_groups = 32, # for nn.GroupNorm- conv_downsample = False,- upsample = "nearest+conv", # options: "deconv", "nearest+conv"- kl_embed_dim = 4,- custom_conv_padding = None,- activation_fn = 'swish',-)--# Others-seed = 42-outputs = "outputs"-wandb = False--# Training-''' NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-'''--epochs = 2000-log_every = 1-ckpt_every = 1000-load = None--batch_size = 4-lr = 1e-4-grad_clip = 1.0diff --git a/configs/vae_3d/vqgan2d_ucf101_config.py b/configs/vae_3d/vqgan2d_ucf101_config.pydeleted file mode 100644index ee61cab..0000000--- a/configs/vae_3d/vqgan2d_ucf101_config.py+++ /dev/null@@ -1,178 +0,0 @@-# Copyright 2023 The videogvt Authors.-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--# pylint: disable=line-too-long-r"""Configs for the VQGAN-2D on the UCF101.--"""--import ml_collections--UCF101_TRAIN_SIZE = 9_537-UCF101_TEST_SIZE = 3_783-NUM_CLASSES = 101-VARIANT = 'VQGAN/2D'---def get_config(config_str='B'):- """Returns the base experiment configuration."""- version, *options = config_str.split('-')-- config = ml_collections.ConfigDict()- config.experiment_name = f'UCF101_{VARIANT}'-- # Overall- config.rng_seed = 0- config.image_size = 128- config.batch_size = 256- config.eval_batch_size = config.get_ref('batch_size') // 4- config.num_training_epochs = 1000- config.lax_precision = 'default'-- # Dataset.- config.dataset_name = 'ucf101'- config.data_dtype_str = 'float32'- config.dataset_configs = ml_collections.ConfigDict()- config.dataset_configs.num_classes = NUM_CLASSES- config.dataset_configs.num_frames = 16- config.dataset_configs.stride = 1- # TODO(Lijun-Yu, roadjiang): test this augmentation and other augmentations.- # min_resize = int(min_resize / 224 * 256)- config.dataset_configs.min_resize = config.get_oneway_ref('image_size')- config.dataset_configs.crop_size = config.get_ref('image_size')- config.dataset_configs.one_hot_label = False- config.dataset_configs.zero_centering = False # Range is 0 to 1- config.dataset_configs.num_test_clips = 1- config.dataset_configs.prefetch_to_device = 2- config.num_train_sampled_frames = 4-- # Model: vqvae- model_class, model_type = VARIANT.split('/')- config.dtype = config.get_ref('data_dtype_str')- config.model_class = model_class-- config.pretrained_image_model = True- config.perceptual_loss_weight = 0.1- config.perceptual_loss_on_logit = True- config.polyak_decay = 0.999 # ema decay factor for generator-- config.vqgan = ml_collections.ConfigDict()- config.vqgan.model_type = model_type- config.vqgan.loss_type = 'non-saturating'- config.vqgan.g_adversarial_loss_weight = 0.1- config.vqgan.gradient_penalty = 'r1' # r1, none- config.vqgan.grad_penalty_cost = 10.0- # finetune decoder- config.vqgan.finetune_decoder = False- config.vqgan.finetune_path = ''-- config.vqvae = ml_collections.ConfigDict()- config.vqvae.architecture = '2dcnn'- config.vqvae.codebook_size = 1024- config.vqvae.entropy_loss_ratio = 0.1- config.vqvae.entropy_temperature = 0.01- config.vqvae.entropy_loss_type = 'softmax'- config.vqvae.commitment_cost = 0.25-- config.vqvae.filters = {'B': 64, 'L': 128}[version]- config.vqvae.num_enc_res_blocks = 2- config.vqvae.num_dec_res_blocks = 2- config.vqvae.channel_multipliers = (1, 2, 2, 4)- config.vqvae.embedding_dim = 256- config.vqvae.conv_downsample = False- config.vqvae.deconv_upsample = False- config.vqvae.activation_fn = 'swish'- config.vqvae.norm_type = 'GN'-- config.discriminator = ml_collections.ConfigDict()- config.discriminator.filters = config.vqvae.get_oneway_ref('filters')- config.discriminator.channel_multipliers = (2, 4, 4, 4, 4)-- # Learning rate.- config.base_lr = 0.0001- config.optimizer = ml_collections.ConfigDict()- config.optimizer.lr = config.get_ref('base_lr')- config.optimizer.beta1 = 0.0- config.optimizer.beta2 = 0.99-- config.optimizer.g_lr = config.get_ref('base_lr')- config.optimizer.d_lr = config.get_ref('base_lr')-- steps_per_epoch = UCF101_TRAIN_SIZE // config.get_ref('batch_size')- total_steps = config.get_ref('num_training_epochs') * steps_per_epoch- config.lr_configs = ml_collections.ConfigDict()- config.lr_configs.learning_rate_schedule = 'compound'- config.lr_configs.factors = 'constant * cosine_decay * linear_warmup'- config.lr_configs.warmup_steps = 1 * steps_per_epoch- config.lr_configs.steps_per_cycle = total_steps- config.lr_configs.base_learning_rate = config.get_ref('base_lr') # placeholder-- # Pretrained model on ImageNet.- config.init_from = ml_collections.ConfigDict()- config.init_from.checkpoint_path = {- 'B': 'gs://magvit/models/imagenet_2d_base',- 'L': 'gs://magvit/models/imagenet_2d_large',- }[version]-- # Evaluation.- config.eval = ml_collections.ConfigDict()- config.eval.enable_inception_score = True- config.eval.enable_frechet_distance = True- config.eval.data_splits = 'train,validation'- config.eval.num_examples = 10000- config.eval.final_num_repeats = 1- config.eval_from = ml_collections.ConfigDict()- config.eval_from.checkpoint_path = None- config.eval_from.step = None-- # Logging.- config.logging = ml_collections.ConfigDict()- config.logging.enable_checkpoint = True- config.logging.checkpoint_steps = 1000- config.logging.checkpoint_kept = 5- config.logging.log_metric_steps = 200- config.logging.log_sample_size = 2-- if 'runlocal' in options:- config.batch_size = 16- config.num_training_epochs = 10- config.eval_batch_split = 1-- config.pretrained_image_model = False- config.perceptual_loss_weight = 0.0-- config.vqvae.filters = 32- config.vqvae.num_enc_res_blocks = 1- config.vqvae.num_dec_res_blocks = 1- config.vqvae.channel_multipliers = (1,) * len(- config.vqvae.channel_multipliers)- config.discriminator.channel_multipliers = (1,) * len(- config.discriminator.channel_multipliers)-- config.logging.enable_checkpoint = False- config.logging.checkpoint_steps = 100- config.logging.log_metric_steps = 20-- del config.init_from-- # Standalone evaluation.- if 'eval' in options:- config.eval_only = True- config.eval_from.checkpoint_path = ''- config.eval_from.step = -1- config.eval_from.legacy_checkpoint = True-- return config--diff --git a/configs/vae_3d/vqgan3d_ucf101_config.py b/configs/vae_3d/vqgan3d_ucf101_config.pydeleted file mode 100644index a3875dc..0000000--- a/configs/vae_3d/vqgan3d_ucf101_config.py+++ /dev/null@@ -1,102 +0,0 @@-# Copyright 2023 The videogvt Authors.-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--# pylint: disable=line-too-long-r"""-Configs for the VQGAN-3D on the UCF101.--usage: `config = vqgan3d_ucf101_config.get_config()`--"""--import ml_collections--from opensora.configs.vae_3d import vqgan2d_ucf101_config--VARIANT = 'VQGAN/3D'---def get_config(config_str='B'):- """Returns the base experiment configuration."""- version, *options = config_str.split('-')-- config = vqgan2d_ucf101_config.get_config(config_str)- config.experiment_name = f'UCF101_{VARIANT}'- model_class, model_type = VARIANT.split('/')-- # Overall- config.batch_size = {'B': 128, 'L': 256, 'H': 256}[version]- config.eval_batch_size = config.get_ref('batch_size') // 4- config.num_training_epochs = {'B': 500, 'L': 2000, 'H': 2000}[version]-- # Dataset.- del config.num_train_sampled_frames-- # Model: vqvae- config.model_class = model_class-- config.pretrained_image_model = True # TODO(Lijun-Yu): 3d perceptual loss-- config.vqgan.model_type = model_type-- config.vqvae.in_channels = 3 # NOTE: SCH: added- config.vqvae.num_groups = 32 # NOTE: SCH: added, for nn.GroupNorm- config.vqvae.architecture = '3dcnn'- config.vqvae.codebook_size = {'B': 1024, 'L': 1024, 'H': 4096}[version]- config.vqvae.filters = {'B': 64, 'L': 128, 'H': 128}[version]- config.vqvae.num_enc_res_blocks = {'B': 2, 'L': 2, 'H': 4}[version]- config.vqvae.num_dec_res_blocks = {'B': 2, 'L': 2, 'H': 4}[version]- config.vqvae.channel_multipliers = (1, 2, 2, 4)- config.vqvae.temporal_downsample = (True, True, False)- config.vqvae.embedding_dim = {'B': 256, 'L': 256, 'H': 512}[version] # z embedding dimension- config.vqvae.conv_downsample = False- config.vqvae.deconv_upsample = False-- # Save memory- config.vqvae.num_enc_remat_blocks = {'B': 0, 'L': 1, 'H': 3}[version]- config.vqvae.num_dec_remat_blocks = config.vqvae.get_ref(- 'num_enc_remat_blocks')- config.discriminator.num_remat_blocks = config.vqvae.get_ref(- 'num_enc_remat_blocks')-- # Pretrained models on ImageNet.- config.init_from = ml_collections.ConfigDict()- config.init_from.inflation = '2d->3d'-- # Standalone evaluation.- if 'eval' in options:- config.eval_only = True- config.eval_from.checkpoint_path = {- 'B': 'gs://magvit/models/ucf_3d_base',- 'L': 'gs://magvit/models/ucf_3d_large',- }[version]- config.eval_from.step = -1- config.eval_from.legacy_checkpoint = True-- if 'runlocal' in options:- config.batch_size = 16- config.num_training_epochs = 10- # gets a small model for debugging- config.vqvae.filters = 32- config.vqvae.embedding_dim = 16- config.vqvae.num_enc_res_blocks = 1- config.vqvae.num_dec_res_blocks = 1- config.discriminator.filters = 1- config.discriminator.channel_multipliers = (1,)- config.vqvae.channel_multipliers = (1,)- config.vqvae.codebook_size = 128-- return config--diff --git a/configs/vae_magvit_v2/inference/16x128x128.py b/configs/vae_magvit_v2/inference/16x128x128.pydeleted file mode 100644index 6d20a91..0000000--- a/configs/vae_magvit_v2/inference/16x128x128.py+++ /dev/null@@ -1,87 +0,0 @@-num_frames = 16-frame_interval = 3-image_size = (128, 128)-fps = 24 // 3-is_vae = True--# Define dataset-root = None-data_path = "CSV_PATH"-max_test_samples = -1-use_image_transform = False-num_workers = 4-video_contains_first_frame = False--# Define acceleration-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1---# Define model--model = dict(- type="VAE_MAGVIT_V2",- in_out_channels = 3,- latent_embed_dim = 256,- filters = 128,- num_res_blocks = 4,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (False, True, True),- num_groups = 32, # for nn.GroupNorm- kl_embed_dim = 64,- activation_fn = 'swish',- separate_first_frame_encoding = False,- custom_conv_padding = None,- encoder_double_z = False,-)---discriminator = dict(- type="DISCRIMINATOR_3D",- image_size = image_size,- num_frames = num_frames,- in_channels = 3,- filters = 128,- channel_multipliers = (2,4,4,4,4) # (2,4,4,4) for 64x64 resolution-)---# loss weights-logvar_init=0.0-kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-# discriminator_loss_weight = 0.5 # for generator adversarial loss-generator_factor = 0.1 # for generator adversarial loss-lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight-discriminator_loss_type="non-saturating"-generator_loss_type="non-saturating"-discriminator_start = 50000 # 50000 NOTE: change to correct val, debug use -1 for now-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use-ema_decay = 0.999 # ema decay factor for generator---# Others-seed = 42-save_dir = "outputs/samples_v2"-wandb = False--# Training-''' NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-'''--epochs = 10-log_every = 1-ckpt_every = 1000-load = None--batch_size = 4-lr = 1e-4-grad_clip = 1.0--calc_loss = False\ No newline at end of filediff --git a/configs/vae_magvit_v2/inference/16x128x128_z=4.py b/configs/vae_magvit_v2/inference/16x128x128_z=4.pydeleted file mode 100644index d1e7c01..0000000--- a/configs/vae_magvit_v2/inference/16x128x128_z=4.py+++ /dev/null@@ -1,87 +0,0 @@-num_frames = 16-frame_interval = 3-image_size = (128, 128)-fps = 24 // 3-is_vae = True--# Define dataset-root = None-data_path = "CSV_PATH"-max_test_samples = -1-use_image_transform = False-num_workers = 4-video_contains_first_frame = False--# Define acceleration-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1---# Define model--model = dict(- type="VAE_MAGVIT_V2",- in_out_channels = 3,- latent_embed_dim = 4,- filters = 128,- num_res_blocks = 4,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (False, True, True),- num_groups = 32, # for nn.GroupNorm- kl_embed_dim = 4,- activation_fn = 'swish',- separate_first_frame_encoding = False,- disable_space = False,- custom_conv_padding = None-)---discriminator = dict(- type="DISCRIMINATOR_3D",- image_size = image_size,- num_frames = num_frames,- in_channels = 3,- filters = 128,- channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution-)---# loss weights-logvar_init=0.0-kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-# discriminator_loss_weight = 0.5 # for generator adversarial loss-generator_factor = 0.1 # for generator adversarial loss-lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight-discriminator_loss_type="non-saturating"-generator_loss_type="non-saturating"-discriminator_start = 50000 # 50000 NOTE: change to correct val, debug use -1 for now-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use-ema_decay = 0.999 # ema decay factor for generator---# Others-seed = 42-save_dir = "outputs/samples_z4"-wandb = False--# Training-''' NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-'''--epochs = 10-log_every = 1-ckpt_every = 1000-load = None--batch_size = 4-lr = 1e-4-grad_clip = 1.0--calc_loss = False\ No newline at end of filediff --git a/configs/vae_magvit_v2/inference/17x128x128_pixabay.py b/configs/vae_magvit_v2/inference/17x128x128_pixabay.pydeleted file mode 100644index b264c2e..0000000--- a/configs/vae_magvit_v2/inference/17x128x128_pixabay.py+++ /dev/null@@ -1,97 +0,0 @@-num_frames = 17--dataset = dict(- type="VideoTextDataset",- data_path=None,- num_frames=num_frames,- frame_interval=3,- image_size=(128, 128),- get_text=False,-)--fps = 24 // 3-is_vae = True--# Define dataset-max_test_samples = -1---# Define acceleration-num_workers = 4-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1--use_pipeline = True-video_contains_first_frame = True---# Define model--vae_2d = dict(- type="VideoAutoencoderKL",- from_pretrained="stabilityai/sd-vae-ft-ema",-)--model = dict(- type="VAE_MAGVIT_V2",- in_out_channels=4,- latent_embed_dim=64,- filters=128,- num_res_blocks=4,- channel_multipliers=(1, 2, 2, 4),- temporal_downsample=(False, True, True),- num_groups=32, # for nn.GroupNorm- kl_embed_dim=4,- activation_fn="swish",- separate_first_frame_encoding=False,- disable_space=True,- custom_conv_padding=None,- encoder_double_z=True,-)--discriminator = dict(- type="DISCRIMINATOR_3D",- image_size=(128, 128),- num_frames=num_frames,- in_channels=3,- filters=128,- channel_multipliers=(2, 4, 4, 4, 4),- # channel_multipliers = (2,4,4), #(2,4,4,4,4) # (2,4,4,4) for 64x64 resolution-)---# loss weights-logvar_init = 0.0-kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-# discriminator_loss_weight = 0.5 # for generator adversarial loss-generator_factor = 0.1 # for generator adversarial loss-lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight-discriminator_loss_type = "non-saturating"-generator_loss_type = "non-saturating"-discriminator_start = 2500 # 50000 NOTE: change to correct val, debug use -1 for now-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use-ema_decay = 0.999 # ema decay factor for generator---# Others-seed = 42-save_dir = "outputs/samples_pixabay_17"-wandb = False--# Training-""" NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-"""---batch_size = 1-lr = 1e-4-grad_clip = 1.0--calc_loss = Truediff --git a/configs/vae_magvit_v2/inference/1x128x128_pixabay.py b/configs/vae_magvit_v2/inference/1x128x128_pixabay.pydeleted file mode 100644index e7511bc..0000000--- a/configs/vae_magvit_v2/inference/1x128x128_pixabay.py+++ /dev/null@@ -1,97 +0,0 @@-num_frames = 1--dataset = dict(- type="VideoTextDataset",- data_path=None,- num_frames=num_frames,- frame_interval=3,- image_size=(128, 128),- get_text=False,-)--fps = 24 // 3-is_vae = True--# Define dataset-max_test_samples = -1---# Define acceleration-num_workers = 4-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1--use_pipeline = True-video_contains_first_frame = True---# Define model--vae_2d = dict(- type="VideoAutoencoderKL",- from_pretrained="stabilityai/sd-vae-ft-ema",-)--model = dict(- type="VAE_MAGVIT_V2",- in_out_channels = 4,- latent_embed_dim = 64,- filters = 128,- num_res_blocks = 4,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (False, True, True),- num_groups = 32, # for nn.GroupNorm- kl_embed_dim = 4,- activation_fn = 'swish',- separate_first_frame_encoding = False,- disable_space = True,- custom_conv_padding = None,- encoder_double_z = True,-)--discriminator = dict(- type="DISCRIMINATOR_3D",- image_size = (128, 128),- num_frames = num_frames,- in_channels = 3,- filters = 128,- channel_multipliers = (2,4,4,4,4),- # channel_multipliers = (2,4,4), #(2,4,4,4,4) # (2,4,4,4) for 64x64 resolution-)---# loss weights-logvar_init=0.0-kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-# discriminator_loss_weight = 0.5 # for generator adversarial loss-generator_factor = 0.1 # for generator adversarial loss-lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight-discriminator_loss_type="non-saturating"-generator_loss_type="non-saturating"-discriminator_start = 2500 # 50000 NOTE: change to correct val, debug use -1 for now-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use-ema_decay = 0.999 # ema decay factor for generator---# Others-seed = 42-save_dir = "outputs/samples_pixabay_17"-wandb = False--# Training-''' NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-'''---batch_size = 1-lr = 1e-4-grad_clip = 1.0--calc_loss = True\ No newline at end of filediff --git a/configs/vae_magvit_v2/inference/pipeline_16x128x128.py b/configs/vae_magvit_v2/inference/pipeline_16x128x128.pydeleted file mode 100644index 540d63d..0000000--- a/configs/vae_magvit_v2/inference/pipeline_16x128x128.py+++ /dev/null@@ -1,96 +0,0 @@-num_frames = 16-frame_interval = 3-image_size = (128, 128)-fps = 24 // 3-is_vae = True-use_pipeline = True--# Define dataset-root = None-data_path = "CSV_PATH"-max_test_samples = -1-use_image_transform = False-num_workers = 4-video_contains_first_frame = False----# Define acceleration-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1---# Define model--vae_2d = dict(- type="VideoAutoencoderKL",- from_pretrained="stabilityai/sd-vae-ft-ema",-)--model = dict(- type="VAE_MAGVIT_V2",- in_out_channels = 4,- latent_embed_dim = 256,- filters = 128,- num_res_blocks = 4,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (False, True, True),- num_groups = 32, # for nn.GroupNorm- kl_embed_dim = 64,- activation_fn = 'swish',- separate_first_frame_encoding = False,- disable_space = True,- custom_conv_padding = None,- encoder_double_z = False,-)---discriminator = dict(- type="DISCRIMINATOR_3D",- image_size = 16,- num_frames = num_frames,- in_channels = 4,- filters = 128,- channel_multipliers = (2,4,4), #(2,4,4,4,4) # (2,4,4,4) for 64x64 resolution-)---# loss weights-logvar_init=0.0-kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-# discriminator_loss_weight = 0.5 # for generator adversarial loss-generator_factor = 0.1 # for generator adversarial loss-lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight-discriminator_loss_type="non-saturating"-generator_loss_type="non-saturating"-discriminator_start = 2500 # 50000 NOTE: change to correct val, debug use -1 for now-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use-ema_decay = 0.999 # ema decay factor for generator---# Others-seed = 42-save_dir = "outputs/pipeline_samples_v2"-wandb = False--# Training-''' NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-'''--epochs = 10-log_every = 1-ckpt_every = 500-load = None--batch_size = 4-lr = 1e-4-grad_clip = 1.0--calc_loss = True\ No newline at end of filediff --git a/configs/vae_magvit_v2/train/16x128x128.py b/configs/vae_magvit_v2/train/16x128x128.pydeleted file mode 100644index 9e71b8d..0000000--- a/configs/vae_magvit_v2/train/16x128x128.py+++ /dev/null@@ -1,84 +0,0 @@-num_frames = 16-frame_interval = 3-image_size = (128, 128)--# Define dataset-root = None-data_path = "CSV_PATH"-use_image_transform = False-num_workers = 4-video_contains_first_frame = False--# Define acceleration-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1---# Define model--model = dict(- type="VAE_MAGVIT_V2",- in_out_channels = 3,- latent_embed_dim = 256,- filters = 128,- num_res_blocks = 4,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (False, True, True),- num_groups = 32, # for nn.GroupNorm- kl_embed_dim = 64,- activation_fn = 'swish',- separate_first_frame_encoding = False,- disable_space = False,- custom_conv_padding = None-)---discriminator = dict(- type="DISCRIMINATOR_3D",- image_size = image_size,- num_frames = num_frames,- in_channels = 3,- filters = 128,- channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution-)---# loss weights-logvar_init=0.0-kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-# discriminator_loss_weight = 0.5 # for generator adversarial loss-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1-lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001-# discriminator_loss_type="non-saturating"-# generator_loss_type="non-saturating"-discriminator_loss_type="hinge"-generator_loss_type="hinge"-discriminator_start = 30000 # 50000 NOTE: change to correct val, debug use -1 for now-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use-ema_decay = 0.999 # ema decay factor for generator---# Others-seed = 42-outputs = "outputs"-wandb = False--# Training-''' NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-'''--epochs = 200-log_every = 1-ckpt_every = 200-load = None--batch_size = 4-lr = 1e-4-grad_clip = 1.0diff --git a/configs/vae_magvit_v2/train/16x128x128_pixabay_8_GPU_pipeline.py b/configs/vae_magvit_v2/train/16x128x128_pixabay_8_GPU_pipeline.pydeleted file mode 100644index 7b2c60c..0000000--- a/configs/vae_magvit_v2/train/16x128x128_pixabay_8_GPU_pipeline.py+++ /dev/null@@ -1,93 +0,0 @@-dataset = dict(- type="VideoTextDataset",- data_path=None,- num_frames=16,- frame_interval=3,- image_size=(128, 128),- get_text=False,-)--# Define acceleration-num_workers = 4-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1--use_pipeline = True-video_contains_first_frame = False--# Define model-vae_2d = dict(- type="VideoAutoencoderKL",- from_pretrained="stabilityai/sd-vae-ft-ema",- # SDXL-)-model = dict(- type="VAE_MAGVIT_V2",- in_out_channels=4,- latent_embed_dim=4,- filters=128,- num_res_blocks=4,- channel_multipliers=(1, 2, 2, 4),- temporal_downsample=(False, True, True),- num_groups=32, # for nn.GroupNorm- kl_embed_dim=4,- activation_fn="swish",- separate_first_frame_encoding=False,- disable_space=True,- encoder_double_z=True,- custom_conv_padding=None,-)---discriminator = dict(- type="DISCRIMINATOR_3D",- image_size=(16, 16), # NOTE: here image size is different- num_frames=16,- in_channels=4,- filters=128,- use_pretrained=True, # NOTE: set to False only if we want to disable load- # channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution- channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z-)---# loss weights-logvar_init = 0.0-kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1-lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001-discriminator_loss_type = "non-saturating"-generator_loss_type = "non-saturating"-# discriminator_loss_type="hinge"-# generator_loss_type="hinge"-discriminator_start = 100 # 8k data / (8*32) = 31 steps per epoch, use around 3 epochs-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use-ema_decay = 0.999 # ema decay factor for generator---# Others-seed = 42-outputs = "outputs"-wandb = False--# Training-""" NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-"""--epochs = 200-log_every = 1--ckpt_every = 50-load = None--batch_size = 32--lr = 1e-4-grad_clip = 1.0diff --git a/configs/vae_magvit_v2/train/16x128x128_trial.py b/configs/vae_magvit_v2/train/16x128x128_trial.pydeleted file mode 100644index e9693c9..0000000--- a/configs/vae_magvit_v2/train/16x128x128_trial.py+++ /dev/null@@ -1,83 +0,0 @@-num_frames = 16-frame_interval = 3-image_size = (128, 128)--# Define dataset-root = None-data_path = "CSV_PATH"-use_image_transform = False-num_workers = 4-video_contains_first_frame = False--# Define acceleration-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1---# Define model--model = dict(- type="VAE_MAGVIT_V2",- in_out_channels = 3,- latent_embed_dim = 256,- filters = 128,- num_res_blocks = 4,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (False, True, True),- num_groups = 32, # for nn.GroupNorm- kl_embed_dim = 64,- activation_fn = 'swish',- separate_first_frame_encoding = False,- disable_space = False,- encoder_double_z = False,- custom_conv_padding = None-)---discriminator = dict(- type="DISCRIMINATOR_3D",- image_size = image_size,- num_frames = num_frames,- in_channels = 3,- filters = 128,- channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution-)---# loss weights-logvar_init=0.0-kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-# discriminator_loss_weight = 0.5 # for generator adversarial loss-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1-lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001-discriminator_loss_type="non-saturating"-generator_loss_type="non-saturating"-discriminator_start = 1000 # 50000 NOTE: change to correct val, debug use -1 for now-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use-ema_decay = 0.999 # ema decay factor for generator---# Others-seed = 42-outputs = "outputs"-wandb = False--# Training-''' NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-'''--epochs = 2000-log_every = 1-ckpt_every = 1000-load = None--batch_size = 4-lr = 1e-4-grad_clip = 1.0diff --git a/configs/vae_magvit_v2/train/16x128x128_z4_LC.py b/configs/vae_magvit_v2/train/16x128x128_z4_LC.pydeleted file mode 100644index 2cb7cb6..0000000--- a/configs/vae_magvit_v2/train/16x128x128_z4_LC.py+++ /dev/null@@ -1,82 +0,0 @@-num_frames = 16-frame_interval = 3-image_size = (128, 128)--# Define dataset-root = None-data_path = "CSV_PATH"-use_image_transform = False-num_workers = 4-video_contains_first_frame = False--# Define acceleration-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1---# Define model--model = dict(- type="VAE_MAGVIT_V2",- in_out_channels = 3,- latent_embed_dim = 4,- filters = 128,- num_res_blocks = 4,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (False, True, True),- num_groups = 32, # for nn.GroupNorm- kl_embed_dim = 4,- activation_fn = 'swish',- separate_first_frame_encoding = False,- disable_space = False,- custom_conv_padding = None-)---discriminator = dict(- type="DISCRIMINATOR_3D",- image_size = image_size,- num_frames = num_frames,- in_channels = 3,- filters = 128,- channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution-)---# loss weights-logvar_init=0.0-kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-# discriminator_loss_weight = 0.5 # for generator adversarial loss-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1-lecam_loss_weight = 0.001 # NOTE: MAVGIT v2 use 0.001-discriminator_loss_type="non-saturating"-generator_loss_type="non-saturating"-discriminator_start = -1 # 50000 NOTE: change to correct val, debug use -1 for now-gradient_penalty_loss_weight = 10 # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use-ema_decay = 0.999 # ema decay for LeCam---# Others-seed = 42-outputs = "outputs"-wandb = False--# Training-''' NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-'''--epochs = 2000-log_every = 1-ckpt_every = 1000-load = None--batch_size = 4-lr = 1e-4-grad_clip = 1.0diff --git a/configs/vae_magvit_v2/train/16x128x128_z=4.py b/configs/vae_magvit_v2/train/16x128x128_z=4.pydeleted file mode 100644index f252903..0000000--- a/configs/vae_magvit_v2/train/16x128x128_z=4.py+++ /dev/null@@ -1,82 +0,0 @@-num_frames = 16-frame_interval = 3-image_size = (128, 128)--# Define dataset-root = None-data_path = "CSV_PATH"-use_image_transform = False-num_workers = 4-video_contains_first_frame = False--# Define acceleration-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1---# Define model--model = dict(- type="VAE_MAGVIT_V2",- in_out_channels = 3,- latent_embed_dim = 4,- filters = 128,- num_res_blocks = 4,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (False, True, True),- num_groups = 32, # for nn.GroupNorm- kl_embed_dim = 4,- activation_fn = 'swish',- separate_first_frame_encoding = False,- disable_space = False,- custom_conv_padding = None-)---discriminator = dict(- type="DISCRIMINATOR_3D",- image_size = image_size,- num_frames = num_frames,- in_channels = 3,- filters = 128,- channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution-)---# loss weights-logvar_init=0.0-kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-# discriminator_loss_weight = 0.5 # for generator adversarial loss-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1-lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001-discriminator_loss_type="non-saturating"-generator_loss_type="non-saturating"-discriminator_start = 1000 # 50000 NOTE: change to correct val, debug use -1 for now-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use-ema_decay = 0.999 # ema decay factor for generator---# Others-seed = 42-outputs = "outputs"-wandb = False--# Training-''' NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-'''--epochs = 2000-log_every = 1-ckpt_every = 1000-load = None--batch_size = 4-lr = 1e-4-grad_clip = 1.0diff --git a/configs/vae_magvit_v2/train/17x128x128.py b/configs/vae_magvit_v2/train/17x128x128.pydeleted file mode 100644index 0397924..0000000--- a/configs/vae_magvit_v2/train/17x128x128.py+++ /dev/null@@ -1,95 +0,0 @@-num_frames = 17-image_size = (128, 128)-dataset = dict(- type="VideoTextDataset",- data_path=None,- num_frames=num_frames,- frame_interval=3,- image_size=image_size,- get_text=False,-)--# Define acceleration-num_workers = 4-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1--use_pipeline = True-video_contains_first_frame = True--# Define model-vae_2d = dict(- type="VideoAutoencoderKL",- from_pretrained="stabilityai/sd-vae-ft-ema",- # SDXL-)-model = dict(- type="VAE_MAGVIT_V2",- in_out_channels=4,- latent_embed_dim=64,- filters=128,- num_res_blocks=4,- channel_multipliers=(1, 2, 2, 4),- temporal_downsample=(False, True, True),- num_groups=32, # for nn.GroupNorm- kl_embed_dim=4,- activation_fn="swish",- separate_first_frame_encoding=False,- disable_space=True,- encoder_double_z=True,- custom_conv_padding=None,-)---discriminator = dict(- type="DISCRIMINATOR_3D",- image_size=image_size, # NOTE: here image size is different- num_frames=num_frames,- in_channels=3,- filters=128,- use_pretrained=True, # NOTE: set to False only if we want to disable load- channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution- # channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z-)---# loss weights-logvar_init = 0.0-kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1-lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001-discriminator_loss_type = "non-saturating"-generator_loss_type = "non-saturating"-# discriminator_loss_type="hinge"-# generator_loss_type="hinge"-discriminator_start = 2000 # 5000 # 8k data / (8*1) = 1000 steps per epoch-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use-ema_decay = 0.999 # ema decay factor for generator---# Others-seed = 42-outputs = "outputs"-wandb = False--# Training-""" NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-"""--epochs = 500-log_every = 1--ckpt_every = 500-load = None--batch_size = 1--lr = 1e-4-grad_clip = 1.0diff --git a/configs/vae_magvit_v2/train/17x128x128_LC_GP.py b/configs/vae_magvit_v2/train/17x128x128_LC_GP.pydeleted file mode 100644index de31bdb..0000000--- a/configs/vae_magvit_v2/train/17x128x128_LC_GP.py+++ /dev/null@@ -1,95 +0,0 @@-num_frames = 17-image_size = (128, 128)-dataset = dict(- type="VideoTextDataset",- data_path=None,- num_frames=num_frames,- frame_interval=3,- image_size=image_size,- get_text=False,-)--# Define acceleration-num_workers = 4-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1--use_pipeline = True-video_contains_first_frame = True--# Define model-vae_2d = dict(- type="VideoAutoencoderKL",- from_pretrained="stabilityai/sd-vae-ft-ema",- # SDXL-)-model = dict(- type="VAE_MAGVIT_V2",- in_out_channels=4,- latent_embed_dim=64,- filters=128,- num_res_blocks=4,- channel_multipliers=(1, 2, 2, 4),- temporal_downsample=(False, True, True),- num_groups=32, # for nn.GroupNorm- kl_embed_dim=4,- activation_fn="swish",- separate_first_frame_encoding=False,- disable_space=True,- encoder_double_z=True,- custom_conv_padding=None,-)---discriminator = dict(- type="DISCRIMINATOR_3D",- image_size=image_size, # NOTE: here image size is different- num_frames=num_frames,- in_channels=3,- filters=128,- use_pretrained=True, # NOTE: set to False only if we want to disable load- channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution- # channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z-)---# loss weights-logvar_init = 0.0-kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1-lecam_loss_weight = 0.001 # NOTE: MAVGIT v2 use 0.001-discriminator_loss_type = "non-saturating"-generator_loss_type = "non-saturating"-# discriminator_loss_type="hinge"-# generator_loss_type="hinge"-discriminator_start = 2000 # 5000 # 8k data / (8*1) = 1000 steps per epoch-gradient_penalty_loss_weight = 10 # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use-ema_decay = 0.999 # ema decay factor for generator---# Others-seed = 42-outputs = "outputs"-wandb = False--# Training-""" NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-"""--epochs = 500-log_every = 1--ckpt_every = 500-load = None--batch_size = 1--lr = 1e-4-grad_clip = 1.0diff --git a/configs/vae_magvit_v2/train/17x128x128_pixabay_8_GPU_pipeline.py b/configs/vae_magvit_v2/train/17x128x128_pixabay_8_GPU_pipeline.pydeleted file mode 100644index fe69e94..0000000--- a/configs/vae_magvit_v2/train/17x128x128_pixabay_8_GPU_pipeline.py+++ /dev/null@@ -1,94 +0,0 @@-num_frames = 17-dataset = dict(- type="VideoTextDataset",- data_path=None,- num_frames=num_frames,- frame_interval=3,- image_size=(128, 128),- get_text=False,-)--# Define acceleration-num_workers = 4-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1--use_pipeline = True-video_contains_first_frame = True--# Define model-vae_2d = dict(- type="VideoAutoencoderKL",- from_pretrained="stabilityai/sd-vae-ft-ema",- # SDXL-)-model = dict(- type="VAE_MAGVIT_V2",- in_out_channels=4,- latent_embed_dim=64,- filters=128,- num_res_blocks=4,- channel_multipliers=(1, 2, 2, 4),- temporal_downsample=(False, True, True),- num_groups=32, # for nn.GroupNorm- kl_embed_dim=4,- activation_fn="swish",- separate_first_frame_encoding=False,- disable_space=True,- encoder_double_z=True,- custom_conv_padding=None,-)---discriminator = dict(- type="DISCRIMINATOR_3D",- image_size=(16, 16), # NOTE: here image size is different- num_frames=num_frames,- in_channels=4,- filters=128,- use_pretrained=True, # NOTE: set to False only if we want to disable load- # channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution- channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z-)---# loss weights-logvar_init = 0.0-kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1-lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001-discriminator_loss_type = "non-saturating"-generator_loss_type = "non-saturating"-# discriminator_loss_type="hinge"-# generator_loss_type="hinge"-discriminator_start = 5000 # 8k data / (8*1) = 1000 steps per epoch-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use-ema_decay = 0.999 # ema decay factor for generator---# Others-seed = 42-outputs = "outputs"-wandb = False--# Training-""" NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-"""--epochs = 20-log_every = 1--ckpt_every = 500-load = None--batch_size = 1--lr = 1e-4-grad_clip = 1.0diff --git a/configs/vae_magvit_v2/train/1x256x256_small_lr.py b/configs/vae_magvit_v2/train/1x256x256_small_lr.pydeleted file mode 100644index 7ceca93..0000000--- a/configs/vae_magvit_v2/train/1x256x256_small_lr.py+++ /dev/null@@ -1,105 +0,0 @@-num_frames = 1-image_size = (256, 256)-dataset = dict(- type="VideoTextDataset",- data_path=None,- num_frames=num_frames,- frame_interval=1,- image_size=image_size,- get_text=False,-)--# Define acceleration-num_workers = 4-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1--use_pipeline = True-video_contains_first_frame = True--# Define model-# vae_2d = dict(-# type="VideoAutoencoderKL",-# from_pretrained="stabilityai/sd-vae-ft-ema",-# # SDXL-# )--# latest-vae_2d = dict(- type="VideoAutoencoderKL",- from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",- subfolder="vae",- micro_batch_size=4,- local_files_only=True,-)--model = dict(- type="VAE_MAGVIT_V2",- in_out_channels=4,- latent_embed_dim=64,- filters=128,- num_res_blocks=4,- channel_multipliers=(1, 2, 2, 4),- temporal_downsample=(False, True, True),- num_groups=32, # for nn.GroupNorm- kl_embed_dim=4,- activation_fn="swish",- separate_first_frame_encoding=False,- disable_space=True,- custom_conv_padding=None,- encoder_double_z=True,-)---discriminator = dict(- type="DISCRIMINATOR_3D",- image_size=image_size, # NOTE: here image size is different- num_frames=num_frames,- in_channels=3,- filters=128,- use_pretrained=True, # NOTE: set to False only if we want to disable load- channel_multipliers=(2, 4, 4, 4, 4), # (2,4,4,4) for 64x64 resolution- # channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z-)---# loss weights-logvar_init = 0.0-kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1-lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001-discriminator_loss_type = "non-saturating"-generator_loss_type = "non-saturating"-# discriminator_loss_type="hinge"-# generator_loss_type="hinge"-discriminator_start = 2000 # 5000 # 8k data / (8*1) = 1000 steps per epoch-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use-ema_decay = 0.999 # ema decay factor for generator---# Others-seed = 42-outputs = "outputs"-wandb = False--# Training-""" NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-"""--epochs = 100-log_every = 1--ckpt_every = 500-load = None--batch_size = 1--lr = 1e-5-grad_clip = 1.0diff --git a/configs/vae_magvit_v2/train/pipeline_16x128x128.py b/configs/vae_magvit_v2/train/pipeline_16x128x128.pydeleted file mode 100644index 3aa6f28..0000000--- a/configs/vae_magvit_v2/train/pipeline_16x128x128.py+++ /dev/null@@ -1,93 +0,0 @@-num_frames = 16-frame_interval = 3-image_size = (128, 128)-use_pipeline = True--# Define dataset-root = None-data_path = "CSV_PATH"-use_image_transform = False-num_workers = 4-video_contains_first_frame = False--# Define acceleration-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1---# Define model-vae_2d = dict(- type="VideoAutoencoderKL",- from_pretrained="stabilityai/sd-vae-ft-ema",- # SDXL-)--model = dict(- type="VAE_MAGVIT_V2",- in_out_channels = 4,- latent_embed_dim = 256,- filters = 128,- num_res_blocks = 4,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (False, True, True),- num_groups = 32, # for nn.GroupNorm- kl_embed_dim = 64,- activation_fn = 'swish',- separate_first_frame_encoding = False,- disable_space = True,- encoder_double_z = False,- custom_conv_padding = None-)---discriminator = dict(- type="DISCRIMINATOR_3D",- image_size = (16, 16), # NOTE: here image size is different- num_frames = num_frames,- in_channels = 4,- filters = 128,- use_pretrained=True, # NOTE: set to False only if we want to fresh train using a different discriminator!- # channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution- channel_multipliers= (2,4,4) # since on intermediate layer, 16 x 16 x 16 dimension z-)---# loss weights-logvar_init=0.0-kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-# discriminator_loss_weight = 0.5 # for generator adversarial loss-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1-lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001-# discriminator_loss_type="non-saturating"-# generator_loss_type="non-saturating"-discriminator_loss_type="hinge"-generator_loss_type="hinge"-discriminator_start = 1000 # 50000 NOTE: change to correct val, debug use -1 for now-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use-ema_decay = 0.999 # ema decay factor for generator---# Others-seed = 42-outputs = "outputs"-wandb = False--# Training-''' NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-'''--epochs = 2000-log_every = 1-ckpt_every = 1000-load = None--batch_size = 4-lr = 1e-4-grad_clip = 1.0diff --git a/configs/vae_magvit_v2/train/pipeline_16x128x128_LC.py b/configs/vae_magvit_v2/train/pipeline_16x128x128_LC.pydeleted file mode 100644index fc3c832..0000000--- a/configs/vae_magvit_v2/train/pipeline_16x128x128_LC.py+++ /dev/null@@ -1,93 +0,0 @@-num_frames = 16-frame_interval = 3-image_size = (128, 128)-use_pipeline = True--# Define dataset-root = None-data_path = "CSV_PATH"-use_image_transform = False-num_workers = 4-video_contains_first_frame = False--# Define acceleration-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1---# Define model-vae_2d = dict(- type="VideoAutoencoderKL",- from_pretrained="stabilityai/sd-vae-ft-ema",- # SDXL-)--model = dict(- type="VAE_MAGVIT_V2",- in_out_channels = 4,- latent_embed_dim = 256,- filters = 128,- num_res_blocks = 4,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (False, True, True),- num_groups = 32, # for nn.GroupNorm- kl_embed_dim = 64,- activation_fn = 'swish',- separate_first_frame_encoding = False,- disable_space = True,- encoder_double_z = False,- custom_conv_padding = None-)---discriminator = dict(- type="DISCRIMINATOR_3D",- image_size = (16, 16), # NOTE: here image size is different- num_frames = num_frames,- in_channels = 4,- filters = 128,- use_pretrained=True, # NOTE: set to False only if we want to fresh train using a different discriminator!- # channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution- channel_multipliers= (2,4,4) # since on intermediate layer, 16 x 16 x 16 dimension z-)---# loss weights-logvar_init=0.0-kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-# discriminator_loss_weight = 0.5 # for generator adversarial loss-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1-lecam_loss_weight = 0.001 # NOTE: MAVGIT v2 use 0.001-# discriminator_loss_type="non-saturating"-# generator_loss_type="non-saturating"-discriminator_loss_type="hinge"-generator_loss_type="hinge"-discriminator_start = 1000 # 50000 NOTE: change to correct val, debug use -1 for now-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use-ema_decay = 0.999 # ema decay factor for generator---# Others-seed = 42-outputs = "outputs"-wandb = False--# Training-''' NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-'''--epochs = 2000-log_every = 1-ckpt_every = 1000-load = None--batch_size = 4-lr = 1e-4-grad_clip = 1.0diff --git a/configs/vae_magvit_v2/train/pipeline_16x128x128_LC_GP.py b/configs/vae_magvit_v2/train/pipeline_16x128x128_LC_GP.pydeleted file mode 100644index ae1f975..0000000--- a/configs/vae_magvit_v2/train/pipeline_16x128x128_LC_GP.py+++ /dev/null@@ -1,93 +0,0 @@-num_frames = 16-frame_interval = 3-image_size = (128, 128)-use_pipeline = True--# Define dataset-root = None-data_path = "CSV_PATH"-use_image_transform = False-num_workers = 4-video_contains_first_frame = False--# Define acceleration-dtype = "bf16"-grad_checkpoint = True-plugin = "zero2"-sp_size = 1---# Define model-vae_2d = dict(- type="VideoAutoencoderKL",- from_pretrained="stabilityai/sd-vae-ft-ema",- # SDXL-)--model = dict(- type="VAE_MAGVIT_V2",- in_out_channels = 4,- latent_embed_dim = 256,- filters = 128,- num_res_blocks = 4,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (False, True, True),- num_groups = 32, # for nn.GroupNorm- kl_embed_dim = 64,- activation_fn = 'swish',- separate_first_frame_encoding = False,- disable_space = True,- encoder_double_z = False,- custom_conv_padding = None-)---discriminator = dict(- type="DISCRIMINATOR_3D",- image_size = (16, 16), # NOTE: here image size is different- num_frames = num_frames,- in_channels = 4,- filters = 128,- use_pretrained=True, # NOTE: set to False only if we want to fresh train using a different discriminator!- # channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution- channel_multipliers= (2,4,4) # since on intermediate layer, 16 x 16 x 16 dimension z-)---# loss weights-logvar_init=0.0-kl_loss_weight = 0.000001-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0-discriminator_factor = 1.0 # for discriminator adversarial loss-# discriminator_loss_weight = 0.5 # for generator adversarial loss-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1-lecam_loss_weight = 0.001 # NOTE: MAVGIT v2 use 0.001-# discriminator_loss_type="non-saturating"-# generator_loss_type="non-saturating"-discriminator_loss_type="hinge"-generator_loss_type="hinge"-discriminator_start = 1000 # 50000 NOTE: change to correct val, debug use -1 for now-gradient_penalty_loss_weight = 10 # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use-ema_decay = 0.999 # ema decay factor for generator---# Others-seed = 42-outputs = "outputs"-wandb = False--# Training-''' NOTE:-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],-3-6 epochs for pexel, from pexel observation its correct-'''--epochs = 2000-log_every = 1-ckpt_every = 1000-load = None--batch_size = 4-lr = 1e-4-grad_clip = 1.0diff --git a/opensora/models/vae/__init__.py b/opensora/models/vae/__init__.pyindex 1dbf956..8b123a2 100644--- a/opensora/models/vae/__init__.py+++ b/opensora/models/vae/__init__.py@@ -1,4 +1,3 @@from .vae import VideoAutoencoderKL, VideoAutoencoderKLTemporalDecoder-from .vae_3d import VAE_3D, VAE_3D_Bfrom .vae_3d_v2 import VAE_3D_V2, VAE_MAGVIT_V2diff --git a/opensora/models/vae/discriminator_3d.py b/opensora/models/vae/discriminator_3d.pydeleted file mode 100644index 34b0bc2..0000000--- a/opensora/models/vae/discriminator_3d.py+++ /dev/null@@ -1,126 +0,0 @@-"""3D StyleGAN discriminator."""--import functools-import math-from typing import Any--import torch-import torch.nn as nn--# TODO: torch.nn.init.xavier_uniform_-# default_kernel_init = nn.initializers.xavier_uniform()--def xavier_uniform_weight_init(m):- if isinstance(m, nn.Conv3d) or isinstance(m, nn.Linear):- nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain('relu'))- if m.bias is not None:- nn.init.zeros_(m.bias)- # print("initialized module to xavier_uniform:", m)--class ResBlock(nn.Module):- """3D StyleGAN ResBlock for D."""-- def __init__(- self,- in_channels,- filters,- activation_fn,- num_groups=32,- device="cpu",- dtype=torch.bfloat16,- ):- super().__init__()-- self.filters = filters- self.activation_fn = activation_fn-- # SCH: NOTE: although paper says conv (X->Y, Y->Y), original code implementation is (X->X, X->Y), we follow code- self.conv1 = nn.Conv3d(in_channels, in_channels, (3,3,3), padding=1, dtype=dtype, device=device) # TODO: need to init to xavier_uniform- self.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)- self.avg_pool_with_t = nn.AvgPool3d((2,2,2),count_include_pad=False)- self.conv2 = nn.Conv3d(in_channels, self.filters,(1,1,1), bias=False, dtype=dtype, device=device) # need to init to xavier_uniform- self.conv3 = nn.Conv3d(in_channels, self.filters, (3,3,3), padding=1, dtype=dtype, device=device) # need to init to xavier_uniform- self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)- self.apply(xavier_uniform_weight_init)-- def forward(self, x):- residual = x- x = self.conv1(x)- x = self.norm1(x)- x = self.activation_fn(x)-- x = self.avg_pool_with_t(x)- residual = self.avg_pool_with_t(residual)- residual = self.conv2(residual)- x = self.conv3(x)- x = self.norm2(x)- x = self.activation_fn(x)- out = (residual + x) / math.sqrt(2)- return out--class StyleGANDiscriminator(nn.Module):- """StyleGAN Discriminator."""- def __init__(- self,- image_size = (128, 128),- num_frames = 17,- in_channels = 3,- filters = 128,- channel_multipliers = (2,4,4,4,4),- num_groups=32,- dtype = torch.bfloat16,- device="cpu",- ):- super().__init__()- self.dtype = dtype- self.input_size = image_size- self.filters = filters- self.activation_fn = nn.LeakyReLU(negative_slope=0.2)- self.channel_multipliers = channel_multipliers-- self.conv1 = nn.Conv3d(in_channels, self.filters, (3, 3, 3), padding=1, dtype=dtype, device=device) # need to init to xavier_uniform-- prev_filters = self.filters # record in_channels- self.num_blocks = len(self.channel_multipliers)- self.res_block_list = []- for i in range(self.num_blocks):- filters = self.filters * self.channel_multipliers[i]- self.res_block_list.append(ResBlock(prev_filters, filters, self.activation_fn, dtype=dtype,device=device))- prev_filters = filters # update in_channels-- self.conv2 = nn.Conv3d(prev_filters, prev_filters, (3,3,3), padding=1, dtype=dtype, device=device) # need to init to xavier_uniform-- self.norm1 = nn.GroupNorm(num_groups, prev_filters, dtype=dtype, device=device)-- scale_factor = 2 ** self.num_blocks- if num_frames % scale_factor != 0: # SCH: NOTE: has first frame which would be padded before usage- time_scaled = num_frames // scale_factor + 1- else:- time_scaled = num_frames / scale_factor- assert self.input_size[0] % scale_factor == 0, f"image width {self.input_size[0]} is not divisible by scale factor {scale_factor}"- assert self.input_size[1] % scale_factor == 0, f"image height {self.input_size[1]} is not divisible by scale factor {scale_factor}"-- w_scaled, h_scaled = self.input_size[0] / scale_factor, self.input_size[1] / scale_factor- in_features = int(prev_filters * time_scaled * w_scaled * h_scaled) # (C*T*W*H)- self.linear1 = nn.Linear(in_features, prev_filters, device=device, dtype=dtype) # need to init to xavier_uniform- self.linear2 = nn.Linear(prev_filters, 1, device=device, dtype=dtype) # need to init to xavier_uniform-- self.apply(xavier_uniform_weight_init)-- def forward(self, x):-- x = self.conv1(x)- x = self.activation_fn(x)-- for i in range(self.num_blocks):- x = self.res_block_list[i](x)-- x = self.conv2(x)- x = self.norm1(x)- x = self.activation_fn(x)- x = x.reshape((x.shape[0], -1)) # SCH: [B, (C * T * W * H)] ?- breakpoint()- x = self.linear1(x)- x = self.activation_fn(x)- x = self.linear2(x)- return xdiff --git a/opensora/models/vae/magvit2.py b/opensora/models/vae/magvit2.pydeleted file mode 100644index 455e128..0000000--- a/opensora/models/vae/magvit2.py+++ /dev/null@@ -1,1905 +0,0 @@-import copy-from pathlib import Path-from math import log2, ceil, sqrt-from functools import wraps, partial--import torch-import torch.nn.functional as F-from torch.cuda.amp import autocast-from torch import nn, einsum, Tensor-from torch.nn import Module, ModuleList-from torch.autograd import grad as torch_grad--import torchvision-from torchvision.models import VGG16_Weights--from collections import namedtuple--from vector_quantize_pytorch import LFQ, FSQ--from einops import rearrange, repeat, reduce, pack, unpack-from einops.layers.torch import Rearrange--from beartype import beartype-from beartype.typing import Union, Tuple, Optional, List--from magvit2_pytorch.attend import Attend-from magvit2_pytorch.version import __version__--from gateloop_transformer import SimpleGateLoopLayer--from taylor_series_linear_attention import TaylorSeriesLinearAttn--from kornia.filters import filter3d--import pickle--# helper--def exists(v):- return v is not None--def default(v, d):- return v if exists(v) else d--def safe_get_index(it, ind, default = None):- if ind < len(it):- return it[ind]- return default--def pair(t):- return t if isinstance(t, tuple) else (t, t)--def identity(t, *args, **kwargs):- return t--def divisible_by(num, den):- return (num % den) == 0--def pack_one(t, pattern):- return pack([t], pattern)--def unpack_one(t, ps, pattern):- return unpack(t, ps, pattern)[0]--def append_dims(t, ndims: int):- return t.reshape(*t.shape, *((1,) * ndims))--def is_odd(n):- return not divisible_by(n, 2)--def maybe_del_attr_(o, attr):- if hasattr(o, attr):- delattr(o, attr)--def cast_tuple(t, length = 1):- return t if isinstance(t, tuple) else ((t,) * length)--# tensor helpers--def l2norm(t):- return F.normalize(t, dim = -1, p = 2)--def pad_at_dim(t, pad, dim = -1, value = 0.):- dims_from_right = (- dim - 1) if dim < 0 else (t.ndim - dim - 1)- zeros = ((0, 0) * dims_from_right)- return F.pad(t, (*zeros, *pad), value = value)--def pick_video_frame(video, frame_indices):- batch, device = video.shape[0], video.device- video = rearrange(video, 'b c f ... -> b f c ...')- batch_indices = torch.arange(batch, device = device)- batch_indices = rearrange(batch_indices, 'b -> b 1')- images = video[batch_indices, frame_indices]- images = rearrange(images, 'b 1 c ... -> b c ...')- return images--# gan related--def gradient_penalty(images, output):- batch_size = images.shape[0]-- gradients = torch_grad(- outputs = output,- inputs = images,- grad_outputs = torch.ones(output.size(), device = images.device),- create_graph = True,- retain_graph = True,- only_inputs = True- )[0]-- gradients = rearrange(gradients, 'b ... -> b (...)')- return ((gradients.norm(2, dim = 1) - 1) ** 2).mean()--def leaky_relu(p = 0.1):- return nn.LeakyReLU(p)--def hinge_discr_loss(fake, real):- return (F.relu(1 + fake) + F.relu(1 - real)).mean()--def hinge_gen_loss(fake):- return -fake.mean()--@autocast(enabled = False)-@beartype-def grad_layer_wrt_loss(- loss: Tensor,- layer: nn.Parameter-):- return torch_grad(- outputs = loss,- inputs = layer,- grad_outputs = torch.ones_like(loss),- retain_graph = True- )[0].detach()--# helper decorators--def remove_vgg(fn):- @wraps(fn)- def inner(self, *args, **kwargs):- has_vgg = hasattr(self, 'vgg')- if has_vgg:- vgg = self.vgg- delattr(self, 'vgg')-- out = fn(self, *args, **kwargs)-- if has_vgg:- self.vgg = vgg-- return out- return inner--# helper classes--def Sequential(*modules):- modules = [*filter(exists, modules)]-- if len(modules) == 0:- return nn.Identity()-- return nn.Sequential(*modules)--class Residual(Module):- @beartype- def __init__(self, fn: Module):- super().__init__()- self.fn = fn-- def forward(self, x, **kwargs):- return self.fn(x, **kwargs) + x--# for a bunch of tensor operations to change tensor to (batch, time, feature dimension) and back--class ToTimeSequence(Module):- @beartype- def __init__(self, fn: Module):- super().__init__()- self.fn = fn-- def forward(self, x, **kwargs):- x = rearrange(x, 'b c f ... -> b ... f c')- x, ps = pack_one(x, '* n c')-- o = self.fn(x, **kwargs)-- o = unpack_one(o, ps, '* n c')- return rearrange(o, 'b ... f c -> b c f ...')---class SqueezeExcite(Module):- # global context network - attention-esque squeeze-excite variant (https://arxiv.org/abs/2012.13375)-- def __init__(- self,- dim,- *,- dim_out = None,- dim_hidden_min = 16,- init_bias = -10- ):- super().__init__()- dim_out = default(dim_out, dim)-- self.to_k = nn.Conv2d(dim, 1, 1)- dim_hidden = max(dim_hidden_min, dim_out // 2)-- self.net = nn.Sequential(- nn.Conv2d(dim, dim_hidden, 1),- nn.LeakyReLU(0.1),- nn.Conv2d(dim_hidden, dim_out, 1),- nn.Sigmoid()- )-- nn.init.zeros_(self.net[-2].weight)- nn.init.constant_(self.net[-2].bias, init_bias)-- def forward(self, x):- orig_input, batch = x, x.shape[0]- is_video = x.ndim == 5-- if is_video:- x = rearrange(x, 'b c f h w -> (b f) c h w')-- context = self.to_k(x)-- context = rearrange(context, 'b c h w -> b c (h w)').softmax(dim = -1)- spatial_flattened_input = rearrange(x, 'b c h w -> b c (h w)')-- out = einsum('b i n, b c n -> b c i', context, spatial_flattened_input)- out = rearrange(out, '... -> ... 1')- gates = self.net(out)-- if is_video:- gates = rearrange(gates, '(b f) c h w -> b c f h w', b = batch)-- return gates * orig_input--# token shifting--class TokenShift(Module):- @beartype- def __init__(self, fn: Module):- super().__init__()- self.fn = fn-- def forward(self, x, **kwargs):- x, x_shift = x.chunk(2, dim = 1)- x_shift = pad_at_dim(x_shift, (1, -1), dim = 2) # shift time dimension- x = torch.cat((x, x_shift), dim = 1)- return self.fn(x, **kwargs)--# rmsnorm--class RMSNorm(Module):- def __init__(- self,- dim,- channel_first = False,- images = False,- bias = False- ):- super().__init__()- broadcastable_dims = (1, 1, 1) if not images else (1, 1)- shape = (dim, *broadcastable_dims) if channel_first else (dim,)-- self.channel_first = channel_first- self.scale = dim ** 0.5- self.gamma = nn.Parameter(torch.ones(shape))- self.bias = nn.Parameter(torch.zeros(shape)) if bias else 0.-- def forward(self, x):- return F.normalize(x, dim = (1 if self.channel_first else -1)) * self.scale * self.gamma + self.bias--class AdaptiveRMSNorm(Module):- def __init__(- self,- dim,- *,- dim_cond,- channel_first = False,- images = False,- bias = False- ):- super().__init__()- broadcastable_dims = (1, 1, 1) if not images else (1, 1)- shape = (dim, *broadcastable_dims) if channel_first else (dim,)-- self.dim_cond = dim_cond- self.channel_first = channel_first- self.scale = dim ** 0.5-- self.to_gamma = nn.Linear(dim_cond, dim)- self.to_bias = nn.Linear(dim_cond, dim) if bias else None-- nn.init.zeros_(self.to_gamma.weight)- nn.init.ones_(self.to_gamma.bias)-- if bias:- nn.init.zeros_(self.to_bias.weight)- nn.init.zeros_(self.to_bias.bias)-- @beartype- def forward(self, x: Tensor, *, cond: Tensor):- batch = x.shape[0]- assert cond.shape == (batch, self.dim_cond)-- gamma = self.to_gamma(cond)-- bias = 0.- if exists(self.to_bias):- bias = self.to_bias(cond)-- if self.channel_first:- gamma = append_dims(gamma, x.ndim - 2)-- if exists(self.to_bias):- bias = append_dims(bias, x.ndim - 2)-- return F.normalize(x, dim = (1 if self.channel_first else -1)) * self.scale * gamma + bias--# attention--class Attention(Module):- @beartype- def __init__(- self,- *,- dim,- dim_cond: Optional[int] = None,- causal = False,- dim_head = 32,- heads = 8,- flash = False,- dropout = 0.,- num_memory_kv = 4- ):- super().__init__()- dim_inner = dim_head * heads-- self.need_cond = exists(dim_cond)-- if self.need_cond:- self.norm = AdaptiveRMSNorm(dim, dim_cond = dim_cond)- else:- self.norm = RMSNorm(dim)-- self.to_qkv = nn.Sequential(- nn.Linear(dim, dim_inner * 3, bias = False),- Rearrange('b n (qkv h d) -> qkv b h n d', qkv = 3, h = heads)- )-- assert num_memory_kv > 0- self.mem_kv = nn.Parameter(torch.randn(2, heads, num_memory_kv, dim_head))-- self.attend = Attend(- causal = causal,- dropout = dropout,- flash = flash- )-- self.to_out = nn.Sequential(- Rearrange('b h n d -> b n (h d)'),- nn.Linear(dim_inner, dim, bias = False)- )-- @beartype- def forward(- self,- x,- mask: Optional[Tensor ] = None,- cond: Optional[Tensor] = None- ):- maybe_cond_kwargs = dict(cond = cond) if self.need_cond else dict()-- x = self.norm(x, **maybe_cond_kwargs)-- q, k, v = self.to_qkv(x)-- mk, mv = map(lambda t: repeat(t, 'h n d -> b h n d', b = q.shape[0]), self.mem_kv)- k = torch.cat((mk, k), dim = -2)- v = torch.cat((mv, v), dim = -2)-- out = self.attend(q, k, v, mask = mask)- return self.to_out(out)--class LinearAttention(Module):- """- using the specific linear attention proposed in https://arxiv.org/abs/2106.09681- """-- @beartype- def __init__(- self,- *,- dim,- dim_cond: Optional[int] = None,- dim_head = 8,- heads = 8,- dropout = 0.- ):- super().__init__()- dim_inner = dim_head * heads-- self.need_cond = exists(dim_cond)-- if self.need_cond:- self.norm = AdaptiveRMSNorm(dim, dim_cond = dim_cond)- else:- self.norm = RMSNorm(dim)-- self.attn = TaylorSeriesLinearAttn(- dim = dim,- dim_head = dim_head,- heads = heads- )-- def forward(- self,- x,- cond: Optional[Tensor] = None- ):- maybe_cond_kwargs = dict(cond = cond) if self.need_cond else dict()-- x = self.norm(x, **maybe_cond_kwargs)-- return self.attn(x)--class LinearSpaceAttention(LinearAttention):- """- SCH: format h & w into a linear dim, do linear attention, then format back- """- def forward(self, x, *args, **kwargs):- x = rearrange(x, 'b c ... h w -> b ... h w c')- x, batch_ps = pack_one(x, '* h w c')- x, seq_ps = pack_one(x, 'b * c')-- x = super().forward(x, *args, **kwargs)-- x = unpack_one(x, seq_ps, 'b * c')- x = unpack_one(x, batch_ps, '* h w c')- return rearrange(x, 'b ... h w c -> b c ... h w')--class SpaceAttention(Attention):- def forward(self, x, *args, **kwargs):- x = rearrange(x, 'b c t h w -> b t h w c')- x, batch_ps = pack_one(x, '* h w c')- x, seq_ps = pack_one(x, 'b * c')-- x = super().forward(x, *args, **kwargs)-- x = unpack_one(x, seq_ps, 'b * c')- x = unpack_one(x, batch_ps, '* h w c')- return rearrange(x, 'b t h w c -> b c t h w')--class TimeAttention(Attention):- def forward(self, x, *args, **kwargs):- x = rearrange(x, 'b c t h w -> b h w t c')- x, batch_ps = pack_one(x, '* t c')-- x = super().forward(x, *args, **kwargs)-- x = unpack_one(x, batch_ps, '* t c')- return rearrange(x, 'b h w t c -> b c t h w')--class GEGLU(Module):- def forward(self, x):- x, gate = x.chunk(2, dim = 1)- return F.gelu(gate) * x--class FeedForward(Module):- @beartype- def __init__(- self,- dim,- *,- dim_cond: Optional[int] = None,- mult = 4,- images = False- ):- super().__init__()- conv_klass = nn.Conv2d if images else nn.Conv3d-- rmsnorm_klass = RMSNorm if not exists(dim_cond) else partial(AdaptiveRMSNorm, dim_cond = dim_cond)-- maybe_adaptive_norm_klass = partial(rmsnorm_klass, channel_first = True, images = images)-- dim_inner = int(dim * mult * 2 / 3)-- self.norm = maybe_adaptive_norm_klass(dim)-- self.net = Sequential(- conv_klass(dim, dim_inner * 2, 1),- GEGLU(),- conv_klass(dim_inner, dim, 1)- )-- @beartype- def forward(- self,- x: Tensor,- *,- cond: Optional[Tensor] = None- ):- maybe_cond_kwargs = dict(cond = cond) if exists(cond) else dict()-- x = self.norm(x, **maybe_cond_kwargs)- return self.net(x)--# discriminator with anti-aliased downsampling (blurpool Zhang et al.)--class Blur(Module):- def __init__(self):- super().__init__()- f = torch.Tensor([1, 2, 1])- self.register_buffer('f', f)-- def forward(- self,- x,- space_only = False,- time_only = False- ):- assert not (space_only and time_only)-- f = self.f-- if space_only:- f = einsum('i, j -> i j', f, f)- f = rearrange(f, '... -> 1 1 ...')- elif time_only:- f = rearrange(f, 'f -> 1 f 1 1')- else:- f = einsum('i, j, k -> i j k', f, f, f)- f = rearrange(f, '... -> 1 ...')-- is_images = x.ndim == 4-- if is_images:- x = rearrange(x, 'b c h w -> b c 1 h w')-- out = filter3d(x, f, normalized = True)-- if is_images:- out = rearrange(out, 'b c 1 h w -> b c h w')-- return out--class DiscriminatorBlock(Module):- def __init__(- self,- input_channels,- filters,- downsample = True,- antialiased_downsample = True- ):- super().__init__()- self.conv_res = nn.Conv2d(input_channels, filters, 1, stride = (2 if downsample else 1))-- self.net = nn.Sequential(- nn.Conv2d(input_channels, filters, 3, padding = 1),- leaky_relu(),- nn.Conv2d(filters, filters, 3, padding = 1),- leaky_relu()- )-- self.maybe_blur = Blur() if antialiased_downsample else None-- self.downsample = nn.Sequential(- Rearrange('b c (h p1) (w p2) -> b (c p1 p2) h w', p1 = 2, p2 = 2),- nn.Conv2d(filters * 4, filters, 1)- ) if downsample else None-- def forward(self, x):- res = self.conv_res(x)-- x = self.net(x)-- if exists(self.downsample):- if exists(self.maybe_blur):- x = self.maybe_blur(x, space_only = True)-- x = self.downsample(x)-- x = (x + res) * (2 ** -0.5)- return x--class Discriminator(Module):- @beartype- def __init__(- self,- *,- dim,- image_size,- channels = 3,- max_dim = 512,- attn_heads = 8,- attn_dim_head = 32,- linear_attn_dim_head = 8,- linear_attn_heads = 16,- ff_mult = 4,- antialiased_downsample = False- ):- super().__init__()- image_size = pair(image_size)- min_image_resolution = min(image_size)-- num_layers = int(log2(min_image_resolution) - 2)-- blocks = []-- layer_dims = [channels] + [(dim * 4) * (2 ** i) for i in range(num_layers + 1)] # SCH: num channels across each layer- layer_dims = [min(layer_dim, max_dim) for layer_dim in layer_dims]- layer_dims_in_out = tuple(zip(layer_dims[:-1], layer_dims[1:]))-- blocks = []- attn_blocks = []-- image_resolution = min_image_resolution-- for ind, (in_chan, out_chan) in enumerate(layer_dims_in_out):- num_layer = ind + 1- is_not_last = ind != (len(layer_dims_in_out) - 1)-- block = DiscriminatorBlock(- in_chan,- out_chan,- downsample = is_not_last,- antialiased_downsample = antialiased_downsample- )-- attn_block = Sequential(- Residual(LinearSpaceAttention(- dim = out_chan,- heads = linear_attn_heads,- dim_head = linear_attn_dim_head- )),- Residual(FeedForward(- dim = out_chan,- mult = ff_mult,- images = True- ))- )-- blocks.append(ModuleList([- block,- attn_block- ]))-- image_resolution //= 2-- self.blocks = ModuleList(blocks)-- dim_last = layer_dims[-1]-- downsample_factor = 2 ** num_layers- last_fmap_size = tuple(map(lambda n: n // downsample_factor, image_size))-- latent_dim = last_fmap_size[0] * last_fmap_size[1] * dim_last-- self.to_logits = Sequential(- nn.Conv2d(dim_last, dim_last, 3, padding = 1),- leaky_relu(),- Rearrange('b ... -> b (...)'),- nn.Linear(latent_dim, 1),- Rearrange('b 1 -> b')- )-- def forward(self, x):-- for block, attn_block in self.blocks:- x = block(x)- x = attn_block(x)-- return self.to_logits(x)--# modulatable conv from Karras et al. Stylegan2-# for conditioning on latents--class Conv3DMod(Module):- @beartype- def __init__(- self,- dim,- *,- spatial_kernel,- time_kernel,- causal = True,- dim_out = None,- demod = True,- eps = 1e-8,- pad_mode = 'zeros'- ):- super().__init__()- dim_out = default(dim_out, dim)-- self.eps = eps-- assert is_odd(spatial_kernel) and is_odd(time_kernel)-- self.spatial_kernel = spatial_kernel- self.time_kernel = time_kernel-- time_padding = (time_kernel - 1, 0) if causal else ((time_kernel // 2,) * 2)-- self.pad_mode = pad_mode- self.padding = (*((spatial_kernel // 2,) * 4), *time_padding)- self.weights = nn.Parameter(torch.randn((dim_out, dim, time_kernel, spatial_kernel, spatial_kernel)))-- self.demod = demod-- nn.init.kaiming_normal_(self.weights, a = 0, mode = 'fan_in', nonlinearity = 'selu')-- @beartype- def forward(- self,- fmap,- cond: Tensor- ):- """- notation-- b - batch- n - convs- o - output- i - input- k - kernel- """-- b = fmap.shape[0]-- # prepare weights for modulation-- weights = self.weights-- # do the modulation, demodulation, as done in stylegan2-- cond = rearrange(cond, 'b i -> b 1 i 1 1 1')-- weights = weights * (cond + 1)-- if self.demod:- inv_norm = reduce(weights ** 2, 'b o i k0 k1 k2 -> b o 1 1 1 1', 'sum').clamp(min = self.eps).rsqrt()- weights = weights * inv_norm-- fmap = rearrange(fmap, 'b c t h w -> 1 (b c) t h w')-- weights = rearrange(weights, 'b o ... -> (b o) ...')-- fmap = F.pad(fmap, self.padding, mode = self.pad_mode)- fmap = F.conv3d(fmap, weights, groups = b)-- return rearrange(fmap, '1 (b o) ... -> b o ...', b = b)--# strided conv downsamples--class SpatialDownsample2x(Module):- def __init__(- self,- dim,- dim_out = None,- kernel_size = 3,- antialias = False- ):- super().__init__()- dim_out = default(dim_out, dim)- self.maybe_blur = Blur() if antialias else identity- self.conv = nn.Conv2d(dim, dim_out, kernel_size, stride = 2, padding = kernel_size // 2)-- def forward(self, x):- x = self.maybe_blur(x, space_only = True)-- x = rearrange(x, 'b c t h w -> b t c h w')- x, ps = pack_one(x, '* c h w')-- out = self.conv(x)-- out = unpack_one(out, ps, '* c h w')- out = rearrange(out, 'b t c h w -> b c t h w')- return out--class TimeDownsample2x(Module):- def __init__(- self,- dim,- dim_out = None,- kernel_size = 3,- antialias = False- ):- super().__init__()- dim_out = default(dim_out, dim)- self.maybe_blur = Blur() if antialias else identity- self.time_causal_padding = (kernel_size - 1, 0)- self.conv = nn.Conv1d(dim, dim_out, kernel_size, stride = 2)-- def forward(self, x):- x = self.maybe_blur(x, time_only = True)-- x = rearrange(x, 'b c t h w -> b h w c t')- x, ps = pack_one(x, '* c t')-- x = F.pad(x, self.time_causal_padding)- out = self.conv(x)-- out = unpack_one(out, ps, '* c t')- out = rearrange(out, 'b h w c t -> b c t h w')- return out--# depth to space upsamples--class SpatialUpsample2x(Module):- def __init__(- self,- dim,- dim_out = None- ):- super().__init__()- dim_out = default(dim_out, dim)- conv = nn.Conv2d(dim, dim_out * 4, 1)-- self.net = nn.Sequential(- conv,- nn.SiLU(),- Rearrange('b (c p1 p2) h w -> b c (h p1) (w p2)', p1 = 2, p2 = 2)- )-- self.init_conv_(conv)-- def init_conv_(self, conv):- o, i, h, w = conv.weight.shape- conv_weight = torch.empty(o // 4, i, h, w)- nn.init.kaiming_uniform_(conv_weight)- conv_weight = repeat(conv_weight, 'o ... -> (o 4) ...')-- conv.weight.data.copy_(conv_weight)- nn.init.zeros_(conv.bias.data)-- def forward(self, x):- x = rearrange(x, 'b c t h w -> b t c h w')- x, ps = pack_one(x, '* c h w')-- out = self.net(x)-- out = unpack_one(out, ps, '* c h w')- out = rearrange(out, 'b t c h w -> b c t h w')- return out--class TimeUpsample2x(Module):- def __init__(- self,- dim,- dim_out = None- ):- super().__init__()- dim_out = default(dim_out, dim)- conv = nn.Conv1d(dim, dim_out * 2, 1)-- self.net = nn.Sequential(- conv,- nn.SiLU(),- Rearrange('b (c p) t -> b c (t p)', p = 2)- )-- self.init_conv_(conv)-- def init_conv_(self, conv):- o, i, t = conv.weight.shape- conv_weight = torch.empty(o // 2, i, t)- nn.init.kaiming_uniform_(conv_weight)- conv_weight = repeat(conv_weight, 'o ... -> (o 2) ...')-- conv.weight.data.copy_(conv_weight)- nn.init.zeros_(conv.bias.data)-- def forward(self, x):- x = rearrange(x, 'b c t h w -> b h w c t')- x, ps = pack_one(x, '* c t')-- out = self.net(x)-- out = unpack_one(out, ps, '* c t')- out = rearrange(out, 'b h w c t -> b c t h w')- return out--# autoencoder - only best variant here offered, with causal conv 3d--def SameConv2d(dim_in, dim_out, kernel_size):- kernel_size = cast_tuple(kernel_size, 2)- padding = [k // 2 for k in kernel_size]- return nn.Conv2d(dim_in, dim_out, kernel_size = kernel_size, padding = padding)--class CausalConv3d(Module):- @beartype- def __init__(- self,- chan_in,- chan_out,- kernel_size: Union[int, Tuple[int, int, int]],- pad_mode = 'constant',- **kwargs- ):- super().__init__()- kernel_size = cast_tuple(kernel_size, 3)-- time_kernel_size, height_kernel_size, width_kernel_size = kernel_size-- assert is_odd(height_kernel_size) and is_odd(width_kernel_size)-- dilation = kwargs.pop('dilation', 1)- stride = kwargs.pop('stride', 1)-- self.pad_mode = pad_mode- time_pad = dilation * (time_kernel_size - 1) + (1 - stride)- height_pad = height_kernel_size // 2- width_pad = width_kernel_size // 2-- self.time_pad = time_pad- self.time_causal_padding = (width_pad, width_pad, height_pad, height_pad, time_pad, 0)-- stride = (stride, 1, 1)- dilation = (dilation, 1, 1)- self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride = stride, dilation = dilation, **kwargs)-- def forward(self, x):- pad_mode = self.pad_mode if self.time_pad < x.shape[2] else 'constant'-- x = F.pad(x, self.time_causal_padding, mode = pad_mode)- return self.conv(x)--@beartype-def ResidualUnit(- dim,- kernel_size: Union[int, Tuple[int, int, int]],- pad_mode: str = 'constant'-):- net = Sequential(- CausalConv3d(dim, dim, kernel_size, pad_mode = pad_mode),- nn.ELU(),- nn.Conv3d(dim, dim, 1),- nn.ELU(),- SqueezeExcite(dim)- )-- return Residual(net)--@beartype-class ResidualUnitMod(Module):- def __init__(- self,- dim,- kernel_size: Union[int, Tuple[int, int, int]],- *,- dim_cond,- pad_mode: str = 'constant',- demod = True- ):- super().__init__()- kernel_size = cast_tuple(kernel_size, 3)- time_kernel_size, height_kernel_size, width_kernel_size = kernel_size- assert height_kernel_size == width_kernel_size-- self.to_cond = nn.Linear(dim_cond, dim)-- self.conv = Conv3DMod(- dim = dim,- spatial_kernel = height_kernel_size,- time_kernel = time_kernel_size,- causal = True,- demod = demod,- pad_mode = pad_mode- )-- self.conv_out = nn.Conv3d(dim, dim, 1)-- @beartype- def forward(- self,- x,- cond: Tensor,- ):- res = x- cond = self.to_cond(cond)-- x = self.conv(x, cond = cond)- x = F.elu(x)- x = self.conv_out(x)- x = F.elu(x)- return x + res--class CausalConvTranspose3d(Module):- def __init__(- self,- chan_in,- chan_out,- kernel_size: Union[int, Tuple[int, int, int]],- *,- time_stride,- **kwargs- ):- super().__init__()- kernel_size = cast_tuple(kernel_size, 3)-- time_kernel_size, height_kernel_size, width_kernel_size = kernel_size-- assert is_odd(height_kernel_size) and is_odd(width_kernel_size)-- self.upsample_factor = time_stride-- height_pad = height_kernel_size // 2- width_pad = width_kernel_size // 2-- stride = (time_stride, 1, 1)- padding = (0, height_pad, width_pad)-- self.conv = nn.ConvTranspose3d(chan_in, chan_out, kernel_size, stride, padding = padding, **kwargs)-- def forward(self, x):- assert x.ndim == 5- t = x.shape[2]-- out = self.conv(x)-- out = out[..., :(t * self.upsample_factor), :, :]- return out--# video tokenizer class--LossBreakdown = namedtuple('LossBreakdown', [- 'recon_loss',- 'lfq_aux_loss',- 'quantizer_loss_breakdown',- 'perceptual_loss',- 'adversarial_gen_loss',- 'adaptive_adversarial_weight',- 'multiscale_gen_losses',- 'multiscale_gen_adaptive_weights'-])--DiscrLossBreakdown = namedtuple('DiscrLossBreakdown', [- 'discr_loss',- 'multiscale_discr_losses',- 'gradient_penalty'-])--class VideoTokenizer(Module):- @beartype- def __init__(- self,- *,- image_size,- layers: Tuple[Union[str, Tuple[str, int]], ...] = (- 'residual',- 'residual',- 'residual'- ),- residual_conv_kernel_size = 3,- # num_codebooks = 1,- # codebook_size: Optional[int] = None,- channels = 3,- init_dim = 64,- max_dim = float('inf'),- dim_cond = None,- dim_cond_expansion_factor = 4.,- input_conv_kernel_size: Tuple[int, int, int] = (7, 7, 7),- output_conv_kernel_size: Tuple[int, int, int] = (3, 3, 3),- pad_mode: str = 'constant',- lfq_entropy_loss_weight = 0.1,- lfq_commitment_loss_weight = 1., # SCH: codebook?- lfq_diversity_gamma = 2.5,- quantizer_aux_loss_weight = 1.,- lfq_activation = nn.Identity(),- use_fsq = False,- fsq_levels: Optional[List[int]] = None,- attn_dim_head = 32,- attn_heads = 8,- attn_dropout = 0.,- linear_attn_dim_head = 8,- linear_attn_heads = 16,- vgg: Optional[Module] = None,- vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,- perceptual_loss_weight = 1e-1,- discr_kwargs: Optional[dict] = None,- multiscale_discrs: Tuple[Module, ...] = tuple(),- use_gan = True,- adversarial_loss_weight = 1.,- grad_penalty_loss_weight = 10.,- multiscale_adversarial_loss_weight = 1.,- flash_attn = True,- separate_first_frame_encoding = False- ):- super().__init__()-- # for autosaving the config-- _locals = locals()- _locals.pop('self', None)- _locals.pop('__class__', None)- self._configs = pickle.dumps(_locals)-- # image size-- self.channels = channels- self.image_size = image_size-- # initial encoder-- self.conv_in = CausalConv3d(channels, init_dim, input_conv_kernel_size, pad_mode = pad_mode)-- # whether to encode the first frame separately or not-- self.conv_in_first_frame = nn.Identity()- self.conv_out_first_frame = nn.Identity()-- if separate_first_frame_encoding:- self.conv_in_first_frame = SameConv2d(channels, init_dim, input_conv_kernel_size[-2:])- self.conv_out_first_frame = SameConv2d(init_dim, channels, output_conv_kernel_size[-2:])-- self.separate_first_frame_encoding = separate_first_frame_encoding-- # encoder and decoder layers-- self.encoder_layers = ModuleList([])- self.decoder_layers = ModuleList([])-- self.conv_out = CausalConv3d(init_dim, channels, output_conv_kernel_size, pad_mode = pad_mode)-- dim = init_dim- dim_out = dim-- layer_fmap_size = image_size # SCH: feaure map size- time_downsample_factor = 1- has_cond_across_layers = [] # SCH: record if the corr. layers has condition-- for layer_def in layers:- layer_type, *layer_params = cast_tuple(layer_def)-- has_cond = False-- if layer_type == 'residual': # SCH: resblock- encoder_layer = ResidualUnit(dim, residual_conv_kernel_size)- decoder_layer = ResidualUnit(dim, residual_conv_kernel_size)-- elif layer_type == 'consecutive_residual':- num_consecutive, = layer_params- encoder_layer = Sequential(*[ResidualUnit(dim, residual_conv_kernel_size) for _ in range(num_consecutive)])- decoder_layer = Sequential(*[ResidualUnit(dim, residual_conv_kernel_size) for _ in range(num_consecutive)])-- elif layer_type == 'cond_residual':- assert exists(dim_cond), 'dim_cond must be passed into VideoTokenizer, if tokenizer is to be conditioned'-- has_cond = True-- encoder_layer = ResidualUnitMod(dim, residual_conv_kernel_size, dim_cond = int(dim_cond * dim_cond_expansion_factor))- decoder_layer = ResidualUnitMod(dim, residual_conv_kernel_size, dim_cond = int(dim_cond * dim_cond_expansion_factor))- dim_out = dim-- elif layer_type == 'compress_space':- dim_out = safe_get_index(layer_params, 0)- dim_out = default(dim_out, dim * 2) # SCH: if dim_out exists, else use dim * 2- dim_out = min(dim_out, max_dim)-- encoder_layer = SpatialDownsample2x(dim, dim_out) # SCH: 2d conv in space dimensions- decoder_layer = SpatialUpsample2x(dim_out, dim) # SCH: 2d conv in space dimensions, use more channel to expand space dim-- assert layer_fmap_size > 1- layer_fmap_size //= 2-- elif layer_type == 'compress_time':- dim_out = safe_get_index(layer_params, 0)- dim_out = default(dim_out, dim * 2)- dim_out = min(dim_out, max_dim)-- encoder_layer = TimeDownsample2x(dim, dim_out) # SCH: 1d conv in time dim to reduce- decoder_layer = TimeUpsample2x(dim_out, dim) # SCH: 1d conv in time dim, use more channels to expand time dim-- time_downsample_factor *= 2-- elif layer_type == 'attend_space':- attn_kwargs = dict(- dim = dim,- dim_head = attn_dim_head,- heads = attn_heads,- dropout = attn_dropout,- flash = flash_attn- )-- encoder_layer = Sequential(- Residual(SpaceAttention(**attn_kwargs)),- Residual(FeedForward(dim))- )-- decoder_layer = Sequential(- Residual(SpaceAttention(**attn_kwargs)),- Residual(FeedForward(dim))- )-- elif layer_type == 'linear_attend_space':- linear_attn_kwargs = dict(- dim = dim,- dim_head = linear_attn_dim_head,- heads = linear_attn_heads- )-- encoder_layer = Sequential(- Residual(LinearSpaceAttention(**linear_attn_kwargs)),- Residual(FeedForward(dim))- )-- decoder_layer = Sequential(- Residual(LinearSpaceAttention(**linear_attn_kwargs)),- Residual(FeedForward(dim))- )-- elif layer_type == 'gateloop_time':- gateloop_kwargs = dict(- use_heinsen = False- )-- encoder_layer = ToTimeSequence(Residual(SimpleGateLoopLayer(dim = dim)))- decoder_layer = ToTimeSequence(Residual(SimpleGateLoopLayer(dim = dim)))-- elif layer_type == 'attend_time':- attn_kwargs = dict(- dim = dim,- dim_head = attn_dim_head,- heads = attn_heads,- dropout = attn_dropout,- causal = True,- flash = flash_attn- )-- encoder_layer = Sequential(- Residual(TokenShift(TimeAttention(**attn_kwargs))),- Residual(TokenShift(FeedForward(dim, dim_cond = dim_cond)))- )-- decoder_layer = Sequential(- Residual(TokenShift(TimeAttention(**attn_kwargs))),- Residual(TokenShift(FeedForward(dim, dim_cond = dim_cond)))- )-- elif layer_type == 'cond_attend_space':- has_cond = True-- attn_kwargs = dict(- dim = dim,- dim_cond = dim_cond,- dim_head = attn_dim_head,- heads = attn_heads,- dropout = attn_dropout,- flash = flash_attn- )-- encoder_layer = Sequential(- Residual(SpaceAttention(**attn_kwargs)),- Residual(FeedForward(dim))- )-- decoder_layer = Sequential(- Residual(SpaceAttention(**attn_kwargs)),- Residual(FeedForward(dim))- )-- elif layer_type == 'cond_linear_attend_space':- has_cond = True-- attn_kwargs = dict(- dim = dim,- dim_cond = dim_cond,- dim_head = attn_dim_head,- heads = attn_heads,- dropout = attn_dropout,- flash = flash_attn- )-- encoder_layer = Sequential(- Residual(LinearSpaceAttention(**attn_kwargs)),- Residual(FeedForward(dim, dim_cond = dim_cond))- )-- decoder_layer = Sequential(- Residual(LinearSpaceAttention(**attn_kwargs)),- Residual(FeedForward(dim, dim_cond = dim_cond))- )-- elif layer_type == 'cond_attend_time':- has_cond = True-- attn_kwargs = dict(- dim = dim,- dim_cond = dim_cond,- dim_head = attn_dim_head,- heads = attn_heads,- dropout = attn_dropout,- causal = True,- flash = flash_attn- )-- encoder_layer = Sequential(- Residual(TokenShift(TimeAttention(**attn_kwargs))),- Residual(TokenShift(FeedForward(dim, dim_cond = dim_cond)))- )-- decoder_layer = Sequential(- Residual(TokenShift(TimeAttention(**attn_kwargs))),- Residual(TokenShift(FeedForward(dim, dim_cond = dim_cond)))- )-- else:- raise ValueError(f'unknown layer type {layer_type}')-- self.encoder_layers.append(encoder_layer)- self.decoder_layers.insert(0, decoder_layer)-- dim = dim_out- has_cond_across_layers.append(has_cond)-- # add a final norm just before quantization layer-- self.encoder_layers.append(Sequential(- Rearrange('b c ... -> b ... c'),- nn.LayerNorm(dim),- Rearrange('b ... c -> b c ...'),- ))-- self.time_downsample_factor = time_downsample_factor- self.time_padding = time_downsample_factor - 1-- self.fmap_size = layer_fmap_size-- # use a MLP stem for conditioning, if needed-- self.has_cond_across_layers = has_cond_across_layers- self.has_cond = any(has_cond_across_layers)-- self.encoder_cond_in = nn.Identity()- self.decoder_cond_in = nn.Identity()-- if has_cond:- self.dim_cond = dim_cond-- self.encoder_cond_in = Sequential(- nn.Linear(dim_cond, int(dim_cond * dim_cond_expansion_factor)),- nn.SiLU()- )-- self.decoder_cond_in = Sequential(- nn.Linear(dim_cond, int(dim_cond * dim_cond_expansion_factor)),- nn.SiLU()- )-- ## SCH: remove quantizer- # # quantizer related-- # self.use_fsq = use_fsq-- # if not use_fsq:- # assert exists(codebook_size) and not exists(fsq_levels), 'if use_fsq is set to False, `codebook_size` must be set (and not `fsq_levels`)'-- # # lookup free quantizer(s) - multiple codebooks is possible- # # each codebook will get its own entropy regularization-- # self.quantizers = LFQ(- # dim = dim,- # codebook_size = codebook_size,- # num_codebooks = num_codebooks,- # entropy_loss_weight = lfq_entropy_loss_weight,- # commitment_loss_weight = lfq_commitment_loss_weight,- # diversity_gamma = lfq_diversity_gamma- # )-- # else:- # assert not exists(codebook_size) and exists(fsq_levels), 'if use_fsq is set to True, `fsq_levels` must be set (and not `codebook_size`). the effective codebook size is the cumulative product of all the FSQ levels'-- # self.quantizers = FSQ(- # fsq_levels,- # dim = dim,- # num_codebooks = num_codebooks- # )-- # self.quantizer_aux_loss_weight = quantizer_aux_loss_weight-- # dummy loss-- self.register_buffer('zero', torch.tensor(0.), persistent = False)-- # perceptual loss related-- use_vgg = channels in {1, 3, 4} and perceptual_loss_weight > 0.-- self.vgg = None- self.perceptual_loss_weight = perceptual_loss_weight-- if use_vgg:- if not exists(vgg):- vgg = torchvision.models.vgg16(- weights = vgg_weights- )-- vgg.classifier = Sequential(*vgg.classifier[:-2])-- self.vgg = vgg-- self.use_vgg = use_vgg-- # main flag for whether to use GAN at all-- self.use_gan = use_gan-- # discriminator-- discr_kwargs = default(discr_kwargs, dict(- dim = dim,- image_size = image_size,- channels = channels,- max_dim = 512- ))-- self.discr = Discriminator(**discr_kwargs)-- self.adversarial_loss_weight = adversarial_loss_weight- self.grad_penalty_loss_weight = grad_penalty_loss_weight-- self.has_gan = use_gan and adversarial_loss_weight > 0.-- # multi-scale discriminators-- self.has_multiscale_gan = use_gan and multiscale_adversarial_loss_weight > 0.-- self.multiscale_discrs = ModuleList([*multiscale_discrs])-- self.multiscale_adversarial_loss_weight = multiscale_adversarial_loss_weight-- self.has_multiscale_discrs = (- use_gan and \- multiscale_adversarial_loss_weight > 0. and \- len(multiscale_discrs) > 0- )-- @property- def device(self):- return self.zero.device-- @classmethod- def init_and_load_from(cls, path, strict = True):- path = Path(path)- assert path.exists()- pkg = torch.load(str(path), map_location = 'cpu')-- assert 'config' in pkg, 'model configs were not found in this saved checkpoint'-- config = pickle.loads(pkg['config'])- tokenizer = cls(**config)- tokenizer.load(path, strict = strict)- return tokenizer-- def parameters(self):- return [- *self.conv_in.parameters(),- *self.conv_in_first_frame.parameters(),- *self.conv_out_first_frame.parameters(),- *self.conv_out.parameters(),- *self.encoder_layers.parameters(),- *self.decoder_layers.parameters(),- *self.encoder_cond_in.parameters(),- *self.decoder_cond_in.parameters(),- *self.quantizers.parameters()- ]-- def discr_parameters(self):- return self.discr.parameters()-- def copy_for_eval(self):- device = self.device- vae_copy = copy.deepcopy(self.cpu())-- maybe_del_attr_(vae_copy, 'discr')- maybe_del_attr_(vae_copy, 'vgg')- maybe_del_attr_(vae_copy, 'multiscale_discrs')-- vae_copy.eval()- return vae_copy.to(device)-- @remove_vgg- def state_dict(self, *args, **kwargs):- return super().state_dict(*args, **kwargs)-- @remove_vgg- def load_state_dict(self, *args, **kwargs):- return super().load_state_dict(*args, **kwargs)-- def save(self, path, overwrite = True):- path = Path(path)- assert overwrite or not path.exists(), f'{str(path)} already exists'-- pkg = dict(- model_state_dict = self.state_dict(),- version = __version__,- config = self._configs- )-- torch.save(pkg, str(path))-- def load(self, path, strict = True):- path = Path(path)- assert path.exists()-- pkg = torch.load(str(path))- state_dict = pkg.get('model_state_dict')- version = pkg.get('version')-- assert exists(state_dict)-- if exists(version):- print(f'loading checkpointed tokenizer from version {version}')-- self.load_state_dict(state_dict, strict = strict)-- @beartype- def encode(- self,- video: Tensor,- quantize = False,- cond: Optional[Tensor] = None,- video_contains_first_frame = True- ):- """- SCH: conv (may sep 1st frame), then pass through self.encoder_layers, then quantize if needed, finish- """- encode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame-- # whether to pad video or not-- if video_contains_first_frame:- video_len = video.shape[2]-- video = pad_at_dim(video, (self.time_padding, 0), value = 0., dim = 2)- video_packed_shape = [torch.Size([self.time_padding]), torch.Size([]), torch.Size([video_len - 1])]-- # conditioning, if needed-- assert (not self.has_cond) or exists(cond), '`cond` must be passed into tokenizer forward method since conditionable layers were specified'-- if exists(cond):- assert cond.shape == (video.shape[0], self.dim_cond)-- cond = self.encoder_cond_in(cond)- cond_kwargs = dict(cond = cond)-- # initial conv- # taking into account whether to encode first frame separately-- if encode_first_frame_separately:- pad, first_frame, video = unpack(video, video_packed_shape, 'b c * h w')- first_frame = self.conv_in_first_frame(first_frame)-- video = self.conv_in(video)-- if encode_first_frame_separately:- video, _ = pack([first_frame, video], 'b c * h w')- video = pad_at_dim(video, (self.time_padding, 0), dim = 2)-- # encoder layers-- for fn, has_cond in zip(self.encoder_layers, self.has_cond_across_layers):-- layer_kwargs = dict()-- if has_cond:- layer_kwargs = cond_kwargs-- video = fn(video, **layer_kwargs)-- maybe_quantize = identity if not quantize else self.quantizers-- return maybe_quantize(video)-- # @beartype- # def decode_from_code_indices(- # self,- # codes: Tensor,- # cond: Optional[Tensor] = None,- # video_contains_first_frame = True- # ):- # assert codes.dtype in (torch.long, torch.int32)-- # if codes.ndim == 2:- # video_code_len = codes.shape[-1]- # assert divisible_by(video_code_len, self.fmap_size ** 2), f'flattened video ids must have a length ({video_code_len}) that is divisible by the fmap size ({self.fmap_size}) squared ({self.fmap_size ** 2})'-- # codes = rearrange(codes, 'b (f h w) -> b f h w', h = self.fmap_size, w = self.fmap_size)-- # quantized = self.quantizers.indices_to_codes(codes)-- # return self.decode(quantized, cond = cond, video_contains_first_frame = video_contains_first_frame)-- @beartype- def decode(- self,- quantized: Tensor,- cond: Optional[Tensor] = None,- video_contains_first_frame = True- ):- decode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame-- batch = quantized.shape[0]-- # conditioning, if needed-- assert (not self.has_cond) or exists(cond), '`cond` must be passed into tokenizer forward method since conditionable layers were specified'-- if exists(cond): # SCH: quantized latents used as control signal following StyleGAN?- assert cond.shape == (batch, self.dim_cond)-- cond = self.decoder_cond_in(cond) # SCH: linear + activation- cond_kwargs = dict(cond = cond)-- # decoder layers-- x = quantized-- for fn, has_cond in zip(self.decoder_layers, reversed(self.has_cond_across_layers)):-- layer_kwargs = dict()-- if has_cond:- layer_kwargs = cond_kwargs-- x = fn(x, **layer_kwargs)-- # to pixels-- if decode_first_frame_separately:- left_pad, xff, x = x[:, :, :self.time_padding], x[:, :, self.time_padding], x[:, :, (self.time_padding + 1):]-- out = self.conv_out(x)- outff = self.conv_out_first_frame(xff)-- video, _ = pack([outff, out], 'b c * h w')-- else:- video = self.conv_out(x)-- # if video were padded, remove padding-- if video_contains_first_frame:- video = video[:, :, self.time_padding:]-- return video-- @torch.no_grad()- def tokenize(self, video):- self.eval()- return self.forward(video, return_codes = True)-- @beartype- def forward(- self,- video_or_images: Tensor,- cond: Optional[Tensor] = None,- return_loss = False,- return_codes = False,- return_recon = False,- return_discr_loss = False,- return_recon_loss_only = False,- apply_gradient_penalty = True,- video_contains_first_frame = True,- adversarial_loss_weight = None,- multiscale_adversarial_loss_weight = None- ):- adversarial_loss_weight = default(adversarial_loss_weight, self.adversarial_loss_weight)- multiscale_adversarial_loss_weight = default(multiscale_adversarial_loss_weight, self.multiscale_adversarial_loss_weight)-- assert (return_loss + return_codes + return_discr_loss) <= 1- assert video_or_images.ndim in {4, 5}-- assert video_or_images.shape[-2:] == (self.image_size, self.image_size)-- # accept images for image pretraining (curriculum learning from images to video)-- is_image = video_or_images.ndim == 4-- if is_image:- video = rearrange(video_or_images, 'b c ... -> b c 1 ...')- video_contains_first_frame = True- else:- video = video_or_images-- batch, channels, frames = video.shape[:3]-- assert divisible_by(frames - int(video_contains_first_frame), self.time_downsample_factor), f'number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}'-- # encoder-- x = self.encode(video, cond = cond, video_contains_first_frame = video_contains_first_frame)--- ## SCH: remove the codebook- # # lookup free quantization-- # if self.use_fsq:- # quantized, codes = self.quantizers(x)-- # aux_losses = self.zero- # quantizer_loss_breakdown = None- # else:- # (quantized, codes, aux_losses), quantizer_loss_breakdown = self.quantizers(x, return_loss_breakdown = True)-- # if return_codes and not return_recon:- # return codes-- # decoder- recon_video = self.decode(x, cond = cond, video_contains_first_frame = video_contains_first_frame)-- # if return_codes:- # return codes, recon_video-- # reconstruction loss-- if not (return_loss or return_discr_loss or return_recon_loss_only):- return recon_video-- recon_loss = F.mse_loss(video, recon_video)-- # for validation, only return recon loss-- if return_recon_loss_only:- return recon_loss, recon_video-- # TODO:- # gan discriminator loss-- if return_discr_loss:- assert self.has_gan- assert exists(self.discr)-- # pick a random frame for image discriminator-- frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices-- real = pick_video_frame(video, frame_indices)-- if apply_gradient_penalty:- real = real.requires_grad_()-- fake = pick_video_frame(recon_video, frame_indices)-- real_logits = self.discr(real)- fake_logits = self.discr(fake.detach())-- discr_loss = hinge_discr_loss(fake_logits, real_logits)-- # multiscale discriminators-- multiscale_discr_losses = []-- if self.has_multiscale_discrs:- for discr in self.multiscale_discrs:- multiscale_real_logits = discr(video)- multiscale_fake_logits = discr(recon_video.detach())-- multiscale_discr_loss = hinge_discr_loss(multiscale_fake_logits, multiscale_real_logits)-- multiscale_discr_losses.append(multiscale_discr_loss)- else:- multiscale_discr_losses.append(self.zero)-- # gradient penalty-- if apply_gradient_penalty:- gradient_penalty_loss = gradient_penalty(real, real_logits)- else:- gradient_penalty_loss = self.zero-- # total loss-- total_loss = discr_loss + \- gradient_penalty_loss * self.grad_penalty_loss_weight + \- sum(multiscale_discr_losses) * self.multiscale_adversarial_loss_weight-- discr_loss_breakdown = DiscrLossBreakdown(- discr_loss,- multiscale_discr_losses,- gradient_penalty_loss- )-- return total_loss, discr_loss_breakdown-- # perceptual loss-- if self.use_vgg:-- frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices-- input_vgg_input = pick_video_frame(video, frame_indices)- recon_vgg_input = pick_video_frame(recon_video, frame_indices)-- if channels == 1:- input_vgg_input = repeat(input_vgg_input, 'b 1 h w -> b c h w', c = 3)- recon_vgg_input = repeat(recon_vgg_input, 'b 1 h w -> b c h w', c = 3)-- elif channels == 4:- input_vgg_input = input_vgg_input[:, :3]- recon_vgg_input = recon_vgg_input[:, :3]-- input_vgg_feats = self.vgg(input_vgg_input)- recon_vgg_feats = self.vgg(recon_vgg_input)-- perceptual_loss = F.mse_loss(input_vgg_feats, recon_vgg_feats)- else:- perceptual_loss = self.zero-- # get gradient with respect to perceptual loss for last decoder layer- # needed for adaptive weighting-- last_dec_layer = self.conv_out.conv.weight-- norm_grad_wrt_perceptual_loss = None-- if self.training and self.use_vgg and (self.has_gan or self.has_multiscale_discrs):- norm_grad_wrt_perceptual_loss = grad_layer_wrt_loss(perceptual_loss, last_dec_layer).norm(p = 2)-- # per-frame image discriminator-- recon_video_frames = None-- if self.has_gan:- frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices- recon_video_frames = pick_video_frame(recon_video, frame_indices)-- fake_logits = self.discr(recon_video_frames)- gen_loss = hinge_gen_loss(fake_logits)-- adaptive_weight = 1.-- if exists(norm_grad_wrt_perceptual_loss):- norm_grad_wrt_gen_loss = grad_layer_wrt_loss(gen_loss, last_dec_layer).norm(p = 2)- adaptive_weight = norm_grad_wrt_perceptual_loss / norm_grad_wrt_gen_loss.clamp(min = 1e-3)- adaptive_weight.clamp_(max = 1e3)-- if torch.isnan(adaptive_weight).any():- adaptive_weight = 1.- else:- gen_loss = self.zero- adaptive_weight = 0.-- # multiscale discriminator losses-- multiscale_gen_losses = []- multiscale_gen_adaptive_weights = []-- if self.has_multiscale_gan and self.has_multiscale_discrs:- if not exists(recon_video_frames):- recon_video_frames = pick_video_frame(recon_video, frame_indices)-- for discr in self.multiscale_discrs:- fake_logits = recon_video_frames- multiscale_gen_loss = hinge_gen_loss(fake_logits)-- multiscale_gen_losses.append(multiscale_gen_loss)-- multiscale_adaptive_weight = 1.-- if exists(norm_grad_wrt_perceptual_loss):- norm_grad_wrt_gen_loss = grad_layer_wrt_loss(multiscale_gen_loss, last_dec_layer).norm(p = 2)- multiscale_adaptive_weight = norm_grad_wrt_perceptual_loss / norm_grad_wrt_gen_loss.clamp(min = 1e-5)- multiscale_adaptive_weight.clamp_(max = 1e3)-- multiscale_gen_adaptive_weights.append(multiscale_adaptive_weight)-- # calculate total loss-- total_loss = recon_loss \- + aux_losses * self.quantizer_aux_loss_weight \- + perceptual_loss * self.perceptual_loss_weight \- + gen_loss * adaptive_weight * adversarial_loss_weight-- if self.has_multiscale_discrs:-- weighted_multiscale_gen_losses = sum(loss * weight for loss, weight in zip(multiscale_gen_losses, multiscale_gen_adaptive_weights))-- total_loss = total_loss + weighted_multiscale_gen_losses * multiscale_adversarial_loss_weight-- # loss breakdown-- loss_breakdown = LossBreakdown(- recon_loss,- aux_losses,- quantizer_loss_breakdown,- perceptual_loss,- gen_loss,- adaptive_weight,- multiscale_gen_losses,- multiscale_gen_adaptive_weights- )-- return total_loss, loss_breakdown--# main class-class MagViT2(Module):- def __init__(self):- super().__init__()-- def forward(self, x):- return xdiff --git a/opensora/models/vae/model_utils.py b/opensora/models/vae/model_utils.pydeleted file mode 100644index c00f886..0000000--- a/opensora/models/vae/model_utils.py+++ /dev/null@@ -1,232 +0,0 @@-import numpy as np-import torch-import torch.nn as nn--# from taming.modules.losses.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers-# from taming.modules.discriminator.model import NLayerDiscriminator, weights_init-from einops import rearrange--"""Stripped version of https://github.com/richzhang/PerceptualSimilarity/tree/master/models"""---## NOTE: not used since we only have 'GN'-# def get_norm_layer(norm_type, dtype):-# if norm_type == 'LN':-# # supply a few args with partial function and pass the rest of the args when this norm_fn is called-# norm_fn = functools.partial(nn.LayerNorm, dtype=dtype)-# elif norm_type == 'GN': #-# norm_fn = functools.partial(nn.GroupNorm, dtype=dtype)-# elif norm_type is None:-# norm_fn = lambda: (lambda x: x)-# else:-# raise NotImplementedError(f'norm_type: {norm_type}')-# return norm_fn---class DiagonalGaussianDistribution(object):- def __init__(- self,- parameters,- deterministic=False,- ):- self.parameters = parameters- self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) # SCH: channels dim- self.logvar = torch.clamp(self.logvar, -30.0, 20.0)- self.deterministic = deterministic- self.std = torch.exp(0.5 * self.logvar)- self.var = torch.exp(self.logvar)- if self.deterministic:- self.var = self.std = torch.zeros_like(self.mean).to(device=self.parameters.device, dtype=self.mean.dtype)-- def sample(self):- # torch.randn: standard normal distribution- x = self.mean + self.std * torch.randn(self.mean.shape).to(device=self.parameters.device, dtype=self.mean.dtype)- return x-- def kl(self, other=None):- if self.deterministic:- return torch.Tensor([0.0])- else:- if other is None: # SCH: assumes other is a standard normal distribution- return 0.5 * torch.sum(torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar, dim=[1, 2, 3, 4])- else:- return 0.5 * torch.sum(- torch.pow(self.mean - other.mean, 2) / other.var- + self.var / other.var- - 1.0- - self.logvar- + other.logvar,- dim=[1, 2, 3, 4],- )-- def nll(self, sample, dims=[1, 2, 3, 4]): # TODO: what does this do?- if self.deterministic:- return torch.Tensor([0.0])- logtwopi = np.log(2.0 * np.pi)- return 0.5 * torch.sum(logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var, dim=dims)-- def mode(self): # SCH: used for vae inference?- return self.mean---class VEA3DLoss(nn.Module):- def __init__(- self,- # disc_start,- logvar_init=0.0,- kl_weight=1.0,- pixelloss_weight=1.0,- perceptual_weight=0.1,- disc_loss="hinge",- ):- super().__init__()- assert disc_loss in ["hinge", "vanilla"]- self.kl_weight = kl_weight- self.pixel_weight = pixelloss_weight- self.perceptual_loss = LPIPS().eval()- self.perceptual_weight = perceptual_weight- # output log variance- self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)-- def forward(- self,- inputs,- reconstructions,- posteriors,- # optimizer_idx,- # global_step,- weights=None,- ):- rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous())- if self.perceptual_weight > 0: # NOTE: need in_channels == 3 in order to use!- assert inputs.size(1) == 3, f"using vgg16 that requires 3 input channels but got {inputs.size(1)}"- # SCH: transform to [(B,T), C, H, W] shape for percetual loss over each frame- B = inputs.shape[0]- inputs = rearrange(inputs, "B C T H W -> (B T) C H W")- reconstructions = rearrange(reconstructions, "B C T H W -> (B T) C H W")- # permutated_input = torch.permute(inputs, (0, 2, 1, 3, 4)) # [B, C, T, H, W] --> [B, T, C, H, W]- # permutated_rec = torch.permute(reconstructions, (0, 2, 1, 3, 4))- # data_shape = permutated_input.size()- # p_loss = self.perceptual_loss(- # permutated_input.reshape(-1, data_shape[-3], data_shape[-2],data_shape[-1]).contiguous(),- # permutated_rec.reshape(-1, data_shape[-3], data_shape[-2],data_shape[-1]).contiguous()- # )- p_loss = self.perceptual_loss(inputs.contiguous(), reconstructions.contiguous())- # SCH: shape back p_loss- # permuted_p_loss = torch.permute(p_loss.reshape(data_shape[0], data_shape[1], 1, 1, 1), (0,2,1,3,4))- # rec_loss = rec_loss + self.perceptual_weight * permuted_p_loss- p_loss = rearrange(p_loss, "(B T) C H W -> B C T H W", B=B)- rec_loss = rec_loss + self.perceptual_weight * p_loss-- nll_loss = rec_loss / torch.exp(self.logvar) + self.logvar- weighted_nll_loss = nll_loss- if weights is not None:- weighted_nll_loss = weights * nll_loss- weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]- nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]- kl_loss = posteriors.kl()- kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]-- loss = weighted_nll_loss + self.kl_weight * kl_loss # TODO: add discriminator loss later-- return loss---class VEA3DLossWithDiscriminator(nn.Module):- def __init__(- self,- # disc_start,- logvar_init=0.0,- kl_weight=1.0,- pixelloss_weight=1.0,- disc_num_layers=3,- disc_in_channels=3,- disc_factor=1.0,- disc_weight=1.0,- perceptual_weight=1.0,- use_actnorm=False,- disc_conditional=False,- disc_loss="hinge",- ):- super().__init__()- assert disc_loss in ["hinge", "vanilla"]- self.kl_weight = kl_weight- self.pixel_weight = pixelloss_weight- self.perceptual_loss = LPIPS().eval()- self.perceptual_weight = perceptual_weight- # output log variance- self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)-- # self.discriminator = NLayerDiscriminator(input_nc=disc_in_channels,- # n_layers=disc_num_layers,- # use_actnorm=use_actnorm- # ).apply(weights_init)- # self.discriminator_iter_start = disc_start- # self.disc_loss = hinge_d_loss if disc_loss == "hinge" else vanilla_d_loss- # self.disc_factor = disc_factor- # self.discriminator_weight = disc_weight- # self.disc_conditional = disc_conditional-- # TODO: for discriminator- # def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer=None):- # if last_layer is not None:- # nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0]- # g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0]- # else:- # nll_grads = torch.autograd.grad(nll_loss, self.last_layer[0], retain_graph=True)[0]- # g_grads = torch.autograd.grad(g_loss, self.last_layer[0], retain_graph=True)[0]-- # d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)- # d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()- # d_weight = d_weight * self.discriminator_weight- # return d_weight-- def forward(- self,- inputs,- reconstructions,- posteriors,- # optimizer_idx,- # global_step,- last_layer=None,- cond=None,- split="train",- weights=None,- ):- rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous())- if self.perceptual_weight > 0: # NOTE: need in_channels == 3 in order to use!- assert inputs.size(1) == 3, f"using vgg16 that requires 3 input channels but got {inputs.size(1)} "- # SCH: transform to [(B,T), C, H, W] shape for percetual loss over each frame- permutated_input = torch.permute(inputs, (0, 2, 1, 3, 4)) # [B, C, T, H, W] --> [B, T, C, H, W]- permutated_rec = torch.permute(reconstructions, (0, 2, 1, 3, 4))- data_shape = permutated_input.size()- p_loss = self.perceptual_loss(- permutated_input.reshape(-1, data_shape[-3], data_shape[-2], data_shape[-1]).contiguous(),- permutated_rec.reshape(-1, data_shape[-3], data_shape[-2], data_shape[-1]).contiguous(),- )- # SCH: shape back p_loss- permuted_p_loss = torch.permute(p_loss.reshape(data_shape[0], data_shape[1], 1, 1, 1), (0, 2, 1, 3, 4))- rec_loss = rec_loss + self.perceptual_weight * permuted_p_loss-- nll_loss = rec_loss / torch.exp(self.logvar) + self.logvar- weighted_nll_loss = nll_loss- if weights is not None:- weighted_nll_loss = weights * nll_loss- weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]- nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]- kl_loss = posteriors.kl()- kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]-- loss = weighted_nll_loss + self.kl_weight * kl_loss # TODO: add discriminator loss later-- # log = {"{}/total_loss".format(split): loss.clone().detach().mean(),- # "{}/logvar".format(split): self.logvar.detach(),- # "{}/kl_loss".format(split): kl_loss.detach().mean(),- # "{}/nll_loss".format(split): nll_loss.detach().mean(),- # "{}/rec_loss".format(split): rec_loss.detach().mean(),- # # "{}/d_weight".format(split): d_weight.detach(),- # # "{}/disc_factor".format(split): torch.tensor(disc_factor),- # # "{}/g_loss".format(split): g_loss.detach().mean(),- # }-- return lossdiff --git a/opensora/models/vae/utils.py b/opensora/models/vae/utils.pynew file mode 100644index 0000000..fd57391--- /dev/null+++ b/opensora/models/vae/utils.py@@ -0,0 +1,70 @@+import numpy as np+import torch+import torch.nn as nn++# from taming.modules.losses.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers+# from taming.modules.discriminator.model import NLayerDiscriminator, weights_init+from einops import rearrange++"""Stripped version of https://github.com/richzhang/PerceptualSimilarity/tree/master/models"""+++## NOTE: not used since we only have 'GN'+# def get_norm_layer(norm_type, dtype):+# if norm_type == 'LN':+# # supply a few args with partial function and pass the rest of the args when this norm_fn is called+# norm_fn = functools.partial(nn.LayerNorm, dtype=dtype)+# elif norm_type == 'GN': #+# norm_fn = functools.partial(nn.GroupNorm, dtype=dtype)+# elif norm_type is None:+# norm_fn = lambda: (lambda x: x)+# else:+# raise NotImplementedError(f'norm_type: {norm_type}')+# return norm_fn+++class DiagonalGaussianDistribution(object):+ def __init__(+ self,+ parameters,+ deterministic=False,+ ):+ self.parameters = parameters+ self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) # SCH: channels dim+ self.logvar = torch.clamp(self.logvar, -30.0, 20.0)+ self.deterministic = deterministic+ self.std = torch.exp(0.5 * self.logvar)+ self.var = torch.exp(self.logvar)+ if self.deterministic:+ self.var = self.std = torch.zeros_like(self.mean).to(device=self.parameters.device, dtype=self.mean.dtype)++ def sample(self):+ # torch.randn: standard normal distribution+ x = self.mean + self.std * torch.randn(self.mean.shape).to(device=self.parameters.device, dtype=self.mean.dtype)+ return x++ def kl(self, other=None):+ if self.deterministic:+ return torch.Tensor([0.0])+ else:+ if other is None: # SCH: assumes other is a standard normal distribution+ return 0.5 * torch.sum(torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar, dim=[1, 2, 3, 4])+ else:+ return 0.5 * torch.sum(+ torch.pow(self.mean - other.mean, 2) / other.var+ + self.var / other.var+ - 1.0+ - self.logvar+ + other.logvar,+ dim=[1, 2, 3, 4],+ )++ def nll(self, sample, dims=[1, 2, 3, 4]): # TODO: what does this do?+ if self.deterministic:+ return torch.Tensor([0.0])+ logtwopi = np.log(2.0 * np.pi)+ return 0.5 * torch.sum(logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var, dim=dims)++ def mode(self): # SCH: used for vae inference?+ return self.mean+diff --git a/opensora/models/vae/vae_3d.py b/opensora/models/vae/vae_3d.pyindex 1907fb4..5b74ed0 100644--- a/opensora/models/vae/vae_3d.py+++ b/opensora/models/vae/vae_3d.py@@ -1,13 +1,20 @@import functools-from typing import Any, Dict, Tuple, Type, Union, Sequence, Optional-from absl import logging+import math+from typing import Tuple, Union++import numpy as npimport torch-import torch.nn as nn-import numpy as np-from numpy import typing as nptyping-from opensora.models.vae import model_utils+import torch.nn as nn+import torch.nn.functional as F+from einops import pack, rearrange, repeat, unpack++from .utils import DiagonalGaussianDistribution+from .lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformersfrom opensora.registry import MODELS-from opensora.utils.ckpt_utils import load_checkpoint+from opensora.utils.ckpt_utils import find_model, load_checkpoint++# from diffusers.models.modeling_utils import ModelMixin+"""Encoder and Decoder stuctures with 3D CNNs."""@@ -19,76 +26,580 @@ NOTE:!!! opensora read video into [B,C,T,H,W] format output-TODO:- check data dimensions format"""++def cast_tuple(t, length=1):+ return t if isinstance(t, tuple) else ((t,) * length)+++def divisible_by(num, den):+ return (num % den) == 0+++def is_odd(n):+ return not divisible_by(n, 2)+++def pad_at_dim(t, pad, dim=-1, value=0.0):+ dims_from_right = (-dim - 1) if dim < 0 else (t.ndim - dim - 1)+ zeros = (0, 0) * dims_from_right+ return F.pad(t, (*zeros, *pad), value=value)+++def pick_video_frame(video, frame_indices):+ """get frame_indices from the video of [B, C, T, H, W] and return images of [B, C, H, W]"""+ batch, device = video.shape[0], video.device+ video = rearrange(video, "b c f ... -> b f c ...")+ batch_indices = torch.arange(batch, device=device)+ batch_indices = rearrange(batch_indices, "b -> b 1")+ images = video[batch_indices, frame_indices]+ images = rearrange(images, "b 1 c ... -> b c ...")+ return images+++def exists(v):+ return v is not None+++# ============== Generator Adversarial Loss Functions ==============+def lecam_reg(real_pred, fake_pred, ema_real_pred, ema_fake_pred):+ assert real_pred.ndim == 0 and ema_fake_pred.ndim == 0+ lecam_loss = torch.mean(torch.pow(nn.ReLU()(real_pred - ema_fake_pred), 2))+ lecam_loss += torch.mean(torch.pow(nn.ReLU()(ema_real_pred - fake_pred), 2))+ return lecam_loss+++# Open-Sora-Plan+# Very bad, do not use+def r1_penalty(real_img, real_pred):+ """R1 regularization for discriminator. The core idea is to+ penalize the gradient on real data alone: when the+ generator distribution produces the true data distribution+ and the discriminator is equal to 0 on the data manifold, the+ gradient penalty ensures that the discriminator cannot create+ a non-zero gradient orthogonal to the data manifold without+ suffering a loss in the GAN game.++ Ref:+ Eq. 9 in Which training methods for GANs do actually converge.+ """+ grad_real = torch.autograd.grad(outputs=real_pred.sum(), inputs=real_img, create_graph=True)[0]+ grad_penalty = grad_real.pow(2).view(grad_real.shape[0], -1).sum(1).mean()+ return grad_penalty+++# Open-Sora-Plan+# Implementation as described by https://arxiv.org/abs/1704.00028 # TODO: checkout the codes+def gradient_penalty_loss(discriminator, real_data, fake_data, weight=None):+ """Calculate gradient penalty for wgan-gp.++ Args:+ discriminator (nn.Module): Network for the discriminator.+ real_data (Tensor): Real input data.+ fake_data (Tensor): Fake input data.+ weight (Tensor): Weight tensor. Default: None.++ Returns:+ Tensor: A tensor for gradient penalty.+ """++ batch_size = real_data.size(0)+ alpha = real_data.new_tensor(torch.rand(batch_size, 1, 1, 1))++ # interpolate between real_data and fake_data+ interpolates = alpha * real_data + (1.0 - alpha) * fake_data+ interpolates = torch.autograd.Variable(interpolates, requires_grad=True)++ disc_interpolates = discriminator(interpolates)+ gradients = torch.autograd.grad(+ outputs=disc_interpolates,+ inputs=interpolates,+ grad_outputs=torch.ones_like(disc_interpolates),+ create_graph=True,+ retain_graph=True,+ only_inputs=True,+ )[0]++ if weight is not None:+ gradients = gradients * weight++ gradients_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean()+ if weight is not None:+ gradients_penalty /= torch.mean(weight)++ return gradients_penalty+++def gradient_penalty_fn(images, output):+ # batch_size = images.shape[0]+ gradients = torch.autograd.grad(+ outputs=output,+ inputs=images,+ grad_outputs=torch.ones(output.size(), device=images.device),+ create_graph=True,+ retain_graph=True,+ only_inputs=True,+ )[0]++ gradients = rearrange(gradients, "b ... -> b (...)")+ return ((gradients.norm(2, dim=1) - 1) ** 2).mean()+++# ============== Discriminator Adversarial Loss Functions ==============+def hinge_d_loss(logits_real, logits_fake):+ loss_real = torch.mean(F.relu(1.0 - logits_real))+ loss_fake = torch.mean(F.relu(1.0 + logits_fake))+ d_loss = 0.5 * (loss_real + loss_fake)+ return d_loss+++def vanilla_d_loss(logits_real, logits_fake):+ d_loss = 0.5 * (+ torch.mean(torch.nn.functional.softplus(-logits_real)) + torch.mean(torch.nn.functional.softplus(logits_fake))+ )+ return d_loss+++# from MAGVIT, used in place hof hinge_d_loss+def sigmoid_cross_entropy_with_logits(labels, logits):+ # The final formulation is: max(x, 0) - x * z + log(1 + exp(-abs(x)))+ zeros = torch.zeros_like(logits, dtype=logits.dtype)+ condition = logits >= zeros+ relu_logits = torch.where(condition, logits, zeros)+ neg_abs_logits = torch.where(condition, -logits, logits)+ return relu_logits - logits * labels + torch.log1p(torch.exp(neg_abs_logits))+++def xavier_uniform_weight_init(m):+ if isinstance(m, nn.Conv3d) or isinstance(m, nn.Linear):+ nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain("relu"))+ if m.bias is not None:+ nn.init.zeros_(m.bias)+ # print("initialized module to xavier_uniform:", m)+++def Sequential(*modules):+ modules = [*filter(exists, modules)]++ if len(modules) == 0:+ return nn.Identity()++ return nn.Sequential(*modules)+++def SameConv2d(dim_in, dim_out, kernel_size):+ kernel_size = cast_tuple(kernel_size, 2)+ padding = [k // 2 for k in kernel_size]+ return nn.Conv2d(dim_in, dim_out, kernel_size=kernel_size, padding=padding)+++def adopt_weight(weight, global_step, threshold=0, value=0.0):+ if global_step < threshold:+ weight = value+ return weight+++class CausalConv3d(nn.Module):+ def __init__(+ self,+ chan_in,+ chan_out,+ kernel_size: Union[int, Tuple[int, int, int]],+ pad_mode="constant",+ strides=None, # allow custom stride+ **kwargs,+ ):+ super().__init__()+ kernel_size = cast_tuple(kernel_size, 3)++ time_kernel_size, height_kernel_size, width_kernel_size = kernel_size++ assert is_odd(height_kernel_size) and is_odd(width_kernel_size)++ dilation = kwargs.pop("dilation", 1)+ stride = strides[0] if strides is not None else kwargs.pop("stride", 1)++ self.pad_mode = pad_mode+ time_pad = dilation * (time_kernel_size - 1) + (1 - stride)+ height_pad = height_kernel_size // 2+ width_pad = width_kernel_size // 2++ self.time_pad = time_pad+ self.time_causal_padding = (width_pad, width_pad, height_pad, height_pad, time_pad, 0)++ stride = strides if strides is not None else (stride, 1, 1)+ # padding = kwargs.pop('padding', 0)+ # if padding == "same" and not all([pad == 1 for pad in padding]):+ # padding = "valid"+ dilation = (dilation, 1, 1)+ self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)++ def forward(self, x):+ pad_mode = self.pad_mode if self.time_pad < x.shape[2] else "constant"++ x = F.pad(x, self.time_causal_padding, mode=pad_mode)+ return self.conv(x)++class ResBlock(nn.Module):def __init__(- self,- in_out_channels, # SCH: added- filters,- # norm_fn, # SCH: removed, use GN- conv_fn,- activation_fn=nn.ReLU,- use_conv_shortcut=False,- num_groups=32,- device="cpu",- dtype=torch.bfloat16,+ self,+ in_channels, # SCH: added+ filters,+ conv_fn,+ activation_fn=nn.SiLU,+ use_conv_shortcut=False,+ num_groups=32,+ device="cpu",+ dtype=torch.bfloat16,):super().__init__()+ self.in_channels = in_channelsself.filters = filtersself.activate = activation_fn()self.use_conv_shortcut = use_conv_shortcut-+# SCH: MAGVIT uses GroupNorm by default- self.norm1 = nn.GroupNorm(num_groups, in_out_channels, device=device, dtype=dtype)- self.conv1 = conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)+ self.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)+ self.conv1 = conv_fn(in_channels, self.filters, kernel_size=(3, 3, 3), bias=False)self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)self.conv2 = conv_fn(self.filters, self.filters, kernel_size=(3, 3, 3), bias=False)- if self.use_conv_shortcut:- self.conv3 = conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)- else:- self.conv3 = conv_fn(in_out_channels, self.filters, kernel_size=(1, 1, 1), bias=False)-+ if in_channels != filters:+ if self.use_conv_shortcut:+ self.conv3 = conv_fn(in_channels, self.filters, kernel_size=(3, 3, 3), bias=False)+ else:+ self.conv3 = conv_fn(in_channels, self.filters, kernel_size=(1, 1, 1), bias=False)def forward(self, x):- device, dtype = x.device, x.dtype- input_dim = x.shape[1]+ # device, dtype = x.device, x.dtype+ # input_dim = x.shape[1]residual = x- x = self.norm1.to(device,dtype)(x)+ x = self.norm1(x)x = self.activate(x)- x = self.conv1.to(device,dtype)(x)- x = self.norm2.to(device, dtype)(x)+ x = self.conv1(x)+ x = self.norm2(x)x = self.activate(x)- x = self.conv2.to(device, dtype)(x)- if input_dim != self.filters: # TODO: what does it do here- residual = self.conv3.to(device, dtype)(residual)- return x + residual--def _get_selected_flags(total_len: int, select_len: int, suffix: bool):- assert select_len <= total_len- selected = np.zeros(total_len, dtype=bool)- if not suffix:- selected[:select_len] = True+ x = self.conv2(x)+ if self.in_channels != self.filters: # SCH: ResBlock X->Y+ residual = self.conv3(residual)+ return x + residual+++# SCH: own implementation modified on top of: discriminator with anti-aliased downsampling (blurpool Zhang et al.)+class BlurPool3D(nn.Module):+ def __init__(+ self,+ channels,+ pad_type="reflect",+ filt_size=3,+ stride=2,+ pad_off=0,+ device="cpu",+ dtype=torch.bfloat16,+ ):+ super(BlurPool3D, self).__init__()+ self.filt_size = filt_size+ self.pad_off = pad_off+ self.pad_sizes = [+ int(1.0 * (filt_size - 1) / 2),+ int(np.ceil(1.0 * (filt_size - 1) / 2)),+ int(1.0 * (filt_size - 1) / 2),+ int(np.ceil(1.0 * (filt_size - 1) / 2)),+ int(1.0 * (filt_size - 1) / 2),+ int(np.ceil(1.0 * (filt_size - 1) / 2)),+ ]+ self.pad_sizes = [pad_size + pad_off for pad_size in self.pad_sizes]+ self.stride = stride+ self.off = int((self.stride - 1) / 2.0)+ self.channels = channels++ if self.filt_size == 1:+ a = np.array(+ [+ 1.0,+ ]+ )+ elif self.filt_size == 2:+ a = np.array([1.0, 1.0])+ elif self.filt_size == 3:+ a = np.array([1.0, 2.0, 1.0])+ elif self.filt_size == 4:+ a = np.array([1.0, 3.0, 3.0, 1.0])+ elif self.filt_size == 5:+ a = np.array([1.0, 4.0, 6.0, 4.0, 1.0])+ elif self.filt_size == 6:+ a = np.array([1.0, 5.0, 10.0, 10.0, 5.0, 1.0])+ elif self.filt_size == 7:+ a = np.array([1.0, 6.0, 15.0, 20.0, 15.0, 6.0, 1.0])++ filt_2d = a[:, None] * a[None, :]+ filt_3d = torch.Tensor(a[:, None, None] * filt_2d[None, :, :]).to(device, dtype)++ filt = filt_3d / torch.sum(filt_3d) # SCH: modified to it 3D+ self.register_buffer("filt", filt[None, None, :, :, :].repeat((self.channels, 1, 1, 1, 1)))++ self.pad = get_pad_layer(pad_type)(self.pad_sizes)++ def forward(self, inp):+ if self.filt_size == 1:+ if self.pad_off == 0:+ return inp[:, :, :: self.stride, :: self.stride]+ else:+ return self.pad(inp)[:, :, :: self.stride, :: self.stride]+ else:+ return F.conv3d(self.pad(inp), self.filt, stride=self.stride, groups=inp.shape[1])+++def get_pad_layer(pad_type):+ if pad_type in ["refl", "reflect"]:+ PadLayer = nn.ReflectionPad3d+ elif pad_type in ["repl", "replicate"]:+ PadLayer = nn.ReplicationPad3d+ elif pad_type == "zero":+ PadLayer = nn.ZeroPad3delse:- selected[-select_len:] = True- return selected+ print("Pad type [%s] not recognized" % pad_type)+ return PadLayer+++class ResBlockDown(nn.Module):+ """3D StyleGAN ResBlock for D."""++ def __init__(+ self,+ in_channels,+ filters,+ activation_fn,+ num_groups=32,+ device="cpu",+ dtype=torch.bfloat16,+ ):+ super().__init__()++ self.filters = filters+ self.activation_fn = activation_fn++ # SCH: NOTE: although paper says conv (X->Y, Y->Y), original code implementation is (X->X, X->Y), we follow code+ self.conv1 = nn.Conv3d(+ in_channels, in_channels, (3, 3, 3), padding=1, device=device, dtype=dtype+ ) # NOTE: init to xavier_uniform+ self.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)++ self.blur = BlurPool3D(in_channels, device=device, dtype=dtype)++ self.conv2 = nn.Conv3d(+ in_channels, self.filters, (1, 1, 1), bias=False, device=device, dtype=dtype+ ) # NOTE: init to xavier_uniform+ self.conv3 = nn.Conv3d(+ in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype+ ) # NOTE: init to xavier_uniform+ self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)++ # self.apply(xavier_uniform_weight_init)++ def forward(self, x):+ residual = x+ x = self.conv1(x)+ x = self.norm1(x)+ x = self.activation_fn(x)++ residual = self.blur(residual)+ residual = self.conv2(residual)++ x = self.blur(x)+ x = self.conv3(x)+ x = self.norm2(x)+ x = self.activation_fn(x)+ out = (residual + x) / math.sqrt(2)+ return out+++# SCH: taken from Open Sora Plan+def n_layer_disc_weights_init(m):+ classname = m.__class__.__name__+ if classname.find("Conv") != -1:+ nn.init.normal_(m.weight.data, 0.0, 0.02)+ elif classname.find("BatchNorm") != -1:+ nn.init.normal_(m.weight.data, 1.0, 0.02)+ nn.init.constant_(m.bias.data, 0)+++class NLayerDiscriminator3D(nn.Module):+ """Defines a 3D PatchGAN discriminator as in Pix2Pix but for 3D inputs."""++ def __init__(self, input_nc=1, ndf=64, n_layers=3, use_actnorm=False):+ """+ Construct a 3D PatchGAN discriminator++ Parameters:+ input_nc (int) -- the number of channels in input volumes+ ndf (int) -- the number of filters in the last conv layer+ n_layers (int) -- the number of conv layers in the discriminator+ use_actnorm (bool) -- flag to use actnorm instead of batchnorm+ """+ super(NLayerDiscriminator3D, self).__init__()+ if not use_actnorm:+ norm_layer = nn.BatchNorm3d+ else:+ raise NotImplementedError("Not implemented.")+ if type(norm_layer) == functools.partial:+ use_bias = norm_layer.func != nn.BatchNorm3d+ else:+ use_bias = norm_layer != nn.BatchNorm3d++ kw = 4+ padw = 1+ sequence = [nn.Conv3d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]+ nf_mult = 1+ nf_mult_prev = 1+ for n in range(1, n_layers): # gradually increase the number of filters+ nf_mult_prev = nf_mult+ nf_mult = min(2**n, 8)+ sequence += [+ nn.Conv3d(+ ndf * nf_mult_prev,+ ndf * nf_mult,+ kernel_size=(kw, kw, kw),+ stride=(1, 2, 2),+ padding=padw,+ bias=use_bias,+ ),+ norm_layer(ndf * nf_mult),+ nn.LeakyReLU(0.2, True),+ ]++ nf_mult_prev = nf_mult+ nf_mult = min(2**n_layers, 8)+ sequence += [+ nn.Conv3d(+ ndf * nf_mult_prev, ndf * nf_mult, kernel_size=(kw, kw, kw), stride=1, padding=padw, bias=use_bias+ ),+ norm_layer(ndf * nf_mult),+ nn.LeakyReLU(0.2, True),+ ]++ sequence += [+ nn.Conv3d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)+ ] # output 1 channel prediction map+ self.main = nn.Sequential(*sequence)++ def forward(self, input):+ """Standard forward."""+ return self.main(input)+++class StyleGANDiscriminatorBlur(nn.Module):+ """StyleGAN Discriminator."""++ """+ SCH: NOTE:+ this discriminator requries the num_frames to be fixed during training;+ in case we pre-train with image then train on video, this disciminator's Linear layer would have to be re-trained!+ """++ def __init__(+ self,+ image_size=(128, 128),+ num_frames=17,+ in_channels=3,+ filters=128,+ channel_multipliers=(2, 4, 4, 4, 4),+ num_groups=32,+ dtype=torch.bfloat16,+ device="cpu",+ ):+ super().__init__()++ self.dtype = dtype+ self.input_size = cast_tuple(image_size, 2)+ self.filters = filters+ self.activation_fn = nn.LeakyReLU(negative_slope=0.2)+ self.channel_multipliers = channel_multipliers++ self.conv1 = nn.Conv3d(+ in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype+ ) # NOTE: init to xavier_uniform++ prev_filters = self.filters # record in_channels+ self.num_blocks = len(self.channel_multipliers)+ self.res_block_list = nn.ModuleList([])+ for i in range(self.num_blocks):+ filters = self.filters * self.channel_multipliers[i]+ self.res_block_list.append(+ ResBlockDown(prev_filters, filters, self.activation_fn, device=device, dtype=dtype).apply(+ xavier_uniform_weight_init+ )+ )+ prev_filters = filters # update in_channels++ self.conv2 = nn.Conv3d(+ prev_filters, prev_filters, (3, 3, 3), padding=1, device=device, dtype=dtype+ ) # NOTE: init to xavier_uniform+ # torch.nn.init.xavier_uniform_(self.conv2.weight)++ self.norm1 = nn.GroupNorm(num_groups, prev_filters, dtype=dtype, device=device)++ scale_factor = 2**self.num_blocks+ if num_frames % scale_factor != 0: # SCH: NOTE: has first frame which would be padded before usage+ time_scaled = num_frames // scale_factor + 1+ else:+ time_scaled = num_frames / scale_factor++ assert (+ self.input_size[0] % scale_factor == 0+ ), f"image width {self.input_size[0]} is not divisible by scale factor {scale_factor}"+ assert (+ self.input_size[1] % scale_factor == 0+ ), f"image height {self.input_size[1]} is not divisible by scale factor {scale_factor}"+ w_scaled, h_scaled = self.input_size[0] / scale_factor, self.input_size[1] / scale_factor+ in_features = int(prev_filters * time_scaled * w_scaled * h_scaled) # (C*T*W*H)+ self.linear1 = nn.Linear(in_features, prev_filters, device=device, dtype=dtype) # NOTE: init to xavier_uniform+ self.linear2 = nn.Linear(prev_filters, 1, device=device, dtype=dtype) # NOTE: init to xavier_uniform++ # self.apply(xavier_uniform_weight_init)++ def forward(self, x):+ x = self.conv1(x)+ # print("discriminator aft conv:", x.size())+ x = self.activation_fn(x)++ for i in range(self.num_blocks):+ x = self.res_block_list[i](x)+ # print("discriminator resblock down:", x.size())++ x = self.conv2(x)+ # print("discriminator aft conv2:", x.size())+ x = self.norm1(x)+ x = self.activation_fn(x)+ x = x.reshape((x.shape[0], -1)) # SCH: [B, (C * T * W * H)] ?++ # print("discriminator reshape:", x.size())+ x = self.linear1(x)+ # print("discriminator aft linear1:", x.size())++ x = self.activation_fn(x)+ x = self.linear2(x)+ # print("discriminator aft linear2:", x.size())+ return xclass Encoder(nn.Module):"""Encoder Blocks."""- def __init__(self,- filters = 64,- num_res_blocks = 2,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (True, True, False),- num_groups = 32, # for nn.GroupNorm- in_out_channels = 3, # SCH: added, in_channels at the start- latent_embed_dim = 256, # num channels for latent vector- conv_downsample = False,- custom_conv_padding = None,- activation_fn = 'swish',++ def __init__(+ self,+ filters=128,+ num_res_blocks=4,+ channel_multipliers=(1, 2, 2, 4),+ temporal_downsample=(False, True, True),+ num_groups=32, # for nn.GroupNorm+ # in_out_channels = 3, # SCH: added, in_channels at the start+ latent_embed_dim=512, # num channels for latent vector+ # conv_downsample = False,+ disable_spatial_downsample=False, # for vae pipeline+ custom_conv_padding=None,+ activation_fn="swish",device="cpu",dtype=torch.bfloat16,):@@ -99,37 +610,27 @@ class Encoder(nn.Module):self.temporal_downsample = temporal_downsampleself.num_groups = num_groups- if isinstance(self.temporal_downsample, int):- self.temporal_downsample = _get_selected_flags(- len(self.channel_multipliers) - 1, self.temporal_downsample, False)-self.embedding_dim = latent_embed_dim- self.conv_downsample = conv_downsample+ self.disable_spatial_downsample = disable_spatial_downsample+ # self.conv_downsample = conv_downsampleself.custom_conv_padding = custom_conv_padding- # self.norm_type = self.config.vqvae.norm_type- # self.num_remat_block = self.config.vqvae.get('num_enc_remat_blocks', 0)- if activation_fn == 'relu':+ if activation_fn == "relu":self.activation_fn = nn.ReLU- elif activation_fn == 'swish':+ elif activation_fn == "swish":self.activation_fn = nn.SiLUelse:raise NotImplementedErrorself.activate = self.activation_fn()self.conv_fn = functools.partial(- nn.Conv3d,+ CausalConv3d,+ # padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorchdtype=dtype,- padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorch- # custom_padding=self.custom_conv_paddingdevice=device,)-- # self.norm_fn = model_utils.get_norm_layer(- # norm_type=self.norm_type, dtype=self.dtype)-+self.block_args = dict(- # norm_fn=self.norm_fn,conv_fn=self.conv_fn,dtype=dtype,activation_fn=self.activation_fn,@@ -137,92 +638,98 @@ class Encoder(nn.Module):num_groups=self.num_groups,device=device,)-- self.conv1 = self.conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)++ # NOTE: moved to VAE for separate first frame processing+ # self.conv1 = self.conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)# ResBlocks and conv downsample- self.block_res_blocks = []+ self.block_res_blocks = nn.ModuleList([])self.num_blocks = len(self.channel_multipliers)- self.conv_blocks = []+ self.conv_blocks = nn.ModuleList([])filters = self.filters- prev_filters = filters # record for in_channels+ prev_filters = filters # record for in_channelsfor i in range(self.num_blocks):# resblock handling- filters = self.filters * self.channel_multipliers[i] # SCH: determine the number out_channels- block_items = []+ filters = self.filters * self.channel_multipliers[i] # SCH: determine the number out_channels+ block_items = nn.ModuleList([])for _ in range(self.num_res_blocks):block_items.append(ResBlock(prev_filters, filters, **self.block_args))- prev_filters = filters # update in_channels+ prev_filters = filters # update in_channelsself.block_res_blocks.append(block_items)-- if i < self.num_blocks - 1:- # conv blocks handling- if self.conv_downsample:- t_stride = 2 if self.temporal_downsample[i] else 1- t_pad = 1 if self.temporal_downsample[i] else 0- self.conv_blocks.append(self.conv_fn(prev_filters, filters, kernel_size=(4, 4, 4), stride=(t_stride, 2, 2)), padding=(t_pad,1,1)) # SCH: should be same in_channel and out_channel- prev_filters = filters # update in_channels- # NOTE: downsample, dimensions T, H, W- self.avg_pool_with_t = nn.AvgPool3d((2,2,2), count_include_pad=False)- self.avg_pool = nn.AvgPool3d((1,2,2), count_include_pad=False)+ if i < self.num_blocks - 1: # SCH: T-Causal Conv 3x3x3, 128->128, stride t x stride s x stride s+ t_stride = 2 if self.temporal_downsample[i] else 1+ s_stride = 2 if not self.disable_spatial_downsample else 1+ self.conv_blocks.append(+ self.conv_fn(prev_filters, filters, kernel_size=(3, 3, 3), strides=(t_stride, s_stride, s_stride))+ ) # SCH: should be same in_channel and out_channel+ prev_filters = filters # update in_channels# last layer res block- self.res_blocks = []+ self.res_blocks = nn.ModuleList([])for _ in range(self.num_res_blocks):self.res_blocks.append(ResBlock(prev_filters, filters, **self.block_args))- prev_filters = filters # update in_channels+ prev_filters = filters # update in_channels# MAGVIT uses Group Normalization- self.norm1 = nn.GroupNorm(self.num_groups, prev_filters, dtype=dtype, device=device) # SCH: separate <prev_filters> channels into 32 groups+ self.norm1 = nn.GroupNorm(+ self.num_groups, prev_filters, dtype=dtype, device=device+ ) # SCH: separate <prev_filters> channels into 32 groups- self.conv2 = self.conv_fn(prev_filters, self.embedding_dim, kernel_size=(1, 1, 1))+ self.conv2 = nn.Conv3d(+ prev_filters, self.embedding_dim, kernel_size=(1, 1, 1), dtype=dtype, device=device, padding="same"+ )def forward(self, x):- dtype, device = x.dtype, x.device- x = self.conv1.to(device, dtype)(x)+ # dtype, device = x.dtype, x.device++ # NOTE: moved to VAE for separate first frame processing+ # x = self.conv1(x)++ # print("encoder:", x.size())+for i in range(self.num_blocks):for j in range(self.num_res_blocks):x = self.block_res_blocks[i][j](x)+ # print("encoder:", x.size())if i < self.num_blocks - 1:- if self.conv_downsample:- x = self.conv_blocks[i].to(device, dtype)(x)- else:- if self.temporal_downsample[i]:- x = self.avg_pool_with_t(x)- else:- x = self.avg_pool(x)+ x = self.conv_blocks[i](x)+ # print("encoder:", x.size())for i in range(self.num_res_blocks):x = self.res_blocks[i](x)+ # print("encoder:", x.size())- x = self.norm1.to(device, dtype)(x)+ x = self.norm1(x)x = self.activate(x)- x = self.conv2.to(device, dtype)(x)+ x = self.conv2(x)+ # print("encoder:", x.size())return x-class Decoder(nn.Module):"""Decoder Blocks."""- def __init__(self,- latent_embed_dim = 256,- filters = 64,- in_out_channels = 4,- num_res_blocks = 2,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (True, True, False),- num_groups = 32, # for nn.GroupNorm- upsample = "nearest+conv", # options: "deconv", "nearest+conv"- custom_conv_padding = None,- activation_fn = 'swish',++ def __init__(+ self,+ latent_embed_dim=512,+ filters=128,+ # in_out_channels = 4,+ num_res_blocks=4,+ channel_multipliers=(1, 2, 2, 4),+ temporal_downsample=(False, True, True),+ num_groups=32, # for nn.GroupNorm+ # upsample = "nearest+conv", # options: "deconv", "nearest+conv"+ disable_spatial_upsample=False, # for vae pipeline+ custom_conv_padding=None,+ activation_fn="swish",device="cpu",dtype=torch.bfloat16,):super().__init__()- self.output_dim = in_out_channels+ # self.output_dim = in_out_channelsself.embedding_dim = latent_embed_dimself.filters = filtersself.num_res_blocks = num_res_blocks@@ -230,38 +737,28 @@ class Decoder(nn.Module):self.temporal_downsample = temporal_downsampleself.num_groups = num_groups- if isinstance(self.temporal_downsample, int):- self.temporal_downsample = _get_selected_flags(- len(self.channel_multipliers) - 1, self.temporal_downsample, False)-- self.upsample = upsample+ # self.upsample = upsample+ self.s_stride = 1 if disable_spatial_upsample else 2 # spatial strideself.custom_conv_padding = custom_conv_padding# self.norm_type = self.config.vqvae.norm_type# self.num_remat_block = self.config.vqvae.get('num_dec_remat_blocks', 0)- if activation_fn == 'relu':+ if activation_fn == "relu":self.activation_fn = nn.ReLU- elif activation_fn == 'swish':- self.activation_fn = nn.SiLU+ elif activation_fn == "swish":+ self.activation_fn = nn.SiLUelse:- raise NotImplementedError+ raise NotImplementedErrorself.activate = self.activation_fn()self.conv_fn = functools.partial(- nn.Conv3d,+ CausalConv3d,dtype=dtype,- padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorch- # custom_padding=self.custom_conv_padding+ # padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorchdevice=device,)- self.conv_t_fn = functools.partial(nn.ConvTranspose3d, dtype=dtype)-- # self.norm_fn = model_utils.get_norm_layer(- # norm_type=self.norm_type, dtype=dtype)-self.block_args = dict(- # norm_fn=self.norm_fn,conv_fn=self.conv_fn,activation_fn=self.activation_fn,use_conv_shortcut=False,@@ -276,102 +773,104 @@ class Decoder(nn.Module):self.conv1 = self.conv_fn(self.embedding_dim, filters, kernel_size=(3, 3, 3), bias=True)# last layer res block- self.res_blocks = []+ self.res_blocks = nn.ModuleList([])for _ in range(self.num_res_blocks):self.res_blocks.append(ResBlock(filters, filters, **self.block_args))+ # TODO: do I need to add adaptive GroupNorm in between each block?- # NOTE: upsample, dimensions T, H, W- self.upsampler_with_t = nn.Upsample(scale_factor=(2,2,2))- self.upsampler = nn.Upsample(scale_factor=(1,2,2))+ # # NOTE: upsample, dimensions T, H, W+ # self.upsampler_with_t = nn.Upsample(scale_factor=(2,2,2))+ # self.upsampler = nn.Upsample(scale_factor=(1,2,2))# ResBlocks and conv upsample- prev_filters = filters # SCH: in_channels- self.block_res_blocks = []+ prev_filters = filters # SCH: in_channels+ self.block_res_blocks = nn.ModuleList([])self.num_blocks = len(self.channel_multipliers)- self.conv_blocks = []+ self.conv_blocks = nn.ModuleList([])# SCH: reverse to keep track of the in_channels, but append also in a reverse direction- for i in reversed(range(self.num_blocks)):+ for i in reversed(range(self.num_blocks)):filters = self.filters * self.channel_multipliers[i]# resblock handling- block_items = []+ block_items = nn.ModuleList([])for _ in range(self.num_res_blocks):block_items.append(ResBlock(prev_filters, filters, **self.block_args))- prev_filters = filters # SCH: update in_channels- self.block_res_blocks.insert(0, block_items) # SCH: append in front-- # conv blocks handling+ prev_filters = filters # SCH: update in_channels+ self.block_res_blocks.insert(0, block_items) # SCH: append in front++ # conv blocks with upsamplingif i > 0:t_stride = 2 if self.temporal_downsample[i - 1] else 1- t_kernel = 4 if self.temporal_downsample[i - 1] else 3 # SCH: hack to keep dimension same- if self.upsample == "deconv":- assert self.custom_conv_padding is None, ('Custom padding not implemented for ConvTranspose')- # SCH: append in front- self.conv_blocks.insert(0,- self.conv_t_fn(prev_filters, filters, kernel_size=(t_kernel, 4, 4), stride=(t_stride, 2, 2), padding=1))- prev_filters = filters # SCH: update in_channels- elif self.upsample == 'nearest+conv':- # SCH: append in front- self.conv_blocks.insert(0, self.conv_fn(prev_filters, filters, kernel_size=(3, 3, 3)))- prev_filters = filters # SCH: update in_channels- else:- raise NotImplementedError(f'Unknown upsampler: {self.upsample}')-+ # SCH: T-Causal Conv 3x3x3, f -> (t_stride * 2 * 2) * f, depth to space t_stride x 2 x 2+ self.conv_blocks.insert(+ 0,+ self.conv_fn(+ prev_filters, prev_filters * t_stride * self.s_stride * self.s_stride, kernel_size=(3, 3, 3)+ ),+ )+self.norm1 = nn.GroupNorm(self.num_groups, prev_filters, device=device, dtype=dtype)- self.conv2 = self.conv_fn(prev_filters, self.output_dim, kernel_size=(3, 3, 3))+ # NOTE: moved to VAE for separate first frame processing+ # self.conv2 = self.conv_fn(prev_filters, self.output_dim, kernel_size=(3, 3, 3))def forward(self,x,**kwargs,):- dtype, device = x.dtype, x.device- x = self.conv1.to(device, dtype)(x)+ # dtype, device = x.dtype, x.device++ x = self.conv1(x)+ # print("decoder:", x.size())for i in range(self.num_res_blocks):x = self.res_blocks[i](x)- for i in reversed(range(self.num_blocks)): # reverse here to make decoder symmetric with encoder+ # print("decoder:", x.size())+ for i in reversed(range(self.num_blocks)): # reverse here to make decoder symmetric with encoderfor j in range(self.num_res_blocks):x = self.block_res_blocks[i][j](x)-+ # print("decoder:", x.size())if i > 0:- if self.upsample == 'deconv':- assert self.custom_conv_padding is None, ('Custom padding not implemented for ConvTranspose')- x = self.conv_blocks[i-1].to(device, dtype)(x)- elif self.upsample == 'nearest+conv':- if self.temporal_downsample[i - 1]:- x = self.upsampler_with_t(x)- else:- x = self.upsampler(x)- x = self.conv_blocks[i-1].to(device, dtype)(x)- else:- raise NotImplementedError(f'Unknown upsampler: {self.upsample}')- x = self.norm1.to(device, dtype)(x)+ t_stride = 2 if self.temporal_downsample[i - 1] else 1+ # SCH: T-Causal Conv 3x3x3, f -> (t_stride * 2 * 2) * f, depth to space t_stride x 2 x 2+ x = self.conv_blocks[i - 1](x)+ x = rearrange(+ x,+ "B (C ts hs ws) T H W -> B C (T ts) (H hs) (W ws)",+ ts=t_stride,+ hs=self.s_stride,+ ws=self.s_stride,+ )+ # print("decoder:", x.size())++ x = self.norm1(x)x = self.activate(x)- x = self.conv2.to(device, dtype)(x)+ # NOTE: moved to VAE for separate first frame processing+ # x = self.conv2(x)return x-+@MODELS.register_module()-class VAE_3D(nn.Module):- """The 3D VAE """+class VAE_3D_V2(nn.Module): # , ModelMixin+ """The 3D VAE"""+def __init__(- self,- latent_embed_dim = 256,- filters = 64,- num_res_blocks = 2,- channel_multipliers = (1, 2, 2, 4),- temporal_downsample = (True, True, False),- num_groups = 32, # for nn.GroupNorm- conv_downsample = False,- upsample = "nearest+conv", # options: "deconv", "nearest+conv"- custom_conv_padding = None,- activation_fn = 'swish',- in_out_channels = 4,- kl_embed_dim = 64,+ self,+ latent_embed_dim=256,+ filters=128,+ num_res_blocks=2,+ separate_first_frame_encoding=False,+ channel_multipliers=(1, 2, 2, 4),+ temporal_downsample=(True, True, False),+ num_groups=32, # for nn.GroupNorm+ disable_space=False,+ custom_conv_padding=None,+ activation_fn="swish",+ in_out_channels=4,+ kl_embed_dim=64,+ encoder_double_z=True,device="cpu",dtype="bf16",- # precision: Any = jax.lax.Precision.DEFAULT):super().__init__()@@ -381,88 +880,469 @@ class VAE_3D(nn.Module):elif dtype == "fp16":dtype = torch.float16else:- raise NotImplementedError(f'dtype: {dtype}')+ raise NotImplementedError(f"dtype: {dtype}")- # Model Initialization+ # ==== Model Params ====+ # self.image_size = cast_tuple(image_size, 2)+ self.time_downsample_factor = 2 ** sum(temporal_downsample)+ self.time_padding = self.time_downsample_factor - 1+ self.separate_first_frame_encoding = separate_first_frame_encoding+ image_down = 2 ** len(temporal_downsample)+ t_down = 2 ** len([x for x in temporal_downsample if x == True])+ self.patch_size = (t_down, image_down, image_down)++ # ==== Model Initialization ====++ # encoder & decoder first and last conv layer+ # SCH: NOTE: following MAGVIT, conv in bias=False in encoder first conv+ self.conv_in = CausalConv3d(+ in_out_channels, filters, kernel_size=(3, 3, 3), bias=False, dtype=dtype, device=device+ )+ self.conv_in_first_frame = nn.Identity()+ self.conv_out_first_frame = nn.Identity()+ if separate_first_frame_encoding:+ self.conv_in_first_frame = SameConv2d(in_out_channels, filters, (3, 3))+ self.conv_out_first_frame = SameConv2d(filters, in_out_channels, (3, 3))+ self.conv_out = CausalConv3d(filters, in_out_channels, 3, dtype=dtype, device=device)self.encoder = Encoder(- filters=filters,- num_res_blocks=num_res_blocks,- channel_multipliers=channel_multipliers,+ filters=filters,+ num_res_blocks=num_res_blocks,+ channel_multipliers=channel_multipliers,temporal_downsample=temporal_downsample,- num_groups = num_groups, # for nn.GroupNorm- in_out_channels = in_out_channels,- latent_embed_dim = latent_embed_dim,- conv_downsample = conv_downsample,- custom_conv_padding = custom_conv_padding,- activation_fn = activation_fn,+ num_groups=num_groups, # for nn.GroupNorm+ # in_out_channels = in_out_channels,+ latent_embed_dim=latent_embed_dim * 2 if encoder_double_z else latent_embed_dim,+ # conv_downsample = conv_downsample,+ disable_spatial_downsample=disable_space,+ custom_conv_padding=custom_conv_padding,+ activation_fn=activation_fn,device=device,dtype=dtype,)self.decoder = Decoder(- latent_embed_dim = latent_embed_dim,- filters = filters,- in_out_channels = in_out_channels,- num_res_blocks = num_res_blocks,- channel_multipliers = channel_multipliers,- temporal_downsample = temporal_downsample,- num_groups = num_groups, # for nn.GroupNorm- upsample = upsample, # options: "deconv", "nearest+conv"- custom_conv_padding = custom_conv_padding,- activation_fn = activation_fn,+ latent_embed_dim=latent_embed_dim,+ filters=filters,+ # in_out_channels = in_out_channels,+ num_res_blocks=num_res_blocks,+ channel_multipliers=channel_multipliers,+ temporal_downsample=temporal_downsample,+ num_groups=num_groups, # for nn.GroupNorm+ # upsample = upsample, # options: "deconv", "nearest+conv"+ disable_spatial_upsample=disable_space,+ custom_conv_padding=custom_conv_padding,+ activation_fn=activation_fn,device=device,dtype=dtype,)- self.quant_conv = nn.Conv3d(latent_embed_dim, 2*kl_embed_dim, 1)- self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1)-- image_down = 2 ** len(temporal_downsample)- t_down = 2 ** len([x for x in temporal_downsample if x == True])- self.patch_size = (t_down, image_down, image_down)+ if encoder_double_z:+ self.quant_conv = nn.Conv3d(2 * latent_embed_dim, 2 * kl_embed_dim, 1, device=device, dtype=dtype)+ else:+ self.quant_conv = nn.Conv3d(latent_embed_dim, 2 * kl_embed_dim, 1, device=device, dtype=dtype)+ self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1, device=device, dtype=dtype)def get_latent_size(self, input_size):for i in range(len(input_size)):assert input_size[i] % self.patch_size[i] == 0, "Input size must be divisible by patch size"input_size = [input_size[i] // self.patch_size[i] for i in range(3)]return input_size-+def encode(self,- x,+ video,+ video_contains_first_frame=True,):- encoded_feature = self.encoder(x)- moments = self.quant_conv(encoded_feature).to(x.dtype)- posterior = model_utils.DiagonalGaussianDistribution(moments)+ encode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame++ # whether to pad video or not+ if video_contains_first_frame:+ video_len = video.shape[2]+ video = pad_at_dim(video, (self.time_padding, 0), value=0.0, dim=2)+ video_packed_shape = [torch.Size([self.time_padding]), torch.Size([]), torch.Size([video_len - 1])]++ # print("pre-encoder:", video.size())++ # NOTE: moved encoder conv1 here for separate first frame encoding+ if encode_first_frame_separately:+ pad, first_frame, video = unpack(video, video_packed_shape, "b c * h w")+ first_frame = self.conv_in_first_frame(first_frame)+ video = self.conv_in(video)++ # print("pre-encoder:", video.size())++ if encode_first_frame_separately:+ video, _ = pack([first_frame, video], "b c * h w")+ video = pad_at_dim(video, (self.time_padding, 0), dim=2)++ encoded_feature = self.encoder(video)++ # print("after encoder:", encoded_feature.size())++ # NOTE: TODO: do we include this before gaussian distri? or go directly to Gaussian distribution+ moments = self.quant_conv(encoded_feature).to(video.dtype)+ posterior = DiagonalGaussianDistribution(moments)++ # print("after encoder moments:", moments.size())+return posterior-+def decode(self,z,- ):- dtype = z.dtype- z = self.post_quant_conv(z).to(dtype)+ video_contains_first_frame=True,+ ):+ # dtype = z.dtype+ decode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame++ z = self.post_quant_conv(z)+ # print("pre decoder, post quant conv:", z.size())+dec = self.decoder(z)- return dec-+ # print("post decoder:", dec.size())++ # SCH: moved decoder last conv layer here for separate first frame decoding+ if decode_first_frame_separately:+ left_pad, dec_ff, dec = (+ dec[:, :, : self.time_padding],+ dec[:, :, self.time_padding],+ dec[:, :, (self.time_padding + 1) :],+ )+ out = self.conv_out(dec)+ outff = self.conv_out_first_frame(dec_ff)+ video, _ = pack([outff, out], "b c * h w")+ else:+ video = self.conv_out(dec)+ # if video were padded, remove padding+ if video_contains_first_frame:+ video = video[:, :, self.time_padding :]++ # print("conv out:", video.size())++ return video++ def get_last_layer(self):+ # CausalConv3d wraps the conv+ return self.conv_out.conv.weight+def forward(self,- input,+ video,sample_posterior=True,- ):- posterior = self.encode(input)+ video_contains_first_frame=True,+ # split = "train",+ ):+ batch, channels, frames = video.shape[:3]+ assert divisible_by(+ frames - int(video_contains_first_frame), self.time_downsample_factor+ ), f"number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}"++ posterior = self.encode(+ video,+ video_contains_first_frame=video_contains_first_frame,+ )+if sample_posterior:z = posterior.sample()else:z = posterior.mode()- dec = self.decode(z)- return dec, posterior-[email protected]_module("VAE_3D_B")-def VAE_3D_B(from_pretrained=None, **kwargs):- model = VAE_3D(**kwargs)+ recon_video = self.decode(z, video_contains_first_frame=video_contains_first_frame)++ return recon_video, posterior+++class VEALoss(nn.Module):+ def __init__(+ self,+ logvar_init=0.0,+ perceptual_loss_weight=0.1,+ kl_loss_weight=0.000001,+ # vgg=None,+ # vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,+ device="cpu",+ dtype="bf16",+ ):+ super().__init__()++ if type(dtype) == str:+ if dtype == "bf16":+ dtype = torch.bfloat16+ elif dtype == "fp16":+ dtype = torch.float16+ else:+ raise NotImplementedError(f"dtype: {dtype}")++ # KL Loss+ self.kl_loss_weight = kl_loss_weight+ # Perceptual Loss+ self.perceptual_loss_fn = LPIPS().eval().to(device, dtype)+ self.perceptual_loss_weight = perceptual_loss_weight+ self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)+++ def forward(+ self,+ video,+ recon_video,+ posterior,+ nll_weights=None,+ split="train",+ ):+ video = rearrange(video, "b c t h w -> (b t) c h w").contiguous()+ recon_video = rearrange(recon_video, "b c t h w -> (b t) c h w").contiguous()++ # reconstruction loss+ recon_loss = torch.abs(video - recon_video)++ # perceptual loss+ if self.perceptual_loss_weight is not None and self.perceptual_loss_weight > 0.0:+ # handle channels+ channels = video.shape[1]+ assert channels in {1, 3, 4}+ if channels == 1:+ input_vgg_input = repeat(video, "b 1 h w -> b c h w", c=3)+ recon_vgg_input = repeat(recon_video, "b 1 h w -> b c h w", c=3)+ elif channels == 4: # SCH: take the first 3 for perceptual loss calc+ input_vgg_input = video[:, :3]+ recon_vgg_input = recon_video[:, :3]+ else:+ input_vgg_input = video+ recon_vgg_input = recon_video++ perceptual_loss = self.perceptual_loss_fn(input_vgg_input, recon_vgg_input)+ recon_loss = recon_loss + self.perceptual_loss_weight * perceptual_loss++ nll_loss = recon_loss / torch.exp(self.logvar) + self.logvar+ weighted_nll_loss = nll_loss+ if nll_weights is not None:+ weighted_nll_loss = nll_weights * nll_loss+ weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]+ nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]+++ # KL Loss+ weighted_kl_loss = 0+ if self.kl_loss_weight is not None and self.kl_loss_weight > 0.0:+ kl_loss = posterior.kl()+ kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]+ weighted_kl_loss = kl_loss * self.kl_loss_weight++ return nll_loss, weighted_nll_loss, weighted_kl_loss+++class AdversarialLoss(nn.Module):+ def __init__(+ self,+ discriminator_factor=1.0,+ discriminator_start=50001,+ generator_factor=0.5,+ generator_loss_type="non-saturating",+ ):+ super().__init__()+ self.discriminator_factor = discriminator_factor+ self.discriminator_start = discriminator_start+ self.generator_factor = generator_factor+ self.generator_loss_type = generator_loss_type++ def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer):+ nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0] # SCH: TODO: debug added creat+ g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[+ 0+ ] # SCH: TODO: debug added create_graph=True+ d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)+ d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()+ d_weight = d_weight * self.generator_factor+ return d_weight++ def forward(+ self,+ fake_logits,+ nll_loss,+ last_layer,+ global_step,+ is_training=True,+ ):+ # NOTE: following MAGVIT to allow non_saturating+ assert self.generator_loss_type in ["hinge", "vanilla", "non-saturating"]++ if self.generator_loss_type == "hinge":+ gen_loss = -torch.mean(fake_logits)+ elif self.generator_loss_type == "non-saturating":+ gen_loss = torch.mean(+ sigmoid_cross_entropy_with_logits(labels=torch.ones_like(fake_logits), logits=fake_logits)+ )+ else:+ raise ValueError("Generator loss {} not supported".format(self.generator_loss_type))++ if self.discriminator_factor is not None and self.discriminator_factor > 0.0:+ try:+ d_weight = self.calculate_adaptive_weight(nll_loss, gen_loss, last_layer)+ except RuntimeError:+ assert not is_training+ d_weight = torch.tensor(0.0)+ else:+ d_weight = torch.tensor(0.0)++ disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)+ weighted_gen_loss = d_weight * disc_factor * gen_loss++ return weighted_gen_loss+++class LeCamEMA:+ def __init__(self, ema_real=0.0, ema_fake=0.0, decay=0.999, dtype=torch.bfloat16, device="cpu"):+ self.decay = decay+ self.ema_real = torch.tensor(ema_real).to(device, dtype)+ self.ema_fake = torch.tensor(ema_fake).to(device, dtype)++ def update(self, ema_real, ema_fake):+ self.ema_real = self.ema_real * self.decay + ema_real * (1 - self.decay)+ self.ema_fake = self.ema_fake * self.decay + ema_fake * (1 - self.decay)++ def get(self):+ return self.ema_real, self.ema_fake+++class DiscriminatorLoss(nn.Module):+ def __init__(+ self,+ discriminator_factor=1.0,+ discriminator_start=50001,+ discriminator_loss_type="non-saturating",+ lecam_loss_weight=None,+ gradient_penalty_loss_weight=None, # SCH: following MAGVIT config.vqgan.grad_penalty_cost+ ):+ super().__init__()++ assert discriminator_loss_type in ["hinge", "vanilla", "non-saturating"]+ self.discriminator_factor = discriminator_factor+ self.discriminator_start = discriminator_start+ self.lecam_loss_weight = lecam_loss_weight+ self.gradient_penalty_loss_weight = gradient_penalty_loss_weight+ self.discriminator_loss_type = discriminator_loss_type++ def forward(+ self,+ real_logits,+ fake_logits,+ global_step,+ lecam_ema_real=None,+ lecam_ema_fake=None,+ real_video=None,+ split="train",+ ):+ if self.discriminator_factor is not None and self.discriminator_factor > 0.0:+ disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)++ if self.discriminator_loss_type == "hinge":+ disc_loss = hinge_d_loss(real_logits, fake_logits)+ elif self.discriminator_loss_type == "non-saturating":+ if real_logits is not None:+ real_loss = sigmoid_cross_entropy_with_logits(+ labels=torch.ones_like(real_logits), logits=real_logits+ )+ else:+ real_loss = 0.0+ if fake_logits is not None:+ fake_loss = sigmoid_cross_entropy_with_logits(+ labels=torch.zeros_like(fake_logits), logits=fake_logits+ )+ else:+ fake_loss = 0.0+ disc_loss = 0.5 * (torch.mean(real_loss) + torch.mean(fake_loss))+ elif self.discriminator_loss_type == "vanilla":+ disc_loss = vanilla_d_loss(real_logits, fake_logits)+ else:+ raise ValueError(f"Unknown GAN loss '{self.discriminator_loss_type}'.")++ weighted_d_adversarial_loss = disc_factor * disc_loss++ else:+ weighted_d_adversarial_loss = 0++ lecam_loss = torch.tensor(0.0)+ if self.lecam_loss_weight is not None and self.lecam_loss_weight > 0.0:+ real_pred = torch.mean(real_logits)+ fake_pred = torch.mean(fake_logits)+ lecam_loss = lecam_reg(real_pred, fake_pred, lecam_ema_real, lecam_ema_fake)+ lecam_loss = lecam_loss * self.lecam_loss_weight++ gradient_penalty = torch.tensor(0.0)+ if self.gradient_penalty_loss_weight is not None and self.gradient_penalty_loss_weight > 0.0:+ assert real_video is not None+ gradient_penalty = gradient_penalty_fn(real_video, real_logits)+ # gradient_penalty = r1_penalty(real_video, real_logits) # MAGVIT uses r1 penalty+ gradient_penalty *= self.gradient_penalty_loss_weight++ # discriminator_loss = weighted_d_adversarial_loss + lecam_loss + gradient_penalty++ # log = {+ # "{}/d_adversarial_loss".format(split): weighted_d_adversarial_loss.detach().mean(),+ # "{}/lecam_loss".format(split): lecam_loss.detach().mean(),+ # "{}/gradient_penalty".format(split): gradient_penalty.detach().mean(),+ # }++ return (weighted_d_adversarial_loss, lecam_loss, gradient_penalty)++[email protected]_module("VAE_MAGVIT_V2")+def VAE_MAGVIT_V2(from_pretrained=None, **kwargs):+ model = VAE_3D_V2(**kwargs)if from_pretrained is not None:- load_checkpoint(model, from_pretrained)- return model\ No newline at end of file+ load_checkpoint(model, from_pretrained, model_name="model")+ return model++[email protected]_module("DISCRIMINATOR_3D")+def DISCRIMINATOR_3D(from_pretrained=None, inflate_from_2d=False, use_pretrained=True, **kwargs):+ model = StyleGANDiscriminatorBlur(**kwargs).apply(xavier_uniform_weight_init)+ if from_pretrained is not None:+ if use_pretrained:+ if inflate_from_2d:+ load_checkpoint_with_inflation(model, from_pretrained)+ else:+ load_checkpoint(model, from_pretrained, model_name="discriminator")+ print(f"loaded discriminator")+ else:+ print(f"discriminator use_pretrained={use_pretrained}, initializing new discriminator")++ return model+[email protected]_module("N_Layer_DISCRIMINATOR_3D")+def DISCRIMINATOR_3D(from_pretrained=None, inflate_from_2d=False, use_pretrained=True, **kwargs):+ model = NLayerDiscriminator3D(input_nc=3, n_layers=3,).apply(n_layer_disc_weights_init)+ if from_pretrained is not None:+ if use_pretrained:+ if inflate_from_2d:+ load_checkpoint_with_inflation(model, from_pretrained)+ else:+ load_checkpoint(model, from_pretrained, model_name="discriminator")+ print(f"loaded discriminator")+ else:+ print(f"discriminator use_pretrained={use_pretrained}, initializing new discriminator")++ return model+++def load_checkpoint_with_inflation(model, ckpt_path):+ """+ pre-train using image, then inflate to 3D videos+ """+ if ckpt_path.endswith(".pt") or ckpt_path.endswith(".pth"):+ state_dict = find_model(ckpt_path)+ with torch.no_grad():+ for key in state_dict:+ if key in model:+ # central inflation+ if state_dict[key].size() == model[key][:, :, 0, :, :].size():+ # temporal dimension+ val = torch.zeros_like(model[key])+ centre = int(model[key].size(2) // 2)+ val[:, :, centre, :, :] = state_dict[key]+ missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)+ print(f"Missing keys: {missing_keys}")+ print(f"Unexpected keys: {unexpected_keys}")+ else:+ load_checkpoint(model, ckpt_path) # use the default functiondiff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.pydeleted file mode 100644index 96a5f9a..0000000--- a/opensora/models/vae/vae_3d_v2.py+++ /dev/null@@ -1,1349 +0,0 @@-import functools-import math-from typing import Tuple, Union--import numpy as np-import torch-import torch.nn as nn-import torch.nn.functional as F-from einops import pack, rearrange, repeat, unpack-from torch import nn--from opensora.models.vae import model_utils-from opensora.models.vae.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers-from opensora.registry import MODELS-from opensora.utils.ckpt_utils import find_model, load_checkpoint--# from diffusers.models.modeling_utils import ModelMixin---"""Encoder and Decoder stuctures with 3D CNNs."""--"""-NOTE:- removed LayerNorm since not used in this arch- GroupNorm: flax uses default `epsilon=1e-06`, whereas torch uses `eps=1e-05`- for average pool and upsample, input shape needs to be [N,C,T,H,W] --> if not, adjust the scale factors accordingly-- !!! opensora read video into [B,C,T,H,W] format output--"""---def cast_tuple(t, length=1):- return t if isinstance(t, tuple) else ((t,) * length)---def divisible_by(num, den):- return (num % den) == 0---def is_odd(n):- return not divisible_by(n, 2)---def pad_at_dim(t, pad, dim=-1, value=0.0):- dims_from_right = (-dim - 1) if dim < 0 else (t.ndim - dim - 1)- zeros = (0, 0) * dims_from_right- return F.pad(t, (*zeros, *pad), value=value)---def pick_video_frame(video, frame_indices):- """get frame_indices from the video of [B, C, T, H, W] and return images of [B, C, H, W]"""- batch, device = video.shape[0], video.device- video = rearrange(video, "b c f ... -> b f c ...")- batch_indices = torch.arange(batch, device=device)- batch_indices = rearrange(batch_indices, "b -> b 1")- images = video[batch_indices, frame_indices]- images = rearrange(images, "b 1 c ... -> b c ...")- return images---def exists(v):- return v is not None---# ============== Generator Adversarial Loss Functions ==============-def lecam_reg(real_pred, fake_pred, ema_real_pred, ema_fake_pred):- assert real_pred.ndim == 0 and ema_fake_pred.ndim == 0- lecam_loss = torch.mean(torch.pow(nn.ReLU()(real_pred - ema_fake_pred), 2))- lecam_loss += torch.mean(torch.pow(nn.ReLU()(ema_real_pred - fake_pred), 2))- return lecam_loss---# Open-Sora-Plan-# Very bad, do not use-def r1_penalty(real_img, real_pred):- """R1 regularization for discriminator. The core idea is to- penalize the gradient on real data alone: when the- generator distribution produces the true data distribution- and the discriminator is equal to 0 on the data manifold, the- gradient penalty ensures that the discriminator cannot create- a non-zero gradient orthogonal to the data manifold without- suffering a loss in the GAN game.-- Ref:- Eq. 9 in Which training methods for GANs do actually converge.- """- grad_real = torch.autograd.grad(outputs=real_pred.sum(), inputs=real_img, create_graph=True)[0]- grad_penalty = grad_real.pow(2).view(grad_real.shape[0], -1).sum(1).mean()- return grad_penalty---# Open-Sora-Plan-# Implementation as described by https://arxiv.org/abs/1704.00028 # TODO: checkout the codes-def gradient_penalty_loss(discriminator, real_data, fake_data, weight=None):- """Calculate gradient penalty for wgan-gp.-- Args:- discriminator (nn.Module): Network for the discriminator.- real_data (Tensor): Real input data.- fake_data (Tensor): Fake input data.- weight (Tensor): Weight tensor. Default: None.-- Returns:- Tensor: A tensor for gradient penalty.- """-- batch_size = real_data.size(0)- alpha = real_data.new_tensor(torch.rand(batch_size, 1, 1, 1))-- # interpolate between real_data and fake_data- interpolates = alpha * real_data + (1.0 - alpha) * fake_data- interpolates = torch.autograd.Variable(interpolates, requires_grad=True)-- disc_interpolates = discriminator(interpolates)- gradients = torch.autograd.grad(- outputs=disc_interpolates,- inputs=interpolates,- grad_outputs=torch.ones_like(disc_interpolates),- create_graph=True,- retain_graph=True,- only_inputs=True,- )[0]-- if weight is not None:- gradients = gradients * weight-- gradients_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean()- if weight is not None:- gradients_penalty /= torch.mean(weight)-- return gradients_penalty---def gradient_penalty_fn(images, output):- # batch_size = images.shape[0]- gradients = torch.autograd.grad(- outputs=output,- inputs=images,- grad_outputs=torch.ones(output.size(), device=images.device),- create_graph=True,- retain_graph=True,- only_inputs=True,- )[0]-- gradients = rearrange(gradients, "b ... -> b (...)")- return ((gradients.norm(2, dim=1) - 1) ** 2).mean()---# ============== Discriminator Adversarial Loss Functions ==============-def hinge_d_loss(logits_real, logits_fake):- loss_real = torch.mean(F.relu(1.0 - logits_real))- loss_fake = torch.mean(F.relu(1.0 + logits_fake))- d_loss = 0.5 * (loss_real + loss_fake)- return d_loss---def vanilla_d_loss(logits_real, logits_fake):- d_loss = 0.5 * (- torch.mean(torch.nn.functional.softplus(-logits_real)) + torch.mean(torch.nn.functional.softplus(logits_fake))- )- return d_loss---# from MAGVIT, used in place hof hinge_d_loss-def sigmoid_cross_entropy_with_logits(labels, logits):- # The final formulation is: max(x, 0) - x * z + log(1 + exp(-abs(x)))- zeros = torch.zeros_like(logits, dtype=logits.dtype)- condition = logits >= zeros- relu_logits = torch.where(condition, logits, zeros)- neg_abs_logits = torch.where(condition, -logits, logits)- return relu_logits - logits * labels + torch.log1p(torch.exp(neg_abs_logits))---def xavier_uniform_weight_init(m):- if isinstance(m, nn.Conv3d) or isinstance(m, nn.Linear):- nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain("relu"))- if m.bias is not None:- nn.init.zeros_(m.bias)- # print("initialized module to xavier_uniform:", m)---def Sequential(*modules):- modules = [*filter(exists, modules)]-- if len(modules) == 0:- return nn.Identity()-- return nn.Sequential(*modules)---def SameConv2d(dim_in, dim_out, kernel_size):- kernel_size = cast_tuple(kernel_size, 2)- padding = [k // 2 for k in kernel_size]- return nn.Conv2d(dim_in, dim_out, kernel_size=kernel_size, padding=padding)---def adopt_weight(weight, global_step, threshold=0, value=0.0):- if global_step < threshold:- weight = value- return weight---class CausalConv3d(nn.Module):- def __init__(- self,- chan_in,- chan_out,- kernel_size: Union[int, Tuple[int, int, int]],- pad_mode="constant",- strides=None, # allow custom stride- **kwargs,- ):- super().__init__()- kernel_size = cast_tuple(kernel_size, 3)-- time_kernel_size, height_kernel_size, width_kernel_size = kernel_size-- assert is_odd(height_kernel_size) and is_odd(width_kernel_size)-- dilation = kwargs.pop("dilation", 1)- stride = strides[0] if strides is not None else kwargs.pop("stride", 1)-- self.pad_mode = pad_mode- time_pad = dilation * (time_kernel_size - 1) + (1 - stride)- height_pad = height_kernel_size // 2- width_pad = width_kernel_size // 2-- self.time_pad = time_pad- self.time_causal_padding = (width_pad, width_pad, height_pad, height_pad, time_pad, 0)-- stride = strides if strides is not None else (stride, 1, 1)- # padding = kwargs.pop('padding', 0)- # if padding == "same" and not all([pad == 1 for pad in padding]):- # padding = "valid"- dilation = (dilation, 1, 1)- self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)-- def forward(self, x):- pad_mode = self.pad_mode if self.time_pad < x.shape[2] else "constant"-- x = F.pad(x, self.time_causal_padding, mode=pad_mode)- return self.conv(x)---class ResBlock(nn.Module):- def __init__(- self,- in_channels, # SCH: added- filters,- conv_fn,- activation_fn=nn.SiLU,- use_conv_shortcut=False,- num_groups=32,- device="cpu",- dtype=torch.bfloat16,- ):- super().__init__()- self.in_channels = in_channels- self.filters = filters- self.activate = activation_fn()- self.use_conv_shortcut = use_conv_shortcut-- # SCH: MAGVIT uses GroupNorm by default- self.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)- self.conv1 = conv_fn(in_channels, self.filters, kernel_size=(3, 3, 3), bias=False)- self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)- self.conv2 = conv_fn(self.filters, self.filters, kernel_size=(3, 3, 3), bias=False)- if in_channels != filters:- if self.use_conv_shortcut:- self.conv3 = conv_fn(in_channels, self.filters, kernel_size=(3, 3, 3), bias=False)- else:- self.conv3 = conv_fn(in_channels, self.filters, kernel_size=(1, 1, 1), bias=False)-- def forward(self, x):- # device, dtype = x.device, x.dtype- # input_dim = x.shape[1]- residual = x- x = self.norm1(x)- x = self.activate(x)- x = self.conv1(x)- x = self.norm2(x)- x = self.activate(x)- x = self.conv2(x)- if self.in_channels != self.filters: # SCH: ResBlock X->Y- residual = self.conv3(residual)- return x + residual---# SCH: own implementation modified on top of: discriminator with anti-aliased downsampling (blurpool Zhang et al.)-class BlurPool3D(nn.Module):- def __init__(- self,- channels,- pad_type="reflect",- filt_size=3,- stride=2,- pad_off=0,- device="cpu",- dtype=torch.bfloat16,- ):- super(BlurPool3D, self).__init__()- self.filt_size = filt_size- self.pad_off = pad_off- self.pad_sizes = [- int(1.0 * (filt_size - 1) / 2),- int(np.ceil(1.0 * (filt_size - 1) / 2)),- int(1.0 * (filt_size - 1) / 2),- int(np.ceil(1.0 * (filt_size - 1) / 2)),- int(1.0 * (filt_size - 1) / 2),- int(np.ceil(1.0 * (filt_size - 1) / 2)),- ]- self.pad_sizes = [pad_size + pad_off for pad_size in self.pad_sizes]- self.stride = stride- self.off = int((self.stride - 1) / 2.0)- self.channels = channels-- if self.filt_size == 1:- a = np.array(- [- 1.0,- ]- )- elif self.filt_size == 2:- a = np.array([1.0, 1.0])- elif self.filt_size == 3:- a = np.array([1.0, 2.0, 1.0])- elif self.filt_size == 4:- a = np.array([1.0, 3.0, 3.0, 1.0])- elif self.filt_size == 5:- a = np.array([1.0, 4.0, 6.0, 4.0, 1.0])- elif self.filt_size == 6:- a = np.array([1.0, 5.0, 10.0, 10.0, 5.0, 1.0])- elif self.filt_size == 7:- a = np.array([1.0, 6.0, 15.0, 20.0, 15.0, 6.0, 1.0])-- filt_2d = a[:, None] * a[None, :]- filt_3d = torch.Tensor(a[:, None, None] * filt_2d[None, :, :]).to(device, dtype)-- filt = filt_3d / torch.sum(filt_3d) # SCH: modified to it 3D- self.register_buffer("filt", filt[None, None, :, :, :].repeat((self.channels, 1, 1, 1, 1)))-- self.pad = get_pad_layer(pad_type)(self.pad_sizes)-- def forward(self, inp):- if self.filt_size == 1:- if self.pad_off == 0:- return inp[:, :, :: self.stride, :: self.stride]- else:- return self.pad(inp)[:, :, :: self.stride, :: self.stride]- else:- return F.conv3d(self.pad(inp), self.filt, stride=self.stride, groups=inp.shape[1])---def get_pad_layer(pad_type):- if pad_type in ["refl", "reflect"]:- PadLayer = nn.ReflectionPad3d- elif pad_type in ["repl", "replicate"]:- PadLayer = nn.ReplicationPad3d- elif pad_type == "zero":- PadLayer = nn.ZeroPad3d- else:- print("Pad type [%s] not recognized" % pad_type)- return PadLayer---class ResBlockDown(nn.Module):- """3D StyleGAN ResBlock for D."""-- def __init__(- self,- in_channels,- filters,- activation_fn,- num_groups=32,- device="cpu",- dtype=torch.bfloat16,- ):- super().__init__()-- self.filters = filters- self.activation_fn = activation_fn-- # SCH: NOTE: although paper says conv (X->Y, Y->Y), original code implementation is (X->X, X->Y), we follow code- self.conv1 = nn.Conv3d(- in_channels, in_channels, (3, 3, 3), padding=1, device=device, dtype=dtype- ) # NOTE: init to xavier_uniform- self.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)-- self.blur = BlurPool3D(in_channels, device=device, dtype=dtype)-- self.conv2 = nn.Conv3d(- in_channels, self.filters, (1, 1, 1), bias=False, device=device, dtype=dtype- ) # NOTE: init to xavier_uniform- self.conv3 = nn.Conv3d(- in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype- ) # NOTE: init to xavier_uniform- self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)-- # self.apply(xavier_uniform_weight_init)-- def forward(self, x):- residual = x- x = self.conv1(x)- x = self.norm1(x)- x = self.activation_fn(x)-- residual = self.blur(residual)- residual = self.conv2(residual)-- x = self.blur(x)- x = self.conv3(x)- x = self.norm2(x)- x = self.activation_fn(x)- out = (residual + x) / math.sqrt(2)- return out---# SCH: taken from Open Sora Plan-def n_layer_disc_weights_init(m):- classname = m.__class__.__name__- if classname.find("Conv") != -1:- nn.init.normal_(m.weight.data, 0.0, 0.02)- elif classname.find("BatchNorm") != -1:- nn.init.normal_(m.weight.data, 1.0, 0.02)- nn.init.constant_(m.bias.data, 0)---class NLayerDiscriminator3D(nn.Module):- """Defines a 3D PatchGAN discriminator as in Pix2Pix but for 3D inputs."""-- def __init__(self, input_nc=1, ndf=64, n_layers=3, use_actnorm=False):- """- Construct a 3D PatchGAN discriminator-- Parameters:- input_nc (int) -- the number of channels in input volumes- ndf (int) -- the number of filters in the last conv layer- n_layers (int) -- the number of conv layers in the discriminator- use_actnorm (bool) -- flag to use actnorm instead of batchnorm- """- super(NLayerDiscriminator3D, self).__init__()- if not use_actnorm:- norm_layer = nn.BatchNorm3d- else:- raise NotImplementedError("Not implemented.")- if type(norm_layer) == functools.partial:- use_bias = norm_layer.func != nn.BatchNorm3d- else:- use_bias = norm_layer != nn.BatchNorm3d-- kw = 4- padw = 1- sequence = [nn.Conv3d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]- nf_mult = 1- nf_mult_prev = 1- for n in range(1, n_layers): # gradually increase the number of filters- nf_mult_prev = nf_mult- nf_mult = min(2**n, 8)- sequence += [- nn.Conv3d(- ndf * nf_mult_prev,- ndf * nf_mult,- kernel_size=(kw, kw, kw),- stride=(1, 2, 2),- padding=padw,- bias=use_bias,- ),- norm_layer(ndf * nf_mult),- nn.LeakyReLU(0.2, True),- ]-- nf_mult_prev = nf_mult- nf_mult = min(2**n_layers, 8)- sequence += [- nn.Conv3d(- ndf * nf_mult_prev, ndf * nf_mult, kernel_size=(kw, kw, kw), stride=1, padding=padw, bias=use_bias- ),- norm_layer(ndf * nf_mult),- nn.LeakyReLU(0.2, True),- ]-- sequence += [- nn.Conv3d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)- ] # output 1 channel prediction map- self.main = nn.Sequential(*sequence)-- def forward(self, input):- """Standard forward."""- return self.main(input)---class StyleGANDiscriminatorBlur(nn.Module):- """StyleGAN Discriminator."""-- """- SCH: NOTE:- this discriminator requries the num_frames to be fixed during training;- in case we pre-train with image then train on video, this disciminator's Linear layer would have to be re-trained!- """-- def __init__(- self,- image_size=(128, 128),- num_frames=17,- in_channels=3,- filters=128,- channel_multipliers=(2, 4, 4, 4, 4),- num_groups=32,- dtype=torch.bfloat16,- device="cpu",- ):- super().__init__()-- self.dtype = dtype- self.input_size = cast_tuple(image_size, 2)- self.filters = filters- self.activation_fn = nn.LeakyReLU(negative_slope=0.2)- self.channel_multipliers = channel_multipliers-- self.conv1 = nn.Conv3d(- in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype- ) # NOTE: init to xavier_uniform-- prev_filters = self.filters # record in_channels- self.num_blocks = len(self.channel_multipliers)- self.res_block_list = nn.ModuleList([])- for i in range(self.num_blocks):- filters = self.filters * self.channel_multipliers[i]- self.res_block_list.append(- ResBlockDown(prev_filters, filters, self.activation_fn, device=device, dtype=dtype).apply(- xavier_uniform_weight_init- )- )- prev_filters = filters # update in_channels-- self.conv2 = nn.Conv3d(- prev_filters, prev_filters, (3, 3, 3), padding=1, device=device, dtype=dtype- ) # NOTE: init to xavier_uniform- # torch.nn.init.xavier_uniform_(self.conv2.weight)-- self.norm1 = nn.GroupNorm(num_groups, prev_filters, dtype=dtype, device=device)-- scale_factor = 2**self.num_blocks- if num_frames % scale_factor != 0: # SCH: NOTE: has first frame which would be padded before usage- time_scaled = num_frames // scale_factor + 1- else:- time_scaled = num_frames / scale_factor-- assert (- self.input_size[0] % scale_factor == 0- ), f"image width {self.input_size[0]} is not divisible by scale factor {scale_factor}"- assert (- self.input_size[1] % scale_factor == 0- ), f"image height {self.input_size[1]} is not divisible by scale factor {scale_factor}"- w_scaled, h_scaled = self.input_size[0] / scale_factor, self.input_size[1] / scale_factor- in_features = int(prev_filters * time_scaled * w_scaled * h_scaled) # (C*T*W*H)- self.linear1 = nn.Linear(in_features, prev_filters, device=device, dtype=dtype) # NOTE: init to xavier_uniform- self.linear2 = nn.Linear(prev_filters, 1, device=device, dtype=dtype) # NOTE: init to xavier_uniform-- # self.apply(xavier_uniform_weight_init)-- def forward(self, x):- x = self.conv1(x)- # print("discriminator aft conv:", x.size())- x = self.activation_fn(x)-- for i in range(self.num_blocks):- x = self.res_block_list[i](x)- # print("discriminator resblock down:", x.size())-- x = self.conv2(x)- # print("discriminator aft conv2:", x.size())- x = self.norm1(x)- x = self.activation_fn(x)- x = x.reshape((x.shape[0], -1)) # SCH: [B, (C * T * W * H)] ?-- # print("discriminator reshape:", x.size())- x = self.linear1(x)- # print("discriminator aft linear1:", x.size())-- x = self.activation_fn(x)- x = self.linear2(x)- # print("discriminator aft linear2:", x.size())- return x---class Encoder(nn.Module):- """Encoder Blocks."""-- def __init__(- self,- filters=128,- num_res_blocks=4,- channel_multipliers=(1, 2, 2, 4),- temporal_downsample=(False, True, True),- num_groups=32, # for nn.GroupNorm- # in_out_channels = 3, # SCH: added, in_channels at the start- latent_embed_dim=512, # num channels for latent vector- # conv_downsample = False,- disable_spatial_downsample=False, # for vae pipeline- custom_conv_padding=None,- activation_fn="swish",- device="cpu",- dtype=torch.bfloat16,- ):- super().__init__()- self.filters = filters- self.num_res_blocks = num_res_blocks- self.channel_multipliers = channel_multipliers- self.temporal_downsample = temporal_downsample- self.num_groups = num_groups-- self.embedding_dim = latent_embed_dim- self.disable_spatial_downsample = disable_spatial_downsample- # self.conv_downsample = conv_downsample- self.custom_conv_padding = custom_conv_padding-- if activation_fn == "relu":- self.activation_fn = nn.ReLU- elif activation_fn == "swish":- self.activation_fn = nn.SiLU- else:- raise NotImplementedError- self.activate = self.activation_fn()-- self.conv_fn = functools.partial(- CausalConv3d,- # padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorch- dtype=dtype,- device=device,- )-- self.block_args = dict(- conv_fn=self.conv_fn,- dtype=dtype,- activation_fn=self.activation_fn,- use_conv_shortcut=False,- num_groups=self.num_groups,- device=device,- )-- # NOTE: moved to VAE for separate first frame processing- # self.conv1 = self.conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)-- # ResBlocks and conv downsample- self.block_res_blocks = nn.ModuleList([])- self.num_blocks = len(self.channel_multipliers)- self.conv_blocks = nn.ModuleList([])-- filters = self.filters- prev_filters = filters # record for in_channels- for i in range(self.num_blocks):- # resblock handling- filters = self.filters * self.channel_multipliers[i] # SCH: determine the number out_channels- block_items = nn.ModuleList([])- for _ in range(self.num_res_blocks):- block_items.append(ResBlock(prev_filters, filters, **self.block_args))- prev_filters = filters # update in_channels- self.block_res_blocks.append(block_items)-- if i < self.num_blocks - 1: # SCH: T-Causal Conv 3x3x3, 128->128, stride t x stride s x stride s- t_stride = 2 if self.temporal_downsample[i] else 1- s_stride = 2 if not self.disable_spatial_downsample else 1- self.conv_blocks.append(- self.conv_fn(prev_filters, filters, kernel_size=(3, 3, 3), strides=(t_stride, s_stride, s_stride))- ) # SCH: should be same in_channel and out_channel- prev_filters = filters # update in_channels-- # last layer res block- self.res_blocks = nn.ModuleList([])- for _ in range(self.num_res_blocks):- self.res_blocks.append(ResBlock(prev_filters, filters, **self.block_args))- prev_filters = filters # update in_channels-- # MAGVIT uses Group Normalization- self.norm1 = nn.GroupNorm(- self.num_groups, prev_filters, dtype=dtype, device=device- ) # SCH: separate <prev_filters> channels into 32 groups-- self.conv2 = nn.Conv3d(- prev_filters, self.embedding_dim, kernel_size=(1, 1, 1), dtype=dtype, device=device, padding="same"- )-- def forward(self, x):- # dtype, device = x.dtype, x.device-- # NOTE: moved to VAE for separate first frame processing- # x = self.conv1(x)-- # print("encoder:", x.size())-- for i in range(self.num_blocks):- for j in range(self.num_res_blocks):- x = self.block_res_blocks[i][j](x)- # print("encoder:", x.size())-- if i < self.num_blocks - 1:- x = self.conv_blocks[i](x)- # print("encoder:", x.size())-- for i in range(self.num_res_blocks):- x = self.res_blocks[i](x)- # print("encoder:", x.size())-- x = self.norm1(x)- x = self.activate(x)- x = self.conv2(x)- # print("encoder:", x.size())- return x---class Decoder(nn.Module):- """Decoder Blocks."""-- def __init__(- self,- latent_embed_dim=512,- filters=128,- # in_out_channels = 4,- num_res_blocks=4,- channel_multipliers=(1, 2, 2, 4),- temporal_downsample=(False, True, True),- num_groups=32, # for nn.GroupNorm- # upsample = "nearest+conv", # options: "deconv", "nearest+conv"- disable_spatial_upsample=False, # for vae pipeline- custom_conv_padding=None,- activation_fn="swish",- device="cpu",- dtype=torch.bfloat16,- ):- super().__init__()- # self.output_dim = in_out_channels- self.embedding_dim = latent_embed_dim- self.filters = filters- self.num_res_blocks = num_res_blocks- self.channel_multipliers = channel_multipliers- self.temporal_downsample = temporal_downsample- self.num_groups = num_groups-- # self.upsample = upsample- self.s_stride = 1 if disable_spatial_upsample else 2 # spatial stride- self.custom_conv_padding = custom_conv_padding- # self.norm_type = self.config.vqvae.norm_type- # self.num_remat_block = self.config.vqvae.get('num_dec_remat_blocks', 0)-- if activation_fn == "relu":- self.activation_fn = nn.ReLU- elif activation_fn == "swish":- self.activation_fn = nn.SiLU- else:- raise NotImplementedError- self.activate = self.activation_fn()-- self.conv_fn = functools.partial(- CausalConv3d,- dtype=dtype,- # padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorch- device=device,- )-- self.block_args = dict(- conv_fn=self.conv_fn,- activation_fn=self.activation_fn,- use_conv_shortcut=False,- num_groups=self.num_groups,- device=device,- dtype=dtype,- )- self.num_blocks = len(self.channel_multipliers)-- filters = self.filters * self.channel_multipliers[-1]-- self.conv1 = self.conv_fn(self.embedding_dim, filters, kernel_size=(3, 3, 3), bias=True)-- # last layer res block- self.res_blocks = nn.ModuleList([])- for _ in range(self.num_res_blocks):- self.res_blocks.append(ResBlock(filters, filters, **self.block_args))-- # TODO: do I need to add adaptive GroupNorm in between each block?-- # # NOTE: upsample, dimensions T, H, W- # self.upsampler_with_t = nn.Upsample(scale_factor=(2,2,2))- # self.upsampler = nn.Upsample(scale_factor=(1,2,2))-- # ResBlocks and conv upsample- prev_filters = filters # SCH: in_channels- self.block_res_blocks = nn.ModuleList([])- self.num_blocks = len(self.channel_multipliers)- self.conv_blocks = nn.ModuleList([])- # SCH: reverse to keep track of the in_channels, but append also in a reverse direction- for i in reversed(range(self.num_blocks)):- filters = self.filters * self.channel_multipliers[i]- # resblock handling- block_items = nn.ModuleList([])- for _ in range(self.num_res_blocks):- block_items.append(ResBlock(prev_filters, filters, **self.block_args))- prev_filters = filters # SCH: update in_channels- self.block_res_blocks.insert(0, block_items) # SCH: append in front-- # conv blocks with upsampling- if i > 0:- t_stride = 2 if self.temporal_downsample[i - 1] else 1- # SCH: T-Causal Conv 3x3x3, f -> (t_stride * 2 * 2) * f, depth to space t_stride x 2 x 2- self.conv_blocks.insert(- 0,- self.conv_fn(- prev_filters, prev_filters * t_stride * self.s_stride * self.s_stride, kernel_size=(3, 3, 3)- ),- )-- self.norm1 = nn.GroupNorm(self.num_groups, prev_filters, device=device, dtype=dtype)-- # NOTE: moved to VAE for separate first frame processing- # self.conv2 = self.conv_fn(prev_filters, self.output_dim, kernel_size=(3, 3, 3))-- def forward(- self,- x,- **kwargs,- ):- # dtype, device = x.dtype, x.device-- x = self.conv1(x)- # print("decoder:", x.size())- for i in range(self.num_res_blocks):- x = self.res_blocks[i](x)- # print("decoder:", x.size())- for i in reversed(range(self.num_blocks)): # reverse here to make decoder symmetric with encoder- for j in range(self.num_res_blocks):- x = self.block_res_blocks[i][j](x)- # print("decoder:", x.size())- if i > 0:- t_stride = 2 if self.temporal_downsample[i - 1] else 1- # SCH: T-Causal Conv 3x3x3, f -> (t_stride * 2 * 2) * f, depth to space t_stride x 2 x 2- x = self.conv_blocks[i - 1](x)- x = rearrange(- x,- "B (C ts hs ws) T H W -> B C (T ts) (H hs) (W ws)",- ts=t_stride,- hs=self.s_stride,- ws=self.s_stride,- )- # print("decoder:", x.size())-- x = self.norm1(x)- x = self.activate(x)- # NOTE: moved to VAE for separate first frame processing- # x = self.conv2(x)- return x--[email protected]_module()-class VAE_3D_V2(nn.Module): # , ModelMixin- """The 3D VAE"""-- def __init__(- self,- latent_embed_dim=256,- filters=128,- num_res_blocks=2,- separate_first_frame_encoding=False,- channel_multipliers=(1, 2, 2, 4),- temporal_downsample=(True, True, False),- num_groups=32, # for nn.GroupNorm- disable_space=False,- custom_conv_padding=None,- activation_fn="swish",- in_out_channels=4,- kl_embed_dim=64,- encoder_double_z=True,- device="cpu",- dtype="bf16",- ):- super().__init__()-- if type(dtype) == str:- if dtype == "bf16":- dtype = torch.bfloat16- elif dtype == "fp16":- dtype = torch.float16- else:- raise NotImplementedError(f"dtype: {dtype}")-- # ==== Model Params ====- # self.image_size = cast_tuple(image_size, 2)- self.time_downsample_factor = 2 ** sum(temporal_downsample)- self.time_padding = self.time_downsample_factor - 1- self.separate_first_frame_encoding = separate_first_frame_encoding-- image_down = 2 ** len(temporal_downsample)- t_down = 2 ** len([x for x in temporal_downsample if x == True])- self.patch_size = (t_down, image_down, image_down)-- # ==== Model Initialization ====-- # encoder & decoder first and last conv layer- # SCH: NOTE: following MAGVIT, conv in bias=False in encoder first conv- self.conv_in = CausalConv3d(- in_out_channels, filters, kernel_size=(3, 3, 3), bias=False, dtype=dtype, device=device- )- self.conv_in_first_frame = nn.Identity()- self.conv_out_first_frame = nn.Identity()- if separate_first_frame_encoding:- self.conv_in_first_frame = SameConv2d(in_out_channels, filters, (3, 3))- self.conv_out_first_frame = SameConv2d(filters, in_out_channels, (3, 3))- self.conv_out = CausalConv3d(filters, in_out_channels, 3, dtype=dtype, device=device)-- self.encoder = Encoder(- filters=filters,- num_res_blocks=num_res_blocks,- channel_multipliers=channel_multipliers,- temporal_downsample=temporal_downsample,- num_groups=num_groups, # for nn.GroupNorm- # in_out_channels = in_out_channels,- latent_embed_dim=latent_embed_dim * 2 if encoder_double_z else latent_embed_dim,- # conv_downsample = conv_downsample,- disable_spatial_downsample=disable_space,- custom_conv_padding=custom_conv_padding,- activation_fn=activation_fn,- device=device,- dtype=dtype,- )- self.decoder = Decoder(- latent_embed_dim=latent_embed_dim,- filters=filters,- # in_out_channels = in_out_channels,- num_res_blocks=num_res_blocks,- channel_multipliers=channel_multipliers,- temporal_downsample=temporal_downsample,- num_groups=num_groups, # for nn.GroupNorm- # upsample = upsample, # options: "deconv", "nearest+conv"- disable_spatial_upsample=disable_space,- custom_conv_padding=custom_conv_padding,- activation_fn=activation_fn,- device=device,- dtype=dtype,- )-- if encoder_double_z:- self.quant_conv = nn.Conv3d(2 * latent_embed_dim, 2 * kl_embed_dim, 1, device=device, dtype=dtype)- else:- self.quant_conv = nn.Conv3d(latent_embed_dim, 2 * kl_embed_dim, 1, device=device, dtype=dtype)- self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1, device=device, dtype=dtype)-- def get_latent_size(self, input_size):- for i in range(len(input_size)):- assert input_size[i] % self.patch_size[i] == 0, "Input size must be divisible by patch size"- input_size = [input_size[i] // self.patch_size[i] for i in range(3)]- return input_size-- def encode(- self,- video,- video_contains_first_frame=True,- ):- encode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame-- # whether to pad video or not- if video_contains_first_frame:- video_len = video.shape[2]- video = pad_at_dim(video, (self.time_padding, 0), value=0.0, dim=2)- video_packed_shape = [torch.Size([self.time_padding]), torch.Size([]), torch.Size([video_len - 1])]-- # print("pre-encoder:", video.size())-- # NOTE: moved encoder conv1 here for separate first frame encoding- if encode_first_frame_separately:- pad, first_frame, video = unpack(video, video_packed_shape, "b c * h w")- first_frame = self.conv_in_first_frame(first_frame)- video = self.conv_in(video)-- # print("pre-encoder:", video.size())-- if encode_first_frame_separately:- video, _ = pack([first_frame, video], "b c * h w")- video = pad_at_dim(video, (self.time_padding, 0), dim=2)-- encoded_feature = self.encoder(video)-- # print("after encoder:", encoded_feature.size())-- # NOTE: TODO: do we include this before gaussian distri? or go directly to Gaussian distribution- moments = self.quant_conv(encoded_feature).to(video.dtype)- posterior = model_utils.DiagonalGaussianDistribution(moments)-- # print("after encoder moments:", moments.size())-- return posterior-- def decode(- self,- z,- video_contains_first_frame=True,- ):- # dtype = z.dtype- decode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame-- z = self.post_quant_conv(z)- # print("pre decoder, post quant conv:", z.size())-- dec = self.decoder(z)- # print("post decoder:", dec.size())-- # SCH: moved decoder last conv layer here for separate first frame decoding- if decode_first_frame_separately:- left_pad, dec_ff, dec = (- dec[:, :, : self.time_padding],- dec[:, :, self.time_padding],- dec[:, :, (self.time_padding + 1) :],- )- out = self.conv_out(dec)- outff = self.conv_out_first_frame(dec_ff)- video, _ = pack([outff, out], "b c * h w")- else:- video = self.conv_out(dec)- # if video were padded, remove padding- if video_contains_first_frame:- video = video[:, :, self.time_padding :]-- # print("conv out:", video.size())-- return video-- def get_last_layer(self):- # CausalConv3d wraps the conv- return self.conv_out.conv.weight-- def forward(- self,- video,- sample_posterior=True,- video_contains_first_frame=True,- # split = "train",- ):- batch, channels, frames = video.shape[:3]- assert divisible_by(- frames - int(video_contains_first_frame), self.time_downsample_factor- ), f"number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}"-- posterior = self.encode(- video,- video_contains_first_frame=video_contains_first_frame,- )-- if sample_posterior:- z = posterior.sample()- else:- z = posterior.mode()-- recon_video = self.decode(z, video_contains_first_frame=video_contains_first_frame)-- return recon_video, posterior---class VEALoss(nn.Module):- def __init__(- self,- logvar_init=0.0,- perceptual_loss_weight=0.1,- kl_loss_weight=0.000001,- # vgg=None,- # vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,- device="cpu",- dtype="bf16",- ):- super().__init__()-- if type(dtype) == str:- if dtype == "bf16":- dtype = torch.bfloat16- elif dtype == "fp16":- dtype = torch.float16- else:- raise NotImplementedError(f"dtype: {dtype}")-- # KL Loss- self.kl_loss_weight = kl_loss_weight- # Perceptual Loss- self.perceptual_loss_fn = LPIPS().eval().to(device, dtype)- self.perceptual_loss_weight = perceptual_loss_weight- self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)--- def forward(- self,- video,- recon_video,- posterior,- nll_weights=None,- split="train",- ):- video = rearrange(video, "b c t h w -> (b t) c h w").contiguous()- recon_video = rearrange(recon_video, "b c t h w -> (b t) c h w").contiguous()-- # reconstruction loss- recon_loss = torch.abs(video - recon_video)-- # perceptual loss- if self.perceptual_loss_weight is not None and self.perceptual_loss_weight > 0.0:- # handle channels- channels = video.shape[1]- assert channels in {1, 3, 4}- if channels == 1:- input_vgg_input = repeat(video, "b 1 h w -> b c h w", c=3)- recon_vgg_input = repeat(recon_video, "b 1 h w -> b c h w", c=3)- elif channels == 4: # SCH: take the first 3 for perceptual loss calc- input_vgg_input = video[:, :3]- recon_vgg_input = recon_video[:, :3]- else:- input_vgg_input = video- recon_vgg_input = recon_video-- perceptual_loss = self.perceptual_loss_fn(input_vgg_input, recon_vgg_input)- recon_loss = recon_loss + self.perceptual_loss_weight * perceptual_loss-- nll_loss = recon_loss / torch.exp(self.logvar) + self.logvar- weighted_nll_loss = nll_loss- if nll_weights is not None:- weighted_nll_loss = nll_weights * nll_loss- weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]- nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]--- # KL Loss- weighted_kl_loss = 0- if self.kl_loss_weight is not None and self.kl_loss_weight > 0.0:- kl_loss = posterior.kl()- kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]- weighted_kl_loss = kl_loss * self.kl_loss_weight-- return nll_loss, weighted_nll_loss, weighted_kl_loss---class AdversarialLoss(nn.Module):- def __init__(- self,- discriminator_factor=1.0,- discriminator_start=50001,- generator_factor=0.5,- generator_loss_type="non-saturating",- ):- super().__init__()- self.discriminator_factor = discriminator_factor- self.discriminator_start = discriminator_start- self.generator_factor = generator_factor- self.generator_loss_type = generator_loss_type-- def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer):- nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0] # SCH: TODO: debug added creat- g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[- 0- ] # SCH: TODO: debug added create_graph=True- d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)- d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()- d_weight = d_weight * self.generator_factor- return d_weight-- def forward(- self,- fake_logits,- nll_loss,- last_layer,- global_step,- is_training=True,- ):- # NOTE: following MAGVIT to allow non_saturating- assert self.generator_loss_type in ["hinge", "vanilla", "non-saturating"]-- if self.generator_loss_type == "hinge":- gen_loss = -torch.mean(fake_logits)- elif self.generator_loss_type == "non-saturating":- gen_loss = torch.mean(- sigmoid_cross_entropy_with_logits(labels=torch.ones_like(fake_logits), logits=fake_logits)- )- else:- raise ValueError("Generator loss {} not supported".format(self.generator_loss_type))-- if self.discriminator_factor is not None and self.discriminator_factor > 0.0:- try:- d_weight = self.calculate_adaptive_weight(nll_loss, gen_loss, last_layer)- except RuntimeError:- assert not is_training- d_weight = torch.tensor(0.0)- else:- d_weight = torch.tensor(0.0)-- disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)- weighted_gen_loss = d_weight * disc_factor * gen_loss-- return weighted_gen_loss---class LeCamEMA:- def __init__(self, ema_real=0.0, ema_fake=0.0, decay=0.999, dtype=torch.bfloat16, device="cpu"):- self.decay = decay- self.ema_real = torch.tensor(ema_real).to(device, dtype)- self.ema_fake = torch.tensor(ema_fake).to(device, dtype)-- def update(self, ema_real, ema_fake):- self.ema_real = self.ema_real * self.decay + ema_real * (1 - self.decay)- self.ema_fake = self.ema_fake * self.decay + ema_fake * (1 - self.decay)-- def get(self):- return self.ema_real, self.ema_fake---class DiscriminatorLoss(nn.Module):- def __init__(- self,- discriminator_factor=1.0,- discriminator_start=50001,- discriminator_loss_type="non-saturating",- lecam_loss_weight=None,- gradient_penalty_loss_weight=None, # SCH: following MAGVIT config.vqgan.grad_penalty_cost- ):- super().__init__()-- assert discriminator_loss_type in ["hinge", "vanilla", "non-saturating"]- self.discriminator_factor = discriminator_factor- self.discriminator_start = discriminator_start- self.lecam_loss_weight = lecam_loss_weight- self.gradient_penalty_loss_weight = gradient_penalty_loss_weight- self.discriminator_loss_type = discriminator_loss_type-- def forward(- self,- real_logits,- fake_logits,- global_step,- lecam_ema_real=None,- lecam_ema_fake=None,- real_video=None,- split="train",- ):- if self.discriminator_factor is not None and self.discriminator_factor > 0.0:- disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)-- if self.discriminator_loss_type == "hinge":- disc_loss = hinge_d_loss(real_logits, fake_logits)- elif self.discriminator_loss_type == "non-saturating":- if real_logits is not None:- real_loss = sigmoid_cross_entropy_with_logits(- labels=torch.ones_like(real_logits), logits=real_logits- )- else:- real_loss = 0.0- if fake_logits is not None:- fake_loss = sigmoid_cross_entropy_with_logits(- labels=torch.zeros_like(fake_logits), logits=fake_logits- )- else:- fake_loss = 0.0- disc_loss = 0.5 * (torch.mean(real_loss) + torch.mean(fake_loss))- elif self.discriminator_loss_type == "vanilla":- disc_loss = vanilla_d_loss(real_logits, fake_logits)- else:- raise ValueError(f"Unknown GAN loss '{self.discriminator_loss_type}'.")-- weighted_d_adversarial_loss = disc_factor * disc_loss-- else:- weighted_d_adversarial_loss = 0-- lecam_loss = torch.tensor(0.0)- if self.lecam_loss_weight is not None and self.lecam_loss_weight > 0.0:- real_pred = torch.mean(real_logits)- fake_pred = torch.mean(fake_logits)- lecam_loss = lecam_reg(real_pred, fake_pred, lecam_ema_real, lecam_ema_fake)- lecam_loss = lecam_loss * self.lecam_loss_weight-- gradient_penalty = torch.tensor(0.0)- if self.gradient_penalty_loss_weight is not None and self.gradient_penalty_loss_weight > 0.0:- assert real_video is not None- gradient_penalty = gradient_penalty_fn(real_video, real_logits)- # gradient_penalty = r1_penalty(real_video, real_logits) # MAGVIT uses r1 penalty- gradient_penalty *= self.gradient_penalty_loss_weight-- # discriminator_loss = weighted_d_adversarial_loss + lecam_loss + gradient_penalty-- # log = {- # "{}/d_adversarial_loss".format(split): weighted_d_adversarial_loss.detach().mean(),- # "{}/lecam_loss".format(split): lecam_loss.detach().mean(),- # "{}/gradient_penalty".format(split): gradient_penalty.detach().mean(),- # }-- return (weighted_d_adversarial_loss, lecam_loss, gradient_penalty)--[email protected]_module("VAE_MAGVIT_V2")-def VAE_MAGVIT_V2(from_pretrained=None, **kwargs):- model = VAE_3D_V2(**kwargs)- if from_pretrained is not None:- load_checkpoint(model, from_pretrained, model_name="model")- return model--[email protected]_module("DISCRIMINATOR_3D")-def DISCRIMINATOR_3D(from_pretrained=None, inflate_from_2d=False, use_pretrained=True, **kwargs):- model = StyleGANDiscriminatorBlur(**kwargs).apply(xavier_uniform_weight_init)- if from_pretrained is not None:- if use_pretrained:- if inflate_from_2d:- load_checkpoint_with_inflation(model, from_pretrained)- else:- load_checkpoint(model, from_pretrained, model_name="discriminator")- print(f"loaded discriminator")- else:- print(f"discriminator use_pretrained={use_pretrained}, initializing new discriminator")-- return model-[email protected]_module("N_Layer_DISCRIMINATOR_3D")-def DISCRIMINATOR_3D(from_pretrained=None, inflate_from_2d=False, use_pretrained=True, **kwargs):- model = NLayerDiscriminator3D(input_nc=3, n_layers=3,).apply(n_layer_disc_weights_init)- if from_pretrained is not None:- if use_pretrained:- if inflate_from_2d:- load_checkpoint_with_inflation(model, from_pretrained)- else:- load_checkpoint(model, from_pretrained, model_name="discriminator")- print(f"loaded discriminator")- else:- print(f"discriminator use_pretrained={use_pretrained}, initializing new discriminator")-- return model---def load_checkpoint_with_inflation(model, ckpt_path):- """- pre-train using image, then inflate to 3D videos- """- if ckpt_path.endswith(".pt") or ckpt_path.endswith(".pth"):- state_dict = find_model(ckpt_path)- with torch.no_grad():- for key in state_dict:- if key in model:- # central inflation- if state_dict[key].size() == model[key][:, :, 0, :, :].size():- # temporal dimension- val = torch.zeros_like(model[key])- centre = int(model[key].size(2) // 2)- val[:, :, centre, :, :] = state_dict[key]- missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)- print(f"Missing keys: {missing_keys}")- print(f"Unexpected keys: {unexpected_keys}")- else:- load_checkpoint(model, ckpt_path) # use the default functiondiff --git a/scripts/inference-vae-3d.py b/scripts/inference-vae-3d.pydeleted file mode 100644index a624a45..0000000--- a/scripts/inference-vae-3d.py+++ /dev/null@@ -1,148 +0,0 @@-import os--import colossalai-import torch-import torch.distributed as dist-from colossalai.cluster import DistCoordinator-from mmengine.runner import set_random_seed--from opensora.acceleration.parallel_states import set_sequence_parallel_group-from opensora.datasets import save_sample-from opensora.registry import MODELS, SCHEDULERS, build_module-from opensora.utils.config_utils import parse_configs-from opensora.utils.misc import to_torch_dtype-from opensora.datasets import DatasetFromCSV, get_transforms_image, get_transforms_video, prepare_dataloader-from opensora.acceleration.parallel_states import (- get_data_parallel_group,- set_data_parallel_group,- set_sequence_parallel_group,-)-from tqdm import tqdm-from opensora.models.vae.model_utils import VEA3DLoss-from colossalai.utils import get_current_device---def main():- # ======================================================- # 1. cfg and init distributed env- # ======================================================- cfg = parse_configs(training=False)- print(cfg)-- # init distributed- colossalai.launch_from_torch({})- coordinator = DistCoordinator()-- # if coordinator.world_size > 1:- # set_sequence_parallel_group(dist.group.WORLD)- # enable_sequence_parallelism = True- # else:- # enable_sequence_parallelism = False-- # ======================================================- # 2. runtime variables- # ======================================================- torch.set_grad_enabled(False)- # torch.backends.cuda.matmul.allow_tf32 = True- # torch.backends.cudnn.allow_tf32 = True- # device = "cuda" if torch.cuda.is_available() else "cpu"- device = get_current_device()- dtype = to_torch_dtype(cfg.dtype)- # set_random_seed(seed=cfg.seed)--- # ======================================================- # 3. build dataset and dataloader- # ======================================================- dataset = DatasetFromCSV(- cfg.data_path,- # TODO: change transforms- transform=(- get_transforms_video(cfg.image_size[0])- if not cfg.use_image_transform- else get_transforms_image(cfg.image_size[0])- ),- num_frames=cfg.num_frames,- frame_interval=cfg.frame_interval,- root=cfg.root,- )-- dataloader = prepare_dataloader(- dataset,- batch_size=cfg.batch_size,- num_workers=cfg.num_workers,- shuffle=False,- drop_last=True,- pin_memory=True,- process_group=get_data_parallel_group(),- )- print(f"Dataset contains {len(dataset):,} videos ({cfg.data_path})")-- total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size- print(f"Total batch size: {total_batch_size}")-- # ======================================================- # 4. build model & load weights- # ======================================================- # 3.1. build model- # input_size = (cfg.num_frames, *cfg.image_size)- vae = build_module(cfg.model, MODELS, device=device)- # latent_size = vae.get_latent_size(input_size)-- # 3.2. move to device & eval- vae = vae.to(device, dtype).eval()--- # 3.4. support for multi-resolution- model_args = dict()- if cfg.multi_resolution:- image_size = cfg.image_size- hw = torch.tensor([image_size], device=device, dtype=dtype).repeat(cfg.batch_size, 1)- ar = torch.tensor([[image_size[0] / image_size[1]]], device=device, dtype=dtype).repeat(cfg.batch_size, 1)- model_args["data_info"] = dict(ar=ar, hw=hw)-- # ======================================================- # 4. inference- # ======================================================- save_dir = cfg.save_dir- os.makedirs(save_dir, exist_ok=True)--- # 4.1. batch generation-- # define loss function- loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)- running_loss = 0.0- loss_steps = 0--- total_steps = len(dataloader)- dataloader_iter = iter(dataloader)-- with tqdm(- range(total_steps),- # desc=f"Avg Loss: {running_loss}",- disable=not coordinator.is_master(),- total=total_steps,- initial=0,- ) as pbar:- for step in pbar:- batch = next(dataloader_iter)- x = batch["video"].to(device, dtype) # [B, C, T, H, W]- reconstructions, posterior = vae(x)- loss = loss_function(x, reconstructions, posterior)- loss_steps += 1- running_loss = loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)-- if coordinator.is_master():- pbar.set_postfix({"loss": loss, "running_loss": running_loss, "step": step})- for idx, sample in enumerate(reconstructions):- pos = step * cfg.batch_size + idx- save_path = os.path.join(save_dir, f"sample_{pos}")- save_sample(sample, fps=cfg.fps, save_path=save_path)-- print("test loss:", running_loss)---if __name__ == "__main__":- main()diff --git a/scripts/inference-vae-v2.py b/scripts/inference-vae.pysimilarity index 100%rename from scripts/inference-vae-v2.pyrename to scripts/inference-vae.pydiff --git a/scripts/train-vae-3d.py b/scripts/train-vae-3d.pydeleted file mode 100644index cb6a213..0000000--- a/scripts/train-vae-3d.py+++ /dev/null@@ -1,271 +0,0 @@-from copy import deepcopy--import colossalai-import torch-import torch.nn as nn-from torch.optim import Optimizer-from torch.optim.lr_scheduler import _LRScheduler-import torch.distributed as dist-import wandb-from colossalai.booster import Booster-from colossalai.booster.plugin import LowLevelZeroPlugin-from colossalai.cluster import DistCoordinator-from colossalai.nn.optimizer import HybridAdam-from colossalai.utils import get_current_device-from tqdm import tqdm-import os--from opensora.acceleration.checkpoint import set_grad_checkpoint-from opensora.acceleration.parallel_states import (- get_data_parallel_group,- set_data_parallel_group,- set_sequence_parallel_group,-)-from opensora.acceleration.plugin import ZeroSeqParallelPlugin-from opensora.datasets import DatasetFromCSV, get_transforms_image, get_transforms_video, prepare_dataloader-from opensora.registry import MODELS, SCHEDULERS, build_module-from opensora.utils.ckpt_utils import create_logger, load_json, save_json, load, model_sharding, record_model_param_shape, save-from opensora.utils.config_utils import (- create_experiment_workspace,- create_tensorboard_writer,- parse_configs,- save_training_config,-)-from opensora.utils.misc import all_reduce_mean, format_numel_str, get_model_numel, requires_grad, to_torch_dtype-from opensora.utils.train_utils import update_ema, MaskGenerator-from opensora.models.vae.model_utils import VEA3DLoss---def main():- # ======================================================- # 1. args & cfg- # ======================================================- cfg = parse_configs(training=True)- print(cfg)- exp_name, exp_dir = create_experiment_workspace(cfg)- save_training_config(cfg._cfg_dict, exp_dir)-- # ======================================================- # 2. runtime variables & colossalai launch- # ======================================================- assert torch.cuda.is_available(), "Training currently requires at least one GPU."- assert cfg.dtype in ["fp16", "bf16"], f"Unknown mixed precision {cfg.dtype}"-- # 2.1. colossalai init distributed training- colossalai.launch_from_torch({})- coordinator = DistCoordinator()- device = get_current_device()- dtype = to_torch_dtype(cfg.dtype)-- # 2.2. init logger, tensorboard & wandb- if not coordinator.is_master():- logger = create_logger(None)- else:- logger = create_logger(exp_dir)- logger.info(f"Experiment directory created at {exp_dir}")-- writer = create_tensorboard_writer(exp_dir)- if cfg.wandb:- wandb.init(project="opensora-vae", name=exp_name, config=cfg._cfg_dict)-- # 2.3. initialize ColossalAI booster- if cfg.plugin == "zero2":- plugin = LowLevelZeroPlugin(- stage=2,- precision=cfg.dtype,- initial_scale=2**16,- max_norm=cfg.grad_clip,- )- set_data_parallel_group(dist.group.WORLD)- elif cfg.plugin == "zero2-seq":- plugin = ZeroSeqParallelPlugin(- sp_size=cfg.sp_size,- stage=2,- precision=cfg.dtype,- initial_scale=2**16,- max_norm=cfg.grad_clip,- )- set_sequence_parallel_group(plugin.sp_group)- set_data_parallel_group(plugin.dp_group)- else:- raise ValueError(f"Unknown plugin {cfg.plugin}")- booster = Booster(plugin=plugin)-- # ======================================================- # 3. build dataset and dataloader- # ======================================================- dataset = DatasetFromCSV(- cfg.data_path,- # TODO: change transforms- transform=(- get_transforms_video(cfg.image_size[0])- if not cfg.use_image_transform- else get_transforms_image(cfg.image_size[0])- ),- num_frames=cfg.num_frames,- frame_interval=cfg.frame_interval,- root=cfg.root,- )-- # TODO: use plugin's prepare dataloader- # a batch contains:- # {- # "video": torch.Tensor, # [B, C, T, H, W],- # "text": List[str],- # }- dataloader = prepare_dataloader(- dataset,- batch_size=cfg.batch_size,- num_workers=cfg.num_workers,- shuffle=True,- drop_last=True,- pin_memory=True,- process_group=get_data_parallel_group(),- )- logger.info(f"Dataset contains {len(dataset):,} videos ({cfg.data_path})")-- total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size- logger.info(f"Total batch size: {total_batch_size}")-- # ======================================================- # 4. build model- # ======================================================- # 4.1. build model- vae = build_module(cfg.model, MODELS, device=device)- vae_numel, vae_numel_trainable = get_model_numel(vae)- logger.info(- f"Trainable vae params: {format_numel_str(vae_numel_trainable)}, Total model params: {format_numel_str(vae_numel)}"- )--- # 4.3. move to device- vae = vae.to(device, dtype)-- # 4.5. setup optimizer- optimizer = HybridAdam(- filter(lambda p: p.requires_grad, vae.parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True- )- lr_scheduler = None-- # 4.6. prepare for training- if cfg.grad_checkpoint:- set_grad_checkpoint(vae)- vae.train()--- # =======================================================- # 5. boost model for distributed training with colossalai- # =======================================================- torch.set_default_dtype(dtype)- vae, optimizer, _, dataloader, lr_scheduler = booster.boost(- model=vae, optimizer=optimizer, lr_scheduler=lr_scheduler, dataloader=dataloader- )- torch.set_default_dtype(torch.float)- num_steps_per_epoch = len(dataloader)- logger.info("Boost vae for distributed training")-- # =======================================================- # 6. training loop- # =======================================================- start_epoch = start_step = log_step = sampler_start_idx = 0- running_loss = 0.0--- # 6.1. resume training- if cfg.load is not None:- logger.info("Loading checkpoint")- booster.load_model(vae, os.path.join(cfg.load, "model"))- booster.load_optimizer(optimizer, os.path.join(cfg.load, "optimizer"))- if lr_scheduler is not None:- booster.load_lr_scheduler(lr_scheduler, os.path.join(cfg.load, "lr_scheduler"))- running_states = load_json(os.path.join(cfg.load, "running_states.json"))- dist.barrier()- start_epoch, start_step, sampler_start_idx = running_states["epoch"], running_states["step"], running_states["sample_start_index"]- logger.info(f"Loaded checkpoint {cfg.load} at epoch {start_epoch} step {start_step}")- logger.info(f"Training for {cfg.epochs} epochs with {num_steps_per_epoch} steps per epoch")-- dataloader.sampler.set_start_index(sampler_start_idx)-- # define loss function- loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)--- # 6.2. training loop- for epoch in range(start_epoch, cfg.epochs):- dataloader.sampler.set_epoch(epoch)- dataloader_iter = iter(dataloader)- logger.info(f"Beginning epoch {epoch}...")-- with tqdm(- range(start_step, num_steps_per_epoch),- desc=f"Epoch {epoch}",- disable=not coordinator.is_master(),- total=num_steps_per_epoch,- initial=start_step,- ) as pbar:- for step in pbar:- batch = next(dataloader_iter)- x = batch["video"].to(device, dtype) # [B, C, T, H, W]-- # loss = vae.get_loss(x)- reconstructions, posterior = vae(x)- loss = loss_function(x, reconstructions, posterior)-- # Backward & update- booster.backward(loss=loss, optimizer=optimizer)- optimizer.step()- optimizer.zero_grad()-- # Log loss values:- all_reduce_mean(loss)- running_loss += loss.item()- global_step = epoch * num_steps_per_epoch + step- log_step += 1-- # Log to tensorboard- if coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:- avg_loss = running_loss / log_step- pbar.set_postfix({"loss": avg_loss, "step": step, "global_step": global_step})- running_loss = 0- log_step = 0- writer.add_scalar("loss", loss.item(), global_step)- if cfg.wandb:- wandb.log(- {- "iter": global_step,- "num_samples": global_step * total_batch_size,- "epoch": epoch,- "loss": loss.item(),- "avg_loss": avg_loss,- },- step=global_step,- )-- # Save checkpoint- if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:- save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")- os.makedirs(os.path.join(save_dir, "model"), exist_ok=True)- # TODO: save in model?- booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)- booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)- if lr_scheduler is not None:- booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))- running_states = {- "epoch": epoch,- "step": step+1,- "global_step": global_step+1,- "sample_start_index": (step+1) * cfg.batch_size,- }- if coordinator.is_master():- save_json(running_states, os.path.join(save_dir, "running_states.json"))- dist.barrier()- logger.info(- f"Saved checkpoint at epoch {epoch} step {step + 1} global_step {global_step + 1} to {exp_dir}"- )-- # the continue epochs are not resumed, so we need to reset the sampler start index and start step- dataloader.sampler.set_start_index(0)- start_step = 0--if __name__ == "__main__":- main()diff --git a/scripts/train-vae-v2.py b/scripts/train-vae.pysimilarity index 100%rename from scripts/train-vae-v2.pyrename to scripts/train-vae.py