Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

clean up code

Shen-Chenhui4d6967166301
diff --git a/configs/vae_magvit_v2/inference/1x256x256_pixabay.py b/configs/vae/inference/1x256x256.py
similarity index 100%
rename from configs/vae_magvit_v2/inference/1x256x256_pixabay.py
rename to configs/vae/inference/1x256x256.py
diff --git a/configs/vae_magvit_v2/train/17x256x256_pixabay.py b/configs/vae/train/17x256x256.py
similarity index 100%
rename from configs/vae_magvit_v2/train/17x256x256_pixabay.py
rename to configs/vae/train/17x256x256.py
diff --git a/configs/vae_magvit_v2/train/1x256x256_pixabay.py b/configs/vae/train/1x256x256.py
similarity index 100%
rename from configs/vae_magvit_v2/train/1x256x256_pixabay.py
rename to configs/vae/train/1x256x256.py
diff --git a/configs/vae_3d/inference/16x256x256.py b/configs/vae_3d/inference/16x256x256.py
deleted file mode 100644
index 5cda9bd..0000000
--- a/configs/vae_3d/inference/16x256x256.py
+++ /dev/null
@@ -1,56 +0,0 @@
-num_frames = 16
-frame_interval = 3
-fps = 24 // 3
-image_size = (256, 256)
-is_vae = True
-
-# Define dataset
-root = None
-data_path = "CSV_PATH"
-use_image_transform = False
-num_workers = 4
-
-# Define acceleration
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-# Define Loss
-kl_weight = 0.000001
-perceptual_weight = 0.1 # TODO: need to change this to 0.1 !!! according to MAGVIT paper
-
-# Define model
-
-model = dict(
- type="VAE_3D_B",
- from_pretrained="PRETRAINED_MODEL", # TODO: double check how this loads the pretrained model
- in_out_channels = 3,
- latent_embed_dim = 256,
- filters = 64,
- num_res_blocks = 2,
- channel_multipliers = (1, 2, 2, 4),
- temporal_downsample = (True, True, False),
- num_groups = 32, # for nn.GroupNorm
- conv_downsample = False,
- upsample = "nearest+conv", # options: "deconv", "nearest+conv"
- kl_embed_dim = 64,
- custom_conv_padding = None,
- activation_fn = 'swish',
-)
-
-
-# Others
-seed = 42
-save_dir = "outputs/samples"
-wandb = False
-
-
-epochs = 10
-log_every = 1
-ckpt_every = 1000
-load = None
-
-batch_size = 8
-grad_clip = 1.0
-lr = 1e-4
\ No newline at end of file
diff --git a/configs/vae_3d/train/16x256x256.py b/configs/vae_3d/train/16x256x256.py
deleted file mode 100644
index ab6e1ce..0000000
--- a/configs/vae_3d/train/16x256x256.py
+++ /dev/null
@@ -1,58 +0,0 @@
-num_frames = 16
-frame_interval = 3
-image_size = (256, 256)
-
-# Define dataset
-root = None
-data_path = "CSV_PATH"
-use_image_transform = False
-num_workers = 4
-
-# Define acceleration
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-# Define Loss
-kl_weight = 0.000001
-perceptual_weight = 0.1 # TODO: need to change this to 0.1 !!! according to MAGVIT paper
-
-# Define model
-
-model = dict(
- type="VAE_3D_B",
- in_out_channels = 3,
- latent_embed_dim = 4,
- filters = 64,
- num_res_blocks = 2,
- channel_multipliers = (1, 2, 2, 4),
- temporal_downsample = (True, True, False),
- num_groups = 32, # for nn.GroupNorm
- conv_downsample = False,
- upsample = "nearest+conv", # options: "deconv", "nearest+conv"
- kl_embed_dim = 4,
- custom_conv_padding = None,
- activation_fn = 'swish',
-)
-
-# Others
-seed = 42
-outputs = "outputs"
-wandb = False
-
-# Training
-''' NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-'''
-
-epochs = 2000
-log_every = 1
-ckpt_every = 1000
-load = None
-
-batch_size = 4
-lr = 1e-4
-grad_clip = 1.0
diff --git a/configs/vae_3d/vqgan2d_ucf101_config.py b/configs/vae_3d/vqgan2d_ucf101_config.py
deleted file mode 100644
index ee61cab..0000000
--- a/configs/vae_3d/vqgan2d_ucf101_config.py
+++ /dev/null
@@ -1,178 +0,0 @@
-# Copyright 2023 The videogvt Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=line-too-long
-r"""Configs for the VQGAN-2D on the UCF101.
-
-"""
-
-import ml_collections
-
-UCF101_TRAIN_SIZE = 9_537
-UCF101_TEST_SIZE = 3_783
-NUM_CLASSES = 101
-VARIANT = 'VQGAN/2D'
-
-
-def get_config(config_str='B'):
- """Returns the base experiment configuration."""
- version, *options = config_str.split('-')
-
- config = ml_collections.ConfigDict()
- config.experiment_name = f'UCF101_{VARIANT}'
-
- # Overall
- config.rng_seed = 0
- config.image_size = 128
- config.batch_size = 256
- config.eval_batch_size = config.get_ref('batch_size') // 4
- config.num_training_epochs = 1000
- config.lax_precision = 'default'
-
- # Dataset.
- config.dataset_name = 'ucf101'
- config.data_dtype_str = 'float32'
- config.dataset_configs = ml_collections.ConfigDict()
- config.dataset_configs.num_classes = NUM_CLASSES
- config.dataset_configs.num_frames = 16
- config.dataset_configs.stride = 1
- # TODO(Lijun-Yu, roadjiang): test this augmentation and other augmentations.
- # min_resize = int(min_resize / 224 * 256)
- config.dataset_configs.min_resize = config.get_oneway_ref('image_size')
- config.dataset_configs.crop_size = config.get_ref('image_size')
- config.dataset_configs.one_hot_label = False
- config.dataset_configs.zero_centering = False # Range is 0 to 1
- config.dataset_configs.num_test_clips = 1
- config.dataset_configs.prefetch_to_device = 2
- config.num_train_sampled_frames = 4
-
- # Model: vqvae
- model_class, model_type = VARIANT.split('/')
- config.dtype = config.get_ref('data_dtype_str')
- config.model_class = model_class
-
- config.pretrained_image_model = True
- config.perceptual_loss_weight = 0.1
- config.perceptual_loss_on_logit = True
- config.polyak_decay = 0.999 # ema decay factor for generator
-
- config.vqgan = ml_collections.ConfigDict()
- config.vqgan.model_type = model_type
- config.vqgan.loss_type = 'non-saturating'
- config.vqgan.g_adversarial_loss_weight = 0.1
- config.vqgan.gradient_penalty = 'r1' # r1, none
- config.vqgan.grad_penalty_cost = 10.0
- # finetune decoder
- config.vqgan.finetune_decoder = False
- config.vqgan.finetune_path = ''
-
- config.vqvae = ml_collections.ConfigDict()
- config.vqvae.architecture = '2dcnn'
- config.vqvae.codebook_size = 1024
- config.vqvae.entropy_loss_ratio = 0.1
- config.vqvae.entropy_temperature = 0.01
- config.vqvae.entropy_loss_type = 'softmax'
- config.vqvae.commitment_cost = 0.25
-
- config.vqvae.filters = {'B': 64, 'L': 128}[version]
- config.vqvae.num_enc_res_blocks = 2
- config.vqvae.num_dec_res_blocks = 2
- config.vqvae.channel_multipliers = (1, 2, 2, 4)
- config.vqvae.embedding_dim = 256
- config.vqvae.conv_downsample = False
- config.vqvae.deconv_upsample = False
- config.vqvae.activation_fn = 'swish'
- config.vqvae.norm_type = 'GN'
-
- config.discriminator = ml_collections.ConfigDict()
- config.discriminator.filters = config.vqvae.get_oneway_ref('filters')
- config.discriminator.channel_multipliers = (2, 4, 4, 4, 4)
-
- # Learning rate.
- config.base_lr = 0.0001
- config.optimizer = ml_collections.ConfigDict()
- config.optimizer.lr = config.get_ref('base_lr')
- config.optimizer.beta1 = 0.0
- config.optimizer.beta2 = 0.99
-
- config.optimizer.g_lr = config.get_ref('base_lr')
- config.optimizer.d_lr = config.get_ref('base_lr')
-
- steps_per_epoch = UCF101_TRAIN_SIZE // config.get_ref('batch_size')
- total_steps = config.get_ref('num_training_epochs') * steps_per_epoch
- config.lr_configs = ml_collections.ConfigDict()
- config.lr_configs.learning_rate_schedule = 'compound'
- config.lr_configs.factors = 'constant * cosine_decay * linear_warmup'
- config.lr_configs.warmup_steps = 1 * steps_per_epoch
- config.lr_configs.steps_per_cycle = total_steps
- config.lr_configs.base_learning_rate = config.get_ref('base_lr') # placeholder
-
- # Pretrained model on ImageNet.
- config.init_from = ml_collections.ConfigDict()
- config.init_from.checkpoint_path = {
- 'B': 'gs://magvit/models/imagenet_2d_base',
- 'L': 'gs://magvit/models/imagenet_2d_large',
- }[version]
-
- # Evaluation.
- config.eval = ml_collections.ConfigDict()
- config.eval.enable_inception_score = True
- config.eval.enable_frechet_distance = True
- config.eval.data_splits = 'train,validation'
- config.eval.num_examples = 10000
- config.eval.final_num_repeats = 1
- config.eval_from = ml_collections.ConfigDict()
- config.eval_from.checkpoint_path = None
- config.eval_from.step = None
-
- # Logging.
- config.logging = ml_collections.ConfigDict()
- config.logging.enable_checkpoint = True
- config.logging.checkpoint_steps = 1000
- config.logging.checkpoint_kept = 5
- config.logging.log_metric_steps = 200
- config.logging.log_sample_size = 2
-
- if 'runlocal' in options:
- config.batch_size = 16
- config.num_training_epochs = 10
- config.eval_batch_split = 1
-
- config.pretrained_image_model = False
- config.perceptual_loss_weight = 0.0
-
- config.vqvae.filters = 32
- config.vqvae.num_enc_res_blocks = 1
- config.vqvae.num_dec_res_blocks = 1
- config.vqvae.channel_multipliers = (1,) * len(
- config.vqvae.channel_multipliers)
- config.discriminator.channel_multipliers = (1,) * len(
- config.discriminator.channel_multipliers)
-
- config.logging.enable_checkpoint = False
- config.logging.checkpoint_steps = 100
- config.logging.log_metric_steps = 20
-
- del config.init_from
-
- # Standalone evaluation.
- if 'eval' in options:
- config.eval_only = True
- config.eval_from.checkpoint_path = ''
- config.eval_from.step = -1
- config.eval_from.legacy_checkpoint = True
-
- return config
-
-
diff --git a/configs/vae_3d/vqgan3d_ucf101_config.py b/configs/vae_3d/vqgan3d_ucf101_config.py
deleted file mode 100644
index a3875dc..0000000
--- a/configs/vae_3d/vqgan3d_ucf101_config.py
+++ /dev/null
@@ -1,102 +0,0 @@
-# Copyright 2023 The videogvt Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=line-too-long
-r"""
-Configs for the VQGAN-3D on the UCF101.
-
-usage: `config = vqgan3d_ucf101_config.get_config()`
-
-"""
-
-import ml_collections
-
-from opensora.configs.vae_3d import vqgan2d_ucf101_config
-
-VARIANT = 'VQGAN/3D'
-
-
-def get_config(config_str='B'):
- """Returns the base experiment configuration."""
- version, *options = config_str.split('-')
-
- config = vqgan2d_ucf101_config.get_config(config_str)
- config.experiment_name = f'UCF101_{VARIANT}'
- model_class, model_type = VARIANT.split('/')
-
- # Overall
- config.batch_size = {'B': 128, 'L': 256, 'H': 256}[version]
- config.eval_batch_size = config.get_ref('batch_size') // 4
- config.num_training_epochs = {'B': 500, 'L': 2000, 'H': 2000}[version]
-
- # Dataset.
- del config.num_train_sampled_frames
-
- # Model: vqvae
- config.model_class = model_class
-
- config.pretrained_image_model = True # TODO(Lijun-Yu): 3d perceptual loss
-
- config.vqgan.model_type = model_type
-
- config.vqvae.in_channels = 3 # NOTE: SCH: added
- config.vqvae.num_groups = 32 # NOTE: SCH: added, for nn.GroupNorm
- config.vqvae.architecture = '3dcnn'
- config.vqvae.codebook_size = {'B': 1024, 'L': 1024, 'H': 4096}[version]
- config.vqvae.filters = {'B': 64, 'L': 128, 'H': 128}[version]
- config.vqvae.num_enc_res_blocks = {'B': 2, 'L': 2, 'H': 4}[version]
- config.vqvae.num_dec_res_blocks = {'B': 2, 'L': 2, 'H': 4}[version]
- config.vqvae.channel_multipliers = (1, 2, 2, 4)
- config.vqvae.temporal_downsample = (True, True, False)
- config.vqvae.embedding_dim = {'B': 256, 'L': 256, 'H': 512}[version] # z embedding dimension
- config.vqvae.conv_downsample = False
- config.vqvae.deconv_upsample = False
-
- # Save memory
- config.vqvae.num_enc_remat_blocks = {'B': 0, 'L': 1, 'H': 3}[version]
- config.vqvae.num_dec_remat_blocks = config.vqvae.get_ref(
- 'num_enc_remat_blocks')
- config.discriminator.num_remat_blocks = config.vqvae.get_ref(
- 'num_enc_remat_blocks')
-
- # Pretrained models on ImageNet.
- config.init_from = ml_collections.ConfigDict()
- config.init_from.inflation = '2d->3d'
-
- # Standalone evaluation.
- if 'eval' in options:
- config.eval_only = True
- config.eval_from.checkpoint_path = {
- 'B': 'gs://magvit/models/ucf_3d_base',
- 'L': 'gs://magvit/models/ucf_3d_large',
- }[version]
- config.eval_from.step = -1
- config.eval_from.legacy_checkpoint = True
-
- if 'runlocal' in options:
- config.batch_size = 16
- config.num_training_epochs = 10
- # gets a small model for debugging
- config.vqvae.filters = 32
- config.vqvae.embedding_dim = 16
- config.vqvae.num_enc_res_blocks = 1
- config.vqvae.num_dec_res_blocks = 1
- config.discriminator.filters = 1
- config.discriminator.channel_multipliers = (1,)
- config.vqvae.channel_multipliers = (1,)
- config.vqvae.codebook_size = 128
-
- return config
-
-
diff --git a/configs/vae_magvit_v2/inference/16x128x128.py b/configs/vae_magvit_v2/inference/16x128x128.py
deleted file mode 100644
index 6d20a91..0000000
--- a/configs/vae_magvit_v2/inference/16x128x128.py
+++ /dev/null
@@ -1,87 +0,0 @@
-num_frames = 16
-frame_interval = 3
-image_size = (128, 128)
-fps = 24 // 3
-is_vae = True
-
-# Define dataset
-root = None
-data_path = "CSV_PATH"
-max_test_samples = -1
-use_image_transform = False
-num_workers = 4
-video_contains_first_frame = False
-
-# Define acceleration
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-
-# Define model
-
-model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels = 3,
- latent_embed_dim = 256,
- filters = 128,
- num_res_blocks = 4,
- channel_multipliers = (1, 2, 2, 4),
- temporal_downsample = (False, True, True),
- num_groups = 32, # for nn.GroupNorm
- kl_embed_dim = 64,
- activation_fn = 'swish',
- separate_first_frame_encoding = False,
- custom_conv_padding = None,
- encoder_double_z = False,
-)
-
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size = image_size,
- num_frames = num_frames,
- in_channels = 3,
- filters = 128,
- channel_multipliers = (2,4,4,4,4) # (2,4,4,4) for 64x64 resolution
-)
-
-
-# loss weights
-logvar_init=0.0
-kl_loss_weight = 0.000001
-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0 # for discriminator adversarial loss
-# discriminator_loss_weight = 0.5 # for generator adversarial loss
-generator_factor = 0.1 # for generator adversarial loss
-lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight
-discriminator_loss_type="non-saturating"
-generator_loss_type="non-saturating"
-discriminator_start = 50000 # 50000 NOTE: change to correct val, debug use -1 for now
-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
-ema_decay = 0.999 # ema decay factor for generator
-
-
-# Others
-seed = 42
-save_dir = "outputs/samples_v2"
-wandb = False
-
-# Training
-''' NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-'''
-
-epochs = 10
-log_every = 1
-ckpt_every = 1000
-load = None
-
-batch_size = 4
-lr = 1e-4
-grad_clip = 1.0
-
-calc_loss = False
\ No newline at end of file
diff --git a/configs/vae_magvit_v2/inference/16x128x128_z=4.py b/configs/vae_magvit_v2/inference/16x128x128_z=4.py
deleted file mode 100644
index d1e7c01..0000000
--- a/configs/vae_magvit_v2/inference/16x128x128_z=4.py
+++ /dev/null
@@ -1,87 +0,0 @@
-num_frames = 16
-frame_interval = 3
-image_size = (128, 128)
-fps = 24 // 3
-is_vae = True
-
-# Define dataset
-root = None
-data_path = "CSV_PATH"
-max_test_samples = -1
-use_image_transform = False
-num_workers = 4
-video_contains_first_frame = False
-
-# Define acceleration
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-
-# Define model
-
-model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels = 3,
- latent_embed_dim = 4,
- filters = 128,
- num_res_blocks = 4,
- channel_multipliers = (1, 2, 2, 4),
- temporal_downsample = (False, True, True),
- num_groups = 32, # for nn.GroupNorm
- kl_embed_dim = 4,
- activation_fn = 'swish',
- separate_first_frame_encoding = False,
- disable_space = False,
- custom_conv_padding = None
-)
-
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size = image_size,
- num_frames = num_frames,
- in_channels = 3,
- filters = 128,
- channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution
-)
-
-
-# loss weights
-logvar_init=0.0
-kl_loss_weight = 0.000001
-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0 # for discriminator adversarial loss
-# discriminator_loss_weight = 0.5 # for generator adversarial loss
-generator_factor = 0.1 # for generator adversarial loss
-lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight
-discriminator_loss_type="non-saturating"
-generator_loss_type="non-saturating"
-discriminator_start = 50000 # 50000 NOTE: change to correct val, debug use -1 for now
-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
-ema_decay = 0.999 # ema decay factor for generator
-
-
-# Others
-seed = 42
-save_dir = "outputs/samples_z4"
-wandb = False
-
-# Training
-''' NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-'''
-
-epochs = 10
-log_every = 1
-ckpt_every = 1000
-load = None
-
-batch_size = 4
-lr = 1e-4
-grad_clip = 1.0
-
-calc_loss = False
\ No newline at end of file
diff --git a/configs/vae_magvit_v2/inference/17x128x128_pixabay.py b/configs/vae_magvit_v2/inference/17x128x128_pixabay.py
deleted file mode 100644
index b264c2e..0000000
--- a/configs/vae_magvit_v2/inference/17x128x128_pixabay.py
+++ /dev/null
@@ -1,97 +0,0 @@
-num_frames = 17
-
-dataset = dict(
- type="VideoTextDataset",
- data_path=None,
- num_frames=num_frames,
- frame_interval=3,
- image_size=(128, 128),
- get_text=False,
-)
-
-fps = 24 // 3
-is_vae = True
-
-# Define dataset
-max_test_samples = -1
-
-
-# Define acceleration
-num_workers = 4
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-use_pipeline = True
-video_contains_first_frame = True
-
-
-# Define model
-
-vae_2d = dict(
- type="VideoAutoencoderKL",
- from_pretrained="stabilityai/sd-vae-ft-ema",
-)
-
-model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels=4,
- latent_embed_dim=64,
- filters=128,
- num_res_blocks=4,
- channel_multipliers=(1, 2, 2, 4),
- temporal_downsample=(False, True, True),
- num_groups=32, # for nn.GroupNorm
- kl_embed_dim=4,
- activation_fn="swish",
- separate_first_frame_encoding=False,
- disable_space=True,
- custom_conv_padding=None,
- encoder_double_z=True,
-)
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size=(128, 128),
- num_frames=num_frames,
- in_channels=3,
- filters=128,
- channel_multipliers=(2, 4, 4, 4, 4),
- # channel_multipliers = (2,4,4), #(2,4,4,4,4) # (2,4,4,4) for 64x64 resolution
-)
-
-
-# loss weights
-logvar_init = 0.0
-kl_loss_weight = 0.000001
-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0 # for discriminator adversarial loss
-# discriminator_loss_weight = 0.5 # for generator adversarial loss
-generator_factor = 0.1 # for generator adversarial loss
-lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight
-discriminator_loss_type = "non-saturating"
-generator_loss_type = "non-saturating"
-discriminator_start = 2500 # 50000 NOTE: change to correct val, debug use -1 for now
-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
-ema_decay = 0.999 # ema decay factor for generator
-
-
-# Others
-seed = 42
-save_dir = "outputs/samples_pixabay_17"
-wandb = False
-
-# Training
-""" NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-"""
-
-
-batch_size = 1
-lr = 1e-4
-grad_clip = 1.0
-
-calc_loss = True
diff --git a/configs/vae_magvit_v2/inference/1x128x128_pixabay.py b/configs/vae_magvit_v2/inference/1x128x128_pixabay.py
deleted file mode 100644
index e7511bc..0000000
--- a/configs/vae_magvit_v2/inference/1x128x128_pixabay.py
+++ /dev/null
@@ -1,97 +0,0 @@
-num_frames = 1
-
-dataset = dict(
- type="VideoTextDataset",
- data_path=None,
- num_frames=num_frames,
- frame_interval=3,
- image_size=(128, 128),
- get_text=False,
-)
-
-fps = 24 // 3
-is_vae = True
-
-# Define dataset
-max_test_samples = -1
-
-
-# Define acceleration
-num_workers = 4
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-use_pipeline = True
-video_contains_first_frame = True
-
-
-# Define model
-
-vae_2d = dict(
- type="VideoAutoencoderKL",
- from_pretrained="stabilityai/sd-vae-ft-ema",
-)
-
-model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels = 4,
- latent_embed_dim = 64,
- filters = 128,
- num_res_blocks = 4,
- channel_multipliers = (1, 2, 2, 4),
- temporal_downsample = (False, True, True),
- num_groups = 32, # for nn.GroupNorm
- kl_embed_dim = 4,
- activation_fn = 'swish',
- separate_first_frame_encoding = False,
- disable_space = True,
- custom_conv_padding = None,
- encoder_double_z = True,
-)
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size = (128, 128),
- num_frames = num_frames,
- in_channels = 3,
- filters = 128,
- channel_multipliers = (2,4,4,4,4),
- # channel_multipliers = (2,4,4), #(2,4,4,4,4) # (2,4,4,4) for 64x64 resolution
-)
-
-
-# loss weights
-logvar_init=0.0
-kl_loss_weight = 0.000001
-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0 # for discriminator adversarial loss
-# discriminator_loss_weight = 0.5 # for generator adversarial loss
-generator_factor = 0.1 # for generator adversarial loss
-lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight
-discriminator_loss_type="non-saturating"
-generator_loss_type="non-saturating"
-discriminator_start = 2500 # 50000 NOTE: change to correct val, debug use -1 for now
-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
-ema_decay = 0.999 # ema decay factor for generator
-
-
-# Others
-seed = 42
-save_dir = "outputs/samples_pixabay_17"
-wandb = False
-
-# Training
-''' NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-'''
-
-
-batch_size = 1
-lr = 1e-4
-grad_clip = 1.0
-
-calc_loss = True
\ No newline at end of file
diff --git a/configs/vae_magvit_v2/inference/pipeline_16x128x128.py b/configs/vae_magvit_v2/inference/pipeline_16x128x128.py
deleted file mode 100644
index 540d63d..0000000
--- a/configs/vae_magvit_v2/inference/pipeline_16x128x128.py
+++ /dev/null
@@ -1,96 +0,0 @@
-num_frames = 16
-frame_interval = 3
-image_size = (128, 128)
-fps = 24 // 3
-is_vae = True
-use_pipeline = True
-
-# Define dataset
-root = None
-data_path = "CSV_PATH"
-max_test_samples = -1
-use_image_transform = False
-num_workers = 4
-video_contains_first_frame = False
-
-
-
-# Define acceleration
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-
-# Define model
-
-vae_2d = dict(
- type="VideoAutoencoderKL",
- from_pretrained="stabilityai/sd-vae-ft-ema",
-)
-
-model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels = 4,
- latent_embed_dim = 256,
- filters = 128,
- num_res_blocks = 4,
- channel_multipliers = (1, 2, 2, 4),
- temporal_downsample = (False, True, True),
- num_groups = 32, # for nn.GroupNorm
- kl_embed_dim = 64,
- activation_fn = 'swish',
- separate_first_frame_encoding = False,
- disable_space = True,
- custom_conv_padding = None,
- encoder_double_z = False,
-)
-
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size = 16,
- num_frames = num_frames,
- in_channels = 4,
- filters = 128,
- channel_multipliers = (2,4,4), #(2,4,4,4,4) # (2,4,4,4) for 64x64 resolution
-)
-
-
-# loss weights
-logvar_init=0.0
-kl_loss_weight = 0.000001
-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0 # for discriminator adversarial loss
-# discriminator_loss_weight = 0.5 # for generator adversarial loss
-generator_factor = 0.1 # for generator adversarial loss
-lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight
-discriminator_loss_type="non-saturating"
-generator_loss_type="non-saturating"
-discriminator_start = 2500 # 50000 NOTE: change to correct val, debug use -1 for now
-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
-ema_decay = 0.999 # ema decay factor for generator
-
-
-# Others
-seed = 42
-save_dir = "outputs/pipeline_samples_v2"
-wandb = False
-
-# Training
-''' NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-'''
-
-epochs = 10
-log_every = 1
-ckpt_every = 500
-load = None
-
-batch_size = 4
-lr = 1e-4
-grad_clip = 1.0
-
-calc_loss = True
\ No newline at end of file
diff --git a/configs/vae_magvit_v2/train/16x128x128.py b/configs/vae_magvit_v2/train/16x128x128.py
deleted file mode 100644
index 9e71b8d..0000000
--- a/configs/vae_magvit_v2/train/16x128x128.py
+++ /dev/null
@@ -1,84 +0,0 @@
-num_frames = 16
-frame_interval = 3
-image_size = (128, 128)
-
-# Define dataset
-root = None
-data_path = "CSV_PATH"
-use_image_transform = False
-num_workers = 4
-video_contains_first_frame = False
-
-# Define acceleration
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-
-# Define model
-
-model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels = 3,
- latent_embed_dim = 256,
- filters = 128,
- num_res_blocks = 4,
- channel_multipliers = (1, 2, 2, 4),
- temporal_downsample = (False, True, True),
- num_groups = 32, # for nn.GroupNorm
- kl_embed_dim = 64,
- activation_fn = 'swish',
- separate_first_frame_encoding = False,
- disable_space = False,
- custom_conv_padding = None
-)
-
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size = image_size,
- num_frames = num_frames,
- in_channels = 3,
- filters = 128,
- channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution
-)
-
-
-# loss weights
-logvar_init=0.0
-kl_loss_weight = 0.000001
-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0 # for discriminator adversarial loss
-# discriminator_loss_weight = 0.5 # for generator adversarial loss
-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1
-lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001
-# discriminator_loss_type="non-saturating"
-# generator_loss_type="non-saturating"
-discriminator_loss_type="hinge"
-generator_loss_type="hinge"
-discriminator_start = 30000 # 50000 NOTE: change to correct val, debug use -1 for now
-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
-ema_decay = 0.999 # ema decay factor for generator
-
-
-# Others
-seed = 42
-outputs = "outputs"
-wandb = False
-
-# Training
-''' NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-'''
-
-epochs = 200
-log_every = 1
-ckpt_every = 200
-load = None
-
-batch_size = 4
-lr = 1e-4
-grad_clip = 1.0
diff --git a/configs/vae_magvit_v2/train/16x128x128_pixabay_8_GPU_pipeline.py b/configs/vae_magvit_v2/train/16x128x128_pixabay_8_GPU_pipeline.py
deleted file mode 100644
index 7b2c60c..0000000
--- a/configs/vae_magvit_v2/train/16x128x128_pixabay_8_GPU_pipeline.py
+++ /dev/null
@@ -1,93 +0,0 @@
-dataset = dict(
- type="VideoTextDataset",
- data_path=None,
- num_frames=16,
- frame_interval=3,
- image_size=(128, 128),
- get_text=False,
-)
-
-# Define acceleration
-num_workers = 4
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-use_pipeline = True
-video_contains_first_frame = False
-
-# Define model
-vae_2d = dict(
- type="VideoAutoencoderKL",
- from_pretrained="stabilityai/sd-vae-ft-ema",
- # SDXL
-)
-model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels=4,
- latent_embed_dim=4,
- filters=128,
- num_res_blocks=4,
- channel_multipliers=(1, 2, 2, 4),
- temporal_downsample=(False, True, True),
- num_groups=32, # for nn.GroupNorm
- kl_embed_dim=4,
- activation_fn="swish",
- separate_first_frame_encoding=False,
- disable_space=True,
- encoder_double_z=True,
- custom_conv_padding=None,
-)
-
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size=(16, 16), # NOTE: here image size is different
- num_frames=16,
- in_channels=4,
- filters=128,
- use_pretrained=True, # NOTE: set to False only if we want to disable load
- # channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution
- channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z
-)
-
-
-# loss weights
-logvar_init = 0.0
-kl_loss_weight = 0.000001
-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0 # for discriminator adversarial loss
-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1
-lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001
-discriminator_loss_type = "non-saturating"
-generator_loss_type = "non-saturating"
-# discriminator_loss_type="hinge"
-# generator_loss_type="hinge"
-discriminator_start = 100 # 8k data / (8*32) = 31 steps per epoch, use around 3 epochs
-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
-ema_decay = 0.999 # ema decay factor for generator
-
-
-# Others
-seed = 42
-outputs = "outputs"
-wandb = False
-
-# Training
-""" NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-"""
-
-epochs = 200
-log_every = 1
-
-ckpt_every = 50
-load = None
-
-batch_size = 32
-
-lr = 1e-4
-grad_clip = 1.0
diff --git a/configs/vae_magvit_v2/train/16x128x128_trial.py b/configs/vae_magvit_v2/train/16x128x128_trial.py
deleted file mode 100644
index e9693c9..0000000
--- a/configs/vae_magvit_v2/train/16x128x128_trial.py
+++ /dev/null
@@ -1,83 +0,0 @@
-num_frames = 16
-frame_interval = 3
-image_size = (128, 128)
-
-# Define dataset
-root = None
-data_path = "CSV_PATH"
-use_image_transform = False
-num_workers = 4
-video_contains_first_frame = False
-
-# Define acceleration
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-
-# Define model
-
-model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels = 3,
- latent_embed_dim = 256,
- filters = 128,
- num_res_blocks = 4,
- channel_multipliers = (1, 2, 2, 4),
- temporal_downsample = (False, True, True),
- num_groups = 32, # for nn.GroupNorm
- kl_embed_dim = 64,
- activation_fn = 'swish',
- separate_first_frame_encoding = False,
- disable_space = False,
- encoder_double_z = False,
- custom_conv_padding = None
-)
-
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size = image_size,
- num_frames = num_frames,
- in_channels = 3,
- filters = 128,
- channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution
-)
-
-
-# loss weights
-logvar_init=0.0
-kl_loss_weight = 0.000001
-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0 # for discriminator adversarial loss
-# discriminator_loss_weight = 0.5 # for generator adversarial loss
-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1
-lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001
-discriminator_loss_type="non-saturating"
-generator_loss_type="non-saturating"
-discriminator_start = 1000 # 50000 NOTE: change to correct val, debug use -1 for now
-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
-ema_decay = 0.999 # ema decay factor for generator
-
-
-# Others
-seed = 42
-outputs = "outputs"
-wandb = False
-
-# Training
-''' NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-'''
-
-epochs = 2000
-log_every = 1
-ckpt_every = 1000
-load = None
-
-batch_size = 4
-lr = 1e-4
-grad_clip = 1.0
diff --git a/configs/vae_magvit_v2/train/16x128x128_z4_LC.py b/configs/vae_magvit_v2/train/16x128x128_z4_LC.py
deleted file mode 100644
index 2cb7cb6..0000000
--- a/configs/vae_magvit_v2/train/16x128x128_z4_LC.py
+++ /dev/null
@@ -1,82 +0,0 @@
-num_frames = 16
-frame_interval = 3
-image_size = (128, 128)
-
-# Define dataset
-root = None
-data_path = "CSV_PATH"
-use_image_transform = False
-num_workers = 4
-video_contains_first_frame = False
-
-# Define acceleration
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-
-# Define model
-
-model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels = 3,
- latent_embed_dim = 4,
- filters = 128,
- num_res_blocks = 4,
- channel_multipliers = (1, 2, 2, 4),
- temporal_downsample = (False, True, True),
- num_groups = 32, # for nn.GroupNorm
- kl_embed_dim = 4,
- activation_fn = 'swish',
- separate_first_frame_encoding = False,
- disable_space = False,
- custom_conv_padding = None
-)
-
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size = image_size,
- num_frames = num_frames,
- in_channels = 3,
- filters = 128,
- channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution
-)
-
-
-# loss weights
-logvar_init=0.0
-kl_loss_weight = 0.000001
-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0 # for discriminator adversarial loss
-# discriminator_loss_weight = 0.5 # for generator adversarial loss
-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1
-lecam_loss_weight = 0.001 # NOTE: MAVGIT v2 use 0.001
-discriminator_loss_type="non-saturating"
-generator_loss_type="non-saturating"
-discriminator_start = -1 # 50000 NOTE: change to correct val, debug use -1 for now
-gradient_penalty_loss_weight = 10 # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
-ema_decay = 0.999 # ema decay for LeCam
-
-
-# Others
-seed = 42
-outputs = "outputs"
-wandb = False
-
-# Training
-''' NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-'''
-
-epochs = 2000
-log_every = 1
-ckpt_every = 1000
-load = None
-
-batch_size = 4
-lr = 1e-4
-grad_clip = 1.0
diff --git a/configs/vae_magvit_v2/train/16x128x128_z=4.py b/configs/vae_magvit_v2/train/16x128x128_z=4.py
deleted file mode 100644
index f252903..0000000
--- a/configs/vae_magvit_v2/train/16x128x128_z=4.py
+++ /dev/null
@@ -1,82 +0,0 @@
-num_frames = 16
-frame_interval = 3
-image_size = (128, 128)
-
-# Define dataset
-root = None
-data_path = "CSV_PATH"
-use_image_transform = False
-num_workers = 4
-video_contains_first_frame = False
-
-# Define acceleration
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-
-# Define model
-
-model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels = 3,
- latent_embed_dim = 4,
- filters = 128,
- num_res_blocks = 4,
- channel_multipliers = (1, 2, 2, 4),
- temporal_downsample = (False, True, True),
- num_groups = 32, # for nn.GroupNorm
- kl_embed_dim = 4,
- activation_fn = 'swish',
- separate_first_frame_encoding = False,
- disable_space = False,
- custom_conv_padding = None
-)
-
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size = image_size,
- num_frames = num_frames,
- in_channels = 3,
- filters = 128,
- channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution
-)
-
-
-# loss weights
-logvar_init=0.0
-kl_loss_weight = 0.000001
-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0 # for discriminator adversarial loss
-# discriminator_loss_weight = 0.5 # for generator adversarial loss
-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1
-lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001
-discriminator_loss_type="non-saturating"
-generator_loss_type="non-saturating"
-discriminator_start = 1000 # 50000 NOTE: change to correct val, debug use -1 for now
-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
-ema_decay = 0.999 # ema decay factor for generator
-
-
-# Others
-seed = 42
-outputs = "outputs"
-wandb = False
-
-# Training
-''' NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-'''
-
-epochs = 2000
-log_every = 1
-ckpt_every = 1000
-load = None
-
-batch_size = 4
-lr = 1e-4
-grad_clip = 1.0
diff --git a/configs/vae_magvit_v2/train/17x128x128.py b/configs/vae_magvit_v2/train/17x128x128.py
deleted file mode 100644
index 0397924..0000000
--- a/configs/vae_magvit_v2/train/17x128x128.py
+++ /dev/null
@@ -1,95 +0,0 @@
-num_frames = 17
-image_size = (128, 128)
-dataset = dict(
- type="VideoTextDataset",
- data_path=None,
- num_frames=num_frames,
- frame_interval=3,
- image_size=image_size,
- get_text=False,
-)
-
-# Define acceleration
-num_workers = 4
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-use_pipeline = True
-video_contains_first_frame = True
-
-# Define model
-vae_2d = dict(
- type="VideoAutoencoderKL",
- from_pretrained="stabilityai/sd-vae-ft-ema",
- # SDXL
-)
-model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels=4,
- latent_embed_dim=64,
- filters=128,
- num_res_blocks=4,
- channel_multipliers=(1, 2, 2, 4),
- temporal_downsample=(False, True, True),
- num_groups=32, # for nn.GroupNorm
- kl_embed_dim=4,
- activation_fn="swish",
- separate_first_frame_encoding=False,
- disable_space=True,
- encoder_double_z=True,
- custom_conv_padding=None,
-)
-
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size=image_size, # NOTE: here image size is different
- num_frames=num_frames,
- in_channels=3,
- filters=128,
- use_pretrained=True, # NOTE: set to False only if we want to disable load
- channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution
- # channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z
-)
-
-
-# loss weights
-logvar_init = 0.0
-kl_loss_weight = 0.000001
-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0 # for discriminator adversarial loss
-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1
-lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001
-discriminator_loss_type = "non-saturating"
-generator_loss_type = "non-saturating"
-# discriminator_loss_type="hinge"
-# generator_loss_type="hinge"
-discriminator_start = 2000 # 5000 # 8k data / (8*1) = 1000 steps per epoch
-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
-ema_decay = 0.999 # ema decay factor for generator
-
-
-# Others
-seed = 42
-outputs = "outputs"
-wandb = False
-
-# Training
-""" NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-"""
-
-epochs = 500
-log_every = 1
-
-ckpt_every = 500
-load = None
-
-batch_size = 1
-
-lr = 1e-4
-grad_clip = 1.0
diff --git a/configs/vae_magvit_v2/train/17x128x128_LC_GP.py b/configs/vae_magvit_v2/train/17x128x128_LC_GP.py
deleted file mode 100644
index de31bdb..0000000
--- a/configs/vae_magvit_v2/train/17x128x128_LC_GP.py
+++ /dev/null
@@ -1,95 +0,0 @@
-num_frames = 17
-image_size = (128, 128)
-dataset = dict(
- type="VideoTextDataset",
- data_path=None,
- num_frames=num_frames,
- frame_interval=3,
- image_size=image_size,
- get_text=False,
-)
-
-# Define acceleration
-num_workers = 4
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-use_pipeline = True
-video_contains_first_frame = True
-
-# Define model
-vae_2d = dict(
- type="VideoAutoencoderKL",
- from_pretrained="stabilityai/sd-vae-ft-ema",
- # SDXL
-)
-model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels=4,
- latent_embed_dim=64,
- filters=128,
- num_res_blocks=4,
- channel_multipliers=(1, 2, 2, 4),
- temporal_downsample=(False, True, True),
- num_groups=32, # for nn.GroupNorm
- kl_embed_dim=4,
- activation_fn="swish",
- separate_first_frame_encoding=False,
- disable_space=True,
- encoder_double_z=True,
- custom_conv_padding=None,
-)
-
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size=image_size, # NOTE: here image size is different
- num_frames=num_frames,
- in_channels=3,
- filters=128,
- use_pretrained=True, # NOTE: set to False only if we want to disable load
- channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution
- # channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z
-)
-
-
-# loss weights
-logvar_init = 0.0
-kl_loss_weight = 0.000001
-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0 # for discriminator adversarial loss
-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1
-lecam_loss_weight = 0.001 # NOTE: MAVGIT v2 use 0.001
-discriminator_loss_type = "non-saturating"
-generator_loss_type = "non-saturating"
-# discriminator_loss_type="hinge"
-# generator_loss_type="hinge"
-discriminator_start = 2000 # 5000 # 8k data / (8*1) = 1000 steps per epoch
-gradient_penalty_loss_weight = 10 # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
-ema_decay = 0.999 # ema decay factor for generator
-
-
-# Others
-seed = 42
-outputs = "outputs"
-wandb = False
-
-# Training
-""" NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-"""
-
-epochs = 500
-log_every = 1
-
-ckpt_every = 500
-load = None
-
-batch_size = 1
-
-lr = 1e-4
-grad_clip = 1.0
diff --git a/configs/vae_magvit_v2/train/17x128x128_pixabay_8_GPU_pipeline.py b/configs/vae_magvit_v2/train/17x128x128_pixabay_8_GPU_pipeline.py
deleted file mode 100644
index fe69e94..0000000
--- a/configs/vae_magvit_v2/train/17x128x128_pixabay_8_GPU_pipeline.py
+++ /dev/null
@@ -1,94 +0,0 @@
-num_frames = 17
-dataset = dict(
- type="VideoTextDataset",
- data_path=None,
- num_frames=num_frames,
- frame_interval=3,
- image_size=(128, 128),
- get_text=False,
-)
-
-# Define acceleration
-num_workers = 4
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-use_pipeline = True
-video_contains_first_frame = True
-
-# Define model
-vae_2d = dict(
- type="VideoAutoencoderKL",
- from_pretrained="stabilityai/sd-vae-ft-ema",
- # SDXL
-)
-model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels=4,
- latent_embed_dim=64,
- filters=128,
- num_res_blocks=4,
- channel_multipliers=(1, 2, 2, 4),
- temporal_downsample=(False, True, True),
- num_groups=32, # for nn.GroupNorm
- kl_embed_dim=4,
- activation_fn="swish",
- separate_first_frame_encoding=False,
- disable_space=True,
- encoder_double_z=True,
- custom_conv_padding=None,
-)
-
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size=(16, 16), # NOTE: here image size is different
- num_frames=num_frames,
- in_channels=4,
- filters=128,
- use_pretrained=True, # NOTE: set to False only if we want to disable load
- # channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution
- channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z
-)
-
-
-# loss weights
-logvar_init = 0.0
-kl_loss_weight = 0.000001
-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0 # for discriminator adversarial loss
-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1
-lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001
-discriminator_loss_type = "non-saturating"
-generator_loss_type = "non-saturating"
-# discriminator_loss_type="hinge"
-# generator_loss_type="hinge"
-discriminator_start = 5000 # 8k data / (8*1) = 1000 steps per epoch
-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
-ema_decay = 0.999 # ema decay factor for generator
-
-
-# Others
-seed = 42
-outputs = "outputs"
-wandb = False
-
-# Training
-""" NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-"""
-
-epochs = 20
-log_every = 1
-
-ckpt_every = 500
-load = None
-
-batch_size = 1
-
-lr = 1e-4
-grad_clip = 1.0
diff --git a/configs/vae_magvit_v2/train/1x256x256_small_lr.py b/configs/vae_magvit_v2/train/1x256x256_small_lr.py
deleted file mode 100644
index 7ceca93..0000000
--- a/configs/vae_magvit_v2/train/1x256x256_small_lr.py
+++ /dev/null
@@ -1,105 +0,0 @@
-num_frames = 1
-image_size = (256, 256)
-dataset = dict(
- type="VideoTextDataset",
- data_path=None,
- num_frames=num_frames,
- frame_interval=1,
- image_size=image_size,
- get_text=False,
-)
-
-# Define acceleration
-num_workers = 4
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-use_pipeline = True
-video_contains_first_frame = True
-
-# Define model
-# vae_2d = dict(
-# type="VideoAutoencoderKL",
-# from_pretrained="stabilityai/sd-vae-ft-ema",
-# # SDXL
-# )
-
-# latest
-vae_2d = dict(
- type="VideoAutoencoderKL",
- from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
- subfolder="vae",
- micro_batch_size=4,
- local_files_only=True,
-)
-
-model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels=4,
- latent_embed_dim=64,
- filters=128,
- num_res_blocks=4,
- channel_multipliers=(1, 2, 2, 4),
- temporal_downsample=(False, True, True),
- num_groups=32, # for nn.GroupNorm
- kl_embed_dim=4,
- activation_fn="swish",
- separate_first_frame_encoding=False,
- disable_space=True,
- custom_conv_padding=None,
- encoder_double_z=True,
-)
-
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size=image_size, # NOTE: here image size is different
- num_frames=num_frames,
- in_channels=3,
- filters=128,
- use_pretrained=True, # NOTE: set to False only if we want to disable load
- channel_multipliers=(2, 4, 4, 4, 4), # (2,4,4,4) for 64x64 resolution
- # channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z
-)
-
-
-# loss weights
-logvar_init = 0.0
-kl_loss_weight = 0.000001
-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0 # for discriminator adversarial loss
-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1
-lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001
-discriminator_loss_type = "non-saturating"
-generator_loss_type = "non-saturating"
-# discriminator_loss_type="hinge"
-# generator_loss_type="hinge"
-discriminator_start = 2000 # 5000 # 8k data / (8*1) = 1000 steps per epoch
-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
-ema_decay = 0.999 # ema decay factor for generator
-
-
-# Others
-seed = 42
-outputs = "outputs"
-wandb = False
-
-# Training
-""" NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-"""
-
-epochs = 100
-log_every = 1
-
-ckpt_every = 500
-load = None
-
-batch_size = 1
-
-lr = 1e-5
-grad_clip = 1.0
diff --git a/configs/vae_magvit_v2/train/pipeline_16x128x128.py b/configs/vae_magvit_v2/train/pipeline_16x128x128.py
deleted file mode 100644
index 3aa6f28..0000000
--- a/configs/vae_magvit_v2/train/pipeline_16x128x128.py
+++ /dev/null
@@ -1,93 +0,0 @@
-num_frames = 16
-frame_interval = 3
-image_size = (128, 128)
-use_pipeline = True
-
-# Define dataset
-root = None
-data_path = "CSV_PATH"
-use_image_transform = False
-num_workers = 4
-video_contains_first_frame = False
-
-# Define acceleration
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-
-# Define model
-vae_2d = dict(
- type="VideoAutoencoderKL",
- from_pretrained="stabilityai/sd-vae-ft-ema",
- # SDXL
-)
-
-model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels = 4,
- latent_embed_dim = 256,
- filters = 128,
- num_res_blocks = 4,
- channel_multipliers = (1, 2, 2, 4),
- temporal_downsample = (False, True, True),
- num_groups = 32, # for nn.GroupNorm
- kl_embed_dim = 64,
- activation_fn = 'swish',
- separate_first_frame_encoding = False,
- disable_space = True,
- encoder_double_z = False,
- custom_conv_padding = None
-)
-
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size = (16, 16), # NOTE: here image size is different
- num_frames = num_frames,
- in_channels = 4,
- filters = 128,
- use_pretrained=True, # NOTE: set to False only if we want to fresh train using a different discriminator!
- # channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution
- channel_multipliers= (2,4,4) # since on intermediate layer, 16 x 16 x 16 dimension z
-)
-
-
-# loss weights
-logvar_init=0.0
-kl_loss_weight = 0.000001
-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0 # for discriminator adversarial loss
-# discriminator_loss_weight = 0.5 # for generator adversarial loss
-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1
-lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001
-# discriminator_loss_type="non-saturating"
-# generator_loss_type="non-saturating"
-discriminator_loss_type="hinge"
-generator_loss_type="hinge"
-discriminator_start = 1000 # 50000 NOTE: change to correct val, debug use -1 for now
-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
-ema_decay = 0.999 # ema decay factor for generator
-
-
-# Others
-seed = 42
-outputs = "outputs"
-wandb = False
-
-# Training
-''' NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-'''
-
-epochs = 2000
-log_every = 1
-ckpt_every = 1000
-load = None
-
-batch_size = 4
-lr = 1e-4
-grad_clip = 1.0
diff --git a/configs/vae_magvit_v2/train/pipeline_16x128x128_LC.py b/configs/vae_magvit_v2/train/pipeline_16x128x128_LC.py
deleted file mode 100644
index fc3c832..0000000
--- a/configs/vae_magvit_v2/train/pipeline_16x128x128_LC.py
+++ /dev/null
@@ -1,93 +0,0 @@
-num_frames = 16
-frame_interval = 3
-image_size = (128, 128)
-use_pipeline = True
-
-# Define dataset
-root = None
-data_path = "CSV_PATH"
-use_image_transform = False
-num_workers = 4
-video_contains_first_frame = False
-
-# Define acceleration
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-
-# Define model
-vae_2d = dict(
- type="VideoAutoencoderKL",
- from_pretrained="stabilityai/sd-vae-ft-ema",
- # SDXL
-)
-
-model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels = 4,
- latent_embed_dim = 256,
- filters = 128,
- num_res_blocks = 4,
- channel_multipliers = (1, 2, 2, 4),
- temporal_downsample = (False, True, True),
- num_groups = 32, # for nn.GroupNorm
- kl_embed_dim = 64,
- activation_fn = 'swish',
- separate_first_frame_encoding = False,
- disable_space = True,
- encoder_double_z = False,
- custom_conv_padding = None
-)
-
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size = (16, 16), # NOTE: here image size is different
- num_frames = num_frames,
- in_channels = 4,
- filters = 128,
- use_pretrained=True, # NOTE: set to False only if we want to fresh train using a different discriminator!
- # channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution
- channel_multipliers= (2,4,4) # since on intermediate layer, 16 x 16 x 16 dimension z
-)
-
-
-# loss weights
-logvar_init=0.0
-kl_loss_weight = 0.000001
-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0 # for discriminator adversarial loss
-# discriminator_loss_weight = 0.5 # for generator adversarial loss
-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1
-lecam_loss_weight = 0.001 # NOTE: MAVGIT v2 use 0.001
-# discriminator_loss_type="non-saturating"
-# generator_loss_type="non-saturating"
-discriminator_loss_type="hinge"
-generator_loss_type="hinge"
-discriminator_start = 1000 # 50000 NOTE: change to correct val, debug use -1 for now
-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
-ema_decay = 0.999 # ema decay factor for generator
-
-
-# Others
-seed = 42
-outputs = "outputs"
-wandb = False
-
-# Training
-''' NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-'''
-
-epochs = 2000
-log_every = 1
-ckpt_every = 1000
-load = None
-
-batch_size = 4
-lr = 1e-4
-grad_clip = 1.0
diff --git a/configs/vae_magvit_v2/train/pipeline_16x128x128_LC_GP.py b/configs/vae_magvit_v2/train/pipeline_16x128x128_LC_GP.py
deleted file mode 100644
index ae1f975..0000000
--- a/configs/vae_magvit_v2/train/pipeline_16x128x128_LC_GP.py
+++ /dev/null
@@ -1,93 +0,0 @@
-num_frames = 16
-frame_interval = 3
-image_size = (128, 128)
-use_pipeline = True
-
-# Define dataset
-root = None
-data_path = "CSV_PATH"
-use_image_transform = False
-num_workers = 4
-video_contains_first_frame = False
-
-# Define acceleration
-dtype = "bf16"
-grad_checkpoint = True
-plugin = "zero2"
-sp_size = 1
-
-
-# Define model
-vae_2d = dict(
- type="VideoAutoencoderKL",
- from_pretrained="stabilityai/sd-vae-ft-ema",
- # SDXL
-)
-
-model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels = 4,
- latent_embed_dim = 256,
- filters = 128,
- num_res_blocks = 4,
- channel_multipliers = (1, 2, 2, 4),
- temporal_downsample = (False, True, True),
- num_groups = 32, # for nn.GroupNorm
- kl_embed_dim = 64,
- activation_fn = 'swish',
- separate_first_frame_encoding = False,
- disable_space = True,
- encoder_double_z = False,
- custom_conv_padding = None
-)
-
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size = (16, 16), # NOTE: here image size is different
- num_frames = num_frames,
- in_channels = 4,
- filters = 128,
- use_pretrained=True, # NOTE: set to False only if we want to fresh train using a different discriminator!
- # channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution
- channel_multipliers= (2,4,4) # since on intermediate layer, 16 x 16 x 16 dimension z
-)
-
-
-# loss weights
-logvar_init=0.0
-kl_loss_weight = 0.000001
-perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
-discriminator_factor = 1.0 # for discriminator adversarial loss
-# discriminator_loss_weight = 0.5 # for generator adversarial loss
-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1
-lecam_loss_weight = 0.001 # NOTE: MAVGIT v2 use 0.001
-# discriminator_loss_type="non-saturating"
-# generator_loss_type="non-saturating"
-discriminator_loss_type="hinge"
-generator_loss_type="hinge"
-discriminator_start = 1000 # 50000 NOTE: change to correct val, debug use -1 for now
-gradient_penalty_loss_weight = 10 # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
-ema_decay = 0.999 # ema decay factor for generator
-
-
-# Others
-seed = 42
-outputs = "outputs"
-wandb = False
-
-# Training
-''' NOTE:
-magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
-==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
-3-6 epochs for pexel, from pexel observation its correct
-'''
-
-epochs = 2000
-log_every = 1
-ckpt_every = 1000
-load = None
-
-batch_size = 4
-lr = 1e-4
-grad_clip = 1.0
diff --git a/opensora/models/vae/__init__.py b/opensora/models/vae/__init__.py
index 1dbf956..8b123a2 100644
--- a/opensora/models/vae/__init__.py
+++ b/opensora/models/vae/__init__.py
@@ -1,4 +1,3 @@
from .vae import VideoAutoencoderKL, VideoAutoencoderKLTemporalDecoder
-from .vae_3d import VAE_3D, VAE_3D_B
from .vae_3d_v2 import VAE_3D_V2, VAE_MAGVIT_V2
diff --git a/opensora/models/vae/discriminator_3d.py b/opensora/models/vae/discriminator_3d.py
deleted file mode 100644
index 34b0bc2..0000000
--- a/opensora/models/vae/discriminator_3d.py
+++ /dev/null
@@ -1,126 +0,0 @@
-"""3D StyleGAN discriminator."""
-
-import functools
-import math
-from typing import Any
-
-import torch
-import torch.nn as nn
-
-# TODO: torch.nn.init.xavier_uniform_
-# default_kernel_init = nn.initializers.xavier_uniform()
-
-def xavier_uniform_weight_init(m):
- if isinstance(m, nn.Conv3d) or isinstance(m, nn.Linear):
- nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain('relu'))
- if m.bias is not None:
- nn.init.zeros_(m.bias)
- # print("initialized module to xavier_uniform:", m)
-
-class ResBlock(nn.Module):
- """3D StyleGAN ResBlock for D."""
-
- def __init__(
- self,
- in_channels,
- filters,
- activation_fn,
- num_groups=32,
- device="cpu",
- dtype=torch.bfloat16,
- ):
- super().__init__()
-
- self.filters = filters
- self.activation_fn = activation_fn
-
- # SCH: NOTE: although paper says conv (X->Y, Y->Y), original code implementation is (X->X, X->Y), we follow code
- self.conv1 = nn.Conv3d(in_channels, in_channels, (3,3,3), padding=1, dtype=dtype, device=device) # TODO: need to init to xavier_uniform
- self.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)
- self.avg_pool_with_t = nn.AvgPool3d((2,2,2),count_include_pad=False)
- self.conv2 = nn.Conv3d(in_channels, self.filters,(1,1,1), bias=False, dtype=dtype, device=device) # need to init to xavier_uniform
- self.conv3 = nn.Conv3d(in_channels, self.filters, (3,3,3), padding=1, dtype=dtype, device=device) # need to init to xavier_uniform
- self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)
- self.apply(xavier_uniform_weight_init)
-
- def forward(self, x):
- residual = x
- x = self.conv1(x)
- x = self.norm1(x)
- x = self.activation_fn(x)
-
- x = self.avg_pool_with_t(x)
- residual = self.avg_pool_with_t(residual)
- residual = self.conv2(residual)
- x = self.conv3(x)
- x = self.norm2(x)
- x = self.activation_fn(x)
- out = (residual + x) / math.sqrt(2)
- return out
-
-class StyleGANDiscriminator(nn.Module):
- """StyleGAN Discriminator."""
- def __init__(
- self,
- image_size = (128, 128),
- num_frames = 17,
- in_channels = 3,
- filters = 128,
- channel_multipliers = (2,4,4,4,4),
- num_groups=32,
- dtype = torch.bfloat16,
- device="cpu",
- ):
- super().__init__()
- self.dtype = dtype
- self.input_size = image_size
- self.filters = filters
- self.activation_fn = nn.LeakyReLU(negative_slope=0.2)
- self.channel_multipliers = channel_multipliers
-
- self.conv1 = nn.Conv3d(in_channels, self.filters, (3, 3, 3), padding=1, dtype=dtype, device=device) # need to init to xavier_uniform
-
- prev_filters = self.filters # record in_channels
- self.num_blocks = len(self.channel_multipliers)
- self.res_block_list = []
- for i in range(self.num_blocks):
- filters = self.filters * self.channel_multipliers[i]
- self.res_block_list.append(ResBlock(prev_filters, filters, self.activation_fn, dtype=dtype,device=device))
- prev_filters = filters # update in_channels
-
- self.conv2 = nn.Conv3d(prev_filters, prev_filters, (3,3,3), padding=1, dtype=dtype, device=device) # need to init to xavier_uniform
-
- self.norm1 = nn.GroupNorm(num_groups, prev_filters, dtype=dtype, device=device)
-
- scale_factor = 2 ** self.num_blocks
- if num_frames % scale_factor != 0: # SCH: NOTE: has first frame which would be padded before usage
- time_scaled = num_frames // scale_factor + 1
- else:
- time_scaled = num_frames / scale_factor
- assert self.input_size[0] % scale_factor == 0, f"image width {self.input_size[0]} is not divisible by scale factor {scale_factor}"
- assert self.input_size[1] % scale_factor == 0, f"image height {self.input_size[1]} is not divisible by scale factor {scale_factor}"
-
- w_scaled, h_scaled = self.input_size[0] / scale_factor, self.input_size[1] / scale_factor
- in_features = int(prev_filters * time_scaled * w_scaled * h_scaled) # (C*T*W*H)
- self.linear1 = nn.Linear(in_features, prev_filters, device=device, dtype=dtype) # need to init to xavier_uniform
- self.linear2 = nn.Linear(prev_filters, 1, device=device, dtype=dtype) # need to init to xavier_uniform
-
- self.apply(xavier_uniform_weight_init)
-
- def forward(self, x):
-
- x = self.conv1(x)
- x = self.activation_fn(x)
-
- for i in range(self.num_blocks):
- x = self.res_block_list[i](x)
-
- x = self.conv2(x)
- x = self.norm1(x)
- x = self.activation_fn(x)
- x = x.reshape((x.shape[0], -1)) # SCH: [B, (C * T * W * H)] ?
- breakpoint()
- x = self.linear1(x)
- x = self.activation_fn(x)
- x = self.linear2(x)
- return x
diff --git a/opensora/models/vae/magvit2.py b/opensora/models/vae/magvit2.py
deleted file mode 100644
index 455e128..0000000
--- a/opensora/models/vae/magvit2.py
+++ /dev/null
@@ -1,1905 +0,0 @@
-import copy
-from pathlib import Path
-from math import log2, ceil, sqrt
-from functools import wraps, partial
-
-import torch
-import torch.nn.functional as F
-from torch.cuda.amp import autocast
-from torch import nn, einsum, Tensor
-from torch.nn import Module, ModuleList
-from torch.autograd import grad as torch_grad
-
-import torchvision
-from torchvision.models import VGG16_Weights
-
-from collections import namedtuple
-
-from vector_quantize_pytorch import LFQ, FSQ
-
-from einops import rearrange, repeat, reduce, pack, unpack
-from einops.layers.torch import Rearrange
-
-from beartype import beartype
-from beartype.typing import Union, Tuple, Optional, List
-
-from magvit2_pytorch.attend import Attend
-from magvit2_pytorch.version import __version__
-
-from gateloop_transformer import SimpleGateLoopLayer
-
-from taylor_series_linear_attention import TaylorSeriesLinearAttn
-
-from kornia.filters import filter3d
-
-import pickle
-
-# helper
-
-def exists(v):
- return v is not None
-
-def default(v, d):
- return v if exists(v) else d
-
-def safe_get_index(it, ind, default = None):
- if ind < len(it):
- return it[ind]
- return default
-
-def pair(t):
- return t if isinstance(t, tuple) else (t, t)
-
-def identity(t, *args, **kwargs):
- return t
-
-def divisible_by(num, den):
- return (num % den) == 0
-
-def pack_one(t, pattern):
- return pack([t], pattern)
-
-def unpack_one(t, ps, pattern):
- return unpack(t, ps, pattern)[0]
-
-def append_dims(t, ndims: int):
- return t.reshape(*t.shape, *((1,) * ndims))
-
-def is_odd(n):
- return not divisible_by(n, 2)
-
-def maybe_del_attr_(o, attr):
- if hasattr(o, attr):
- delattr(o, attr)
-
-def cast_tuple(t, length = 1):
- return t if isinstance(t, tuple) else ((t,) * length)
-
-# tensor helpers
-
-def l2norm(t):
- return F.normalize(t, dim = -1, p = 2)
-
-def pad_at_dim(t, pad, dim = -1, value = 0.):
- dims_from_right = (- dim - 1) if dim < 0 else (t.ndim - dim - 1)
- zeros = ((0, 0) * dims_from_right)
- return F.pad(t, (*zeros, *pad), value = value)
-
-def pick_video_frame(video, frame_indices):
- batch, device = video.shape[0], video.device
- video = rearrange(video, 'b c f ... -> b f c ...')
- batch_indices = torch.arange(batch, device = device)
- batch_indices = rearrange(batch_indices, 'b -> b 1')
- images = video[batch_indices, frame_indices]
- images = rearrange(images, 'b 1 c ... -> b c ...')
- return images
-
-# gan related
-
-def gradient_penalty(images, output):
- batch_size = images.shape[0]
-
- gradients = torch_grad(
- outputs = output,
- inputs = images,
- grad_outputs = torch.ones(output.size(), device = images.device),
- create_graph = True,
- retain_graph = True,
- only_inputs = True
- )[0]
-
- gradients = rearrange(gradients, 'b ... -> b (...)')
- return ((gradients.norm(2, dim = 1) - 1) ** 2).mean()
-
-def leaky_relu(p = 0.1):
- return nn.LeakyReLU(p)
-
-def hinge_discr_loss(fake, real):
- return (F.relu(1 + fake) + F.relu(1 - real)).mean()
-
-def hinge_gen_loss(fake):
- return -fake.mean()
-
-@autocast(enabled = False)
-@beartype
-def grad_layer_wrt_loss(
- loss: Tensor,
- layer: nn.Parameter
-):
- return torch_grad(
- outputs = loss,
- inputs = layer,
- grad_outputs = torch.ones_like(loss),
- retain_graph = True
- )[0].detach()
-
-# helper decorators
-
-def remove_vgg(fn):
- @wraps(fn)
- def inner(self, *args, **kwargs):
- has_vgg = hasattr(self, 'vgg')
- if has_vgg:
- vgg = self.vgg
- delattr(self, 'vgg')
-
- out = fn(self, *args, **kwargs)
-
- if has_vgg:
- self.vgg = vgg
-
- return out
- return inner
-
-# helper classes
-
-def Sequential(*modules):
- modules = [*filter(exists, modules)]
-
- if len(modules) == 0:
- return nn.Identity()
-
- return nn.Sequential(*modules)
-
-class Residual(Module):
- @beartype
- def __init__(self, fn: Module):
- super().__init__()
- self.fn = fn
-
- def forward(self, x, **kwargs):
- return self.fn(x, **kwargs) + x
-
-# for a bunch of tensor operations to change tensor to (batch, time, feature dimension) and back
-
-class ToTimeSequence(Module):
- @beartype
- def __init__(self, fn: Module):
- super().__init__()
- self.fn = fn
-
- def forward(self, x, **kwargs):
- x = rearrange(x, 'b c f ... -> b ... f c')
- x, ps = pack_one(x, '* n c')
-
- o = self.fn(x, **kwargs)
-
- o = unpack_one(o, ps, '* n c')
- return rearrange(o, 'b ... f c -> b c f ...')
-
-
-class SqueezeExcite(Module):
- # global context network - attention-esque squeeze-excite variant (https://arxiv.org/abs/2012.13375)
-
- def __init__(
- self,
- dim,
- *,
- dim_out = None,
- dim_hidden_min = 16,
- init_bias = -10
- ):
- super().__init__()
- dim_out = default(dim_out, dim)
-
- self.to_k = nn.Conv2d(dim, 1, 1)
- dim_hidden = max(dim_hidden_min, dim_out // 2)
-
- self.net = nn.Sequential(
- nn.Conv2d(dim, dim_hidden, 1),
- nn.LeakyReLU(0.1),
- nn.Conv2d(dim_hidden, dim_out, 1),
- nn.Sigmoid()
- )
-
- nn.init.zeros_(self.net[-2].weight)
- nn.init.constant_(self.net[-2].bias, init_bias)
-
- def forward(self, x):
- orig_input, batch = x, x.shape[0]
- is_video = x.ndim == 5
-
- if is_video:
- x = rearrange(x, 'b c f h w -> (b f) c h w')
-
- context = self.to_k(x)
-
- context = rearrange(context, 'b c h w -> b c (h w)').softmax(dim = -1)
- spatial_flattened_input = rearrange(x, 'b c h w -> b c (h w)')
-
- out = einsum('b i n, b c n -> b c i', context, spatial_flattened_input)
- out = rearrange(out, '... -> ... 1')
- gates = self.net(out)
-
- if is_video:
- gates = rearrange(gates, '(b f) c h w -> b c f h w', b = batch)
-
- return gates * orig_input
-
-# token shifting
-
-class TokenShift(Module):
- @beartype
- def __init__(self, fn: Module):
- super().__init__()
- self.fn = fn
-
- def forward(self, x, **kwargs):
- x, x_shift = x.chunk(2, dim = 1)
- x_shift = pad_at_dim(x_shift, (1, -1), dim = 2) # shift time dimension
- x = torch.cat((x, x_shift), dim = 1)
- return self.fn(x, **kwargs)
-
-# rmsnorm
-
-class RMSNorm(Module):
- def __init__(
- self,
- dim,
- channel_first = False,
- images = False,
- bias = False
- ):
- super().__init__()
- broadcastable_dims = (1, 1, 1) if not images else (1, 1)
- shape = (dim, *broadcastable_dims) if channel_first else (dim,)
-
- self.channel_first = channel_first
- self.scale = dim ** 0.5
- self.gamma = nn.Parameter(torch.ones(shape))
- self.bias = nn.Parameter(torch.zeros(shape)) if bias else 0.
-
- def forward(self, x):
- return F.normalize(x, dim = (1 if self.channel_first else -1)) * self.scale * self.gamma + self.bias
-
-class AdaptiveRMSNorm(Module):
- def __init__(
- self,
- dim,
- *,
- dim_cond,
- channel_first = False,
- images = False,
- bias = False
- ):
- super().__init__()
- broadcastable_dims = (1, 1, 1) if not images else (1, 1)
- shape = (dim, *broadcastable_dims) if channel_first else (dim,)
-
- self.dim_cond = dim_cond
- self.channel_first = channel_first
- self.scale = dim ** 0.5
-
- self.to_gamma = nn.Linear(dim_cond, dim)
- self.to_bias = nn.Linear(dim_cond, dim) if bias else None
-
- nn.init.zeros_(self.to_gamma.weight)
- nn.init.ones_(self.to_gamma.bias)
-
- if bias:
- nn.init.zeros_(self.to_bias.weight)
- nn.init.zeros_(self.to_bias.bias)
-
- @beartype
- def forward(self, x: Tensor, *, cond: Tensor):
- batch = x.shape[0]
- assert cond.shape == (batch, self.dim_cond)
-
- gamma = self.to_gamma(cond)
-
- bias = 0.
- if exists(self.to_bias):
- bias = self.to_bias(cond)
-
- if self.channel_first:
- gamma = append_dims(gamma, x.ndim - 2)
-
- if exists(self.to_bias):
- bias = append_dims(bias, x.ndim - 2)
-
- return F.normalize(x, dim = (1 if self.channel_first else -1)) * self.scale * gamma + bias
-
-# attention
-
-class Attention(Module):
- @beartype
- def __init__(
- self,
- *,
- dim,
- dim_cond: Optional[int] = None,
- causal = False,
- dim_head = 32,
- heads = 8,
- flash = False,
- dropout = 0.,
- num_memory_kv = 4
- ):
- super().__init__()
- dim_inner = dim_head * heads
-
- self.need_cond = exists(dim_cond)
-
- if self.need_cond:
- self.norm = AdaptiveRMSNorm(dim, dim_cond = dim_cond)
- else:
- self.norm = RMSNorm(dim)
-
- self.to_qkv = nn.Sequential(
- nn.Linear(dim, dim_inner * 3, bias = False),
- Rearrange('b n (qkv h d) -> qkv b h n d', qkv = 3, h = heads)
- )
-
- assert num_memory_kv > 0
- self.mem_kv = nn.Parameter(torch.randn(2, heads, num_memory_kv, dim_head))
-
- self.attend = Attend(
- causal = causal,
- dropout = dropout,
- flash = flash
- )
-
- self.to_out = nn.Sequential(
- Rearrange('b h n d -> b n (h d)'),
- nn.Linear(dim_inner, dim, bias = False)
- )
-
- @beartype
- def forward(
- self,
- x,
- mask: Optional[Tensor ] = None,
- cond: Optional[Tensor] = None
- ):
- maybe_cond_kwargs = dict(cond = cond) if self.need_cond else dict()
-
- x = self.norm(x, **maybe_cond_kwargs)
-
- q, k, v = self.to_qkv(x)
-
- mk, mv = map(lambda t: repeat(t, 'h n d -> b h n d', b = q.shape[0]), self.mem_kv)
- k = torch.cat((mk, k), dim = -2)
- v = torch.cat((mv, v), dim = -2)
-
- out = self.attend(q, k, v, mask = mask)
- return self.to_out(out)
-
-class LinearAttention(Module):
- """
- using the specific linear attention proposed in https://arxiv.org/abs/2106.09681
- """
-
- @beartype
- def __init__(
- self,
- *,
- dim,
- dim_cond: Optional[int] = None,
- dim_head = 8,
- heads = 8,
- dropout = 0.
- ):
- super().__init__()
- dim_inner = dim_head * heads
-
- self.need_cond = exists(dim_cond)
-
- if self.need_cond:
- self.norm = AdaptiveRMSNorm(dim, dim_cond = dim_cond)
- else:
- self.norm = RMSNorm(dim)
-
- self.attn = TaylorSeriesLinearAttn(
- dim = dim,
- dim_head = dim_head,
- heads = heads
- )
-
- def forward(
- self,
- x,
- cond: Optional[Tensor] = None
- ):
- maybe_cond_kwargs = dict(cond = cond) if self.need_cond else dict()
-
- x = self.norm(x, **maybe_cond_kwargs)
-
- return self.attn(x)
-
-class LinearSpaceAttention(LinearAttention):
- """
- SCH: format h & w into a linear dim, do linear attention, then format back
- """
- def forward(self, x, *args, **kwargs):
- x = rearrange(x, 'b c ... h w -> b ... h w c')
- x, batch_ps = pack_one(x, '* h w c')
- x, seq_ps = pack_one(x, 'b * c')
-
- x = super().forward(x, *args, **kwargs)
-
- x = unpack_one(x, seq_ps, 'b * c')
- x = unpack_one(x, batch_ps, '* h w c')
- return rearrange(x, 'b ... h w c -> b c ... h w')
-
-class SpaceAttention(Attention):
- def forward(self, x, *args, **kwargs):
- x = rearrange(x, 'b c t h w -> b t h w c')
- x, batch_ps = pack_one(x, '* h w c')
- x, seq_ps = pack_one(x, 'b * c')
-
- x = super().forward(x, *args, **kwargs)
-
- x = unpack_one(x, seq_ps, 'b * c')
- x = unpack_one(x, batch_ps, '* h w c')
- return rearrange(x, 'b t h w c -> b c t h w')
-
-class TimeAttention(Attention):
- def forward(self, x, *args, **kwargs):
- x = rearrange(x, 'b c t h w -> b h w t c')
- x, batch_ps = pack_one(x, '* t c')
-
- x = super().forward(x, *args, **kwargs)
-
- x = unpack_one(x, batch_ps, '* t c')
- return rearrange(x, 'b h w t c -> b c t h w')
-
-class GEGLU(Module):
- def forward(self, x):
- x, gate = x.chunk(2, dim = 1)
- return F.gelu(gate) * x
-
-class FeedForward(Module):
- @beartype
- def __init__(
- self,
- dim,
- *,
- dim_cond: Optional[int] = None,
- mult = 4,
- images = False
- ):
- super().__init__()
- conv_klass = nn.Conv2d if images else nn.Conv3d
-
- rmsnorm_klass = RMSNorm if not exists(dim_cond) else partial(AdaptiveRMSNorm, dim_cond = dim_cond)
-
- maybe_adaptive_norm_klass = partial(rmsnorm_klass, channel_first = True, images = images)
-
- dim_inner = int(dim * mult * 2 / 3)
-
- self.norm = maybe_adaptive_norm_klass(dim)
-
- self.net = Sequential(
- conv_klass(dim, dim_inner * 2, 1),
- GEGLU(),
- conv_klass(dim_inner, dim, 1)
- )
-
- @beartype
- def forward(
- self,
- x: Tensor,
- *,
- cond: Optional[Tensor] = None
- ):
- maybe_cond_kwargs = dict(cond = cond) if exists(cond) else dict()
-
- x = self.norm(x, **maybe_cond_kwargs)
- return self.net(x)
-
-# discriminator with anti-aliased downsampling (blurpool Zhang et al.)
-
-class Blur(Module):
- def __init__(self):
- super().__init__()
- f = torch.Tensor([1, 2, 1])
- self.register_buffer('f', f)
-
- def forward(
- self,
- x,
- space_only = False,
- time_only = False
- ):
- assert not (space_only and time_only)
-
- f = self.f
-
- if space_only:
- f = einsum('i, j -> i j', f, f)
- f = rearrange(f, '... -> 1 1 ...')
- elif time_only:
- f = rearrange(f, 'f -> 1 f 1 1')
- else:
- f = einsum('i, j, k -> i j k', f, f, f)
- f = rearrange(f, '... -> 1 ...')
-
- is_images = x.ndim == 4
-
- if is_images:
- x = rearrange(x, 'b c h w -> b c 1 h w')
-
- out = filter3d(x, f, normalized = True)
-
- if is_images:
- out = rearrange(out, 'b c 1 h w -> b c h w')
-
- return out
-
-class DiscriminatorBlock(Module):
- def __init__(
- self,
- input_channels,
- filters,
- downsample = True,
- antialiased_downsample = True
- ):
- super().__init__()
- self.conv_res = nn.Conv2d(input_channels, filters, 1, stride = (2 if downsample else 1))
-
- self.net = nn.Sequential(
- nn.Conv2d(input_channels, filters, 3, padding = 1),
- leaky_relu(),
- nn.Conv2d(filters, filters, 3, padding = 1),
- leaky_relu()
- )
-
- self.maybe_blur = Blur() if antialiased_downsample else None
-
- self.downsample = nn.Sequential(
- Rearrange('b c (h p1) (w p2) -> b (c p1 p2) h w', p1 = 2, p2 = 2),
- nn.Conv2d(filters * 4, filters, 1)
- ) if downsample else None
-
- def forward(self, x):
- res = self.conv_res(x)
-
- x = self.net(x)
-
- if exists(self.downsample):
- if exists(self.maybe_blur):
- x = self.maybe_blur(x, space_only = True)
-
- x = self.downsample(x)
-
- x = (x + res) * (2 ** -0.5)
- return x
-
-class Discriminator(Module):
- @beartype
- def __init__(
- self,
- *,
- dim,
- image_size,
- channels = 3,
- max_dim = 512,
- attn_heads = 8,
- attn_dim_head = 32,
- linear_attn_dim_head = 8,
- linear_attn_heads = 16,
- ff_mult = 4,
- antialiased_downsample = False
- ):
- super().__init__()
- image_size = pair(image_size)
- min_image_resolution = min(image_size)
-
- num_layers = int(log2(min_image_resolution) - 2)
-
- blocks = []
-
- layer_dims = [channels] + [(dim * 4) * (2 ** i) for i in range(num_layers + 1)] # SCH: num channels across each layer
- layer_dims = [min(layer_dim, max_dim) for layer_dim in layer_dims]
- layer_dims_in_out = tuple(zip(layer_dims[:-1], layer_dims[1:]))
-
- blocks = []
- attn_blocks = []
-
- image_resolution = min_image_resolution
-
- for ind, (in_chan, out_chan) in enumerate(layer_dims_in_out):
- num_layer = ind + 1
- is_not_last = ind != (len(layer_dims_in_out) - 1)
-
- block = DiscriminatorBlock(
- in_chan,
- out_chan,
- downsample = is_not_last,
- antialiased_downsample = antialiased_downsample
- )
-
- attn_block = Sequential(
- Residual(LinearSpaceAttention(
- dim = out_chan,
- heads = linear_attn_heads,
- dim_head = linear_attn_dim_head
- )),
- Residual(FeedForward(
- dim = out_chan,
- mult = ff_mult,
- images = True
- ))
- )
-
- blocks.append(ModuleList([
- block,
- attn_block
- ]))
-
- image_resolution //= 2
-
- self.blocks = ModuleList(blocks)
-
- dim_last = layer_dims[-1]
-
- downsample_factor = 2 ** num_layers
- last_fmap_size = tuple(map(lambda n: n // downsample_factor, image_size))
-
- latent_dim = last_fmap_size[0] * last_fmap_size[1] * dim_last
-
- self.to_logits = Sequential(
- nn.Conv2d(dim_last, dim_last, 3, padding = 1),
- leaky_relu(),
- Rearrange('b ... -> b (...)'),
- nn.Linear(latent_dim, 1),
- Rearrange('b 1 -> b')
- )
-
- def forward(self, x):
-
- for block, attn_block in self.blocks:
- x = block(x)
- x = attn_block(x)
-
- return self.to_logits(x)
-
-# modulatable conv from Karras et al. Stylegan2
-# for conditioning on latents
-
-class Conv3DMod(Module):
- @beartype
- def __init__(
- self,
- dim,
- *,
- spatial_kernel,
- time_kernel,
- causal = True,
- dim_out = None,
- demod = True,
- eps = 1e-8,
- pad_mode = 'zeros'
- ):
- super().__init__()
- dim_out = default(dim_out, dim)
-
- self.eps = eps
-
- assert is_odd(spatial_kernel) and is_odd(time_kernel)
-
- self.spatial_kernel = spatial_kernel
- self.time_kernel = time_kernel
-
- time_padding = (time_kernel - 1, 0) if causal else ((time_kernel // 2,) * 2)
-
- self.pad_mode = pad_mode
- self.padding = (*((spatial_kernel // 2,) * 4), *time_padding)
- self.weights = nn.Parameter(torch.randn((dim_out, dim, time_kernel, spatial_kernel, spatial_kernel)))
-
- self.demod = demod
-
- nn.init.kaiming_normal_(self.weights, a = 0, mode = 'fan_in', nonlinearity = 'selu')
-
- @beartype
- def forward(
- self,
- fmap,
- cond: Tensor
- ):
- """
- notation
-
- b - batch
- n - convs
- o - output
- i - input
- k - kernel
- """
-
- b = fmap.shape[0]
-
- # prepare weights for modulation
-
- weights = self.weights
-
- # do the modulation, demodulation, as done in stylegan2
-
- cond = rearrange(cond, 'b i -> b 1 i 1 1 1')
-
- weights = weights * (cond + 1)
-
- if self.demod:
- inv_norm = reduce(weights ** 2, 'b o i k0 k1 k2 -> b o 1 1 1 1', 'sum').clamp(min = self.eps).rsqrt()
- weights = weights * inv_norm
-
- fmap = rearrange(fmap, 'b c t h w -> 1 (b c) t h w')
-
- weights = rearrange(weights, 'b o ... -> (b o) ...')
-
- fmap = F.pad(fmap, self.padding, mode = self.pad_mode)
- fmap = F.conv3d(fmap, weights, groups = b)
-
- return rearrange(fmap, '1 (b o) ... -> b o ...', b = b)
-
-# strided conv downsamples
-
-class SpatialDownsample2x(Module):
- def __init__(
- self,
- dim,
- dim_out = None,
- kernel_size = 3,
- antialias = False
- ):
- super().__init__()
- dim_out = default(dim_out, dim)
- self.maybe_blur = Blur() if antialias else identity
- self.conv = nn.Conv2d(dim, dim_out, kernel_size, stride = 2, padding = kernel_size // 2)
-
- def forward(self, x):
- x = self.maybe_blur(x, space_only = True)
-
- x = rearrange(x, 'b c t h w -> b t c h w')
- x, ps = pack_one(x, '* c h w')
-
- out = self.conv(x)
-
- out = unpack_one(out, ps, '* c h w')
- out = rearrange(out, 'b t c h w -> b c t h w')
- return out
-
-class TimeDownsample2x(Module):
- def __init__(
- self,
- dim,
- dim_out = None,
- kernel_size = 3,
- antialias = False
- ):
- super().__init__()
- dim_out = default(dim_out, dim)
- self.maybe_blur = Blur() if antialias else identity
- self.time_causal_padding = (kernel_size - 1, 0)
- self.conv = nn.Conv1d(dim, dim_out, kernel_size, stride = 2)
-
- def forward(self, x):
- x = self.maybe_blur(x, time_only = True)
-
- x = rearrange(x, 'b c t h w -> b h w c t')
- x, ps = pack_one(x, '* c t')
-
- x = F.pad(x, self.time_causal_padding)
- out = self.conv(x)
-
- out = unpack_one(out, ps, '* c t')
- out = rearrange(out, 'b h w c t -> b c t h w')
- return out
-
-# depth to space upsamples
-
-class SpatialUpsample2x(Module):
- def __init__(
- self,
- dim,
- dim_out = None
- ):
- super().__init__()
- dim_out = default(dim_out, dim)
- conv = nn.Conv2d(dim, dim_out * 4, 1)
-
- self.net = nn.Sequential(
- conv,
- nn.SiLU(),
- Rearrange('b (c p1 p2) h w -> b c (h p1) (w p2)', p1 = 2, p2 = 2)
- )
-
- self.init_conv_(conv)
-
- def init_conv_(self, conv):
- o, i, h, w = conv.weight.shape
- conv_weight = torch.empty(o // 4, i, h, w)
- nn.init.kaiming_uniform_(conv_weight)
- conv_weight = repeat(conv_weight, 'o ... -> (o 4) ...')
-
- conv.weight.data.copy_(conv_weight)
- nn.init.zeros_(conv.bias.data)
-
- def forward(self, x):
- x = rearrange(x, 'b c t h w -> b t c h w')
- x, ps = pack_one(x, '* c h w')
-
- out = self.net(x)
-
- out = unpack_one(out, ps, '* c h w')
- out = rearrange(out, 'b t c h w -> b c t h w')
- return out
-
-class TimeUpsample2x(Module):
- def __init__(
- self,
- dim,
- dim_out = None
- ):
- super().__init__()
- dim_out = default(dim_out, dim)
- conv = nn.Conv1d(dim, dim_out * 2, 1)
-
- self.net = nn.Sequential(
- conv,
- nn.SiLU(),
- Rearrange('b (c p) t -> b c (t p)', p = 2)
- )
-
- self.init_conv_(conv)
-
- def init_conv_(self, conv):
- o, i, t = conv.weight.shape
- conv_weight = torch.empty(o // 2, i, t)
- nn.init.kaiming_uniform_(conv_weight)
- conv_weight = repeat(conv_weight, 'o ... -> (o 2) ...')
-
- conv.weight.data.copy_(conv_weight)
- nn.init.zeros_(conv.bias.data)
-
- def forward(self, x):
- x = rearrange(x, 'b c t h w -> b h w c t')
- x, ps = pack_one(x, '* c t')
-
- out = self.net(x)
-
- out = unpack_one(out, ps, '* c t')
- out = rearrange(out, 'b h w c t -> b c t h w')
- return out
-
-# autoencoder - only best variant here offered, with causal conv 3d
-
-def SameConv2d(dim_in, dim_out, kernel_size):
- kernel_size = cast_tuple(kernel_size, 2)
- padding = [k // 2 for k in kernel_size]
- return nn.Conv2d(dim_in, dim_out, kernel_size = kernel_size, padding = padding)
-
-class CausalConv3d(Module):
- @beartype
- def __init__(
- self,
- chan_in,
- chan_out,
- kernel_size: Union[int, Tuple[int, int, int]],
- pad_mode = 'constant',
- **kwargs
- ):
- super().__init__()
- kernel_size = cast_tuple(kernel_size, 3)
-
- time_kernel_size, height_kernel_size, width_kernel_size = kernel_size
-
- assert is_odd(height_kernel_size) and is_odd(width_kernel_size)
-
- dilation = kwargs.pop('dilation', 1)
- stride = kwargs.pop('stride', 1)
-
- self.pad_mode = pad_mode
- time_pad = dilation * (time_kernel_size - 1) + (1 - stride)
- height_pad = height_kernel_size // 2
- width_pad = width_kernel_size // 2
-
- self.time_pad = time_pad
- self.time_causal_padding = (width_pad, width_pad, height_pad, height_pad, time_pad, 0)
-
- stride = (stride, 1, 1)
- dilation = (dilation, 1, 1)
- self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride = stride, dilation = dilation, **kwargs)
-
- def forward(self, x):
- pad_mode = self.pad_mode if self.time_pad < x.shape[2] else 'constant'
-
- x = F.pad(x, self.time_causal_padding, mode = pad_mode)
- return self.conv(x)
-
-@beartype
-def ResidualUnit(
- dim,
- kernel_size: Union[int, Tuple[int, int, int]],
- pad_mode: str = 'constant'
-):
- net = Sequential(
- CausalConv3d(dim, dim, kernel_size, pad_mode = pad_mode),
- nn.ELU(),
- nn.Conv3d(dim, dim, 1),
- nn.ELU(),
- SqueezeExcite(dim)
- )
-
- return Residual(net)
-
-@beartype
-class ResidualUnitMod(Module):
- def __init__(
- self,
- dim,
- kernel_size: Union[int, Tuple[int, int, int]],
- *,
- dim_cond,
- pad_mode: str = 'constant',
- demod = True
- ):
- super().__init__()
- kernel_size = cast_tuple(kernel_size, 3)
- time_kernel_size, height_kernel_size, width_kernel_size = kernel_size
- assert height_kernel_size == width_kernel_size
-
- self.to_cond = nn.Linear(dim_cond, dim)
-
- self.conv = Conv3DMod(
- dim = dim,
- spatial_kernel = height_kernel_size,
- time_kernel = time_kernel_size,
- causal = True,
- demod = demod,
- pad_mode = pad_mode
- )
-
- self.conv_out = nn.Conv3d(dim, dim, 1)
-
- @beartype
- def forward(
- self,
- x,
- cond: Tensor,
- ):
- res = x
- cond = self.to_cond(cond)
-
- x = self.conv(x, cond = cond)
- x = F.elu(x)
- x = self.conv_out(x)
- x = F.elu(x)
- return x + res
-
-class CausalConvTranspose3d(Module):
- def __init__(
- self,
- chan_in,
- chan_out,
- kernel_size: Union[int, Tuple[int, int, int]],
- *,
- time_stride,
- **kwargs
- ):
- super().__init__()
- kernel_size = cast_tuple(kernel_size, 3)
-
- time_kernel_size, height_kernel_size, width_kernel_size = kernel_size
-
- assert is_odd(height_kernel_size) and is_odd(width_kernel_size)
-
- self.upsample_factor = time_stride
-
- height_pad = height_kernel_size // 2
- width_pad = width_kernel_size // 2
-
- stride = (time_stride, 1, 1)
- padding = (0, height_pad, width_pad)
-
- self.conv = nn.ConvTranspose3d(chan_in, chan_out, kernel_size, stride, padding = padding, **kwargs)
-
- def forward(self, x):
- assert x.ndim == 5
- t = x.shape[2]
-
- out = self.conv(x)
-
- out = out[..., :(t * self.upsample_factor), :, :]
- return out
-
-# video tokenizer class
-
-LossBreakdown = namedtuple('LossBreakdown', [
- 'recon_loss',
- 'lfq_aux_loss',
- 'quantizer_loss_breakdown',
- 'perceptual_loss',
- 'adversarial_gen_loss',
- 'adaptive_adversarial_weight',
- 'multiscale_gen_losses',
- 'multiscale_gen_adaptive_weights'
-])
-
-DiscrLossBreakdown = namedtuple('DiscrLossBreakdown', [
- 'discr_loss',
- 'multiscale_discr_losses',
- 'gradient_penalty'
-])
-
-class VideoTokenizer(Module):
- @beartype
- def __init__(
- self,
- *,
- image_size,
- layers: Tuple[Union[str, Tuple[str, int]], ...] = (
- 'residual',
- 'residual',
- 'residual'
- ),
- residual_conv_kernel_size = 3,
- # num_codebooks = 1,
- # codebook_size: Optional[int] = None,
- channels = 3,
- init_dim = 64,
- max_dim = float('inf'),
- dim_cond = None,
- dim_cond_expansion_factor = 4.,
- input_conv_kernel_size: Tuple[int, int, int] = (7, 7, 7),
- output_conv_kernel_size: Tuple[int, int, int] = (3, 3, 3),
- pad_mode: str = 'constant',
- lfq_entropy_loss_weight = 0.1,
- lfq_commitment_loss_weight = 1., # SCH: codebook?
- lfq_diversity_gamma = 2.5,
- quantizer_aux_loss_weight = 1.,
- lfq_activation = nn.Identity(),
- use_fsq = False,
- fsq_levels: Optional[List[int]] = None,
- attn_dim_head = 32,
- attn_heads = 8,
- attn_dropout = 0.,
- linear_attn_dim_head = 8,
- linear_attn_heads = 16,
- vgg: Optional[Module] = None,
- vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,
- perceptual_loss_weight = 1e-1,
- discr_kwargs: Optional[dict] = None,
- multiscale_discrs: Tuple[Module, ...] = tuple(),
- use_gan = True,
- adversarial_loss_weight = 1.,
- grad_penalty_loss_weight = 10.,
- multiscale_adversarial_loss_weight = 1.,
- flash_attn = True,
- separate_first_frame_encoding = False
- ):
- super().__init__()
-
- # for autosaving the config
-
- _locals = locals()
- _locals.pop('self', None)
- _locals.pop('__class__', None)
- self._configs = pickle.dumps(_locals)
-
- # image size
-
- self.channels = channels
- self.image_size = image_size
-
- # initial encoder
-
- self.conv_in = CausalConv3d(channels, init_dim, input_conv_kernel_size, pad_mode = pad_mode)
-
- # whether to encode the first frame separately or not
-
- self.conv_in_first_frame = nn.Identity()
- self.conv_out_first_frame = nn.Identity()
-
- if separate_first_frame_encoding:
- self.conv_in_first_frame = SameConv2d(channels, init_dim, input_conv_kernel_size[-2:])
- self.conv_out_first_frame = SameConv2d(init_dim, channels, output_conv_kernel_size[-2:])
-
- self.separate_first_frame_encoding = separate_first_frame_encoding
-
- # encoder and decoder layers
-
- self.encoder_layers = ModuleList([])
- self.decoder_layers = ModuleList([])
-
- self.conv_out = CausalConv3d(init_dim, channels, output_conv_kernel_size, pad_mode = pad_mode)
-
- dim = init_dim
- dim_out = dim
-
- layer_fmap_size = image_size # SCH: feaure map size
- time_downsample_factor = 1
- has_cond_across_layers = [] # SCH: record if the corr. layers has condition
-
- for layer_def in layers:
- layer_type, *layer_params = cast_tuple(layer_def)
-
- has_cond = False
-
- if layer_type == 'residual': # SCH: resblock
- encoder_layer = ResidualUnit(dim, residual_conv_kernel_size)
- decoder_layer = ResidualUnit(dim, residual_conv_kernel_size)
-
- elif layer_type == 'consecutive_residual':
- num_consecutive, = layer_params
- encoder_layer = Sequential(*[ResidualUnit(dim, residual_conv_kernel_size) for _ in range(num_consecutive)])
- decoder_layer = Sequential(*[ResidualUnit(dim, residual_conv_kernel_size) for _ in range(num_consecutive)])
-
- elif layer_type == 'cond_residual':
- assert exists(dim_cond), 'dim_cond must be passed into VideoTokenizer, if tokenizer is to be conditioned'
-
- has_cond = True
-
- encoder_layer = ResidualUnitMod(dim, residual_conv_kernel_size, dim_cond = int(dim_cond * dim_cond_expansion_factor))
- decoder_layer = ResidualUnitMod(dim, residual_conv_kernel_size, dim_cond = int(dim_cond * dim_cond_expansion_factor))
- dim_out = dim
-
- elif layer_type == 'compress_space':
- dim_out = safe_get_index(layer_params, 0)
- dim_out = default(dim_out, dim * 2) # SCH: if dim_out exists, else use dim * 2
- dim_out = min(dim_out, max_dim)
-
- encoder_layer = SpatialDownsample2x(dim, dim_out) # SCH: 2d conv in space dimensions
- decoder_layer = SpatialUpsample2x(dim_out, dim) # SCH: 2d conv in space dimensions, use more channel to expand space dim
-
- assert layer_fmap_size > 1
- layer_fmap_size //= 2
-
- elif layer_type == 'compress_time':
- dim_out = safe_get_index(layer_params, 0)
- dim_out = default(dim_out, dim * 2)
- dim_out = min(dim_out, max_dim)
-
- encoder_layer = TimeDownsample2x(dim, dim_out) # SCH: 1d conv in time dim to reduce
- decoder_layer = TimeUpsample2x(dim_out, dim) # SCH: 1d conv in time dim, use more channels to expand time dim
-
- time_downsample_factor *= 2
-
- elif layer_type == 'attend_space':
- attn_kwargs = dict(
- dim = dim,
- dim_head = attn_dim_head,
- heads = attn_heads,
- dropout = attn_dropout,
- flash = flash_attn
- )
-
- encoder_layer = Sequential(
- Residual(SpaceAttention(**attn_kwargs)),
- Residual(FeedForward(dim))
- )
-
- decoder_layer = Sequential(
- Residual(SpaceAttention(**attn_kwargs)),
- Residual(FeedForward(dim))
- )
-
- elif layer_type == 'linear_attend_space':
- linear_attn_kwargs = dict(
- dim = dim,
- dim_head = linear_attn_dim_head,
- heads = linear_attn_heads
- )
-
- encoder_layer = Sequential(
- Residual(LinearSpaceAttention(**linear_attn_kwargs)),
- Residual(FeedForward(dim))
- )
-
- decoder_layer = Sequential(
- Residual(LinearSpaceAttention(**linear_attn_kwargs)),
- Residual(FeedForward(dim))
- )
-
- elif layer_type == 'gateloop_time':
- gateloop_kwargs = dict(
- use_heinsen = False
- )
-
- encoder_layer = ToTimeSequence(Residual(SimpleGateLoopLayer(dim = dim)))
- decoder_layer = ToTimeSequence(Residual(SimpleGateLoopLayer(dim = dim)))
-
- elif layer_type == 'attend_time':
- attn_kwargs = dict(
- dim = dim,
- dim_head = attn_dim_head,
- heads = attn_heads,
- dropout = attn_dropout,
- causal = True,
- flash = flash_attn
- )
-
- encoder_layer = Sequential(
- Residual(TokenShift(TimeAttention(**attn_kwargs))),
- Residual(TokenShift(FeedForward(dim, dim_cond = dim_cond)))
- )
-
- decoder_layer = Sequential(
- Residual(TokenShift(TimeAttention(**attn_kwargs))),
- Residual(TokenShift(FeedForward(dim, dim_cond = dim_cond)))
- )
-
- elif layer_type == 'cond_attend_space':
- has_cond = True
-
- attn_kwargs = dict(
- dim = dim,
- dim_cond = dim_cond,
- dim_head = attn_dim_head,
- heads = attn_heads,
- dropout = attn_dropout,
- flash = flash_attn
- )
-
- encoder_layer = Sequential(
- Residual(SpaceAttention(**attn_kwargs)),
- Residual(FeedForward(dim))
- )
-
- decoder_layer = Sequential(
- Residual(SpaceAttention(**attn_kwargs)),
- Residual(FeedForward(dim))
- )
-
- elif layer_type == 'cond_linear_attend_space':
- has_cond = True
-
- attn_kwargs = dict(
- dim = dim,
- dim_cond = dim_cond,
- dim_head = attn_dim_head,
- heads = attn_heads,
- dropout = attn_dropout,
- flash = flash_attn
- )
-
- encoder_layer = Sequential(
- Residual(LinearSpaceAttention(**attn_kwargs)),
- Residual(FeedForward(dim, dim_cond = dim_cond))
- )
-
- decoder_layer = Sequential(
- Residual(LinearSpaceAttention(**attn_kwargs)),
- Residual(FeedForward(dim, dim_cond = dim_cond))
- )
-
- elif layer_type == 'cond_attend_time':
- has_cond = True
-
- attn_kwargs = dict(
- dim = dim,
- dim_cond = dim_cond,
- dim_head = attn_dim_head,
- heads = attn_heads,
- dropout = attn_dropout,
- causal = True,
- flash = flash_attn
- )
-
- encoder_layer = Sequential(
- Residual(TokenShift(TimeAttention(**attn_kwargs))),
- Residual(TokenShift(FeedForward(dim, dim_cond = dim_cond)))
- )
-
- decoder_layer = Sequential(
- Residual(TokenShift(TimeAttention(**attn_kwargs))),
- Residual(TokenShift(FeedForward(dim, dim_cond = dim_cond)))
- )
-
- else:
- raise ValueError(f'unknown layer type {layer_type}')
-
- self.encoder_layers.append(encoder_layer)
- self.decoder_layers.insert(0, decoder_layer)
-
- dim = dim_out
- has_cond_across_layers.append(has_cond)
-
- # add a final norm just before quantization layer
-
- self.encoder_layers.append(Sequential(
- Rearrange('b c ... -> b ... c'),
- nn.LayerNorm(dim),
- Rearrange('b ... c -> b c ...'),
- ))
-
- self.time_downsample_factor = time_downsample_factor
- self.time_padding = time_downsample_factor - 1
-
- self.fmap_size = layer_fmap_size
-
- # use a MLP stem for conditioning, if needed
-
- self.has_cond_across_layers = has_cond_across_layers
- self.has_cond = any(has_cond_across_layers)
-
- self.encoder_cond_in = nn.Identity()
- self.decoder_cond_in = nn.Identity()
-
- if has_cond:
- self.dim_cond = dim_cond
-
- self.encoder_cond_in = Sequential(
- nn.Linear(dim_cond, int(dim_cond * dim_cond_expansion_factor)),
- nn.SiLU()
- )
-
- self.decoder_cond_in = Sequential(
- nn.Linear(dim_cond, int(dim_cond * dim_cond_expansion_factor)),
- nn.SiLU()
- )
-
- ## SCH: remove quantizer
- # # quantizer related
-
- # self.use_fsq = use_fsq
-
- # if not use_fsq:
- # assert exists(codebook_size) and not exists(fsq_levels), 'if use_fsq is set to False, `codebook_size` must be set (and not `fsq_levels`)'
-
- # # lookup free quantizer(s) - multiple codebooks is possible
- # # each codebook will get its own entropy regularization
-
- # self.quantizers = LFQ(
- # dim = dim,
- # codebook_size = codebook_size,
- # num_codebooks = num_codebooks,
- # entropy_loss_weight = lfq_entropy_loss_weight,
- # commitment_loss_weight = lfq_commitment_loss_weight,
- # diversity_gamma = lfq_diversity_gamma
- # )
-
- # else:
- # assert not exists(codebook_size) and exists(fsq_levels), 'if use_fsq is set to True, `fsq_levels` must be set (and not `codebook_size`). the effective codebook size is the cumulative product of all the FSQ levels'
-
- # self.quantizers = FSQ(
- # fsq_levels,
- # dim = dim,
- # num_codebooks = num_codebooks
- # )
-
- # self.quantizer_aux_loss_weight = quantizer_aux_loss_weight
-
- # dummy loss
-
- self.register_buffer('zero', torch.tensor(0.), persistent = False)
-
- # perceptual loss related
-
- use_vgg = channels in {1, 3, 4} and perceptual_loss_weight > 0.
-
- self.vgg = None
- self.perceptual_loss_weight = perceptual_loss_weight
-
- if use_vgg:
- if not exists(vgg):
- vgg = torchvision.models.vgg16(
- weights = vgg_weights
- )
-
- vgg.classifier = Sequential(*vgg.classifier[:-2])
-
- self.vgg = vgg
-
- self.use_vgg = use_vgg
-
- # main flag for whether to use GAN at all
-
- self.use_gan = use_gan
-
- # discriminator
-
- discr_kwargs = default(discr_kwargs, dict(
- dim = dim,
- image_size = image_size,
- channels = channels,
- max_dim = 512
- ))
-
- self.discr = Discriminator(**discr_kwargs)
-
- self.adversarial_loss_weight = adversarial_loss_weight
- self.grad_penalty_loss_weight = grad_penalty_loss_weight
-
- self.has_gan = use_gan and adversarial_loss_weight > 0.
-
- # multi-scale discriminators
-
- self.has_multiscale_gan = use_gan and multiscale_adversarial_loss_weight > 0.
-
- self.multiscale_discrs = ModuleList([*multiscale_discrs])
-
- self.multiscale_adversarial_loss_weight = multiscale_adversarial_loss_weight
-
- self.has_multiscale_discrs = (
- use_gan and \
- multiscale_adversarial_loss_weight > 0. and \
- len(multiscale_discrs) > 0
- )
-
- @property
- def device(self):
- return self.zero.device
-
- @classmethod
- def init_and_load_from(cls, path, strict = True):
- path = Path(path)
- assert path.exists()
- pkg = torch.load(str(path), map_location = 'cpu')
-
- assert 'config' in pkg, 'model configs were not found in this saved checkpoint'
-
- config = pickle.loads(pkg['config'])
- tokenizer = cls(**config)
- tokenizer.load(path, strict = strict)
- return tokenizer
-
- def parameters(self):
- return [
- *self.conv_in.parameters(),
- *self.conv_in_first_frame.parameters(),
- *self.conv_out_first_frame.parameters(),
- *self.conv_out.parameters(),
- *self.encoder_layers.parameters(),
- *self.decoder_layers.parameters(),
- *self.encoder_cond_in.parameters(),
- *self.decoder_cond_in.parameters(),
- *self.quantizers.parameters()
- ]
-
- def discr_parameters(self):
- return self.discr.parameters()
-
- def copy_for_eval(self):
- device = self.device
- vae_copy = copy.deepcopy(self.cpu())
-
- maybe_del_attr_(vae_copy, 'discr')
- maybe_del_attr_(vae_copy, 'vgg')
- maybe_del_attr_(vae_copy, 'multiscale_discrs')
-
- vae_copy.eval()
- return vae_copy.to(device)
-
- @remove_vgg
- def state_dict(self, *args, **kwargs):
- return super().state_dict(*args, **kwargs)
-
- @remove_vgg
- def load_state_dict(self, *args, **kwargs):
- return super().load_state_dict(*args, **kwargs)
-
- def save(self, path, overwrite = True):
- path = Path(path)
- assert overwrite or not path.exists(), f'{str(path)} already exists'
-
- pkg = dict(
- model_state_dict = self.state_dict(),
- version = __version__,
- config = self._configs
- )
-
- torch.save(pkg, str(path))
-
- def load(self, path, strict = True):
- path = Path(path)
- assert path.exists()
-
- pkg = torch.load(str(path))
- state_dict = pkg.get('model_state_dict')
- version = pkg.get('version')
-
- assert exists(state_dict)
-
- if exists(version):
- print(f'loading checkpointed tokenizer from version {version}')
-
- self.load_state_dict(state_dict, strict = strict)
-
- @beartype
- def encode(
- self,
- video: Tensor,
- quantize = False,
- cond: Optional[Tensor] = None,
- video_contains_first_frame = True
- ):
- """
- SCH: conv (may sep 1st frame), then pass through self.encoder_layers, then quantize if needed, finish
- """
- encode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame
-
- # whether to pad video or not
-
- if video_contains_first_frame:
- video_len = video.shape[2]
-
- video = pad_at_dim(video, (self.time_padding, 0), value = 0., dim = 2)
- video_packed_shape = [torch.Size([self.time_padding]), torch.Size([]), torch.Size([video_len - 1])]
-
- # conditioning, if needed
-
- assert (not self.has_cond) or exists(cond), '`cond` must be passed into tokenizer forward method since conditionable layers were specified'
-
- if exists(cond):
- assert cond.shape == (video.shape[0], self.dim_cond)
-
- cond = self.encoder_cond_in(cond)
- cond_kwargs = dict(cond = cond)
-
- # initial conv
- # taking into account whether to encode first frame separately
-
- if encode_first_frame_separately:
- pad, first_frame, video = unpack(video, video_packed_shape, 'b c * h w')
- first_frame = self.conv_in_first_frame(first_frame)
-
- video = self.conv_in(video)
-
- if encode_first_frame_separately:
- video, _ = pack([first_frame, video], 'b c * h w')
- video = pad_at_dim(video, (self.time_padding, 0), dim = 2)
-
- # encoder layers
-
- for fn, has_cond in zip(self.encoder_layers, self.has_cond_across_layers):
-
- layer_kwargs = dict()
-
- if has_cond:
- layer_kwargs = cond_kwargs
-
- video = fn(video, **layer_kwargs)
-
- maybe_quantize = identity if not quantize else self.quantizers
-
- return maybe_quantize(video)
-
- # @beartype
- # def decode_from_code_indices(
- # self,
- # codes: Tensor,
- # cond: Optional[Tensor] = None,
- # video_contains_first_frame = True
- # ):
- # assert codes.dtype in (torch.long, torch.int32)
-
- # if codes.ndim == 2:
- # video_code_len = codes.shape[-1]
- # assert divisible_by(video_code_len, self.fmap_size ** 2), f'flattened video ids must have a length ({video_code_len}) that is divisible by the fmap size ({self.fmap_size}) squared ({self.fmap_size ** 2})'
-
- # codes = rearrange(codes, 'b (f h w) -> b f h w', h = self.fmap_size, w = self.fmap_size)
-
- # quantized = self.quantizers.indices_to_codes(codes)
-
- # return self.decode(quantized, cond = cond, video_contains_first_frame = video_contains_first_frame)
-
- @beartype
- def decode(
- self,
- quantized: Tensor,
- cond: Optional[Tensor] = None,
- video_contains_first_frame = True
- ):
- decode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame
-
- batch = quantized.shape[0]
-
- # conditioning, if needed
-
- assert (not self.has_cond) or exists(cond), '`cond` must be passed into tokenizer forward method since conditionable layers were specified'
-
- if exists(cond): # SCH: quantized latents used as control signal following StyleGAN?
- assert cond.shape == (batch, self.dim_cond)
-
- cond = self.decoder_cond_in(cond) # SCH: linear + activation
- cond_kwargs = dict(cond = cond)
-
- # decoder layers
-
- x = quantized
-
- for fn, has_cond in zip(self.decoder_layers, reversed(self.has_cond_across_layers)):
-
- layer_kwargs = dict()
-
- if has_cond:
- layer_kwargs = cond_kwargs
-
- x = fn(x, **layer_kwargs)
-
- # to pixels
-
- if decode_first_frame_separately:
- left_pad, xff, x = x[:, :, :self.time_padding], x[:, :, self.time_padding], x[:, :, (self.time_padding + 1):]
-
- out = self.conv_out(x)
- outff = self.conv_out_first_frame(xff)
-
- video, _ = pack([outff, out], 'b c * h w')
-
- else:
- video = self.conv_out(x)
-
- # if video were padded, remove padding
-
- if video_contains_first_frame:
- video = video[:, :, self.time_padding:]
-
- return video
-
- @torch.no_grad()
- def tokenize(self, video):
- self.eval()
- return self.forward(video, return_codes = True)
-
- @beartype
- def forward(
- self,
- video_or_images: Tensor,
- cond: Optional[Tensor] = None,
- return_loss = False,
- return_codes = False,
- return_recon = False,
- return_discr_loss = False,
- return_recon_loss_only = False,
- apply_gradient_penalty = True,
- video_contains_first_frame = True,
- adversarial_loss_weight = None,
- multiscale_adversarial_loss_weight = None
- ):
- adversarial_loss_weight = default(adversarial_loss_weight, self.adversarial_loss_weight)
- multiscale_adversarial_loss_weight = default(multiscale_adversarial_loss_weight, self.multiscale_adversarial_loss_weight)
-
- assert (return_loss + return_codes + return_discr_loss) <= 1
- assert video_or_images.ndim in {4, 5}
-
- assert video_or_images.shape[-2:] == (self.image_size, self.image_size)
-
- # accept images for image pretraining (curriculum learning from images to video)
-
- is_image = video_or_images.ndim == 4
-
- if is_image:
- video = rearrange(video_or_images, 'b c ... -> b c 1 ...')
- video_contains_first_frame = True
- else:
- video = video_or_images
-
- batch, channels, frames = video.shape[:3]
-
- assert divisible_by(frames - int(video_contains_first_frame), self.time_downsample_factor), f'number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}'
-
- # encoder
-
- x = self.encode(video, cond = cond, video_contains_first_frame = video_contains_first_frame)
-
-
- ## SCH: remove the codebook
- # # lookup free quantization
-
- # if self.use_fsq:
- # quantized, codes = self.quantizers(x)
-
- # aux_losses = self.zero
- # quantizer_loss_breakdown = None
- # else:
- # (quantized, codes, aux_losses), quantizer_loss_breakdown = self.quantizers(x, return_loss_breakdown = True)
-
- # if return_codes and not return_recon:
- # return codes
-
- # decoder
- recon_video = self.decode(x, cond = cond, video_contains_first_frame = video_contains_first_frame)
-
- # if return_codes:
- # return codes, recon_video
-
- # reconstruction loss
-
- if not (return_loss or return_discr_loss or return_recon_loss_only):
- return recon_video
-
- recon_loss = F.mse_loss(video, recon_video)
-
- # for validation, only return recon loss
-
- if return_recon_loss_only:
- return recon_loss, recon_video
-
- # TODO:
- # gan discriminator loss
-
- if return_discr_loss:
- assert self.has_gan
- assert exists(self.discr)
-
- # pick a random frame for image discriminator
-
- frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices
-
- real = pick_video_frame(video, frame_indices)
-
- if apply_gradient_penalty:
- real = real.requires_grad_()
-
- fake = pick_video_frame(recon_video, frame_indices)
-
- real_logits = self.discr(real)
- fake_logits = self.discr(fake.detach())
-
- discr_loss = hinge_discr_loss(fake_logits, real_logits)
-
- # multiscale discriminators
-
- multiscale_discr_losses = []
-
- if self.has_multiscale_discrs:
- for discr in self.multiscale_discrs:
- multiscale_real_logits = discr(video)
- multiscale_fake_logits = discr(recon_video.detach())
-
- multiscale_discr_loss = hinge_discr_loss(multiscale_fake_logits, multiscale_real_logits)
-
- multiscale_discr_losses.append(multiscale_discr_loss)
- else:
- multiscale_discr_losses.append(self.zero)
-
- # gradient penalty
-
- if apply_gradient_penalty:
- gradient_penalty_loss = gradient_penalty(real, real_logits)
- else:
- gradient_penalty_loss = self.zero
-
- # total loss
-
- total_loss = discr_loss + \
- gradient_penalty_loss * self.grad_penalty_loss_weight + \
- sum(multiscale_discr_losses) * self.multiscale_adversarial_loss_weight
-
- discr_loss_breakdown = DiscrLossBreakdown(
- discr_loss,
- multiscale_discr_losses,
- gradient_penalty_loss
- )
-
- return total_loss, discr_loss_breakdown
-
- # perceptual loss
-
- if self.use_vgg:
-
- frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices
-
- input_vgg_input = pick_video_frame(video, frame_indices)
- recon_vgg_input = pick_video_frame(recon_video, frame_indices)
-
- if channels == 1:
- input_vgg_input = repeat(input_vgg_input, 'b 1 h w -> b c h w', c = 3)
- recon_vgg_input = repeat(recon_vgg_input, 'b 1 h w -> b c h w', c = 3)
-
- elif channels == 4:
- input_vgg_input = input_vgg_input[:, :3]
- recon_vgg_input = recon_vgg_input[:, :3]
-
- input_vgg_feats = self.vgg(input_vgg_input)
- recon_vgg_feats = self.vgg(recon_vgg_input)
-
- perceptual_loss = F.mse_loss(input_vgg_feats, recon_vgg_feats)
- else:
- perceptual_loss = self.zero
-
- # get gradient with respect to perceptual loss for last decoder layer
- # needed for adaptive weighting
-
- last_dec_layer = self.conv_out.conv.weight
-
- norm_grad_wrt_perceptual_loss = None
-
- if self.training and self.use_vgg and (self.has_gan or self.has_multiscale_discrs):
- norm_grad_wrt_perceptual_loss = grad_layer_wrt_loss(perceptual_loss, last_dec_layer).norm(p = 2)
-
- # per-frame image discriminator
-
- recon_video_frames = None
-
- if self.has_gan:
- frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices
- recon_video_frames = pick_video_frame(recon_video, frame_indices)
-
- fake_logits = self.discr(recon_video_frames)
- gen_loss = hinge_gen_loss(fake_logits)
-
- adaptive_weight = 1.
-
- if exists(norm_grad_wrt_perceptual_loss):
- norm_grad_wrt_gen_loss = grad_layer_wrt_loss(gen_loss, last_dec_layer).norm(p = 2)
- adaptive_weight = norm_grad_wrt_perceptual_loss / norm_grad_wrt_gen_loss.clamp(min = 1e-3)
- adaptive_weight.clamp_(max = 1e3)
-
- if torch.isnan(adaptive_weight).any():
- adaptive_weight = 1.
- else:
- gen_loss = self.zero
- adaptive_weight = 0.
-
- # multiscale discriminator losses
-
- multiscale_gen_losses = []
- multiscale_gen_adaptive_weights = []
-
- if self.has_multiscale_gan and self.has_multiscale_discrs:
- if not exists(recon_video_frames):
- recon_video_frames = pick_video_frame(recon_video, frame_indices)
-
- for discr in self.multiscale_discrs:
- fake_logits = recon_video_frames
- multiscale_gen_loss = hinge_gen_loss(fake_logits)
-
- multiscale_gen_losses.append(multiscale_gen_loss)
-
- multiscale_adaptive_weight = 1.
-
- if exists(norm_grad_wrt_perceptual_loss):
- norm_grad_wrt_gen_loss = grad_layer_wrt_loss(multiscale_gen_loss, last_dec_layer).norm(p = 2)
- multiscale_adaptive_weight = norm_grad_wrt_perceptual_loss / norm_grad_wrt_gen_loss.clamp(min = 1e-5)
- multiscale_adaptive_weight.clamp_(max = 1e3)
-
- multiscale_gen_adaptive_weights.append(multiscale_adaptive_weight)
-
- # calculate total loss
-
- total_loss = recon_loss \
- + aux_losses * self.quantizer_aux_loss_weight \
- + perceptual_loss * self.perceptual_loss_weight \
- + gen_loss * adaptive_weight * adversarial_loss_weight
-
- if self.has_multiscale_discrs:
-
- weighted_multiscale_gen_losses = sum(loss * weight for loss, weight in zip(multiscale_gen_losses, multiscale_gen_adaptive_weights))
-
- total_loss = total_loss + weighted_multiscale_gen_losses * multiscale_adversarial_loss_weight
-
- # loss breakdown
-
- loss_breakdown = LossBreakdown(
- recon_loss,
- aux_losses,
- quantizer_loss_breakdown,
- perceptual_loss,
- gen_loss,
- adaptive_weight,
- multiscale_gen_losses,
- multiscale_gen_adaptive_weights
- )
-
- return total_loss, loss_breakdown
-
-# main class
-class MagViT2(Module):
- def __init__(self):
- super().__init__()
-
- def forward(self, x):
- return x
diff --git a/opensora/models/vae/model_utils.py b/opensora/models/vae/model_utils.py
deleted file mode 100644
index c00f886..0000000
--- a/opensora/models/vae/model_utils.py
+++ /dev/null
@@ -1,232 +0,0 @@
-import numpy as np
-import torch
-import torch.nn as nn
-
-# from taming.modules.losses.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers
-# from taming.modules.discriminator.model import NLayerDiscriminator, weights_init
-from einops import rearrange
-
-"""Stripped version of https://github.com/richzhang/PerceptualSimilarity/tree/master/models"""
-
-
-## NOTE: not used since we only have 'GN'
-# def get_norm_layer(norm_type, dtype):
-# if norm_type == 'LN':
-# # supply a few args with partial function and pass the rest of the args when this norm_fn is called
-# norm_fn = functools.partial(nn.LayerNorm, dtype=dtype)
-# elif norm_type == 'GN': #
-# norm_fn = functools.partial(nn.GroupNorm, dtype=dtype)
-# elif norm_type is None:
-# norm_fn = lambda: (lambda x: x)
-# else:
-# raise NotImplementedError(f'norm_type: {norm_type}')
-# return norm_fn
-
-
-class DiagonalGaussianDistribution(object):
- def __init__(
- self,
- parameters,
- deterministic=False,
- ):
- self.parameters = parameters
- self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) # SCH: channels dim
- self.logvar = torch.clamp(self.logvar, -30.0, 20.0)
- self.deterministic = deterministic
- self.std = torch.exp(0.5 * self.logvar)
- self.var = torch.exp(self.logvar)
- if self.deterministic:
- self.var = self.std = torch.zeros_like(self.mean).to(device=self.parameters.device, dtype=self.mean.dtype)
-
- def sample(self):
- # torch.randn: standard normal distribution
- x = self.mean + self.std * torch.randn(self.mean.shape).to(device=self.parameters.device, dtype=self.mean.dtype)
- return x
-
- def kl(self, other=None):
- if self.deterministic:
- return torch.Tensor([0.0])
- else:
- if other is None: # SCH: assumes other is a standard normal distribution
- return 0.5 * torch.sum(torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar, dim=[1, 2, 3, 4])
- else:
- return 0.5 * torch.sum(
- torch.pow(self.mean - other.mean, 2) / other.var
- + self.var / other.var
- - 1.0
- - self.logvar
- + other.logvar,
- dim=[1, 2, 3, 4],
- )
-
- def nll(self, sample, dims=[1, 2, 3, 4]): # TODO: what does this do?
- if self.deterministic:
- return torch.Tensor([0.0])
- logtwopi = np.log(2.0 * np.pi)
- return 0.5 * torch.sum(logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var, dim=dims)
-
- def mode(self): # SCH: used for vae inference?
- return self.mean
-
-
-class VEA3DLoss(nn.Module):
- def __init__(
- self,
- # disc_start,
- logvar_init=0.0,
- kl_weight=1.0,
- pixelloss_weight=1.0,
- perceptual_weight=0.1,
- disc_loss="hinge",
- ):
- super().__init__()
- assert disc_loss in ["hinge", "vanilla"]
- self.kl_weight = kl_weight
- self.pixel_weight = pixelloss_weight
- self.perceptual_loss = LPIPS().eval()
- self.perceptual_weight = perceptual_weight
- # output log variance
- self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)
-
- def forward(
- self,
- inputs,
- reconstructions,
- posteriors,
- # optimizer_idx,
- # global_step,
- weights=None,
- ):
- rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous())
- if self.perceptual_weight > 0: # NOTE: need in_channels == 3 in order to use!
- assert inputs.size(1) == 3, f"using vgg16 that requires 3 input channels but got {inputs.size(1)}"
- # SCH: transform to [(B,T), C, H, W] shape for percetual loss over each frame
- B = inputs.shape[0]
- inputs = rearrange(inputs, "B C T H W -> (B T) C H W")
- reconstructions = rearrange(reconstructions, "B C T H W -> (B T) C H W")
- # permutated_input = torch.permute(inputs, (0, 2, 1, 3, 4)) # [B, C, T, H, W] --> [B, T, C, H, W]
- # permutated_rec = torch.permute(reconstructions, (0, 2, 1, 3, 4))
- # data_shape = permutated_input.size()
- # p_loss = self.perceptual_loss(
- # permutated_input.reshape(-1, data_shape[-3], data_shape[-2],data_shape[-1]).contiguous(),
- # permutated_rec.reshape(-1, data_shape[-3], data_shape[-2],data_shape[-1]).contiguous()
- # )
- p_loss = self.perceptual_loss(inputs.contiguous(), reconstructions.contiguous())
- # SCH: shape back p_loss
- # permuted_p_loss = torch.permute(p_loss.reshape(data_shape[0], data_shape[1], 1, 1, 1), (0,2,1,3,4))
- # rec_loss = rec_loss + self.perceptual_weight * permuted_p_loss
- p_loss = rearrange(p_loss, "(B T) C H W -> B C T H W", B=B)
- rec_loss = rec_loss + self.perceptual_weight * p_loss
-
- nll_loss = rec_loss / torch.exp(self.logvar) + self.logvar
- weighted_nll_loss = nll_loss
- if weights is not None:
- weighted_nll_loss = weights * nll_loss
- weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]
- nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]
- kl_loss = posteriors.kl()
- kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]
-
- loss = weighted_nll_loss + self.kl_weight * kl_loss # TODO: add discriminator loss later
-
- return loss
-
-
-class VEA3DLossWithDiscriminator(nn.Module):
- def __init__(
- self,
- # disc_start,
- logvar_init=0.0,
- kl_weight=1.0,
- pixelloss_weight=1.0,
- disc_num_layers=3,
- disc_in_channels=3,
- disc_factor=1.0,
- disc_weight=1.0,
- perceptual_weight=1.0,
- use_actnorm=False,
- disc_conditional=False,
- disc_loss="hinge",
- ):
- super().__init__()
- assert disc_loss in ["hinge", "vanilla"]
- self.kl_weight = kl_weight
- self.pixel_weight = pixelloss_weight
- self.perceptual_loss = LPIPS().eval()
- self.perceptual_weight = perceptual_weight
- # output log variance
- self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)
-
- # self.discriminator = NLayerDiscriminator(input_nc=disc_in_channels,
- # n_layers=disc_num_layers,
- # use_actnorm=use_actnorm
- # ).apply(weights_init)
- # self.discriminator_iter_start = disc_start
- # self.disc_loss = hinge_d_loss if disc_loss == "hinge" else vanilla_d_loss
- # self.disc_factor = disc_factor
- # self.discriminator_weight = disc_weight
- # self.disc_conditional = disc_conditional
-
- # TODO: for discriminator
- # def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer=None):
- # if last_layer is not None:
- # nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0]
- # g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0]
- # else:
- # nll_grads = torch.autograd.grad(nll_loss, self.last_layer[0], retain_graph=True)[0]
- # g_grads = torch.autograd.grad(g_loss, self.last_layer[0], retain_graph=True)[0]
-
- # d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)
- # d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()
- # d_weight = d_weight * self.discriminator_weight
- # return d_weight
-
- def forward(
- self,
- inputs,
- reconstructions,
- posteriors,
- # optimizer_idx,
- # global_step,
- last_layer=None,
- cond=None,
- split="train",
- weights=None,
- ):
- rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous())
- if self.perceptual_weight > 0: # NOTE: need in_channels == 3 in order to use!
- assert inputs.size(1) == 3, f"using vgg16 that requires 3 input channels but got {inputs.size(1)} "
- # SCH: transform to [(B,T), C, H, W] shape for percetual loss over each frame
- permutated_input = torch.permute(inputs, (0, 2, 1, 3, 4)) # [B, C, T, H, W] --> [B, T, C, H, W]
- permutated_rec = torch.permute(reconstructions, (0, 2, 1, 3, 4))
- data_shape = permutated_input.size()
- p_loss = self.perceptual_loss(
- permutated_input.reshape(-1, data_shape[-3], data_shape[-2], data_shape[-1]).contiguous(),
- permutated_rec.reshape(-1, data_shape[-3], data_shape[-2], data_shape[-1]).contiguous(),
- )
- # SCH: shape back p_loss
- permuted_p_loss = torch.permute(p_loss.reshape(data_shape[0], data_shape[1], 1, 1, 1), (0, 2, 1, 3, 4))
- rec_loss = rec_loss + self.perceptual_weight * permuted_p_loss
-
- nll_loss = rec_loss / torch.exp(self.logvar) + self.logvar
- weighted_nll_loss = nll_loss
- if weights is not None:
- weighted_nll_loss = weights * nll_loss
- weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]
- nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]
- kl_loss = posteriors.kl()
- kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]
-
- loss = weighted_nll_loss + self.kl_weight * kl_loss # TODO: add discriminator loss later
-
- # log = {"{}/total_loss".format(split): loss.clone().detach().mean(),
- # "{}/logvar".format(split): self.logvar.detach(),
- # "{}/kl_loss".format(split): kl_loss.detach().mean(),
- # "{}/nll_loss".format(split): nll_loss.detach().mean(),
- # "{}/rec_loss".format(split): rec_loss.detach().mean(),
- # # "{}/d_weight".format(split): d_weight.detach(),
- # # "{}/disc_factor".format(split): torch.tensor(disc_factor),
- # # "{}/g_loss".format(split): g_loss.detach().mean(),
- # }
-
- return loss
diff --git a/opensora/models/vae/utils.py b/opensora/models/vae/utils.py
new file mode 100644
index 0000000..fd57391
--- /dev/null
+++ b/opensora/models/vae/utils.py
@@ -0,0 +1,70 @@
+import numpy as np
+import torch
+import torch.nn as nn
+
+# from taming.modules.losses.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers
+# from taming.modules.discriminator.model import NLayerDiscriminator, weights_init
+from einops import rearrange
+
+"""Stripped version of https://github.com/richzhang/PerceptualSimilarity/tree/master/models"""
+
+
+## NOTE: not used since we only have 'GN'
+# def get_norm_layer(norm_type, dtype):
+# if norm_type == 'LN':
+# # supply a few args with partial function and pass the rest of the args when this norm_fn is called
+# norm_fn = functools.partial(nn.LayerNorm, dtype=dtype)
+# elif norm_type == 'GN': #
+# norm_fn = functools.partial(nn.GroupNorm, dtype=dtype)
+# elif norm_type is None:
+# norm_fn = lambda: (lambda x: x)
+# else:
+# raise NotImplementedError(f'norm_type: {norm_type}')
+# return norm_fn
+
+
+class DiagonalGaussianDistribution(object):
+ def __init__(
+ self,
+ parameters,
+ deterministic=False,
+ ):
+ self.parameters = parameters
+ self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) # SCH: channels dim
+ self.logvar = torch.clamp(self.logvar, -30.0, 20.0)
+ self.deterministic = deterministic
+ self.std = torch.exp(0.5 * self.logvar)
+ self.var = torch.exp(self.logvar)
+ if self.deterministic:
+ self.var = self.std = torch.zeros_like(self.mean).to(device=self.parameters.device, dtype=self.mean.dtype)
+
+ def sample(self):
+ # torch.randn: standard normal distribution
+ x = self.mean + self.std * torch.randn(self.mean.shape).to(device=self.parameters.device, dtype=self.mean.dtype)
+ return x
+
+ def kl(self, other=None):
+ if self.deterministic:
+ return torch.Tensor([0.0])
+ else:
+ if other is None: # SCH: assumes other is a standard normal distribution
+ return 0.5 * torch.sum(torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar, dim=[1, 2, 3, 4])
+ else:
+ return 0.5 * torch.sum(
+ torch.pow(self.mean - other.mean, 2) / other.var
+ + self.var / other.var
+ - 1.0
+ - self.logvar
+ + other.logvar,
+ dim=[1, 2, 3, 4],
+ )
+
+ def nll(self, sample, dims=[1, 2, 3, 4]): # TODO: what does this do?
+ if self.deterministic:
+ return torch.Tensor([0.0])
+ logtwopi = np.log(2.0 * np.pi)
+ return 0.5 * torch.sum(logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var, dim=dims)
+
+ def mode(self): # SCH: used for vae inference?
+ return self.mean
+
diff --git a/opensora/models/vae/vae_3d.py b/opensora/models/vae/vae_3d.py
index 1907fb4..5b74ed0 100644
--- a/opensora/models/vae/vae_3d.py
+++ b/opensora/models/vae/vae_3d.py
@@ -1,13 +1,20 @@
import functools
-from typing import Any, Dict, Tuple, Type, Union, Sequence, Optional
-from absl import logging
+import math
+from typing import Tuple, Union
+
+import numpy as np
import torch
-import torch.nn as nn
-import numpy as np
-from numpy import typing as nptyping
-from opensora.models.vae import model_utils
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import pack, rearrange, repeat, unpack
+
+from .utils import DiagonalGaussianDistribution
+from .lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers
from opensora.registry import MODELS
-from opensora.utils.ckpt_utils import load_checkpoint
+from opensora.utils.ckpt_utils import find_model, load_checkpoint
+
+# from diffusers.models.modeling_utils import ModelMixin
+
"""Encoder and Decoder stuctures with 3D CNNs."""
@@ -19,76 +26,580 @@ NOTE:
!!! opensora read video into [B,C,T,H,W] format output
-TODO:
- check data dimensions format
"""
+
+def cast_tuple(t, length=1):
+ return t if isinstance(t, tuple) else ((t,) * length)
+
+
+def divisible_by(num, den):
+ return (num % den) == 0
+
+
+def is_odd(n):
+ return not divisible_by(n, 2)
+
+
+def pad_at_dim(t, pad, dim=-1, value=0.0):
+ dims_from_right = (-dim - 1) if dim < 0 else (t.ndim - dim - 1)
+ zeros = (0, 0) * dims_from_right
+ return F.pad(t, (*zeros, *pad), value=value)
+
+
+def pick_video_frame(video, frame_indices):
+ """get frame_indices from the video of [B, C, T, H, W] and return images of [B, C, H, W]"""
+ batch, device = video.shape[0], video.device
+ video = rearrange(video, "b c f ... -> b f c ...")
+ batch_indices = torch.arange(batch, device=device)
+ batch_indices = rearrange(batch_indices, "b -> b 1")
+ images = video[batch_indices, frame_indices]
+ images = rearrange(images, "b 1 c ... -> b c ...")
+ return images
+
+
+def exists(v):
+ return v is not None
+
+
+# ============== Generator Adversarial Loss Functions ==============
+def lecam_reg(real_pred, fake_pred, ema_real_pred, ema_fake_pred):
+ assert real_pred.ndim == 0 and ema_fake_pred.ndim == 0
+ lecam_loss = torch.mean(torch.pow(nn.ReLU()(real_pred - ema_fake_pred), 2))
+ lecam_loss += torch.mean(torch.pow(nn.ReLU()(ema_real_pred - fake_pred), 2))
+ return lecam_loss
+
+
+# Open-Sora-Plan
+# Very bad, do not use
+def r1_penalty(real_img, real_pred):
+ """R1 regularization for discriminator. The core idea is to
+ penalize the gradient on real data alone: when the
+ generator distribution produces the true data distribution
+ and the discriminator is equal to 0 on the data manifold, the
+ gradient penalty ensures that the discriminator cannot create
+ a non-zero gradient orthogonal to the data manifold without
+ suffering a loss in the GAN game.
+
+ Ref:
+ Eq. 9 in Which training methods for GANs do actually converge.
+ """
+ grad_real = torch.autograd.grad(outputs=real_pred.sum(), inputs=real_img, create_graph=True)[0]
+ grad_penalty = grad_real.pow(2).view(grad_real.shape[0], -1).sum(1).mean()
+ return grad_penalty
+
+
+# Open-Sora-Plan
+# Implementation as described by https://arxiv.org/abs/1704.00028 # TODO: checkout the codes
+def gradient_penalty_loss(discriminator, real_data, fake_data, weight=None):
+ """Calculate gradient penalty for wgan-gp.
+
+ Args:
+ discriminator (nn.Module): Network for the discriminator.
+ real_data (Tensor): Real input data.
+ fake_data (Tensor): Fake input data.
+ weight (Tensor): Weight tensor. Default: None.
+
+ Returns:
+ Tensor: A tensor for gradient penalty.
+ """
+
+ batch_size = real_data.size(0)
+ alpha = real_data.new_tensor(torch.rand(batch_size, 1, 1, 1))
+
+ # interpolate between real_data and fake_data
+ interpolates = alpha * real_data + (1.0 - alpha) * fake_data
+ interpolates = torch.autograd.Variable(interpolates, requires_grad=True)
+
+ disc_interpolates = discriminator(interpolates)
+ gradients = torch.autograd.grad(
+ outputs=disc_interpolates,
+ inputs=interpolates,
+ grad_outputs=torch.ones_like(disc_interpolates),
+ create_graph=True,
+ retain_graph=True,
+ only_inputs=True,
+ )[0]
+
+ if weight is not None:
+ gradients = gradients * weight
+
+ gradients_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean()
+ if weight is not None:
+ gradients_penalty /= torch.mean(weight)
+
+ return gradients_penalty
+
+
+def gradient_penalty_fn(images, output):
+ # batch_size = images.shape[0]
+ gradients = torch.autograd.grad(
+ outputs=output,
+ inputs=images,
+ grad_outputs=torch.ones(output.size(), device=images.device),
+ create_graph=True,
+ retain_graph=True,
+ only_inputs=True,
+ )[0]
+
+ gradients = rearrange(gradients, "b ... -> b (...)")
+ return ((gradients.norm(2, dim=1) - 1) ** 2).mean()
+
+
+# ============== Discriminator Adversarial Loss Functions ==============
+def hinge_d_loss(logits_real, logits_fake):
+ loss_real = torch.mean(F.relu(1.0 - logits_real))
+ loss_fake = torch.mean(F.relu(1.0 + logits_fake))
+ d_loss = 0.5 * (loss_real + loss_fake)
+ return d_loss
+
+
+def vanilla_d_loss(logits_real, logits_fake):
+ d_loss = 0.5 * (
+ torch.mean(torch.nn.functional.softplus(-logits_real)) + torch.mean(torch.nn.functional.softplus(logits_fake))
+ )
+ return d_loss
+
+
+# from MAGVIT, used in place hof hinge_d_loss
+def sigmoid_cross_entropy_with_logits(labels, logits):
+ # The final formulation is: max(x, 0) - x * z + log(1 + exp(-abs(x)))
+ zeros = torch.zeros_like(logits, dtype=logits.dtype)
+ condition = logits >= zeros
+ relu_logits = torch.where(condition, logits, zeros)
+ neg_abs_logits = torch.where(condition, -logits, logits)
+ return relu_logits - logits * labels + torch.log1p(torch.exp(neg_abs_logits))
+
+
+def xavier_uniform_weight_init(m):
+ if isinstance(m, nn.Conv3d) or isinstance(m, nn.Linear):
+ nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain("relu"))
+ if m.bias is not None:
+ nn.init.zeros_(m.bias)
+ # print("initialized module to xavier_uniform:", m)
+
+
+def Sequential(*modules):
+ modules = [*filter(exists, modules)]
+
+ if len(modules) == 0:
+ return nn.Identity()
+
+ return nn.Sequential(*modules)
+
+
+def SameConv2d(dim_in, dim_out, kernel_size):
+ kernel_size = cast_tuple(kernel_size, 2)
+ padding = [k // 2 for k in kernel_size]
+ return nn.Conv2d(dim_in, dim_out, kernel_size=kernel_size, padding=padding)
+
+
+def adopt_weight(weight, global_step, threshold=0, value=0.0):
+ if global_step < threshold:
+ weight = value
+ return weight
+
+
+class CausalConv3d(nn.Module):
+ def __init__(
+ self,
+ chan_in,
+ chan_out,
+ kernel_size: Union[int, Tuple[int, int, int]],
+ pad_mode="constant",
+ strides=None, # allow custom stride
+ **kwargs,
+ ):
+ super().__init__()
+ kernel_size = cast_tuple(kernel_size, 3)
+
+ time_kernel_size, height_kernel_size, width_kernel_size = kernel_size
+
+ assert is_odd(height_kernel_size) and is_odd(width_kernel_size)
+
+ dilation = kwargs.pop("dilation", 1)
+ stride = strides[0] if strides is not None else kwargs.pop("stride", 1)
+
+ self.pad_mode = pad_mode
+ time_pad = dilation * (time_kernel_size - 1) + (1 - stride)
+ height_pad = height_kernel_size // 2
+ width_pad = width_kernel_size // 2
+
+ self.time_pad = time_pad
+ self.time_causal_padding = (width_pad, width_pad, height_pad, height_pad, time_pad, 0)
+
+ stride = strides if strides is not None else (stride, 1, 1)
+ # padding = kwargs.pop('padding', 0)
+ # if padding == "same" and not all([pad == 1 for pad in padding]):
+ # padding = "valid"
+ dilation = (dilation, 1, 1)
+ self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)
+
+ def forward(self, x):
+ pad_mode = self.pad_mode if self.time_pad < x.shape[2] else "constant"
+
+ x = F.pad(x, self.time_causal_padding, mode=pad_mode)
+ return self.conv(x)
+
+
class ResBlock(nn.Module):
def __init__(
- self,
- in_out_channels, # SCH: added
- filters,
- # norm_fn, # SCH: removed, use GN
- conv_fn,
- activation_fn=nn.ReLU,
- use_conv_shortcut=False,
- num_groups=32,
- device="cpu",
- dtype=torch.bfloat16,
+ self,
+ in_channels, # SCH: added
+ filters,
+ conv_fn,
+ activation_fn=nn.SiLU,
+ use_conv_shortcut=False,
+ num_groups=32,
+ device="cpu",
+ dtype=torch.bfloat16,
):
super().__init__()
+ self.in_channels = in_channels
self.filters = filters
self.activate = activation_fn()
self.use_conv_shortcut = use_conv_shortcut
-
+
# SCH: MAGVIT uses GroupNorm by default
- self.norm1 = nn.GroupNorm(num_groups, in_out_channels, device=device, dtype=dtype)
- self.conv1 = conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)
+ self.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)
+ self.conv1 = conv_fn(in_channels, self.filters, kernel_size=(3, 3, 3), bias=False)
self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)
self.conv2 = conv_fn(self.filters, self.filters, kernel_size=(3, 3, 3), bias=False)
- if self.use_conv_shortcut:
- self.conv3 = conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)
- else:
- self.conv3 = conv_fn(in_out_channels, self.filters, kernel_size=(1, 1, 1), bias=False)
-
+ if in_channels != filters:
+ if self.use_conv_shortcut:
+ self.conv3 = conv_fn(in_channels, self.filters, kernel_size=(3, 3, 3), bias=False)
+ else:
+ self.conv3 = conv_fn(in_channels, self.filters, kernel_size=(1, 1, 1), bias=False)
def forward(self, x):
- device, dtype = x.device, x.dtype
- input_dim = x.shape[1]
+ # device, dtype = x.device, x.dtype
+ # input_dim = x.shape[1]
residual = x
- x = self.norm1.to(device,dtype)(x)
+ x = self.norm1(x)
x = self.activate(x)
- x = self.conv1.to(device,dtype)(x)
- x = self.norm2.to(device, dtype)(x)
+ x = self.conv1(x)
+ x = self.norm2(x)
x = self.activate(x)
- x = self.conv2.to(device, dtype)(x)
- if input_dim != self.filters: # TODO: what does it do here
- residual = self.conv3.to(device, dtype)(residual)
- return x + residual
-
-def _get_selected_flags(total_len: int, select_len: int, suffix: bool):
- assert select_len <= total_len
- selected = np.zeros(total_len, dtype=bool)
- if not suffix:
- selected[:select_len] = True
+ x = self.conv2(x)
+ if self.in_channels != self.filters: # SCH: ResBlock X->Y
+ residual = self.conv3(residual)
+ return x + residual
+
+
+# SCH: own implementation modified on top of: discriminator with anti-aliased downsampling (blurpool Zhang et al.)
+class BlurPool3D(nn.Module):
+ def __init__(
+ self,
+ channels,
+ pad_type="reflect",
+ filt_size=3,
+ stride=2,
+ pad_off=0,
+ device="cpu",
+ dtype=torch.bfloat16,
+ ):
+ super(BlurPool3D, self).__init__()
+ self.filt_size = filt_size
+ self.pad_off = pad_off
+ self.pad_sizes = [
+ int(1.0 * (filt_size - 1) / 2),
+ int(np.ceil(1.0 * (filt_size - 1) / 2)),
+ int(1.0 * (filt_size - 1) / 2),
+ int(np.ceil(1.0 * (filt_size - 1) / 2)),
+ int(1.0 * (filt_size - 1) / 2),
+ int(np.ceil(1.0 * (filt_size - 1) / 2)),
+ ]
+ self.pad_sizes = [pad_size + pad_off for pad_size in self.pad_sizes]
+ self.stride = stride
+ self.off = int((self.stride - 1) / 2.0)
+ self.channels = channels
+
+ if self.filt_size == 1:
+ a = np.array(
+ [
+ 1.0,
+ ]
+ )
+ elif self.filt_size == 2:
+ a = np.array([1.0, 1.0])
+ elif self.filt_size == 3:
+ a = np.array([1.0, 2.0, 1.0])
+ elif self.filt_size == 4:
+ a = np.array([1.0, 3.0, 3.0, 1.0])
+ elif self.filt_size == 5:
+ a = np.array([1.0, 4.0, 6.0, 4.0, 1.0])
+ elif self.filt_size == 6:
+ a = np.array([1.0, 5.0, 10.0, 10.0, 5.0, 1.0])
+ elif self.filt_size == 7:
+ a = np.array([1.0, 6.0, 15.0, 20.0, 15.0, 6.0, 1.0])
+
+ filt_2d = a[:, None] * a[None, :]
+ filt_3d = torch.Tensor(a[:, None, None] * filt_2d[None, :, :]).to(device, dtype)
+
+ filt = filt_3d / torch.sum(filt_3d) # SCH: modified to it 3D
+ self.register_buffer("filt", filt[None, None, :, :, :].repeat((self.channels, 1, 1, 1, 1)))
+
+ self.pad = get_pad_layer(pad_type)(self.pad_sizes)
+
+ def forward(self, inp):
+ if self.filt_size == 1:
+ if self.pad_off == 0:
+ return inp[:, :, :: self.stride, :: self.stride]
+ else:
+ return self.pad(inp)[:, :, :: self.stride, :: self.stride]
+ else:
+ return F.conv3d(self.pad(inp), self.filt, stride=self.stride, groups=inp.shape[1])
+
+
+def get_pad_layer(pad_type):
+ if pad_type in ["refl", "reflect"]:
+ PadLayer = nn.ReflectionPad3d
+ elif pad_type in ["repl", "replicate"]:
+ PadLayer = nn.ReplicationPad3d
+ elif pad_type == "zero":
+ PadLayer = nn.ZeroPad3d
else:
- selected[-select_len:] = True
- return selected
+ print("Pad type [%s] not recognized" % pad_type)
+ return PadLayer
+
+
+class ResBlockDown(nn.Module):
+ """3D StyleGAN ResBlock for D."""
+
+ def __init__(
+ self,
+ in_channels,
+ filters,
+ activation_fn,
+ num_groups=32,
+ device="cpu",
+ dtype=torch.bfloat16,
+ ):
+ super().__init__()
+
+ self.filters = filters
+ self.activation_fn = activation_fn
+
+ # SCH: NOTE: although paper says conv (X->Y, Y->Y), original code implementation is (X->X, X->Y), we follow code
+ self.conv1 = nn.Conv3d(
+ in_channels, in_channels, (3, 3, 3), padding=1, device=device, dtype=dtype
+ ) # NOTE: init to xavier_uniform
+ self.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)
+
+ self.blur = BlurPool3D(in_channels, device=device, dtype=dtype)
+
+ self.conv2 = nn.Conv3d(
+ in_channels, self.filters, (1, 1, 1), bias=False, device=device, dtype=dtype
+ ) # NOTE: init to xavier_uniform
+ self.conv3 = nn.Conv3d(
+ in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype
+ ) # NOTE: init to xavier_uniform
+ self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)
+
+ # self.apply(xavier_uniform_weight_init)
+
+ def forward(self, x):
+ residual = x
+ x = self.conv1(x)
+ x = self.norm1(x)
+ x = self.activation_fn(x)
+
+ residual = self.blur(residual)
+ residual = self.conv2(residual)
+
+ x = self.blur(x)
+ x = self.conv3(x)
+ x = self.norm2(x)
+ x = self.activation_fn(x)
+ out = (residual + x) / math.sqrt(2)
+ return out
+
+
+# SCH: taken from Open Sora Plan
+def n_layer_disc_weights_init(m):
+ classname = m.__class__.__name__
+ if classname.find("Conv") != -1:
+ nn.init.normal_(m.weight.data, 0.0, 0.02)
+ elif classname.find("BatchNorm") != -1:
+ nn.init.normal_(m.weight.data, 1.0, 0.02)
+ nn.init.constant_(m.bias.data, 0)
+
+
+class NLayerDiscriminator3D(nn.Module):
+ """Defines a 3D PatchGAN discriminator as in Pix2Pix but for 3D inputs."""
+
+ def __init__(self, input_nc=1, ndf=64, n_layers=3, use_actnorm=False):
+ """
+ Construct a 3D PatchGAN discriminator
+
+ Parameters:
+ input_nc (int) -- the number of channels in input volumes
+ ndf (int) -- the number of filters in the last conv layer
+ n_layers (int) -- the number of conv layers in the discriminator
+ use_actnorm (bool) -- flag to use actnorm instead of batchnorm
+ """
+ super(NLayerDiscriminator3D, self).__init__()
+ if not use_actnorm:
+ norm_layer = nn.BatchNorm3d
+ else:
+ raise NotImplementedError("Not implemented.")
+ if type(norm_layer) == functools.partial:
+ use_bias = norm_layer.func != nn.BatchNorm3d
+ else:
+ use_bias = norm_layer != nn.BatchNorm3d
+
+ kw = 4
+ padw = 1
+ sequence = [nn.Conv3d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]
+ nf_mult = 1
+ nf_mult_prev = 1
+ for n in range(1, n_layers): # gradually increase the number of filters
+ nf_mult_prev = nf_mult
+ nf_mult = min(2**n, 8)
+ sequence += [
+ nn.Conv3d(
+ ndf * nf_mult_prev,
+ ndf * nf_mult,
+ kernel_size=(kw, kw, kw),
+ stride=(1, 2, 2),
+ padding=padw,
+ bias=use_bias,
+ ),
+ norm_layer(ndf * nf_mult),
+ nn.LeakyReLU(0.2, True),
+ ]
+
+ nf_mult_prev = nf_mult
+ nf_mult = min(2**n_layers, 8)
+ sequence += [
+ nn.Conv3d(
+ ndf * nf_mult_prev, ndf * nf_mult, kernel_size=(kw, kw, kw), stride=1, padding=padw, bias=use_bias
+ ),
+ norm_layer(ndf * nf_mult),
+ nn.LeakyReLU(0.2, True),
+ ]
+
+ sequence += [
+ nn.Conv3d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)
+ ] # output 1 channel prediction map
+ self.main = nn.Sequential(*sequence)
+
+ def forward(self, input):
+ """Standard forward."""
+ return self.main(input)
+
+
+class StyleGANDiscriminatorBlur(nn.Module):
+ """StyleGAN Discriminator."""
+
+ """
+ SCH: NOTE:
+ this discriminator requries the num_frames to be fixed during training;
+ in case we pre-train with image then train on video, this disciminator's Linear layer would have to be re-trained!
+ """
+
+ def __init__(
+ self,
+ image_size=(128, 128),
+ num_frames=17,
+ in_channels=3,
+ filters=128,
+ channel_multipliers=(2, 4, 4, 4, 4),
+ num_groups=32,
+ dtype=torch.bfloat16,
+ device="cpu",
+ ):
+ super().__init__()
+
+ self.dtype = dtype
+ self.input_size = cast_tuple(image_size, 2)
+ self.filters = filters
+ self.activation_fn = nn.LeakyReLU(negative_slope=0.2)
+ self.channel_multipliers = channel_multipliers
+
+ self.conv1 = nn.Conv3d(
+ in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype
+ ) # NOTE: init to xavier_uniform
+
+ prev_filters = self.filters # record in_channels
+ self.num_blocks = len(self.channel_multipliers)
+ self.res_block_list = nn.ModuleList([])
+ for i in range(self.num_blocks):
+ filters = self.filters * self.channel_multipliers[i]
+ self.res_block_list.append(
+ ResBlockDown(prev_filters, filters, self.activation_fn, device=device, dtype=dtype).apply(
+ xavier_uniform_weight_init
+ )
+ )
+ prev_filters = filters # update in_channels
+
+ self.conv2 = nn.Conv3d(
+ prev_filters, prev_filters, (3, 3, 3), padding=1, device=device, dtype=dtype
+ ) # NOTE: init to xavier_uniform
+ # torch.nn.init.xavier_uniform_(self.conv2.weight)
+
+ self.norm1 = nn.GroupNorm(num_groups, prev_filters, dtype=dtype, device=device)
+
+ scale_factor = 2**self.num_blocks
+ if num_frames % scale_factor != 0: # SCH: NOTE: has first frame which would be padded before usage
+ time_scaled = num_frames // scale_factor + 1
+ else:
+ time_scaled = num_frames / scale_factor
+
+ assert (
+ self.input_size[0] % scale_factor == 0
+ ), f"image width {self.input_size[0]} is not divisible by scale factor {scale_factor}"
+ assert (
+ self.input_size[1] % scale_factor == 0
+ ), f"image height {self.input_size[1]} is not divisible by scale factor {scale_factor}"
+ w_scaled, h_scaled = self.input_size[0] / scale_factor, self.input_size[1] / scale_factor
+ in_features = int(prev_filters * time_scaled * w_scaled * h_scaled) # (C*T*W*H)
+ self.linear1 = nn.Linear(in_features, prev_filters, device=device, dtype=dtype) # NOTE: init to xavier_uniform
+ self.linear2 = nn.Linear(prev_filters, 1, device=device, dtype=dtype) # NOTE: init to xavier_uniform
+
+ # self.apply(xavier_uniform_weight_init)
+
+ def forward(self, x):
+ x = self.conv1(x)
+ # print("discriminator aft conv:", x.size())
+ x = self.activation_fn(x)
+
+ for i in range(self.num_blocks):
+ x = self.res_block_list[i](x)
+ # print("discriminator resblock down:", x.size())
+
+ x = self.conv2(x)
+ # print("discriminator aft conv2:", x.size())
+ x = self.norm1(x)
+ x = self.activation_fn(x)
+ x = x.reshape((x.shape[0], -1)) # SCH: [B, (C * T * W * H)] ?
+
+ # print("discriminator reshape:", x.size())
+ x = self.linear1(x)
+ # print("discriminator aft linear1:", x.size())
+
+ x = self.activation_fn(x)
+ x = self.linear2(x)
+ # print("discriminator aft linear2:", x.size())
+ return x
class Encoder(nn.Module):
"""Encoder Blocks."""
- def __init__(self,
- filters = 64,
- num_res_blocks = 2,
- channel_multipliers = (1, 2, 2, 4),
- temporal_downsample = (True, True, False),
- num_groups = 32, # for nn.GroupNorm
- in_out_channels = 3, # SCH: added, in_channels at the start
- latent_embed_dim = 256, # num channels for latent vector
- conv_downsample = False,
- custom_conv_padding = None,
- activation_fn = 'swish',
+
+ def __init__(
+ self,
+ filters=128,
+ num_res_blocks=4,
+ channel_multipliers=(1, 2, 2, 4),
+ temporal_downsample=(False, True, True),
+ num_groups=32, # for nn.GroupNorm
+ # in_out_channels = 3, # SCH: added, in_channels at the start
+ latent_embed_dim=512, # num channels for latent vector
+ # conv_downsample = False,
+ disable_spatial_downsample=False, # for vae pipeline
+ custom_conv_padding=None,
+ activation_fn="swish",
device="cpu",
dtype=torch.bfloat16,
):
@@ -99,37 +610,27 @@ class Encoder(nn.Module):
self.temporal_downsample = temporal_downsample
self.num_groups = num_groups
- if isinstance(self.temporal_downsample, int):
- self.temporal_downsample = _get_selected_flags(
- len(self.channel_multipliers) - 1, self.temporal_downsample, False)
-
self.embedding_dim = latent_embed_dim
- self.conv_downsample = conv_downsample
+ self.disable_spatial_downsample = disable_spatial_downsample
+ # self.conv_downsample = conv_downsample
self.custom_conv_padding = custom_conv_padding
- # self.norm_type = self.config.vqvae.norm_type
- # self.num_remat_block = self.config.vqvae.get('num_enc_remat_blocks', 0)
- if activation_fn == 'relu':
+ if activation_fn == "relu":
self.activation_fn = nn.ReLU
- elif activation_fn == 'swish':
+ elif activation_fn == "swish":
self.activation_fn = nn.SiLU
else:
raise NotImplementedError
self.activate = self.activation_fn()
self.conv_fn = functools.partial(
- nn.Conv3d,
+ CausalConv3d,
+ # padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorch
dtype=dtype,
- padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorch
- # custom_padding=self.custom_conv_padding
device=device,
)
-
- # self.norm_fn = model_utils.get_norm_layer(
- # norm_type=self.norm_type, dtype=self.dtype)
-
+
self.block_args = dict(
- # norm_fn=self.norm_fn,
conv_fn=self.conv_fn,
dtype=dtype,
activation_fn=self.activation_fn,
@@ -137,92 +638,98 @@ class Encoder(nn.Module):
num_groups=self.num_groups,
device=device,
)
-
- self.conv1 = self.conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)
+
+ # NOTE: moved to VAE for separate first frame processing
+ # self.conv1 = self.conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)
# ResBlocks and conv downsample
- self.block_res_blocks = []
+ self.block_res_blocks = nn.ModuleList([])
self.num_blocks = len(self.channel_multipliers)
- self.conv_blocks = []
+ self.conv_blocks = nn.ModuleList([])
filters = self.filters
- prev_filters = filters # record for in_channels
+ prev_filters = filters # record for in_channels
for i in range(self.num_blocks):
# resblock handling
- filters = self.filters * self.channel_multipliers[i] # SCH: determine the number out_channels
- block_items = []
+ filters = self.filters * self.channel_multipliers[i] # SCH: determine the number out_channels
+ block_items = nn.ModuleList([])
for _ in range(self.num_res_blocks):
block_items.append(ResBlock(prev_filters, filters, **self.block_args))
- prev_filters = filters # update in_channels
+ prev_filters = filters # update in_channels
self.block_res_blocks.append(block_items)
-
- if i < self.num_blocks - 1:
- # conv blocks handling
- if self.conv_downsample:
- t_stride = 2 if self.temporal_downsample[i] else 1
- t_pad = 1 if self.temporal_downsample[i] else 0
- self.conv_blocks.append(self.conv_fn(prev_filters, filters, kernel_size=(4, 4, 4), stride=(t_stride, 2, 2)), padding=(t_pad,1,1)) # SCH: should be same in_channel and out_channel
- prev_filters = filters # update in_channels
- # NOTE: downsample, dimensions T, H, W
- self.avg_pool_with_t = nn.AvgPool3d((2,2,2), count_include_pad=False)
- self.avg_pool = nn.AvgPool3d((1,2,2), count_include_pad=False)
+ if i < self.num_blocks - 1: # SCH: T-Causal Conv 3x3x3, 128->128, stride t x stride s x stride s
+ t_stride = 2 if self.temporal_downsample[i] else 1
+ s_stride = 2 if not self.disable_spatial_downsample else 1
+ self.conv_blocks.append(
+ self.conv_fn(prev_filters, filters, kernel_size=(3, 3, 3), strides=(t_stride, s_stride, s_stride))
+ ) # SCH: should be same in_channel and out_channel
+ prev_filters = filters # update in_channels
# last layer res block
- self.res_blocks = []
+ self.res_blocks = nn.ModuleList([])
for _ in range(self.num_res_blocks):
self.res_blocks.append(ResBlock(prev_filters, filters, **self.block_args))
- prev_filters = filters # update in_channels
+ prev_filters = filters # update in_channels
# MAGVIT uses Group Normalization
- self.norm1 = nn.GroupNorm(self.num_groups, prev_filters, dtype=dtype, device=device) # SCH: separate <prev_filters> channels into 32 groups
+ self.norm1 = nn.GroupNorm(
+ self.num_groups, prev_filters, dtype=dtype, device=device
+ ) # SCH: separate <prev_filters> channels into 32 groups
- self.conv2 = self.conv_fn(prev_filters, self.embedding_dim, kernel_size=(1, 1, 1))
+ self.conv2 = nn.Conv3d(
+ prev_filters, self.embedding_dim, kernel_size=(1, 1, 1), dtype=dtype, device=device, padding="same"
+ )
def forward(self, x):
- dtype, device = x.dtype, x.device
- x = self.conv1.to(device, dtype)(x)
+ # dtype, device = x.dtype, x.device
+
+ # NOTE: moved to VAE for separate first frame processing
+ # x = self.conv1(x)
+
+ # print("encoder:", x.size())
+
for i in range(self.num_blocks):
for j in range(self.num_res_blocks):
x = self.block_res_blocks[i][j](x)
+ # print("encoder:", x.size())
if i < self.num_blocks - 1:
- if self.conv_downsample:
- x = self.conv_blocks[i].to(device, dtype)(x)
- else:
- if self.temporal_downsample[i]:
- x = self.avg_pool_with_t(x)
- else:
- x = self.avg_pool(x)
+ x = self.conv_blocks[i](x)
+ # print("encoder:", x.size())
for i in range(self.num_res_blocks):
x = self.res_blocks[i](x)
+ # print("encoder:", x.size())
- x = self.norm1.to(device, dtype)(x)
+ x = self.norm1(x)
x = self.activate(x)
- x = self.conv2.to(device, dtype)(x)
+ x = self.conv2(x)
+ # print("encoder:", x.size())
return x
-
class Decoder(nn.Module):
"""Decoder Blocks."""
- def __init__(self,
- latent_embed_dim = 256,
- filters = 64,
- in_out_channels = 4,
- num_res_blocks = 2,
- channel_multipliers = (1, 2, 2, 4),
- temporal_downsample = (True, True, False),
- num_groups = 32, # for nn.GroupNorm
- upsample = "nearest+conv", # options: "deconv", "nearest+conv"
- custom_conv_padding = None,
- activation_fn = 'swish',
+
+ def __init__(
+ self,
+ latent_embed_dim=512,
+ filters=128,
+ # in_out_channels = 4,
+ num_res_blocks=4,
+ channel_multipliers=(1, 2, 2, 4),
+ temporal_downsample=(False, True, True),
+ num_groups=32, # for nn.GroupNorm
+ # upsample = "nearest+conv", # options: "deconv", "nearest+conv"
+ disable_spatial_upsample=False, # for vae pipeline
+ custom_conv_padding=None,
+ activation_fn="swish",
device="cpu",
dtype=torch.bfloat16,
):
super().__init__()
- self.output_dim = in_out_channels
+ # self.output_dim = in_out_channels
self.embedding_dim = latent_embed_dim
self.filters = filters
self.num_res_blocks = num_res_blocks
@@ -230,38 +737,28 @@ class Decoder(nn.Module):
self.temporal_downsample = temporal_downsample
self.num_groups = num_groups
- if isinstance(self.temporal_downsample, int):
- self.temporal_downsample = _get_selected_flags(
- len(self.channel_multipliers) - 1, self.temporal_downsample, False)
-
- self.upsample = upsample
+ # self.upsample = upsample
+ self.s_stride = 1 if disable_spatial_upsample else 2 # spatial stride
self.custom_conv_padding = custom_conv_padding
# self.norm_type = self.config.vqvae.norm_type
# self.num_remat_block = self.config.vqvae.get('num_dec_remat_blocks', 0)
- if activation_fn == 'relu':
+ if activation_fn == "relu":
self.activation_fn = nn.ReLU
- elif activation_fn == 'swish':
- self.activation_fn = nn.SiLU
+ elif activation_fn == "swish":
+ self.activation_fn = nn.SiLU
else:
- raise NotImplementedError
+ raise NotImplementedError
self.activate = self.activation_fn()
self.conv_fn = functools.partial(
- nn.Conv3d,
+ CausalConv3d,
dtype=dtype,
- padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorch
- # custom_padding=self.custom_conv_padding
+ # padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorch
device=device,
)
- self.conv_t_fn = functools.partial(nn.ConvTranspose3d, dtype=dtype)
-
- # self.norm_fn = model_utils.get_norm_layer(
- # norm_type=self.norm_type, dtype=dtype)
-
self.block_args = dict(
- # norm_fn=self.norm_fn,
conv_fn=self.conv_fn,
activation_fn=self.activation_fn,
use_conv_shortcut=False,
@@ -276,102 +773,104 @@ class Decoder(nn.Module):
self.conv1 = self.conv_fn(self.embedding_dim, filters, kernel_size=(3, 3, 3), bias=True)
# last layer res block
- self.res_blocks = []
+ self.res_blocks = nn.ModuleList([])
for _ in range(self.num_res_blocks):
self.res_blocks.append(ResBlock(filters, filters, **self.block_args))
+ # TODO: do I need to add adaptive GroupNorm in between each block?
- # NOTE: upsample, dimensions T, H, W
- self.upsampler_with_t = nn.Upsample(scale_factor=(2,2,2))
- self.upsampler = nn.Upsample(scale_factor=(1,2,2))
+ # # NOTE: upsample, dimensions T, H, W
+ # self.upsampler_with_t = nn.Upsample(scale_factor=(2,2,2))
+ # self.upsampler = nn.Upsample(scale_factor=(1,2,2))
# ResBlocks and conv upsample
- prev_filters = filters # SCH: in_channels
- self.block_res_blocks = []
+ prev_filters = filters # SCH: in_channels
+ self.block_res_blocks = nn.ModuleList([])
self.num_blocks = len(self.channel_multipliers)
- self.conv_blocks = []
+ self.conv_blocks = nn.ModuleList([])
# SCH: reverse to keep track of the in_channels, but append also in a reverse direction
- for i in reversed(range(self.num_blocks)):
+ for i in reversed(range(self.num_blocks)):
filters = self.filters * self.channel_multipliers[i]
# resblock handling
- block_items = []
+ block_items = nn.ModuleList([])
for _ in range(self.num_res_blocks):
block_items.append(ResBlock(prev_filters, filters, **self.block_args))
- prev_filters = filters # SCH: update in_channels
- self.block_res_blocks.insert(0, block_items) # SCH: append in front
-
- # conv blocks handling
+ prev_filters = filters # SCH: update in_channels
+ self.block_res_blocks.insert(0, block_items) # SCH: append in front
+
+ # conv blocks with upsampling
if i > 0:
t_stride = 2 if self.temporal_downsample[i - 1] else 1
- t_kernel = 4 if self.temporal_downsample[i - 1] else 3 # SCH: hack to keep dimension same
- if self.upsample == "deconv":
- assert self.custom_conv_padding is None, ('Custom padding not implemented for ConvTranspose')
- # SCH: append in front
- self.conv_blocks.insert(0,
- self.conv_t_fn(prev_filters, filters, kernel_size=(t_kernel, 4, 4), stride=(t_stride, 2, 2), padding=1))
- prev_filters = filters # SCH: update in_channels
- elif self.upsample == 'nearest+conv':
- # SCH: append in front
- self.conv_blocks.insert(0, self.conv_fn(prev_filters, filters, kernel_size=(3, 3, 3)))
- prev_filters = filters # SCH: update in_channels
- else:
- raise NotImplementedError(f'Unknown upsampler: {self.upsample}')
-
+ # SCH: T-Causal Conv 3x3x3, f -> (t_stride * 2 * 2) * f, depth to space t_stride x 2 x 2
+ self.conv_blocks.insert(
+ 0,
+ self.conv_fn(
+ prev_filters, prev_filters * t_stride * self.s_stride * self.s_stride, kernel_size=(3, 3, 3)
+ ),
+ )
+
self.norm1 = nn.GroupNorm(self.num_groups, prev_filters, device=device, dtype=dtype)
- self.conv2 = self.conv_fn(prev_filters, self.output_dim, kernel_size=(3, 3, 3))
+ # NOTE: moved to VAE for separate first frame processing
+ # self.conv2 = self.conv_fn(prev_filters, self.output_dim, kernel_size=(3, 3, 3))
def forward(
self,
x,
**kwargs,
):
- dtype, device = x.dtype, x.device
- x = self.conv1.to(device, dtype)(x)
+ # dtype, device = x.dtype, x.device
+
+ x = self.conv1(x)
+ # print("decoder:", x.size())
for i in range(self.num_res_blocks):
x = self.res_blocks[i](x)
- for i in reversed(range(self.num_blocks)): # reverse here to make decoder symmetric with encoder
+ # print("decoder:", x.size())
+ for i in reversed(range(self.num_blocks)): # reverse here to make decoder symmetric with encoder
for j in range(self.num_res_blocks):
x = self.block_res_blocks[i][j](x)
-
+ # print("decoder:", x.size())
if i > 0:
- if self.upsample == 'deconv':
- assert self.custom_conv_padding is None, ('Custom padding not implemented for ConvTranspose')
- x = self.conv_blocks[i-1].to(device, dtype)(x)
- elif self.upsample == 'nearest+conv':
- if self.temporal_downsample[i - 1]:
- x = self.upsampler_with_t(x)
- else:
- x = self.upsampler(x)
- x = self.conv_blocks[i-1].to(device, dtype)(x)
- else:
- raise NotImplementedError(f'Unknown upsampler: {self.upsample}')
- x = self.norm1.to(device, dtype)(x)
+ t_stride = 2 if self.temporal_downsample[i - 1] else 1
+ # SCH: T-Causal Conv 3x3x3, f -> (t_stride * 2 * 2) * f, depth to space t_stride x 2 x 2
+ x = self.conv_blocks[i - 1](x)
+ x = rearrange(
+ x,
+ "B (C ts hs ws) T H W -> B C (T ts) (H hs) (W ws)",
+ ts=t_stride,
+ hs=self.s_stride,
+ ws=self.s_stride,
+ )
+ # print("decoder:", x.size())
+
+ x = self.norm1(x)
x = self.activate(x)
- x = self.conv2.to(device, dtype)(x)
+ # NOTE: moved to VAE for separate first frame processing
+ # x = self.conv2(x)
return x
-
+
@MODELS.register_module()
-class VAE_3D(nn.Module):
- """The 3D VAE """
+class VAE_3D_V2(nn.Module): # , ModelMixin
+ """The 3D VAE"""
+
def __init__(
- self,
- latent_embed_dim = 256,
- filters = 64,
- num_res_blocks = 2,
- channel_multipliers = (1, 2, 2, 4),
- temporal_downsample = (True, True, False),
- num_groups = 32, # for nn.GroupNorm
- conv_downsample = False,
- upsample = "nearest+conv", # options: "deconv", "nearest+conv"
- custom_conv_padding = None,
- activation_fn = 'swish',
- in_out_channels = 4,
- kl_embed_dim = 64,
+ self,
+ latent_embed_dim=256,
+ filters=128,
+ num_res_blocks=2,
+ separate_first_frame_encoding=False,
+ channel_multipliers=(1, 2, 2, 4),
+ temporal_downsample=(True, True, False),
+ num_groups=32, # for nn.GroupNorm
+ disable_space=False,
+ custom_conv_padding=None,
+ activation_fn="swish",
+ in_out_channels=4,
+ kl_embed_dim=64,
+ encoder_double_z=True,
device="cpu",
dtype="bf16",
- # precision: Any = jax.lax.Precision.DEFAULT
):
super().__init__()
@@ -381,88 +880,469 @@ class VAE_3D(nn.Module):
elif dtype == "fp16":
dtype = torch.float16
else:
- raise NotImplementedError(f'dtype: {dtype}')
+ raise NotImplementedError(f"dtype: {dtype}")
- # Model Initialization
+ # ==== Model Params ====
+ # self.image_size = cast_tuple(image_size, 2)
+ self.time_downsample_factor = 2 ** sum(temporal_downsample)
+ self.time_padding = self.time_downsample_factor - 1
+ self.separate_first_frame_encoding = separate_first_frame_encoding
+ image_down = 2 ** len(temporal_downsample)
+ t_down = 2 ** len([x for x in temporal_downsample if x == True])
+ self.patch_size = (t_down, image_down, image_down)
+
+ # ==== Model Initialization ====
+
+ # encoder & decoder first and last conv layer
+ # SCH: NOTE: following MAGVIT, conv in bias=False in encoder first conv
+ self.conv_in = CausalConv3d(
+ in_out_channels, filters, kernel_size=(3, 3, 3), bias=False, dtype=dtype, device=device
+ )
+ self.conv_in_first_frame = nn.Identity()
+ self.conv_out_first_frame = nn.Identity()
+ if separate_first_frame_encoding:
+ self.conv_in_first_frame = SameConv2d(in_out_channels, filters, (3, 3))
+ self.conv_out_first_frame = SameConv2d(filters, in_out_channels, (3, 3))
+ self.conv_out = CausalConv3d(filters, in_out_channels, 3, dtype=dtype, device=device)
self.encoder = Encoder(
- filters=filters,
- num_res_blocks=num_res_blocks,
- channel_multipliers=channel_multipliers,
+ filters=filters,
+ num_res_blocks=num_res_blocks,
+ channel_multipliers=channel_multipliers,
temporal_downsample=temporal_downsample,
- num_groups = num_groups, # for nn.GroupNorm
- in_out_channels = in_out_channels,
- latent_embed_dim = latent_embed_dim,
- conv_downsample = conv_downsample,
- custom_conv_padding = custom_conv_padding,
- activation_fn = activation_fn,
+ num_groups=num_groups, # for nn.GroupNorm
+ # in_out_channels = in_out_channels,
+ latent_embed_dim=latent_embed_dim * 2 if encoder_double_z else latent_embed_dim,
+ # conv_downsample = conv_downsample,
+ disable_spatial_downsample=disable_space,
+ custom_conv_padding=custom_conv_padding,
+ activation_fn=activation_fn,
device=device,
dtype=dtype,
)
self.decoder = Decoder(
- latent_embed_dim = latent_embed_dim,
- filters = filters,
- in_out_channels = in_out_channels,
- num_res_blocks = num_res_blocks,
- channel_multipliers = channel_multipliers,
- temporal_downsample = temporal_downsample,
- num_groups = num_groups, # for nn.GroupNorm
- upsample = upsample, # options: "deconv", "nearest+conv"
- custom_conv_padding = custom_conv_padding,
- activation_fn = activation_fn,
+ latent_embed_dim=latent_embed_dim,
+ filters=filters,
+ # in_out_channels = in_out_channels,
+ num_res_blocks=num_res_blocks,
+ channel_multipliers=channel_multipliers,
+ temporal_downsample=temporal_downsample,
+ num_groups=num_groups, # for nn.GroupNorm
+ # upsample = upsample, # options: "deconv", "nearest+conv"
+ disable_spatial_upsample=disable_space,
+ custom_conv_padding=custom_conv_padding,
+ activation_fn=activation_fn,
device=device,
dtype=dtype,
)
- self.quant_conv = nn.Conv3d(latent_embed_dim, 2*kl_embed_dim, 1)
- self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1)
-
- image_down = 2 ** len(temporal_downsample)
- t_down = 2 ** len([x for x in temporal_downsample if x == True])
- self.patch_size = (t_down, image_down, image_down)
+ if encoder_double_z:
+ self.quant_conv = nn.Conv3d(2 * latent_embed_dim, 2 * kl_embed_dim, 1, device=device, dtype=dtype)
+ else:
+ self.quant_conv = nn.Conv3d(latent_embed_dim, 2 * kl_embed_dim, 1, device=device, dtype=dtype)
+ self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1, device=device, dtype=dtype)
def get_latent_size(self, input_size):
for i in range(len(input_size)):
assert input_size[i] % self.patch_size[i] == 0, "Input size must be divisible by patch size"
input_size = [input_size[i] // self.patch_size[i] for i in range(3)]
return input_size
-
+
def encode(
self,
- x,
+ video,
+ video_contains_first_frame=True,
):
- encoded_feature = self.encoder(x)
- moments = self.quant_conv(encoded_feature).to(x.dtype)
- posterior = model_utils.DiagonalGaussianDistribution(moments)
+ encode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame
+
+ # whether to pad video or not
+ if video_contains_first_frame:
+ video_len = video.shape[2]
+ video = pad_at_dim(video, (self.time_padding, 0), value=0.0, dim=2)
+ video_packed_shape = [torch.Size([self.time_padding]), torch.Size([]), torch.Size([video_len - 1])]
+
+ # print("pre-encoder:", video.size())
+
+ # NOTE: moved encoder conv1 here for separate first frame encoding
+ if encode_first_frame_separately:
+ pad, first_frame, video = unpack(video, video_packed_shape, "b c * h w")
+ first_frame = self.conv_in_first_frame(first_frame)
+ video = self.conv_in(video)
+
+ # print("pre-encoder:", video.size())
+
+ if encode_first_frame_separately:
+ video, _ = pack([first_frame, video], "b c * h w")
+ video = pad_at_dim(video, (self.time_padding, 0), dim=2)
+
+ encoded_feature = self.encoder(video)
+
+ # print("after encoder:", encoded_feature.size())
+
+ # NOTE: TODO: do we include this before gaussian distri? or go directly to Gaussian distribution
+ moments = self.quant_conv(encoded_feature).to(video.dtype)
+ posterior = DiagonalGaussianDistribution(moments)
+
+ # print("after encoder moments:", moments.size())
+
return posterior
-
+
def decode(
self,
z,
- ):
- dtype = z.dtype
- z = self.post_quant_conv(z).to(dtype)
+ video_contains_first_frame=True,
+ ):
+ # dtype = z.dtype
+ decode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame
+
+ z = self.post_quant_conv(z)
+ # print("pre decoder, post quant conv:", z.size())
+
dec = self.decoder(z)
- return dec
-
+ # print("post decoder:", dec.size())
+
+ # SCH: moved decoder last conv layer here for separate first frame decoding
+ if decode_first_frame_separately:
+ left_pad, dec_ff, dec = (
+ dec[:, :, : self.time_padding],
+ dec[:, :, self.time_padding],
+ dec[:, :, (self.time_padding + 1) :],
+ )
+ out = self.conv_out(dec)
+ outff = self.conv_out_first_frame(dec_ff)
+ video, _ = pack([outff, out], "b c * h w")
+ else:
+ video = self.conv_out(dec)
+ # if video were padded, remove padding
+ if video_contains_first_frame:
+ video = video[:, :, self.time_padding :]
+
+ # print("conv out:", video.size())
+
+ return video
+
+ def get_last_layer(self):
+ # CausalConv3d wraps the conv
+ return self.conv_out.conv.weight
+
def forward(
self,
- input,
+ video,
sample_posterior=True,
- ):
- posterior = self.encode(input)
+ video_contains_first_frame=True,
+ # split = "train",
+ ):
+ batch, channels, frames = video.shape[:3]
+ assert divisible_by(
+ frames - int(video_contains_first_frame), self.time_downsample_factor
+ ), f"number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}"
+
+ posterior = self.encode(
+ video,
+ video_contains_first_frame=video_contains_first_frame,
+ )
+
if sample_posterior:
z = posterior.sample()
else:
z = posterior.mode()
- dec = self.decode(z)
- return dec, posterior
-
[email protected]_module("VAE_3D_B")
-def VAE_3D_B(from_pretrained=None, **kwargs):
- model = VAE_3D(**kwargs)
+ recon_video = self.decode(z, video_contains_first_frame=video_contains_first_frame)
+
+ return recon_video, posterior
+
+
+class VEALoss(nn.Module):
+ def __init__(
+ self,
+ logvar_init=0.0,
+ perceptual_loss_weight=0.1,
+ kl_loss_weight=0.000001,
+ # vgg=None,
+ # vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,
+ device="cpu",
+ dtype="bf16",
+ ):
+ super().__init__()
+
+ if type(dtype) == str:
+ if dtype == "bf16":
+ dtype = torch.bfloat16
+ elif dtype == "fp16":
+ dtype = torch.float16
+ else:
+ raise NotImplementedError(f"dtype: {dtype}")
+
+ # KL Loss
+ self.kl_loss_weight = kl_loss_weight
+ # Perceptual Loss
+ self.perceptual_loss_fn = LPIPS().eval().to(device, dtype)
+ self.perceptual_loss_weight = perceptual_loss_weight
+ self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)
+
+
+ def forward(
+ self,
+ video,
+ recon_video,
+ posterior,
+ nll_weights=None,
+ split="train",
+ ):
+ video = rearrange(video, "b c t h w -> (b t) c h w").contiguous()
+ recon_video = rearrange(recon_video, "b c t h w -> (b t) c h w").contiguous()
+
+ # reconstruction loss
+ recon_loss = torch.abs(video - recon_video)
+
+ # perceptual loss
+ if self.perceptual_loss_weight is not None and self.perceptual_loss_weight > 0.0:
+ # handle channels
+ channels = video.shape[1]
+ assert channels in {1, 3, 4}
+ if channels == 1:
+ input_vgg_input = repeat(video, "b 1 h w -> b c h w", c=3)
+ recon_vgg_input = repeat(recon_video, "b 1 h w -> b c h w", c=3)
+ elif channels == 4: # SCH: take the first 3 for perceptual loss calc
+ input_vgg_input = video[:, :3]
+ recon_vgg_input = recon_video[:, :3]
+ else:
+ input_vgg_input = video
+ recon_vgg_input = recon_video
+
+ perceptual_loss = self.perceptual_loss_fn(input_vgg_input, recon_vgg_input)
+ recon_loss = recon_loss + self.perceptual_loss_weight * perceptual_loss
+
+ nll_loss = recon_loss / torch.exp(self.logvar) + self.logvar
+ weighted_nll_loss = nll_loss
+ if nll_weights is not None:
+ weighted_nll_loss = nll_weights * nll_loss
+ weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]
+ nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]
+
+
+ # KL Loss
+ weighted_kl_loss = 0
+ if self.kl_loss_weight is not None and self.kl_loss_weight > 0.0:
+ kl_loss = posterior.kl()
+ kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]
+ weighted_kl_loss = kl_loss * self.kl_loss_weight
+
+ return nll_loss, weighted_nll_loss, weighted_kl_loss
+
+
+class AdversarialLoss(nn.Module):
+ def __init__(
+ self,
+ discriminator_factor=1.0,
+ discriminator_start=50001,
+ generator_factor=0.5,
+ generator_loss_type="non-saturating",
+ ):
+ super().__init__()
+ self.discriminator_factor = discriminator_factor
+ self.discriminator_start = discriminator_start
+ self.generator_factor = generator_factor
+ self.generator_loss_type = generator_loss_type
+
+ def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer):
+ nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0] # SCH: TODO: debug added creat
+ g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[
+ 0
+ ] # SCH: TODO: debug added create_graph=True
+ d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)
+ d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()
+ d_weight = d_weight * self.generator_factor
+ return d_weight
+
+ def forward(
+ self,
+ fake_logits,
+ nll_loss,
+ last_layer,
+ global_step,
+ is_training=True,
+ ):
+ # NOTE: following MAGVIT to allow non_saturating
+ assert self.generator_loss_type in ["hinge", "vanilla", "non-saturating"]
+
+ if self.generator_loss_type == "hinge":
+ gen_loss = -torch.mean(fake_logits)
+ elif self.generator_loss_type == "non-saturating":
+ gen_loss = torch.mean(
+ sigmoid_cross_entropy_with_logits(labels=torch.ones_like(fake_logits), logits=fake_logits)
+ )
+ else:
+ raise ValueError("Generator loss {} not supported".format(self.generator_loss_type))
+
+ if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
+ try:
+ d_weight = self.calculate_adaptive_weight(nll_loss, gen_loss, last_layer)
+ except RuntimeError:
+ assert not is_training
+ d_weight = torch.tensor(0.0)
+ else:
+ d_weight = torch.tensor(0.0)
+
+ disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)
+ weighted_gen_loss = d_weight * disc_factor * gen_loss
+
+ return weighted_gen_loss
+
+
+class LeCamEMA:
+ def __init__(self, ema_real=0.0, ema_fake=0.0, decay=0.999, dtype=torch.bfloat16, device="cpu"):
+ self.decay = decay
+ self.ema_real = torch.tensor(ema_real).to(device, dtype)
+ self.ema_fake = torch.tensor(ema_fake).to(device, dtype)
+
+ def update(self, ema_real, ema_fake):
+ self.ema_real = self.ema_real * self.decay + ema_real * (1 - self.decay)
+ self.ema_fake = self.ema_fake * self.decay + ema_fake * (1 - self.decay)
+
+ def get(self):
+ return self.ema_real, self.ema_fake
+
+
+class DiscriminatorLoss(nn.Module):
+ def __init__(
+ self,
+ discriminator_factor=1.0,
+ discriminator_start=50001,
+ discriminator_loss_type="non-saturating",
+ lecam_loss_weight=None,
+ gradient_penalty_loss_weight=None, # SCH: following MAGVIT config.vqgan.grad_penalty_cost
+ ):
+ super().__init__()
+
+ assert discriminator_loss_type in ["hinge", "vanilla", "non-saturating"]
+ self.discriminator_factor = discriminator_factor
+ self.discriminator_start = discriminator_start
+ self.lecam_loss_weight = lecam_loss_weight
+ self.gradient_penalty_loss_weight = gradient_penalty_loss_weight
+ self.discriminator_loss_type = discriminator_loss_type
+
+ def forward(
+ self,
+ real_logits,
+ fake_logits,
+ global_step,
+ lecam_ema_real=None,
+ lecam_ema_fake=None,
+ real_video=None,
+ split="train",
+ ):
+ if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
+ disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)
+
+ if self.discriminator_loss_type == "hinge":
+ disc_loss = hinge_d_loss(real_logits, fake_logits)
+ elif self.discriminator_loss_type == "non-saturating":
+ if real_logits is not None:
+ real_loss = sigmoid_cross_entropy_with_logits(
+ labels=torch.ones_like(real_logits), logits=real_logits
+ )
+ else:
+ real_loss = 0.0
+ if fake_logits is not None:
+ fake_loss = sigmoid_cross_entropy_with_logits(
+ labels=torch.zeros_like(fake_logits), logits=fake_logits
+ )
+ else:
+ fake_loss = 0.0
+ disc_loss = 0.5 * (torch.mean(real_loss) + torch.mean(fake_loss))
+ elif self.discriminator_loss_type == "vanilla":
+ disc_loss = vanilla_d_loss(real_logits, fake_logits)
+ else:
+ raise ValueError(f"Unknown GAN loss '{self.discriminator_loss_type}'.")
+
+ weighted_d_adversarial_loss = disc_factor * disc_loss
+
+ else:
+ weighted_d_adversarial_loss = 0
+
+ lecam_loss = torch.tensor(0.0)
+ if self.lecam_loss_weight is not None and self.lecam_loss_weight > 0.0:
+ real_pred = torch.mean(real_logits)
+ fake_pred = torch.mean(fake_logits)
+ lecam_loss = lecam_reg(real_pred, fake_pred, lecam_ema_real, lecam_ema_fake)
+ lecam_loss = lecam_loss * self.lecam_loss_weight
+
+ gradient_penalty = torch.tensor(0.0)
+ if self.gradient_penalty_loss_weight is not None and self.gradient_penalty_loss_weight > 0.0:
+ assert real_video is not None
+ gradient_penalty = gradient_penalty_fn(real_video, real_logits)
+ # gradient_penalty = r1_penalty(real_video, real_logits) # MAGVIT uses r1 penalty
+ gradient_penalty *= self.gradient_penalty_loss_weight
+
+ # discriminator_loss = weighted_d_adversarial_loss + lecam_loss + gradient_penalty
+
+ # log = {
+ # "{}/d_adversarial_loss".format(split): weighted_d_adversarial_loss.detach().mean(),
+ # "{}/lecam_loss".format(split): lecam_loss.detach().mean(),
+ # "{}/gradient_penalty".format(split): gradient_penalty.detach().mean(),
+ # }
+
+ return (weighted_d_adversarial_loss, lecam_loss, gradient_penalty)
+
+
[email protected]_module("VAE_MAGVIT_V2")
+def VAE_MAGVIT_V2(from_pretrained=None, **kwargs):
+ model = VAE_3D_V2(**kwargs)
if from_pretrained is not None:
- load_checkpoint(model, from_pretrained)
- return model
\ No newline at end of file
+ load_checkpoint(model, from_pretrained, model_name="model")
+ return model
+
+
[email protected]_module("DISCRIMINATOR_3D")
+def DISCRIMINATOR_3D(from_pretrained=None, inflate_from_2d=False, use_pretrained=True, **kwargs):
+ model = StyleGANDiscriminatorBlur(**kwargs).apply(xavier_uniform_weight_init)
+ if from_pretrained is not None:
+ if use_pretrained:
+ if inflate_from_2d:
+ load_checkpoint_with_inflation(model, from_pretrained)
+ else:
+ load_checkpoint(model, from_pretrained, model_name="discriminator")
+ print(f"loaded discriminator")
+ else:
+ print(f"discriminator use_pretrained={use_pretrained}, initializing new discriminator")
+
+ return model
+
[email protected]_module("N_Layer_DISCRIMINATOR_3D")
+def DISCRIMINATOR_3D(from_pretrained=None, inflate_from_2d=False, use_pretrained=True, **kwargs):
+ model = NLayerDiscriminator3D(input_nc=3, n_layers=3,).apply(n_layer_disc_weights_init)
+ if from_pretrained is not None:
+ if use_pretrained:
+ if inflate_from_2d:
+ load_checkpoint_with_inflation(model, from_pretrained)
+ else:
+ load_checkpoint(model, from_pretrained, model_name="discriminator")
+ print(f"loaded discriminator")
+ else:
+ print(f"discriminator use_pretrained={use_pretrained}, initializing new discriminator")
+
+ return model
+
+
+def load_checkpoint_with_inflation(model, ckpt_path):
+ """
+ pre-train using image, then inflate to 3D videos
+ """
+ if ckpt_path.endswith(".pt") or ckpt_path.endswith(".pth"):
+ state_dict = find_model(ckpt_path)
+ with torch.no_grad():
+ for key in state_dict:
+ if key in model:
+ # central inflation
+ if state_dict[key].size() == model[key][:, :, 0, :, :].size():
+ # temporal dimension
+ val = torch.zeros_like(model[key])
+ centre = int(model[key].size(2) // 2)
+ val[:, :, centre, :, :] = state_dict[key]
+ missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)
+ print(f"Missing keys: {missing_keys}")
+ print(f"Unexpected keys: {unexpected_keys}")
+ else:
+ load_checkpoint(model, ckpt_path) # use the default function
diff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.py
deleted file mode 100644
index 96a5f9a..0000000
--- a/opensora/models/vae/vae_3d_v2.py
+++ /dev/null
@@ -1,1349 +0,0 @@
-import functools
-import math
-from typing import Tuple, Union
-
-import numpy as np
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from einops import pack, rearrange, repeat, unpack
-from torch import nn
-
-from opensora.models.vae import model_utils
-from opensora.models.vae.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers
-from opensora.registry import MODELS
-from opensora.utils.ckpt_utils import find_model, load_checkpoint
-
-# from diffusers.models.modeling_utils import ModelMixin
-
-
-"""Encoder and Decoder stuctures with 3D CNNs."""
-
-"""
-NOTE:
- removed LayerNorm since not used in this arch
- GroupNorm: flax uses default `epsilon=1e-06`, whereas torch uses `eps=1e-05`
- for average pool and upsample, input shape needs to be [N,C,T,H,W] --> if not, adjust the scale factors accordingly
-
- !!! opensora read video into [B,C,T,H,W] format output
-
-"""
-
-
-def cast_tuple(t, length=1):
- return t if isinstance(t, tuple) else ((t,) * length)
-
-
-def divisible_by(num, den):
- return (num % den) == 0
-
-
-def is_odd(n):
- return not divisible_by(n, 2)
-
-
-def pad_at_dim(t, pad, dim=-1, value=0.0):
- dims_from_right = (-dim - 1) if dim < 0 else (t.ndim - dim - 1)
- zeros = (0, 0) * dims_from_right
- return F.pad(t, (*zeros, *pad), value=value)
-
-
-def pick_video_frame(video, frame_indices):
- """get frame_indices from the video of [B, C, T, H, W] and return images of [B, C, H, W]"""
- batch, device = video.shape[0], video.device
- video = rearrange(video, "b c f ... -> b f c ...")
- batch_indices = torch.arange(batch, device=device)
- batch_indices = rearrange(batch_indices, "b -> b 1")
- images = video[batch_indices, frame_indices]
- images = rearrange(images, "b 1 c ... -> b c ...")
- return images
-
-
-def exists(v):
- return v is not None
-
-
-# ============== Generator Adversarial Loss Functions ==============
-def lecam_reg(real_pred, fake_pred, ema_real_pred, ema_fake_pred):
- assert real_pred.ndim == 0 and ema_fake_pred.ndim == 0
- lecam_loss = torch.mean(torch.pow(nn.ReLU()(real_pred - ema_fake_pred), 2))
- lecam_loss += torch.mean(torch.pow(nn.ReLU()(ema_real_pred - fake_pred), 2))
- return lecam_loss
-
-
-# Open-Sora-Plan
-# Very bad, do not use
-def r1_penalty(real_img, real_pred):
- """R1 regularization for discriminator. The core idea is to
- penalize the gradient on real data alone: when the
- generator distribution produces the true data distribution
- and the discriminator is equal to 0 on the data manifold, the
- gradient penalty ensures that the discriminator cannot create
- a non-zero gradient orthogonal to the data manifold without
- suffering a loss in the GAN game.
-
- Ref:
- Eq. 9 in Which training methods for GANs do actually converge.
- """
- grad_real = torch.autograd.grad(outputs=real_pred.sum(), inputs=real_img, create_graph=True)[0]
- grad_penalty = grad_real.pow(2).view(grad_real.shape[0], -1).sum(1).mean()
- return grad_penalty
-
-
-# Open-Sora-Plan
-# Implementation as described by https://arxiv.org/abs/1704.00028 # TODO: checkout the codes
-def gradient_penalty_loss(discriminator, real_data, fake_data, weight=None):
- """Calculate gradient penalty for wgan-gp.
-
- Args:
- discriminator (nn.Module): Network for the discriminator.
- real_data (Tensor): Real input data.
- fake_data (Tensor): Fake input data.
- weight (Tensor): Weight tensor. Default: None.
-
- Returns:
- Tensor: A tensor for gradient penalty.
- """
-
- batch_size = real_data.size(0)
- alpha = real_data.new_tensor(torch.rand(batch_size, 1, 1, 1))
-
- # interpolate between real_data and fake_data
- interpolates = alpha * real_data + (1.0 - alpha) * fake_data
- interpolates = torch.autograd.Variable(interpolates, requires_grad=True)
-
- disc_interpolates = discriminator(interpolates)
- gradients = torch.autograd.grad(
- outputs=disc_interpolates,
- inputs=interpolates,
- grad_outputs=torch.ones_like(disc_interpolates),
- create_graph=True,
- retain_graph=True,
- only_inputs=True,
- )[0]
-
- if weight is not None:
- gradients = gradients * weight
-
- gradients_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean()
- if weight is not None:
- gradients_penalty /= torch.mean(weight)
-
- return gradients_penalty
-
-
-def gradient_penalty_fn(images, output):
- # batch_size = images.shape[0]
- gradients = torch.autograd.grad(
- outputs=output,
- inputs=images,
- grad_outputs=torch.ones(output.size(), device=images.device),
- create_graph=True,
- retain_graph=True,
- only_inputs=True,
- )[0]
-
- gradients = rearrange(gradients, "b ... -> b (...)")
- return ((gradients.norm(2, dim=1) - 1) ** 2).mean()
-
-
-# ============== Discriminator Adversarial Loss Functions ==============
-def hinge_d_loss(logits_real, logits_fake):
- loss_real = torch.mean(F.relu(1.0 - logits_real))
- loss_fake = torch.mean(F.relu(1.0 + logits_fake))
- d_loss = 0.5 * (loss_real + loss_fake)
- return d_loss
-
-
-def vanilla_d_loss(logits_real, logits_fake):
- d_loss = 0.5 * (
- torch.mean(torch.nn.functional.softplus(-logits_real)) + torch.mean(torch.nn.functional.softplus(logits_fake))
- )
- return d_loss
-
-
-# from MAGVIT, used in place hof hinge_d_loss
-def sigmoid_cross_entropy_with_logits(labels, logits):
- # The final formulation is: max(x, 0) - x * z + log(1 + exp(-abs(x)))
- zeros = torch.zeros_like(logits, dtype=logits.dtype)
- condition = logits >= zeros
- relu_logits = torch.where(condition, logits, zeros)
- neg_abs_logits = torch.where(condition, -logits, logits)
- return relu_logits - logits * labels + torch.log1p(torch.exp(neg_abs_logits))
-
-
-def xavier_uniform_weight_init(m):
- if isinstance(m, nn.Conv3d) or isinstance(m, nn.Linear):
- nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain("relu"))
- if m.bias is not None:
- nn.init.zeros_(m.bias)
- # print("initialized module to xavier_uniform:", m)
-
-
-def Sequential(*modules):
- modules = [*filter(exists, modules)]
-
- if len(modules) == 0:
- return nn.Identity()
-
- return nn.Sequential(*modules)
-
-
-def SameConv2d(dim_in, dim_out, kernel_size):
- kernel_size = cast_tuple(kernel_size, 2)
- padding = [k // 2 for k in kernel_size]
- return nn.Conv2d(dim_in, dim_out, kernel_size=kernel_size, padding=padding)
-
-
-def adopt_weight(weight, global_step, threshold=0, value=0.0):
- if global_step < threshold:
- weight = value
- return weight
-
-
-class CausalConv3d(nn.Module):
- def __init__(
- self,
- chan_in,
- chan_out,
- kernel_size: Union[int, Tuple[int, int, int]],
- pad_mode="constant",
- strides=None, # allow custom stride
- **kwargs,
- ):
- super().__init__()
- kernel_size = cast_tuple(kernel_size, 3)
-
- time_kernel_size, height_kernel_size, width_kernel_size = kernel_size
-
- assert is_odd(height_kernel_size) and is_odd(width_kernel_size)
-
- dilation = kwargs.pop("dilation", 1)
- stride = strides[0] if strides is not None else kwargs.pop("stride", 1)
-
- self.pad_mode = pad_mode
- time_pad = dilation * (time_kernel_size - 1) + (1 - stride)
- height_pad = height_kernel_size // 2
- width_pad = width_kernel_size // 2
-
- self.time_pad = time_pad
- self.time_causal_padding = (width_pad, width_pad, height_pad, height_pad, time_pad, 0)
-
- stride = strides if strides is not None else (stride, 1, 1)
- # padding = kwargs.pop('padding', 0)
- # if padding == "same" and not all([pad == 1 for pad in padding]):
- # padding = "valid"
- dilation = (dilation, 1, 1)
- self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)
-
- def forward(self, x):
- pad_mode = self.pad_mode if self.time_pad < x.shape[2] else "constant"
-
- x = F.pad(x, self.time_causal_padding, mode=pad_mode)
- return self.conv(x)
-
-
-class ResBlock(nn.Module):
- def __init__(
- self,
- in_channels, # SCH: added
- filters,
- conv_fn,
- activation_fn=nn.SiLU,
- use_conv_shortcut=False,
- num_groups=32,
- device="cpu",
- dtype=torch.bfloat16,
- ):
- super().__init__()
- self.in_channels = in_channels
- self.filters = filters
- self.activate = activation_fn()
- self.use_conv_shortcut = use_conv_shortcut
-
- # SCH: MAGVIT uses GroupNorm by default
- self.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)
- self.conv1 = conv_fn(in_channels, self.filters, kernel_size=(3, 3, 3), bias=False)
- self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)
- self.conv2 = conv_fn(self.filters, self.filters, kernel_size=(3, 3, 3), bias=False)
- if in_channels != filters:
- if self.use_conv_shortcut:
- self.conv3 = conv_fn(in_channels, self.filters, kernel_size=(3, 3, 3), bias=False)
- else:
- self.conv3 = conv_fn(in_channels, self.filters, kernel_size=(1, 1, 1), bias=False)
-
- def forward(self, x):
- # device, dtype = x.device, x.dtype
- # input_dim = x.shape[1]
- residual = x
- x = self.norm1(x)
- x = self.activate(x)
- x = self.conv1(x)
- x = self.norm2(x)
- x = self.activate(x)
- x = self.conv2(x)
- if self.in_channels != self.filters: # SCH: ResBlock X->Y
- residual = self.conv3(residual)
- return x + residual
-
-
-# SCH: own implementation modified on top of: discriminator with anti-aliased downsampling (blurpool Zhang et al.)
-class BlurPool3D(nn.Module):
- def __init__(
- self,
- channels,
- pad_type="reflect",
- filt_size=3,
- stride=2,
- pad_off=0,
- device="cpu",
- dtype=torch.bfloat16,
- ):
- super(BlurPool3D, self).__init__()
- self.filt_size = filt_size
- self.pad_off = pad_off
- self.pad_sizes = [
- int(1.0 * (filt_size - 1) / 2),
- int(np.ceil(1.0 * (filt_size - 1) / 2)),
- int(1.0 * (filt_size - 1) / 2),
- int(np.ceil(1.0 * (filt_size - 1) / 2)),
- int(1.0 * (filt_size - 1) / 2),
- int(np.ceil(1.0 * (filt_size - 1) / 2)),
- ]
- self.pad_sizes = [pad_size + pad_off for pad_size in self.pad_sizes]
- self.stride = stride
- self.off = int((self.stride - 1) / 2.0)
- self.channels = channels
-
- if self.filt_size == 1:
- a = np.array(
- [
- 1.0,
- ]
- )
- elif self.filt_size == 2:
- a = np.array([1.0, 1.0])
- elif self.filt_size == 3:
- a = np.array([1.0, 2.0, 1.0])
- elif self.filt_size == 4:
- a = np.array([1.0, 3.0, 3.0, 1.0])
- elif self.filt_size == 5:
- a = np.array([1.0, 4.0, 6.0, 4.0, 1.0])
- elif self.filt_size == 6:
- a = np.array([1.0, 5.0, 10.0, 10.0, 5.0, 1.0])
- elif self.filt_size == 7:
- a = np.array([1.0, 6.0, 15.0, 20.0, 15.0, 6.0, 1.0])
-
- filt_2d = a[:, None] * a[None, :]
- filt_3d = torch.Tensor(a[:, None, None] * filt_2d[None, :, :]).to(device, dtype)
-
- filt = filt_3d / torch.sum(filt_3d) # SCH: modified to it 3D
- self.register_buffer("filt", filt[None, None, :, :, :].repeat((self.channels, 1, 1, 1, 1)))
-
- self.pad = get_pad_layer(pad_type)(self.pad_sizes)
-
- def forward(self, inp):
- if self.filt_size == 1:
- if self.pad_off == 0:
- return inp[:, :, :: self.stride, :: self.stride]
- else:
- return self.pad(inp)[:, :, :: self.stride, :: self.stride]
- else:
- return F.conv3d(self.pad(inp), self.filt, stride=self.stride, groups=inp.shape[1])
-
-
-def get_pad_layer(pad_type):
- if pad_type in ["refl", "reflect"]:
- PadLayer = nn.ReflectionPad3d
- elif pad_type in ["repl", "replicate"]:
- PadLayer = nn.ReplicationPad3d
- elif pad_type == "zero":
- PadLayer = nn.ZeroPad3d
- else:
- print("Pad type [%s] not recognized" % pad_type)
- return PadLayer
-
-
-class ResBlockDown(nn.Module):
- """3D StyleGAN ResBlock for D."""
-
- def __init__(
- self,
- in_channels,
- filters,
- activation_fn,
- num_groups=32,
- device="cpu",
- dtype=torch.bfloat16,
- ):
- super().__init__()
-
- self.filters = filters
- self.activation_fn = activation_fn
-
- # SCH: NOTE: although paper says conv (X->Y, Y->Y), original code implementation is (X->X, X->Y), we follow code
- self.conv1 = nn.Conv3d(
- in_channels, in_channels, (3, 3, 3), padding=1, device=device, dtype=dtype
- ) # NOTE: init to xavier_uniform
- self.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)
-
- self.blur = BlurPool3D(in_channels, device=device, dtype=dtype)
-
- self.conv2 = nn.Conv3d(
- in_channels, self.filters, (1, 1, 1), bias=False, device=device, dtype=dtype
- ) # NOTE: init to xavier_uniform
- self.conv3 = nn.Conv3d(
- in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype
- ) # NOTE: init to xavier_uniform
- self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)
-
- # self.apply(xavier_uniform_weight_init)
-
- def forward(self, x):
- residual = x
- x = self.conv1(x)
- x = self.norm1(x)
- x = self.activation_fn(x)
-
- residual = self.blur(residual)
- residual = self.conv2(residual)
-
- x = self.blur(x)
- x = self.conv3(x)
- x = self.norm2(x)
- x = self.activation_fn(x)
- out = (residual + x) / math.sqrt(2)
- return out
-
-
-# SCH: taken from Open Sora Plan
-def n_layer_disc_weights_init(m):
- classname = m.__class__.__name__
- if classname.find("Conv") != -1:
- nn.init.normal_(m.weight.data, 0.0, 0.02)
- elif classname.find("BatchNorm") != -1:
- nn.init.normal_(m.weight.data, 1.0, 0.02)
- nn.init.constant_(m.bias.data, 0)
-
-
-class NLayerDiscriminator3D(nn.Module):
- """Defines a 3D PatchGAN discriminator as in Pix2Pix but for 3D inputs."""
-
- def __init__(self, input_nc=1, ndf=64, n_layers=3, use_actnorm=False):
- """
- Construct a 3D PatchGAN discriminator
-
- Parameters:
- input_nc (int) -- the number of channels in input volumes
- ndf (int) -- the number of filters in the last conv layer
- n_layers (int) -- the number of conv layers in the discriminator
- use_actnorm (bool) -- flag to use actnorm instead of batchnorm
- """
- super(NLayerDiscriminator3D, self).__init__()
- if not use_actnorm:
- norm_layer = nn.BatchNorm3d
- else:
- raise NotImplementedError("Not implemented.")
- if type(norm_layer) == functools.partial:
- use_bias = norm_layer.func != nn.BatchNorm3d
- else:
- use_bias = norm_layer != nn.BatchNorm3d
-
- kw = 4
- padw = 1
- sequence = [nn.Conv3d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]
- nf_mult = 1
- nf_mult_prev = 1
- for n in range(1, n_layers): # gradually increase the number of filters
- nf_mult_prev = nf_mult
- nf_mult = min(2**n, 8)
- sequence += [
- nn.Conv3d(
- ndf * nf_mult_prev,
- ndf * nf_mult,
- kernel_size=(kw, kw, kw),
- stride=(1, 2, 2),
- padding=padw,
- bias=use_bias,
- ),
- norm_layer(ndf * nf_mult),
- nn.LeakyReLU(0.2, True),
- ]
-
- nf_mult_prev = nf_mult
- nf_mult = min(2**n_layers, 8)
- sequence += [
- nn.Conv3d(
- ndf * nf_mult_prev, ndf * nf_mult, kernel_size=(kw, kw, kw), stride=1, padding=padw, bias=use_bias
- ),
- norm_layer(ndf * nf_mult),
- nn.LeakyReLU(0.2, True),
- ]
-
- sequence += [
- nn.Conv3d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)
- ] # output 1 channel prediction map
- self.main = nn.Sequential(*sequence)
-
- def forward(self, input):
- """Standard forward."""
- return self.main(input)
-
-
-class StyleGANDiscriminatorBlur(nn.Module):
- """StyleGAN Discriminator."""
-
- """
- SCH: NOTE:
- this discriminator requries the num_frames to be fixed during training;
- in case we pre-train with image then train on video, this disciminator's Linear layer would have to be re-trained!
- """
-
- def __init__(
- self,
- image_size=(128, 128),
- num_frames=17,
- in_channels=3,
- filters=128,
- channel_multipliers=(2, 4, 4, 4, 4),
- num_groups=32,
- dtype=torch.bfloat16,
- device="cpu",
- ):
- super().__init__()
-
- self.dtype = dtype
- self.input_size = cast_tuple(image_size, 2)
- self.filters = filters
- self.activation_fn = nn.LeakyReLU(negative_slope=0.2)
- self.channel_multipliers = channel_multipliers
-
- self.conv1 = nn.Conv3d(
- in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype
- ) # NOTE: init to xavier_uniform
-
- prev_filters = self.filters # record in_channels
- self.num_blocks = len(self.channel_multipliers)
- self.res_block_list = nn.ModuleList([])
- for i in range(self.num_blocks):
- filters = self.filters * self.channel_multipliers[i]
- self.res_block_list.append(
- ResBlockDown(prev_filters, filters, self.activation_fn, device=device, dtype=dtype).apply(
- xavier_uniform_weight_init
- )
- )
- prev_filters = filters # update in_channels
-
- self.conv2 = nn.Conv3d(
- prev_filters, prev_filters, (3, 3, 3), padding=1, device=device, dtype=dtype
- ) # NOTE: init to xavier_uniform
- # torch.nn.init.xavier_uniform_(self.conv2.weight)
-
- self.norm1 = nn.GroupNorm(num_groups, prev_filters, dtype=dtype, device=device)
-
- scale_factor = 2**self.num_blocks
- if num_frames % scale_factor != 0: # SCH: NOTE: has first frame which would be padded before usage
- time_scaled = num_frames // scale_factor + 1
- else:
- time_scaled = num_frames / scale_factor
-
- assert (
- self.input_size[0] % scale_factor == 0
- ), f"image width {self.input_size[0]} is not divisible by scale factor {scale_factor}"
- assert (
- self.input_size[1] % scale_factor == 0
- ), f"image height {self.input_size[1]} is not divisible by scale factor {scale_factor}"
- w_scaled, h_scaled = self.input_size[0] / scale_factor, self.input_size[1] / scale_factor
- in_features = int(prev_filters * time_scaled * w_scaled * h_scaled) # (C*T*W*H)
- self.linear1 = nn.Linear(in_features, prev_filters, device=device, dtype=dtype) # NOTE: init to xavier_uniform
- self.linear2 = nn.Linear(prev_filters, 1, device=device, dtype=dtype) # NOTE: init to xavier_uniform
-
- # self.apply(xavier_uniform_weight_init)
-
- def forward(self, x):
- x = self.conv1(x)
- # print("discriminator aft conv:", x.size())
- x = self.activation_fn(x)
-
- for i in range(self.num_blocks):
- x = self.res_block_list[i](x)
- # print("discriminator resblock down:", x.size())
-
- x = self.conv2(x)
- # print("discriminator aft conv2:", x.size())
- x = self.norm1(x)
- x = self.activation_fn(x)
- x = x.reshape((x.shape[0], -1)) # SCH: [B, (C * T * W * H)] ?
-
- # print("discriminator reshape:", x.size())
- x = self.linear1(x)
- # print("discriminator aft linear1:", x.size())
-
- x = self.activation_fn(x)
- x = self.linear2(x)
- # print("discriminator aft linear2:", x.size())
- return x
-
-
-class Encoder(nn.Module):
- """Encoder Blocks."""
-
- def __init__(
- self,
- filters=128,
- num_res_blocks=4,
- channel_multipliers=(1, 2, 2, 4),
- temporal_downsample=(False, True, True),
- num_groups=32, # for nn.GroupNorm
- # in_out_channels = 3, # SCH: added, in_channels at the start
- latent_embed_dim=512, # num channels for latent vector
- # conv_downsample = False,
- disable_spatial_downsample=False, # for vae pipeline
- custom_conv_padding=None,
- activation_fn="swish",
- device="cpu",
- dtype=torch.bfloat16,
- ):
- super().__init__()
- self.filters = filters
- self.num_res_blocks = num_res_blocks
- self.channel_multipliers = channel_multipliers
- self.temporal_downsample = temporal_downsample
- self.num_groups = num_groups
-
- self.embedding_dim = latent_embed_dim
- self.disable_spatial_downsample = disable_spatial_downsample
- # self.conv_downsample = conv_downsample
- self.custom_conv_padding = custom_conv_padding
-
- if activation_fn == "relu":
- self.activation_fn = nn.ReLU
- elif activation_fn == "swish":
- self.activation_fn = nn.SiLU
- else:
- raise NotImplementedError
- self.activate = self.activation_fn()
-
- self.conv_fn = functools.partial(
- CausalConv3d,
- # padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorch
- dtype=dtype,
- device=device,
- )
-
- self.block_args = dict(
- conv_fn=self.conv_fn,
- dtype=dtype,
- activation_fn=self.activation_fn,
- use_conv_shortcut=False,
- num_groups=self.num_groups,
- device=device,
- )
-
- # NOTE: moved to VAE for separate first frame processing
- # self.conv1 = self.conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)
-
- # ResBlocks and conv downsample
- self.block_res_blocks = nn.ModuleList([])
- self.num_blocks = len(self.channel_multipliers)
- self.conv_blocks = nn.ModuleList([])
-
- filters = self.filters
- prev_filters = filters # record for in_channels
- for i in range(self.num_blocks):
- # resblock handling
- filters = self.filters * self.channel_multipliers[i] # SCH: determine the number out_channels
- block_items = nn.ModuleList([])
- for _ in range(self.num_res_blocks):
- block_items.append(ResBlock(prev_filters, filters, **self.block_args))
- prev_filters = filters # update in_channels
- self.block_res_blocks.append(block_items)
-
- if i < self.num_blocks - 1: # SCH: T-Causal Conv 3x3x3, 128->128, stride t x stride s x stride s
- t_stride = 2 if self.temporal_downsample[i] else 1
- s_stride = 2 if not self.disable_spatial_downsample else 1
- self.conv_blocks.append(
- self.conv_fn(prev_filters, filters, kernel_size=(3, 3, 3), strides=(t_stride, s_stride, s_stride))
- ) # SCH: should be same in_channel and out_channel
- prev_filters = filters # update in_channels
-
- # last layer res block
- self.res_blocks = nn.ModuleList([])
- for _ in range(self.num_res_blocks):
- self.res_blocks.append(ResBlock(prev_filters, filters, **self.block_args))
- prev_filters = filters # update in_channels
-
- # MAGVIT uses Group Normalization
- self.norm1 = nn.GroupNorm(
- self.num_groups, prev_filters, dtype=dtype, device=device
- ) # SCH: separate <prev_filters> channels into 32 groups
-
- self.conv2 = nn.Conv3d(
- prev_filters, self.embedding_dim, kernel_size=(1, 1, 1), dtype=dtype, device=device, padding="same"
- )
-
- def forward(self, x):
- # dtype, device = x.dtype, x.device
-
- # NOTE: moved to VAE for separate first frame processing
- # x = self.conv1(x)
-
- # print("encoder:", x.size())
-
- for i in range(self.num_blocks):
- for j in range(self.num_res_blocks):
- x = self.block_res_blocks[i][j](x)
- # print("encoder:", x.size())
-
- if i < self.num_blocks - 1:
- x = self.conv_blocks[i](x)
- # print("encoder:", x.size())
-
- for i in range(self.num_res_blocks):
- x = self.res_blocks[i](x)
- # print("encoder:", x.size())
-
- x = self.norm1(x)
- x = self.activate(x)
- x = self.conv2(x)
- # print("encoder:", x.size())
- return x
-
-
-class Decoder(nn.Module):
- """Decoder Blocks."""
-
- def __init__(
- self,
- latent_embed_dim=512,
- filters=128,
- # in_out_channels = 4,
- num_res_blocks=4,
- channel_multipliers=(1, 2, 2, 4),
- temporal_downsample=(False, True, True),
- num_groups=32, # for nn.GroupNorm
- # upsample = "nearest+conv", # options: "deconv", "nearest+conv"
- disable_spatial_upsample=False, # for vae pipeline
- custom_conv_padding=None,
- activation_fn="swish",
- device="cpu",
- dtype=torch.bfloat16,
- ):
- super().__init__()
- # self.output_dim = in_out_channels
- self.embedding_dim = latent_embed_dim
- self.filters = filters
- self.num_res_blocks = num_res_blocks
- self.channel_multipliers = channel_multipliers
- self.temporal_downsample = temporal_downsample
- self.num_groups = num_groups
-
- # self.upsample = upsample
- self.s_stride = 1 if disable_spatial_upsample else 2 # spatial stride
- self.custom_conv_padding = custom_conv_padding
- # self.norm_type = self.config.vqvae.norm_type
- # self.num_remat_block = self.config.vqvae.get('num_dec_remat_blocks', 0)
-
- if activation_fn == "relu":
- self.activation_fn = nn.ReLU
- elif activation_fn == "swish":
- self.activation_fn = nn.SiLU
- else:
- raise NotImplementedError
- self.activate = self.activation_fn()
-
- self.conv_fn = functools.partial(
- CausalConv3d,
- dtype=dtype,
- # padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorch
- device=device,
- )
-
- self.block_args = dict(
- conv_fn=self.conv_fn,
- activation_fn=self.activation_fn,
- use_conv_shortcut=False,
- num_groups=self.num_groups,
- device=device,
- dtype=dtype,
- )
- self.num_blocks = len(self.channel_multipliers)
-
- filters = self.filters * self.channel_multipliers[-1]
-
- self.conv1 = self.conv_fn(self.embedding_dim, filters, kernel_size=(3, 3, 3), bias=True)
-
- # last layer res block
- self.res_blocks = nn.ModuleList([])
- for _ in range(self.num_res_blocks):
- self.res_blocks.append(ResBlock(filters, filters, **self.block_args))
-
- # TODO: do I need to add adaptive GroupNorm in between each block?
-
- # # NOTE: upsample, dimensions T, H, W
- # self.upsampler_with_t = nn.Upsample(scale_factor=(2,2,2))
- # self.upsampler = nn.Upsample(scale_factor=(1,2,2))
-
- # ResBlocks and conv upsample
- prev_filters = filters # SCH: in_channels
- self.block_res_blocks = nn.ModuleList([])
- self.num_blocks = len(self.channel_multipliers)
- self.conv_blocks = nn.ModuleList([])
- # SCH: reverse to keep track of the in_channels, but append also in a reverse direction
- for i in reversed(range(self.num_blocks)):
- filters = self.filters * self.channel_multipliers[i]
- # resblock handling
- block_items = nn.ModuleList([])
- for _ in range(self.num_res_blocks):
- block_items.append(ResBlock(prev_filters, filters, **self.block_args))
- prev_filters = filters # SCH: update in_channels
- self.block_res_blocks.insert(0, block_items) # SCH: append in front
-
- # conv blocks with upsampling
- if i > 0:
- t_stride = 2 if self.temporal_downsample[i - 1] else 1
- # SCH: T-Causal Conv 3x3x3, f -> (t_stride * 2 * 2) * f, depth to space t_stride x 2 x 2
- self.conv_blocks.insert(
- 0,
- self.conv_fn(
- prev_filters, prev_filters * t_stride * self.s_stride * self.s_stride, kernel_size=(3, 3, 3)
- ),
- )
-
- self.norm1 = nn.GroupNorm(self.num_groups, prev_filters, device=device, dtype=dtype)
-
- # NOTE: moved to VAE for separate first frame processing
- # self.conv2 = self.conv_fn(prev_filters, self.output_dim, kernel_size=(3, 3, 3))
-
- def forward(
- self,
- x,
- **kwargs,
- ):
- # dtype, device = x.dtype, x.device
-
- x = self.conv1(x)
- # print("decoder:", x.size())
- for i in range(self.num_res_blocks):
- x = self.res_blocks[i](x)
- # print("decoder:", x.size())
- for i in reversed(range(self.num_blocks)): # reverse here to make decoder symmetric with encoder
- for j in range(self.num_res_blocks):
- x = self.block_res_blocks[i][j](x)
- # print("decoder:", x.size())
- if i > 0:
- t_stride = 2 if self.temporal_downsample[i - 1] else 1
- # SCH: T-Causal Conv 3x3x3, f -> (t_stride * 2 * 2) * f, depth to space t_stride x 2 x 2
- x = self.conv_blocks[i - 1](x)
- x = rearrange(
- x,
- "B (C ts hs ws) T H W -> B C (T ts) (H hs) (W ws)",
- ts=t_stride,
- hs=self.s_stride,
- ws=self.s_stride,
- )
- # print("decoder:", x.size())
-
- x = self.norm1(x)
- x = self.activate(x)
- # NOTE: moved to VAE for separate first frame processing
- # x = self.conv2(x)
- return x
-
-
-class VAE_3D_V2(nn.Module): # , ModelMixin
- """The 3D VAE"""
-
- def __init__(
- self,
- latent_embed_dim=256,
- filters=128,
- num_res_blocks=2,
- separate_first_frame_encoding=False,
- channel_multipliers=(1, 2, 2, 4),
- temporal_downsample=(True, True, False),
- num_groups=32, # for nn.GroupNorm
- disable_space=False,
- custom_conv_padding=None,
- activation_fn="swish",
- in_out_channels=4,
- kl_embed_dim=64,
- encoder_double_z=True,
- device="cpu",
- dtype="bf16",
- ):
- super().__init__()
-
- if type(dtype) == str:
- if dtype == "bf16":
- dtype = torch.bfloat16
- elif dtype == "fp16":
- dtype = torch.float16
- else:
- raise NotImplementedError(f"dtype: {dtype}")
-
- # ==== Model Params ====
- # self.image_size = cast_tuple(image_size, 2)
- self.time_downsample_factor = 2 ** sum(temporal_downsample)
- self.time_padding = self.time_downsample_factor - 1
- self.separate_first_frame_encoding = separate_first_frame_encoding
-
- image_down = 2 ** len(temporal_downsample)
- t_down = 2 ** len([x for x in temporal_downsample if x == True])
- self.patch_size = (t_down, image_down, image_down)
-
- # ==== Model Initialization ====
-
- # encoder & decoder first and last conv layer
- # SCH: NOTE: following MAGVIT, conv in bias=False in encoder first conv
- self.conv_in = CausalConv3d(
- in_out_channels, filters, kernel_size=(3, 3, 3), bias=False, dtype=dtype, device=device
- )
- self.conv_in_first_frame = nn.Identity()
- self.conv_out_first_frame = nn.Identity()
- if separate_first_frame_encoding:
- self.conv_in_first_frame = SameConv2d(in_out_channels, filters, (3, 3))
- self.conv_out_first_frame = SameConv2d(filters, in_out_channels, (3, 3))
- self.conv_out = CausalConv3d(filters, in_out_channels, 3, dtype=dtype, device=device)
-
- self.encoder = Encoder(
- filters=filters,
- num_res_blocks=num_res_blocks,
- channel_multipliers=channel_multipliers,
- temporal_downsample=temporal_downsample,
- num_groups=num_groups, # for nn.GroupNorm
- # in_out_channels = in_out_channels,
- latent_embed_dim=latent_embed_dim * 2 if encoder_double_z else latent_embed_dim,
- # conv_downsample = conv_downsample,
- disable_spatial_downsample=disable_space,
- custom_conv_padding=custom_conv_padding,
- activation_fn=activation_fn,
- device=device,
- dtype=dtype,
- )
- self.decoder = Decoder(
- latent_embed_dim=latent_embed_dim,
- filters=filters,
- # in_out_channels = in_out_channels,
- num_res_blocks=num_res_blocks,
- channel_multipliers=channel_multipliers,
- temporal_downsample=temporal_downsample,
- num_groups=num_groups, # for nn.GroupNorm
- # upsample = upsample, # options: "deconv", "nearest+conv"
- disable_spatial_upsample=disable_space,
- custom_conv_padding=custom_conv_padding,
- activation_fn=activation_fn,
- device=device,
- dtype=dtype,
- )
-
- if encoder_double_z:
- self.quant_conv = nn.Conv3d(2 * latent_embed_dim, 2 * kl_embed_dim, 1, device=device, dtype=dtype)
- else:
- self.quant_conv = nn.Conv3d(latent_embed_dim, 2 * kl_embed_dim, 1, device=device, dtype=dtype)
- self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1, device=device, dtype=dtype)
-
- def get_latent_size(self, input_size):
- for i in range(len(input_size)):
- assert input_size[i] % self.patch_size[i] == 0, "Input size must be divisible by patch size"
- input_size = [input_size[i] // self.patch_size[i] for i in range(3)]
- return input_size
-
- def encode(
- self,
- video,
- video_contains_first_frame=True,
- ):
- encode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame
-
- # whether to pad video or not
- if video_contains_first_frame:
- video_len = video.shape[2]
- video = pad_at_dim(video, (self.time_padding, 0), value=0.0, dim=2)
- video_packed_shape = [torch.Size([self.time_padding]), torch.Size([]), torch.Size([video_len - 1])]
-
- # print("pre-encoder:", video.size())
-
- # NOTE: moved encoder conv1 here for separate first frame encoding
- if encode_first_frame_separately:
- pad, first_frame, video = unpack(video, video_packed_shape, "b c * h w")
- first_frame = self.conv_in_first_frame(first_frame)
- video = self.conv_in(video)
-
- # print("pre-encoder:", video.size())
-
- if encode_first_frame_separately:
- video, _ = pack([first_frame, video], "b c * h w")
- video = pad_at_dim(video, (self.time_padding, 0), dim=2)
-
- encoded_feature = self.encoder(video)
-
- # print("after encoder:", encoded_feature.size())
-
- # NOTE: TODO: do we include this before gaussian distri? or go directly to Gaussian distribution
- moments = self.quant_conv(encoded_feature).to(video.dtype)
- posterior = model_utils.DiagonalGaussianDistribution(moments)
-
- # print("after encoder moments:", moments.size())
-
- return posterior
-
- def decode(
- self,
- z,
- video_contains_first_frame=True,
- ):
- # dtype = z.dtype
- decode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame
-
- z = self.post_quant_conv(z)
- # print("pre decoder, post quant conv:", z.size())
-
- dec = self.decoder(z)
- # print("post decoder:", dec.size())
-
- # SCH: moved decoder last conv layer here for separate first frame decoding
- if decode_first_frame_separately:
- left_pad, dec_ff, dec = (
- dec[:, :, : self.time_padding],
- dec[:, :, self.time_padding],
- dec[:, :, (self.time_padding + 1) :],
- )
- out = self.conv_out(dec)
- outff = self.conv_out_first_frame(dec_ff)
- video, _ = pack([outff, out], "b c * h w")
- else:
- video = self.conv_out(dec)
- # if video were padded, remove padding
- if video_contains_first_frame:
- video = video[:, :, self.time_padding :]
-
- # print("conv out:", video.size())
-
- return video
-
- def get_last_layer(self):
- # CausalConv3d wraps the conv
- return self.conv_out.conv.weight
-
- def forward(
- self,
- video,
- sample_posterior=True,
- video_contains_first_frame=True,
- # split = "train",
- ):
- batch, channels, frames = video.shape[:3]
- assert divisible_by(
- frames - int(video_contains_first_frame), self.time_downsample_factor
- ), f"number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}"
-
- posterior = self.encode(
- video,
- video_contains_first_frame=video_contains_first_frame,
- )
-
- if sample_posterior:
- z = posterior.sample()
- else:
- z = posterior.mode()
-
- recon_video = self.decode(z, video_contains_first_frame=video_contains_first_frame)
-
- return recon_video, posterior
-
-
-class VEALoss(nn.Module):
- def __init__(
- self,
- logvar_init=0.0,
- perceptual_loss_weight=0.1,
- kl_loss_weight=0.000001,
- # vgg=None,
- # vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,
- device="cpu",
- dtype="bf16",
- ):
- super().__init__()
-
- if type(dtype) == str:
- if dtype == "bf16":
- dtype = torch.bfloat16
- elif dtype == "fp16":
- dtype = torch.float16
- else:
- raise NotImplementedError(f"dtype: {dtype}")
-
- # KL Loss
- self.kl_loss_weight = kl_loss_weight
- # Perceptual Loss
- self.perceptual_loss_fn = LPIPS().eval().to(device, dtype)
- self.perceptual_loss_weight = perceptual_loss_weight
- self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)
-
-
- def forward(
- self,
- video,
- recon_video,
- posterior,
- nll_weights=None,
- split="train",
- ):
- video = rearrange(video, "b c t h w -> (b t) c h w").contiguous()
- recon_video = rearrange(recon_video, "b c t h w -> (b t) c h w").contiguous()
-
- # reconstruction loss
- recon_loss = torch.abs(video - recon_video)
-
- # perceptual loss
- if self.perceptual_loss_weight is not None and self.perceptual_loss_weight > 0.0:
- # handle channels
- channels = video.shape[1]
- assert channels in {1, 3, 4}
- if channels == 1:
- input_vgg_input = repeat(video, "b 1 h w -> b c h w", c=3)
- recon_vgg_input = repeat(recon_video, "b 1 h w -> b c h w", c=3)
- elif channels == 4: # SCH: take the first 3 for perceptual loss calc
- input_vgg_input = video[:, :3]
- recon_vgg_input = recon_video[:, :3]
- else:
- input_vgg_input = video
- recon_vgg_input = recon_video
-
- perceptual_loss = self.perceptual_loss_fn(input_vgg_input, recon_vgg_input)
- recon_loss = recon_loss + self.perceptual_loss_weight * perceptual_loss
-
- nll_loss = recon_loss / torch.exp(self.logvar) + self.logvar
- weighted_nll_loss = nll_loss
- if nll_weights is not None:
- weighted_nll_loss = nll_weights * nll_loss
- weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]
- nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]
-
-
- # KL Loss
- weighted_kl_loss = 0
- if self.kl_loss_weight is not None and self.kl_loss_weight > 0.0:
- kl_loss = posterior.kl()
- kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]
- weighted_kl_loss = kl_loss * self.kl_loss_weight
-
- return nll_loss, weighted_nll_loss, weighted_kl_loss
-
-
-class AdversarialLoss(nn.Module):
- def __init__(
- self,
- discriminator_factor=1.0,
- discriminator_start=50001,
- generator_factor=0.5,
- generator_loss_type="non-saturating",
- ):
- super().__init__()
- self.discriminator_factor = discriminator_factor
- self.discriminator_start = discriminator_start
- self.generator_factor = generator_factor
- self.generator_loss_type = generator_loss_type
-
- def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer):
- nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0] # SCH: TODO: debug added creat
- g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[
- 0
- ] # SCH: TODO: debug added create_graph=True
- d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)
- d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()
- d_weight = d_weight * self.generator_factor
- return d_weight
-
- def forward(
- self,
- fake_logits,
- nll_loss,
- last_layer,
- global_step,
- is_training=True,
- ):
- # NOTE: following MAGVIT to allow non_saturating
- assert self.generator_loss_type in ["hinge", "vanilla", "non-saturating"]
-
- if self.generator_loss_type == "hinge":
- gen_loss = -torch.mean(fake_logits)
- elif self.generator_loss_type == "non-saturating":
- gen_loss = torch.mean(
- sigmoid_cross_entropy_with_logits(labels=torch.ones_like(fake_logits), logits=fake_logits)
- )
- else:
- raise ValueError("Generator loss {} not supported".format(self.generator_loss_type))
-
- if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
- try:
- d_weight = self.calculate_adaptive_weight(nll_loss, gen_loss, last_layer)
- except RuntimeError:
- assert not is_training
- d_weight = torch.tensor(0.0)
- else:
- d_weight = torch.tensor(0.0)
-
- disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)
- weighted_gen_loss = d_weight * disc_factor * gen_loss
-
- return weighted_gen_loss
-
-
-class LeCamEMA:
- def __init__(self, ema_real=0.0, ema_fake=0.0, decay=0.999, dtype=torch.bfloat16, device="cpu"):
- self.decay = decay
- self.ema_real = torch.tensor(ema_real).to(device, dtype)
- self.ema_fake = torch.tensor(ema_fake).to(device, dtype)
-
- def update(self, ema_real, ema_fake):
- self.ema_real = self.ema_real * self.decay + ema_real * (1 - self.decay)
- self.ema_fake = self.ema_fake * self.decay + ema_fake * (1 - self.decay)
-
- def get(self):
- return self.ema_real, self.ema_fake
-
-
-class DiscriminatorLoss(nn.Module):
- def __init__(
- self,
- discriminator_factor=1.0,
- discriminator_start=50001,
- discriminator_loss_type="non-saturating",
- lecam_loss_weight=None,
- gradient_penalty_loss_weight=None, # SCH: following MAGVIT config.vqgan.grad_penalty_cost
- ):
- super().__init__()
-
- assert discriminator_loss_type in ["hinge", "vanilla", "non-saturating"]
- self.discriminator_factor = discriminator_factor
- self.discriminator_start = discriminator_start
- self.lecam_loss_weight = lecam_loss_weight
- self.gradient_penalty_loss_weight = gradient_penalty_loss_weight
- self.discriminator_loss_type = discriminator_loss_type
-
- def forward(
- self,
- real_logits,
- fake_logits,
- global_step,
- lecam_ema_real=None,
- lecam_ema_fake=None,
- real_video=None,
- split="train",
- ):
- if self.discriminator_factor is not None and self.discriminator_factor > 0.0:
- disc_factor = adopt_weight(self.discriminator_factor, global_step, threshold=self.discriminator_start)
-
- if self.discriminator_loss_type == "hinge":
- disc_loss = hinge_d_loss(real_logits, fake_logits)
- elif self.discriminator_loss_type == "non-saturating":
- if real_logits is not None:
- real_loss = sigmoid_cross_entropy_with_logits(
- labels=torch.ones_like(real_logits), logits=real_logits
- )
- else:
- real_loss = 0.0
- if fake_logits is not None:
- fake_loss = sigmoid_cross_entropy_with_logits(
- labels=torch.zeros_like(fake_logits), logits=fake_logits
- )
- else:
- fake_loss = 0.0
- disc_loss = 0.5 * (torch.mean(real_loss) + torch.mean(fake_loss))
- elif self.discriminator_loss_type == "vanilla":
- disc_loss = vanilla_d_loss(real_logits, fake_logits)
- else:
- raise ValueError(f"Unknown GAN loss '{self.discriminator_loss_type}'.")
-
- weighted_d_adversarial_loss = disc_factor * disc_loss
-
- else:
- weighted_d_adversarial_loss = 0
-
- lecam_loss = torch.tensor(0.0)
- if self.lecam_loss_weight is not None and self.lecam_loss_weight > 0.0:
- real_pred = torch.mean(real_logits)
- fake_pred = torch.mean(fake_logits)
- lecam_loss = lecam_reg(real_pred, fake_pred, lecam_ema_real, lecam_ema_fake)
- lecam_loss = lecam_loss * self.lecam_loss_weight
-
- gradient_penalty = torch.tensor(0.0)
- if self.gradient_penalty_loss_weight is not None and self.gradient_penalty_loss_weight > 0.0:
- assert real_video is not None
- gradient_penalty = gradient_penalty_fn(real_video, real_logits)
- # gradient_penalty = r1_penalty(real_video, real_logits) # MAGVIT uses r1 penalty
- gradient_penalty *= self.gradient_penalty_loss_weight
-
- # discriminator_loss = weighted_d_adversarial_loss + lecam_loss + gradient_penalty
-
- # log = {
- # "{}/d_adversarial_loss".format(split): weighted_d_adversarial_loss.detach().mean(),
- # "{}/lecam_loss".format(split): lecam_loss.detach().mean(),
- # "{}/gradient_penalty".format(split): gradient_penalty.detach().mean(),
- # }
-
- return (weighted_d_adversarial_loss, lecam_loss, gradient_penalty)
-
-
[email protected]_module("VAE_MAGVIT_V2")
-def VAE_MAGVIT_V2(from_pretrained=None, **kwargs):
- model = VAE_3D_V2(**kwargs)
- if from_pretrained is not None:
- load_checkpoint(model, from_pretrained, model_name="model")
- return model
-
-
[email protected]_module("DISCRIMINATOR_3D")
-def DISCRIMINATOR_3D(from_pretrained=None, inflate_from_2d=False, use_pretrained=True, **kwargs):
- model = StyleGANDiscriminatorBlur(**kwargs).apply(xavier_uniform_weight_init)
- if from_pretrained is not None:
- if use_pretrained:
- if inflate_from_2d:
- load_checkpoint_with_inflation(model, from_pretrained)
- else:
- load_checkpoint(model, from_pretrained, model_name="discriminator")
- print(f"loaded discriminator")
- else:
- print(f"discriminator use_pretrained={use_pretrained}, initializing new discriminator")
-
- return model
-
[email protected]_module("N_Layer_DISCRIMINATOR_3D")
-def DISCRIMINATOR_3D(from_pretrained=None, inflate_from_2d=False, use_pretrained=True, **kwargs):
- model = NLayerDiscriminator3D(input_nc=3, n_layers=3,).apply(n_layer_disc_weights_init)
- if from_pretrained is not None:
- if use_pretrained:
- if inflate_from_2d:
- load_checkpoint_with_inflation(model, from_pretrained)
- else:
- load_checkpoint(model, from_pretrained, model_name="discriminator")
- print(f"loaded discriminator")
- else:
- print(f"discriminator use_pretrained={use_pretrained}, initializing new discriminator")
-
- return model
-
-
-def load_checkpoint_with_inflation(model, ckpt_path):
- """
- pre-train using image, then inflate to 3D videos
- """
- if ckpt_path.endswith(".pt") or ckpt_path.endswith(".pth"):
- state_dict = find_model(ckpt_path)
- with torch.no_grad():
- for key in state_dict:
- if key in model:
- # central inflation
- if state_dict[key].size() == model[key][:, :, 0, :, :].size():
- # temporal dimension
- val = torch.zeros_like(model[key])
- centre = int(model[key].size(2) // 2)
- val[:, :, centre, :, :] = state_dict[key]
- missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)
- print(f"Missing keys: {missing_keys}")
- print(f"Unexpected keys: {unexpected_keys}")
- else:
- load_checkpoint(model, ckpt_path) # use the default function
diff --git a/scripts/inference-vae-3d.py b/scripts/inference-vae-3d.py
deleted file mode 100644
index a624a45..0000000
--- a/scripts/inference-vae-3d.py
+++ /dev/null
@@ -1,148 +0,0 @@
-import os
-
-import colossalai
-import torch
-import torch.distributed as dist
-from colossalai.cluster import DistCoordinator
-from mmengine.runner import set_random_seed
-
-from opensora.acceleration.parallel_states import set_sequence_parallel_group
-from opensora.datasets import save_sample
-from opensora.registry import MODELS, SCHEDULERS, build_module
-from opensora.utils.config_utils import parse_configs
-from opensora.utils.misc import to_torch_dtype
-from opensora.datasets import DatasetFromCSV, get_transforms_image, get_transforms_video, prepare_dataloader
-from opensora.acceleration.parallel_states import (
- get_data_parallel_group,
- set_data_parallel_group,
- set_sequence_parallel_group,
-)
-from tqdm import tqdm
-from opensora.models.vae.model_utils import VEA3DLoss
-from colossalai.utils import get_current_device
-
-
-def main():
- # ======================================================
- # 1. cfg and init distributed env
- # ======================================================
- cfg = parse_configs(training=False)
- print(cfg)
-
- # init distributed
- colossalai.launch_from_torch({})
- coordinator = DistCoordinator()
-
- # if coordinator.world_size > 1:
- # set_sequence_parallel_group(dist.group.WORLD)
- # enable_sequence_parallelism = True
- # else:
- # enable_sequence_parallelism = False
-
- # ======================================================
- # 2. runtime variables
- # ======================================================
- torch.set_grad_enabled(False)
- # torch.backends.cuda.matmul.allow_tf32 = True
- # torch.backends.cudnn.allow_tf32 = True
- # device = "cuda" if torch.cuda.is_available() else "cpu"
- device = get_current_device()
- dtype = to_torch_dtype(cfg.dtype)
- # set_random_seed(seed=cfg.seed)
-
-
- # ======================================================
- # 3. build dataset and dataloader
- # ======================================================
- dataset = DatasetFromCSV(
- cfg.data_path,
- # TODO: change transforms
- transform=(
- get_transforms_video(cfg.image_size[0])
- if not cfg.use_image_transform
- else get_transforms_image(cfg.image_size[0])
- ),
- num_frames=cfg.num_frames,
- frame_interval=cfg.frame_interval,
- root=cfg.root,
- )
-
- dataloader = prepare_dataloader(
- dataset,
- batch_size=cfg.batch_size,
- num_workers=cfg.num_workers,
- shuffle=False,
- drop_last=True,
- pin_memory=True,
- process_group=get_data_parallel_group(),
- )
- print(f"Dataset contains {len(dataset):,} videos ({cfg.data_path})")
-
- total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size
- print(f"Total batch size: {total_batch_size}")
-
- # ======================================================
- # 4. build model & load weights
- # ======================================================
- # 3.1. build model
- # input_size = (cfg.num_frames, *cfg.image_size)
- vae = build_module(cfg.model, MODELS, device=device)
- # latent_size = vae.get_latent_size(input_size)
-
- # 3.2. move to device & eval
- vae = vae.to(device, dtype).eval()
-
-
- # 3.4. support for multi-resolution
- model_args = dict()
- if cfg.multi_resolution:
- image_size = cfg.image_size
- hw = torch.tensor([image_size], device=device, dtype=dtype).repeat(cfg.batch_size, 1)
- ar = torch.tensor([[image_size[0] / image_size[1]]], device=device, dtype=dtype).repeat(cfg.batch_size, 1)
- model_args["data_info"] = dict(ar=ar, hw=hw)
-
- # ======================================================
- # 4. inference
- # ======================================================
- save_dir = cfg.save_dir
- os.makedirs(save_dir, exist_ok=True)
-
-
- # 4.1. batch generation
-
- # define loss function
- loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)
- running_loss = 0.0
- loss_steps = 0
-
-
- total_steps = len(dataloader)
- dataloader_iter = iter(dataloader)
-
- with tqdm(
- range(total_steps),
- # desc=f"Avg Loss: {running_loss}",
- disable=not coordinator.is_master(),
- total=total_steps,
- initial=0,
- ) as pbar:
- for step in pbar:
- batch = next(dataloader_iter)
- x = batch["video"].to(device, dtype) # [B, C, T, H, W]
- reconstructions, posterior = vae(x)
- loss = loss_function(x, reconstructions, posterior)
- loss_steps += 1
- running_loss = loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)
-
- if coordinator.is_master():
- pbar.set_postfix({"loss": loss, "running_loss": running_loss, "step": step})
- for idx, sample in enumerate(reconstructions):
- pos = step * cfg.batch_size + idx
- save_path = os.path.join(save_dir, f"sample_{pos}")
- save_sample(sample, fps=cfg.fps, save_path=save_path)
-
- print("test loss:", running_loss)
-
-
-if __name__ == "__main__":
- main()
diff --git a/scripts/inference-vae-v2.py b/scripts/inference-vae.py
similarity index 100%
rename from scripts/inference-vae-v2.py
rename to scripts/inference-vae.py
diff --git a/scripts/train-vae-3d.py b/scripts/train-vae-3d.py
deleted file mode 100644
index cb6a213..0000000
--- a/scripts/train-vae-3d.py
+++ /dev/null
@@ -1,271 +0,0 @@
-from copy import deepcopy
-
-import colossalai
-import torch
-import torch.nn as nn
-from torch.optim import Optimizer
-from torch.optim.lr_scheduler import _LRScheduler
-import torch.distributed as dist
-import wandb
-from colossalai.booster import Booster
-from colossalai.booster.plugin import LowLevelZeroPlugin
-from colossalai.cluster import DistCoordinator
-from colossalai.nn.optimizer import HybridAdam
-from colossalai.utils import get_current_device
-from tqdm import tqdm
-import os
-
-from opensora.acceleration.checkpoint import set_grad_checkpoint
-from opensora.acceleration.parallel_states import (
- get_data_parallel_group,
- set_data_parallel_group,
- set_sequence_parallel_group,
-)
-from opensora.acceleration.plugin import ZeroSeqParallelPlugin
-from opensora.datasets import DatasetFromCSV, get_transforms_image, get_transforms_video, prepare_dataloader
-from opensora.registry import MODELS, SCHEDULERS, build_module
-from opensora.utils.ckpt_utils import create_logger, load_json, save_json, load, model_sharding, record_model_param_shape, save
-from opensora.utils.config_utils import (
- create_experiment_workspace,
- create_tensorboard_writer,
- parse_configs,
- save_training_config,
-)
-from opensora.utils.misc import all_reduce_mean, format_numel_str, get_model_numel, requires_grad, to_torch_dtype
-from opensora.utils.train_utils import update_ema, MaskGenerator
-from opensora.models.vae.model_utils import VEA3DLoss
-
-
-def main():
- # ======================================================
- # 1. args & cfg
- # ======================================================
- cfg = parse_configs(training=True)
- print(cfg)
- exp_name, exp_dir = create_experiment_workspace(cfg)
- save_training_config(cfg._cfg_dict, exp_dir)
-
- # ======================================================
- # 2. runtime variables & colossalai launch
- # ======================================================
- assert torch.cuda.is_available(), "Training currently requires at least one GPU."
- assert cfg.dtype in ["fp16", "bf16"], f"Unknown mixed precision {cfg.dtype}"
-
- # 2.1. colossalai init distributed training
- colossalai.launch_from_torch({})
- coordinator = DistCoordinator()
- device = get_current_device()
- dtype = to_torch_dtype(cfg.dtype)
-
- # 2.2. init logger, tensorboard & wandb
- if not coordinator.is_master():
- logger = create_logger(None)
- else:
- logger = create_logger(exp_dir)
- logger.info(f"Experiment directory created at {exp_dir}")
-
- writer = create_tensorboard_writer(exp_dir)
- if cfg.wandb:
- wandb.init(project="opensora-vae", name=exp_name, config=cfg._cfg_dict)
-
- # 2.3. initialize ColossalAI booster
- if cfg.plugin == "zero2":
- plugin = LowLevelZeroPlugin(
- stage=2,
- precision=cfg.dtype,
- initial_scale=2**16,
- max_norm=cfg.grad_clip,
- )
- set_data_parallel_group(dist.group.WORLD)
- elif cfg.plugin == "zero2-seq":
- plugin = ZeroSeqParallelPlugin(
- sp_size=cfg.sp_size,
- stage=2,
- precision=cfg.dtype,
- initial_scale=2**16,
- max_norm=cfg.grad_clip,
- )
- set_sequence_parallel_group(plugin.sp_group)
- set_data_parallel_group(plugin.dp_group)
- else:
- raise ValueError(f"Unknown plugin {cfg.plugin}")
- booster = Booster(plugin=plugin)
-
- # ======================================================
- # 3. build dataset and dataloader
- # ======================================================
- dataset = DatasetFromCSV(
- cfg.data_path,
- # TODO: change transforms
- transform=(
- get_transforms_video(cfg.image_size[0])
- if not cfg.use_image_transform
- else get_transforms_image(cfg.image_size[0])
- ),
- num_frames=cfg.num_frames,
- frame_interval=cfg.frame_interval,
- root=cfg.root,
- )
-
- # TODO: use plugin's prepare dataloader
- # a batch contains:
- # {
- # "video": torch.Tensor, # [B, C, T, H, W],
- # "text": List[str],
- # }
- dataloader = prepare_dataloader(
- dataset,
- batch_size=cfg.batch_size,
- num_workers=cfg.num_workers,
- shuffle=True,
- drop_last=True,
- pin_memory=True,
- process_group=get_data_parallel_group(),
- )
- logger.info(f"Dataset contains {len(dataset):,} videos ({cfg.data_path})")
-
- total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size
- logger.info(f"Total batch size: {total_batch_size}")
-
- # ======================================================
- # 4. build model
- # ======================================================
- # 4.1. build model
- vae = build_module(cfg.model, MODELS, device=device)
- vae_numel, vae_numel_trainable = get_model_numel(vae)
- logger.info(
- f"Trainable vae params: {format_numel_str(vae_numel_trainable)}, Total model params: {format_numel_str(vae_numel)}"
- )
-
-
- # 4.3. move to device
- vae = vae.to(device, dtype)
-
- # 4.5. setup optimizer
- optimizer = HybridAdam(
- filter(lambda p: p.requires_grad, vae.parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True
- )
- lr_scheduler = None
-
- # 4.6. prepare for training
- if cfg.grad_checkpoint:
- set_grad_checkpoint(vae)
- vae.train()
-
-
- # =======================================================
- # 5. boost model for distributed training with colossalai
- # =======================================================
- torch.set_default_dtype(dtype)
- vae, optimizer, _, dataloader, lr_scheduler = booster.boost(
- model=vae, optimizer=optimizer, lr_scheduler=lr_scheduler, dataloader=dataloader
- )
- torch.set_default_dtype(torch.float)
- num_steps_per_epoch = len(dataloader)
- logger.info("Boost vae for distributed training")
-
- # =======================================================
- # 6. training loop
- # =======================================================
- start_epoch = start_step = log_step = sampler_start_idx = 0
- running_loss = 0.0
-
-
- # 6.1. resume training
- if cfg.load is not None:
- logger.info("Loading checkpoint")
- booster.load_model(vae, os.path.join(cfg.load, "model"))
- booster.load_optimizer(optimizer, os.path.join(cfg.load, "optimizer"))
- if lr_scheduler is not None:
- booster.load_lr_scheduler(lr_scheduler, os.path.join(cfg.load, "lr_scheduler"))
- running_states = load_json(os.path.join(cfg.load, "running_states.json"))
- dist.barrier()
- start_epoch, start_step, sampler_start_idx = running_states["epoch"], running_states["step"], running_states["sample_start_index"]
- logger.info(f"Loaded checkpoint {cfg.load} at epoch {start_epoch} step {start_step}")
- logger.info(f"Training for {cfg.epochs} epochs with {num_steps_per_epoch} steps per epoch")
-
- dataloader.sampler.set_start_index(sampler_start_idx)
-
- # define loss function
- loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)
-
-
- # 6.2. training loop
- for epoch in range(start_epoch, cfg.epochs):
- dataloader.sampler.set_epoch(epoch)
- dataloader_iter = iter(dataloader)
- logger.info(f"Beginning epoch {epoch}...")
-
- with tqdm(
- range(start_step, num_steps_per_epoch),
- desc=f"Epoch {epoch}",
- disable=not coordinator.is_master(),
- total=num_steps_per_epoch,
- initial=start_step,
- ) as pbar:
- for step in pbar:
- batch = next(dataloader_iter)
- x = batch["video"].to(device, dtype) # [B, C, T, H, W]
-
- # loss = vae.get_loss(x)
- reconstructions, posterior = vae(x)
- loss = loss_function(x, reconstructions, posterior)
-
- # Backward & update
- booster.backward(loss=loss, optimizer=optimizer)
- optimizer.step()
- optimizer.zero_grad()
-
- # Log loss values:
- all_reduce_mean(loss)
- running_loss += loss.item()
- global_step = epoch * num_steps_per_epoch + step
- log_step += 1
-
- # Log to tensorboard
- if coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:
- avg_loss = running_loss / log_step
- pbar.set_postfix({"loss": avg_loss, "step": step, "global_step": global_step})
- running_loss = 0
- log_step = 0
- writer.add_scalar("loss", loss.item(), global_step)
- if cfg.wandb:
- wandb.log(
- {
- "iter": global_step,
- "num_samples": global_step * total_batch_size,
- "epoch": epoch,
- "loss": loss.item(),
- "avg_loss": avg_loss,
- },
- step=global_step,
- )
-
- # Save checkpoint
- if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:
- save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")
- os.makedirs(os.path.join(save_dir, "model"), exist_ok=True)
- # TODO: save in model?
- booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)
- booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)
- if lr_scheduler is not None:
- booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))
- running_states = {
- "epoch": epoch,
- "step": step+1,
- "global_step": global_step+1,
- "sample_start_index": (step+1) * cfg.batch_size,
- }
- if coordinator.is_master():
- save_json(running_states, os.path.join(save_dir, "running_states.json"))
- dist.barrier()
- logger.info(
- f"Saved checkpoint at epoch {epoch} step {step + 1} global_step {global_step + 1} to {exp_dir}"
- )
-
- # the continue epochs are not resumed, so we need to reset the sampler start index and start step
- dataloader.sampler.set_start_index(0)
- start_step = 0
-
-if __name__ == "__main__":
- main()
diff --git a/scripts/train-vae-v2.py b/scripts/train-vae.py
similarity index 100%
rename from scripts/train-vae-v2.py
rename to scripts/train-vae.py