Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[wip] try vae

zhengzangwb2f302b379be
diff --git a/.gitignore b/.gitignore
index 4d5dc97..b9f8121 100644
--- a/.gitignore
+++ b/.gitignore
@@ -181,4 +181,3 @@ cache/
hostfile
gradio_cached_examples/
wandb/
-taming/
diff --git a/configs/vae/train/17x256x256.py b/configs/vae/train/17x256x256.py
index d5a49d7..2b24de2 100644
--- a/configs/vae/train/17x256x256.py
+++ b/configs/vae/train/17x256x256.py
@@ -1,31 +1,22 @@
num_frames = 17
image_size = (256, 256)
+
+# Define dataset
dataset = dict(
type="VideoTextDataset",
data_path=None,
num_frames=num_frames,
frame_interval=1,
image_size=image_size,
- get_text=False,
)
# Define acceleration
-num_workers = 4
+num_workers = 16
dtype = "bf16"
grad_checkpoint = True
plugin = "zero2"
sp_size = 1
-use_pipeline = True
-video_contains_first_frame = True
-
-# Define model
-# vae_2d = dict(
-# type="VideoAutoencoderKL",
-# from_pretrained="stabilityai/sd-vae-ft-ema",
-# # SDXL
-# )
-
# latest
vae_2d = dict(
type="VideoAutoencoderKL",
@@ -36,33 +27,20 @@ vae_2d = dict(
)
model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels=4,
- latent_embed_dim=64,
- filters=128,
- num_res_blocks=4,
- channel_multipliers=(1, 2, 2, 4),
- temporal_downsample=(False, True, True),
- num_groups=32, # for nn.GroupNorm
- kl_embed_dim=4,
- activation_fn="swish",
- separate_first_frame_encoding=False,
- disable_space=True,
- encoder_double_z=True,
- custom_conv_padding=None,
+ type="VAE_Temporal_SD",
)
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size=image_size, # NOTE: here image size is different
- num_frames=num_frames,
- in_channels=3,
- filters=128,
- use_pretrained=True, # NOTE: set to False only if we want to disable load
- channel_multipliers = (2,4,4,4,4), # (2,4,4,4) for 64x64 resolution
- # channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z
-)
+# discriminator = dict(
+# type="DISCRIMINATOR_3D",
+# image_size=image_size, # NOTE: here image size is different
+# num_frames=num_frames,
+# in_channels=3,
+# filters=128,
+# use_pretrained=True, # NOTE: set to False only if we want to disable load
+# channel_multipliers=(2, 4, 4, 4, 4), # (2,4,4,4) for 64x64 resolution
+# # channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z
+# )
# loss weights
@@ -76,7 +54,7 @@ discriminator_loss_type = "non-saturating"
generator_loss_type = "non-saturating"
# discriminator_loss_type="hinge"
# generator_loss_type="hinge"
-discriminator_start = 2000 # 5000 # 8k data / (8*1) = 1000 steps per epoch
+discriminator_start = 2000 # 5000 # 8k data / (8*1) = 1000 steps per epoch
gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
ema_decay = 0.999 # ema decay factor for generator
@@ -95,11 +73,9 @@ magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
epochs = 100
log_every = 1
-
-ckpt_every = 500
+ckpt_every = 1000
load = None
-batch_size = 1
-
-lr = 1e-4
+batch_size = 4
+lr = 1e-5
grad_clip = 1.0
diff --git a/configs/vae/train/1x256x256.py b/configs/vae/train/1x256x256.py
index 91d99a9..582bbcb 100644
--- a/configs/vae/train/1x256x256.py
+++ b/configs/vae/train/1x256x256.py
@@ -1,31 +1,22 @@
num_frames = 1
image_size = (256, 256)
+
+# Define dataset
dataset = dict(
type="VideoTextDataset",
data_path=None,
num_frames=num_frames,
frame_interval=1,
image_size=image_size,
- get_text=False,
)
# Define acceleration
-num_workers = 4
+num_workers = 16
dtype = "bf16"
-grad_checkpoint = True
+grad_checkpoint = False
plugin = "zero2"
sp_size = 1
-use_pipeline = True
-video_contains_first_frame = True
-
-# Define model
-# vae_2d = dict(
-# type="VideoAutoencoderKL",
-# from_pretrained="stabilityai/sd-vae-ft-ema",
-# # SDXL
-# )
-
# latest
vae_2d = dict(
type="VideoAutoencoderKL",
@@ -36,33 +27,19 @@ vae_2d = dict(
)
model = dict(
- type="VAE_MAGVIT_V2",
- in_out_channels=4,
- latent_embed_dim=64,
- filters=128,
- num_res_blocks=4,
- channel_multipliers=(1, 2, 2, 4),
- temporal_downsample=(False, True, True),
- num_groups=32, # for nn.GroupNorm
- kl_embed_dim=4,
- activation_fn="swish",
- separate_first_frame_encoding=False,
- disable_space=True,
- custom_conv_padding=None,
- encoder_double_z=True,
+ type="VAE_Temporal_SD",
)
-
-discriminator = dict(
- type="DISCRIMINATOR_3D",
- image_size=image_size, # NOTE: here image size is different
- num_frames=num_frames,
- in_channels=3,
- filters=128,
- use_pretrained=True, # NOTE: set to False only if we want to disable load
- channel_multipliers=(2, 4, 4, 4, 4), # (2,4,4,4) for 64x64 resolution
- # channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z
-)
+# discriminator = dict(
+# type="DISCRIMINATOR_3D",
+# image_size=image_size, # NOTE: here image size is different
+# num_frames=num_frames,
+# in_channels=3,
+# filters=128,
+# use_pretrained=True, # NOTE: set to False only if we want to disable load
+# channel_multipliers=(2, 4, 4, 4, 4), # (2,4,4,4) for 64x64 resolution
+# # channel_multipliers=(2, 4, 4), # since on intermediate layer dimension ofs z
+# )
# loss weights
@@ -70,14 +47,12 @@ logvar_init = 0.0
kl_loss_weight = 0.000001
perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
discriminator_factor = 1.0 # for discriminator adversarial loss
-generator_factor = 0.1 # SCH: generator adversarial loss, MAGVIT v2 uses 0.1
+generator_factor = 0.1 # generator adversarial loss, MAGVIT v2 uses 0.1
lecam_loss_weight = None # NOTE: MAVGIT v2 use 0.001
discriminator_loss_type = "non-saturating"
generator_loss_type = "non-saturating"
-# discriminator_loss_type="hinge"
-# generator_loss_type="hinge"
-discriminator_start = 2000 # 5000 # 8k data / (8*1) = 1000 steps per epoch
-gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
+discriminator_start = 2000
+gradient_penalty_loss_weight = None # MAGVIT uses 10, opensora plan doesn't use
ema_decay = 0.999 # ema decay factor for generator
@@ -95,11 +70,9 @@ magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
epochs = 100
log_every = 1
-
-ckpt_every = 500
+ckpt_every = 1000
load = None
-batch_size = 1
-
-lr = 1e-4
+batch_size = 4
+lr = 1e-5
grad_clip = 1.0
diff --git a/opensora/datasets/datasets.py b/opensora/datasets/datasets.py
index d320ff7..b9d7f05 100644
--- a/opensora/datasets/datasets.py
+++ b/opensora/datasets/datasets.py
@@ -29,10 +29,10 @@ class VideoTextDataset(torch.utils.data.Dataset):
frame_interval=1,
image_size=(256, 256),
transform_name="center",
- get_text=True,
):
self.data_path = data_path
self.data = read_file(data_path)
+ self.get_text = "text" in self.data.columns
self.num_frames = num_frames
self.frame_interval = frame_interval
self.image_size = image_size
@@ -40,7 +40,6 @@ class VideoTextDataset(torch.utils.data.Dataset):
"image": get_transforms_image(transform_name, image_size),
"video": get_transforms_video(transform_name, image_size),
}
- self.get_text = get_text
def _print_data_number(self):
num_videos = 0
@@ -63,7 +62,6 @@ class VideoTextDataset(torch.utils.data.Dataset):
def getitem(self, index):
sample = self.data.iloc[index]
path = sample["path"]
- text = sample["text"] if self.get_text else None
file_type = self.get_type(path)
if file_type == "video":
@@ -90,10 +88,10 @@ class VideoTextDataset(torch.utils.data.Dataset):
# TCHW -> CTHW
video = video.permute(1, 0, 2, 3)
+ ret = {"video": video}
if self.get_text:
- return {"video": video, "text": text}
- else:
- return {"video": video}
+ ret["text"] = sample["text"]
+ return ret
def __getitem__(self, index):
for _ in range(10):
@@ -135,7 +133,6 @@ class VariableVideoTextDataset(VideoTextDataset):
sample = self.data.iloc[index]
path = sample["path"]
- text = sample["text"]
file_type = self.get_type(path)
ar = width / height
@@ -166,12 +163,14 @@ class VariableVideoTextDataset(VideoTextDataset):
# TCHW -> CTHW
video = video.permute(1, 0, 2, 3)
- return {
+ ret = {
"video": video,
- "text": text,
"num_frames": num_frames,
"height": height,
"width": width,
"ar": ar,
"fps": video_fps,
}
+ if self.get_text:
+ ret["text"] = sample["text"]
+ return ret
diff --git a/opensora/models/vae/__init__.py b/opensora/models/vae/__init__.py
index c46c015..78a24fd 100644
--- a/opensora/models/vae/__init__.py
+++ b/opensora/models/vae/__init__.py
@@ -1,2 +1,3 @@
+from .discriminator import DISCRIMINATOR_3D
from .vae import VideoAutoencoderKL, VideoAutoencoderKLTemporalDecoder
-from .vae_3d import VAE_3D_V2, VAE_MAGVIT_V2
+from .vae_3d import VAE_Temporal
diff --git a/opensora/models/vae/discriminator.py b/opensora/models/vae/discriminator.py
new file mode 100644
index 0000000..e510387
--- /dev/null
+++ b/opensora/models/vae/discriminator.py
@@ -0,0 +1,364 @@
+import functools
+import math
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from opensora.registry import MODELS
+from opensora.utils.ckpt_utils import find_model, load_checkpoint
+
+
+def cast_tuple(t, length=1):
+ return t if isinstance(t, tuple) else ((t,) * length)
+
+
+def xavier_uniform_weight_init(m):
+ if isinstance(m, nn.Conv3d) or isinstance(m, nn.Linear):
+ nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain("relu"))
+ if m.bias is not None:
+ nn.init.zeros_(m.bias)
+ # print("initialized module to xavier_uniform:", m)
+
+
+# SCH: taken from Open Sora Plan
+def n_layer_disc_weights_init(m):
+ classname = m.__class__.__name__
+ if classname.find("Conv") != -1:
+ nn.init.normal_(m.weight.data, 0.0, 0.02)
+ elif classname.find("BatchNorm") != -1:
+ nn.init.normal_(m.weight.data, 1.0, 0.02)
+ nn.init.constant_(m.bias.data, 0)
+
+
+# SCH: own implementation modified on top of: discriminator with anti-aliased downsampling (blurpool Zhang et al.)
+class BlurPool3D(nn.Module):
+ def __init__(
+ self,
+ channels,
+ pad_type="reflect",
+ filt_size=3,
+ stride=2,
+ pad_off=0,
+ device="cpu",
+ dtype=torch.bfloat16,
+ ):
+ super(BlurPool3D, self).__init__()
+ self.filt_size = filt_size
+ self.pad_off = pad_off
+ self.pad_sizes = [
+ int(1.0 * (filt_size - 1) / 2),
+ int(np.ceil(1.0 * (filt_size - 1) / 2)),
+ int(1.0 * (filt_size - 1) / 2),
+ int(np.ceil(1.0 * (filt_size - 1) / 2)),
+ int(1.0 * (filt_size - 1) / 2),
+ int(np.ceil(1.0 * (filt_size - 1) / 2)),
+ ]
+ self.pad_sizes = [pad_size + pad_off for pad_size in self.pad_sizes]
+ self.stride = stride
+ self.off = int((self.stride - 1) / 2.0)
+ self.channels = channels
+
+ if self.filt_size == 1:
+ a = np.array(
+ [
+ 1.0,
+ ]
+ )
+ elif self.filt_size == 2:
+ a = np.array([1.0, 1.0])
+ elif self.filt_size == 3:
+ a = np.array([1.0, 2.0, 1.0])
+ elif self.filt_size == 4:
+ a = np.array([1.0, 3.0, 3.0, 1.0])
+ elif self.filt_size == 5:
+ a = np.array([1.0, 4.0, 6.0, 4.0, 1.0])
+ elif self.filt_size == 6:
+ a = np.array([1.0, 5.0, 10.0, 10.0, 5.0, 1.0])
+ elif self.filt_size == 7:
+ a = np.array([1.0, 6.0, 15.0, 20.0, 15.0, 6.0, 1.0])
+
+ filt_2d = a[:, None] * a[None, :]
+ filt_3d = torch.Tensor(a[:, None, None] * filt_2d[None, :, :]).to(device, dtype)
+
+ filt = filt_3d / torch.sum(filt_3d) # SCH: modified to it 3D
+ self.register_buffer("filt", filt[None, None, :, :, :].repeat((self.channels, 1, 1, 1, 1)))
+
+ self.pad = get_pad_layer(pad_type)(self.pad_sizes)
+
+ def forward(self, inp):
+ if self.filt_size == 1:
+ if self.pad_off == 0:
+ return inp[:, :, :: self.stride, :: self.stride]
+ else:
+ return self.pad(inp)[:, :, :: self.stride, :: self.stride]
+ else:
+ return F.conv3d(self.pad(inp), self.filt, stride=self.stride, groups=inp.shape[1])
+
+
+class ResBlockDown(nn.Module):
+ """3D StyleGAN ResBlock for D."""
+
+ def __init__(
+ self,
+ in_channels,
+ filters,
+ activation_fn,
+ num_groups=32,
+ device="cpu",
+ dtype=torch.bfloat16,
+ ):
+ super().__init__()
+
+ self.filters = filters
+ self.activation_fn = activation_fn
+
+ # SCH: NOTE: although paper says conv (X->Y, Y->Y), original code implementation is (X->X, X->Y), we follow code
+ self.conv1 = nn.Conv3d(
+ in_channels, in_channels, (3, 3, 3), padding=1, device=device, dtype=dtype
+ ) # NOTE: init to xavier_uniform
+ self.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)
+
+ self.blur = BlurPool3D(in_channels, device=device, dtype=dtype)
+
+ self.conv2 = nn.Conv3d(
+ in_channels, self.filters, (1, 1, 1), bias=False, device=device, dtype=dtype
+ ) # NOTE: init to xavier_uniform
+ self.conv3 = nn.Conv3d(
+ in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype
+ ) # NOTE: init to xavier_uniform
+ self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)
+
+ # self.apply(xavier_uniform_weight_init)
+
+ def forward(self, x):
+ residual = x
+ x = self.conv1(x)
+ x = self.norm1(x)
+ x = self.activation_fn(x)
+
+ residual = self.blur(residual)
+ residual = self.conv2(residual)
+
+ x = self.blur(x)
+ x = self.conv3(x)
+ x = self.norm2(x)
+ x = self.activation_fn(x)
+ out = (residual + x) / math.sqrt(2)
+ return out
+
+
+class NLayerDiscriminator3D(nn.Module):
+ """Defines a 3D PatchGAN discriminator as in Pix2Pix but for 3D inputs."""
+
+ def __init__(self, input_nc=1, ndf=64, n_layers=3, use_actnorm=False):
+ """
+ Construct a 3D PatchGAN discriminator
+
+ Parameters:
+ input_nc (int) -- the number of channels in input volumes
+ ndf (int) -- the number of filters in the last conv layer
+ n_layers (int) -- the number of conv layers in the discriminator
+ use_actnorm (bool) -- flag to use actnorm instead of batchnorm
+ """
+ super(NLayerDiscriminator3D, self).__init__()
+ if not use_actnorm:
+ norm_layer = nn.BatchNorm3d
+ else:
+ raise NotImplementedError("Not implemented.")
+ if type(norm_layer) == functools.partial:
+ use_bias = norm_layer.func != nn.BatchNorm3d
+ else:
+ use_bias = norm_layer != nn.BatchNorm3d
+
+ kw = 4
+ padw = 1
+ sequence = [nn.Conv3d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]
+ nf_mult = 1
+ nf_mult_prev = 1
+ for n in range(1, n_layers): # gradually increase the number of filters
+ nf_mult_prev = nf_mult
+ nf_mult = min(2**n, 8)
+ sequence += [
+ nn.Conv3d(
+ ndf * nf_mult_prev,
+ ndf * nf_mult,
+ kernel_size=(kw, kw, kw),
+ stride=(1, 2, 2),
+ padding=padw,
+ bias=use_bias,
+ ),
+ norm_layer(ndf * nf_mult),
+ nn.LeakyReLU(0.2, True),
+ ]
+
+ nf_mult_prev = nf_mult
+ nf_mult = min(2**n_layers, 8)
+ sequence += [
+ nn.Conv3d(
+ ndf * nf_mult_prev, ndf * nf_mult, kernel_size=(kw, kw, kw), stride=1, padding=padw, bias=use_bias
+ ),
+ norm_layer(ndf * nf_mult),
+ nn.LeakyReLU(0.2, True),
+ ]
+
+ sequence += [
+ nn.Conv3d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)
+ ] # output 1 channel prediction map
+ self.main = nn.Sequential(*sequence)
+
+ def forward(self, input):
+ """Standard forward."""
+ return self.main(input)
+
+
+class StyleGANDiscriminatorBlur(nn.Module):
+ """StyleGAN Discriminator.
+
+ SCH: NOTE:
+ this discriminator requries the num_frames to be fixed during training;
+ in case we pre-train with image then train on video, this disciminator's Linear layer would have to be re-trained!
+ """
+
+ def __init__(
+ self,
+ image_size=(128, 128),
+ num_frames=17,
+ in_channels=3,
+ filters=128,
+ channel_multipliers=(2, 4, 4, 4, 4),
+ num_groups=32,
+ dtype=torch.bfloat16,
+ device="cpu",
+ ):
+ super().__init__()
+
+ self.dtype = dtype
+ self.input_size = cast_tuple(image_size, 2)
+ self.filters = filters
+ self.activation_fn = nn.LeakyReLU(negative_slope=0.2)
+ self.channel_multipliers = channel_multipliers
+
+ self.conv1 = nn.Conv3d(
+ in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype
+ ) # NOTE: init to xavier_uniform
+
+ prev_filters = self.filters # record in_channels
+ self.num_blocks = len(self.channel_multipliers)
+ self.res_block_list = nn.ModuleList([])
+ for i in range(self.num_blocks):
+ filters = self.filters * self.channel_multipliers[i]
+ self.res_block_list.append(
+ ResBlockDown(prev_filters, filters, self.activation_fn, device=device, dtype=dtype).apply(
+ xavier_uniform_weight_init
+ )
+ )
+ prev_filters = filters # update in_channels
+
+ self.conv2 = nn.Conv3d(
+ prev_filters, prev_filters, (3, 3, 3), padding=1, device=device, dtype=dtype
+ ) # NOTE: init to xavier_uniform
+ # torch.nn.init.xavier_uniform_(self.conv2.weight)
+
+ self.norm1 = nn.GroupNorm(num_groups, prev_filters, dtype=dtype, device=device)
+
+ scale_factor = 2**self.num_blocks
+ if num_frames % scale_factor != 0: # SCH: NOTE: has first frame which would be padded before usage
+ time_scaled = num_frames // scale_factor + 1
+ else:
+ time_scaled = num_frames / scale_factor
+
+ assert (
+ self.input_size[0] % scale_factor == 0
+ ), f"image width {self.input_size[0]} is not divisible by scale factor {scale_factor}"
+ assert (
+ self.input_size[1] % scale_factor == 0
+ ), f"image height {self.input_size[1]} is not divisible by scale factor {scale_factor}"
+ w_scaled, h_scaled = self.input_size[0] / scale_factor, self.input_size[1] / scale_factor
+ in_features = int(prev_filters * time_scaled * w_scaled * h_scaled) # (C*T*W*H)
+ self.linear1 = nn.Linear(in_features, prev_filters, device=device, dtype=dtype) # NOTE: init to xavier_uniform
+ self.linear2 = nn.Linear(prev_filters, 1, device=device, dtype=dtype) # NOTE: init to xavier_uniform
+
+ # self.apply(xavier_uniform_weight_init)
+
+ def forward(self, x):
+ x = self.conv1(x)
+ # print("discriminator aft conv:", x.size())
+ x = self.activation_fn(x)
+
+ for i in range(self.num_blocks):
+ x = self.res_block_list[i](x)
+ # print("discriminator resblock down:", x.size())
+
+ x = self.conv2(x)
+ # print("discriminator aft conv2:", x.size())
+ x = self.norm1(x)
+ x = self.activation_fn(x)
+ x = x.reshape((x.shape[0], -1)) # SCH: [B, (C * T * W * H)] ?
+
+ # print("discriminator reshape:", x.size())
+ x = self.linear1(x)
+ # print("discriminator aft linear1:", x.size())
+
+ x = self.activation_fn(x)
+ x = self.linear2(x)
+ # print("discriminator aft linear2:", x.size())
+ return x
+
+
+def load_checkpoint_with_inflation(model, ckpt_path):
+ """
+ pre-train using image, then inflate to 3D videos
+ """
+ if ckpt_path.endswith(".pt") or ckpt_path.endswith(".pth"):
+ state_dict = find_model(ckpt_path)
+ with torch.no_grad():
+ for key in state_dict:
+ if key in model:
+ # central inflation
+ if state_dict[key].size() == model[key][:, :, 0, :, :].size():
+ # temporal dimension
+ val = torch.zeros_like(model[key])
+ centre = int(model[key].size(2) // 2)
+ val[:, :, centre, :, :] = state_dict[key]
+ missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)
+ print(f"Missing keys: {missing_keys}")
+ print(f"Unexpected keys: {unexpected_keys}")
+ else:
+ load_checkpoint(model, ckpt_path) # use the default function
+
+
[email protected]_module("DISCRIMINATOR_3D")
+def DISCRIMINATOR_3D(from_pretrained=None, inflate_from_2d=False, use_pretrained=True, **kwargs):
+ model = StyleGANDiscriminatorBlur(**kwargs).apply(xavier_uniform_weight_init)
+ if from_pretrained is not None:
+ if use_pretrained:
+ if inflate_from_2d:
+ load_checkpoint_with_inflation(model, from_pretrained)
+ else:
+ load_checkpoint(model, from_pretrained, model_name="discriminator")
+ print("loaded discriminator")
+ else:
+ print(f"discriminator use_pretrained={use_pretrained}, initializing new discriminator")
+
+ return model
+
+
[email protected]_module("N_Layer_DISCRIMINATOR_3D")
+def DISCRIMINATOR_3D_N_Layer(from_pretrained=None, inflate_from_2d=False, use_pretrained=True, **kwargs):
+ model = NLayerDiscriminator3D(
+ input_nc=3,
+ n_layers=3,
+ ).apply(n_layer_disc_weights_init)
+ if from_pretrained is not None:
+ if use_pretrained:
+ if inflate_from_2d:
+ load_checkpoint_with_inflation(model, from_pretrained)
+ else:
+ load_checkpoint(model, from_pretrained, model_name="discriminator")
+ print("loaded discriminator")
+ else:
+ print(f"discriminator use_pretrained={use_pretrained}, initializing new discriminator")
+
+ return model
diff --git a/opensora/models/vae/losses.py b/opensora/models/vae/losses.py
index 4adc85c..68d555f 100644
--- a/opensora/models/vae/losses.py
+++ b/opensora/models/vae/losses.py
@@ -1,9 +1,11 @@
-import torch.nn as nn
import torch
+import torch.nn as nn
import torch.nn.functional as F
-from .lpips import LPIPS
from einops import rearrange, repeat
+from .lpips import LPIPS
+
+
def hinge_d_loss(logits_real, logits_fake):
loss_real = torch.mean(F.relu(1.0 - logits_real))
loss_fake = torch.mean(F.relu(1.0 + logits_fake))
@@ -27,12 +29,14 @@ def sigmoid_cross_entropy_with_logits(labels, logits):
neg_abs_logits = torch.where(condition, -logits, logits)
return relu_logits - logits * labels + torch.log1p(torch.exp(neg_abs_logits))
+
def lecam_reg(real_pred, fake_pred, ema_real_pred, ema_fake_pred):
assert real_pred.ndim == 0 and ema_fake_pred.ndim == 0
lecam_loss = torch.mean(torch.pow(nn.ReLU()(real_pred - ema_fake_pred), 2))
lecam_loss += torch.mean(torch.pow(nn.ReLU()(ema_real_pred - fake_pred), 2))
return lecam_loss
+
def gradient_penalty_fn(images, output):
gradients = torch.autograd.grad(
outputs=output,
@@ -46,7 +50,8 @@ def gradient_penalty_fn(images, output):
gradients = rearrange(gradients, "b ... -> b (...)")
return ((gradients.norm(2, dim=1) - 1) ** 2).mean()
-class VEALoss(nn.Module):
+
+class VAELoss(nn.Module):
def __init__(
self,
logvar_init=0.0,
@@ -72,7 +77,6 @@ class VEALoss(nn.Module):
self.perceptual_loss_weight = perceptual_loss_weight
self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)
-
def forward(
self,
video,
@@ -111,7 +115,6 @@ class VEALoss(nn.Module):
weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]
nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]
-
# KL Loss
weighted_kl_loss = 0
if self.kl_loss_weight is not None and self.kl_loss_weight > 0.0:
@@ -121,11 +124,13 @@ class VEALoss(nn.Module):
return nll_loss, weighted_nll_loss, weighted_kl_loss
+
def adopt_weight(weight, global_step, threshold=0, value=0.0):
if global_step < threshold:
weight = value
return weight
+
class AdversarialLoss(nn.Module):
def __init__(
self,
@@ -141,10 +146,8 @@ class AdversarialLoss(nn.Module):
self.generator_loss_type = generator_loss_type
def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer):
- nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0]
- g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[
- 0
- ]
+ nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0]
+ g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0]
d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)
d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()
d_weight = d_weight * self.generator_factor
@@ -269,4 +272,4 @@ class DiscriminatorLoss(nn.Module):
gradient_penalty = gradient_penalty_fn(real_video, real_logits)
gradient_penalty *= self.gradient_penalty_loss_weight
- return (weighted_d_adversarial_loss, lecam_loss, gradient_penalty)
\ No newline at end of file
+ return (weighted_d_adversarial_loss, lecam_loss, gradient_penalty)
diff --git a/opensora/models/vae/lpips.py b/opensora/models/vae/lpips.py
index b22cd0d..d1f00c7 100644
--- a/opensora/models/vae/lpips.py
+++ b/opensora/models/vae/lpips.py
@@ -1,28 +1,26 @@
+import hashlib
+import os
+from collections import namedtuple
+
+import requests
import torch
import torch.nn as nn
from torchvision import models
-from collections import namedtuple
-import os, hashlib
-import requests
from tqdm import tqdm
-URL_MAP = {
- "vgg_lpips": "https://heibox.uni-heidelberg.de/f/607503859c864bc1b30b/?dl=1"
-}
+URL_MAP = {"vgg_lpips": "https://heibox.uni-heidelberg.de/f/607503859c864bc1b30b/?dl=1"}
-CKPT_MAP = {
- "vgg_lpips": "vgg.pth"
-}
+CKPT_MAP = {"vgg_lpips": "vgg.pth"}
+
+MD5_MAP = {"vgg_lpips": "d507d7349b931f0638a25a48a722f98a"}
-MD5_MAP = {
- "vgg_lpips": "d507d7349b931f0638a25a48a722f98a"
-}
def md5_hash(path):
with open(path, "rb") as f:
content = f.read()
return hashlib.md5(content).hexdigest()
+
def download(url, local_path, chunk_size=1024):
os.makedirs(os.path.split(local_path)[0], exist_ok=True)
with requests.get(url, stream=True) as r:
@@ -34,6 +32,7 @@ def download(url, local_path, chunk_size=1024):
f.write(data)
pbar.update(chunk_size)
+
def get_ckpt_path(name, root, check=False):
assert name in URL_MAP
path = os.path.join(root, CKPT_MAP[name])
@@ -45,14 +44,13 @@ def get_ckpt_path(name, root, check=False):
return path
-
class LPIPS(nn.Module):
# Learned perceptual metric
def __init__(self, use_dropout=True):
super().__init__()
self.scaling_layer = ScalingLayer()
self.chns = [64, 128, 256, 512, 512] # vg16 features
- self.net = vgg16(pretrained=True, requires_grad=False) # NOTE: TODO: need in_channels = 4 to use
+ self.net = vgg16(pretrained=True, requires_grad=False) # NOTE: TODO: need in_channels = 4 to use
self.lin0 = NetLinLayer(self.chns[0], use_dropout=use_dropout)
self.lin1 = NetLinLayer(self.chns[1], use_dropout=use_dropout)
self.lin2 = NetLinLayer(self.chns[2], use_dropout=use_dropout)
@@ -63,7 +61,7 @@ class LPIPS(nn.Module):
param.requires_grad = False
def load_from_pretrained(self, name="vgg_lpips"):
- ckpt = get_ckpt_path(name, "taming/modules/autoencoder/lpips")
+ ckpt = get_ckpt_path(name, "taming/modules/autoencoder/lpips", root="pretrained_models")
self.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False)
# print("loaded pretrained LPIPS loss from {}".format(ckpt))
@@ -92,23 +90,32 @@ class LPIPS(nn.Module):
return val
-# SCH: TODO: this channel shift & scale may need to be changed
+# SCH: TODO: this channel shift & scale may need to be changed
class ScalingLayer(nn.Module):
def __init__(self):
super(ScalingLayer, self).__init__()
- self.register_buffer('shift', torch.Tensor([-.030, -.088, -.188])[None, :, None, None])
- self.register_buffer('scale', torch.Tensor([.458, .448, .450])[None, :, None, None])
+ self.register_buffer("shift", torch.Tensor([-0.030, -0.088, -0.188])[None, :, None, None])
+ self.register_buffer("scale", torch.Tensor([0.458, 0.448, 0.450])[None, :, None, None])
def forward(self, inp):
return (inp - self.shift) / self.scale
class NetLinLayer(nn.Module):
- """ A single linear layer which does a 1x1 conv """
+ """A single linear layer which does a 1x1 conv"""
+
def __init__(self, chn_in, chn_out=1, use_dropout=False):
super(NetLinLayer, self).__init__()
- layers = [nn.Dropout(), ] if (use_dropout) else []
- layers += [nn.Conv2d(chn_in, chn_out, 1, stride=1, padding=0, bias=False), ]
+ layers = (
+ [
+ nn.Dropout(),
+ ]
+ if (use_dropout)
+ else []
+ )
+ layers += [
+ nn.Conv2d(chn_in, chn_out, 1, stride=1, padding=0, bias=False),
+ ]
self.model = nn.Sequential(*layers)
@@ -147,15 +154,15 @@ class vgg16(torch.nn.Module):
h_relu4_3 = h
h = self.slice5(h)
h_relu5_3 = h
- vgg_outputs = namedtuple("VggOutputs", ['relu1_2', 'relu2_2', 'relu3_3', 'relu4_3', 'relu5_3'])
+ vgg_outputs = namedtuple("VggOutputs", ["relu1_2", "relu2_2", "relu3_3", "relu4_3", "relu5_3"])
out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3, h_relu5_3)
return out
-def normalize_tensor(x,eps=1e-10):
- norm_factor = torch.sqrt(torch.sum(x**2,dim=1,keepdim=True))
- return x/(norm_factor+eps)
+def normalize_tensor(x, eps=1e-10):
+ norm_factor = torch.sqrt(torch.sum(x**2, dim=1, keepdim=True))
+ return x / (norm_factor + eps)
def spatial_average(x, keepdim=True):
- return x.mean([2,3],keepdim=keepdim)
\ No newline at end of file
+ return x.mean([2, 3], keepdim=keepdim)
diff --git a/opensora/models/vae/vae_3d.py b/opensora/models/vae/vae_3d.py
index d4a47ad..db0db61 100644
--- a/opensora/models/vae/vae_3d.py
+++ b/opensora/models/vae/vae_3d.py
@@ -1,15 +1,13 @@
import functools
-import math
from typing import Tuple, Union
-import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
-from einops import pack, rearrange
+from einops import rearrange
from opensora.registry import MODELS
-from opensora.utils.ckpt_utils import find_model, load_checkpoint
+from opensora.utils.ckpt_utils import load_checkpoint
from .utils import DiagonalGaussianDistribution
@@ -29,7 +27,7 @@ def is_odd(n):
def pad_at_dim(t, pad, dim=-1, value=0.0):
dims_from_right = (-dim - 1) if dim < 0 else (t.ndim - dim - 1)
zeros = (0, 0) * dims_from_right
- return F.pad(t, (*zeros, *pad), value=value)
+ return F.pad(t, (*zeros, *pad), mode="replicate")
def pick_video_frame(video, frame_indices):
@@ -47,29 +45,6 @@ def exists(v):
return v is not None
-def xavier_uniform_weight_init(m):
- if isinstance(m, nn.Conv3d) or isinstance(m, nn.Linear):
- nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain("relu"))
- if m.bias is not None:
- nn.init.zeros_(m.bias)
- # print("initialized module to xavier_uniform:", m)
-
-
-def Sequential(*modules):
- modules = [*filter(exists, modules)]
-
- if len(modules) == 0:
- return nn.Identity()
-
- return nn.Sequential(*modules)
-
-
-def SameConv2d(dim_in, dim_out, kernel_size):
- kernel_size = cast_tuple(kernel_size, 2)
- padding = [k // 2 for k in kernel_size]
- return nn.Conv2d(dim_in, dim_out, kernel_size=kernel_size, padding=padding)
-
-
class CausalConv3d(nn.Module):
def __init__(
self,
@@ -99,17 +74,14 @@ class CausalConv3d(nn.Module):
self.time_causal_padding = (width_pad, width_pad, height_pad, height_pad, time_pad, 0)
stride = strides if strides is not None else (stride, 1, 1)
- # padding = kwargs.pop('padding', 0)
- # if padding == "same" and not all([pad == 1 for pad in padding]):
- # padding = "valid"
dilation = (dilation, 1, 1)
self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)
def forward(self, x):
- pad_mode = self.pad_mode if self.time_pad < x.shape[2] else "constant"
-
+ pad_mode = "replicate"
x = F.pad(x, self.time_causal_padding, mode=pad_mode)
- return self.conv(x)
+ x = self.conv(x)
+ return x
class ResBlock(nn.Module):
@@ -156,302 +128,14 @@ class ResBlock(nn.Module):
return x + residual
-# SCH: own implementation modified on top of: discriminator with anti-aliased downsampling (blurpool Zhang et al.)
-class BlurPool3D(nn.Module):
- def __init__(
- self,
- channels,
- pad_type="reflect",
- filt_size=3,
- stride=2,
- pad_off=0,
- device="cpu",
- dtype=torch.bfloat16,
- ):
- super(BlurPool3D, self).__init__()
- self.filt_size = filt_size
- self.pad_off = pad_off
- self.pad_sizes = [
- int(1.0 * (filt_size - 1) / 2),
- int(np.ceil(1.0 * (filt_size - 1) / 2)),
- int(1.0 * (filt_size - 1) / 2),
- int(np.ceil(1.0 * (filt_size - 1) / 2)),
- int(1.0 * (filt_size - 1) / 2),
- int(np.ceil(1.0 * (filt_size - 1) / 2)),
- ]
- self.pad_sizes = [pad_size + pad_off for pad_size in self.pad_sizes]
- self.stride = stride
- self.off = int((self.stride - 1) / 2.0)
- self.channels = channels
-
- if self.filt_size == 1:
- a = np.array(
- [
- 1.0,
- ]
- )
- elif self.filt_size == 2:
- a = np.array([1.0, 1.0])
- elif self.filt_size == 3:
- a = np.array([1.0, 2.0, 1.0])
- elif self.filt_size == 4:
- a = np.array([1.0, 3.0, 3.0, 1.0])
- elif self.filt_size == 5:
- a = np.array([1.0, 4.0, 6.0, 4.0, 1.0])
- elif self.filt_size == 6:
- a = np.array([1.0, 5.0, 10.0, 10.0, 5.0, 1.0])
- elif self.filt_size == 7:
- a = np.array([1.0, 6.0, 15.0, 20.0, 15.0, 6.0, 1.0])
-
- filt_2d = a[:, None] * a[None, :]
- filt_3d = torch.Tensor(a[:, None, None] * filt_2d[None, :, :]).to(device, dtype)
-
- filt = filt_3d / torch.sum(filt_3d) # SCH: modified to it 3D
- self.register_buffer("filt", filt[None, None, :, :, :].repeat((self.channels, 1, 1, 1, 1)))
-
- self.pad = get_pad_layer(pad_type)(self.pad_sizes)
-
- def forward(self, inp):
- if self.filt_size == 1:
- if self.pad_off == 0:
- return inp[:, :, :: self.stride, :: self.stride]
- else:
- return self.pad(inp)[:, :, :: self.stride, :: self.stride]
- else:
- return F.conv3d(self.pad(inp), self.filt, stride=self.stride, groups=inp.shape[1])
-
-
-def get_pad_layer(pad_type):
- if pad_type in ["refl", "reflect"]:
- PadLayer = nn.ReflectionPad3d
- elif pad_type in ["repl", "replicate"]:
- PadLayer = nn.ReplicationPad3d
- elif pad_type == "zero":
- PadLayer = nn.ZeroPad3d
+def get_activation_fn(activation):
+ if activation == "relu":
+ activation_fn = nn.ReLU
+ elif activation == "swish":
+ activation_fn = nn.SiLU
else:
- print("Pad type [%s] not recognized" % pad_type)
- return PadLayer
-
-
-class ResBlockDown(nn.Module):
- """3D StyleGAN ResBlock for D."""
-
- def __init__(
- self,
- in_channels,
- filters,
- activation_fn,
- num_groups=32,
- device="cpu",
- dtype=torch.bfloat16,
- ):
- super().__init__()
-
- self.filters = filters
- self.activation_fn = activation_fn
-
- # SCH: NOTE: although paper says conv (X->Y, Y->Y), original code implementation is (X->X, X->Y), we follow code
- self.conv1 = nn.Conv3d(
- in_channels, in_channels, (3, 3, 3), padding=1, device=device, dtype=dtype
- ) # NOTE: init to xavier_uniform
- self.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)
-
- self.blur = BlurPool3D(in_channels, device=device, dtype=dtype)
-
- self.conv2 = nn.Conv3d(
- in_channels, self.filters, (1, 1, 1), bias=False, device=device, dtype=dtype
- ) # NOTE: init to xavier_uniform
- self.conv3 = nn.Conv3d(
- in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype
- ) # NOTE: init to xavier_uniform
- self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)
-
- # self.apply(xavier_uniform_weight_init)
-
- def forward(self, x):
- residual = x
- x = self.conv1(x)
- x = self.norm1(x)
- x = self.activation_fn(x)
-
- residual = self.blur(residual)
- residual = self.conv2(residual)
-
- x = self.blur(x)
- x = self.conv3(x)
- x = self.norm2(x)
- x = self.activation_fn(x)
- out = (residual + x) / math.sqrt(2)
- return out
-
-
-# SCH: taken from Open Sora Plan
-def n_layer_disc_weights_init(m):
- classname = m.__class__.__name__
- if classname.find("Conv") != -1:
- nn.init.normal_(m.weight.data, 0.0, 0.02)
- elif classname.find("BatchNorm") != -1:
- nn.init.normal_(m.weight.data, 1.0, 0.02)
- nn.init.constant_(m.bias.data, 0)
-
-
-class NLayerDiscriminator3D(nn.Module):
- """Defines a 3D PatchGAN discriminator as in Pix2Pix but for 3D inputs."""
-
- def __init__(self, input_nc=1, ndf=64, n_layers=3, use_actnorm=False):
- """
- Construct a 3D PatchGAN discriminator
-
- Parameters:
- input_nc (int) -- the number of channels in input volumes
- ndf (int) -- the number of filters in the last conv layer
- n_layers (int) -- the number of conv layers in the discriminator
- use_actnorm (bool) -- flag to use actnorm instead of batchnorm
- """
- super(NLayerDiscriminator3D, self).__init__()
- if not use_actnorm:
- norm_layer = nn.BatchNorm3d
- else:
- raise NotImplementedError("Not implemented.")
- if type(norm_layer) == functools.partial:
- use_bias = norm_layer.func != nn.BatchNorm3d
- else:
- use_bias = norm_layer != nn.BatchNorm3d
-
- kw = 4
- padw = 1
- sequence = [nn.Conv3d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]
- nf_mult = 1
- nf_mult_prev = 1
- for n in range(1, n_layers): # gradually increase the number of filters
- nf_mult_prev = nf_mult
- nf_mult = min(2**n, 8)
- sequence += [
- nn.Conv3d(
- ndf * nf_mult_prev,
- ndf * nf_mult,
- kernel_size=(kw, kw, kw),
- stride=(1, 2, 2),
- padding=padw,
- bias=use_bias,
- ),
- norm_layer(ndf * nf_mult),
- nn.LeakyReLU(0.2, True),
- ]
-
- nf_mult_prev = nf_mult
- nf_mult = min(2**n_layers, 8)
- sequence += [
- nn.Conv3d(
- ndf * nf_mult_prev, ndf * nf_mult, kernel_size=(kw, kw, kw), stride=1, padding=padw, bias=use_bias
- ),
- norm_layer(ndf * nf_mult),
- nn.LeakyReLU(0.2, True),
- ]
-
- sequence += [
- nn.Conv3d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)
- ] # output 1 channel prediction map
- self.main = nn.Sequential(*sequence)
-
- def forward(self, input):
- """Standard forward."""
- return self.main(input)
-
-
-class StyleGANDiscriminatorBlur(nn.Module):
- """StyleGAN Discriminator."""
-
- """
- SCH: NOTE:
- this discriminator requries the num_frames to be fixed during training;
- in case we pre-train with image then train on video, this disciminator's Linear layer would have to be re-trained!
- """
-
- def __init__(
- self,
- image_size=(128, 128),
- num_frames=17,
- in_channels=3,
- filters=128,
- channel_multipliers=(2, 4, 4, 4, 4),
- num_groups=32,
- dtype=torch.bfloat16,
- device="cpu",
- ):
- super().__init__()
-
- self.dtype = dtype
- self.input_size = cast_tuple(image_size, 2)
- self.filters = filters
- self.activation_fn = nn.LeakyReLU(negative_slope=0.2)
- self.channel_multipliers = channel_multipliers
-
- self.conv1 = nn.Conv3d(
- in_channels, self.filters, (3, 3, 3), padding=1, device=device, dtype=dtype
- ) # NOTE: init to xavier_uniform
-
- prev_filters = self.filters # record in_channels
- self.num_blocks = len(self.channel_multipliers)
- self.res_block_list = nn.ModuleList([])
- for i in range(self.num_blocks):
- filters = self.filters * self.channel_multipliers[i]
- self.res_block_list.append(
- ResBlockDown(prev_filters, filters, self.activation_fn, device=device, dtype=dtype).apply(
- xavier_uniform_weight_init
- )
- )
- prev_filters = filters # update in_channels
-
- self.conv2 = nn.Conv3d(
- prev_filters, prev_filters, (3, 3, 3), padding=1, device=device, dtype=dtype
- ) # NOTE: init to xavier_uniform
- # torch.nn.init.xavier_uniform_(self.conv2.weight)
-
- self.norm1 = nn.GroupNorm(num_groups, prev_filters, dtype=dtype, device=device)
-
- scale_factor = 2**self.num_blocks
- if num_frames % scale_factor != 0: # SCH: NOTE: has first frame which would be padded before usage
- time_scaled = num_frames // scale_factor + 1
- else:
- time_scaled = num_frames / scale_factor
-
- assert (
- self.input_size[0] % scale_factor == 0
- ), f"image width {self.input_size[0]} is not divisible by scale factor {scale_factor}"
- assert (
- self.input_size[1] % scale_factor == 0
- ), f"image height {self.input_size[1]} is not divisible by scale factor {scale_factor}"
- w_scaled, h_scaled = self.input_size[0] / scale_factor, self.input_size[1] / scale_factor
- in_features = int(prev_filters * time_scaled * w_scaled * h_scaled) # (C*T*W*H)
- self.linear1 = nn.Linear(in_features, prev_filters, device=device, dtype=dtype) # NOTE: init to xavier_uniform
- self.linear2 = nn.Linear(prev_filters, 1, device=device, dtype=dtype) # NOTE: init to xavier_uniform
-
- # self.apply(xavier_uniform_weight_init)
-
- def forward(self, x):
- x = self.conv1(x)
- # print("discriminator aft conv:", x.size())
- x = self.activation_fn(x)
-
- for i in range(self.num_blocks):
- x = self.res_block_list[i](x)
- # print("discriminator resblock down:", x.size())
-
- x = self.conv2(x)
- # print("discriminator aft conv2:", x.size())
- x = self.norm1(x)
- x = self.activation_fn(x)
- x = x.reshape((x.shape[0], -1)) # SCH: [B, (C * T * W * H)] ?
-
- # print("discriminator reshape:", x.size())
- x = self.linear1(x)
- # print("discriminator aft linear1:", x.size())
-
- x = self.activation_fn(x)
- x = self.linear2(x)
- # print("discriminator aft linear2:", x.size())
- return x
+ raise NotImplementedError
+ return activation_fn
class Encoder(nn.Module):
@@ -459,16 +143,13 @@ class Encoder(nn.Module):
def __init__(
self,
+ in_out_channels=4,
+ latent_embed_dim=512, # num channels for latent vector
filters=128,
num_res_blocks=4,
channel_multipliers=(1, 2, 2, 4),
temporal_downsample=(False, True, True),
num_groups=32, # for nn.GroupNorm
- # in_out_channels = 3, # SCH: added, in_channels at the start
- latent_embed_dim=512, # num channels for latent vector
- # conv_downsample = False,
- disable_spatial_downsample=False, # for vae pipeline
- custom_conv_padding=None,
activation_fn="swish",
device="cpu",
dtype=torch.bfloat16,
@@ -476,30 +157,19 @@ class Encoder(nn.Module):
super().__init__()
self.filters = filters
self.num_res_blocks = num_res_blocks
+ self.num_blocks = len(channel_multipliers)
self.channel_multipliers = channel_multipliers
self.temporal_downsample = temporal_downsample
self.num_groups = num_groups
-
self.embedding_dim = latent_embed_dim
- self.disable_spatial_downsample = disable_spatial_downsample
- # self.conv_downsample = conv_downsample
- self.custom_conv_padding = custom_conv_padding
-
- if activation_fn == "relu":
- self.activation_fn = nn.ReLU
- elif activation_fn == "swish":
- self.activation_fn = nn.SiLU
- else:
- raise NotImplementedError
- self.activate = self.activation_fn()
+ self.activation_fn = get_activation_fn(activation_fn)
+ self.activate = self.activation_fn()
self.conv_fn = functools.partial(
CausalConv3d,
- # padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorch
dtype=dtype,
device=device,
)
-
self.block_args = dict(
conv_fn=self.conv_fn,
dtype=dtype,
@@ -509,40 +179,44 @@ class Encoder(nn.Module):
device=device,
)
- # NOTE: moved to VAE for separate first frame processing
- # self.conv1 = self.conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)
+ # first layer conv
+ self.conv_in = self.conv_fn(
+ in_out_channels,
+ filters,
+ kernel_size=(3, 3, 3),
+ bias=False,
+ dtype=dtype,
+ device=device,
+ )
# ResBlocks and conv downsample
self.block_res_blocks = nn.ModuleList([])
- self.num_blocks = len(self.channel_multipliers)
self.conv_blocks = nn.ModuleList([])
filters = self.filters
prev_filters = filters # record for in_channels
for i in range(self.num_blocks):
- # resblock handling
- filters = self.filters * self.channel_multipliers[i] # SCH: determine the number out_channels
+ filters = self.filters * self.channel_multipliers[i]
block_items = nn.ModuleList([])
for _ in range(self.num_res_blocks):
block_items.append(ResBlock(prev_filters, filters, **self.block_args))
prev_filters = filters # update in_channels
self.block_res_blocks.append(block_items)
- if i < self.num_blocks - 1: # SCH: T-Causal Conv 3x3x3, 128->128, stride t x stride s x stride s
- if self.temporal_downsample[i]:
+ if i < self.num_blocks - 1:
+ if self.temporal_downsample[i]:
t_stride = 2 if self.temporal_downsample[i] else 1
- s_stride = 2 if not self.disable_spatial_downsample else 1
+ s_stride = 1
self.conv_blocks.append(
- self.conv_fn(prev_filters, filters, kernel_size=(3, 3, 3), strides=(t_stride, s_stride, s_stride))
- ) # SCH: should be same in_channel and out_channel
- prev_filters = filters # update in_channels
- else: # if no t downsample, don't add since this does nothing for pipeline models
- self.conv_blocks.append( # Identity
- nn.Identity(prev_filters)
+ self.conv_fn(
+ prev_filters, filters, kernel_size=(3, 3, 3), strides=(t_stride, s_stride, s_stride)
+ )
)
prev_filters = filters # update in_channels
-
-
+ else:
+ # if no t downsample, don't add since this does nothing for pipeline models
+ self.conv_blocks.append(nn.Identity(prev_filters)) # Identity
+ prev_filters = filters # update in_channels
# last layer res block
self.res_blocks = nn.ModuleList([])
@@ -553,40 +227,26 @@ class Encoder(nn.Module):
# MAGVIT uses Group Normalization
self.norm1 = nn.GroupNorm(
self.num_groups, prev_filters, dtype=dtype, device=device
- ) # SCH: separate <prev_filters> channels into 32 groups
+ ) # separate <prev_filters> channels into 32 groups
- self.conv2 = nn.Conv3d(
+ self.conv2 = self.conv_fn(
prev_filters, self.embedding_dim, kernel_size=(1, 1, 1), dtype=dtype, device=device, padding="same"
)
def forward(self, x):
- # dtype, device = x.dtype, x.device
-
- # NOTE: moved to VAE for separate first frame processing
- # x = self.conv1(x)
-
- # print("encoder:", x.size())
-
- try:
- for i in range(self.num_blocks):
- for j in range(self.num_res_blocks):
- x = self.block_res_blocks[i][j](x)
- # print("encoder:", x.size())
+ x = self.conv_in(x)
- if i < self.num_blocks - 1:
- x = self.conv_blocks[i](x)
- # print("encoder:", x.size())
-
- for i in range(self.num_res_blocks):
- x = self.res_blocks[i](x)
- # print("encoder:", x.size())
- except:
- breakpoint()
+ for i in range(self.num_blocks):
+ for j in range(self.num_res_blocks):
+ x = self.block_res_blocks[i][j](x)
+ if i < self.num_blocks - 1:
+ x = self.conv_blocks[i](x)
+ for i in range(self.num_res_blocks):
+ x = self.res_blocks[i](x)
x = self.norm1(x)
x = self.activate(x)
x = self.conv2(x)
- # print("encoder:", x.size())
return x
@@ -595,50 +255,34 @@ class Decoder(nn.Module):
def __init__(
self,
+ in_out_channels=4,
latent_embed_dim=512,
filters=128,
- # in_out_channels = 4,
num_res_blocks=4,
channel_multipliers=(1, 2, 2, 4),
temporal_downsample=(False, True, True),
num_groups=32, # for nn.GroupNorm
- # upsample = "nearest+conv", # options: "deconv", "nearest+conv"
- disable_spatial_upsample=False, # for vae pipeline
- custom_conv_padding=None,
activation_fn="swish",
device="cpu",
dtype=torch.bfloat16,
):
super().__init__()
- # self.output_dim = in_out_channels
- self.embedding_dim = latent_embed_dim
self.filters = filters
self.num_res_blocks = num_res_blocks
+ self.num_blocks = len(channel_multipliers)
self.channel_multipliers = channel_multipliers
self.temporal_downsample = temporal_downsample
self.num_groups = num_groups
+ self.embedding_dim = latent_embed_dim
+ self.s_stride = 1
- # self.upsample = upsample
- self.s_stride = 1 if disable_spatial_upsample else 2 # spatial stride
- self.custom_conv_padding = custom_conv_padding
- # self.norm_type = self.config.vqvae.norm_type
- # self.num_remat_block = self.config.vqvae.get('num_dec_remat_blocks', 0)
-
- if activation_fn == "relu":
- self.activation_fn = nn.ReLU
- elif activation_fn == "swish":
- self.activation_fn = nn.SiLU
- else:
- raise NotImplementedError
+ self.activation_fn = get_activation_fn(activation_fn)
self.activate = self.activation_fn()
-
self.conv_fn = functools.partial(
CausalConv3d,
dtype=dtype,
- # padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorch
device=device,
)
-
self.block_args = dict(
conv_fn=self.conv_fn,
activation_fn=self.activation_fn,
@@ -647,10 +291,11 @@ class Decoder(nn.Module):
device=device,
dtype=dtype,
)
- self.num_blocks = len(self.channel_multipliers)
filters = self.filters * self.channel_multipliers[-1]
+ prev_filters = filters
+ # last conv
self.conv1 = self.conv_fn(self.embedding_dim, filters, kernel_size=(3, 3, 3), bias=True)
# last layer res block
@@ -658,18 +303,11 @@ class Decoder(nn.Module):
for _ in range(self.num_res_blocks):
self.res_blocks.append(ResBlock(filters, filters, **self.block_args))
- # TODO: do I need to add adaptive GroupNorm in between each block?
-
- # # NOTE: upsample, dimensions T, H, W
- # self.upsampler_with_t = nn.Upsample(scale_factor=(2,2,2))
- # self.upsampler = nn.Upsample(scale_factor=(1,2,2))
-
# ResBlocks and conv upsample
- prev_filters = filters # SCH: in_channels
self.block_res_blocks = nn.ModuleList([])
self.num_blocks = len(self.channel_multipliers)
self.conv_blocks = nn.ModuleList([])
- # SCH: reverse to keep track of the in_channels, but append also in a reverse direction
+ # reverse to keep track of the in_channels, but append also in a reverse direction
for i in reversed(range(self.num_blocks)):
filters = self.filters * self.channel_multipliers[i]
# resblock handling
@@ -681,7 +319,7 @@ class Decoder(nn.Module):
# conv blocks with upsampling
if i > 0:
- if self.temporal_downsample[i]:
+ if self.temporal_downsample[i - 1]:
t_stride = 2 if self.temporal_downsample[i - 1] else 1
# SCH: T-Causal Conv 3x3x3, f -> (t_stride * 2 * 2) * f, depth to space t_stride x 2 x 2
self.conv_blocks.insert(
@@ -693,36 +331,22 @@ class Decoder(nn.Module):
else:
self.conv_blocks.insert(
0,
- nn.Identity(
- prev_filters
- ),
+ nn.Identity(prev_filters),
)
-
self.norm1 = nn.GroupNorm(self.num_groups, prev_filters, device=device, dtype=dtype)
- # NOTE: moved to VAE for separate first frame processing
- # self.conv2 = self.conv_fn(prev_filters, self.output_dim, kernel_size=(3, 3, 3))
-
- def forward(
- self,
- x,
- **kwargs,
- ):
- # dtype, device = x.dtype, x.device
+ self.conv_out = self.conv_fn(filters, in_out_channels, 3, dtype=dtype, device=device)
+ def forward(self, x):
x = self.conv1(x)
- # print("decoder:", x.size())
for i in range(self.num_res_blocks):
x = self.res_blocks[i](x)
- # print("decoder:", x.size())
- for i in reversed(range(self.num_blocks)): # reverse here to make decoder symmetric with encoder
+ for i in reversed(range(self.num_blocks)):
for j in range(self.num_res_blocks):
x = self.block_res_blocks[i][j](x)
- # print("decoder:", x.size())
if i > 0:
t_stride = 2 if self.temporal_downsample[i - 1] else 1
- # SCH: T-Causal Conv 3x3x3, f -> (t_stride * 2 * 2) * f, depth to space t_stride x 2 x 2
x = self.conv_blocks[i - 1](x)
x = rearrange(
x,
@@ -731,278 +355,105 @@ class Decoder(nn.Module):
hs=self.s_stride,
ws=self.s_stride,
)
- # print("decoder:", x.size())
x = self.norm1(x)
x = self.activate(x)
- # NOTE: moved to VAE for separate first frame processing
- # x = self.conv2(x)
+ x = self.conv_out(x)
return x
@MODELS.register_module()
-class VAE_3D_V2(nn.Module): # , ModelMixin
- """The 3D VAE"""
-
+class VAE_Temporal(nn.Module):
def __init__(
self,
- latent_embed_dim=256,
+ in_out_channels=4,
+ latent_embed_dim=4,
+ embed_dim=4,
filters=128,
- num_res_blocks=2,
- separate_first_frame_encoding=False,
+ num_res_blocks=4,
channel_multipliers=(1, 2, 2, 4),
temporal_downsample=(True, True, False),
num_groups=32, # for nn.GroupNorm
- disable_space=False,
- custom_conv_padding=None,
activation_fn="swish",
- in_out_channels=4,
- kl_embed_dim=64,
- encoder_double_z=True,
device="cpu",
- dtype="bf16",
+ dtype=torch.bfloat16,
):
super().__init__()
- if type(dtype) == str:
- if dtype == "bf16":
- dtype = torch.bfloat16
- elif dtype == "fp16":
- dtype = torch.float16
- else:
- raise NotImplementedError(f"dtype: {dtype}")
-
- # ==== Model Params ====
- # self.image_size = cast_tuple(image_size, 2)
self.time_downsample_factor = 2 ** sum(temporal_downsample)
self.time_padding = self.time_downsample_factor - 1
- self.separate_first_frame_encoding = separate_first_frame_encoding
-
- image_down = 2 ** len(temporal_downsample)
- t_down = 2 ** len([x for x in temporal_downsample if x == True])
- self.patch_size = (t_down, image_down, image_down)
-
- # ==== Model Initialization ====
-
- # encoder & decoder first and last conv layer
- # SCH: NOTE: following MAGVIT, conv in bias=False in encoder first conv
- self.conv_in = CausalConv3d(
- in_out_channels, filters, kernel_size=(3, 3, 3), bias=False, dtype=dtype, device=device
- )
- self.conv_in_first_frame = nn.Identity()
- self.conv_out_first_frame = nn.Identity()
- if separate_first_frame_encoding:
- self.conv_in_first_frame = SameConv2d(in_out_channels, filters, (3, 3))
- self.conv_out_first_frame = SameConv2d(filters, in_out_channels, (3, 3))
- self.conv_out = CausalConv3d(filters, in_out_channels, 3, dtype=dtype, device=device)
+ self.patch_size = (self.time_downsample_factor, 1, 1)
+ # NOTE: following MAGVIT, conv in bias=False in encoder first conv
self.encoder = Encoder(
+ in_out_channels=in_out_channels,
+ latent_embed_dim=latent_embed_dim * 2,
filters=filters,
num_res_blocks=num_res_blocks,
channel_multipliers=channel_multipliers,
temporal_downsample=temporal_downsample,
num_groups=num_groups, # for nn.GroupNorm
- # in_out_channels = in_out_channels,
- latent_embed_dim=latent_embed_dim * 2 if encoder_double_z else latent_embed_dim,
- # conv_downsample = conv_downsample,
- disable_spatial_downsample=disable_space,
- custom_conv_padding=custom_conv_padding,
activation_fn=activation_fn,
device=device,
dtype=dtype,
)
+ self.quant_conv = CausalConv3d(2 * latent_embed_dim, 2 * embed_dim, 1, device=device, dtype=dtype)
+
+ self.post_quant_conv = CausalConv3d(embed_dim, latent_embed_dim, 1, device=device, dtype=dtype)
self.decoder = Decoder(
+ in_out_channels=in_out_channels,
latent_embed_dim=latent_embed_dim,
filters=filters,
- # in_out_channels = in_out_channels,
num_res_blocks=num_res_blocks,
channel_multipliers=channel_multipliers,
temporal_downsample=temporal_downsample,
num_groups=num_groups, # for nn.GroupNorm
- # upsample = upsample, # options: "deconv", "nearest+conv"
- disable_spatial_upsample=disable_space,
- custom_conv_padding=custom_conv_padding,
activation_fn=activation_fn,
device=device,
dtype=dtype,
)
- if encoder_double_z:
- self.quant_conv = nn.Conv3d(2 * latent_embed_dim, 2 * kl_embed_dim, 1, device=device, dtype=dtype)
- else:
- self.quant_conv = nn.Conv3d(latent_embed_dim, 2 * kl_embed_dim, 1, device=device, dtype=dtype)
- self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1, device=device, dtype=dtype)
-
def get_latent_size(self, input_size):
for i in range(len(input_size)):
assert input_size[i] % self.patch_size[i] == 0, "Input size must be divisible by patch size"
input_size = [input_size[i] // self.patch_size[i] for i in range(3)]
return input_size
- def encode(
- self,
- video,
- video_contains_first_frame=True,
- ):
- self.separate_first_frame_encoding and video_contains_first_frame
-
- # whether to pad video or not
- # if video_contains_first_frame:
- # video_len = video.shape[2]
- # video = pad_at_dim(video, (self.time_padding, 0), value=0.0, dim=2)
- # video_packed_shape = [torch.Size([self.time_padding]), torch.Size([]), torch.Size([video_len - 1])]
-
- # print("pre-encoder:", video.size())
-
- # NOTE: moved encoder conv1 here for separate first frame encoding
- # if encode_first_frame_separately:
- # pad, first_frame, video = unpack(video, video_packed_shape, "b c * h w")
- # first_frame = self.conv_in_first_frame(first_frame)
- video = self.conv_in(video)
-
- # print("pre-encoder:", video.size())
-
- # if encode_first_frame_separately:
- # video, _ = pack([first_frame, video], "b c * h w")
- # video = pad_at_dim(video, (self.time_padding, 0), dim=2)
-
- encoded_feature = self.encoder(video)
-
- # print("after encoder:", encoded_feature.size())
-
- # NOTE: TODO: do we include this before gaussian distri? or go directly to Gaussian distribution
- moments = self.quant_conv(encoded_feature).to(video.dtype)
+ def encode(self, x):
+ x = pad_at_dim(x, (self.time_padding, 0), dim=2)
+ encoded_feature = self.encoder(x)
+ moments = self.quant_conv(encoded_feature).to(x.dtype)
posterior = DiagonalGaussianDistribution(moments)
-
- # print("after encoder moments:", moments.size())
-
return posterior
- def decode(
- self,
- z,
- video_contains_first_frame=True,
- ):
- # dtype = z.dtype
- decode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame
-
+ def decode(self, z):
z = self.post_quant_conv(z)
- # print("pre decoder, post quant conv:", z.size())
-
- dec = self.decoder(z)
- # print("post decoder:", dec.size())
-
- # SCH: moved decoder last conv layer here for separate first frame decoding
- if decode_first_frame_separately:
- left_pad, dec_ff, dec = (
- dec[:, :, : self.time_padding],
- dec[:, :, self.time_padding],
- dec[:, :, (self.time_padding + 1) :],
- )
- out = self.conv_out(dec)
- outff = self.conv_out_first_frame(dec_ff)
- video, _ = pack([outff, out], "b c * h w")
- else:
- video = self.conv_out(dec)
- # if video were padded, remove padding
- if video_contains_first_frame:
- video = video[:, :, self.time_padding :]
-
- # print("conv out:", video.size())
-
- return video
-
- def get_last_layer(self):
- # CausalConv3d wraps the conv
- return self.conv_out.conv.weight
-
- def forward(
- self,
- video,
- sample_posterior=True,
- video_contains_first_frame=True,
- # split = "train",
- ):
- batch, channels, frames = video.shape[:3]
- # assert divisible_by(
- # frames - int(video_contains_first_frame), self.time_downsample_factor
- # ), f"number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}"
-
- posterior = self.encode(
- video,
- video_contains_first_frame=video_contains_first_frame,
- )
+ x = self.decoder(z)
+ x = x[:, :, self.time_padding :]
+ return x
+ def forward(self, x, sample_posterior=True):
+ posterior = self.encode(x)
if sample_posterior:
z = posterior.sample()
else:
z = posterior.mode()
+ recon_video = self.decode(z)
+ return recon_video, posterior, z
- recon_video = self.decode(z, video_contains_first_frame=video_contains_first_frame)
-
- return recon_video, posterior
-
-
[email protected]_module("VAE_MAGVIT_V2")
-def VAE_MAGVIT_V2(from_pretrained=None, **kwargs):
- model = VAE_3D_V2(**kwargs)
- if from_pretrained is not None:
- load_checkpoint(model, from_pretrained, model_name="model")
- return model
-
-
[email protected]_module("DISCRIMINATOR_3D")
-def DISCRIMINATOR_3D(from_pretrained=None, inflate_from_2d=False, use_pretrained=True, **kwargs):
- model = StyleGANDiscriminatorBlur(**kwargs).apply(xavier_uniform_weight_init)
- if from_pretrained is not None:
- if use_pretrained:
- if inflate_from_2d:
- load_checkpoint_with_inflation(model, from_pretrained)
- else:
- load_checkpoint(model, from_pretrained, model_name="discriminator")
- print(f"loaded discriminator")
- else:
- print(f"discriminator use_pretrained={use_pretrained}, initializing new discriminator")
- return model
-
-
[email protected]_module("N_Layer_DISCRIMINATOR_3D")
-def DISCRIMINATOR_3D(from_pretrained=None, inflate_from_2d=False, use_pretrained=True, **kwargs):
- model = NLayerDiscriminator3D(
- input_nc=3,
- n_layers=3,
- ).apply(n_layer_disc_weights_init)
[email protected]_module("VAE_Temporal_SD")
+def VAE_Temporal_SD(from_pretrained=None, **kwargs):
+ model = VAE_Temporal(
+ in_out_channels=4,
+ latent_embed_dim=4,
+ embed_dim=4,
+ filters=128,
+ num_res_blocks=3,
+ channel_multipliers=(1, 2, 2),
+ temporal_downsample=(True, True),
+ **kwargs,
+ )
if from_pretrained is not None:
- if use_pretrained:
- if inflate_from_2d:
- load_checkpoint_with_inflation(model, from_pretrained)
- else:
- load_checkpoint(model, from_pretrained, model_name="discriminator")
- print(f"loaded discriminator")
- else:
- print(f"discriminator use_pretrained={use_pretrained}, initializing new discriminator")
-
+ load_checkpoint(model, from_pretrained)
return model
-
-
-def load_checkpoint_with_inflation(model, ckpt_path):
- """
- pre-train using image, then inflate to 3D videos
- """
- if ckpt_path.endswith(".pt") or ckpt_path.endswith(".pth"):
- state_dict = find_model(ckpt_path)
- with torch.no_grad():
- for key in state_dict:
- if key in model:
- # central inflation
- if state_dict[key].size() == model[key][:, :, 0, :, :].size():
- # temporal dimension
- val = torch.zeros_like(model[key])
- centre = int(model[key].size(2) // 2)
- val[:, :, centre, :, :] = state_dict[key]
- missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)
- print(f"Missing keys: {missing_keys}")
- print(f"Unexpected keys: {unexpected_keys}")
- else:
- load_checkpoint(model, ckpt_path) # use the default function
diff --git a/scripts/inference-vae.py b/scripts/inference-vae.py
index 63030db..7011807 100644
--- a/scripts/inference-vae.py
+++ b/scripts/inference-vae.py
@@ -9,8 +9,8 @@ from tqdm import tqdm
from opensora.acceleration.parallel_states import get_data_parallel_group
from opensora.datasets import prepare_dataloader, save_sample
-from opensora.models.vae.losses import AdversarialLoss, DiscriminatorLoss, VEALoss
-from opensora.models.vae.vae_3d import LeCamEMA, pad_at_dim
+from opensora.models.vae.losses import AdversarialLoss, DiscriminatorLoss, LeCamEMA, VAELoss
+from opensora.models.vae.vae_3d import pad_at_dim
from opensora.registry import DATASETS, MODELS, build_module
from opensora.utils.config_utils import parse_configs
from opensora.utils.misc import to_torch_dtype
@@ -87,7 +87,7 @@ def main():
# define loss function
if cfg.calc_loss:
- vae_loss_fn = VEALoss(
+ vae_loss_fn = VAELoss(
logvar_init=cfg.logvar_init,
perceptual_loss_weight=cfg.perceptual_loss_weight,
kl_loss_weight=cfg.kl_loss_weight,
diff --git a/scripts/train-vae.py b/scripts/train-vae.py
index aef65ac..5200cba 100644
--- a/scripts/train-vae.py
+++ b/scripts/train-vae.py
@@ -1,18 +1,17 @@
import os
-from glob import glob
+from datetime import timedelta
+from pprint import pprint
-import colossalai
import torch
import torch.distributed as dist
-import wandb
from colossalai.booster import Booster
from colossalai.booster.plugin import LowLevelZeroPlugin
from colossalai.cluster import DistCoordinator
from colossalai.nn.optimizer import HybridAdam
-from colossalai.utils import get_current_device
-from einops import rearrange
+from colossalai.utils import get_current_device, set_seed
from tqdm import tqdm
+import wandb
from opensora.acceleration.checkpoint import set_grad_checkpoint
from opensora.acceleration.parallel_states import (
get_data_parallel_group,
@@ -20,9 +19,8 @@ from opensora.acceleration.parallel_states import (
set_sequence_parallel_group,
)
from opensora.acceleration.plugin import ZeroSeqParallelPlugin
-from opensora.datasets import prepare_dataloader, prepare_variable_dataloader
-from opensora.models.vae.vae_3d import pad_at_dim
-from opensora.models.vae.losses import AdversarialLoss, DiscriminatorLoss, VEALoss, LeCamEMA
+from opensora.datasets import prepare_dataloader
+from opensora.models.vae.losses import AdversarialLoss, DiscriminatorLoss, VAELoss
from opensora.registry import DATASETS, MODELS, build_module
from opensora.utils.ckpt_utils import create_logger, load_json, save_json
from opensora.utils.config_utils import (
@@ -39,49 +37,36 @@ def main():
# 1. args & cfg
# ======================================================
cfg = parse_configs(training=True)
+ exp_name, exp_dir = create_experiment_workspace(cfg)
+ save_training_config(cfg._cfg_dict, exp_dir)
# ======================================================
# 2. runtime variables & colossalai launch
# ======================================================
assert torch.cuda.is_available(), "Training currently requires at least one GPU."
+ assert cfg.dtype in ["fp16", "bf16"], f"Unknown mixed precision {cfg.dtype}"
# 2.1. colossalai init distributed training
- colossalai.launch_from_torch({})
+ # we set a very large timeout to avoid some processes exit early
+ dist.init_process_group(backend="nccl", timeout=timedelta(hours=24))
+ torch.cuda.set_device(dist.get_rank() % torch.cuda.device_count())
+ set_seed(1024)
coordinator = DistCoordinator()
-
- exp_dir = None
- if coordinator.is_master(): # only create directory for master
- exp_name, exp_dir = create_experiment_workspace(cfg)
- save_training_config(cfg._cfg_dict, exp_dir)
- dist.barrier()
-
- # get exp dir for non-master process
- if exp_dir is None:
- # experiment_index = len(glob(f"{cfg.outputs}/*")) - 1
- # model_name = cfg.model["type"].replace("/", "-")
- # exp_name = f"{experiment_index:03d}-{model_name}"
- # exp_dir = f"{cfg.outputs}/{exp_name}"
- exp_name, exp_dir = create_experiment_workspace(cfg, get_last_workspace=True)
- assert os.path.exists(exp_dir)
-
device = get_current_device()
- assert cfg.dtype in ["fp16", "bf16"], f"Unknown mixed precision {cfg.dtype}"
dtype = to_torch_dtype(cfg.dtype)
# 2.2. init logger, tensorboard & wandb
if not coordinator.is_master():
logger = create_logger(None)
else:
- print(cfg)
+ print("Training configuration:")
+ pprint(cfg._cfg_dict)
logger = create_logger(exp_dir)
logger.info(f"Experiment directory created at {exp_dir}")
writer = create_tensorboard_writer(exp_dir)
if cfg.wandb:
- # wandb.init(project="opensora-vae", name=exp_name, config=cfg._cfg_dict)
- # NOTE: here we use the outputs folder name to store running records of different experiments (since frequent interruption)
- name = os.path.basename(os.path.normpath(cfg.outputs))
- wandb.init(project="opensora-vae", name=name, config=cfg._cfg_dict)
+ wandb.init(project="minisora", name=exp_name, config=cfg._cfg_dict)
# 2.3. initialize ColossalAI booster
if cfg.plugin == "zero2":
@@ -109,6 +94,7 @@ def main():
# ======================================================
# 3. build dataset and dataloader
# ======================================================
+ assert cfg.dataset.type == "VideoTextDataset", "Only support VideoTextDataset for now"
dataset = build_module(cfg.dataset, DATASETS)
logger.info(f"Dataset contains {len(dataset)} samples.")
dataloader_args = dict(
@@ -122,111 +108,129 @@ def main():
process_group=get_data_parallel_group(),
)
# TODO: use plugin's prepare dataloader
- if cfg.bucket_config is None:
- dataloader = prepare_dataloader(**dataloader_args)
- else:
- dataloader = prepare_variable_dataloader(
- bucket_config=cfg.bucket_config,
- num_bucket_build_workers=cfg.num_bucket_build_workers,
- **dataloader_args,
- )
- if cfg.dataset.type == "VideoTextDataset":
- total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size
- logger.info(f"Total batch size: {total_batch_size}")
+ dataloader = prepare_dataloader(**dataloader_args)
+ total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size
+ logger.info(f"Total batch size: {total_batch_size}")
# ======================================================
# 4. build model
# ======================================================
# 4.1. build model
-
- if cfg.get("use_pipeline") == True:
- # use 2D VAE, then temporal VAE
+ if cfg.get("vae_2d", None) is not None:
vae_2d = build_module(cfg.vae_2d, MODELS)
+ vae_2d.to(device, dtype).eval()
- vae = build_module(cfg.model, MODELS, device=device)
- vae_numel, vae_numel_trainable = get_model_numel(vae)
- logger.info(
- f"Trainable vae params: {format_numel_str(vae_numel_trainable)}, Total model params: {format_numel_str(vae_numel)}"
+ model = build_module(
+ cfg.model,
+ MODELS,
+ device=device,
+ dtype=dtype,
)
-
- discriminator = build_module(cfg.discriminator, MODELS, device=device)
- discriminator_numel, discriminator_numel_trainable = get_model_numel(discriminator)
+ model_numel, model_numel_trainable = get_model_numel(model)
logger.info(
- f"Trainable discriminator params: {format_numel_str(discriminator_numel_trainable)}, Total model params: {format_numel_str(discriminator_numel)}"
+ f"Trainable model params: {format_numel_str(model_numel_trainable)}, Total model params: {format_numel_str(model_numel)}"
)
- # LeCam Initialization
- lecam_ema = LeCamEMA(decay=cfg.ema_decay, dtype=dtype, device=device)
+ # discriminator = build_module(cfg.discriminator, MODELS, device=device)
+ # discriminator_numel, discriminator_numel_trainable = get_model_numel(discriminator)
+ # logger.info(
+ # f"Trainable discriminator params: {format_numel_str(discriminator_numel_trainable)}, Total model params: {format_numel_str(discriminator_numel)}"
+ # )
+
+ # # LeCam Initialization
+ # lecam_ema = LeCamEMA(decay=cfg.ema_decay, dtype=dtype, device=device)
# 4.3. move to device
- if cfg.get("use_pipeline") == True:
- vae_2d.to(device, dtype).eval() # eval mode, not training!
+ model = model.to(device, dtype)
+ # discriminator = discriminator.to(device, dtype)
+
+ # 4.4 loss functions
+ vae_loss_fn = VAELoss(
+ logvar_init=cfg.logvar_init,
+ perceptual_loss_weight=cfg.perceptual_loss_weight,
+ kl_loss_weight=cfg.kl_loss_weight,
+ device=device,
+ dtype=dtype,
+ )
+
+ adversarial_loss_fn = AdversarialLoss(
+ discriminator_factor=cfg.discriminator_factor,
+ discriminator_start=cfg.discriminator_start,
+ generator_factor=cfg.generator_factor,
+ generator_loss_type=cfg.generator_loss_type,
+ )
- vae = vae.to(device, dtype)
- discriminator = discriminator.to(device, dtype)
+ disc_loss_fn = DiscriminatorLoss(
+ discriminator_factor=cfg.discriminator_factor,
+ discriminator_start=cfg.discriminator_start,
+ discriminator_loss_type=cfg.discriminator_loss_type,
+ lecam_loss_weight=cfg.lecam_loss_weight,
+ gradient_penalty_loss_weight=cfg.gradient_penalty_loss_weight,
+ )
# 4.5. setup optimizer
# vae optimizer
optimizer = HybridAdam(
- filter(lambda p: p.requires_grad, vae.parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True
+ filter(lambda p: p.requires_grad, model.parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True
)
lr_scheduler = None
# disc optimizer
- disc_optimizer = HybridAdam(
- filter(lambda p: p.requires_grad, discriminator.parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True
- )
- disc_lr_scheduler = None
+ # disc_optimizer = HybridAdam(
+ # filter(lambda p: p.requires_grad, discriminator.parameters()), lr=cfg.lr, weight_decay=0, adamw_mode=True
+ # )
+ # disc_lr_scheduler = None
# 4.6. prepare for training
if cfg.grad_checkpoint:
- set_grad_checkpoint(vae)
- set_grad_checkpoint(discriminator)
- vae.train()
- discriminator.train()
+ set_grad_checkpoint(model)
+ # set_grad_checkpoint(discriminator)
+ model.train()
+ # discriminator.train()
# =======================================================
# 5. boost model for distributed training with colossalai
# =======================================================
torch.set_default_dtype(dtype)
- vae, optimizer, _, dataloader, lr_scheduler = booster.boost(
- model=vae, optimizer=optimizer, lr_scheduler=lr_scheduler, dataloader=dataloader
+ model, optimizer, _, dataloader, lr_scheduler = booster.boost(
+ model=model,
+ optimizer=optimizer,
+ lr_scheduler=lr_scheduler,
+ dataloader=dataloader,
)
torch.set_default_dtype(torch.float)
num_steps_per_epoch = len(dataloader)
- logger.info("Boost vae for distributed training")
+ logger.info("Boost model for distributed training")
+ num_steps_per_epoch = len(dataloader)
- discriminator, disc_optimizer, _, _, disc_lr_scheduler = booster.boost(
- model=discriminator, optimizer=disc_optimizer, lr_scheduler=disc_lr_scheduler
- )
- logger.info("Boost discriminator for distributed training")
+ # discriminator, disc_optimizer, _, _, disc_lr_scheduler = booster.boost(
+ # model=discriminator, optimizer=disc_optimizer, lr_scheduler=disc_lr_scheduler
+ # )
+ # logger.info("Boost discriminator for distributed training")
# =======================================================
# 6. training loop
# =======================================================
start_epoch = start_step = log_step = sampler_start_idx = 0
+ acc_step = 0
running_loss = 0.0
running_disc_loss = 0.0
-
# 6.1. resume training
if cfg.load is not None:
logger.info("Loading checkpoint")
- booster.load_model(vae, os.path.join(cfg.load, "model"))
- booster.load_model(discriminator, os.path.join(cfg.load, "discriminator"))
+ booster.load_model(model, os.path.join(cfg.load, "model"))
booster.load_optimizer(optimizer, os.path.join(cfg.load, "optimizer"))
- booster.load_optimizer(disc_optimizer, os.path.join(cfg.load, "disc_optimizer"))
- if lr_scheduler is not None:
- booster.load_lr_scheduler(lr_scheduler, os.path.join(cfg.load, "lr_scheduler"))
- if disc_lr_scheduler is not None:
- booster.load_lr_scheduler(disc_lr_scheduler, os.path.join(cfg.load, "disc_lr_scheduler"))
+
+ # booster.load_model(discriminator, os.path.join(cfg.load, "discriminator"))
+ # booster.load_optimizer(disc_optimizer, os.path.join(cfg.load, "disc_optimizer"))
# LeCam EMA for discriminator
- lecam_path = os.path.join(cfg.load, "lecam_states.json")
- if cfg.lecam_loss_weight is not None and os.path.exists(lecam_path):
- lecam_state = load_json(lecam_path)
- lecam_ema_real, lecam_ema_fake = lecam_state["lecam_ema_real"], lecam_state["lecam_ema_fake"]
- lecam_ema = LeCamEMA(
- decay=cfg.ema_decay, ema_real=lecam_ema_real, ema_fake=lecam_ema_fake, dtype=dtype, device=device
- )
+ # lecam_path = os.path.join(cfg.load, "lecam_states.json")
+ # if cfg.lecam_loss_weight is not None and os.path.exists(lecam_path):
+ # lecam_state = load_json(lecam_path)
+ # lecam_ema_real, lecam_ema_fake = lecam_state["lecam_ema_real"], lecam_state["lecam_ema_fake"]
+ # lecam_ema = LeCamEMA(
+ # decay=cfg.ema_decay, ema_real=lecam_ema_real, ema_fake=lecam_ema_fake, dtype=dtype, device=device
+ # )
running_states = load_json(os.path.join(cfg.load, "running_states.json"))
dist.barrier()
@@ -240,39 +244,15 @@ def main():
dataloader.sampler.set_start_index(sampler_start_idx)
- # 6.2 Define loss functions
- vae_loss_fn = VEALoss(
- logvar_init=cfg.logvar_init,
- perceptual_loss_weight=cfg.perceptual_loss_weight,
- kl_loss_weight=cfg.kl_loss_weight,
- device=device,
- dtype=dtype,
- )
-
- adversarial_loss_fn = AdversarialLoss(
- discriminator_factor=cfg.discriminator_factor,
- discriminator_start=cfg.discriminator_start,
- generator_factor=cfg.generator_factor,
- generator_loss_type=cfg.generator_loss_type,
- )
-
- disc_loss_fn = DiscriminatorLoss(
- discriminator_factor=cfg.discriminator_factor,
- discriminator_start=cfg.discriminator_start,
- discriminator_loss_type=cfg.discriminator_loss_type,
- lecam_loss_weight=cfg.lecam_loss_weight,
- gradient_penalty_loss_weight=cfg.gradient_penalty_loss_weight,
- )
-
# 6.3. training loop
# calculate discriminator_time_padding
- disc_time_downsample_factor = 2 ** len(cfg.discriminator.channel_multipliers)
- if cfg.dataset.num_frames % disc_time_downsample_factor != 0:
- disc_time_padding = disc_time_downsample_factor - cfg.dataset.num_frames % disc_time_downsample_factor
- else:
- disc_time_padding = 0
- video_contains_first_frame = cfg.video_contains_first_frame
+ # disc_time_downsample_factor = 2 ** len(cfg.discriminator.channel_multipliers)
+ # if cfg.dataset.num_frames % disc_time_downsample_factor != 0:
+ # disc_time_padding = disc_time_downsample_factor - cfg.dataset.num_frames % disc_time_downsample_factor
+ # else:
+ # disc_time_padding = 0
+ # video_contains_first_frame = cfg.video_contains_first_frame
for epoch in range(start_epoch, cfg.epochs):
dataloader.sampler.set_epoch(epoch)
@@ -280,88 +260,47 @@ def main():
logger.info(f"Beginning epoch {epoch}...")
with tqdm(
- range(start_step, num_steps_per_epoch),
+ enumerate(dataloader_iter, start=start_step),
desc=f"Epoch {epoch}",
disable=not coordinator.is_master(),
total=num_steps_per_epoch,
initial=start_step,
) as pbar:
- # with profile(
- # activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA],
- # schedule=torch.profiler.schedule(
- # wait=1,
- # warmup=1,
- # active=2,
- # repeat=2,
- # ),
- # on_trace_ready=torch.profiler.tensorboard_trace_handler('/home/shenchenhui/log'),
- # with_stack=True,
- # record_shapes=True,
- # profile_memory=True,
- # ) as p: # trace efficiency
-
- for step in pbar:
- # with profile(
- # activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA],
- # with_stack=True,
- # ) as p: # trace efficiency
-
- # SCH: calc global step at the start
- global_step = epoch * num_steps_per_epoch + step
-
- batch = next(dataloader_iter)
+ for step, batch in pbar:
x = batch["video"].to(device, dtype) # [B, C, T, H, W]
- # supprt for image or video inputs
- assert x.ndim in {4, 5}, f"received input of {x.ndim} dimensions" # either image or video
- assert (
- x.shape[-2:] == cfg.dataset.image_size
- ), f"received input size {x.shape[-2:]}, but config image size is {cfg.dataset.image_size}"
- is_image = x.ndim == 4
- if is_image:
- video = rearrange(x, "b c ... -> b c 1 ...")
- video_contains_first_frame = True
- else:
- video = x
-
# ===== Spatial VAE =====
- if cfg.get("use_pipeline") == True:
+ if cfg.get("vae_2d", None) is not None:
with torch.no_grad():
- t_video = vae_2d.encode(video)
+ x_z = vae_2d.encode(x)
# ====== VAE ======
- t_recon_video, posterior = vae(
- t_video,
- video_contains_first_frame=video_contains_first_frame,
- )
-
- recon_video = vae_2d.decode(t_recon_video)
+ x_z_rec, posterior, z = model(x_z)
+ x_rec = vae_2d.decode(x_z_rec)
# ====== Generator Loss ======
# simple nll loss
- nll_loss, weighted_nll_loss, weighted_kl_loss = vae_loss_fn(
- video, recon_video, posterior
- )
-
- _, weighted_z_nll_loss, _ = vae_loss_fn(
- t_video, t_recon_video, posterior
- )
+ _, weighted_nll_loss, weighted_kl_loss = vae_loss_fn(x, x_rec, posterior)
+ _, weighted_z_nll_loss, _ = vae_loss_fn(x_z, x_z_rec, posterior)
+ # _, image_identity_loss, _ = vae_loss_fn(x_z, z, posterior)
- adversarial_loss = torch.tensor(0.0)
+ # adversarial_loss = torch.tensor(0.0)
# adversarial loss
- if global_step > cfg.discriminator_start:
- # padded videos for GAN
- fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value=0.0, dim=2)
- fake_logits = discriminator(fake_video.contiguous())
- adversarial_loss = adversarial_loss_fn(
- fake_logits,
- nll_loss,
- vae.module.get_last_layer(),
- global_step,
- is_training=vae.training,
- )
-
- vae_loss = weighted_nll_loss + weighted_kl_loss + adversarial_loss + weighted_z_nll_loss
+ # if global_step > cfg.discriminator_start:
+ # # padded videos for GAN
+ # fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value=0.0, dim=2)
+ # fake_logits = discriminator(fake_video.contiguous())
+ # adversarial_loss = adversarial_loss_fn(
+ # fake_logits,
+ # nll_loss,
+ # vae.module.get_last_layer(),
+ # global_step,
+ # is_training=vae.training,
+ # )
+
+ # vae_loss = weighted_nll_loss + weighted_kl_loss + adversarial_loss + weighted_z_nll_loss
+ # vae_loss = weighted_nll_loss + weighted_kl_loss + weighted_z_nll_loss + image_identity_loss
+ vae_loss = weighted_nll_loss + weighted_kl_loss + weighted_z_nll_loss
optimizer.zero_grad()
# Backward & update
@@ -373,59 +312,60 @@ def main():
# Log loss values:
all_reduce_mean(vae_loss) # NOTE: this is to get average loss for logging
running_loss += vae_loss.item()
+ global_step = epoch * num_steps_per_epoch + step
+ log_step += 1
+ acc_step += 1
# ====== Discriminator Loss ======
- if global_step > cfg.discriminator_start:
- # if video_contains_first_frame:
- # Since we don't have enough T frames, pad anyways
- real_video = pad_at_dim(video, (disc_time_padding, 0), value=0.0, dim=2)
- fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value=0.0, dim=2)
-
- if cfg.gradient_penalty_loss_weight is not None and cfg.gradient_penalty_loss_weight > 0.0:
- real_video = real_video.requires_grad_()
- real_logits = discriminator(
- real_video.contiguous()
- ) # SCH: not detached for now for gradient_penalty calculation
- else:
- real_logits = discriminator(real_video.contiguous().detach())
-
- fake_logits = discriminator(fake_video.contiguous().detach())
-
- lecam_ema_real, lecam_ema_fake = lecam_ema.get()
-
- weighted_d_adversarial_loss, lecam_loss, gradient_penalty_loss = disc_loss_fn(
- real_logits,
- fake_logits,
- global_step,
- lecam_ema_real=lecam_ema_real,
- lecam_ema_fake=lecam_ema_fake,
- real_video=real_video if cfg.gradient_penalty_loss_weight is not None else None,
- )
- disc_loss = weighted_d_adversarial_loss + lecam_loss + gradient_penalty_loss
- if cfg.lecam_loss_weight is not None:
- ema_real = torch.mean(real_logits.clone().detach()).to(device, dtype)
- ema_fake = torch.mean(fake_logits.clone().detach()).to(device, dtype)
- all_reduce_mean(ema_real)
- all_reduce_mean(ema_fake)
- lecam_ema.update(ema_real, ema_fake)
-
- disc_optimizer.zero_grad()
- # Backward & update
- booster.backward(loss=disc_loss, optimizer=disc_optimizer)
- # # NOTE: TODO: clip gradients? this is done in Open-Sora-Plan
- # torch.nn.utils.clip_grad_norm_(discriminator.parameters(), 1) # NOTE: done by grad_clip
- disc_optimizer.step()
-
- # Log loss values:
- all_reduce_mean(disc_loss)
- running_disc_loss += disc_loss.item()
- else:
- disc_loss = torch.tensor(0.0)
- weighted_d_adversarial_loss = torch.tensor(0.0)
- lecam_loss = torch.tensor(0.0)
- gradient_penalty_loss = torch.tensor(0.0)
-
- log_step += 1
+ # if global_step > cfg.discriminator_start:
+ # # if video_contains_first_frame:
+ # # Since we don't have enough T frames, pad anyways
+ # real_video = pad_at_dim(video, (disc_time_padding, 0), value=0.0, dim=2)
+ # fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value=0.0, dim=2)
+
+ # if cfg.gradient_penalty_loss_weight is not None and cfg.gradient_penalty_loss_weight > 0.0:
+ # real_video = real_video.requires_grad_()
+ # real_logits = discriminator(
+ # real_video.contiguous()
+ # ) # SCH: not detached for now for gradient_penalty calculation
+ # else:
+ # real_logits = discriminator(real_video.contiguous().detach())
+
+ # fake_logits = discriminator(fake_video.contiguous().detach())
+
+ # lecam_ema_real, lecam_ema_fake = lecam_ema.get()
+
+ # weighted_d_adversarial_loss, lecam_loss, gradient_penalty_loss = disc_loss_fn(
+ # real_logits,
+ # fake_logits,
+ # global_step,
+ # lecam_ema_real=lecam_ema_real,
+ # lecam_ema_fake=lecam_ema_fake,
+ # real_video=real_video if cfg.gradient_penalty_loss_weight is not None else None,
+ # )
+ # disc_loss = weighted_d_adversarial_loss + lecam_loss + gradient_penalty_loss
+ # if cfg.lecam_loss_weight is not None:
+ # ema_real = torch.mean(real_logits.clone().detach()).to(device, dtype)
+ # ema_fake = torch.mean(fake_logits.clone().detach()).to(device, dtype)
+ # all_reduce_mean(ema_real)
+ # all_reduce_mean(ema_fake)
+ # lecam_ema.update(ema_real, ema_fake)
+
+ # disc_optimizer.zero_grad()
+ # # Backward & update
+ # booster.backward(loss=disc_loss, optimizer=disc_optimizer)
+ # # # NOTE: TODO: clip gradients? this is done in Open-Sora-Plan
+ # # torch.nn.utils.clip_grad_norm_(discriminator.parameters(), 1) # NOTE: done by grad_clip
+ # disc_optimizer.step()
+
+ # # Log loss values:
+ # all_reduce_mean(disc_loss)
+ # running_disc_loss += disc_loss.item()
+ # else:
+ # disc_loss = torch.tensor(0.0)
+ # weighted_d_adversarial_loss = torch.tensor(0.0)
+ # lecam_loss = torch.tensor(0.0)
+ # gradient_penalty_loss = torch.tensor(0.0)
# Log to tensorboard
if coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:
@@ -446,11 +386,13 @@ def main():
"epoch": epoch,
"loss": vae_loss.item(),
"kl_loss": weighted_kl_loss.item(),
- "gen_adv_loss": adversarial_loss.item(),
- "disc_loss": disc_loss.item(),
- "lecam_loss": lecam_loss.item(),
- "r1_grad_penalty": gradient_penalty_loss.item(),
+ # "gen_adv_loss": adversarial_loss.item(),
+ # "disc_loss": disc_loss.item(),
+ # "lecam_loss": lecam_loss.item(),
+ # "r1_grad_penalty": gradient_penalty_loss.item(),
"nll_loss": weighted_nll_loss.item(),
+ "z_nll_loss": weighted_z_nll_loss.item(),
+ # "image_identity_loss": image_identity_loss.item(),
"avg_loss": avg_loss,
},
step=global_step,
@@ -460,19 +402,14 @@ def main():
if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:
save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")
os.makedirs(os.path.join(save_dir, "model"), exist_ok=True) # already handled in booster save_model
- booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)
- booster.save_model(discriminator, os.path.join(save_dir, "discriminator"), shard=True)
+ booster.save_model(model, os.path.join(save_dir, "model"), shard=True)
+ # booster.save_model(discriminator, os.path.join(save_dir, "discriminator"), shard=True)
booster.save_optimizer(
optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096
)
- booster.save_optimizer(
- disc_optimizer, os.path.join(save_dir, "disc_optimizer"), shard=True, size_per_shard=4096
- )
-
- if lr_scheduler is not None:
- booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))
- if disc_lr_scheduler is not None:
- booster.save_lr_scheduler(disc_lr_scheduler, os.path.join(save_dir, "disc_lr_scheduler"))
+ # booster.save_optimizer(
+ # disc_optimizer, os.path.join(save_dir, "disc_optimizer"), shard=True, size_per_shard=4096
+ # )
running_states = {
"epoch": epoch,
@@ -481,15 +418,15 @@ def main():
"sample_start_index": (step + 1) * cfg.batch_size,
}
- lecam_ema_real, lecam_ema_fake = lecam_ema.get()
- lecam_state = {
- "lecam_ema_real": lecam_ema_real.item(),
- "lecam_ema_fake": lecam_ema_fake.item(),
- }
+ # lecam_ema_real, lecam_ema_fake = lecam_ema.get()
+ # lecam_state = {
+ # "lecam_ema_real": lecam_ema_real.item(),
+ # "lecam_ema_fake": lecam_ema_fake.item(),
+ # }
if coordinator.is_master():
save_json(running_states, os.path.join(save_dir, "running_states.json"))
- if cfg.lecam_loss_weight is not None:
- save_json(lecam_state, os.path.join(save_dir, "lecam_states.json"))
+ # if cfg.lecam_loss_weight is not None:
+ # save_json(lecam_state, os.path.join(save_dir, "lecam_states.json"))
dist.barrier()
logger.info(