Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

magvit v2 enc dec arc

Shen-Chenhuia4025b0ea70e
diff --git a/configs/vae_3d/train/16x256x256.py b/configs/vae_3d/train/16x256x256.py
index bbfb422..da0f1fb 100644
--- a/configs/vae_3d/train/16x256x256.py
+++ b/configs/vae_3d/train/16x256x256.py
@@ -48,7 +48,7 @@ magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
3-6 epochs for pexel, from pexel observation its correct
'''
-epochs = 5
+epochs = 10
log_every = 1
ckpt_every = 1000
load = None
diff --git a/configs/vae_magvit_v2/train/17x128x128.py b/configs/vae_magvit_v2/train/17x128x128.py
new file mode 100644
index 0000000..a5e8c03
--- /dev/null
+++ b/configs/vae_magvit_v2/train/17x128x128.py
@@ -0,0 +1,58 @@
+num_frames = 16
+frame_interval = 3
+image_size = (128, 128)
+
+# Define dataset
+root = None
+data_path = "CSV_PATH"
+use_image_transform = False
+num_workers = 4
+
+# Define acceleration
+dtype = "bf16"
+grad_checkpoint = True
+plugin = "zero2"
+sp_size = 1
+
+# Define Loss
+kl_weight = 0.000001
+perceptual_weight = 0.1 # TODO: need to change this to 0.1 !!! according to MAGVIT paper
+
+# Define model
+
+model = dict(
+ type="VAE_3D_B",
+ in_out_channels = 3,
+ latent_embed_dim = 16,
+ filters = 128,
+ num_res_blocks = 4,
+ channel_multipliers = (1, 2, 2, 4),
+ temporal_downsample = (False, True, True),
+ num_groups = 32, # for nn.GroupNorm
+ conv_downsample = False,
+ upsample = "nearest+conv", # options: "deconv", "nearest+conv"
+ kl_embed_dim = 64,
+ custom_conv_padding = None,
+ activation_fn = 'swish',
+)
+
+# Others
+seed = 42
+outputs = "outputs"
+wandb = False
+
+# Training
+''' NOTE:
+magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
+==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
+3-6 epochs for pexel, from pexel observation its correct
+'''
+
+epochs = 10
+log_every = 1
+ckpt_every = 1000
+load = None
+
+batch_size = 8
+lr = 1e-4
+grad_clip = 1.0
diff --git a/opensora/models/vae/README.md b/opensora/models/vae/README.md
index f0a83af..0a6094c 100644
--- a/opensora/models/vae/README.md
+++ b/opensora/models/vae/README.md
@@ -1,42 +1,43 @@
## Commands
-### 1. Train
+## 1. VAE 3D
+### 1.1 Train
```yaml
# train on pexel dataset
WANDB_API_KEY=<wandb_api_key> CUDA_VISIBLE_DEVICES=<n> torchrun --master_port=<port_num> --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/train.csv --wandb True
```
-### 2. Inference
+### 1.2 Inference
```yaml
CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/train_pexel_028/epoch3-global_step20000/ --data-path /home/shenchenhui/data/pexels/debug.csv --save-dir outputs/pexel
-CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/004-F16S3-VAE_3D_B/epoch0-global_step1000 --data-path /home/shenchenhui/data/pexels/debug.csv --save-dir outputs/pexel
-
-CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/004-F16S3-VAE_3D_B/epoch0-global_step2000 --data-path /home/shenchenhui/data/pexels/debug.csv --save-dir outputs/pexel
-
-
-# debug train and inference on the same few samples
-WANDB_API_KEY=7bc1ce71b2dc0b8cd40c500eb256747583f6c07e CUDA_VISIBLE_DEVICES=5 torchrun --master_port=29530 --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/debug.csv --wandb True
-
-CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/006-F16S3-VAE_3D_B/epoch49-global_step50 --data-path /home/shenchenhui/data/pexels/debug.csv
# resume training debug
-WANDB_API_KEY=7bc1ce71b2dc0b8cd40c500eb256747583f6c07e CUDA_VISIBLE_DEVICES=5 torchrun --master_port=29530 --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/debug.csv --load /home/shenchenhui/Open-Sora-dev/outputs/006-F16S3-VAE_3D_B/epoch49-global_step50 --wandb True
+CUDA_VISIBLE_DEVICES=5 torchrun --master_port=29530 --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/debug.csv --load /home/shenchenhui/Open-Sora-dev/outputs/006-F16S3-VAE_3D_B/epoch49-global_step50
```
-# inference
-
-CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/train_pexel_028/epoch9-global_step52000/ --data-path /home/shenchenhui/data/pexels/test.csv
-
-
-
-```yaml
-scp -P 31081 [email protected]:/home/shenchenhui/Open-Sora-dev/outputs/samples/sample_0.mp4 /Users/shenchenhui/Desktop
-
-scp -P 31081 [email protected]:/home/shenchenhui/data/pexels/test.csv /Users/shenchenhui/Desktop
+## 2. MAGVIT-v2
+### 2.1 dependencies
+```
+'accelerate>=0.24.0',
+'beartype',
+'einops>=0.7.0',
+'ema-pytorch>=0.2.4',
+'pytorch-warmup',
+'gateloop-transformer>=0.2.2',
+'kornia',
+'opencv-python',
+'pillow',
+'pytorch-custom-utils>=0.0.9',
+'numpy',
+'vector-quantize-pytorch>=1.11.8',
+'taylor-series-linear-attention>=0.1.5',
+'torch',
+'torchvision',
+'x-transformers'
+```
-```
\ No newline at end of file
diff --git a/opensora/models/vae/magvit2.py b/opensora/models/vae/magvit2.py
new file mode 100644
index 0000000..455e128
--- /dev/null
+++ b/opensora/models/vae/magvit2.py
@@ -0,0 +1,1905 @@
+import copy
+from pathlib import Path
+from math import log2, ceil, sqrt
+from functools import wraps, partial
+
+import torch
+import torch.nn.functional as F
+from torch.cuda.amp import autocast
+from torch import nn, einsum, Tensor
+from torch.nn import Module, ModuleList
+from torch.autograd import grad as torch_grad
+
+import torchvision
+from torchvision.models import VGG16_Weights
+
+from collections import namedtuple
+
+from vector_quantize_pytorch import LFQ, FSQ
+
+from einops import rearrange, repeat, reduce, pack, unpack
+from einops.layers.torch import Rearrange
+
+from beartype import beartype
+from beartype.typing import Union, Tuple, Optional, List
+
+from magvit2_pytorch.attend import Attend
+from magvit2_pytorch.version import __version__
+
+from gateloop_transformer import SimpleGateLoopLayer
+
+from taylor_series_linear_attention import TaylorSeriesLinearAttn
+
+from kornia.filters import filter3d
+
+import pickle
+
+# helper
+
+def exists(v):
+ return v is not None
+
+def default(v, d):
+ return v if exists(v) else d
+
+def safe_get_index(it, ind, default = None):
+ if ind < len(it):
+ return it[ind]
+ return default
+
+def pair(t):
+ return t if isinstance(t, tuple) else (t, t)
+
+def identity(t, *args, **kwargs):
+ return t
+
+def divisible_by(num, den):
+ return (num % den) == 0
+
+def pack_one(t, pattern):
+ return pack([t], pattern)
+
+def unpack_one(t, ps, pattern):
+ return unpack(t, ps, pattern)[0]
+
+def append_dims(t, ndims: int):
+ return t.reshape(*t.shape, *((1,) * ndims))
+
+def is_odd(n):
+ return not divisible_by(n, 2)
+
+def maybe_del_attr_(o, attr):
+ if hasattr(o, attr):
+ delattr(o, attr)
+
+def cast_tuple(t, length = 1):
+ return t if isinstance(t, tuple) else ((t,) * length)
+
+# tensor helpers
+
+def l2norm(t):
+ return F.normalize(t, dim = -1, p = 2)
+
+def pad_at_dim(t, pad, dim = -1, value = 0.):
+ dims_from_right = (- dim - 1) if dim < 0 else (t.ndim - dim - 1)
+ zeros = ((0, 0) * dims_from_right)
+ return F.pad(t, (*zeros, *pad), value = value)
+
+def pick_video_frame(video, frame_indices):
+ batch, device = video.shape[0], video.device
+ video = rearrange(video, 'b c f ... -> b f c ...')
+ batch_indices = torch.arange(batch, device = device)
+ batch_indices = rearrange(batch_indices, 'b -> b 1')
+ images = video[batch_indices, frame_indices]
+ images = rearrange(images, 'b 1 c ... -> b c ...')
+ return images
+
+# gan related
+
+def gradient_penalty(images, output):
+ batch_size = images.shape[0]
+
+ gradients = torch_grad(
+ outputs = output,
+ inputs = images,
+ grad_outputs = torch.ones(output.size(), device = images.device),
+ create_graph = True,
+ retain_graph = True,
+ only_inputs = True
+ )[0]
+
+ gradients = rearrange(gradients, 'b ... -> b (...)')
+ return ((gradients.norm(2, dim = 1) - 1) ** 2).mean()
+
+def leaky_relu(p = 0.1):
+ return nn.LeakyReLU(p)
+
+def hinge_discr_loss(fake, real):
+ return (F.relu(1 + fake) + F.relu(1 - real)).mean()
+
+def hinge_gen_loss(fake):
+ return -fake.mean()
+
+@autocast(enabled = False)
+@beartype
+def grad_layer_wrt_loss(
+ loss: Tensor,
+ layer: nn.Parameter
+):
+ return torch_grad(
+ outputs = loss,
+ inputs = layer,
+ grad_outputs = torch.ones_like(loss),
+ retain_graph = True
+ )[0].detach()
+
+# helper decorators
+
+def remove_vgg(fn):
+ @wraps(fn)
+ def inner(self, *args, **kwargs):
+ has_vgg = hasattr(self, 'vgg')
+ if has_vgg:
+ vgg = self.vgg
+ delattr(self, 'vgg')
+
+ out = fn(self, *args, **kwargs)
+
+ if has_vgg:
+ self.vgg = vgg
+
+ return out
+ return inner
+
+# helper classes
+
+def Sequential(*modules):
+ modules = [*filter(exists, modules)]
+
+ if len(modules) == 0:
+ return nn.Identity()
+
+ return nn.Sequential(*modules)
+
+class Residual(Module):
+ @beartype
+ def __init__(self, fn: Module):
+ super().__init__()
+ self.fn = fn
+
+ def forward(self, x, **kwargs):
+ return self.fn(x, **kwargs) + x
+
+# for a bunch of tensor operations to change tensor to (batch, time, feature dimension) and back
+
+class ToTimeSequence(Module):
+ @beartype
+ def __init__(self, fn: Module):
+ super().__init__()
+ self.fn = fn
+
+ def forward(self, x, **kwargs):
+ x = rearrange(x, 'b c f ... -> b ... f c')
+ x, ps = pack_one(x, '* n c')
+
+ o = self.fn(x, **kwargs)
+
+ o = unpack_one(o, ps, '* n c')
+ return rearrange(o, 'b ... f c -> b c f ...')
+
+
+class SqueezeExcite(Module):
+ # global context network - attention-esque squeeze-excite variant (https://arxiv.org/abs/2012.13375)
+
+ def __init__(
+ self,
+ dim,
+ *,
+ dim_out = None,
+ dim_hidden_min = 16,
+ init_bias = -10
+ ):
+ super().__init__()
+ dim_out = default(dim_out, dim)
+
+ self.to_k = nn.Conv2d(dim, 1, 1)
+ dim_hidden = max(dim_hidden_min, dim_out // 2)
+
+ self.net = nn.Sequential(
+ nn.Conv2d(dim, dim_hidden, 1),
+ nn.LeakyReLU(0.1),
+ nn.Conv2d(dim_hidden, dim_out, 1),
+ nn.Sigmoid()
+ )
+
+ nn.init.zeros_(self.net[-2].weight)
+ nn.init.constant_(self.net[-2].bias, init_bias)
+
+ def forward(self, x):
+ orig_input, batch = x, x.shape[0]
+ is_video = x.ndim == 5
+
+ if is_video:
+ x = rearrange(x, 'b c f h w -> (b f) c h w')
+
+ context = self.to_k(x)
+
+ context = rearrange(context, 'b c h w -> b c (h w)').softmax(dim = -1)
+ spatial_flattened_input = rearrange(x, 'b c h w -> b c (h w)')
+
+ out = einsum('b i n, b c n -> b c i', context, spatial_flattened_input)
+ out = rearrange(out, '... -> ... 1')
+ gates = self.net(out)
+
+ if is_video:
+ gates = rearrange(gates, '(b f) c h w -> b c f h w', b = batch)
+
+ return gates * orig_input
+
+# token shifting
+
+class TokenShift(Module):
+ @beartype
+ def __init__(self, fn: Module):
+ super().__init__()
+ self.fn = fn
+
+ def forward(self, x, **kwargs):
+ x, x_shift = x.chunk(2, dim = 1)
+ x_shift = pad_at_dim(x_shift, (1, -1), dim = 2) # shift time dimension
+ x = torch.cat((x, x_shift), dim = 1)
+ return self.fn(x, **kwargs)
+
+# rmsnorm
+
+class RMSNorm(Module):
+ def __init__(
+ self,
+ dim,
+ channel_first = False,
+ images = False,
+ bias = False
+ ):
+ super().__init__()
+ broadcastable_dims = (1, 1, 1) if not images else (1, 1)
+ shape = (dim, *broadcastable_dims) if channel_first else (dim,)
+
+ self.channel_first = channel_first
+ self.scale = dim ** 0.5
+ self.gamma = nn.Parameter(torch.ones(shape))
+ self.bias = nn.Parameter(torch.zeros(shape)) if bias else 0.
+
+ def forward(self, x):
+ return F.normalize(x, dim = (1 if self.channel_first else -1)) * self.scale * self.gamma + self.bias
+
+class AdaptiveRMSNorm(Module):
+ def __init__(
+ self,
+ dim,
+ *,
+ dim_cond,
+ channel_first = False,
+ images = False,
+ bias = False
+ ):
+ super().__init__()
+ broadcastable_dims = (1, 1, 1) if not images else (1, 1)
+ shape = (dim, *broadcastable_dims) if channel_first else (dim,)
+
+ self.dim_cond = dim_cond
+ self.channel_first = channel_first
+ self.scale = dim ** 0.5
+
+ self.to_gamma = nn.Linear(dim_cond, dim)
+ self.to_bias = nn.Linear(dim_cond, dim) if bias else None
+
+ nn.init.zeros_(self.to_gamma.weight)
+ nn.init.ones_(self.to_gamma.bias)
+
+ if bias:
+ nn.init.zeros_(self.to_bias.weight)
+ nn.init.zeros_(self.to_bias.bias)
+
+ @beartype
+ def forward(self, x: Tensor, *, cond: Tensor):
+ batch = x.shape[0]
+ assert cond.shape == (batch, self.dim_cond)
+
+ gamma = self.to_gamma(cond)
+
+ bias = 0.
+ if exists(self.to_bias):
+ bias = self.to_bias(cond)
+
+ if self.channel_first:
+ gamma = append_dims(gamma, x.ndim - 2)
+
+ if exists(self.to_bias):
+ bias = append_dims(bias, x.ndim - 2)
+
+ return F.normalize(x, dim = (1 if self.channel_first else -1)) * self.scale * gamma + bias
+
+# attention
+
+class Attention(Module):
+ @beartype
+ def __init__(
+ self,
+ *,
+ dim,
+ dim_cond: Optional[int] = None,
+ causal = False,
+ dim_head = 32,
+ heads = 8,
+ flash = False,
+ dropout = 0.,
+ num_memory_kv = 4
+ ):
+ super().__init__()
+ dim_inner = dim_head * heads
+
+ self.need_cond = exists(dim_cond)
+
+ if self.need_cond:
+ self.norm = AdaptiveRMSNorm(dim, dim_cond = dim_cond)
+ else:
+ self.norm = RMSNorm(dim)
+
+ self.to_qkv = nn.Sequential(
+ nn.Linear(dim, dim_inner * 3, bias = False),
+ Rearrange('b n (qkv h d) -> qkv b h n d', qkv = 3, h = heads)
+ )
+
+ assert num_memory_kv > 0
+ self.mem_kv = nn.Parameter(torch.randn(2, heads, num_memory_kv, dim_head))
+
+ self.attend = Attend(
+ causal = causal,
+ dropout = dropout,
+ flash = flash
+ )
+
+ self.to_out = nn.Sequential(
+ Rearrange('b h n d -> b n (h d)'),
+ nn.Linear(dim_inner, dim, bias = False)
+ )
+
+ @beartype
+ def forward(
+ self,
+ x,
+ mask: Optional[Tensor ] = None,
+ cond: Optional[Tensor] = None
+ ):
+ maybe_cond_kwargs = dict(cond = cond) if self.need_cond else dict()
+
+ x = self.norm(x, **maybe_cond_kwargs)
+
+ q, k, v = self.to_qkv(x)
+
+ mk, mv = map(lambda t: repeat(t, 'h n d -> b h n d', b = q.shape[0]), self.mem_kv)
+ k = torch.cat((mk, k), dim = -2)
+ v = torch.cat((mv, v), dim = -2)
+
+ out = self.attend(q, k, v, mask = mask)
+ return self.to_out(out)
+
+class LinearAttention(Module):
+ """
+ using the specific linear attention proposed in https://arxiv.org/abs/2106.09681
+ """
+
+ @beartype
+ def __init__(
+ self,
+ *,
+ dim,
+ dim_cond: Optional[int] = None,
+ dim_head = 8,
+ heads = 8,
+ dropout = 0.
+ ):
+ super().__init__()
+ dim_inner = dim_head * heads
+
+ self.need_cond = exists(dim_cond)
+
+ if self.need_cond:
+ self.norm = AdaptiveRMSNorm(dim, dim_cond = dim_cond)
+ else:
+ self.norm = RMSNorm(dim)
+
+ self.attn = TaylorSeriesLinearAttn(
+ dim = dim,
+ dim_head = dim_head,
+ heads = heads
+ )
+
+ def forward(
+ self,
+ x,
+ cond: Optional[Tensor] = None
+ ):
+ maybe_cond_kwargs = dict(cond = cond) if self.need_cond else dict()
+
+ x = self.norm(x, **maybe_cond_kwargs)
+
+ return self.attn(x)
+
+class LinearSpaceAttention(LinearAttention):
+ """
+ SCH: format h & w into a linear dim, do linear attention, then format back
+ """
+ def forward(self, x, *args, **kwargs):
+ x = rearrange(x, 'b c ... h w -> b ... h w c')
+ x, batch_ps = pack_one(x, '* h w c')
+ x, seq_ps = pack_one(x, 'b * c')
+
+ x = super().forward(x, *args, **kwargs)
+
+ x = unpack_one(x, seq_ps, 'b * c')
+ x = unpack_one(x, batch_ps, '* h w c')
+ return rearrange(x, 'b ... h w c -> b c ... h w')
+
+class SpaceAttention(Attention):
+ def forward(self, x, *args, **kwargs):
+ x = rearrange(x, 'b c t h w -> b t h w c')
+ x, batch_ps = pack_one(x, '* h w c')
+ x, seq_ps = pack_one(x, 'b * c')
+
+ x = super().forward(x, *args, **kwargs)
+
+ x = unpack_one(x, seq_ps, 'b * c')
+ x = unpack_one(x, batch_ps, '* h w c')
+ return rearrange(x, 'b t h w c -> b c t h w')
+
+class TimeAttention(Attention):
+ def forward(self, x, *args, **kwargs):
+ x = rearrange(x, 'b c t h w -> b h w t c')
+ x, batch_ps = pack_one(x, '* t c')
+
+ x = super().forward(x, *args, **kwargs)
+
+ x = unpack_one(x, batch_ps, '* t c')
+ return rearrange(x, 'b h w t c -> b c t h w')
+
+class GEGLU(Module):
+ def forward(self, x):
+ x, gate = x.chunk(2, dim = 1)
+ return F.gelu(gate) * x
+
+class FeedForward(Module):
+ @beartype
+ def __init__(
+ self,
+ dim,
+ *,
+ dim_cond: Optional[int] = None,
+ mult = 4,
+ images = False
+ ):
+ super().__init__()
+ conv_klass = nn.Conv2d if images else nn.Conv3d
+
+ rmsnorm_klass = RMSNorm if not exists(dim_cond) else partial(AdaptiveRMSNorm, dim_cond = dim_cond)
+
+ maybe_adaptive_norm_klass = partial(rmsnorm_klass, channel_first = True, images = images)
+
+ dim_inner = int(dim * mult * 2 / 3)
+
+ self.norm = maybe_adaptive_norm_klass(dim)
+
+ self.net = Sequential(
+ conv_klass(dim, dim_inner * 2, 1),
+ GEGLU(),
+ conv_klass(dim_inner, dim, 1)
+ )
+
+ @beartype
+ def forward(
+ self,
+ x: Tensor,
+ *,
+ cond: Optional[Tensor] = None
+ ):
+ maybe_cond_kwargs = dict(cond = cond) if exists(cond) else dict()
+
+ x = self.norm(x, **maybe_cond_kwargs)
+ return self.net(x)
+
+# discriminator with anti-aliased downsampling (blurpool Zhang et al.)
+
+class Blur(Module):
+ def __init__(self):
+ super().__init__()
+ f = torch.Tensor([1, 2, 1])
+ self.register_buffer('f', f)
+
+ def forward(
+ self,
+ x,
+ space_only = False,
+ time_only = False
+ ):
+ assert not (space_only and time_only)
+
+ f = self.f
+
+ if space_only:
+ f = einsum('i, j -> i j', f, f)
+ f = rearrange(f, '... -> 1 1 ...')
+ elif time_only:
+ f = rearrange(f, 'f -> 1 f 1 1')
+ else:
+ f = einsum('i, j, k -> i j k', f, f, f)
+ f = rearrange(f, '... -> 1 ...')
+
+ is_images = x.ndim == 4
+
+ if is_images:
+ x = rearrange(x, 'b c h w -> b c 1 h w')
+
+ out = filter3d(x, f, normalized = True)
+
+ if is_images:
+ out = rearrange(out, 'b c 1 h w -> b c h w')
+
+ return out
+
+class DiscriminatorBlock(Module):
+ def __init__(
+ self,
+ input_channels,
+ filters,
+ downsample = True,
+ antialiased_downsample = True
+ ):
+ super().__init__()
+ self.conv_res = nn.Conv2d(input_channels, filters, 1, stride = (2 if downsample else 1))
+
+ self.net = nn.Sequential(
+ nn.Conv2d(input_channels, filters, 3, padding = 1),
+ leaky_relu(),
+ nn.Conv2d(filters, filters, 3, padding = 1),
+ leaky_relu()
+ )
+
+ self.maybe_blur = Blur() if antialiased_downsample else None
+
+ self.downsample = nn.Sequential(
+ Rearrange('b c (h p1) (w p2) -> b (c p1 p2) h w', p1 = 2, p2 = 2),
+ nn.Conv2d(filters * 4, filters, 1)
+ ) if downsample else None
+
+ def forward(self, x):
+ res = self.conv_res(x)
+
+ x = self.net(x)
+
+ if exists(self.downsample):
+ if exists(self.maybe_blur):
+ x = self.maybe_blur(x, space_only = True)
+
+ x = self.downsample(x)
+
+ x = (x + res) * (2 ** -0.5)
+ return x
+
+class Discriminator(Module):
+ @beartype
+ def __init__(
+ self,
+ *,
+ dim,
+ image_size,
+ channels = 3,
+ max_dim = 512,
+ attn_heads = 8,
+ attn_dim_head = 32,
+ linear_attn_dim_head = 8,
+ linear_attn_heads = 16,
+ ff_mult = 4,
+ antialiased_downsample = False
+ ):
+ super().__init__()
+ image_size = pair(image_size)
+ min_image_resolution = min(image_size)
+
+ num_layers = int(log2(min_image_resolution) - 2)
+
+ blocks = []
+
+ layer_dims = [channels] + [(dim * 4) * (2 ** i) for i in range(num_layers + 1)] # SCH: num channels across each layer
+ layer_dims = [min(layer_dim, max_dim) for layer_dim in layer_dims]
+ layer_dims_in_out = tuple(zip(layer_dims[:-1], layer_dims[1:]))
+
+ blocks = []
+ attn_blocks = []
+
+ image_resolution = min_image_resolution
+
+ for ind, (in_chan, out_chan) in enumerate(layer_dims_in_out):
+ num_layer = ind + 1
+ is_not_last = ind != (len(layer_dims_in_out) - 1)
+
+ block = DiscriminatorBlock(
+ in_chan,
+ out_chan,
+ downsample = is_not_last,
+ antialiased_downsample = antialiased_downsample
+ )
+
+ attn_block = Sequential(
+ Residual(LinearSpaceAttention(
+ dim = out_chan,
+ heads = linear_attn_heads,
+ dim_head = linear_attn_dim_head
+ )),
+ Residual(FeedForward(
+ dim = out_chan,
+ mult = ff_mult,
+ images = True
+ ))
+ )
+
+ blocks.append(ModuleList([
+ block,
+ attn_block
+ ]))
+
+ image_resolution //= 2
+
+ self.blocks = ModuleList(blocks)
+
+ dim_last = layer_dims[-1]
+
+ downsample_factor = 2 ** num_layers
+ last_fmap_size = tuple(map(lambda n: n // downsample_factor, image_size))
+
+ latent_dim = last_fmap_size[0] * last_fmap_size[1] * dim_last
+
+ self.to_logits = Sequential(
+ nn.Conv2d(dim_last, dim_last, 3, padding = 1),
+ leaky_relu(),
+ Rearrange('b ... -> b (...)'),
+ nn.Linear(latent_dim, 1),
+ Rearrange('b 1 -> b')
+ )
+
+ def forward(self, x):
+
+ for block, attn_block in self.blocks:
+ x = block(x)
+ x = attn_block(x)
+
+ return self.to_logits(x)
+
+# modulatable conv from Karras et al. Stylegan2
+# for conditioning on latents
+
+class Conv3DMod(Module):
+ @beartype
+ def __init__(
+ self,
+ dim,
+ *,
+ spatial_kernel,
+ time_kernel,
+ causal = True,
+ dim_out = None,
+ demod = True,
+ eps = 1e-8,
+ pad_mode = 'zeros'
+ ):
+ super().__init__()
+ dim_out = default(dim_out, dim)
+
+ self.eps = eps
+
+ assert is_odd(spatial_kernel) and is_odd(time_kernel)
+
+ self.spatial_kernel = spatial_kernel
+ self.time_kernel = time_kernel
+
+ time_padding = (time_kernel - 1, 0) if causal else ((time_kernel // 2,) * 2)
+
+ self.pad_mode = pad_mode
+ self.padding = (*((spatial_kernel // 2,) * 4), *time_padding)
+ self.weights = nn.Parameter(torch.randn((dim_out, dim, time_kernel, spatial_kernel, spatial_kernel)))
+
+ self.demod = demod
+
+ nn.init.kaiming_normal_(self.weights, a = 0, mode = 'fan_in', nonlinearity = 'selu')
+
+ @beartype
+ def forward(
+ self,
+ fmap,
+ cond: Tensor
+ ):
+ """
+ notation
+
+ b - batch
+ n - convs
+ o - output
+ i - input
+ k - kernel
+ """
+
+ b = fmap.shape[0]
+
+ # prepare weights for modulation
+
+ weights = self.weights
+
+ # do the modulation, demodulation, as done in stylegan2
+
+ cond = rearrange(cond, 'b i -> b 1 i 1 1 1')
+
+ weights = weights * (cond + 1)
+
+ if self.demod:
+ inv_norm = reduce(weights ** 2, 'b o i k0 k1 k2 -> b o 1 1 1 1', 'sum').clamp(min = self.eps).rsqrt()
+ weights = weights * inv_norm
+
+ fmap = rearrange(fmap, 'b c t h w -> 1 (b c) t h w')
+
+ weights = rearrange(weights, 'b o ... -> (b o) ...')
+
+ fmap = F.pad(fmap, self.padding, mode = self.pad_mode)
+ fmap = F.conv3d(fmap, weights, groups = b)
+
+ return rearrange(fmap, '1 (b o) ... -> b o ...', b = b)
+
+# strided conv downsamples
+
+class SpatialDownsample2x(Module):
+ def __init__(
+ self,
+ dim,
+ dim_out = None,
+ kernel_size = 3,
+ antialias = False
+ ):
+ super().__init__()
+ dim_out = default(dim_out, dim)
+ self.maybe_blur = Blur() if antialias else identity
+ self.conv = nn.Conv2d(dim, dim_out, kernel_size, stride = 2, padding = kernel_size // 2)
+
+ def forward(self, x):
+ x = self.maybe_blur(x, space_only = True)
+
+ x = rearrange(x, 'b c t h w -> b t c h w')
+ x, ps = pack_one(x, '* c h w')
+
+ out = self.conv(x)
+
+ out = unpack_one(out, ps, '* c h w')
+ out = rearrange(out, 'b t c h w -> b c t h w')
+ return out
+
+class TimeDownsample2x(Module):
+ def __init__(
+ self,
+ dim,
+ dim_out = None,
+ kernel_size = 3,
+ antialias = False
+ ):
+ super().__init__()
+ dim_out = default(dim_out, dim)
+ self.maybe_blur = Blur() if antialias else identity
+ self.time_causal_padding = (kernel_size - 1, 0)
+ self.conv = nn.Conv1d(dim, dim_out, kernel_size, stride = 2)
+
+ def forward(self, x):
+ x = self.maybe_blur(x, time_only = True)
+
+ x = rearrange(x, 'b c t h w -> b h w c t')
+ x, ps = pack_one(x, '* c t')
+
+ x = F.pad(x, self.time_causal_padding)
+ out = self.conv(x)
+
+ out = unpack_one(out, ps, '* c t')
+ out = rearrange(out, 'b h w c t -> b c t h w')
+ return out
+
+# depth to space upsamples
+
+class SpatialUpsample2x(Module):
+ def __init__(
+ self,
+ dim,
+ dim_out = None
+ ):
+ super().__init__()
+ dim_out = default(dim_out, dim)
+ conv = nn.Conv2d(dim, dim_out * 4, 1)
+
+ self.net = nn.Sequential(
+ conv,
+ nn.SiLU(),
+ Rearrange('b (c p1 p2) h w -> b c (h p1) (w p2)', p1 = 2, p2 = 2)
+ )
+
+ self.init_conv_(conv)
+
+ def init_conv_(self, conv):
+ o, i, h, w = conv.weight.shape
+ conv_weight = torch.empty(o // 4, i, h, w)
+ nn.init.kaiming_uniform_(conv_weight)
+ conv_weight = repeat(conv_weight, 'o ... -> (o 4) ...')
+
+ conv.weight.data.copy_(conv_weight)
+ nn.init.zeros_(conv.bias.data)
+
+ def forward(self, x):
+ x = rearrange(x, 'b c t h w -> b t c h w')
+ x, ps = pack_one(x, '* c h w')
+
+ out = self.net(x)
+
+ out = unpack_one(out, ps, '* c h w')
+ out = rearrange(out, 'b t c h w -> b c t h w')
+ return out
+
+class TimeUpsample2x(Module):
+ def __init__(
+ self,
+ dim,
+ dim_out = None
+ ):
+ super().__init__()
+ dim_out = default(dim_out, dim)
+ conv = nn.Conv1d(dim, dim_out * 2, 1)
+
+ self.net = nn.Sequential(
+ conv,
+ nn.SiLU(),
+ Rearrange('b (c p) t -> b c (t p)', p = 2)
+ )
+
+ self.init_conv_(conv)
+
+ def init_conv_(self, conv):
+ o, i, t = conv.weight.shape
+ conv_weight = torch.empty(o // 2, i, t)
+ nn.init.kaiming_uniform_(conv_weight)
+ conv_weight = repeat(conv_weight, 'o ... -> (o 2) ...')
+
+ conv.weight.data.copy_(conv_weight)
+ nn.init.zeros_(conv.bias.data)
+
+ def forward(self, x):
+ x = rearrange(x, 'b c t h w -> b h w c t')
+ x, ps = pack_one(x, '* c t')
+
+ out = self.net(x)
+
+ out = unpack_one(out, ps, '* c t')
+ out = rearrange(out, 'b h w c t -> b c t h w')
+ return out
+
+# autoencoder - only best variant here offered, with causal conv 3d
+
+def SameConv2d(dim_in, dim_out, kernel_size):
+ kernel_size = cast_tuple(kernel_size, 2)
+ padding = [k // 2 for k in kernel_size]
+ return nn.Conv2d(dim_in, dim_out, kernel_size = kernel_size, padding = padding)
+
+class CausalConv3d(Module):
+ @beartype
+ def __init__(
+ self,
+ chan_in,
+ chan_out,
+ kernel_size: Union[int, Tuple[int, int, int]],
+ pad_mode = 'constant',
+ **kwargs
+ ):
+ super().__init__()
+ kernel_size = cast_tuple(kernel_size, 3)
+
+ time_kernel_size, height_kernel_size, width_kernel_size = kernel_size
+
+ assert is_odd(height_kernel_size) and is_odd(width_kernel_size)
+
+ dilation = kwargs.pop('dilation', 1)
+ stride = kwargs.pop('stride', 1)
+
+ self.pad_mode = pad_mode
+ time_pad = dilation * (time_kernel_size - 1) + (1 - stride)
+ height_pad = height_kernel_size // 2
+ width_pad = width_kernel_size // 2
+
+ self.time_pad = time_pad
+ self.time_causal_padding = (width_pad, width_pad, height_pad, height_pad, time_pad, 0)
+
+ stride = (stride, 1, 1)
+ dilation = (dilation, 1, 1)
+ self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride = stride, dilation = dilation, **kwargs)
+
+ def forward(self, x):
+ pad_mode = self.pad_mode if self.time_pad < x.shape[2] else 'constant'
+
+ x = F.pad(x, self.time_causal_padding, mode = pad_mode)
+ return self.conv(x)
+
+@beartype
+def ResidualUnit(
+ dim,
+ kernel_size: Union[int, Tuple[int, int, int]],
+ pad_mode: str = 'constant'
+):
+ net = Sequential(
+ CausalConv3d(dim, dim, kernel_size, pad_mode = pad_mode),
+ nn.ELU(),
+ nn.Conv3d(dim, dim, 1),
+ nn.ELU(),
+ SqueezeExcite(dim)
+ )
+
+ return Residual(net)
+
+@beartype
+class ResidualUnitMod(Module):
+ def __init__(
+ self,
+ dim,
+ kernel_size: Union[int, Tuple[int, int, int]],
+ *,
+ dim_cond,
+ pad_mode: str = 'constant',
+ demod = True
+ ):
+ super().__init__()
+ kernel_size = cast_tuple(kernel_size, 3)
+ time_kernel_size, height_kernel_size, width_kernel_size = kernel_size
+ assert height_kernel_size == width_kernel_size
+
+ self.to_cond = nn.Linear(dim_cond, dim)
+
+ self.conv = Conv3DMod(
+ dim = dim,
+ spatial_kernel = height_kernel_size,
+ time_kernel = time_kernel_size,
+ causal = True,
+ demod = demod,
+ pad_mode = pad_mode
+ )
+
+ self.conv_out = nn.Conv3d(dim, dim, 1)
+
+ @beartype
+ def forward(
+ self,
+ x,
+ cond: Tensor,
+ ):
+ res = x
+ cond = self.to_cond(cond)
+
+ x = self.conv(x, cond = cond)
+ x = F.elu(x)
+ x = self.conv_out(x)
+ x = F.elu(x)
+ return x + res
+
+class CausalConvTranspose3d(Module):
+ def __init__(
+ self,
+ chan_in,
+ chan_out,
+ kernel_size: Union[int, Tuple[int, int, int]],
+ *,
+ time_stride,
+ **kwargs
+ ):
+ super().__init__()
+ kernel_size = cast_tuple(kernel_size, 3)
+
+ time_kernel_size, height_kernel_size, width_kernel_size = kernel_size
+
+ assert is_odd(height_kernel_size) and is_odd(width_kernel_size)
+
+ self.upsample_factor = time_stride
+
+ height_pad = height_kernel_size // 2
+ width_pad = width_kernel_size // 2
+
+ stride = (time_stride, 1, 1)
+ padding = (0, height_pad, width_pad)
+
+ self.conv = nn.ConvTranspose3d(chan_in, chan_out, kernel_size, stride, padding = padding, **kwargs)
+
+ def forward(self, x):
+ assert x.ndim == 5
+ t = x.shape[2]
+
+ out = self.conv(x)
+
+ out = out[..., :(t * self.upsample_factor), :, :]
+ return out
+
+# video tokenizer class
+
+LossBreakdown = namedtuple('LossBreakdown', [
+ 'recon_loss',
+ 'lfq_aux_loss',
+ 'quantizer_loss_breakdown',
+ 'perceptual_loss',
+ 'adversarial_gen_loss',
+ 'adaptive_adversarial_weight',
+ 'multiscale_gen_losses',
+ 'multiscale_gen_adaptive_weights'
+])
+
+DiscrLossBreakdown = namedtuple('DiscrLossBreakdown', [
+ 'discr_loss',
+ 'multiscale_discr_losses',
+ 'gradient_penalty'
+])
+
+class VideoTokenizer(Module):
+ @beartype
+ def __init__(
+ self,
+ *,
+ image_size,
+ layers: Tuple[Union[str, Tuple[str, int]], ...] = (
+ 'residual',
+ 'residual',
+ 'residual'
+ ),
+ residual_conv_kernel_size = 3,
+ # num_codebooks = 1,
+ # codebook_size: Optional[int] = None,
+ channels = 3,
+ init_dim = 64,
+ max_dim = float('inf'),
+ dim_cond = None,
+ dim_cond_expansion_factor = 4.,
+ input_conv_kernel_size: Tuple[int, int, int] = (7, 7, 7),
+ output_conv_kernel_size: Tuple[int, int, int] = (3, 3, 3),
+ pad_mode: str = 'constant',
+ lfq_entropy_loss_weight = 0.1,
+ lfq_commitment_loss_weight = 1., # SCH: codebook?
+ lfq_diversity_gamma = 2.5,
+ quantizer_aux_loss_weight = 1.,
+ lfq_activation = nn.Identity(),
+ use_fsq = False,
+ fsq_levels: Optional[List[int]] = None,
+ attn_dim_head = 32,
+ attn_heads = 8,
+ attn_dropout = 0.,
+ linear_attn_dim_head = 8,
+ linear_attn_heads = 16,
+ vgg: Optional[Module] = None,
+ vgg_weights: VGG16_Weights = VGG16_Weights.DEFAULT,
+ perceptual_loss_weight = 1e-1,
+ discr_kwargs: Optional[dict] = None,
+ multiscale_discrs: Tuple[Module, ...] = tuple(),
+ use_gan = True,
+ adversarial_loss_weight = 1.,
+ grad_penalty_loss_weight = 10.,
+ multiscale_adversarial_loss_weight = 1.,
+ flash_attn = True,
+ separate_first_frame_encoding = False
+ ):
+ super().__init__()
+
+ # for autosaving the config
+
+ _locals = locals()
+ _locals.pop('self', None)
+ _locals.pop('__class__', None)
+ self._configs = pickle.dumps(_locals)
+
+ # image size
+
+ self.channels = channels
+ self.image_size = image_size
+
+ # initial encoder
+
+ self.conv_in = CausalConv3d(channels, init_dim, input_conv_kernel_size, pad_mode = pad_mode)
+
+ # whether to encode the first frame separately or not
+
+ self.conv_in_first_frame = nn.Identity()
+ self.conv_out_first_frame = nn.Identity()
+
+ if separate_first_frame_encoding:
+ self.conv_in_first_frame = SameConv2d(channels, init_dim, input_conv_kernel_size[-2:])
+ self.conv_out_first_frame = SameConv2d(init_dim, channels, output_conv_kernel_size[-2:])
+
+ self.separate_first_frame_encoding = separate_first_frame_encoding
+
+ # encoder and decoder layers
+
+ self.encoder_layers = ModuleList([])
+ self.decoder_layers = ModuleList([])
+
+ self.conv_out = CausalConv3d(init_dim, channels, output_conv_kernel_size, pad_mode = pad_mode)
+
+ dim = init_dim
+ dim_out = dim
+
+ layer_fmap_size = image_size # SCH: feaure map size
+ time_downsample_factor = 1
+ has_cond_across_layers = [] # SCH: record if the corr. layers has condition
+
+ for layer_def in layers:
+ layer_type, *layer_params = cast_tuple(layer_def)
+
+ has_cond = False
+
+ if layer_type == 'residual': # SCH: resblock
+ encoder_layer = ResidualUnit(dim, residual_conv_kernel_size)
+ decoder_layer = ResidualUnit(dim, residual_conv_kernel_size)
+
+ elif layer_type == 'consecutive_residual':
+ num_consecutive, = layer_params
+ encoder_layer = Sequential(*[ResidualUnit(dim, residual_conv_kernel_size) for _ in range(num_consecutive)])
+ decoder_layer = Sequential(*[ResidualUnit(dim, residual_conv_kernel_size) for _ in range(num_consecutive)])
+
+ elif layer_type == 'cond_residual':
+ assert exists(dim_cond), 'dim_cond must be passed into VideoTokenizer, if tokenizer is to be conditioned'
+
+ has_cond = True
+
+ encoder_layer = ResidualUnitMod(dim, residual_conv_kernel_size, dim_cond = int(dim_cond * dim_cond_expansion_factor))
+ decoder_layer = ResidualUnitMod(dim, residual_conv_kernel_size, dim_cond = int(dim_cond * dim_cond_expansion_factor))
+ dim_out = dim
+
+ elif layer_type == 'compress_space':
+ dim_out = safe_get_index(layer_params, 0)
+ dim_out = default(dim_out, dim * 2) # SCH: if dim_out exists, else use dim * 2
+ dim_out = min(dim_out, max_dim)
+
+ encoder_layer = SpatialDownsample2x(dim, dim_out) # SCH: 2d conv in space dimensions
+ decoder_layer = SpatialUpsample2x(dim_out, dim) # SCH: 2d conv in space dimensions, use more channel to expand space dim
+
+ assert layer_fmap_size > 1
+ layer_fmap_size //= 2
+
+ elif layer_type == 'compress_time':
+ dim_out = safe_get_index(layer_params, 0)
+ dim_out = default(dim_out, dim * 2)
+ dim_out = min(dim_out, max_dim)
+
+ encoder_layer = TimeDownsample2x(dim, dim_out) # SCH: 1d conv in time dim to reduce
+ decoder_layer = TimeUpsample2x(dim_out, dim) # SCH: 1d conv in time dim, use more channels to expand time dim
+
+ time_downsample_factor *= 2
+
+ elif layer_type == 'attend_space':
+ attn_kwargs = dict(
+ dim = dim,
+ dim_head = attn_dim_head,
+ heads = attn_heads,
+ dropout = attn_dropout,
+ flash = flash_attn
+ )
+
+ encoder_layer = Sequential(
+ Residual(SpaceAttention(**attn_kwargs)),
+ Residual(FeedForward(dim))
+ )
+
+ decoder_layer = Sequential(
+ Residual(SpaceAttention(**attn_kwargs)),
+ Residual(FeedForward(dim))
+ )
+
+ elif layer_type == 'linear_attend_space':
+ linear_attn_kwargs = dict(
+ dim = dim,
+ dim_head = linear_attn_dim_head,
+ heads = linear_attn_heads
+ )
+
+ encoder_layer = Sequential(
+ Residual(LinearSpaceAttention(**linear_attn_kwargs)),
+ Residual(FeedForward(dim))
+ )
+
+ decoder_layer = Sequential(
+ Residual(LinearSpaceAttention(**linear_attn_kwargs)),
+ Residual(FeedForward(dim))
+ )
+
+ elif layer_type == 'gateloop_time':
+ gateloop_kwargs = dict(
+ use_heinsen = False
+ )
+
+ encoder_layer = ToTimeSequence(Residual(SimpleGateLoopLayer(dim = dim)))
+ decoder_layer = ToTimeSequence(Residual(SimpleGateLoopLayer(dim = dim)))
+
+ elif layer_type == 'attend_time':
+ attn_kwargs = dict(
+ dim = dim,
+ dim_head = attn_dim_head,
+ heads = attn_heads,
+ dropout = attn_dropout,
+ causal = True,
+ flash = flash_attn
+ )
+
+ encoder_layer = Sequential(
+ Residual(TokenShift(TimeAttention(**attn_kwargs))),
+ Residual(TokenShift(FeedForward(dim, dim_cond = dim_cond)))
+ )
+
+ decoder_layer = Sequential(
+ Residual(TokenShift(TimeAttention(**attn_kwargs))),
+ Residual(TokenShift(FeedForward(dim, dim_cond = dim_cond)))
+ )
+
+ elif layer_type == 'cond_attend_space':
+ has_cond = True
+
+ attn_kwargs = dict(
+ dim = dim,
+ dim_cond = dim_cond,
+ dim_head = attn_dim_head,
+ heads = attn_heads,
+ dropout = attn_dropout,
+ flash = flash_attn
+ )
+
+ encoder_layer = Sequential(
+ Residual(SpaceAttention(**attn_kwargs)),
+ Residual(FeedForward(dim))
+ )
+
+ decoder_layer = Sequential(
+ Residual(SpaceAttention(**attn_kwargs)),
+ Residual(FeedForward(dim))
+ )
+
+ elif layer_type == 'cond_linear_attend_space':
+ has_cond = True
+
+ attn_kwargs = dict(
+ dim = dim,
+ dim_cond = dim_cond,
+ dim_head = attn_dim_head,
+ heads = attn_heads,
+ dropout = attn_dropout,
+ flash = flash_attn
+ )
+
+ encoder_layer = Sequential(
+ Residual(LinearSpaceAttention(**attn_kwargs)),
+ Residual(FeedForward(dim, dim_cond = dim_cond))
+ )
+
+ decoder_layer = Sequential(
+ Residual(LinearSpaceAttention(**attn_kwargs)),
+ Residual(FeedForward(dim, dim_cond = dim_cond))
+ )
+
+ elif layer_type == 'cond_attend_time':
+ has_cond = True
+
+ attn_kwargs = dict(
+ dim = dim,
+ dim_cond = dim_cond,
+ dim_head = attn_dim_head,
+ heads = attn_heads,
+ dropout = attn_dropout,
+ causal = True,
+ flash = flash_attn
+ )
+
+ encoder_layer = Sequential(
+ Residual(TokenShift(TimeAttention(**attn_kwargs))),
+ Residual(TokenShift(FeedForward(dim, dim_cond = dim_cond)))
+ )
+
+ decoder_layer = Sequential(
+ Residual(TokenShift(TimeAttention(**attn_kwargs))),
+ Residual(TokenShift(FeedForward(dim, dim_cond = dim_cond)))
+ )
+
+ else:
+ raise ValueError(f'unknown layer type {layer_type}')
+
+ self.encoder_layers.append(encoder_layer)
+ self.decoder_layers.insert(0, decoder_layer)
+
+ dim = dim_out
+ has_cond_across_layers.append(has_cond)
+
+ # add a final norm just before quantization layer
+
+ self.encoder_layers.append(Sequential(
+ Rearrange('b c ... -> b ... c'),
+ nn.LayerNorm(dim),
+ Rearrange('b ... c -> b c ...'),
+ ))
+
+ self.time_downsample_factor = time_downsample_factor
+ self.time_padding = time_downsample_factor - 1
+
+ self.fmap_size = layer_fmap_size
+
+ # use a MLP stem for conditioning, if needed
+
+ self.has_cond_across_layers = has_cond_across_layers
+ self.has_cond = any(has_cond_across_layers)
+
+ self.encoder_cond_in = nn.Identity()
+ self.decoder_cond_in = nn.Identity()
+
+ if has_cond:
+ self.dim_cond = dim_cond
+
+ self.encoder_cond_in = Sequential(
+ nn.Linear(dim_cond, int(dim_cond * dim_cond_expansion_factor)),
+ nn.SiLU()
+ )
+
+ self.decoder_cond_in = Sequential(
+ nn.Linear(dim_cond, int(dim_cond * dim_cond_expansion_factor)),
+ nn.SiLU()
+ )
+
+ ## SCH: remove quantizer
+ # # quantizer related
+
+ # self.use_fsq = use_fsq
+
+ # if not use_fsq:
+ # assert exists(codebook_size) and not exists(fsq_levels), 'if use_fsq is set to False, `codebook_size` must be set (and not `fsq_levels`)'
+
+ # # lookup free quantizer(s) - multiple codebooks is possible
+ # # each codebook will get its own entropy regularization
+
+ # self.quantizers = LFQ(
+ # dim = dim,
+ # codebook_size = codebook_size,
+ # num_codebooks = num_codebooks,
+ # entropy_loss_weight = lfq_entropy_loss_weight,
+ # commitment_loss_weight = lfq_commitment_loss_weight,
+ # diversity_gamma = lfq_diversity_gamma
+ # )
+
+ # else:
+ # assert not exists(codebook_size) and exists(fsq_levels), 'if use_fsq is set to True, `fsq_levels` must be set (and not `codebook_size`). the effective codebook size is the cumulative product of all the FSQ levels'
+
+ # self.quantizers = FSQ(
+ # fsq_levels,
+ # dim = dim,
+ # num_codebooks = num_codebooks
+ # )
+
+ # self.quantizer_aux_loss_weight = quantizer_aux_loss_weight
+
+ # dummy loss
+
+ self.register_buffer('zero', torch.tensor(0.), persistent = False)
+
+ # perceptual loss related
+
+ use_vgg = channels in {1, 3, 4} and perceptual_loss_weight > 0.
+
+ self.vgg = None
+ self.perceptual_loss_weight = perceptual_loss_weight
+
+ if use_vgg:
+ if not exists(vgg):
+ vgg = torchvision.models.vgg16(
+ weights = vgg_weights
+ )
+
+ vgg.classifier = Sequential(*vgg.classifier[:-2])
+
+ self.vgg = vgg
+
+ self.use_vgg = use_vgg
+
+ # main flag for whether to use GAN at all
+
+ self.use_gan = use_gan
+
+ # discriminator
+
+ discr_kwargs = default(discr_kwargs, dict(
+ dim = dim,
+ image_size = image_size,
+ channels = channels,
+ max_dim = 512
+ ))
+
+ self.discr = Discriminator(**discr_kwargs)
+
+ self.adversarial_loss_weight = adversarial_loss_weight
+ self.grad_penalty_loss_weight = grad_penalty_loss_weight
+
+ self.has_gan = use_gan and adversarial_loss_weight > 0.
+
+ # multi-scale discriminators
+
+ self.has_multiscale_gan = use_gan and multiscale_adversarial_loss_weight > 0.
+
+ self.multiscale_discrs = ModuleList([*multiscale_discrs])
+
+ self.multiscale_adversarial_loss_weight = multiscale_adversarial_loss_weight
+
+ self.has_multiscale_discrs = (
+ use_gan and \
+ multiscale_adversarial_loss_weight > 0. and \
+ len(multiscale_discrs) > 0
+ )
+
+ @property
+ def device(self):
+ return self.zero.device
+
+ @classmethod
+ def init_and_load_from(cls, path, strict = True):
+ path = Path(path)
+ assert path.exists()
+ pkg = torch.load(str(path), map_location = 'cpu')
+
+ assert 'config' in pkg, 'model configs were not found in this saved checkpoint'
+
+ config = pickle.loads(pkg['config'])
+ tokenizer = cls(**config)
+ tokenizer.load(path, strict = strict)
+ return tokenizer
+
+ def parameters(self):
+ return [
+ *self.conv_in.parameters(),
+ *self.conv_in_first_frame.parameters(),
+ *self.conv_out_first_frame.parameters(),
+ *self.conv_out.parameters(),
+ *self.encoder_layers.parameters(),
+ *self.decoder_layers.parameters(),
+ *self.encoder_cond_in.parameters(),
+ *self.decoder_cond_in.parameters(),
+ *self.quantizers.parameters()
+ ]
+
+ def discr_parameters(self):
+ return self.discr.parameters()
+
+ def copy_for_eval(self):
+ device = self.device
+ vae_copy = copy.deepcopy(self.cpu())
+
+ maybe_del_attr_(vae_copy, 'discr')
+ maybe_del_attr_(vae_copy, 'vgg')
+ maybe_del_attr_(vae_copy, 'multiscale_discrs')
+
+ vae_copy.eval()
+ return vae_copy.to(device)
+
+ @remove_vgg
+ def state_dict(self, *args, **kwargs):
+ return super().state_dict(*args, **kwargs)
+
+ @remove_vgg
+ def load_state_dict(self, *args, **kwargs):
+ return super().load_state_dict(*args, **kwargs)
+
+ def save(self, path, overwrite = True):
+ path = Path(path)
+ assert overwrite or not path.exists(), f'{str(path)} already exists'
+
+ pkg = dict(
+ model_state_dict = self.state_dict(),
+ version = __version__,
+ config = self._configs
+ )
+
+ torch.save(pkg, str(path))
+
+ def load(self, path, strict = True):
+ path = Path(path)
+ assert path.exists()
+
+ pkg = torch.load(str(path))
+ state_dict = pkg.get('model_state_dict')
+ version = pkg.get('version')
+
+ assert exists(state_dict)
+
+ if exists(version):
+ print(f'loading checkpointed tokenizer from version {version}')
+
+ self.load_state_dict(state_dict, strict = strict)
+
+ @beartype
+ def encode(
+ self,
+ video: Tensor,
+ quantize = False,
+ cond: Optional[Tensor] = None,
+ video_contains_first_frame = True
+ ):
+ """
+ SCH: conv (may sep 1st frame), then pass through self.encoder_layers, then quantize if needed, finish
+ """
+ encode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame
+
+ # whether to pad video or not
+
+ if video_contains_first_frame:
+ video_len = video.shape[2]
+
+ video = pad_at_dim(video, (self.time_padding, 0), value = 0., dim = 2)
+ video_packed_shape = [torch.Size([self.time_padding]), torch.Size([]), torch.Size([video_len - 1])]
+
+ # conditioning, if needed
+
+ assert (not self.has_cond) or exists(cond), '`cond` must be passed into tokenizer forward method since conditionable layers were specified'
+
+ if exists(cond):
+ assert cond.shape == (video.shape[0], self.dim_cond)
+
+ cond = self.encoder_cond_in(cond)
+ cond_kwargs = dict(cond = cond)
+
+ # initial conv
+ # taking into account whether to encode first frame separately
+
+ if encode_first_frame_separately:
+ pad, first_frame, video = unpack(video, video_packed_shape, 'b c * h w')
+ first_frame = self.conv_in_first_frame(first_frame)
+
+ video = self.conv_in(video)
+
+ if encode_first_frame_separately:
+ video, _ = pack([first_frame, video], 'b c * h w')
+ video = pad_at_dim(video, (self.time_padding, 0), dim = 2)
+
+ # encoder layers
+
+ for fn, has_cond in zip(self.encoder_layers, self.has_cond_across_layers):
+
+ layer_kwargs = dict()
+
+ if has_cond:
+ layer_kwargs = cond_kwargs
+
+ video = fn(video, **layer_kwargs)
+
+ maybe_quantize = identity if not quantize else self.quantizers
+
+ return maybe_quantize(video)
+
+ # @beartype
+ # def decode_from_code_indices(
+ # self,
+ # codes: Tensor,
+ # cond: Optional[Tensor] = None,
+ # video_contains_first_frame = True
+ # ):
+ # assert codes.dtype in (torch.long, torch.int32)
+
+ # if codes.ndim == 2:
+ # video_code_len = codes.shape[-1]
+ # assert divisible_by(video_code_len, self.fmap_size ** 2), f'flattened video ids must have a length ({video_code_len}) that is divisible by the fmap size ({self.fmap_size}) squared ({self.fmap_size ** 2})'
+
+ # codes = rearrange(codes, 'b (f h w) -> b f h w', h = self.fmap_size, w = self.fmap_size)
+
+ # quantized = self.quantizers.indices_to_codes(codes)
+
+ # return self.decode(quantized, cond = cond, video_contains_first_frame = video_contains_first_frame)
+
+ @beartype
+ def decode(
+ self,
+ quantized: Tensor,
+ cond: Optional[Tensor] = None,
+ video_contains_first_frame = True
+ ):
+ decode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame
+
+ batch = quantized.shape[0]
+
+ # conditioning, if needed
+
+ assert (not self.has_cond) or exists(cond), '`cond` must be passed into tokenizer forward method since conditionable layers were specified'
+
+ if exists(cond): # SCH: quantized latents used as control signal following StyleGAN?
+ assert cond.shape == (batch, self.dim_cond)
+
+ cond = self.decoder_cond_in(cond) # SCH: linear + activation
+ cond_kwargs = dict(cond = cond)
+
+ # decoder layers
+
+ x = quantized
+
+ for fn, has_cond in zip(self.decoder_layers, reversed(self.has_cond_across_layers)):
+
+ layer_kwargs = dict()
+
+ if has_cond:
+ layer_kwargs = cond_kwargs
+
+ x = fn(x, **layer_kwargs)
+
+ # to pixels
+
+ if decode_first_frame_separately:
+ left_pad, xff, x = x[:, :, :self.time_padding], x[:, :, self.time_padding], x[:, :, (self.time_padding + 1):]
+
+ out = self.conv_out(x)
+ outff = self.conv_out_first_frame(xff)
+
+ video, _ = pack([outff, out], 'b c * h w')
+
+ else:
+ video = self.conv_out(x)
+
+ # if video were padded, remove padding
+
+ if video_contains_first_frame:
+ video = video[:, :, self.time_padding:]
+
+ return video
+
+ @torch.no_grad()
+ def tokenize(self, video):
+ self.eval()
+ return self.forward(video, return_codes = True)
+
+ @beartype
+ def forward(
+ self,
+ video_or_images: Tensor,
+ cond: Optional[Tensor] = None,
+ return_loss = False,
+ return_codes = False,
+ return_recon = False,
+ return_discr_loss = False,
+ return_recon_loss_only = False,
+ apply_gradient_penalty = True,
+ video_contains_first_frame = True,
+ adversarial_loss_weight = None,
+ multiscale_adversarial_loss_weight = None
+ ):
+ adversarial_loss_weight = default(adversarial_loss_weight, self.adversarial_loss_weight)
+ multiscale_adversarial_loss_weight = default(multiscale_adversarial_loss_weight, self.multiscale_adversarial_loss_weight)
+
+ assert (return_loss + return_codes + return_discr_loss) <= 1
+ assert video_or_images.ndim in {4, 5}
+
+ assert video_or_images.shape[-2:] == (self.image_size, self.image_size)
+
+ # accept images for image pretraining (curriculum learning from images to video)
+
+ is_image = video_or_images.ndim == 4
+
+ if is_image:
+ video = rearrange(video_or_images, 'b c ... -> b c 1 ...')
+ video_contains_first_frame = True
+ else:
+ video = video_or_images
+
+ batch, channels, frames = video.shape[:3]
+
+ assert divisible_by(frames - int(video_contains_first_frame), self.time_downsample_factor), f'number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}'
+
+ # encoder
+
+ x = self.encode(video, cond = cond, video_contains_first_frame = video_contains_first_frame)
+
+
+ ## SCH: remove the codebook
+ # # lookup free quantization
+
+ # if self.use_fsq:
+ # quantized, codes = self.quantizers(x)
+
+ # aux_losses = self.zero
+ # quantizer_loss_breakdown = None
+ # else:
+ # (quantized, codes, aux_losses), quantizer_loss_breakdown = self.quantizers(x, return_loss_breakdown = True)
+
+ # if return_codes and not return_recon:
+ # return codes
+
+ # decoder
+ recon_video = self.decode(x, cond = cond, video_contains_first_frame = video_contains_first_frame)
+
+ # if return_codes:
+ # return codes, recon_video
+
+ # reconstruction loss
+
+ if not (return_loss or return_discr_loss or return_recon_loss_only):
+ return recon_video
+
+ recon_loss = F.mse_loss(video, recon_video)
+
+ # for validation, only return recon loss
+
+ if return_recon_loss_only:
+ return recon_loss, recon_video
+
+ # TODO:
+ # gan discriminator loss
+
+ if return_discr_loss:
+ assert self.has_gan
+ assert exists(self.discr)
+
+ # pick a random frame for image discriminator
+
+ frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices
+
+ real = pick_video_frame(video, frame_indices)
+
+ if apply_gradient_penalty:
+ real = real.requires_grad_()
+
+ fake = pick_video_frame(recon_video, frame_indices)
+
+ real_logits = self.discr(real)
+ fake_logits = self.discr(fake.detach())
+
+ discr_loss = hinge_discr_loss(fake_logits, real_logits)
+
+ # multiscale discriminators
+
+ multiscale_discr_losses = []
+
+ if self.has_multiscale_discrs:
+ for discr in self.multiscale_discrs:
+ multiscale_real_logits = discr(video)
+ multiscale_fake_logits = discr(recon_video.detach())
+
+ multiscale_discr_loss = hinge_discr_loss(multiscale_fake_logits, multiscale_real_logits)
+
+ multiscale_discr_losses.append(multiscale_discr_loss)
+ else:
+ multiscale_discr_losses.append(self.zero)
+
+ # gradient penalty
+
+ if apply_gradient_penalty:
+ gradient_penalty_loss = gradient_penalty(real, real_logits)
+ else:
+ gradient_penalty_loss = self.zero
+
+ # total loss
+
+ total_loss = discr_loss + \
+ gradient_penalty_loss * self.grad_penalty_loss_weight + \
+ sum(multiscale_discr_losses) * self.multiscale_adversarial_loss_weight
+
+ discr_loss_breakdown = DiscrLossBreakdown(
+ discr_loss,
+ multiscale_discr_losses,
+ gradient_penalty_loss
+ )
+
+ return total_loss, discr_loss_breakdown
+
+ # perceptual loss
+
+ if self.use_vgg:
+
+ frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices
+
+ input_vgg_input = pick_video_frame(video, frame_indices)
+ recon_vgg_input = pick_video_frame(recon_video, frame_indices)
+
+ if channels == 1:
+ input_vgg_input = repeat(input_vgg_input, 'b 1 h w -> b c h w', c = 3)
+ recon_vgg_input = repeat(recon_vgg_input, 'b 1 h w -> b c h w', c = 3)
+
+ elif channels == 4:
+ input_vgg_input = input_vgg_input[:, :3]
+ recon_vgg_input = recon_vgg_input[:, :3]
+
+ input_vgg_feats = self.vgg(input_vgg_input)
+ recon_vgg_feats = self.vgg(recon_vgg_input)
+
+ perceptual_loss = F.mse_loss(input_vgg_feats, recon_vgg_feats)
+ else:
+ perceptual_loss = self.zero
+
+ # get gradient with respect to perceptual loss for last decoder layer
+ # needed for adaptive weighting
+
+ last_dec_layer = self.conv_out.conv.weight
+
+ norm_grad_wrt_perceptual_loss = None
+
+ if self.training and self.use_vgg and (self.has_gan or self.has_multiscale_discrs):
+ norm_grad_wrt_perceptual_loss = grad_layer_wrt_loss(perceptual_loss, last_dec_layer).norm(p = 2)
+
+ # per-frame image discriminator
+
+ recon_video_frames = None
+
+ if self.has_gan:
+ frame_indices = torch.randn((batch, frames)).topk(1, dim = -1).indices
+ recon_video_frames = pick_video_frame(recon_video, frame_indices)
+
+ fake_logits = self.discr(recon_video_frames)
+ gen_loss = hinge_gen_loss(fake_logits)
+
+ adaptive_weight = 1.
+
+ if exists(norm_grad_wrt_perceptual_loss):
+ norm_grad_wrt_gen_loss = grad_layer_wrt_loss(gen_loss, last_dec_layer).norm(p = 2)
+ adaptive_weight = norm_grad_wrt_perceptual_loss / norm_grad_wrt_gen_loss.clamp(min = 1e-3)
+ adaptive_weight.clamp_(max = 1e3)
+
+ if torch.isnan(adaptive_weight).any():
+ adaptive_weight = 1.
+ else:
+ gen_loss = self.zero
+ adaptive_weight = 0.
+
+ # multiscale discriminator losses
+
+ multiscale_gen_losses = []
+ multiscale_gen_adaptive_weights = []
+
+ if self.has_multiscale_gan and self.has_multiscale_discrs:
+ if not exists(recon_video_frames):
+ recon_video_frames = pick_video_frame(recon_video, frame_indices)
+
+ for discr in self.multiscale_discrs:
+ fake_logits = recon_video_frames
+ multiscale_gen_loss = hinge_gen_loss(fake_logits)
+
+ multiscale_gen_losses.append(multiscale_gen_loss)
+
+ multiscale_adaptive_weight = 1.
+
+ if exists(norm_grad_wrt_perceptual_loss):
+ norm_grad_wrt_gen_loss = grad_layer_wrt_loss(multiscale_gen_loss, last_dec_layer).norm(p = 2)
+ multiscale_adaptive_weight = norm_grad_wrt_perceptual_loss / norm_grad_wrt_gen_loss.clamp(min = 1e-5)
+ multiscale_adaptive_weight.clamp_(max = 1e3)
+
+ multiscale_gen_adaptive_weights.append(multiscale_adaptive_weight)
+
+ # calculate total loss
+
+ total_loss = recon_loss \
+ + aux_losses * self.quantizer_aux_loss_weight \
+ + perceptual_loss * self.perceptual_loss_weight \
+ + gen_loss * adaptive_weight * adversarial_loss_weight
+
+ if self.has_multiscale_discrs:
+
+ weighted_multiscale_gen_losses = sum(loss * weight for loss, weight in zip(multiscale_gen_losses, multiscale_gen_adaptive_weights))
+
+ total_loss = total_loss + weighted_multiscale_gen_losses * multiscale_adversarial_loss_weight
+
+ # loss breakdown
+
+ loss_breakdown = LossBreakdown(
+ recon_loss,
+ aux_losses,
+ quantizer_loss_breakdown,
+ perceptual_loss,
+ gen_loss,
+ adaptive_weight,
+ multiscale_gen_losses,
+ multiscale_gen_adaptive_weights
+ )
+
+ return total_loss, loss_breakdown
+
+# main class
+class MagViT2(Module):
+ def __init__(self):
+ super().__init__()
+
+ def forward(self, x):
+ return x
diff --git a/opensora/models/vae/vae_3d.py b/opensora/models/vae/vae_3d.py
index 58ec853..29abaec 100644
--- a/opensora/models/vae/vae_3d.py
+++ b/opensora/models/vae/vae_3d.py
@@ -80,12 +80,12 @@ class Encoder(nn.Module):
"""Encoder Blocks."""
def __init__(self,
filters = 64,
- num_res_blocks = 3,
+ num_res_blocks = 2,
channel_multipliers = (1, 2, 2, 4),
temporal_downsample = (True, True, False),
num_groups = 32, # for nn.GroupNorm
in_out_channels = 3, # SCH: added, in_channels at the start
- latent_embed_dim = 256,
+ latent_embed_dim = 256, # num channels for latent vector
conv_downsample = False,
custom_conv_padding = None,
activation_fn = 'swish',
diff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.py
new file mode 100644
index 0000000..d9fa2ff
--- /dev/null
+++ b/opensora/models/vae/vae_3d_v2.py
@@ -0,0 +1,464 @@
+import functools
+from typing import Any, Dict, Tuple, Type, Union, Sequence, Optional
+from absl import logging
+import torch
+import torch.nn as nn
+import numpy as np
+from numpy import typing as nptyping
+from opensora.models.vae import model_utils
+from opensora.registry import MODELS
+from opensora.utils.ckpt_utils import load_checkpoint
+from einops.layers.torch import Rearrange # NOTE: this is a torch module
+from einops import rearrange
+
+"""Encoder and Decoder stuctures with 3D CNNs."""
+
+"""
+NOTE:
+ removed LayerNorm since not used in this arch
+ GroupNorm: flax uses default `epsilon=1e-06`, whereas torch uses `eps=1e-05`
+ for average pool and upsample, input shape needs to be [N,C,T,H,W] --> if not, adjust the scale factors accordingly
+
+ !!! opensora read video into [B,C,T,H,W] format output
+
+TODO:
+ check data dimensions format
+"""
+def cast_tuple(t, length = 1):
+ return t if isinstance(t, tuple) else ((t,) * length)
+
+def divisible_by(num, den):
+ return (num % den) == 0
+
+def is_odd(n):
+ return not divisible_by(n, 2)
+
+class CausalConv3d(nn.Module):
+ def __init__(
+ self,
+ chan_in,
+ chan_out,
+ kernel_size: Union[int, Tuple[int, int, int]],
+ pad_mode = 'constant',
+ **kwargs
+ ):
+ super().__init__()
+ kernel_size = cast_tuple(kernel_size, 3)
+
+ time_kernel_size, height_kernel_size, width_kernel_size = kernel_size
+
+ assert is_odd(height_kernel_size) and is_odd(width_kernel_size)
+
+ dilation = kwargs.pop('dilation', 1)
+ stride = kwargs.pop('stride', 1)
+
+ self.pad_mode = pad_mode
+ time_pad = dilation * (time_kernel_size - 1) + (1 - stride)
+ height_pad = height_kernel_size // 2
+ width_pad = width_kernel_size // 2
+
+ self.time_pad = time_pad
+ self.time_causal_padding = (width_pad, width_pad, height_pad, height_pad, time_pad, 0)
+
+ stride = (stride, 1, 1)
+ dilation = (dilation, 1, 1)
+ self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride = stride, dilation = dilation, **kwargs)
+
+ def forward(self, x):
+ pad_mode = self.pad_mode if self.time_pad < x.shape[2] else 'constant'
+
+ x = nn.F.pad(x, self.time_causal_padding, mode = pad_mode)
+ return self.conv(x)
+
+# TODO: CausalConvTranspose3d
+
+class ResBlock(nn.Module):
+ def __init__(
+ self,
+ in_channels, # SCH: added
+ filters,
+ conv_fn,
+ activation_fn=nn.SiLU,
+ use_conv_shortcut=False,
+ num_groups=32,
+ device="cpu",
+ dtype=torch.bfloat16,
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ self.filters = filters
+ self.activate = activation_fn()
+ self.use_conv_shortcut = use_conv_shortcut
+
+ # SCH: MAGVIT uses GroupNorm by default
+ self.norm1 = nn.GroupNorm(num_groups, in_channels, device=device, dtype=dtype)
+ self.conv1 = conv_fn(in_channels, self.filters, kernel_size=(3, 3, 3), bias=False)
+ self.norm2 = nn.GroupNorm(num_groups, self.filters, device=device, dtype=dtype)
+ self.conv2 = conv_fn(self.filters, self.filters, kernel_size=(3, 3, 3), bias=False)
+ if in_channels != filters:
+ if self.use_conv_shortcut:
+ self.conv3 = conv_fn(in_channels, self.filters, kernel_size=(3, 3, 3), bias=False)
+ else:
+ self.conv3 = conv_fn(in_channels, self.filters, kernel_size=(1, 1, 1), bias=False)
+
+
+ def forward(self, x):
+ # device, dtype = x.device, x.dtype
+ # input_dim = x.shape[1]
+ residual = x
+ x = self.norm1(x)
+ x = self.activate(x)
+ x = self.conv1(x)
+ x = self.norm2(x)
+ x = self.activate(x)
+ x = self.conv2(x)
+ if self.in_channels != self.filters: # SCH: ResBlock X->Y
+ residual = self.conv3(residual)
+ return x + residual
+
+
+class Encoder(nn.Module):
+ """Encoder Blocks."""
+ def __init__(self,
+ filters = 128,
+ num_res_blocks = 4,
+ channel_multipliers = (1, 2, 2, 4),
+ temporal_downsample = (False, True, True),
+ num_groups = 32, # for nn.GroupNorm
+ in_out_channels = 3, # SCH: added, in_channels at the start
+ latent_embed_dim = 512, # num channels for latent vector
+ conv_downsample = False,
+ custom_conv_padding = None,
+ activation_fn = 'swish',
+ device="cpu",
+ dtype=torch.bfloat16,
+ ):
+ super().__init__()
+ self.filters = filters
+ self.num_res_blocks = num_res_blocks
+ self.channel_multipliers = channel_multipliers
+ self.temporal_downsample = temporal_downsample
+ self.num_groups = num_groups
+
+ self.embedding_dim = latent_embed_dim
+ self.conv_downsample = conv_downsample
+ self.custom_conv_padding = custom_conv_padding
+
+ if activation_fn == 'relu':
+ self.activation_fn = nn.ReLU
+ elif activation_fn == 'swish':
+ self.activation_fn = nn.SiLU
+ else:
+ raise NotImplementedError
+ self.activate = self.activation_fn()
+
+ self.conv_fn = functools.partial(
+ CausalConv3d,
+ dtype=dtype,
+ padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorch
+ device=device,
+ )
+
+ self.block_args = dict(
+ conv_fn=self.conv_fn,
+ dtype=dtype,
+ activation_fn=self.activation_fn,
+ use_conv_shortcut=False,
+ num_groups=self.num_groups,
+ device=device,
+ )
+
+ self.conv1 = self.conv_fn(in_out_channels, self.filters, kernel_size=(3, 3, 3), bias=False)
+
+ # ResBlocks and conv downsample
+ self.block_res_blocks = []
+ self.num_blocks = len(self.channel_multipliers)
+ self.conv_blocks = []
+
+ filters = self.filters
+ prev_filters = filters # record for in_channels
+ for i in range(self.num_blocks):
+ # resblock handling
+ filters = self.filters * self.channel_multipliers[i] # SCH: determine the number out_channels
+ block_items = []
+ for _ in range(self.num_res_blocks):
+ block_items.append(ResBlock(prev_filters, filters, **self.block_args))
+ prev_filters = filters # update in_channels
+ self.block_res_blocks.append(block_items)
+
+ if i < self.num_blocks - 1: # SCH: T-Causal Conv 3x3x3, 128->128, stride t x 2 x 2
+ t_stride = 2 if self.temporal_downsample[i] else 1
+ self.conv_blocks.append(self.conv_fn(prev_filters, filters, kernel_size=(3, 3, 3), strides=(t_stride, 2, 2))) # SCH: should be same in_channel and out_channel
+ prev_filters = filters # update in_channels
+
+
+ # last layer res block
+ self.res_blocks = []
+ for _ in range(self.num_res_blocks):
+ self.res_blocks.append(ResBlock(prev_filters, filters, **self.block_args))
+ prev_filters = filters # update in_channels
+
+ # MAGVIT uses Group Normalization
+ self.norm1 = nn.GroupNorm(self.num_groups, prev_filters, dtype=dtype, device=device) # SCH: separate <prev_filters> channels into 32 groups
+
+ # TODO: check if this is correct ??
+ self.conv2 = nn.Conv3d(prev_filters, self.embedding_dim, kernel_size=(1, 1, 1), dtype=dtype, device=device, padding="same")
+
+ def forward(self, x):
+ # dtype, device = x.dtype, x.device
+ x = self.conv1(x)
+ for i in range(self.num_blocks):
+ for j in range(self.num_res_blocks):
+ x = self.block_res_blocks[i][j](x)
+
+ if i < self.num_blocks - 1:
+ x = self.conv_blocks[i](x)
+
+
+ for i in range(self.num_res_blocks):
+ x = self.res_blocks[i](x)
+
+ x = self.norm1(x)
+ x = self.activate(x)
+ x = self.conv2(x)
+ return x
+
+
+
+class Decoder(nn.Module):
+ """Decoder Blocks."""
+ def __init__(self,
+ latent_embed_dim = 512,
+ filters = 128,
+ in_out_channels = 4,
+ num_res_blocks = 4,
+ channel_multipliers = (1, 2, 2, 4),
+ temporal_downsample = (False, True, True),
+ num_groups = 32, # for nn.GroupNorm
+ # upsample = "nearest+conv", # options: "deconv", "nearest+conv"
+ custom_conv_padding = None,
+ activation_fn = 'swish',
+ device="cpu",
+ dtype=torch.bfloat16,
+ ):
+ super().__init__()
+ self.output_dim = in_out_channels
+ self.embedding_dim = latent_embed_dim
+ self.filters = filters
+ self.num_res_blocks = num_res_blocks
+ self.channel_multipliers = channel_multipliers
+ self.temporal_downsample = temporal_downsample
+ self.num_groups = num_groups
+
+ # self.upsample = upsample
+ self.custom_conv_padding = custom_conv_padding
+ # self.norm_type = self.config.vqvae.norm_type
+ # self.num_remat_block = self.config.vqvae.get('num_dec_remat_blocks', 0)
+
+ if activation_fn == 'relu':
+ self.activation_fn = nn.ReLU
+ elif activation_fn == 'swish':
+ self.activation_fn = nn.SiLU
+ else:
+ raise NotImplementedError
+ self.activate = self.activation_fn()
+
+ self.conv_fn = functools.partial(
+ CausalConv3d,
+ dtype=dtype,
+ padding='valid' if self.custom_conv_padding is not None else 'same', # SCH: lower letter for pytorch
+ device=device,
+ )
+
+ self.block_args = dict(
+ conv_fn=self.conv_fn,
+ activation_fn=self.activation_fn,
+ use_conv_shortcut=False,
+ num_groups=self.num_groups,
+ device=device,
+ dtype=dtype,
+ )
+ self.num_blocks = len(self.channel_multipliers)
+
+ filters = self.filters * self.channel_multipliers[-1]
+
+ self.conv1 = self.conv_fn(self.embedding_dim, filters, kernel_size=(3, 3, 3), bias=True)
+
+ # last layer res block
+ self.res_blocks = []
+ for _ in range(self.num_res_blocks):
+ self.res_blocks.append(ResBlock(filters, filters, **self.block_args))
+
+ # TODO: do I need to add adaptive GroupNorm in between each block?
+
+ # # NOTE: upsample, dimensions T, H, W
+ # self.upsampler_with_t = nn.Upsample(scale_factor=(2,2,2))
+ # self.upsampler = nn.Upsample(scale_factor=(1,2,2))
+
+ # ResBlocks and conv upsample
+ prev_filters = filters # SCH: in_channels
+ self.block_res_blocks = []
+ self.num_blocks = len(self.channel_multipliers)
+ self.conv_blocks = []
+ # SCH: reverse to keep track of the in_channels, but append also in a reverse direction
+ for i in reversed(range(self.num_blocks)):
+ filters = self.filters * self.channel_multipliers[i]
+ # resblock handling
+ block_items = []
+ for _ in range(self.num_res_blocks):
+ block_items.append(ResBlock(prev_filters, filters, **self.block_args))
+ prev_filters = filters # SCH: update in_channels
+ self.block_res_blocks.insert(0, block_items) # SCH: append in front
+
+ # conv blocks with upsampling
+ if i > 0:
+ t_stride = 2 if self.temporal_downsample[i - 1] else 1
+ # SCH: T-Causal Conv 3x3x3, f -> (t_stride * 2 * 2) * f, depth to space t_stride x 2 x 2
+ self.conv_blocks.insert(0,
+ self.conv_fn(prev_filters, prev_filters * t_stride * 4, kernel_size=(3,3,3))
+ )
+
+ self.norm1 = nn.GroupNorm(self.num_groups, prev_filters, device=device, dtype=dtype)
+ self.conv2 = self.conv_fn(prev_filters, self.output_dim, kernel_size=(3, 3, 3))
+
+
+ def forward(
+ self,
+ x,
+ **kwargs,
+ ):
+ dtype, device = x.dtype, x.device
+ x = self.conv1(x)
+ for i in range(self.num_res_blocks):
+ x = self.res_blocks[i](x)
+ for i in reversed(range(self.num_blocks)): # reverse here to make decoder symmetric with encoder
+ for j in range(self.num_res_blocks):
+ x = self.block_res_blocks[i][j](x)
+
+ if i > 0:
+ t_stride = 2 if self.temporal_downsample[i - 1] else 1
+ # SCH: T-Causal Conv 3x3x3, f -> (t_stride * 2 * 2) * f, depth to space t_stride x 2 x 2
+ x = self.conv_blocks[i-1](x)
+ x = rearrange(x, "B (C ts hs ws) T H W -> B C (T ts) (H hs) (W ws)", ts=t_stride, hs=2, ws=2)
+
+
+ x = self.norm1(x)
+ x = self.activate(x)
+ x = self.conv2(x)
+ return x
+
+
+class VAE_3D(nn.Module):
+ """The 3D VAE """
+ def __init__(
+ self,
+ latent_embed_dim = 256,
+ filters = 64,
+ num_res_blocks = 2,
+ channel_multipliers = (1, 2, 2, 4),
+ temporal_downsample = (True, True, False),
+ num_groups = 32, # for nn.GroupNorm
+ conv_downsample = False,
+ upsample = "nearest+conv", # options: "deconv", "nearest+conv"
+ custom_conv_padding = None,
+ activation_fn = 'swish',
+ in_out_channels = 4,
+ kl_embed_dim = 64,
+ device="cpu",
+ dtype="bf16",
+ # precision: Any = jax.lax.Precision.DEFAULT
+ ):
+ super().__init__()
+
+ if type(dtype) == str:
+ if dtype == "bf16":
+ dtype = torch.bfloat16
+ elif dtype == "fp16":
+ dtype = torch.float16
+ else:
+ raise NotImplementedError(f'dtype: {dtype}')
+
+ # Model Initialization
+
+
+ self.encoder = Encoder(
+ filters=filters,
+ num_res_blocks=num_res_blocks,
+ channel_multipliers=channel_multipliers,
+ temporal_downsample=temporal_downsample,
+ num_groups = num_groups, # for nn.GroupNorm
+ in_out_channels = in_out_channels,
+ latent_embed_dim = latent_embed_dim,
+ conv_downsample = conv_downsample,
+ custom_conv_padding = custom_conv_padding,
+ activation_fn = activation_fn,
+ device=device,
+ dtype=dtype,
+ )
+ self.decoder = Decoder(
+ latent_embed_dim = latent_embed_dim,
+ filters = filters,
+ in_out_channels = in_out_channels,
+ num_res_blocks = num_res_blocks,
+ channel_multipliers = channel_multipliers,
+ temporal_downsample = temporal_downsample,
+ num_groups = num_groups, # for nn.GroupNorm
+ upsample = upsample, # options: "deconv", "nearest+conv"
+ custom_conv_padding = custom_conv_padding,
+ activation_fn = activation_fn,
+ device=device,
+ dtype=dtype,
+ )
+
+ self.quant_conv = nn.Conv3d(latent_embed_dim, 2*kl_embed_dim, 1)
+ self.post_quant_conv = nn.Conv3d(kl_embed_dim, latent_embed_dim, 1)
+
+ image_down = 2 ** len(temporal_downsample)
+ t_down = 2 ** len([x for x in temporal_downsample if x == True])
+ self.patch_size = (t_down, image_down, image_down)
+
+ def get_latent_size(self, input_size):
+ for i in range(len(input_size)):
+ assert input_size[i] % self.patch_size[i] == 0, "Input size must be divisible by patch size"
+ input_size = [input_size[i] // self.patch_size[i] for i in range(3)]
+ return input_size
+
+ def encode(
+ self,
+ x,
+ ):
+ encoded_feature = self.encoder(x)
+ moments = self.quant_conv(encoded_feature).to(x.dtype)
+ posterior = model_utils.DiagonalGaussianDistribution(moments)
+ return posterior
+
+ def decode(
+ self,
+ z,
+ ):
+ dtype = z.dtype
+ z = self.post_quant_conv(z).to(dtype)
+ dec = self.decoder(z)
+ return dec
+
+ def forward(
+ self,
+ input,
+ sample_posterior=True,
+ ):
+ posterior = self.encode(input)
+ if sample_posterior:
+ z = posterior.sample()
+ else:
+ z = posterior.mode()
+ dec = self.decode(z)
+ return dec, posterior
+
+
[email protected]_module("VAE_3D_B")
+def VAE_3D_B(from_pretrained=None, **kwargs):
+ model = VAE_3D(**kwargs)
+ if from_pretrained is not None:
+ load_checkpoint(model, from_pretrained)
+ return model
\ No newline at end of file
diff --git a/scripts/inference-vae.py b/scripts/inference-vae-3d.py
similarity index 98%
rename from scripts/inference-vae.py
rename to scripts/inference-vae-3d.py
index 6b096f2..a624a45 100644
--- a/scripts/inference-vae.py
+++ b/scripts/inference-vae-3d.py
@@ -135,6 +135,7 @@ def main():
running_loss = loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)
if coordinator.is_master():
+ pbar.set_postfix({"loss": loss, "running_loss": running_loss, "step": step})
for idx, sample in enumerate(reconstructions):
pos = step * cfg.batch_size + idx
save_path = os.path.join(save_dir, f"sample_{pos}")
diff --git a/scripts/train-vae.py b/scripts/train-vae-3d.py
similarity index 100%
rename from scripts/train-vae.py
rename to scripts/train-vae-3d.py