Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

inference v2

Shen-Chenhuibecff9d86fa0
diff --git a/configs/vae_3d/inference/16x256x256.py b/configs/vae_3d/inference/16x256x256.py
index 5cda9bd..89591d3 100644
--- a/configs/vae_3d/inference/16x256x256.py
+++ b/configs/vae_3d/inference/16x256x256.py
@@ -53,4 +53,6 @@ load = None
batch_size = 8
grad_clip = 1.0
-lr = 1e-4
\ No newline at end of file
+lr = 1e-4
+
+calc_loss = False
\ No newline at end of file
diff --git a/configs/vae_magvit_v2/inference/16x128x128.py b/configs/vae_magvit_v2/inference/16x128x128.py
new file mode 100644
index 0000000..4ed6d2d
--- /dev/null
+++ b/configs/vae_magvit_v2/inference/16x128x128.py
@@ -0,0 +1,81 @@
+num_frames = 16
+frame_interval = 3
+image_size = (128, 128)
+
+# Define dataset
+root = None
+data_path = "CSV_PATH"
+use_image_transform = False
+num_workers = 4
+video_contains_first_frame = False
+
+# Define acceleration
+dtype = "bf16"
+grad_checkpoint = True
+plugin = "zero2"
+sp_size = 1
+
+
+# Define model
+
+model = dict(
+ type="VAE_MAGVIT_V2",
+ in_out_channels = 3,
+ latent_embed_dim = 256,
+ filters = 128,
+ num_res_blocks = 4,
+ channel_multipliers = (1, 2, 2, 4),
+ temporal_downsample = (False, True, True),
+ num_groups = 32, # for nn.GroupNorm
+ kl_embed_dim = 64,
+ activation_fn = 'swish',
+ separate_first_frame_encoding = False,
+ custom_conv_padding = None
+)
+
+
+discriminator = dict(
+ type="DISCRIMINATOR_3D",
+ image_size = image_size,
+ num_frames = num_frames,
+ in_channels = 3,
+ filters = 128,
+ channel_multipliers = (2,4,4,4,4) # (2,4,4,4) for 64x64 resolution
+)
+
+
+# loss weights
+logvar_init=0.0
+kl_loss_weight = 0.000001
+perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
+discriminator_factor = 1.0 # for discriminator adversarial loss
+# discriminator_loss_weight = 0.5 # for generator adversarial loss
+generator_factor = 0.5 # for generator adversarial loss
+lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight
+discriminator_loss_type="non-saturating"
+generator_loss_type="non-saturating"
+discriminator_start = 50000 # 50000 NOTE: change to correct val, debug use -1 for now
+gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
+ema_decay = 0.999 # ema decay factor for generator
+
+
+# Others
+seed = 42
+outputs = "outputs/samples_v2"
+wandb = False
+
+# Training
+''' NOTE:
+magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
+==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
+3-6 epochs for pexel, from pexel observation its correct
+'''
+
+epochs = 10
+log_every = 1
+ckpt_every = 1000
+load = None
+
+batch_size = 4
+lr = 1e-4
+grad_clip = 1.0
diff --git a/scripts/inference-debug.py b/scripts/inference-debug.py
deleted file mode 100644
index 18183cf..0000000
--- a/scripts/inference-debug.py
+++ /dev/null
@@ -1,151 +0,0 @@
-import os
-
-import colossalai
-import torch
-import torch.distributed as dist
-from colossalai.cluster import DistCoordinator
-from mmengine.runner import set_random_seed
-
-from opensora.acceleration.parallel_states import set_sequence_parallel_group
-from opensora.datasets import save_sample
-from opensora.registry import MODELS, SCHEDULERS, build_module
-from opensora.utils.config_utils import parse_configs
-from opensora.utils.misc import to_torch_dtype
-from opensora.datasets import DatasetFromCSV, get_transforms_image, get_transforms_video, prepare_dataloader
-from opensora.acceleration.parallel_states import (
- get_data_parallel_group,
- set_data_parallel_group,
- set_sequence_parallel_group,
-)
-from tqdm import tqdm
-from opensora.models.vae.model_utils import VEA3DLoss
-
-# DEBUG
-from colossalai.utils import get_current_device
-
-
-def main():
- # ======================================================
- # 1. cfg and init distributed env
- # ======================================================
- cfg = parse_configs(training=False)
- print(cfg)
-
- # init distributed
- colossalai.launch_from_torch({})
- coordinator = DistCoordinator()
-
- # if coordinator.world_size > 1:
- # set_sequence_parallel_group(dist.group.WORLD)
- # enable_sequence_parallelism = True
- # else:
- # enable_sequence_parallelism = False
-
- # ======================================================
- # 2. runtime variables
- # ======================================================
- torch.set_grad_enabled(False)
- # torch.backends.cuda.matmul.allow_tf32 = True
- # torch.backends.cudnn.allow_tf32 = True
- # device = "cuda" if torch.cuda.is_available() else "cpu"
- device = get_current_device()
- dtype = to_torch_dtype(cfg.dtype)
- # set_random_seed(seed=cfg.seed) # Issue is this line !!!!!!!
-
-
-
-
- # ======================================================
- # 3. build dataset and dataloader
- # ======================================================
- dataset = DatasetFromCSV(
- cfg.data_path,
- # TODO: change transforms
- transform=(
- get_transforms_video(cfg.image_size[0])
- if not cfg.use_image_transform
- else get_transforms_image(cfg.image_size[0])
- ),
- num_frames=cfg.num_frames,
- frame_interval=cfg.frame_interval,
- root=cfg.root,
- )
-
- dataloader = prepare_dataloader(
- dataset,
- batch_size=cfg.batch_size,
- num_workers=cfg.num_workers,
- shuffle=True,
- drop_last=True,
- pin_memory=True,
- process_group=get_data_parallel_group(),
- )
- print(f"Dataset contains {len(dataset):,} videos ({cfg.data_path})")
-
- total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size
- print(f"Total batch size: {total_batch_size}")
-
- # ======================================================
- # 4. build model & load weights
- # ======================================================
- # 3.1. build model
- # input_size = (cfg.num_frames, *cfg.image_size)
- vae = build_module(cfg.model, MODELS, device=device)
- # latent_size = vae.get_latent_size(input_size)
-
- # 3.2. move to device & eval
- vae = vae.to(device, dtype).eval()
-
-
- # 3.4. support for multi-resolution
- model_args = dict()
- if cfg.multi_resolution:
- image_size = cfg.image_size
- hw = torch.tensor([image_size], device=device, dtype=dtype).repeat(cfg.batch_size, 1)
- ar = torch.tensor([[image_size[0] / image_size[1]]], device=device, dtype=dtype).repeat(cfg.batch_size, 1)
- model_args["data_info"] = dict(ar=ar, hw=hw)
-
- # ======================================================
- # 4. inference
- # ======================================================
- save_dir = cfg.save_dir
- os.makedirs(save_dir, exist_ok=True)
-
-
- # 4.1. batch generation
-
- # define loss function
- loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)
- running_loss = 0.0
- loss_steps = 0
-
-
- total_steps = len(dataloader)
- dataloader_iter = iter(dataloader)
-
- with tqdm(
- range(total_steps),
- # desc=f"Avg Loss: {running_loss}",
- disable=not coordinator.is_master(),
- total=total_steps,
- initial=0,
- ) as pbar:
- for step in pbar:
- batch = next(dataloader_iter)
- x = batch["video"].to(device, dtype) # [B, C, T, H, W]
- reconstructions, posterior = vae(x)
- loss = loss_function(x, reconstructions, posterior)
- loss_steps += 1
- running_loss = loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)
-
- if coordinator.is_master():
- for idx, sample in enumerate(reconstructions):
- pos = step * cfg.batch_size + idx
- save_path = os.path.join(save_dir, f"sample_{pos}")
- save_sample(sample, fps=cfg.fps, save_path=save_path)
-
- print("test loss:", running_loss)
-
-
-if __name__ == "__main__":
- main()
diff --git a/scripts/inference-vae-v2.py b/scripts/inference-vae-v2.py
new file mode 100644
index 0000000..47e4159
--- /dev/null
+++ b/scripts/inference-vae-v2.py
@@ -0,0 +1,240 @@
+import os
+
+import colossalai
+import torch
+import torch.distributed as dist
+from colossalai.cluster import DistCoordinator
+from mmengine.runner import set_random_seed
+
+from opensora.acceleration.parallel_states import set_sequence_parallel_group
+from opensora.datasets import save_sample
+from opensora.registry import MODELS, SCHEDULERS, build_module
+from opensora.utils.config_utils import parse_configs
+from opensora.utils.misc import to_torch_dtype
+from opensora.datasets import DatasetFromCSV, get_transforms_image, get_transforms_video, prepare_dataloader
+from opensora.acceleration.parallel_states import (
+ get_data_parallel_group,
+ set_data_parallel_group,
+ set_sequence_parallel_group,
+)
+from tqdm import tqdm
+from opensora.models.vae.vae_3d_v2 import VEALoss, DiscriminatorLoss, AdversarialLoss, pad_at_dim
+
+from einops import rearrange
+from colossalai.utils import get_current_device
+
+
+def main():
+ # ======================================================
+ # 1. cfg and init distributed env
+ # ======================================================
+ cfg = parse_configs(training=False)
+ print(cfg)
+
+ # init distributed
+ colossalai.launch_from_torch({})
+ coordinator = DistCoordinator()
+
+ # if coordinator.world_size > 1:
+ # set_sequence_parallel_group(dist.group.WORLD)
+ # enable_sequence_parallelism = True
+ # else:
+ # enable_sequence_parallelism = False
+
+ # ======================================================
+ # 2. runtime variables
+ # ======================================================
+ torch.set_grad_enabled(False)
+ device = get_current_device()
+ dtype = to_torch_dtype(cfg.dtype)
+
+
+
+
+ # ======================================================
+ # 3. build dataset and dataloader
+ # ======================================================
+ dataset = DatasetFromCSV(
+ cfg.data_path,
+ transform=(
+ get_transforms_video(cfg.image_size[0])
+ if not cfg.use_image_transform
+ else get_transforms_image(cfg.image_size[0])
+ ),
+ num_frames=cfg.num_frames,
+ frame_interval=cfg.frame_interval,
+ root=cfg.root,
+ )
+
+ dataloader = prepare_dataloader(
+ dataset,
+ batch_size=cfg.batch_size,
+ num_workers=cfg.num_workers,
+ shuffle=True,
+ drop_last=True,
+ pin_memory=True,
+ process_group=get_data_parallel_group(),
+ )
+ print(f"Dataset contains {len(dataset):,} videos ({cfg.data_path})")
+
+ total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size
+ print(f"Total batch size: {total_batch_size}")
+
+ # ======================================================
+ # 4. build model & load weights
+ # ======================================================
+ # 3.1. build model
+ vae = build_module(cfg.model, MODELS, device=device)
+ discriminator = build_module(cfg.discriminator, MODELS, device=device)
+
+ # 3.2. move to device & eval
+ vae = vae.to(device, dtype).eval()
+ discriminator = discriminator.to(device, dtype).eval()
+
+ # 3.4. support for multi-resolution
+ model_args = dict()
+ if cfg.multi_resolution:
+ image_size = cfg.image_size
+ hw = torch.tensor([image_size], device=device, dtype=dtype).repeat(cfg.batch_size, 1)
+ ar = torch.tensor([[image_size[0] / image_size[1]]], device=device, dtype=dtype).repeat(cfg.batch_size, 1)
+ model_args["data_info"] = dict(ar=ar, hw=hw)
+
+ # ======================================================
+ # 4. inference
+ # ======================================================
+ save_dir = cfg.save_dir
+ os.makedirs(save_dir, exist_ok=True)
+
+
+
+ # 4.1. batch generation
+
+ # define loss function
+ if cfg.calc_loss:
+ vae_loss_fn = VEALoss(
+ logvar_init=cfg.logvar_init,
+ perceptual_loss_weight = cfg.perceptual_loss_weight,
+ kl_loss_weight = cfg.kl_loss_weight,
+ device=device,
+ dtype=dtype,
+ )
+
+ adversarial_loss_fn = AdversarialLoss(
+ discriminator_factor = cfg.discriminator_factor,
+ discriminator_start = cfg.discriminator_start,
+ generator_factor = cfg.generator_factor,
+ generator_loss_type = cfg.generator_loss_type,
+ )
+
+ disc_loss_fn = DiscriminatorLoss(
+ discriminator_factor = cfg.discriminator_factor,
+ discriminator_start = cfg.discriminator_start,
+ discriminator_loss_type = cfg.discriminator_loss_type,
+ lecam_loss_weight = cfg.lecam_loss_weight,
+ gradient_penalty_loss_weight = cfg.gradient_penalty_loss_weight,
+ )
+
+
+ running_loss = 0.0
+ running_disc_loss = 0.0
+ loss_steps = 0
+
+ disc_time_downsample_factor = 2 ** len(cfg.discriminator.channel_multipliers)
+ disc_time_padding = disc_time_downsample_factor - cfg.num_frames % disc_time_downsample_factor
+ video_contains_first_frame = cfg.video_contains_first_frame
+
+ lecam_ema_real = torch.tensor(0.0)
+ lecam_ema_fake = torch.tensor(0.0)
+
+ total_steps = len(dataloader)
+ dataloader_iter = iter(dataloader)
+
+ with tqdm(
+ range(total_steps),
+ # desc=f"Avg Loss: {running_loss}",
+ disable=not coordinator.is_master(),
+ total=total_steps,
+ initial=0,
+ ) as pbar:
+ for step in pbar:
+ batch = next(dataloader_iter)
+ x = batch["video"].to(device, dtype) # [B, C, T, H, W]
+
+ is_image = x.ndim == 4
+ if is_image:
+ video = rearrange(x, 'b c ... -> b c 1 ...')
+ video_contains_first_frame = True
+ else:
+ video = x
+
+ recon_video, posterior = vae(
+ video,
+ video_contains_first_frame = video_contains_first_frame
+ )
+
+ if cfg.calc_loss:
+ # ====== Calc Loss ======
+ # simple nll loss
+ nll_loss, weighted_nll_loss, weighted_kl_loss = vae_loss_fn(
+ video,
+ recon_video,
+ posterior,
+ split = "eval"
+ )
+
+ fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)
+ fake_logits = discriminator(fake_video.contiguous()) # TODO: take out contiguous?
+ adversarial_loss = adversarial_loss_fn(
+ fake_logits,
+ nll_loss,
+ vae.module.get_last_layer(),
+ cfg.discriminator_start+1, # Hack to use discriminator
+ is_training = vae.training,
+ )
+
+ vae_loss = weighted_nll_loss + weighted_kl_loss + adversarial_loss
+
+ # ====== Discriminator Loss ======
+ real_video = pad_at_dim(video, (disc_time_padding, 0), value = 0., dim = 2)
+ fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)
+
+ if cfg.gradient_penalty_loss_weight is not None and cfg.gradient_penalty_loss_weight > 0.0:
+ real_video = real_video.requires_grad_()
+ real_logits = discriminator(real_video.contiguous()) # SCH: not detached for now for gradient_penalty calculation
+ else:
+ real_logits = discriminator(real_video.contiguous().detach())
+
+ fake_logits = discriminator(fake_video.contiguous().detach())
+ disc_loss = disc_loss_fn(
+ real_logits,
+ fake_logits,
+ cfg.discriminator_start+1, # Hack to use discriminator
+ lecam_ema_real = lecam_ema_real,
+ lecam_ema_fake = lecam_ema_fake,
+ real_video = real_video if cfg.gradient_penalty_loss_weight is not None else None,
+ )
+
+ if cfg.ema_decay is not None:
+ # SCH: TODO: is this written properly like this for moving average? e.g. distributed training etc.
+ lecam_ema_real = lecam_ema_real * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(real_logits.clone().detach())
+ lecam_ema_fake = lecam_ema_fake * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(fake_logits.clone().detach())
+
+
+ loss_steps += 1
+ running_disc_loss = disc_loss.item()/loss_steps + running_loss * ((loss_steps - 1) / loss_steps)
+ running_loss = vae_loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)
+
+ if coordinator.is_master():
+ for idx, sample in enumerate(recon_video):
+ pos = step * cfg.batch_size + idx
+ save_path = os.path.join(save_dir, f"sample_{pos}")
+ save_sample(sample, fps=cfg.fps, save_path=save_path)
+
+ if cfg.calc_loss:
+ print("test vae loss:", running_loss)
+ print("test disc loss:", running_disc_loss)
+
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/train-vae-v2.py b/scripts/train-vae-v2.py
index 3963ea5..4a0116a 100644
--- a/scripts/train-vae-v2.py
+++ b/scripts/train-vae-v2.py
@@ -353,10 +353,8 @@ def main():
# ====== Discriminator Loss ======
if global_step > cfg.discriminator_start:
-
# if video_contains_first_frame:
# Since we don't have enough T frames, pad anyways
-
real_video = pad_at_dim(video, (disc_time_padding, 0), value = 0., dim = 2)
fake_video = pad_at_dim(recon_video, (disc_time_padding, 0), value = 0., dim = 2)