Open-Sora: Democratizing Efficient Video Production for All
Python
add inference config add readme complete inference script TODO: DEBUG inference
Shen-Chenhuidfdf8db49144
diff --git a/configs/vae_3d/inference/16x256x256.py b/configs/vae_3d/inference/16x256x256.pynew file mode 100644index 0000000..53e1970--- /dev/null+++ b/configs/vae_3d/inference/16x256x256.py@@ -0,0 +1,45 @@+num_frames = 16+frame_interval = 3+fps = 24 // 3+image_size = (256, 256)++# Define dataset+root = None+data_path = "CSV_PATH"+use_image_transform = False+num_workers = 4++# Define acceleration+dtype = "fp16"+grad_checkpoint = True+plugin = "zero2"+sp_size = 1++# Define Loss+kl_weight = 0.000001+perceptual_weight = 1.0 # TODO: need to change this to 0.1 !!! according to MAGVIT paper++# Define model++model = dict(+ type="VAE_3D",+ from_pretrained="PRETRAINED_MODEL", # TODO: double check how this loads the pretrained model+ in_out_channels = 3,+ latent_embed_dim = 256,+ filters = 64,+ num_res_blocks = 2,+ channel_multipliers = (1, 2, 2, 4),+ temporal_downsample = (True, True, False),+ num_groups = 32, # for nn.GroupNorm+ conv_downsample = False,+ upsample = "nearest+conv", # options: "deconv", "nearest+conv"+ kl_embed_dim = 64,+ custom_conv_padding = None,+ activation_fn = 'swish',+)+++# Others+seed = 42+save_dir = "outputs/samples"+diff --git a/configs/vae_3d/train/16x256x256.py b/configs/vae_3d/train/16x256x256.pyindex 41f6af5..c3fe27f 100644--- a/configs/vae_3d/train/16x256x256.py+++ b/configs/vae_3d/train/16x256x256.py@@ -16,7 +16,7 @@ sp_size = 1# Define Losskl_weight = 0.000001-perceptual_weight = 1.0+perceptual_weight = 1.0 # TODO: need to change this to 0.1 !!! according to MAGVIT paper# Define model@@ -36,17 +36,16 @@ model = dict(activation_fn = 'swish',)-# TODO: check all these settings--scheduler = dict(- type="iddpm",- timestep_respacing="",-)-# Othersseed = 42outputs = "outputs"-wandb = False++# Training+''' NOTE:+magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128+==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],+3-6 epochs for pexel, from pexel observation its correct+'''epochs = 3log_every = 1diff --git a/opensora/models/vae/README.md b/opensora/models/vae/README.mdnew file mode 100644index 0000000..bc060b1--- /dev/null+++ b/opensora/models/vae/README.md@@ -0,0 +1,15 @@+## Commands+++### 1. Train++```yaml+# train on pexel dataset+WANDB_API_KEY=<wandb_api_key> CUDA_VISIBLE_DEVICES=<n> torchrun --master_port=<port_num> --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/train.csv --wandb True+```++### 2. Inference++```yaml+CUDA_VISIBLE_DEVICES=<n> torchrun --master_port=<port_num> --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/inference/16x256x256.py --data-path /home/shenchenhui/data/pexels/train.csv --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/028-F16S3-VAE_3D/epoch3-global_step20000/vae+```\ No newline at end of filediff --git a/opensora/models/vae/model_utils.py b/opensora/models/vae/model_utils.pyindex 979caa1..f5c487e 100644--- a/opensora/models/vae/model_utils.py+++ b/opensora/models/vae/model_utils.py@@ -42,6 +42,7 @@ class DiagonalGaussianDistribution(object):self.var = self.std = torch.zeros_like(self.mean).to(device=self.parameters.device, dtype=self.mean.dtype)def sample(self):+ # torch.randn: standard normal distributionx = self.mean + self.std * torch.randn(self.mean.shape).to(device=self.parameters.device, dtype=self.mean.dtype)return x@@ -49,17 +50,17 @@ class DiagonalGaussianDistribution(object):if self.deterministic:return torch.Tensor([0.])else:- if other is None:+ if other is None: # SCH: assumes other is a standard normal distributionreturn 0.5 * torch.sum(torch.pow(self.mean, 2)+ self.var - 1.0 - self.logvar,- dim=[1, 2, 3, 4]) # TODO: check dimensions+ dim=[1, 2, 3, 4])else:return 0.5 * torch.sum(torch.pow(self.mean - other.mean, 2) / other.var+ self.var / other.var - 1.0 - self.logvar + other.logvar,- dim=[1, 2, 3, 4]) # TODO: check dimensions+ dim=[1, 2, 3, 4])- def nll(self, sample, dims=[1,2,3,4]): # TODO: check dimensions+ def nll(self, sample, dims=[1,2,3,4]): # TODO: what does this do?if self.deterministic:return torch.Tensor([0.])logtwopi = np.log(2.0 * np.pi)@@ -67,7 +68,7 @@ class DiagonalGaussianDistribution(object):logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var,dim=dims)- def mode(self):+ def mode(self): # SCH: used for vae inference?return self.meandiff --git a/scripts/inference-vae.py b/scripts/inference-vae.pyindex fe111b8..2e033bb 100644--- a/scripts/inference-vae.py+++ b/scripts/inference-vae.py@@ -11,6 +11,14 @@ from opensora.datasets import save_samplefrom opensora.registry import MODELS, SCHEDULERS, build_modulefrom opensora.utils.config_utils import parse_configsfrom opensora.utils.misc import to_torch_dtype+from opensora.datasets import DatasetFromCSV, get_transforms_image, get_transforms_video, prepare_dataloader+from opensora.acceleration.parallel_states import (+ get_data_parallel_group,+ set_data_parallel_group,+ set_sequence_parallel_group,+)+from tqdm import tqdm+from opensora.models.vae.model_utils import VEA3DLossdef main():@@ -41,13 +49,44 @@ def main():set_random_seed(seed=cfg.seed)prompts = cfg.prompt++ # ======================================================+ # 3. build dataset and dataloader+ # ======================================================+ dataset = DatasetFromCSV(+ cfg.data_path,+ # TODO: change transforms+ transform=(+ get_transforms_video(cfg.image_size[0])+ if not cfg.use_image_transform+ else get_transforms_image(cfg.image_size[0])+ ),+ num_frames=cfg.num_frames,+ frame_interval=cfg.frame_interval,+ root=cfg.root,+ )++ dataloader = prepare_dataloader(+ dataset,+ batch_size=cfg.batch_size,+ num_workers=cfg.num_workers,+ shuffle=True,+ drop_last=True,+ pin_memory=True,+ process_group=get_data_parallel_group(),+ )+ print(f"Dataset contains {len(dataset):,} videos ({cfg.data_path})")++ total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size+ print(f"Total batch size: {total_batch_size}")+# ======================================================- # 3. build model & load weights+ # 4. build model & load weights# ======================================================# 3.1. build model- input_size = (cfg.num_frames, *cfg.image_size)- vae = build_module(cfg.vae, MODELS)- latent_size = vae.get_latent_size(input_size)+ # input_size = (cfg.num_frames, *cfg.image_size)+ vae = build_module(cfg.model, MODELS)+ # latent_size = vae.get_latent_size(input_size)# 3.2. move to device & evalvae = vae.to(device, dtype).eval()@@ -66,40 +105,43 @@ def main():# ======================================================# 4. inference# ======================================================- sample_idx = 0save_dir = cfg.save_diros.makedirs(save_dir, exist_ok=True)# 4.1. batch generation- # TODO: read input, sample, then decode ??? =========- for i in range(0, len(prompts), cfg.batch_size):- # 4.2 sample in hidden space-- z = torch.randn(len(batch_prompts), vae.out_channels, *latent_size, device=device, dtype=dtype)-- # 4.3. diffusion sampling- samples = scheduler.sample(- model,- text_encoder,- z=z,- prompts=batch_prompts,- device=device,- additional_args=model_args,- )--- samples = vae.decode(samples.to(dtype))-- if coordinator.is_master():- for idx, sample in enumerate(samples):- print(f"Prompt: {batch_prompts[idx]}")- save_path = os.path.join(save_dir, f"sample_{sample_idx}")- save_sample(sample, fps=cfg.fps, save_path=save_path)- sample_idx += 1-- # TODO: read input, sample, then decode ??? =========+ # define loss function+ loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)+ running_loss = 0.0+ loss_steps = 0+++ total_steps = len(dataloader)+ dataloader_iter = iter(dataloader)++ with tqdm(+ range(total_steps),+ desc=f"Avg Loss: {running_loss}",+ disable=not coordinator.is_master(),+ total=total_steps,+ initial=0,+ ) as pbar:+ for step in pbar:+ batch = next(dataloader_iter)+ x = batch["video"].to(device, dtype) # [B, C, T, H, W]+ reconstructions, posterior = vae(x)+ loss = loss_function(x, reconstructions, posterior)+ loss_steps += 1+ running_loss = loss.item()/ loss_steps + running_loss * (loss_steps - 1 / loss_steps)++ if coordinator.is_master():+ for idx, sample in enumerate(reconstructions):+ save_path = os.path.join(save_dir, f"sample_{idx}")+ save_sample(sample, fps=cfg.fps, save_path=save_path)++ print("test loss:", running_loss)+if __name__ == "__main__":main()diff --git a/scripts/train-vae.py b/scripts/train-vae.pyindex 2e004ac..56e33ef 100644--- a/scripts/train-vae.py+++ b/scripts/train-vae.py@@ -170,8 +170,6 @@ def main():start_epoch = start_step = log_step = sampler_start_idx = 0running_loss = 0.0- # TODO: allow resume training without ema-# 6.1. resume trainingif cfg.load is not None:@@ -247,7 +245,7 @@ def main():if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")os.makedirs(os.path.join(save_dir, "model"), exist_ok=True)-+ # TODO: save in model?booster.save_model(vae, os.path.join(save_dir, "vae"), shard=True)booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)if lr_scheduler is not None: