Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

add inference config add readme complete inference script TODO: DEBUG inference

Shen-Chenhuidfdf8db49144
diff --git a/configs/vae_3d/inference/16x256x256.py b/configs/vae_3d/inference/16x256x256.py
new file mode 100644
index 0000000..53e1970
--- /dev/null
+++ b/configs/vae_3d/inference/16x256x256.py
@@ -0,0 +1,45 @@
+num_frames = 16
+frame_interval = 3
+fps = 24 // 3
+image_size = (256, 256)
+
+# Define dataset
+root = None
+data_path = "CSV_PATH"
+use_image_transform = False
+num_workers = 4
+
+# Define acceleration
+dtype = "fp16"
+grad_checkpoint = True
+plugin = "zero2"
+sp_size = 1
+
+# Define Loss
+kl_weight = 0.000001
+perceptual_weight = 1.0 # TODO: need to change this to 0.1 !!! according to MAGVIT paper
+
+# Define model
+
+model = dict(
+ type="VAE_3D",
+ from_pretrained="PRETRAINED_MODEL", # TODO: double check how this loads the pretrained model
+ in_out_channels = 3,
+ latent_embed_dim = 256,
+ filters = 64,
+ num_res_blocks = 2,
+ channel_multipliers = (1, 2, 2, 4),
+ temporal_downsample = (True, True, False),
+ num_groups = 32, # for nn.GroupNorm
+ conv_downsample = False,
+ upsample = "nearest+conv", # options: "deconv", "nearest+conv"
+ kl_embed_dim = 64,
+ custom_conv_padding = None,
+ activation_fn = 'swish',
+)
+
+
+# Others
+seed = 42
+save_dir = "outputs/samples"
+
diff --git a/configs/vae_3d/train/16x256x256.py b/configs/vae_3d/train/16x256x256.py
index 41f6af5..c3fe27f 100644
--- a/configs/vae_3d/train/16x256x256.py
+++ b/configs/vae_3d/train/16x256x256.py
@@ -16,7 +16,7 @@ sp_size = 1
# Define Loss
kl_weight = 0.000001
-perceptual_weight = 1.0
+perceptual_weight = 1.0 # TODO: need to change this to 0.1 !!! according to MAGVIT paper
# Define model
@@ -36,17 +36,16 @@ model = dict(
activation_fn = 'swish',
)
-# TODO: check all these settings
-
-scheduler = dict(
- type="iddpm",
- timestep_respacing="",
-)
-
# Others
seed = 42
outputs = "outputs"
-wandb = False
+
+# Training
+''' NOTE:
+magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
+==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
+3-6 epochs for pexel, from pexel observation its correct
+'''
epochs = 3
log_every = 1
diff --git a/opensora/models/vae/README.md b/opensora/models/vae/README.md
new file mode 100644
index 0000000..bc060b1
--- /dev/null
+++ b/opensora/models/vae/README.md
@@ -0,0 +1,15 @@
+## Commands
+
+
+### 1. Train
+
+```yaml
+# train on pexel dataset
+WANDB_API_KEY=<wandb_api_key> CUDA_VISIBLE_DEVICES=<n> torchrun --master_port=<port_num> --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/train.csv --wandb True
+```
+
+### 2. Inference
+
+```yaml
+CUDA_VISIBLE_DEVICES=<n> torchrun --master_port=<port_num> --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/inference/16x256x256.py --data-path /home/shenchenhui/data/pexels/train.csv --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/028-F16S3-VAE_3D/epoch3-global_step20000/vae
+```
\ No newline at end of file
diff --git a/opensora/models/vae/model_utils.py b/opensora/models/vae/model_utils.py
index 979caa1..f5c487e 100644
--- a/opensora/models/vae/model_utils.py
+++ b/opensora/models/vae/model_utils.py
@@ -42,6 +42,7 @@ class DiagonalGaussianDistribution(object):
self.var = self.std = torch.zeros_like(self.mean).to(device=self.parameters.device, dtype=self.mean.dtype)
def sample(self):
+ # torch.randn: standard normal distribution
x = self.mean + self.std * torch.randn(self.mean.shape).to(device=self.parameters.device, dtype=self.mean.dtype)
return x
@@ -49,17 +50,17 @@ class DiagonalGaussianDistribution(object):
if self.deterministic:
return torch.Tensor([0.])
else:
- if other is None:
+ if other is None: # SCH: assumes other is a standard normal distribution
return 0.5 * torch.sum(torch.pow(self.mean, 2)
+ self.var - 1.0 - self.logvar,
- dim=[1, 2, 3, 4]) # TODO: check dimensions
+ dim=[1, 2, 3, 4])
else:
return 0.5 * torch.sum(
torch.pow(self.mean - other.mean, 2) / other.var
+ self.var / other.var - 1.0 - self.logvar + other.logvar,
- dim=[1, 2, 3, 4]) # TODO: check dimensions
+ dim=[1, 2, 3, 4])
- def nll(self, sample, dims=[1,2,3,4]): # TODO: check dimensions
+ def nll(self, sample, dims=[1,2,3,4]): # TODO: what does this do?
if self.deterministic:
return torch.Tensor([0.])
logtwopi = np.log(2.0 * np.pi)
@@ -67,7 +68,7 @@ class DiagonalGaussianDistribution(object):
logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var,
dim=dims)
- def mode(self):
+ def mode(self): # SCH: used for vae inference?
return self.mean
diff --git a/scripts/inference-vae.py b/scripts/inference-vae.py
index fe111b8..2e033bb 100644
--- a/scripts/inference-vae.py
+++ b/scripts/inference-vae.py
@@ -11,6 +11,14 @@ from opensora.datasets import save_sample
from opensora.registry import MODELS, SCHEDULERS, build_module
from opensora.utils.config_utils import parse_configs
from opensora.utils.misc import to_torch_dtype
+from opensora.datasets import DatasetFromCSV, get_transforms_image, get_transforms_video, prepare_dataloader
+from opensora.acceleration.parallel_states import (
+ get_data_parallel_group,
+ set_data_parallel_group,
+ set_sequence_parallel_group,
+)
+from tqdm import tqdm
+from opensora.models.vae.model_utils import VEA3DLoss
def main():
@@ -41,13 +49,44 @@ def main():
set_random_seed(seed=cfg.seed)
prompts = cfg.prompt
+
+ # ======================================================
+ # 3. build dataset and dataloader
+ # ======================================================
+ dataset = DatasetFromCSV(
+ cfg.data_path,
+ # TODO: change transforms
+ transform=(
+ get_transforms_video(cfg.image_size[0])
+ if not cfg.use_image_transform
+ else get_transforms_image(cfg.image_size[0])
+ ),
+ num_frames=cfg.num_frames,
+ frame_interval=cfg.frame_interval,
+ root=cfg.root,
+ )
+
+ dataloader = prepare_dataloader(
+ dataset,
+ batch_size=cfg.batch_size,
+ num_workers=cfg.num_workers,
+ shuffle=True,
+ drop_last=True,
+ pin_memory=True,
+ process_group=get_data_parallel_group(),
+ )
+ print(f"Dataset contains {len(dataset):,} videos ({cfg.data_path})")
+
+ total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.sp_size
+ print(f"Total batch size: {total_batch_size}")
+
# ======================================================
- # 3. build model & load weights
+ # 4. build model & load weights
# ======================================================
# 3.1. build model
- input_size = (cfg.num_frames, *cfg.image_size)
- vae = build_module(cfg.vae, MODELS)
- latent_size = vae.get_latent_size(input_size)
+ # input_size = (cfg.num_frames, *cfg.image_size)
+ vae = build_module(cfg.model, MODELS)
+ # latent_size = vae.get_latent_size(input_size)
# 3.2. move to device & eval
vae = vae.to(device, dtype).eval()
@@ -66,40 +105,43 @@ def main():
# ======================================================
# 4. inference
# ======================================================
- sample_idx = 0
save_dir = cfg.save_dir
os.makedirs(save_dir, exist_ok=True)
# 4.1. batch generation
- # TODO: read input, sample, then decode ??? =========
- for i in range(0, len(prompts), cfg.batch_size):
- # 4.2 sample in hidden space
-
- z = torch.randn(len(batch_prompts), vae.out_channels, *latent_size, device=device, dtype=dtype)
-
- # 4.3. diffusion sampling
- samples = scheduler.sample(
- model,
- text_encoder,
- z=z,
- prompts=batch_prompts,
- device=device,
- additional_args=model_args,
- )
-
-
- samples = vae.decode(samples.to(dtype))
-
- if coordinator.is_master():
- for idx, sample in enumerate(samples):
- print(f"Prompt: {batch_prompts[idx]}")
- save_path = os.path.join(save_dir, f"sample_{sample_idx}")
- save_sample(sample, fps=cfg.fps, save_path=save_path)
- sample_idx += 1
-
- # TODO: read input, sample, then decode ??? =========
+ # define loss function
+ loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)
+ running_loss = 0.0
+ loss_steps = 0
+
+
+ total_steps = len(dataloader)
+ dataloader_iter = iter(dataloader)
+
+ with tqdm(
+ range(total_steps),
+ desc=f"Avg Loss: {running_loss}",
+ disable=not coordinator.is_master(),
+ total=total_steps,
+ initial=0,
+ ) as pbar:
+ for step in pbar:
+ batch = next(dataloader_iter)
+ x = batch["video"].to(device, dtype) # [B, C, T, H, W]
+ reconstructions, posterior = vae(x)
+ loss = loss_function(x, reconstructions, posterior)
+ loss_steps += 1
+ running_loss = loss.item()/ loss_steps + running_loss * (loss_steps - 1 / loss_steps)
+
+ if coordinator.is_master():
+ for idx, sample in enumerate(reconstructions):
+ save_path = os.path.join(save_dir, f"sample_{idx}")
+ save_sample(sample, fps=cfg.fps, save_path=save_path)
+
+ print("test loss:", running_loss)
+
if __name__ == "__main__":
main()
diff --git a/scripts/train-vae.py b/scripts/train-vae.py
index 2e004ac..56e33ef 100644
--- a/scripts/train-vae.py
+++ b/scripts/train-vae.py
@@ -170,8 +170,6 @@ def main():
start_epoch = start_step = log_step = sampler_start_idx = 0
running_loss = 0.0
- # TODO: allow resume training without ema
-
# 6.1. resume training
if cfg.load is not None:
@@ -247,7 +245,7 @@ def main():
if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:
save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")
os.makedirs(os.path.join(save_dir, "model"), exist_ok=True)
-
+ # TODO: save in model?
booster.save_model(vae, os.path.join(save_dir, "vae"), shard=True)
booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)
if lr_scheduler is not None: