Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

modify inference

Shen-Chenhui7b134d71dcb6
diff --git a/configs/vae_magvit_v2/inference/17x128x128_pixabay.py b/configs/vae_magvit_v2/inference/17x128x128_pixabay.py
new file mode 100644
index 0000000..9517f12
--- /dev/null
+++ b/configs/vae_magvit_v2/inference/17x128x128_pixabay.py
@@ -0,0 +1,104 @@
+num_frames = 17
+
+dataset = dict(
+ type="VideoTextDataset",
+ data_path=None,
+ num_frames=num_frames,
+ frame_interval=3,
+ image_size=(128, 128),
+ get_text=False,
+)
+
+fps = 24 // 3
+is_vae = True
+use_pipeline = True
+
+# Define dataset
+root = None
+data_path = "CSV_PATH"
+max_test_samples = -1
+use_image_transform = False
+num_workers = 4
+video_contains_first_frame = True
+
+
+
+# Define acceleration
+dtype = "bf16"
+grad_checkpoint = True
+plugin = "zero2"
+sp_size = 1
+
+
+# Define model
+
+vae_2d = dict(
+ type="VideoAutoencoderKL",
+ from_pretrained="stabilityai/sd-vae-ft-ema",
+)
+
+model = dict(
+ type="VAE_MAGVIT_V2",
+ in_out_channels = 4,
+ latent_embed_dim = 64,
+ filters = 128,
+ num_res_blocks = 4,
+ channel_multipliers = (1, 2, 2, 4),
+ temporal_downsample = (False, True, True),
+ num_groups = 32, # for nn.GroupNorm
+ kl_embed_dim = 4,
+ activation_fn = 'swish',
+ separate_first_frame_encoding = False,
+ disable_space = True,
+ custom_conv_padding = None,
+ encoder_double_z = False,
+ custom_conv_padding=None,
+)
+
+discriminator = dict(
+ type="DISCRIMINATOR_3D",
+ image_size = 16,
+ num_frames = num_frames,
+ in_channels = 4,
+ filters = 128,
+ channel_multipliers = (2,4,4), #(2,4,4,4,4) # (2,4,4,4) for 64x64 resolution
+)
+
+
+# loss weights
+logvar_init=0.0
+kl_loss_weight = 0.000001
+perceptual_loss_weight = 0.1 # use vgg is not None and more than 0
+discriminator_factor = 1.0 # for discriminator adversarial loss
+# discriminator_loss_weight = 0.5 # for generator adversarial loss
+generator_factor = 0.1 # for generator adversarial loss
+lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight
+discriminator_loss_type="non-saturating"
+generator_loss_type="non-saturating"
+discriminator_start = 2500 # 50000 NOTE: change to correct val, debug use -1 for now
+gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use
+ema_decay = 0.999 # ema decay factor for generator
+
+
+# Others
+seed = 42
+save_dir = "outputs/samples_pixabay_17"
+wandb = False
+
+# Training
+''' NOTE:
+magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128
+==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],
+3-6 epochs for pexel, from pexel observation its correct
+'''
+
+epochs = 10
+log_every = 1
+ckpt_every = 500
+load = None
+
+batch_size = 4
+lr = 1e-4
+grad_clip = 1.0
+
+calc_loss = True
\ No newline at end of file
diff --git a/configs/vae_magvit_v2/train/17x128x128_pixabay_8_GPU_pipeline.py b/configs/vae_magvit_v2/train/17x128x128_pixabay_8_GPU_pipeline.py
index 3926367..fe69e94 100644
--- a/configs/vae_magvit_v2/train/17x128x128_pixabay_8_GPU_pipeline.py
+++ b/configs/vae_magvit_v2/train/17x128x128_pixabay_8_GPU_pipeline.py
@@ -1,7 +1,8 @@
+num_frames = 17
dataset = dict(
type="VideoTextDataset",
data_path=None,
- num_frames=17,
+ num_frames=num_frames,
frame_interval=3,
image_size=(128, 128),
get_text=False,
@@ -44,7 +45,7 @@ model = dict(
discriminator = dict(
type="DISCRIMINATOR_3D",
image_size=(16, 16), # NOTE: here image size is different
- num_frames=17,
+ num_frames=num_frames,
in_channels=4,
filters=128,
use_pretrained=True, # NOTE: set to False only if we want to disable load
diff --git a/scripts/inference-vae-v2.py b/scripts/inference-vae-v2.py
index 65cb411..5a41d79 100644
--- a/scripts/inference-vae-v2.py
+++ b/scripts/inference-vae-v2.py
@@ -11,7 +11,8 @@ from opensora.datasets import save_sample
from opensora.registry import MODELS, SCHEDULERS, build_module
from opensora.utils.config_utils import parse_configs
from opensora.utils.misc import to_torch_dtype
-from opensora.datasets import DatasetFromCSV, get_transforms_image, get_transforms_video, prepare_dataloader
+from opensora.datasets import prepare_dataloader, prepare_variable_dataloader
+from opensora.datasets import DATASETS, MODELS, build_module
from opensora.acceleration.parallel_states import (
get_data_parallel_group,
set_data_parallel_group,
@@ -54,18 +55,7 @@ def main():
# ======================================================
# 3. build dataset and dataloader
# ======================================================
- dataset = DatasetFromCSV(
- cfg.data_path,
- transform=(
- get_transforms_video(cfg.image_size[0])
- if not cfg.use_image_transform
- else get_transforms_image(cfg.image_size[0])
- ),
- num_frames=cfg.num_frames,
- frame_interval=cfg.frame_interval,
- root=cfg.root,
- )
-
+ dataset = build_module(cfg.dataset, DATASETS)
dataloader = prepare_dataloader(
dataset,
@@ -100,7 +90,7 @@ def main():
# 3.4. support for multi-resolution
model_args = dict()
if cfg.multi_resolution:
- image_size = cfg.image_size
+ image_size = cfg.dataset.image_size
hw = torch.tensor([image_size], device=device, dtype=dtype).repeat(cfg.batch_size, 1)
ar = torch.tensor([[image_size[0] / image_size[1]]], device=device, dtype=dtype).repeat(cfg.batch_size, 1)
model_args["data_info"] = dict(ar=ar, hw=hw)
@@ -146,8 +136,8 @@ def main():
loss_steps = 0
disc_time_downsample_factor = 2 ** len(cfg.discriminator.channel_multipliers)
- if cfg.num_frames % disc_time_downsample_factor != 0:
- disc_time_padding = disc_time_downsample_factor - cfg.num_frames % disc_time_downsample_factor
+ if cfg.datasets.num_frames % disc_time_downsample_factor != 0:
+ disc_time_padding = disc_time_downsample_factor - cfg.datasets.num_frames % disc_time_downsample_factor
else:
disc_time_padding = 0
video_contains_first_frame = cfg.video_contains_first_frame