Open-Sora: Democratizing Efficient Video Production for All
Python
modify inference
Shen-Chenhui7b134d71dcb6
diff --git a/configs/vae_magvit_v2/inference/17x128x128_pixabay.py b/configs/vae_magvit_v2/inference/17x128x128_pixabay.pynew file mode 100644index 0000000..9517f12--- /dev/null+++ b/configs/vae_magvit_v2/inference/17x128x128_pixabay.py@@ -0,0 +1,104 @@+num_frames = 17++dataset = dict(+ type="VideoTextDataset",+ data_path=None,+ num_frames=num_frames,+ frame_interval=3,+ image_size=(128, 128),+ get_text=False,+)++fps = 24 // 3+is_vae = True+use_pipeline = True++# Define dataset+root = None+data_path = "CSV_PATH"+max_test_samples = -1+use_image_transform = False+num_workers = 4+video_contains_first_frame = True++++# Define acceleration+dtype = "bf16"+grad_checkpoint = True+plugin = "zero2"+sp_size = 1+++# Define model++vae_2d = dict(+ type="VideoAutoencoderKL",+ from_pretrained="stabilityai/sd-vae-ft-ema",+)++model = dict(+ type="VAE_MAGVIT_V2",+ in_out_channels = 4,+ latent_embed_dim = 64,+ filters = 128,+ num_res_blocks = 4,+ channel_multipliers = (1, 2, 2, 4),+ temporal_downsample = (False, True, True),+ num_groups = 32, # for nn.GroupNorm+ kl_embed_dim = 4,+ activation_fn = 'swish',+ separate_first_frame_encoding = False,+ disable_space = True,+ custom_conv_padding = None,+ encoder_double_z = False,+ custom_conv_padding=None,+)++discriminator = dict(+ type="DISCRIMINATOR_3D",+ image_size = 16,+ num_frames = num_frames,+ in_channels = 4,+ filters = 128,+ channel_multipliers = (2,4,4), #(2,4,4,4,4) # (2,4,4,4) for 64x64 resolution+)+++# loss weights+logvar_init=0.0+kl_loss_weight = 0.000001+perceptual_loss_weight = 0.1 # use vgg is not None and more than 0+discriminator_factor = 1.0 # for discriminator adversarial loss+# discriminator_loss_weight = 0.5 # for generator adversarial loss+generator_factor = 0.1 # for generator adversarial loss+lecam_loss_weight = None # NOTE: not clear in MAGVIT what is the weight+discriminator_loss_type="non-saturating"+generator_loss_type="non-saturating"+discriminator_start = 2500 # 50000 NOTE: change to correct val, debug use -1 for now+gradient_penalty_loss_weight = None # 10 # SCH: MAGVIT uses 10, opensora plan doesn't use+ema_decay = 0.999 # ema decay factor for generator+++# Others+seed = 42+save_dir = "outputs/samples_pixabay_17"+wandb = False++# Training+''' NOTE:+magvit uses about # samples (K) * epochs ~ 2-5 K, num_frames = 4, reso = 128+==> ours num_frams = 16, reso = 256, so samples (K) * epochs ~ [500 - 1200],+3-6 epochs for pexel, from pexel observation its correct+'''++epochs = 10+log_every = 1+ckpt_every = 500+load = None++batch_size = 4+lr = 1e-4+grad_clip = 1.0++calc_loss = True\ No newline at end of filediff --git a/configs/vae_magvit_v2/train/17x128x128_pixabay_8_GPU_pipeline.py b/configs/vae_magvit_v2/train/17x128x128_pixabay_8_GPU_pipeline.pyindex 3926367..fe69e94 100644--- a/configs/vae_magvit_v2/train/17x128x128_pixabay_8_GPU_pipeline.py+++ b/configs/vae_magvit_v2/train/17x128x128_pixabay_8_GPU_pipeline.py@@ -1,7 +1,8 @@+num_frames = 17dataset = dict(type="VideoTextDataset",data_path=None,- num_frames=17,+ num_frames=num_frames,frame_interval=3,image_size=(128, 128),get_text=False,@@ -44,7 +45,7 @@ model = dict(discriminator = dict(type="DISCRIMINATOR_3D",image_size=(16, 16), # NOTE: here image size is different- num_frames=17,+ num_frames=num_frames,in_channels=4,filters=128,use_pretrained=True, # NOTE: set to False only if we want to disable loaddiff --git a/scripts/inference-vae-v2.py b/scripts/inference-vae-v2.pyindex 65cb411..5a41d79 100644--- a/scripts/inference-vae-v2.py+++ b/scripts/inference-vae-v2.py@@ -11,7 +11,8 @@ from opensora.datasets import save_samplefrom opensora.registry import MODELS, SCHEDULERS, build_modulefrom opensora.utils.config_utils import parse_configsfrom opensora.utils.misc import to_torch_dtype-from opensora.datasets import DatasetFromCSV, get_transforms_image, get_transforms_video, prepare_dataloader+from opensora.datasets import prepare_dataloader, prepare_variable_dataloader+from opensora.datasets import DATASETS, MODELS, build_modulefrom opensora.acceleration.parallel_states import (get_data_parallel_group,set_data_parallel_group,@@ -54,18 +55,7 @@ def main():# ======================================================# 3. build dataset and dataloader# ======================================================- dataset = DatasetFromCSV(- cfg.data_path,- transform=(- get_transforms_video(cfg.image_size[0])- if not cfg.use_image_transform- else get_transforms_image(cfg.image_size[0])- ),- num_frames=cfg.num_frames,- frame_interval=cfg.frame_interval,- root=cfg.root,- )-+ dataset = build_module(cfg.dataset, DATASETS)dataloader = prepare_dataloader(dataset,@@ -100,7 +90,7 @@ def main():# 3.4. support for multi-resolutionmodel_args = dict()if cfg.multi_resolution:- image_size = cfg.image_size+ image_size = cfg.dataset.image_sizehw = torch.tensor([image_size], device=device, dtype=dtype).repeat(cfg.batch_size, 1)ar = torch.tensor([[image_size[0] / image_size[1]]], device=device, dtype=dtype).repeat(cfg.batch_size, 1)model_args["data_info"] = dict(ar=ar, hw=hw)@@ -146,8 +136,8 @@ def main():loss_steps = 0disc_time_downsample_factor = 2 ** len(cfg.discriminator.channel_multipliers)- if cfg.num_frames % disc_time_downsample_factor != 0:- disc_time_padding = disc_time_downsample_factor - cfg.num_frames % disc_time_downsample_factor+ if cfg.datasets.num_frames % disc_time_downsample_factor != 0:+ disc_time_padding = disc_time_downsample_factor - cfg.datasets.num_frames % disc_time_downsample_factorelse:disc_time_padding = 0video_contains_first_frame = cfg.video_contains_first_frame