Open-Sora: Democratizing Efficient Video Production for All
Python
Exp/image mix (#20)
Zheng Zangwei (Alex Zheng)7612d22fc639
diff --git a/configs/dit/train/1x256x256.py b/configs/dit/train/1x256x256.pyindex 667e0a8..c423b24 100644--- a/configs/dit/train/1x256x256.py+++ b/configs/dit/train/1x256x256.py@@ -1,14 +1,15 @@-num_frames = 1-frame_interval = 1-image_size = (256, 256)-# Define dataset-root = None-data_path = "CSV_PATH"-use_image_transform = True-num_workers = 4+dataset = dict(+ type="VideoTextDataset",+ data_path=None,+ num_frames=1,+ frame_interval=1,+ image_size=(256, 256),+ transform_name="center",+)# Define acceleration+num_workers = 4dtype = "bf16"grad_checkpoint = Falseplugin = "zero2"diff --git a/configs/opensora-v1-1/inference/Vx1024.py b/configs/opensora-v1-1/inference/image.pysimilarity index 91%rename from configs/opensora-v1-1/inference/Vx1024.pyrename to configs/opensora-v1-1/inference/image.pyindex 649d9e5..96d6783 100644--- a/configs/opensora-v1-1/inference/Vx1024.py+++ b/configs/opensora-v1-1/inference/image.py@@ -1,6 +1,7 @@-num_frames = 4+num_frames = 1fps = 24 // 3-image_size = (704, 1472)+# image_size = (1358, 680)+image_size = (2160, 3840)multi_resolution = "STDiT2"# Define modeldiff --git a/configs/opensora-v1-1/train/test.py b/configs/opensora-v1-1/train/benchmark.pysimilarity index 73%rename from configs/opensora-v1-1/train/test.pyrename to configs/opensora-v1-1/train/benchmark.pyindex 87f4d3b..d3b60cb 100644--- a/configs/opensora-v1-1/train/test.py+++ b/configs/opensora-v1-1/train/benchmark.py@@ -4,15 +4,22 @@ dataset = dict(data_path=None,num_frames=None,frame_interval=3,- image_size=(None, None, None),+ image_size=(None, None),transform_name="resize_crop",)bucket_config = {- "256": {1: (1.0, 64)}, # 32 ok, 64 broken+ # "256": {1: (1.0, 256)}, # 4.5s/it+ # "512": {1: (1.0, 96)}, # 4.7s/it+ # "512": {1: (1.0, 128)}, # 6.3s/it+ # "480p": {1: (1.0, 50)}, # 4.0s/it+ # "1024": {1: (1.0, 32)}, # 6.8s/it+ # "1024": {1: (1.0, 20)}, # 4.3s/it+ # "1080p": {1: (1.0, 16)}, # 8.6s/it+ "1080p": {1: (1.0, 8)}, # 4.4s/it}# Define acceleration-num_workers = 0+num_workers = 4dtype = "bf16"grad_checkpoint = Trueplugin = "zero2"diff --git a/configs/opensora-v1-1/train/image.py b/configs/opensora-v1-1/train/image.pyindex 588f3c2..fe9a671 100644--- a/configs/opensora-v1-1/train/image.py+++ b/configs/opensora-v1-1/train/image.py@@ -4,15 +4,15 @@ dataset = dict(data_path=None,num_frames=None,frame_interval=3,- image_size=(None, None, None),+ image_size=(None, None),transform_name="resize_crop",)-bucket_config = {- "256": {1: (1.0, 128)},- "512": {1: (1.0, 2)},- "480p": {1: (1.0, 2)},- "1024": {1: (1.0, 2)},- "1080p": {1: (1.0, 2)},+bucket_config = { # 6s/it+ "256": {1: (1.0, 256)},+ "512": {1: (1.0, 80)},+ "480p": {1: (1.0, 52)},+ "1024": {1: (1.0, 20)},+ "1080p": {1: (1.0, 8)},}# Define acceleration@@ -53,7 +53,7 @@ wandb = Falseepochs = 1000log_every = 10-ckpt_every = 1000+ckpt_every = 500load = Nonebatch_size = 10 # only for loggingdiff --git a/opensora/datasets/aspect.py b/opensora/datasets/aspect.pyindex 0d772c9..7588cde 100644--- a/opensora/datasets/aspect.py+++ b/opensora/datasets/aspect.py@@ -3,10 +3,12 @@ import math# Ours-def get_h_w(a, ts):+def get_h_w(a, ts, eps=1e-4):h = (ts * a) ** 0.5+ h = h + epsh = math.ceil(h) if math.ceil(h) % 2 == 0 else math.floor(h)w = h / a+ w = w + epsw = math.ceil(w) if math.ceil(w) % 2 == 0 else math.floor(w)return h, w@@ -93,11 +95,11 @@ ASPECT_RATIO_720P = {"0.67": (784, 1176),"0.75": (832, 1110),"1.00": (960, 960),- "1.33": (1108, 831),+ "1.33": (1108, 832),"1.50": (1176, 784),"1.78": (1280, 720),"1.89": (1320, 698),- "2.00": (1358, 679),+ "2.00": (1358, 680),"2.08": (1386, 666),}@@ -116,7 +118,7 @@ ASPECT_RATIO_480P = {"1.52": (790, 520),"1.78": (854, 480),"1.92": (888, 462),- "2.00": (906, 453),+ "2.00": (906, 454),"2.10": (928, 442),}@@ -130,14 +132,14 @@ ASPECT_RATIO_360P = {"0.54": (470, 870),"0.56": (480, 854), # base"0.62": (506, 810),- "0.67": (522, 783),+ "0.67": (522, 784),"0.75": (554, 738),"1.00": (640, 640),"1.33": (740, 555),"1.50": (784, 522),"1.78": (854, 480),"1.89": (880, 466),- "2.00": (906, 453),+ "2.00": (906, 454),"2.08": (924, 444),}diff --git a/opensora/datasets/sampler.py b/opensora/datasets/sampler.pyindex 08e8930..1478ff2 100644--- a/opensora/datasets/sampler.py+++ b/opensora/datasets/sampler.py@@ -4,6 +4,7 @@ from pprint import pprintfrom typing import Iterator, List, Optionalimport torch+import torch.distributed as distfrom torch.utils.data import DistributedSamplerfrom .bucket import Bucket@@ -29,6 +30,27 @@ class VariableVideoBatchSampler(DistributedSampler):self.bucket = Bucket(bucket_config)self.verbose = verboseself.last_micro_batch_access_index = 0+ self.approximate_num_batch = None++ def get_num_batch(self) -> int:+ g = torch.Generator()+ g.manual_seed(self.seed + self.epoch)+ bucket_sample_dict = OrderedDict()++ # group by bucket+ # each data sample is put into a bucket with a similar image/video size+ for i in range(len(self.dataset)):+ t, h, w = self.dataset.get_data_info(i)+ bucket_id = self.bucket.get_bucket_id(t, h, w, self.dataset.frame_interval, g)+ if bucket_id is None:+ continue+ if bucket_id not in bucket_sample_dict:+ bucket_sample_dict[bucket_id] = []+ bucket_sample_dict[bucket_id].append(i)++ # calculate the number of batches+ self._print_bucket_info(bucket_sample_dict)+ return self.approximate_num_batchdef __iter__(self) -> Iterator[List[int]]:g = torch.Generator()@@ -155,8 +177,9 @@ class VariableVideoBatchSampler(DistributedSampler):def load_state_dict(self, state_dict: dict) -> None:self.__dict__.update(state_dict)- def _print_bucket_info(self, bucket_sample_dict: dict) -> None:+ def _print_bucket_info(self, bucket_sample_dict: dict, verbose=True) -> None:total_samples = 0+ num_batch = 0num_dict = {}num_aspect_dict = defaultdict(int)num_hwt_dict = defaultdict(int)@@ -166,13 +189,17 @@ class VariableVideoBatchSampler(DistributedSampler):num_dict[k] = sizenum_aspect_dict[k[-1]] += sizenum_hwt_dict[k[:-1]] += size- print(f"Total training samples: {total_samples}, num buckets: {len(num_dict)}")- print("Bucket samples:")- pprint(num_dict)- print("Bucket samples by HxWxT:")- pprint(num_hwt_dict)- print("Bucket samples by aspect ratio:")- pprint(num_aspect_dict)+ num_batch += size // self.bucket.get_batch_size(k[:-1])+ if dist.get_rank() == 0 and verbose:+ print(f"Total training samples: {total_samples}, num buckets: {len(num_dict)}")+ print("Bucket samples:")+ pprint(num_dict)+ print("Bucket samples by HxWxT:")+ pprint(num_hwt_dict)+ print("Bucket samples by aspect ratio:")+ pprint(num_aspect_dict)+ print(f"Number of batches: {num_batch}")+ self.approximate_num_batch = num_batchdef set_epoch(self, epoch: int) -> None:super().set_epoch(epoch)diff --git a/opensora/models/vae/vae.py b/opensora/models/vae/vae.pyindex e6e1441..79f78b1 100644--- a/opensora/models/vae/vae.py+++ b/opensora/models/vae/vae.py@@ -53,9 +53,9 @@ class VideoAutoencoderKL(nn.Module):def get_latent_size(self, input_size):latent_size = []for i in range(3):- assert (- input_size[i] is None or input_size[i] % self.patch_size[i] == 0- ), "Input size must be divisible by patch size"+ # assert (+ # input_size[i] is None or input_size[i] % self.patch_size[i] == 0+ # ), "Input size must be divisible by patch size"latent_size.append(input_size[i] // self.patch_size[i] if input_size[i] is not None else None)return latent_size@@ -89,9 +89,9 @@ class VideoAutoencoderKLTemporalDecoder(nn.Module):def get_latent_size(self, input_size):latent_size = []for i in range(3):- assert (- input_size[i] is None or input_size[i] % self.patch_size[i] == 0- ), "Input size must be divisible by patch size"+ # assert (+ # input_size[i] is None or input_size[i] % self.patch_size[i] == 0+ # ), "Input size must be divisible by patch size"latent_size.append(input_size[i] // self.patch_size[i] if input_size[i] is not None else None)return latent_sizediff --git a/scripts/train.py b/scripts/train.pyindex 439b461..8ddf9c6 100644--- a/scripts/train.py+++ b/scripts/train.py@@ -37,8 +37,6 @@ def main():# 1. args & cfg# ======================================================cfg = parse_configs(training=True)- print("Training configuration:")- pprint(cfg._cfg_dict)exp_name, exp_dir = create_experiment_workspace(cfg)save_training_config(cfg._cfg_dict, exp_dir)@@ -58,6 +56,8 @@ def main():if not coordinator.is_master():logger = create_logger(None)else:+ print("Training configuration:")+ pprint(cfg._cfg_dict)logger = create_logger(exp_dir)logger.info(f"Experiment directory created at {exp_dir}")@@ -173,13 +173,16 @@ def main():dataloader=dataloader,)torch.set_default_dtype(torch.float)- num_steps_per_epoch = len(dataloader)logger.info("Boost model for distributed training")+ if cfg.dataset.type == "VariableVideoTextDataset":+ num_steps_per_epoch = dataloader.batch_sampler.get_num_batch() // dist.get_world_size()+ else:+ num_steps_per_epoch = len(dataloader)# =======================================================# 6. training loop# =======================================================- start_epoch = start_step = log_step = sampler_start_idx = 0+ start_epoch = start_step = log_step = sampler_start_idx = acc_step = 0running_loss = 0.0sampler_to_io = dataloader.batch_sampler if cfg.dataset.type == "VariableVideoTextDataset" else None# 6.1. resume training@@ -254,6 +257,7 @@ def main():running_loss += loss.item()global_step = epoch * num_steps_per_epoch + steplog_step += 1+ acc_step += 1# Log to tensorboardif coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:@@ -269,6 +273,7 @@ def main():"epoch": epoch,"loss": loss.item(),"avg_loss": avg_loss,+ "acc_step": acc_step,},step=global_step,)