Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[feature] add batch size search script (#47)

Hongxin Liu2393c6308396
diff --git a/configs/opensora-v1-1/train/benchmark.py b/configs/opensora-v1-1/train/benchmark.py
index e0e14c7..18b1c4a 100644
--- a/configs/opensora-v1-1/train/benchmark.py
+++ b/configs/opensora-v1-1/train/benchmark.py
@@ -1,19 +1,29 @@
+# this file is only for batch size search and is not used for training
+
# Define dataset
dataset = dict(
type="VariableVideoTextDataset",
data_path=None,
num_frames=None,
- frame_interval=1,
+ frame_interval=3,
image_size=(None, None),
transform_name="resize_crop",
)
+
+# bucket config format:
+# 1. { resolution: {num_frames: (prob, batch_size)} }, in this case batch_size is ignored when searching
+# 2. { resolution: {num_frames: (prob, (max_batch_size, ))} }, batch_size is searched in the range [batch_size_start, max_batch_size), batch_size_start is configured via CLI
+# 3. { resolution: {num_frames: (prob, (min_batch_size, max_batch_size))} }, batch_size is searched in the range [min_batch_size, max_batch_size)
+# 4. { resolution: {num_frames: (prob, (min_batch_size, max_batch_size, step_size))} }, batch_size is searched in the range [min_batch_size, max_batch_size) with step_size (grid search)
+# 5. { resolution: {num_frames: (0.0, None)} }, this bucket will not be used
+
bucket_config = {
# "240p": {128: (1.0, 2)}, # 4.28s/it
# "240p": {64: (1.0, 4)},
# "240p": {32: (1.0, 8)}, # 4.6s/it
# "240p": {16: (1.0, 16)}, # 4.6s/it
# "480p": {16: (1.0, 4)}, # 4.6s/it
- "720p": {16: (1.0, 2)}, # 5.89s/it
+ # "720p": {16: (1.0, 2)}, # 5.89s/it
# "256": {1: (1.0, 256)}, # 4.5s/it
# "512": {1: (1.0, 96)}, # 4.7s/it
# "512": {1: (1.0, 128)}, # 6.3s/it
@@ -22,6 +32,18 @@ bucket_config = {
# "1024": {1: (1.0, 20)}, # 4.3s/it
# "1080p": {1: (1.0, 16)}, # 8.6s/it
# "1080p": {1: (1.0, 8)}, # 4.4s/it
+ "240p": {
+ 16: (1.0, (2, 32)),
+ 32: (1.0, (2, 16)),
+ 64: (1.0, (2, 8)),
+ 128: (1.0, (2, 6)),
+ },
+ "256": {1: (1.0, (128, 300))},
+ "512": {1: (0.5, (64, 128))},
+ "480p": {1: (0.4, (32, 128)), 16: (0.4, (2, 32)), 32: (0.0, None)},
+ "720p": {16: (0.1, (2, 16)), 32: (0.0, None)}, # No examples now
+ "1024": {1: (0.3, (8, 64))},
+ "1080p": {1: (0.3, (2, 32))},
}
# mask_ratios = {
# "mask_no": 0.0,
diff --git a/docs/commands.md b/docs/commands.md
index 83fb42c..7827cd9 100644
--- a/docs/commands.md
+++ b/docs/commands.md
@@ -90,3 +90,49 @@ You can modify corresponding config files to change the training settings. See m
### Training Hyperparameters
1. `dtype` is the data type for training. Only `fp16` and `bf16` are supported. ColossalAI automatically enables the mixed precision training for `fp16` and `bf16`. During training, we find `bf16` more stable.
+
+## Search batch size for buckets
+
+To search the batch size for buckets, run the following command.
+
+```bash
+torchrun --standalone --nproc_per_node 1 scripts/search_bs.py configs/opensora-v1-1/train/benchmark.py --data-path YOUR_CSV_PATH -o YOUR_OUTPUT_CONFIG_PATH --base-resolution 240p --base-frames 128 --batch-size-start 2 --batch-size-end 256 --batch-size-step 2
+```
+
+If your dataset is extremely large, you extract a subset of the dataset for the search.
+
+```bash
+# each bucket contains 1000 samples
+python tools/datasets/split.py YOUR_CSV_PATH -o YOUR_SUBSET_CSV_PATH -c configs/opensora-v1-1/train/video.py -l 1000
+```
+
+If you want to control the batch size search more granularly, you can configure batch size start, end, and step in the config file.
+
+Bucket config format:
+1. `{ resolution: {num_frames: (prob, batch_size)} }`, in this case batch_size is ignored when searching
+2. `{ resolution: {num_frames: (prob, (max_batch_size, ))} }`, batch_size is searched in the range `[batch_size_start, max_batch_size)`, batch_size_start is configured via CLI
+3. `{ resolution: {num_frames: (prob, (min_batch_size, max_batch_size))} }`, batch_size is searched in the range `[min_batch_size, max_batch_size)`
+4. `{ resolution: {num_frames: (prob, (min_batch_size, max_batch_size, step_size))} }`, batch_size is searched in the range `[min_batch_size, max_batch_size)` with step_size (grid search)
+5. `{ resolution: {num_frames: (0.0, None)} }`, this bucket will not be used
+
+Here is an example of the bucket config:
+
+```python
+bucket_config = {
+
+ "240p": {
+ 16: (1.0, (2, 32)),
+ 32: (1.0, (2, 16)),
+ 64: (1.0, (2, 8)),
+ 128: (1.0, (2, 6)),
+ },
+ "256": {1: (1.0, (128, 300))},
+ "512": {1: (0.5, (64, 128))},
+ "480p": {1: (0.4, (32, 128)), 16: (0.4, (2, 32)), 32: (0.0, None)},
+ "720p": {16: (0.1, (2, 16)), 32: (0.0, None)}, # No examples now
+ "1024": {1: (0.3, (8, 64))},
+ "1080p": {1: (0.3, (2, 32))},
+}
+```
+
+It will print the best batch size (and corresponding step time) for each bucket and save the output config file.
\ No newline at end of file
diff --git a/scripts/search_bs.py b/scripts/search_bs.py
new file mode 100644
index 0000000..af224e9
--- /dev/null
+++ b/scripts/search_bs.py
@@ -0,0 +1,492 @@
+import argparse
+import time
+import traceback
+from copy import deepcopy
+
+import colossalai
+import torch
+import torch.distributed as dist
+from colossalai.booster import Booster
+from colossalai.booster.plugin import LowLevelZeroPlugin
+from colossalai.cluster import DistCoordinator
+from colossalai.nn.optimizer import HybridAdam
+from colossalai.utils import get_current_device
+from mmengine.config import Config
+from tqdm import tqdm
+
+from opensora.acceleration.checkpoint import set_grad_checkpoint
+from opensora.acceleration.parallel_states import (
+ get_data_parallel_group,
+ set_data_parallel_group,
+ set_sequence_parallel_group,
+)
+from opensora.acceleration.plugin import ZeroSeqParallelPlugin
+from opensora.datasets import prepare_variable_dataloader
+from opensora.registry import DATASETS, MODELS, SCHEDULERS, build_module
+from opensora.utils.ckpt_utils import model_sharding
+from opensora.utils.config_utils import merge_args, parse_configs
+from opensora.utils.misc import (
+ format_numel_str,
+ get_model_numel,
+ requires_grad,
+ to_torch_dtype,
+)
+from opensora.utils.train_utils import MaskGenerator, update_ema
+
+
+class BColors:
+ HEADER = "\033[95m"
+ OKBLUE = "\033[94m"
+ OKCYAN = "\033[96m"
+ OKGREEN = "\033[92m"
+ WARNING = "\033[93m"
+ FAIL = "\033[91m"
+ ENDC = "\033[0m"
+ BOLD = "\033[1m"
+ UNDERLINE = "\033[4m"
+
+
+# BUCKETS = [
+# ("240p", 16),
+# ("240p", 32),
+# ("240p", 64),
+# ("240p", 128),
+# ("256", 1),
+# ("512", 1),
+# ("480p", 1),
+# ("480p", 16),
+# ("480p", 32),
+# ("720p", 16),
+# ("720p", 32),
+# ("1024", 1),
+# ("1080p", 1),
+# ]
+
+
+def parse_configs():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("config", help="model config file path")
+ parser.add_argument(
+ "-o", "--output", help="output config file path", default="output_config.py"
+ )
+
+ parser.add_argument("--seed", default=42, type=int, help="generation seed")
+ parser.add_argument(
+ "--ckpt-path",
+ type=str,
+ help="path to model ckpt; will overwrite cfg.ckpt_path if specified",
+ )
+ parser.add_argument(
+ "--data-path", default=None, type=str, help="path to data csv", required=True
+ )
+ parser.add_argument("--warmup-steps", default=1, type=int, help="warmup steps")
+ parser.add_argument("--active-steps", default=1, type=int, help="active steps")
+ parser.add_argument(
+ "--base-resolution", default="240p", type=str, help="base resolution"
+ )
+ parser.add_argument("--base-frames", default=128, type=int, help="base frames")
+ parser.add_argument(
+ "--batch-size-start", default=2, type=int, help="batch size start"
+ )
+ parser.add_argument(
+ "--batch-size-end", default=256, type=int, help="batch size end"
+ )
+ parser.add_argument(
+ "--batch-size-step", default=2, type=int, help="batch size step"
+ )
+ args = parser.parse_args()
+ cfg = Config.fromfile(args.config)
+ cfg = merge_args(cfg, args, training=True)
+ return cfg, args
+
+
+def rewrite_config(cfg, resolution, num_frames, batch_size):
+ cfg.bucket_config = {resolution: {num_frames: (1.0, batch_size)}}
+ return cfg
+
+
+def update_bucket_config_bs(bucket_config, resolution, num_frames, batch_size):
+ p, _ = bucket_config[resolution][num_frames]
+ bucket_config[resolution][num_frames] = (p, batch_size)
+
+
+def main():
+ # ======================================================
+ # 1. args & cfg
+ # ======================================================
+ cfg, args = parse_configs()
+ print(cfg)
+ assert (
+ cfg.dataset.type == "VariableVideoTextDataset"
+ ), "Only VariableVideoTextDataset is supported"
+
+ # ======================================================
+ # 2. runtime variables & colossalai launch
+ # ======================================================
+ assert torch.cuda.is_available(), "Training currently requires at least one GPU."
+ assert cfg.dtype in ["fp16", "bf16"], f"Unknown mixed precision {cfg.dtype}"
+
+ # 2.1. colossalai init distributed training
+ colossalai.launch_from_torch({})
+ coordinator = DistCoordinator()
+ device = get_current_device()
+ dtype = to_torch_dtype(cfg.dtype)
+
+ # 2.3. initialize ColossalAI booster
+ if cfg.plugin == "zero2":
+ plugin = LowLevelZeroPlugin(
+ stage=2,
+ precision=cfg.dtype,
+ initial_scale=2**16,
+ max_norm=cfg.grad_clip,
+ )
+ set_data_parallel_group(dist.group.WORLD)
+ elif cfg.plugin == "zero2-seq":
+ plugin = ZeroSeqParallelPlugin(
+ sp_size=cfg.sp_size,
+ stage=2,
+ precision=cfg.dtype,
+ initial_scale=2**16,
+ max_norm=cfg.grad_clip,
+ )
+ set_sequence_parallel_group(plugin.sp_group)
+ set_data_parallel_group(plugin.dp_group)
+ else:
+ raise ValueError(f"Unknown plugin {cfg.plugin}")
+ booster = Booster(plugin=plugin)
+
+ # ======================================================
+ # 4. build model
+ # ======================================================
+ # 4.1. build model
+ text_encoder = build_module(cfg.text_encoder, MODELS, device=device)
+ vae = build_module(cfg.vae, MODELS)
+ input_size = (cfg.dataset.num_frames, *cfg.dataset.image_size)
+ latent_size = vae.get_latent_size(input_size)
+ model = build_module(
+ cfg.model,
+ MODELS,
+ input_size=latent_size,
+ in_channels=vae.out_channels,
+ caption_channels=text_encoder.output_dim,
+ model_max_length=text_encoder.model_max_length,
+ dtype=dtype,
+ )
+ model_numel, model_numel_trainable = get_model_numel(model)
+ coordinator.print_on_master(
+ f"Trainable model params: {format_numel_str(model_numel_trainable)}, Total model params: {format_numel_str(model_numel)}"
+ )
+
+ # 4.2. create ema
+ ema = deepcopy(model).to(torch.float32).to(device)
+ requires_grad(ema, False)
+
+ # 4.3. move to device
+ vae = vae.to(device, dtype)
+ model = model.to(device, dtype)
+
+ # 4.4. build scheduler
+ scheduler = build_module(cfg.scheduler, SCHEDULERS)
+
+ # 4.5. setup optimizer
+ optimizer = HybridAdam(
+ filter(lambda p: p.requires_grad, model.parameters()),
+ lr=cfg.lr,
+ weight_decay=0,
+ adamw_mode=True,
+ )
+ lr_scheduler = None
+
+ # 4.6. prepare for training
+ if cfg.grad_checkpoint:
+ set_grad_checkpoint(model)
+ model.train()
+ update_ema(ema, model, decay=0, sharded=False)
+ ema.eval()
+ if cfg.mask_ratios is not None:
+ mask_generator = MaskGenerator(cfg.mask_ratios)
+ else:
+ mask_generator = None
+
+ # =======================================================
+ # 5. boost model for distributed training with colossalai
+ # =======================================================
+ torch.set_default_dtype(dtype)
+ model, optimizer, _, _, lr_scheduler = booster.boost(
+ model=model,
+ optimizer=optimizer,
+ lr_scheduler=lr_scheduler,
+ )
+ torch.set_default_dtype(torch.float)
+ coordinator.print_on_master("Boost model for distributed training")
+
+ model_sharding(ema)
+
+ buckets = [
+ (res, f)
+ for res, d in cfg.bucket_config.items()
+ for f, (p, bs) in d.items()
+ if bs is not None and p > 0.0
+ ]
+ output_bucket_cfg = deepcopy(cfg.bucket_config)
+ # find the base batch size
+ assert (args.base_resolution, args.base_frames) in buckets
+ del buckets[buckets.index((args.base_resolution, args.base_frames))]
+ base_batch_size, base_step_time = benchmark(
+ args,
+ cfg,
+ args.base_resolution,
+ args.base_frames,
+ device,
+ dtype,
+ booster,
+ vae,
+ text_encoder,
+ model,
+ mask_generator,
+ scheduler,
+ optimizer,
+ ema,
+ )
+ update_bucket_config_bs(
+ output_bucket_cfg, args.base_resolution, args.base_frames, base_batch_size
+ )
+ coordinator.print_on_master(
+ f"{BColors.OKBLUE}Base resolution: {args.base_resolution}, Base frames: {args.base_frames}, Batch size: {base_batch_size}, Base step time: {base_step_time}{BColors.ENDC}"
+ )
+ result_table = [
+ f"{args.base_resolution}, {args.base_frames}, {base_batch_size}, {base_step_time:.2f}"
+ ]
+ for resolution, frames in buckets:
+ try:
+ batch_size, step_time = benchmark(
+ args,
+ cfg,
+ resolution,
+ frames,
+ device,
+ dtype,
+ booster,
+ vae,
+ text_encoder,
+ model,
+ mask_generator,
+ scheduler,
+ optimizer,
+ ema,
+ target_step_time=base_step_time,
+ )
+ coordinator.print_on_master(
+ f"{BColors.OKBLUE}Resolution: {resolution}, Frames: {frames}, Batch size: {batch_size}, Step time: {step_time}{BColors.ENDC}"
+ )
+ update_bucket_config_bs(output_bucket_cfg, resolution, frames, batch_size)
+ result_table.append(
+ f"{resolution}, {frames}, {batch_size}, {step_time:.2f}"
+ )
+ except RuntimeError:
+ pass
+ result_table = "\n".join(result_table)
+ coordinator.print_on_master(
+ f"{BColors.OKBLUE}Resolution, Frames, Batch size, Step time\n{result_table}{BColors.ENDC}"
+ )
+ coordinator.print_on_master(f"{BColors.OKBLUE}{output_bucket_cfg}{BColors.ENDC}")
+ if coordinator.is_master():
+ cfg.bucket_config = output_bucket_cfg
+ cfg.dump(args.output)
+
+
+def benchmark(
+ args,
+ cfg,
+ resolution,
+ num_frames,
+ device,
+ dtype,
+ booster,
+ vae,
+ text_encoder,
+ model,
+ mask_generator,
+ scheduler,
+ optimizer,
+ ema,
+ target_step_time=None,
+):
+ batch_sizes = []
+ step_times = []
+
+ def run_step(bs) -> float:
+ step_time = train(
+ args,
+ cfg,
+ resolution,
+ num_frames,
+ bs,
+ device,
+ dtype,
+ booster,
+ vae,
+ text_encoder,
+ model,
+ mask_generator,
+ scheduler,
+ optimizer,
+ ema,
+ )
+ step_times.append(step_time)
+ batch_sizes.append(bs)
+ return step_time
+
+ orig_bs = cfg.bucket_config[resolution][num_frames][1]
+ lower_bound = args.batch_size_start
+ upper_bound = args.batch_size_end
+ step_size = args.batch_size_step
+ if isinstance(orig_bs, tuple):
+ if len(orig_bs) == 1:
+ upper_bound = orig_bs[0]
+ elif len(orig_bs) == 2:
+ lower_bound, upper_bound = orig_bs
+ elif len(orig_bs) == 3:
+ lower_bound, upper_bound, step_size = orig_bs
+ batch_start_size = lower_bound
+
+ while lower_bound < upper_bound:
+ mid = (lower_bound + upper_bound) // 2
+ try:
+ step_time = run_step(mid)
+ lower_bound = mid + 1
+ except Exception:
+ traceback.print_exc()
+ upper_bound = mid
+
+ for batch_size in range(batch_start_size, upper_bound, step_size):
+ if batch_size in batch_sizes:
+ continue
+ step_time = run_step(batch_size)
+ if len(step_times) == 0:
+ raise RuntimeError("No valid batch size found")
+ if target_step_time is None:
+ # find the fastest batch size
+ throughputs = [
+ batch_size / step_time
+ for step_time, batch_size in zip(step_times, batch_sizes)
+ ]
+ max_throughput = max(throughputs)
+ target_batch_size = batch_sizes[throughputs.index(max_throughput)]
+ step_time = step_times[throughputs.index(max_throughput)]
+ else:
+ # find the batch size that meets the target step time
+ diff = [abs(t - target_step_time) for t in step_times]
+ closest_step_time = min(diff)
+ target_batch_size = batch_sizes[diff.index(closest_step_time)]
+ step_time = step_times[diff.index(closest_step_time)]
+ return target_batch_size, step_time
+
+
+def train(
+ args,
+ cfg,
+ resolution,
+ num_frames,
+ batch_size,
+ device,
+ dtype,
+ booster,
+ vae,
+ text_encoder,
+ model,
+ mask_generator,
+ scheduler,
+ optimizer,
+ ema,
+):
+ total_steps = args.warmup_steps + args.active_steps
+ cfg = rewrite_config(deepcopy(cfg), resolution, num_frames, batch_size)
+
+ dataset = build_module(cfg.dataset, DATASETS)
+ dataset.dummy = True
+ dataloader_args = dict(
+ dataset=dataset,
+ batch_size=cfg.batch_size,
+ num_workers=cfg.num_workers,
+ shuffle=True,
+ drop_last=True,
+ pin_memory=True,
+ process_group=get_data_parallel_group(),
+ )
+ dataloader = prepare_variable_dataloader(
+ bucket_config=cfg.bucket_config,
+ **dataloader_args,
+ )
+ dataloader_iter = iter(dataloader)
+ num_steps_per_epoch = (
+ dataloader.batch_sampler.get_num_batch() // dist.get_world_size()
+ )
+
+ assert (
+ num_steps_per_epoch >= total_steps
+ ), f"num_steps_per_epoch={num_steps_per_epoch} < total_steps={total_steps}"
+ duration = 0
+ # this is essential for the first iteration after OOM
+ optimizer._grad_store.reset_all_gradients()
+ optimizer._bucket_store.reset_num_elements_in_bucket()
+ optimizer._bucket_store.grad_to_param_mapping = dict()
+ optimizer._bucket_store._grad_in_bucket = dict()
+ optimizer._bucket_store._param_list = []
+ optimizer._bucket_store._padding_size = []
+ for rank in range(optimizer._bucket_store._world_size):
+ optimizer._bucket_store._grad_in_bucket[rank] = []
+ optimizer._bucket_store.offset_list = [0]
+ optimizer.zero_grad()
+ for step, batch in tqdm(
+ enumerate(dataloader_iter),
+ desc=f"{resolution}:{num_frames} bs={batch_size}",
+ total=total_steps,
+ ):
+ if step >= total_steps:
+ break
+ if step >= args.warmup_steps:
+ start = time.time()
+
+ x = batch.pop("video").to(device, dtype) # [B, C, T, H, W]
+ y = batch.pop("text")
+ # Visual and text encoding
+ with torch.no_grad():
+ # Prepare visual inputs
+ x = vae.encode(x) # [B, C, T, H/P, W/P]
+ # Prepare text inputs
+ model_args = text_encoder.encode(y)
+
+ # Mask
+ if cfg.mask_ratios is not None:
+ mask = mask_generator.get_masks(x)
+ model_args["x_mask"] = mask
+ else:
+ mask = None
+
+ # Video info
+ for k, v in batch.items():
+ model_args[k] = v.to(device, dtype)
+
+ # Diffusion
+ t = torch.randint(0, scheduler.num_timesteps, (x.shape[0],), device=device)
+ loss_dict = scheduler.training_losses(model, x, t, model_args, mask=mask)
+
+ # Backward & update
+ loss = loss_dict["loss"].mean()
+ booster.backward(loss=loss, optimizer=optimizer)
+ optimizer.step()
+ optimizer.zero_grad()
+
+ # Update EMA
+ update_ema(ema, model.module, optimizer=optimizer)
+ if step >= args.warmup_steps:
+ end = time.time()
+ duration += end - start
+
+ avg_step_time = duration / args.active_steps
+ return avg_step_time
+
+
+if __name__ == "__main__":
+ main()