Open-Sora: Democratizing Efficient Video Production for All
Python
add extract_vae_feat
pxy5585e3dae3a4
diff --git a/scripts/misc/extract_vae_feat.py b/scripts/misc/extract_vae_feat.pynew file mode 100644index 0000000..f99d233--- /dev/null+++ b/scripts/misc/extract_vae_feat.py@@ -0,0 +1,284 @@+import os+from copy import deepcopy+from datetime import timedelta+from pprint import pformat++import torch+import torch.distributed as dist+from colossalai.booster import Booster+from colossalai.cluster import DistCoordinator+from colossalai.nn.optimizer import HybridAdam+from colossalai.utils import get_current_device, set_seed+from tqdm import tqdm++import wandb+from opensora.acceleration.checkpoint import set_grad_checkpoint+from opensora.acceleration.parallel_states import get_data_parallel_group+from opensora.datasets import prepare_dataloader, prepare_variable_dataloader+from opensora.datasets.utils import collate_fn_ignore_none+from opensora.registry import DATASETS, MODELS, SCHEDULERS, build_module+from opensora.utils.ckpt_utils import load, model_gathering, model_sharding, record_model_param_shape, save+from opensora.utils.config_utils import define_experiment_workspace, parse_configs, save_training_config+from opensora.utils.misc import (+ all_reduce_mean,+ create_logger,+ create_tensorboard_writer,+ format_numel_str,+ get_model_numel,+ requires_grad,+ to_torch_dtype,+)+from opensora.utils.train_utils import MaskGenerator, create_colossalai_plugin, update_ema++DEFAULT_DATASET_NAME = "VideoTextDataset"+++def main():+ # ======================================================+ # 1. configs & runtime variables+ # ======================================================+ # == parse configs ==+ cfg = parse_configs(training=False)++ # == device and dtype ==+ assert torch.cuda.is_available(), "Training currently requires at least one GPU."+ cfg_dtype = cfg.get("dtype", "bf16")+ assert cfg_dtype in ["fp16", "bf16"], f"Unknown mixed precision {cfg_dtype}"+ dtype = to_torch_dtype(cfg.get("dtype", "bf16"))++ # == colossalai init distributed training ==+ # NOTE: A very large timeout is set to avoid some processes exit early+ dist.init_process_group(backend="nccl", timeout=timedelta(hours=24))+ torch.cuda.set_device(dist.get_rank() % torch.cuda.device_count())+ set_seed(cfg.get("seed", 1024))+ coordinator = DistCoordinator()+ device = get_current_device()++ # == init exp_dir ==+ exp_name, exp_dir = define_experiment_workspace(cfg)+ coordinator.block_all()+ if coordinator.is_master():+ os.makedirs(exp_dir, exist_ok=True)+ save_training_config(cfg.to_dict(), exp_dir)+ coordinator.block_all()++ # == init logger, tensorboard & wandb ==+ logger = create_logger(exp_dir)+ logger.info("Experiment directory created at %s", exp_dir)+ logger.info("Training configuration:\n %s", pformat(cfg.to_dict()))+ if coordinator.is_master():+ tb_writer = create_tensorboard_writer(exp_dir)+ if cfg.get("wandb", False):+ wandb.init(project="minisora", name=exp_name, config=cfg.to_dict(), dir="./outputs/wandb")++ # == init ColossalAI booster ==+ plugin = create_colossalai_plugin(+ plugin=cfg.get("plugin", "zero2"),+ dtype=cfg_dtype,+ grad_clip=cfg.get("grad_clip", 0),+ sp_size=cfg.get("sp_size", 1),+ )+ booster = Booster(plugin=plugin)++ # ======================================================+ # 2. build dataset and dataloader+ # ======================================================+ logger.info("Building dataset...")+ # == build dataset ==+ dataset = build_module(cfg.dataset, DATASETS)+ logger.info("Dataset contains %s samples.", len(dataset))++ # == build dataloader ==+ dataloader_args = dict(+ dataset=dataset,+ batch_size=cfg.get("batch_size", None),+ num_workers=cfg.get("num_workers", 4),+ seed=cfg.get("seed", 1024),+ shuffle=True,+ drop_last=True,+ pin_memory=True,+ process_group=get_data_parallel_group(),+ collate_fn=collate_fn_ignore_none,+ )+ if cfg.dataset.type == DEFAULT_DATASET_NAME:+ dataloader = prepare_dataloader(**dataloader_args)+ total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.get("sp_size", 1)+ logger.info("Total batch size: %s", total_batch_size)+ num_steps_per_epoch = len(dataloader)+ sampler_to_io = None+ else:+ dataloader = prepare_variable_dataloader(+ bucket_config=cfg.get("bucket_config", None),+ num_bucket_build_workers=cfg.get("num_bucket_build_workers", 1),+ **dataloader_args,+ )+ num_steps_per_epoch = dataloader.batch_sampler.get_num_batch() // dist.get_world_size()+ sampler_to_io = None if cfg.get("start_from_scratch ", False) else dataloader.batch_sampler++ # ======================================================+ # 3. build model+ # ======================================================+ logger.info("Building models...")+ # == build text-encoder and vae ==+ text_encoder = build_module(cfg.text_encoder, MODELS, device=device, dtype=dtype)+ vae = build_module(cfg.vae, MODELS).to(device, dtype).eval()++ # == build diffusion model ==+ input_size = (dataset.num_frames, *dataset.image_size)+ latent_size = vae.get_latent_size(input_size)+ model = (+ build_module(+ cfg.model,+ MODELS,+ input_size=latent_size,+ in_channels=vae.out_channels,+ caption_channels=text_encoder.output_dim,+ model_max_length=text_encoder.model_max_length,+ )+ .to(device, dtype)+ .train()+ )+ model_numel, model_numel_trainable = get_model_numel(model)+ logger.info(+ "[Diffusion] Trainable model params: %s, Total model params: %s",+ format_numel_str(model_numel_trainable),+ format_numel_str(model_numel),+ )++ # == build ema for diffusion model ==+ ema = deepcopy(model).to(torch.float32).to(device)+ requires_grad(ema, False)+ ema_shape_dict = record_model_param_shape(ema)+ ema.eval()+ update_ema(ema, model, decay=0, sharded=False)++ # == setup loss function, build scheduler ==+ scheduler = build_module(cfg.scheduler, SCHEDULERS)++ # == setup optimizer ==+ optimizer = HybridAdam(+ filter(lambda p: p.requires_grad, model.parameters()),+ adamw_mode=True,+ lr=cfg.get("lr", 1e-4),+ weight_decay=cfg.get("weight_decay", 0),+ eps=cfg.get("adam_eps", 1e-8),+ )+ lr_scheduler = None++ # == additional preparation ==+ if cfg.get("grad_checkpoint", False):+ set_grad_checkpoint(model)+ if cfg.get("mask_ratios", None) is not None:+ mask_generator = MaskGenerator(cfg.mask_ratios)++ # =======================================================+ # 4. distributed training preparation with colossalai+ # =======================================================+ logger.info("Preparing for distributed training...")+ # == boosting ==+ # NOTE: we set dtype first to make initialization of model consistent with the dtype; then reset it to the fp32 as we make diffusion scheduler in fp32+ torch.set_default_dtype(dtype)+ model, optimizer, _, dataloader, lr_scheduler = booster.boost(+ model=model,+ optimizer=optimizer,+ lr_scheduler=lr_scheduler,+ dataloader=dataloader,+ )+ torch.set_default_dtype(torch.float)+ logger.info("Boosting model for distributed training")++ # == global variables ==+ cfg_epochs = cfg.get("epochs", 1000)+ start_epoch = start_step = log_step = sampler_start_idx = acc_step = 0+ running_loss = 0.0+ logger.info("Training for %s epochs with %s steps per epoch", cfg_epochs, num_steps_per_epoch)++ # == resume ==+ if cfg.get("load", None) is not None:+ logger.info("Loading checkpoint")+ ret = load(+ booster,+ cfg.load,+ model=model,+ ema=ema,+ optimizer=optimizer,+ lr_scheduler=lr_scheduler,+ sampler=sampler_to_io,+ )+ if not cfg.get("start_from_scratch ", False):+ start_epoch, start_step, sampler_start_idx = ret+ logger.info("Loaded checkpoint %s at epoch %s step %s", cfg.load, start_epoch, start_step)+ if cfg.dataset.type == DEFAULT_DATASET_NAME:+ dataloader.sampler.set_start_index(sampler_start_idx)++ model_sharding(ema)++ # =======================================================+ # 5. training loop+ # =======================================================+ dist.barrier()+ for epoch in range(1):+ # == set dataloader to new epoch ==+ if cfg.dataset.type == DEFAULT_DATASET_NAME:+ dataloader.sampler.set_epoch(epoch)+ dataloader_iter = iter(dataloader)+ logger.info("Beginning epoch %s...", epoch)++ # == training loop in an epoch ==+ save_root = f'/mnt/nfs-207/sora_data/webvid-10M/feat'+ data_list = []+ cnt = 0+ max_len = 10++ with tqdm(+ enumerate(dataloader_iter, start=start_step),+ desc=f"Epoch {epoch}",+ disable=not coordinator.is_master(),+ initial=start_step,+ total=num_steps_per_epoch,+ ) as pbar:+ for step, batch in pbar:+ x = batch.pop("video").to(device, dtype) # [B, C, T, H, W]+ y = batch.pop("text")++ # == visual and text encoding ==+ with torch.no_grad():+ # Prepare visual inputs+ x = vae.encode(x) # [B, C, T, H/P, W/P]+ # Prepare text inputs++ model_args = {}+ # == video meta info ==+ for k, v in batch.items():+ model_args[k] = v.to(device, dtype)++ # keys to save: ['x', 'y', 'mask', 'fps']+ data_i = {+ 'x': x.detach().cpu(),+ 'fps': model_args['fps'].detach().cpu(),+ }++ save_text_feat = [True, False]+ if save_text_feat:+ with torch.no_grad():+ model_args_text = text_encoder.encode(y)+ data_i.update({+ 'y': model_args_text['y'].detach().cpu(),+ 'mask': model_args_text['mask'].detach().cpu(),+ })+ breakpoint()+ else:+ data_i['text'] = y++ data_list.append(data_i)++ if len(data_list) == max_len:+ save_path = os.path.join(save_root, f'data/{cnt}.bin')+ torch.save(data_list, save_path)++ data_list = []+ cnt += 1+++if __name__ == "__main__":+ main()