Open-Sora: Democratizing Efficient Video Production for All
Python
[feat] extract feature
zhengzangw5aad90acaf84
diff --git a/configs/opensora-v1-2/misc/extract.py b/configs/opensora-v1-2/misc/extract.pyindex fa1c436..368838d 100644--- a/configs/opensora-v1-2/misc/extract.py+++ b/configs/opensora-v1-2/misc/extract.py@@ -36,7 +36,7 @@ wandb = False# Model settingsmodel = dict(type="STDiT3-XL/2",- from_pretrained=None,+ from_pretrained="/mnt/nfs-206/zangwei/opensora/outputs/1091-STDiT3-XL-2/epoch0-global_step8500",qk_norm=True,enable_flash_attn=True,enable_layernorm_kernel=True,@@ -45,7 +45,7 @@ vae = dict(type="OpenSoraVAE_V1_2",from_pretrained="pretrained_models/vae-pipeline",micro_frame_size=17,- micro_batch_size=4,+ micro_batch_size=32,)text_encoder = dict(type="t5",@@ -58,3 +58,6 @@ text_encoder = dict(# feature extraction settingssave_text_features = Truesave_compressed_text_features = True+bin_size = 10+save_dir = f"/mnt/nfs-207/sora_data/feat/test_b{bin_size}"+log_time = Falsediff --git a/opensora/datasets/sampler.py b/opensora/datasets/sampler.pyindex 7fd4d77..bee2a6c 100644--- a/opensora/datasets/sampler.py+++ b/opensora/datasets/sampler.py@@ -192,7 +192,7 @@ class VariableVideoBatchSampler(DistributedSampler):bucket_access_list = bucket_id_access_order[i * self.num_replicas : (i + 1) * self.num_replicas]self.last_micro_batch_access_index += self.num_replicas- # comppute the data samples consumed by each access+ # compute the data samples consumed by each accessbucket_access_boundaries = []for bucket_id in bucket_access_list:bucket_bs = self.bucket.get_batch_size(bucket_id)diff --git a/opensora/utils/misc.py b/opensora/utils/misc.pyindex 4be047c..61e00f4 100644--- a/opensora/utils/misc.py+++ b/opensora/utils/misc.py@@ -388,9 +388,10 @@ def get_tensor_memory(tensor, human_readable=True):class FeatureSaver:- def __init__(self, save_dir, bin_size=10):+ def __init__(self, save_dir, bin_size=10, start_bin=0):self.save_dir = save_dirself.bin_size = bin_size+ self.bin_cnt = start_binself.data_list = []self.cnt = 0@@ -398,3 +399,13 @@ class FeatureSaver:def update(self, data):self.data_list.append(data)self.cnt += 1++ if self.cnt % self.bin_size == 0:+ self.save()++ def save(self):+ save_path = os.path.join(self.save_dir, f"{self.bin_cnt:08}.bin")+ torch.save(self.data_list, save_path)+ get_logger().info("Saved to %s", save_path)+ self.data_list = []+ self.bin_cnt += 1diff --git a/scripts/misc/extract_feat.py b/scripts/misc/extract_feat.pyindex 476e203..ac2bef9 100644--- a/scripts/misc/extract_feat.py+++ b/scripts/misc/extract_feat.py@@ -1,18 +1,16 @@import osfrom pprint import pformat+import colossalaiimport torch-import torch.distributed as distfrom tqdm import tqdmfrom opensora.acceleration.parallel_states import get_data_parallel_group-from opensora.datasets import prepare_dataloader, prepare_variable_dataloader+from opensora.datasets import prepare_variable_dataloaderfrom opensora.datasets.utils import collate_fn_ignore_nonefrom opensora.registry import DATASETS, MODELS, build_module-from opensora.utils.config_utils import parse_configs-from opensora.utils.misc import FeatureSaver, create_logger, format_numel_str, get_model_numel, to_torch_dtype--DEFAULT_DATASET_NAME = "VideoTextDataset"+from opensora.utils.config_utils import parse_configs, save_training_config+from opensora.utils.misc import FeatureSaver, Timer, create_logger, format_numel_str, get_model_numel, to_torch_dtypedef main():@@ -37,6 +35,8 @@ def main():torch.backends.cuda.matmul.allow_tf32 = Truetorch.backends.cudnn.allow_tf32 = True+ colossalai.launch_from_torch({})+# == init logger, tensorboard & wandb ==logger = create_logger()logger.info("Configuration:\n %s", pformat(cfg.to_dict()))@@ -61,18 +61,12 @@ def main():process_group=get_data_parallel_group(),collate_fn=collate_fn_ignore_none,)- if cfg.dataset.type == DEFAULT_DATASET_NAME:- dataloader = prepare_dataloader(**dataloader_args)- total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.get("sp_size", 1)- logger.info("Total batch size: %s", total_batch_size)- num_steps_per_epoch = len(dataloader)- else:- dataloader = prepare_variable_dataloader(- bucket_config=cfg.get("bucket_config", None),- num_bucket_build_workers=cfg.get("num_bucket_build_workers", 1),- **dataloader_args,- )- num_steps_per_epoch = dataloader.batch_sampler.get_num_batch()+ dataloader = prepare_variable_dataloader(+ bucket_config=cfg.get("bucket_config", None),+ num_bucket_build_workers=cfg.get("num_bucket_build_workers", 1),+ **dataloader_args,+ )+ num_batch = dataloader.batch_sampler.get_num_batch()# ======================================================# 3. build model@@ -104,63 +98,69 @@ def main():format_numel_str(model_numel),)- # =======================================================- # 4. distributed training preparation with colossalai- # =======================================================- # == global variables ==- start_step = sampler_start_idx = 0- logger.info("Training for with %s steps per epoch", num_steps_per_epoch)-- if cfg.dataset.type == DEFAULT_DATASET_NAME:- dataloader.sampler.set_start_index(sampler_start_idx)-# =======================================================# 5. training loop# =======================================================- dist.barrier()- for epoch in range(1):- # == set dataloader to new epoch ==- if cfg.dataset.type == DEFAULT_DATASET_NAME:- dataloader.sampler.set_epoch(epoch)- dataloader_iter = iter(dataloader)- logger.info("Beginning epoch %s...", epoch)-- # == training loop in an epoch ==- assert cfg.get("save_dir", None) is not None, "Please specify the save_dir in the config file."- os.makedirs(cfg.save_dir, exist_ok=True)- saver = FeatureSaver(cfg.save_dir)- save_text_features = cfg.get("save_text_features", False)- save_compressed_text_features = cfg.get("save_compressed_text_features", False)-- with tqdm(- enumerate(dataloader_iter, start=start_step),- desc=f"Epoch {epoch}",- initial=start_step,- total=num_steps_per_epoch,- ) as pbar:- for step, batch in pbar:+ # == global variables ==+ bin_size = cfg.bin_size+ save_text_features = cfg.get("save_text_features", False)+ save_compressed_text_features = cfg.get("save_compressed_text_features", False)++ # == number of bins ==+ num_bin = num_batch // bin_size+ logger.info("Number of batches: %s", num_batch)+ logger.info("Bin size: %s", bin_size)+ logger.info("Number of bins: %s", num_bin)++ # resume from a specific batch index+ start_index = cfg.get("start_index", 0)+ end_index = cfg.get("end_index", num_bin)+ dataloader.batch_sampler.load_state_dict({"last_micro_batch_access_index": start_index})+ num_bin_to_process = min(num_bin, end_index) - start_index+ logger.info("Start index: %s", start_index)+ logger.info("End index: %s", end_index)+ logger.info("Number of batches to process: %s", num_bin_to_process)++ # create save directory+ assert cfg.get("save_dir", None) is not None, "Please specify the save_dir in the config file."+ save_dir = os.path.join(cfg.save_dir, f"s{start_index}_e{end_index}")+ os.makedirs(save_dir, exist_ok=True)+ save_training_config(cfg.to_dict(), save_dir)+ logger.info("Saving features to %s", save_dir)++ saver = FeatureSaver(save_dir, bin_size, start_bin=start_index)++ # == training loop in an epoch ==+ dataloader_iter = iter(dataloader)+ log_time = cfg.get("log_time", False)+ for i in tqdm(range(0, num_bin_to_process)):+ with Timer("step", log=log_time):+ with Timer("data loading", log=log_time):+ batch = next(dataloader_iter)x = batch.pop("video").to(device, dtype) # [B, C, T, H, W]y = batch.pop("text")- x = vae.encode(x).cpu() # [B, C, T, H/P, W/P]- fps = batch["fps"].to(dtype)- batch_dict = {"x": x, "fps": fps}+ with Timer("vae", log=log_time):+ x = vae.encode(x)+ with Timer("feature to cpu", log=log_time):+ x = x.cpu()+ fps = batch["fps"].to(dtype)+ batch_dict = {"x": x, "fps": fps}- if save_text_features:+ if save_text_features:+ with Timer("text", log=log_time):text_infos = text_encoder.encode(y)y_feat = text_infos["y"]y_mask = text_infos["mask"]- if not save_compressed_text_features:- y_feat = y_feat.cpu()- y_mask = y_mask.cpu()- else:+ if save_compressed_text_features:y_feat, y_mask = model.encode_text(y_feat, y_mask)- y_feat = y_feat.cpu()y_mask = torch.tensor(y_mask)- breakpoint()- batch_dict.update({"y": y_feat, "mask": y_mask})+ with Timer("feature to cpu", log=log_time):+ y_feat = y_feat.cpu()+ y_mask = y_mask.cpu()+ batch_dict.update({"y": y_feat, "mask": y_mask})- saver.update(batch_dict)+ saver.update(batch_dict)if __name__ == "__main__":diff --git a/scripts/misc/launch_extract_feat.sh b/scripts/misc/launch_extract_feat.shnew file mode 100644index 0000000..50f9212--- /dev/null+++ b/scripts/misc/launch_extract_feat.sh@@ -0,0 +1,19 @@+#!/bin/bash++set -x+set -e++START_SPLIT=0+NUM_SPLIT=100++DATA_PATH=$1+DATA_ARG="--data-path $DATA_PATH"++CMD="torchrun --standalone --nproc_per_node 1 scripts/misc/extract_feat.py configs/opensora-v1-2/misc/extract.py $DATA_ARG"+declare -a GPUS=(0 1 2 3 4 5 6 7)++mkdir -p logs/extract_feat++for i in "${GPUS[@]}"; do+ CUDA_VISIBLE_DEVICES=$i $CMD --start-index $(($START_SPLIT + i * $NUM_SPLIT)) --end-index $(($START_SPLIT + (i + 1) * $NUM_SPLIT)) >logs/extract_feat/$i.log 2>&1 &+done