Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[feat] extract feature

zhengzangw5aad90acaf84
diff --git a/configs/opensora-v1-2/misc/extract.py b/configs/opensora-v1-2/misc/extract.py
index fa1c436..368838d 100644
--- a/configs/opensora-v1-2/misc/extract.py
+++ b/configs/opensora-v1-2/misc/extract.py
@@ -36,7 +36,7 @@ wandb = False
# Model settings
model = dict(
type="STDiT3-XL/2",
- from_pretrained=None,
+ from_pretrained="/mnt/nfs-206/zangwei/opensora/outputs/1091-STDiT3-XL-2/epoch0-global_step8500",
qk_norm=True,
enable_flash_attn=True,
enable_layernorm_kernel=True,
@@ -45,7 +45,7 @@ vae = dict(
type="OpenSoraVAE_V1_2",
from_pretrained="pretrained_models/vae-pipeline",
micro_frame_size=17,
- micro_batch_size=4,
+ micro_batch_size=32,
)
text_encoder = dict(
type="t5",
@@ -58,3 +58,6 @@ text_encoder = dict(
# feature extraction settings
save_text_features = True
save_compressed_text_features = True
+bin_size = 10
+save_dir = f"/mnt/nfs-207/sora_data/feat/test_b{bin_size}"
+log_time = False
diff --git a/opensora/datasets/sampler.py b/opensora/datasets/sampler.py
index 7fd4d77..bee2a6c 100644
--- a/opensora/datasets/sampler.py
+++ b/opensora/datasets/sampler.py
@@ -192,7 +192,7 @@ class VariableVideoBatchSampler(DistributedSampler):
bucket_access_list = bucket_id_access_order[i * self.num_replicas : (i + 1) * self.num_replicas]
self.last_micro_batch_access_index += self.num_replicas
- # comppute the data samples consumed by each access
+ # compute the data samples consumed by each access
bucket_access_boundaries = []
for bucket_id in bucket_access_list:
bucket_bs = self.bucket.get_batch_size(bucket_id)
diff --git a/opensora/utils/misc.py b/opensora/utils/misc.py
index 4be047c..61e00f4 100644
--- a/opensora/utils/misc.py
+++ b/opensora/utils/misc.py
@@ -388,9 +388,10 @@ def get_tensor_memory(tensor, human_readable=True):
class FeatureSaver:
- def __init__(self, save_dir, bin_size=10):
+ def __init__(self, save_dir, bin_size=10, start_bin=0):
self.save_dir = save_dir
self.bin_size = bin_size
+ self.bin_cnt = start_bin
self.data_list = []
self.cnt = 0
@@ -398,3 +399,13 @@ class FeatureSaver:
def update(self, data):
self.data_list.append(data)
self.cnt += 1
+
+ if self.cnt % self.bin_size == 0:
+ self.save()
+
+ def save(self):
+ save_path = os.path.join(self.save_dir, f"{self.bin_cnt:08}.bin")
+ torch.save(self.data_list, save_path)
+ get_logger().info("Saved to %s", save_path)
+ self.data_list = []
+ self.bin_cnt += 1
diff --git a/scripts/misc/extract_feat.py b/scripts/misc/extract_feat.py
index 476e203..ac2bef9 100644
--- a/scripts/misc/extract_feat.py
+++ b/scripts/misc/extract_feat.py
@@ -1,18 +1,16 @@
import os
from pprint import pformat
+import colossalai
import torch
-import torch.distributed as dist
from tqdm import tqdm
from opensora.acceleration.parallel_states import get_data_parallel_group
-from opensora.datasets import prepare_dataloader, prepare_variable_dataloader
+from opensora.datasets import prepare_variable_dataloader
from opensora.datasets.utils import collate_fn_ignore_none
from opensora.registry import DATASETS, MODELS, build_module
-from opensora.utils.config_utils import parse_configs
-from opensora.utils.misc import FeatureSaver, create_logger, format_numel_str, get_model_numel, to_torch_dtype
-
-DEFAULT_DATASET_NAME = "VideoTextDataset"
+from opensora.utils.config_utils import parse_configs, save_training_config
+from opensora.utils.misc import FeatureSaver, Timer, create_logger, format_numel_str, get_model_numel, to_torch_dtype
def main():
@@ -37,6 +35,8 @@ def main():
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.allow_tf32 = True
+ colossalai.launch_from_torch({})
+
# == init logger, tensorboard & wandb ==
logger = create_logger()
logger.info("Configuration:\n %s", pformat(cfg.to_dict()))
@@ -61,18 +61,12 @@ def main():
process_group=get_data_parallel_group(),
collate_fn=collate_fn_ignore_none,
)
- if cfg.dataset.type == DEFAULT_DATASET_NAME:
- dataloader = prepare_dataloader(**dataloader_args)
- total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.get("sp_size", 1)
- logger.info("Total batch size: %s", total_batch_size)
- num_steps_per_epoch = len(dataloader)
- else:
- dataloader = prepare_variable_dataloader(
- bucket_config=cfg.get("bucket_config", None),
- num_bucket_build_workers=cfg.get("num_bucket_build_workers", 1),
- **dataloader_args,
- )
- num_steps_per_epoch = dataloader.batch_sampler.get_num_batch()
+ dataloader = prepare_variable_dataloader(
+ bucket_config=cfg.get("bucket_config", None),
+ num_bucket_build_workers=cfg.get("num_bucket_build_workers", 1),
+ **dataloader_args,
+ )
+ num_batch = dataloader.batch_sampler.get_num_batch()
# ======================================================
# 3. build model
@@ -104,63 +98,69 @@ def main():
format_numel_str(model_numel),
)
- # =======================================================
- # 4. distributed training preparation with colossalai
- # =======================================================
- # == global variables ==
- start_step = sampler_start_idx = 0
- logger.info("Training for with %s steps per epoch", num_steps_per_epoch)
-
- if cfg.dataset.type == DEFAULT_DATASET_NAME:
- dataloader.sampler.set_start_index(sampler_start_idx)
-
# =======================================================
# 5. training loop
# =======================================================
- dist.barrier()
- for epoch in range(1):
- # == set dataloader to new epoch ==
- if cfg.dataset.type == DEFAULT_DATASET_NAME:
- dataloader.sampler.set_epoch(epoch)
- dataloader_iter = iter(dataloader)
- logger.info("Beginning epoch %s...", epoch)
-
- # == training loop in an epoch ==
- assert cfg.get("save_dir", None) is not None, "Please specify the save_dir in the config file."
- os.makedirs(cfg.save_dir, exist_ok=True)
- saver = FeatureSaver(cfg.save_dir)
- save_text_features = cfg.get("save_text_features", False)
- save_compressed_text_features = cfg.get("save_compressed_text_features", False)
-
- with tqdm(
- enumerate(dataloader_iter, start=start_step),
- desc=f"Epoch {epoch}",
- initial=start_step,
- total=num_steps_per_epoch,
- ) as pbar:
- for step, batch in pbar:
+ # == global variables ==
+ bin_size = cfg.bin_size
+ save_text_features = cfg.get("save_text_features", False)
+ save_compressed_text_features = cfg.get("save_compressed_text_features", False)
+
+ # == number of bins ==
+ num_bin = num_batch // bin_size
+ logger.info("Number of batches: %s", num_batch)
+ logger.info("Bin size: %s", bin_size)
+ logger.info("Number of bins: %s", num_bin)
+
+ # resume from a specific batch index
+ start_index = cfg.get("start_index", 0)
+ end_index = cfg.get("end_index", num_bin)
+ dataloader.batch_sampler.load_state_dict({"last_micro_batch_access_index": start_index})
+ num_bin_to_process = min(num_bin, end_index) - start_index
+ logger.info("Start index: %s", start_index)
+ logger.info("End index: %s", end_index)
+ logger.info("Number of batches to process: %s", num_bin_to_process)
+
+ # create save directory
+ assert cfg.get("save_dir", None) is not None, "Please specify the save_dir in the config file."
+ save_dir = os.path.join(cfg.save_dir, f"s{start_index}_e{end_index}")
+ os.makedirs(save_dir, exist_ok=True)
+ save_training_config(cfg.to_dict(), save_dir)
+ logger.info("Saving features to %s", save_dir)
+
+ saver = FeatureSaver(save_dir, bin_size, start_bin=start_index)
+
+ # == training loop in an epoch ==
+ dataloader_iter = iter(dataloader)
+ log_time = cfg.get("log_time", False)
+ for i in tqdm(range(0, num_bin_to_process)):
+ with Timer("step", log=log_time):
+ with Timer("data loading", log=log_time):
+ batch = next(dataloader_iter)
x = batch.pop("video").to(device, dtype) # [B, C, T, H, W]
y = batch.pop("text")
- x = vae.encode(x).cpu() # [B, C, T, H/P, W/P]
- fps = batch["fps"].to(dtype)
- batch_dict = {"x": x, "fps": fps}
+ with Timer("vae", log=log_time):
+ x = vae.encode(x)
+ with Timer("feature to cpu", log=log_time):
+ x = x.cpu()
+ fps = batch["fps"].to(dtype)
+ batch_dict = {"x": x, "fps": fps}
- if save_text_features:
+ if save_text_features:
+ with Timer("text", log=log_time):
text_infos = text_encoder.encode(y)
y_feat = text_infos["y"]
y_mask = text_infos["mask"]
- if not save_compressed_text_features:
- y_feat = y_feat.cpu()
- y_mask = y_mask.cpu()
- else:
+ if save_compressed_text_features:
y_feat, y_mask = model.encode_text(y_feat, y_mask)
- y_feat = y_feat.cpu()
y_mask = torch.tensor(y_mask)
- breakpoint()
- batch_dict.update({"y": y_feat, "mask": y_mask})
+ with Timer("feature to cpu", log=log_time):
+ y_feat = y_feat.cpu()
+ y_mask = y_mask.cpu()
+ batch_dict.update({"y": y_feat, "mask": y_mask})
- saver.update(batch_dict)
+ saver.update(batch_dict)
if __name__ == "__main__":
diff --git a/scripts/misc/launch_extract_feat.sh b/scripts/misc/launch_extract_feat.sh
new file mode 100644
index 0000000..50f9212
--- /dev/null
+++ b/scripts/misc/launch_extract_feat.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -x
+set -e
+
+START_SPLIT=0
+NUM_SPLIT=100
+
+DATA_PATH=$1
+DATA_ARG="--data-path $DATA_PATH"
+
+CMD="torchrun --standalone --nproc_per_node 1 scripts/misc/extract_feat.py configs/opensora-v1-2/misc/extract.py $DATA_ARG"
+declare -a GPUS=(0 1 2 3 4 5 6 7)
+
+mkdir -p logs/extract_feat
+
+for i in "${GPUS[@]}"; do
+ CUDA_VISIBLE_DEVICES=$i $CMD --start-index $(($START_SPLIT + i * $NUM_SPLIT)) --end-index $(($START_SPLIT + (i + 1) * $NUM_SPLIT)) >logs/extract_feat/$i.log 2>&1 &
+done