Open-Sora: Democratizing Efficient Video Production for All
Python
add parallel pandas
Zangwei Zheng5e3eca2d0f02
diff --git a/tools/datasets/README.md b/tools/datasets/README.mdindex fadf484..1ec2965 100644--- a/tools/datasets/README.md+++ b/tools/datasets/README.md@@ -1,16 +1,18 @@# Dataset Download and Management-## HD-VG-130M+## Dataset Download++### HD-VG-130MThis dataset comprises 130M text-video pairs. You can download the dataset and prepare it for training according to [the dataset repository's instructions](https://github.com/daooshee/HD-VG-130M). There is a README.md file in the Google Drive link that provides instructions on how to download and cut the videos. For this version, we directly use the dataset provided by the authors.-## VidProM+### VidProM```bashpython -m tools.datasets.convert_dataset vidprom VIDPROM_FOLDER --info VidProM_semantic_unique.csv```-## Demo Dataset+### Demo DatasetYou can use ImageNet and UCF101 for a quick demo. After downloading the datasets, you can use the following command to prepare the csv file for the dataset:@@ -54,3 +56,9 @@ python -m tools.datasets.csvutil DATA.csv --remove-caption-prefix# generate DATA_root.csv with absolute pathpython -m tools.datasets.csvutil DATA.csv --abspath /absolute/path/to/dataset```++To accelerate processing speed, you can install [pandarallel](https://github.com/nalepae/pandarallel):++```bash+pip install pandarallel+```diff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.pyindex a9cdb76..2d4d997 100644--- a/tools/datasets/csvutil.py+++ b/tools/datasets/csvutil.py@@ -1,8 +1,6 @@import argparse-import csvimport htmlimport os-import reimport numpy as npimport pandas as pd@@ -10,6 +8,20 @@ from tqdm import tqdmtqdm.pandas()+try:+ from pandarallel import pandarallel++ pandarallel.initialize(progress_bar=True)+ pandas_has_parallel = True+except ImportError:+ pandas_has_parallel = False+++def apply(df, func):+ if pandas_has_parallel:+ return df.parallel_apply(func)+ return df.progress_apply(func)+def get_video_length(path):import cv2@@ -61,6 +73,7 @@ def parse_args():parser = argparse.ArgumentParser()parser.add_argument("input", type=str, nargs="+")parser.add_argument("--output", type=str, default=None)+ parser.add_argument("--disable-parallel", action="store_true")# special caseparser.add_argument("--shard", type=int, default=None)@@ -148,18 +161,18 @@ def main(args):# processingif args.abspath is not None:assert args.relpath is None- data["path"] = data["path"].progress_apply(lambda x: os.path.join(args.abspath, x))+ data["path"] = apply(data["path"], lambda x: os.path.join(args.abspath, x))if args.relpath is not None:assert args.abspath is None- data["path"] = data["path"].progress_apply(lambda x: os.path.relpath(x, args.relpath))+ data["path"] = apply(data["path"], lambda x: os.path.relpath(x, args.relpath))if args.remove_caption_prefix:assert "text" in data.columns- data["text"] = data["text"].progress_apply(remove_caption_prefix)+ data["text"] = apply(data["text"], remove_caption_prefix)if args.unescape:assert "text" in data.columns- data["text"] = data["text"].progress_apply(html.unescape)+ data["text"] = apply(data["text"], html.unescape)if args.relength:- data["num_frames"] = data["path"].progress_apply(get_video_length)+ data["num_frames"] = apply(data["path"], get_video_length)# filteringif args.remove_empty_caption:@@ -170,7 +183,7 @@ def main(args):data = data[~data["text"].str.contains(r"(?P<url>https?://[^\s]+)", regex=True)]if args.lang is not None:assert "text" in data.columns- data = data[data["text"].progress_apply(detect_lang)]+ data = data[data["text"].progress_apply(detect_lang)] # cannot parallelizeif args.fmin is not None:assert "num_frames" in data.columnsdata = data[data["num_frames"] >= args.fmin]@@ -196,4 +209,6 @@ def main(args):if __name__ == "__main__":args = parse_args()+ if args.disable_parallel:+ pandas_has_parallel = Falsemain(args)