Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

add parallel pandas

Zangwei Zheng5e3eca2d0f02
diff --git a/tools/datasets/README.md b/tools/datasets/README.md
index fadf484..1ec2965 100644
--- a/tools/datasets/README.md
+++ b/tools/datasets/README.md
@@ -1,16 +1,18 @@
# Dataset Download and Management
-## HD-VG-130M
+## Dataset Download
+
+### HD-VG-130M
This dataset comprises 130M text-video pairs. You can download the dataset and prepare it for training according to [the dataset repository's instructions](https://github.com/daooshee/HD-VG-130M). There is a README.md file in the Google Drive link that provides instructions on how to download and cut the videos. For this version, we directly use the dataset provided by the authors.
-## VidProM
+### VidProM
```bash
python -m tools.datasets.convert_dataset vidprom VIDPROM_FOLDER --info VidProM_semantic_unique.csv
```
-## Demo Dataset
+### Demo Dataset
You can use ImageNet and UCF101 for a quick demo. After downloading the datasets, you can use the following command to prepare the csv file for the dataset:
@@ -54,3 +56,9 @@ python -m tools.datasets.csvutil DATA.csv --remove-caption-prefix
# generate DATA_root.csv with absolute path
python -m tools.datasets.csvutil DATA.csv --abspath /absolute/path/to/dataset
```
+
+To accelerate processing speed, you can install [pandarallel](https://github.com/nalepae/pandarallel):
+
+```bash
+pip install pandarallel
+```
diff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.py
index a9cdb76..2d4d997 100644
--- a/tools/datasets/csvutil.py
+++ b/tools/datasets/csvutil.py
@@ -1,8 +1,6 @@
import argparse
-import csv
import html
import os
-import re
import numpy as np
import pandas as pd
@@ -10,6 +8,20 @@ from tqdm import tqdm
tqdm.pandas()
+try:
+ from pandarallel import pandarallel
+
+ pandarallel.initialize(progress_bar=True)
+ pandas_has_parallel = True
+except ImportError:
+ pandas_has_parallel = False
+
+
+def apply(df, func):
+ if pandas_has_parallel:
+ return df.parallel_apply(func)
+ return df.progress_apply(func)
+
def get_video_length(path):
import cv2
@@ -61,6 +73,7 @@ def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("input", type=str, nargs="+")
parser.add_argument("--output", type=str, default=None)
+ parser.add_argument("--disable-parallel", action="store_true")
# special case
parser.add_argument("--shard", type=int, default=None)
@@ -148,18 +161,18 @@ def main(args):
# processing
if args.abspath is not None:
assert args.relpath is None
- data["path"] = data["path"].progress_apply(lambda x: os.path.join(args.abspath, x))
+ data["path"] = apply(data["path"], lambda x: os.path.join(args.abspath, x))
if args.relpath is not None:
assert args.abspath is None
- data["path"] = data["path"].progress_apply(lambda x: os.path.relpath(x, args.relpath))
+ data["path"] = apply(data["path"], lambda x: os.path.relpath(x, args.relpath))
if args.remove_caption_prefix:
assert "text" in data.columns
- data["text"] = data["text"].progress_apply(remove_caption_prefix)
+ data["text"] = apply(data["text"], remove_caption_prefix)
if args.unescape:
assert "text" in data.columns
- data["text"] = data["text"].progress_apply(html.unescape)
+ data["text"] = apply(data["text"], html.unescape)
if args.relength:
- data["num_frames"] = data["path"].progress_apply(get_video_length)
+ data["num_frames"] = apply(data["path"], get_video_length)
# filtering
if args.remove_empty_caption:
@@ -170,7 +183,7 @@ def main(args):
data = data[~data["text"].str.contains(r"(?P<url>https?://[^\s]+)", regex=True)]
if args.lang is not None:
assert "text" in data.columns
- data = data[data["text"].progress_apply(detect_lang)]
+ data = data[data["text"].progress_apply(detect_lang)] # cannot parallelize
if args.fmin is not None:
assert "num_frames" in data.columns
data = data[data["num_frames"] >= args.fmin]
@@ -196,4 +209,6 @@ def main(args):
if __name__ == "__main__":
args = parse_args()
+ if args.disable_parallel:
+ pandas_has_parallel = False
main(args)