Open-Sora: Democratizing Efficient Video Production for All
Python
[feat] support parquet
Zangwei Zhengfff002bede02
diff --git a/opensora/datasets/datasets.py b/opensora/datasets/datasets.pyindex ff7dff5..e0ee243 100644--- a/opensora/datasets/datasets.py+++ b/opensora/datasets/datasets.py@@ -1,14 +1,13 @@import osimport numpy as np-import pandas as pdimport torchimport torchvisionfrom torchvision.datasets.folder import IMG_EXTENSIONS, pil_loaderfrom opensora.registry import DATASETS-from .utils import VID_EXTENSIONS, get_transforms_image, get_transforms_video, temporal_random_crop+from .utils import VID_EXTENSIONS, get_transforms_image, get_transforms_video, read_file, temporal_random_crop@DATASETS.register_module()@@ -30,7 +29,7 @@ class VideoTextDataset(torch.utils.data.Dataset):transform_name="center",):self.data_path = data_path- self.data = pd.read_csv(data_path)+ self.data = read_file(data_path)self.num_frames = num_framesself.frame_interval = frame_intervalself.image_size = image_sizediff --git a/opensora/datasets/utils.py b/opensora/datasets/utils.pyindex 1b36d5d..ba205cb 100644--- a/opensora/datasets/utils.py+++ b/opensora/datasets/utils.py@@ -1,6 +1,7 @@import osimport numpy as np+import pandas as pdimport torchimport torchvisionimport torchvision.transforms as transforms@@ -14,6 +15,15 @@ from . import video_transformsVID_EXTENSIONS = (".mp4", ".avi", ".mov", ".mkv")+def read_file(input_path):+ if input_path.endswith(".csv"):+ return pd.read_csv(input_path)+ elif input_path.endswith(".parquet"):+ return pd.read_parquet(input_path)+ else:+ raise NotImplementedError(f"Unsupported file format: {input_path}")++def temporal_random_crop(vframes, num_frames, frame_interval):temporal_sample = video_transforms.TemporalRandomCrop(num_frames * frame_interval)total_frames = len(vframes)diff --git a/tools/datasets/README.md b/tools/datasets/README.mdindex fc685ba..d5d6bbb 100644--- a/tools/datasets/README.md+++ b/tools/datasets/README.md@@ -13,7 +13,6 @@- [Frame extraction](#frame-extraction)- [Crop Midjourney 4 grid](#crop-midjourney-4-grid)- [Analyze datasets](#analyze-datasets)- - [Frame extraction speed](#frame-extraction-speed)After preparing the raw dataset according to the [instructions](/docs/datasets.md), you can use the following commands to manage the dataset.@@ -44,11 +43,14 @@ path, text, num_frames, width, height, aspect_ratio/absolute/path/to/video2.mp4, caption, 20, 256, 256, 1```-We use pandas to manage the CSV files. The following code is for reading and writing the CSV files:+We use pandas to manage the `.csv` or `.parquet` files. The following code is for reading and writing files:```pythondf = pd.read_csv(input_path)df = df.to_csv(output_path, index=False)+# or use parquet, which is smaller+df = pd.read_parquet(input_path)+df = df.to_parquet(output_path, index=False)```## Dataset to CSV@@ -72,16 +74,7 @@ python -m tools.datasets.convert vidprom VIDPROM_FOLDER --info VidProM_semantic_## Manage datasets-You can easily get basic information about the dataset by using the following commands:--```bash-# examine the first 10 rows of the CSV file-head -n 10 DATA1.csv-# count the number of data in the CSV file (approximately)-wc -l DATA1.csv-```--Additionally, Ww provide `csvutil.py` to manage the CSV files.+Use `csvutil` to manage the dataset.### Requirement@@ -105,7 +98,7 @@ pip install lingua-language-detector### Basic Usage-You can use the following commands to process the CSV files. The output csv file will be saved in the same directory as the input csv file, with different suffixes indicating the processed method.+You can use the following commands to process the `csv` or `parquet` files. The output file will be saved in the same directory as the input, with different suffixes indicating the processed method.```bash# csvutil takes multiple CSV files as input and merge them into one CSV file@@ -169,7 +162,7 @@ You can also use `python -m tools.datasets.csvutil --help` to see usage.| `--format FORMAT` | | Output format (csv, parquet, parquet.gzip) || `--disable-parallel` | | Disable `pandarallel` || `--seed SEED` | | Random seed |-| `--shard SHARD` | `_0`,`_1` | Shard the dataset |+| `--shard SHARD` | `_0`,`_1`, ... | Shard the dataset || `--sort KEY` | `_sort` | Sort the dataset by KEY || `--sort-descending KEY` | `_sort` | Sort the dataset by KEY in descending order || `--difference DATA.csv` | | Remove the paths in DATA.csv from the dataset |@@ -189,6 +182,7 @@ You can also use `python -m tools.datasets.csvutil --help` to see usage.| `--unescape` | `_unescape` | Unescape the caption || `--merge-cmotion` | `_cmotion` | Merge the camera motion to the caption || `--count-num-token` | `_ntoken` | Count the number of tokens in the caption |+| `--load-caption EXT` | `_load` | Load the caption from the file || `--fmin FMIN` | `_fmin` | Filter the dataset by minimum number of frames || `--fmax FMAX` | `_fmax` | Filter the dataset by maximum number of frames || `--hwmax HWMAX` | `_hwmax` | Filter the dataset by maximum height x width |@@ -230,25 +224,17 @@ python -m tools.datasets.transform img_rand_crop meta.csv /path/to/raw/data /pat## Analyze datasets-Since the dataset is provided in a CSV file, you can easily analyze the dataset using pandas (after applying `--info`). Here are some examples:--```python-import pandas as pd-import matplotlib.pyplot as plt+You can easily get basic information about a `.csv` dataset by using the following commands:-data = pd.read_csv('meta.csv')-data.hist(column="resolution")-plt.savefig('outputs/samples/info.jpg')+```bash+# examine the first 10 rows of the CSV file+head -n 10 DATA1.csv+# count the number of data in the CSV file (approximately)+wc -l DATA1.csv```-## Frame extraction speed+For the dataset provided in a `.csv` or `.parquet` file, you can easily analyze the dataset using the following commands. Plots will be automatically saved.-We use three libraries to extract frames from videos: `opencv`, `pyav` and `decord`. Our benchmark results of loading 256 video's middle frames are as follows:--| Library | Time (s) |-| ------- | -------- |-| opencv | 33 |-| decord | 28 |-| pyav | 10 |--Although `pyav` is the fastest, it can only extract the key frames instead of frames at any time. Therefore, we use `decord` as the default library for frame extraction. For dataset management, without a bottleneck on loading speed, we choose `opencv` as the default library for video information extraction.+```python+pyhton -m tools.datasets.analyze DATA_info.csv+```diff --git a/tools/datasets/analyze.py b/tools/datasets/analyze.pyindex 134db45..701d716 100644--- a/tools/datasets/analyze.py+++ b/tools/datasets/analyze.py@@ -1,8 +1,87 @@+import argparse+import os++import matplotlib.pyplot as plt+import pandas as pd+++def read_file(input_path):+ if input_path.endswith(".csv"):+ return pd.read_csv(input_path)+ elif input_path.endswith(".parquet"):+ return pd.read_parquet(input_path)+ else:+ raise NotImplementedError(f"Unsupported file format: {input_path}")+++def parse_args():+ parser = argparse.ArgumentParser()+ parser.add_argument("input", type=str, help="Path to the input dataset")+ parser.add_argument("--save-img", type=str, default="outputs/samples/infos/", help="Path to save the image")+ return parser.parse_args()+++def plot_data(data, column, bins, name):+ plt.clf()+ data.hist(column=column, bins=bins)+ os.makedirs(os.path.dirname(name), exist_ok=True)+ plt.savefig(name)+ print(f"Saved {name}")+++def plot_categorical_data(data, column, name):+ plt.clf()+ data[column].value_counts().plot(kind="bar")+ os.makedirs(os.path.dirname(name), exist_ok=True)+ plt.savefig(name)+ print(f"Saved {name}")+++COLUMNS = {+ "num_frames": 100,+ "resolution": 100,+ "text_len": 100,+ "aes": 100,+ "match": 100,+ "flow": 100,+ "cmotion": None,+}+++def main(args):+ data = read_file(args.input)++ image_index = data["num_frames"] == 1+ if image_index.sum() > 0:+ print("=== Image Data Info ===")+ img_data = data[image_index]+ print(f"Number of images: {len(img_data)}")+ print(img_data.head())+ print(img_data.describe())+ if args.save_img:+ for column in COLUMNS:+ if column in img_data.columns and column not in ["num_frames", "cmotion"]:+ if COLUMNS[column] is None:+ plot_categorical_data(img_data, column, os.path.join(args.save_img, f"image_{column}.png"))+ else:+ plot_data(img_data, column, COLUMNS[column], os.path.join(args.save_img, f"image_{column}.png"))+ if not image_index.all():+ print("=== Video Data Info ===")+ video_data = data[~image_index]+ print(f"Number of videos: {len(video_data)}")+ print(video_data.head())+ print(video_data.describe())+ if args.save_img:+ for column in COLUMNS:+ if column in video_data.columns:+ if COLUMNS[column] is None:+ plot_categorical_data(video_data, column, os.path.join(args.save_img, f"video_{column}.png"))+ else:+ plot_data(+ video_data, column, COLUMNS[column], os.path.join(args.save_img, f"video_{column}.png")+ )++if __name__ == "__main__":args = parse_args()- if args.disable_parallel:- pandas_has_parallel = False- if args.seed is not None:- random.seed(args.seed)- np.random.seed(args.seed)main(args)diff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.pyindex 9b21dac..addf2a3 100644--- a/tools/datasets/csvutil.py+++ b/tools/datasets/csvutil.py@@ -46,7 +46,7 @@ def apply(df, func, **kwargs):# ======================================================-def get_info(path):+def get_info_cv2(path):import cv2try:@@ -72,6 +72,13 @@ def get_info(path):return 0, 0, 0, np.nan, np.nan, np.nan+def get_info_ffmpeg(path):+ import ffmpeg++ ffmpeg.probe(path)+ breakpoint()++# ======================================================# --remove-corrupted# ======================================================@@ -467,7 +474,7 @@ def main(args):assert "text" in data.columnsdata["text_len"] = apply(data["text"], lambda x: len(tokenizer(x)["input_ids"]))if args.info:- info = apply(data["path"], get_info)+ info = apply(data["path"], get_info_ffmpeg)(data["num_frames"],data["height"],@@ -528,7 +535,7 @@ def main(args):def parse_args():parser = argparse.ArgumentParser()- parser.add_argument("input", type=str, nargs="+")+ parser.add_argument("input", type=str, nargs="+", help="path to the input dataset")parser.add_argument("--output", type=str, default=None, help="output path")parser.add_argument("--format", type=str, default="csv", help="output format", choices=["csv", "parquet"])parser.add_argument("--disable-parallel", action="store_true", help="disable parallel processing")