Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[feat] support parquet

Zangwei Zhengfff002bede02
diff --git a/opensora/datasets/datasets.py b/opensora/datasets/datasets.py
index ff7dff5..e0ee243 100644
--- a/opensora/datasets/datasets.py
+++ b/opensora/datasets/datasets.py
@@ -1,14 +1,13 @@
import os
import numpy as np
-import pandas as pd
import torch
import torchvision
from torchvision.datasets.folder import IMG_EXTENSIONS, pil_loader
from opensora.registry import DATASETS
-from .utils import VID_EXTENSIONS, get_transforms_image, get_transforms_video, temporal_random_crop
+from .utils import VID_EXTENSIONS, get_transforms_image, get_transforms_video, read_file, temporal_random_crop
@DATASETS.register_module()
@@ -30,7 +29,7 @@ class VideoTextDataset(torch.utils.data.Dataset):
transform_name="center",
):
self.data_path = data_path
- self.data = pd.read_csv(data_path)
+ self.data = read_file(data_path)
self.num_frames = num_frames
self.frame_interval = frame_interval
self.image_size = image_size
diff --git a/opensora/datasets/utils.py b/opensora/datasets/utils.py
index 1b36d5d..ba205cb 100644
--- a/opensora/datasets/utils.py
+++ b/opensora/datasets/utils.py
@@ -1,6 +1,7 @@
import os
import numpy as np
+import pandas as pd
import torch
import torchvision
import torchvision.transforms as transforms
@@ -14,6 +15,15 @@ from . import video_transforms
VID_EXTENSIONS = (".mp4", ".avi", ".mov", ".mkv")
+def read_file(input_path):
+ if input_path.endswith(".csv"):
+ return pd.read_csv(input_path)
+ elif input_path.endswith(".parquet"):
+ return pd.read_parquet(input_path)
+ else:
+ raise NotImplementedError(f"Unsupported file format: {input_path}")
+
+
def temporal_random_crop(vframes, num_frames, frame_interval):
temporal_sample = video_transforms.TemporalRandomCrop(num_frames * frame_interval)
total_frames = len(vframes)
diff --git a/tools/datasets/README.md b/tools/datasets/README.md
index fc685ba..d5d6bbb 100644
--- a/tools/datasets/README.md
+++ b/tools/datasets/README.md
@@ -13,7 +13,6 @@
- [Frame extraction](#frame-extraction)
- [Crop Midjourney 4 grid](#crop-midjourney-4-grid)
- [Analyze datasets](#analyze-datasets)
- - [Frame extraction speed](#frame-extraction-speed)
After preparing the raw dataset according to the [instructions](/docs/datasets.md), you can use the following commands to manage the dataset.
@@ -44,11 +43,14 @@ path, text, num_frames, width, height, aspect_ratio
/absolute/path/to/video2.mp4, caption, 20, 256, 256, 1
```
-We use pandas to manage the CSV files. The following code is for reading and writing the CSV files:
+We use pandas to manage the `.csv` or `.parquet` files. The following code is for reading and writing files:
```python
df = pd.read_csv(input_path)
df = df.to_csv(output_path, index=False)
+# or use parquet, which is smaller
+df = pd.read_parquet(input_path)
+df = df.to_parquet(output_path, index=False)
```
## Dataset to CSV
@@ -72,16 +74,7 @@ python -m tools.datasets.convert vidprom VIDPROM_FOLDER --info VidProM_semantic_
## Manage datasets
-You can easily get basic information about the dataset by using the following commands:
-
-```bash
-# examine the first 10 rows of the CSV file
-head -n 10 DATA1.csv
-# count the number of data in the CSV file (approximately)
-wc -l DATA1.csv
-```
-
-Additionally, Ww provide `csvutil.py` to manage the CSV files.
+Use `csvutil` to manage the dataset.
### Requirement
@@ -105,7 +98,7 @@ pip install lingua-language-detector
### Basic Usage
-You can use the following commands to process the CSV files. The output csv file will be saved in the same directory as the input csv file, with different suffixes indicating the processed method.
+You can use the following commands to process the `csv` or `parquet` files. The output file will be saved in the same directory as the input, with different suffixes indicating the processed method.
```bash
# csvutil takes multiple CSV files as input and merge them into one CSV file
@@ -169,7 +162,7 @@ You can also use `python -m tools.datasets.csvutil --help` to see usage.
| `--format FORMAT` | | Output format (csv, parquet, parquet.gzip) |
| `--disable-parallel` | | Disable `pandarallel` |
| `--seed SEED` | | Random seed |
-| `--shard SHARD` | `_0`,`_1` | Shard the dataset |
+| `--shard SHARD` | `_0`,`_1`, ... | Shard the dataset |
| `--sort KEY` | `_sort` | Sort the dataset by KEY |
| `--sort-descending KEY` | `_sort` | Sort the dataset by KEY in descending order |
| `--difference DATA.csv` | | Remove the paths in DATA.csv from the dataset |
@@ -189,6 +182,7 @@ You can also use `python -m tools.datasets.csvutil --help` to see usage.
| `--unescape` | `_unescape` | Unescape the caption |
| `--merge-cmotion` | `_cmotion` | Merge the camera motion to the caption |
| `--count-num-token` | `_ntoken` | Count the number of tokens in the caption |
+| `--load-caption EXT` | `_load` | Load the caption from the file |
| `--fmin FMIN` | `_fmin` | Filter the dataset by minimum number of frames |
| `--fmax FMAX` | `_fmax` | Filter the dataset by maximum number of frames |
| `--hwmax HWMAX` | `_hwmax` | Filter the dataset by maximum height x width |
@@ -230,25 +224,17 @@ python -m tools.datasets.transform img_rand_crop meta.csv /path/to/raw/data /pat
## Analyze datasets
-Since the dataset is provided in a CSV file, you can easily analyze the dataset using pandas (after applying `--info`). Here are some examples:
-
-```python
-import pandas as pd
-import matplotlib.pyplot as plt
+You can easily get basic information about a `.csv` dataset by using the following commands:
-data = pd.read_csv('meta.csv')
-data.hist(column="resolution")
-plt.savefig('outputs/samples/info.jpg')
+```bash
+# examine the first 10 rows of the CSV file
+head -n 10 DATA1.csv
+# count the number of data in the CSV file (approximately)
+wc -l DATA1.csv
```
-## Frame extraction speed
+For the dataset provided in a `.csv` or `.parquet` file, you can easily analyze the dataset using the following commands. Plots will be automatically saved.
-We use three libraries to extract frames from videos: `opencv`, `pyav` and `decord`. Our benchmark results of loading 256 video's middle frames are as follows:
-
-| Library | Time (s) |
-| ------- | -------- |
-| opencv | 33 |
-| decord | 28 |
-| pyav | 10 |
-
-Although `pyav` is the fastest, it can only extract the key frames instead of frames at any time. Therefore, we use `decord` as the default library for frame extraction. For dataset management, without a bottleneck on loading speed, we choose `opencv` as the default library for video information extraction.
+```python
+pyhton -m tools.datasets.analyze DATA_info.csv
+```
diff --git a/tools/datasets/analyze.py b/tools/datasets/analyze.py
index 134db45..701d716 100644
--- a/tools/datasets/analyze.py
+++ b/tools/datasets/analyze.py
@@ -1,8 +1,87 @@
+import argparse
+import os
+
+import matplotlib.pyplot as plt
+import pandas as pd
+
+
+def read_file(input_path):
+ if input_path.endswith(".csv"):
+ return pd.read_csv(input_path)
+ elif input_path.endswith(".parquet"):
+ return pd.read_parquet(input_path)
+ else:
+ raise NotImplementedError(f"Unsupported file format: {input_path}")
+
+
+def parse_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("input", type=str, help="Path to the input dataset")
+ parser.add_argument("--save-img", type=str, default="outputs/samples/infos/", help="Path to save the image")
+ return parser.parse_args()
+
+
+def plot_data(data, column, bins, name):
+ plt.clf()
+ data.hist(column=column, bins=bins)
+ os.makedirs(os.path.dirname(name), exist_ok=True)
+ plt.savefig(name)
+ print(f"Saved {name}")
+
+
+def plot_categorical_data(data, column, name):
+ plt.clf()
+ data[column].value_counts().plot(kind="bar")
+ os.makedirs(os.path.dirname(name), exist_ok=True)
+ plt.savefig(name)
+ print(f"Saved {name}")
+
+
+COLUMNS = {
+ "num_frames": 100,
+ "resolution": 100,
+ "text_len": 100,
+ "aes": 100,
+ "match": 100,
+ "flow": 100,
+ "cmotion": None,
+}
+
+
+def main(args):
+ data = read_file(args.input)
+
+ image_index = data["num_frames"] == 1
+ if image_index.sum() > 0:
+ print("=== Image Data Info ===")
+ img_data = data[image_index]
+ print(f"Number of images: {len(img_data)}")
+ print(img_data.head())
+ print(img_data.describe())
+ if args.save_img:
+ for column in COLUMNS:
+ if column in img_data.columns and column not in ["num_frames", "cmotion"]:
+ if COLUMNS[column] is None:
+ plot_categorical_data(img_data, column, os.path.join(args.save_img, f"image_{column}.png"))
+ else:
+ plot_data(img_data, column, COLUMNS[column], os.path.join(args.save_img, f"image_{column}.png"))
+ if not image_index.all():
+ print("=== Video Data Info ===")
+ video_data = data[~image_index]
+ print(f"Number of videos: {len(video_data)}")
+ print(video_data.head())
+ print(video_data.describe())
+ if args.save_img:
+ for column in COLUMNS:
+ if column in video_data.columns:
+ if COLUMNS[column] is None:
+ plot_categorical_data(video_data, column, os.path.join(args.save_img, f"video_{column}.png"))
+ else:
+ plot_data(
+ video_data, column, COLUMNS[column], os.path.join(args.save_img, f"video_{column}.png")
+ )
+
+
if __name__ == "__main__":
args = parse_args()
- if args.disable_parallel:
- pandas_has_parallel = False
- if args.seed is not None:
- random.seed(args.seed)
- np.random.seed(args.seed)
main(args)
diff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.py
index 9b21dac..addf2a3 100644
--- a/tools/datasets/csvutil.py
+++ b/tools/datasets/csvutil.py
@@ -46,7 +46,7 @@ def apply(df, func, **kwargs):
# ======================================================
-def get_info(path):
+def get_info_cv2(path):
import cv2
try:
@@ -72,6 +72,13 @@ def get_info(path):
return 0, 0, 0, np.nan, np.nan, np.nan
+def get_info_ffmpeg(path):
+ import ffmpeg
+
+ ffmpeg.probe(path)
+ breakpoint()
+
+
# ======================================================
# --remove-corrupted
# ======================================================
@@ -467,7 +474,7 @@ def main(args):
assert "text" in data.columns
data["text_len"] = apply(data["text"], lambda x: len(tokenizer(x)["input_ids"]))
if args.info:
- info = apply(data["path"], get_info)
+ info = apply(data["path"], get_info_ffmpeg)
(
data["num_frames"],
data["height"],
@@ -528,7 +535,7 @@ def main(args):
def parse_args():
parser = argparse.ArgumentParser()
- parser.add_argument("input", type=str, nargs="+")
+ parser.add_argument("input", type=str, nargs="+", help="path to the input dataset")
parser.add_argument("--output", type=str, default=None, help="output path")
parser.add_argument("--format", type=str, default="csv", help="output format", choices=["csv", "parquet"])
parser.add_argument("--disable-parallel", action="store_true", help="disable parallel processing")