Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

complete data format

Zangwei Zheng3c5ce7f74346
diff --git a/tools/datasets/README.md b/tools/datasets/README.md
index d94900e..fadf484 100644
--- a/tools/datasets/README.md
+++ b/tools/datasets/README.md
@@ -1,15 +1,5 @@
# Dataset Download and Management
-## Dataset Format
-
-The training data should be provided in a CSV file, with each row containing the following information:
-
-```csv
-path, text, num_frames, aesthetic_score
-/absolute/path/to/image1.jpg, caption1, num_of_frames
-/absolute/path/to/video2.mp4, caption2, num_of_frames
-```
-
## HD-VG-130M
This dataset comprises 130M text-video pairs. You can download the dataset and prepare it for training according to [the dataset repository's instructions](https://github.com/daooshee/HD-VG-130M). There is a README.md file in the Google Drive link that provides instructions on how to download and cut the videos. For this version, we directly use the dataset provided by the authors.
@@ -31,25 +21,36 @@ python -m tools.datasets.convert_dataset imagenet IMAGENET_FOLDER --split train
python -m tools.datasets.convert_dataset ucf101 UCF101_FOLDER --split videos
```
+## Dataset Format
+
+The dataset should be provided in a CSV file, which is used both for training and data preprocessing. The CSV file should only contain the following columns (can be optional):
+
+```csv
+path, text, num_frames, aesthetic_score, fps, width, height, aspect_ratio
+/absolute/path/to/image1.jpg, caption1, num_of_frames, score1
+/absolute/path/to/video2.mp4, caption2, num_of_frames, score2
+```
+
+We use pandas to manage the CSV files. You can use the following code to read and write the CSV files:
+
+```python
+df = pd.read_csv(input_path)
+df = df.to_csv(output_path, index=False)
+```
+
## Manage datasets
We provide `csvutils.py` to manage the CSV files. You can use the following commands to process the CSV files:
```bash
-# generate DATA_fmin_128_fmax_256.csv with frames between 128 and 256
-python -m tools.datasets.csvutil DATA.csv --fmin 128 --fmax 256
-# generate DATA_root.csv with absolute path
-python -m tools.datasets.csvutil DATA.csv --root /absolute/path/to/dataset
-# remove videos with no captions
-python -m tools.datasets.csvutil DATA.csv --remove-empty-caption
+# csvutil takes multiple CSV files as input and merge them into one CSV file
+python -m tools.datasets.csvutil DATA1.csv DATA2.csv
+# filter frames between 128 and 256, with captions
+python -m tools.datasets.csvutil DATA.csv --fmin 128 --fmax 256 --remove-empty-caption
# compute the number of frames for each video
python -m tools.datasets.csvutil DATA.csv --relength
# remove caption prefix
python -m tools.datasets.csvutil DATA.csv --remove-caption-prefix
-```
-
-To merge multiple CSV files, you can use the following command:
-
-```bash
-cat *csv > combined.csv
+# generate DATA_root.csv with absolute path
+python -m tools.datasets.csvutil DATA.csv --abspath /absolute/path/to/dataset
```
diff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.py
index a4ac929..a9cdb76 100644
--- a/tools/datasets/csvutil.py
+++ b/tools/datasets/csvutil.py
@@ -4,10 +4,21 @@ import html
import os
import re
+import numpy as np
+import pandas as pd
from tqdm import tqdm
-# path, name, #frames
-PREFIX = [
+tqdm.pandas()
+
+
+def get_video_length(path):
+ import cv2
+
+ cap = cv2.VideoCapture(path)
+ return int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+
+
+LLAVA_PREFIX = [
"The video shows",
"The video captures",
"The video features",
@@ -19,136 +30,170 @@ PREFIX = [
]
-def get_video_length(path):
- import cv2
+def remove_caption_prefix(caption):
+ for prefix in LLAVA_PREFIX:
+ if caption.startswith(prefix):
+ caption = caption[len(prefix) :].strip()
+ if caption[0].islower():
+ caption = caption[0].upper() + caption[1:]
+ return caption
- cap = cv2.VideoCapture(path)
- return int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+def build_lang_detector(lang_to_detect):
+ from lingua import Language, LanguageDetectorBuilder
-def shard_csv(data, shard, input_path):
- n = len(data)
- num_shard = n // shard
- for i in range(shard):
- start = i * num_shard
- end = (i + 1) * num_shard if i != shard - 1 else n
- file_name, ext = os.path.splitext(input_path)
- output_path = os.path.join(os.path.join(os.path.dirname(file_name)), f"{os.path.basename(file_name)}_{i}{ext}")
- with open(output_path, "w") as f:
- writer = csv.writer(f)
- writer.writerows(data[start:end])
- print(f"Saved {end - start} samples to {output_path}.")
- print("Done.")
+ lang_dict = dict(en=Language.ENGLISH)
+ assert lang_to_detect in lang_dict
+ valid_lang = lang_dict[lang_to_detect]
+ detector = LanguageDetectorBuilder.from_all_spoken_languages().with_low_accuracy_mode().build()
+ def detect_lang(caption):
+ confidence_values = detector.compute_language_confidence_values(caption)
+ confidence = [x.language for x in confidence_values[:5]]
+ if valid_lang not in confidence:
+ return False
+ return True
-def main(args):
- input_path = args.input
- output_path = args.output
- if output_path is None:
- name = os.path.basename(input_path)
- name, ext = os.path.splitext(name)
- if args.fmin is not None:
- name += f"_fmin_{args.fmin}"
- if args.fmax is not None:
- name += f"_fmax_{args.fmax}"
- if args.remove_empty_caption:
- name += "_rec"
- if args.remove_caption_prefix:
- name += "_rcp"
- if args.root is not None:
- name += f"_root"
- if args.relength:
- name += "_relength"
- if args.clean:
- name += "_clean"
- if args.unescape:
- name += "_unescape"
- if args.remove_url:
- name += "_nourl"
- if args.lang is not None:
- name += f"_{args.lang}"
- from lingua import Language, LanguageDetectorBuilder
-
- lang_dict = dict(en=Language.ENGLISH)
- assert args.lang in lang_dict
- valid_lang = lang_dict[args.lang]
- detector = LanguageDetectorBuilder.from_all_spoken_languages().with_low_accuracy_mode().build()
- output_path = os.path.join(os.path.dirname(input_path), name + ext)
-
- with open(input_path, "r") as f:
- reader = csv.reader(f)
- data = list(reader)
- print("Number of videos before filtering:", len(data))
+ return detect_lang
- if args.shard is not None:
- shard_csv(data, args.shard, input_path)
- return
-
- data_new = []
- for i, row in tqdm(enumerate(data)):
- path = row[0]
- caption = row[1]
- if len(row) >= 3:
- n_frames = int(row[2])
- if args.fmin is not None and n_frames < args.fmin:
- continue
- if args.fmax is not None and n_frames > args.fmax:
- continue
- if args.remove_empty_caption and len(caption) == 0:
- continue
- if args.clean:
- caption = caption.strip()
- row[1] = caption
- if args.unescape:
- caption = html.unescape(caption)
- row[1] = caption
- if args.lang is not None:
- confidence_values = detector.compute_language_confidence_values(caption)
- confidence = [x.language for x in confidence_values[:5]]
- if valid_lang not in confidence:
- continue
- if args.remove_url:
- if re.search("(?P<url>https?://[^\s]+)", caption) is not None:
- continue
- if args.remove_caption_prefix:
- for prefix in PREFIX:
- if caption.startswith(prefix):
- caption = caption[len(prefix) :].strip()
- if caption[0].islower():
- caption = caption[0].upper() + caption[1:]
- row[1] = caption
- break
- if args.root is not None:
- row[0] = os.path.join(args.root, path)
- if args.relength:
- n_frames = get_video_length(row[0])
- if len(row) < 3:
- row.append(n_frames)
- else:
- row[2] = n_frames
- data_new.append(row)
-
- print("Number of videos after filtering:", len(data_new))
- with open(output_path, "w") as f:
- writer = csv.writer(f)
- writer.writerows(data_new)
- print("Output saved to", output_path)
-
-if __name__ == "__main__":
+def parse_args():
parser = argparse.ArgumentParser()
- parser.add_argument("input", type=str)
+ parser.add_argument("input", type=str, nargs="+")
parser.add_argument("--output", type=str, default=None)
- parser.add_argument("--fmin", type=int, default=None)
- parser.add_argument("--fmax", type=int, default=None)
- parser.add_argument("--root", type=str, default=None)
- parser.add_argument("--remove-empty-caption", action="store_true")
- parser.add_argument("--remove-caption-prefix", action="store_true")
- parser.add_argument("--relength", action="store_true")
+ # special case
parser.add_argument("--shard", type=int, default=None)
+
+ # path processing
+ parser.add_argument("--abspath", type=str, default=None)
+ parser.add_argument("--relpath", type=str, default=None)
+ # caption filtering
+ parser.add_argument("--remove-empty-caption", action="store_true")
parser.add_argument("--lang", type=str, default=None)
- parser.add_argument("--clean", action="store_true")
- parser.add_argument("--unescape", action="store_true")
parser.add_argument("--remove-url", action="store_true")
- args = parser.parse_args()
+ # caption processing
+ parser.add_argument("--remove-caption-prefix", action="store_true")
+ parser.add_argument("--unescape", action="store_true")
+ # num_frames processing
+ parser.add_argument("--relength", action="store_true")
+ # num_frames filtering
+ parser.add_argument("--fmin", type=int, default=None)
+ parser.add_argument("--fmax", type=int, default=None)
+ # aesthetic filtering
+ parser.add_argument("--aesmin", type=float, default=None)
+
+ return parser.parse_args()
+
+
+def get_output_path(args, input_name):
+ if args.output is not None:
+ return args.output
+
+ name = input_name
+ dir_path = os.path.dirname(args.input[0])
+
+ # path processing
+ if args.abspath is not None:
+ name += "_abspath"
+ if args.relpath is not None:
+ name += "_relpath"
+ # caption filtering
+ if args.remove_empty_caption:
+ name += "_rec"
+ if args.lang is not None:
+ name += f"_{args.lang}"
+ if args.remove_url:
+ name += "_nourl"
+ # caption processing
+ if args.remove_caption_prefix:
+ name += "_rcp"
+ if args.unescape:
+ name += "_unescape"
+ # num_frames processing
+ if args.relength:
+ name += "_relength"
+ # num_frames filtering
+ if args.fmin is not None:
+ name += f"_fmin_{args.fmin}"
+ if args.fmax is not None:
+ name += f"_fmax_{args.fmax}"
+ # aesthetic filtering
+ if args.aesmin is not None:
+ name += f"_aesmin_{args.aesmin}"
+
+ output_path = os.path.join(dir_path, f"{name}.csv")
+ return output_path
+
+
+def main(args):
+ # reading data
+ data = []
+ input_name = ""
+ for i, input_path in enumerate(args.input):
+ data.append(pd.read_csv(input_path))
+ input_name += os.path.basename(input_path).split(".")[0]
+ if i != len(args.input) - 1:
+ input_name += "|"
+ print(f"Loaded {len(data[-1])} samples from {input_path}.")
+ data = pd.concat(data, ignore_index=True, sort=False)
+ print(f"Total number of samples: {len(data)}.")
+
+ # get output path
+ output_path = get_output_path(args, input_name)
+
+ # preparation
+ if args.lang is not None:
+ detect_lang = build_lang_detector(args.lang)
+
+ # processing
+ if args.abspath is not None:
+ assert args.relpath is None
+ data["path"] = data["path"].progress_apply(lambda x: os.path.join(args.abspath, x))
+ if args.relpath is not None:
+ assert args.abspath is None
+ data["path"] = data["path"].progress_apply(lambda x: os.path.relpath(x, args.relpath))
+ if args.remove_caption_prefix:
+ assert "text" in data.columns
+ data["text"] = data["text"].progress_apply(remove_caption_prefix)
+ if args.unescape:
+ assert "text" in data.columns
+ data["text"] = data["text"].progress_apply(html.unescape)
+ if args.relength:
+ data["num_frames"] = data["path"].progress_apply(get_video_length)
+
+ # filtering
+ if args.remove_empty_caption:
+ assert "text" in data.columns
+ data = data[data["text"].str.len() > 0]
+ if args.remove_url:
+ assert "text" in data.columns
+ data = data[~data["text"].str.contains(r"(?P<url>https?://[^\s]+)", regex=True)]
+ if args.lang is not None:
+ assert "text" in data.columns
+ data = data[data["text"].progress_apply(detect_lang)]
+ if args.fmin is not None:
+ assert "num_frames" in data.columns
+ data = data[data["num_frames"] >= args.fmin]
+ if args.fmax is not None:
+ assert "num_frames" in data.columns
+ data = data[data["num_frames"] <= args.fmax]
+ if args.aesmin is not None:
+ assert "aesthetic_score" in data.columns
+ data = data[data["aesthetic_score"] >= args.aesmin]
+ print(f"Filtered number of samples: {len(data)}.")
+
+ # shard data
+ if args.shard is not None:
+ sharded_data = np.array_split(data, args.shard)
+ for i in range(args.shard):
+ output_path = output_path.replace(".csv", f"_{i}.csv")
+ sharded_data[i].to_csv(output_path, index=False)
+ print(f"Saved {len(sharded_data[i])} samples to {output_path}.")
+ else:
+ data.to_csv(output_path, index=False)
+ print(f"Saved {len(data)} samples to {output_path}.")
+
+
+if __name__ == "__main__":
+ args = parse_args()
main(args)