Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

added checks for video corruption (#30)

Frank Lee04c502f859a5
diff --git a/tools/datasets/README.md b/tools/datasets/README.md
index 6b34287..3774000 100644
--- a/tools/datasets/README.md
+++ b/tools/datasets/README.md
@@ -101,14 +101,20 @@ You can use the following commands to process the CSV files. The output csv file
# csvutil takes multiple CSV files as input and merge them into one CSV file
# output: DATA1+DATA2.csv
python -m tools.datasets.csvutil DATA1.csv DATA2.csv
+
# shard CSV files into multiple CSV files
# output: DATA1_0.csv, DATA1_1.csv, ...
python -m tools.datasets.csvutil DATA1.csv --shard 10
+
# filter frames between 128 and 256, with captions
# output: DATA1_fmin_128_fmax_256.csv
python -m tools.datasets.csvutil DATA.csv --fmin 128 --fmax 256
+
# Disable parallel processing
python -m tools.datasets.csvutil DATA.csv --fmin 128 --fmax 256 --disable-parallel
+
+# Remove corrupted video from the csv
+python -m tools.datasets.csvutil DATA.csv --remove-corrupted
```
Here are more examples:
@@ -117,6 +123,7 @@ Here are more examples:
# modify the path to absolute path by root given
# output: DATA_abspath.csv
python -m tools.datasets.csvutil DATA.csv --abspath /absolute/path/to/dataset
+
# modify the path to relative path by root given
# output: DATA_relpath.csv
python -m tools.datasets.csvutil DATA.csv --relpath /relative/path/to/dataset
@@ -124,15 +131,19 @@ python -m tools.datasets.csvutil DATA.csv --relpath /relative/path/to/dataset
# remove the rows with empty captions
# output: DATA_noempty.csv
python -m tools.datasets.csvutil DATA.csv --remove-empty-caption
+
# remove the rows with urls
# output: DATA_nourl.csv
python -m tools.datasets.csvutil DATA.csv --remove-url
+
# unescape the caption
# output: DATA_unescape.csv
python -m tools.datasets.csvutil DATA.csv --unescape
+
# modify LLaVA caption
# output: DATA_rcp.csv
python -m tools.datasets.csvutil DATA.csv --remove-caption-prefix
+
# keep only the rows with english captions
# output: DATA_en.csv
python -m tools.datasets.csvutil DATA.csv --lang en
diff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.py
index e7eb0a5..7078257 100644
--- a/tools/datasets/csvutil.py
+++ b/tools/datasets/csvutil.py
@@ -253,6 +253,16 @@ def text_preprocessing(text, use_text_preprocessing: bool = True):
return text.lower().strip()
+def is_video_valid(path):
+ import decord
+
+ try:
+ decord.VideoReader(path, num_threads=1)
+ return True
+ except:
+ return False
+
+
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("input", type=str, nargs="+")
@@ -268,21 +278,28 @@ def parse_args():
# path processing
parser.add_argument("--relpath", type=str, default=None)
parser.add_argument("--abspath", type=str, default=None)
+
# path filtering
parser.add_argument("--ext", action="store_true")
+
# caption filtering
parser.add_argument("--remove-empty-caption", action="store_true")
parser.add_argument("--lang", type=str, default=None)
parser.add_argument("--remove-url", action="store_true")
+ parser.add_argument("--remove-corrupted", action="store_true")
+
# caption processing
parser.add_argument("--remove-caption-prefix", action="store_true")
parser.add_argument("--unescape", action="store_true")
parser.add_argument("--clean-caption", action="store_true")
+
# num_frames processing
parser.add_argument("--info", action="store_true")
+
# num_frames filtering
parser.add_argument("--fmin", type=int, default=None)
parser.add_argument("--fmax", type=int, default=None)
+
# aesthetic filtering
parser.add_argument("--aesmin", type=float, default=None)
parser.add_argument("--matchmin", type=float, default=None)
@@ -340,6 +357,8 @@ def get_output_path(args, input_name):
if args.sort_ascending is not None:
assert args.sort_descending is None
name += "_sort"
+ if args.remove_corrupted:
+ name += "_remove_corrupted"
output_path = os.path.join(dir_path, f"{name}.csv")
return output_path
@@ -399,6 +418,9 @@ def main(args):
assert "text" in data.columns
data = data[data["text"].str.len() > 0]
data = data[~data["text"].isna()]
+ if args.remove_corrupted:
+ assert "path" in data.columns
+ data = data[apply(data["path"], is_video_valid)]
# processing
if args.relpath is not None: