Open-Sora: Democratizing Efficient Video Production for All
Python
added checks for video corruption (#30)
Frank Lee04c502f859a5
diff --git a/tools/datasets/README.md b/tools/datasets/README.mdindex 6b34287..3774000 100644--- a/tools/datasets/README.md+++ b/tools/datasets/README.md@@ -101,14 +101,20 @@ You can use the following commands to process the CSV files. The output csv file# csvutil takes multiple CSV files as input and merge them into one CSV file# output: DATA1+DATA2.csvpython -m tools.datasets.csvutil DATA1.csv DATA2.csv+# shard CSV files into multiple CSV files# output: DATA1_0.csv, DATA1_1.csv, ...python -m tools.datasets.csvutil DATA1.csv --shard 10+# filter frames between 128 and 256, with captions# output: DATA1_fmin_128_fmax_256.csvpython -m tools.datasets.csvutil DATA.csv --fmin 128 --fmax 256+# Disable parallel processingpython -m tools.datasets.csvutil DATA.csv --fmin 128 --fmax 256 --disable-parallel++# Remove corrupted video from the csv+python -m tools.datasets.csvutil DATA.csv --remove-corrupted```Here are more examples:@@ -117,6 +123,7 @@ Here are more examples:# modify the path to absolute path by root given# output: DATA_abspath.csvpython -m tools.datasets.csvutil DATA.csv --abspath /absolute/path/to/dataset+# modify the path to relative path by root given# output: DATA_relpath.csvpython -m tools.datasets.csvutil DATA.csv --relpath /relative/path/to/dataset@@ -124,15 +131,19 @@ python -m tools.datasets.csvutil DATA.csv --relpath /relative/path/to/dataset# remove the rows with empty captions# output: DATA_noempty.csvpython -m tools.datasets.csvutil DATA.csv --remove-empty-caption+# remove the rows with urls# output: DATA_nourl.csvpython -m tools.datasets.csvutil DATA.csv --remove-url+# unescape the caption# output: DATA_unescape.csvpython -m tools.datasets.csvutil DATA.csv --unescape+# modify LLaVA caption# output: DATA_rcp.csvpython -m tools.datasets.csvutil DATA.csv --remove-caption-prefix+# keep only the rows with english captions# output: DATA_en.csvpython -m tools.datasets.csvutil DATA.csv --lang endiff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.pyindex e7eb0a5..7078257 100644--- a/tools/datasets/csvutil.py+++ b/tools/datasets/csvutil.py@@ -253,6 +253,16 @@ def text_preprocessing(text, use_text_preprocessing: bool = True):return text.lower().strip()+def is_video_valid(path):+ import decord++ try:+ decord.VideoReader(path, num_threads=1)+ return True+ except:+ return False++def parse_args():parser = argparse.ArgumentParser()parser.add_argument("input", type=str, nargs="+")@@ -268,21 +278,28 @@ def parse_args():# path processingparser.add_argument("--relpath", type=str, default=None)parser.add_argument("--abspath", type=str, default=None)+# path filteringparser.add_argument("--ext", action="store_true")+# caption filteringparser.add_argument("--remove-empty-caption", action="store_true")parser.add_argument("--lang", type=str, default=None)parser.add_argument("--remove-url", action="store_true")+ parser.add_argument("--remove-corrupted", action="store_true")+# caption processingparser.add_argument("--remove-caption-prefix", action="store_true")parser.add_argument("--unescape", action="store_true")parser.add_argument("--clean-caption", action="store_true")+# num_frames processingparser.add_argument("--info", action="store_true")+# num_frames filteringparser.add_argument("--fmin", type=int, default=None)parser.add_argument("--fmax", type=int, default=None)+# aesthetic filteringparser.add_argument("--aesmin", type=float, default=None)parser.add_argument("--matchmin", type=float, default=None)@@ -340,6 +357,8 @@ def get_output_path(args, input_name):if args.sort_ascending is not None:assert args.sort_descending is Nonename += "_sort"+ if args.remove_corrupted:+ name += "_remove_corrupted"output_path = os.path.join(dir_path, f"{name}.csv")return output_path@@ -399,6 +418,9 @@ def main(args):assert "text" in data.columnsdata = data[data["text"].str.len() > 0]data = data[~data["text"].isna()]+ if args.remove_corrupted:+ assert "path" in data.columns+ data = data[apply(data["path"], is_video_valid)]# processingif args.relpath is not None: