Open-Sora: Democratizing Efficient Video Production for All
Python
add remove duplicates by path function
Shen-Chenhui0d7c226ce676
diff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.pyindex 64fc33c..d332df0 100644--- a/tools/datasets/csvutil.py+++ b/tools/datasets/csvutil.py@@ -286,6 +286,7 @@ def parse_args():# caption filteringparser.add_argument("--remove-empty-caption", action="store_true")+ parser.add_argument("--remove-duplicate-path", action="store_true")parser.add_argument("--lang", type=str, default=None)parser.add_argument("--remove-url", action="store_true")parser.add_argument("--remove-corrupted", action="store_true")@@ -328,6 +329,8 @@ def get_output_path(args, input_name):# caption filteringif args.remove_empty_caption:name += "_noempty"+ if args.remove_duplicate_path:+ name += "_noduppath"if args.lang is not None:name += f"_{args.lang}"if args.remove_url:@@ -430,6 +433,9 @@ def main(args):assert "text" in data.columnsdata = data[data["text"].str.len() > 0]data = data[~data["text"].isna()]+ if args.remove_duplicate_path:+ assert "path" in data.columns+ data = data.drop_duplicates(subset=['path'])if args.remove_corrupted:assert "path" in data.columnsdata = data[apply(data["path"], is_video_valid)]