Open-Sora: Democratizing Efficient Video Production for All
Python
deduplicate
Tom Young43ef5759a322
diff --git a/tools/datasets/convert.py b/tools/datasets/convert.pyindex 49acf72..f35d91e 100644--- a/tools/datasets/convert.py+++ b/tools/datasets/convert.py@@ -106,6 +106,7 @@ def process_general_images(root, output):def process_general_videos(root, output):root = os.path.expanduser(root)path_list = get_filelist(root, VID_EXTENSIONS)+ path_list = list(set(path_list)) # remove duplicatesfname_list = [os.path.splitext(os.path.basename(x))[0] for x in path_list]relpath_list = [os.path.relpath(x, root) for x in path_list]df = pd.DataFrame(dict(path=path_list, id=fname_list, relpath=relpath_list))