Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

deduplicate

Tom Young43ef5759a322
diff --git a/tools/datasets/convert.py b/tools/datasets/convert.py
index 49acf72..f35d91e 100644
--- a/tools/datasets/convert.py
+++ b/tools/datasets/convert.py
@@ -106,6 +106,7 @@ def process_general_images(root, output):
def process_general_videos(root, output):
root = os.path.expanduser(root)
path_list = get_filelist(root, VID_EXTENSIONS)
+ path_list = list(set(path_list)) # remove duplicates
fname_list = [os.path.splitext(os.path.basename(x))[0] for x in path_list]
relpath_list = [os.path.relpath(x, root) for x in path_list]
df = pd.DataFrame(dict(path=path_list, id=fname_list, relpath=relpath_list))