Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

update data processing

xyupengca53b1a21b06
diff --git a/tools/optical_flow/README.md b/tools/optical_flow/README.md
deleted file mode 100644
index 414cc6b..0000000
--- a/tools/optical_flow/README.md
+++ /dev/null
@@ -1 +0,0 @@
-`python tools/optical_flow/inference.py --meta_path ./data/Panda-70M/processed/meta/test_intact_cut_head-100.csv`
\ No newline at end of file
diff --git a/tools/scoring/README.md b/tools/scoring/README.md
new file mode 100644
index 0000000..64cb73e
--- /dev/null
+++ b/tools/scoring/README.md
@@ -0,0 +1,8 @@
+## Data Scoring and Filtering
+
+### Aesthetic Score
+
+### Optical Flow Score
+`python tools/scoring/optical_flow/inference.py --meta_path ./data/Panda-70M/processed/meta/test_intact_cut_head-100.csv`
+
+### Matching Score
\ No newline at end of file
diff --git a/tools/aesthetic/__init__.py b/tools/scoring/__init__.py
similarity index 100%
rename from tools/aesthetic/__init__.py
rename to tools/scoring/__init__.py
diff --git a/tools/aesthetic/README.md b/tools/scoring/aesthetic/README.md
similarity index 100%
rename from tools/aesthetic/README.md
rename to tools/scoring/aesthetic/README.md
diff --git a/tools/optical_flow/__init__.py b/tools/scoring/aesthetic/__init__.py
similarity index 100%
rename from tools/optical_flow/__init__.py
rename to tools/scoring/aesthetic/__init__.py
diff --git a/tools/aesthetic/inference.py b/tools/scoring/aesthetic/inference.py
similarity index 100%
rename from tools/aesthetic/inference.py
rename to tools/scoring/aesthetic/inference.py
diff --git a/tools/scoring/optical_flow/__init__.py b/tools/scoring/optical_flow/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tools/optical_flow/inference.py b/tools/scoring/optical_flow/inference.py
similarity index 100%
rename from tools/optical_flow/inference.py
rename to tools/scoring/optical_flow/inference.py
index 6965add..2ccb132 100644
--- a/tools/optical_flow/inference.py
+++ b/tools/scoring/optical_flow/inference.py
@@ -4,7 +4,6 @@ print(f'Current working directory: {os.getcwd()}')
import argparse
import av
-import decord
import numpy as np
import pandas as pd
from einops import rearrange
@@ -16,6 +15,7 @@ import torch.nn.functional as F
from torch.utils.data import Dataset
from torchvision.transforms.functional import pil_to_tensor
+import decord
from unimatch import UniMatch
diff --git a/tools/optical_flow/unimatch/__init__.py b/tools/scoring/optical_flow/unimatch/__init__.py
similarity index 100%
rename from tools/optical_flow/unimatch/__init__.py
rename to tools/scoring/optical_flow/unimatch/__init__.py
diff --git a/tools/optical_flow/unimatch/attention.py b/tools/scoring/optical_flow/unimatch/attention.py
similarity index 100%
rename from tools/optical_flow/unimatch/attention.py
rename to tools/scoring/optical_flow/unimatch/attention.py
diff --git a/tools/optical_flow/unimatch/backbone.py b/tools/scoring/optical_flow/unimatch/backbone.py
similarity index 100%
rename from tools/optical_flow/unimatch/backbone.py
rename to tools/scoring/optical_flow/unimatch/backbone.py
diff --git a/tools/optical_flow/unimatch/geometry.py b/tools/scoring/optical_flow/unimatch/geometry.py
similarity index 100%
rename from tools/optical_flow/unimatch/geometry.py
rename to tools/scoring/optical_flow/unimatch/geometry.py
diff --git a/tools/optical_flow/unimatch/matching.py b/tools/scoring/optical_flow/unimatch/matching.py
similarity index 100%
rename from tools/optical_flow/unimatch/matching.py
rename to tools/scoring/optical_flow/unimatch/matching.py
diff --git a/tools/optical_flow/unimatch/position.py b/tools/scoring/optical_flow/unimatch/position.py
similarity index 100%
rename from tools/optical_flow/unimatch/position.py
rename to tools/scoring/optical_flow/unimatch/position.py
diff --git a/tools/optical_flow/unimatch/reg_refine.py b/tools/scoring/optical_flow/unimatch/reg_refine.py
similarity index 100%
rename from tools/optical_flow/unimatch/reg_refine.py
rename to tools/scoring/optical_flow/unimatch/reg_refine.py
diff --git a/tools/optical_flow/unimatch/transformer.py b/tools/scoring/optical_flow/unimatch/transformer.py
similarity index 100%
rename from tools/optical_flow/unimatch/transformer.py
rename to tools/scoring/optical_flow/unimatch/transformer.py
diff --git a/tools/optical_flow/unimatch/trident_conv.py b/tools/scoring/optical_flow/unimatch/trident_conv.py
similarity index 100%
rename from tools/optical_flow/unimatch/trident_conv.py
rename to tools/scoring/optical_flow/unimatch/trident_conv.py
diff --git a/tools/optical_flow/unimatch/unimatch.py b/tools/scoring/optical_flow/unimatch/unimatch.py
similarity index 100%
rename from tools/optical_flow/unimatch/unimatch.py
rename to tools/scoring/optical_flow/unimatch/unimatch.py
diff --git a/tools/optical_flow/unimatch/utils.py b/tools/scoring/optical_flow/unimatch/utils.py
similarity index 100%
rename from tools/optical_flow/unimatch/utils.py
rename to tools/scoring/optical_flow/unimatch/utils.py
diff --git a/tools/splitting/README.md b/tools/splitting/README.md
new file mode 100644
index 0000000..0e3e0ab
--- /dev/null
+++ b/tools/splitting/README.md
@@ -0,0 +1,7 @@
+## Scene Detection and Video Splitting
+
+### Scene Detection
+TODO
+
+### Video Splitting
+TODO
diff --git a/tools/splitting/__init__.py b/tools/splitting/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tools/scenedetect/README.md b/tools/splitting/scenedetect/README.md
similarity index 100%
rename from tools/scenedetect/README.md
rename to tools/splitting/scenedetect/README.md
diff --git a/tools/scenedetect/scene_detect.py b/tools/splitting/scenedetect/scene_detect.py
similarity index 100%
rename from tools/scenedetect/scene_detect.py
rename to tools/splitting/scenedetect/scene_detect.py
diff --git a/tools/scenedetect/utils.py b/tools/splitting/scenedetect/utils.py
similarity index 100%
rename from tools/scenedetect/utils.py
rename to tools/splitting/scenedetect/utils.py