Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

add camera motion detection (#34)

Zheng Zangwei (Alex Zheng)c9b81d8fd6f0
diff --git a/tools/caption/camera_motion_detect.py b/tools/caption/camera_motion_detect.py
new file mode 100644
index 0000000..142159f
--- /dev/null
+++ b/tools/caption/camera_motion_detect.py
@@ -0,0 +1,131 @@
+# ref: https://github.com/antiboredom/camera-motion-detector
+
+import argparse
+
+import cv2
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+
+tqdm.pandas()
+
+
+def apply(df, func, **kwargs):
+ if pandas_has_parallel:
+ return df.parallel_apply(func, **kwargs)
+ return df.progress_apply(func, **kwargs)
+
+
+try:
+ from pandarallel import pandarallel
+
+ pandarallel.initialize(progress_bar=True)
+ pandas_has_parallel = True
+except ImportError:
+ pandas_has_parallel = False
+
+
+def make_empty(new_w, new_h):
+ empty = []
+ for y in range(new_h):
+ xvals = []
+ for x in range(new_w):
+ xvals.append([x, y])
+ empty.append(xvals)
+
+ empty = np.array(empty)
+ return empty
+
+
+def get_type(mag, ang, zoom_in, tau_static=1.0, tau_zoom=(0.4, 0.6)):
+ if mag < tau_static:
+ return "static"
+ if zoom_in < tau_zoom[0]:
+ return "zoom out"
+ if zoom_in > tau_zoom[1]:
+ return "zoom in"
+ if ang < 45 or ang >= 315:
+ return "pan left"
+ if 45 <= ang < 135:
+ return "tilt up"
+ if 135 <= ang < 225:
+ return "pan right"
+ if 225 <= ang < 315:
+ return "tilt down"
+ return "unknown"
+
+
+def get_video_type(frame_types):
+ # count the number of each type
+ counts = {}
+ max_count = 0
+ max_type = None
+ for frame_type in frame_types:
+ if frame_type not in counts:
+ counts[frame_type] = 0
+ counts[frame_type] += 1
+ if counts[frame_type] > max_count:
+ max_count = counts[frame_type]
+ max_type = frame_type
+ if max_count > len(frame_types) / 2:
+ return max_type
+ if "static" in counts:
+ return "unknown"
+ if "zoom in" not in counts and "zoom out" not in counts:
+ return "pan/tilt"
+ return "dynamic"
+
+
+def process(path: str, frame_interval=15) -> str:
+ cap = cv2.VideoCapture(path)
+ count = 0
+ prvs = None
+ frame_types = []
+ while cap.isOpened():
+ ret, frame = cap.read()
+ if ret:
+ frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+ if count == 0:
+ prvs = frame
+ h, w = frame.shape
+ empty = make_empty(w, h)
+ empty_dists = np.sqrt(
+ np.square(empty.ravel()[::2] - (w / 2)) + np.square(empty.ravel()[1::2] - (h / 2))
+ )
+ else:
+ flow = cv2.calcOpticalFlowFarneback(prvs, frame, None, 0.5, 3, 15, 3, 5, 1.2, 0)
+ mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1], angleInDegrees=True)
+ mean_mag = np.median(mag)
+ mean_ang = np.median(ang)
+
+ flow_coords = flow + empty
+ xvals = flow_coords.ravel()[::2] - (w / 2)
+ yvals = flow_coords.ravel()[1::2] - (h / 2)
+ dists = np.sqrt(np.square(xvals) + np.square(yvals))
+ dist_diff = dists >= empty_dists
+ zoom_in_factor = np.count_nonzero(dist_diff) / len(dist_diff)
+ frame_types.append(get_type(mean_mag, mean_ang, zoom_in_factor))
+ count += frame_interval
+ cap.set(cv2.CAP_PROP_POS_FRAMES, count)
+ else:
+ cap.release()
+ break
+ video_type = get_video_type(frame_types)
+ return video_type
+
+
+def main(args):
+ output_file = args.input.replace(".csv", "_cmotion.csv")
+ data = pd.read_csv(args.input)
+ data["cmotion"] = apply(data["path"], process)
+ data.to_csv(output_file, index=False)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("input", type=str)
+ parser.add_argument("--disable-parallel", action="store_true")
+ args = parser.parse_args()
+ if args.disable_parallel:
+ pandas_has_parallel = False
+ main(args)
diff --git a/tools/datasets/README.md b/tools/datasets/README.md
index 158da68..c24a0be 100644
--- a/tools/datasets/README.md
+++ b/tools/datasets/README.md
@@ -44,6 +44,7 @@ The columns are defined as follows:
- `aspect_ratio`: the aspect ratio of the video frame (height divided by width). Optional.
- `aes`: the aesthetic score by [asethetic scorer](/tools/aesthetic/README.md). Optional.
- `match`: the clip score by [clip scorer](/tools/clip/README.md). Optional.
+- `cmotion`: the camera motion
## Dataset to CSV
diff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.py
index d332df0..abddcb6 100644
--- a/tools/datasets/csvutil.py
+++ b/tools/datasets/csvutil.py
@@ -1,6 +1,7 @@
import argparse
import html
import os
+import random
import re
from functools import partial
from glob import glob
@@ -85,8 +86,6 @@ LLAVA_PREFIX = [
def remove_caption_prefix(caption):
for prefix in LLAVA_PREFIX:
- if isinstance(caption, float):
- breakpoint()
if caption.startswith(prefix):
caption = caption[len(prefix) :].strip()
if caption[0].islower():
@@ -94,6 +93,41 @@ def remove_caption_prefix(caption):
return caption
+CMOTION_TEXT = {
+ "static": "The camera is static.",
+ "dynamic": "The camera is moving.",
+ "unknown": None,
+ "zoom in": "The camera is zooming in.",
+ "zoom out": "The camera is zooming out.",
+ "pan left": "The camera is panning left.",
+ "pan right": "The camera is panning right.",
+ "tilt up": "The camera is tilting up.",
+ "tilt down": "The camera is tilting down.",
+ "pan/tilt": "The camera is panning.",
+}
+CMOTION_PROBS = {
+ # hard-coded probabilities
+ "static": 0.1,
+ "dynamic": 0.25,
+ "unknown": 0.0,
+ "zoom in": 0.8,
+ "zoom out": 1.0,
+ "pan left": 1.0,
+ "pan right": 0.1,
+ "tilt up": 1.0,
+ "tilt down": 1.0,
+ "pan/tilt": 1.0,
+}
+
+
+def merge_cmotion(caption, cmotion):
+ text = CMOTION_TEXT[cmotion]
+ prob = CMOTION_PROBS[cmotion]
+ if text is not None and random.random() < prob:
+ caption = f"{caption} {text}"
+ return caption
+
+
def build_lang_detector(lang_to_detect):
from lingua import Language, LanguageDetectorBuilder
@@ -251,9 +285,11 @@ def text_preprocessing(text, use_text_preprocessing: bool = True):
return text
else:
return text.lower().strip()
-
+
+
def get_key_val_given_path(path, key, df):
- return df.loc[df["path"]==path, key].item()
+ return df.loc[df["path"] == path, key].item()
+
def is_video_valid(path):
import decord
@@ -270,6 +306,7 @@ def parse_args():
parser.add_argument("input", type=str, nargs="+")
parser.add_argument("--output", type=str, default=None)
parser.add_argument("--disable-parallel", action="store_true")
+ parser.add_argument("--seed", type=int, default=None)
# special case
parser.add_argument("--shard", type=int, default=None)
parser.add_argument("--sort-descending", type=str, default=None)
@@ -295,6 +332,8 @@ def parse_args():
parser.add_argument("--remove-caption-prefix", action="store_true")
parser.add_argument("--unescape", action="store_true")
parser.add_argument("--clean-caption", action="store_true")
+ parser.add_argument("--merge-cmotion", action="store_true")
+ parser.add_argument("--count-text-token", type=str, choices=["t5"], default=None)
# num_frames processing
parser.add_argument("--info", action="store_true")
@@ -342,6 +381,10 @@ def get_output_path(args, input_name):
name += "_unescape"
if args.clean_caption:
name += "_clean"
+ if args.merge_cmotion:
+ name += "_cmcaption"
+ if args.count_text_token:
+ name += "_textlen"
# num_frames processing
if args.info:
name += "_info"
@@ -403,12 +446,12 @@ def main(args):
print(f"Intersection csv contains {len(data_int)} samples.")
data = data[data["path"].isin(data_int["path"])]
input_name += f"-{os.path.basename(args.intersection).split('.')[0]}"
- # get additional data column from intersection
+ # get additional data column from intersection
int_new_keys = [k for k in data_int.keys() if not k in data]
for k in int_new_keys:
- # get corresponding values
- data[k] = apply(data['path'], lambda x: get_key_val_given_path(x, k, data_int))
-
+ # get corresponding values
+ data[k] = apply(data["path"], lambda x: get_key_val_given_path(x, k, data_int))
+
print(f"added {len(int_new_keys)} keys from {args.intersection}: {int_new_keys}")
print(f"Filtered number of samples: {len(data)}.")
@@ -418,6 +461,10 @@ def main(args):
# preparation
if args.lang is not None:
detect_lang = build_lang_detector(args.lang)
+ if args.count_text_token == "t5":
+ from transformers import AutoTokenizer
+
+ tokenizer = AutoTokenizer.from_pretrained("DeepFloyd/t5-v1_1-xxl")
# filtering
if args.ext:
@@ -457,6 +504,11 @@ def main(args):
data["text"],
partial(text_preprocessing, use_text_preprocessing=True),
)
+ if args.merge_cmotion:
+ data["text"] = apply(data, lambda x: merge_cmotion(x["text"], x["cmotion"]), axis=1)
+ if args.count_text_token is not None:
+ assert "text" in data.columns
+ data["text_len"] = apply(data["text"], lambda x: len(tokenizer(x)["input_ids"]))
if args.info:
info = apply(data["path"], get_video_info)
(
@@ -512,4 +564,7 @@ if __name__ == "__main__":
args = parse_args()
if args.disable_parallel:
pandas_has_parallel = False
+ if args.seed is not None:
+ random.seed(args.seed)
+ np.random.seed(args.seed)
main(args)