Open-Sora: Democratizing Efficient Video Production for All
Python
add camera motion detection (#34)
Zheng Zangwei (Alex Zheng)c9b81d8fd6f0
diff --git a/tools/caption/camera_motion_detect.py b/tools/caption/camera_motion_detect.pynew file mode 100644index 0000000..142159f--- /dev/null+++ b/tools/caption/camera_motion_detect.py@@ -0,0 +1,131 @@+# ref: https://github.com/antiboredom/camera-motion-detector++import argparse++import cv2+import numpy as np+import pandas as pd+from tqdm import tqdm++tqdm.pandas()+++def apply(df, func, **kwargs):+ if pandas_has_parallel:+ return df.parallel_apply(func, **kwargs)+ return df.progress_apply(func, **kwargs)+++try:+ from pandarallel import pandarallel++ pandarallel.initialize(progress_bar=True)+ pandas_has_parallel = True+except ImportError:+ pandas_has_parallel = False+++def make_empty(new_w, new_h):+ empty = []+ for y in range(new_h):+ xvals = []+ for x in range(new_w):+ xvals.append([x, y])+ empty.append(xvals)++ empty = np.array(empty)+ return empty+++def get_type(mag, ang, zoom_in, tau_static=1.0, tau_zoom=(0.4, 0.6)):+ if mag < tau_static:+ return "static"+ if zoom_in < tau_zoom[0]:+ return "zoom out"+ if zoom_in > tau_zoom[1]:+ return "zoom in"+ if ang < 45 or ang >= 315:+ return "pan left"+ if 45 <= ang < 135:+ return "tilt up"+ if 135 <= ang < 225:+ return "pan right"+ if 225 <= ang < 315:+ return "tilt down"+ return "unknown"+++def get_video_type(frame_types):+ # count the number of each type+ counts = {}+ max_count = 0+ max_type = None+ for frame_type in frame_types:+ if frame_type not in counts:+ counts[frame_type] = 0+ counts[frame_type] += 1+ if counts[frame_type] > max_count:+ max_count = counts[frame_type]+ max_type = frame_type+ if max_count > len(frame_types) / 2:+ return max_type+ if "static" in counts:+ return "unknown"+ if "zoom in" not in counts and "zoom out" not in counts:+ return "pan/tilt"+ return "dynamic"+++def process(path: str, frame_interval=15) -> str:+ cap = cv2.VideoCapture(path)+ count = 0+ prvs = None+ frame_types = []+ while cap.isOpened():+ ret, frame = cap.read()+ if ret:+ frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)+ if count == 0:+ prvs = frame+ h, w = frame.shape+ empty = make_empty(w, h)+ empty_dists = np.sqrt(+ np.square(empty.ravel()[::2] - (w / 2)) + np.square(empty.ravel()[1::2] - (h / 2))+ )+ else:+ flow = cv2.calcOpticalFlowFarneback(prvs, frame, None, 0.5, 3, 15, 3, 5, 1.2, 0)+ mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1], angleInDegrees=True)+ mean_mag = np.median(mag)+ mean_ang = np.median(ang)++ flow_coords = flow + empty+ xvals = flow_coords.ravel()[::2] - (w / 2)+ yvals = flow_coords.ravel()[1::2] - (h / 2)+ dists = np.sqrt(np.square(xvals) + np.square(yvals))+ dist_diff = dists >= empty_dists+ zoom_in_factor = np.count_nonzero(dist_diff) / len(dist_diff)+ frame_types.append(get_type(mean_mag, mean_ang, zoom_in_factor))+ count += frame_interval+ cap.set(cv2.CAP_PROP_POS_FRAMES, count)+ else:+ cap.release()+ break+ video_type = get_video_type(frame_types)+ return video_type+++def main(args):+ output_file = args.input.replace(".csv", "_cmotion.csv")+ data = pd.read_csv(args.input)+ data["cmotion"] = apply(data["path"], process)+ data.to_csv(output_file, index=False)+++if __name__ == "__main__":+ parser = argparse.ArgumentParser()+ parser.add_argument("input", type=str)+ parser.add_argument("--disable-parallel", action="store_true")+ args = parser.parse_args()+ if args.disable_parallel:+ pandas_has_parallel = False+ main(args)diff --git a/tools/datasets/README.md b/tools/datasets/README.mdindex 158da68..c24a0be 100644--- a/tools/datasets/README.md+++ b/tools/datasets/README.md@@ -44,6 +44,7 @@ The columns are defined as follows:- `aspect_ratio`: the aspect ratio of the video frame (height divided by width). Optional.- `aes`: the aesthetic score by [asethetic scorer](/tools/aesthetic/README.md). Optional.- `match`: the clip score by [clip scorer](/tools/clip/README.md). Optional.+- `cmotion`: the camera motion## Dataset to CSVdiff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.pyindex d332df0..abddcb6 100644--- a/tools/datasets/csvutil.py+++ b/tools/datasets/csvutil.py@@ -1,6 +1,7 @@import argparseimport htmlimport os+import randomimport refrom functools import partialfrom glob import glob@@ -85,8 +86,6 @@ LLAVA_PREFIX = [def remove_caption_prefix(caption):for prefix in LLAVA_PREFIX:- if isinstance(caption, float):- breakpoint()if caption.startswith(prefix):caption = caption[len(prefix) :].strip()if caption[0].islower():@@ -94,6 +93,41 @@ def remove_caption_prefix(caption):return caption+CMOTION_TEXT = {+ "static": "The camera is static.",+ "dynamic": "The camera is moving.",+ "unknown": None,+ "zoom in": "The camera is zooming in.",+ "zoom out": "The camera is zooming out.",+ "pan left": "The camera is panning left.",+ "pan right": "The camera is panning right.",+ "tilt up": "The camera is tilting up.",+ "tilt down": "The camera is tilting down.",+ "pan/tilt": "The camera is panning.",+}+CMOTION_PROBS = {+ # hard-coded probabilities+ "static": 0.1,+ "dynamic": 0.25,+ "unknown": 0.0,+ "zoom in": 0.8,+ "zoom out": 1.0,+ "pan left": 1.0,+ "pan right": 0.1,+ "tilt up": 1.0,+ "tilt down": 1.0,+ "pan/tilt": 1.0,+}+++def merge_cmotion(caption, cmotion):+ text = CMOTION_TEXT[cmotion]+ prob = CMOTION_PROBS[cmotion]+ if text is not None and random.random() < prob:+ caption = f"{caption} {text}"+ return caption++def build_lang_detector(lang_to_detect):from lingua import Language, LanguageDetectorBuilder@@ -251,9 +285,11 @@ def text_preprocessing(text, use_text_preprocessing: bool = True):return textelse:return text.lower().strip()-++def get_key_val_given_path(path, key, df):- return df.loc[df["path"]==path, key].item()+ return df.loc[df["path"] == path, key].item()+def is_video_valid(path):import decord@@ -270,6 +306,7 @@ def parse_args():parser.add_argument("input", type=str, nargs="+")parser.add_argument("--output", type=str, default=None)parser.add_argument("--disable-parallel", action="store_true")+ parser.add_argument("--seed", type=int, default=None)# special caseparser.add_argument("--shard", type=int, default=None)parser.add_argument("--sort-descending", type=str, default=None)@@ -295,6 +332,8 @@ def parse_args():parser.add_argument("--remove-caption-prefix", action="store_true")parser.add_argument("--unescape", action="store_true")parser.add_argument("--clean-caption", action="store_true")+ parser.add_argument("--merge-cmotion", action="store_true")+ parser.add_argument("--count-text-token", type=str, choices=["t5"], default=None)# num_frames processingparser.add_argument("--info", action="store_true")@@ -342,6 +381,10 @@ def get_output_path(args, input_name):name += "_unescape"if args.clean_caption:name += "_clean"+ if args.merge_cmotion:+ name += "_cmcaption"+ if args.count_text_token:+ name += "_textlen"# num_frames processingif args.info:name += "_info"@@ -403,12 +446,12 @@ def main(args):print(f"Intersection csv contains {len(data_int)} samples.")data = data[data["path"].isin(data_int["path"])]input_name += f"-{os.path.basename(args.intersection).split('.')[0]}"- # get additional data column from intersection+ # get additional data column from intersectionint_new_keys = [k for k in data_int.keys() if not k in data]for k in int_new_keys:- # get corresponding values- data[k] = apply(data['path'], lambda x: get_key_val_given_path(x, k, data_int))-+ # get corresponding values+ data[k] = apply(data["path"], lambda x: get_key_val_given_path(x, k, data_int))+print(f"added {len(int_new_keys)} keys from {args.intersection}: {int_new_keys}")print(f"Filtered number of samples: {len(data)}.")@@ -418,6 +461,10 @@ def main(args):# preparationif args.lang is not None:detect_lang = build_lang_detector(args.lang)+ if args.count_text_token == "t5":+ from transformers import AutoTokenizer++ tokenizer = AutoTokenizer.from_pretrained("DeepFloyd/t5-v1_1-xxl")# filteringif args.ext:@@ -457,6 +504,11 @@ def main(args):data["text"],partial(text_preprocessing, use_text_preprocessing=True),)+ if args.merge_cmotion:+ data["text"] = apply(data, lambda x: merge_cmotion(x["text"], x["cmotion"]), axis=1)+ if args.count_text_token is not None:+ assert "text" in data.columns+ data["text_len"] = apply(data["text"], lambda x: len(tokenizer(x)["input_ids"]))if args.info:info = apply(data["path"], get_video_info)(@@ -512,4 +564,7 @@ if __name__ == "__main__":args = parse_args()if args.disable_parallel:pandas_has_parallel = False+ if args.seed is not None:+ random.seed(args.seed)+ np.random.seed(args.seed)main(args)