Open-Sora: Democratizing Efficient Video Production for All
Python
new camera motion detection (#65)
Tom Young335e8a3eedbc
diff --git a/tools/caption/README.md b/tools/caption/README.mdindex b30856c..da55638 100644--- a/tools/caption/README.md+++ b/tools/caption/README.md@@ -93,3 +93,35 @@ python -m tools.caption.caption_gpt4 DATA.csv --key $OPENAI_API_KEY```The cost is approximately $0.01 per video (3 frames per video).++## Camera Motion Detection++Install additional required packages: `tools/caption/camera_motion/requirements.txt`.++Run the following command to classify camera motion:++```bash+# output: meta_cmotion.csv+python -m tools.caption.camera_motion.detect tools/caption/camera_motion/meta.csv+```++You may additionally specify `threshold` to indicate how "sensitive" the detection should be as below. For example `threshold = 0.2` means that the video is only counted as `tilt_up` when the pixels moved down by `>20%` of video height between the starting and ending frames.+```bash+# output: meta_cmotion.csv+python -m tools.caption.camera_motion.detect tools/caption/camera_motion/meta.csv --threshold 0.2+```+++Each video is classified according to 8 categories:+ `pan_right,+ pan_left,+ tilt_up,+ tilt_down,+ zoom_in,+ zoom_out,+ static,+ unclassified`.+Categories of `tilt`, `pan` and `zoom` can overlap with each other.+++diff --git a/tools/caption/camera_motion/__init__.py b/tools/caption/camera_motion/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/tools/caption/camera_motion/camera_motion.py b/tools/caption/camera_motion/camera_motion.pynew file mode 100644index 0000000..21ac818--- /dev/null+++ b/tools/caption/camera_motion/camera_motion.py@@ -0,0 +1,150 @@+import torch+import os+import numpy as np+from tqdm import tqdm++from .visualizer import Visualizer+from .utils import load_video+++def transform(vector):+ x = np.mean([item[0] for item in vector])+ y = np.mean([item[1] for item in vector])+ return [x, y]+++class CameraPredict:+ def __init__(self, device, submodules_list, factor=0.25):+ self.device = device+ self.grid_size = 10+ self.factor = factor+ try:+ self.model = torch.hub.load(submodules_list["repo"], submodules_list["model"]).to(self.device)+ except:+ # workaround for CERTIFICATE_VERIFY_FAILED (see: https://github.com/pytorch/pytorch/issues/33288#issuecomment-954160699)+ import ssl+ ssl._create_default_https_context = ssl._create_unverified_context+ self.model = torch.hub.load(submodules_list["repo"], submodules_list["model"]).to(self.device)++ def infer(self, video_path, save_video=False, save_dir="./saved_videos"):+ # load video+ video = load_video(video_path, return_tensor=False)+ # set scale+ height, width = video.shape[1], video.shape[2]+ self.scale = min(height, width)+ video = torch.from_numpy(video).permute(0, 3, 1, 2)[None].float().to(self.device) # B T C H W+ pred_tracks, pred_visibility = self.model(video, grid_size=self.grid_size) # B T N 2, B T N 1++ if save_video:+ video_name = os.path.basename(video_path)[:-4]+ vis = Visualizer(save_dir=save_dir, pad_value=120, linewidth=3)+ vis.visualize(video, pred_tracks, pred_visibility, filename=video_name)++ return pred_tracks[0].long().detach().cpu().numpy()++ def transform_class(self, vector, min_reso): # 768*0.05+ scale = min_reso * self.factor+ x, y = vector+ direction = []+ if x > scale:+ direction.append("right")+ elif x < -scale:+ direction.append("left")++ if y > scale:+ direction.append("down")+ elif y < -scale:+ direction.append("up")++ return direction if direction else ["static"]++ def get_edge_point(self, track):+ middle = self.grid_size // 2+ top = [list(track[0, i, :]) for i in range(middle-2, middle+2)]+ down = [list(track[self.grid_size-1, i, :]) for i in range(middle-2, middle+2)]+ left = [list(track[i, 0, :]) for i in range(middle-2, middle+2)]+ right = [list(track[i, self.grid_size-1, :]) for i in range(middle-2, middle+2)]++ return top, down, left, right+++ def get_edge_direction(self, track1, track2):+ edge_points1 = self.get_edge_point(track1)+ edge_points2 = self.get_edge_point(track2)++ vector_results = []+ for points1, points2 in zip(edge_points1, edge_points2):+ vectors = [[end[0]-start[0], end[1]-start[1]] for start, end in zip(points1, points2)]+ vector_results.append(vectors)+ vector_results = list(map(transform, vector_results))+ class_results = [self.transform_class(vector, min_reso=self.scale) for vector in vector_results]++ return class_results+++ def classify_top_down(self, top, down):+ results = []+ classes = [f"{item_t}_{item_d}" for item_t in top for item_d in down]++ results_mapping = {+ "left_left": "pan_right",+ "right_right": "pan_left",+ "down_down": "tilt_up",+ "up_up": "tilt_down",+ "up_down": "zoom_in",+ "down_up": "zoom_out",+ "static_static": "static"+ }+ results = [results_mapping.get(cls) for cls in classes if cls in results_mapping]+ return results if results else ["None"]+++ def classify_left_right(self, left, right):+ results = []+ classes = [f"{item_l}_{item_r}" for item_l in left for item_r in right]+ results_mapping = {+ "left_left": "pan_right",+ "right_right": "pan_left",+ "down_down": "tilt_up",+ "up_up": "tilt_down",+ "left_right": "zoom_in",+ "right_left": "zoom_out",+ "static_static": "static"+ }+ results = [results_mapping.get(cls) for cls in classes if cls in results_mapping]+ return results if results else ["None"]+++ def camera_classify(self, track1, track2):+ top, down, left, right = self.get_edge_direction(track1, track2)++ top_results = self.classify_top_down(top, down)+ left_results = self.classify_left_right(left, right)++ results = list(set(top_results+left_results))+ if "None" in results and len(results)>1:+ results.remove("None")+ if "static" in results and len(results)>1:+ results.remove("static")+ if len(results) == 1 and results[0] == "None": # Tom added this to deal with edge cases+ results = ["Undetermined"]+ return results+++ def predict(self, video_path):+ pred_track = self.infer(video_path)+ track1 = pred_track[0].reshape((self.grid_size, self.grid_size, 2))+ track2 = pred_track[-1].reshape((self.grid_size, self.grid_size, 2))+ results = self.camera_classify(track1, track2)+ return results+++def compute_camera_motion(device, submodules_dict, video_paths, factor):+ camera = CameraPredict(device, submodules_dict, factor)+ # predict_results = camera.predict(video_path)+ # return predict_results+ all_predictions = []+ for video_path in video_paths:+ camera_motion_types = camera.predict(video_path)+ all_predictions.append('+'.join(camera_motion_types))+ return all_predictionsdiff --git a/tools/caption/camera_motion/detect.py b/tools/caption/camera_motion/detect.pynew file mode 100644index 0000000..7f41b11--- /dev/null+++ b/tools/caption/camera_motion/detect.py@@ -0,0 +1,30 @@+# Originally developed by https://github.com/Vchitect/VBench based on https://github.com/facebookresearch/co-tracker.++import argparse+from .camera_motion import compute_camera_motion+import pandas as pd+from typing import List+import ptvsd+++def process(paths: List[str], threshold: float) -> List[str]:+ device = "cuda"+ submodules = {'repo': 'facebookresearch/co-tracker', 'model': 'cotracker2'}+ camera_motion_types = compute_camera_motion(device, submodules, paths, factor=threshold)+ return camera_motion_types+++def main(args):+ output_file = args.input.replace(".csv", "_cmotion.csv")+ data = pd.read_csv(args.input)+ data["cmotion"] = process(data["path"], args.threshold)+ data.to_csv(output_file, index=False)+ print(f"Output saved to {output_file}")+++if __name__ == "__main__":+ parser = argparse.ArgumentParser()+ parser.add_argument("input", type=str)+ parser.add_argument("--threshold", type=float, default=0.25)+ args = parser.parse_args()+ main(args)diff --git a/tools/caption/camera_motion/requirements.txt b/tools/caption/camera_motion/requirements.txtnew file mode 100644index 0000000..ce5c4bd--- /dev/null+++ b/tools/caption/camera_motion/requirements.txt@@ -0,0 +1,3 @@+decord+ptvsd+imageio-ffmpeg\ No newline at end of filediff --git a/tools/caption/camera_motion/utils.py b/tools/caption/camera_motion/utils.pynew file mode 100755index 0000000..e05fadd--- /dev/null+++ b/tools/caption/camera_motion/utils.py@@ -0,0 +1,112 @@+import numpy as np+import torch+from PIL import Image, ImageSequence+from decord import VideoReader+++def get_frame_indices(num_frames, vlen, sample='rand', fix_start=None, input_fps=1, max_num_frames=-1):+ if sample in ["rand", "middle"]: # uniform sampling+ acc_samples = min(num_frames, vlen)+ # split the video into `acc_samples` intervals, and sample from each interval.+ intervals = np.linspace(start=0, stop=vlen, num=acc_samples + 1).astype(int)+ ranges = []+ for idx, interv in enumerate(intervals[:-1]):+ ranges.append((interv, intervals[idx + 1] - 1))+ if sample == 'rand':+ try:+ frame_indices = [random.choice(range(x[0], x[1])) for x in ranges]+ except:+ frame_indices = np.random.permutation(vlen)[:acc_samples]+ frame_indices.sort()+ frame_indices = list(frame_indices)+ elif fix_start is not None:+ frame_indices = [x[0] + fix_start for x in ranges]+ elif sample == 'middle':+ frame_indices = [(x[0] + x[1]) // 2 for x in ranges]+ else:+ raise NotImplementedError++ if len(frame_indices) < num_frames: # padded with last frame+ padded_frame_indices = [frame_indices[-1]] * num_frames+ padded_frame_indices[:len(frame_indices)] = frame_indices+ frame_indices = padded_frame_indices+ elif "fps" in sample: # fps0.5, sequentially sample frames at 0.5 fps+ output_fps = float(sample[3:])+ duration = float(vlen) / input_fps+ delta = 1 / output_fps # gap between frames, this is also the clip length each frame represents+ frame_seconds = np.arange(0 + delta / 2, duration + delta / 2, delta)+ frame_indices = np.around(frame_seconds * input_fps).astype(int)+ frame_indices = [e for e in frame_indices if e < vlen]+ if max_num_frames > 0 and len(frame_indices) > max_num_frames:+ frame_indices = frame_indices[:max_num_frames]+ # frame_indices = np.linspace(0 + delta / 2, duration + delta / 2, endpoint=False, num=max_num_frames)+ else:+ raise ValueError+ return frame_indices++def load_video(video_path, data_transform=None, num_frames=None, return_tensor=True, width=None, height=None):+ """+ Load a video from a given path and apply optional data transformations.++ The function supports loading video in GIF (.gif), PNG (.png), and MP4 (.mp4) formats.+ Depending on the format, it processes and extracts frames accordingly.++ Parameters:+ - video_path (str): The file path to the video or image to be loaded.+ - data_transform (callable, optional): A function that applies transformations to the video data.++ Returns:+ - frames (torch.Tensor): A tensor containing the video frames with shape (T, C, H, W),+ where T is the number of frames, C is the number of channels, H is the height, and W is the width.++ Raises:+ - NotImplementedError: If the video format is not supported.++ The function first determines the format of the video file by its extension.+ For GIFs, it iterates over each frame and converts them to RGB.+ For PNGs, it reads the single frame, converts it to RGB.+ For MP4s, it reads the frames using the VideoReader class and converts them to NumPy arrays.+ If a data_transform is provided, it is applied to the buffer before converting it to a tensor.+ Finally, the tensor is permuted to match the expected (T, C, H, W) format.+ """+ if video_path.endswith('.gif'):+ frame_ls = []+ img = Image.open(video_path)+ for frame in ImageSequence.Iterator(img):+ frame = frame.convert('RGB')+ frame = np.array(frame).astype(np.uint8)+ frame_ls.append(frame)+ buffer = np.array(frame_ls).astype(np.uint8)+ elif video_path.endswith('.png'):+ frame = Image.open(video_path)+ frame = frame.convert('RGB')+ frame = np.array(frame).astype(np.uint8)+ frame_ls = [frame]+ buffer = np.array(frame_ls)+ elif video_path.endswith('.mp4'):+ import decord+ decord.bridge.set_bridge('native')+ if width:+ video_reader = VideoReader(video_path, width=width, height=height, num_threads=1)+ else:+ video_reader = VideoReader(video_path, num_threads=1)+ frames = video_reader.get_batch(range(len(video_reader))) # (T, H, W, C), torch.uint8++ buffer = frames.asnumpy().astype(np.uint8)+ else:+ raise NotImplementedError++ frames = buffer+ if num_frames:+ frame_indices = get_frame_indices(+ num_frames, len(frames), sample="middle"+ )+ frames = frames[frame_indices]++ if data_transform:+ frames = data_transform(frames)+ elif return_tensor:+ frames = torch.Tensor(frames)+ frames = frames.permute(0, 3, 1, 2) # (T, C, H, W), torch.uint8++ return framesdiff --git a/tools/caption/camera_motion/visualizer.py b/tools/caption/camera_motion/visualizer.pynew file mode 100644index 0000000..edb14f8--- /dev/null+++ b/tools/caption/camera_motion/visualizer.py@@ -0,0 +1,346 @@+# Copyright (c) Meta Platforms, Inc. and affiliates.+# All rights reserved.++# This source code is licensed under the license found in the cotracker github repo. https://github.com/facebookresearch/co-tracker.+import os+import numpy as np+import imageio+import torch++from matplotlib import cm+import torch.nn.functional as F+import torchvision.transforms as transforms+import matplotlib.pyplot as plt+from PIL import Image, ImageDraw+++def read_video_from_path(path):+ try:+ reader = imageio.get_reader(path)+ except Exception as e:+ print("Error opening video file: ", e)+ return None+ frames = []+ for i, im in enumerate(reader):+ frames.append(np.array(im))+ return np.stack(frames)+++def draw_circle(rgb, coord, radius, color=(255, 0, 0), visible=True):+ # Create a draw object+ draw = ImageDraw.Draw(rgb)+ # Calculate the bounding box of the circle+ left_up_point = (coord[0] - radius, coord[1] - radius)+ right_down_point = (coord[0] + radius, coord[1] + radius)+ # Draw the circle+ draw.ellipse(+ [left_up_point, right_down_point],+ fill=tuple(color) if visible else None,+ outline=tuple(color),+ )+ return rgb+++def draw_line(rgb, coord_y, coord_x, color, linewidth):+ draw = ImageDraw.Draw(rgb)+ draw.line(+ (coord_y[0], coord_y[1], coord_x[0], coord_x[1]),+ fill=tuple(color),+ width=linewidth,+ )+ return rgb+++def add_weighted(rgb, alpha, original, beta, gamma):+ return (rgb * alpha + original * beta + gamma).astype("uint8")+++class Visualizer:+ def __init__(+ self,+ save_dir: str = "./results",+ grayscale: bool = False,+ pad_value: int = 0,+ fps: int = 10,+ mode: str = "rainbow", # 'cool', 'optical_flow'+ linewidth: int = 2,+ show_first_frame: int = 10,+ tracks_leave_trace: int = 0, # -1 for infinite+ ):+ self.mode = mode+ self.save_dir = save_dir+ if mode == "rainbow":+ self.color_map = cm.get_cmap("gist_rainbow")+ elif mode == "cool":+ self.color_map = cm.get_cmap(mode)+ self.show_first_frame = show_first_frame+ self.grayscale = grayscale+ self.tracks_leave_trace = tracks_leave_trace+ self.pad_value = pad_value+ self.linewidth = linewidth+ self.fps = fps++ def visualize(+ self,+ video: torch.Tensor, # (B,T,C,H,W)+ tracks: torch.Tensor, # (B,T,N,2)+ visibility: torch.Tensor = None, # (B, T, N, 1) bool+ gt_tracks: torch.Tensor = None, # (B,T,N,2)+ segm_mask: torch.Tensor = None, # (B,1,H,W)+ filename: str = "video",+ writer=None, # tensorboard Summary Writer, used for visualization during training+ step: int = 0,+ query_frame: int = 0,+ save_video: bool = True,+ compensate_for_camera_motion: bool = False,+ ):+ if compensate_for_camera_motion:+ assert segm_mask is not None+ if segm_mask is not None:+ coords = tracks[0, query_frame].round().long()+ segm_mask = segm_mask[0, query_frame][coords[:, 1], coords[:, 0]].long()++ video = F.pad(+ video,+ (self.pad_value, self.pad_value, self.pad_value, self.pad_value),+ "constant",+ 255,+ )+ print("video shape after pad is: ", video.shape)+ tracks = tracks + self.pad_value++ print(tracks)+ print("tracks shape after pad is: ", tracks.shape)++ if self.grayscale:+ transform = transforms.Grayscale()+ video = transform(video)+ video = video.repeat(1, 1, 3, 1, 1)++ res_video = self.draw_tracks_on_video(+ video=video,+ tracks=tracks,+ visibility=visibility,+ segm_mask=segm_mask,+ gt_tracks=gt_tracks,+ query_frame=query_frame,+ compensate_for_camera_motion=compensate_for_camera_motion,+ )+ if save_video:+ self.save_video(res_video, filename=filename, writer=writer, step=step)+ return res_video++ def save_video(self, video, filename, writer=None, step=0):+ if writer is not None:+ writer.add_video(+ filename,+ video.to(torch.uint8),+ global_step=step,+ fps=self.fps,+ )+ else:+ os.makedirs(self.save_dir, exist_ok=True)+ wide_list = list(video.unbind(1))+ wide_list = [wide[0].permute(1, 2, 0).cpu().numpy() for wide in wide_list]++ # Prepare the video file path+ save_path = os.path.join(self.save_dir, f"{filename}.mp4")++ # Create a writer object+ video_writer = imageio.get_writer(save_path, fps=self.fps)++ # Write frames to the video file+ for frame in wide_list[2:-1]:+ video_writer.append_data(frame)++ video_writer.close()++ print(f"Video saved to {save_path}")++ def draw_tracks_on_video(+ self,+ video: torch.Tensor,+ tracks: torch.Tensor,+ visibility: torch.Tensor = None,+ segm_mask: torch.Tensor = None,+ gt_tracks=None,+ query_frame: int = 0,+ compensate_for_camera_motion=False,+ ):+ B, T, C, H, W = video.shape+ _, _, N, D = tracks.shape++ assert D == 2+ assert C == 3+ video = video[0].permute(0, 2, 3, 1).byte().detach().cpu().numpy() # S, H, W, C+ tracks = tracks[0].long().detach().cpu().numpy() # S, N, 2+ if gt_tracks is not None:+ gt_tracks = gt_tracks[0].detach().cpu().numpy()++ res_video = []++ # process input video+ for rgb in video:+ res_video.append(rgb.copy())+ vector_colors = np.zeros((T, N, 3))++ if self.mode == "optical_flow":+ import flow_vis++ vector_colors = flow_vis.flow_to_color(tracks - tracks[query_frame][None])+ elif segm_mask is None:+ if self.mode == "rainbow":+ y_min, y_max = (+ tracks[query_frame, :, 1].min(),+ tracks[query_frame, :, 1].max(),+ )+ norm = plt.Normalize(y_min, y_max)+ for n in range(N):+ color = self.color_map(norm(tracks[query_frame, n, 1]))+ color = np.array(color[:3])[None] * 255+ vector_colors[:, n] = np.repeat(color, T, axis=0)+ else:+ # color changes with time+ for t in range(T):+ color = np.array(self.color_map(t / T)[:3])[None] * 255+ vector_colors[t] = np.repeat(color, N, axis=0)+ else:+ if self.mode == "rainbow":+ vector_colors[:, segm_mask <= 0, :] = 255++ y_min, y_max = (+ tracks[0, segm_mask > 0, 1].min(),+ tracks[0, segm_mask > 0, 1].max(),+ )+ norm = plt.Normalize(y_min, y_max)+ for n in range(N):+ if segm_mask[n] > 0:+ color = self.color_map(norm(tracks[0, n, 1]))+ color = np.array(color[:3])[None] * 255+ vector_colors[:, n] = np.repeat(color, T, axis=0)++ else:+ # color changes with segm class+ segm_mask = segm_mask.cpu()+ color = np.zeros((segm_mask.shape[0], 3), dtype=np.float32)+ color[segm_mask > 0] = np.array(self.color_map(1.0)[:3]) * 255.0+ color[segm_mask <= 0] = np.array(self.color_map(0.0)[:3]) * 255.0+ vector_colors = np.repeat(color[None], T, axis=0)++ # draw tracks+ if self.tracks_leave_trace != 0:+ for t in range(query_frame + 1, T):+ first_ind = (+ max(0, t - self.tracks_leave_trace) if self.tracks_leave_trace >= 0 else 0+ )+ curr_tracks = tracks[first_ind : t + 1]+ curr_colors = vector_colors[first_ind : t + 1]+ if compensate_for_camera_motion:+ diff = (+ tracks[first_ind : t + 1, segm_mask <= 0]+ - tracks[t : t + 1, segm_mask <= 0]+ ).mean(1)[:, None]++ curr_tracks = curr_tracks - diff+ curr_tracks = curr_tracks[:, segm_mask > 0]+ curr_colors = curr_colors[:, segm_mask > 0]++ res_video[t] = self._draw_pred_tracks(+ res_video[t],+ curr_tracks,+ curr_colors,+ )+ if gt_tracks is not None:+ res_video[t] = self._draw_gt_tracks(res_video[t], gt_tracks[first_ind : t + 1])++ # draw points+ for t in range(query_frame, T):+ img = Image.fromarray(np.uint8(res_video[t]))+ for i in range(N):+ coord = (tracks[t, i, 0], tracks[t, i, 1])+ visibile = True+ if visibility is not None:+ visibile = visibility[0, t, i]+ if coord[0] != 0 and coord[1] != 0:+ if not compensate_for_camera_motion or (+ compensate_for_camera_motion and segm_mask[i] > 0+ ):+ img = draw_circle(+ img,+ coord=coord,+ radius=int(self.linewidth * 2),+ color=vector_colors[t, i].astype(int),+ visible=visibile,+ )+ res_video[t] = np.array(img)++ # construct the final rgb sequence+ if self.show_first_frame > 0:+ res_video = [res_video[0]] * self.show_first_frame + res_video[1:]+ return torch.from_numpy(np.stack(res_video)).permute(0, 3, 1, 2)[None].byte()++ def _draw_pred_tracks(+ self,+ rgb: np.ndarray, # H x W x 3+ tracks: np.ndarray, # T x 2+ vector_colors: np.ndarray,+ alpha: float = 0.5,+ ):+ T, N, _ = tracks.shape+ rgb = Image.fromarray(np.uint8(rgb))+ for s in range(T - 1):+ vector_color = vector_colors[s]+ original = rgb.copy()+ alpha = (s / T) ** 2+ for i in range(N):+ coord_y = (int(tracks[s, i, 0]), int(tracks[s, i, 1]))+ coord_x = (int(tracks[s + 1, i, 0]), int(tracks[s + 1, i, 1]))+ if coord_y[0] != 0 and coord_y[1] != 0:+ rgb = draw_line(+ rgb,+ coord_y,+ coord_x,+ vector_color[i].astype(int),+ self.linewidth,+ )+ if self.tracks_leave_trace > 0:+ rgb = Image.fromarray(+ np.uint8(add_weighted(np.array(rgb), alpha, np.array(original), 1 - alpha, 0))+ )+ rgb = np.array(rgb)+ return rgb++ def _draw_gt_tracks(+ self,+ rgb: np.ndarray, # H x W x 3,+ gt_tracks: np.ndarray, # T x 2+ ):+ T, N, _ = gt_tracks.shape+ color = np.array((211, 0, 0))+ rgb = Image.fromarray(np.uint8(rgb))+ for t in range(T):+ for i in range(N):+ gt_tracks = gt_tracks[t][i]+ # draw a red cross+ if gt_tracks[0] > 0 and gt_tracks[1] > 0:+ length = self.linewidth * 3+ coord_y = (int(gt_tracks[0]) + length, int(gt_tracks[1]) + length)+ coord_x = (int(gt_tracks[0]) - length, int(gt_tracks[1]) - length)+ rgb = draw_line(+ rgb,+ coord_y,+ coord_x,+ color,+ self.linewidth,+ )+ coord_y = (int(gt_tracks[0]) - length, int(gt_tracks[1]) + length)+ coord_x = (int(gt_tracks[0]) + length, int(gt_tracks[1]) - length)+ rgb = draw_line(+ rgb,+ coord_y,+ coord_x,+ color,+ self.linewidth,+ )+ rgb = np.array(rgb)+ return rgb