Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

new camera motion detection (#65)

Tom Young335e8a3eedbc
diff --git a/tools/caption/README.md b/tools/caption/README.md
index b30856c..da55638 100644
--- a/tools/caption/README.md
+++ b/tools/caption/README.md
@@ -93,3 +93,35 @@ python -m tools.caption.caption_gpt4 DATA.csv --key $OPENAI_API_KEY
```
The cost is approximately $0.01 per video (3 frames per video).
+
+## Camera Motion Detection
+
+Install additional required packages: `tools/caption/camera_motion/requirements.txt`.
+
+Run the following command to classify camera motion:
+
+```bash
+# output: meta_cmotion.csv
+python -m tools.caption.camera_motion.detect tools/caption/camera_motion/meta.csv
+```
+
+You may additionally specify `threshold` to indicate how "sensitive" the detection should be as below. For example `threshold = 0.2` means that the video is only counted as `tilt_up` when the pixels moved down by `>20%` of video height between the starting and ending frames.
+```bash
+# output: meta_cmotion.csv
+python -m tools.caption.camera_motion.detect tools/caption/camera_motion/meta.csv --threshold 0.2
+```
+
+
+Each video is classified according to 8 categories:
+ `pan_right,
+ pan_left,
+ tilt_up,
+ tilt_down,
+ zoom_in,
+ zoom_out,
+ static,
+ unclassified`.
+Categories of `tilt`, `pan` and `zoom` can overlap with each other.
+
+
+
diff --git a/tools/caption/camera_motion/__init__.py b/tools/caption/camera_motion/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tools/caption/camera_motion/camera_motion.py b/tools/caption/camera_motion/camera_motion.py
new file mode 100644
index 0000000..21ac818
--- /dev/null
+++ b/tools/caption/camera_motion/camera_motion.py
@@ -0,0 +1,150 @@
+import torch
+import os
+import numpy as np
+from tqdm import tqdm
+
+from .visualizer import Visualizer
+from .utils import load_video
+
+
+def transform(vector):
+ x = np.mean([item[0] for item in vector])
+ y = np.mean([item[1] for item in vector])
+ return [x, y]
+
+
+class CameraPredict:
+ def __init__(self, device, submodules_list, factor=0.25):
+ self.device = device
+ self.grid_size = 10
+ self.factor = factor
+ try:
+ self.model = torch.hub.load(submodules_list["repo"], submodules_list["model"]).to(self.device)
+ except:
+ # workaround for CERTIFICATE_VERIFY_FAILED (see: https://github.com/pytorch/pytorch/issues/33288#issuecomment-954160699)
+ import ssl
+ ssl._create_default_https_context = ssl._create_unverified_context
+ self.model = torch.hub.load(submodules_list["repo"], submodules_list["model"]).to(self.device)
+
+ def infer(self, video_path, save_video=False, save_dir="./saved_videos"):
+ # load video
+ video = load_video(video_path, return_tensor=False)
+ # set scale
+ height, width = video.shape[1], video.shape[2]
+ self.scale = min(height, width)
+ video = torch.from_numpy(video).permute(0, 3, 1, 2)[None].float().to(self.device) # B T C H W
+ pred_tracks, pred_visibility = self.model(video, grid_size=self.grid_size) # B T N 2, B T N 1
+
+ if save_video:
+ video_name = os.path.basename(video_path)[:-4]
+ vis = Visualizer(save_dir=save_dir, pad_value=120, linewidth=3)
+ vis.visualize(video, pred_tracks, pred_visibility, filename=video_name)
+
+ return pred_tracks[0].long().detach().cpu().numpy()
+
+ def transform_class(self, vector, min_reso): # 768*0.05
+ scale = min_reso * self.factor
+ x, y = vector
+ direction = []
+ if x > scale:
+ direction.append("right")
+ elif x < -scale:
+ direction.append("left")
+
+ if y > scale:
+ direction.append("down")
+ elif y < -scale:
+ direction.append("up")
+
+ return direction if direction else ["static"]
+
+ def get_edge_point(self, track):
+ middle = self.grid_size // 2
+ top = [list(track[0, i, :]) for i in range(middle-2, middle+2)]
+ down = [list(track[self.grid_size-1, i, :]) for i in range(middle-2, middle+2)]
+ left = [list(track[i, 0, :]) for i in range(middle-2, middle+2)]
+ right = [list(track[i, self.grid_size-1, :]) for i in range(middle-2, middle+2)]
+
+ return top, down, left, right
+
+
+ def get_edge_direction(self, track1, track2):
+ edge_points1 = self.get_edge_point(track1)
+ edge_points2 = self.get_edge_point(track2)
+
+ vector_results = []
+ for points1, points2 in zip(edge_points1, edge_points2):
+ vectors = [[end[0]-start[0], end[1]-start[1]] for start, end in zip(points1, points2)]
+ vector_results.append(vectors)
+ vector_results = list(map(transform, vector_results))
+ class_results = [self.transform_class(vector, min_reso=self.scale) for vector in vector_results]
+
+ return class_results
+
+
+ def classify_top_down(self, top, down):
+ results = []
+ classes = [f"{item_t}_{item_d}" for item_t in top for item_d in down]
+
+ results_mapping = {
+ "left_left": "pan_right",
+ "right_right": "pan_left",
+ "down_down": "tilt_up",
+ "up_up": "tilt_down",
+ "up_down": "zoom_in",
+ "down_up": "zoom_out",
+ "static_static": "static"
+ }
+ results = [results_mapping.get(cls) for cls in classes if cls in results_mapping]
+ return results if results else ["None"]
+
+
+ def classify_left_right(self, left, right):
+ results = []
+ classes = [f"{item_l}_{item_r}" for item_l in left for item_r in right]
+ results_mapping = {
+ "left_left": "pan_right",
+ "right_right": "pan_left",
+ "down_down": "tilt_up",
+ "up_up": "tilt_down",
+ "left_right": "zoom_in",
+ "right_left": "zoom_out",
+ "static_static": "static"
+ }
+ results = [results_mapping.get(cls) for cls in classes if cls in results_mapping]
+ return results if results else ["None"]
+
+
+ def camera_classify(self, track1, track2):
+ top, down, left, right = self.get_edge_direction(track1, track2)
+
+ top_results = self.classify_top_down(top, down)
+ left_results = self.classify_left_right(left, right)
+
+ results = list(set(top_results+left_results))
+ if "None" in results and len(results)>1:
+ results.remove("None")
+ if "static" in results and len(results)>1:
+ results.remove("static")
+ if len(results) == 1 and results[0] == "None": # Tom added this to deal with edge cases
+ results = ["Undetermined"]
+ return results
+
+
+ def predict(self, video_path):
+ pred_track = self.infer(video_path)
+ track1 = pred_track[0].reshape((self.grid_size, self.grid_size, 2))
+ track2 = pred_track[-1].reshape((self.grid_size, self.grid_size, 2))
+ results = self.camera_classify(track1, track2)
+ return results
+
+
+def compute_camera_motion(device, submodules_dict, video_paths, factor):
+ camera = CameraPredict(device, submodules_dict, factor)
+ # predict_results = camera.predict(video_path)
+ # return predict_results
+ all_predictions = []
+ for video_path in video_paths:
+ camera_motion_types = camera.predict(video_path)
+ all_predictions.append('+'.join(camera_motion_types))
+ return all_predictions
diff --git a/tools/caption/camera_motion/detect.py b/tools/caption/camera_motion/detect.py
new file mode 100644
index 0000000..7f41b11
--- /dev/null
+++ b/tools/caption/camera_motion/detect.py
@@ -0,0 +1,30 @@
+# Originally developed by https://github.com/Vchitect/VBench based on https://github.com/facebookresearch/co-tracker.
+
+import argparse
+from .camera_motion import compute_camera_motion
+import pandas as pd
+from typing import List
+import ptvsd
+
+
+def process(paths: List[str], threshold: float) -> List[str]:
+ device = "cuda"
+ submodules = {'repo': 'facebookresearch/co-tracker', 'model': 'cotracker2'}
+ camera_motion_types = compute_camera_motion(device, submodules, paths, factor=threshold)
+ return camera_motion_types
+
+
+def main(args):
+ output_file = args.input.replace(".csv", "_cmotion.csv")
+ data = pd.read_csv(args.input)
+ data["cmotion"] = process(data["path"], args.threshold)
+ data.to_csv(output_file, index=False)
+ print(f"Output saved to {output_file}")
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("input", type=str)
+ parser.add_argument("--threshold", type=float, default=0.25)
+ args = parser.parse_args()
+ main(args)
diff --git a/tools/caption/camera_motion/requirements.txt b/tools/caption/camera_motion/requirements.txt
new file mode 100644
index 0000000..ce5c4bd
--- /dev/null
+++ b/tools/caption/camera_motion/requirements.txt
@@ -0,0 +1,3 @@
+decord
+ptvsd
+imageio-ffmpeg
\ No newline at end of file
diff --git a/tools/caption/camera_motion/utils.py b/tools/caption/camera_motion/utils.py
new file mode 100755
index 0000000..e05fadd
--- /dev/null
+++ b/tools/caption/camera_motion/utils.py
@@ -0,0 +1,112 @@
+import numpy as np
+import torch
+from PIL import Image, ImageSequence
+from decord import VideoReader
+
+
+def get_frame_indices(num_frames, vlen, sample='rand', fix_start=None, input_fps=1, max_num_frames=-1):
+ if sample in ["rand", "middle"]: # uniform sampling
+ acc_samples = min(num_frames, vlen)
+ # split the video into `acc_samples` intervals, and sample from each interval.
+ intervals = np.linspace(start=0, stop=vlen, num=acc_samples + 1).astype(int)
+ ranges = []
+ for idx, interv in enumerate(intervals[:-1]):
+ ranges.append((interv, intervals[idx + 1] - 1))
+ if sample == 'rand':
+ try:
+ frame_indices = [random.choice(range(x[0], x[1])) for x in ranges]
+ except:
+ frame_indices = np.random.permutation(vlen)[:acc_samples]
+ frame_indices.sort()
+ frame_indices = list(frame_indices)
+ elif fix_start is not None:
+ frame_indices = [x[0] + fix_start for x in ranges]
+ elif sample == 'middle':
+ frame_indices = [(x[0] + x[1]) // 2 for x in ranges]
+ else:
+ raise NotImplementedError
+
+ if len(frame_indices) < num_frames: # padded with last frame
+ padded_frame_indices = [frame_indices[-1]] * num_frames
+ padded_frame_indices[:len(frame_indices)] = frame_indices
+ frame_indices = padded_frame_indices
+ elif "fps" in sample: # fps0.5, sequentially sample frames at 0.5 fps
+ output_fps = float(sample[3:])
+ duration = float(vlen) / input_fps
+ delta = 1 / output_fps # gap between frames, this is also the clip length each frame represents
+ frame_seconds = np.arange(0 + delta / 2, duration + delta / 2, delta)
+ frame_indices = np.around(frame_seconds * input_fps).astype(int)
+ frame_indices = [e for e in frame_indices if e < vlen]
+ if max_num_frames > 0 and len(frame_indices) > max_num_frames:
+ frame_indices = frame_indices[:max_num_frames]
+ # frame_indices = np.linspace(0 + delta / 2, duration + delta / 2, endpoint=False, num=max_num_frames)
+ else:
+ raise ValueError
+ return frame_indices
+
+def load_video(video_path, data_transform=None, num_frames=None, return_tensor=True, width=None, height=None):
+ """
+ Load a video from a given path and apply optional data transformations.
+
+ The function supports loading video in GIF (.gif), PNG (.png), and MP4 (.mp4) formats.
+ Depending on the format, it processes and extracts frames accordingly.
+
+ Parameters:
+ - video_path (str): The file path to the video or image to be loaded.
+ - data_transform (callable, optional): A function that applies transformations to the video data.
+
+ Returns:
+ - frames (torch.Tensor): A tensor containing the video frames with shape (T, C, H, W),
+ where T is the number of frames, C is the number of channels, H is the height, and W is the width.
+
+ Raises:
+ - NotImplementedError: If the video format is not supported.
+
+ The function first determines the format of the video file by its extension.
+ For GIFs, it iterates over each frame and converts them to RGB.
+ For PNGs, it reads the single frame, converts it to RGB.
+ For MP4s, it reads the frames using the VideoReader class and converts them to NumPy arrays.
+ If a data_transform is provided, it is applied to the buffer before converting it to a tensor.
+ Finally, the tensor is permuted to match the expected (T, C, H, W) format.
+ """
+ if video_path.endswith('.gif'):
+ frame_ls = []
+ img = Image.open(video_path)
+ for frame in ImageSequence.Iterator(img):
+ frame = frame.convert('RGB')
+ frame = np.array(frame).astype(np.uint8)
+ frame_ls.append(frame)
+ buffer = np.array(frame_ls).astype(np.uint8)
+ elif video_path.endswith('.png'):
+ frame = Image.open(video_path)
+ frame = frame.convert('RGB')
+ frame = np.array(frame).astype(np.uint8)
+ frame_ls = [frame]
+ buffer = np.array(frame_ls)
+ elif video_path.endswith('.mp4'):
+ import decord
+ decord.bridge.set_bridge('native')
+ if width:
+ video_reader = VideoReader(video_path, width=width, height=height, num_threads=1)
+ else:
+ video_reader = VideoReader(video_path, num_threads=1)
+ frames = video_reader.get_batch(range(len(video_reader))) # (T, H, W, C), torch.uint8
+
+ buffer = frames.asnumpy().astype(np.uint8)
+ else:
+ raise NotImplementedError
+
+ frames = buffer
+ if num_frames:
+ frame_indices = get_frame_indices(
+ num_frames, len(frames), sample="middle"
+ )
+ frames = frames[frame_indices]
+
+ if data_transform:
+ frames = data_transform(frames)
+ elif return_tensor:
+ frames = torch.Tensor(frames)
+ frames = frames.permute(0, 3, 1, 2) # (T, C, H, W), torch.uint8
+
+ return frames
diff --git a/tools/caption/camera_motion/visualizer.py b/tools/caption/camera_motion/visualizer.py
new file mode 100644
index 0000000..edb14f8
--- /dev/null
+++ b/tools/caption/camera_motion/visualizer.py
@@ -0,0 +1,346 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+
+# This source code is licensed under the license found in the cotracker github repo. https://github.com/facebookresearch/co-tracker.
+import os
+import numpy as np
+import imageio
+import torch
+
+from matplotlib import cm
+import torch.nn.functional as F
+import torchvision.transforms as transforms
+import matplotlib.pyplot as plt
+from PIL import Image, ImageDraw
+
+
+def read_video_from_path(path):
+ try:
+ reader = imageio.get_reader(path)
+ except Exception as e:
+ print("Error opening video file: ", e)
+ return None
+ frames = []
+ for i, im in enumerate(reader):
+ frames.append(np.array(im))
+ return np.stack(frames)
+
+
+def draw_circle(rgb, coord, radius, color=(255, 0, 0), visible=True):
+ # Create a draw object
+ draw = ImageDraw.Draw(rgb)
+ # Calculate the bounding box of the circle
+ left_up_point = (coord[0] - radius, coord[1] - radius)
+ right_down_point = (coord[0] + radius, coord[1] + radius)
+ # Draw the circle
+ draw.ellipse(
+ [left_up_point, right_down_point],
+ fill=tuple(color) if visible else None,
+ outline=tuple(color),
+ )
+ return rgb
+
+
+def draw_line(rgb, coord_y, coord_x, color, linewidth):
+ draw = ImageDraw.Draw(rgb)
+ draw.line(
+ (coord_y[0], coord_y[1], coord_x[0], coord_x[1]),
+ fill=tuple(color),
+ width=linewidth,
+ )
+ return rgb
+
+
+def add_weighted(rgb, alpha, original, beta, gamma):
+ return (rgb * alpha + original * beta + gamma).astype("uint8")
+
+
+class Visualizer:
+ def __init__(
+ self,
+ save_dir: str = "./results",
+ grayscale: bool = False,
+ pad_value: int = 0,
+ fps: int = 10,
+ mode: str = "rainbow", # 'cool', 'optical_flow'
+ linewidth: int = 2,
+ show_first_frame: int = 10,
+ tracks_leave_trace: int = 0, # -1 for infinite
+ ):
+ self.mode = mode
+ self.save_dir = save_dir
+ if mode == "rainbow":
+ self.color_map = cm.get_cmap("gist_rainbow")
+ elif mode == "cool":
+ self.color_map = cm.get_cmap(mode)
+ self.show_first_frame = show_first_frame
+ self.grayscale = grayscale
+ self.tracks_leave_trace = tracks_leave_trace
+ self.pad_value = pad_value
+ self.linewidth = linewidth
+ self.fps = fps
+
+ def visualize(
+ self,
+ video: torch.Tensor, # (B,T,C,H,W)
+ tracks: torch.Tensor, # (B,T,N,2)
+ visibility: torch.Tensor = None, # (B, T, N, 1) bool
+ gt_tracks: torch.Tensor = None, # (B,T,N,2)
+ segm_mask: torch.Tensor = None, # (B,1,H,W)
+ filename: str = "video",
+ writer=None, # tensorboard Summary Writer, used for visualization during training
+ step: int = 0,
+ query_frame: int = 0,
+ save_video: bool = True,
+ compensate_for_camera_motion: bool = False,
+ ):
+ if compensate_for_camera_motion:
+ assert segm_mask is not None
+ if segm_mask is not None:
+ coords = tracks[0, query_frame].round().long()
+ segm_mask = segm_mask[0, query_frame][coords[:, 1], coords[:, 0]].long()
+
+ video = F.pad(
+ video,
+ (self.pad_value, self.pad_value, self.pad_value, self.pad_value),
+ "constant",
+ 255,
+ )
+ print("video shape after pad is: ", video.shape)
+ tracks = tracks + self.pad_value
+
+ print(tracks)
+ print("tracks shape after pad is: ", tracks.shape)
+
+ if self.grayscale:
+ transform = transforms.Grayscale()
+ video = transform(video)
+ video = video.repeat(1, 1, 3, 1, 1)
+
+ res_video = self.draw_tracks_on_video(
+ video=video,
+ tracks=tracks,
+ visibility=visibility,
+ segm_mask=segm_mask,
+ gt_tracks=gt_tracks,
+ query_frame=query_frame,
+ compensate_for_camera_motion=compensate_for_camera_motion,
+ )
+ if save_video:
+ self.save_video(res_video, filename=filename, writer=writer, step=step)
+ return res_video
+
+ def save_video(self, video, filename, writer=None, step=0):
+ if writer is not None:
+ writer.add_video(
+ filename,
+ video.to(torch.uint8),
+ global_step=step,
+ fps=self.fps,
+ )
+ else:
+ os.makedirs(self.save_dir, exist_ok=True)
+ wide_list = list(video.unbind(1))
+ wide_list = [wide[0].permute(1, 2, 0).cpu().numpy() for wide in wide_list]
+
+ # Prepare the video file path
+ save_path = os.path.join(self.save_dir, f"{filename}.mp4")
+
+ # Create a writer object
+ video_writer = imageio.get_writer(save_path, fps=self.fps)
+
+ # Write frames to the video file
+ for frame in wide_list[2:-1]:
+ video_writer.append_data(frame)
+
+ video_writer.close()
+
+ print(f"Video saved to {save_path}")
+
+ def draw_tracks_on_video(
+ self,
+ video: torch.Tensor,
+ tracks: torch.Tensor,
+ visibility: torch.Tensor = None,
+ segm_mask: torch.Tensor = None,
+ gt_tracks=None,
+ query_frame: int = 0,
+ compensate_for_camera_motion=False,
+ ):
+ B, T, C, H, W = video.shape
+ _, _, N, D = tracks.shape
+
+ assert D == 2
+ assert C == 3
+ video = video[0].permute(0, 2, 3, 1).byte().detach().cpu().numpy() # S, H, W, C
+ tracks = tracks[0].long().detach().cpu().numpy() # S, N, 2
+ if gt_tracks is not None:
+ gt_tracks = gt_tracks[0].detach().cpu().numpy()
+
+ res_video = []
+
+ # process input video
+ for rgb in video:
+ res_video.append(rgb.copy())
+ vector_colors = np.zeros((T, N, 3))
+
+ if self.mode == "optical_flow":
+ import flow_vis
+
+ vector_colors = flow_vis.flow_to_color(tracks - tracks[query_frame][None])
+ elif segm_mask is None:
+ if self.mode == "rainbow":
+ y_min, y_max = (
+ tracks[query_frame, :, 1].min(),
+ tracks[query_frame, :, 1].max(),
+ )
+ norm = plt.Normalize(y_min, y_max)
+ for n in range(N):
+ color = self.color_map(norm(tracks[query_frame, n, 1]))
+ color = np.array(color[:3])[None] * 255
+ vector_colors[:, n] = np.repeat(color, T, axis=0)
+ else:
+ # color changes with time
+ for t in range(T):
+ color = np.array(self.color_map(t / T)[:3])[None] * 255
+ vector_colors[t] = np.repeat(color, N, axis=0)
+ else:
+ if self.mode == "rainbow":
+ vector_colors[:, segm_mask <= 0, :] = 255
+
+ y_min, y_max = (
+ tracks[0, segm_mask > 0, 1].min(),
+ tracks[0, segm_mask > 0, 1].max(),
+ )
+ norm = plt.Normalize(y_min, y_max)
+ for n in range(N):
+ if segm_mask[n] > 0:
+ color = self.color_map(norm(tracks[0, n, 1]))
+ color = np.array(color[:3])[None] * 255
+ vector_colors[:, n] = np.repeat(color, T, axis=0)
+
+ else:
+ # color changes with segm class
+ segm_mask = segm_mask.cpu()
+ color = np.zeros((segm_mask.shape[0], 3), dtype=np.float32)
+ color[segm_mask > 0] = np.array(self.color_map(1.0)[:3]) * 255.0
+ color[segm_mask <= 0] = np.array(self.color_map(0.0)[:3]) * 255.0
+ vector_colors = np.repeat(color[None], T, axis=0)
+
+ # draw tracks
+ if self.tracks_leave_trace != 0:
+ for t in range(query_frame + 1, T):
+ first_ind = (
+ max(0, t - self.tracks_leave_trace) if self.tracks_leave_trace >= 0 else 0
+ )
+ curr_tracks = tracks[first_ind : t + 1]
+ curr_colors = vector_colors[first_ind : t + 1]
+ if compensate_for_camera_motion:
+ diff = (
+ tracks[first_ind : t + 1, segm_mask <= 0]
+ - tracks[t : t + 1, segm_mask <= 0]
+ ).mean(1)[:, None]
+
+ curr_tracks = curr_tracks - diff
+ curr_tracks = curr_tracks[:, segm_mask > 0]
+ curr_colors = curr_colors[:, segm_mask > 0]
+
+ res_video[t] = self._draw_pred_tracks(
+ res_video[t],
+ curr_tracks,
+ curr_colors,
+ )
+ if gt_tracks is not None:
+ res_video[t] = self._draw_gt_tracks(res_video[t], gt_tracks[first_ind : t + 1])
+
+ # draw points
+ for t in range(query_frame, T):
+ img = Image.fromarray(np.uint8(res_video[t]))
+ for i in range(N):
+ coord = (tracks[t, i, 0], tracks[t, i, 1])
+ visibile = True
+ if visibility is not None:
+ visibile = visibility[0, t, i]
+ if coord[0] != 0 and coord[1] != 0:
+ if not compensate_for_camera_motion or (
+ compensate_for_camera_motion and segm_mask[i] > 0
+ ):
+ img = draw_circle(
+ img,
+ coord=coord,
+ radius=int(self.linewidth * 2),
+ color=vector_colors[t, i].astype(int),
+ visible=visibile,
+ )
+ res_video[t] = np.array(img)
+
+ # construct the final rgb sequence
+ if self.show_first_frame > 0:
+ res_video = [res_video[0]] * self.show_first_frame + res_video[1:]
+ return torch.from_numpy(np.stack(res_video)).permute(0, 3, 1, 2)[None].byte()
+
+ def _draw_pred_tracks(
+ self,
+ rgb: np.ndarray, # H x W x 3
+ tracks: np.ndarray, # T x 2
+ vector_colors: np.ndarray,
+ alpha: float = 0.5,
+ ):
+ T, N, _ = tracks.shape
+ rgb = Image.fromarray(np.uint8(rgb))
+ for s in range(T - 1):
+ vector_color = vector_colors[s]
+ original = rgb.copy()
+ alpha = (s / T) ** 2
+ for i in range(N):
+ coord_y = (int(tracks[s, i, 0]), int(tracks[s, i, 1]))
+ coord_x = (int(tracks[s + 1, i, 0]), int(tracks[s + 1, i, 1]))
+ if coord_y[0] != 0 and coord_y[1] != 0:
+ rgb = draw_line(
+ rgb,
+ coord_y,
+ coord_x,
+ vector_color[i].astype(int),
+ self.linewidth,
+ )
+ if self.tracks_leave_trace > 0:
+ rgb = Image.fromarray(
+ np.uint8(add_weighted(np.array(rgb), alpha, np.array(original), 1 - alpha, 0))
+ )
+ rgb = np.array(rgb)
+ return rgb
+
+ def _draw_gt_tracks(
+ self,
+ rgb: np.ndarray, # H x W x 3,
+ gt_tracks: np.ndarray, # T x 2
+ ):
+ T, N, _ = gt_tracks.shape
+ color = np.array((211, 0, 0))
+ rgb = Image.fromarray(np.uint8(rgb))
+ for t in range(T):
+ for i in range(N):
+ gt_tracks = gt_tracks[t][i]
+ # draw a red cross
+ if gt_tracks[0] > 0 and gt_tracks[1] > 0:
+ length = self.linewidth * 3
+ coord_y = (int(gt_tracks[0]) + length, int(gt_tracks[1]) + length)
+ coord_x = (int(gt_tracks[0]) - length, int(gt_tracks[1]) - length)
+ rgb = draw_line(
+ rgb,
+ coord_y,
+ coord_x,
+ color,
+ self.linewidth,
+ )
+ coord_y = (int(gt_tracks[0]) - length, int(gt_tracks[1]) + length)
+ coord_x = (int(gt_tracks[0]) + length, int(gt_tracks[1]) - length)
+ rgb = draw_line(
+ rgb,
+ coord_y,
+ coord_x,
+ color,
+ self.linewidth,
+ )
+ rgb = np.array(rgb)
+ return rgb