Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[feature] support dp for optical flow scoring (#23)

Hongxin Liu810ebeeb6af7
diff --git a/tools/scoring/README.md b/tools/scoring/README.md
index a229660..1460f3c 100644
--- a/tools/scoring/README.md
+++ b/tools/scoring/README.md
@@ -47,6 +47,8 @@ With `meta.csv` containing the paths to the videos, run the following command:
```bash
python -m tools.scoring.optical_flow.inference /path/to/meta.csv
+# or run in parallel
+torchrun --standalone --nproc_per_node 8 -m tools.scoring.optical_flow.inference_parallel /path/to/meta.csv
```
The output should be `/path/to/meta_flow.csv` with column `flow`. Higher optical flow scores indicate larger movement.
diff --git a/tools/scoring/optical_flow/inference.py b/tools/scoring/optical_flow/inference.py
index d48ecf7..db62b35 100644
--- a/tools/scoring/optical_flow/inference.py
+++ b/tools/scoring/optical_flow/inference.py
@@ -21,7 +21,9 @@ def extract_frames_av(video_path, frame_inds=[0, 10, 20, 30]):
for idx in frame_inds:
if idx >= total_frames:
idx = total_frames - 1
- target_timestamp = int(idx * av.time_base / container.streams.video[0].average_rate)
+ target_timestamp = int(
+ idx * av.time_base / container.streams.video[0].average_rate
+ )
container.seek(target_timestamp)
frame = next(container.decode(video=0)).to_image()
frames.append(frame)
@@ -56,7 +58,9 @@ class VideoTextDataset(torch.utils.data.Dataset):
H, W = images.shape[-2:]
if H > W:
images = rearrange(images, "N C H W -> N C W H")
- images = F.interpolate(images, size=(320, 576), mode="bilinear", align_corners=True)
+ images = F.interpolate(
+ images, size=(320, 576), mode="bilinear", align_corners=True
+ )
return images
@@ -65,6 +69,8 @@ class VideoTextDataset(torch.utils.data.Dataset):
def main():
+ torch.backends.cudnn.deterministic = True
+ torch.backends.cudnn.benchmark = False
parser = argparse.ArgumentParser()
parser.add_argument("meta_path", type=str, help="Path to the input CSV file")
parser.add_argument("--bs", type=int, default=4, help="Batch size")
@@ -86,8 +92,10 @@ def main():
num_transformer_layers=6,
reg_refine=True,
task="flow",
+ ).eval()
+ ckpt = torch.load(
+ "./pretrained_models/unimatch/gmflow-scale2-regrefine6-mixdata-train320x576-4e7b215d.pth"
)
- ckpt = torch.load("./pretrained_models/unimatch/gmflow-scale2-regrefine6-mixdata-train320x576-4e7b215d.pth")
model.load_state_dict(ckpt["model"])
model = model.to(device)
# model = torch.nn.DataParallel(model)
diff --git a/tools/scoring/optical_flow/inference_parallel.py b/tools/scoring/optical_flow/inference_parallel.py
new file mode 100644
index 0000000..c64b840
--- /dev/null
+++ b/tools/scoring/optical_flow/inference_parallel.py
@@ -0,0 +1,179 @@
+import argparse
+import os
+
+import av
+import colossalai
+import numpy as np
+import pandas as pd
+import torch
+import torch.distributed as dist
+import torch.nn.functional as F
+from einops import rearrange
+from torch.utils.data import DataLoader, DistributedSampler
+from tqdm import tqdm
+
+from .unimatch import UniMatch
+
+import decord # isort: skip
+
+
+def extract_frames_av(video_path, frame_inds=[0, 10, 20, 30]):
+ container = av.open(video_path)
+ total_frames = container.streams.video[0].frames
+ frames = []
+ for idx in frame_inds:
+ if idx >= total_frames:
+ idx = total_frames - 1
+ target_timestamp = int(
+ idx * av.time_base / container.streams.video[0].average_rate
+ )
+ container.seek(target_timestamp)
+ frame = next(container.decode(video=0)).to_image()
+ frames.append(frame)
+ return frames
+
+
+def extract_frames(video_path, frame_inds=[0, 10, 20, 30]):
+ container = decord.VideoReader(video_path, num_threads=1)
+ total_frames = len(container)
+ # avg_fps = container.get_avg_fps()
+
+ frame_inds = np.array(frame_inds).astype(np.int32)
+ frame_inds[frame_inds >= total_frames] = total_frames - 1
+ frames = container.get_batch(frame_inds).asnumpy() # [N, H, W, C]
+ return frames
+
+
+def merge_scores(gathered_list: list, meta: pd.DataFrame):
+ # reorder
+ indices_list = list(map(lambda x: x[0], gathered_list))
+ flow_scores_list = list(map(lambda x: x[1], gathered_list))
+ flat_indices = []
+ for x in zip(*indices_list):
+ flat_indices.extend(x)
+ flat_flow_scores = []
+ for x in zip(*flow_scores_list):
+ flat_flow_scores.extend(x)
+ flat_indices = np.array(flat_indices)
+ flat_flow_scores = np.array(flat_flow_scores)
+ # filter duplicates
+ unique_indices, unique_indices_idx = np.unique(flat_indices, return_index=True)
+ meta.loc[unique_indices, "flow"] = flat_flow_scores[unique_indices_idx]
+
+
+class VideoTextDataset(torch.utils.data.Dataset):
+ def __init__(self, meta_path, frame_inds=[0, 10, 20, 30]):
+ self.meta_path = meta_path
+ self.meta = pd.read_csv(meta_path)
+ self.frame_inds = frame_inds
+
+ def __getitem__(self, index):
+ row = self.meta.iloc[index]
+ images = extract_frames(row["path"], frame_inds=self.frame_inds)
+ # images = [pil_to_tensor(x) for x in images] # [C, H, W]
+
+ # transform
+ images = torch.from_numpy(images).float()
+ images = rearrange(images, "N H W C -> N C H W")
+ H, W = images.shape[-2:]
+ if H > W:
+ images = rearrange(images, "N C H W -> N C W H")
+ images = F.interpolate(
+ images, size=(320, 576), mode="bilinear", align_corners=True
+ )
+
+ return images, index
+
+ def __len__(self):
+ return len(self.meta)
+
+
+def main():
+ torch.backends.cudnn.deterministic = True
+ torch.backends.cudnn.benchmark = False
+ colossalai.launch_from_torch({})
+ parser = argparse.ArgumentParser()
+ parser.add_argument("meta_path", type=str, help="Path to the input CSV file")
+ parser.add_argument("--bs", type=int, default=4, help="Batch size")
+ parser.add_argument("--num_workers", type=int, default=16, help="Number of workers")
+ args = parser.parse_args()
+
+ meta_path = args.meta_path
+ wo_ext, ext = os.path.splitext(meta_path)
+ out_path = f"{wo_ext}_flow{ext}"
+
+ # build model
+ device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+ model = UniMatch(
+ feature_channels=128,
+ num_scales=2,
+ upsample_factor=4,
+ num_head=1,
+ ffn_dim_expansion=4,
+ num_transformer_layers=6,
+ reg_refine=True,
+ task="flow",
+ ).eval()
+ ckpt = torch.load(
+ "./pretrained_models/unimatch/gmflow-scale2-regrefine6-mixdata-train320x576-4e7b215d.pth"
+ )
+ model.load_state_dict(ckpt["model"])
+ model = model.to(device)
+ # model = torch.nn.DataParallel(model)
+
+ # build dataset
+ dataset = VideoTextDataset(meta_path=meta_path, frame_inds=[0, 10, 20, 30])
+ dataloader = DataLoader(
+ dataset,
+ batch_size=args.bs,
+ num_workers=args.num_workers,
+ sampler=DistributedSampler(
+ dataset,
+ num_replicas=dist.get_world_size(),
+ rank=dist.get_rank(),
+ shuffle=False,
+ drop_last=False,
+ ),
+ )
+
+ # compute optical flow scores
+ dataset.meta["flow"] = np.nan
+ indices_list = []
+ flow_scores_list = []
+ for images, indices in tqdm(dataloader, disable=dist.get_rank() != 0):
+ images = images.to(device)
+ B = images.shape[0]
+
+ batch_0 = rearrange(images[:, :-1], "B N C H W -> (B N) C H W").contiguous()
+ batch_1 = rearrange(images[:, 1:], "B N C H W -> (B N) C H W").contiguous()
+
+ with torch.no_grad():
+ res = model(
+ batch_0,
+ batch_1,
+ attn_type="swin",
+ attn_splits_list=[2, 8],
+ corr_radius_list=[-1, 4],
+ prop_radius_list=[-1, 1],
+ num_reg_refine=6,
+ task="flow",
+ pred_bidir_flow=False,
+ )
+ flow_maps = res["flow_preds"][-1].cpu() # [B * (N-1), 2, H, W]
+ flow_maps = rearrange(flow_maps, "(B N) C H W -> B N H W C", B=B)
+ flow_scores = flow_maps.abs().mean(dim=[1, 2, 3, 4])
+ flow_scores = flow_scores.tolist()
+
+ indices_list.extend(indices)
+ flow_scores_list.extend(flow_scores)
+
+ gathered_list = [None] * dist.get_world_size()
+ dist.all_gather_object(gathered_list, (indices_list, flow_scores_list))
+ if dist.get_rank() == 0:
+ merge_scores(gathered_list, dataset.meta)
+ dataset.meta.to_csv(out_path, index=False)
+ print(f"New meta with optical flow scores saved to '{out_path}'.")
+
+
+if __name__ == "__main__":
+ main()