Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

Dev/pxy (#49)

xyupeng7ebe6c2d8712
diff --git a/tools/datasets/transform.py b/tools/datasets/transform.py
index e59c316..e377681 100644
--- a/tools/datasets/transform.py
+++ b/tools/datasets/transform.py
@@ -103,7 +103,7 @@ def extract_frames(video_path, input_dir, output, point):
def extract_frames_new(
video_path,
- frame_inds=[0, 10, 20, 30],
+ frame_inds=None,
points=None,
backend='opencv',
return_length=False,
diff --git a/tools/scoring/README.md b/tools/scoring/README.md
index b462783..500d52b 100644
--- a/tools/scoring/README.md
+++ b/tools/scoring/README.md
@@ -51,7 +51,7 @@ wget https://s3.eu-central-1.amazonaws.com/avg-projects/unimatch/pretrained/gmfl
Then run:
```bash
-torchrun --standalone --nproc_per_node 8 tools/scoring/optical_flow/inference_parallel.py /path/to/meta.csv
+torchrun --standalone --nproc_per_node 8 tools/scoring/optical_flow/inference.py /path/to/meta.csv
```
The output should be `/path/to/meta_flow.csv` with column `flow`.
@@ -64,7 +64,7 @@ For videos, we compute the matching score of the middle frame and the caption.
**Make sure** meta files contain the column `text`, which is the caption of the sample. Then run:
```bash
-torchrun --standalone --nproc_per_node 8 tools/scoring/matching/inference_parallel.py /path/to/meta.csv
+torchrun --standalone --nproc_per_node 8 tools/scoring/matching/inference.py /path/to/meta.csv
```
The output should be `/path/to/meta_match.csv` with column `match`. Higher matching scores indicate better image-text/video-text alignment.
diff --git a/tools/scoring/matching/inference.py b/tools/scoring/matching/inference.py
index 0953899..23ddddc 100644
--- a/tools/scoring/matching/inference.py
+++ b/tools/scoring/matching/inference.py
@@ -3,13 +3,18 @@ import os
import av
import clip
+import colossalai
import numpy as np
import pandas as pd
import torch
+import torch.distributed as dist
import torch.nn.functional as F
+from torch.utils.data import DataLoader, DistributedSampler
from torchvision.datasets.folder import pil_loader
from tqdm import tqdm
+from tools.datasets.transform import extract_frames_new
+
IMG_EXTENSIONS = (
".jpg",
".jpeg",
@@ -29,28 +34,6 @@ def is_video(filename):
return ext in VID_EXTENSIONS
-IMG_EXTENSIONS = (".jpg", ".jpeg", ".png", ".ppm", ".bmp", ".pgm", ".tif", ".tiff", ".webp")
-VID_EXTENSIONS = (".mp4", ".avi", ".mov", ".mkv")
-
-
-def is_video(filename):
- ext = os.path.splitext(filename)[-1].lower()
- return ext in VID_EXTENSIONS
-
-
-def extract_frames(video_path, points=[0.5]):
- container = av.open(video_path)
- total_frames = container.streams.video[0].frames
- frames = []
- for point in points:
- target_frame = total_frames * point
- target_timestamp = int((target_frame * av.time_base) / container.streams.video[0].average_rate)
- container.seek(target_timestamp)
- frame = next(container.decode(video=0)).to_image()
- frames.append(frame)
- return frames
-
-
class VideoTextDataset(torch.utils.data.Dataset):
def __init__(self, meta_path, transform):
self.meta_path = meta_path
@@ -62,21 +45,40 @@ class VideoTextDataset(torch.utils.data.Dataset):
path = row["path"]
if is_video(path):
- img = extract_frames(path, points=[0.5])[0]
+ img = extract_frames_new(path, points=[0.5], backend='opencv')[0]
else:
img = pil_loader(path)
+
img = self.transform(img)
text = row["text"]
text = clip.tokenize(text, truncate=True).squeeze()
- return img, text
+ return img, text, index
def __len__(self):
return len(self.meta)
+def merge_scores(gathered_list: list, meta: pd.DataFrame):
+ # reorder
+ indices_list = list(map(lambda x: x[0], gathered_list))
+ scores_list = list(map(lambda x: x[1], gathered_list))
+ flat_indices = []
+ for x in zip(*indices_list):
+ flat_indices.extend(x)
+ flat_scores = []
+ for x in zip(*scores_list):
+ flat_scores.extend(x)
+ flat_indices = np.array(flat_indices)
+ flat_scores = np.array(flat_scores)
+ # filter duplicates
+ unique_indices, unique_indices_idx = np.unique(flat_indices, return_index=True)
+ meta.loc[unique_indices, "match"] = flat_scores[unique_indices_idx]
+
+
def main():
+ colossalai.launch_from_torch({})
parser = argparse.ArgumentParser()
parser.add_argument("meta_path", type=str, help="Path to the input CSV file")
parser.add_argument("--bs", type=int, default=16, help="Batch size")
@@ -91,25 +93,30 @@ def main():
device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
model, preprocess = clip.load("ViT-L/14", device=device)
logit_scale = model.logit_scale.exp().item()
- # model = torch.nn.DataParallel(model)
# build dataset
dataset = VideoTextDataset(meta_path=meta_path, transform=preprocess)
- dataloader = torch.utils.data.DataLoader(
+ dataloader = DataLoader(
dataset,
batch_size=args.bs,
num_workers=args.num_workers,
- shuffle=False,
+ sampler=DistributedSampler(
+ dataset,
+ num_replicas=dist.get_world_size(),
+ rank=dist.get_rank(),
+ shuffle=False,
+ drop_last=False,
+ ),
)
# compute scores
dataset.meta["match"] = np.nan
- index = 0
+ indices_list = []
+ scores_list = []
model.eval()
- for imgs, text in tqdm(dataloader):
+ for imgs, text, indices in tqdm(dataloader, disable=dist.get_rank() != 0):
imgs = imgs.to(device)
text = text.to(device)
- B = imgs.shape[0]
with torch.no_grad():
feat_img = model.encode_image(imgs)
@@ -118,13 +125,16 @@ def main():
feat_img = F.normalize(feat_img, dim=1)
feat_text = F.normalize(feat_text, dim=1)
clip_scores = logit_scale * (feat_img * feat_text).sum(dim=1)
- clip_scores_np = clip_scores.cpu().numpy()
-
- dataset.meta.loc[index : index + B - 1, "match"] = clip_scores_np
- index += B
-
- dataset.meta.to_csv(out_path, index=False)
- print(f"New meta with matching scores saved to '{out_path}'.")
+ clip_scores = clip_scores.cpu().tolist()
+ indices_list.extend(indices)
+ scores_list.extend(clip_scores)
+
+ gathered_list = [None] * dist.get_world_size()
+ dist.all_gather_object(gathered_list, (indices_list, scores_list))
+ if dist.get_rank() == 0:
+ merge_scores(gathered_list, dataset.meta)
+ dataset.meta.to_csv(out_path, index=False)
+ print(f"New meta with matching scores saved to '{out_path}'.")
if __name__ == "__main__":
diff --git a/tools/scoring/matching/inference_parallel.py b/tools/scoring/matching/inference_parallel.py
deleted file mode 100644
index 23ddddc..0000000
--- a/tools/scoring/matching/inference_parallel.py
+++ /dev/null
@@ -1,141 +0,0 @@
-import argparse
-import os
-
-import av
-import clip
-import colossalai
-import numpy as np
-import pandas as pd
-import torch
-import torch.distributed as dist
-import torch.nn.functional as F
-from torch.utils.data import DataLoader, DistributedSampler
-from torchvision.datasets.folder import pil_loader
-from tqdm import tqdm
-
-from tools.datasets.transform import extract_frames_new
-
-IMG_EXTENSIONS = (
- ".jpg",
- ".jpeg",
- ".png",
- ".ppm",
- ".bmp",
- ".pgm",
- ".tif",
- ".tiff",
- ".webp",
-)
-VID_EXTENSIONS = (".mp4", ".avi", ".mov", ".mkv")
-
-
-def is_video(filename):
- ext = os.path.splitext(filename)[-1].lower()
- return ext in VID_EXTENSIONS
-
-
-class VideoTextDataset(torch.utils.data.Dataset):
- def __init__(self, meta_path, transform):
- self.meta_path = meta_path
- self.meta = pd.read_csv(meta_path)
- self.transform = transform
-
- def __getitem__(self, index):
- row = self.meta.iloc[index]
- path = row["path"]
-
- if is_video(path):
- img = extract_frames_new(path, points=[0.5], backend='opencv')[0]
- else:
- img = pil_loader(path)
-
- img = self.transform(img)
-
- text = row["text"]
- text = clip.tokenize(text, truncate=True).squeeze()
-
- return img, text, index
-
- def __len__(self):
- return len(self.meta)
-
-
-def merge_scores(gathered_list: list, meta: pd.DataFrame):
- # reorder
- indices_list = list(map(lambda x: x[0], gathered_list))
- scores_list = list(map(lambda x: x[1], gathered_list))
- flat_indices = []
- for x in zip(*indices_list):
- flat_indices.extend(x)
- flat_scores = []
- for x in zip(*scores_list):
- flat_scores.extend(x)
- flat_indices = np.array(flat_indices)
- flat_scores = np.array(flat_scores)
- # filter duplicates
- unique_indices, unique_indices_idx = np.unique(flat_indices, return_index=True)
- meta.loc[unique_indices, "match"] = flat_scores[unique_indices_idx]
-
-
-def main():
- colossalai.launch_from_torch({})
- parser = argparse.ArgumentParser()
- parser.add_argument("meta_path", type=str, help="Path to the input CSV file")
- parser.add_argument("--bs", type=int, default=16, help="Batch size")
- parser.add_argument("--num_workers", type=int, default=16, help="Number of workers")
- args = parser.parse_args()
-
- meta_path = args.meta_path
- wo_ext, ext = os.path.splitext(meta_path)
- out_path = f"{wo_ext}_match{ext}"
-
- # build model
- device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
- model, preprocess = clip.load("ViT-L/14", device=device)
- logit_scale = model.logit_scale.exp().item()
-
- # build dataset
- dataset = VideoTextDataset(meta_path=meta_path, transform=preprocess)
- dataloader = DataLoader(
- dataset,
- batch_size=args.bs,
- num_workers=args.num_workers,
- sampler=DistributedSampler(
- dataset,
- num_replicas=dist.get_world_size(),
- rank=dist.get_rank(),
- shuffle=False,
- drop_last=False,
- ),
- )
-
- # compute scores
- dataset.meta["match"] = np.nan
- indices_list = []
- scores_list = []
- model.eval()
- for imgs, text, indices in tqdm(dataloader, disable=dist.get_rank() != 0):
- imgs = imgs.to(device)
- text = text.to(device)
-
- with torch.no_grad():
- feat_img = model.encode_image(imgs)
- feat_text = model.encode_text(text)
-
- feat_img = F.normalize(feat_img, dim=1)
- feat_text = F.normalize(feat_text, dim=1)
- clip_scores = logit_scale * (feat_img * feat_text).sum(dim=1)
- clip_scores = clip_scores.cpu().tolist()
- indices_list.extend(indices)
- scores_list.extend(clip_scores)
-
- gathered_list = [None] * dist.get_world_size()
- dist.all_gather_object(gathered_list, (indices_list, scores_list))
- if dist.get_rank() == 0:
- merge_scores(gathered_list, dataset.meta)
- dataset.meta.to_csv(out_path, index=False)
- print(f"New meta with matching scores saved to '{out_path}'.")
-
-
-if __name__ == "__main__":
- main()
diff --git a/tools/scoring/optical_flow/inference_parallel.py b/tools/scoring/optical_flow/inference.py
similarity index 100%
rename from tools/scoring/optical_flow/inference_parallel.py
rename to tools/scoring/optical_flow/inference.py