Open-Sora: Democratizing Efficient Video Production for All
Python
accelerate aesthetic
Zangwei Zheng4d3b68e3ad3c
diff --git a/tools/aesthetic/inference.py b/tools/aesthetic/inference.pyindex 99287af..1e2a3f7 100644--- a/tools/aesthetic/inference.py+++ b/tools/aesthetic/inference.py@@ -1,52 +1,47 @@# adapted from https://github.com/christophschuhmann/improved-aesthetic-predictor/blob/main/simple_inference.py+import argparse++import avimport clip-import cv2+import numpy as npimport pandas as pdimport torchimport torch.nn as nnimport torch.nn.functional as F-from PIL import Image+from einops import rearrangefrom tqdm import tqdm-def get_video_length(cap):- return int(cap.get(cv2.CAP_PROP_FRAME_COUNT))---def extract_frames(video_path, points=(0.5,)):- cap = cv2.VideoCapture(video_path)- length = get_video_length(cap)- points = [int(length * point) for point in points]+def extract_frames(video_path, points=(0.0, 0.5, 0.9)):+ container = av.open(video_path)+ total_frames = container.streams.video[0].framesframes = []- if length < 3:- return frames, lengthfor point in points:- cap.set(cv2.CAP_PROP_POS_FRAMES, point)- ret, frame = cap.read()- frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)- frame = Image.fromarray(frame)+ target_frame = total_frames * point+ target_timestamp = int((target_frame * av.time_base) / container.streams.video[0].average_rate)+ container.seek(target_timestamp)+ frame = next(container.decode(video=0)).to_image()frames.append(frame)- if len(frames) == 1:- frames = frames[0]- return frames, length+ return framesclass VideoTextDataset(torch.utils.data.Dataset):- def __init__(self, csv_path, transform=None):+ def __init__(self, csv_path, transform=None, points=(0.1, 0.5, 0.9)):self.csv_path = csv_path- self.samples = pd.read_csv(csv_path, header=None)+ self.data = pd.read_csv(csv_path)self.transform = transform+ self.points = pointsdef getitem(self, index):- sample = self.samples.iloc[index]- img = extract_frames(sample[0])[0]- img = self.transform(img)- text = sample[1]+ sample = self.data.iloc[index]+ images = extract_frames(sample["path"], points=self.points)+ images = [self.transform(img) for img in images]+ images = torch.stack(images)- return dict(index=index, image=img, text=text)+ return dict(index=index, image=images)def __len__(self):- return len(self.samples)+ return len(self.data)def __getitem__(self, index):return self.getitem(index)@@ -87,29 +82,48 @@ class AestheticScorer(nn.Module):return self.mlp(image_features)-def main():+def main(args):+ output_file = args.input.replace(".csv", "_aes.csv")++ # build modeldevice = "cuda" if torch.cuda.is_available() else "cpu"model = AestheticScorer(768, device)+ preprocess = model.preprocess+ model = torch.nn.DataParallel(model)- dataset = VideoTextDataset(- "/mnt/hdd/data/VidProM/VidProM_pika/meta/vidprom_relength_fmin_48_clean_en_unescape_nourl.csv",- transform=model.preprocess,+ # build dataset+ dataset = VideoTextDataset(args.input, transform=preprocess, points=(0.5,))+ dataloader = torch.utils.data.DataLoader(+ dataset,+ batch_size=args.bs,+ shuffle=False,+ num_workers=args.num_workers,+ pin_memory=True,+ prefetch_factor=args.prefetch_factor,)- dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=False, num_workers=16, pin_memory=True)- dataset.samples["aesthetic"] = ""- model = torch.nn.DataParallel(model)- output_file = "vidprom_aes.csv"++ # compute aesthetic scores+ dataset.data["aesthetic"] = np.nanindex = 0for batch in tqdm(dataloader):- image = batch["image"].to(device)+ images = batch["image"].to(device)+ B = images.shape[0]+ images = rearrange(images, "b p c h w -> (b p) c h w")with torch.no_grad():- score = model(image)- dataset.samples.loc[index : index + len(score) - 1, "aesthetic"] = score.cpu().numpy().flatten()- index += len(score)-- dataset.samples.to_csv(output_file, index=False, header=False)- print(f"Saved {index} samples")+ scores = model(images)+ scores = rearrange(scores, "(b p) 1 -> b p", b=B)+ scores = scores.mean(dim=1)+ scores_np = scores.cpu().numpy()+ dataset.data.loc[index : index + len(scores_np) - 1, "aesthetic"] = scores_np+ index += len(images)+ dataset.data.to_csv(output_file, index=False)if __name__ == "__main__":- main()+ parser = argparse.ArgumentParser()+ parser.add_argument("input", type=str, help="Path to the input CSV file")+ parser.add_argument("--bs", type=int, default=512, help="Batch size")+ parser.add_argument("--num_workers", type=int, default=64, help="Number of workers")+ parser.add_argument("--prefetch_factor", type=int, default=8, help="Prefetch factor")+ args = parser.parse_args()+ main(args)