Open-Sora: Democratizing Efficient Video Production for All
Python
Dev/pxy (#100)
xyupeng3c9d13de8fcc
diff --git a/docs/data_processing.md b/docs/data_processing.mdindex 0d1c195..e9cf362 100644--- a/docs/data_processing.md+++ b/docs/data_processing.md@@ -38,16 +38,13 @@ python -m tools.datasets.convert video ${ROOT_CLIPS} --output ${ROOT_META}/meta_# 2.4 Get clips information and remove broken ones. This should output ${ROOT_META}/meta_clips_info_fmin1.csvpython -m tools.datasets.datautil ${ROOT_META}/meta_clips.csv --info --fmin 1-# 3.1 Predict aesthetic scores. This should output ${ROOT_META}/meta_clips_info_fmin1_aes_part*.csv+# 3.1 Predict aesthetic scores. This should output ${ROOT_META}/meta_clips_info_fmin1_aes.csvtorchrun --nproc_per_node 8 -m tools.scoring.aesthetic.inference \${ROOT_META}/meta_clips_info_fmin1.csv \--bs 1024 \--num_workers 16-# 3.2 Merge files; This should output ${ROOT_META}/meta_clips_info_fmin1_aes.csv-python -m tools.datasets.datautil ${ROOT_META}/meta_clips_info_fmin1_aes_part*.csv --output ${ROOT_META}/meta_clips_info_fmin1_aes.csv--# 3.3 Filter by aesthetic scores. This should output ${ROOT_META}/meta_clips_info_fmin1_aes_aesmin5.csv+# 3.2 Filter by aesthetic scores. This should output ${ROOT_META}/meta_clips_info_fmin1_aes_aesmin5.csvpython -m tools.datasets.datautil ${ROOT_META}/meta_clips_info_fmin1_aes.csv --aesmin 5# 4.1 Generate caption. This should output ${ROOT_META}/meta_clips_info_fmin1_aes_aesmin5_caption_part*.csvdiff --git a/tools/datasets/convert.py b/tools/datasets/convert.pyindex 2c0db28..49acf72 100644--- a/tools/datasets/convert.py+++ b/tools/datasets/convert.py@@ -94,20 +94,22 @@ def process_vidprom(root, info):def process_general_images(root, output):root = os.path.expanduser(root)- image_lists = get_filelist(root, IMG_EXTENSIONS)- df = pd.DataFrame(dict(path=image_lists))- if output is None:- output = "images.csv"+ path_list = get_filelist(root, IMG_EXTENSIONS)+ fname_list = [os.path.splitext(os.path.basename(x))[0] for x in path_list]+ df = pd.DataFrame(dict(id=fname_list, path=path_list))++ os.makedirs(os.path.dirname(output), exist_ok=True)df.to_csv(output, index=False)print(f"Saved {len(df)} samples to {output}.")def process_general_videos(root, output):root = os.path.expanduser(root)- video_lists = get_filelist(root, VID_EXTENSIONS)- df = pd.DataFrame(dict(path=video_lists))- if output is None:- output = "videos.csv"+ path_list = get_filelist(root, VID_EXTENSIONS)+ fname_list = [os.path.splitext(os.path.basename(x))[0] for x in path_list]+ relpath_list = [os.path.relpath(x, root) for x in path_list]+ df = pd.DataFrame(dict(path=path_list, id=fname_list, relpath=relpath_list))+os.makedirs(os.path.dirname(output), exist_ok=True)df.to_csv(output, index=False)print(f"Saved {len(df)} samples to {output}.")@@ -119,7 +121,7 @@ if __name__ == "__main__":parser.add_argument("root", type=str)parser.add_argument("--split", type=str, default="train")parser.add_argument("--info", type=str, default=None)- parser.add_argument("--output", type=str, default=None)+ parser.add_argument("--output", type=str, default=None, required=True, help='Output path')args = parser.parse_args()if args.dataset == "imagenet":diff --git a/tools/datasets/datautil.py b/tools/datasets/datautil.pyindex d06b22f..b7d0b5f 100644--- a/tools/datasets/datautil.py+++ b/tools/datasets/datautil.py@@ -54,7 +54,7 @@ def get_info(path):if ext in IMG_EXTENSIONS:im = cv2.imread(path)if im is None:- return 0, 0, 0, np.nan, np.nan+ return 0, 0, 0, np.nan, np.nan, np.nanheight, width = im.shape[:2]num_frames, fps = 1, np.nanelse:@@ -415,8 +415,12 @@ def main(args):data_new = pd.read_csv(args.intersection)print(f"Intersection csv contains {len(data_new)} samples.")cols_to_use = data_new.columns.difference(data.columns)- cols_to_use = cols_to_use.insert(0, "path")- data = pd.merge(data, data_new[cols_to_use], on="path", how="inner")++ col_on = 'path'+ # if 'id' in data.columns and 'id' in data_new.columns:+ # col_on = 'id'+ cols_to_use = cols_to_use.insert(0, col_on)+ data = pd.merge(data, data_new[cols_to_use], on=col_on, how="inner")print(f"Intersection number of samples: {len(data)}.")# train columns@@ -484,6 +488,8 @@ def main(args):data["path"] = apply(data["path"], lambda x: os.path.relpath(x, args.relpath))if args.abspath is not None:data["path"] = apply(data["path"], lambda x: os.path.join(args.abspath, x))+ if args.path_to_id:+ data["id"] = apply(data["path"], lambda x: os.path.splitext(os.path.basename(x))[0])if args.merge_cmotion:data["text"] = apply(data, lambda x: merge_cmotion(x["text"], x["cmotion"]), axis=1)if args.refine_llm_caption:@@ -581,6 +587,7 @@ def parse_args():# path processingparser.add_argument("--relpath", type=str, default=None, help="modify the path to relative path by root given")parser.add_argument("--abspath", type=str, default=None, help="modify the path to absolute path by root given")+ parser.add_argument("--path-to-id", action='store_true', help="add id based on path")# caption filteringparser.add_argument(diff --git a/tools/scene_cut/cut.py b/tools/scene_cut/cut.pyindex 0e02aed..0b9c9e6 100644--- a/tools/scene_cut/cut.py+++ b/tools/scene_cut/cut.py@@ -50,10 +50,10 @@ def split_video(video_path,scene_list,save_dir,- min_seconds=2.0,- max_seconds=15.0,+ min_seconds=2,+ max_seconds=15,target_fps=30,- shorter_size=720,+ shorter_size=None,verbose=False,logger=None,):@@ -134,8 +134,8 @@ def parse_args():help='if not None, clip shorter than min_seconds is ignored')parser.add_argument("--max_seconds", type=float, default=None,help='if not None, clip longer than max_seconds is truncated')- parser.add_argument("--target_fps", type=int, default=30, help='target fps of clips')- parser.add_argument("--shorter_size", type=int, default=720, help='resize the shorter size by keeping ratio')+ parser.add_argument("--target_fps", type=int, default=None, help='target fps of clips')+ parser.add_argument("--shorter_size", type=int, default=None, help='resize the shorter size by keeping ratio')parser.add_argument("--num_workers", type=int, default=None, help='#workers for pandarallel')args = parser.parse_args()diff --git a/tools/scoring/optical_flow/inference.py b/tools/scoring/optical_flow/inference.pyindex 170b076..499ff06 100644--- a/tools/scoring/optical_flow/inference.py+++ b/tools/scoring/optical_flow/inference.py@@ -1,7 +1,7 @@import argparseimport os+from datetime import timedelta-import colossalaiimport numpy as npimport pandas as pdimport torch@@ -13,25 +13,27 @@ from torchvision.transforms.functional import pil_to_tensorfrom tqdm import tqdmfrom tools.datasets.utils import extract_frames-from .unimatch import UniMatch-def merge_scores(gathered_list: list, meta: pd.DataFrame):+def merge_scores(gathered_list: list, meta: pd.DataFrame, column):# reorderindices_list = list(map(lambda x: x[0], gathered_list))- flow_scores_list = list(map(lambda x: x[1], gathered_list))+ scores_list = list(map(lambda x: x[1], gathered_list))+flat_indices = []for x in zip(*indices_list):flat_indices.extend(x)- flat_flow_scores = []- for x in zip(*flow_scores_list):- flat_flow_scores.extend(x)+ flat_scores = []+ for x in zip(*scores_list):+ flat_scores.extend(x)flat_indices = np.array(flat_indices)- flat_flow_scores = np.array(flat_flow_scores)+ flat_scores = np.array(flat_scores)+# filter duplicatesunique_indices, unique_indices_idx = np.unique(flat_indices, return_index=True)- meta.loc[unique_indices, "flow"] = flat_flow_scores[unique_indices_idx]+ meta.loc[unique_indices, column] = flat_scores[unique_indices_idx]+ return metaclass VideoTextDataset(torch.utils.data.Dataset):@@ -61,17 +63,19 @@ class VideoTextDataset(torch.utils.data.Dataset):def parse_args():parser = argparse.ArgumentParser()parser.add_argument("meta_path", type=str, help="Path to the input CSV file")- parser.add_argument("--bs", type=int, default=4, help="Batch size")+ parser.add_argument("--bs", type=int, default=4, help="Batch size") # don't use too large bs for unimatchparser.add_argument("--num_workers", type=int, default=16, help="Number of workers")args = parser.parse_args()return argsdef main():+ args = parse_args()+torch.backends.cudnn.deterministic = Truetorch.backends.cudnn.benchmark = False- colossalai.launch_from_torch({})- args = parse_args()+ dist.init_process_group(backend="nccl", timeout=timedelta(hours=24))+ torch.cuda.set_device(dist.get_rank() % torch.cuda.device_count())meta_path = args.meta_pathwo_ext, ext = os.path.splitext(meta_path)@@ -88,11 +92,10 @@ def main():num_transformer_layers=6,reg_refine=True,task="flow",- ).eval()+ )ckpt = torch.load("./pretrained_models/unimatch/gmflow-scale2-regrefine6-mixdata-train320x576-4e7b215d.pth")model.load_state_dict(ckpt["model"])model = model.to(device)- # model = torch.nn.DataParallel(model)# build datasetdataset = VideoTextDataset(meta_path=meta_path, frame_inds=[0, 10, 20, 30])@@ -110,9 +113,9 @@ def main():)# compute optical flow scores- dataset.meta["flow"] = np.nanindices_list = []- flow_scores_list = []+ scores_list = []+ model.eval()for images, indices in tqdm(dataloader, disable=dist.get_rank() != 0):images = images.to(device)B = images.shape[0]@@ -138,13 +141,13 @@ def main():flow_scores = flow_scores.tolist()indices_list.extend(indices)- flow_scores_list.extend(flow_scores)+ scores_list.extend(flow_scores)gathered_list = [None] * dist.get_world_size()- dist.all_gather_object(gathered_list, (indices_list, flow_scores_list))+ dist.all_gather_object(gathered_list, (indices_list, scores_list))if dist.get_rank() == 0:- merge_scores(gathered_list, dataset.meta)- dataset.meta.to_csv(out_path, index=False)+ meta_new = merge_scores(gathered_list, dataset.meta, column='flow')+ meta_new.to_csv(out_path, index=False)print(f"New meta with optical flow scores saved to '{out_path}'.")