Open-Sora: Democratizing Efficient Video Production for All
Python
update vbench script
zhengzangw67f03c6cdaf2
diff --git a/.gitignore b/.gitignoreindex b4237e0..57b6f55 100644--- a/.gitignore+++ b/.gitignore@@ -199,4 +199,5 @@ tools/caption/pllava_dir/PLLaVA/# vbenchvbench+!eval/vbenchvbench2_beta_i2vdiff --git a/eval/loss/launch.sh b/eval/loss/launch.shindex a80f791..67d230a 100644--- a/eval/loss/launch.sh+++ b/eval/loss/launch.sh@@ -1,8 +1,5 @@#!/bin/bash-set -x-set -e-CMD="torchrun --standalone --nproc_per_node 1 eval/loss/eval_loss.py configs/opensora-v1-2/misc/eval_loss.py"CKPT_PATH=$1MODEL_NAME=$2diff --git a/eval/vbench/calc_vbench.py b/eval/vbench/calc_vbench.pynew file mode 100644index 0000000..b2505bf--- /dev/null+++ b/eval/vbench/calc_vbench.py@@ -0,0 +1,66 @@+import argparse+import os+from vbench import VBench+import torch+import time++full_info_path = "eval/vbench/VBench_full_info.json"+dimensions = [+ # Quality Score+ "subject_consistency",+ "background_consistency",+ "motion_smoothness",+ "dynamic_degree",+ "aesthetic_quality",+ "imaging_quality",+ "temporal_flickering",+ # Semantic Score+ "object_class",+ "multiple_objects",+ "color",+ "spatial_relationship",+ "scene",+ "temporal_style",+ "overall_consistency",+ "human_action",+ "appearance_style",+]++def parse_args():+ parser = argparse.ArgumentParser()+ parser.add_argument("video_folder", type=str) # samples/samples..._vbench/eval+ parser.add_argument("model_ckpt", type=str)+ parser.add_argument("--start", type=int, default=0) # start index of dimension to be evaluated+ parser.add_argument("--end", type=int, default=-1) # start index of dimension to be evaluated++ args = parser.parse_args()+ return args+++if __name__ == "__main__":+ args = parse_args()+ output_dir = os.path.join(args.model_ckpt, "vbench")+ os.makedirs(output_dir, exist_ok=True)+ video_path = args.video_folder++ kwargs = {}+ kwargs['imaging_quality_preprocessing_mode'] = 'longer' # use VBench/evaluate.py default++ start_time = time.time()++ # NOTE: important to use torch.device("cuda"), else will have issue with object_class third_party module+ my_VBench = VBench(torch.device("cuda"), full_info_path, output_dir)+ if args.end == -1: # adjust end accordingly+ args.end = len(dimensions)+ for dim in dimensions[args.start:args.end]:+ my_VBench.evaluate(+ videos_path=video_path,+ name=dim,+ local=False,+ read_frame=False,+ dimension_list=[dim],+ mode='vbench_standard',+ **kwargs+ )++ print("Runtime: %s seconds " % (time.time() - start_time))\ No newline at end of filediff --git a/eval/vbench/launch.sh b/eval/vbench/launch.shindex d45c9fb..0cb342e 100644--- a/eval/vbench/launch.sh+++ b/eval/vbench/launch.sh@@ -1,8 +1,5 @@# !/bin/bash-set -x-set -e-CKPT=$1NUM_FRAMES=$2MODEL_NAME=$3diff --git a/eval/vbench/launch_calc.sh b/eval/vbench/launch_calc.shnew file mode 100644index 0000000..53114b9--- /dev/null+++ b/eval/vbench/launch_calc.sh@@ -0,0 +1,17 @@+# !/bin/bash++VIDEO_DIR=$1+CKPT_DIR=$2+LOG_BASE=$CKPT_DIR+mkdir -p $LOG_BASE+echo "Logging to $LOG_BASE"++GPUS=(0 1 2 3 4 5 6 7)+START_INDEX_LIST=(0 2 4 6 8 10 12 14)+END_INDEX_LIST=(2 4 6 8 10 12 14 16)+TASK_ID_LIST=(calc_vbench_a calc_vbench_b calc_vbench_c calc_vbench_d calc_vbench_e calc_vbench_f calc_vbench_g calc_vbench_h) # for log records only+++for i in "${!GPUS[@]}"; do+ CUDA_VISIBLE_DEVICES=${GPUS[i]} python eval/vbench/calc_vbench.py $VIDEO_DIR $CKPT_DIR --start ${START_INDEX_LIST[i]} --end ${END_INDEX_LIST[i]} > ${LOG_BASE}/${TASK_ID_LIST[i]}.log 2>&1 &+done