Open-Sora: Democratizing Efficient Video Production for All
Python
automatically calculate scaled scores
Shen-Chenhuib8a93d5abac9
diff --git a/eval/README.md b/eval/README.mdindex b80ab0c..701db17 100644--- a/eval/README.md+++ b/eval/README.md@@ -42,9 +42,9 @@ First, generate the relevant videos with the following commands:```bash# vbench tasks (4a 4b 4c ...)-bash eval/sample.sh /path/to/ckpt -4a+bash eval/sample.sh /path/to/ckpt num_frames model_name_for_log -4a# launch 8 jobs at once (you must read the script to understand the details)-bash eval/vbench/launch.sh /path/to/ckpt+bash eval/vbench/launch.sh /path/to/ckpt num_frames model_name```After generation, install the VBench package following our [installation](../docs/installation.md)'s sections of "Evaluation Dependencies". Then, run the following commands to evaluate the generated samples.@@ -53,11 +53,31 @@ After generation, install the VBench package following our [installation](../docbash eval/vbench/vbench.sh /path/to/video_folder```+Finally, we obtain the scaled scores for the model by:+```bash+python eval/vbench/tabulate_vbench_scores.py --score_dir path/to/evaluation_results/dir+```+## VBench-i2v[VBench-i2v](https://github.com/Vchitect/VBench/tree/master/vbench2_beta_i2v) is a benchmark for short image to video generation (beta version).+Similarly, install the VBench package following our [installation](../docs/installation.md)'s sections of "Evaluation Dependencies". Then, run the following commands to evaluate the generated samples.++```bash+# Step 1: generate the relevant videos+# vbench i2v tasks (5a 5b 5c ...)+bash eval/sample.sh /path/to/ckpt num_frames model_name_for_log -5a+# launch 8 jobs at once+bash eval/vbench_i2v/launch.sh /path/to/ckpt num_frames model_name++# Step 2: run vbench to evaluate the generated samples+python eval/vbench_i2v/vbench_i2v.py+python eval/vbench_i2v/vbench_video_quality.py-TBD+# Step 3: obtain the scaled scores+python eval/vbench_i2v/tabulate_vbench_i2v_scores.py --score_dir path/to/evaluation_results/dir++```## VAEdiff --git a/eval/vbench/launch.sh b/eval/vbench/launch.shindex 0ee4009..436a937 100644--- a/eval/vbench/launch.sh+++ b/eval/vbench/launch.sh@@ -5,7 +5,7 @@ set -eCKPT=$1MODEL_NAME=$2-NUM_FRAMES=51+NUM_FRAMES=$3if [[ $CKPT == *"ema"* ]]; thenparentdir=$(dirname $CKPT)diff --git a/eval/vbench/tabulate_vbench_scores.py b/eval/vbench/tabulate_vbench_scores.pyindex 3df9805..c7f27b8 100644--- a/eval/vbench/tabulate_vbench_scores.py+++ b/eval/vbench/tabulate_vbench_scores.py@@ -3,6 +3,68 @@ import jsonimport osfrom ast import literal_eval+SEMANTIC_WEIGHT = 1+QUALITY_WEIGHT = 4++QUALITY_LIST = [+ "subject consistency",+ "background consistency",+ "temporal flickering",+ "motion smoothness",+ "aesthetic quality",+ "imaging quality",+ "dynamic degree",]++SEMANTIC_LIST = [+ "object class",+ "multiple objects",+ "human action",+ "color",+ "spatial relationship",+ "scene",+ "appearance style",+ "temporal style",+ "overall consistency"+]++NORMALIZE_DIC = {+ "subject consistency": {"Min": 0.1462, "Max": 1.0},+ "background consistency": {"Min": 0.2615, "Max": 1.0},+ "temporal flickering": {"Min": 0.6293, "Max": 1.0},+ "motion smoothness": {"Min": 0.706, "Max": 0.9975},+ "dynamic degree": {"Min": 0.0, "Max": 1.0},+ "aesthetic quality": {"Min": 0.0, "Max": 1.0},+ "imaging quality": {"Min": 0.0, "Max": 1.0},+ "object class": {"Min": 0.0, "Max": 1.0},+ "multiple objects": {"Min": 0.0, "Max": 1.0},+ "human action": {"Min": 0.0, "Max": 1.0},+ "color": {"Min": 0.0, "Max": 1.0},+ "spatial relationship": {"Min": 0.0, "Max": 1.0},+ "scene": {"Min": 0.0, "Max": 0.8222},+ "appearance style": {"Min": 0.0009, "Max": 0.2855},+ "temporal style": {"Min": 0.0, "Max": 0.364},+ "overall consistency": {"Min": 0.0, "Max": 0.364}+}++DIM_WEIGHT = {+"subject consistency":1,+"background consistency":1,+"temporal flickering":1,+"motion smoothness":1,+"aesthetic quality":1,+"imaging quality":1,+"dynamic degree":0.5,+"object class":1,+"multiple objects":1,+"human action":1,+"color":1,+"spatial relationship":1,+"scene":1,+"appearance style":1,+"temporal style":1,+"overall consistency":1+}+def parse_args():parser = argparse.ArgumentParser()parser.add_argument("--score_dir", type=str) # evaluation_results/samples_...@@ -19,7 +81,6 @@ if __name__ == "__main__":assert len(res_files) == len(info_files), f"got {len(res_files)} res files, but {len(info_files)} info files"full_results = {}-for res_file in res_files:# first check if results is normalinfo_file = res_file.split(res_postfix)[0] + info_postfix@@ -30,12 +91,37 @@ if __name__ == "__main__":with open(os.path.join(args.score_dir, res_file), "r", encoding="utf-8") as f:data = json.load(f)for key, val in data.items():- full_results[key] = format(val[0]*100, ".2f")+ full_results[key] = format(val[0], ".4f")++ scaled_results = {}+ dims = set()+ for key, val in full_results.items():+ dim = key.replace("_", " ") if "_" in key else key+ scaled_score = (float(val) - NORMALIZE_DIC[dim]["Min"]) / (NORMALIZE_DIC[dim]["Max"] - NORMALIZE_DIC[dim]["Min"])+ scaled_score *= DIM_WEIGHT[dim]+ scaled_results[dim] = scaled_score+ dims.add(dim)+ assert len(dims) == len(NORMALIZE_DIC), f"{set(NORMALIZE_DIC.keys())-dims} not calculated yet"++ quality_score = sum([scaled_results[i] for i in QUALITY_LIST]) / sum([DIM_WEIGHT[i] for i in QUALITY_LIST])+ semantic_score = sum([scaled_results[i] for i in SEMANTIC_LIST]) / sum([DIM_WEIGHT[i] for i in SEMANTIC_LIST])+ scaled_results["quality score"] = quality_score+ scaled_results["semantic score"] = semantic_score+ scaled_results["total score"] = (quality_score * QUALITY_WEIGHT + semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT)++ formated_scaled_results = {}+ for key,val in scaled_results.items():+ formated_scaled_results[key] = format(val*100, ".2f")+"%"output_file_path = os.path.join(args.score_dir, "all_results.json")with open(output_file_path, "w") as outfile:json.dump(full_results, outfile, indent=4, sort_keys=True)print(f"results saved to: {output_file_path}")++ scaled_file_path = os.path.join(args.score_dir, "scaled_results.json")+ with open(scaled_file_path, "w") as outfile:+ json.dump(formated_scaled_results, outfile, indent=4, sort_keys=True)+ print(f"results saved to: {scaled_file_path}")diff --git a/eval/vbench_i2v/launch.sh b/eval/vbench_i2v/launch.shindex 8900b90..5a183e6 100644--- a/eval/vbench_i2v/launch.sh+++ b/eval/vbench_i2v/launch.sh@@ -4,8 +4,8 @@ set -xset -eCKPT=$1-NUM_FRAMES=51-MODEL_NAME=$2+NUM_FRAMES=$2+MODEL_NAME=$3if [[ $CKPT == *"ema"* ]]; thenparentdir=$(dirname $CKPT)@@ -13,7 +13,6 @@ if [[ $CKPT == *"ema"* ]]; thenelseCKPT_BASE=$(basename $CKPT)fi-# LOG_BASE=logs/sample/${MODEL_NAME}_${CKPT_BASE}LOG_BASE=$(dirname $CKPT)/evalecho "Logging to $LOG_BASE"diff --git a/eval/vbench_i2v/tabulate_vbench_i2v_scores.py b/eval/vbench_i2v/tabulate_vbench_i2v_scores.pynew file mode 100644index 0000000..9f6dc77--- /dev/null+++ b/eval/vbench_i2v/tabulate_vbench_i2v_scores.py@@ -0,0 +1,110 @@+import argparse+import json+import os+from ast import literal_eval++I2V_WEIGHT = 1.0+I2V_QUALITY_WEIGHT = 1.0++I2V_LIST = [+ "Video-Image Subject Consistency",+ "Video-Image Background Consistency",+]++I2V_QUALITY_LIST = [+ "Subject Consistency",+ "Background Consistency",+ "Motion Smoothness",+ "Dynamic Degree",+ "Aesthetic Quality",+ "Imaging Quality",+ "Temporal Flickering"+]++DIM_WEIGHT_I2V = {+"Video-Text Camera Motion": 0.1,+"Video-Image Subject Consistency": 1,+"Video-Image Background Consistency": 1,+"Subject Consistency": 1,+"Background Consistency": 1,+"Motion Smoothness": 1,+"Dynamic Degree": 0.5,+"Aesthetic Quality": 1,+"Imaging Quality": 1,+"Temporal Flickering": 1+}++NORMALIZE_DIC_I2V = {+ "Video-Text Camera Motion" :{"Min": 0.0, "Max":1.0 },+ "Video-Image Subject Consistency":{"Min": 0.1462, "Max": 1.0},+ "Video-Image Background Consistency":{"Min": 0.2615, "Max":1.0 },+ "Subject Consistency":{"Min": 0.1462, "Max": 1.0},+ "Background Consistency":{"Min": 0.2615, "Max": 1.0 },+ "Motion Smoothness":{"Min": 0.7060, "Max": 0.9975},+ "Dynamic Degree":{"Min": 0.0, "Max": 1.0},+ "Aesthetic Quality":{"Min": 0.0, "Max": 1.0},+ "Imaging Quality":{"Min": 0.0, "Max": 1.0},+ "Temporal Flickering":{"Min":0.6293, "Max": 1.0}+}+++def parse_args():+ parser = argparse.ArgumentParser()+ parser.add_argument("--score_dir", type=str) # evaluation_results/samples_...+ args = parser.parse_args()+ return args++if __name__ == "__main__":+ args = parse_args()+ res_postfix = "_eval_results.json"+ info_postfix = "_full_info.json"+ files = os.listdir(args.score_dir)+ res_files = [x for x in files if res_postfix in x]+ info_files = [x for x in files if info_postfix in x]+ assert len(res_files) == len(info_files), f"got {len(res_files)} res files, but {len(info_files)} info files"++ full_results = {}+ for res_file in res_files:+ # first check if results is normal+ info_file = res_file.split(res_postfix)[0] + info_postfix+ with open(os.path.join(args.score_dir, info_file), "r", encoding="utf-8") as f:+ info = json.load(f)+ assert len(info[0]["video_list"]) > 0, f"Error: {info_file} has 0 video list"+ # read results+ with open(os.path.join(args.score_dir, res_file), "r", encoding="utf-8") as f:+ data = json.load(f)+ for key, val in data.items():+ full_results[key] = format(val[0], ".4f")++ scaled_results = {}+ dims = set()+ for key, val in full_results.items():+ dim = key+ scaled_score = (float(val) - NORMALIZE_DIC_I2V[dim]["Min"]) / (NORMALIZE_DIC_I2V[dim]["Max"] - NORMALIZE_DIC_I2V[dim]["Min"])+ scaled_score *= DIM_WEIGHT_I2V[dim]+ scaled_results[dim] = scaled_score+ dims.add(dim)++ assert len(dims) == len(NORMALIZE_DIC_I2V), f"{set(NORMALIZE_DIC_I2V.keys())-dims} not calculated yet"++ quality_score = sum([scaled_results[i] for i in I2V_QUALITY_LIST]) / sum([DIM_WEIGHT_I2V[i] for i in I2V_QUALITY_LIST])+ i2v_score = sum([scaled_results[i] for i in I2V_LIST]) / sum([DIM_WEIGHT_I2V[i] for i in I2V_LIST])++ scaled_results["quality score"] = quality_score+ scaled_results["i2v score"] = i2v_score+ scaled_results["total score"] = (quality_score * I2V_QUALITY_WEIGHT + i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT)++ formated_scaled_results = {}+ for key,val in scaled_results.items():+ formated_scaled_results[key] = format(val*100, ".2f")+"%"++ output_file_path = os.path.join(args.score_dir, "all_results.json")+ with open(output_file_path, "w") as outfile:+ json.dump(full_results, outfile, indent=4, sort_keys=True)+ print(f"results saved to: {output_file_path}")+++ scaled_file_path = os.path.join(args.score_dir, "scaled_results.json")+ with open(scaled_file_path, "w") as outfile:+ json.dump(formated_scaled_results, outfile, indent=4, sort_keys=True)+ print(f"results saved to: {scaled_file_path}")\ No newline at end of filediff --git a/eval/vbench_i2v/vbench_video_quality.py b/eval/vbench_i2v/vbench_video_quality.pyindex 01be7bc..a291426 100644--- a/eval/vbench_i2v/vbench_video_quality.py+++ b/eval/vbench_i2v/vbench_video_quality.py@@ -2,7 +2,7 @@ from vbench import VBenchVIDEO_PATH = ""-DIMENSIONS = ["subject consistency", "background_consistency", "motion_smoothness", "dynamic_degree", "aesthetic_quality", "imaging_quality"]+DIMENSIONS = ["subject_consistency", "background_consistency", "motion_smoothness", "dynamic_degree", "aesthetic_quality", "imaging_quality", "temporal_flickering"]my_VBench = VBench("cuda", "vbench2_beta_i2v/vbench2_i2v_full_info.json", "evaluation_results")my_VBench.evaluate(