Open-Sora: Democratizing Efficient Video Production for All
Python
format
Shen-Chenhui53b9ccba1d14
diff --git a/docs/installation.md b/docs/installation.mdindex 71dae21..e17d0ef 100644--- a/docs/installation.md+++ b/docs/installation.md@@ -71,14 +71,34 @@ pip install -v .[eval]### Step 2: Install VBench-You need to manually install [VBench](https://github.com/Vchitect/VBench):+<!-- You need to manually install [VBench](https://github.com/Vchitect/VBench):```bash-pip install --no-deps vbench==0.1.2+pip install --no-deps vbench==0.1.1# If the installation shows a warning about the intalled vbench not in PATH, you need to add it by:export PATH="/path/to/vbench:$PATH"+``` -->++You need to install VBench mannually by:+```bash+# first clone their repo+cd ..+git clone https://github.com/Vchitect/VBench.git+cd VBench+git checkout v0.1.2++# next, fix their hard-coded path isse+vim vbench2_beta_i2v/utils.py+# find `image_root` in the `load_i2v_dimension_info` function, change it to point to your appropriate image folder++# last, create softlinks+cd Open-Sora # or `cd Open-Sora-dev` for development+ln -s ../VBench/vbench vbench+ln -s ../VBench/vbench2_beta_i2v vbench2_beta_i2v+# later you need to make sure to call evaluatio from your Open-Sora folder, else vbench, vbench2_beta_i2v cannot be found```+### Step 3: Install `cupy` for Potential VAE ErrorsYou need to mannually install [cupy](https://docs.cupy.dev/en/stable/install.html).diff --git a/eval/README.md b/eval/README.mdindex 15852bf..9bc70d0 100644--- a/eval/README.md+++ b/eval/README.md@@ -54,8 +54,12 @@ bash eval/vbench/launch.sh /path/to/ckpt num_frames model_nameAfter generation, install the VBench package following our [installation](../docs/installation.md)'s sections of "Evaluation Dependencies". Then, run the following commands to evaluate the generated samples.-```bash+<!-- ```bashbash eval/vbench/vbench.sh /path/to/video_folder /path/to/model/ckpt+``` -->++```bash+python eval/vbench/calc_vbench.py /path/to/video_folder /path/to/model/ckpt```Finally, we obtain the scaled scores for the model by:@@ -66,7 +70,7 @@ python eval/vbench/tabulate_vbench_scores.py --score_dir path/to/score/dir## VBench-i2v[VBench-i2v](https://github.com/Vchitect/VBench/tree/master/vbench2_beta_i2v) is a benchmark for short image to video generation (beta version).-Similarly, install the VBench package following our [installation](../docs/installation.md)'s sections of "Evaluation Dependencies". Then, run the following commands to evaluate the generated samples.+```bash# Step 1: generate the relevant videos@@ -76,12 +80,12 @@ bash eval/sample.sh /path/to/ckpt num_frames model_name_for_log -5abash eval/vbench_i2v/launch.sh /path/to/ckpt num_frames model_name# Step 2: run vbench to evaluate the generated samples-python eval/vbench_i2v/vbench_i2v.py+python eval/vbench_i2v/vbench_i2v.py /path/to/video_folder /path/to/model/ckpt+# Note that if you need to go to `VBench/vbench2_beta_i2v/utils.py` and change the harded-coded var `image_root` in the `load_i2v_dimension_info` function to your corresponding image folder.# Step 3: obtain the scaled scores-def load_i2v_dimension_info(json_dir, dimension, lang, resolution):-# Note that if you need to go to `your_conda_env_path/lib/python3.x/site-packages/vbench2_beta_i2v/utils.py` and change the harded-coded var `image_root` in the `load_i2v_dimension_info` function to your appropriate image folder.-python eval/vbench_i2v/tabulate_vbench_i2v_scores.py --score_dir path/to/evaluation_results/dir+python eval/vbench_i2v/tabulate_vbench_i2v_scores.py path/to/videos/folder path/to/your/model/ckpt+# this will store the results under `eval/vbench_i2v` in the path/to/your/model/ckpt```diff --git a/eval/loss/tabulate_rl_loss.py b/eval/loss/tabulate_rl_loss.pyindex d9b3a19..48c930e 100644--- a/eval/loss/tabulate_rl_loss.py+++ b/eval/loss/tabulate_rl_loss.py@@ -15,7 +15,6 @@ from ast import literal_evaldef parse_args():parser = argparse.ArgumentParser()parser.add_argument("--log_dir", type=str)- parser.add_argument("--ckpt_name", type=str)args = parser.parse_args()return args@@ -30,7 +29,8 @@ if __name__ == "__main__":os.makedirs(output_dir)files = os.listdir(args.log_dir)- files = ["img.log", "video_144p.log", "video_240p.log", "video_360p.log", "video_480p.log", "video_720p.log"]+ # files = ["img.log", "video_144p.log", "video_240p.log", "video_360p.log", "video_480p.log", "video_720p.log"]+ files = ["img.log", "144p_vid.log", "240p_vid.log", "360p_vid.log", "480p_vid.log", "720p_vid.log"]loss_info = {}@@ -47,7 +47,7 @@ if __name__ == "__main__":loss_info[resolution][frame] = format(loss, ".4f")# Convert and write JSON object to file- output_file_path = os.path.join(output_dir, args.ckpt_name + "_loss.json")+ output_file_path = os.path.join(output_dir, "loss.json")with open(output_file_path, "w") as outfile:json.dump(loss_info, outfile, indent=4, sort_keys=True)print(f"results saved to: {output_file_path}")diff --git a/eval/vbench/vbench.sh b/eval/vbench/vbench.shindex 3e1ff49..9b66791 100644--- a/eval/vbench/vbench.sh+++ b/eval/vbench/vbench.sh@@ -5,7 +5,7 @@ set -e# Base path for videosvideos_path=$1model_path=$2-output_path=${model_path}/eval/vbench+output_path=$(dirname $model_path)/eval/vbenchmkdir -p ${output_path}# videos_base=$(basename $videos_path)json_path=./eval/vbench/VBench_full_info.jsondiff --git a/eval/vbench_i2v/calc_vbench_i2v.py b/eval/vbench_i2v/calc_vbench_i2v.pynew file mode 100644index 0000000..fbbdd11--- /dev/null+++ b/eval/vbench_i2v/calc_vbench_i2v.py@@ -0,0 +1,45 @@+import argparse+import os++from vbench import VBench+from vbench2_beta_i2v import VBenchI2V++full_info_path = "eval/vbench_i2v/vbench2_i2v_full_info.json"+video_quality_dimensions = [+ "subject_consistency",+ "background_consistency",+ "motion_smoothness",+ "dynamic_degree",+ "aesthetic_quality",+ "imaging_quality",+ "temporal_flickering",+]+i2v_dimensions = ["i2v_subject", "i2v_background", "camera_motion"]+++def parse_args():+ parser = argparse.ArgumentParser()+ parser.add_argument("video_folder", type=str) # samples/samples..._vbench_i2v/+ parser.add_argument("model_ckpt", type=str)+ args = parser.parse_args()+ return args+++if __name__ == "__main__":+ args = parse_args()+ output_dir = os.path.join(args.model_ckpt, "eval", "vbench_i2v")+ os.makedirs(output_dir, exist_ok=True)+ video_path = args.video_folder++ my_VBench_I2V = VBenchI2V("cuda", full_info_path, output_dir)+ for i2v_dim in i2v_dimensions:+ my_VBench_I2V.evaluate(videos_path=video_path, name=i2v_dim, dimension_list=[i2v_dim], resolution="1-1")++ kwargs = {}+ kwargs["imaging_quality_preprocessing_mode"] = "longer" # use VBench/evaluate.py default++ my_VBench = VBench("cuda", full_info_path, output_dir)+ for quality_dim in video_quality_dimensions:+ my_VBench.evaluate(+ videos_path=video_path, name=quality_dim, dimension_list=[quality_dim], mode="vbench_standard", **kwargs+ )diff --git a/eval/vbench_i2v/tabulate_vbench_i2v_scores.py b/eval/vbench_i2v/tabulate_vbench_i2v_scores.pyindex 758171c..fb6a518 100644--- a/eval/vbench_i2v/tabulate_vbench_i2v_scores.py+++ b/eval/vbench_i2v/tabulate_vbench_i2v_scores.py@@ -6,44 +6,44 @@ I2V_WEIGHT = 1.0I2V_QUALITY_WEIGHT = 1.0I2V_LIST = [- "Video-Image Subject Consistency",- "Video-Image Background Consistency",+ "i2v_subject",+ "i2v_background",]I2V_QUALITY_LIST = [- "Subject Consistency",- "Background Consistency",- "Motion Smoothness",- "Dynamic Degree",- "Aesthetic Quality",- "Imaging Quality",- "Temporal Flickering",+ "subject_consistency",+ "background_consistency",+ "temporal_flickering",+ "motion_smoothness",+ "aesthetic_quality",+ "imaging_quality",+ "dynamic_degree",]DIM_WEIGHT_I2V = {- "Video-Text Camera Motion": 0.1,- "Video-Image Subject Consistency": 1,- "Video-Image Background Consistency": 1,- "Subject Consistency": 1,- "Background Consistency": 1,- "Motion Smoothness": 1,- "Dynamic Degree": 0.5,- "Aesthetic Quality": 1,- "Imaging Quality": 1,- "Temporal Flickering": 1,+ "camera_motion": 0.1,+ "i2v_subject": 1,+ "i2v_background_consistency": 1,+ "subject_consistency": 1,+ "background_consistency": 1,+ "motion_smoothness": 1,+ "dynamic_degree": 0.5,+ "aesthetic_quality": 1,+ "imaging_quality": 1,+ "temporal_flickering": 1,}NORMALIZE_DIC_I2V = {- "Video-Text Camera Motion": {"Min": 0.0, "Max": 1.0},- "Video-Image Subject Consistency": {"Min": 0.1462, "Max": 1.0},- "Video-Image Background Consistency": {"Min": 0.2615, "Max": 1.0},- "Subject Consistency": {"Min": 0.1462, "Max": 1.0},- "Background Consistency": {"Min": 0.2615, "Max": 1.0},- "Motion Smoothness": {"Min": 0.7060, "Max": 0.9975},- "Dynamic Degree": {"Min": 0.0, "Max": 1.0},- "Aesthetic Quality": {"Min": 0.0, "Max": 1.0},- "Imaging Quality": {"Min": 0.0, "Max": 1.0},- "Temporal Flickering": {"Min": 0.6293, "Max": 1.0},+ "camera_motion": {"Min": 0.0, "Max": 1.0},+ "i2v_subject": {"Min": 0.1462, "Max": 1.0},+ "i2v_background": {"Min": 0.2615, "Max": 1.0},+ "subject_consistency": {"Min": 0.1462, "Max": 1.0},+ "background_consistency": {"Min": 0.2615, "Max": 1.0},+ "motion_smoothness": {"Min": 0.7060, "Max": 0.9975},+ "dynamic_degree": {"Min": 0.0, "Max": 1.0},+ "aesthetic_quality": {"Min": 0.0, "Max": 1.0},+ "imaging_quality": {"Min": 0.0, "Max": 1.0},+ "temporal_flickering": {"Min": 0.6293, "Max": 1.0},}diff --git a/eval/vbench_i2v/vbench_i2v.py b/eval/vbench_i2v/vbench_i2v.pydeleted file mode 100644index ac82ab9..0000000--- a/eval/vbench_i2v/vbench_i2v.py+++ /dev/null@@ -1,41 +0,0 @@-import argparse-import os--from vbench2_beta_i2v import VBenchI2V--FULL_INFO_PATH = "eval/vbench_i2v/vbench2_i2v_full_info.json"-VIDEO_QUALITY_DIMENSIONS = [- "subject_consistency",- "background_consistency",- "motion_smoothness",- "dynamic_degree",- "aesthetic_quality",- "imaging_quality",- "temporal_flickering",-]-I2V_DIMENSIONS = ["i2v_subject", "i2v_background", "camera_motion"]---def parse_args():- parser = argparse.ArgumentParser()- parser.add_argument("video_folder", type=str) # samples/samples..._vbench_i2v/- parser.add_argument("model_ckpt", type=str)- args = parser.parse_args()- return args---if __name__ == "__main__":- args = parse_args()- output_dir = os.path.join(args.model_ckpt, "eval", "vbench_i2v")- os.makedirs(output_dir, exist_ok=True)- video_path = args.video_folder-- my_VBench_I2V = VBenchI2V("cuda", FULL_INFO_PATH, output_dir)- my_VBench_I2V.evaluate(videos_path=video_path, name="vbench_i2v", dimension_list=I2V_DIMENSIONS, resolution="1-1")-- # my_VBench = VBench("cuda", FULL_INFO_PATH, output_dir)- # my_VBench.evaluate(- # videos_path=video_path,- # name="vbench_video_quality",- # dimension_list=VIDEO_QUALITY_DIMENSIONS,- # )