Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

format

Shen-Chenhui53b9ccba1d14
diff --git a/docs/installation.md b/docs/installation.md
index 71dae21..e17d0ef 100644
--- a/docs/installation.md
+++ b/docs/installation.md
@@ -71,14 +71,34 @@ pip install -v .[eval]
### Step 2: Install VBench
-You need to manually install [VBench](https://github.com/Vchitect/VBench):
+<!-- You need to manually install [VBench](https://github.com/Vchitect/VBench):
```bash
-pip install --no-deps vbench==0.1.2
+pip install --no-deps vbench==0.1.1
# If the installation shows a warning about the intalled vbench not in PATH, you need to add it by:
export PATH="/path/to/vbench:$PATH"
+``` -->
+
+You need to install VBench mannually by:
+```bash
+# first clone their repo
+cd ..
+git clone https://github.com/Vchitect/VBench.git
+cd VBench
+git checkout v0.1.2
+
+# next, fix their hard-coded path isse
+vim vbench2_beta_i2v/utils.py
+# find `image_root` in the `load_i2v_dimension_info` function, change it to point to your appropriate image folder
+
+# last, create softlinks
+cd Open-Sora # or `cd Open-Sora-dev` for development
+ln -s ../VBench/vbench vbench
+ln -s ../VBench/vbench2_beta_i2v vbench2_beta_i2v
+# later you need to make sure to call evaluatio from your Open-Sora folder, else vbench, vbench2_beta_i2v cannot be found
```
+
### Step 3: Install `cupy` for Potential VAE Errors
You need to mannually install [cupy](https://docs.cupy.dev/en/stable/install.html).
diff --git a/eval/README.md b/eval/README.md
index 15852bf..9bc70d0 100644
--- a/eval/README.md
+++ b/eval/README.md
@@ -54,8 +54,12 @@ bash eval/vbench/launch.sh /path/to/ckpt num_frames model_name
After generation, install the VBench package following our [installation](../docs/installation.md)'s sections of "Evaluation Dependencies". Then, run the following commands to evaluate the generated samples.
-```bash
+<!-- ```bash
bash eval/vbench/vbench.sh /path/to/video_folder /path/to/model/ckpt
+``` -->
+
+```bash
+python eval/vbench/calc_vbench.py /path/to/video_folder /path/to/model/ckpt
```
Finally, we obtain the scaled scores for the model by:
@@ -66,7 +70,7 @@ python eval/vbench/tabulate_vbench_scores.py --score_dir path/to/score/dir
## VBench-i2v
[VBench-i2v](https://github.com/Vchitect/VBench/tree/master/vbench2_beta_i2v) is a benchmark for short image to video generation (beta version).
-Similarly, install the VBench package following our [installation](../docs/installation.md)'s sections of "Evaluation Dependencies". Then, run the following commands to evaluate the generated samples.
+
```bash
# Step 1: generate the relevant videos
@@ -76,12 +80,12 @@ bash eval/sample.sh /path/to/ckpt num_frames model_name_for_log -5a
bash eval/vbench_i2v/launch.sh /path/to/ckpt num_frames model_name
# Step 2: run vbench to evaluate the generated samples
-python eval/vbench_i2v/vbench_i2v.py
+python eval/vbench_i2v/vbench_i2v.py /path/to/video_folder /path/to/model/ckpt
+# Note that if you need to go to `VBench/vbench2_beta_i2v/utils.py` and change the harded-coded var `image_root` in the `load_i2v_dimension_info` function to your corresponding image folder.
# Step 3: obtain the scaled scores
-def load_i2v_dimension_info(json_dir, dimension, lang, resolution):
-# Note that if you need to go to `your_conda_env_path/lib/python3.x/site-packages/vbench2_beta_i2v/utils.py` and change the harded-coded var `image_root` in the `load_i2v_dimension_info` function to your appropriate image folder.
-python eval/vbench_i2v/tabulate_vbench_i2v_scores.py --score_dir path/to/evaluation_results/dir
+python eval/vbench_i2v/tabulate_vbench_i2v_scores.py path/to/videos/folder path/to/your/model/ckpt
+# this will store the results under `eval/vbench_i2v` in the path/to/your/model/ckpt
```
diff --git a/eval/loss/tabulate_rl_loss.py b/eval/loss/tabulate_rl_loss.py
index d9b3a19..48c930e 100644
--- a/eval/loss/tabulate_rl_loss.py
+++ b/eval/loss/tabulate_rl_loss.py
@@ -15,7 +15,6 @@ from ast import literal_eval
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--log_dir", type=str)
- parser.add_argument("--ckpt_name", type=str)
args = parser.parse_args()
return args
@@ -30,7 +29,8 @@ if __name__ == "__main__":
os.makedirs(output_dir)
files = os.listdir(args.log_dir)
- files = ["img.log", "video_144p.log", "video_240p.log", "video_360p.log", "video_480p.log", "video_720p.log"]
+ # files = ["img.log", "video_144p.log", "video_240p.log", "video_360p.log", "video_480p.log", "video_720p.log"]
+ files = ["img.log", "144p_vid.log", "240p_vid.log", "360p_vid.log", "480p_vid.log", "720p_vid.log"]
loss_info = {}
@@ -47,7 +47,7 @@ if __name__ == "__main__":
loss_info[resolution][frame] = format(loss, ".4f")
# Convert and write JSON object to file
- output_file_path = os.path.join(output_dir, args.ckpt_name + "_loss.json")
+ output_file_path = os.path.join(output_dir, "loss.json")
with open(output_file_path, "w") as outfile:
json.dump(loss_info, outfile, indent=4, sort_keys=True)
print(f"results saved to: {output_file_path}")
diff --git a/eval/vbench/vbench.sh b/eval/vbench/vbench.sh
index 3e1ff49..9b66791 100644
--- a/eval/vbench/vbench.sh
+++ b/eval/vbench/vbench.sh
@@ -5,7 +5,7 @@ set -e
# Base path for videos
videos_path=$1
model_path=$2
-output_path=${model_path}/eval/vbench
+output_path=$(dirname $model_path)/eval/vbench
mkdir -p ${output_path}
# videos_base=$(basename $videos_path)
json_path=./eval/vbench/VBench_full_info.json
diff --git a/eval/vbench_i2v/calc_vbench_i2v.py b/eval/vbench_i2v/calc_vbench_i2v.py
new file mode 100644
index 0000000..fbbdd11
--- /dev/null
+++ b/eval/vbench_i2v/calc_vbench_i2v.py
@@ -0,0 +1,45 @@
+import argparse
+import os
+
+from vbench import VBench
+from vbench2_beta_i2v import VBenchI2V
+
+full_info_path = "eval/vbench_i2v/vbench2_i2v_full_info.json"
+video_quality_dimensions = [
+ "subject_consistency",
+ "background_consistency",
+ "motion_smoothness",
+ "dynamic_degree",
+ "aesthetic_quality",
+ "imaging_quality",
+ "temporal_flickering",
+]
+i2v_dimensions = ["i2v_subject", "i2v_background", "camera_motion"]
+
+
+def parse_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("video_folder", type=str) # samples/samples..._vbench_i2v/
+ parser.add_argument("model_ckpt", type=str)
+ args = parser.parse_args()
+ return args
+
+
+if __name__ == "__main__":
+ args = parse_args()
+ output_dir = os.path.join(args.model_ckpt, "eval", "vbench_i2v")
+ os.makedirs(output_dir, exist_ok=True)
+ video_path = args.video_folder
+
+ my_VBench_I2V = VBenchI2V("cuda", full_info_path, output_dir)
+ for i2v_dim in i2v_dimensions:
+ my_VBench_I2V.evaluate(videos_path=video_path, name=i2v_dim, dimension_list=[i2v_dim], resolution="1-1")
+
+ kwargs = {}
+ kwargs["imaging_quality_preprocessing_mode"] = "longer" # use VBench/evaluate.py default
+
+ my_VBench = VBench("cuda", full_info_path, output_dir)
+ for quality_dim in video_quality_dimensions:
+ my_VBench.evaluate(
+ videos_path=video_path, name=quality_dim, dimension_list=[quality_dim], mode="vbench_standard", **kwargs
+ )
diff --git a/eval/vbench_i2v/tabulate_vbench_i2v_scores.py b/eval/vbench_i2v/tabulate_vbench_i2v_scores.py
index 758171c..fb6a518 100644
--- a/eval/vbench_i2v/tabulate_vbench_i2v_scores.py
+++ b/eval/vbench_i2v/tabulate_vbench_i2v_scores.py
@@ -6,44 +6,44 @@ I2V_WEIGHT = 1.0
I2V_QUALITY_WEIGHT = 1.0
I2V_LIST = [
- "Video-Image Subject Consistency",
- "Video-Image Background Consistency",
+ "i2v_subject",
+ "i2v_background",
]
I2V_QUALITY_LIST = [
- "Subject Consistency",
- "Background Consistency",
- "Motion Smoothness",
- "Dynamic Degree",
- "Aesthetic Quality",
- "Imaging Quality",
- "Temporal Flickering",
+ "subject_consistency",
+ "background_consistency",
+ "temporal_flickering",
+ "motion_smoothness",
+ "aesthetic_quality",
+ "imaging_quality",
+ "dynamic_degree",
]
DIM_WEIGHT_I2V = {
- "Video-Text Camera Motion": 0.1,
- "Video-Image Subject Consistency": 1,
- "Video-Image Background Consistency": 1,
- "Subject Consistency": 1,
- "Background Consistency": 1,
- "Motion Smoothness": 1,
- "Dynamic Degree": 0.5,
- "Aesthetic Quality": 1,
- "Imaging Quality": 1,
- "Temporal Flickering": 1,
+ "camera_motion": 0.1,
+ "i2v_subject": 1,
+ "i2v_background_consistency": 1,
+ "subject_consistency": 1,
+ "background_consistency": 1,
+ "motion_smoothness": 1,
+ "dynamic_degree": 0.5,
+ "aesthetic_quality": 1,
+ "imaging_quality": 1,
+ "temporal_flickering": 1,
}
NORMALIZE_DIC_I2V = {
- "Video-Text Camera Motion": {"Min": 0.0, "Max": 1.0},
- "Video-Image Subject Consistency": {"Min": 0.1462, "Max": 1.0},
- "Video-Image Background Consistency": {"Min": 0.2615, "Max": 1.0},
- "Subject Consistency": {"Min": 0.1462, "Max": 1.0},
- "Background Consistency": {"Min": 0.2615, "Max": 1.0},
- "Motion Smoothness": {"Min": 0.7060, "Max": 0.9975},
- "Dynamic Degree": {"Min": 0.0, "Max": 1.0},
- "Aesthetic Quality": {"Min": 0.0, "Max": 1.0},
- "Imaging Quality": {"Min": 0.0, "Max": 1.0},
- "Temporal Flickering": {"Min": 0.6293, "Max": 1.0},
+ "camera_motion": {"Min": 0.0, "Max": 1.0},
+ "i2v_subject": {"Min": 0.1462, "Max": 1.0},
+ "i2v_background": {"Min": 0.2615, "Max": 1.0},
+ "subject_consistency": {"Min": 0.1462, "Max": 1.0},
+ "background_consistency": {"Min": 0.2615, "Max": 1.0},
+ "motion_smoothness": {"Min": 0.7060, "Max": 0.9975},
+ "dynamic_degree": {"Min": 0.0, "Max": 1.0},
+ "aesthetic_quality": {"Min": 0.0, "Max": 1.0},
+ "imaging_quality": {"Min": 0.0, "Max": 1.0},
+ "temporal_flickering": {"Min": 0.6293, "Max": 1.0},
}
diff --git a/eval/vbench_i2v/vbench_i2v.py b/eval/vbench_i2v/vbench_i2v.py
deleted file mode 100644
index ac82ab9..0000000
--- a/eval/vbench_i2v/vbench_i2v.py
+++ /dev/null
@@ -1,41 +0,0 @@
-import argparse
-import os
-
-from vbench2_beta_i2v import VBenchI2V
-
-FULL_INFO_PATH = "eval/vbench_i2v/vbench2_i2v_full_info.json"
-VIDEO_QUALITY_DIMENSIONS = [
- "subject_consistency",
- "background_consistency",
- "motion_smoothness",
- "dynamic_degree",
- "aesthetic_quality",
- "imaging_quality",
- "temporal_flickering",
-]
-I2V_DIMENSIONS = ["i2v_subject", "i2v_background", "camera_motion"]
-
-
-def parse_args():
- parser = argparse.ArgumentParser()
- parser.add_argument("video_folder", type=str) # samples/samples..._vbench_i2v/
- parser.add_argument("model_ckpt", type=str)
- args = parser.parse_args()
- return args
-
-
-if __name__ == "__main__":
- args = parse_args()
- output_dir = os.path.join(args.model_ckpt, "eval", "vbench_i2v")
- os.makedirs(output_dir, exist_ok=True)
- video_path = args.video_folder
-
- my_VBench_I2V = VBenchI2V("cuda", FULL_INFO_PATH, output_dir)
- my_VBench_I2V.evaluate(videos_path=video_path, name="vbench_i2v", dimension_list=I2V_DIMENSIONS, resolution="1-1")
-
- # my_VBench = VBench("cuda", FULL_INFO_PATH, output_dir)
- # my_VBench.evaluate(
- # videos_path=video_path,
- # name="vbench_video_quality",
- # dimension_list=VIDEO_QUALITY_DIMENSIONS,
- # )