Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

update installation

zhengzangw509f4cd36b09
diff --git a/configs/opensora-v1-2/train/stage1.py b/configs/opensora-v1-2/train/stage1.py
index edef68a..faea0e3 100644
--- a/configs/opensora-v1-2/train/stage1.py
+++ b/configs/opensora-v1-2/train/stage1.py
@@ -4,7 +4,7 @@ dataset = dict(
transform_name="resize_crop",
)
-# h800
+# backup
# bucket_config = { # 20s/it
# "144p": {1: (1.0, 100), 51: (1.0, 30), 102: (1.0, 20), 204: (1.0, 8), 408: (1.0, 4)},
# # ---
diff --git a/configs/opensora-v1-2/train/stage2.py b/configs/opensora-v1-2/train/stage2.py
new file mode 100644
index 0000000..4ad4dce
--- /dev/null
+++ b/configs/opensora-v1-2/train/stage2.py
@@ -0,0 +1,90 @@
+# Dataset settings
+dataset = dict(
+ type="VariableVideoTextDataset",
+ transform_name="resize_crop",
+)
+
+# webvid
+bucket_config = { # 12s/it
+ "144p": {1: (1.0, 475), 51: (1.0, 51), 102: ((1.0, 0.33), 27), 204: ((1.0, 0.1), 13), 408: ((1.0, 0.1), 6)},
+ # ---
+ "256": {1: (0.4, 297), 51: (0.5, 20), 102: ((0.5, 0.33), 10), 204: ((0.5, 0.1), 5), 408: ((0.5, 0.1), 2)},
+ "240p": {1: (0.3, 297), 51: (0.4, 20), 102: ((0.4, 0.33), 10), 204: ((0.4, 0.1), 5), 408: ((0.4, 0.1), 2)},
+ # ---
+ "360p": {1: (0.2, 141), 51: (0.15, 8), 102: ((0.15, 0.33), 4), 204: ((0.15, 0.1), 2), 408: ((0.15, 0.1), 1)},
+ "512": {1: (0.2, 141), 51: (0.15, 8), 102: ((0.15, 0.33), 4), 204: ((0.15, 0.1), 2), 408: ((0.15, 0.1), 1)},
+ # ---
+ "480p": {1: (0.1, 89), 51: (0.1, 5), 102: (0.1, 2), 204: (0.1, 1)},
+ # ---
+ "720p": {1: (0.05, 36), 51: (0.1, 1)},
+ "1024": {1: (0.05, 36), 51: (0.1, 1)},
+ # ---
+ "1080p": {1: (0.1, 5)},
+ # ---
+ "2048": {1: (0.1, 5)},
+}
+
+grad_checkpoint = True
+
+# Acceleration settings
+num_workers = 8
+num_bucket_build_workers = 16
+dtype = "bf16"
+plugin = "zero2"
+
+# Model settings
+model = dict(
+ type="STDiT3-XL/2",
+ from_pretrained=None,
+ qk_norm=True,
+ enable_flash_attn=True,
+ enable_layernorm_kernel=True,
+ freeze_y_embedder=True,
+)
+vae = dict(
+ type="OpenSoraVAE_V1_2",
+ from_pretrained="/mnt/jfs/sora_checkpoints/vae-pipeline",
+ micro_frame_size=17,
+ micro_batch_size=4,
+)
+text_encoder = dict(
+ type="t5",
+ from_pretrained="DeepFloyd/t5-v1_1-xxl",
+ model_max_length=300,
+ shardformer=True,
+ local_files_only=True,
+)
+scheduler = dict(
+ type="rflow",
+ use_timestep_transform=True,
+ sample_method="logit-normal",
+)
+
+# Mask settings
+mask_ratios = {
+ "random": 0.05,
+ "intepolate": 0.005,
+ "quarter_random": 0.005,
+ "quarter_head": 0.005,
+ "quarter_tail": 0.005,
+ "quarter_head_tail": 0.005,
+ "image_random": 0.025,
+ "image_head": 0.05,
+ "image_tail": 0.025,
+ "image_head_tail": 0.025,
+}
+
+# Log settings
+seed = 42
+outputs = "outputs"
+wandb = False
+epochs = 1000
+log_every = 10
+ckpt_every = 200
+
+# optimization settings
+load = None
+grad_clip = 1.0
+lr = 1e-4
+ema_decay = 0.99
+adam_eps = 1e-15
diff --git a/docs/installation.md b/docs/installation.md
index 2a81261..21027f9 100644
--- a/docs/installation.md
+++ b/docs/installation.md
@@ -26,6 +26,7 @@ pip install -r requirements/requirements-cu121.txt
```
If you are using different CUDA versions, you need to manually install `torch`, `torchvision` and `xformers`. You can find the compatible distributions according to the links below.
+
- PyTorch: choose install commands from [PyTorch installation page](https://pytorch.org/get-started/locally/) based on your own CUDA version.
- xformers: choose install commands from [xformers repo](https://github.com/facebookresearch/xformers?tab=readme-ov-file#installing-xformers) based on your own CUDA version.
@@ -57,12 +58,10 @@ pip install flash-attn --no-build-isolation
pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" git+https://github.com/NVIDIA/apex.git
```
-
-
## Evaluation
+### Step 1: Install Requirements
-## Step 1: Install Requirements
To conduct evaluation, run the following command to install requirements:
```bash
@@ -70,7 +69,8 @@ pip install -v .[eval]
# For development:`pip install -v -e .[eval]`
```
-## Step 2: Install VBench
+### Step 2: Install VBench
+
You need to manually install [VBench](https://github.com/Vchitect/VBench):
```bash
@@ -79,7 +79,7 @@ pip install --no-deps vbench==0.1.1
export PATH="/path/to/vbench:$PATH"
```
-#### Step 3: Install `cupy` for Potential VAE Errors
+### Step 3: Install `cupy` for Potential VAE Errors
You need to mannually install [cupy](https://docs.cupy.dev/en/stable/install.html).
@@ -95,10 +95,8 @@ import torchvision.transforms.functional_tensor as F_t
import torchvision.transforms._functional_tensor as F_t
```
-
## Data Processing
-
### Step 1: Install Requirements
First, run the following command to install requirements:
diff --git a/requirements/requirements-eval.txt b/requirements/requirements-eval.txt
index 31dd449..9a70aa6 100644
--- a/requirements/requirements-eval.txt
+++ b/requirements/requirements-eval.txt
@@ -3,9 +3,9 @@ detectron2 @ git+https://github.com/facebookresearch/detectron2.git@ff53992
imageio>=2.34.1
pyiqa==0.1.10
scikit-learn>=1.4.2
-scikit-image==0.23.2
+scikit-image>=0.20.0
lvis==0.5.3
-boto3==1.34.113
+boto3>=1.34.113
# [vae]
decord==0.6.0
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 78eb65a..d8771c5 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -9,7 +9,7 @@ accelerate==0.29.2 # for t5
av>=12.0.0 # for video loading
# [gradio]
-gradio>=4.31.3
+gradio>=4.26.0
spaces>=0.28.3
# [notebook]