Open-Sora: Democratizing Efficient Video Production for All
Python
update installation
zhengzangw509f4cd36b09
diff --git a/configs/opensora-v1-2/train/stage1.py b/configs/opensora-v1-2/train/stage1.pyindex edef68a..faea0e3 100644--- a/configs/opensora-v1-2/train/stage1.py+++ b/configs/opensora-v1-2/train/stage1.py@@ -4,7 +4,7 @@ dataset = dict(transform_name="resize_crop",)-# h800+# backup# bucket_config = { # 20s/it# "144p": {1: (1.0, 100), 51: (1.0, 30), 102: (1.0, 20), 204: (1.0, 8), 408: (1.0, 4)},# # ---diff --git a/configs/opensora-v1-2/train/stage2.py b/configs/opensora-v1-2/train/stage2.pynew file mode 100644index 0000000..4ad4dce--- /dev/null+++ b/configs/opensora-v1-2/train/stage2.py@@ -0,0 +1,90 @@+# Dataset settings+dataset = dict(+ type="VariableVideoTextDataset",+ transform_name="resize_crop",+)++# webvid+bucket_config = { # 12s/it+ "144p": {1: (1.0, 475), 51: (1.0, 51), 102: ((1.0, 0.33), 27), 204: ((1.0, 0.1), 13), 408: ((1.0, 0.1), 6)},+ # ---+ "256": {1: (0.4, 297), 51: (0.5, 20), 102: ((0.5, 0.33), 10), 204: ((0.5, 0.1), 5), 408: ((0.5, 0.1), 2)},+ "240p": {1: (0.3, 297), 51: (0.4, 20), 102: ((0.4, 0.33), 10), 204: ((0.4, 0.1), 5), 408: ((0.4, 0.1), 2)},+ # ---+ "360p": {1: (0.2, 141), 51: (0.15, 8), 102: ((0.15, 0.33), 4), 204: ((0.15, 0.1), 2), 408: ((0.15, 0.1), 1)},+ "512": {1: (0.2, 141), 51: (0.15, 8), 102: ((0.15, 0.33), 4), 204: ((0.15, 0.1), 2), 408: ((0.15, 0.1), 1)},+ # ---+ "480p": {1: (0.1, 89), 51: (0.1, 5), 102: (0.1, 2), 204: (0.1, 1)},+ # ---+ "720p": {1: (0.05, 36), 51: (0.1, 1)},+ "1024": {1: (0.05, 36), 51: (0.1, 1)},+ # ---+ "1080p": {1: (0.1, 5)},+ # ---+ "2048": {1: (0.1, 5)},+}++grad_checkpoint = True++# Acceleration settings+num_workers = 8+num_bucket_build_workers = 16+dtype = "bf16"+plugin = "zero2"++# Model settings+model = dict(+ type="STDiT3-XL/2",+ from_pretrained=None,+ qk_norm=True,+ enable_flash_attn=True,+ enable_layernorm_kernel=True,+ freeze_y_embedder=True,+)+vae = dict(+ type="OpenSoraVAE_V1_2",+ from_pretrained="/mnt/jfs/sora_checkpoints/vae-pipeline",+ micro_frame_size=17,+ micro_batch_size=4,+)+text_encoder = dict(+ type="t5",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",+ model_max_length=300,+ shardformer=True,+ local_files_only=True,+)+scheduler = dict(+ type="rflow",+ use_timestep_transform=True,+ sample_method="logit-normal",+)++# Mask settings+mask_ratios = {+ "random": 0.05,+ "intepolate": 0.005,+ "quarter_random": 0.005,+ "quarter_head": 0.005,+ "quarter_tail": 0.005,+ "quarter_head_tail": 0.005,+ "image_random": 0.025,+ "image_head": 0.05,+ "image_tail": 0.025,+ "image_head_tail": 0.025,+}++# Log settings+seed = 42+outputs = "outputs"+wandb = False+epochs = 1000+log_every = 10+ckpt_every = 200++# optimization settings+load = None+grad_clip = 1.0+lr = 1e-4+ema_decay = 0.99+adam_eps = 1e-15diff --git a/docs/installation.md b/docs/installation.mdindex 2a81261..21027f9 100644--- a/docs/installation.md+++ b/docs/installation.md@@ -26,6 +26,7 @@ pip install -r requirements/requirements-cu121.txt```If you are using different CUDA versions, you need to manually install `torch`, `torchvision` and `xformers`. You can find the compatible distributions according to the links below.+- PyTorch: choose install commands from [PyTorch installation page](https://pytorch.org/get-started/locally/) based on your own CUDA version.- xformers: choose install commands from [xformers repo](https://github.com/facebookresearch/xformers?tab=readme-ov-file#installing-xformers) based on your own CUDA version.@@ -57,12 +58,10 @@ pip install flash-attn --no-build-isolationpip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" git+https://github.com/NVIDIA/apex.git```--## Evaluation+### Step 1: Install Requirements-## Step 1: Install RequirementsTo conduct evaluation, run the following command to install requirements:```bash@@ -70,7 +69,8 @@ pip install -v .[eval]# For development:`pip install -v -e .[eval]````-## Step 2: Install VBench+### Step 2: Install VBench+You need to manually install [VBench](https://github.com/Vchitect/VBench):```bash@@ -79,7 +79,7 @@ pip install --no-deps vbench==0.1.1export PATH="/path/to/vbench:$PATH"```-#### Step 3: Install `cupy` for Potential VAE Errors+### Step 3: Install `cupy` for Potential VAE ErrorsYou need to mannually install [cupy](https://docs.cupy.dev/en/stable/install.html).@@ -95,10 +95,8 @@ import torchvision.transforms.functional_tensor as F_timport torchvision.transforms._functional_tensor as F_t```-## Data Processing-### Step 1: Install RequirementsFirst, run the following command to install requirements:diff --git a/requirements/requirements-eval.txt b/requirements/requirements-eval.txtindex 31dd449..9a70aa6 100644--- a/requirements/requirements-eval.txt+++ b/requirements/requirements-eval.txt@@ -3,9 +3,9 @@ detectron2 @ git+https://github.com/facebookresearch/detectron2.git@ff53992imageio>=2.34.1pyiqa==0.1.10scikit-learn>=1.4.2-scikit-image==0.23.2+scikit-image>=0.20.0lvis==0.5.3-boto3==1.34.113+boto3>=1.34.113# [vae]decord==0.6.0diff --git a/requirements/requirements.txt b/requirements/requirements.txtindex 78eb65a..d8771c5 100644--- a/requirements/requirements.txt+++ b/requirements/requirements.txt@@ -9,7 +9,7 @@ accelerate==0.29.2 # for t5av>=12.0.0 # for video loading# [gradio]-gradio>=4.31.3+gradio>=4.26.0spaces>=0.28.3# [notebook]