Open-Sora: Democratizing Efficient Video Production for All
Python
Docs/fix (#211)
Zheng Zangwei (Alex Zheng)b454751f8f02
diff --git a/configs/opensora/train/16x512x512.py b/configs/opensora/train/16x512x512.pyindex 885aad1..22031dc 100644--- a/configs/opensora/train/16x512x512.py+++ b/configs/opensora/train/16x512x512.py@@ -30,7 +30,7 @@ vae = dict()text_encoder = dict(type="t5",- from_pretrained="./pretrained_models/t5_ckpts",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",model_max_length=120,shardformer=True,)diff --git a/configs/opensora/train/360x512x512.py b/configs/opensora/train/360x512x512.pyindex 7a6f759..0f4e170 100644--- a/configs/opensora/train/360x512x512.py+++ b/configs/opensora/train/360x512x512.py@@ -31,7 +31,7 @@ vae = dict()text_encoder = dict(type="t5",- from_pretrained="./pretrained_models/t5_ckpts",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",model_max_length=120,shardformer=True,)diff --git a/configs/opensora/train/64x512x512-sp.py b/configs/opensora/train/64x512x512-sp.pyindex b0b9062..f9b138e 100644--- a/configs/opensora/train/64x512x512-sp.py+++ b/configs/opensora/train/64x512x512-sp.py@@ -30,7 +30,7 @@ vae = dict()text_encoder = dict(type="t5",- from_pretrained="./pretrained_models/t5_ckpts",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",model_max_length=120,shardformer=True,)diff --git a/configs/opensora/train/64x512x512.py b/configs/opensora/train/64x512x512.pyindex dfcdcc0..81162de 100644--- a/configs/opensora/train/64x512x512.py+++ b/configs/opensora/train/64x512x512.py@@ -30,7 +30,7 @@ vae = dict()text_encoder = dict(type="t5",- from_pretrained="./pretrained_models/t5_ckpts",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",model_max_length=120,shardformer=True,)diff --git a/configs/pixart/inference/16x256x256.py b/configs/pixart/inference/16x256x256.pyindex 6fc8ee6..fed26f5 100644--- a/configs/pixart/inference/16x256x256.py+++ b/configs/pixart/inference/16x256x256.py@@ -15,7 +15,7 @@ vae = dict()text_encoder = dict(type="t5",- from_pretrained="./pretrained_models/t5_ckpts",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",model_max_length=120,)scheduler = dict(diff --git a/configs/pixart/inference/1x1024MS.py b/configs/pixart/inference/1x1024MS.pyindex 41cc97a..3b34ebf 100644--- a/configs/pixart/inference/1x1024MS.py+++ b/configs/pixart/inference/1x1024MS.py@@ -17,7 +17,7 @@ vae = dict()text_encoder = dict(type="t5",- from_pretrained="./pretrained_models/t5_ckpts",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",model_max_length=120,)scheduler = dict(diff --git a/configs/pixart/inference/1x256x256.py b/configs/pixart/inference/1x256x256.pyindex 11e06d7..44ce10b 100644--- a/configs/pixart/inference/1x256x256.py+++ b/configs/pixart/inference/1x256x256.py@@ -16,7 +16,7 @@ vae = dict()text_encoder = dict(type="t5",- from_pretrained="./pretrained_models/t5_ckpts",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",model_max_length=120,)scheduler = dict(diff --git a/configs/pixart/inference/1x512x512.py b/configs/pixart/inference/1x512x512.pyindex 5674259..243af4a 100644--- a/configs/pixart/inference/1x512x512.py+++ b/configs/pixart/inference/1x512x512.py@@ -16,7 +16,7 @@ vae = dict()text_encoder = dict(type="t5",- from_pretrained="./pretrained_models/t5_ckpts",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",model_max_length=120,)scheduler = dict(diff --git a/configs/pixart/train/16x256x256.py b/configs/pixart/train/16x256x256.pyindex b47731e..dc91937 100644--- a/configs/pixart/train/16x256x256.py+++ b/configs/pixart/train/16x256x256.py@@ -29,7 +29,7 @@ vae = dict()text_encoder = dict(type="t5",- from_pretrained="./pretrained_models/t5_ckpts",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",model_max_length=120,shardformer=True,)diff --git a/configs/pixart/train/1x512x512.py b/configs/pixart/train/1x512x512.pyindex 619c9aa..ec73b8c 100644--- a/configs/pixart/train/1x512x512.py+++ b/configs/pixart/train/1x512x512.py@@ -30,7 +30,7 @@ vae = dict()text_encoder = dict(type="t5",- from_pretrained="./pretrained_models/t5_ckpts",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",model_max_length=120,shardformer=True,)diff --git a/configs/pixart/train/64x512x512.py b/configs/pixart/train/64x512x512.pyindex 628cf25..deebdf6 100644--- a/configs/pixart/train/64x512x512.py+++ b/configs/pixart/train/64x512x512.py@@ -30,7 +30,7 @@ vae = dict()text_encoder = dict(type="t5",- from_pretrained="./pretrained_models/t5_ckpts",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",model_max_length=120,shardformer=True,)diff --git a/docs/report_v1.md b/docs/report_v1.mdindex b3b8073..edb3644 100644--- a/docs/report_v1.md+++ b/docs/report_v1.md@@ -14,7 +14,7 @@ As shown in the figure, we insert a temporal attention right after each spatialTo focus on video generation, we hope to train the model based on a powerful image generation model. [PixArt-α](https://github.com/PixArt-alpha/PixArt-alpha) is an efficiently trained high-quality image generation model with T5-conditioned DiT structure. We initialize our model with PixArt-α and initialize the projection layer of inserted temporal attention with zero. This initialization preserves model's ability of image generation at beginning, while Latte's architecture cannot. The inserted attention increases the number of parameter from 580M to 724M.-+Drawing from the success of PixArt-α and Stable Video Diffusion, we also adopt a progressive training strategy: 16x256x256 on 366K pretraining datasets, and then 16x256x256, 16x512x512, and 64x512x512 on 20K datasets. With scaled position embedding, this strategy greatly reduces the computational cost.diff --git a/docs/structure.md b/docs/structure.mdindex 2d83620..3eb7c69 100644--- a/docs/structure.md+++ b/docs/structure.md@@ -100,7 +100,7 @@ vae = dict()text_encoder = dict(type="t5", # Select text encoder type (t5, clip)- from_pretrained="./pretrained_models/t5_ckpts", # Load from pretrained text encoder+ from_pretrained="DeepFloyd/t5-v1_1-xxl", # Load from pretrained text encodermodel_max_length=120, # Maximum length of input text)scheduler = dict(@@ -153,7 +153,7 @@ vae = dict()text_encoder = dict(type="t5",- from_pretrained="./pretrained_models/t5_ckpts",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",model_max_length=120,shardformer=True, # Enable shardformer for T5 acceleration)diff --git a/docs/zh_CN/structure.md b/docs/zh_CN/structure.mdindex 2d83620..3eb7c69 100644--- a/docs/zh_CN/structure.md+++ b/docs/zh_CN/structure.md@@ -100,7 +100,7 @@ vae = dict()text_encoder = dict(type="t5", # Select text encoder type (t5, clip)- from_pretrained="./pretrained_models/t5_ckpts", # Load from pretrained text encoder+ from_pretrained="DeepFloyd/t5-v1_1-xxl", # Load from pretrained text encodermodel_max_length=120, # Maximum length of input text)scheduler = dict(@@ -153,7 +153,7 @@ vae = dict()text_encoder = dict(type="t5",- from_pretrained="./pretrained_models/t5_ckpts",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",model_max_length=120,shardformer=True, # Enable shardformer for T5 acceleration)