Open-Sora: Democratizing Efficient Video Production for All
Python
adapted pretrained model to training (#371)
Frank Leed599fd75b323
diff --git a/opensora/models/stdit/stdit2.py b/opensora/models/stdit/stdit2.pyindex afcc0c1..5de1769 100644--- a/opensora/models/stdit/stdit2.py+++ b/opensora/models/stdit/stdit2.py@@ -1,6 +1,7 @@import numpy as npimport torchimport torch.nn as nn+import osfrom einops import rearrangefrom rotary_embedding_torch import RotaryEmbeddingfrom timm.models.layers import DropPath@@ -23,6 +24,7 @@ from opensora.models.layers.blocks import ()from opensora.registry import MODELSfrom transformers import PretrainedConfig, PreTrainedModel+from opensora.utils.ckpt_utils import load_checkpointclass STDiT2Block(nn.Module):@@ -502,8 +504,22 @@ class STDiT2(PreTrainedModel):@MODELS.register_module("STDiT2-XL/2")def STDiT2_XL_2(from_pretrained=None, **kwargs):if from_pretrained is not None:- model = STDiT2.from_pretrained(from_pretrained, **kwargs)+ if os.path.isdir(from_pretrained) or os.path.isfile(from_pretrained):+ # if it is a directory or a file, we load the checkpoint manually+ config = STDiT2Config(+ depth=28,+ hidden_size=1152,+ patch_size=(1, 2, 2),+ num_heads=16, **kwargs+ )+ model = STDiT2(config)+ load_checkpoint(model, from_pretrained)+ return model+ else:+ # otherwise, we load the model from hugging face hub+ return STDiT2.from_pretrained(from_pretrained)else:+ # create a new modelconfig = STDiT2Config(depth=28,hidden_size=1152,diff --git a/scripts/train.py b/scripts/train.pyindex aed7b45..bfb6d39 100644--- a/scripts/train.py+++ b/scripts/train.py@@ -133,8 +133,7 @@ def main():input_size=latent_size,in_channels=vae.out_channels,caption_channels=text_encoder.output_dim,- model_max_length=text_encoder.model_max_length,- dtype=dtype,+ model_max_length=text_encoder.model_max_length)model_numel, model_numel_trainable = get_model_numel(model)logger.info(