Official repository for LTX-Video
Python diffusion-modelsditimage-to-videoimage-to-video-generationtext-to-videotext-to-video-generation
Inference: Load checkpoint from single safetensors file.
Sapir Weissbuchee2488cc096b
diff --git a/README.md b/README.mdindex aac777a..f2b8cbf 100644--- a/README.md+++ b/README.md@@ -76,13 +76,13 @@ To use our model, please follow the inference code in [inference.py](./inference#### For text-to-video generation:```bash-python inference.py --ckpt_dir 'PATH' --prompt "PROMPT" --height HEIGHT --width WIDTH --num_frames NUM_FRAMES --seed SEED+python inference.py --ckpt_path 'PATH' --prompt "PROMPT" --height HEIGHT --width WIDTH --num_frames NUM_FRAMES --seed SEED```#### For image-to-video generation:```bash-python inference.py --ckpt_dir 'PATH' --prompt "PROMPT" --input_image_path IMAGE_PATH --height HEIGHT --width WIDTH --num_frames NUM_FRAMES --seed SEED+python inference.py --ckpt_path 'PATH' --prompt "PROMPT" --input_image_path IMAGE_PATH --height HEIGHT --width WIDTH --num_frames NUM_FRAMES --seed SEED```## ComfyUI Integrationdiff --git a/inference.py b/inference.pyindex dcf0683..5e111cd 100644--- a/inference.py+++ b/inference.py@@ -8,7 +8,7 @@ from diffusers.utils import loggingimport imageioimport numpy as np-import safetensors.torch+from safetensors import safe_openimport torchimport torch.nn.functional as Ffrom PIL import Image@@ -29,34 +29,33 @@ MAX_WIDTH = 1280MAX_NUM_FRAMES = 257-def load_vae(vae_dir):- vae_ckpt_path = vae_dir / "vae_diffusion_pytorch_model.safetensors"- vae_config_path = vae_dir / "config.json"- with open(vae_config_path, "r") as f:- vae_config = json.load(f)+def load_vae(vae_config, ckpt):vae = CausalVideoAutoencoder.from_config(vae_config)- vae_state_dict = safetensors.torch.load_file(vae_ckpt_path)+ vae_state_dict = {+ key.replace("vae.", ""): value+ for key, value in ckpt.items()+ if key.startswith("vae.")+ }vae.load_state_dict(vae_state_dict)if torch.cuda.is_available():vae = vae.cuda()return vae.to(torch.bfloat16)-def load_unet(unet_dir):- unet_ckpt_path = unet_dir / "unet_diffusion_pytorch_model.safetensors"- unet_config_path = unet_dir / "config.json"- transformer_config = Transformer3DModel.load_config(unet_config_path)+def load_transformer(transformer_config, ckpt):transformer = Transformer3DModel.from_config(transformer_config)- unet_state_dict = safetensors.torch.load_file(unet_ckpt_path)- transformer.load_state_dict(unet_state_dict, strict=True)+ transformer_state_dict = {+ key.replace("model.diffusion_model.", ""): value+ for key, value in ckpt.items()+ if key.startswith("model.diffusion_model.")+ }+ transformer.load_state_dict(transformer_state_dict, strict=True)if torch.cuda.is_available():transformer = transformer.cuda()return transformer-def load_scheduler(scheduler_dir):- scheduler_config_path = scheduler_dir / "scheduler_config.json"- scheduler_config = RectifiedFlowScheduler.load_config(scheduler_config_path)+def load_scheduler(scheduler_config):return RectifiedFlowScheduler.from_config(scheduler_config)@@ -168,10 +167,10 @@ def main():# Directoriesparser.add_argument(- "--ckpt_dir",+ "--ckpt_path",type=str,required=True,- help="Path to the directory containing unet, vae, and scheduler subdirectories",+ help="Path to a safetensors file that contains all model parts.",)parser.add_argument("--input_video_path",@@ -303,15 +302,22 @@ def main():media_items = None# Paths for the separate mode directories- ckpt_dir = Path(args.ckpt_dir)- unet_dir = ckpt_dir / "unet"- vae_dir = ckpt_dir / "vae"- scheduler_dir = ckpt_dir / "scheduler"+ ckpt_path = Path(args.ckpt_path)+ ckpt = {}+ with safe_open(ckpt_path, framework="pt", device="cpu") as f:+ metadata = f.metadata()+ for k in f.keys():+ ckpt[k] = f.get_tensor(k)++ configs = json.loads(metadata["config"])+ vae_config = configs["vae"]+ transformer_config = configs["transformer"]+ scheduler_config = configs["scheduler"]# Load models- vae = load_vae(vae_dir)- unet = load_unet(unet_dir)- scheduler = load_scheduler(scheduler_dir)+ vae = load_vae(vae_config, ckpt)+ transformer = load_transformer(transformer_config, ckpt)+ scheduler = load_scheduler(scheduler_config)patchifier = SymmetricPatchifier(patch_size=1)text_encoder = T5EncoderModel.from_pretrained("PixArt-alpha/PixArt-XL-2-1024-MS", subfolder="text_encoder"@@ -322,12 +328,12 @@ def main():"PixArt-alpha/PixArt-XL-2-1024-MS", subfolder="tokenizer")- if args.bfloat16 and unet.dtype != torch.bfloat16:- unet = unet.to(torch.bfloat16)+ if args.bfloat16 and transformer.dtype != torch.bfloat16:+ transformer = transformer.to(torch.bfloat16)# Use submodels for the pipelinesubmodel_dict = {- "transformer": unet,+ "transformer": transformer,"patchifier": patchifier,"text_encoder": text_encoder,"tokenizer": tokenizer,