Official repository for LTX-Video
Python diffusion-modelsditimage-to-videoimage-to-video-generationtext-to-videotext-to-video-generation
Inference: Support distilled model and update default params
Yoav HaCoheneab705c6493a
diff --git a/README.md b/README.mdindex ae2e664..f71642e 100644--- a/README.md+++ b/README.md@@ -31,7 +31,7 @@ This is the official repository for LTX-Video.# IntroductionLTX-Video is the first DiT-based video generation model that can generate high-quality videos in *real-time*.-It can generate 24 FPS videos at 768x512 resolution, faster than it takes to watch them.+It can generate 30 FPS videos at 1216×704 resolution, faster than it takes to watch them.The model is trained on a large-scale dataset of diverse videos and can generate high-resolution videoswith realistic and diverse content.@@ -46,6 +46,18 @@ The model supports text-to-image, image-to-video, keyframe-based animation, vide# News+## April, 15th, 2025: New checkpoints v0.9.6:+- Release a new checkpoint [ltxv-2b-0.9.6-dev-04-25](https://huggingface.co/Lightricks/LTX-Video/blob/main/ltxv-2b-0.9.6-dev-04-25.safetensors) with improved quality+- Release a new distilled model [ltxv-2b-0.9.6-distilled-04-25](https://huggingface.co/Lightricks/LTX-Video/blob/main/ltxv-2b-0.9.6-distilled-04-25.safetensors)+ * 15x faster inference than non-distilled model.+ * Does not require classifier-free guidance and spatio-temporal guidance.+ * Supports sampling with 8 (recommended), 4, 2 or 1 diffusion steps.+- Improved prompt adherence, motion quality and fine details.+- New default resolution and FPS: 1216 × 704 pixels at 30 FPS+ * Still real time on H100 with the distilled model.+ * Other resolutions and FPS are still supported.+- Support stochastic inference (can improve visual quality when using the distilled model)+## March, 5th, 2025: New checkpoint v0.9.5- New license for commercial use ([OpenRail-M](https://huggingface.co/Lightricks/LTX-Video/ltx-video-2b-v0.9.5.license.txt))- Release a new checkpoint v0.9.5 with improved quality@@ -111,7 +123,12 @@ Then, download the model from [Hugging Face](https://huggingface.co/Lightricks/Lfrom huggingface_hub import hf_hub_downloadmodel_dir = 'MODEL_DIR' # The local directory to save downloaded checkpoint-hf_hub_download(repo_id="Lightricks/LTX-Video", filename="ltx-video-2b-v0.9.5.safetensors", local_dir=model_dir, local_dir_use_symlinks=False, repo_type='model')++# Distilled model:+hf_hub_download(repo_id="Lightricks/LTX-Video", filename="ltxv-2b-0.9.6-distilled-04-25.safetensors", local_dir=model_dir, local_dir_use_symlinks=False, repo_type='model')++# Full model:+hf_hub_download(repo_id="Lightricks/LTX-Video", filename="ltxv-2b-0.9.6-dev-04-25.safetensors", local_dir=model_dir, local_dir_use_symlinks=False, repo_type='model')```### Inferencediff --git a/inference.py b/inference.pyindex 105bf2a..34e94ba 100644--- a/inference.py+++ b/inference.py@@ -7,8 +7,10 @@ from diffusers.utils import loggingfrom typing import Optional, List, Unionimport imageio+import jsonimport numpy as npimport torch+from safetensors import safe_openfrom PIL import Imagefrom transformers import (T5EncoderModel,@@ -237,13 +239,13 @@ def main():parser.add_argument("--height",type=int,- default=480,+ default=704,help="Height of the output video frames. Optional if an input image provided.",)parser.add_argument("--width",type=int,- default=704,+ default=1216,help="Width of the output video frames. If None will infer from input image.",)parser.add_argument(@@ -253,7 +255,7 @@ def main():help="Number of frames to generate in the output video",)parser.add_argument(- "--frame_rate", type=int, default=25, help="Frame rate for the output video"+ "--frame_rate", type=int, default=30, help="Frame rate for the output video")parser.add_argument("--device",@@ -271,13 +273,13 @@ def main():parser.add_argument("--decode_timestep",type=float,- default=0.025,+ default=0.05,help="Timestep for decoding noise",)parser.add_argument("--decode_noise_scale",type=float,- default=0.0125,+ default=0.025,help="Noise level for decoding noise",)@@ -353,6 +355,11 @@ def main():default="unsloth/Llama-3.2-3B-Instruct",help="Path to the LLM model, default is Llama-3.2-3B-Instruct, but you can use other models like Llama-3.1-8B-Instruct, or other models supported by Hugging Face",)+ parser.add_argument(+ "--stochastic_sampling",+ action="store_true",+ help="Runs inference with stochastic sampling.",+ )args = parser.parse_args()logger.warning(f"Running generation with arguments: {args}")@@ -373,6 +380,13 @@ def create_ltx_video_pipeline(assert os.path.exists(ckpt_path), f"Ckpt path provided (--ckpt_path) {ckpt_path} does not exist"++ with safe_open(ckpt_path, framework="pt") as f:+ metadata = f.metadata()+ config_str = metadata.get("config")+ configs = json.loads(config_str)+ allowed_inference_steps = configs.get("allowed_inference_steps", None)+vae = CausalVideoAutoencoder.from_pretrained(ckpt_path)transformer = Transformer3DModel.from_pretrained(ckpt_path)@@ -433,6 +447,7 @@ def create_ltx_video_pipeline("prompt_enhancer_image_caption_processor": prompt_enhancer_image_caption_processor,"prompt_enhancer_llm_model": prompt_enhancer_llm_model,"prompt_enhancer_llm_tokenizer": prompt_enhancer_llm_tokenizer,+ "allowed_inference_steps": allowed_inference_steps,}pipeline = LTXVideoPipeline(**submodel_dict)@@ -471,6 +486,7 @@ def infer(prompt_enhancement_words_threshold: int = 50,prompt_enhancer_image_caption_model_name_or_path: str = "MiaoshouAI/Florence-2-large-PromptGen-v2.0",prompt_enhancer_llm_model_name_or_path: str = "unsloth/Llama-3.2-3B-Instruct",+ stochastic_sampling: bool = False,**kwargs,):if kwargs.get("input_image_path", None):@@ -620,6 +636,7 @@ def infer(offload_to_cpu=offload_to_cpu,device=device,enhance_prompt=enhance_prompt,+ stochastic_sampling=stochastic_sampling,).images# Crop the padded images to the desired resolution and number of framesdiff --git a/ltx_video/pipelines/pipeline_ltx_video.py b/ltx_video/pipelines/pipeline_ltx_video.pyindex 01b23e8..f2f9cc2 100644--- a/ltx_video/pipelines/pipeline_ltx_video.py+++ b/ltx_video/pipelines/pipeline_ltx_video.py@@ -237,6 +237,7 @@ class LTXVideoPipeline(DiffusionPipeline):prompt_enhancer_image_caption_processor: AutoProcessor,prompt_enhancer_llm_model: AutoModelForCausalLM,prompt_enhancer_llm_tokenizer: AutoTokenizer,+ allowed_inference_steps: Optional[List[float]] = None,):super().__init__()@@ -258,6 +259,8 @@ class LTXVideoPipeline(DiffusionPipeline):)self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)+ self.allowed_inference_steps = allowed_inference_steps+def mask_text_embeddings(self, emb, mask):if emb.shape[0] == 1:keep_index = mask.sum().item()@@ -683,6 +686,7 @@ class LTXVideoPipeline(DiffusionPipeline):offload_to_cpu: bool = False,enhance_prompt: bool = False,text_encoder_max_tokens: int = 256,+ stochastic_sampling: bool = False,**kwargs,) -> Union[ImagePipelineOutput, Tuple]:"""@@ -751,6 +755,8 @@ class LTXVideoPipeline(DiffusionPipeline):If set to `True`, the prompt is enhanced using a LLM model.text_encoder_max_tokens (`int`, *optional*, defaults to `256`):The maximum number of tokens to use for the text encoder.+ stochastic_sampling (`bool`, *optional*, defaults to `False`):+ If set to `True`, the sampling is stochastic. If set to `False`, the sampling is deterministic.Examples:@@ -934,6 +940,11 @@ class LTXVideoPipeline(DiffusionPipeline):timesteps,**retrieve_timesteps_kwargs,)+ if self.allowed_inference_steps is not None:+ for timestep in [round(x, 4) for x in timesteps.tolist()]:+ assert (+ timestep in self.allowed_inference_steps+ ), f"Invalid inference timestep {timestep}. Allowed timesteps are {self.allowed_inference_steps}."# 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipelineextra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)@@ -1063,6 +1074,7 @@ class LTXVideoPipeline(DiffusionPipeline):orig_conditioning_mask,t,extra_step_kwargs,+ stochastic_sampling=stochastic_sampling,)# call the callback, if provided@@ -1137,6 +1149,7 @@ class LTXVideoPipeline(DiffusionPipeline):t: float,extra_step_kwargs,t_eps=1e-6,+ stochastic_sampling=False,):"""Perform the denoising step for the required tokens, based on the current timestep and@@ -1153,6 +1166,7 @@ class LTXVideoPipeline(DiffusionPipeline):latents,**extra_step_kwargs,return_dict=False,+ stochastic_sampling=stochastic_sampling,)[0]if conditioning_mask is None:@@ -1236,7 +1250,7 @@ class LTXVideoPipeline(DiffusionPipeline):media_item.to(dtype=self.vae.dtype, device=self.vae.device),self.vae,vae_per_channel_normalize=vae_per_channel_normalize,- ).to(dtype=self.transformer.dtype)+ ).to(dtype=init_latents.dtype)# Handle the different conditioning casesif media_frame_number == 0:diff --git a/ltx_video/schedulers/rf.py b/ltx_video/schedulers/rf.pyindex ed2f6c5..5c56f9e 100644--- a/ltx_video/schedulers/rf.py+++ b/ltx_video/schedulers/rf.py@@ -182,6 +182,7 @@ class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):base_resolution: int = 32**2,target_shift_terminal: Optional[float] = None,sampler: Optional[str] = "Uniform",+ shift: Optional[float] = None,):super().__init__()self.init_noise_sigma = 1.0@@ -190,13 +191,25 @@ class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):self.shifting = shiftingself.base_resolution = base_resolutionself.target_shift_terminal = target_shift_terminal- self.timesteps = self.sigmas = self.get_initial_timesteps(num_train_timesteps)+ self.timesteps = self.sigmas = self.get_initial_timesteps(+ num_train_timesteps, shift=shift+ )+ self.shift = shift- def get_initial_timesteps(self, num_timesteps: int) -> Tensor:+ def get_initial_timesteps(+ self, num_timesteps: int, shift: Optional[float] = None+ ) -> Tensor:if self.sampler == "Uniform":return torch.linspace(1, 1 / num_timesteps, num_timesteps)elif self.sampler == "LinearQuadratic":return linear_quadratic_schedule(num_timesteps)+ elif self.sampler == "Constant":+ assert (+ shift is not None+ ), "Shift must be provided for constant time shift sampler."+ return time_shift(+ shift, 1, torch.linspace(1, 1 / num_timesteps, num_timesteps)+ )def shift_timesteps(self, samples: Tensor, timesteps: Tensor) -> Tensor:if self.shifting == "SD3":@@ -224,7 +237,9 @@ class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):device (`Union[str, torch.device]`, *optional*): The device to which the timesteps tensor will be moved."""num_inference_steps = min(self.config.num_train_timesteps, num_inference_steps)- self.timesteps = self.get_initial_timesteps(num_inference_steps).to(device)+ self.timesteps = self.get_initial_timesteps(+ num_inference_steps, shift=self.shift+ ).to(device)self.timesteps = self.shift_timesteps(samples, self.timesteps)self.num_inference_steps = num_inference_stepsself.sigmas = self.timesteps@@ -277,6 +292,7 @@ class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):timestep: torch.FloatTensor,sample: torch.FloatTensor,return_dict: bool = True,+ stochastic_sampling: Optional[bool] = False,**kwargs,) -> Union[RectifiedFlowSchedulerOutput, Tuple]:"""@@ -295,6 +311,8 @@ class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):A current latent tokens to be de-noised.return_dict (`bool`, *optional*, defaults to `True`):Whether or not to return a [`~schedulers.scheduling_ddim.DDIMSchedulerOutput`] or `tuple`.+ stochastic_sampling (`bool`, *optional*, defaults to `False`):+ Whether to use stochastic sampling for the sampling process.Returns:[`~schedulers.scheduling_utils.RectifiedFlowSchedulerOutput`] or `tuple`:@@ -327,7 +345,12 @@ class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):dt = (timestep - lower_timestep)[..., None]# Compute previous sample- prev_sample = sample - dt * model_output+ if stochastic_sampling:+ x0 = sample - timestep[..., None] * model_output+ next_timestep = timestep[..., None] - dt+ prev_sample = self.add_noise(x0, torch.randn_like(sample), next_timestep)+ else:+ prev_sample = sample - dt * model_outputif not return_dict:return (prev_sample,)diff --git a/pyproject.toml b/pyproject.tomlindex 5452a4d..277e7aa 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -20,7 +20,7 @@ dependencies = ["diffusers>=0.28.2","transformers>=4.47.2","sentencepiece>=0.1.96",- "huggingface-hub~=0.25.2",+ "huggingface-hub~=0.30","einops","timm"]diff --git a/tests/conftest.py b/tests/conftest.pyindex 41d1921..48a95ce 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -13,6 +13,15 @@ def test_paths(request, pytestconfig):)input_image_path = pytestconfig.getoption("input_image_path")input_video_path = pytestconfig.getoption("input_video_path")+ prompt_enhancer_image_caption_model_name_or_path = pytestconfig.getoption(+ "prompt_enhancer_image_caption_model_name_or_path"+ )+ prompt_enhancer_llm_model_name_or_path = pytestconfig.getoption(+ "prompt_enhancer_llm_model_name_or_path"+ )+ prompt_enhancement_words_threshold = pytestconfig.getoption(+ "prompt_enhancement_words_threshold"+ )config = {"ckpt_path": ckpt_path,@@ -20,6 +29,9 @@ def test_paths(request, pytestconfig):"input_video_path": input_video_path,"output_path": output_path,"text_encoder_model_name_or_path": text_encoder_model_name_or_path,+ "prompt_enhancer_image_caption_model_name_or_path": prompt_enhancer_image_caption_model_name_or_path,+ "prompt_enhancer_llm_model_name_or_path": prompt_enhancer_llm_model_name_or_path,+ "prompt_enhancement_words_threshold": prompt_enhancement_words_threshold,}yield config@@ -60,3 +72,21 @@ def pytest_addoption(parser):default="tests/utils/woman.mp4",help="Path to input video file.",)+ parser.addoption(+ "--prompt_enhancer_image_caption_model_name_or_path",+ action="store",+ default="MiaoshouAI/Florence-2-large-PromptGen-v2.0",+ help="Path to prompt_enhancer_image_caption_model.",+ )+ parser.addoption(+ "--prompt_enhancer_llm_model_name_or_path",+ action="store",+ default="unsloth/Llama-3.2-3B-Instruct",+ help="Path to LLM model for prompt enhancement.",+ )+ parser.addoption(+ "--prompt_enhancement_words_threshold",+ type=int,+ default=50,+ help="Enable prompt enhancement only if input prompt has fewer words than this threshold. Set to 0 to disable enhancement completely.",+ )diff --git a/tests/test_inference.py b/tests/test_inference.pyindex cc6789e..7fe5e1f 100644--- a/tests/test_inference.py+++ b/tests/test_inference.py@@ -64,8 +64,6 @@ def test_infer_runs_on_real_path(test_paths, conditioning_test_mode):"negative_prompt": "worst quality, inconsistent motion, blurry, jittery, distorted","offload_to_cpu": False,"prompt_enhancement_max_words": 1000,- "prompt_enhancer_image_caption_model_name_or_path": "MiaoshouAI/Florence-2-large-PromptGen-v2.0",- "prompt_enhancer_llm_model_name_or_path": "unsloth/Llama-3.2-3B-Instruct",}infer(**{**conditioning_params, **test_paths, **params})@@ -99,9 +97,13 @@ def test_pipeline_on_batch(test_paths):device=device,precision="bfloat16",text_encoder_model_name_or_path=test_paths["text_encoder_model_name_or_path"],- enhance_prompt=True,- prompt_enhancer_image_caption_model_name_or_path="MiaoshouAI/Florence-2-large-PromptGen-v2.0",- prompt_enhancer_llm_model_name_or_path="unsloth/Llama-3.2-3B-Instruct",+ enhance_prompt=False,+ prompt_enhancer_image_caption_model_name_or_path=test_paths[+ "prompt_enhancer_image_caption_model_name_or_path"+ ],+ prompt_enhancer_llm_model_name_or_path=test_paths[+ "prompt_enhancer_llm_model_name_or_path"+ ],)params = {@@ -175,8 +177,12 @@ def test_prompt_enhancement(test_paths, monkeypatch):precision="bfloat16",text_encoder_model_name_or_path=test_paths["text_encoder_model_name_or_path"],enhance_prompt=True,- prompt_enhancer_image_caption_model_name_or_path="MiaoshouAI/Florence-2-large-PromptGen-v2.0",- prompt_enhancer_llm_model_name_or_path="unsloth/Llama-3.2-3B-Instruct",+ prompt_enhancer_image_caption_model_name_or_path=test_paths[+ "prompt_enhancer_image_caption_model_name_or_path"+ ],+ prompt_enhancer_llm_model_name_or_path=test_paths[+ "prompt_enhancer_llm_model_name_or_path"+ ],)original_prompt = "A cat sitting on a windowsill"