Skip to content

Official repository for LTX-Video

Inference: Support distilled model and update default params

Yoav HaCoheneab705c6493a
diff --git a/README.md b/README.md
index ae2e664..f71642e 100644
--- a/README.md
+++ b/README.md
@@ -31,7 +31,7 @@ This is the official repository for LTX-Video.
# Introduction
LTX-Video is the first DiT-based video generation model that can generate high-quality videos in *real-time*.
-It can generate 24 FPS videos at 768x512 resolution, faster than it takes to watch them.
+It can generate 30 FPS videos at 1216×704 resolution, faster than it takes to watch them.
The model is trained on a large-scale dataset of diverse videos and can generate high-resolution videos
with realistic and diverse content.
@@ -46,6 +46,18 @@ The model supports text-to-image, image-to-video, keyframe-based animation, vide
# News
+## April, 15th, 2025: New checkpoints v0.9.6:
+- Release a new checkpoint [ltxv-2b-0.9.6-dev-04-25](https://huggingface.co/Lightricks/LTX-Video/blob/main/ltxv-2b-0.9.6-dev-04-25.safetensors) with improved quality
+- Release a new distilled model [ltxv-2b-0.9.6-distilled-04-25](https://huggingface.co/Lightricks/LTX-Video/blob/main/ltxv-2b-0.9.6-distilled-04-25.safetensors)
+ * 15x faster inference than non-distilled model.
+ * Does not require classifier-free guidance and spatio-temporal guidance.
+ * Supports sampling with 8 (recommended), 4, 2 or 1 diffusion steps.
+- Improved prompt adherence, motion quality and fine details.
+- New default resolution and FPS: 1216 × 704 pixels at 30 FPS
+ * Still real time on H100 with the distilled model.
+ * Other resolutions and FPS are still supported.
+- Support stochastic inference (can improve visual quality when using the distilled model)
+
## March, 5th, 2025: New checkpoint v0.9.5
- New license for commercial use ([OpenRail-M](https://huggingface.co/Lightricks/LTX-Video/ltx-video-2b-v0.9.5.license.txt))
- Release a new checkpoint v0.9.5 with improved quality
@@ -111,7 +123,12 @@ Then, download the model from [Hugging Face](https://huggingface.co/Lightricks/L
from huggingface_hub import hf_hub_download
model_dir = 'MODEL_DIR' # The local directory to save downloaded checkpoint
-hf_hub_download(repo_id="Lightricks/LTX-Video", filename="ltx-video-2b-v0.9.5.safetensors", local_dir=model_dir, local_dir_use_symlinks=False, repo_type='model')
+
+# Distilled model:
+hf_hub_download(repo_id="Lightricks/LTX-Video", filename="ltxv-2b-0.9.6-distilled-04-25.safetensors", local_dir=model_dir, local_dir_use_symlinks=False, repo_type='model')
+
+# Full model:
+hf_hub_download(repo_id="Lightricks/LTX-Video", filename="ltxv-2b-0.9.6-dev-04-25.safetensors", local_dir=model_dir, local_dir_use_symlinks=False, repo_type='model')
```
### Inference
diff --git a/inference.py b/inference.py
index 105bf2a..34e94ba 100644
--- a/inference.py
+++ b/inference.py
@@ -7,8 +7,10 @@ from diffusers.utils import logging
from typing import Optional, List, Union
import imageio
+import json
import numpy as np
import torch
+from safetensors import safe_open
from PIL import Image
from transformers import (
T5EncoderModel,
@@ -237,13 +239,13 @@ def main():
parser.add_argument(
"--height",
type=int,
- default=480,
+ default=704,
help="Height of the output video frames. Optional if an input image provided.",
)
parser.add_argument(
"--width",
type=int,
- default=704,
+ default=1216,
help="Width of the output video frames. If None will infer from input image.",
)
parser.add_argument(
@@ -253,7 +255,7 @@ def main():
help="Number of frames to generate in the output video",
)
parser.add_argument(
- "--frame_rate", type=int, default=25, help="Frame rate for the output video"
+ "--frame_rate", type=int, default=30, help="Frame rate for the output video"
)
parser.add_argument(
"--device",
@@ -271,13 +273,13 @@ def main():
parser.add_argument(
"--decode_timestep",
type=float,
- default=0.025,
+ default=0.05,
help="Timestep for decoding noise",
)
parser.add_argument(
"--decode_noise_scale",
type=float,
- default=0.0125,
+ default=0.025,
help="Noise level for decoding noise",
)
@@ -353,6 +355,11 @@ def main():
default="unsloth/Llama-3.2-3B-Instruct",
help="Path to the LLM model, default is Llama-3.2-3B-Instruct, but you can use other models like Llama-3.1-8B-Instruct, or other models supported by Hugging Face",
)
+ parser.add_argument(
+ "--stochastic_sampling",
+ action="store_true",
+ help="Runs inference with stochastic sampling.",
+ )
args = parser.parse_args()
logger.warning(f"Running generation with arguments: {args}")
@@ -373,6 +380,13 @@ def create_ltx_video_pipeline(
assert os.path.exists(
ckpt_path
), f"Ckpt path provided (--ckpt_path) {ckpt_path} does not exist"
+
+ with safe_open(ckpt_path, framework="pt") as f:
+ metadata = f.metadata()
+ config_str = metadata.get("config")
+ configs = json.loads(config_str)
+ allowed_inference_steps = configs.get("allowed_inference_steps", None)
+
vae = CausalVideoAutoencoder.from_pretrained(ckpt_path)
transformer = Transformer3DModel.from_pretrained(ckpt_path)
@@ -433,6 +447,7 @@ def create_ltx_video_pipeline(
"prompt_enhancer_image_caption_processor": prompt_enhancer_image_caption_processor,
"prompt_enhancer_llm_model": prompt_enhancer_llm_model,
"prompt_enhancer_llm_tokenizer": prompt_enhancer_llm_tokenizer,
+ "allowed_inference_steps": allowed_inference_steps,
}
pipeline = LTXVideoPipeline(**submodel_dict)
@@ -471,6 +486,7 @@ def infer(
prompt_enhancement_words_threshold: int = 50,
prompt_enhancer_image_caption_model_name_or_path: str = "MiaoshouAI/Florence-2-large-PromptGen-v2.0",
prompt_enhancer_llm_model_name_or_path: str = "unsloth/Llama-3.2-3B-Instruct",
+ stochastic_sampling: bool = False,
**kwargs,
):
if kwargs.get("input_image_path", None):
@@ -620,6 +636,7 @@ def infer(
offload_to_cpu=offload_to_cpu,
device=device,
enhance_prompt=enhance_prompt,
+ stochastic_sampling=stochastic_sampling,
).images
# Crop the padded images to the desired resolution and number of frames
diff --git a/ltx_video/pipelines/pipeline_ltx_video.py b/ltx_video/pipelines/pipeline_ltx_video.py
index 01b23e8..f2f9cc2 100644
--- a/ltx_video/pipelines/pipeline_ltx_video.py
+++ b/ltx_video/pipelines/pipeline_ltx_video.py
@@ -237,6 +237,7 @@ class LTXVideoPipeline(DiffusionPipeline):
prompt_enhancer_image_caption_processor: AutoProcessor,
prompt_enhancer_llm_model: AutoModelForCausalLM,
prompt_enhancer_llm_tokenizer: AutoTokenizer,
+ allowed_inference_steps: Optional[List[float]] = None,
):
super().__init__()
@@ -258,6 +259,8 @@ class LTXVideoPipeline(DiffusionPipeline):
)
self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
+ self.allowed_inference_steps = allowed_inference_steps
+
def mask_text_embeddings(self, emb, mask):
if emb.shape[0] == 1:
keep_index = mask.sum().item()
@@ -683,6 +686,7 @@ class LTXVideoPipeline(DiffusionPipeline):
offload_to_cpu: bool = False,
enhance_prompt: bool = False,
text_encoder_max_tokens: int = 256,
+ stochastic_sampling: bool = False,
**kwargs,
) -> Union[ImagePipelineOutput, Tuple]:
"""
@@ -751,6 +755,8 @@ class LTXVideoPipeline(DiffusionPipeline):
If set to `True`, the prompt is enhanced using a LLM model.
text_encoder_max_tokens (`int`, *optional*, defaults to `256`):
The maximum number of tokens to use for the text encoder.
+ stochastic_sampling (`bool`, *optional*, defaults to `False`):
+ If set to `True`, the sampling is stochastic. If set to `False`, the sampling is deterministic.
Examples:
@@ -934,6 +940,11 @@ class LTXVideoPipeline(DiffusionPipeline):
timesteps,
**retrieve_timesteps_kwargs,
)
+ if self.allowed_inference_steps is not None:
+ for timestep in [round(x, 4) for x in timesteps.tolist()]:
+ assert (
+ timestep in self.allowed_inference_steps
+ ), f"Invalid inference timestep {timestep}. Allowed timesteps are {self.allowed_inference_steps}."
# 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
@@ -1063,6 +1074,7 @@ class LTXVideoPipeline(DiffusionPipeline):
orig_conditioning_mask,
t,
extra_step_kwargs,
+ stochastic_sampling=stochastic_sampling,
)
# call the callback, if provided
@@ -1137,6 +1149,7 @@ class LTXVideoPipeline(DiffusionPipeline):
t: float,
extra_step_kwargs,
t_eps=1e-6,
+ stochastic_sampling=False,
):
"""
Perform the denoising step for the required tokens, based on the current timestep and
@@ -1153,6 +1166,7 @@ class LTXVideoPipeline(DiffusionPipeline):
latents,
**extra_step_kwargs,
return_dict=False,
+ stochastic_sampling=stochastic_sampling,
)[0]
if conditioning_mask is None:
@@ -1236,7 +1250,7 @@ class LTXVideoPipeline(DiffusionPipeline):
media_item.to(dtype=self.vae.dtype, device=self.vae.device),
self.vae,
vae_per_channel_normalize=vae_per_channel_normalize,
- ).to(dtype=self.transformer.dtype)
+ ).to(dtype=init_latents.dtype)
# Handle the different conditioning cases
if media_frame_number == 0:
diff --git a/ltx_video/schedulers/rf.py b/ltx_video/schedulers/rf.py
index ed2f6c5..5c56f9e 100644
--- a/ltx_video/schedulers/rf.py
+++ b/ltx_video/schedulers/rf.py
@@ -182,6 +182,7 @@ class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):
base_resolution: int = 32**2,
target_shift_terminal: Optional[float] = None,
sampler: Optional[str] = "Uniform",
+ shift: Optional[float] = None,
):
super().__init__()
self.init_noise_sigma = 1.0
@@ -190,13 +191,25 @@ class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):
self.shifting = shifting
self.base_resolution = base_resolution
self.target_shift_terminal = target_shift_terminal
- self.timesteps = self.sigmas = self.get_initial_timesteps(num_train_timesteps)
+ self.timesteps = self.sigmas = self.get_initial_timesteps(
+ num_train_timesteps, shift=shift
+ )
+ self.shift = shift
- def get_initial_timesteps(self, num_timesteps: int) -> Tensor:
+ def get_initial_timesteps(
+ self, num_timesteps: int, shift: Optional[float] = None
+ ) -> Tensor:
if self.sampler == "Uniform":
return torch.linspace(1, 1 / num_timesteps, num_timesteps)
elif self.sampler == "LinearQuadratic":
return linear_quadratic_schedule(num_timesteps)
+ elif self.sampler == "Constant":
+ assert (
+ shift is not None
+ ), "Shift must be provided for constant time shift sampler."
+ return time_shift(
+ shift, 1, torch.linspace(1, 1 / num_timesteps, num_timesteps)
+ )
def shift_timesteps(self, samples: Tensor, timesteps: Tensor) -> Tensor:
if self.shifting == "SD3":
@@ -224,7 +237,9 @@ class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):
device (`Union[str, torch.device]`, *optional*): The device to which the timesteps tensor will be moved.
"""
num_inference_steps = min(self.config.num_train_timesteps, num_inference_steps)
- self.timesteps = self.get_initial_timesteps(num_inference_steps).to(device)
+ self.timesteps = self.get_initial_timesteps(
+ num_inference_steps, shift=self.shift
+ ).to(device)
self.timesteps = self.shift_timesteps(samples, self.timesteps)
self.num_inference_steps = num_inference_steps
self.sigmas = self.timesteps
@@ -277,6 +292,7 @@ class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):
timestep: torch.FloatTensor,
sample: torch.FloatTensor,
return_dict: bool = True,
+ stochastic_sampling: Optional[bool] = False,
**kwargs,
) -> Union[RectifiedFlowSchedulerOutput, Tuple]:
"""
@@ -295,6 +311,8 @@ class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):
A current latent tokens to be de-noised.
return_dict (`bool`, *optional*, defaults to `True`):
Whether or not to return a [`~schedulers.scheduling_ddim.DDIMSchedulerOutput`] or `tuple`.
+ stochastic_sampling (`bool`, *optional*, defaults to `False`):
+ Whether to use stochastic sampling for the sampling process.
Returns:
[`~schedulers.scheduling_utils.RectifiedFlowSchedulerOutput`] or `tuple`:
@@ -327,7 +345,12 @@ class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):
dt = (timestep - lower_timestep)[..., None]
# Compute previous sample
- prev_sample = sample - dt * model_output
+ if stochastic_sampling:
+ x0 = sample - timestep[..., None] * model_output
+ next_timestep = timestep[..., None] - dt
+ prev_sample = self.add_noise(x0, torch.randn_like(sample), next_timestep)
+ else:
+ prev_sample = sample - dt * model_output
if not return_dict:
return (prev_sample,)
diff --git a/pyproject.toml b/pyproject.toml
index 5452a4d..277e7aa 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -20,7 +20,7 @@ dependencies = [
"diffusers>=0.28.2",
"transformers>=4.47.2",
"sentencepiece>=0.1.96",
- "huggingface-hub~=0.25.2",
+ "huggingface-hub~=0.30",
"einops",
"timm"
]
diff --git a/tests/conftest.py b/tests/conftest.py
index 41d1921..48a95ce 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -13,6 +13,15 @@ def test_paths(request, pytestconfig):
)
input_image_path = pytestconfig.getoption("input_image_path")
input_video_path = pytestconfig.getoption("input_video_path")
+ prompt_enhancer_image_caption_model_name_or_path = pytestconfig.getoption(
+ "prompt_enhancer_image_caption_model_name_or_path"
+ )
+ prompt_enhancer_llm_model_name_or_path = pytestconfig.getoption(
+ "prompt_enhancer_llm_model_name_or_path"
+ )
+ prompt_enhancement_words_threshold = pytestconfig.getoption(
+ "prompt_enhancement_words_threshold"
+ )
config = {
"ckpt_path": ckpt_path,
@@ -20,6 +29,9 @@ def test_paths(request, pytestconfig):
"input_video_path": input_video_path,
"output_path": output_path,
"text_encoder_model_name_or_path": text_encoder_model_name_or_path,
+ "prompt_enhancer_image_caption_model_name_or_path": prompt_enhancer_image_caption_model_name_or_path,
+ "prompt_enhancer_llm_model_name_or_path": prompt_enhancer_llm_model_name_or_path,
+ "prompt_enhancement_words_threshold": prompt_enhancement_words_threshold,
}
yield config
@@ -60,3 +72,21 @@ def pytest_addoption(parser):
default="tests/utils/woman.mp4",
help="Path to input video file.",
)
+ parser.addoption(
+ "--prompt_enhancer_image_caption_model_name_or_path",
+ action="store",
+ default="MiaoshouAI/Florence-2-large-PromptGen-v2.0",
+ help="Path to prompt_enhancer_image_caption_model.",
+ )
+ parser.addoption(
+ "--prompt_enhancer_llm_model_name_or_path",
+ action="store",
+ default="unsloth/Llama-3.2-3B-Instruct",
+ help="Path to LLM model for prompt enhancement.",
+ )
+ parser.addoption(
+ "--prompt_enhancement_words_threshold",
+ type=int,
+ default=50,
+ help="Enable prompt enhancement only if input prompt has fewer words than this threshold. Set to 0 to disable enhancement completely.",
+ )
diff --git a/tests/test_inference.py b/tests/test_inference.py
index cc6789e..7fe5e1f 100644
--- a/tests/test_inference.py
+++ b/tests/test_inference.py
@@ -64,8 +64,6 @@ def test_infer_runs_on_real_path(test_paths, conditioning_test_mode):
"negative_prompt": "worst quality, inconsistent motion, blurry, jittery, distorted",
"offload_to_cpu": False,
"prompt_enhancement_max_words": 1000,
- "prompt_enhancer_image_caption_model_name_or_path": "MiaoshouAI/Florence-2-large-PromptGen-v2.0",
- "prompt_enhancer_llm_model_name_or_path": "unsloth/Llama-3.2-3B-Instruct",
}
infer(**{**conditioning_params, **test_paths, **params})
@@ -99,9 +97,13 @@ def test_pipeline_on_batch(test_paths):
device=device,
precision="bfloat16",
text_encoder_model_name_or_path=test_paths["text_encoder_model_name_or_path"],
- enhance_prompt=True,
- prompt_enhancer_image_caption_model_name_or_path="MiaoshouAI/Florence-2-large-PromptGen-v2.0",
- prompt_enhancer_llm_model_name_or_path="unsloth/Llama-3.2-3B-Instruct",
+ enhance_prompt=False,
+ prompt_enhancer_image_caption_model_name_or_path=test_paths[
+ "prompt_enhancer_image_caption_model_name_or_path"
+ ],
+ prompt_enhancer_llm_model_name_or_path=test_paths[
+ "prompt_enhancer_llm_model_name_or_path"
+ ],
)
params = {
@@ -175,8 +177,12 @@ def test_prompt_enhancement(test_paths, monkeypatch):
precision="bfloat16",
text_encoder_model_name_or_path=test_paths["text_encoder_model_name_or_path"],
enhance_prompt=True,
- prompt_enhancer_image_caption_model_name_or_path="MiaoshouAI/Florence-2-large-PromptGen-v2.0",
- prompt_enhancer_llm_model_name_or_path="unsloth/Llama-3.2-3B-Instruct",
+ prompt_enhancer_image_caption_model_name_or_path=test_paths[
+ "prompt_enhancer_image_caption_model_name_or_path"
+ ],
+ prompt_enhancer_llm_model_name_or_path=test_paths[
+ "prompt_enhancer_llm_model_name_or_path"
+ ],
)
original_prompt = "A cat sitting on a windowsill"