Skip to content

Official repository for LTX-Video

inference: Refactor to expose a simple infer function

Yoav HaCohen4ba32692bf6d
diff --git a/inference.py b/inference.py
index 771e4c6..d0dc50b 100644
--- a/inference.py
+++ b/inference.py
@@ -4,11 +4,11 @@ import random
from datetime import datetime
from pathlib import Path
from diffusers.utils import logging
+from typing import Optional, Union
import imageio
import numpy as np
import torch
-import torch.nn.functional as F
from PIL import Image
from transformers import T5EncoderModel, T5Tokenizer
@@ -303,6 +303,13 @@ def main():
help="Offloading unnecessary computations to CPU.",
)
+ parser.add_argument(
+ "--text_encoder_model_name_or_path",
+ type=str,
+ default="PixArt-alpha/PixArt-XL-2-1024-MS",
+ help="Local path or model identifier for both the tokenizer and text encoder. Defaults to pretrained model on Hugging Face.",
+ )
+
# Add to the argument parser section
parser.add_argument(
"--sampler",
@@ -313,91 +320,38 @@ def main():
)
args = parser.parse_args()
-
logger.warning(f"Running generation with arguments: {args}")
-
- seed_everething(args.seed)
-
- offload_to_cpu = False if not args.offload_to_cpu else get_total_gpu_memory() < 30
-
- output_dir = (
- Path(args.output_path)
- if args.output_path
- else Path(f"outputs/{datetime.today().strftime('%Y-%m-%d')}")
- )
- output_dir.mkdir(parents=True, exist_ok=True)
-
- # Load image
- if args.input_image_path:
- media_items_prepad = load_image_to_tensor_with_resize_and_crop(
- args.input_image_path, args.height, args.width
- )
- else:
- media_items_prepad = None
-
- height = args.height if args.height else media_items_prepad.shape[-2]
- width = args.width if args.width else media_items_prepad.shape[-1]
- num_frames = args.num_frames
-
- if height > MAX_HEIGHT or width > MAX_WIDTH or num_frames > MAX_NUM_FRAMES:
- logger.warning(
- f"Input resolution or number of frames {height}x{width}x{num_frames} is too big, it is suggested to use the resolution below {MAX_HEIGHT}x{MAX_WIDTH}x{MAX_NUM_FRAMES}."
- )
-
- # Adjust dimensions to be divisible by 32 and num_frames to be (N * 8 + 1)
- height_padded = ((height - 1) // 32 + 1) * 32
- width_padded = ((width - 1) // 32 + 1) * 32
- num_frames_padded = ((num_frames - 2) // 8 + 1) * 8 + 1
-
- padding = calculate_padding(height, width, height_padded, width_padded)
-
- logger.warning(
- f"Padded dimensions: {height_padded}x{width_padded}x{num_frames_padded}"
- )
-
- if media_items_prepad is not None:
- media_items = F.pad(
- media_items_prepad, padding, mode="constant", value=-1
- ) # -1 is the value for padding since the image is normalized to -1, 1
- else:
- media_items = None
-
- device = args.device or get_device()
-
- if device == "mps":
- if torch.__version__ in ["2.4", "2.5"]:
- logger.warning(
- "PyTorch versions 2.4 and 2.5 have a known bug with MPS. Please update to PyTorch 2.6. More info: https://github.com/pytorch/pytorch/issues/141471"
- )
-
- transformer_dtype = (
- torch.bfloat16 if args.precision == "bfloat16" else torch.float32
- )
-
- ckpt_path = Path(args.ckpt_path)
- vae = CausalVideoAutoencoder.from_pretrained(
- ckpt_path, device=device, torch_dtype=torch.bfloat16
- )
- transformer = Transformer3DModel.from_pretrained(ckpt_path, device=device)
- transformer.to(transformer_dtype)
+ infer(**vars(args))
+
+
+def create_ltx_video_pipeline(
+ ckpt_path: str,
+ precision: str,
+ text_encoder_model_name_or_path: str,
+ sampler: Optional[str] = None,
+ device: Optional[str] = None,
+) -> LTXVideoPipeline:
+ ckpt_path = Path(ckpt_path)
+ assert os.path.exists(
+ ckpt_path
+ ), f"Ckpt path provided (--ckpt_path) {ckpt_path} does not exist"
+ vae = CausalVideoAutoencoder.from_pretrained(ckpt_path, torch_dtype=torch.bfloat16)
+ transformer = Transformer3DModel.from_pretrained(ckpt_path)
# Use constructor if sampler is specified, otherwise use from_pretrained
- if args.sampler:
+ if sampler:
scheduler = RectifiedFlowScheduler(
- sampler=(
- "Uniform" if args.sampler.lower() == "uniform" else "LinearQuadratic"
- )
+ sampler=("Uniform" if sampler.lower() == "uniform" else "LinearQuadratic")
)
else:
scheduler = RectifiedFlowScheduler.from_pretrained(ckpt_path)
text_encoder = T5EncoderModel.from_pretrained(
- "PixArt-alpha/PixArt-XL-2-1024-MS", subfolder="text_encoder"
+ text_encoder_model_name_or_path, subfolder="text_encoder"
)
- text_encoder.to(device)
patchifier = SymmetricPatchifier(patch_size=1)
tokenizer = T5Tokenizer.from_pretrained(
- "PixArt-alpha/PixArt-XL-2-1024-MS", subfolder="tokenizer"
+ text_encoder_model_name_or_path, subfolder="tokenizer"
)
transformer = transformer.to(device)
@@ -405,23 +359,10 @@ def main():
text_encoder = text_encoder.to(device)
vae = vae.to(torch.bfloat16)
- if args.precision == "bfloat16" and transformer.dtype != torch.bfloat16:
+ if precision == "bfloat16" and transformer.dtype != torch.bfloat16:
transformer = transformer.to(torch.bfloat16)
text_encoder = text_encoder.to(torch.bfloat16)
- # Set spatiotemporal guidance
- skip_block_list = [int(x.strip()) for x in args.stg_skip_layers.split(",")]
- if args.stg_mode.lower() == "stg_a" or args.stg_mode.lower() == "attention":
- skip_layer_strategy = SkipLayerStrategy.Attention
- elif args.stg_mode.lower() == "stg_r" or args.stg_mode.lower() == "residual":
- skip_layer_strategy = SkipLayerStrategy.Residual
- elif (
- args.stg_mode.lower() == "stg_t" or args.stg_mode.lower() == "transformer_block"
- ):
- skip_layer_strategy = SkipLayerStrategy.TransformerBlock
- else:
- raise ValueError(f"Invalid spatiotemporal guidance mode: {args.stg_mode}")
-
# Use submodels for the pipeline
submodel_dict = {
"transformer": transformer,
@@ -434,47 +375,145 @@ def main():
pipeline = LTXVideoPipeline(**submodel_dict)
pipeline = pipeline.to(device)
+ return pipeline
+
+
+def infer(
+ ckpt_path: str,
+ output_path: Optional[str],
+ seed: int,
+ num_inference_steps: int,
+ num_images_per_prompt: int,
+ guidance_scale: float,
+ stg_scale: float,
+ stg_rescale: float,
+ stg_mode: str,
+ stg_skip_layers: str,
+ image_cond_noise_scale: float,
+ height: Optional[int],
+ width: Optional[int],
+ num_frames: int,
+ frame_rate: int,
+ precision: str,
+ decode_timestep: float,
+ decode_noise_scale: float,
+ prompt: str,
+ negative_prompt: str,
+ offload_to_cpu: bool,
+ text_encoder_model_name_or_path: str,
+ input_image_path: Optional[str] = None,
+ sampler: Optional[str] = None,
+ device: Optional[str] = None,
+ **kwargs,
+):
+ device = device or get_device()
+
+ if device == "mps":
+ if torch.__version__ in ["2.4", "2.5"]:
+ logger.warning(
+ "PyTorch versions 2.4 and 2.5 have a known bug with MPS. Please update to PyTorch 2.6. More info: https://github.com/pytorch/pytorch/issues/141471"
+ )
+
+ generator = torch.Generator(device=device).manual_seed(seed)
+
+ seed_everething(seed)
+ if offload_to_cpu and not device == "cpu":
+ logger.warning(
+ "offload_to_cpu is set to True, but offloading will not occur since the model is already running on CPU."
+ )
+ offload_to_cpu = False
+ else:
+ offload_to_cpu = offload_to_cpu and get_total_gpu_memory() < 30
+
+ output_dir = (
+ Path(output_path)
+ if output_path
+ else Path(f"outputs/{datetime.today().strftime('%Y-%m-%d')}")
+ )
+ output_dir.mkdir(parents=True, exist_ok=True)
+
+ # Adjust dimensions to be divisible by 32 and num_frames to be (N * 8 + 1)
+ height_padded = ((height - 1) // 32 + 1) * 32
+ width_padded = ((width - 1) // 32 + 1) * 32
+ num_frames_padded = ((num_frames - 2) // 8 + 1) * 8 + 1
+
+ padding = calculate_padding(height, width, height_padded, width_padded)
+
+ logger.warning(
+ f"Padded dimensions: {height_padded}x{width_padded}x{num_frames_padded}"
+ )
+
+ pipeline = create_ltx_video_pipeline(
+ ckpt_path=ckpt_path,
+ precision=precision,
+ text_encoder_model_name_or_path=text_encoder_model_name_or_path,
+ sampler=sampler,
+ device=kwargs.get("device", get_device()),
+ )
+
+ conditioning_item = (
+ prepare_conditioning(
+ path=input_image_path,
+ height=height,
+ width=width,
+ padding=padding,
+ )
+ if input_image_path
+ else None
+ )
+
+ # Set spatiotemporal guidance
+ skip_block_list = [int(x.strip()) for x in stg_skip_layers.split(",")]
+ if stg_mode.lower() == "stg_a" or stg_mode.lower() == "attention":
+ skip_layer_strategy = SkipLayerStrategy.Attention
+ elif stg_mode.lower() == "stg_r" or stg_mode.lower() == "residual":
+ skip_layer_strategy = SkipLayerStrategy.Residual
+ elif stg_mode.lower() == "stg_t" or stg_mode.lower() == "transformer_block":
+ skip_layer_strategy = SkipLayerStrategy.TransformerBlock
+ else:
+ raise ValueError(f"Invalid spatiotemporal guidance mode: {stg_mode}")
+
+ skip_block_list = [int(x.strip()) for x in stg_skip_layers.split(",")]
# Prepare input for the pipeline
sample = {
- "prompt": args.prompt,
+ "prompt": prompt,
"prompt_attention_mask": None,
- "negative_prompt": args.negative_prompt,
+ "negative_prompt": negative_prompt,
"negative_prompt_attention_mask": None,
- "media_items": media_items,
+ "media_items": conditioning_item,
}
- generator = torch.Generator(device=device).manual_seed(args.seed)
-
images = pipeline(
- num_inference_steps=args.num_inference_steps,
- num_images_per_prompt=args.num_images_per_prompt,
- guidance_scale=args.guidance_scale,
+ num_inference_steps=num_inference_steps,
+ num_images_per_prompt=num_images_per_prompt,
+ guidance_scale=guidance_scale,
skip_layer_strategy=skip_layer_strategy,
skip_block_list=skip_block_list,
- stg_scale=args.stg_scale,
- do_rescaling=args.stg_rescale != 1,
- rescaling_scale=args.stg_rescale,
+ stg_scale=stg_scale,
+ do_rescaling=stg_rescale != 1,
+ rescaling_scale=stg_rescale,
generator=generator,
output_type="pt",
callback_on_step_end=None,
height=height_padded,
width=width_padded,
num_frames=num_frames_padded,
- frame_rate=args.frame_rate,
+ frame_rate=frame_rate,
**sample,
is_video=True,
vae_per_channel_normalize=True,
conditioning_method=(
ConditioningMethod.FIRST_FRAME
- if media_items is not None
+ if conditioning_item is not None
else ConditioningMethod.UNCONDITIONAL
),
- image_cond_noise_scale=args.image_cond_noise_scale,
- decode_timestep=args.decode_timestep,
- decode_noise_scale=args.decode_noise_scale,
- mixed_precision=(args.precision == "mixed_precision"),
+ image_cond_noise_scale=image_cond_noise_scale,
+ decode_timestep=decode_timestep,
+ decode_noise_scale=decode_noise_scale,
+ mixed_precision=(precision == "mixed_precision"),
offload_to_cpu=offload_to_cpu,
+ device=device,
).images
# Crop the padded images to the desired resolution and number of frames
@@ -492,29 +531,25 @@ def main():
video_np = images[i].permute(1, 2, 3, 0).cpu().float().numpy()
# Unnormalizing images to [0, 255] range
video_np = (video_np * 255).astype(np.uint8)
- fps = args.frame_rate
+ fps = frame_rate
height, width = video_np.shape[1:3]
# In case a single image is generated
if video_np.shape[0] == 1:
output_filename = get_unique_filename(
f"image_output_{i}",
".png",
- prompt=args.prompt,
- seed=args.seed,
+ prompt=prompt,
+ seed=seed,
resolution=(height, width, num_frames),
dir=output_dir,
)
imageio.imwrite(output_filename, video_np[0])
else:
- if args.input_image_path:
- base_filename = f"img_to_vid_{i}"
- else:
- base_filename = f"text_to_vid_{i}"
output_filename = get_unique_filename(
- base_filename,
+ f"video_output_{i}",
".mp4",
- prompt=args.prompt,
- seed=args.seed,
+ prompt=prompt,
+ seed=seed,
resolution=(height, width, num_frames),
dir=output_dir,
)
@@ -524,30 +559,32 @@ def main():
for frame in video_np:
video.append_data(frame)
- # Write condition image
- if args.input_image_path:
- reference_image = (
- (
- media_items_prepad[0, :, 0].permute(1, 2, 0).cpu().data.numpy()
- + 1.0
- )
- / 2.0
- * 255
- )
- imageio.imwrite(
- get_unique_filename(
- base_filename,
- ".png",
- prompt=args.prompt,
- seed=args.seed,
- resolution=(height, width, num_frames),
- dir=output_dir,
- endswith="_condition",
- ),
- reference_image.astype(np.uint8),
- )
logger.warning(f"Output saved to {output_dir}")
+def prepare_conditioning(
+ path: str,
+ height: int,
+ width: int,
+ padding: tuple[int, int, int, int],
+) -> torch.Tensor:
+ """Prepare conditioning items based on input media paths and their parameters.
+
+ Args:
+ path: Path to conditioning image
+ strength: Strength for the conditioning media
+ height: Height of the output frames
+ width: Width of the output frames
+ padding: Padding to apply to the frames
+
+ Returns:
+ A list of ConditioningItem objects.
+ """
+
+ frame_tensor = load_image_to_tensor_with_resize_and_crop(path, height, width)
+ frame_tensor = torch.nn.functional.pad(frame_tensor, padding)
+ frame_tensor
+
+
if __name__ == "__main__":
main()
diff --git a/pyproject.toml b/pyproject.toml
index 589ec9c..e45f120 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -31,3 +31,6 @@ inference-script = [
"matplotlib",
"imageio[ffmpeg]"
]
+test = [
+ "pytest",
+]