Free, open-source AI video generator for Mac. Run LTX-2.5 and MiniMax H3 (Hailuo) locally on Apple Silicon: text-to-video and image-to-video with sound, lip-sync, character LoRA training, songs from lyrics (YuE2) and image editing (Qwen-Image-Edit). One-click install via Pinokio. Offline: no cloud, no API key, no NVIDIA GPU.
Python ai-filmmakingai-videoai-video-generatorapple-siliconhailuoimage-to-videolip-synclocal-ailora-trainingltx-2ltx-2-5ltx-videomacosminimax-h3mlxmusic-generationoffline-aipinokiotext-to-videovideo-generation
"""Image-only preprocessor for LTX-2 MLX trainer.
Encodes still images into 1-frame VAE latents and captions into Gemma
embeddings, written in the layout `PrecomputedDataset` expects:
output_dir/
.precomputed/
latents/
latent_0000.safetensors {latents:[128,1,h,w], num_frames=1, height, width, fps}
latent_0001.safetensors
conditions/
condition_0000.safetensors {video_prompt_embeds, audio_prompt_embeds, prompt_attention_mask}
...
Caption encoding is delegated to the existing video preprocessor's helper —
the path is identical and we don't fork that code.
Usage::
./scripts/run.sh python -m lora_lab.preprocess_images \
--images dataset/images \
--captions dataset/captions \
--output dataset \
--width 512 --height 320
The model dir defaults to the same LTX-2.3 Q4 snapshot Phosphene uses.
"""
from __future__ import annotations
import argparse
import logging
import os
import sys
from pathlib import Path
import mlx.core as mx
import numpy as np
from PIL import Image
from safetensors.numpy import save_file as save_safetensors
from ltx_core_mlx.components.patchifiers import compute_video_latent_shape
from ltx_core_mlx.utils.memory import aggressive_cleanup
from ltx_trainer_mlx.preprocess import _encode_all_captions, _resolve_model_dir
logger = logging.getLogger(__name__)
IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".bmp"}
# Resolved dynamically — see lora_lab/__init__.py for the resolution
# order. Public Pinokio installs use the vendored mlx_models dir; the
# authoring tree falls back to the HF hub cache.
from lora_lab import resolve_default_model_dir
DEFAULT_MODEL_DIR = resolve_default_model_dir()
def _force_eval(*arrays: mx.array) -> None:
mx.eval(*arrays)
def _apply_gemma_max_length_override() -> int | None:
"""Honour ``LTX2_GEMMA_MAX_LENGTH`` for the caption encode, the way the
render pipelines do.
The vendored ``_encode_all_captions`` calls ``encode_all_layers(caption)``
with the default 1024-token pad, and on some chips the macOS GPU watchdog
kills that command buffer partway through the caption list (#61: SIGABRT
on the 1st, 7th or 11th caption of different runs — a race on accumulated
GPU pressure, not a bad caption). The render helper survives the same
kill by retrying once at a shorter encode; the panel relaunches the
trainer the same way and passes the length through this variable. A
256-token pad holds every caption the trainer writes (50-80 words) with
room to spare, and the transformer already trains and renders against
that length on the render side. Returns the applied length, or None when
the variable is unset (the default 1024 stays).
"""
raw = (os.environ.get("LTX2_GEMMA_MAX_LENGTH") or "").strip()
if not raw:
return None
try:
n = max(64, int(raw))
except ValueError:
return None
from ltx_core_mlx.text_encoders.gemma.encoders import base_encoder as _be
cls = _be.GemmaLanguageModel
orig = cls.encode_all_layers
if getattr(orig, "_ltx_max_length_override", None) == n:
return n
def encode_all_layers(self, text: str, max_length: int = 1024):
return orig(self, text, min(int(max_length), n))
encode_all_layers._ltx_max_length_override = n # type: ignore[attr-defined]
cls.encode_all_layers = encode_all_layers
print(f"caption encode: LTX2_GEMMA_MAX_LENGTH={n} (default 1024)")
return n
def _reconcile_precomputed(precomputed: Path, image_files: list, captions: list,
canvas: tuple) -> str:
"""Drop cached tensors that no longer match the inputs, and say what was done.
The vendored encoders skip work by INDEX FILENAME — "condition_0007 exists,
skipping" — with no idea what produced it. Re-training from the same folder
with a new trigger therefore trained on the OLD captions; dropping a photo
shifted every later index so latents paired with the wrong captions; a new
canvas kept the old latents. Issue #62: three triggers, one folder, one
result. A manifest of (image name/size/mtime, caption hash, canvas, encode
length) is written BEFORE encoding, so a crash mid-way resumes correctly
and any change to the inputs invalidates exactly what it must.
"""
import hashlib
import json
import shutil
def _h(text: str) -> str:
return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]
cur = {
"images": [[f.name, f.stat().st_size, int(f.stat().st_mtime)] for f in image_files],
"captions": [_h(c) for c in captions],
"canvas": list(canvas),
"gemma_max_length": os.environ.get("LTX2_GEMMA_MAX_LENGTH", ""),
}
latents_dir = precomputed / "latents"
conditions_dir = precomputed / "conditions"
manifest = precomputed / "manifest.json"
try:
prev = json.loads(manifest.read_text(encoding="utf-8"))
except (OSError, ValueError):
prev = None
def _has_files(d: Path) -> bool:
return d.is_dir() and any(d.iterdir())
if prev is None:
if _has_files(latents_dir) or _has_files(conditions_dir):
shutil.rmtree(latents_dir, ignore_errors=True)
shutil.rmtree(conditions_dir, ignore_errors=True)
action = "cache predates manifests, cannot prove it matches these inputs - re-encoding everything"
else:
action = "fresh"
elif prev.get("images") != cur["images"] or prev.get("canvas") != cur["canvas"]:
shutil.rmtree(latents_dir, ignore_errors=True)
shutil.rmtree(conditions_dir, ignore_errors=True)
action = "images or canvas changed - re-encoding everything"
elif (prev.get("gemma_max_length") != cur["gemma_max_length"]
or len(prev.get("captions") or []) != len(cur["captions"])):
shutil.rmtree(conditions_dir, ignore_errors=True)
action = "caption encode settings changed - re-encoding captions"
elif prev.get("captions") != cur["captions"]:
n = 0
for i, (a, b) in enumerate(zip(prev["captions"], cur["captions"])):
if a != b:
(conditions_dir / f"condition_{i:04d}.safetensors").unlink(missing_ok=True)
n += 1
action = f"{n} caption(s) changed - re-encoding those"
else:
action = "inputs unchanged - reusing cached tensors"
precomputed.mkdir(parents=True, exist_ok=True)
manifest.write_text(json.dumps(cur), encoding="utf-8")
return action
def _resolve_captions(image_files: list[Path], captions_dir: Path | None, caption_ext: str) -> list[str]:
captions: list[str] = []
if captions_dir is None:
for img in image_files:
captions.append(img.stem.replace("_", " ").replace("-", " "))
print(" no captions dir provided; using filenames as captions")
return captions
for img in image_files:
cap_file = captions_dir / f"{img.stem}{caption_ext}"
if cap_file.exists():
captions.append(cap_file.read_text().strip())
else:
fallback = img.stem.replace("_", " ").replace("-", " ")
logger.warning("no caption file for %s, using filename: '%s'", img.name, fallback)
captions.append(fallback)
return captions
def _load_image_as_1frame_tensor(image_path: Path, target_h: int, target_w: int, crop_anchor: str = "center") -> mx.array:
"""Load `image_path`, scale-and-crop to (target_h, target_w), return shape (1, C, 1, H, W) in [-1, 1].
Critical: this is a CROP (preserves face aspect ratio), not a stretch. Earlier
versions used Image.resize() directly, which non-uniformly scaled portrait
selfies into squares — squashed the face geometry, and the LoRA only worked
at inference aspect ratios that undid the same distortion. Center-crop is
the right fix.
`crop_anchor` controls vertical positioning when source is taller than target
after scale-fit (i.e. portrait sources cropped to landscape target):
- "center" (default): symmetric crop top+bottom — backward-compatible
- "top": anchor at top — keeps upper portion (face for portrait sources)
- "bottom": anchor at bottom — keeps lower portion
For character LoRAs trained at landscape from portrait sources, "top" is
usually right: faces sit in the upper third of portraits, legs in the lower
third. Center-crop on a 3:4 portrait → 16:9 landscape can clip the top of
the head AND the chin simultaneously.
"""
img = Image.open(image_path).convert("RGB")
src_w, src_h = img.size
# Resize so the SHORTER side fits the target, preserving aspect ratio.
scale = max(target_w / src_w, target_h / src_h)
new_w = int(round(src_w * scale))
new_h = int(round(src_h * scale))
img = img.resize((new_w, new_h), Image.LANCZOS)
# Crop the LONGER side down to target. Horizontal always centers;
# vertical position is controlled by crop_anchor.
left = (new_w - target_w) // 2
if crop_anchor == "top":
top = 0
elif crop_anchor == "bottom":
top = new_h - target_h
else: # "center"
top = (new_h - target_h) // 2
img = img.crop((left, top, left + target_w, top + target_h))
arr = np.array(img, dtype=np.float32) / 255.0 # (H, W, 3) in [0, 1]
arr = arr.transpose(2, 0, 1) # (3, H, W)
arr = arr[:, None, :, :] # (3, 1, H, W) — F=1
arr = arr[None] # (1, 3, 1, H, W)
arr = arr * 2.0 - 1.0 # to [-1, 1] for VAE
return mx.array(arr).astype(mx.bfloat16)
def _encode_all_images(
image_files: list[Path],
latents_dir: Path,
model_dir: str,
target_height: int,
target_width: int,
crop_anchor: str = "center",
) -> None:
from ltx_trainer_mlx.model_loader import load_video_vae_encoder
if target_height % 32 != 0 or target_width % 32 != 0:
raise ValueError(f"width ({target_width}) and height ({target_height}) must be divisible by 32")
needed: list[int] = []
for i in range(len(image_files)):
out_path = latents_dir / f"latent_{i:04d}.safetensors"
if out_path.exists():
print(f" [{i + 1}/{len(image_files)}] skipping (exists): {out_path.name}")
else:
needed.append(i)
if not needed:
print(" all image latents already encoded")
return
vae_encoder = load_video_vae_encoder(model_dir=model_dir)
vae_encoder.freeze()
F_lat, H_lat, W_lat = compute_video_latent_shape(1, target_height, target_width)
assert F_lat == 1, f"expected F_lat=1 for F=1, got {F_lat}"
print(f" target latent shape per image: [128, {F_lat}, {H_lat}, {W_lat}]")
for i in needed:
img_path = image_files[i]
out_path = latents_dir / f"latent_{i:04d}.safetensors"
print(f" [{i + 1}/{len(image_files)}] encoding: {img_path.name}")
try:
video = _load_image_as_1frame_tensor(img_path, target_height, target_width, crop_anchor=crop_anchor)
latent = vae_encoder.encode(video)
_force_eval(latent)
save_safetensors(
{
"latents": np.array(latent[0].astype(mx.float32)), # [128, 1, H_lat, W_lat]
"num_frames": np.array([F_lat], dtype=np.int32),
"height": np.array([H_lat], dtype=np.int32),
"width": np.array([W_lat], dtype=np.int32),
"fps": np.array([24.0], dtype=np.float32),
},
str(out_path),
)
except Exception as exc:
logger.error("failed to encode %s: %s", img_path.name, exc)
continue
if i % 5 == 0:
aggressive_cleanup()
del vae_encoder
aggressive_cleanup()
print(" image encoding complete")
def preprocess_images(
images_dir: str,
output_dir: str,
model_dir: str = DEFAULT_MODEL_DIR,
gemma_model_id: str | None = None,
target_height: int = 320,
target_width: int = 512,
captions_dir: str | None = None,
caption_ext: str = ".txt",
crop_anchor: str = "center",
) -> None:
# Default to the locally-resolved Gemma path (vendored install ships
# mlx_models/gemma-3-12b-it-4bit; the panel sets LTX_MODELS_DIR so
# the resolver lands there). Falling back to the HF repo id, the way
# the previous hard-coded default did, would silently duplicate-
# download ~6 GB to HF_HOME on a clean Pinokio install AND fail on
# offline rigs that have the local Gemma already cached.
if gemma_model_id is None:
from lora_lab import resolve_default_text_encoder
gemma_model_id = resolve_default_text_encoder()
_apply_gemma_max_length_override()
mx.set_cache_limit(mx.device_info()["memory_size"])
model_dir = _resolve_model_dir(model_dir)
images_path = Path(images_dir)
if not images_path.exists():
raise FileNotFoundError(f"images dir not found: {images_path}")
image_files = sorted(
f for f in images_path.iterdir() if f.suffix.lower() in IMAGE_EXTENSIONS and f.is_file()
)
if not image_files:
raise ValueError(f"no images found in {images_path}")
print(f"found {len(image_files)} images in {images_path}")
precomputed = Path(output_dir) / ".precomputed"
latents_dir = precomputed / "latents"
conditions_dir = precomputed / "conditions"
latents_dir.mkdir(parents=True, exist_ok=True)
conditions_dir.mkdir(parents=True, exist_ok=True)
captions = _resolve_captions(
image_files,
Path(captions_dir) if captions_dir else None,
caption_ext,
)
print("cache: " + _reconcile_precomputed(
precomputed, image_files, captions,
(int(target_width), int(target_height), str(crop_anchor))))
latents_dir.mkdir(parents=True, exist_ok=True)
conditions_dir.mkdir(parents=True, exist_ok=True)
print("Phase 1: encoding text captions (Gemma)...")
_encode_all_captions(
captions=captions,
conditions_dir=conditions_dir,
model_dir=model_dir,
gemma_model_id=gemma_model_id,
)
print(f"Phase 2: encoding image latents at {target_width}x{target_height} (1 frame each, crop_anchor={crop_anchor})...")
_encode_all_images(
image_files=image_files,
latents_dir=latents_dir,
model_dir=model_dir,
target_height=target_height,
target_width=target_width,
crop_anchor=crop_anchor,
)
print(f"\npreprocessing complete. {len(image_files)} samples written to {precomputed}")
print(f" latents: {latents_dir}")
print(f" conditions: {conditions_dir}")
def main() -> int:
p = argparse.ArgumentParser(description="LTX-2 image-only LoRA preprocessor")
p.add_argument("--images", required=True, help="directory of source images")
p.add_argument("--captions", default=None, help="directory of caption .txt files (one per image stem)")
p.add_argument("--caption-ext", default=".txt")
p.add_argument("--output", required=True, help="output root — preprocessed data goes to <output>/.precomputed/")
p.add_argument("--width", type=int, default=512, help="target width (divisible by 32)")
p.add_argument("--height", type=int, default=320, help="target height (divisible by 32)")
p.add_argument("--crop-anchor", default="center", choices=["center", "top", "bottom"],
help="vertical crop position when source is taller than target after scale-fit "
"(center=default+backward-compatible; top=preserves face on portrait sources "
"going to landscape target; bottom=preserves lower body)")
p.add_argument("--model-dir", default=DEFAULT_MODEL_DIR)
# Default left as None so the preprocess_images() body's local
# resolver (resolve_default_text_encoder) picks the right path.
# Hardcoding the HF repo id here would duplicate-download Gemma to
# HF_HOME on a vendored install that already has it under
# mlx_models/gemma-3-12b-it-4bit/.
p.add_argument("--gemma", default=None,
help="Gemma model dir or repo id. Default: resolved "
"via LTX_MODELS_DIR or vendored mlx_models/.")
p.add_argument("-v", "--verbose", action="store_true")
args = p.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.INFO,
format="%(asctime)s %(levelname)s %(name)s | %(message)s",
)
preprocess_images(
images_dir=args.images,
captions_dir=args.captions,
caption_ext=args.caption_ext,
output_dir=args.output,
target_height=args.height,
target_width=args.width,
model_dir=args.model_dir,
gemma_model_id=args.gemma,
crop_anchor=args.crop_anchor,
)
return 0
if __name__ == "__main__":
sys.exit(main())