Skip to content

Free, open-source AI video generator for Mac. Run LTX-2.5 and MiniMax H3 (Hailuo) locally on Apple Silicon: text-to-video and image-to-video with sound, lip-sync, character LoRA training, songs from lyrics (YuE2) and image editing (Qwen-Image-Edit). One-click install via Pinokio. Offline: no cloud, no API key, no NVIDIA GPU.

23.8 KB
Raw Blame
"""Thin CLI wrapper around ``LtxvTrainer`` for image-only character LoRAs.

Why a wrapper instead of using ``ltx-2-mlx`` directly:

- The default ``TextToVideoConfig.first_frame_conditioning_p`` is 0.1. With
  1-frame image samples, when this triggers the entire image is treated as
  conditioning, the loss mask becomes all-zero, ``mean(loss_mask) == 0``, and
  the masked-MSE loss in ``TextToVideoStrategy.compute_loss`` divides by zero
  → NaN. We force it to 0.0 for image-only training.
- We want first/last training-step timing logged to stdout so M3 prints a
  honest per-step number on this machine.

Usage::

    ./scripts/run.sh python -m lora_lab.train --config configs/char_image_lora.yaml
"""

from __future__ import annotations

import argparse
import logging
import sys
import time
from pathlib import Path
from typing import Any

import yaml

import ltx_trainer_mlx.model_loader as ml_module
import ltx_trainer_mlx.training_strategies as ts_module
from ltx_trainer_mlx.config import LtxTrainerConfig
from ltx_trainer_mlx.trainer import LtxvTrainer

logger = logging.getLogger(__name__)


# The dev transformer the patched loader ACTUALLY chose, recorded so the
# sidecar can state the truth. Before this existed, train_character.py stamped
# a hardcoded "base_model" literal into every sidecar — which is exactly the
# provenance users pasted into issue #62 while we tried to diagnose their
# training runs against a fiction. None until the first load resolves.
RESOLVED_BASE: dict | None = None


def _patch_loader_prefer_dev_transformer() -> None:
    """Force `load_transformer` to pick `transformer-dev.safetensors` when present.

    Default auto-detect order is `transformer.safetensors` → `transformer-distilled.safetensors`
    → `transformer-dev.safetensors`. In the `dgrauet/ltx-2.3-mlx-q4` model dir, the
    distilled file exists, so dev never gets picked.

    Per LTX-2 issue #175 (Lightricks/LTX-2): the distilled checkpoint uses a different
    sigma schedule (8-step DISTILLED_SIGMAS) than the standard flow-matching trainer
    objective. Training a character LoRA against the distilled weights produces gradients
    that point toward the wrong target — coarse features (hair/age/beard) survive the
    mismatch but fine face geometry doesn't. Switching the trainer to the dev transformer
    eliminates the objective mismatch.

    Q4/Q8 sibling fallback (2026-05-18):
        The trainer config hardcodes the Q4 model dir, but on a Q8-only install
        the dev transformer lives at ``<panel>/mlx_models/ltx-2.3-mlx-q8/
        transformer-dev.safetensors`` and the Q4 dir doesn't exist (or has only
        the distilled file). The panel's preflight (_train_required_models in
        mlx_ltx_panel.py) already accepts either location via ``extra_dirs=
        [q8_local_dir]``, but the trainer process — running in a separate venv
        — has to do its own probing. We look at both ``<model_dir>`` and the
        sibling ``ltx-2.3-mlx-q8/`` next to it, and pin to whichever holds a
        **full-precision** dev. If NEITHER exists we raise a clear, actionable
        error instead of letting the loader silently fall back to the distilled
        file — silent fallback was the original bug: identity capture quietly
        broke because the trainer was optimizing against the wrong sigma schedule.

    Precision guard (2026-07-20, #35/#36):
        A 4-bit *quantized* dev transformer (~11 GB) can sit in the Q4 dir and
        pass a bare exists() check — but training against quantized weights also
        yields a near-zero LoRA. So we compare file sizes: a full-precision dev
        is ~19 GB, quantized ~11 GB. We prefer the largest dev ≥ 15 GB and warn
        loudly if a quantized one is shadowing it. Reported by @saved-j in #36.
    """
    from pathlib import Path

    # A full-precision dev transformer is ~19 GB; a 4-bit quantized dev is ~11 GB.
    # Training against the quantized one produces a near-zero LoRA (#35/#36), so we
    # must never pick it while a full-precision dev exists anywhere. 15 GB cleanly
    # separates the two.
    FULL_DEV_MIN_BYTES = 15 * 1000**3

    def _dev_size(p: "Path") -> int:
        try:
            return p.stat().st_size if p.exists() else 0
        except OSError:
            return 0

    _orig = ml_module.load_transformer

    def patched(model_dir, transformer_file=None):
        if transformer_file is None:
            primary = Path(model_dir) / "transformer-dev.safetensors"
            sibling = Path(model_dir).parent / "ltx-2.3-mlx-q8" / "transformer-dev.safetensors"
            p_size, s_size = _dev_size(primary), _dev_size(sibling)

            # Each candidate is (dir_to_pass_to_loader, size_bytes, human_label).
            # The sibling repoints the loader at the Q8 DIR as `model_dir` so
            # `load_split_safetensors` resolves the shards relative to the right
            # snapshot — passing only the basename would still join against the
            # Q4 dir and miss the sibling's shard files.
            candidates = []
            if p_size:
                candidates.append((model_dir, p_size, str(primary.parent)))
            if s_size:
                candidates.append((str(sibling.parent), s_size, str(sibling.parent)))

            full = [c for c in candidates if c[1] >= FULL_DEV_MIN_BYTES]
            if full:
                chosen = max(full, key=lambda c: c[1])
                if p_size and p_size < FULL_DEV_MIN_BYTES:
                    # A quantized dev is shadowing the full one in the Q4 dir —
                    # this is the exact #35/#36 silent failure. Skip it, loudly.
                    logger.warning(
                        "training-side override: IGNORING quantized transformer-dev.safetensors "
                        "(%.1f GB) in %s — it would train a near-zero LoRA (#35/#36). Using the "
                        "full-precision dev (%.1f GB) from %s instead.",
                        p_size / 1e9, primary.parent, chosen[1] / 1e9, chosen[2],
                    )
                else:
                    logger.info(
                        "training-side override: using full-precision transformer-dev.safetensors "
                        "(%.1f GB) from %s (not distilled)",
                        chosen[1] / 1e9, chosen[2],
                    )
                global RESOLVED_BASE
                RESOLVED_BASE = {"dir": str(chosen[2]), "bytes": int(chosen[1]),
                                 "file": "transformer-dev.safetensors",
                                 "full_precision": True}
                return _orig(chosen[0], transformer_file="transformer-dev.safetensors")

            if candidates:
                # Only a suspiciously small (likely quantized) dev exists anywhere.
                # Better than the distilled base, but identity capture will be weak —
                # use it and shout so the user knows to re-download the full dev.
                chosen = max(candidates, key=lambda c: c[1])
                logger.warning(
                    "training-side override: only a small transformer-dev.safetensors (%.1f GB) "
                    "found at %s — expected the full-precision dev (~19 GB). Character identity "
                    "capture will be weak; re-download it via Train Character -> Preflight.",
                    chosen[1] / 1e9, chosen[2],
                )
                RESOLVED_BASE = {"dir": str(chosen[2]), "bytes": int(chosen[1]),
                                 "file": "transformer-dev.safetensors",
                                 "full_precision": False}
                return _orig(chosen[0], transformer_file="transformer-dev.safetensors")

            raise FileNotFoundError(
                f"transformer-dev.safetensors not found in {primary.parent} "
                f"or {sibling.parent}. Open Phosphene → Train Character → "
                f"Preflight to download the full-precision dev transformer (~19 GB)."
            )
        return _orig(model_dir, transformer_file=transformer_file)

    ml_module.load_transformer = patched


# Moved to lora_lab.failure (light imports) so the panel can ask the SAME
# question before it queues a run: the Train tab's own check used to accept
# Hugging Face cache copies the trainer never loads, and Start was not gated
# by it at all (fleet 2 Nov 2026: one Mac failed 22 runs in an hour on it).
from lora_lab.failure import training_weights_problem  # noqa: E402,F401  (re-exported)


def _patch_compute_video_positions_fps_kwarg() -> None:
    """Translate ``fps=`` → ``frame_rate=`` across the trainer.

    Upstream rename 2026-05-14 (same family as the rename in
    ``_decode_and_save_video``): ``ltx_core_mlx.utils.positions`` renamed
    ``fps`` → ``frame_rate`` on both ``compute_video_positions`` and
    ``compute_audio_token_count``. The trainer still calls with ``fps=``
    in multiple places (base_strategy, trainer, validation_sampler,
    text_to_video, video_to_video), so training raises
    ``TypeError: ... unexpected keyword argument 'fps'`` at the first
    training step.

    We wrap both functions to accept either kwarg name, then rebind the
    symbols in every trainer-side module that imported them by name.
    Lora-lab CLAUDE rule 1 says don't modify phosphene's tree — so we
    monkey-patch at runtime instead.
    """
    import ltx_core_mlx.utils.positions as _pos

    _orig_vid = _pos.compute_video_positions
    _orig_aud = _pos.compute_audio_token_count

    def patched_video(num_frames, height, width, *, fps=None, frame_rate=None):
        if frame_rate is None:
            frame_rate = fps if fps is not None else 24.0
        return _orig_vid(num_frames=num_frames, height=height, width=width,
                          frame_rate=frame_rate)

    def patched_audio(num_video_frames, *, fps=None, frame_rate=None):
        if frame_rate is None:
            frame_rate = fps if fps is not None else 24.0
        return _orig_aud(num_video_frames=num_video_frames, frame_rate=frame_rate)

    # Patch the source module.
    _pos.compute_video_positions = patched_video
    _pos.compute_audio_token_count = patched_audio

    # Rebind in every trainer-side module that imported the names directly.
    _consumer_modules = (
        "ltx_trainer_mlx.trainer",
        "ltx_trainer_mlx.validation_sampler",
        "ltx_trainer_mlx.training_strategies.base_strategy",
        "ltx_trainer_mlx.training_strategies.text_to_video",
        "ltx_trainer_mlx.training_strategies.video_to_video",
    )
    import importlib
    for name in _consumer_modules:
        try:
            mod = importlib.import_module(name)
        except ImportError:
            continue
        if hasattr(mod, "compute_video_positions"):
            mod.compute_video_positions = patched_video
        if hasattr(mod, "compute_audio_token_count"):
            mod.compute_audio_token_count = patched_audio


def _patch_lora_target_exclude_audio(enabled: bool = True) -> None:
    """Exclude audio attention/FF layers from LoRA target matching.

    The trainer's ``_find_lora_targets`` (trainer.py:917) does suffix matching:
    it splits each module path by ``.`` and checks if ANY part is in
    ``target_modules``. With the canonical face-LoRA target_modules
    ``[to_q, to_k, to_v, to_out]``, the matcher catches both
    ``transformer_blocks.N.attn1.to_q`` (video attention — wanted) AND
    ``transformer_blocks.N.audio_attn1.to_q`` (audio attention — unwanted).

    Training on still images with no audio signal pushes random gradients
    into the audio attention layers. The saved face LoRA then carries those
    random "audio_attn" deltas. When stacked with the audio LoRA at inference,
    those random deltas corrupt the model's audio attention path → lips don't
    animate during dialogue, voice is distorted. Discovered 2026-05-15 on
    Aria v2 — face LoRA had 1152 audio_attn deltas that broke lip-sync until
    they were filtered out post-hoc.

    Fix: wrap ``_find_lora_targets`` so any path containing an ``audio_*``
    submodule is skipped at training time. The LoRA never includes those
    weights, never wastes gradient updates on them, and the saved file is
    clean by construction (no post-hoc filter needed).

    Lora-lab CLAUDE rule 1 says don't mutate phosphene's tree — so we
    monkey-patch at runtime instead.

    ``enabled=False`` restores the vendored matcher, which is the Lightricks
    trainer's default target set: q/k/v/out on all six attention families,
    1152 modules — the same set ``bizarrotrn_v2`` carries (#62 trainer
    comparison). Idempotent either way.
    """
    import ltx_trainer_mlx.trainer as _trainer
    _orig = getattr(_trainer, "_phos_orig_find_lora_targets", None)
    if _orig is None:
        _orig = _trainer._find_lora_targets
        _trainer._phos_orig_find_lora_targets = _orig
    if not enabled:
        _trainer._find_lora_targets = _orig
        logger.info("lora target filter: off (all attention families, audio included)")
        return

    EXCLUDE_SUBSTRINGS = ("audio_attn", "audio_ff")

    def patched(model, target_names):
        results = _orig(model, target_names)
        filtered = [(path, mod) for (path, mod) in results
                    if not any(s in path for s in EXCLUDE_SUBSTRINGS)]
        dropped = len(results) - len(filtered)
        if dropped:
            logger.info(
                "lora target filter: kept %d video targets, "
                "dropped %d audio targets (audio_attn / audio_ff)",
                len(filtered), dropped,
            )
        return filtered

    _trainer._find_lora_targets = patched


def _patch_contiguous_checkpoint_save() -> None:
    """Make every tensor the trainer saves C-contiguous. THE #62 root cause.

    ``ltx_trainer_mlx.trainer._save_checkpoint`` converts each LoRA factor to
    the ComfyUI layout with ``np.array(mx.transpose(param).astype(mx.float32))``.
    The params are already float32, so ``astype`` is a no-op and the transpose
    stays a VIEW: numpy receives a correctly-strided but non-contiguous array.
    ``safetensors.numpy.save_file`` (0.8.0 in this venv) serializes
    ``tensor.ctypes.data`` for ``tensor.nbytes`` — the raw buffer, strides
    ignored; its own docstring says tensors "need to be contiguous". So every
    ``lora_A.weight`` on disk is ``lora_a``'s bytes under the transposed shape,
    not ``lora_a.T``, and likewise ``lora_B``.

    The file keeps every element value, so it attaches cleanly and moves the
    output, but its delta is unrelated to the trained one (cosine -0.0002 at
    4096x32) and, on a trained adapter, about half its strength:
    ``valeriosan_v2``-``v5`` read 6.7-7.1e-4 on the identity family as saved
    and 1.33-1.49e-3 unscrambled — the band of the characters that work. That
    is the #62 signature exactly — clean attach, "half strength", output
    perturbed, no identity — and why ``probe_fit.py`` measured ~0 fit on
    adapters whose training loss had fallen 37-68%: unscrambled, the same
    2-image adapter fits its images by 41-83%. Character and voice LoRAs both
    save through here.

    Wrapping the trainer module's ``save_safetensors`` binding fixes every save
    site at once and is a no-op on arrays that are already contiguous.
    """
    import numpy as np
    import ltx_trainer_mlx.trainer as _trainer

    orig = getattr(_trainer, "_phos_orig_save_safetensors", None)
    if orig is None:
        orig = _trainer.save_safetensors
        _trainer._phos_orig_save_safetensors = orig

    def save_contiguous(tensors: dict, filename: Any, metadata: Any = None) -> None:
        tensors = {k: np.ascontiguousarray(v) for k, v in tensors.items()}
        if metadata is None:
            return orig(tensors, filename)
        return orig(tensors, filename, metadata)

    _trainer.save_safetensors = save_contiguous


def _patch_adam_bias_correction(enabled: bool) -> None:
    """Give the trainer's AdamW the bias correction every PyTorch AdamW has.

    ``ltx_trainer_mlx.trainer._init_optimizer`` builds ``mlx.optimizers.AdamW``
    with only ``learning_rate`` and ``weight_decay``, and MLX defaults
    ``bias_correction`` to False (checked on mlx 0.31.1). PyTorch's AdamW —
    which the Lightricks trainer and ai-toolkit (bitsandbytes AdamW8bit) both
    use — always corrects. Uncorrected, Adam's step is
    ``(1 - b1**t) / sqrt(1 - b2**t)`` times the corrected one: 3.2x at step 1,
    6.5x around steps 10-15, 3.3x at 100, 1.6x at 500, 1.26x at 1000. So the
    "lr 1e-4" recipe spent its first few hundred steps at several times that,
    with batch-1 gradients that are ~92% noise draw (#62, trainer regression
    check). ``enabled=False`` restores the old optimizer exactly; it is how a
    run reproduces an adapter trained before iter7.
    """
    import ltx_trainer_mlx.trainer as _trainer

    cls = _trainer.LtxvTrainer
    orig = getattr(cls, "_phos_orig_init_optimizer", None)
    if orig is None:
        orig = cls._init_optimizer
        cls._phos_orig_init_optimizer = orig

    def patched(self) -> None:
        orig(self)
        # Adam.apply_single reads self.bias_correction on every step, so
        # setting it after construction is the same as passing it in.
        self._optimizer.bias_correction = bool(enabled)
        logger.info("optimizer: AdamW bias_correction=%s", bool(enabled))

    cls._init_optimizer = patched


# Image-only training feeds the joint audio+video DiT a stand-in audio stream:
# trainer.py `_build_loss_fn` builds ONE all-zero audio token with audio
# timesteps of 0 — "clean, finished audio" — next to a noisy video. Neither
# reference trainer does that (#62 trainer comparison):
#   - Lightricks ltx-trainer passes audio=None, and ltx-core then skips the
#     audio<->video cross-attention entirely (transformer.py run_a2v/run_v2a);
#   - ai-toolkit passes a zero audio latent at the SAME sigma as the video, as
#     every real render does.
# The MLX model also drives the a2v gate from the VIDEO sigma where ltx-core
# uses the audio one (transformer_args._prepare_cross_attention_timestep), so
# clean-audio-plus-noisy-video is a combination the reference model never
# computes. These are the three modes.
IMAGE_AUDIO_MODES = ("skip_a2v", "matched_sigma", "clean_zero")


class _ImageOnlyAudioTransformer:
    """Stand-in for ``trainer._transformer`` inside one loss evaluation.

    The vendored loss function looks ``self._transformer`` up at call time and
    calls it with the dummy-audio kwargs. Swapping a plain forwarding object in
    for the duration of that call lets us change only the audio arguments
    without copying the vendored loss function. It is not an ``nn.Module``:
    ``value_and_grad`` was built on the real model and still differentiates
    its trainable parameters, which this object calls straight through to.
    """

    def __init__(self, model: Any, mode: str) -> None:
        self._model = model
        self._mode = mode

    def __call__(self, **kwargs: Any) -> Any:
        import mlx.core as mx

        if self._mode == "skip_a2v":
            from ltx_core_mlx.guidance.perturbations import (
                BatchedPerturbationConfig,
                Perturbation,
                PerturbationConfig,
                PerturbationType,
            )

            # Both directions, every block: the audio stream can no longer
            # reach the video loss, which is what audio=None means upstream.
            # The block multiplies the cross-attention output by 0, so values
            # AND gradients match skipping it (a2v LoRAs stay at zero).
            per_sample = PerturbationConfig([
                Perturbation(PerturbationType.SKIP_A2V_CROSS_ATTN, None),
                Perturbation(PerturbationType.SKIP_V2A_CROSS_ATTN, None),
            ])
            batch = int(kwargs["video_latent"].shape[0])
            kwargs["perturbations"] = BatchedPerturbationConfig([per_sample] * batch)
        elif self._mode == "matched_sigma":
            # ai-toolkit: same zero latent, but at the video's noise level.
            ats = kwargs["audio_timesteps"]
            sigma = kwargs["timestep"].astype(ats.dtype).reshape(-1, 1)
            kwargs["audio_timesteps"] = mx.broadcast_to(sigma, ats.shape)
        return self._model(**kwargs)


def _patch_image_only_audio(mode: str) -> None:
    """Choose how image-only training handles the joint model's audio stream.

    ``skip_a2v`` matches the Lightricks trainer, ``matched_sigma`` matches
    ai-toolkit, ``clean_zero`` is the pre-iter7 behaviour (no patch). Applies
    only to the dummy-audio branch: a run that trains real audio
    (``generate_audio``) is left exactly as the vendored trainer builds it.
    """
    if mode not in IMAGE_AUDIO_MODES:
        raise ValueError(f"image_audio must be one of {IMAGE_AUDIO_MODES}, got {mode!r}")
    import ltx_trainer_mlx.trainer as _trainer

    cls = _trainer.LtxvTrainer
    orig = getattr(cls, "_phos_orig_build_loss_fn", None)
    if orig is None:
        orig = cls._build_loss_fn
        cls._phos_orig_build_loss_fn = orig

    if mode == "clean_zero":
        cls._build_loss_fn = orig
        return

    def patched(self) -> Any:
        loss_fn = orig(self)
        if self._training_strategy.requires_audio:
            return loss_fn

        def image_only_loss_fn(batch: Any) -> Any:
            real = self._transformer
            self._transformer = _ImageOnlyAudioTransformer(real, mode)
            try:
                return loss_fn(batch)
            finally:
                self._transformer = real

        return image_only_loss_fn

    cls._build_loss_fn = patched
    logger.info("image-only audio stream: %s", mode)


def _patch_strategy_for_image_only() -> None:
    """Force `first_frame_conditioning_p=0.0` for image-only training.

    The Pydantic config doesn't expose this knob; intercept the factory to
    rebuild the strategy config with our override.
    """
    _orig = ts_module.get_training_strategy

    def patched(config):
        name = getattr(config, "name", None)
        if name == "text_to_video":
            generate_audio = bool(getattr(config, "generate_audio", False))
            native_cfg = ts_module.TextToVideoConfig(
                first_frame_conditioning_p=0.0,
                with_audio=generate_audio,
            )
            strategy = ts_module.TextToVideoStrategy(native_cfg)
            logger.info(
                "image-only override: first_frame_conditioning_p=0.0, with_audio=%s",
                generate_audio,
            )
            return strategy
        return _orig(config)

    ts_module.get_training_strategy = patched


def main() -> int:
    p = argparse.ArgumentParser(description="LTX-2 image-only character LoRA trainer")
    p.add_argument("--config", required=True, help="path to YAML config (see configs/char_image_lora.yaml)")
    p.add_argument("-v", "--verbose", action="store_true")
    args = p.parse_args()

    logging.basicConfig(
        level=logging.DEBUG if args.verbose else logging.INFO,
        format="%(asctime)s %(levelname)s %(name)s | %(message)s",
    )

    cfg_path = Path(args.config).resolve()
    with cfg_path.open() as f:
        cfg_dict = yaml.safe_load(f)

    config = LtxTrainerConfig.model_validate(cfg_dict)

    _patch_loader_prefer_dev_transformer()
    _patch_strategy_for_image_only()
    _patch_compute_video_positions_fps_kwarg()
    _patch_lora_target_exclude_audio()
    _patch_contiguous_checkpoint_save()
    # The defaults of train_character.RECIPE_SWITCHES (the legacy recipe),
    # stated explicitly; the yaml still owns scheduler_type on this path.
    _patch_adam_bias_correction(False)
    _patch_image_only_audio("clean_zero")

    t0 = time.time()
    trainer = LtxvTrainer(config)
    print(f"trainer init complete in {time.time() - t0:.1f}s")

    t0 = time.time()
    saved_path, stats = trainer.train(disable_progress_bars=False)
    print(f"\ntraining wall: {time.time() - t0:.1f}s")
    print(f"checkpoint: {saved_path}")
    print(f"per-step: {1.0 / stats.steps_per_second:.2f}s  ({stats.steps_per_second:.3f} steps/s)")
    print(f"peak memory: {stats.peak_memory_gb:.2f} GB")
    return 0


if __name__ == "__main__":
    sys.exit(main())