Skip to content

Free, open-source AI video generator for Mac. Run LTX-2.5 and MiniMax H3 (Hailuo) locally on Apple Silicon: text-to-video and image-to-video with sound, lip-sync, character LoRA training, songs from lyrics (YuE2) and image editing (Qwen-Image-Edit). One-click install via Pinokio. Offline: no cloud, no API key, no NVIDIA GPU.

54.2 KB
Raw Blame
"""Single-shot CLI: run the full Character LoRA pipeline from a job-spec JSON.

This is the orchestration backend for Phosphene's "Train Character" panel.
The panel writes a spec JSON and invokes this module as a subprocess:

    ./scripts/run.sh python -m lora_lab.train_character path/to/spec.json

The CLI streams JSON-lines progress to stdout (one JSON object per line) so
the panel can tail it. The final `.safetensors` is written to the path in
the spec, with a `<lora>.safetensors.json` sidecar of training metadata.

------------------------------------------------------------------------
Stdout protocol (one JSON object per line; ordered, append-only):

    {"event":"start","job_id":"..."}
    {"event":"plan","preset":"medium","rank":32,"alpha":32,"steps":3000,
     "lr":0.0001,"resolution":576,"caption_strategy":"class_word",
     "estimated_wall_s":7234,"image_count":24}
    {"event":"crop_start","total":24}
    {"event":"crop_progress","done":1,"total":24,"path":"..."}
    ...
    {"event":"crop_done"}
    {"event":"preprocess_start","total":24}
    {"event":"preprocess_progress","done":1,"total":24}
    ...
    {"event":"preprocess_done"}
    {"event":"train_start","total_steps":3000,"estimated_wall_s":7200}
    {"event":"train_progress","step":250,"loss":0.512,"eta_s":6300}
    ...
    {"event":"train_done","checkpoint":"/path/to/lora.safetensors"}
    {"event":"sidecar_written","path":"/path/to/lora.safetensors.json"}
    {"event":"done"}

On any failure:

    {"event":"error","stage":"preprocess","message":"..."}
    -> exit code 1

------------------------------------------------------------------------
Spec JSON schema (everything except `advanced` is required):

    {
      "job_id": "char_20260511_134552",
      "trigger": "mrbz07",
      "preset": "quick" | "medium" | "high",
      "images_dir": ".../state/train_character/<job_id>/images",
      "output_lora_path": ".../mlx_models/loras_local/<job_id>/<job_id>.safetensors",
      "advanced": {
        "rank": null | int,
        "alpha": null | int,
        "steps": null | int,
        "lr": null | float,
        "resolution": null | int,
        "caption_strategy": null | "class_word" | "trigger_only" | "auto_caption",
        "crop_strategy": null | "center",  // "face_centered" not yet implemented
        // iter7 recipe switches (RECIPE_SWITCHES; null = default):
        "adam_bias_correction": null | true | false,
        "scheduler_type": null | "linear" | "constant",
        "image_audio": null | "skip_a2v" | "matched_sigma" | "clean_zero",
        "lora_target_families": null | "video" | "all_attention"
      }
    }

Null fields fall back to the preset defaults below.

------------------------------------------------------------------------
Presets (validated recipes — do not invent new ones):

    | Field             | quick      | medium     | high       |
    |-------------------|------------|------------|------------|
    | rank              | 16         | 32         | 32         |
    | alpha             | 16         | 32         | 32         |
    | steps             | 1500       | 3000       | 5000       |
    | lr                | 1.0e-4     | 1.0e-4     | 1.0e-4     |
    | resolution        | 576        | 576        | 576        |
    | caption_strategy  | class_word | class_word | class_word |
    | target_modules    | to_q/k/v/out (all)                  |

------------------------------------------------------------------------
Wall-time estimator (panel must mirror this byte-for-byte):

    base_step_s   = 2.4                  # M4 Max 64 GB, 576x576 rank 32
    res_scale     = (res / 576.0) ** 2   # per-step time ~ tokens ~ res^2
    rank_scale    = 0.95 if rank <= 16 else 1.0
    per_step_s    = base_step_s * res_scale * rank_scale

    preprocess_s  = 90 + 2.0 * image_count   # model load + VAE encode + Gemma
    train_s       = steps * per_step_s
    total_s       = preprocess_s + train_s
"""

from __future__ import annotations

import argparse
import datetime as _dt
import json
import logging
import os
import shutil
import subprocess
import sys
import tempfile
import time
import traceback
from pathlib import Path
from typing import Any

import yaml

from lora_lab.failure import (NAN_LOSS_ABORT_STEPS, NonFiniteLoss, classify_exception,
                              is_finite_loss)

logger = logging.getLogger(__name__)


# ----------------------------------------------------------------------
# CLI-ONLY DEFAULTS. NOT a mirror of the panel's presets — they have never
# matched, and the comment that said they must is what made a change to
# either one look safe.
#
# The panel writes every hyperparameter it cares about into spec.json
# (rank / alpha / steps / lr / resolution), and `resolve_preset` lets those
# `advanced` values win — so on a panel-driven run this table is dead code.
# It is live on the CLI path only, where `--preset quick` trains rank 16
# while the panel's Quick trains rank 8, and the panel derives steps from
# `epochs x image_count` while these are fixed. Same word, two recipes: read
# them as this module's defaults and nothing else. If they ever need to agree,
# make one of them import the other rather than restating it here.
# ----------------------------------------------------------------------

PRESETS: dict[str, dict[str, Any]] = {
    "quick": {
        "rank": 16, "alpha": 16, "steps": 1500, "lr": 1.0e-4,
        "resolution": 576, "caption_strategy": "class_word",
    },
    "medium": {
        "rank": 32, "alpha": 32, "steps": 3000, "lr": 1.0e-4,
        "resolution": 576, "caption_strategy": "class_word",
    },
    "high": {
        "rank": 32, "alpha": 32, "steps": 5000, "lr": 1.0e-4,
        "resolution": 576, "caption_strategy": "class_word",
    },
}

TARGET_MODULES = ["to_q", "to_k", "to_v", "to_out"]

# Bumped when the trainer's behaviour changes, so a sidecar dates its file.
# "iter5" had been frozen in every sidecar since the yaml era regardless of
# what changed; iter6 = truthful base_model + width/height honored; iter7 =
# adapters saved contiguous (train._patch_contiguous_checkpoint_save — every
# earlier adapter on safetensors 0.8.0 was written scrambled, #62) plus the
# RECIPE_SWITCHES below.
LORA_LAB_VERSION = "iter7"

# Training-recipe switches from the #62 comparison against the trainers known
# to bind a face (Lightricks ltx-trainer, ostris ai-toolkit) — see the PR #89
# investigation log (tanis2000/Phosphene, branch issue-62,
# docs/ISSUE_62_VALERIOSAN_2026-09-30.md), "Comparison against known-working
# trainers". Each maps to (default, allowed values). The DEFAULT is the legacy
# recipe, the one every adapter before iter7 trained with: with the save fixed
# it fits its training images and renders a learned person, and the screen
# ("Screen results with the save fixed") found the reference-trainer values
# halve fit at equal steps with no gain in trigger binding. The other values
# stay available so a spec can A/B one switch at a time.
#
#   adam_bias_correction  PyTorch AdamW always bias-corrects; MLX's AdamW
#                         defaults to not, and the vendored trainer never
#                         asked (lora_lab.train._patch_adam_bias_correction).
#   scheduler_type        Lightricks default: linear decay to 0.1x over the
#                         run. ai-toolkit keeps it constant.
#   image_audio           how the stand-in audio stream of an image-only run
#                         reaches the video: skipped (Lightricks), at the
#                         video's sigma (ai-toolkit) or clean at sigma 0 (old).
#   lora_target_families  "all_attention" = the Lightricks default and
#                         bizarrotrn_v2's 1152 modules. Under skip_a2v the
#                         audio families get no gradient and stay exactly
#                         zero; under the other two modes they train on the
#                         stand-in audio — the 2026-05-15 Aria v2 lip-sync
#                         breakage (the screen's arm C trained audio_attn2
#                         hardest).
RECIPE_SWITCHES: dict[str, tuple[Any, tuple[Any, ...]]] = {
    "adam_bias_correction": (False, (True, False)),
    "scheduler_type": ("constant", ("linear", "constant")),
    "image_audio": ("clean_zero", ("skip_a2v", "matched_sigma", "clean_zero")),
    "lora_target_families": ("video", ("all_attention", "video")),
}


def resolve_recipe(cfg: dict[str, Any]) -> dict[str, Any]:
    """The recipe switches for one run: spec value if given, else the default.

    Raises ValueError on a value outside the allowed set rather than guessing,
    so a typo in a hand-edited spec cannot silently run the default arm.
    """
    out: dict[str, Any] = {}
    for key, (default, allowed) in RECIPE_SWITCHES.items():
        value = cfg.get(key)
        if value is None:
            value = default
        if isinstance(default, bool) and isinstance(value, str):
            value = value.strip().lower() in ("1", "true", "yes", "on")
        if value not in allowed:
            raise ValueError(f"{key} must be one of {list(allowed)}, got {value!r}")
        out[key] = value
    return out


def _resolved_base_label() -> str:
    """What the loader ACTUALLY trained against, from train.RESOLVED_BASE."""
    # In a real training run train.py is already imported (it owns the
    # loader patch), so sys.modules has it. Import errors here mean we are
    # NOT in a training process (train.py needs ltx_trainer_mlx, which only
    # exists in the trainer venv) — fall back to the honest unknown label
    # rather than crashing the sidecar write.
    try:
        try:
            from . import train as _train_mod
        except ImportError:
            import train as _train_mod  # script-style invocation
    except Exception:
        _train_mod = None
    rb = getattr(_train_mod, "RESOLVED_BASE", None) if _train_mod else None
    if not rb:
        return "Lightricks/LTX-2.3 (dev transformer — resolution not recorded)"
    gb = rb.get("bytes", 0) / 1e9
    prec = "full-precision" if rb.get("full_precision") else "QUANTIZED (weak identity)"
    return (f"Lightricks/LTX-2.3 dev transformer · {rb.get('dir', '?')} · "
            f"{gb:.1f} GB · {prec}")

# Estimator constants for THIS module's estimate. The panel's ETA is a
# different formula, not a copy of this one: it uses a per-preset
# `seconds_per_step` with no fixed term, and `3.0 * image_count` for
# preprocessing. Nothing gates the two against each other because they are
# not the same estimate.
_BASE_STEP_S = 2.4            # M4 Max 64 GB, 576x576 rank 32
_PREPROCESS_FIXED_S = 90.0    # model load + Gemma init
_PREPROCESS_PER_IMAGE_S = 2.0 # VAE encode per image


def _normalize_target_modules(value: Any) -> list[str]:
    """Accept panel-supplied LoRA target modules while keeping a safe allowlist."""
    if value is None:
        return list(TARGET_MODULES)
    if isinstance(value, str):
        raw = [x.strip() for x in value.split(",") if x.strip()]
    elif isinstance(value, (list, tuple)):
        raw = [str(x).strip() for x in value if str(x).strip()]
    else:
        raw = []
    allowed = set(TARGET_MODULES)
    mods = [m for m in raw if m in allowed]
    return mods or list(TARGET_MODULES)


def resolve_preset(preset: str, advanced: dict[str, Any] | None) -> dict[str, Any]:
    """Merge a preset with optional `advanced` overrides. None values fall through."""
    if preset not in PRESETS:
        raise ValueError(f"unknown preset {preset!r}; expected one of {list(PRESETS)}")
    cfg = dict(PRESETS[preset])
    if advanced:
        # `k in cfg` gated overrides to keys the preset already declared —
        # and PRESETS has never declared width/height, so the aspect-matched
        # training that commit 3f49ca3 shipped (the fix for "each eye gets
        # ~1 latent token at 512²") was silently dropped here and every
        # panel-driven training fell back to resolution×resolution. An
        # explicit allowlist instead: preset keys plus the geometry pair.
        _extra_ok = ("width", "height", "checkpoint_keep_last_n",
                     "enable_gradient_checkpointing", *RECIPE_SWITCHES)
        for k, v in advanced.items():
            if v is not None and (k in cfg or k in _extra_ok):
                cfg[k] = v
        if advanced.get("target_modules") is not None:
            cfg["target_modules"] = _normalize_target_modules(advanced.get("target_modules"))
    if "target_modules" not in cfg:
        cfg["target_modules"] = list(TARGET_MODULES)
    return cfg


def _wh_from_advanced(cfg: dict[str, Any]) -> tuple[int, int]:
    """Resolve (width, height) from a cfg dict.

    Backwards-compat: if `width` and `height` aren't both present, falls
    back to a square `resolution × resolution` canvas (the legacy v2
    behavior). The 2026-05-20 widescreen-training change adds independent
    W/H so character LoRAs can train at the same aspect as inference
    (1024×576 widescreen) without baking square + letterbox-bar artifacts
    into the weights.
    """
    w = cfg.get("width")
    h = cfg.get("height")
    if w and h:
        return int(w), int(h)
    res = int(cfg.get("resolution") or 576)
    return res, res


def estimate_wall_seconds(image_count: int, preset: str, advanced: dict[str, Any] | None) -> float:
    """Estimate end-to-end wall seconds for a job.

    Formula (panel JS must mirror this exactly):
        per_step_s   = 2.4 * (W*H / 576^2) * (0.95 if rank <= 16 else 1.0)
        preprocess_s = 90 + 2.0 * image_count
        total_s      = preprocess_s + steps * per_step_s

    Pre-2026-05-20 this used (res/576)^2 assuming square training.
    Widescreen training (e.g. 1024×576) needs the token-budget scale
    expressed as the actual pixel ratio to match per-step compute.
    """
    cfg = resolve_preset(preset, advanced)
    w, h = _wh_from_advanced({**cfg, **(advanced or {})})
    rank = int(cfg["rank"])
    steps = int(cfg["steps"])
    pixel_scale = (w * h) / (576.0 * 576.0)
    rank_scale = 0.95 if rank <= 16 else 1.0
    per_step_s = _BASE_STEP_S * pixel_scale * rank_scale
    preprocess_s = _PREPROCESS_FIXED_S + _PREPROCESS_PER_IMAGE_S * image_count
    return preprocess_s + steps * per_step_s


# ----------------------------------------------------------------------
# Stdout JSON-lines emitter
# ----------------------------------------------------------------------

# Capture the ORIGINAL stdout at import time. The training stage temporarily
# swaps sys.stdout to silence the trainer's prints; emit() must keep writing
# to the real panel-facing stream regardless of those swaps.
_REAL_STDOUT = sys.stdout


def emit(event: str, **kwargs: Any) -> None:
    """Write one JSON event line to the real stdout, flushed."""
    obj = {"event": event, **kwargs}
    _REAL_STDOUT.write(json.dumps(obj, separators=(",", ":")) + "\n")
    _REAL_STDOUT.flush()


def emit_error_and_exit(stage: str, message: str, code: int = 1,
                        exc: BaseException | None = None) -> None:
    # `error_class` is the closed word the panel may report (lora_lab.failure);
    # `message` stays on this Mac (the Logs tab and <job>/train.log).
    emit("error", stage=stage, message=message,
         error_class=classify_exception(exc))
    sys.exit(code)


# ----------------------------------------------------------------------
# Spec validation
# ----------------------------------------------------------------------

_REQUIRED_SPEC_KEYS = ("job_id", "trigger", "preset", "images_dir", "output_lora_path")


def load_spec(spec_path: Path) -> dict[str, Any]:
    if not spec_path.exists():
        raise FileNotFoundError(f"spec not found: {spec_path}")
    spec = json.loads(spec_path.read_text())

    # ---- compat shim: Phosphene's panel emits a flat-spec shape with
    # different key names than this CLI's canonical schema. Normalize
    # without losing data so both shapes work end-to-end.
    #
    # Panel writes:        CLI canonical:
    #   output_path          → output_lora_path
    #   rank/steps/lr/...    → advanced.{rank,steps,lr,...}  (flat → nested)
    #   caption_strategy:    → caption_strategy in {class_word, trigger_only,
    #     "trigger_simple"        auto_caption}; "trigger_simple" was the
    #                             panel agent's name for class_word.
    if "output_lora_path" not in spec and "output_path" in spec:
        spec["output_lora_path"] = spec["output_path"]
    if not isinstance(spec.get("advanced"), dict):
        spec["advanced"] = {}
    for k in ("rank", "alpha", "steps", "lr", "resolution",
              "width", "height", "checkpoint_keep_last_n",
              "enable_gradient_checkpointing",
              "target_modules", "caption_strategy", "crop_strategy",
              *RECIPE_SWITCHES):
        if k in spec and k not in spec["advanced"]:
            spec["advanced"][k] = spec[k]
    cs = spec["advanced"].get("caption_strategy")
    if cs == "trigger_simple":
        # Panel-side alias; means "trigger word + a tiny framing hint",
        # which is what class_word does (e.g. "mychar man, close-up portrait").
        spec["advanced"]["caption_strategy"] = "class_word"

    for k in _REQUIRED_SPEC_KEYS:
        if k not in spec:
            raise ValueError(f"spec missing required key: {k!r}")
    return spec


# ----------------------------------------------------------------------
# Cropping + caption authoring
# ----------------------------------------------------------------------

IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".bmp"}


def list_source_images(images_dir: Path) -> list[Path]:
    """Sorted list of source image files. Sort by filename to be deterministic."""
    if not images_dir.exists():
        raise FileNotFoundError(f"images dir does not exist: {images_dir}")
    files = sorted(
        f for f in images_dir.iterdir()
        if f.is_file() and f.suffix.lower() in IMAGE_EXTENSIONS
    )
    if not files:
        raise ValueError(f"no images found in {images_dir}")
    return files


def _framing_for_image(width: int, height: int) -> str:
    return "close-up portrait" if height > width else "medium shot"


def _make_caption(trigger: str, strategy: str, width: int, height: int) -> str:
    # Emit LTX's canonical `[VISUAL]: ...\n[TEXT]: None\n` format so the
    # fallback rows match what caption_with_gemma.py writes for the auto-
    # caption path. Mixed-format datasets (some rows naked `<trigger> man`,
    # others `[VISUAL]:...`) caused identity drift on the fallback images
    # because the trainer's tokenizer saw two distinct prompt styles for
    # the same character. See ~/.claude/skills/ltx-lora/SKILL.md for the
    # validated v2 recipe.
    if strategy == "trigger_only":
        body = "a photo"
    else:
        # class_word (default) and auto_caption fallback
        body = _framing_for_image(width, height)
    return f"[VISUAL]: {trigger}, {body}\n[TEXT]: None\n"


def crop_and_caption(
    source_files: list[Path],
    cropped_dir: Path,
    captions_dir: Path,
    images_renamed_dir: Path,
    trigger: str,
    caption_strategy: str,
    resolution: int | None = None,
    crop_strategy: str = "center",
    *,
    width: int | None = None,
    height: int | None = None,
) -> tuple[list[Path], list[str]]:
    """Scale + (optionally) crop sources to a `(width × height)` canvas,
    rename sequentially, write captions.

    Backwards-compat: callers that pass `resolution=N` get a square N×N
    canvas (legacy v2 behavior). New callers pass `width` + `height`
    independently to produce non-square canvases that match the
    inference aspect (e.g. 1024×576 widescreen).

    `crop_strategy` ∈ {"center", "letterbox"} controls how non-square sources
    fit into the square training canvas:

      "center" (default, legacy behavior):
          Scale so the SHORTER source dim = target, then center-crop the
          longer dim to target. Image fills the square; content on the
          longer dim is clipped. Good for character close-ups where the
          face is centered. Bad for wide shots — long-shot proportions are
          never seen by the model, which is why generated long shots tend
          to look mushy / wrong-scale.

      "letterbox":
          Scale so the LONGER source dim = target, then pad the shorter
          dim with black to fill the square. Image content keeps its
          native aspect ratio inside the square. Model learns to ignore
          the consistent black bars (they're a constant factor across
          training samples). Trade-off: ~30% pixel-budget waste on bars
          for portraits/landscapes, but generated long shots / wide
          shots reproduce real proportions.

    Returns (renamed_image_paths, caption_strings) in matching order.

    Writes:
      images_renamed_dir/char_NNN.png          full-resolution training input (lossless)
      cropped_dir/char_NNN.jpg                 1:1 thumbnail (256px) for the UI gallery
      captions_dir/char_NNN.txt                caption matching each image stem
    """
    from PIL import Image

    cropped_dir.mkdir(parents=True, exist_ok=True)
    captions_dir.mkdir(parents=True, exist_ok=True)
    images_renamed_dir.mkdir(parents=True, exist_ok=True)

    if crop_strategy not in ("center", "letterbox"):
        raise ValueError(
            f"unknown crop_strategy: {crop_strategy!r} "
            f"(want 'center' or 'letterbox')"
        )

    # Resolve target canvas dims. New callers pass width + height
    # explicitly; legacy callers pass resolution (square fallback).
    if width and height:
        target_w, target_h = int(width), int(height)
    elif resolution:
        target_w = target_h = int(resolution)
    else:
        raise ValueError("crop_and_caption requires width+height or resolution")

    emit("crop_start", total=len(source_files), crop_strategy=crop_strategy,
         width=target_w, height=target_h)

    out_paths: list[Path] = []
    out_captions: list[str] = []

    for i, src in enumerate(source_files, start=1):
        try:
            img = Image.open(src).convert("RGB")
            src_w, src_h = img.size

            if crop_strategy == "letterbox":
                # Scale so the source FITS INSIDE (target_w × target_h)
                # without distortion. Pad with black to fill the canvas.
                # On non-square canvases this can produce smaller bars
                # (or even none, when source aspect matches target).
                scale = min(target_w / src_w, target_h / src_h)
                new_w = int(round(src_w * scale))
                new_h = int(round(src_h * scale))
                scaled = img.resize((new_w, new_h), Image.LANCZOS)
                cropped_full = Image.new("RGB", (target_w, target_h), (0, 0, 0))
                paste_x = (target_w - new_w) // 2
                paste_y = (target_h - new_h) // 2
                cropped_full.paste(scaled, (paste_x, paste_y))
            else:
                # "center" — scale-and-center-crop to fill (target_w ×
                # target_h). Mirrors preprocess_images._load_image_as_1frame_tensor:
                # scale so the SHORTER side hits its target, then crop
                # the longer side. On widescreen targets this is the
                # right call — for portrait sources it center-crops
                # vertically (head usually survives unless the source
                # is unusually tall).
                scale = max(target_w / src_w, target_h / src_h)
                new_w = int(round(src_w * scale))
                new_h = int(round(src_h * scale))
                scaled = img.resize((new_w, new_h), Image.LANCZOS)
                left = (new_w - target_w) // 2
                top = (new_h - target_h) // 2
                cropped_full = scaled.crop((left, top, left + target_w, top + target_h))

            stem = f"char_{i:03d}"
            png_path = images_renamed_dir / f"{stem}.png"
            cropped_full.save(png_path, format="PNG")

            # Thumbnail for the UI gallery (256x256, jpg, compact).
            thumb = cropped_full.copy()
            thumb.thumbnail((256, 256), Image.LANCZOS)
            thumb_path = cropped_dir / f"{stem}.jpg"
            thumb.save(thumb_path, format="JPEG", quality=88)

            # Honour user-provided captions written to captions_dir by the
            # panel (filename matches the source image stem OR the renamed
            # `char_NNN` stem the panel pre-mapped to). This lets the Train
            # tab ship industry-standard `image_001.png` + `image_001.txt`
            # pairs in LTX `[VISUAL]:` / `[TEXT]:` format without the
            # trainer stomping them with the dumbest-strongest fallback.
            user_cap_paths = [
                captions_dir / f"{stem}.txt",                  # post-rename match
                captions_dir / f"{src.stem}.txt",              # original-stem match
            ]
            user_cap = next((p for p in user_cap_paths if p.is_file()), None)
            if user_cap is not None:
                caption = user_cap.read_text(encoding="utf-8", errors="replace")
                # Re-write at the renamed stem so the preprocess step finds it.
                (captions_dir / f"{stem}.txt").write_text(caption, encoding="utf-8")
            else:
                caption = _make_caption(trigger, caption_strategy, src_w, src_h)
                (captions_dir / f"{stem}.txt").write_text(caption)

            out_paths.append(png_path)
            out_captions.append(caption)

            emit("crop_progress", done=i, total=len(source_files), path=str(png_path))
        except Exception as exc:  # noqa: BLE001
            # One unreadable / corrupt image must not kill the whole pass.
            # Mirrors preprocess_images.preprocess_images_to_latents which
            # logs+continues on per-image encode failure (preprocess_images.py
            # :174-176). Re-raising used to fail the entire crop stage, so
            # users with one bad JPEG in a 30-image dataset lost the whole run.
            logger.warning(
                "failed to crop %s: %s — skipping this image", src.name, exc
            )
            emit(
                "warning",
                stage="crop",
                message=f"skipped {src.name}: {exc}",
                file=src.name,
            )
            continue

    # A previous run's frames that this run did not write (ov-review D-01).
    # The stems are renumbered 1..N every run, and the preprocess step encodes
    # EVERY png in images_renamed_dir: remove a photo from a 12-photo dataset
    # and re-train, and the old char_012.png was still there - "preprocess
    # wrote 12 latents; expected 11" on every retry. A photo that failed to
    # crop this run left its old frame behind the same way, to be trained
    # under another photo's caption. Derived files only; the sources and the
    # captions are untouched.
    keep = {p.name for p in out_paths}
    for stale in images_renamed_dir.glob("char_*.png"):
        if stale.name not in keep:
            stale.unlink(missing_ok=True)
            (cropped_dir / f"{stale.stem}.jpg").unlink(missing_ok=True)

    emit("crop_done")
    return out_paths, out_captions


# ----------------------------------------------------------------------
# Preprocess (call the existing module — same env, same VAE encode + Gemma)
# ----------------------------------------------------------------------

def run_preprocess(
    images_dir: Path,
    captions_dir: Path,
    output_root: Path,
    resolution: int | None = None,
    total: int = 0,
    *,
    width: int | None = None,
    height: int | None = None,
) -> None:
    """Encode images + captions into the layout PrecomputedDataset expects.

    Calls preprocess_images.preprocess_images() in-process so progress lines
    can be intermixed cleanly. We emit a coarse start/end pair; the inner
    preprocessor logs its own prints to stderr (we route them away from
    stdout to keep the JSON-lines stream pure).

    Resolution params: pass `width` and `height` for non-square training
    (1024×576 widescreen etc.), or pass `resolution=N` for the legacy
    square N×N canvas. The preprocessor already exposes target_height
    and target_width independently so this is a contract change only.
    """
    from lora_lab import preprocess_images as pp  # local import — heavy deps

    if width and height:
        target_w, target_h = int(width), int(height)
    elif resolution:
        target_w = target_h = int(resolution)
    else:
        raise ValueError("run_preprocess requires width+height or resolution")

    emit("preprocess_start", total=total, width=target_w, height=target_h)

    # Pipe inner module's stdout into stderr so JSON-lines on our stdout
    # stay clean. The preprocessor uses bare print(); duplicate fd2.
    orig_stdout = sys.stdout
    try:
        sys.stdout = sys.stderr  # redirect prints inside the call
        pp.preprocess_images(
            images_dir=str(images_dir),
            output_dir=str(output_root),
            # Pass the locally-resolved Gemma path explicitly. Without
            # this the preprocessor falls back to the HF repo id and
            # may duplicate-download ~6 GB to HF_HOME (or fail offline).
            gemma_model_id=DEFAULT_TEXT_ENCODER,
            target_height=target_h,
            target_width=target_w,
            captions_dir=str(captions_dir),
            caption_ext=".txt",
        )
    finally:
        sys.stdout = orig_stdout

    # The inner module already wrote one latent_NNNN.safetensors per image
    # into output_root/.precomputed/latents/. Verify count.
    latents_dir = output_root / ".precomputed" / "latents"
    written = sorted(latents_dir.glob("latent_*.safetensors"))
    if len(written) != total:
        raise RuntimeError(
            f"preprocess wrote {len(written)} latents; expected {total}"
        )
    emit("preprocess_progress", done=total, total=total)
    emit("preprocess_done")


# ----------------------------------------------------------------------
# Training (build a temp YAML, hand to the existing trainer wrapper logic)
# ----------------------------------------------------------------------

# Resolved dynamically — see lora_lab/__init__.py. The previous hard-
# coded HF cache snapshot path was correct for the maintainer's machine
# only; clean Pinokio installs have the model at mlx_models/ltx-2.3-mlx-q4/
# inside the panel install dir, and ``LTX_MODELS_DIR`` (set by the panel
# when it spawns this script) points the resolver there.
from lora_lab import resolve_default_model_dir, resolve_default_text_encoder
DEFAULT_MODEL_PATH = resolve_default_model_dir()
DEFAULT_TEXT_ENCODER = resolve_default_text_encoder()


def build_trainer_config(
    *,
    cfg: dict[str, Any],
    data_root: Path,
    output_dir: Path,
) -> dict[str, Any]:
    """Compose the LtxTrainerConfig dict from a resolved preset + paths."""
    recipe = resolve_recipe(cfg)
    return {
        "model": {
            "model_path": DEFAULT_MODEL_PATH,
            "text_encoder_path": DEFAULT_TEXT_ENCODER,
            "training_mode": "lora",
        },
        "lora": {
            "rank": int(cfg["rank"]),
            "alpha": int(cfg["alpha"]),
            "dropout": 0.0,
            "target_modules": _normalize_target_modules(cfg.get("target_modules")),
        },
        "optimization": {
            "learning_rate": float(cfg["lr"]),
            "steps": int(cfg["steps"]),
            "batch_size": 1,
            "gradient_accumulation_steps": 1,
            "max_grad_norm": 1.0,
            "weight_decay": 0.0,
            # "constant" is the legacy default; "linear" uses the trainer's
            # defaults, start 1.0 -> end 0.1 over `steps` — the Lightricks
            # t2v_lora.yaml schedule (RECIPE_SWITCHES).
            "scheduler_type": recipe["scheduler_type"],
            # Per-block recompute (ltx_core_mlx transformer model.py, which
            # passes the LoRA params into mx.checkpoint explicitly so they
            # still get gradients). Off by default: 512 px fits without it.
            # At 768x768 on a 64 GB Mac the uncheckpointed run hit a 54.8 GB
            # footprint, swapped ~31 GB of GPU memory and logged no step in
            # 28 minutes; recompute trades speed for that memory.
            "enable_gradient_checkpointing": bool(cfg.get("enable_gradient_checkpointing") or False),
        },
        "data": {
            "preprocessed_data_root": str(data_root),
        },
        "training_strategy": {
            "name": "text_to_video",
            "generate_audio": False,
        },
        "flow_matching": {
            "timestep_sampling_mode": "shifted_logit_normal",
        },
        "validation": {
            "prompts": [],
            "interval": None,
            "skip_initial_validation": True,
            "generate_audio": False,
        },
        "checkpoints": {
            # Save every ~10% of total steps so the user has fallbacks if a long
            # run is interrupted, but keep at least one mid-run checkpoint.
            "interval": max(1, int(cfg["steps"]) // 5),
            # keep_last_n=2 left only the final checkpoint of the #62 v2/v3
            # runs on disk, so "was it over-trained?" could not be answered
            # after the fact. A spec may raise it; use -1 to keep every save.
            # NOT 5: when steps is a multiple of the interval the final step
            # is recorded twice (interval save + end-of-run save, trainer.py
            # :216/:285), so 5 silently pruned the first checkpoint.
            "keep_last_n": int(cfg.get("checkpoint_keep_last_n") or 2),
        },
        "seed": 42,
        "output_dir": str(output_dir),
    }


def run_training(
    *,
    cfg: dict[str, Any],
    data_root: Path,
    output_dir: Path,
    estimated_wall_s: float,
) -> tuple[Path, float]:
    """Run the trainer in-process. Returns (final_checkpoint_path, wall_seconds).

    We call the trainer with a step_callback so we can emit per-step progress
    without parsing tqdm output.
    """
    # Late imports — keep CLI start cheap.
    from ltx_trainer_mlx.config import LtxTrainerConfig
    from ltx_trainer_mlx.trainer import LtxvTrainer
    from lora_lab import train as lab_train

    config_dict = build_trainer_config(cfg=cfg, data_root=data_root, output_dir=output_dir)
    config = LtxTrainerConfig.model_validate(config_dict)
    recipe = resolve_recipe(cfg)

    # Apply the same patches train.py uses: dev transformer + image-only safety
    # + fps→frame_rate kwarg shim + audio-attn exclusion, then the iter7
    # recipe switches. These all run as cheap monkey-patches before
    # LtxvTrainer is instantiated.
    lab_train._patch_loader_prefer_dev_transformer()
    lab_train._patch_strategy_for_image_only()
    lab_train._patch_compute_video_positions_fps_kwarg()
    # Not a recipe switch: without it the saved adapter is scrambled (#62).
    lab_train._patch_contiguous_checkpoint_save()
    lab_train._patch_lora_target_exclude_audio(recipe["lora_target_families"] == "video")
    lab_train._patch_adam_bias_correction(recipe["adam_bias_correction"])
    lab_train._patch_image_only_audio(recipe["image_audio"])
    emit("log", line="recipe: " + ", ".join(f"{k}={v}" for k, v in recipe.items()))
    if recipe["lora_target_families"] == "all_attention" and recipe["image_audio"] == "skip_a2v":
        emit(
            "warning",
            stage="config",
            message=(
                "lora_target_families=all_attention with image_audio=skip_a2v: "
                "the audio-family and a2v adapters get no gradient and are saved "
                "as exact zeros. Pair all_attention with matched_sigma to train them."
            ),
        )

    # Pin Python/numpy RNG before LtxvTrainer touches them. The upstream
    # trainer seeds `mx.random` with `cfg.seed=42` (see ltx_trainer_mlx/
    # trainer.py:114) but the dataset shuffle at trainer.py:978 uses
    # Python's stdlib `random.shuffle(indices)` which is NEVER seeded by
    # the trainer. Result: with batch=1 over 5000 steps, two identical-
    # recipe runs traverse the dataset in different orders and land at
    # different minima — exactly the variance Mr Bizarro saw between the
    # OLD chartest_v2 (good) and the 2026-05-18 retrain (worse). Pinning
    # stdlib + numpy RNG here closes the biggest reproducibility hole.
    # (MLX/Metal matmul nondeterminism still remains as a smaller second-
    # order source — documented, unfixable today.)
    import random as _stdlib_random
    import numpy as _np
    _seed = int(cfg.get("seed", 42))
    _stdlib_random.seed(_seed)
    _np.random.seed(_seed)

    total_steps = int(cfg["steps"])
    emit("train_start", total_steps=total_steps, estimated_wall_s=int(estimated_wall_s))

    train_start = time.time()
    last_emit_step = 0
    # Roughly every ~2% of total steps, with bounds [10, 250].
    emit_every = max(10, min(250, total_steps // 50 or 10))
    last_loss: float | None = None
    # Mean over the steps since the last emit. The emitted loss used to be
    # the single most recent step's, i.e. one sample at a random sigma every
    # ~2% of the run: timestep variance swamped any trend, and the #62 v2/v3
    # curves could not say whether training converged. Batch 1 makes the
    # per-step value this noisy; the interval mean is what is readable.
    loss_sum = 0.0
    loss_n = 0

    # Sniff per-step loss without forking the trainer: monkey-patch
    # TrainingProgress.update_training. It always gets called (even with
    # disable_progress_bars=True the method runs and just no-ops the UI).
    import ltx_trainer_mlx.progress as tp_mod  # type: ignore

    _orig_update = tp_mod.TrainingProgress.update_training

    nonfinite_streak = 0

    def _sniff_update(self, *, loss, lr, step_time, advance=True):  # noqa: ANN001
        nonlocal last_loss, loss_sum, loss_n, nonfinite_streak
        try:
            value = float(loss)
        except Exception:  # noqa: BLE001
            value = None
        if value is not None:
            if is_finite_loss(value):
                last_loss = value
                loss_sum += value
                loss_n += 1
                nonfinite_streak = 0
            else:
                # One NaN step poisons every LoRA parameter through the
                # optimizer; every step after it is NaN as well. Stop instead
                # of spending hours writing a file of NaNs (4.19.1: the fleet
                # could not tell this from any other "exit 1").
                nonfinite_streak += 1
                if nonfinite_streak >= NAN_LOSS_ABORT_STEPS:
                    raise NonFiniteLoss(
                        f"the training loss stopped being a number for "
                        f"{nonfinite_streak} steps in a row - the adapter "
                        f"cannot learn from here")
        return _orig_update(self, loss=loss, lr=lr, step_time=step_time, advance=advance)

    tp_mod.TrainingProgress.update_training = _sniff_update

    def _cb(step: int, total: int, _videos: list[Path]) -> None:
        nonlocal last_emit_step, loss_sum, loss_n
        if step - last_emit_step < emit_every and step != total:
            return
        last_emit_step = step
        elapsed = time.time() - train_start
        # Defensive: avoid div-by-zero on step 0.
        eta = 0
        if step > 0:
            per_step = elapsed / step
            eta = max(0, int(per_step * (total - step)))
        if loss_n:
            loss_val = round(loss_sum / loss_n, 4)
        else:
            loss_val = round(last_loss, 4) if last_loss is not None else None
        n_val = loss_n
        loss_sum, loss_n = 0.0, 0
        emit(
            "train_progress",
            step=step,
            loss=loss_val,
            loss_n=n_val,
            eta_s=eta,
        )
        # Panel-side alias: mlx_ltx_panel.py parses {"event":"step"} with
        # {step, total, loss} fields and renders the progress bar from it.
        emit("step", step=step, total=total, loss=loss_val, loss_n=n_val, eta_s=eta)

    # Silence the trainer's own logging to stdout — only our JSON-lines should
    # reach stdout. Send everything else to stderr so it's still capturable.
    for noisy in (
        "ltx_trainer_mlx",
        "ltx_trainer_mlx.trainer",
        "ltx_core_mlx",
    ):
        logging.getLogger(noisy).propagate = False

    # Redirect generic prints from the trainer to stderr.
    orig_stdout = sys.stdout
    try:
        sys.stdout = sys.stderr
        trainer = LtxvTrainer(config)
        saved_path, _stats = trainer.train(disable_progress_bars=True, step_callback=_cb)
    finally:
        sys.stdout = orig_stdout

    train_wall = time.time() - train_start
    emit("train_done", checkpoint=str(saved_path))
    # Panel-side alias for the same milestone.
    emit("checkpoint", path=str(saved_path))
    return Path(saved_path), train_wall


# ----------------------------------------------------------------------
# Sidecar
# ----------------------------------------------------------------------

def report_adapter_strength(lora_path: Path) -> dict[str, Any] | None:
    """Measure what the finished adapter can actually do, and say so.

    A training run that completes is not a training run that worked. In August
    2026 two users (issues #61, #62) trained characters that finished clean,
    saved a 132–500 MB file, attached ``576/576`` modules at render — and moved
    nothing a human eye could see. Nothing anywhere in the pipeline had ever
    measured the one quantity that decides it: the size of the low-rank delta
    the file carries. So the run ended with "done ✓" and the user spent days
    proving a negative.

    This emits that number at the end of every training run, next to the file
    it describes, and shouts when it lands below the band every adapter that
    has ever carried an identity here sits in (``lora_compat.WEAK_DELTA_RMS``,
    calibrated against the shipped characters and third-party LoRAs). It does
    not fail the job: the file is still saved, still usable, and the owner may
    want a deliberately light touch. It just stops the silence.
    """
    try:
        from lora_compat import measure_adapter_effect
    except ImportError:  # pragma: no cover — panel root is always on PYTHONPATH
        return None
    try:
        effect = measure_adapter_effect(lora_path)
    except Exception as exc:  # noqa: BLE001 — a probe must never fail a run
        emit("log", line=f"adapter strength: not measured ({exc})")
        return None
    if effect is None:
        return None

    payload = {
        "modules": effect.modules,
        "carrying_modules": effect.modules - effect.zero_modules,
        "delta_rms_median": effect.median_rms,
        "delta_rms_max": effect.max_rms,
        "floor": effect.floor,
        "verdict": "inert" if effect.inert else ("weak" if effect.weak else "ok"),
    }
    emit("adapter_strength", **payload)
    emit("log", line=f"adapter strength: {effect.summary}")
    if effect.inert:
        emit(
            "warning",
            stage="verify",
            message=(
                f"{lora_path.name} carries no weight delta at all — every "
                "low-rank product is zero, so it cannot change a render. "
                "Training moved nothing; treat this run as failed."
            ),
        )
    elif effect.weak:
        emit(
            "warning",
            stage="verify",
            message=(
                f"{lora_path.name} is much weaker than the adapters that carry "
                f"identity here (measured {effect.median_rms:.2e}, floor "
                f"{effect.floor:.1e}, validated recipe ~1.4e-03). Expect little "
                "or no visible effect. The High preset (rank 32, ~100 epochs, "
                "lr 1e-4) is the only recipe graded for identity."
            ),
        )
    return payload


def write_sidecar(
    *,
    lora_path: Path,
    cfg: dict[str, Any],
    trigger: str,
    image_count: int,
    training_wall_s: float,
    strength: dict[str, Any] | None = None,
) -> Path:
    sidecar_path = lora_path.with_suffix(lora_path.suffix + ".json")
    payload = {
        "trigger": trigger,
        "preset": cfg.get("preset_name"),
        "rank": int(cfg["rank"]),
        "alpha": int(cfg["alpha"]),
        "steps": int(cfg["steps"]),
        "lr": float(cfg["lr"]),
        # Legacy `resolution` field — kept for backwards-compat with
        # consumers that still read it. Set to width when widescreen,
        # to the single square value when legacy.
        "resolution": int(cfg.get("width") or cfg.get("resolution") or 576),
        "width": int(cfg.get("width") or cfg.get("resolution") or 576),
        "height": int(cfg.get("height") or cfg.get("resolution") or 576),
        "image_count": image_count,
        "caption_strategy": cfg["caption_strategy"],
        "target_modules": _normalize_target_modules(cfg.get("target_modules")),
        "training_wall_seconds": round(float(training_wall_s), 1),
        "created_at": _dt.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
        # The RESOLVED base, not a literal. The old hardcoded string named
        # the q4 dev regardless of what loaded — the exact provenance users
        # pasted into #62 while their real base was unknown.
        "base_model": _resolved_base_label(),
        "training_resolution": [
            int(cfg.get("width") or cfg.get("resolution") or 576),
            int(cfg.get("height") or cfg.get("resolution") or 576),
        ],
        "lora_lab_version": LORA_LAB_VERSION,
        # Which arm of the #62 recipe switches trained this file. Absent on
        # anything before iter7, which all trained as the legacy values.
        "recipe": resolve_recipe(cfg),
        "loadable_via": "ltx_core_mlx.loader.fuse_loras.apply_loras",
    }
    if strength is not None:
        # Kept with the adapter so a file that turns up months later can be
        # judged without re-measuring, and so the panel can show the verdict
        # next to the character it belongs to.
        payload["adapter_strength"] = strength
    sidecar_path.write_text(json.dumps(payload, indent=2))
    emit("sidecar_written", path=str(sidecar_path))
    return sidecar_path


# ----------------------------------------------------------------------
# Main pipeline
# ----------------------------------------------------------------------

def run_pipeline(spec_path: Path) -> int:
    spec = load_spec(spec_path)
    job_id: str = spec["job_id"]
    trigger: str = spec["trigger"]
    preset: str = spec["preset"]
    images_dir = Path(spec["images_dir"]).resolve()
    output_lora_path = Path(spec["output_lora_path"]).resolve()
    advanced = spec.get("advanced") or {}

    emit("start", job_id=job_id)

    # Weights first: a missing pack is a ten-second answer, not a failure
    # minutes into the caption encode (4.19.1, lora_lab.train).
    from lora_lab import train as lab_train
    _missing = lab_train.training_weights_problem(DEFAULT_MODEL_PATH)
    if _missing:
        emit_error_and_exit("models", _missing, exc=FileNotFoundError(_missing))

    # Resolve preset + overrides.
    cfg = resolve_preset(preset, advanced)
    cfg["preset_name"] = preset
    caption_strategy = cfg["caption_strategy"]
    # The Phosphene panel speaks a slightly different caption_strategy
    # vocabulary than the trainer (historical reasons — panel evolved
    # around CivitAI conventions; trainer around dreambooth). Map the
    # panel names to ours:
    #   * 'trigger_simple' — panel's name for the fallback that writes
    #     `<trigger> man` for every image. Equivalent to our 'class_word'.
    #   * 'user_provided'  — panel asserts every image has a `.txt`
    #     sidecar (verified in /train/start). The crop step above already
    #     honors those files; `caption_strategy` here only governs the
    #     fallback path for images WITHOUT a sidecar, which by definition
    #     won't fire when 'user_provided' is set. Aliasing to 'class_word'
    #     keeps the fallback sensible if the contract is ever broken
    #     (e.g. a sidecar is deleted between /train/start validation and
    #     the crop loop reading it).
    _PANEL_CAPTION_STRATEGY_ALIASES = {
        "trigger_simple": "class_word",
        "user_provided": "class_word",
    }
    if caption_strategy in _PANEL_CAPTION_STRATEGY_ALIASES:
        original = caption_strategy
        caption_strategy = _PANEL_CAPTION_STRATEGY_ALIASES[caption_strategy]
        cfg["caption_strategy"] = caption_strategy
        emit(
            "log",
            line=f"caption_strategy: panel-alias {original!r} → {caption_strategy!r}",
        )
    if caption_strategy == "auto_caption":
        # Deferred — fall back to class_word with a warning.
        emit(
            "warning",
            stage="caption",
            message="auto_caption deferred; falling back to class_word",
        )
        cfg["caption_strategy"] = "class_word"
        caption_strategy = "class_word"
    if caption_strategy not in {"class_word", "trigger_only"}:
        emit_error_and_exit(
            "caption", f"unknown caption_strategy: {caption_strategy!r}"
        )

    crop_strategy = advanced.get("crop_strategy") or "center"
    if crop_strategy not in ("center", "letterbox"):
        emit(
            "warning",
            stage="crop",
            message=f"crop_strategy={crop_strategy!r} not implemented; using center",
        )
        crop_strategy = "center"

    # Resolve target dims (width × height). Spec may provide `width`
    # and `height` independently (widescreen training, 2026-05-20+) OR
    # a legacy `resolution` which is interpreted as a square canvas.
    target_w, target_h = _wh_from_advanced(cfg)
    if target_w % 32 != 0 or target_h % 32 != 0:
        emit_error_and_exit(
            "config",
            f"target dims {target_w}×{target_h} must both be divisible by 32",
        )
    # Stamp resolved dims back onto cfg so downstream sidecar + estimator
    # see consistent values even when only `resolution` was set.
    cfg["width"] = target_w
    cfg["height"] = target_h
    # Legacy single-int — keep for any consumer that still reads it.
    resolution = target_w  # NB: when widescreen, this is just the WIDTH

    # Source images.
    try:
        source_files = list_source_images(images_dir)
    except Exception as exc:  # noqa: BLE001
        emit_error_and_exit("ingest", str(exc), exc=exc)

    image_count = len(source_files)

    try:
        recipe = resolve_recipe(cfg)
    except ValueError as exc:
        emit_error_and_exit("config", str(exc), exc=exc)

    estimated_wall_s = estimate_wall_seconds(image_count, preset, advanced)
    emit(
        "plan",
        recipe=recipe,
        preset=preset,
        rank=int(cfg["rank"]),
        alpha=int(cfg["alpha"]),
        steps=int(cfg["steps"]),
        lr=float(cfg["lr"]),
        resolution=resolution,
        caption_strategy=caption_strategy,
        estimated_wall_s=int(estimated_wall_s),
        image_count=image_count,
    )

    # Lay out the job workspace alongside the source images. The panel writes
    # spec.json into state/train_character/<job_id>/, and `images_dir` lives
    # at .../<job_id>/images. We park our derived data under the same parent.
    job_root = images_dir.parent
    cropped_dir = job_root / "cropped"
    captions_dir = job_root / "captions"
    images_renamed_dir = job_root / "images_renamed"
    data_root = job_root / "training_data"
    train_out_dir = job_root / "train_output"

    try:
        renamed, _captions = crop_and_caption(
            source_files=source_files,
            cropped_dir=cropped_dir,
            captions_dir=captions_dir,
            images_renamed_dir=images_renamed_dir,
            trigger=trigger,
            caption_strategy=caption_strategy,
            width=target_w,
            height=target_h,
            crop_strategy=crop_strategy,
        )
    except Exception as exc:  # noqa: BLE001
        traceback.print_exc(file=sys.stderr)
        emit_error_and_exit("crop", str(exc), exc=exc)

    try:
        run_preprocess(
            images_dir=images_renamed_dir,
            captions_dir=captions_dir,
            output_root=data_root,
            width=target_w,
            height=target_h,
            total=len(renamed),
        )
    except Exception as exc:  # noqa: BLE001
        traceback.print_exc(file=sys.stderr)
        emit_error_and_exit("preprocess", str(exc), exc=exc)

    try:
        checkpoint_path, training_wall_s = run_training(
            cfg=cfg,
            data_root=data_root,
            output_dir=train_out_dir,
            estimated_wall_s=estimated_wall_s,
        )
    except Exception as exc:  # noqa: BLE001
        traceback.print_exc(file=sys.stderr)
        emit_error_and_exit("train", str(exc), exc=exc)

    # Move the checkpoint to the spec's output path.
    output_lora_path.parent.mkdir(parents=True, exist_ok=True)
    try:
        shutil.copy2(str(checkpoint_path), str(output_lora_path))
    except Exception as exc:  # noqa: BLE001
        emit_error_and_exit("publish", f"failed to copy checkpoint: {exc}", exc=exc)

    # Measure before the sidecar so the strength verdict is on record even if
    # sidecar writing fails — the number is the part a stuck user needs.
    strength = report_adapter_strength(output_lora_path)

    try:
        write_sidecar(
            lora_path=output_lora_path,
            cfg=cfg,
            trigger=trigger,
            image_count=image_count,
            training_wall_s=training_wall_s,
            strength=strength,
        )
    except Exception as exc:  # noqa: BLE001
        emit_error_and_exit("sidecar", str(exc), exc=exc)

    emit("done")
    return 0


def main(argv: list[str] | None = None) -> int:
    p = argparse.ArgumentParser(description="LoRA-lab Train Character pipeline")
    p.add_argument("spec", nargs="?", help="path to job-spec JSON (positional)")
    p.add_argument("--spec", dest="spec_flag", help="path to job-spec JSON (flag form, used by Phosphene panel)")
    p.add_argument("--job-id", dest="job_id", default=None,
                   help="optional job id label (used by Phosphene panel; the spec JSON's own job_id field is authoritative)")
    p.add_argument("-v", "--verbose", action="store_true")
    args = p.parse_args(argv)
    if args.spec_flag and not args.spec:
        args.spec = args.spec_flag
    if not args.spec:
        p.error("must provide a job-spec JSON path (positional or via --spec)")

    logging.basicConfig(
        level=logging.DEBUG if args.verbose else logging.INFO,
        format="%(asctime)s %(levelname)s %(name)s | %(message)s",
        stream=sys.stderr,  # NEVER pollute stdout — JSON-lines protocol
    )

    try:
        return run_pipeline(Path(args.spec))
    except SystemExit:
        raise
    except Exception as exc:  # noqa: BLE001
        traceback.print_exc(file=sys.stderr)
        emit("error", stage="unknown", message=str(exc),
             error_class=classify_exception(exc))
        return 1


if __name__ == "__main__":
    sys.exit(main())