Skip to content

Free, open-source AI video generator for Mac. Run LTX-2.5 and MiniMax H3 (Hailuo) locally on Apple Silicon: text-to-video and image-to-video with sound, lip-sync, character LoRA training, songs from lyrics (YuE2) and image editing (Qwen-Image-Edit). One-click install via Pinokio. Offline: no cloud, no API key, no NVIDIA GPU.

33.0 KB
Raw Blame
"""LoRA training support: dataset and caption-map helpers, completed-run listing, model preflight, failure classification.

The helpers around a training job (the job itself, run_train_job_inner, stays in the panel): dataset folders and caption maps, the dev-transformer install, time estimates, advice, and turning a runner event or exit code into a plain-English error.

Moved verbatim out of mlx_ltx_panel.py. The body of each function is unchanged
except that panel globals and this module's own functions are spelled
`P.<name>` (the same mechanical rename panel/routes_*.py uses). `P` is the
RUNNING panel module, assigned by mlx_ltx_panel where it imports this module;
it is never imported here because the panel usually runs as __main__ and
importing it by name would execute the whole file a second time. Routing every
cross-reference through `P` keeps test patches effective: a test that patches
`P.USAGE_LOG_FILE` or `P._usage_fleet_query_one` still reaches the code that
uses it. Constants and locks stay defined in the panel.

The names below stay importable from mlx_ltx_panel, which re-exports them, so
panel/routes_*.py and the tests are unchanged.
"""
from __future__ import annotations

from pathlib import Path
import json
import os
import re
import threading
import time

P = None  # the running mlx_ltx_panel module; assigned where the panel imports this


def _train_dataset_dir(job_id: str) -> Path:
    """Per-job dataset folder. Holds images/, captions/, spec.json."""
    p = P.TRAIN_DIR / job_id
    return p


def _train_caption_map_path(dataset_dir: Path) -> Path:
    return dataset_dir / P._TRAIN_CAPTION_MAP_NAME


def _train_read_caption_map(dataset_dir: Path) -> dict[str, str]:
    p = P._train_caption_map_path(dataset_dir)
    if not p.is_file():
        return {}
    try:
        data = json.loads(p.read_text(encoding="utf-8"))
        if isinstance(data, dict):
            return {str(k): str(v) for k, v in data.items()
                    if isinstance(k, str) and isinstance(v, str)}
    except (OSError, json.JSONDecodeError):
        pass
    return {}


def _train_write_caption_map(dataset_dir: Path, mapping: dict[str, str]) -> None:
    p = P._train_caption_map_path(dataset_dir)
    try:
        p.write_text(json.dumps(mapping, indent=2), encoding="utf-8")
    except OSError:
        pass


def _train_set_caption_map_entry(dataset_dir: Path,
                                 original_stem: str,
                                 saved_stem: str) -> None:
    m = P._train_read_caption_map(dataset_dir)
    m[original_stem] = saved_stem
    P._train_write_caption_map(dataset_dir, m)


def _train_reconcile_one(dataset_dir: Path,
                         original_stem: str,
                         saved_stem: str) -> bool:
    """If a caption file exists under `original_stem.txt`, rename it to
    `saved_stem.txt`. Used when the user uploads `image_001.txt` BEFORE its
    matching `image_001.png` arrives. Returns True if a paired caption now
    sits under `<captions>/<saved_stem>.txt`."""
    captions_dir = dataset_dir / "captions"
    saved_path = captions_dir / f"{saved_stem}.txt"
    if saved_path.is_file():
        return True
    if original_stem == saved_stem:
        return False
    parked = captions_dir / f"{original_stem}.txt"
    if parked.is_file():
        try:
            parked.rename(saved_path)
            return True
        except OSError:
            return False
    return False


def _train_reconcile_captions(dataset_dir: Path) -> tuple[int, int, list[str]]:
    """Called on training start. Walks images/ + captions/, ensures every
    caption under an *original* stem in caption_map.json is moved to its
    saved stem so the trainer's per-image lookup hits. Returns
    (paired_count, total_images, warnings)."""
    images_dir = dataset_dir / "images"
    captions_dir = dataset_dir / "captions"
    cmap = P._train_read_caption_map(dataset_dir)
    if not images_dir.is_dir():
        return 0, 0, ["images dir missing"]
    images = [p for p in sorted(images_dir.iterdir())
              if p.is_file() and p.suffix.lower() in P.TRAIN_IMAGE_EXTS]
    total = len(images)
    warnings: list[str] = []
    if not captions_dir.is_dir():
        return 0, total, warnings
    for original, saved in cmap.items():
        if original == saved:
            continue
        parked = captions_dir / f"{original}.txt"
        target = captions_dir / f"{saved}.txt"
        if parked.is_file() and not target.is_file():
            try:
                parked.rename(target)
            except OSError as e:
                warnings.append(f"could not move {parked.name} → {target.name}: {e}")
    paired = sum(1 for img in images
                 if (captions_dir / f"{img.stem}.txt").is_file())
    return paired, total, warnings


def _train_estimate_seconds(preset: str, image_count: int, *,
                            steps_override: int | None = None,
                            is_style: bool = False) -> int:
    """Wall-time estimate in seconds for a training run. The image count
    has a small overhead for the preprocess pass (image → VAE latent
    cache, ~3 s/image on M4 Max), then `steps * seconds_per_step` for
    the train loop itself.

    `is_style` switches the preset lookup table — style training pulls
    its seconds_per_step from TRAIN_STYLE_PRESETS, not TRAIN_PRESETS."""
    table = P.TRAIN_STYLE_PRESETS if is_style else P.TRAIN_PRESETS
    cfg = table.get(preset) or table["quick"]
    # Steps derive from epochs × image_count; advanced override wins.
    steps = steps_override if steps_override else P._preset_steps_for(cfg, image_count)
    preprocess = 3.0 * max(0, image_count)
    train = float(steps) * float(cfg["seconds_per_step"])
    return int(preprocess + train + 30)  # +30 s for setup + checkpoint write


def _train_list_completed() -> list[dict]:
    """Scan mlx_models/loras/ for .safetensors with a Phosphene-trained
    sidecar (`kind=train_character` OR `kind=train_style`) and return one
    record per discovered run for the Trained-LoRAs list in the Train tab.
    Surfacing both kinds in a single list lets the user see their full
    training inventory in one place; the UI filters by `train_type` to show
    the kind that matches the active train-type toggle (and lists everything
    when "All" is selected)."""
    out: list[dict] = []
    if P.LORAS_DIR.exists():
        for p in sorted(P.LORAS_DIR.iterdir(),
                        key=lambda x: x.stat().st_mtime if x.exists() else 0,
                        reverse=True):
            if p.suffix.lower() != ".safetensors":
                continue
            # Style LoRAs use a `.style.safetensors` suffix, so the sidecar
            # is `.style.json` not `.json`. Probe both paths.
            sidecar_candidates = [p.with_suffix(".json")]
            if p.name.endswith(".style.safetensors"):
                sidecar_candidates.insert(
                    0, p.with_name(p.name[: -len(".safetensors")] + ".json")
                )
            sidecar = next((s for s in sidecar_candidates if s.is_file()), None)
            if sidecar is None:
                continue
            try:
                meta = json.loads(sidecar.read_text(encoding="utf-8"))
            except Exception:
                continue
            kind = meta.get("kind")
            if kind not in ("train_character", "train_style"):
                continue
            try:
                stat = p.stat()
            except OSError:
                continue
            verdict = P._adapter_verdict(meta)
            out.append({
                "id": meta.get("job_id") or p.stem,
                "name": meta.get("name") or p.stem,
                "trigger": meta.get("trigger") or "",
                "preset": meta.get("preset") or "",
                "kind": kind,
                "train_type": meta.get("train_type")
                                or ("style" if kind == "train_style"
                                    else "character"),
                "path": str(p),
                "filename": p.name,
                "size_mb": round(stat.st_size / (1024 * 1024), 1),
                "created_at": meta.get("created_at") or stat.st_mtime,
                "image_count": meta.get("image_count") or 0,
                # WHETHER THIS FILE CAN DO ANYTHING, carried out of the
                # sidecar so the library can warn before somebody spends a
                # render finding out. "unknown" for anything trained by a
                # build that did not measure — silence is not weakness, and a
                # chip on every older LoRA would be noise nobody could act on.
                "adapter_verdict": verdict,
                # AND WHAT TO DO ABOUT IT. A verdict the user cannot act on
                # sends them to a GitHub thread to find out what their own
                # number meant (#62); the remedy travels with the verdict, and
                # is hardware-aware because on a sub-64 GB Mac "use High" is
                # not a thing that machine can do.
                "adapter_advice": (
                    P._train_weak_advice(
                        verdict,
                        str(meta.get("preset") or ""),
                        kind == "train_style",
                    )
                    if verdict not in ("ok", "unknown") else ""
                ),
            })
    return out


def _train_weak_advice(verdict: str, preset: str = "", is_style: bool = False) -> str:
    """One sentence a user can ACT on when their adapter measures weak/inert.

    The measurement shipped in v4.6.0 and the number was correct; what it did
    not do was tell anyone what to do about it. A first-time user on the Quick
    preset read "1.98e-04", found nothing in the panel that explained it, and
    ended up reading a 21-comment GitHub thread to learn that rank 8 has never
    carried a face (#62). The verdict without the remedy is the actual product
    failure in that issue.

    The remedy is hardware-dependent, which is why this is a function and not
    a constant: on a Mac under 64 GB `_select_train_profile` rewrites the whole
    table and *High is rank 8 at 500 steps on two projections*, so "train again
    on High" would be advice that machine cannot honour.
    """
    verdict = (verdict or "").lower()
    if verdict == "inert":
        return (
            "This adapter carries no weight delta at all — every low-rank "
            "product is exactly zero, so it cannot change a single pixel. "
            "Treat the run as failed rather than as a weak result: nothing "
            "was learned. Check the Logs for a trainer error, then train "
            "again."
        )
    compact = bool(P.TRAIN_PROFILE.get("compact"))
    if is_style:
        head = ("This style adapter came out weak: the deltas it learned may "
                "be too small to visibly change a render.")
    else:
        head = ("This character came out weak: the deltas it learned are "
                "below the level any adapter that has carried a face here "
                "has ever measured, so renders will most likely look "
                "LoRA-free.")
    if compact:
        ram = P.TRAIN_PROFILE.get("ram_gb")
        where = f"{ram} GB" if ram else "under 64 GB"
        return (
            f"{head} This Mac has {where}, so training runs the compact "
            "profile — even High is rank 8 / 500 steps / 448px on two of the "
            "four attention projections, not the rank-32 recipe that was "
            "graded on a face. No preset on this machine reaches that recipe, "
            "so this is the hardware and not your photos. More images and "
            "sharper, more varied ones still help; a 64 GB-class Mac is what "
            "unlocks the graded recipe."
        )
    if (preset or "").lower() != "high":
        return (
            f"{head} Train again on the High preset — rank 8 (Quick) and "
            "rank 16 (Medium) have never carried an identity here, and High "
            "is the only recipe that has. Raising the character strength "
            "toward 2.0 helps a weak adapter a little; it cannot rescue one "
            "this far under the floor."
        )
    return (
        f"{head} This was already the High preset, so the next levers are the "
        "dataset rather than the recipe: more images (20-40), sharper and more "
        "varied in pose and lighting, and captions that name the trigger plainly. "
        "Please attach the training log to https://github.com/mrbizarro/phosphene/issues/62 "
        "— a weak High run is the case that issue is still open on."
    )


def _big_enough(path: Path, min_bytes: int) -> bool:
    try:
        return path.exists() and path.stat().st_size >= min_bytes
    except OSError:
        return False


def _train_required_models() -> list[dict]:
    """Return the list of model files Train Character needs and whether each
    is present locally. Surfaces missing models to the UI so the user can
    one-click download them via the existing hf flow.

    Train Character requires the FULL-PRECISION `transformer-dev.safetensors`
    from the LTX-2.3 Q8 repo (~21 GB). The dev base is required because
    distilled is structurally wrong for training (different sigma schedule
    than the trainer's flow-matching objective) — and it must be the Q8
    repo's full-precision copy because the Q4 repo's transformer-dev is
    QUANTIZED (~11 GB): the trainer refuses it (train.py's
    FULL_DEV_MIN_BYTES) since it silently trains a near-zero LoRA (#35).
    The default install ships neither, to save disk for inference-only
    users.

    Gemma + the rest of the Q4 stack (vae, audio, etc.) are already required
    for inference so they should already be present, but we check anyway.
    """
    q4_local_dir = P.MODELS_DIR / "ltx-2.3-mlx-q4"
    q8_local_dir = P.MODELS_DIR / "ltx-2.3-mlx-q8"
    gemma_local_dir = P.MODELS_DIR / "gemma-3-12b-it-4bit"
    gemma_repo_id = "mlx-community/gemma-3-12b-it-4bit"

    def _file_present(local_dir: Path, repo_id: str, filename: str,
                      *, extra_dirs: list[Path] | None = None,
                      min_bytes: int = P._MIN_FILE_BYTES,
                      caches: bool = True) -> bool:
        """Check known locations for a model file in this order:

          1. The primary local_dir (where the helper expects it by default).
          2. Any `extra_dirs` callers list explicitly — used for files that
             can legitimately live in more than one place. The dev
             transformer is the canonical example: original downloads
             land in mlx_models/ltx-2.3-mlx-q4/ but Phosphene also ships
             a Q8-friendly copy under mlx_models/ltx-2.3-mlx-q8/, and the
             trainer happily loads either. Without this fallback the
             preflight reports "missing" on installs that only have the
             Q8-side copy and prompts an unnecessary 11 GB redownload.
          3. The configured HF_HOME hub for repo_id.
          4. The user-default HF cache (~/.cache/huggingface/hub) — covers
             installs where HF_HOME wasn't set at first download time and
             weights landed in the default location.
        """
        candidates: list[Path] = [local_dir / filename]
        for d in (extra_dirs or []):
            candidates.append(d / filename)
        if not caches:      # the file must be where its loader reads it
            return any(_big_enough(c, min_bytes) for c in candidates)
        cache_dir = P._repo_hf_cache_dir(repo_id)
        if cache_dir is not None:
            candidates.append(cache_dir / filename)
        # Last-resort: default HF cache, regardless of HF_HOME. Matches what
        # huggingface_hub does when neither env var is set. Old downloads
        # often live here even when HF_HOME is now pointing elsewhere.
        safe = repo_id.replace("/", "--")
        default_root = Path.home() / ".cache" / "huggingface" / "hub" / f"models--{safe}" / "snapshots"
        try:
            if default_root.is_dir():
                snaps = sorted(default_root.iterdir(),
                               key=lambda d: d.stat().st_mtime, reverse=True)
                if snaps:
                    candidates.append(snaps[0] / filename)
        except OSError:
            pass
        for c in candidates:
            try:
                if c.exists() and c.stat().st_size >= min_bytes:
                    return True
            except OSError:
                continue
        return False

    # Mirror of lora_lab.failure.FULL_DEV_MIN_BYTES: anything smaller is a
    # quantized transformer-dev, which the trainer refuses. The preflight
    # must not show green on a file the trainer will reject (#35).
    _FULL_DEV_MIN_BYTES = 15 * 1000**3

    items = [
        {
            # THE ASSUMPTION THAT EXPIRED. This function's own docstring says
            # "the rest of the Q4 stack (vae, audio, etc.) are already required
            # for inference so they should already be present" — true right up
            # until v4.0, when a fresh install stopped fetching LTX-2.3 at all.
            # The trainer runs against 2.3 and needs its VAE, audio VAE and
            # vocoder; without this row the Train tab would have looked ready
            # and failed at load time on a pack nobody told the user about.
            "key": "ltx23_base",
            "label": "LTX-2.3 base pack (training runs against 2.3)",
            "blurb": "The trainer runs against LTX-2.3, not 2.5 — it needs 2.3's "
                     "VAE, audio VAE and vocoder. Your clips still render on "
                     "LTX-2.5; this is training-only. A fresh v4.0 install does "
                     "not download it, so it is fetched when you want to train.",
            "repo_id": "dgrauet/ltx-2.3-mlx-q4",
            "filename": "vae_decoder.safetensors",
            "local_dir": str(q4_local_dir),
            "size_gb": 20.48,
            # Local folder only: the trainer loads 2.3 from
            # mlx_models/ltx-2.3-mlx-q4 and nowhere else (lora_lab.failure.
            # training_weights_problem). A Hugging Face cache copy used to
            # read "ready" here while every Start died in a second.
            "ready": _file_present(q4_local_dir, "dgrauet/ltx-2.3-mlx-q4",
                                   "vae_decoder.safetensors", caches=False),
        },
        {
            "key": "ltx_dev_transformer",
            "label": "LTX-2.3 dev transformer (training-only, full precision)",
            # SYS-15/SYS-42: "the dev transformer has the right flow-matching
            # schedule for LoRA-from-images training" was lab jargon a
            # creator can't act on. Outcome language: what it's for, and
            # what goes wrong if the wrong copy is used.
            "blurb": "Required for training — a different file from the one "
                     "your renders already use. Must be the full-precision "
                     "copy (~21 GB): a smaller, compressed copy trains a "
                     "LoRA that looks fine in the log and does nothing when "
                     "you use it.",
            "repo_id": "dgrauet/ltx-2.3-mlx-q8",
            "filename": "transformer-dev.safetensors",
            "local_dir": str(q8_local_dir),
            "size_gb": 20.6,
            # Size-gated: an 11 GB quantized dev (the old preflight's Q4
            # download, still on many disks) must show NOT-ready, because
            # the trainer refuses it and the LoRA comes out dead (#35).
            # The trainer's own two places, never a Hugging Face cache: fleet
            # 2 Nov 2026, a Train tab that read "ready" (or a Start that was
            # never gated) let one Mac fail 22 runs in an hour, each in under
            # a second, on "the dev transformer is not downloaded".
            "ready": _file_present(q8_local_dir, "dgrauet/ltx-2.3-mlx-q8",
                                   "transformer-dev.safetensors",
                                   extra_dirs=[q4_local_dir],
                                   min_bytes=_FULL_DEV_MIN_BYTES,
                                   caches=False),
        },
        {
            "key": "gemma_text_encoder",
            "label": "Gemma 3 12B (text encoder)",
            "blurb": "Encodes prompts for both inference and training. Already "
                     "required by Phosphene's renderer, so this is usually green.",
            "repo_id": gemma_repo_id,
            "filename": "config.json",
            "local_dir": str(gemma_local_dir),
            "size_gb": 6.0,
            "ready": _file_present(gemma_local_dir, gemma_repo_id, "config.json"),
        },
    ]
    return items


def _train_install_dev_transformer(push_log) -> dict:
    """Trigger an hf download for just `transformer-dev.safetensors` from
    the LTX-2.3 Q8 repo — the FULL-PRECISION copy the trainer requires.
    (Until v3.4.1 this pulled the Q4 repo's transformer-dev, which is
    quantized: the trainer refuses it and the LoRA trains dead — #35.)
    Reuses the same `hf download` binary the existing
    HF download flow uses; runs synchronously in a worker thread so the
    caller can stream progress (the panel currently invokes this from a
    /train/install endpoint that returns quickly + then polls /status).

    Returns {"ok": True, "started": True, "repo_id": ...} or {"ok": False,
    "error": "..."} if the download couldn't be started (already running,
    binary missing, etc.).
    """
    import subprocess
    if P.DOWNLOAD["active"]:
        return {"ok": False,
                "error": f"another download is already active "
                         f"({P.DOWNLOAD.get('repo_id', '?')})."}

    repo_id = "dgrauet/ltx-2.3-mlx-q8"
    local_dir = P.MODELS_DIR / "ltx-2.3-mlx-q8"
    local_dir.mkdir(parents=True, exist_ok=True)

    hf_bin = P.HF_BIN if P.HF_BIN is not None else P._resolve_hf()
    if hf_bin is None:
        return {"ok": False, "error": "hf CLI not found on PATH."}

    cmd = [str(hf_bin), "download", repo_id,
           "--include", "transformer-dev.safetensors",
           "--local-dir", str(local_dir)]

    env = dict(os.environ)
    hf_token = P._active_hf_token()
    if hf_token:
        env["HF_TOKEN"] = hf_token
        env["HUGGING_FACE_HUB_TOKEN"] = hf_token

    with P.DOWNLOAD_LOCK:
        P.DOWNLOAD["active"] = True
        P.DOWNLOAD["key"] = "ltx_dev_transformer"
        P.DOWNLOAD["repo_id"] = repo_id
        P.DOWNLOAD["started_ts"] = time.time()
        P.DOWNLOAD["last_line"] = ""

    def _runner():
        try:
            push_log(f"[hf] dev transformer download started "
                     f"({repo_id} / transformer-dev.safetensors, ~21 GB — "
                     f"the full-precision copy training requires).")
            proc = subprocess.Popen(
                cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                text=True, errors="replace", bufsize=1, env=env, start_new_session=True)
            if not P._download_register(proc):
                push_log("[hf] dev transformer download cancelled.")
                return
            buf = ""
            assert proc.stdout is not None
            while True:
                ch = proc.stdout.read(1)
                if not ch:
                    break
                if ch in ("\n", "\r"):
                    line = buf.strip()
                    buf = ""
                    if line:
                        with P.DOWNLOAD_LOCK:
                            P.DOWNLOAD["last_line"] = line[:200]
                        push_log(f"[hf:dev] {line[:300]}")
                else:
                    buf += ch
            rc = proc.wait()
            if rc == 0:
                push_log("[hf] dev transformer downloaded successfully.")
            else:
                push_log(f"[hf] dev transformer download exited with code {rc}.")
        except Exception as exc:
            push_log(f"[hf] dev transformer crashed: {exc}")
        finally:
            with P.DOWNLOAD_LOCK:
                P.DOWNLOAD["active"] = False
                P.DOWNLOAD["key"] = None
                P.DOWNLOAD["repo_id"] = None
                P.DOWNLOAD["started_ts"] = None
                P.DOWNLOAD["last_line"] = ""
                P.DOWNLOAD["proc"] = None
                P.DOWNLOAD["pgid"] = None
                P.DOWNLOAD["cancelled"] = False

    threading.Thread(target=_runner, daemon=True, name="hf-dev-download").start()
    return {"ok": True, "started": True, "repo_id": repo_id,
            "filename": "transformer-dev.safetensors"}


def _train_phase_from_line(line: str, payload: "dict | None", current: str) -> str:
    """Which phase the trainer is in, from the lines it prints.

    The preprocessor's phase banners are plain text (they reach the panel
    on the merged stdout/stderr pipe); a training step is a JSON event.
    Anything else keeps the current phase. Phases: start -> text_encode ->
    image_encode -> train.
    """
    if isinstance(payload, dict):
        if (payload.get("event") or "step") in ("step", "train_progress"):
            return "train"
        return current
    s = line or ""
    if "Phase 1: encoding text captions" in s:
        return "text_encode"
    if "Phase 2: encoding image latents" in s:
        return "image_encode"
    return current


def _train_encode_retry_wanted(rc: int, watchdog_seen: bool, phase: str,
                               already_retried: bool) -> bool:
    """One automatic relaunch, and only for the kill this can fix (#61):
    SIGABRT with a Metal-watchdog signature in the log, during the caption
    encode, not already retried. A kill during image encode or training
    is the canvas-sized problem the existing error names; a crash without
    the signature is not a GPU timeout at all."""
    return (rc == -6 and bool(watchdog_seen) and phase == "text_encode"
            and not already_retried)


def _train_gemma_fallback_max_length() -> int:
    """The shorter Gemma pad the relaunch uses — the same knob and default
    (256, floor 64) as the render helper's fallback, so one env var tunes
    both: LTX_GEMMA_FALLBACK_MAX_LENGTH."""
    try:
        return max(64, int(os.environ.get("LTX_GEMMA_FALLBACK_MAX_LENGTH", "256") or 256))
    except ValueError:
        return 256


def _train_wipe_caption_conditions(dataset_dir: Path) -> int:
    """Discard the caption encodings a killed run wrote, so the relaunch
    re-encodes every caption at one length — the preprocessor skips a
    condition file that exists, and a dataset half at 1024 tokens and half
    at 256 would not batch. Image latents live beside them and are left
    alone. Returns how many files went."""
    cond = Path(dataset_dir) / "training_data" / ".precomputed" / "conditions"
    if not cond.is_dir():
        return 0
    n = 0
    for f in cond.iterdir():
        if f.is_file() and f.name.startswith("condition_"):
            try:
                f.unlink()
                n += 1
            except OSError:
                pass
    return n


def train_error_from_event(payload: dict) -> dict:
    """The trainer's `error` event, its class held to the closed list."""
    cls = str(payload.get("error_class") or "other")
    return {
        "stage": str(payload.get("stage") or "unknown")[:40],
        "message": str(payload.get("message") or "")[:2000],
        "error_class": cls if cls in P.TRAIN_ERROR_CLASSES else "other",
    }


def train_error_for_exit(rc: int, trainer_error: dict | None, tail=None) -> str:
    """The class of a trainer that exited non-zero. The trainer's own word
    wins; a process killed by SIGKILL without one is macOS reclaiming memory
    (jetsam), the one signal death that has a single cause. A trainer that
    crashed before it could send its word is classed from its last traceback
    line (an ImportError at startup is `import_error`, not `other`)."""
    if trainer_error and trainer_error.get("error_class") in P.TRAIN_ERROR_CLASSES:
        return trainer_error["error_class"]
    if rc == -9:
        return "oom"
    exc = train_exception_from_tail(tail)
    if exc:
        return train_class_from_exception_name(*exc)
    return "other"


def train_stage_word(trainer_error: dict | None) -> str:
    """The stage the trainer stopped in, from lora_lab.failure.TRAIN_STAGES
    only (it rides on analytics as `train_stage`); "unknown" without one."""
    from lora_lab.failure import TRAIN_STAGES
    st = str((trainer_error or {}).get("stage") or "unknown")
    return st if st in TRAIN_STAGES else "unknown"


#: Opens every "can't start" refusal for a training run. The analytics
#: refusal table matches on it (slug `train_models`), so it must not change
#: without that table.
TRAIN_WEIGHTS_REFUSAL_PREFIX = "Training can't start yet: "


def train_weights_refusal() -> str | None:
    """Why a training run cannot start on this Mac, or None (fleet 2 Nov 2026).

    The SAME question the trainer asks first (lora_lab.failure.
    training_weights_problem), asked before a run is queued, retried or
    spawned. Until now only the trainer asked it: the run was queued, the
    trainer stopped in under a second, the card said "training exited with
    code 1", and the same Start (or Retry) did it again - 22 times in an hour
    on one Mac, 31 failed runs from 3 people in a day."""
    from lora_lab.failure import training_weights_problem
    problem = training_weights_problem(P.MODELS_DIR / "ltx-2.3-mlx-q4")
    if not problem:
        return None
    return TRAIN_WEIGHTS_REFUSAL_PREFIX + problem[0].upper() + problem[1:]


_TRAIN_STAGE_WORDS = {
    "models": "before it started",
    "caption": "while preparing the captions",
    "config": "while reading the settings",
    "ingest": "while reading the photos",
    "crop": "while cropping the photos",
    "preprocess": "while encoding the photos and captions",
    "train": "while training",
    "publish": "while saving the result",
    "sidecar": "while saving the result",
    "validate": "while checking the voice clip",
    "verify": "while checking the result",
}

# The last exception line of a Python traceback: "SomeError: text" or
# "pkg.mod.SomeError: text", at the start of the line.
_EXC_LINE = re.compile(r"^((?:[A-Za-z_][\w]*\.)*[A-Za-z_]\w*(?:Error|Exception|Exit|Interrupt))(?::\s*(.*))?$")


def train_exception_from_tail(lines) -> tuple[str, str] | None:
    """(type name, message) of the last traceback line in the trainer's plain
    output, or None. For a trainer that died before it could send its own
    `error` event (an import that failed, a crash in setup)."""
    for line in reversed(list(lines or [])):
        m = _EXC_LINE.match(str(line).strip())
        if m:
            return m.group(1).rsplit(".", 1)[-1], (m.group(2) or "").strip()
    return None


def train_class_from_exception_name(name: str, message: str = "") -> str:
    """lora_lab.failure's closed class from a bare exception type name and its
    text - the same reading classify_exception makes of a live exception."""
    from lora_lab.failure import _DISK_FRAGMENTS, _OOM_FRAGMENTS
    text = (message or "").lower()
    if name == "MemoryError" or any(f in text for f in _OOM_FRAGMENTS):
        return "oom"
    if any(f in text for f in _DISK_FRAGMENTS):
        return "disk_full"
    if name in ("ImportError", "ModuleNotFoundError"):
        return "import_error"
    if name == "FileNotFoundError":
        return "missing_file"
    return "other"


def train_failure_message(rc: int, trainer_error: dict | None, tail=None) -> str:
    """What the failed training card says (fleet 2 Nov 2026).

    The trainer already says why it stopped, in a sentence written for the
    user; the panel used to drop that and show "training exited with code 1 -
    see Logs for stack trace", the only text the user and the fleet ever saw.
    The message stays on this Mac like every job error: analytics sends the
    closed class and stage (train_error, train_stage), never this text."""
    # A message of only spaces falls through to the exit-code paths below
    # (msg[0] on an empty string would turn the card into "string index out of range").
    if trainer_error and str(trainer_error.get("message") or "").strip():
        where = _TRAIN_STAGE_WORDS.get(str(trainer_error.get("stage") or ""), "")
        msg = str(trainer_error["message"]).strip()
        msg = msg[0].upper() + msg[1:]
        if trainer_error.get("stage") == "models":
            return TRAIN_WEIGHTS_REFUSAL_PREFIX + msg
        return f"Training stopped{(' ' + where) if where else ''}: {msg}"
    if rc == -15:
        return ("Training was stopped from outside the app (SIGTERM): Phosphene "
                "or the Mac was shutting down, or another program ended it. "
                "Nothing is wrong with your photos; start it again.")
    exc = train_exception_from_tail(tail)
    if exc:
        name, text = exc
        return (f"Training stopped (code {rc}): {name}"
                f"{(': ' + text) if text else ''}. The full trace is in Logs.")
    return f"training exited with code {rc} — see Logs for stack trace"


def _train_release_render_memory() -> None:
    """Stop the warm render helper before a training run (4.19.1)."""
    try:
        if P.HELPER.is_alive():
            P.push("[train] releasing the render engine's memory for training "
                 "(it reloads on the next render)")
            P.HELPER.kill()
    except Exception as e:  # never let housekeeping fail a training run
        P.push(f"[train] could not release the render engine: {e}")