Free, open-source AI video generator for Mac. Run LTX-2.5 and MiniMax H3 (Hailuo) locally on Apple Silicon: text-to-video and image-to-video with sound, lip-sync, character LoRA training, songs from lyrics (YuE2) and image editing (Qwen-Image-Edit). One-click install via Pinokio. Offline: no cloud, no API key, no NVIDIA GPU.
Python ai-filmmakingai-videoai-video-generatorapple-siliconhailuoimage-to-videolip-synclocal-ailora-trainingltx-2ltx-2-5ltx-videomacosminimax-h3mlxmusic-generationoffline-aipinokiotext-to-videovideo-generation
"""Auto-caption a Train Character dataset with the local Gemma 3 12B
multimodal model.
Standalone subprocess called by the panel's `/train/auto-caption` endpoint.
Reads images from `<dataset>/images/`, generates one `[VISUAL]: <trigger>, …`
caption per image describing what VARIES (pose / framing / clothing /
lighting / setting / mood) while explicitly skipping identity features
(face, hair, age, ethnicity) so the LoRA learns identity from pixels and
not from prose. Mirrors the caption recipe Bizarro v2 was trained on.
Why a subprocess + not the warm helper:
Lightricks' Gemma wrapper (used by the helper for prompt enhancement)
is text-only — it explicitly does not pass images through the vision
tower. We need `mlx_vlm`, which uses the same Gemma 3 weights but
invokes the `Gemma3ForConditionalGeneration` path. Running it in a
short-lived subprocess means Gemma's ~6 GB RSS frees the moment
captioning finishes; nothing accumulates on top of the dev
transformer the trainer needs next.
Protocol (mirrors lora_lab.train_character):
Emits one JSON object per line on stdout. The panel tails stdout and
pushes each line into STATE['log']. Events:
{"event":"loading"} model load start
{"event":"loaded","elapsed_sec":3.2} model ready
{"event":"progress","i":1,"n":37,"file":"…","caption":"…"}
{"event":"done","elapsed_sec":85.0,"count":37}
{"event":"error","message":"…"} fatal — exit non-zero
CLI:
python -m caption_with_gemma \
--dataset <state/train_character/donaldtrn> \
--trigger donaldtrn \
[--gemma-path <mlx_models/gemma-3-12b-it-4bit>] \
[--max-tokens 200] [--temperature 0.3]
"""
from __future__ import annotations
import argparse
import json
import pathlib
import re
import sys
import time
import traceback
def _emit(event: str, **kwargs) -> None:
"""Write one JSON event to stdout + flush. The panel reads these
line-by-line and surfaces them in the log pane."""
payload = {"event": event, **kwargs}
sys.stdout.write(json.dumps(payload, ensure_ascii=False) + "\n")
sys.stdout.flush()
# v2 caption recipe — describe what VARIES, skip identity. The "ONE
# sentence" + "50-80 words" constraints are loose targets (Gemma comes in
# slightly shorter in practice, ~30-50 words); that's fine, still well
# above the 3-word trigger_simple fallback the panel uses without us.
_SYSTEM_TMPL = (
"You are captioning images for a video character LoRA. The trigger "
"token is {trigger}. "
"Write ONE single sentence, 50-80 words, in this exact format:\n"
"[VISUAL]: {trigger}, <description>\n"
"Describe ONLY what VARIES across shots: pose, framing, clothing, "
"setting, lighting, camera angle, mood. "
"DO NOT describe facial features, hair color, age, ethnicity, or "
"identity — the LoRA absorbs those from pixels. "
"DO NOT mention the trigger word inside the description. "
"Plain prose, no bullet points, no markdown, no quotes."
)
_USER_TMPL = "Caption this image following the spec."
# Images extensions the panel accepts under <dataset>/images/. Mirrors
# TRAIN_IMAGE_EXTS in mlx_ltx_panel.py — keep in sync.
_IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp"}
def _normalise_caption(text: str, trigger: str) -> str:
"""Clean up Gemma's output to the canonical `[VISUAL]: <trigger>, body`
format. Handles every drift mode we've seen in practice:
* leading/trailing quotes the model occasionally wraps
* missing `[VISUAL]:` prefix (model jumps straight to `tok, body`)
* doubled trigger in body (model re-emits the trigger after the
prefix it was told to start with — pre-fix this produced
`[VISUAL]: tok, tok, body` on disk for a couple of images)
* extra `[VISUAL]:` segments deeper in the text
"""
text = (text or "").strip()
text = re.sub(r'^["\']|["\']$', '', text).strip()
# Step 1: extract the body by stripping ALL leading occurrences of
# `[VISUAL]:` and `<trigger>,` (with optional whitespace) — handles
# `[VISUAL]: tok, tok, body`, `tok, body`, and the bare body case in
# one pass.
pattern = re.compile(
r"^\s*(?:\[VISUAL\]\s*:\s*)?" # optional [VISUAL]:
r"(?:" + re.escape(trigger) + r"\s*,\s*)*", # zero+ trigger,
flags=re.IGNORECASE,
)
body = pattern.sub("", text, count=1).strip().lstrip(",").strip()
# Step 2: re-attach the canonical prefix exactly once.
return f"[VISUAL]: {trigger}, {body}"
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--dataset", required=True,
help="Path to the Train Character dataset dir "
"(contains images/ and captions/).")
ap.add_argument("--trigger", required=True,
help="Trigger token to embed in every caption.")
ap.add_argument("--gemma-path", default=None,
help="Override Gemma 3 model dir (default: "
"mlx_models/gemma-3-12b-it-4bit next to this script).")
ap.add_argument("--max-tokens", type=int, default=200)
ap.add_argument("--temperature", type=float, default=0.3,
help="Low temp keeps the format stable; higher = "
"more varied prose but more drift off-format.")
args = ap.parse_args()
dataset = pathlib.Path(args.dataset).resolve()
images_dir = dataset / "images"
captions_dir = dataset / "captions"
if not images_dir.is_dir():
_emit("error", message=f"images dir not found: {images_dir}")
return 1
captions_dir.mkdir(parents=True, exist_ok=True)
images = sorted(p for p in images_dir.iterdir()
if p.is_file() and p.suffix.lower() in _IMAGE_EXTS)
if not images:
_emit("error", message=f"no images in {images_dir}")
return 1
# Default Gemma path is the same one the panel's preflight already
# downloads — no extra disk required. Resolve via lora_lab's shared
# helper so this honors `LTX_MODELS_DIR` the same way every other
# lora_lab module does; anchoring to `__file__.parent` previously broke
# when this script ran from a Pinokio install where the panel had set
# LTX_MODELS_DIR to a different root than the script's directory.
if args.gemma_path:
gemma_path = pathlib.Path(args.gemma_path).resolve()
else:
try:
from lora_lab import resolve_default_text_encoder
resolved = resolve_default_text_encoder()
except ImportError:
# lora_lab not on PYTHONPATH (shouldn't happen via the panel's
# spawn path, but keep a sane fallback so the standalone CLI
# still works in a clean checkout).
resolved = str(pathlib.Path(__file__).resolve().parent
/ "mlx_models" / "gemma-3-12b-it-4bit")
gemma_path = pathlib.Path(resolved).resolve()
if not gemma_path.is_dir():
_emit("error",
message=f"Gemma model dir not found: {gemma_path} — "
f"run /train/preflight via the panel, or set "
f"LTX_MODELS_DIR to a root that contains "
f"gemma-3-12b-it-4bit/.")
return 1
# mlx_vlm + Gemma 3. Imports here (not module top) so the JSON-event
# protocol still works if the package is missing: we can emit a clean
# "error" event before crashing.
t_load = time.time()
_emit("loading")
try:
from mlx_vlm import load, generate
from mlx_vlm.prompt_utils import apply_chat_template
except ImportError as e:
_emit("error", message=f"mlx_vlm not installed: {e}. "
f"Run: pip install --no-deps 'mlx-vlm==0.4.4'")
return 1
# WHICH GEMMA IS THIS. Two folders live side by side in mlx_models and
# only one of them can see an image:
#
# gemma-3-12b-it-4bit model_type "gemma3" <- the VLM
# gemma4-12b-ltx25-q4 model_type "gemma4_unified" <- LTX-2.5's text
# encoder, no
# vision tower
#
# Handing the second one to mlx_vlm produces "Model type gemma4_unified
# not supported" — 19 captioning failures across 4 installs and five
# releases in the fleet, and nothing in that sentence says which folder
# was wrong or which one is right. One read of config.json does.
try:
_cfg = json.loads((gemma_path / "config.json").read_text())
_mtype = str(_cfg.get("model_type") or "")
except Exception: # noqa: BLE001
_mtype = ""
if _mtype and not _mtype.startswith("gemma3"):
_emit("error",
message=f"{gemma_path.name} is not the captioning model — its "
f"config says model_type \"{_mtype}\", which has no "
f"vision tower (gemma4-12b-ltx25-q4 is LTX-2.5's text "
f"encoder). Captioning needs gemma-3-12b-it-4bit. "
f"Run /train/preflight from the panel to fetch it, or "
f"point LTX_GEMMA_PATH at that folder.")
return 1
try:
model, processor = load(str(gemma_path))
except Exception as e:
_emit("error", message=f"failed to load Gemma: "
f"{type(e).__name__}: {e}")
return 1
_emit("loaded", elapsed_sec=round(time.time() - t_load, 2))
system = _SYSTEM_TMPL.format(trigger=args.trigger)
n = len(images)
t_start = time.time()
# Counted, not assumed. A per-image failure `continue`s below, so the
# number of images is NOT the number of captions on disk. Reporting the
# former made the panel print "done - 24 captions" over a dataset that
# had 20, and the four images with no caption then trained on the
# 3-word `<trigger> man` fallback without anyone being told. Same class
# as the zip filter that dropped 6 of 18 images in #61: a silent loss
# under a green summary.
written = 0
for i, img_path in enumerate(images, 1):
t_step = time.time()
messages = [
{"role": "system", "content": system},
{"role": "user", "content": _USER_TMPL},
]
try:
formatted = apply_chat_template(
processor, model.config, messages, num_images=1)
out = generate(
model, processor, formatted, image=[str(img_path)],
max_tokens=args.max_tokens, verbose=False,
temperature=args.temperature,
)
except Exception as e:
# Don't abort the whole run on a single bad image — log and
# keep going. The image will fall back to trigger_simple at
# train time (panel writes "<trigger> man" for missing caps).
_emit("error", message=f"caption failed for {img_path.name}: "
f"{type(e).__name__}: {e}",
i=i, n=n, file=img_path.name)
continue
text = (getattr(out, "text", out) or "")
caption = _normalise_caption(text, args.trigger)
cap_path = captions_dir / f"{img_path.stem}.txt"
try:
cap_path.write_text(caption + "\n", encoding="utf-8")
except OSError as e:
_emit("error", message=f"could not write {cap_path}: {e}",
i=i, n=n, file=img_path.name)
continue
written += 1
_emit("progress",
i=i, n=n,
file=img_path.name,
elapsed_sec=round(time.time() - t_step, 2),
caption=caption)
# Nothing on disk is not a run that finished, whatever the exit code
# said. Emitting `done` here left the panel green while the whole
# dataset fell through to the fallback caption at train time.
if written == 0:
_emit("error",
message=f"no captions were written for any of the {n} images "
f"— every image failed to caption. The dataset is "
f"unchanged; see the errors above for the cause.",
count=0, failed=n, total=n)
return 1
_emit("done",
elapsed_sec=round(time.time() - t_start, 2),
count=written,
failed=n - written,
total=n)
return 0
if __name__ == "__main__":
try:
raise SystemExit(main())
except KeyboardInterrupt:
_emit("error", message="cancelled (SIGINT)")
raise SystemExit(130)
except Exception as exc:
_emit("error", message=f"{type(exc).__name__}: {exc}",
traceback=traceback.format_exc())
raise SystemExit(1)