Free, open-source AI video generator for Mac. Run LTX-2.5 and MiniMax H3 (Hailuo) locally on Apple Silicon: text-to-video and image-to-video with sound, lip-sync, character LoRA training, songs from lyrics (YuE2) and image editing (Qwen-Image-Edit). One-click install via Pinokio. Offline: no cloud, no API key, no NVIDIA GPU.
Python ai-filmmakingai-videoai-video-generatorapple-siliconhailuoimage-to-videolip-synclocal-ailora-trainingltx-2ltx-2-5ltx-videomacosminimax-h3mlxmusic-generationoffline-aipinokiotext-to-videovideo-generation
✨ Prompt enhance via Gemma (warm-helper integration)
mrbizarro49eee4d10c1d
diff --git a/mlx_ltx_panel.py b/mlx_ltx_panel.pyindex 85d16e8..e062d8c 100644--- a/mlx_ltx_panel.py+++ b/mlx_ltx_panel.py@@ -2042,6 +2042,42 @@ class Handler(BaseHTTPRequestHandler):push(f"[hf] cancel requested for {rid}")self._json({"ok": True}); return+ if path == "/prompt/enhance":+ # Gemma-driven prompt enhancement, routed through the warm+ # helper subprocess. First call after panel start eats a+ # ~10-15s Gemma load; cached afterwards (subsequent enhances+ # ~3-5s). Helper's release_pipelines frees Gemma when a real+ # render comes in, so memory doesn't accumulate on top of+ # the dev transformer.+ user_prompt = (form.get("prompt", [""])[0] or "").strip()+ mode = (form.get("mode", ["t2v"])[0] or "t2v").lower()+ if mode not in ("t2v", "i2v"):+ mode = "t2v"+ if not user_prompt:+ self._json({"error": "no prompt provided"}, 400); return+ push(f"[enhance] {mode}: {user_prompt[:80]}…")+ try:+ result = HELPER.run({+ "action": "enhance_prompt",+ "id": f"enh-{int(time.time()*1000)}",+ "params": {"prompt": user_prompt, "mode": mode, "seed": 10},+ })+ except Exception as exc:+ push(f"[enhance] failed: {exc}")+ self._json({"error": str(exc)}, 500); return+ enhanced = result.get("enhanced", "").strip()+ if not enhanced:+ self._json({"error": "Gemma returned empty result"}, 500); return+ push(f"[enhance] → {enhanced[:120]}… ({result.get('elapsed_sec','?')}s)")+ self._json({+ "ok": True,+ "original": user_prompt,+ "enhanced": enhanced,+ "mode": mode,+ "elapsed_sec": result.get("elapsed_sec"),+ })+ return+self.send_error(404)@@ -2814,6 +2850,12 @@ HTML = r"""<!doctype html><h2>Prompt</h2><textarea name="prompt" id="prompt" placeholder="What should happen in the video..."></textarea>+ <!-- Gemma-driven prompt enhancement (upstream's `ltx-2-mlx enhance`).+ Rewrites your prompt with the structure/keywords LTX 2.3 trained+ on. ~12-15s on cold start (Gemma needs to load), ~5s warm. -->+ <div class="row-actions" style="margin-top:6px">+ <button type="button" class="ghost-btn" id="enhanceBtn" onclick="enhancePrompt()" title="Use Gemma to rewrite your prompt in the style LTX 2.3 was trained on">✨ Enhance prompt with Gemma</button>+ </div><!-- Mode-specific: audio (i2v_clean_audio only — accessed via Advanced) --><details>@@ -3079,6 +3121,40 @@ document.querySelectorAll('#qualityGroup .pill-btn').forEach(b => b.onclick = ()document.querySelectorAll('#aspectGroup .pill-btn').forEach(b => b.onclick = () => setAspect(b.dataset.aspect));document.querySelectorAll('#extendModeGroup .pill-btn').forEach(b => b.onclick = () => setExtendMode(b.dataset.extendMode));+// Prompt enhancement via Gemma — wraps the upstream CLI's `enhance`+// subcommand. Cold start ~12-15s (Gemma load), warm ~5s. Blocks the UI+// during the request (just the button — rest of the form stays usable).+async function enhancePrompt() {+ const ta = document.getElementById('prompt');+ const original = ta.value.trim();+ if (!original) { alert('Type a prompt before enhancing it.'); return; }+ const mode = (currentMode === 'i2v' || currentMode === 'keyframe' || currentMode === 'extend') ? 'i2v' : 't2v';+ const btn = document.getElementById('enhanceBtn');+ const originalLabel = btn.textContent;+ btn.disabled = true;+ btn.textContent = '✨ Loading Gemma… (~15s on cold start)';+ let res;+ try {+ const fd = new URLSearchParams({ prompt: original, mode });+ const r = await fetch('/prompt/enhance', { method: 'POST', body: fd });+ res = await r.json();+ } catch (e) {+ alert('Enhance request failed: ' + (e.message || e));+ btn.disabled = false; btn.textContent = originalLabel;+ return;+ }+ btn.disabled = false; btn.textContent = originalLabel;+ if (res.error) { alert('Enhance failed: ' + res.error); return; }+ // Show diff in a confirm so the user can decide whether to accept.+ const accept = confirm(+ `Original:\n${res.original}\n\nEnhanced:\n${res.enhanced}\n\nReplace your prompt with the enhanced version?`+ );+ if (accept) {+ ta.value = res.enhanced;+ ta.dispatchEvent(new Event('input', { bubbles: true }));+ }+}+// Extend duration: user types seconds, we convert to latent frames behind// the scenes. Each latent = 8 video frames; at 24 fps that's 0.333 s.// Round-up so the user gets at least the seconds they asked for.diff --git a/mlx_warm_helper.py b/mlx_warm_helper.pyindex b600a9f..e09caa3 100644--- a/mlx_warm_helper.py+++ b/mlx_warm_helper.py@@ -114,6 +114,7 @@ def release_pipelines(keep_kind=None):Caller must hold _pipe_lock."""global _t2v_pipe, _i2v_pipe, _extend_pipe, _hq_pipe, _kf_pipe, _hq_model_dir, _kf_model_dir+ global _gemma_lmtry:from ltx_core_mlx.utils.memory import aggressive_cleanupexcept Exception:@@ -130,6 +131,11 @@ def release_pipelines(keep_kind=None):_hq_pipe = None; _hq_model_dir = None; freed.append("HQ")if keep_kind != "keyframe" and _kf_pipe is not None:_kf_pipe = None; _kf_model_dir = None; freed.append("Keyframe")+ # Always free Gemma LanguageModel when releasing for any pipeline —+ # ~6 GB persistent that competes with the dev transformer's headroom.+ # Re-loaded on demand by the next enhance call (one-time ~10s cost).+ if keep_kind != "gemma_lm" and _gemma_lm is not None:+ _gemma_lm = None; freed.append("GemmaLM")if freed:aggressive_cleanup()emit({"event": "log", "line": f"Released pipelines: {', '.join(freed)} (freeing RAM before next load)"})@@ -221,6 +227,42 @@ def get_kf_pipe(model_dir: str):return _kf_pipe+# ---- prompt enhancement (Gemma language model) ------------------------------+# Separate from the pipeline's TextEncoder wrapper — same weights file, but+# the LanguageModel class supports `.enhance_t2v(prompt, seed)` /+# `.enhance_i2v(prompt, seed)` for prompt rewriting. Loaded lazily on first+# enhance request. Held warm across calls; freed by `release_pipelines`+# when a render starts to keep memory below the 64 GB ceiling.+_gemma_lm = None+++def get_gemma_lm():+ global _gemma_lm+ if _gemma_lm is None:+ from ltx_core_mlx.text_encoders.gemma.encoders.base_encoder import GemmaLanguageModel+ emit({"event": "log", "line": "Loading Gemma language model for prompt enhancement (~10-15s)…"})+ with _pipe_lock:+ # Free any active pipeline first — Gemma is ~6 GB, the dev+ # transformer is ~12-19 GB, having both resident risks pushing+ # us past 64 GB on standard tier.+ release_pipelines(keep_kind=None)+ _gemma_lm = GemmaLanguageModel()+ _gemma_lm.load(GEMMA_PATH)+ emit({"event": "log", "line": "Gemma loaded — subsequent enhances will be fast."})+ return _gemma_lm+++def free_gemma_lm():+ global _gemma_lm+ if _gemma_lm is not None:+ _gemma_lm = None+ try:+ from ltx_core_mlx.utils.memory import aggressive_cleanup+ aggressive_cleanup()+ except Exception:+ pass++# ---- image preprocessing -----------------------------------------------------def cover_crop_to_size(src_path: str, w: int, h: int) -> str:@@ -492,6 +534,46 @@ for line in sys.__stdin__:_is_busy = Falsecontinue+ if action == "enhance_prompt":+ # Gemma-driven prompt rewriting. Same model file as the pipeline's+ # text encoder, but loaded as a `GemmaLanguageModel` (the wrapper+ # that knows how to do `enhance_t2v` / `enhance_i2v`). First call+ # eats a ~10-15s Gemma load; cached afterwards. release_pipelines+ # frees Gemma when a real render comes in, so memory doesn't pile up.+ job_id = msg.get("id", "?")+ p = msg.get("params", {}) or {}+ user_prompt = (p.get("prompt") or "").strip()+ mode = (p.get("mode") or "t2v").lower()+ if mode not in ("t2v", "i2v"):+ mode = "t2v"+ seed = int(p.get("seed", 10))+ if not user_prompt:+ emit({"event": "error", "id": job_id, "error": "empty prompt"})+ continue+ _is_busy = True+ try:+ t0 = time.time()+ lm = get_gemma_lm()+ if mode == "t2v":+ enhanced = lm.enhance_t2v(user_prompt, seed=seed)+ else:+ enhanced = lm.enhance_i2v(user_prompt, seed=seed)+ elapsed = round(time.time() - t0, 2)+ _last_activity = time.time()+ emit({+ "event": "done", "id": job_id,+ "enhanced": enhanced,+ "original": user_prompt,+ "mode": mode,+ "elapsed_sec": elapsed,+ })+ except Exception as exc:+ _last_activity = time.time()+ emit({"event": "error", "id": job_id, "error": str(exc), "trace": traceback.format_exc()})+ finally:+ _is_busy = False+ continue+emit({"event": "error", "error": f"unknown action: {action}"})emit({"event": "exit", "reason": "stdin_closed"})