Skip to content

Free, open-source AI video generator for Mac. Run LTX-2.5 and MiniMax H3 (Hailuo) locally on Apple Silicon: text-to-video and image-to-video with sound, lip-sync, character LoRA training, songs from lyrics (YuE2) and image editing (Qwen-Image-Edit). One-click install via Pinokio. Offline: no cloud, no API key, no NVIDIA GPU.

✨ Prompt enhance via Gemma (warm-helper integration)

mrbizarro49eee4d10c1d
diff --git a/mlx_ltx_panel.py b/mlx_ltx_panel.py
index 85d16e8..e062d8c 100644
--- a/mlx_ltx_panel.py
+++ b/mlx_ltx_panel.py
@@ -2042,6 +2042,42 @@ class Handler(BaseHTTPRequestHandler):
push(f"[hf] cancel requested for {rid}")
self._json({"ok": True}); return
+ if path == "/prompt/enhance":
+ # Gemma-driven prompt enhancement, routed through the warm
+ # helper subprocess. First call after panel start eats a
+ # ~10-15s Gemma load; cached afterwards (subsequent enhances
+ # ~3-5s). Helper's release_pipelines frees Gemma when a real
+ # render comes in, so memory doesn't accumulate on top of
+ # the dev transformer.
+ user_prompt = (form.get("prompt", [""])[0] or "").strip()
+ mode = (form.get("mode", ["t2v"])[0] or "t2v").lower()
+ if mode not in ("t2v", "i2v"):
+ mode = "t2v"
+ if not user_prompt:
+ self._json({"error": "no prompt provided"}, 400); return
+ push(f"[enhance] {mode}: {user_prompt[:80]}…")
+ try:
+ result = HELPER.run({
+ "action": "enhance_prompt",
+ "id": f"enh-{int(time.time()*1000)}",
+ "params": {"prompt": user_prompt, "mode": mode, "seed": 10},
+ })
+ except Exception as exc:
+ push(f"[enhance] failed: {exc}")
+ self._json({"error": str(exc)}, 500); return
+ enhanced = result.get("enhanced", "").strip()
+ if not enhanced:
+ self._json({"error": "Gemma returned empty result"}, 500); return
+ push(f"[enhance] → {enhanced[:120]}… ({result.get('elapsed_sec','?')}s)")
+ self._json({
+ "ok": True,
+ "original": user_prompt,
+ "enhanced": enhanced,
+ "mode": mode,
+ "elapsed_sec": result.get("elapsed_sec"),
+ })
+ return
+
self.send_error(404)
@@ -2814,6 +2850,12 @@ HTML = r"""<!doctype html>
<h2>Prompt</h2>
<textarea name="prompt" id="prompt" placeholder="What should happen in the video..."></textarea>
+ <!-- Gemma-driven prompt enhancement (upstream's `ltx-2-mlx enhance`).
+ Rewrites your prompt with the structure/keywords LTX 2.3 trained
+ on. ~12-15s on cold start (Gemma needs to load), ~5s warm. -->
+ <div class="row-actions" style="margin-top:6px">
+ <button type="button" class="ghost-btn" id="enhanceBtn" onclick="enhancePrompt()" title="Use Gemma to rewrite your prompt in the style LTX 2.3 was trained on">✨ Enhance prompt with Gemma</button>
+ </div>
<!-- Mode-specific: audio (i2v_clean_audio only — accessed via Advanced) -->
<details>
@@ -3079,6 +3121,40 @@ document.querySelectorAll('#qualityGroup .pill-btn').forEach(b => b.onclick = ()
document.querySelectorAll('#aspectGroup .pill-btn').forEach(b => b.onclick = () => setAspect(b.dataset.aspect));
document.querySelectorAll('#extendModeGroup .pill-btn').forEach(b => b.onclick = () => setExtendMode(b.dataset.extendMode));
+// Prompt enhancement via Gemma — wraps the upstream CLI's `enhance`
+// subcommand. Cold start ~12-15s (Gemma load), warm ~5s. Blocks the UI
+// during the request (just the button — rest of the form stays usable).
+async function enhancePrompt() {
+ const ta = document.getElementById('prompt');
+ const original = ta.value.trim();
+ if (!original) { alert('Type a prompt before enhancing it.'); return; }
+ const mode = (currentMode === 'i2v' || currentMode === 'keyframe' || currentMode === 'extend') ? 'i2v' : 't2v';
+ const btn = document.getElementById('enhanceBtn');
+ const originalLabel = btn.textContent;
+ btn.disabled = true;
+ btn.textContent = '✨ Loading Gemma… (~15s on cold start)';
+ let res;
+ try {
+ const fd = new URLSearchParams({ prompt: original, mode });
+ const r = await fetch('/prompt/enhance', { method: 'POST', body: fd });
+ res = await r.json();
+ } catch (e) {
+ alert('Enhance request failed: ' + (e.message || e));
+ btn.disabled = false; btn.textContent = originalLabel;
+ return;
+ }
+ btn.disabled = false; btn.textContent = originalLabel;
+ if (res.error) { alert('Enhance failed: ' + res.error); return; }
+ // Show diff in a confirm so the user can decide whether to accept.
+ const accept = confirm(
+ `Original:\n${res.original}\n\nEnhanced:\n${res.enhanced}\n\nReplace your prompt with the enhanced version?`
+ );
+ if (accept) {
+ ta.value = res.enhanced;
+ ta.dispatchEvent(new Event('input', { bubbles: true }));
+ }
+}
+
// Extend duration: user types seconds, we convert to latent frames behind
// the scenes. Each latent = 8 video frames; at 24 fps that's 0.333 s.
// Round-up so the user gets at least the seconds they asked for.
diff --git a/mlx_warm_helper.py b/mlx_warm_helper.py
index b600a9f..e09caa3 100644
--- a/mlx_warm_helper.py
+++ b/mlx_warm_helper.py
@@ -114,6 +114,7 @@ def release_pipelines(keep_kind=None):
Caller must hold _pipe_lock.
"""
global _t2v_pipe, _i2v_pipe, _extend_pipe, _hq_pipe, _kf_pipe, _hq_model_dir, _kf_model_dir
+ global _gemma_lm
try:
from ltx_core_mlx.utils.memory import aggressive_cleanup
except Exception:
@@ -130,6 +131,11 @@ def release_pipelines(keep_kind=None):
_hq_pipe = None; _hq_model_dir = None; freed.append("HQ")
if keep_kind != "keyframe" and _kf_pipe is not None:
_kf_pipe = None; _kf_model_dir = None; freed.append("Keyframe")
+ # Always free Gemma LanguageModel when releasing for any pipeline —
+ # ~6 GB persistent that competes with the dev transformer's headroom.
+ # Re-loaded on demand by the next enhance call (one-time ~10s cost).
+ if keep_kind != "gemma_lm" and _gemma_lm is not None:
+ _gemma_lm = None; freed.append("GemmaLM")
if freed:
aggressive_cleanup()
emit({"event": "log", "line": f"Released pipelines: {', '.join(freed)} (freeing RAM before next load)"})
@@ -221,6 +227,42 @@ def get_kf_pipe(model_dir: str):
return _kf_pipe
+# ---- prompt enhancement (Gemma language model) ------------------------------
+# Separate from the pipeline's TextEncoder wrapper — same weights file, but
+# the LanguageModel class supports `.enhance_t2v(prompt, seed)` /
+# `.enhance_i2v(prompt, seed)` for prompt rewriting. Loaded lazily on first
+# enhance request. Held warm across calls; freed by `release_pipelines`
+# when a render starts to keep memory below the 64 GB ceiling.
+_gemma_lm = None
+
+
+def get_gemma_lm():
+ global _gemma_lm
+ if _gemma_lm is None:
+ from ltx_core_mlx.text_encoders.gemma.encoders.base_encoder import GemmaLanguageModel
+ emit({"event": "log", "line": "Loading Gemma language model for prompt enhancement (~10-15s)…"})
+ with _pipe_lock:
+ # Free any active pipeline first — Gemma is ~6 GB, the dev
+ # transformer is ~12-19 GB, having both resident risks pushing
+ # us past 64 GB on standard tier.
+ release_pipelines(keep_kind=None)
+ _gemma_lm = GemmaLanguageModel()
+ _gemma_lm.load(GEMMA_PATH)
+ emit({"event": "log", "line": "Gemma loaded — subsequent enhances will be fast."})
+ return _gemma_lm
+
+
+def free_gemma_lm():
+ global _gemma_lm
+ if _gemma_lm is not None:
+ _gemma_lm = None
+ try:
+ from ltx_core_mlx.utils.memory import aggressive_cleanup
+ aggressive_cleanup()
+ except Exception:
+ pass
+
+
# ---- image preprocessing -----------------------------------------------------
def cover_crop_to_size(src_path: str, w: int, h: int) -> str:
@@ -492,6 +534,46 @@ for line in sys.__stdin__:
_is_busy = False
continue
+ if action == "enhance_prompt":
+ # Gemma-driven prompt rewriting. Same model file as the pipeline's
+ # text encoder, but loaded as a `GemmaLanguageModel` (the wrapper
+ # that knows how to do `enhance_t2v` / `enhance_i2v`). First call
+ # eats a ~10-15s Gemma load; cached afterwards. release_pipelines
+ # frees Gemma when a real render comes in, so memory doesn't pile up.
+ job_id = msg.get("id", "?")
+ p = msg.get("params", {}) or {}
+ user_prompt = (p.get("prompt") or "").strip()
+ mode = (p.get("mode") or "t2v").lower()
+ if mode not in ("t2v", "i2v"):
+ mode = "t2v"
+ seed = int(p.get("seed", 10))
+ if not user_prompt:
+ emit({"event": "error", "id": job_id, "error": "empty prompt"})
+ continue
+ _is_busy = True
+ try:
+ t0 = time.time()
+ lm = get_gemma_lm()
+ if mode == "t2v":
+ enhanced = lm.enhance_t2v(user_prompt, seed=seed)
+ else:
+ enhanced = lm.enhance_i2v(user_prompt, seed=seed)
+ elapsed = round(time.time() - t0, 2)
+ _last_activity = time.time()
+ emit({
+ "event": "done", "id": job_id,
+ "enhanced": enhanced,
+ "original": user_prompt,
+ "mode": mode,
+ "elapsed_sec": elapsed,
+ })
+ except Exception as exc:
+ _last_activity = time.time()
+ emit({"event": "error", "id": job_id, "error": str(exc), "trace": traceback.format_exc()})
+ finally:
+ _is_busy = False
+ continue
+
emit({"event": "error", "error": f"unknown action: {action}"})
emit({"event": "exit", "reason": "stdin_closed"})