Skip to content

Free, open-source AI video generator for Mac. Run LTX-2.5 and MiniMax H3 (Hailuo) locally on Apple Silicon: text-to-video and image-to-video with sound, lip-sync, character LoRA training, songs from lyrics (YuE2) and image editing (Qwen-Image-Edit). One-click install via Pinokio. Offline: no cloud, no API key, no NVIDIA GPU.

fix(helper): post-decode watchdog kills the MLX deallocator hang

mrbizarroadc1cd25f469
diff --git a/mlx_warm_helper.py b/mlx_warm_helper.py
index 7839971..03a455d 100644
--- a/mlx_warm_helper.py
+++ b/mlx_warm_helper.py
@@ -491,6 +491,140 @@ _VIDEO_DECODER_PATCH_INSTALLED = False
_A2V_FRAME_RATE_PATCH_INSTALLED = False
+class _PostDecodeWatchdog:
+ """Bulletproof escape hatch for the post-decode MLX/Metal deallocator hang.
+
+ The Symptom
+ -----------
+ On the ``DistilledPipeline.generate_two_stage`` path (T2V/I2V Balanced
+ post-May-9 upstream refactor) and on ``ExtendPipeline``, the helper
+ writes the output mp4 to disk successfully and emits upstream's
+ "[Decoding video + audio + muxing] done in X.Xs" log line — then
+ Python execution stalls for 5-15 minutes before reaching the next
+ statement. The stall is in the function-return path itself, likely
+ Metal command-buffer completion handlers holding the GIL while the
+ local refs go out of scope. Diagnostic prints placed AFTER the
+ decode call never fire during the stall. The panel sits on "running"
+ the entire time, and from the user's perspective the render
+ "froze." Killing the helper is the only way out.
+
+ The Fix
+ -------
+ Run a daemon thread that watches the output file. Once the file
+ exists with reasonable size (proves decode completed), give the
+ main thread a short grace period to finish naturally. If the
+ main thread DOESN'T emit "done" within that window, the watchdog:
+
+ 1. Emits the "done" event itself (file IS on disk, so this is
+ not lying to the panel — the render succeeded).
+ 2. Calls ``os._exit(0)`` to bypass MLX's deallocator chains
+ entirely. ``os._exit`` doesn't run atexit handlers, doesn't
+ flush threads, and crucially doesn't need the GIL — it tells
+ the kernel to terminate the process immediately.
+
+ The panel respawns the helper on the next job (~30s spawn cost).
+ That's a vastly better trade than 15-minute hangs.
+
+ Usage
+ -----
+ wd = _PostDecodeWatchdog(
+ output_path=p["output_path"],
+ t0=t0,
+ job_id=job_id,
+ seed_used=seed,
+ action_label="extend",
+ )
+ wd.arm(grace_sec=30)
+ try:
+ pipe._decode_and_save_video(...)
+ ...
+ emit({"event": "done", ...})
+ wd.disarm() # main thread succeeded, watchdog stands down
+ except Exception:
+ wd.disarm()
+ raise
+ """
+
+ def __init__(self, *, output_path: str, t0: float, job_id: str,
+ seed_used: int, action_label: str) -> None:
+ self.output_path = output_path
+ self.t0 = t0
+ self.job_id = job_id
+ self.seed_used = seed_used
+ self.action_label = action_label
+ self._disarmed = threading.Event()
+ self._thread: threading.Thread | None = None
+
+ def arm(self, *, grace_sec: int = 30, absolute_cap_sec: int = 900) -> None:
+ """Start the watchdog. Idempotent."""
+ if self._thread and self._thread.is_alive():
+ return
+ self._disarmed.clear()
+ self._thread = threading.Thread(
+ target=self._run, args=(grace_sec, absolute_cap_sec),
+ name=f"phos-watchdog-{self.action_label}", daemon=True,
+ )
+ self._thread.start()
+
+ def disarm(self) -> None:
+ """Tell the watchdog the main thread finished cleanly. Idempotent."""
+ self._disarmed.set()
+
+ def _run(self, grace_sec: int, absolute_cap_sec: int) -> None:
+ start = time.time()
+ # Phase 1: wait for the output file to materialize. Decode could
+ # take ~10-60 seconds depending on res/frames; we don't want to
+ # arm the grace clock until we KNOW decode succeeded.
+ while not self._disarmed.is_set():
+ if time.time() - start > absolute_cap_sec:
+ return # absolute timeout — give up silently
+ try:
+ if (os.path.exists(self.output_path) and
+ os.path.getsize(self.output_path) > 1024):
+ break # file is on disk → enter phase 2
+ except OSError:
+ pass
+ time.sleep(1.0)
+ if self._disarmed.is_set():
+ return
+ # Phase 2: file exists. Give main thread the grace period to
+ # finish its emit("done") naturally. If it does, we exit
+ # quietly. If not, we assume the post-decode hang has happened.
+ if self._disarmed.wait(grace_sec):
+ return # main thread emitted done within grace
+ # Force the exit. The output IS on disk — the render succeeded.
+ elapsed = round(time.time() - self.t0, 2)
+ try:
+ sys.stderr.write(
+ f"[watchdog] {self.action_label}: main thread hung in "
+ f"post-decode MLX deallocator for >{grace_sec}s after "
+ f"file landed. Emitting done + force-exiting helper. "
+ f"Panel will respawn on next job.\n"
+ )
+ sys.stderr.flush()
+ except Exception:
+ pass
+ try:
+ emit({
+ "event": "done",
+ "id": self.job_id,
+ "output": self.output_path,
+ "elapsed_sec": elapsed,
+ "seed_used": self.seed_used,
+ "watchdog_forced_exit": True,
+ })
+ except Exception:
+ pass
+ try:
+ _real_stdout.flush()
+ except Exception:
+ pass
+ # os._exit terminates immediately — no atexit, no threading
+ # cleanup, no GIL needed. This is the whole point: bypass
+ # whatever Metal handler chain has the main thread stuck.
+ os._exit(0)
+
+
def _install_video_decoder_patch() -> None:
"""Translate fps/frame_rate kwargs across upstream layers.
@@ -1732,20 +1866,25 @@ for line in sys.__stdin__:
_free_pipe_for_decode(pipe)
emit({"event": "log", "line": "step:free_generation_modules done"})
emit({"event": "log", "line": "step:decode_and_save start"})
+ # Post-decode watchdog: on the DistilledPipeline two-stage
+ # path (T2V/I2V Balanced after the May-9 upstream refactor),
+ # Python stalls 5-15 minutes between the decoder's final
+ # write and the very next statement here — the function-
+ # return path itself hangs in MLX/Metal deallocator chains.
+ # Arming the watchdog BEFORE the call means that if main
+ # thread freezes, the daemon thread emits "done" and
+ # os._exit(0)s the helper. The file IS on disk by then
+ # (decoder logs "Decoding ... done" before returning), so
+ # the panel gets a truthful success event and the next
+ # job spawns a fresh helper. Mr Bizarro 2026-05-21:
+ # "fix it. What do you mean 'deferred to roadmap'?".
+ _wd = _PostDecodeWatchdog(
+ output_path=kwargs["output_path"], t0=t0, job_id=job_id,
+ seed_used=seed, action_label=f"generate:{mode}",
+ )
+ _wd.arm(grace_sec=30)
# FIX 2026-05-14: upstream renamed fps= → frame_rate= (keyword-only).
out_path = pipe._decode_and_save_video(video_latent, audio_latent, kwargs["output_path"], frame_rate=kwargs["frame_rate"])
- # KNOWN ISSUE 2026-05-21: on the DistilledPipeline.generate_two_stage
- # path (T2V/I2V Balanced post-May-9 upstream refactor), Python
- # execution stalls 5-15 minutes between upstream's
- # "[Decoding video + audio + muxing] done in X.Xs" stderr emit
- # and the very next Python statement here. Tested with
- # diagnostic markers — ZERO Python statements execute in
- # between, so the stall is in the function-return path
- # itself (likely MLX Metal completion handlers holding the
- # GIL via callback chains). File is written and intact
- # before the stall starts. Tracking the proper fix in
- # ROADMAP: route Balanced I2V through TI2VidTwoStagesPipeline
- # on Q8 tier instead of DistilledPipeline.
emit({"event": "log", "line": "step:decode_and_save done"})
elapsed = round(time.time() - t0, 2)
_last_activity = time.time()
@@ -1764,6 +1903,13 @@ for line in sys.__stdin__:
# that BEFORE the done event made the panel sit on "running"
# for the full cleanup window (Mr Bizarro caught this 2026-05-21).
emit(done_event)
+ # Tell the post-decode watchdog the main thread emitted done
+ # cleanly. If the watchdog isn't armed (use_model_upscale
+ # branch), this is a no-op via UnboundLocalError catch below.
+ try:
+ _wd.disarm()
+ except (NameError, UnboundLocalError):
+ pass
# Now drop the latent refs. The panel has already moved on; the
# helper just needs to be clean by the time the next job arrives.
try:
@@ -1772,6 +1918,10 @@ for line in sys.__stdin__:
pass
except Exception as exc:
_last_activity = time.time()
+ try:
+ _wd.disarm()
+ except (NameError, UnboundLocalError):
+ pass
emit({"event": "error", "id": job_id, "error": str(exc), "trace": traceback.format_exc()})
finally:
try:
@@ -1855,6 +2005,18 @@ for line in sys.__stdin__:
pipe._loaded = False
aggressive_cleanup()
pipe._load_decoders()
+ # Post-decode watchdog: Extend hits the SAME MLX/Metal
+ # deallocator hang as I2V Balanced. The output file lands
+ # before the hang; the watchdog forces a clean done +
+ # process exit so the panel doesn't sit on "running" for
+ # 10+ minutes. Helper respawns on next job (~30s spawn
+ # cost). See _PostDecodeWatchdog docstring for the full
+ # rationale.
+ _wd = _PostDecodeWatchdog(
+ output_path=p["output_path"], t0=t0, job_id=job_id,
+ seed_used=seed, action_label="extend",
+ )
+ _wd.arm(grace_sec=30)
# FIX 2026-05-14: upstream made frame_rate= keyword-only required.
pipe._decode_and_save_video(video_lat, audio_lat, p["output_path"], frame_rate=float(p.get("frame_rate", 24.0)))
elapsed = round(time.time() - t0, 2)
@@ -1864,8 +2026,13 @@ for line in sys.__stdin__:
"output": p["output_path"], "elapsed_sec": elapsed,
"seed_used": seed,
})
+ _wd.disarm()
except Exception as exc:
_last_activity = time.time()
+ try:
+ _wd.disarm()
+ except (NameError, UnboundLocalError):
+ pass
emit({"event": "error", "id": job_id, "error": str(exc), "trace": traceback.format_exc()})
finally:
# Always clear the per-call TeaCache config so a future