Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Report the video progress the GPU has actually done (#11004)

Daniel Han6ba103be1fa3
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.py
index 03ba20e6a1f..3c7d85d79d8 100644
--- a/studio/backend/core/inference/diffusion_cuda_graph.py
+++ b/studio/backend/core/inference/diffusion_cuda_graph.py
@@ -10,9 +10,11 @@ Not a hook: hooks move ``nn.Module._call_impl`` off its fast path and offload ow
from __future__ import annotations
+import contextlib
import gc
import inspect
import os
+import threading
import traceback
import weakref
from functools import update_wrapper
@@ -40,6 +42,70 @@ def cuda_graph_disabled() -> bool:
return os.environ.get(CUDA_GRAPH_DISABLE_ENV, "").strip().lower() in _TRUE_TOKENS
+# ``torch.cuda.graph`` begins its capture in CUDA's default global mode, which prohibits the
+# "potentially unsafe" calls -- cudaEventQuery among them -- from EVERY thread for as long as it
+# records. The denoise progress poller queries a CUDA event ten times a second and the denoiser
+# captures its graphs during the first steps of a render, so the two really do overlap.
+#
+# A flag read before the query would be a TOCTOU: the reader can see "no capture", the capture can
+# begin, and the query then lands inside it and invalidates it. So the lock has to span the CUDA
+# call itself, not just the depth. Capture raises the depth UNDER the lock before it enters
+# ``torch.cuda.graph``, and a querier holds the same lock ACROSS its query, which leaves only two
+# orderings and both are safe: either the querier holds the lock and capture entry waits the few
+# microseconds a query takes, or capture got there first and the querier sees a non-zero depth and
+# skips. The depth is not a flag because nothing says two pipelines cannot be capturing at once,
+# and neither holds the lock while it records, so captures do not serialise against each other.
+_CAPTURE_LOCK = threading.Lock()
+_CAPTURE_DEPTH = 0
+
+
+def _capturing():
+ """Mark a capture as recording for the duration of the block.
+
+ The depth MUST be raised before ``torch.cuda.graph`` is entered, which is what makes the
+ ordering above hold; keep this the outermost of the two context managers.
+ """
+ global _CAPTURE_DEPTH
+ with _CAPTURE_LOCK:
+ _CAPTURE_DEPTH += 1
+ try:
+ yield
+ finally:
+ with _CAPTURE_LOCK:
+ _CAPTURE_DEPTH -= 1
+
+
+def hold_off_capture():
+ """Yield True while it is safe to make a CUDA call a recording capture would prohibit.
+
+ While a True is held, no capture can ENTER ``torch.cuda.graph``: capture entry takes the same
+ lock. Yields False, having taken nothing, when a capture is already recording or is entering
+ right now -- the caller must then skip its CUDA call entirely.
+
+ The acquire is non-blocking on purpose. This is polled at 10 Hz for progress reporting, and a
+ render must never wait on a progress tick; a skipped poll costs a tenth of a second of bar.
+ """
+ if not _CAPTURE_LOCK.acquire(blocking = False):
+ yield False
+ return
+ try:
+ yield _CAPTURE_DEPTH == 0
+ finally:
+ _CAPTURE_LOCK.release()
+
+
+def capture_in_progress() -> bool:
+ """Whether a capture is recording, as a snapshot for reporting.
+
+ NOT safe to gate a CUDA call on: by the time the caller acts the answer can have changed.
+ Use ``hold_off_capture`` for that.
+ """
+ with _CAPTURE_LOCK:
+ return _CAPTURE_DEPTH > 0
+
+
# Not ``torch.utils._pytree``: it makes an unregistered object a LEAF, which must stay visible.
@@ -391,7 +457,7 @@ class GraphedForward:
graph = torch.cuda.CUDAGraph()
# ``pool = None`` is identical to omitting the argument, so both captures take one path.
- with torch.cuda.graph(graph, pool = _POOL_BOX[0]):
+ with _capturing(), torch.cuda.graph(graph, pool = _POOL_BOX[0]):
out = self.orig(*static_args, **static_kwargs)
if _POOL_BOX[0] is None:
try:
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index c6b174092b4..f560adce767 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -445,14 +445,29 @@ class _VideoGenerationCancelled(Exception):
@contextlib.contextmanager
-def _scheduler_step_progress(pipe: Any, on_step: Any):
+def _scheduler_step_progress(
+ pipe: Any,
+ on_step: Any,
+ on_step_done: Any = None,
+):
"""Progress + cancellation for pipelines WITHOUT callback_on_step_end.
HunyuanVideo15Pipeline exposes no per-step callback, but every denoise step
makes exactly one ``scheduler.step`` call, so wrapping that method gives the
- same per-step tick the callback path gets. ``on_step`` receives the 1-based
- step count and may raise (_VideoGenerationCancelled) to abort the loop. The
- original method is always restored, even when the pipeline raises.
+ same per-step tick the callback path gets. The original method is always
+ restored, even when the pipeline raises.
+
+ Two hooks, because they want opposite sides of the call:
+
+ ``on_step`` runs BEFORE the scheduler step and receives the 1-based step count. It is the
+ cancellation check, and may raise (_VideoGenerationCancelled) to abort the loop -- which has to
+ happen before the step's work is submitted, not after.
+
+ ``on_step_done`` runs AFTER the original step returns, with the same count. It is the progress
+ tick, and it belongs here because ``scheduler.step`` is what enqueues the step's latent update
+ (and on MiniMax-H3 the audio scheduler update after it). Marking the step from the pre-hook
+ would place the marker ahead of kernels that step still owes, so the marker could signal while
+ that step was still running -- the same lie this wrapper exists to remove, one step wide.
"""
scheduler = pipe.scheduler
original = scheduler.step
@@ -461,7 +476,10 @@ def _scheduler_step_progress(pipe: Any, on_step: Any):
def _step(*args: Any, **kwargs: Any) -> Any:
count["n"] += 1
on_step(count["n"])
- return original(*args, **kwargs)
+ result = original(*args, **kwargs)
+ if on_step_done is not None:
+ on_step_done(count["n"])
+ return result
scheduler.step = _step
try:
@@ -470,6 +488,263 @@ def _scheduler_step_progress(pipe: Any, on_step: Any):
scheduler.step = original
+def _record_cuda_event() -> Any:
+ """A ``torch.cuda.Event`` recorded on the current stream, or None when that is not possible.
+
+ Never raises and never synchronises: an event recorded here is only ever polled with
+ ``query()``, so it costs one enqueued marker and nothing on the critical path. Returns None on
+ a CPU/MPS host, and refuses to record inside a CUDA-graph capture -- an event recorded there
+ becomes part of the graph, and querying such an event is prohibited in every capture mode.
+ """
+ try:
+ import torch
+
+ if not torch.cuda.is_available():
+ return None
+ if torch.cuda.is_current_stream_capturing():
+ return None
+ event = torch.cuda.Event()
+ event.record()
+ return event
+ except Exception: # noqa: BLE001 -- progress reporting must never fail a render
+ return None
+
+
+class _CompletedStepTicker:
+ """The denoise step the GPU has FINISHED, not the one the CPU has enqueued.
+
+ With the denoiser's forward replayed from a captured CUDA graph the Python loop enqueues every
+ step and returns long before the GPU has run them: on MiniMax-H3 at 960x544x124 over 30 steps
+ the 29 host-side ticks land in 0.9 s and the GPU is still 26 s from done. A counter driven by
+ ``scheduler.step`` or ``callback_on_step_end`` therefore runs the bar to the end of the denoise
+ while no denoising has happened, and the user watches an apparent freeze at the far right of a
+ bar that claims a second remains.
+
+ So mark each step's place in the stream with a ``torch.cuda.Event`` and report the last one
+ that has completed. ``query()`` does not block and does not serialise, which is the whole
+ point: a ``synchronize()`` per step would give the same honest number and hand back the
+ CUDA-graph win that the video speed path exists for.
+
+ Falls back to the enqueued count on any host without usable CUDA events (CPU, MPS), which is
+ exactly today's behaviour and is correct there -- without a graph to run ahead of, the host
+ count IS the progress.
+ """
+
+ def __init__(self, total: int) -> None:
+ self.total = max(0, int(total))
+ self._lock = threading.Lock()
+ self._events: list = []
+ self._event_backed = False
+ self._enqueued = 0
+ self._done = 0
+ self._scanned = 0
+ self._boundary: Any = None
+ self._boundary_marked = False
+
+ @property
+ def event_backed(self) -> bool:
+ """Whether any step is being tracked by a real GPU event."""
+ with self._lock:
+ return self._event_backed
+
+ def mark_boundary(self) -> bool:
+ """Mark the end of the denoise in the stream, from the host position that knows it.
+
+ Returns True when a real event was recorded, meaning the caller must WAIT for
+ ``boundary_reached()`` before calling the denoise finished. False means there is no event
+ to wait on (a host with no CUDA events), and the boundary is reached the moment it is
+ marked, because there is no queue for the host to have run ahead of.
+ """
+ event = _record_cuda_event()
+ with self._lock:
+ self._boundary = event
+ self._boundary_marked = True
+ return event is not None
+
+ @property
+ def boundary_marked(self) -> bool:
+ """Whether the end of the denoise has been marked yet."""
+ with self._lock:
+ return self._boundary_marked
+
+ def boundary_reached(self) -> bool:
+ """Whether the GPU has actually reached the marked end of the denoise.
+
+ True when nothing was marked or the marker cannot be read: this gates a phase label, so an
+ unreadable marker must not strand the bar in the denoise phase forever.
+ """
+ with self._lock:
+ if not self._boundary_marked:
+ return False
+ event = self._boundary
+ if event is None:
+ return True
+ try:
+ return bool(event.query())
+ except Exception: # noqa: BLE001 -- an unqueryable marker counts as reached
+ return True
+
+ def record(self, enqueued: int) -> None:
+ """Note that step ``enqueued`` (1-based) has been submitted to the device."""
+ enqueued = int(enqueued)
+ with self._lock:
+ if enqueued <= self._enqueued:
+ return
+ self._enqueued = enqueued
+ if len(self._events) >= self.total:
+ return
+ event = _record_cuda_event()
+ if event is None:
+ return
+ with self._lock:
+ # The step number travels WITH the event, so a step that could not be marked costs its
+ # own tick and never shifts the ones after it.
+ self._events.append((enqueued, event))
+ self._event_backed = True
+
+ def completed(self) -> int:
+ """Steps the GPU has finished. Never decreases, never exceeds what was enqueued, never
+ blocks, and never raises."""
+ with self._lock:
+ if not self._event_backed:
+ return self._enqueued
+ pending = self._events[self._scanned :]
+ done, scanned = self._done, self._scanned
+ for enqueued, event in pending:
+ try:
+ if not event.query():
+ break
+ except Exception: # noqa: BLE001 -- an unqueryable event just stops the scan
+ break
+ done = max(done, enqueued)
+ scanned += 1
+ with self._lock:
+ self._done = max(self._done, done)
+ self._scanned = max(self._scanned, scanned)
+ return self._done
+
+
+def _hold_off_cuda_graph_capture():
+ """Yield True while it is safe to query a CUDA event, holding off any capture that would start.
+
+ Delegates to the CUDA-graph layer, which owns the lock capture entry takes. A missing graph
+ layer means there is no capture to collide with, so the answer is True.
+ """
+ try:
+ from . import diffusion_cuda_graph
+ guard = diffusion_cuda_graph.hold_off_capture
+ except Exception: # noqa: BLE001 -- no graph layer means no capture to avoid
+ yield True
+ return
+ with guard() as clear:
+ yield clear
+
+
+def _completed_step_poller(pump: Any, poll_seconds: float = 0.1):
+ """Keep advancing the reported progress from the GPU while the caller sits inside ``pipe()``.
+
+ The host-side ticks stop the moment the loop has enqueued its last step, so after that nothing
+ would ever ask the events again -- which is precisely the stretch the bar used to lie through.
+ A daemon thread polling at 10 Hz costs a couple of ``cudaEventQuery`` calls per tick. ``pump``
+ must not raise, but is guarded anyway: no progress update may fail a render.
+
+ The whole pump runs inside the capture hold-off, not just a check before it. Reading a flag and
+ then querying would leave a window for a capture to begin in between, and that query would land
+ inside the capture and invalidate it, poisoning the graph wrapper and dropping the load to the
+ eager path for the rest of its life.
+ """
+ stop = threading.Event()
+
+ def _run() -> None:
+ while not stop.is_set():
+ stop.wait(poll_seconds)
+ try:
+ with _hold_off_cuda_graph_capture() as clear:
+ if not clear:
+ continue
+ pump()
+ except Exception: # noqa: BLE001 -- progress must never fail a render
+ pass
+
+ thread = threading.Thread(target = _run, name = "video-denoise-progress", daemon = True)
+ thread.start()
+ try:
+ yield
+ finally:
+ stop.set()
+ thread.join(timeout = 2.0)
+
+
+# The decoders a video pipeline may run after its denoise loop, in the order the modular MiniMax-H3
+# workflow runs them. Wrapping the bound method is the one hook every family shares.
+_DECODE_ATTRS = ("vae", "audio_vae")
+
+# How hard the end-of-denoise marker tries before it gives up and flips the phase at the host
+# position. The hold-off refuses without blocking, so most refusals are the 10 Hz poller holding
+# the lock for a couple of event queries; 20 tries 20 ms apart covers several poll ticks and costs
+# nothing when uncontended, while a real capture outlasts it and takes the host-position fallback.
+_BOUNDARY_MARK_ATTEMPTS = 20
+_BOUNDARY_MARK_RETRY_SECONDS = 0.02
+
+
+def _decode_phase(pipe: Any, on_decode: Any):
+ """Flip the reported phase to "decode" the instant the decoder is entered.
+
+ HunyuanVideo-1.5, Wan and MiniMax-H3's modular workflow all run the decode INSIDE the pipeline
+ call with nothing between the denoise loop and it, so a phase set only after ``pipe()`` returns
+ reports the whole decode as the last denoise step. On H3 that decode plus its post-processing
+ is ~3.9 s of the render, and the VAE decode is the memory peak.
+
+ Note what this hook is and is not: it is the one HOST position that knows the denoise loop is
+ over, and nothing more. On a family where the host runs ahead of the device, the denoise
+ kernels can still be queued when it fires, so the caller must treat it as "mark the boundary",
+ not as "the denoise finished" -- see ``_CompletedStepTicker.mark_boundary``.
+
+ ``on_decode`` fires at most once per generation and must not raise. Every wrapper installed
+ here is removed again, including when the decode raises -- and restored to whatever was there,
+ since the speed layer may have already put a compiled decode in the instance ``__dict__``.
+ """
+ fired = {"done": False}
+ restore: list = []
+
+ def _wrap(original: Any):
+ def _decode(*args: Any, **kwargs: Any) -> Any:
+ if not fired["done"]:
+ fired["done"] = True
+ on_decode()
+ return original(*args, **kwargs)
+
+ return _decode
+
+ for name in _DECODE_ATTRS:
+ owner = getattr(pipe, name, None)
+ original = getattr(owner, "decode", None) if owner is not None else None
+ if not callable(original):
+ continue
+ had_own = "decode" in getattr(owner, "__dict__", {})
+ try:
+ owner.decode = _wrap(original)
+ except Exception: # noqa: BLE001 -- a decoder that refuses assignment goes unreported
+ continue
+ restore.append((owner, original, had_own))
+ try:
+ yield
+ finally:
+ for owner, original, had_own in restore:
+ try:
+ if had_own:
+ owner.decode = original
+ else:
+ # Nothing was shadowing the class method, so leave nothing behind -- a bound
+ # method parked in a module's __dict__ is a reference cycle back to the module.
+ del owner.decode
+ except Exception: # noqa: BLE001 -- cleanup is best-effort
+ pass
+
+
def _assert_pick_is_not_speech(
repo_id: str,
gguf_filename: Optional[str],
@@ -5671,36 +5946,124 @@ class VideoBackend:
"error": None,
}
- def _tick(done: int) -> None:
+ ticker = _CompletedStepTicker(steps)
+
+ def _report(done: int) -> None:
+ """Publish a step the GPU has actually finished. Monotonic, and silent once the
+ denoise is over so a late poll cannot walk the bar back under a later phase."""
+ done = max(0, min(int(done), steps))
+ if self._gen.get("phase") != "denoise":
+ return
+ if done <= int(self._gen.get("step") or 0):
+ return
elapsed = time.monotonic() - started
self._gen.update(
step = done,
eta_seconds = (elapsed / max(1, done)) * max(0, steps - done),
)
+ def _tick(enqueued: int) -> None:
+ """One denoise step has now been fully SUBMITTED, latent update included. What
+ gets reported is what the GPU has completed; only a host with no usable CUDA
+ events falls back to this count.
+
+ Inside the capture hold-off for the same reason the poller is: recording and
+ querying an event are both prohibited while ANOTHER thread is capturing a graph,
+ because torch captures in cudaStreamCaptureModeGlobal, under which a concurrent
+ capture in any thread bars every thread from potentially unsafe calls. This
+ pipeline's own capture is caught by ``is_current_stream_capturing``; a second
+ pipeline's is not, and Studio runs image and video renders side by side. A tick
+ skipped here costs that step its marker and nothing else: the step number travels
+ with the event, so the later ones do not shift, and the poller keeps reporting."""
+ with _hold_off_cuda_graph_capture() as clear:
+ if not clear:
+ return
+ ticker.record(enqueued)
+ _report(ticker.completed())
+
+ def _finish_denoise() -> None:
+ """The denoise is provably over, so complete the bar rather than leaving it
+ wherever the last event landed. Also why the bar no longer stops one short:
+ MiniMax-H3's loop makes 29 ``scheduler.step`` calls for a 30-step render, and
+ nothing else ever reported the last step."""
+ self._gen.update(step = steps, eta_seconds = None)
+
+ def _enter_decode_phase() -> None:
+ _finish_denoise()
+ self._gen.update(phase = "decode", eta_seconds = None)
+
+ def _on_decode() -> None:
+ """The decoder was entered, which is a HOST position: on a family whose host
+ runs ahead the denoise kernels may still be queued, and flipping here would
+ jump the bar to steps/steps while the GPU was still denoising. So mark the
+ boundary in the stream and let the poller flip when the GPU reaches it. With no
+ event to wait on there is no queue to have run ahead of, so flip at once.
+
+ Marking records an event, so it takes the capture hold-off too, and it RETRIES:
+ the hold-off acquires its lock without blocking, so a refusal can mean nothing
+ worse than the 10 Hz poller holding it for its own queries at that instant, and
+ giving up there would flip the bar to steps/steps with the denoise queue still
+ draining, which is the exact lie this boundary exists to remove. The retries are
+ short and only run when contended. A capture that outlasts them really does leave
+ nothing to wait on, and the host position is then the only answer there is."""
+ marked = False
+ for _ in range(_BOUNDARY_MARK_ATTEMPTS):
+ with _hold_off_cuda_graph_capture() as clear:
+ if clear:
+ marked = ticker.mark_boundary()
+ break
+ time.sleep(_BOUNDARY_MARK_RETRY_SECONDS)
+ if not marked:
+ _enter_decode_phase()
+
+ def _pump() -> None:
+ """One poll, inside the capture hold-off. Advances the step from the GPU, and
+ takes the denoise to complete only once the GPU has reached the marked end."""
+ if self._gen.get("phase") != "denoise":
+ return
+ if ticker.boundary_marked and ticker.boundary_reached():
+ _enter_decode_phase()
+ return
+ _report(ticker.completed())
+
def _on_step(p, step_index, timestep, callback_kwargs):
+ # diffusers calls this at the END of a loop iteration, after scheduler.step, so
+ # the step's latent update is already submitted when the marker goes down.
if cancel.is_set():
p._interrupt = True
return callback_kwargs
_tick(step_index + 1)
return callback_kwargs
- def _on_scheduler_step(done: int) -> None:
- # No cooperative _interrupt here, so cancellation must unwind the denoise loop via an exception
+ def _on_scheduler_step_cancel(done: int) -> None:
+ # Runs BEFORE the scheduler step. No cooperative _interrupt here, so
+ # cancellation must unwind the denoise loop via an exception, and it has to do
+ # that before the step's work is submitted rather than after.
if cancel.is_set():
raise _VideoGenerationCancelled()
- _tick(done)
- if "callback_on_step_end" in call_params:
+ has_step_callback = "callback_on_step_end" in call_params
+ if has_step_callback:
kwargs["callback_on_step_end"] = _on_step
- progress_ctx = contextlib.nullcontext()
- else:
- # HunyuanVideo-1.5 has no step callback, so wrap scheduler.step for progress + cancel and restore
- # afterwards. Same for H3's modular workflow: ModularPipeline takes no callback (an unknown input is
- # only warned about), but MiniMaxH3LoopSchedulerStep calls components.scheduler.step --
- # pipe.scheduler -- once per denoise step, so the wrapper ticks and can unwind a multi-minute run on
- # Cancel.
- progress_ctx = _scheduler_step_progress(pipe, _on_scheduler_step)
+
+ @contextlib.contextmanager
+ def progress_ctx():
+ """Everything that reports on a denoise, entered as one unit around ``pipe()``."""
+ with contextlib.ExitStack() as stack:
+ if not has_step_callback:
+ # HunyuanVideo-1.5 has no step callback, so wrap scheduler.step for progress + cancel and
+ # restore afterwards. Same for H3's modular workflow: ModularPipeline takes no callback (an
+ # unknown input is only warned about), but MiniMaxH3LoopSchedulerStep calls
+ # components.scheduler.step -- pipe.scheduler -- once per denoise step, so the wrapper ticks
+ # and can unwind a multi-minute run on Cancel. Cancel before the step, tick after it.
+ stack.enter_context(
+ _scheduler_step_progress(pipe, _on_scheduler_step_cancel, _tick)
+ )
+ # Family-agnostic: no video family has a callback between its denoise loop and its decode, so
+ # every one of them gets its decode phase from the decoder itself.
+ stack.enter_context(_decode_phase(pipe, _on_decode))
+ stack.enter_context(_completed_step_poller(_pump))
+ yield
# Re-check an AUTO cache decision against the ACTUAL step count; explicit choices never toggle
if state.cache_auto:
@@ -5727,7 +6090,7 @@ class VideoBackend:
if state.transformer_cache:
self._reset_step_cache(pipe)
try:
- with torch.inference_mode(), progress_ctx, sigma_ctx:
+ with torch.inference_mode(), progress_ctx(), sigma_ctx:
output = pipe(**kwargs)
except _VideoGenerationCancelled:
# Unwinding by exception skips maybe_free_model_hooks(); under offload the onloaded modules would
@@ -5742,6 +6105,9 @@ class VideoBackend:
if cancel.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
+ # The pipeline returned, so every step ran whatever the last event said. This is
+ # also the only place a latent-only render (no decoder entered) can complete.
+ _finish_denoise()
self._gen.update(phase = "export", eta_seconds = None)
if fam.modular_workflow:
video_frames = output["videos"][0]
diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py
index ea0b80f4eef..42fd3c9ec11 100644
--- a/studio/backend/tests/test_video_backend.py
+++ b/studio/backend/tests/test_video_backend.py
@@ -291,6 +291,19 @@ class _FakeWanVae:
def __init__(self) -> None:
self.tiled = False
self.decoder = _FakeWanDecoder()
+ self.decodes = 0
+ # Test hooks fired from INSIDE the real decode, where the reported phase must already say
+ # "decode": that is the whole stretch the progress used to report as the last denoise step.
+ self.on_decode = None
+ self.decode_raises = None
+
+ def decode(self, *args, **kwargs):
+ self.decodes += 1
+ if self.on_decode is not None:
+ self.on_decode()
+ if self.decode_raises is not None:
+ raise self.decode_raises
+ return object()
def enable_tiling(self) -> None:
self.tiled = True
@@ -331,6 +344,9 @@ class _FakeWanPipeBase:
callback_on_step_end(self, step, 0, {})
if self._interrupt:
break
+ if not self._interrupt:
+ # Like every other family, the decode runs inside __call__ with no callback around it.
+ self.vae.decode(object())
frames = [[object() for _ in range(int(num_frames or 1))]]
return types.SimpleNamespace(frames = frames, audio = None)
@@ -483,6 +499,9 @@ class _FakeHV15Pipe:
pass
for _ in range(int(num_inference_steps or 1)):
self.scheduler.step()
+ # The real pipeline decodes inside __call__ with no callback around it, which is why the
+ # decode phase has to come from the decoder itself.
+ self.vae.decode(object())
frames = [[object() for _ in range(int(num_frames or 1))]]
return types.SimpleNamespace(frames = frames, audio = None)
@@ -4323,8 +4342,10 @@ def test_h3_modular_generation_ticks_and_cancels_through_the_scheduler(fake_runt
pipe.scheduler.on_step = lambda n: steps_seen.append(backend._gen.get("step"))
result = backend.generate(prompt = "a fox", steps = 4)
assert "callback_on_step_end" not in pipe.last_kwargs
- # One wrapped tick per denoise step, then the original method back in place.
- assert steps_seen == [1, 2, 3, 4]
+ # One wrapped tick per denoise step, then the original method back in place. The reading taken
+ # from INSIDE step n is n-1, not n: scheduler.step is what enqueues the step's latent update,
+ # so the tick for step n lands after the original returns rather than before it.
+ assert steps_seen == [0, 1, 2, 3]
assert pipe.scheduler.step.__func__ is _FakeH3Scheduler.step
assert result["num_frames"] == 124 and result["has_audio"] is False
@@ -8910,3 +8931,623 @@ def test_h3_generate_non_oom_error_leaves_the_graphs_alone(fake_runtime):
with pytest.raises(RuntimeError, match = "shape mismatch"):
backend.generate(prompt = "a fox", steps = 4)
assert handle.resets == 0
+
+
+# ── Progress truth: the bar must track the GPU, not the host queue ─────────────
+# Measured on MiniMax-H3 (960x544x124, 30 steps, int8 DiT + int8 TE, speed=default with a captured
+# CUDA graph, B200): the 29 host-side scheduler.step calls land in 0.9 s and the GPU is still
+# 25.9 s from finishing the denoise, then spends another 2.6 s in the video VAE decode and 1.3 s in
+# post-processing. The bar used to reach "step 29/30, ~1s left" in a second and then sit there for
+# half a minute.
+
+
+class _FakeCudaEvent:
+ """A ``torch.cuda.Event`` stand-in whose completion the test controls."""
+
+ def __init__(self) -> None:
+ self.done = False
+ self.queries = 0
+
+ def query(self) -> bool:
+ self.queries += 1
+ return self.done
+
+
+class _ClassDecoder:
+ """A decoder whose ``decode`` lives on the CLASS, as an untouched diffusers VAE's does."""
+
+ def __init__(self) -> None:
+ self.calls = 0
+
+ def decode(self, *args, **kwargs):
+ self.calls += 1
+ return "decoded"
+
+
+def _ticker(monkeypatch, total):
+ """A ticker whose events the test owns, plus the list they land in."""
+ import core.inference.video as video_mod
+
+ events: list = []
+
+ def _fake_event():
+ event = _FakeCudaEvent()
+ events.append(event)
+ return event
+
+ monkeypatch.setattr(video_mod, "_record_cuda_event", _fake_event)
+ return video_mod._CompletedStepTicker(total), events
+
+
+def test_ticker_reports_only_steps_the_gpu_finished(monkeypatch):
+ ticker, events = _ticker(monkeypatch, 8)
+ for step in range(1, 9):
+ ticker.record(step)
+ # The host has enqueued `step` steps and the GPU has finished none of them.
+ assert ticker.completed() == 0
+ assert ticker.event_backed is True
+ events[0].done = True
+ events[1].done = True
+ assert ticker.completed() == 2
+ # Still nothing beyond the contiguous run of finished events, even if a later one lands first.
+ events[5].done = True
+ assert ticker.completed() == 2
+ for event in events:
+ event.done = True
+ assert ticker.completed() == 8
+
+
+def test_ticker_never_walks_backwards_or_past_the_total(monkeypatch):
+ ticker, events = _ticker(monkeypatch, 4)
+ for step in range(1, 5):
+ ticker.record(step)
+ for event in events:
+ event.done = True
+ assert ticker.completed() == 4
+ # A duplicated tick, and an event that starts failing its query, must not lower the bar.
+ ticker.record(2)
+ for event in events:
+ event.done = False
+ assert ticker.completed() == 4
+ # Records past the total are counted as enqueued but mark no further events.
+ ticker.record(9)
+ assert len(events) == 4
+ assert ticker.completed() == 4
+
+
+def test_ticker_falls_back_to_the_host_count_without_cuda_events(monkeypatch):
+ # A CPU or MPS host has no events to poll, and there is no graph to run ahead of either, so
+ # the host count IS the progress. This is exactly the pre-existing behaviour.
+ import core.inference.video as video_mod
+
+ monkeypatch.setattr(video_mod, "_record_cuda_event", lambda: None)
+ ticker = video_mod._CompletedStepTicker(5)
+ for step in range(1, 6):
+ ticker.record(step)
+ assert ticker.completed() == step
+ assert ticker.event_backed is False
+
+
+def test_ticker_unmarkable_step_does_not_shift_the_ones_after_it(monkeypatch):
+ # A step whose event could not be recorded (a capture was in flight) costs its own tick and
+ # nothing more: the step number travels with the event.
+ import core.inference.video as video_mod
+
+ events: list = []
+ refuse = {"step": 2}
+
+ def _fake_event():
+ if len(events) + 1 == refuse["step"]:
+ refuse["step"] = None
+ return None
+ event = _FakeCudaEvent()
+ events.append(event)
+ return event
+
+ monkeypatch.setattr(video_mod, "_record_cuda_event", _fake_event)
+ ticker = video_mod._CompletedStepTicker(4)
+ for step in range(1, 5):
+ ticker.record(step)
+ assert len(events) == 3 # steps 1, 3, 4
+ events[0].done = True
+ assert ticker.completed() == 1
+ events[1].done = True
+ assert ticker.completed() == 3 # the event carries its own step number, so 3, never 2
+
+
+def test_completed_step_poller_advances_with_no_host_ticks(monkeypatch):
+ # The whole point: the host stops ticking the moment it has enqueued the last step, so
+ # something has to keep asking the GPU.
+ import core.inference.video as video_mod
+
+ ticker, events = _ticker(monkeypatch, 3)
+ for step in range(1, 4):
+ ticker.record(step)
+ seen: list = []
+ with video_mod._completed_step_poller(lambda: seen.append(ticker.completed()), 0.01):
+ for event in events:
+ event.done = True
+ deadline = time.monotonic() + 5.0
+ while time.monotonic() < deadline and 3 not in seen:
+ time.sleep(0.01)
+ assert 3 in seen
+ # And the thread is gone with the context.
+ assert not [t for t in threading.enumerate() if t.name == "video-denoise-progress"]
+
+
+def test_completed_step_poller_stands_back_during_a_graph_capture(monkeypatch):
+ # cudaEventQuery is one of the calls CUDA prohibits from EVERY thread while a capture begun in
+ # the default global mode is recording, and the denoiser captures during the first steps.
+ import core.inference.video as video_mod
+
+ ticker, events = _ticker(monkeypatch, 2)
+ for step in (1, 2):
+ ticker.record(step)
+ for event in events:
+ event.done = True
+ from core.inference import diffusion_cuda_graph as cg
+
+ seen: list = []
+ with cg._capturing(): # a capture really is recording for the whole poll window
+ with video_mod._completed_step_poller(lambda: seen.append(ticker.completed()), 0.01):
+ time.sleep(0.2)
+ assert seen == []
+ assert all(event.queries == 0 for event in events)
+
+
+def test_cuda_graph_capture_flag_tracks_the_recording_window():
+ from core.inference import diffusion_cuda_graph as cg
+
+ assert cg.capture_in_progress() is False
+ with cg._capturing():
+ assert cg.capture_in_progress() is True
+ with cg._capturing(): # nothing says two pipelines cannot capture at once
+ assert cg.capture_in_progress() is True
+ assert cg.capture_in_progress() is True
+ assert cg.capture_in_progress() is False
+ # And it is released when the capture raises.
+ with pytest.raises(RuntimeError, match = "capture failed"):
+ with cg._capturing():
+ raise RuntimeError("capture failed")
+ assert cg.capture_in_progress() is False
+
+
+def test_decode_phase_fires_once_and_restores_a_class_decode():
+ import core.inference.video as video_mod
+
+ vae = _ClassDecoder()
+ pipe = types.SimpleNamespace(vae = vae)
+ fired: list = []
+ with video_mod._decode_phase(pipe, lambda: fired.append("decode")):
+ assert vae.decode(1) == "decoded"
+ assert vae.decode(2) == "decoded"
+ assert fired == ["decode"] # at most once per generation
+ assert vae.calls == 2
+ # Nothing left shadowing the class method: a bound method parked in a module's __dict__ is a
+ # reference cycle back to the module.
+ assert "decode" not in vae.__dict__
+ assert vae.decode(3) == "decoded"
+
+
+def test_decode_phase_restores_a_compiled_decode_it_found():
+ # The speed layer puts torch.compile(vae.decode) in the instance __dict__ for U-Net families
+ # (_compile_vae_decode), so the wrapper must put THAT back, not fall through to the class.
+ import core.inference.video as video_mod
+
+ vae = _ClassDecoder()
+ compiled = lambda *a, **k: "compiled" # noqa: E731 - a stand-in for the compiled callable
+ vae.decode = compiled
+ pipe = types.SimpleNamespace(vae = vae)
+ with video_mod._decode_phase(pipe, lambda: None):
+ assert vae.decode() == "compiled"
+ assert vae.decode is compiled
+
+
+def test_decode_phase_restores_the_wrapper_when_the_decode_raises():
+ import core.inference.video as video_mod
+
+ vae = _ClassDecoder()
+ pipe = types.SimpleNamespace(vae = vae)
+ fired: list = []
+
+ def _boom(*args, **kwargs):
+ raise RuntimeError("out of memory in decode")
+
+ vae.decode = _boom
+ with pytest.raises(RuntimeError, match = "out of memory in decode"):
+ with video_mod._decode_phase(pipe, lambda: fired.append("decode")):
+ vae.decode()
+ # The phase was reported before the decode blew up, and the wrapper came off anyway.
+ assert fired == ["decode"]
+ assert vae.decode is _boom
+
+
+def test_decode_phase_covers_the_audio_decoder_too():
+ # MiniMax-H3 decodes video then audio; whichever runs first names the phase.
+ import core.inference.video as video_mod
+
+ audio_vae = _ClassDecoder()
+ pipe = types.SimpleNamespace(vae = None, audio_vae = audio_vae)
+ fired: list = []
+ with video_mod._decode_phase(pipe, lambda: fired.append("decode")):
+ audio_vae.decode()
+ assert fired == ["decode"]
+ assert "decode" not in audio_vae.__dict__
+
+
+def _patch_events(monkeypatch, *, done: bool = False):
+ """Hand out fake CUDA events, optionally already complete. Returns the list they land in."""
+ import core.inference.video as video_mod
+
+ events: list = []
+
+ def _fake_event():
+ event = _FakeCudaEvent()
+ event.done = done
+ events.append(event)
+ return event
+
+ monkeypatch.setattr(video_mod, "_record_cuda_event", _fake_event)
+ return events
+
+
+def test_step_marker_goes_down_after_the_scheduler_step_not_before(fake_runtime, monkeypatch):
+ # scheduler.step is what enqueues the step's latent update (and H3's audio scheduler update
+ # after it), so a marker recorded before it sits AHEAD of kernels that step still owes and can
+ # signal while the step is still running. The tick must come after the original returns.
+ events = _patch_events(monkeypatch)
+ backend = VideoBackend()
+ backend.load_pipeline(
+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"
+ )
+ pipe = _FakeHV15Pipeline.instance
+ marked_when_step_ran: list = []
+ # The fake fires this from INSIDE the original scheduler.step, i.e. while step n's latent
+ # update is being submitted.
+ pipe.scheduler.on_step = lambda n: marked_when_step_ran.append((n, len(events)))
+
+ backend.generate(prompt = "a fox", steps = 5, num_frames = 9, fps = 24)
+
+ # While step n is submitting, only the n-1 steps BEFORE it are marked. The old ordering made
+ # this (n, n): step n was already marked before its own work was enqueued.
+ assert marked_when_step_ran == [(1, 0), (2, 1), (3, 2), (4, 3), (5, 4)]
+ assert len(events) == 6 # five steps plus the end-of-denoise boundary marker
+
+
+def test_cancellation_is_still_checked_before_the_step_runs(fake_runtime, monkeypatch):
+ # Moving the tick after the call must not move the cancel check with it: cancelling has to
+ # unwind before the next step's work is submitted, not after it has run.
+ events = _patch_events(monkeypatch)
+ backend = VideoBackend()
+ backend.load_pipeline(
+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"
+ )
+ pipe = _FakeHV15Pipeline.instance
+ pipe.scheduler.on_step = lambda n: backend.cancel_generate() if n == 1 else None
+ with pytest.raises(RuntimeError, match = VIDEO_CANCELLED_MSG):
+ backend.generate(prompt = "a fox", steps = 4)
+ # Step 2 never ran: the pre-hook raised ahead of it.
+ assert pipe.scheduler.calls == 1
+ # And exactly one step was marked, the one that really was submitted. No boundary marker
+ # either, since the decoder was never reached.
+ assert len(events) == 1
+
+
+def test_hv15_bar_never_outruns_the_gpu_and_holds_the_phase(fake_runtime, monkeypatch):
+ # End to end on the scheduler-wrap path (HunyuanVideo-1.5 exposes no step callback, like H3's
+ # modular workflow). The host enqueues every step and enters the decoder while the GPU has
+ # finished nothing, which is exactly the Wan / LTX-2 shape.
+ events = _patch_events(monkeypatch)
+ backend = VideoBackend()
+ backend.load_pipeline(
+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"
+ )
+ pipe = _FakeHV15Pipeline.instance
+ seen: list = []
+
+ def _watch(enqueued):
+ seen.append((enqueued, int(backend._gen.get("step") or 0), sum(e.done for e in events)))
+
+ pipe.scheduler.on_step = _watch
+ at_decode: dict = {}
+ pipe.vae.on_decode = lambda: at_decode.update(backend._gen)
+
+ backend.generate(prompt = "a fox", steps = 8, num_frames = 9, fps = 24)
+
+ assert [enqueued for enqueued, _, _ in seen] == list(range(1, 9))
+ # The invariant: never more steps reported than the GPU has actually completed.
+ for enqueued, reported, finished in seen:
+ assert reported <= finished, (enqueued, reported, finished)
+ # With the GPU finishing nothing, the bar stayed at 0 through the whole host-side loop, where
+ # it used to read 8/8 while no denoising had happened.
+ assert {reported for _, reported, _ in seen} == {0}
+ # And entering the decoder did NOT complete the bar or flip the phase, because the boundary
+ # marker has not completed: the GPU is still denoising.
+ assert at_decode.get("phase") == "denoise"
+ assert at_decode.get("step") == 0
+
+
+def test_decode_phase_lands_once_the_gpu_reaches_the_boundary(fake_runtime, monkeypatch):
+ # Same path, but now the device keeps up with the host, so the boundary marker is complete when
+ # the decoder is entered and the phase flips there.
+ _patch_events(monkeypatch, done = True)
+ backend = VideoBackend()
+ backend.load_pipeline(
+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"
+ )
+ pipe = _FakeHV15Pipeline.instance
+ at_decode: dict = {}
+ pipe.vae.on_decode = lambda: _settle(backend, at_decode)
+
+ backend.generate(prompt = "a fox", steps = 8, num_frames = 9, fps = 24)
+
+ assert at_decode.get("phase") == "decode"
+ assert at_decode.get("step") == 8
+ assert at_decode.get("total") == 8
+ assert at_decode.get("eta_seconds") is None
+
+
+def _settle(
+ backend,
+ out,
+ timeout = 5.0,
+):
+ """Wait for the poller to publish the decode phase, then snapshot it.
+
+ The flip is the poller's job precisely because the decoder-entry position cannot know whether
+ the GPU has caught up, so a test that reads _gen the instant the decoder is entered is racing
+ the poll interval rather than testing anything.
+ """
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ if backend._gen.get("phase") == "decode":
+ break
+ time.sleep(0.01)
+ out.update(backend._gen)
+
+
+def test_a_capture_cannot_start_while_an_event_query_is_in_flight(monkeypatch):
+ # The TOCTOU this replaced: a poll read "no capture", the capture began, and the poll's
+ # prohibited cudaEventQuery landed inside it. The lock has to span the query itself.
+ import core.inference.video as video_mod
+ from core.inference import diffusion_cuda_graph as cg
+
+ in_query = threading.Event()
+ finish_query = threading.Event()
+
+ class _BlockingEvent:
+ def query(self):
+ in_query.set()
+ finish_query.wait(5.0)
+ return True
+
+ monkeypatch.setattr(video_mod, "_record_cuda_event", lambda: _BlockingEvent())
+ ticker = video_mod._CompletedStepTicker(1)
+ ticker.record(1)
+
+ entered_capture = threading.Event()
+
+ def _capture():
+ with cg._capturing():
+ entered_capture.set()
+
+ with video_mod._completed_step_poller(ticker.completed, 0.01):
+ assert in_query.wait(5.0), "the poller never reached the query"
+ capturer = threading.Thread(target = _capture, daemon = True)
+ capturer.start()
+ # A capture must not be able to raise the depth, and so must not be able to enter
+ # torch.cuda.graph, while a query is in flight.
+ assert not entered_capture.wait(0.5)
+ finish_query.set()
+ assert entered_capture.wait(5.0)
+ capturer.join(timeout = 5.0)
+ assert cg.capture_in_progress() is False
+
+
+def test_hold_off_capture_yields_false_rather_than_waiting_on_a_capture():
+ from core.inference import diffusion_cuda_graph as cg
+
+ with cg.hold_off_capture() as clear:
+ assert clear is True
+ with cg._capturing():
+ with cg.hold_off_capture() as clear:
+ assert clear is False
+ with cg.hold_off_capture() as clear:
+ assert clear is True
+
+
+def test_denoise_completes_even_when_the_loop_ticks_fewer_times_than_steps(fake_runtime):
+ # MiniMax-H3's loop makes 29 scheduler.step calls for a 30-step render, so a bar driven purely
+ # by ticks stops at 29/30 forever. Reaching the decode proves the denoise finished.
+ backend = VideoBackend()
+ pipe = _load_h3_modular(backend)
+ seen: dict = {}
+ original_call = pipe.__class__.__call__
+
+ def _one_tick_short(
+ self,
+ *,
+ num_inference_steps = None,
+ **kwargs,
+ ):
+ out = original_call(self, num_inference_steps = int(num_inference_steps) - 1, **kwargs)
+ seen["at_end_of_loop"] = int(backend._gen.get("step") or 0)
+ return out
+
+ pipe.__class__.__call__ = _one_tick_short
+ try:
+ backend.generate(prompt = "a fox", steps = 30, num_frames = 9, fps = 24)
+ finally:
+ pipe.__class__.__call__ = original_call
+ assert pipe.scheduler.calls == 29
+ assert seen["at_end_of_loop"] < 30
+ # generate() completed the bar once the pipeline returned, so the user never sees 29/30 stick.
+ assert backend._gen == {"active": False}
+
+
+def test_hv15_cancel_still_unwinds_and_leaves_nothing_installed(fake_runtime):
+ backend = VideoBackend()
+ backend.load_pipeline(
+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"
+ )
+ pipe = _FakeHV15Pipeline.instance
+ pipe.scheduler.on_step = lambda n: backend.cancel_generate() if n == 1 else None
+ with pytest.raises(RuntimeError, match = VIDEO_CANCELLED_MSG):
+ backend.generate(prompt = "a fox", steps = 4)
+ assert pipe.scheduler.calls == 1
+ assert pipe.scheduler.step.__func__ is _FakeHV15Scheduler.step
+ # The decode never ran, its wrapper is off, and the poller thread is gone.
+ assert pipe.vae.decodes == 0
+ assert "decode" not in pipe.vae.__dict__
+ assert not [t for t in threading.enumerate() if t.name == "video-denoise-progress"]
+
+
+def test_decode_failure_surfaces_and_leaves_nothing_installed(fake_runtime):
+ backend = VideoBackend()
+ backend.load_pipeline(
+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"
+ )
+ pipe = _FakeHV15Pipeline.instance
+ pipe.vae.decode_raises = RuntimeError("CUDA out of memory in decode")
+ with pytest.raises(RuntimeError, match = "CUDA out of memory in decode"):
+ backend.generate(prompt = "a fox", steps = 4)
+ assert "decode" not in pipe.vae.__dict__
+ assert not [t for t in threading.enumerate() if t.name == "video-denoise-progress"]
+
+
+def test_callback_family_keeps_its_own_callback(fake_runtime, monkeypatch):
+ # Wan exposes callback_on_step_end, so nothing wraps its scheduler. Only the tick changed:
+ # the reported step now comes from completed events rather than the callback's index.
+ import core.inference.video as video_mod
+
+ events = _patch_events(monkeypatch, done = True)
+ wrapped: list = []
+ original_wrap = video_mod._scheduler_step_progress
+ monkeypatch.setattr(
+ video_mod,
+ "_scheduler_step_progress",
+ lambda *a, **kw: wrapped.append(a[0]) or original_wrap(*a, **kw),
+ )
+
+ backend = VideoBackend()
+ backend.load_pipeline("Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline")
+ pipe = backend._state.pipe
+ at_decode: dict = {}
+ pipe.vae.on_decode = lambda: _settle(backend, at_decode)
+
+ backend.generate(prompt = "a sloth", steps = 6, num_frames = 9, fps = 24)
+
+ # The scheduler was never wrapped: this family drives its own callback and still does.
+ assert wrapped == []
+ # One marker per callback tick, so the callback really ran, plus the boundary marker.
+ assert len(events) == 7
+ assert pipe.vae.decodes == 1
+ assert at_decode.get("phase") == "decode"
+ assert at_decode.get("step") == 6
+
+
+# ── the step tick and the boundary marker are CUDA calls too ────────────────────────────────
+
+
+def test_a_capture_in_another_pipeline_stops_the_tick_from_touching_events(
+ fake_runtime, monkeypatch
+):
+ # torch captures in cudaStreamCaptureModeGlobal, under which a capture recording in ANY
+ # thread bars every thread from potentially unsafe calls -- recording and querying an event
+ # among them. is_current_stream_capturing only sees this pipeline's own capture, so a second
+ # pipeline capturing while this one ticks is exactly the case the poller is guarded for, and
+ # the tick and the boundary marker have to take the same hold-off.
+ import core.inference.video as video_mod
+
+ events = _patch_events(monkeypatch, done = True)
+
+ @contextlib.contextmanager
+ def _capture_running():
+ yield False
+
+ monkeypatch.setattr(video_mod, "_hold_off_cuda_graph_capture", _capture_running)
+
+ backend = VideoBackend()
+ backend.load_pipeline(
+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"
+ )
+ pipe = _FakeHV15Pipeline.instance
+ # Read inside the decode, since _gen is cleared once generate() returns.
+ at_decode: dict = {}
+ pipe.vae.on_decode = lambda: at_decode.update(backend._gen)
+
+ backend.generate(prompt = "a fox", steps = 5, num_frames = 9, fps = 24)
+
+ # Not one event was recorded: neither the five step markers nor the end-of-denoise boundary.
+ assert events == []
+ # And the bar is still honest at the end. With no marker to wait on there is nothing to wait
+ # FOR, so the decode flip falls back to the host position, which is what this family did
+ # before any of this and is correct when the queue cannot be measured.
+ assert at_decode.get("phase") == "decode"
+ assert at_decode.get("step") == 5
+
+
+def test_the_tick_marks_every_step_again_once_the_capture_is_over(fake_runtime, monkeypatch):
+ # The control for the test above: the guard must not cost a marker when nothing is capturing,
+ # or it would quietly turn the whole feature off.
+ events = _patch_events(monkeypatch, done = True)
+
+ backend = VideoBackend()
+ backend.load_pipeline(
+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"
+ )
+ backend.generate(prompt = "a fox", steps = 5, num_frames = 9, fps = 24)
+
+ assert len(events) == 6 # five steps plus the boundary
+
+
+def test_the_boundary_marker_waits_out_a_busy_capture_lock(fake_runtime, monkeypatch):
+ # hold_off_capture acquires without blocking, so a refusal can mean nothing worse than the
+ # poller holding the lock for its own queries at that instant. Giving up on the first refusal
+ # would flip the bar to steps/steps with the queue still draining, so the marker retries.
+ import core.inference.video as video_mod
+
+ _patch_events(monkeypatch, done = True)
+
+ # Refuses everything for a window that covers the decode entry, then frees up. Shorter than
+ # the retry budget (20 x 20 ms), so a marker that waits gets its event and one that gives up
+ # never calls mark_boundary at all.
+ busy_until = {"t": 0.0}
+
+ @contextlib.contextmanager
+ def _busy_window():
+ yield time.monotonic() >= busy_until["t"]
+
+ marks = {"calls": 0, "ok": None}
+ original_mark = video_mod._CompletedStepTicker.mark_boundary
+
+ def _counted(self):
+ marks["calls"] += 1
+ marks["ok"] = original_mark(self)
+ return marks["ok"]
+
+ monkeypatch.setattr(video_mod._CompletedStepTicker, "mark_boundary", _counted)
+
+ backend = VideoBackend()
+ backend.load_pipeline(
+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"
+ )
+ pipe = _FakeHV15Pipeline.instance
+ at_decode: dict = {}
+ pipe.vae.on_decode = lambda: _settle(backend, at_decode)
+
+ def _contend(n):
+ # Armed on the last step, so the lock is busy exactly across the decode entry.
+ if n == 5:
+ busy_until["t"] = time.monotonic() + 0.2
+ monkeypatch.setattr(video_mod, "_hold_off_cuda_graph_capture", _busy_window)
+
+ pipe.scheduler.on_step = _contend
+
+ backend.generate(prompt = "a fox", steps = 5, num_frames = 9, fps = 24)
+
+ # It waited the lock out and marked a real boundary instead of flipping blind. Before the
+ # retry, a refusal at this instant meant mark_boundary was never called at all.
+ assert marks["calls"] == 1
+ assert marks["ok"] is True
+ assert at_decode.get("phase") == "decode"