Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Report the video progress the GPU has actually done (#11004)
Daniel Han6ba103be1fa3
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.pyindex 03ba20e6a1f..3c7d85d79d8 100644--- a/studio/backend/core/inference/diffusion_cuda_graph.py+++ b/studio/backend/core/inference/diffusion_cuda_graph.py@@ -10,9 +10,11 @@ Not a hook: hooks move ``nn.Module._call_impl`` off its fast path and offload owfrom __future__ import annotations+import contextlibimport gcimport inspectimport os+import threadingimport tracebackimport weakreffrom functools import update_wrapper@@ -40,6 +42,70 @@ def cuda_graph_disabled() -> bool:return os.environ.get(CUDA_GRAPH_DISABLE_ENV, "").strip().lower() in _TRUE_TOKENS+# ``torch.cuda.graph`` begins its capture in CUDA's default global mode, which prohibits the+# "potentially unsafe" calls -- cudaEventQuery among them -- from EVERY thread for as long as it+# records. The denoise progress poller queries a CUDA event ten times a second and the denoiser+# captures its graphs during the first steps of a render, so the two really do overlap.+#+# A flag read before the query would be a TOCTOU: the reader can see "no capture", the capture can+# begin, and the query then lands inside it and invalidates it. So the lock has to span the CUDA+# call itself, not just the depth. Capture raises the depth UNDER the lock before it enters+# ``torch.cuda.graph``, and a querier holds the same lock ACROSS its query, which leaves only two+# orderings and both are safe: either the querier holds the lock and capture entry waits the few+# microseconds a query takes, or capture got there first and the querier sees a non-zero depth and+# skips. The depth is not a flag because nothing says two pipelines cannot be capturing at once,+# and neither holds the lock while it records, so captures do not serialise against each other.+_CAPTURE_LOCK = threading.Lock()+_CAPTURE_DEPTH = 0+++def _capturing():+ """Mark a capture as recording for the duration of the block.++ The depth MUST be raised before ``torch.cuda.graph`` is entered, which is what makes the+ ordering above hold; keep this the outermost of the two context managers.+ """+ global _CAPTURE_DEPTH+ with _CAPTURE_LOCK:+ _CAPTURE_DEPTH += 1+ try:+ yield+ finally:+ with _CAPTURE_LOCK:+ _CAPTURE_DEPTH -= 1+++def hold_off_capture():+ """Yield True while it is safe to make a CUDA call a recording capture would prohibit.++ While a True is held, no capture can ENTER ``torch.cuda.graph``: capture entry takes the same+ lock. Yields False, having taken nothing, when a capture is already recording or is entering+ right now -- the caller must then skip its CUDA call entirely.++ The acquire is non-blocking on purpose. This is polled at 10 Hz for progress reporting, and a+ render must never wait on a progress tick; a skipped poll costs a tenth of a second of bar.+ """+ if not _CAPTURE_LOCK.acquire(blocking = False):+ yield False+ return+ try:+ yield _CAPTURE_DEPTH == 0+ finally:+ _CAPTURE_LOCK.release()+++def capture_in_progress() -> bool:+ """Whether a capture is recording, as a snapshot for reporting.++ NOT safe to gate a CUDA call on: by the time the caller acts the answer can have changed.+ Use ``hold_off_capture`` for that.+ """+ with _CAPTURE_LOCK:+ return _CAPTURE_DEPTH > 0++# Not ``torch.utils._pytree``: it makes an unregistered object a LEAF, which must stay visible.@@ -391,7 +457,7 @@ class GraphedForward:graph = torch.cuda.CUDAGraph()# ``pool = None`` is identical to omitting the argument, so both captures take one path.- with torch.cuda.graph(graph, pool = _POOL_BOX[0]):+ with _capturing(), torch.cuda.graph(graph, pool = _POOL_BOX[0]):out = self.orig(*static_args, **static_kwargs)if _POOL_BOX[0] is None:try:diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex c6b174092b4..f560adce767 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -445,14 +445,29 @@ class _VideoGenerationCancelled(Exception):@contextlib.contextmanager-def _scheduler_step_progress(pipe: Any, on_step: Any):+def _scheduler_step_progress(+ pipe: Any,+ on_step: Any,+ on_step_done: Any = None,+):"""Progress + cancellation for pipelines WITHOUT callback_on_step_end.HunyuanVideo15Pipeline exposes no per-step callback, but every denoise stepmakes exactly one ``scheduler.step`` call, so wrapping that method gives the- same per-step tick the callback path gets. ``on_step`` receives the 1-based- step count and may raise (_VideoGenerationCancelled) to abort the loop. The- original method is always restored, even when the pipeline raises.+ same per-step tick the callback path gets. The original method is always+ restored, even when the pipeline raises.++ Two hooks, because they want opposite sides of the call:++ ``on_step`` runs BEFORE the scheduler step and receives the 1-based step count. It is the+ cancellation check, and may raise (_VideoGenerationCancelled) to abort the loop -- which has to+ happen before the step's work is submitted, not after.++ ``on_step_done`` runs AFTER the original step returns, with the same count. It is the progress+ tick, and it belongs here because ``scheduler.step`` is what enqueues the step's latent update+ (and on MiniMax-H3 the audio scheduler update after it). Marking the step from the pre-hook+ would place the marker ahead of kernels that step still owes, so the marker could signal while+ that step was still running -- the same lie this wrapper exists to remove, one step wide."""scheduler = pipe.scheduleroriginal = scheduler.step@@ -461,7 +476,10 @@ def _scheduler_step_progress(pipe: Any, on_step: Any):def _step(*args: Any, **kwargs: Any) -> Any:count["n"] += 1on_step(count["n"])- return original(*args, **kwargs)+ result = original(*args, **kwargs)+ if on_step_done is not None:+ on_step_done(count["n"])+ return resultscheduler.step = _steptry:@@ -470,6 +488,263 @@ def _scheduler_step_progress(pipe: Any, on_step: Any):scheduler.step = original+def _record_cuda_event() -> Any:+ """A ``torch.cuda.Event`` recorded on the current stream, or None when that is not possible.++ Never raises and never synchronises: an event recorded here is only ever polled with+ ``query()``, so it costs one enqueued marker and nothing on the critical path. Returns None on+ a CPU/MPS host, and refuses to record inside a CUDA-graph capture -- an event recorded there+ becomes part of the graph, and querying such an event is prohibited in every capture mode.+ """+ try:+ import torch++ if not torch.cuda.is_available():+ return None+ if torch.cuda.is_current_stream_capturing():+ return None+ event = torch.cuda.Event()+ event.record()+ return event+ except Exception: # noqa: BLE001 -- progress reporting must never fail a render+ return None+++class _CompletedStepTicker:+ """The denoise step the GPU has FINISHED, not the one the CPU has enqueued.++ With the denoiser's forward replayed from a captured CUDA graph the Python loop enqueues every+ step and returns long before the GPU has run them: on MiniMax-H3 at 960x544x124 over 30 steps+ the 29 host-side ticks land in 0.9 s and the GPU is still 26 s from done. A counter driven by+ ``scheduler.step`` or ``callback_on_step_end`` therefore runs the bar to the end of the denoise+ while no denoising has happened, and the user watches an apparent freeze at the far right of a+ bar that claims a second remains.++ So mark each step's place in the stream with a ``torch.cuda.Event`` and report the last one+ that has completed. ``query()`` does not block and does not serialise, which is the whole+ point: a ``synchronize()`` per step would give the same honest number and hand back the+ CUDA-graph win that the video speed path exists for.++ Falls back to the enqueued count on any host without usable CUDA events (CPU, MPS), which is+ exactly today's behaviour and is correct there -- without a graph to run ahead of, the host+ count IS the progress.+ """++ def __init__(self, total: int) -> None:+ self.total = max(0, int(total))+ self._lock = threading.Lock()+ self._events: list = []+ self._event_backed = False+ self._enqueued = 0+ self._done = 0+ self._scanned = 0+ self._boundary: Any = None+ self._boundary_marked = False++ @property+ def event_backed(self) -> bool:+ """Whether any step is being tracked by a real GPU event."""+ with self._lock:+ return self._event_backed++ def mark_boundary(self) -> bool:+ """Mark the end of the denoise in the stream, from the host position that knows it.++ Returns True when a real event was recorded, meaning the caller must WAIT for+ ``boundary_reached()`` before calling the denoise finished. False means there is no event+ to wait on (a host with no CUDA events), and the boundary is reached the moment it is+ marked, because there is no queue for the host to have run ahead of.+ """+ event = _record_cuda_event()+ with self._lock:+ self._boundary = event+ self._boundary_marked = True+ return event is not None++ @property+ def boundary_marked(self) -> bool:+ """Whether the end of the denoise has been marked yet."""+ with self._lock:+ return self._boundary_marked++ def boundary_reached(self) -> bool:+ """Whether the GPU has actually reached the marked end of the denoise.++ True when nothing was marked or the marker cannot be read: this gates a phase label, so an+ unreadable marker must not strand the bar in the denoise phase forever.+ """+ with self._lock:+ if not self._boundary_marked:+ return False+ event = self._boundary+ if event is None:+ return True+ try:+ return bool(event.query())+ except Exception: # noqa: BLE001 -- an unqueryable marker counts as reached+ return True++ def record(self, enqueued: int) -> None:+ """Note that step ``enqueued`` (1-based) has been submitted to the device."""+ enqueued = int(enqueued)+ with self._lock:+ if enqueued <= self._enqueued:+ return+ self._enqueued = enqueued+ if len(self._events) >= self.total:+ return+ event = _record_cuda_event()+ if event is None:+ return+ with self._lock:+ # The step number travels WITH the event, so a step that could not be marked costs its+ # own tick and never shifts the ones after it.+ self._events.append((enqueued, event))+ self._event_backed = True++ def completed(self) -> int:+ """Steps the GPU has finished. Never decreases, never exceeds what was enqueued, never+ blocks, and never raises."""+ with self._lock:+ if not self._event_backed:+ return self._enqueued+ pending = self._events[self._scanned :]+ done, scanned = self._done, self._scanned+ for enqueued, event in pending:+ try:+ if not event.query():+ break+ except Exception: # noqa: BLE001 -- an unqueryable event just stops the scan+ break+ done = max(done, enqueued)+ scanned += 1+ with self._lock:+ self._done = max(self._done, done)+ self._scanned = max(self._scanned, scanned)+ return self._done+++def _hold_off_cuda_graph_capture():+ """Yield True while it is safe to query a CUDA event, holding off any capture that would start.++ Delegates to the CUDA-graph layer, which owns the lock capture entry takes. A missing graph+ layer means there is no capture to collide with, so the answer is True.+ """+ try:+ from . import diffusion_cuda_graph+ guard = diffusion_cuda_graph.hold_off_capture+ except Exception: # noqa: BLE001 -- no graph layer means no capture to avoid+ yield True+ return+ with guard() as clear:+ yield clear+++def _completed_step_poller(pump: Any, poll_seconds: float = 0.1):+ """Keep advancing the reported progress from the GPU while the caller sits inside ``pipe()``.++ The host-side ticks stop the moment the loop has enqueued its last step, so after that nothing+ would ever ask the events again -- which is precisely the stretch the bar used to lie through.+ A daemon thread polling at 10 Hz costs a couple of ``cudaEventQuery`` calls per tick. ``pump``+ must not raise, but is guarded anyway: no progress update may fail a render.++ The whole pump runs inside the capture hold-off, not just a check before it. Reading a flag and+ then querying would leave a window for a capture to begin in between, and that query would land+ inside the capture and invalidate it, poisoning the graph wrapper and dropping the load to the+ eager path for the rest of its life.+ """+ stop = threading.Event()++ def _run() -> None:+ while not stop.is_set():+ stop.wait(poll_seconds)+ try:+ with _hold_off_cuda_graph_capture() as clear:+ if not clear:+ continue+ pump()+ except Exception: # noqa: BLE001 -- progress must never fail a render+ pass++ thread = threading.Thread(target = _run, name = "video-denoise-progress", daemon = True)+ thread.start()+ try:+ yield+ finally:+ stop.set()+ thread.join(timeout = 2.0)+++# The decoders a video pipeline may run after its denoise loop, in the order the modular MiniMax-H3+# workflow runs them. Wrapping the bound method is the one hook every family shares.+_DECODE_ATTRS = ("vae", "audio_vae")++# How hard the end-of-denoise marker tries before it gives up and flips the phase at the host+# position. The hold-off refuses without blocking, so most refusals are the 10 Hz poller holding+# the lock for a couple of event queries; 20 tries 20 ms apart covers several poll ticks and costs+# nothing when uncontended, while a real capture outlasts it and takes the host-position fallback.+_BOUNDARY_MARK_ATTEMPTS = 20+_BOUNDARY_MARK_RETRY_SECONDS = 0.02+++def _decode_phase(pipe: Any, on_decode: Any):+ """Flip the reported phase to "decode" the instant the decoder is entered.++ HunyuanVideo-1.5, Wan and MiniMax-H3's modular workflow all run the decode INSIDE the pipeline+ call with nothing between the denoise loop and it, so a phase set only after ``pipe()`` returns+ reports the whole decode as the last denoise step. On H3 that decode plus its post-processing+ is ~3.9 s of the render, and the VAE decode is the memory peak.++ Note what this hook is and is not: it is the one HOST position that knows the denoise loop is+ over, and nothing more. On a family where the host runs ahead of the device, the denoise+ kernels can still be queued when it fires, so the caller must treat it as "mark the boundary",+ not as "the denoise finished" -- see ``_CompletedStepTicker.mark_boundary``.++ ``on_decode`` fires at most once per generation and must not raise. Every wrapper installed+ here is removed again, including when the decode raises -- and restored to whatever was there,+ since the speed layer may have already put a compiled decode in the instance ``__dict__``.+ """+ fired = {"done": False}+ restore: list = []++ def _wrap(original: Any):+ def _decode(*args: Any, **kwargs: Any) -> Any:+ if not fired["done"]:+ fired["done"] = True+ on_decode()+ return original(*args, **kwargs)++ return _decode++ for name in _DECODE_ATTRS:+ owner = getattr(pipe, name, None)+ original = getattr(owner, "decode", None) if owner is not None else None+ if not callable(original):+ continue+ had_own = "decode" in getattr(owner, "__dict__", {})+ try:+ owner.decode = _wrap(original)+ except Exception: # noqa: BLE001 -- a decoder that refuses assignment goes unreported+ continue+ restore.append((owner, original, had_own))+ try:+ yield+ finally:+ for owner, original, had_own in restore:+ try:+ if had_own:+ owner.decode = original+ else:+ # Nothing was shadowing the class method, so leave nothing behind -- a bound+ # method parked in a module's __dict__ is a reference cycle back to the module.+ del owner.decode+ except Exception: # noqa: BLE001 -- cleanup is best-effort+ pass++def _assert_pick_is_not_speech(repo_id: str,gguf_filename: Optional[str],@@ -5671,36 +5946,124 @@ class VideoBackend:"error": None,}- def _tick(done: int) -> None:+ ticker = _CompletedStepTicker(steps)++ def _report(done: int) -> None:+ """Publish a step the GPU has actually finished. Monotonic, and silent once the+ denoise is over so a late poll cannot walk the bar back under a later phase."""+ done = max(0, min(int(done), steps))+ if self._gen.get("phase") != "denoise":+ return+ if done <= int(self._gen.get("step") or 0):+ returnelapsed = time.monotonic() - startedself._gen.update(step = done,eta_seconds = (elapsed / max(1, done)) * max(0, steps - done),)+ def _tick(enqueued: int) -> None:+ """One denoise step has now been fully SUBMITTED, latent update included. What+ gets reported is what the GPU has completed; only a host with no usable CUDA+ events falls back to this count.++ Inside the capture hold-off for the same reason the poller is: recording and+ querying an event are both prohibited while ANOTHER thread is capturing a graph,+ because torch captures in cudaStreamCaptureModeGlobal, under which a concurrent+ capture in any thread bars every thread from potentially unsafe calls. This+ pipeline's own capture is caught by ``is_current_stream_capturing``; a second+ pipeline's is not, and Studio runs image and video renders side by side. A tick+ skipped here costs that step its marker and nothing else: the step number travels+ with the event, so the later ones do not shift, and the poller keeps reporting."""+ with _hold_off_cuda_graph_capture() as clear:+ if not clear:+ return+ ticker.record(enqueued)+ _report(ticker.completed())++ def _finish_denoise() -> None:+ """The denoise is provably over, so complete the bar rather than leaving it+ wherever the last event landed. Also why the bar no longer stops one short:+ MiniMax-H3's loop makes 29 ``scheduler.step`` calls for a 30-step render, and+ nothing else ever reported the last step."""+ self._gen.update(step = steps, eta_seconds = None)++ def _enter_decode_phase() -> None:+ _finish_denoise()+ self._gen.update(phase = "decode", eta_seconds = None)++ def _on_decode() -> None:+ """The decoder was entered, which is a HOST position: on a family whose host+ runs ahead the denoise kernels may still be queued, and flipping here would+ jump the bar to steps/steps while the GPU was still denoising. So mark the+ boundary in the stream and let the poller flip when the GPU reaches it. With no+ event to wait on there is no queue to have run ahead of, so flip at once.++ Marking records an event, so it takes the capture hold-off too, and it RETRIES:+ the hold-off acquires its lock without blocking, so a refusal can mean nothing+ worse than the 10 Hz poller holding it for its own queries at that instant, and+ giving up there would flip the bar to steps/steps with the denoise queue still+ draining, which is the exact lie this boundary exists to remove. The retries are+ short and only run when contended. A capture that outlasts them really does leave+ nothing to wait on, and the host position is then the only answer there is."""+ marked = False+ for _ in range(_BOUNDARY_MARK_ATTEMPTS):+ with _hold_off_cuda_graph_capture() as clear:+ if clear:+ marked = ticker.mark_boundary()+ break+ time.sleep(_BOUNDARY_MARK_RETRY_SECONDS)+ if not marked:+ _enter_decode_phase()++ def _pump() -> None:+ """One poll, inside the capture hold-off. Advances the step from the GPU, and+ takes the denoise to complete only once the GPU has reached the marked end."""+ if self._gen.get("phase") != "denoise":+ return+ if ticker.boundary_marked and ticker.boundary_reached():+ _enter_decode_phase()+ return+ _report(ticker.completed())+def _on_step(p, step_index, timestep, callback_kwargs):+ # diffusers calls this at the END of a loop iteration, after scheduler.step, so+ # the step's latent update is already submitted when the marker goes down.if cancel.is_set():p._interrupt = Truereturn callback_kwargs_tick(step_index + 1)return callback_kwargs- def _on_scheduler_step(done: int) -> None:- # No cooperative _interrupt here, so cancellation must unwind the denoise loop via an exception+ def _on_scheduler_step_cancel(done: int) -> None:+ # Runs BEFORE the scheduler step. No cooperative _interrupt here, so+ # cancellation must unwind the denoise loop via an exception, and it has to do+ # that before the step's work is submitted rather than after.if cancel.is_set():raise _VideoGenerationCancelled()- _tick(done)- if "callback_on_step_end" in call_params:+ has_step_callback = "callback_on_step_end" in call_params+ if has_step_callback:kwargs["callback_on_step_end"] = _on_step- progress_ctx = contextlib.nullcontext()- else:- # HunyuanVideo-1.5 has no step callback, so wrap scheduler.step for progress + cancel and restore- # afterwards. Same for H3's modular workflow: ModularPipeline takes no callback (an unknown input is- # only warned about), but MiniMaxH3LoopSchedulerStep calls components.scheduler.step --- # pipe.scheduler -- once per denoise step, so the wrapper ticks and can unwind a multi-minute run on- # Cancel.- progress_ctx = _scheduler_step_progress(pipe, _on_scheduler_step)++ @contextlib.contextmanager+ def progress_ctx():+ """Everything that reports on a denoise, entered as one unit around ``pipe()``."""+ with contextlib.ExitStack() as stack:+ if not has_step_callback:+ # HunyuanVideo-1.5 has no step callback, so wrap scheduler.step for progress + cancel and+ # restore afterwards. Same for H3's modular workflow: ModularPipeline takes no callback (an+ # unknown input is only warned about), but MiniMaxH3LoopSchedulerStep calls+ # components.scheduler.step -- pipe.scheduler -- once per denoise step, so the wrapper ticks+ # and can unwind a multi-minute run on Cancel. Cancel before the step, tick after it.+ stack.enter_context(+ _scheduler_step_progress(pipe, _on_scheduler_step_cancel, _tick)+ )+ # Family-agnostic: no video family has a callback between its denoise loop and its decode, so+ # every one of them gets its decode phase from the decoder itself.+ stack.enter_context(_decode_phase(pipe, _on_decode))+ stack.enter_context(_completed_step_poller(_pump))+ yield# Re-check an AUTO cache decision against the ACTUAL step count; explicit choices never toggleif state.cache_auto:@@ -5727,7 +6090,7 @@ class VideoBackend:if state.transformer_cache:self._reset_step_cache(pipe)try:- with torch.inference_mode(), progress_ctx, sigma_ctx:+ with torch.inference_mode(), progress_ctx(), sigma_ctx:output = pipe(**kwargs)except _VideoGenerationCancelled:# Unwinding by exception skips maybe_free_model_hooks(); under offload the onloaded modules would@@ -5742,6 +6105,9 @@ class VideoBackend:if cancel.is_set():raise RuntimeError(VIDEO_CANCELLED_MSG)+ # The pipeline returned, so every step ran whatever the last event said. This is+ # also the only place a latent-only render (no decoder entered) can complete.+ _finish_denoise()self._gen.update(phase = "export", eta_seconds = None)if fam.modular_workflow:video_frames = output["videos"][0]diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.pyindex ea0b80f4eef..42fd3c9ec11 100644--- a/studio/backend/tests/test_video_backend.py+++ b/studio/backend/tests/test_video_backend.py@@ -291,6 +291,19 @@ class _FakeWanVae:def __init__(self) -> None:self.tiled = Falseself.decoder = _FakeWanDecoder()+ self.decodes = 0+ # Test hooks fired from INSIDE the real decode, where the reported phase must already say+ # "decode": that is the whole stretch the progress used to report as the last denoise step.+ self.on_decode = None+ self.decode_raises = None++ def decode(self, *args, **kwargs):+ self.decodes += 1+ if self.on_decode is not None:+ self.on_decode()+ if self.decode_raises is not None:+ raise self.decode_raises+ return object()def enable_tiling(self) -> None:self.tiled = True@@ -331,6 +344,9 @@ class _FakeWanPipeBase:callback_on_step_end(self, step, 0, {})if self._interrupt:break+ if not self._interrupt:+ # Like every other family, the decode runs inside __call__ with no callback around it.+ self.vae.decode(object())frames = [[object() for _ in range(int(num_frames or 1))]]return types.SimpleNamespace(frames = frames, audio = None)@@ -483,6 +499,9 @@ class _FakeHV15Pipe:passfor _ in range(int(num_inference_steps or 1)):self.scheduler.step()+ # The real pipeline decodes inside __call__ with no callback around it, which is why the+ # decode phase has to come from the decoder itself.+ self.vae.decode(object())frames = [[object() for _ in range(int(num_frames or 1))]]return types.SimpleNamespace(frames = frames, audio = None)@@ -4323,8 +4342,10 @@ def test_h3_modular_generation_ticks_and_cancels_through_the_scheduler(fake_runtpipe.scheduler.on_step = lambda n: steps_seen.append(backend._gen.get("step"))result = backend.generate(prompt = "a fox", steps = 4)assert "callback_on_step_end" not in pipe.last_kwargs- # One wrapped tick per denoise step, then the original method back in place.- assert steps_seen == [1, 2, 3, 4]+ # One wrapped tick per denoise step, then the original method back in place. The reading taken+ # from INSIDE step n is n-1, not n: scheduler.step is what enqueues the step's latent update,+ # so the tick for step n lands after the original returns rather than before it.+ assert steps_seen == [0, 1, 2, 3]assert pipe.scheduler.step.__func__ is _FakeH3Scheduler.stepassert result["num_frames"] == 124 and result["has_audio"] is False@@ -8910,3 +8931,623 @@ def test_h3_generate_non_oom_error_leaves_the_graphs_alone(fake_runtime):with pytest.raises(RuntimeError, match = "shape mismatch"):backend.generate(prompt = "a fox", steps = 4)assert handle.resets == 0+++# ── Progress truth: the bar must track the GPU, not the host queue ─────────────+# Measured on MiniMax-H3 (960x544x124, 30 steps, int8 DiT + int8 TE, speed=default with a captured+# CUDA graph, B200): the 29 host-side scheduler.step calls land in 0.9 s and the GPU is still+# 25.9 s from finishing the denoise, then spends another 2.6 s in the video VAE decode and 1.3 s in+# post-processing. The bar used to reach "step 29/30, ~1s left" in a second and then sit there for+# half a minute.+++class _FakeCudaEvent:+ """A ``torch.cuda.Event`` stand-in whose completion the test controls."""++ def __init__(self) -> None:+ self.done = False+ self.queries = 0++ def query(self) -> bool:+ self.queries += 1+ return self.done+++class _ClassDecoder:+ """A decoder whose ``decode`` lives on the CLASS, as an untouched diffusers VAE's does."""++ def __init__(self) -> None:+ self.calls = 0++ def decode(self, *args, **kwargs):+ self.calls += 1+ return "decoded"+++def _ticker(monkeypatch, total):+ """A ticker whose events the test owns, plus the list they land in."""+ import core.inference.video as video_mod++ events: list = []++ def _fake_event():+ event = _FakeCudaEvent()+ events.append(event)+ return event++ monkeypatch.setattr(video_mod, "_record_cuda_event", _fake_event)+ return video_mod._CompletedStepTicker(total), events+++def test_ticker_reports_only_steps_the_gpu_finished(monkeypatch):+ ticker, events = _ticker(monkeypatch, 8)+ for step in range(1, 9):+ ticker.record(step)+ # The host has enqueued `step` steps and the GPU has finished none of them.+ assert ticker.completed() == 0+ assert ticker.event_backed is True+ events[0].done = True+ events[1].done = True+ assert ticker.completed() == 2+ # Still nothing beyond the contiguous run of finished events, even if a later one lands first.+ events[5].done = True+ assert ticker.completed() == 2+ for event in events:+ event.done = True+ assert ticker.completed() == 8+++def test_ticker_never_walks_backwards_or_past_the_total(monkeypatch):+ ticker, events = _ticker(monkeypatch, 4)+ for step in range(1, 5):+ ticker.record(step)+ for event in events:+ event.done = True+ assert ticker.completed() == 4+ # A duplicated tick, and an event that starts failing its query, must not lower the bar.+ ticker.record(2)+ for event in events:+ event.done = False+ assert ticker.completed() == 4+ # Records past the total are counted as enqueued but mark no further events.+ ticker.record(9)+ assert len(events) == 4+ assert ticker.completed() == 4+++def test_ticker_falls_back_to_the_host_count_without_cuda_events(monkeypatch):+ # A CPU or MPS host has no events to poll, and there is no graph to run ahead of either, so+ # the host count IS the progress. This is exactly the pre-existing behaviour.+ import core.inference.video as video_mod++ monkeypatch.setattr(video_mod, "_record_cuda_event", lambda: None)+ ticker = video_mod._CompletedStepTicker(5)+ for step in range(1, 6):+ ticker.record(step)+ assert ticker.completed() == step+ assert ticker.event_backed is False+++def test_ticker_unmarkable_step_does_not_shift_the_ones_after_it(monkeypatch):+ # A step whose event could not be recorded (a capture was in flight) costs its own tick and+ # nothing more: the step number travels with the event.+ import core.inference.video as video_mod++ events: list = []+ refuse = {"step": 2}++ def _fake_event():+ if len(events) + 1 == refuse["step"]:+ refuse["step"] = None+ return None+ event = _FakeCudaEvent()+ events.append(event)+ return event++ monkeypatch.setattr(video_mod, "_record_cuda_event", _fake_event)+ ticker = video_mod._CompletedStepTicker(4)+ for step in range(1, 5):+ ticker.record(step)+ assert len(events) == 3 # steps 1, 3, 4+ events[0].done = True+ assert ticker.completed() == 1+ events[1].done = True+ assert ticker.completed() == 3 # the event carries its own step number, so 3, never 2+++def test_completed_step_poller_advances_with_no_host_ticks(monkeypatch):+ # The whole point: the host stops ticking the moment it has enqueued the last step, so+ # something has to keep asking the GPU.+ import core.inference.video as video_mod++ ticker, events = _ticker(monkeypatch, 3)+ for step in range(1, 4):+ ticker.record(step)+ seen: list = []+ with video_mod._completed_step_poller(lambda: seen.append(ticker.completed()), 0.01):+ for event in events:+ event.done = True+ deadline = time.monotonic() + 5.0+ while time.monotonic() < deadline and 3 not in seen:+ time.sleep(0.01)+ assert 3 in seen+ # And the thread is gone with the context.+ assert not [t for t in threading.enumerate() if t.name == "video-denoise-progress"]+++def test_completed_step_poller_stands_back_during_a_graph_capture(monkeypatch):+ # cudaEventQuery is one of the calls CUDA prohibits from EVERY thread while a capture begun in+ # the default global mode is recording, and the denoiser captures during the first steps.+ import core.inference.video as video_mod++ ticker, events = _ticker(monkeypatch, 2)+ for step in (1, 2):+ ticker.record(step)+ for event in events:+ event.done = True+ from core.inference import diffusion_cuda_graph as cg++ seen: list = []+ with cg._capturing(): # a capture really is recording for the whole poll window+ with video_mod._completed_step_poller(lambda: seen.append(ticker.completed()), 0.01):+ time.sleep(0.2)+ assert seen == []+ assert all(event.queries == 0 for event in events)+++def test_cuda_graph_capture_flag_tracks_the_recording_window():+ from core.inference import diffusion_cuda_graph as cg++ assert cg.capture_in_progress() is False+ with cg._capturing():+ assert cg.capture_in_progress() is True+ with cg._capturing(): # nothing says two pipelines cannot capture at once+ assert cg.capture_in_progress() is True+ assert cg.capture_in_progress() is True+ assert cg.capture_in_progress() is False+ # And it is released when the capture raises.+ with pytest.raises(RuntimeError, match = "capture failed"):+ with cg._capturing():+ raise RuntimeError("capture failed")+ assert cg.capture_in_progress() is False+++def test_decode_phase_fires_once_and_restores_a_class_decode():+ import core.inference.video as video_mod++ vae = _ClassDecoder()+ pipe = types.SimpleNamespace(vae = vae)+ fired: list = []+ with video_mod._decode_phase(pipe, lambda: fired.append("decode")):+ assert vae.decode(1) == "decoded"+ assert vae.decode(2) == "decoded"+ assert fired == ["decode"] # at most once per generation+ assert vae.calls == 2+ # Nothing left shadowing the class method: a bound method parked in a module's __dict__ is a+ # reference cycle back to the module.+ assert "decode" not in vae.__dict__+ assert vae.decode(3) == "decoded"+++def test_decode_phase_restores_a_compiled_decode_it_found():+ # The speed layer puts torch.compile(vae.decode) in the instance __dict__ for U-Net families+ # (_compile_vae_decode), so the wrapper must put THAT back, not fall through to the class.+ import core.inference.video as video_mod++ vae = _ClassDecoder()+ compiled = lambda *a, **k: "compiled" # noqa: E731 - a stand-in for the compiled callable+ vae.decode = compiled+ pipe = types.SimpleNamespace(vae = vae)+ with video_mod._decode_phase(pipe, lambda: None):+ assert vae.decode() == "compiled"+ assert vae.decode is compiled+++def test_decode_phase_restores_the_wrapper_when_the_decode_raises():+ import core.inference.video as video_mod++ vae = _ClassDecoder()+ pipe = types.SimpleNamespace(vae = vae)+ fired: list = []++ def _boom(*args, **kwargs):+ raise RuntimeError("out of memory in decode")++ vae.decode = _boom+ with pytest.raises(RuntimeError, match = "out of memory in decode"):+ with video_mod._decode_phase(pipe, lambda: fired.append("decode")):+ vae.decode()+ # The phase was reported before the decode blew up, and the wrapper came off anyway.+ assert fired == ["decode"]+ assert vae.decode is _boom+++def test_decode_phase_covers_the_audio_decoder_too():+ # MiniMax-H3 decodes video then audio; whichever runs first names the phase.+ import core.inference.video as video_mod++ audio_vae = _ClassDecoder()+ pipe = types.SimpleNamespace(vae = None, audio_vae = audio_vae)+ fired: list = []+ with video_mod._decode_phase(pipe, lambda: fired.append("decode")):+ audio_vae.decode()+ assert fired == ["decode"]+ assert "decode" not in audio_vae.__dict__+++def _patch_events(monkeypatch, *, done: bool = False):+ """Hand out fake CUDA events, optionally already complete. Returns the list they land in."""+ import core.inference.video as video_mod++ events: list = []++ def _fake_event():+ event = _FakeCudaEvent()+ event.done = done+ events.append(event)+ return event++ monkeypatch.setattr(video_mod, "_record_cuda_event", _fake_event)+ return events+++def test_step_marker_goes_down_after_the_scheduler_step_not_before(fake_runtime, monkeypatch):+ # scheduler.step is what enqueues the step's latent update (and H3's audio scheduler update+ # after it), so a marker recorded before it sits AHEAD of kernels that step still owes and can+ # signal while the step is still running. The tick must come after the original returns.+ events = _patch_events(monkeypatch)+ backend = VideoBackend()+ backend.load_pipeline(+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"+ )+ pipe = _FakeHV15Pipeline.instance+ marked_when_step_ran: list = []+ # The fake fires this from INSIDE the original scheduler.step, i.e. while step n's latent+ # update is being submitted.+ pipe.scheduler.on_step = lambda n: marked_when_step_ran.append((n, len(events)))++ backend.generate(prompt = "a fox", steps = 5, num_frames = 9, fps = 24)++ # While step n is submitting, only the n-1 steps BEFORE it are marked. The old ordering made+ # this (n, n): step n was already marked before its own work was enqueued.+ assert marked_when_step_ran == [(1, 0), (2, 1), (3, 2), (4, 3), (5, 4)]+ assert len(events) == 6 # five steps plus the end-of-denoise boundary marker+++def test_cancellation_is_still_checked_before_the_step_runs(fake_runtime, monkeypatch):+ # Moving the tick after the call must not move the cancel check with it: cancelling has to+ # unwind before the next step's work is submitted, not after it has run.+ events = _patch_events(monkeypatch)+ backend = VideoBackend()+ backend.load_pipeline(+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"+ )+ pipe = _FakeHV15Pipeline.instance+ pipe.scheduler.on_step = lambda n: backend.cancel_generate() if n == 1 else None+ with pytest.raises(RuntimeError, match = VIDEO_CANCELLED_MSG):+ backend.generate(prompt = "a fox", steps = 4)+ # Step 2 never ran: the pre-hook raised ahead of it.+ assert pipe.scheduler.calls == 1+ # And exactly one step was marked, the one that really was submitted. No boundary marker+ # either, since the decoder was never reached.+ assert len(events) == 1+++def test_hv15_bar_never_outruns_the_gpu_and_holds_the_phase(fake_runtime, monkeypatch):+ # End to end on the scheduler-wrap path (HunyuanVideo-1.5 exposes no step callback, like H3's+ # modular workflow). The host enqueues every step and enters the decoder while the GPU has+ # finished nothing, which is exactly the Wan / LTX-2 shape.+ events = _patch_events(monkeypatch)+ backend = VideoBackend()+ backend.load_pipeline(+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"+ )+ pipe = _FakeHV15Pipeline.instance+ seen: list = []++ def _watch(enqueued):+ seen.append((enqueued, int(backend._gen.get("step") or 0), sum(e.done for e in events)))++ pipe.scheduler.on_step = _watch+ at_decode: dict = {}+ pipe.vae.on_decode = lambda: at_decode.update(backend._gen)++ backend.generate(prompt = "a fox", steps = 8, num_frames = 9, fps = 24)++ assert [enqueued for enqueued, _, _ in seen] == list(range(1, 9))+ # The invariant: never more steps reported than the GPU has actually completed.+ for enqueued, reported, finished in seen:+ assert reported <= finished, (enqueued, reported, finished)+ # With the GPU finishing nothing, the bar stayed at 0 through the whole host-side loop, where+ # it used to read 8/8 while no denoising had happened.+ assert {reported for _, reported, _ in seen} == {0}+ # And entering the decoder did NOT complete the bar or flip the phase, because the boundary+ # marker has not completed: the GPU is still denoising.+ assert at_decode.get("phase") == "denoise"+ assert at_decode.get("step") == 0+++def test_decode_phase_lands_once_the_gpu_reaches_the_boundary(fake_runtime, monkeypatch):+ # Same path, but now the device keeps up with the host, so the boundary marker is complete when+ # the decoder is entered and the phase flips there.+ _patch_events(monkeypatch, done = True)+ backend = VideoBackend()+ backend.load_pipeline(+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"+ )+ pipe = _FakeHV15Pipeline.instance+ at_decode: dict = {}+ pipe.vae.on_decode = lambda: _settle(backend, at_decode)++ backend.generate(prompt = "a fox", steps = 8, num_frames = 9, fps = 24)++ assert at_decode.get("phase") == "decode"+ assert at_decode.get("step") == 8+ assert at_decode.get("total") == 8+ assert at_decode.get("eta_seconds") is None+++def _settle(+ backend,+ out,+ timeout = 5.0,+):+ """Wait for the poller to publish the decode phase, then snapshot it.++ The flip is the poller's job precisely because the decoder-entry position cannot know whether+ the GPU has caught up, so a test that reads _gen the instant the decoder is entered is racing+ the poll interval rather than testing anything.+ """+ deadline = time.monotonic() + timeout+ while time.monotonic() < deadline:+ if backend._gen.get("phase") == "decode":+ break+ time.sleep(0.01)+ out.update(backend._gen)+++def test_a_capture_cannot_start_while_an_event_query_is_in_flight(monkeypatch):+ # The TOCTOU this replaced: a poll read "no capture", the capture began, and the poll's+ # prohibited cudaEventQuery landed inside it. The lock has to span the query itself.+ import core.inference.video as video_mod+ from core.inference import diffusion_cuda_graph as cg++ in_query = threading.Event()+ finish_query = threading.Event()++ class _BlockingEvent:+ def query(self):+ in_query.set()+ finish_query.wait(5.0)+ return True++ monkeypatch.setattr(video_mod, "_record_cuda_event", lambda: _BlockingEvent())+ ticker = video_mod._CompletedStepTicker(1)+ ticker.record(1)++ entered_capture = threading.Event()++ def _capture():+ with cg._capturing():+ entered_capture.set()++ with video_mod._completed_step_poller(ticker.completed, 0.01):+ assert in_query.wait(5.0), "the poller never reached the query"+ capturer = threading.Thread(target = _capture, daemon = True)+ capturer.start()+ # A capture must not be able to raise the depth, and so must not be able to enter+ # torch.cuda.graph, while a query is in flight.+ assert not entered_capture.wait(0.5)+ finish_query.set()+ assert entered_capture.wait(5.0)+ capturer.join(timeout = 5.0)+ assert cg.capture_in_progress() is False+++def test_hold_off_capture_yields_false_rather_than_waiting_on_a_capture():+ from core.inference import diffusion_cuda_graph as cg++ with cg.hold_off_capture() as clear:+ assert clear is True+ with cg._capturing():+ with cg.hold_off_capture() as clear:+ assert clear is False+ with cg.hold_off_capture() as clear:+ assert clear is True+++def test_denoise_completes_even_when_the_loop_ticks_fewer_times_than_steps(fake_runtime):+ # MiniMax-H3's loop makes 29 scheduler.step calls for a 30-step render, so a bar driven purely+ # by ticks stops at 29/30 forever. Reaching the decode proves the denoise finished.+ backend = VideoBackend()+ pipe = _load_h3_modular(backend)+ seen: dict = {}+ original_call = pipe.__class__.__call__++ def _one_tick_short(+ self,+ *,+ num_inference_steps = None,+ **kwargs,+ ):+ out = original_call(self, num_inference_steps = int(num_inference_steps) - 1, **kwargs)+ seen["at_end_of_loop"] = int(backend._gen.get("step") or 0)+ return out++ pipe.__class__.__call__ = _one_tick_short+ try:+ backend.generate(prompt = "a fox", steps = 30, num_frames = 9, fps = 24)+ finally:+ pipe.__class__.__call__ = original_call+ assert pipe.scheduler.calls == 29+ assert seen["at_end_of_loop"] < 30+ # generate() completed the bar once the pipeline returned, so the user never sees 29/30 stick.+ assert backend._gen == {"active": False}+++def test_hv15_cancel_still_unwinds_and_leaves_nothing_installed(fake_runtime):+ backend = VideoBackend()+ backend.load_pipeline(+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"+ )+ pipe = _FakeHV15Pipeline.instance+ pipe.scheduler.on_step = lambda n: backend.cancel_generate() if n == 1 else None+ with pytest.raises(RuntimeError, match = VIDEO_CANCELLED_MSG):+ backend.generate(prompt = "a fox", steps = 4)+ assert pipe.scheduler.calls == 1+ assert pipe.scheduler.step.__func__ is _FakeHV15Scheduler.step+ # The decode never ran, its wrapper is off, and the poller thread is gone.+ assert pipe.vae.decodes == 0+ assert "decode" not in pipe.vae.__dict__+ assert not [t for t in threading.enumerate() if t.name == "video-denoise-progress"]+++def test_decode_failure_surfaces_and_leaves_nothing_installed(fake_runtime):+ backend = VideoBackend()+ backend.load_pipeline(+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"+ )+ pipe = _FakeHV15Pipeline.instance+ pipe.vae.decode_raises = RuntimeError("CUDA out of memory in decode")+ with pytest.raises(RuntimeError, match = "CUDA out of memory in decode"):+ backend.generate(prompt = "a fox", steps = 4)+ assert "decode" not in pipe.vae.__dict__+ assert not [t for t in threading.enumerate() if t.name == "video-denoise-progress"]+++def test_callback_family_keeps_its_own_callback(fake_runtime, monkeypatch):+ # Wan exposes callback_on_step_end, so nothing wraps its scheduler. Only the tick changed:+ # the reported step now comes from completed events rather than the callback's index.+ import core.inference.video as video_mod++ events = _patch_events(monkeypatch, done = True)+ wrapped: list = []+ original_wrap = video_mod._scheduler_step_progress+ monkeypatch.setattr(+ video_mod,+ "_scheduler_step_progress",+ lambda *a, **kw: wrapped.append(a[0]) or original_wrap(*a, **kw),+ )++ backend = VideoBackend()+ backend.load_pipeline("Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline")+ pipe = backend._state.pipe+ at_decode: dict = {}+ pipe.vae.on_decode = lambda: _settle(backend, at_decode)++ backend.generate(prompt = "a sloth", steps = 6, num_frames = 9, fps = 24)++ # The scheduler was never wrapped: this family drives its own callback and still does.+ assert wrapped == []+ # One marker per callback tick, so the callback really ran, plus the boundary marker.+ assert len(events) == 7+ assert pipe.vae.decodes == 1+ assert at_decode.get("phase") == "decode"+ assert at_decode.get("step") == 6+++# ── the step tick and the boundary marker are CUDA calls too ────────────────────────────────+++def test_a_capture_in_another_pipeline_stops_the_tick_from_touching_events(+ fake_runtime, monkeypatch+):+ # torch captures in cudaStreamCaptureModeGlobal, under which a capture recording in ANY+ # thread bars every thread from potentially unsafe calls -- recording and querying an event+ # among them. is_current_stream_capturing only sees this pipeline's own capture, so a second+ # pipeline capturing while this one ticks is exactly the case the poller is guarded for, and+ # the tick and the boundary marker have to take the same hold-off.+ import core.inference.video as video_mod++ events = _patch_events(monkeypatch, done = True)++ @contextlib.contextmanager+ def _capture_running():+ yield False++ monkeypatch.setattr(video_mod, "_hold_off_cuda_graph_capture", _capture_running)++ backend = VideoBackend()+ backend.load_pipeline(+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"+ )+ pipe = _FakeHV15Pipeline.instance+ # Read inside the decode, since _gen is cleared once generate() returns.+ at_decode: dict = {}+ pipe.vae.on_decode = lambda: at_decode.update(backend._gen)++ backend.generate(prompt = "a fox", steps = 5, num_frames = 9, fps = 24)++ # Not one event was recorded: neither the five step markers nor the end-of-denoise boundary.+ assert events == []+ # And the bar is still honest at the end. With no marker to wait on there is nothing to wait+ # FOR, so the decode flip falls back to the host position, which is what this family did+ # before any of this and is correct when the queue cannot be measured.+ assert at_decode.get("phase") == "decode"+ assert at_decode.get("step") == 5+++def test_the_tick_marks_every_step_again_once_the_capture_is_over(fake_runtime, monkeypatch):+ # The control for the test above: the guard must not cost a marker when nothing is capturing,+ # or it would quietly turn the whole feature off.+ events = _patch_events(monkeypatch, done = True)++ backend = VideoBackend()+ backend.load_pipeline(+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"+ )+ backend.generate(prompt = "a fox", steps = 5, num_frames = 9, fps = 24)++ assert len(events) == 6 # five steps plus the boundary+++def test_the_boundary_marker_waits_out_a_busy_capture_lock(fake_runtime, monkeypatch):+ # hold_off_capture acquires without blocking, so a refusal can mean nothing worse than the+ # poller holding the lock for its own queries at that instant. Giving up on the first refusal+ # would flip the bar to steps/steps with the queue still draining, so the marker retries.+ import core.inference.video as video_mod++ _patch_events(monkeypatch, done = True)++ # Refuses everything for a window that covers the decode entry, then frees up. Shorter than+ # the retry budget (20 x 20 ms), so a marker that waits gets its event and one that gives up+ # never calls mark_boundary at all.+ busy_until = {"t": 0.0}++ @contextlib.contextmanager+ def _busy_window():+ yield time.monotonic() >= busy_until["t"]++ marks = {"calls": 0, "ok": None}+ original_mark = video_mod._CompletedStepTicker.mark_boundary++ def _counted(self):+ marks["calls"] += 1+ marks["ok"] = original_mark(self)+ return marks["ok"]++ monkeypatch.setattr(video_mod._CompletedStepTicker, "mark_boundary", _counted)++ backend = VideoBackend()+ backend.load_pipeline(+ "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", model_kind = "pipeline"+ )+ pipe = _FakeHV15Pipeline.instance+ at_decode: dict = {}+ pipe.vae.on_decode = lambda: _settle(backend, at_decode)++ def _contend(n):+ # Armed on the last step, so the lock is busy exactly across the decode entry.+ if n == 5:+ busy_until["t"] = time.monotonic() + 0.2+ monkeypatch.setattr(video_mod, "_hold_off_cuda_graph_capture", _busy_window)++ pipe.scheduler.on_step = _contend++ backend.generate(prompt = "a fox", steps = 5, num_frames = 9, fps = 24)++ # It waited the lock out and marked a real boundary instead of flipping blind. Before the+ # retry, a refusal at this instant meant mark_boundary was never called at all.+ assert marks["calls"] == 1+ assert marks["ok"] is True+ assert at_decode.get("phase") == "decode"