Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: encode Wan clips while the VAE decodes (#14405)

Daniel Hanffd64b1e3800
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index f97c417db6e..47301289c84 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -279,6 +279,7 @@ from .video_minimax_h3_te import (
)
from .video_encode import encode_x264
from .video_nvenc import encode_nvenc, nvenc_gpu
+from .video_stream_encode import claim_stream, offer_stream, open_stream_encode
from utils.hardware import clear_gpu_cache
# Shared with the image backend so both pin every loader call to the same live cache root
@@ -11461,6 +11462,8 @@ class VideoBackend:
bg_token = bg_compile._FORCE_EAGER.set(True) if bg_eager else None
# Closed in the finally so the pipeline is text-to-video again on return, cancel or failure.
i2v_swap = contextlib.ExitStack()
+ # Encodes Wan chunks while the VAE decodes; closed in the finally on cancel or failure.
+ stream_session = None
try:
# FIRST, before any device object exists. begin_generate runs this on a fresh daemon thread, so until it
# is pinned the un-indexed state.device below -- the H3 memory probe and every torch.Generator --
@@ -12106,6 +12109,8 @@ class VideoBackend:
stack.enter_context(_decode_phase(pipe, _on_decode))
# The decoded clip becomes uint8 on the GPU, bit-identical to the np / pil export (video_frames).
stack.enter_context(uint8_video_frames(pipe))
+ if stream_session is not None:
+ stack.enter_context(stream_session.hooks())
stack.enter_context(_completed_step_poller(_pump))
yield
@@ -12164,6 +12169,14 @@ class VideoBackend:
if not _render_under_no_grad(state)
else torch.no_grad()
)
+ stream_session = open_stream_encode(
+ pipe,
+ out_fps,
+ engage = not fam.modular_workflow
+ and not fam.has_audio
+ and ltx25_render_call is None,
+ logger = logger,
+ )
previewer = LatentPreviewer.create(
family = fam.name,
requested = live_preview,
@@ -12223,18 +12236,20 @@ class VideoBackend:
audio = getattr(output, "audio", None)
audio_track = audio[0] if fam.has_audio and audio is not None else None
audio_sample_rate = None
- if audio_sample_rate is None:
- mp4_bytes = self._encode_mp4(
- video_frames, out_fps, audio_track, pipe if fam.has_audio else None
- )
- else:
- mp4_bytes = self._encode_mp4(
- video_frames,
- out_fps,
- audio_track,
- pipe if fam.has_audio else None,
- audio_sample_rate = int(audio_sample_rate),
- )
+ # Encoded during the decode when its frames are exactly these (_encode_mp4 claims it).
+ with offer_stream(stream_session if audio_track is None else None):
+ if audio_sample_rate is None:
+ mp4_bytes = self._encode_mp4(
+ video_frames, out_fps, audio_track, pipe if fam.has_audio else None
+ )
+ else:
+ mp4_bytes = self._encode_mp4(
+ video_frames,
+ out_fps,
+ audio_track,
+ pipe if fam.has_audio else None,
+ audio_sample_rate = int(audio_sample_rate),
+ )
# A cancel during the blocking export/mux must still discard the clip; re-check before it is persisted.
if cancel.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
@@ -12304,6 +12319,8 @@ class VideoBackend:
_log_failed_generation(request_shape, exc)
raise
finally:
+ if stream_session is not None:
+ stream_session.close()
end_cfg_render(getattr(state, "pipe", None))
end_cfg_batch_render(getattr(state, "pipe", None))
try:
@@ -12968,6 +12985,10 @@ class VideoBackend:
"""Encode frames (+ optional audio) to H.264 MP4 bytes via diffusers' PyAV
exporter. A temp file bridges the exporter's path-based API; the bytes are
what the gallery persists."""
+ if audio is None:
+ streamed = claim_stream(video_frames, fps)
+ if streamed is not None:
+ return streamed
from diffusers.utils.export_utils import encode_video
tmp = tempfile.NamedTemporaryFile(suffix = ".mp4", delete = False)
diff --git a/studio/backend/core/inference/video_stream_encode.py b/studio/backend/core/inference/video_stream_encode.py
new file mode 100644
index 00000000000..4644d702d5e
--- /dev/null
+++ b/studio/backend/core/inference/video_stream_encode.py
@@ -0,0 +1,377 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Encode a Wan clip while its VAE is still decoding.
+
+A decoder forward hook converts each chunk to uint8 with the pipeline's exact ops, copies it to pinned memory on a
+side stream and feeds a libx264 thread. The streamed mp4 is used only when its frames equal the final frames bit for
+bit; otherwise the clip is encoded as before. ``UNSLOTH_STUDIO_VIDEO_STREAM_ENCODE=0`` turns it off.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import queue
+import sys
+import tempfile
+import threading
+from pathlib import Path
+from typing import Any, Iterator, Optional
+
+STREAM_ENCODE_ENV = "UNSLOTH_STUDIO_VIDEO_STREAM_ENCODE"
+QUEUE_DEPTH = 8
+_STOP = object()
+
+STATS = {"streamed": 0, "fallback": 0, "last": ""}
+
+
+def stream_encode_enabled() -> bool:
+ return os.environ.get(STREAM_ENCODE_ENV, "").strip().lower() not in ("0", "false", "no", "off")
+
+
+def _note(outcome: str, reason: str, logger: Any) -> None:
+ STATS[outcome] += 1
+ STATS["last"] = reason
+ if logger is not None:
+ logger.info("video.stream_encode: %s (%s)", outcome, reason)
+
+
+class StreamEncode:
+ """One render's overlapped encode. Hooks are live only inside ``hooks()``; ``take`` or ``close`` ends it."""
+
+ def __init__(
+ self,
+ vae: Any,
+ processor: Any,
+ fps: int,
+ logger: Any = None,
+ ) -> None:
+ import torch
+
+ self._torch = torch
+ self.vae = vae
+ self.processor = processor
+ self.fps = int(fps)
+ self.logger = logger
+ patch = getattr(getattr(vae, "config", None), "patch_size", None)
+ self._unpatchify = (
+ getattr(sys.modules.get(type(vae).__module__), "unpatchify", None) if patch else None
+ )
+ self._patch = patch
+ self._side = None
+ self.chunks: list = []
+ self.reason = ""
+ self.valid = True
+ self._decoding = False
+ self._decoded = False
+ self._queue: "queue.Queue[Any]" = queue.Queue(maxsize = QUEUE_DEPTH)
+ self._failed = False
+ self._thread: Optional[threading.Thread] = None
+ handle = tempfile.NamedTemporaryFile(suffix = ".mp4", delete = False)
+ handle.close()
+ self.path = handle.name
+
+ def invalidate(self, reason: str) -> None:
+ if self.valid:
+ self.valid = False
+ self.reason = reason
+
+ def _on_chunk(self, _module: Any, args: Any, kwargs: Any, output: Any) -> None:
+ if not self.valid or not self._decoding:
+ return
+ torch = self._torch
+ try:
+ if kwargs.get("first_chunk") and self.chunks:
+ # The fp16 decode's fp32 rerun (or any second pass) starts over: the chunks so far are stale.
+ self.invalidate("decoder restarted")
+ return
+ if not torch.is_tensor(output) or output.ndim != 5 or output.shape[0] != 1:
+ self.invalidate("unexpected decoder output")
+ return
+ if self._failed:
+ self.invalidate("encoder failed")
+ return
+ x = output
+ if self._patch:
+ if self._unpatchify is None:
+ self.invalidate("no unpatchify")
+ return
+ x = self._unpatchify(x, patch_size = self._patch)
+ x = torch.clamp(x, min = -1.0, max = 1.0)
+ x = self.processor.postprocess(x[0].permute(1, 0, 2, 3), "pt")
+ x = (
+ (x.to(torch.float32) * 255)
+ .round_()
+ .to(torch.uint8)
+ .permute(0, 2, 3, 1)
+ .contiguous()
+ )
+ device = x.device
+ done = None
+ if device.type == "cuda":
+ if self._side is None:
+ self._side = torch.cuda.Stream(device = device)
+ ready = torch.cuda.Event()
+ ready.record(torch.cuda.current_stream(device))
+ host = torch.empty(x.shape, dtype = torch.uint8, pin_memory = True)
+ with torch.cuda.stream(self._side):
+ self._side.wait_event(ready)
+ host.copy_(x, non_blocking = True)
+ done = torch.cuda.Event()
+ done.record(self._side)
+ x.record_stream(self._side)
+ else:
+ host = x.cpu() if device.type != "cpu" else x
+ except Exception as exc: # noqa: BLE001 - streaming is an optimisation; the normal encode still runs
+ self.invalidate(f"chunk failed: {type(exc).__name__}")
+ return
+ self.chunks.append((host, done))
+ if self._thread is None:
+ self._thread = threading.Thread(
+ target = self._encode, name = "video-stream-encode", daemon = True
+ )
+ self._thread.start()
+ self._queue.put((host, done))
+
+ @contextlib.contextmanager
+ def hooks(self) -> Iterator[None]:
+ """Arm the decoder hook for the outermost ``vae.decode``; a tiled decode disarms it for this render."""
+ vae = self.vae
+ restore: list = []
+
+ def _wrap(name: str, make: Any) -> None:
+ original = getattr(vae, name, None)
+ if not callable(original):
+ return
+ had_own = name in getattr(vae, "__dict__", {})
+ setattr(vae, name, make(original))
+ restore.append((name, original, had_own))
+
+ def _decode_wrapper(original: Any) -> Any:
+ def decode(*args: Any, **kwargs: Any) -> Any:
+ if self._decoding:
+ return original(*args, **kwargs)
+ if self._decoded:
+ self.invalidate("second decode")
+ self._decoding = True
+ try:
+ return original(*args, **kwargs)
+ finally:
+ self._decoding = False
+ self._decoded = True
+
+ return decode
+
+ def _tiled_wrapper(original: Any) -> Any:
+ def tiled_decode(*args: Any, **kwargs: Any) -> Any:
+ self.invalidate("tiled decode")
+ return original(*args, **kwargs)
+
+ return tiled_decode
+
+ handle = None
+ try:
+ _wrap("decode", _decode_wrapper)
+ _wrap("tiled_decode", _tiled_wrapper)
+ handle = vae.decoder.register_forward_hook(self._on_chunk, with_kwargs = True)
+ except Exception: # noqa: BLE001 - a VAE that refuses the hooks just encodes afterwards
+ self.invalidate("hooks refused")
+ try:
+ yield
+ except BaseException:
+ self.close()
+ raise
+ finally:
+ if handle is not None:
+ handle.remove()
+ for name, original, had_own in reversed(restore):
+ try:
+ if had_own:
+ setattr(vae, name, original)
+ else:
+ delattr(vae, name)
+ except Exception: # noqa: BLE001 - cleanup is best-effort
+ pass
+
+ def _encode(self) -> None:
+ container = None
+ try:
+ import av
+ while True:
+ item = self._queue.get()
+ if item is _STOP:
+ break
+ if self._failed or not self.valid:
+ continue
+ host, done = item
+ if done is not None:
+ done.synchronize()
+ frames = host.numpy()
+ if container is None:
+ container = av.open(self.path, mode = "w", format = "mp4")
+ stream = container.add_stream("libx264", rate = self.fps)
+ stream.width = int(frames.shape[2])
+ stream.height = int(frames.shape[1])
+ stream.pix_fmt = "yuv420p"
+ stream.codec_context.thread_type = "FRAME"
+ for frame in frames:
+ for packet in stream.encode(av.VideoFrame.from_ndarray(frame, format = "rgb24")):
+ container.mux(packet)
+ if container is not None and self.valid and not self._failed:
+ for packet in stream.encode():
+ container.mux(packet)
+ except Exception: # noqa: BLE001 - drain so the render thread never blocks on a full queue
+ self._failed = True
+ while self._queue.get() is not _STOP:
+ pass
+ finally:
+ if container is not None:
+ try:
+ container.close()
+ except Exception: # noqa: BLE001
+ self._failed = True
+
+ def _join(self) -> None:
+ if self._thread is not None:
+ self._queue.put(_STOP)
+ self._thread.join()
+ self._thread = None
+
+ def _matches(self, video_frames: Any) -> bool:
+ torch = self._torch
+ if not isinstance(video_frames, torch.Tensor) or video_frames.dtype != torch.uint8:
+ self.invalidate("final frames are not uint8")
+ return False
+ if not self.chunks:
+ self.invalidate("no chunks")
+ return False
+ for _, done in self.chunks:
+ if done is not None:
+ done.synchronize()
+ streamed = [host for host, _ in self.chunks]
+ if sum(len(h) for h in streamed) != len(video_frames):
+ self.invalidate("frame count differs")
+ return False
+ import numpy as np
+
+ final = video_frames.numpy()
+ start = 0
+ for host in streamed:
+ # numpy, not torch.equal: torch's parallel compare costs seconds on a loaded many-core host.
+ if not np.array_equal(host.numpy(), final[start : start + len(host)]):
+ self.invalidate("frames differ")
+ return False
+ start += len(host)
+ return True
+
+ def take(self, video_frames: Any, fps: int) -> Optional[bytes]:
+ """The streamed mp4 when its frames are exactly ``video_frames`` at ``fps``; None (caller encodes) otherwise."""
+ try:
+ if self.valid and int(fps) != self.fps:
+ self.invalidate("fps differs")
+ if self.valid and not self._decoded:
+ self.invalidate("no decode")
+ if self.valid and self._matches(video_frames):
+ self._join()
+ if not self._failed and self.valid:
+ data = Path(self.path).read_bytes()
+ count = _mp4_frame_count(data, video_frames.shape[1:3])
+ if count == len(video_frames):
+ _note("streamed", f"{len(video_frames)} frames", self.logger)
+ return data
+ self.invalidate(f"streamed file holds {count} frames")
+ else:
+ self.invalidate(self.reason or "encoder failed")
+ _note("fallback", self.reason or "unknown", self.logger)
+ return None
+ finally:
+ self.close()
+
+ def close(self) -> None:
+ self.invalidate(self.reason or "closed")
+ self._join()
+ self.chunks = []
+ try:
+ os.unlink(self.path)
+ except OSError:
+ pass
+
+
+def _mp4_frame_count(data: bytes, size: Any) -> Optional[int]:
+ """Video packets in ``data`` when its video stream is ``size`` (H, W); None when it does not demux."""
+ if not data:
+ return None
+ try:
+ import io
+
+ import av
+ with av.open(io.BytesIO(data), mode = "r", format = "mp4") as container:
+ stream = container.streams.video[0]
+ if (stream.codec_context.height, stream.codec_context.width) != tuple(
+ int(x) for x in size
+ ):
+ return None
+ return sum(1 for packet in container.demux(stream) if packet.size)
+ except Exception: # noqa: BLE001 - an unreadable file is never used
+ return None
+
+
+_offered = threading.local()
+
+
+def offer_stream(session: Optional[StreamEncode]) -> Iterator[None]:
+ """Make ``session`` the mp4 source for ``VideoBackend._encode_mp4`` calls on this thread.
+
+ The export stays the single entry point, so whatever wraps ``_encode_mp4`` still sees every clip."""
+ previous = getattr(_offered, "session", None)
+ _offered.session = session
+ try:
+ yield
+ finally:
+ _offered.session = previous
+
+
+def claim_stream(video_frames: Any, fps: int) -> Optional[bytes]:
+ """The offered session's mp4 for exactly these frames, once; None (encode normally) otherwise."""
+ session = getattr(_offered, "session", None)
+ if session is None:
+ return None
+ _offered.session = None
+ return session.take(video_frames, fps)
+
+
+def open_stream_encode(
+ pipe: Any,
+ fps: int,
+ *,
+ engage: bool,
+ logger: Any = None,
+) -> Optional[StreamEncode]:
+ """A session for a resident CUDA ``AutoencoderKLWan`` with libx264 frame threads, or None."""
+ if not engage or not stream_encode_enabled():
+ return None
+ from .video_encode import frame_threads_enabled
+ from .video_nvenc import nvenc_requested
+
+ if not frame_threads_enabled() or nvenc_requested():
+ return None
+ vae = getattr(pipe, "vae", None)
+ processor = getattr(pipe, "video_processor", None)
+ if type(vae).__name__ != "AutoencoderKLWan" or not callable(
+ getattr(processor, "postprocess", None)
+ ):
+ return None
+ decoder = getattr(vae, "decoder", None)
+ if not callable(getattr(decoder, "register_forward_hook", None)):
+ return None
+ try:
+ import av # noqa: F401
+ import torch
+
+ if not torch.cuda.is_available() or next(decoder.parameters()).device.type != "cuda":
+ return None
+ return StreamEncode(vae, processor, fps, logger)
+ except Exception: # noqa: BLE001 - no PyAV, no CUDA or no parameters: encode afterwards
+ return None
diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py
index 7c0c8990d42..31be1b67a2f 100644
--- a/studio/backend/tests/test_video_backend.py
+++ b/studio/backend/tests/test_video_backend.py
@@ -2274,6 +2274,46 @@ def test_wan_ti2v_does_not_thread_cfg2(fake_runtime):
assert "guidance_scale_2" not in call
+_REAL_ENCODE_MP4 = VideoBackend.__dict__["_encode_mp4"].__func__
+
+
+def test_a_streamed_clip_still_goes_through_encode_mp4(fake_runtime, monkeypatch):
+ # The streamed mp4 is handed out by _encode_mp4 itself, so a wrapper around it (the in-process bench captures the
+ # frames there) still sees every clip; the session is consumed once and closed after the render.
+ import core.inference.video as vid
+
+ class _Session:
+ def __init__(self):
+ self.took, self.closed = [], 0
+
+ def hooks(self):
+ return contextlib.nullcontext()
+
+ def take(self, frames, fps):
+ self.took.append(fps)
+ return b"STREAMED"
+
+ def close(self):
+ self.closed += 1
+
+ session = _Session()
+ monkeypatch.setattr(vid, "open_stream_encode", lambda *_a, **_k: session)
+ seen = []
+
+ def grab(frames, fps, audio, pipe, **kwargs):
+ data = _REAL_ENCODE_MP4(frames, fps, audio, pipe, **kwargs)
+ seen.append((frames, fps))
+ return data
+
+ monkeypatch.setattr(VideoBackend, "_encode_mp4", staticmethod(grab))
+ backend = VideoBackend()
+ backend.load_pipeline("Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline")
+ res = backend.generate(prompt = "a sloth")
+ assert res["mp4_bytes"] == b"STREAMED"
+ assert len(seen) == 1 and seen[0][0] is not None
+ assert session.took == [seen[0][1]] and session.closed == 1
+
+
def test_wan_a14b_dual_dit_pipeline_loads(fake_runtime):
# The A14B repo builds a dual-DiT MoE pipeline (transformer + transformer_2).
backend = VideoBackend()
diff --git a/studio/backend/tests/test_video_stream_encode.py b/studio/backend/tests/test_video_stream_encode.py
new file mode 100644
index 00000000000..555d9afb03a
--- /dev/null
+++ b/studio/backend/tests/test_video_stream_encode.py
@@ -0,0 +1,195 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The streamed encode must hand back exactly the frames the pipeline exports, and fall back otherwise."""
+
+from __future__ import annotations
+
+import os
+
+import numpy as np
+import pytest
+
+torch = pytest.importorskip("torch")
+av = pytest.importorskip("av")
+diffusers = pytest.importorskip("diffusers")
+
+from diffusers import AutoencoderKLWan # noqa: E402
+from diffusers.video_processor import VideoProcessor # noqa: E402
+
+from core.inference import video_stream_encode as vse # noqa: E402
+from core.inference.video_frames import uint8_video_frames # noqa: E402
+
+
[email protected](autouse = True)
+def _few_threads():
+ # The tiny convs run in milliseconds on a few threads and minutes on a loaded many-core host.
+ threads = torch.get_num_threads()
+ torch.set_num_threads(min(4, threads))
+ yield
+ torch.set_num_threads(threads)
+
+
+def _vae(patch = False):
+ torch.manual_seed(0)
+ kw = dict(
+ base_dim = 16,
+ z_dim = 4,
+ dim_mult = [1, 1, 1, 1],
+ num_res_blocks = 1,
+ temperal_downsample = [False, True, True],
+ latents_mean = [0.0] * 4,
+ latents_std = [1.0] * 4,
+ )
+ if patch:
+ kw.update(
+ decoder_base_dim = 16, is_residual = True, in_channels = 12, out_channels = 12, patch_size = 2
+ )
+ vae = AutoencoderKLWan(**kw).eval()
+ vae.disable_tiling()
+ return vae
+
+
+class _Pipe:
+ def __init__(self, vae):
+ self.vae = vae
+ self.video_processor = VideoProcessor(vae_scale_factor = 8)
+
+ def __call__(self, z):
+ # WanPipeline's tail: decode, then postprocess_video to np.
+ video = self.vae.decode(z, return_dict = False)[0]
+ return self.video_processor.postprocess_video(video, output_type = "np")[0]
+
+
+def _render(pipe, session, z):
+ with torch.inference_mode(), uint8_video_frames(pipe), session.hooks():
+ return pipe(z)
+
+
+def _mp4_frames(data, tmp_path):
+ path = tmp_path / "out.mp4"
+ path.write_bytes(data)
+ with av.open(str(path)) as c:
+ return [f.to_ndarray(format = "rgb24") for f in c.decode(video = 0)]
+
+
[email protected]("patch", [False, True])
+def test_streamed_frames_are_the_exported_frames(patch, tmp_path):
+ pipe = _Pipe(_vae(patch))
+ z = torch.randn(1, 4, 4, 8, 8)
+ with torch.inference_mode():
+ reference = pipe.video_processor.postprocess_video(
+ pipe.vae.decode(z, return_dict = False)[0], output_type = "np"
+ )[0]
+ session = vse.StreamEncode(pipe.vae, pipe.video_processor, 8)
+ frames = _render(pipe, session, z)
+ assert isinstance(frames, torch.Tensor) and frames.dtype == torch.uint8
+ # 1 + 3 x 4 frames, streamed as 4 decoder chunks
+ assert len(session.chunks) == 4 and len(frames) == 13
+ assert np.array_equal(frames.numpy(), (reference * 255).round().astype(np.uint8))
+ before = vse.STATS["streamed"]
+ data = session.take(frames, 8)
+ assert data is not None and vse.STATS["streamed"] == before + 1
+ assert len(_mp4_frames(data, tmp_path)) == 13
+ assert not os.path.exists(session.path)
+ # hooks are removed with the session
+ assert "decode" not in vars(pipe.vae) and not pipe.vae.decoder._forward_hooks
+
+
+def test_tiled_decode_falls_back():
+ vae = _vae()
+ vae.enable_tiling(tile_sample_min_height = 32, tile_sample_min_width = 32)
+ pipe = _Pipe(vae)
+ session = vse.StreamEncode(vae, pipe.video_processor, 8)
+ frames = _render(pipe, session, torch.randn(1, 4, 2, 8, 8))
+ assert session.take(frames, 8) is None
+ assert vse.STATS["last"] == "tiled decode"
+ assert not os.path.exists(session.path)
+
+
+def test_any_differing_byte_or_fps_falls_back():
+ pipe = _Pipe(_vae())
+ z = torch.randn(1, 4, 3, 8, 8)
+ session = vse.StreamEncode(pipe.vae, pipe.video_processor, 8)
+ frames = _render(pipe, session, z).clone()
+ frames[5, 3, 3, 1] ^= 1
+ assert session.take(frames, 8) is None and vse.STATS["last"] == "frames differ"
+ session = vse.StreamEncode(pipe.vae, pipe.video_processor, 8)
+ frames = _render(pipe, session, z)
+ assert session.take(frames, 16) is None and vse.STATS["last"] == "fps differs"
+ session = vse.StreamEncode(pipe.vae, pipe.video_processor, 8)
+ frames = _render(pipe, session, z)
+ assert session.take(frames[:-1], 8) is None and vse.STATS["last"] == "frame count differs"
+
+
+def test_a_restarted_decode_falls_back():
+ # The fp16 decode reruns in fp32 when its output is not finite: the first pass's chunks are stale.
+ pipe = _Pipe(_vae())
+ original = pipe.vae.decode
+
+ def decode_twice(z, *args, **kwargs):
+ original(z, *args, **kwargs)
+ return original(z, *args, **kwargs)
+
+ pipe.vae.decode = decode_twice
+ session = vse.StreamEncode(pipe.vae, pipe.video_processor, 8)
+ frames = _render(pipe, session, torch.randn(1, 4, 2, 8, 8))
+ assert session.take(frames, 8) is None and vse.STATS["last"] == "decoder restarted"
+ assert pipe.vae.decode is decode_twice
+
+
+def test_failure_inside_the_render_closes_the_session():
+ pipe = _Pipe(_vae())
+ session = vse.StreamEncode(pipe.vae, pipe.video_processor, 8)
+ with pytest.raises(RuntimeError):
+ with torch.inference_mode(), uint8_video_frames(pipe), session.hooks():
+ pipe(torch.randn(1, 4, 2, 8, 8))
+ raise RuntimeError("cancelled")
+ assert not session.valid and session._thread is None
+ assert not os.path.exists(session.path)
+
+
+def test_engagement_gates(monkeypatch):
+ pipe = _Pipe(_vae())
+ monkeypatch.setenv(vse.STREAM_ENCODE_ENV, "0")
+ assert vse.open_stream_encode(pipe, 8, engage = True) is None
+ monkeypatch.delenv(vse.STREAM_ENCODE_ENV)
+ assert vse.open_stream_encode(pipe, 8, engage = False) is None
+ monkeypatch.setenv("UNSLOTH_STUDIO_VIDEO_FRAME_THREADS", "0")
+ assert vse.open_stream_encode(pipe, 8, engage = True) is None
+ monkeypatch.delenv("UNSLOTH_STUDIO_VIDEO_FRAME_THREADS")
+ # a CPU decoder never engages; the hook is only worth it beside a GPU decode
+ assert vse.open_stream_encode(pipe, 8, engage = True) is None
+
+
+def test_a_streamed_file_that_does_not_hold_the_clip_is_never_used(monkeypatch):
+ pipe = _Pipe(_vae())
+ session = vse.StreamEncode(pipe.vae, pipe.video_processor, 8)
+ frames = _render(pipe, session, torch.randn(1, 4, 3, 8, 8))
+ join = session._join
+
+ def join_then_truncate():
+ join()
+ with open(session.path, "r+b") as f:
+ f.truncate(os.path.getsize(session.path) // 2)
+
+ monkeypatch.setattr(session, "_join", join_then_truncate)
+ assert session.take(frames, 8) is None
+ assert vse.STATS["last"].startswith("streamed file holds")
+
+
+def test_encode_mp4_claims_an_offered_stream_once():
+ from core.inference.video import VideoBackend
+
+ pipe = _Pipe(_vae())
+ session = vse.StreamEncode(pipe.vae, pipe.video_processor, 8)
+ frames = _render(pipe, session, torch.randn(1, 4, 3, 8, 8))
+ before = vse.STATS["streamed"]
+ with vse.offer_stream(session):
+ data = VideoBackend._encode_mp4(frames, 8, None, None)
+ assert vse.STATS["streamed"] == before + 1
+ # a second export in the same scope encodes normally
+ again = VideoBackend._encode_mp4(frames, 8, None, None)
+ assert vse._mp4_frame_count(data, frames.shape[1:3]) == len(frames)
+ assert vse._mp4_frame_count(again, frames.shape[1:3]) == len(frames)
+ assert vse.STATS["streamed"] == before + 1