Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: convert decoded video frames to uint8 on the GPU before the mp4 encode (#11879)
Daniel Hanff8b04fbc4e4
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 786dc1b210b..ce28b480c4d 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -143,6 +143,7 @@ from .video_families import (video_family_prequant_repo,video_family_prequant_schemes,)+from .video_frames import uint8_video_framesfrom .video_minimax_h3 import (H3_ANCHOR_FIRST,H3_ANCHOR_LAST,@@ -6280,6 +6281,8 @@ class VideoBackend:# Family-agnostic: no video family has a callback between its denoise loop and its decode, so# every one of them gets its decode phase from the decoder itself.stack.enter_context(_decode_phase(pipe, _on_decode))+ # The decoded clip becomes uint8 on the GPU, bit-identical to the np / pil export (video_frames).+ stack.enter_context(uint8_video_frames(pipe))stack.enter_context(_completed_step_poller(_pump))yielddiff --git a/studio/backend/core/inference/video_frames.py b/studio/backend/core/inference/video_frames.pynew file mode 100644index 00000000000..3ff4a2be740--- /dev/null+++ b/studio/backend/core/inference/video_frames.py@@ -0,0 +1,99 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Convert a decoded clip to uint8 RGB frames on its own device, bit-identical to the np / pil export+(float32 ``(x * 255).round()`` matches numpy); out-of-range clips or non CUDA / CPU devices use the original path."""++from __future__ import annotations++import contextlib+from typing import Any, Iterator, Optional++_SLICE_BYTES = 64 << 20+++def device_uint8(frames: Any) -> Optional[Any]:+ """(F, C, H, W) float frames in [0, 1] -> CPU uint8 (F, H, W, C); None when the fast path does not apply."""+ import torch++ device = frames.device+ if device.type not in ("cuda", "cpu") or frames.numel() == 0:+ return None+ count, channels, height, width = frames.shape+ shape = (count, height, width, channels)+ host = None+ if device.type == "cuda":+ try:+ host = torch.empty(shape, dtype = torch.uint8, device = "cpu", pin_memory = True)+ except Exception: # noqa: BLE001 - pinned allocation refused (host limits); a pageable copy is still exact+ host = None+ pinned = host is not None+ if host is None:+ host = torch.empty(shape, dtype = torch.uint8, device = "cpu")+ step = max(1, _SLICE_BYTES // max(1, channels * height * width * 4))+ in_range = None+ try:+ for start in range(0, count, step):+ piece = frames[start : start + step].to(torch.float32)+ # Range-checked per slice: a whole-clip reduction over the permuted view copies the clip first.+ lo, hi = torch.aminmax(piece)+ fits = (lo >= 0) & (hi <= 1)+ in_range = fits if in_range is None else in_range & fits+ piece = (piece * 255).round_().to(torch.uint8).permute(0, 2, 3, 1)+ host[start : start + step].copy_(piece, non_blocking = pinned)+ if pinned:+ torch.cuda.current_stream(device).synchronize()+ except torch.cuda.OutOfMemoryError:+ # np / pil copy straight to host, so fall back rather than fail a render.+ return None+ return host if bool(in_range) else None+++def uint8_video_frames(pipe: Any) -> Iterator[None]:+ """While active, ``pipe.video_processor``'s np / pil ``postprocess_video`` returns CPU uint8 (B, F, H, W, C)."""+ proc = getattr(pipe, "video_processor", None)+ original = getattr(proc, "postprocess_video", None)+ postprocess = getattr(proc, "postprocess", None)+ if not callable(original) or not callable(postprocess):+ yield+ return+ import torch++ had_own = "postprocess_video" in vars(proc)++ def postprocess_video(+ video: Any,+ output_type: str = "np",+ **kwargs: Any,+ ) -> Any:+ if (+ output_type not in ("np", "pil")+ or not isinstance(video, torch.Tensor)+ or video.ndim != 5+ ):+ return original(video, output_type, **kwargs)+ clips = []+ for batch in range(video.shape[0]):+ # "pt" stops right after the denormalize the np / pil paths apply, on the same view they use.+ clip = device_uint8(postprocess(video[batch].permute(1, 0, 2, 3), "pt", **kwargs))+ if clip is None:+ return original(video, output_type, **kwargs)+ clips.append(clip)+ return clips[0].unsqueeze(0) if len(clips) == 1 else torch.stack(clips)++ try:+ proc.postprocess_video = postprocess_video+ except Exception: # noqa: BLE001 - a processor that refuses instance attributes keeps its own path+ yield+ return+ try:+ yield+ finally:+ if had_own:+ proc.postprocess_video = original+ else:+ try:+ del proc.postprocess_video+ except AttributeError:+ passdiff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.pyindex 5a8df661b71..70f254cdebc 100644--- a/studio/backend/tests/test_video_backend.py+++ b/studio/backend/tests/test_video_backend.py@@ -9,6 +9,7 @@ stack loads."""import builtinsimport contextlibimport dataclasses+import functoolsimport sysimport threadingimport time@@ -4500,6 +4501,45 @@ def test_h3_modular_generation_ticks_and_cancels_through_the_scheduler(fake_runtassert pipe.scheduler.step.__func__ is _FakeH3Scheduler.step+def test_the_pipeline_call_runs_with_uint8_frames_on_every_family(+ fake_runtime, tmp_path, monkeypatch+):+ active: list = []++ @contextlib.contextmanager+ def _frames(pipe):+ active.append(pipe)+ try:+ yield+ finally:+ active.remove(pipe)++ monkeypatch.setattr("core.inference.video.uint8_video_frames", _frames)+ seen_inside: list = []+ for backend, pipe in (+ (_load_ltx23_from_dir(tmp_path), None),+ (VideoBackend(), "h3"),+ ):+ if pipe == "h3":+ pipe = _load_h3_modular(backend)+ else:+ pipe = backend._state.pipe+ original_call = type(pipe).__call__++ def _call(+ self,+ *a,+ _orig = original_call,+ **k,+ ):+ seen_inside.append(list(active) == [self])+ return _orig(self, *a, **k)++ monkeypatch.setattr(type(pipe), "__call__", functools.wraps(original_call)(_call))+ backend.generate(prompt = "a fox", steps = 2)+ assert seen_inside == [True, True] and active == []++def test_h3_native_transcode_is_torch_free_and_keeps_audio(monkeypatch, tmp_path):import ioimport mathdiff --git a/studio/backend/tests/test_video_frames.py b/studio/backend/tests/test_video_frames.pynew file mode 100644index 00000000000..ae1d30627b9--- /dev/null+++ b/studio/backend/tests/test_video_frames.py@@ -0,0 +1,147 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""video_frames must match diffusers' np / pil export exactly, else fall back."""++import types++import numpy as np+import pytest++torch = pytest.importorskip("torch")+pytest.importorskip("diffusers")+from diffusers.image_processor import VaeImageProcessor # noqa: E402+from diffusers.video_processor import VideoProcessor # noqa: E402++from core.inference.video_frames import device_uint8, uint8_video_frames # noqa: E402+++def _clip(+ dtype,+ *,+ normalized,+ frames = 5,+ height = 16,+ width = 24,+ seed = 0,+):+ g = torch.Generator().manual_seed(seed)+ video = torch.rand((1, 3, frames, height, width), generator = g)+ # Exact rounding boundaries (k + 0.5) / 255 plus range ends.+ edge = (+ (torch.arange(height * width) % 256)+ .float()+ .add(0.5)+ .div(255)+ .clamp(0, 1)+ .view(height, width)+ )+ video[0, 0, 0] = edge+ video[0, 1, 0] = 0.0+ video[0, 2, 0] = 1.0+ if normalized:+ video = video * 2 - 1+ return video.to(dtype)+++def _encode_video_uint8(np_frames):+ # diffusers.utils.export_utils.encode_video's np branch.+ return (np_frames * 255).round().astype("uint8")+++def _run(vp, video, output_type):+ with uint8_video_frames(types.SimpleNamespace(video_processor = vp)):+ return vp.postprocess_video(video, output_type = output_type)++[email protected]("dtype", [torch.float32, torch.bfloat16, torch.float16])[email protected]("normalized", [False, True])+def test_matches_the_np_export(dtype, normalized):+ vp = VideoProcessor(vae_scale_factor = 8, do_normalize = normalized)+ video = _clip(dtype, normalized = normalized)+ expected = _encode_video_uint8(vp.postprocess_video(video, output_type = "np"))+ got = _run(vp, video, "np")+ assert got.dtype == torch.uint8 and got.shape == expected.shape+ assert np.array_equal(got.numpy(), expected)+++def test_matches_the_pil_export_for_every_clip_in_a_batch():+ vp = VideoProcessor(vae_scale_factor = 8, do_normalize = True)+ video = torch.cat([_clip(torch.bfloat16, normalized = True, seed = s) for s in (3, 4)])+ expected = np.stack(+ [+ np.stack([np.array(im) for im in clip])+ for clip in vp.postprocess_video(video, output_type = "pil")+ ]+ )+ got = _run(vp, video, "pil")+ assert np.array_equal(got.numpy(), expected)+ assert np.array_equal(got[1].numpy(), expected[1])+++def test_every_float32_near_a_rounding_boundary():+ k = torch.arange(256, dtype = torch.float32)+ bits = torch.cat([(k + 0.5) / 255, k / 255]).clamp(0, 1).view(torch.int32)+ values = torch.cat([bits + d for d in range(-512, 513)]).view(torch.float32)+ values = values[(values >= 0) & (values <= 1)]+ n = values.numel() - values.numel() % 3+ frames = values[:n].view(-1, 3, 1, 1)+ got = device_uint8(frames).reshape(-1).numpy()+ expected = _encode_video_uint8(VaeImageProcessor.pt_to_numpy(frames)).reshape(-1)+ assert np.array_equal(got, expected)++[email protected]("bad", [float("nan"), 1.5, -0.25])[email protected]("output_type", ["np", "pil"])+def test_out_of_range_takes_the_original_postprocess(bad, output_type):+ vp = VideoProcessor(vae_scale_factor = 8, do_normalize = False)+ video = _clip(torch.float32, normalized = False)+ video[0, 0, 1, 2, 3] = bad+ expected = vp.postprocess_video(video, output_type = output_type)+ got = _run(vp, video, output_type)+ if output_type == "np":+ assert isinstance(got, np.ndarray) and np.array_equal(got, expected, equal_nan = True)+ else:+ assert isinstance(got[0][1], type(expected[0][1]))+ assert all(np.array_equal(np.array(a), np.array(b)) for a, b in zip(got[0], expected[0]))+++def test_other_output_types_and_the_processor_are_untouched():+ vp = VideoProcessor(vae_scale_factor = 8, do_normalize = True)+ video = _clip(torch.float32, normalized = True)+ assert torch.equal(_run(vp, video, "pt"), vp.postprocess_video(video, output_type = "pt"))+ assert "postprocess_video" not in vars(vp)+ with uint8_video_frames(types.SimpleNamespace()):+ pass+++def test_restored_when_the_pipeline_raises():+ vp = VideoProcessor(vae_scale_factor = 8)+ with pytest.raises(RuntimeError):+ with uint8_video_frames(types.SimpleNamespace(video_processor = vp)):+ assert "postprocess_video" in vars(vp)+ raise RuntimeError("boom")+ assert "postprocess_video" not in vars(vp)++[email protected](not torch.cuda.is_available(), reason = "needs a CUDA / ROCm device")+def test_cuda_matches_numpy_and_lands_in_pinned_memory():+ vp = VideoProcessor(vae_scale_factor = 8, do_normalize = False)+ video = _clip(torch.float32, normalized = False, frames = 40, height = 64, width = 96)+ expected = _encode_video_uint8(vp.postprocess_video(video, output_type = "np"))+ got = _run(vp, video.cuda(), "np")+ assert got.device.type == "cpu" and got.is_pinned()+ assert np.array_equal(got.numpy(), expected)+++def test_a_card_too_full_for_a_slice_takes_the_original_postprocess(monkeypatch):+ vp = VideoProcessor(vae_scale_factor = 8, do_normalize = False)+ video = _clip(torch.float32, normalized = False)+ expected = vp.postprocess_video(video, output_type = "np")++ def _oom(*args, **kwargs):+ raise torch.cuda.OutOfMemoryError("CUDA out of memory")++ monkeypatch.setattr(torch, "aminmax", _oom)+ got = _run(vp, video, "np")+ assert isinstance(got, np.ndarray) and np.array_equal(got, expected)