Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio video: denormalize decoded frames slice by slice in the uint8 export (#16748)

Daniel Han3fc6b699e65e
diff --git a/studio/backend/core/inference/video_frames.py b/studio/backend/core/inference/video_frames.py
index 0383365ec76..02f202ad0bf 100644
--- a/studio/backend/core/inference/video_frames.py
+++ b/studio/backend/core/inference/video_frames.py
@@ -16,8 +16,9 @@ _FINITE_SLICE_ELEMS = 16 << 20
NONFINITE_CHECK_ENV = "UNSLOTH_VIDEO_NONFINITE_CHECK"
-def device_uint8(frames: Any) -> Optional[Any]:
- """(F, C, H, W) float frames in [0, 1] -> CPU uint8 (F, H, W, C); None when the fast path does not apply."""
+def device_uint8(frames: Any, prepare: Any = None) -> Optional[Any]:
+ """(F, C, H, W) float frames in [0, 1] -> CPU uint8 (F, H, W, C); None when the fast path does not apply.
+ ``prepare(piece, start)`` maps each raw slice first, keeping float temporaries slice-sized."""
import torch
device = frames.device
@@ -40,7 +41,10 @@ def device_uint8(frames: Any) -> Optional[Any]:
in_range = None
try:
for start in range(0, count, step):
- piece = frames[start : start + step].to(torch.float32)
+ piece = frames[start : start + step]
+ if prepare is not None:
+ piece = prepare(piece, start)
+ piece = piece.to(torch.float32)
# Range-checked per slice: a whole-clip reduction over the permuted view copies the clip first.
lo, hi = torch.aminmax(piece)
fits = (lo >= 0) & (hi <= 1)
@@ -175,6 +179,35 @@ def finite_video_decode(pipe: Any, attrs: "tuple[str, ...]" = ("vae",)) -> Itera
pass
+_FRAME_WISE_METHODS = ("postprocess", "_denormalize_conditionally", "denormalize")
+
+
+def _frame_wise_prepare(proc: Any, postprocess: Any, frames: Any, kwargs: dict) -> Any:
+ """Per-slice ``postprocess(slice, "pt")`` prepare, or None. Only diffusers' own unoverridden VaeImageProcessor
+ methods are frame-wise; a subclass or instance override may mix frames, so it keeps the whole-clip call."""
+ try:
+ from diffusers.image_processor import VaeImageProcessor
+ except Exception: # noqa: BLE001
+ return None
+ own = vars(proc) if hasattr(proc, "__dict__") else {}
+ for name in _FRAME_WISE_METHODS:
+ if name in own or getattr(type(proc), name, None) is not getattr(
+ VaeImageProcessor, name, None
+ ):
+ return None
+ if set(kwargs) - {"do_denormalize"}:
+ return None
+ flags = kwargs.get("do_denormalize")
+ if flags is None:
+ return lambda piece, _start: postprocess(piece, "pt")
+ if not isinstance(flags, (list, tuple)) or len(flags) < frames.shape[0]:
+ return None
+ flags = list(flags)
+ return lambda piece, start: postprocess(
+ piece, "pt", do_denormalize = flags[start : start + piece.shape[0]]
+ )
+
+
@contextlib.contextmanager
def uint8_video_frames(pipe: Any) -> Iterator[None]:
"""While active, ``pipe.video_processor``'s np / pil ``postprocess_video`` returns CPU uint8 (B, F, H, W, C), and
@@ -207,7 +240,12 @@ def uint8_video_frames(pipe: Any) -> Iterator[None]:
clips = []
for batch in range(video.shape[0]):
# "pt" stops right after the denormalize the np / pil paths apply, on the same view they use.
- clip = device_uint8(postprocess(video[batch].permute(1, 0, 2, 3), "pt", **kwargs))
+ frames = video[batch].permute(1, 0, 2, 3)
+ prepare = _frame_wise_prepare(proc, postprocess, frames, kwargs)
+ if prepare is not None:
+ clip = device_uint8(frames, prepare)
+ else:
+ clip = device_uint8(postprocess(frames, "pt", **kwargs))
if clip is None:
return original(video, output_type, **kwargs)
clips.append(clip)
diff --git a/studio/backend/tests/test_video_frames.py b/studio/backend/tests/test_video_frames.py
index e1de5e06c11..5267099245d 100644
--- a/studio/backend/tests/test_video_frames.py
+++ b/studio/backend/tests/test_video_frames.py
@@ -145,3 +145,81 @@ def test_a_card_too_full_for_a_slice_takes_the_original_postprocess(monkeypatch)
monkeypatch.setattr(torch, "aminmax", _oom)
got = _run(vp, video, "np")
assert isinstance(got, np.ndarray) and np.array_equal(got, expected)
+
+
+def _denormalize_spy(monkeypatch):
+ # Patched on VaeImageProcessor itself, so the processor still resolves to diffusers' own frame-wise method.
+ seen = []
+ original = VaeImageProcessor._denormalize_conditionally
+
+ def denormalize(
+ self,
+ images,
+ do_denormalize = None,
+ ):
+ seen.append(int(images.shape[0]))
+ return original(self, images, do_denormalize)
+
+ monkeypatch.setattr(VaeImageProcessor, "_denormalize_conditionally", denormalize)
+ return seen
+
+
[email protected]("dtype", [torch.float32, torch.float16])
[email protected]("output_type", ["np", "pil"])
+def test_denormalize_runs_per_slice_not_on_the_whole_clip(dtype, output_type, monkeypatch):
+ import core.inference.video_frames as vf
+
+ vp = VideoProcessor(vae_scale_factor = 8, do_normalize = True)
+ # 9 frames in slices of 2: odd tail, two clips in the batch
+ video = torch.cat([_clip(dtype, normalized = True, frames = 9, seed = s) for s in (5, 6)])
+ expected = _encode_video_uint8(vp.postprocess_video(video, output_type = "np"))
+ monkeypatch.setattr(vf, "_SLICE_BYTES", 2 * 3 * 16 * 24 * 4)
+ seen = _denormalize_spy(monkeypatch)
+ got = _run(vp, video, output_type)
+ assert np.array_equal(got.numpy(), expected)
+ assert seen and max(seen) <= 2 and sum(seen) == 18
+
+
+def test_per_frame_do_denormalize_list_is_sliced_with_the_frames(monkeypatch):
+ import core.inference.video_frames as vf
+
+ vp = VideoProcessor(vae_scale_factor = 8, do_normalize = True)
+ video = _clip(torch.float32, normalized = True, frames = 7)
+ flags = [True, False, True, True, False, False, True]
+ video[0, :, [1, 4, 5]] = video[0, :, [1, 4, 5]].clamp(
+ 0, 1
+ ) # undenormalized frames stay in range
+ expected = _encode_video_uint8(
+ vp.postprocess_video(video, output_type = "np", do_denormalize = flags)
+ )
+ monkeypatch.setattr(vf, "_SLICE_BYTES", 3 * 3 * 16 * 24 * 4)
+ seen = _denormalize_spy(monkeypatch)
+ with uint8_video_frames(types.SimpleNamespace(video_processor = vp)):
+ got = vp.postprocess_video(video, output_type = "np", do_denormalize = flags)
+ assert isinstance(got, torch.Tensor) and np.array_equal(got.numpy(), expected)
+ assert seen == [3, 3, 1]
+
+
+def test_a_processor_that_overrides_postprocess_keeps_the_whole_clip_call(monkeypatch):
+ calls = []
+
+ class Mixing(VideoProcessor):
+ def postprocess(
+ self,
+ image,
+ output_type = "pil",
+ **kwargs,
+ ):
+ calls.append(int(image.shape[0]))
+ return super().postprocess(image, output_type, **kwargs)
+
+ vp = Mixing(vae_scale_factor = 8, do_normalize = True)
+ video = _clip(torch.float32, normalized = True, frames = 5)
+ expected = _encode_video_uint8(VideoProcessor.postprocess_video(vp, video, output_type = "np"))
+ calls.clear()
+ import core.inference.video_frames as vf
+
+ monkeypatch.setattr(vf, "_SLICE_BYTES", 3 * 16 * 24 * 4)
+ got = _run(vp, video, "np")
+ assert np.array_equal(got.numpy(), expected)
+ assert calls == [5]