Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: encode video mp4 with x264 frame threads (export 1.25 to 2.5x faster, same settings) (#12648)
Daniel Han65629d0987af
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 95a6716264f..b9ad66468ca 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -223,6 +223,7 @@ from .video_minimax_h3_te import (h3_te_quant_scheme,h3_te_resident_gb,)+from .video_encode import encode_x264from .video_nvenc import encode_nvenc, nvenc_gpufrom utils.hardware import clear_gpu_cache@@ -9359,7 +9360,14 @@ class VideoBackend:encode_kwargs.get("audio"),encode_kwargs.get("audio_sample_rate"),):- encode_video(video_frames, fps, tmp.name, **encode_kwargs)+ if not encode_x264(+ video_frames,+ fps,+ tmp.name,+ encode_kwargs.get("audio"),+ encode_kwargs.get("audio_sample_rate"),+ ):+ encode_video(video_frames, fps, tmp.name, **encode_kwargs)return Path(tmp.name).read_bytes()finally:try:diff --git a/studio/backend/core/inference/video_encode.py b/studio/backend/core/inference/video_encode.pynew file mode 100644index 00000000000..304d3441e7a--- /dev/null+++ b/studio/backend/core/inference/video_encode.py@@ -0,0 +1,61 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""diffusers' ``encode_video`` stream (libx264 defaults, yuv420p, same audio helpers) with x264 frame threads:+PyAV opens codecs with ``thread_type = SLICE``, so every frame waits for its slowest slice.+``UNSLOTH_STUDIO_VIDEO_FRAME_THREADS=0`` restores the diffusers call."""++from __future__ import annotations++import os+from typing import Any++FRAME_THREADS_ENV = "UNSLOTH_STUDIO_VIDEO_FRAME_THREADS"+++def frame_threads_enabled() -> bool:+ return os.environ.get(FRAME_THREADS_ENV, "").strip().lower() not in ("0", "false", "no", "off")+++def encode_x264(+ video: Any,+ fps: int,+ path: str,+ audio: Any = None,+ audio_sample_rate: Any = None,+) -> bool:+ """Write ``path`` with frame-threaded libx264; False (caller runs ``encode_video``) when anything is unsure."""+ if not frame_threads_enabled():+ return False+ from .video_nvenc import _uint8_frames++ frames = _uint8_frames(video)+ if frames is None or frames.ndim != 4 or frames.shape[-1] != 3 or len(frames) == 0:+ return False+ if audio is not None and audio_sample_rate is None:+ return False+ try:+ import av+ from diffusers.utils.export_utils import _prepare_audio_stream, _write_audio+ except Exception: # noqa: BLE001 - no PyAV or an older diffusers: the stock exporter decides+ return False+ try:+ with av.open(path, mode = "w", format = "mp4") as container:+ stream = container.add_stream("libx264", rate = int(fps))+ stream.width = int(frames.shape[2])+ stream.height = int(frames.shape[1])+ stream.pix_fmt = "yuv420p"+ stream.codec_context.thread_type = "FRAME"+ audio_stream = (+ _prepare_audio_stream(container, audio_sample_rate) if audio is not None else None+ )+ for frame in frames:+ for packet in stream.encode(av.VideoFrame.from_ndarray(frame, format = "rgb24")):+ container.mux(packet)+ for packet in stream.encode():+ container.mux(packet)+ if audio_stream is not None:+ _write_audio(container, audio_stream, audio, audio_sample_rate, av)+ return True+ except Exception: # noqa: BLE001 - a partial file is overwritten by the stock exporter+ return Falsediff --git a/studio/backend/tests/test_video_encode.py b/studio/backend/tests/test_video_encode.pynew file mode 100644index 00000000000..62f7405913b--- /dev/null+++ b/studio/backend/tests/test_video_encode.py@@ -0,0 +1,162 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+++from __future__ import annotations++import re++import numpy as np+import pytest++torch = pytest.importorskip("torch")+av = pytest.importorskip("av")+eu = pytest.importorskip("diffusers.utils.export_utils")++from core.inference import video_encode as ve # noqa: E402++# x264 options that only describe threading; every other field must match diffusers' encode_video.+_THREADING = {"threads", "lookahead_threads", "sliced_threads", "slices", "sync_lookahead"}+++def _clip(+ frames = 24,+ height = 128,+ width = 192,+):+ yy, xx = np.mgrid[0:height, 0:width]+ return np.stack(+ [+ np.stack([(xx + 4 * t) % 256, (yy + 2 * t) % 256, (xx + yy) % 256], -1)+ for t in range(frames)+ ]+ ).astype(np.uint8)+++def _x264_options(path):+ data = open(path, "rb").read()+ match = re.search(rb"x264 - core .*? options: ([^\x00]+?)\x00", data, re.S)+ assert match, "no x264 SEI"+ return dict(kv.split("=", 1) for kv in match.group(1).decode().split() if "=" in kv)+++def _decode(path):+ with av.open(path) as container:+ streams = {s.type: s for s in container.streams}+ video = streams["video"]+ info = (+ video.codec_context.name,+ video.codec_context.pix_fmt,+ video.average_rate,+ set(streams),+ )+ frames = np.stack([f.to_ndarray(format = "rgb24") for f in container.decode(video = 0)])+ return info, frames+++def _psnr(a, b):+ mse = np.mean((a.astype(np.float64) - b.astype(np.float64)) ** 2)+ return 10 * np.log10(255**2 / mse)++[email protected]("with_audio", [False, True])+def test_same_stream_and_settings_as_encode_video(tmp_path, monkeypatch, with_audio):+ monkeypatch.delenv(ve.FRAME_THREADS_ENV, raising = False)+ clip = _clip()+ audio = torch.zeros(2, 48000) if with_audio else None+ rate = 48000 if with_audio else None+ ours, stock = str(tmp_path / "ours.mp4"), str(tmp_path / "stock.mp4")+ assert ve.encode_x264(torch.from_numpy(clip), 24, ours, audio, rate)+ eu.encode_video(+ torch.from_numpy(clip),+ 24,+ stock,+ **({"audio": audio, "audio_sample_rate": rate} if with_audio else {}),+ )+ info_ours, frames_ours = _decode(ours)+ info_stock, frames_stock = _decode(stock)+ assert info_ours == info_stock+ assert info_ours[0] == "h264" and info_ours[1] == "yuv420p"+ assert ("audio" in info_ours[3]) == with_audio+ assert frames_ours.shape == frames_stock.shape == clip.shape+ assert _psnr(frames_ours, clip) > 30+ opts_ours, opts_stock = _x264_options(ours), _x264_options(stock)+ assert opts_ours["sliced_threads"] == "0" and opts_ours["crf"] == "23.0"+ keys = (set(opts_ours) | set(opts_stock)) - _THREADING+ assert {k: opts_ours.get(k) for k in keys} == {k: opts_stock.get(k) for k in keys}+++def test_float_pil_and_uint8_inputs_encode_the_same_frames(tmp_path):+ from PIL import Image++ clip = _clip(frames = 8)+ inputs = {+ "u8": torch.from_numpy(clip),+ "np": clip.astype(np.float32) / 255,+ "pil": [Image.fromarray(f) for f in clip],+ }+ decoded = {}+ for name, value in inputs.items():+ path = str(tmp_path / f"{name}.mp4")+ assert ve.encode_x264(value, 24, path)+ decoded[name] = _decode(path)[1]+ assert np.array_equal(decoded["u8"], decoded["np"]) and np.array_equal(+ decoded["u8"], decoded["pil"]+ )++[email protected]("value", ["0", "false", "off"])+def test_kill_switch(tmp_path, monkeypatch, value):+ monkeypatch.setenv(ve.FRAME_THREADS_ENV, value)+ assert ve.encode_x264(_clip(frames = 2), 24, str(tmp_path / "x.mp4")) is False+++def test_unsure_input_and_failures_hand_back(tmp_path, monkeypatch):+ path = str(tmp_path / "x.mp4")+ # Out of range float: encode_video's own warn-and-use-as-is branch decides.+ assert ve.encode_x264(np.full((2, 16, 16, 3), 2.0, dtype = np.float32), 24, path) is False+ assert ve.encode_x264(np.zeros((2, 16, 16, 4), dtype = np.uint8), 24, path) is False+ assert (+ ve.encode_x264(np.zeros((2, 16, 16, 3), dtype = np.uint8), 24, path, torch.zeros(2, 10))+ is False+ )++ def _boom(*args, **kwargs):+ raise RuntimeError("encoder open failed")++ monkeypatch.setattr(av, "open", _boom)+ assert ve.encode_x264(np.zeros((2, 16, 16, 3), dtype = np.uint8), 24, path) is False++[email protected]("x264_ok, expect", [(True, ["frame"]), (False, ["frame", "stock"])])+def test_encode_mp4_routes_to_frame_threads_first(monkeypatch, x264_ok, expect):+ from core.inference import video as video_mod++ used: list = []++ def _stock(frames, fps, path, **kwargs):+ used.append("stock")+ open(path, "wb").write(b"stock")++ def _frame(frames, fps, path, audio, rate):+ used.append("frame")+ if x264_ok:+ open(path, "wb").write(b"frame")+ return x264_ok++ monkeypatch.setattr(eu, "encode_video", _stock)+ monkeypatch.setattr(video_mod, "nvenc_gpu", lambda logger = None: None)+ monkeypatch.setattr(video_mod, "encode_x264", _frame)+ out = video_mod.VideoBackend._encode_mp4(+ np.zeros((2, 16, 16, 3), dtype = np.float32), 24, None, None+ )+ assert used == expect and out == (b"frame" if x264_ok else b"stock")+++def test_encode_mp4_end_to_end_default_is_frame_threaded(monkeypatch):+ from core.inference import video as video_mod++ monkeypatch.delenv(ve.FRAME_THREADS_ENV, raising = False)+ monkeypatch.setattr(video_mod, "nvenc_gpu", lambda logger = None: None)+ data = video_mod.VideoBackend._encode_mp4(torch.from_numpy(_clip()), 24, None, None)+ assert b"sliced_threads=0" in datadiff --git a/studio/backend/tests/test_video_nvenc.py b/studio/backend/tests/test_video_nvenc.pyindex 23e52534bd7..f789960b12a 100644--- a/studio/backend/tests/test_video_nvenc.py+++ b/studio/backend/tests/test_video_nvenc.py@@ -146,6 +146,7 @@ def test_encode_mp4_falls_back_to_libx264(monkeypatch, gpu, nvenc_ok, expect):monkeypatch.setattr(eu, "encode_video", _x264)monkeypatch.setattr(video_mod, "nvenc_gpu", lambda logger = None: gpu)monkeypatch.setattr(video_mod, "encode_nvenc", _nvenc)+ monkeypatch.setattr(video_mod, "encode_x264", lambda *args: False)out = video_mod.VideoBackend._encode_mp4(np.zeros((2, 16, 16, 3), dtype = np.float32), 24, None, None)