Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: encode video mp4 with x264 frame threads (export 1.25 to 2.5x faster, same settings) (#12648)

Daniel Han65629d0987af
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 95a6716264f..b9ad66468ca 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -223,6 +223,7 @@ from .video_minimax_h3_te import (
h3_te_quant_scheme,
h3_te_resident_gb,
)
+from .video_encode import encode_x264
from .video_nvenc import encode_nvenc, nvenc_gpu
from utils.hardware import clear_gpu_cache
@@ -9359,7 +9360,14 @@ class VideoBackend:
encode_kwargs.get("audio"),
encode_kwargs.get("audio_sample_rate"),
):
- encode_video(video_frames, fps, tmp.name, **encode_kwargs)
+ if not encode_x264(
+ video_frames,
+ fps,
+ tmp.name,
+ encode_kwargs.get("audio"),
+ encode_kwargs.get("audio_sample_rate"),
+ ):
+ encode_video(video_frames, fps, tmp.name, **encode_kwargs)
return Path(tmp.name).read_bytes()
finally:
try:
diff --git a/studio/backend/core/inference/video_encode.py b/studio/backend/core/inference/video_encode.py
new file mode 100644
index 00000000000..304d3441e7a
--- /dev/null
+++ b/studio/backend/core/inference/video_encode.py
@@ -0,0 +1,61 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""diffusers' ``encode_video`` stream (libx264 defaults, yuv420p, same audio helpers) with x264 frame threads:
+PyAV opens codecs with ``thread_type = SLICE``, so every frame waits for its slowest slice.
+``UNSLOTH_STUDIO_VIDEO_FRAME_THREADS=0`` restores the diffusers call."""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+FRAME_THREADS_ENV = "UNSLOTH_STUDIO_VIDEO_FRAME_THREADS"
+
+
+def frame_threads_enabled() -> bool:
+ return os.environ.get(FRAME_THREADS_ENV, "").strip().lower() not in ("0", "false", "no", "off")
+
+
+def encode_x264(
+ video: Any,
+ fps: int,
+ path: str,
+ audio: Any = None,
+ audio_sample_rate: Any = None,
+) -> bool:
+ """Write ``path`` with frame-threaded libx264; False (caller runs ``encode_video``) when anything is unsure."""
+ if not frame_threads_enabled():
+ return False
+ from .video_nvenc import _uint8_frames
+
+ frames = _uint8_frames(video)
+ if frames is None or frames.ndim != 4 or frames.shape[-1] != 3 or len(frames) == 0:
+ return False
+ if audio is not None and audio_sample_rate is None:
+ return False
+ try:
+ import av
+ from diffusers.utils.export_utils import _prepare_audio_stream, _write_audio
+ except Exception: # noqa: BLE001 - no PyAV or an older diffusers: the stock exporter decides
+ return False
+ try:
+ with av.open(path, mode = "w", format = "mp4") as container:
+ stream = container.add_stream("libx264", rate = int(fps))
+ stream.width = int(frames.shape[2])
+ stream.height = int(frames.shape[1])
+ stream.pix_fmt = "yuv420p"
+ stream.codec_context.thread_type = "FRAME"
+ audio_stream = (
+ _prepare_audio_stream(container, audio_sample_rate) if audio is not None else None
+ )
+ for frame in frames:
+ for packet in stream.encode(av.VideoFrame.from_ndarray(frame, format = "rgb24")):
+ container.mux(packet)
+ for packet in stream.encode():
+ container.mux(packet)
+ if audio_stream is not None:
+ _write_audio(container, audio_stream, audio, audio_sample_rate, av)
+ return True
+ except Exception: # noqa: BLE001 - a partial file is overwritten by the stock exporter
+ return False
diff --git a/studio/backend/tests/test_video_encode.py b/studio/backend/tests/test_video_encode.py
new file mode 100644
index 00000000000..62f7405913b
--- /dev/null
+++ b/studio/backend/tests/test_video_encode.py
@@ -0,0 +1,162 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+
+from __future__ import annotations
+
+import re
+
+import numpy as np
+import pytest
+
+torch = pytest.importorskip("torch")
+av = pytest.importorskip("av")
+eu = pytest.importorskip("diffusers.utils.export_utils")
+
+from core.inference import video_encode as ve # noqa: E402
+
+# x264 options that only describe threading; every other field must match diffusers' encode_video.
+_THREADING = {"threads", "lookahead_threads", "sliced_threads", "slices", "sync_lookahead"}
+
+
+def _clip(
+ frames = 24,
+ height = 128,
+ width = 192,
+):
+ yy, xx = np.mgrid[0:height, 0:width]
+ return np.stack(
+ [
+ np.stack([(xx + 4 * t) % 256, (yy + 2 * t) % 256, (xx + yy) % 256], -1)
+ for t in range(frames)
+ ]
+ ).astype(np.uint8)
+
+
+def _x264_options(path):
+ data = open(path, "rb").read()
+ match = re.search(rb"x264 - core .*? options: ([^\x00]+?)\x00", data, re.S)
+ assert match, "no x264 SEI"
+ return dict(kv.split("=", 1) for kv in match.group(1).decode().split() if "=" in kv)
+
+
+def _decode(path):
+ with av.open(path) as container:
+ streams = {s.type: s for s in container.streams}
+ video = streams["video"]
+ info = (
+ video.codec_context.name,
+ video.codec_context.pix_fmt,
+ video.average_rate,
+ set(streams),
+ )
+ frames = np.stack([f.to_ndarray(format = "rgb24") for f in container.decode(video = 0)])
+ return info, frames
+
+
+def _psnr(a, b):
+ mse = np.mean((a.astype(np.float64) - b.astype(np.float64)) ** 2)
+ return 10 * np.log10(255**2 / mse)
+
+
[email protected]("with_audio", [False, True])
+def test_same_stream_and_settings_as_encode_video(tmp_path, monkeypatch, with_audio):
+ monkeypatch.delenv(ve.FRAME_THREADS_ENV, raising = False)
+ clip = _clip()
+ audio = torch.zeros(2, 48000) if with_audio else None
+ rate = 48000 if with_audio else None
+ ours, stock = str(tmp_path / "ours.mp4"), str(tmp_path / "stock.mp4")
+ assert ve.encode_x264(torch.from_numpy(clip), 24, ours, audio, rate)
+ eu.encode_video(
+ torch.from_numpy(clip),
+ 24,
+ stock,
+ **({"audio": audio, "audio_sample_rate": rate} if with_audio else {}),
+ )
+ info_ours, frames_ours = _decode(ours)
+ info_stock, frames_stock = _decode(stock)
+ assert info_ours == info_stock
+ assert info_ours[0] == "h264" and info_ours[1] == "yuv420p"
+ assert ("audio" in info_ours[3]) == with_audio
+ assert frames_ours.shape == frames_stock.shape == clip.shape
+ assert _psnr(frames_ours, clip) > 30
+ opts_ours, opts_stock = _x264_options(ours), _x264_options(stock)
+ assert opts_ours["sliced_threads"] == "0" and opts_ours["crf"] == "23.0"
+ keys = (set(opts_ours) | set(opts_stock)) - _THREADING
+ assert {k: opts_ours.get(k) for k in keys} == {k: opts_stock.get(k) for k in keys}
+
+
+def test_float_pil_and_uint8_inputs_encode_the_same_frames(tmp_path):
+ from PIL import Image
+
+ clip = _clip(frames = 8)
+ inputs = {
+ "u8": torch.from_numpy(clip),
+ "np": clip.astype(np.float32) / 255,
+ "pil": [Image.fromarray(f) for f in clip],
+ }
+ decoded = {}
+ for name, value in inputs.items():
+ path = str(tmp_path / f"{name}.mp4")
+ assert ve.encode_x264(value, 24, path)
+ decoded[name] = _decode(path)[1]
+ assert np.array_equal(decoded["u8"], decoded["np"]) and np.array_equal(
+ decoded["u8"], decoded["pil"]
+ )
+
+
[email protected]("value", ["0", "false", "off"])
+def test_kill_switch(tmp_path, monkeypatch, value):
+ monkeypatch.setenv(ve.FRAME_THREADS_ENV, value)
+ assert ve.encode_x264(_clip(frames = 2), 24, str(tmp_path / "x.mp4")) is False
+
+
+def test_unsure_input_and_failures_hand_back(tmp_path, monkeypatch):
+ path = str(tmp_path / "x.mp4")
+ # Out of range float: encode_video's own warn-and-use-as-is branch decides.
+ assert ve.encode_x264(np.full((2, 16, 16, 3), 2.0, dtype = np.float32), 24, path) is False
+ assert ve.encode_x264(np.zeros((2, 16, 16, 4), dtype = np.uint8), 24, path) is False
+ assert (
+ ve.encode_x264(np.zeros((2, 16, 16, 3), dtype = np.uint8), 24, path, torch.zeros(2, 10))
+ is False
+ )
+
+ def _boom(*args, **kwargs):
+ raise RuntimeError("encoder open failed")
+
+ monkeypatch.setattr(av, "open", _boom)
+ assert ve.encode_x264(np.zeros((2, 16, 16, 3), dtype = np.uint8), 24, path) is False
+
+
[email protected]("x264_ok, expect", [(True, ["frame"]), (False, ["frame", "stock"])])
+def test_encode_mp4_routes_to_frame_threads_first(monkeypatch, x264_ok, expect):
+ from core.inference import video as video_mod
+
+ used: list = []
+
+ def _stock(frames, fps, path, **kwargs):
+ used.append("stock")
+ open(path, "wb").write(b"stock")
+
+ def _frame(frames, fps, path, audio, rate):
+ used.append("frame")
+ if x264_ok:
+ open(path, "wb").write(b"frame")
+ return x264_ok
+
+ monkeypatch.setattr(eu, "encode_video", _stock)
+ monkeypatch.setattr(video_mod, "nvenc_gpu", lambda logger = None: None)
+ monkeypatch.setattr(video_mod, "encode_x264", _frame)
+ out = video_mod.VideoBackend._encode_mp4(
+ np.zeros((2, 16, 16, 3), dtype = np.float32), 24, None, None
+ )
+ assert used == expect and out == (b"frame" if x264_ok else b"stock")
+
+
+def test_encode_mp4_end_to_end_default_is_frame_threaded(monkeypatch):
+ from core.inference import video as video_mod
+
+ monkeypatch.delenv(ve.FRAME_THREADS_ENV, raising = False)
+ monkeypatch.setattr(video_mod, "nvenc_gpu", lambda logger = None: None)
+ data = video_mod.VideoBackend._encode_mp4(torch.from_numpy(_clip()), 24, None, None)
+ assert b"sliced_threads=0" in data
diff --git a/studio/backend/tests/test_video_nvenc.py b/studio/backend/tests/test_video_nvenc.py
index 23e52534bd7..f789960b12a 100644
--- a/studio/backend/tests/test_video_nvenc.py
+++ b/studio/backend/tests/test_video_nvenc.py
@@ -146,6 +146,7 @@ def test_encode_mp4_falls_back_to_libx264(monkeypatch, gpu, nvenc_ok, expect):
monkeypatch.setattr(eu, "encode_video", _x264)
monkeypatch.setattr(video_mod, "nvenc_gpu", lambda logger = None: gpu)
monkeypatch.setattr(video_mod, "encode_nvenc", _nvenc)
+ monkeypatch.setattr(video_mod, "encode_x264", lambda *args: False)
out = video_mod.VideoBackend._encode_mp4(
np.zeros((2, 16, 16, 3), dtype = np.float32), 24, None, None
)