Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: opt-in NVENC for the video mp4 export (#11881)

Daniel Hane06f019e04d2
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 1ff7973958b..df78c1001bb 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -169,6 +169,7 @@ from .video_minimax_h3_te import (
h3_te_quant_scheme,
h3_te_resident_gb,
)
+from .video_nvenc import encode_nvenc, nvenc_gpu
from utils.hardware import clear_gpu_cache
# Shared with the image backend so both pin every loader call to the same live cache root
@@ -6808,7 +6809,16 @@ class VideoBackend:
)
if sample_rate:
encode_kwargs["audio_sample_rate"] = int(sample_rate)
- encode_video(video_frames, fps, tmp.name, **encode_kwargs)
+ gpu = nvenc_gpu(logger = logger)
+ if gpu is None or not encode_nvenc(
+ video_frames,
+ fps,
+ tmp.name,
+ gpu,
+ encode_kwargs.get("audio"),
+ encode_kwargs.get("audio_sample_rate"),
+ ):
+ encode_video(video_frames, fps, tmp.name, **encode_kwargs)
return Path(tmp.name).read_bytes()
finally:
try:
diff --git a/studio/backend/core/inference/video_nvenc.py b/studio/backend/core/inference/video_nvenc.py
new file mode 100644
index 00000000000..370fd6586d2
--- /dev/null
+++ b/studio/backend/core/inference/video_nvenc.py
@@ -0,0 +1,173 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Opt-in NVENC H.264 mp4 export (``UNSLOTH_STUDIO_VIDEO_ENCODER=nvenc``); libx264 on any failure. Not the default:
+larger files, and slower than libx264 on many-core hosts."""
+
+from __future__ import annotations
+
+import io
+import os
+import re
+import threading
+import time
+from typing import Any, Optional
+
+ENCODER_ENV = "UNSLOTH_STUDIO_VIDEO_ENCODER"
+CODEC = "h264_nvenc"
+# b=0 lifts FFmpeg's 2 Mbit/s default bitrate cap, which otherwise overrides the cq quality target.
+NVENC_OPTIONS = {
+ "preset": "p5",
+ "tune": "hq",
+ "rc": "vbr",
+ "cq": "24",
+ "b": "0",
+ "spatial-aq": "1",
+ "temporal-aq": "1",
+ "bf": "3",
+}
+# No NVENC engine (NVIDIA encode support matrix); skip the probe since a failed open costs seconds.
+_NO_NVENC_GPUS = (
+ "A100",
+ "A30",
+ "A800",
+ "H100",
+ "H800",
+ "H200",
+ "H20",
+ "B100",
+ "B200",
+ "B300",
+ "GB200",
+ "GB300",
+ "GH200",
+)
+# Failed probes may be transient (GeForce session limit held elsewhere), so retry after this.
+_PROBE_RETRY_S = 600.0
+
+_probe_lock = threading.Lock()
+_probed: dict = {}
+
+
+def nvenc_requested() -> bool:
+ return os.environ.get(ENCODER_ENV, "").strip().lower() == "nvenc"
+
+
+def _encode(
+ container: Any,
+ frames: Any,
+ fps: int,
+ gpu: int,
+ audio: Any = None,
+ audio_sample_rate: Any = None,
+) -> None:
+ import av
+
+ stream = container.add_stream(CODEC, rate = int(fps), options = {**NVENC_OPTIONS, "gpu": str(gpu)})
+ stream.width = int(frames.shape[2])
+ stream.height = int(frames.shape[1])
+ stream.pix_fmt = "yuv420p"
+ audio_stream = None
+ if audio is not None:
+ from diffusers.utils.export_utils import _prepare_audio_stream
+ audio_stream = _prepare_audio_stream(container, audio_sample_rate)
+ for frame in frames:
+ for packet in stream.encode(av.VideoFrame.from_ndarray(frame, format = "rgb24")):
+ container.mux(packet)
+ for packet in stream.encode():
+ container.mux(packet)
+ if audio_stream is not None:
+ from diffusers.utils.export_utils import _write_audio
+ _write_audio(container, audio_stream, audio, audio_sample_rate, av)
+
+
+def _probe(gpu: int) -> bool:
+ import av
+ import numpy as np
+ try:
+ with av.open(io.BytesIO(), mode = "w", format = "mp4") as container:
+ _encode(container, np.zeros((1, 256, 256, 3), dtype = np.uint8), 24, gpu)
+ return True
+ except Exception: # noqa: BLE001
+ return False
+
+
+def nvenc_gpu(device: Any = None, logger: Any = None) -> Optional[int]:
+ """CUDA ordinal to encode on if NVENC is requested and works, else None."""
+ if not nvenc_requested():
+ return None
+ try:
+ import torch
+
+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
+ return None
+ index = torch.device(device).index if device is not None else None
+ if index is None:
+ index = torch.cuda.current_device()
+ name = torch.cuda.get_device_name(index)
+ except Exception: # noqa: BLE001
+ return None
+ key = (index, name, os.environ.get("CUDA_VISIBLE_DEVICES"))
+ with _probe_lock:
+ cached = _probed.get(key)
+ if cached is None or (
+ cached[1] is not None and time.monotonic() - cached[1] >= _PROBE_RETRY_S
+ ):
+ if set(re.split(r"[\s\-_/]+", name.upper())) & set(_NO_NVENC_GPUS):
+ ok, failed_at = False, None
+ else:
+ ok = _probe(index)
+ failed_at = None if ok else time.monotonic()
+ _probed[key] = (ok, failed_at)
+ if logger is not None and (cached is None or cached[0] != ok):
+ logger.info(
+ "video.encoder: %s on %s (%s)",
+ "h264_nvenc" if ok else "libx264",
+ name,
+ f"{ENCODER_ENV}=nvenc" + ("" if ok else "; NVENC unavailable, keeping libx264"),
+ )
+ return index if _probed[key][0] else None
+
+
+def _uint8_frames(video: Any) -> Any:
+ """Frames as diffusers' encode_video would convert them; None when unsure."""
+ import numpy as np
+
+ if isinstance(video, list):
+ return np.stack([np.asarray(frame) for frame in video]) if video else None
+ if isinstance(video, np.ndarray):
+ if video.dtype == np.uint8:
+ return video
+ if not (np.all(video >= 0) and np.all(video <= 1)):
+ return None
+ return (video * 255).round().astype("uint8")
+ try:
+ import torch
+ if isinstance(video, torch.Tensor) and video.dtype == torch.uint8:
+ return video.cpu().numpy()
+ except Exception: # noqa: BLE001
+ pass
+ return None
+
+
+def encode_nvenc(
+ video: Any,
+ fps: int,
+ path: str,
+ gpu: int,
+ audio: Any = None,
+ audio_sample_rate: Any = None,
+) -> bool:
+ """Write ``path`` with NVENC; False (nothing usable written) when the caller should encode with libx264."""
+ frames = _uint8_frames(video)
+ if frames is None or frames.ndim != 4 or frames.shape[-1] != 3 or len(frames) == 0:
+ return False
+ if audio is not None and audio_sample_rate is None:
+ return False
+ try:
+ import av
+ with av.open(path, mode = "w", format = "mp4") as container:
+ _encode(container, frames, fps, gpu, audio, audio_sample_rate)
+ return True
+ except Exception: # noqa: BLE001
+ return False
diff --git a/studio/backend/tests/test_video_nvenc.py b/studio/backend/tests/test_video_nvenc.py
new file mode 100644
index 00000000000..23e52534bd7
--- /dev/null
+++ b/studio/backend/tests/test_video_nvenc.py
@@ -0,0 +1,184 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+
+from __future__ import annotations
+
+
+import numpy as np
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import video_nvenc as vn # noqa: E402
+
+
[email protected](autouse = True)
+def _fresh(monkeypatch):
+ monkeypatch.setattr(vn, "_probed", {})
+
+
+def _fake_cuda(
+ monkeypatch,
+ name,
+ *,
+ hip = None,
+):
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(torch.cuda, "get_device_name", lambda index = None: name)
+ monkeypatch.setattr(torch.version, "hip", hip, raising = False)
+ probes: list = []
+
+ def _probe(gpu):
+ probes.append(gpu)
+ return True
+
+ monkeypatch.setattr(vn, "_probe", _probe)
+ return probes
+
+
+def test_off_unless_requested(monkeypatch):
+ monkeypatch.delenv(vn.ENCODER_ENV, raising = False)
+ probes = _fake_cuda(monkeypatch, "NVIDIA L4")
+ assert vn.nvenc_gpu() is None and probes == []
+ monkeypatch.setenv(vn.ENCODER_ENV, "x264")
+ assert vn.nvenc_gpu() is None and probes == []
+
+
+ "name",
+ [
+ "NVIDIA B200",
+ "NVIDIA B100",
+ "NVIDIA A100-SXM4-40GB",
+ "NVIDIA H100 80GB HBM3",
+ "NVIDIA GB200",
+ "NVIDIA H20",
+ "NVIDIA GH200 480GB",
+ ],
+)
+def test_gpus_without_an_encoder_are_never_probed(monkeypatch, name):
+ monkeypatch.setenv(vn.ENCODER_ENV, "nvenc")
+ probes = _fake_cuda(monkeypatch, name)
+ assert vn.nvenc_gpu() is None and probes == []
+
+
+def test_probed_once_per_gpu(monkeypatch):
+ monkeypatch.setenv(vn.ENCODER_ENV, " NVENC ")
+ probes = _fake_cuda(monkeypatch, "NVIDIA RTX A3000 Laptop GPU")
+ assert vn.nvenc_gpu() == 0
+ assert vn.nvenc_gpu() == 0
+ assert probes == [0]
+
+
+def test_a_failed_probe_is_retried_after_the_window(monkeypatch):
+ monkeypatch.setenv(vn.ENCODER_ENV, "nvenc")
+ _fake_cuda(monkeypatch, "NVIDIA GeForce RTX 4090")
+ results = iter([False, True])
+ probes: list = []
+ monkeypatch.setattr(vn, "_probe", lambda gpu: probes.append(gpu) or next(results))
+ now = [1000.0]
+ monkeypatch.setattr(vn.time, "monotonic", lambda: now[0])
+ assert vn.nvenc_gpu() is None
+ now[0] += vn._PROBE_RETRY_S - 1
+ assert vn.nvenc_gpu() is None and probes == [0]
+ now[0] += 1
+ assert vn.nvenc_gpu() == 0 and probes == [0, 0]
+ now[0] += 10 * vn._PROBE_RETRY_S
+ assert vn.nvenc_gpu() == 0 and probes == [0, 0]
+
+
+def test_rocm_is_never_tried(monkeypatch):
+ monkeypatch.setenv(vn.ENCODER_ENV, "nvenc")
+ probes = _fake_cuda(monkeypatch, "AMD Radeon 8060S", hip = "7.1")
+ assert vn.nvenc_gpu() is None and probes == []
+
+
+def test_frames_match_what_encode_video_would_encode():
+ rng = np.random.default_rng(0)
+ clip = rng.random((3, 8, 12, 3), dtype = np.float32)
+ assert np.array_equal(vn._uint8_frames(clip), (clip * 255).round().astype("uint8"))
+ u8 = (clip * 255).round().astype("uint8")
+ assert np.array_equal(vn._uint8_frames(torch.from_numpy(u8)), u8)
+ from PIL import Image
+
+ assert np.array_equal(vn._uint8_frames([Image.fromarray(f) for f in u8]), u8)
+ clip[0, 0, 0, 0] = np.nan
+ assert vn._uint8_frames(clip) is None
+
+
+def test_a_failed_encode_asks_for_libx264(monkeypatch, tmp_path):
+ def _boom(*args, **kwargs):
+ raise RuntimeError("OpenEncodeSessionEx failed: incompatible client key (21)")
+
+ monkeypatch.setattr(vn, "_encode", _boom)
+ clip = np.zeros((2, 16, 16, 3), dtype = np.uint8)
+ assert vn.encode_nvenc(clip, 24, str(tmp_path / "x.mp4"), 0) is False
+ assert (
+ vn.encode_nvenc(
+ np.full((2, 16, 16, 3), 2.0, dtype = np.float32), 24, str(tmp_path / "x.mp4"), 0
+ )
+ is False
+ )
+
+
+ "gpu, nvenc_ok, expect", [(None, None, "x264"), (0, True, "nvenc"), (0, False, "x264")]
+)
+def test_encode_mp4_falls_back_to_libx264(monkeypatch, gpu, nvenc_ok, expect):
+ eu = pytest.importorskip("diffusers.utils.export_utils")
+
+ from core.inference import video as video_mod
+
+ used: list = []
+
+ def _x264(frames, fps, path, **kwargs):
+ used.append("x264")
+ open(path, "wb").write(b"x264")
+
+ def _nvenc(frames, fps, path, gpu_index, audio, sample_rate):
+ if nvenc_ok:
+ used.append("nvenc")
+ open(path, "wb").write(b"nvenc")
+ return nvenc_ok
+
+ monkeypatch.setattr(eu, "encode_video", _x264)
+ monkeypatch.setattr(video_mod, "nvenc_gpu", lambda logger = None: gpu)
+ monkeypatch.setattr(video_mod, "encode_nvenc", _nvenc)
+ out = video_mod.VideoBackend._encode_mp4(
+ np.zeros((2, 16, 16, 3), dtype = np.float32), 24, None, None
+ )
+ assert used == [expect] and out == expect.encode()
+
+
+def _nvenc_here():
+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
+ return False
+ import re
+ return not set(re.split(r"[\s\-_/]+", torch.cuda.get_device_name(0).upper())) & set(
+ vn._NO_NVENC_GPUS
+ )
+
+
[email protected](not _nvenc_here(), reason = "needs an NVIDIA GPU with an NVENC engine")
+def test_real_nvenc_clip_round_trips(monkeypatch, tmp_path):
+ av = pytest.importorskip("av")
+ monkeypatch.setenv(vn.ENCODER_ENV, "nvenc")
+ gpu = vn.nvenc_gpu()
+ if gpu is None:
+ pytest.skip("NVENC did not open here (driver / container)")
+ yy, xx = np.mgrid[0:128, 0:192]
+ clip = np.stack(
+ [np.stack([(xx + 4 * t) % 256, (yy + 2 * t) % 256, (xx + yy) % 256], -1) for t in range(24)]
+ )
+ clip = clip.astype(np.uint8)
+ audio = torch.zeros(2, 48000)
+ path = str(tmp_path / "nv.mp4")
+ assert vn.encode_nvenc(clip, 24, path, gpu, audio, 48000)
+ with av.open(path) as container:
+ codecs = {s.type: s.codec_context.name for s in container.streams}
+ frames = [f.to_ndarray(format = "rgb24") for f in container.decode(video = 0)]
+ assert codecs["video"] == "h264" and "audio" in codecs and len(frames) == 24
+ mse = np.mean((np.stack(frames).astype(np.float64) - clip) ** 2)
+ assert 10 * np.log10(255**2 / mse) > 30