Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

feat(linux): hardware H264 on AMD and Intel GPUs through VA-API (#3262)

Richie McIlroye3ad34fddcd3
diff --git a/crates/enc-ffmpeg/src/video/h264.rs b/crates/enc-ffmpeg/src/video/h264.rs
index cbe05805f..01db20997 100644
--- a/crates/enc-ffmpeg/src/video/h264.rs
+++ b/crates/enc-ffmpeg/src/video/h264.rs
@@ -12,6 +12,7 @@ use ffmpeg::{
use tracing::{debug, error, info, trace, warn};
use crate::base::{EncoderBase, PacketSink};
+use crate::video::HwUpload;
use crate::video::h264_packet::H264PacketEncoder;
#[cfg(target_os = "macos")]
use crate::video::videotoolbox_hw::VideoToolboxHwFrames;
@@ -60,12 +61,14 @@ pub enum H264EncoderError {
SelfTest(String),
#[error("VideoToolbox hardware input unavailable: {0}")]
VideoToolboxHwInput(String),
+ #[error("VA-API: {0}")]
+ Vaapi(String),
}
fn is_hardware_h264(codec_name: &str) -> bool {
matches!(
codec_name,
- "h264_videotoolbox" | "h264_nvenc" | "h264_qsv" | "h264_amf" | "h264_mf"
+ "h264_videotoolbox" | "h264_nvenc" | "h264_qsv" | "h264_amf" | "h264_mf" | "h264_vaapi"
)
}
@@ -270,6 +273,7 @@ impl H264EncoderBuilder {
input_format,
input_width,
input_height,
+ hw_upload,
} = open_video_encoder(
codec,
encoder_options,
@@ -298,6 +302,7 @@ impl H264EncoderBuilder {
input_width,
input_height,
converted_frame_pool,
+ hw_upload,
#[cfg(target_os = "macos")]
videotoolbox_hw: None,
})
@@ -432,6 +437,7 @@ impl H264EncoderBuilder {
input_width: input_config.width,
input_height: input_config.height,
converted_frame_pool: None,
+ hw_upload: HwUpload::default(),
videotoolbox_hw: Some(Arc::new(hw_frames)),
})
}
@@ -567,6 +573,8 @@ pub(crate) struct OpenedVideoEncoder {
pub input_format: format::Pixel,
pub input_width: u32,
pub input_height: u32,
+ /// Where software frames go before the encoder takes them.
+ pub hw_upload: HwUpload,
}
#[allow(clippy::too_many_arguments)]
@@ -640,11 +648,18 @@ fn open_video_encoder_inner(
crf: Option<u8>,
global_header: bool,
) -> Result<OpenedVideoEncoder, H264EncoderError> {
- let encoder_supports_input_format = codec
- .video()
- .ok()
- .and_then(|codec_video| codec_video.formats())
- .is_some_and(|mut formats| formats.any(|f| f == input_config.pixel_format));
+ // `h264_vaapi` lists only VA surfaces; the frames it is handed are NV12
+ // that gets uploaded into one.
+ let vaapi = codec.name() == "h264_vaapi";
+ let encoder_supports_input_format = if vaapi {
+ input_config.pixel_format == ffmpeg::format::Pixel::NV12
+ } else {
+ codec
+ .video()
+ .ok()
+ .and_then(|codec_video| codec_video.formats())
+ .is_some_and(|mut formats| formats.any(|f| f == input_config.pixel_format))
+ };
let mut needs_pixel_conversion = false;
@@ -749,14 +764,26 @@ fn open_video_encoder_inner(
.map(|v| v.get())
.unwrap_or(1);
- let encoder = {
+ let open = |hw_upload: &HwUpload,
+ encoder_options: Dictionary<'static>|
+ -> Result<encoder::Video, ffmpeg::Error> {
let mut encoder_ctx = context::Context::new_with_codec(codec);
encoder_ctx.set_threading(Config::count(thread_count));
let mut encoder = encoder_ctx.encoder().video()?;
encoder.set_width(output_width);
encoder.set_height(output_height);
- encoder.set_format(output_format);
+ encoder.set_format(if vaapi {
+ ffmpeg::format::Pixel::VAAPI
+ } else {
+ output_format
+ });
+ #[cfg(target_os = "linux")]
+ if let Some(frames) = &hw_upload.vaapi {
+ unsafe { frames.attach(encoder.as_mut_ptr()) };
+ }
+ #[cfg(not(target_os = "linux"))]
+ let _ = hw_upload;
encoder.set_time_base(input_config.time_base);
encoder.set_frame_rate(Some(input_config.frame_rate));
encoder.set_colorspace(color::Space::BT709);
@@ -784,7 +811,22 @@ fn open_video_encoder_inner(
encoder.set_max_bit_rate(bitrate * 3 / 2);
}
- encoder.open_as_with(codec, encoder_options)?
+ encoder.open_as_with(codec, encoder_options)
+ };
+
+ #[cfg(target_os = "linux")]
+ let (encoder, hw_upload) = if vaapi {
+ open_on_a_vaapi_node(output_width, output_height, |hw_upload| {
+ open(hw_upload, clone_options(&encoder_options))
+ })?
+ } else {
+ let hw_upload = HwUpload::default();
+ (open(&hw_upload, encoder_options)?, hw_upload)
+ };
+ #[cfg(not(target_os = "linux"))]
+ let (encoder, hw_upload) = {
+ let hw_upload = HwUpload::default();
+ (open(&hw_upload, encoder_options)?, hw_upload)
};
let converted_frame_pool = if converter.is_some() {
@@ -807,9 +849,45 @@ fn open_video_encoder_inner(
input_format: input_config.pixel_format,
input_width: input_config.width,
input_height: input_config.height,
+ hw_upload,
})
}
+/// Opens the encoder on the first render node whose VA-API driver both makes
+/// NV12 surfaces and encodes H264: a GPU can do the first and not the second.
+#[cfg(target_os = "linux")]
+fn open_on_a_vaapi_node(
+ width: u32,
+ height: u32,
+ mut open: impl FnMut(&HwUpload) -> Result<encoder::Video, ffmpeg::Error>,
+) -> Result<(encoder::Video, HwUpload), H264EncoderError> {
+ let mut errors = Vec::new();
+ for node in super::vaapi::render_nodes() {
+ let frames = match super::vaapi::VaapiFrames::open(&node, width, height) {
+ Ok(frames) => frames,
+ Err(error) => {
+ errors.push(format!("{}: {error}", node.display()));
+ continue;
+ }
+ };
+ let hw_upload = HwUpload {
+ vaapi: Some(frames),
+ };
+ match open(&hw_upload) {
+ Ok(encoder) => {
+ debug!(node = %node.display(), "H264 encodes through VA-API");
+ return Ok((encoder, hw_upload));
+ }
+ Err(error) => errors.push(format!("{}: {error}", node.display())),
+ }
+ }
+ Err(H264EncoderError::Vaapi(if errors.is_empty() {
+ "no render node with a VA-API driver".to_string()
+ } else {
+ errors.join("; ")
+ }))
+}
+
pub struct H264Encoder {
base: EncoderBase,
encoder: encoder::Video,
@@ -821,6 +899,7 @@ pub struct H264Encoder {
input_width: u32,
input_height: u32,
converted_frame_pool: Option<frame::Video>,
+ hw_upload: HwUpload,
#[cfg(target_os = "macos")]
videotoolbox_hw: Option<Arc<VideoToolboxHwFrames>>,
}
@@ -930,6 +1009,10 @@ impl H264Encoder {
} else {
&frame
};
+ let frame_to_send = self
+ .hw_upload
+ .frame(frame_to_send)
+ .map_err(QueueFrameError::Encode)?;
self.base
.send_frame(frame_to_send, output, &mut self.encoder)
@@ -959,6 +1042,10 @@ impl H264Encoder {
} else {
frame as &frame::Video
};
+ let frame_to_send = self
+ .hw_upload
+ .frame(frame_to_send)
+ .map_err(QueueFrameError::Encode)?;
self.base
.send_frame(frame_to_send, output, &mut self.encoder)
@@ -985,9 +1072,13 @@ impl H264Encoder {
self.base
.update_pts(&mut frame, timestamp, &mut self.encoder);
+ let frame = self
+ .hw_upload
+ .frame(&frame)
+ .map_err(QueueFrameError::Encode)?;
self.base
- .send_frame(&frame, output, &mut self.encoder)
+ .send_frame(frame, output, &mut self.encoder)
.map_err(QueueFrameError::Encode)?;
Ok(())
@@ -1131,11 +1222,16 @@ fn requires_software_encoder(config: &VideoInfo, preset: H264Preset, is_export:
}
#[cfg(target_os = "linux")]
-fn linux_encoder_priority(nvidia_device_available: bool) -> &'static [&'static str] {
- if nvidia_device_available {
- &["h264_nvenc", "libx264"]
- } else {
- &["libx264"]
+fn linux_encoder_priority(
+ nvidia_device_available: bool,
+ vaapi_device_available: bool,
+) -> &'static [&'static str] {
+ match (nvidia_device_available, vaapi_device_available) {
+ // A laptop's NVIDIA GPU first, its integrated GPU if NVENC fails.
+ (true, true) => &["h264_nvenc", "h264_vaapi", "libx264"],
+ (true, false) => &["h264_nvenc", "libx264"],
+ (false, true) => &["h264_vaapi", "libx264"],
+ (false, false) => &["libx264"],
}
}
@@ -1172,7 +1268,10 @@ fn get_default_encoder_priority(_config: &VideoInfo) -> &'static [&'static str]
#[cfg(target_os = "linux")]
{
- linux_encoder_priority(std::path::Path::new("/dev/nvidiactl").exists())
+ linux_encoder_priority(
+ std::path::Path::new("/dev/nvidiactl").exists(),
+ !super::vaapi::render_nodes().is_empty(),
+ )
}
#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))]
@@ -1315,6 +1414,10 @@ fn get_codec_and_options(
}
options.set("g", &keyframe_interval_str);
}
+ "h264_vaapi" => {
+ options.set("profile", "high");
+ options.set("g", &keyframe_interval_str);
+ }
"h264_mf" => {
options.set("hw_encoding", "true");
if is_export {
@@ -1404,7 +1507,8 @@ fn cached_hardware_self_test(
sync::{Mutex, OnceLock},
};
- if !(cfg!(target_os = "windows") || (cfg!(target_os = "linux") && codec.name() == "h264_nvenc"))
+ if !(cfg!(target_os = "windows")
+ || (cfg!(target_os = "linux") && matches!(codec.name(), "h264_nvenc" | "h264_vaapi")))
{
return Ok(());
}
@@ -1502,6 +1606,7 @@ fn hardware_encoder_self_test(
)
.map_err(|e| format!("test encoder open: {e}"))?;
let mut encoder = opened.encoder;
+ let mut hw_upload = opened.hw_upload;
// Mid-gray is neutral in both RGB (128,128,128) and YUV (Y=128, U=V=128)
// representations, so filling every plane with 128 produces a valid gray
@@ -1532,8 +1637,11 @@ fn hardware_encoder_self_test(
const TEST_FRAME_COUNT: i64 = 8;
for i in 0..TEST_FRAME_COUNT {
test_frame.set_pts(Some(i * pts_step));
+ let frame = hw_upload
+ .frame(&test_frame)
+ .map_err(|e| format!("upload: {e}"))?;
encoder
- .send_frame(&test_frame)
+ .send_frame(frame)
.map_err(|e| format!("send_frame: {e}"))?;
drain_packets(&mut encoder, &mut packets);
}
@@ -1738,13 +1846,30 @@ mod self_test_tests {
#[cfg(target_os = "linux")]
#[test]
fn linux_nvidia_device_prefers_nvenc_and_keeps_software_fallback() {
- assert_eq!(linux_encoder_priority(true), &["h264_nvenc", "libx264"]);
+ assert_eq!(
+ linux_encoder_priority(true, false),
+ &["h264_nvenc", "libx264"]
+ );
}
#[cfg(target_os = "linux")]
#[test]
- fn linux_without_nvidia_preserves_software_encoder() {
- assert_eq!(linux_encoder_priority(false), &["libx264"]);
+ fn linux_amd_and_intel_gpus_encode_through_va_api() {
+ assert_eq!(
+ linux_encoder_priority(false, true),
+ &["h264_vaapi", "libx264"]
+ );
+ // A hybrid laptop falls back to its integrated GPU before software.
+ assert_eq!(
+ linux_encoder_priority(true, true),
+ &["h264_nvenc", "h264_vaapi", "libx264"]
+ );
+ }
+
+ #[cfg(target_os = "linux")]
+ #[test]
+ fn linux_without_a_hardware_encoder_preserves_software_encoder() {
+ assert_eq!(linux_encoder_priority(false, false), &["libx264"]);
}
#[cfg(target_os = "linux")]
@@ -1756,7 +1881,7 @@ mod self_test_tests {
get_encoder_priority_with_override(
&config,
H264Preset::HighThroughput,
- Some(linux_encoder_priority(true)),
+ Some(linux_encoder_priority(true, true)),
false,
),
&["libx264"],
@@ -1771,7 +1896,7 @@ mod self_test_tests {
let codecs = get_codec_and_options(
&config,
H264Preset::Ultrafast,
- Some(linux_encoder_priority(true)),
+ Some(linux_encoder_priority(true, true)),
true,
Some(23),
);
@@ -1807,6 +1932,52 @@ mod self_test_tests {
cached_hardware_self_test(codec, &options, &config, 160, 120, 0.3, None)
.expect("real NVIDIA hardware preserves neutral gray through encode and decode");
}
+
+ #[cfg(target_os = "linux")]
+ #[test]
+ fn linux_vaapi_self_test_decodes_real_gray_frames_when_available() {
+ if super::super::vaapi::render_nodes().is_empty() {
+ return;
+ }
+
+ ffmpeg::init().ok();
+ let Some(codec) = encoder::find_by_name("h264_vaapi") else {
+ return;
+ };
+ let config = VideoInfo::from_raw(cap_media_info::RawVideoFormat::Bgra, 640, 360, 30);
+ let options = get_codec_and_options(
+ &config,
+ H264Preset::Ultrafast,
+ Some(&["h264_vaapi", "libx264"]),
+ false,
+ None,
+ )
+ .into_iter()
+ .find(|(candidate, _)| candidate.name() == "h264_vaapi")
+ .expect("available VA-API encoder is configured")
+ .1;
+ // Only hardware that opens an H264 encoder is tested (a render node
+ // can be inaccessible, render-only, or lack its VA-API driver), but
+ // one that opens must keep the gray.
+ if open_video_encoder_inner(
+ codec,
+ clone_options(&options),
+ &config,
+ 640,
+ 360,
+ 0.3,
+ true,
+ None,
+ false,
+ )
+ .is_err()
+ {
+ return;
+ }
+
+ cached_hardware_self_test(codec, &options, &config, 640, 360, 0.3, None)
+ .expect("a VA-API GPU preserves neutral gray through upload, encode and decode");
+ }
}
#[cfg(test)]
diff --git a/crates/enc-ffmpeg/src/video/h264_packet.rs b/crates/enc-ffmpeg/src/video/h264_packet.rs
index dd0fe0e8a..9f702f1a2 100644
--- a/crates/enc-ffmpeg/src/video/h264_packet.rs
+++ b/crates/enc-ffmpeg/src/video/h264_packet.rs
@@ -7,12 +7,14 @@ use ffmpeg::{
frame,
};
+use crate::video::HwUpload;
use crate::video::h264::OpenedVideoEncoder;
pub struct H264PacketEncoder {
encoder: encoder::Video,
converter: Option<ffmpeg::software::scaling::Context>,
converted_frame_pool: Option<frame::Video>,
+ hw_upload: HwUpload,
output_format: format::Pixel,
output_width: u32,
output_height: u32,
@@ -56,6 +58,7 @@ impl H264PacketEncoder {
encoder: opened.encoder,
converter: opened.converter,
converted_frame_pool: opened.converted_frame_pool,
+ hw_upload: opened.hw_upload,
output_format: opened.output_format,
output_width: opened.output_width,
output_height: opened.output_height,
@@ -148,6 +151,10 @@ impl H264PacketEncoder {
&frame
};
+ let frame_to_send = self
+ .hw_upload
+ .frame(frame_to_send)
+ .map_err(EncodePacketError::Encode)?;
self.encoder
.send_frame(frame_to_send)
.map_err(EncodePacketError::Encode)?;
@@ -180,6 +187,10 @@ impl H264PacketEncoder {
frame as &frame::Video
};
+ let frame_to_send = self
+ .hw_upload
+ .frame(frame_to_send)
+ .map_err(EncodePacketError::Encode)?;
self.encoder
.send_frame(frame_to_send)
.map_err(EncodePacketError::Encode)?;
diff --git a/crates/enc-ffmpeg/src/video/mod.rs b/crates/enc-ffmpeg/src/video/mod.rs
index e69e981ed..c4fb740e8 100644
--- a/crates/enc-ffmpeg/src/video/mod.rs
+++ b/crates/enc-ffmpeg/src/video/mod.rs
@@ -7,9 +7,32 @@ pub mod hevc;
mod packed_yuv;
pub mod prores;
mod rgb_to_yuv;
+#[cfg(target_os = "linux")]
+pub(crate) mod vaapi;
#[cfg(target_os = "macos")]
pub mod videotoolbox_hw;
+/// Hands an encoder the frame it takes: a GPU surface for encoders that only
+/// encode those (VA-API), the software frame itself for every other.
+#[derive(Default)]
+pub(crate) struct HwUpload {
+ #[cfg(target_os = "linux")]
+ pub(crate) vaapi: Option<vaapi::VaapiFrames>,
+}
+
+impl HwUpload {
+ pub(crate) fn frame<'a>(
+ &'a mut self,
+ frame: &'a frame::Video,
+ ) -> Result<&'a frame::Video, ffmpeg::Error> {
+ #[cfg(target_os = "linux")]
+ if let Some(vaapi) = &mut self.vaapi {
+ return vaapi.upload(frame);
+ }
+ Ok(frame)
+ }
+}
+
pub(crate) fn convert_or_scale(
src: &frame::Video,
dst: &mut frame::Video,
diff --git a/crates/enc-ffmpeg/src/video/vaapi.rs b/crates/enc-ffmpeg/src/video/vaapi.rs
new file mode 100644
index 000000000..6e8a3970b
--- /dev/null
+++ b/crates/enc-ffmpeg/src/video/vaapi.rs
@@ -0,0 +1,158 @@
+//! H.264 on Linux AMD and Intel GPUs through VA-API.
+//!
+//! `h264_vaapi` only encodes frames that already live in a VA surface, so the
+//! encoder gets a pool of NV12 surfaces on a render node and each software
+//! NV12 frame is uploaded into one just before it is sent.
+
+use std::{
+ ffi::CString,
+ path::{Path, PathBuf},
+ ptr,
+};
+
+use ffmpeg::{ffi, frame};
+
+pub(crate) struct VaapiFrames {
+ device: *mut ffi::AVBufferRef,
+ frames: *mut ffi::AVBufferRef,
+ surface: frame::Video,
+}
+
+// The buffers are reference counted by FFmpeg and only touched from the
+// thread that owns the encoder.
+unsafe impl Send for VaapiFrames {}
+
+impl VaapiFrames {
+ pub(crate) fn open(node: &Path, width: u32, height: u32) -> Result<Self, String> {
+ let path = CString::new(node.as_os_str().as_encoded_bytes())
+ .map_err(|_| "render node path has a NUL byte".to_string())?;
+ let mut device = ptr::null_mut();
+ let ret = unsafe {
+ ffi::av_hwdevice_ctx_create(
+ &mut device,
+ ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI,
+ path.as_ptr(),
+ ptr::null_mut(),
+ 0,
+ )
+ };
+ if ret < 0 {
+ return Err(format!("VA-API device: {}", ffmpeg::Error::from(ret)));
+ }
+ let mut frames = unsafe { ffi::av_hwframe_ctx_alloc(device) };
+ if frames.is_null() {
+ unsafe { ffi::av_buffer_unref(&mut device) };
+ return Err("VA-API frame pool allocation failed".to_string());
+ }
+ unsafe {
+ let context = (*frames).data.cast::<ffi::AVHWFramesContext>();
+ (*context).format = ffi::AVPixelFormat::AV_PIX_FMT_VAAPI;
+ (*context).sw_format = ffi::AVPixelFormat::AV_PIX_FMT_NV12;
+ (*context).width = width as i32;
+ (*context).height = height as i32;
+ (*context).initial_pool_size = 20;
+ }
+ let ret = unsafe { ffi::av_hwframe_ctx_init(frames) };
+ if ret < 0 {
+ unsafe {
+ ffi::av_buffer_unref(&mut frames);
+ ffi::av_buffer_unref(&mut device);
+ }
+ return Err(format!(
+ "VA-API NV12 surfaces: {}",
+ ffmpeg::Error::from(ret)
+ ));
+ }
+ Ok(Self {
+ device,
+ frames,
+ surface: frame::Video::empty(),
+ })
+ }
+
+ /// # Safety
+ /// `encoder` must be an unopened codec context.
+ pub(crate) unsafe fn attach(&self, encoder: *mut ffi::AVCodecContext) {
+ unsafe { (*encoder).hw_frames_ctx = ffi::av_buffer_ref(self.frames) };
+ }
+
+ pub(crate) fn upload(&mut self, source: &frame::Video) -> Result<&frame::Video, ffmpeg::Error> {
+ unsafe {
+ let surface = self.surface.as_mut_ptr();
+ ffi::av_frame_unref(surface);
+ let ret = ffi::av_hwframe_get_buffer(self.frames, surface, 0);
+ if ret < 0 {
+ return Err(ffmpeg::Error::from(ret));
+ }
+ let ret = ffi::av_hwframe_transfer_data(surface, source.as_ptr(), 0);
+ if ret < 0 {
+ return Err(ffmpeg::Error::from(ret));
+ }
+ let ret = ffi::av_frame_copy_props(surface, source.as_ptr());
+ if ret < 0 {
+ return Err(ffmpeg::Error::from(ret));
+ }
+ }
+ Ok(&self.surface)
+ }
+}
+
+impl Drop for VaapiFrames {
+ fn drop(&mut self) {
+ unsafe {
+ ffi::av_frame_unref(self.surface.as_mut_ptr());
+ ffi::av_buffer_unref(&mut self.frames);
+ ffi::av_buffer_unref(&mut self.device);
+ }
+ }
+}
+
+/// Render nodes whose kernel driver has a VA-API driver: not NVIDIA (whose
+/// encoder is NVENC), and not the virtual vgem and vkms devices.
+pub(crate) fn render_nodes() -> Vec<PathBuf> {
+ let Ok(entries) = std::fs::read_dir("/dev/dri") else {
+ return Vec::new();
+ };
+ let mut nodes = entries
+ .filter_map(Result::ok)
+ .map(|entry| entry.path())
+ .filter(|path| {
+ path.file_name()
+ .and_then(|name| name.to_str())
+ .is_some_and(|name| name.starts_with("renderD"))
+ })
+ .filter(|path| vaapi_driver(kernel_driver(path).as_deref()))
+ .collect::<Vec<_>>();
+ nodes.sort();
+ nodes
+}
+
+fn kernel_driver(node: &Path) -> Option<String> {
+ let name = node.file_name()?;
+ let link =
+ std::fs::read_link(Path::new("/sys/class/drm").join(name).join("device/driver")).ok()?;
+ Some(link.file_name()?.to_string_lossy().into_owned())
+}
+
+fn vaapi_driver(driver: Option<&str>) -> bool {
+ match driver {
+ None => false,
+ Some(driver) => !(driver.starts_with("nvidia") || matches!(driver, "vgem" | "vkms")),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn only_gpus_with_a_va_api_driver_are_tried() {
+ for driver in ["amdgpu", "radeon", "i915", "xe", "nouveau", "virtio-pci"] {
+ assert!(vaapi_driver(Some(driver)), "{driver}");
+ }
+ for driver in ["nvidia", "nvidia-drm", "vgem", "vkms"] {
+ assert!(!vaapi_driver(Some(driver)), "{driver}");
+ }
+ assert!(!vaapi_driver(None));
+ }
+}