Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat(media): video decodes on the platform's hardware behind a toggle

Jareer46337f52683a
diff --git a/src/Cargo.lock b/src/Cargo.lock
index a09670c4..c4e5eda4 100644
--- a/src/Cargo.lock
+++ b/src/Cargo.lock
@@ -1317,6 +1317,7 @@ dependencies = [
"concat-core",
"ffmpeg-the-third",
"libc",
+ "log",
"thiserror 2.0.20",
]
diff --git a/src/crates/concat-host/tests/export.rs b/src/crates/concat-host/tests/export.rs
index 4f33157e..3aefa22f 100644
--- a/src/crates/concat-host/tests/export.rs
+++ b/src/crates/concat-host/tests/export.rs
@@ -41,7 +41,7 @@ use concat_host::{media, projects};
use concat_media::audio::{self as sound, AudioClip};
use concat_media::{
AudioDecoder, AudioOptions, DecodeOptions, Decoder, EncodeOptions, Encoder,
- Error as MediaError, FrameSink, FrameSource, SampleFormat, VideoCodec,
+ Error as MediaError, FrameSink, FrameSource, HwDevice, SampleFormat, VideoCodec,
};
use concat_project::animation;
use concat_project::commands::{ClipMove, ClipPatch, Command, TrackFlag, TrimEdge};
@@ -1190,3 +1190,71 @@ fn the_edges_export_too() {
exported.expect_second(end - 0.5, 3);
exported.expect_tone(start + 4.0);
}
+
+/// Hardware decode preferred, the way the settings toggle leaves it: the
+/// export decodes its sources on the platform's device where there is one
+/// and reads the same picture and sound back. The preference is the
+/// process's, so the other scenarios running alongside share it for the
+/// moment, and must not mind.
+#[test]
+fn an_export_decodes_on_the_hardware_when_preferred() {
+ struct Preferred;
+ impl Drop for Preferred {
+ fn drop(&mut self) {
+ concat_media::set_hardware_decode(false);
+ }
+ }
+ let _preferred = Preferred;
+ concat_media::set_hardware_decode(true);
+ assert_eq!(
+ concat_media::hardware_decode(),
+ HwDevice::platform_default().is_some(),
+ "on where the platform has a device, off where it has none"
+ );
+
+ let scratch = Scratch::new("hardware");
+ let sources = Sources::make(scratch.path());
+ // A reader opened the way the engine opens them follows the preference
+ // onto the device, on a machine that has one.
+ let mut reader = Decoder::open(&sources.peek, &DecodeOptions::default()).expect("opens");
+ reader.next_frame().expect("decodes").expect("a frame");
+ assert_eq!(reader.hardware(), HwDevice::platform_default());
+ drop(reader);
+ let mut studio = Studio::new(scratch.path(), "Hardware", video(WIDTH, HEIGHT, 30, 1));
+ let peek = studio.import(&sources.peek);
+ let cam = studio.import(&sources.cam);
+ studio.apply(Command::AddClipAtFirstFree {
+ media_id: peek,
+ start: 0.0,
+ });
+ let clip = studio
+ .apply(Command::AddClipAtFirstFree {
+ media_id: cam,
+ start: 6.0,
+ })
+ .expect("the clip has an id");
+ // A retimed clip on the device: the pacing sits on top of it exactly
+ // as it does the CPU.
+ studio.apply(Command::SetClipSpeed {
+ clip_id: clip.clone(),
+ speed: 2.0,
+ });
+ let start = studio.clip(&clip).start;
+ let end = studio.end();
+ assert!(
+ (end - start - 2.0).abs() < 1e-6,
+ "four seconds at double is two: {start} to {end}"
+ );
+
+ for (num, den) in [(30, 1), (60000, 1001)] {
+ let exported = studio.export_at(&format!("hardware at {num}/{den}"), Some((num, den)));
+ exported.expect_length(end);
+ exported.expect_sound(end);
+ exported.expect_second(0.5, 0);
+ exported.expect_second(5.9, 5);
+ exported.expect_second(start + 0.25, 0);
+ exported.expect_second(start + 1.75, 3);
+ exported.expect_tone(5.5);
+ exported.expect_quiet(4.5);
+ }
+}
diff --git a/src/crates/concat-media/Cargo.toml b/src/crates/concat-media/Cargo.toml
index 1ac203b3..9b509499 100644
--- a/src/crates/concat-media/Cargo.toml
+++ b/src/crates/concat-media/Cargo.toml
@@ -15,6 +15,7 @@ concat-core.workspace = true
# display-rotation side data is read from where 7.0 moved it.
ffmpeg-the-third = "6"
libc = "0.2"
+log = "0.4"
thiserror.workspace = true
[lints]
diff --git a/src/crates/concat-media/src/decode.rs b/src/crates/concat-media/src/decode.rs
index a6f914e4..fc310a48 100644
--- a/src/crates/concat-media/src/decode.rs
+++ b/src/crates/concat-media/src/decode.rs
@@ -9,9 +9,19 @@
//! by duplicating and dropping, repeats a still forever, and stops after a
//! frame budget. Every frame comes with its real presentation timestamp.
//!
+//! The codec runs on the platform's video hardware when the reader's
+//! [`HwPolicy`] and the process-wide preference say so, and on the CPU
+//! otherwise; see [`crate::hardware`]. A frame decoded on the device is
+//! copied back before the filtergraph sees it, so everything from there on
+//! is the same path, and a device that fails at any point hands the reader
+//! to software from the frame it was at. The frames that come out are the
+//! same either way, in count, size and timing; only the pixels may differ
+//! by the rounding of a different colour conversion.
+//!
//! ## On the unsafe in here
//!
-//! There is none: the wrapper crate owns every pointer. What this module
+//! The wrapper crate owns every pointer, and the one raw write here is the
+//! threading setting the wrapper has no accessor for. What this module
//! owns is the order things happen in.
use std::path::{Path, PathBuf};
@@ -26,6 +36,7 @@ use ffmpeg_the_third::util::frame::video::Video;
use crate::error::{Error, Result};
use crate::ffi;
+use crate::hardware::{self, Device, HwDevice, HwPolicy};
/// Anything that yields frames in order.
///
@@ -101,6 +112,13 @@ pub struct DecodeOptions {
/// decoded frame. Never for anything a person will watch or export: the
/// frame is near the instant, not at it.
pub keyframes_only: bool,
+ /// Whether to decode on the platform's video hardware. The default
+ /// follows the process-wide preference, [`crate::set_hardware_decode`],
+ /// so the window's toggle reaches every reader without being passed
+ /// through each one; a reader can insist on software or on a device.
+ /// A device that fails is never an error: the reader carries on in
+ /// software. See [`crate::hardware`].
+ pub hardware: HwPolicy,
}
impl DecodeOptions {
@@ -153,6 +171,19 @@ impl DecodeOptions {
self.keyframes_only = true;
self
}
+
+ /// Decodes on `device`, whatever the preference says, and in software
+ /// if the device will not. See [`DecodeOptions::hardware`].
+ pub fn accelerated(mut self, device: HwDevice) -> Self {
+ self.hardware = HwPolicy::Device(device);
+ self
+ }
+
+ /// Decodes in software, whatever the preference says.
+ pub fn in_software(mut self) -> Self {
+ self.hardware = HwPolicy::Software;
+ self
+ }
}
/// How a stream says its colour is encoded: the tags in its container,
@@ -285,6 +316,15 @@ pub struct Decoder {
/// Frames before this instant are decoded and discarded - the exact
/// half of a seek, after the container landed on a keyframe.
discard_before: Option<Rational>,
+ /// Frames at or before this instant are discarded too: where a reader
+ /// that moved to software mid-file picks up, after the frame the
+ /// device last gave it.
+ discard_through: Option<Rational>,
+ /// The device the codec is on and the pixel format its frames arrive
+ /// in, or `None` in software, which is also where a failed device ends.
+ hardware: Option<(HwDevice, Pixel)>,
+ /// The last picture's timestamp, for picking up in software after it.
+ last_pts: Option<Rational>,
/// The output frame instant the pacer is at, when pacing.
tick: u64,
origin: Rational,
@@ -320,31 +360,25 @@ impl Decoder {
let stream_index = stream.index();
let time_base = stream.time_base();
let rotation = ffi::rotation(&stream);
- let parameters = stream.parameters();
- let coded = (parameters.width(), parameters.height());
+ let coded = {
+ let parameters = stream.parameters();
+ (parameters.width(), parameters.height())
+ };
- let mut context = ffmpeg::codec::Context::from_parameters(parameters)
- .map_err(|error| ffi::fail("codec parameters", path, error))?;
- // Every core. libavcodec's API opens a decoder on one thread unless
- // told otherwise (the `ffmpeg` tool turns threads on for itself), and
- // a seek decodes every frame from the keyframe before it: on one
- // thread a 4K H.264 file decoded at about 87 frames a second, and a
- // scrub waited 400 ms for its frame. Frame threads for that walk,
- // slice threads for files cut into slices; zero lets the codec count.
- // SAFETY: the context is not opened yet, which is when threading is
- // set, and both fields are plain integers.
- unsafe {
- let raw = context.as_mut_ptr();
- (*raw).thread_type = ffmpeg::sys::FF_THREAD_FRAME | ffmpeg::sys::FF_THREAD_SLICE;
- (*raw).thread_count = 0;
- }
- let mut decoder = context
- .decoder()
- .video()
- .map_err(|error| ffi::fail("open decoder", path, error))?;
- if options.keyframes_only {
- decoder.skip_frame(ffmpeg::Discard::NonKey);
- }
+ // The device the policy asks for, if the process has one. A codec
+ // that will not open on it opens in software instead, and the
+ // reader is none the wiser past a line in the log.
+ let device = options.hardware.device().and_then(hardware::device);
+ let (decoder, accelerated) = match device {
+ Some(device) => match Self::open_codec(path, &stream, options, Some(&device)) {
+ Ok(opened) => opened,
+ Err(error) => {
+ hardware::note_fallback(device.kind(), "open the decoder", &error.to_string());
+ Self::open_codec(path, &stream, options, None)?
+ }
+ },
+ None => Self::open_codec(path, &stream, options, None)?,
+ };
let color = ColorSignal {
primaries: decoder.color_primaries(),
transfer: decoder.color_transfer_characteristic(),
@@ -375,6 +409,9 @@ impl Decoder {
height,
graph: None,
discard_before: None,
+ discard_through: None,
+ hardware: accelerated,
+ last_pts: None,
tick: 0,
origin: Rational::ZERO,
current: None,
@@ -392,11 +429,90 @@ impl Decoder {
Ok(this)
}
+ /// Opens the stream's codec, on `device` where the codec can decode on
+ /// it. Gives back the opened decoder and, when it is on the device, the
+ /// device and the pixel format its frames arrive in.
+ fn open_codec(
+ path: &Path,
+ stream: &ffmpeg::format::stream::Stream<'_>,
+ options: &DecodeOptions,
+ device: Option<&Device>,
+ ) -> Result<(decoder::Video, Option<(HwDevice, Pixel)>)> {
+ let mut context = ffmpeg::codec::Context::from_parameters(stream.parameters())
+ .map_err(|error| ffi::fail("codec parameters", path, error))?;
+ // Every core. libavcodec's API opens a decoder on one thread unless
+ // told otherwise (the `ffmpeg` tool turns threads on for itself), and
+ // a seek decodes every frame from the keyframe before it: on one
+ // thread a 4K H.264 file decoded at about 87 frames a second, and a
+ // scrub waited 400 ms for its frame. Frame threads for that walk,
+ // slice threads for files cut into slices; zero lets the codec count.
+ // SAFETY: the context is not opened yet, which is when threading is
+ // set, and both fields are plain integers.
+ unsafe {
+ let raw = context.as_mut_ptr();
+ (*raw).thread_type = ffmpeg::sys::FF_THREAD_FRAME | ffmpeg::sys::FF_THREAD_SLICE;
+ (*raw).thread_count = 0;
+ }
+ let accelerated = device.and_then(|device| {
+ device
+ .attach(&mut context)
+ .map(|format| (device.kind(), format))
+ });
+ let mut decoder = context
+ .decoder()
+ .video()
+ .map_err(|error| ffi::fail("open decoder", path, error))?;
+ if options.keyframes_only {
+ decoder.skip_frame(ffmpeg::Discard::NonKey);
+ }
+ Ok((decoder, accelerated))
+ }
+
/// How many frames have been produced so far.
pub const fn produced(&self) -> u64 {
self.produced
}
+ /// The device the codec is decoding on, or `None` in software: also
+ /// the answer once a device has failed and the reader has moved on
+ /// without it, or once the codec has shown, with its first frame, that
+ /// it declined the device for this stream.
+ pub fn hardware(&self) -> Option<HwDevice> {
+ self.hardware.map(|(device, _)| device)
+ }
+
+ /// Reopens the codec in software and carries on from `from`: the frame
+ /// at that instant is the next one out, or the one after it with
+ /// `after`. The pacer's state is kept, so what comes out is what would
+ /// have come out of the device. `None` starts over from the origin,
+ /// which is the best a stream with no timestamps allows.
+ fn resume_in_software(&mut self, from: Option<Rational>, after: bool) -> Result<()> {
+ let decoder = {
+ let stream = self
+ .input
+ .stream(self.stream)
+ .ok_or_else(|| Error::NoVideoStream {
+ path: self.path.clone(),
+ })?;
+ Self::open_codec(&self.path, &stream, &self.options, None)?.0
+ };
+ let at = from.unwrap_or(self.origin);
+ let target = ffi::av_ticks(at);
+ self.input
+ .seek(target, ..=target)
+ .map_err(|error| ffi::fail("seek", &self.path, error))?;
+ self.decoder = decoder;
+ self.hardware = None;
+ if !self.options.keyframes_only {
+ if after {
+ self.discard_through = Some(at);
+ } else {
+ self.discard_before = Some(self.discard_before.map_or(at, |before| before.max(at)));
+ }
+ }
+ Ok(())
+ }
+
/// The container seek, and the state reset that goes with it.
fn jump(&mut self, to: Rational) -> Result<()> {
let target = ffi::av_ticks(to);
@@ -408,6 +524,8 @@ impl Decoder {
// wanted, and discarding up to the target would throw it away and
// wait for the *next* keyframe, a whole group of pictures late.
self.discard_before = (!self.options.keyframes_only).then_some(to);
+ self.discard_through = None;
+ self.last_pts = None;
self.origin = to;
self.tick = 0;
self.current = None;
@@ -418,7 +536,8 @@ impl Decoder {
}
/// The next picture out of the codec, or `None` at the end of the file.
- /// Frames before the discard point never come out of here.
+ /// Frames before the discard point never come out of here, and a frame
+ /// from the device is copied back to memory before it does.
fn next_source(&mut self) -> Result<Option<Source>> {
loop {
let mut frame = Video::empty();
@@ -427,16 +546,58 @@ impl Decoder {
let pts = frame
.timestamp()
.and_then(|ticks| ffi::seconds(ticks, self.time_base));
+ if let Some((device, format)) = self.hardware {
+ if frame.format() == format {
+ frame = match hardware::download(&frame) {
+ Ok(copy) => copy,
+ Err(error) => {
+ hardware::note_fallback(
+ device,
+ "copy a frame back",
+ &error.to_string(),
+ );
+ self.resume_in_software(pts, false)?;
+ continue;
+ }
+ };
+ } else {
+ // The codec set the device up for this stream
+ // and the device said no, so the codec went on
+ // in software by itself; the frames are fine.
+ hardware::note_fallback(
+ device,
+ "take this stream",
+ "the codec decoded it in software",
+ );
+ self.hardware = None;
+ }
+ }
if let (Some(before), Some(pts)) = (self.discard_before, pts)
&& pts < before
{
continue;
}
+ if let (Some(through), Some(pts)) = (self.discard_through, pts)
+ && pts <= through
+ {
+ continue;
+ }
self.discard_before = None;
+ self.discard_through = None;
+ self.last_pts = pts;
return Ok(Some(Source { frame, pts }));
}
Err(ffmpeg::Error::Eof) => return Ok(None),
Err(error) if ffi::is_again(&error) => {}
+ Err(error) if self.hardware.is_some() => {
+ // The device choked on a picture it had accepted the
+ // stream for. Software takes over after the last frame
+ // the reader was given.
+ let (device, _) = self.hardware.take().expect("checked");
+ hardware::note_fallback(device, "decode a frame", &error.to_string());
+ self.resume_in_software(self.last_pts, self.last_pts.is_some())?;
+ continue;
+ }
Err(error) => return Err(ffi::fail("decode", &self.path, error)),
}
@@ -453,6 +614,16 @@ impl Decoder {
match self.decoder.send_packet(&packet) {
Ok(()) => break,
Err(error) if ffi::is_again(&error) => break,
+ Err(error) if self.hardware.is_some() => {
+ let (device, _) = self.hardware.take().expect("checked");
+ hardware::note_fallback(
+ device,
+ "take a packet",
+ &error.to_string(),
+ );
+ self.resume_in_software(self.last_pts, self.last_pts.is_some())?;
+ break;
+ }
Err(error) => return Err(ffi::fail("send packet", &self.path, error)),
}
}
@@ -891,4 +1062,201 @@ mod tests {
fn a_missing_file_is_an_error_not_a_panic() {
assert!(Decoder::open("does-not-exist.mp4", &DecodeOptions::default()).is_err());
}
+
+ /// A thirty-frame H.264 file whose every frame is its own flat grey,
+ /// for telling frames apart.
+ fn grey_steps(name: &str) -> PathBuf {
+ use crate::{EncodeOptions, Encoder, FrameSink};
+ let path =
+ std::env::temp_dir().join(format!("concat-decode-{name}-{}.mp4", std::process::id()));
+ let mut encoder = Encoder::create(
+ &path,
+ 64,
+ 64,
+ FrameRate::THIRTY,
+ &EncodeOptions {
+ preset: "ultrafast".to_owned(),
+ ..EncodeOptions::default()
+ },
+ )
+ .expect("the linked FFmpeg encodes h264");
+ for index in 0..30u8 {
+ let pixels = [index * 8, index * 8, index * 8, 255].repeat(64 * 64);
+ let frame = Frame::from_rgba(64, 64, pixels).expect("a frame");
+ encoder.write_frame(&frame).expect("writes");
+ }
+ encoder.finish().expect("finishes");
+ path
+ }
+
+ /// Every frame and its instant, to the end.
+ fn drain(decoder: &mut Decoder) -> Vec<(Option<Rational>, Frame)> {
+ let mut frames = Vec::new();
+ while let Some(frame) = decoder.next_frame().expect("decodes") {
+ frames.push((decoder.position(), frame));
+ }
+ frames
+ }
+
+ /// The largest difference between two frames, over every channel.
+ fn worst_difference(a: &Frame, b: &Frame) -> u32 {
+ a.pixels()
+ .iter()
+ .zip(b.pixels())
+ .map(|(a, b)| u32::from(a.abs_diff(*b)))
+ .max()
+ .unwrap_or(0)
+ }
+
+ /// Insisting on software gives software, whatever the preference, and
+ /// the frames are the frames.
+ #[test]
+ fn software_is_software_whatever_the_preference() {
+ let path = grey_steps("software");
+ let mut decoder =
+ Decoder::open(&path, &DecodeOptions::default().in_software()).expect("opens");
+ assert_eq!(decoder.hardware(), None);
+ let frames = drain(&mut decoder);
+ let _ = std::fs::remove_file(&path);
+ assert_eq!(frames.len(), 30);
+ assert_eq!(decoder.hardware(), None);
+ }
+
+ /// The device decodes the same frames at the same instants as the CPU:
+ /// the same count, the same timestamps, and pixels within the rounding
+ /// of a different colour conversion. A seek on the device lands where
+ /// a seek on the CPU does. On a machine with the device, that is; on
+ /// one without, the reader opens in software and the file still
+ /// decodes whole.
+ #[test]
+ fn hardware_decode_matches_software_in_count_and_timing() {
+ let path = grey_steps("hardware");
+ let Some(device) = HwDevice::platform_default() else {
+ return;
+ };
+ let software = DecodeOptions::default().scaled_to(64, 64).in_software();
+ let hardware = DecodeOptions::default()
+ .scaled_to(64, 64)
+ .accelerated(device);
+
+ let mut cpu = Decoder::open(&path, &software).expect("opens in software");
+ let cpu_frames = drain(&mut cpu);
+ let mut gpu = Decoder::open(&path, &hardware).expect("opens on the device");
+ let gpu_frames = drain(&mut gpu);
+ if cfg!(target_os = "macos") {
+ assert_eq!(
+ gpu.hardware(),
+ Some(HwDevice::VideoToolbox),
+ "a Mac decodes H.264 on VideoToolbox"
+ );
+ }
+
+ assert_eq!(
+ gpu_frames.len(),
+ cpu_frames.len(),
+ "the same number of frames"
+ );
+ for (index, ((cpu_at, cpu_frame), (gpu_at, gpu_frame))) in
+ cpu_frames.iter().zip(&gpu_frames).enumerate()
+ {
+ assert_eq!(gpu_at, cpu_at, "frame {index} at the same instant");
+ assert_eq!(
+ (gpu_frame.width(), gpu_frame.height()),
+ (cpu_frame.width(), cpu_frame.height())
+ );
+ let difference = worst_difference(cpu_frame, gpu_frame);
+ assert!(
+ difference <= 8,
+ "frame {index} differs by {difference} levels between the device and the CPU"
+ );
+ }
+
+ // A seek on the device lands on the same frame as one on the CPU.
+ let mut sought = Decoder::open(&path, &hardware.clone().starting_at(Rational::new(1, 2)))
+ .expect("opens at half a second on the device");
+ let landed = sought.next_frame().expect("decodes").expect("a frame");
+ let _ = std::fs::remove_file(&path);
+ let (_, fifteenth) = &cpu_frames[15];
+ let difference = worst_difference(fifteenth, &landed);
+ assert!(
+ difference <= 8,
+ "the seek landed {difference} levels from frame 15"
+ );
+ }
+
+ /// The pacer sits on top of the device exactly as it does the CPU: a
+ /// file paced to twice its rate gives the same frames either way.
+ #[test]
+ fn a_paced_hardware_decode_matches_a_paced_software_one() {
+ let path = grey_steps("paced");
+ let Some(device) = HwDevice::platform_default() else {
+ return;
+ };
+ let rate = FrameRate::new(Rational::from_int(60));
+ let software = DecodeOptions::default()
+ .scaled_to(64, 64)
+ .at_rate(rate)
+ .in_software();
+ let hardware = DecodeOptions::default()
+ .scaled_to(64, 64)
+ .at_rate(rate)
+ .accelerated(device);
+ let cpu_frames = drain(&mut Decoder::open(&path, &software).expect("opens"));
+ let gpu_frames = drain(&mut Decoder::open(&path, &hardware).expect("opens"));
+ let _ = std::fs::remove_file(&path);
+ assert!(
+ cpu_frames.len() >= 58,
+ "thirty frames at sixty a second: {}",
+ cpu_frames.len()
+ );
+ assert_eq!(gpu_frames.len(), cpu_frames.len());
+ for (index, ((cpu_at, cpu_frame), (gpu_at, gpu_frame))) in
+ cpu_frames.iter().zip(&gpu_frames).enumerate()
+ {
+ assert_eq!(gpu_at, cpu_at, "frame {index} at the same instant");
+ assert!(worst_difference(cpu_frame, gpu_frame) <= 8, "frame {index}");
+ }
+ }
+
+ /// A codec the device has no decoder for - a JPEG still, here - opens
+ /// in software straight away, with the device asked for, and decodes.
+ #[test]
+ fn a_codec_the_device_lacks_decodes_in_software() {
+ let path =
+ std::env::temp_dir().join(format!("concat-decode-jpeg-{}.jpg", std::process::id()));
+ let mut frame = Frame::black(64, 64);
+ frame.fill([200, 40, 40, 255]);
+ let bytes = crate::encode::jpeg(&frame, 90).expect("a jpeg");
+ std::fs::write(&path, bytes).expect("writes the still");
+ let device = HwDevice::platform_default().unwrap_or(HwDevice::VideoToolbox);
+ let mut decoder = Decoder::open(&path, &DecodeOptions::default().accelerated(device))
+ .expect("the still opens whatever the device says");
+ assert_eq!(
+ decoder.hardware(),
+ None,
+ "a JPEG is not the device's to decode"
+ );
+ let decoded = decoder.next_frame().expect("decodes").expect("the picture");
+ let _ = std::fs::remove_file(&path);
+ assert_eq!((decoded.width(), decoded.height()), (64, 64));
+ let red = decoded.pixels()[0];
+ assert!(red > 150, "the picture came through: red {red}");
+ }
+
+ /// A device the linked FFmpeg does not carry is asked for and quietly
+ /// not used: the file decodes in software.
+ #[test]
+ fn a_device_the_library_lacks_means_software() {
+ let path = grey_steps("absent");
+ let absent = HwDevice::ALL
+ .into_iter()
+ .find(|device| !device.linked())
+ .unwrap_or(HwDevice::Vaapi);
+ let mut decoder =
+ Decoder::open(&path, &DecodeOptions::default().accelerated(absent)).expect("opens");
+ let frames = drain(&mut decoder);
+ let _ = std::fs::remove_file(&path);
+ assert_eq!(frames.len(), 30);
+ assert_ne!(decoder.hardware(), Some(absent));
+ }
}
diff --git a/src/crates/concat-media/src/hardware.rs b/src/crates/concat-media/src/hardware.rs
new file mode 100644
index 00000000..0e23269d
--- /dev/null
+++ b/src/crates/concat-media/src/hardware.rs
@@ -0,0 +1,493 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors
+
+//! Decoding on the platform's own video hardware.
+//!
+//! Every platform the app ships on has a fixed-function video decoder that
+//! turns H.264 and HEVC into pictures at a fraction of what the CPU spends,
+//! and libavcodec drives each of them through one door: a device context
+//! from `av_hwdevice_ctx_create`, handed to the codec before it opens, and
+//! a `get_format` callback that picks the device's pixel format when the
+//! codec offers it. The frames then come out of the codec in the device's
+//! memory, and `av_hwframe_transfer_data` copies each one back into system
+//! memory, where the filtergraph in [`crate::decode`] takes over exactly as
+//! it does for a software frame.
+//!
+//! ## What decides
+//!
+//! A reader's [`DecodeOptions`](crate::DecodeOptions) carries an
+//! [`HwPolicy`]. The default, [`HwPolicy::Preference`], follows one
+//! process-wide setting, [`set_hardware_decode`], so the window's toggle
+//! reaches every reader the pool, the preview, the export and the
+//! thumbnails open without each of them being told. A reader can insist on
+//! software or on a named device instead, which is what the tests do.
+//!
+//! The device is the platform's own: VideoToolbox on macOS and iOS, D3D11VA
+//! on Windows, MediaCodec on Android. VAAPI on Linux is never chosen by
+//! default and only ever by name, because on an Intel i915 it has hung the
+//! GPU for every process on the machine until a reboot (issue #70).
+//!
+//! ## What happens when it does not work
+//!
+//! Everything falls back to software, and nothing a person sees changes but
+//! the speed. A codec the device has no decoder for goes straight to
+//! software with no message, since that is most codecs and a warning per
+//! file would be noise. A device that cannot be created, a codec that will
+//! not open with it, or a frame that will not copy back is a real failure,
+//! logged once per process, and the reader carries on in software from the
+//! frame it was at.
+//!
+//! ## What this is not, yet
+//!
+//! The frame is copied back to the CPU. On macOS the decoded picture lives
+//! in an IOSurface the GPU could sample directly, and the compositor's wgpu
+//! device could take it without the copy; that zero-copy path is the
+//! follow-up to this module, and needs the frame to stay a hardware frame
+//! all the way to the render.
+
+use std::collections::HashMap;
+use std::ptr;
+use std::sync::atomic::{AtomicU8, Ordering};
+use std::sync::{Mutex, Once};
+
+use ffmpeg_the_third as ffmpeg;
+use ffmpeg_the_third::codec::decoder;
+use ffmpeg_the_third::format::Pixel;
+use ffmpeg_the_third::sys;
+use ffmpeg_the_third::util::frame::video::Video;
+
+/// A platform's video decode hardware, as libavcodec names it.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub enum HwDevice {
+ /// Apple's, on macOS and iOS.
+ VideoToolbox,
+ /// Direct3D 11 video acceleration, on Windows.
+ D3d11va,
+ /// Android's.
+ MediaCodec,
+ /// The Linux video acceleration API. Never the default; see the module
+ /// doc and issue #70.
+ Vaapi,
+}
+
+impl HwDevice {
+ /// Every device, in the order a menu would list them.
+ pub const ALL: [HwDevice; 4] = [
+ HwDevice::VideoToolbox,
+ HwDevice::D3d11va,
+ HwDevice::MediaCodec,
+ HwDevice::Vaapi,
+ ];
+
+ /// The device this binary's platform has, or `None` where there is no
+ /// device worth choosing unasked: Linux, whose VAAPI is opt-in only.
+ pub fn platform_default() -> Option<Self> {
+ if cfg!(any(target_os = "macos", target_os = "ios")) {
+ Some(HwDevice::VideoToolbox)
+ } else if cfg!(target_os = "windows") {
+ Some(HwDevice::D3d11va)
+ } else if cfg!(target_os = "android") {
+ Some(HwDevice::MediaCodec)
+ } else {
+ None
+ }
+ }
+
+ /// What the device is called on a label.
+ pub fn label(self) -> &'static str {
+ match self {
+ HwDevice::VideoToolbox => "VideoToolbox",
+ HwDevice::D3d11va => "Direct3D 11",
+ HwDevice::MediaCodec => "MediaCodec",
+ HwDevice::Vaapi => "VAAPI",
+ }
+ }
+
+ /// What libavcodec calls the device: the name `-hwaccel` takes.
+ pub fn name(self) -> &'static str {
+ match self {
+ HwDevice::VideoToolbox => "videotoolbox",
+ HwDevice::D3d11va => "d3d11va",
+ HwDevice::MediaCodec => "mediacodec",
+ HwDevice::Vaapi => "vaapi",
+ }
+ }
+
+ /// Whether the linked FFmpeg was built with this device at all. A
+ /// device that is not here cannot be created, whatever the platform.
+ pub fn linked(self) -> bool {
+ crate::ffi::init();
+ let wanted = self.av_type();
+ let mut kind = sys::AVHWDeviceType::NONE;
+ loop {
+ // SAFETY: iterating a static table; NONE ends it.
+ kind = unsafe { sys::av_hwdevice_iterate_types(kind) };
+ if kind == sys::AVHWDeviceType::NONE {
+ return false;
+ }
+ if kind == wanted {
+ return true;
+ }
+ }
+ }
+
+ fn av_type(self) -> sys::AVHWDeviceType {
+ match self {
+ HwDevice::VideoToolbox => sys::AVHWDeviceType::VIDEOTOOLBOX,
+ HwDevice::D3d11va => sys::AVHWDeviceType::D3D11VA,
+ HwDevice::MediaCodec => sys::AVHWDeviceType::MEDIACODEC,
+ HwDevice::Vaapi => sys::AVHWDeviceType::VAAPI,
+ }
+ }
+}
+
+/// How a reader chooses between the hardware and the CPU.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub enum HwPolicy {
+ /// Follow the process-wide preference; see [`set_hardware_decode`].
+ /// The default, and what every reader the app opens uses.
+ #[default]
+ Preference,
+ /// Software, whatever the preference says.
+ Software,
+ /// This device, whatever the preference says; software if it fails.
+ Device(HwDevice),
+}
+
+impl HwPolicy {
+ /// The device this policy asks for right now, or `None` for software.
+ pub fn device(self) -> Option<HwDevice> {
+ match self {
+ HwPolicy::Preference => hardware_device(),
+ HwPolicy::Software => None,
+ HwPolicy::Device(device) => Some(device),
+ }
+ }
+}
+
+/// The process-wide preference: `0` for software, else one past the
+/// device's index in [`HwDevice::ALL`]. An atomic rather than a lock
+/// because every `Decoder::open` reads it.
+static PREFERENCE: AtomicU8 = AtomicU8::new(0);
+
+/// Turns hardware decode on or off for every reader opened from now on
+/// that follows the preference, which is all of them unless they say
+/// otherwise. On means the platform's own device, so on Linux it means
+/// nothing until [`set_hardware_device`] names one. Readers already open
+/// keep whatever they opened with.
+pub fn set_hardware_decode(on: bool) {
+ set_hardware_device(if on {
+ HwDevice::platform_default()
+ } else {
+ None
+ });
+}
+
+/// Whether the preference is currently for a device.
+pub fn hardware_decode() -> bool {
+ hardware_device().is_some()
+}
+
+/// Names the device readers that follow the preference should use, or
+/// `None` for software. The way to opt into VAAPI.
+pub fn set_hardware_device(device: Option<HwDevice>) {
+ let code = device.map_or(0, |device| {
+ HwDevice::ALL
+ .iter()
+ .position(|candidate| *candidate == device)
+ .map_or(0, |index| index as u8 + 1)
+ });
+ PREFERENCE.store(code, Ordering::Relaxed);
+}
+
+/// The device the preference names, or `None` for software.
+pub fn hardware_device() -> Option<HwDevice> {
+ match PREFERENCE.load(Ordering::Relaxed) {
+ 0 => None,
+ code => HwDevice::ALL.get(usize::from(code) - 1).copied(),
+ }
+}
+
+/// One created device context: a reference to libavutil's
+/// `AVHWDeviceContext`, shared by every reader on the device.
+pub(crate) struct Device {
+ kind: HwDevice,
+ context: *mut sys::AVBufferRef,
+}
+
+// SAFETY: an AVBufferRef's count is atomic and the device context behind
+// it is immutable once created; libavcodec itself shares one device across
+// its decoding threads. The reference is only ever cloned or dropped here
+// and read by the codec.
+unsafe impl Send for Device {}
+unsafe impl Sync for Device {}
+
+impl Device {
+ fn create(kind: HwDevice) -> Result<Self, ffmpeg::Error> {
+ let mut context: *mut sys::AVBufferRef = ptr::null_mut();
+ // SAFETY: the out-pointer is a valid local; a null device name and
+ // no options ask for the platform's default device, which is the
+ // only one a laptop has.
+ let ret = unsafe {
+ sys::av_hwdevice_ctx_create(
+ &mut context,
+ kind.av_type(),
+ ptr::null(),
+ ptr::null_mut(),
+ 0,
+ )
+ };
+ if ret < 0 || context.is_null() {
+ return Err(ffmpeg::Error::from(ret));
+ }
+ Ok(Self { kind, context })
+ }
+
+ /// Which device this is.
+ pub(crate) fn kind(&self) -> HwDevice {
+ self.kind
+ }
+
+ /// The pixel format `codec` decodes to on this device, if it can
+ /// decode on it at all.
+ fn pixel_format(&self, codec: &ffmpeg::Codec) -> Option<Pixel> {
+ hardware_format(codec.as_ptr(), self.kind.av_type()).map(Pixel::from)
+ }
+
+ /// Hands the codec context this device, before it opens. Returns the
+ /// device's pixel format when the codec can decode on it, and `None`,
+ /// having touched nothing, when it cannot: that codec is a software
+ /// one whatever the preference.
+ pub(crate) fn attach(&self, context: &mut ffmpeg::codec::Context) -> Option<Pixel> {
+ let codec = decoder::find(context.id())?;
+ let format = self.pixel_format(&codec)?;
+ // SAFETY: the context is not opened yet. The codec takes its own
+ // reference to the device and releases it when it closes.
+ unsafe {
+ let raw = context.as_mut_ptr();
+ (*raw).hw_device_ctx = sys::av_buffer_ref(self.context);
+ if (*raw).hw_device_ctx.is_null() {
+ return None;
+ }
+ (*raw).get_format = Some(pick_format);
+ }
+ Some(format)
+ }
+}
+
+impl Clone for Device {
+ fn clone(&self) -> Self {
+ // SAFETY: a new reference to a live buffer; null only when out of
+ // memory, which is not a case this process survives anyway.
+ let context = unsafe { sys::av_buffer_ref(self.context) };
+ assert!(!context.is_null(), "out of memory cloning a device context");
+ Self {
+ kind: self.kind,
+ context,
+ }
+ }
+}
+
+impl Drop for Device {
+ fn drop(&mut self) {
+ // SAFETY: our own reference, released once.
+ unsafe { sys::av_buffer_unref(&mut self.context) };
+ }
+}
+
+/// The devices created so far, one per kind, and the kinds that could not
+/// be: a device that failed once is not tried again, and not logged again.
+static DEVICES: Mutex<Option<HashMap<HwDevice, Option<Device>>>> = Mutex::new(None);
+
+/// The process's device of this kind, created on first use, or `None`
+/// when the platform has none to give.
+pub(crate) fn device(kind: HwDevice) -> Option<Device> {
+ crate::ffi::init();
+ let mut guard = DEVICES
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ let devices = guard.get_or_insert_with(HashMap::new);
+ devices
+ .entry(kind)
+ .or_insert_with(|| match Device::create(kind) {
+ Ok(device) => {
+ log::info!("video decode on {}", kind.label());
+ Some(device)
+ }
+ Err(error) => {
+ log::warn!(
+ "{} is not available, decoding in software: {error}",
+ kind.label()
+ );
+ None
+ }
+ })
+ .clone()
+}
+
+/// The pixel format `codec` decodes to on a device of type `kind`, from
+/// its table of hardware configurations, or `None` when the table has no
+/// entry for the device.
+fn hardware_format(
+ codec: *const sys::AVCodec,
+ kind: sys::AVHWDeviceType,
+) -> Option<sys::AVPixelFormat> {
+ if codec.is_null() {
+ return None;
+ }
+ let mut index = 0;
+ loop {
+ // SAFETY: the codec is a static libavcodec table entry, and the
+ // config table is walked by index until a null ends it.
+ let config = unsafe { sys::avcodec_get_hw_config(codec, index) };
+ if config.is_null() {
+ return None;
+ }
+ // SAFETY: non-null, and static like the codec.
+ let config = unsafe { &*config };
+ if config.device_type == kind
+ && config.methods & sys::AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX.0 as libc::c_int != 0
+ {
+ return Some(config.pix_fmt);
+ }
+ index += 1;
+ }
+}
+
+/// The codec's `get_format`: the device's pixel format when the codec
+/// offers it, else what libavcodec would have picked on its own. The
+/// codec calls this again, without the device's format, when setting the
+/// device up for the stream failed, and that second call is how a stream
+/// the hardware refuses ends up decoding in software with no help from us.
+unsafe extern "C" fn pick_format(
+ context: *mut sys::AVCodecContext,
+ formats: *const sys::AVPixelFormat,
+) -> sys::AVPixelFormat {
+ // SAFETY: libavcodec hands a live context and a NONE-terminated list.
+ unsafe {
+ let device = (*context).hw_device_ctx;
+ let wanted = if device.is_null() {
+ None
+ } else {
+ let kind = (*((*device).data as *const sys::AVHWDeviceContext)).type_;
+ hardware_format((*context).codec, kind)
+ };
+ if let Some(wanted) = wanted {
+ let mut cursor = formats;
+ while *cursor != sys::AVPixelFormat::NONE {
+ if *cursor == wanted {
+ return wanted;
+ }
+ cursor = cursor.add(1);
+ }
+ }
+ sys::avcodec_default_get_format(context, formats)
+ }
+}
+
+/// Copies a frame out of the device's memory into a fresh software frame,
+/// in whatever format the device hands back (NV12, for the most part), with
+/// its timestamp and colour tags intact.
+pub(crate) fn download(frame: &Video) -> Result<Video, ffmpeg::Error> {
+ let mut copy = Video::empty();
+ // SAFETY: both frames are live; the destination is empty, so the
+ // transfer allocates it in the device's first software format.
+ unsafe {
+ let ret = sys::av_hwframe_transfer_data(copy.as_mut_ptr(), frame.as_ptr(), 0);
+ if ret < 0 {
+ return Err(ffmpeg::Error::from(ret));
+ }
+ let ret = sys::av_frame_copy_props(copy.as_mut_ptr(), frame.as_ptr());
+ if ret < 0 {
+ return Err(ffmpeg::Error::from(ret));
+ }
+ }
+ Ok(copy)
+}
+
+/// Writes down, once per process, that a reader gave up on the hardware
+/// and why. Once, because the reason is the same for every file after the
+/// first and the log is a person's, not a profiler's.
+pub(crate) fn note_fallback(device: HwDevice, stage: &str, detail: &str) {
+ static ONCE: Once = Once::new();
+ ONCE.call_once(|| {
+ log::warn!(
+ "{} could not {stage}; decoding in software from here on: {detail}",
+ device.label()
+ );
+ });
+}
+
+/// The name libavutil gives a device type.
+#[cfg(test)]
+fn type_name(kind: HwDevice) -> String {
+ // SAFETY: a pointer to a static string, or null for an unknown type.
+ let raw = unsafe { sys::av_hwdevice_get_type_name(kind.av_type()) };
+ if raw.is_null() {
+ return kind.name().to_owned();
+ }
+ // SAFETY: checked non-null; a NUL-terminated static string.
+ unsafe { std::ffi::CStr::from_ptr(raw) }
+ .to_string_lossy()
+ .into_owned()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn the_platform_default_is_never_vaapi() {
+ assert_ne!(HwDevice::platform_default(), Some(HwDevice::Vaapi));
+ if cfg!(target_os = "macos") {
+ assert_eq!(HwDevice::platform_default(), Some(HwDevice::VideoToolbox));
+ }
+ }
+
+ #[test]
+ fn the_preference_round_trips() {
+ // Other tests in this process read the preference through their
+ // own readers, so this one only sets what it found.
+ let before = hardware_device();
+ set_hardware_device(Some(HwDevice::Vaapi));
+ assert_eq!(hardware_device(), Some(HwDevice::Vaapi));
+ assert_eq!(HwPolicy::Preference.device(), Some(HwDevice::Vaapi));
+ assert_eq!(HwPolicy::Software.device(), None);
+ assert_eq!(
+ HwPolicy::Device(HwDevice::D3d11va).device(),
+ Some(HwDevice::D3d11va)
+ );
+ set_hardware_device(before);
+ assert_eq!(hardware_device(), before);
+ }
+
+ #[test]
+ fn the_names_match_libavutil() {
+ for device in HwDevice::ALL {
+ assert_eq!(type_name(device), device.name(), "{device:?}");
+ }
+ }
+
+ #[test]
+ fn a_device_the_library_lacks_is_not_created() {
+ // The linked FFmpeg on a Mac has VideoToolbox and nothing else;
+ // asking for a device it lacks is a clean None, once.
+ let absent = HwDevice::ALL.into_iter().find(|device| !device.linked());
+ if let Some(absent) = absent {
+ assert!(device(absent).is_none());
+ assert!(device(absent).is_none());
+ }
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn videotoolbox_decodes_h264_and_not_jpeg() {
+ assert!(HwDevice::VideoToolbox.linked());
+ let device = device(HwDevice::VideoToolbox).expect("a Mac has VideoToolbox");
+ assert_eq!(device.kind(), HwDevice::VideoToolbox);
+ let h264 = decoder::find(ffmpeg::codec::Id::H264).expect("an H.264 decoder");
+ assert_eq!(device.pixel_format(&h264), Some(Pixel::VIDEOTOOLBOX));
+ let jpeg = decoder::find(ffmpeg::codec::Id::MJPEG).expect("a JPEG decoder");
+ assert_eq!(device.pixel_format(&jpeg), None);
+ }
+}
diff --git a/src/crates/concat-media/src/lib.rs b/src/crates/concat-media/src/lib.rs
index f09e8cfb..c64842ce 100644
--- a/src/crates/concat-media/src/lib.rs
+++ b/src/crates/concat-media/src/lib.rs
@@ -18,6 +18,7 @@ pub mod decode;
pub mod encode;
pub mod error;
mod ffi;
+pub mod hardware;
pub mod peaks;
pub mod pool;
pub mod probe;
@@ -28,6 +29,7 @@ pub use decode::{ColorSignal, DecodeOptions, Decoder, FrameSource, SeekableSourc
pub use encode::{EncodeOptions, Encoder, FrameSink, VideoCodec, jpeg};
pub use error::{Error, Result};
pub use ffi::{init, linked_version};
+pub use hardware::{HwDevice, HwPolicy, hardware_decode, set_hardware_decode};
pub use peaks::Peaks;
pub use pool::{FrameCache, ReaderPool};
pub use probe::{AudioStream, MediaInfo, VideoStream, probe};