The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
feat(media): video decodes on the platform's hardware behind a toggle
Jareer46337f52683a
diff --git a/src/Cargo.lock b/src/Cargo.lockindex a09670c4..c4e5eda4 100644--- a/src/Cargo.lock+++ b/src/Cargo.lock@@ -1317,6 +1317,7 @@ dependencies = ["concat-core","ffmpeg-the-third","libc",+ "log","thiserror 2.0.20",]diff --git a/src/crates/concat-host/tests/export.rs b/src/crates/concat-host/tests/export.rsindex 4f33157e..3aefa22f 100644--- a/src/crates/concat-host/tests/export.rs+++ b/src/crates/concat-host/tests/export.rs@@ -41,7 +41,7 @@ use concat_host::{media, projects};use concat_media::audio::{self as sound, AudioClip};use concat_media::{AudioDecoder, AudioOptions, DecodeOptions, Decoder, EncodeOptions, Encoder,- Error as MediaError, FrameSink, FrameSource, SampleFormat, VideoCodec,+ Error as MediaError, FrameSink, FrameSource, HwDevice, SampleFormat, VideoCodec,};use concat_project::animation;use concat_project::commands::{ClipMove, ClipPatch, Command, TrackFlag, TrimEdge};@@ -1190,3 +1190,71 @@ fn the_edges_export_too() {exported.expect_second(end - 0.5, 3);exported.expect_tone(start + 4.0);}++/// Hardware decode preferred, the way the settings toggle leaves it: the+/// export decodes its sources on the platform's device where there is one+/// and reads the same picture and sound back. The preference is the+/// process's, so the other scenarios running alongside share it for the+/// moment, and must not mind.+#[test]+fn an_export_decodes_on_the_hardware_when_preferred() {+ struct Preferred;+ impl Drop for Preferred {+ fn drop(&mut self) {+ concat_media::set_hardware_decode(false);+ }+ }+ let _preferred = Preferred;+ concat_media::set_hardware_decode(true);+ assert_eq!(+ concat_media::hardware_decode(),+ HwDevice::platform_default().is_some(),+ "on where the platform has a device, off where it has none"+ );++ let scratch = Scratch::new("hardware");+ let sources = Sources::make(scratch.path());+ // A reader opened the way the engine opens them follows the preference+ // onto the device, on a machine that has one.+ let mut reader = Decoder::open(&sources.peek, &DecodeOptions::default()).expect("opens");+ reader.next_frame().expect("decodes").expect("a frame");+ assert_eq!(reader.hardware(), HwDevice::platform_default());+ drop(reader);+ let mut studio = Studio::new(scratch.path(), "Hardware", video(WIDTH, HEIGHT, 30, 1));+ let peek = studio.import(&sources.peek);+ let cam = studio.import(&sources.cam);+ studio.apply(Command::AddClipAtFirstFree {+ media_id: peek,+ start: 0.0,+ });+ let clip = studio+ .apply(Command::AddClipAtFirstFree {+ media_id: cam,+ start: 6.0,+ })+ .expect("the clip has an id");+ // A retimed clip on the device: the pacing sits on top of it exactly+ // as it does the CPU.+ studio.apply(Command::SetClipSpeed {+ clip_id: clip.clone(),+ speed: 2.0,+ });+ let start = studio.clip(&clip).start;+ let end = studio.end();+ assert!(+ (end - start - 2.0).abs() < 1e-6,+ "four seconds at double is two: {start} to {end}"+ );++ for (num, den) in [(30, 1), (60000, 1001)] {+ let exported = studio.export_at(&format!("hardware at {num}/{den}"), Some((num, den)));+ exported.expect_length(end);+ exported.expect_sound(end);+ exported.expect_second(0.5, 0);+ exported.expect_second(5.9, 5);+ exported.expect_second(start + 0.25, 0);+ exported.expect_second(start + 1.75, 3);+ exported.expect_tone(5.5);+ exported.expect_quiet(4.5);+ }+}diff --git a/src/crates/concat-media/Cargo.toml b/src/crates/concat-media/Cargo.tomlindex 1ac203b3..9b509499 100644--- a/src/crates/concat-media/Cargo.toml+++ b/src/crates/concat-media/Cargo.toml@@ -15,6 +15,7 @@ concat-core.workspace = true# display-rotation side data is read from where 7.0 moved it.ffmpeg-the-third = "6"libc = "0.2"+log = "0.4"thiserror.workspace = true[lints]diff --git a/src/crates/concat-media/src/decode.rs b/src/crates/concat-media/src/decode.rsindex a6f914e4..fc310a48 100644--- a/src/crates/concat-media/src/decode.rs+++ b/src/crates/concat-media/src/decode.rs@@ -9,9 +9,19 @@//! by duplicating and dropping, repeats a still forever, and stops after a//! frame budget. Every frame comes with its real presentation timestamp.//!+//! The codec runs on the platform's video hardware when the reader's+//! [`HwPolicy`] and the process-wide preference say so, and on the CPU+//! otherwise; see [`crate::hardware`]. A frame decoded on the device is+//! copied back before the filtergraph sees it, so everything from there on+//! is the same path, and a device that fails at any point hands the reader+//! to software from the frame it was at. The frames that come out are the+//! same either way, in count, size and timing; only the pixels may differ+//! by the rounding of a different colour conversion.+//!//! ## On the unsafe in here//!-//! There is none: the wrapper crate owns every pointer. What this module+//! The wrapper crate owns every pointer, and the one raw write here is the+//! threading setting the wrapper has no accessor for. What this module//! owns is the order things happen in.use std::path::{Path, PathBuf};@@ -26,6 +36,7 @@ use ffmpeg_the_third::util::frame::video::Video;use crate::error::{Error, Result};use crate::ffi;+use crate::hardware::{self, Device, HwDevice, HwPolicy};/// Anything that yields frames in order.///@@ -101,6 +112,13 @@ pub struct DecodeOptions {/// decoded frame. Never for anything a person will watch or export: the/// frame is near the instant, not at it.pub keyframes_only: bool,+ /// Whether to decode on the platform's video hardware. The default+ /// follows the process-wide preference, [`crate::set_hardware_decode`],+ /// so the window's toggle reaches every reader without being passed+ /// through each one; a reader can insist on software or on a device.+ /// A device that fails is never an error: the reader carries on in+ /// software. See [`crate::hardware`].+ pub hardware: HwPolicy,}impl DecodeOptions {@@ -153,6 +171,19 @@ impl DecodeOptions {self.keyframes_only = true;self}++ /// Decodes on `device`, whatever the preference says, and in software+ /// if the device will not. See [`DecodeOptions::hardware`].+ pub fn accelerated(mut self, device: HwDevice) -> Self {+ self.hardware = HwPolicy::Device(device);+ self+ }++ /// Decodes in software, whatever the preference says.+ pub fn in_software(mut self) -> Self {+ self.hardware = HwPolicy::Software;+ self+ }}/// How a stream says its colour is encoded: the tags in its container,@@ -285,6 +316,15 @@ pub struct Decoder {/// Frames before this instant are decoded and discarded - the exact/// half of a seek, after the container landed on a keyframe.discard_before: Option<Rational>,+ /// Frames at or before this instant are discarded too: where a reader+ /// that moved to software mid-file picks up, after the frame the+ /// device last gave it.+ discard_through: Option<Rational>,+ /// The device the codec is on and the pixel format its frames arrive+ /// in, or `None` in software, which is also where a failed device ends.+ hardware: Option<(HwDevice, Pixel)>,+ /// The last picture's timestamp, for picking up in software after it.+ last_pts: Option<Rational>,/// The output frame instant the pacer is at, when pacing.tick: u64,origin: Rational,@@ -320,31 +360,25 @@ impl Decoder {let stream_index = stream.index();let time_base = stream.time_base();let rotation = ffi::rotation(&stream);- let parameters = stream.parameters();- let coded = (parameters.width(), parameters.height());+ let coded = {+ let parameters = stream.parameters();+ (parameters.width(), parameters.height())+ };- let mut context = ffmpeg::codec::Context::from_parameters(parameters)- .map_err(|error| ffi::fail("codec parameters", path, error))?;- // Every core. libavcodec's API opens a decoder on one thread unless- // told otherwise (the `ffmpeg` tool turns threads on for itself), and- // a seek decodes every frame from the keyframe before it: on one- // thread a 4K H.264 file decoded at about 87 frames a second, and a- // scrub waited 400 ms for its frame. Frame threads for that walk,- // slice threads for files cut into slices; zero lets the codec count.- // SAFETY: the context is not opened yet, which is when threading is- // set, and both fields are plain integers.- unsafe {- let raw = context.as_mut_ptr();- (*raw).thread_type = ffmpeg::sys::FF_THREAD_FRAME | ffmpeg::sys::FF_THREAD_SLICE;- (*raw).thread_count = 0;- }- let mut decoder = context- .decoder()- .video()- .map_err(|error| ffi::fail("open decoder", path, error))?;- if options.keyframes_only {- decoder.skip_frame(ffmpeg::Discard::NonKey);- }+ // The device the policy asks for, if the process has one. A codec+ // that will not open on it opens in software instead, and the+ // reader is none the wiser past a line in the log.+ let device = options.hardware.device().and_then(hardware::device);+ let (decoder, accelerated) = match device {+ Some(device) => match Self::open_codec(path, &stream, options, Some(&device)) {+ Ok(opened) => opened,+ Err(error) => {+ hardware::note_fallback(device.kind(), "open the decoder", &error.to_string());+ Self::open_codec(path, &stream, options, None)?+ }+ },+ None => Self::open_codec(path, &stream, options, None)?,+ };let color = ColorSignal {primaries: decoder.color_primaries(),transfer: decoder.color_transfer_characteristic(),@@ -375,6 +409,9 @@ impl Decoder {height,graph: None,discard_before: None,+ discard_through: None,+ hardware: accelerated,+ last_pts: None,tick: 0,origin: Rational::ZERO,current: None,@@ -392,11 +429,90 @@ impl Decoder {Ok(this)}+ /// Opens the stream's codec, on `device` where the codec can decode on+ /// it. Gives back the opened decoder and, when it is on the device, the+ /// device and the pixel format its frames arrive in.+ fn open_codec(+ path: &Path,+ stream: &ffmpeg::format::stream::Stream<'_>,+ options: &DecodeOptions,+ device: Option<&Device>,+ ) -> Result<(decoder::Video, Option<(HwDevice, Pixel)>)> {+ let mut context = ffmpeg::codec::Context::from_parameters(stream.parameters())+ .map_err(|error| ffi::fail("codec parameters", path, error))?;+ // Every core. libavcodec's API opens a decoder on one thread unless+ // told otherwise (the `ffmpeg` tool turns threads on for itself), and+ // a seek decodes every frame from the keyframe before it: on one+ // thread a 4K H.264 file decoded at about 87 frames a second, and a+ // scrub waited 400 ms for its frame. Frame threads for that walk,+ // slice threads for files cut into slices; zero lets the codec count.+ // SAFETY: the context is not opened yet, which is when threading is+ // set, and both fields are plain integers.+ unsafe {+ let raw = context.as_mut_ptr();+ (*raw).thread_type = ffmpeg::sys::FF_THREAD_FRAME | ffmpeg::sys::FF_THREAD_SLICE;+ (*raw).thread_count = 0;+ }+ let accelerated = device.and_then(|device| {+ device+ .attach(&mut context)+ .map(|format| (device.kind(), format))+ });+ let mut decoder = context+ .decoder()+ .video()+ .map_err(|error| ffi::fail("open decoder", path, error))?;+ if options.keyframes_only {+ decoder.skip_frame(ffmpeg::Discard::NonKey);+ }+ Ok((decoder, accelerated))+ }+/// How many frames have been produced so far.pub const fn produced(&self) -> u64 {self.produced}+ /// The device the codec is decoding on, or `None` in software: also+ /// the answer once a device has failed and the reader has moved on+ /// without it, or once the codec has shown, with its first frame, that+ /// it declined the device for this stream.+ pub fn hardware(&self) -> Option<HwDevice> {+ self.hardware.map(|(device, _)| device)+ }++ /// Reopens the codec in software and carries on from `from`: the frame+ /// at that instant is the next one out, or the one after it with+ /// `after`. The pacer's state is kept, so what comes out is what would+ /// have come out of the device. `None` starts over from the origin,+ /// which is the best a stream with no timestamps allows.+ fn resume_in_software(&mut self, from: Option<Rational>, after: bool) -> Result<()> {+ let decoder = {+ let stream = self+ .input+ .stream(self.stream)+ .ok_or_else(|| Error::NoVideoStream {+ path: self.path.clone(),+ })?;+ Self::open_codec(&self.path, &stream, &self.options, None)?.0+ };+ let at = from.unwrap_or(self.origin);+ let target = ffi::av_ticks(at);+ self.input+ .seek(target, ..=target)+ .map_err(|error| ffi::fail("seek", &self.path, error))?;+ self.decoder = decoder;+ self.hardware = None;+ if !self.options.keyframes_only {+ if after {+ self.discard_through = Some(at);+ } else {+ self.discard_before = Some(self.discard_before.map_or(at, |before| before.max(at)));+ }+ }+ Ok(())+ }+/// The container seek, and the state reset that goes with it.fn jump(&mut self, to: Rational) -> Result<()> {let target = ffi::av_ticks(to);@@ -408,6 +524,8 @@ impl Decoder {// wanted, and discarding up to the target would throw it away and// wait for the *next* keyframe, a whole group of pictures late.self.discard_before = (!self.options.keyframes_only).then_some(to);+ self.discard_through = None;+ self.last_pts = None;self.origin = to;self.tick = 0;self.current = None;@@ -418,7 +536,8 @@ impl Decoder {}/// The next picture out of the codec, or `None` at the end of the file.- /// Frames before the discard point never come out of here.+ /// Frames before the discard point never come out of here, and a frame+ /// from the device is copied back to memory before it does.fn next_source(&mut self) -> Result<Option<Source>> {loop {let mut frame = Video::empty();@@ -427,16 +546,58 @@ impl Decoder {let pts = frame.timestamp().and_then(|ticks| ffi::seconds(ticks, self.time_base));+ if let Some((device, format)) = self.hardware {+ if frame.format() == format {+ frame = match hardware::download(&frame) {+ Ok(copy) => copy,+ Err(error) => {+ hardware::note_fallback(+ device,+ "copy a frame back",+ &error.to_string(),+ );+ self.resume_in_software(pts, false)?;+ continue;+ }+ };+ } else {+ // The codec set the device up for this stream+ // and the device said no, so the codec went on+ // in software by itself; the frames are fine.+ hardware::note_fallback(+ device,+ "take this stream",+ "the codec decoded it in software",+ );+ self.hardware = None;+ }+ }if let (Some(before), Some(pts)) = (self.discard_before, pts)&& pts < before{continue;}+ if let (Some(through), Some(pts)) = (self.discard_through, pts)+ && pts <= through+ {+ continue;+ }self.discard_before = None;+ self.discard_through = None;+ self.last_pts = pts;return Ok(Some(Source { frame, pts }));}Err(ffmpeg::Error::Eof) => return Ok(None),Err(error) if ffi::is_again(&error) => {}+ Err(error) if self.hardware.is_some() => {+ // The device choked on a picture it had accepted the+ // stream for. Software takes over after the last frame+ // the reader was given.+ let (device, _) = self.hardware.take().expect("checked");+ hardware::note_fallback(device, "decode a frame", &error.to_string());+ self.resume_in_software(self.last_pts, self.last_pts.is_some())?;+ continue;+ }Err(error) => return Err(ffi::fail("decode", &self.path, error)),}@@ -453,6 +614,16 @@ impl Decoder {match self.decoder.send_packet(&packet) {Ok(()) => break,Err(error) if ffi::is_again(&error) => break,+ Err(error) if self.hardware.is_some() => {+ let (device, _) = self.hardware.take().expect("checked");+ hardware::note_fallback(+ device,+ "take a packet",+ &error.to_string(),+ );+ self.resume_in_software(self.last_pts, self.last_pts.is_some())?;+ break;+ }Err(error) => return Err(ffi::fail("send packet", &self.path, error)),}}@@ -891,4 +1062,201 @@ mod tests {fn a_missing_file_is_an_error_not_a_panic() {assert!(Decoder::open("does-not-exist.mp4", &DecodeOptions::default()).is_err());}++ /// A thirty-frame H.264 file whose every frame is its own flat grey,+ /// for telling frames apart.+ fn grey_steps(name: &str) -> PathBuf {+ use crate::{EncodeOptions, Encoder, FrameSink};+ let path =+ std::env::temp_dir().join(format!("concat-decode-{name}-{}.mp4", std::process::id()));+ let mut encoder = Encoder::create(+ &path,+ 64,+ 64,+ FrameRate::THIRTY,+ &EncodeOptions {+ preset: "ultrafast".to_owned(),+ ..EncodeOptions::default()+ },+ )+ .expect("the linked FFmpeg encodes h264");+ for index in 0..30u8 {+ let pixels = [index * 8, index * 8, index * 8, 255].repeat(64 * 64);+ let frame = Frame::from_rgba(64, 64, pixels).expect("a frame");+ encoder.write_frame(&frame).expect("writes");+ }+ encoder.finish().expect("finishes");+ path+ }++ /// Every frame and its instant, to the end.+ fn drain(decoder: &mut Decoder) -> Vec<(Option<Rational>, Frame)> {+ let mut frames = Vec::new();+ while let Some(frame) = decoder.next_frame().expect("decodes") {+ frames.push((decoder.position(), frame));+ }+ frames+ }++ /// The largest difference between two frames, over every channel.+ fn worst_difference(a: &Frame, b: &Frame) -> u32 {+ a.pixels()+ .iter()+ .zip(b.pixels())+ .map(|(a, b)| u32::from(a.abs_diff(*b)))+ .max()+ .unwrap_or(0)+ }++ /// Insisting on software gives software, whatever the preference, and+ /// the frames are the frames.+ #[test]+ fn software_is_software_whatever_the_preference() {+ let path = grey_steps("software");+ let mut decoder =+ Decoder::open(&path, &DecodeOptions::default().in_software()).expect("opens");+ assert_eq!(decoder.hardware(), None);+ let frames = drain(&mut decoder);+ let _ = std::fs::remove_file(&path);+ assert_eq!(frames.len(), 30);+ assert_eq!(decoder.hardware(), None);+ }++ /// The device decodes the same frames at the same instants as the CPU:+ /// the same count, the same timestamps, and pixels within the rounding+ /// of a different colour conversion. A seek on the device lands where+ /// a seek on the CPU does. On a machine with the device, that is; on+ /// one without, the reader opens in software and the file still+ /// decodes whole.+ #[test]+ fn hardware_decode_matches_software_in_count_and_timing() {+ let path = grey_steps("hardware");+ let Some(device) = HwDevice::platform_default() else {+ return;+ };+ let software = DecodeOptions::default().scaled_to(64, 64).in_software();+ let hardware = DecodeOptions::default()+ .scaled_to(64, 64)+ .accelerated(device);++ let mut cpu = Decoder::open(&path, &software).expect("opens in software");+ let cpu_frames = drain(&mut cpu);+ let mut gpu = Decoder::open(&path, &hardware).expect("opens on the device");+ let gpu_frames = drain(&mut gpu);+ if cfg!(target_os = "macos") {+ assert_eq!(+ gpu.hardware(),+ Some(HwDevice::VideoToolbox),+ "a Mac decodes H.264 on VideoToolbox"+ );+ }++ assert_eq!(+ gpu_frames.len(),+ cpu_frames.len(),+ "the same number of frames"+ );+ for (index, ((cpu_at, cpu_frame), (gpu_at, gpu_frame))) in+ cpu_frames.iter().zip(&gpu_frames).enumerate()+ {+ assert_eq!(gpu_at, cpu_at, "frame {index} at the same instant");+ assert_eq!(+ (gpu_frame.width(), gpu_frame.height()),+ (cpu_frame.width(), cpu_frame.height())+ );+ let difference = worst_difference(cpu_frame, gpu_frame);+ assert!(+ difference <= 8,+ "frame {index} differs by {difference} levels between the device and the CPU"+ );+ }++ // A seek on the device lands on the same frame as one on the CPU.+ let mut sought = Decoder::open(&path, &hardware.clone().starting_at(Rational::new(1, 2)))+ .expect("opens at half a second on the device");+ let landed = sought.next_frame().expect("decodes").expect("a frame");+ let _ = std::fs::remove_file(&path);+ let (_, fifteenth) = &cpu_frames[15];+ let difference = worst_difference(fifteenth, &landed);+ assert!(+ difference <= 8,+ "the seek landed {difference} levels from frame 15"+ );+ }++ /// The pacer sits on top of the device exactly as it does the CPU: a+ /// file paced to twice its rate gives the same frames either way.+ #[test]+ fn a_paced_hardware_decode_matches_a_paced_software_one() {+ let path = grey_steps("paced");+ let Some(device) = HwDevice::platform_default() else {+ return;+ };+ let rate = FrameRate::new(Rational::from_int(60));+ let software = DecodeOptions::default()+ .scaled_to(64, 64)+ .at_rate(rate)+ .in_software();+ let hardware = DecodeOptions::default()+ .scaled_to(64, 64)+ .at_rate(rate)+ .accelerated(device);+ let cpu_frames = drain(&mut Decoder::open(&path, &software).expect("opens"));+ let gpu_frames = drain(&mut Decoder::open(&path, &hardware).expect("opens"));+ let _ = std::fs::remove_file(&path);+ assert!(+ cpu_frames.len() >= 58,+ "thirty frames at sixty a second: {}",+ cpu_frames.len()+ );+ assert_eq!(gpu_frames.len(), cpu_frames.len());+ for (index, ((cpu_at, cpu_frame), (gpu_at, gpu_frame))) in+ cpu_frames.iter().zip(&gpu_frames).enumerate()+ {+ assert_eq!(gpu_at, cpu_at, "frame {index} at the same instant");+ assert!(worst_difference(cpu_frame, gpu_frame) <= 8, "frame {index}");+ }+ }++ /// A codec the device has no decoder for - a JPEG still, here - opens+ /// in software straight away, with the device asked for, and decodes.+ #[test]+ fn a_codec_the_device_lacks_decodes_in_software() {+ let path =+ std::env::temp_dir().join(format!("concat-decode-jpeg-{}.jpg", std::process::id()));+ let mut frame = Frame::black(64, 64);+ frame.fill([200, 40, 40, 255]);+ let bytes = crate::encode::jpeg(&frame, 90).expect("a jpeg");+ std::fs::write(&path, bytes).expect("writes the still");+ let device = HwDevice::platform_default().unwrap_or(HwDevice::VideoToolbox);+ let mut decoder = Decoder::open(&path, &DecodeOptions::default().accelerated(device))+ .expect("the still opens whatever the device says");+ assert_eq!(+ decoder.hardware(),+ None,+ "a JPEG is not the device's to decode"+ );+ let decoded = decoder.next_frame().expect("decodes").expect("the picture");+ let _ = std::fs::remove_file(&path);+ assert_eq!((decoded.width(), decoded.height()), (64, 64));+ let red = decoded.pixels()[0];+ assert!(red > 150, "the picture came through: red {red}");+ }++ /// A device the linked FFmpeg does not carry is asked for and quietly+ /// not used: the file decodes in software.+ #[test]+ fn a_device_the_library_lacks_means_software() {+ let path = grey_steps("absent");+ let absent = HwDevice::ALL+ .into_iter()+ .find(|device| !device.linked())+ .unwrap_or(HwDevice::Vaapi);+ let mut decoder =+ Decoder::open(&path, &DecodeOptions::default().accelerated(absent)).expect("opens");+ let frames = drain(&mut decoder);+ let _ = std::fs::remove_file(&path);+ assert_eq!(frames.len(), 30);+ assert_ne!(decoder.hardware(), Some(absent));+ }}diff --git a/src/crates/concat-media/src/hardware.rs b/src/crates/concat-media/src/hardware.rsnew file mode 100644index 00000000..0e23269d--- /dev/null+++ b/src/crates/concat-media/src/hardware.rs@@ -0,0 +1,493 @@+// SPDX-License-Identifier: AGPL-3.0-or-later+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors++//! Decoding on the platform's own video hardware.+//!+//! Every platform the app ships on has a fixed-function video decoder that+//! turns H.264 and HEVC into pictures at a fraction of what the CPU spends,+//! and libavcodec drives each of them through one door: a device context+//! from `av_hwdevice_ctx_create`, handed to the codec before it opens, and+//! a `get_format` callback that picks the device's pixel format when the+//! codec offers it. The frames then come out of the codec in the device's+//! memory, and `av_hwframe_transfer_data` copies each one back into system+//! memory, where the filtergraph in [`crate::decode`] takes over exactly as+//! it does for a software frame.+//!+//! ## What decides+//!+//! A reader's [`DecodeOptions`](crate::DecodeOptions) carries an+//! [`HwPolicy`]. The default, [`HwPolicy::Preference`], follows one+//! process-wide setting, [`set_hardware_decode`], so the window's toggle+//! reaches every reader the pool, the preview, the export and the+//! thumbnails open without each of them being told. A reader can insist on+//! software or on a named device instead, which is what the tests do.+//!+//! The device is the platform's own: VideoToolbox on macOS and iOS, D3D11VA+//! on Windows, MediaCodec on Android. VAAPI on Linux is never chosen by+//! default and only ever by name, because on an Intel i915 it has hung the+//! GPU for every process on the machine until a reboot (issue #70).+//!+//! ## What happens when it does not work+//!+//! Everything falls back to software, and nothing a person sees changes but+//! the speed. A codec the device has no decoder for goes straight to+//! software with no message, since that is most codecs and a warning per+//! file would be noise. A device that cannot be created, a codec that will+//! not open with it, or a frame that will not copy back is a real failure,+//! logged once per process, and the reader carries on in software from the+//! frame it was at.+//!+//! ## What this is not, yet+//!+//! The frame is copied back to the CPU. On macOS the decoded picture lives+//! in an IOSurface the GPU could sample directly, and the compositor's wgpu+//! device could take it without the copy; that zero-copy path is the+//! follow-up to this module, and needs the frame to stay a hardware frame+//! all the way to the render.++use std::collections::HashMap;+use std::ptr;+use std::sync::atomic::{AtomicU8, Ordering};+use std::sync::{Mutex, Once};++use ffmpeg_the_third as ffmpeg;+use ffmpeg_the_third::codec::decoder;+use ffmpeg_the_third::format::Pixel;+use ffmpeg_the_third::sys;+use ffmpeg_the_third::util::frame::video::Video;++/// A platform's video decode hardware, as libavcodec names it.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]+pub enum HwDevice {+ /// Apple's, on macOS and iOS.+ VideoToolbox,+ /// Direct3D 11 video acceleration, on Windows.+ D3d11va,+ /// Android's.+ MediaCodec,+ /// The Linux video acceleration API. Never the default; see the module+ /// doc and issue #70.+ Vaapi,+}++impl HwDevice {+ /// Every device, in the order a menu would list them.+ pub const ALL: [HwDevice; 4] = [+ HwDevice::VideoToolbox,+ HwDevice::D3d11va,+ HwDevice::MediaCodec,+ HwDevice::Vaapi,+ ];++ /// The device this binary's platform has, or `None` where there is no+ /// device worth choosing unasked: Linux, whose VAAPI is opt-in only.+ pub fn platform_default() -> Option<Self> {+ if cfg!(any(target_os = "macos", target_os = "ios")) {+ Some(HwDevice::VideoToolbox)+ } else if cfg!(target_os = "windows") {+ Some(HwDevice::D3d11va)+ } else if cfg!(target_os = "android") {+ Some(HwDevice::MediaCodec)+ } else {+ None+ }+ }++ /// What the device is called on a label.+ pub fn label(self) -> &'static str {+ match self {+ HwDevice::VideoToolbox => "VideoToolbox",+ HwDevice::D3d11va => "Direct3D 11",+ HwDevice::MediaCodec => "MediaCodec",+ HwDevice::Vaapi => "VAAPI",+ }+ }++ /// What libavcodec calls the device: the name `-hwaccel` takes.+ pub fn name(self) -> &'static str {+ match self {+ HwDevice::VideoToolbox => "videotoolbox",+ HwDevice::D3d11va => "d3d11va",+ HwDevice::MediaCodec => "mediacodec",+ HwDevice::Vaapi => "vaapi",+ }+ }++ /// Whether the linked FFmpeg was built with this device at all. A+ /// device that is not here cannot be created, whatever the platform.+ pub fn linked(self) -> bool {+ crate::ffi::init();+ let wanted = self.av_type();+ let mut kind = sys::AVHWDeviceType::NONE;+ loop {+ // SAFETY: iterating a static table; NONE ends it.+ kind = unsafe { sys::av_hwdevice_iterate_types(kind) };+ if kind == sys::AVHWDeviceType::NONE {+ return false;+ }+ if kind == wanted {+ return true;+ }+ }+ }++ fn av_type(self) -> sys::AVHWDeviceType {+ match self {+ HwDevice::VideoToolbox => sys::AVHWDeviceType::VIDEOTOOLBOX,+ HwDevice::D3d11va => sys::AVHWDeviceType::D3D11VA,+ HwDevice::MediaCodec => sys::AVHWDeviceType::MEDIACODEC,+ HwDevice::Vaapi => sys::AVHWDeviceType::VAAPI,+ }+ }+}++/// How a reader chooses between the hardware and the CPU.+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]+pub enum HwPolicy {+ /// Follow the process-wide preference; see [`set_hardware_decode`].+ /// The default, and what every reader the app opens uses.+ #[default]+ Preference,+ /// Software, whatever the preference says.+ Software,+ /// This device, whatever the preference says; software if it fails.+ Device(HwDevice),+}++impl HwPolicy {+ /// The device this policy asks for right now, or `None` for software.+ pub fn device(self) -> Option<HwDevice> {+ match self {+ HwPolicy::Preference => hardware_device(),+ HwPolicy::Software => None,+ HwPolicy::Device(device) => Some(device),+ }+ }+}++/// The process-wide preference: `0` for software, else one past the+/// device's index in [`HwDevice::ALL`]. An atomic rather than a lock+/// because every `Decoder::open` reads it.+static PREFERENCE: AtomicU8 = AtomicU8::new(0);++/// Turns hardware decode on or off for every reader opened from now on+/// that follows the preference, which is all of them unless they say+/// otherwise. On means the platform's own device, so on Linux it means+/// nothing until [`set_hardware_device`] names one. Readers already open+/// keep whatever they opened with.+pub fn set_hardware_decode(on: bool) {+ set_hardware_device(if on {+ HwDevice::platform_default()+ } else {+ None+ });+}++/// Whether the preference is currently for a device.+pub fn hardware_decode() -> bool {+ hardware_device().is_some()+}++/// Names the device readers that follow the preference should use, or+/// `None` for software. The way to opt into VAAPI.+pub fn set_hardware_device(device: Option<HwDevice>) {+ let code = device.map_or(0, |device| {+ HwDevice::ALL+ .iter()+ .position(|candidate| *candidate == device)+ .map_or(0, |index| index as u8 + 1)+ });+ PREFERENCE.store(code, Ordering::Relaxed);+}++/// The device the preference names, or `None` for software.+pub fn hardware_device() -> Option<HwDevice> {+ match PREFERENCE.load(Ordering::Relaxed) {+ 0 => None,+ code => HwDevice::ALL.get(usize::from(code) - 1).copied(),+ }+}++/// One created device context: a reference to libavutil's+/// `AVHWDeviceContext`, shared by every reader on the device.+pub(crate) struct Device {+ kind: HwDevice,+ context: *mut sys::AVBufferRef,+}++// SAFETY: an AVBufferRef's count is atomic and the device context behind+// it is immutable once created; libavcodec itself shares one device across+// its decoding threads. The reference is only ever cloned or dropped here+// and read by the codec.+unsafe impl Send for Device {}+unsafe impl Sync for Device {}++impl Device {+ fn create(kind: HwDevice) -> Result<Self, ffmpeg::Error> {+ let mut context: *mut sys::AVBufferRef = ptr::null_mut();+ // SAFETY: the out-pointer is a valid local; a null device name and+ // no options ask for the platform's default device, which is the+ // only one a laptop has.+ let ret = unsafe {+ sys::av_hwdevice_ctx_create(+ &mut context,+ kind.av_type(),+ ptr::null(),+ ptr::null_mut(),+ 0,+ )+ };+ if ret < 0 || context.is_null() {+ return Err(ffmpeg::Error::from(ret));+ }+ Ok(Self { kind, context })+ }++ /// Which device this is.+ pub(crate) fn kind(&self) -> HwDevice {+ self.kind+ }++ /// The pixel format `codec` decodes to on this device, if it can+ /// decode on it at all.+ fn pixel_format(&self, codec: &ffmpeg::Codec) -> Option<Pixel> {+ hardware_format(codec.as_ptr(), self.kind.av_type()).map(Pixel::from)+ }++ /// Hands the codec context this device, before it opens. Returns the+ /// device's pixel format when the codec can decode on it, and `None`,+ /// having touched nothing, when it cannot: that codec is a software+ /// one whatever the preference.+ pub(crate) fn attach(&self, context: &mut ffmpeg::codec::Context) -> Option<Pixel> {+ let codec = decoder::find(context.id())?;+ let format = self.pixel_format(&codec)?;+ // SAFETY: the context is not opened yet. The codec takes its own+ // reference to the device and releases it when it closes.+ unsafe {+ let raw = context.as_mut_ptr();+ (*raw).hw_device_ctx = sys::av_buffer_ref(self.context);+ if (*raw).hw_device_ctx.is_null() {+ return None;+ }+ (*raw).get_format = Some(pick_format);+ }+ Some(format)+ }+}++impl Clone for Device {+ fn clone(&self) -> Self {+ // SAFETY: a new reference to a live buffer; null only when out of+ // memory, which is not a case this process survives anyway.+ let context = unsafe { sys::av_buffer_ref(self.context) };+ assert!(!context.is_null(), "out of memory cloning a device context");+ Self {+ kind: self.kind,+ context,+ }+ }+}++impl Drop for Device {+ fn drop(&mut self) {+ // SAFETY: our own reference, released once.+ unsafe { sys::av_buffer_unref(&mut self.context) };+ }+}++/// The devices created so far, one per kind, and the kinds that could not+/// be: a device that failed once is not tried again, and not logged again.+static DEVICES: Mutex<Option<HashMap<HwDevice, Option<Device>>>> = Mutex::new(None);++/// The process's device of this kind, created on first use, or `None`+/// when the platform has none to give.+pub(crate) fn device(kind: HwDevice) -> Option<Device> {+ crate::ffi::init();+ let mut guard = DEVICES+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner());+ let devices = guard.get_or_insert_with(HashMap::new);+ devices+ .entry(kind)+ .or_insert_with(|| match Device::create(kind) {+ Ok(device) => {+ log::info!("video decode on {}", kind.label());+ Some(device)+ }+ Err(error) => {+ log::warn!(+ "{} is not available, decoding in software: {error}",+ kind.label()+ );+ None+ }+ })+ .clone()+}++/// The pixel format `codec` decodes to on a device of type `kind`, from+/// its table of hardware configurations, or `None` when the table has no+/// entry for the device.+fn hardware_format(+ codec: *const sys::AVCodec,+ kind: sys::AVHWDeviceType,+) -> Option<sys::AVPixelFormat> {+ if codec.is_null() {+ return None;+ }+ let mut index = 0;+ loop {+ // SAFETY: the codec is a static libavcodec table entry, and the+ // config table is walked by index until a null ends it.+ let config = unsafe { sys::avcodec_get_hw_config(codec, index) };+ if config.is_null() {+ return None;+ }+ // SAFETY: non-null, and static like the codec.+ let config = unsafe { &*config };+ if config.device_type == kind+ && config.methods & sys::AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX.0 as libc::c_int != 0+ {+ return Some(config.pix_fmt);+ }+ index += 1;+ }+}++/// The codec's `get_format`: the device's pixel format when the codec+/// offers it, else what libavcodec would have picked on its own. The+/// codec calls this again, without the device's format, when setting the+/// device up for the stream failed, and that second call is how a stream+/// the hardware refuses ends up decoding in software with no help from us.+unsafe extern "C" fn pick_format(+ context: *mut sys::AVCodecContext,+ formats: *const sys::AVPixelFormat,+) -> sys::AVPixelFormat {+ // SAFETY: libavcodec hands a live context and a NONE-terminated list.+ unsafe {+ let device = (*context).hw_device_ctx;+ let wanted = if device.is_null() {+ None+ } else {+ let kind = (*((*device).data as *const sys::AVHWDeviceContext)).type_;+ hardware_format((*context).codec, kind)+ };+ if let Some(wanted) = wanted {+ let mut cursor = formats;+ while *cursor != sys::AVPixelFormat::NONE {+ if *cursor == wanted {+ return wanted;+ }+ cursor = cursor.add(1);+ }+ }+ sys::avcodec_default_get_format(context, formats)+ }+}++/// Copies a frame out of the device's memory into a fresh software frame,+/// in whatever format the device hands back (NV12, for the most part), with+/// its timestamp and colour tags intact.+pub(crate) fn download(frame: &Video) -> Result<Video, ffmpeg::Error> {+ let mut copy = Video::empty();+ // SAFETY: both frames are live; the destination is empty, so the+ // transfer allocates it in the device's first software format.+ unsafe {+ let ret = sys::av_hwframe_transfer_data(copy.as_mut_ptr(), frame.as_ptr(), 0);+ if ret < 0 {+ return Err(ffmpeg::Error::from(ret));+ }+ let ret = sys::av_frame_copy_props(copy.as_mut_ptr(), frame.as_ptr());+ if ret < 0 {+ return Err(ffmpeg::Error::from(ret));+ }+ }+ Ok(copy)+}++/// Writes down, once per process, that a reader gave up on the hardware+/// and why. Once, because the reason is the same for every file after the+/// first and the log is a person's, not a profiler's.+pub(crate) fn note_fallback(device: HwDevice, stage: &str, detail: &str) {+ static ONCE: Once = Once::new();+ ONCE.call_once(|| {+ log::warn!(+ "{} could not {stage}; decoding in software from here on: {detail}",+ device.label()+ );+ });+}++/// The name libavutil gives a device type.+#[cfg(test)]+fn type_name(kind: HwDevice) -> String {+ // SAFETY: a pointer to a static string, or null for an unknown type.+ let raw = unsafe { sys::av_hwdevice_get_type_name(kind.av_type()) };+ if raw.is_null() {+ return kind.name().to_owned();+ }+ // SAFETY: checked non-null; a NUL-terminated static string.+ unsafe { std::ffi::CStr::from_ptr(raw) }+ .to_string_lossy()+ .into_owned()+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn the_platform_default_is_never_vaapi() {+ assert_ne!(HwDevice::platform_default(), Some(HwDevice::Vaapi));+ if cfg!(target_os = "macos") {+ assert_eq!(HwDevice::platform_default(), Some(HwDevice::VideoToolbox));+ }+ }++ #[test]+ fn the_preference_round_trips() {+ // Other tests in this process read the preference through their+ // own readers, so this one only sets what it found.+ let before = hardware_device();+ set_hardware_device(Some(HwDevice::Vaapi));+ assert_eq!(hardware_device(), Some(HwDevice::Vaapi));+ assert_eq!(HwPolicy::Preference.device(), Some(HwDevice::Vaapi));+ assert_eq!(HwPolicy::Software.device(), None);+ assert_eq!(+ HwPolicy::Device(HwDevice::D3d11va).device(),+ Some(HwDevice::D3d11va)+ );+ set_hardware_device(before);+ assert_eq!(hardware_device(), before);+ }++ #[test]+ fn the_names_match_libavutil() {+ for device in HwDevice::ALL {+ assert_eq!(type_name(device), device.name(), "{device:?}");+ }+ }++ #[test]+ fn a_device_the_library_lacks_is_not_created() {+ // The linked FFmpeg on a Mac has VideoToolbox and nothing else;+ // asking for a device it lacks is a clean None, once.+ let absent = HwDevice::ALL.into_iter().find(|device| !device.linked());+ if let Some(absent) = absent {+ assert!(device(absent).is_none());+ assert!(device(absent).is_none());+ }+ }++ #[cfg(target_os = "macos")]+ #[test]+ fn videotoolbox_decodes_h264_and_not_jpeg() {+ assert!(HwDevice::VideoToolbox.linked());+ let device = device(HwDevice::VideoToolbox).expect("a Mac has VideoToolbox");+ assert_eq!(device.kind(), HwDevice::VideoToolbox);+ let h264 = decoder::find(ffmpeg::codec::Id::H264).expect("an H.264 decoder");+ assert_eq!(device.pixel_format(&h264), Some(Pixel::VIDEOTOOLBOX));+ let jpeg = decoder::find(ffmpeg::codec::Id::MJPEG).expect("a JPEG decoder");+ assert_eq!(device.pixel_format(&jpeg), None);+ }+}diff --git a/src/crates/concat-media/src/lib.rs b/src/crates/concat-media/src/lib.rsindex f09e8cfb..c64842ce 100644--- a/src/crates/concat-media/src/lib.rs+++ b/src/crates/concat-media/src/lib.rs@@ -18,6 +18,7 @@ pub mod decode;pub mod encode;pub mod error;mod ffi;+pub mod hardware;pub mod peaks;pub mod pool;pub mod probe;@@ -28,6 +29,7 @@ pub use decode::{ColorSignal, DecodeOptions, Decoder, FrameSource, SeekableSourcpub use encode::{EncodeOptions, Encoder, FrameSink, VideoCodec, jpeg};pub use error::{Error, Result};pub use ffi::{init, linked_version};+pub use hardware::{HwDevice, HwPolicy, hardware_decode, set_hardware_decode};pub use peaks::Peaks;pub use pool::{FrameCache, ReaderPool};pub use probe::{AudioStream, MediaInfo, VideoStream, probe};