The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
perf(decode): each stream goes to whichever of the CPU and the chip decodes it faster
Jareer62273e4f27e8
diff --git a/src/crates/concat-host/tests/export.rs b/src/crates/concat-host/tests/export.rsindex 6f3b28d2..92f47ce2 100644--- a/src/crates/concat-host/tests/export.rs+++ b/src/crates/concat-host/tests/export.rs@@ -83,9 +83,9 @@ fn second_of_colour(rgb: [u8; 3]) -> Option<u32> {/// A silent picture of `seconds` at `rate`, each frame the colour of its/// source second.-fn picture(path: &Path, rate: FrameRate, seconds: u32) {+fn picture(path: &Path, rate: FrameRate, seconds: u32, codec: VideoCodec) {let options = EncodeOptions {- codec: VideoCodec::H264,+ codec,preset: "ultrafast".to_owned(),crf: 16,rate_mode: RateMode::Vbr,@@ -187,25 +187,30 @@ struct Sources {impl Sources {fn make(dir: &Path) -> Sources {+ Sources::make_in(dir, VideoCodec::H264)+ }++ /// The same set with every picture in `codec`.+ fn make_in(dir: &Path, codec: VideoCodec) -> Sources {let wav = dir.join("clock.wav");clock_wav(&wav, 6);let aac = dir.join("clock.m4a");clock_aac(&wav, &aac, 6);let peek_picture = dir.join("peek-picture.mp4");- picture(&peek_picture, FrameRate::SIXTY, 6);+ picture(&peek_picture, FrameRate::SIXTY, 6, codec);let peek = dir.join("peek.mp4");sound::mux(&peek_picture, &aac, &peek).expect("joins the screen recording");let cam_picture = dir.join("cam-picture.mp4");- picture(&cam_picture, FrameRate::THIRTY, 4);+ picture(&cam_picture, FrameRate::THIRTY, 4, codec);let cam_sound = dir.join("cam.m4a");clock_aac(&wav, &cam_sound, 4);let cam = dir.join("cam.mp4");sound::mux(&cam_picture, &cam_sound, &cam).expect("joins the camera clip");let silent = dir.join("silent.mp4");- picture(&silent, FrameRate::FILM, 3);+ picture(&silent, FrameRate::FILM, 3, codec);let still_path = dir.join("still.jpg");still(&still_path);@@ -1270,7 +1275,10 @@ fn an_export_decodes_on_the_hardware_when_preferred() {);let scratch = Scratch::new("hardware");- let sources = Sources::make(scratch.path());+ // HEVC, not the H.264 the other scenarios use: 8-bit H.264 is the one+ // stream the preference leaves on the CPU, which decodes it faster+ // (concat_media::hardware::hardware_wins).+ let sources = Sources::make_in(scratch.path(), VideoCodec::Hevc);// A reader opened the way the engine opens them follows the preference// onto the device, on a machine that has one.let mut reader = Decoder::open(&sources.peek, &DecodeOptions::default()).expect("opens");diff --git a/src/crates/concat-media/src/decode.rs b/src/crates/concat-media/src/decode.rsindex f033fe48..eda0593c 100644--- a/src/crates/concat-media/src/decode.rs+++ b/src/crates/concat-media/src/decode.rs@@ -556,10 +556,24 @@ impl Decoder {(parameters.width(), parameters.height())};- // The device the policy asks for, if the process has one. A codec- // that will not open on it opens in software instead, and the+ // The device the policy asks for this stream, if the process has+ // one; see `hardware::hardware_wins` for which streams it wants. A+ // codec that will not open on it opens in software instead, and the// reader is none the wiser past a line in the log.- let device = options.hardware.device().and_then(hardware::device);+ let (codec, format) = {+ let parameters = stream.parameters();+ // SAFETY: a live stream's codec parameters; for a video stream+ // `format` holds an AVPixelFormat.+ let raw = unsafe { (*parameters.as_ptr()).format };+ (+ parameters.id(),+ Pixel::from(ffmpeg::sys::AVPixelFormat(raw)),+ )+ };+ let device = options+ .hardware+ .device_for(codec, format)+ .and_then(hardware::device);let (decoder, accelerated) = match device {Some(device) => match Self::open_codec(path, &stream, options, Some(&device)) {Ok(opened) => opened,diff --git a/src/crates/concat-media/src/hardware.rs b/src/crates/concat-media/src/hardware.rsindex 0e23269d..10adc50a 100644--- a/src/crates/concat-media/src/hardware.rs+++ b/src/crates/concat-media/src/hardware.rs@@ -17,10 +17,13 @@//!//! A reader's [`DecodeOptions`](crate::DecodeOptions) carries an//! [`HwPolicy`]. The default, [`HwPolicy::Preference`], follows one-//! process-wide setting, [`set_hardware_decode`], so the window's toggle-//! reaches every reader the pool, the preview, the export and the-//! thumbnails open without each of them being told. A reader can insist on-//! software or on a named device instead, which is what the tests do.+//! process-wide setting, [`set_hardware_decode`], which the window turns on+//! at start and nothing turns off, so every reader the pool, the preview,+//! the export and the thumbnails open follows it without being told. It+//! asks for the device stream by stream, only where the hardware is faster+//! ([`hardware_wins`]): not for 8-bit H.264, which the CPU decodes at twice+//! the chip's rate or better. A reader can insist on software or on a named+//! device instead, which is what the tests do.//!//! The device is the platform's own: VideoToolbox on macOS and iOS, D3D11VA//! on Windows, MediaCodec on Android. VAAPI on Linux is never chosen by@@ -163,6 +166,47 @@ impl HwPolicy {HwPolicy::Device(device) => Some(device),}}++ /// The device this policy asks for a stream of `codec` in `format`:+ /// [`HwPolicy::device`], except that the preference asks only where+ /// the hardware is the faster road; see [`hardware_wins`]. A named+ /// device is asked for whatever the stream.+ pub fn device_for(self, codec: ffmpeg::codec::Id, format: Pixel) -> Option<HwDevice> {+ match self {+ HwPolicy::Preference => {+ hardware_device().filter(|device| hardware_wins(*device, codec, format))+ }+ HwPolicy::Software | HwPolicy::Device(_) => self.device(),+ }+ }+}++/// Whether `device` decodes a stream of `codec` in `format` faster than+/// the CPU does, counting the copy back to memory every hardware frame+/// still pays (see the module's last section).+///+/// Measured on an M5, 4K frames a second, software on every core against+/// VideoToolbox: H.264 114 against 56, 8-bit HEVC 164 against 132, 10-bit+/// HEVC 58 against 117, ProRes 422 78 against 108. H.264 is so cheap in+/// software that the chip, plus the copy, loses by half, and 1080p the same+/// (493 against 144). Everything heavier wins on the hardware, and even at+/// parity the hardware leaves the cores to the rest of the app: 8-bit HEVC+/// took 0.9 of a core on the chip and 6.8 on the CPU. So the chip takes+/// everything but 8-bit H.264; a codec it has no decoder for goes to+/// software regardless, as before.+///+/// A phone's cores are no match for its decoder, and MediaCodec takes+/// everything it can.+pub fn hardware_wins(device: HwDevice, codec: ffmpeg::codec::Id, format: Pixel) -> bool {+ if device == HwDevice::MediaCodec {+ return true;+ }+ // SAFETY: a descriptor points into libavutil's static table, and every+ // pixel format has at least one component.+ let deep = format+ .descriptor()+ .is_some_and(|descriptor| unsafe { (*descriptor.as_ptr()).comp[0].depth } > 8);+ codec != ffmpeg::codec::Id::H264 || deep}/// The process-wide preference: `0` for software, else one past the@@ -436,6 +480,26 @@ fn type_name(kind: HwDevice) -> String {mod tests {use super::*;+ /// The CPU keeps 8-bit H.264, which it decodes faster than the chip+ /// can hand frames back; everything heavier goes to the chip, and a+ /// phone sends it everything.+ #[test]+ fn the_hardware_takes_what_it_decodes_faster() {+ use ffmpeg::codec::Id;+ let vt = HwDevice::VideoToolbox;+ assert!(!hardware_wins(vt, Id::H264, Pixel::YUV420P));+ assert!(!hardware_wins(HwDevice::D3d11va, Id::H264, Pixel::YUV420P));+ assert!(hardware_wins(vt, Id::H264, Pixel::YUV420P10LE));+ assert!(hardware_wins(vt, Id::HEVC, Pixel::YUV420P));+ assert!(hardware_wins(vt, Id::HEVC, Pixel::YUV420P10LE));+ assert!(hardware_wins(vt, Id::PRORES, Pixel::YUV422P10LE));+ assert!(hardware_wins(+ HwDevice::MediaCodec,+ Id::H264,+ Pixel::YUV420P+ ));+ }+#[test]fn the_platform_default_is_never_vaapi() {assert_ne!(HwDevice::platform_default(), Some(HwDevice::Vaapi));