Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

perf(decode): each stream goes to whichever of the CPU and the chip decodes it faster

Jareer62273e4f27e8
diff --git a/src/crates/concat-host/tests/export.rs b/src/crates/concat-host/tests/export.rs
index 6f3b28d2..92f47ce2 100644
--- a/src/crates/concat-host/tests/export.rs
+++ b/src/crates/concat-host/tests/export.rs
@@ -83,9 +83,9 @@ fn second_of_colour(rgb: [u8; 3]) -> Option<u32> {
/// A silent picture of `seconds` at `rate`, each frame the colour of its
/// source second.
-fn picture(path: &Path, rate: FrameRate, seconds: u32) {
+fn picture(path: &Path, rate: FrameRate, seconds: u32, codec: VideoCodec) {
let options = EncodeOptions {
- codec: VideoCodec::H264,
+ codec,
preset: "ultrafast".to_owned(),
crf: 16,
rate_mode: RateMode::Vbr,
@@ -187,25 +187,30 @@ struct Sources {
impl Sources {
fn make(dir: &Path) -> Sources {
+ Sources::make_in(dir, VideoCodec::H264)
+ }
+
+ /// The same set with every picture in `codec`.
+ fn make_in(dir: &Path, codec: VideoCodec) -> Sources {
let wav = dir.join("clock.wav");
clock_wav(&wav, 6);
let aac = dir.join("clock.m4a");
clock_aac(&wav, &aac, 6);
let peek_picture = dir.join("peek-picture.mp4");
- picture(&peek_picture, FrameRate::SIXTY, 6);
+ picture(&peek_picture, FrameRate::SIXTY, 6, codec);
let peek = dir.join("peek.mp4");
sound::mux(&peek_picture, &aac, &peek).expect("joins the screen recording");
let cam_picture = dir.join("cam-picture.mp4");
- picture(&cam_picture, FrameRate::THIRTY, 4);
+ picture(&cam_picture, FrameRate::THIRTY, 4, codec);
let cam_sound = dir.join("cam.m4a");
clock_aac(&wav, &cam_sound, 4);
let cam = dir.join("cam.mp4");
sound::mux(&cam_picture, &cam_sound, &cam).expect("joins the camera clip");
let silent = dir.join("silent.mp4");
- picture(&silent, FrameRate::FILM, 3);
+ picture(&silent, FrameRate::FILM, 3, codec);
let still_path = dir.join("still.jpg");
still(&still_path);
@@ -1270,7 +1275,10 @@ fn an_export_decodes_on_the_hardware_when_preferred() {
);
let scratch = Scratch::new("hardware");
- let sources = Sources::make(scratch.path());
+ // HEVC, not the H.264 the other scenarios use: 8-bit H.264 is the one
+ // stream the preference leaves on the CPU, which decodes it faster
+ // (concat_media::hardware::hardware_wins).
+ let sources = Sources::make_in(scratch.path(), VideoCodec::Hevc);
// A reader opened the way the engine opens them follows the preference
// onto the device, on a machine that has one.
let mut reader = Decoder::open(&sources.peek, &DecodeOptions::default()).expect("opens");
diff --git a/src/crates/concat-media/src/decode.rs b/src/crates/concat-media/src/decode.rs
index f033fe48..eda0593c 100644
--- a/src/crates/concat-media/src/decode.rs
+++ b/src/crates/concat-media/src/decode.rs
@@ -556,10 +556,24 @@ impl Decoder {
(parameters.width(), parameters.height())
};
- // The device the policy asks for, if the process has one. A codec
- // that will not open on it opens in software instead, and the
+ // The device the policy asks for this stream, if the process has
+ // one; see `hardware::hardware_wins` for which streams it wants. A
+ // codec that will not open on it opens in software instead, and the
// reader is none the wiser past a line in the log.
- let device = options.hardware.device().and_then(hardware::device);
+ let (codec, format) = {
+ let parameters = stream.parameters();
+ // SAFETY: a live stream's codec parameters; for a video stream
+ // `format` holds an AVPixelFormat.
+ let raw = unsafe { (*parameters.as_ptr()).format };
+ (
+ parameters.id(),
+ Pixel::from(ffmpeg::sys::AVPixelFormat(raw)),
+ )
+ };
+ let device = options
+ .hardware
+ .device_for(codec, format)
+ .and_then(hardware::device);
let (decoder, accelerated) = match device {
Some(device) => match Self::open_codec(path, &stream, options, Some(&device)) {
Ok(opened) => opened,
diff --git a/src/crates/concat-media/src/hardware.rs b/src/crates/concat-media/src/hardware.rs
index 0e23269d..10adc50a 100644
--- a/src/crates/concat-media/src/hardware.rs
+++ b/src/crates/concat-media/src/hardware.rs
@@ -17,10 +17,13 @@
//!
//! A reader's [`DecodeOptions`](crate::DecodeOptions) carries an
//! [`HwPolicy`]. The default, [`HwPolicy::Preference`], follows one
-//! process-wide setting, [`set_hardware_decode`], so the window's toggle
-//! reaches every reader the pool, the preview, the export and the
-//! thumbnails open without each of them being told. A reader can insist on
-//! software or on a named device instead, which is what the tests do.
+//! process-wide setting, [`set_hardware_decode`], which the window turns on
+//! at start and nothing turns off, so every reader the pool, the preview,
+//! the export and the thumbnails open follows it without being told. It
+//! asks for the device stream by stream, only where the hardware is faster
+//! ([`hardware_wins`]): not for 8-bit H.264, which the CPU decodes at twice
+//! the chip's rate or better. A reader can insist on software or on a named
+//! device instead, which is what the tests do.
//!
//! The device is the platform's own: VideoToolbox on macOS and iOS, D3D11VA
//! on Windows, MediaCodec on Android. VAAPI on Linux is never chosen by
@@ -163,6 +166,47 @@ impl HwPolicy {
HwPolicy::Device(device) => Some(device),
}
}
+
+ /// The device this policy asks for a stream of `codec` in `format`:
+ /// [`HwPolicy::device`], except that the preference asks only where
+ /// the hardware is the faster road; see [`hardware_wins`]. A named
+ /// device is asked for whatever the stream.
+ pub fn device_for(self, codec: ffmpeg::codec::Id, format: Pixel) -> Option<HwDevice> {
+ match self {
+ HwPolicy::Preference => {
+ hardware_device().filter(|device| hardware_wins(*device, codec, format))
+ }
+ HwPolicy::Software | HwPolicy::Device(_) => self.device(),
+ }
+ }
+}
+
+/// Whether `device` decodes a stream of `codec` in `format` faster than
+/// the CPU does, counting the copy back to memory every hardware frame
+/// still pays (see the module's last section).
+///
+/// Measured on an M5, 4K frames a second, software on every core against
+/// VideoToolbox: H.264 114 against 56, 8-bit HEVC 164 against 132, 10-bit
+/// HEVC 58 against 117, ProRes 422 78 against 108. H.264 is so cheap in
+/// software that the chip, plus the copy, loses by half, and 1080p the same
+/// (493 against 144). Everything heavier wins on the hardware, and even at
+/// parity the hardware leaves the cores to the rest of the app: 8-bit HEVC
+/// took 0.9 of a core on the chip and 6.8 on the CPU. So the chip takes
+/// everything but 8-bit H.264; a codec it has no decoder for goes to
+/// software regardless, as before.
+///
+/// A phone's cores are no match for its decoder, and MediaCodec takes
+/// everything it can.
+pub fn hardware_wins(device: HwDevice, codec: ffmpeg::codec::Id, format: Pixel) -> bool {
+ if device == HwDevice::MediaCodec {
+ return true;
+ }
+ // SAFETY: a descriptor points into libavutil's static table, and every
+ // pixel format has at least one component.
+ let deep = format
+ .descriptor()
+ .is_some_and(|descriptor| unsafe { (*descriptor.as_ptr()).comp[0].depth } > 8);
+ codec != ffmpeg::codec::Id::H264 || deep
}
/// The process-wide preference: `0` for software, else one past the
@@ -436,6 +480,26 @@ fn type_name(kind: HwDevice) -> String {
mod tests {
use super::*;
+ /// The CPU keeps 8-bit H.264, which it decodes faster than the chip
+ /// can hand frames back; everything heavier goes to the chip, and a
+ /// phone sends it everything.
+ #[test]
+ fn the_hardware_takes_what_it_decodes_faster() {
+ use ffmpeg::codec::Id;
+ let vt = HwDevice::VideoToolbox;
+ assert!(!hardware_wins(vt, Id::H264, Pixel::YUV420P));
+ assert!(!hardware_wins(HwDevice::D3d11va, Id::H264, Pixel::YUV420P));
+ assert!(hardware_wins(vt, Id::H264, Pixel::YUV420P10LE));
+ assert!(hardware_wins(vt, Id::HEVC, Pixel::YUV420P));
+ assert!(hardware_wins(vt, Id::HEVC, Pixel::YUV420P10LE));
+ assert!(hardware_wins(vt, Id::PRORES, Pixel::YUV422P10LE));
+ assert!(hardware_wins(
+ HwDevice::MediaCodec,
+ Id::H264,
+ Pixel::YUV420P
+ ));
+ }
+
#[test]
fn the_platform_default_is_never_vaapi() {
assert_ne!(HwDevice::platform_default(), Some(HwDevice::Vaapi));