Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

perf(android): video decodes on MediaCodec, which it never did

Jareer2969e024e310
diff --git a/src/crates/concat-media/src/decode.rs b/src/crates/concat-media/src/decode.rs
index 3936417c..14feb04d 100644
--- a/src/crates/concat-media/src/decode.rs
+++ b/src/crates/concat-media/src/decode.rs
@@ -35,8 +35,39 @@ use ffmpeg_the_third::format::{self, Pixel};
use ffmpeg_the_third::util::frame::video::Video;
use crate::error::{Error, Result};
+use crate::feed;
use crate::ffi;
-use crate::hardware::{self, Device, HwDevice, HwPolicy};
+use crate::hardware::{self, Device, HwDevice, HwPolicy, Seat};
+
+/// How [`Decoder::open_codec`] opens the codec.
+enum Codec<'a> {
+ /// libavcodec's own decoder for the stream, on the CPU.
+ Software,
+ /// libavcodec's own decoder, with this hwaccel device attached.
+ On(&'a Device),
+ /// A decoder by name in place of the built-in one: the platform's
+ /// decoder wrapped as a codec, MediaCodec's.
+ Named(&'static str),
+}
+
+/// Where a reader's codec runs when it is not plain software.
+enum OnChip {
+ /// A hwaccel: frames arrive in the device's memory, in this format,
+ /// and are copied back.
+ Accel(HwDevice, Pixel),
+ /// The platform's own decoder behind a codec of its own: frames arrive
+ /// in memory already. Holds its seat on the chip for as long as it
+ /// lasts; see [`hardware::MEDIACODEC_READERS`].
+ Wrapped { device: HwDevice, _seat: Seat },
+}
+
+impl OnChip {
+ fn device(&self) -> HwDevice {
+ match self {
+ OnChip::Accel(device, _) | OnChip::Wrapped { device, .. } => *device,
+ }
+ }
+}
/// Anything that yields frames in order.
///
@@ -573,9 +604,18 @@ pub struct Decoder {
/// that moved to software mid-file picks up, after the frame the
/// device last gave it.
discard_through: Option<Rational>,
- /// The device the codec is on and the pixel format its frames arrive
- /// in, or `None` in software, which is also where a failed device ends.
- hardware: Option<(HwDevice, Pixel)>,
+ /// Where the codec runs when it is not on the CPU, or `None` in
+ /// software, which is also where a failed device ends.
+ hardware: Option<OnChip>,
+ /// A packet the codec refused while full, to send again; see
+ /// [`crate::feed`].
+ held: feed::Held<ffmpeg::Packet>,
+ /// Whether the codec has been handed anything since it opened or was
+ /// last flushed. A codec with nothing in it is not flushed: MediaCodec
+ /// flushed before its first output has lost the stream's parameter
+ /// sets and must be handed them again, which a seek straight after
+ /// opening - every reader opened at a start point - would do.
+ fed: bool,
/// The last picture's timestamp, for picking up in software after it.
last_pts: Option<Rational>,
/// The output frame instant the pacer is at, when pacing.
@@ -633,32 +673,94 @@ impl Decoder {
Pixel::from(ffmpeg::sys::AVPixelFormat(raw)),
)
};
- let device = options
- .hardware
- .device_for(codec, format)
- .and_then(hardware::device);
- let (decoder, accelerated) = match device {
- Some(device) => match Self::open_codec(path, &stream, options, Some(&device)) {
- Ok(opened) => opened,
- Err(error) => {
- hardware::note_fallback(device.kind(), "open the decoder", &error.to_string());
- Self::open_codec(path, &stream, options, None)?
+ let software = || Self::open_codec(path, &stream, options, Codec::Software);
+ let (decoder, accelerated) = match options.hardware.device_for(codec, format) {
+ // MediaCodec: its own decoder, by name, when a seat is free.
+ // Not for a keyframes-only reader: the wrapper ignores
+ // `skip_frame`, so a filmstrip tile would decode its whole
+ // group of pictures instead of one frame, and its pipeline
+ // holds the first picture back until several packets are in -
+ // a one-frame grab is the chip's worst case and the CPU's best.
+ Some(device @ HwDevice::MediaCodec) => {
+ let wrapped = hardware::mediacodec_decoder(codec, format)
+ .filter(|_| !options.keyframes_only)
+ .filter(|&name| {
+ hardware::mediacodec_works(name, || {
+ first_picture_within(path, name, hardware::MEDIACODEC_FIRST_PICTURE)
+ })
+ })
+ .and_then(|name| hardware::mediacodec_seat().map(|seat| (name, seat)));
+ match wrapped {
+ Some((name, seat)) => {
+ match Self::open_codec(path, &stream, options, Codec::Named(name)) {
+ Ok((decoder, _)) => {
+ hardware::note_wrapped(device, name);
+ (
+ decoder,
+ Some(OnChip::Wrapped {
+ device,
+ _seat: seat,
+ }),
+ )
+ }
+ Err(error) => {
+ // Most often the phone's sessions are all
+ // taken, by this app or another.
+ hardware::note_fallback(
+ device,
+ "open the decoder",
+ &error.to_string(),
+ );
+ software()?
+ }
+ }
+ }
+ None => software()?,
+ }
+ }
+ Some(device) => match hardware::device(device) {
+ Some(device) => {
+ match Self::open_codec(path, &stream, options, Codec::On(&device)) {
+ Ok(opened) => opened,
+ Err(error) => {
+ hardware::note_fallback(
+ device.kind(),
+ "open the decoder",
+ &error.to_string(),
+ );
+ software()?
+ }
+ }
}
+ None => software()?,
},
- None => Self::open_codec(path, &stream, options, None)?,
+ None => software()?,
};
// The caller's word over the file's, where the caller has one: the
// file's tag is what the override exists to correct. Where neither
// says, the codec's convention does; see `ColorRange::implied`.
- let tagged = decoder.color_range();
+ // Read off the stream, not the opened codec: MediaCodec's wrapper
+ // writes the chip's own idea of the colour into the codec as it
+ // starts - limited range on a stream that says nothing, NV12 for
+ // the pixels - which is not what the file says.
+ // SAFETY: a live stream's codec parameters; plain enum fields.
+ let (tagged, primaries, transfer, matrix) = unsafe {
+ let raw = stream.parameters().as_ptr();
+ (
+ ffmpeg::color::Range::from((*raw).color_range),
+ ffmpeg::color::Primaries::from((*raw).color_primaries),
+ ffmpeg::color::TransferCharacteristic::from((*raw).color_trc),
+ ffmpeg::color::Space::from((*raw).color_space),
+ )
+ };
let implied = options.color_range.is_none() && tagged == ffmpeg::color::Range::Unspecified;
let color = ColorSignal {
- primaries: decoder.color_primaries(),
- transfer: decoder.color_transfer_characteristic(),
- matrix: decoder.color_space(),
+ primaries,
+ transfer,
+ matrix,
range: match options.color_range {
Some(range) => range.as_ffmpeg(),
- None if implied => ColorRange::implied(decoder.id(), decoder.format()).as_ffmpeg(),
+ None if implied => ColorRange::implied(codec, format).as_ffmpeg(),
None => tagged,
},
implied,
@@ -690,6 +792,8 @@ impl Decoder {
discard_before: None,
discard_through: None,
hardware: accelerated,
+ held: None,
+ fed: false,
last_pts: None,
tick: 0,
origin: Rational::ZERO,
@@ -708,15 +812,14 @@ impl Decoder {
Ok(this)
}
- /// Opens the stream's codec, on `device` where the codec can decode on
- /// it. Gives back the opened decoder and, when it is on the device, the
- /// device and the pixel format its frames arrive in.
+ /// Opens the stream's codec as `codec` says. Gives back the opened
+ /// decoder and, when it is on a hwaccel device, where.
fn open_codec(
path: &Path,
stream: &ffmpeg::format::stream::Stream<'_>,
options: &DecodeOptions,
- device: Option<&Device>,
- ) -> Result<(decoder::Video, Option<(HwDevice, Pixel)>)> {
+ codec: Codec<'_>,
+ ) -> Result<(decoder::Video, Option<OnChip>)> {
let mut context = ffmpeg::codec::Context::from_parameters(stream.parameters())
.map_err(|error| ffi::fail("codec parameters", path, error))?;
// Every core. libavcodec's API opens a decoder on one thread unless
@@ -733,15 +836,38 @@ impl Decoder {
(*raw).thread_type = ffmpeg::sys::FF_THREAD_FRAME | ffmpeg::sys::FF_THREAD_SLICE;
(*raw).thread_count = i32::from(options.threads);
}
- let accelerated = device.and_then(|device| {
- device
- .attach(&mut context)
- .map(|format| (device.kind(), format))
- });
- let mut decoder = context
- .decoder()
- .video()
- .map_err(|error| ffi::fail("open decoder", path, error))?;
+ let (accelerated, named) = match codec {
+ Codec::Software => (None, None),
+ Codec::On(device) => (
+ device
+ .attach(&mut context)
+ .map(|format| OnChip::Accel(device.kind(), format)),
+ None,
+ ),
+ Codec::Named(name) => {
+ let found = decoder::find_by_name(name).ok_or_else(|| Error::Missing {
+ what: "decoder",
+ name: name.to_owned(),
+ })?;
+ // MediaCodec counts time in microseconds; told the stream's
+ // clock, the wrapper converts each packet's timestamp there
+ // and every frame's back, exactly for any clock coarser than
+ // a microsecond, which is every one a container uses.
+ // SAFETY: the context is not opened yet; a plain rational.
+ unsafe {
+ (*context.as_mut_ptr()).pkt_timebase = stream.time_base().into();
+ }
+ (None, Some(found))
+ }
+ };
+ let opened = match named {
+ Some(found) => context
+ .decoder()
+ .open_as(found)
+ .and_then(|opened| opened.video()),
+ None => context.decoder().video(),
+ };
+ let mut decoder = opened.map_err(|error| ffi::fail("open decoder", path, error))?;
if options.keyframes_only {
decoder.skip_frame(ffmpeg::Discard::NonKey);
}
@@ -758,7 +884,7 @@ impl Decoder {
/// without it, or once the codec has shown, with its first frame, that
/// it declined the device for this stream.
pub fn hardware(&self) -> Option<HwDevice> {
- self.hardware.map(|(device, _)| device)
+ self.hardware.as_ref().map(OnChip::device)
}
/// Reopens the codec in software and carries on from `from`: the frame
@@ -774,7 +900,7 @@ impl Decoder {
.ok_or_else(|| Error::NoVideoStream {
path: self.path.clone(),
})?;
- Self::open_codec(&self.path, &stream, &self.options, None)?.0
+ Self::open_codec(&self.path, &stream, &self.options, Codec::Software)?.0
};
let at = from.unwrap_or(self.origin);
let target = ffi::av_ticks(at + self.start);
@@ -782,7 +908,10 @@ impl Decoder {
.seek(target, ..=target)
.map_err(|error| ffi::fail("seek", &self.path, error))?;
self.decoder = decoder;
+ // The old codec, and its seat on the chip, go now.
self.hardware = None;
+ self.held = None;
+ self.fed = false;
if !self.options.keyframes_only {
if after {
self.discard_through = Some(at);
@@ -799,7 +928,14 @@ impl Decoder {
self.input
.seek(target, ..=target)
.map_err(|error| ffi::fail("seek", &self.path, error))?;
- self.decoder.flush();
+ // Every picture and packet the codec holds is from before the
+ // seek: out, and so is the packet it refused. For MediaCodec this
+ // is the codec's own flush, which it needs before a jump.
+ if self.fed {
+ self.decoder.flush();
+ self.fed = false;
+ }
+ self.held = None;
// Keyframes only: the picture the container landed on is the one
// wanted, and discarding up to the target would throw it away and
// wait for the *next* keyframe, a whole group of pictures late.
@@ -817,104 +953,78 @@ impl Decoder {
/// The next picture out of the codec, or `None` at the end of the file.
/// Frames before the discard point never come out of here, and a frame
- /// from the device is copied back to memory before it does.
+ /// from a hwaccel device is copied back to memory before it does.
fn next_source(&mut self) -> Result<Option<Source>> {
loop {
- let mut frame = Video::empty();
- match self.decoder.receive_frame(&mut frame) {
- Ok(()) => {
- let pts = frame
- .timestamp()
- .and_then(|ticks| ffi::seconds(ticks, self.time_base))
- .map(|pts| pts - self.start);
- if let Some((device, format)) = self.hardware {
- if frame.format() == format {
- frame = match hardware::download(&frame) {
- Ok(copy) => copy,
- Err(error) => {
- hardware::note_fallback(
- device,
- "copy a frame back",
- &error.to_string(),
- );
- self.resume_in_software(pts, false)?;
- continue;
- }
- };
- } else {
- // The codec set the device up for this stream
- // and the device said no, so the codec went on
- // in software by itself; the frames are fine.
- hardware::note_fallback(
- device,
- "take this stream",
- "the codec decoded it in software",
- );
- self.hardware = None;
- }
- }
- if let (Some(before), Some(pts)) = (self.discard_before, pts)
- && pts < before
- {
- continue;
- }
- if let (Some(through), Some(pts)) = (self.discard_through, pts)
- && pts <= through
- {
- continue;
- }
- self.discard_before = None;
- self.discard_through = None;
- self.last_pts = pts;
- return Ok(Some(Source { frame, pts }));
- }
- Err(ffmpeg::Error::Eof) => return Ok(None),
- Err(error) if ffi::is_again(&error) => {}
- Err(error) if self.hardware.is_some() => {
- // The device choked on a picture it had accepted the
- // stream for. Software takes over after the last frame
- // the reader was given.
- let (device, _) = self.hardware.take().expect("checked");
- hardware::note_fallback(device, "decode a frame", &error.to_string());
+ // Packets for other streams are skipped; the end of the file
+ // drains the codec so its last few frames come out at all; a
+ // packet the codec refuses while full is kept and sent again.
+ self.fed = true;
+ let read = || feed::read_packet(&mut self.input, self.stream);
+ let mut frame = match feed::next_frame(&mut self.decoder, &mut self.held, read) {
+ Ok(Some(frame)) => frame,
+ Ok(None) => return Ok(None),
+ Err(feed::Stall::Read(error)) => return Err(ffi::fail("read", &self.path, error)),
+ Err(stall) if self.hardware.is_some() => {
+ // The device choked on a stream it had accepted.
+ // Software takes over after the last frame the reader
+ // was given.
+ let device = self.hardware.take().expect("checked").device();
+ let (stage, detail) = match &stall {
+ feed::Stall::Send(error) => ("take a packet", error.to_string()),
+ feed::Stall::Receive(error) => ("decode a frame", error.to_string()),
+ _ => ("keep decoding", "it stopped taking packets".to_owned()),
+ };
+ hardware::note_fallback(device, stage, &detail);
self.resume_in_software(self.last_pts, self.last_pts.is_some())?;
continue;
}
- Err(error) => return Err(ffi::fail("decode", &self.path, error)),
- }
-
- // The codec wants another packet. Packets for other streams are
- // skipped; the end of the file flushes the codec so its last few
- // frames come out at all.
- loop {
- let mut packet = ffmpeg::Packet::empty();
- match packet.read(&mut self.input) {
- Ok(()) => {
- if packet.stream() != self.stream {
+ Err(stall) => return Err(feed::fail(stall, &self.path)),
+ };
+ let pts = frame
+ .timestamp()
+ .and_then(|ticks| ffi::seconds(ticks, self.time_base))
+ .map(|pts| pts - self.start);
+ if let Some(&OnChip::Accel(device, format)) = self.hardware.as_ref() {
+ if frame.format() == format {
+ frame = match hardware::download(&frame) {
+ Ok(copy) => copy,
+ Err(error) => {
+ hardware::note_fallback(
+ device,
+ "copy a frame back",
+ &error.to_string(),
+ );
+ self.resume_in_software(pts, false)?;
continue;
}
- match self.decoder.send_packet(&packet) {
- Ok(()) => break,
- Err(error) if ffi::is_again(&error) => break,
- Err(error) if self.hardware.is_some() => {
- let (device, _) = self.hardware.take().expect("checked");
- hardware::note_fallback(
- device,
- "take a packet",
- &error.to_string(),
- );
- self.resume_in_software(self.last_pts, self.last_pts.is_some())?;
- break;
- }
- Err(error) => return Err(ffi::fail("send packet", &self.path, error)),
- }
- }
- Err(ffmpeg::Error::Eof) => {
- let _ = self.decoder.send_eof();
- break;
- }
- Err(error) => return Err(ffi::fail("read", &self.path, error)),
+ };
+ } else {
+ // The codec set the device up for this stream and the
+ // device said no, so the codec went on in software by
+ // itself; the frames are fine.
+ hardware::note_fallback(
+ device,
+ "take this stream",
+ "the codec decoded it in software",
+ );
+ self.hardware = None;
}
}
+ if let (Some(before), Some(pts)) = (self.discard_before, pts)
+ && pts < before
+ {
+ continue;
+ }
+ if let (Some(through), Some(pts)) = (self.discard_through, pts)
+ && pts <= through
+ {
+ continue;
+ }
+ self.discard_before = None;
+ self.discard_through = None;
+ self.last_pts = pts;
+ return Ok(Some(Source { frame, pts }));
}
}
@@ -1125,6 +1235,51 @@ impl FrameSource for Decoder {
}
}
+/// Whether MediaCodec's decoder `name` gives the first picture of
+/// `path`'s video within `wait` - [`hardware::MEDIACODEC_FIRST_PICTURE`] for a
+/// reader; see [`hardware::Verdicts`] for why a decoder is asked before it
+/// is trusted.
+///
+/// The try runs on a thread of its own, with its own demuxer and codec,
+/// because a decoder that wedges never returns control to the caller; a
+/// wedged try is left behind on its thread, and the reader never waits on
+/// it past the deadline. A try that answers late still has its say, for
+/// the readers that come after.
+fn first_picture_within(path: &Path, name: &'static str, wait: std::time::Duration) -> bool {
+ let path = path.to_path_buf();
+ let (answer, answered) = std::sync::mpsc::channel();
+ let tried = std::thread::Builder::new()
+ .name("mediacodec-first-picture".to_owned())
+ .spawn(move || {
+ let attempt = || -> Option<()> {
+ let mut input = ffmpeg::format::input(&path).ok()?;
+ let (mut codec, index) = {
+ let stream = input.streams().best(ffmpeg::media::Type::Video)?;
+ let (codec, _) = Decoder::open_codec(
+ &path,
+ &stream,
+ &DecodeOptions::default(),
+ Codec::Named(name),
+ )
+ .ok()?;
+ (codec, stream.index())
+ };
+ let mut held = None;
+ let read = || feed::read_packet(&mut input, index);
+ feed::next_frame(&mut codec, &mut held, read)
+ .ok()?
+ .map(|_| ())
+ };
+ let works = attempt().is_some();
+ let _ = answer.send(works);
+ // Late or not, what it found is the verdict from now on: a
+ // decoder that was only slow to start is not written off for
+ // the life of the process.
+ hardware::mediacodec_answered(name, works);
+ });
+ tried.is_ok() && answered.recv_timeout(wait).unwrap_or(false)
+}
+
/// A second reference to `frame`'s picture: the same buffers, counted,
/// so a graph that takes the reference (`buffersrc` does) takes nothing
/// from the frame's owner. `Video::clone` copies the pixels - fifty
@@ -1678,6 +1833,127 @@ mod tests {
);
}
+ /// On a phone or the emulator, with a real file in
+ /// `CONCAT_DEVICE_SAMPLE`: MediaCodec decodes it to the frames the CPU
+ /// does - every frame of the first `CONCAT_DEVICE_FRAMES` (default 90)
+ /// at the same instant, within the rounding of a colour conversion -
+ /// and a seek forward, back and to the very start lands on the frame
+ /// the CPU's seek lands on. Prints how long each took, which on the
+ /// emulator measures its software MediaCodec and nothing a phone
+ /// does. With `CONCAT_DEVICE_OUT`, writes the chip's frame at each
+ /// seek there as a JPEG to look at.
+ ///
+ /// Built with `cargo test -p concat-media --no-run --target
+ /// aarch64-linux-android`, pushed with the file to /data/local/tmp,
+ /// and run there with `--ignored mediacodec`.
+ #[test]
+ #[ignore = "needs Android and a sample in CONCAT_DEVICE_SAMPLE"]
+ fn mediacodec_decodes_a_real_file_as_the_cpu_does() {
+ use std::time::Instant;
+ let Some(path) = std::env::var_os("CONCAT_DEVICE_SAMPLE").map(PathBuf::from) else {
+ return;
+ };
+ let count: usize = std::env::var("CONCAT_DEVICE_FRAMES")
+ .ok()
+ .and_then(|count| count.parse().ok())
+ .unwrap_or(90);
+ let out = std::env::var_os("CONCAT_DEVICE_OUT").map(PathBuf::from);
+ let chip = DecodeOptions::default().accelerated(HwDevice::MediaCodec);
+ let cpu = DecodeOptions::default().in_software();
+
+ let take = |options: &DecodeOptions| {
+ let started = Instant::now();
+ let mut decoder = Decoder::open(&path, options).expect("opens");
+ let mut frames = Vec::new();
+ while frames.len() < count {
+ let Some(frame) = decoder.next_frame().expect("decodes") else {
+ break;
+ };
+ frames.push((decoder.position(), frame));
+ }
+ (decoder.hardware(), frames, started.elapsed())
+ };
+ // How long the platform's decoder takes to its first picture, the
+ // number the first-picture deadline is measured against.
+ {
+ let started = Instant::now();
+ let name = hardware::mediacodec_decoder(
+ ffmpeg::format::input(&path)
+ .expect("opens")
+ .streams()
+ .best(ffmpeg::media::Type::Video)
+ .expect("a video stream")
+ .parameters()
+ .id(),
+ Pixel::YUV420P,
+ )
+ .expect("a codec MediaCodec takes");
+ let answered = first_picture_within(&path, name, std::time::Duration::from_secs(60));
+ eprintln!(
+ "{name}: first picture {} in {:?}",
+ if answered { "came" } else { "did not come" },
+ started.elapsed()
+ );
+ }
+ let (on_chip, chip_frames, chip_took) = take(&chip);
+ let (_, cpu_frames, cpu_took) = take(&cpu);
+ assert_eq!(
+ on_chip,
+ Some(HwDevice::MediaCodec),
+ "still on the chip at the end"
+ );
+ eprintln!(
+ "{} frames of {}x{}: MediaCodec {chip_took:?}, CPU {cpu_took:?}",
+ chip_frames.len(),
+ chip_frames[0].1.width(),
+ chip_frames[0].1.height()
+ );
+ assert_eq!(chip_frames.len(), cpu_frames.len());
+ for (index, ((chip_at, chip_frame), (cpu_at, cpu_frame))) in
+ chip_frames.iter().zip(&cpu_frames).enumerate()
+ {
+ assert_eq!(chip_at, cpu_at, "frame {index} at the same instant");
+ let difference = worst_difference(chip_frame, cpu_frame);
+ assert!(difference <= 8, "frame {index} differs by {difference}");
+ }
+
+ // Seeks on one open reader each, the way the pool's warm readers
+ // take them: forward, back, forward again, and to the start.
+ let mut chip_reader = Decoder::open(&path, &chip).expect("opens");
+ let mut cpu_reader = Decoder::open(&path, &cpu).expect("opens");
+ for (step, to) in [(7, 3), (1, 1), (5, 2), (0, 1), (11, 4)]
+ .into_iter()
+ .enumerate()
+ {
+ let to = Rational::new(to.0, to.1);
+ let started = Instant::now();
+ chip_reader.seek(to).expect("seeks");
+ let landed = chip_reader.next_frame().expect("decodes").expect("a frame");
+ let took = started.elapsed();
+ cpu_reader.seek(to).expect("seeks");
+ let wanted = cpu_reader.next_frame().expect("decodes").expect("a frame");
+ assert_eq!(
+ chip_reader.position(),
+ cpu_reader.position(),
+ "seek to {to:?}"
+ );
+ let difference = worst_difference(&landed, &wanted);
+ eprintln!(
+ "seek to {to:?} landed at {:?} in {took:?}, {difference} levels off",
+ chip_reader.position()
+ );
+ assert!(
+ difference <= 8,
+ "the seek to {to:?} landed {difference} levels off"
+ );
+ if let Some(out) = &out {
+ let jpeg = crate::encode::jpeg(&landed, 90).expect("encodes");
+ std::fs::write(out.join(format!("seek-{step}.jpg")), jpeg).expect("writes");
+ }
+ }
+ assert_eq!(chip_reader.hardware(), Some(HwDevice::MediaCodec));
+ }
+
/// The pacer sits on top of the device exactly as it does the CPU: a
/// file paced to twice its rate gives the same frames either way.
#[test]
diff --git a/src/crates/concat-media/src/hardware.rs b/src/crates/concat-media/src/hardware.rs
index 10adc50a..60aec0bd 100644
--- a/src/crates/concat-media/src/hardware.rs
+++ b/src/crates/concat-media/src/hardware.rs
@@ -30,6 +30,13 @@
//! default and only ever by name, because on an Intel i915 it has hung the
//! GPU for every process on the machine until a reboot (issue #70).
//!
+//! MediaCodec goes through a different door. It is no hwaccel: FFmpeg
+//! reaches it only through its own decoders, `h264_mediacodec` and the
+//! rest, opened by name in place of the built-in one, which hand frames
+//! back in memory with no copy to make ([`mediacodec_decoder`]). A phone
+//! has few decoder sessions to give, so at most [`MEDIACODEC_READERS`]
+//! readers hold one at a time and the rest decode in software.
+//!
//! ## What happens when it does not work
//!
//! Everything falls back to software, and nothing a person sees changes but
@@ -50,8 +57,9 @@
use std::collections::HashMap;
use std::ptr;
-use std::sync::atomic::{AtomicU8, Ordering};
+use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering};
use std::sync::{Mutex, Once};
+use std::time::Duration;
use ffmpeg_the_third as ffmpeg;
use ffmpeg_the_third::codec::decoder;
@@ -196,10 +204,10 @@ impl HwPolicy {
/// software regardless, as before.
///
/// A phone's cores are no match for its decoder, and MediaCodec takes
-/// everything it can.
+/// everything [`mediacodec_decoder`] has a decoder for.
pub fn hardware_wins(device: HwDevice, codec: ffmpeg::codec::Id, format: Pixel) -> bool {
if device == HwDevice::MediaCodec {
- return true;
+ return mediacodec_decoder(codec, format).is_some();
}
// SAFETY: a descriptor points into libavutil's static table, and every
// pixel format has at least one component.
@@ -209,6 +217,222 @@ pub fn hardware_wins(device: HwDevice, codec: ffmpeg::codec::Id, format: Pixel)
codec != ffmpeg::codec::Id::H264 || deep
}
+/// The libavcodec decoder that drives MediaCodec for a stream of `codec`
+/// in `format`, or `None` where the stream stays in software.
+///
+/// MediaCodec is not a hwaccel the way the desktop devices are: FFmpeg's
+/// own H.264 and HEVC decoders know nothing of it, and a device context
+/// handed to them changes nothing. It is a set of separate decoders, one
+/// per codec, opened by name, which feed the packets to the phone's chip
+/// and, with no output surface, hand back each picture in memory as NV12
+/// or planar 4:2:0 with the chip's stride and crop already undone -
+/// frames the filtergraph takes as it takes a software decoder's.
+///
+/// Eight-bit 4:2:0 only. Ten-bit HEVC comes back from the chip as P010,
+/// which FFmpeg's wrapper has no mapping for (it fails), or squeezed to
+/// eight bits by some vendors (which would quietly lose the HDR the deep
+/// path keeps); it stays in software until that is settled on a phone.
+/// VP8, MPEG-4 Part 2 and MPEG-2 have wrappers too, but are rare on a
+/// phone and cheap in software. AV1 goes to the chip, or to the
+/// platform's own software decoder where the chip has none: this FFmpeg
+/// has no AV1 decoder of its own.
+pub fn mediacodec_decoder(codec: ffmpeg::codec::Id, format: Pixel) -> Option<&'static str> {
+ use ffmpeg::codec::Id;
+ if !matches!(format, Pixel::YUV420P | Pixel::YUVJ420P) {
+ return None;
+ }
+ match codec {
+ Id::H264 => Some("h264_mediacodec"),
+ Id::HEVC => Some("hevc_mediacodec"),
+ Id::VP9 => Some("vp9_mediacodec"),
+ Id::AV1 => Some("av1_mediacodec"),
+ _ => None,
+ }
+}
+
+/// Readers decoding on MediaCodec at once, at most; the rest decode in
+/// software.
+///
+/// A phone has a fixed budget of decoder sessions - a count, and under it
+/// a ceiling on pixels a second across all of them, which a mid-range
+/// chip spends on two or three 4K streams. Past it, `configure` fails and
+/// the reader falls back anyway, but only after the attempt; and every
+/// session spent on a reader sitting idle in the pool is one the camera,
+/// the export's MediaCodec encoder or another app cannot have. Concat
+/// keeps up to eight warm readers plus the filmstrip's and the export's,
+/// while what a person watches needs few at once: the clip under the
+/// playhead, the next one ready at a cut, and one more for a layer over
+/// them or a scrub elsewhere. So three; the fourth reader on decodes on
+/// the CPU as before, and a seat comes free when its reader closes or
+/// falls back.
+pub const MEDIACODEC_READERS: usize = 3;
+
+/// A bounded count of readers holding something scarce.
+pub(crate) struct Seats {
+ taken: AtomicUsize,
+ limit: usize,
+}
+
+impl Seats {
+ pub(crate) const fn new(limit: usize) -> Self {
+ Self {
+ taken: AtomicUsize::new(0),
+ limit,
+ }
+ }
+
+ /// A seat, or `None` when all of them are taken. The seat is given
+ /// back when it drops.
+ pub(crate) fn take(&'static self) -> Option<Seat> {
+ self.taken
+ .fetch_update(Ordering::AcqRel, Ordering::Acquire, |taken| {
+ (taken < self.limit).then_some(taken + 1)
+ })
+ .ok()
+ .map(|_| Seat { seats: self })
+ }
+
+ /// How many seats are taken right now.
+ #[cfg(test)]
+ pub(crate) fn taken(&self) -> usize {
+ self.taken.load(Ordering::Acquire)
+ }
+}
+
+/// One taken seat; see [`Seats`].
+pub(crate) struct Seat {
+ seats: &'static Seats,
+}
+
+impl Drop for Seat {
+ fn drop(&mut self) {
+ self.seats.taken.fetch_sub(1, Ordering::AcqRel);
+ }
+}
+
+impl std::fmt::Debug for Seat {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str("Seat")
+ }
+}
+
+/// The readers on MediaCodec; see [`MEDIACODEC_READERS`].
+static MEDIACODEC_SEATS: Seats = Seats::new(MEDIACODEC_READERS);
+
+/// A seat on MediaCodec for a reader about to open one of its decoders,
+/// or `None` when [`MEDIACODEC_READERS`] already have one - said once a
+/// process, since a busy timeline says it all the time.
+pub(crate) fn mediacodec_seat() -> Option<Seat> {
+ let seat = MEDIACODEC_SEATS.take();
+ if seat.is_none() {
+ static ONCE: Once = Once::new();
+ ONCE.call_once(|| {
+ log::info!(
+ "{MEDIACODEC_READERS} readers already decode on MediaCodec; more decode in software"
+ );
+ });
+ }
+ seat
+}
+
+/// How long a MediaCodec decoder has to give its first picture the first
+/// time it is tried. A phone's chip gives it in tens of milliseconds and
+/// the platform's software decoders in a few hundred; the wait is only
+/// ever spent in full on a decoder that is never going to answer.
+pub const MEDIACODEC_FIRST_PICTURE: Duration = Duration::from_secs(2);
+
+/// What each MediaCodec decoder did the first time this process tried
+/// it: gave a picture, or did not.
+///
+/// The first try matters because a platform decoder that wedges does not
+/// fail, it hangs: FFmpeg's wrapper waits for an input or an output
+/// buffer that never comes, in a loop inside `avcodec_send_packet` that
+/// nothing outside it can break. The emulator's own goldfish H.264
+/// decoder does exactly that with no host GPU, and a reader that met it
+/// froze the preview for good. So each decoder is tried once, off to the
+/// side with a deadline (see [`MEDIACODEC_FIRST_PICTURE`]), before any
+/// reader trusts it.
+pub(crate) struct Verdicts {
+ said: Mutex<Vec<(&'static str, bool)>>,
+}
+
+impl Verdicts {
+ pub(crate) const fn new() -> Self {
+ Self {
+ said: Mutex::new(Vec::new()),
+ }
+ }
+
+ /// Whether `name` works: what it did when first tried, or, the first
+ /// time, what `try_it` finds - once, with every other reader asking
+ /// meanwhile waiting for the answer rather than trying it again.
+ pub(crate) fn works(&self, name: &'static str, try_it: impl FnOnce() -> bool) -> bool {
+ let mut said = self
+ .said
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ if let Some(&(_, works)) = said.iter().find(|(said, _)| *said == name) {
+ return works;
+ }
+ let works = try_it();
+ said.push((name, works));
+ works
+ }
+
+ /// Records what a try found after its deadline had passed: a decoder
+ /// that was only slow the first time - a phone just woken, a codec
+ /// service starting cold - is trusted from here on. Waits for the
+ /// first verdict to be written, so the late one is the one that stays.
+ pub(crate) fn amend(&self, name: &'static str, works: bool) {
+ let mut said = self
+ .said
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ match said.iter_mut().find(|(said, _)| *said == name) {
+ Some(verdict) => verdict.1 = works,
+ None => said.push((name, works)),
+ }
+ }
+}
+
+/// Every MediaCodec decoder's first try; see [`Verdicts`].
+static MEDIACODEC_VERDICTS: Verdicts = Verdicts::new();
+
+/// Whether MediaCodec's `name` can be trusted with a reader: what it did
+/// the first time `try_it` tried it in this process. A decoder that did
+/// not give a picture in time is said once and not tried again, unless
+/// that first try answers late; see [`mediacodec_answered`].
+pub(crate) fn mediacodec_works(name: &'static str, try_it: impl FnOnce() -> bool) -> bool {
+ MEDIACODEC_VERDICTS.works(name, || {
+ let works = try_it();
+ if !works {
+ log::warn!(
+ "MediaCodec's {name} gave no picture within {:?}; that codec decodes in software",
+ MEDIACODEC_FIRST_PICTURE
+ );
+ }
+ works
+ })
+}
+
+/// What a first try of `name` found once it finished, deadline or not;
+/// see [`Verdicts::amend`].
+pub(crate) fn mediacodec_answered(name: &'static str, works: bool) {
+ MEDIACODEC_VERDICTS.amend(name, works);
+}
+
+/// Writes down, once per decoder, that a reader really is decoding on the
+/// platform's decoder through `name`: the chip is said to be in use only
+/// once a codec has opened on it.
+pub(crate) fn note_wrapped(device: HwDevice, name: &'static str) {
+ static SAID: Mutex<Vec<&'static str>> = Mutex::new(Vec::new());
+ let mut said = SAID.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+ if !said.contains(&name) {
+ said.push(name);
+ log::info!("video decode on {} through {name}", device.label());
+ }
+}
+
/// The process-wide preference: `0` for software, else one past the
/// device's index in [`HwDevice::ALL`]. An atomic rather than a lock
/// because every `Decoder::open` reads it.
@@ -493,11 +717,114 @@ mod tests {
assert!(hardware_wins(vt, Id::HEVC, Pixel::YUV420P));
assert!(hardware_wins(vt, Id::HEVC, Pixel::YUV420P10LE));
assert!(hardware_wins(vt, Id::PRORES, Pixel::YUV422P10LE));
- assert!(hardware_wins(
- HwDevice::MediaCodec,
- Id::H264,
- Pixel::YUV420P
- ));
+ let phone = HwDevice::MediaCodec;
+ assert!(hardware_wins(phone, Id::H264, Pixel::YUV420P));
+ assert!(hardware_wins(phone, Id::HEVC, Pixel::YUVJ420P));
+ assert!(hardware_wins(phone, Id::AV1, Pixel::YUV420P));
+ // Ten bits and 4:2:2 stay on the CPU, and so do codecs it has no
+ // decoder for here.
+ assert!(!hardware_wins(phone, Id::HEVC, Pixel::YUV420P10LE));
+ assert!(!hardware_wins(phone, Id::H264, Pixel::YUV422P));
+ assert!(!hardware_wins(phone, Id::PRORES, Pixel::YUV422P10LE));
+ assert!(!hardware_wins(phone, Id::MPEG4, Pixel::YUV420P));
+ }
+
+ /// MediaCodec's decoders are opened by name, never found by codec id:
+ /// the built-in H.264 decoder has no MediaCodec entry at all, which is
+ /// how every phone decoded on its CPU while the log said otherwise.
+ #[test]
+ fn mediacodec_decoders_are_the_named_wrappers() {
+ use ffmpeg::codec::Id;
+ assert_eq!(
+ mediacodec_decoder(Id::H264, Pixel::YUV420P),
+ Some("h264_mediacodec")
+ );
+ assert_eq!(
+ mediacodec_decoder(Id::HEVC, Pixel::YUV420P),
+ Some("hevc_mediacodec")
+ );
+ assert_eq!(mediacodec_decoder(Id::HEVC, Pixel::YUV420P10LE), None);
+ crate::ffi::init();
+ let h264 = decoder::find(Id::H264).expect("an H.264 decoder");
+ assert_eq!(
+ hardware_format(h264.as_ptr(), sys::AVHWDeviceType::MEDIACODEC),
+ None,
+ "the built-in decoder has grown a MediaCodec hwaccel; revisit"
+ );
+ }
+
+ #[test]
+ fn seats_run_out_and_come_back() {
+ static SEATS: Seats = Seats::new(2);
+ let first = SEATS.take().expect("a seat");
+ let second = SEATS.take().expect("a second seat");
+ assert!(SEATS.take().is_none(), "two is the limit");
+ assert_eq!(SEATS.taken(), 2);
+ drop(first);
+ assert_eq!(SEATS.taken(), 1);
+ let third = SEATS.take().expect("the freed seat");
+ assert!(SEATS.take().is_none());
+ drop((second, third));
+ assert_eq!(SEATS.taken(), 0);
+ }
+
+ /// A decoder is tried once; its verdict stands, good or bad, and the
+ /// readers asking at the same time wait for it rather than each
+ /// trying it again.
+ #[test]
+ fn a_decoder_is_tried_once_and_the_verdict_stands() {
+ static VERDICTS: Verdicts = Verdicts::new();
+ let tries = AtomicUsize::new(0);
+ let try_it = |answer: bool| {
+ let tries = &tries;
+ move || {
+ tries.fetch_add(1, Ordering::SeqCst);
+ std::thread::sleep(Duration::from_millis(20));
+ answer
+ }
+ };
+ std::thread::scope(|scope| {
+ for _ in 0..4 {
+ scope.spawn(|| assert!(!VERDICTS.works("wedged", try_it(false))));
+ }
+ });
+ assert_eq!(tries.load(Ordering::SeqCst), 1);
+ assert!(VERDICTS.works("fine", try_it(true)));
+ assert!(
+ VERDICTS.works("fine", try_it(false)),
+ "the first verdict stands"
+ );
+ assert!(!VERDICTS.works("wedged", try_it(true)));
+ assert_eq!(tries.load(Ordering::SeqCst), 2);
+
+ // A try that answers after its deadline is believed from then on.
+ VERDICTS.amend("wedged", true);
+ assert!(VERDICTS.works("wedged", try_it(false)));
+ assert_eq!(tries.load(Ordering::SeqCst), 2);
+ }
+
+ #[test]
+ fn seats_hold_under_contention() {
+ static SEATS: Seats = Seats::new(3);
+ let peak = std::sync::Arc::new(AtomicUsize::new(0));
+ let threads: Vec<_> = (0..8)
+ .map(|_| {
+ let peak = std::sync::Arc::clone(&peak);
+ std::thread::spawn(move || {
+ for _ in 0..2000 {
+ if let Some(seat) = SEATS.take() {
+ peak.fetch_max(SEATS.taken(), Ordering::AcqRel);
+ drop(seat);
+ }
+ }
+ })
+ })
+ .collect();
+ for thread in threads {
+ thread.join().expect("joins");
+ }
+ assert!(peak.load(Ordering::Acquire) <= 3);
+ assert_eq!(SEATS.taken(), 0);
}
#[test]