The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
perf(android): video decodes on MediaCodec, which it never did
Jareer2969e024e310
diff --git a/src/crates/concat-media/src/decode.rs b/src/crates/concat-media/src/decode.rsindex 3936417c..14feb04d 100644--- a/src/crates/concat-media/src/decode.rs+++ b/src/crates/concat-media/src/decode.rs@@ -35,8 +35,39 @@ use ffmpeg_the_third::format::{self, Pixel};use ffmpeg_the_third::util::frame::video::Video;use crate::error::{Error, Result};+use crate::feed;use crate::ffi;-use crate::hardware::{self, Device, HwDevice, HwPolicy};+use crate::hardware::{self, Device, HwDevice, HwPolicy, Seat};++/// How [`Decoder::open_codec`] opens the codec.+enum Codec<'a> {+ /// libavcodec's own decoder for the stream, on the CPU.+ Software,+ /// libavcodec's own decoder, with this hwaccel device attached.+ On(&'a Device),+ /// A decoder by name in place of the built-in one: the platform's+ /// decoder wrapped as a codec, MediaCodec's.+ Named(&'static str),+}++/// Where a reader's codec runs when it is not plain software.+enum OnChip {+ /// A hwaccel: frames arrive in the device's memory, in this format,+ /// and are copied back.+ Accel(HwDevice, Pixel),+ /// The platform's own decoder behind a codec of its own: frames arrive+ /// in memory already. Holds its seat on the chip for as long as it+ /// lasts; see [`hardware::MEDIACODEC_READERS`].+ Wrapped { device: HwDevice, _seat: Seat },+}++impl OnChip {+ fn device(&self) -> HwDevice {+ match self {+ OnChip::Accel(device, _) | OnChip::Wrapped { device, .. } => *device,+ }+ }+}/// Anything that yields frames in order.///@@ -573,9 +604,18 @@ pub struct Decoder {/// that moved to software mid-file picks up, after the frame the/// device last gave it.discard_through: Option<Rational>,- /// The device the codec is on and the pixel format its frames arrive- /// in, or `None` in software, which is also where a failed device ends.- hardware: Option<(HwDevice, Pixel)>,+ /// Where the codec runs when it is not on the CPU, or `None` in+ /// software, which is also where a failed device ends.+ hardware: Option<OnChip>,+ /// A packet the codec refused while full, to send again; see+ /// [`crate::feed`].+ held: feed::Held<ffmpeg::Packet>,+ /// Whether the codec has been handed anything since it opened or was+ /// last flushed. A codec with nothing in it is not flushed: MediaCodec+ /// flushed before its first output has lost the stream's parameter+ /// sets and must be handed them again, which a seek straight after+ /// opening - every reader opened at a start point - would do.+ fed: bool,/// The last picture's timestamp, for picking up in software after it.last_pts: Option<Rational>,/// The output frame instant the pacer is at, when pacing.@@ -633,32 +673,94 @@ impl Decoder {Pixel::from(ffmpeg::sys::AVPixelFormat(raw)),)};- let device = options- .hardware- .device_for(codec, format)- .and_then(hardware::device);- let (decoder, accelerated) = match device {- Some(device) => match Self::open_codec(path, &stream, options, Some(&device)) {- Ok(opened) => opened,- Err(error) => {- hardware::note_fallback(device.kind(), "open the decoder", &error.to_string());- Self::open_codec(path, &stream, options, None)?+ let software = || Self::open_codec(path, &stream, options, Codec::Software);+ let (decoder, accelerated) = match options.hardware.device_for(codec, format) {+ // MediaCodec: its own decoder, by name, when a seat is free.+ // Not for a keyframes-only reader: the wrapper ignores+ // `skip_frame`, so a filmstrip tile would decode its whole+ // group of pictures instead of one frame, and its pipeline+ // holds the first picture back until several packets are in -+ // a one-frame grab is the chip's worst case and the CPU's best.+ Some(device @ HwDevice::MediaCodec) => {+ let wrapped = hardware::mediacodec_decoder(codec, format)+ .filter(|_| !options.keyframes_only)+ .filter(|&name| {+ hardware::mediacodec_works(name, || {+ first_picture_within(path, name, hardware::MEDIACODEC_FIRST_PICTURE)+ })+ })+ .and_then(|name| hardware::mediacodec_seat().map(|seat| (name, seat)));+ match wrapped {+ Some((name, seat)) => {+ match Self::open_codec(path, &stream, options, Codec::Named(name)) {+ Ok((decoder, _)) => {+ hardware::note_wrapped(device, name);+ (+ decoder,+ Some(OnChip::Wrapped {+ device,+ _seat: seat,+ }),+ )+ }+ Err(error) => {+ // Most often the phone's sessions are all+ // taken, by this app or another.+ hardware::note_fallback(+ device,+ "open the decoder",+ &error.to_string(),+ );+ software()?+ }+ }+ }+ None => software()?,+ }+ }+ Some(device) => match hardware::device(device) {+ Some(device) => {+ match Self::open_codec(path, &stream, options, Codec::On(&device)) {+ Ok(opened) => opened,+ Err(error) => {+ hardware::note_fallback(+ device.kind(),+ "open the decoder",+ &error.to_string(),+ );+ software()?+ }+ }}+ None => software()?,},- None => Self::open_codec(path, &stream, options, None)?,+ None => software()?,};// The caller's word over the file's, where the caller has one: the// file's tag is what the override exists to correct. Where neither// says, the codec's convention does; see `ColorRange::implied`.- let tagged = decoder.color_range();+ // Read off the stream, not the opened codec: MediaCodec's wrapper+ // writes the chip's own idea of the colour into the codec as it+ // starts - limited range on a stream that says nothing, NV12 for+ // the pixels - which is not what the file says.+ // SAFETY: a live stream's codec parameters; plain enum fields.+ let (tagged, primaries, transfer, matrix) = unsafe {+ let raw = stream.parameters().as_ptr();+ (+ ffmpeg::color::Range::from((*raw).color_range),+ ffmpeg::color::Primaries::from((*raw).color_primaries),+ ffmpeg::color::TransferCharacteristic::from((*raw).color_trc),+ ffmpeg::color::Space::from((*raw).color_space),+ )+ };let implied = options.color_range.is_none() && tagged == ffmpeg::color::Range::Unspecified;let color = ColorSignal {- primaries: decoder.color_primaries(),- transfer: decoder.color_transfer_characteristic(),- matrix: decoder.color_space(),+ primaries,+ transfer,+ matrix,range: match options.color_range {Some(range) => range.as_ffmpeg(),- None if implied => ColorRange::implied(decoder.id(), decoder.format()).as_ffmpeg(),+ None if implied => ColorRange::implied(codec, format).as_ffmpeg(),None => tagged,},implied,@@ -690,6 +792,8 @@ impl Decoder {discard_before: None,discard_through: None,hardware: accelerated,+ held: None,+ fed: false,last_pts: None,tick: 0,origin: Rational::ZERO,@@ -708,15 +812,14 @@ impl Decoder {Ok(this)}- /// Opens the stream's codec, on `device` where the codec can decode on- /// it. Gives back the opened decoder and, when it is on the device, the- /// device and the pixel format its frames arrive in.+ /// Opens the stream's codec as `codec` says. Gives back the opened+ /// decoder and, when it is on a hwaccel device, where.fn open_codec(path: &Path,stream: &ffmpeg::format::stream::Stream<'_>,options: &DecodeOptions,- device: Option<&Device>,- ) -> Result<(decoder::Video, Option<(HwDevice, Pixel)>)> {+ codec: Codec<'_>,+ ) -> Result<(decoder::Video, Option<OnChip>)> {let mut context = ffmpeg::codec::Context::from_parameters(stream.parameters()).map_err(|error| ffi::fail("codec parameters", path, error))?;// Every core. libavcodec's API opens a decoder on one thread unless@@ -733,15 +836,38 @@ impl Decoder {(*raw).thread_type = ffmpeg::sys::FF_THREAD_FRAME | ffmpeg::sys::FF_THREAD_SLICE;(*raw).thread_count = i32::from(options.threads);}- let accelerated = device.and_then(|device| {- device- .attach(&mut context)- .map(|format| (device.kind(), format))- });- let mut decoder = context- .decoder()- .video()- .map_err(|error| ffi::fail("open decoder", path, error))?;+ let (accelerated, named) = match codec {+ Codec::Software => (None, None),+ Codec::On(device) => (+ device+ .attach(&mut context)+ .map(|format| OnChip::Accel(device.kind(), format)),+ None,+ ),+ Codec::Named(name) => {+ let found = decoder::find_by_name(name).ok_or_else(|| Error::Missing {+ what: "decoder",+ name: name.to_owned(),+ })?;+ // MediaCodec counts time in microseconds; told the stream's+ // clock, the wrapper converts each packet's timestamp there+ // and every frame's back, exactly for any clock coarser than+ // a microsecond, which is every one a container uses.+ // SAFETY: the context is not opened yet; a plain rational.+ unsafe {+ (*context.as_mut_ptr()).pkt_timebase = stream.time_base().into();+ }+ (None, Some(found))+ }+ };+ let opened = match named {+ Some(found) => context+ .decoder()+ .open_as(found)+ .and_then(|opened| opened.video()),+ None => context.decoder().video(),+ };+ let mut decoder = opened.map_err(|error| ffi::fail("open decoder", path, error))?;if options.keyframes_only {decoder.skip_frame(ffmpeg::Discard::NonKey);}@@ -758,7 +884,7 @@ impl Decoder {/// without it, or once the codec has shown, with its first frame, that/// it declined the device for this stream.pub fn hardware(&self) -> Option<HwDevice> {- self.hardware.map(|(device, _)| device)+ self.hardware.as_ref().map(OnChip::device)}/// Reopens the codec in software and carries on from `from`: the frame@@ -774,7 +900,7 @@ impl Decoder {.ok_or_else(|| Error::NoVideoStream {path: self.path.clone(),})?;- Self::open_codec(&self.path, &stream, &self.options, None)?.0+ Self::open_codec(&self.path, &stream, &self.options, Codec::Software)?.0};let at = from.unwrap_or(self.origin);let target = ffi::av_ticks(at + self.start);@@ -782,7 +908,10 @@ impl Decoder {.seek(target, ..=target).map_err(|error| ffi::fail("seek", &self.path, error))?;self.decoder = decoder;+ // The old codec, and its seat on the chip, go now.self.hardware = None;+ self.held = None;+ self.fed = false;if !self.options.keyframes_only {if after {self.discard_through = Some(at);@@ -799,7 +928,14 @@ impl Decoder {self.input.seek(target, ..=target).map_err(|error| ffi::fail("seek", &self.path, error))?;- self.decoder.flush();+ // Every picture and packet the codec holds is from before the+ // seek: out, and so is the packet it refused. For MediaCodec this+ // is the codec's own flush, which it needs before a jump.+ if self.fed {+ self.decoder.flush();+ self.fed = false;+ }+ self.held = None;// Keyframes only: the picture the container landed on is the one// wanted, and discarding up to the target would throw it away and// wait for the *next* keyframe, a whole group of pictures late.@@ -817,104 +953,78 @@ impl Decoder {/// The next picture out of the codec, or `None` at the end of the file./// Frames before the discard point never come out of here, and a frame- /// from the device is copied back to memory before it does.+ /// from a hwaccel device is copied back to memory before it does.fn next_source(&mut self) -> Result<Option<Source>> {loop {- let mut frame = Video::empty();- match self.decoder.receive_frame(&mut frame) {- Ok(()) => {- let pts = frame- .timestamp()- .and_then(|ticks| ffi::seconds(ticks, self.time_base))- .map(|pts| pts - self.start);- if let Some((device, format)) = self.hardware {- if frame.format() == format {- frame = match hardware::download(&frame) {- Ok(copy) => copy,- Err(error) => {- hardware::note_fallback(- device,- "copy a frame back",- &error.to_string(),- );- self.resume_in_software(pts, false)?;- continue;- }- };- } else {- // The codec set the device up for this stream- // and the device said no, so the codec went on- // in software by itself; the frames are fine.- hardware::note_fallback(- device,- "take this stream",- "the codec decoded it in software",- );- self.hardware = None;- }- }- if let (Some(before), Some(pts)) = (self.discard_before, pts)- && pts < before- {- continue;- }- if let (Some(through), Some(pts)) = (self.discard_through, pts)- && pts <= through- {- continue;- }- self.discard_before = None;- self.discard_through = None;- self.last_pts = pts;- return Ok(Some(Source { frame, pts }));- }- Err(ffmpeg::Error::Eof) => return Ok(None),- Err(error) if ffi::is_again(&error) => {}- Err(error) if self.hardware.is_some() => {- // The device choked on a picture it had accepted the- // stream for. Software takes over after the last frame- // the reader was given.- let (device, _) = self.hardware.take().expect("checked");- hardware::note_fallback(device, "decode a frame", &error.to_string());+ // Packets for other streams are skipped; the end of the file+ // drains the codec so its last few frames come out at all; a+ // packet the codec refuses while full is kept and sent again.+ self.fed = true;+ let read = || feed::read_packet(&mut self.input, self.stream);+ let mut frame = match feed::next_frame(&mut self.decoder, &mut self.held, read) {+ Ok(Some(frame)) => frame,+ Ok(None) => return Ok(None),+ Err(feed::Stall::Read(error)) => return Err(ffi::fail("read", &self.path, error)),+ Err(stall) if self.hardware.is_some() => {+ // The device choked on a stream it had accepted.+ // Software takes over after the last frame the reader+ // was given.+ let device = self.hardware.take().expect("checked").device();+ let (stage, detail) = match &stall {+ feed::Stall::Send(error) => ("take a packet", error.to_string()),+ feed::Stall::Receive(error) => ("decode a frame", error.to_string()),+ _ => ("keep decoding", "it stopped taking packets".to_owned()),+ };+ hardware::note_fallback(device, stage, &detail);self.resume_in_software(self.last_pts, self.last_pts.is_some())?;continue;}- Err(error) => return Err(ffi::fail("decode", &self.path, error)),- }-- // The codec wants another packet. Packets for other streams are- // skipped; the end of the file flushes the codec so its last few- // frames come out at all.- loop {- let mut packet = ffmpeg::Packet::empty();- match packet.read(&mut self.input) {- Ok(()) => {- if packet.stream() != self.stream {+ Err(stall) => return Err(feed::fail(stall, &self.path)),+ };+ let pts = frame+ .timestamp()+ .and_then(|ticks| ffi::seconds(ticks, self.time_base))+ .map(|pts| pts - self.start);+ if let Some(&OnChip::Accel(device, format)) = self.hardware.as_ref() {+ if frame.format() == format {+ frame = match hardware::download(&frame) {+ Ok(copy) => copy,+ Err(error) => {+ hardware::note_fallback(+ device,+ "copy a frame back",+ &error.to_string(),+ );+ self.resume_in_software(pts, false)?;continue;}- match self.decoder.send_packet(&packet) {- Ok(()) => break,- Err(error) if ffi::is_again(&error) => break,- Err(error) if self.hardware.is_some() => {- let (device, _) = self.hardware.take().expect("checked");- hardware::note_fallback(- device,- "take a packet",- &error.to_string(),- );- self.resume_in_software(self.last_pts, self.last_pts.is_some())?;- break;- }- Err(error) => return Err(ffi::fail("send packet", &self.path, error)),- }- }- Err(ffmpeg::Error::Eof) => {- let _ = self.decoder.send_eof();- break;- }- Err(error) => return Err(ffi::fail("read", &self.path, error)),+ };+ } else {+ // The codec set the device up for this stream and the+ // device said no, so the codec went on in software by+ // itself; the frames are fine.+ hardware::note_fallback(+ device,+ "take this stream",+ "the codec decoded it in software",+ );+ self.hardware = None;}}+ if let (Some(before), Some(pts)) = (self.discard_before, pts)+ && pts < before+ {+ continue;+ }+ if let (Some(through), Some(pts)) = (self.discard_through, pts)+ && pts <= through+ {+ continue;+ }+ self.discard_before = None;+ self.discard_through = None;+ self.last_pts = pts;+ return Ok(Some(Source { frame, pts }));}}@@ -1125,6 +1235,51 @@ impl FrameSource for Decoder {}}+/// Whether MediaCodec's decoder `name` gives the first picture of+/// `path`'s video within `wait` - [`hardware::MEDIACODEC_FIRST_PICTURE`] for a+/// reader; see [`hardware::Verdicts`] for why a decoder is asked before it+/// is trusted.+///+/// The try runs on a thread of its own, with its own demuxer and codec,+/// because a decoder that wedges never returns control to the caller; a+/// wedged try is left behind on its thread, and the reader never waits on+/// it past the deadline. A try that answers late still has its say, for+/// the readers that come after.+fn first_picture_within(path: &Path, name: &'static str, wait: std::time::Duration) -> bool {+ let path = path.to_path_buf();+ let (answer, answered) = std::sync::mpsc::channel();+ let tried = std::thread::Builder::new()+ .name("mediacodec-first-picture".to_owned())+ .spawn(move || {+ let attempt = || -> Option<()> {+ let mut input = ffmpeg::format::input(&path).ok()?;+ let (mut codec, index) = {+ let stream = input.streams().best(ffmpeg::media::Type::Video)?;+ let (codec, _) = Decoder::open_codec(+ &path,+ &stream,+ &DecodeOptions::default(),+ Codec::Named(name),+ )+ .ok()?;+ (codec, stream.index())+ };+ let mut held = None;+ let read = || feed::read_packet(&mut input, index);+ feed::next_frame(&mut codec, &mut held, read)+ .ok()?+ .map(|_| ())+ };+ let works = attempt().is_some();+ let _ = answer.send(works);+ // Late or not, what it found is the verdict from now on: a+ // decoder that was only slow to start is not written off for+ // the life of the process.+ hardware::mediacodec_answered(name, works);+ });+ tried.is_ok() && answered.recv_timeout(wait).unwrap_or(false)+}+/// A second reference to `frame`'s picture: the same buffers, counted,/// so a graph that takes the reference (`buffersrc` does) takes nothing/// from the frame's owner. `Video::clone` copies the pixels - fifty@@ -1678,6 +1833,127 @@ mod tests {);}+ /// On a phone or the emulator, with a real file in+ /// `CONCAT_DEVICE_SAMPLE`: MediaCodec decodes it to the frames the CPU+ /// does - every frame of the first `CONCAT_DEVICE_FRAMES` (default 90)+ /// at the same instant, within the rounding of a colour conversion -+ /// and a seek forward, back and to the very start lands on the frame+ /// the CPU's seek lands on. Prints how long each took, which on the+ /// emulator measures its software MediaCodec and nothing a phone+ /// does. With `CONCAT_DEVICE_OUT`, writes the chip's frame at each+ /// seek there as a JPEG to look at.+ ///+ /// Built with `cargo test -p concat-media --no-run --target+ /// aarch64-linux-android`, pushed with the file to /data/local/tmp,+ /// and run there with `--ignored mediacodec`.+ #[test]+ #[ignore = "needs Android and a sample in CONCAT_DEVICE_SAMPLE"]+ fn mediacodec_decodes_a_real_file_as_the_cpu_does() {+ use std::time::Instant;+ let Some(path) = std::env::var_os("CONCAT_DEVICE_SAMPLE").map(PathBuf::from) else {+ return;+ };+ let count: usize = std::env::var("CONCAT_DEVICE_FRAMES")+ .ok()+ .and_then(|count| count.parse().ok())+ .unwrap_or(90);+ let out = std::env::var_os("CONCAT_DEVICE_OUT").map(PathBuf::from);+ let chip = DecodeOptions::default().accelerated(HwDevice::MediaCodec);+ let cpu = DecodeOptions::default().in_software();++ let take = |options: &DecodeOptions| {+ let started = Instant::now();+ let mut decoder = Decoder::open(&path, options).expect("opens");+ let mut frames = Vec::new();+ while frames.len() < count {+ let Some(frame) = decoder.next_frame().expect("decodes") else {+ break;+ };+ frames.push((decoder.position(), frame));+ }+ (decoder.hardware(), frames, started.elapsed())+ };+ // How long the platform's decoder takes to its first picture, the+ // number the first-picture deadline is measured against.+ {+ let started = Instant::now();+ let name = hardware::mediacodec_decoder(+ ffmpeg::format::input(&path)+ .expect("opens")+ .streams()+ .best(ffmpeg::media::Type::Video)+ .expect("a video stream")+ .parameters()+ .id(),+ Pixel::YUV420P,+ )+ .expect("a codec MediaCodec takes");+ let answered = first_picture_within(&path, name, std::time::Duration::from_secs(60));+ eprintln!(+ "{name}: first picture {} in {:?}",+ if answered { "came" } else { "did not come" },+ started.elapsed()+ );+ }+ let (on_chip, chip_frames, chip_took) = take(&chip);+ let (_, cpu_frames, cpu_took) = take(&cpu);+ assert_eq!(+ on_chip,+ Some(HwDevice::MediaCodec),+ "still on the chip at the end"+ );+ eprintln!(+ "{} frames of {}x{}: MediaCodec {chip_took:?}, CPU {cpu_took:?}",+ chip_frames.len(),+ chip_frames[0].1.width(),+ chip_frames[0].1.height()+ );+ assert_eq!(chip_frames.len(), cpu_frames.len());+ for (index, ((chip_at, chip_frame), (cpu_at, cpu_frame))) in+ chip_frames.iter().zip(&cpu_frames).enumerate()+ {+ assert_eq!(chip_at, cpu_at, "frame {index} at the same instant");+ let difference = worst_difference(chip_frame, cpu_frame);+ assert!(difference <= 8, "frame {index} differs by {difference}");+ }++ // Seeks on one open reader each, the way the pool's warm readers+ // take them: forward, back, forward again, and to the start.+ let mut chip_reader = Decoder::open(&path, &chip).expect("opens");+ let mut cpu_reader = Decoder::open(&path, &cpu).expect("opens");+ for (step, to) in [(7, 3), (1, 1), (5, 2), (0, 1), (11, 4)]+ .into_iter()+ .enumerate()+ {+ let to = Rational::new(to.0, to.1);+ let started = Instant::now();+ chip_reader.seek(to).expect("seeks");+ let landed = chip_reader.next_frame().expect("decodes").expect("a frame");+ let took = started.elapsed();+ cpu_reader.seek(to).expect("seeks");+ let wanted = cpu_reader.next_frame().expect("decodes").expect("a frame");+ assert_eq!(+ chip_reader.position(),+ cpu_reader.position(),+ "seek to {to:?}"+ );+ let difference = worst_difference(&landed, &wanted);+ eprintln!(+ "seek to {to:?} landed at {:?} in {took:?}, {difference} levels off",+ chip_reader.position()+ );+ assert!(+ difference <= 8,+ "the seek to {to:?} landed {difference} levels off"+ );+ if let Some(out) = &out {+ let jpeg = crate::encode::jpeg(&landed, 90).expect("encodes");+ std::fs::write(out.join(format!("seek-{step}.jpg")), jpeg).expect("writes");+ }+ }+ assert_eq!(chip_reader.hardware(), Some(HwDevice::MediaCodec));+ }+/// The pacer sits on top of the device exactly as it does the CPU: a/// file paced to twice its rate gives the same frames either way.#[test]diff --git a/src/crates/concat-media/src/hardware.rs b/src/crates/concat-media/src/hardware.rsindex 10adc50a..60aec0bd 100644--- a/src/crates/concat-media/src/hardware.rs+++ b/src/crates/concat-media/src/hardware.rs@@ -30,6 +30,13 @@//! default and only ever by name, because on an Intel i915 it has hung the//! GPU for every process on the machine until a reboot (issue #70).//!+//! MediaCodec goes through a different door. It is no hwaccel: FFmpeg+//! reaches it only through its own decoders, `h264_mediacodec` and the+//! rest, opened by name in place of the built-in one, which hand frames+//! back in memory with no copy to make ([`mediacodec_decoder`]). A phone+//! has few decoder sessions to give, so at most [`MEDIACODEC_READERS`]+//! readers hold one at a time and the rest decode in software.+//!//! ## What happens when it does not work//!//! Everything falls back to software, and nothing a person sees changes but@@ -50,8 +57,9 @@use std::collections::HashMap;use std::ptr;-use std::sync::atomic::{AtomicU8, Ordering};+use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering};use std::sync::{Mutex, Once};+use std::time::Duration;use ffmpeg_the_third as ffmpeg;use ffmpeg_the_third::codec::decoder;@@ -196,10 +204,10 @@ impl HwPolicy {/// software regardless, as before.////// A phone's cores are no match for its decoder, and MediaCodec takes-/// everything it can.+/// everything [`mediacodec_decoder`] has a decoder for.pub fn hardware_wins(device: HwDevice, codec: ffmpeg::codec::Id, format: Pixel) -> bool {if device == HwDevice::MediaCodec {- return true;+ return mediacodec_decoder(codec, format).is_some();}// SAFETY: a descriptor points into libavutil's static table, and every// pixel format has at least one component.@@ -209,6 +217,222 @@ pub fn hardware_wins(device: HwDevice, codec: ffmpeg::codec::Id, format: Pixel)codec != ffmpeg::codec::Id::H264 || deep}+/// The libavcodec decoder that drives MediaCodec for a stream of `codec`+/// in `format`, or `None` where the stream stays in software.+///+/// MediaCodec is not a hwaccel the way the desktop devices are: FFmpeg's+/// own H.264 and HEVC decoders know nothing of it, and a device context+/// handed to them changes nothing. It is a set of separate decoders, one+/// per codec, opened by name, which feed the packets to the phone's chip+/// and, with no output surface, hand back each picture in memory as NV12+/// or planar 4:2:0 with the chip's stride and crop already undone -+/// frames the filtergraph takes as it takes a software decoder's.+///+/// Eight-bit 4:2:0 only. Ten-bit HEVC comes back from the chip as P010,+/// which FFmpeg's wrapper has no mapping for (it fails), or squeezed to+/// eight bits by some vendors (which would quietly lose the HDR the deep+/// path keeps); it stays in software until that is settled on a phone.+/// VP8, MPEG-4 Part 2 and MPEG-2 have wrappers too, but are rare on a+/// phone and cheap in software. AV1 goes to the chip, or to the+/// platform's own software decoder where the chip has none: this FFmpeg+/// has no AV1 decoder of its own.+pub fn mediacodec_decoder(codec: ffmpeg::codec::Id, format: Pixel) -> Option<&'static str> {+ use ffmpeg::codec::Id;+ if !matches!(format, Pixel::YUV420P | Pixel::YUVJ420P) {+ return None;+ }+ match codec {+ Id::H264 => Some("h264_mediacodec"),+ Id::HEVC => Some("hevc_mediacodec"),+ Id::VP9 => Some("vp9_mediacodec"),+ Id::AV1 => Some("av1_mediacodec"),+ _ => None,+ }+}++/// Readers decoding on MediaCodec at once, at most; the rest decode in+/// software.+///+/// A phone has a fixed budget of decoder sessions - a count, and under it+/// a ceiling on pixels a second across all of them, which a mid-range+/// chip spends on two or three 4K streams. Past it, `configure` fails and+/// the reader falls back anyway, but only after the attempt; and every+/// session spent on a reader sitting idle in the pool is one the camera,+/// the export's MediaCodec encoder or another app cannot have. Concat+/// keeps up to eight warm readers plus the filmstrip's and the export's,+/// while what a person watches needs few at once: the clip under the+/// playhead, the next one ready at a cut, and one more for a layer over+/// them or a scrub elsewhere. So three; the fourth reader on decodes on+/// the CPU as before, and a seat comes free when its reader closes or+/// falls back.+pub const MEDIACODEC_READERS: usize = 3;++/// A bounded count of readers holding something scarce.+pub(crate) struct Seats {+ taken: AtomicUsize,+ limit: usize,+}++impl Seats {+ pub(crate) const fn new(limit: usize) -> Self {+ Self {+ taken: AtomicUsize::new(0),+ limit,+ }+ }++ /// A seat, or `None` when all of them are taken. The seat is given+ /// back when it drops.+ pub(crate) fn take(&'static self) -> Option<Seat> {+ self.taken+ .fetch_update(Ordering::AcqRel, Ordering::Acquire, |taken| {+ (taken < self.limit).then_some(taken + 1)+ })+ .ok()+ .map(|_| Seat { seats: self })+ }++ /// How many seats are taken right now.+ #[cfg(test)]+ pub(crate) fn taken(&self) -> usize {+ self.taken.load(Ordering::Acquire)+ }+}++/// One taken seat; see [`Seats`].+pub(crate) struct Seat {+ seats: &'static Seats,+}++impl Drop for Seat {+ fn drop(&mut self) {+ self.seats.taken.fetch_sub(1, Ordering::AcqRel);+ }+}++impl std::fmt::Debug for Seat {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.write_str("Seat")+ }+}++/// The readers on MediaCodec; see [`MEDIACODEC_READERS`].+static MEDIACODEC_SEATS: Seats = Seats::new(MEDIACODEC_READERS);++/// A seat on MediaCodec for a reader about to open one of its decoders,+/// or `None` when [`MEDIACODEC_READERS`] already have one - said once a+/// process, since a busy timeline says it all the time.+pub(crate) fn mediacodec_seat() -> Option<Seat> {+ let seat = MEDIACODEC_SEATS.take();+ if seat.is_none() {+ static ONCE: Once = Once::new();+ ONCE.call_once(|| {+ log::info!(+ "{MEDIACODEC_READERS} readers already decode on MediaCodec; more decode in software"+ );+ });+ }+ seat+}++/// How long a MediaCodec decoder has to give its first picture the first+/// time it is tried. A phone's chip gives it in tens of milliseconds and+/// the platform's software decoders in a few hundred; the wait is only+/// ever spent in full on a decoder that is never going to answer.+pub const MEDIACODEC_FIRST_PICTURE: Duration = Duration::from_secs(2);++/// What each MediaCodec decoder did the first time this process tried+/// it: gave a picture, or did not.+///+/// The first try matters because a platform decoder that wedges does not+/// fail, it hangs: FFmpeg's wrapper waits for an input or an output+/// buffer that never comes, in a loop inside `avcodec_send_packet` that+/// nothing outside it can break. The emulator's own goldfish H.264+/// decoder does exactly that with no host GPU, and a reader that met it+/// froze the preview for good. So each decoder is tried once, off to the+/// side with a deadline (see [`MEDIACODEC_FIRST_PICTURE`]), before any+/// reader trusts it.+pub(crate) struct Verdicts {+ said: Mutex<Vec<(&'static str, bool)>>,+}++impl Verdicts {+ pub(crate) const fn new() -> Self {+ Self {+ said: Mutex::new(Vec::new()),+ }+ }++ /// Whether `name` works: what it did when first tried, or, the first+ /// time, what `try_it` finds - once, with every other reader asking+ /// meanwhile waiting for the answer rather than trying it again.+ pub(crate) fn works(&self, name: &'static str, try_it: impl FnOnce() -> bool) -> bool {+ let mut said = self+ .said+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner());+ if let Some(&(_, works)) = said.iter().find(|(said, _)| *said == name) {+ return works;+ }+ let works = try_it();+ said.push((name, works));+ works+ }++ /// Records what a try found after its deadline had passed: a decoder+ /// that was only slow the first time - a phone just woken, a codec+ /// service starting cold - is trusted from here on. Waits for the+ /// first verdict to be written, so the late one is the one that stays.+ pub(crate) fn amend(&self, name: &'static str, works: bool) {+ let mut said = self+ .said+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner());+ match said.iter_mut().find(|(said, _)| *said == name) {+ Some(verdict) => verdict.1 = works,+ None => said.push((name, works)),+ }+ }+}++/// Every MediaCodec decoder's first try; see [`Verdicts`].+static MEDIACODEC_VERDICTS: Verdicts = Verdicts::new();++/// Whether MediaCodec's `name` can be trusted with a reader: what it did+/// the first time `try_it` tried it in this process. A decoder that did+/// not give a picture in time is said once and not tried again, unless+/// that first try answers late; see [`mediacodec_answered`].+pub(crate) fn mediacodec_works(name: &'static str, try_it: impl FnOnce() -> bool) -> bool {+ MEDIACODEC_VERDICTS.works(name, || {+ let works = try_it();+ if !works {+ log::warn!(+ "MediaCodec's {name} gave no picture within {:?}; that codec decodes in software",+ MEDIACODEC_FIRST_PICTURE+ );+ }+ works+ })+}++/// What a first try of `name` found once it finished, deadline or not;+/// see [`Verdicts::amend`].+pub(crate) fn mediacodec_answered(name: &'static str, works: bool) {+ MEDIACODEC_VERDICTS.amend(name, works);+}++/// Writes down, once per decoder, that a reader really is decoding on the+/// platform's decoder through `name`: the chip is said to be in use only+/// once a codec has opened on it.+pub(crate) fn note_wrapped(device: HwDevice, name: &'static str) {+ static SAID: Mutex<Vec<&'static str>> = Mutex::new(Vec::new());+ let mut said = SAID.lock().unwrap_or_else(|poisoned| poisoned.into_inner());+ if !said.contains(&name) {+ said.push(name);+ log::info!("video decode on {} through {name}", device.label());+ }+}+/// The process-wide preference: `0` for software, else one past the/// device's index in [`HwDevice::ALL`]. An atomic rather than a lock/// because every `Decoder::open` reads it.@@ -493,11 +717,114 @@ mod tests {assert!(hardware_wins(vt, Id::HEVC, Pixel::YUV420P));assert!(hardware_wins(vt, Id::HEVC, Pixel::YUV420P10LE));assert!(hardware_wins(vt, Id::PRORES, Pixel::YUV422P10LE));- assert!(hardware_wins(- HwDevice::MediaCodec,- Id::H264,- Pixel::YUV420P- ));+ let phone = HwDevice::MediaCodec;+ assert!(hardware_wins(phone, Id::H264, Pixel::YUV420P));+ assert!(hardware_wins(phone, Id::HEVC, Pixel::YUVJ420P));+ assert!(hardware_wins(phone, Id::AV1, Pixel::YUV420P));+ // Ten bits and 4:2:2 stay on the CPU, and so do codecs it has no+ // decoder for here.+ assert!(!hardware_wins(phone, Id::HEVC, Pixel::YUV420P10LE));+ assert!(!hardware_wins(phone, Id::H264, Pixel::YUV422P));+ assert!(!hardware_wins(phone, Id::PRORES, Pixel::YUV422P10LE));+ assert!(!hardware_wins(phone, Id::MPEG4, Pixel::YUV420P));+ }++ /// MediaCodec's decoders are opened by name, never found by codec id:+ /// the built-in H.264 decoder has no MediaCodec entry at all, which is+ /// how every phone decoded on its CPU while the log said otherwise.+ #[test]+ fn mediacodec_decoders_are_the_named_wrappers() {+ use ffmpeg::codec::Id;+ assert_eq!(+ mediacodec_decoder(Id::H264, Pixel::YUV420P),+ Some("h264_mediacodec")+ );+ assert_eq!(+ mediacodec_decoder(Id::HEVC, Pixel::YUV420P),+ Some("hevc_mediacodec")+ );+ assert_eq!(mediacodec_decoder(Id::HEVC, Pixel::YUV420P10LE), None);+ crate::ffi::init();+ let h264 = decoder::find(Id::H264).expect("an H.264 decoder");+ assert_eq!(+ hardware_format(h264.as_ptr(), sys::AVHWDeviceType::MEDIACODEC),+ None,+ "the built-in decoder has grown a MediaCodec hwaccel; revisit"+ );+ }++ #[test]+ fn seats_run_out_and_come_back() {+ static SEATS: Seats = Seats::new(2);+ let first = SEATS.take().expect("a seat");+ let second = SEATS.take().expect("a second seat");+ assert!(SEATS.take().is_none(), "two is the limit");+ assert_eq!(SEATS.taken(), 2);+ drop(first);+ assert_eq!(SEATS.taken(), 1);+ let third = SEATS.take().expect("the freed seat");+ assert!(SEATS.take().is_none());+ drop((second, third));+ assert_eq!(SEATS.taken(), 0);+ }++ /// A decoder is tried once; its verdict stands, good or bad, and the+ /// readers asking at the same time wait for it rather than each+ /// trying it again.+ #[test]+ fn a_decoder_is_tried_once_and_the_verdict_stands() {+ static VERDICTS: Verdicts = Verdicts::new();+ let tries = AtomicUsize::new(0);+ let try_it = |answer: bool| {+ let tries = &tries;+ move || {+ tries.fetch_add(1, Ordering::SeqCst);+ std::thread::sleep(Duration::from_millis(20));+ answer+ }+ };+ std::thread::scope(|scope| {+ for _ in 0..4 {+ scope.spawn(|| assert!(!VERDICTS.works("wedged", try_it(false))));+ }+ });+ assert_eq!(tries.load(Ordering::SeqCst), 1);+ assert!(VERDICTS.works("fine", try_it(true)));+ assert!(+ VERDICTS.works("fine", try_it(false)),+ "the first verdict stands"+ );+ assert!(!VERDICTS.works("wedged", try_it(true)));+ assert_eq!(tries.load(Ordering::SeqCst), 2);++ // A try that answers after its deadline is believed from then on.+ VERDICTS.amend("wedged", true);+ assert!(VERDICTS.works("wedged", try_it(false)));+ assert_eq!(tries.load(Ordering::SeqCst), 2);+ }++ #[test]+ fn seats_hold_under_contention() {+ static SEATS: Seats = Seats::new(3);+ let peak = std::sync::Arc::new(AtomicUsize::new(0));+ let threads: Vec<_> = (0..8)+ .map(|_| {+ let peak = std::sync::Arc::clone(&peak);+ std::thread::spawn(move || {+ for _ in 0..2000 {+ if let Some(seat) = SEATS.take() {+ peak.fetch_max(SEATS.taken(), Ordering::AcqRel);+ drop(seat);+ }+ }+ })+ })+ .collect();+ for thread in threads {+ thread.join().expect("joins");+ }+ assert!(peak.load(Ordering::Acquire) <= 3);+ assert_eq!(SEATS.taken(), 0);}#[test]