The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
fix(export): a phone's encoder that says "try again" is waited for, frame by frame and at the end (#286)
Jareer447570a618f2
diff --git a/src/crates/concat-media/src/encode.rs b/src/crates/concat-media/src/encode.rsindex f600f6d4..0e2ba730 100644--- a/src/crates/concat-media/src/encode.rs+++ b/src/crates/concat-media/src/encode.rs@@ -1284,8 +1284,74 @@ impl Encoder {self.written}+ /// Hands the encoder a frame, or the end of the stream with `None`, as+ /// FFmpeg's send-and-receive contract asks: an encoder that answers+ /// "try again" has finished packets waiting to be read, and takes the+ /// same frame once they are. Android's MediaCodec encoders say so+ /// whenever their input buffers are full, which on a phone is most+ /// exports; treated as a failure, it ended every H.264 export there+ /// a fraction of a second in. An encoder that keeps saying it for+ /// [`ENCODER_PATIENCE`] is stuck, and the export fails saying so.+ fn send(&mut self, frame: Option<&Video>) -> Result<()> {+ let sent = patiently(+ self,+ |this| match frame {+ Some(frame) => this.encoder.send_frame(frame),+ None => this.encoder.send_eof(),+ },+ ffi::is_again,+ Self::drain,+ ENCODER_PATIENCE,+ )?;+ sent.map_err(|refused| match refused {+ Refused::Error(error) => ffi::fail("encode", &self.path, error),+ Refused::Stalled => Error::Ffi {+ operation: "encode",+ path: self.path.clone(),+ detail: format!(+ "the encoder took no {} for {} seconds",+ if frame.is_some() {+ "frame"+ } else {+ "end of the stream"+ },+ ENCODER_PATIENCE.as_secs()+ ),+ },+ })+ }++ /// Reads out what the encoder still holds once it has been told the+ /// stream ended, until it says it has handed back the last packet.+ /// FFmpeg's contract has no "try again" here, but its MediaCodec+ /// wrapper waits a few milliseconds for each output buffer and says+ /// "try again" when none has come yet, frames still inside it; stopping+ /// there cut the end off a phone's export with no error. An encoder+ /// that hands back nothing for [`ENCODER_PATIENCE`] is stuck, and the+ /// export fails saying so rather than writing a file short of its end.+ fn flush(&mut self) -> Result<()> {+ if until_ended(self, Self::read_out, ENCODER_PATIENCE)? {+ return Ok(());+ }+ Err(Error::Ffi {+ operation: "encode",+ path: self.path.clone(),+ detail: format!(+ "the encoder held its last frames for {} seconds",+ ENCODER_PATIENCE.as_secs()+ ),+ })+ }+/// Writes every packet the encoder has ready.fn drain(&mut self) -> Result<()> {+ self.read_out().map(|_| ())+ }++ /// Writes every packet the encoder has ready, and says whether that was+ /// its last or it has nothing more for now.+ fn read_out(&mut self) -> Result<Drained> {+ let mut packets = 0;loop {let mut packet = ffmpeg::Packet::empty();match self.encoder.receive_packet(&mut packet) {@@ -1304,15 +1370,90 @@ impl Encoder {packet.write_interleaved(&mut self.output).map_err(|error| ffi::fail("write packet", &self.path, error))?;+ packets += 1;}- Err(ffmpeg::Error::Eof) => return Ok(()),- Err(error) if ffi::is_again(&error) => return Ok(()),+ Err(ffmpeg::Error::Eof) => return Ok(Drained::Ended),+ Err(error) if ffi::is_again(&error) => return Ok(Drained::Again { packets }),Err(error) => return Err(ffi::fail("encode", &self.path, error)),}}}}+/// How long an encoder may keep answering "try again" to one frame, its+/// finished packets read out between asks, before the export calls it+/// stuck. A phone's MediaCodec encoder frees an input buffer within a few+/// milliseconds of its output being read; ten seconds is a stall.+const ENCODER_PATIENCE: std::time::Duration = std::time::Duration::from_secs(10);+/// The pause between two asks, so a busy hardware encoder is waited for+/// rather than spun on.+const ENCODER_WAIT: std::time::Duration = std::time::Duration::from_millis(2);++/// Why [`patiently`] gave up on a send.+#[derive(Debug, PartialEq)]+enum Refused<E> {+ /// The encoder refused it for a reason of its own.+ Error(E),+ /// The encoder kept saying "try again" for the whole of the patience.+ Stalled,+}++/// `send` until it is taken: on an error `again` calls a "try again",+/// `drain` reads out what the encoder has finished and `send` is asked+/// once more, for up to `patience`. Any other error ends it at once. A+/// failure to drain is the outer error; the send's own outcome the inner.+fn patiently<S, E>(+ state: &mut S,+ mut send: impl FnMut(&mut S) -> std::result::Result<(), E>,+ again: impl Fn(&E) -> bool,+ mut drain: impl FnMut(&mut S) -> Result<()>,+ patience: std::time::Duration,+) -> Result<std::result::Result<(), Refused<E>>> {+ let started = std::time::Instant::now();+ loop {+ match send(state) {+ Ok(()) => return Ok(Ok(())),+ Err(error) if !again(&error) => return Ok(Err(Refused::Error(error))),+ Err(_) if started.elapsed() >= patience => return Ok(Err(Refused::Stalled)),+ Err(_) => {+ drain(state)?;+ std::thread::sleep(ENCODER_WAIT);+ }+ }+ }+}++/// What reading an encoder's packets out came to.+#[derive(Debug, PartialEq)]+enum Drained {+ /// It has handed back its last packet.+ Ended,+ /// It has nothing more for now; `packets` came out before it said so.+ Again { packets: usize },+}++/// `read_out` until the encoder says it has handed back its last packet,+/// pausing [`ENCODER_WAIT`] between reads that end in "try again". The+/// patience runs from the last packet, so an encoder still handing frames+/// back is waited on however long its tail is; one that hands back nothing+/// for `patience` gives `false`.+fn until_ended<S>(+ state: &mut S,+ mut read_out: impl FnMut(&mut S) -> Result<Drained>,+ patience: std::time::Duration,+) -> Result<bool> {+ let mut last = std::time::Instant::now();+ loop {+ match read_out(state)? {+ Drained::Ended => return Ok(true),+ Drained::Again { packets } if packets > 0 => last = std::time::Instant::now(),+ Drained::Again { .. } if last.elapsed() >= patience => return Ok(false),+ Drained::Again { .. } => {}+ }+ std::thread::sleep(ENCODER_WAIT);+ }+}+impl FrameSink for Encoder {fn write_frame(&mut self, frame: &Frame) -> Result<()> {if frame.width() != self.width || frame.height() != self.height {@@ -1371,9 +1512,7 @@ impl FrameSink for Encoder {(*converted.as_mut_ptr()).color_range = self.color_range.into();}- self.encoder- .send_frame(&converted)- .map_err(|error| ffi::fail("encode", &self.path, error))?;+ self.send(Some(&converted))?;self.written += 1;self.drain()}@@ -1383,10 +1522,8 @@ impl FrameSink for Encoder {return Ok(());}self.finished = true;- self.encoder- .send_eof()- .map_err(|error| ffi::fail("encode", &self.path, error))?;- self.drain()?;+ self.send(None)?;+ self.flush()?;if let Some(light) = &self.light {// The light the file holds, measured as it was written, in the// container ('clli'), which the muxer writes with the index.@@ -1494,6 +1631,131 @@ pub fn jpeg(frame: &Frame, quality: u8) -> Result<Vec<u8>> {mod tests {use super::*;+ /// A "try again" is not a failure: the finished packets are read out+ /// and the same frame is sent again, as often as it takes (#286: every+ /// H.264 export on an Android phone ended there). Any other error ends+ /// the send at once, and an encoder that never takes the frame is+ /// called stuck rather than waited on for ever.+ #[test]+ fn a_frame_the_encoder_is_not_ready_for_is_sent_again() {+ #[derive(Default)]+ struct Fake {+ refusals: u32,+ sends: u32,+ drains: u32,+ }+ let again = |error: &&str| *error == "again";+ let patience = std::time::Duration::from_secs(5);++ let mut busy = Fake {+ refusals: 3,+ ..Fake::default()+ };+ let sent = patiently(+ &mut busy,+ |fake| {+ fake.sends += 1;+ if fake.refusals > 0 {+ fake.refusals -= 1;+ Err("again")+ } else {+ Ok(())+ }+ },+ again,+ |fake| {+ fake.drains += 1;+ Ok(())+ },+ patience,+ )+ .unwrap();+ assert_eq!(sent, Ok(()));+ assert_eq!((busy.sends, busy.drains), (4, 3));++ let mut broken = Fake::default();+ let sent = patiently(+ &mut broken,+ |fake| {+ fake.sends += 1;+ Err("invalid")+ },+ again,+ |fake| {+ fake.drains += 1;+ Ok(())+ },+ patience,+ )+ .unwrap();+ assert_eq!(sent, Err(Refused::Error("invalid")));+ assert_eq!((broken.sends, broken.drains), (1, 0));++ let mut stuck = Fake::default();+ let sent = patiently(+ &mut stuck,+ |_| Err("again"),+ again,+ |_| Ok(()),+ std::time::Duration::ZERO,+ )+ .unwrap();+ assert_eq!(sent, Err(Refused::Stalled));+ }++ /// After the end of the stream, an encoder that says "try again" with+ /// frames still inside it (MediaCodec's) is read again until it says it+ /// has handed back the last, and one that hands back nothing for the+ /// whole of the patience is called stuck rather than taken as done.+ #[test]+ fn the_last_frames_are_waited_for() {+ let patience = std::time::Duration::from_secs(5);++ // Two empty reads, one with the tail, two more empty, then the end.+ let mut reads = vec![+ Drained::Ended,+ Drained::Again { packets: 0 },+ Drained::Again { packets: 0 },+ Drained::Again { packets: 3 },+ Drained::Again { packets: 0 },+ Drained::Again { packets: 0 },+ ];+ let mut asked = 0;+ let ended = until_ended(+ &mut reads,+ |reads| {+ asked += 1;+ Ok(reads.pop().expect("read past the end"))+ },+ patience,+ )+ .unwrap();+ assert!(ended);+ assert_eq!(asked, 6);++ let ended = until_ended(+ &mut (),+ |_| Ok(Drained::Again { packets: 0 }),+ std::time::Duration::ZERO,+ )+ .unwrap();+ assert!(!ended);++ let mut failing = ();+ let failed = until_ended(+ &mut failing,+ |_| {+ Err(Error::Ffi {+ operation: "encode",+ path: std::path::PathBuf::new(),+ detail: "broken".into(),+ })+ },+ patience,+ );+ assert!(failed.is_err());+ }+/// A constant-bitrate export opens with each software encoder and comes/// out near the rate asked for. Noise, so the content costs more than/// the target and the encoder has to hold it rather than coast under.