The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
fix(media): a packet the codec refuses while full is sent again, not dropped
Jareerac8a6548b031
diff --git a/src/crates/concat-media/src/audio.rs b/src/crates/concat-media/src/audio.rsindex 71c446d2..7f904e30 100644--- a/src/crates/concat-media/src/audio.rs+++ b/src/crates/concat-media/src/audio.rs@@ -337,6 +337,8 @@ struct MixInput {input: ffmpeg::format::context::Input,stream: usize,decoder: ffmpeg::codec::decoder::Audio,+ /// A packet the codec refused while full; see [`crate::feed`].+ held: crate::feed::Held<ffmpeg::Packet>,label: String,done: bool,/// Samples already sent to the graph for this clip.@@ -351,37 +353,14 @@ impl MixInput {if self.done {return Ok(None);}- loop {- let mut frame = Audio::empty();- match self.decoder.receive_frame(&mut frame) {- Ok(()) => return Ok(Some(frame)),- Err(ffmpeg::Error::Eof) => {- self.done = true;- return Ok(None);- }- Err(error) if ffi::is_again(&error) => {}- Err(error) => return Err(ffi::fail("decode", &self.path, error)),- }- loop {- let mut packet = ffmpeg::Packet::empty();- match packet.read(&mut self.input) {- Ok(()) => {- if packet.stream() != self.stream {- continue;- }- match self.decoder.send_packet(&packet) {- Ok(()) => break,- Err(error) if ffi::is_again(&error) => break,- Err(error) => return Err(ffi::fail("send packet", &self.path, error)),- }- }- Err(ffmpeg::Error::Eof) => {- let _ = self.decoder.send_eof();- break;- }- Err(error) => return Err(ffi::fail("read", &self.path, error)),- }+ let read = || crate::feed::read_packet(&mut self.input, self.stream);+ match crate::feed::next_frame(&mut self.decoder, &mut self.held, read) {+ Ok(Some(frame)) => Ok(Some(frame)),+ Ok(None) => {+ self.done = true;+ Ok(None)}+ Err(stall) => Err(crate::feed::fail(stall, &self.path)),}}}@@ -438,6 +417,7 @@ pub fn mix_to_file(clips: &[AudioClip], duration: f64, destination: &Path) -> Reinput,stream: stream_index,decoder,+ held: None,label,done: false,samples_sent: 0,diff --git a/src/crates/concat-media/src/feed.rs b/src/crates/concat-media/src/feed.rsnew file mode 100644index 00000000..d29ede2e--- /dev/null+++ b/src/crates/concat-media/src/feed.rs@@ -0,0 +1,376 @@+// SPDX-License-Identifier: AGPL-3.0-or-later+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors++//! Keeping a codec fed: the send-packet, receive-frame loop every decoder+//! in this crate runs, written once.+//!+//! libavcodec's contract is that `avcodec_send_packet` may refuse a packet+//! with `EAGAIN` when the codec has frames to hand back first, and that the+//! refused packet is to be sent again once they have been taken. FFmpeg's+//! own software decoders almost never say it, so the loops here used to+//! read past the refusal and move on to the next packet - dropping the+//! refused one. A codec that wraps the platform's decoder says it all the+//! time: MediaCodec's input queue holds a handful of packets, and every+//! packet dropped when it was full was a hole in the stream, a smear of+//! grey blocks until the next keyframe. [`next_frame`] keeps the packet+//! and sends it again after draining what the codec had.++use ffmpeg_the_third as ffmpeg;+use ffmpeg_the_third::codec::decoder;+use ffmpeg_the_third::util::frame::{audio::Audio, video::Video};++use std::path::Path;++use crate::error::Error;+use crate::ffi;++/// What goes into a codec: a packet, or the end of the stream.+#[derive(Debug)]+pub(crate) enum Input<P> {+ Packet(P),+ End,+}++/// A packet the codec refused with `EAGAIN`, kept to be sent again; or+/// nothing. Cleared whenever the codec is flushed or replaced, since the+/// packet belonged to the stream position before.+pub(crate) type Held<P> = Option<Input<P>>;++/// The two halves of a codec's API, as [`next_frame`] drives them.+pub(crate) trait Codec {+ type Packet;+ type Frame;+ /// Hands the codec a packet, or tells it the stream is over.+ fn send(&mut self, input: &Input<Self::Packet>) -> Result<(), ffmpeg::Error>;+ /// Takes a decoded frame out of the codec.+ fn receive(&mut self) -> Result<Self::Frame, ffmpeg::Error>;+}++/// Why [`next_frame`] stopped without a frame or the end.+#[derive(Debug)]+pub(crate) enum Stall<E> {+ /// The codec refused to hand back a frame.+ Receive(ffmpeg::Error),+ /// The codec refused a packet, for a reason other than being full.+ Send(ffmpeg::Error),+ /// Reading the next packet failed.+ Read(E),+ /// The codec refused every packet and gave nothing back, round after+ /// round. libavcodec promises that never happens; a broken platform+ /// decoder is the one way it could, and a stuck reader is worse than an+ /// error the caller can fall back from.+ Stuck,+}++/// Rounds in a row with the codec neither taking a packet nor giving a+/// frame before it counts as stuck. MediaCodec's receive waits for its+/// output on each round, so this is seconds, not a busy spin.+const STUCK_AFTER: u32 = 1000;++/// The next frame out of `codec`, or `None` once it has given its last.+///+/// Feeds it from `held` first, then from `read`, and when the codec says+/// `EAGAIN` to a packet, keeps that packet in `held` and takes frames out+/// before trying it again. `read` gives [`Input::End`] at the end of the+/// file, as often as it is asked.+pub(crate) fn next_frame<C: Codec, E>(+ codec: &mut C,+ held: &mut Held<C::Packet>,+ mut read: impl FnMut() -> Result<Input<C::Packet>, E>,+) -> Result<Option<C::Frame>, Stall<E>> {+ let mut refused = 0;+ loop {+ match codec.receive() {+ Ok(frame) => return Ok(Some(frame)),+ Err(ffmpeg::Error::Eof) => return Ok(None),+ Err(error) if ffi::is_again(&error) => {}+ Err(error) => return Err(Stall::Receive(error)),+ }+ let input = match held.take() {+ Some(input) => input,+ None => read().map_err(Stall::Read)?,+ };+ match codec.send(&input) {+ Ok(()) => refused = 0,+ // Told twice the stream is over: it heard the first time.+ Err(ffmpeg::Error::Eof) if matches!(input, Input::End) => {}+ Err(error) if ffi::is_again(&error) => {+ *held = Some(input);+ refused += 1;+ if refused >= STUCK_AFTER {+ return Err(Stall::Stuck);+ }+ }+ Err(error) => return Err(Stall::Send(error)),+ }+ }+}++/// The error a stall is for a reader of `path`, named the way the loops+/// this replaced named theirs.+pub(crate) fn fail(stall: Stall<ffmpeg::Error>, path: &Path) -> Error {+ match stall {+ Stall::Receive(error) => ffi::fail("decode", path, error),+ Stall::Send(error) => ffi::fail("send packet", path, error),+ Stall::Read(error) => ffi::fail("read", path, error),+ Stall::Stuck => Error::Ffi {+ operation: "decode",+ path: path.to_path_buf(),+ detail: "the codec stopped taking packets and giving frames".to_owned(),+ },+ }+}++/// The next packet of `stream` in `input`, or [`Input::End`] at the end of+/// the file. Packets of every other stream are skipped.+pub(crate) fn read_packet(+ input: &mut ffmpeg::format::context::Input,+ stream: usize,+) -> Result<Input<ffmpeg::Packet>, ffmpeg::Error> {+ loop {+ let mut packet = ffmpeg::Packet::empty();+ match packet.read(input) {+ Ok(()) if packet.stream() == stream => return Ok(Input::Packet(packet)),+ Ok(()) => {}+ Err(ffmpeg::Error::Eof) => return Ok(Input::End),+ Err(error) => return Err(error),+ }+ }+}++fn send(codec: &mut decoder::Opened, input: &Input<ffmpeg::Packet>) -> Result<(), ffmpeg::Error> {+ match input {+ Input::Packet(packet) => codec.send_packet(packet),+ Input::End => codec.send_eof(),+ }+}++impl Codec for decoder::Video {+ type Packet = ffmpeg::Packet;+ type Frame = Video;++ fn send(&mut self, input: &Input<ffmpeg::Packet>) -> Result<(), ffmpeg::Error> {+ send(self, input)+ }++ fn receive(&mut self) -> Result<Video, ffmpeg::Error> {+ let mut frame = Video::empty();+ self.receive_frame(&mut frame).map(|()| frame)+ }+}++impl Codec for decoder::Audio {+ type Packet = ffmpeg::Packet;+ type Frame = Audio;++ fn send(&mut self, input: &Input<ffmpeg::Packet>) -> Result<(), ffmpeg::Error> {+ send(self, input)+ }++ fn receive(&mut self) -> Result<Audio, ffmpeg::Error> {+ let mut frame = Audio::empty();+ self.receive_frame(&mut frame).map(|()| frame)+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use std::collections::VecDeque;++ fn again() -> ffmpeg::Error {+ ffmpeg::Error::Other {+ errno: libc::EAGAIN,+ }+ }++ /// A codec shaped like MediaCodec: an input queue of `room` packets,+ /// one frame out per packet in, refusing packets while the queue is+ /// full, and giving a frame back only once it has refused one (or the+ /// stream is over) - the worst case for a loop that drops refusals.+ struct Queue {+ room: usize,+ queue: VecDeque<u32>,+ blocked: bool,+ ended: bool,+ refusals: usize,+ }++ impl Queue {+ fn new(room: usize) -> Self {+ Self {+ room,+ queue: VecDeque::new(),+ blocked: false,+ ended: false,+ refusals: 0,+ }+ }+ }++ impl Codec for Queue {+ type Packet = u32;+ type Frame = u32;++ fn send(&mut self, input: &Input<u32>) -> Result<(), ffmpeg::Error> {+ match input {+ _ if self.queue.len() >= self.room => {+ self.blocked = true;+ self.refusals += 1;+ Err(again())+ }+ Input::Packet(_) if self.ended => Err(ffmpeg::Error::Eof),+ Input::Packet(packet) => {+ self.queue.push_back(*packet);+ Ok(())+ }+ Input::End if self.ended => Err(ffmpeg::Error::Eof),+ Input::End => {+ self.ended = true;+ Ok(())+ }+ }+ }++ fn receive(&mut self) -> Result<u32, ffmpeg::Error> {+ if !self.blocked && !self.ended {+ return Err(again());+ }+ match self.queue.pop_front() {+ Some(frame) => {+ self.blocked = false;+ Ok(frame)+ }+ None if self.ended => Err(ffmpeg::Error::Eof),+ None => Err(again()),+ }+ }+ }++ /// A codec that, like a full MediaCodec queue, refuses a packet once+ /// and hands back a frame first. Every packet must still come out.+ struct Refuses {+ sent: Vec<u32>,+ ready: VecDeque<u32>,+ refuse_next: bool,+ ended: bool,+ }++ impl Codec for Refuses {+ type Packet = u32;+ type Frame = u32;++ fn send(&mut self, input: &Input<u32>) -> Result<(), ffmpeg::Error> {+ if self.refuse_next {+ self.refuse_next = false;+ return Err(again());+ }+ match input {+ Input::Packet(packet) => {+ self.sent.push(*packet);+ self.ready.push_back(*packet);+ // Every other packet is refused the next time round.+ self.refuse_next = packet % 2 == 0;+ }+ Input::End => self.ended = true,+ }+ Ok(())+ }++ fn receive(&mut self) -> Result<u32, ffmpeg::Error> {+ match self.ready.pop_front() {+ Some(frame) => Ok(frame),+ None if self.ended => Err(ffmpeg::Error::Eof),+ None => Err(again()),+ }+ }+ }++ fn drain<C: Codec<Packet = u32, Frame = u32>>(codec: &mut C, packets: u32) -> Vec<u32> {+ let mut source = 0..packets;+ let mut held = None;+ let mut out = Vec::new();+ loop {+ let read = || -> Result<Input<u32>, ()> {+ Ok(source.next().map_or(Input::End, Input::Packet))+ };+ match next_frame(codec, &mut held, read) {+ Ok(Some(frame)) => out.push(frame),+ Ok(None) => return out,+ Err(stall) => panic!("stalled: {stall:?}"),+ }+ }+ }++ #[test]+ fn a_refused_packet_is_sent_again_not_dropped() {+ let mut codec = Refuses {+ sent: Vec::new(),+ ready: VecDeque::new(),+ refuse_next: false,+ ended: false,+ };+ let out = drain(&mut codec, 10);+ assert_eq!(out, (0..10).collect::<Vec<_>>());+ assert_eq!(codec.sent, (0..10).collect::<Vec<_>>());+ }++ #[test]+ fn a_full_input_queue_loses_nothing_and_drains_at_the_end() {+ for room in [1, 2, 4, 8] {+ let mut codec = Queue::new(room);+ let out = drain(&mut codec, 30);+ assert_eq!(out, (0..30).collect::<Vec<_>>(), "room {room}");+ assert!(codec.refusals > 0, "room {room} was never full");+ }+ }++ #[test]+ fn a_codec_that_takes_nothing_and_gives_nothing_is_stuck_not_spinning() {+ struct Wedged;+ impl Codec for Wedged {+ type Packet = u32;+ type Frame = u32;+ fn send(&mut self, _: &Input<u32>) -> Result<(), ffmpeg::Error> {+ Err(again())+ }+ fn receive(&mut self) -> Result<u32, ffmpeg::Error> {+ Err(again())+ }+ }+ let mut held = None;+ let mut reads = 0;+ let result = next_frame(&mut Wedged, &mut held, || -> Result<Input<u32>, ()> {+ reads += 1;+ Ok(Input::Packet(7))+ });+ assert!(matches!(result, Err(Stall::Stuck)));+ // The one packet it kept refusing was read once and is still held.+ assert_eq!(reads, 1);+ assert!(matches!(held, Some(Input::Packet(7))));+ }++ #[test]+ fn errors_say_which_side_failed() {+ struct Broken;+ impl Codec for Broken {+ type Packet = u32;+ type Frame = u32;+ fn send(&mut self, _: &Input<u32>) -> Result<(), ffmpeg::Error> {+ Err(ffmpeg::Error::InvalidData)+ }+ fn receive(&mut self) -> Result<u32, ffmpeg::Error> {+ Err(again())+ }+ }+ let mut held = None;+ let sent = next_frame(&mut Broken, &mut held, || -> Result<Input<u32>, ()> {+ Ok(Input::Packet(1))+ });+ assert!(matches!(sent, Err(Stall::Send(ffmpeg::Error::InvalidData))));+ let read = next_frame(&mut Broken, &mut held, || -> Result<Input<u32>, &str> {+ Err("disk")+ });+ assert!(matches!(read, Err(Stall::Read("disk"))));+ }+}diff --git a/src/crates/concat-media/src/lib.rs b/src/crates/concat-media/src/lib.rsindex 74dc488d..e2a93287 100644--- a/src/crates/concat-media/src/lib.rs+++ b/src/crates/concat-media/src/lib.rs@@ -17,6 +17,7 @@ pub mod audio;pub mod decode;pub mod encode;pub mod error;+mod feed;mod ffi;pub mod hardware;pub mod peaks;diff --git a/src/crates/concat-media/src/samples.rs b/src/crates/concat-media/src/samples.rsindex 44af16ba..c0996885 100644--- a/src/crates/concat-media/src/samples.rs+++ b/src/crates/concat-media/src/samples.rs@@ -134,6 +134,8 @@ pub struct AudioDecoder {/// is, the way the picture and the export mix measure it.origin: f64,decoder: decoder::Audio,+ /// A packet the codec refused while full; see [`crate::feed`].+ held: crate::feed::Held<ffmpeg::Packet>,options: AudioOptions,graph: Option<filter::Graph>,/// The codec has been drained and the graph flushed; only what the sink@@ -200,6 +202,7 @@ impl AudioDecoder {time_base,origin,decoder,+ held: None,options: options.clone(),graph: None,flushed: false,@@ -258,35 +261,9 @@ impl AudioDecoder {/// One decoded frame out of the codec, or `None` when the file is over.fn next_source(&mut self) -> Result<Option<Audio>> {- loop {- let mut frame = Audio::empty();- match self.decoder.receive_frame(&mut frame) {- Ok(()) => return Ok(Some(frame)),- Err(ffmpeg::Error::Eof) => return Ok(None),- Err(error) if ffi::is_again(&error) => {}- Err(error) => return Err(ffi::fail("decode", &self.path, error)),- }- loop {- let mut packet = ffmpeg::Packet::empty();- match packet.read(&mut self.input) {- Ok(()) => {- if packet.stream() != self.stream {- continue;- }- match self.decoder.send_packet(&packet) {- Ok(()) => break,- Err(error) if ffi::is_again(&error) => break,- Err(error) => return Err(ffi::fail("send packet", &self.path, error)),- }- }- Err(ffmpeg::Error::Eof) => {- let _ = self.decoder.send_eof();- break;- }- Err(error) => return Err(ffi::fail("read", &self.path, error)),- }- }- }+ let read = || crate::feed::read_packet(&mut self.input, self.stream);+ crate::feed::next_frame(&mut self.decoder, &mut self.held, read)+ .map_err(|stall| crate::feed::fail(stall, &self.path))}/// The next frame out of the graph: interleaved samples at the requested