Skip to content

🎥 Make videos programmatically with React

24.8 KB
Raw Blame
use std::{io::ErrorKind, time::SystemTime};

use ffmpeg_next::{ffi::AVSEEK_FLAG_ANY, Rational};
use remotionffmpeg::{codec::Id, frame::Video, media::Type, Dictionary, StreamMut};
extern crate ffmpeg_next as remotionffmpeg;
use std::time::UNIX_EPOCH;

use crate::{
    errors::ErrorWithBacktrace,
    frame_cache::{get_frame_cache_id, FrameCacheItem},
    frame_cache_manager::FrameCacheManager,
    global_printer::_print_verbose,
    rotation,
    scalable_frame::{NotRgbFrame, Rotate, ScalableFrame},
    tone_map::FilterGraph,
};

pub struct OpenedStream {
    pub stream_index: usize,
    pub original_width: u32,
    pub original_height: u32,
    pub scaled_width: u32,
    pub scaled_height: u32,
    pub video: remotionffmpeg::codec::decoder::Video,
    pub src: String,
    pub original_src: String,
    pub input: remotionffmpeg::format::context::Input,
    pub last_position: Option<i64>,
    pub duration_or_zero: i64,
    pub duration_in_seconds: Option<f64>,
    pub reached_eof: bool,
    pub transparent: bool,
    pub rotation: Rotate,
    pub filter_graph: FilterGraph,
    pub time_base: Rational,
    pub fps: Rational,
}

#[derive(Clone, Copy)]
pub struct LastFrameInfo {
    index: usize,
    pts: i64,
}

pub fn calc_position(time: f64, time_base: Rational) -> i64 {
    // Container timestamps use integer time-base ticks. Use the nearest tick so
    // floating-point imprecision does not put an exact frame time one tick early.
    (time * time_base.1 as f64 / time_base.0 as f64).round() as i64
}

pub fn get_time() -> u128 {
    SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .expect("time went backwards")
        .as_millis()
}

const MAX_DIVERGING_SEEK: u8 = 3;

impl OpenedStream {
    pub fn receive_frame(&mut self) -> Result<Option<Video>, ErrorWithBacktrace> {
        let mut frame = Video::empty();

        let res = self.video.receive_frame(&mut frame);

        match res {
            Err(err) => {
                if err.to_string().contains("Resource temporarily unavailable") {
                    Ok(None)
                } else if err.to_string().contains("End of file") {
                    self.reached_eof = true;
                    Ok(None)
                } else {
                    Err(std::io::Error::new(ErrorKind::Other, err.to_string()))?
                }
            }
            Ok(_) => Ok(Some(frame)),
        }
    }

    pub fn handle_eof(
        &mut self,
        position: i64,
        one_frame_in_time_base: i64,
        previous_pts: Option<i64>,
        tone_mapped: bool,
        frame_cache_manager: &mut FrameCacheManager,
        thread_index: usize,
    ) -> Result<Option<LastFrameInfo>, ErrorWithBacktrace> {
        self.video.send_eof()?;

        let mut latest_frame: Option<LastFrameInfo> = None;
        let mut looped_pts = previous_pts;
        let mut offset = 0;

        loop {
            let result = self.receive_frame();

            match result {
                Ok(Some(video)) => {
                    let frame_cache_id = get_frame_cache_id();

                    let mut size: u64 = 0;

                    let amount_of_planes = video.planes();
                    for i in 0..amount_of_planes {
                        size += video.data(i).len() as u64;
                    }

                    let frame = NotRgbFrame {
                        original_width: self.original_width,
                        original_height: self.original_height,
                        scaled_height: self.scaled_height,
                        scaled_width: self.scaled_width,
                        rotate: self.rotation,
                        original_src: self.original_src.clone(),
                        size,
                        unscaled_frame: video.clone(),
                        tone_mapped,
                        filter_graph: self.filter_graph,
                        colorspace: video.color_space(),
                        src_range: video.color_range(),
                    };

                    offset = offset + one_frame_in_time_base;

                    let item = FrameCacheItem {
                        resolved_pts: video.pts().expect("pts"),
                        frame: ScalableFrame::new(frame, self.transparent),
                        id: frame_cache_id,
                        asked_time: position,
                        last_used: get_time(),
                        previous_pts: looped_pts,
                    };

                    looped_pts = video.pts();
                    frame_cache_manager.add_to_cache(
                        &self.src,
                        &self.original_src,
                        self.transparent,
                        tone_mapped,
                        item,
                        thread_index,
                    );
                    latest_frame = Some(LastFrameInfo {
                        index: frame_cache_id,
                        pts: video.pts().expect("pts"),
                    });
                }
                Ok(None) => {
                    if self.reached_eof {
                        break;
                    }
                }
                Err(err) => {
                    return Err(err);
                }
            }
        }
        Ok(latest_frame)
    }

    pub fn get_frame(
        &mut self,
        stream_index: usize,
        time: f64,
        target_position: i64,
        time_base: Rational,
        one_frame_in_time_base: i64,
        _threshold: i64,
        tone_mapped: bool,
        frame_cache_manager: &mut FrameCacheManager,
        thread_index: usize,
        max_cache_size_in_bytes: u64,
    ) -> Result<usize, ErrorWithBacktrace> {
        let mut freshly_seeked = false;
        let position_to_seek_to = match self.duration_or_zero {
            0 => target_position,
            _ => self.duration_or_zero.min(target_position),
        };

        let should_seek =
        // Beyond the position in the video
         target_position < self.last_position.unwrap_or(0)
            ||
            // Less than 3 seconds before the position we want to be
            self.last_position.unwrap_or(0) < calc_position(time - 1.0, time_base)

            ;

        if should_seek {
            _print_verbose(&format!(
                "Seeking to {} from dts = {:?}, duration = {}, target position = {}",
                position_to_seek_to, self.last_position, self.duration_or_zero, target_position
            ))?;
            self.video.flush();
            match self.input.seek(
                self.stream_index as i32,
                0,
                target_position,
                position_to_seek_to,
                0,
            ) {
                Ok(_) => Ok(()),
                Err(err) => {
                    if err.to_string().contains("Operation not permitted") {
                        _print_verbose(&format!(
                            "Could not perform seek. Got 'Operation not permitted' error."
                        ))?;
                        _print_verbose(&format!(
                            "Attempting to seek to the beginning of the file."
                        ))?;
                        match self.input.seek(
                            self.stream_index as i32,
                            0,
                            target_position,
                            0,
                            AVSEEK_FLAG_ANY,
                        ) {
                            Ok(_) => Ok(()),
                            Err(err) => Err(err),
                        }
                    } else {
                        Err(err)
                    }
                }
            }?;

            freshly_seeked = true;
            self.last_position = None
        }

        let mut last_frame_received: Option<LastFrameInfo> = None;
        let mut stop_after_n_diverging_pts: Option<u8> = None;
        let mut retried_from_start_after_empty_seek = false;

        let mut items_in_loop = 0;

        loop {
            if stop_after_n_diverging_pts.is_some() && stop_after_n_diverging_pts.unwrap() == 0 {
                break;
            }
            match last_frame_received {
                Some(_) => {
                    if stop_after_n_diverging_pts.is_none() {
                        stop_after_n_diverging_pts = Some(MAX_DIVERGING_SEEK);
                    }
                }
                None => {}
            }

            let (stream, packet) = match self.input.get_next_packet() {
                Err(remotionffmpeg::Error::Eof) => {
                    let data = self.handle_eof(
                        target_position,
                        one_frame_in_time_base,
                        match freshly_seeked || self.last_position.is_none() {
                            true => None,
                            false => Some(self.last_position.unwrap_or(0)),
                        },
                        tone_mapped,
                        frame_cache_manager,
                        thread_index,
                    )?;
                    if let Some(last_frame) = data {
                        last_frame_received = Some(last_frame);
                        frame_cache_manager.set_last_frame(
                            &self.src,
                            &self.original_src,
                            self.transparent,
                            tone_mapped,
                            last_frame.index,
                        );
                    } else {
                        frame_cache_manager.set_biggest_frame_as_last_frame(
                            &self.src,
                            &self.original_src,
                            self.transparent,
                            tone_mapped,
                        );
                    }

                    let cache_is_empty = frame_cache_manager.is_empty(
                        &self.src,
                        &self.original_src,
                        self.transparent,
                        tone_mapped,
                    )?;

                    if should_seek
                        && !retried_from_start_after_empty_seek
                        && last_frame_received.is_none()
                        && cache_is_empty
                        && position_to_seek_to > 0
                    {
                        _print_verbose(&format!(
                            "No frame decoded after seeking to {}; retrying from the beginning",
                            target_position
                        ))?;
                        self.video.flush();
                        self.reached_eof = false;
                        self.input
                            .seek(self.stream_index as i32, 0, 0, 0, AVSEEK_FLAG_ANY)?;
                        freshly_seeked = true;
                        self.last_position = None;
                        stop_after_n_diverging_pts = None;
                        retried_from_start_after_empty_seek = true;
                        continue;
                    }

                    break;
                }
                Ok(packet) => packet,
                Err(err) => Err(std::io::Error::new(ErrorKind::Other, err.to_string()))?,
            };

            if stream.parameters().medium() != Type::Video {
                continue;
            }
            if stream.index() != self.stream_index {
                continue;
            }

            _print_verbose(&format!(
                "Thread {}, stream {} - got packet, dts = {:?} pts = {:?} key = {}",
                thread_index,
                stream_index,
                packet.dts(),
                packet.pts(),
                packet.is_key()
            ))?;
            if freshly_seeked {
                if packet.is_key() {
                    freshly_seeked = false
                } else {
                    match packet.pts() {
                        Some(pts) => {
                            let new_position_to_seek_to = pts - 1;
                            stop_after_n_diverging_pts = None;

                            let res = self.input.seek(
                                self.stream_index as i32,
                                0,
                                pts,
                                new_position_to_seek_to,
                                0,
                            );
                            match res {
                                Ok(_) => {}
                                Err(err) => {
                                    if err.to_string().contains("Operation not permitted") {
                                        _print_verbose(&format!(
                                            "Could not perform seek. Got 'Operation not permitted' error."
                                        ))?;
                                        _print_verbose(&format!(
                                            "Attempting to seek to the beginning of the file."
                                        ))?;
                                        self.input.seek(
                                            self.stream_index as i32,
                                            0,
                                            pts,
                                            new_position_to_seek_to,
                                            AVSEEK_FLAG_ANY,
                                        )?;
                                    } else {
                                        panic!("{}", err.to_string());
                                    }
                                }
                            }
                        }
                        None => {}
                    };
                    continue;
                }
            }

            // Don't throw an error here, sometimes it will still work!
            // For example, loop melting-0.5x.webm from <AnimatedEmoji /> component
            // and we will get the error "Invalid data found when processing input"
            // but it'll still work!
            let res = self.video.send_packet(&packet);
            if res.is_err() {
                if res
                    .err()
                    .unwrap()
                    .to_string()
                    .contains("Resource temporarily unavailable")
                {
                    _print_verbose(&format!("Need to send another packet"))?;
                }
                _print_verbose(&format!("Error sending packet: {}", res.err().unwrap()))?;
            }

            loop {
                let result = self.receive_frame();

                match result {
                    Ok(Some(unfiltered)) => {
                        _print_verbose(&format!(
                            "Thread {}, stream {} - received frame, tone_mapped = {}",
                            thread_index, stream_index, tone_mapped
                        ))?;

                        let frame_cache_id = get_frame_cache_id();

                        let mut size: u64 = 0;

                        let amount_of_planes = unfiltered.planes();
                        for i in 0..amount_of_planes {
                            size += unfiltered.data(i).len() as u64;
                        }

                        let frame = NotRgbFrame {
                            original_height: self.original_height,
                            original_width: self.original_width,
                            scaled_height: self.scaled_height,
                            scaled_width: self.scaled_width,
                            rotate: self.rotation,
                            original_src: self.original_src.clone(),
                            size,
                            unscaled_frame: unfiltered.clone(),
                            tone_mapped,
                            filter_graph: self.filter_graph,
                            colorspace: unfiltered.color_space(),
                            src_range: unfiltered.color_range(),
                        };

                        let previous_pts = match freshly_seeked || self.last_position.is_none() {
                            true => None,
                            false => Some(self.last_position.unwrap_or(0)),
                        };
                        let item = FrameCacheItem {
                            resolved_pts: unfiltered.pts().expect("expected pts"),
                            frame: ScalableFrame::new(frame, self.transparent),
                            id: frame_cache_id,
                            asked_time: target_position,
                            last_used: get_time(),
                            previous_pts,
                        };

                        self.last_position = Some(unfiltered.pts().expect("expected pts"));
                        freshly_seeked = false;
                        frame_cache_manager.add_to_cache(
                            &self.src,
                            &self.original_src,
                            self.transparent,
                            tone_mapped,
                            item,
                            thread_index,
                        );

                        items_in_loop += 1;

                        if items_in_loop % 5 == 0 {
                            frame_cache_manager.prune_on_thread(max_cache_size_in_bytes)?;
                        }

                        match stop_after_n_diverging_pts {
                            Some(stop) => match last_frame_received {
                                Some(last_frame) => {
                                    let prev_difference = (last_frame.pts - target_position).abs();
                                    let new_difference =
                                        (unfiltered.pts().expect("pts") - target_position).abs();

                                    if new_difference >= prev_difference {
                                        stop_after_n_diverging_pts = Some(stop.saturating_sub(1));
                                    } else if prev_difference > new_difference {
                                        // Fixing test video crazy1.mp4, frames 240-259
                                        stop_after_n_diverging_pts =
                                            Some((stop + 1).min(MAX_DIVERGING_SEEK));
                                    }
                                }
                                None => {}
                            },
                            None => {}
                        }

                        last_frame_received = Some(LastFrameInfo {
                            index: frame_cache_id,
                            pts: unfiltered.pts().expect("pts"),
                        });
                    }
                    Ok(None) => {
                        break;
                    }
                    Err(err) => {
                        return Err(err);
                    }
                }
            }
        }

        // Allow a lot of threshold here, because otherwise we would crash.
        // Better to have a frame with a lot of deviation than to crash.
        let final_frame = frame_cache_manager.get_cache_item_id(
            &self.src,
            &self.original_src,
            self.transparent,
            tone_mapped,
            target_position,
            1000000000,
            true,
        )?;

        if final_frame.is_none() {
            return Err(std::io::Error::new(
                ErrorKind::Other,
                format!(
                    "No frame found at position {} for source {} (original source = {}). See https://remotion.dev/docs/troubleshooting/no-frame-found-at-position for troubleshooting help.",
                    target_position, self.src, self.original_src
                ),
            ))?;
        }

        return Ok(final_frame.unwrap());
    }
}

pub fn calculate_display_video_size(
    dar_x: i32,
    dar_y: i32,
    sar_x: i32,
    sar_y: i32,
    x: u32,
    y: u32,
) -> (u32, u32) {
    if dar_x == 0 || dar_y == 0 {
        return (x, y);
    }

    let new_width = ((x as f64 * sar_x as f64 / sar_y as f64).round()).max(x as f64);
    let new_height = (new_width / (dar_x as f64 / dar_y as f64)).ceil();

    (new_width as u32, new_height as u32)
}

pub fn get_display_aspect_ratio(mut_stream: &StreamMut) -> Rational {
    unsafe {
        let asp = mut_stream.get_display_aspect_ratio();
        return Rational::new(asp.numerator(), asp.denominator());
    }
}

pub fn open_stream(
    src: &str,
    original_src: &str,
    transparent: bool,
) -> Result<OpenedStream, ErrorWithBacktrace> {
    let mut dictionary = Dictionary::new();
    dictionary.set("fflags", "+genpts");
    let mut input = remotionffmpeg::format::input_with_dictionary(&src, dictionary)?;

    let mut_stream = match input
        .streams_mut()
        .find(|s| s.parameters().medium() == Type::Video)
    {
        Some(stream) => stream,
        None => {
            return Err(ErrorWithBacktrace::from(format!(
                "No video stream found in input file {}. Is this a video file?",
                original_src
            )));
        }
    };

    let stream_index = mut_stream.index();

    let duration_or_zero = mut_stream.duration().max(0);

    let time_base = mut_stream.time_base();
    let duration_in_seconds = match duration_or_zero {
        0 => None,
        _ => Some(duration_or_zero as f64 * time_base.1 as f64 / time_base.0 as f64),
    };
    let parameters = mut_stream.parameters();
    let side_data = mut_stream.side_data();

    let mut rotate = Rotate::Rotate0;

    for data in side_data {
        if data.kind() == remotionffmpeg::codec::packet::side_data::Type::DisplayMatrix {
            let value = data.data();
            let rotate_value = rotation::get_from_side_data(value)?;
            if rotate_value != 0.0 {
                _print_verbose(&format!("Detected rotation in {}: {}", src, rotate_value))?;
                if rotate_value == 90.0 {
                    rotate = Rotate::Rotate90;
                } else if rotate_value == 180.0 {
                    rotate = Rotate::Rotate180;
                } else if rotate_value == 270.0 || rotate_value == -90.0 {
                    rotate = Rotate::Rotate270;
                } else {
                    return Err(ErrorWithBacktrace::from(format!(
                        "Unsupported rotation value {}",
                        rotate_value
                    )));
                }
            }
        }
    }

    let is_vp8_or_vp9 = match transparent {
        true => unsafe {
            let codec_id = (*(*(mut_stream).as_ptr()).codecpar).codec_id;

            if codec_id == remotionffmpeg::codec::id::get_av_codec_id(Id::VP8) {
                Some("vp8")
            } else if codec_id == remotionffmpeg::codec::id::get_av_codec_id(Id::VP9) {
                Some("vp9")
            } else {
                None
            }
        },
        false => None,
    };

    let video = remotionffmpeg::codec::context::Context::from_parameters(parameters.clone())?;

    let decoder = match is_vp8_or_vp9 {
        Some("vp8") => video.decoder().video_with_codec("libvpx")?,
        Some("vp9") => video.decoder().video_with_codec("libvpx-vp9")?,
        Some(_) => unreachable!(),
        None => video.decoder().video()?,
    };

    let original_width = decoder.width();
    let original_height = decoder.height();

    let sar_x;
    let sar_y;
    unsafe {
        sar_x = (*mut_stream.as_ptr()).sample_aspect_ratio.num;
        sar_y = (*mut_stream.as_ptr()).sample_aspect_ratio.den;
    }

    let fps = mut_stream.avg_frame_rate();

    let display_aspect_ratio = get_display_aspect_ratio(&mut_stream);

    _print_verbose(&format!("Display aspect ratio: {:?}", display_aspect_ratio))?;
    _print_verbose(&format!("Sample aspect ratio: {} {}", sar_x, sar_y))?;
    _print_verbose(&format!(
        "Original width: {}, original height: {}",
        original_width, original_height
    ))?;

    let (scaled_width, scaled_height) = calculate_display_video_size(
        display_aspect_ratio.0,
        display_aspect_ratio.1,
        sar_x,
        sar_y,
        original_width,
        original_height,
    );
    _print_verbose(&format!(
        "new width: {}, new height: {}",
        scaled_width, scaled_height
    ))?;

    let filter_graph = FilterGraph {
        original_width,
        original_height,
        format: decoder.format(),
        time_base,
        video_primaries: decoder.color_primaries(),
        transfer_characteristic: decoder.color_transfer_characteristic(),
        color_space: decoder.color_space(),
        color_range: decoder.color_range(),
        aspect_ratio: decoder.aspect_ratio(),
    };

    let opened_stream = OpenedStream {
        stream_index,
        original_height,
        original_width,
        scaled_height,
        scaled_width,
        video: decoder,
        src: src.to_string(),
        input,
        last_position: None,
        duration_or_zero,
        duration_in_seconds,
        reached_eof: false,
        transparent,
        rotation: rotate,
        original_src: original_src.to_string(),
        filter_graph,
        fps,
        time_base,
    };

    Ok(opened_stream)
}