🎥 Make videos programmatically with React
extern crate ffmpeg_next as remotionffmpeg;
use std::sync::atomic::{AtomicUsize, Ordering};
use ffmpeg_next::Rational;
use serde::{Deserialize, Serialize};
use crate::{
errors::ErrorWithBacktrace,
global_printer::{_print_debug, _print_verbose},
max_cache_size,
opened_stream::get_time,
scalable_frame::ScalableFrame,
};
pub fn get_frame_cache_id() -> usize {
static COUNTER: AtomicUsize = AtomicUsize::new(1);
COUNTER.fetch_add(1, Ordering::Relaxed)
}
pub struct FrameCacheItem {
pub resolved_pts: i64,
pub previous_pts: Option<i64>,
pub asked_time: i64,
pub frame: ScalableFrame,
pub id: usize,
pub last_used: u128,
}
pub struct FrameCache {
pub items: Vec<FrameCacheItem>,
pub last_frame: Option<usize>,
}
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct FrameCacheReference {
pub id: usize,
pub last_used: u128,
pub src: String,
pub original_src: String,
pub transparent: bool,
pub tone_mapped: bool,
pub thread_index: usize,
pub size: u64,
}
#[derive(Clone, Copy)]
struct LastFrameFoundSoFar {
pub id: usize,
pub pts: i64,
}
impl FrameCache {
pub fn new() -> Self {
Self {
items: Vec::new(),
last_frame: None,
}
}
pub fn get_references(
&self,
src: String,
original_src: String,
transparent: bool,
tone_mapped: bool,
thread_index: usize,
) -> Result<Vec<FrameCacheReference>, ErrorWithBacktrace> {
let mut references: Vec<FrameCacheReference> = Vec::new();
for item in &self.items {
references.push(FrameCacheReference {
id: item.id,
last_used: item.last_used,
src: src.clone(),
original_src: original_src.clone(),
transparent,
tone_mapped,
thread_index,
size: item.frame.get_size(),
});
}
Ok(references)
}
pub fn add_item(&mut self, item: FrameCacheItem, thread_index: usize) {
max_cache_size::get_instance()
.lock()
.unwrap()
.add_to_current_cache_size(thread_index, item.frame.get_size().into());
self.items.push(item);
}
pub fn set_last_frame(&mut self, id: usize) {
self.last_frame = Some(id);
}
pub fn get_last_frame_in_second(&mut self, time_base: Rational) -> Option<f64> {
match self.last_frame {
Some(last_frame_id) => {
for i in 0..self.items.len() {
if self.items[i].id == last_frame_id {
let pts = self.items[i].resolved_pts as f64;
let in_seconds = pts / (time_base.1 as f64 / time_base.0 as f64);
return Some(in_seconds);
}
}
None
}
None => None,
}
}
pub fn set_biggest_frame_as_last_frame(&mut self) {
let mut last_frame_found: Option<LastFrameFoundSoFar> = None;
for i in 0..self.items.len() {
match last_frame_found {
Some(exists) => {
if self.items[i].resolved_pts > exists.pts {
last_frame_found = Some(LastFrameFoundSoFar {
id: self.items[i].id,
pts: self.items[i].resolved_pts,
});
}
}
None => {
last_frame_found = Some(LastFrameFoundSoFar {
id: self.items[i].id,
pts: self.items[i].resolved_pts,
});
}
}
}
match last_frame_found {
Some(biggest_frame) => {
self.set_last_frame(biggest_frame.id);
}
None => {}
}
}
pub fn get_item_from_id(
&mut self,
id: usize,
thread_index: usize,
) -> Result<Option<Vec<u8>>, ErrorWithBacktrace> {
let mut data: Option<Vec<u8>> = None;
for i in 0..self.items.len() {
if self.items[i].id == id {
self.items[i].frame.ensure_data(thread_index)?;
self.items[i].last_used = get_time();
data = Some(self.items[i].frame.get_data()?);
break;
}
}
Ok(data)
}
pub fn remove_item_by_id(
&mut self,
id: usize,
thread_index: usize,
) -> Result<(), ErrorWithBacktrace> {
for i in 0..self.items.len() {
if self.items[i].id == id {
if self.last_frame.is_some() && id == self.last_frame.expect("last_frame") {
self.last_frame = None;
}
max_cache_size::get_instance()
.lock()
.unwrap()
.remove_from_cache_size(thread_index, self.items[i].frame.get_size().into());
self.items.remove(i);
break;
}
}
Ok(())
}
pub fn is_empty(&self) -> bool {
self.items.is_empty()
}
pub fn get_cache_item_count(&self) -> usize {
self.items.len()
}
pub fn get_local_size_in_bytes(&self) -> u64 {
let mut size: u64 = 0;
for item in &self.items {
size += item.frame.get_size();
}
size
}
pub fn get_item_id(
&mut self,
time: i64,
threshold: i64,
thread_index: usize,
allow_no_pts_after: bool,
) -> Result<Option<usize>, ErrorWithBacktrace> {
let mut best_item: Option<usize> = None;
let mut best_distance = std::i64::MAX;
// Is last frame or beyond
for i in 0..self.items.len() {
match self.last_frame {
Some(last_frame_id) => {
if self.items[i].id == last_frame_id && self.items[i].resolved_pts < time {
self.items[i].frame.ensure_data(thread_index)?;
return Ok(Some(self.items[i].id));
}
}
None => {}
}
}
// Exact same time as requested
for i in 0..self.items.len() {
if self.items[i].resolved_pts == time {
self.items[i].frame.ensure_data(thread_index)?;
return Ok(Some(self.items[i].id));
}
}
// A video frame is displayed from its PTS until the PTS of the next frame.
// Therefore, select the latest frame at or before the requested timestamp,
// instead of the frame with the nearest PTS. If the request is before the
// first frame, fall back to the earliest frame after the timestamp.
for i in 0..self.items.len() {
if self.items[i].resolved_pts > time {
continue;
}
let distance = time - self.items[i].resolved_pts;
// LTE: If multiple items have the same PTS, take the most recently
// decoded one.
if distance <= best_distance {
best_distance = distance;
best_item = Some(i);
}
}
if best_item.is_none() {
for i in 0..self.items.len() {
let distance = self.items[i].resolved_pts - time;
if distance >= 0 && distance <= best_distance {
best_distance = distance;
best_item = Some(i);
}
}
}
let has_pts_after = self.items.iter().any(|item| item.resolved_pts >= time);
// If this happens, then `last_frame` has not worked correctly. Do not match
if !has_pts_after && !allow_no_pts_after {
return Ok(None);
}
if best_distance > threshold {
let has_no_items_before = self.items.iter().all(|item| item.resolved_pts > time);
if has_no_items_before {
// best_item is None when self.items is empty. The downstream
// get_frame() in opened_stream.rs already converts None into a
// clean error pointing at the troubleshooting docs, so return
// Ok(None) here rather than panicking on the unwrap() below.
let Some(best) = best_item else {
return Ok(None);
};
let has_asked_time_before = self
.items
.iter()
.find(|item| item.asked_time <= time)
.is_some();
let asked_time_difference = (self.items[best].asked_time - time).abs();
if has_asked_time_before && self.items[best].asked_time <= time {
_print_debug(&format!(
"Found timestamp with too much threshold, but will let it pass because no closer frames found. difference = {}, threshold = {}",
asked_time_difference, threshold
))?;
// Fall through to success case
} else {
return Ok(None);
}
} else {
let closest_item_after_this = self
.items
.iter()
.filter(|item| item.resolved_pts >= time)
.min_by_key(|item| item.resolved_pts);
let is_inbetween_next_frame = match closest_item_after_this {
Some(closest_item_after_this) => {
// Comparing DTS and PTS - not perfect but that is the assumption for variable videos
closest_item_after_this.resolved_pts > time
&& closest_item_after_this.previous_pts.is_some()
&& closest_item_after_this.previous_pts.unwrap() < time
}
None => false,
};
let is_inbetween_and_after = best_item.is_some()
&& self.items[best_item.unwrap()].previous_pts.is_some()
&& self.items[best_item.unwrap()].previous_pts.unwrap() < time
&& self.items[best_item.unwrap()].resolved_pts > time;
if is_inbetween_next_frame {
_print_verbose(&format!(
"Found timestamp with too much threshold, but will let it pass because it is the best match and there is a frame after the current one that follows one with a lower timestamp. threshold = {}, after this pts = {} after this previous pts = {:?}",
threshold,
closest_item_after_this.unwrap().resolved_pts,
closest_item_after_this.unwrap().previous_pts,
))?;
// Fall through to success
} else if is_inbetween_and_after {
_print_verbose(&format!(
"Found timestamp with too much threshold, but will let it pass because it is inbetween frames. threshold = {}",
threshold
))?;
// Fall through to success
} else {
return Ok(None);
}
}
}
match best_item {
Some(best_item) => {
self.items[best_item].frame.ensure_data(thread_index)?;
Ok(Some(self.items[best_item].id))
}
None => Ok(None),
}
}
}