Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

18.5 KB
Raw Blame
use std::{cell::RefCell, collections::VecDeque};

use cap_camera::CapturedFrame;
use cap_camera_windows::PixelFormat;
use ffmpeg::{Packet, format::Pixel, frame::Video as FFVideo};

use crate::CapturedFrameExt;

#[derive(thiserror::Error, Debug)]
pub enum AsFFmpegError {
    #[error("FailedToGetBytes: {0}")]
    FailedToGetBytes(windows_core::Error),
    #[error("Empty")]
    Empty,
    #[error("MJPEG decode error: {0}")]
    MjpegDecodeError(String),
    #[error("H264 decode error: {0}")]
    H264DecodeError(String),
    #[error("H264 decoder needs more data (non-fatal)")]
    H264NeedMoreData,
}

struct MjpegDecoder {
    decoder: ffmpeg::codec::decoder::Video,
}

impl MjpegDecoder {
    fn new() -> Result<Self, AsFFmpegError> {
        let codec = ffmpeg::codec::decoder::find(ffmpeg::codec::Id::MJPEG)
            .ok_or_else(|| AsFFmpegError::MjpegDecodeError("MJPEG codec not found".to_string()))?;

        let decoder_context = ffmpeg::codec::context::Context::new_with_codec(codec);

        let decoder = decoder_context.decoder().video().map_err(|e| {
            AsFFmpegError::MjpegDecodeError(format!("Failed to create decoder: {e}"))
        })?;

        Ok(Self { decoder })
    }

    fn decode(&mut self, bytes: &[u8]) -> Result<FFVideo, AsFFmpegError> {
        let packet = Packet::copy(bytes);
        self.decoder
            .send_packet(&packet)
            .map_err(|e| AsFFmpegError::MjpegDecodeError(format!("Failed to send packet: {e}")))?;

        let mut decoded_frame = FFVideo::empty();
        self.decoder
            .receive_frame(&mut decoded_frame)
            .map_err(|e| {
                AsFFmpegError::MjpegDecodeError(format!("Failed to receive frame: {e}"))
            })?;

        Ok(decoded_frame)
    }
}

thread_local! {
    static MJPEG_DECODER: RefCell<Option<MjpegDecoder>> = const { RefCell::new(None) };
}

fn decode_mjpeg(bytes: &[u8]) -> Result<FFVideo, AsFFmpegError> {
    MJPEG_DECODER.with(|decoder_cell| {
        let mut decoder_opt = decoder_cell.borrow_mut();

        if decoder_opt.is_none() {
            *decoder_opt = Some(MjpegDecoder::new()?);
        }

        decoder_opt.as_mut().unwrap().decode(bytes)
    })
}

pub struct H264Decoder {
    decoder: ffmpeg::codec::decoder::Video,
    received_keyframe: bool,
    frame_buffer: VecDeque<FFVideo>,
}

impl H264Decoder {
    pub fn new() -> Result<Self, AsFFmpegError> {
        let codec = ffmpeg::codec::decoder::find(ffmpeg::codec::Id::H264)
            .ok_or_else(|| AsFFmpegError::H264DecodeError("H264 codec not found".to_string()))?;

        let decoder_context = ffmpeg::codec::context::Context::new_with_codec(codec);

        let decoder = decoder_context.decoder().video().map_err(|e| {
            AsFFmpegError::H264DecodeError(format!("Failed to create decoder: {e}"))
        })?;

        Ok(Self {
            decoder,
            received_keyframe: false,
            frame_buffer: VecDeque::new(),
        })
    }

    pub fn decode(&mut self, bytes: &[u8]) -> Result<Option<FFVideo>, AsFFmpegError> {
        if let Some(frame) = self.frame_buffer.pop_front() {
            return Ok(Some(frame));
        }

        if !self.received_keyframe && !Self::contains_keyframe(bytes) {
            return Ok(None);
        }

        if Self::contains_keyframe(bytes) {
            self.received_keyframe = true;
        }

        let packet = Packet::copy(bytes);

        loop {
            match self.decoder.send_packet(&packet) {
                Ok(()) => break,
                Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::error::EAGAIN => {
                    self.drain_frames()?;
                }
                Err(e) => {
                    return Err(AsFFmpegError::H264DecodeError(format!(
                        "Failed to send packet: {e}"
                    )));
                }
            }
        }

        self.drain_frames()?;

        Ok(self.frame_buffer.pop_front())
    }

    pub fn flush(&mut self) -> Result<Vec<FFVideo>, AsFFmpegError> {
        if let Err(e) = self.decoder.send_eof()
            && !matches!(e, ffmpeg::Error::Eof)
        {
            return Err(AsFFmpegError::H264DecodeError(format!(
                "Failed to send EOF: {e}"
            )));
        }

        self.drain_frames()?;

        Ok(self.frame_buffer.drain(..).collect())
    }

    pub fn reset(&mut self) -> Result<(), AsFFmpegError> {
        *self = Self::new()?;
        Ok(())
    }

    fn drain_frames(&mut self) -> Result<(), AsFFmpegError> {
        loop {
            let mut decoded_frame = FFVideo::empty();
            match self.decoder.receive_frame(&mut decoded_frame) {
                Ok(()) => {
                    self.frame_buffer.push_back(decoded_frame);
                }
                Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::error::EAGAIN => {
                    return Ok(());
                }
                Err(ffmpeg::Error::Eof) => {
                    return Ok(());
                }
                Err(e) => {
                    return Err(AsFFmpegError::H264DecodeError(format!(
                        "Failed to receive frame: {e}"
                    )));
                }
            }
        }
    }

    fn contains_keyframe(bytes: &[u8]) -> bool {
        let mut i = 0;
        while i + 4 < bytes.len() {
            if bytes[i] == 0 && bytes[i + 1] == 0 {
                let (start_code_len, nal_start) = if bytes[i + 2] == 1 {
                    (3, i + 3)
                } else if bytes[i + 2] == 0 && i + 3 < bytes.len() && bytes[i + 3] == 1 {
                    (4, i + 4)
                } else {
                    i += 1;
                    continue;
                };

                if nal_start < bytes.len() {
                    let nal_unit_type = bytes[nal_start] & 0x1F;
                    match nal_unit_type {
                        5 | 7 | 8 => return true,
                        _ => {}
                    }
                }

                i += start_code_len;
            } else {
                i += 1;
            }
        }
        false
    }
}

thread_local! {
    static H264_DECODER: RefCell<Option<H264Decoder>> = const { RefCell::new(None) };
}

fn decode_h264(bytes: &[u8]) -> Result<FFVideo, AsFFmpegError> {
    H264_DECODER.with(|decoder_cell| {
        let mut decoder_opt = decoder_cell.borrow_mut();

        if decoder_opt.is_none() {
            *decoder_opt = Some(H264Decoder::new()?);
        }

        let decoder = decoder_opt.as_mut().unwrap();
        decoder
            .decode(bytes)?
            .ok_or(AsFFmpegError::H264NeedMoreData)
    })
}

pub fn reset_h264_decoder() {
    H264_DECODER.with(|decoder_cell| {
        *decoder_cell.borrow_mut() = None;
    });
}

fn src_y(y: usize, height: usize, is_bottom_up: bool) -> usize {
    if is_bottom_up { height - y - 1 } else { y }
}

impl CapturedFrameExt for CapturedFrame {
    fn as_ffmpeg(&self) -> Result<ffmpeg::frame::Video, AsFFmpegError> {
        let native = self.native();
        let width = native.width;
        let height = native.height;
        let is_bottom_up = native.is_bottom_up;

        if width == 0 || height == 0 {
            return Err(AsFFmpegError::Empty);
        }

        let bytes = native.bytes().map_err(AsFFmpegError::FailedToGetBytes)?;

        if bytes.is_empty() {
            return Err(AsFFmpegError::Empty);
        }

        Ok(match native.pixel_format {
            PixelFormat::YUV420P => {
                let mut ff_frame = FFVideo::new(Pixel::YUV420P, width as u32, height as u32);

                let stride = ff_frame.stride(0);
                for y in 0..height {
                    let row_width = width;
                    let src_row = &bytes[src_y(y, height, is_bottom_up) * row_width..];
                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
                    dest_row[0..row_width].copy_from_slice(&src_row[0..row_width]);
                }

                let stride = ff_frame.stride(1);
                let half_height = height / 2;
                for y in 0..half_height {
                    let row_width = width / 2;
                    let src_row =
                        &bytes[width * height + src_y(y, half_height, is_bottom_up) * row_width..];
                    let dest_row = &mut ff_frame.data_mut(1)[y * stride..];
                    dest_row[0..row_width].copy_from_slice(&src_row[0..row_width]);
                }

                let stride = ff_frame.stride(2);
                for y in 0..half_height {
                    let row_width = width / 2;
                    let src_row = &bytes[width * height
                        + width * height / 4
                        + src_y(y, half_height, is_bottom_up) * row_width..];
                    let dest_row = &mut ff_frame.data_mut(2)[y * stride..];
                    dest_row[0..row_width].copy_from_slice(&src_row[0..row_width]);
                }

                ff_frame
            }
            PixelFormat::NV12 => {
                let mut ff_frame = FFVideo::new(Pixel::NV12, width as u32, height as u32);

                let stride = ff_frame.stride(0);
                for y in 0..height {
                    let src_row = &bytes[src_y(y, height, is_bottom_up) * width..];
                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
                    dest_row[0..width].copy_from_slice(&src_row[0..width]);
                }

                let stride = ff_frame.stride(1);
                let uv_base = &bytes[width * height..];
                let half_height = height / 2;
                for y in 0..half_height {
                    let row_width = width;
                    let src_row = &uv_base[src_y(y, half_height, is_bottom_up) * row_width..];
                    let dest_row = &mut ff_frame.data_mut(1)[y * stride..];
                    dest_row[0..row_width].copy_from_slice(&src_row[0..row_width]);
                }

                ff_frame
            }
            PixelFormat::ARGB => {
                let mut ff_frame = FFVideo::new(Pixel::BGRA, width as u32, height as u32);

                let stride = ff_frame.stride(0);
                for y in 0..height {
                    let row_width = width * 4;
                    let src_row = &bytes[src_y(y, height, is_bottom_up) * row_width..];
                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
                    dest_row[0..row_width].copy_from_slice(&src_row[0..row_width]);
                }

                ff_frame
            }
            PixelFormat::RGB24 => {
                let mut ff_frame = FFVideo::new(Pixel::BGR24, width as u32, height as u32);

                let stride = ff_frame.stride(0);
                let src_stride = width * 3;
                for y in 0..height {
                    let src_row = &bytes[src_y(y, height, is_bottom_up) * src_stride..];
                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
                    dest_row[0..src_stride].copy_from_slice(&src_row[0..src_stride]);
                }

                ff_frame
            }
            PixelFormat::RGB32 => {
                let mut ff_frame = FFVideo::new(Pixel::BGRA, width as u32, height as u32);

                let stride = ff_frame.stride(0);
                for y in 0..height {
                    let row_width = width * 4;
                    let src_row = &bytes[src_y(y, height, is_bottom_up) * row_width..];
                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
                    dest_row[0..row_width].copy_from_slice(&src_row[0..row_width]);
                }

                ff_frame
            }
            PixelFormat::YUYV422 => {
                let mut ff_frame = FFVideo::new(Pixel::YUYV422, width as u32, height as u32);

                let stride = ff_frame.stride(0);
                for y in 0..height {
                    let row_width = width * 2;
                    let src_row = &bytes[src_y(y, height, is_bottom_up) * row_width..];
                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
                    dest_row[0..row_width].copy_from_slice(&src_row[0..row_width]);
                }

                ff_frame
            }
            PixelFormat::UYVY422 => {
                let mut ff_frame = FFVideo::new(Pixel::UYVY422, width as u32, height as u32);

                let stride = ff_frame.stride(0);
                for y in 0..height {
                    let row_width = width * 2;
                    let src_row = &bytes[src_y(y, height, is_bottom_up) * row_width..];
                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
                    dest_row[0..row_width].copy_from_slice(&src_row[0..row_width]);
                }

                ff_frame
            }
            PixelFormat::MJPEG => decode_mjpeg(&bytes)?,
            PixelFormat::YV12 => {
                let mut ff_frame = FFVideo::new(Pixel::YUV420P, width as u32, height as u32);

                let stride = ff_frame.stride(0);
                for y in 0..height {
                    let row_width = width;
                    let src_row = &bytes[src_y(y, height, is_bottom_up) * row_width..];
                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
                    dest_row[0..row_width].copy_from_slice(&src_row[0..row_width]);
                }

                let v_offset = width * height;
                let u_offset = v_offset + (width / 2) * (height / 2);
                let stride_u = ff_frame.stride(1);
                let stride_v = ff_frame.stride(2);
                let half_height = height / 2;

                for y in 0..half_height {
                    let row_width = width / 2;
                    let sy = src_y(y, half_height, is_bottom_up);
                    let src_v = &bytes[v_offset + sy * row_width..];
                    let src_u = &bytes[u_offset + sy * row_width..];
                    ff_frame.data_mut(1)[y * stride_u..][0..row_width]
                        .copy_from_slice(&src_u[0..row_width]);
                    ff_frame.data_mut(2)[y * stride_v..][0..row_width]
                        .copy_from_slice(&src_v[0..row_width]);
                }

                ff_frame
            }
            PixelFormat::BGR24 => {
                let mut ff_frame = FFVideo::new(Pixel::BGR24, width as u32, height as u32);

                let stride = ff_frame.stride(0);
                let src_stride = width * 3;
                for y in 0..height {
                    let src_row = &bytes[src_y(y, height, is_bottom_up) * src_stride..];
                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
                    dest_row[0..src_stride].copy_from_slice(&src_row[0..src_stride]);
                }

                ff_frame
            }
            PixelFormat::GRAY8 => {
                let mut ff_frame = FFVideo::new(Pixel::GRAY8, width as u32, height as u32);

                let stride = ff_frame.stride(0);
                for y in 0..height {
                    let row_width = width;
                    let src_row = &bytes[src_y(y, height, is_bottom_up) * row_width..];
                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
                    dest_row[0..row_width].copy_from_slice(&src_row[0..row_width]);
                }

                ff_frame
            }
            PixelFormat::GRAY16 => {
                let mut ff_frame = FFVideo::new(Pixel::GRAY16LE, width as u32, height as u32);

                let stride = ff_frame.stride(0);
                let src_stride = width * 2;
                for y in 0..height {
                    let src_row = &bytes[src_y(y, height, is_bottom_up) * src_stride..];
                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
                    dest_row[0..src_stride].copy_from_slice(&src_row[0..src_stride]);
                }

                ff_frame
            }
            PixelFormat::NV21 => {
                let mut ff_frame = FFVideo::new(Pixel::NV12, width as u32, height as u32);

                let stride = ff_frame.stride(0);
                for y in 0..height {
                    let src_row = &bytes[src_y(y, height, is_bottom_up) * width..];
                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
                    dest_row[0..width].copy_from_slice(&src_row[0..width]);
                }

                let stride = ff_frame.stride(1);
                let src_uv = &bytes[width * height..];
                let half_height = height / 2;

                for y in 0..half_height {
                    let row_width = width;
                    let src_row = &src_uv[src_y(y, half_height, is_bottom_up) * row_width..];
                    let dest_row = &mut ff_frame.data_mut(1)[y * stride..];
                    for x in 0..width / 2 {
                        dest_row[x * 2] = src_row[x * 2 + 1];
                        dest_row[x * 2 + 1] = src_row[x * 2];
                    }
                }

                ff_frame
            }
            PixelFormat::RGB565 => {
                let mut ff_frame = FFVideo::new(Pixel::RGB565LE, width as u32, height as u32);

                let stride = ff_frame.stride(0);
                let src_stride = width * 2;
                for y in 0..height {
                    let src_row = &bytes[src_y(y, height, is_bottom_up) * src_stride..];
                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
                    dest_row[0..src_stride].copy_from_slice(&src_row[0..src_stride]);
                }

                ff_frame
            }
            PixelFormat::P010 => {
                let mut ff_frame = FFVideo::new(Pixel::P010LE, width as u32, height as u32);

                let stride = ff_frame.stride(0);
                let src_stride = width * 2;
                for y in 0..height {
                    let src_row = &bytes[src_y(y, height, is_bottom_up) * src_stride..];
                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
                    dest_row[0..src_stride].copy_from_slice(&src_row[0..src_stride]);
                }

                let stride = ff_frame.stride(1);
                let uv_offset = width * height * 2;
                let src_stride = width * 2;
                let half_height = height / 2;

                for y in 0..half_height {
                    let src_row =
                        &bytes[uv_offset + src_y(y, half_height, is_bottom_up) * src_stride..];
                    let dest_row = &mut ff_frame.data_mut(1)[y * stride..];
                    dest_row[0..src_stride].copy_from_slice(&src_row[0..src_stride]);
                }

                ff_frame
            }
            PixelFormat::H264 => decode_h264(&bytes)?,
        })
    }
}