Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

16.2 KB
Raw Blame
use cap_camera::CapturedFrame;
use cap_camera_avfoundation::ImageBufExt;
use cidre::*;
use ffmpeg::{format::Pixel, software::scaling};
use std::{
    cell::RefCell,
    sync::atomic::{AtomicBool, Ordering},
};

use crate::CapturedFrameExt;

type CachedScaler = (Pixel, u32, u32, scaling::Context, ffmpeg::frame::Video);

#[derive(thiserror::Error, Debug)]
pub enum AsFFmpegError {
    #[error("Unsupported media subtype '{0}'")]
    UnsupportedSubType(String),
    #[error("Insufficient plane count for format '{format}': expected {expected}, found {found}")]
    InsufficientPlaneCount {
        format: String,
        expected: usize,
        found: usize,
    },
    #[error("Swscale fallback failed for format '{format}': {reason}")]
    SwscaleFallbackFailed { format: String, reason: String },
    #[error("{0}")]
    Native(#[from] cidre::os::Error),
    #[error("No image buffer available")]
    NoImageBuffer,
}

struct FourccInfo {
    pixel: Pixel,
    bytes_per_pixel: usize,
}

fn fourcc_to_pixel_format(fourcc: &str) -> Option<FourccInfo> {
    match fourcc {
        "ABGR" => Some(FourccInfo {
            pixel: Pixel::ABGR,
            bytes_per_pixel: 4,
        }),
        "b64a" => Some(FourccInfo {
            pixel: Pixel::RGBA64BE,
            bytes_per_pixel: 8,
        }),
        "b48r" => Some(FourccInfo {
            pixel: Pixel::RGB48BE,
            bytes_per_pixel: 6,
        }),
        "L016" => Some(FourccInfo {
            pixel: Pixel::GRAY16LE,
            bytes_per_pixel: 2,
        }),
        _ => None,
    }
}

static FALLBACK_WARNING_LOGGED: AtomicBool = AtomicBool::new(false);

impl CapturedFrameExt for CapturedFrame {
    fn as_ffmpeg(&self) -> Result<ffmpeg::frame::Video, AsFFmpegError> {
        sample_buf_as_ffmpeg(self.native().sample_buf())
    }
}

// Standalone so consumers holding a bare retained CMSampleBuffer (e.g. the
// native camera preview's CPU fallback) can reuse the same conversion.
pub fn sample_buf_as_ffmpeg(
    sample_buf: &cm::SampleBuf,
) -> Result<ffmpeg::frame::Video, AsFFmpegError> {
    {
        let mut image_buf = sample_buf
            .image_buf()
            .map(|b| b.retained())
            .ok_or(AsFFmpegError::NoImageBuffer)?;

        let width = image_buf.width();
        let height = image_buf.height();
        let plane0_stride = image_buf.plane_bytes_per_row(0);
        let plane1_stride = image_buf.plane_bytes_per_row(1);
        let plane_count = image_buf.plane_count();
        let plane_info: [(usize, usize, usize); 3] = [
            (
                image_buf.plane_bytes_per_row(0),
                image_buf.plane_height(0),
                image_buf.plane_width(0),
            ),
            (
                image_buf.plane_bytes_per_row(1),
                image_buf.plane_height(1),
                image_buf.plane_width(1),
            ),
            (
                image_buf.plane_bytes_per_row(2),
                image_buf.plane_height(2),
                image_buf.plane_width(2),
            ),
        ];

        let format_desc = sample_buf
            .format_desc()
            .ok_or(AsFFmpegError::NoImageBuffer)?;

        let bytes_lock = ImageBufExt::base_addr_lock(
            image_buf.as_mut(),
            cv::pixel_buffer::LockFlags::READ_ONLY,
        )?;

        let res = match cidre::four_cc_to_str(&mut format_desc.media_sub_type().to_be_bytes()) {
            "2vuy" => {
                let mut ff_frame = ffmpeg::frame::Video::new(
                    ffmpeg::format::Pixel::UYVY422,
                    width as u32,
                    height as u32,
                );

                let src_stride = plane0_stride;
                let dest_stride = ff_frame.stride(0);

                let src_bytes = bytes_lock.plane_data(0);
                let dest_bytes = &mut ff_frame.data_mut(0);

                for y in 0..height {
                    let row_width = width * 2;
                    let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
                    let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];

                    dest_row.copy_from_slice(src_row);
                }

                ff_frame
            }
            format @ ("420v" | "420f") => {
                let mut ff_frame = ffmpeg::frame::Video::new(
                    ffmpeg::format::Pixel::NV12,
                    width as u32,
                    height as u32,
                );
                ff_frame.set_color_range(if format == "420f" {
                    ffmpeg::color::Range::JPEG
                } else {
                    ffmpeg::color::Range::MPEG
                });

                let src_stride = plane0_stride;
                let dest_stride = ff_frame.stride(0);

                let src_bytes = bytes_lock.plane_data(0);
                let dest_bytes = &mut ff_frame.data_mut(0);

                for y in 0..height {
                    let row_width = width;
                    let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
                    let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];

                    dest_row.copy_from_slice(src_row);
                }

                let src_stride = plane1_stride;
                let dest_stride = ff_frame.stride(1);

                let src_bytes = bytes_lock.plane_data(1);
                let dest_bytes = &mut ff_frame.data_mut(1);

                for y in 0..height / 2 {
                    let row_width = width;
                    let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
                    let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];

                    dest_row.copy_from_slice(src_row);
                }

                ff_frame
            }
            "yuvs" => {
                let mut ff_frame = ffmpeg::frame::Video::new(
                    ffmpeg::format::Pixel::YUYV422,
                    width as u32,
                    height as u32,
                );

                let src_stride = plane0_stride;
                let dest_stride = ff_frame.stride(0);

                let src_bytes = bytes_lock.plane_data(0);
                let dest_bytes = &mut ff_frame.data_mut(0);

                for y in 0..height {
                    let row_width = width * 2;
                    let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
                    let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];

                    dest_row.copy_from_slice(src_row);
                }

                ff_frame
            }
            "BGRA" => {
                let mut ff_frame = ffmpeg::frame::Video::new(
                    ffmpeg::format::Pixel::BGRA,
                    width as u32,
                    height as u32,
                );

                let src_stride = plane0_stride;
                let dest_stride = ff_frame.stride(0);

                let src_bytes = bytes_lock.plane_data(0);
                let dest_bytes = &mut ff_frame.data_mut(0);

                for y in 0..height {
                    let row_width = width * 4;
                    let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
                    let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];

                    dest_row.copy_from_slice(src_row);
                }

                ff_frame
            }
            "ARGB" => {
                let mut ff_frame = ffmpeg::frame::Video::new(
                    ffmpeg::format::Pixel::ARGB,
                    width as u32,
                    height as u32,
                );

                let src_stride = plane0_stride;
                let dest_stride = ff_frame.stride(0);

                let src_bytes = bytes_lock.plane_data(0);
                let dest_bytes = &mut ff_frame.data_mut(0);

                for y in 0..height {
                    let row_width = width * 4;
                    let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
                    let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];

                    dest_row.copy_from_slice(src_row);
                }

                ff_frame
            }
            "RGBA" => {
                let mut ff_frame = ffmpeg::frame::Video::new(
                    ffmpeg::format::Pixel::RGBA,
                    width as u32,
                    height as u32,
                );

                let src_stride = plane0_stride;
                let dest_stride = ff_frame.stride(0);

                let src_bytes = bytes_lock.plane_data(0);
                let dest_bytes = &mut ff_frame.data_mut(0);

                for y in 0..height {
                    let row_width = width * 4;
                    let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
                    let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];

                    dest_row.copy_from_slice(src_row);
                }

                ff_frame
            }
            "24BG" => {
                let mut ff_frame = ffmpeg::frame::Video::new(
                    ffmpeg::format::Pixel::BGR24,
                    width as u32,
                    height as u32,
                );

                let src_stride = plane0_stride;
                let dest_stride = ff_frame.stride(0);

                let src_bytes = bytes_lock.plane_data(0);
                let dest_bytes = &mut ff_frame.data_mut(0);

                for y in 0..height {
                    let row_width = width * 3;
                    let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
                    let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];

                    dest_row.copy_from_slice(src_row);
                }

                ff_frame
            }
            "24RG" => {
                let mut ff_frame = ffmpeg::frame::Video::new(
                    ffmpeg::format::Pixel::RGB24,
                    width as u32,
                    height as u32,
                );

                let src_stride = plane0_stride;
                let dest_stride = ff_frame.stride(0);

                let src_bytes = bytes_lock.plane_data(0);
                let dest_bytes = &mut ff_frame.data_mut(0);

                for y in 0..height {
                    let row_width = width * 3;
                    let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
                    let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];

                    dest_row.copy_from_slice(src_row);
                }

                ff_frame
            }
            "y420" => {
                if plane_count < 3 {
                    return Err(AsFFmpegError::InsufficientPlaneCount {
                        format: "y420".to_string(),
                        expected: 3,
                        found: plane_count,
                    });
                }

                let mut ff_frame = ffmpeg::frame::Video::new(
                    ffmpeg::format::Pixel::YUV420P,
                    width as u32,
                    height as u32,
                );

                for (plane, &(src_stride, plane_height, row_width)) in plane_info.iter().enumerate()
                {
                    let dest_stride = ff_frame.stride(plane);

                    let src_bytes = bytes_lock.plane_data(plane);
                    let dest_bytes = &mut ff_frame.data_mut(plane);

                    for y in 0..plane_height {
                        let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
                        let dest_row =
                            &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];
                        dest_row.copy_from_slice(src_row);
                    }
                }

                ff_frame
            }
            "L008" | "GRAY" => {
                let mut ff_frame = ffmpeg::frame::Video::new(
                    ffmpeg::format::Pixel::GRAY8,
                    width as u32,
                    height as u32,
                );

                let src_stride = plane0_stride;
                let dest_stride = ff_frame.stride(0);

                let src_bytes = bytes_lock.plane_data(0);
                let dest_bytes = &mut ff_frame.data_mut(0);

                for y in 0..height {
                    let row_width = width;
                    let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
                    let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];

                    dest_row.copy_from_slice(src_row);
                }

                ff_frame
            }
            format => {
                if let Some(info) = fourcc_to_pixel_format(format) {
                    if !FALLBACK_WARNING_LOGGED.swap(true, Ordering::Relaxed) {
                        tracing::warn!(
                            "Using swscale fallback for camera format '{}' - this may impact performance",
                            format
                        );
                    }

                    let mut src_frame =
                        ffmpeg::frame::Video::new(info.pixel, width as u32, height as u32);

                    let src_stride = plane0_stride;
                    let dest_stride = src_frame.stride(0);
                    let src_bytes = bytes_lock.plane_data(0);
                    let dest_bytes = &mut src_frame.data_mut(0);

                    let row_width = width * info.bytes_per_pixel;
                    for y in 0..height {
                        let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
                        let dest_row =
                            &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];
                        dest_row.copy_from_slice(src_row);
                    }

                    thread_local! {
                        static FALLBACK_SCALER: RefCell<Option<CachedScaler>> = const { RefCell::new(None) };
                    }

                    FALLBACK_SCALER.with(|cell| {
                        let mut cached = cell.borrow_mut();

                        let w = width as u32;
                        let h = height as u32;

                        let needs_reinit = cached.as_ref().is_none_or(|(p, cw, ch, _, _)| {
                            *p != info.pixel || *cw != w || *ch != h
                        });

                        if needs_reinit {
                            let scaler = scaling::Context::get(
                                info.pixel,
                                w,
                                h,
                                Pixel::RGBA,
                                w,
                                h,
                                scaling::flag::Flags::FAST_BILINEAR,
                            )
                            .map_err(|e| {
                                AsFFmpegError::SwscaleFallbackFailed {
                                    format: format.to_string(),
                                    reason: format!("Failed to create scaler: {e}"),
                                }
                            })?;

                            let out = ffmpeg::frame::Video::new(Pixel::RGBA, w, h);
                            *cached = Some((info.pixel, w, h, scaler, out));
                        }

                        let (_, _, _, scaler, output) = cached.as_mut().unwrap();

                        scaler.run(&src_frame, output).map_err(|e| {
                            AsFFmpegError::SwscaleFallbackFailed {
                                format: format.to_string(),
                                reason: format!("Conversion failed: {e}"),
                            }
                        })?;

                        let mut cloned = ffmpeg::frame::Video::new(Pixel::RGBA, w, h);
                        let src_data = output.data(0);
                        let dest_data = cloned.data_mut(0);
                        dest_data[..src_data.len()].copy_from_slice(src_data);
                        Ok::<_, AsFFmpegError>(cloned)
                    })?
                } else {
                    return Err(AsFFmpegError::UnsupportedSubType(format.to_string()));
                }
            }
        };

        Ok(res)
    }
}