Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

13.1 KB
Raw Blame
use super::*;

use cap_camera_avfoundation::*;
use cidre::*;
use objc2_av_foundation::*;

pub(super) fn list_cameras_impl() -> impl Iterator<Item = CameraInfo> {
    // ar_pool: called from pool-less tokio threads on a polling cadence; the
    // unique_id/localized_name accessors autorelease NSStrings that would
    // otherwise accumulate for the process lifetime.
    objc::ar_pool(|| {
        let devices = cap_camera_avfoundation::list_video_devices();
        devices
            .iter()
            .map(|d| CameraInfo {
                device_id: d.unique_id().to_string(),
                model_id: ModelID::from_avfoundation(d),
                display_name: d.localized_name().to_string(),
            })
            .collect::<Vec<_>>()
    })
    .into_iter()
}

impl CameraInfo {
    pub(super) fn formats_impl(&self) -> Option<Vec<Format>> {
        let device = find_device(self)?;

        let mut ret = vec![];

        for format in device.formats().iter() {
            let desc = format.format_desc();
            let width = desc.dims().width as u32;
            let height = desc.dims().height as u32;
            let pixel_format = fourcc_display(desc.media_sub_type());

            for fr_range in format.video_supported_frame_rate_ranges().iter() {
                // SAFETY: trust me bro it crashes on intel mac otherwise
                let fr_range = unsafe {
                    &*(fr_range as *const av::capture::device::FrameRateRange)
                        .cast::<AVFrameRateRange>()
                };

                let min = unsafe { fr_range.minFrameDuration() };

                ret.push(Format {
                    native: format.retained(),
                    info: FormatInfo {
                        width,
                        height,
                        frame_rate: min.timescale as f32 / min.value as f32,
                    },
                    pixel_format: Some(pixel_format.clone()),
                })
            }
        }

        Some(ret)
    }
}

impl ModelID {
    fn from_avfoundation(device: &cidre::av::capture::Device) -> Option<Self> {
        let unique_id = device.unique_id().to_string();
        Self::from_avfoundation_unique_id(&unique_id)
    }

    fn from_avfoundation_unique_id(unique_id: &str) -> Option<Self> {
        let suffix = unique_id.get(unique_id.len().checked_sub(8)?..)?;
        let vid = suffix.get(..4)?;
        let pid = suffix.get(4..)?;

        if vid == "0000" && pid == "0001" {
            return None;
        }

        Some(Self {
            vid: vid.to_string(),
            pid: pid.to_string(),
        })
    }
}

pub type NativeFormat = arc::R<av::capture::device::Format>;

pub type NativeCaptureHandle = AVFoundationRecordingHandle;

static AVFOUNDATION_SESSION_LIFECYCLE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());

fn avfoundation_session_lifecycle_guard() -> std::sync::MutexGuard<'static, ()> {
    AVFOUNDATION_SESSION_LIFECYCLE_LOCK
        .lock()
        .unwrap_or_else(|err| err.into_inner())
}

fn find_device(info: &CameraInfo) -> Option<arc::R<av::CaptureDevice>> {
    let devices = list_video_devices();
    devices
        .iter()
        .find(
            |d| match (ModelID::from_avfoundation(d).as_ref(), info.model_id()) {
                (Some(a), Some(b)) => a == b,
                (None, None) => d.unique_id().to_string() == info.device_id(),
                _ => false,
            },
        )
        .map(|v| v.retained())
}

fn fourcc_display(fourcc: u32) -> String {
    let mut bytes = fourcc.to_be_bytes();
    cidre::four_cc_to_str(&mut bytes).to_string()
}

/// The min frame duration of the format's frame-rate range matching the
/// selected rate. Formats are enumerated per frame-rate range, so without
/// this the device free-runs at the format's default rate (often the
/// fastest range, e.g. 60fps) while the pipeline assumes the selected one.
fn min_frame_duration_for_rate(
    format: &av::capture::device::Format,
    frame_rate: f32,
) -> Option<cm::Time> {
    let mut best: Option<(f32, cm::Time)> = None;

    for fr_range in format.video_supported_frame_rate_ranges().iter() {
        // SAFETY: trust me bro it crashes on intel mac otherwise
        let fr_range = unsafe {
            &*(fr_range as *const av::capture::device::FrameRateRange).cast::<AVFrameRateRange>()
        };

        let min = unsafe { fr_range.minFrameDuration() };
        if min.value <= 0 || min.timescale <= 0 {
            continue;
        }

        let range_rate = min.timescale as f32 / min.value as f32;
        let rate_diff = (range_rate - frame_rate).abs();
        if best.is_none_or(|(best_diff, _)| rate_diff < best_diff) {
            best = Some((rate_diff, cm::Time::new(min.value, min.timescale)));
        }
    }

    best.and_then(|(rate_diff, duration)| (rate_diff < 1.0).then_some(duration))
}

pub(super) fn start_capturing_impl(
    camera: &CameraInfo,
    format: Format,
    mode: CaptureMode,
    mut callback: impl FnMut(CapturedFrame) + 'static,
) -> Result<AVFoundationRecordingHandle, StartCapturingError> {
    let mut device = find_device(camera)
        .ok_or(StartCapturingError::DeviceNotFound)?
        .retained();

    let input =
        av::capture::DeviceInput::with_device(&device).map_err(AVFoundationError::Static)?;

    let queue = dispatch::Queue::new();
    let mut missing_image_bufs: u64 = 0;
    let delegate =
        CallbackOutputDelegate::with(CallbackOutputDelegateInner::new(Box::new(move |data| {
            if data.sample_buf.image_buf().is_none() {
                missing_image_bufs += 1;
                if missing_image_bufs == 1 || missing_image_bufs.is_multiple_of(100) {
                    tracing::warn!(
                        count = missing_image_bufs,
                        "Camera output delivered sample buffer(s) without an image buffer"
                    );
                }
                return;
            };

            callback(CapturedFrame {
                native: NativeCapturedFrame(data.sample_buf.retained()),
                timestamp: data.timestamp,
            });
        })));

    let mut output = av::capture::VideoDataOutput::new();
    let mut session = av::capture::Session::new();
    let mut added_input = false;

    session.configure(|s| {
        if s.can_add_input(&input) {
            s.add_input(&input);
            added_input = true;
        } else {
            return;
        }

        s.add_output(&output);
    });

    if !added_input {
        return Err(AVFoundationError::Message(
            "Failed to add camera input to AVFoundation session".to_string(),
        )
        .into());
    }

    if mode == CaptureMode::Native {
        let pixel_format = format.native().format_desc().media_sub_type();
        let is_available = output
            .available_video_cv_pixel_formats()
            .iter()
            .any(|n| n.as_u32() == pixel_format);

        if is_available {
            let video_settings = ns::Dictionary::with_keys_values(
                &[cv::pixel_buffer_keys::pixel_format().as_ns()],
                &[ns::Number::with_u32(pixel_format).as_id_ref()],
            );
            if let Err(err) = output.set_video_settings(Some(video_settings.as_ref())) {
                tracing::warn!(
                    pixel_format = %fourcc_display(pixel_format),
                    "Failed to request native camera pixel format, using default conversion: {err}"
                );
            }
        } else {
            tracing::warn!(
                pixel_format = %fourcc_display(pixel_format),
                "Native camera pixel format not offered by video output, using default conversion"
            );
        }
    }

    output.set_sample_buf_delegate(Some(delegate.as_ref()), Some(&queue));

    {
        let _session_lifecycle_guard = avfoundation_session_lifecycle_guard();

        match mode {
            CaptureMode::Native => {
                // The device config must stay locked while running starts,
                // otherwise start_running can overwrite the active format on macOS
                // https://stackoverflow.com/questions/36689578/avfoundation-capturing-video-with-custom-resolution
                let mut _lock = device.config_lock().map_err(AVFoundationError::Retained)?;

                _lock.set_active_format(format.native());

                // Setting the active format resets the frame durations to the
                // format's defaults, which can be a faster rate than the
                // selected one (the device would then deliver e.g. 60fps
                // while the pipeline records it as 30fps, stretching the
                // recording). Cap delivery at the selected rate; leave the
                // max duration alone so low-light rate reduction still works.
                if let Some(duration) =
                    min_frame_duration_for_rate(format.native(), format.frame_rate())
                {
                    if let Err(err) = _lock.set_active_video_min_frame_duration(duration) {
                        tracing::warn!(
                            frame_rate = format.frame_rate(),
                            "Failed to set camera min frame duration: {err}"
                        );
                    }
                } else {
                    tracing::warn!(
                        frame_rate = format.frame_rate(),
                        "No frame-rate range matches the selected camera rate, using device default"
                    );
                }

                session.start_running();
            }
            CaptureMode::Compatibility => {
                session.start_running();
            }
        }
    }

    Ok(AVFoundationRecordingHandle {
        _delegate: delegate,
        session,
        output,
        input,
        _device: device,
        is_torn_down: false,
    })
}

pub struct AVFoundationRecordingHandle {
    _delegate: arc::R<cap_camera_avfoundation::CallbackOutputDelegate>,
    session: arc::R<cidre::av::capture::Session>,
    output: arc::R<av::CaptureVideoDataOutput>,
    input: arc::R<av::CaptureDeviceInput>,
    _device: arc::R<av::CaptureDevice>,
    is_torn_down: bool,
}

impl AVFoundationRecordingHandle {
    pub fn stop_capturing(mut self) -> Result<(), String> {
        self.teardown();
        Ok(())
    }

    fn teardown(&mut self) {
        if self.is_torn_down {
            return;
        }

        self.is_torn_down = true;
        let _session_lifecycle_guard = avfoundation_session_lifecycle_guard();

        self.session.stop_running();
        self.output
            .set_sample_buf_delegate::<CallbackOutputDelegate>(None, None);
        let output = self.output.clone();
        let input = self.input.clone();
        self.session.configure(move |s| {
            s.remove_output(&output);
            s.remove_input(&input);
        });
    }
}

impl Drop for AVFoundationRecordingHandle {
    fn drop(&mut self) {
        self.teardown();
    }
}

#[derive(thiserror::Error)]
pub enum AVFoundationError {
    #[error("{0}")]
    Static(&'static cidre::ns::Error),
    #[error("{0}")]
    Retained(cidre::arc::R<cidre::ns::Error>),
    #[error("{0}")]
    Message(String),
}

impl From<&'static cidre::ns::Error> for AVFoundationError {
    fn from(err: &'static cidre::ns::Error) -> Self {
        AVFoundationError::Static(err)
    }
}

impl From<cidre::arc::R<cidre::ns::Error>> for AVFoundationError {
    fn from(err: cidre::arc::R<cidre::ns::Error>) -> Self {
        AVFoundationError::Retained(err)
    }
}

impl Deref for AVFoundationError {
    type Target = cidre::ns::Error;

    fn deref(&self) -> &Self::Target {
        match self {
            AVFoundationError::Static(err) => err,
            AVFoundationError::Retained(err) => err,
            AVFoundationError::Message(_) => unreachable!(),
        }
    }
}

impl Debug for AVFoundationError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            AVFoundationError::Static(err) => write!(f, "{err}"),
            AVFoundationError::Retained(err) => write!(f, "{err}"),
            AVFoundationError::Message(err) => write!(f, "{err}"),
        }
    }
}

#[derive(Debug, Clone)]
pub struct NativeCapturedFrame(arc::R<cm::SampleBuf>);

impl NativeCapturedFrame {
    pub fn image_buf(&self) -> Option<arc::R<cv::ImageBuf>> {
        self.0.image_buf().map(|b| b.retained())
    }

    pub fn sample_buf(&self) -> &arc::R<cm::SampleBuf> {
        &self.0
    }
}

#[cfg(test)]
mod model_id_tests {
    use super::ModelID;

    #[test]
    fn usb_camera_ids_preserve_the_vendor_and_product_suffix() {
        for unique_id in ["0x12340000046d082d", "046d082d", "カメラ046d082d"] {
            let model = ModelID::from_avfoundation_unique_id(unique_id).unwrap();
            assert_eq!(model.vid, "046d");
            assert_eq!(model.pid, "082d");
        }
    }

    #[test]
    fn malformed_camera_ids_fall_back_to_the_device_id() {
        for unique_id in ["", "short", "cameraé123", "é1234567", "123é456"] {
            assert!(ModelID::from_avfoundation_unique_id(unique_id).is_none());
        }
    }

    #[test]
    fn builtin_camera_ids_still_fall_back_to_the_device_id() {
        assert!(ModelID::from_avfoundation_unique_id("0x1234000000000001").is_none());
    }
}