Skip to content

Your Android phone and your Windows PC, working as one. Local-first, end-to-end encrypted, no account.

19.7 KB
Raw Blame
// SPDX-License-Identifier: GPL-3.0-or-later
//! MP4 recording for mirrored phone screens and app windows using Windows
//! Media Foundation's `IMFSinkWriter`.
//!
//! Video is muxed as H.264 passthrough (`MFVideoFormat_H264` on both input and
//! output streams of the sink writer) so zero video re-encoding occurs on the
//! PC. When phone audio (16-bit PCM) is present, it is encoded to AAC-LC
//! (`MFAudioFormat_AAC`) into the same `.mp4` container.

use crate::i18n::tr;
use std::{
    path::{Path, PathBuf},
    time::{Instant, SystemTime, UNIX_EPOCH},
};

use windows::{
    Win32::{
        Media::MediaFoundation::{
            IMFMediaBuffer, IMFMediaType, IMFSample, IMFSinkWriter, MF_MT_ALL_SAMPLES_INDEPENDENT,
            MF_MT_AUDIO_AVG_BYTES_PER_SECOND, MF_MT_AUDIO_BITS_PER_SAMPLE, MF_MT_AUDIO_BLOCK_ALIGNMENT,
            MF_MT_AUDIO_NUM_CHANNELS, MF_MT_AUDIO_SAMPLES_PER_SECOND, MF_MT_AVG_BITRATE, MF_MT_FRAME_RATE,
            MF_MT_FRAME_SIZE, MF_MT_INTERLACE_MODE, MF_MT_MAJOR_TYPE, MF_MT_MPEG_SEQUENCE_HEADER,
            MF_MT_PIXEL_ASPECT_RATIO, MF_MT_SUBTYPE, MFAudioFormat_AAC, MFAudioFormat_PCM, MFCreateMediaType,
            MFCreateMemoryBuffer, MFCreateSample, MFCreateSinkWriterFromURL, MFMediaType_Audio,
            MFMediaType_Video, MFSampleExtension_CleanPoint, MFVideoFormat_H264,
            MFVideoInterlace_Progressive,
        },
        System::SystemInformation::GetLocalTime,
    },
    core::HSTRING,
};

use super::h264::ensure_mf_started;

/// Formats `<prefix> YYYY-MM-DD HH.MM.SS.<ext>` using the PC's local clock.
pub fn local_timestamp_filename(prefix: &str, ext: &str) -> String {
    // SAFETY: GetLocalTime always succeeds and writes a SYSTEMTIME struct.
    let st = unsafe { GetLocalTime() };
    format!(
        "{prefix} {:04}-{:02}-{:02} {:02}.{:02}.{:02}.{ext}",
        st.wYear, st.wMonth, st.wDay, st.wHour, st.wMinute, st.wSecond
    )
}

struct ActiveWriter {
    writer: IMFSinkWriter,
    video_stream: u32,
    audio_stream: Option<(u32, u32, u8)>,
    first_video_us: Option<u64>,
    last_video_hns: i64,
    video_frames_written: u64,
    first_audio_us: Option<u64>,
    last_audio_hns: i64,
    audio_samples_written: u64,
}

// SAFETY: Media Foundation runs in the multithreaded apartment (`COINIT_MULTITHREADED`
// via `ensure_mf_started`), and all access to `ActiveWriter` is serialized through `&mut self`.
unsafe impl Send for ActiveWriter {}

/// Records a mirrored H.264 stream (and optional PCM audio stream) into an `.mp4` file.
pub struct MirrorRecorder {
    path: PathBuf,
    width: u32,
    height: u32,
    fps: u32,
    audio_cfg: Option<(u32, u8)>,
    sps_pps: Vec<u8>,
    writer: Option<ActiveWriter>,
    started_at: Instant,
    started_epoch_ms: i64,
}

impl std::fmt::Debug for MirrorRecorder {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("MirrorRecorder")
            .field("path", &self.path)
            .field("width", &self.width)
            .field("height", &self.height)
            .field("active", &self.writer.is_some())
            .finish_non_exhaustive()
    }
}

impl MirrorRecorder {
    pub fn new(
        path: PathBuf,
        width: u32,
        height: u32,
        fps: u32,
        audio_cfg: Option<(u32, u8)>,
    ) -> MirrorRecorder {
        let started_epoch_ms =
            SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |d| d.as_millis() as i64);
        MirrorRecorder {
            path,
            width,
            height,
            fps: fps.max(1),
            audio_cfg,
            sps_pps: Vec::new(),
            writer: None,
            started_at: Instant::now(),
            started_epoch_ms,
        }
    }

    pub fn started_at(&self) -> Instant {
        self.started_at
    }

    pub fn started_epoch_ms(&self) -> i64 {
        self.started_epoch_ms
    }

    /// Seeds cached SPS/PPS NAL units collected before recording started.
    pub fn set_sps_pps(&mut self, sps_pps: &[u8]) {
        if !sps_pps.is_empty() {
            merge_sps_pps(&mut self.sps_pps, sps_pps);
        }
    }

    /// Updates stream dimensions before the first keyframe initializes the writer.
    pub fn on_config(&mut self, width: u32, height: u32) {
        if self.writer.is_none() && width > 0 && height > 0 {
            self.width = width;
            self.height = height;
        }
    }

    /// Updates audio configuration before the first keyframe initializes the writer.
    pub fn on_audio_config(&mut self, rate: u32, channels: u8) {
        if self.writer.is_none() && rate > 0 && channels > 0 {
            self.audio_cfg = Some((rate, channels));
        }
    }

    /// Processes an incoming Annex-B H.264 access unit.
    pub fn on_packet(&mut self, keyframe: bool, time_us: u64, data: &[u8]) -> Result<(), String> {
        if data.is_empty() {
            return Ok(());
        }
        merge_sps_pps(&mut self.sps_pps, data);

        // If this packet only carries parameter sets (SPS/PPS) without a coded frame,
        // keep them for the upcoming keyframe without writing an empty video sample.
        if is_parameter_only(data) {
            return Ok(());
        }

        let is_keyframe = keyframe || has_idr_slice(data);

        if self.writer.is_none() {
            if !is_keyframe || self.width == 0 || self.height == 0 {
                return Ok(());
            }
            let active = ActiveWriter::open(
                &self.path,
                self.width,
                self.height,
                self.fps,
                &self.sps_pps,
                self.audio_cfg,
            )
            .map_err(|e| e.to_string())?;
            self.writer = Some(active);
        }

        let Some(w) = self.writer.as_mut() else {
            return Ok(());
        };

        ensure_mf_started().map_err(|e| e.to_string())?;
        let first_us = *w.first_video_us.get_or_insert(time_us);
        let mut hns = (time_us.saturating_sub(first_us) * 10) as i64;
        if w.video_frames_written > 0 && hns <= w.last_video_hns {
            hns = w.last_video_hns + 10_000;
        }
        w.last_video_hns = hns;
        let dur_hns = (10_000_000 / u64::from(self.fps.max(1))) as i64;

        let mut combined = Vec::new();
        let payload: &[u8] = if is_keyframe && !data_has_sps_pps(data) && !self.sps_pps.is_empty() {
            combined.reserve(self.sps_pps.len() + data.len());
            combined.extend_from_slice(&self.sps_pps);
            combined.extend_from_slice(data);
            &combined
        } else {
            data
        };

        write_sample(&w.writer, w.video_stream, payload, hns, dur_hns, is_keyframe)
            .map_err(|e| e.to_string())?;
        w.video_frames_written += 1;
        Ok(())
    }

    /// Processes an incoming 16-bit interleaved PCM audio packet.
    pub fn on_audio(&mut self, time_us: u64, pcm: &[u8]) -> Result<(), String> {
        if pcm.is_empty() {
            return Ok(());
        }
        let Some(w) = self.writer.as_mut() else {
            // Wait until the first video keyframe initializes the writer so audio and video start at t=0.
            return Ok(());
        };
        let Some((stream_idx, rate, channels)) = w.audio_stream else {
            return Ok(());
        };

        ensure_mf_started().map_err(|e| e.to_string())?;
        let first_us = *w.first_audio_us.get_or_insert(time_us);
        let mut hns = (time_us.saturating_sub(first_us) * 10) as i64;
        if w.audio_samples_written > 0 && hns <= w.last_audio_hns {
            hns = w.last_audio_hns + 1;
        }
        let bytes_per_frame = (usize::from(channels.max(1)) * 2).max(2);
        let frames = pcm.len() / bytes_per_frame;
        if frames == 0 {
            return Ok(());
        }
        let dur_hns = ((frames as u64) * 10_000_000 / u64::from(rate.max(1))).max(1) as i64;
        w.last_audio_hns = hns + dur_hns;

        write_sample(&w.writer, stream_idx, pcm, hns, dur_hns, false).map_err(|e| e.to_string())?;
        w.audio_samples_written += 1;
        Ok(())
    }

    /// Finalizes the MP4 container and flushes it to disk.
    pub fn finish(mut self) -> Result<PathBuf, String> {
        if let Some(w) = self.writer.take()
            && w.video_frames_written > 0
        {
            ensure_mf_started().map_err(|e| e.to_string())?;
            if let Some((stream_idx, rate, channels)) = w.audio_stream
                && w.audio_samples_written == 0
            {
                // Media Foundation's MP4 sink rejects Finalize() with MF_E_SINK_NO_SAMPLES_PROCESSED
                // if a declared stream received 0 samples; write a single 20 ms silent frame.
                let silence_len = (rate as usize / 50).max(1) * usize::from(channels.max(1)) * 2;
                let silence = vec![0u8; silence_len];
                let _ = write_sample(&w.writer, stream_idx, &silence, 0, 200_000, false);
            }
            // SAFETY: Finalize flushes and closes the MP4 sink writer.
            unsafe { w.writer.Finalize() }.map_err(|e| e.to_string())?;
            drop(w);
            return Ok(self.path);
        }
        let _ = std::fs::remove_file(&self.path);
        Err(tr!("Mp4", "No video frames were recorded yet."))
    }
}

impl ActiveWriter {
    fn open(
        path: &Path,
        width: u32,
        height: u32,
        fps: u32,
        sps_pps: &[u8],
        audio_cfg: Option<(u32, u8)>,
    ) -> windows::core::Result<ActiveWriter> {
        ensure_mf_started()?;
        if let Some(parent) = path.parent() {
            let _ = std::fs::create_dir_all(parent);
        }
        let _ = std::fs::remove_file(path);

        // SAFETY: Media Foundation COM calls with owned parameters.
        unsafe {
            let writer: IMFSinkWriter =
                MFCreateSinkWriterFromURL(&HSTRING::from(path.as_os_str()), None, None)?;

            let video_out: IMFMediaType = MFCreateMediaType()?;
            video_out.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?;
            video_out.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_H264)?;
            video_out.SetUINT32(&MF_MT_AVG_BITRATE, 8_000_000)?;
            video_out.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)?;
            video_out.SetUINT64(&MF_MT_FRAME_SIZE, (u64::from(width) << 32) | u64::from(height))?;
            video_out.SetUINT64(&MF_MT_FRAME_RATE, (u64::from(fps.max(1)) << 32) | 1u64)?;
            video_out.SetUINT64(&MF_MT_PIXEL_ASPECT_RATIO, (1u64 << 32) | 1u64)?;
            if !sps_pps.is_empty() {
                video_out.SetBlob(&MF_MT_MPEG_SEQUENCE_HEADER, sps_pps)?;
            }

            let video_stream = writer.AddStream(&video_out)?;
            writer.SetInputMediaType(video_stream, &video_out, None)?;

            let mut audio_stream = None;
            if let Some((rate, channels)) = audio_cfg
                && rate > 0
                && channels > 0
            {
                match add_aac_stream(&writer, rate, channels) {
                    Ok(idx) => audio_stream = Some((idx, rate, channels)),
                    Err(e) => {
                        tracing::warn!(error = %e, "couldn't add AAC audio stream to MP4; recording video only");
                    }
                }
            }

            writer.BeginWriting()?;

            Ok(ActiveWriter {
                writer,
                video_stream,
                audio_stream,
                first_video_us: None,
                last_video_hns: 0,
                video_frames_written: 0,
                first_audio_us: None,
                last_audio_hns: 0,
                audio_samples_written: 0,
            })
        }
    }
}

unsafe fn add_aac_stream(writer: &IMFSinkWriter, rate: u32, channels: u8) -> windows::core::Result<u32> {
    let ch = u32::from(channels);
    let block_align = ch * 2;
    // SAFETY: Media Foundation media type creation and stream setup.
    unsafe {
        let audio_out: IMFMediaType = MFCreateMediaType()?;
        audio_out.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Audio)?;
        audio_out.SetGUID(&MF_MT_SUBTYPE, &MFAudioFormat_AAC)?;
        audio_out.SetUINT32(&MF_MT_AUDIO_SAMPLES_PER_SECOND, rate)?;
        audio_out.SetUINT32(&MF_MT_AUDIO_NUM_CHANNELS, ch)?;
        audio_out.SetUINT32(&MF_MT_AUDIO_BITS_PER_SAMPLE, 16)?;
        audio_out.SetUINT32(&MF_MT_AUDIO_AVG_BYTES_PER_SECOND, 20_000)?;

        let stream_idx = writer.AddStream(&audio_out)?;

        let audio_in: IMFMediaType = MFCreateMediaType()?;
        audio_in.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Audio)?;
        audio_in.SetGUID(&MF_MT_SUBTYPE, &MFAudioFormat_PCM)?;
        audio_in.SetUINT32(&MF_MT_AUDIO_SAMPLES_PER_SECOND, rate)?;
        audio_in.SetUINT32(&MF_MT_AUDIO_NUM_CHANNELS, ch)?;
        audio_in.SetUINT32(&MF_MT_AUDIO_BITS_PER_SAMPLE, 16)?;
        audio_in.SetUINT32(&MF_MT_AUDIO_BLOCK_ALIGNMENT, block_align)?;
        audio_in.SetUINT32(&MF_MT_AUDIO_AVG_BYTES_PER_SECOND, rate * block_align)?;
        audio_in.SetUINT32(&MF_MT_ALL_SAMPLES_INDEPENDENT, 1)?;

        writer.SetInputMediaType(stream_idx, &audio_in, None)?;
        Ok(stream_idx)
    }
}

fn write_sample(
    writer: &IMFSinkWriter,
    stream_idx: u32,
    bytes: &[u8],
    hns: i64,
    dur_hns: i64,
    clean_point: bool,
) -> windows::core::Result<()> {
    // SAFETY: buffer is sized to `bytes.len()`, locked while copied, and unlocked before WriteSample.
    unsafe {
        let buf: IMFMediaBuffer = MFCreateMemoryBuffer(bytes.len() as u32)?;
        let mut dst = std::ptr::null_mut();
        buf.Lock(&mut dst, None, None)?;
        std::ptr::copy_nonoverlapping(bytes.as_ptr(), dst, bytes.len());
        buf.Unlock()?;
        buf.SetCurrentLength(bytes.len() as u32)?;

        let sample: IMFSample = MFCreateSample()?;
        sample.AddBuffer(&buf)?;
        sample.SetSampleTime(hns)?;
        sample.SetSampleDuration(dur_hns.max(1))?;
        if clean_point {
            sample.SetUINT32(&MFSampleExtension_CleanPoint, 1)?;
        }
        writer.WriteSample(stream_idx, &sample)
    }
}

/// Splits an Annex-B buffer into `(nal_type, nal_payload_without_start_code)` slices.
fn annex_b_nals(data: &[u8]) -> Vec<(u8, &[u8])> {
    let mut starts = Vec::new();
    let mut i = 0;
    while i + 3 <= data.len() {
        if i + 4 <= data.len() && data[i..i + 4] == [0, 0, 0, 1] {
            starts.push((i, i + 4));
            i += 4;
        } else if data[i..i + 3] == [0, 0, 1] {
            starts.push((i, i + 3));
            i += 3;
        } else {
            i += 1;
        }
    }
    let mut out = Vec::with_capacity(starts.len());
    for (idx, &(_, payload_start)) in starts.iter().enumerate() {
        let payload_end = starts.get(idx + 1).map_or(data.len(), |&(next_code, _)| next_code);
        if payload_start < payload_end {
            let payload = &data[payload_start..payload_end];
            out.push((payload[0] & 0x1F, payload));
        }
    }
    out
}

/// Extracts any SPS (NAL 7) and PPS (NAL 8) units from `data` and merges them
/// into `sps_pps` with 4-byte `00 00 00 01` Annex-B start codes.
pub fn merge_sps_pps(sps_pps: &mut Vec<u8>, data: &[u8]) {
    let nals = annex_b_nals(data);
    if !nals.iter().any(|&(t, _)| t == 7 || t == 8) {
        return;
    }
    let existing = annex_b_nals(sps_pps);
    let mut sps: Option<Vec<u8>> = existing.iter().find(|&&(t, _)| t == 7).map(|&(_, p)| p.to_vec());
    let mut pps: Option<Vec<u8>> = existing.iter().find(|&&(t, _)| t == 8).map(|&(_, p)| p.to_vec());
    for (t, payload) in nals {
        if t == 7 {
            sps = Some(payload.to_vec());
        } else if t == 8 {
            pps = Some(payload.to_vec());
        }
    }
    sps_pps.clear();
    if let Some(s) = sps {
        sps_pps.extend_from_slice(&[0, 0, 0, 1]);
        sps_pps.extend_from_slice(&s);
    }
    if let Some(p) = pps {
        sps_pps.extend_from_slice(&[0, 0, 0, 1]);
        sps_pps.extend_from_slice(&p);
    }
}

fn data_has_sps_pps(data: &[u8]) -> bool {
    let nals = annex_b_nals(data);
    nals.iter().any(|&(t, _)| t == 7) && nals.iter().any(|&(t, _)| t == 8)
}

fn has_idr_slice(data: &[u8]) -> bool {
    annex_b_nals(data).iter().any(|&(t, _)| t == 5)
}

fn is_parameter_only(data: &[u8]) -> bool {
    let nals = annex_b_nals(data);
    !nals.is_empty() && nals.iter().all(|&(t, _)| t == 7 || t == 8)
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Splits an Annex-B stream into access units at its access unit delimiters (NAL type 9).
    fn access_units(stream: &[u8]) -> Vec<&[u8]> {
        let starts: Vec<usize> =
            (0..stream.len()).filter(|&i| stream[i..].starts_with(&[0, 0, 0, 1, 9])).collect();
        starts
            .iter()
            .enumerate()
            .map(|(n, &s)| &stream[s..*starts.get(n + 1).unwrap_or(&stream.len())])
            .collect()
    }

    fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool {
        haystack.windows(needle.len()).any(|w| w == needle)
    }

    #[test]
    fn muxes_h264_passthrough_to_mp4() {
        let dir = tempfile::tempdir().unwrap();
        let mp4_path = dir.path().join("passthrough.mp4");
        let units = access_units(include_bytes!("testdata/testsrc.h264"));
        assert!(!units.is_empty());

        let mut rec = MirrorRecorder::new(mp4_path.clone(), 320, 240, 30, None);
        for (i, unit) in units.iter().enumerate() {
            let keyframe = i % 5 == 0;
            rec.on_packet(keyframe, (i as u64) * 33_333, unit).unwrap();
        }
        let saved = rec.finish().unwrap();
        assert_eq!(saved, mp4_path);
        let bytes = std::fs::read(&saved).unwrap();
        assert!(bytes.len() > 200, "unexpectedly small MP4: {} B", bytes.len());
        assert_eq!(&bytes[4..8], b"ftyp");
        assert!(contains_subslice(&bytes, b"moov"), "missing moov box");
        assert!(contains_subslice(&bytes, b"mdat"), "missing mdat box");
        assert!(contains_subslice(&bytes, b"avc1"), "missing avc1 sample entry");
        let _ = std::fs::remove_file(&saved);
    }

    #[test]
    fn muxes_h264_and_pcm_audio_to_mp4() {
        let dir = tempfile::tempdir().unwrap();
        let units = access_units(include_bytes!("testdata/testsrc.h264"));

        // 1. Video + 48 kHz stereo PCM audio frames.
        let mp4_with_audio = dir.path().join("with_audio.mp4");
        let mut rec = MirrorRecorder::new(mp4_with_audio.clone(), 320, 240, 30, Some((48_000, 2)));
        let pcm_20ms = vec![0u8; (48_000 / 50) * 2 * 2];
        for (i, unit) in units.iter().enumerate() {
            let t_us = (i as u64) * 33_333;
            rec.on_packet(i % 5 == 0, t_us, unit).unwrap();
            rec.on_audio(t_us, &pcm_20ms).unwrap();
        }
        let saved = rec.finish().unwrap();
        let bytes = std::fs::read(&saved).unwrap();
        assert!(bytes.len() > 200);
        assert_eq!(&bytes[4..8], b"ftyp");
        assert!(contains_subslice(&bytes, b"moov"));
        assert!(contains_subslice(&bytes, b"mdat"));
        assert!(contains_subslice(&bytes, b"avc1"));
        assert!(contains_subslice(&bytes, b"mp4a"));
        let _ = std::fs::remove_file(&saved);

        // 2. Audio stream declared (`Some((48_000, 2))`), but 0 audio packets arrive:
        // the silent-frame safeguard must still allow Finalize() to succeed cleanly.
        let mp4_silent_safeguard = dir.path().join("silent_safeguard.mp4");
        let mut rec2 = MirrorRecorder::new(mp4_silent_safeguard.clone(), 320, 240, 30, Some((48_000, 2)));
        for (i, unit) in units.iter().enumerate() {
            rec2.on_packet(i % 5 == 0, (i as u64) * 33_333, unit).unwrap();
        }
        let saved2 = rec2.finish().unwrap();
        let bytes2 = std::fs::read(&saved2).unwrap();
        assert!(bytes2.len() > 200);
        assert_eq!(&bytes2[4..8], b"ftyp");
        assert!(contains_subslice(&bytes2, b"moov"));
        assert!(contains_subslice(&bytes2, b"avc1"));
        assert!(contains_subslice(&bytes2, b"mp4a"));
        let _ = std::fs::remove_file(&saved2);
    }
}