Skip to content

A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.

TypeScript
11.4 KB
Raw Blame
//! The device's shared-mode mix format, as `IAudioClient::GetMixFormat` reports it, and the
//! conversion of its packets to the 16 kHz mono f32 every recognizer consumes. Parsed from the
//! raw `WAVEFORMATEX` bytes so it is tested on every platform, without a device.

use crate::voice::audio::{self, Resampler};
use crate::voice::capture_thread::failed_to_start;
use crate::voice::VoiceError;

// Format tags (mmreg.h). An extensible format names its sample type in `SubFormat`, a GUID whose
// first field is one of the plain tags and the rest is the same base for every one of them.
const WAVE_FORMAT_PCM: u16 = 1;
const WAVE_FORMAT_IEEE_FLOAT: u16 = 3;
const WAVE_FORMAT_EXTENSIBLE: u16 = 0xFFFE;
/// `KSDATAFORMAT_SUBTYPE_*` after its first field (`-0000-0010-8000-00aa00389b71`), in memory order.
const SUBTYPE_BASE: [u8; 12] = [0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71];
/// `sizeof(WAVEFORMATEX)`, and the extensible tail after it: valid bits, channel mask, SubFormat.
pub(crate) const WAVEFORMATEX_BYTES: usize = 18;
const EXTENSIBLE_TAIL: usize = 22;

/// How one sample of one channel is stored.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Sample {
    Float32,
    Float64,
    /// Unsigned, centred on 128.
    Pcm8,
    Pcm16,
    /// Packed in three bytes.
    Pcm24,
    /// Also 24-bit audio in a 32-bit container, which is left-justified, so full scale is the same.
    Pcm32,
}

impl Sample {
    fn bytes(self) -> usize {
        match self {
            Sample::Pcm8 => 1,
            Sample::Pcm16 => 2,
            Sample::Pcm24 => 3,
            Sample::Float32 | Sample::Pcm32 => 4,
            Sample::Float64 => 8,
        }
    }

    /// One little-endian sample, `bytes()` long, scaled to -1..1.
    fn decode(self, b: &[u8]) -> f32 {
        match self {
            Sample::Float32 => f32::from_le_bytes([b[0], b[1], b[2], b[3]]),
            Sample::Float64 => f64::from_le_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]) as f32,
            Sample::Pcm8 => (f32::from(b[0]) - 128.0) / 128.0,
            Sample::Pcm16 => f32::from(i16::from_le_bytes([b[0], b[1]])) / 32_768.0,
            Sample::Pcm24 => (i32::from_le_bytes([0, b[0], b[1], b[2]]) >> 8) as f32 / 8_388_608.0,
            Sample::Pcm32 => i32::from_le_bytes([b[0], b[1], b[2], b[3]]) as f32 / 2_147_483_648.0,
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct MixFormat {
    pub sample: Sample,
    pub channels: usize,
    pub rate: u32,
    /// Bytes per frame: one sample of every channel, interleaved.
    pub block_align: usize,
}

impl MixFormat {
    /// Reads a `WAVEFORMATEX` and, for `WAVE_FORMAT_EXTENSIBLE`, the `WAVEFORMATEXTENSIBLE` tail
    /// after it: `bytes` holds the 18-byte header and then `cbSize` more.
    pub fn parse(bytes: &[u8]) -> Result<Self, VoiceError> {
        let unsupported = |what: String| failed_to_start(format!("the microphone's format is not supported ({what})"));
        let u16_at = |at: usize| u16::from_le_bytes([bytes[at], bytes[at + 1]]);
        if bytes.len() < WAVEFORMATEX_BYTES {
            return Err(unsupported(format!("{} bytes", bytes.len())));
        }
        let (tag, channels, bits, block_align) = (u16_at(0), u16_at(2), u16_at(14), u16_at(12));
        let rate = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]);
        let kind = if tag == WAVE_FORMAT_EXTENSIBLE {
            let tail = &bytes[WAVEFORMATEX_BYTES..];
            if u16_at(16) < EXTENSIBLE_TAIL as u16 || tail.len() < EXTENSIBLE_TAIL {
                return Err(unsupported("a truncated extensible format".into()));
            }
            let valid = u16::from_le_bytes([tail[0], tail[1]]);
            if valid == 0 || valid > bits {
                return Err(unsupported(format!("{valid} valid bits in {bits}")));
            }
            let subformat = &tail[6..22];
            if subformat[4..] != SUBTYPE_BASE || subformat[2..4] != [0, 0] {
                return Err(unsupported(format!("subformat {subformat:02x?}")));
            }
            u16::from_le_bytes([subformat[0], subformat[1]])
        } else {
            tag
        };
        let sample = match (kind, bits) {
            (WAVE_FORMAT_IEEE_FLOAT, 32) => Sample::Float32,
            (WAVE_FORMAT_IEEE_FLOAT, 64) => Sample::Float64,
            (WAVE_FORMAT_PCM, 8) => Sample::Pcm8,
            (WAVE_FORMAT_PCM, 16) => Sample::Pcm16,
            (WAVE_FORMAT_PCM, 24) => Sample::Pcm24,
            (WAVE_FORMAT_PCM, 32) => Sample::Pcm32,
            _ => return Err(unsupported(format!("type {kind}, {bits} bits"))),
        };
        let (channels, block_align) = (usize::from(channels), usize::from(block_align));
        if channels == 0 || rate == 0 || block_align != channels * sample.bytes() {
            return Err(unsupported(format!("{channels} channels at {rate} Hz, {block_align} bytes a frame")));
        }
        Ok(MixFormat { sample, channels, rate, block_align })
    }
}

/// Turns packets in the mix format into `audio::SAMPLE_RATE` mono f32, keeping the resampler's
/// position across packets.
pub(crate) struct Converter {
    format: MixFormat,
    resampler: Resampler,
}

impl Converter {
    pub fn new(format: MixFormat) -> Self {
        Converter { format, resampler: Resampler::new(f64::from(format.rate)) }
    }

    /// One packet of interleaved frames; a trailing partial frame is ignored.
    pub fn packet(&mut self, data: &[u8]) -> Vec<f32> {
        let MixFormat { sample, channels, block_align, .. } = self.format;
        let frames = data.len() / block_align;
        let mut split = vec![Vec::with_capacity(frames); channels];
        for frame in data.chunks_exact(block_align) {
            for (channel, bytes) in split.iter_mut().zip(frame.chunks_exact(sample.bytes())) {
                channel.push(sample.decode(bytes));
            }
        }
        let refs: Vec<&[f32]> = split.iter().map(Vec::as_slice).collect();
        self.resampler.process(&audio::mono(&refs))
    }

    /// A packet WASAPI flagged silent: `frames` frames of nothing, at the device rate.
    pub fn silence(&mut self, frames: usize) -> Vec<f32> {
        self.resampler.process(&vec![0.0; frames])
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// A `WAVEFORMATEX`, with the extensible tail when `subformat` is given.
    fn wave(tag: u16, channels: u16, rate: u32, bits: u16, subformat: Option<u16>) -> Vec<u8> {
        let block_align = channels * bits / 8;
        let mut bytes = Vec::new();
        bytes.extend(tag.to_le_bytes());
        bytes.extend(channels.to_le_bytes());
        bytes.extend(rate.to_le_bytes());
        bytes.extend((rate * u32::from(block_align)).to_le_bytes());
        bytes.extend(block_align.to_le_bytes());
        bytes.extend(bits.to_le_bytes());
        match subformat {
            None => bytes.extend(0u16.to_le_bytes()),
            Some(kind) => {
                bytes.extend(22u16.to_le_bytes());
                bytes.extend(bits.to_le_bytes());
                bytes.extend(0x3u32.to_le_bytes());
                bytes.extend(u32::from(kind).to_le_bytes());
                bytes.extend(SUBTYPE_BASE);
            }
        }
        bytes
    }

    fn extensible(channels: u16, rate: u32, bits: u16, kind: u16) -> Vec<u8> {
        wave(WAVE_FORMAT_EXTENSIBLE, channels, rate, bits, Some(kind))
    }

    #[test]
    fn valid_bits_may_be_fewer_than_the_container_never_more() {
        let with_valid = |valid: u16| {
            let mut bytes = extensible(2, 48_000, 32, 1);
            bytes[18..20].copy_from_slice(&valid.to_le_bytes());
            MixFormat::parse(&bytes).map(|f| f.sample)
        };
        // 24 valid bits in a 32-bit container are left-justified, so they read as 32-bit PCM.
        assert_eq!(with_valid(24), Ok(Sample::Pcm32));
        assert!(with_valid(40).is_err());
        assert!(with_valid(0).is_err());
    }

    #[test]
    fn the_mix_format_names_its_sample_type_channels_and_rate() {
        let parsed = |bytes: Vec<u8>| MixFormat::parse(&bytes).map(|f| (f.sample, f.channels, f.rate, f.block_align));
        assert_eq!(parsed(extensible(2, 48_000, 32, 3)), Ok((Sample::Float32, 2, 48_000, 8)));
        assert_eq!(parsed(extensible(2, 44_100, 16, 1)), Ok((Sample::Pcm16, 2, 44_100, 4)));
        assert_eq!(parsed(extensible(1, 96_000, 24, 1)), Ok((Sample::Pcm24, 1, 96_000, 3)));
        assert_eq!(parsed(extensible(4, 48_000, 32, 1)), Ok((Sample::Pcm32, 4, 48_000, 16)));
        assert_eq!(parsed(wave(WAVE_FORMAT_IEEE_FLOAT, 1, 16_000, 32, None)), Ok((Sample::Float32, 1, 16_000, 4)));
        assert_eq!(parsed(wave(WAVE_FORMAT_PCM, 2, 22_050, 16, None)), Ok((Sample::Pcm16, 2, 22_050, 4)));
    }

    #[test]
    fn a_mix_format_it_cannot_read_is_an_error_not_silence() {
        let error = |bytes: &[u8]| MixFormat::parse(bytes).unwrap_err().to_string();
        assert!(error(&extensible(2, 48_000, 12, 1)).contains("type 1, 12 bits"));
        let mut ac3 = extensible(2, 48_000, 16, 0x92);
        assert!(error(&ac3).contains("type 146"));
        ac3[30] ^= 0xff; // Not a KSDATAFORMAT_SUBTYPE GUID at all.
        assert!(error(&ac3).contains("subformat"));
        assert!(error(&extensible(2, 48_000, 32, 3)[..30]).contains("truncated"));
        assert!(error(&wave(WAVE_FORMAT_PCM, 0, 48_000, 16, None)).contains("0 channels"));
        assert!(error(&[0; 4]).starts_with("The microphone could not start: "));
    }

    #[test]
    fn samples_of_every_type_scale_to_full_range() {
        assert_eq!(Sample::Float32.decode(&0.5f32.to_le_bytes()), 0.5);
        assert_eq!(Sample::Float64.decode(&(-0.25f64).to_le_bytes()), -0.25);
        assert_eq!(Sample::Pcm8.decode(&[192]), 0.5);
        assert_eq!(Sample::Pcm16.decode(&i16::MIN.to_le_bytes()), -1.0);
        assert_eq!(Sample::Pcm16.decode(&16_384i16.to_le_bytes()), 0.5);
        assert_eq!(Sample::Pcm24.decode(&[0x00, 0x00, 0xC0]), -0.5);
        assert_eq!(Sample::Pcm24.decode(&[0x00, 0x00, 0x40]), 0.5);
        assert_eq!(Sample::Pcm32.decode(&(1i32 << 30).to_le_bytes()), 0.5);
    }

    #[test]
    fn a_48_khz_stereo_float_packet_becomes_16_khz_mono() {
        let format = MixFormat::parse(&extensible(2, 48_000, 32, 3)).unwrap();
        let mut converter = Converter::new(format);
        // 10 ms: left at 0.5, right at -0.1, so the mix is 0.2.
        let packet: Vec<u8> = (0..480).flat_map(|_| [0.5f32.to_le_bytes(), (-0.1f32).to_le_bytes()]).flatten().collect();
        let out = converter.packet(&packet);
        assert_eq!(out.len(), 160);
        assert!(out.iter().all(|s| (s - 0.2).abs() < 1e-6), "{:?}", &out[..4]);
        assert_eq!(converter.silence(480), vec![0.0; 160], "a silent packet keeps the timing");
    }

    #[test]
    fn a_44_1_khz_four_channel_24_bit_packet_becomes_16_khz_mono_across_packets() {
        let format = MixFormat::parse(&extensible(4, 44_100, 24, 1)).unwrap();
        assert_eq!(format.block_align, 12);
        // Two channels at half scale, two silent: the mix is a quarter.
        let frame = [[0x00, 0x00, 0x40], [0x00, 0x00, 0x40], [0, 0, 0], [0, 0, 0]].concat();
        let second: Vec<u8> = frame.iter().copied().cycle().take(frame.len() * 4_410).collect();
        let mut converter = Converter::new(format);
        let (first, rest) = second.split_at(frame.len() * 1_000);
        let mut out = converter.packet(first);
        out.extend(converter.packet(rest));
        assert_eq!(out.len(), 1_600, "a tenth of a second at 16 kHz");
        assert!(out.iter().all(|s| (s - 0.25).abs() < 1e-6));
    }
}