Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Upgrade Rubato and test audio resampling contracts

Manav Rathib850ca4815f8
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 175bede711..c4127ac674 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -563,6 +563,39 @@ dependencies = [
"syn 2.0.117",
]
+[[package]]
+name = "audio-codec-algorithms"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1254ebf6529f3763c491acfb5ab6e960809e3c75a38584cc664f8c5667fb7107"
+
+[[package]]
+name = "audioadapter"
+version = "5.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1292ef9edf681b7426ed089004b021a42896492f58bd0c909ad44e5faec9ac4"
+
+[[package]]
+name = "audioadapter-buffers"
+version = "5.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46289a81a3bfa26d0f8b2415f8ca9decde9a66308a150568af5c102381b3c8ce"
+dependencies = [
+ "audioadapter",
+ "audioadapter-sample",
+ "num-traits",
+]
+
+[[package]]
+name = "audioadapter-sample"
+version = "5.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5258faecf4edbe35ec483bda9b1f7438e6b391891819269c6b5461aeaa0e8d7d"
+dependencies = [
+ "audio-codec-algorithms",
+ "num-traits",
+]
+
[[package]]
name = "autocfg"
version = "1.5.0"
@@ -7972,14 +8005,18 @@ dependencies = [
[[package]]
name = "rubato"
-version = "0.16.2"
+version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5258099699851cfd0082aeb645feb9c084d9a5e1f1b8d5372086b989fc5e56a1"
+checksum = "a7cb1ffaf8738df50aab642a7f6465df81c6ba9e2818268053487165298114be"
dependencies = [
+ "audioadapter",
+ "audioadapter-buffers",
"num-complex",
"num-integer",
"num-traits",
"realfft",
+ "visibility",
+ "windowfunctions",
]
[[package]]
@@ -10677,6 +10714,17 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+[[package]]
+name = "visibility"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
[[package]]
name = "vswhom"
version = "0.1.0"
@@ -11033,6 +11081,15 @@ dependencies = [
"windows-version",
]
+[[package]]
+name = "windowfunctions"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90628d739333b7c5d2ee0b70210b97b8cddc38440c682c96fd9e2c24c2db5f3a"
+dependencies = [
+ "num-traits",
+]
+
[[package]]
name = "windows"
version = "0.61.3"
diff --git a/rust/crates/ensu/Cargo.toml b/rust/crates/ensu/Cargo.toml
index 087b289d1c..715b83daff 100644
--- a/rust/crates/ensu/Cargo.toml
+++ b/rust/crates/ensu/Cargo.toml
@@ -9,7 +9,7 @@ ente-ensu-crypto.workspace = true
ente-image.workspace = true
memmap2 = "0.9.9"
regex.workspace = true
-rubato = { version = "0.16.0", optional = true }
+rubato = { version = "5.0.0", optional = true }
rusqlite = { workspace = true, default-features = true, optional = true }
self_cell = "1.0.0"
serde.workspace = true
diff --git a/rust/crates/ensu/src/transcription/audio.rs b/rust/crates/ensu/src/transcription/audio.rs
index 5fd996e9fc..f8bb38792d 100644
--- a/rust/crates/ensu/src/transcription/audio.rs
+++ b/rust/crates/ensu/src/transcription/audio.rs
@@ -1,7 +1,8 @@
use std::path::Path;
use std::time::Duration;
-use rubato::{FftFixedIn, Resampler};
+use rubato::audioadapter_buffers::direct::InterleavedSlice;
+use rubato::{Fft, FixedSync, Resampler, WindowFunction};
use transcribe_rs::vad::{SileroVad, SmoothedVad, Vad};
use crate::transcription::{Result, TranscriptionError};
@@ -96,7 +97,7 @@ fn extract_speech(
}
struct FrameResampler {
- resampler: Option<FftFixedIn<f32>>,
+ resampler: Option<Fft<f32>>,
chunk_in: usize,
in_buf: Vec<f32>,
frame_samples: usize,
@@ -115,12 +116,14 @@ impl FrameResampler {
let resampler = if in_hz == out_hz {
None
} else {
- Some(FftFixedIn::<f32>::new(
+ Some(Fft::<f32>::new_custom(
in_hz,
out_hz,
RESAMPLER_CHUNK_SIZE,
1,
1,
+ WindowFunction::BlackmanHarris2,
+ FixedSync::Input,
)?)
};
@@ -147,9 +150,10 @@ impl FrameResampler {
if self.in_buf.len() == self.chunk_in {
if let Some(resampler) = self.resampler.as_mut()
- && let Ok(out) = resampler.process(&[&self.in_buf[..]], None)
+ && let Ok(input) = InterleavedSlice::new(&self.in_buf, 1, self.chunk_in)
+ && let Ok(out) = resampler.process(&input, None)
{
- self.emit_frames(&out[0], &mut emit);
+ self.emit_frames(&out.take_data(), &mut emit);
}
self.in_buf.clear();
}
@@ -161,8 +165,10 @@ impl FrameResampler {
&& !self.in_buf.is_empty()
{
self.in_buf.resize(self.chunk_in, 0.0);
- if let Ok(out) = resampler.process(&[&self.in_buf[..]], None) {
- self.emit_frames(&out[0], &mut emit);
+ if let Ok(input) = InterleavedSlice::new(&self.in_buf, 1, self.chunk_in)
+ && let Ok(out) = resampler.process(&input, None)
+ {
+ self.emit_frames(&out.take_data(), &mut emit);
}
}
@@ -187,3 +193,85 @@ impl FrameResampler {
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use std::f32::consts::TAU;
+
+ use super::*;
+
+ fn resample(input: &[f32], sample_rate: usize, push_size: usize) -> Vec<f32> {
+ let mut resampler =
+ FrameResampler::new(sample_rate, 16_000, Duration::from_millis(30)).unwrap();
+ let mut output = Vec::new();
+ let mut emit = |frame: &[f32]| {
+ assert_eq!(frame.len(), 480);
+ output.extend_from_slice(frame);
+ };
+ for chunk in input.chunks(push_size) {
+ resampler.push(chunk, &mut emit);
+ }
+ resampler.finish(emit);
+ output
+ }
+
+ #[test]
+ fn native_rate_preserves_samples_and_pads_the_last_frame() {
+ let input: Vec<_> = (0..961).map(|i| i as f32 / 1000.0 - 0.5).collect();
+ let output = resample(&input, 16_000, 137);
+
+ assert_eq!(output.len(), 1440);
+ assert_eq!(&output[..input.len()], input);
+ assert!(output[input.len()..].iter().all(|&sample| sample == 0.0));
+ }
+
+ #[test]
+ fn resampling_is_independent_of_push_boundaries() {
+ for sample_rate in [44_100, 48_000] {
+ let input: Vec<_> = (0..sample_rate / 4 + 19)
+ .map(|i| (TAU * 1000.0 * i as f32 / sample_rate as f32).sin())
+ .collect();
+ let expected = resample(&input, sample_rate, input.len());
+ assert!(!expected.is_empty());
+
+ for push_size in [1, 257, 1023, 1024, 1025] {
+ assert_eq!(
+ resample(&input, sample_rate, push_size),
+ expected,
+ "{sample_rate} Hz, {push_size} samples per push"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn downsampling_preserves_speech_frequencies_and_rejects_aliasing() {
+ fn amplitude(samples: &[f32], frequency: f32) -> f32 {
+ let (sin, cos) =
+ samples
+ .iter()
+ .enumerate()
+ .fold((0.0, 0.0), |(sin, cos), (i, &sample)| {
+ let phase = TAU * frequency * i as f32 / 16_000.0;
+ (sin + sample * phase.sin(), cos + sample * phase.cos())
+ });
+ 2.0 * sin.hypot(cos) / samples.len() as f32
+ }
+
+ for sample_rate in [44_100, 48_000] {
+ let input: Vec<_> = (0..sample_rate)
+ .map(|i| {
+ let phase = TAU * i as f32 / sample_rate as f32;
+ 0.5 * (1000.0 * phase).sin() + 0.5 * (12_000.0 * phase).sin()
+ })
+ .collect();
+ let output = resample(&input, sample_rate, 257);
+ let settled = &output[1600..9600];
+ let speech = amplitude(settled, 1000.0);
+ let alias = amplitude(settled, 4000.0);
+
+ assert!((speech - 0.5).abs() < 0.01, "{sample_rate} Hz: {speech}");
+ assert!(alias < 0.01, "{sample_rate} Hz: {alias}");
+ }
+ }
+}