Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

fix(speech): transcription aborted itself before the first encode

Jareerbf59ca11c8c2
diff --git a/engine/crates/concat-speech/src/transcribe.rs b/engine/crates/concat-speech/src/transcribe.rs
index f488e8e4..55de6901 100644
--- a/engine/crates/concat-speech/src/transcribe.rs
+++ b/engine/crates/concat-speech/src/transcribe.rs
@@ -20,7 +20,7 @@
//! segments come back relative to the window.
use std::path::PathBuf;
-use std::sync::atomic::Ordering;
+use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use concat_host::{AppDirs, SingleFlight};
@@ -122,6 +122,14 @@ fn models_dir(dirs: &AppDirs) -> Result<PathBuf, String> {
Ok(dir)
}
+/// whisper's abort callback: `user_data` is the run's cancel flag, and a
+/// true here stops the computation where it stands. See `transcribe`.
+unsafe extern "C" fn abort_when_set(user_data: *mut std::ffi::c_void) -> bool {
+ // SAFETY: the caller passed `Arc::as_ptr` of an `AtomicBool` it keeps
+ // alive for as long as whisper may call this.
+ unsafe { (*(user_data as *const AtomicBool)).load(Ordering::Relaxed) }
+}
+
fn model_file(dirs: &AppDirs, id: &str) -> Result<PathBuf, String> {
// Ids come from our own table; anything else is a bug, not input.
known(id).ok_or_else(|| format!("unknown model {id:?}"))?;
@@ -369,16 +377,35 @@ impl Transcriber {
.unwrap_or(4);
let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 1 });
params.set_n_threads(threads as i32);
- // The language is whisper's to hear: it detects it from the first
- // window, and an English-only model ignores the setting anyway.
- params.set_language(Some("auto"));
+ // The language is whisper's to hear, when it can hear more than
+ // one: a multilingual model detects it from the first window. An
+ // English-only model asked for "auto" still runs the detection,
+ // over a vocabulary it was never trained on, and "hears" Thai at
+ // one percent - so it is simply told English.
+ params.set_language(Some(if context.is_multilingual() {
+ "auto"
+ } else {
+ "en"
+ }));
params.set_print_special(false);
params.set_print_progress(false);
params.set_print_realtime(false);
params.set_print_timestamps(false);
params.set_suppress_blank(true);
- let abort = Arc::clone(&cancel);
- params.set_abort_callback_safe(move || abort.load(Ordering::Relaxed));
+ // The raw hooks, not `set_abort_callback_safe`. whisper-rs 0.16's
+ // safe one hands whisper a pointer to a boxed trait object with a
+ // trampoline typed for the closure itself, so whisper reads the
+ // box's two words as the closure and the answer is whatever byte
+ // sits there: true, as a rule, and every encode was aborted before
+ // it began - "failed to encode", error -6, on the first clip. The
+ // flag itself is what whisper is given here, and it outlives the
+ // call: `cancel` is held to the end of this function.
+ // SAFETY: `abort_when_set` reads `user_data` as the `AtomicBool`
+ // inside `cancel`, which lives until `full` has returned.
+ unsafe {
+ params.set_abort_callback(Some(abort_when_set));
+ params.set_abort_callback_user_data(Arc::as_ptr(&cancel) as *mut std::ffi::c_void);
+ }
params.set_progress_callback_safe(progress);
state
@@ -461,3 +488,39 @@ mod tests {
let _ = std::fs::remove_dir_all(&scratch);
}
}
+
+#[cfg(test)]
+mod real_audio {
+ use super::*;
+
+ /// The whole path through whisper on a real file with a downloaded
+ /// model - the thing the unit tests cannot cover, and what the abort
+ /// hook broke. Ignored: it wants the app's model directory and a file
+ /// named in CONCAT_TRANSCRIBE_AUDIO.
+ ///
+ /// CONCAT_TRANSCRIBE_AUDIO=speech.wav cargo test -p concat-speech -- --ignored real_audio
+ #[test]
+ #[ignore = "needs a downloaded tiny.en and CONCAT_TRANSCRIBE_AUDIO"]
+ fn transcribes_a_file_with_the_installed_model() {
+ let Ok(audio) = std::env::var("CONCAT_TRANSCRIBE_AUDIO") else {
+ eprintln!("CONCAT_TRANSCRIBE_AUDIO not set; nothing to transcribe");
+ return;
+ };
+ let dirs = AppDirs::locate().expect("the app's directories");
+ let transcriber = Transcriber::new();
+ let request = TranscribeRequest {
+ path: audio,
+ audio_stream: None,
+ source_start: 0.0,
+ window: 30.0,
+ model_id: "tiny.en".to_owned(),
+ };
+ let segments = transcriber
+ .transcribe(&dirs, &request, |percent| eprintln!("{percent}%"))
+ .expect("transcription");
+ for segment in &segments {
+ eprintln!("{:.2}-{:.2} {}", segment.start, segment.end, segment.text);
+ }
+ assert!(!segments.is_empty(), "no words heard");
+ }
+}