Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

fix(captions): each caption starts when its words do, not where the one before it ended (#438)

Jareer2922614cd310
diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md
index 74e01f6f..f8a2bf7e 100644
--- a/THIRD_PARTY_NOTICES.md
+++ b/THIRD_PARTY_NOTICES.md
@@ -26,6 +26,12 @@ originate at https://huggingface.co/ggerganov/whisper.cpp (MIT); Concat
mirrors them and downloads them on demand from that mirror, and never
bundles them. See "The model mirror" below.
+The one model Concat does bundle is Silero VAD v6.2.0 in whisper.cpp's
+format (`ggml-silero-v6.2.0.bin`, from https://huggingface.co/ggml-org/whisper-vad,
+MIT; Silero VAD itself is MIT, https://github.com/snakers4/silero-vad,
+Copyright (c) 2020-present Silero Team). It finds where the speech is, so
+captions are timed from it.
+
## Slint (used under GPL-3.0-only)
The `concat` crate (`src/crates/concat`) builds against
diff --git a/src/crates/concat-speech/assets/ggml-silero-v6.2.0.bin b/src/crates/concat-speech/assets/ggml-silero-v6.2.0.bin
new file mode 100644
index 00000000..e08fad67
Binary files /dev/null and b/src/crates/concat-speech/assets/ggml-silero-v6.2.0.bin differ
diff --git a/src/crates/concat-speech/src/transcribe.rs b/src/crates/concat-speech/src/transcribe.rs
index bb16dd53..a3ee01d6 100644
--- a/src/crates/concat-speech/src/transcribe.rs
+++ b/src/crates/concat-speech/src/transcribe.rs
@@ -26,7 +26,10 @@ use std::sync::{Arc, Mutex};
use concat_host::{AppDirs, SingleFlight};
use concat_media::{AudioDecoder, AudioOptions, SampleFormat};
use serde::{Deserialize, Serialize};
-use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters};
+use whisper_rs::{
+ FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters, WhisperVadContext,
+ WhisperVadContextParams, WhisperVadParams,
+};
use crate::{DownloadProgress, JobStage};
@@ -224,6 +227,77 @@ fn model_file(dirs: &AppDirs, id: &str) -> Result<PathBuf, String> {
Ok(models_dir(dirs)?.join(format!("ggml-{id}.bin")))
}
+/// Silero VAD v6.2.0 in whisper.cpp's format, from ggml-org/whisper-vad
+/// (MIT; sha256 2aa269b7...6987). It tells whisper where the speech is, so
+/// whisper hears only that and times each caption from where its words
+/// begin. Without it a caption started where the one before had ended:
+/// the silence between them went to the next line, which came up seconds
+/// before anyone spoke (#438). Under a megabyte, so it ships inside the
+/// app rather than as a second download.
+const VAD_MODEL: &[u8] = include_bytes!("../assets/ggml-silero-v6.2.0.bin");
+
+/// The spans of `samples` (16 kHz mono) that are speech, in seconds, by
+/// Silero VAD. Empty when the model cannot be had or will not run: the
+/// captions then keep whisper's own times, as they always did.
+fn speech_spans(dirs: &AppDirs, samples: &[f32], threads: usize) -> Vec<(f64, f64)> {
+ let spans = vad_model(dirs).and_then(|path| {
+ let path = path
+ .to_str()
+ .ok_or_else(|| format!("{} is not a path whisper can open", path.display()))?
+ .to_owned();
+ let mut context_params = WhisperVadContextParams::new();
+ context_params.set_n_threads(threads as i32);
+ context_params.set_use_gpu(false);
+ let mut context = WhisperVadContext::new(&path, context_params)
+ .map_err(|error| format!("could not load {path}: {error}"))?;
+ let segments = context
+ .segments_from_samples(WhisperVadParams::new(), samples)
+ .map_err(|error| format!("voice detection failed: {error}"))?;
+ // Centiseconds, like whisper's own timestamps.
+ Ok(segments
+ .map(|span| (f64::from(span.start) / 100.0, f64::from(span.end) / 100.0))
+ .collect())
+ });
+ spans.unwrap_or_else(|error: String| {
+ log::warn!("transcribe: captions keep whisper's times: {error}");
+ Vec::new()
+ })
+}
+
+/// A caption's span drawn in to the speech it covers. Whisper starts a
+/// caption where the one before it ended, so the silence between two lines
+/// went to the second, which came up seconds before it was said (#438); the
+/// voice detector knows where the words begin and end. A caption over no
+/// detected speech - a whisper too quiet for the detector, say - keeps the
+/// times whisper gave it.
+fn on_speech(start: f64, end: f64, speech: &[(f64, f64)]) -> (f64, f64) {
+ let mut covered = speech
+ .iter()
+ .filter(|(from, to)| *from < end && *to > start);
+ let Some(&(first, mut last)) = covered.next() else {
+ return (start, end);
+ };
+ if let Some(&(_, to)) = covered.last() {
+ last = to;
+ }
+ (start.max(first), end.min(last))
+}
+
+/// The VAD model as a file whisper can open, written beside the
+/// transcription models the first time it is wanted.
+fn vad_model(dirs: &AppDirs) -> Result<PathBuf, String> {
+ let path = models_dir(dirs)?.join("ggml-silero-v6.2.0.bin");
+ let whole = std::fs::metadata(&path)
+ .is_ok_and(|meta| meta.len() == VAD_MODEL.len() as u64);
+ if !whole {
+ let part = path.with_extension("part");
+ std::fs::write(&part, VAD_MODEL)
+ .and_then(|()| std::fs::rename(&part, &path))
+ .map_err(|error| format!("could not write {}: {error}", path.display()))?;
+ }
+ Ok(path)
+}
+
/// One model, as the settings panel shows it.
#[derive(Serialize, Clone, Debug)]
#[serde(rename_all = "camelCase")]
@@ -524,10 +598,15 @@ impl Transcriber {
});
});
+ // Where the speech is, for timing the captions; see `VAD_MODEL`.
+ // Without it the captions come out as whisper timed them.
+ let speech = speech_spans(dirs, &samples, threads);
+ let run = state.full(params, &samples);
+
// An abort ends whisper's run as an error of its own - "failed to
// encode", -6 - so a Cancel read as a failure, and the person was
// told transcription failed when they had stopped it (#388).
- if let Err(error) = state.full(params, &samples) {
+ if let Err(error) = run {
if cancel.load(Ordering::Relaxed) {
return Err("transcription cancelled".to_owned());
}
@@ -543,6 +622,7 @@ impl Transcriber {
// Whisper's timestamps are centiseconds.
let start = segment.start_timestamp() as f64 / 100.0;
let end = segment.end_timestamp() as f64 / 100.0;
+ let (start, end) = on_speech(start, end, &speech);
let text = segment.to_str_lossy().ok()?.trim().to_owned();
keep_segment(start, end, text)
})
@@ -574,6 +654,54 @@ fn keep_segment(start: f64, end: f64, text: String) -> Option<Segment> {
mod tests {
use super::*;
+ /// Prints the captions a downloaded model makes of a file, for timing
+ /// them by hand against the speech: `CONCAT_CAPTION_FILE=talk.wav
+ /// CONCAT_CAPTION_MODEL=base.en cargo test -p concat-speech --
+ /// --ignored --nocapture captions_of_a_file`.
+ #[test]
+ #[ignore = "needs a downloaded model and a file to caption"]
+ fn captions_of_a_file() {
+ let path = std::env::var("CONCAT_CAPTION_FILE").expect("CONCAT_CAPTION_FILE");
+ let model_id = std::env::var("CONCAT_CAPTION_MODEL").unwrap_or("base.en".to_owned());
+ let dirs = AppDirs::locate().expect("app dirs");
+ // Past the end is fine: the decoder stops where the sound does.
+ let window = 3600.0;
+ let segments = Transcriber::new()
+ .transcribe(
+ &dirs,
+ &TranscribeRequest {
+ path,
+ audio_stream: None,
+ source_start: 0.0,
+ window,
+ model_id,
+ },
+ |_| {},
+ )
+ .expect("transcribes");
+ for segment in segments {
+ println!("{:7.2} {:7.2} {}", segment.start, segment.end, segment.text);
+ }
+ }
+
+ /// The reporter's clip in #438: five lines, each after a silence.
+ /// Whisper started each where the last had ended; the speech says
+ /// where it starts.
+ #[test]
+ fn a_caption_starts_and_ends_with_the_speech_it_covers() {
+ let speech = [(3.07, 4.93), (14.08, 16.48), (25.09, 27.52)];
+ assert_eq!(on_speech(0.0, 7.0, &speech), (3.07, 4.93));
+ assert_eq!(on_speech(7.0, 18.0, &speech), (14.08, 16.48));
+ // A line whisper split in two keeps its split.
+ assert_eq!(on_speech(25.0, 26.0, &speech), (25.09, 26.0));
+ assert_eq!(on_speech(26.0, 30.0, &speech), (26.0, 27.52));
+ // Two spans under one caption: from the first to the last.
+ assert_eq!(on_speech(0.0, 20.0, &speech), (3.07, 16.48));
+ // Over no detected speech, or with no detector: whisper's times.
+ assert_eq!(on_speech(40.0, 45.0, &speech), (40.0, 45.0));
+ assert_eq!(on_speech(0.0, 7.0, &[]), (0.0, 7.0));
+ }
+
#[test]
fn stage_directions_and_degenerate_spans_are_not_captions() {
assert_eq!(