A free, open source, and extensible speech-to-text application that works completely offline.
feat: pass custom words via transcribe-cpp vocabulary API (#2284)
CJ Pais4150a4a8d4e1
diff --git a/src-tauri/src/engine_supervisor/protocol.rs b/src-tauri/src/engine_supervisor/protocol.rsindex 9e353c95..4e2726a6 100644--- a/src-tauri/src/engine_supervisor/protocol.rs+++ b/src-tauri/src/engine_supervisor/protocol.rs@@ -153,6 +153,7 @@ pub struct LoadedInfo {pub on_gpu: bool,pub capabilities: Capabilities,pub supports_initial_prompt: bool,+ pub supports_vocabulary: bool,}/// Encode one frame. Refuses a section the reader would refuse, so andiff --git a/src-tauri/src/engine_supervisor/worker.rs b/src-tauri/src/engine_supervisor/worker.rsindex 6774c973..3277268d 100644--- a/src-tauri/src/engine_supervisor/worker.rs+++ b/src-tauri/src/engine_supervisor/worker.rs@@ -313,6 +313,7 @@ fn load(.capabilities().map_err(|e| format!("Failed to read model capabilities: {e}"))?,supports_initial_prompt: model.supports(Feature::InitialPrompt),+ supports_vocabulary: model.supports(Feature::Vocabulary),};Ok((session, info))}diff --git a/src-tauri/src/managers/transcription.rs b/src-tauri/src/managers/transcription.rsindex 904a9425..3a4a6b4c 100644--- a/src-tauri/src/managers/transcription.rs+++ b/src-tauri/src/managers/transcription.rs@@ -23,7 +23,7 @@ use std::thread;use std::time::{Duration, Instant, SystemTime};use tauri::{AppHandle, Emitter, Manager};use tauri_specta::Event;-use transcribe_cpp::{Backend, RunExtension, RunOptions, StreamOptions, Task, WhisperRunOptions};+use transcribe_cpp::{Backend, RunOptions, StreamOptions, Task};const STREAM_PERF_LOG_INTERVAL: Duration = Duration::from_secs(5);@@ -102,6 +102,8 @@ struct FinalizedStreamText {output_language: OutputLanguageEvidence,/// The streaming model's supported languages, for text-based detection.supported_languages: Vec<String>,+ /// Custom words went to the model as vocabulary; skip fuzzy correction.+ custom_words_prompted: bool,}/// Routes real-time audio frames to the active streaming worker. Shared between@@ -794,10 +796,13 @@ impl TranscriptionManager {&languages,run_plan.target_language.as_deref() == Some("en"),);+ let vocabulary = vocabulary_for_model(info.supports_vocabulary, &settings.custom_words);+ let custom_words_prompted = !vocabulary.is_empty();let run_options = RunOptions {task: run_plan.task,language: run_plan.language,target_language: run_plan.target_language,+ vocabulary,..Default::default()};@@ -879,6 +884,7 @@ impl TranscriptionManager {text: finalized.text.full,output_language,supported_languages: languages.clone(),+ custom_words_prompted,}))}Ok(None) => {@@ -936,12 +942,11 @@ impl TranscriptionManager {};let settings = get_settings(&self.app_handle);- // Streaming models do not receive a decode prompt, so custom words- // always go through the shared fuzzy post-correction path.+ // Skip fuzzy correction if the model already got the custom words.let filtered = post_process_transcription_text(finalized.text,&settings,- false,+ finalized.custom_words_prompted,&finalized.output_language,&finalized.supported_languages,);@@ -1054,14 +1059,11 @@ impl TranscriptionManager {);debug!("Output language evidence: {:?}", output_language);- // Apply fuzzy word correction if custom words are configured — UNLESS the- // words were already handed to the model as an initial prompt (whisper- // family). We don't pass a prompt to non-whisper models (it requires the- // whisper-kind run extension), so they still get fuzzy correction here.+ // Skip fuzzy correction if the model already got the custom words.let filtered_result = post_process_transcription_text(run.text,&settings,- run.model_is_whisper,+ run.custom_words_prompted,&output_language,&run.languages,);@@ -1102,9 +1104,7 @@ impl TranscriptionManager {/// transcribe-cpp: the model runs in the engine's worker process, which/// recovers from crashes and hangs itself. The loaded model's live/// capabilities (cheap GGUF-metadata reads) are the source of truth, not- /// the ModelManager copy. The whisper run extension is kind-tagged, so- /// non-whisper archs (parakeet, voxtral, …) reject it with INVALID_ARG;- /// attach it — and translate — only where supported.+ /// the ModelManager copy; translate is gated on what the model advertises.fn transcribe_cpp(&self,info: LoadedInfo,@@ -1113,37 +1113,22 @@ impl TranscriptionManager {validated_language: &str,active_model: &str,) -> Result<RunOutcome> {- // Whether the model advertises Feature::InitialPrompt. Informational- // (logged below); the whisper run extension and the fuzzy-correction- // skip are gated on `model_is_whisper` instead, since non-whisper- // archs can advertise the feature while rejecting the whisper-kind- // extension.- let model_takes_initial_prompt = info.supports_initial_prompt;- let model_is_whisper = info.arch == "whisper";let model_supports_translate = info.capabilities.supports_translate;let languages = info.capabilities.languages;++ let vocabulary = vocabulary_for_model(info.supports_vocabulary, &settings.custom_words);+ let custom_words_prompted = !vocabulary.is_empty();+debug!(- "transcribe-cpp model '{}' on '{}': initial_prompt={}, translate={}, languages={:?}",+ "transcribe-cpp model '{}' on '{}': initial_prompt={}, vocabulary={}, translate={}, languages={:?}",settings.selected_model,info.backend,- model_takes_initial_prompt,+ info.supports_initial_prompt,+ info.supports_vocabulary,model_supports_translate,languages);- // Custom words become the initial prompt ONLY for models that accept- // one (whisper family). Attaching the whisper run extension to a- // non-whisper arch is rejected with INVALID_ARG, so skip it there and- // let the fuzzy post-correction handle custom words instead.- let family = if settings.custom_words.is_empty() || !model_is_whisper {- None- } else {- Some(RunExtension::Whisper(WhisperRunOptions {- initial_prompt: Some(settings.custom_words.join(", ")),- ..Default::default()- }))- };-let run_plan = transcribe_cpp_run_plan(settings.translate_to_english,validated_language,@@ -1157,15 +1142,15 @@ impl TranscriptionManager {task: run_plan.task,language: run_plan.language,target_language: run_plan.target_language,- family,+ vocabulary,..Default::default()};debug!(- "transcribe-cpp run: task={:?}, language={:?}, initial_prompt={}",+ "transcribe-cpp run: task={:?}, language={:?}, vocabulary_terms={}",run_options.task,run_options.language,- run_options.family.is_some()+ run_options.vocabulary.len());let transcript = match self.engine.transcribe(audio, run_options) {@@ -1187,7 +1172,7 @@ impl TranscriptionManager {// Whisper's audio-based LID (auto mode only; `None` when a// language hint was passed).model_detected_language: transcript.language,- model_is_whisper,+ custom_words_prompted,})}}@@ -1202,12 +1187,8 @@ struct RunOutcome {output_was_translated: bool,/// The language the model itself detected, if it reported one.model_detected_language: Option<String>,- /// Whether the loaded model is actually whisper-family (arch string).- /// Non-whisper archs (e.g. Voxtral Small) can advertise- /// Feature::InitialPrompt yet reject the whisper-kind run extension with- /// INVALID_ARG, so the whisper extension must be gated on the arch, not- /// on the feature (see #1601).- model_is_whisper: bool,+ /// Custom words went to the model as vocabulary; skip fuzzy correction.+ custom_words_prompted: bool,}fn emit_stream_text(app_handle: &AppHandle, committed: &str, tentative: &str) {@@ -1404,6 +1385,17 @@ fn with_model_detected_language(}}+/// Custom words for transcribe-cpp's `vocabulary` slot, in priority order.+/// Empty for models without Feature::Vocabulary, which would warn on every run+/// and rely on fuzzy correction instead.+fn vocabulary_for_model(supports_vocabulary: bool, custom_words: &[String]) -> Vec<String> {+ if supports_vocabulary {+ custom_words.to_vec()+ } else {+ Vec::new()+ }+}+struct TranscribeCppRunPlan {task: Task,language: Option<String>,@@ -2143,6 +2135,37 @@ mod tests {assert_eq!(plan.language.as_deref(), Some("es"));assert_eq!(plan.target_language, None);}++ #[test]+ fn fuzzy_correction_is_skipped_exactly_when_vocabulary_was_sent() {+ let settings = AppSettings {+ selected_language: "en".to_string(),+ custom_words: vec!["Handy".to_string()],+ ..Default::default()+ };+ let evidence = OutputLanguageEvidence::UserSelected("en".to_string());+ let supported = languages(&["en"]);++ // The model was already prompted with the term: leave its output alone.+ let prompted = post_process_transcription_text(+ "i use handy every day".to_string(),+ &settings,+ true,+ &evidence,+ &supported,+ );+ assert_eq!(prompted, "i use handy every day");++ // No vocabulary sent: the fuzzy post-correction fixes the casing.+ let corrected = post_process_transcription_text(+ "i use handy every day".to_string(),+ &settings,+ false,+ &evidence,+ &supported,+ );+ assert_eq!(corrected, "i use Handy every day");+ }}impl Drop for TranscriptionManager {