Skip to content

armgit / Handy

Public Imported from GitHub
0

A free, open source, and extensible speech-to-text application that works completely offline.

feat: pass custom words via transcribe-cpp vocabulary API (#2284)

CJ Pais4150a4a8d4e1
diff --git a/src-tauri/src/engine_supervisor/protocol.rs b/src-tauri/src/engine_supervisor/protocol.rs
index 9e353c95..4e2726a6 100644
--- a/src-tauri/src/engine_supervisor/protocol.rs
+++ b/src-tauri/src/engine_supervisor/protocol.rs
@@ -153,6 +153,7 @@ pub struct LoadedInfo {
pub on_gpu: bool,
pub capabilities: Capabilities,
pub supports_initial_prompt: bool,
+ pub supports_vocabulary: bool,
}
/// Encode one frame. Refuses a section the reader would refuse, so an
diff --git a/src-tauri/src/engine_supervisor/worker.rs b/src-tauri/src/engine_supervisor/worker.rs
index 6774c973..3277268d 100644
--- a/src-tauri/src/engine_supervisor/worker.rs
+++ b/src-tauri/src/engine_supervisor/worker.rs
@@ -313,6 +313,7 @@ fn load(
.capabilities()
.map_err(|e| format!("Failed to read model capabilities: {e}"))?,
supports_initial_prompt: model.supports(Feature::InitialPrompt),
+ supports_vocabulary: model.supports(Feature::Vocabulary),
};
Ok((session, info))
}
diff --git a/src-tauri/src/managers/transcription.rs b/src-tauri/src/managers/transcription.rs
index 904a9425..3a4a6b4c 100644
--- a/src-tauri/src/managers/transcription.rs
+++ b/src-tauri/src/managers/transcription.rs
@@ -23,7 +23,7 @@ use std::thread;
use std::time::{Duration, Instant, SystemTime};
use tauri::{AppHandle, Emitter, Manager};
use tauri_specta::Event;
-use transcribe_cpp::{Backend, RunExtension, RunOptions, StreamOptions, Task, WhisperRunOptions};
+use transcribe_cpp::{Backend, RunOptions, StreamOptions, Task};
const STREAM_PERF_LOG_INTERVAL: Duration = Duration::from_secs(5);
@@ -102,6 +102,8 @@ struct FinalizedStreamText {
output_language: OutputLanguageEvidence,
/// The streaming model's supported languages, for text-based detection.
supported_languages: Vec<String>,
+ /// Custom words went to the model as vocabulary; skip fuzzy correction.
+ custom_words_prompted: bool,
}
/// Routes real-time audio frames to the active streaming worker. Shared between
@@ -794,10 +796,13 @@ impl TranscriptionManager {
&languages,
run_plan.target_language.as_deref() == Some("en"),
);
+ let vocabulary = vocabulary_for_model(info.supports_vocabulary, &settings.custom_words);
+ let custom_words_prompted = !vocabulary.is_empty();
let run_options = RunOptions {
task: run_plan.task,
language: run_plan.language,
target_language: run_plan.target_language,
+ vocabulary,
..Default::default()
};
@@ -879,6 +884,7 @@ impl TranscriptionManager {
text: finalized.text.full,
output_language,
supported_languages: languages.clone(),
+ custom_words_prompted,
}))
}
Ok(None) => {
@@ -936,12 +942,11 @@ impl TranscriptionManager {
};
let settings = get_settings(&self.app_handle);
- // Streaming models do not receive a decode prompt, so custom words
- // always go through the shared fuzzy post-correction path.
+ // Skip fuzzy correction if the model already got the custom words.
let filtered = post_process_transcription_text(
finalized.text,
&settings,
- false,
+ finalized.custom_words_prompted,
&finalized.output_language,
&finalized.supported_languages,
);
@@ -1054,14 +1059,11 @@ impl TranscriptionManager {
);
debug!("Output language evidence: {:?}", output_language);
- // Apply fuzzy word correction if custom words are configured — UNLESS the
- // words were already handed to the model as an initial prompt (whisper
- // family). We don't pass a prompt to non-whisper models (it requires the
- // whisper-kind run extension), so they still get fuzzy correction here.
+ // Skip fuzzy correction if the model already got the custom words.
let filtered_result = post_process_transcription_text(
run.text,
&settings,
- run.model_is_whisper,
+ run.custom_words_prompted,
&output_language,
&run.languages,
);
@@ -1102,9 +1104,7 @@ impl TranscriptionManager {
/// transcribe-cpp: the model runs in the engine's worker process, which
/// recovers from crashes and hangs itself. The loaded model's live
/// capabilities (cheap GGUF-metadata reads) are the source of truth, not
- /// the ModelManager copy. The whisper run extension is kind-tagged, so
- /// non-whisper archs (parakeet, voxtral, …) reject it with INVALID_ARG;
- /// attach it — and translate — only where supported.
+ /// the ModelManager copy; translate is gated on what the model advertises.
fn transcribe_cpp(
&self,
info: LoadedInfo,
@@ -1113,37 +1113,22 @@ impl TranscriptionManager {
validated_language: &str,
active_model: &str,
) -> Result<RunOutcome> {
- // Whether the model advertises Feature::InitialPrompt. Informational
- // (logged below); the whisper run extension and the fuzzy-correction
- // skip are gated on `model_is_whisper` instead, since non-whisper
- // archs can advertise the feature while rejecting the whisper-kind
- // extension.
- let model_takes_initial_prompt = info.supports_initial_prompt;
- let model_is_whisper = info.arch == "whisper";
let model_supports_translate = info.capabilities.supports_translate;
let languages = info.capabilities.languages;
+
+ let vocabulary = vocabulary_for_model(info.supports_vocabulary, &settings.custom_words);
+ let custom_words_prompted = !vocabulary.is_empty();
+
debug!(
- "transcribe-cpp model '{}' on '{}': initial_prompt={}, translate={}, languages={:?}",
+ "transcribe-cpp model '{}' on '{}': initial_prompt={}, vocabulary={}, translate={}, languages={:?}",
settings.selected_model,
info.backend,
- model_takes_initial_prompt,
+ info.supports_initial_prompt,
+ info.supports_vocabulary,
model_supports_translate,
languages
);
- // Custom words become the initial prompt ONLY for models that accept
- // one (whisper family). Attaching the whisper run extension to a
- // non-whisper arch is rejected with INVALID_ARG, so skip it there and
- // let the fuzzy post-correction handle custom words instead.
- let family = if settings.custom_words.is_empty() || !model_is_whisper {
- None
- } else {
- Some(RunExtension::Whisper(WhisperRunOptions {
- initial_prompt: Some(settings.custom_words.join(", ")),
- ..Default::default()
- }))
- };
-
let run_plan = transcribe_cpp_run_plan(
settings.translate_to_english,
validated_language,
@@ -1157,15 +1142,15 @@ impl TranscriptionManager {
task: run_plan.task,
language: run_plan.language,
target_language: run_plan.target_language,
- family,
+ vocabulary,
..Default::default()
};
debug!(
- "transcribe-cpp run: task={:?}, language={:?}, initial_prompt={}",
+ "transcribe-cpp run: task={:?}, language={:?}, vocabulary_terms={}",
run_options.task,
run_options.language,
- run_options.family.is_some()
+ run_options.vocabulary.len()
);
let transcript = match self.engine.transcribe(audio, run_options) {
@@ -1187,7 +1172,7 @@ impl TranscriptionManager {
// Whisper's audio-based LID (auto mode only; `None` when a
// language hint was passed).
model_detected_language: transcript.language,
- model_is_whisper,
+ custom_words_prompted,
})
}
}
@@ -1202,12 +1187,8 @@ struct RunOutcome {
output_was_translated: bool,
/// The language the model itself detected, if it reported one.
model_detected_language: Option<String>,
- /// Whether the loaded model is actually whisper-family (arch string).
- /// Non-whisper archs (e.g. Voxtral Small) can advertise
- /// Feature::InitialPrompt yet reject the whisper-kind run extension with
- /// INVALID_ARG, so the whisper extension must be gated on the arch, not
- /// on the feature (see #1601).
- model_is_whisper: bool,
+ /// Custom words went to the model as vocabulary; skip fuzzy correction.
+ custom_words_prompted: bool,
}
fn emit_stream_text(app_handle: &AppHandle, committed: &str, tentative: &str) {
@@ -1404,6 +1385,17 @@ fn with_model_detected_language(
}
}
+/// Custom words for transcribe-cpp's `vocabulary` slot, in priority order.
+/// Empty for models without Feature::Vocabulary, which would warn on every run
+/// and rely on fuzzy correction instead.
+fn vocabulary_for_model(supports_vocabulary: bool, custom_words: &[String]) -> Vec<String> {
+ if supports_vocabulary {
+ custom_words.to_vec()
+ } else {
+ Vec::new()
+ }
+}
+
struct TranscribeCppRunPlan {
task: Task,
language: Option<String>,
@@ -2143,6 +2135,37 @@ mod tests {
assert_eq!(plan.language.as_deref(), Some("es"));
assert_eq!(plan.target_language, None);
}
+
+ #[test]
+ fn fuzzy_correction_is_skipped_exactly_when_vocabulary_was_sent() {
+ let settings = AppSettings {
+ selected_language: "en".to_string(),
+ custom_words: vec!["Handy".to_string()],
+ ..Default::default()
+ };
+ let evidence = OutputLanguageEvidence::UserSelected("en".to_string());
+ let supported = languages(&["en"]);
+
+ // The model was already prompted with the term: leave its output alone.
+ let prompted = post_process_transcription_text(
+ "i use handy every day".to_string(),
+ &settings,
+ true,
+ &evidence,
+ &supported,
+ );
+ assert_eq!(prompted, "i use handy every day");
+
+ // No vocabulary sent: the fuzzy post-correction fixes the casing.
+ let corrected = post_process_transcription_text(
+ "i use handy every day".to_string(),
+ &settings,
+ false,
+ &evidence,
+ &supported,
+ );
+ assert_eq!(corrected, "i use Handy every day");
+ }
}
impl Drop for TranscriptionManager {