Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(audio): transcription resilience + correctness (2/n) — whisper panic, deepgram keyterm, openai retry (#4236)

Louis Beaumont5c663485b63c
diff --git a/crates/screenpipe-audio/src/transcription/deepgram/batch.rs b/crates/screenpipe-audio/src/transcription/deepgram/batch.rs
index 10814278b..c95b8bbbe 100644
--- a/crates/screenpipe-audio/src/transcription/deepgram/batch.rs
+++ b/crates/screenpipe-audio/src/transcription/deepgram/batch.rs
@@ -145,12 +145,14 @@ fn create_query_params(languages: Vec<Language>, vocabulary: &[VocabularyEntry])
}
}
- // Add vocabulary as Deepgram keyterms (Nova-3 uses `keyterm` instead of `keywords`)
+ // Add vocabulary as Deepgram nova-3 keyterms. nova-3 keyterm prompting takes
+ // plain terms with NO `:intensifier` (that's the older nova-2 `keywords`
+ // syntax) — a trailing `:2` would be sent as part of the literal term.
for entry in vocabulary.iter().take(100) {
let keyword = entry.replacement.as_deref().unwrap_or(&entry.word);
- // Simple percent-encode spaces for the query string
- let encoded = keyword.replace(' ', "%20");
- query_params.push_str(&format!("&keyterm={}:2", encoded));
+ // Percent-encode spaces (and the comma, which separates keyterms).
+ let encoded = keyword.replace(' ', "%20").replace(',', "%2C");
+ query_params.push_str(&format!("&keyterm={}", encoded));
}
query_params
@@ -669,6 +671,25 @@ mod tests {
assert!(!params.contains("detect_language"));
}
+ #[test]
+ fn vocabulary_becomes_plain_keyterms_without_intensifier() {
+ let vocab = vec![
+ VocabularyEntry {
+ word: "Screenpipe".into(),
+ replacement: None,
+ },
+ VocabularyEntry {
+ word: "Core Audio".into(),
+ replacement: None,
+ },
+ ];
+ let params = create_query_params(vec![], &vocab);
+ assert!(params.contains("&keyterm=Screenpipe"), "got: {params}");
+ // nova-3 keyterms take no `:intensifier` (that's nova-2 keywords).
+ assert!(!params.contains(":2"), "got: {params}");
+ assert!(params.contains("&keyterm=Core%20Audio"), "got: {params}");
+ }
+
#[test]
fn transient_error_text_flags_network_blips_only() {
// retryable transport failures
diff --git a/crates/screenpipe-audio/src/transcription/openai_compatible/batch.rs b/crates/screenpipe-audio/src/transcription/openai_compatible/batch.rs
index e45a7a93a..e89f10c6b 100644
--- a/crates/screenpipe-audio/src/transcription/openai_compatible/batch.rs
+++ b/crates/screenpipe-audio/src/transcription/openai_compatible/batch.rs
@@ -71,62 +71,99 @@ pub async fn transcribe_with_openai_compatible(
.build()?,
),
};
- // Build multipart form
- let mut form = multipart::Form::new()
- .text("model", model.to_string())
- .text("response_format", "json".to_string())
- .part(
- "file",
- multipart::Part::bytes(audio_bytes)
- .file_name(file_name.to_string())
- .mime_str(mime_type)?,
- );
+ // Send with bounded retry on transient transport failures (timeouts,
+ // connection resets, "error sending request" blips — the recurring
+ // openai-compatible failures in Sentry, often a local server briefly
+ // unavailable). The multipart form can't be reused across attempts, so
+ // rebuild it each time; audio bytes are cloned only on a retry. HTTP status
+ // errors come back as Ok(Response) and are handled by handle_response, so a
+ // reqwest::Error here is always transport-level.
+ const MAX_ATTEMPTS: u32 = 3;
+ let mut last_err: Option<reqwest::Error> = None;
+ for attempt in 0..MAX_ATTEMPTS {
+ let mut form = multipart::Form::new()
+ .text("model", model.to_string())
+ .text("response_format", "json".to_string())
+ .part(
+ "file",
+ multipart::Part::bytes(audio_bytes.clone())
+ .file_name(file_name.to_string())
+ .mime_str(mime_type)?,
+ );
+
+ if !languages.is_empty() {
+ form = form.text("language", languages[0].as_lang_code().to_string());
+ }
- // Add language if specified
- if !languages.is_empty() {
- // Use the first language as the primary language hint
- let lang_code = languages[0].as_lang_code();
- form = form.text("language", lang_code.to_string());
- }
+ // Vocabulary/hotwords for biasing: `prompt` = OpenAI initial_prompt,
+ // `context` = mlx-audio hotwords. Send both so it works on either server.
+ if !vocabulary.is_empty() {
+ let prompt = vocabulary.join(", ");
+ form = form.text("prompt", prompt.clone()).text("context", prompt);
+ }
- // Pass vocabulary/hotwords for transcription biasing.
- // - OpenAI Whisper API: uses `prompt` as initial_prompt
- // - mlx-audio (VibeVoice-ASR): uses `context` for hotwords (ignores `prompt`)
- // Send both fields so it works regardless of which server is running.
- if !vocabulary.is_empty() {
- let prompt = vocabulary.join(", ");
- debug!("passing vocabulary as prompt + context: {}", prompt);
- form = form.text("prompt", prompt.clone());
- form = form.text("context", prompt);
- }
+ let mut request = client
+ .post(format!("{}/v1/audio/transcriptions", endpoint))
+ .multipart(form);
- // Build request with optional authentication
- let mut request = client
- .post(format!("{}/v1/audio/transcriptions", endpoint))
- .multipart(form);
+ if let Some(key) = api_key {
+ if !key.is_empty() {
+ request = request.bearer_auth(key);
+ }
+ }
- if let Some(key) = api_key {
- if !key.is_empty() {
- request = request.bearer_auth(key);
+ if let Some(headers) = custom_headers {
+ for (name, value) in headers {
+ if let (Ok(header_name), Ok(header_value)) = (
+ reqwest::header::HeaderName::from_bytes(name.as_bytes()),
+ reqwest::header::HeaderValue::from_str(value),
+ ) {
+ request = request.header(header_name, header_value);
+ }
+ }
}
- }
- // Add custom headers
- if let Some(headers) = custom_headers {
- for (name, value) in headers {
- if let (Ok(header_name), Ok(header_value)) = (
- reqwest::header::HeaderName::from_bytes(name.as_bytes()),
- reqwest::header::HeaderValue::from_str(value),
- ) {
- request = request.header(header_name, header_value);
+ match request.send().await {
+ Ok(response) => return handle_response(response, device).await,
+ Err(e) => {
+ if !openai_error_is_transient(&e) || attempt + 1 == MAX_ATTEMPTS {
+ return Err(e.into());
+ }
+ let delay = std::time::Duration::from_millis(300 * 2u64.pow(attempt));
+ debug!(
+ "openai-compatible request failed (attempt {}/{}): {} — retrying in {:?}",
+ attempt + 1,
+ MAX_ATTEMPTS,
+ e,
+ delay
+ );
+ last_err = Some(e);
+ tokio::time::sleep(delay).await;
}
}
}
+ Err(last_err
+ .expect("retry loop ran with at least one attempt")
+ .into())
+}
- // Send request
- let response = request.send().await?;
+/// Whether a transport-level error is transient (worth retrying) rather than
+/// deterministic. HTTP status errors don't reach here (they're `Ok(Response)`).
+fn openai_error_is_transient(err: &reqwest::Error) -> bool {
+ err.is_timeout() || err.is_connect() || transient_error_text(&format!("{err:?}"))
+}
- handle_response(response, device).await
+/// String-level fallback for transient transport failures reqwest doesn't flag
+/// via `is_timeout`/`is_connect`.
+fn transient_error_text(debug: &str) -> bool {
+ let d = debug.to_lowercase();
+ d.contains("timed out")
+ || d.contains("timeout")
+ || d.contains("connection reset")
+ || d.contains("connection closed")
+ || d.contains("broken pipe")
+ || d.contains("connection refused")
+ || d.contains("error sending request")
}
/// Create WAV data from f32 audio samples (uncompressed PCM).
@@ -297,6 +334,17 @@ async fn handle_response(response: Response, device: &str) -> Result<String> {
mod tests {
use super::*;
+ #[test]
+ fn transient_error_text_flags_blips_only() {
+ assert!(transient_error_text(
+ "reqwest::Error { kind: Request, source: error sending request }"
+ ));
+ assert!(transient_error_text("operation timed out"));
+ assert!(transient_error_text("Connection refused (os error 61)"));
+ assert!(!transient_error_text("invalid api key"));
+ assert!(!transient_error_text("404 not found"));
+ }
+
#[test]
fn test_create_mp3_data() {
let audio_data: Vec<f32> = vec![0.0, 0.5, 0.0, -0.5, 0.0];
diff --git a/crates/screenpipe-audio/src/transcription/whisper/batch.rs b/crates/screenpipe-audio/src/transcription/whisper/batch.rs
index be59def11..88575c6eb 100644
--- a/crates/screenpipe-audio/src/transcription/whisper/batch.rs
+++ b/crates/screenpipe-audio/src/transcription/whisper/batch.rs
@@ -16,6 +16,33 @@ use whisper_rs::{FullParams, SamplingStrategy, WhisperState};
/// white noise at 0.1 amplitude (RMS~0.071), normal speech (RMS~0.05-0.3).
const MIN_RMS_ENERGY: f32 = 0.015;
+/// Char budget for the Whisper initial_prompt — roughly Whisper's ~224-token
+/// prompt window. Entries past this are dropped (whole entries only).
+const INITIAL_PROMPT_CHAR_BUDGET: usize = 800;
+
+/// Build the Whisper initial_prompt from vocabulary, capped to `budget` chars by
+/// adding *whole* comma-joined entries. Never byte-slices a joined string: a cut
+/// at a non-char boundary panics, and a unicode term (e.g. an accented attendee
+/// name now seeded as a keyterm) can land exactly there.
+fn build_initial_prompt(vocabulary: &[VocabularyEntry], budget: usize) -> String {
+ let mut prompt = String::new();
+ let mut char_len = 0usize;
+ for entry in vocabulary {
+ let term = entry.replacement.as_deref().unwrap_or(&entry.word);
+ let sep = if prompt.is_empty() { 0 } else { 2 }; // ", "
+ let term_len = term.chars().count();
+ if char_len + sep + term_len > budget {
+ break;
+ }
+ if !prompt.is_empty() {
+ prompt.push_str(", ");
+ }
+ prompt.push_str(term);
+ char_len += sep + term_len;
+ }
+ prompt
+}
+
/// Processes audio data using the Whisper model to generate transcriptions.
///
/// # Returns
@@ -71,21 +98,13 @@ pub async fn process_with_whisper(
params.set_debug_mode(false);
params.set_translate(false);
- // Set initial_prompt from vocabulary to bias Whisper toward custom words
+ // Set initial_prompt from vocabulary to bias Whisper toward custom words.
if !vocabulary.is_empty() {
- let prompt: String = vocabulary
- .iter()
- .map(|v| v.replacement.as_deref().unwrap_or(&v.word))
- .collect::<Vec<_>>()
- .join(", ");
- // Truncate to ~800 chars to stay within Whisper's 224 token limit
- let prompt = if prompt.len() > 800 {
- &prompt[..800]
- } else {
- &prompt
- };
- debug!("whisper initial_prompt: {}", prompt);
- params.set_initial_prompt(prompt);
+ let prompt = build_initial_prompt(vocabulary, INITIAL_PROMPT_CHAR_BUDGET);
+ if !prompt.is_empty() {
+ debug!("whisper initial_prompt: {}", prompt);
+ params.set_initial_prompt(prompt.as_str());
+ }
}
whisper_state
@@ -106,3 +125,44 @@ pub async fn process_with_whisper(
Ok(transcript)
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn vocab(words: &[&str]) -> Vec<VocabularyEntry> {
+ words
+ .iter()
+ .map(|w| VocabularyEntry {
+ word: w.to_string(),
+ replacement: None,
+ })
+ .collect()
+ }
+
+ #[test]
+ fn joins_entries_within_budget() {
+ let p = build_initial_prompt(&vocab(&["Arvind", "Vercel", "Screenpipe"]), 800);
+ assert_eq!(p, "Arvind, Vercel, Screenpipe");
+ assert!(build_initial_prompt(&[], 800).is_empty());
+ }
+
+ #[test]
+ fn drops_whole_entries_past_budget() {
+ // budget 10: "Arvind" (6) fits; ", Vercel" would be 14 > 10 → dropped.
+ assert_eq!(
+ build_initial_prompt(&vocab(&["Arvind", "Vercel"]), 10),
+ "Arvind"
+ );
+ }
+
+ #[test]
+ fn never_panics_on_unicode_at_the_boundary() {
+ // Accented multibyte names near the budget must not byte-slice/panic.
+ let names = vocab(&["André", "Müller", "Søren", "naïve", "Zoë"]);
+ for budget in 0..40 {
+ let p = build_initial_prompt(&names, budget);
+ assert!(p.chars().count() <= budget.max(0));
+ }
+ }
+}