YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(audio): transcription resilience + correctness (2/n) — whisper panic, deepgram keyterm, openai retry (#4236)
Louis Beaumont5c663485b63c
diff --git a/crates/screenpipe-audio/src/transcription/deepgram/batch.rs b/crates/screenpipe-audio/src/transcription/deepgram/batch.rsindex 10814278b..c95b8bbbe 100644--- a/crates/screenpipe-audio/src/transcription/deepgram/batch.rs+++ b/crates/screenpipe-audio/src/transcription/deepgram/batch.rs@@ -145,12 +145,14 @@ fn create_query_params(languages: Vec<Language>, vocabulary: &[VocabularyEntry])}}- // Add vocabulary as Deepgram keyterms (Nova-3 uses `keyterm` instead of `keywords`)+ // Add vocabulary as Deepgram nova-3 keyterms. nova-3 keyterm prompting takes+ // plain terms with NO `:intensifier` (that's the older nova-2 `keywords`+ // syntax) — a trailing `:2` would be sent as part of the literal term.for entry in vocabulary.iter().take(100) {let keyword = entry.replacement.as_deref().unwrap_or(&entry.word);- // Simple percent-encode spaces for the query string- let encoded = keyword.replace(' ', "%20");- query_params.push_str(&format!("&keyterm={}:2", encoded));+ // Percent-encode spaces (and the comma, which separates keyterms).+ let encoded = keyword.replace(' ', "%20").replace(',', "%2C");+ query_params.push_str(&format!("&keyterm={}", encoded));}query_params@@ -669,6 +671,25 @@ mod tests {assert!(!params.contains("detect_language"));}+ #[test]+ fn vocabulary_becomes_plain_keyterms_without_intensifier() {+ let vocab = vec![+ VocabularyEntry {+ word: "Screenpipe".into(),+ replacement: None,+ },+ VocabularyEntry {+ word: "Core Audio".into(),+ replacement: None,+ },+ ];+ let params = create_query_params(vec![], &vocab);+ assert!(params.contains("&keyterm=Screenpipe"), "got: {params}");+ // nova-3 keyterms take no `:intensifier` (that's nova-2 keywords).+ assert!(!params.contains(":2"), "got: {params}");+ assert!(params.contains("&keyterm=Core%20Audio"), "got: {params}");+ }+#[test]fn transient_error_text_flags_network_blips_only() {// retryable transport failuresdiff --git a/crates/screenpipe-audio/src/transcription/openai_compatible/batch.rs b/crates/screenpipe-audio/src/transcription/openai_compatible/batch.rsindex e45a7a93a..e89f10c6b 100644--- a/crates/screenpipe-audio/src/transcription/openai_compatible/batch.rs+++ b/crates/screenpipe-audio/src/transcription/openai_compatible/batch.rs@@ -71,62 +71,99 @@ pub async fn transcribe_with_openai_compatible(.build()?,),};- // Build multipart form- let mut form = multipart::Form::new()- .text("model", model.to_string())- .text("response_format", "json".to_string())- .part(- "file",- multipart::Part::bytes(audio_bytes)- .file_name(file_name.to_string())- .mime_str(mime_type)?,- );+ // Send with bounded retry on transient transport failures (timeouts,+ // connection resets, "error sending request" blips — the recurring+ // openai-compatible failures in Sentry, often a local server briefly+ // unavailable). The multipart form can't be reused across attempts, so+ // rebuild it each time; audio bytes are cloned only on a retry. HTTP status+ // errors come back as Ok(Response) and are handled by handle_response, so a+ // reqwest::Error here is always transport-level.+ const MAX_ATTEMPTS: u32 = 3;+ let mut last_err: Option<reqwest::Error> = None;+ for attempt in 0..MAX_ATTEMPTS {+ let mut form = multipart::Form::new()+ .text("model", model.to_string())+ .text("response_format", "json".to_string())+ .part(+ "file",+ multipart::Part::bytes(audio_bytes.clone())+ .file_name(file_name.to_string())+ .mime_str(mime_type)?,+ );++ if !languages.is_empty() {+ form = form.text("language", languages[0].as_lang_code().to_string());+ }- // Add language if specified- if !languages.is_empty() {- // Use the first language as the primary language hint- let lang_code = languages[0].as_lang_code();- form = form.text("language", lang_code.to_string());- }+ // Vocabulary/hotwords for biasing: `prompt` = OpenAI initial_prompt,+ // `context` = mlx-audio hotwords. Send both so it works on either server.+ if !vocabulary.is_empty() {+ let prompt = vocabulary.join(", ");+ form = form.text("prompt", prompt.clone()).text("context", prompt);+ }- // Pass vocabulary/hotwords for transcription biasing.- // - OpenAI Whisper API: uses `prompt` as initial_prompt- // - mlx-audio (VibeVoice-ASR): uses `context` for hotwords (ignores `prompt`)- // Send both fields so it works regardless of which server is running.- if !vocabulary.is_empty() {- let prompt = vocabulary.join(", ");- debug!("passing vocabulary as prompt + context: {}", prompt);- form = form.text("prompt", prompt.clone());- form = form.text("context", prompt);- }+ let mut request = client+ .post(format!("{}/v1/audio/transcriptions", endpoint))+ .multipart(form);- // Build request with optional authentication- let mut request = client- .post(format!("{}/v1/audio/transcriptions", endpoint))- .multipart(form);+ if let Some(key) = api_key {+ if !key.is_empty() {+ request = request.bearer_auth(key);+ }+ }- if let Some(key) = api_key {- if !key.is_empty() {- request = request.bearer_auth(key);+ if let Some(headers) = custom_headers {+ for (name, value) in headers {+ if let (Ok(header_name), Ok(header_value)) = (+ reqwest::header::HeaderName::from_bytes(name.as_bytes()),+ reqwest::header::HeaderValue::from_str(value),+ ) {+ request = request.header(header_name, header_value);+ }+ }}- }- // Add custom headers- if let Some(headers) = custom_headers {- for (name, value) in headers {- if let (Ok(header_name), Ok(header_value)) = (- reqwest::header::HeaderName::from_bytes(name.as_bytes()),- reqwest::header::HeaderValue::from_str(value),- ) {- request = request.header(header_name, header_value);+ match request.send().await {+ Ok(response) => return handle_response(response, device).await,+ Err(e) => {+ if !openai_error_is_transient(&e) || attempt + 1 == MAX_ATTEMPTS {+ return Err(e.into());+ }+ let delay = std::time::Duration::from_millis(300 * 2u64.pow(attempt));+ debug!(+ "openai-compatible request failed (attempt {}/{}): {} — retrying in {:?}",+ attempt + 1,+ MAX_ATTEMPTS,+ e,+ delay+ );+ last_err = Some(e);+ tokio::time::sleep(delay).await;}}}+ Err(last_err+ .expect("retry loop ran with at least one attempt")+ .into())+}- // Send request- let response = request.send().await?;+/// Whether a transport-level error is transient (worth retrying) rather than+/// deterministic. HTTP status errors don't reach here (they're `Ok(Response)`).+fn openai_error_is_transient(err: &reqwest::Error) -> bool {+ err.is_timeout() || err.is_connect() || transient_error_text(&format!("{err:?}"))+}- handle_response(response, device).await+/// String-level fallback for transient transport failures reqwest doesn't flag+/// via `is_timeout`/`is_connect`.+fn transient_error_text(debug: &str) -> bool {+ let d = debug.to_lowercase();+ d.contains("timed out")+ || d.contains("timeout")+ || d.contains("connection reset")+ || d.contains("connection closed")+ || d.contains("broken pipe")+ || d.contains("connection refused")+ || d.contains("error sending request")}/// Create WAV data from f32 audio samples (uncompressed PCM).@@ -297,6 +334,17 @@ async fn handle_response(response: Response, device: &str) -> Result<String> {mod tests {use super::*;+ #[test]+ fn transient_error_text_flags_blips_only() {+ assert!(transient_error_text(+ "reqwest::Error { kind: Request, source: error sending request }"+ ));+ assert!(transient_error_text("operation timed out"));+ assert!(transient_error_text("Connection refused (os error 61)"));+ assert!(!transient_error_text("invalid api key"));+ assert!(!transient_error_text("404 not found"));+ }+#[test]fn test_create_mp3_data() {let audio_data: Vec<f32> = vec![0.0, 0.5, 0.0, -0.5, 0.0];diff --git a/crates/screenpipe-audio/src/transcription/whisper/batch.rs b/crates/screenpipe-audio/src/transcription/whisper/batch.rsindex be59def11..88575c6eb 100644--- a/crates/screenpipe-audio/src/transcription/whisper/batch.rs+++ b/crates/screenpipe-audio/src/transcription/whisper/batch.rs@@ -16,6 +16,33 @@ use whisper_rs::{FullParams, SamplingStrategy, WhisperState};/// white noise at 0.1 amplitude (RMS~0.071), normal speech (RMS~0.05-0.3).const MIN_RMS_ENERGY: f32 = 0.015;+/// Char budget for the Whisper initial_prompt — roughly Whisper's ~224-token+/// prompt window. Entries past this are dropped (whole entries only).+const INITIAL_PROMPT_CHAR_BUDGET: usize = 800;++/// Build the Whisper initial_prompt from vocabulary, capped to `budget` chars by+/// adding *whole* comma-joined entries. Never byte-slices a joined string: a cut+/// at a non-char boundary panics, and a unicode term (e.g. an accented attendee+/// name now seeded as a keyterm) can land exactly there.+fn build_initial_prompt(vocabulary: &[VocabularyEntry], budget: usize) -> String {+ let mut prompt = String::new();+ let mut char_len = 0usize;+ for entry in vocabulary {+ let term = entry.replacement.as_deref().unwrap_or(&entry.word);+ let sep = if prompt.is_empty() { 0 } else { 2 }; // ", "+ let term_len = term.chars().count();+ if char_len + sep + term_len > budget {+ break;+ }+ if !prompt.is_empty() {+ prompt.push_str(", ");+ }+ prompt.push_str(term);+ char_len += sep + term_len;+ }+ prompt+}+/// Processes audio data using the Whisper model to generate transcriptions.////// # Returns@@ -71,21 +98,13 @@ pub async fn process_with_whisper(params.set_debug_mode(false);params.set_translate(false);- // Set initial_prompt from vocabulary to bias Whisper toward custom words+ // Set initial_prompt from vocabulary to bias Whisper toward custom words.if !vocabulary.is_empty() {- let prompt: String = vocabulary- .iter()- .map(|v| v.replacement.as_deref().unwrap_or(&v.word))- .collect::<Vec<_>>()- .join(", ");- // Truncate to ~800 chars to stay within Whisper's 224 token limit- let prompt = if prompt.len() > 800 {- &prompt[..800]- } else {- &prompt- };- debug!("whisper initial_prompt: {}", prompt);- params.set_initial_prompt(prompt);+ let prompt = build_initial_prompt(vocabulary, INITIAL_PROMPT_CHAR_BUDGET);+ if !prompt.is_empty() {+ debug!("whisper initial_prompt: {}", prompt);+ params.set_initial_prompt(prompt.as_str());+ }}whisper_state@@ -106,3 +125,44 @@ pub async fn process_with_whisper(Ok(transcript)}++#[cfg(test)]+mod tests {+ use super::*;++ fn vocab(words: &[&str]) -> Vec<VocabularyEntry> {+ words+ .iter()+ .map(|w| VocabularyEntry {+ word: w.to_string(),+ replacement: None,+ })+ .collect()+ }++ #[test]+ fn joins_entries_within_budget() {+ let p = build_initial_prompt(&vocab(&["Arvind", "Vercel", "Screenpipe"]), 800);+ assert_eq!(p, "Arvind, Vercel, Screenpipe");+ assert!(build_initial_prompt(&[], 800).is_empty());+ }++ #[test]+ fn drops_whole_entries_past_budget() {+ // budget 10: "Arvind" (6) fits; ", Vercel" would be 14 > 10 → dropped.+ assert_eq!(+ build_initial_prompt(&vocab(&["Arvind", "Vercel"]), 10),+ "Arvind"+ );+ }++ #[test]+ fn never_panics_on_unicode_at_the_boundary() {+ // Accented multibyte names near the budget must not byte-slice/panic.+ let names = vocab(&["André", "Müller", "Søren", "naïve", "Zoë"]);+ for budget in 0..40 {+ let p = build_initial_prompt(&names, budget);+ assert!(p.chars().count() <= budget.max(0));+ }+ }+}