YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(audio): retry rate-limited OpenAI-compatible transcription (#7242)
Ezra Ellette6661cc543625
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rsindex b4cfb92de..da6781d32 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs@@ -1554,13 +1554,34 @@ mod tests {#[tokio::test]async fn transcription_gateway_failure_reaches_support_after_log_rotation() {+ assert_transcription_failure_reaches_support_after_log_rotation(+ 504,+ "upstream inference timed out",+ 1,+ )+ .await;+ }++ #[tokio::test]+ async fn transcription_rate_limit_reaches_support_after_log_rotation() {+ assert_transcription_failure_reaches_support_after_log_rotation(+ 429,+ "queue full or wait deadline; retry later",+ 3,+ )+ .await;+ }++ async fn assert_transcription_failure_reaches_support_after_log_rotation(+ status: u16,+ cause: &str,+ attempts: u64,+ ) {let server = MockServer::start().await;Mock::given(method("POST")).and(path("/v1/audio/transcriptions"))- .respond_with(- ResponseTemplate::new(504).set_body_string("upstream inference timed out"),- )- .expect(1)+ .respond_with(ResponseTemplate::new(status).set_body_string(cause))+ .expect(attempts).mount(&server).await;let failure = screenpipe_audio::transcription::openai_compatible::batch::transcribe_with_openai_compatible(@@ -1623,8 +1644,12 @@ mod tests {"{report}");assert!(report.contains("audio=120s, timeout=120s"), "{report}");- assert!(report.contains("504 Gateway Timeout"), "{report}");- assert!(report.contains("upstream inference timed out"), "{report}");+ assert!(+ report.contains(&reqwest::StatusCode::from_u16(status).unwrap().to_string()),+ "{report}"+ );+ assert!(report.contains(cause), "{report}");+ assert!(report.contains(&format!("attempt={attempts}")), "{report}");assert!(!report.contains("hunter2"));}diff --git a/crates/screenpipe-audio/src/transcription/openai_compatible/batch.rs b/crates/screenpipe-audio/src/transcription/openai_compatible/batch.rsindex 1a9372ddf..de0fa1554 100644--- a/crates/screenpipe-audio/src/transcription/openai_compatible/batch.rs+++ b/crates/screenpipe-audio/src/transcription/openai_compatible/batch.rs@@ -10,7 +10,8 @@ use serde_json::Value;use std::collections::HashMap;use std::mem::MaybeUninit;use std::sync::Arc;-use tracing::{debug, error, info};+use std::time::Duration;+use tracing::{debug, error, info, warn};use crate::transcription::stt::OPENAI_COMPATIBLE_TIMEOUT_SECS;@@ -76,13 +77,10 @@ pub async fn transcribe_with_openai_compatible(.build()?,),};- // Send with bounded retry on transient transport failures (timeouts,- // connection resets, "error sending request" blips — the recurring- // openai-compatible failures in Sentry, often a local server briefly- // unavailable). The multipart form can't be reused across attempts, so- // rebuild it each time; audio bytes are cloned only on a retry. HTTP status- // errors come back as Ok(Response) and are handled by handle_response, so a- // reqwest::Error here is always transport-level.+ // Retry transient transport failures and HTTP 429 within the same bounded+ // attempt budget. Rebuild the multipart form for each upload. Rate limits+ // use exponential backoff and respect Retry-After; other HTTP failures+ // still go directly to handle_response.const MAX_ATTEMPTS: u32 = 3;let mut last_err: Option<reqwest::Error> = None;for attempt in 0..MAX_ATTEMPTS {@@ -131,11 +129,25 @@ pub async fn transcribe_with_openai_compatible(match request.send().await {Ok(response) => {+ if response.status() == reqwest::StatusCode::TOO_MANY_REQUESTS+ && attempt + 1 < MAX_ATTEMPTS+ {+ if let Some(delay) = rate_limit_retry_delay(response.headers(), attempt) {+ let body = response.text().await.unwrap_or_default();+ warn!(+ "device: {}, openai-compatible transcription rate limited (HTTP 429, attempt {}/{}): {} — retrying in {:?}",+ device, attempt + 1, MAX_ATTEMPTS, body, delay+ );+ tokio::time::sleep(delay).await;+ continue;+ }+ }return handle_response(response, device).await.map_err(|e| {anyhow::anyhow!(- "OpenAI-compatible transcription failed (audio={}s, timeout={}s): {:#}",+ "OpenAI-compatible transcription failed (audio={}s, timeout={}s, attempt={}): {:#}",audio_duration_secs,timeout_secs,+ attempt + 1,e)});@@ -169,6 +181,35 @@ pub async fn transcribe_with_openai_compatible(.into())}+/// Wait at least 1s, then 2s, or the provider's Retry-After if longer. A hint+/// beyond one minute ends this request's retries instead of parking the worker+/// indefinitely or retrying sooner than the provider permits.+fn rate_limit_retry_delay(headers: &reqwest::header::HeaderMap, attempt: u32) -> Option<Duration> {+ let backoff = Duration::from_secs(2u64.pow(attempt));+ let retry_after = headers+ .get(reqwest::header::RETRY_AFTER)+ .and_then(|value| value.to_str().ok())+ .and_then(|value| {+ let value = value.trim();+ value+ .parse::<u64>()+ .ok()+ .map(Duration::from_secs)+ .or_else(|| {+ chrono::DateTime::parse_from_rfc2822(value)+ .ok()+ .map(|date| {+ (date.with_timezone(&chrono::Utc) - chrono::Utc::now())+ .to_std()+ .unwrap_or_default()+ })+ })+ })+ .unwrap_or_default();+ let delay = backoff.max(retry_after);+ (delay <= Duration::from_secs(60)).then_some(delay)+}+/// Build the OpenAI-compatible transcription URL from a user-provided base/// endpoint. Keeping this in the shared request path means the settings/// verification request and production transcription use the exact same URL.@@ -365,12 +406,211 @@ async fn handle_response(response: Response, device: &str) -> Result<String> {#[cfg(test)]mod tests {use super::*;+ use std::sync::Mutex;use std::time::Duration;+ use tracing::instrument::WithSubscriber;use wiremock::{matchers::{method, path},Mock, MockServer, ResponseTemplate,};+ #[tokio::test]+ async fn rate_limits_back_off_then_recover_without_error_logs() {+ let server = MockServer::start().await;+ let arrivals = Arc::new(Mutex::new(Vec::new()));+ let observed = arrivals.clone();+ Mock::given(method("POST"))+ .and(path("/v1/audio/transcriptions"))+ .respond_with(move |_: &wiremock::Request| {+ let mut arrivals = observed.lock().unwrap();+ arrivals.push(std::time::Instant::now());+ if arrivals.len() < 3 {+ ResponseTemplate::new(429).set_body_json(serde_json::json!({+ "error": "queue full or wait deadline; retry later"+ }))+ } else {+ ResponseTemplate::new(200)+ .set_body_json(serde_json::json!({"text": "recovered transcript"}))+ }+ })+ .expect(3)+ .mount(&server)+ .await;+ let log = tempfile::NamedTempFile::new().unwrap();+ let subscriber = tracing_subscriber::fmt()+ .with_ansi(false)+ .without_time()+ .with_writer(log.reopen().unwrap())+ .finish();+ let result = transcribe_with_openai_compatible(+ None,+ &server.uri(),+ Some("test-key"),+ "whisper-1",+ &[0.0; 160],+ "test",+ 16000,+ vec![],+ &["screenpipe".into()],+ None,+ true,+ )+ .with_subscriber(subscriber)+ .await+ .unwrap();+ assert_eq!(result, "recovered transcript");+ {+ let arrivals = arrivals.lock().unwrap();+ assert!(arrivals[1].duration_since(arrivals[0]) >= Duration::from_secs(1));+ assert!(arrivals[2].duration_since(arrivals[1]) >= Duration::from_secs(2));+ }+ let logs = std::fs::read_to_string(log.path()).unwrap();+ assert_eq!(logs.matches("WARN").count(), 2, "{logs}");+ assert!(+ !logs.contains("ERROR"),+ "a recovered 429 must not emit a Sentry error: {logs}"+ );+ for request in server.received_requests().await.unwrap() {+ assert_eq!(request.headers["authorization"], "Bearer test-key");+ let body = String::from_utf8_lossy(&request.body);+ for expected in ["whisper-1", "screenpipe", "audio.wav", "RIFF"] {+ assert!(body.contains(expected), "retry lost {expected}");+ }+ }+ }++ #[tokio::test]+ async fn rate_limit_retry_after_is_a_minimum_wait() {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .respond_with(ResponseTemplate::new(429).insert_header("Retry-After", "2"))+ .up_to_n_times(1)+ .expect(1)+ .with_priority(1)+ .mount(&server)+ .await;+ Mock::given(method("POST"))+ .respond_with(+ ResponseTemplate::new(200).set_body_json(serde_json::json!({"text": "ok"})),+ )+ .expect(1)+ .with_priority(2)+ .mount(&server)+ .await;+ let start = std::time::Instant::now();+ let result = transcribe_with_openai_compatible(+ None,+ &server.uri(),+ None,+ "whisper-1",+ &[0.0; 160],+ "test",+ 16000,+ vec![],+ &[],+ None,+ false,+ )+ .await+ .unwrap();+ assert_eq!(result, "ok");+ assert!(start.elapsed() >= Duration::from_secs(2));+ }++ #[tokio::test]+ async fn persistent_rate_limit_keeps_the_cause_after_three_attempts() {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .respond_with(ResponseTemplate::new(429).set_body_json(serde_json::json!({+ "error": "queue full or wait deadline; retry later"+ })))+ .expect(3)+ .mount(&server)+ .await;+ let error = transcribe_with_openai_compatible(+ None,+ &server.uri(),+ None,+ "whisper-1",+ &[0.0; 160],+ "test",+ 16000,+ vec![],+ &[],+ None,+ true,+ )+ .await+ .unwrap_err()+ .to_string();+ for expected in [+ "429 Too Many Requests",+ "queue full or wait deadline",+ "attempt=3",+ "audio=1s, timeout=30s",+ ] {+ assert!(error.contains(expected), "missing {expected}: {error}");+ }+ }++ #[tokio::test]+ async fn other_http_failures_and_long_cooldowns_do_not_retry() {+ for status in [400, 401, 403, 404, 429, 500, 504] {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .respond_with(+ ResponseTemplate::new(status)+ .insert_header("Retry-After", "3600")+ .set_body_string("provider rejection"),+ )+ .expect(1)+ .mount(&server)+ .await;+ let error = tokio::time::timeout(+ Duration::from_secs(5),+ transcribe_with_openai_compatible(+ None,+ &server.uri(),+ None,+ "whisper-1",+ &[0.0; 160],+ "test",+ 16000,+ vec![],+ &[],+ None,+ true,+ ),+ )+ .await+ .expect("must not wait out a long provider cooldown")+ .unwrap_err()+ .to_string();+ assert!(error.contains(&status.to_string()), "{error}");+ assert!(error.contains("provider rejection"), "{error}");+ assert!(error.contains("attempt=1"), "{error}");+ }+ }++ #[test]+ fn rate_limit_retry_after_handles_dates_and_invalid_hints() {+ let mut headers = reqwest::header::HeaderMap::new();+ for value in ["garbage", "-1", "NaN", "0", "Wed, 21 Oct 2015 07:28:00 GMT"] {+ headers.insert(reqwest::header::RETRY_AFTER, value.parse().unwrap());+ assert_eq!(+ rate_limit_retry_delay(&headers, 1),+ Some(Duration::from_secs(2))+ );+ }+ let date = (chrono::Utc::now() + chrono::Duration::seconds(30)).to_rfc2822();+ headers.insert(reqwest::header::RETRY_AFTER, date.parse().unwrap());+ let delay = rate_limit_retry_delay(&headers, 0).unwrap();+ assert!((Duration::from_secs(28)..=Duration::from_secs(30)).contains(&delay));+ let date = (chrono::Utc::now() + chrono::Duration::hours(1)).to_rfc2822();+ headers.insert(reqwest::header::RETRY_AFTER, date.parse().unwrap());+ assert_eq!(rate_limit_retry_delay(&headers, 0), None);+ }+#[tokio::test]async fn long_audio_can_finish_after_the_old_client_deadline() {let server = MockServer::start().await;