Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(health): stop false audio stall/503 during post-meeting transcription catch-up (#5011)

Ansh Groverfffa84efc850
diff --git a/crates/screenpipe-engine/src/routes/health.rs b/crates/screenpipe-engine/src/routes/health.rs
index da9e35ed1..f21f5be1d 100644
--- a/crates/screenpipe-engine/src/routes/health.rs
+++ b/crates/screenpipe-engine/src/routes/health.rs
@@ -41,16 +41,38 @@ const AUDIO_RECONCILIATION_LOOKBACK_HOURS: i64 = 24 * 7;
const AUDIO_RECONCILIATION_FRESHNESS_DELAY_SECS: i64 = 10 * 60;
const AUDIO_RECONCILIATION_BACKLOG_CACHE_TTL_SECS: i64 = 30;
+/// How long the transcription pipeline may go without landing a single
+/// successful write before a deep+old backlog counts as a real stall.
+///
+/// Batch mode intentionally parks audio during a live session, then drains it
+/// with a reconciliation sweep that runs every 120s (see the sweep loop in
+/// `audio_manager::manager`). During that normal post-meeting catch-up the
+/// sweep writes a row per chunk, so `last_db_write_ts` advances at least once
+/// per sweep. This threshold must therefore sit comfortably above one sweep
+/// interval so a healthy-but-catching-up queue never trips the alarm; only a
+/// genuinely wedged sweep (or an engine that is up but writing nothing) goes
+/// this long with zero progress.
+const AUDIO_TRANSCRIPTION_NO_PROGRESS_SECS: u64 = 5 * 60;
+
/// Decide whether the audio transcription backlog should be flagged as a real
-/// stall. Real stall = the reconciliation worker has fallen behind. A "stall"
-/// is intentionally NOT flagged when batch mode is parking the queue while a
-/// live audio session owns the engine — that's expected, not broken.
+/// stall. Real stall = the reconciliation worker has fallen behind AND is not
+/// making progress. Two things are intentionally NOT flagged:
+///
+/// 1. Batch mode parking the queue while a live audio session owns the engine
+/// (`intentionally_deferring`) — expected, not broken.
+/// 2. Normal post-meeting catch-up, where a deep backlog exists but the sweep
+/// is actively draining it (`last_db_write_ts` is fresh). The meeting flag
+/// flips off the instant a call ends, well before the several-minute drain
+/// completes, so a purely state-based check (old backlog + no live meeting)
+/// false-fires a 503 after every meeting. Gating on *progress* is what
+/// distinguishes "catching up" from "stuck".
///
/// Returning `false` here is what makes the difference between the user
-/// seeing a calm "ok" response and a misleading 503/degraded mid-meeting.
+/// seeing a calm "ok" response and a misleading 503/degraded after a meeting.
fn audio_backlog_is_stalled(
pending_count: u64,
oldest_pending_age_secs: u64,
+ transcription_progress_age_secs: u64,
intentionally_deferring: bool,
) -> bool {
if intentionally_deferring {
@@ -61,9 +83,14 @@ fn audio_backlog_is_stalled(
// only when there's a real backlog AND the oldest chunk has been waiting
// noticeably longer than the freshness delay (>2x = should have been
// picked up by the last sweep).
- pending_count > 20
+ let backlog_deep_and_old = pending_count > 20
&& oldest_pending_age_secs
- > (AUDIO_RECONCILIATION_FRESHNESS_DELAY_SECS as u64).saturating_mul(2)
+ > (AUDIO_RECONCILIATION_FRESHNESS_DELAY_SECS as u64).saturating_mul(2);
+ // ...AND the pipeline is not draining it. During normal catch-up the sweep
+ // lands a write every <=120s, keeping this age small; a wedged sweep or a
+ // silently-not-writing engine lets it grow past the no-progress window.
+ let no_recent_progress = transcription_progress_age_secs > AUDIO_TRANSCRIPTION_NO_PROGRESS_SECS;
+ backlog_deep_and_old && no_recent_progress
}
/// Describe the most likely cause of a DB-write stall from pool stats.
@@ -800,19 +827,30 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
// behavior, not a real stall.
//
// A real stall now means: the reconciliation worker has pending
- // chunks older than the freshness window — i.e. they should have
- // been processed by now and haven't. The `intentionally_deferring`
- // gate (handled by audio_backlog_is_stalled) prevents the same
- // false-positive class during a live audio session.
+ // chunks older than the freshness window AND isn't draining them. The
+ // `intentionally_deferring` gate (handled by audio_backlog_is_stalled)
+ // suppresses the false positive during a live audio session, and the
+ // progress-age gate suppresses it during normal post-meeting catch-up
+ // (a deep backlog that the sweep is actively working through).
let backlog = audio_reconciliation_backlog.unwrap_or((0, None));
let pending_count = backlog.0;
let oldest_pending_age_secs = backlog
.1
.map(|ts| (now.timestamp() - ts.timestamp()).max(0) as u64)
.unwrap_or(0);
+ // Seconds since the last successful transcription write (live or
+ // reconciliation — both call `record_db_insert`). `last_db_write_ts==0`
+ // means nothing has ever landed, which with a deep old backlog is a
+ // genuine stall, so treat "never" as maximally stale.
+ let transcription_progress_age_secs = if audio_snap.last_db_write_ts == 0 {
+ u64::MAX
+ } else {
+ now_ts.saturating_sub(audio_snap.last_db_write_ts)
+ };
let stalled = audio_backlog_is_stalled(
pending_count,
oldest_pending_age_secs,
+ transcription_progress_age_secs,
intentionally_deferring,
);
if stalled {
@@ -821,12 +859,14 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
let prev = LAST_AUDIO_STALL_LOG.load(Ordering::Relaxed);
if now_ts.saturating_sub(prev) >= 60 {
LAST_AUDIO_STALL_LOG.store(now_ts, Ordering::Relaxed);
- let (rs, ri, ws, wi) = state.db.pool_stats();
+ // Report the transcription-progress age (the signal this stall
+ // is actually based on), NOT the SQLite pool stats — the pool is
+ // unrelated to the reconciliation sweep and only misleads triage.
warn!(
- "health_check: audio transcription backlog stalled — {} chunk(s) pending, oldest {}s old | pool: read={}/{} idle, write={}/{} idle",
+ "health_check: audio transcription backlog stalled — {} chunk(s) pending, oldest {}s old, no successful transcription in {}s (reconciliation sweep not draining the queue)",
pending_count,
oldest_pending_age_secs,
- ri, rs, wi, ws,
+ transcription_progress_age_secs,
);
}
}
@@ -988,24 +1028,26 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
// Audio degradation: chunks_channel_full > 0 means the Whisper consumer
// couldn't keep up and audio was dropped even after a 30s backpressure wait.
- // A reconciliation backlog means audio exists but transcript has not landed
- // yet, which should be visible instead of reported as healthy — *unless*
- // the backlog is the expected result of batch mode deferring during a
- // live session, in which case it's not a problem to surface.
+ //
+ // A pending reconciliation backlog on its own does NOT mean degraded: batch
+ // mode builds a backlog during every meeting and drains it over the next few
+ // minutes, and the meeting flag clears the instant a call ends (before the
+ // drain finishes). Flagging any non-empty backlog therefore 503'd after
+ // every meeting. `audio_db_write_stalled` already captures the genuine case
+ // — deep + old + not draining — so a healthy catch-up stays 200 and only a
+ // truly stuck queue is surfaced as degraded.
let audio_degraded = if !state.audio_disabled
&& !transcription_engine_disabled
&& audio_snap.uptime_secs > 120.0
{
let channel_full = audio_snap.chunks_channel_full > 0;
- let transcription_backlog =
- pending_transcription_segments.is_some() && !intentionally_deferring;
if channel_full {
warn!(
"health_check: {} audio chunk(s) dropped (transcription engine too slow)",
audio_snap.chunks_channel_full
);
}
- channel_full || audio_db_write_stalled || transcription_backlog
+ channel_full || audio_db_write_stalled
} else {
false
};
@@ -1588,21 +1630,45 @@ mod tests {
fn audio_backlog_stall_gate() {
let freshness = AUDIO_RECONCILIATION_FRESHNESS_DELAY_SECS as u64;
let way_past = freshness * 3;
+ let no_progress = AUDIO_TRANSCRIPTION_NO_PROGRESS_SECS + 60; // past the window
+ let progressing = 30; // a write landed 30s ago — sweep is draining
- // Real stall: big backlog, old, no live session — must flag.
- assert!(audio_backlog_is_stalled(200, way_past, false));
+ // Real stall: big backlog, old, no live session, and NOT draining
+ // (no successful transcription for longer than the no-progress window).
+ assert!(audio_backlog_is_stalled(200, way_past, no_progress, false));
- // Same numbers but live session in flight — must NOT flag.
- // (Mid-meeting false-positive: batch mode parks the queue while live
- // transcription owns the engine, and the old check misread that as
- // a broken pipeline.)
- assert!(!audio_backlog_is_stalled(200, way_past, true));
+ // Same deep+old backlog but the sweep IS draining it (fresh write) —
+ // this is normal post-meeting catch-up, must NOT flag. This is the
+ // regression this fix targets: the meeting flag clears the instant a
+ // call ends, so the old state-based check 503'd during every drain.
+ assert!(!audio_backlog_is_stalled(200, way_past, progressing, false));
- // Small backlog within the freshness window — never a stall.
- assert!(!audio_backlog_is_stalled(5, freshness / 2, false));
+ // Deep+old+not-draining but a live session is in flight — must NOT flag.
+ // (Mid-meeting false-positive: batch mode parks the queue while live
+ // transcription owns the engine.)
+ assert!(!audio_backlog_is_stalled(200, way_past, no_progress, true));
+
+ // Small backlog within the freshness window — never a stall, even if no
+ // recent write (10 min of in-flight audio is expected).
+ assert!(!audio_backlog_is_stalled(
+ 5,
+ freshness / 2,
+ no_progress,
+ false
+ ));
// Big count but young enough — not a stall yet.
- assert!(!audio_backlog_is_stalled(200, freshness, false));
+ assert!(!audio_backlog_is_stalled(
+ 200,
+ freshness,
+ no_progress,
+ false
+ ));
+
+ // Never-written engine (last_db_write_ts == 0 → u64::MAX age) with a
+ // deep old backlog IS a genuine stall (e.g. engine came up but writes
+ // nothing).
+ assert!(audio_backlog_is_stalled(200, way_past, u64::MAX, false));
}
/// Healthy, actively-capturing mic with no recent timeout, varying only the