Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

feat(health): report observed dependency and startup diagnostics (#7261)

Louis Beaumont0eb3a09da538
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/health.rs b/apps/screenpipe-app-tauri/src-tauri/src/health.rs
index b5ba84cf0..6f41c66cd 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/health.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/health.rs
@@ -109,6 +109,7 @@ fn now_epoch() -> u64 {
}
pub fn set_boot_phase(phase: &str, message: Option<&str>) {
+ screenpipe_core::health_diagnostics::startup_phase(phase);
let mut guard = BOOT_PHASE.write().unwrap_or_else(|e| e.into_inner());
// Don't reset since_epoch if the phase is unchanged (no-op writes)
if guard.phase != phase {
@@ -125,6 +126,7 @@ pub fn set_boot_phase(phase: &str, message: Option<&str>) {
}
pub fn set_boot_error(err: &str) {
+ screenpipe_core::health_diagnostics::startup_failure("unknown", err, None);
let mut guard = BOOT_PHASE.write().unwrap_or_else(|e| e.into_inner());
guard.phase = "error".to_string();
guard.error = Some(err.to_string());
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
index 1b6a0ca54..ef8c5c915 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
@@ -352,6 +352,14 @@ impl ServerCore {
workflow_catalog_dir: Option<std::path::PathBuf>,
runtime_lifetime: tokio::sync::oneshot::Sender<()>,
) -> Result<Self, String> {
+ screenpipe_core::health_diagnostics::begin_startup(&config.data_dir);
+ screenpipe_core::health_diagnostics::configure_media_required(
+ !config.disable_audio
+ || !config.disable_snapshot_compaction
+ || (!config.disable_vision
+ && config.hd_recording_default
+ == screenpipe_engine::high_fps_controller::DefaultMode::Always),
+ );
info!("Starting server core on port {}", config.port);
crate::health::set_boot_phase("starting", Some("starting server"));
let ai_gateway_url = crate::config::screenpipe_ai_gateway_url()?;
diff --git a/crates/screenpipe-audio/src/speaker/mod.rs b/crates/screenpipe-audio/src/speaker/mod.rs
index 305d96e1c..ef54decb6 100644
--- a/crates/screenpipe-audio/src/speaker/mod.rs
+++ b/crates/screenpipe-audio/src/speaker/mod.rs
@@ -41,7 +41,11 @@ fn create_session_with_embedding_ep<P: AsRef<Path>>(
// degrades to an error and the caller falls back (diarization off) instead
// of bricking startup. See `utils::ort_watchdog`.
crate::utils::ort_watchdog::run_with_timeout(
- "ort session init",
+ if embedding_ep {
+ "speaker_embedding"
+ } else {
+ "speaker_segmentation"
+ },
crate::utils::ort_watchdog::ORT_INIT_TIMEOUT,
move || {
catch_panic_into_error("ort session init", || {
diff --git a/crates/screenpipe-audio/src/transcription/engine.rs b/crates/screenpipe-audio/src/transcription/engine.rs
index ee25e5e1b..e391370b8 100644
--- a/crates/screenpipe-audio/src/transcription/engine.rs
+++ b/crates/screenpipe-audio/src/transcription/engine.rs
@@ -211,6 +211,52 @@ impl TranscriptionEngine {
openai_compatible_config: Option<crate::transcription::stt::OpenAICompatibleConfig>,
languages: Vec<Language>,
vocabulary: Vec<VocabularyEntry>,
+ ) -> Result<Self> {
+ let requested = config.clone();
+ let model_generation =
+ screenpipe_core::health_diagnostics::begin_transcription_model(&config.to_string());
+ let mut observed_status = "disabled";
+ let mut observed_path = None;
+ let result = Self::new_observed(
+ config,
+ deepgram_config,
+ openai_compatible_config,
+ languages,
+ vocabulary,
+ model_generation,
+ &mut observed_status,
+ &mut observed_path,
+ )
+ .await;
+ let status = match &result {
+ Ok(Self::Disabled) => observed_status,
+ Ok(Self::Deepgram { .. } | Self::OpenAICompatible { .. }) => "not_required",
+ Ok(_) => "ready",
+ Err(_) => "load_failed",
+ };
+ if status != "downloading" {
+ screenpipe_core::health_diagnostics::transcription_model_state(
+ model_generation,
+ status,
+ observed_path,
+ result.as_ref().err().map(ToString::to_string).or_else(|| {
+ (status == "unsupported").then(|| format!("{} requires AVX2", requested))
+ }),
+ );
+ }
+ result
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ async fn new_observed(
+ config: Arc<AudioTranscriptionEngine>,
+ deepgram_config: Option<DeepgramTranscriptionConfig>,
+ openai_compatible_config: Option<crate::transcription::stt::OpenAICompatibleConfig>,
+ languages: Vec<Language>,
+ vocabulary: Vec<VocabularyEntry>,
+ model_generation: u64,
+ observed_status: &mut &'static str,
+ observed_path: &mut Option<std::path::PathBuf>,
) -> Result<Self> {
// Whisper (ggml) and Qwen3 (antirez kernels) are statically compiled
// with AVX2 on Windows/Linux x64 release builds — initializing them on
@@ -235,6 +281,7 @@ impl TranscriptionEngine {
} else {
tracing::debug!("{msg}");
}
+ *observed_status = "unsupported";
return Ok(Self::Disabled);
}
@@ -307,6 +354,13 @@ impl TranscriptionEngine {
warn!(
"qwen3-asr weights not in Hugging Face cache yet; transcription disabled until download completes"
);
+ *observed_status = "downloading";
+ screenpipe_core::health_diagnostics::transcription_model_state(
+ model_generation,
+ "downloading",
+ None,
+ None,
+ );
audiopipe::Model::spawn_pretrained_download(MODEL_NAME.to_string());
Ok(Self::Disabled)
}
@@ -354,6 +408,13 @@ impl TranscriptionEngine {
warn!(
"parakeet-mlx weights not in Hugging Face cache yet; transcription disabled until download completes"
);
+ *observed_status = "downloading";
+ screenpipe_core::health_diagnostics::transcription_model_state(
+ model_generation,
+ "downloading",
+ None,
+ None,
+ );
audiopipe::Model::spawn_pretrained_download(MODEL_NAME.to_string());
Ok(Self::Disabled)
}
@@ -381,6 +442,13 @@ impl TranscriptionEngine {
warn!(
"parakeet weights not in Hugging Face cache yet; transcription disabled until download completes"
);
+ *observed_status = "downloading";
+ screenpipe_core::health_diagnostics::transcription_model_state(
+ model_generation,
+ "downloading",
+ None,
+ None,
+ );
audiopipe::Model::spawn_pretrained_download(MODEL_NAME.to_string());
Ok(Self::Disabled)
}
@@ -424,6 +492,13 @@ impl TranscriptionEngine {
warn!(
"parakeet-mlx weights not in Hugging Face cache yet; transcription disabled until download completes"
);
+ *observed_status = "downloading";
+ screenpipe_core::health_diagnostics::transcription_model_state(
+ model_generation,
+ "downloading",
+ None,
+ None,
+ );
audiopipe::Model::spawn_pretrained_download(MODEL_NAME.to_string());
Ok(Self::Disabled)
}
@@ -448,6 +523,13 @@ impl TranscriptionEngine {
"whisper model is not available locally yet for {:?}; audio transcription disabled until download completes",
config
);
+ *observed_status = "downloading";
+ screenpipe_core::health_diagnostics::transcription_model_state(
+ model_generation,
+ "downloading",
+ None,
+ None,
+ );
let config_for_download = config.clone();
tokio::spawn(async move {
match tokio::task::spawn_blocking(move || {
@@ -456,21 +538,42 @@ impl TranscriptionEngine {
.await
{
Ok(Ok(path)) => {
+ screenpipe_core::health_diagnostics::transcription_model_state(
+ model_generation,
+ "downloaded",
+ Some(path.clone()),
+ None,
+ );
info!("whisper model downloaded in background: {:?}", path)
}
Ok(Err(error)) => {
+ screenpipe_core::health_diagnostics::transcription_model_state(
+ model_generation,
+ "download_failed",
+ None,
+ Some(error.to_string()),
+ );
warn!("whisper background download failed: {}", error)
}
- Err(join_error) => warn!(
- "whisper background download task panicked: {}",
- join_error
- ),
+ Err(join_error) => {
+ screenpipe_core::health_diagnostics::transcription_model_state(
+ model_generation,
+ "download_failed",
+ None,
+ Some(join_error.to_string()),
+ );
+ warn!(
+ "whisper background download task panicked: {}",
+ join_error
+ );
+ }
}
});
return Ok(Self::Disabled);
}
};
+ *observed_path = Some(quantized_path.clone());
info!("whisper model available: {:?}", quantized_path);
let Some(context) =
diff --git a/crates/screenpipe-audio/src/utils/ffmpeg.rs b/crates/screenpipe-audio/src/utils/ffmpeg.rs
index 781ba79d1..c423558ac 100644
--- a/crates/screenpipe-audio/src/utils/ffmpeg.rs
+++ b/crates/screenpipe-audio/src/utils/ffmpeg.rs
@@ -18,6 +18,25 @@ fn encode_single_audio(
channels: u16,
output_path: &Path,
overwrite_reserved_path: bool,
+) -> anyhow::Result<()> {
+ let operation = screenpipe_core::health_diagnostics::MediaOperation::start("audio");
+ let result = encode_single_audio_inner(
+ data,
+ sample_rate,
+ channels,
+ output_path,
+ overwrite_reserved_path,
+ );
+ operation.finish(&result);
+ result
+}
+
+fn encode_single_audio_inner(
+ data: &[u8],
+ sample_rate: u32,
+ channels: u16,
+ output_path: &Path,
+ overwrite_reserved_path: bool,
) -> anyhow::Result<()> {
debug!("Starting FFmpeg process");
@@ -282,6 +301,63 @@ mod path_tests {
use super::{reserve_new_file_path_with_timestamp, write_audio_to_new_file};
use chrono::TimeZone;
+ /// Compare identical real encodes with/without the diagnostics wrapper.
+ /// Synthetic PCM avoids microphone access and private recordings. Alternate
+ /// order to reduce warm-cache/thermal bias; compare decoded PCM, not only
+ /// file existence. This is a local timing sample, not a CI timing gate.
+ #[test]
+ #[ignore = "requires ffmpeg; manual encode/output/performance eval"]
+ fn health_diagnostics_audio_encode_eval() {
+ use super::{encode_single_audio, encode_single_audio_inner};
+ use std::time::Instant;
+ let ffmpeg = screenpipe_core::find_ffmpeg_path().expect("ffmpeg required for eval");
+ let dir = tempfile::tempdir().unwrap();
+ let pcm: Vec<u8> = (0..160_000)
+ .flat_map(|i| {
+ let sample = (i as f32 * 440.0 * std::f32::consts::TAU / 16_000.0).sin() * 0.2;
+ sample.to_le_bytes()
+ })
+ .collect();
+ let mut baseline = vec![];
+ let mut observed = vec![];
+ for round in 0..22 {
+ let mut decoded = vec![];
+ for index in 0..2 {
+ let instrumented = (round + index) % 2 == 0;
+ let path = dir.path().join(format!("{round}-{index}.mp4"));
+ let started = Instant::now();
+ if instrumented {
+ encode_single_audio(&pcm, 16_000, 1, &path, false).unwrap();
+ } else {
+ encode_single_audio_inner(&pcm, 16_000, 1, &path, false).unwrap();
+ }
+ let elapsed = started.elapsed().as_secs_f64() * 1000.0;
+ if round >= 2 {
+ if instrumented {
+ observed.push(elapsed);
+ } else {
+ baseline.push(elapsed);
+ }
+ }
+ let output = screenpipe_core::ffmpeg_cmd(&ffmpeg)
+ .args(["-v", "error", "-i"])
+ .arg(&path)
+ .args(["-f", "f32le", "-acodec", "pcm_f32le", "pipe:1"])
+ .output()
+ .unwrap();
+ assert!(output.status.success());
+ assert!(output.stdout.len() >= pcm.len());
+ decoded.push(output.stdout);
+ }
+ assert_eq!(decoded[0], decoded[1], "diagnostics changed decoded audio");
+ }
+ baseline.sort_by(f64::total_cmp);
+ observed.sort_by(f64::total_cmp);
+ eprintln!("10s PCM, 20 pairs: baseline median_ms={:.3} p95_ms={:.3}; observed median_ms={:.3} p95_ms={:.3}; median_delta_pct={:.2}; all 22 decoded pairs identical",
+ baseline[10], baseline[18], observed[10], observed[18],
+ (observed[10] / baseline[10] - 1.0) * 100.0);
+ }
+
#[test]
fn reserves_distinct_paths_for_same_device_and_second() {
let dir = tempfile::tempdir().expect("tempdir");
diff --git a/crates/screenpipe-audio/src/utils/ort_watchdog.rs b/crates/screenpipe-audio/src/utils/ort_watchdog.rs
index 167eee304..066120ed4 100644
--- a/crates/screenpipe-audio/src/utils/ort_watchdog.rs
+++ b/crates/screenpipe-audio/src/utils/ort_watchdog.rs
@@ -53,15 +53,51 @@ where
})
.map_err(|e| anyhow!("{context}: failed to spawn watchdog thread: {e}"))?;
- match rx.recv_timeout(timeout) {
- Ok(result) => result,
- Err(mpsc::RecvTimeoutError::Timeout) => Err(anyhow!(
+ let (result, status) = match rx.recv_timeout(timeout) {
+ Ok(Ok(value)) => (Ok(value), "ok"),
+ Ok(Err(error)) => (Err(error), "init_failed"),
+ Err(mpsc::RecvTimeoutError::Timeout) => (
+ Err(anyhow!(
"{context}: timed out after {timeout:?} (likely an ONNX Runtime init hang on this host)"
)),
- Err(mpsc::RecvTimeoutError::Disconnected) => Err(anyhow!(
- "{context}: watchdog worker exited without a result"
- )),
+ "init_timed_out",
+ ),
+ Err(mpsc::RecvTimeoutError::Disconnected) => (
+ Err(anyhow!(
+ "{context}: watchdog worker exited without a result"
+ )),
+ "init_failed",
+ ),
+ };
+ let component = if context == "silero vad init" {
+ "silero_vad"
+ } else {
+ context
+ };
+ screenpipe_core::health_diagnostics::component_state(
+ "runtime",
+ component,
+ status,
+ None,
+ result.as_ref().err().map(ToString::to_string),
+ );
+ if matches!(
+ component,
+ "silero_vad" | "speaker_embedding" | "speaker_segmentation"
+ ) {
+ screenpipe_core::health_diagnostics::component_state(
+ "models",
+ if component == "silero_vad" {
+ "vad"
+ } else {
+ component
+ },
+ if status == "ok" { "ready" } else { status },
+ None,
+ result.as_ref().err().map(ToString::to_string),
+ );
}
+ result
}
#[cfg(test)]
diff --git a/crates/screenpipe-core/src/ffmpeg.rs b/crates/screenpipe-core/src/ffmpeg.rs
index de46921de..96c938ebd 100644
--- a/crates/screenpipe-core/src/ffmpeg.rs
+++ b/crates/screenpipe-core/src/ffmpeg.rs
@@ -32,7 +32,7 @@ const EXECUTABLE_NAME: &str = "ffmpeg.exe";
// until the app restarts — the entire session's audio is lost, never persisted.
// See SCREENPIPE-CLI-V8 (337×/day on Windows) and SCREENPIPE-CLI-VC.
static FFMPEG_PATH: Lazy<RwLock<Option<PathBuf>>> =
- Lazy::new(|| RwLock::new(find_ffmpeg_path_internal()));
+ Lazy::new(|| RwLock::new(resolve_observed_ffmpeg()));
// Path discovery remains eager so a manual or bundled install is picked up on
// the next call. Only the network-backed auto-install is rate-limited after a
@@ -95,7 +95,7 @@ pub fn find_ffmpeg_path() -> Option<PathBuf> {
return Some(p.clone());
}
}
- let resolved = find_ffmpeg_path_internal();
+ let resolved = resolve_observed_ffmpeg();
if resolved.is_some() {
debug!("re-resolved ffmpeg path (cache was stale/empty): {resolved:?}");
}
@@ -103,6 +103,98 @@ pub fn find_ffmpeg_path() -> Option<PathBuf> {
resolved
}
+// Observe discovery outside /health. The bounded version probe runs only when
+// the resolved path changes and never delays capture or the resolver.
+fn resolve_observed_ffmpeg() -> Option<PathBuf> {
+ let path = find_ffmpeg_path_internal()?;
+ let probe = [path.with_file_name(if cfg!(windows) {
+ "ffprobe.exe"
+ } else {
+ "ffprobe"
+ })]
+ .into_iter()
+ .find(|p| p.is_file())
+ .or_else(|| which("ffprobe").ok());
+ let source = if std::env::current_exe()
+ .ok()
+ .and_then(|p| p.parent().map(Path::to_path_buf))
+ .is_some_and(|dir| path.starts_with(&dir) || path.starts_with(dir.join("../Resources")))
+ {
+ "bundled"
+ } else if which(EXECUTABLE_NAME).ok().as_ref() == Some(&path) {
+ "path"
+ } else {
+ "sidecar_download"
+ };
+ crate::health_diagnostics::binary_resolved(&path, source, probe);
+ static PROBED: Lazy<Mutex<Option<PathBuf>>> = Lazy::new(|| Mutex::new(None));
+ let mut probed = PROBED.lock().unwrap_or_else(|e| e.into_inner());
+ if probed.as_ref() != Some(&path) {
+ *probed = Some(path.clone());
+ let probe_path = path.clone();
+ let _ = std::thread::Builder::new()
+ .name("ffmpeg-version".into())
+ .spawn(move || {
+ use std::io::Read;
+ use std::process::Stdio;
+ let result = (|| -> std::io::Result<String> {
+ let mut child = ffmpeg_cmd(&probe_path)
+ .arg("-version")
+ .stdout(Stdio::piped())
+ .stderr(Stdio::null())
+ .spawn()?;
+ let deadline = Instant::now() + Duration::from_secs(2);
+ loop {
+ match child.try_wait() {
+ Ok(Some(status)) => {
+ if !status.success() {
+ return Err(std::io::Error::other(format!(
+ "ffmpeg -version exited with {status}"
+ )));
+ }
+ let mut text = String::new();
+ if let Some(stdout) = child.stdout.take() {
+ stdout.take(8192).read_to_string(&mut text)?;
+ }
+ return Ok(text
+ .lines()
+ .next()
+ .unwrap_or_default()
+ .chars()
+ .take(256)
+ .collect());
+ }
+ Ok(None) if Instant::now() < deadline => {
+ std::thread::sleep(Duration::from_millis(20))
+ }
+ other => {
+ let _ = child.kill();
+ let _ = child.wait();
+ return Err(other.err().unwrap_or_else(|| {
+ std::io::Error::new(
+ std::io::ErrorKind::TimedOut,
+ "ffmpeg version probe timed out",
+ )
+ }));
+ }
+ }
+ }
+ })();
+ match result {
+ Ok(version) => {
+ crate::health_diagnostics::binary_version(&probe_path, Some(version), None)
+ }
+ Err(error) => crate::health_diagnostics::binary_version(
+ &probe_path,
+ None,
+ Some(error.to_string()),
+ ),
+ }
+ });
+ }
+ Some(path)
+}
+
/// Create a `std::process::Command` for ffmpeg with `CREATE_NO_WINDOW` on Windows.
pub fn ffmpeg_cmd(path: impl AsRef<std::ffi::OsStr>) -> std::process::Command {
#[cfg(not(windows))]
@@ -366,7 +458,9 @@ fn find_ffmpeg_path_internal() -> Option<PathBuf> {
debug!("ffmpeg not found. installing...");
+ crate::health_diagnostics::binary_installing();
if let Err(error) = handle_ffmpeg_installation() {
+ crate::health_diagnostics::binary_install_failed(&error.to_string(), 60);
ffmpeg_install_retry().record_failure(Instant::now());
error!("failed to install ffmpeg: {}", error);
return None;
@@ -403,6 +497,10 @@ fn find_ffmpeg_path_internal() -> Option<PathBuf> {
}
ffmpeg_install_retry().record_failure(Instant::now());
+ crate::health_diagnostics::binary_install_failed(
+ "ffmpeg not found even after installation",
+ 60,
+ );
error!("ffmpeg not found even after installation");
None // Return None if ffmpeg is not found
}
diff --git a/crates/screenpipe-core/src/health_diagnostics.rs b/crates/screenpipe-core/src/health_diagnostics.rs
new file mode 100644
index 000000000..6ffedd765
--- /dev/null
+++ b/crates/screenpipe-core/src/health_diagnostics.rs
@@ -0,0 +1,974 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+//! Observed dependency and startup state. Snapshot reads never discover binaries,
+//! spawn processes, download models, or touch disk. Instrumentation belongs at
+//! startup and process/model lifecycle boundaries, never in capture callbacks.
+
+use serde::{Deserialize, Serialize};
+use std::collections::{BTreeMap, VecDeque};
+use std::path::{Path, PathBuf};
+use std::sync::{LazyLock, Mutex};
+
+fn now() -> i64 {
+ chrono::Utc::now().timestamp()
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Fault {
+ pub subsystem: String,
+ pub code: String,
+ pub severity: String,
+ pub detail: String,
+ pub remedy: String,
+ pub since: Option<i64>,
+}
+impl Fault {
+ pub fn new(subsystem: &str, code: &str, detail: &str, remedy: &str) -> Self {
+ Self {
+ subsystem: subsystem.into(),
+ code: code.into(),
+ severity: "warning".into(),
+ detail: detail.into(),
+ remedy: remedy.into(),
+ since: None,
+ }
+ }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Component {
+ pub status: String,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub engine: Option<String>,
+ pub path: Option<PathBuf>,
+ pub source: String,
+ pub version: Option<String>,
+ pub error: Option<String>,
+ pub error_at: Option<i64>,
+}
+impl Default for Component {
+ fn default() -> Self {
+ Self {
+ status: "unknown".into(),
+ engine: None,
+ path: None,
+ source: "unknown".into(),
+ version: None,
+ error: None,
+ error_at: None,
+ }
+ }
+}
+
+#[derive(Clone, Debug, Default, Serialize)]
+pub struct ProcessSnapshot {
+ pub hd_encoders_running: u64,
+ pub audio_encodes_running: u64,
+ pub audio_encodes_last_minute: u64,
+ pub audio_encode_failures_last_minute: u64,
+ pub extractions_running: u64,
+ pub extractions_queued: Option<u64>,
+ pub compaction_running: bool,
+ pub stalls_total: u64,
+ pub last_stall_at: Option<i64>,
+}
+
+#[derive(Clone, Debug, Default, Serialize)]
+pub struct FfmpegSnapshot {
+ #[serde(flatten)]
+ pub binary: Component,
+ pub version_probe_error: Option<String>,
+ pub install_in_progress: bool,
+ pub install_retry_after: Option<i64>,
+ pub last_spawn_error: Option<String>,
+ pub last_spawn_error_at: Option<i64>,
+ pub processes: ProcessSnapshot,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct FailedStart {
+ pub at: i64,
+ pub phase: String,
+ pub reason: String,
+ pub detail: String,
+ pub exit_code: Option<i32>,
+}
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Preflight {
+ pub check: String,
+ pub result: String,
+ pub detail: Option<String>,
+ pub remedy: Option<String>,
+}
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
+pub struct Environment {
+ pub data_dir_writable: Option<bool>,
+ pub path_has_ffmpeg: Option<bool>,
+ pub env_overrides: Vec<String>,
+}
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Startup {
+ pub phase: String,
+ pub phase_since: i64,
+ pub engine_started_at: Option<i64>,
+ pub preflight: Vec<Preflight>,
+ pub last_failed_start: Option<FailedStart>,
+ pub environment: Environment,
+}
+impl Default for Startup {
+ fn default() -> Self {
+ Self {
+ phase: "idle".into(),
+ phase_since: now(),
+ engine_started_at: None,
+ preflight: vec![],
+ last_failed_start: None,
+ environment: Environment::default(),
+ }
+ }
+}
+
+#[derive(Clone, Default)]
+struct State {
+ ffmpeg: FfmpegSnapshot,
+ ffprobe: Component,
+ models: BTreeMap<String, Component>,
+ runtime: BTreeMap<String, Component>,
+ startup: Startup,
+ startup_path: Option<PathBuf>,
+ audio_completions: VecDeque<(i64, bool)>,
+ compactions: u64,
+ media_required: Option<bool>,
+ model_generation: u64,
+ last_hd_success: Option<i64>,
+ // A success in one pipeline must not clear a failure in another.
+ media_failures: BTreeMap<String, (i64, String)>,
+}
+static STATE: LazyLock<Mutex<State>> = LazyLock::new(|| Mutex::new(State::default()));
+// Only startup writers acquire this lock. Media instrumentation and health reads
+// never wait for checkpoint I/O. Lock order is STARTUP_WRITER then STATE.
+static STARTUP_WRITER: Mutex<()> = Mutex::new(());
+fn state() -> std::sync::MutexGuard<'static, State> {
+ STATE.lock().unwrap_or_else(|e| e.into_inner())
+}
+
+pub fn configure_media_required(required: bool) {
+ state().media_required = Some(required);
+}
+
+pub fn binary_resolved(path: &Path, source: &str, probe: Option<PathBuf>) {
+ let mut s = state();
+ if s.ffmpeg.binary.path.as_deref() != Some(path) {
+ s.ffmpeg.binary.version = None;
+ }
+ s.ffmpeg.binary.path = Some(path.into());
+ s.ffmpeg.binary.source = source.into();
+ s.ffmpeg.binary.status = "ok".into();
+ s.ffmpeg.binary.error = None;
+ s.ffmpeg.binary.error_at = None;
+ s.ffmpeg.version_probe_error = None;
+ s.ffmpeg.install_in_progress = false;
+ s.ffmpeg.install_retry_after = None;
+ s.ffprobe.status = if probe.is_some() { "ok" } else { "missing" }.into();
+ s.ffprobe.path = probe;
+}
+pub fn binary_version(path: &Path, version: Option<String>, error: Option<String>) {
+ let mut s = state();
+ if s.ffmpeg.binary.path.as_deref() != Some(path) {
+ return;
+ }
+ s.ffmpeg.binary.version = version;
+ // A metadata probe timeout does not establish a recording failure. Actual
+ // encode failures are tracked separately and cannot be cleared by this probe.
+ s.ffmpeg.version_probe_error = error;
+}
+pub fn binary_installing() {
+ let mut s = state();
+ s.ffmpeg.binary.status = "installing".into();
+ s.ffmpeg.install_in_progress = true;
+}
+pub fn binary_install_failed(error: &str, retry_secs: i64) {
+ let mut s = state();
+ s.ffmpeg.binary.status = "install_failed".into();
+ s.ffmpeg.binary.error = Some(bounded_detail(error));
+ s.ffmpeg.binary.error_at = Some(now());
+ s.ffmpeg.install_in_progress = false;
+ s.ffmpeg.install_retry_after = Some(now() + retry_secs);
+}
+
+/// Scope a process lifetime, including cancellation. Only spawn/finish boundaries
+/// take a lock; no per-frame counters or allocations are added.
+pub struct MediaOperation {
+ kind: &'static str,
+}
+impl MediaOperation {
+ pub fn start(kind: &'static str) -> Self {
+ let mut s = state();
+ s.media_required = Some(true);
+ match kind {
+ "audio" => s.ffmpeg.processes.audio_encodes_running += 1,
+ "hd" => s.ffmpeg.processes.hd_encoders_running += 1,
+ "extraction" => s.ffmpeg.processes.extractions_running += 1,
+ "compaction" => s.compactions += 1,
+ _ => {}
+ }
+ Self { kind }
+ }
+ pub fn finish<T, E: std::fmt::Display>(&self, result: &Result<T, E>) {
+ // Error formatting may run arbitrary Display code; keep it outside the
+ // shared lock, along with clock access.
+ let error = result
+ .as_ref()
+ .err()
+ .map(|e| bounded_detail(&e.to_string()));
+ let at = now();
+ let mut s = state();
+ if self.kind == "audio" {
+ while s
+ .audio_completions
+ .front()
+ .is_some_and(|(t, _)| at - t >= 60)
+ {
+ s.audio_completions.pop_front();
+ }
+ if s.audio_completions.len() == 4096 {
+ s.audio_completions.pop_front();
+ }
+ s.audio_completions.push_back((at, result.is_err()));
+ }
+ match result {
+ Ok(_) => {
+ s.media_failures.remove(self.kind);
+ if self.kind == "hd" {
+ s.last_hd_success = Some(at);
+ }
+ }
+ Err(_) => {
+ let error = error.unwrap_or_default();
+ s.ffmpeg.last_spawn_error = Some(error.clone());
+ s.ffmpeg.last_spawn_error_at = Some(at);
+ s.media_failures.insert(self.kind.into(), (at, error));
+ }
+ }
+ }
+}
+impl Drop for MediaOperation {
+ fn drop(&mut self) {
+ let mut s = state();
+ match self.kind {
+ "audio" => {
+ s.ffmpeg.processes.audio_encodes_running =
+ s.ffmpeg.processes.audio_encodes_running.saturating_sub(1)
+ }
+ "hd" => {
+ s.ffmpeg.processes.hd_encoders_running =
+ s.ffmpeg.processes.hd_encoders_running.saturating_sub(1)
+ }
+ "extraction" => {
+ s.ffmpeg.processes.extractions_running =
+ s.ffmpeg.processes.extractions_running.saturating_sub(1)
+ }
+ "compaction" => s.compactions = s.compactions.saturating_sub(1),
+ _ => {}
+ }
+ }
+}
+pub fn media_stall() {
+ let mut s = state();
+ s.ffmpeg.processes.stalls_total += 1;
+ s.ffmpeg.processes.last_stall_at = Some(now());
+}
+
+pub fn begin_transcription_model(engine: &str) -> u64 {
+ let mut s = state();
+ s.model_generation += 1;
+ s.models.insert(
+ "transcription".into(),
+ Component {
+ status: "initializing".into(),
+ engine: Some(engine.into()),
+ ..Component::default()
+ },
+ );
+ s.model_generation
+}
+pub fn transcription_model_state(
+ generation: u64,
+ status: &str,
+ path: Option<PathBuf>,
+ error: Option<String>,
+) {
+ let mut s = state();
+ if s.model_generation != generation {
+ return;
+ }
+ let c = s.models.entry("transcription".into()).or_default();
+ c.status = status.into();
+ c.path = path;
+ c.error_at = error.as_ref().map(|_| now());
+ c.error = error.map(|s| bounded_detail(&s));
+}
+
+pub fn component_state(
+ group: &str,
+ name: &str,
+ status: &str,
+ path: Option<PathBuf>,
+ error: Option<String>,
+) {
+ let mut s = state();
+ let map = if group == "models" {
+ &mut s.models
+ } else {
+ &mut s.runtime
+ };
+ let c = map.entry(name.into()).or_default();
+ c.status = status.into();
+ c.path = path;
+ c.error_at = error.as_ref().map(|_| now());
+ c.error = error.map(|s| bounded_detail(&s));
+}
+
+fn write_startup(path: &Path, snapshot: &Startup) -> std::io::Result<()> {
+ let mut f = tempfile::NamedTempFile::new_in(path.parent().unwrap())?;
+ serde_json::to_writer(&mut f, snapshot)?;
+ f.as_file().sync_all()?;
+ f.persist(path).map_err(|e| e.error)?;
+ Ok(())
+}
+fn update_startup(update: impl FnOnce(&mut State) -> bool) {
+ update_startup_with_writer(update, write_startup);
+}
+
+fn update_startup_with_writer(
+ update: impl FnOnce(&mut State) -> bool,
+ write: impl FnOnce(&Path, &Startup) -> std::io::Result<()>,
+) {
+ // Serialize mutations as well as writes so an older phase cannot overwrite
+ // a newer checkpoint. Never retain STATE across disk I/O or serialization.
+ let _writer = STARTUP_WRITER.lock().unwrap_or_else(|e| e.into_inner());
+ let checkpoint = {
+ let mut s = state();
+ if !update(&mut s) {
+ return;
+ }
+ s.startup_path.clone().map(|path| (path, s.startup.clone()))
+ };
+ if let Some((path, mut startup)) = checkpoint {
+ // A successfully persisted checkpoint should report its own write as
+ // successful, including the first write after a disk error.
+ startup.environment.data_dir_writable = Some(true);
+ let ok = write(&path, &startup).is_ok();
+ state().startup.environment.data_dir_writable = Some(ok);
+ }
+}
+
+fn bounded_detail(detail: &str) -> String {
+ detail.chars().take(2048).collect()
+}
+fn recover_startup(previous: Option<Startup>) -> Option<FailedStart> {
+ previous.and_then(|p| {
+ if !matches!(p.phase.as_str(), "ready" | "idle" | "error") {
+ Some(FailedStart {
+ at: p.phase_since,
+ phase: p.phase,
+ reason: "unknown".into(),
+ detail: "previous startup did not reach ready; termination cause was not observed"
+ .into(),
+ exit_code: None,
+ })
+ } else {
+ p.last_failed_start
+ }
+ })
+}
+pub fn begin_startup(data_dir: &Path) {
+ let path = data_dir.join("health-startup.json");
+ let previous = std::fs::File::open(&path)
+ .ok()
+ .and_then(|f| serde_json::from_reader(f).ok());
+ let environment = Environment {
+ data_dir_writable: None,
+ path_has_ffmpeg: Some(which::which("ffmpeg").is_ok()),
+ env_overrides: [
+ "SCREENPIPE_FFMPEG_PATH",
+ "ORT_DYLIB_PATH",
+ "HF_HOME",
+ "HF_HUB_CACHE",
+ ]
+ .into_iter()
+ .filter(|name| std::env::var_os(name).is_some())
+ .map(String::from)
+ .collect(),
+ };
+ update_startup(|s| {
+ s.models.clear();
+ s.runtime.clear();
+ s.model_generation += 1;
+ s.media_failures.clear();
+ s.startup = Startup {
+ phase: "starting".into(),
+ engine_started_at: Some(now()),
+ last_failed_start: recover_startup(previous),
+ environment,
+ ..Startup::default()
+ };
+ s.startup_path = Some(path);
+ true
+ });
+}
+pub fn startup_phase(phase: &str) {
+ update_startup(|s| {
+ if s.startup.phase == phase {
+ return false;
+ }
+ let completed = match (s.startup.phase.as_str(), phase) {
+ ("migrating_database", "building_audio") => Some("database"),
+ ("building_audio", "starting_pipes") => Some("audio_initialization"),
+ ("starting_pipes", "ready") => Some("server_start"),
+ _ => None,
+ };
+ if let Some(check) = completed {
+ s.startup.preflight.retain(|p| p.check != check);
+ s.startup.preflight.push(Preflight {
+ check: check.into(),
+ result: "ok".into(),
+ detail: None,
+ remedy: None,
+ });
+ }
+ s.startup.phase = phase.into();
+ s.startup.phase_since = now();
+ true
+ });
+}
+pub fn startup_failure(reason: &str, detail: &str, exit_code: Option<i32>) {
+ let detail = bounded_detail(detail);
+ update_startup(|s| {
+ if matches!(s.startup.phase.as_str(), "idle" | "ready" | "error") {
+ return false;
+ }
+ s.startup.last_failed_start = Some(FailedStart {
+ at: now(),
+ phase: s.startup.phase.clone(),
+ reason: reason.into(),
+ detail,
+ exit_code,
+ });
+ s.startup.phase = "error".into();
+ s.startup.phase_since = now();
+ true
+ });
+}
+pub fn preflight(check: &str, result: &str, detail: Option<String>, remedy: Option<String>) {
+ update_startup(|s| {
+ s.startup.preflight.retain(|v| v.check != check);
+ s.startup.preflight.push(Preflight {
+ check: check.into(),
+ result: result.into(),
+ detail: detail.map(|s| bounded_detail(&s)),
+ remedy,
+ });
+ true
+ });
+}
+
+#[derive(Clone, Serialize)]
+pub struct Diagnostics {
+ pub dependencies: serde_json::Value,
+ pub runtime: serde_json::Value,
+ pub startup: Startup,
+ pub unhealthy_reasons: Vec<Fault>,
+}
+pub fn snapshot(audio_disabled: bool) -> Diagnostics {
+ let observed = state().clone();
+ snapshot_from(&observed, audio_disabled)
+}
+
+fn snapshot_from(s: &State, audio_disabled: bool) -> Diagnostics {
+ let at = now();
+ let mut ffmpeg = s.ffmpeg.clone();
+ ffmpeg.processes.compaction_running = s.compactions > 0;
+ ffmpeg.processes.audio_encodes_last_minute = s
+ .audio_completions
+ .iter()
+ .filter(|(t, _)| at - t < 60)
+ .count() as u64;
+ ffmpeg.processes.audio_encode_failures_last_minute = s
+ .audio_completions
+ .iter()
+ .filter(|(t, failed)| *failed && at - t < 60)
+ .count() as u64;
+ if let Some((failed_at, error)) = s
+ .media_failures
+ .iter()
+ .filter(|(kind, (failed_at, _))| {
+ at - failed_at < 180 && !(audio_disabled && *kind == "audio")
+ })
+ .map(|(_, failure)| failure)
+ .max_by_key(|(failed_at, _)| *failed_at)
+ {
+ ffmpeg.binary.status = "spawn_failing".into();
+ ffmpeg.binary.error = Some(error.clone());
+ ffmpeg.binary.error_at = Some(*failed_at);
+ }
+ if ffmpeg
+ .processes
+ .last_stall_at
+ .is_some_and(|t| at - t < 180 && s.last_hd_success.is_none_or(|success| success < t))
+ {
+ ffmpeg.binary.status = "stalled".into();
+ ffmpeg.binary.error = Some("HD encoder write timed out".into());
+ ffmpeg.binary.error_at = ffmpeg.processes.last_stall_at;
+ }
+ if s.media_required == Some(false) {
+ ffmpeg.binary.status = "not_required".into();
+ }
+ let mut ffprobe = s.ffprobe.clone();
+ if s.media_required == Some(false) {
+ ffprobe.status = "not_required".into();
+ }
+ let mut models = s.models.clone();
+ let mut runtime = s.runtime.clone();
+ for name in [
+ "transcription",
+ "vad",
+ "speaker_segmentation",
+ "speaker_embedding",
+ ] {
+ let c = models.entry(name.into()).or_default();
+ if audio_disabled {
+ c.status = "not_required".into();
+ }
+ }
+ if audio_disabled {
+ for c in runtime.values_mut() {
+ c.status = "not_required".into();
+ }
+ }
+ let mut faults = vec![];
+ // Unknown and first-run downloading are observations, not proven failures.
+ // ffprobe discovery is metadata only. Its absence does not establish that
+ // any enabled recording pipeline is failing.
+ for (name, c) in [("ffmpeg", &ffmpeg.binary)] {
+ if matches!(
+ c.status.as_str(),
+ "missing" | "install_failed" | "spawn_failing" | "stalled"
+ ) {
+ let mut fault = Fault::new(
+ "dependencies",
+ &format!("{name}_{}", c.status),
+ c.error
+ .as_deref()
+ .or(ffmpeg.last_spawn_error.as_deref())
+ .unwrap_or("media dependency unavailable"),
+ "Install or repair the bundled ffmpeg, then retry recording.",
+ );
+ fault.since = c.error_at.or(ffmpeg.last_spawn_error_at);
+ faults.push(fault);
+ }
+ }
+ for (group, components) in [("dependencies", &models), ("runtime", &runtime)] {
+ for (name, c) in components {
+ if matches!(
+ c.status.as_str(),
+ "load_failed"
+ | "init_failed"
+ | "download_failed"
+ | "init_timed_out"
+ | "unsupported"
+ ) {
+ let mut fault = Fault::new(group, &format!("{name}_{}", c.status), c.error.as_deref().unwrap_or("component unavailable"), "Check the configured local model and native runtime; retry initialization or select a supported transcription engine.");
+ fault.since = c.error_at;
+ faults.push(fault);
+ }
+ }
+ }
+ let compat = !crate::cpu_features::has_avx2();
+ let avx2 = cfg!(target_arch = "x86_64").then(crate::cpu_features::has_avx2);
+ let degraded_features: Vec<_> = runtime
+ .iter()
+ .filter(|(_, c)| matches!(c.status.as_str(), "init_failed" | "init_timed_out"))
+ .map(|(name, _)| name.clone())
+ .collect();
+ let onnx_status = if audio_disabled {
+ "not_required"
+ } else if runtime.values().any(|c| c.status == "init_timed_out") {
+ "init_timed_out"
+ } else if !degraded_features.is_empty() {
+ "init_failed"
+ } else if !runtime.is_empty() && runtime.values().all(|c| c.status == "ok") {
+ "ok"
+ } else {
+ "unknown"
+ };
+ Diagnostics {
+ dependencies: serde_json::json!({"ffmpeg": ffmpeg, "ffprobe": ffprobe, "models": models}),
+ runtime: serde_json::json!({
+ "status": if faults.iter().any(|f| f.subsystem == "runtime") { "degraded" }
+ else if onnx_status == "ok" || onnx_status == "not_required" { "ok" } else { "unknown" },
+ "onnxruntime": {
+ "status": onnx_status, "path": null, "source": "unknown", "version": null,
+ "init_timed_out": onnx_status == "init_timed_out",
+ "degraded_features": degraded_features, "initializations": runtime,
+ },
+ "cpu": { "avx2": avx2, "compat_mode": compat },
+ "msvc_runtime": { "status": if cfg!(windows) { "unknown" } else { "not_required" } },
+ "swift_runtime": { "status": if cfg!(target_os = "macos") { "unknown" } else { "not_required" } },
+ "crash_dump_helper": { "status": if cfg!(windows) { "unknown" } else { "not_required" } },
+ }),
+ startup: s.startup.clone(),
+ unhealthy_reasons: faults,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ // Tests using process-global observations must not race each other.
+ static GLOBAL_TEST: Mutex<()> = Mutex::new(());
+ #[test]
+ fn health_diagnostics_process_guards_release_and_recover_per_pipeline() {
+ let _test = GLOBAL_TEST.lock().unwrap();
+ *state() = State::default();
+ let operation = MediaOperation::start("audio");
+ operation.finish::<(), _>(&Err("synthetic spawn failure"));
+ assert_eq!(
+ snapshot(false).dependencies["ffmpeg"]["processes"]["audio_encodes_running"],
+ 1
+ );
+ let extraction = MediaOperation::start("extraction");
+ extraction.finish::<_, &str>(&Ok(()));
+ assert!(!snapshot(false).unhealthy_reasons.is_empty());
+ drop(extraction);
+ operation.finish::<_, &str>(&Ok(()));
+ assert!(snapshot(false).unhealthy_reasons.is_empty());
+ drop(operation);
+ assert_eq!(
+ snapshot(false).dependencies["ffmpeg"]["processes"]["audio_encodes_running"],
+ 0
+ );
+ let hd = MediaOperation::start("hd");
+ media_stall();
+ assert!(snapshot(false)
+ .unhealthy_reasons
+ .iter()
+ .any(|f| f.code == "ffmpeg_stalled"));
+ hd.finish::<_, &str>(&Ok(()));
+ assert!(snapshot(false).unhealthy_reasons.is_empty());
+ drop(hd);
+ let old = begin_transcription_model("old engine");
+ let current = begin_transcription_model("current engine");
+ transcription_model_state(current, "ready", None, None);
+ transcription_model_state(old, "download_failed", None, Some("late result".into()));
+ assert_eq!(
+ snapshot(false).dependencies["models"]["transcription"]["status"],
+ "ready"
+ );
+ assert_eq!(
+ snapshot(false).dependencies["models"]["transcription"]["engine"],
+ "current engine"
+ );
+ }
+
+ #[test]
+ fn health_diagnostics_slow_checkpoint_does_not_block_media_or_health() {
+ use std::sync::mpsc;
+ use std::time::Duration;
+ let _test = GLOBAL_TEST.lock().unwrap();
+ let dir = tempfile::tempdir().unwrap();
+ *state() = State::default();
+ begin_startup(dir.path());
+ let (entered_tx, entered_rx) = mpsc::channel();
+ let (release_tx, release_rx) = mpsc::channel();
+ let writer = std::thread::spawn(move || {
+ update_startup_with_writer(
+ |s| {
+ s.startup.phase = "building_audio".into();
+ true
+ },
+ |path, checkpoint| {
+ entered_tx.send(()).unwrap();
+ release_rx.recv_timeout(Duration::from_secs(10)).unwrap();
+ write_startup(path, checkpoint)
+ },
+ );
+ });
+ entered_rx.recv_timeout(Duration::from_secs(2)).unwrap();
+ let (done_tx, done_rx) = mpsc::channel();
+ let media = std::thread::spawn(move || {
+ for _ in 0..100 {
+ let op = MediaOperation::start("audio");
+ op.finish::<_, &str>(&Ok(()));
+ drop(op);
+ assert_eq!(snapshot(false).startup.phase, "building_audio");
+ }
+ done_tx.send(()).unwrap();
+ });
+ // On the old implementation this times out until the disk writer is
+ // released. Always release it before asserting, so a regression cannot
+ // leave a hung test process.
+ let completed_while_disk_blocked = done_rx.recv_timeout(Duration::from_secs(2));
+ release_tx.send(()).unwrap();
+ writer.join().unwrap();
+ media.join().unwrap();
+ assert!(completed_while_disk_blocked.is_ok());
+ startup_phase("ready");
+ let persisted: Startup = serde_json::from_reader(
+ std::fs::File::open(dir.path().join("health-startup.json")).unwrap(),
+ )
+ .unwrap();
+ assert_eq!(persisted.phase, "ready");
+ assert_eq!(
+ snapshot(false).dependencies["ffmpeg"]["processes"]["audio_encodes_running"],
+ 0
+ );
+ *state() = State::default();
+ }
+
+ #[test]
+ fn health_diagnostics_checkpoint_failure_is_observed_and_recovers() {
+ let _test = GLOBAL_TEST.lock().unwrap();
+ let dir = tempfile::tempdir().unwrap();
+ *state() = State::default();
+ begin_startup(dir.path());
+ update_startup_with_writer(
+ |_| true,
+ |_, _| Err(std::io::Error::other("disk unavailable")),
+ );
+ assert_eq!(
+ snapshot(false).startup.environment.data_dir_writable,
+ Some(false)
+ );
+ let op = MediaOperation::start("audio");
+ op.finish::<_, &str>(&Ok(()));
+ drop(op);
+ startup_phase("ready");
+ assert_eq!(
+ snapshot(false).startup.environment.data_dir_writable,
+ Some(true)
+ );
+ *state() = State::default();
+ }
+
+ #[test]
+ fn health_diagnostics_optional_probe_failure_does_not_degrade_recording() {
+ let _test = GLOBAL_TEST.lock().unwrap();
+ *state() = State::default();
+ let path = Path::new("test-ffmpeg");
+ binary_resolved(path, "path", None);
+ binary_version(path, None, Some("probe timeout".into()));
+ let observation = snapshot(false);
+ assert_eq!(observation.dependencies["ffmpeg"]["status"], "ok");
+ assert_eq!(observation.dependencies["ffprobe"]["status"], "missing");
+ assert_eq!(
+ observation.dependencies["ffmpeg"]["version_probe_error"],
+ "probe timeout"
+ );
+ assert!(observation.unhealthy_reasons.is_empty());
+ let op = MediaOperation::start("audio");
+ op.finish::<(), _>(&Err("actual encode failure"));
+ binary_version(path, Some("7.0".into()), None);
+ assert!(snapshot(false)
+ .unhealthy_reasons
+ .iter()
+ .any(|f| f.code == "ffmpeg_spawn_failing"));
+ op.finish::<_, &str>(&Ok(()));
+ drop(op);
+ assert!(snapshot(false).unhealthy_reasons.is_empty());
+ *state() = State::default();
+ }
+
+ #[test]
+ #[ignore = "manual timing eval; run with --ignored --nocapture"]
+ fn health_diagnostics_lifecycle_latency_eval() {
+ use std::time::Instant;
+ let _test = GLOBAL_TEST.lock().unwrap();
+ *state() = State::default();
+ let mut timings = Vec::with_capacity(20_000);
+ for i in 0..20_000 {
+ let started = Instant::now();
+ let op = MediaOperation::start("audio");
+ op.finish::<_, &str>(&Ok(()));
+ drop(op);
+ timings.push(started.elapsed().as_nanos());
+ if i % 100 == 0 {
+ std::hint::black_box(snapshot(false));
+ }
+ }
+ timings.sort_unstable();
+ let started = Instant::now();
+ for _ in 0..1000 {
+ std::hint::black_box(snapshot(false));
+ }
+ eprintln!("diagnostics lifecycle n=20000 p50_ns={} p95_ns={} p99_ns={}; full_snapshot_mean_us={:.2}; retained_events={}",
+ timings[10_000], timings[19_000], timings[19_800],
+ started.elapsed().as_secs_f64() * 1000.0, state().audio_completions.len());
+ assert_eq!(state().audio_completions.len(), 4096);
+ assert_eq!(state().ffmpeg.processes.audio_encodes_running, 0);
+ *state() = State::default();
+ }
+
+ #[test]
+ fn health_diagnostics_downloads_and_unknowns_are_not_faults() {
+ let mut s = State::default();
+ s.models.insert(
+ "transcription".into(),
+ Component {
+ status: "downloading".into(),
+ ..Component::default()
+ },
+ );
+ assert!(snapshot_from(&s, false).unhealthy_reasons.is_empty());
+ s.models.get_mut("transcription").unwrap().status = "download_failed".into();
+ assert_eq!(
+ snapshot_from(&s, false).unhealthy_reasons[0].code,
+ "transcription_download_failed"
+ );
+ assert!(snapshot_from(&s, true).unhealthy_reasons.is_empty());
+ s.models.get_mut("transcription").unwrap().status = "ready".into();
+ assert!(snapshot_from(&s, false).unhealthy_reasons.is_empty());
+ }
+
+ #[test]
+ fn health_diagnostics_dependency_status_matrix() {
+ for (status, fault) in [
+ ("unknown", false),
+ ("ok", false),
+ ("installing", false),
+ ("missing", true),
+ ("install_failed", true),
+ ] {
+ let mut s = State::default();
+ s.ffmpeg.binary.status = status.into();
+ s.ffmpeg.binary.error_at = Some(42);
+ let observation = snapshot_from(&s, false);
+ assert_eq!(!observation.unhealthy_reasons.is_empty(), fault, "{status}");
+ if fault {
+ assert_eq!(observation.unhealthy_reasons[0].since, Some(42));
+ }
+ s.media_required = Some(false);
+ assert!(snapshot_from(&s, false).unhealthy_reasons.is_empty());
+ }
+ for (status, fault) in [
+ ("unknown", false),
+ ("initializing", false),
+ ("downloading", false),
+ ("downloaded", false),
+ ("ready", false),
+ ("not_required", false),
+ ("load_failed", true),
+ ("download_failed", true),
+ ("init_timed_out", true),
+ ("unsupported", true),
+ ] {
+ let mut s = State::default();
+ s.models.insert(
+ "transcription".into(),
+ Component {
+ status: status.into(),
+ error_at: Some(42),
+ ..Component::default()
+ },
+ );
+ let observation = snapshot_from(&s, false);
+ assert_eq!(!observation.unhealthy_reasons.is_empty(), fault, "{status}");
+ if fault {
+ assert_eq!(observation.unhealthy_reasons[0].since, Some(42));
+ }
+ assert!(snapshot_from(&s, true).unhealthy_reasons.is_empty());
+ }
+ }
+
+ #[test]
+ fn health_diagnostics_failure_history_does_not_degrade_current_start() {
+ let mut s = State::default();
+ s.startup.last_failed_start = Some(FailedStart {
+ at: 1,
+ phase: "building_audio".into(),
+ reason: "unknown".into(),
+ detail: "old error".into(),
+ exit_code: None,
+ });
+ assert!(snapshot_from(&s, false).unhealthy_reasons.is_empty());
+ }
+
+ #[test]
+ fn health_diagnostics_unneeded_and_old_media_failures_are_benign() {
+ let mut s = State::default();
+ s.ffmpeg.binary.status = "install_failed".into();
+ s.media_required = Some(false);
+ assert!(snapshot_from(&s, true).unhealthy_reasons.is_empty());
+ s.media_required = Some(true);
+ assert_eq!(
+ snapshot_from(&s, true).unhealthy_reasons[0].code,
+ "ffmpeg_install_failed"
+ );
+ s.ffmpeg.binary.status = "ok".into();
+ s.media_failures
+ .insert("audio".into(), (now(), "spawn failed".into()));
+ assert!(snapshot_from(&s, true).unhealthy_reasons.is_empty());
+ assert!(!snapshot_from(&s, false).unhealthy_reasons.is_empty());
+ s.media_failures
+ .insert("audio".into(), (now() - 181, "old failure".into()));
+ assert!(snapshot_from(&s, false).unhealthy_reasons.is_empty());
+ }
+
+ #[test]
+ fn health_diagnostics_audio_counters_use_a_rolling_minute() {
+ let mut s = State::default();
+ s.audio_completions
+ .extend([(now() - 70, true), (now(), true), (now(), false)]);
+ let snapshot = snapshot_from(&s, false);
+ assert_eq!(
+ snapshot.dependencies["ffmpeg"]["processes"]["audio_encodes_last_minute"],
+ 2
+ );
+ assert_eq!(
+ snapshot.dependencies["ffmpeg"]["processes"]["audio_encode_failures_last_minute"],
+ 1
+ );
+ }
+
+ #[test]
+ fn startup_failure_survives_successful_restart() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("health-startup.json");
+ let failure = FailedStart {
+ at: 1,
+ phase: "building_audio".into(),
+ reason: "unknown".into(),
+ detail: "init failed".into(),
+ exit_code: Some(1),
+ };
+ let old = Startup {
+ phase: "ready".into(),
+ last_failed_start: Some(failure),
+ ..Startup::default()
+ };
+ write_startup(&path, &old).unwrap();
+ let read = serde_json::from_reader(std::fs::File::open(path).unwrap()).unwrap();
+ assert_eq!(recover_startup(Some(read)).unwrap().detail, "init failed");
+ }
+ #[test]
+ fn interrupted_start_does_not_invent_a_termination_cause() {
+ let old = Startup {
+ phase: "migrating_database".into(),
+ ..Startup::default()
+ };
+ let failed = recover_startup(Some(old)).unwrap();
+ assert_eq!(failed.reason, "unknown");
+ assert_eq!(failed.phase, "migrating_database");
+ assert_eq!(failed.exit_code, None);
+ }
+ #[test]
+ fn fresh_and_ready_starts_have_no_failure() {
+ assert!(recover_startup(None).is_none());
+ assert!(recover_startup(Some(Startup {
+ phase: "ready".into(),
+ ..Startup::default()
+ }))
+ .is_none());
+ }
+}
diff --git a/crates/screenpipe-core/src/lib.rs b/crates/screenpipe-core/src/lib.rs
index ef8a8e20c..85160407f 100644
--- a/crates/screenpipe-core/src/lib.rs
+++ b/crates/screenpipe-core/src/lib.rs
@@ -35,6 +35,7 @@ pub mod display_topology;
// cycle via screenpipe-secrets). Re-exported here as the canonical path.
pub use screenpipe_cpu_features as cpu_features;
pub mod ffmpeg;
+pub mod health_diagnostics;
pub mod memories;
// Single source of truth for CREATE_NO_WINDOW. Spawning a console program from
// the GUI-subsystem app flashes a terminal on Windows unless every call site
diff --git a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs b/crates/screenpipe-engine/src/bin/screenpipe-engine.rs
index 44144de83..c19e30682 100644
--- a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs
+++ b/crates/screenpipe-engine/src/bin/screenpipe-engine.rs
@@ -527,6 +527,7 @@ async fn main() -> anyhow::Result<()> {
}
}
};
+ screenpipe_core::health_diagnostics::begin_startup(&local_data_dir);
let local_data_dir_clone = local_data_dir.clone();
// One-time, TTY-only equivalent of the desktop onboarding's "connect all
@@ -861,6 +862,13 @@ async fn main() -> anyhow::Result<()> {
}));
}
+ screenpipe_core::health_diagnostics::configure_media_required(
+ !config.disable_audio
+ || !config.disable_snapshot_compaction
+ || (!config.disable_vision
+ && config.hd_recording_default
+ == screenpipe_engine::high_fps_controller::DefaultMode::Always),
+ );
// Only require ffmpeg when audio recording is enabled. Vision-only recording
// should not attempt network installs (important for offline / locked-down
// Windows environments).
@@ -873,10 +881,22 @@ async fn main() -> anyhow::Result<()> {
eprintln!(
"ffmpeg not found and installation failed. please install ffmpeg manually."
);
+ screenpipe_core::health_diagnostics::startup_failure(
+ "ffmpeg_missing",
+ "ffmpeg not found and installation failed",
+ Some(1),
+ );
std::process::exit(1);
}
}
+ screenpipe_core::health_diagnostics::preflight("ffmpeg", "ok", None, None);
} else {
+ screenpipe_core::health_diagnostics::preflight(
+ "ffmpeg",
+ "not_checked",
+ Some("audio disabled; discovery deferred until a media pipeline needs it".into()),
+ None,
+ );
debug!("audio disabled; skipping ffmpeg preflight");
}
@@ -947,6 +967,11 @@ async fn main() -> anyhow::Result<()> {
if start.elapsed() > timeout {
eprintln!("timed out waiting for permissions.");
if !permissions::preflight_check(need_screen, need_audio) {
+ screenpipe_core::health_diagnostics::startup_failure(
+ "permission_denied",
+ "required screen or microphone permission was not granted",
+ Some(1),
+ );
std::process::exit(1);
}
break;
@@ -975,12 +1000,22 @@ async fn main() -> anyhow::Result<()> {
#[cfg(not(target_os = "macos"))]
{
if !permissions::preflight_check(need_screen, need_audio) {
+ screenpipe_core::health_diagnostics::startup_failure(
+ "permission_denied",
+ "required recording permission was not granted",
+ Some(1),
+ );
std::process::exit(1);
}
}
}
if !is_local_ipv4_port_free(config.port) {
+ screenpipe_core::health_diagnostics::startup_failure(
+ "port_in_use",
+ "configured server port is already in use",
+ None,
+ );
error!(
"you're likely already running screenpipe instance in a different environment, e.g. terminal/ide, close it and restart or use different port"
);
@@ -1019,6 +1054,7 @@ async fn main() -> anyhow::Result<()> {
// This helps track users who may have screen capture issues due to old macOS
analytics::check_macos_version();
+ screenpipe_core::health_diagnostics::startup_phase("migrating_database");
let database_path =
screenpipe_db::storage::resolve_database_path(&local_data_dir.join("db.sqlite"))?;
let (db, startup_guard) = loop {
@@ -1146,6 +1182,7 @@ async fn main() -> anyhow::Result<()> {
Some(detector)
};
+ screenpipe_core::health_diagnostics::startup_phase("building_audio");
let mut audio_manager_builder = config.to_audio_manager_builder(
PathBuf::from(output_path_clone.clone().to_string()),
audio_devices,
@@ -1197,6 +1234,7 @@ async fn main() -> anyhow::Result<()> {
Arc::new(manager)
}
Err(e) => {
+ screenpipe_core::health_diagnostics::startup_failure("unknown", &e.to_string(), None);
error!("{e}");
return Ok(());
}
@@ -1516,6 +1554,7 @@ async fn main() -> anyhow::Result<()> {
};
// Initialize pipe manager
+ screenpipe_core::health_diagnostics::startup_phase("starting_pipes");
let pipes_dir = local_data_dir.join("pipes");
std::fs::create_dir_all(&pipes_dir).ok();
@@ -1992,7 +2031,22 @@ async fn main() -> anyhow::Result<()> {
// mDNS behavior from SCServer::start.
let router = server.try_create_router().await?;
let server_address = SocketAddr::new(IpAddr::V4(config.listen_address), config.port);
- let listener = screenpipe_engine::server::bind_listener(server_address).await?;
+ let listener = screenpipe_engine::server::bind_listener(server_address)
+ .await
+ .map_err(|error| {
+ screenpipe_core::health_diagnostics::startup_failure(
+ if error.kind() == std::io::ErrorKind::AddrInUse {
+ "port_in_use"
+ } else {
+ "unknown"
+ },
+ &error.to_string(),
+ None,
+ );
+ error
+ })?;
+ screenpipe_core::health_diagnostics::preflight("server_port", "ok", None, None);
+ screenpipe_core::health_diagnostics::startup_phase("ready");
drop(startup_guard);
info!("Server listening on {}", server_address);
if server.advertise_mdns {
diff --git a/crates/screenpipe-engine/src/hd_recorder.rs b/crates/screenpipe-engine/src/hd_recorder.rs
index 293b324ff..6da750ba7 100644
--- a/crates/screenpipe-engine/src/hd_recorder.rs
+++ b/crates/screenpipe-engine/src/hd_recorder.rs
@@ -279,6 +279,7 @@ mod macos {
match tokio::time::timeout(write_timeout, writer.write_all(jpeg)).await {
Ok(result) => result?,
Err(_) => {
+ screenpipe_core::health_diagnostics::media_stall();
return Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
"hd ffmpeg stdin write timed out",
@@ -470,7 +471,10 @@ mod macos {
));
let file_str = file.to_string_lossy().to_string();
- let mut ffmpeg = start_hd_ffmpeg(&file, actual_fps)?;
+ let operation = screenpipe_core::health_diagnostics::MediaOperation::start("hd");
+ let mut ffmpeg = start_hd_ffmpeg(&file, actual_fps).inspect_err(|error| {
+ operation.finish::<(), _>(&Err(error));
+ })?;
let mut stdin = ffmpeg.stdin.take().context("hd ffmpeg stdin missing")?;
// Queue the chunk row concurrently. Neither SQLite admission nor its
// completion is awaited by the media writer.
@@ -601,6 +605,7 @@ mod macos {
write_failed = true;
}
Err(_) => {
+ screenpipe_core::health_diagnostics::media_stall();
warn!("hd recorder: ffmpeg finalize timed out on monitor {monitor_id}");
write_failed = true;
let _ = ffmpeg.kill().await;
@@ -608,6 +613,12 @@ mod macos {
}
Ok(Ok(_)) => {}
}
+ operation.finish(&if write_failed {
+ Err("HD encoder failed; inspect encoder log")
+ } else {
+ Ok(())
+ });
+ drop(operation);
drop(stream);
spawn_hd_index_persist(
diff --git a/crates/screenpipe-engine/src/routes/health.rs b/crates/screenpipe-engine/src/routes/health.rs
index bfea650cd..b2ae1350e 100644
--- a/crates/screenpipe-engine/src/routes/health.rs
+++ b/crates/screenpipe-engine/src/routes/health.rs
@@ -645,6 +645,16 @@ pub struct MonitorInfo {
pub struct HealthCheckResponse {
pub status: String,
pub status_code: u16,
+ /// Cached observations. Health polling never installs or probes dependencies.
+ #[serde(default)]
+ pub dependencies: serde_json::Value,
+ #[serde(default)]
+ pub runtime: serde_json::Value,
+ #[serde(default)]
+ pub startup: serde_json::Value,
+ /// Stable causes and remedies; legacy message text is retained unchanged.
+ #[serde(default)]
+ pub unhealthy_reasons: Vec<serde_json::Value>,
pub last_frame_timestamp: Option<chrono::DateTime<Utc>>,
pub last_audio_timestamp: Option<chrono::DateTime<Utc>>,
pub frame_status: String,
@@ -984,6 +994,12 @@ fn degraded_response() -> HealthCheckResponse {
HealthCheckResponse {
status: "degraded".to_string(),
status_code: 503,
+ dependencies: json!({"status": "unknown"}),
+ runtime: json!({"status": "unknown"}),
+ startup: json!({"phase": "unknown"}),
+ unhealthy_reasons: vec![
+ json!({"subsystem": "startup", "code": "health_snapshot_timeout", "severity": "warning", "detail": "health snapshot timed out", "remedy": "Retry the health request and inspect the engine log if it persists.", "since": null}),
+ ],
last_frame_timestamp: None,
last_audio_timestamp: None,
frame_status: "unknown".to_string(),
@@ -1846,7 +1862,16 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
// so remote monitoring can see degradation and engine-restart requests.
let wqh = state.db.write_queue_health();
- HealthCheckResponse {
+ let diagnostics = screenpipe_core::health_diagnostics::snapshot(state.audio_disabled);
+ let mut response = HealthCheckResponse {
+ dependencies: diagnostics.dependencies,
+ runtime: diagnostics.runtime,
+ startup: serde_json::to_value(diagnostics.startup).unwrap_or_default(),
+ unhealthy_reasons: diagnostics
+ .unhealthy_reasons
+ .into_iter()
+ .map(|f| serde_json::to_value(f).unwrap_or_default())
+ .collect(),
status: overall_status.to_string(),
status_code,
last_frame_timestamp: last_frame,
@@ -1989,6 +2014,118 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
schedule_paused: crate::schedule_monitor::schedule_paused(),
hostname: hostname::get().ok().and_then(|h| h.into_string().ok()),
version: Some(env!("CARGO_PKG_VERSION").to_string()),
+ };
+ add_health_reasons(&mut response);
+ response
+}
+
+/// Derive faults from the same typed state as the legacy response, never by
+/// parsing its human-readable message. This also covers timeout fallback above.
+fn add_health_reasons(response: &mut HealthCheckResponse) {
+ let mut add = |subsystem: &str, code: &str, detail: &str, remedy: &str| {
+ response
+ .unhealthy_reasons
+ .push(json!({"subsystem": subsystem, "code": code,
+ "severity": "warning", "detail": detail, "remedy": remedy, "since": null}));
+ };
+ if !matches!(response.frame_status.as_str(), "ok" | "disabled") {
+ add("vision", &format!("vision_{}", response.vision_reason), &response.frame_status,
+ "Inspect vision_reason and capture permissions; check selected displays and the capture loop.");
+ } else if matches!(
+ response.vision_reason.as_str(),
+ "permission_denied" | "capture_stalled" | "not_started" | "ocr_unavailable"
+ ) {
+ add(
+ "vision",
+ &format!("vision_{}", response.vision_reason),
+ &response.vision_reason,
+ "Inspect capture permissions and the local OCR installation.",
+ );
+ }
+ if !matches!(
+ response.audio_status.as_str(),
+ "ok" | "disabled" | "no_input_device" | "waiting_for_meeting"
+ ) {
+ add(
+ "audio",
+ &format!("audio_{}", response.audio_status),
+ &response.audio_status,
+ "Check selected audio devices, permissions, and the transcription engine.",
+ );
+ }
+ if response.write_queue_degraded {
+ add(
+ "database",
+ "write_queue_degraded",
+ "database writer reported repeated failures",
+ "Check free disk space and the database write error in the engine log.",
+ );
+ }
+ if response.vision_db_write_stalled {
+ add(
+ "vision",
+ "vision_write_stalled",
+ "vision persistence is stalled",
+ "Inspect capture loop and database pipeline counters.",
+ );
+ }
+ if response.audio_db_write_stalled {
+ add(
+ "audio",
+ "transcription_write_stalled",
+ "audio transcription persistence is stalled",
+ "Inspect the transcription engine and pending segment backlog.",
+ );
+ }
+ if let Some(p) = &response.pipeline {
+ if response.status == "degraded" && p.frame_drop_rate > 0.5 && p.uptime_secs > 120.0 {
+ add(
+ "vision",
+ "frame_drop_rate_high",
+ "capture frames are being dropped",
+ "Inspect capture pipeline errors and database latency.",
+ );
+ }
+ if response.status == "degraded" && p.avg_db_latency_ms > 10_000.0 && p.uptime_secs > 120.0
+ {
+ add(
+ "database",
+ "write_latency_high",
+ "vision database writes are slow",
+ "Inspect disk latency and database lock contention.",
+ );
+ }
+ }
+ if let Some(p) = &response.audio_pipeline {
+ if response.status == "degraded" && p.chunks_channel_full > 0 && p.uptime_secs > 120.0 {
+ add(
+ "audio",
+ "transcription_queue_full",
+ "transcription queue dropped audio chunks",
+ "Inspect transcription throughput and engine errors.",
+ );
+ }
+ }
+ if response
+ .ui_recorder
+ .as_ref()
+ .is_some_and(|ui| ui.configured && !ui.running)
+ {
+ add(
+ "ui_recorder",
+ "ui_recorder_not_running",
+ "UI capture is configured but not running",
+ "Check accessibility/input monitoring permissions and UI recorder startup errors.",
+ );
+ }
+ // Preserve the contract even if a future degradation condition has not yet
+ // received its own code; tests require every degraded response to explain it.
+ if response.status == "degraded" && response.unhealthy_reasons.is_empty() {
+ response.unhealthy_reasons.push(json!({"subsystem":"startup", "code":"pipeline_degraded", "severity":"warning", "detail":"a recording pipeline is degraded", "remedy":"Inspect the detailed pipeline counters and engine log.", "since":null}));
+ }
+ if !response.unhealthy_reasons.is_empty() {
+ response.status = "degraded".into();
+ response.status_code = 503;
}
}
@@ -2503,9 +2640,86 @@ mod tests {
);
}
+ #[test]
+ fn health_diagnostics_preserve_legacy_messages_and_explain_faults() {
+ let mut response = dummy_response("healthy");
+ let old_message = response.message.clone();
+ add_health_reasons(&mut response);
+ assert!(response.unhealthy_reasons.is_empty());
+ assert_eq!(response.status, "healthy");
+ response.write_queue_degraded = true;
+ add_health_reasons(&mut response);
+ assert_eq!(response.status, "degraded");
+ assert_eq!(response.status_code, 503);
+ assert_eq!(response.message, old_message);
+ assert_eq!(
+ response.unhealthy_reasons[0]["code"],
+ "write_queue_degraded"
+ );
+ }
+
+ #[test]
+ fn health_diagnostics_intentional_idle_states_remain_healthy() {
+ for audio in ["ok", "disabled", "no_input_device", "waiting_for_meeting"] {
+ for vision in ["ok", "disabled"] {
+ let mut response = dummy_response("healthy");
+ response.audio_status = audio.into();
+ response.frame_status = vision.into();
+ add_health_reasons(&mut response);
+ assert_eq!(response.status, "healthy", "{audio}/{vision}");
+ assert_eq!(response.status_code, 200);
+ assert!(response.unhealthy_reasons.is_empty());
+ }
+ }
+ }
+
+ #[test]
+ fn health_diagnostics_observed_dependency_fault_changes_status_only() {
+ let mut response = dummy_response("healthy");
+ let original = response.clone();
+ response.unhealthy_reasons.push(json!({
+ "subsystem": "dependencies", "code": "ffmpeg_spawn_failing",
+ "severity": "warning", "detail": "encode failed", "remedy": "repair ffmpeg", "since": 42
+ }));
+ add_health_reasons(&mut response);
+ assert_eq!(response.status, "degraded");
+ assert_eq!(response.status_code, 503);
+ assert_eq!(response.message, original.message);
+ assert_eq!(response.verbose_instructions, original.verbose_instructions);
+ assert_eq!(response.audio_status, original.audio_status);
+ assert_eq!(response.frame_status, original.frame_status);
+ assert_eq!(response.unhealthy_reasons.len(), 1);
+ assert_eq!(response.unhealthy_reasons[0]["since"], 42);
+ }
+
+ #[test]
+ fn health_diagnostics_legacy_responses_deserialize() {
+ let mut value = serde_json::to_value(dummy_response("healthy")).unwrap();
+ for key in ["dependencies", "runtime", "startup", "unhealthy_reasons"] {
+ value.as_object_mut().unwrap().remove(key);
+ }
+ let response: HealthCheckResponse = serde_json::from_value(value).unwrap();
+ assert!(response.unhealthy_reasons.is_empty());
+ }
+
+ #[test]
+ fn health_diagnostics_timeout_and_unknown_degradation_have_reasons() {
+ assert_eq!(
+ degraded_response().unhealthy_reasons[0]["code"],
+ "health_snapshot_timeout"
+ );
+ let mut response = dummy_response("degraded");
+ add_health_reasons(&mut response);
+ assert!(!response.unhealthy_reasons.is_empty());
+ }
+
fn dummy_response(status: &str) -> HealthCheckResponse {
HealthCheckResponse {
status: status.to_string(),
+ dependencies: json!({}),
+ runtime: json!({}),
+ startup: json!({}),
+ unhealthy_reasons: vec![],
status_code: 200,
last_frame_timestamp: None,
last_audio_timestamp: None,
diff --git a/crates/screenpipe-engine/src/snapshot_compaction.rs b/crates/screenpipe-engine/src/snapshot_compaction.rs
index c3a4dcaef..d63e663fa 100644
--- a/crates/screenpipe-engine/src/snapshot_compaction.rs
+++ b/crates/screenpipe-engine/src/snapshot_compaction.rs
@@ -585,7 +585,12 @@ async fn compact_chunk(
);
// Encode JPEGs → MP4 via ffmpeg (low-priority, capped threads, JPEG passthrough)
- let (mut child, encoder) = start_ffmpeg_lowpri(&mp4_path_str, fps, video_quality).await?;
+ let operation = screenpipe_core::health_diagnostics::MediaOperation::start("compaction");
+ let started = start_ffmpeg_lowpri(&mp4_path_str, fps, video_quality).await;
+ if started.is_err() {
+ operation.finish(&started);
+ }
+ let (mut child, encoder) = started?;
let mut stdin = child
.stdin
.take()
@@ -682,6 +687,12 @@ async fn compact_chunk(
// retries next cycle (the output guard removes the partial MP4).
let encode_ok = write_error.is_none() && exit_status.is_some_and(|status| status.success());
encoder.record_encode_outcome(encode_ok);
+ operation.finish(&if encode_ok {
+ Ok(())
+ } else {
+ Err("snapshot encoder did not finish successfully")
+ });
+ drop(operation);
if let Some(error) = write_error {
return Err(anyhow::anyhow!(
"failed while feeding snapshots to ffmpeg: {error} — keeping source JPEGs for retry"
diff --git a/crates/screenpipe-engine/src/video_utils.rs b/crates/screenpipe-engine/src/video_utils.rs
index 3b3ef4606..40262583a 100644
--- a/crates/screenpipe-engine/src/video_utils.rs
+++ b/crates/screenpipe-engine/src/video_utils.rs
@@ -1189,7 +1189,12 @@ pub async fn extract_frame_from_video(
debug!("ffmpeg command: {:?}", command);
- let output = command.output().await?;
+ let operation = screenpipe_core::health_diagnostics::MediaOperation::start("extraction");
+ let output = command.output().await;
+ operation.finish(&output);
+ let output = output?;
+ // Nonzero exit can mean a corrupt user media file, not a missing runtime.
+ // Keep that request-specific error in the existing API response.
if !output.status.success() {
let error_message = String::from_utf8_lossy(&output.stderr);
diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md
index b6d6d9d05..2b5fcba73 100644
--- a/docs/COVERAGE.md
+++ b/docs/COVERAGE.md
@@ -37,16 +37,16 @@ results and `cargo llvm-cov` data on top when judging release confidence.
### Core Engine
- Mapped suites: 35
-- Mapped Rust files: 370
-- Active test blocks: 3622
-- Ignored/manual test blocks: 158
-- Weighted coverage points: 2992.8
+- Mapped Rust files: 371
+- Active test blocks: 3654
+- Ignored/manual test blocks: 164
+- Weighted coverage points: 3015.5
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 31 | 3476 | 144 | 2926.5 | 21 | 11 | 100% |
-| macos | 31 | 3536 | 124 | 2938.8 | 22 | 11 | 100% |
-| linux | 28 | 3093 | 124 | 2573.7 | 20 | 11 | 100% |
+| windows | 31 | 3508 | 150 | 2949.2 | 21 | 11 | 100% |
+| macos | 31 | 3553 | 125 | 2951.0 | 22 | 11 | 100% |
+| linux | 28 | 3110 | 125 | 2585.9 | 20 | 11 | 100% |
## Refresh
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 2a1d70bb6..ab4b2e824 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -8,11 +8,11 @@ confidence, and criticality.
- Manifest: `docs/coverage/core-engine-map.json`
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs
- Mapped suites: 35
-- Mapped Rust files: 370
-- Active test blocks: 3622
-- Ignored/manual test blocks: 158
-- Declared test blocks: 3780
-- Weighted coverage points: 2992.8
+- Mapped Rust files: 371
+- Active test blocks: 3654
+- Ignored/manual test blocks: 164
+- Declared test blocks: 3818
+- Weighted coverage points: 3015.5
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,20 +23,20 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 31 | 3476 | 144 | 2926.5 | 21 | 11 | 100% |
-| macos | 31 | 3536 | 124 | 2938.8 | 22 | 11 | 100% |
-| linux | 28 | 3093 | 124 | 2573.7 | 20 | 11 | 100% |
+| windows | 31 | 3508 | 150 | 2949.2 | 21 | 11 | 100% |
+| macos | 31 | 3553 | 125 | 2951.0 | 22 | 11 | 100% |
+| linux | 28 | 3110 | 125 | 2585.9 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| screenpipe-engine | 12 | 21 | 123 | 1765 | 51 | 1359.3 | 10 |
+| screenpipe-engine | 12 | 21 | 123 | 1771 | 51 | 1363.5 | 10 |
| screenpipe-db | 5 | 62 | 25 | 577 | 25 | 549.1 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
-| screenpipe-audio | 6 | 25 | 51 | 635 | 44 | 558.4 | 5 |
+| screenpipe-audio | 6 | 25 | 51 | 645 | 45 | 565.4 | 5 |
| screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |
-| screenpipe-a11y | 4 | 2 | 30 | 358 | 29 | 263.8 | 3 |
+| screenpipe-a11y | 4 | 2 | 31 | 374 | 34 | 275.3 | 3 |
| screenpipe-fs | 1 | 0 | 1 | 3 | 0 | 3.0 | 3 |
## Line Coverage
@@ -62,28 +62,28 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| Layer | windows | macos | linux |
| --- | --- | --- | --- |
-| accessibility | 4 suites / 355 active / 30 ignored / 325.6 pts | 4 suites / 411 active / 10 ignored / 332.4 pts | 4 suites / 330 active / 7 ignored / 302.5 pts |
-| audio | 8 suites / 751 active / 45 ignored / 674.4 pts | 8 suites / 751 active / 45 ignored / 674.4 pts | 7 suites / 672 active / 44 ignored / 619.1 pts |
+| accessibility | 4 suites / 371 active / 35 ignored / 337.1 pts | 4 suites / 412 active / 10 ignored / 333.4 pts | 4 suites / 331 active / 7 ignored / 303.5 pts |
+| audio | 8 suites / 761 active / 46 ignored / 681.4 pts | 8 suites / 761 active / 46 ignored / 681.4 pts | 7 suites / 682 active / 45 ignored / 626.1 pts |
| audio-device | 2 suites / 226 active / 7 ignored / 202.3 pts | 2 suites / 226 active / 7 ignored / 202.3 pts | 1 suites / 147 active / 6 ignored / 147.0 pts |
| configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |
| database | 7 suites / 531 active / 21 ignored / 503.1 pts | 7 suites / 531 active / 21 ignored / 503.1 pts | 7 suites / 531 active / 21 ignored / 503.1 pts |
| db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts |
| engine-lifecycle | 6 suites / 247 active / 1 ignored / 222.0 pts | 6 suites / 247 active / 1 ignored / 222.0 pts | 6 suites / 243 active / 9 ignored / 221.1 pts |
-| local-api | 3 suites / 457 active / 10 ignored / 322.3 pts | 3 suites / 457 active / 10 ignored / 322.3 pts | 3 suites / 457 active / 10 ignored / 322.3 pts |
-| meeting | 6 suites / 1654 active / 21 ignored / 1351.3 pts | 6 suites / 1654 active / 21 ignored / 1351.3 pts | 4 suites / 1305 active / 17 ignored / 1026.0 pts |
+| local-api | 3 suites / 462 active / 10 ignored / 325.8 pts | 3 suites / 462 active / 10 ignored / 325.8 pts | 3 suites / 462 active / 10 ignored / 325.8 pts |
+| meeting | 6 suites / 1659 active / 21 ignored / 1354.8 pts | 6 suites / 1659 active / 21 ignored / 1354.8 pts | 4 suites / 1310 active / 17 ignored / 1029.5 pts |
| ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1586 active / 76 ignored / 1404.4 pts | 14 suites / 1694 active / 80 ignored / 1447.6 pts | 13 suites / 1586 active / 76 ignored / 1404.4 pts |
+| performance | 13 suites / 1598 active / 77 ignored / 1413.1 pts | 14 suites / 1706 active / 81 ignored / 1456.3 pts | 13 suites / 1598 active / 77 ignored / 1413.1 pts |
| pipes | 1 suites / 515 active / 3 ignored / 360.5 pts | 1 suites / 515 active / 3 ignored / 360.5 pts | 1 suites / 515 active / 3 ignored / 360.5 pts |
-| privacy | 5 suites / 949 active / 37 ignored / 768.7 pts | 5 suites / 1005 active / 17 ignored / 775.5 pts | 5 suites / 924 active / 14 ignored / 745.5 pts |
+| privacy | 5 suites / 965 active / 42 ignored / 780.2 pts | 5 suites / 1006 active / 17 ignored / 776.5 pts | 5 suites / 925 active / 14 ignored / 746.5 pts |
| real-app | - | 1 suites / 108 active / 4 ignored / 43.2 pts | - |
| speaker | 2 suites / 375 active / 9 ignored / 375.0 pts | 2 suites / 375 active / 9 ignored / 375.0 pts | 2 suites / 375 active / 9 ignored / 375.0 pts |
-| storage | 4 suites / 659 active / 38 ignored / 546.8 pts | 4 suites / 659 active / 38 ignored / 546.8 pts | 4 suites / 659 active / 38 ignored / 546.8 pts |
+| storage | 4 suites / 660 active / 38 ignored / 547.5 pts | 4 suites / 660 active / 38 ignored / 547.5 pts | 4 suites / 660 active / 38 ignored / 547.5 pts |
| sync | 1 suites / 515 active / 3 ignored / 360.5 pts | 1 suites / 515 active / 3 ignored / 360.5 pts | 1 suites / 515 active / 3 ignored / 360.5 pts |
-| timeline | 4 suites / 1192 active / 43 ignored / 977.2 pts | 4 suites / 1192 active / 43 ignored / 977.2 pts | 4 suites / 1192 active / 43 ignored / 977.2 pts |
-| transcription | 5 suites / 824 active / 42 ignored / 646.6 pts | 5 suites / 824 active / 42 ignored / 646.6 pts | 5 suites / 824 active / 42 ignored / 646.6 pts |
-| ui-events | 4 suites / 765 active / 29 ignored / 581.1 pts | 3 suites / 713 active / 5 ignored / 544.7 pts | 3 suites / 713 active / 5 ignored / 544.7 pts |
-| vision-capture | 6 suites / 552 active / 32 ignored / 436.8 pts | 6 suites / 556 active / 32 ignored / 442.3 pts | 5 suites / 547 active / 31 ignored / 433.3 pts |
+| timeline | 4 suites / 1198 active / 43 ignored / 981.4 pts | 4 suites / 1198 active / 43 ignored / 981.4 pts | 4 suites / 1198 active / 43 ignored / 981.4 pts |
+| transcription | 5 suites / 839 active / 43 ignored / 657.1 pts | 5 suites / 839 active / 43 ignored / 657.1 pts | 5 suites / 839 active / 43 ignored / 657.1 pts |
+| ui-events | 4 suites / 781 active / 34 ignored / 592.6 pts | 3 suites / 714 active / 5 ignored / 545.7 pts | 3 suites / 714 active / 5 ignored / 545.7 pts |
+| vision-capture | 6 suites / 553 active / 32 ignored / 437.5 pts | 6 suites / 557 active / 32 ignored / 443.0 pts | 5 suites / 548 active / 31 ignored / 434.0 pts |
## Critical Flow Matrix
@@ -119,23 +119,23 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| Suite | Crate | Platforms | Layers | Flows | Criticality | Confidence | Kind | Files | Active | Ignored | Notes |
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| a11y-core-tree-cross-platform | screenpipe-a11y | windows, macos, linux | accessibility, ui-events, privacy, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | strong | unit | 14 | 171 | 0 | Cross-platform accessibility config, tree normalization, cache, privacy title matching, events, budget, and activity feed units. |
+| a11y-core-tree-cross-platform | screenpipe-a11y | windows, macos, linux | accessibility, ui-events, privacy, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | strong | unit | 14 | 172 | 0 | Cross-platform accessibility config, tree normalization, cache, privacy title matching, events, budget, and activity feed units. |
| a11y-linux-tree | screenpipe-a11y | linux | accessibility, privacy | accessibility-ui-events, privacy-and-redaction | medium | partial | unit | 4 | 27 | 1 | Linux-specific accessibility/incognito normalization tests. |
| a11y-macos-tree | screenpipe-a11y | macos | accessibility, privacy, real-app, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | conditional | mixed | 8 | 108 | 4 | macOS AX unit coverage, a scored 100-case click-attribution policy eval, and real TextEdit/Finder/Obsidian probes. Click attribution and Obsidian live tests are ignored by default when they require a logged-in desktop, app install, or AX permission. |
-| a11y-windows-tree | screenpipe-a11y | windows | accessibility, privacy, ui-events | accessibility-ui-events, privacy-and-redaction | high | partial | unit | 6 | 52 | 24 | Windows UIA/accessibility parsing and privacy matching; some UIA tests are ignored where they require a live desktop. |
+| a11y-windows-tree | screenpipe-a11y | windows | accessibility, privacy, ui-events | accessibility-ui-events, privacy-and-redaction | high | partial | unit | 7 | 67 | 29 | Windows UIA/accessibility parsing and privacy matching; some UIA tests are ignored where they require a live desktop. |
| audio-device-stream-health | screenpipe-audio | windows, macos, linux | audio-device, audio, performance | audio-device-health, audio-record-transcribe, performance-liveness | high | strong | mixed | 14 | 147 | 6 | Device monitor, device manager, stream buffering, source lag, audio metrics, Bluetooth gap/hallucination regressions, and cross-platform process-tap watchdog counters (process_tap.rs split into src/core/process_tap/ modules). |
| audio-meetings-speakers-dedup | screenpipe-audio | windows, macos, linux | audio, meeting, speaker, transcription | audio-record-transcribe, meeting-live-notes, performance-liveness | high | strong | mixed | 26 | 262 | 8 | Meeting streaming config/controller logic, speaker embedding state, cross-device dedupe simulations, and overlap cleanup coverage. |
| audio-models-filtering | screenpipe-audio | windows, macos, linux | audio, transcription, privacy | audio-record-transcribe, privacy-and-redaction | medium | partial | mixed | 6 | 20 | 10 | Model-download/TLS guards, ONNX startup smoke, and music-versus-speech filtering. |
| audio-pipeline-benchmarks | screenpipe-audio | windows, macos, linux | audio, transcription, performance | audio-record-transcribe, meeting-live-notes, performance-liveness | medium | partial | benchmark | 8 | 22 | 12 | Benchmark-backed regression probes for VAD, smart mode, meeting audio, quality, cross-device, and end-to-end pipeline timing. |
| audio-platform-output-capture | screenpipe-audio | windows, macos | audio-device, audio, meeting | audio-device-health, audio-record-transcribe, meeting-live-notes | high | partial | unit | 7 | 79 | 1 | OS-specific output/system-audio capture: CoreAudio process tap and SCK output watchdog plus VPIO health policy on macOS, per-process meeting audio taps on both platforms, and the Windows follow-the-audio output watchdog. Platform impl files are cfg-gated to their target OS. |
-| audio-transcription-pipeline | screenpipe-audio | windows, macos, linux | audio, transcription, performance | audio-record-transcribe, meeting-live-notes, performance-liveness | high | partial | mixed | 15 | 105 | 7 | Batch deferral, cleanup, language detection, result normalization, and real recording/transcription tests. Hardware/model-heavy tests are ignored by default. Also covers ffmpeg encode child reaping on error paths. |
+| audio-transcription-pipeline | screenpipe-audio | windows, macos, linux | audio, transcription, performance | audio-record-transcribe, meeting-live-notes, performance-liveness | high | partial | mixed | 15 | 115 | 8 | Batch deferral, cleanup, language detection, result normalization, and real recording/transcription tests. Hardware/model-heavy tests are ignored by default. Also covers ffmpeg encode child reaping on error paths. |
| db-accessibility-ui-events | screenpipe-db | windows, macos, linux | database, configuration, accessibility, ui-events, performance | settings-to-engine-config, accessibility-ui-events, performance-liveness | medium | partial | integration | 7 | 27 | 2 | Elements bulk insert, on-screen filtering, UI event batching, DB tier config, and ignored heavy-read real-DB probes. |
| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. |
| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 16 | 47 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 36 | 285 | 12 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
-| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 41 | 415 | 5 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
-| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
+| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 41 | 420 | 5 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
+| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 302 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |
| engine-db-recovery-cli | screenpipe-engine | windows, macos, linux | database, engine-lifecycle | engine-health-lifecycle, performance-liveness | high | strong | unit | 1 | 29 | 0 | Exact DB/WAL/SHM working-copy preservation, rollback on archive failure, and restart repair for crashes during the multi-file generation swap. |
| engine-focus-os | screenpipe-engine | windows, macos | engine-lifecycle, os-integration | engine-health-lifecycle, performance-liveness | medium | conditional | unit | 3 | 6 | 0 | Platform focus-tracker parsing/helpers. These files are cfg-gated and only execute on their target OS. |
@@ -173,8 +173,9 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| a11y-macos-tree | screenpipe-a11y | src/platform/macos_click_attribution_e2e.rs | source | 0 | 1 | 1 |
| a11y-macos-tree | screenpipe-a11y | src/platform/macos_click_attribution_eval.rs | source | 2 | 0 | 2 |
| a11y-macos-tree | screenpipe-a11y | src/platform/macos.rs | source | 23 | 0 | 23 |
+| a11y-windows-tree | screenpipe-a11y | src/platform/retained_tree.rs | source | 12 | 0 | 12 |
| a11y-windows-tree | screenpipe-a11y | src/platform/windows_uia_tests.rs | source | 0 | 12 | 12 |
-| a11y-windows-tree | screenpipe-a11y | src/platform/windows_uia.rs | source | 13 | 12 | 25 |
+| a11y-windows-tree | screenpipe-a11y | src/platform/windows_uia.rs | source | 15 | 12 | 27 |
| a11y-windows-tree | screenpipe-a11y | src/platform/windows.rs | source | 23 | 0 | 23 |
| a11y-core-tree-cross-platform | screenpipe-a11y | src/scroll.rs | source | 9 | 0 | 9 |
| a11y-core-tree-cross-platform | screenpipe-a11y | src/tree/app_version.rs | source | 3 | 0 | 3 |
@@ -185,9 +186,9 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| a11y-linux-tree | screenpipe-a11y | src/tree/linux.rs | source | 13 | 0 | 13 |
| a11y-macos-tree | screenpipe-a11y | src/tree/macos_lines.rs | source | 12 | 0 | 12 |
| a11y-macos-tree | screenpipe-a11y | src/tree/macos.rs | source | 52 | 0 | 52 |
-| a11y-core-tree-cross-platform | screenpipe-a11y | src/tree/mod.rs | source | 37 | 0 | 37 |
+| a11y-core-tree-cross-platform | screenpipe-a11y | src/tree/mod.rs | source | 38 | 0 | 38 |
| a11y-windows-tree | screenpipe-a11y | src/tree/windows_lines.rs | source | 2 | 0 | 2 |
-| a11y-windows-tree | screenpipe-a11y | src/tree/windows.rs | source | 12 | 0 | 12 |
+| a11y-windows-tree | screenpipe-a11y | src/tree/windows.rs | source | 13 | 5 | 18 |
| a11y-core-tree-cross-platform | screenpipe-a11y | src/url_filter.rs | source | 14 | 0 | 14 |
| a11y-macos-tree | screenpipe-a11y | tests/e2e_obsidian.rs | integration | 0 | 3 | 3 |
| a11y-macos-tree | screenpipe-a11y | tests/e2e_tree_walker.rs | integration | 8 | 0 | 8 |
@@ -207,7 +208,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio-device-stream-health | screenpipe-audio | src/core/process_tap/counters.rs | source | 2 | 0 | 2 |
| audio-platform-output-capture | screenpipe-audio | src/core/process_tap/macos.rs | source | 28 | 1 | 29 |
| audio-platform-output-capture | screenpipe-audio | src/core/process_tap/windows.rs | source | 12 | 0 | 12 |
-| audio-transcription-pipeline | screenpipe-audio | src/core/run_record_and_transcribe.rs | source | 10 | 0 | 10 |
+| audio-transcription-pipeline | screenpipe-audio | src/core/run_record_and_transcribe.rs | source | 15 | 0 | 15 |
| audio-platform-output-capture | screenpipe-audio | src/core/sck_output_watchdog.rs | source | 12 | 0 | 12 |
| audio-device-stream-health | screenpipe-audio | src/core/source_buffer.rs | source | 7 | 0 | 7 |
| audio-device-stream-health | screenpipe-audio | src/core/stream.rs | source | 12 | 0 | 12 |
@@ -233,14 +234,14 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio-transcription-pipeline | screenpipe-audio | src/transcription/deepgram/batch.rs | source | 11 | 0 | 11 |
| audio-transcription-pipeline | screenpipe-audio | src/transcription/deepgram/mod.rs | source | 1 | 0 | 1 |
| audio-transcription-pipeline | screenpipe-audio | src/transcription/engine.rs | source | 4 | 0 | 4 |
-| audio-transcription-pipeline | screenpipe-audio | src/transcription/openai_compatible/batch.rs | source | 7 | 0 | 7 |
+| audio-transcription-pipeline | screenpipe-audio | src/transcription/openai_compatible/batch.rs | source | 12 | 0 | 12 |
| audio-transcription-pipeline | screenpipe-audio | src/transcription/transcription_result.rs | source | 9 | 0 | 9 |
| audio-transcription-pipeline | screenpipe-audio | src/transcription/whisper/batch.rs | source | 4 | 0 | 4 |
| audio-transcription-pipeline | screenpipe-audio | src/transcription/whisper/detect_language.rs | source | 4 | 0 | 4 |
| audio-transcription-pipeline | screenpipe-audio | src/utils/audio/convert.rs | source | 8 | 0 | 8 |
| audio-models-filtering | screenpipe-audio | src/utils/audio/music_detection.rs | source | 6 | 0 | 6 |
| audio-transcription-pipeline | screenpipe-audio | src/utils/audio/resample.rs | source | 11 | 0 | 11 |
-| audio-transcription-pipeline | screenpipe-audio | src/utils/ffmpeg.rs | source | 4 | 0 | 4 |
+| audio-transcription-pipeline | screenpipe-audio | src/utils/ffmpeg.rs | source | 4 | 1 | 5 |
| audio-models-filtering | screenpipe-audio | src/utils/ort_watchdog.rs | source | 3 | 0 | 3 |
| audio-transcription-pipeline | screenpipe-audio | tests/accuracy_test.rs | integration | 0 | 1 | 1 |
| audio-pipeline-benchmarks | screenpipe-audio | tests/audio_pipeline_benchmark/audio_fixtures.rs | integration | 6 | 0 | 6 |
@@ -385,7 +386,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| engine-telemetry-observability | screenpipe-engine | src/crash_log.rs | source | 4 | 0 | 4 |
| engine-retention-storage | screenpipe-engine | src/disk_pressure.rs | source | 11 | 0 | 11 |
| engine-meeting-privacy-sync | screenpipe-engine | src/drm_detector.rs | source | 19 | 2 | 21 |
-| engine-capture-timeline | screenpipe-engine | src/event_driven_capture.rs | source | 94 | 0 | 94 |
+| engine-capture-timeline | screenpipe-engine | src/event_driven_capture.rs | source | 95 | 0 | 95 |
| engine-meeting-privacy-sync | screenpipe-engine | src/external_memory_sync.rs | source | 13 | 0 | 13 |
| engine-capture-timeline | screenpipe-engine | src/focus_aware_controller.rs | source | 14 | 0 | 14 |
| engine-focus-os | screenpipe-engine | src/focus_tracker/darwin.rs | source | 3 | 0 | 3 |
@@ -440,7 +441,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| engine-api-routes | screenpipe-engine | src/routes/data.rs | source | 2 | 0 | 2 |
| engine-api-routes | screenpipe-engine | src/routes/elements.rs | source | 25 | 1 | 26 |
| engine-api-routes | screenpipe-engine | src/routes/frames.rs | source | 12 | 1 | 13 |
-| engine-api-routes | screenpipe-engine | src/routes/health.rs | source | 48 | 0 | 48 |
+| engine-api-routes | screenpipe-engine | src/routes/health.rs | source | 53 | 0 | 53 |
| engine-api-routes | screenpipe-engine | src/routes/live_views.rs | source | 1 | 0 | 1 |
| engine-api-routes | screenpipe-engine | src/routes/meeting_summary_status.rs | source | 10 | 0 | 10 |
| engine-api-routes | screenpipe-engine | src/routes/meetings.rs | source | 7 | 0 | 7 |
diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.json
index 741fcb069..bd8b04bac 100644
--- a/docs/coverage/core-engine-map.json
+++ b/docs/coverage/core-engine-map.json
@@ -1344,6 +1344,7 @@
"crate": "screenpipe-a11y",
"files": [
"src/incognito/windows.rs",
+ "src/platform/retained_tree.rs",
"src/platform/windows.rs",
"src/platform/windows_uia.rs",
"src/platform/windows_uia_tests.rs",
diff --git a/docs/health-diagnostics.md b/docs/health-diagnostics.md
new file mode 100644
index 000000000..651fd716a
--- /dev/null
+++ b/docs/health-diagnostics.md
@@ -0,0 +1,102 @@
+# Health dependency and startup diagnostics
+
+<!-- doc-covers: crates/screenpipe-core/src/health_diagnostics.rs, crates/screenpipe-engine/src/routes/health.rs -->
+<!-- doc-verified: f02a3d069 -->
+
+`GET /health` adds `dependencies`, `runtime`, `startup`, and
+`unhealthy_reasons`. Existing response fields deserialize unchanged and existing
+`message` text is preserved. New consumers should use the fault array for
+causes and remedies; a newly detected dependency fault can make `status`
+`degraded` and `status_code` 503 while the legacy pipeline message stays the same.
+
+Health reads copy observed state. They do not discover or install binaries,
+start model downloads, enumerate processes, or read startup records from disk.
+The existing one-second health cache applies to these fields too. Snapshot
+serialization happens after releasing the observation lock. Startup checkpoint
+writers serialize with a separate mutex, and release the observation lock before
+filesystem operations, so a slow checkpoint cannot hold up audio lifecycle
+updates or health reads. Audio completion history is capped at 4,096 events and
+expired entries are removed from the front without rescanning the whole buffer.
+Checkpoint writes still run synchronously on the startup caller; they do not run
+in audio callbacks.
+
+## Observed state
+
+- `dependencies.ffmpeg`: resolved path and source, one bounded background
+ version probe per changed path, installation state and retry deadline,
+ separate non-fatal version-probe errors, latest process failure, running HD/audio/extraction operations, rolling
+ audio completion/failure counts, compaction activity, and HD timeout count.
+- `dependencies.ffprobe`: sibling/PATH discovery when FFmpeg resolves. Missing
+ ffprobe is informational; it does not establish a recording failure.
+- `dependencies.models`: selected transcription engine, initialization,
+ local-model download requests, Whisper download completion/failure, and
+ VAD/speaker session initialization outcomes. `downloaded` means weights
+ arrived; only a successful engine initialization reports `ready`.
+- `runtime.onnxruntime`: actual guarded session outcomes, timeouts, and affected
+ features. A model/session initialization error is `init_failed`, not a claim
+ that a particular DLL is missing. CPU compatibility mode is independent.
+- `startup`: boot phase, completed initialization checks, environment override
+ **names only**, and the most recent failed or interrupted start. Timestamps
+ in this contract are Unix seconds. The atomic `health-startup.json` checkpoint
+ resides in the configured data directory and survives restarts. A previous
+ attempt that never reached ready has an `unknown` termination cause; it does
+ not invent a signal or exit code. Previous failures do not degrade a healthy
+ current session.
+- `unhealthy_reasons`: stable snake_case code, subsystem, warning severity,
+ detail, remedy, and observed timestamp when available. It covers current
+ media/model/runtime faults and the existing vision, audio, writer, and
+ configured-but-stopped UI-recorder states. Snapshot timeout has its own code.
+
+Process guards release counters on cancellation. A successful operation clears
+only its own pipeline's failure. Media failures without a later observation
+age out of current degradation after three minutes; historical error details
+remain. Downloading, unknown states, and version-probe timeouts do not by
+themselves cause degradation. A successful metadata probe cannot clear an actual
+encode failure.
+Audio-disabled runs suppress audio model/runtime faults. No capture admission,
+encoding, download, or recovery policy is changed by this instrumentation.
+
+## Remaining coverage
+
+This is the first implementation of the broader diagnostics request, not the
+complete platform dependency contract:
+
+- MSVC, Swift, crash-helper presence, and exact loaded ONNX path/version/source
+ remain `unknown` on applicable platforms. A process killed by the OS loader
+ before `main` cannot write a checkpoint; it requires launcher-side reporting.
+- Audiopipe's detached Parakeet/Qwen download API has no completion callback
+ here. State is refreshed when the engine next initializes; only Whisper's
+ download callback currently publishes a definitive download failure.
+- Extraction running counts cover timeline frame extraction. Queue depth is
+ `null` (unobserved), and export/merge/probe call sites are not yet instrumented.
+ A corrupt input file remains a request error, not an installation diagnosis.
+- Bootstrap is observed at the existing CLI/desktop boundaries; the HTTP server
+ is not moved ahead of database/audio initialization. During a failed boot,
+ inspect the checkpoint or logs; the next successful boot exposes its history.
+- Runtime settings changes can leave dependency requirement flags conservative
+ until the next recording-engine restart. Observed media operations still mark
+ FFmpeg as required when used.
+
+The domain allow/block request is already implemented separately in PR #6685;
+this change does not backport it or change domain filtering.
+
+## Focused validation
+
+The diagnostics regression suite exercises blocked checkpoint I/O, failed-write
+recovery, bounded counters, per-pipeline fault recovery, stale model callbacks,
+and the healthy/degraded status matrix. Two opt-in local evals avoid timing gates
+on shared CI hosts:
+
+```sh
+cargo test -p screenpipe-core health_diagnostics --lib
+cargo test -p screenpipe-engine --lib routes::health
+cargo test -p screenpipe-audio --lib utils::
+cargo test -p screenpipe-core health_diagnostics_lifecycle_latency_eval --lib -- --ignored --nocapture
+cargo test -p screenpipe-audio --lib health_diagnostics_audio_encode_eval -- --ignored --nocapture
+```
+
+The encoder eval requires FFmpeg, encodes synthetic ten-second PCM in alternating
+instrumented/uninstrumented pairs, and checks that decoded audio is identical.
+It opens no audio devices and uses no private recordings. These measurements do
+not replace platform E2E, microphone/device-switch testing, or speech-quality
+evals.