Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

Reduce healthy diagnostic telemetry while preserving failure reports

Louis Beaumontda8c9c67b12f
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
index 0adf82294..b1f20516c 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
@@ -649,7 +649,10 @@ impl ServerCore {
// --- Resource + sleep monitors (long-lived) ---
let resource_reporter = ResourceTelemetryReporter::new(config.analytics_enabled);
- resource_reporter.start_monitoring(Duration::from_secs(30), Some(Duration::from_secs(60)));
+ resource_reporter.start_monitoring(
+ Duration::from_secs(30),
+ Some(screenpipe_engine::telemetry_reporting::DIAGNOSTIC_REPORT_INTERVAL),
+ );
start_sleep_monitor();
// --- HTTP server ---
diff --git a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs b/crates/screenpipe-engine/src/bin/screenpipe-engine.rs
index 6843f9323..2f4c6e636 100644
--- a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs
+++ b/crates/screenpipe-engine/src/bin/screenpipe-engine.rs
@@ -1067,7 +1067,10 @@ async fn run(cli: Cli, record_arg_sources: RecordArgSources) -> anyhow::Result<(
let audio_devices_clone = audio_devices.clone();
let resource_reporter = ResourceTelemetryReporter::new(config.analytics_enabled);
- resource_reporter.start_monitoring(Duration::from_secs(30), Some(Duration::from_secs(60)));
+ resource_reporter.start_monitoring(
+ Duration::from_secs(30),
+ Some(screenpipe_engine::telemetry_reporting::DIAGNOSTIC_REPORT_INTERVAL),
+ );
// Initialize analytics for API tracking
analytics::init(config.analytics_enabled);
diff --git a/crates/screenpipe-engine/src/lib.rs b/crates/screenpipe-engine/src/lib.rs
index be3590617..4a1f86488 100644
--- a/crates/screenpipe-engine/src/lib.rs
+++ b/crates/screenpipe-engine/src/lib.rs
@@ -100,6 +100,7 @@ pub mod structured_outputs;
mod sync_api;
pub mod sync_provider;
pub mod telemetry_context;
+pub mod telemetry_reporting;
mod telemetry_tasks;
pub mod ui_recorder;
// Exposed publicly so the commercial `screenpipe-sdk` (screenpipe/sdk repo)
diff --git a/crates/screenpipe-engine/src/resource_monitor.rs b/crates/screenpipe-engine/src/resource_monitor.rs
index bcfe5e988..d0759a486 100644
--- a/crates/screenpipe-engine/src/resource_monitor.rs
+++ b/crates/screenpipe-engine/src/resource_monitor.rs
@@ -792,7 +792,8 @@ impl ResourceTelemetryReporter {
// analytics/debug/file logging are disabled.
let monitor = Arc::clone(self);
let posthog_interval = posthog_interval.unwrap_or(interval);
- let mut last_posthog_update = Instant::now();
+ // Keep a startup sample for short sessions, then send window summaries.
+ let mut last_posthog_update: Option<Instant> = None;
let mut window = ResourceReportWindow::default();
tokio::spawn(async move {
@@ -835,10 +836,11 @@ impl ResourceTelemetryReporter {
snapshot.phys_footprint_gb,
);
let now = Instant::now();
- let should_send_to_posthog = now.duration_since(last_posthog_update) >= posthog_interval;
+ let should_send_to_posthog = last_posthog_update
+ .is_none_or(|last| now.duration_since(last) >= posthog_interval);
if should_send_to_posthog {
- last_posthog_update = now;
+ last_posthog_update = Some(now);
monitor.log_status(&snapshot, &std::mem::take(&mut window)).await;
} else {
monitor.log_status_local(&snapshot).await;
diff --git a/crates/screenpipe-engine/src/server.rs b/crates/screenpipe-engine/src/server.rs
index 35d8de3b7..c6e597279 100644
--- a/crates/screenpipe-engine/src/server.rs
+++ b/crates/screenpipe-engine/src/server.rs
@@ -13,6 +13,8 @@ use screenpipe_audio::audio_manager::AudioManager;
use screenpipe_core::sync::SyncServiceHandle;
use tracing::{debug, error, info};
+use crate::telemetry_reporting::HealthReportSchedule;
+
use crate::{
analytics,
history_access::HistoryAccessPolicy,
@@ -652,15 +654,22 @@ impl SCServer {
}
});
- // Spawn periodic vision pipeline metrics reporter (every 60 seconds)
+ // Check pipeline failures every minute; report healthy summaries every 15 minutes.
let metrics_for_posthog = self.vision_metrics.clone();
telemetry_tasks.spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(60));
+ let started = std::time::Instant::now();
+ let mut reports = HealthReportSchedule::new();
loop {
interval.tick().await;
let snap = metrics_for_posthog.snapshot();
// Only report if the pipeline has captured any frames
- if snap.frames_captured > 0 {
+ if snap.frames_captured > 0
+ && reports.should_report(
+ started.elapsed(),
+ [snap.frames_dropped, snap.pipeline_stall_count],
+ )
+ {
// Recording-coverage reliability metric: what % of the
// user's working time had healthy capture. Sampled
// independently (5s sampler); snapshotted here so the
@@ -670,6 +679,7 @@ impl SCServer {
"vision_pipeline_health",
json!({
"uptime_secs": snap.uptime_secs,
+ "telemetry_report_interval_secs": crate::telemetry_reporting::DIAGNOSTIC_REPORT_INTERVAL.as_secs(),
"frames_captured": snap.frames_captured,
"frames_skipped": snap.frames_skipped,
"ocr_completed": snap.ocr_completed,
@@ -706,26 +716,42 @@ impl SCServer {
}
});
- // Spawn periodic audio pipeline metrics reporter (every 60 seconds)
+ // Keep minute-level failure/device checks without sending healthy polls.
let audio_metrics_for_posthog = self.audio_metrics.clone();
let audio_manager_for_posthog = self.audio_manager.clone();
telemetry_tasks.spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(60));
+ let started = std::time::Instant::now();
+ let mut reports = HealthReportSchedule::new();
loop {
interval.tick().await;
let snap = audio_metrics_for_posthog.snapshot();
// Only report if the pipeline has processed any chunks
if snap.chunks_sent > 0 || snap.vad_rejected > 0 {
- let devices: Vec<String> = audio_manager_for_posthog
+ let mut devices: Vec<String> = audio_manager_for_posthog
.current_devices()
.iter()
.map(|d| d.to_string())
.collect();
+ devices.sort_unstable();
let device_count = devices.len();
+ let failure_counts = [
+ snap.chunks_channel_full,
+ snap.stream_timeouts,
+ snap.chunks_lagged,
+ snap.process_errors,
+ snap.transcription_errors,
+ ];
+ if !reports
+ .should_report(started.elapsed(), (failure_counts, devices.clone()))
+ {
+ continue;
+ }
analytics::capture_event_nonblocking(
"audio_pipeline_health",
json!({
"uptime_secs": snap.uptime_secs,
+ "telemetry_report_interval_secs": crate::telemetry_reporting::DIAGNOSTIC_REPORT_INTERVAL.as_secs(),
"chunks_sent": snap.chunks_sent,
"chunks_received": snap.chunks_received,
"chunks_channel_full": snap.chunks_channel_full,
diff --git a/crates/screenpipe-engine/src/telemetry_reporting.rs b/crates/screenpipe-engine/src/telemetry_reporting.rs
new file mode 100644
index 000000000..d75fee9ed
--- /dev/null
+++ b/crates/screenpipe-engine/src/telemetry_reporting.rs
@@ -0,0 +1,90 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+use std::time::Duration;
+
+/// Healthy diagnostics are summaries, not activity events. Local sampling and
+/// failure checks keep their existing cadence.
+pub const DIAGNOSTIC_REPORT_INTERVAL: Duration = Duration::from_secs(15 * 60);
+
+/// Report startup, changed failure/device state, and a periodic heartbeat.
+/// State contains only failure counters and device identity, never work counters:
+/// increasing frames/chunks must not turn every healthy sample into a report.
+pub(crate) struct HealthReportSchedule<T> {
+ last_report: Option<(Duration, T)>,
+}
+
+impl<T: PartialEq> HealthReportSchedule<T> {
+ pub(crate) fn new() -> Self {
+ Self { last_report: None }
+ }
+
+ pub(crate) fn should_report(&mut self, elapsed: Duration, state: T) -> bool {
+ let report = self.last_report.as_ref().is_none_or(|(last, previous)| {
+ elapsed.saturating_sub(*last) >= DIAGNOSTIC_REPORT_INTERVAL || *previous != state
+ });
+ if report {
+ self.last_report = Some((elapsed, state));
+ }
+ report
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn healthy_minute_checks_emit_four_snapshots_per_hour() {
+ let mut schedule = HealthReportSchedule::new();
+ let reports: Vec<_> = (0..60)
+ .filter(|minute| schedule.should_report(Duration::from_secs(minute * 60), [0, 0]))
+ .collect();
+ assert_eq!(reports, [0, 15, 30, 45]);
+ }
+
+ #[test]
+ fn new_failures_report_on_the_next_check_without_waiting_for_heartbeat() {
+ let mut schedule = HealthReportSchedule::new();
+ assert!(schedule.should_report(Duration::ZERO, [0, 0]));
+ assert!(schedule.should_report(Duration::from_secs(60), [1, 0]));
+ assert!(!schedule.should_report(Duration::from_secs(120), [1, 0]));
+ assert!(schedule.should_report(Duration::from_secs(180), [1, 1]));
+ }
+
+ #[test]
+ fn repeated_failures_are_not_sampled_away() {
+ let mut schedule = HealthReportSchedule::new();
+ for minute in 0..60 {
+ assert!(schedule.should_report(Duration::from_secs(minute * 60), [minute]));
+ }
+ }
+
+ #[test]
+ fn device_replacement_with_same_count_is_reported() {
+ let mut schedule = HealthReportSchedule::new();
+ assert!(schedule.should_report(Duration::ZERO, ([0], vec!["mic-a"])));
+ assert!(schedule.should_report(Duration::from_secs(60), ([0], vec!["mic-b"])));
+ assert!(!schedule.should_report(Duration::from_secs(120), ([0], vec!["mic-b"])));
+ }
+
+ #[test]
+ fn counter_reset_and_fresh_reporter_are_reported() {
+ let mut schedule = HealthReportSchedule::new();
+ assert!(schedule.should_report(Duration::from_secs(300), [10]));
+ assert!(schedule.should_report(Duration::from_secs(360), [0]));
+ let mut restarted = HealthReportSchedule::new();
+ assert!(restarted.should_report(Duration::ZERO, [0]));
+ }
+
+ #[test]
+ fn exact_boundary_and_long_sleep_report_once() {
+ let mut schedule = HealthReportSchedule::new();
+ assert!(schedule.should_report(Duration::ZERO, [0]));
+ assert!(!schedule.should_report(DIAGNOSTIC_REPORT_INTERVAL - Duration::from_secs(1), [0]));
+ assert!(schedule.should_report(DIAGNOSTIC_REPORT_INTERVAL, [0]));
+ let after_sleep = Duration::from_secs(24 * 3600);
+ assert!(schedule.should_report(after_sleep, [0]));
+ assert!(!schedule.should_report(after_sleep, [0]));
+ }
+}
diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md
index 0645d43ff..d757f1ff4 100644
--- a/docs/COVERAGE.md
+++ b/docs/COVERAGE.md
@@ -37,16 +37,16 @@ results and `cargo llvm-cov` data on top when judging release confidence.
### Core Engine
- Mapped suites: 37
-- Mapped Rust files: 406
-- Active test blocks: 3952
+- Mapped Rust files: 407
+- Active test blocks: 3958
- Ignored/manual test blocks: 189
-- Weighted coverage points: 3257.2
+- Weighted coverage points: 3261.4
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 33 | 3789 | 171 | 3184.2 | 21 | 11 | 100% |
-| macos | 32 | 3832 | 150 | 3179.5 | 22 | 11 | 100% |
-| linux | 29 | 3367 | 146 | 2802.5 | 20 | 11 | 100% |
+| windows | 33 | 3795 | 171 | 3188.4 | 21 | 11 | 100% |
+| macos | 32 | 3838 | 150 | 3183.7 | 22 | 11 | 100% |
+| linux | 29 | 3373 | 146 | 2806.7 | 20 | 11 | 100% |
## Refresh
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 18cccd33a..831fb34e4 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -8,11 +8,11 @@ confidence, and criticality.
- Manifest: `docs/coverage/core-engine-map.json`
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs
- Mapped suites: 37
-- Mapped Rust files: 406
-- Active test blocks: 3952
+- Mapped Rust files: 407
+- Active test blocks: 3958
- Ignored/manual test blocks: 189
-- Declared test blocks: 4141
-- Weighted coverage points: 3257.2
+- Declared test blocks: 4147
+- Weighted coverage points: 3261.4
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,15 +23,15 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 33 | 3789 | 171 | 3184.2 | 21 | 11 | 100% |
-| macos | 32 | 3832 | 150 | 3179.5 | 22 | 11 | 100% |
-| linux | 29 | 3367 | 146 | 2802.5 | 20 | 11 | 100% |
+| windows | 33 | 3795 | 171 | 3188.4 | 21 | 11 | 100% |
+| macos | 32 | 3838 | 150 | 3183.7 | 22 | 11 | 100% |
+| linux | 29 | 3373 | 146 | 2806.7 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| screenpipe-engine | 12 | 22 | 132 | 1882 | 53 | 1448.1 | 10 |
+| screenpipe-engine | 12 | 22 | 133 | 1888 | 53 | 1452.3 | 10 |
| screenpipe-db | 5 | 69 | 29 | 640 | 32 | 609.2 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
| screenpipe-audio | 8 | 29 | 54 | 706 | 56 | 617.0 | 5 |
@@ -68,12 +68,12 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| configuration | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts |
| database | 7 suites / 596 active / 28 ignored / 565.2 pts | 7 suites / 596 active / 28 ignored / 565.2 pts | 7 suites / 596 active / 28 ignored / 565.2 pts |
| db-search | 2 suites / 120 active / 9 ignored / 120.0 pts | 2 suites / 120 active / 9 ignored / 120.0 pts | 2 suites / 120 active / 9 ignored / 120.0 pts |
-| engine-lifecycle | 6 suites / 274 active / 1 ignored / 244.8 pts | 6 suites / 274 active / 1 ignored / 244.8 pts | 6 suites / 270 active / 9 ignored / 243.9 pts |
+| engine-lifecycle | 6 suites / 280 active / 1 ignored / 249.0 pts | 6 suites / 280 active / 1 ignored / 249.0 pts | 6 suites / 276 active / 9 ignored / 248.1 pts |
| local-api | 3 suites / 494 active / 12 ignored / 349.7 pts | 3 suites / 494 active / 12 ignored / 349.7 pts | 3 suites / 494 active / 12 ignored / 349.7 pts |
| meeting | 6 suites / 1765 active / 32 ignored / 1442.8 pts | 6 suites / 1765 active / 32 ignored / 1442.8 pts | 4 suites / 1411 active / 28 ignored / 1112.5 pts |
| ocr | 4 suites / 141 active / 7 ignored / 130.8 pts | 4 suites / 134 active / 7 ignored / 128.6 pts | 3 suites / 125 active / 6 ignored / 119.6 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1730 active / 87 ignored / 1528.3 pts | 14 suites / 1855 active / 95 ignored / 1578.3 pts | 13 suites / 1730 active / 87 ignored / 1528.3 pts |
+| performance | 13 suites / 1736 active / 87 ignored / 1532.5 pts | 14 suites / 1861 active / 95 ignored / 1582.5 pts | 13 suites / 1736 active / 87 ignored / 1532.5 pts |
| pipes | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts |
| privacy | 5 suites / 1031 active / 43 ignored / 833.9 pts | 5 suites / 1087 active / 22 ignored / 835.6 pts | 5 suites / 989 active / 15 ignored / 798.8 pts |
| real-app | - | 1 suites / 125 active / 8 ignored / 50.0 pts | - |
@@ -146,7 +146,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| engine-meeting-privacy-sync | screenpipe-engine | windows, macos, linux | meeting, privacy, ui-events, pipes, sync | meeting-live-notes, privacy-and-redaction, accessibility-ui-events, performance-liveness | medium | strong | unit | 33 | 554 | 3 | Unit-heavy coverage for privacy filter policy, capture exclusions, UI recorder safety, pipes/live-view/structured-output helpers, sync helpers, and CLI parsing. Meeting detection moved to the engine-meeting-watcher suite. |
| engine-meeting-watcher | screenpipe-engine | windows, macos | meeting | meeting-live-notes | high | strong | mixed | 10 | 275 | 3 | Successor of src/meeting_detector.rs and src/meeting_telemetry.rs. Audio-process and UI-scan backend state machines, candidate resolution, shared telemetry, and a scored trajectory eval. ui_scan/macos.rs and ui_scan/windows.rs are cfg-gated and only execute on their target OS; both backends are null on Linux. |
| engine-retention-storage | screenpipe-engine | windows, macos, linux | storage, engine-lifecycle, performance | engine-health-lifecycle, performance-liveness | medium | strong | mixed | 5 | 43 | 0 | Local retention deletion (including lean-mode heavy-text stripping), cloud archive watermarking, atomic state-file replacement, and the low-disk capture monitor. |
-| engine-telemetry-observability | screenpipe-engine | windows, macos, linux | engine-lifecycle, performance | engine-health-lifecycle, performance-liveness | medium | strong | unit | 8 | 40 | 0 | PostHog capture gating, telemetry context shaping, piggyback telemetry forwarding, crash-log helpers, resource monitoring, and the recording-coverage reliability metric. |
+| engine-telemetry-observability | screenpipe-engine | windows, macos, linux | engine-lifecycle, performance | engine-health-lifecycle, performance-liveness | medium | strong | unit | 9 | 46 | 0 | PostHog capture gating, telemetry context shaping, piggyback telemetry forwarding, crash-log helpers, resource monitoring, and the recording-coverage reliability metric. |
| engine-workflows | screenpipe-engine | windows, macos, linux | local-api, storage | local-api-search | high | partial | mixed | 7 | 40 | 0 | Catalog identity and persistence, revision-scoped feedback, workflow evidence and pipeline contracts, and denial of agent-granted rollout access. Does not assert live cloud discovery quality. Workflow publication orchestration deadlines and nested query backpressure are covered with virtual-time router tests. |
| filesystem-durable-sync | screenpipe-fs | macos, linux, windows | database, vision-capture, audio | capture-ocr-pipeline, audio-record-transcribe, engine-health-lifecycle | high | strong | unit | 1 | 3 | 0 | Durable flushes preserve local full-sync behavior, fall back to fsync only for macOS ENOTSUP, and propagate real I/O or fallback failures. Real SMB migration and continued recording are covered by the marked-volume integration test. |
| screen-capture-ocr-contract | screenpipe-screen | windows, macos, linux | vision-capture, ocr | capture-ocr-pipeline | high | partial | unit | 3 | 18 | 0 | Cross-platform cached-OCR unit coverage for RawCaptureResult to CaptureResult metadata, browser URL, focus state, window-to-screen OCR coordinate transformation, Tesseract output parsing, and contour-based text-region detection for the meeting OCR gate. |
@@ -502,6 +502,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| engine-meeting-privacy-sync | screenpipe-engine | src/sync_api.rs | source | 13 | 0 | 13 |
| engine-meeting-privacy-sync | screenpipe-engine | src/sync_provider.rs | source | 4 | 0 | 4 |
| engine-telemetry-observability | screenpipe-engine | src/telemetry_context.rs | source | 8 | 0 | 8 |
+| engine-telemetry-observability | screenpipe-engine | src/telemetry_reporting.rs | source | 6 | 0 | 6 |
| engine-telemetry-observability | screenpipe-engine | src/telemetry_tasks.rs | source | 2 | 0 | 2 |
| engine-meeting-privacy-sync | screenpipe-engine | src/ui_recorder.rs | source | 65 | 0 | 65 |
| engine-capture-timeline | screenpipe-engine | src/video_cache.rs | source | 4 | 0 | 4 |
diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.json
index 629564f9c..92b63c404 100644
--- a/docs/coverage/core-engine-map.json
+++ b/docs/coverage/core-engine-map.json
@@ -587,6 +587,7 @@
"src/recording_coverage.rs",
"src/resource_monitor.rs",
"src/telemetry_context.rs",
+ "src/telemetry_reporting.rs",
"src/telemetry_tasks.rs"
],
"platforms": [