Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

Sample repetitive unattributed API search diagnostics (#7503)

Louis Beaumonta7ee891cec6e
diff --git a/crates/screenpipe-engine/src/analytics.rs b/crates/screenpipe-engine/src/analytics.rs
index 926ad72ed..3469b36ea 100644
--- a/crates/screenpipe-engine/src/analytics.rs
+++ b/crates/screenpipe-engine/src/analytics.rs
@@ -5,7 +5,8 @@
use once_cell::sync::Lazy;
use reqwest::Client;
use serde_json::{json, Value};
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::time::{Duration, Instant};
use tracing::{debug, trace};
use crate::telemetry_context::TelemetryContext;
@@ -22,6 +23,73 @@ static TELEMETRY_ENABLED: AtomicBool = AtomicBool::new(false);
static ANALYTICS: Lazy<Analytics> = Lazy::new(Analytics::new);
+pub(crate) const SEARCH_REPORT_INTERVAL: Duration = Duration::from_secs(60);
+
+static SEARCH_REPORT_START: Lazy<Instant> = Lazy::new(Instant::now);
+static SEARCH_REPORTS: SearchReportLimiter = SearchReportLimiter::new();
+
+/// Only unattributed, explicitly API-originated searches share a report budget.
+/// Empty/non-empty and parsed/ordinary results have separate slots so an empty
+/// poll cannot hide the first successful retrieval or parsed-search presence.
+/// Fixed storage keeps untrusted request properties out of the limiter's keys.
+pub(crate) struct SearchReportLimiter {
+ windows: [AtomicU64; 4],
+}
+
+impl SearchReportLimiter {
+ pub(crate) const fn new() -> Self {
+ Self {
+ windows: [const { AtomicU64::new(0) }; 4],
+ }
+ }
+
+ pub(crate) fn should_report(
+ &self,
+ sampled: bool,
+ parsed: bool,
+ non_empty: bool,
+ window: u64,
+ ) -> bool {
+ if !sampled {
+ return true;
+ }
+ let slot = usize::from(parsed) * 2 + usize::from(non_empty);
+ claim_search_window(&self.windows[slot], window)
+ }
+}
+
+fn claim_search_window(last_window: &AtomicU64, window: u64) -> bool {
+ last_window
+ .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |last| {
+ (window > last).then_some(window)
+ })
+ .is_ok()
+}
+
+/// Let the first request after UTC midnight report even when it shares the
+/// previous day's monotonic minute. This preserves daily activity presence.
+/// The high bits keep progress monotonic if the wall clock moves backwards;
+/// the low bits distinguish days within a minute. Both fields fit for thousands
+/// of years, and one atomic claim still arbitrates concurrent requests.
+fn search_report_window(elapsed: Duration, utc_timestamp: i64) -> u64 {
+ let minute = elapsed.as_secs() / SEARCH_REPORT_INTERVAL.as_secs();
+ let utc_day = utc_timestamp.div_euclid(86_400).max(0) as u64 + 1;
+ (minute << 32) | (utc_day & u64::from(u32::MAX))
+}
+
+/// Consent is checked before claiming a slot or constructing a payload.
+/// App, MCP and named-agent requests bypass the API diagnostic limiter.
+pub(crate) fn should_capture_search(sampled: bool, parsed: bool, non_empty: bool) -> bool {
+ if !is_enabled() {
+ return false;
+ }
+ let window = search_report_window(
+ SEARCH_REPORT_START.elapsed(),
+ chrono::Utc::now().timestamp(),
+ );
+ SEARCH_REPORTS.should_report(sampled, parsed, non_empty, window)
+}
+
pub struct Analytics {
client: Client,
distinct_id: String,
@@ -241,6 +309,87 @@ mod tests {
use super::*;
use std::sync::Mutex;
+ #[test]
+ fn api_polling_preserves_each_result_kind_and_unlimited_unsampled_requests() {
+ let reports = SearchReportLimiter::new();
+ assert!(reports.should_report(true, false, false, 1));
+ for _ in 0..100_000 {
+ assert!(!reports.should_report(true, false, false, 1));
+ assert!(reports.should_report(false, false, false, 1));
+ }
+ // The empty API flood must not hide successful or parsed results.
+ assert!(reports.should_report(true, false, true, 1));
+ assert!(reports.should_report(true, true, false, 1));
+ assert!(reports.should_report(true, true, true, 1));
+ for parsed in [false, true] {
+ for non_empty in [false, true] {
+ assert!(!reports.should_report(true, parsed, non_empty, 1));
+ assert!(reports.should_report(true, parsed, non_empty, 2));
+ }
+ }
+ }
+
+ #[test]
+ fn unsampled_searches_do_not_consume_the_api_budget() {
+ let reports = SearchReportLimiter::new();
+ for _ in 0..100 {
+ assert!(reports.should_report(false, false, true, 1));
+ }
+ assert!(reports.should_report(true, false, true, 1));
+ }
+
+ #[test]
+ fn midnight_preserves_daily_presence_and_clock_rollback_does_not_stall_reporting() {
+ let reports = SearchReportLimiter::new();
+ let before_midnight = search_report_window(Duration::from_secs(10), 86_399);
+ let after_midnight = search_report_window(Duration::from_secs(11), 86_400);
+ assert!(reports.should_report(true, false, true, before_midnight));
+ assert!(reports.should_report(true, false, true, after_midnight));
+ assert!(!reports.should_report(true, false, true, after_midnight));
+ // A delayed previous-day caller cannot reopen an already used slot.
+ assert!(!reports.should_report(true, false, true, before_midnight));
+ // A clock correction cannot suppress reports beyond the current minute.
+ let rolled_back = search_report_window(Duration::from_secs(60), 86_000);
+ assert!(reports.should_report(true, false, true, rolled_back));
+ assert!(!reports.should_report(true, false, true, rolled_back));
+ }
+
+ #[test]
+ fn search_reports_bound_a_flood_and_allow_the_next_window() {
+ let window = AtomicU64::new(0);
+ assert!(claim_search_window(&window, 1));
+ for _ in 0..100_000 {
+ assert!(!claim_search_window(&window, 1));
+ }
+ assert!(claim_search_window(&window, 2));
+ // A delayed concurrent caller must not move the window backwards.
+ assert!(!claim_search_window(&window, 1));
+ assert!(!claim_search_window(&window, 2));
+ assert!(claim_search_window(&window, 180));
+ assert!(!claim_search_window(&window, 180));
+ }
+
+ #[test]
+ fn concurrent_searches_share_one_report_budget() {
+ let window = AtomicU64::new(0);
+ let barrier = std::sync::Barrier::new(16);
+ let emitted = std::thread::scope(|scope| {
+ let workers: Vec<_> = (0..16)
+ .map(|_| {
+ scope.spawn(|| {
+ barrier.wait();
+ claim_search_window(&window, 1)
+ })
+ })
+ .collect();
+ workers
+ .into_iter()
+ .map(|worker| usize::from(worker.join().unwrap()))
+ .sum::<usize>()
+ });
+ assert_eq!(emitted, 1);
+ }
+
// The three vars telemetry_disabled_by_env inspects. CI runners set
// CI/GITHUB_ACTIONS for real, so every test saves and restores them.
static ENV_LOCK: Mutex<()> = Mutex::new(());
diff --git a/crates/screenpipe-engine/src/routes/request_origin.rs b/crates/screenpipe-engine/src/routes/request_origin.rs
index 12bca42f6..b5d501eaf 100644
--- a/crates/screenpipe-engine/src/routes/request_origin.rs
+++ b/crates/screenpipe-engine/src/routes/request_origin.rs
@@ -31,6 +31,11 @@ impl ExplicitApiClient {
self.source == RequestSource::Api
}
+ /// Preserve app/MCP engagement and all named-agent attribution in full.
+ pub(crate) fn sample_search_diagnostics(&self) -> bool {
+ self.is_direct_api() && self.agent == AgentClient::Unknown
+ }
+
pub(crate) fn source_label(&self) -> &'static str {
match self.source {
RequestSource::App => "app",
@@ -129,6 +134,34 @@ mod tests {
assert_eq!(extract(None).await.source_label(), "app");
}
+ #[tokio::test]
+ async fn api_flood_cannot_suppress_app_mcp_or_named_agent_searches() {
+ let reports = crate::analytics::SearchReportLimiter::new();
+ let anonymous_api = extract(Some("api")).await;
+ assert!(anonymous_api.sample_search_diagnostics());
+ assert!(reports.should_report(anonymous_api.sample_search_diagnostics(), false, true, 1));
+ assert!(!reports.should_report(anonymous_api.sample_search_diagnostics(), false, true, 1));
+
+ for (source, agent) in [
+ (None, None),
+ (Some("app"), None),
+ (Some("mcp"), None),
+ (Some("MCP"), Some("unknown")),
+ (Some("api"), Some("codex")),
+ (Some("API"), Some("claude-code")),
+ ] {
+ let client = extract_agent(source, agent).await;
+ assert!(!client.sample_search_diagnostics());
+ for _ in 0..10 {
+ assert!(reports.should_report(client.sample_search_diagnostics(), false, true, 1));
+ }
+ }
+ // Untrusted agent labels cannot create unbounded per-agent buckets.
+ assert!(extract_agent(Some("api"), Some("private-project"))
+ .await
+ .sample_search_diagnostics());
+ }
+
#[tokio::test]
async fn attributes_only_fixed_app_names_on_explicit_external_requests() {
for source in ["api", "mcp"] {
diff --git a/crates/screenpipe-engine/src/routes/search.rs b/crates/screenpipe-engine/src/routes/search.rs
index 386b1c7be..537d63e25 100644
--- a/crates/screenpipe-engine/src/routes/search.rs
+++ b/crates/screenpipe-engine/src/routes/search.rs
@@ -1640,31 +1640,39 @@ pub(crate) async fn search(
debug!("search completed: found {} results", total);
+ // Only repetitive, unattributed API diagnostics are sampled. App/MCP and
+ // named-agent searches keep their existing events and occurrence timestamps.
+ // All requests still contribute to api_usage_5min; qualified outcomes are
+ // independent. Decide before building the diagnostic payload or spawning.
+ let sampled = api_client.sample_search_diagnostics();
let response_non_empty = !content_items.is_empty();
- let parsed_parser_profiles = if parsed_search {
- parsed_parser_profiles(&content_items)
- } else {
- Vec::new()
- };
+ if analytics::should_capture_search(sampled, parsed_search, response_non_empty) {
+ let parsed_parser_profiles = if parsed_search {
+ parsed_parser_profiles(&content_items)
+ } else {
+ Vec::new()
+ };
- // Track search analytics
- analytics::capture_event_nonblocking(
- "search_performed",
- serde_json::json!({
- "query_length": query.q.as_ref().map(|q| q.len()).unwrap_or(0),
- "content_type": format!("{:?}", query.content_type),
- "request_source": api_client.source_label(),
- "agent_client": api_client.agent_client().as_str(),
- "has_date_filter": query.start_time.is_some() || query.end_time.is_some(),
- "has_app_filter": query.app_name.is_some(),
- "result_count": total,
- "non_empty": response_non_empty,
- "parsed_non_empty": parsed_search && response_non_empty,
- "parsed_parser_profiles": parsed_parser_profiles,
- "limit": query.pagination.limit,
- "offset": query.pagination.offset,
- }),
- );
+ analytics::capture_event_nonblocking(
+ "search_performed",
+ serde_json::json!({
+ "telemetry_sample_interval_seconds": if sampled { analytics::SEARCH_REPORT_INTERVAL.as_secs() } else { 0 },
+ "telemetry_sampling": if sampled { "first_per_interval_per_result_kind" } else { "none" },
+ "query_length": query.q.as_ref().map(|q| q.len()).unwrap_or(0),
+ "content_type": format!("{:?}", query.content_type),
+ "request_source": api_client.source_label(),
+ "agent_client": api_client.agent_client().as_str(),
+ "has_date_filter": query.start_time.is_some() || query.end_time.is_some(),
+ "has_app_filter": query.app_name.is_some(),
+ "result_count": total,
+ "non_empty": response_non_empty,
+ "parsed_non_empty": parsed_search && response_non_empty,
+ "parsed_parser_profiles": parsed_parser_profiles,
+ "limit": query.pagination.limit,
+ "offset": query.pagination.offset,
+ }),
+ );
+ }
// Get cloud search metadata
let time_range = match (query.start_time, query.end_time) {
diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md
index 4289e3498..e72873490 100644
--- a/docs/COVERAGE.md
+++ b/docs/COVERAGE.md
@@ -38,15 +38,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.
- Mapped suites: 36
- Mapped Rust files: 383
-- Active test blocks: 3784
+- Active test blocks: 3790
- Ignored/manual test blocks: 170
-- Weighted coverage points: 3121.5
+- Weighted coverage points: 3125.7
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 32 | 3638 | 156 | 3055.2 | 21 | 11 | 100% |
-| macos | 31 | 3664 | 131 | 3043.7 | 22 | 11 | 100% |
-| linux | 28 | 3221 | 131 | 2678.6 | 20 | 11 | 100% |
+| windows | 32 | 3644 | 156 | 3059.4 | 21 | 11 | 100% |
+| macos | 31 | 3670 | 131 | 3047.9 | 22 | 11 | 100% |
+| linux | 28 | 3227 | 131 | 2682.8 | 20 | 11 | 100% |
## Refresh
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index f45d01e77..957dcc4b5 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -9,10 +9,10 @@ confidence, and criticality.
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs
- Mapped suites: 36
- Mapped Rust files: 383
-- Active test blocks: 3784
+- Active test blocks: 3790
- Ignored/manual test blocks: 170
-- Declared test blocks: 3954
-- Weighted coverage points: 3121.5
+- Declared test blocks: 3960
+- Weighted coverage points: 3125.7
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,15 +23,15 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 32 | 3638 | 156 | 3055.2 | 21 | 11 | 100% |
-| macos | 31 | 3664 | 131 | 3043.7 | 22 | 11 | 100% |
-| linux | 28 | 3221 | 131 | 2678.6 | 20 | 11 | 100% |
+| windows | 32 | 3644 | 156 | 3059.4 | 21 | 11 | 100% |
+| macos | 31 | 3670 | 131 | 3047.9 | 22 | 11 | 100% |
+| linux | 28 | 3227 | 131 | 2682.8 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| screenpipe-engine | 12 | 21 | 127 | 1824 | 52 | 1404.5 | 10 |
+| screenpipe-engine | 12 | 21 | 127 | 1830 | 52 | 1408.7 | 10 |
| screenpipe-db | 5 | 65 | 26 | 595 | 26 | 566.6 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
| screenpipe-audio | 7 | 26 | 53 | 675 | 49 | 590.3 | 5 |
@@ -68,20 +68,20 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| configuration | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts |
| database | 7 suites / 548 active / 22 ignored / 519.6 pts | 7 suites / 548 active / 22 ignored / 519.6 pts | 7 suites / 548 active / 22 ignored / 519.6 pts |
| db-search | 2 suites / 118 active / 9 ignored / 118.0 pts | 2 suites / 118 active / 9 ignored / 118.0 pts | 2 suites / 118 active / 9 ignored / 118.0 pts |
-| engine-lifecycle | 6 suites / 256 active / 1 ignored / 231.0 pts | 6 suites / 256 active / 1 ignored / 231.0 pts | 6 suites / 252 active / 9 ignored / 230.1 pts |
-| local-api | 3 suites / 477 active / 11 ignored / 337.5 pts | 3 suites / 477 active / 11 ignored / 337.5 pts | 3 suites / 477 active / 11 ignored / 337.5 pts |
-| meeting | 6 suites / 1706 active / 24 ignored / 1391.6 pts | 6 suites / 1706 active / 24 ignored / 1391.6 pts | 4 suites / 1357 active / 20 ignored / 1066.3 pts |
+| engine-lifecycle | 6 suites / 261 active / 1 ignored / 234.5 pts | 6 suites / 261 active / 1 ignored / 234.5 pts | 6 suites / 257 active / 9 ignored / 233.6 pts |
+| local-api | 3 suites / 478 active / 11 ignored / 338.2 pts | 3 suites / 478 active / 11 ignored / 338.2 pts | 3 suites / 478 active / 11 ignored / 338.2 pts |
+| meeting | 6 suites / 1707 active / 24 ignored / 1392.3 pts | 6 suites / 1707 active / 24 ignored / 1392.3 pts | 4 suites / 1358 active / 20 ignored / 1067.0 pts |
| ocr | 4 suites / 140 active / 7 ignored / 129.8 pts | 4 suites / 133 active / 7 ignored / 127.6 pts | 3 suites / 124 active / 6 ignored / 118.6 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1650 active / 80 ignored / 1460.1 pts | 14 suites / 1758 active / 84 ignored / 1503.3 pts | 13 suites / 1650 active / 80 ignored / 1460.1 pts |
+| performance | 13 suites / 1655 active / 80 ignored / 1463.6 pts | 14 suites / 1763 active / 84 ignored / 1506.8 pts | 13 suites / 1655 active / 80 ignored / 1463.6 pts |
| pipes | 1 suites / 540 active / 3 ignored / 378.0 pts | 1 suites / 540 active / 3 ignored / 378.0 pts | 1 suites / 540 active / 3 ignored / 378.0 pts |
| privacy | 5 suites / 1002 active / 42 ignored / 809.1 pts | 5 suites / 1041 active / 17 ignored / 804.0 pts | 5 suites / 960 active / 14 ignored / 774.0 pts |
| real-app | - | 1 suites / 108 active / 4 ignored / 43.2 pts | - |
| speaker | 2 suites / 388 active / 11 ignored / 388.0 pts | 2 suites / 388 active / 11 ignored / 388.0 pts | 2 suites / 388 active / 11 ignored / 388.0 pts |
| storage | 4 suites / 676 active / 39 ignored / 561.7 pts | 4 suites / 676 active / 39 ignored / 561.7 pts | 4 suites / 676 active / 39 ignored / 561.7 pts |
| sync | 1 suites / 540 active / 3 ignored / 378.0 pts | 1 suites / 540 active / 3 ignored / 378.0 pts | 1 suites / 540 active / 3 ignored / 378.0 pts |
-| timeline | 4 suites / 1223 active / 45 ignored / 1002.5 pts | 4 suites / 1223 active / 45 ignored / 1002.5 pts | 4 suites / 1223 active / 45 ignored / 1002.5 pts |
-| transcription | 6 suites / 870 active / 48 ignored / 680.2 pts | 5 suites / 864 active / 48 ignored / 676.0 pts | 5 suites / 864 active / 48 ignored / 676.0 pts |
+| timeline | 4 suites / 1224 active / 45 ignored / 1003.2 pts | 4 suites / 1224 active / 45 ignored / 1003.2 pts | 4 suites / 1224 active / 45 ignored / 1003.2 pts |
+| transcription | 6 suites / 871 active / 48 ignored / 680.9 pts | 5 suites / 865 active / 48 ignored / 676.8 pts | 5 suites / 865 active / 48 ignored / 676.8 pts |
| ui-events | 4 suites / 817 active / 34 ignored / 620.0 pts | 3 suites / 748 active / 5 ignored / 571.7 pts | 3 suites / 748 active / 5 ignored / 571.7 pts |
| vision-capture | 6 suites / 576 active / 32 ignored / 453.6 pts | 6 suites / 569 active / 32 ignored / 451.4 pts | 5 suites / 560 active / 31 ignored / 442.4 pts |
@@ -135,7 +135,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 16 | 47 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 14 | 106 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 37 | 295 | 13 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
-| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 44 | 429 | 6 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
+| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 44 | 430 | 6 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 306 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 13 | 125 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |
| engine-db-recovery-cli | screenpipe-engine | windows, macos, linux | database, engine-lifecycle | engine-health-lifecycle, performance-liveness | high | strong | unit | 1 | 29 | 0 | Exact DB/WAL/SHM working-copy preservation, rollback on archive failure, and restart repair for crashes during the multi-file generation swap. |
@@ -145,7 +145,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| engine-meeting-privacy-sync | screenpipe-engine | windows, macos, linux | meeting, privacy, ui-events, pipes, sync | meeting-live-notes, privacy-and-redaction, accessibility-ui-events, performance-liveness | medium | strong | unit | 32 | 540 | 3 | Unit-heavy coverage for privacy filter policy, capture exclusions, UI recorder safety, pipes/live-view/structured-output helpers, sync helpers, and CLI parsing. Meeting detection moved to the engine-meeting-watcher suite. |
| engine-meeting-watcher | screenpipe-engine | windows, macos | meeting | meeting-live-notes | high | strong | mixed | 10 | 270 | 3 | Successor of src/meeting_detector.rs and src/meeting_telemetry.rs. Audio-process and UI-scan backend state machines, candidate resolution, shared telemetry, and a scored trajectory eval. ui_scan/macos.rs and ui_scan/windows.rs are cfg-gated and only execute on their target OS; both backends are null on Linux. |
| engine-retention-storage | screenpipe-engine | windows, macos, linux | storage, engine-lifecycle, performance | engine-health-lifecycle, performance-liveness | medium | strong | mixed | 5 | 39 | 0 | Local retention deletion (including lean-mode heavy-text stripping), cloud archive watermarking, atomic state-file replacement, and the low-disk capture monitor. |
-| engine-telemetry-observability | screenpipe-engine | windows, macos, linux | engine-lifecycle, performance | engine-health-lifecycle, performance-liveness | medium | strong | unit | 6 | 30 | 0 | PostHog capture gating, telemetry context shaping, piggyback telemetry forwarding, crash-log helpers, resource monitoring, and the recording-coverage reliability metric. |
+| engine-telemetry-observability | screenpipe-engine | windows, macos, linux | engine-lifecycle, performance | engine-health-lifecycle, performance-liveness | medium | strong | unit | 6 | 35 | 0 | PostHog capture gating, telemetry context shaping, piggyback telemetry forwarding, crash-log helpers, resource monitoring, and the recording-coverage reliability metric. |
| engine-workflows | screenpipe-engine | windows, macos, linux | local-api, storage | local-api-search | high | partial | mixed | 7 | 36 | 0 | Catalog identity and persistence, revision-scoped feedback, workflow evidence and pipeline contracts, and denial of agent-granted rollout access. Does not assert live cloud discovery quality. Workflow publication orchestration deadlines and nested query backpressure are covered with virtual-time router tests. |
| filesystem-durable-sync | screenpipe-fs | macos, linux, windows | database, vision-capture, audio | capture-ocr-pipeline, audio-record-transcribe, engine-health-lifecycle | high | strong | unit | 1 | 3 | 0 | Durable flushes preserve local full-sync behavior, fall back to fsync only for macOS ENOTSUP, and propagate real I/O or fallback failures. Real SMB migration and continued recording are covered by the marked-volume integration test. |
| screen-capture-ocr-contract | screenpipe-screen | windows, macos, linux | vision-capture, ocr | capture-ocr-pipeline | high | partial | unit | 3 | 18 | 0 | Cross-platform cached-OCR unit coverage for RawCaptureResult to CaptureResult metadata, browser URL, focus state, window-to-screen OCR coordinate transformation, Tesseract output parsing, and contour-based text-region detection for the meeting OCR gate. |
@@ -366,7 +366,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| engine-api-routes | screenpipe-engine | src/activity_ledger.rs | source | 6 | 0 | 6 |
| engine-api-routes | screenpipe-engine | src/agent_profile.rs | source | 4 | 0 | 4 |
| engine-api-routes | screenpipe-engine | src/agent_skills.rs | source | 4 | 0 | 4 |
-| engine-telemetry-observability | screenpipe-engine | src/analytics.rs | source | 5 | 0 | 5 |
+| engine-telemetry-observability | screenpipe-engine | src/analytics.rs | source | 10 | 0 | 10 |
| engine-retention-storage | screenpipe-engine | src/archive.rs | source | 14 | 0 | 14 |
| engine-retention-storage | screenpipe-engine | src/atomic_file.rs | source | 4 | 0 | 4 |
| engine-api-routes | screenpipe-engine | src/auth_key.rs | source | 8 | 0 | 8 |
@@ -457,7 +457,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| engine-api-routes | screenpipe-engine | src/routes/meetings.rs | source | 7 | 0 | 7 |
| engine-api-routes | screenpipe-engine | src/routes/memories.rs | source | 5 | 0 | 5 |
| engine-api-routes | screenpipe-engine | src/routes/pipe_store.rs | source | 4 | 0 | 4 |
-| engine-api-routes | screenpipe-engine | src/routes/request_origin.rs | source | 2 | 0 | 2 |
+| engine-api-routes | screenpipe-engine | src/routes/request_origin.rs | source | 3 | 0 | 3 |
| engine-api-routes | screenpipe-engine | src/routes/response_format.rs | source | 6 | 0 | 6 |
| engine-api-routes | screenpipe-engine | src/routes/retranscribe.rs | source | 4 | 0 | 4 |
| engine-api-routes | screenpipe-engine | src/routes/search.rs | source | 29 | 0 | 29 |
diff --git a/packages/screenpipe-mcp/README.md b/packages/screenpipe-mcp/README.md
index eaf88e010..210188f42 100644
--- a/packages/screenpipe-mcp/README.md
+++ b/packages/screenpipe-mcp/README.md
@@ -31,6 +31,24 @@ These are self-reported identifiers, not verified app identities; copied skills
and callers that omit the headers cannot be reliably attributed. Historical
generic API events cannot be assigned an app retroactively.
+The engine preserves every existing uncached app, MCP and named-agent
+`search_performed` event. Only explicitly API-originated requests with an unknown
+agent are sampled: the first event per 60-second monotonic window per engine
+process, separately for empty/non-empty and parsed/ordinary results (at most four
+events per window, plus the first event in each category after UTC midnight to
+preserve daily activity). An empty background poll cannot consume the successful-result
+slot, and API traffic cannot consume an app, MCP or named-agent slot.
+
+Sampled events carry `telemetry_sampling: "first_per_interval_per_result_kind"`
+and `telemetry_sample_interval_seconds: 60`; unsampled events carry `"none"` and
+`0`. Sampled events are presence diagnostics, not exact search counts or an
+unbiased sample of query properties. Their counts and property breakdowns are
+not comparable to earlier releases. `api_usage_5min.request_count` still counts
+aggregate API traffic across all routes, and `qualified_value_event` retains the
+existing successful-retrieval path, including cached searches. No new events are
+added for cache hits. Health/resource telemetry and local monitoring retain their
+existing cadence.
+
Attribution adds fields to existing events and uses the existing analytics
opt-out. It does not scan running apps or send prompts, search text, or raw
client names. HTTP MCP now reports successful searches through the same