YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(telemetry): cap search and qualified-value event floods (#7874)
Ezra Ellette85f466e0214e
diff --git a/crates/screenpipe-engine/src/analytics.rs b/crates/screenpipe-engine/src/analytics.rsindex f3bddabaf..d0e999ae8 100644--- a/crates/screenpipe-engine/src/analytics.rs+++ b/crates/screenpipe-engine/src/analytics.rs@@ -9,6 +9,7 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};use std::time::{Duration, Instant};use tracing::{debug, trace};+use crate::qualified_value::AgentClient;use crate::telemetry_context::TelemetryContext;#[cfg(target_os = "macos")]@@ -27,33 +28,69 @@ pub(crate) const SEARCH_REPORT_INTERVAL: Duration = Duration::from_secs(60);static SEARCH_REPORT_START: Lazy<Instant> = Lazy::new(Instant::now);static SEARCH_REPORTS: SearchReportLimiter = SearchReportLimiter::new();+static VALUE_REPORTS: ValueReportLimiter = ValueReportLimiter::new();-/// Only unattributed, explicitly API-originated searches share a report budget.-/// Empty/non-empty and parsed/ordinary results have separate slots so an empty-/// poll cannot hide the first successful retrieval or parsed-search presence.-/// Fixed storage keeps untrusted request properties out of the limiter's keys.+/// Each source/client/result kind gets its first report per minute. No request+/// headers can bypass sampling or create unbounded limiter keys.pub(crate) struct SearchReportLimiter {- windows: [AtomicU64; 4],+ windows: [AtomicU64; 3 * AgentClient::COUNT * 4],}impl SearchReportLimiter {pub(crate) const fn new() -> Self {Self {- windows: [const { AtomicU64::new(0) }; 4],+ windows: [const { AtomicU64::new(0) }; 3 * AgentClient::COUNT * 4],}}pub(crate) fn should_report(&self,- sampled: bool,+ source: &str,+ client: AgentClient,parsed: bool,non_empty: bool,window: u64,) -> bool {- if !sampled {- return true;+ let source = match source {+ "API" => 0,+ "MCP" => 1,+ _ => 2,+ };+ let slot = (source * AgentClient::COUNT + client as usize) * 4+ + usize::from(parsed) * 2+ + usize::from(non_empty);+ claim_search_window(&self.windows[slot], window)+ }+}++/// Independent budgets preserve the first successful outcome even if search+/// diagnostics or a different retrieval type already exhausted their window.+struct ValueReportLimiter {+ windows: [AtomicU64; 2 * AgentClient::COUNT * 4],+}++impl ValueReportLimiter {+ const fn new() -> Self {+ Self {+ windows: [const { AtomicU64::new(0) }; 2 * AgentClient::COUNT * 4],}- let slot = usize::from(parsed) * 2 + usize::from(non_empty);+ }++ fn should_report(&self, properties: &Value, window: u64) -> bool {+ let surface = match properties["surface"].as_str() {+ Some("api") => 0,+ Some("mcp") => 1,+ _ => return true,+ };+ let action = match properties["action"].as_str() {+ Some("search") => 0,+ Some("meeting") => 1,+ _ if properties["value_strength"] == "completed" => 3,+ _ => 2,+ };+ let client =+ AgentClient::from_name(properties["agent_client"].as_str().unwrap_or_default());+ let slot = (surface * AgentClient::COUNT + client as usize) * 4 + action;claim_search_window(&self.windows[slot], window)}}@@ -77,17 +114,22 @@ fn search_report_window(elapsed: Duration, utc_timestamp: i64) -> u64 {(minute << 32) | (utc_day & u64::from(u32::MAX))}-/// Consent is checked before claiming a slot or constructing a payload.-/// App, MCP and named-agent requests bypass the API diagnostic limiter.-pub(crate) fn should_capture_search(sampled: bool, parsed: bool, non_empty: bool) -> bool {- if !is_enabled() {- return false;- }- let window = search_report_window(+fn current_report_window() -> u64 {+ search_report_window(SEARCH_REPORT_START.elapsed(),chrono::Utc::now().timestamp(),- );- SEARCH_REPORTS.should_report(sampled, parsed, non_empty, window)+ )+}++/// Consent is checked before claiming a slot or constructing a payload.+pub(crate) fn should_capture_search(+ source: &str,+ client: AgentClient,+ parsed: bool,+ non_empty: bool,+) -> bool {+ is_enabled()+ && SEARCH_REPORTS.should_report(source, client, parsed, non_empty, current_report_window())}pub struct Analytics {@@ -170,6 +212,17 @@ pub async fn capture_event(event: &str, properties: Value) {}let mut props = properties;+ // Bound every engine API/MCP outcome here, including cached searches,+ // activity summaries and the MCP adapter. Explicit frontend actions are+ // separate and retain their full-fidelity events.+ if event == "qualified_value_event" && matches!(props["surface"].as_str(), Some("api" | "mcp"))+ {+ if !VALUE_REPORTS.should_report(&props, current_report_window()) {+ return;+ }+ props["telemetry_sample_interval_seconds"] = json!(SEARCH_REPORT_INTERVAL.as_secs());+ props["telemetry_sampling"] = json!("first_per_interval_per_surface_client_value_kind");+ }if let Some(obj) = props.as_object_mut() {obj.insert("distinct_id".to_string(), json!(ANALYTICS.distinct_id));obj.insert("$lib".to_string(), json!("screenpipe-engine"));@@ -319,48 +372,118 @@ mod tests {use std::sync::Mutex;#[test]- fn api_polling_preserves_each_result_kind_and_unlimited_unsampled_requests() {+ fn search_floods_are_bounded_for_every_source_client_and_result_kind() {let reports = SearchReportLimiter::new();- assert!(reports.should_report(true, false, false, 1));- for _ in 0..100_000 {- assert!(!reports.should_report(true, false, false, 1));- assert!(reports.should_report(false, false, false, 1));+ for source in ["app", "API", "MCP"] {+ for client in [+ AgentClient::Unknown,+ AgentClient::Claude,+ AgentClient::Codex,+ ] {+ assert!(reports.should_report(source, client, false, false, 1));+ for _ in 0..100_000 {+ assert!(!reports.should_report(source, client, false, false, 1));+ }+ // Empty polling cannot hide successful or parsed retrievals.+ assert!(reports.should_report(source, client, false, true, 1));+ assert!(reports.should_report(source, client, true, false, 1));+ assert!(reports.should_report(source, client, true, true, 1));+ for parsed in [false, true] {+ for non_empty in [false, true] {+ assert!(!reports.should_report(source, client, parsed, non_empty, 1));+ assert!(reports.should_report(source, client, parsed, non_empty, 2));+ }+ }+ }+ }+ }++ #[test]+ fn qualified_outcomes_have_independent_bounded_budgets() {+ use crate::qualified_value::{api_outcome_properties, ApiOutcomeKind, McpOutcome};++ let searches = SearchReportLimiter::new();+ let values = ValueReportLimiter::new();+ assert!(searches.should_report("API", AgentClient::Unknown, false, true, 1));+ let mut outcomes = Vec::new();+ for client in [+ AgentClient::Unknown,+ AgentClient::Claude,+ AgentClient::Codex,+ ] {+ for kind in [+ ApiOutcomeKind::SearchResult,+ ApiOutcomeKind::ActivitySummary,+ ] {+ outcomes.push(api_outcome_properties(kind, client));+ }+ for kind in [+ "search_result",+ "meeting_result",+ "artifact_result",+ "artifact_created",+ ] {+ let outcome: McpOutcome = serde_json::from_value(json!({+ "outcome": kind,+ "client": client.as_str(),+ }))+ .unwrap();+ outcomes.push(outcome.into_properties());+ }}- // The empty API flood must not hide successful or parsed results.- assert!(reports.should_report(true, false, true, 1));- assert!(reports.should_report(true, true, false, 1));- assert!(reports.should_report(true, true, true, 1));- for parsed in [false, true] {- for non_empty in [false, true] {- assert!(!reports.should_report(true, parsed, non_empty, 1));- assert!(reports.should_report(true, parsed, non_empty, 2));+ for properties in outcomes {+ assert!(values.should_report(&properties, 1));+ for _ in 0..1000 {+ assert!(!values.should_report(&properties, 1));}+ assert!(values.should_report(&properties, 2));}}#[test]- fn unsampled_searches_do_not_consume_the_api_budget() {- let reports = SearchReportLimiter::new();- for _ in 0..100 {- assert!(reports.should_report(false, false, true, 1));+ fn unrecognized_value_clients_share_the_unknown_budget() {+ let reports = ValueReportLimiter::new();+ let mut properties = crate::qualified_value::api_outcome_properties(+ crate::qualified_value::ApiOutcomeKind::SearchResult,+ AgentClient::Unknown,+ );+ assert!(reports.should_report(&properties, 1));+ for client in [+ "unknown",+ "direct_api",+ "private-project",+ "arbitrary-header",+ ] {+ properties["agent_client"] = json!(client);+ assert!(!reports.should_report(&properties, 1));}- assert!(reports.should_report(true, false, true, 1));}#[test]fn midnight_preserves_daily_presence_and_clock_rollback_does_not_stall_reporting() {let reports = SearchReportLimiter::new();+ let values = ValueReportLimiter::new();+ let properties = crate::qualified_value::api_outcome_properties(+ crate::qualified_value::ApiOutcomeKind::SearchResult,+ AgentClient::Unknown,+ );let before_midnight = search_report_window(Duration::from_secs(10), 86_399);let after_midnight = search_report_window(Duration::from_secs(11), 86_400);- assert!(reports.should_report(true, false, true, before_midnight));- assert!(reports.should_report(true, false, true, after_midnight));- assert!(!reports.should_report(true, false, true, after_midnight));+ for window in [before_midnight, after_midnight] {+ assert!(reports.should_report("app", AgentClient::Unknown, false, true, window));+ assert!(values.should_report(&properties, window));+ assert!(!reports.should_report("app", AgentClient::Unknown, false, true, window));+ assert!(!values.should_report(&properties, window));+ }// A delayed previous-day caller cannot reopen an already used slot.- assert!(!reports.should_report(true, false, true, before_midnight));+ assert!(!reports.should_report("app", AgentClient::Unknown, false, true, before_midnight));+ assert!(!values.should_report(&properties, before_midnight));// A clock correction cannot suppress reports beyond the current minute.let rolled_back = search_report_window(Duration::from_secs(60), 86_000);- assert!(reports.should_report(true, false, true, rolled_back));- assert!(!reports.should_report(true, false, true, rolled_back));+ assert!(reports.should_report("app", AgentClient::Unknown, false, true, rolled_back));+ assert!(values.should_report(&properties, rolled_back));+ assert!(!reports.should_report("app", AgentClient::Unknown, false, true, rolled_back));+ assert!(!values.should_report(&properties, rolled_back));}#[test]diff --git a/crates/screenpipe-engine/src/qualified_value.rs b/crates/screenpipe-engine/src/qualified_value.rsindex 515da0eb9..6251492e8 100644--- a/crates/screenpipe-engine/src/qualified_value.rs+++ b/crates/screenpipe-engine/src/qualified_value.rs@@ -31,6 +31,8 @@ pub(crate) enum AgentClient {}impl AgentClient {+ pub(crate) const COUNT: usize = Self::Unknown as usize + 1;+/// Accept only fixed app identifiers; never retain arbitrary header values.pub(crate) fn from_name(value: &str) -> Self {Self::deserialize(serde::de::value::StrDeserializer::<serde::de::value::Error>::new(value))diff --git a/crates/screenpipe-engine/src/routes/request_origin.rs b/crates/screenpipe-engine/src/routes/request_origin.rsindex b5d501eaf..d8eb8fa44 100644--- a/crates/screenpipe-engine/src/routes/request_origin.rs+++ b/crates/screenpipe-engine/src/routes/request_origin.rs@@ -31,11 +31,6 @@ impl ExplicitApiClient {self.source == RequestSource::Api}- /// Preserve app/MCP engagement and all named-agent attribution in full.- pub(crate) fn sample_search_diagnostics(&self) -> bool {- self.is_direct_api() && self.agent == AgentClient::Unknown- }-pub(crate) fn source_label(&self) -> &'static str {match self.source {RequestSource::App => "app",@@ -135,31 +130,56 @@ mod tests {}#[tokio::test]- async fn api_flood_cannot_suppress_app_mcp_or_named_agent_searches() {+ async fn every_request_origin_is_sampled_without_hiding_other_clients() {let reports = crate::analytics::SearchReportLimiter::new();- let anonymous_api = extract(Some("api")).await;- assert!(anonymous_api.sample_search_diagnostics());- assert!(reports.should_report(anonymous_api.sample_search_diagnostics(), false, true, 1));- assert!(!reports.should_report(anonymous_api.sample_search_diagnostics(), false, true, 1));-for (source, agent) in [(None, None),- (Some("app"), None),+ (Some("api"), None),(Some("mcp"), None),- (Some("MCP"), Some("unknown")),(Some("api"), Some("codex")),(Some("API"), Some("claude-code")),] {let client = extract_agent(source, agent).await;- assert!(!client.sample_search_diagnostics());- for _ in 0..10 {- assert!(reports.should_report(client.sample_search_diagnostics(), false, true, 1));+ assert!(reports.should_report(+ client.source_label(),+ client.agent_client(),+ false,+ true,+ 1,+ ));+ for _ in 0..1000 {+ assert!(!reports.should_report(+ client.source_label(),+ client.agent_client(),+ false,+ true,+ 1,+ ));}+ assert!(reports.should_report(+ client.source_label(),+ client.agent_client(),+ false,+ true,+ 2,+ ));+ }+ // Missing and untrusted headers share existing buckets, not bypasses.+ for (source, agent) in [+ (Some("app"), None),+ (Some("private-project"), None),+ (Some("api"), Some("private-project")),+ (Some("MCP"), Some("unknown")),+ ] {+ let client = extract_agent(source, agent).await;+ assert!(!reports.should_report(+ client.source_label(),+ client.agent_client(),+ false,+ true,+ 2,+ ));}- // Untrusted agent labels cannot create unbounded per-agent buckets.- assert!(extract_agent(Some("api"), Some("private-project"))- .await- .sample_search_diagnostics());}#[tokio::test]diff --git a/crates/screenpipe-engine/src/routes/search.rs b/crates/screenpipe-engine/src/routes/search.rsindex 34c4873ab..c12383a41 100644--- a/crates/screenpipe-engine/src/routes/search.rs+++ b/crates/screenpipe-engine/src/routes/search.rs@@ -1688,13 +1688,16 @@ pub(crate) async fn search(debug!("search completed: found {} results", total);- // Only repetitive, unattributed API diagnostics are sampled. App/MCP and- // named-agent searches keep their existing events and occurrence timestamps.- // All requests still contribute to api_usage_5min; qualified outcomes are- // independent. Decide before building the diagnostic payload or spawning.- let sampled = api_client.sample_search_diagnostics();+ // Bound diagnostics for every source, including callers with no headers.+ // All requests still contribute to api_usage_5min; qualified outcomes have+ // their own budget. Decide before building the payload or spawning.let response_non_empty = !content_items.is_empty();- if analytics::should_capture_search(sampled, parsed_search, response_non_empty) {+ if analytics::should_capture_search(+ api_client.source_label(),+ api_client.agent_client(),+ parsed_search,+ response_non_empty,+ ) {let parsed_parser_profiles = if parsed_search {parsed_parser_profiles(&content_items)} else {@@ -1704,8 +1707,8 @@ pub(crate) async fn search(analytics::capture_event_nonblocking("search_performed",serde_json::json!({- "telemetry_sample_interval_seconds": if sampled { analytics::SEARCH_REPORT_INTERVAL.as_secs() } else { 0 },- "telemetry_sampling": if sampled { "first_per_interval_per_result_kind" } else { "none" },+ "telemetry_sample_interval_seconds": analytics::SEARCH_REPORT_INTERVAL.as_secs(),+ "telemetry_sampling": "first_per_interval_per_source_client_result_kind","query_length": query.q.as_ref().map(|q| q.len()).unwrap_or(0),"content_type": format!("{:?}", query.content_type),"request_source": api_client.source_label(),diff --git a/packages/screenpipe-mcp/README.md b/packages/screenpipe-mcp/README.mdindex 797adc60f..4e82dc915 100644--- a/packages/screenpipe-mcp/README.md+++ b/packages/screenpipe-mcp/README.md@@ -31,23 +31,26 @@ These are self-reported identifiers, not verified app identities; copied skillsand callers that omit the headers cannot be reliably attributed. Historicalgeneric API events cannot be assigned an app retroactively.-The engine preserves every existing uncached app, MCP and named-agent-`search_performed` event. Only explicitly API-originated requests with an unknown-agent are sampled: the first event per 60-second monotonic window per engine-process, separately for empty/non-empty and parsed/ordinary results (at most four-events per window, plus the first event in each category after UTC midnight to-preserve daily activity). An empty background poll cannot consume the successful-result-slot, and API traffic cannot consume an app, MCP or named-agent slot.--Sampled events carry `telemetry_sampling: "first_per_interval_per_result_kind"`-and `telemetry_sample_interval_seconds: 60`; unsampled events carry `"none"` and-`0`. Sampled events are presence diagnostics, not exact search counts or an-unbiased sample of query properties. Their counts and property breakdowns are-not comparable to earlier releases. `api_usage_5min.request_count` still counts-aggregate API traffic across all routes, and `qualified_value_event` retains the-existing successful-retrieval path, including cached searches. No new events are-added for cache hits. Health/resource telemetry and local monitoring retain their-existing cadence.+The engine samples `search_performed` for every source, including app, MCP,+named agents and callers without attribution headers. Each fixed source/client+pair gets its first event per 60-second monotonic window per engine process,+separately for empty/non-empty and parsed/ordinary results (at most four events+per pair per window). The first event in each category after UTC midnight is+also preserved. One client's polling cannot consume another client's budget.++API/MCP `qualified_value_event` emissions have an independent first-per-minute+budget per surface, fixed client and value kind: search, meeting, artifact+retrieved or artifact created. This covers cached API searches, activity+summaries and MCP outcomes. Explicit frontend value actions are not sampled.++Both event families carry `telemetry_sample_interval_seconds: 60` and+`telemetry_sampling`: `"first_per_interval_per_source_client_result_kind"` for+search diagnostics, or `"first_per_interval_per_surface_client_value_kind"` for+qualified values. These are presence observations, not exact retrieval counts+or unbiased samples of query properties. Counts and property breakdowns are not+comparable to earlier releases. `api_usage_5min.request_count` still counts+aggregate traffic across all routes. No search diagnostics are added for cache+hits; actual searches, recording and local monitoring are unaffected.Resource events additionally report `telemetry_window_samples`,`window_cpu_percent_mean`, `window_cpu_percent_max`, `window_memory_gb_mean`,@@ -56,11 +59,11 @@ since the previous reporting attempt. These supplement the existing point-in-timproperties. Weight window means by their sample counts when aggregating them.Transport remains best effort; these summaries are not a durable sample archive.-Cost reductions must preserve successful-retrieval counts and cohort-relative-retention windows. A future reduction in diagnostic cadence needs a validated-replacement for fault timelines, short-session coverage, and resource peaks,-including shutdown and failed-delivery handling. This change does not reduce-PostHog event volume or claim billing savings.+Sampling preserves the first successful retrieval for daily presence and+cohort-relative retention, not every occurrence. It reduces outgoing analytics+volume after clients upgrade; actual billing savings depend on the account's+pricing and usage tiers. Health/resource reporting is unchanged by retrieval+sampling.Attribution adds fields to existing events and uses the existing analyticsopt-out. It does not scan running apps or send prompts, search text, or raw