YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
feat(enterprise): ingest and search activities (#6877)
Ezra Ellette44145d6a3e51
diff --git a/apps/screenpipe-app-tauri/lib/hooks/__tests__/use-enterprise-policy.test.tsx b/apps/screenpipe-app-tauri/lib/hooks/__tests__/use-enterprise-policy.test.tsxindex 5ae0805cc..fb1dbb9e1 100644--- a/apps/screenpipe-app-tauri/lib/hooks/__tests__/use-enterprise-policy.test.tsx+++ b/apps/screenpipe-app-tauri/lib/hooks/__tests__/use-enterprise-policy.test.tsx@@ -44,6 +44,7 @@ const mocks = vi.hoisted(() => {setEnterpriseRecordingAuthorized: vi.fn(async () => ({ status: "ok", data: null })),applyEnterpriseUiVisibility: vi.fn(async () => undefined),setSyncStreams: vi.fn(async () => undefined),+ setActivitySyncEnabled: vi.fn(async () => undefined),saveEnterpriseTeamConfig: vi.fn(async () => null),syncManagedTeamSkills: vi.fn(async () => ({ status: "ok" as const, data: [] })),},@@ -235,6 +236,7 @@ describe("enterprise policy runtime manual activation", () => {});mocks.commands.applyEnterpriseUiVisibility.mockResolvedValue(undefined);mocks.commands.setSyncStreams.mockResolvedValue(undefined);+ mocks.commands.setActivitySyncEnabled.mockResolvedValue(undefined);mocks.commands.saveEnterpriseTeamConfig.mockResolvedValue(null);mocks.commands.syncManagedTeamSkills.mockResolvedValue({ status: "ok", data: [] });mocks.platform.mockReturnValue("windows");diff --git a/apps/screenpipe-app-tauri/lib/hooks/use-enterprise-policy.ts b/apps/screenpipe-app-tauri/lib/hooks/use-enterprise-policy.tsindex ac9e7c27d..0e6c46704 100644--- a/apps/screenpipe-app-tauri/lib/hooks/use-enterprise-policy.ts+++ b/apps/screenpipe-app-tauri/lib/hooks/use-enterprise-policy.ts@@ -985,6 +985,11 @@ export function useEnterprisePolicyRuntime() {),LOCAL_POLICY_COMMAND_TIMEOUT_MS);+ await withTimeout(+ "enterprise setActivitySyncEnabled",+ commands.setActivitySyncEnabled(streams.activities === true),+ LOCAL_POLICY_COMMAND_TIMEOUT_MS+ );} catch (e) {console.warn("[enterprise] failed to push sync streams to Rust:", e);}diff --git a/apps/screenpipe-app-tauri/lib/utils/tauri.ts b/apps/screenpipe-app-tauri/lib/utils/tauri.tsindex b17be1d64..3c8f3964f 100644--- a/apps/screenpipe-app-tauri/lib/utils/tauri.ts+++ b/apps/screenpipe-app-tauri/lib/utils/tauri.ts@@ -2379,6 +2379,9 @@ async searchNavigateToTimeline(timestamp: string, frameId: number | null, searchelse return { status: "error", error: e as any };}},+async setActivitySyncEnabled(activities: boolean) : Promise<void> {+ await TAURI_INVOKE("set_activity_sync_enabled", { activities });+},/*** Persist the user's explicit Settings choice. Automatic launch reconciliation* skips opted-out targets until the user explicitly connects them again.diff --git a/apps/screenpipe-app-tauri/src-tauri/src/activity_history.rs b/apps/screenpipe-app-tauri/src-tauri/src/activity_history.rsindex 474a8b1e5..1e160df31 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/activity_history.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/activity_history.rs@@ -418,6 +418,16 @@ fn read_all(app: &AppHandle) -> Result<PersistedActivityHistory, String> {.unwrap_or_default())}+pub(crate) fn entries_ending_after(+ app: &AppHandle,+ since: DateTime<Utc>,+) -> Result<Vec<ActivityHistoryEntry>, String> {+ let mut entries = read_all(app)?.entries;+ entries.retain(|entry| parse_time(&entry.end_at).is_some_and(|end| end > since));+ entries.sort_by(|left, right| left.end_at.cmp(&right.end_at).then(left.id.cmp(&right.id)));+ Ok(entries)+}+fn write_all(app: &AppHandle, history: &PersistedActivityHistory) -> Result<(), String> {let store = store::get_store(app, None).map_err(|error| error.to_string())?;store.set(diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/policy.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/policy.rsindex b8735ddca..6caad27ca 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/policy.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/policy.rs@@ -28,9 +28,10 @@ static ENFORCE_AUTO_START: Lazy<AtomicBool> =Lazy::new(|| AtomicBool::new(read_persisted_enforce_auto_start()));static POLICY_UPDATE_LOCK: Lazy<Mutex<()>> = Lazy::new(|| Mutex::new(()));-/// Per-stream sync policy. Defaults to all-true so an unconfigured device-/// behaves exactly like before this feature shipped. The frontend pulls the-/// admin's choices from `GET /api/enterprise/policy` (`syncStreams` field) on+/// Per-stream sync policy. Established streams preserve their historical+/// defaults, while richer semantic streams require an explicit opt-in. The+/// frontend pulls the admin's choices from `GET /api/enterprise/policy`+/// (`syncStreams` field) on/// the 5-min poll and pushes them in via `set_sync_streams`. The sync state/// machine in `enterprise_sync::run_one_sync` reads this on every tick./// How many frame images (screenshots) may leave this device — the org's@@ -105,6 +106,7 @@ impl FeedbackSyncMode {pub struct SyncStreams {pub frames: bool,pub parsed: bool,+ pub activities: bool,pub audio: bool,pub ui_events: bool,pub memories: bool,@@ -118,6 +120,7 @@ impl Default for SyncStreams {Self {frames: true,parsed: false,+ activities: false,audio: true,ui_events: true,memories: true,@@ -392,9 +395,14 @@ pub fn set_sync_streams() {let feedback = FeedbackSyncMode::parse(&feedback);let frame_images = FrameImagesMode::parse(&frame_images);+ let activities = SYNC_STREAMS+ .read()+ .map(|guard| guard.activities)+ .unwrap_or(false);let next = SyncStreams {frames,parsed,+ activities,audio,ui_events,memories,@@ -405,9 +413,10 @@ pub fn set_sync_streams(if let Ok(mut guard) = SYNC_STREAMS.write() {if *guard != next {tracing::info!(- "enterprise: sync streams updated frames={} parsed={} audio={} ui={} memories={} snapshots={} feedback={} frame_images={}",+ "enterprise: sync streams updated frames={} parsed={} activities={} audio={} ui={} memories={} snapshots={} feedback={} frame_images={}",frames,parsed,+ activities,audio,ui_events,memories,@@ -420,10 +429,20 @@ pub fn set_sync_streams(}}+#[tauri::command]+#[specta::specta]+pub fn set_activity_sync_enabled(activities: bool) {+ if let Ok(mut guard) = SYNC_STREAMS.write() {+ if guard.activities != activities {+ tracing::info!("enterprise: activity sync stream updated activities={activities}");+ guard.activities = activities;+ }+ }+}+/// Snapshot of the current per-stream sync policy. Read by the sync state-/// machine on every tick. Returns the defaults (all-true) if the lock is-/// poisoned — fail-open here mirrors the centralized-data master-switch-/// behavior: the ingest endpoint will still enforce policy server-side.+/// machine on every tick. Returns the per-stream defaults if the lock is+/// poisoned; the ingest endpoint still enforces policy server-side.//// Available under `enterprise-build` (the sync state machine in// enterprise_sync::run_one_sync reads it on every tick) and under `test`.@@ -571,6 +590,7 @@ mod tests {let s = SyncStreams::default();assert!(s.frames);assert!(!s.parsed);+ assert!(!s.activities);assert!(s.audio);assert!(s.ui_events);assert!(s.memories);diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rsindex 1886eef86..ea2b2d3b3 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs@@ -4,9 +4,10 @@//! Enterprise telemetry sync.//!-//! Periodically pulls new screen + audio + UI activity from the local screenpipe-//! API and POSTs it as JSONL to the screenpipe enterprise ingest endpoint,-//! authenticated with an org license key. Server-side it lands in R2 under+//! Periodically pulls new screen, interpreted Activities, audio, and UI data+//! from the local screenpipe API and POSTs it as JSONL to the screenpipe+//! enterprise ingest endpoint, authenticated with an org license key.+//! Server-side it lands in R2 under//! `enterprise-telemetry/{license_id}/{device_id}/{ts}.jsonl` and feeds the//! org's admin chat dashboard.//!@@ -248,6 +249,7 @@ struct CursorBoundary {ui: u32,memories: u32,parsed: u32,+ activity_ts: Option<String>,/// Feedback supports a true `(updated_at, id)` keyset cursor because its/// IDs are stable strings and its local route merges DB and legacy rows.feedback_id: Option<String>,@@ -380,6 +382,14 @@ pub trait LocalApiClient: Send + Sync {Ok(Vec::new())}+ /// Fetch AI-interpreted Activities ending after `since_ts`.+ async fn fetch_activities_since(+ &self,+ _since_ts: Option<&str>,+ ) -> Result<Vec<ActivityRow>, EnterpriseSyncError> {+ Ok(Vec::new())+ }+/// Fetch one frame's full-resolution JPEG by id — the same image the/// local `/frames/{id}` route serves, which means capture-time PII/// redaction has already been applied when the org policy enables it.@@ -401,7 +411,8 @@ pub trait LocalApiClient: Send + Sync {// same types — so it lives in `screenpipe-telemetry-wire`. Re-exported here// so the desktop shim keeps importing everything from `ee_sync::`.pub use screenpipe_telemetry_wire::{- AudioRow, FeedbackRow, FrameRow, MemoryRow, ParsedRow, SnapshotRow, TelemetryRecord, UiEventRow,+ ActivityRow, AudioRow, FeedbackRow, FrameRow, MemoryRow, ParsedRow, SnapshotRow,+ TelemetryRecord, UiEventRow,};// ─── Errors ─────────────────────────────────────────────────────────────────@@ -596,6 +607,10 @@ async fn run_one_sync_inner(cursor.last_parsed_ts = Some(cutoff.to_rfc3339());cursor.boundary.parsed = 0;}+ if cursor.boundary.activity_ts.is_none() {+ let cutoff = chrono::Utc::now() - chrono::Duration::from_std(SAFE_BACKFILL).unwrap();+ cursor.boundary.activity_ts = Some(cutoff.to_rfc3339());+ }// Per-stream sync policy is fetched fresh on every tick — the admin can// flip toggles in the dashboard and the device picks them up on the next@@ -728,6 +743,20 @@ async fn run_one_sync_inner(} else {Vec::new()};+ let activities = if streams.activities {+ match local+ .fetch_activities_since(cursor.boundary.activity_ts.as_deref())+ .await+ {+ Ok(rows) => rows,+ Err(error) => {+ warn!("enterprise sync: activity fetch failed (skipping): {}", error);+ Vec::new()+ }+ }+ } else {+ Vec::new()+ };if frames.is_empty()&& audio.is_empty()@@ -735,17 +764,19 @@ async fn run_one_sync_inner(&& snapshots.is_empty()&& memories.is_empty()&& parsed.is_empty()+ && activities.is_empty()&& feedback.is_empty(){debug!("enterprise sync: nothing new since last tick");return Ok(SyncTickReport::default());}- let mut body = build_jsonl_with_parsed(+ let mut body = screenpipe_telemetry_wire::build_jsonl_with_semantic_streams(&cfg.device_id,&cfg.device_label,&frames,&parsed,+ &activities,&audio,&ui,&snapshots,@@ -765,6 +796,9 @@ async fn run_one_sync_inner(&frames,|row| &row.timestamp,);+ if let Some(latest) = activities.last() {+ next_cursor.boundary.activity_ts = Some(latest.end_at.clone());+ }advance_timestamp_boundary(&mut next_cursor.last_audio_ts,&mut next_cursor.boundary.audio,@@ -804,6 +838,7 @@ async fn run_one_sync_inner(let counts = DirectUploadRecordCounts {frames: frames.len(),parsed: parsed.len(),+ activities: activities.len(),audio: audio.len(),ui: ui.len(),snapshots: snapshots.len(),@@ -824,6 +859,7 @@ async fn run_one_sync_inner(let counts = DirectUploadRecordCounts {frames: frames.len(),parsed: parsed.len(),+ activities: activities.len(),audio: audio.len(),ui: ui.len(),snapshots: snapshots.len(),@@ -852,6 +888,7 @@ async fn run_one_sync_inner(Ok(SyncTickReport {frames: frames.len(),parsed: parsed.len(),+ activities: activities.len(),audio: audio.len(),ui: ui.len(),snapshots: snapshots.len(),@@ -889,6 +926,7 @@ fn advance_timestamp_boundary<T>(pub struct SyncTickReport {pub frames: usize,pub parsed: usize,+ pub activities: usize,pub audio: usize,pub ui: usize,pub snapshots: usize,@@ -2174,6 +2212,20 @@ mod tests {}}+ fn activity(id: &str, start: &str, end: &str) -> ActivityRow {+ ActivityRow {+ activity_id: id.to_string(),+ activity_kind: "work".to_string(),+ meeting_id: None,+ timestamp: end.to_string(),+ start_at: start.to_string(),+ end_at: end.to_string(),+ title: "Prepared enterprise rollout".to_string(),+ summary: "Reviewed policy and completed the rollout checklist.".to_string(),+ evidence: Vec::new(),+ }+ }+// ─── truncate_on_char_boundary (UTF-8 safety) ───────────────────────#[test]@@ -2601,11 +2653,13 @@ mod tests {memories_to_yield: Mutex<Vec<Vec<MemoryRow>>>,feedback_to_yield: Mutex<Vec<Vec<FeedbackRow>>>,parsed_to_yield: Mutex<Vec<Vec<ParsedRow>>>,+ activities_to_yield: Mutex<Vec<Vec<ActivityRow>>>,last_frames_since: Mutex<Option<String>>,last_audio_since: Mutex<Option<String>>,last_memories_since: Mutex<Option<String>>,last_feedback_since: Mutex<Option<String>>,last_parsed_since: Mutex<Option<String>>,+ last_activity_since: Mutex<Option<String>>,}impl MockLocal {@@ -2616,11 +2670,13 @@ mod tests {memories_to_yield: Mutex::new(Vec::new()),feedback_to_yield: Mutex::new(Vec::new()),parsed_to_yield: Mutex::new(Vec::new()),+ activities_to_yield: Mutex::new(Vec::new()),last_frames_since: Mutex::new(None),last_audio_since: Mutex::new(None),last_memories_since: Mutex::new(None),last_feedback_since: Mutex::new(None),last_parsed_since: Mutex::new(None),+ last_activity_since: Mutex::new(None),}}@@ -2638,6 +2694,11 @@ mod tests {self.parsed_to_yield = Mutex::new(parsed);self}++ fn with_activities(mut self, activities: Vec<Vec<ActivityRow>>) -> Self {+ self.activities_to_yield = Mutex::new(activities);+ self+ }}/// Models the local `/search` and `/memories` pagination contract: the@@ -2740,6 +2801,7 @@ mod tests {"off".to_string(),"off".to_string(),);+ crate::enterprise_policy::set_activity_sync_enabled(false);}}@@ -2819,6 +2881,18 @@ mod tests {.pop().unwrap_or_default())}+ async fn fetch_activities_since(+ &self,+ since_ts: Option<&str>,+ ) -> Result<Vec<ActivityRow>, EnterpriseSyncError> {+ *self.last_activity_since.lock().unwrap() = since_ts.map(str::to_string);+ Ok(self+ .activities_to_yield+ .lock()+ .unwrap()+ .pop()+ .unwrap_or_default())+ }}fn test_cfg(dir: &TempDir, ingest_url: String) -> EnterpriseSyncConfig {@@ -3855,6 +3929,60 @@ mod tests {);}+ #[tokio::test]+ async fn activity_rows_upload_only_when_the_separate_stream_is_enabled() {+ let _guard = crate::enterprise_policy::sync_streams_test_lock();+ let _restore = RestoreDefaultSyncStreams;+ crate::enterprise_policy::set_sync_streams(+ false,+ false,+ false,+ false,+ false,+ false,+ "off".to_string(),+ "off".to_string(),+ );++ let server = wiremock::MockServer::start().await;+ wiremock::Mock::given(wiremock::matchers::method("POST"))+ .and(wiremock::matchers::path("/ingest"))+ .respond_with(wiremock::ResponseTemplate::new(200))+ .expect(1)+ .mount(&server)+ .await;+ let dir = TempDir::new().unwrap();+ let cfg = test_cfg(&dir, format!("{}/ingest", server.uri()));+ let mut cursor = Cursor::default();+ cursor.boundary.activity_ts = Some("2026-09-02T18:00:00Z".to_string());+ let local = MockLocal::new(vec![], vec![]).with_activities(vec![vec![activity(+ "activity-1",+ "2026-09-02T18:00:00Z",+ "2026-09-02T18:15:00Z",+ )]]);++ let disabled_report =+ run_one_sync(&cfg, &mut cursor, &local, &reqwest::Client::new())+ .await+ .unwrap();+ assert_eq!(disabled_report.activities, 0);+ assert!(local.last_activity_since.lock().unwrap().is_none());+ assert!(server.received_requests().await.unwrap().is_empty());++ crate::enterprise_policy::set_activity_sync_enabled(true);+ let report = run_one_sync(&cfg, &mut cursor, &local, &reqwest::Client::new())+ .await+ .unwrap();+ assert_eq!(report.activities, 1);+ assert_eq!(+ cursor.boundary.activity_ts.as_deref(),+ Some("2026-09-02T18:15:00Z")+ );+ let requests = server.received_requests().await.unwrap();+ let batch = screenpipe_telemetry_wire::parse_jsonl(&requests[0].body);+ assert!(matches!(batch.records.as_slice(), [TelemetryRecord::Activity { .. }]));+ }+#[tokio::test]async fn direct_upload_success_advances_cursor_after_complete() {let server = wiremock::MockServer::start().await;diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/upload.rsindex b7905d43a..e92c54ac3 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/upload.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/upload.rs@@ -348,6 +348,7 @@ pub fn direct_upload_cursors(cursor: &Cursor) -> DirectUploadCursors {DirectUploadCursors {last_frame_ts: cursor.last_frame_ts.clone(),last_parsed_ts: cursor.last_parsed_ts.clone(),+ last_activity_ts: cursor.boundary.activity_ts.clone(),last_audio_ts: cursor.last_audio_ts.clone(),last_ui_ts: cursor.last_ui_ts.clone(),last_memory_ts: cursor.last_memory_ts.clone(),@@ -664,6 +665,7 @@ mod tests {DirectUploadRecordCounts {frames: 1,parsed: 0,+ activities: 0,audio: 0,ui: 0,snapshots: 0,@@ -673,6 +675,7 @@ mod tests {DirectUploadCursors {last_frame_ts: Some("2026-05-13T18:00:00Z".to_string()),last_parsed_ts: None,+ last_activity_ts: None,last_audio_ts: None,last_ui_ts: None,last_memory_ts: None,diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rsindex 47fa7d1a5..25e09dfc3 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs@@ -25,9 +25,10 @@ mod imp {use crate::recording::local_api_context_from_app;use base64::Engine;use ee_sync::{- AudioRow, EnterpriseSyncConfig, EnterpriseSyncError, FeedbackRow, FrameRow, LocalApiClient,- MemoryRow, ParsedRow, SnapshotRow, UiEventRow,+ ActivityRow, AudioRow, EnterpriseSyncConfig, EnterpriseSyncError, FeedbackRow, FrameRow,+ LocalApiClient, MemoryRow, ParsedRow, SnapshotRow, UiEventRow,};+ use screenpipe_telemetry_wire::ActivityEvidenceRow;use serde::Deserialize;use sha2::{Digest, Sha256};use std::collections::HashMap;@@ -331,6 +332,43 @@ mod imp {Ok(out)}+ async fn fetch_activities_since(+ &self,+ since_ts: Option<&str>,+ ) -> Result<Vec<ActivityRow>, EnterpriseSyncError> {+ let since = since_ts+ .and_then(|value| chrono::DateTime::parse_from_rfc3339(value).ok())+ .map(|value| value.with_timezone(&chrono::Utc))+ .unwrap_or_else(chrono::Utc::now);+ let entries = crate::activity_history::entries_ending_after(&self.app, since)+ .map_err(EnterpriseSyncError::LocalApi)?;+ Ok(entries+ .into_iter()+ .map(|entry| ActivityRow {+ activity_id: entry.id,+ activity_kind: entry.kind,+ meeting_id: entry.meeting_id,+ timestamp: entry.end_at.clone(),+ start_at: entry.start_at,+ end_at: entry.end_at,+ title: entry.title,+ summary: entry.summary,+ evidence: entry+ .evidence+ .into_iter()+ .map(|evidence| ActivityEvidenceRow {+ kind: evidence.kind,+ at: evidence.at,+ frame_id: evidence.frame_id,+ meeting_id: evidence.meeting_id,+ app_name: evidence.app_name,+ label: evidence.label,+ })+ .collect(),+ })+ .collect())+ }+async fn fetch_ui_events_since(&self,since_ts: Option<&str>,@@ -764,6 +802,8 @@ mod imp {frames: bool,#[serde(default)]parsed: bool,+ #[serde(default)]+ activities: bool,#[serde(default = "default_enabled_sync_stream")]audio: bool,#[serde(default = "default_enabled_sync_stream")]@@ -783,6 +823,7 @@ mod imp {Self {frames: true,parsed: false,+ activities: false,audio: true,ui_events: true,memories: true,@@ -824,6 +865,7 @@ mod imp {self.feedback_mode(),self.frame_images_mode(),);+ crate::enterprise_policy::set_activity_sync_enabled(self.activities);}}@@ -1809,6 +1851,7 @@ mod imp {sync_streams: NativeSyncStreams {frames: true,parsed: true,+ activities: true,audio: false,ui_events: true,memories: false,@@ -1831,6 +1874,7 @@ mod imp {assert!(policy.require_account_login);assert!(!policy.recording_allowed);assert!(policy.sync_streams.parsed);+ assert!(policy.sync_streams.activities);assert!(!policy.sync_streams.audio);assert_eq!(policy.sync_streams.feedback_mode(), "full");assert_eq!(policy.sync_streams.frame_images_mode(), "all");diff --git a/apps/screenpipe-app-tauri/src-tauri/src/main.rs b/apps/screenpipe-app-tauri/src-tauri/src/main.rsindex 466993252..356199943 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/main.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/main.rs@@ -168,7 +168,7 @@ pub use server::spawn_server;pub use enterprise_install_metadata::get_enterprise_install_metadata;pub use enterprise_host_identity::get_enterprise_host_identity;pub use enterprise_policy::set_enterprise_policy;-pub use enterprise_policy::set_sync_streams;+pub use enterprise_policy::{set_activity_sync_enabled, set_sync_streams};pub use enterprise_recording_access::set_enterprise_recording_authorized;pub use permissions::do_permissions_check;pub use permissions::open_permission_settings;diff --git a/crates/screenpipe-gateway/src/api.rs b/crates/screenpipe-gateway/src/api.rsindex 6675d191b..afa4f70da 100644--- a/crates/screenpipe-gateway/src/api.rs+++ b/crates/screenpipe-gateway/src/api.rs@@ -757,6 +757,71 @@ async fn query_parsed(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Va.collect())}+async fn query_activities(+ db: &DatabaseManager,+ kq: &KindQuery<'_>,+) -> Result<Vec<Value>, sqlx::Error> {+ let fts = kq.q.and_then(fts_expression);+ let order = if kq.newest_first { "DESC" } else { "ASC" };+ let sql = format!(+ r#"SELECT a.rowid, a.device_id, a.device_label, a.timestamp, a.title, a.summary,+ a.activity_id, a.payload_json+ FROM gateway_activity_records a+ {fts_join}+ WHERE datetime(a.timestamp) >= datetime(?1) AND datetime(a.timestamp) <= datetime(?2)+ AND (?3 IS NULL OR a.device_id = ?3)+ AND (?4 IS NULL OR datetime(a.timestamp) {cmp} datetime(?4)+ OR (datetime(a.timestamp) = datetime(?4) AND printf('activity:%020d', a.rowid) {cmp} ?5))+ ORDER BY datetime(a.timestamp) {order}, printf('activity:%020d', a.rowid) {order}+ LIMIT ?6"#,+ cmp = if kq.newest_first { "<" } else { ">" },+ fts_join = if fts.is_some() {+ "JOIN gateway_activity_records_fts fts ON fts.rowid = a.rowid AND gateway_activity_records_fts MATCH ?7"+ } else {+ ""+ },+ );+ let mut query = sqlx::query_as::<+ _,+ (i64, String, String, String, String, String, String, String),+ >(sqlx::AssertSqlSafe(sql))+ .bind(&kq.since)+ .bind(&kq.until)+ .bind(kq.device_id)+ .bind(kq.cursor_t)+ .bind(kq.cursor_id)+ .bind(kq.limit);+ if let Some(expression) = fts {+ query = query.bind(expression);+ }+ Ok(query+ .fetch_all(&db.pool)+ .await?+ .into_iter()+ .map(+ |(rowid, device_id, label, timestamp, title, summary, activity_id, payload)| {+ let activity =+ serde_json::from_str::<Value>(&payload).unwrap_or_else(|_| json!({}));+ with_cursor_id(+ json!({+ "kind": "activity",+ "t": timestamp,+ "device": label,+ "device_id": device_id,+ "app": null,+ "window": null,+ "url": null,+ "text": format!("{title}\n{summary}"),+ "activity_id": activity_id,+ "activity": activity,+ }),+ format!("activity:{rowid:020}"),+ )+ },+ )+ .collect())+}+async fn query_audio(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Value>, sqlx::Error> {let fts = kq.q.and_then(fts_expression);let order = if kq.newest_first { "DESC" } else { "ASC" };@@ -1114,6 +1179,7 @@ async fn query_kinds(let mut part = match *kind {"frame" => query_frames(db, kq).await?,"parsed" => query_parsed(db, kq).await?,+ "activity" => query_activities(db, kq).await?,"audio" => query_audio(db, kq).await?,"ui" => query_ui(db, kq).await?,"memory" => query_memories(db, kq).await?,@@ -1283,7 +1349,9 @@ async fn search(};let mut results = match query_kinds(&state.db,- &["frame", "parsed", "audio", "ui", "memory", "feedback"],+ &[+ "frame", "parsed", "activity", "audio", "ui", "memory", "feedback",+ ],&kq,).await@@ -1384,7 +1452,9 @@ async fn records(Err(resp) => return *resp,};let kinds: Vec<&str> = match kind_filter.as_str() {- "all" => vec!["frame", "parsed", "audio", "ui", "memory", "feedback"],+ "all" => vec![+ "frame", "parsed", "activity", "audio", "ui", "memory", "feedback",+ ],k => vec![k],};let kq = KindQuery {@@ -2441,6 +2511,55 @@ mod tests {}));}+ #[tokio::test]+ async fn activity_records_are_searchable_and_kind_filterable() {+ let dir = tempfile::tempdir().unwrap();+ let router = seeded_router(&dir).await;+ let writable = sqlx::sqlite::SqlitePoolOptions::new()+ .max_connections(1)+ .connect_with(+ sqlx::sqlite::SqliteConnectOptions::new()+ .filename(dir.path().join("gateway.db"))+ .read_only(false),+ )+ .await+ .unwrap();+ sqlx::query(+ r#"INSERT INTO gateway_activity_records+ (sync_id, device_id, device_label, activity_id, timestamp, start_at, title, summary, payload_json)+ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"#,+ )+ .bind("dev-a:activity:activity-1")+ .bind("dev-a")+ .bind("Alice Mac")+ .bind("activity-1")+ .bind("2026-07-22T10:15:00Z")+ .bind("2026-07-22T10:00:00Z")+ .bind("Prepared enterprise rollout")+ .bind("Completed the deployment checklist")+ .bind(r#"{"activity_id":"activity-1","title":"Prepared enterprise rollout"}"#)+ .execute(&writable)+ .await+ .unwrap();++ let (status, body) = get_json(+ &router,+ "/api/enterprise/v1/search?q=deployment&since=2026-07-22T00:00:00Z",+ )+ .await;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(body["results"][0]["kind"], "activity");+ assert_eq!(body["results"][0]["activity_id"], "activity-1");++ let (status, body) = get_json(+ &router,+ "/api/enterprise/v1/records?kind=activity&since=2026-07-22T00:00:00Z",+ )+ .await;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(body["records"].as_array().unwrap().len(), 1);+ }+#[tokio::test]async fn devices_lists_both_with_labels() {let dir = tempfile::tempdir().unwrap();diff --git a/crates/screenpipe-gateway/src/bin/seed.rs b/crates/screenpipe-gateway/src/bin/seed.rsindex 462798d04..d7d35bd1b 100644--- a/crates/screenpipe-gateway/src/bin/seed.rs+++ b/crates/screenpipe-gateway/src/bin/seed.rs@@ -205,6 +205,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {// compute_batch_id, so they have to match what was actually written.frames: 2 + usize::from(device.stale_sentinel),parsed: 1,+ activities: 0,audio: 1,ui: 1,snapshots: 0,@@ -214,6 +215,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {let cursors = DirectUploadCursors {last_frame_ts: Some(format!("2026-07-22T{:02}:05:00Z", device.hour)),last_parsed_ts: Some(format!("2026-07-22T{:02}:06:00Z", device.hour)),+ last_activity_ts: None,last_audio_ts: Some(format!("2026-07-22T{:02}:02:00Z", device.hour)),last_ui_ts: Some(format!("2026-07-22T{:02}:03:00Z", device.hour)),last_memory_ts: Some(format!("2026-07-22T{:02}:08:00Z", device.hour)),diff --git a/crates/screenpipe-gateway/src/ingest.rs b/crates/screenpipe-gateway/src/ingest.rsindex fc48f3696..e2f4a0ab5 100644--- a/crates/screenpipe-gateway/src/ingest.rs+++ b/crates/screenpipe-gateway/src/ingest.rs@@ -30,6 +30,7 @@//! |---|---|---|//! | frame | `video_chunks` (virtual, `gw://` path) + `frames` (`full_text`) | `frames.machine_id` |//! | parsed | `gateway_parsed_records` + FTS (full wire payload retained) | `device_id` |+//! | activity | `gateway_activity_records` + FTS (full wire payload retained) | `device_id` |//! | audio | `audio_chunks` (`gw://` path) + `audio_transcriptions` (+ speaker by name) | `audio_chunks.machine_id` |//! | ui | `ui_events` | `ui_events.machine_id` |//! | snapshot | JPEG on disk + `frames.snapshot_path` on the matching frame row | via frame row |@@ -48,8 +49,8 @@ use base64::Engine;use screenpipe_db::DatabaseManager;use screenpipe_sync::{BlobSource, ListRequest};use screenpipe_telemetry_wire::{- org_telemetry_prefix, parse_jsonl, parse_telemetry_key, AudioRow, FeedbackRow, FrameRow,- MemoryRow, ParsedRow, SnapshotRow, TelemetryRecord, UiEventRow,+ org_telemetry_prefix, parse_jsonl, parse_telemetry_key, ActivityRow, AudioRow, FeedbackRow,+ FrameRow, MemoryRow, ParsedRow, SnapshotRow, TelemetryRecord, UiEventRow,};use tracing::{debug, info, warn};@@ -205,6 +206,35 @@ pub async fn ensure_gateway_schema(db: &DatabaseManager) -> Result<(), GatewayEr).execute(&mut **tx.conn()).await?;+ sqlx::query(+ r#"CREATE TABLE IF NOT EXISTS gateway_activity_records (+ sync_id TEXT PRIMARY KEY,+ device_id TEXT NOT NULL,+ device_label TEXT NOT NULL,+ activity_id TEXT NOT NULL,+ timestamp TEXT NOT NULL,+ start_at TEXT NOT NULL,+ title TEXT NOT NULL,+ summary TEXT NOT NULL,+ payload_json TEXT NOT NULL+ )"#,+ )+ .execute(&mut **tx.conn())+ .await?;+ sqlx::query(+ r#"CREATE VIRTUAL TABLE IF NOT EXISTS gateway_activity_records_fts+ USING fts5(title, summary, content='gateway_activity_records', content_rowid='rowid')"#,+ )+ .execute(&mut **tx.conn())+ .await?;+ sqlx::query(+ r#"CREATE TRIGGER IF NOT EXISTS gateway_activity_records_ai AFTER INSERT ON gateway_activity_records BEGIN+ INSERT INTO gateway_activity_records_fts(rowid, title, summary)+ VALUES (new.rowid, new.title, new.summary);+ END"#,+ )+ .execute(&mut **tx.conn())+ .await?;tx.commit().await?;Ok(())}@@ -456,6 +486,11 @@ impl Ingestor {device_label,parsed,} => insert_parsed(conn, device_id, device_label, parsed).await?,+ TelemetryRecord::Activity {+ device_id,+ device_label,+ activity,+ } => insert_activity(conn, device_id, device_label, activity).await?,TelemetryRecord::Audio {device_id, audio, ..} => insert_audio(conn, device_id, audio).await?,@@ -550,6 +585,34 @@ async fn insert_parsed(Ok(result.rows_affected() > 0)}+async fn insert_activity(+ conn: &mut Conn,+ device_id: &str,+ device_label: &str,+ activity: &ActivityRow,+) -> Result<bool, GatewayError> {+ let sync_id = format!("{device_id}:activity:{}", activity.activity_id);+ let payload_json = serde_json::to_string(activity)+ .map_err(|error| GatewayError::DbWrite(format!("serialize activity row: {error}")))?;+ let result = sqlx::query(+ r#"INSERT OR IGNORE INTO gateway_activity_records+ (sync_id, device_id, device_label, activity_id, timestamp, start_at, title, summary, payload_json)+ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"#,+ )+ .bind(sync_id)+ .bind(device_id)+ .bind(device_label)+ .bind(&activity.activity_id)+ .bind(&activity.end_at)+ .bind(&activity.start_at)+ .bind(&activity.title)+ .bind(&activity.summary)+ .bind(payload_json)+ .execute(&mut **conn)+ .await?;+ Ok(result.rows_affected() > 0)+}+/// frame → virtual `gw://` video chunk + frames row (full_text carries the/// merged accessibility/OCR text; frames_fts triggers index it). Mirrors/// screenpipe-db's SyncInsertFrame + SyncInsertOcr, with `gw://` instead ofdiff --git a/crates/screenpipe-telemetry-wire/src/lib.rs b/crates/screenpipe-telemetry-wire/src/lib.rsindex d146b2a17..0f4da10be 100644--- a/crates/screenpipe-telemetry-wire/src/lib.rs+++ b/crates/screenpipe-telemetry-wire/src/lib.rs@@ -35,6 +35,7 @@ pub use manifest::{DIRECT_UPLOAD_CONTENT_TYPE, DIRECT_UPLOAD_READABLE_MODE, DIRECT_UPLOAD_WRITE_ONLY_MODE,};pub use records::{- build_feedback_jsonl, build_jsonl, build_jsonl_with_parsed, parse_jsonl, AudioRow, FeedbackRow,- FrameRow, MemoryRow, ParsedBatch, ParsedRow, SnapshotRow, TelemetryRecord, UiEventRow,+ build_feedback_jsonl, build_jsonl, build_jsonl_with_parsed, build_jsonl_with_semantic_streams,+ parse_jsonl, ActivityEvidenceRow, ActivityRow, AudioRow, FeedbackRow, FrameRow, MemoryRow,+ ParsedBatch, ParsedRow, SnapshotRow, TelemetryRecord, UiEventRow,};diff --git a/crates/screenpipe-telemetry-wire/src/manifest.rs b/crates/screenpipe-telemetry-wire/src/manifest.rsindex 73dd46aa4..63fdbf9bb 100644--- a/crates/screenpipe-telemetry-wire/src/manifest.rs+++ b/crates/screenpipe-telemetry-wire/src/manifest.rs@@ -33,6 +33,8 @@ pub struct DirectUploadRecordCounts {pub frames: usize,#[serde(default)]pub parsed: usize,+ #[serde(default)]+ pub activities: usize,pub audio: usize,pub ui: usize,pub snapshots: usize,@@ -47,6 +49,8 @@ pub struct DirectUploadCursors {pub last_frame_ts: Option<String>,#[serde(default)]pub last_parsed_ts: Option<String>,+ #[serde(default)]+ pub last_activity_ts: Option<String>,pub last_audio_ts: Option<String>,pub last_ui_ts: Option<String>,#[serde(default)]@@ -111,6 +115,7 @@ mod tests {DirectUploadRecordCounts {frames: 1,parsed: 0,+ activities: 0,audio: 0,ui: 0,snapshots: 0,@@ -120,6 +125,7 @@ mod tests {DirectUploadCursors {last_frame_ts: Some("2026-05-13T18:00:00Z".to_string()),last_parsed_ts: None,+ last_activity_ts: None,last_audio_ts: None,last_ui_ts: None,last_memory_ts: None,@@ -183,8 +189,10 @@ mod tests {})).unwrap();assert_eq!(counts.parsed, 0);+ assert_eq!(counts.activities, 0);assert_eq!(counts.memories, 0);assert_eq!(cursors.last_parsed_ts, None);+ assert_eq!(cursors.last_activity_ts, None);assert_eq!(cursors.last_memory_ts, None);}}diff --git a/crates/screenpipe-telemetry-wire/src/records.rs b/crates/screenpipe-telemetry-wire/src/records.rsindex 5229306d2..06056330b 100644--- a/crates/screenpipe-telemetry-wire/src/records.rs+++ b/crates/screenpipe-telemetry-wire/src/records.rs@@ -5,7 +5,7 @@//! JSONL record schema for enterprise telemetry batches.//!//! One record per line, tagged by `kind` so mixed streams stay trivially-//! parseable: `kind: "frame" | "parsed" | "audio" | "ui" | "snapshot" | "memory" | "feedback"`.+//! parseable: `kind: "frame" | "parsed" | "activity" | "audio" | "ui" | "snapshot" | "memory" | "feedback"`.//! Every record carries the originating `device_id` + `device_label` at the//! top level (flattened next to the kind-specific row fields).//!@@ -59,6 +59,31 @@ pub struct ParsedRow {pub actors: Vec<serde_json::Value>,}+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]+pub struct ActivityEvidenceRow {+ pub kind: String,+ pub at: String,+ pub frame_id: Option<i64>,+ pub meeting_id: Option<i64>,+ pub app_name: Option<String>,+ pub label: String,+}++/// One AI-interpreted Activity persisted by the desktop scheduler.+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]+pub struct ActivityRow {+ pub activity_id: String,+ pub activity_kind: String,+ pub meeting_id: Option<i64>,+ /// Primary record timestamp used by generic enterprise readers.+ pub timestamp: String,+ pub start_at: String,+ pub end_at: String,+ pub title: String,+ pub summary: String,+ pub evidence: Vec<ActivityEvidenceRow>,+}+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]pub struct AudioRow {pub transcription_id: i64,@@ -154,7 +179,7 @@ pub struct FeedbackRow {}/// One JSONL line. Tagged enum keeps mixed streams trivially parseable —-/// `kind: "frame" | "parsed" | "audio" | "ui" | "snapshot" | "memory" | "feedback"`.+/// `kind: "frame" | "parsed" | "activity" | "audio" | "ui" | "snapshot" | "memory" | "feedback"`.#[derive(Debug, Clone, Serialize, Deserialize)]#[serde(tag = "kind", rename_all = "lowercase")]pub enum TelemetryRecord {@@ -170,6 +195,12 @@ pub enum TelemetryRecord {#[serde(flatten)]parsed: ParsedRow,},+ Activity {+ device_id: String,+ device_label: String,+ #[serde(flatten)]+ activity: ActivityRow,+ },Audio {device_id: String,device_label: String,@@ -207,6 +238,7 @@ impl TelemetryRecord {match self {Self::Frame { device_id, .. }| Self::Parsed { device_id, .. }+ | Self::Activity { device_id, .. }| Self::Audio { device_id, .. }| Self::Ui { device_id, .. }| Self::Snapshot { device_id, .. }@@ -219,6 +251,7 @@ impl TelemetryRecord {match self {Self::Frame { device_label, .. }| Self::Parsed { device_label, .. }+ | Self::Activity { device_label, .. }| Self::Audio { device_label, .. }| Self::Ui { device_label, .. }| Self::Snapshot { device_label, .. }@@ -233,6 +266,7 @@ impl TelemetryRecord {match self {Self::Frame { frame, .. } => &frame.timestamp,Self::Parsed { parsed, .. } => &parsed.timestamp,+ Self::Activity { activity, .. } => &activity.end_at,Self::Audio { audio, .. } => &audio.timestamp,Self::Ui { ui, .. } => &ui.timestamp,Self::Snapshot { snapshot, .. } => &snapshot.timestamp,@@ -246,6 +280,7 @@ impl TelemetryRecord {match self {Self::Frame { .. } => "frame",Self::Parsed { .. } => "parsed",+ Self::Activity { .. } => "activity",Self::Audio { .. } => "audio",Self::Ui { .. } => "ui",Self::Snapshot { .. } => "snapshot",@@ -317,9 +352,35 @@ pub fn build_jsonl_with_parsed(ui: &[UiEventRow],snapshots: &[SnapshotRow],memories: &[MemoryRow],+) -> Vec<u8> {+ build_jsonl_with_semantic_streams(+ device_id,+ device_label,+ frames,+ parsed,+ &[],+ audio,+ ui,+ snapshots,+ memories,+ )+}++/// Serialize a batch including all separately gated semantic streams.+pub fn build_jsonl_with_semantic_streams(+ device_id: &str,+ device_label: &str,+ frames: &[FrameRow],+ parsed: &[ParsedRow],+ activities: &[ActivityRow],+ audio: &[AudioRow],+ ui: &[UiEventRow],+ snapshots: &[SnapshotRow],+ memories: &[MemoryRow],) -> Vec<u8> {let mut out = Vec::with_capacity(- (frames.len() + parsed.len() + audio.len() + ui.len() + memories.len()) * 256+ (frames.len() + parsed.len() + activities.len() + audio.len() + ui.len() + memories.len())+ * 256+ snapshots.len() * 50_000,);let mut push = |record: &TelemetryRecord, id: i64| match serde_json::to_vec(record) {@@ -361,6 +422,14 @@ pub fn build_jsonl_with_parsed(p.frame_id,);}+ for activity in activities {+ let record = TelemetryRecord::Activity {+ device_id: device_id.to_string(),+ device_label: device_label.to_string(),+ activity: activity.clone(),+ };+ push(&record, 0);+ }for a in audio {push(&TelemetryRecord::Audio {@@ -460,6 +529,38 @@ mod tests {}}+ #[test]+ fn activity_stream_round_trips_as_its_own_kind() {+ let activity = ActivityRow {+ activity_id: "activity-1".to_string(),+ activity_kind: "work".to_string(),+ meeting_id: None,+ timestamp: "2026-09-02T18:15:00Z".to_string(),+ start_at: "2026-09-02T18:00:00Z".to_string(),+ end_at: "2026-09-02T18:15:00Z".to_string(),+ title: "Prepared rollout".to_string(),+ summary: "Completed the deployment checklist".to_string(),+ evidence: Vec::new(),+ };+ let body = build_jsonl_with_semantic_streams(+ "dev-a",+ "Alice Mac",+ &[],+ &[],+ &[activity],+ &[],+ &[],+ &[],+ &[],+ );+ let parsed = parse_jsonl(&body);+ assert_eq!(parsed.skipped_lines, 0);+ assert!(matches!(+ parsed.records.as_slice(),+ [TelemetryRecord::Activity { activity, .. }] if activity.activity_id == "activity-1"+ ));+ }+#[test]fn jsonl_round_trips_across_devices() {let mut body = build_jsonl(