Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

feat(enterprise): auto-submit device logs when enrolled but not uploading (#4563)

Louis Beaumont95c490e1aa65
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs
index 983c7dd33..c975a2c51 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs
@@ -526,6 +526,21 @@ mod imp {
cfg.device_id, cfg.device_label, cfg.ingest_url
);
+ // Point the stalled-upload watchdog at the real app-log dirs. App logs
+ // live in the screenpipe data dir (RollingFileAppender), NOT Tauri's
+ // app_data_dir — so the from_env default would miss them.
+ let mut cfg = cfg;
+ let mut log_dirs = Vec::new();
+ if let Ok(d) = crate::log_files::get_screenpipe_data_dir(app) {
+ log_dirs.push(d);
+ }
+ if let Ok(d) = crate::log_files::get_data_dir(app) {
+ log_dirs.push(d);
+ }
+ if !log_dirs.is_empty() {
+ cfg.log_dirs = log_dirs;
+ }
+
let api = local_api_context_from_app(app);
let api_url_base = api.url("");
// NB: don't capture `api.api_key` here — at spawn-time the
diff --git a/ee/desktop-rust/enterprise_sync.rs b/ee/desktop-rust/enterprise_sync.rs
index 1c40b3e40..f8936f8f9 100644
--- a/ee/desktop-rust/enterprise_sync.rs
+++ b/ee/desktop-rust/enterprise_sync.rs
@@ -64,6 +64,15 @@ const BACKOFF_MAX: Duration = Duration::from_secs(60 * 60);
/// retrying every interval.
const RETRY_AFTER_AUTH_FAIL: Duration = Duration::from_secs(60 * 60);
+/// Stalled-upload watchdog: an enrolled device that's been failing to land data
+/// in the org's storage for this long auto-submits its logs to support (the
+/// same endpoint the in-app "send logs" button uses — no UI, so it works on
+/// "run hidden" managed devices). Several missed 5-min ticks past first-run lag.
+const UPLOAD_STALL_THRESHOLD: Duration = Duration::from_secs(30 * 60);
+/// Re-arm window for the auto-submit, so a persistently-broken device reports at
+/// most ~twice a day instead of every tick.
+const AUTO_LOG_COOLDOWN: Duration = Duration::from_secs(12 * 60 * 60);
+
/// Default endpoint. Overridable via `SCREENPIPE_ENTERPRISE_INGEST_URL` for
/// staging / on-prem.
pub const DEFAULT_INGEST_URL: &str = "https://screenpipe.com/api/enterprise/ingest";
@@ -87,6 +96,10 @@ pub struct EnterpriseSyncConfig {
pub cursor_path: PathBuf,
/// Hosted plaintext ingest or direct encrypted customer-storage upload.
pub upload_mode: EnterpriseUploadMode,
+ /// Directories to scan for `*.log` files when the stalled-upload watchdog
+ /// auto-submits diagnostics. Empty = watchdog can still fire but ships a
+ /// "no log files found" marker. Set by the caller to the app's log dirs.
+ pub log_dirs: Vec<PathBuf>,
}
impl EnterpriseSyncConfig {
@@ -152,6 +165,9 @@ impl EnterpriseSyncConfig {
ingest_url,
cursor_path,
upload_mode,
+ // Logs live in the app data dir by default; the caller may extend
+ // this (e.g. the second tracing dir) after construction.
+ log_dirs: vec![app_data_dir],
})
}
@@ -1112,6 +1128,203 @@ pub async fn fulfill_frame_requests(
/// Run the sync forever (or until shutdown signal fires). Resilient to all
/// transient errors. Idempotent across restarts via the cursor file.
+/// Decide whether to auto-submit diagnostic logs for a stalled upload pipeline.
+/// Pure so it unit-tests without a clock. Fires only when ALL hold:
+/// - the sync has run long enough to have had a fair shot (startup grace);
+/// - no data has actually landed in the org's storage within the stall window
+/// (`since_last_data` = None means it never has);
+/// - we've seen at least one REAL upload failure since data last landed — so a
+/// genuinely idle/paused device (uploads succeed with nothing new) never
+/// phones home; and
+/// - we're past the cooldown since the last auto-submit.
+fn should_auto_submit_stall_logs(
+ running_for: Duration,
+ since_last_data: Option<Duration>,
+ saw_failure_since_data: bool,
+ since_last_submit: Option<Duration>,
+) -> bool {
+ if running_for < UPLOAD_STALL_THRESHOLD {
+ return false;
+ }
+ let stale = since_last_data.map_or(true, |d| d >= UPLOAD_STALL_THRESHOLD);
+ if !stale || !saw_failure_since_data {
+ return false;
+ }
+ since_last_submit.map_or(true, |d| d >= AUTO_LOG_COOLDOWN)
+}
+
+/// Path of the persisted last-auto-submit marker (next to the sync cursor). The
+/// cooldown is persisted so it survives app restarts — otherwise a crash-looping
+/// or frequently-restarting stuck device would re-submit logs on every boot.
+fn stall_marker_path(cfg: &EnterpriseSyncConfig) -> PathBuf {
+ cfg.cursor_path.with_file_name(".enterprise-stall-log")
+}
+
+/// Load the persisted last-auto-submit time (unix seconds). None when never
+/// submitted / unreadable / malformed.
+fn load_last_auto_submit(cfg: &EnterpriseSyncConfig) -> Option<std::time::SystemTime> {
+ let secs: u64 = std::fs::read_to_string(stall_marker_path(cfg))
+ .ok()?
+ .trim()
+ .parse()
+ .ok()?;
+ Some(std::time::UNIX_EPOCH + Duration::from_secs(secs))
+}
+
+/// Persist the last-auto-submit time (best-effort; a write failure only risks an
+/// earlier retry, never a crash).
+fn save_last_auto_submit(cfg: &EnterpriseSyncConfig, when: std::time::SystemTime) {
+ if let Ok(d) = when.duration_since(std::time::UNIX_EPOCH) {
+ let _ = std::fs::write(stall_marker_path(cfg), d.as_secs().to_string());
+ }
+}
+
+/// Origin (`scheme://host[:port]`) of the ingest URL — the logs endpoints live
+/// on the same host. Falls back to prod if the URL is malformed.
+fn logs_base_url(ingest_url: &str) -> String {
+ if let Some(scheme_end) = ingest_url.find("://") {
+ let rest = &ingest_url[scheme_end + 3..];
+ let host_len = rest.find('/').unwrap_or(rest.len());
+ return ingest_url[..scheme_end + 3 + host_len].to_string();
+ }
+ "https://screenpi.pe".to_string()
+}
+
+/// Stable, regex-safe identifier (`^[A-Za-z0-9._:-]+$`, ≤128) for the logs API.
+fn stall_log_identifier(device_id: &str) -> String {
+ let safe: String = device_id
+ .chars()
+ .filter(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | ':' | '-'))
+ .collect();
+ format!("enterprise-auto-{safe}").chars().take(128).collect()
+}
+
+/// UTF-8-boundary-safe tail of `s`, at most `max_bytes`.
+fn tail_str(s: &str, max_bytes: usize) -> &str {
+ if s.len() <= max_bytes {
+ return s;
+ }
+ let mut start = s.len() - max_bytes;
+ while start < s.len() && !s.is_char_boundary(start) {
+ start += 1;
+ }
+ &s[start..]
+}
+
+/// Read recent `*.log` content from `dirs`, newest file first, bounded total.
+async fn collect_log_text(dirs: &[PathBuf]) -> String {
+ const MAX_TOTAL: usize = 256 * 1024;
+ let mut files: Vec<(std::time::SystemTime, PathBuf)> = Vec::new();
+ for dir in dirs {
+ if let Ok(mut rd) = tokio::fs::read_dir(dir).await {
+ while let Ok(Some(entry)) = rd.next_entry().await {
+ let path = entry.path();
+ if path.extension().map(|e| e == "log").unwrap_or(false) {
+ let modified = entry
+ .metadata()
+ .await
+ .ok()
+ .and_then(|m| m.modified().ok())
+ .unwrap_or(std::time::SystemTime::UNIX_EPOCH);
+ files.push((modified, path));
+ }
+ }
+ }
+ }
+ files.sort_by_key(|(m, _)| std::cmp::Reverse(*m));
+
+ let mut out = String::new();
+ for (_, path) in files {
+ if out.len() >= MAX_TOTAL {
+ break;
+ }
+ if let Ok(content) = tokio::fs::read_to_string(&path).await {
+ let name = path.file_name().unwrap_or_default().to_string_lossy();
+ let budget = MAX_TOTAL - out.len();
+ out.push_str(&format!("\n=== {} ===\n{}", name, tail_str(&content, budget)));
+ }
+ }
+ if out.is_empty() {
+ out.push_str("[no log files found]");
+ }
+ out
+}
+
+/// Best-effort: ship the device's app logs to support via the same public
+/// endpoint the in-app "send logs" button uses. No UI required, so it works on
+/// "run hidden" managed devices. Never panics; failures just warn and are
+/// retried next cooldown.
+async fn submit_stall_logs(cfg: &EnterpriseSyncConfig, http: &reqwest::Client) {
+ let base = logs_base_url(&cfg.ingest_url);
+ let identifier = stall_log_identifier(&cfg.device_id);
+
+ // 1. signed upload URL
+ let signed: serde_json::Value = match http
+ .post(format!("{base}/api/logs"))
+ .json(&serde_json::json!({ "identifier": identifier, "type": "machine" }))
+ .send()
+ .await
+ .and_then(|r| r.error_for_status())
+ {
+ Ok(r) => match r.json().await {
+ Ok(v) => v,
+ Err(e) => {
+ warn!("enterprise sync: stall-log url decode failed: {e}");
+ return;
+ }
+ },
+ Err(e) => {
+ warn!("enterprise sync: stall-log url request failed: {e}");
+ return;
+ }
+ };
+ let signed_url = signed["data"]["signedUrl"].as_str();
+ let path = signed["data"]["path"].as_str();
+ let (signed_url, path) = match (signed_url, path) {
+ (Some(u), Some(p)) => (u.to_string(), p.to_string()),
+ _ => {
+ warn!("enterprise sync: stall-log url response missing fields");
+ return;
+ }
+ };
+
+ // 2. upload the log bytes
+ let body = collect_log_text(&cfg.log_dirs).await;
+ if let Err(e) = http
+ .put(&signed_url)
+ .header("Content-Type", "text/plain")
+ .body(body)
+ .send()
+ .await
+ .and_then(|r| r.error_for_status())
+ {
+ warn!("enterprise sync: stall-log upload failed: {e}");
+ return;
+ }
+
+ // 3. confirm (this is what files it for support)
+ let feedback = format!(
+ "auto: enterprise device recording but not landing data in org storage (license {}, device {}, mode {:?})",
+ cfg.license_key,
+ cfg.device_id,
+ std::mem::discriminant(&cfg.upload_mode)
+ );
+ let _ = http
+ .post(format!("{base}/api/logs/confirm"))
+ .json(&serde_json::json!({
+ "path": path,
+ "identifier": identifier,
+ "type": "machine",
+ "os": std::env::consts::OS,
+ "os_version": "",
+ "app_version": option_env!("CARGO_PKG_VERSION").unwrap_or("unknown"),
+ "feedback_text": feedback,
+ }))
+ .send()
+ .await;
+ info!("enterprise sync: auto-submitted diagnostic logs (device enrolled but not uploading)");
+}
+
pub async fn run(
cfg: EnterpriseSyncConfig,
local: Arc<dyn LocalApiClient>,
@@ -1130,8 +1343,51 @@ pub async fn run(
let mut cursor = Cursor::load(&cfg.cursor_path);
let mut backoff = BACKOFF_INITIAL;
+ // Stalled-upload watchdog state. `last_auto_submit` is persisted (wall-clock)
+ // so the cooldown survives app restarts; the rest is in-memory (a fresh start
+ // re-applies the startup grace, which is the desired behavior).
+ let started = std::time::Instant::now();
+ let mut last_data_upload: Option<std::time::Instant> = None;
+ let mut saw_failure_since_data = false;
+ let mut last_auto_submit: Option<std::time::SystemTime> = load_last_auto_submit(&cfg);
+
loop {
- match run_one_sync(&cfg, &mut cursor, local.as_ref(), &http).await {
+ let result = run_one_sync(&cfg, &mut cursor, local.as_ref(), &http).await;
+
+ // Watchdog bookkeeping: a data-bearing success clears the failure flag;
+ // any error counts as a real upload failure. Then maybe phone home.
+ match &result {
+ Ok(report) if report.bytes > 0 => {
+ last_data_upload = Some(std::time::Instant::now());
+ saw_failure_since_data = false;
+ }
+ Ok(_) => {}
+ Err(_) => saw_failure_since_data = true,
+ }
+ // Wall-clock elapsed since the persisted last submit. A clock that moved
+ // backwards (Err) is treated as "just submitted" (Duration::ZERO) so we
+ // never spam on a clock glitch.
+ let since_last_submit = last_auto_submit.map(|t| {
+ std::time::SystemTime::now()
+ .duration_since(t)
+ .unwrap_or(Duration::ZERO)
+ });
+ if should_auto_submit_stall_logs(
+ started.elapsed(),
+ last_data_upload.map(|t| t.elapsed()),
+ saw_failure_since_data,
+ since_last_submit,
+ ) {
+ submit_stall_logs(&cfg, &http).await;
+ // Persist BEFORE updating memory so a restart right after still honors
+ // the cooldown. Set regardless of submit success → a fully-offline
+ // device retries next window rather than hammering every tick.
+ let now = std::time::SystemTime::now();
+ save_last_auto_submit(&cfg, now);
+ last_auto_submit = Some(now);
+ }
+
+ match result {
Ok(report) => {
if report.frames > 0
|| report.audio > 0
@@ -1223,6 +1479,109 @@ mod tests {
use std::sync::Mutex;
use tempfile::TempDir;
+ const MIN: Duration = Duration::from_secs(60);
+
+ #[test]
+ fn watchdog_fires_when_stalled_with_failures_past_grace() {
+ // running long enough, no data ever, saw a failure, never submitted
+ assert!(should_auto_submit_stall_logs(40 * MIN, None, true, None));
+ // last data is old + a failure since → fire
+ assert!(should_auto_submit_stall_logs(
+ 2 * 60 * MIN,
+ Some(35 * MIN),
+ true,
+ None
+ ));
+ }
+
+ #[test]
+ fn watchdog_silent_during_startup_grace() {
+ // only just started — give the first ticks a chance even with no data
+ assert!(!should_auto_submit_stall_logs(5 * MIN, None, true, None));
+ }
+
+ #[test]
+ fn watchdog_silent_for_idle_device_no_failures() {
+ // no data recently but NO upload failure → genuinely idle/paused, not broken
+ assert!(!should_auto_submit_stall_logs(60 * MIN, Some(40 * MIN), false, None));
+ assert!(!should_auto_submit_stall_logs(60 * MIN, None, false, None));
+ }
+
+ #[test]
+ fn watchdog_silent_when_data_is_flowing() {
+ // recent successful upload → healthy even if a failure was seen
+ assert!(!should_auto_submit_stall_logs(60 * MIN, Some(2 * MIN), true, None));
+ }
+
+ #[test]
+ fn watchdog_respects_cooldown() {
+ // stalled + failing, but submitted recently → wait
+ assert!(!should_auto_submit_stall_logs(
+ 5 * 60 * MIN,
+ Some(40 * MIN),
+ true,
+ Some(60 * MIN)
+ ));
+ // cooldown elapsed → fire again
+ assert!(should_auto_submit_stall_logs(
+ 20 * 60 * MIN,
+ Some(40 * MIN),
+ true,
+ Some(13 * 60 * MIN)
+ ));
+ }
+
+ #[test]
+ fn cooldown_persists_across_restarts() {
+ // The crash-loop guard: a restart must NOT reset the cooldown, or a
+ // stuck device would re-submit on every boot.
+ let dir = TempDir::new().unwrap();
+ let cfg = test_cfg(&dir, "http://x".into());
+ assert!(load_last_auto_submit(&cfg).is_none()); // nothing yet
+
+ let now = std::time::SystemTime::now();
+ save_last_auto_submit(&cfg, now);
+
+ // a fresh process reloads the marker and is still within cooldown
+ let loaded = load_last_auto_submit(&cfg).expect("persisted");
+ let since = std::time::SystemTime::now()
+ .duration_since(loaded)
+ .unwrap_or(Duration::ZERO);
+ assert!(since < AUTO_LOG_COOLDOWN);
+ assert!(!should_auto_submit_stall_logs(60 * MIN, None, true, Some(since)));
+ }
+
+ #[test]
+ fn logs_base_url_takes_origin() {
+ assert_eq!(
+ logs_base_url("https://screenpi.pe/api/enterprise/ingest"),
+ "https://screenpi.pe"
+ );
+ assert_eq!(
+ logs_base_url("http://localhost:3000/api/enterprise/ingest"),
+ "http://localhost:3000"
+ );
+ assert_eq!(logs_base_url("not a url"), "https://screenpi.pe");
+ }
+
+ #[test]
+ fn stall_log_identifier_is_regex_safe() {
+ let id = stall_log_identifier("AB-12 34/xy");
+ assert!(id.starts_with("enterprise-auto-"));
+ assert!(id.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | ':' | '-')));
+ assert!(id.len() <= 128);
+ }
+
+ #[test]
+ fn tail_str_is_char_boundary_safe() {
+ assert_eq!(tail_str("hello", 100), "hello");
+ assert_eq!(tail_str("hello", 2), "lo");
+ // multi-byte: never split a char
+ let s = "aé"; // 'é' is 2 bytes
+ let t = tail_str(s, 1); // would land mid-'é' → step forward to boundary
+ assert!(s.ends_with(t) || t.is_empty());
+ }
+
fn frame(id: i64, ts: &str, app: &str, text: &str) -> FrameRow {
FrameRow {
frame_id: id,
@@ -1796,6 +2155,7 @@ mod tests {
ingest_url,
cursor_path: dir.path().join(CURSOR_FILENAME),
upload_mode: EnterpriseUploadMode::HostedIngest,
+ log_dirs: vec![dir.path().to_path_buf()],
}
}
@@ -2509,6 +2869,7 @@ mod tests {
ingest_url: format!("{server_uri}/api/enterprise/ingest"),
cursor_path: tmp.path().join("cursor.json"),
upload_mode: EnterpriseUploadMode::HostedIngest,
+ log_dirs: vec![tmp.path().to_path_buf()],
}
}