YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(telemetry): report phys_footprint so cold-memory leaks are visible (+ bound OCR throttle map) (#4662)
Louis Beaumont2e8fb8d8bcc5
diff --git a/crates/screenpipe-engine/src/event_driven_capture.rs b/crates/screenpipe-engine/src/event_driven_capture.rsindex 4bd183daf..aaa31c05d 100644--- a/crates/screenpipe-engine/src/event_driven_capture.rs+++ b/crates/screenpipe-engine/src/event_driven_capture.rs@@ -1830,13 +1830,23 @@ fn terminal_ocr_throttled(app_name: &str) -> bool {Err(_) => return false,};let now = Instant::now();- match guard.get(&app_key) {+ let throttled = match guard.get(&app_key) {Some(&last) if now.duration_since(last) < INTERVAL => true,_ => {guard.insert(app_key, now);false}+ };++ // Bound the map: keys are app names, so in practice this is the small+ // `is_ocr_heavy_app` allow-list — but it's a process-lifetime static, so+ // drop entries older than INTERVAL (they're stale and can't throttle+ // anything anymore) before it can grow without limit.+ if guard.len() > 64 {+ guard.retain(|_, &mut last| now.duration_since(last) < INTERVAL);}++ throttled}/// Apps whose accessibility tree tends to be thin, making OCR fallback expensive.diff --git a/crates/screenpipe-engine/src/resource_monitor.rs b/crates/screenpipe-engine/src/resource_monitor.rsindex d00a3e4bf..fb57ae65c 100644--- a/crates/screenpipe-engine/src/resource_monitor.rs+++ b/crates/screenpipe-engine/src/resource_monitor.rs@@ -18,6 +18,63 @@ use tracing::{error, info, warn};use crate::telemetry_context::TelemetryContext;+/// Read this process's physical memory footprint (bytes) via+/// `proc_pid_rusage(RUSAGE_INFO_V0)`. `ri_phys_footprint` is the exact value+/// Activity Monitor reports under "Memory" — it includes compressed and+/// swapped-out dirty pages, unlike resident set size. That makes it the metric+/// that actually surfaces a cold-memory leak (pages written once then paged+/// out), which RSS-based telemetry misses entirely. Returns `None` if the+/// syscall fails (never panics; caller falls back to RSS).+#[cfg(target_os = "macos")]+fn macos_phys_footprint_bytes() -> Option<u64> {+ // rusage_info_v0 layout from <sys/resource.h>. `ri_phys_footprint` is+ // present from v0, so we don't need a newer flavor. Fields after it are+ // included only to size the struct correctly for the kernel copy-out.+ #[repr(C)]+ #[derive(Default, Clone, Copy)]+ struct RUsageInfoV0 {+ ri_uuid: [u8; 16],+ ri_user_time: u64,+ ri_system_time: u64,+ ri_pkg_idle_wkups: u64,+ ri_interrupt_wkups: u64,+ ri_pageins: u64,+ ri_wired_size: u64,+ ri_resident_size: u64,+ ri_phys_footprint: u64,+ ri_proc_start_abstime: u64,+ ri_proc_exit_abstime: u64,+ ri_child_user_time: u64,+ ri_child_system_time: u64,+ ri_child_pkg_idle_wkups: u64,+ ri_child_interrupt_wkups: u64,+ ri_child_pageins: u64,+ ri_child_elapsed_abstime: u64,+ ri_diskio_bytesread: u64,+ ri_diskio_byteswritten: u64,+ }++ extern "C" {+ fn proc_pid_rusage(pid: i32, flavor: i32, buffer: *mut std::ffi::c_void) -> i32;+ }+ const RUSAGE_INFO_V0: i32 = 0;++ let mut info = RUsageInfoV0::default();+ let pid = std::process::id() as i32;+ let rc = unsafe {+ proc_pid_rusage(+ pid,+ RUSAGE_INFO_V0,+ &mut info as *mut RUsageInfoV0 as *mut std::ffi::c_void,+ )+ };+ if rc == 0 {+ Some(info.ri_phys_footprint)+ } else {+ None+ }+}+pub struct ResourceMonitor {start_time: Instant,resource_log_file: Option<String>, // analyse output here: https://colab.research.google.com/drive/1zELlGdzGdjChWKikSqZTHekm5XRxY-1r?usp=sharing@@ -320,6 +377,7 @@ impl ResourceMonitor {total_memory_gb: f64,system_total_memory: f64,total_cpu: f32,+ phys_footprint_gb: f64,) {let Some(client) = &self.posthog_client else {return;@@ -330,6 +388,10 @@ impl ResourceMonitor {properties.insert("distinct_id".to_string(), json!(&self.distinct_id));properties.insert("$lib".to_string(), json!("rust-reqwest"));properties.insert("total_memory_gb".to_string(), json!(total_memory_gb));+ // Activity-Monitor "Memory" — surfaces cold/compressed/swapped leaks+ // that resident set size (`total_memory_gb`) hides. See+ // `macos_phys_footprint_bytes`.+ properties.insert("phys_footprint_gb".to_string(), json!(phys_footprint_gb));properties.insert("system_total_memory_gb".to_string(),json!(system_total_memory),@@ -376,7 +438,7 @@ impl ResourceMonitor {}}- async fn collect_metrics(&self, sys: &System) -> (f64, f64, f64, f32, f64, Duration) {+ async fn collect_metrics(&self, sys: &System) -> (f64, f64, f64, f32, f64, Duration, f64) {let pid = std::process::id();let mut total_memory = 0.0;let mut max_virtual_memory = 0.0; // Changed from total to max@@ -408,6 +470,20 @@ impl ResourceMonitor {let memory_usage_percent = (total_memory / system_total_memory) * 100.0;let runtime = self.start_time.elapsed();+ // Physical footprint = the "Memory" number Activity Monitor shows, and+ // the one that actually tracks a leak: it counts compressed + swapped+ // dirty pages, which `memory()` (resident set size) does NOT. A cold+ // leak (pages written once, never re-read) gets compressed/swapped out,+ // so RSS stays flat while footprint climbs to the GBs users report.+ // On non-macOS we fall back to RSS (≈ footprint there, and `malloc_trim`+ // keeps Linux RSS honest), so the field is always populated.+ #[cfg(target_os = "macos")]+ let phys_footprint_gb = macos_phys_footprint_bytes()+ .map(|b| b as f64 / (1024.0 * 1024.0 * 1024.0))+ .unwrap_or(total_memory);+ #[cfg(not(target_os = "macos"))]+ let phys_footprint_gb = total_memory;+(total_memory,system_total_memory,@@ -415,13 +491,14 @@ impl ResourceMonitor {total_cpu,max_virtual_memory,runtime,+ phys_footprint_gb,)}/// Max resource log file size (10 MB). When exceeded the file is truncated.const MAX_RESOURCE_LOG_BYTES: u64 = 10 * 1024 * 1024;- async fn log_to_file(&self, metrics: (f64, f64, f64, f32, f64, Duration)) {+ async fn log_to_file(&self, metrics: (f64, f64, f64, f32, f64, Duration, f64)) {let (total_memory_gb,system_total_memory,@@ -429,6 +506,7 @@ impl ResourceMonitor {total_cpu,total_virtual_memory_gb,runtime,+ phys_footprint_gb,) = metrics;if let Some(ref filename) = self.resource_log_file {@@ -440,6 +518,7 @@ impl ResourceMonitor {"memory_usage_percent": memory_usage_percent,"total_cpu_percent": total_cpu,"total_virtual_memory_gb": total_virtual_memory_gb,+ "phys_footprint_gb": phys_footprint_gb,});// Append-only JSONL: one JSON object per line, no read-back needed.@@ -475,16 +554,18 @@ impl ResourceMonitor {total_cpu,total_virtual_memory_gb,runtime,+ phys_footprint_gb,) = metrics;// Log to console with virtual memory. Let tracing format lazily so// release builds with debug logging disabled avoid the String allocation.debug!(- "Runtime: {}s, Memory: {:.0}% ({:.2} GB / {:.2} GB), Virtual: {:.2} GB, CPU: {:.0}%",+ "Runtime: {}s, Memory: {:.0}% ({:.2} GB / {:.2} GB), Footprint: {:.2} GB, Virtual: {:.2} GB, CPU: {:.0}%",runtime.as_secs(),memory_usage_percent,total_memory_gb,system_total_memory,+ phys_footprint_gb,total_virtual_memory_gb,total_cpu);@@ -495,7 +576,7 @@ impl ResourceMonitor {// Send to PostHog if enabledif self.posthog_enabled {tokio::select! {- _ = self.send_to_posthog(total_memory_gb, system_total_memory, total_cpu) => {},+ _ = self.send_to_posthog(total_memory_gb, system_total_memory, total_cpu, phys_footprint_gb) => {},_ = tokio::time::sleep(Duration::from_secs(5)) => {warn!("PostHog request timed out");}@@ -592,16 +673,18 @@ impl ResourceMonitor {total_cpu,total_virtual_memory_gb,runtime,+ phys_footprint_gb,) = metrics;// Log to console with virtual memory. Let tracing format lazily so// release builds with debug logging disabled avoid the String allocation.debug!(- "Runtime: {}s, Memory: {:.0}% ({:.2} GB / {:.2} GB), Virtual: {:.2} GB, CPU: {:.0}%",+ "Runtime: {}s, Memory: {:.0}% ({:.2} GB / {:.2} GB), Footprint: {:.2} GB, Virtual: {:.2} GB, CPU: {:.0}%",runtime.as_secs(),memory_usage_percent,total_memory_gb,system_total_memory,+ phys_footprint_gb,total_virtual_memory_gb,total_cpu);@@ -622,3 +705,25 @@ impl ResourceMonitor {}}}++#[cfg(test)]+mod tests {+ /// Validates the `proc_pid_rusage` FFI: a live process always has a+ /// non-zero physical footprint. Guards against a wrong `rusage_info_v0`+ /// struct layout (which would silently read garbage / the wrong field).+ #[cfg(target_os = "macos")]+ #[test]+ fn phys_footprint_is_plausible() {+ let bytes = super::macos_phys_footprint_bytes()+ .expect("proc_pid_rusage(RUSAGE_INFO_V0) should succeed for self");+ // A running test process is comfortably above 1 MB and below 100 GB.+ assert!(+ bytes > 1024 * 1024,+ "phys_footprint suspiciously small ({bytes} bytes) — struct layout likely wrong"+ );+ assert!(+ bytes < 100 * 1024 * 1024 * 1024,+ "phys_footprint suspiciously large ({bytes} bytes) — struct layout likely wrong"+ );+ }+}diff --git a/crates/screenpipe-screen/src/apple.rs b/crates/screenpipe-screen/src/apple.rsindex 90012d1d6..c8cc08a81 100644--- a/crates/screenpipe-screen/src/apple.rs+++ b/crates/screenpipe-screen/src/apple.rs@@ -373,4 +373,103 @@ mod tests {assert_eq!(r[2].0, 5);assert_eq!(r[2].1, 1);}++ /// Reproduction for the macOS slow memory leak reported 2026-06-29+ /// (user `ingvar2424`, screenpipe v2.5.79 at 19.6 GB RSS after weeks of+ /// idle background running on DEFAULT settings → ~1 GB/day steady growth).+ ///+ /// Hypothesis under test: the per-frame **Apple Vision OCR** path+ /// (`perform_ocr_apple`) leaks native memory each call. Apple's Vision+ /// (`VNRecognizeTextRequest`) is a known source of CoreFoundation / MLModel+ /// buffer growth; such CF (Create-rule) allocations are NOT reclaimed by the+ /// `cidre::objc::ar_pool(...)` wrapper, which only drains *autoreleased*+ /// objects. Screen capture runs OCR continuously at the default cadence, so+ /// even a small per-call leak compounds to GB-scale over weeks.+ ///+ /// This drives `perform_ocr_apple` in a tight loop on a fresh synthetic+ /// 1280x800 frame each iteration and prints peak RSS (`ru_maxrss`) at+ /// checkpoints. Interpretation:+ /// - CLEAN path: RSS rises during warmup (model load) then PLATEAUS.+ /// - LEAK: RSS keeps climbing roughly linearly with iterations.+ ///+ /// Unlike the SCK capture repro, this needs **no Screen Recording TCC+ /// permission** — Vision runs on an in-memory image — so it reproduces in+ /// CI / headless. Kept `#[ignore]` (perf/memory repro, not correctness):+ /// cargo test -p screenpipe-screen --lib apple::tests::repro_apple_ocr_leak -- --ignored --nocapture+ #[cfg(target_os = "macos")]+ #[test]+ #[ignore = "macOS memory-leak repro; prints RSS deltas"]+ fn repro_apple_ocr_leak() {+ use super::perform_ocr_apple;+ use image::{DynamicImage, Rgba, RgbaImage};++ /// Peak resident memory in bytes (ru_maxrss is bytes on Darwin).+ fn peak_rss_bytes() -> u64 {+ unsafe {+ let mut ru: libc::rusage = std::mem::zeroed();+ libc::getrusage(libc::RUSAGE_SELF, &mut ru);+ ru.ru_maxrss as u64+ }+ }+ fn mb(b: u64) -> f64 {+ (b as f64) / (1024.0 * 1024.0)+ }++ // Build a representative capture-sized frame with high-contrast+ // "text-like" bars so the Vision recognizer engages its full pipeline.+ // Content varies by `seed` so the OS can't trivially short-circuit a+ // repeated identical input.+ fn make_frame(seed: u32) -> DynamicImage {+ let (w, h) = (1280u32, 800u32);+ let mut img = RgbaImage::from_pixel(w, h, Rgba([255, 255, 255, 255]));+ for line in 0..20u32 {+ let y0 = 20 + line * 38 + (seed % 7);+ let bar_h = 16u32;+ let x_end = 80 + ((line * 53 + seed) % (w - 160));+ for y in y0..(y0 + bar_h).min(h) {+ for x in 60..x_end.min(w) {+ // dashes to mimic words/letters+ if (x / 9) % 2 == 0 {+ img.put_pixel(x, y, Rgba([10, 10, 10, 255]));+ }+ }+ }+ }+ DynamicImage::ImageRgba8(img)+ }++ const N: usize = 3_000;+ const CHECKPOINT: usize = 250;++ let baseline = peak_rss_bytes();+ eprintln!("[ocr-repro] baseline peak RSS: {:.1} MB", mb(baseline));++ let mut last = baseline;+ for i in 0..N {+ let frame = make_frame(i as u32);+ let _ = perform_ocr_apple(&frame, &[]);++ if (i + 1) % CHECKPOINT == 0 {+ let now = peak_rss_bytes();+ eprintln!(+ "[ocr-repro] after {:>5} OCRs: peak RSS {:.1} MB (+{:.1} MB since last, +{:.1} MB total)",+ i + 1,+ mb(now),+ mb(now.saturating_sub(last)),+ mb(now.saturating_sub(baseline)),+ );+ last = now;+ }+ }++ let total = peak_rss_bytes().saturating_sub(baseline);+ eprintln!(+ "[ocr-repro] TOTAL peak-RSS growth over {} OCRs: {:.1} MB (~{:.1} KB/call)",+ N,+ mb(total),+ (total as f64) / 1024.0 / (N as f64),+ );+ // Diagnostic only — we want the printed curve, not a hard assertion that+ // could flake on shared CI. A clean path plateaus; a leak climbs.+ }}