YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(capture): exclude unfocused incognito windows from monitor capture (#6260)
Louis Beaumontde8ac0781b1b
diff --git a/crates/screenpipe-engine/src/capture_exclusions.rs b/crates/screenpipe-engine/src/capture_exclusions.rsindex a41bacdd4..436713dfd 100644--- a/crates/screenpipe-engine/src/capture_exclusions.rs+++ b/crates/screenpipe-engine/src/capture_exclusions.rs@@ -19,7 +19,10 @@//! configured — the caller must skip the frame's pixels rather than risk//! storing a window the user asked to hide.-use screenpipe_screen::capture_screenshot_by_window::{get_excluded_sck_window_ids, WindowFilters};+use screenpipe_a11y::tree::TreeWalkerConfig;+use screenpipe_screen::capture_screenshot_by_window::{+ get_excluded_sck_window_ids, ExclusionSources, WindowFilters,+};use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};use std::sync::{Mutex, OnceLock};use std::time::{Duration, Instant};@@ -66,7 +69,10 @@ fn exclusion_cache() -> &'static Mutex<ExclusionCache> {/// (in flight elsewhere, timed out, or task failure). The blocking closure/// always writes the cache and clears the in-flight flag on completion, even/// after its awaiting caller gave up.-async fn refresh_excluded_sck_window_ids(window_filters: &WindowFilters) -> Option<Vec<u32>> {+async fn refresh_excluded_sck_window_ids(+ window_filters: &WindowFilters,+ sources: ExclusionSources,+) -> Option<Vec<u32>> {if EXCLUSION_REFRESH_IN_FLIGHT.swap(true, Ordering::AcqRel) {// A refresh (possibly stuck in CGWindowListCopyWindowInfo) is already// running; don't pile another blocked thread behind it.@@ -75,7 +81,7 @@ async fn refresh_excluded_sck_window_ids(window_filters: &WindowFilters) -> Optilet filters = window_filters.clone();let handle = tokio::task::spawn_blocking(move || {- let mut ids = get_excluded_sck_window_ids(&filters);+ let mut ids = get_excluded_sck_window_ids(&filters, sources);ids.sort_unstable();ids.dedup();{@@ -116,15 +122,30 @@ async fn refresh_excluded_sck_window_ids(window_filters: &WindowFilters) -> Opti}}+/// Privacy sources implied by the current tree-walker config.+///+/// The a11y gates apply these only to the focus-owning window; the exclusion+/// resolver applies them to every on-screen window, which is what keeps an+/// unfocused private window out of a monitor-wide frame.+pub(crate) fn exclusion_sources(config: &TreeWalkerConfig) -> ExclusionSources {+ ExclusionSources {+ ignore_incognito_windows: config.ignore_incognito_windows,+ enhanced_incognito_detection: config.enhanced_incognito_detection,+ }+}+/// Exclusion ids for the visual-change PROBE (frame diffing only — nothing is/// stored). Best effort: fresh ids when available, cached ids of any age/// otherwise. Over-inclusive probe pixels merely risk a spurious/// visual-change trigger; the storage capture applies its own stricter rule.-pub(crate) async fn probe_excluded_sck_window_ids(window_filters: &WindowFilters) -> Vec<u32> {- if window_filters.is_empty() {+pub(crate) async fn probe_excluded_sck_window_ids(+ window_filters: &WindowFilters,+ sources: ExclusionSources,+) -> Vec<u32> {+ if window_filters.is_empty() && sources.is_empty() {return Vec::new();}- if let Some(ids) = refresh_excluded_sck_window_ids(window_filters).await {+ if let Some(ids) = refresh_excluded_sck_window_ids(window_filters, sources).await {return ids;}exclusion_cache()@@ -143,11 +164,12 @@ pub(crate) async fn probe_excluded_sck_window_ids(window_filters: &WindowFilters/// privacy property without freezing the loop.pub(crate) async fn storage_excluded_sck_window_ids(window_filters: &WindowFilters,+ sources: ExclusionSources,) -> Option<Vec<u32>> {- if window_filters.is_empty() {+ if window_filters.is_empty() && sources.is_empty() {return Some(Vec::new());}- if let Some(ids) = refresh_excluded_sck_window_ids(window_filters).await {+ if let Some(ids) = refresh_excluded_sck_window_ids(window_filters, sources).await {return Some(ids);}let cache = exclusion_cache().lock().unwrap_or_else(|e| e.into_inner());@@ -272,7 +294,9 @@ mod tests {// of spawning another blocking task behind it.assert!(!EXCLUSION_REFRESH_IN_FLIGHT.swap(true, Ordering::AcqRel));let started = Instant::now();- let result = refresh_excluded_sck_window_ids(&excluding_filters()).await;+ let result =+ refresh_excluded_sck_window_ids(&excluding_filters(), ExclusionSources::default())+ .await;assert_eq!(result, None);// Must return via the fast path, never the 5s timeout path.assert!(started.elapsed() < EXCLUSION_REFRESH_TIMEOUT / 2);@@ -294,7 +318,7 @@ mod tests {}let filters = excluding_filters();assert_eq!(- storage_excluded_sck_window_ids(&filters).await,+ storage_excluded_sck_window_ids(&filters, ExclusionSources::default()).await,Some(vec![42]));@@ -305,8 +329,14 @@ mod tests {let mut cache = exclusion_cache().lock().unwrap_or_else(|e| e.into_inner());cache.refreshed_at = instant_secs_ago(600);}- assert_eq!(storage_excluded_sck_window_ids(&filters).await, None);- assert_eq!(probe_excluded_sck_window_ids(&filters).await, vec![42]);+ assert_eq!(+ storage_excluded_sck_window_ids(&filters, ExclusionSources::default()).await,+ None+ );+ assert_eq!(+ probe_excluded_sck_window_ids(&filters, ExclusionSources::default()).await,+ vec![42]+ );EXCLUSION_REFRESH_IN_FLIGHT.store(false, Ordering::Release);}@@ -315,12 +345,63 @@ mod tests {let filters = WindowFilters::new(&[], &[], &[]);assert!(filters.is_empty());assert_eq!(- storage_excluded_sck_window_ids(&filters).await,+ storage_excluded_sck_window_ids(&filters, ExclusionSources::default()).await,Some(Vec::new()));assert_eq!(- probe_excluded_sck_window_ids(&filters).await,+ probe_excluded_sck_window_ids(&filters, ExclusionSources::default()).await,Vec::<u32>::new());}++ /// Most users configure no ignore patterns but keep incognito on (it is+ /// the default). If the empty-filters short circuit still applied, the+ /// resolver would never run and screenpipe/screenpipe#6198 would persist+ /// for exactly the default configuration.+ #[tokio::test]+ async fn incognito_alone_defeats_the_empty_filters_short_circuit() {+ let _guard = SHARED_STATE_TEST_LOCK+ .lock()+ .unwrap_or_else(|e| e.into_inner());+ let filters = WindowFilters::new(&[], &[], &[]);+ let sources = ExclusionSources {+ ignore_incognito_windows: true,+ enhanced_incognito_detection: false,+ };+ assert!(filters.is_empty());+ assert!(!sources.is_empty());++ // Hold the in-flight flag so no real WindowServer call happens: the+ // point is that resolution is ATTEMPTED rather than short circuited,+ // which here means falling through to the fail-closed cache rule+ // instead of returning `Some(vec![])`.+ assert!(!EXCLUSION_REFRESH_IN_FLIGHT.swap(true, Ordering::AcqRel));+ {+ let mut cache = exclusion_cache().lock().unwrap_or_else(|e| e.into_inner());+ cache.ids = Vec::new();+ cache.refreshed_at = None;+ }+ assert_eq!(+ storage_excluded_sck_window_ids(&filters, sources).await,+ None+ );+ EXCLUSION_REFRESH_IN_FLIGHT.store(false, Ordering::Release);+ }++ #[test]+ fn exclusion_sources_mirror_the_tree_walker_config() {+ let mut config = TreeWalkerConfig::default();+ config.ignore_incognito_windows = true;+ config.enhanced_incognito_detection = true;+ assert_eq!(+ exclusion_sources(&config),+ ExclusionSources {+ ignore_incognito_windows: true,+ enhanced_incognito_detection: true,+ }+ );++ config.ignore_incognito_windows = false;+ assert!(exclusion_sources(&config).is_empty());+ }}diff --git a/crates/screenpipe-engine/src/event_driven_capture.rs b/crates/screenpipe-engine/src/event_driven_capture.rsindex 63963fa6e..ca0bcc209 100644--- a/crates/screenpipe-engine/src/event_driven_capture.rs+++ b/crates/screenpipe-engine/src/event_driven_capture.rs@@ -8,7 +8,9 @@//! Captures happen only on meaningful user events: app switch, window focus,//! click, typing pause, scroll stop, clipboard, and periodic idle fallback.-use crate::capture_exclusions::{probe_excluded_sck_window_ids, storage_excluded_sck_window_ids};+use crate::capture_exclusions::{+ exclusion_sources, probe_excluded_sck_window_ids, storage_excluded_sck_window_ids,+};use crate::hot_frame_cache::{HotFrame, HotFrameCache};use crate::power::PowerProfile;use crate::semantic_worker::{SemanticCaptureGap, SemanticProjectionJob, SemanticProjectionSender};@@ -1794,7 +1796,11 @@ pub(crate) async fn event_driven_capture_loop(&monitor_liveness,screenpipe_screen::CaptureLoopStage::ExclusionProbe,);- let fresh_ids = probe_excluded_sck_window_ids(&capture_params.window_filters).await;+ let fresh_ids = probe_excluded_sck_window_ids(+ &capture_params.window_filters,+ exclusion_sources(capture_params.tree_walker_config),+ )+ .await;if fresh_ids != cached_excluded_ids {cached_excluded_ids = fresh_ids;}@@ -2821,7 +2827,11 @@ async fn do_capture(let storage_exclusions = if screenshot_disabled {Some(Vec::new())} else {- storage_excluded_sck_window_ids(¶ms.window_filters).await+ storage_excluded_sck_window_ids(+ ¶ms.window_filters,+ exclusion_sources(params.tree_walker_config),+ )+ .await};let skip_pixels_for_unknown_exclusions = storage_exclusions.is_none();if skip_pixels_for_unknown_exclusions {diff --git a/crates/screenpipe-engine/src/hd_recorder.rs b/crates/screenpipe-engine/src/hd_recorder.rsindex 02165f0c3..8f649f144 100644--- a/crates/screenpipe-engine/src/hd_recorder.rs+++ b/crates/screenpipe-engine/src/hd_recorder.rs@@ -64,6 +64,12 @@ pub struct HdRecorderConfig {/// Ignored-URL patterns — windows showing a blocked URL are excluded from/// HD capture too (URL privacy parity with the event-driven capture loop).pub ignored_urls: Vec<String>,+ /// Exclude browser private/incognito windows from HD capture. Without this+ /// the HD stream keeps recording a private window that the event-driven+ /// loop excludes, since HD frames bypass the a11y gates entirely.+ pub ignore_incognito_windows: bool,+ /// Let the incognito detector use browser-native APIs (macOS Automation).+ pub enhanced_incognito_detection: bool,}/// Per-monitor HD recorder loop. Idles until an HD session is active, then@@ -344,7 +350,7 @@ mod macos {stop_signal: &Arc<AtomicBool>,) -> Result<()> {use screenpipe_screen::capture_screenshot_by_window::{- get_excluded_sck_window_ids, WindowFilters,+ get_excluded_sck_window_ids, ExclusionSources, WindowFilters,};// Read the HD rate live from the controller so tray changes (10↔30fps)@@ -358,7 +364,13 @@ mod macos {&config.included_windows,&config.ignored_urls,);- let mut excluded = get_excluded_sck_window_ids(&filters);+ let mut excluded = get_excluded_sck_window_ids(+ &filters,+ ExclusionSources {+ ignore_incognito_windows: config.ignore_incognito_windows,+ enhanced_incognito_detection: config.enhanced_incognito_detection,+ },+ );excluded.sort_unstable();excluded.dedup();diff --git a/crates/screenpipe-engine/src/vision_manager/manager.rs b/crates/screenpipe-engine/src/vision_manager/manager.rsindex e8770646c..9bb7b1868 100644--- a/crates/screenpipe-engine/src/vision_manager/manager.rs+++ b/crates/screenpipe-engine/src/vision_manager/manager.rs@@ -721,6 +721,8 @@ impl VisionManager {ignored_windows: self.config.ignored_windows.clone(),included_windows: self.config.included_windows.clone(),ignored_urls: self.config.ignored_urls.clone(),+ ignore_incognito_windows: self.config.ignore_incognito_windows,+ enhanced_incognito_detection: self.config.enhanced_incognito_detection,};let hd_handle = self.vision_handlediff --git a/crates/screenpipe-screen/src/capture_screenshot_by_window.rs b/crates/screenpipe-screen/src/capture_screenshot_by_window.rsindex 25397a532..54817a5ab 100644--- a/crates/screenpipe-screen/src/capture_screenshot_by_window.rs+++ b/crates/screenpipe-screen/src/capture_screenshot_by_window.rs@@ -935,29 +935,76 @@ fn get_all_windows() -> Result<Vec<WindowData>, Box<dyn Error>> {.collect())}-/// Resolve ignored/included window filters to SCK window IDs (macOS only).+/// Privacy sources that contribute windows to the SCK exclusion set, beyond+/// the user's own ignore/include patterns.///-/// Enumerates all on-screen windows, checks each against `WindowFilters::is_valid()`,-/// and returns the SCK window IDs of windows that should be excluded from capture.-/// These IDs can be passed to `capture_image_excluding()` so ScreenCaptureKit-/// never renders those windows into the capture buffer.+/// Incognito lives here rather than inside [`WindowFilters`] because it is not+/// a pattern: deciding it needs the browser-native detector, which may run+/// AppleScript and is therefore only safe on the bounded, single-flight+/// exclusion-refresh path.+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+pub struct ExclusionSources {+ /// Exclude browser private/incognito windows anywhere on screen, not just+ /// the focused one.+ pub ignore_incognito_windows: bool,+ /// Let the incognito detector use browser-native APIs (macOS Automation).+ /// Without this, Chromium private windows are undetectable on macOS — their+ /// window name is the bare page title (see [`get_excluded_sck_window_ids`]).+ pub enhanced_incognito_detection: bool,+}++impl ExclusionSources {+ /// True when no source here can contribute a window, so a WindowServer+ /// enumeration is only worth doing if the user configured patterns.+ pub fn is_empty(&self) -> bool {+ !self.ignore_incognito_windows+ }+}++/// Resolve ignored/included window filters and privacy sources to SCK window+/// IDs (macOS only).+///+/// Enumerates all on-screen windows, checks each against+/// `WindowFilters::is_valid()` and the incognito detector, and returns the SCK+/// window IDs of windows that should be excluded from capture. These IDs can be+/// passed to `capture_image_excluding()` so ScreenCaptureKit never renders+/// those windows into the capture buffer.+///+/// Why incognito is resolved here rather than at the focused-window gate: the+/// screenshot is monitor-wide, so a *visible but unfocused* private window is+/// in the frame even though the focus-scoped a11y gates never see it. Removing+/// it from the SCK composite means those pixels are never rendered at all,+/// rather than captured and then discarded (screenpipe/screenpipe#6198).+///+/// Detection caveat: on macOS a Chromium private window's CGWindow name is the+/// bare page title, with no marker to match — verified against Chrome 141, both+/// `kCGWindowName` and AppleScript `name of window` return e.g. `Example+/// Domain`. Title matching alone therefore cannot catch it, and+/// `enhanced_incognito_detection` (which asks the browser directly) is required+/// for Chromium. Firefox and Safari do carry the marker in the title and work+/// in either mode.///-/// Returns an empty vec if no windows match the filters or on error.+/// Returns an empty vec if no windows match or on error.#[cfg(target_os = "macos")]-pub fn get_excluded_sck_window_ids(window_filters: &WindowFilters) -> Vec<u32> {- if window_filters.ignore_patterns.is_empty()- && window_filters.include_patterns.is_empty()- && window_filters.ignored_urls.is_empty()- {+pub fn get_excluded_sck_window_ids(+ window_filters: &WindowFilters,+ sources: ExclusionSources,+) -> Vec<u32> {+ if window_filters.is_empty() && sources.is_empty() {return Vec::new();}+ let detector = sources+ .ignore_incognito_windows+ .then(|| screenpipe_a11y::incognito::create_detector(sources.enhanced_incognito_detection));+let (cg_windows, _) = get_cg_window_list();- let excluded = excluded_sck_window_ids_from_cg_windows(&cg_windows, window_filters);+ let excluded =+ excluded_sck_window_ids_from_cg_windows(&cg_windows, window_filters, detector.as_deref());if !excluded.is_empty() {debug!(- "resolved {} ignored window(s) to SCK IDs: {:?}",+ "resolved {} excluded window(s) to SCK IDs: {:?}",excluded.len(),excluded);@@ -974,18 +1021,37 @@ pub fn get_excluded_sck_window_ids(window_filters: &WindowFilters) -> Vec<u32> {fn excluded_sck_window_ids_from_cg_windows(cg_windows: &[CGWindowInfo],window_filters: &WindowFilters,+ incognito_detector: Option<&dyn screenpipe_a11y::incognito::IncognitoDetector>,) -> Vec<u32> {cg_windows.iter().filter(|window| window.window_id != 0 && !window.owner_name.is_empty())- .filter(|window| !window_filters.is_valid(&window.owner_name, &window.window_name))+ .filter(|window| {+ if !window_filters.is_valid(&window.owner_name, &window.window_name) {+ return true;+ }+ // Layer 0 is a normal application window. Menubar extras, overlays+ // and system chrome sit above it and never host browsing content,+ // so asking the detector about them only costs AppleScript calls.+ window.layer == 0+ && incognito_detector.is_some_and(|detector| {+ detector.is_incognito(&window.owner_name, window.pid, &window.window_name)+ })+ }).map(|window| window.window_id).collect()}/// Non-macOS stub — SCK exclusion is not available.+///+/// Windows and Linux have no way to drop another process's window from a+/// monitor capture, so the unfocused-private-window exposure remains there and+/// needs a different remedy.#[cfg(not(target_os = "macos"))]-pub fn get_excluded_sck_window_ids(_window_filters: &WindowFilters) -> Vec<u32> {+pub fn get_excluded_sck_window_ids(+ _window_filters: &WindowFilters,+ _sources: ExclusionSources,+) -> Vec<u32> {Vec::new()}@@ -1785,6 +1851,27 @@ mod tests {);}+ /// Stands in for the real detector so these tests stay hermetic: the+ /// macOS detector shells out to AppleScript and its answer depends on+ /// which browsers happen to be running on the machine.+ struct FakeIncognitoDetector {+ private_titles: Vec<String>,+ }++ impl screenpipe_a11y::incognito::IncognitoDetector for FakeIncognitoDetector {+ fn is_incognito(&self, _app: &str, _pid: i32, window_title: &str) -> bool {+ self.private_titles.iter().any(|t| t == window_title)+ }+ }++ fn detector(+ private_titles: &[&str],+ ) -> Box<dyn screenpipe_a11y::incognito::IncognitoDetector> {+ Box::new(FakeIncognitoDetector {+ private_titles: private_titles.iter().map(|t| t.to_string()).collect(),+ })+ }+#[test]fn test_excluded_sck_ids_respect_ignore_patterns() {let windows = vec![@@ -1794,7 +1881,7 @@ mod tests {let filters = WindowFilters::new(&["password".to_string()], &[], &[]);assert_eq!(- excluded_sck_window_ids_from_cg_windows(&windows, &filters),+ excluded_sck_window_ids_from_cg_windows(&windows, &filters, None),vec![111]);}@@ -1808,7 +1895,7 @@ mod tests {let filters = WindowFilters::new(&[], &["terminal".to_string()], &[]);assert_eq!(- excluded_sck_window_ids_from_cg_windows(&windows, &filters),+ excluded_sck_window_ids_from_cg_windows(&windows, &filters, None),vec![222]);}@@ -1822,11 +1909,98 @@ mod tests {assert!(excluded_sck_window_ids_from_cg_windows(&[missing_owner, missing_id],- &filters+ &filters,+ None+ )+ .is_empty());+ }++ /// The bug in screenpipe/screenpipe#6198: the private window is visible+ /// but not focused, so every focus-scoped gate misses it. It must still+ /// leave the SCK composite.+ #[test]+ fn test_excluded_sck_ids_exclude_unfocused_incognito_window() {+ let windows = vec![+ make_window(111, 0, 0, 0, 800, 600, "Google Chrome", "Example Domain"),+ make_window(222, 0, 0, 0, 800, 600, "Google Chrome", "New Tab"),+ ];+ let filters = WindowFilters::new(&[], &[], &[]);++ assert_eq!(+ excluded_sck_window_ids_from_cg_windows(+ &windows,+ &filters,+ Some(detector(&["Example Domain"]).as_ref())+ ),+ vec![111]+ );+ }++ /// Incognito exclusion is opt-in: with the setting off, a private+ /// window is treated like any other window.+ #[test]+ fn test_excluded_sck_ids_keep_incognito_when_detector_absent() {+ let windows = vec![make_window(+ 111,+ 0,+ 0,+ 0,+ 800,+ 600,+ "Google Chrome",+ "Example Domain",+ )];+ let filters = WindowFilters::new(&[], &[], &[]);++ assert!(excluded_sck_window_ids_from_cg_windows(&windows, &filters, None).is_empty());+ }++ /// Menubar extras and overlays sit above layer 0 and never host+ /// browsing content; asking the detector about them would only spend+ /// AppleScript calls (and risk excluding unrelated system chrome).+ #[test]+ fn test_excluded_sck_ids_ignore_incognito_above_layer_zero() {+ let windows = vec![make_window(+ 111,+ 25,+ 0,+ 0,+ 300,+ 40,+ "Google Chrome",+ "Example Domain",+ )];+ let filters = WindowFilters::new(&[], &[], &[]);++ assert!(excluded_sck_window_ids_from_cg_windows(+ &windows,+ &filters,+ Some(detector(&["Example Domain"]).as_ref())).is_empty());}+ /// A pattern-excluded window and a private window must both drop out+ /// in the same pass — the two sources are independent.+ #[test]+ fn test_excluded_sck_ids_combine_patterns_and_incognito() {+ let windows = vec![+ make_window(111, 0, 0, 0, 800, 600, "Password Manager", "Vault"),+ make_window(222, 0, 0, 0, 800, 600, "Google Chrome", "Example Domain"),+ make_window(333, 0, 0, 0, 800, 600, "Notes", "Daily note"),+ ];+ let filters = WindowFilters::new(&["password".to_string()], &[], &[]);++ assert_eq!(+ excluded_sck_window_ids_from_cg_windows(+ &windows,+ &filters,+ Some(detector(&["Example Domain"]).as_ref())+ ),+ vec![111, 222]+ );+ }+#[test]fn test_topmost_skips_overlay_layers() {// Wispr Flow overlay (layer 25) is first in z-order but should be skippeddiff --git a/crates/screenpipe-screen/src/core.rs b/crates/screenpipe-screen/src/core.rsindex 12cf1e645..4f44cd5ff 100644--- a/crates/screenpipe-screen/src/core.rs+++ b/crates/screenpipe-screen/src/core.rs@@ -5,7 +5,7 @@#[cfg(target_os = "macos")]use crate::apple::perform_ocr_apple;use crate::capture_screenshot_by_window::{- get_excluded_sck_window_ids, CapturedWindow, WindowFilters,+ get_excluded_sck_window_ids, CapturedWindow, ExclusionSources, WindowFilters,};use crate::custom_ocr::perform_ocr_custom;use crate::frame_comparison::{FrameComparer, FrameComparisonConfig};@@ -209,8 +209,14 @@ pub async fn continuous_capture(let mut captured = None;for attempt in 0..=MAX_CAPTURE_RETRIES {- match capture_monitor_image(&monitor, &get_excluded_sck_window_ids(&window_filters))- .await+ // Pattern filters only: this legacy loop carries no incognito+ // config. The production path is `event_driven_capture`, which+ // resolves privacy sources via `exclusion_sources`.+ match capture_monitor_image(+ &monitor,+ &get_excluded_sck_window_ids(&window_filters, ExclusionSources::default()),+ )+ .await{Ok(result) => {if attempt > 0 {