YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix: preserve desktop context when excluding windows
Louis Beaumont24e140304415
diff --git a/crates/screenpipe-engine/src/event_driven_capture.rs b/crates/screenpipe-engine/src/event_driven_capture.rsindex 2cbbed512..2a7b098be 100644--- a/crates/screenpipe-engine/src/event_driven_capture.rs+++ b/crates/screenpipe-engine/src/event_driven_capture.rs@@ -1223,8 +1223,12 @@ pub(crate) async fn event_driven_capture_loop({Ok(Ok(output)) => {state.mark_captured();- if let Some(ref mut comparer) = frame_comparer {- let _ = comparer.compare(&output.image);+ if !capture_params.window_filters.has_window_patterns()+ && !capture_params.window_filters.has_url_allowlist()+ {+ if let Some(ref mut comparer) = frame_comparer {+ let _ = comparer.compare(&output.image);+ }}if let Some(ref result) = output.result {last_content_hash = result.content_hash;@@ -2137,11 +2141,16 @@ pub(crate) async fn event_driven_capture_loop(consecutive_capture_errors = 0;}- // Feed the captured frame to comparer so we don't- // re-trigger on the same visual state (reuses capture- // image — no extra screenshot needed)- if let Some(ref mut comparer) = frame_comparer {- let _ = comparer.compare(&output.image);+ // The visual probe observes the display stream. A+ // redacted capture is a different image of a static+ // desktop; seeding it here would retrigger capture on+ // every probe. Let probes maintain their own reference.+ if !capture_params.window_filters.has_window_patterns()+ && !capture_params.window_filters.has_url_allowlist()+ {+ if let Some(ref mut comparer) = frame_comparer {+ let _ = comparer.compare(&output.image);+ }}// Frozen-stream detection: a live persistent stream@@ -3054,6 +3063,81 @@ fn bypasses_capture_throttles(trigger: &CaptureTrigger) -> bool {)}+/// Preserve the redacted desktop when only the focused window is filtered.+/// URL, incognito, built-in exclusions and DRM retain their existing gates.+fn permits_redacted_desktop_without_focus(+ filters: &WindowFilters,+ config: &screenpipe_a11y::tree::TreeWalkerConfig,+ pixels_verified: bool,+ pause_on_drm: bool,+) -> bool {+ pixels_verified+ && filters.has_window_patterns()+ && config.ignored_urls.is_empty()+ && config.included_urls.is_empty()+ && !pause_on_drm+}++#[derive(Debug, PartialEq, Eq)]+enum RedactedFocusDecision {+ Continue,+ PixelsOnly,+ Withhold,+}++fn redacted_focus_decision(+ filters: &WindowFilters,+ focus: Option<screenpipe_a11y::tree::FocusedWindowFilterResult>,+ pixels_only_permitted: bool,+) -> RedactedFocusDecision {+ use screenpipe_a11y::tree::FocusedWindowFilterResult;+ match focus {+ Some(FocusedWindowFilterResult::Allowed {+ app_name,+ window_name,+ }) if !app_name.is_empty() => {+ if filters.is_valid(&app_name, &window_name) {+ RedactedFocusDecision::Continue+ } else if pixels_only_permitted {+ RedactedFocusDecision::PixelsOnly+ } else {+ RedactedFocusDecision::Withhold+ }+ }+ // A rejected or unavailable focus check must not fall through to the+ // unknown-monitor OCR path, which has no focused metadata to gate.+ _ => RedactedFocusDecision::Withhold,+ }+}++async fn redacted_focus_check(+ filters: &WindowFilters,+ mut config: screenpipe_a11y::tree::TreeWalkerConfig,+ pixels_only_permitted: bool,+) -> RedactedFocusDecision {+ // Retain a permit inside the blocking call even after timeout, so a stuck+ // native metadata query cannot accumulate unbounded background workers.+ static QUERY: std::sync::OnceLock<Arc<tokio::sync::Semaphore>> = std::sync::OnceLock::new();+ let Ok(permit) = QUERY+ .get_or_init(|| Arc::new(tokio::sync::Semaphore::new(1)))+ .clone()+ .try_acquire_owned()+ else {+ return RedactedFocusDecision::Withhold;+ };+ config.ignored_windows.clear();+ config.included_windows.clear();+ let query = tokio::task::spawn_blocking(move || {+ let _permit = permit;+ screenpipe_a11y::tree::check_focused_window_filters(config)+ });+ let focus = match tokio::time::timeout(Duration::from_millis(250), query).await {+ Ok(Ok(Ok(focus))) => Some(focus),+ _ => None,+ };+ redacted_focus_decision(filters, focus, pixels_only_permitted)+}+/// Perform a single event-driven capture.////// When `previous_content_hash` is `Some` and matches the current accessibility@@ -3200,7 +3284,10 @@ async fn do_capture(// (prevents re-triggering on the same bad frame). The caller records the// matching telemetry counter from `corrupt`.if !screenshot_disabled && !skip_pixels_for_unknown_exclusions {- if let Some(kind) = frame_corruption(&image) {+ if let Some(kind) = frame_corruption_with_privacy_padding(+ &image,+ params.window_filters.has_window_patterns(),+ ) {match kind {// Green is the notable, rarer signal — surface it at warn so it// shows up in shared logs. Black is common (DRM / excluded window).@@ -3222,6 +3309,76 @@ async fn do_capture(}}+ // The native compositor has already removed every excluded window. An+ // excluded focused window must not discard the other allowed windows.+ // Persist only verified pixels: no focused identity, AX, OCR, URL, document+ // path or reused element reference can escape through this narrow path.+ let verified_filtered_pixels = !screenshot_disabled+ && !skip_pixels_for_unknown_exclusions+ && params.window_filters.has_window_patterns();+ let focus_decision = if verified_filtered_pixels {+ redacted_focus_check(+ ¶ms.window_filters,+ params.tree_walker_config.clone(),+ permits_redacted_desktop_without_focus(+ ¶ms.window_filters,+ params.tree_walker_config,+ true,+ params.pause_on_drm_content,+ ),+ )+ .await+ } else {+ RedactedFocusDecision::Continue+ };+ if focus_decision == RedactedFocusDecision::Withhold {+ return Ok(CaptureOutput {+ result: None,+ image,+ elements_deduped: false,+ corrupt: None,+ });+ }+ if focus_decision == RedactedFocusDecision::PixelsOnly {+ let _ = params+ .tree_walker+ .suspend_with_timeout(Duration::from_millis(250))+ .await;+ let ctx = CaptureContext {+ db: params.db,+ snapshot_writer: params.snapshot_writer,+ image: Arc::new(image),+ captured_at,+ monitor_id: params.monitor_id,+ device_name: params.device_name,+ app_name: None,+ window_name: None,+ browser_url: None,+ document_path: None,+ focused: false,+ capture_trigger: trigger.as_str(),+ use_pii_removal: params.use_pii_removal,+ languages: params.languages.to_vec(),+ elements_ref_frame_id: None,+ screenshot_disabled: false,+ in_meeting,+ monitor_hosts_focus: false,+ ax_screenshot_coherent: false,+ focused_window_bounds: None,+ };+ let result = paired_capture_deferred(&ctx).await?;+ if let Some(sender) = params.semantic_tx {+ sender.record_capture_gap(result.frame_id, SemanticCaptureGap::TreeMissing);+ }+ let image = Arc::try_unwrap(ctx.image).unwrap_or_else(|arc| (*arc).clone());+ return Ok(CaptureOutput {+ result: Some(result),+ image,+ elements_deduped: false,+ corrupt: None,+ });+ }+// Walk accessibility tree on blocking thread (AX APIs are synchronous).// Apply adaptive budget overrides: expensive apps (Electron/Discord) get// reduced max_nodes and timeout to avoid blocking their UI thread.@@ -4162,6 +4319,26 @@ fn frame_corruption(image: &image::DynamicImage) -> Option<CorruptKind> {}}+/// Privacy composition intentionally paints excluded surfaces black. A small+/// allowed foreground window must still be durably recorded even when it+/// occupies less than the corruption sampler's five-percent threshold.+fn frame_corruption_with_privacy_padding(+ image: &image::DynamicImage,+ isolated_windows: bool,+) -> Option<CorruptKind> {+ let kind = frame_corruption(image)?;+ if isolated_windows && matches!(kind, CorruptKind::Black) {+ use image::GenericImageView;+ if image+ .pixels()+ .any(|(_, _, p)| u16::from(p[0]) + u16::from(p[1]) + u16::from(p[2]) >= 15)+ {+ return None;+ }+ }+ Some(kind)+}+/// Convenience predicate used by tests.#[cfg(test)]fn is_frame_corrupt(image: &image::DynamicImage) -> bool {@@ -4172,6 +4349,70 @@ fn is_frame_corrupt(image: &image::DynamicImage) -> bool {mod tests {use super::*;+ #[test]+ fn redacted_focus_rejections_cannot_fall_through_unknown_monitor_capture() {+ use screenpipe_a11y::tree::{FocusedWindowFilterResult as Focus, SkipReason};+ let filters = WindowFilters::new(&["PrivacyBlocked".into()], &[], &[]);+ let allowed = |title: &str| {+ Some(Focus::Allowed {+ app_name: "Fixture".into(),+ window_name: title.into(),+ })+ };+ assert_eq!(+ redacted_focus_decision(&filters, allowed("PrivacyAllowed"), true),+ RedactedFocusDecision::Continue+ );+ assert_eq!(+ redacted_focus_decision(&filters, allowed("PrivacyBlocked"), true),+ RedactedFocusDecision::PixelsOnly+ );+ // URL/DRM policies cannot use the pixels-only exception.+ assert_eq!(+ redacted_focus_decision(&filters, allowed("PrivacyBlocked"), false),+ RedactedFocusDecision::Withhold+ );+ for focus in [+ None,+ Some(Focus::NotFound),+ Some(Focus::Skipped(SkipReason::Incognito)),+ Some(Focus::Skipped(SkipReason::ExcludedApp)),+ ] {+ assert_eq!(+ redacted_focus_decision(&filters, focus, true),+ RedactedFocusDecision::Withhold+ );+ }+ }++ #[test]+ fn redacted_desktop_requires_verified_pixels_and_window_only_policy() {+ let filters = WindowFilters::new(&["PrivacyBlocked".into()], &[], &[]);+ let empty = WindowFilters::new(&[], &[], &[]);+ let mut config = screenpipe_a11y::tree::TreeWalkerConfig::default();+ assert!(permits_redacted_desktop_without_focus(+ &filters, &config, true, false+ ));+ assert!(!permits_redacted_desktop_without_focus(+ &filters, &config, false, false+ ));+ assert!(!permits_redacted_desktop_without_focus(+ &filters, &config, true, true+ ));+ assert!(!permits_redacted_desktop_without_focus(+ &empty, &config, true, false+ ));+ config.ignored_urls = serde_json::from_str(r#"["private.example"]"#).unwrap();+ assert!(!permits_redacted_desktop_without_focus(+ &filters, &config, true, false+ ));+ config.ignored_urls.clear();+ config.included_urls = serde_json::from_str(r#"[{"domain":"work.example"}]"#).unwrap();+ assert!(!permits_redacted_desktop_without_focus(+ &filters, &config, true, false+ ));+ }+#[test]fn url_allowlist_masks_everything_outside_focused_window() {use image::{GenericImageView, Pixel};@@ -5894,6 +6135,31 @@ mod tests {assert!(is_frame_corrupt(&img));}+ #[test]+ fn privacy_redaction_preserves_small_allowed_foreground() {+ let mut pixels = image::RgbaImage::from_pixel(1920, 1080, image::Rgba([0, 0, 0, 255]));+ for y in 30..60 {+ for x in 50..100 {+ pixels.put_pixel(x, y, image::Rgba([200, 220, 230, 255]));+ }+ }+ let image = image::DynamicImage::ImageRgba8(pixels);+ assert_eq!(+ frame_corruption_with_privacy_padding(&image, false),+ Some(CorruptKind::Black)+ );+ assert_eq!(frame_corruption_with_privacy_padding(&image, true), None);+ assert_eq!(+ frame_corruption_with_privacy_padding(&solid(1920, 1080, [0, 0, 0]), true),+ Some(CorruptKind::Black)+ );+ let green = frame_with_bottom_band(1920, 1080, 648, [120, 130, 140], [0, 150, 0]);+ assert_eq!(+ frame_corruption_with_privacy_padding(&green, true),+ Some(CorruptKind::GreenBand)+ );+ }+#[test]fn frame_corruption_reports_kind_for_telemetry() {// Black and green resolve to distinct kinds so the caller bumps thediff --git a/crates/screenpipe-screen/examples/screenshot_redaction_probe.rs b/crates/screenpipe-screen/examples/screenshot_redaction_probe.rsnew file mode 100644index 000000000..e8085c7bd--- /dev/null+++ b/crates/screenpipe-screen/examples/screenshot_redaction_probe.rs@@ -0,0 +1,124 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++//! Native acceptance/soak probe. Run only on an isolated synthetic desktop.+//! OUT MODE MONITOR FRAMES INTERVAL_MS [MAX_WIDTH]; MODE=exclude/include/none/empty.+use screenpipe_screen::{+ capture_screenshot_by_window::{ExclusionSources, WindowFilters},+ monitor::{get_default_monitor, get_monitor_by_id},+ screenshot_privacy::capture_monitor_image_filtered,+};+use std::{+ path::PathBuf,+ time::{Duration, Instant},+};++#[tokio::main]+async fn main() -> anyhow::Result<()> {+ let args: Vec<String> = std::env::args().collect();+ let out = PathBuf::from(args.get(1).expect("output directory"));+ std::fs::create_dir_all(&out)?;+ let mode = args.get(2).map(String::as_str).unwrap_or("exclude");+ let id: u32 = args.get(3).map(|s| s.parse()).transpose()?.unwrap_or(0);+ let count: u32 = args.get(4).map(|s| s.parse()).transpose()?.unwrap_or(10);+ let interval: u64 = args.get(5).map(|s| s.parse()).transpose()?.unwrap_or(100);+ if let Some(cap) = args.get(6) {+ #[cfg(target_os = "macos")]+ screenpipe_screen::monitor::set_sck_capture_max_width(cap.parse()?);+ #[cfg(not(target_os = "macos"))]+ anyhow::bail!("MAX_WIDTH is unsupported on this platform: {cap}");+ }+ let monitor = if id == 0 {+ get_default_monitor().await+ } else {+ get_monitor_by_id(id).await+ }+ .ok_or_else(|| anyhow::anyhow!("missing monitor"))?;+ let filters = match mode {+ "exclude" => WindowFilters::new(&["::PrivacyBlocked".into()], &[], &[]),+ "none" => WindowFilters::new(&[], &[], &[]),+ "include" => WindowFilters::new(+ &["::PrivacyBlocked".into()],+ &["::PrivacyAllowed".into()],+ &[],+ ),+ "includeapp" => WindowFilters::new(&[], &["windows_privacy_fixture".into()], &[]),+ "blank" => WindowFilters::new(+ &["::PrivacyBlocked".into()],+ &["windows_privacy_fixture".into()],+ &[],+ ),+ "shell-exclude" => WindowFilters::new(&["Windows Explorer".into()], &[], &[]),+ "empty" => WindowFilters::new(&[], &["::NoSuchWindow".into()], &[]),+ "nomatch" => WindowFilters::new(&["::NeverPresentWindow".into()], &[], &[]),+ _ => anyhow::bail!("unknown mode: {mode}"),+ };+ let writer =+ screenpipe_screen::snapshot_writer::SnapshotWriter::new(out.join("snapshots"), 90, 0);+ let start = Instant::now();+ let mut saved = 0;+ let mut rejected = 0;+ let mut leaked = 0;+ for index in 0..count {+ match capture_monitor_image_filtered(+ &monitor,+ &filters,+ ExclusionSources::default(),+ &[],+ true,+ )+ .await+ {+ Ok((image, time)) => {+ let write_start = Instant::now();+ let path = writer.write(&image, chrono::Utc::now(), monitor.id())?;+ let write_ms = write_start.elapsed().as_secs_f64() * 1000.0;+ let read_start = Instant::now();+ let stored = image::open(&path)?.to_rgb8();+ let read_ms = read_start.elapsed().as_secs_f64() * 1000.0;+ let rgb = image.to_rgb8();+ let marker = |p: &&image::Rgb<u8>| p[0] > 200 && p[1] < 60 && p[2] > 200;+ let allowed_marker = |p: &&image::Rgb<u8>| p[0] < 30 && p[1] > 220 && p[2] > 220;+ let partial_alpha_marker = |p: &&image::Rgb<u8>| {+ p[0] < 30 && (90..=170).contains(&p[1]) && (90..=170).contains(&p[2])+ };+ let forbidden = rgb.pixels().filter(marker).count();+ let stored_forbidden = stored.pixels().filter(marker).count();+ let allowed = rgb.pixels().filter(allowed_marker).count();+ let stored_allowed = stored.pixels().filter(allowed_marker).count();+ let partial_alpha = rgb.pixels().filter(partial_alpha_marker).count();+ if forbidden > 0 || stored_forbidden > 0 {+ leaked += 1;+ }+ if saved < 2 {+ image.save(out.join(format!("frame-{index}.png")))?;+ }+ println!(+ "{}",+ serde_json::json!({"index":index,"saved":true,"path":path,"width":image.width(),"height":image.height(),"capture_ms":time.as_secs_f64()*1000.0,"write_ms":write_ms,"read_ms":read_ms,"forbidden_pixels":forbidden,"stored_forbidden_pixels":stored_forbidden,"allowed_pixels":allowed,"stored_allowed_pixels":stored_allowed,"partial_alpha_pixels":partial_alpha})+ );+ saved += 1;+ }+ Err(error) => {+ rejected += 1;+ println!(+ "{}",+ serde_json::json!({"index":index,"saved":false,"error":error.to_string()})+ );+ }+ }+ if interval > 0 {+ tokio::time::sleep(Duration::from_millis(interval)).await;+ }+ }+ println!(+ "{}",+ serde_json::json!({"summary":true,"saved":saved,"rejected":rejected,"leaked":leaked,"elapsed_ms":start.elapsed().as_millis()})+ );+ anyhow::ensure!(saved > 0, "no screenshots stored");+ anyhow::ensure!(+ !matches!(mode, "exclude" | "include" | "empty" | "blank") || leaked == 0,+ "excluded marker persisted in {leaked} frames"+ );+ Ok(())+}diff --git a/crates/screenpipe-screen/examples/windows_privacy_fixture.rs b/crates/screenpipe-screen/examples/windows_privacy_fixture.rsindex 4a1f74915..b75f5b00d 100644--- a/crates/screenpipe-screen/examples/windows_privacy_fixture.rs+++ b/crates/screenpipe-screen/examples/windows_privacy_fixture.rs@@ -18,9 +18,11 @@ mod windows_harness {System::LibraryLoader::GetModuleHandleW,UI::WindowsAndMessaging::{CreateWindowExW, DefWindowProcW, DispatchMessageW, GetClientRect, GetMessageW,- LoadCursorW, RegisterClassW, ShowWindow, TranslateMessage, CS_HREDRAW, CS_VREDRAW,- IDC_ARROW, MSG, SW_SHOW, WINDOW_EX_STYLE, WINDOW_LONG_PTR_INDEX, WM_DESTROY,- WM_PAINT, WM_PRINT, WM_PRINTCLIENT, WNDCLASSW, WS_OVERLAPPEDWINDOW, WS_VISIBLE,+ LoadCursorW, RegisterClassW, SetLayeredWindowAttributes, ShowWindow,+ TranslateMessage, CS_HREDRAW, CS_VREDRAW, IDC_ARROW, LWA_ALPHA, MSG, SW_MAXIMIZE,+ SW_SHOW, WINDOW_EX_STYLE, WINDOW_LONG_PTR_INDEX, WM_DESTROY, WM_PAINT, WM_PRINT,+ WM_PRINTCLIENT, WNDCLASSW, WS_EX_LAYERED, WS_OVERLAPPEDWINDOW, WS_POPUP,+ WS_VISIBLE,},},};@@ -51,14 +53,14 @@ mod windows_harness {WM_PAINT => {let mut paint = PAINTSTRUCT::default();let dc = BeginPaint(hwnd, &mut paint);- let color = if windows::Win32::UI::WindowsAndMessaging::GetWindowLongPtrW(+ let state = windows::Win32::UI::WindowsAndMessaging::GetWindowLongPtrW(hwnd,WINDOW_LONG_PTR_INDEX(0),- ) == 1- {- BLOCKED- } else {- ALLOWED+ );+ let color = match state {+ 1 => BLOCKED,+ 2 => COLORREF(0x0000_00ff),+ _ => ALLOWED,};let brush = CreateSolidBrush(color);FillRect(dc, &paint.rcPaint, brush);@@ -69,14 +71,14 @@ mod windows_harness {WM_PRINT | WM_PRINTCLIENT => {let mut client = RECT::default();let _ = GetClientRect(hwnd, &mut client);- let color = if windows::Win32::UI::WindowsAndMessaging::GetWindowLongPtrW(+ let state = windows::Win32::UI::WindowsAndMessaging::GetWindowLongPtrW(hwnd,WINDOW_LONG_PTR_INDEX(0),- ) == 1- {- BLOCKED- } else {- ALLOWED+ );+ let color = match state {+ 1 => BLOCKED,+ 2 => COLORREF(0x0000_00ff),+ _ => ALLOWED,};let brush = CreateSolidBrush(color);FillRect(HDC(wparam.0 as *mut _), &client, brush);@@ -88,8 +90,9 @@ mod windows_harness {}}- pub fn run_window(blocked: bool, x: i32, y: i32) -> Result<()> {+ pub fn run_window(kind: &str, x: i32, y: i32) -> Result<()> {unsafe {+ let blocked = matches!(kind, "blocked" | "blocked-red" | "churn");let module = GetModuleHandleW(None).unwrap_or_default();let class_name = if blocked {w!("PrivacyBlockedFixture")@@ -106,17 +109,29 @@ mod windows_harness {..Default::default()};let _ = RegisterClassW(&class);- let title = if blocked {+ let title = if kind == "blank" {+ ""+ } else if blocked {"PrivacyBlocked"} else {"PrivacyAllowed"};let title_wide: Vec<u16> = title.encode_utf16().chain(Some(0)).collect();+ let ex_style = if kind == "transparent" {+ WS_EX_LAYERED+ } else {+ WINDOW_EX_STYLE::default()+ };+ let style = if kind == "borderless" || kind == "transparent" {+ WS_POPUP | WS_VISIBLE+ } else {+ WS_OVERLAPPEDWINDOW | WS_VISIBLE+ };let hwnd = CreateWindowExW(- WINDOW_EX_STYLE::default(),+ ex_style,class_name,windows::core::PCWSTR(title_wide.as_ptr()),- WS_OVERLAPPEDWINDOW | WS_VISIBLE,+ style,x,y,520,@@ -130,10 +145,20 @@ mod windows_harness {windows::Win32::UI::WindowsAndMessaging::SetWindowLongPtrW(hwnd,WINDOW_LONG_PTR_INDEX(0),- isize::from(blocked),+ if kind == "blocked-red" {+ 2+ } else {+ isize::from(blocked)+ },);+ if kind == "transparent" {+ let _ = SetLayeredWindowAttributes(hwnd, COLORREF(0), 128, LWA_ALPHA);+ }let _ = ShowWindow(hwnd, SW_SHOW);- if blocked {+ if kind == "maximized" {+ let _ = ShowWindow(hwnd, SW_MAXIMIZE);+ }+ if kind == "churn" {let raw = hwnd.0 as usize;thread::spawn(move || loop {// Keep the synthetic fixture above the test runner while@@ -168,10 +193,22 @@ mod windows_harness {fn main() -> anyhow::Result<()> {let mut args = std::env::args().skip(1);let kind = args.next().ok_or_else(|| {- anyhow::anyhow!("usage: windows_privacy_fixture <allowed|blocked> <x> <y>")+ anyhow::anyhow!(+ "usage: windows_privacy_fixture <allowed|blocked|blocked-red|blank|borderless|transparent|maximized|churn> <x> <y>"+ )})?;anyhow::ensure!(- kind == "allowed" || kind == "blocked",+ matches!(+ kind.as_str(),+ "allowed"+ | "blocked"+ | "blocked-red"+ | "blank"+ | "borderless"+ | "transparent"+ | "maximized"+ | "churn"+ ),"unknown fixture window");let x = args@@ -182,7 +219,7 @@ fn main() -> anyhow::Result<()> {.next().ok_or_else(|| anyhow::anyhow!("missing y"))?.parse()?;- windows_harness::run_window(kind == "blocked", x, y)+ windows_harness::run_window(&kind, x, y)}#[cfg(not(target_os = "windows"))]diff --git a/crates/screenpipe-screen/src/screenshot_privacy.rs b/crates/screenpipe-screen/src/screenshot_privacy.rsindex 481d8a70a..3648d1915 100644--- a/crates/screenpipe-screen/src/screenshot_privacy.rs+++ b/crates/screenpipe-screen/src/screenshot_privacy.rs@@ -11,6 +11,9 @@ use crate::monitor::SafeMonitor;use image::DynamicImage;use std::time::{Duration, Instant};+#[cfg(any(target_os = "macos", target_os = "linux", test))]+mod scene;+#[cfg(target_os = "linux")]mod linux;#[cfg(target_os = "macos")]diff --git a/crates/screenpipe-screen/src/screenshot_privacy/linux.rs b/crates/screenpipe-screen/src/screenshot_privacy/linux.rsindex 4a12ea19d..3cb9cc445 100644--- a/crates/screenpipe-screen/src/screenshot_privacy/linux.rs+++ b/crates/screenpipe-screen/src/screenshot_privacy/linux.rs@@ -1,12 +1,15 @@// screenpipe — AI that knows everything you've seen, said, or heard// https://screenpipe.com-use super::*;+use super::{+ scene::{self, Rect},+ *,+};use anyhow::{anyhow, ensure};use once_cell::sync::Lazy;use std::sync::Arc;use tokio::sync::Semaphore;-use xcb::{composite, x, XidNew};+use xcb::{composite, x, Xid, XidNew};static CAPTURE_SLOT: Lazy<Arc<Semaphore>> = Lazy::new(|| Arc::new(Semaphore::new(1)));@@ -16,8 +19,8 @@ pub(crate) async fn capture_filtered(sources: ExclusionSources,_fresh: bool,) -> anyhow::Result<DynamicImage> {- // XWayland enumeration cannot account for native Wayland windows. Never- // substitute a portal/whole-monitor capture when a window policy is set.+ // XWayland cannot enumerate native Wayland surfaces. Never fall back to a+ // portal/monitor image when isolated pixels are required.ensure!(std::env::var_os("WAYLAND_DISPLAY").is_none()&& std::env::var("XDG_SESSION_TYPE").unwrap_or_default() != "wayland",@@ -37,6 +40,198 @@ pub(crate) async fn capture_filtered(.map_err(|_| anyhow!("window-filtered capture timed out"))??}+fn atom(conn: &xcb::Connection, name: &str) -> anyhow::Result<x::Atom> {+ Ok(conn+ .wait_for_reply(conn.send_request(&x::InternAtom {+ only_if_exists: true,+ name: name.as_bytes(),+ }))?+ .atom())+}++fn property(+ conn: &xcb::Connection,+ window: x::Window,+ name: &str,+) -> anyhow::Result<x::GetPropertyReply> {+ let property = atom(conn, name)?;+ ensure!(property != x::ATOM_NONE, "missing {name}");+ Ok(conn.wait_for_reply(conn.send_request(&x::GetProperty {+ delete: false,+ window,+ property,+ r#type: x::ATOM_ANY,+ long_offset: 0,+ long_length: 16384,+ }))?)+}++fn text_property(conn: &xcb::Connection, window: x::Window, name: &str) -> String {+ property(conn, window, name)+ .ok()+ .filter(|p| p.format() == 8)+ .map(|p| String::from_utf8_lossy(p.value::<u8>()).into_owned())+ .unwrap_or_default()+}++#[derive(Debug, Clone, PartialEq)]+struct Surface {+ id: u32,+ client: u32,+ pid: u32,+ app: String,+ title: String,+ rect: Rect,+ opacity: u32,+}++/// Actual root children are bottom-to-top, including unmanaged menus, WM+/// decorations, dock and desktop windows. The old client-only list omitted+/// these and could reveal a lower allowed window under a blocked popup.+fn snapshot(+ conn: &xcb::Connection,+ root: x::Window,+ overlay: x::Window,+) -> anyhow::Result<Vec<Surface>> {+ let clients = property(conn, root, "_NET_CLIENT_LIST_STACKING")+ .ok()+ .filter(|p| p.format() == 32)+ .map(|p| p.value::<u32>().to_vec())+ .unwrap_or_default();+ let mut frame_clients = std::collections::HashMap::new();+ for id in clients {+ let mut current = x::Window::new(id);+ for _ in 0..32 {+ let Ok(tree) =+ conn.wait_for_reply(conn.send_request(&x::QueryTree { window: current }))+ else {+ break;+ };+ if tree.parent() == root {+ frame_clients.insert(current.resource_id(), id);+ break;+ }+ if tree.parent().resource_id() == 0 {+ break;+ }+ current = tree.parent();+ }+ }+ let tree = conn.wait_for_reply(conn.send_request(&x::QueryTree { window: root }))?;+ let mut result = Vec::new();+ for &window in tree.children() {+ if window == overlay {+ continue;+ }+ let attrs = conn.wait_for_reply(conn.send_request(&x::GetWindowAttributes { window }))?;+ if attrs.map_state() != x::MapState::Viewable || attrs.class() == x::WindowClass::InputOnly+ {+ continue;+ }+ let geometry = conn.wait_for_reply(conn.send_request(&x::GetGeometry {+ drawable: x::Drawable::Window(window),+ }))?;+ let position = conn.wait_for_reply(conn.send_request(&x::TranslateCoordinates {+ src_window: window,+ dst_window: root,+ src_x: 0,+ src_y: 0,+ }))?;+ let client = *frame_clients+ .get(&window.resource_id())+ .unwrap_or(&window.resource_id());+ let native = x::Window::new(client);+ let class = text_property(conn, native, "WM_CLASS");+ let app = class.split('\0').nth(1).unwrap_or("").to_string();+ let mut title = text_property(conn, native, "_NET_WM_NAME");+ if title.is_empty() {+ title = text_property(conn, native, "WM_NAME");+ }+ let cardinal = |name| {+ property(conn, native, name)+ .ok()+ .filter(|p| p.format() == 32)+ .and_then(|p| p.value::<u32>().first().copied())+ };+ result.push(Surface {+ id: window.resource_id(),+ client,+ pid: cardinal("_NET_WM_PID").unwrap_or(0),+ app,+ title,+ rect: Rect {+ x: i64::from(position.dst_x()),+ y: i64::from(position.dst_y()),+ width: u32::from(geometry.width()),+ height: u32::from(geometry.height()),+ },+ opacity: cardinal("_NET_WM_WINDOW_OPACITY").unwrap_or(u32::MAX),+ });+ }+ Ok(result)+}++fn read_pixmap(+ conn: &xcb::Connection,+ pixmap: x::Pixmap,+ width: u32,+ height: u32,+) -> anyhow::Result<image::RgbaImage> {+ ensure!(+ width > 0 && height > 0 && width <= 16384 && height <= 16384,+ "invalid surface dimensions"+ );+ let reply = conn.wait_for_reply(conn.send_request(&x::GetImage {+ format: x::ImageFormat::ZPixmap,+ drawable: x::Drawable::Pixmap(pixmap),+ x: 0,+ y: 0,+ width: width as u16,+ height: height as u16,+ plane_mask: u32::MAX,+ }))?;+ let setup = conn.get_setup();+ let format = setup+ .pixmap_formats()+ .iter()+ .find(|f| f.depth() == reply.depth())+ .ok_or_else(|| anyhow!("missing surface pixel format"))?;+ ensure!(+ format.bits_per_pixel() == 32+ && format.scanline_pad() == 32+ && setup.image_byte_order() == x::ImageOrder::LsbFirst+ && matches!(reply.depth(), 24 | 32),+ "unsupported surface pixel format"+ );+ let data = reply.data();+ ensure!(+ data.len() == width as usize * height as usize * 4,+ "invalid surface pixel storage"+ );+ let mut image = image::RgbaImage::new(width, height);+ for (s, d) in data.chunks_exact(4).zip(image.pixels_mut()) {+ *d = image::Rgba([+ s[2],+ s[1],+ s[0],+ if reply.depth() == 32 { s[3] } else { 255 },+ ]);+ }+ Ok(image)+}++fn background_id(conn: &xcb::Connection, root: x::Window) -> Option<u32> {+ ["_XROOTPMAP_ID", "ESETROOT_PMAP_ID"]+ .into_iter()+ .find_map(|name| {+ property(conn, root, name)+ .ok()+ .filter(|p| p.format() == 32 && p.r#type() == x::ATOM_PIXMAP)+ .and_then(|p| p.value::<u32>().first().copied())+ .filter(|id| *id != 0)+ })+}+fn capture(monitor: &crate::monitor::MonitorData,filters: &WindowFilters,@@ -45,47 +240,133 @@ fn capture(let detector = sources.ignore_incognito_windows.then(|| screenpipe_a11y::incognito::create_detector(sources.enhanced_incognito_detection));- let (conn, _) = xcb::Connection::connect(None)?;+ let (conn, index) = xcb::Connection::connect(None)?;conn.wait_for_reply(conn.send_request(&composite::QueryVersion {client_major_version: 0,client_minor_version: 4,}))?;- let mut windows = xcap::Window::all()?;- // xcap orders front to back; draw back to front on an opaque black canvas.- windows.reverse();+ let setup = conn.get_setup();+ let screen = setup+ .roots()+ .nth(index as usize)+ .ok_or_else(|| anyhow!("missing X11 screen"))?;+ let root = screen.root();+ let overlay = conn+ .wait_for_reply(conn.send_request(&composite::GetOverlayWindow { window: root }))?+ .overlay_win();+ conn.check_request(+ conn.send_request_checked(&composite::ReleaseOverlayWindow { window: root }),+ )?;+ conn.check_request(conn.send_request_checked(&x::ChangeWindowAttributes {+ window: root,+ value_list: &[x::Cw::EventMask(+ x::EventMask::SUBSTRUCTURE_NOTIFY | x::EventMask::PROPERTY_CHANGE,+ )],+ }))?;+ let initial = snapshot(&conn, root, overlay)?;+ for surface in &initial {+ for id in [surface.id, surface.client] {+ conn.check_request(conn.send_request_checked(&x::ChangeWindowAttributes {+ window: x::Window::new(id),+ value_list: &[x::Cw::EventMask(+ x::EventMask::STRUCTURE_NOTIFY | x::EventMask::PROPERTY_CHANGE,+ )],+ }))?;+ }+ }+ let before = snapshot(&conn, root, overlay)?;+ ensure!(+ initial == before,+ "desktop scene changed during privacy setup"+ );+ let policy_atoms = [+ "_NET_WM_NAME",+ "WM_NAME",+ "WM_CLASS",+ "_NET_WM_PID",+ "_NET_WM_WINDOW_OPACITY",+ "_NET_WM_STATE",+ "_XROOTPMAP_ID",+ "ESETROOT_PMAP_ID",+ ]+ .iter()+ .map(|name| atom(&conn, name))+ .collect::<anyhow::Result<Vec<_>>>()?;+ ensure!(+ !scene_changed(&conn, &policy_atoms)?,+ "desktop changed before privacy capture"+ );+ let wallpaper = background_id(&conn, root);let mut canvas =image::RgbaImage::from_pixel(monitor.width, monitor.height, image::Rgba([0, 0, 0, 255]));- let mut captured = 0;- for window in windows {- let (Ok(id), Ok(app), Ok(title), Ok(false)) = (- window.id(),- window.app_name(),- window.title(),- window.is_minimized(),- ) else {- continue;- };- // _NET_WM_PID is optional in X11 (e.g. Tk does not publish it).- let pid = window.pid().unwrap_or(0);- if !window_allowed(filters, detector.as_deref(), &app, &title, pid as i32) {+ // This is the wallpaper resource, never the root framebuffer containing+ // other clients. A desktop manager's own window is also captured below.+ if filters.is_valid("Desktop", "") {+ if let Some(id) = wallpaper {+ let pixmap = x::Pixmap::new(id);+ let size = conn.wait_for_reply(conn.send_request(&x::GetGeometry {+ drawable: x::Drawable::Pixmap(pixmap),+ }))?;+ let image = read_pixmap(+ &conn,+ pixmap,+ u32::from(size.width()),+ u32::from(size.height()),+ )?;+ // X11 tiles a root background pixmap; it may be only 1x1.+ // Use root coordinates so adjacent/negative-origin monitors agree.+ for (x, y, pixel) in canvas.enumerate_pixels_mut() {+ let sx = (i64::from(x) + i64::from(monitor.x)).rem_euclid(i64::from(image.width()))+ as u32;+ let sy = (i64::from(y) + i64::from(monitor.y)).rem_euclid(i64::from(image.height()))+ as u32;+ *pixel = *image.get_pixel(sx, sy);+ pixel[3] = 255;+ }+ }+ }+ let monitor_rect = Rect {+ x: i64::from(monitor.x),+ y: i64::from(monitor.y),+ width: monitor.width,+ height: monitor.height,+ };+ for surface in &before {+ if !surface.rect.intersects(monitor_rect) {continue;}- let (wx, wy, width, height) = (window.x()?, window.y()?, window.width()?, window.height()?);- if i64::from(wx) + i64::from(width) <= i64::from(monitor.x)- || i64::from(wy) + i64::from(height) <= i64::from(monitor.y)- || i64::from(wx) >= i64::from(monitor.x) + i64::from(monitor.width)- || i64::from(wy) >= i64::from(monitor.y) + i64::from(monitor.height)- {+ let dest = Rect {+ x: surface.rect.x - i64::from(monitor.x),+ y: surface.rect.y - i64::from(monitor.y),+ ..surface.rect+ };+ if !window_allowed(+ filters,+ detector.as_deref(),+ &surface.app,+ &surface.title,+ surface.pid as i32,+ ) {+ scene::paint_black(&mut canvas, dest);continue;}+ let native = x::Window::new(surface.id);+ let attrs =+ conn.wait_for_reply(conn.send_request(&x::GetWindowAttributes { window: native }))?;ensure!(- width > 0 && height > 0 && width <= 16384 && height <= 16384,- "invalid window dimensions"+ setup+ .roots()+ .any(|s| s+ .allowed_depths()+ .any(+ |d| d.visuals().iter().any(|v| v.visual_id() == attrs.visual()+ && v.class() == x::VisualClass::TrueColor+ && v.red_mask() == 0xff0000+ && v.green_mask() == 0xff00+ && v.blue_mask() == 0xff)+ )),+ "unsupported surface visual");- let native = x::Window::new(id);- // XGetImage(window) can contain undefined pixels where another window- // obscures it. Read the window's own Composite backing pixmap instead.- // Connection teardown releases our redirects on every success/error.conn.check_request(conn.send_request_checked(&composite::RedirectWindow {window: native,update: composite::Redirect::Automatic,@@ -95,78 +376,49 @@ fn capture(window: native,pixmap,}))?;- let reply = conn.wait_for_reply(conn.send_request(&x::GetImage {- format: x::ImageFormat::ZPixmap,- drawable: x::Drawable::Pixmap(pixmap),- x: 0,- y: 0,- width: width as u16,- height: height as u16,- plane_mask: u32::MAX,- }))?;- conn.send_request(&x::FreePixmap { pixmap });- let setup = conn.get_setup();- let format = setup- .pixmap_formats()- .iter()- .find(|f| f.depth() == reply.depth())- .ok_or_else(|| anyhow!("missing window pixel format"))?;- ensure!(- format.bits_per_pixel() == 32- && format.scanline_pad() == 32- && setup.image_byte_order() == x::ImageOrder::LsbFirst- && matches!(reply.depth(), 24 | 32),- "unsupported window pixel format"- );- let attrs =- conn.wait_for_reply(conn.send_request(&x::GetWindowAttributes { window: native }))?;- let true_color = setup.roots().any(|s| {- s.allowed_depths().any(|d| {- d.visuals().iter().any(|v| {- v.visual_id() == attrs.visual()- && v.class() == x::VisualClass::TrueColor- && v.red_mask() == 0xff0000- && v.green_mask() == 0xff00- && v.blue_mask() == 0xff- })- })- });- ensure!(true_color, "unsupported window visual");- // A close/reuse/title/geometry transition invalidates the whole frame.- ensure!(- window.app_name()? == app- && window.title()? == title- && window.pid().unwrap_or(0) == pid- && !window.is_minimized()?- && (window.x()?, window.y()?, window.width()?, window.height()?)- == (wx, wy, width, height),- "allowed window changed during capture"- );- let data = reply.data();- ensure!(- data.len() == width as usize * height as usize * 4,- "invalid window pixel storage"- );- for y in 0..height {- let dest_y = i64::from(wy) - i64::from(monitor.y) + i64::from(y);- if dest_y < 0 || dest_y >= i64::from(monitor.height) {- continue;- }- for x in 0..width {- let dest_x = i64::from(wx) - i64::from(monitor.x) + i64::from(x);- if dest_x < 0 || dest_x >= i64::from(monitor.width) {- continue;+ let mut image = read_pixmap(&conn, pixmap, surface.rect.width, surface.rect.height)?;+ conn.check_request(conn.send_request_checked(&x::FreePixmap { pixmap }))?;+ if surface.opacity != u32::MAX {+ for pixel in image.pixels_mut() {+ for c in &mut pixel.0 {+ *c = ((u64::from(*c) * u64::from(surface.opacity) + u64::from(u32::MAX) / 2)+ / u64::from(u32::MAX)) as u8;}- let p = &data[(y as usize * width as usize + x as usize) * 4..][..4];- canvas.put_pixel(- dest_x as u32,- dest_y as u32,- image::Rgba([p[2], p[1], p[0], 255]),- );}}- captured += 1;+ scene::overlay_premultiplied(&mut canvas, &image, dest.x, dest.y, None);}- ensure!(captured > 0, "no verified allowed windows on monitor");+ // Includes blocked/unknown surfaces and stacking order. Never persist a+ // partially reconstructed scene after close/reuse/move/resize/reorder.+ ensure!(+ snapshot(&conn, root, overlay)? == before && background_id(&conn, root) == wallpaper,+ "desktop scene changed during privacy capture"+ );+ // Events retain transient title/geometry changes even if a window returns+ // to the same identity before revalidation (an ABA transition).+ ensure!(+ !scene_changed(&conn, &policy_atoms)?,+ "desktop changed during privacy capture"+ );Ok(DynamicImage::ImageRgba8(canvas))}++fn scene_changed(conn: &xcb::Connection, policy_atoms: &[x::Atom]) -> anyhow::Result<bool> {+ let mut changed = false;+ while let Some(event) = conn.poll_for_event()? {+ if let xcb::Event::X(event) = event {+ changed |= match event {+ x::Event::PropertyNotify(e) => policy_atoms.contains(&e.atom()),+ x::Event::CreateNotify(_)+ | x::Event::DestroyNotify(_)+ | x::Event::MapNotify(_)+ | x::Event::UnmapNotify(_)+ | x::Event::ConfigureNotify(_)+ | x::Event::ReparentNotify(_)+ | x::Event::CirculateNotify(_) => true,+ _ => false,+ };+ }+ }+ Ok(changed)+}diff --git a/crates/screenpipe-screen/src/screenshot_privacy/macos.rs b/crates/screenpipe-screen/src/screenshot_privacy/macos.rsindex c88ceac44..e0d53954e 100644--- a/crates/screenpipe-screen/src/screenshot_privacy/macos.rs+++ b/crates/screenpipe-screen/src/screenshot_privacy/macos.rs@@ -43,11 +43,232 @@ pub(crate) async fn capture_filtered(.map_err(|_| anyhow!("window-filtered capture timed out"))??}+#[derive(Clone, Debug, PartialEq)]+struct Surface {+ id: u32,+ pid: i32,+ app: String,+ title: String,+ x: f64,+ y: f64,+ width: f64,+ height: f64,+ alpha: f64,+ layer: i64,+}++/// WindowServer order is authoritative; SCShareableContent does not promise+/// stacking order. Include desktop and shell surfaces, not just layer-zero apps.+fn snapshot(display: cidre::cg::Rect) -> anyhow::Result<Vec<Surface>> {+ window_list(+ display,+ core_graphics::window::kCGWindowListOptionOnScreenOnly,+ )+}++fn window_list(display: cidre::cg::Rect, options: u32) -> anyhow::Result<Vec<Surface>> {+ use core_foundation::{+ base::{CFType, TCFType},+ dictionary::CFDictionary,+ number::CFNumber,+ string::CFString,+ };+ use core_graphics::window::{copy_window_info, kCGNullWindowID};+ let list = copy_window_info(options, kCGNullWindowID)+ .ok_or_else(|| anyhow!("window scene unavailable"))?;+ let mut result = Vec::new();+ for ptr in list.iter() {+ let dict: CFDictionary<CFString, CFType> =+ unsafe { CFDictionary::wrap_under_get_rule(*ptr as _) };+ let number = |key: &str| -> Option<f64> {+ dict.find(CFString::new(key))?+ .downcast::<CFNumber>()?+ .to_f64()+ };+ let text = |key: &str| -> String {+ dict.find(CFString::new(key))+ .and_then(|v| v.downcast::<CFString>())+ .map(|s| s.to_string())+ .unwrap_or_default()+ };+ let bounds = dict+ .find(CFString::new("kCGWindowBounds"))+ .and_then(|v| v.downcast::<CFDictionary>())+ .map(|d| unsafe {+ CFDictionary::<CFString, CFType>::wrap_under_get_rule(d.as_concrete_TypeRef())+ })+ .ok_or_else(|| anyhow!("window geometry unavailable"))?;+ let bound = |key: &str| -> anyhow::Result<f64> {+ bounds+ .find(CFString::new(key))+ .and_then(|v| v.downcast::<CFNumber>())+ .and_then(|n| n.to_f64())+ .ok_or_else(|| anyhow!("invalid window geometry"))+ };+ let (x, y, width, height) = (bound("X")?, bound("Y")?, bound("Width")?, bound("Height")?);+ let alpha = number("kCGWindowAlpha").unwrap_or(1.0);+ let layer = number("kCGWindowLayer").unwrap_or(0.0) as i64;+ ensure!(+ [x, y, width, height, alpha].iter().all(|v| v.is_finite()),+ "non-finite window geometry"+ );+ if width <= 0.0 || height <= 0.0 || alpha <= 0.0 || layer == 2147483630 {+ continue;+ }+ if options != core_graphics::window::kCGWindowListOptionAll+ && (x + width <= display.origin.x+ || y + height <= display.origin.y+ || x >= display.origin.x + display.size.width+ || y >= display.origin.y + display.size.height)+ {+ continue;+ }+ result.push(Surface {+ id: number("kCGWindowNumber").ok_or_else(|| anyhow!("window id unavailable"))? as u32,+ pid: number("kCGWindowOwnerPID").unwrap_or(0.0) as i32,+ app: text("kCGWindowOwnerName"),+ title: text("kCGWindowName"),+ x,+ y,+ width,+ height,+ alpha,+ layer,+ });+ }+ result.reverse(); // bottom-to-top+ Ok(result)+}++/// Dock thumbnails can reproduce excluded windows. If native tile inspection+/// fails, omit Dock coverage instead of trusting its mirrored app pixels.+fn dock_masks(+ dock_pid: Option<i32>,+ display: cidre::cg::Rect,+ filters: &WindowFilters,+ detector: Option<&dyn screenpipe_a11y::incognito::IncognitoDetector>,+) -> Option<Vec<Surface>> {+ use cidre::{ax, cf};+ let Some(pid) = dock_pid else {+ return Some(Vec::new());+ };+ if !ax::is_process_trusted() {+ return None;+ }+ let app = ax::UiElement::with_app_pid(pid);+ app.set_messaging_timeout_secs(0.1).ok()?;+ let groups = app.children().ok()?;+ let windows = window_list(display, core_graphics::window::kCGWindowListOptionAll).ok()?;+ let text = |element: &ax::UiElement, attr: &ax::Attr| -> Option<String> {+ let value = element.attr_value(attr).ok()?;+ if value.get_type_id() != cf::String::type_id() {+ return None;+ }+ let value: &cf::String = unsafe { std::mem::transmute(&*value) };+ Some(value.to_string())+ };+ let mut masks = Vec::new();+ let mut seen = 0;+ for group in groups.iter() {+ group.set_messaging_timeout_secs(0.1).ok()?;+ let children = group.children().ok()?;+ for tile in children.iter() {+ seen += 1;+ if seen > 512 {+ return None;+ }+ tile.set_messaging_timeout_secs(0.1).ok()?;+ if text(tile, ax::attr::subrole()).as_deref() != Some("AXMinimizedWindowDockItem") {+ continue;+ }+ let title = text(tile, ax::attr::title())?;+ let matches: Vec<_> = windows.iter().filter(|w| w.title == title).collect();+ if !matches.is_empty()+ && matches+ .iter()+ .all(|w| window_allowed(filters, detector, &w.app, &w.title, w.pid))+ {+ continue;+ }+ let pos = tile.pos().ok()?.cg_point()?;+ let size = tile.size().ok()?.cg_size()?;+ if ![pos.x, pos.y, size.width, size.height]+ .iter()+ .all(|v| v.is_finite())+ || size.width <= 0.0+ || size.height <= 0.0+ {+ return None;+ }+ masks.push(Surface {+ id: 0,+ pid,+ app: "Dock".into(),+ title,+ x: pos.x,+ y: pos.y,+ width: size.width,+ height: size.height,+ alpha: 1.0,+ layer: 20,+ });+ }+ }+ Some(masks)+}++fn output_rect(+ surface: &Surface,+ display: cidre::cg::Rect,+ width: u32,+ height: u32,+) -> super::scene::Rect {+ let sx = f64::from(width) / display.size.width;+ let sy = f64::from(height) / display.size.height;+ let x = ((surface.x - display.origin.x) * sx).floor();+ let y = ((surface.y - display.origin.y) * sy).floor();+ super::scene::Rect {+ x: x as i64,+ y: y as i64,+ width: (((surface.x + surface.width - display.origin.x) * sx).ceil() - x).max(0.0) as u32,+ height: (((surface.y + surface.height - display.origin.y) * sy).ceil() - y).max(0.0) as u32,+ }+}++async fn take(+ filter: &sc::ContentFilter,+ width: usize,+ height: usize,+ window: bool,+) -> anyhow::Result<image::RgbaImage> {+ let mut cfg = sc::StreamCfg::new();+ cfg.set_width(width);+ cfg.set_height(height);+ cfg.set_pixel_format(cv::PixelFormat::_32_BGRA);+ cfg.set_shows_cursor(false);+ cfg.set_scales_to_fit(true);+ if window {+ cfg.set_ignore_shadows_single_window(true);+ }+ if api::macos_available("15.0") {+ cfg.set_show_mouse_clicks(false);+ }+ let sample = sc::ScreenshotManager::capture_sample_buf(filter, &cfg)+ .await+ .map_err(|e| anyhow!("window-filtered screenshot: {e:?}"))?;+ let mut pixels = sample+ .image_buf()+ .ok_or_else(|| anyhow!("missing screenshot pixels"))?+ .retained();+ Ok(rgba(&mut pixels)?.into_rgba8())+}+async fn capture(id: u32,filters: &WindowFilters,sources: ExclusionSources,) -> anyhow::Result<DynamicImage> {+ use super::scene::{self, Rect};let detector = sources.ignore_incognito_windows.then(|| screenpipe_a11y::incognito::create_detector(sources.enhanced_incognito_detection));@@ -59,81 +280,149 @@ async fn capture(.iter().find(|d| d.display_id().0 == id).ok_or_else(|| anyhow!("filtered display disappeared"))?;- let mut selected = Vec::new();- let mut identities = Vec::new();- for window in content.windows().iter() {- if !window.is_on_screen() || window.window_layer() != 0 {- continue;- }- let Some(app) = window.owning_app() else {- continue;- };- let Some(title) = window.title() else {- continue;- };- let name = app.app_name().to_string();- let title = title.to_string();- if window_allowed(- filters,- detector.as_deref(),- &name,- &title,- app.process_id(),- ) {- identities.push((window.id(), app.process_id(), name, title));- selected.push(window.retained());- }- }- ensure!(!selected.is_empty(), "no verified allowed windows");+ let frame = display.frame();+ let before = snapshot(frame)?;+ let available = content.windows();+ let native = |surface: &Surface| {+ available.iter().find(|w| {+ w.id() == surface.id+ && w.is_on_screen()+ && w.owning_app()+ .is_some_and(|a| a.process_id() == surface.pid)+ })+ };+ let dock_pid = before.iter().filter(|s| s.layer == 20).find_map(|s| {+ native(s)+ .filter(|w| {+ w.owning_app()+ .is_some_and(|a| a.bundle_id().to_string() == "com.apple.dock")+ })+ .map(|_| s.pid)+ });+ let dock_before = dock_masks(dock_pid, frame, filters, detector.as_deref());+ let allowed: Vec<bool> = before+ .iter()+ .map(|s| {+ native(s).is_some()+ && !(Some(s.pid) == dock_pid && s.layer == 20 && dock_before.is_none())+ && window_allowed(filters, detector.as_deref(), &s.app, &s.title, s.pid)+ })+ .collect();+ let selected: Vec<_> = before+ .iter()+ .zip(&allowed)+ .filter_map(|(s, ok)| {+ if *ok {+ native(s).map(|w| w.retained())+ } else {+ None+ }+ })+ .collect();let windows = ns::Array::from_slice_retained(&selected);- let filter = sc::ContentFilter::with_display_including_windows(display, &windows);- let mut cfg = sc::StreamCfg::new();- let native_width = display.width() as usize;- let native_height = display.height() as usize;+ let mut filter = sc::ContentFilter::with_display_including_windows(display, &windows);+ // SCDisplay dimensions are logical points on Retina displays. Preserve the+ // native pixel density before applying the user's explicit resolution cap.+ let scale = filter.point_pixel_scale() as f64;+ ensure!(+ scale.is_finite() && scale > 0.0,+ "invalid display pixel scale"+ );+ let nw = (frame.size.width * scale).round() as usize;+ let nh = (frame.size.height * scale).round() as usize;+ ensure!(+ nw > 0 && nh > 0 && nw <= 16384 && nh <= 16384,+ "invalid display dimensions"+ );let cap = crate::monitor::sck_capture_max_width() as usize;- let width = if cap == 0 {- native_width+ let width = if cap == 0 { nw } else { nw.min(cap) };+ let height = (nh * width / nw).max(1);+ let mut canvas = if selected.is_empty() {+ image::RgbaImage::from_pixel(width as u32, height as u32, image::Rgba([0, 0, 0, 255]))} else {- native_width.min(cap)+ if api::macos_available("14.2") {+ filter.set_include_menu_bar(before.iter().zip(&allowed).any(|(s, ok)| {+ *ok && s.layer == 24 && s.app == "Window Server" && s.title == "Menubar"+ }));+ }+ let pixels = take(&filter, width, height, false).await?;+ let mut canvas =+ image::RgbaImage::from_pixel(width as u32, height as u32, image::Rgba([0, 0, 0, 255]));+ scene::overlay_premultiplied(&mut canvas, &pixels, 0, 0, None);+ canvas+ };+ let bounds = Rect {+ x: 0,+ y: 0,+ width: width as u32,+ height: height as u32,};+ let mut redacted = Vec::<Rect>::new();+ for (surface, ok) in before.iter().zip(&allowed) {+ let rect = output_rect(surface, frame, width as u32, height as u32);+ if !*ok {+ // The system Dock owns a display-sized transparent overlay. Its+ // rectangle is not its visible coverage. Use only native alpha as+ // a mask; none of this excluded surface's RGB enters the output.+ let dock = native(surface).filter(|w| {+ surface.layer == 20+ && w.owning_app()+ .is_some_and(|a| a.bundle_id().to_string() == "com.apple.dock")+ });+ if let Some(dock) = dock {+ let filter = sc::ContentFilter::with_desktop_independent_window(dock);+ let coverage =+ take(&filter, rect.width as usize, rect.height as usize, true).await?;+ scene::paint_black_coverage(&mut canvas, &coverage, rect.x, rect.y);+ } else {+ scene::paint_black(&mut canvas, rect);+ }+ if let Some(clipped) = rect.intersection(bounds) {+ scene::add_region(&mut redacted, clipped);+ }+ } else if redacted.iter().any(|r| rect.intersects(*r)) {+ // Restore only the affected part of an allowed foreground surface.+ // Capturing its native pixels preserves translucent corners without+ // revealing excluded pixels or repainting unrelated foreground UI.+ let window = native(surface).ok_or_else(|| anyhow!("allowed window disappeared"))?;+ let filter = sc::ContentFilter::with_desktop_independent_window(window);+ let pixels = take(&filter, rect.width as usize, rect.height as usize, true).await?;+ for region in &redacted {+ scene::overlay_premultiplied(&mut canvas, &pixels, rect.x, rect.y, Some(*region));+ }+ }+ }+ if let Some(masks) = &dock_before {+ for mask in masks {+ scene::paint_black(+ &mut canvas,+ output_rect(mask, frame, width as u32, height as u32),+ );+ }+ }ensure!(- native_width > 0 && native_height > 0,- "invalid display dimensions"+ dock_masks(dock_pid, frame, filters, detector.as_deref()) == dock_before,+ "Dock thumbnail policy changed during capture"+ );+ ensure!(+ snapshot(frame)? == before,+ "desktop scene changed during privacy capture");- cfg.set_width(width);- cfg.set_height((native_height * width / native_width).max(1));- cfg.set_pixel_format(cv::PixelFormat::_32_BGRA);- cfg.set_shows_cursor(false);- // Match the existing captureMaxWidth contract: scale the entire display,- // rather than cropping its right/bottom edges at smaller output sizes.- cfg.set_scales_to_fit(true);- if api::macos_available("15.0") {- cfg.set_show_mouse_clicks(false);- }- let sample = sc::ScreenshotManager::capture_sample_buf(&filter, &cfg)- .await- .map_err(|e| anyhow!("window-filtered screenshot: {e:?}"))?;- // Do not accept a frame across window reuse/title changes. A newly created- // unrelated window is absent from the positive filter even during animation.let after = sc::ShareableContent::current().await.map_err(|e| anyhow!("window revalidation: {e:?}"))?;let after_windows = after.windows();- for (id, pid, name, title) in identities {- let same = after_windows.iter().any(|w| {- w.id() == id- && w.is_on_screen()- && w.title().is_some_and(|t| t.to_string() == title)- && w.owning_app()- .is_some_and(|a| a.process_id() == pid && a.app_name().to_string() == name)- });- ensure!(same, "allowed window changed during capture");+ for (s, ok) in before.iter().zip(&allowed) {+ if *ok {+ ensure!(+ after_windows.iter().any(|w| w.id() == s.id+ && w.is_on_screen()+ && w.owning_app().is_some_and(|a| a.process_id() == s.pid)),+ "allowed window changed during capture"+ );+ }}- let mut pixels = sample- .image_buf()- .ok_or_else(|| anyhow!("missing screenshot pixels"))?- .retained();- rgba(&mut pixels)+ Ok(DynamicImage::ImageRgba8(canvas))}fn rgba(buf: &mut cv::ImageBuf) -> anyhow::Result<DynamicImage> {@@ -172,7 +461,7 @@ fn rgba(buf: &mut cv::ImageBuf) -> anyhow::Result<DynamicImage> {.chunks_exact(4).zip(target.chunks_exact_mut(4)){- t.copy_from_slice(&[s[2], s[1], s[0], 255]);+ t.copy_from_slice(&[s[2], s[1], s[0], s[3]]);}}Ok(DynamicImage::ImageRgba8(image))diff --git a/crates/screenpipe-screen/src/screenshot_privacy/scene.rs b/crates/screenpipe-screen/src/screenshot_privacy/scene.rsnew file mode 100644index 000000000..e6640caa9--- /dev/null+++ b/crates/screenpipe-screen/src/screenshot_privacy/scene.rs@@ -0,0 +1,246 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++//! Pixel operations for native, positively selected window surfaces. Rectangles+//! use output pixels; no operation reads an unfiltered monitor image.+use image::{Rgba, RgbaImage};++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub(super) struct Rect {+ pub x: i64,+ pub y: i64,+ pub width: u32,+ pub height: u32,+}++impl Rect {+ pub fn intersection(self, other: Self) -> Option<Self> {+ let x = self.x.max(other.x);+ let y = self.y.max(other.y);+ let right = (self.x + i64::from(self.width)).min(other.x + i64::from(other.width));+ let bottom = (self.y + i64::from(self.height)).min(other.y + i64::from(other.height));+ (right > x && bottom > y).then_some(Self {+ x,+ y,+ width: (right - x).max(0) as u32,+ height: (bottom - y).max(0) as u32,+ })+ }++ pub fn intersects(self, other: Self) -> bool {+ self.intersection(other).is_some()+ }+}++pub(super) fn paint_black(image: &mut RgbaImage, rect: Rect) {+ let Some(rect) = rect.intersection(Rect {+ x: 0,+ y: 0,+ width: image.width(),+ height: image.height(),+ }) else {+ return;+ };+ for y in rect.y as u32..rect.y as u32 + rect.height {+ for x in rect.x as u32..rect.x as u32 + rect.width {+ image.put_pixel(x, y, Rgba([0, 0, 0, 255]));+ }+ }+}++/// Use only coverage from a native shell surface; never copy excluded RGB.+#[cfg(any(target_os = "macos", test))]+pub(super) fn paint_black_coverage(canvas: &mut RgbaImage, mask: &RgbaImage, x: i64, y: i64) {+ for (sx, sy, pixel) in mask.enumerate_pixels() {+ let dx = x + i64::from(sx);+ let dy = y + i64::from(sy);+ if pixel[3] > 0+ && dx >= 0+ && dy >= 0+ && dx < i64::from(canvas.width())+ && dy < i64::from(canvas.height())+ {+ canvas.put_pixel(dx as u32, dy as u32, Rgba([0, 0, 0, 255]));+ }+ }+}++/// Blend premultiplied native RGBA onto an opaque canvas. An optional region+/// limits restoration to a redacted area, leaving other foreground pixels alone.+pub(super) fn overlay_premultiplied(+ canvas: &mut RgbaImage,+ source: &RgbaImage,+ x: i64,+ y: i64,+ region: Option<Rect>,+) {+ let source_rect = Rect {+ x,+ y,+ width: source.width(),+ height: source.height(),+ };+ let bounds = Rect {+ x: 0,+ y: 0,+ width: canvas.width(),+ height: canvas.height(),+ };+ let Some(mut clip) = source_rect.intersection(bounds) else {+ return;+ };+ if let Some(region) = region {+ let Some(intersection) = clip.intersection(region) else {+ return;+ };+ clip = intersection;+ }+ for dy in clip.y..clip.y + i64::from(clip.height) {+ for dx in clip.x..clip.x + i64::from(clip.width) {+ let s = source.get_pixel((dx - x) as u32, (dy - y) as u32);+ if s[3] == 0 {+ continue;+ } // unused RGB bytes are not content+ let d = canvas.get_pixel_mut(dx as u32, dy as u32);+ let inverse = 255 - u16::from(s[3]);+ for c in 0..3 {+ d[c] = (u16::from(s[c]) + (u16::from(d[c]) * inverse + 127) / 255).min(255) as u8;+ }+ d[3] = 255;+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ fn rect(x: i64, y: i64, w: u32, h: u32) -> Rect {+ Rect {+ x,+ y,+ width: w,+ height: h,+ }+ }+ #[test]+ fn excluded_shell_coverage_preserves_transparent_desktop_area() {+ let mut canvas = RgbaImage::from_pixel(3, 1, Rgba([0, 255, 255, 255]));+ let mask =+ RgbaImage::from_raw(3, 1, vec![255, 0, 255, 0, 255, 0, 255, 1, 255, 0, 255, 255])+ .unwrap();+ paint_black_coverage(&mut canvas, &mask, 0, 0);+ assert_eq!(canvas.get_pixel(0, 0).0, [0, 255, 255, 255]);+ assert_eq!(canvas.get_pixel(1, 0).0, [0, 0, 0, 255]);+ assert_eq!(canvas.get_pixel(2, 0).0, [0, 0, 0, 255]);+ }+ #[test]+ fn overlapping_exclusions_restore_transparency_once_per_pixel() {+ let mut regions = Vec::new();+ add_region(&mut regions, rect(0, 0, 3, 3));+ add_region(&mut regions, rect(1, 1, 3, 3));+ add_region(&mut regions, rect(0, 0, 3, 3));+ for (i, a) in regions.iter().enumerate() {+ for b in regions.iter().skip(i + 1) {+ assert!(!a.intersects(*b));+ }+ }+ let mut canvas = RgbaImage::from_pixel(4, 4, Rgba([0, 0, 0, 255]));+ let source = RgbaImage::from_pixel(4, 4, Rgba([64, 0, 0, 128]));+ for r in regions {+ overlay_premultiplied(&mut canvas, &source, 0, 0, Some(r));+ }+ assert_eq!(canvas.get_pixel(1, 1).0, [64, 0, 0, 255]);+ assert_eq!(canvas.get_pixel(3, 0).0, [0, 0, 0, 255]);+ assert_eq!(canvas.pixels().filter(|p| p[0] == 64).count(), 14);+ }++ #[test]+ fn excluded_middle_layer_keeps_allowed_foreground_and_hides_background() {+ let mut canvas = RgbaImage::from_pixel(12, 8, Rgba([0, 200, 0, 255]));+ paint_black(&mut canvas, rect(3, 2, 6, 4));+ let front = RgbaImage::from_pixel(3, 3, Rgba([200, 0, 0, 255]));+ overlay_premultiplied(&mut canvas, &front, 7, 3, None);+ assert_eq!(canvas.get_pixel(2, 3).0, [0, 200, 0, 255]);+ assert_eq!(canvas.get_pixel(4, 3).0, [0, 0, 0, 255]);+ assert_eq!(canvas.get_pixel(8, 3).0, [200, 0, 0, 255]);+ // An excluded window raised above that foreground must erase it.+ paint_black(&mut canvas, rect(3, 2, 6, 4));+ assert_eq!(canvas.get_pixel(8, 3).0, [0, 0, 0, 255]);+ }+ #[test]+ fn transparent_foreground_reveals_black_redaction_not_hidden_background() {+ let mut canvas = RgbaImage::from_pixel(2, 1, Rgba([0, 0, 0, 255]));+ let source = RgbaImage::from_raw(2, 1, vec![64, 32, 16, 128, 255, 255, 255, 0]).unwrap();+ overlay_premultiplied(&mut canvas, &source, 0, 0, None);+ assert_eq!(canvas.get_pixel(0, 0).0, [64, 32, 16, 255]);+ assert_eq!(canvas.get_pixel(1, 0).0, [0, 0, 0, 255]);+ }+ #[test]+ fn negative_origin_and_spanning_window_clip_to_each_display() {+ let mut left = RgbaImage::from_pixel(4, 3, Rgba([255; 4]));+ paint_black(&mut left, rect(-2, -1, 4, 3));+ assert_eq!(left.get_pixel(0, 0).0, [0, 0, 0, 255]);+ assert_eq!(left.get_pixel(2, 0).0, [255; 4]);+ assert_eq!(left.get_pixel(0, 2).0, [255; 4]);+ assert!(!rect(0, 0, 4, 3).intersects(rect(4, 0, 2, 2)));+ assert!(rect(0, 0, 4, 3).intersects(rect(3, 0, 2, 2)));+ }+ #[test]+ fn restore_only_redacted_intersection_does_not_overpaint_other_windows() {+ let mut canvas = RgbaImage::from_pixel(4, 4, Rgba([0, 0, 200, 255]));+ let region = rect(0, 0, 2, 2);+ paint_black(&mut canvas, region);+ let source = RgbaImage::from_pixel(4, 4, Rgba([200, 0, 0, 255]));+ overlay_premultiplied(&mut canvas, &source, 0, 0, Some(region));+ assert_eq!(canvas.get_pixel(1, 1).0, [200, 0, 0, 255]);+ assert_eq!(canvas.get_pixel(2, 1).0, [0, 0, 200, 255]);+ }+}++/// Keep an exact disjoint union so translucent foreground pixels are never+/// blended twice where two excluded windows overlap.+#[cfg(any(target_os = "macos", test))]+pub(super) fn add_region(regions: &mut Vec<Rect>, rect: Rect) {+ let mut pending = vec![rect];+ for existing in regions.iter() {+ let mut next = Vec::new();+ for r in pending {+ let Some(i) = r.intersection(*existing) else {+ next.push(r);+ continue;+ };+ for part in [+ Rect {+ x: r.x,+ y: r.y,+ width: r.width,+ height: (i.y - r.y) as u32,+ },+ Rect {+ x: r.x,+ y: i.y + i64::from(i.height),+ width: r.width,+ height: (r.y + i64::from(r.height) - i.y - i64::from(i.height)) as u32,+ },+ Rect {+ x: r.x,+ y: i.y,+ width: (i.x - r.x) as u32,+ height: i.height,+ },+ Rect {+ x: i.x + i64::from(i.width),+ y: i.y,+ width: (r.x + i64::from(r.width) - i.x - i64::from(i.width)) as u32,+ height: i.height,+ },+ ] {+ if part.width > 0 && part.height > 0 {+ next.push(part);+ }+ }+ }+ pending = next;+ }+ regions.extend(pending);+}diff --git a/crates/screenpipe-screen/src/screenshot_privacy/windows.rs b/crates/screenpipe-screen/src/screenshot_privacy/windows.rsindex de6ffacb0..279035f5a 100644--- a/crates/screenpipe-screen/src/screenshot_privacy/windows.rs+++ b/crates/screenpipe-screen/src/screenshot_privacy/windows.rs@@ -7,46 +7,158 @@ use crate::{wgc_capture::PersistentCapture,};use anyhow::{anyhow, bail, Context, Result};-use image::{imageops, DynamicImage, Rgba, RgbaImage};+use image::{DynamicImage, Rgba, RgbaImage};use once_cell::sync::Lazy;use screenpipe_a11y::incognito::create_detector;-use std::{sync::Arc, time::Duration};+use std::{collections::HashMap, sync::Arc, time::Duration};use tokio::sync::Semaphore;+use windows::Win32::{+ Foundation::{GetLastError, SetLastError, BOOL, COLORREF, ERROR_SUCCESS, HWND, LPARAM, RECT},+ Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_CLOAKED, DWMWA_EXTENDED_FRAME_BOUNDS},+ UI::WindowsAndMessaging::{+ EnumWindows, GetClassNameW, GetLayeredWindowAttributes, GetShellWindow, GetWindowLongW,+ GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId, IsIconic, IsWindowVisible,+ GWL_EXSTYLE, LAYERED_WINDOW_ATTRIBUTES_FLAGS, LWA_ALPHA, LWA_COLORKEY, WS_EX_LAYERED,+ },+};use xcap::Window;#[derive(Debug, Clone, PartialEq, Eq)]-struct WindowMetadata {- id: u32,+struct PolicyMetadata {pid: u32,app_name: String,title: String,+}+#[derive(Debug, Clone, PartialEq, Eq)]+enum SurfacePolicy {+ Allowed,+ Blocked,+ Unknown,+}+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum SurfaceOpacity {+ Ordinary,+ PerPixel,+ Constant {+ alpha: u8,+ color_key: Option<[u8; 3]>,+ },+}++impl SurfaceOpacity {+ fn alpha(self) -> u8 {+ match self {+ Self::Constant { alpha, .. } => alpha,+ _ => 255,+ }+ }++ fn color_key(self) -> Option<[u8; 3]> {+ match self {+ Self::Constant { color_key, .. } => color_key,+ _ => None,+ }+ }+}++fn surface_opacity(hwnd: HWND) -> Result<SurfaceOpacity> {+ unsafe { SetLastError(ERROR_SUCCESS) };+ let style = unsafe { GetWindowLongW(hwnd, GWL_EXSTYLE) } as u32;+ if style == 0 && unsafe { GetLastError() } != ERROR_SUCCESS {+ bail!("unable to read native window style");+ }+ if style & WS_EX_LAYERED.0 == 0 {+ return Ok(SurfaceOpacity::Ordinary);+ }+ let mut alpha = 255;+ let mut key = COLORREF(0);+ let mut flags = LAYERED_WINDOW_ATTRIBUTES_FLAGS(0);+ if unsafe {+ GetLayeredWindowAttributes(hwnd, Some(&mut key), Some(&mut alpha), Some(&mut flags))+ }+ .is_err()+ {+ return Ok(SurfaceOpacity::PerPixel);+ }+ let color_key = (flags.0 & LWA_COLORKEY.0 != 0).then_some([+ key.0 as u8,+ (key.0 >> 8) as u8,+ (key.0 >> 16) as u8,+ ]);+ Ok(SurfaceOpacity::Constant {+ alpha: if flags.0 & LWA_ALPHA.0 != 0 {+ alpha+ } else {+ 255+ },+ color_key,+ })+}+#[derive(Debug, Clone, PartialEq, Eq)]+struct NativeWindow {+ id: u32,+ pid: u32,+ class_name: String,+ title: String,x: i32,y: i32,width: u32,height: u32,+ policy: SurfacePolicy,+ opacity: SurfaceOpacity,}-fn metadata(window: &Window) -> Result<WindowMetadata> {- let value = WindowMetadata {- id: window.id().context("window id")?,- pid: window.pid().context("window pid")?,- app_name: window.app_name().context("window app name")?,- title: window.title().context("window title")?,- x: window.x().context("window x")?,- y: window.y().context("window y")?,- width: window.width().context("window width")?,- height: window.height().context("window height")?,- };- if value.pid == 0 || value.app_name.trim().is_empty() || value.title.trim().is_empty() {- bail!("window {} has incomplete privacy metadata", value.id);+fn wide_text(mut read: impl FnMut(&mut [u16]) -> i32, capacity: usize) -> Result<String> {+ let mut buffer = vec![0; capacity.max(1)];+ unsafe { SetLastError(ERROR_SUCCESS) };+ let length = read(&mut buffer);+ if length == 0 && unsafe { GetLastError() } != ERROR_SUCCESS {+ bail!("native window text read failed");}- if value.width == 0 || value.height == 0 || value.width > 16384 || value.height > 16384 {- bail!("window {} has invalid dimensions", value.id);+ Ok(String::from_utf16_lossy(+ &buffer[..(length.max(0) as usize).min(buffer.len())],+ ))+}++unsafe extern "system" fn enum_window(hwnd: HWND, lparam: LPARAM) -> BOOL {+ unsafe { (&mut *(lparam.0 as *mut Vec<HWND>)).push(hwnd) };+ true.into()+}++fn enumerate_hwnds() -> Result<Vec<HWND>> {+ let mut ids = Vec::new();+ unsafe {+ EnumWindows(+ Some(enum_window),+ LPARAM(&mut ids as *mut Vec<HWND> as isize),+ )+ }+ .ok()+ .context("EnumWindows")?;+ Ok(ids)+}++fn policy_metadata() -> Result<HashMap<u32, PolicyMetadata>> {+ let mut result = HashMap::new();+ for window in Window::all().context("enumerate policy windows")? {+ let Ok(id) = window.id() else { continue };+ let value = (|| {+ Ok::<_, anyhow::Error>(PolicyMetadata {+ pid: window.pid().context("window pid")?,+ app_name: window.app_name().context("window app name")?,+ title: window.title().context("window title")?,+ })+ })();+ if let Ok(value) = value {+ if value.pid != 0 && !value.app_name.trim().is_empty() {+ result.insert(id, value);+ }+ }}- Ok(value)+ Ok(result)}-fn overlaps_monitor(window: &WindowMetadata, monitor: &MonitorData) -> bool {+fn overlaps_monitor(window: &NativeWindow, monitor: &MonitorData) -> bool {let right = i64::from(window.x) + i64::from(window.width);let bottom = i64::from(window.y) + i64::from(window.height);let monitor_right = i64::from(monitor.x) + i64::from(monitor.width);@@ -57,77 +169,272 @@ fn overlaps_monitor(window: &WindowMetadata, monitor: &MonitorData) -> bool {&& bottom > i64::from(monitor.y)}-fn capture_filtered_sync(+fn snapshot(monitor: &MonitorData,filters: &WindowFilters,sources: ExclusionSources,-) -> Result<DynamicImage> {- let mut windows = Window::all().context("enumerate visible windows")?;+) -> Result<Vec<NativeWindow>> {+ let metadata = policy_metadata()?;let detector = sources.ignore_incognito_windows.then(|| create_detector(sources.enhanced_incognito_detection));- let mut output = RgbaImage::from_pixel(monitor.width, monitor.height, Rgba([0, 0, 0, 255]));-- // xcap enumerates top-to-bottom. Paint bottom-to-top so allowed overlap is faithful.- windows.reverse();- let mut captured = 0;- for window in windows {- // Unknown windows are never selected. An error on an already selected- // window below invalidates the entire frame.- if !matches!(window.is_minimized(), Ok(false)) {+ let shell = unsafe { GetShellWindow() };+ let mut shell_pid = 0;+ unsafe { GetWindowThreadProcessId(shell, Some(&mut shell_pid)) };+ let mut result = Vec::new();+ // EnumWindows is top-to-bottom. Snapshot order is security-relevant.+ for hwnd in enumerate_hwnds()? {+ if !unsafe { IsWindowVisible(hwnd) }.as_bool() || unsafe { IsIconic(hwnd) }.as_bool() {continue;}- let Ok(before) = metadata(&window) else {+ let mut cloaked = 0u32;+ unsafe {+ DwmGetWindowAttribute(+ hwnd,+ DWMWA_CLOAKED,+ &mut cloaked as *mut u32 as *mut _,+ std::mem::size_of::<u32>() as u32,+ )+ }+ .context("DwmGetWindowAttribute(DWMWA_CLOAKED)")?;+ if cloaked != 0 {+ continue;+ }+ let mut rect = RECT::default();+ if unsafe {+ DwmGetWindowAttribute(+ hwnd,+ DWMWA_EXTENDED_FRAME_BOUNDS,+ &mut rect as *mut RECT as *mut _,+ std::mem::size_of::<RECT>() as u32,+ )+ }+ .is_err()+ {+ // An uncertain visible surface must never be reconstructed from guessed+ // geometry. Withhold the complete frame instead.+ bail!(+ "unable to resolve DWM frame bounds for window {}",+ hwnd.0 as usize+ );+ }+ let width = rect.right.saturating_sub(rect.left) as u32;+ let height = rect.bottom.saturating_sub(rect.top) as u32;+ if width == 0 || height == 0 {continue;+ }+ if !overlaps_monitor(+ &NativeWindow {+ id: 0,+ pid: 0,+ class_name: String::new(),+ title: String::new(),+ x: rect.left,+ y: rect.top,+ width,+ height,+ policy: SurfacePolicy::Unknown,+ opacity: SurfaceOpacity::Ordinary,+ },+ monitor,+ ) {+ continue;+ }+ if width > 32768 || height > 32768 {+ bail!("oversized visible window intersects monitor");+ }+ let mut pid = 0;+ unsafe { GetWindowThreadProcessId(hwnd, Some(&mut pid)) };+ let class_name = wide_text(|buffer| unsafe { GetClassNameW(hwnd, buffer) }, 256)?;+ if class_name.is_empty() {+ bail!("native window class is unavailable");+ }+ unsafe { SetLastError(ERROR_SUCCESS) };+ let title_length = unsafe { GetWindowTextLengthW(hwnd) };+ if title_length == 0 && unsafe { GetLastError() } != ERROR_SUCCESS {+ bail!("native window title length is unavailable");+ }+ let title_capacity = title_length.max(0) as usize + 1;+ let title = wide_text(+ |buffer| unsafe { GetWindowTextW(hwnd, buffer) },+ title_capacity,+ )?;+ let opacity = surface_opacity(hwnd)?;+ let id = hwnd.0 as usize as u32;+ // Shell-owned switchers and previews can contain live pixels copied from a+ // different source window. Their own PID/title is not authoritative for a+ // tile, so never trust them as ordinary Explorer/XAML surfaces.+ let unverified_preview = matches!(+ class_name.as_str(),+ "TaskListThumbnailWnd"+ | "MultitaskingViewFrame"+ | "XamlExplorerHostIslandWindow"+ | "ForegroundStaging"+ | "Shell_DimLayer"+ ) || class_name.contains("Thumbnail");+ let desktop_surface =+ pid != 0 && pid == shell_pid && matches!(class_name.as_str(), "Progman" | "WorkerW");+ let (policy, geometry) = if unverified_preview {+ (SurfacePolicy::Blocked, (rect.left, rect.top, width, height))+ } else if desktop_surface {+ let allowed = super::window_allowed(+ filters,+ detector.as_deref(),+ "Windows Explorer",+ if title.is_empty() {+ "Program Manager"+ } else {+ &title+ },+ pid as i32,+ );+ (+ if allowed {+ SurfacePolicy::Allowed+ } else {+ SurfacePolicy::Blocked+ },+ (rect.left, rect.top, width, height),+ )+ } else if let Some(meta) = metadata+ .get(&id)+ .filter(|meta| meta.pid == pid && meta.title == title)+ {+ if super::window_allowed(+ filters,+ detector.as_deref(),+ &meta.app_name,+ &meta.title,+ meta.pid as i32,+ ) {+ (SurfacePolicy::Allowed, (rect.left, rect.top, width, height))+ } else {+ // Redact the conservative native region for blocked surfaces.+ (SurfacePolicy::Blocked, (rect.left, rect.top, width, height))+ }+ } else {+ (SurfacePolicy::Unknown, (rect.left, rect.top, width, height))};- if !overlaps_monitor(&before, monitor) {+ let window = NativeWindow {+ id,+ pid,+ class_name,+ title,+ x: geometry.0,+ y: geometry.1,+ width: geometry.2,+ height: geometry.3,+ policy,+ opacity,+ };+ if overlaps_monitor(&window, monitor) {+ result.push(window);+ }+ }+ Ok(result)+}++fn paint_black(output: &mut RgbaImage, window: &NativeWindow, monitor: &MonitorData) {+ let left = (i64::from(window.x) - i64::from(monitor.x)).max(0) as u32;+ let top = (i64::from(window.y) - i64::from(monitor.y)).max(0) as u32;+ let right = (i64::from(window.x) + i64::from(window.width) - i64::from(monitor.x))+ .clamp(0, i64::from(monitor.width)) as u32;+ let bottom = (i64::from(window.y) + i64::from(window.height) - i64::from(monitor.y))+ .clamp(0, i64::from(monitor.height)) as u32;+ for row in output+ .rows_mut()+ .skip(top as usize)+ .take(bottom.saturating_sub(top) as usize)+ {+ for pixel in row+ .skip(left as usize)+ .take(right.saturating_sub(left) as usize)+ {+ *pixel = Rgba([0, 0, 0, 255]);+ }+ }+}++/// WGC B8G8R8A8 surfaces are premultiplied. `imageops::overlay` expects straight+/// alpha and would multiply RGB a second time, darkening translucent windows.+fn overlay_wgc_premultiplied(+ output: &mut RgbaImage,+ foreground: &RgbaImage,+ x: i64,+ y: i64,+ opacity: SurfaceOpacity,+) {+ for (sx, sy, source) in foreground.enumerate_pixels() {+ let dx = x + i64::from(sx);+ let dy = y + i64::from(sy);+ if dx < 0 || dy < 0 || dx >= i64::from(output.width()) || dy >= i64::from(output.height()) {continue;}- let allowed = super::window_allowed(- filters,- detector.as_deref(),- &before.app_name,- &before.title,- before.pid as i32,- );- if !allowed {+ if opacity.color_key().is_some_and(|key| source.0[..3] == key) {+ continue;+ }+ let global_alpha = u32::from(opacity.alpha());+ let alpha = (u32::from(source[3]) * global_alpha + 127) / 255;+ if alpha == 0 {continue;}+ let background = output.get_pixel_mut(dx as u32, dy as u32);+ let inverse = 255 - alpha;+ for channel in 0..3 {+ background[channel] = ((u32::from(source[channel]) * global_alpha + 127) / 255+ + (u32::from(background[channel]) * inverse + 127) / 255)+ .min(255) as u8;+ }+ background[3] = 255;+ }+}- // Use Screenpipe's native window capture on every architecture. This- // never falls back to monitor pixels and supports software D3D devices.- let capture = PersistentCapture::new_window(before.id)?;- // The session was just created for this capture. Its primed first frame- // is fresh even if it arrived before this read, and static windows need- // not produce a second compositor update to become capturable.- let image = capture- .get_latest_image_streaming(Duration::from_secs(2))?- .into_rgba8();- drop(capture);- if image.width() != before.width || image.height() != before.height {- bail!("window {} capture dimensions changed", before.id);- }- // Preserve WGC alpha while compositing onto an opaque black canvas;- // transparent pixels must not expose their unused RGB storage.- let after = metadata(&window)?;- if after != before || window.is_minimized()? {- bail!("window {} changed during privacy capture", before.id);- }- imageops::overlay(- &mut output,- &image,- i64::from(before.x) - i64::from(monitor.x),- i64::from(before.y) - i64::from(monitor.y),- );- captured += 1;+fn capture_filtered_sync(+ monitor: &MonitorData,+ filters: &WindowFilters,+ sources: ExclusionSources,+) -> Result<DynamicImage> {+ let before = snapshot(monitor, filters, sources)?;+ let mut output = RgbaImage::from_pixel(monitor.width, monitor.height, Rgba([0, 0, 0, 255]));+ for window in before.iter().rev() {+ match window.policy {+ SurfacePolicy::Blocked | SurfacePolicy::Unknown => {+ paint_black(&mut output, window, monitor)+ }+ SurfacePolicy::Allowed => {+ let capture = PersistentCapture::new_privacy_window(window.id, window.pid)?;+ let image = capture+ .get_latest_image_streaming(Duration::from_secs(2))?+ .into_rgba8();+ drop(capture);+ if (image.width(), image.height()) != (window.width, window.height) {+ bail!(+ "window {} ({}) capture geometry {}x{} != native {}x{}",+ window.id,+ window.class_name,+ image.width(),+ image.height(),+ window.width,+ window.height+ );+ }+ // Blend alpha only onto pixels already proven safe; unused RGB cannot leak.+ overlay_wgc_premultiplied(+ &mut output,+ &image,+ i64::from(window.x) - i64::from(monitor.x),+ i64::from(window.y) - i64::from(monitor.y),+ window.opacity,+ );+ }+ }}- if captured == 0 {- bail!("no verified allowed windows on monitor");+ if snapshot(monitor, filters, sources)? != before {+ bail!("window scene changed during privacy capture");}Ok(DynamicImage::ImageRgba8(output))}-// A timed-out native call retains its permit so it cannot accumulate workers.static CAPTURE_SLOT: Lazy<Arc<Semaphore>> = Lazy::new(|| Arc::new(Semaphore::new(1)));pub(crate) async fn capture_filtered(@@ -154,18 +461,19 @@ pub(crate) async fn capture_filtered(#[cfg(test)]mod tests {use super::*;-#[test]fn overlap_math_rejects_adjacent_windows() {- let window = WindowMetadata {+ let window = NativeWindow {id: 1,pid: 1,- app_name: "test".into(),+ class_name: "test".into(),title: "test".into(),x: 100,y: 0,width: 10,height: 10,+ policy: SurfacePolicy::Allowed,+ opacity: SurfaceOpacity::Ordinary,};let monitor = MonitorData {name: "test".into(),@@ -177,4 +485,76 @@ mod tests {};assert!(!overlaps_monitor(&window, &monitor));}++ #[test]+ fn premultiplied_alpha_is_not_multiplied_twice() {+ let mut output = RgbaImage::from_pixel(1, 1, Rgba([0, 0, 0, 255]));+ let foreground = RgbaImage::from_pixel(1, 1, Rgba([100, 50, 25, 128]));+ overlay_wgc_premultiplied(&mut output, &foreground, 0, 0, SurfaceOpacity::Ordinary);+ assert_eq!(output.get_pixel(0, 0), &Rgba([100, 50, 25, 255]));+ }++ #[test]+ fn alpha_zero_ignores_rgb_payload() {+ let mut output = RgbaImage::from_pixel(1, 1, Rgba([7, 8, 9, 255]));+ let foreground = RgbaImage::from_pixel(1, 1, Rgba([255, 1, 200, 0]));+ overlay_wgc_premultiplied(&mut output, &foreground, 0, 0, SurfaceOpacity::Ordinary);+ assert_eq!(output.get_pixel(0, 0), &Rgba([7, 8, 9, 255]));+ }++ #[test]+ fn global_opacity_preserves_half_cyan_over_black() {+ let mut output = RgbaImage::from_pixel(1, 1, Rgba([0, 0, 0, 255]));+ let foreground = RgbaImage::from_pixel(1, 1, Rgba([0, 255, 255, 255]));+ overlay_wgc_premultiplied(+ &mut output,+ &foreground,+ 0,+ 0,+ SurfaceOpacity::Constant {+ alpha: 128,+ color_key: None,+ },+ );+ assert_eq!(output.get_pixel(0, 0), &Rgba([0, 128, 128, 255]));+ }++ #[test]+ fn global_opacity_scales_premultiplied_rgb_once() {+ let mut output = RgbaImage::from_pixel(1, 1, Rgba([0, 0, 0, 255]));+ let foreground = RgbaImage::from_pixel(1, 1, Rgba([100, 50, 25, 128]));+ overlay_wgc_premultiplied(+ &mut output,+ &foreground,+ 0,+ 0,+ SurfaceOpacity::Constant {+ alpha: 128,+ color_key: None,+ },+ );+ assert_eq!(output.get_pixel(0, 0), &Rgba([50, 25, 13, 255]));+ }++ #[test]+ #[ignore = "requires a live unlocked Windows console"]+ fn filtered_capture_keeps_full_monitor_dimensions() {+ let monitor = xcap::Monitor::all()+ .expect("enumerate monitors")+ .into_iter()+ .next()+ .expect("monitor");+ let info = MonitorData {+ name: monitor.name().expect("monitor name"),+ width: monitor.width().expect("monitor width"),+ height: monitor.height().expect("monitor height"),+ x: monitor.x().expect("monitor x"),+ y: monitor.y().expect("monitor y"),+ is_primary: monitor.is_primary().expect("monitor primary"),+ };+ let filters = WindowFilters::new(&["__screenpipe_no_match__".into()], &[], &[]);+ let image = capture_filtered_sync(&info, &filters, ExclusionSources::default())+ .expect("filtered capture");+ assert_eq!((image.width(), image.height()), (info.width, info.height));+ }}diff --git a/crates/screenpipe-screen/src/wgc_capture.rs b/crates/screenpipe-screen/src/wgc_capture.rsindex 4e9f2a813..27f974428 100644--- a/crates/screenpipe-screen/src/wgc_capture.rs+++ b/crates/screenpipe-screen/src/wgc_capture.rs@@ -33,8 +33,9 @@use anyhow::{anyhow, Result};use image::{DynamicImage, RgbaImage};+use std::collections::VecDeque;use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};-use std::sync::{Arc, Condvar, Mutex, MutexGuard};+use std::sync::{Arc, Condvar, Mutex, MutexGuard, OnceLock};use std::time::{Duration, Instant};use windows::{core::{factory, IInspectable, Interface},@@ -373,6 +374,68 @@ struct LatestFrame {demand: CopyDemand,}+// CreateForWindow can retain an ALPC port after the returned item is released.+// Cache only a bounded set of keyed native targets, never sessions or pixels.+struct PrivacyWindowItem {+ window_id: u32,+ pid: u32,+ item: GraphicsCaptureItem,+ closed: Arc<AtomicBool>,+ closed_token: EventRegistrationToken,+}++impl Drop for PrivacyWindowItem {+ fn drop(&mut self) {+ let _ = self.item.RemoveClosed(self.closed_token);+ }+}++fn privacy_window_items() -> &'static Mutex<VecDeque<PrivacyWindowItem>> {+ static ITEMS: OnceLock<Mutex<VecDeque<PrivacyWindowItem>>> = OnceLock::new();+ ITEMS.get_or_init(|| Mutex::new(VecDeque::new()))+}++fn privacy_window_item(window_id: u32, pid: u32) -> Result<GraphicsCaptureItem> {+ let mut items = privacy_window_items()+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner());+ items.retain(|entry| {+ !entry.closed.load(Ordering::Acquire) && (entry.window_id != window_id || entry.pid == pid)+ });+ if let Some(index) = items+ .iter()+ .position(|entry| entry.window_id == window_id && entry.pid == pid)+ {+ let entry = items.remove(index).expect("located item");+ let item = entry.item.clone();+ items.push_back(entry);+ return Ok(item);+ }+ let interop = factory::<GraphicsCaptureItem, IGraphicsCaptureItemInterop>()?;+ let item: GraphicsCaptureItem =+ unsafe { interop.CreateForWindow(HWND(window_id as usize as *mut std::ffi::c_void))? };+ let closed = Arc::new(AtomicBool::new(false));+ let flag = closed.clone();+ let closed_token = item.Closed(+ &TypedEventHandler::<GraphicsCaptureItem, IInspectable>::new(move |_, _| {+ flag.store(true, Ordering::Release);+ Ok(())+ }),+ )?;+ const MAX_PRIVACY_ITEMS: usize = 8;+ if items.len() == MAX_PRIVACY_ITEMS {+ items.pop_front();+ }+ items.push_back(PrivacyWindowItem {+ window_id,+ pid,+ item: item.clone(),+ closed,+ closed_token,+ });+ Ok(item)+}+/// Persistent WGC capture session that keeps a single GraphicsCaptureSession alive,/// eliminating the orange border flash caused by per-frame session create/destroy.pub struct PersistentCapture {@@ -411,17 +474,18 @@ impl PersistentCapture {Self::from_item(item, false)}- /// Capture only this window's compositor surface, on every Windows architecture.- /// Native item dimensions avoid applying process DPI scaling to physical pixels.- pub(crate) fn new_window(window_id: u32) -> Result<Self> {- let interop = factory::<GraphicsCaptureItem, IGraphicsCaptureItemInterop>()- .map_err(|e| anyhow!("failed to get GraphicsCaptureItem factory: {}", e))?;- let item: GraphicsCaptureItem = unsafe {- interop- .CreateForWindow(HWND(window_id as usize as *mut std::ffi::c_void))- .map_err(|e| anyhow!("CreateForWindow failed: {}", e))?- };- Self::from_item(item, true)+ /// Capture only this window's compositor surface. Reuse only the keyed native+ /// target; every call creates a new session and never retains pixels.+ pub(crate) fn new_privacy_window(window_id: u32, pid: u32) -> Result<Self> {+ let item = privacy_window_item(window_id, pid)?;+ let result = Self::from_item(item, true);+ if result.is_err() {+ privacy_window_items()+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner())+ .retain(|entry| entry.window_id != window_id || entry.pid != pid);+ }+ result}fn from_item(item: GraphicsCaptureItem, capture_initial_frame: bool) -> Result<Self> {diff --git a/docs/COVERAGE.md b/docs/COVERAGE.mdindex b2401d212..70423487e 100644--- a/docs/COVERAGE.md+++ b/docs/COVERAGE.md@@ -37,16 +37,16 @@ results and `cargo llvm-cov` data on top when judging release confidence.### Core Engine- Mapped suites: 37-- Mapped Rust files: 404-- Active test blocks: 3936-- Ignored/manual test blocks: 188-- Weighted coverage points: 3243.0+- Mapped Rust files: 405+- Active test blocks: 3949+- Ignored/manual test blocks: 189+- Weighted coverage points: 3255.1| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 33 | 3773 | 170 | 3170.0 | 21 | 11 | 100% |-| macos | 32 | 3816 | 149 | 3165.3 | 22 | 11 | 100% |-| linux | 29 | 3351 | 145 | 2788.3 | 20 | 11 | 100% |+| windows | 33 | 3786 | 171 | 3182.1 | 21 | 11 | 100% |+| macos | 32 | 3829 | 150 | 3177.4 | 22 | 11 | 100% |+| linux | 29 | 3364 | 146 | 2800.4 | 20 | 11 | 100% |## Refreshdiff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex 1553160bd..270a2bb1e 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -8,11 +8,11 @@ confidence, and criticality.- Manifest: `docs/coverage/core-engine-map.json`- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs- Mapped suites: 37-- Mapped Rust files: 404-- Active test blocks: 3936-- Ignored/manual test blocks: 188-- Declared test blocks: 4124-- Weighted coverage points: 3243.0+- Mapped Rust files: 405+- Active test blocks: 3949+- Ignored/manual test blocks: 189+- Declared test blocks: 4138+- Weighted coverage points: 3255.1Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -23,19 +23,19 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 33 | 3773 | 170 | 3170.0 | 21 | 11 | 100% |-| macos | 32 | 3816 | 149 | 3165.3 | 22 | 11 | 100% |-| linux | 29 | 3351 | 145 | 2788.3 | 20 | 11 | 100% |+| windows | 33 | 3786 | 171 | 3182.1 | 21 | 11 | 100% |+| macos | 32 | 3829 | 150 | 3177.4 | 22 | 11 | 100% |+| linux | 29 | 3364 | 146 | 2800.4 | 20 | 11 | 100% |## Crate Summary| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows || --- | --- | --- | --- | --- | --- | --- | --- |-| screenpipe-engine | 12 | 22 | 131 | 1876 | 53 | 1443.9 | 10 |+| screenpipe-engine | 12 | 22 | 131 | 1879 | 53 | 1446.0 | 10 || screenpipe-db | 5 | 69 | 29 | 640 | 32 | 609.2 | 9 || screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 || screenpipe-audio | 8 | 29 | 54 | 706 | 56 | 617.0 | 5 |-| screenpipe-screen | 6 | 9 | 21 | 283 | 9 | 251.4 | 4 |+| screenpipe-screen | 6 | 9 | 22 | 293 | 10 | 261.4 | 4 || screenpipe-a11y | 4 | 2 | 34 | 401 | 38 | 291.5 | 3 || screenpipe-fs | 1 | 0 | 1 | 3 | 0 | 3.0 | 3 |@@ -73,17 +73,17 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| meeting | 6 suites / 1762 active / 32 ignored / 1440.7 pts | 6 suites / 1762 active / 32 ignored / 1440.7 pts | 4 suites / 1408 active / 28 ignored / 1110.4 pts || ocr | 4 suites / 141 active / 7 ignored / 130.8 pts | 4 suites / 134 active / 7 ignored / 128.6 pts | 3 suites / 125 active / 6 ignored / 119.6 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |-| performance | 13 suites / 1717 active / 86 ignored / 1516.2 pts | 14 suites / 1842 active / 94 ignored / 1566.2 pts | 13 suites / 1717 active / 86 ignored / 1516.2 pts |+| performance | 13 suites / 1730 active / 87 ignored / 1528.3 pts | 14 suites / 1855 active / 95 ignored / 1578.3 pts | 13 suites / 1730 active / 87 ignored / 1528.3 pts || pipes | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts |-| privacy | 5 suites / 1021 active / 42 ignored / 823.9 pts | 5 suites / 1077 active / 21 ignored / 825.6 pts | 5 suites / 979 active / 14 ignored / 788.8 pts |+| privacy | 5 suites / 1031 active / 43 ignored / 833.9 pts | 5 suites / 1087 active / 22 ignored / 835.6 pts | 5 suites / 989 active / 15 ignored / 798.8 pts || real-app | - | 1 suites / 125 active / 8 ignored / 50.0 pts | - || speaker | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts |-| storage | 4 suites / 705 active / 41 ignored / 588.0 pts | 4 suites / 705 active / 41 ignored / 588.0 pts | 4 suites / 705 active / 41 ignored / 588.0 pts |+| storage | 4 suites / 708 active / 41 ignored / 590.1 pts | 4 suites / 708 active / 41 ignored / 590.1 pts | 4 suites / 708 active / 41 ignored / 590.1 pts || sync | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts |-| timeline | 4 suites / 1258 active / 48 ignored / 1034.5 pts | 4 suites / 1258 active / 48 ignored / 1034.5 pts | 4 suites / 1258 active / 48 ignored / 1034.5 pts |+| timeline | 4 suites / 1271 active / 49 ignored / 1046.6 pts | 4 suites / 1271 active / 49 ignored / 1046.6 pts | 4 suites / 1271 active / 49 ignored / 1046.6 pts || transcription | 7 suites / 901 active / 53 ignored / 704.3 pts | 6 suites / 895 active / 53 ignored / 700.1 pts | 6 suites / 895 active / 53 ignored / 700.1 pts || ui-events | 4 suites / 831 active / 34 ignored / 629.8 pts | 3 suites / 762 active / 5 ignored / 581.5 pts | 3 suites / 762 active / 5 ignored / 581.5 pts |-| vision-capture | 6 suites / 584 active / 32 ignored / 460.3 pts | 6 suites / 577 active / 32 ignored / 458.1 pts | 5 suites / 568 active / 31 ignored / 449.1 pts |+| vision-capture | 6 suites / 597 active / 33 ignored / 472.4 pts | 6 suites / 590 active / 33 ignored / 470.2 pts | 5 suites / 581 active / 32 ignored / 461.2 pts |## Critical Flow Matrix@@ -137,7 +137,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 14 | 107 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. || db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 40 | 315 | 15 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. || engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 46 | 438 | 7 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |-| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 307 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |+| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 310 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. || engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 13 | 125 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. || engine-db-recovery-cli | screenpipe-engine | windows, macos, linux | database, engine-lifecycle | engine-health-lifecycle, performance-liveness | high | strong | unit | 1 | 33 | 0 | Exact DB/WAL/SHM working-copy preservation, rollback on archive failure, and restart repair for crashes during the multi-file generation swap. || engine-focus-os | screenpipe-engine | windows, macos | engine-lifecycle, os-integration | engine-health-lifecycle, performance-liveness | medium | conditional | unit | 3 | 6 | 0 | Platform focus-tracker parsing/helpers. These files are cfg-gated and only execute on their target OS. The separately invoked harness=false tests/native_multi_monitor.rs acceptance is documented in scripts/native-test-fixtures/macos/README.md and is not counted as an automatic Rust test block. |@@ -150,7 +150,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| engine-workflows | screenpipe-engine | windows, macos, linux | local-api, storage | local-api-search | high | partial | mixed | 7 | 40 | 0 | Catalog identity and persistence, revision-scoped feedback, workflow evidence and pipeline contracts, and denial of agent-granted rollout access. Does not assert live cloud discovery quality. Workflow publication orchestration deadlines and nested query backpressure are covered with virtual-time router tests. || filesystem-durable-sync | screenpipe-fs | macos, linux, windows | database, vision-capture, audio | capture-ocr-pipeline, audio-record-transcribe, engine-health-lifecycle | high | strong | unit | 1 | 3 | 0 | Durable flushes preserve local full-sync behavior, fall back to fsync only for macOS ENOTSUP, and propagate real I/O or fallback failures. Real SMB migration and continued recording are covered by the marked-volume integration test. || screen-capture-ocr-contract | screenpipe-screen | windows, macos, linux | vision-capture, ocr | capture-ocr-pipeline | high | partial | unit | 3 | 18 | 0 | Cross-platform cached-OCR unit coverage for RawCaptureResult to CaptureResult metadata, browser URL, focus state, window-to-screen OCR coordinate transformation, Tesseract output parsing, and contour-based text-region detection for the meeting OCR gate. |-| screen-capture-windowing | screenpipe-screen | windows, macos, linux | vision-capture, timeline, performance, privacy | capture-ocr-pipeline, timeline-streaming, privacy-and-redaction, performance-liveness | high | strong | mixed | 16 | 198 | 0 | Window filtering, empty-window regressions, retry policy, URL timing, monitor cache, OCR cache, snapshots, and image comparison. |+| screen-capture-windowing | screenpipe-screen | windows, macos, linux | vision-capture, timeline, performance, privacy | capture-ocr-pipeline, timeline-streaming, privacy-and-redaction, performance-liveness | high | strong | mixed | 17 | 208 | 1 | Window filtering, empty-window regressions, retry policy, URL timing, monitor cache, OCR cache, snapshots, and image comparison. || screen-custom-ocr | screenpipe-screen | windows, macos, linux | ocr | capture-ocr-pipeline | medium | conditional | manual | 1 | 0 | 2 | Custom OCR tests are ignored by default and only contribute when explicitly run. || screen-macos-ocr | screenpipe-screen | macos | ocr, vision-capture | capture-ocr-pipeline | high | strong | mixed | 2 | 9 | 1 | Apple Vision OCR source/unit coverage and fixture OCR assertions. || screen-monitor-platform | screenpipe-screen | windows, macos, linux | vision-capture | capture-ocr-pipeline | medium | partial | unit | 5 | 42 | 5 | Per-OS monitor enumeration (Windows, macOS, Wayland/portal on Linux) and the persistent Windows.Graphics.Capture session. Each file is cfg-gated and only executes on its target OS. |@@ -409,7 +409,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| engine-telemetry-observability | screenpipe-engine | src/crash_log.rs | source | 4 | 0 | 4 || engine-retention-storage | screenpipe-engine | src/disk_pressure.rs | source | 11 | 0 | 11 || engine-meeting-privacy-sync | screenpipe-engine | src/drm_detector.rs | source | 19 | 2 | 21 |-| engine-capture-timeline | screenpipe-engine | src/event_driven_capture.rs | source | 98 | 0 | 98 |+| engine-capture-timeline | screenpipe-engine | src/event_driven_capture.rs | source | 101 | 0 | 101 || engine-meeting-privacy-sync | screenpipe-engine | src/external_memory_sync.rs | source | 13 | 0 | 13 || engine-capture-timeline | screenpipe-engine | src/focus_aware_controller.rs | source | 15 | 0 | 15 || engine-focus-os | screenpipe-engine | src/focus_tracker/darwin.rs | source | 3 | 0 | 3 |@@ -547,7 +547,8 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| screen-capture-windowing | screenpipe-screen | src/ocr_cache.rs | source | 15 | 0 | 15 || screen-windows-ocr | screenpipe-screen | src/ocr_worker.rs | source | 3 | 0 | 3 || screen-capture-windowing | screenpipe-screen | src/screenshot_privacy.rs | source | 4 | 0 | 4 |-| screen-capture-windowing | screenpipe-screen | src/screenshot_privacy/windows.rs | source | 1 | 0 | 1 |+| screen-capture-windowing | screenpipe-screen | src/screenshot_privacy/scene.rs | source | 6 | 0 | 6 |+| screen-capture-windowing | screenpipe-screen | src/screenshot_privacy/windows.rs | source | 5 | 1 | 6 || screen-capture-windowing | screenpipe-screen | src/snapshot_writer.rs | source | 5 | 0 | 5 || screen-capture-ocr-contract | screenpipe-screen | src/tesseract.rs | source | 6 | 0 | 6 || screen-capture-ocr-contract | screenpipe-screen | src/text_regions.rs | source | 11 | 0 | 11 |diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.jsonindex 09153dcd2..bb809936c 100644--- a/docs/coverage/core-engine-map.json+++ b/docs/coverage/core-engine-map.json@@ -1158,6 +1158,7 @@"src/ocr_cache.rs","src/snapshot_writer.rs","src/screenshot_privacy.rs",+ "src/screenshot_privacy/scene.rs","src/screenshot_privacy/windows.rs","src/utils.rs","tests/capture_error_test.rs",