Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix: preserve desktop context when excluding windows

Louis Beaumont24e140304415
diff --git a/crates/screenpipe-engine/src/event_driven_capture.rs b/crates/screenpipe-engine/src/event_driven_capture.rs
index 2cbbed512..2a7b098be 100644
--- a/crates/screenpipe-engine/src/event_driven_capture.rs
+++ b/crates/screenpipe-engine/src/event_driven_capture.rs
@@ -1223,8 +1223,12 @@ pub(crate) async fn event_driven_capture_loop(
{
Ok(Ok(output)) => {
state.mark_captured();
- if let Some(ref mut comparer) = frame_comparer {
- let _ = comparer.compare(&output.image);
+ if !capture_params.window_filters.has_window_patterns()
+ && !capture_params.window_filters.has_url_allowlist()
+ {
+ if let Some(ref mut comparer) = frame_comparer {
+ let _ = comparer.compare(&output.image);
+ }
}
if let Some(ref result) = output.result {
last_content_hash = result.content_hash;
@@ -2137,11 +2141,16 @@ pub(crate) async fn event_driven_capture_loop(
consecutive_capture_errors = 0;
}
- // Feed the captured frame to comparer so we don't
- // re-trigger on the same visual state (reuses capture
- // image — no extra screenshot needed)
- if let Some(ref mut comparer) = frame_comparer {
- let _ = comparer.compare(&output.image);
+ // The visual probe observes the display stream. A
+ // redacted capture is a different image of a static
+ // desktop; seeding it here would retrigger capture on
+ // every probe. Let probes maintain their own reference.
+ if !capture_params.window_filters.has_window_patterns()
+ && !capture_params.window_filters.has_url_allowlist()
+ {
+ if let Some(ref mut comparer) = frame_comparer {
+ let _ = comparer.compare(&output.image);
+ }
}
// Frozen-stream detection: a live persistent stream
@@ -3054,6 +3063,81 @@ fn bypasses_capture_throttles(trigger: &CaptureTrigger) -> bool {
)
}
+/// Preserve the redacted desktop when only the focused window is filtered.
+/// URL, incognito, built-in exclusions and DRM retain their existing gates.
+fn permits_redacted_desktop_without_focus(
+ filters: &WindowFilters,
+ config: &screenpipe_a11y::tree::TreeWalkerConfig,
+ pixels_verified: bool,
+ pause_on_drm: bool,
+) -> bool {
+ pixels_verified
+ && filters.has_window_patterns()
+ && config.ignored_urls.is_empty()
+ && config.included_urls.is_empty()
+ && !pause_on_drm
+}
+
+#[derive(Debug, PartialEq, Eq)]
+enum RedactedFocusDecision {
+ Continue,
+ PixelsOnly,
+ Withhold,
+}
+
+fn redacted_focus_decision(
+ filters: &WindowFilters,
+ focus: Option<screenpipe_a11y::tree::FocusedWindowFilterResult>,
+ pixels_only_permitted: bool,
+) -> RedactedFocusDecision {
+ use screenpipe_a11y::tree::FocusedWindowFilterResult;
+ match focus {
+ Some(FocusedWindowFilterResult::Allowed {
+ app_name,
+ window_name,
+ }) if !app_name.is_empty() => {
+ if filters.is_valid(&app_name, &window_name) {
+ RedactedFocusDecision::Continue
+ } else if pixels_only_permitted {
+ RedactedFocusDecision::PixelsOnly
+ } else {
+ RedactedFocusDecision::Withhold
+ }
+ }
+ // A rejected or unavailable focus check must not fall through to the
+ // unknown-monitor OCR path, which has no focused metadata to gate.
+ _ => RedactedFocusDecision::Withhold,
+ }
+}
+
+async fn redacted_focus_check(
+ filters: &WindowFilters,
+ mut config: screenpipe_a11y::tree::TreeWalkerConfig,
+ pixels_only_permitted: bool,
+) -> RedactedFocusDecision {
+ // Retain a permit inside the blocking call even after timeout, so a stuck
+ // native metadata query cannot accumulate unbounded background workers.
+ static QUERY: std::sync::OnceLock<Arc<tokio::sync::Semaphore>> = std::sync::OnceLock::new();
+ let Ok(permit) = QUERY
+ .get_or_init(|| Arc::new(tokio::sync::Semaphore::new(1)))
+ .clone()
+ .try_acquire_owned()
+ else {
+ return RedactedFocusDecision::Withhold;
+ };
+ config.ignored_windows.clear();
+ config.included_windows.clear();
+ let query = tokio::task::spawn_blocking(move || {
+ let _permit = permit;
+ screenpipe_a11y::tree::check_focused_window_filters(config)
+ });
+ let focus = match tokio::time::timeout(Duration::from_millis(250), query).await {
+ Ok(Ok(Ok(focus))) => Some(focus),
+ _ => None,
+ };
+ redacted_focus_decision(filters, focus, pixels_only_permitted)
+}
+
/// Perform a single event-driven capture.
///
/// When `previous_content_hash` is `Some` and matches the current accessibility
@@ -3200,7 +3284,10 @@ async fn do_capture(
// (prevents re-triggering on the same bad frame). The caller records the
// matching telemetry counter from `corrupt`.
if !screenshot_disabled && !skip_pixels_for_unknown_exclusions {
- if let Some(kind) = frame_corruption(&image) {
+ if let Some(kind) = frame_corruption_with_privacy_padding(
+ &image,
+ params.window_filters.has_window_patterns(),
+ ) {
match kind {
// Green is the notable, rarer signal — surface it at warn so it
// shows up in shared logs. Black is common (DRM / excluded window).
@@ -3222,6 +3309,76 @@ async fn do_capture(
}
}
+ // The native compositor has already removed every excluded window. An
+ // excluded focused window must not discard the other allowed windows.
+ // Persist only verified pixels: no focused identity, AX, OCR, URL, document
+ // path or reused element reference can escape through this narrow path.
+ let verified_filtered_pixels = !screenshot_disabled
+ && !skip_pixels_for_unknown_exclusions
+ && params.window_filters.has_window_patterns();
+ let focus_decision = if verified_filtered_pixels {
+ redacted_focus_check(
+ &params.window_filters,
+ params.tree_walker_config.clone(),
+ permits_redacted_desktop_without_focus(
+ &params.window_filters,
+ params.tree_walker_config,
+ true,
+ params.pause_on_drm_content,
+ ),
+ )
+ .await
+ } else {
+ RedactedFocusDecision::Continue
+ };
+ if focus_decision == RedactedFocusDecision::Withhold {
+ return Ok(CaptureOutput {
+ result: None,
+ image,
+ elements_deduped: false,
+ corrupt: None,
+ });
+ }
+ if focus_decision == RedactedFocusDecision::PixelsOnly {
+ let _ = params
+ .tree_walker
+ .suspend_with_timeout(Duration::from_millis(250))
+ .await;
+ let ctx = CaptureContext {
+ db: params.db,
+ snapshot_writer: params.snapshot_writer,
+ image: Arc::new(image),
+ captured_at,
+ monitor_id: params.monitor_id,
+ device_name: params.device_name,
+ app_name: None,
+ window_name: None,
+ browser_url: None,
+ document_path: None,
+ focused: false,
+ capture_trigger: trigger.as_str(),
+ use_pii_removal: params.use_pii_removal,
+ languages: params.languages.to_vec(),
+ elements_ref_frame_id: None,
+ screenshot_disabled: false,
+ in_meeting,
+ monitor_hosts_focus: false,
+ ax_screenshot_coherent: false,
+ focused_window_bounds: None,
+ };
+ let result = paired_capture_deferred(&ctx).await?;
+ if let Some(sender) = params.semantic_tx {
+ sender.record_capture_gap(result.frame_id, SemanticCaptureGap::TreeMissing);
+ }
+ let image = Arc::try_unwrap(ctx.image).unwrap_or_else(|arc| (*arc).clone());
+ return Ok(CaptureOutput {
+ result: Some(result),
+ image,
+ elements_deduped: false,
+ corrupt: None,
+ });
+ }
+
// Walk accessibility tree on blocking thread (AX APIs are synchronous).
// Apply adaptive budget overrides: expensive apps (Electron/Discord) get
// reduced max_nodes and timeout to avoid blocking their UI thread.
@@ -4162,6 +4319,26 @@ fn frame_corruption(image: &image::DynamicImage) -> Option<CorruptKind> {
}
}
+/// Privacy composition intentionally paints excluded surfaces black. A small
+/// allowed foreground window must still be durably recorded even when it
+/// occupies less than the corruption sampler's five-percent threshold.
+fn frame_corruption_with_privacy_padding(
+ image: &image::DynamicImage,
+ isolated_windows: bool,
+) -> Option<CorruptKind> {
+ let kind = frame_corruption(image)?;
+ if isolated_windows && matches!(kind, CorruptKind::Black) {
+ use image::GenericImageView;
+ if image
+ .pixels()
+ .any(|(_, _, p)| u16::from(p[0]) + u16::from(p[1]) + u16::from(p[2]) >= 15)
+ {
+ return None;
+ }
+ }
+ Some(kind)
+}
+
/// Convenience predicate used by tests.
#[cfg(test)]
fn is_frame_corrupt(image: &image::DynamicImage) -> bool {
@@ -4172,6 +4349,70 @@ fn is_frame_corrupt(image: &image::DynamicImage) -> bool {
mod tests {
use super::*;
+ #[test]
+ fn redacted_focus_rejections_cannot_fall_through_unknown_monitor_capture() {
+ use screenpipe_a11y::tree::{FocusedWindowFilterResult as Focus, SkipReason};
+ let filters = WindowFilters::new(&["PrivacyBlocked".into()], &[], &[]);
+ let allowed = |title: &str| {
+ Some(Focus::Allowed {
+ app_name: "Fixture".into(),
+ window_name: title.into(),
+ })
+ };
+ assert_eq!(
+ redacted_focus_decision(&filters, allowed("PrivacyAllowed"), true),
+ RedactedFocusDecision::Continue
+ );
+ assert_eq!(
+ redacted_focus_decision(&filters, allowed("PrivacyBlocked"), true),
+ RedactedFocusDecision::PixelsOnly
+ );
+ // URL/DRM policies cannot use the pixels-only exception.
+ assert_eq!(
+ redacted_focus_decision(&filters, allowed("PrivacyBlocked"), false),
+ RedactedFocusDecision::Withhold
+ );
+ for focus in [
+ None,
+ Some(Focus::NotFound),
+ Some(Focus::Skipped(SkipReason::Incognito)),
+ Some(Focus::Skipped(SkipReason::ExcludedApp)),
+ ] {
+ assert_eq!(
+ redacted_focus_decision(&filters, focus, true),
+ RedactedFocusDecision::Withhold
+ );
+ }
+ }
+
+ #[test]
+ fn redacted_desktop_requires_verified_pixels_and_window_only_policy() {
+ let filters = WindowFilters::new(&["PrivacyBlocked".into()], &[], &[]);
+ let empty = WindowFilters::new(&[], &[], &[]);
+ let mut config = screenpipe_a11y::tree::TreeWalkerConfig::default();
+ assert!(permits_redacted_desktop_without_focus(
+ &filters, &config, true, false
+ ));
+ assert!(!permits_redacted_desktop_without_focus(
+ &filters, &config, false, false
+ ));
+ assert!(!permits_redacted_desktop_without_focus(
+ &filters, &config, true, true
+ ));
+ assert!(!permits_redacted_desktop_without_focus(
+ &empty, &config, true, false
+ ));
+ config.ignored_urls = serde_json::from_str(r#"["private.example"]"#).unwrap();
+ assert!(!permits_redacted_desktop_without_focus(
+ &filters, &config, true, false
+ ));
+ config.ignored_urls.clear();
+ config.included_urls = serde_json::from_str(r#"[{"domain":"work.example"}]"#).unwrap();
+ assert!(!permits_redacted_desktop_without_focus(
+ &filters, &config, true, false
+ ));
+ }
+
#[test]
fn url_allowlist_masks_everything_outside_focused_window() {
use image::{GenericImageView, Pixel};
@@ -5894,6 +6135,31 @@ mod tests {
assert!(is_frame_corrupt(&img));
}
+ #[test]
+ fn privacy_redaction_preserves_small_allowed_foreground() {
+ let mut pixels = image::RgbaImage::from_pixel(1920, 1080, image::Rgba([0, 0, 0, 255]));
+ for y in 30..60 {
+ for x in 50..100 {
+ pixels.put_pixel(x, y, image::Rgba([200, 220, 230, 255]));
+ }
+ }
+ let image = image::DynamicImage::ImageRgba8(pixels);
+ assert_eq!(
+ frame_corruption_with_privacy_padding(&image, false),
+ Some(CorruptKind::Black)
+ );
+ assert_eq!(frame_corruption_with_privacy_padding(&image, true), None);
+ assert_eq!(
+ frame_corruption_with_privacy_padding(&solid(1920, 1080, [0, 0, 0]), true),
+ Some(CorruptKind::Black)
+ );
+ let green = frame_with_bottom_band(1920, 1080, 648, [120, 130, 140], [0, 150, 0]);
+ assert_eq!(
+ frame_corruption_with_privacy_padding(&green, true),
+ Some(CorruptKind::GreenBand)
+ );
+ }
+
#[test]
fn frame_corruption_reports_kind_for_telemetry() {
// Black and green resolve to distinct kinds so the caller bumps the
diff --git a/crates/screenpipe-screen/examples/screenshot_redaction_probe.rs b/crates/screenpipe-screen/examples/screenshot_redaction_probe.rs
new file mode 100644
index 000000000..e8085c7bd
--- /dev/null
+++ b/crates/screenpipe-screen/examples/screenshot_redaction_probe.rs
@@ -0,0 +1,124 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+//! Native acceptance/soak probe. Run only on an isolated synthetic desktop.
+//! OUT MODE MONITOR FRAMES INTERVAL_MS [MAX_WIDTH]; MODE=exclude/include/none/empty.
+use screenpipe_screen::{
+ capture_screenshot_by_window::{ExclusionSources, WindowFilters},
+ monitor::{get_default_monitor, get_monitor_by_id},
+ screenshot_privacy::capture_monitor_image_filtered,
+};
+use std::{
+ path::PathBuf,
+ time::{Duration, Instant},
+};
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+ let args: Vec<String> = std::env::args().collect();
+ let out = PathBuf::from(args.get(1).expect("output directory"));
+ std::fs::create_dir_all(&out)?;
+ let mode = args.get(2).map(String::as_str).unwrap_or("exclude");
+ let id: u32 = args.get(3).map(|s| s.parse()).transpose()?.unwrap_or(0);
+ let count: u32 = args.get(4).map(|s| s.parse()).transpose()?.unwrap_or(10);
+ let interval: u64 = args.get(5).map(|s| s.parse()).transpose()?.unwrap_or(100);
+ if let Some(cap) = args.get(6) {
+ #[cfg(target_os = "macos")]
+ screenpipe_screen::monitor::set_sck_capture_max_width(cap.parse()?);
+ #[cfg(not(target_os = "macos"))]
+ anyhow::bail!("MAX_WIDTH is unsupported on this platform: {cap}");
+ }
+ let monitor = if id == 0 {
+ get_default_monitor().await
+ } else {
+ get_monitor_by_id(id).await
+ }
+ .ok_or_else(|| anyhow::anyhow!("missing monitor"))?;
+ let filters = match mode {
+ "exclude" => WindowFilters::new(&["::PrivacyBlocked".into()], &[], &[]),
+ "none" => WindowFilters::new(&[], &[], &[]),
+ "include" => WindowFilters::new(
+ &["::PrivacyBlocked".into()],
+ &["::PrivacyAllowed".into()],
+ &[],
+ ),
+ "includeapp" => WindowFilters::new(&[], &["windows_privacy_fixture".into()], &[]),
+ "blank" => WindowFilters::new(
+ &["::PrivacyBlocked".into()],
+ &["windows_privacy_fixture".into()],
+ &[],
+ ),
+ "shell-exclude" => WindowFilters::new(&["Windows Explorer".into()], &[], &[]),
+ "empty" => WindowFilters::new(&[], &["::NoSuchWindow".into()], &[]),
+ "nomatch" => WindowFilters::new(&["::NeverPresentWindow".into()], &[], &[]),
+ _ => anyhow::bail!("unknown mode: {mode}"),
+ };
+ let writer =
+ screenpipe_screen::snapshot_writer::SnapshotWriter::new(out.join("snapshots"), 90, 0);
+ let start = Instant::now();
+ let mut saved = 0;
+ let mut rejected = 0;
+ let mut leaked = 0;
+ for index in 0..count {
+ match capture_monitor_image_filtered(
+ &monitor,
+ &filters,
+ ExclusionSources::default(),
+ &[],
+ true,
+ )
+ .await
+ {
+ Ok((image, time)) => {
+ let write_start = Instant::now();
+ let path = writer.write(&image, chrono::Utc::now(), monitor.id())?;
+ let write_ms = write_start.elapsed().as_secs_f64() * 1000.0;
+ let read_start = Instant::now();
+ let stored = image::open(&path)?.to_rgb8();
+ let read_ms = read_start.elapsed().as_secs_f64() * 1000.0;
+ let rgb = image.to_rgb8();
+ let marker = |p: &&image::Rgb<u8>| p[0] > 200 && p[1] < 60 && p[2] > 200;
+ let allowed_marker = |p: &&image::Rgb<u8>| p[0] < 30 && p[1] > 220 && p[2] > 220;
+ let partial_alpha_marker = |p: &&image::Rgb<u8>| {
+ p[0] < 30 && (90..=170).contains(&p[1]) && (90..=170).contains(&p[2])
+ };
+ let forbidden = rgb.pixels().filter(marker).count();
+ let stored_forbidden = stored.pixels().filter(marker).count();
+ let allowed = rgb.pixels().filter(allowed_marker).count();
+ let stored_allowed = stored.pixels().filter(allowed_marker).count();
+ let partial_alpha = rgb.pixels().filter(partial_alpha_marker).count();
+ if forbidden > 0 || stored_forbidden > 0 {
+ leaked += 1;
+ }
+ if saved < 2 {
+ image.save(out.join(format!("frame-{index}.png")))?;
+ }
+ println!(
+ "{}",
+ serde_json::json!({"index":index,"saved":true,"path":path,"width":image.width(),"height":image.height(),"capture_ms":time.as_secs_f64()*1000.0,"write_ms":write_ms,"read_ms":read_ms,"forbidden_pixels":forbidden,"stored_forbidden_pixels":stored_forbidden,"allowed_pixels":allowed,"stored_allowed_pixels":stored_allowed,"partial_alpha_pixels":partial_alpha})
+ );
+ saved += 1;
+ }
+ Err(error) => {
+ rejected += 1;
+ println!(
+ "{}",
+ serde_json::json!({"index":index,"saved":false,"error":error.to_string()})
+ );
+ }
+ }
+ if interval > 0 {
+ tokio::time::sleep(Duration::from_millis(interval)).await;
+ }
+ }
+ println!(
+ "{}",
+ serde_json::json!({"summary":true,"saved":saved,"rejected":rejected,"leaked":leaked,"elapsed_ms":start.elapsed().as_millis()})
+ );
+ anyhow::ensure!(saved > 0, "no screenshots stored");
+ anyhow::ensure!(
+ !matches!(mode, "exclude" | "include" | "empty" | "blank") || leaked == 0,
+ "excluded marker persisted in {leaked} frames"
+ );
+ Ok(())
+}
diff --git a/crates/screenpipe-screen/examples/windows_privacy_fixture.rs b/crates/screenpipe-screen/examples/windows_privacy_fixture.rs
index 4a1f74915..b75f5b00d 100644
--- a/crates/screenpipe-screen/examples/windows_privacy_fixture.rs
+++ b/crates/screenpipe-screen/examples/windows_privacy_fixture.rs
@@ -18,9 +18,11 @@ mod windows_harness {
System::LibraryLoader::GetModuleHandleW,
UI::WindowsAndMessaging::{
CreateWindowExW, DefWindowProcW, DispatchMessageW, GetClientRect, GetMessageW,
- LoadCursorW, RegisterClassW, ShowWindow, TranslateMessage, CS_HREDRAW, CS_VREDRAW,
- IDC_ARROW, MSG, SW_SHOW, WINDOW_EX_STYLE, WINDOW_LONG_PTR_INDEX, WM_DESTROY,
- WM_PAINT, WM_PRINT, WM_PRINTCLIENT, WNDCLASSW, WS_OVERLAPPEDWINDOW, WS_VISIBLE,
+ LoadCursorW, RegisterClassW, SetLayeredWindowAttributes, ShowWindow,
+ TranslateMessage, CS_HREDRAW, CS_VREDRAW, IDC_ARROW, LWA_ALPHA, MSG, SW_MAXIMIZE,
+ SW_SHOW, WINDOW_EX_STYLE, WINDOW_LONG_PTR_INDEX, WM_DESTROY, WM_PAINT, WM_PRINT,
+ WM_PRINTCLIENT, WNDCLASSW, WS_EX_LAYERED, WS_OVERLAPPEDWINDOW, WS_POPUP,
+ WS_VISIBLE,
},
},
};
@@ -51,14 +53,14 @@ mod windows_harness {
WM_PAINT => {
let mut paint = PAINTSTRUCT::default();
let dc = BeginPaint(hwnd, &mut paint);
- let color = if windows::Win32::UI::WindowsAndMessaging::GetWindowLongPtrW(
+ let state = windows::Win32::UI::WindowsAndMessaging::GetWindowLongPtrW(
hwnd,
WINDOW_LONG_PTR_INDEX(0),
- ) == 1
- {
- BLOCKED
- } else {
- ALLOWED
+ );
+ let color = match state {
+ 1 => BLOCKED,
+ 2 => COLORREF(0x0000_00ff),
+ _ => ALLOWED,
};
let brush = CreateSolidBrush(color);
FillRect(dc, &paint.rcPaint, brush);
@@ -69,14 +71,14 @@ mod windows_harness {
WM_PRINT | WM_PRINTCLIENT => {
let mut client = RECT::default();
let _ = GetClientRect(hwnd, &mut client);
- let color = if windows::Win32::UI::WindowsAndMessaging::GetWindowLongPtrW(
+ let state = windows::Win32::UI::WindowsAndMessaging::GetWindowLongPtrW(
hwnd,
WINDOW_LONG_PTR_INDEX(0),
- ) == 1
- {
- BLOCKED
- } else {
- ALLOWED
+ );
+ let color = match state {
+ 1 => BLOCKED,
+ 2 => COLORREF(0x0000_00ff),
+ _ => ALLOWED,
};
let brush = CreateSolidBrush(color);
FillRect(HDC(wparam.0 as *mut _), &client, brush);
@@ -88,8 +90,9 @@ mod windows_harness {
}
}
- pub fn run_window(blocked: bool, x: i32, y: i32) -> Result<()> {
+ pub fn run_window(kind: &str, x: i32, y: i32) -> Result<()> {
unsafe {
+ let blocked = matches!(kind, "blocked" | "blocked-red" | "churn");
let module = GetModuleHandleW(None).unwrap_or_default();
let class_name = if blocked {
w!("PrivacyBlockedFixture")
@@ -106,17 +109,29 @@ mod windows_harness {
..Default::default()
};
let _ = RegisterClassW(&class);
- let title = if blocked {
+ let title = if kind == "blank" {
+ ""
+ } else if blocked {
"PrivacyBlocked"
} else {
"PrivacyAllowed"
};
let title_wide: Vec<u16> = title.encode_utf16().chain(Some(0)).collect();
+ let ex_style = if kind == "transparent" {
+ WS_EX_LAYERED
+ } else {
+ WINDOW_EX_STYLE::default()
+ };
+ let style = if kind == "borderless" || kind == "transparent" {
+ WS_POPUP | WS_VISIBLE
+ } else {
+ WS_OVERLAPPEDWINDOW | WS_VISIBLE
+ };
let hwnd = CreateWindowExW(
- WINDOW_EX_STYLE::default(),
+ ex_style,
class_name,
windows::core::PCWSTR(title_wide.as_ptr()),
- WS_OVERLAPPEDWINDOW | WS_VISIBLE,
+ style,
x,
y,
520,
@@ -130,10 +145,20 @@ mod windows_harness {
windows::Win32::UI::WindowsAndMessaging::SetWindowLongPtrW(
hwnd,
WINDOW_LONG_PTR_INDEX(0),
- isize::from(blocked),
+ if kind == "blocked-red" {
+ 2
+ } else {
+ isize::from(blocked)
+ },
);
+ if kind == "transparent" {
+ let _ = SetLayeredWindowAttributes(hwnd, COLORREF(0), 128, LWA_ALPHA);
+ }
let _ = ShowWindow(hwnd, SW_SHOW);
- if blocked {
+ if kind == "maximized" {
+ let _ = ShowWindow(hwnd, SW_MAXIMIZE);
+ }
+ if kind == "churn" {
let raw = hwnd.0 as usize;
thread::spawn(move || loop {
// Keep the synthetic fixture above the test runner while
@@ -168,10 +193,22 @@ mod windows_harness {
fn main() -> anyhow::Result<()> {
let mut args = std::env::args().skip(1);
let kind = args.next().ok_or_else(|| {
- anyhow::anyhow!("usage: windows_privacy_fixture <allowed|blocked> <x> <y>")
+ anyhow::anyhow!(
+ "usage: windows_privacy_fixture <allowed|blocked|blocked-red|blank|borderless|transparent|maximized|churn> <x> <y>"
+ )
})?;
anyhow::ensure!(
- kind == "allowed" || kind == "blocked",
+ matches!(
+ kind.as_str(),
+ "allowed"
+ | "blocked"
+ | "blocked-red"
+ | "blank"
+ | "borderless"
+ | "transparent"
+ | "maximized"
+ | "churn"
+ ),
"unknown fixture window"
);
let x = args
@@ -182,7 +219,7 @@ fn main() -> anyhow::Result<()> {
.next()
.ok_or_else(|| anyhow::anyhow!("missing y"))?
.parse()?;
- windows_harness::run_window(kind == "blocked", x, y)
+ windows_harness::run_window(&kind, x, y)
}
#[cfg(not(target_os = "windows"))]
diff --git a/crates/screenpipe-screen/src/screenshot_privacy.rs b/crates/screenpipe-screen/src/screenshot_privacy.rs
index 481d8a70a..3648d1915 100644
--- a/crates/screenpipe-screen/src/screenshot_privacy.rs
+++ b/crates/screenpipe-screen/src/screenshot_privacy.rs
@@ -11,6 +11,9 @@ use crate::monitor::SafeMonitor;
use image::DynamicImage;
use std::time::{Duration, Instant};
+#[cfg(any(target_os = "macos", target_os = "linux", test))]
+mod scene;
+
#[cfg(target_os = "linux")]
mod linux;
#[cfg(target_os = "macos")]
diff --git a/crates/screenpipe-screen/src/screenshot_privacy/linux.rs b/crates/screenpipe-screen/src/screenshot_privacy/linux.rs
index 4a12ea19d..3cb9cc445 100644
--- a/crates/screenpipe-screen/src/screenshot_privacy/linux.rs
+++ b/crates/screenpipe-screen/src/screenshot_privacy/linux.rs
@@ -1,12 +1,15 @@
// screenpipe — AI that knows everything you've seen, said, or heard
// https://screenpipe.com
-use super::*;
+use super::{
+ scene::{self, Rect},
+ *,
+};
use anyhow::{anyhow, ensure};
use once_cell::sync::Lazy;
use std::sync::Arc;
use tokio::sync::Semaphore;
-use xcb::{composite, x, XidNew};
+use xcb::{composite, x, Xid, XidNew};
static CAPTURE_SLOT: Lazy<Arc<Semaphore>> = Lazy::new(|| Arc::new(Semaphore::new(1)));
@@ -16,8 +19,8 @@ pub(crate) async fn capture_filtered(
sources: ExclusionSources,
_fresh: bool,
) -> anyhow::Result<DynamicImage> {
- // XWayland enumeration cannot account for native Wayland windows. Never
- // substitute a portal/whole-monitor capture when a window policy is set.
+ // XWayland cannot enumerate native Wayland surfaces. Never fall back to a
+ // portal/monitor image when isolated pixels are required.
ensure!(
std::env::var_os("WAYLAND_DISPLAY").is_none()
&& std::env::var("XDG_SESSION_TYPE").unwrap_or_default() != "wayland",
@@ -37,6 +40,198 @@ pub(crate) async fn capture_filtered(
.map_err(|_| anyhow!("window-filtered capture timed out"))??
}
+fn atom(conn: &xcb::Connection, name: &str) -> anyhow::Result<x::Atom> {
+ Ok(conn
+ .wait_for_reply(conn.send_request(&x::InternAtom {
+ only_if_exists: true,
+ name: name.as_bytes(),
+ }))?
+ .atom())
+}
+
+fn property(
+ conn: &xcb::Connection,
+ window: x::Window,
+ name: &str,
+) -> anyhow::Result<x::GetPropertyReply> {
+ let property = atom(conn, name)?;
+ ensure!(property != x::ATOM_NONE, "missing {name}");
+ Ok(conn.wait_for_reply(conn.send_request(&x::GetProperty {
+ delete: false,
+ window,
+ property,
+ r#type: x::ATOM_ANY,
+ long_offset: 0,
+ long_length: 16384,
+ }))?)
+}
+
+fn text_property(conn: &xcb::Connection, window: x::Window, name: &str) -> String {
+ property(conn, window, name)
+ .ok()
+ .filter(|p| p.format() == 8)
+ .map(|p| String::from_utf8_lossy(p.value::<u8>()).into_owned())
+ .unwrap_or_default()
+}
+
+#[derive(Debug, Clone, PartialEq)]
+struct Surface {
+ id: u32,
+ client: u32,
+ pid: u32,
+ app: String,
+ title: String,
+ rect: Rect,
+ opacity: u32,
+}
+
+/// Actual root children are bottom-to-top, including unmanaged menus, WM
+/// decorations, dock and desktop windows. The old client-only list omitted
+/// these and could reveal a lower allowed window under a blocked popup.
+fn snapshot(
+ conn: &xcb::Connection,
+ root: x::Window,
+ overlay: x::Window,
+) -> anyhow::Result<Vec<Surface>> {
+ let clients = property(conn, root, "_NET_CLIENT_LIST_STACKING")
+ .ok()
+ .filter(|p| p.format() == 32)
+ .map(|p| p.value::<u32>().to_vec())
+ .unwrap_or_default();
+ let mut frame_clients = std::collections::HashMap::new();
+ for id in clients {
+ let mut current = x::Window::new(id);
+ for _ in 0..32 {
+ let Ok(tree) =
+ conn.wait_for_reply(conn.send_request(&x::QueryTree { window: current }))
+ else {
+ break;
+ };
+ if tree.parent() == root {
+ frame_clients.insert(current.resource_id(), id);
+ break;
+ }
+ if tree.parent().resource_id() == 0 {
+ break;
+ }
+ current = tree.parent();
+ }
+ }
+ let tree = conn.wait_for_reply(conn.send_request(&x::QueryTree { window: root }))?;
+ let mut result = Vec::new();
+ for &window in tree.children() {
+ if window == overlay {
+ continue;
+ }
+ let attrs = conn.wait_for_reply(conn.send_request(&x::GetWindowAttributes { window }))?;
+ if attrs.map_state() != x::MapState::Viewable || attrs.class() == x::WindowClass::InputOnly
+ {
+ continue;
+ }
+ let geometry = conn.wait_for_reply(conn.send_request(&x::GetGeometry {
+ drawable: x::Drawable::Window(window),
+ }))?;
+ let position = conn.wait_for_reply(conn.send_request(&x::TranslateCoordinates {
+ src_window: window,
+ dst_window: root,
+ src_x: 0,
+ src_y: 0,
+ }))?;
+ let client = *frame_clients
+ .get(&window.resource_id())
+ .unwrap_or(&window.resource_id());
+ let native = x::Window::new(client);
+ let class = text_property(conn, native, "WM_CLASS");
+ let app = class.split('\0').nth(1).unwrap_or("").to_string();
+ let mut title = text_property(conn, native, "_NET_WM_NAME");
+ if title.is_empty() {
+ title = text_property(conn, native, "WM_NAME");
+ }
+ let cardinal = |name| {
+ property(conn, native, name)
+ .ok()
+ .filter(|p| p.format() == 32)
+ .and_then(|p| p.value::<u32>().first().copied())
+ };
+ result.push(Surface {
+ id: window.resource_id(),
+ client,
+ pid: cardinal("_NET_WM_PID").unwrap_or(0),
+ app,
+ title,
+ rect: Rect {
+ x: i64::from(position.dst_x()),
+ y: i64::from(position.dst_y()),
+ width: u32::from(geometry.width()),
+ height: u32::from(geometry.height()),
+ },
+ opacity: cardinal("_NET_WM_WINDOW_OPACITY").unwrap_or(u32::MAX),
+ });
+ }
+ Ok(result)
+}
+
+fn read_pixmap(
+ conn: &xcb::Connection,
+ pixmap: x::Pixmap,
+ width: u32,
+ height: u32,
+) -> anyhow::Result<image::RgbaImage> {
+ ensure!(
+ width > 0 && height > 0 && width <= 16384 && height <= 16384,
+ "invalid surface dimensions"
+ );
+ let reply = conn.wait_for_reply(conn.send_request(&x::GetImage {
+ format: x::ImageFormat::ZPixmap,
+ drawable: x::Drawable::Pixmap(pixmap),
+ x: 0,
+ y: 0,
+ width: width as u16,
+ height: height as u16,
+ plane_mask: u32::MAX,
+ }))?;
+ let setup = conn.get_setup();
+ let format = setup
+ .pixmap_formats()
+ .iter()
+ .find(|f| f.depth() == reply.depth())
+ .ok_or_else(|| anyhow!("missing surface pixel format"))?;
+ ensure!(
+ format.bits_per_pixel() == 32
+ && format.scanline_pad() == 32
+ && setup.image_byte_order() == x::ImageOrder::LsbFirst
+ && matches!(reply.depth(), 24 | 32),
+ "unsupported surface pixel format"
+ );
+ let data = reply.data();
+ ensure!(
+ data.len() == width as usize * height as usize * 4,
+ "invalid surface pixel storage"
+ );
+ let mut image = image::RgbaImage::new(width, height);
+ for (s, d) in data.chunks_exact(4).zip(image.pixels_mut()) {
+ *d = image::Rgba([
+ s[2],
+ s[1],
+ s[0],
+ if reply.depth() == 32 { s[3] } else { 255 },
+ ]);
+ }
+ Ok(image)
+}
+
+fn background_id(conn: &xcb::Connection, root: x::Window) -> Option<u32> {
+ ["_XROOTPMAP_ID", "ESETROOT_PMAP_ID"]
+ .into_iter()
+ .find_map(|name| {
+ property(conn, root, name)
+ .ok()
+ .filter(|p| p.format() == 32 && p.r#type() == x::ATOM_PIXMAP)
+ .and_then(|p| p.value::<u32>().first().copied())
+ .filter(|id| *id != 0)
+ })
+}
+
fn capture(
monitor: &crate::monitor::MonitorData,
filters: &WindowFilters,
@@ -45,47 +240,133 @@ fn capture(
let detector = sources
.ignore_incognito_windows
.then(|| screenpipe_a11y::incognito::create_detector(sources.enhanced_incognito_detection));
- let (conn, _) = xcb::Connection::connect(None)?;
+ let (conn, index) = xcb::Connection::connect(None)?;
conn.wait_for_reply(conn.send_request(&composite::QueryVersion {
client_major_version: 0,
client_minor_version: 4,
}))?;
- let mut windows = xcap::Window::all()?;
- // xcap orders front to back; draw back to front on an opaque black canvas.
- windows.reverse();
+ let setup = conn.get_setup();
+ let screen = setup
+ .roots()
+ .nth(index as usize)
+ .ok_or_else(|| anyhow!("missing X11 screen"))?;
+ let root = screen.root();
+ let overlay = conn
+ .wait_for_reply(conn.send_request(&composite::GetOverlayWindow { window: root }))?
+ .overlay_win();
+ conn.check_request(
+ conn.send_request_checked(&composite::ReleaseOverlayWindow { window: root }),
+ )?;
+ conn.check_request(conn.send_request_checked(&x::ChangeWindowAttributes {
+ window: root,
+ value_list: &[x::Cw::EventMask(
+ x::EventMask::SUBSTRUCTURE_NOTIFY | x::EventMask::PROPERTY_CHANGE,
+ )],
+ }))?;
+ let initial = snapshot(&conn, root, overlay)?;
+ for surface in &initial {
+ for id in [surface.id, surface.client] {
+ conn.check_request(conn.send_request_checked(&x::ChangeWindowAttributes {
+ window: x::Window::new(id),
+ value_list: &[x::Cw::EventMask(
+ x::EventMask::STRUCTURE_NOTIFY | x::EventMask::PROPERTY_CHANGE,
+ )],
+ }))?;
+ }
+ }
+ let before = snapshot(&conn, root, overlay)?;
+ ensure!(
+ initial == before,
+ "desktop scene changed during privacy setup"
+ );
+ let policy_atoms = [
+ "_NET_WM_NAME",
+ "WM_NAME",
+ "WM_CLASS",
+ "_NET_WM_PID",
+ "_NET_WM_WINDOW_OPACITY",
+ "_NET_WM_STATE",
+ "_XROOTPMAP_ID",
+ "ESETROOT_PMAP_ID",
+ ]
+ .iter()
+ .map(|name| atom(&conn, name))
+ .collect::<anyhow::Result<Vec<_>>>()?;
+ ensure!(
+ !scene_changed(&conn, &policy_atoms)?,
+ "desktop changed before privacy capture"
+ );
+ let wallpaper = background_id(&conn, root);
let mut canvas =
image::RgbaImage::from_pixel(monitor.width, monitor.height, image::Rgba([0, 0, 0, 255]));
- let mut captured = 0;
- for window in windows {
- let (Ok(id), Ok(app), Ok(title), Ok(false)) = (
- window.id(),
- window.app_name(),
- window.title(),
- window.is_minimized(),
- ) else {
- continue;
- };
- // _NET_WM_PID is optional in X11 (e.g. Tk does not publish it).
- let pid = window.pid().unwrap_or(0);
- if !window_allowed(filters, detector.as_deref(), &app, &title, pid as i32) {
+ // This is the wallpaper resource, never the root framebuffer containing
+ // other clients. A desktop manager's own window is also captured below.
+ if filters.is_valid("Desktop", "") {
+ if let Some(id) = wallpaper {
+ let pixmap = x::Pixmap::new(id);
+ let size = conn.wait_for_reply(conn.send_request(&x::GetGeometry {
+ drawable: x::Drawable::Pixmap(pixmap),
+ }))?;
+ let image = read_pixmap(
+ &conn,
+ pixmap,
+ u32::from(size.width()),
+ u32::from(size.height()),
+ )?;
+ // X11 tiles a root background pixmap; it may be only 1x1.
+ // Use root coordinates so adjacent/negative-origin monitors agree.
+ for (x, y, pixel) in canvas.enumerate_pixels_mut() {
+ let sx = (i64::from(x) + i64::from(monitor.x)).rem_euclid(i64::from(image.width()))
+ as u32;
+ let sy = (i64::from(y) + i64::from(monitor.y)).rem_euclid(i64::from(image.height()))
+ as u32;
+ *pixel = *image.get_pixel(sx, sy);
+ pixel[3] = 255;
+ }
+ }
+ }
+ let monitor_rect = Rect {
+ x: i64::from(monitor.x),
+ y: i64::from(monitor.y),
+ width: monitor.width,
+ height: monitor.height,
+ };
+ for surface in &before {
+ if !surface.rect.intersects(monitor_rect) {
continue;
}
- let (wx, wy, width, height) = (window.x()?, window.y()?, window.width()?, window.height()?);
- if i64::from(wx) + i64::from(width) <= i64::from(monitor.x)
- || i64::from(wy) + i64::from(height) <= i64::from(monitor.y)
- || i64::from(wx) >= i64::from(monitor.x) + i64::from(monitor.width)
- || i64::from(wy) >= i64::from(monitor.y) + i64::from(monitor.height)
- {
+ let dest = Rect {
+ x: surface.rect.x - i64::from(monitor.x),
+ y: surface.rect.y - i64::from(monitor.y),
+ ..surface.rect
+ };
+ if !window_allowed(
+ filters,
+ detector.as_deref(),
+ &surface.app,
+ &surface.title,
+ surface.pid as i32,
+ ) {
+ scene::paint_black(&mut canvas, dest);
continue;
}
+ let native = x::Window::new(surface.id);
+ let attrs =
+ conn.wait_for_reply(conn.send_request(&x::GetWindowAttributes { window: native }))?;
ensure!(
- width > 0 && height > 0 && width <= 16384 && height <= 16384,
- "invalid window dimensions"
+ setup
+ .roots()
+ .any(|s| s
+ .allowed_depths()
+ .any(
+ |d| d.visuals().iter().any(|v| v.visual_id() == attrs.visual()
+ && v.class() == x::VisualClass::TrueColor
+ && v.red_mask() == 0xff0000
+ && v.green_mask() == 0xff00
+ && v.blue_mask() == 0xff)
+ )),
+ "unsupported surface visual"
);
- let native = x::Window::new(id);
- // XGetImage(window) can contain undefined pixels where another window
- // obscures it. Read the window's own Composite backing pixmap instead.
- // Connection teardown releases our redirects on every success/error.
conn.check_request(conn.send_request_checked(&composite::RedirectWindow {
window: native,
update: composite::Redirect::Automatic,
@@ -95,78 +376,49 @@ fn capture(
window: native,
pixmap,
}))?;
- let reply = conn.wait_for_reply(conn.send_request(&x::GetImage {
- format: x::ImageFormat::ZPixmap,
- drawable: x::Drawable::Pixmap(pixmap),
- x: 0,
- y: 0,
- width: width as u16,
- height: height as u16,
- plane_mask: u32::MAX,
- }))?;
- conn.send_request(&x::FreePixmap { pixmap });
- let setup = conn.get_setup();
- let format = setup
- .pixmap_formats()
- .iter()
- .find(|f| f.depth() == reply.depth())
- .ok_or_else(|| anyhow!("missing window pixel format"))?;
- ensure!(
- format.bits_per_pixel() == 32
- && format.scanline_pad() == 32
- && setup.image_byte_order() == x::ImageOrder::LsbFirst
- && matches!(reply.depth(), 24 | 32),
- "unsupported window pixel format"
- );
- let attrs =
- conn.wait_for_reply(conn.send_request(&x::GetWindowAttributes { window: native }))?;
- let true_color = setup.roots().any(|s| {
- s.allowed_depths().any(|d| {
- d.visuals().iter().any(|v| {
- v.visual_id() == attrs.visual()
- && v.class() == x::VisualClass::TrueColor
- && v.red_mask() == 0xff0000
- && v.green_mask() == 0xff00
- && v.blue_mask() == 0xff
- })
- })
- });
- ensure!(true_color, "unsupported window visual");
- // A close/reuse/title/geometry transition invalidates the whole frame.
- ensure!(
- window.app_name()? == app
- && window.title()? == title
- && window.pid().unwrap_or(0) == pid
- && !window.is_minimized()?
- && (window.x()?, window.y()?, window.width()?, window.height()?)
- == (wx, wy, width, height),
- "allowed window changed during capture"
- );
- let data = reply.data();
- ensure!(
- data.len() == width as usize * height as usize * 4,
- "invalid window pixel storage"
- );
- for y in 0..height {
- let dest_y = i64::from(wy) - i64::from(monitor.y) + i64::from(y);
- if dest_y < 0 || dest_y >= i64::from(monitor.height) {
- continue;
- }
- for x in 0..width {
- let dest_x = i64::from(wx) - i64::from(monitor.x) + i64::from(x);
- if dest_x < 0 || dest_x >= i64::from(monitor.width) {
- continue;
+ let mut image = read_pixmap(&conn, pixmap, surface.rect.width, surface.rect.height)?;
+ conn.check_request(conn.send_request_checked(&x::FreePixmap { pixmap }))?;
+ if surface.opacity != u32::MAX {
+ for pixel in image.pixels_mut() {
+ for c in &mut pixel.0 {
+ *c = ((u64::from(*c) * u64::from(surface.opacity) + u64::from(u32::MAX) / 2)
+ / u64::from(u32::MAX)) as u8;
}
- let p = &data[(y as usize * width as usize + x as usize) * 4..][..4];
- canvas.put_pixel(
- dest_x as u32,
- dest_y as u32,
- image::Rgba([p[2], p[1], p[0], 255]),
- );
}
}
- captured += 1;
+ scene::overlay_premultiplied(&mut canvas, &image, dest.x, dest.y, None);
}
- ensure!(captured > 0, "no verified allowed windows on monitor");
+ // Includes blocked/unknown surfaces and stacking order. Never persist a
+ // partially reconstructed scene after close/reuse/move/resize/reorder.
+ ensure!(
+ snapshot(&conn, root, overlay)? == before && background_id(&conn, root) == wallpaper,
+ "desktop scene changed during privacy capture"
+ );
+ // Events retain transient title/geometry changes even if a window returns
+ // to the same identity before revalidation (an ABA transition).
+ ensure!(
+ !scene_changed(&conn, &policy_atoms)?,
+ "desktop changed during privacy capture"
+ );
Ok(DynamicImage::ImageRgba8(canvas))
}
+
+fn scene_changed(conn: &xcb::Connection, policy_atoms: &[x::Atom]) -> anyhow::Result<bool> {
+ let mut changed = false;
+ while let Some(event) = conn.poll_for_event()? {
+ if let xcb::Event::X(event) = event {
+ changed |= match event {
+ x::Event::PropertyNotify(e) => policy_atoms.contains(&e.atom()),
+ x::Event::CreateNotify(_)
+ | x::Event::DestroyNotify(_)
+ | x::Event::MapNotify(_)
+ | x::Event::UnmapNotify(_)
+ | x::Event::ConfigureNotify(_)
+ | x::Event::ReparentNotify(_)
+ | x::Event::CirculateNotify(_) => true,
+ _ => false,
+ };
+ }
+ }
+ Ok(changed)
+}
diff --git a/crates/screenpipe-screen/src/screenshot_privacy/macos.rs b/crates/screenpipe-screen/src/screenshot_privacy/macos.rs
index c88ceac44..e0d53954e 100644
--- a/crates/screenpipe-screen/src/screenshot_privacy/macos.rs
+++ b/crates/screenpipe-screen/src/screenshot_privacy/macos.rs
@@ -43,11 +43,232 @@ pub(crate) async fn capture_filtered(
.map_err(|_| anyhow!("window-filtered capture timed out"))??
}
+#[derive(Clone, Debug, PartialEq)]
+struct Surface {
+ id: u32,
+ pid: i32,
+ app: String,
+ title: String,
+ x: f64,
+ y: f64,
+ width: f64,
+ height: f64,
+ alpha: f64,
+ layer: i64,
+}
+
+/// WindowServer order is authoritative; SCShareableContent does not promise
+/// stacking order. Include desktop and shell surfaces, not just layer-zero apps.
+fn snapshot(display: cidre::cg::Rect) -> anyhow::Result<Vec<Surface>> {
+ window_list(
+ display,
+ core_graphics::window::kCGWindowListOptionOnScreenOnly,
+ )
+}
+
+fn window_list(display: cidre::cg::Rect, options: u32) -> anyhow::Result<Vec<Surface>> {
+ use core_foundation::{
+ base::{CFType, TCFType},
+ dictionary::CFDictionary,
+ number::CFNumber,
+ string::CFString,
+ };
+ use core_graphics::window::{copy_window_info, kCGNullWindowID};
+ let list = copy_window_info(options, kCGNullWindowID)
+ .ok_or_else(|| anyhow!("window scene unavailable"))?;
+ let mut result = Vec::new();
+ for ptr in list.iter() {
+ let dict: CFDictionary<CFString, CFType> =
+ unsafe { CFDictionary::wrap_under_get_rule(*ptr as _) };
+ let number = |key: &str| -> Option<f64> {
+ dict.find(CFString::new(key))?
+ .downcast::<CFNumber>()?
+ .to_f64()
+ };
+ let text = |key: &str| -> String {
+ dict.find(CFString::new(key))
+ .and_then(|v| v.downcast::<CFString>())
+ .map(|s| s.to_string())
+ .unwrap_or_default()
+ };
+ let bounds = dict
+ .find(CFString::new("kCGWindowBounds"))
+ .and_then(|v| v.downcast::<CFDictionary>())
+ .map(|d| unsafe {
+ CFDictionary::<CFString, CFType>::wrap_under_get_rule(d.as_concrete_TypeRef())
+ })
+ .ok_or_else(|| anyhow!("window geometry unavailable"))?;
+ let bound = |key: &str| -> anyhow::Result<f64> {
+ bounds
+ .find(CFString::new(key))
+ .and_then(|v| v.downcast::<CFNumber>())
+ .and_then(|n| n.to_f64())
+ .ok_or_else(|| anyhow!("invalid window geometry"))
+ };
+ let (x, y, width, height) = (bound("X")?, bound("Y")?, bound("Width")?, bound("Height")?);
+ let alpha = number("kCGWindowAlpha").unwrap_or(1.0);
+ let layer = number("kCGWindowLayer").unwrap_or(0.0) as i64;
+ ensure!(
+ [x, y, width, height, alpha].iter().all(|v| v.is_finite()),
+ "non-finite window geometry"
+ );
+ if width <= 0.0 || height <= 0.0 || alpha <= 0.0 || layer == 2147483630 {
+ continue;
+ }
+ if options != core_graphics::window::kCGWindowListOptionAll
+ && (x + width <= display.origin.x
+ || y + height <= display.origin.y
+ || x >= display.origin.x + display.size.width
+ || y >= display.origin.y + display.size.height)
+ {
+ continue;
+ }
+ result.push(Surface {
+ id: number("kCGWindowNumber").ok_or_else(|| anyhow!("window id unavailable"))? as u32,
+ pid: number("kCGWindowOwnerPID").unwrap_or(0.0) as i32,
+ app: text("kCGWindowOwnerName"),
+ title: text("kCGWindowName"),
+ x,
+ y,
+ width,
+ height,
+ alpha,
+ layer,
+ });
+ }
+ result.reverse(); // bottom-to-top
+ Ok(result)
+}
+
+/// Dock thumbnails can reproduce excluded windows. If native tile inspection
+/// fails, omit Dock coverage instead of trusting its mirrored app pixels.
+fn dock_masks(
+ dock_pid: Option<i32>,
+ display: cidre::cg::Rect,
+ filters: &WindowFilters,
+ detector: Option<&dyn screenpipe_a11y::incognito::IncognitoDetector>,
+) -> Option<Vec<Surface>> {
+ use cidre::{ax, cf};
+ let Some(pid) = dock_pid else {
+ return Some(Vec::new());
+ };
+ if !ax::is_process_trusted() {
+ return None;
+ }
+ let app = ax::UiElement::with_app_pid(pid);
+ app.set_messaging_timeout_secs(0.1).ok()?;
+ let groups = app.children().ok()?;
+ let windows = window_list(display, core_graphics::window::kCGWindowListOptionAll).ok()?;
+ let text = |element: &ax::UiElement, attr: &ax::Attr| -> Option<String> {
+ let value = element.attr_value(attr).ok()?;
+ if value.get_type_id() != cf::String::type_id() {
+ return None;
+ }
+ let value: &cf::String = unsafe { std::mem::transmute(&*value) };
+ Some(value.to_string())
+ };
+ let mut masks = Vec::new();
+ let mut seen = 0;
+ for group in groups.iter() {
+ group.set_messaging_timeout_secs(0.1).ok()?;
+ let children = group.children().ok()?;
+ for tile in children.iter() {
+ seen += 1;
+ if seen > 512 {
+ return None;
+ }
+ tile.set_messaging_timeout_secs(0.1).ok()?;
+ if text(tile, ax::attr::subrole()).as_deref() != Some("AXMinimizedWindowDockItem") {
+ continue;
+ }
+ let title = text(tile, ax::attr::title())?;
+ let matches: Vec<_> = windows.iter().filter(|w| w.title == title).collect();
+ if !matches.is_empty()
+ && matches
+ .iter()
+ .all(|w| window_allowed(filters, detector, &w.app, &w.title, w.pid))
+ {
+ continue;
+ }
+ let pos = tile.pos().ok()?.cg_point()?;
+ let size = tile.size().ok()?.cg_size()?;
+ if ![pos.x, pos.y, size.width, size.height]
+ .iter()
+ .all(|v| v.is_finite())
+ || size.width <= 0.0
+ || size.height <= 0.0
+ {
+ return None;
+ }
+ masks.push(Surface {
+ id: 0,
+ pid,
+ app: "Dock".into(),
+ title,
+ x: pos.x,
+ y: pos.y,
+ width: size.width,
+ height: size.height,
+ alpha: 1.0,
+ layer: 20,
+ });
+ }
+ }
+ Some(masks)
+}
+
+fn output_rect(
+ surface: &Surface,
+ display: cidre::cg::Rect,
+ width: u32,
+ height: u32,
+) -> super::scene::Rect {
+ let sx = f64::from(width) / display.size.width;
+ let sy = f64::from(height) / display.size.height;
+ let x = ((surface.x - display.origin.x) * sx).floor();
+ let y = ((surface.y - display.origin.y) * sy).floor();
+ super::scene::Rect {
+ x: x as i64,
+ y: y as i64,
+ width: (((surface.x + surface.width - display.origin.x) * sx).ceil() - x).max(0.0) as u32,
+ height: (((surface.y + surface.height - display.origin.y) * sy).ceil() - y).max(0.0) as u32,
+ }
+}
+
+async fn take(
+ filter: &sc::ContentFilter,
+ width: usize,
+ height: usize,
+ window: bool,
+) -> anyhow::Result<image::RgbaImage> {
+ let mut cfg = sc::StreamCfg::new();
+ cfg.set_width(width);
+ cfg.set_height(height);
+ cfg.set_pixel_format(cv::PixelFormat::_32_BGRA);
+ cfg.set_shows_cursor(false);
+ cfg.set_scales_to_fit(true);
+ if window {
+ cfg.set_ignore_shadows_single_window(true);
+ }
+ if api::macos_available("15.0") {
+ cfg.set_show_mouse_clicks(false);
+ }
+ let sample = sc::ScreenshotManager::capture_sample_buf(filter, &cfg)
+ .await
+ .map_err(|e| anyhow!("window-filtered screenshot: {e:?}"))?;
+ let mut pixels = sample
+ .image_buf()
+ .ok_or_else(|| anyhow!("missing screenshot pixels"))?
+ .retained();
+ Ok(rgba(&mut pixels)?.into_rgba8())
+}
+
async fn capture(
id: u32,
filters: &WindowFilters,
sources: ExclusionSources,
) -> anyhow::Result<DynamicImage> {
+ use super::scene::{self, Rect};
let detector = sources
.ignore_incognito_windows
.then(|| screenpipe_a11y::incognito::create_detector(sources.enhanced_incognito_detection));
@@ -59,81 +280,149 @@ async fn capture(
.iter()
.find(|d| d.display_id().0 == id)
.ok_or_else(|| anyhow!("filtered display disappeared"))?;
- let mut selected = Vec::new();
- let mut identities = Vec::new();
- for window in content.windows().iter() {
- if !window.is_on_screen() || window.window_layer() != 0 {
- continue;
- }
- let Some(app) = window.owning_app() else {
- continue;
- };
- let Some(title) = window.title() else {
- continue;
- };
- let name = app.app_name().to_string();
- let title = title.to_string();
- if window_allowed(
- filters,
- detector.as_deref(),
- &name,
- &title,
- app.process_id(),
- ) {
- identities.push((window.id(), app.process_id(), name, title));
- selected.push(window.retained());
- }
- }
- ensure!(!selected.is_empty(), "no verified allowed windows");
+ let frame = display.frame();
+ let before = snapshot(frame)?;
+ let available = content.windows();
+ let native = |surface: &Surface| {
+ available.iter().find(|w| {
+ w.id() == surface.id
+ && w.is_on_screen()
+ && w.owning_app()
+ .is_some_and(|a| a.process_id() == surface.pid)
+ })
+ };
+ let dock_pid = before.iter().filter(|s| s.layer == 20).find_map(|s| {
+ native(s)
+ .filter(|w| {
+ w.owning_app()
+ .is_some_and(|a| a.bundle_id().to_string() == "com.apple.dock")
+ })
+ .map(|_| s.pid)
+ });
+ let dock_before = dock_masks(dock_pid, frame, filters, detector.as_deref());
+ let allowed: Vec<bool> = before
+ .iter()
+ .map(|s| {
+ native(s).is_some()
+ && !(Some(s.pid) == dock_pid && s.layer == 20 && dock_before.is_none())
+ && window_allowed(filters, detector.as_deref(), &s.app, &s.title, s.pid)
+ })
+ .collect();
+ let selected: Vec<_> = before
+ .iter()
+ .zip(&allowed)
+ .filter_map(|(s, ok)| {
+ if *ok {
+ native(s).map(|w| w.retained())
+ } else {
+ None
+ }
+ })
+ .collect();
let windows = ns::Array::from_slice_retained(&selected);
- let filter = sc::ContentFilter::with_display_including_windows(display, &windows);
- let mut cfg = sc::StreamCfg::new();
- let native_width = display.width() as usize;
- let native_height = display.height() as usize;
+ let mut filter = sc::ContentFilter::with_display_including_windows(display, &windows);
+ // SCDisplay dimensions are logical points on Retina displays. Preserve the
+ // native pixel density before applying the user's explicit resolution cap.
+ let scale = filter.point_pixel_scale() as f64;
+ ensure!(
+ scale.is_finite() && scale > 0.0,
+ "invalid display pixel scale"
+ );
+ let nw = (frame.size.width * scale).round() as usize;
+ let nh = (frame.size.height * scale).round() as usize;
+ ensure!(
+ nw > 0 && nh > 0 && nw <= 16384 && nh <= 16384,
+ "invalid display dimensions"
+ );
let cap = crate::monitor::sck_capture_max_width() as usize;
- let width = if cap == 0 {
- native_width
+ let width = if cap == 0 { nw } else { nw.min(cap) };
+ let height = (nh * width / nw).max(1);
+ let mut canvas = if selected.is_empty() {
+ image::RgbaImage::from_pixel(width as u32, height as u32, image::Rgba([0, 0, 0, 255]))
} else {
- native_width.min(cap)
+ if api::macos_available("14.2") {
+ filter.set_include_menu_bar(before.iter().zip(&allowed).any(|(s, ok)| {
+ *ok && s.layer == 24 && s.app == "Window Server" && s.title == "Menubar"
+ }));
+ }
+ let pixels = take(&filter, width, height, false).await?;
+ let mut canvas =
+ image::RgbaImage::from_pixel(width as u32, height as u32, image::Rgba([0, 0, 0, 255]));
+ scene::overlay_premultiplied(&mut canvas, &pixels, 0, 0, None);
+ canvas
+ };
+ let bounds = Rect {
+ x: 0,
+ y: 0,
+ width: width as u32,
+ height: height as u32,
};
+ let mut redacted = Vec::<Rect>::new();
+ for (surface, ok) in before.iter().zip(&allowed) {
+ let rect = output_rect(surface, frame, width as u32, height as u32);
+ if !*ok {
+ // The system Dock owns a display-sized transparent overlay. Its
+ // rectangle is not its visible coverage. Use only native alpha as
+ // a mask; none of this excluded surface's RGB enters the output.
+ let dock = native(surface).filter(|w| {
+ surface.layer == 20
+ && w.owning_app()
+ .is_some_and(|a| a.bundle_id().to_string() == "com.apple.dock")
+ });
+ if let Some(dock) = dock {
+ let filter = sc::ContentFilter::with_desktop_independent_window(dock);
+ let coverage =
+ take(&filter, rect.width as usize, rect.height as usize, true).await?;
+ scene::paint_black_coverage(&mut canvas, &coverage, rect.x, rect.y);
+ } else {
+ scene::paint_black(&mut canvas, rect);
+ }
+ if let Some(clipped) = rect.intersection(bounds) {
+ scene::add_region(&mut redacted, clipped);
+ }
+ } else if redacted.iter().any(|r| rect.intersects(*r)) {
+ // Restore only the affected part of an allowed foreground surface.
+ // Capturing its native pixels preserves translucent corners without
+ // revealing excluded pixels or repainting unrelated foreground UI.
+ let window = native(surface).ok_or_else(|| anyhow!("allowed window disappeared"))?;
+ let filter = sc::ContentFilter::with_desktop_independent_window(window);
+ let pixels = take(&filter, rect.width as usize, rect.height as usize, true).await?;
+ for region in &redacted {
+ scene::overlay_premultiplied(&mut canvas, &pixels, rect.x, rect.y, Some(*region));
+ }
+ }
+ }
+ if let Some(masks) = &dock_before {
+ for mask in masks {
+ scene::paint_black(
+ &mut canvas,
+ output_rect(mask, frame, width as u32, height as u32),
+ );
+ }
+ }
ensure!(
- native_width > 0 && native_height > 0,
- "invalid display dimensions"
+ dock_masks(dock_pid, frame, filters, detector.as_deref()) == dock_before,
+ "Dock thumbnail policy changed during capture"
+ );
+ ensure!(
+ snapshot(frame)? == before,
+ "desktop scene changed during privacy capture"
);
- cfg.set_width(width);
- cfg.set_height((native_height * width / native_width).max(1));
- cfg.set_pixel_format(cv::PixelFormat::_32_BGRA);
- cfg.set_shows_cursor(false);
- // Match the existing captureMaxWidth contract: scale the entire display,
- // rather than cropping its right/bottom edges at smaller output sizes.
- cfg.set_scales_to_fit(true);
- if api::macos_available("15.0") {
- cfg.set_show_mouse_clicks(false);
- }
- let sample = sc::ScreenshotManager::capture_sample_buf(&filter, &cfg)
- .await
- .map_err(|e| anyhow!("window-filtered screenshot: {e:?}"))?;
- // Do not accept a frame across window reuse/title changes. A newly created
- // unrelated window is absent from the positive filter even during animation.
let after = sc::ShareableContent::current()
.await
.map_err(|e| anyhow!("window revalidation: {e:?}"))?;
let after_windows = after.windows();
- for (id, pid, name, title) in identities {
- let same = after_windows.iter().any(|w| {
- w.id() == id
- && w.is_on_screen()
- && w.title().is_some_and(|t| t.to_string() == title)
- && w.owning_app()
- .is_some_and(|a| a.process_id() == pid && a.app_name().to_string() == name)
- });
- ensure!(same, "allowed window changed during capture");
+ for (s, ok) in before.iter().zip(&allowed) {
+ if *ok {
+ ensure!(
+ after_windows.iter().any(|w| w.id() == s.id
+ && w.is_on_screen()
+ && w.owning_app().is_some_and(|a| a.process_id() == s.pid)),
+ "allowed window changed during capture"
+ );
+ }
}
- let mut pixels = sample
- .image_buf()
- .ok_or_else(|| anyhow!("missing screenshot pixels"))?
- .retained();
- rgba(&mut pixels)
+ Ok(DynamicImage::ImageRgba8(canvas))
}
fn rgba(buf: &mut cv::ImageBuf) -> anyhow::Result<DynamicImage> {
@@ -172,7 +461,7 @@ fn rgba(buf: &mut cv::ImageBuf) -> anyhow::Result<DynamicImage> {
.chunks_exact(4)
.zip(target.chunks_exact_mut(4))
{
- t.copy_from_slice(&[s[2], s[1], s[0], 255]);
+ t.copy_from_slice(&[s[2], s[1], s[0], s[3]]);
}
}
Ok(DynamicImage::ImageRgba8(image))
diff --git a/crates/screenpipe-screen/src/screenshot_privacy/scene.rs b/crates/screenpipe-screen/src/screenshot_privacy/scene.rs
new file mode 100644
index 000000000..e6640caa9
--- /dev/null
+++ b/crates/screenpipe-screen/src/screenshot_privacy/scene.rs
@@ -0,0 +1,246 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+//! Pixel operations for native, positively selected window surfaces. Rectangles
+//! use output pixels; no operation reads an unfiltered monitor image.
+use image::{Rgba, RgbaImage};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(super) struct Rect {
+ pub x: i64,
+ pub y: i64,
+ pub width: u32,
+ pub height: u32,
+}
+
+impl Rect {
+ pub fn intersection(self, other: Self) -> Option<Self> {
+ let x = self.x.max(other.x);
+ let y = self.y.max(other.y);
+ let right = (self.x + i64::from(self.width)).min(other.x + i64::from(other.width));
+ let bottom = (self.y + i64::from(self.height)).min(other.y + i64::from(other.height));
+ (right > x && bottom > y).then_some(Self {
+ x,
+ y,
+ width: (right - x).max(0) as u32,
+ height: (bottom - y).max(0) as u32,
+ })
+ }
+
+ pub fn intersects(self, other: Self) -> bool {
+ self.intersection(other).is_some()
+ }
+}
+
+pub(super) fn paint_black(image: &mut RgbaImage, rect: Rect) {
+ let Some(rect) = rect.intersection(Rect {
+ x: 0,
+ y: 0,
+ width: image.width(),
+ height: image.height(),
+ }) else {
+ return;
+ };
+ for y in rect.y as u32..rect.y as u32 + rect.height {
+ for x in rect.x as u32..rect.x as u32 + rect.width {
+ image.put_pixel(x, y, Rgba([0, 0, 0, 255]));
+ }
+ }
+}
+
+/// Use only coverage from a native shell surface; never copy excluded RGB.
+#[cfg(any(target_os = "macos", test))]
+pub(super) fn paint_black_coverage(canvas: &mut RgbaImage, mask: &RgbaImage, x: i64, y: i64) {
+ for (sx, sy, pixel) in mask.enumerate_pixels() {
+ let dx = x + i64::from(sx);
+ let dy = y + i64::from(sy);
+ if pixel[3] > 0
+ && dx >= 0
+ && dy >= 0
+ && dx < i64::from(canvas.width())
+ && dy < i64::from(canvas.height())
+ {
+ canvas.put_pixel(dx as u32, dy as u32, Rgba([0, 0, 0, 255]));
+ }
+ }
+}
+
+/// Blend premultiplied native RGBA onto an opaque canvas. An optional region
+/// limits restoration to a redacted area, leaving other foreground pixels alone.
+pub(super) fn overlay_premultiplied(
+ canvas: &mut RgbaImage,
+ source: &RgbaImage,
+ x: i64,
+ y: i64,
+ region: Option<Rect>,
+) {
+ let source_rect = Rect {
+ x,
+ y,
+ width: source.width(),
+ height: source.height(),
+ };
+ let bounds = Rect {
+ x: 0,
+ y: 0,
+ width: canvas.width(),
+ height: canvas.height(),
+ };
+ let Some(mut clip) = source_rect.intersection(bounds) else {
+ return;
+ };
+ if let Some(region) = region {
+ let Some(intersection) = clip.intersection(region) else {
+ return;
+ };
+ clip = intersection;
+ }
+ for dy in clip.y..clip.y + i64::from(clip.height) {
+ for dx in clip.x..clip.x + i64::from(clip.width) {
+ let s = source.get_pixel((dx - x) as u32, (dy - y) as u32);
+ if s[3] == 0 {
+ continue;
+ } // unused RGB bytes are not content
+ let d = canvas.get_pixel_mut(dx as u32, dy as u32);
+ let inverse = 255 - u16::from(s[3]);
+ for c in 0..3 {
+ d[c] = (u16::from(s[c]) + (u16::from(d[c]) * inverse + 127) / 255).min(255) as u8;
+ }
+ d[3] = 255;
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ fn rect(x: i64, y: i64, w: u32, h: u32) -> Rect {
+ Rect {
+ x,
+ y,
+ width: w,
+ height: h,
+ }
+ }
+ #[test]
+ fn excluded_shell_coverage_preserves_transparent_desktop_area() {
+ let mut canvas = RgbaImage::from_pixel(3, 1, Rgba([0, 255, 255, 255]));
+ let mask =
+ RgbaImage::from_raw(3, 1, vec![255, 0, 255, 0, 255, 0, 255, 1, 255, 0, 255, 255])
+ .unwrap();
+ paint_black_coverage(&mut canvas, &mask, 0, 0);
+ assert_eq!(canvas.get_pixel(0, 0).0, [0, 255, 255, 255]);
+ assert_eq!(canvas.get_pixel(1, 0).0, [0, 0, 0, 255]);
+ assert_eq!(canvas.get_pixel(2, 0).0, [0, 0, 0, 255]);
+ }
+ #[test]
+ fn overlapping_exclusions_restore_transparency_once_per_pixel() {
+ let mut regions = Vec::new();
+ add_region(&mut regions, rect(0, 0, 3, 3));
+ add_region(&mut regions, rect(1, 1, 3, 3));
+ add_region(&mut regions, rect(0, 0, 3, 3));
+ for (i, a) in regions.iter().enumerate() {
+ for b in regions.iter().skip(i + 1) {
+ assert!(!a.intersects(*b));
+ }
+ }
+ let mut canvas = RgbaImage::from_pixel(4, 4, Rgba([0, 0, 0, 255]));
+ let source = RgbaImage::from_pixel(4, 4, Rgba([64, 0, 0, 128]));
+ for r in regions {
+ overlay_premultiplied(&mut canvas, &source, 0, 0, Some(r));
+ }
+ assert_eq!(canvas.get_pixel(1, 1).0, [64, 0, 0, 255]);
+ assert_eq!(canvas.get_pixel(3, 0).0, [0, 0, 0, 255]);
+ assert_eq!(canvas.pixels().filter(|p| p[0] == 64).count(), 14);
+ }
+
+ #[test]
+ fn excluded_middle_layer_keeps_allowed_foreground_and_hides_background() {
+ let mut canvas = RgbaImage::from_pixel(12, 8, Rgba([0, 200, 0, 255]));
+ paint_black(&mut canvas, rect(3, 2, 6, 4));
+ let front = RgbaImage::from_pixel(3, 3, Rgba([200, 0, 0, 255]));
+ overlay_premultiplied(&mut canvas, &front, 7, 3, None);
+ assert_eq!(canvas.get_pixel(2, 3).0, [0, 200, 0, 255]);
+ assert_eq!(canvas.get_pixel(4, 3).0, [0, 0, 0, 255]);
+ assert_eq!(canvas.get_pixel(8, 3).0, [200, 0, 0, 255]);
+ // An excluded window raised above that foreground must erase it.
+ paint_black(&mut canvas, rect(3, 2, 6, 4));
+ assert_eq!(canvas.get_pixel(8, 3).0, [0, 0, 0, 255]);
+ }
+ #[test]
+ fn transparent_foreground_reveals_black_redaction_not_hidden_background() {
+ let mut canvas = RgbaImage::from_pixel(2, 1, Rgba([0, 0, 0, 255]));
+ let source = RgbaImage::from_raw(2, 1, vec![64, 32, 16, 128, 255, 255, 255, 0]).unwrap();
+ overlay_premultiplied(&mut canvas, &source, 0, 0, None);
+ assert_eq!(canvas.get_pixel(0, 0).0, [64, 32, 16, 255]);
+ assert_eq!(canvas.get_pixel(1, 0).0, [0, 0, 0, 255]);
+ }
+ #[test]
+ fn negative_origin_and_spanning_window_clip_to_each_display() {
+ let mut left = RgbaImage::from_pixel(4, 3, Rgba([255; 4]));
+ paint_black(&mut left, rect(-2, -1, 4, 3));
+ assert_eq!(left.get_pixel(0, 0).0, [0, 0, 0, 255]);
+ assert_eq!(left.get_pixel(2, 0).0, [255; 4]);
+ assert_eq!(left.get_pixel(0, 2).0, [255; 4]);
+ assert!(!rect(0, 0, 4, 3).intersects(rect(4, 0, 2, 2)));
+ assert!(rect(0, 0, 4, 3).intersects(rect(3, 0, 2, 2)));
+ }
+ #[test]
+ fn restore_only_redacted_intersection_does_not_overpaint_other_windows() {
+ let mut canvas = RgbaImage::from_pixel(4, 4, Rgba([0, 0, 200, 255]));
+ let region = rect(0, 0, 2, 2);
+ paint_black(&mut canvas, region);
+ let source = RgbaImage::from_pixel(4, 4, Rgba([200, 0, 0, 255]));
+ overlay_premultiplied(&mut canvas, &source, 0, 0, Some(region));
+ assert_eq!(canvas.get_pixel(1, 1).0, [200, 0, 0, 255]);
+ assert_eq!(canvas.get_pixel(2, 1).0, [0, 0, 200, 255]);
+ }
+}
+
+/// Keep an exact disjoint union so translucent foreground pixels are never
+/// blended twice where two excluded windows overlap.
+#[cfg(any(target_os = "macos", test))]
+pub(super) fn add_region(regions: &mut Vec<Rect>, rect: Rect) {
+ let mut pending = vec![rect];
+ for existing in regions.iter() {
+ let mut next = Vec::new();
+ for r in pending {
+ let Some(i) = r.intersection(*existing) else {
+ next.push(r);
+ continue;
+ };
+ for part in [
+ Rect {
+ x: r.x,
+ y: r.y,
+ width: r.width,
+ height: (i.y - r.y) as u32,
+ },
+ Rect {
+ x: r.x,
+ y: i.y + i64::from(i.height),
+ width: r.width,
+ height: (r.y + i64::from(r.height) - i.y - i64::from(i.height)) as u32,
+ },
+ Rect {
+ x: r.x,
+ y: i.y,
+ width: (i.x - r.x) as u32,
+ height: i.height,
+ },
+ Rect {
+ x: i.x + i64::from(i.width),
+ y: i.y,
+ width: (r.x + i64::from(r.width) - i.x - i64::from(i.width)) as u32,
+ height: i.height,
+ },
+ ] {
+ if part.width > 0 && part.height > 0 {
+ next.push(part);
+ }
+ }
+ }
+ pending = next;
+ }
+ regions.extend(pending);
+}
diff --git a/crates/screenpipe-screen/src/screenshot_privacy/windows.rs b/crates/screenpipe-screen/src/screenshot_privacy/windows.rs
index de6ffacb0..279035f5a 100644
--- a/crates/screenpipe-screen/src/screenshot_privacy/windows.rs
+++ b/crates/screenpipe-screen/src/screenshot_privacy/windows.rs
@@ -7,46 +7,158 @@ use crate::{
wgc_capture::PersistentCapture,
};
use anyhow::{anyhow, bail, Context, Result};
-use image::{imageops, DynamicImage, Rgba, RgbaImage};
+use image::{DynamicImage, Rgba, RgbaImage};
use once_cell::sync::Lazy;
use screenpipe_a11y::incognito::create_detector;
-use std::{sync::Arc, time::Duration};
+use std::{collections::HashMap, sync::Arc, time::Duration};
use tokio::sync::Semaphore;
+use windows::Win32::{
+ Foundation::{GetLastError, SetLastError, BOOL, COLORREF, ERROR_SUCCESS, HWND, LPARAM, RECT},
+ Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_CLOAKED, DWMWA_EXTENDED_FRAME_BOUNDS},
+ UI::WindowsAndMessaging::{
+ EnumWindows, GetClassNameW, GetLayeredWindowAttributes, GetShellWindow, GetWindowLongW,
+ GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId, IsIconic, IsWindowVisible,
+ GWL_EXSTYLE, LAYERED_WINDOW_ATTRIBUTES_FLAGS, LWA_ALPHA, LWA_COLORKEY, WS_EX_LAYERED,
+ },
+};
use xcap::Window;
#[derive(Debug, Clone, PartialEq, Eq)]
-struct WindowMetadata {
- id: u32,
+struct PolicyMetadata {
pid: u32,
app_name: String,
title: String,
+}
+#[derive(Debug, Clone, PartialEq, Eq)]
+enum SurfacePolicy {
+ Allowed,
+ Blocked,
+ Unknown,
+}
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum SurfaceOpacity {
+ Ordinary,
+ PerPixel,
+ Constant {
+ alpha: u8,
+ color_key: Option<[u8; 3]>,
+ },
+}
+
+impl SurfaceOpacity {
+ fn alpha(self) -> u8 {
+ match self {
+ Self::Constant { alpha, .. } => alpha,
+ _ => 255,
+ }
+ }
+
+ fn color_key(self) -> Option<[u8; 3]> {
+ match self {
+ Self::Constant { color_key, .. } => color_key,
+ _ => None,
+ }
+ }
+}
+
+fn surface_opacity(hwnd: HWND) -> Result<SurfaceOpacity> {
+ unsafe { SetLastError(ERROR_SUCCESS) };
+ let style = unsafe { GetWindowLongW(hwnd, GWL_EXSTYLE) } as u32;
+ if style == 0 && unsafe { GetLastError() } != ERROR_SUCCESS {
+ bail!("unable to read native window style");
+ }
+ if style & WS_EX_LAYERED.0 == 0 {
+ return Ok(SurfaceOpacity::Ordinary);
+ }
+ let mut alpha = 255;
+ let mut key = COLORREF(0);
+ let mut flags = LAYERED_WINDOW_ATTRIBUTES_FLAGS(0);
+ if unsafe {
+ GetLayeredWindowAttributes(hwnd, Some(&mut key), Some(&mut alpha), Some(&mut flags))
+ }
+ .is_err()
+ {
+ return Ok(SurfaceOpacity::PerPixel);
+ }
+ let color_key = (flags.0 & LWA_COLORKEY.0 != 0).then_some([
+ key.0 as u8,
+ (key.0 >> 8) as u8,
+ (key.0 >> 16) as u8,
+ ]);
+ Ok(SurfaceOpacity::Constant {
+ alpha: if flags.0 & LWA_ALPHA.0 != 0 {
+ alpha
+ } else {
+ 255
+ },
+ color_key,
+ })
+}
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct NativeWindow {
+ id: u32,
+ pid: u32,
+ class_name: String,
+ title: String,
x: i32,
y: i32,
width: u32,
height: u32,
+ policy: SurfacePolicy,
+ opacity: SurfaceOpacity,
}
-fn metadata(window: &Window) -> Result<WindowMetadata> {
- let value = WindowMetadata {
- id: window.id().context("window id")?,
- pid: window.pid().context("window pid")?,
- app_name: window.app_name().context("window app name")?,
- title: window.title().context("window title")?,
- x: window.x().context("window x")?,
- y: window.y().context("window y")?,
- width: window.width().context("window width")?,
- height: window.height().context("window height")?,
- };
- if value.pid == 0 || value.app_name.trim().is_empty() || value.title.trim().is_empty() {
- bail!("window {} has incomplete privacy metadata", value.id);
+fn wide_text(mut read: impl FnMut(&mut [u16]) -> i32, capacity: usize) -> Result<String> {
+ let mut buffer = vec![0; capacity.max(1)];
+ unsafe { SetLastError(ERROR_SUCCESS) };
+ let length = read(&mut buffer);
+ if length == 0 && unsafe { GetLastError() } != ERROR_SUCCESS {
+ bail!("native window text read failed");
}
- if value.width == 0 || value.height == 0 || value.width > 16384 || value.height > 16384 {
- bail!("window {} has invalid dimensions", value.id);
+ Ok(String::from_utf16_lossy(
+ &buffer[..(length.max(0) as usize).min(buffer.len())],
+ ))
+}
+
+unsafe extern "system" fn enum_window(hwnd: HWND, lparam: LPARAM) -> BOOL {
+ unsafe { (&mut *(lparam.0 as *mut Vec<HWND>)).push(hwnd) };
+ true.into()
+}
+
+fn enumerate_hwnds() -> Result<Vec<HWND>> {
+ let mut ids = Vec::new();
+ unsafe {
+ EnumWindows(
+ Some(enum_window),
+ LPARAM(&mut ids as *mut Vec<HWND> as isize),
+ )
+ }
+ .ok()
+ .context("EnumWindows")?;
+ Ok(ids)
+}
+
+fn policy_metadata() -> Result<HashMap<u32, PolicyMetadata>> {
+ let mut result = HashMap::new();
+ for window in Window::all().context("enumerate policy windows")? {
+ let Ok(id) = window.id() else { continue };
+ let value = (|| {
+ Ok::<_, anyhow::Error>(PolicyMetadata {
+ pid: window.pid().context("window pid")?,
+ app_name: window.app_name().context("window app name")?,
+ title: window.title().context("window title")?,
+ })
+ })();
+ if let Ok(value) = value {
+ if value.pid != 0 && !value.app_name.trim().is_empty() {
+ result.insert(id, value);
+ }
+ }
}
- Ok(value)
+ Ok(result)
}
-fn overlaps_monitor(window: &WindowMetadata, monitor: &MonitorData) -> bool {
+fn overlaps_monitor(window: &NativeWindow, monitor: &MonitorData) -> bool {
let right = i64::from(window.x) + i64::from(window.width);
let bottom = i64::from(window.y) + i64::from(window.height);
let monitor_right = i64::from(monitor.x) + i64::from(monitor.width);
@@ -57,77 +169,272 @@ fn overlaps_monitor(window: &WindowMetadata, monitor: &MonitorData) -> bool {
&& bottom > i64::from(monitor.y)
}
-fn capture_filtered_sync(
+fn snapshot(
monitor: &MonitorData,
filters: &WindowFilters,
sources: ExclusionSources,
-) -> Result<DynamicImage> {
- let mut windows = Window::all().context("enumerate visible windows")?;
+) -> Result<Vec<NativeWindow>> {
+ let metadata = policy_metadata()?;
let detector = sources
.ignore_incognito_windows
.then(|| create_detector(sources.enhanced_incognito_detection));
- let mut output = RgbaImage::from_pixel(monitor.width, monitor.height, Rgba([0, 0, 0, 255]));
-
- // xcap enumerates top-to-bottom. Paint bottom-to-top so allowed overlap is faithful.
- windows.reverse();
- let mut captured = 0;
- for window in windows {
- // Unknown windows are never selected. An error on an already selected
- // window below invalidates the entire frame.
- if !matches!(window.is_minimized(), Ok(false)) {
+ let shell = unsafe { GetShellWindow() };
+ let mut shell_pid = 0;
+ unsafe { GetWindowThreadProcessId(shell, Some(&mut shell_pid)) };
+ let mut result = Vec::new();
+ // EnumWindows is top-to-bottom. Snapshot order is security-relevant.
+ for hwnd in enumerate_hwnds()? {
+ if !unsafe { IsWindowVisible(hwnd) }.as_bool() || unsafe { IsIconic(hwnd) }.as_bool() {
continue;
}
- let Ok(before) = metadata(&window) else {
+ let mut cloaked = 0u32;
+ unsafe {
+ DwmGetWindowAttribute(
+ hwnd,
+ DWMWA_CLOAKED,
+ &mut cloaked as *mut u32 as *mut _,
+ std::mem::size_of::<u32>() as u32,
+ )
+ }
+ .context("DwmGetWindowAttribute(DWMWA_CLOAKED)")?;
+ if cloaked != 0 {
+ continue;
+ }
+ let mut rect = RECT::default();
+ if unsafe {
+ DwmGetWindowAttribute(
+ hwnd,
+ DWMWA_EXTENDED_FRAME_BOUNDS,
+ &mut rect as *mut RECT as *mut _,
+ std::mem::size_of::<RECT>() as u32,
+ )
+ }
+ .is_err()
+ {
+ // An uncertain visible surface must never be reconstructed from guessed
+ // geometry. Withhold the complete frame instead.
+ bail!(
+ "unable to resolve DWM frame bounds for window {}",
+ hwnd.0 as usize
+ );
+ }
+ let width = rect.right.saturating_sub(rect.left) as u32;
+ let height = rect.bottom.saturating_sub(rect.top) as u32;
+ if width == 0 || height == 0 {
continue;
+ }
+ if !overlaps_monitor(
+ &NativeWindow {
+ id: 0,
+ pid: 0,
+ class_name: String::new(),
+ title: String::new(),
+ x: rect.left,
+ y: rect.top,
+ width,
+ height,
+ policy: SurfacePolicy::Unknown,
+ opacity: SurfaceOpacity::Ordinary,
+ },
+ monitor,
+ ) {
+ continue;
+ }
+ if width > 32768 || height > 32768 {
+ bail!("oversized visible window intersects monitor");
+ }
+ let mut pid = 0;
+ unsafe { GetWindowThreadProcessId(hwnd, Some(&mut pid)) };
+ let class_name = wide_text(|buffer| unsafe { GetClassNameW(hwnd, buffer) }, 256)?;
+ if class_name.is_empty() {
+ bail!("native window class is unavailable");
+ }
+ unsafe { SetLastError(ERROR_SUCCESS) };
+ let title_length = unsafe { GetWindowTextLengthW(hwnd) };
+ if title_length == 0 && unsafe { GetLastError() } != ERROR_SUCCESS {
+ bail!("native window title length is unavailable");
+ }
+ let title_capacity = title_length.max(0) as usize + 1;
+ let title = wide_text(
+ |buffer| unsafe { GetWindowTextW(hwnd, buffer) },
+ title_capacity,
+ )?;
+ let opacity = surface_opacity(hwnd)?;
+ let id = hwnd.0 as usize as u32;
+ // Shell-owned switchers and previews can contain live pixels copied from a
+ // different source window. Their own PID/title is not authoritative for a
+ // tile, so never trust them as ordinary Explorer/XAML surfaces.
+ let unverified_preview = matches!(
+ class_name.as_str(),
+ "TaskListThumbnailWnd"
+ | "MultitaskingViewFrame"
+ | "XamlExplorerHostIslandWindow"
+ | "ForegroundStaging"
+ | "Shell_DimLayer"
+ ) || class_name.contains("Thumbnail");
+ let desktop_surface =
+ pid != 0 && pid == shell_pid && matches!(class_name.as_str(), "Progman" | "WorkerW");
+ let (policy, geometry) = if unverified_preview {
+ (SurfacePolicy::Blocked, (rect.left, rect.top, width, height))
+ } else if desktop_surface {
+ let allowed = super::window_allowed(
+ filters,
+ detector.as_deref(),
+ "Windows Explorer",
+ if title.is_empty() {
+ "Program Manager"
+ } else {
+ &title
+ },
+ pid as i32,
+ );
+ (
+ if allowed {
+ SurfacePolicy::Allowed
+ } else {
+ SurfacePolicy::Blocked
+ },
+ (rect.left, rect.top, width, height),
+ )
+ } else if let Some(meta) = metadata
+ .get(&id)
+ .filter(|meta| meta.pid == pid && meta.title == title)
+ {
+ if super::window_allowed(
+ filters,
+ detector.as_deref(),
+ &meta.app_name,
+ &meta.title,
+ meta.pid as i32,
+ ) {
+ (SurfacePolicy::Allowed, (rect.left, rect.top, width, height))
+ } else {
+ // Redact the conservative native region for blocked surfaces.
+ (SurfacePolicy::Blocked, (rect.left, rect.top, width, height))
+ }
+ } else {
+ (SurfacePolicy::Unknown, (rect.left, rect.top, width, height))
};
- if !overlaps_monitor(&before, monitor) {
+ let window = NativeWindow {
+ id,
+ pid,
+ class_name,
+ title,
+ x: geometry.0,
+ y: geometry.1,
+ width: geometry.2,
+ height: geometry.3,
+ policy,
+ opacity,
+ };
+ if overlaps_monitor(&window, monitor) {
+ result.push(window);
+ }
+ }
+ Ok(result)
+}
+
+fn paint_black(output: &mut RgbaImage, window: &NativeWindow, monitor: &MonitorData) {
+ let left = (i64::from(window.x) - i64::from(monitor.x)).max(0) as u32;
+ let top = (i64::from(window.y) - i64::from(monitor.y)).max(0) as u32;
+ let right = (i64::from(window.x) + i64::from(window.width) - i64::from(monitor.x))
+ .clamp(0, i64::from(monitor.width)) as u32;
+ let bottom = (i64::from(window.y) + i64::from(window.height) - i64::from(monitor.y))
+ .clamp(0, i64::from(monitor.height)) as u32;
+ for row in output
+ .rows_mut()
+ .skip(top as usize)
+ .take(bottom.saturating_sub(top) as usize)
+ {
+ for pixel in row
+ .skip(left as usize)
+ .take(right.saturating_sub(left) as usize)
+ {
+ *pixel = Rgba([0, 0, 0, 255]);
+ }
+ }
+}
+
+/// WGC B8G8R8A8 surfaces are premultiplied. `imageops::overlay` expects straight
+/// alpha and would multiply RGB a second time, darkening translucent windows.
+fn overlay_wgc_premultiplied(
+ output: &mut RgbaImage,
+ foreground: &RgbaImage,
+ x: i64,
+ y: i64,
+ opacity: SurfaceOpacity,
+) {
+ for (sx, sy, source) in foreground.enumerate_pixels() {
+ let dx = x + i64::from(sx);
+ let dy = y + i64::from(sy);
+ if dx < 0 || dy < 0 || dx >= i64::from(output.width()) || dy >= i64::from(output.height()) {
continue;
}
- let allowed = super::window_allowed(
- filters,
- detector.as_deref(),
- &before.app_name,
- &before.title,
- before.pid as i32,
- );
- if !allowed {
+ if opacity.color_key().is_some_and(|key| source.0[..3] == key) {
+ continue;
+ }
+ let global_alpha = u32::from(opacity.alpha());
+ let alpha = (u32::from(source[3]) * global_alpha + 127) / 255;
+ if alpha == 0 {
continue;
}
+ let background = output.get_pixel_mut(dx as u32, dy as u32);
+ let inverse = 255 - alpha;
+ for channel in 0..3 {
+ background[channel] = ((u32::from(source[channel]) * global_alpha + 127) / 255
+ + (u32::from(background[channel]) * inverse + 127) / 255)
+ .min(255) as u8;
+ }
+ background[3] = 255;
+ }
+}
- // Use Screenpipe's native window capture on every architecture. This
- // never falls back to monitor pixels and supports software D3D devices.
- let capture = PersistentCapture::new_window(before.id)?;
- // The session was just created for this capture. Its primed first frame
- // is fresh even if it arrived before this read, and static windows need
- // not produce a second compositor update to become capturable.
- let image = capture
- .get_latest_image_streaming(Duration::from_secs(2))?
- .into_rgba8();
- drop(capture);
- if image.width() != before.width || image.height() != before.height {
- bail!("window {} capture dimensions changed", before.id);
- }
- // Preserve WGC alpha while compositing onto an opaque black canvas;
- // transparent pixels must not expose their unused RGB storage.
- let after = metadata(&window)?;
- if after != before || window.is_minimized()? {
- bail!("window {} changed during privacy capture", before.id);
- }
- imageops::overlay(
- &mut output,
- &image,
- i64::from(before.x) - i64::from(monitor.x),
- i64::from(before.y) - i64::from(monitor.y),
- );
- captured += 1;
+fn capture_filtered_sync(
+ monitor: &MonitorData,
+ filters: &WindowFilters,
+ sources: ExclusionSources,
+) -> Result<DynamicImage> {
+ let before = snapshot(monitor, filters, sources)?;
+ let mut output = RgbaImage::from_pixel(monitor.width, monitor.height, Rgba([0, 0, 0, 255]));
+ for window in before.iter().rev() {
+ match window.policy {
+ SurfacePolicy::Blocked | SurfacePolicy::Unknown => {
+ paint_black(&mut output, window, monitor)
+ }
+ SurfacePolicy::Allowed => {
+ let capture = PersistentCapture::new_privacy_window(window.id, window.pid)?;
+ let image = capture
+ .get_latest_image_streaming(Duration::from_secs(2))?
+ .into_rgba8();
+ drop(capture);
+ if (image.width(), image.height()) != (window.width, window.height) {
+ bail!(
+ "window {} ({}) capture geometry {}x{} != native {}x{}",
+ window.id,
+ window.class_name,
+ image.width(),
+ image.height(),
+ window.width,
+ window.height
+ );
+ }
+ // Blend alpha only onto pixels already proven safe; unused RGB cannot leak.
+ overlay_wgc_premultiplied(
+ &mut output,
+ &image,
+ i64::from(window.x) - i64::from(monitor.x),
+ i64::from(window.y) - i64::from(monitor.y),
+ window.opacity,
+ );
+ }
+ }
}
- if captured == 0 {
- bail!("no verified allowed windows on monitor");
+ if snapshot(monitor, filters, sources)? != before {
+ bail!("window scene changed during privacy capture");
}
Ok(DynamicImage::ImageRgba8(output))
}
-// A timed-out native call retains its permit so it cannot accumulate workers.
static CAPTURE_SLOT: Lazy<Arc<Semaphore>> = Lazy::new(|| Arc::new(Semaphore::new(1)));
pub(crate) async fn capture_filtered(
@@ -154,18 +461,19 @@ pub(crate) async fn capture_filtered(
#[cfg(test)]
mod tests {
use super::*;
-
#[test]
fn overlap_math_rejects_adjacent_windows() {
- let window = WindowMetadata {
+ let window = NativeWindow {
id: 1,
pid: 1,
- app_name: "test".into(),
+ class_name: "test".into(),
title: "test".into(),
x: 100,
y: 0,
width: 10,
height: 10,
+ policy: SurfacePolicy::Allowed,
+ opacity: SurfaceOpacity::Ordinary,
};
let monitor = MonitorData {
name: "test".into(),
@@ -177,4 +485,76 @@ mod tests {
};
assert!(!overlaps_monitor(&window, &monitor));
}
+
+ #[test]
+ fn premultiplied_alpha_is_not_multiplied_twice() {
+ let mut output = RgbaImage::from_pixel(1, 1, Rgba([0, 0, 0, 255]));
+ let foreground = RgbaImage::from_pixel(1, 1, Rgba([100, 50, 25, 128]));
+ overlay_wgc_premultiplied(&mut output, &foreground, 0, 0, SurfaceOpacity::Ordinary);
+ assert_eq!(output.get_pixel(0, 0), &Rgba([100, 50, 25, 255]));
+ }
+
+ #[test]
+ fn alpha_zero_ignores_rgb_payload() {
+ let mut output = RgbaImage::from_pixel(1, 1, Rgba([7, 8, 9, 255]));
+ let foreground = RgbaImage::from_pixel(1, 1, Rgba([255, 1, 200, 0]));
+ overlay_wgc_premultiplied(&mut output, &foreground, 0, 0, SurfaceOpacity::Ordinary);
+ assert_eq!(output.get_pixel(0, 0), &Rgba([7, 8, 9, 255]));
+ }
+
+ #[test]
+ fn global_opacity_preserves_half_cyan_over_black() {
+ let mut output = RgbaImage::from_pixel(1, 1, Rgba([0, 0, 0, 255]));
+ let foreground = RgbaImage::from_pixel(1, 1, Rgba([0, 255, 255, 255]));
+ overlay_wgc_premultiplied(
+ &mut output,
+ &foreground,
+ 0,
+ 0,
+ SurfaceOpacity::Constant {
+ alpha: 128,
+ color_key: None,
+ },
+ );
+ assert_eq!(output.get_pixel(0, 0), &Rgba([0, 128, 128, 255]));
+ }
+
+ #[test]
+ fn global_opacity_scales_premultiplied_rgb_once() {
+ let mut output = RgbaImage::from_pixel(1, 1, Rgba([0, 0, 0, 255]));
+ let foreground = RgbaImage::from_pixel(1, 1, Rgba([100, 50, 25, 128]));
+ overlay_wgc_premultiplied(
+ &mut output,
+ &foreground,
+ 0,
+ 0,
+ SurfaceOpacity::Constant {
+ alpha: 128,
+ color_key: None,
+ },
+ );
+ assert_eq!(output.get_pixel(0, 0), &Rgba([50, 25, 13, 255]));
+ }
+
+ #[test]
+ #[ignore = "requires a live unlocked Windows console"]
+ fn filtered_capture_keeps_full_monitor_dimensions() {
+ let monitor = xcap::Monitor::all()
+ .expect("enumerate monitors")
+ .into_iter()
+ .next()
+ .expect("monitor");
+ let info = MonitorData {
+ name: monitor.name().expect("monitor name"),
+ width: monitor.width().expect("monitor width"),
+ height: monitor.height().expect("monitor height"),
+ x: monitor.x().expect("monitor x"),
+ y: monitor.y().expect("monitor y"),
+ is_primary: monitor.is_primary().expect("monitor primary"),
+ };
+ let filters = WindowFilters::new(&["__screenpipe_no_match__".into()], &[], &[]);
+ let image = capture_filtered_sync(&info, &filters, ExclusionSources::default())
+ .expect("filtered capture");
+ assert_eq!((image.width(), image.height()), (info.width, info.height));
+ }
}
diff --git a/crates/screenpipe-screen/src/wgc_capture.rs b/crates/screenpipe-screen/src/wgc_capture.rs
index 4e9f2a813..27f974428 100644
--- a/crates/screenpipe-screen/src/wgc_capture.rs
+++ b/crates/screenpipe-screen/src/wgc_capture.rs
@@ -33,8 +33,9 @@
use anyhow::{anyhow, Result};
use image::{DynamicImage, RgbaImage};
+use std::collections::VecDeque;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
-use std::sync::{Arc, Condvar, Mutex, MutexGuard};
+use std::sync::{Arc, Condvar, Mutex, MutexGuard, OnceLock};
use std::time::{Duration, Instant};
use windows::{
core::{factory, IInspectable, Interface},
@@ -373,6 +374,68 @@ struct LatestFrame {
demand: CopyDemand,
}
+// CreateForWindow can retain an ALPC port after the returned item is released.
+// Cache only a bounded set of keyed native targets, never sessions or pixels.
+struct PrivacyWindowItem {
+ window_id: u32,
+ pid: u32,
+ item: GraphicsCaptureItem,
+ closed: Arc<AtomicBool>,
+ closed_token: EventRegistrationToken,
+}
+
+impl Drop for PrivacyWindowItem {
+ fn drop(&mut self) {
+ let _ = self.item.RemoveClosed(self.closed_token);
+ }
+}
+
+fn privacy_window_items() -> &'static Mutex<VecDeque<PrivacyWindowItem>> {
+ static ITEMS: OnceLock<Mutex<VecDeque<PrivacyWindowItem>>> = OnceLock::new();
+ ITEMS.get_or_init(|| Mutex::new(VecDeque::new()))
+}
+
+fn privacy_window_item(window_id: u32, pid: u32) -> Result<GraphicsCaptureItem> {
+ let mut items = privacy_window_items()
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ items.retain(|entry| {
+ !entry.closed.load(Ordering::Acquire) && (entry.window_id != window_id || entry.pid == pid)
+ });
+ if let Some(index) = items
+ .iter()
+ .position(|entry| entry.window_id == window_id && entry.pid == pid)
+ {
+ let entry = items.remove(index).expect("located item");
+ let item = entry.item.clone();
+ items.push_back(entry);
+ return Ok(item);
+ }
+ let interop = factory::<GraphicsCaptureItem, IGraphicsCaptureItemInterop>()?;
+ let item: GraphicsCaptureItem =
+ unsafe { interop.CreateForWindow(HWND(window_id as usize as *mut std::ffi::c_void))? };
+ let closed = Arc::new(AtomicBool::new(false));
+ let flag = closed.clone();
+ let closed_token = item.Closed(
+ &TypedEventHandler::<GraphicsCaptureItem, IInspectable>::new(move |_, _| {
+ flag.store(true, Ordering::Release);
+ Ok(())
+ }),
+ )?;
+ const MAX_PRIVACY_ITEMS: usize = 8;
+ if items.len() == MAX_PRIVACY_ITEMS {
+ items.pop_front();
+ }
+ items.push_back(PrivacyWindowItem {
+ window_id,
+ pid,
+ item: item.clone(),
+ closed,
+ closed_token,
+ });
+ Ok(item)
+}
+
/// Persistent WGC capture session that keeps a single GraphicsCaptureSession alive,
/// eliminating the orange border flash caused by per-frame session create/destroy.
pub struct PersistentCapture {
@@ -411,17 +474,18 @@ impl PersistentCapture {
Self::from_item(item, false)
}
- /// Capture only this window's compositor surface, on every Windows architecture.
- /// Native item dimensions avoid applying process DPI scaling to physical pixels.
- pub(crate) fn new_window(window_id: u32) -> Result<Self> {
- let interop = factory::<GraphicsCaptureItem, IGraphicsCaptureItemInterop>()
- .map_err(|e| anyhow!("failed to get GraphicsCaptureItem factory: {}", e))?;
- let item: GraphicsCaptureItem = unsafe {
- interop
- .CreateForWindow(HWND(window_id as usize as *mut std::ffi::c_void))
- .map_err(|e| anyhow!("CreateForWindow failed: {}", e))?
- };
- Self::from_item(item, true)
+ /// Capture only this window's compositor surface. Reuse only the keyed native
+ /// target; every call creates a new session and never retains pixels.
+ pub(crate) fn new_privacy_window(window_id: u32, pid: u32) -> Result<Self> {
+ let item = privacy_window_item(window_id, pid)?;
+ let result = Self::from_item(item, true);
+ if result.is_err() {
+ privacy_window_items()
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ .retain(|entry| entry.window_id != window_id || entry.pid != pid);
+ }
+ result
}
fn from_item(item: GraphicsCaptureItem, capture_initial_frame: bool) -> Result<Self> {
diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md
index b2401d212..70423487e 100644
--- a/docs/COVERAGE.md
+++ b/docs/COVERAGE.md
@@ -37,16 +37,16 @@ results and `cargo llvm-cov` data on top when judging release confidence.
### Core Engine
- Mapped suites: 37
-- Mapped Rust files: 404
-- Active test blocks: 3936
-- Ignored/manual test blocks: 188
-- Weighted coverage points: 3243.0
+- Mapped Rust files: 405
+- Active test blocks: 3949
+- Ignored/manual test blocks: 189
+- Weighted coverage points: 3255.1
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 33 | 3773 | 170 | 3170.0 | 21 | 11 | 100% |
-| macos | 32 | 3816 | 149 | 3165.3 | 22 | 11 | 100% |
-| linux | 29 | 3351 | 145 | 2788.3 | 20 | 11 | 100% |
+| windows | 33 | 3786 | 171 | 3182.1 | 21 | 11 | 100% |
+| macos | 32 | 3829 | 150 | 3177.4 | 22 | 11 | 100% |
+| linux | 29 | 3364 | 146 | 2800.4 | 20 | 11 | 100% |
## Refresh
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 1553160bd..270a2bb1e 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -8,11 +8,11 @@ confidence, and criticality.
- Manifest: `docs/coverage/core-engine-map.json`
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs
- Mapped suites: 37
-- Mapped Rust files: 404
-- Active test blocks: 3936
-- Ignored/manual test blocks: 188
-- Declared test blocks: 4124
-- Weighted coverage points: 3243.0
+- Mapped Rust files: 405
+- Active test blocks: 3949
+- Ignored/manual test blocks: 189
+- Declared test blocks: 4138
+- Weighted coverage points: 3255.1
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,19 +23,19 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 33 | 3773 | 170 | 3170.0 | 21 | 11 | 100% |
-| macos | 32 | 3816 | 149 | 3165.3 | 22 | 11 | 100% |
-| linux | 29 | 3351 | 145 | 2788.3 | 20 | 11 | 100% |
+| windows | 33 | 3786 | 171 | 3182.1 | 21 | 11 | 100% |
+| macos | 32 | 3829 | 150 | 3177.4 | 22 | 11 | 100% |
+| linux | 29 | 3364 | 146 | 2800.4 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| screenpipe-engine | 12 | 22 | 131 | 1876 | 53 | 1443.9 | 10 |
+| screenpipe-engine | 12 | 22 | 131 | 1879 | 53 | 1446.0 | 10 |
| screenpipe-db | 5 | 69 | 29 | 640 | 32 | 609.2 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
| screenpipe-audio | 8 | 29 | 54 | 706 | 56 | 617.0 | 5 |
-| screenpipe-screen | 6 | 9 | 21 | 283 | 9 | 251.4 | 4 |
+| screenpipe-screen | 6 | 9 | 22 | 293 | 10 | 261.4 | 4 |
| screenpipe-a11y | 4 | 2 | 34 | 401 | 38 | 291.5 | 3 |
| screenpipe-fs | 1 | 0 | 1 | 3 | 0 | 3.0 | 3 |
@@ -73,17 +73,17 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| meeting | 6 suites / 1762 active / 32 ignored / 1440.7 pts | 6 suites / 1762 active / 32 ignored / 1440.7 pts | 4 suites / 1408 active / 28 ignored / 1110.4 pts |
| ocr | 4 suites / 141 active / 7 ignored / 130.8 pts | 4 suites / 134 active / 7 ignored / 128.6 pts | 3 suites / 125 active / 6 ignored / 119.6 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1717 active / 86 ignored / 1516.2 pts | 14 suites / 1842 active / 94 ignored / 1566.2 pts | 13 suites / 1717 active / 86 ignored / 1516.2 pts |
+| performance | 13 suites / 1730 active / 87 ignored / 1528.3 pts | 14 suites / 1855 active / 95 ignored / 1578.3 pts | 13 suites / 1730 active / 87 ignored / 1528.3 pts |
| pipes | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts |
-| privacy | 5 suites / 1021 active / 42 ignored / 823.9 pts | 5 suites / 1077 active / 21 ignored / 825.6 pts | 5 suites / 979 active / 14 ignored / 788.8 pts |
+| privacy | 5 suites / 1031 active / 43 ignored / 833.9 pts | 5 suites / 1087 active / 22 ignored / 835.6 pts | 5 suites / 989 active / 15 ignored / 798.8 pts |
| real-app | - | 1 suites / 125 active / 8 ignored / 50.0 pts | - |
| speaker | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts |
-| storage | 4 suites / 705 active / 41 ignored / 588.0 pts | 4 suites / 705 active / 41 ignored / 588.0 pts | 4 suites / 705 active / 41 ignored / 588.0 pts |
+| storage | 4 suites / 708 active / 41 ignored / 590.1 pts | 4 suites / 708 active / 41 ignored / 590.1 pts | 4 suites / 708 active / 41 ignored / 590.1 pts |
| sync | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts |
-| timeline | 4 suites / 1258 active / 48 ignored / 1034.5 pts | 4 suites / 1258 active / 48 ignored / 1034.5 pts | 4 suites / 1258 active / 48 ignored / 1034.5 pts |
+| timeline | 4 suites / 1271 active / 49 ignored / 1046.6 pts | 4 suites / 1271 active / 49 ignored / 1046.6 pts | 4 suites / 1271 active / 49 ignored / 1046.6 pts |
| transcription | 7 suites / 901 active / 53 ignored / 704.3 pts | 6 suites / 895 active / 53 ignored / 700.1 pts | 6 suites / 895 active / 53 ignored / 700.1 pts |
| ui-events | 4 suites / 831 active / 34 ignored / 629.8 pts | 3 suites / 762 active / 5 ignored / 581.5 pts | 3 suites / 762 active / 5 ignored / 581.5 pts |
-| vision-capture | 6 suites / 584 active / 32 ignored / 460.3 pts | 6 suites / 577 active / 32 ignored / 458.1 pts | 5 suites / 568 active / 31 ignored / 449.1 pts |
+| vision-capture | 6 suites / 597 active / 33 ignored / 472.4 pts | 6 suites / 590 active / 33 ignored / 470.2 pts | 5 suites / 581 active / 32 ignored / 461.2 pts |
## Critical Flow Matrix
@@ -137,7 +137,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 14 | 107 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 40 | 315 | 15 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 46 | 438 | 7 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
-| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 307 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
+| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 310 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 13 | 125 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |
| engine-db-recovery-cli | screenpipe-engine | windows, macos, linux | database, engine-lifecycle | engine-health-lifecycle, performance-liveness | high | strong | unit | 1 | 33 | 0 | Exact DB/WAL/SHM working-copy preservation, rollback on archive failure, and restart repair for crashes during the multi-file generation swap. |
| engine-focus-os | screenpipe-engine | windows, macos | engine-lifecycle, os-integration | engine-health-lifecycle, performance-liveness | medium | conditional | unit | 3 | 6 | 0 | Platform focus-tracker parsing/helpers. These files are cfg-gated and only execute on their target OS. The separately invoked harness=false tests/native_multi_monitor.rs acceptance is documented in scripts/native-test-fixtures/macos/README.md and is not counted as an automatic Rust test block. |
@@ -150,7 +150,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| engine-workflows | screenpipe-engine | windows, macos, linux | local-api, storage | local-api-search | high | partial | mixed | 7 | 40 | 0 | Catalog identity and persistence, revision-scoped feedback, workflow evidence and pipeline contracts, and denial of agent-granted rollout access. Does not assert live cloud discovery quality. Workflow publication orchestration deadlines and nested query backpressure are covered with virtual-time router tests. |
| filesystem-durable-sync | screenpipe-fs | macos, linux, windows | database, vision-capture, audio | capture-ocr-pipeline, audio-record-transcribe, engine-health-lifecycle | high | strong | unit | 1 | 3 | 0 | Durable flushes preserve local full-sync behavior, fall back to fsync only for macOS ENOTSUP, and propagate real I/O or fallback failures. Real SMB migration and continued recording are covered by the marked-volume integration test. |
| screen-capture-ocr-contract | screenpipe-screen | windows, macos, linux | vision-capture, ocr | capture-ocr-pipeline | high | partial | unit | 3 | 18 | 0 | Cross-platform cached-OCR unit coverage for RawCaptureResult to CaptureResult metadata, browser URL, focus state, window-to-screen OCR coordinate transformation, Tesseract output parsing, and contour-based text-region detection for the meeting OCR gate. |
-| screen-capture-windowing | screenpipe-screen | windows, macos, linux | vision-capture, timeline, performance, privacy | capture-ocr-pipeline, timeline-streaming, privacy-and-redaction, performance-liveness | high | strong | mixed | 16 | 198 | 0 | Window filtering, empty-window regressions, retry policy, URL timing, monitor cache, OCR cache, snapshots, and image comparison. |
+| screen-capture-windowing | screenpipe-screen | windows, macos, linux | vision-capture, timeline, performance, privacy | capture-ocr-pipeline, timeline-streaming, privacy-and-redaction, performance-liveness | high | strong | mixed | 17 | 208 | 1 | Window filtering, empty-window regressions, retry policy, URL timing, monitor cache, OCR cache, snapshots, and image comparison. |
| screen-custom-ocr | screenpipe-screen | windows, macos, linux | ocr | capture-ocr-pipeline | medium | conditional | manual | 1 | 0 | 2 | Custom OCR tests are ignored by default and only contribute when explicitly run. |
| screen-macos-ocr | screenpipe-screen | macos | ocr, vision-capture | capture-ocr-pipeline | high | strong | mixed | 2 | 9 | 1 | Apple Vision OCR source/unit coverage and fixture OCR assertions. |
| screen-monitor-platform | screenpipe-screen | windows, macos, linux | vision-capture | capture-ocr-pipeline | medium | partial | unit | 5 | 42 | 5 | Per-OS monitor enumeration (Windows, macOS, Wayland/portal on Linux) and the persistent Windows.Graphics.Capture session. Each file is cfg-gated and only executes on its target OS. |
@@ -409,7 +409,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| engine-telemetry-observability | screenpipe-engine | src/crash_log.rs | source | 4 | 0 | 4 |
| engine-retention-storage | screenpipe-engine | src/disk_pressure.rs | source | 11 | 0 | 11 |
| engine-meeting-privacy-sync | screenpipe-engine | src/drm_detector.rs | source | 19 | 2 | 21 |
-| engine-capture-timeline | screenpipe-engine | src/event_driven_capture.rs | source | 98 | 0 | 98 |
+| engine-capture-timeline | screenpipe-engine | src/event_driven_capture.rs | source | 101 | 0 | 101 |
| engine-meeting-privacy-sync | screenpipe-engine | src/external_memory_sync.rs | source | 13 | 0 | 13 |
| engine-capture-timeline | screenpipe-engine | src/focus_aware_controller.rs | source | 15 | 0 | 15 |
| engine-focus-os | screenpipe-engine | src/focus_tracker/darwin.rs | source | 3 | 0 | 3 |
@@ -547,7 +547,8 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| screen-capture-windowing | screenpipe-screen | src/ocr_cache.rs | source | 15 | 0 | 15 |
| screen-windows-ocr | screenpipe-screen | src/ocr_worker.rs | source | 3 | 0 | 3 |
| screen-capture-windowing | screenpipe-screen | src/screenshot_privacy.rs | source | 4 | 0 | 4 |
-| screen-capture-windowing | screenpipe-screen | src/screenshot_privacy/windows.rs | source | 1 | 0 | 1 |
+| screen-capture-windowing | screenpipe-screen | src/screenshot_privacy/scene.rs | source | 6 | 0 | 6 |
+| screen-capture-windowing | screenpipe-screen | src/screenshot_privacy/windows.rs | source | 5 | 1 | 6 |
| screen-capture-windowing | screenpipe-screen | src/snapshot_writer.rs | source | 5 | 0 | 5 |
| screen-capture-ocr-contract | screenpipe-screen | src/tesseract.rs | source | 6 | 0 | 6 |
| screen-capture-ocr-contract | screenpipe-screen | src/text_regions.rs | source | 11 | 0 | 11 |
diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.json
index 09153dcd2..bb809936c 100644
--- a/docs/coverage/core-engine-map.json
+++ b/docs/coverage/core-engine-map.json
@@ -1158,6 +1158,7 @@
"src/ocr_cache.rs",
"src/snapshot_writer.rs",
"src/screenshot_privacy.rs",
+ "src/screenshot_privacy/scene.rs",
"src/screenshot_privacy/windows.rs",
"src/utils.rs",
"tests/capture_error_test.rs",