YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
Add opt-in focused-window screenshot scope
Louis Beaumontb55306196de1
diff --git a/apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx b/apps/screenpipe-app-tauri/lib/hooks/use-settings.tsxindex 8bba573e9..a16758e05 100644--- a/apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx+++ b/apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx@@ -822,6 +822,7 @@ let DEFAULT_SETTINGS: Settings = {starSessionShortcut: "Control+Super+B",disableVision: false,disableScreenshots: false,+ focusedWindowScreenshots: false,enableSemanticContext: true,remoteControlPreferences: {...NEW_INSTALL_REMOTE_CONTROL_PREFERENCES,diff --git a/apps/screenpipe-app-tauri/lib/utils/tauri.ts b/apps/screenpipe-app-tauri/lib/utils/tauri.tsindex 183d5a26b..d7ee5b5b5 100644--- a/apps/screenpipe-app-tauri/lib/utils/tauri.ts+++ b/apps/screenpipe-app-tauri/lib/utils/tauri.ts@@ -3935,6 +3935,10 @@ disableVision: boolean;* UI events are enough and screen pixels are too expensive or sensitive.*/disableScreenshots?: boolean;+/**+ * Limit screenshot pixels to the focused window. Off preserves desktop context.+ */+focusedWindowScreenshots?: boolean;/*** Build normalized semantic context from captured accessibility trees.* Experimental and opt-in. False preserves the historical capture pathdiff --git a/apps/screenpipe-app-tauri/lib/utils/validation.ts b/apps/screenpipe-app-tauri/lib/utils/validation.tsindex f64894178..d1f846aee 100644--- a/apps/screenpipe-app-tauri/lib/utils/validation.ts+++ b/apps/screenpipe-app-tauri/lib/utils/validation.ts@@ -120,6 +120,7 @@ export const settingsStoreSchema = z.object({includedUrls: z.array(domainRuleSchema).optional(),disableVision: z.boolean(),disableScreenshots: z.boolean().optional(),+ focusedWindowScreenshots: z.boolean().optional(),enableSemanticContext: z.boolean().optional(),remoteControlPreferences: remoteControlPreferencesSchema.optional(),remoteControlPolicy: remoteControlPolicySchema.optional(),diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/managed-settings.json b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/managed-settings.jsonindex a9be2cacc..cf21c9c3b 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/managed-settings.json+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/managed-settings.json@@ -15,6 +15,7 @@{"policyKey": "ignoredMeetingApps", "deviceKey": "ignoredMeetingApps", "apply": "engine", "kind": "string-array", "defaultValue": []},{"policyKey": "disableVision", "deviceKey": "disableVision", "apply": "engine", "kind": "boolean", "defaultValue": false},{"policyKey": "disableScreenshots", "deviceKey": "disableScreenshots", "apply": "engine", "kind": "boolean", "defaultValue": false},+ {"policyKey": "focusedWindowScreenshots", "deviceKey": "focusedWindowScreenshots", "apply": "engine", "kind": "boolean", "defaultValue": false},{"policyKey": "semanticContextMode", "deviceKey": "semanticContextMode", "apply": "engine", "kind": "enum", "values": ["memory", "computerUse", "both"], "defaultValue": "memory"},{"policyKey": "disableTimeline", "deviceKey": "disableTimeline", "apply": "engine", "kind": "boolean", "defaultValue": false},{"policyKey": "useAllMonitors", "deviceKey": "useAllMonitors", "apply": "engine", "kind": "boolean", "defaultValue": true},diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/managed_settings.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/managed_settings.rsindex 9c4d1b876..a04c8508d 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/managed_settings.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/managed_settings.rs@@ -465,7 +465,8 @@ mod tests {"disableAudio": "true", "disableVision": "true","ignoredWindows": ["Private"], "audioChunkDuration": 60,"localRetentionEnabled": "true", "localRetentionDays": 7,- "listen_on_lan": "true", "captureMaxWidth": 1920+ "listen_on_lan": "true", "captureMaxWidth": 1920,+ "focusedWindowScreenshots": "true"}));assert!(persist_store(store.as_ref(), &locked, false, || store.save()@@ -480,6 +481,7 @@ mod tests {assert!(config.listen_on_lan);assert_eq!(config.audio_chunk_duration, 60);assert_eq!(config.capture_max_width, 1920);+ assert!(config.focused_window_screenshots);assert_eq!(config.ignored_windows, vec!["Private"]);assert_eq!(saved["settings"]["localRetentionDays"], 7);assert_eq!(saved["settings"]["listenOnLan"], true);diff --git a/crates/screenpipe-a11y/src/tree/linux.rs b/crates/screenpipe-a11y/src/tree/linux.rsindex 3243210ca..60e74cc35 100644--- a/crates/screenpipe-a11y/src/tree/linux.rs+++ b/crates/screenpipe-a11y/src/tree/linux.rs@@ -1325,6 +1325,19 @@ impl TreeWalkerPlatform for LinuxTreeWalker {return Ok(TreeWalkResult::Skipped(SkipReason::UserIgnored));}+ // Reuse the extents already read above; no additional AT-SPI calls on+ // the capture path. Keep negative monitor-relative origins intact.+ let window_bounds = (self.config.capture_window_bounds+ && state.monitor_w > 0.0+ && state.monitor_h > 0.0+ && state.window_w > 0.0+ && state.window_h > 0.0)+ .then(|| super::WindowBounds {+ x: (state.window_x - state.monitor_x) / state.monitor_w,+ y: (state.window_y - state.monitor_y) / state.monitor_h,+ width: state.window_w / state.monitor_w,+ height: state.window_h / state.monitor_h,+ });let text_content = state.text_buffer;// Truncate if needed@@ -1394,7 +1407,7 @@ impl TreeWalkerPlatform for LinuxTreeWalker {truncation_reason: state.truncation_reason,retained_work_pending: false,max_depth_reached: state.max_depth_reached,- window_bounds: None,+ window_bounds,}))}}diff --git a/crates/screenpipe-a11y/src/tree/mod.rs b/crates/screenpipe-a11y/src/tree/mod.rsindex 5da2f9de0..d0fb6f08f 100644--- a/crates/screenpipe-a11y/src/tree/mod.rs+++ b/crates/screenpipe-a11y/src/tree/mod.rs@@ -515,6 +515,9 @@ pub struct TreeWalkerConfig {pub monitor_width: f64,/// Monitor height in screen points.pub monitor_height: f64,+ /// Export Linux window extents for opt-in focused screenshot scoping.+ /// False preserves the historical Linux full-frame OCR behavior.+ pub capture_window_bounds: bool,/// Automatically detect and skip incognito / private browsing windows.pub ignore_incognito_windows: bool,/// Use browser-native APIs for more reliable incognito detection where@@ -578,6 +581,7 @@ impl Default for TreeWalkerConfig {monitor_y: 0.0,monitor_width: 0.0,monitor_height: 0.0,+ capture_window_bounds: false,ignore_incognito_windows: true,enhanced_incognito_detection: false,capture_app_identity: false,diff --git a/crates/screenpipe-config/src/recording.rs b/crates/screenpipe-config/src/recording.rsindex 9b178441b..770e58672 100644--- a/crates/screenpipe-config/src/recording.rs+++ b/crates/screenpipe-config/src/recording.rs@@ -428,6 +428,11 @@ pub struct RecordingSettings {#[serde(rename = "disableScreenshots", default)]pub disable_screenshots: bool,+ /// Black out screenshot pixels outside the focused window area, preserving+ /// monitor dimensions and text coordinates. Off preserves desktop context.+ #[serde(rename = "focusedWindowScreenshots", default)]+ pub focused_window_screenshots: bool,+/// Build normalized semantic context from captured accessibility trees./// Experimental and opt-in. False preserves the historical capture path/// without starting a parser worker or writing semantic tables.@@ -969,6 +974,7 @@ impl Default for RecordingSettings {vocabulary: vec![],disable_vision: false,disable_screenshots: false,+ focused_window_screenshots: false,enable_semantic_context: false,semantic_context_mode: SemanticContextMode::Memory,disable_timeline: false,@@ -1346,6 +1352,22 @@ mod tests {assert_eq!(settings.effective_aec_flags(), (false, true, false));}+ #[test]+ fn focused_window_screenshots_default_off_and_round_trip() {+ let legacy: RecordingSettings = serde_json::from_str("{}").unwrap();+ assert!(!legacy.focused_window_screenshots);+ for value in [true, false] {+ let settings: RecordingSettings =+ serde_json::from_value(serde_json::json!({"focusedWindowScreenshots": value}))+ .unwrap();+ assert_eq!(settings.focused_window_screenshots, value);+ assert_eq!(+ serde_json::to_value(settings).unwrap()["focusedWindowScreenshots"],+ value+ );+ }+ }+#[test]fn deserializes_legacy_extra_fields() {// These fields used to live in SettingsStore.extra HashMapdiff --git a/crates/screenpipe-engine/src/cli/mod.rs b/crates/screenpipe-engine/src/cli/mod.rsindex 86dc8698d..a88581734 100644--- a/crates/screenpipe-engine/src/cli/mod.rs+++ b/crates/screenpipe-engine/src/cli/mod.rs@@ -626,6 +626,12 @@ pub struct RecordArgs {#[arg(long, default_value_t = false)]pub disable_screenshots: bool,+ /// Black out pixels outside the focused window area, keeping image dimensions.+ /// False keeps desktop context. Missing focus/bounds suppresses screenshots.+ #[arg(long, action = ArgAction::Set, num_args = 0..=1,+ default_value_t = false, default_missing_value = "true")]+ pub focused_window_screenshots: bool,+/// Select structured app context: off, memory, automation, or both./// Memory builds compact context for retrieval and pipes; automation keeps/// action controls, state, and bounds for computer-use agents. New CLI@@ -942,6 +948,7 @@ pub struct RecordArgSources {pub filter_music: bool,pub disable_vision: bool,pub disable_screenshots: bool,+ pub focused_window_screenshots: bool,pub app_context: bool,pub enable_semantic_context: bool,pub ignored_windows: bool,@@ -1011,6 +1018,7 @@ impl RecordArgSources {filter_music: from_command_line(record, "filter_music"),disable_vision: from_command_line(record, "disable_vision"),disable_screenshots: from_command_line(record, "disable_screenshots"),+ focused_window_screenshots: from_command_line(record, "focused_window_screenshots"),app_context: from_command_line(record, "app_context"),enable_semantic_context: matches!(record.value_source("enable_semantic_context"),@@ -1072,6 +1080,7 @@ impl RecordArgSources {|| self.filter_music|| self.disable_vision|| self.disable_screenshots+ || self.focused_window_screenshots|| self.app_context|| self.enable_semantic_context|| self.ignored_windows@@ -1238,6 +1247,7 @@ impl RecordArgs {disable_audio: self.disable_audio,disable_vision: self.disable_vision,disable_screenshots: self.disable_screenshots,+ focused_window_screenshots: self.focused_window_screenshots,enable_semantic_context,semantic_context_mode,// CLI has no --disable-timeline flag; the desktop app drives this@@ -1564,6 +1574,9 @@ impl RecordArgs {if sources.disable_screenshots {settings.disable_screenshots = self.disable_screenshots;}+ if sources.focused_window_screenshots {+ settings.focused_window_screenshots = self.focused_window_screenshots;+ }if sources.app_context {let (enabled, mode) = self.app_context@@ -3306,6 +3319,46 @@ mod tests {}}+ #[test]+ fn focused_window_screenshots_explicit_on_off_and_saved_choice() {+ for (argv, saved, expected) in [+ (vec!["screenpipe", "record"], true, true),+ (+ vec!["screenpipe", "record", "--focused-window-screenshots"],+ false,+ true,+ ),+ (+ vec!["screenpipe", "record", "--focused-window-screenshots=false"],+ true,+ false,+ ),+ ] {+ let sources = record_sources(argv.clone());+ let cli = Cli::try_parse_from(argv).unwrap();+ let mut settings = screenpipe_config::RecordingSettings {+ focused_window_screenshots: saved,+ ..Default::default()+ };+ let Command::Record(args) = cli.command else {+ panic!("expected record")+ };+ args.apply_explicit_overrides(&mut settings, &sources);+ assert_eq!(settings.focused_window_screenshots, expected);+ assert_eq!(+ args.to_recording_settings().focused_window_screenshots,+ args.focused_window_screenshots+ );+ }+ let Command::Record(args) = Cli::try_parse_from(["screenpipe", "record"])+ .unwrap()+ .command+ else {+ panic!("expected record")+ };+ assert!(!args.focused_window_screenshots);+ }+#[test]fn test_record_arg_sources_only_mark_explicit_cli_args() {let matches = Cli::command()diff --git a/crates/screenpipe-engine/src/event_driven_capture.rs b/crates/screenpipe-engine/src/event_driven_capture.rsindex 2a7b098be..50fab420d 100644--- a/crates/screenpipe-engine/src/event_driven_capture.rs+++ b/crates/screenpipe-engine/src/event_driven_capture.rs@@ -322,6 +322,7 @@ pub(crate) struct CaptureParams<'a> {pub ignored_patterns: Vec<WindowPattern>,pub use_pii_removal: bool,pub pause_on_drm_content: bool,+ pub focused_window_screenshots: bool,pub languages: &'a [screenpipe_core::Language],/// Opt-in background semantic parser. None preserves historical capture.pub semantic_tx: Option<&'a SemanticProjectionSender>,@@ -579,6 +580,7 @@ pub struct EventDrivenCaptureConfig {/// When true, visual checks, full screenshot capture, JPEG writes, and OCR/// fallback are skipped.pub disable_screenshots: bool,+ pub focused_window_screenshots: bool,/// User-pinned guaranteed-capture floor (ms). When `Some`, this is an/// explicit "always capture at least every N ms" choice and it must/// survive PowerProfile transitions — a battery-saver switch (whose@@ -600,6 +602,7 @@ impl Default for EventDrivenCaptureConfig {visual_check_interval_ms: 3_000, // check every 3 secondsvisual_change_threshold: 0.05, // ~5% difference triggers capturedisable_screenshots: false,+ focused_window_screenshots: false,idle_capture_interval_override_ms: None, // follow PowerProfile unless pinned}}@@ -1044,6 +1047,7 @@ pub(crate) async fn event_driven_capture_loop();let screenshots_disabled_by_config = config.disable_screenshots;+ let focused_window_screenshots = config.focused_window_screenshots;let mut screenshot_disabled = screenshots_disabled_by_config;// Publish the effective state so /health can name an intentional pixel// pause instead of blaming Screen Recording permission for it (#5808).@@ -1180,6 +1184,7 @@ pub(crate) async fn event_driven_capture_loop(ignored_patterns: WindowPattern::parse_list(&tree_walker_config.ignored_windows),use_pii_removal,pause_on_drm_content,+ focused_window_screenshots,languages: &languages,semantic_tx: semantic_tx.as_ref(),};@@ -2507,6 +2512,23 @@ fn mask_image_to_focused_window(Some(image::DynamicImage::ImageRgba8(masked))}+/// Reuse monitor coordinates for both pixels and accessibility/OCR overlays.+/// An unavailable focused window must never silently widen a scoped image.+fn apply_screenshot_scope(+ image: image::DynamicImage,+ bounds: Option<screenpipe_capture::paired_capture::FocusedWindowBounds>,+ focused_only: bool,+ screenshots_disabled: bool,+) -> (image::DynamicImage, bool) {+ if !focused_only || screenshots_disabled {+ return (image, false);+ }+ match bounds.and_then(|bounds| mask_image_to_focused_window(&image, bounds)) {+ Some(masked) => (masked, false),+ None => (image::DynamicImage::new_rgba8(1, 1), true),+ }+}+#[cfg(target_os = "macos")]fn get_focused_pid_fresh() -> Option<i32> {screenpipe_a11y::platform::macos::get_focused_pid_fresh()@@ -3162,6 +3184,16 @@ async fn do_capture(monitor_hosts_focus: bool,#[allow(unused_variables)] last_screenshot_focus_pid: &mut Option<i32>,) -> Result<CaptureOutput> {+ // Only the monitor hosting focus can provide an image in this mode.+ // Avoid taking and then discarding screenshots of the other monitors.+ if params.focused_window_screenshots && !monitor_hosts_focus {+ return Ok(CaptureOutput {+ result: None,+ image: image::DynamicImage::new_rgba8(1, 1),+ elements_deduped: false,+ corrupt: None,+ });+ }let captured_at = Utc::now();let bypass_capture_throttles = bypasses_capture_throttles(trigger);#[cfg(target_os = "windows")]@@ -3331,7 +3363,10 @@ async fn do_capture(} else {RedactedFocusDecision::Continue};- if focus_decision == RedactedFocusDecision::Withhold {+ if focus_decision == RedactedFocusDecision::Withhold+ || (params.focused_window_screenshots+ && focus_decision == RedactedFocusDecision::PixelsOnly)+ {return Ok(CaptureOutput {result: None,image,@@ -3888,6 +3923,14 @@ async fn do_capture(.is_some_and(|app| app.eq_ignore_ascii_case(&snap.app_name))}).and_then(|snap| snap.window_bounds)+ .filter(|b| {+ b.x.is_finite()+ && b.y.is_finite()+ && b.width.is_finite()+ && b.height.is_finite()+ && b.width > 0.0+ && b.height > 0.0+ }).map(|b| screenpipe_capture::paired_capture::FocusedWindowBounds {x: (b.x * frame_w as f64).round() as i32,@@ -3898,27 +3941,17 @@ async fn do_capture()};- // With a URL allowlist, an allowed native app must not expose a blocked- // browser elsewhere on the monitor. Black out pixels outside the- // positively identified focused window for both native apps and browsers.+ // A URL allowlist and the opt-in focused-window mode both restrict pixels+ // to a positively identified focused window. Preserve monitor dimensions+ // so accessibility bounds, OCR boxes, and replay retain their coordinates.// If the platform cannot provide trustworthy bounds, keep the allowed snapshot but// suppress pixels and OCR for this frame rather than widening capture.- let mut allowlist_pixels_unverified = false;- let image = if params.window_filters.has_url_allowlist()- && !screenshot_disabled- && !skip_pixels_for_unknown_exclusions- {- match focused_window_bounds.and_then(|bounds| mask_image_to_focused_window(&image, bounds))- {- Some(masked) => masked,- None => {- allowlist_pixels_unverified = true;- image::DynamicImage::new_rgba8(1, 1)- }- }- } else {- image- };+ let (image, scoped_pixels_unverified) = apply_screenshot_scope(+ image,+ focused_window_bounds,+ params.focused_window_screenshots || params.window_filters.has_url_allowlist(),+ screenshot_disabled || skip_pixels_for_unknown_exclusions,+ );let elements_ref_frame_id = matching_elements_reference(trigger,@@ -3951,7 +3984,7 @@ async fn do_capture(// only, no snapshot written for the 1x1 placeholder.screenshot_disabled: screenshot_disabled|| skip_pixels_for_unknown_exclusions- || allowlist_pixels_unverified,+ || scoped_pixels_unverified,in_meeting,monitor_hosts_focus,ax_screenshot_coherent,@@ -4413,6 +4446,52 @@ mod tests {));}+ #[test]+ fn focused_window_screenshots_preserve_coordinates_and_hide_desktop() {+ use image::GenericImageView;+ let source = image::DynamicImage::ImageRgba8(image::RgbaImage::from_pixel(+ 4,+ 3,+ image::Rgba([10, 20, 30, 255]),+ ));+ let bounds = screenpipe_capture::paired_capture::FocusedWindowBounds {+ x: -1,+ y: 1,+ width: 3,+ height: 5,+ };+ let (desktop, suppressed) = apply_screenshot_scope(source.clone(), None, false, false);+ assert!(!suppressed);+ assert_eq!(desktop.as_bytes(), source.as_bytes());+ let (focused, suppressed) =+ apply_screenshot_scope(source.clone(), Some(bounds), true, false);+ assert!(!suppressed);+ assert_eq!(focused.dimensions(), source.dimensions());+ for y in 0..3 {+ for x in 0..4 {+ let expected = if x < 2 && y >= 1 {+ [10, 20, 30, 255]+ } else {+ [0, 0, 0, 255]+ };+ assert_eq!(focused.get_pixel(x, y).0, expected);+ }+ }+ // Missing focus/bounds (including another monitor) cannot save a desktop image.+ assert!(apply_screenshot_scope(source.clone(), None, true, false).1);+ let invalid = screenpipe_capture::paired_capture::FocusedWindowBounds {+ x: 9,+ y: 9,+ width: 1,+ height: 1,+ };+ assert!(apply_screenshot_scope(source.clone(), Some(invalid), true, false).1);+ // Image capture being off must not turn it back on or do masking work.+ let (disabled, suppressed) = apply_screenshot_scope(source.clone(), None, true, true);+ assert!(!suppressed);+ assert_eq!(disabled.as_bytes(), source.as_bytes());+ }+#[test]fn url_allowlist_masks_everything_outside_focused_window() {use image::{GenericImageView, Pixel};diff --git a/crates/screenpipe-engine/src/recording_config.rs b/crates/screenpipe-engine/src/recording_config.rsindex 4ea7e8077..8651ba912 100644--- a/crates/screenpipe-engine/src/recording_config.rs+++ b/crates/screenpipe-engine/src/recording_config.rs@@ -39,6 +39,7 @@ pub struct RecordingConfig {/// capture alive. This skips visual-diff images, full screenshot capture,/// JPEG writes, and OCR fallback.pub disable_screenshots: bool,+ pub focused_window_screenshots: bool,/// Opt in to the bounded semantic projection worker. False preserves the/// historical capture and database behavior.pub enable_semantic_context: bool,@@ -306,6 +307,7 @@ impl RecordingConfig {|| settings.audio_capture_mode.eq_ignore_ascii_case("disabled"),disable_vision: settings.disable_vision,disable_screenshots: settings.disable_screenshots,+ focused_window_screenshots: settings.focused_window_screenshots,enable_semantic_context: settings.enable_semantic_context,semantic_context_mode: settings.semantic_context_mode,disable_timeline: settings.timeline_cache_disabled(),@@ -570,6 +572,7 @@ impl RecordingConfig {video_quality: self.video_quality.clone(),capture_max_width: self.capture_max_width,disable_screenshots: self.disable_screenshots,+ focused_window_screenshots: self.focused_window_screenshots,enable_semantic_context: self.enable_semantic_context,semantic_context_mode: self.semantic_context_mode,idle_capture_interval_ms: self.idle_capture_interval_ms,@@ -892,6 +895,7 @@ mod tests {assert_eq!(vision.video_quality, "high");assert_eq!(vision.capture_max_width, 2560);assert!(!vision.disable_screenshots);+ assert!(!vision.focused_window_screenshots);assert_eq!(vision.idle_capture_interval_ms, Some(2_000));assert_eq!(vision.visual_check_interval_ms, Some(350));assert_eq!(vision.visual_change_threshold, Some(0.18));@@ -900,6 +904,21 @@ mod tests {assert_eq!(vision.capture_on_clipboard, Some(true));}+ #[test]+ fn focused_window_screenshots_reach_vision_config() {+ let settings = screenpipe_config::RecordingSettings {+ focused_window_screenshots: true,+ ..Default::default()+ };+ let config = build(&settings);+ assert!(config.focused_window_screenshots);+ let vision = config.to_vision_manager_config(+ "capture-output".into(),+ std::sync::Arc::new(PipelineMetrics::new()),+ );+ assert!(vision.focused_window_screenshots);+ }+/// OCR reads the captured frame, so no storage preset may shrink the/// capture by default (#7393).#[test]diff --git a/crates/screenpipe-engine/src/vision_manager/manager.rs b/crates/screenpipe-engine/src/vision_manager/manager.rsindex 7199cab0d..200af828b 100644--- a/crates/screenpipe-engine/src/vision_manager/manager.rs+++ b/crates/screenpipe-engine/src/vision_manager/manager.rs@@ -65,6 +65,7 @@ pub struct VisionManagerConfig {pub capture_max_width: u32,/// Skip screenshot pixels/JPEG/OCR while keeping accessibility-tree capture.pub disable_screenshots: bool,+ pub focused_window_screenshots: bool,/// Enable the bounded semantic projection worker. Off by default.pub enable_semantic_context: bool,/// Choose memory, computer-use, or both projections over one capture.@@ -657,6 +658,7 @@ impl VisionManager {monitor_y: monitor.y() as f64,monitor_width: monitor.width() as f64,monitor_height: monitor.height() as f64,+ capture_window_bounds: self.config.focused_window_screenshots,ignore_incognito_windows: self.config.ignore_incognito_windows,enhanced_incognito_detection: self.config.enhanced_incognito_detection,capture_app_identity: self.config.enable_semantic_context@@ -675,6 +677,7 @@ impl VisionManager {let mut capture_config = EventDrivenCaptureConfig {jpeg_quality: baseline_q,disable_screenshots: self.config.disable_screenshots,+ focused_window_screenshots: self.config.focused_window_screenshots,..EventDrivenCaptureConfig::default()};// Mitsukeru fork: apply per-parameter CLI / settings overrides if any.@@ -1056,6 +1059,7 @@ mod tests {video_quality: "balanced".to_string(),capture_max_width: 0,disable_screenshots: false,+ focused_window_screenshots: false,enable_semantic_context,semantic_context_mode: SemanticContextMode::Memory,idle_capture_interval_ms: None,diff --git a/docs/COVERAGE.md b/docs/COVERAGE.mdindex 70423487e..4df225aad 100644--- a/docs/COVERAGE.md+++ b/docs/COVERAGE.md@@ -38,15 +38,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.- Mapped suites: 37- Mapped Rust files: 405-- Active test blocks: 3949+- Active test blocks: 3952- Ignored/manual test blocks: 189-- Weighted coverage points: 3255.1+- Weighted coverage points: 3257.5| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 33 | 3786 | 171 | 3182.1 | 21 | 11 | 100% |-| macos | 32 | 3829 | 150 | 3177.4 | 22 | 11 | 100% |-| linux | 29 | 3364 | 146 | 2800.4 | 20 | 11 | 100% |+| windows | 33 | 3789 | 171 | 3184.5 | 21 | 11 | 100% |+| macos | 32 | 3832 | 150 | 3179.8 | 22 | 11 | 100% |+| linux | 29 | 3367 | 146 | 2802.8 | 20 | 11 | 100% |## Refreshdiff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex 270a2bb1e..d440c4eac 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -9,10 +9,10 @@ confidence, and criticality.- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs- Mapped suites: 37- Mapped Rust files: 405-- Active test blocks: 3949+- Active test blocks: 3952- Ignored/manual test blocks: 189-- Declared test blocks: 4138-- Weighted coverage points: 3255.1+- Declared test blocks: 4141+- Weighted coverage points: 3257.5Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -23,15 +23,15 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 33 | 3786 | 171 | 3182.1 | 21 | 11 | 100% |-| macos | 32 | 3829 | 150 | 3177.4 | 22 | 11 | 100% |-| linux | 29 | 3364 | 146 | 2800.4 | 20 | 11 | 100% |+| windows | 33 | 3789 | 171 | 3184.5 | 21 | 11 | 100% |+| macos | 32 | 3832 | 150 | 3179.8 | 22 | 11 | 100% |+| linux | 29 | 3367 | 146 | 2802.8 | 20 | 11 | 100% |## Crate Summary| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows || --- | --- | --- | --- | --- | --- | --- | --- |-| screenpipe-engine | 12 | 22 | 131 | 1879 | 53 | 1446.0 | 10 |+| screenpipe-engine | 12 | 22 | 131 | 1882 | 53 | 1448.4 | 10 || screenpipe-db | 5 | 69 | 29 | 640 | 32 | 609.2 | 9 || screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 || screenpipe-audio | 8 | 29 | 54 | 706 | 56 | 617.0 | 5 |@@ -65,25 +65,25 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| accessibility | 4 suites / 384 active / 35 ignored / 349.0 pts | 4 suites / 440 active / 14 ignored / 350.7 pts | 4 suites / 342 active / 7 ignored / 314.0 pts || audio | 10 suites / 844 active / 60 ignored / 755.0 pts | 9 suites / 838 active / 60 ignored / 750.8 pts | 8 suites / 759 active / 59 ignored / 695.5 pts || audio-device | 2 suites / 243 active / 10 ignored / 219.3 pts | 2 suites / 243 active / 10 ignored / 219.3 pts | 1 suites / 164 active / 9 ignored / 164.0 pts |-| configuration | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts |+| configuration | 2 suites / 154 active / 3 ignored / 139.7 pts | 2 suites / 154 active / 3 ignored / 139.7 pts | 2 suites / 154 active / 3 ignored / 139.7 pts || database | 7 suites / 596 active / 28 ignored / 565.2 pts | 7 suites / 596 active / 28 ignored / 565.2 pts | 7 suites / 596 active / 28 ignored / 565.2 pts || db-search | 2 suites / 120 active / 9 ignored / 120.0 pts | 2 suites / 120 active / 9 ignored / 120.0 pts | 2 suites / 120 active / 9 ignored / 120.0 pts |-| engine-lifecycle | 6 suites / 274 active / 1 ignored / 244.8 pts | 6 suites / 274 active / 1 ignored / 244.8 pts | 6 suites / 270 active / 9 ignored / 243.9 pts |+| engine-lifecycle | 6 suites / 275 active / 1 ignored / 245.8 pts | 6 suites / 275 active / 1 ignored / 245.8 pts | 6 suites / 271 active / 9 ignored / 244.9 pts || local-api | 3 suites / 491 active / 12 ignored / 347.6 pts | 3 suites / 491 active / 12 ignored / 347.6 pts | 3 suites / 491 active / 12 ignored / 347.6 pts |-| meeting | 6 suites / 1762 active / 32 ignored / 1440.7 pts | 6 suites / 1762 active / 32 ignored / 1440.7 pts | 4 suites / 1408 active / 28 ignored / 1110.4 pts |+| meeting | 6 suites / 1763 active / 32 ignored / 1441.4 pts | 6 suites / 1763 active / 32 ignored / 1441.4 pts | 4 suites / 1409 active / 28 ignored / 1111.1 pts || ocr | 4 suites / 141 active / 7 ignored / 130.8 pts | 4 suites / 134 active / 7 ignored / 128.6 pts | 3 suites / 125 active / 6 ignored / 119.6 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |-| performance | 13 suites / 1730 active / 87 ignored / 1528.3 pts | 14 suites / 1855 active / 95 ignored / 1578.3 pts | 13 suites / 1730 active / 87 ignored / 1528.3 pts |-| pipes | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts |-| privacy | 5 suites / 1031 active / 43 ignored / 833.9 pts | 5 suites / 1087 active / 22 ignored / 835.6 pts | 5 suites / 989 active / 15 ignored / 798.8 pts |+| performance | 13 suites / 1732 active / 87 ignored / 1530.0 pts | 14 suites / 1857 active / 95 ignored / 1580.0 pts | 13 suites / 1732 active / 87 ignored / 1530.0 pts |+| pipes | 1 suites / 555 active / 3 ignored / 388.5 pts | 1 suites / 555 active / 3 ignored / 388.5 pts | 1 suites / 555 active / 3 ignored / 388.5 pts |+| privacy | 5 suites / 1032 active / 43 ignored / 834.6 pts | 5 suites / 1088 active / 22 ignored / 836.3 pts | 5 suites / 990 active / 15 ignored / 799.5 pts || real-app | - | 1 suites / 125 active / 8 ignored / 50.0 pts | - || speaker | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts |-| storage | 4 suites / 708 active / 41 ignored / 590.1 pts | 4 suites / 708 active / 41 ignored / 590.1 pts | 4 suites / 708 active / 41 ignored / 590.1 pts |-| sync | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts |-| timeline | 4 suites / 1271 active / 49 ignored / 1046.6 pts | 4 suites / 1271 active / 49 ignored / 1046.6 pts | 4 suites / 1271 active / 49 ignored / 1046.6 pts |+| storage | 4 suites / 709 active / 41 ignored / 590.8 pts | 4 suites / 709 active / 41 ignored / 590.8 pts | 4 suites / 709 active / 41 ignored / 590.8 pts |+| sync | 1 suites / 555 active / 3 ignored / 388.5 pts | 1 suites / 555 active / 3 ignored / 388.5 pts | 1 suites / 555 active / 3 ignored / 388.5 pts |+| timeline | 4 suites / 1272 active / 49 ignored / 1047.3 pts | 4 suites / 1272 active / 49 ignored / 1047.3 pts | 4 suites / 1272 active / 49 ignored / 1047.3 pts || transcription | 7 suites / 901 active / 53 ignored / 704.3 pts | 6 suites / 895 active / 53 ignored / 700.1 pts | 6 suites / 895 active / 53 ignored / 700.1 pts |-| ui-events | 4 suites / 831 active / 34 ignored / 629.8 pts | 3 suites / 762 active / 5 ignored / 581.5 pts | 3 suites / 762 active / 5 ignored / 581.5 pts |-| vision-capture | 6 suites / 597 active / 33 ignored / 472.4 pts | 6 suites / 590 active / 33 ignored / 470.2 pts | 5 suites / 581 active / 32 ignored / 461.2 pts |+| ui-events | 4 suites / 832 active / 34 ignored / 630.5 pts | 3 suites / 763 active / 5 ignored / 582.2 pts | 3 suites / 763 active / 5 ignored / 582.2 pts |+| vision-capture | 6 suites / 598 active / 33 ignored / 473.1 pts | 6 suites / 591 active / 33 ignored / 470.9 pts | 5 suites / 582 active / 32 ignored / 461.9 pts |## Critical Flow Matrix@@ -137,13 +137,13 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 14 | 107 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. || db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 40 | 315 | 15 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. || engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 46 | 438 | 7 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |-| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 310 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |-| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 13 | 125 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |+| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 311 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |+| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 13 | 126 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. || engine-db-recovery-cli | screenpipe-engine | windows, macos, linux | database, engine-lifecycle | engine-health-lifecycle, performance-liveness | high | strong | unit | 1 | 33 | 0 | Exact DB/WAL/SHM working-copy preservation, rollback on archive failure, and restart repair for crashes during the multi-file generation swap. || engine-focus-os | screenpipe-engine | windows, macos | engine-lifecycle, os-integration | engine-health-lifecycle, performance-liveness | medium | conditional | unit | 3 | 6 | 0 | Platform focus-tracker parsing/helpers. These files are cfg-gated and only execute on their target OS. The separately invoked harness=false tests/native_multi_monitor.rs acceptance is documented in scripts/native-test-fixtures/macos/README.md and is not counted as an automatic Rust test block. || engine-linux-keep-awake | screenpipe-engine | linux | engine-lifecycle | engine-health-lifecycle | high | conditional | mixed | 1 | 2 | 8 | Existing isolated-process missing/stalled bus regressions plus ignored live logind inhibitor tests. Live tests require logind and explicit execution. || engine-local-api-search-integration | screenpipe-engine | windows, macos, linux | local-api, db-search | local-api-search | high | strong | integration | 1 | 13 | 5 | Active /search route test builds an audio-disabled router, seeds captured-screen-shaped OCR data into an in-memory DB, and asserts the HTTP response and pagination. |-| engine-meeting-privacy-sync | screenpipe-engine | windows, macos, linux | meeting, privacy, ui-events, pipes, sync | meeting-live-notes, privacy-and-redaction, accessibility-ui-events, performance-liveness | medium | strong | unit | 33 | 554 | 3 | Unit-heavy coverage for privacy filter policy, capture exclusions, UI recorder safety, pipes/live-view/structured-output helpers, sync helpers, and CLI parsing. Meeting detection moved to the engine-meeting-watcher suite. |+| engine-meeting-privacy-sync | screenpipe-engine | windows, macos, linux | meeting, privacy, ui-events, pipes, sync | meeting-live-notes, privacy-and-redaction, accessibility-ui-events, performance-liveness | medium | strong | unit | 33 | 555 | 3 | Unit-heavy coverage for privacy filter policy, capture exclusions, UI recorder safety, pipes/live-view/structured-output helpers, sync helpers, and CLI parsing. Meeting detection moved to the engine-meeting-watcher suite. || engine-meeting-watcher | screenpipe-engine | windows, macos | meeting | meeting-live-notes | high | strong | mixed | 10 | 275 | 3 | Successor of src/meeting_detector.rs and src/meeting_telemetry.rs. Audio-process and UI-scan backend state machines, candidate resolution, shared telemetry, and a scored trajectory eval. ui_scan/macos.rs and ui_scan/windows.rs are cfg-gated and only execute on their target OS; both backends are null on Linux. || engine-retention-storage | screenpipe-engine | windows, macos, linux | storage, engine-lifecycle, performance | engine-health-lifecycle, performance-liveness | medium | strong | mixed | 5 | 43 | 0 | Local retention deletion (including lean-mode heavy-text stripping), cloud archive watermarking, atomic state-file replacement, and the low-disk capture monitor. || engine-telemetry-observability | screenpipe-engine | windows, macos, linux | engine-lifecycle, performance | engine-health-lifecycle, performance-liveness | medium | strong | unit | 8 | 40 | 0 | PostHog capture gating, telemetry context shaping, piggyback telemetry forwarding, crash-log helpers, resource monitoring, and the recording-coverage reliability metric. |@@ -392,7 +392,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| engine-db-recovery-cli | screenpipe-engine | src/cli/db.rs | source | 33 | 0 | 33 || engine-meeting-privacy-sync | screenpipe-engine | src/cli/install.rs | source | 4 | 0 | 4 || engine-meeting-privacy-sync | screenpipe-engine | src/cli/login.rs | source | 1 | 0 | 1 |-| engine-meeting-privacy-sync | screenpipe-engine | src/cli/mod.rs | source | 50 | 0 | 50 |+| engine-meeting-privacy-sync | screenpipe-engine | src/cli/mod.rs | source | 51 | 0 | 51 || engine-meeting-privacy-sync | screenpipe-engine | src/cli/presets.rs | source | 9 | 0 | 9 || engine-meeting-privacy-sync | screenpipe-engine | src/cli/profile.rs | source | 3 | 0 | 3 || engine-meeting-privacy-sync | screenpipe-engine | src/cli/search.rs | source | 7 | 0 | 7 |@@ -409,7 +409,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| engine-telemetry-observability | screenpipe-engine | src/crash_log.rs | source | 4 | 0 | 4 || engine-retention-storage | screenpipe-engine | src/disk_pressure.rs | source | 11 | 0 | 11 || engine-meeting-privacy-sync | screenpipe-engine | src/drm_detector.rs | source | 19 | 2 | 21 |-| engine-capture-timeline | screenpipe-engine | src/event_driven_capture.rs | source | 101 | 0 | 101 |+| engine-capture-timeline | screenpipe-engine | src/event_driven_capture.rs | source | 102 | 0 | 102 || engine-meeting-privacy-sync | screenpipe-engine | src/external_memory_sync.rs | source | 13 | 0 | 13 || engine-capture-timeline | screenpipe-engine | src/focus_aware_controller.rs | source | 15 | 0 | 15 || engine-focus-os | screenpipe-engine | src/focus_tracker/darwin.rs | source | 3 | 0 | 3 |@@ -452,7 +452,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| engine-meeting-privacy-sync | screenpipe-engine | src/privacy_filter.rs | source | 3 | 0 | 3 || engine-focus-os | screenpipe-engine | src/process_priority.rs | source | 2 | 0 | 2 || engine-meeting-privacy-sync | screenpipe-engine | src/qualified_value.rs | source | 5 | 0 | 5 |-| engine-config-lifecycle | screenpipe-engine | src/recording_config.rs | source | 18 | 0 | 18 |+| engine-config-lifecycle | screenpipe-engine | src/recording_config.rs | source | 19 | 0 | 19 || engine-telemetry-observability | screenpipe-engine | src/recording_coverage.rs | source | 5 | 0 | 5 || engine-config-lifecycle | screenpipe-engine | src/recording_detail.rs | source | 6 | 0 | 6 || engine-telemetry-observability | screenpipe-engine | src/resource_monitor.rs | source | 7 | 0 | 7 |diff --git a/docs/pr-assets/focused-window-screenshots/README.md b/docs/pr-assets/focused-window-screenshots/README.mdnew file mode 100644index 000000000..57474c6b6--- /dev/null+++ b/docs/pr-assets/focused-window-screenshots/README.md@@ -0,0 +1,24 @@+<!-- screenpipe — AI that knows everything you've seen, said, or heard -->+<!-- https://screenpipe.com -->++# Focused-window screenshot setting++The two PNGs are synthetic pixel fixtures, not native application captures.+They were produced by an isolated Rust probe that extracted+`mask_image_to_focused_window` and `apply_screenshot_scope` verbatim from+`crates/screenpipe-engine/src/event_driven_capture.rs`. The default-off result+retains the original image allocation. The enabled result retains the+1920 × 1080 dimensions and original pixels within the focused rectangle,+with opaque black outside it.++The source unit test also checks missing/invalid bounds, partial off-monitor+windows, and screenshots already disabled. Native capture, multi-monitor+behavior, focus races, and a running enterprise policy rollout require+separate platform validation. This setting masks a rectangle from the+already privacy-filtered desktop; allowed overlays inside that rectangle+can remain visible. It is not a tightly cropped or isolated native-window image.++An isolated release-mode helper probe on this Mac, while other cold builds+were running, measured 100 iterations at 1920 × 1080: median 7.455 ms,+p95 263.942 ms. This includes only the masking helper, not native capture,+OCR, JPEG encoding, or database writes, and is not a recorder benchmark.