YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
[codex] Add Hyprland Wayland attribution fallback
Milan Petrović200b0df7bedd
diff --git a/crates/screenpipe-a11y/src/platform/linux.rs b/crates/screenpipe-a11y/src/platform/linux.rsindex 5963b0c0b..4bedddff0 100644--- a/crates/screenpipe-a11y/src/platform/linux.rs+++ b/crates/screenpipe-a11y/src/platform/linux.rs@@ -16,7 +16,7 @@//!//! ## Window Tracking//!-//! Uses `xdotool` (X11) or D-Bus (Wayland) to track the active window.+//! Uses Hyprland IPC (Wayland) or `xdotool` (X11) to track the active window.//!//! ## Clipboard//!@@ -31,10 +31,11 @@ use crossbeam_channel::{bounded, Receiver, Sender};use parking_lot::Mutex;use screenpipe_core::pii_removal::remove_pii;use std::path::Path;+use std::process::{Command, Output, Stdio};use std::sync::atomic::{AtomicBool, Ordering};-use std::sync::Arc;+use std::sync::{Arc, OnceLock};use std::thread;-use std::time::Instant;+use std::time::{Duration, Instant};use tracing::{debug, warn};/// Permission status for UI capture on Linux.@@ -202,7 +203,7 @@ impl UiRecorder {);}- // Thread 2: App/window observer (always runs — uses xdotool, no evdev needed)+ // Thread 2: App/window observer (always runs — uses compositor/X11 metadata, no evdev needed)let tx2 = tx.clone();let stop2 = stop.clone();let config2 = self.config.clone();@@ -989,14 +990,100 @@ fn run_window_observer(}/// Get active window info: (app_name, window_title, pid).-/// Tries xdotool (X11) first, falls back to D-Bus.-fn get_active_window_info() -> Option<(String, String, i32)> {+/// Uses Hyprland IPC on Hyprland/Wayland, then falls back to xdotool for X11.+pub fn get_active_window_info() -> Option<(String, String, i32)> {+ const ACTIVE_WINDOW_CACHE_TTL: Duration = Duration::from_secs(1);+ static CACHE: OnceLock<Mutex<Option<(Option<(String, String, i32)>, Instant)>>> =+ OnceLock::new();++ let cache = CACHE.get_or_init(|| Mutex::new(None));+ let now = Instant::now();+ {+ let cached = cache.lock();+ if let Some((value, captured_at)) = &*cached {+ if now.saturating_duration_since(*captured_at) < ACTIVE_WINDOW_CACHE_TTL {+ return value.clone();+ }+ }+ }++ let fresh = get_active_window_info_fresh();+ *cache.lock() = Some((fresh.clone(), Instant::now()));+ fresh+}++/// Get active window info without using the shared observer cache.+///+/// Use this when stale focused-window metadata would affect privacy gates.+pub fn get_active_window_info_fresh() -> Option<(String, String, i32)> {+ get_hyprland_active_window_info().or_else(get_x11_active_window_info)+}++fn get_hyprland_active_window_info() -> Option<(String, String, i32)> {+ if std::env::var_os("HYPRLAND_INSTANCE_SIGNATURE").is_none()+ && std::env::var("XDG_CURRENT_DESKTOP").map_or(true, |desktop| {+ !desktop.to_ascii_lowercase().contains("hyprland")+ })+ {+ return None;+ }++ let output = command_output_with_timeout(+ "hyprctl",+ &["-j", "activewindow"],+ Duration::from_millis(200),+ )+ .filter(|o| o.status.success())?;++ parse_hyprland_active_window_json(&output.stdout)+}++fn parse_hyprland_active_window_json(raw: &[u8]) -> Option<(String, String, i32)> {+ let value: serde_json::Value = serde_json::from_slice(raw).ok()?;+ let app_name = value+ .get("class")+ .and_then(|v| v.as_str())+ .map(str::trim)+ .filter(|s| !s.is_empty())+ .or_else(|| {+ value+ .get("initialClass")+ .and_then(|v| v.as_str())+ .map(str::trim)+ .filter(|s| !s.is_empty())+ })?+ .to_string();++ let window_title = value+ .get("title")+ .and_then(|v| v.as_str())+ .map(clean_hyprland_title)+ .unwrap_or_default();++ let pid = value.get("pid").and_then(|v| v.as_i64()).unwrap_or(0) as i32;++ Some((app_name, window_title, pid))+}++fn clean_hyprland_title(title: &str) -> String {+ title+ .chars()+ .filter(|&c| {+ !matches!(+ c,+ '\u{061c}' | '\u{200e}' | '\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}'+ )+ })+ .collect::<String>()+ .trim()+ .to_string()+}++fn get_x11_active_window_info() -> Option<(String, String, i32)> {// Try xdotool (works on X11 and XWayland)- let wid_output = std::process::Command::new("xdotool")- .arg("getactivewindow")- .output()- .ok()- .filter(|o| o.status.success())?;+ let wid_output =+ command_output_with_timeout("xdotool", &["getactivewindow"], Duration::from_millis(300))+ .filter(|o| o.status.success())?;let wid = String::from_utf8_lossy(&wid_output.stdout).trim()@@ -1006,21 +1093,23 @@ fn get_active_window_info() -> Option<(String, String, i32)> {}// Get window name- let name_output = std::process::Command::new("xdotool")- .args(["getwindowname", &wid])- .output()- .ok()- .filter(|o| o.status.success())?;+ let name_output = command_output_with_timeout(+ "xdotool",+ &["getwindowname", &wid],+ Duration::from_millis(300),+ )+ .filter(|o| o.status.success())?;let title = String::from_utf8_lossy(&name_output.stdout).trim().to_string();// Get PID- let pid_output = std::process::Command::new("xdotool")- .args(["getwindowpid", &wid])- .output()- .ok()- .filter(|o| o.status.success());+ let pid_output = command_output_with_timeout(+ "xdotool",+ &["getwindowpid", &wid],+ Duration::from_millis(300),+ )+ .filter(|o| o.status.success());let pid: i32 = pid_output.map(|o| {@@ -1041,6 +1130,34 @@ fn get_active_window_info() -> Option<(String, String, i32)> {Some((app_name, title, pid))}+fn command_output_with_timeout(command: &str, args: &[&str], timeout: Duration) -> Option<Output> {+ let mut child = Command::new(command)+ .args(args)+ .stdout(Stdio::piped())+ .stderr(Stdio::piped())+ .spawn()+ .ok()?;+ let start = Instant::now();++ loop {+ match child.try_wait() {+ Ok(Some(_)) => return child.wait_with_output().ok(),+ Ok(None) => {}+ Err(_) => {+ let _ = child.kill();+ let _ = child.wait();+ return None;+ }+ }+ if start.elapsed() >= timeout {+ let _ = child.kill();+ let _ = child.wait();+ return None;+ }+ thread::sleep(Duration::from_millis(10));+ }+}+/// Get process name from PID by reading /proc/<pid>/comm.pub fn get_process_name(pid: u32) -> Option<String> {std::fs::read_to_string(format!("/proc/{}/comm", pid))@@ -1395,6 +1512,46 @@ mod tests {assert_eq!(truncate("hello world", 8), "hello...");}+ #[test]+ fn test_parse_hyprland_active_window_json() {+ let raw = br#"{+ "class": "org.telegram.desktop",+ "title": "\u061c\u200e\u2068Lucern Clinic\u2069 \u2013 (106539)",+ "initialClass": "Telegram",+ "pid": 2800,+ "xwayland": false+ }"#;++ let (app_name, window_title, pid) = parse_hyprland_active_window_json(raw).unwrap();++ assert_eq!(app_name, "org.telegram.desktop");+ assert_eq!(window_title, "Lucern Clinic \u{2013} (106539)");+ assert_eq!(pid, 2800);+ }++ #[test]+ fn test_clean_hyprland_title_strips_bidi_controls() {+ let title = "\u{061c}\u{200e}\u{2068}Telegram\u{2069}\u{202e}";++ assert_eq!(clean_hyprland_title(title), "Telegram");+ }++ #[test]+ fn test_parse_hyprland_active_window_json_falls_back_to_initial_class() {+ let raw = br#"{+ "class": "",+ "title": "Terminal",+ "initialClass": "Alacritty",+ "pid": 123+ }"#;++ let (app_name, window_title, pid) = parse_hyprland_active_window_json(raw).unwrap();++ assert_eq!(app_name, "Alacritty");+ assert_eq!(window_title, "Terminal");+ assert_eq!(pid, 123);+ }+#[test]fn test_get_process_name() {// PID 1 (init/systemd) should exist on Linuxdiff --git a/crates/screenpipe-engine/src/event_driven_capture.rs b/crates/screenpipe-engine/src/event_driven_capture.rsindex aaa31c05d..6a890d4ec 100644--- a/crates/screenpipe-engine/src/event_driven_capture.rs+++ b/crates/screenpipe-engine/src/event_driven_capture.rs@@ -1738,10 +1738,42 @@ struct CaptureOutput {corrupt: Option<CorruptKind>,}+#[derive(Debug, Clone)]+struct LightweightFocusedMetadata {+ app_name: Option<String>,+ window_name: Option<String>,+}++fn normalize_metadata_value(value: Option<&str>) -> Option<String> {+ value+ .map(str::trim)+ .filter(|value| !value.is_empty())+ .map(str::to_string)+}+fn resolve_capture_metadata(tree_snapshot: Option<&screenpipe_a11y::tree::TreeSnapshot>,trigger: &CaptureTrigger,- lightweight_app_name: Option<&str>,+ lightweight_metadata: Option<&LightweightFocusedMetadata>,+) -> (+ Option<String>,+ Option<String>,+ Option<String>,+ Option<String>,+) {+ resolve_capture_metadata_with_policy(+ tree_snapshot,+ trigger,+ lightweight_metadata,+ cfg!(target_os = "linux"),+ )+}++fn resolve_capture_metadata_with_policy(+ tree_snapshot: Option<&screenpipe_a11y::tree::TreeSnapshot>,+ trigger: &CaptureTrigger,+ lightweight_metadata: Option<&LightweightFocusedMetadata>,+ prefer_lightweight_metadata: bool,) -> (Option<String>,Option<String>,@@ -1758,14 +1790,22 @@ fn resolve_capture_metadata(None => (None, None, None, None),};- // Fallback to the lightweight focused-app query when the tree walk returned- // nothing (focused_window AX query failed, e.g. Zoom during meetings).- // Without this, captures triggered by click/visual_change/idle would be- // stored with null app_name even though we know the focused app.- if app_name.is_none() {- if let Some(name) = lightweight_app_name {- if !name.is_empty() {- app_name = Some(name.to_string());+ app_name = normalize_metadata_value(app_name.as_deref());+ window_name = normalize_metadata_value(window_name.as_deref());++ // Prefer focused-window metadata on Linux Wayland, where AT-SPI can return+ // stale or generic app names (for example "electron") while Hyprland knows+ // the actual focused app/window. Other platforms keep the older fallback+ // behavior because their lightweight source may itself be cached.+ if let Some(metadata) = lightweight_metadata {+ if let Some(name) = normalize_metadata_value(metadata.app_name.as_deref()) {+ if prefer_lightweight_metadata || app_name.is_none() {+ app_name = Some(name);+ }+ }+ if let Some(name) = normalize_metadata_value(metadata.window_name.as_deref()) {+ if prefer_lightweight_metadata || window_name.is_none() {+ window_name = Some(name);}}}@@ -2102,22 +2142,34 @@ async fn do_capture(// reduced max_nodes and timeout to avoid blocking their UI thread.let mut config = params.tree_walker_config.clone();- // Get the focused app name for budget decisions. AppSwitch triggers carry+ // Get focused metadata for budget decisions. AppSwitch triggers carry// the name directly; for all other triggers (visual change, idle, manual)- // we do a lightweight AX query to get the focused app. This ensures the- // walk budget applies to ALL captures, not just app switches.- let trigger_app = match trigger {- CaptureTrigger::AppSwitch { app_name, .. } => Some(app_name.clone()),- _ => {- #[cfg(target_os = "macos")]- {- get_focused_app_name_lightweight()+ // we do a lightweight platform query. This ensures the walk budget applies+ // to ALL captures, not just app switches.+ let lightweight_focused_metadata = match trigger {+ CaptureTrigger::AppSwitch { .. } => None,+ _ => match tokio::time::timeout(+ Duration::from_secs(1),+ tokio::task::spawn_blocking(get_focused_metadata_lightweight),+ )+ .await+ {+ Ok(Ok(metadata)) => metadata,+ Ok(Err(err)) => {+ debug!("focused metadata lookup task failed: {}", err);+ None}- #[cfg(not(target_os = "macos"))]- {+ Err(_) => {+ debug!("focused metadata lookup timed out");None}- }+ },+ };+ let trigger_app = match trigger {+ CaptureTrigger::AppSwitch { app_name, .. } => Some(app_name.clone()),+ _ => lightweight_focused_metadata+ .as_ref()+ .and_then(|metadata| metadata.app_name.clone()),};// Terminal OCR rate-limit: wezterm/alacritty/kitty/hyper/warp all bypass AX@@ -2333,7 +2385,11 @@ async fn do_capture(// Use tree metadata by default, but for focus-change triggers prefer the// event payload when the tree lags or reports the wrong frontmost target.let (app_name_owned, window_name_owned, browser_url_owned, document_path_owned) =- resolve_capture_metadata(tree_snapshot.as_ref(), trigger, trigger_app.as_deref());+ resolve_capture_metadata(+ tree_snapshot.as_ref(),+ trigger,+ lightweight_focused_metadata.as_ref(),+ );// Skip lock screen / screensaver — these waste disk and pollute timeline.// Also update the global SCREEN_IS_LOCKED flag so subsequent loop iterations@@ -2449,7 +2505,7 @@ async fn do_capture(})}-/// Cheaply get the focused app name. Used to tag captures and to apply+/// Cheaply get focused app/window metadata. Used to tag captures and to apply/// per-app throttles (walk budget, terminal OCR, Obsidian OCR).////// Tries NSWorkspace first: filters `running_apps()` to the one with@@ -2467,7 +2523,7 @@ async fn do_capture(/// TTL keeps staleness bounded to something no human perceives while/// collapsing the common case to a single atomic load.#[cfg(target_os = "macos")]-fn get_focused_app_name_lightweight() -> Option<String> {+fn get_focused_metadata_lightweight() -> Option<LightweightFocusedMetadata> {use arc_swap::ArcSwap;use std::sync::OnceLock;use std::time::{Duration, Instant};@@ -2476,7 +2532,8 @@ fn get_focused_app_name_lightweight() -> Option<String> {// (name, captured_at). ArcSwap gives lock-free reads; in the common// case the whole function is one atomic load + a clock read + clone.- static CACHE: OnceLock<ArcSwap<(Option<String>, Instant)>> = OnceLock::new();+ static CACHE: OnceLock<ArcSwap<(Option<LightweightFocusedMetadata>, Instant)>> =+ OnceLock::new();let cache = CACHE.get_or_init(|| {ArcSwap::from_pointee((None, Instant::now() - CACHE_TTL - Duration::from_secs(1)))});@@ -2489,11 +2546,37 @@ fn get_focused_app_name_lightweight() -> Option<String> {}}- let fresh = query_frontmost_app_name_uncached();+ let fresh = query_frontmost_app_name_uncached().map(|app_name| LightweightFocusedMetadata {+ app_name: Some(app_name),+ window_name: None,+ });cache.store(std::sync::Arc::new((fresh.clone(), now)));fresh}+#[cfg(target_os = "linux")]+fn get_focused_metadata_lightweight() -> Option<LightweightFocusedMetadata> {+ let (app_name, window_name, _) =+ screenpipe_a11y::platform::linux::get_active_window_info_fresh()?;+ Some(LightweightFocusedMetadata {+ app_name: if app_name.is_empty() {+ None+ } else {+ Some(app_name)+ },+ window_name: if window_name.is_empty() {+ None+ } else {+ Some(window_name)+ },+ })+}++#[cfg(not(any(target_os = "macos", target_os = "linux")))]+fn get_focused_metadata_lightweight() -> Option<LightweightFocusedMetadata> {+ None+}+#[cfg(target_os = "macos")]fn query_frontmost_app_name_uncached() -> Option<String> {use cidre::{ax, ns, objc};@@ -2759,6 +2842,166 @@ mod tests {assert_eq!(CaptureTrigger::Manual.as_str(), "manual");}+ #[test]+ fn resolve_capture_metadata_uses_lightweight_app_and_window_without_tree() {+ let metadata = LightweightFocusedMetadata {+ app_name: Some("Telegram".into()),+ window_name: Some("Lucern Clinic".into()),+ };++ let (app_name, window_name, browser_url, document_path) =+ resolve_capture_metadata(None, &CaptureTrigger::Idle, Some(&metadata));++ assert_eq!(app_name.as_deref(), Some("Telegram"));+ assert_eq!(window_name.as_deref(), Some("Lucern Clinic"));+ assert_eq!(browser_url, None);+ assert_eq!(document_path, None);+ }++ #[test]+ fn resolve_capture_metadata_uses_lightweight_metadata_when_tree_values_are_blank() {+ let snapshot = screenpipe_a11y::tree::TreeSnapshot {+ app_name: " ".into(),+ window_name: "".into(),+ text_content: "visible text".into(),+ nodes: Vec::new(),+ browser_url: None,+ document_path: None,+ timestamp: Utc::now(),+ node_count: 0,+ walk_duration: Duration::from_millis(1),+ content_hash: 0,+ simhash: 0,+ truncated: false,+ truncation_reason: screenpipe_a11y::tree::TruncationReason::None,+ max_depth_reached: 0,+ };+ let metadata = LightweightFocusedMetadata {+ app_name: Some(" org.telegram.desktop ".into()),+ window_name: Some(" Telegram ".into()),+ };++ let (app_name, window_name, _, _) =+ resolve_capture_metadata(Some(&snapshot), &CaptureTrigger::Idle, Some(&metadata));++ assert_eq!(app_name.as_deref(), Some("org.telegram.desktop"));+ assert_eq!(window_name.as_deref(), Some("Telegram"));+ }++ #[test]+ fn resolve_capture_metadata_prefers_lightweight_metadata_over_stale_tree_values() {+ let snapshot = screenpipe_a11y::tree::TreeSnapshot {+ app_name: "electron".into(),+ window_name: "stale browser title".into(),+ text_content: "visible text".into(),+ nodes: Vec::new(),+ browser_url: None,+ document_path: None,+ timestamp: Utc::now(),+ node_count: 0,+ walk_duration: Duration::from_millis(1),+ content_hash: 0,+ simhash: 0,+ truncated: false,+ truncation_reason: screenpipe_a11y::tree::TruncationReason::None,+ max_depth_reached: 0,+ };+ let metadata = LightweightFocusedMetadata {+ app_name: Some("Alacritty".into()),+ window_name: Some("screenpipe-wayland-fix".into()),+ };++ let (app_name, window_name, _, _) = resolve_capture_metadata_with_policy(+ Some(&snapshot),+ &CaptureTrigger::Idle,+ Some(&metadata),+ true,+ );++ assert_eq!(app_name.as_deref(), Some("Alacritty"));+ assert_eq!(window_name.as_deref(), Some("screenpipe-wayland-fix"));+ }++ #[test]+ fn resolve_capture_metadata_keeps_tree_values_when_lightweight_is_fallback_only() {+ let snapshot = screenpipe_a11y::tree::TreeSnapshot {+ app_name: "Safari".into(),+ window_name: "Current Page".into(),+ text_content: "visible text".into(),+ nodes: Vec::new(),+ browser_url: None,+ document_path: None,+ timestamp: Utc::now(),+ node_count: 0,+ walk_duration: Duration::from_millis(1),+ content_hash: 0,+ simhash: 0,+ truncated: false,+ truncation_reason: screenpipe_a11y::tree::TruncationReason::None,+ max_depth_reached: 0,+ };+ let metadata = LightweightFocusedMetadata {+ app_name: Some("Terminal".into()),+ window_name: Some("Stale Terminal".into()),+ };++ let (app_name, window_name, _, _) = resolve_capture_metadata_with_policy(+ Some(&snapshot),+ &CaptureTrigger::Idle,+ Some(&metadata),+ false,+ );++ assert_eq!(app_name.as_deref(), Some("Safari"));+ assert_eq!(window_name.as_deref(), Some("Current Page"));+ }++ #[test]+ fn resolve_capture_metadata_normalizes_blank_tree_values_without_lightweight_metadata() {+ let snapshot = screenpipe_a11y::tree::TreeSnapshot {+ app_name: " ".into(),+ window_name: "".into(),+ text_content: "visible text".into(),+ nodes: Vec::new(),+ browser_url: None,+ document_path: None,+ timestamp: Utc::now(),+ node_count: 0,+ walk_duration: Duration::from_millis(1),+ content_hash: 0,+ simhash: 0,+ truncated: false,+ truncation_reason: screenpipe_a11y::tree::TruncationReason::None,+ max_depth_reached: 0,+ };++ let (app_name, window_name, _, _) =+ resolve_capture_metadata(Some(&snapshot), &CaptureTrigger::Idle, None);++ assert_eq!(app_name, None);+ assert_eq!(window_name, None);+ }++ #[test]+ fn resolve_capture_metadata_prefers_window_focus_trigger_title() {+ let metadata = LightweightFocusedMetadata {+ app_name: Some("Telegram".into()),+ window_name: Some("Stale Title".into()),+ };++ let (app_name, window_name, _, _) = resolve_capture_metadata(+ None,+ &CaptureTrigger::WindowFocus {+ window_name: "Fresh Title".into(),+ target: None,+ },+ Some(&metadata),+ );++ assert_eq!(app_name.as_deref(), Some("Telegram"));+ assert_eq!(window_name.as_deref(), Some("Fresh Title"));+ }+#[test]fn workflow_checkpoint_trigger_classification() {assert!(is_workflow_checkpoint_trigger(&CaptureTrigger::AppSwitch {