YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix: use grim capture on wlroots wayland (#4533)
Louis Beaumont46b40df47707
diff --git a/crates/screenpipe-redact/src/adapters/rfdetr.rs b/crates/screenpipe-redact/src/adapters/rfdetr.rsindex 504af8961..151b81d34 100644--- a/crates/screenpipe-redact/src/adapters/rfdetr.rs+++ b/crates/screenpipe-redact/src/adapters/rfdetr.rs@@ -80,7 +80,7 @@ const CLASSES: [SpanLabel; NUM_CLASSES] = [/// Configuration for [`RfdetrRedactor`].#[derive(Debug, Clone)]pub struct RfdetrConfig {- /// Path to `rfdetr_vN.onnx`. We default to `~/.screenpipe/models/rfdetr_v12.onnx`+ /// Path to `rfdetr_vN.onnx`. We default to `~/.screenpipe/models/rfdetr_v13.onnx`/// in [`Self::default_model_path`] but callers may override (e.g./// for an INT8-quantized variant in the future).pub model_path: PathBuf,@@ -201,7 +201,7 @@ impl RfdetrConfig {tracing::info!(target = %self.model_path.display(),bytes = bytes.len(),- "rfdetr_v12.onnx ready"+ "rfdetr_v13.onnx ready");Ok(())}@@ -537,8 +537,15 @@ mod tests {#[test]fn default_path_lives_under_screenpipe_dir() {let p = RfdetrConfig::default_model_path();- let s = p.to_string_lossy();- assert!(s.contains(".screenpipe/models/rfdetr_v12.onnx"));+ let expected_suffix = Path::new(".screenpipe")+ .join("models")+ .join("rfdetr_v13.onnx");+ assert!(+ p.ends_with(&expected_suffix),+ "default path {} should end with {}",+ p.display(),+ expected_suffix.display()+ );}#[test]@@ -598,7 +605,7 @@ mod tests {// (Real download path is exercised by integration tests off// the unit-test harness.)let d = tempdir().unwrap();- let p = d.path().join("models").join("rfdetr_v12.onnx");+ let p = d.path().join("models").join("rfdetr_v13.onnx");std::fs::create_dir_all(p.parent().unwrap()).unwrap();std::fs::write(&p, b"not the real model").unwrap();let cfg = RfdetrConfig {diff --git a/crates/screenpipe-screen/src/monitor.rs b/crates/screenpipe-screen/src/monitor.rsindex 69df731b9..10c13ff9e 100644--- a/crates/screenpipe-screen/src/monitor.rs+++ b/crates/screenpipe-screen/src/monitor.rs@@ -8,9 +8,21 @@ use anyhow::Result;use image::DynamicImage;use once_cell::sync::Lazy;use std::fmt;+#[cfg(target_os = "linux")]+use std::io::Read;+#[cfg(target_os = "linux")]+use std::os::unix::fs::PermissionsExt;+#[cfg(target_os = "linux")]+use std::process::{Command, Stdio};#[cfg(target_os = "windows")]use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};+#[cfg(target_os = "linux")]+use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering};+#[cfg(target_os = "linux")]+use std::sync::Once;use std::sync::{Arc, RwLock};+#[cfg(target_os = "linux")]+use std::time::{Duration, Instant};use tracing;/// Cached monitor descriptions updated by the monitor watcher every 5s.@@ -18,6 +30,55 @@ use tracing;static CACHED_MONITOR_DESCRIPTIONS: Lazy<RwLock<Vec<String>>> =Lazy::new(|| RwLock::new(Vec::new()));+#[cfg(any(target_os = "linux", test))]+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum WaylandCaptureMode {+ Auto,+ Grim,+ Xcap,+}++#[cfg(any(target_os = "linux", test))]+fn parse_wayland_capture_mode(value: Option<&str>) -> WaylandCaptureMode {+ match value+ .map(str::trim)+ .filter(|value| !value.is_empty())+ .map(str::to_ascii_lowercase)+ .as_deref()+ {+ Some("grim" | "wlr" | "wlroots") => WaylandCaptureMode::Grim,+ Some("xcap" | "dbus" | "portal" | "off" | "false" | "0") => WaylandCaptureMode::Xcap,+ _ => WaylandCaptureMode::Auto,+ }+}++#[cfg(any(target_os = "linux", test))]+fn is_wayland_session(session_type: Option<&str>, wayland_display: Option<&str>) -> bool {+ session_type.is_some_and(|value| value.eq_ignore_ascii_case("wayland"))+ || wayland_display.is_some_and(|value| !value.trim().is_empty())+}++#[cfg(any(target_os = "linux", test))]+fn desktop_prefers_grim(desktop_hint: &str) -> bool {+ let normalized = desktop_hint.to_ascii_lowercase();+ [+ "dwl", "hyprland", "labwc", "niri", "river", "sway", "wayfire", "wlroots",+ ]+ .iter()+ .any(|needle| normalized.contains(needle))+}++#[cfg(target_os = "linux")]+static GRIM_CAPTURE_COUNTER: AtomicU64 = AtomicU64::new(0);+#[cfg(target_os = "linux")]+static GRIM_CAPTURE_LOG_ONCE: Once = Once::new();+#[cfg(target_os = "linux")]+static GRIM_CAPTURE_FALLBACK_LOG_ONCE: Once = Once::new();+#[cfg(target_os = "linux")]+static GRIM_AVAILABLE: Lazy<bool> = Lazy::new(|| command_in_path("grim"));+#[cfg(target_os = "linux")]+const GRIM_CAPTURE_TIMEOUT: Duration = Duration::from_secs(5);+/// macOS display capture is mediated by WindowServer/replayd. Serializing these/// calls avoids concurrent multi-monitor spikes while preserving capture order.#[cfg(target_os = "macos")]@@ -174,6 +235,89 @@ pub mod macos_version {#[cfg(target_os = "macos")]use macos_version::use_sck_rs;+#[cfg(target_os = "linux")]+fn command_in_path(command: &str) -> bool {+ std::env::var_os("PATH").is_some_and(|paths| {+ std::env::split_paths(&paths).any(|dir| {+ let candidate = dir.join(command);+ candidate.is_file()+ && candidate+ .metadata()+ .map(|metadata| metadata.permissions().mode() & 0o111 != 0)+ .unwrap_or(false)+ })+ })+}++#[cfg(target_os = "linux")]+fn current_desktop_hint() -> String {+ [+ std::env::var("XDG_CURRENT_DESKTOP").ok(),+ std::env::var("XDG_SESSION_DESKTOP").ok(),+ std::env::var("DESKTOP_SESSION").ok(),+ ]+ .into_iter()+ .flatten()+ .collect::<Vec<_>>()+ .join(":")+}++#[cfg(target_os = "linux")]+fn should_try_grim_capture() -> bool {+ if !is_wayland_session(+ std::env::var("XDG_SESSION_TYPE").ok().as_deref(),+ std::env::var("WAYLAND_DISPLAY").ok().as_deref(),+ ) {+ return false;+ }++ match parse_wayland_capture_mode(std::env::var("SCREENPIPE_WAYLAND_CAPTURE").ok().as_deref()) {+ WaylandCaptureMode::Grim => true,+ WaylandCaptureMode::Xcap => false,+ WaylandCaptureMode::Auto => {+ *GRIM_AVAILABLE && desktop_prefers_grim(¤t_desktop_hint())+ }+ }+}++#[cfg(any(target_os = "linux", test))]+fn grim_geometry(data: &MonitorData) -> String {+ format!("{},{} {}x{}", data.x, data.y, data.width, data.height)+}++#[cfg(target_os = "linux")]+fn wait_with_timeout(+ child: &mut std::process::Child,+ timeout: Duration,+) -> std::io::Result<std::process::ExitStatus> {+ let started = Instant::now();+ loop {+ if let Some(status) = child.try_wait()? {+ return Ok(status);+ }++ if started.elapsed() >= timeout {+ let _ = child.kill();+ let _ = child.wait();+ return Err(std::io::Error::new(+ std::io::ErrorKind::TimedOut,+ format!("process timed out after {timeout:?}"),+ ));+ }++ std::thread::sleep(Duration::from_millis(10));+ }+}++#[cfg(target_os = "linux")]+fn read_child_stderr(child: &mut std::process::Child) -> String {+ let mut stderr = String::new();+ if let Some(mut pipe) = child.stderr.take() {+ let _ = pipe.read_to_string(&mut stderr);+ }+ stderr.trim().to_string()+}+impl SafeMonitor {// macOS: Create from sck-rs monitor#[cfg(target_os = "macos")]@@ -488,7 +632,23 @@ impl SafeMonitor {#[cfg(not(target_os = "windows"))]{let cached_idx = self.cached_monitor_index.clone();+ let monitor_data = self.monitor_data.as_ref().clone();let image = tokio::task::spawn_blocking(move || -> Result<DynamicImage> {+ #[cfg(target_os = "linux")]+ if should_try_grim_capture() {+ match Self::capture_with_grim(monitor_id, &monitor_data) {+ Ok(image) => return Ok(image),+ Err(err) => {+ GRIM_CAPTURE_FALLBACK_LOG_ONCE.call_once(|| {+ tracing::warn!(+ "grim Wayland capture failed ({}); falling back to xcap",+ err+ );+ });+ }+ }+ }+Self::per_frame_capture_with_cache(monitor_id, cached_idx)}).await@@ -550,6 +710,57 @@ impl SafeMonitor {.map(DynamicImage::ImageRgba8)}+ #[cfg(target_os = "linux")]+ fn capture_with_grim(monitor_id: u32, data: &MonitorData) -> Result<DynamicImage> {+ if data.width == 0 || data.height == 0 {+ return Err(anyhow::anyhow!("invalid monitor dimensions"));+ }++ GRIM_CAPTURE_LOG_ONCE.call_once(|| {+ tracing::info!(+ "using grim for Wayland screen capture on wlroots-compatible compositor"+ );+ });++ let geometry = grim_geometry(data);+ let sequence = GRIM_CAPTURE_COUNTER.fetch_add(1, AtomicOrdering::Relaxed);+ let path = std::env::temp_dir().join(format!(+ "screenpipe-grim-{}-{monitor_id}-{sequence}.png",+ std::process::id()+ ));++ let result = (|| -> Result<DynamicImage> {+ let mut child = Command::new("grim")+ .arg("-g")+ .arg(&geometry)+ .arg("-t")+ .arg("png")+ .arg(&path)+ .stdout(Stdio::null())+ .stderr(Stdio::piped())+ .spawn()+ .map_err(|e| anyhow::anyhow!("failed to spawn grim: {e}"))?;++ let status = wait_with_timeout(&mut child, GRIM_CAPTURE_TIMEOUT)+ .map_err(|e| anyhow::anyhow!("grim capture failed: {e}"))?;+ let stderr = read_child_stderr(&mut child);+ if !status.success() {+ return Err(anyhow::anyhow!(+ "grim exited with status {status}; stderr: {stderr}"+ ));+ }++ let bytes = std::fs::read(&path)+ .map_err(|e| anyhow::anyhow!("failed to read grim output: {e}"))?;+ image::load_from_memory(&bytes)+ .map(|image| DynamicImage::ImageRgba8(image.to_rgba8()))+ .map_err(|e| anyhow::anyhow!("failed to decode grim output: {e}"))+ })();++ let _ = std::fs::remove_file(&path);+ result+ }+/// Refresh the cached monitor handle by re-enumerating all monitors./// Call this after a capture failure — the monitor may have been reconnected/// with different properties (resolution change, etc.).@@ -1214,6 +1425,63 @@ mod tests {assert_eq!(prefix, prefix2);}+ #[test]+ fn test_wayland_session_detection() {+ assert!(is_wayland_session(Some("wayland"), None));+ assert!(is_wayland_session(Some("Wayland"), None));+ assert!(is_wayland_session(Some("x11"), Some("wayland-1")));+ assert!(!is_wayland_session(Some("x11"), None));+ assert!(!is_wayland_session(None, Some("")));+ }++ #[test]+ fn test_wayland_capture_mode_parsing() {+ assert_eq!(parse_wayland_capture_mode(None), WaylandCaptureMode::Auto);+ assert_eq!(+ parse_wayland_capture_mode(Some("")),+ WaylandCaptureMode::Auto+ );+ assert_eq!(+ parse_wayland_capture_mode(Some("grim")),+ WaylandCaptureMode::Grim+ );+ assert_eq!(+ parse_wayland_capture_mode(Some("wlroots")),+ WaylandCaptureMode::Grim+ );+ assert_eq!(+ parse_wayland_capture_mode(Some("xcap")),+ WaylandCaptureMode::Xcap+ );+ assert_eq!(+ parse_wayland_capture_mode(Some("portal")),+ WaylandCaptureMode::Xcap+ );+ }++ #[test]+ fn test_wlroots_desktops_prefer_grim() {+ assert!(desktop_prefers_grim("Hyprland"));+ assert!(desktop_prefers_grim("niri"));+ assert!(desktop_prefers_grim("sway:wlroots"));+ assert!(desktop_prefers_grim("river"));+ assert!(!desktop_prefers_grim("GNOME"));+ assert!(!desktop_prefers_grim("KDE"));+ }++ #[test]+ fn test_grim_geometry_format() {+ let data = MonitorData {+ width: 2560,+ height: 1440,+ x: -2560,+ y: 0,+ name: "left".to_string(),+ is_primary: false,+ };+ assert_eq!(grim_geometry(&data), "-2560,0 2560x1440");+ }+#[cfg(target_os = "windows")]#[test]fn test_persistent_capture_disables_after_three_failures() {