Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix: use grim capture on wlroots wayland (#4533)

Louis Beaumont46b40df47707
diff --git a/crates/screenpipe-redact/src/adapters/rfdetr.rs b/crates/screenpipe-redact/src/adapters/rfdetr.rs
index 504af8961..151b81d34 100644
--- a/crates/screenpipe-redact/src/adapters/rfdetr.rs
+++ b/crates/screenpipe-redact/src/adapters/rfdetr.rs
@@ -80,7 +80,7 @@ const CLASSES: [SpanLabel; NUM_CLASSES] = [
/// Configuration for [`RfdetrRedactor`].
#[derive(Debug, Clone)]
pub struct RfdetrConfig {
- /// Path to `rfdetr_vN.onnx`. We default to `~/.screenpipe/models/rfdetr_v12.onnx`
+ /// Path to `rfdetr_vN.onnx`. We default to `~/.screenpipe/models/rfdetr_v13.onnx`
/// in [`Self::default_model_path`] but callers may override (e.g.
/// for an INT8-quantized variant in the future).
pub model_path: PathBuf,
@@ -201,7 +201,7 @@ impl RfdetrConfig {
tracing::info!(
target = %self.model_path.display(),
bytes = bytes.len(),
- "rfdetr_v12.onnx ready"
+ "rfdetr_v13.onnx ready"
);
Ok(())
}
@@ -537,8 +537,15 @@ mod tests {
#[test]
fn default_path_lives_under_screenpipe_dir() {
let p = RfdetrConfig::default_model_path();
- let s = p.to_string_lossy();
- assert!(s.contains(".screenpipe/models/rfdetr_v12.onnx"));
+ let expected_suffix = Path::new(".screenpipe")
+ .join("models")
+ .join("rfdetr_v13.onnx");
+ assert!(
+ p.ends_with(&expected_suffix),
+ "default path {} should end with {}",
+ p.display(),
+ expected_suffix.display()
+ );
}
#[test]
@@ -598,7 +605,7 @@ mod tests {
// (Real download path is exercised by integration tests off
// the unit-test harness.)
let d = tempdir().unwrap();
- let p = d.path().join("models").join("rfdetr_v12.onnx");
+ let p = d.path().join("models").join("rfdetr_v13.onnx");
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
std::fs::write(&p, b"not the real model").unwrap();
let cfg = RfdetrConfig {
diff --git a/crates/screenpipe-screen/src/monitor.rs b/crates/screenpipe-screen/src/monitor.rs
index 69df731b9..10c13ff9e 100644
--- a/crates/screenpipe-screen/src/monitor.rs
+++ b/crates/screenpipe-screen/src/monitor.rs
@@ -8,9 +8,21 @@ use anyhow::Result;
use image::DynamicImage;
use once_cell::sync::Lazy;
use std::fmt;
+#[cfg(target_os = "linux")]
+use std::io::Read;
+#[cfg(target_os = "linux")]
+use std::os::unix::fs::PermissionsExt;
+#[cfg(target_os = "linux")]
+use std::process::{Command, Stdio};
#[cfg(target_os = "windows")]
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
+#[cfg(target_os = "linux")]
+use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering};
+#[cfg(target_os = "linux")]
+use std::sync::Once;
use std::sync::{Arc, RwLock};
+#[cfg(target_os = "linux")]
+use std::time::{Duration, Instant};
use tracing;
/// Cached monitor descriptions updated by the monitor watcher every 5s.
@@ -18,6 +30,55 @@ use tracing;
static CACHED_MONITOR_DESCRIPTIONS: Lazy<RwLock<Vec<String>>> =
Lazy::new(|| RwLock::new(Vec::new()));
+#[cfg(any(target_os = "linux", test))]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum WaylandCaptureMode {
+ Auto,
+ Grim,
+ Xcap,
+}
+
+#[cfg(any(target_os = "linux", test))]
+fn parse_wayland_capture_mode(value: Option<&str>) -> WaylandCaptureMode {
+ match value
+ .map(str::trim)
+ .filter(|value| !value.is_empty())
+ .map(str::to_ascii_lowercase)
+ .as_deref()
+ {
+ Some("grim" | "wlr" | "wlroots") => WaylandCaptureMode::Grim,
+ Some("xcap" | "dbus" | "portal" | "off" | "false" | "0") => WaylandCaptureMode::Xcap,
+ _ => WaylandCaptureMode::Auto,
+ }
+}
+
+#[cfg(any(target_os = "linux", test))]
+fn is_wayland_session(session_type: Option<&str>, wayland_display: Option<&str>) -> bool {
+ session_type.is_some_and(|value| value.eq_ignore_ascii_case("wayland"))
+ || wayland_display.is_some_and(|value| !value.trim().is_empty())
+}
+
+#[cfg(any(target_os = "linux", test))]
+fn desktop_prefers_grim(desktop_hint: &str) -> bool {
+ let normalized = desktop_hint.to_ascii_lowercase();
+ [
+ "dwl", "hyprland", "labwc", "niri", "river", "sway", "wayfire", "wlroots",
+ ]
+ .iter()
+ .any(|needle| normalized.contains(needle))
+}
+
+#[cfg(target_os = "linux")]
+static GRIM_CAPTURE_COUNTER: AtomicU64 = AtomicU64::new(0);
+#[cfg(target_os = "linux")]
+static GRIM_CAPTURE_LOG_ONCE: Once = Once::new();
+#[cfg(target_os = "linux")]
+static GRIM_CAPTURE_FALLBACK_LOG_ONCE: Once = Once::new();
+#[cfg(target_os = "linux")]
+static GRIM_AVAILABLE: Lazy<bool> = Lazy::new(|| command_in_path("grim"));
+#[cfg(target_os = "linux")]
+const GRIM_CAPTURE_TIMEOUT: Duration = Duration::from_secs(5);
+
/// macOS display capture is mediated by WindowServer/replayd. Serializing these
/// calls avoids concurrent multi-monitor spikes while preserving capture order.
#[cfg(target_os = "macos")]
@@ -174,6 +235,89 @@ pub mod macos_version {
#[cfg(target_os = "macos")]
use macos_version::use_sck_rs;
+#[cfg(target_os = "linux")]
+fn command_in_path(command: &str) -> bool {
+ std::env::var_os("PATH").is_some_and(|paths| {
+ std::env::split_paths(&paths).any(|dir| {
+ let candidate = dir.join(command);
+ candidate.is_file()
+ && candidate
+ .metadata()
+ .map(|metadata| metadata.permissions().mode() & 0o111 != 0)
+ .unwrap_or(false)
+ })
+ })
+}
+
+#[cfg(target_os = "linux")]
+fn current_desktop_hint() -> String {
+ [
+ std::env::var("XDG_CURRENT_DESKTOP").ok(),
+ std::env::var("XDG_SESSION_DESKTOP").ok(),
+ std::env::var("DESKTOP_SESSION").ok(),
+ ]
+ .into_iter()
+ .flatten()
+ .collect::<Vec<_>>()
+ .join(":")
+}
+
+#[cfg(target_os = "linux")]
+fn should_try_grim_capture() -> bool {
+ if !is_wayland_session(
+ std::env::var("XDG_SESSION_TYPE").ok().as_deref(),
+ std::env::var("WAYLAND_DISPLAY").ok().as_deref(),
+ ) {
+ return false;
+ }
+
+ match parse_wayland_capture_mode(std::env::var("SCREENPIPE_WAYLAND_CAPTURE").ok().as_deref()) {
+ WaylandCaptureMode::Grim => true,
+ WaylandCaptureMode::Xcap => false,
+ WaylandCaptureMode::Auto => {
+ *GRIM_AVAILABLE && desktop_prefers_grim(&current_desktop_hint())
+ }
+ }
+}
+
+#[cfg(any(target_os = "linux", test))]
+fn grim_geometry(data: &MonitorData) -> String {
+ format!("{},{} {}x{}", data.x, data.y, data.width, data.height)
+}
+
+#[cfg(target_os = "linux")]
+fn wait_with_timeout(
+ child: &mut std::process::Child,
+ timeout: Duration,
+) -> std::io::Result<std::process::ExitStatus> {
+ let started = Instant::now();
+ loop {
+ if let Some(status) = child.try_wait()? {
+ return Ok(status);
+ }
+
+ if started.elapsed() >= timeout {
+ let _ = child.kill();
+ let _ = child.wait();
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::TimedOut,
+ format!("process timed out after {timeout:?}"),
+ ));
+ }
+
+ std::thread::sleep(Duration::from_millis(10));
+ }
+}
+
+#[cfg(target_os = "linux")]
+fn read_child_stderr(child: &mut std::process::Child) -> String {
+ let mut stderr = String::new();
+ if let Some(mut pipe) = child.stderr.take() {
+ let _ = pipe.read_to_string(&mut stderr);
+ }
+ stderr.trim().to_string()
+}
+
impl SafeMonitor {
// macOS: Create from sck-rs monitor
#[cfg(target_os = "macos")]
@@ -488,7 +632,23 @@ impl SafeMonitor {
#[cfg(not(target_os = "windows"))]
{
let cached_idx = self.cached_monitor_index.clone();
+ let monitor_data = self.monitor_data.as_ref().clone();
let image = tokio::task::spawn_blocking(move || -> Result<DynamicImage> {
+ #[cfg(target_os = "linux")]
+ if should_try_grim_capture() {
+ match Self::capture_with_grim(monitor_id, &monitor_data) {
+ Ok(image) => return Ok(image),
+ Err(err) => {
+ GRIM_CAPTURE_FALLBACK_LOG_ONCE.call_once(|| {
+ tracing::warn!(
+ "grim Wayland capture failed ({}); falling back to xcap",
+ err
+ );
+ });
+ }
+ }
+ }
+
Self::per_frame_capture_with_cache(monitor_id, cached_idx)
})
.await
@@ -550,6 +710,57 @@ impl SafeMonitor {
.map(DynamicImage::ImageRgba8)
}
+ #[cfg(target_os = "linux")]
+ fn capture_with_grim(monitor_id: u32, data: &MonitorData) -> Result<DynamicImage> {
+ if data.width == 0 || data.height == 0 {
+ return Err(anyhow::anyhow!("invalid monitor dimensions"));
+ }
+
+ GRIM_CAPTURE_LOG_ONCE.call_once(|| {
+ tracing::info!(
+ "using grim for Wayland screen capture on wlroots-compatible compositor"
+ );
+ });
+
+ let geometry = grim_geometry(data);
+ let sequence = GRIM_CAPTURE_COUNTER.fetch_add(1, AtomicOrdering::Relaxed);
+ let path = std::env::temp_dir().join(format!(
+ "screenpipe-grim-{}-{monitor_id}-{sequence}.png",
+ std::process::id()
+ ));
+
+ let result = (|| -> Result<DynamicImage> {
+ let mut child = Command::new("grim")
+ .arg("-g")
+ .arg(&geometry)
+ .arg("-t")
+ .arg("png")
+ .arg(&path)
+ .stdout(Stdio::null())
+ .stderr(Stdio::piped())
+ .spawn()
+ .map_err(|e| anyhow::anyhow!("failed to spawn grim: {e}"))?;
+
+ let status = wait_with_timeout(&mut child, GRIM_CAPTURE_TIMEOUT)
+ .map_err(|e| anyhow::anyhow!("grim capture failed: {e}"))?;
+ let stderr = read_child_stderr(&mut child);
+ if !status.success() {
+ return Err(anyhow::anyhow!(
+ "grim exited with status {status}; stderr: {stderr}"
+ ));
+ }
+
+ let bytes = std::fs::read(&path)
+ .map_err(|e| anyhow::anyhow!("failed to read grim output: {e}"))?;
+ image::load_from_memory(&bytes)
+ .map(|image| DynamicImage::ImageRgba8(image.to_rgba8()))
+ .map_err(|e| anyhow::anyhow!("failed to decode grim output: {e}"))
+ })();
+
+ let _ = std::fs::remove_file(&path);
+ result
+ }
+
/// Refresh the cached monitor handle by re-enumerating all monitors.
/// Call this after a capture failure — the monitor may have been reconnected
/// with different properties (resolution change, etc.).
@@ -1214,6 +1425,63 @@ mod tests {
assert_eq!(prefix, prefix2);
}
+ #[test]
+ fn test_wayland_session_detection() {
+ assert!(is_wayland_session(Some("wayland"), None));
+ assert!(is_wayland_session(Some("Wayland"), None));
+ assert!(is_wayland_session(Some("x11"), Some("wayland-1")));
+ assert!(!is_wayland_session(Some("x11"), None));
+ assert!(!is_wayland_session(None, Some("")));
+ }
+
+ #[test]
+ fn test_wayland_capture_mode_parsing() {
+ assert_eq!(parse_wayland_capture_mode(None), WaylandCaptureMode::Auto);
+ assert_eq!(
+ parse_wayland_capture_mode(Some("")),
+ WaylandCaptureMode::Auto
+ );
+ assert_eq!(
+ parse_wayland_capture_mode(Some("grim")),
+ WaylandCaptureMode::Grim
+ );
+ assert_eq!(
+ parse_wayland_capture_mode(Some("wlroots")),
+ WaylandCaptureMode::Grim
+ );
+ assert_eq!(
+ parse_wayland_capture_mode(Some("xcap")),
+ WaylandCaptureMode::Xcap
+ );
+ assert_eq!(
+ parse_wayland_capture_mode(Some("portal")),
+ WaylandCaptureMode::Xcap
+ );
+ }
+
+ #[test]
+ fn test_wlroots_desktops_prefer_grim() {
+ assert!(desktop_prefers_grim("Hyprland"));
+ assert!(desktop_prefers_grim("niri"));
+ assert!(desktop_prefers_grim("sway:wlroots"));
+ assert!(desktop_prefers_grim("river"));
+ assert!(!desktop_prefers_grim("GNOME"));
+ assert!(!desktop_prefers_grim("KDE"));
+ }
+
+ #[test]
+ fn test_grim_geometry_format() {
+ let data = MonitorData {
+ width: 2560,
+ height: 1440,
+ x: -2560,
+ y: 0,
+ name: "left".to_string(),
+ is_primary: false,
+ };
+ assert_eq!(grim_geometry(&data), "-2560,0 2560x1440");
+ }
+
#[cfg(target_os = "windows")]
#[test]
fn test_persistent_capture_disables_after_three_failures() {