YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix: avoid wedged SCK topology workers (#6733)
Ezra Ellette7d815e952f3f
diff --git a/Cargo.lock b/Cargo.lockindex 0c7958f9a..f3aea878d 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -8496,7 +8496,7 @@ dependencies = [[[package]]name = "sck-rs"version = "0.1.0"-source = "git+https://github.com/screenpipe/sck-rs?rev=db22c13ff506d4d289a8bbcd1408640d1c1c119f#db22c13ff506d4d289a8bbcd1408640d1c1c119f"+source = "git+https://github.com/screenpipe/sck-rs?rev=758ee8b826763cba68da6eb3d204e8ba2d357109#758ee8b826763cba68da6eb3d204e8ba2d357109"dependencies = ["cidre 0.16.1 (git+https://github.com/yury/cidre.git?rev=9c20f5ee5abaad4598843a33e13d6e48822d1f30)","image",diff --git a/crates/screenpipe-engine/src/vision_manager/monitor_watcher.rs b/crates/screenpipe-engine/src/vision_manager/monitor_watcher.rsindex c1ad8b138..3bb59e330 100644--- a/crates/screenpipe-engine/src/vision_manager/monitor_watcher.rs+++ b/crates/screenpipe-engine/src/vision_manager/monitor_watcher.rs@@ -412,7 +412,7 @@ fn restart_cooldown_elapsed(}#[cfg(target_os = "macos")]-async fn wait_for_stopped_vision_retry(+async fn wait_for_monitor_retry(unlock: &tokio::sync::Notify,screen_locked: bool,unlocked_backstop: Duration,@@ -811,12 +811,25 @@ pub async fn start_monitor_watcher(}// ── Normal monitor polling ──────────────────────────────────────+ // A display reconfiguration while locked still invalidates the+ // topology cache, but SCK cannot provide useful fresh topology in+ // that state. Keep the cache dirty and wait for the exact unlock+ // transition instead of turning the reconfiguration notification+ // (or 60s backstop) into a replayd retry loop.+ #[cfg(target_os = "macos")]+ if crate::sleep_monitor::screen_is_locked() {+ let unlock = crate::sleep_monitor::screen_unlock_notify();+ let _ = wait_for_monitor_retry(unlock, true, Duration::from_secs(5)).await;+ info!("screen unlocked — refreshing monitor topology");+ continue;+ }+// If stopped (e.g. no monitors after undock/wake), retry start().if vision_manager.status().await != VisionManagerStatus::Running {#[cfg(target_os = "macos")]{let unlock = crate::sleep_monitor::screen_unlock_notify();- if wait_for_stopped_vision_retry(+ if wait_for_monitor_retry(unlock,crate::sleep_monitor::screen_is_locked(),Duration::from_secs(5),@@ -1286,9 +1299,9 @@ mod tests {#[cfg(target_os = "macos")]#[tokio::test]- async fn locked_vision_retry_waits_for_the_unlock_event() {+ async fn locked_monitor_retry_waits_for_the_unlock_event() {let unlock = tokio::sync::Notify::new();- let wait = wait_for_stopped_vision_retry(&unlock, true, Duration::from_millis(1));+ let wait = wait_for_monitor_retry(&unlock, true, Duration::from_millis(1));tokio::pin!(wait);assert!(diff --git a/crates/screenpipe-screen/Cargo.toml b/crates/screenpipe-screen/Cargo.tomlindex cfe590ce8..d9c378489 100644--- a/crates/screenpipe-screen/Cargo.toml+++ b/crates/screenpipe-screen/Cargo.toml@@ -82,7 +82,7 @@ harness = false# macOS: sck-rs for 12.3+ (ScreenCaptureKit), xcap as fallback for older versions[target.'cfg(target_os = "macos")'.dependencies]-sck-rs = { git = "https://github.com/screenpipe/sck-rs" , rev = "db22c13ff506d4d289a8bbcd1408640d1c1c119f"}+sck-rs = { git = "https://github.com/screenpipe/sck-rs" , rev = "758ee8b826763cba68da6eb3d204e8ba2d357109"}xcap = { workspace = true }libc = { workspace = true }cidre = { git = "https://github.com/yury/cidre.git", rev = "8481f3f0dc7dd39bb5be80d9424bfac0cad3801a", features = ["vn", "cv", "cg", "app"] }diff --git a/crates/screenpipe-screen/src/monitor/macos.rs b/crates/screenpipe-screen/src/monitor/macos.rsindex 51bb2ab94..99cda1076 100644--- a/crates/screenpipe-screen/src/monitor/macos.rs+++ b/crates/screenpipe-screen/src/monitor/macos.rs@@ -35,12 +35,11 @@ const MACOS_CAPTURE_TIMEOUT: Duration = Duration::from_secs(12);const MACOS_CG_FALLBACK_TIMEOUT: Duration = Duration::from_secs(2);/// ScreenCaptureKit can stop delivering the `SCShareableContent` completion-/// callback after a capture teardown or update. Serialize healthy calls, but-/// reserve one worker for a fresh retry after the first Apple callback wedges.+/// callback after a capture teardown or update. Serialize healthy calls; the+/// async sck-rs API makes a timed-out enumeration cancellable without leaving+/// a blocking worker behind.static SCK_MONITOR_ENUMERATION_SERIALIZER: Lazy<tokio::sync::Mutex<()>> =Lazy::new(|| tokio::sync::Mutex::new(()));-static SCK_MONITOR_ENUMERATION_WORKERS: Lazy<Arc<tokio::sync::Semaphore>> =- Lazy::new(|| Arc::new(tokio::sync::Semaphore::new(2)));static XCAP_MONITOR_ENUMERATION_SERIALIZER: Lazy<tokio::sync::Mutex<()>> =Lazy::new(|| tokio::sync::Mutex::new(()));static XCAP_MONITOR_ENUMERATION_WORKERS: Lazy<Arc<tokio::sync::Semaphore>> =@@ -208,7 +207,7 @@ where{// Bounding only the holder leaves the wait unbounded, so a queued caller// inherits the sum of every holder ahead of it. See the same repair and- // reasoning in `run_bounded_sck_enumeration`; this is the capture-path twin+ // reasoning in `run_async_sck_enumeration`; this is the capture-path twin// and sits directly under the capture loop's own bounded await.// See tests::capture_wait_stays_bounded_when_callers_queue.let Ok(_serial_guard) = tokio::time::timeout(timeout, serializer.lock()).await else {@@ -466,28 +465,18 @@ fn is_clamshell_inactive_builtin(display_id: u32) -> bool {screenpipe_core::display_topology::is_clamshell_inactive_builtin(display_id)}-async fn run_bounded_sck_enumeration<T, F>(+async fn run_async_sck_enumeration<T, F, Fut>(serializer: &tokio::sync::Mutex<()>,- workers: Arc<tokio::sync::Semaphore>,timeout: Duration,enumerate: F,) -> std::result::Result<T, MonitorListError>where- T: Send + 'static,- F: FnOnce() -> std::result::Result<T, MonitorListError> + Send + 'static,+ F: FnOnce() -> Fut,+ Fut: std::future::Future<Output = std::result::Result<T, MonitorListError>>,{- // Healthy enumeration stays single-file. After a timeout this guard is- // released, allowing one genuinely fresh SCK request to recover capture.- //- // Bounding only the holder is not enough. Each holder is capped at- // `timeout`, but an unbounded wait here lets a queued caller inherit the- // sum of every holder ahead of it: with callbacks that are slow yet do- // return, permits recycle, nobody fast-fails, and the Nth caller waits- // N * timeout. That is the #3939 shape — a 250ms-bounded await in the- // capture loop reported frozen for 73-299s, which is 5-20 holders deep at- // the 15s production timeout. Cap the wait so a caller never blocks for- // more than roughly twice the timeout it asked for.- // See tests::enumeration_wait_stays_bounded_when_callers_queue.+ // Bound both the serializer queue and the Apple callback. Dropping the+ // async sck-rs future cancels our wait while cidre keeps any escaping+ // Objective-C block state alive for a late callback.let _serial_guard = match tokio::time::timeout(timeout, serializer.lock()).await {Ok(guard) => guard,Err(_) => {@@ -498,32 +487,8 @@ where}};- // A timed-out spawn_blocking task cannot be cancelled because the Apple- // callback is outside Rust's control. Two permits bound the damage while- // still leaving capacity for one fresh recovery attempt.- let permit = workers.try_acquire_owned().map_err(|e| match e {- tokio::sync::TryAcquireError::NoPermits => MonitorListError::Other(- "ScreenCaptureKit monitor enumeration retry budget exhausted; two Apple callbacks remain blocked, restart screenpipe to recover"- .to_string(),- ),- tokio::sync::TryAcquireError::Closed => MonitorListError::Other(- "ScreenCaptureKit monitor enumeration worker pool closed".to_string(),- ),- })?;-- let task = tokio::task::spawn_blocking(move || {- // Keep the permit inside the blocking task. If the async caller times- // out, the OS call may still be running and must continue to consume- // one of the two bounded worker slots until it really returns.- let _permit = permit;- enumerate()- });-- match tokio::time::timeout(timeout, task).await {- Ok(Ok(result)) => result,- Ok(Err(e)) => Err(MonitorListError::Other(format!(- "macOS monitor enumeration task failed: {e}"- ))),+ match tokio::time::timeout(timeout, enumerate()).await {+ Ok(result) => result,Err(_) => {let message = format!("macOS monitor enumeration timed out after {}s; ScreenCaptureKit/replayd did not reply",@@ -535,7 +500,7 @@ where}}-fn enumerate_sck_monitors() -> std::result::Result<Vec<SafeMonitor>, MonitorListError> {+async fn enumerate_sck_monitors() -> std::result::Result<Vec<SafeMonitor>, MonitorListError> {#[cfg(feature = "e2e")]if std::env::var("SCREENPIPE_E2E_SCK_RELAUNCH_POISONED").ok()@@ -559,13 +524,11 @@ fn enumerate_sck_monitors() -> std::result::Result<Vec<SafeMonitor>, MonitorList&& !SCK_E2E_HANG_INJECTED.swap(true, std::sync::atomic::Ordering::SeqCst){tracing::warn!("e2e: injecting one blocked ScreenCaptureKit monitor enumeration callback");- loop {- std::thread::park();- }+ std::future::pending::<()>().await;}tracing::debug!("Using sck-rs for screen capture (macOS 12.3+)");- match SckMonitor::all() {+ match SckMonitor::all_async().await {Ok(monitors) if monitors.is_empty() => Err(MonitorListError::NoMonitorsFound),Ok(monitors) => Ok(monitors.into_iter()@@ -619,7 +582,8 @@ pub fn e2e_arm_sck_lookup_hang_fault() -> bool {}#[cfg(debug_assertions)]-fn enumerate_sck_monitors_for_lookup() -> std::result::Result<Vec<SafeMonitor>, MonitorListError> {+async fn enumerate_sck_monitors_for_lookup(+) -> std::result::Result<Vec<SafeMonitor>, MonitorListError> {if e2e_lookup_hang_enabled()&& SCK_E2E_LOOKUP_HANG_ARMED.load(std::sync::atomic::Ordering::SeqCst)&& !SCK_E2E_LOOKUP_HANG_INJECTED.swap(true, std::sync::atomic::Ordering::SeqCst)@@ -631,16 +595,15 @@ fn enumerate_sck_monitors_for_lookup() -> std::result::Result<Vec<SafeMonitor>,b"1",);}- loop {- std::thread::park();- }+ std::future::pending::<()>().await;}- enumerate_sck_monitors()+ enumerate_sck_monitors().await}#[cfg(not(debug_assertions))]-fn enumerate_sck_monitors_for_lookup() -> std::result::Result<Vec<SafeMonitor>, MonitorListError> {- enumerate_sck_monitors()+async fn enumerate_sck_monitors_for_lookup(+) -> std::result::Result<Vec<SafeMonitor>, MonitorListError> {+ enumerate_sck_monitors().await}/// How long a successful enumeration may answer `get_monitor_by_id`.@@ -863,15 +826,14 @@ fn sck_monitor_error_allows_fallback(error: &MonitorListError) -> bool {/// List monitors with detailed error information (permission denied vs no monitors)pub async fn list_monitors_detailed() -> std::result::Result<Vec<SafeMonitor>, MonitorListError> {- // Wrap the ObjC call paths in an autorelease pool. Tokio blocking workers- // are long-lived; without a per-call drain these objects accumulate.- // See monitor::tests::repro_list_monitors_autorelease_leak.+ // sck-rs initiates the callback and converts the returned topology inside+ // scoped autorelease pools. The legacy xcap path retains its blocking+ // worker pool below.let result: std::result::Result<Vec<SafeMonitor>, MonitorListError> = if use_sck_rs() {- let sck_result = run_bounded_sck_enumeration(+ let sck_result = run_async_sck_enumeration(&SCK_MONITOR_ENUMERATION_SERIALIZER,- SCK_MONITOR_ENUMERATION_WORKERS.clone(),MONITOR_ENUMERATION_TIMEOUT,- || cidre::objc::ar_pool(enumerate_sck_monitors),+ enumerate_sck_monitors,).await;match sck_result {@@ -912,11 +874,10 @@ pub async fn list_monitors_cached() -> Vec<SafeMonitor> {pub async fn get_default_monitor() -> Option<SafeMonitor> {if use_sck_rs() {- let sck_result = run_bounded_sck_enumeration(+ let sck_result = run_async_sck_enumeration(&SCK_MONITOR_ENUMERATION_SERIALIZER,- SCK_MONITOR_ENUMERATION_WORKERS.clone(),monitor_lookup_timeout(),- || cidre::objc::ar_pool(enumerate_sck_monitors_for_lookup),+ enumerate_sck_monitors_for_lookup,).await;match sck_result {@@ -944,11 +905,10 @@ pub async fn get_monitor_by_id(id: u32) -> Option<SafeMonitor> {return Some(monitor);}if use_sck_rs() {- match run_bounded_sck_enumeration(+ match run_async_sck_enumeration(&SCK_MONITOR_ENUMERATION_SERIALIZER,- SCK_MONITOR_ENUMERATION_WORKERS.clone(),monitor_lookup_timeout(),- || cidre::objc::ar_pool(enumerate_sck_monitors_for_lookup),+ enumerate_sck_monitors_for_lookup,).await{@@ -1274,18 +1234,9 @@ mod tests {#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn monitor_enumeration_timeout_allows_one_bounded_retry() {let serializer = tokio::sync::Mutex::new(());- let workers = Arc::new(tokio::sync::Semaphore::new(2));- let (release_tx, release_rx) = std::sync::mpsc::channel();-- let first = run_bounded_sck_enumeration(- &serializer,- workers.clone(),- Duration::from_millis(25),- move || {- release_rx.recv().expect("release first enumeration");- Ok(1u8)- },- )+ let first = run_async_sck_enumeration(&serializer, Duration::from_millis(25), || async {+ std::future::pending::<std::result::Result<u8, MonitorListError>>().await+ }).await;assert!(matches!(first,@@ -1293,108 +1244,32 @@ mod tests {if message.contains("ScreenCaptureKit/replayd did not reply")));- let second = run_bounded_sck_enumeration(- &serializer,- workers.clone(),- Duration::from_secs(1),- || Ok(2u8),- )- .await;- assert!(matches!(second, Ok(2)), "a fresh retry should recover");-- release_tx.send(()).expect("unblock first enumeration");- let recovered =- run_bounded_sck_enumeration(&serializer, workers, Duration::from_secs(1), || Ok(3u8))+ let second =+ run_async_sck_enumeration(&serializer, Duration::from_secs(1), || async { Ok(2u8) }).await;- assert!(matches!(recovered, Ok(3)));- }-- #[tokio::test(flavor = "multi_thread", worker_threads = 2)]- async fn monitor_enumeration_never_exceeds_two_blocked_workers() {- let serializer = tokio::sync::Mutex::new(());- let workers = Arc::new(tokio::sync::Semaphore::new(2));- let (release_first_tx, release_first_rx) = std::sync::mpsc::channel();- let (release_second_tx, release_second_rx) = std::sync::mpsc::channel();-- for release_rx in [release_first_rx, release_second_rx] {- let result = run_bounded_sck_enumeration(- &serializer,- workers.clone(),- Duration::from_millis(25),- move || {- release_rx.recv().expect("release blocked enumeration");- Ok(1u8)- },- )- .await;- assert!(matches!(- result,- Err(MonitorListError::Other(ref message))- if message.contains("ScreenCaptureKit/replayd did not reply")- ));- }-- let third_ran = Arc::new(AtomicBool::new(false));- let third_ran_in_task = third_ran.clone();- let third = run_bounded_sck_enumeration(- &serializer,- workers.clone(),- Duration::from_secs(1),- move || {- third_ran_in_task.store(true, Ordering::SeqCst);- Ok(3u8)- },- )- .await;- assert!(matches!(- third,- Err(MonitorListError::Other(ref message))- if message.contains("retry budget exhausted")- ));- assert!(!third_ran.load(Ordering::SeqCst));-- release_first_tx.send(()).expect("release first worker");- release_second_tx.send(()).expect("release second worker");- tokio::time::timeout(Duration::from_secs(1), async {- while workers.available_permits() < 2 {- tokio::time::sleep(Duration::from_millis(5)).await;- }- })- .await- .expect("blocked worker permits should be released");- let recovered =- run_bounded_sck_enumeration(&serializer, workers, Duration::from_secs(1), || Ok(4u8))- .await;- assert!(matches!(recovered, Ok(4)));+ assert!(matches!(second, Ok(2)), "a fresh retry should recover");}/// Production #3939 freezes report 73-299s frozen in a 250ms-bounded await.- /// `run_bounded_sck_enumeration` bounds the *holder* at `timeout`, but the- /// wait for `serializer` is unbounded, so queued callers inherit the sum of- /// every holder ahead of them. With callbacks that are slow but do return,- /// permits recycle, nobody fast-fails, and the queue grows without limit.+ /// The enumeration wrapper must bound both the holder and the serializer+ /// wait so queued callers cannot inherit every deadline ahead of them.////// A caller must never wait for more than a small multiple of the timeout/// it asked for, however many callers are queued.#[tokio::test(flavor = "multi_thread", worker_threads = 4)]async fn enumeration_wait_stays_bounded_when_callers_queue() {const TIMEOUT: Duration = Duration::from_millis(100);- // Longer than TIMEOUT so every holder times out, but finite so the- // permit recycles and later callers never hit the retry budget.const CALLBACK: Duration = Duration::from_millis(160);const CALLERS: usize = 6;let serializer: &'static tokio::sync::Mutex<()> =Box::leak(Box::new(tokio::sync::Mutex::new(())));- let workers = Arc::new(tokio::sync::Semaphore::new(2));-let started = Instant::now();let mut handles = Vec::new();for _ in 0..CALLERS {- let workers = workers.clone();handles.push(tokio::spawn(async move {- let _ = run_bounded_sck_enumeration(serializer, workers, TIMEOUT, move || {- std::thread::sleep(CALLBACK);+ let _ = run_async_sck_enumeration(serializer, TIMEOUT, move || async move {+ tokio::time::sleep(CALLBACK).await;Ok(1u8)}).await;@@ -1407,14 +1282,13 @@ mod tests {worst = worst.max(handle.await.expect("caller task panicked"));}- // Two permits let two holders overlap, so ~2x TIMEOUT is the honest- // ceiling for a bounded design. Allow 3x for scheduling slack.+ // One timeout for the active holder, one for this caller's own future,+ // plus scheduling slack on loaded CI hosts.let ceiling = TIMEOUT * 3;assert!(worst <= ceiling,"queued caller waited {worst:?} for a {TIMEOUT:?} bounded call \- ({CALLERS} callers, ceiling {ceiling:?}) — the serializer wait is unbounded, \- so waiters inherit every holder ahead of them"+ ({CALLERS} callers, ceiling {ceiling:?})");}@@ -1464,53 +1338,6 @@ mod tests {);}- /// The other regime, for contrast: when callbacks never return, both- /// permits stay pinned and later callers fast-fail on the retry budget- /// instead of queueing. This caps the wait at ~2x timeout, which means a- /// permanent wedge alone cannot explain a multi-minute production freeze.- #[tokio::test(flavor = "multi_thread", worker_threads = 4)]- async fn enumeration_fast_fails_once_both_permits_are_pinned() {- const TIMEOUT: Duration = Duration::from_millis(100);- const CALLERS: usize = 6;-- let serializer: &'static tokio::sync::Mutex<()> =- Box::leak(Box::new(tokio::sync::Mutex::new(())));- let workers = Arc::new(tokio::sync::Semaphore::new(2));- let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();- let release_rx = Arc::new(std::sync::Mutex::new(release_rx));-- let started = Instant::now();- let mut handles = Vec::new();- for _ in 0..CALLERS {- let workers = workers.clone();- let release_rx = release_rx.clone();- handles.push(tokio::spawn(async move {- let _ = run_bounded_sck_enumeration(serializer, workers, TIMEOUT, move || {- // Park until the test releases us; models an Apple callback- // that never comes back.- let _ = release_rx.lock().expect("release lock").recv();- Ok(1u8)- })- .await;- started.elapsed()- }));- }-- let mut worst = Duration::ZERO;- for handle in handles {- worst = worst.max(handle.await.expect("caller task panicked"));- }- for _ in 0..CALLERS {- let _ = release_tx.send(());- }-- assert!(- worst <= TIMEOUT * 4,- "with both permits pinned every later caller should fast-fail, \- but the worst wait was {worst:?}"- );- }-#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn capture_timeout_keeps_permit_and_caps_abandoned_workers() {let serializer = tokio::sync::Mutex::new(());@@ -1650,14 +1477,11 @@ mod tests {let baseline = peak_rss_bytes();eprintln!("[repro] baseline peak RSS: {}", fmt_mb(baseline));- // -- Phase 1: drive the code path AS-WRITTEN (should leak) --- // We call `SckMonitor::all()` directly on a tokio blocking worker,- // matching what `list_monitors_detailed` does today.+ // -- Phase 1: reproduce the historical unpooled sync path --let before_phase1 = peak_rss_bytes();for _ in 0..N {tokio::task::spawn_blocking(|| {- // No ar_pool — matches current production path in- // list_monitors_detailed on macOS.+ // No ar_pool — the pre-fix production shape.let _ = sck_rs::Monitor::all();}).await