Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(windows): make WGC copies demand-driven (#5154)

Ezra Ellettef2f6692a2542
diff --git a/crates/screenpipe-screen/examples/wgc_cpu_probe.rs b/crates/screenpipe-screen/examples/wgc_cpu_probe.rs
index 581b6b503..a8a9152f1 100644
--- a/crates/screenpipe-screen/examples/wgc_cpu_probe.rs
+++ b/crates/screenpipe-screen/examples/wgc_cpu_probe.rs
@@ -95,6 +95,7 @@ mod windows_main {
capture
.get_latest_image(Duration::from_secs(2))
.expect("failed to get warm-up frame");
+ let stats_start = capture.stats();
let sample_interval = Duration::from_millis(interval_ms);
let run_for = Duration::from_secs(duration_secs);
@@ -118,6 +119,7 @@ mod windows_main {
let cpu_elapsed = cpu_time_now() - cpu_start;
let wall_elapsed = wall_start.elapsed();
let cores = cpu_elapsed.as_secs_f64() / wall_elapsed.as_secs_f64();
+ let stats_end = capture.stats();
capture.stop();
@@ -128,5 +130,11 @@ mod windows_main {
cpu_elapsed.as_secs_f64()
);
println!("process CPU load: {:.1}% of one core", cores * 100.0);
+ println!(
+ "WGC callbacks: {}, GPU copies: {}, image requests: {}",
+ stats_end.frame_arrivals - stats_start.frame_arrivals,
+ stats_end.copy_submissions - stats_start.copy_submissions,
+ stats_end.image_requests - stats_start.image_requests
+ );
}
}
diff --git a/crates/screenpipe-screen/src/wgc_capture.rs b/crates/screenpipe-screen/src/wgc_capture.rs
index a87a79ffe..a14442563 100644
--- a/crates/screenpipe-screen/src/wgc_capture.rs
+++ b/crates/screenpipe-screen/src/wgc_capture.rs
@@ -13,10 +13,11 @@
//! See https://github.com/screenpipe/screenpipe/issues/4840.
//!
//! Design:
-//! - `FrameArrived` only does a GPU-side `CopyResource` into a persistent "latest"
-//! texture (cheap: a queued GPU command, no CPU readback). The expensive staging
-//! texture + `Map` + swizzle work is deferred to `get_latest_image()`, which runs
-//! only when a caller actually wants a frame.
+//! - `get_latest_image()` publishes a monotonically increasing request generation.
+//! `FrameArrived` keeps draining compositor frames, but only does a GPU-side
+//! `CopyResource` when a request is pending. One copy can satisfy every caller that
+//! was waiting at that instant. The expensive staging texture + `Map` + swizzle
+//! work is likewise performed only for callers that actually want a frame.
//! - The D3D11 device is shared across sessions but recreatable: it is validated with
//! `GetDeviceRemovedReason` on every session init and dropped from the cache when a
//! GPU reset / driver update removes it, so the reinit loop in monitor/windows.rs
@@ -32,7 +33,7 @@
use anyhow::{anyhow, Result};
use image::{DynamicImage, RgbaImage};
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Condvar, Mutex, MutexGuard};
use std::time::{Duration, Instant};
use windows::{
@@ -67,14 +68,77 @@ use windows::{
},
};
-/// Caps the `FrameArrived` callback rate at ~60Hz. Purely a bonus on top of lazy
-/// readback: it reduces GPU `CopyResource` submissions on high-refresh-rate displays.
+/// Caps the `FrameArrived` callback rate at ~60Hz. Purely a bonus on top of
+/// demand-driven copying: it reduces callback overhead on high-refresh-rate displays.
/// Best-effort — `SetMinUpdateInterval` needs Win11 24H2+ and is a no-op if unsupported,
-/// since the callback is already cheap (one GPU copy) without it.
+/// since callbacks without a pending request only drain and inspect the frame.
const MIN_UPDATE_INTERVAL_100NS: i64 = 166_667;
const FRAME_POOL_BUFFERS: i32 = 2;
+/// Compact counters used by the Windows capture probe and future resource telemetry.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct CaptureStats {
+ pub frame_arrivals: u64,
+ pub copy_submissions: u64,
+ pub image_requests: u64,
+}
+
+#[derive(Default)]
+struct CaptureCounters {
+ frame_arrivals: AtomicU64,
+ copy_submissions: AtomicU64,
+ image_requests: AtomicU64,
+}
+
+impl CaptureCounters {
+ fn snapshot(&self) -> CaptureStats {
+ CaptureStats {
+ frame_arrivals: self.frame_arrivals.load(Ordering::Relaxed),
+ copy_submissions: self.copy_submissions.load(Ordering::Relaxed),
+ image_requests: self.image_requests.load(Ordering::Relaxed),
+ }
+ }
+}
+
+/// Generations make outstanding demand explicit and allow one frame to satisfy all
+/// callers that were waiting when it arrived. Access is protected by `LatestFrame`'s
+/// mutex, so no atomics are needed here.
+#[derive(Debug, Default)]
+struct CopyDemand {
+ requested: u64,
+ completed: u64,
+}
+
+impl CopyDemand {
+ fn request(&mut self) -> u64 {
+ if self.requested == u64::MAX {
+ // While the final generation is pending, additional callers can safely
+ // join it. Once complete, no waiter remains and both counters can reset.
+ if self.completed != u64::MAX {
+ return u64::MAX;
+ }
+ self.requested = 0;
+ self.completed = 0;
+ }
+ self.requested += 1;
+ self.requested
+ }
+
+ fn pending_generation(&self) -> Option<u64> {
+ (self.requested != self.completed).then_some(self.requested)
+ }
+
+ fn complete(&mut self, generation: u64) {
+ debug_assert_eq!(generation, self.requested);
+ self.completed = generation;
+ }
+
+ fn is_complete(&self, generation: u64) -> bool {
+ self.completed >= generation
+ }
+}
+
/// Shared D3D11 device state for all persistent captures. One device serves every
/// monitor's session; it lives in `SHARED_D3D` so it can be dropped and recreated
/// after device removal, unlike a `LazyLock` (which would also poison forever if the
@@ -288,6 +352,7 @@ struct LatestFrame {
/// differs are stale pre-`Recreate` deliveries and are dropped; frames whose
/// `ContentSize` differs signal a display mode change and trigger a `Recreate`.
pool_size: (i32, i32),
+ demand: CopyDemand,
}
/// Persistent WGC capture session that keeps a single GraphicsCaptureSession alive,
@@ -300,6 +365,7 @@ pub struct PersistentCapture {
frame_arrived_token: EventRegistrationToken,
closed_token: EventRegistrationToken,
latest: Arc<(Mutex<LatestFrame>, Condvar)>,
+ stats: Arc<CaptureCounters>,
/// Set when the capture item reports `Closed` (monitor disconnect, session
/// invalidated) or `stop()` runs. Checked *before* the cached frame in
/// `get_latest_image` — a closed session errors out (triggering reinit in
@@ -346,14 +412,17 @@ impl PersistentCapture {
width: 0,
height: 0,
pool_size: (item_size.Width, item_size.Height),
+ demand: CopyDemand::default(),
}),
Condvar::new(),
));
let closed = Arc::new(AtomicBool::new(false));
+ let stats = Arc::new(CaptureCounters::default());
let handler_latest = latest.clone();
let handler_closed = closed.clone();
let handler_d3d = d3d.clone();
+ let handler_stats = stats.clone();
let frame_arrived_token = frame_pool
.FrameArrived(
&TypedEventHandler::<Direct3D11CaptureFramePool, IInspectable>::new(
@@ -364,6 +433,7 @@ impl PersistentCapture {
&handler_d3d,
&handler_latest,
&handler_closed,
+ &handler_stats,
) {
tracing::debug!("wgc FrameArrived: {}", e);
}
@@ -457,6 +527,7 @@ impl PersistentCapture {
frame_arrived_token,
closed_token,
latest,
+ stats,
closed,
stopped: AtomicBool::new(false),
staging: Mutex::new(None),
@@ -469,12 +540,14 @@ impl PersistentCapture {
d3d: &Arc<D3dContext>,
latest: &Arc<(Mutex<LatestFrame>, Condvar)>,
closed: &Arc<AtomicBool>,
+ stats: &Arc<CaptureCounters>,
) -> Result<()> {
+ stats.frame_arrivals.fetch_add(1, Ordering::Relaxed);
let frame = frame_pool
.TryGetNextFrame()
.map_err(|e| anyhow!("TryGetNextFrame failed: {}", e))?;
- let stored = Self::store_frame(&frame, frame_pool, d3d, latest, closed);
+ let stored = Self::store_frame(&frame, frame_pool, d3d, latest, closed, stats);
if let Err(e) = frame.Close() {
tracing::debug!("Direct3D11CaptureFrame::Close failed: {:?}", e);
@@ -495,30 +568,11 @@ impl PersistentCapture {
d3d: &Arc<D3dContext>,
latest: &Arc<(Mutex<LatestFrame>, Condvar)>,
closed: &Arc<AtomicBool>,
+ stats: &Arc<CaptureCounters>,
) -> Result<bool> {
let content = frame
.ContentSize()
.map_err(|e| anyhow!("ContentSize failed: {}", e))?;
- let surface = frame
- .Surface()
- .map_err(|e| anyhow!("Direct3D11CaptureFrame::Surface failed: {}", e))?;
- let access = surface
- .cast::<IDirect3DDxgiInterfaceAccess>()
- .map_err(|e| {
- anyhow!(
- "failed to cast surface to IDirect3DDxgiInterfaceAccess: {}",
- e
- )
- })?;
- let source_texture: ID3D11Texture2D = unsafe {
- access
- .GetInterface()
- .map_err(|e| anyhow!("IDirect3DDxgiInterfaceAccess::GetInterface failed: {}", e))?
- };
-
- let mut desc = D3D11_TEXTURE2D_DESC::default();
- unsafe { source_texture.GetDesc(&mut desc) };
- let frame_size = (desc.Width as i32, desc.Height as i32);
let content_size = (content.Width, content.Height);
let mut slot = latest
@@ -526,11 +580,6 @@ impl PersistentCapture {
.lock()
.map_err(|_| anyhow!("latest frame mutex poisoned"))?;
- if frame_size != slot.pool_size {
- // Frame from a buffer generation before the last Recreate — drop it.
- return Ok(false);
- }
-
if content_size != slot.pool_size {
// Display mode changed (resolution/scaling): recreate the pool at the new
// size in place. Keeping the session alive avoids re-flashing the capture
@@ -562,10 +611,42 @@ impl PersistentCapture {
return Err(e);
}
slot.pool_size = content_size;
+ slot.texture = None;
+ slot.width = 0;
+ slot.height = 0;
// Skip this old-size frame; the next delivery uses the recreated buffers.
return Ok(false);
}
+ let Some(request_generation) = slot.demand.pending_generation() else {
+ return Ok(false);
+ };
+
+ let surface = frame
+ .Surface()
+ .map_err(|e| anyhow!("Direct3D11CaptureFrame::Surface failed: {}", e))?;
+ let access = surface
+ .cast::<IDirect3DDxgiInterfaceAccess>()
+ .map_err(|e| {
+ anyhow!(
+ "failed to cast surface to IDirect3DDxgiInterfaceAccess: {}",
+ e
+ )
+ })?;
+ let source_texture: ID3D11Texture2D = unsafe {
+ access
+ .GetInterface()
+ .map_err(|e| anyhow!("IDirect3DDxgiInterfaceAccess::GetInterface failed: {}", e))?
+ };
+
+ let mut desc = D3D11_TEXTURE2D_DESC::default();
+ unsafe { source_texture.GetDesc(&mut desc) };
+ let frame_size = (desc.Width as i32, desc.Height as i32);
+ if frame_size != slot.pool_size {
+ // Frame from a buffer generation before the last Recreate — drop it.
+ return Ok(false);
+ }
+
if slot.texture.is_none() || slot.width != desc.Width || slot.height != desc.Height {
let texture = create_texture_like(&d3d.device, &desc, D3D11_USAGE_DEFAULT, 0)
.map_err(|e| d3d.describe_err("create latest texture", e))?;
@@ -585,13 +666,15 @@ impl PersistentCapture {
let _context_guard = d3d.lock_context();
unsafe { d3d.context.CopyResource(&dst_resource, &src_resource) };
}
+ slot.demand.complete(request_generation);
+ stats.copy_submissions.fetch_add(1, Ordering::Relaxed);
Ok(true)
}
- /// Get the latest captured frame. On the first call after init, polls until a frame
- /// arrives or timeout expires. Errors as soon as the session is closed (monitor
- /// disconnect, sleep/wake invalidation, stop()) so monitor/windows.rs reinits
- /// rather than receiving stale frames.
+ /// Request and read a frame newer than this call. Waits until the callback fulfills
+ /// this request generation or the timeout expires. Errors as soon as the session is
+ /// closed (monitor disconnect, sleep/wake invalidation, stop()) so
+ /// monitor/windows.rs reinits rather than receiving stale frames.
///
/// This is where the CPU cost lives now: staging texture + `Map` readback +
/// BGRA->RGBA swizzle happen here, lazily, only when a frame is actually needed —
@@ -603,6 +686,14 @@ impl PersistentCapture {
.lock()
.map_err(|e| anyhow!("frame mutex poisoned: {}", e))?;
+ if self.closed.load(Ordering::Acquire) {
+ return Err(anyhow!(
+ "WGC session closed (capture item closed or stopped)"
+ ));
+ }
+ let request_generation = slot.demand.request();
+ self.stats.image_requests.fetch_add(1, Ordering::Relaxed);
+
loop {
// Closed takes precedence over any cached frame: a dead session must
// error (=> reinit upstream), never serve its last frame forever.
@@ -611,7 +702,7 @@ impl PersistentCapture {
"WGC session closed (capture item closed or stopped)"
));
}
- if slot.texture.is_some() {
+ if slot.demand.is_complete(request_generation) && slot.texture.is_some() {
break;
}
let now = Instant::now();
@@ -635,6 +726,10 @@ impl PersistentCapture {
self.readback(&texture, width, height)
}
+ pub fn stats(&self) -> CaptureStats {
+ self.stats.snapshot()
+ }
+
fn readback(&self, texture: &ID3D11Texture2D, width: u32, height: u32) -> Result<DynamicImage> {
let row_bytes = width as usize * 4;
let mut rgba = vec![0u8; row_bytes * height as usize];
@@ -762,6 +857,90 @@ mod tests {
use super::*;
use xcap::Monitor as XcapMonitor;
+ fn deliver_frame(demand: &mut CopyDemand) -> bool {
+ let Some(generation) = demand.pending_generation() else {
+ return false;
+ };
+ demand.complete(generation);
+ true
+ }
+
+ #[test]
+ fn frame_arrivals_without_demand_do_not_copy() {
+ let mut demand = CopyDemand::default();
+ assert_eq!((0..1_000).filter(|_| deliver_frame(&mut demand)).count(), 0);
+ }
+
+ #[test]
+ fn one_request_produces_exactly_one_copy() {
+ let mut demand = CopyDemand::default();
+ let generation = demand.request();
+
+ assert!(deliver_frame(&mut demand));
+ assert!(demand.is_complete(generation));
+ assert!(!deliver_frame(&mut demand));
+ }
+
+ #[test]
+ fn later_request_produces_one_additional_copy() {
+ let mut demand = CopyDemand::default();
+ let first = demand.request();
+ assert!(deliver_frame(&mut demand));
+ assert!(demand.is_complete(first));
+
+ let second = demand.request();
+ assert!(!demand.is_complete(second));
+ assert!(deliver_frame(&mut demand));
+ assert!(demand.is_complete(second));
+ assert!(!deliver_frame(&mut demand));
+ }
+
+ #[test]
+ fn one_copy_satisfies_concurrent_requests() {
+ let mut demand = CopyDemand::default();
+ let first = demand.request();
+ let second = demand.request();
+
+ assert!(deliver_frame(&mut demand));
+ assert!(demand.is_complete(first));
+ assert!(demand.is_complete(second));
+ assert!(!deliver_frame(&mut demand));
+ }
+
+ #[test]
+ fn abandoned_request_leaves_at_most_one_copy() {
+ let mut demand = CopyDemand::default();
+ let _abandoned_generation = demand.request();
+
+ assert!(deliver_frame(&mut demand));
+ assert!(!deliver_frame(&mut demand));
+ }
+
+ #[test]
+ fn completed_generations_wrap_only_without_waiters() {
+ let mut demand = CopyDemand {
+ requested: u64::MAX,
+ completed: u64::MAX,
+ };
+
+ let generation = demand.request();
+ assert_eq!(generation, 1);
+ assert!(deliver_frame(&mut demand));
+ assert!(demand.is_complete(generation));
+ }
+
+ #[test]
+ fn callers_join_the_final_generation_until_it_completes() {
+ let mut demand = CopyDemand {
+ requested: u64::MAX,
+ completed: u64::MAX - 1,
+ };
+
+ assert_eq!(demand.request(), u64::MAX);
+ assert!(deliver_frame(&mut demand));
+ assert!(demand.is_complete(u64::MAX));
+ }
+
/// Resolve the first monitor through xcap, exactly like production code in
/// monitor/windows.rs does. This also exercises the id contract that
/// find_hmonitor re-derives (xcap Monitor::id() == HMONITOR value as u32) —
@@ -799,8 +978,7 @@ mod tests {
"captured height != monitor height"
);
- // A second call should hit the already-populated "latest" slot immediately
- // (no waiting for a fresh compositor frame) and still decode cleanly.
+ // A second call must wait for and decode a fresh compositor frame.
let image2 = capture
.get_latest_image(Duration::from_millis(500))
.expect("failed to get second frame");