Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(remote-desktop): 让 Windows 远程光标在本地即时移动 (#4565)

Lizi1a627a5e97e4
diff --git a/apps/desktop/native/remote-desktop/windows-host/src/capture.rs b/apps/desktop/native/remote-desktop/windows-host/src/capture.rs
index 6bd3d6bce3..15d0673303 100644
--- a/apps/desktop/native/remote-desktop/windows-host/src/capture.rs
+++ b/apps/desktop/native/remote-desktop/windows-host/src/capture.rs
@@ -8,13 +8,114 @@ use windows_sys::Win32::{
};
#[path = "../../windows-input/src/desktop.rs"]
mod desktop;
+const OVERLAY_JPEG_LIMIT: usize = 1_000_000;
+const LEGACY_JPEG_LIMIT: usize = 180_000;
+const OVERLAY_MAX_EDGE: i32 = 4096;
+const LEGACY_MAX_EDGE: i32 = 1280;
+const OVERLAY_FALLBACK_EDGE: i32 = 1280;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+struct OverlayBudget {
+ quality: u8,
+ max_edge: i32,
+}
+
+impl OverlayBudget {
+ fn new(quality: u8) -> Self {
+ Self {
+ quality,
+ max_edge: OVERLAY_MAX_EDGE,
+ }
+ }
+
+ fn keep_quality(&mut self, quality: u8) {
+ self.quality = quality;
+ }
+
+ fn keep_fallback_edge(&mut self) {
+ self.max_edge = OVERLAY_FALLBACK_EDGE;
+ self.quality = 10;
+ }
+}
+
+fn overlay_qualities(quality: u8) -> Vec<u8> {
+ let mut qualities = Vec::new();
+ for candidate in [quality, 45, 25, 10] {
+ if candidate <= quality && !qualities.contains(&candidate) {
+ qualities.push(candidate);
+ }
+ }
+ qualities
+}
+
+fn scale_size(width: i32, height: i32, max_edge: i32) -> (i32, i32) {
+ let scale = (max_edge as f64 / width.max(height) as f64).min(1.0);
+ (
+ (width as f64 * scale).round().max(1.0) as i32,
+ (height as f64 * scale).round().max(1.0) as i32,
+ )
+}
+
+fn resize_bgra(data: &[u8], width: i32, height: i32, next_width: i32, next_height: i32) -> Vec<u8> {
+ let mut resized = vec![0; (next_width * next_height * 4) as usize];
+ for y in 0..next_height {
+ for x in 0..next_width {
+ let from = ((y * height / next_height * width + x * width / next_width) * 4) as usize;
+ let to = ((y * next_width + x) * 4) as usize;
+ resized[to..to + 4].copy_from_slice(&data[from..from + 4]);
+ }
+ }
+ resized
+}
+
+fn encode_line(
+ overlay: bool,
+ data: &[u8],
+ width: i32,
+ height: i32,
+ quality: u8,
+ cursor: &Option<serde_json::Value>,
+) -> Result<Option<Vec<u8>>> {
+ let mut jpeg = Vec::new();
+ jpeg_encoder::Encoder::new(&mut jpeg, quality)
+ .encode(
+ data,
+ width as u16,
+ height as u16,
+ jpeg_encoder::ColorType::Bgra,
+ )
+ .map_err(|_| std::io::Error::from(std::io::ErrorKind::InvalidData))?;
+ if jpeg.len()
+ > if overlay {
+ OVERLAY_JPEG_LIMIT
+ } else {
+ LEGACY_JPEG_LIMIT
+ }
+ {
+ return Ok(None);
+ }
+ let encoded = base64::engine::general_purpose::STANDARD.encode(jpeg);
+ let mut line = if overlay {
+ serde_json::to_vec(&serde_json::json!({"jpeg":encoded, "cursor":cursor}))?
+ } else {
+ encoded.into_bytes()
+ };
+ line.push(b'\n');
+ Ok(Some(line))
+}
+
pub struct Capture {
desktop: desktop::InputDesktop,
rect: [i32; 4],
+ overlay: bool,
+ budget: OverlayBudget,
}
impl Capture {
- pub fn new(rect: &serde_json::Value) -> Result<Self> {
- let rect = rect.as_array().filter(|r| r.len() == 4).ok_or_else(error)?;
+ pub fn new(init: &serde_json::Value) -> Result<Self> {
+ let rect = init["rect"]
+ .as_array()
+ .filter(|r| r.len() == 4)
+ .ok_or_else(error)?;
let mut values = [0; 4];
for (i, v) in rect.iter().enumerate() {
values[i] = v
@@ -35,6 +136,12 @@ impl Capture {
Ok(Self {
desktop: desktop::InputDesktop::new(),
rect: values,
+ overlay: init["cursorOverlay"] == true,
+ budget: OverlayBudget::new(match init["bitrate"].as_u64() {
+ Some(20_000_000) => 95,
+ Some(8_000_000) => 80,
+ _ => 65,
+ }),
})
}
pub fn frame(&mut self) -> Result<Vec<u8>> {
@@ -57,10 +164,19 @@ impl Capture {
{
return denied();
}
- let scale = 1280.0 / (width.max(height) as f64);
- let scale = scale.min(1.0);
- let w = (width as f64 * scale).round().max(1.0) as i32;
- let h = (height as f64 * scale).round().max(1.0) as i32;
+ // The negotiated path is also the normal video source. Do not force
+ // it through the legacy 1280px compatibility-preview ceiling.
+ // After a high-detail overflow, later frames reuse the kept edge.
+ let (w, h) = scale_size(
+ width,
+ height,
+ if self.overlay {
+ self.budget.max_edge
+ } else {
+ LEGACY_MAX_EDGE
+ },
+ );
+ let scale = w as f64 / width as f64;
let source = GetDC(ptr::null_mut());
if source.is_null() {
return Err(error());
@@ -115,7 +231,11 @@ impl Capture {
// The cursor is not necessarily included in the screen DC.
let mut cursor: CURSORINFO = mem::zeroed();
cursor.cbSize = mem::size_of::<CURSORINFO>() as u32;
- if ok != 0 && GetCursorInfo(&mut cursor) != 0 && cursor.flags == CURSOR_SHOWING {
+ if !self.overlay
+ && ok != 0
+ && GetCursorInfo(&mut cursor) != 0
+ && cursor.flags == CURSOR_SHOWING
+ {
let mut icon: ICONINFO = mem::zeroed();
if GetIconInfo(cursor.hCursor, &mut icon) != 0 {
DrawIconEx(
@@ -148,20 +268,159 @@ impl Capture {
DeleteDC(target);
ReleaseDC(ptr::null_mut(), source);
let data = data.ok_or_else(error)?;
- for quality in [65, 45, 25, 10] {
- let mut jpeg = Vec::new();
- jpeg_encoder::Encoder::new(&mut jpeg, quality)
- .encode(&data, w as u16, h as u16, jpeg_encoder::ColorType::Bgra)
- .map_err(|_| std::io::Error::from(std::io::ErrorKind::InvalidData))?;
- if jpeg.len() <= 180000 {
- let mut line = base64::engine::general_purpose::STANDARD
- .encode(jpeg)
- .into_bytes();
- line.push(b'\n');
- return Ok(line);
+ let cursor = if self.overlay {
+ crate::cursor::read(self.rect)
+ } else {
+ None
+ };
+ encode_frame(self.overlay, data, w, h, &mut self.budget, &cursor)
+ }
+ }
+}
+
+fn fit_overlay_frame(
+ width: i32,
+ height: i32,
+ budget: &mut OverlayBudget,
+ mut fits: impl FnMut(i32, i32, u8) -> Result<bool>,
+) -> Result<Option<(i32, i32, u8)>> {
+ let qualities = overlay_qualities(budget.quality);
+ let downscale = width.max(height) > OVERLAY_FALLBACK_EDGE;
+ let full_res: Vec<u8> = if downscale {
+ qualities
+ .into_iter()
+ .filter(|quality| *quality != 10)
+ .collect()
+ } else {
+ qualities
+ };
+ for quality in full_res {
+ if fits(width, height, quality)? {
+ budget.keep_quality(quality);
+ return Ok(Some((width, height, quality)));
+ }
+ }
+ if downscale {
+ let (next_width, next_height) = scale_size(width, height, OVERLAY_FALLBACK_EDGE);
+ if fits(next_width, next_height, 10)? {
+ budget.keep_fallback_edge();
+ return Ok(Some((next_width, next_height, 10)));
+ }
+ }
+ Ok(None)
+}
+
+fn encode_frame(
+ overlay: bool,
+ data: Vec<u8>,
+ width: i32,
+ height: i32,
+ budget: &mut OverlayBudget,
+ cursor: &Option<serde_json::Value>,
+) -> Result<Vec<u8>> {
+ if overlay {
+ let mut line = None;
+ let chosen =
+ fit_overlay_frame(width, height, budget, |next_width, next_height, quality| {
+ let resized = if next_width == width && next_height == height {
+ None
+ } else {
+ Some(resize_bgra(&data, width, height, next_width, next_height))
+ };
+ let pixels = resized.as_deref().unwrap_or(&data);
+ match encode_line(true, pixels, next_width, next_height, quality, cursor)? {
+ Some(encoded) => {
+ line = Some(encoded);
+ Ok(true)
+ }
+ None => Ok(false),
}
- }
- Err(std::io::Error::from(std::io::ErrorKind::InvalidData))
+ })?;
+ return line
+ .filter(|_| chosen.is_some())
+ .ok_or_else(|| std::io::Error::from(std::io::ErrorKind::InvalidData));
+ }
+ for quality in [65, 45, 25, 10] {
+ if let Some(encoded) = encode_line(false, &data, width, height, quality, cursor)? {
+ return Ok(encoded);
}
}
+ Err(std::io::Error::from(std::io::ErrorKind::InvalidData))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn overlay_quality_ladder_drops_full_res_retries_after_first_overflow() {
+ let mut budget = OverlayBudget::new(95);
+ let mut attempts = Vec::new();
+ let chosen = fit_overlay_frame(3840, 2160, &mut budget, |width, height, quality| {
+ attempts.push((width, height, quality));
+ Ok(quality == 45)
+ })
+ .unwrap();
+ assert_eq!(chosen, Some((3840, 2160, 45)));
+ assert_eq!(attempts, [(3840, 2160, 95), (3840, 2160, 45)]);
+ assert_eq!(
+ budget,
+ OverlayBudget {
+ quality: 45,
+ max_edge: OVERLAY_MAX_EDGE
+ }
+ );
+
+ attempts.clear();
+ let chosen = fit_overlay_frame(3840, 2160, &mut budget, |width, height, quality| {
+ attempts.push((width, height, quality));
+ Ok(true)
+ })
+ .unwrap();
+ assert_eq!(chosen, Some((3840, 2160, 45)));
+ assert_eq!(attempts, [(3840, 2160, 45)]);
+ }
+
+ #[test]
+ fn overlay_fallback_resolution_is_kept_for_the_connection() {
+ let mut budget = OverlayBudget::new(80);
+ let mut attempts = Vec::new();
+ let chosen = fit_overlay_frame(3840, 2160, &mut budget, |width, height, quality| {
+ attempts.push((width, height, quality));
+ Ok(width.max(height) <= OVERLAY_FALLBACK_EDGE)
+ })
+ .unwrap();
+ assert_eq!(chosen, Some((1280, 720, 10)));
+ assert_eq!(
+ attempts,
+ [
+ (3840, 2160, 80),
+ (3840, 2160, 45),
+ (3840, 2160, 25),
+ (1280, 720, 10),
+ ]
+ );
+ assert_eq!(
+ budget,
+ OverlayBudget {
+ quality: 10,
+ max_edge: OVERLAY_FALLBACK_EDGE
+ }
+ );
+
+ attempts.clear();
+ let (width, height) = scale_size(3840, 2160, budget.max_edge);
+ let chosen = fit_overlay_frame(
+ width,
+ height,
+ &mut budget,
+ |next_width, next_height, quality| {
+ attempts.push((next_width, next_height, quality));
+ Ok(true)
+ },
+ )
+ .unwrap();
+ assert_eq!(chosen, Some((1280, 720, 10)));
+ assert_eq!(attempts, [(1280, 720, 10)]);
+ }
}
diff --git a/apps/desktop/native/remote-desktop/windows-host/src/capture_protocol.rs b/apps/desktop/native/remote-desktop/windows-host/src/capture_protocol.rs
new file mode 100644
index 0000000000..371b78e482
--- /dev/null
+++ b/apps/desktop/native/remote-desktop/windows-host/src/capture_protocol.rs
@@ -0,0 +1,33 @@
+//! Local broker framing only. Remote viewers keep the existing cursorOverlay wire format.
+pub const OVERLAY_RESPONSE_LIMIT: usize = 1_750_000;
+
+pub fn response_limit(init: &serde_json::Value) -> usize {
+ if init["mode"] == "capture" && init["cursorOverlay"] == true {
+ OVERLAY_RESPONSE_LIMIT
+ } else {
+ 240001
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[test]
+ fn only_explicit_capture_overlay_gets_the_larger_frame_budget() {
+ for init in [
+ json!({}),
+ json!({"mode":"capture"}),
+ json!({"mode":"capture", "cursorOverlay":false}),
+ json!({"mode":"capture", "cursorOverlay":"true"}),
+ json!({"mode":"input", "cursorOverlay":true}),
+ ] {
+ assert_eq!(response_limit(&init), 240001);
+ }
+ assert_eq!(
+ response_limit(&json!({"mode":"capture", "cursorOverlay":true})),
+ OVERLAY_RESPONSE_LIMIT
+ );
+ }
+}
diff --git a/apps/desktop/native/remote-desktop/windows-host/src/cursor.rs b/apps/desktop/native/remote-desktop/windows-host/src/cursor.rs
new file mode 100644
index 0000000000..83906e3482
--- /dev/null
+++ b/apps/desktop/native/remote-desktop/windows-host/src/cursor.rs
@@ -0,0 +1,282 @@
+//! Read the host cursor without hiding, moving, or changing the physical pointer.
+//! BGRA stays on the authenticated local pipe; Main encodes the existing PNG wire shape.
+use base64::Engine;
+use std::{mem, ptr};
+use windows_sys::Win32::{Graphics::Gdi::*, UI::WindowsAndMessaging::*};
+
+struct Icon(ICONINFO);
+impl Drop for Icon {
+ fn drop(&mut self) {
+ unsafe {
+ if !self.0.hbmMask.is_null() {
+ DeleteObject(self.0.hbmMask);
+ }
+ if !self.0.hbmColor.is_null() {
+ DeleteObject(self.0.hbmColor);
+ }
+ }
+ }
+}
+
+struct Surface {
+ dc: HDC,
+ bitmap: HBITMAP,
+ old: HGDIOBJ,
+}
+impl Drop for Surface {
+ fn drop(&mut self) {
+ unsafe {
+ SelectObject(self.dc, self.old);
+ DeleteObject(self.bitmap);
+ DeleteDC(self.dc);
+ }
+ }
+}
+impl Surface {
+ unsafe fn new(width: i32, height: i32) -> Option<Self> {
+ let dc = CreateCompatibleDC(ptr::null_mut());
+ if dc.is_null() {
+ return None;
+ }
+ let mut info: BITMAPINFO = mem::zeroed();
+ info.bmiHeader = BITMAPINFOHEADER {
+ biSize: mem::size_of::<BITMAPINFOHEADER>() as u32,
+ biWidth: width,
+ biHeight: -height,
+ biPlanes: 1,
+ biBitCount: 32,
+ biCompression: BI_RGB,
+ ..mem::zeroed()
+ };
+ let mut pixels = ptr::null_mut();
+ let bitmap = CreateDIBSection(dc, &info, DIB_RGB_COLORS, &mut pixels, ptr::null_mut(), 0);
+ if bitmap.is_null() || pixels.is_null() {
+ if !bitmap.is_null() {
+ DeleteObject(bitmap);
+ }
+ DeleteDC(dc);
+ return None;
+ }
+ let old = SelectObject(dc, bitmap);
+ if old.is_null() || old as isize == -1 {
+ DeleteObject(bitmap);
+ DeleteDC(dc);
+ return None;
+ }
+ Some(Self { dc, bitmap, old })
+ }
+
+ unsafe fn draw(
+ &self,
+ handle: HCURSOR,
+ width: i32,
+ height: i32,
+ background: u8,
+ ) -> Option<Vec<u8>> {
+ // Own the pixel buffer in Rust. CreateDIBSection's out-pointer starts
+ // null; CodeQL cannot prove the FFI write, so do not dereference it.
+ let mut info: BITMAPINFO = mem::zeroed();
+ info.bmiHeader = BITMAPINFOHEADER {
+ biSize: mem::size_of::<BITMAPINFOHEADER>() as u32,
+ biWidth: width,
+ biHeight: -height,
+ biPlanes: 1,
+ biBitCount: 32,
+ biCompression: BI_RGB,
+ ..mem::zeroed()
+ };
+ let mut pixels = vec![background; (width as usize) * (height as usize) * 4];
+ if SetDIBits(
+ self.dc,
+ self.bitmap,
+ 0,
+ height as u32,
+ pixels.as_ptr().cast(),
+ &info,
+ DIB_RGB_COLORS,
+ ) == 0
+ || DrawIconEx(
+ self.dc,
+ 0,
+ 0,
+ handle,
+ width,
+ height,
+ 0,
+ ptr::null_mut(),
+ DI_NORMAL,
+ ) == 0
+ {
+ return None;
+ }
+ GdiFlush();
+ if GetDIBits(
+ self.dc,
+ self.bitmap,
+ 0,
+ height as u32,
+ pixels.as_mut_ptr().cast(),
+ &mut info,
+ DIB_RGB_COLORS,
+ ) == 0
+ {
+ return None;
+ }
+ Some(pixels)
+ }
+}
+
+// Rendering against both backgrounds handles legacy AND/XOR masks as well as
+// alpha cursors. Output is premultiplied BGRA, as Electron's bitmap API expects.
+fn composite(black: &[u8], white: &[u8], width: usize) -> Vec<u8> {
+ let mut result = black.to_vec();
+ let mut inverted = Vec::new();
+ for (i, (b, w)) in black.chunks_exact(4).zip(white.chunks_exact(4)).enumerate() {
+ let out = &mut result[i * 4..i * 4 + 4];
+ if (0..3).any(|c| w[c] < b[c]) {
+ // A PNG cannot XOR arbitrary remote content. Use an outlined solid
+ // silhouette for these pixels so legacy I-beams stay visible on both backgrounds.
+ out.copy_from_slice(&[0, 0, 0, 255]);
+ inverted.push(i);
+ } else {
+ out[3] = 255 - (0..3).map(|c| w[c] - b[c]).max().unwrap_or(255);
+ }
+ }
+ let height = result.len() / 4 / width;
+ for i in inverted {
+ let x = i % width;
+ let y = i / width;
+ for ny in y.saturating_sub(1)..=(y + 1).min(height - 1) {
+ for nx in x.saturating_sub(1)..=(x + 1).min(width - 1) {
+ let pixel = &mut result[(ny * width + nx) * 4..(ny * width + nx) * 4 + 4];
+ if pixel[3] == 0 {
+ pixel.copy_from_slice(&[255, 255, 255, 255]);
+ }
+ }
+ }
+ }
+ result
+}
+
+unsafe fn raster(handle: HCURSOR) -> Option<(i32, i32, u32, u32, Vec<u8>)> {
+ let mut icon = Icon(mem::zeroed());
+ if GetIconInfo(handle, &mut icon.0) == 0 {
+ return None;
+ }
+ let monochrome = icon.0.hbmColor.is_null();
+ let bitmap = if monochrome {
+ icon.0.hbmMask
+ } else {
+ icon.0.hbmColor
+ };
+ let mut info: BITMAP = mem::zeroed();
+ if GetObjectW(
+ bitmap,
+ mem::size_of::<BITMAP>() as i32,
+ (&mut info as *mut BITMAP).cast(),
+ ) == 0
+ {
+ return None;
+ }
+ let width = info.bmWidth;
+ let height = if monochrome {
+ info.bmHeight / 2
+ } else {
+ info.bmHeight
+ };
+ if !(1..=256).contains(&width)
+ || !(1..=256).contains(&height)
+ || icon.0.xHotspot >= width as u32
+ || icon.0.yHotspot >= height as u32
+ {
+ return None;
+ }
+ let surface = Surface::new(width, height)?;
+ let black = surface.draw(handle, width, height, 0)?;
+ let white = surface.draw(handle, width, height, 255)?;
+ Some((
+ width,
+ height,
+ icon.0.xHotspot,
+ icon.0.yHotspot,
+ composite(&black, &white, width as usize),
+ ))
+}
+
+pub fn read(rect: [i32; 4]) -> Option<serde_json::Value> {
+ unsafe {
+ let mut cursor: CURSORINFO = mem::zeroed();
+ cursor.cbSize = mem::size_of::<CURSORINFO>() as u32;
+ if GetCursorInfo(&mut cursor) == 0 || cursor.hCursor.is_null() {
+ return None;
+ }
+ let (width, height, hot_x, hot_y, pixels) = raster(cursor.hCursor)?;
+ let [x, y, w, h] = rect;
+ let cx = (cursor.ptScreenPos.x as f64 - x as f64) / w as f64;
+ let cy = (cursor.ptScreenPos.y as f64 - y as f64) / h as f64;
+ Some(serde_json::json!({
+ "visible": cursor.flags == CURSOR_SHOWING && (0.0..1.0).contains(&cx) && (0.0..1.0).contains(&cy),
+ "x": cx.clamp(0.0, 1.0), "y": cy.clamp(0.0, 1.0),
+ "width": width, "height": height, "hotX": hot_x, "hotY": hot_y,
+ "bgra": base64::engine::general_purpose::STANDARD.encode(pixels)
+ }))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn preserves_transparency_opaque_color_and_premultiplied_alpha() {
+ let black = [0, 0, 0, 0, 20, 30, 40, 0, 10, 20, 30, 0];
+ let white = [255, 255, 255, 0, 20, 30, 40, 0, 137, 147, 157, 0];
+ assert_eq!(
+ composite(&black, &white, 3),
+ [0, 0, 0, 0, 20, 30, 40, 255, 10, 20, 30, 128]
+ );
+ }
+
+ #[test]
+ fn inverse_pixels_have_a_contrasting_outline() {
+ let mut black = [0; 36];
+ let mut white = [255; 36];
+ black[16..20].fill(255);
+ white[16..20].fill(0);
+ let pixels = composite(&black, &white, 3);
+ assert_eq!(&pixels[16..20], &[0, 0, 0, 255]);
+ assert!(pixels
+ .chunks_exact(4)
+ .enumerate()
+ .all(|(i, p)| i == 4 || p == [255, 255, 255, 255]));
+ }
+
+ #[test]
+ fn rasterizes_system_shapes_without_moving_the_pointer() {
+ unsafe {
+ for name in [IDC_ARROW, IDC_IBEAM, IDC_HAND, IDC_WAIT, IDC_SIZEWE] {
+ let handle = LoadCursorW(ptr::null_mut(), name);
+ assert!(!handle.is_null());
+ let (w, h, x, y, pixels) = raster(handle).expect("system cursor raster");
+ assert_eq!(pixels.len(), (w * h * 4) as usize);
+ assert!(x < w as u32 && y < h as u32);
+ assert!(pixels.chunks_exact(4).any(|p| p[3] > 0));
+ }
+ }
+ }
+
+ #[test]
+ fn repeated_cursor_reads_release_gdi_bitmaps_and_contexts() {
+ use windows_sys::Win32::System::Threading::{GetGuiResources, GR_GDIOBJECTS};
+ unsafe {
+ let handle = LoadCursorW(ptr::null_mut(), IDC_ARROW);
+ raster(handle).expect("warm cursor cache");
+ let process = windows_sys::Win32::System::Threading::GetCurrentProcess();
+ let before = GetGuiResources(process, GR_GDIOBJECTS);
+ for _ in 0..200 {
+ raster(handle).expect("cursor raster");
+ }
+ assert_eq!(GetGuiResources(process, GR_GDIOBJECTS), before);
+ }
+ }
+}
diff --git a/apps/desktop/native/remote-desktop/windows-host/src/lib.rs b/apps/desktop/native/remote-desktop/windows-host/src/lib.rs
index 0506598b05..d6fafcbdf6 100644
--- a/apps/desktop/native/remote-desktop/windows-host/src/lib.rs
+++ b/apps/desktop/native/remote-desktop/windows-host/src/lib.rs
@@ -1,5 +1,6 @@
//! Main-only Node-API transport: the OS sees the actual Cindy PID opening the
//! pipe. A child process or a self-reported parent PID is never authorization.
+mod capture_protocol;
mod pipe;
mod win;
use napi_derive::napi;
@@ -19,6 +20,7 @@ fn failure(_: impl std::fmt::Debug) -> napi::Error {
pub struct DesktopConnection {
pipe: Arc<Mutex<Option<pipe::Pipe>>>,
cancel: Arc<AtomicBool>,
+ response_limit: usize,
}
#[napi]
impl DesktopConnection {
@@ -73,6 +75,8 @@ impl DesktopConnection {
if init.len() > 1023 || init.contains('\n') {
return Err(failure("init"));
}
+ let parsed: serde_json::Value = serde_json::from_str(&init).map_err(failure)?;
+ let response_limit = capture_protocol::response_limit(&parsed);
pipe.write(format!("{init}\n").as_bytes())
.map_err(failure)?;
if pipe.line(1024).map_err(failure)? != b"ready\n" {
@@ -82,6 +86,7 @@ impl DesktopConnection {
Ok(Self {
pipe: Arc::new(Mutex::new(Some(pipe))),
cancel,
+ response_limit,
})
})
.await
@@ -94,13 +99,14 @@ impl DesktopConnection {
}
let pipe = self.pipe.clone();
let cancel = self.cancel.clone();
+ let response_limit = self.response_limit;
tokio::task::spawn_blocking(move || {
let mut guard = pipe.lock().map_err(failure)?;
let result = (|| {
let pipe = guard.as_mut().ok_or_else(|| failure("closed"))?;
pipe.write(format!("{line}\n").as_bytes())
.map_err(failure)?;
- let bytes = pipe.line(240001).map_err(failure)?;
+ let bytes = pipe.line(response_limit).map_err(failure)?;
String::from_utf8(bytes).map_err(failure)
})();
if result.is_err() || cancel.load(Ordering::SeqCst) {
diff --git a/apps/desktop/native/remote-desktop/windows-host/src/main.rs b/apps/desktop/native/remote-desktop/windows-host/src/main.rs
index 944a45f944..1a99e0c7ab 100644
--- a/apps/desktop/native/remote-desktop/windows-host/src/main.rs
+++ b/apps/desktop/native/remote-desktop/windows-host/src/main.rs
@@ -1,4 +1,6 @@
mod capture;
+mod capture_protocol;
+mod cursor;
mod pipe;
mod security;
mod service;
diff --git a/apps/desktop/native/remote-desktop/windows-host/src/pipe.rs b/apps/desktop/native/remote-desktop/windows-host/src/pipe.rs
index 492f074665..ac83dde9f2 100644
--- a/apps/desktop/native/remote-desktop/windows-host/src/pipe.rs
+++ b/apps/desktop/native/remote-desktop/windows-host/src/pipe.rs
@@ -169,7 +169,15 @@ impl Pipe {
}
}
pub fn write(&self, data: &[u8]) -> Result<()> {
- if data.len() > 240001 {
+ self.write_bounded(data, 240001)
+ }
+ /// Both capture-worker and broker responses use the negotiated budget.
+ /// Ordinary requests and unnegotiated responses retain the original limit.
+ pub fn write_response(&self, data: &[u8], init: &serde_json::Value) -> Result<()> {
+ self.write_bounded(data, crate::capture_protocol::response_limit(init))
+ }
+ fn write_bounded(&self, data: &[u8], limit: usize) -> Result<()> {
+ if data.len() > limit {
return denied();
}
let deadline = Instant::now() + Duration::from_secs(5);
@@ -236,3 +244,74 @@ impl Pipe {
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::capture_protocol::OVERLAY_RESPONSE_LIMIT;
+ use serde_json::json;
+
+ #[test]
+ fn transfers_large_negotiated_frames_without_raising_other_write_limits() {
+ let nonce = std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap()
+ .as_nanos();
+ let name = format!(r"\\.\pipe\cindy-cursor-test-{}-{nonce}", std::process::id());
+ // A caller-owned test pipe needs no SYSTEM/interactive service token.
+ // Keep the production service ACL unchanged; only exercise framing here.
+ let handle = Handle::new(unsafe {
+ CreateNamedPipeW(
+ wide(&name).as_ptr(),
+ PIPE_ACCESS_DUPLEX | FILE_FLAG_OVERLAPPED | FILE_FLAG_FIRST_PIPE_INSTANCE,
+ PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT | PIPE_REJECT_REMOTE_CLIENTS,
+ 1,
+ 262144,
+ 32768,
+ 0,
+ ptr::null(),
+ )
+ })
+ .unwrap();
+ let server = Pipe {
+ handle,
+ buffered: Vec::new(),
+ cancel: Arc::new(AtomicBool::new(false)),
+ shutdown: None,
+ };
+ let writer = std::thread::spawn(move || {
+ server.accept(5000).unwrap();
+ let mut frame = vec![b'x'; OVERLAY_RESPONSE_LIMIT];
+ *frame.last_mut().unwrap() = b'\n';
+ assert_eq!(
+ server.write(&frame).unwrap_err().kind(),
+ io::ErrorKind::PermissionDenied
+ );
+ for init in [
+ json!({"mode":"capture"}),
+ json!({"mode":"capture", "cursorOverlay":false}),
+ json!({"mode":"input", "cursorOverlay":true}),
+ ] {
+ assert_eq!(
+ server.write_response(&frame, &init).unwrap_err().kind(),
+ io::ErrorKind::PermissionDenied
+ );
+ }
+ let overlay = json!({"mode":"capture", "cursorOverlay":true});
+ assert_eq!(
+ server
+ .write_response(&vec![b'x'; OVERLAY_RESPONSE_LIMIT + 1], &overlay)
+ .unwrap_err()
+ .kind(),
+ io::ErrorKind::PermissionDenied
+ );
+ server.write_response(&frame, &overlay).unwrap();
+ });
+ let mut client = Pipe::client(&name).unwrap();
+ let frame = client.line(OVERLAY_RESPONSE_LIMIT).unwrap();
+ assert_eq!(frame.len(), OVERLAY_RESPONSE_LIMIT);
+ assert_eq!(frame.last(), Some(&b'\n'));
+ assert!(frame[..frame.len() - 1].iter().all(|byte| *byte == b'x'));
+ writer.join().unwrap();
+ }
+}
diff --git a/apps/desktop/native/remote-desktop/windows-host/src/service.rs b/apps/desktop/native/remote-desktop/windows-host/src/service.rs
index 06742be738..80f5def279 100644
--- a/apps/desktop/native/remote-desktop/windows-host/src/service.rs
+++ b/apps/desktop/native/remote-desktop/windows-host/src/service.rs
@@ -533,7 +533,7 @@ fn serve(mut client: Pipe) -> Result<()> {
}
}
worker.write(&request)?;
- let response = worker.line(240001)?;
+ let response = worker.line(crate::capture_protocol::response_limit(&parsed))?;
if session != unsafe { WTSGetActiveConsoleSessionId() } || STOP.load(Ordering::SeqCst) {
return denied();
}
@@ -546,7 +546,7 @@ fn serve(mut client: Pipe) -> Result<()> {
return denied();
}
}
- client.write(&response)?;
+ client.write_response(&response, &parsed)?;
}
}
struct Worker {
@@ -604,13 +604,13 @@ pub fn worker(name: &str) -> Result<()> {
}
let init: serde_json::Value = serde_json::from_slice(&pipe.line(1024)?)?;
if init["mode"] == "capture" {
- let mut capture = crate::capture::Capture::new(&init["rect"])?;
+ let mut capture = crate::capture::Capture::new(&init)?;
pipe.write(b"ready\n")?;
loop {
if pipe.line(16)? != b"f\n" {
return denied();
}
- pipe.write(&capture.frame()?)?;
+ pipe.write_response(&capture.frame()?, &init)?;
}
}
if init["mode"] != "input" {
diff --git a/apps/desktop/src/main/remote-desktop/__tests__/captureBoundary.test.ts b/apps/desktop/src/main/remote-desktop/__tests__/captureBoundary.test.ts
index 67e3009cfa..7cc87e2708 100644
--- a/apps/desktop/src/main/remote-desktop/__tests__/captureBoundary.test.ts
+++ b/apps/desktop/src/main/remote-desktop/__tests__/captureBoundary.test.ts
@@ -573,3 +573,54 @@ it('routes native input failure to control release rather than capture teardown'
expect(h.stop).not.toHaveBeenCalled();
expect(h.dispose).not.toHaveBeenCalled();
});
+
+it.each(['darwin', 'win32'])(
+ 'negotiates cursor-free capture on %s only when requested',
+ async (platform) => {
+ vi.stubGlobal('process', { ...process, platform });
+ expect((await h.deps.capabilities()).cursorOverlay).toBe(true);
+ for (const overlay of [false, true]) {
+ const pending = h.deps.offer(
+ { lease: h.lease, display: { id: '1' } },
+ 'sdp',
+ undefined,
+ overlay,
+ 'attempt',
+ );
+ h.handlers.get(DESKTOP_LOCAL.REGISTER)(event());
+ await flush();
+ const command = h.owner.send.mock.calls[0][1];
+ expect(command.cursorOverlay).toBe(overlay);
+ expect(command.nativeCapture).toBe(true);
+ h.handlers.get(DESKTOP_LOCAL.REPLY)(event(), command.id, 'answer');
+ await pending;
+ await h.handlers.get(DESKTOP_LOCAL.NATIVE_FRAME)(event(), h.lease);
+ expect(h.nativeFrame).toHaveBeenLastCalledWith('1', overlay, undefined);
+ await h.deps.frame('1', overlay);
+ expect(h.nativeFrame).toHaveBeenLastCalledWith('1', overlay, undefined);
+ }
+ },
+);
+
+it('does not advertise or select Windows overlays without a ready native service', async () => {
+ vi.stubGlobal('process', { ...process, platform: 'win32' });
+ const { readWindowsDesktopSupport } = await import('../windowsHost');
+ vi.mocked(readWindowsDesktopSupport)
+ .mockResolvedValueOnce('missing')
+ .mockResolvedValueOnce('missing');
+ expect((await h.deps.capabilities()).cursorOverlay).toBe(false);
+ const pending = h.deps.offer(
+ { lease: h.lease, display: { id: '1' } },
+ 'sdp',
+ undefined,
+ true,
+ 'attempt',
+ );
+ h.handlers.get(DESKTOP_LOCAL.REGISTER)(event());
+ await flush();
+ const command = h.owner.send.mock.calls[0][1];
+ expect(command.cursorOverlay).toBe(false);
+ expect(command.nativeCapture).toBe(false);
+ h.handlers.get(DESKTOP_LOCAL.REPLY)(event(), command.id, 'answer');
+ await pending;
+});
diff --git a/apps/desktop/src/main/remote-desktop/__tests__/nativeCapture.test.ts b/apps/desktop/src/main/remote-desktop/__tests__/nativeCapture.test.ts
new file mode 100644
index 0000000000..6f7f18c537
--- /dev/null
+++ b/apps/desktop/src/main/remote-desktop/__tests__/nativeCapture.test.ts
@@ -0,0 +1,117 @@
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { NativeDesktopCapture } from '../nativeCapture';
+import { openWindowsDesktopConnection } from '../windowsHost';
+
+vi.mock('electron', () => ({
+ app: {},
+ nativeImage: { createFromBitmap: () => ({ toPNG: () => Buffer.from('iVBORw0KGgo=', 'base64') }) },
+ screen: {
+ getAllDisplays: () => [
+ { id: 1, scaleFactor: 2, bounds: { x: -960, y: 0, width: 960, height: 540 } },
+ ],
+ dipToScreenRect: () => ({ x: -1920, y: 0, width: 1920, height: 1080 }),
+ },
+}));
+vi.mock('../windowsHost', () => ({ openWindowsDesktopConnection: vi.fn() }));
+let capture: NativeDesktopCapture;
+const connections: Array<{ request: ReturnType<typeof vi.fn>; close: ReturnType<typeof vi.fn> }> =
+ [];
+const packet = JSON.stringify({ jpeg: 'anBlZw==', cursor: null });
+beforeEach(() => {
+ vi.stubGlobal('process', { ...process, platform: 'win32' });
+ connections.length = 0;
+ vi.mocked(openWindowsDesktopConnection)
+ .mockReset()
+ .mockImplementation(async () => {
+ const connection = { request: vi.fn(async () => packet), close: vi.fn() };
+ connections.push(connection);
+ return connection;
+ });
+ capture = new NativeDesktopCapture();
+});
+afterEach(() => {
+ capture.stop();
+ vi.unstubAllGlobals();
+});
+
+it('negotiates only the overlay path and preserves legacy init and response', async () => {
+ await expect(
+ capture.frame('1', true, { fps: 60, bitrate: 8_000_000, audio: false }),
+ ).resolves.toEqual({ jpeg: 'anBlZw==', cursor: null });
+ expect(openWindowsDesktopConnection).toHaveBeenLastCalledWith({
+ mode: 'capture',
+ rect: [-1920, 0, 1920, 1080],
+ cursorOverlay: true,
+ bitrate: 8_000_000,
+ });
+ const overlay = connections[0];
+ vi.mocked(openWindowsDesktopConnection).mockImplementationOnce(async () => ({
+ request: vi.fn(async () => 'anBlZw==\n'),
+ close: vi.fn(),
+ }));
+ await expect(capture.frame('1')).resolves.toBe('anBlZw==');
+ expect(overlay.close).toHaveBeenCalledOnce();
+ expect(openWindowsDesktopConnection).toHaveBeenLastCalledWith({
+ mode: 'capture',
+ rect: [-1920, 0, 1920, 1080],
+ });
+});
+it('reuses capture until video quality changes, then closes the old connection', async () => {
+ await capture.frame('1', true);
+ await capture.frame('1', true);
+ expect(connections).toHaveLength(1);
+ await capture.frame('1', true, { fps: 30, bitrate: 20_000_000, audio: true });
+ expect(connections).toHaveLength(2);
+ expect(connections[0].close).toHaveBeenCalledOnce();
+});
+it('closes an opening connection if its lease was stopped and never requests a frame', async () => {
+ let finish!: (value: any) => void;
+ vi.mocked(openWindowsDesktopConnection).mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ finish = resolve;
+ }),
+ );
+ const pending = capture.frame('1', true);
+ capture.stop();
+ const connection = { request: vi.fn(), close: vi.fn() };
+ finish(connection);
+ await expect(pending).resolves.toBeNull();
+ expect(connection.close).toHaveBeenCalledOnce();
+ expect(connection.request).not.toHaveBeenCalled();
+});
+it('ignores a late frame without clearing the replacement read backpressure', async () => {
+ await capture.frame('1', true);
+ let oldDone!: (value: string) => void;
+ connections[0].request.mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ oldDone = resolve;
+ }),
+ );
+ const old = capture.frame('1', true);
+ await expect(capture.frame('1', true)).resolves.toBeNull();
+ capture.stop();
+ await capture.frame('1', true);
+ let nextDone!: (value: string) => void;
+ connections[1].request.mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ nextDone = resolve;
+ }),
+ );
+ const next = capture.frame('1', true);
+ oldDone(packet);
+ await expect(old).resolves.toBeNull();
+ await expect(capture.frame('1', true)).resolves.toBeNull();
+ expect(connections[1].request).toHaveBeenCalledTimes(2);
+ nextDone(packet);
+ await expect(next).resolves.toEqual({ jpeg: 'anBlZw==', cursor: null });
+});
+it('closes malformed frame streams but does not close a stream for an unavailable cursor', async () => {
+ await capture.frame('1', true);
+ expect(connections[0].close).not.toHaveBeenCalled();
+ connections[0].request.mockResolvedValueOnce('{"jpeg":42}');
+ await expect(capture.frame('1', true)).resolves.toBeNull();
+ expect(connections[0].close).toHaveBeenCalledOnce();
+});
diff --git a/apps/desktop/src/main/remote-desktop/__tests__/windowsCursorFrame.test.ts b/apps/desktop/src/main/remote-desktop/__tests__/windowsCursorFrame.test.ts
new file mode 100644
index 0000000000..8249479074
--- /dev/null
+++ b/apps/desktop/src/main/remote-desktop/__tests__/windowsCursorFrame.test.ts
@@ -0,0 +1,87 @@
+import { describe, expect, it, vi } from 'vitest';
+import { decodeWindowsCursorFrame } from '../windowsCursorFrame';
+
+const png = Buffer.from(
+ 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVQIHWP4z8DwHwAFgAI/ScLbtAAAAABJRU5ErkJggg==',
+ 'base64',
+);
+const cursor = {
+ visible: true,
+ x: 0.25,
+ y: 0.75,
+ width: 4,
+ height: 6,
+ hotX: 2,
+ hotY: 4,
+ bgra: Buffer.alloc(4 * 6 * 4).toString('base64'),
+};
+const frame = (value: unknown = cursor) => JSON.stringify({ jpeg: 'anBlZw==', cursor: value });
+
+describe('Windows local cursor frames', () => {
+ it.each([1, 1.25, 1.5, 2])(
+ 'preserves raster and position while converting %s DPI geometry to points',
+ (scale) => {
+ const encode = vi.fn(() => png);
+ expect(decodeWindowsCursorFrame(frame(), scale, encode)).toEqual({
+ jpeg: 'anBlZw==',
+ cursor: {
+ visible: true,
+ x: 0.25,
+ y: 0.75,
+ width: 4 / scale,
+ height: 6 / scale,
+ hotX: 2 / scale,
+ hotY: 4 / scale,
+ png: png.toString('base64'),
+ },
+ });
+ expect(encode).toHaveBeenCalledWith(Buffer.alloc(96), { width: 4, height: 6 });
+ },
+ );
+ it('keeps the old helper JPEG path and hidden cursor state', () => {
+ expect(decodeWindowsCursorFrame('anBlZw==', 1, () => png)).toBe('anBlZw==');
+ expect(
+ decodeWindowsCursorFrame(frame({ ...cursor, visible: false }), 1, () => png),
+ ).toMatchObject({ cursor: { visible: false } });
+ });
+ it.each([
+ null,
+ {},
+ { ...cursor, width: 257 },
+ { ...cursor, height: 0 },
+ { ...cursor, x: 2 },
+ { ...cursor, hotX: -1 },
+ { ...cursor, visible: 'true' },
+ { ...cursor, bgra: 'YQ==' },
+ { ...cursor, bgra: '!'.repeat(128) },
+ ])('drops an invalid or missing cursor without losing the picture: %j', (value) => {
+ const encode = vi.fn(() => png);
+ expect(decodeWindowsCursorFrame(frame(value), 1, encode)).toEqual({
+ jpeg: 'anBlZw==',
+ cursor: null,
+ });
+ expect(encode).not.toHaveBeenCalled();
+ });
+ it.each([
+ () => Buffer.alloc(49153),
+ () => Buffer.from('not png'),
+ () => {
+ throw new Error('encode');
+ },
+ ])('retains video when cursor encoding fails', (encode) => {
+ expect(decodeWindowsCursorFrame(frame(), 1, encode)).toEqual({
+ jpeg: 'anBlZw==',
+ cursor: null,
+ });
+ });
+ it.each([
+ '[]',
+ '{}',
+ '{"jpeg":false}',
+ '{"jpeg":"data:foo"}',
+ 'x'.repeat(1_750_001),
+ JSON.stringify({ jpeg: 'x'.repeat(1_333_337) }),
+ ])('bounds malformed frame payloads', (text) => {
+ expect(() => decodeWindowsCursorFrame(text, 1, () => png)).toThrow();
+ });
+});
diff --git a/apps/desktop/src/main/remote-desktop/index.ts b/apps/desktop/src/main/remote-desktop/index.ts
index 24be77ef7f..b250706302 100644
--- a/apps/desktop/src/main/remote-desktop/index.ts
+++ b/apps/desktop/src/main/remote-desktop/index.ts
@@ -216,7 +216,7 @@ async function offer(
if (settings?.audio && !supportsSystemAudio) throw new Error('DESKTOP_AUDIO_UNAVAILABLE');
captureGrant = source ? { source, lease: lease.lease, audio: settings?.audio === true } : null;
nativeDisplay = nativeAvailable ? lease.display.id : null;
- nativeOverlay = cursorOverlay === true && process.platform === 'darwin';
+ nativeOverlay = cursorOverlay === true && nativeAvailable;
nativeSettings = settings;
setVideoLease(lease.lease);
videoAttempt = attemptId;
@@ -346,7 +346,7 @@ export const remoteDesktop = new RemoteDesktopController({
const viewerDisplay = enabled && (await viewerDisplaySupported());
return {
version: 1,
- cursorOverlay: process.platform === 'darwin',
+ cursorOverlay: process.platform === 'darwin' || (process.platform === 'win32' && windowsAvailable),
lockOnExit: process.platform === 'darwin',
clipboardContent: process.platform === 'darwin' || process.platform === 'win32',
clipboardText: process.platform === 'darwin' || process.platform === 'win32',
@@ -385,7 +385,7 @@ export const remoteDesktop = new RemoteDesktopController({
if (process.platform === 'darwin' || windowsAvailable) {
const frame = await nativeCapture.frame(
displayId,
- cursorOverlay === true && process.platform === 'darwin',
+ cursorOverlay === true,
nativeSettings,
);
return encodeNativeRelayFrame(frame, (jpeg) => nativeImage.createFromBuffer(jpeg));
diff --git a/apps/desktop/src/main/remote-desktop/nativeCapture.ts b/apps/desktop/src/main/remote-desktop/nativeCapture.ts
index ce3178574d..7c616ff7ec 100644
--- a/apps/desktop/src/main/remote-desktop/nativeCapture.ts
+++ b/apps/desktop/src/main/remote-desktop/nativeCapture.ts
@@ -1,11 +1,12 @@
import { isRemoteDesktopCursor, type RemoteDesktopCursorFrame, type RemoteDesktopVideoSettings } from '@cindy/device-link';
-import { app, screen } from 'electron';
+import { app, nativeImage, screen } from 'electron';
import { execFile, spawn, type ChildProcessWithoutNullStreams } from 'node:child_process';
import { createHash } from 'node:crypto';
import fs from 'node:fs/promises';
import path from 'node:path';
import { promisify } from 'node:util';
import { openWindowsDesktopConnection, type WindowsDesktopConnection } from './windowsHost';
+import { decodeWindowsCursorFrame } from './windowsCursorFrame';
const exec = promisify(execFile);
const name = 'cindy-macos-desktop-capture';
@@ -84,7 +85,8 @@ export class NativeDesktopCapture {
async frame(display: string, overlay = false, settings?: RemoteDesktopVideoSettings): Promise<string | RemoteDesktopCursorFrame | null> {
if (process.platform === 'win32') {
if (this.busy) return null;
- if (this.display && this.display !== display) this.stop();
+ const config = overlay ? 'overlay:' + (settings?.bitrate ?? 0) : '';
+ if (this.windows && (this.display !== display || this.config !== config)) this.stop();
const selected = screen.getAllDisplays().find((item) => String(item.id) === display);
if (!selected) return null;
this.busy = true;
@@ -95,6 +97,7 @@ export class NativeDesktopCapture {
const connection = await openWindowsDesktopConnection({
mode: 'capture',
rect: [bounds.x, bounds.y, bounds.width, bounds.height],
+ ...(overlay ? { cursorOverlay: true, bitrate: settings?.bitrate ?? 0 } : {}),
});
if (generation !== this.generation) {
connection.close();
@@ -102,8 +105,14 @@ export class NativeDesktopCapture {
}
this.windows = connection;
this.display = display;
+ this.config = config;
}
const jpeg = (await this.windows.request('f')).trim();
+ if (generation !== this.generation) return null;
+ if (overlay) {
+ return decodeWindowsCursorFrame(jpeg, selected.scaleFactor,
+ (pixels, size) => nativeImage.createFromBitmap(pixels, size).toPNG());
+ }
return generation === this.generation &&
jpeg.length <= 240000 &&
/^[A-Za-z0-9+/]+={0,2}$/.test(jpeg)
diff --git a/apps/desktop/src/main/remote-desktop/windowsCursorFrame.ts b/apps/desktop/src/main/remote-desktop/windowsCursorFrame.ts
new file mode 100644
index 0000000000..de29d906c4
--- /dev/null
+++ b/apps/desktop/src/main/remote-desktop/windowsCursorFrame.ts
@@ -0,0 +1,77 @@
+import { isRemoteDesktopCursor, type RemoteDesktopCursorFrame } from '@cindy/device-link';
+
+type EncodeCursor = (bgra: Buffer, size: { width: number; height: number }) => Buffer;
+const base64 = /^[A-Za-z0-9+/]+={0,2}$/;
+
+/** Convert bounded local BGRA into the existing cross-device PNG cursor contract.
+ * Invalid/unavailable cursor metadata drops only the overlay, never the desktop picture.
+ */
+export function decodeWindowsCursorFrame(
+ text: string,
+ scaleFactor: number,
+ encode: EncodeCursor,
+): string | RemoteDesktopCursorFrame {
+ if (text.length <= 240000 && base64.test(text)) return text; // older installed helper
+ if (text.length > 1_750_000) throw new Error('INVALID_CURSOR_FRAME');
+ const frame: unknown = JSON.parse(text);
+ if (
+ !frame ||
+ typeof frame !== 'object' ||
+ !('jpeg' in frame) ||
+ typeof frame.jpeg !== 'string' ||
+ frame.jpeg.length > 1_333_336 ||
+ !base64.test(frame.jpeg)
+ )
+ throw new Error('INVALID_CURSOR_FRAME');
+ const result: RemoteDesktopCursorFrame = { jpeg: frame.jpeg, cursor: null };
+ if (!('cursor' in frame) || !frame.cursor || typeof frame.cursor !== 'object') return result;
+ const raw = frame.cursor as Record<string, unknown>;
+ const { width, height, bgra } = raw;
+ if (
+ typeof width !== 'number' ||
+ typeof height !== 'number' ||
+ !Number.isInteger(width) ||
+ !Number.isInteger(height) ||
+ width < 1 ||
+ width > 256 ||
+ height < 1 ||
+ height > 256 ||
+ typeof bgra !== 'string' ||
+ bgra.length !== Math.ceil((width * height * 4) / 3) * 4 ||
+ !base64.test(bgra)
+ )
+ return result;
+ const geometry = [raw.x, raw.y, raw.hotX, raw.hotY];
+ if (
+ !geometry.every((value) => typeof value === 'number' && Number.isFinite(value)) ||
+ typeof raw.visible !== 'boolean'
+ )
+ return result;
+ const x = raw.x as number,
+ y = raw.y as number,
+ hotX = raw.hotX as number,
+ hotY = raw.hotY as number;
+ if (x < 0 || x > 1 || y < 0 || y > 1 || hotX < 0 || hotX > width || hotY < 0 || hotY > height)
+ return result;
+ const pixels = Buffer.from(bgra, 'base64');
+ if (pixels.length !== width * height * 4) return result;
+ try {
+ const png = encode(pixels, { width, height });
+ if (png.length > 49152) return result;
+ const scale = Number.isFinite(scaleFactor) && scaleFactor > 0 ? scaleFactor : 1;
+ const cursor = {
+ visible: raw.visible,
+ x,
+ y,
+ width: width / scale,
+ height: height / scale,
+ hotX: hotX / scale,
+ hotY: hotY / scale,
+ png: png.toString('base64'),
+ };
+ if (isRemoteDesktopCursor(cursor)) result.cursor = cursor;
+ } catch {
+ /* Cursor conversion must not tear down video or input. */
+ }
+ return result;
+}
diff --git a/apps/desktop/src/main/remote-desktop/windowsHost.ts b/apps/desktop/src/main/remote-desktop/windowsHost.ts
index 436cd7b5e9..f7169aff38 100644
--- a/apps/desktop/src/main/remote-desktop/windowsHost.ts
+++ b/apps/desktop/src/main/remote-desktop/windowsHost.ts
@@ -51,7 +51,7 @@ export async function configureWindowsDesktopSupport(enabled: boolean): Promise<
throw new Error('DESKTOP_SYSTEM_SERVICE_UNAVAILABLE');
}
export async function openWindowsDesktopConnection(
- init: { mode: 'input' | 'probe' } | { mode: 'capture'; rect: number[] },
+ init: { mode: 'input' | 'probe' } | { mode: 'capture'; rect: number[]; cursorOverlay?: boolean; bitrate?: number },
): Promise<WindowsDesktopConnection> {
if (process.platform !== 'win32' || !app.isPackaged)
throw new Error('DESKTOP_SYSTEM_SERVICE_UNAVAILABLE');
diff --git a/apps/desktop/src/renderer/features/remote-desktop/__tests__/RemoteDesktopHost.test.tsx b/apps/desktop/src/renderer/features/remote-desktop/__tests__/RemoteDesktopHost.test.tsx
index e5d3298e0d..0528fcff5a 100644
--- a/apps/desktop/src/renderer/features/remote-desktop/__tests__/RemoteDesktopHost.test.tsx
+++ b/apps/desktop/src/renderer/features/remote-desktop/__tests__/RemoteDesktopHost.test.tsx
@@ -18,6 +18,100 @@ afterEach(() => {
vi.useRealTimers();
});
+it('keeps system audio with cursor-free native video and stops cursor updates with the lease', async () => {
+ vi.useFakeTimers();
+ const audio = { kind: 'audio', stop: vi.fn() };
+ const chromiumVideo = { kind: 'video', stop: vi.fn() };
+ const nativeVideo = { kind: 'video', stop: vi.fn() };
+ const tracks = [nativeVideo];
+ const stream = {
+ getTracks: () => tracks,
+ getVideoTracks: () => [nativeVideo],
+ getAudioTracks: () => tracks.filter((track) => track.kind === 'audio'),
+ addTrack: (track: typeof audio) => tracks.push(track),
+ };
+ let cursorUpdate!: (value: any) => void;
+ const stopNative = vi.fn();
+ vi.mocked(nativeCaptureStream).mockImplementationOnce(async (_read, _alive, _failed, update) => {
+ cursorUpdate = update!;
+ return { stream, stop: stopNative, clear: vi.fn() } as any;
+ });
+ vi.stubGlobal('navigator', {
+ mediaDevices: {
+ getDisplayMedia: vi.fn(async () => ({
+ getTracks: () => [audio, chromiumVideo],
+ getVideoTracks: () => [chromiumVideo],
+ getAudioTracks: () => [audio],
+ })),
+ },
+ });
+ let peer: any;
+ class Peer {
+ localDescription = { sdp: 'answer' };
+ iceGatheringState = 'complete';
+ close = vi.fn();
+ addTrack = vi.fn();
+ constructor() {
+ peer = this;
+ }
+ getSenders() {
+ return [];
+ }
+ async setRemoteDescription() {}
+ async setLocalDescription() {}
+ async createAnswer() {
+ return {};
+ }
+ }
+ vi.stubGlobal('RTCPeerConnection', Peer);
+ let command!: (value: any) => void;
+ const api = {
+ onCommand: (fn: typeof command) => {
+ command = fn;
+ return () => {};
+ },
+ registerHost: vi.fn(async () => {}),
+ reply: vi.fn(async () => {}),
+ };
+ disposers.push(startDesktopCaptureHost(api as any));
+ await act(async () =>
+ command({
+ op: 'offer',
+ id: 'offer',
+ lease: 'lease',
+ sdp: 'offer',
+ attemptId: 'a',
+ sourceId: 'screen:1',
+ nativeCapture: true,
+ cursorOverlay: true,
+ settings: { audio: true, fps: 60 },
+ }),
+ );
+ expect(api.reply).toHaveBeenCalledWith('offer', 'answer');
+ expect(peer.addTrack.mock.calls.map(([track]: any[]) => track)).toEqual([nativeVideo, audio]);
+ expect(chromiumVideo.stop).toHaveBeenCalledOnce();
+ expect(audio.stop).not.toHaveBeenCalled();
+ const channel = { label: 'input-v1', readyState: 'open', bufferedAmount: 0, send: vi.fn() };
+ peer.ondatachannel({ channel });
+ cursorUpdate({ x: 0.5 });
+ await act(() => vi.advanceTimersByTimeAsync(50));
+ expect(channel.send).toHaveBeenLastCalledWith(
+ JSON.stringify({ type: 'cursor', cursor: { x: 0.5 } }),
+ );
+ channel.bufferedAmount = 65537;
+ cursorUpdate({ x: 0.6 });
+ await act(() => vi.advanceTimersByTimeAsync(50));
+ expect(channel.send).toHaveBeenCalledTimes(1);
+ command({ op: 'stop' });
+ channel.bufferedAmount = 0;
+ cursorUpdate({ x: 0.7 });
+ await act(() => vi.advanceTimersByTimeAsync(100));
+ expect(channel.send).toHaveBeenCalledTimes(1);
+ expect(audio.stop).toHaveBeenCalledOnce();
+ expect(nativeVideo.stop).toHaveBeenCalledOnce();
+ expect(stopNative).toHaveBeenCalledOnce();
+});
+
it.each(['rejected', 'missing'] as const)(
'rejects native video with %s requested audio and releases capture',
async (audio) => {
@@ -134,7 +228,12 @@ it('exchanges replayable candidates without recapturing, tolerates transient dis
};
await act(async () => command(offer));
expect(reply).toHaveBeenCalledWith('offer', 'answer'); // No gather delay for new endpoints.
- const channel = { label: 'input-v1', readyState: 'open', send: vi.fn(), onmessage: (_event: any) => {} };
+ const channel = {
+ label: 'input-v1',
+ readyState: 'open',
+ send: vi.fn(),
+ onmessage: (_event: any) => {},
+ };
peers[0].ondatachannel({ channel });
await act(() => vi.advanceTimersByTimeAsync(8000));
expect(channel.send).toHaveBeenCalledTimes(1);
diff --git a/apps/desktop/src/renderer/features/remote-desktop/__tests__/viewerInput.test.ts b/apps/desktop/src/renderer/features/remote-desktop/__tests__/viewerInput.test.ts
index e4d013209c..b3747354f7 100644
--- a/apps/desktop/src/renderer/features/remote-desktop/__tests__/viewerInput.test.ts
+++ b/apps/desktop/src/renderer/features/remote-desktop/__tests__/viewerInput.test.ts
@@ -68,6 +68,33 @@ function events() {
m.type === 'input' ? (m.events as Record<string, unknown>[]) : [],
);
}
+
+it('moves the separate cursor before input is sent and ignores delayed host positions while moving', () => {
+ const cursor = {
+ visible: true,
+ x: 0.1,
+ y: 0.1,
+ width: 32,
+ height: 32,
+ hotX: 2,
+ hotY: 3,
+ png: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVQIHWP4z8DwHwAFgAI/ScLbtAAAAABJRU5ErkJggg==',
+ };
+ viewer.receive({ type: 'frame', jpeg: 'frame', cursor });
+ pointer('pointermove', 800, 452);
+ const overlay = document.getElementById('cursor')!;
+ expect(overlay.style.display).toBe('block');
+ expect(overlay.style.left).toBe('798px');
+ expect(overlay.style.top).toBe('397px');
+ expect(events()).toEqual([]);
+ viewer.receive({ type: 'frame', jpeg: 'frame', cursor: { ...cursor, x: 0.2, y: 0.2 } });
+ expect(overlay.style.left).toBe('798px');
+ vi.advanceTimersByTime(34);
+ expect(events()).toContainEqual({ kind: 'move', x: 0.8, y: 400 / 600 });
+ viewer.receive({ type: 'control', enabled: false });
+ viewer.receive({ type: 'frame', jpeg: 'frame', cursor });
+ expect(overlay.style.left).toBe('98px');
+});
it('scopes cursor hiding to the remote picture regardless of window focus or cursor metadata', () => {
expect(stage.style.cursor).toBe('none');
viewer.receive({ type: 'frame', jpeg: 'frame' });
diff --git a/docs/remote-desktop.md b/docs/remote-desktop.md
index afe1cbf2f4..30f62208c7 100644
--- a/docs/remote-desktop.md
+++ b/docs/remote-desktop.md
@@ -583,7 +583,7 @@ Validation: mobile/desktop TypeScript checks, iOS device compilation, macOS
helper typecheck and Windows cross-compilation check. No actual clipboard content
was read and no physical-device or Windows application paste was exercised.
-### macOS local cursor overlay
+### macOS and Windows local cursor overlay
Optional `cursorOverlay` on capabilities, offer and compatibility-frame requests
negotiates cursor-free capture; omitted flags retain the original protocol.
@@ -591,6 +591,42 @@ The macOS helper reads the global NSCursor image/hotspot and the public legacy
CGCursorIsVisible symbol, with normalized position within the selected display.
No cursor-hiding call is applied to the user's physical desktop.
+Windows advertises the same capability only when its installed native service is
+ready. Negotiated capture omits DrawIconEx from the picture and reads the cursor
+image, hotspot, visibility and monitor-relative position separately. The existing
+viewer moves that raster locally before the next input batch is sent; delayed
+host positions do not replace it during active movement. Omitted/false flags
+retain the legacy screenshot/video path, including its embedded cursor.
+
+The Windows worker sends bounded premultiplied BGRA only through its authenticated
+local broker pipe. Main validates geometry and byte length, converts the raster
+to PNG using Electron, and converts physical cursor dimensions/hotspot to desktop
+points using the selected monitor's DPI. Only negotiated capture gets the larger
+1.75 MB local response budget; the existing remote PNG/frame limits and all input,
+owner, console-session and lease checks remain unchanged. Older helpers returning
+plain JPEG still work. No new dependencies, persisted pixels, IPC channels, device-link
+messages or Mobile native changes are needed.
+
+Legacy Windows AND/XOR cursors are rendered against black and white to recover
+transparency. Pixels that invert the background cannot be represented exactly by
+a PNG; they use a solid silhouette with a contrasting outline (including I-beams).
+Other cursor colors/alpha are retained. Missing or invalid cursor data does not
+stop video. Windows overlay capture supports up to 4096px, the requested video
+quality and existing 30/60 fps caps; the old compatibility path stays at 1280px.
+Highly detailed overlay frames lower JPEG quality/resolution to keep the 1 MB
+native JPEG bound, and the capture connection keeps that reduced quality or the
+1280px fallback for later frames. This is not a guarantee of the delivered frame rate.
+
+Windows regression coverage: [cursor raster tests](../apps/desktop/native/remote-desktop/windows-host/src/cursor.rs),
+[large-frame pipe transfer](../apps/desktop/native/remote-desktop/windows-host/src/pipe.rs),
+[capture lifecycle](../apps/desktop/src/main/remote-desktop/__tests__/nativeCapture.test.ts),
+[DPI and bounds](../apps/desktop/src/main/remote-desktop/__tests__/windowsCursorFrame.test.ts),
+and [immediate viewer movement](../apps/desktop/src/renderer/features/remote-desktop/__tests__/viewerInput.test.ts).
+Native cursor tests use system shapes without moving the pointer; compiling these
+tests needs cargo feature windows-sys/Win32_UI_Input_KeyboardAndMouse for the
+existing input-desktop test module. Actual two-device control, lock/UAC transitions,
+Light/Dark viewing and sustained video performance require manual validation.
+
Current Electron does not advertise the cursor media constraint. The negotiated
path therefore uses native cursor-free video (up to 4096 pixels on the long edge,
selected 30/60 fps as capture caps). It retains Chromium system audio when requested.