Skip to content

Your Android phone and your Windows PC, working as one. Local-first, end-to-end encrypted, no account.

Play on phone: the PC's side

Bee96e407544579
diff --git a/core/nectarlink-core/src/features.rs b/core/nectarlink-core/src/features.rs
index b4e25c2..7e4e4e8 100644
--- a/core/nectarlink-core/src/features.rs
+++ b/core/nectarlink-core/src/features.rs
@@ -847,6 +847,12 @@ pub const FEATURES: &[FeatureDef] = &[
requires: &[desktop("media.control", UPDATE), phone("media.remote", UPDATE), DeviceToggle("media")],
partial: None,
},
+ FeatureDef {
+ id: "media.play_on_phone",
+ group: FeatureGroup::Media,
+ requires: &[desktop("media.stream", UPDATE), phone("media.watch", UPDATE), DeviceToggle("media")],
+ partial: None,
+ },
FeatureDef {
id: "media.pc_audio",
group: FeatureGroup::Media,
diff --git a/desktop/app/Cargo.toml b/desktop/app/Cargo.toml
index 8d115ce..edd4e7c 100644
--- a/desktop/app/Cargo.toml
+++ b/desktop/app/Cargo.toml
@@ -43,6 +43,7 @@ windows = { workspace = true, features = [
"Foundation_Collections",
"Media",
"Media_Control",
+ "Security_Authorization_AppCapabilityAccess",
"Security_Cryptography",
"Storage",
"Storage_Provider",
@@ -58,6 +59,10 @@ windows = { workspace = true, features = [
"Devices_Bluetooth_Rfcomm",
"Devices_Enumeration",
"Gaming_Input",
+ "Graphics",
+ "Graphics_Capture",
+ "Graphics_DirectX",
+ "Graphics_DirectX_Direct3D11",
"UI_Notifications",
"Win32_Foundation",
"Win32_UI_Input_XboxController",
@@ -85,6 +90,10 @@ windows = { workspace = true, features = [
"Win32_System_SystemInformation",
"Win32_System_Threading",
"Win32_System_WinRT",
+ "Win32_System_WinRT_Direct3D11",
+ "Win32_System_WinRT_Graphics_Capture",
+ "Win32_UI_Accessibility",
+ "Win32_UI_HiDpi",
"Win32_UI_Input_KeyboardAndMouse",
"Win32_UI_Shell",
"Win32_UI_Shell_Common",
diff --git a/desktop/app/qml/components/NowPlaying.qml b/desktop/app/qml/components/NowPlaying.qml
index 8bd70e0..4329712 100644
--- a/desktop/app/qml/components/NowPlaying.qml
+++ b/desktop/app/qml/components/NowPlaying.qml
@@ -382,6 +382,17 @@ Card {
anchors.verticalCenter: parent.verticalCenter
spacing: 2
+ // Play on phone: this PC's player, on the phone (streamed, or in the service's app).
+ IconButton {
+ visible: card.isLocalPc && MediaList.playOnPhoneName !== ""
+ width: 24; height: 24
+ iconSize: 13
+ iconPath: Icons.phone
+ iconColor: Theme.surfaceContentVariant
+ label: qsTr("Play on %1").arg(MediaList.playOnPhoneName)
+ onClicked: card.send("play_on_phone")
+ }
+
IconButton {
visible: !Theme.graphite
width: 24; height: 24
diff --git a/desktop/app/src/bridge/media.rs b/desktop/app/src/bridge/media.rs
index 6302100..a4abaac 100644
--- a/desktop/app/src/bridge/media.rs
+++ b/desktop/app/src/bridge/media.rs
@@ -42,6 +42,8 @@ pub mod qobject {
#[qproperty(i32, revision)]
#[qproperty(QString, active_art, cxx_name = "activeArt")]
#[qproperty(QString, active_colors, cxx_name = "activeColors")]
+ /// The phone Play on phone plays on (its name), or "" when none is online.
+ #[qproperty(QString, play_on_phone_name, cxx_name = "playOnPhoneName")]
type MediaList = super::MediaListRust;
}
@@ -87,7 +89,8 @@ pub mod qobject {
fn row_count(self: &MediaList, parent: &QModelIndex) -> i32;
/// Runs "play", "pause", "next", "previous" or "seek" (to
- /// `position_ms`) on a device's player.
+ /// `position_ms`) on a device's player, or for this PC's,
+ /// "play_on_phone".
#[qinvokable]
fn command(self: &MediaList, device: &QString, player: &QString, action: &QString, position_ms: f64);
/// The row of a device's first player (the one to show), or -1.
@@ -110,6 +113,7 @@ pub struct MediaListRust {
revision: i32,
active_art: QString,
active_colors: QString,
+ play_on_phone_name: QString,
rows: Vec<PlayerView>,
}
@@ -180,7 +184,11 @@ fn role_value(row: &PlayerView, role: &str) -> QVariant {
impl cxx_qt::Initialize for qobject::MediaList {
fn initialize(self: Pin<&mut Self>) {
- super::subscribe(self.qt_thread(), Changes::MEDIA, Self::refresh);
+ super::subscribe(
+ self.qt_thread(),
+ Changes::MEDIA | Changes::DEVICES | Changes::CAPABILITIES,
+ Self::refresh,
+ );
}
}
@@ -231,6 +239,10 @@ impl qobject::MediaList {
if *self.active_colors() != colors_json {
self.as_mut().set_active_colors(colors_json);
}
+ let phone = QString::from(crate::play_on_phone::phone().map(|(_, name)| name).unwrap_or_default());
+ if *self.play_on_phone_name() != phone {
+ self.as_mut().set_play_on_phone_name(phone);
+ }
let revision = self.revision.wrapping_add(1);
self.set_revision(revision);
}
@@ -282,6 +294,12 @@ impl qobject::MediaList {
let resolved_device =
if device_str == "local" { Some(media::LOCAL_DEVICE) } else { super::parse_device(device) };
let Some(device) = resolved_device else { return };
+ if action_str == "play_on_phone" {
+ if device == media::LOCAL_DEVICE {
+ crate::play_on_phone::play(String::from(player));
+ }
+ return;
+ }
if action_str == "volume" {
let level = (position_ms.round() as i32).clamp(0, 100) as u8;
if device == media::LOCAL_DEVICE {
diff --git a/desktop/app/src/core_host.rs b/desktop/app/src/core_host.rs
index e767e9f..a0b514a 100644
--- a/desktop/app/src/core_host.rs
+++ b/desktop/app/src/core_host.rs
@@ -118,6 +118,7 @@ pub fn compute_capabilities(node: Option<&Node>) -> Vec<String> {
nectarlink_core::STORAGE_READ.into(),
nectarlink_core::STORAGE_WRITE.into(),
nectarlink_core::WEBCAM_VIRTUAL.into(),
+ nectarlink_core::MEDIA_STREAM.into(),
];
let mut pc_mirror_enabled = crate::pc_mirror::is_active();
let mut remote_input_enabled = false;
diff --git a/desktop/app/src/main.rs b/desktop/app/src/main.rs
index c9500ac..613f283 100644
--- a/desktop/app/src/main.rs
+++ b/desktop/app/src/main.rs
@@ -38,6 +38,7 @@ mod pc_mirror;
mod pc_notifications;
mod pc_storage;
mod photos;
+mod play_on_phone;
mod qr;
mod recordings;
mod remote;
diff --git a/desktop/app/src/pc_audio_stream.rs b/desktop/app/src/pc_audio_stream.rs
index 96f685f..26732e1 100644
--- a/desktop/app/src/pc_audio_stream.rs
+++ b/desktop/app/src/pc_audio_stream.rs
@@ -31,6 +31,11 @@ static WORKER_RUNNING: AtomicBool = AtomicBool::new(false);
/// when the PC turned it on (or that phone is away), to the first phone that is.
static TARGET: std::sync::Mutex<Option<nectarlink_core::DeviceId>> = std::sync::Mutex::new(None);
+/// The phone the sound goes to first, if one asked.
+pub fn target() -> Option<nectarlink_core::DeviceId> {
+ *TARGET.lock().unwrap_or_else(|e| e.into_inner())
+}
+
/// Records which phone asked to be the speaker.
pub fn set_target(phone: Option<nectarlink_core::DeviceId>) {
*TARGET.lock().unwrap_or_else(|e| e.into_inner()) = phone;
diff --git a/desktop/app/src/pc_mirror.rs b/desktop/app/src/pc_mirror.rs
index 75a7985..c4a550c 100644
--- a/desktop/app/src/pc_mirror.rs
+++ b/desktop/app/src/pc_mirror.rs
@@ -19,7 +19,7 @@ use nectarlink_core::{
DeviceId, KeyMod, MirrorConfig, MirrorInput, MirrorSend, MirrorStart, PacketKind, mirror_keys,
};
use windows::Win32::{
- Foundation::POINT,
+ Foundation::{POINT, RECT},
Graphics::Gdi::{
BI_RGB, BITMAPINFO, BITMAPINFOHEADER, CreateCompatibleBitmap, CreateCompatibleDC, DESKTOPHORZRES,
DESKTOPVERTRES, DeleteDC, DeleteObject, GetDC, GetDIBits, GetDeviceCaps, HALFTONE, ReleaseDC,
@@ -56,6 +56,8 @@ struct ActiveSession {
peer: DeviceId,
stop: Arc<AtomicBool>,
force_keyframe: Arc<AtomicBool>,
+ /// Play on phone: the phone watches a player, and its input is ignored.
+ watching: bool,
}
static ACTIVE: Mutex<Option<ActiveSession>> = Mutex::new(None);
@@ -94,6 +96,7 @@ pub fn start(peer: DeviceId, options: MirrorStart) {
peer,
stop: Arc::clone(&stop),
force_keyframe: Arc::clone(&force_keyframe),
+ watching: options.watch.is_some(),
});
}
core_host::host().hub.changed(crate::state::Changes::REMOTE);
@@ -166,7 +169,8 @@ pub fn handle_input(peer: &DeviceId, session: u32, input: MirrorInput) {
if session != nectarlink_core::MIRROR_SCREEN {
return;
}
- if active_peer() != Some(*peer) {
+ let watching = ACTIVE.lock().unwrap_or_else(|e| e.into_inner()).as_ref().is_some_and(|s| s.watching);
+ if active_peer() != Some(*peer) || watching {
return;
}
let remote_input_on = core_host::node()
@@ -540,19 +544,27 @@ fn has_h264_sps(data: &[u8]) -> bool {
data.windows(4).any(|w| w[..3] == [0, 0, 1] && (w[3] & 0x1f) == 7)
}
-/// Captures the primary desktop monitor into `bgra` scaled to `(dst_w, dst_h)`
-/// and returns the current physical screen dimensions `(src_w, src_h)`.
-fn capture_primary_screen(dst_w: u32, dst_h: u32, bgra: &mut Vec<u8>) -> Option<(u32, u32)> {
+/// Captures the primary desktop monitor into `bgra` scaled to `(dst_w, dst_h)`,
+/// with the pointer, and returns the current physical screen dimensions
+/// `(src_w, src_h)`. With `region` (physical screen pixels), captures only
+/// that, without the pointer (Play on phone), and returns its size.
+fn capture_screen(region: Option<RECT>, dst_w: u32, dst_h: u32, bgra: &mut Vec<u8>) -> Option<(u32, u32)> {
// SAFETY: Win32 GDI device context and bitmap calls with balanced cleanup.
unsafe {
let hdc_screen = GetDC(None);
if hdc_screen.is_invalid() {
return None;
}
- let phys_w = GetDeviceCaps(Some(hdc_screen), DESKTOPHORZRES);
- let phys_h = GetDeviceCaps(Some(hdc_screen), DESKTOPVERTRES);
- let src_w = if phys_w > 0 { phys_w } else { GetSystemMetrics(SM_CXSCREEN) }.max(1) as u32;
- let src_h = if phys_h > 0 { phys_h } else { GetSystemMetrics(SM_CYSCREEN) }.max(1) as u32;
+ let (src_x, src_y, src_w, src_h) = match region {
+ Some(r) => (r.left, r.top, (r.right - r.left).max(1) as u32, (r.bottom - r.top).max(1) as u32),
+ None => {
+ let phys_w = GetDeviceCaps(Some(hdc_screen), DESKTOPHORZRES);
+ let phys_h = GetDeviceCaps(Some(hdc_screen), DESKTOPVERTRES);
+ let w = if phys_w > 0 { phys_w } else { GetSystemMetrics(SM_CXSCREEN) }.max(1) as u32;
+ let h = if phys_h > 0 { phys_h } else { GetSystemMetrics(SM_CYSCREEN) }.max(1) as u32;
+ (0, 0, w, h)
+ }
+ };
let hdc_mem = CreateCompatibleDC(Some(hdc_screen));
if hdc_mem.is_invalid() {
@@ -574,8 +586,8 @@ fn capture_primary_screen(dst_w: u32, dst_h: u32, bgra: &mut Vec<u8>) -> Option<
dst_w as i32,
dst_h as i32,
Some(hdc_screen),
- 0,
- 0,
+ src_x,
+ src_y,
src_w as i32,
src_h as i32,
SRCCOPY,
@@ -589,7 +601,7 @@ fn capture_primary_screen(dst_w: u32, dst_h: u32, bgra: &mut Vec<u8>) -> Option<
hCursor: Default::default(),
ptScreenPos: POINT::default(),
};
- if GetCursorInfo(&mut ci).is_ok() && (ci.flags.0 & CURSOR_SHOWING.0) != 0 {
+ if region.is_none() && GetCursorInfo(&mut ci).is_ok() && (ci.flags.0 & CURSOR_SHOWING.0) != 0 {
let cx = ((ci.ptScreenPos.x as i64) * (dst_w as i64) / (src_w as i64)) as i32;
let cy = ((ci.ptScreenPos.y as i64) * (dst_h as i64) / (src_h as i64)) as i32;
let _ = DrawIconEx(hdc_mem, cx, cy, ci.hCursor.into(), 0, 0, 0, None, DI_NORMAL);
@@ -670,11 +682,50 @@ fn run_capture_loop(
let mut last_rate_change = Instant::now();
let mut stats = Stats::default();
let mut stats_since = Instant::now();
- tracing::info!(%peer, fps, bitrate, "sharing this PC's screen");
+ // Play on phone: the player's picture instead of the screen, and the
+ // PC's sound to that phone while it watches. The picture's place comes
+ // in physical pixels; so does this thread's capture.
+ if options.watch.is_some() {
+ win::browser::use_physical_pixels();
+ }
+ let watched = options.watch.as_deref().map(crate::play_on_phone::Watched::follow);
+ let mut window_capture: Option<win::window_capture::WindowCapture> = None;
+ let _sound = watched.is_some().then(|| SoundToPhone::start(peer));
+ if watched.is_some() {
+ tracing::info!(%peer, fps, bitrate, "streaming a player to the phone");
+ } else {
+ tracing::info!(%peer, fps, bitrate, "sharing this PC's screen");
+ }
while !stop.load(Ordering::Relaxed) {
let tick_start = Instant::now();
+ let place = watched.as_ref().and_then(crate::play_on_phone::Watched::place);
+ if watched.is_some() {
+ let Some(place) = place else {
+ // Not found yet: nothing to show.
+ std::thread::sleep(frame_interval);
+ continue;
+ };
+ let r = place.on_screen;
+ let dims =
+ target_dimensions((r.right - r.left) as u32, (r.bottom - r.top) as u32, options.max_size);
+ if stream_dims != Some(dims) {
+ // A new size (the window moved to full screen, say): a new stream config.
+ stream_dims = Some(dims);
+ encoder = None;
+ }
+ // The window's own pixels, whatever is in front of it; the
+ // screen there when Windows can't capture the window.
+ if window_capture.as_ref().is_none_or(|c| c.window() != place.window()) {
+ window_capture = win::window_capture::WindowCapture::new(place.window())
+ .inspect_err(
+ |e| tracing::info!(error = %e, "can't capture the window; capturing the screen"),
+ )
+ .ok();
+ }
+ }
+
let (dst_w, dst_h) = match stream_dims {
Some(dims) => dims,
None => {
@@ -726,9 +777,12 @@ fn run_capture_loop(
}
}
- if capture_primary_screen(dst_w, dst_h, &mut bgra).is_some()
- && let Some(enc) = encoder.as_mut()
- {
+ let captured = match (place, window_capture.as_mut()) {
+ (Some(place), Some(capture)) => capture.capture(place.in_window, dst_w, dst_h, &mut bgra),
+ (Some(place), None) => capture_screen(Some(place.on_screen), dst_w, dst_h, &mut bgra).is_some(),
+ (None, _) => capture_screen(None, dst_w, dst_h, &mut bgra).is_some(),
+ };
+ if captured && let Some(enc) = encoder.as_mut() {
bgra_to_nv12(&bgra, dst_w as usize, dst_h as usize, &mut nv12);
let time_us = started.elapsed().as_micros() as u64;
let hns = (time_us * 10) as i64;
@@ -815,6 +869,32 @@ fn run_capture_loop(
cleanup_session(&stop);
}
+/// The PC's sound going to a phone that watches one of its players (as
+/// Phone speaker sends it), for as long as this lives; then as before.
+struct SoundToPhone {
+ was_on: bool,
+ was_target: Option<DeviceId>,
+}
+
+impl SoundToPhone {
+ fn start(peer: DeviceId) -> SoundToPhone {
+ let sound = SoundToPhone {
+ was_on: crate::pc_audio_stream::is_enabled(),
+ was_target: crate::pc_audio_stream::target(),
+ };
+ crate::pc_audio_stream::set_target(Some(peer));
+ crate::pc_audio_stream::set_enabled(true);
+ sound
+ }
+}
+
+impl Drop for SoundToPhone {
+ fn drop(&mut self) {
+ crate::pc_audio_stream::set_target(self.was_target);
+ crate::pc_audio_stream::set_enabled(self.was_on);
+ }
+}
+
fn cleanup_session(stop: &Arc<AtomicBool>) {
let removed = {
let mut guard = ACTIVE.lock().unwrap_or_else(|e| e.into_inner());
@@ -871,6 +951,33 @@ mod tests {
assert!(decoded_any, "decoder must reconstruct at least one frame from H264Encoder");
}
+ /// With a Chromium browser showing a video on a page whose title has
+ /// `NECTARLINK_TEST_TAB` in it, in view: captures just the video, as
+ /// Play on phone does.
+ #[test]
+ #[ignore]
+ fn captures_a_browsers_video() {
+ use crate::win::browser::{Automation, window_for};
+ let wanted = std::env::var("NECTARLINK_TEST_TAB").expect("NECTARLINK_TEST_TAB");
+ // SAFETY: COM for this test's thread.
+ let _ = unsafe {
+ windows::Win32::System::Com::CoInitializeEx(
+ None,
+ windows::Win32::System::Com::COINIT_MULTITHREADED,
+ )
+ };
+ crate::win::browser::use_physical_pixels();
+ let window = window_for("Chrome", &wanted).expect("the test tab's window");
+ let rect = Automation::new().unwrap().video_rect(&window).expect("the video");
+ let (sw, sh) = ((rect.right - rect.left) as u32, (rect.bottom - rect.top) as u32);
+ let (w, h) = target_dimensions(sw, sh, 1280);
+ let mut bgra = Vec::new();
+ assert_eq!(capture_screen(Some(rect), w, h, &mut bgra), Some((sw, sh)));
+ let distinct: std::collections::HashSet<&[u8; 4]> = bgra.as_chunks::<4>().0.iter().collect();
+ eprintln!("{rect:?} as {w}x{h}: {} colors", distinct.len());
+ assert!(distinct.len() > 50, "a moving test pattern, not a blank");
+ }
+
#[test]
fn keyframes_come_on_request_without_rebuilding_the_encoder() {
let (w, h) = target_dimensions(1280, 720, 1280);
diff --git a/desktop/app/src/play_on_phone.rs b/desktop/app/src/play_on_phone.rs
new file mode 100644
index 0000000..f654773
--- /dev/null
+++ b/desktop/app/src/play_on_phone.rs
@@ -0,0 +1,205 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+//! Play on phone, on this PC (docs/protocol/media.md §4): moves what one of
+//! its players plays to a phone. A protected streaming service opens on
+//! the phone in its app and the PC pauses; anything else the phone watches
+//! streamed from the PC, which keeps playing it.
+
+#![allow(unsafe_code)]
+
+use std::{
+ sync::{
+ Arc, Mutex,
+ atomic::{AtomicBool, Ordering},
+ },
+ time::Duration,
+};
+
+use nectarlink_core::{DeviceId, Error, FeatureState, LinkState, MediaAction, PlayOnPhone};
+use windows::Win32::{
+ Foundation::{HWND, RECT},
+ System::Com::{COINIT_MULTITHREADED, CoInitializeEx},
+};
+
+use crate::{
+ bridge::app::{describe, show_message},
+ core_host,
+ win::browser::{self, Automation, Window},
+};
+
+/// The phone to play on: the first one online that can watch a PC's
+/// playback, else the first one online.
+pub fn phone() -> Option<(DeviceId, String)> {
+ core_host::host().hub.read(|s| {
+ let online: Vec<_> =
+ s.devices.iter().filter(|d| matches!(d.link, LinkState::Online { .. })).collect();
+ let watches = |id: &DeviceId| {
+ s.matrices.get(id).and_then(|m| m.state("media.play_on_phone")) == Some(FeatureState::Available)
+ };
+ online.iter().find(|d| watches(&d.id)).or_else(|| online.first()).map(|d| (d.id, d.info.name.clone()))
+ })
+}
+
+/// What this PC's `player` plays: its title, and where it is, in seconds.
+fn now_playing(player: &str) -> (String, Option<u32>) {
+ let views = crate::media::local_views();
+ let Some(view) = views.into_iter().find(|v| v.player.id == player) else {
+ return (String::new(), None);
+ };
+ let position = view.player.position.map(|ms| {
+ let since =
+ if view.player.playing { view.at.elapsed().map_or(0, |d| d.as_millis() as u64) } else { 0 };
+ ((ms + since) / 1000) as u32
+ });
+ (view.player.title.unwrap_or_default(), position.filter(|&s| s > 0))
+}
+
+/// Plays what this PC's `player` plays on a phone (the Play on phone
+/// button on the PC's media card).
+pub fn play(player: String) {
+ let Some((phone, name)) = phone() else {
+ return show_message("Connect your phone to play on it.");
+ };
+ let spawned = std::thread::Builder::new().name("play-on-phone".into()).spawn(move || {
+ // SAFETY: COM for this short-lived thread (UI Automation).
+ let _ = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) };
+ let (title, position) = now_playing(&player);
+ let window = browser::window_for(&player, &title);
+ let url = window
+ .as_ref()
+ .filter(|w| browser::is_browser(&w.program))
+ .and_then(|w| Automation::new().ok()?.page_url(w));
+ let plan = nectarlink_core::play_on_phone::plan(url.as_deref(), position);
+ let Some(node) = core_host::node() else { return };
+ core_host::spawn(async move {
+ match plan {
+ PlayOnPhone::Open { url, service } => match node.open_link(phone, url).await {
+ Ok(()) => {
+ crate::media::command(crate::media::LOCAL_DEVICE, player, MediaAction::Pause, None);
+ show_message(format!("Opening in {service} on {name}. Tap the notification there."));
+ }
+ Err(e) => show_message(failed(&e, &name)),
+ },
+ PlayOnPhone::Stream { open_instead } => {
+ let shown = (!title.is_empty()).then_some(title);
+ match node.offer_watch(phone, player.clone(), shown).await {
+ Ok(()) => show_message(format!("Tap the notification on {name} to watch.")),
+ // A phone that can't watch yet: the page opens there instead.
+ Err(Error::Unsupported) if let Some(url) = open_instead => {
+ match node.open_link(phone, url).await {
+ Ok(()) => {
+ crate::media::command(
+ crate::media::LOCAL_DEVICE,
+ player,
+ MediaAction::Pause,
+ None,
+ );
+ show_message(format!("Sent to {name}. Tap the notification there."));
+ }
+ Err(e) => show_message(failed(&e, &name)),
+ }
+ }
+ Err(e) => show_message(failed(&e, &name)),
+ }
+ }
+ }
+ });
+ });
+ if let Err(e) = spawned {
+ tracing::warn!(error = %e, "can't start Play on phone");
+ }
+}
+
+fn failed(error: &Error, phone: &str) -> String {
+ match error {
+ Error::Unsupported => format!("Update NectarLink on {phone} to play on it."),
+ Error::Denied => format!("Media is turned off for {phone}."),
+ other => describe(other),
+ }
+}
+
+/// Where a watched player's picture is, kept up to date while a phone
+/// watches it: the video moves when the page scrolls, the window moves or
+/// goes full screen.
+pub struct Watched {
+ place: Arc<Mutex<Option<Place>>>,
+ stop: Arc<AtomicBool>,
+}
+
+/// A watched player's picture: its window, and where in it the picture is.
+#[derive(Debug, Clone, Copy)]
+pub struct Place {
+ /// The window's handle (as a number: handles go between threads).
+ window: isize,
+ /// The picture, in physical screen pixels.
+ pub on_screen: RECT,
+ /// The picture, in the window's own pixels (from its frame's top-left),
+ /// as a capture of the window has it.
+ pub in_window: RECT,
+}
+
+/// How often the picture's place is looked up again.
+const FOLLOW_EVERY: Duration = Duration::from_millis(500);
+
+impl Place {
+ /// The window the picture is in.
+ pub fn window(&self) -> HWND {
+ HWND(self.window as *mut core::ffi::c_void)
+ }
+}
+
+impl Watched {
+ /// Starts following `player`'s picture.
+ pub fn follow(player: &str) -> Watched {
+ let place = Arc::new(Mutex::new(None));
+ let stop = Arc::new(AtomicBool::new(false));
+ let (found, stopped, player) = (place.clone(), stop.clone(), player.to_owned());
+ let spawned = std::thread::Builder::new().name("play-on-phone-follow".into()).spawn(move || {
+ // SAFETY: COM for this thread, which ends with the watching.
+ let _ = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) };
+ // Places in the pixels the capture uses.
+ browser::use_physical_pixels();
+ let automation = Automation::new().ok();
+ let mut window: Option<Window> = None;
+ while !stopped.load(Ordering::Relaxed) {
+ // The window it played in, while it's still there; else find it again.
+ if window.as_ref().is_none_or(|w| browser::frame_of(w.hwnd).is_none()) {
+ let (title, _) = now_playing(&player);
+ window = browser::window_for(&player, &title);
+ }
+ let place = window.as_ref().and_then(|w| {
+ let frame = browser::frame_of(w.hwnd)?;
+ let on_screen = match &automation {
+ Some(a) if browser::is_browser(&w.program) => a.video_rect(w)?,
+ _ => frame,
+ };
+ let in_window = RECT {
+ left: on_screen.left - frame.left,
+ top: on_screen.top - frame.top,
+ right: on_screen.right - frame.left,
+ bottom: on_screen.bottom - frame.top,
+ };
+ Some(Place { window: w.hwnd.0 as isize, on_screen, in_window })
+ });
+ if place.is_some() {
+ *found.lock().unwrap_or_else(|e| e.into_inner()) = place;
+ }
+ std::thread::sleep(FOLLOW_EVERY);
+ }
+ });
+ if let Err(e) = spawned {
+ tracing::warn!(error = %e, "can't follow the watched player");
+ }
+ Watched { place, stop }
+ }
+
+ /// Where the picture is now, once known.
+ pub fn place(&self) -> Option<Place> {
+ *self.place.lock().unwrap_or_else(|e| e.into_inner())
+ }
+}
+
+impl Drop for Watched {
+ fn drop(&mut self) {
+ self.stop.store(true, Ordering::Relaxed);
+ }
+}
diff --git a/desktop/app/src/win/browser.rs b/desktop/app/src/win/browser.rs
new file mode 100644
index 0000000..3babbed
--- /dev/null
+++ b/desktop/app/src/win/browser.rs
@@ -0,0 +1,348 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+//! The window a player shows in, for Play on phone: which top-level window
+//! belongs to a media session, the address of the page a browser plays in,
+//! and where on screen the video is, read through UI Automation.
+//!
+//! Chromium browsers (Chrome, Edge, Brave, Opera, Vivaldi) expose their
+//! address bar as an edit field of class `OmniboxViewViews` and a page's
+//! elements with their HTML classes, so YouTube's player
+//! (`video-stream html5-main-video`) can be found exactly; Firefox's
+//! address bar has the automation ID `urlbar-input`. Nothing here reads a
+//! page's text: only the address of the tab that plays.
+
+use windows::{
+ Win32::{
+ Foundation::{CloseHandle, HWND, LPARAM, RECT},
+ Graphics::Dwm::{DWMWA_CLOAKED, DWMWA_EXTENDED_FRAME_BOUNDS, DwmGetWindowAttribute},
+ System::{
+ Com::{CLSCTX_INPROC_SERVER, CoCreateInstance},
+ Threading::{
+ OpenProcess, PROCESS_NAME_WIN32, PROCESS_QUERY_LIMITED_INFORMATION,
+ QueryFullProcessImageNameW,
+ },
+ Variant::VARIANT,
+ },
+ UI::{
+ Accessibility::{
+ CUIAutomation, IUIAutomation, IUIAutomationElement, TreeScope_Descendants,
+ UIA_AutomationIdPropertyId, UIA_ClassNamePropertyId, UIA_ControlTypePropertyId,
+ UIA_DocumentControlTypeId, UIA_PROPERTY_ID, UIA_ValueValuePropertyId,
+ },
+ HiDpi::{DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2, SetThreadDpiAwarenessContext},
+ WindowsAndMessaging::{
+ EnumWindows, GW_OWNER, GetWindow, GetWindowTextW, GetWindowThreadProcessId, IsIconic,
+ IsWindowVisible,
+ },
+ },
+ },
+ core::{BOOL, BSTR},
+};
+
+/// The programs a media session's app ID stands for. Browsers report a
+/// short name (or, for Firefox, a hash of its install path); desktop apps
+/// often report their executable.
+pub fn programs_for(player: &str) -> Vec<String> {
+ let lower = player.to_ascii_lowercase();
+ let known: &[(&str, &str)] = &[
+ ("chrome", "chrome.exe"),
+ ("msedge", "msedge.exe"),
+ ("edge", "msedge.exe"),
+ ("brave", "brave.exe"),
+ ("opera", "opera.exe"),
+ ("vivaldi", "vivaldi.exe"),
+ ("firefox", "firefox.exe"),
+ // Firefox's app ID is a hash of where it's installed; this is the
+ // one for its default location.
+ ("308046b0af4a39cb", "firefox.exe"),
+ ];
+ let mut programs: Vec<String> =
+ known.iter().filter(|(id, _)| lower.contains(id)).map(|(_, exe)| (*exe).to_owned()).collect();
+ // "vlc.exe", or a path, or "VideoLAN.VLC" (a desktop app ID).
+ let last = lower.rsplit(['\\', '/']).next().unwrap_or(&lower);
+ if last.ends_with(".exe") {
+ programs.push(last.to_owned());
+ } else if let Some((_, name)) = last.rsplit_once('.')
+ && !name.is_empty()
+ && !last.contains('!')
+ {
+ programs.push(format!("{name}.exe"));
+ }
+ programs.dedup();
+ programs
+}
+
+/// Whether `program` (an executable's file name) is a browser whose
+/// address bar and page this module can read.
+pub fn is_browser(program: &str) -> bool {
+ matches!(program, "chrome.exe" | "msedge.exe" | "brave.exe" | "opera.exe" | "vivaldi.exe" | "firefox.exe")
+}
+
+/// A visible top-level window.
+#[derive(Debug, Clone)]
+pub struct Window {
+ pub hwnd: HWND,
+ /// Its program's file name, lowercase ("chrome.exe").
+ pub program: String,
+ pub title: String,
+}
+
+/// The visible, uncloaked top-level windows that aren't minimized, front
+/// to back.
+pub fn windows() -> Vec<Window> {
+ unsafe extern "system" fn each(hwnd: HWND, found: LPARAM) -> BOOL {
+ // SAFETY: `found` is the Vec passed to EnumWindows below, alive for the call.
+ let found = unsafe { &mut *(found.0 as *mut Vec<HWND>) };
+ found.push(hwnd);
+ BOOL(1)
+ }
+ let mut handles: Vec<HWND> = Vec::new();
+ // SAFETY: the callback only pushes to `handles`, which outlives the call.
+ let _ = unsafe { EnumWindows(Some(each), LPARAM(&mut handles as *mut Vec<HWND> as isize)) };
+ handles
+ .into_iter()
+ .filter(|&hwnd| shown(hwnd))
+ .filter_map(|hwnd| Some(Window { hwnd, program: program_of(hwnd)?, title: title_of(hwnd) }))
+ .collect()
+}
+
+fn shown(hwnd: HWND) -> bool {
+ // SAFETY: plain queries on a window handle (a stale one just fails).
+ unsafe {
+ if !IsWindowVisible(hwnd).as_bool() || IsIconic(hwnd).as_bool() {
+ return false;
+ }
+ if GetWindow(hwnd, GW_OWNER).is_ok_and(|owner| !owner.is_invalid()) {
+ return false;
+ }
+ let mut cloaked = 0u32;
+ let _ = DwmGetWindowAttribute(
+ hwnd,
+ DWMWA_CLOAKED,
+ (&mut cloaked as *mut u32).cast(),
+ size_of::<u32>() as u32,
+ );
+ cloaked == 0
+ }
+}
+
+fn title_of(hwnd: HWND) -> String {
+ let mut buffer = [0u16; 512];
+ // SAFETY: the buffer's length bounds what's written.
+ let len = unsafe { GetWindowTextW(hwnd, &mut buffer) };
+ String::from_utf16_lossy(&buffer[..len.max(0) as usize])
+}
+
+fn program_of(hwnd: HWND) -> Option<String> {
+ let mut pid = 0u32;
+ // SAFETY: plain process queries; the handle is closed below.
+ unsafe {
+ GetWindowThreadProcessId(hwnd, Some(&mut pid));
+ let process = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid).ok()?;
+ let mut buffer = [0u16; 1024];
+ let mut len = buffer.len() as u32;
+ let named = QueryFullProcessImageNameW(
+ process,
+ PROCESS_NAME_WIN32,
+ windows::core::PWSTR(buffer.as_mut_ptr()),
+ &mut len,
+ );
+ let _ = CloseHandle(process);
+ named.ok()?;
+ let path = String::from_utf16_lossy(&buffer[..len as usize]);
+ Some(path.rsplit('\\').next().unwrap_or(&path).to_ascii_lowercase())
+ }
+}
+
+/// The window that plays `title` for the media session `player`: one of
+/// its program's windows whose title has the media's title in it (a
+/// browser's shows its active tab's), or its only window.
+pub fn window_for(player: &str, title: &str) -> Option<Window> {
+ let programs = programs_for(player);
+ let candidates: Vec<Window> = windows().into_iter().filter(|w| programs.contains(&w.program)).collect();
+ pick(candidates, title)
+}
+
+fn pick(candidates: Vec<Window>, title: &str) -> Option<Window> {
+ let wanted = title.trim().to_lowercase();
+ if !wanted.is_empty()
+ && let Some(found) = candidates.iter().find(|w| w.title.to_lowercase().contains(&wanted))
+ {
+ return Some(found.clone());
+ }
+ match candidates.as_slice() {
+ [only] => Some(only.clone()),
+ _ => None,
+ }
+}
+
+/// Has this thread see the screen in physical pixels, as screen captures
+/// do, whatever the process's DPI awareness: UI Automation's places then
+/// match what a capture of the screen shows.
+pub fn use_physical_pixels() {
+ // SAFETY: changes this thread's DPI awareness only.
+ let _ = unsafe { SetThreadDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2) };
+}
+
+/// UI Automation, on a thread that has COM.
+pub struct Automation(IUIAutomation);
+
+impl Automation {
+ pub fn new() -> windows::core::Result<Automation> {
+ // SAFETY: creates the system's UI Automation client.
+ unsafe { CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) }.map(Automation)
+ }
+
+ fn find(
+ &self,
+ root: &IUIAutomationElement,
+ property: UIA_PROPERTY_ID,
+ value: VARIANT,
+ ) -> Option<IUIAutomationElement> {
+ // SAFETY: COM calls on live interfaces.
+ unsafe {
+ let condition = self.0.CreatePropertyCondition(property, &value).ok()?;
+ root.FindFirst(TreeScope_Descendants, &condition).ok()
+ }
+ }
+
+ fn root(&self, window: &Window) -> Option<IUIAutomationElement> {
+ // SAFETY: a stale handle just fails.
+ unsafe { self.0.ElementFromHandle(window.hwnd).ok() }
+ }
+
+ /// The address of the page the browser `window` shows, with its scheme
+ /// (address bars leave "https://" out).
+ pub fn page_url(&self, window: &Window) -> Option<String> {
+ let root = self.root(window)?;
+ let bar = self
+ .find(&root, UIA_ClassNamePropertyId, VARIANT::from("OmniboxViewViews"))
+ .or_else(|| self.find(&root, UIA_AutomationIdPropertyId, VARIANT::from("urlbar-input")))?;
+ // SAFETY: a COM call on a live element.
+ let value = unsafe { bar.GetCurrentPropertyValue(UIA_ValueValuePropertyId) }.ok()?;
+ let text = BSTR::try_from(&value).ok()?.to_string();
+ with_scheme(&text)
+ }
+
+ /// Where on screen the video plays in `window`, in physical pixels:
+ /// YouTube's player when it's there, else the page, else the window.
+ pub fn video_rect(&self, window: &Window) -> Option<RECT> {
+ let root = self.root(window);
+ let element = root.as_ref().and_then(|root| {
+ self.find(root, UIA_ClassNamePropertyId, VARIANT::from("video-stream html5-main-video")).or_else(
+ || self.find(root, UIA_ControlTypePropertyId, VARIANT::from(UIA_DocumentControlTypeId.0)),
+ )
+ });
+ // SAFETY: a COM call on a live element.
+ let rect = element.and_then(|e| unsafe { e.CurrentBoundingRectangle() }.ok());
+ rect.filter(|r| r.right - r.left >= 64 && r.bottom - r.top >= 64).or_else(|| frame_of(window.hwnd))
+ }
+}
+
+/// A window's frame on screen, without its invisible resize borders.
+pub fn frame_of(hwnd: HWND) -> Option<RECT> {
+ let mut rect = RECT::default();
+ // SAFETY: writes one RECT.
+ unsafe {
+ DwmGetWindowAttribute(
+ hwnd,
+ DWMWA_EXTENDED_FRAME_BOUNDS,
+ (&mut rect as *mut RECT).cast(),
+ size_of::<RECT>() as u32,
+ )
+ }
+ .ok()?;
+ (rect.right > rect.left && rect.bottom > rect.top).then_some(rect)
+}
+
+/// An address bar's text as a web address: "youtube.com/watch?v=x" becomes
+/// "https://youtube.com/watch?v=x". Anything that isn't a web page (a
+/// search being typed, a browser's own pages) is `None`.
+fn with_scheme(text: &str) -> Option<String> {
+ let text = text.trim();
+ if text.is_empty() || text.contains(char::is_whitespace) {
+ return None;
+ }
+ let lower = text.to_ascii_lowercase();
+ if lower.starts_with("https://") || lower.starts_with("http://") {
+ return Some(text.to_owned());
+ }
+ if lower.contains("://") || lower.starts_with("about:") {
+ return None;
+ }
+ let host = lower.split(['/', '?', '#']).next().unwrap_or("");
+ let host = host.split(':').next().unwrap_or("");
+ (host.contains('.') || host == "localhost").then(|| format!("https://{text}"))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn players_map_to_their_programs() {
+ assert_eq!(programs_for("Chrome"), ["chrome.exe"]);
+ assert_eq!(programs_for("MSEdge"), ["msedge.exe"]);
+ assert_eq!(programs_for("308046B0AF4A39CB"), ["firefox.exe"]);
+ assert_eq!(programs_for("vlc.exe"), ["vlc.exe"]);
+ assert_eq!(programs_for(r"C:\Program Files\VideoLAN\VLC\vlc.exe"), ["vlc.exe"]);
+ assert_eq!(programs_for("VideoLAN.VLC"), ["vlc.exe"]);
+ // A packaged app's ID: no program to find a window by.
+ assert!(programs_for("SpotifyAB.SpotifyMusic_zpdnekdrzrea0!Spotify").is_empty());
+ }
+
+ #[test]
+ fn address_bar_text_becomes_a_web_address() {
+ assert_eq!(
+ with_scheme("youtube.com/watch?v=abc").as_deref(),
+ Some("https://youtube.com/watch?v=abc")
+ );
+ assert_eq!(
+ with_scheme("https://www.netflix.com/watch/1").as_deref(),
+ Some("https://www.netflix.com/watch/1")
+ );
+ assert_eq!(with_scheme("127.0.0.1:8765").as_deref(), Some("https://127.0.0.1:8765"));
+ assert_eq!(with_scheme("cats on skateboards"), None, "a search being typed");
+ assert_eq!(with_scheme("chrome://settings"), None);
+ assert_eq!(with_scheme("about:blank"), None);
+ assert_eq!(with_scheme("settings"), None);
+ assert_eq!(with_scheme(""), None);
+ }
+
+ fn window(title: &str) -> Window {
+ Window { hwnd: HWND::default(), program: "chrome.exe".into(), title: title.into() }
+ }
+
+ #[test]
+ fn the_window_with_the_title_is_the_player() {
+ let picked =
+ pick(vec![window("Mail - Google Chrome"), window("A Song - YouTube - Google Chrome")], "a song");
+ assert_eq!(picked.unwrap().title, "A Song - YouTube - Google Chrome");
+ // The playing tab isn't in front in any window: can't tell which.
+ assert!(pick(vec![window("Mail"), window("News")], "A Song").is_none());
+ // One window: that's it, whatever its tab.
+ assert_eq!(pick(vec![window("Mail")], "A Song").unwrap().title, "Mail");
+ }
+
+ /// With a Chromium browser showing a page whose title has
+ /// `NECTARLINK_TEST_TAB` in it: reads its address and its video's place.
+ #[test]
+ #[ignore]
+ fn reads_a_real_browser() {
+ let wanted = std::env::var("NECTARLINK_TEST_TAB").expect("NECTARLINK_TEST_TAB");
+ // SAFETY: COM for this test's thread.
+ let _ = unsafe {
+ windows::Win32::System::Com::CoInitializeEx(
+ None,
+ windows::Win32::System::Com::COINIT_MULTITHREADED,
+ )
+ };
+ use_physical_pixels();
+ let window = window_for("Chrome", &wanted).expect("the test tab's window");
+ let automation = Automation::new().unwrap();
+ let url = automation.page_url(&window);
+ let rect = automation.video_rect(&window).unwrap();
+ eprintln!("{:?} {url:?} {rect:?}", window.title);
+ assert!(url.is_some_and(|u| u.starts_with("http")));
+ assert!(rect.right - rect.left >= 64);
+ }
+}
diff --git a/desktop/app/src/win/mod.rs b/desktop/app/src/win/mod.rs
index 0aab4ea..eb7ae48 100644
--- a/desktop/app/src/win/mod.rs
+++ b/desktop/app/src/win/mod.rs
@@ -9,6 +9,7 @@ pub mod audio;
pub mod audio_out;
pub mod autostart;
pub mod ble_beacon;
+pub mod browser;
pub mod bt_hfp;
pub mod clipboard;
pub mod doctor;
@@ -33,6 +34,7 @@ pub mod tray;
pub mod vcam;
pub mod wallpaper;
pub mod widget;
+pub mod window_capture;
pub mod wol;
use windows::{
diff --git a/desktop/app/src/win/window_capture.rs b/desktop/app/src/win/window_capture.rs
new file mode 100644
index 0000000..c2ee891
--- /dev/null
+++ b/desktop/app/src/win/window_capture.rs
@@ -0,0 +1,335 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+//! One window's own pixels, with Windows Graphics Capture: what the window
+//! shows even with other windows in front of it, and never theirs. Play on
+//! phone streams a player's window this way, cropped to its video.
+
+use windows::{
+ Graphics::{
+ Capture::{
+ Direct3D11CaptureFramePool, GraphicsCaptureAccess, GraphicsCaptureAccessKind,
+ GraphicsCaptureItem, GraphicsCaptureSession,
+ },
+ DirectX::{Direct3D11::IDirect3DDevice, DirectXPixelFormat},
+ SizeInt32,
+ },
+ Win32::{
+ Foundation::{HWND, RECT},
+ Graphics::{
+ Direct3D::D3D_DRIVER_TYPE_HARDWARE,
+ Direct3D11::{
+ D3D11_BOX, D3D11_CPU_ACCESS_READ, D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_MAP_READ,
+ D3D11_MAPPED_SUBRESOURCE, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, D3D11_USAGE_STAGING,
+ D3D11CreateDevice, ID3D11Device, ID3D11DeviceContext, ID3D11Texture2D,
+ },
+ Dxgi::{Common::DXGI_FORMAT_B8G8R8A8_UNORM, Common::DXGI_SAMPLE_DESC, IDXGIDevice},
+ },
+ System::{
+ Com::{COINIT_MULTITHREADED, CoInitializeEx},
+ WinRT::{
+ Direct3D11::{CreateDirect3D11DeviceFromDXGIDevice, IDirect3DDxgiInterfaceAccess},
+ Graphics::Capture::IGraphicsCaptureItemInterop,
+ },
+ },
+ },
+ core::{Interface, factory},
+};
+
+/// Frames the pool holds: one being read, one arriving.
+const BUFFERS: i32 = 2;
+
+/// A running capture of one window.
+pub struct WindowCapture {
+ hwnd: HWND,
+ device: ID3D11Device,
+ context: ID3D11DeviceContext,
+ winrt_device: IDirect3DDevice,
+ pool: Direct3D11CaptureFramePool,
+ session: GraphicsCaptureSession,
+ size: SizeInt32,
+ staging: Option<(ID3D11Texture2D, u32, u32)>,
+ /// The latest crop, kept for when no new frame came (nothing changed).
+ last: Vec<u8>,
+ last_size: (u32, u32),
+}
+
+impl WindowCapture {
+ /// Starts capturing `hwnd`, without the pointer and, where Windows
+ /// allows it, without the yellow capture border.
+ pub fn new(hwnd: HWND) -> windows::core::Result<WindowCapture> {
+ // SAFETY: COM for this thread (WinRT needs it; a second call is harmless).
+ let _ = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) };
+ if !GraphicsCaptureSession::IsSupported()? {
+ return Err(windows::core::Error::from(windows::Win32::Foundation::E_NOTIMPL));
+ }
+ let mut device = None;
+ let mut context = None;
+ // SAFETY: plain Direct3D and WinRT interop calls with owned values.
+ unsafe {
+ D3D11CreateDevice(
+ None,
+ D3D_DRIVER_TYPE_HARDWARE,
+ Default::default(),
+ D3D11_CREATE_DEVICE_BGRA_SUPPORT,
+ None,
+ D3D11_SDK_VERSION,
+ Some(&mut device),
+ None,
+ Some(&mut context),
+ )?;
+ }
+ let device: ID3D11Device =
+ device.ok_or_else(|| windows::core::Error::from(windows::Win32::Foundation::E_FAIL))?;
+ let context =
+ context.ok_or_else(|| windows::core::Error::from(windows::Win32::Foundation::E_FAIL))?;
+ // SAFETY: as above.
+ let (winrt_device, item) = unsafe {
+ let dxgi: IDXGIDevice = device.cast()?;
+ let winrt_device: IDirect3DDevice = CreateDirect3D11DeviceFromDXGIDevice(&dxgi)?.cast()?;
+ let interop = factory::<GraphicsCaptureItem, IGraphicsCaptureItemInterop>()?;
+ let item: GraphicsCaptureItem = interop.CreateForWindow(hwnd)?;
+ (winrt_device, item)
+ };
+ let size = item.Size()?;
+ let pool = Direct3D11CaptureFramePool::CreateFreeThreaded(
+ &winrt_device,
+ DirectXPixelFormat::B8G8R8A8UIntNormalized,
+ BUFFERS,
+ size,
+ )?;
+ let session = pool.CreateCaptureSession(&item)?;
+ let _ = session.SetIsCursorCaptureEnabled(false);
+ // Windows 11 can leave the border off; earlier versions always draw it.
+ if GraphicsCaptureAccess::RequestAccessAsync(GraphicsCaptureAccessKind::Borderless)
+ .and_then(|asked| asked.join())
+ .is_ok()
+ {
+ let _ = session.SetIsBorderRequired(false);
+ }
+ session.StartCapture()?;
+ Ok(WindowCapture {
+ hwnd,
+ device,
+ context,
+ winrt_device,
+ pool,
+ session,
+ size,
+ staging: None,
+ last: Vec::new(),
+ last_size: (0, 0),
+ })
+ }
+
+ /// The window this captures.
+ pub fn window(&self) -> HWND {
+ self.hwnd
+ }
+
+ /// The window's latest picture of `crop` (in the window's own pixels,
+ /// from its top-left), scaled to `dst_w` × `dst_h` into `bgra`. Without
+ /// a new frame (nothing changed on screen), the last one again. False
+ /// before the first frame.
+ pub fn capture(&mut self, crop: RECT, dst_w: u32, dst_h: u32, bgra: &mut Vec<u8>) -> bool {
+ if let Err(e) = self.latest(crop) {
+ tracing::debug!(error = %e, "window capture frame failed");
+ }
+ let (w, h) = self.last_size;
+ if w == 0 || h == 0 {
+ return false;
+ }
+ scale_bgra(&self.last, w, h, dst_w, dst_h, bgra);
+ true
+ }
+
+ /// Reads the newest frame, if one came, into `last`.
+ fn latest(&mut self, crop: RECT) -> windows::core::Result<()> {
+ // Only the newest matters: older frames are dropped.
+ let mut newest = None;
+ while let Ok(frame) = self.pool.TryGetNextFrame() {
+ newest = Some(frame);
+ }
+ let Some(frame) = newest else { return Ok(()) };
+ let content = frame.ContentSize()?;
+ if content.Width != self.size.Width || content.Height != self.size.Height {
+ // The window changed size: frames come at the new size from the next one.
+ self.size = content;
+ self.pool.Recreate(
+ &self.winrt_device,
+ DirectXPixelFormat::B8G8R8A8UIntNormalized,
+ BUFFERS,
+ content,
+ )?;
+ }
+ // SAFETY: COM calls on live interfaces; the staging texture is
+ // mapped for reading and unmapped before it's used again.
+ unsafe {
+ let access: IDirect3DDxgiInterfaceAccess = frame.Surface()?.cast()?;
+ let texture: ID3D11Texture2D = access.GetInterface()?;
+ let mut desc = D3D11_TEXTURE2D_DESC::default();
+ texture.GetDesc(&mut desc);
+ let (tw, th) =
+ (desc.Width.min(content.Width.max(0) as u32), desc.Height.min(content.Height.max(0) as u32));
+ let left = crop.left.clamp(0, tw as i32) as u32;
+ let top = crop.top.clamp(0, th as i32) as u32;
+ let right = crop.right.clamp(0, tw as i32) as u32;
+ let bottom = crop.bottom.clamp(0, th as i32) as u32;
+ if right <= left || bottom <= top {
+ return Ok(());
+ }
+ let (w, h) = (right - left, bottom - top);
+ let staging = self.staging(w, h)?;
+ let area = D3D11_BOX { left, top, front: 0, right, bottom, back: 1 };
+ self.context.CopySubresourceRegion(&staging, 0, 0, 0, 0, &texture, 0, Some(&area));
+ let mut mapped = D3D11_MAPPED_SUBRESOURCE::default();
+ self.context.Map(&staging, 0, D3D11_MAP_READ, 0, Some(&mut mapped))?;
+ let row = w as usize * 4;
+ self.last.resize(row * h as usize, 0);
+ let source =
+ std::slice::from_raw_parts(mapped.pData.cast::<u8>(), mapped.RowPitch as usize * h as usize);
+ for (y, line) in self.last.chunks_exact_mut(row).enumerate() {
+ line.copy_from_slice(&source[y * mapped.RowPitch as usize..][..row]);
+ }
+ self.context.Unmap(&staging, 0);
+ self.last_size = (w, h);
+ }
+ Ok(())
+ }
+
+ /// A texture the CPU can read, `w` × `h`, reused while the size stays.
+ fn staging(&mut self, w: u32, h: u32) -> windows::core::Result<ID3D11Texture2D> {
+ if let Some((texture, sw, sh)) = &self.staging
+ && (*sw, *sh) == (w, h)
+ {
+ return Ok(texture.clone());
+ }
+ let desc = D3D11_TEXTURE2D_DESC {
+ Width: w,
+ Height: h,
+ MipLevels: 1,
+ ArraySize: 1,
+ Format: DXGI_FORMAT_B8G8R8A8_UNORM,
+ SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 },
+ Usage: D3D11_USAGE_STAGING,
+ BindFlags: 0,
+ CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32,
+ MiscFlags: 0,
+ };
+ let mut texture = None;
+ // SAFETY: creates a texture from a filled-in description.
+ unsafe { self.device.CreateTexture2D(&desc, None, Some(&mut texture))? };
+ let texture =
+ texture.ok_or_else(|| windows::core::Error::from(windows::Win32::Foundation::E_FAIL))?;
+ self.staging = Some((texture.clone(), w, h));
+ Ok(texture)
+ }
+}
+
+impl Drop for WindowCapture {
+ fn drop(&mut self) {
+ let _ = self.session.Close();
+ let _ = self.pool.Close();
+ }
+}
+
+/// Scales a `w` × `h` BGRA picture to `dst_w` × `dst_h` (bilinear), or
+/// copies it when the size is the same.
+pub(crate) fn scale_bgra(src: &[u8], w: u32, h: u32, dst_w: u32, dst_h: u32, out: &mut Vec<u8>) {
+ let (w, h, dw, dh) = (w as usize, h as usize, dst_w as usize, dst_h as usize);
+ out.resize(dw * dh * 4, 0);
+ if (w, h) == (dw, dh) {
+ out.copy_from_slice(&src[..w * h * 4]);
+ return;
+ }
+ // Where each output column samples from: left pixel and weight (of 256).
+ let columns: Vec<(usize, u32)> = (0..dw)
+ .map(|x| {
+ let at = ((x as f32 + 0.5) * w as f32 / dw as f32 - 0.5).clamp(0.0, (w - 1) as f32);
+ (at as usize, ((at - at.floor()) * 256.0) as u32)
+ })
+ .collect();
+ for (y, line) in out.chunks_exact_mut(dw * 4).enumerate() {
+ let at = ((y as f32 + 0.5) * h as f32 / dh as f32 - 0.5).clamp(0.0, (h - 1) as f32);
+ let (y0, wy) = (at as usize, ((at - at.floor()) * 256.0) as u32);
+ let y1 = (y0 + 1).min(h - 1);
+ let (top, bottom) = (&src[y0 * w * 4..][..w * 4], &src[y1 * w * 4..][..w * 4]);
+ for (x, pixel) in line.as_chunks_mut::<4>().0.iter_mut().enumerate() {
+ let (x0, wx) = columns[x];
+ let x1 = (x0 + 1).min(w - 1);
+ for c in 0..4 {
+ let row = |r: &[u8]| u32::from(r[x0 * 4 + c]) * (256 - wx) + u32::from(r[x1 * 4 + c]) * wx;
+ let value = (row(top) * (256 - wy) + row(bottom) * wy + (1 << 15)) >> 16;
+ pixel[c] = value as u8;
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn same_size_is_a_copy() {
+ let src: Vec<u8> = (0..4 * 3 * 4).map(|i| i as u8).collect();
+ let mut out = Vec::new();
+ scale_bgra(&src, 4, 3, 4, 3, &mut out);
+ assert_eq!(out, src);
+ }
+
+ #[test]
+ fn halving_averages_and_a_flat_picture_stays_flat() {
+ // 4x2: left half black, right half white.
+ let mut src = Vec::new();
+ for _ in 0..2 {
+ for x in 0..4 {
+ let v = if x < 2 { 0 } else { 255 };
+ src.extend_from_slice(&[v, v, v, 255]);
+ }
+ }
+ let mut out = Vec::new();
+ scale_bgra(&src, 4, 2, 2, 1, &mut out);
+ assert_eq!(out.len(), 2 * 4);
+ assert!(out[0] < 40 && out[4] > 215, "{out:?}");
+ assert_eq!(out[3], 255);
+
+ let flat = vec![77u8; 10 * 6 * 4];
+ scale_bgra(&flat, 10, 6, 7, 3, &mut out);
+ assert!(out.iter().all(|&v| v == 77));
+ }
+
+ /// With a Chromium browser showing a page whose title has
+ /// `NECTARLINK_TEST_TAB` in it: captures its window's video, covered
+ /// or not, and writes it to `NECTARLINK_TEST_DUMP` (raw BGRA) if set.
+ #[test]
+ #[ignore]
+ fn captures_a_window_behind_others() {
+ use crate::win::browser::{Automation, frame_of, use_physical_pixels, window_for};
+ let wanted = std::env::var("NECTARLINK_TEST_TAB").expect("NECTARLINK_TEST_TAB");
+ // SAFETY: COM for this test's thread.
+ let _ = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) };
+ use_physical_pixels();
+ let window = window_for("Chrome", &wanted).expect("the test tab's window");
+ let video = Automation::new().unwrap().video_rect(&window).expect("the video");
+ let frame = frame_of(window.hwnd).unwrap();
+ let crop = RECT {
+ left: video.left - frame.left,
+ top: video.top - frame.top,
+ right: video.right - frame.left,
+ bottom: video.bottom - frame.top,
+ };
+ let (w, h) = ((crop.right - crop.left) as u32, (crop.bottom - crop.top) as u32);
+ let mut capture = WindowCapture::new(window.hwnd).unwrap();
+ let mut bgra = Vec::new();
+ let started = std::time::Instant::now();
+ while !capture.capture(crop, w, h, &mut bgra) {
+ assert!(started.elapsed() < std::time::Duration::from_secs(3), "no frame");
+ std::thread::sleep(std::time::Duration::from_millis(30));
+ }
+ let distinct: std::collections::HashSet<&[u8; 4]> = bgra.as_chunks::<4>().0.iter().collect();
+ eprintln!("{crop:?} {w}x{h}: {} colors", distinct.len());
+ if let Ok(path) = std::env::var("NECTARLINK_TEST_DUMP") {
+ std::fs::write(path, &bgra).unwrap();
+ }
+ assert!(distinct.len() > 50);
+ }
+}