Your Android phone and your Windows PC, working as one. Local-first, end-to-end encrypted, no account.
Play on phone: the PC's side
Bee96e407544579
diff --git a/core/nectarlink-core/src/features.rs b/core/nectarlink-core/src/features.rsindex b4e25c2..7e4e4e8 100644--- a/core/nectarlink-core/src/features.rs+++ b/core/nectarlink-core/src/features.rs@@ -847,6 +847,12 @@ pub const FEATURES: &[FeatureDef] = &[requires: &[desktop("media.control", UPDATE), phone("media.remote", UPDATE), DeviceToggle("media")],partial: None,},+ FeatureDef {+ id: "media.play_on_phone",+ group: FeatureGroup::Media,+ requires: &[desktop("media.stream", UPDATE), phone("media.watch", UPDATE), DeviceToggle("media")],+ partial: None,+ },FeatureDef {id: "media.pc_audio",group: FeatureGroup::Media,diff --git a/desktop/app/Cargo.toml b/desktop/app/Cargo.tomlindex 8d115ce..edd4e7c 100644--- a/desktop/app/Cargo.toml+++ b/desktop/app/Cargo.toml@@ -43,6 +43,7 @@ windows = { workspace = true, features = ["Foundation_Collections","Media","Media_Control",+ "Security_Authorization_AppCapabilityAccess","Security_Cryptography","Storage","Storage_Provider",@@ -58,6 +59,10 @@ windows = { workspace = true, features = ["Devices_Bluetooth_Rfcomm","Devices_Enumeration","Gaming_Input",+ "Graphics",+ "Graphics_Capture",+ "Graphics_DirectX",+ "Graphics_DirectX_Direct3D11","UI_Notifications","Win32_Foundation","Win32_UI_Input_XboxController",@@ -85,6 +90,10 @@ windows = { workspace = true, features = ["Win32_System_SystemInformation","Win32_System_Threading","Win32_System_WinRT",+ "Win32_System_WinRT_Direct3D11",+ "Win32_System_WinRT_Graphics_Capture",+ "Win32_UI_Accessibility",+ "Win32_UI_HiDpi","Win32_UI_Input_KeyboardAndMouse","Win32_UI_Shell","Win32_UI_Shell_Common",diff --git a/desktop/app/qml/components/NowPlaying.qml b/desktop/app/qml/components/NowPlaying.qmlindex 8bd70e0..4329712 100644--- a/desktop/app/qml/components/NowPlaying.qml+++ b/desktop/app/qml/components/NowPlaying.qml@@ -382,6 +382,17 @@ Card {anchors.verticalCenter: parent.verticalCenterspacing: 2+ // Play on phone: this PC's player, on the phone (streamed, or in the service's app).+ IconButton {+ visible: card.isLocalPc && MediaList.playOnPhoneName !== ""+ width: 24; height: 24+ iconSize: 13+ iconPath: Icons.phone+ iconColor: Theme.surfaceContentVariant+ label: qsTr("Play on %1").arg(MediaList.playOnPhoneName)+ onClicked: card.send("play_on_phone")+ }+IconButton {visible: !Theme.graphitewidth: 24; height: 24diff --git a/desktop/app/src/bridge/media.rs b/desktop/app/src/bridge/media.rsindex 6302100..a4abaac 100644--- a/desktop/app/src/bridge/media.rs+++ b/desktop/app/src/bridge/media.rs@@ -42,6 +42,8 @@ pub mod qobject {#[qproperty(i32, revision)]#[qproperty(QString, active_art, cxx_name = "activeArt")]#[qproperty(QString, active_colors, cxx_name = "activeColors")]+ /// The phone Play on phone plays on (its name), or "" when none is online.+ #[qproperty(QString, play_on_phone_name, cxx_name = "playOnPhoneName")]type MediaList = super::MediaListRust;}@@ -87,7 +89,8 @@ pub mod qobject {fn row_count(self: &MediaList, parent: &QModelIndex) -> i32;/// Runs "play", "pause", "next", "previous" or "seek" (to- /// `position_ms`) on a device's player.+ /// `position_ms`) on a device's player, or for this PC's,+ /// "play_on_phone".#[qinvokable]fn command(self: &MediaList, device: &QString, player: &QString, action: &QString, position_ms: f64);/// The row of a device's first player (the one to show), or -1.@@ -110,6 +113,7 @@ pub struct MediaListRust {revision: i32,active_art: QString,active_colors: QString,+ play_on_phone_name: QString,rows: Vec<PlayerView>,}@@ -180,7 +184,11 @@ fn role_value(row: &PlayerView, role: &str) -> QVariant {impl cxx_qt::Initialize for qobject::MediaList {fn initialize(self: Pin<&mut Self>) {- super::subscribe(self.qt_thread(), Changes::MEDIA, Self::refresh);+ super::subscribe(+ self.qt_thread(),+ Changes::MEDIA | Changes::DEVICES | Changes::CAPABILITIES,+ Self::refresh,+ );}}@@ -231,6 +239,10 @@ impl qobject::MediaList {if *self.active_colors() != colors_json {self.as_mut().set_active_colors(colors_json);}+ let phone = QString::from(crate::play_on_phone::phone().map(|(_, name)| name).unwrap_or_default());+ if *self.play_on_phone_name() != phone {+ self.as_mut().set_play_on_phone_name(phone);+ }let revision = self.revision.wrapping_add(1);self.set_revision(revision);}@@ -282,6 +294,12 @@ impl qobject::MediaList {let resolved_device =if device_str == "local" { Some(media::LOCAL_DEVICE) } else { super::parse_device(device) };let Some(device) = resolved_device else { return };+ if action_str == "play_on_phone" {+ if device == media::LOCAL_DEVICE {+ crate::play_on_phone::play(String::from(player));+ }+ return;+ }if action_str == "volume" {let level = (position_ms.round() as i32).clamp(0, 100) as u8;if device == media::LOCAL_DEVICE {diff --git a/desktop/app/src/core_host.rs b/desktop/app/src/core_host.rsindex e767e9f..a0b514a 100644--- a/desktop/app/src/core_host.rs+++ b/desktop/app/src/core_host.rs@@ -118,6 +118,7 @@ pub fn compute_capabilities(node: Option<&Node>) -> Vec<String> {nectarlink_core::STORAGE_READ.into(),nectarlink_core::STORAGE_WRITE.into(),nectarlink_core::WEBCAM_VIRTUAL.into(),+ nectarlink_core::MEDIA_STREAM.into(),];let mut pc_mirror_enabled = crate::pc_mirror::is_active();let mut remote_input_enabled = false;diff --git a/desktop/app/src/main.rs b/desktop/app/src/main.rsindex c9500ac..613f283 100644--- a/desktop/app/src/main.rs+++ b/desktop/app/src/main.rs@@ -38,6 +38,7 @@ mod pc_mirror;mod pc_notifications;mod pc_storage;mod photos;+mod play_on_phone;mod qr;mod recordings;mod remote;diff --git a/desktop/app/src/pc_audio_stream.rs b/desktop/app/src/pc_audio_stream.rsindex 96f685f..26732e1 100644--- a/desktop/app/src/pc_audio_stream.rs+++ b/desktop/app/src/pc_audio_stream.rs@@ -31,6 +31,11 @@ static WORKER_RUNNING: AtomicBool = AtomicBool::new(false);/// when the PC turned it on (or that phone is away), to the first phone that is.static TARGET: std::sync::Mutex<Option<nectarlink_core::DeviceId>> = std::sync::Mutex::new(None);+/// The phone the sound goes to first, if one asked.+pub fn target() -> Option<nectarlink_core::DeviceId> {+ *TARGET.lock().unwrap_or_else(|e| e.into_inner())+}+/// Records which phone asked to be the speaker.pub fn set_target(phone: Option<nectarlink_core::DeviceId>) {*TARGET.lock().unwrap_or_else(|e| e.into_inner()) = phone;diff --git a/desktop/app/src/pc_mirror.rs b/desktop/app/src/pc_mirror.rsindex 75a7985..c4a550c 100644--- a/desktop/app/src/pc_mirror.rs+++ b/desktop/app/src/pc_mirror.rs@@ -19,7 +19,7 @@ use nectarlink_core::{DeviceId, KeyMod, MirrorConfig, MirrorInput, MirrorSend, MirrorStart, PacketKind, mirror_keys,};use windows::Win32::{- Foundation::POINT,+ Foundation::{POINT, RECT},Graphics::Gdi::{BI_RGB, BITMAPINFO, BITMAPINFOHEADER, CreateCompatibleBitmap, CreateCompatibleDC, DESKTOPHORZRES,DESKTOPVERTRES, DeleteDC, DeleteObject, GetDC, GetDIBits, GetDeviceCaps, HALFTONE, ReleaseDC,@@ -56,6 +56,8 @@ struct ActiveSession {peer: DeviceId,stop: Arc<AtomicBool>,force_keyframe: Arc<AtomicBool>,+ /// Play on phone: the phone watches a player, and its input is ignored.+ watching: bool,}static ACTIVE: Mutex<Option<ActiveSession>> = Mutex::new(None);@@ -94,6 +96,7 @@ pub fn start(peer: DeviceId, options: MirrorStart) {peer,stop: Arc::clone(&stop),force_keyframe: Arc::clone(&force_keyframe),+ watching: options.watch.is_some(),});}core_host::host().hub.changed(crate::state::Changes::REMOTE);@@ -166,7 +169,8 @@ pub fn handle_input(peer: &DeviceId, session: u32, input: MirrorInput) {if session != nectarlink_core::MIRROR_SCREEN {return;}- if active_peer() != Some(*peer) {+ let watching = ACTIVE.lock().unwrap_or_else(|e| e.into_inner()).as_ref().is_some_and(|s| s.watching);+ if active_peer() != Some(*peer) || watching {return;}let remote_input_on = core_host::node()@@ -540,19 +544,27 @@ fn has_h264_sps(data: &[u8]) -> bool {data.windows(4).any(|w| w[..3] == [0, 0, 1] && (w[3] & 0x1f) == 7)}-/// Captures the primary desktop monitor into `bgra` scaled to `(dst_w, dst_h)`-/// and returns the current physical screen dimensions `(src_w, src_h)`.-fn capture_primary_screen(dst_w: u32, dst_h: u32, bgra: &mut Vec<u8>) -> Option<(u32, u32)> {+/// Captures the primary desktop monitor into `bgra` scaled to `(dst_w, dst_h)`,+/// with the pointer, and returns the current physical screen dimensions+/// `(src_w, src_h)`. With `region` (physical screen pixels), captures only+/// that, without the pointer (Play on phone), and returns its size.+fn capture_screen(region: Option<RECT>, dst_w: u32, dst_h: u32, bgra: &mut Vec<u8>) -> Option<(u32, u32)> {// SAFETY: Win32 GDI device context and bitmap calls with balanced cleanup.unsafe {let hdc_screen = GetDC(None);if hdc_screen.is_invalid() {return None;}- let phys_w = GetDeviceCaps(Some(hdc_screen), DESKTOPHORZRES);- let phys_h = GetDeviceCaps(Some(hdc_screen), DESKTOPVERTRES);- let src_w = if phys_w > 0 { phys_w } else { GetSystemMetrics(SM_CXSCREEN) }.max(1) as u32;- let src_h = if phys_h > 0 { phys_h } else { GetSystemMetrics(SM_CYSCREEN) }.max(1) as u32;+ let (src_x, src_y, src_w, src_h) = match region {+ Some(r) => (r.left, r.top, (r.right - r.left).max(1) as u32, (r.bottom - r.top).max(1) as u32),+ None => {+ let phys_w = GetDeviceCaps(Some(hdc_screen), DESKTOPHORZRES);+ let phys_h = GetDeviceCaps(Some(hdc_screen), DESKTOPVERTRES);+ let w = if phys_w > 0 { phys_w } else { GetSystemMetrics(SM_CXSCREEN) }.max(1) as u32;+ let h = if phys_h > 0 { phys_h } else { GetSystemMetrics(SM_CYSCREEN) }.max(1) as u32;+ (0, 0, w, h)+ }+ };let hdc_mem = CreateCompatibleDC(Some(hdc_screen));if hdc_mem.is_invalid() {@@ -574,8 +586,8 @@ fn capture_primary_screen(dst_w: u32, dst_h: u32, bgra: &mut Vec<u8>) -> Option<dst_w as i32,dst_h as i32,Some(hdc_screen),- 0,- 0,+ src_x,+ src_y,src_w as i32,src_h as i32,SRCCOPY,@@ -589,7 +601,7 @@ fn capture_primary_screen(dst_w: u32, dst_h: u32, bgra: &mut Vec<u8>) -> Option<hCursor: Default::default(),ptScreenPos: POINT::default(),};- if GetCursorInfo(&mut ci).is_ok() && (ci.flags.0 & CURSOR_SHOWING.0) != 0 {+ if region.is_none() && GetCursorInfo(&mut ci).is_ok() && (ci.flags.0 & CURSOR_SHOWING.0) != 0 {let cx = ((ci.ptScreenPos.x as i64) * (dst_w as i64) / (src_w as i64)) as i32;let cy = ((ci.ptScreenPos.y as i64) * (dst_h as i64) / (src_h as i64)) as i32;let _ = DrawIconEx(hdc_mem, cx, cy, ci.hCursor.into(), 0, 0, 0, None, DI_NORMAL);@@ -670,11 +682,50 @@ fn run_capture_loop(let mut last_rate_change = Instant::now();let mut stats = Stats::default();let mut stats_since = Instant::now();- tracing::info!(%peer, fps, bitrate, "sharing this PC's screen");+ // Play on phone: the player's picture instead of the screen, and the+ // PC's sound to that phone while it watches. The picture's place comes+ // in physical pixels; so does this thread's capture.+ if options.watch.is_some() {+ win::browser::use_physical_pixels();+ }+ let watched = options.watch.as_deref().map(crate::play_on_phone::Watched::follow);+ let mut window_capture: Option<win::window_capture::WindowCapture> = None;+ let _sound = watched.is_some().then(|| SoundToPhone::start(peer));+ if watched.is_some() {+ tracing::info!(%peer, fps, bitrate, "streaming a player to the phone");+ } else {+ tracing::info!(%peer, fps, bitrate, "sharing this PC's screen");+ }while !stop.load(Ordering::Relaxed) {let tick_start = Instant::now();+ let place = watched.as_ref().and_then(crate::play_on_phone::Watched::place);+ if watched.is_some() {+ let Some(place) = place else {+ // Not found yet: nothing to show.+ std::thread::sleep(frame_interval);+ continue;+ };+ let r = place.on_screen;+ let dims =+ target_dimensions((r.right - r.left) as u32, (r.bottom - r.top) as u32, options.max_size);+ if stream_dims != Some(dims) {+ // A new size (the window moved to full screen, say): a new stream config.+ stream_dims = Some(dims);+ encoder = None;+ }+ // The window's own pixels, whatever is in front of it; the+ // screen there when Windows can't capture the window.+ if window_capture.as_ref().is_none_or(|c| c.window() != place.window()) {+ window_capture = win::window_capture::WindowCapture::new(place.window())+ .inspect_err(+ |e| tracing::info!(error = %e, "can't capture the window; capturing the screen"),+ )+ .ok();+ }+ }+let (dst_w, dst_h) = match stream_dims {Some(dims) => dims,None => {@@ -726,9 +777,12 @@ fn run_capture_loop(}}- if capture_primary_screen(dst_w, dst_h, &mut bgra).is_some()- && let Some(enc) = encoder.as_mut()- {+ let captured = match (place, window_capture.as_mut()) {+ (Some(place), Some(capture)) => capture.capture(place.in_window, dst_w, dst_h, &mut bgra),+ (Some(place), None) => capture_screen(Some(place.on_screen), dst_w, dst_h, &mut bgra).is_some(),+ (None, _) => capture_screen(None, dst_w, dst_h, &mut bgra).is_some(),+ };+ if captured && let Some(enc) = encoder.as_mut() {bgra_to_nv12(&bgra, dst_w as usize, dst_h as usize, &mut nv12);let time_us = started.elapsed().as_micros() as u64;let hns = (time_us * 10) as i64;@@ -815,6 +869,32 @@ fn run_capture_loop(cleanup_session(&stop);}+/// The PC's sound going to a phone that watches one of its players (as+/// Phone speaker sends it), for as long as this lives; then as before.+struct SoundToPhone {+ was_on: bool,+ was_target: Option<DeviceId>,+}++impl SoundToPhone {+ fn start(peer: DeviceId) -> SoundToPhone {+ let sound = SoundToPhone {+ was_on: crate::pc_audio_stream::is_enabled(),+ was_target: crate::pc_audio_stream::target(),+ };+ crate::pc_audio_stream::set_target(Some(peer));+ crate::pc_audio_stream::set_enabled(true);+ sound+ }+}++impl Drop for SoundToPhone {+ fn drop(&mut self) {+ crate::pc_audio_stream::set_target(self.was_target);+ crate::pc_audio_stream::set_enabled(self.was_on);+ }+}+fn cleanup_session(stop: &Arc<AtomicBool>) {let removed = {let mut guard = ACTIVE.lock().unwrap_or_else(|e| e.into_inner());@@ -871,6 +951,33 @@ mod tests {assert!(decoded_any, "decoder must reconstruct at least one frame from H264Encoder");}+ /// With a Chromium browser showing a video on a page whose title has+ /// `NECTARLINK_TEST_TAB` in it, in view: captures just the video, as+ /// Play on phone does.+ #[test]+ #[ignore]+ fn captures_a_browsers_video() {+ use crate::win::browser::{Automation, window_for};+ let wanted = std::env::var("NECTARLINK_TEST_TAB").expect("NECTARLINK_TEST_TAB");+ // SAFETY: COM for this test's thread.+ let _ = unsafe {+ windows::Win32::System::Com::CoInitializeEx(+ None,+ windows::Win32::System::Com::COINIT_MULTITHREADED,+ )+ };+ crate::win::browser::use_physical_pixels();+ let window = window_for("Chrome", &wanted).expect("the test tab's window");+ let rect = Automation::new().unwrap().video_rect(&window).expect("the video");+ let (sw, sh) = ((rect.right - rect.left) as u32, (rect.bottom - rect.top) as u32);+ let (w, h) = target_dimensions(sw, sh, 1280);+ let mut bgra = Vec::new();+ assert_eq!(capture_screen(Some(rect), w, h, &mut bgra), Some((sw, sh)));+ let distinct: std::collections::HashSet<&[u8; 4]> = bgra.as_chunks::<4>().0.iter().collect();+ eprintln!("{rect:?} as {w}x{h}: {} colors", distinct.len());+ assert!(distinct.len() > 50, "a moving test pattern, not a blank");+ }+#[test]fn keyframes_come_on_request_without_rebuilding_the_encoder() {let (w, h) = target_dimensions(1280, 720, 1280);diff --git a/desktop/app/src/play_on_phone.rs b/desktop/app/src/play_on_phone.rsnew file mode 100644index 0000000..f654773--- /dev/null+++ b/desktop/app/src/play_on_phone.rs@@ -0,0 +1,205 @@+// SPDX-License-Identifier: GPL-3.0-or-later+//! Play on phone, on this PC (docs/protocol/media.md §4): moves what one of+//! its players plays to a phone. A protected streaming service opens on+//! the phone in its app and the PC pauses; anything else the phone watches+//! streamed from the PC, which keeps playing it.++#![allow(unsafe_code)]++use std::{+ sync::{+ Arc, Mutex,+ atomic::{AtomicBool, Ordering},+ },+ time::Duration,+};++use nectarlink_core::{DeviceId, Error, FeatureState, LinkState, MediaAction, PlayOnPhone};+use windows::Win32::{+ Foundation::{HWND, RECT},+ System::Com::{COINIT_MULTITHREADED, CoInitializeEx},+};++use crate::{+ bridge::app::{describe, show_message},+ core_host,+ win::browser::{self, Automation, Window},+};++/// The phone to play on: the first one online that can watch a PC's+/// playback, else the first one online.+pub fn phone() -> Option<(DeviceId, String)> {+ core_host::host().hub.read(|s| {+ let online: Vec<_> =+ s.devices.iter().filter(|d| matches!(d.link, LinkState::Online { .. })).collect();+ let watches = |id: &DeviceId| {+ s.matrices.get(id).and_then(|m| m.state("media.play_on_phone")) == Some(FeatureState::Available)+ };+ online.iter().find(|d| watches(&d.id)).or_else(|| online.first()).map(|d| (d.id, d.info.name.clone()))+ })+}++/// What this PC's `player` plays: its title, and where it is, in seconds.+fn now_playing(player: &str) -> (String, Option<u32>) {+ let views = crate::media::local_views();+ let Some(view) = views.into_iter().find(|v| v.player.id == player) else {+ return (String::new(), None);+ };+ let position = view.player.position.map(|ms| {+ let since =+ if view.player.playing { view.at.elapsed().map_or(0, |d| d.as_millis() as u64) } else { 0 };+ ((ms + since) / 1000) as u32+ });+ (view.player.title.unwrap_or_default(), position.filter(|&s| s > 0))+}++/// Plays what this PC's `player` plays on a phone (the Play on phone+/// button on the PC's media card).+pub fn play(player: String) {+ let Some((phone, name)) = phone() else {+ return show_message("Connect your phone to play on it.");+ };+ let spawned = std::thread::Builder::new().name("play-on-phone".into()).spawn(move || {+ // SAFETY: COM for this short-lived thread (UI Automation).+ let _ = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) };+ let (title, position) = now_playing(&player);+ let window = browser::window_for(&player, &title);+ let url = window+ .as_ref()+ .filter(|w| browser::is_browser(&w.program))+ .and_then(|w| Automation::new().ok()?.page_url(w));+ let plan = nectarlink_core::play_on_phone::plan(url.as_deref(), position);+ let Some(node) = core_host::node() else { return };+ core_host::spawn(async move {+ match plan {+ PlayOnPhone::Open { url, service } => match node.open_link(phone, url).await {+ Ok(()) => {+ crate::media::command(crate::media::LOCAL_DEVICE, player, MediaAction::Pause, None);+ show_message(format!("Opening in {service} on {name}. Tap the notification there."));+ }+ Err(e) => show_message(failed(&e, &name)),+ },+ PlayOnPhone::Stream { open_instead } => {+ let shown = (!title.is_empty()).then_some(title);+ match node.offer_watch(phone, player.clone(), shown).await {+ Ok(()) => show_message(format!("Tap the notification on {name} to watch.")),+ // A phone that can't watch yet: the page opens there instead.+ Err(Error::Unsupported) if let Some(url) = open_instead => {+ match node.open_link(phone, url).await {+ Ok(()) => {+ crate::media::command(+ crate::media::LOCAL_DEVICE,+ player,+ MediaAction::Pause,+ None,+ );+ show_message(format!("Sent to {name}. Tap the notification there."));+ }+ Err(e) => show_message(failed(&e, &name)),+ }+ }+ Err(e) => show_message(failed(&e, &name)),+ }+ }+ }+ });+ });+ if let Err(e) = spawned {+ tracing::warn!(error = %e, "can't start Play on phone");+ }+}++fn failed(error: &Error, phone: &str) -> String {+ match error {+ Error::Unsupported => format!("Update NectarLink on {phone} to play on it."),+ Error::Denied => format!("Media is turned off for {phone}."),+ other => describe(other),+ }+}++/// Where a watched player's picture is, kept up to date while a phone+/// watches it: the video moves when the page scrolls, the window moves or+/// goes full screen.+pub struct Watched {+ place: Arc<Mutex<Option<Place>>>,+ stop: Arc<AtomicBool>,+}++/// A watched player's picture: its window, and where in it the picture is.+#[derive(Debug, Clone, Copy)]+pub struct Place {+ /// The window's handle (as a number: handles go between threads).+ window: isize,+ /// The picture, in physical screen pixels.+ pub on_screen: RECT,+ /// The picture, in the window's own pixels (from its frame's top-left),+ /// as a capture of the window has it.+ pub in_window: RECT,+}++/// How often the picture's place is looked up again.+const FOLLOW_EVERY: Duration = Duration::from_millis(500);++impl Place {+ /// The window the picture is in.+ pub fn window(&self) -> HWND {+ HWND(self.window as *mut core::ffi::c_void)+ }+}++impl Watched {+ /// Starts following `player`'s picture.+ pub fn follow(player: &str) -> Watched {+ let place = Arc::new(Mutex::new(None));+ let stop = Arc::new(AtomicBool::new(false));+ let (found, stopped, player) = (place.clone(), stop.clone(), player.to_owned());+ let spawned = std::thread::Builder::new().name("play-on-phone-follow".into()).spawn(move || {+ // SAFETY: COM for this thread, which ends with the watching.+ let _ = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) };+ // Places in the pixels the capture uses.+ browser::use_physical_pixels();+ let automation = Automation::new().ok();+ let mut window: Option<Window> = None;+ while !stopped.load(Ordering::Relaxed) {+ // The window it played in, while it's still there; else find it again.+ if window.as_ref().is_none_or(|w| browser::frame_of(w.hwnd).is_none()) {+ let (title, _) = now_playing(&player);+ window = browser::window_for(&player, &title);+ }+ let place = window.as_ref().and_then(|w| {+ let frame = browser::frame_of(w.hwnd)?;+ let on_screen = match &automation {+ Some(a) if browser::is_browser(&w.program) => a.video_rect(w)?,+ _ => frame,+ };+ let in_window = RECT {+ left: on_screen.left - frame.left,+ top: on_screen.top - frame.top,+ right: on_screen.right - frame.left,+ bottom: on_screen.bottom - frame.top,+ };+ Some(Place { window: w.hwnd.0 as isize, on_screen, in_window })+ });+ if place.is_some() {+ *found.lock().unwrap_or_else(|e| e.into_inner()) = place;+ }+ std::thread::sleep(FOLLOW_EVERY);+ }+ });+ if let Err(e) = spawned {+ tracing::warn!(error = %e, "can't follow the watched player");+ }+ Watched { place, stop }+ }++ /// Where the picture is now, once known.+ pub fn place(&self) -> Option<Place> {+ *self.place.lock().unwrap_or_else(|e| e.into_inner())+ }+}++impl Drop for Watched {+ fn drop(&mut self) {+ self.stop.store(true, Ordering::Relaxed);+ }+}diff --git a/desktop/app/src/win/browser.rs b/desktop/app/src/win/browser.rsnew file mode 100644index 0000000..3babbed--- /dev/null+++ b/desktop/app/src/win/browser.rs@@ -0,0 +1,348 @@+// SPDX-License-Identifier: GPL-3.0-or-later+//! The window a player shows in, for Play on phone: which top-level window+//! belongs to a media session, the address of the page a browser plays in,+//! and where on screen the video is, read through UI Automation.+//!+//! Chromium browsers (Chrome, Edge, Brave, Opera, Vivaldi) expose their+//! address bar as an edit field of class `OmniboxViewViews` and a page's+//! elements with their HTML classes, so YouTube's player+//! (`video-stream html5-main-video`) can be found exactly; Firefox's+//! address bar has the automation ID `urlbar-input`. Nothing here reads a+//! page's text: only the address of the tab that plays.++use windows::{+ Win32::{+ Foundation::{CloseHandle, HWND, LPARAM, RECT},+ Graphics::Dwm::{DWMWA_CLOAKED, DWMWA_EXTENDED_FRAME_BOUNDS, DwmGetWindowAttribute},+ System::{+ Com::{CLSCTX_INPROC_SERVER, CoCreateInstance},+ Threading::{+ OpenProcess, PROCESS_NAME_WIN32, PROCESS_QUERY_LIMITED_INFORMATION,+ QueryFullProcessImageNameW,+ },+ Variant::VARIANT,+ },+ UI::{+ Accessibility::{+ CUIAutomation, IUIAutomation, IUIAutomationElement, TreeScope_Descendants,+ UIA_AutomationIdPropertyId, UIA_ClassNamePropertyId, UIA_ControlTypePropertyId,+ UIA_DocumentControlTypeId, UIA_PROPERTY_ID, UIA_ValueValuePropertyId,+ },+ HiDpi::{DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2, SetThreadDpiAwarenessContext},+ WindowsAndMessaging::{+ EnumWindows, GW_OWNER, GetWindow, GetWindowTextW, GetWindowThreadProcessId, IsIconic,+ IsWindowVisible,+ },+ },+ },+ core::{BOOL, BSTR},+};++/// The programs a media session's app ID stands for. Browsers report a+/// short name (or, for Firefox, a hash of its install path); desktop apps+/// often report their executable.+pub fn programs_for(player: &str) -> Vec<String> {+ let lower = player.to_ascii_lowercase();+ let known: &[(&str, &str)] = &[+ ("chrome", "chrome.exe"),+ ("msedge", "msedge.exe"),+ ("edge", "msedge.exe"),+ ("brave", "brave.exe"),+ ("opera", "opera.exe"),+ ("vivaldi", "vivaldi.exe"),+ ("firefox", "firefox.exe"),+ // Firefox's app ID is a hash of where it's installed; this is the+ // one for its default location.+ ("308046b0af4a39cb", "firefox.exe"),+ ];+ let mut programs: Vec<String> =+ known.iter().filter(|(id, _)| lower.contains(id)).map(|(_, exe)| (*exe).to_owned()).collect();+ // "vlc.exe", or a path, or "VideoLAN.VLC" (a desktop app ID).+ let last = lower.rsplit(['\\', '/']).next().unwrap_or(&lower);+ if last.ends_with(".exe") {+ programs.push(last.to_owned());+ } else if let Some((_, name)) = last.rsplit_once('.')+ && !name.is_empty()+ && !last.contains('!')+ {+ programs.push(format!("{name}.exe"));+ }+ programs.dedup();+ programs+}++/// Whether `program` (an executable's file name) is a browser whose+/// address bar and page this module can read.+pub fn is_browser(program: &str) -> bool {+ matches!(program, "chrome.exe" | "msedge.exe" | "brave.exe" | "opera.exe" | "vivaldi.exe" | "firefox.exe")+}++/// A visible top-level window.+#[derive(Debug, Clone)]+pub struct Window {+ pub hwnd: HWND,+ /// Its program's file name, lowercase ("chrome.exe").+ pub program: String,+ pub title: String,+}++/// The visible, uncloaked top-level windows that aren't minimized, front+/// to back.+pub fn windows() -> Vec<Window> {+ unsafe extern "system" fn each(hwnd: HWND, found: LPARAM) -> BOOL {+ // SAFETY: `found` is the Vec passed to EnumWindows below, alive for the call.+ let found = unsafe { &mut *(found.0 as *mut Vec<HWND>) };+ found.push(hwnd);+ BOOL(1)+ }+ let mut handles: Vec<HWND> = Vec::new();+ // SAFETY: the callback only pushes to `handles`, which outlives the call.+ let _ = unsafe { EnumWindows(Some(each), LPARAM(&mut handles as *mut Vec<HWND> as isize)) };+ handles+ .into_iter()+ .filter(|&hwnd| shown(hwnd))+ .filter_map(|hwnd| Some(Window { hwnd, program: program_of(hwnd)?, title: title_of(hwnd) }))+ .collect()+}++fn shown(hwnd: HWND) -> bool {+ // SAFETY: plain queries on a window handle (a stale one just fails).+ unsafe {+ if !IsWindowVisible(hwnd).as_bool() || IsIconic(hwnd).as_bool() {+ return false;+ }+ if GetWindow(hwnd, GW_OWNER).is_ok_and(|owner| !owner.is_invalid()) {+ return false;+ }+ let mut cloaked = 0u32;+ let _ = DwmGetWindowAttribute(+ hwnd,+ DWMWA_CLOAKED,+ (&mut cloaked as *mut u32).cast(),+ size_of::<u32>() as u32,+ );+ cloaked == 0+ }+}++fn title_of(hwnd: HWND) -> String {+ let mut buffer = [0u16; 512];+ // SAFETY: the buffer's length bounds what's written.+ let len = unsafe { GetWindowTextW(hwnd, &mut buffer) };+ String::from_utf16_lossy(&buffer[..len.max(0) as usize])+}++fn program_of(hwnd: HWND) -> Option<String> {+ let mut pid = 0u32;+ // SAFETY: plain process queries; the handle is closed below.+ unsafe {+ GetWindowThreadProcessId(hwnd, Some(&mut pid));+ let process = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid).ok()?;+ let mut buffer = [0u16; 1024];+ let mut len = buffer.len() as u32;+ let named = QueryFullProcessImageNameW(+ process,+ PROCESS_NAME_WIN32,+ windows::core::PWSTR(buffer.as_mut_ptr()),+ &mut len,+ );+ let _ = CloseHandle(process);+ named.ok()?;+ let path = String::from_utf16_lossy(&buffer[..len as usize]);+ Some(path.rsplit('\\').next().unwrap_or(&path).to_ascii_lowercase())+ }+}++/// The window that plays `title` for the media session `player`: one of+/// its program's windows whose title has the media's title in it (a+/// browser's shows its active tab's), or its only window.+pub fn window_for(player: &str, title: &str) -> Option<Window> {+ let programs = programs_for(player);+ let candidates: Vec<Window> = windows().into_iter().filter(|w| programs.contains(&w.program)).collect();+ pick(candidates, title)+}++fn pick(candidates: Vec<Window>, title: &str) -> Option<Window> {+ let wanted = title.trim().to_lowercase();+ if !wanted.is_empty()+ && let Some(found) = candidates.iter().find(|w| w.title.to_lowercase().contains(&wanted))+ {+ return Some(found.clone());+ }+ match candidates.as_slice() {+ [only] => Some(only.clone()),+ _ => None,+ }+}++/// Has this thread see the screen in physical pixels, as screen captures+/// do, whatever the process's DPI awareness: UI Automation's places then+/// match what a capture of the screen shows.+pub fn use_physical_pixels() {+ // SAFETY: changes this thread's DPI awareness only.+ let _ = unsafe { SetThreadDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2) };+}++/// UI Automation, on a thread that has COM.+pub struct Automation(IUIAutomation);++impl Automation {+ pub fn new() -> windows::core::Result<Automation> {+ // SAFETY: creates the system's UI Automation client.+ unsafe { CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) }.map(Automation)+ }++ fn find(+ &self,+ root: &IUIAutomationElement,+ property: UIA_PROPERTY_ID,+ value: VARIANT,+ ) -> Option<IUIAutomationElement> {+ // SAFETY: COM calls on live interfaces.+ unsafe {+ let condition = self.0.CreatePropertyCondition(property, &value).ok()?;+ root.FindFirst(TreeScope_Descendants, &condition).ok()+ }+ }++ fn root(&self, window: &Window) -> Option<IUIAutomationElement> {+ // SAFETY: a stale handle just fails.+ unsafe { self.0.ElementFromHandle(window.hwnd).ok() }+ }++ /// The address of the page the browser `window` shows, with its scheme+ /// (address bars leave "https://" out).+ pub fn page_url(&self, window: &Window) -> Option<String> {+ let root = self.root(window)?;+ let bar = self+ .find(&root, UIA_ClassNamePropertyId, VARIANT::from("OmniboxViewViews"))+ .or_else(|| self.find(&root, UIA_AutomationIdPropertyId, VARIANT::from("urlbar-input")))?;+ // SAFETY: a COM call on a live element.+ let value = unsafe { bar.GetCurrentPropertyValue(UIA_ValueValuePropertyId) }.ok()?;+ let text = BSTR::try_from(&value).ok()?.to_string();+ with_scheme(&text)+ }++ /// Where on screen the video plays in `window`, in physical pixels:+ /// YouTube's player when it's there, else the page, else the window.+ pub fn video_rect(&self, window: &Window) -> Option<RECT> {+ let root = self.root(window);+ let element = root.as_ref().and_then(|root| {+ self.find(root, UIA_ClassNamePropertyId, VARIANT::from("video-stream html5-main-video")).or_else(+ || self.find(root, UIA_ControlTypePropertyId, VARIANT::from(UIA_DocumentControlTypeId.0)),+ )+ });+ // SAFETY: a COM call on a live element.+ let rect = element.and_then(|e| unsafe { e.CurrentBoundingRectangle() }.ok());+ rect.filter(|r| r.right - r.left >= 64 && r.bottom - r.top >= 64).or_else(|| frame_of(window.hwnd))+ }+}++/// A window's frame on screen, without its invisible resize borders.+pub fn frame_of(hwnd: HWND) -> Option<RECT> {+ let mut rect = RECT::default();+ // SAFETY: writes one RECT.+ unsafe {+ DwmGetWindowAttribute(+ hwnd,+ DWMWA_EXTENDED_FRAME_BOUNDS,+ (&mut rect as *mut RECT).cast(),+ size_of::<RECT>() as u32,+ )+ }+ .ok()?;+ (rect.right > rect.left && rect.bottom > rect.top).then_some(rect)+}++/// An address bar's text as a web address: "youtube.com/watch?v=x" becomes+/// "https://youtube.com/watch?v=x". Anything that isn't a web page (a+/// search being typed, a browser's own pages) is `None`.+fn with_scheme(text: &str) -> Option<String> {+ let text = text.trim();+ if text.is_empty() || text.contains(char::is_whitespace) {+ return None;+ }+ let lower = text.to_ascii_lowercase();+ if lower.starts_with("https://") || lower.starts_with("http://") {+ return Some(text.to_owned());+ }+ if lower.contains("://") || lower.starts_with("about:") {+ return None;+ }+ let host = lower.split(['/', '?', '#']).next().unwrap_or("");+ let host = host.split(':').next().unwrap_or("");+ (host.contains('.') || host == "localhost").then(|| format!("https://{text}"))+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn players_map_to_their_programs() {+ assert_eq!(programs_for("Chrome"), ["chrome.exe"]);+ assert_eq!(programs_for("MSEdge"), ["msedge.exe"]);+ assert_eq!(programs_for("308046B0AF4A39CB"), ["firefox.exe"]);+ assert_eq!(programs_for("vlc.exe"), ["vlc.exe"]);+ assert_eq!(programs_for(r"C:\Program Files\VideoLAN\VLC\vlc.exe"), ["vlc.exe"]);+ assert_eq!(programs_for("VideoLAN.VLC"), ["vlc.exe"]);+ // A packaged app's ID: no program to find a window by.+ assert!(programs_for("SpotifyAB.SpotifyMusic_zpdnekdrzrea0!Spotify").is_empty());+ }++ #[test]+ fn address_bar_text_becomes_a_web_address() {+ assert_eq!(+ with_scheme("youtube.com/watch?v=abc").as_deref(),+ Some("https://youtube.com/watch?v=abc")+ );+ assert_eq!(+ with_scheme("https://www.netflix.com/watch/1").as_deref(),+ Some("https://www.netflix.com/watch/1")+ );+ assert_eq!(with_scheme("127.0.0.1:8765").as_deref(), Some("https://127.0.0.1:8765"));+ assert_eq!(with_scheme("cats on skateboards"), None, "a search being typed");+ assert_eq!(with_scheme("chrome://settings"), None);+ assert_eq!(with_scheme("about:blank"), None);+ assert_eq!(with_scheme("settings"), None);+ assert_eq!(with_scheme(""), None);+ }++ fn window(title: &str) -> Window {+ Window { hwnd: HWND::default(), program: "chrome.exe".into(), title: title.into() }+ }++ #[test]+ fn the_window_with_the_title_is_the_player() {+ let picked =+ pick(vec![window("Mail - Google Chrome"), window("A Song - YouTube - Google Chrome")], "a song");+ assert_eq!(picked.unwrap().title, "A Song - YouTube - Google Chrome");+ // The playing tab isn't in front in any window: can't tell which.+ assert!(pick(vec![window("Mail"), window("News")], "A Song").is_none());+ // One window: that's it, whatever its tab.+ assert_eq!(pick(vec![window("Mail")], "A Song").unwrap().title, "Mail");+ }++ /// With a Chromium browser showing a page whose title has+ /// `NECTARLINK_TEST_TAB` in it: reads its address and its video's place.+ #[test]+ #[ignore]+ fn reads_a_real_browser() {+ let wanted = std::env::var("NECTARLINK_TEST_TAB").expect("NECTARLINK_TEST_TAB");+ // SAFETY: COM for this test's thread.+ let _ = unsafe {+ windows::Win32::System::Com::CoInitializeEx(+ None,+ windows::Win32::System::Com::COINIT_MULTITHREADED,+ )+ };+ use_physical_pixels();+ let window = window_for("Chrome", &wanted).expect("the test tab's window");+ let automation = Automation::new().unwrap();+ let url = automation.page_url(&window);+ let rect = automation.video_rect(&window).unwrap();+ eprintln!("{:?} {url:?} {rect:?}", window.title);+ assert!(url.is_some_and(|u| u.starts_with("http")));+ assert!(rect.right - rect.left >= 64);+ }+}diff --git a/desktop/app/src/win/mod.rs b/desktop/app/src/win/mod.rsindex 0aab4ea..eb7ae48 100644--- a/desktop/app/src/win/mod.rs+++ b/desktop/app/src/win/mod.rs@@ -9,6 +9,7 @@ pub mod audio;pub mod audio_out;pub mod autostart;pub mod ble_beacon;+pub mod browser;pub mod bt_hfp;pub mod clipboard;pub mod doctor;@@ -33,6 +34,7 @@ pub mod tray;pub mod vcam;pub mod wallpaper;pub mod widget;+pub mod window_capture;pub mod wol;use windows::{diff --git a/desktop/app/src/win/window_capture.rs b/desktop/app/src/win/window_capture.rsnew file mode 100644index 0000000..c2ee891--- /dev/null+++ b/desktop/app/src/win/window_capture.rs@@ -0,0 +1,335 @@+// SPDX-License-Identifier: GPL-3.0-or-later+//! One window's own pixels, with Windows Graphics Capture: what the window+//! shows even with other windows in front of it, and never theirs. Play on+//! phone streams a player's window this way, cropped to its video.++use windows::{+ Graphics::{+ Capture::{+ Direct3D11CaptureFramePool, GraphicsCaptureAccess, GraphicsCaptureAccessKind,+ GraphicsCaptureItem, GraphicsCaptureSession,+ },+ DirectX::{Direct3D11::IDirect3DDevice, DirectXPixelFormat},+ SizeInt32,+ },+ Win32::{+ Foundation::{HWND, RECT},+ Graphics::{+ Direct3D::D3D_DRIVER_TYPE_HARDWARE,+ Direct3D11::{+ D3D11_BOX, D3D11_CPU_ACCESS_READ, D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_MAP_READ,+ D3D11_MAPPED_SUBRESOURCE, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, D3D11_USAGE_STAGING,+ D3D11CreateDevice, ID3D11Device, ID3D11DeviceContext, ID3D11Texture2D,+ },+ Dxgi::{Common::DXGI_FORMAT_B8G8R8A8_UNORM, Common::DXGI_SAMPLE_DESC, IDXGIDevice},+ },+ System::{+ Com::{COINIT_MULTITHREADED, CoInitializeEx},+ WinRT::{+ Direct3D11::{CreateDirect3D11DeviceFromDXGIDevice, IDirect3DDxgiInterfaceAccess},+ Graphics::Capture::IGraphicsCaptureItemInterop,+ },+ },+ },+ core::{Interface, factory},+};++/// Frames the pool holds: one being read, one arriving.+const BUFFERS: i32 = 2;++/// A running capture of one window.+pub struct WindowCapture {+ hwnd: HWND,+ device: ID3D11Device,+ context: ID3D11DeviceContext,+ winrt_device: IDirect3DDevice,+ pool: Direct3D11CaptureFramePool,+ session: GraphicsCaptureSession,+ size: SizeInt32,+ staging: Option<(ID3D11Texture2D, u32, u32)>,+ /// The latest crop, kept for when no new frame came (nothing changed).+ last: Vec<u8>,+ last_size: (u32, u32),+}++impl WindowCapture {+ /// Starts capturing `hwnd`, without the pointer and, where Windows+ /// allows it, without the yellow capture border.+ pub fn new(hwnd: HWND) -> windows::core::Result<WindowCapture> {+ // SAFETY: COM for this thread (WinRT needs it; a second call is harmless).+ let _ = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) };+ if !GraphicsCaptureSession::IsSupported()? {+ return Err(windows::core::Error::from(windows::Win32::Foundation::E_NOTIMPL));+ }+ let mut device = None;+ let mut context = None;+ // SAFETY: plain Direct3D and WinRT interop calls with owned values.+ unsafe {+ D3D11CreateDevice(+ None,+ D3D_DRIVER_TYPE_HARDWARE,+ Default::default(),+ D3D11_CREATE_DEVICE_BGRA_SUPPORT,+ None,+ D3D11_SDK_VERSION,+ Some(&mut device),+ None,+ Some(&mut context),+ )?;+ }+ let device: ID3D11Device =+ device.ok_or_else(|| windows::core::Error::from(windows::Win32::Foundation::E_FAIL))?;+ let context =+ context.ok_or_else(|| windows::core::Error::from(windows::Win32::Foundation::E_FAIL))?;+ // SAFETY: as above.+ let (winrt_device, item) = unsafe {+ let dxgi: IDXGIDevice = device.cast()?;+ let winrt_device: IDirect3DDevice = CreateDirect3D11DeviceFromDXGIDevice(&dxgi)?.cast()?;+ let interop = factory::<GraphicsCaptureItem, IGraphicsCaptureItemInterop>()?;+ let item: GraphicsCaptureItem = interop.CreateForWindow(hwnd)?;+ (winrt_device, item)+ };+ let size = item.Size()?;+ let pool = Direct3D11CaptureFramePool::CreateFreeThreaded(+ &winrt_device,+ DirectXPixelFormat::B8G8R8A8UIntNormalized,+ BUFFERS,+ size,+ )?;+ let session = pool.CreateCaptureSession(&item)?;+ let _ = session.SetIsCursorCaptureEnabled(false);+ // Windows 11 can leave the border off; earlier versions always draw it.+ if GraphicsCaptureAccess::RequestAccessAsync(GraphicsCaptureAccessKind::Borderless)+ .and_then(|asked| asked.join())+ .is_ok()+ {+ let _ = session.SetIsBorderRequired(false);+ }+ session.StartCapture()?;+ Ok(WindowCapture {+ hwnd,+ device,+ context,+ winrt_device,+ pool,+ session,+ size,+ staging: None,+ last: Vec::new(),+ last_size: (0, 0),+ })+ }++ /// The window this captures.+ pub fn window(&self) -> HWND {+ self.hwnd+ }++ /// The window's latest picture of `crop` (in the window's own pixels,+ /// from its top-left), scaled to `dst_w` × `dst_h` into `bgra`. Without+ /// a new frame (nothing changed on screen), the last one again. False+ /// before the first frame.+ pub fn capture(&mut self, crop: RECT, dst_w: u32, dst_h: u32, bgra: &mut Vec<u8>) -> bool {+ if let Err(e) = self.latest(crop) {+ tracing::debug!(error = %e, "window capture frame failed");+ }+ let (w, h) = self.last_size;+ if w == 0 || h == 0 {+ return false;+ }+ scale_bgra(&self.last, w, h, dst_w, dst_h, bgra);+ true+ }++ /// Reads the newest frame, if one came, into `last`.+ fn latest(&mut self, crop: RECT) -> windows::core::Result<()> {+ // Only the newest matters: older frames are dropped.+ let mut newest = None;+ while let Ok(frame) = self.pool.TryGetNextFrame() {+ newest = Some(frame);+ }+ let Some(frame) = newest else { return Ok(()) };+ let content = frame.ContentSize()?;+ if content.Width != self.size.Width || content.Height != self.size.Height {+ // The window changed size: frames come at the new size from the next one.+ self.size = content;+ self.pool.Recreate(+ &self.winrt_device,+ DirectXPixelFormat::B8G8R8A8UIntNormalized,+ BUFFERS,+ content,+ )?;+ }+ // SAFETY: COM calls on live interfaces; the staging texture is+ // mapped for reading and unmapped before it's used again.+ unsafe {+ let access: IDirect3DDxgiInterfaceAccess = frame.Surface()?.cast()?;+ let texture: ID3D11Texture2D = access.GetInterface()?;+ let mut desc = D3D11_TEXTURE2D_DESC::default();+ texture.GetDesc(&mut desc);+ let (tw, th) =+ (desc.Width.min(content.Width.max(0) as u32), desc.Height.min(content.Height.max(0) as u32));+ let left = crop.left.clamp(0, tw as i32) as u32;+ let top = crop.top.clamp(0, th as i32) as u32;+ let right = crop.right.clamp(0, tw as i32) as u32;+ let bottom = crop.bottom.clamp(0, th as i32) as u32;+ if right <= left || bottom <= top {+ return Ok(());+ }+ let (w, h) = (right - left, bottom - top);+ let staging = self.staging(w, h)?;+ let area = D3D11_BOX { left, top, front: 0, right, bottom, back: 1 };+ self.context.CopySubresourceRegion(&staging, 0, 0, 0, 0, &texture, 0, Some(&area));+ let mut mapped = D3D11_MAPPED_SUBRESOURCE::default();+ self.context.Map(&staging, 0, D3D11_MAP_READ, 0, Some(&mut mapped))?;+ let row = w as usize * 4;+ self.last.resize(row * h as usize, 0);+ let source =+ std::slice::from_raw_parts(mapped.pData.cast::<u8>(), mapped.RowPitch as usize * h as usize);+ for (y, line) in self.last.chunks_exact_mut(row).enumerate() {+ line.copy_from_slice(&source[y * mapped.RowPitch as usize..][..row]);+ }+ self.context.Unmap(&staging, 0);+ self.last_size = (w, h);+ }+ Ok(())+ }++ /// A texture the CPU can read, `w` × `h`, reused while the size stays.+ fn staging(&mut self, w: u32, h: u32) -> windows::core::Result<ID3D11Texture2D> {+ if let Some((texture, sw, sh)) = &self.staging+ && (*sw, *sh) == (w, h)+ {+ return Ok(texture.clone());+ }+ let desc = D3D11_TEXTURE2D_DESC {+ Width: w,+ Height: h,+ MipLevels: 1,+ ArraySize: 1,+ Format: DXGI_FORMAT_B8G8R8A8_UNORM,+ SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 },+ Usage: D3D11_USAGE_STAGING,+ BindFlags: 0,+ CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32,+ MiscFlags: 0,+ };+ let mut texture = None;+ // SAFETY: creates a texture from a filled-in description.+ unsafe { self.device.CreateTexture2D(&desc, None, Some(&mut texture))? };+ let texture =+ texture.ok_or_else(|| windows::core::Error::from(windows::Win32::Foundation::E_FAIL))?;+ self.staging = Some((texture.clone(), w, h));+ Ok(texture)+ }+}++impl Drop for WindowCapture {+ fn drop(&mut self) {+ let _ = self.session.Close();+ let _ = self.pool.Close();+ }+}++/// Scales a `w` × `h` BGRA picture to `dst_w` × `dst_h` (bilinear), or+/// copies it when the size is the same.+pub(crate) fn scale_bgra(src: &[u8], w: u32, h: u32, dst_w: u32, dst_h: u32, out: &mut Vec<u8>) {+ let (w, h, dw, dh) = (w as usize, h as usize, dst_w as usize, dst_h as usize);+ out.resize(dw * dh * 4, 0);+ if (w, h) == (dw, dh) {+ out.copy_from_slice(&src[..w * h * 4]);+ return;+ }+ // Where each output column samples from: left pixel and weight (of 256).+ let columns: Vec<(usize, u32)> = (0..dw)+ .map(|x| {+ let at = ((x as f32 + 0.5) * w as f32 / dw as f32 - 0.5).clamp(0.0, (w - 1) as f32);+ (at as usize, ((at - at.floor()) * 256.0) as u32)+ })+ .collect();+ for (y, line) in out.chunks_exact_mut(dw * 4).enumerate() {+ let at = ((y as f32 + 0.5) * h as f32 / dh as f32 - 0.5).clamp(0.0, (h - 1) as f32);+ let (y0, wy) = (at as usize, ((at - at.floor()) * 256.0) as u32);+ let y1 = (y0 + 1).min(h - 1);+ let (top, bottom) = (&src[y0 * w * 4..][..w * 4], &src[y1 * w * 4..][..w * 4]);+ for (x, pixel) in line.as_chunks_mut::<4>().0.iter_mut().enumerate() {+ let (x0, wx) = columns[x];+ let x1 = (x0 + 1).min(w - 1);+ for c in 0..4 {+ let row = |r: &[u8]| u32::from(r[x0 * 4 + c]) * (256 - wx) + u32::from(r[x1 * 4 + c]) * wx;+ let value = (row(top) * (256 - wy) + row(bottom) * wy + (1 << 15)) >> 16;+ pixel[c] = value as u8;+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn same_size_is_a_copy() {+ let src: Vec<u8> = (0..4 * 3 * 4).map(|i| i as u8).collect();+ let mut out = Vec::new();+ scale_bgra(&src, 4, 3, 4, 3, &mut out);+ assert_eq!(out, src);+ }++ #[test]+ fn halving_averages_and_a_flat_picture_stays_flat() {+ // 4x2: left half black, right half white.+ let mut src = Vec::new();+ for _ in 0..2 {+ for x in 0..4 {+ let v = if x < 2 { 0 } else { 255 };+ src.extend_from_slice(&[v, v, v, 255]);+ }+ }+ let mut out = Vec::new();+ scale_bgra(&src, 4, 2, 2, 1, &mut out);+ assert_eq!(out.len(), 2 * 4);+ assert!(out[0] < 40 && out[4] > 215, "{out:?}");+ assert_eq!(out[3], 255);++ let flat = vec![77u8; 10 * 6 * 4];+ scale_bgra(&flat, 10, 6, 7, 3, &mut out);+ assert!(out.iter().all(|&v| v == 77));+ }++ /// With a Chromium browser showing a page whose title has+ /// `NECTARLINK_TEST_TAB` in it: captures its window's video, covered+ /// or not, and writes it to `NECTARLINK_TEST_DUMP` (raw BGRA) if set.+ #[test]+ #[ignore]+ fn captures_a_window_behind_others() {+ use crate::win::browser::{Automation, frame_of, use_physical_pixels, window_for};+ let wanted = std::env::var("NECTARLINK_TEST_TAB").expect("NECTARLINK_TEST_TAB");+ // SAFETY: COM for this test's thread.+ let _ = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) };+ use_physical_pixels();+ let window = window_for("Chrome", &wanted).expect("the test tab's window");+ let video = Automation::new().unwrap().video_rect(&window).expect("the video");+ let frame = frame_of(window.hwnd).unwrap();+ let crop = RECT {+ left: video.left - frame.left,+ top: video.top - frame.top,+ right: video.right - frame.left,+ bottom: video.bottom - frame.top,+ };+ let (w, h) = ((crop.right - crop.left) as u32, (crop.bottom - crop.top) as u32);+ let mut capture = WindowCapture::new(window.hwnd).unwrap();+ let mut bgra = Vec::new();+ let started = std::time::Instant::now();+ while !capture.capture(crop, w, h, &mut bgra) {+ assert!(started.elapsed() < std::time::Duration::from_secs(3), "no frame");+ std::thread::sleep(std::time::Duration::from_millis(30));+ }+ let distinct: std::collections::HashSet<&[u8; 4]> = bgra.as_chunks::<4>().0.iter().collect();+ eprintln!("{crop:?} {w}x{h}: {} colors", distinct.len());+ if let Ok(path) = std::env::var("NECTARLINK_TEST_DUMP") {+ std::fs::write(path, &bgra).unwrap();+ }+ assert!(distinct.len() > 50);+ }+}