Skip to content

Your Android phone and your Windows PC, working as one. Local-first, end-to-end encrypted, no account.

17.5 KB
Raw Blame
// SPDX-License-Identifier: GPL-3.0-or-later
//! The window a player shows in, for Play on phone: which top-level window
//! belongs to a media session, the address of the page a browser plays in,
//! and where on screen the video is, read through UI Automation.
//!
//! Chromium browsers (Chrome, Edge, Brave, Opera, Vivaldi) expose their
//! address bar as an edit field of class `OmniboxViewViews` and a page's
//! elements with their HTML classes, so YouTube's player
//! (`video-stream html5-main-video`) can be found exactly; Firefox's
//! address bar has the automation ID `urlbar-input`, and Firefox-based
//! browsers (Zen, Floorp...) show only their tab's area, not the page. Nothing here reads a
//! page's text: only the address of the tab that plays.

use windows::{
    Win32::{
        Foundation::{CloseHandle, HWND, LPARAM, RECT},
        Graphics::Dwm::{DWMWA_CLOAKED, DWMWA_EXTENDED_FRAME_BOUNDS, DwmGetWindowAttribute},
        System::{
            Com::{CLSCTX_INPROC_SERVER, CoCreateInstance},
            Threading::{
                OpenProcess, PROCESS_NAME_WIN32, PROCESS_QUERY_LIMITED_INFORMATION,
                QueryFullProcessImageNameW,
            },
            Variant::VARIANT,
        },
        UI::{
            Accessibility::{
                CUIAutomation, IUIAutomation, IUIAutomationElement, TreeScope_Descendants,
                UIA_AutomationIdPropertyId, UIA_ClassNamePropertyId, UIA_ControlTypePropertyId,
                UIA_DocumentControlTypeId, UIA_PROPERTY_ID, UIA_ValueValuePropertyId,
            },
            HiDpi::{DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2, SetThreadDpiAwarenessContext},
            WindowsAndMessaging::{
                EnumWindows, GW_OWNER, GetWindow, GetWindowTextW, GetWindowThreadProcessId, HWND_BOTTOM,
                IsIconic, IsWindowVisible, SW_SHOWNOACTIVATE, SWP_NOACTIVATE, SWP_NOMOVE, SWP_NOSIZE,
                SetWindowPos, ShowWindow,
            },
        },
    },
    core::{BOOL, BSTR},
};

/// The programs a media session's app ID stands for. Browsers report a
/// short name (or, for Firefox, a hash of its install path); desktop apps
/// often report their executable.
pub fn programs_for(player: &str) -> Vec<String> {
    let lower = player.to_ascii_lowercase();
    let known: &[(&str, &str)] = &[
        ("chrome", "chrome.exe"),
        ("msedge", "msedge.exe"),
        ("edge", "msedge.exe"),
        ("brave", "brave.exe"),
        ("opera", "opera.exe"),
        ("vivaldi", "vivaldi.exe"),
        ("firefox", "firefox.exe"),
        ("zen", "zen.exe"),
        ("floorp", "floorp.exe"),
        ("librewolf", "librewolf.exe"),
        ("waterfox", "waterfox.exe"),
        // Firefox's app ID is a hash of where it's installed; this is the
        // one for its default location. Other installs (and Firefox-based
        // browsers) have hashes of their own: their window is found by its
        // title (see `window_for`).
        ("308046b0af4a39cb", "firefox.exe"),
    ];
    let mut programs: Vec<String> =
        known.iter().filter(|(id, _)| lower.contains(id)).map(|(_, exe)| (*exe).to_owned()).collect();
    // "vlc.exe", or a path, or "VideoLAN.VLC" (a desktop app ID).
    let last = lower.rsplit(['\\', '/']).next().unwrap_or(&lower);
    if last.ends_with(".exe") {
        programs.push(last.to_owned());
    } else if let Some((_, name)) = last.rsplit_once('.')
        && !name.is_empty()
        && !last.contains('!')
    {
        programs.push(format!("{name}.exe"));
    }
    programs.dedup();
    programs
}

/// Whether `program` (an executable's file name) is a browser whose
/// address bar and page this module can read.
pub fn is_browser(program: &str) -> bool {
    matches!(
        program,
        "chrome.exe"
            | "msedge.exe"
            | "brave.exe"
            | "opera.exe"
            | "vivaldi.exe"
            | "firefox.exe"
            | "zen.exe"
            | "floorp.exe"
            | "librewolf.exe"
            | "waterfox.exe"
    )
}

/// A visible top-level window.
#[derive(Debug, Clone)]
pub struct Window {
    pub hwnd: HWND,
    /// Its program's file name, lowercase ("chrome.exe").
    pub program: String,
    pub title: String,
}

/// The visible, uncloaked top-level windows that aren't minimized, front
/// to back.
pub fn windows() -> Vec<Window> {
    listed(false)
}

/// As [`windows`], with minimized ones too: a minimized player still plays.
fn windows_and_minimized() -> Vec<Window> {
    listed(true)
}

fn listed(minimized: bool) -> Vec<Window> {
    unsafe extern "system" fn each(hwnd: HWND, found: LPARAM) -> BOOL {
        // SAFETY: `found` is the Vec passed to EnumWindows below, alive for the call.
        let found = unsafe { &mut *(found.0 as *mut Vec<HWND>) };
        found.push(hwnd);
        BOOL(1)
    }
    let mut handles: Vec<HWND> = Vec::new();
    // SAFETY: the callback only pushes to `handles`, which outlives the call.
    let _ = unsafe { EnumWindows(Some(each), LPARAM(&mut handles as *mut Vec<HWND> as isize)) };
    handles
        .into_iter()
        .filter(|&hwnd| shown(hwnd, minimized))
        .filter_map(|hwnd| Some(Window { hwnd, program: program_of(hwnd)?, title: title_of(hwnd) }))
        .collect()
}

fn shown(hwnd: HWND, minimized: bool) -> bool {
    // SAFETY: plain queries on a window handle (a stale one just fails).
    unsafe {
        if !IsWindowVisible(hwnd).as_bool() || (!minimized && IsIconic(hwnd).as_bool()) {
            return false;
        }
        if GetWindow(hwnd, GW_OWNER).is_ok_and(|owner| !owner.is_invalid()) {
            return false;
        }
        let mut cloaked = 0u32;
        let _ = DwmGetWindowAttribute(
            hwnd,
            DWMWA_CLOAKED,
            (&mut cloaked as *mut u32).cast(),
            size_of::<u32>() as u32,
        );
        cloaked == 0
    }
}

fn title_of(hwnd: HWND) -> String {
    let mut buffer = [0u16; 512];
    // SAFETY: the buffer's length bounds what's written.
    let len = unsafe { GetWindowTextW(hwnd, &mut buffer) };
    String::from_utf16_lossy(&buffer[..len.max(0) as usize])
}

fn program_of(hwnd: HWND) -> Option<String> {
    let mut pid = 0u32;
    // SAFETY: plain process queries; the handle is closed below.
    unsafe {
        GetWindowThreadProcessId(hwnd, Some(&mut pid));
        let process = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid).ok()?;
        let mut buffer = [0u16; 1024];
        let mut len = buffer.len() as u32;
        let named = QueryFullProcessImageNameW(
            process,
            PROCESS_NAME_WIN32,
            windows::core::PWSTR(buffer.as_mut_ptr()),
            &mut len,
        );
        let _ = CloseHandle(process);
        named.ok()?;
        let path = String::from_utf16_lossy(&buffer[..len as usize]);
        Some(path.rsplit('\\').next().unwrap_or(&path).to_ascii_lowercase())
    }
}

/// The window that plays `title` for the media session `player`: one of
/// its program's windows whose title has the media's title in it (a
/// browser's shows its active tab's), or its only window.
pub fn window_for(player: &str, title: &str) -> Option<Window> {
    let programs = programs_for(player);
    let all = windows_and_minimized();
    let candidates: Vec<Window> = all.iter().filter(|w| programs.contains(&w.program)).cloned().collect();
    pick(candidates, title).or_else(|| by_title(all, title))
}

/// The one window (of any program but this app) whose title has the
/// media's title in it, for players whose app ID names no program (a hash,
/// as Firefox-based browsers report).
fn by_title(windows: Vec<Window>, title: &str) -> Option<Window> {
    let wanted = title.trim().to_lowercase();
    if wanted.chars().count() < 3 {
        return None;
    }
    let mut matching = windows
        .into_iter()
        .filter(|w| w.program != "nectarlink-desktop.exe" && w.title.to_lowercase().contains(&wanted));
    let found = matching.next()?;
    // Two windows with that title: can't tell which one plays.
    matching.next().is_none().then_some(found)
}

fn pick(candidates: Vec<Window>, title: &str) -> Option<Window> {
    let wanted = title.trim().to_lowercase();
    if !wanted.is_empty()
        && let Some(found) = candidates.iter().find(|w| w.title.to_lowercase().contains(&wanted))
    {
        return Some(found.clone());
    }
    match candidates.as_slice() {
        [only] => Some(only.clone()),
        _ => None,
    }
}

/// Has this thread see the screen in physical pixels, as screen captures
/// do, whatever the process's DPI awareness: UI Automation's places then
/// match what a capture of the screen shows.
pub fn use_physical_pixels() {
    // SAFETY: changes this thread's DPI awareness only.
    let _ = unsafe { SetThreadDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2) };
}

/// UI Automation, on a thread that has COM.
pub struct Automation(IUIAutomation);

impl Automation {
    pub fn new() -> windows::core::Result<Automation> {
        // SAFETY: creates the system's UI Automation client.
        unsafe { CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) }.map(Automation)
    }

    fn find(
        &self,
        root: &IUIAutomationElement,
        property: UIA_PROPERTY_ID,
        value: VARIANT,
    ) -> Option<IUIAutomationElement> {
        // SAFETY: COM calls on live interfaces.
        unsafe {
            let condition = self.0.CreatePropertyCondition(property, &value).ok()?;
            root.FindFirst(TreeScope_Descendants, &condition).ok()
        }
    }

    fn root(&self, window: &Window) -> Option<IUIAutomationElement> {
        // SAFETY: a stale handle just fails.
        unsafe { self.0.ElementFromHandle(window.hwnd).ok() }
    }

    /// The address of the page the browser `window` shows, with its scheme
    /// (address bars leave "https://" out).
    pub fn page_url(&self, window: &Window) -> Option<String> {
        let root = self.root(window)?;
        let bar = self
            .find(&root, UIA_ClassNamePropertyId, VARIANT::from("OmniboxViewViews"))
            .or_else(|| self.find(&root, UIA_AutomationIdPropertyId, VARIANT::from("urlbar-input")))?;
        // SAFETY: a COM call on a live element.
        let value = unsafe { bar.GetCurrentPropertyValue(UIA_ValueValuePropertyId) }.ok()?;
        let text = BSTR::try_from(&value).ok()?.to_string();
        with_scheme(&text)
    }

    /// Where on screen the video plays in `window`, in physical pixels:
    /// YouTube's player when it's there, else the page, else the window.
    #[cfg(test)]
    pub fn video_rect(&self, window: &Window) -> Option<RECT> {
        self.player_rect(window).or_else(|| self.page_rect(window)).or_else(|| frame_of(window.hwnd))
    }

    /// Where YouTube's player is in `window`, once the browser exposes its
    /// page (a moment after it's first asked).
    pub fn player_rect(&self, window: &Window) -> Option<RECT> {
        let root = self.root(window)?;
        self.rect_of(self.find(
            &root,
            UIA_ClassNamePropertyId,
            VARIANT::from("video-stream html5-main-video"),
        ))
    }

    /// Where the page is in `window`: Chromium's page, or the selected
    /// tab's area in Firefox-based browsers (whose pages UI Automation
    /// doesn't show, so their video can't be found).
    pub fn page_rect(&self, window: &Window) -> Option<RECT> {
        let root = self.root(window)?;
        self.rect_of(self.find(&root, UIA_ControlTypePropertyId, VARIANT::from(UIA_DocumentControlTypeId.0)))
            .or_else(|| {
                self.rect_of(self.find(
                    &root,
                    UIA_ClassNamePropertyId,
                    VARIANT::from("browserSidebarContainer deck-selected"),
                ))
            })
    }

    fn rect_of(&self, element: Option<IUIAutomationElement>) -> Option<RECT> {
        // SAFETY: a COM call on a live element.
        let rect = element.and_then(|e| unsafe { e.CurrentBoundingRectangle() }.ok());
        rect.filter(|r| r.right - r.left >= 64 && r.bottom - r.top >= 64)
    }
}

/// A window's frame on screen, without its invisible resize borders.
/// Whether `hwnd` is minimized: Windows draws nothing of it then, so a
/// capture of it stays on its last picture.
pub fn is_minimized(hwnd: HWND) -> bool {
    // SAFETY: a plain query on a window handle.
    unsafe { IsIconic(hwnd).as_bool() }
}

/// Restores a minimized window behind every other one, without bringing it
/// forward or taking the focus: it draws again, out of the way.
pub fn restore_behind(hwnd: HWND) {
    // SAFETY: plain window calls on a handle (a stale one just fails).
    unsafe {
        let _ = ShowWindow(hwnd, SW_SHOWNOACTIVATE);
        let _ = SetWindowPos(hwnd, Some(HWND_BOTTOM), 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE | SWP_NOACTIVATE);
    }
}

pub fn frame_of(hwnd: HWND) -> Option<RECT> {
    let mut rect = RECT::default();
    // SAFETY: writes one RECT.
    unsafe {
        DwmGetWindowAttribute(
            hwnd,
            DWMWA_EXTENDED_FRAME_BOUNDS,
            (&mut rect as *mut RECT).cast(),
            size_of::<RECT>() as u32,
        )
    }
    .ok()?;
    (rect.right > rect.left && rect.bottom > rect.top).then_some(rect)
}

/// An address bar's text as a web address: "youtube.com/watch?v=x" becomes
/// "https://youtube.com/watch?v=x". Anything that isn't a web page (a
/// search being typed, a browser's own pages) is `None`.
fn with_scheme(text: &str) -> Option<String> {
    let text = text.trim();
    if text.is_empty() || text.contains(char::is_whitespace) {
        return None;
    }
    let lower = text.to_ascii_lowercase();
    if lower.starts_with("https://") || lower.starts_with("http://") {
        return Some(text.to_owned());
    }
    if lower.contains("://") || lower.starts_with("about:") {
        return None;
    }
    let host = lower.split(['/', '?', '#']).next().unwrap_or("");
    let host = host.split(':').next().unwrap_or("");
    (host.contains('.') || host == "localhost").then(|| format!("https://{text}"))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn players_map_to_their_programs() {
        assert_eq!(programs_for("Chrome"), ["chrome.exe"]);
        assert_eq!(programs_for("MSEdge"), ["msedge.exe"]);
        assert_eq!(programs_for("308046B0AF4A39CB"), ["firefox.exe"]);
        assert_eq!(programs_for("vlc.exe"), ["vlc.exe"]);
        assert_eq!(programs_for(r"C:\Program Files\VideoLAN\VLC\vlc.exe"), ["vlc.exe"]);
        assert_eq!(programs_for("VideoLAN.VLC"), ["vlc.exe"]);
        // A packaged app's ID: no program to find a window by.
        assert!(programs_for("SpotifyAB.SpotifyMusic_zpdnekdrzrea0!Spotify").is_empty());
    }

    #[test]
    fn address_bar_text_becomes_a_web_address() {
        assert_eq!(
            with_scheme("youtube.com/watch?v=abc").as_deref(),
            Some("https://youtube.com/watch?v=abc")
        );
        assert_eq!(
            with_scheme("https://www.netflix.com/watch/1").as_deref(),
            Some("https://www.netflix.com/watch/1")
        );
        assert_eq!(with_scheme("127.0.0.1:8765").as_deref(), Some("https://127.0.0.1:8765"));
        assert_eq!(with_scheme("cats on skateboards"), None, "a search being typed");
        assert_eq!(with_scheme("chrome://settings"), None);
        assert_eq!(with_scheme("about:blank"), None);
        assert_eq!(with_scheme("settings"), None);
        assert_eq!(with_scheme(""), None);
    }

    fn window(title: &str) -> Window {
        Window { hwnd: HWND::default(), program: "chrome.exe".into(), title: title.into() }
    }

    #[test]
    fn a_hashed_app_id_finds_its_window_by_title() {
        let zen = Window {
            hwnd: HWND::default(),
            program: "zen.exe".into(),
            title: "A Trailer - YouTube - Zen Browser".into(),
        };
        let mail = window("Mail - Google Chrome");
        assert_eq!(by_title(vec![mail.clone(), zen.clone()], "A Trailer").unwrap().program, "zen.exe");
        // Ambiguous or too short: none.
        assert!(by_title(vec![zen.clone(), zen.clone()], "A Trailer").is_none());
        assert!(by_title(vec![zen], "A").is_none());
        assert!(programs_for("F0DC299D809B9700").is_empty());
        assert!(is_browser("zen.exe"));
    }

    #[test]
    fn the_window_with_the_title_is_the_player() {
        let picked =
            pick(vec![window("Mail - Google Chrome"), window("A Song - YouTube - Google Chrome")], "a song");
        assert_eq!(picked.unwrap().title, "A Song - YouTube - Google Chrome");
        // The playing tab isn't in front in any window: can't tell which.
        assert!(pick(vec![window("Mail"), window("News")], "A Song").is_none());
        // One window: that's it, whatever its tab.
        assert_eq!(pick(vec![window("Mail")], "A Song").unwrap().title, "Mail");
    }

    /// With a Chromium browser showing a page whose title has
    /// `NECTARLINK_TEST_TAB` in it: reads its address and its video's place.
    #[test]
    #[ignore = "needs a browser tab titled NECTARLINK_TEST_TAB"]
    fn reads_a_real_browser() {
        let wanted = std::env::var("NECTARLINK_TEST_TAB").expect("NECTARLINK_TEST_TAB");
        // SAFETY: COM for this test's thread.
        let _ = unsafe {
            windows::Win32::System::Com::CoInitializeEx(
                None,
                windows::Win32::System::Com::COINIT_MULTITHREADED,
            )
        };
        use_physical_pixels();
        let window = window_for("Chrome", &wanted).expect("the test tab's window");
        let automation = Automation::new().unwrap();
        let url = automation.page_url(&window);
        let rect = automation.video_rect(&window).unwrap();
        eprintln!("{:?} {url:?} {rect:?}", window.title);
        assert!(url.is_some_and(|u| u.starts_with("http")));
        assert!(rect.right - rect.left >= 64);
    }
}