Your Android phone and your Windows PC, working as one. Local-first, end-to-end encrypted, no account.
Play on phone: the protocol and the core's part
Beefbffb28ab0b7
diff --git a/android/app/src/main/kotlin/app/nectarlink/android/core/Core.kt b/android/app/src/main/kotlin/app/nectarlink/android/core/Core.ktindex 486f0e8..4ad1f7c 100644--- a/android/app/src/main/kotlin/app/nectarlink/android/core/Core.kt+++ b/android/app/src/main/kotlin/app/nectarlink/android/core/Core.kt@@ -1891,14 +1891,24 @@ class Core(context: Context, private val scope: CoroutineScope) : EventListenerremoteOps.trySend { runCatching { it.remoteLaser(id, on, x, y) } }}- /** Asks a paired PC to start streaming its desktop screen to this phone (`mirror.start`, session 0). */- fun startPcMirror(id: String, maxSize: Int = 1280, fps: Int = 30, bitrate: Int = 4_000_000) {+ /**+ * Asks a paired PC to start streaming its desktop screen to this phone (`mirror.start`,+ * session 0), or with [watch], what that PC player plays, with its sound (Play on phone).+ */+ fun startPcMirror(+ id: String,+ maxSize: Int = 1280,+ fps: Int = 30,+ bitrate: Int = 4_000_000,+ watch: String? = null,+ ) {command {it.startPcMirror(id,maxSize.coerceIn(320, 1920).toUInt(),fps.coerceIn(10, 60).toUInt(),bitrate.coerceIn(500_000, 20_000_000).toUInt(),+ watch,)}}diff --git a/core/nectarlink-cli/src/main.rs b/core/nectarlink-cli/src/main.rsindex 1726af6..a554ff6 100644--- a/core/nectarlink-cli/src/main.rs+++ b/core/nectarlink-cli/src/main.rs@@ -3122,6 +3122,7 @@ async fn run(cli: &Cli, node: &Node) -> Result<()> {app: app.clone(),stay_awake: false,screen_off: false,+ watch: None,};node.mirror_start(id, options).await.context("the phone didn't ask its user")?;println!(diff --git a/core/nectarlink-core/src/config.rs b/core/nectarlink-core/src/config.rsindex 1df9c8f..b8a7ce9 100644--- a/core/nectarlink-core/src/config.rs+++ b/core/nectarlink-core/src/config.rs@@ -320,6 +320,18 @@ pub trait Platform: Send + Sync + 'static {Err("this device has no contacts".into())}+ /// Play on phone: a paired PC offers to stream what `player` plays+ /// (`title`). The phone shows the offer; taking it starts mirroring with+ /// `watch` set to `player`.+ fn watch_offered(+ &self,+ _from: &nectarlink_protocol::DeviceId,+ _player: &str,+ _title: Option<&str>,+ ) -> Result<(), String> {+ Err("this device doesn't show a PC's playback".into())+ }+/// Run a command on one of this device's media players (a paired device/// asked); `position` is set for [`MediaAction::Seek`].fn media_command(diff --git a/core/nectarlink-core/src/lib.rs b/core/nectarlink-core/src/lib.rsindex 52c767e..4f5a8ed 100644--- a/core/nectarlink-core/src/lib.rs+++ b/core/nectarlink-core/src/lib.rs@@ -29,6 +29,7 @@ pub mod otp;mod pairing;pub mod palette;mod photos;+pub mod play_on_phone;pub mod remote;mod session;mod sms;@@ -101,6 +102,7 @@ pub use nectarlink_protocol::{},format_mac, ipv4_broadcast, is_package_name, is_valid_storage_dir_path, is_valid_storage_id,is_valid_storage_name, is_valid_storage_path, magic_packet,+ media_watch::{STREAM as MEDIA_STREAM, WATCH as MEDIA_WATCH},mirror::{AUDIO as MIRROR_AUDIO, AUDIO_PLAYBACK as MIRROR_AUDIO_PLAYBACK, AV1 as MIRROR_AV1,CAPTURE as MIRROR_CAPTURE, H264 as MIRROR_H264, H265 as MIRROR_H265, INPUT as MIRROR_INPUT,@@ -140,6 +142,7 @@ pub use palette::{achromatic_scheme, downsample_rgb, downsample_rgba, extract_artwork_scheme_rgb,extract_artwork_scheme_rgba, scheme_from_seed_rgb, scheme_json, seed_from_grid, seed_from_pixels,};+pub use play_on_phone::PlayOnPhone;pub use remote::{ButtonAction, INPUT_INJECT, KeyMod, MouseButton, RemoteInput, SlideAction, remote_keys};pub use storage::{FolderStorage, StorageError, StorageReadFile, TOGGLE as STORAGE_TOGGLE};pub use store::{diff --git a/core/nectarlink-core/src/media.rs b/core/nectarlink-core/src/media.rsindex 5a1f295..a8311f8 100644--- a/core/nectarlink-core/src/media.rs+++ b/core/nectarlink-core/src/media.rs@@ -14,10 +14,16 @@ use std::{use nectarlink_protocol::{DeviceId, Envelope, ErrorCode,- messages::{MediaCommand, MediaPlayer, MediaState, media_actions, media_limits::PLAYERS, types},+ messages::{+ MediaCommand, MediaPlayer, MediaState, MediaWatch, media_actions, media_limits::PLAYERS, media_watch,+ types,+ },};-use crate::{Result, events::NodeEvent, node::Shared, session::Session};+use crate::{Error, Result, events::NodeEvent, node::Shared, session::Session};++/// How long a Play on phone offer stays open for the phone to take.+const WATCH_OFFER: std::time::Duration = std::time::Duration::from_secs(10 * 60);/// The device toggle that allows media for a device.pub(crate) const TOGGLE: &str = "media";@@ -209,6 +215,10 @@ pub(crate) async fn handle(shared: &Arc<Shared>, session: &Arc<Session>, env: &E}}types::MEDIA_SYNC => shared.send_media_state(session).await,+ types::MEDIA_WATCH => {+ let reply = watch_offered(shared, &peer, env.body()?).await;+ session.send(reply.reply_to(env.id)).await?;+ }types::MEDIA_COMMAND => {let command: MediaCommand = env.body()?;let reply = run_command(shared, &peer, command).await;@@ -219,6 +229,101 @@ pub(crate) async fn handle(shared: &Arc<Shared>, session: &Arc<Session>, env: &EOk(true)}+/// Play on phone, on the PC: offers `peer` what `player` plays. The offer+/// is what lets that phone start mirroring with `watch` (see+/// [`watch_allowed`]).+pub(crate) async fn offer_watch(+ shared: &Arc<Shared>,+ peer: DeviceId,+ player: String,+ title: Option<String>,+) -> Result<()> {+ let watch = MediaWatch { player, title };+ if !watch.is_valid() {+ return Err(Error::Internal("not a player to offer".into()));+ }+ if !shared.local_capabilities().iter().any(|c| c == media_watch::STREAM) {+ return Err(Error::Unsupported);+ }+ if !shared.store.get_peer(&peer)?.is_some_and(|p| p.caps.contains(media_watch::WATCH)) {+ return Err(Error::Unsupported);+ }+ if !shared.toggle_on(&peer, TOGGLE) {+ return Err(Error::Denied);+ }+ let session = shared.session(&peer).ok_or(Error::Offline)?;+ let player = watch.player.clone();+ shared+ .watch_offers+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .insert(peer, (player, std::time::Instant::now()));+ let env = Envelope::new(types::MEDIA_WATCH, &watch)?;+ let answer =+ session.request(env, crate::session::REQUEST_TIMEOUT).await.and_then(|r| Ok(r.expect(types::OK)?));+ if answer.is_err() {+ shared.watch_offers.lock().unwrap_or_else(|e| e.into_inner()).remove(&peer);+ }+ answer.map(|_| ())+}++/// Whether `peer` may watch `player`: this PC offered it, and the phone+/// takes it (or comes back to it) in time. Taking it renews the offer.+pub(crate) fn watch_allowed(shared: &Shared, peer: &DeviceId, player: &str) -> bool {+ let mut offers = shared.watch_offers.lock().unwrap_or_else(|e| e.into_inner());+ match offers.get_mut(peer) {+ Some((offered, at)) if offered == player && at.elapsed() < WATCH_OFFER => {+ *at = std::time::Instant::now();+ true+ }+ _ => false,+ }+}++/// Whether `peer` has a Play on phone offer from this PC open (so this PC+/// may open the stream to it).+pub(crate) fn watch_open(shared: &Shared, peer: &DeviceId) -> bool {+ shared+ .watch_offers+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .get(peer)+ .is_some_and(|(_, at)| at.elapsed() < WATCH_OFFER)+}++/// Watching stopped: the offer stays open for a while from now.+pub(crate) fn renew_watch(shared: &Shared, peer: &DeviceId) {+ if let Some((_, at)) = shared.watch_offers.lock().unwrap_or_else(|e| e.into_inner()).get_mut(peer) {+ *at = std::time::Instant::now();+ }+}++/// The phone's side of `media.watch`: show the offer.+async fn watch_offered(shared: &Arc<Shared>, peer: &DeviceId, watch: MediaWatch) -> Envelope {+ if !shared.local_capabilities().iter().any(|c| c == media_watch::WATCH) {+ return Envelope::error(ErrorCode::Unsupported, "this device doesn't show a PC's playback");+ }+ if !shared.toggle_on(peer, TOGGLE) {+ return Envelope::error(ErrorCode::Denied, "media is off for this device");+ }+ if !watch.is_valid() {+ return Envelope::error(ErrorCode::BadMessage, "invalid watch offer");+ }+ let platform = shared.platform.clone();+ let peer = *peer;+ let MediaWatch { player, title } = watch;+ match tokio::task::spawn_blocking(move || platform.watch_offered(&peer, &player, title.as_deref()))+ .await+ .unwrap_or_else(|e| Err(e.to_string()))+ {+ Ok(()) => Envelope::empty(types::OK),+ Err(reason) => {+ tracing::warn!(reason, "can't show a watch offer");+ Envelope::error(ErrorCode::Internal, "can't show it")+ }+ }+}+async fn run_command(shared: &Arc<Shared>, peer: &DeviceId, command: MediaCommand) -> Envelope {if !shared.local_capabilities().iter().any(|c| c == SHARES) {return Envelope::error(ErrorCode::Unsupported, "media control is not available");diff --git a/core/nectarlink-core/src/mirror.rs b/core/nectarlink-core/src/mirror.rsindex 988ce68..4aedd04 100644--- a/core/nectarlink-core/src/mirror.rs+++ b/core/nectarlink-core/src/mirror.rs@@ -333,14 +333,25 @@ pub(crate) async fn handle(shared: &Arc<Shared>, session: &Arc<Session>, env: &Ematch env.t.as_str() {types::MIRROR_START => {let options: MirrorStart = env.body()?;- let needs = if options.app.is_some() { mirror::VIRTUAL_DISPLAY } else { mirror::CAPTURE };let desktop = is_local_desktop(shared);- let allowed =- if desktop { pc_mirror_allowed(shared, &peer) } else { shared.toggle_on(&peer, TOGGLE) };+ let needs = if options.app.is_some() {+ mirror::VIRTUAL_DISPLAY+ } else if desktop && options.watch.is_some() {+ nectarlink_protocol::messages::media_watch::STREAM+ } else {+ mirror::CAPTURE+ };+ let allowed = match &options.watch {+ // Play on phone: only what this PC offered that phone.+ Some(player) if desktop => crate::media::watch_allowed(shared, &peer, player),+ Some(_) => false,+ None if desktop => pc_mirror_allowed(shared, &peer),+ None => shared.toggle_on(&peer, TOGGLE),+ };let reply = if !options.is_valid() {Envelope::error(ErrorCode::BadMessage, "invalid mirroring options")} else if !allowed {- if desktop && options.app.is_none() {+ if desktop && options.app.is_none() && options.watch.is_none() {shared.emit(NodeEvent::PcMirrorRequested { device: peer, options: options.clone() });}Envelope::error(ErrorCode::Denied, "mirroring is off for this device")@@ -370,6 +381,9 @@ pub(crate) async fn handle(shared: &Arc<Shared>, session: &Arc<Session>, env: &Eshared.stop_showing(&peer, mirroring);if mirroring == mirror::SCREEN {shared.pc_mirror_sessions.lock().unwrap_or_else(|e| e.into_inner()).remove(&peer);+ // Watching stopped: the phone may come back to it for a while+ // (the screen rotated, the app went to the background).+ crate::media::renew_watch(shared, &peer);}shared.emit(NodeEvent::Mirroring { device: peer, session: mirroring, on: false });tokio::task::spawn_blocking(move || platform.mirror_stop_requested(&peer, mirroring));@@ -543,7 +557,7 @@ impl MirrorStream {/// sound stream to one that asked for the sound too.pub(crate) async fn open(shared: &Arc<Shared>, session: &Session, audio: bool) -> Result<MirrorStream> {let allowed = if is_local_desktop(shared) {- pc_mirror_allowed(shared, &session.peer)+ pc_mirror_allowed(shared, &session.peer) || crate::media::watch_open(shared, &session.peer)} else {shared.toggle_on(&session.peer, TOGGLE)};diff --git a/core/nectarlink-core/src/node.rs b/core/nectarlink-core/src/node.rsindex b28a62c..9862c7d 100644--- a/core/nectarlink-core/src/node.rs+++ b/core/nectarlink-core/src/node.rs@@ -150,6 +150,8 @@ pub(crate) struct Shared {pub(crate) mirror_stops: Mutex<HashMap<MirrorStop, Arc<tokio::sync::Notify>>>,/// Peers granted one-time reverse-mirror consent for the current session.pub(crate) pc_mirror_sessions: Mutex<std::collections::HashSet<DeviceId>>,+ /// Play on phone offers this PC made, by phone: the player and when.+ pub(crate) watch_offers: Mutex<HashMap<DeviceId, (String, std::time::Instant)>>,/// Stop signals for phone webcam streams arriving here, by phone.pub(crate) webcam_stops: Mutex<HashMap<DeviceId, Arc<tokio::sync::Notify>>>,/// Per-peer rate limiters and prompt state for remote input.@@ -1083,6 +1085,7 @@ impl Shared {if current {sessions.remove(&ended.peer);lock(&self.pc_mirror_sessions).remove(&ended.peer);+ lock(&self.watch_offers).remove(&ended.peer);lock(&self.storage).remove_peer(&ended.peer);self.toggles.remove_peer(&ended.peer);self.deck.remove_peer(&ended.peer);@@ -1406,6 +1409,7 @@ impl Node {deck: Default::default(),mirror_stops: Mutex::new(HashMap::new()),pc_mirror_sessions: Mutex::new(std::collections::HashSet::new()),+ watch_offers: Mutex::new(HashMap::new()),webcam_stops: Mutex::new(HashMap::new()),remote: Mutex::new(Default::default()),storage: Mutex::new(Default::default()),@@ -2850,6 +2854,14 @@ impl Node {self.shared.media_changed(players).await;}+ /// Play on phone: offers `peer` (a phone) what this PC's `player` plays,+ /// streamed (`docs/protocol/media.md` §4). The phone shows the offer;+ /// taking it starts mirroring with `watch`, which this PC then allows+ /// for that phone and player.+ pub async fn offer_watch(&self, peer: DeviceId, player: String, title: Option<String>) -> Result<()> {+ crate::media::offer_watch(&self.shared, peer, player, title).await+ }+/// Runs a command on a paired device's media player.pub async fn media_command(&self,diff --git a/core/nectarlink-core/src/play_on_phone.rs b/core/nectarlink-core/src/play_on_phone.rsnew file mode 100644index 0000000..e553c67--- /dev/null+++ b/core/nectarlink-core/src/play_on_phone.rs@@ -0,0 +1,171 @@+// SPDX-License-Identifier: MPL-2.0+//! Play on phone: how to move what a PC is playing to a phone.+//!+//! Most playback is streamed from the PC as it plays (its picture and its+//! sound), so whatever the PC does to it carries over: a browser's ad+//! blocker on YouTube, for one. Streaming services protect their video+//! (captured, it comes out black), so for those the phone opens the same+//! page, which Android hands to the service's app, and the PC pauses.++use crate::handoff::with_video_timestamp;++/// What Play on phone does for what's playing.+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum PlayOnPhone {+ /// Stream the PC's playback to the phone; the PC keeps playing it.+ Stream {+ /// The page can also be opened on the phone instead (in its app),+ /// at this URL: the PC's page with its position, when known.+ open_instead: Option<String>,+ },+ /// Open this URL on the phone (the service's app takes it) and pause+ /// the PC.+ Open {+ url: String,+ /// The service, for the PC to say where it went ("Netflix").+ service: &'static str,+ },+}++/// Services whose video is protected: (host, or a host it ends with, name).+const PROTECTED: &[(&str, &str)] = &[+ ("netflix.com", "Netflix"),+ ("primevideo.com", "Prime Video"),+ ("hotstar.com", "JioHotstar"),+ ("jiohotstar.com", "JioHotstar"),+ ("disneyplus.com", "Disney+"),+ ("max.com", "Max"),+ ("hbomax.com", "Max"),+ ("tv.apple.com", "Apple TV"),+ ("hulu.com", "Hulu"),+ ("peacocktv.com", "Peacock"),+ ("paramountplus.com", "Paramount+"),+ ("sonyliv.com", "Sony LIV"),+ ("zee5.com", "ZEE5"),+ ("crunchyroll.com", "Crunchyroll"),+];++/// Decides for a page at `url` (the browser's address, when the player is+/// a browser tab and the PC could read it), playing at `position_secs`.+#[must_use]+pub fn plan(url: Option<&str>, position_secs: Option<u32>) -> PlayOnPhone {+ let Some(url) = url.map(str::trim).filter(|u| is_web(u)) else {+ return PlayOnPhone::Stream { open_instead: None };+ };+ let host = host_of(url);+ if let Some(service) = protected_service(&host, url) {+ return PlayOnPhone::Open { url: url.to_owned(), service };+ }+ let open_instead = match position_secs {+ Some(secs) if is_youtube(&host) => with_video_timestamp(url, secs),+ _ => url.to_owned(),+ };+ PlayOnPhone::Stream { open_instead: Some(open_instead) }+}++fn is_web(url: &str) -> bool {+ let lower = url.to_ascii_lowercase();+ lower.starts_with("https://") || lower.starts_with("http://")+}++/// The lowercase host of a web URL, without a port, user info or "www.".+fn host_of(url: &str) -> String {+ let rest = url.split_once("://").map_or(url, |(_, rest)| rest);+ let authority = rest.split(['/', '?', '#']).next().unwrap_or("");+ let host = authority.rsplit_once('@').map_or(authority, |(_, host)| host);+ let host = host.split(':').next().unwrap_or("").to_ascii_lowercase();+ host.strip_prefix("www.").map(str::to_owned).unwrap_or(host)+}++fn is_host(host: &str, domain: &str) -> bool {+ host == domain || host.strip_suffix(domain).is_some_and(|sub| sub.ends_with('.'))+}++fn is_youtube(host: &str) -> bool {+ is_host(host, "youtube.com") || host == "youtu.be"+}++fn protected_service(host: &str, url: &str) -> Option<&'static str> {+ if let Some(&(_, name)) = PROTECTED.iter().find(|(domain, _)| is_host(host, domain)) {+ return Some(name);+ }+ // Prime Video also plays on Amazon's own sites (amazon.com/gp/video).+ let amazon = host.split('.').next() == Some("amazon") || host.starts_with("amazon.");+ let path = url.split_once("://").map_or(url, |(_, rest)| rest);+ let path = path.find('/').map_or("", |i| &path[i..]).to_ascii_lowercase();+ (amazon && path.starts_with("/gp/video")).then_some("Prime Video")+}++#[cfg(test)]+mod tests {+ use super::*;++ fn stream(open: Option<&str>) -> PlayOnPhone {+ PlayOnPhone::Stream { open_instead: open.map(str::to_owned) }+ }++ #[test]+ fn youtube_streams_from_the_pc_and_can_open_at_the_same_second() {+ assert_eq!(+ plan(Some("https://www.youtube.com/watch?v=abc"), Some(90)),+ stream(Some("https://www.youtube.com/watch?v=abc&t=90"))+ );+ assert_eq!(+ plan(Some("https://m.youtube.com/watch?v=abc&t=5"), Some(42)),+ stream(Some("https://m.youtube.com/watch?v=abc&t=42"))+ );+ assert_eq!(plan(Some("https://youtu.be/abc"), None), stream(Some("https://youtu.be/abc")));+ }++ #[test]+ fn protected_services_open_in_their_app() {+ assert_eq!(+ plan(Some("https://www.netflix.com/watch/8118?trackId=1"), Some(600)),+ PlayOnPhone::Open {+ url: "https://www.netflix.com/watch/8118?trackId=1".into(),+ service: "Netflix"+ }+ );+ for (url, service) in [+ ("https://www.primevideo.com/detail/0ABC", "Prime Video"),+ ("https://www.amazon.co.uk/gp/video/detail/B0X", "Prime Video"),+ ("https://www.amazon.com/gp/video/detail/B0X", "Prime Video"),+ ("https://www.hotstar.com/in/shows/x/123", "JioHotstar"),+ ("https://www.jiohotstar.com/movies/y/456", "JioHotstar"),+ ("https://www.disneyplus.com/video/1", "Disney+"),+ ("https://play.max.com/video/watch/1", "Max"),+ ("https://tv.apple.com/us/show/x", "Apple TV"),+ ] {+ assert_eq!(plan(Some(url), None), PlayOnPhone::Open { url: url.into(), service }, "{url}");+ }+ }++ #[test]+ fn lookalike_hosts_are_not_the_service() {+ assert!(matches!(plan(Some("https://notnetflix.com/watch/1"), None), PlayOnPhone::Stream { .. }));+ assert!(matches!(plan(Some("https://netflix.com.example.org/"), None), PlayOnPhone::Stream { .. }));+ assert!(matches!(plan(Some("https://www.amazon.com/dp/B0X"), None), PlayOnPhone::Stream { .. }));+ assert!(matches!(plan(Some("https://climax.com/"), None), PlayOnPhone::Stream { .. }));+ }++ #[test]+ fn the_host_ignores_case_ports_and_user_info() {+ assert!(matches!(plan(Some("HTTPS://WWW.NETFLIX.COM:443/watch/1"), None), PlayOnPhone::Open { .. }));+ assert!(matches!(+ plan(Some("https://[email protected]/watch/1"), None),+ PlayOnPhone::Open { service: "Netflix", .. }+ ));+ }++ #[test]+ fn anything_else_streams() {+ // An app, not a browser tab: there's no page to open.+ assert_eq!(plan(None, Some(30)), stream(None));+ assert_eq!(plan(Some("file:///C:/Videos/a.mp4"), None), stream(None));+ assert_eq!(+ plan(Some("https://vimeo.com/123"), Some(30)),+ stream(Some("https://vimeo.com/123")),+ "only YouTube's position is put in the link"+ );+ }+}diff --git a/core/nectarlink-core/tests/two_devices.rs b/core/nectarlink-core/tests/two_devices.rsindex 9c09fb6..4f57f60 100644--- a/core/nectarlink-core/tests/two_devices.rs+++ b/core/nectarlink-core/tests/two_devices.rs@@ -40,6 +40,7 @@ struct RecordingPlatform {mms_sent: Mutex<Vec<MmsRecord>>,mirror_asks: Mutex<Vec<String>>,mirror_reject: std::sync::atomic::AtomicBool,+ watch_offers: Mutex<Vec<String>>,/// What a PC got of a phone's screen.screen: Arc<ScreenSink>,webcam_asks: Mutex<Vec<String>>,@@ -131,7 +132,17 @@ impl Platform for RecordingPlatform {}let sound = if options.audio { " with sound" } else { "" };let app = options.app.as_deref().map(|a| format!(" {a} as {}", options.session)).unwrap_or_default();- self.mirror_asks.lock().unwrap().push(format!("start {}{sound}{app}", options.max_size));+ let watch = options.watch.as_deref().map(|p| format!(" watching {p}")).unwrap_or_default();+ self.mirror_asks.lock().unwrap().push(format!("start {}{sound}{app}{watch}", options.max_size));+ Ok(())+ }+ fn watch_offered(+ &self,+ _from: &nectarlink_core::DeviceId,+ player: &str,+ title: Option<&str>,+ ) -> Result<(), String> {+ self.watch_offers.lock().unwrap().push(format!("{player}: {}", title.unwrap_or("")));Ok(())}fn mirror_stop_requested(&self, _peer: &nectarlink_core::DeviceId, session: u32) {@@ -1683,6 +1694,7 @@ async fn the_phone_screen_streams_to_the_pc() {app: None,stay_awake: false,screen_off: false,+ watch: None,};with_timeout("start", pc.node.mirror_start(phone_id, options)).await.unwrap();assert_eq!(*phone.platform.mirror_asks.lock().unwrap(), ["start 1920 with sound"]);@@ -1802,6 +1814,7 @@ async fn phone_apps_open_in_windows_of_their_own() {app: Some("com.example.chat".into()),stay_awake: false,screen_off: false,+ watch: None,};// Not Elevated yet: no apps, no app windows.@@ -4954,6 +4967,7 @@ async fn usb_transport_carries_features_when_wifi_unavailable() {app: None,stay_awake: false,screen_off: false,+ watch: None,};with_timeout("start mirror over USB", pc.node.mirror_start(phone_id, options)).await.unwrap();let stream = Arc::new(with_timeout("open mirror over USB", phone.node.mirror_open(pc_id)).await.unwrap());@@ -5009,6 +5023,7 @@ async fn reverse_mirror_requires_pc_consent_and_remote_input_toggle() {app: None,stay_awake: false,screen_off: false,+ watch: None,};// (a) A paired phone calling `mirror_start` against a PC is denied by default@@ -5572,3 +5587,75 @@ async fn pc_audio_reaches_the_phone_over_both_paths() {pc.node.shutdown().await;phone.node.shutdown().await;}++#[tokio::test(flavor = "multi_thread")]+async fn play_on_phone_lets_the_phone_watch_only_what_the_pc_offered() {+ use nectarlink_core::{MEDIA_STREAM, MEDIA_WATCH, MIRROR_SCREEN, MirrorStart};++ let mut pc = device_with("Desktop", DeviceKind::Desktop, &[MEDIA_STREAM]).await;+ let mut phone =+ device_with("Pixel", DeviceKind::Phone, &[MEDIA_WATCH, nectarlink_core::MIRROR_VIEW]).await;+ pair_qr(&mut pc, &mut phone).await;+ let (pc_id, phone_id) = (pc.node.device_id(), phone.node.device_id());+ let watch = |player: &str| MirrorStart {+ max_size: 1280,+ fps: 30,+ bitrate: 4_000_000,+ audio: true,+ session: MIRROR_SCREEN,+ app: None,+ stay_awake: false,+ screen_off: false,+ watch: Some(player.into()),+ };++ // Not offered: the phone can't watch, and the PC isn't asked to share its screen.+ let early = phone.node.mirror_start(pc_id, watch("Chrome")).await;+ assert!(matches!(early, Err(Error::Denied)), "{early:?}");+ assert!(pc.platform.mirror_asks.lock().unwrap().is_empty());++ with_timeout("offer", pc.node.offer_watch(phone_id, "Chrome".into(), Some("A video".into())))+ .await+ .unwrap();+ assert_eq!(*phone.platform.watch_offers.lock().unwrap(), ["Chrome: A video"]);++ // Another player than the one offered: no.+ let other = phone.node.mirror_start(pc_id, watch("Spotify")).await;+ assert!(matches!(other, Err(Error::Denied)), "{other:?}");++ // The one offered: the PC streams it, with its sound.+ with_timeout("watch", phone.node.mirror_start(pc_id, watch("Chrome"))).await.unwrap();+ assert_eq!(*pc.platform.mirror_asks.lock().unwrap(), ["start 1280 with sound watching Chrome"]);+ // The PC may open the stream to it, though screen sharing is off for the phone.+ let stream = with_timeout("open", pc.node.mirror_open(phone_id)).await.unwrap();+ drop(stream);++ // Stopped and started again (the phone rotated): still allowed.+ with_timeout("stop", phone.node.mirror_stop(pc_id, MIRROR_SCREEN)).await;+ with_timeout("watch again", phone.node.mirror_start(pc_id, watch("Chrome"))).await.unwrap();++ // The screen itself stays off limits.+ let screen = MirrorStart { watch: None, audio: false, ..watch("Chrome") };+ let denied = phone.node.mirror_start(pc_id, screen).await;+ assert!(matches!(denied, Err(Error::Denied)), "{denied:?}");++ // With media off for the phone, the PC can't offer.+ pc.node.set_device_toggle(phone_id, "media", false).unwrap();+ let off = pc.node.offer_watch(phone_id, "Chrome".into(), None).await;+ assert!(matches!(off, Err(Error::Denied)), "{off:?}");++ pc.node.shutdown().await;+ phone.node.shutdown().await;+}++#[tokio::test(flavor = "multi_thread")]+async fn play_on_phone_needs_a_phone_that_can_watch() {+ let mut pc = device_with("Desktop", DeviceKind::Desktop, &[nectarlink_core::MEDIA_STREAM]).await;+ let mut phone = device_with("Pixel", DeviceKind::Phone, &[]).await;+ pair_qr(&mut pc, &mut phone).await;+ let offer = pc.node.offer_watch(phone.node.device_id(), "Chrome".into(), None).await;+ assert!(matches!(offer, Err(Error::Unsupported)), "{offer:?}");+ assert!(phone.platform.watch_offers.lock().unwrap().is_empty());+ pc.node.shutdown().await;+ phone.node.shutdown().await;+}diff --git a/core/nectarlink-ffi/src/lib.rs b/core/nectarlink-ffi/src/lib.rsindex ed07eb9..ad98093 100644--- a/core/nectarlink-ffi/src/lib.rs+++ b/core/nectarlink-ffi/src/lib.rs@@ -3885,19 +3885,29 @@ impl NectarlinkNode {.await}- /// Asks a paired PC to stream its screen to this phone (`mirror.start`, `session == 0`).- pub async fn start_pc_mirror(&self, pc_id: String, max_size: u32, fps: u32, bitrate: u32) -> Result<()> {+ /// Asks a paired PC to stream its screen to this phone (`mirror.start`, `session == 0`),+ /// or with `watch`, what that PC player plays, with its sound (Play on phone, after the+ /// PC offered it).+ pub async fn start_pc_mirror(+ &self,+ pc_id: String,+ max_size: u32,+ fps: u32,+ bitrate: u32,+ watch: Option<String>,+ ) -> Result<()> {let id = parse_id(&pc_id)?;let node = self.node.clone();let options = core::MirrorStart {max_size,fps,bitrate,- audio: false,+ audio: watch.is_some(),session: core::MIRROR_SCREEN,app: None,stay_awake: false,screen_off: false,+ watch,};self.run(async move { Ok(node.mirror_start(id, options).await?) }).await}diff --git a/core/nectarlink-protocol/src/messages.rs b/core/nectarlink-protocol/src/messages.rsindex 439dd24..c2e9218 100644--- a/core/nectarlink-protocol/src/messages.rs+++ b/core/nectarlink-protocol/src/messages.rs@@ -38,6 +38,7 @@ pub mod types {pub const MEDIA_STATE: &str = "media.state";pub const MEDIA_SYNC: &str = "media.sync";pub const MEDIA_COMMAND: &str = "media.command";+ pub const MEDIA_WATCH: &str = "media.watch";pub const PC_POWER: &str = "pc.power";pub const PC_WAKE_INFO: &str = "pc.wake_info";pub const LINK_OPEN: &str = "link.open";@@ -1004,6 +1005,32 @@ pub struct MediaCommand {pub position: Option<u64>,}+/// Play on phone (`docs/protocol/media.md` §4).+pub mod media_watch {+ /// Offered by phones that show a PC's playback streamed to them.+ pub const WATCH: &str = "media.watch";+ /// Offered by PCs that stream a player's picture and sound on request.+ pub const STREAM: &str = "media.stream";+}++/// Body of `media.watch`: a PC offers to stream what a player of its own+/// plays to the phone.+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct MediaWatch {+ /// The PC's ID for the player, as in `media.state`.+ pub player: String,+ /// What it plays, to show with the offer.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub title: Option<String>,+}++impl MediaWatch {+ pub fn is_valid(&self) -> bool {+ (1..=media_limits::ID_BYTES).contains(&self.player.len())+ && self.title.as_ref().is_none_or(|t| t.chars().count() <= media_limits::TEXT_CHARS)+ }+}+// ---- Actions (docs/protocol/actions.md) ----/// Body of `pc.power`: `lock` or `sleep`.@@ -1278,17 +1305,30 @@ pub struct MirrorStart {/// Turn the phone's physical screen off while mirroring (`session == SCREEN`, Elevated).#[serde(default, skip_serializing_if = "std::ops::Not::not")]pub screen_off: bool,+ /// Play on phone: show what this PC player (its ID, as in `media.state`)+ /// plays instead of the PC's screen (`session == SCREEN`, sent to a PC+ /// that offered it with `media.watch`).+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub watch: Option<String>,}impl MirrorStart {/// The screen without an app, or an app with a session of its own.pub fn is_valid(&self) -> bool {- match &self.app {- None => self.session == mirror::SCREEN,- Some(app) => {- self.session != mirror::SCREEN && is_package_name(app) && !self.stay_awake && !self.screen_off+ let watch_ok = self+ .watch+ .as_ref()+ .is_none_or(|p| self.app.is_none() && (1..=media_limits::ID_BYTES).contains(&p.len()));+ watch_ok+ && match &self.app {+ None => self.session == mirror::SCREEN,+ Some(app) => {+ self.session != mirror::SCREEN+ && is_package_name(app)+ && !self.stay_awake+ && !self.screen_off+ }}- }}}@@ -5012,6 +5052,40 @@ mod tests {assert!(!StorageRename { from: "a.txt".into(), to: "a.txt".into() }.is_valid());}+ #[test]+ fn play_on_phone_messages_validate() {+ let offer = MediaWatch { player: "Chrome".into(), title: Some("A video".into()) };+ assert!(offer.is_valid());+ assert!(!MediaWatch { player: String::new(), title: None }.is_valid());+ assert!(!MediaWatch { player: "p".repeat(257), title: None }.is_valid());+ assert!(!MediaWatch { player: "Chrome".into(), title: Some("t".repeat(257)) }.is_valid());++ let screen = MirrorStart {+ max_size: 1280,+ fps: 30,+ bitrate: 4_000_000,+ audio: true,+ session: mirror::SCREEN,+ app: None,+ stay_awake: false,+ screen_off: false,+ watch: Some("Chrome".into()),+ };+ assert!(screen.is_valid());+ // Watching is of the PC's screen session only, never with an app.+ assert!(!MirrorStart { session: 3, ..screen.clone() }.is_valid());+ assert!(+ !MirrorStart { session: 3, app: Some("com.example.app".into()), ..screen.clone() }.is_valid()+ );+ assert!(!MirrorStart { watch: Some(String::new()), ..screen.clone() }.is_valid());+ // Older PCs and phones leave it out.+ let plain = MirrorStart { watch: None, ..screen };+ let mut bytes = Vec::new();+ ciborium::into_writer(&plain, &mut bytes).unwrap();+ let back: MirrorStart = ciborium::from_reader(bytes.as_slice()).unwrap();+ assert_eq!(back, plain);+ }+#[test]fn webcam_messages_validate_and_round_trip() {let start = WebcamStart::default();diff --git a/desktop/app/src/mirror.rs b/desktop/app/src/mirror.rsindex bc9d5d2..c0b75b4 100644--- a/desktop/app/src/mirror.rs+++ b/desktop/app/src/mirror.rs@@ -48,6 +48,7 @@ const OPTIONS: MirrorStart = MirrorStart {app: None,stay_awake: false,screen_off: false,+ watch: None,};/// Packets waiting for the decoder, at most (about two seconds).const BACKLOG: usize = 120;diff --git a/docs/protocol/media.md b/docs/protocol/media.mdindex 4285467..be69cf4 100644--- a/docs/protocol/media.md+++ b/docs/protocol/media.md@@ -13,6 +13,8 @@ phone's media controls. Both directions use the same messages.|---|---|---|| `media.control` | both | Shares its players and takes commands for them (a phone needs notification access to read other apps' players) || `media.remote` | both | Shows other devices' players and sends commands to them |+| `media.watch` | phone | Shows a PC's playback streamed to it (Play on phone, §4) |+| `media.stream` | PC | Streams a player's picture and sound on request (§4) |A device sends its players only to devices that offer `media.remote`, andonly while the user allows media for that device (the `media` device@@ -76,3 +78,43 @@ players. Receivers drop actions they don't know.player (Windows' flyout session, Android's media session) out of what itshares.- Implementations **MUST NOT** log titles, artists or albums (v0 §11).++## 4. Play on phone++A PC can move what one of its players plays to a phone. How depends on+what plays, and the PC decides:++- **Streaming services that protect their video** (Netflix, Prime Video,+ JioHotstar, Disney+ and the like) come out black when captured. The PC+ sends the page's address with `link.open` ([actions.md](actions.md)),+ which Android hands to the service's app, and pauses its own player.+- **Everything else** is streamed from the PC as it plays, picture and+ sound, so whatever the PC does to it carries over (a browser's ad+ blocker on YouTube, for one). The PC keeps playing.++Streaming starts with an offer from the PC:++```+t = "media.watch" id = n b = { player: text, title?: text } // PC to phone+t = "ok" re = n+```++`player` is the PC's ID for the player, as in `media.state`; `title` is+what it plays, up to 256 characters, to show with the offer. The phone+shows the offer (Android doesn't let an app open a screen from the+background, so usually as a notification) and answers `ok`. Errors:+`UNSUPPORTED` when the phone doesn't offer `media.watch`, `DENIED` when+media is off for the PC, `BAD_MESSAGE` for an empty or too long `player`+or `title`.++To watch, the phone sends `mirror.start` with `watch` set to that+player, `session` 0 and `audio` true ([mirror.md](mirror.md) §2). The PC+allows it only for a player it offered that phone, while the offer is+open: for 10 minutes after the offer or after watching last stopped, so+the phone can come back to it, and for as long as the connection lasts.+It then streams the part of its screen the player's picture shows in (a+browser's video, else its page, else its window) on the mirroring video+stream, and its sound as Phone speaker does. Input from the phone is+ignored while it watches. Anything else in that `mirror.start` is as for+the PC's screen; without an offer the answer is `DENIED`, and the PC+doesn't ask its user to share the screen.diff --git a/docs/protocol/mirror.md b/docs/protocol/mirror.mdindex 30f0845..4f34df9 100644--- a/docs/protocol/mirror.md+++ b/docs/protocol/mirror.md@@ -31,7 +31,7 @@ On the control stream.```t = "mirror.start" id = n b = { max_size: uint, fps: uint, bitrate: uint, ? audio: bool,? session: uint, ? app: text,- ? stay_awake: bool, ? screen_off: bool }+ ? stay_awake: bool, ? screen_off: bool, ? watch: text }t = "mirror.stop" id = n b = { ? session: uint }t = "mirror.power" b = { ? stay_awake: bool, ? screen_off: bool }t = "mirror.keyframe" b = { ? session: uint }@@ -55,6 +55,9 @@ means 0, so PCs and phones that predate sessions work as before.user agrees, and sound only if the phone can (it may share the screenwithout it). Errors: `DENIED` (mirroring is off for the PC),`UNSUPPORTED` (no `mirror.capture`).+- `watch` (phone → PC, `session` 0 only, never with `app`): Play on phone.+ Show what this PC player plays instead of the PC's screen, after the PC+ offered it with `media.watch` ([media.md](media.md) §4).- `mirror.stop` (either way): stop. The PC stops reading the video stream;the phone stops sharing, restores normal display power, and finishes it.- `mirror.power` (PC → phone): update `stay_awake` and `screen_off` while