Skip to content

Your Android phone and your Windows PC, working as one. Local-first, end-to-end encrypted, no account.

Play on phone: the protocol and the core's part

Beefbffb28ab0b7
diff --git a/android/app/src/main/kotlin/app/nectarlink/android/core/Core.kt b/android/app/src/main/kotlin/app/nectarlink/android/core/Core.kt
index 486f0e8..4ad1f7c 100644
--- a/android/app/src/main/kotlin/app/nectarlink/android/core/Core.kt
+++ b/android/app/src/main/kotlin/app/nectarlink/android/core/Core.kt
@@ -1891,14 +1891,24 @@ class Core(context: Context, private val scope: CoroutineScope) : EventListener
remoteOps.trySend { runCatching { it.remoteLaser(id, on, x, y) } }
}
- /** Asks a paired PC to start streaming its desktop screen to this phone (`mirror.start`, session 0). */
- fun startPcMirror(id: String, maxSize: Int = 1280, fps: Int = 30, bitrate: Int = 4_000_000) {
+ /**
+ * Asks a paired PC to start streaming its desktop screen to this phone (`mirror.start`,
+ * session 0), or with [watch], what that PC player plays, with its sound (Play on phone).
+ */
+ fun startPcMirror(
+ id: String,
+ maxSize: Int = 1280,
+ fps: Int = 30,
+ bitrate: Int = 4_000_000,
+ watch: String? = null,
+ ) {
command {
it.startPcMirror(
id,
maxSize.coerceIn(320, 1920).toUInt(),
fps.coerceIn(10, 60).toUInt(),
bitrate.coerceIn(500_000, 20_000_000).toUInt(),
+ watch,
)
}
}
diff --git a/core/nectarlink-cli/src/main.rs b/core/nectarlink-cli/src/main.rs
index 1726af6..a554ff6 100644
--- a/core/nectarlink-cli/src/main.rs
+++ b/core/nectarlink-cli/src/main.rs
@@ -3122,6 +3122,7 @@ async fn run(cli: &Cli, node: &Node) -> Result<()> {
app: app.clone(),
stay_awake: false,
screen_off: false,
+ watch: None,
};
node.mirror_start(id, options).await.context("the phone didn't ask its user")?;
println!(
diff --git a/core/nectarlink-core/src/config.rs b/core/nectarlink-core/src/config.rs
index 1df9c8f..b8a7ce9 100644
--- a/core/nectarlink-core/src/config.rs
+++ b/core/nectarlink-core/src/config.rs
@@ -320,6 +320,18 @@ pub trait Platform: Send + Sync + 'static {
Err("this device has no contacts".into())
}
+ /// Play on phone: a paired PC offers to stream what `player` plays
+ /// (`title`). The phone shows the offer; taking it starts mirroring with
+ /// `watch` set to `player`.
+ fn watch_offered(
+ &self,
+ _from: &nectarlink_protocol::DeviceId,
+ _player: &str,
+ _title: Option<&str>,
+ ) -> Result<(), String> {
+ Err("this device doesn't show a PC's playback".into())
+ }
+
/// Run a command on one of this device's media players (a paired device
/// asked); `position` is set for [`MediaAction::Seek`].
fn media_command(
diff --git a/core/nectarlink-core/src/lib.rs b/core/nectarlink-core/src/lib.rs
index 52c767e..4f5a8ed 100644
--- a/core/nectarlink-core/src/lib.rs
+++ b/core/nectarlink-core/src/lib.rs
@@ -29,6 +29,7 @@ pub mod otp;
mod pairing;
pub mod palette;
mod photos;
+pub mod play_on_phone;
pub mod remote;
mod session;
mod sms;
@@ -101,6 +102,7 @@ pub use nectarlink_protocol::{
},
format_mac, ipv4_broadcast, is_package_name, is_valid_storage_dir_path, is_valid_storage_id,
is_valid_storage_name, is_valid_storage_path, magic_packet,
+ media_watch::{STREAM as MEDIA_STREAM, WATCH as MEDIA_WATCH},
mirror::{
AUDIO as MIRROR_AUDIO, AUDIO_PLAYBACK as MIRROR_AUDIO_PLAYBACK, AV1 as MIRROR_AV1,
CAPTURE as MIRROR_CAPTURE, H264 as MIRROR_H264, H265 as MIRROR_H265, INPUT as MIRROR_INPUT,
@@ -140,6 +142,7 @@ pub use palette::{
achromatic_scheme, downsample_rgb, downsample_rgba, extract_artwork_scheme_rgb,
extract_artwork_scheme_rgba, scheme_from_seed_rgb, scheme_json, seed_from_grid, seed_from_pixels,
};
+pub use play_on_phone::PlayOnPhone;
pub use remote::{ButtonAction, INPUT_INJECT, KeyMod, MouseButton, RemoteInput, SlideAction, remote_keys};
pub use storage::{FolderStorage, StorageError, StorageReadFile, TOGGLE as STORAGE_TOGGLE};
pub use store::{
diff --git a/core/nectarlink-core/src/media.rs b/core/nectarlink-core/src/media.rs
index 5a1f295..a8311f8 100644
--- a/core/nectarlink-core/src/media.rs
+++ b/core/nectarlink-core/src/media.rs
@@ -14,10 +14,16 @@ use std::{
use nectarlink_protocol::{
DeviceId, Envelope, ErrorCode,
- messages::{MediaCommand, MediaPlayer, MediaState, media_actions, media_limits::PLAYERS, types},
+ messages::{
+ MediaCommand, MediaPlayer, MediaState, MediaWatch, media_actions, media_limits::PLAYERS, media_watch,
+ types,
+ },
};
-use crate::{Result, events::NodeEvent, node::Shared, session::Session};
+use crate::{Error, Result, events::NodeEvent, node::Shared, session::Session};
+
+/// How long a Play on phone offer stays open for the phone to take.
+const WATCH_OFFER: std::time::Duration = std::time::Duration::from_secs(10 * 60);
/// The device toggle that allows media for a device.
pub(crate) const TOGGLE: &str = "media";
@@ -209,6 +215,10 @@ pub(crate) async fn handle(shared: &Arc<Shared>, session: &Arc<Session>, env: &E
}
}
types::MEDIA_SYNC => shared.send_media_state(session).await,
+ types::MEDIA_WATCH => {
+ let reply = watch_offered(shared, &peer, env.body()?).await;
+ session.send(reply.reply_to(env.id)).await?;
+ }
types::MEDIA_COMMAND => {
let command: MediaCommand = env.body()?;
let reply = run_command(shared, &peer, command).await;
@@ -219,6 +229,101 @@ pub(crate) async fn handle(shared: &Arc<Shared>, session: &Arc<Session>, env: &E
Ok(true)
}
+/// Play on phone, on the PC: offers `peer` what `player` plays. The offer
+/// is what lets that phone start mirroring with `watch` (see
+/// [`watch_allowed`]).
+pub(crate) async fn offer_watch(
+ shared: &Arc<Shared>,
+ peer: DeviceId,
+ player: String,
+ title: Option<String>,
+) -> Result<()> {
+ let watch = MediaWatch { player, title };
+ if !watch.is_valid() {
+ return Err(Error::Internal("not a player to offer".into()));
+ }
+ if !shared.local_capabilities().iter().any(|c| c == media_watch::STREAM) {
+ return Err(Error::Unsupported);
+ }
+ if !shared.store.get_peer(&peer)?.is_some_and(|p| p.caps.contains(media_watch::WATCH)) {
+ return Err(Error::Unsupported);
+ }
+ if !shared.toggle_on(&peer, TOGGLE) {
+ return Err(Error::Denied);
+ }
+ let session = shared.session(&peer).ok_or(Error::Offline)?;
+ let player = watch.player.clone();
+ shared
+ .watch_offers
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .insert(peer, (player, std::time::Instant::now()));
+ let env = Envelope::new(types::MEDIA_WATCH, &watch)?;
+ let answer =
+ session.request(env, crate::session::REQUEST_TIMEOUT).await.and_then(|r| Ok(r.expect(types::OK)?));
+ if answer.is_err() {
+ shared.watch_offers.lock().unwrap_or_else(|e| e.into_inner()).remove(&peer);
+ }
+ answer.map(|_| ())
+}
+
+/// Whether `peer` may watch `player`: this PC offered it, and the phone
+/// takes it (or comes back to it) in time. Taking it renews the offer.
+pub(crate) fn watch_allowed(shared: &Shared, peer: &DeviceId, player: &str) -> bool {
+ let mut offers = shared.watch_offers.lock().unwrap_or_else(|e| e.into_inner());
+ match offers.get_mut(peer) {
+ Some((offered, at)) if offered == player && at.elapsed() < WATCH_OFFER => {
+ *at = std::time::Instant::now();
+ true
+ }
+ _ => false,
+ }
+}
+
+/// Whether `peer` has a Play on phone offer from this PC open (so this PC
+/// may open the stream to it).
+pub(crate) fn watch_open(shared: &Shared, peer: &DeviceId) -> bool {
+ shared
+ .watch_offers
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .get(peer)
+ .is_some_and(|(_, at)| at.elapsed() < WATCH_OFFER)
+}
+
+/// Watching stopped: the offer stays open for a while from now.
+pub(crate) fn renew_watch(shared: &Shared, peer: &DeviceId) {
+ if let Some((_, at)) = shared.watch_offers.lock().unwrap_or_else(|e| e.into_inner()).get_mut(peer) {
+ *at = std::time::Instant::now();
+ }
+}
+
+/// The phone's side of `media.watch`: show the offer.
+async fn watch_offered(shared: &Arc<Shared>, peer: &DeviceId, watch: MediaWatch) -> Envelope {
+ if !shared.local_capabilities().iter().any(|c| c == media_watch::WATCH) {
+ return Envelope::error(ErrorCode::Unsupported, "this device doesn't show a PC's playback");
+ }
+ if !shared.toggle_on(peer, TOGGLE) {
+ return Envelope::error(ErrorCode::Denied, "media is off for this device");
+ }
+ if !watch.is_valid() {
+ return Envelope::error(ErrorCode::BadMessage, "invalid watch offer");
+ }
+ let platform = shared.platform.clone();
+ let peer = *peer;
+ let MediaWatch { player, title } = watch;
+ match tokio::task::spawn_blocking(move || platform.watch_offered(&peer, &player, title.as_deref()))
+ .await
+ .unwrap_or_else(|e| Err(e.to_string()))
+ {
+ Ok(()) => Envelope::empty(types::OK),
+ Err(reason) => {
+ tracing::warn!(reason, "can't show a watch offer");
+ Envelope::error(ErrorCode::Internal, "can't show it")
+ }
+ }
+}
+
async fn run_command(shared: &Arc<Shared>, peer: &DeviceId, command: MediaCommand) -> Envelope {
if !shared.local_capabilities().iter().any(|c| c == SHARES) {
return Envelope::error(ErrorCode::Unsupported, "media control is not available");
diff --git a/core/nectarlink-core/src/mirror.rs b/core/nectarlink-core/src/mirror.rs
index 988ce68..4aedd04 100644
--- a/core/nectarlink-core/src/mirror.rs
+++ b/core/nectarlink-core/src/mirror.rs
@@ -333,14 +333,25 @@ pub(crate) async fn handle(shared: &Arc<Shared>, session: &Arc<Session>, env: &E
match env.t.as_str() {
types::MIRROR_START => {
let options: MirrorStart = env.body()?;
- let needs = if options.app.is_some() { mirror::VIRTUAL_DISPLAY } else { mirror::CAPTURE };
let desktop = is_local_desktop(shared);
- let allowed =
- if desktop { pc_mirror_allowed(shared, &peer) } else { shared.toggle_on(&peer, TOGGLE) };
+ let needs = if options.app.is_some() {
+ mirror::VIRTUAL_DISPLAY
+ } else if desktop && options.watch.is_some() {
+ nectarlink_protocol::messages::media_watch::STREAM
+ } else {
+ mirror::CAPTURE
+ };
+ let allowed = match &options.watch {
+ // Play on phone: only what this PC offered that phone.
+ Some(player) if desktop => crate::media::watch_allowed(shared, &peer, player),
+ Some(_) => false,
+ None if desktop => pc_mirror_allowed(shared, &peer),
+ None => shared.toggle_on(&peer, TOGGLE),
+ };
let reply = if !options.is_valid() {
Envelope::error(ErrorCode::BadMessage, "invalid mirroring options")
} else if !allowed {
- if desktop && options.app.is_none() {
+ if desktop && options.app.is_none() && options.watch.is_none() {
shared.emit(NodeEvent::PcMirrorRequested { device: peer, options: options.clone() });
}
Envelope::error(ErrorCode::Denied, "mirroring is off for this device")
@@ -370,6 +381,9 @@ pub(crate) async fn handle(shared: &Arc<Shared>, session: &Arc<Session>, env: &E
shared.stop_showing(&peer, mirroring);
if mirroring == mirror::SCREEN {
shared.pc_mirror_sessions.lock().unwrap_or_else(|e| e.into_inner()).remove(&peer);
+ // Watching stopped: the phone may come back to it for a while
+ // (the screen rotated, the app went to the background).
+ crate::media::renew_watch(shared, &peer);
}
shared.emit(NodeEvent::Mirroring { device: peer, session: mirroring, on: false });
tokio::task::spawn_blocking(move || platform.mirror_stop_requested(&peer, mirroring));
@@ -543,7 +557,7 @@ impl MirrorStream {
/// sound stream to one that asked for the sound too.
pub(crate) async fn open(shared: &Arc<Shared>, session: &Session, audio: bool) -> Result<MirrorStream> {
let allowed = if is_local_desktop(shared) {
- pc_mirror_allowed(shared, &session.peer)
+ pc_mirror_allowed(shared, &session.peer) || crate::media::watch_open(shared, &session.peer)
} else {
shared.toggle_on(&session.peer, TOGGLE)
};
diff --git a/core/nectarlink-core/src/node.rs b/core/nectarlink-core/src/node.rs
index b28a62c..9862c7d 100644
--- a/core/nectarlink-core/src/node.rs
+++ b/core/nectarlink-core/src/node.rs
@@ -150,6 +150,8 @@ pub(crate) struct Shared {
pub(crate) mirror_stops: Mutex<HashMap<MirrorStop, Arc<tokio::sync::Notify>>>,
/// Peers granted one-time reverse-mirror consent for the current session.
pub(crate) pc_mirror_sessions: Mutex<std::collections::HashSet<DeviceId>>,
+ /// Play on phone offers this PC made, by phone: the player and when.
+ pub(crate) watch_offers: Mutex<HashMap<DeviceId, (String, std::time::Instant)>>,
/// Stop signals for phone webcam streams arriving here, by phone.
pub(crate) webcam_stops: Mutex<HashMap<DeviceId, Arc<tokio::sync::Notify>>>,
/// Per-peer rate limiters and prompt state for remote input.
@@ -1083,6 +1085,7 @@ impl Shared {
if current {
sessions.remove(&ended.peer);
lock(&self.pc_mirror_sessions).remove(&ended.peer);
+ lock(&self.watch_offers).remove(&ended.peer);
lock(&self.storage).remove_peer(&ended.peer);
self.toggles.remove_peer(&ended.peer);
self.deck.remove_peer(&ended.peer);
@@ -1406,6 +1409,7 @@ impl Node {
deck: Default::default(),
mirror_stops: Mutex::new(HashMap::new()),
pc_mirror_sessions: Mutex::new(std::collections::HashSet::new()),
+ watch_offers: Mutex::new(HashMap::new()),
webcam_stops: Mutex::new(HashMap::new()),
remote: Mutex::new(Default::default()),
storage: Mutex::new(Default::default()),
@@ -2850,6 +2854,14 @@ impl Node {
self.shared.media_changed(players).await;
}
+ /// Play on phone: offers `peer` (a phone) what this PC's `player` plays,
+ /// streamed (`docs/protocol/media.md` §4). The phone shows the offer;
+ /// taking it starts mirroring with `watch`, which this PC then allows
+ /// for that phone and player.
+ pub async fn offer_watch(&self, peer: DeviceId, player: String, title: Option<String>) -> Result<()> {
+ crate::media::offer_watch(&self.shared, peer, player, title).await
+ }
+
/// Runs a command on a paired device's media player.
pub async fn media_command(
&self,
diff --git a/core/nectarlink-core/src/play_on_phone.rs b/core/nectarlink-core/src/play_on_phone.rs
new file mode 100644
index 0000000..e553c67
--- /dev/null
+++ b/core/nectarlink-core/src/play_on_phone.rs
@@ -0,0 +1,171 @@
+// SPDX-License-Identifier: MPL-2.0
+//! Play on phone: how to move what a PC is playing to a phone.
+//!
+//! Most playback is streamed from the PC as it plays (its picture and its
+//! sound), so whatever the PC does to it carries over: a browser's ad
+//! blocker on YouTube, for one. Streaming services protect their video
+//! (captured, it comes out black), so for those the phone opens the same
+//! page, which Android hands to the service's app, and the PC pauses.
+
+use crate::handoff::with_video_timestamp;
+
+/// What Play on phone does for what's playing.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum PlayOnPhone {
+ /// Stream the PC's playback to the phone; the PC keeps playing it.
+ Stream {
+ /// The page can also be opened on the phone instead (in its app),
+ /// at this URL: the PC's page with its position, when known.
+ open_instead: Option<String>,
+ },
+ /// Open this URL on the phone (the service's app takes it) and pause
+ /// the PC.
+ Open {
+ url: String,
+ /// The service, for the PC to say where it went ("Netflix").
+ service: &'static str,
+ },
+}
+
+/// Services whose video is protected: (host, or a host it ends with, name).
+const PROTECTED: &[(&str, &str)] = &[
+ ("netflix.com", "Netflix"),
+ ("primevideo.com", "Prime Video"),
+ ("hotstar.com", "JioHotstar"),
+ ("jiohotstar.com", "JioHotstar"),
+ ("disneyplus.com", "Disney+"),
+ ("max.com", "Max"),
+ ("hbomax.com", "Max"),
+ ("tv.apple.com", "Apple TV"),
+ ("hulu.com", "Hulu"),
+ ("peacocktv.com", "Peacock"),
+ ("paramountplus.com", "Paramount+"),
+ ("sonyliv.com", "Sony LIV"),
+ ("zee5.com", "ZEE5"),
+ ("crunchyroll.com", "Crunchyroll"),
+];
+
+/// Decides for a page at `url` (the browser's address, when the player is
+/// a browser tab and the PC could read it), playing at `position_secs`.
+#[must_use]
+pub fn plan(url: Option<&str>, position_secs: Option<u32>) -> PlayOnPhone {
+ let Some(url) = url.map(str::trim).filter(|u| is_web(u)) else {
+ return PlayOnPhone::Stream { open_instead: None };
+ };
+ let host = host_of(url);
+ if let Some(service) = protected_service(&host, url) {
+ return PlayOnPhone::Open { url: url.to_owned(), service };
+ }
+ let open_instead = match position_secs {
+ Some(secs) if is_youtube(&host) => with_video_timestamp(url, secs),
+ _ => url.to_owned(),
+ };
+ PlayOnPhone::Stream { open_instead: Some(open_instead) }
+}
+
+fn is_web(url: &str) -> bool {
+ let lower = url.to_ascii_lowercase();
+ lower.starts_with("https://") || lower.starts_with("http://")
+}
+
+/// The lowercase host of a web URL, without a port, user info or "www.".
+fn host_of(url: &str) -> String {
+ let rest = url.split_once("://").map_or(url, |(_, rest)| rest);
+ let authority = rest.split(['/', '?', '#']).next().unwrap_or("");
+ let host = authority.rsplit_once('@').map_or(authority, |(_, host)| host);
+ let host = host.split(':').next().unwrap_or("").to_ascii_lowercase();
+ host.strip_prefix("www.").map(str::to_owned).unwrap_or(host)
+}
+
+fn is_host(host: &str, domain: &str) -> bool {
+ host == domain || host.strip_suffix(domain).is_some_and(|sub| sub.ends_with('.'))
+}
+
+fn is_youtube(host: &str) -> bool {
+ is_host(host, "youtube.com") || host == "youtu.be"
+}
+
+fn protected_service(host: &str, url: &str) -> Option<&'static str> {
+ if let Some(&(_, name)) = PROTECTED.iter().find(|(domain, _)| is_host(host, domain)) {
+ return Some(name);
+ }
+ // Prime Video also plays on Amazon's own sites (amazon.com/gp/video).
+ let amazon = host.split('.').next() == Some("amazon") || host.starts_with("amazon.");
+ let path = url.split_once("://").map_or(url, |(_, rest)| rest);
+ let path = path.find('/').map_or("", |i| &path[i..]).to_ascii_lowercase();
+ (amazon && path.starts_with("/gp/video")).then_some("Prime Video")
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn stream(open: Option<&str>) -> PlayOnPhone {
+ PlayOnPhone::Stream { open_instead: open.map(str::to_owned) }
+ }
+
+ #[test]
+ fn youtube_streams_from_the_pc_and_can_open_at_the_same_second() {
+ assert_eq!(
+ plan(Some("https://www.youtube.com/watch?v=abc"), Some(90)),
+ stream(Some("https://www.youtube.com/watch?v=abc&t=90"))
+ );
+ assert_eq!(
+ plan(Some("https://m.youtube.com/watch?v=abc&t=5"), Some(42)),
+ stream(Some("https://m.youtube.com/watch?v=abc&t=42"))
+ );
+ assert_eq!(plan(Some("https://youtu.be/abc"), None), stream(Some("https://youtu.be/abc")));
+ }
+
+ #[test]
+ fn protected_services_open_in_their_app() {
+ assert_eq!(
+ plan(Some("https://www.netflix.com/watch/8118?trackId=1"), Some(600)),
+ PlayOnPhone::Open {
+ url: "https://www.netflix.com/watch/8118?trackId=1".into(),
+ service: "Netflix"
+ }
+ );
+ for (url, service) in [
+ ("https://www.primevideo.com/detail/0ABC", "Prime Video"),
+ ("https://www.amazon.co.uk/gp/video/detail/B0X", "Prime Video"),
+ ("https://www.amazon.com/gp/video/detail/B0X", "Prime Video"),
+ ("https://www.hotstar.com/in/shows/x/123", "JioHotstar"),
+ ("https://www.jiohotstar.com/movies/y/456", "JioHotstar"),
+ ("https://www.disneyplus.com/video/1", "Disney+"),
+ ("https://play.max.com/video/watch/1", "Max"),
+ ("https://tv.apple.com/us/show/x", "Apple TV"),
+ ] {
+ assert_eq!(plan(Some(url), None), PlayOnPhone::Open { url: url.into(), service }, "{url}");
+ }
+ }
+
+ #[test]
+ fn lookalike_hosts_are_not_the_service() {
+ assert!(matches!(plan(Some("https://notnetflix.com/watch/1"), None), PlayOnPhone::Stream { .. }));
+ assert!(matches!(plan(Some("https://netflix.com.example.org/"), None), PlayOnPhone::Stream { .. }));
+ assert!(matches!(plan(Some("https://www.amazon.com/dp/B0X"), None), PlayOnPhone::Stream { .. }));
+ assert!(matches!(plan(Some("https://climax.com/"), None), PlayOnPhone::Stream { .. }));
+ }
+
+ #[test]
+ fn the_host_ignores_case_ports_and_user_info() {
+ assert!(matches!(plan(Some("HTTPS://WWW.NETFLIX.COM:443/watch/1"), None), PlayOnPhone::Open { .. }));
+ assert!(matches!(
+ plan(Some("https://[email protected]/watch/1"), None),
+ PlayOnPhone::Open { service: "Netflix", .. }
+ ));
+ }
+
+ #[test]
+ fn anything_else_streams() {
+ // An app, not a browser tab: there's no page to open.
+ assert_eq!(plan(None, Some(30)), stream(None));
+ assert_eq!(plan(Some("file:///C:/Videos/a.mp4"), None), stream(None));
+ assert_eq!(
+ plan(Some("https://vimeo.com/123"), Some(30)),
+ stream(Some("https://vimeo.com/123")),
+ "only YouTube's position is put in the link"
+ );
+ }
+}
diff --git a/core/nectarlink-core/tests/two_devices.rs b/core/nectarlink-core/tests/two_devices.rs
index 9c09fb6..4f57f60 100644
--- a/core/nectarlink-core/tests/two_devices.rs
+++ b/core/nectarlink-core/tests/two_devices.rs
@@ -40,6 +40,7 @@ struct RecordingPlatform {
mms_sent: Mutex<Vec<MmsRecord>>,
mirror_asks: Mutex<Vec<String>>,
mirror_reject: std::sync::atomic::AtomicBool,
+ watch_offers: Mutex<Vec<String>>,
/// What a PC got of a phone's screen.
screen: Arc<ScreenSink>,
webcam_asks: Mutex<Vec<String>>,
@@ -131,7 +132,17 @@ impl Platform for RecordingPlatform {
}
let sound = if options.audio { " with sound" } else { "" };
let app = options.app.as_deref().map(|a| format!(" {a} as {}", options.session)).unwrap_or_default();
- self.mirror_asks.lock().unwrap().push(format!("start {}{sound}{app}", options.max_size));
+ let watch = options.watch.as_deref().map(|p| format!(" watching {p}")).unwrap_or_default();
+ self.mirror_asks.lock().unwrap().push(format!("start {}{sound}{app}{watch}", options.max_size));
+ Ok(())
+ }
+ fn watch_offered(
+ &self,
+ _from: &nectarlink_core::DeviceId,
+ player: &str,
+ title: Option<&str>,
+ ) -> Result<(), String> {
+ self.watch_offers.lock().unwrap().push(format!("{player}: {}", title.unwrap_or("")));
Ok(())
}
fn mirror_stop_requested(&self, _peer: &nectarlink_core::DeviceId, session: u32) {
@@ -1683,6 +1694,7 @@ async fn the_phone_screen_streams_to_the_pc() {
app: None,
stay_awake: false,
screen_off: false,
+ watch: None,
};
with_timeout("start", pc.node.mirror_start(phone_id, options)).await.unwrap();
assert_eq!(*phone.platform.mirror_asks.lock().unwrap(), ["start 1920 with sound"]);
@@ -1802,6 +1814,7 @@ async fn phone_apps_open_in_windows_of_their_own() {
app: Some("com.example.chat".into()),
stay_awake: false,
screen_off: false,
+ watch: None,
};
// Not Elevated yet: no apps, no app windows.
@@ -4954,6 +4967,7 @@ async fn usb_transport_carries_features_when_wifi_unavailable() {
app: None,
stay_awake: false,
screen_off: false,
+ watch: None,
};
with_timeout("start mirror over USB", pc.node.mirror_start(phone_id, options)).await.unwrap();
let stream = Arc::new(with_timeout("open mirror over USB", phone.node.mirror_open(pc_id)).await.unwrap());
@@ -5009,6 +5023,7 @@ async fn reverse_mirror_requires_pc_consent_and_remote_input_toggle() {
app: None,
stay_awake: false,
screen_off: false,
+ watch: None,
};
// (a) A paired phone calling `mirror_start` against a PC is denied by default
@@ -5572,3 +5587,75 @@ async fn pc_audio_reaches_the_phone_over_both_paths() {
pc.node.shutdown().await;
phone.node.shutdown().await;
}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn play_on_phone_lets_the_phone_watch_only_what_the_pc_offered() {
+ use nectarlink_core::{MEDIA_STREAM, MEDIA_WATCH, MIRROR_SCREEN, MirrorStart};
+
+ let mut pc = device_with("Desktop", DeviceKind::Desktop, &[MEDIA_STREAM]).await;
+ let mut phone =
+ device_with("Pixel", DeviceKind::Phone, &[MEDIA_WATCH, nectarlink_core::MIRROR_VIEW]).await;
+ pair_qr(&mut pc, &mut phone).await;
+ let (pc_id, phone_id) = (pc.node.device_id(), phone.node.device_id());
+ let watch = |player: &str| MirrorStart {
+ max_size: 1280,
+ fps: 30,
+ bitrate: 4_000_000,
+ audio: true,
+ session: MIRROR_SCREEN,
+ app: None,
+ stay_awake: false,
+ screen_off: false,
+ watch: Some(player.into()),
+ };
+
+ // Not offered: the phone can't watch, and the PC isn't asked to share its screen.
+ let early = phone.node.mirror_start(pc_id, watch("Chrome")).await;
+ assert!(matches!(early, Err(Error::Denied)), "{early:?}");
+ assert!(pc.platform.mirror_asks.lock().unwrap().is_empty());
+
+ with_timeout("offer", pc.node.offer_watch(phone_id, "Chrome".into(), Some("A video".into())))
+ .await
+ .unwrap();
+ assert_eq!(*phone.platform.watch_offers.lock().unwrap(), ["Chrome: A video"]);
+
+ // Another player than the one offered: no.
+ let other = phone.node.mirror_start(pc_id, watch("Spotify")).await;
+ assert!(matches!(other, Err(Error::Denied)), "{other:?}");
+
+ // The one offered: the PC streams it, with its sound.
+ with_timeout("watch", phone.node.mirror_start(pc_id, watch("Chrome"))).await.unwrap();
+ assert_eq!(*pc.platform.mirror_asks.lock().unwrap(), ["start 1280 with sound watching Chrome"]);
+ // The PC may open the stream to it, though screen sharing is off for the phone.
+ let stream = with_timeout("open", pc.node.mirror_open(phone_id)).await.unwrap();
+ drop(stream);
+
+ // Stopped and started again (the phone rotated): still allowed.
+ with_timeout("stop", phone.node.mirror_stop(pc_id, MIRROR_SCREEN)).await;
+ with_timeout("watch again", phone.node.mirror_start(pc_id, watch("Chrome"))).await.unwrap();
+
+ // The screen itself stays off limits.
+ let screen = MirrorStart { watch: None, audio: false, ..watch("Chrome") };
+ let denied = phone.node.mirror_start(pc_id, screen).await;
+ assert!(matches!(denied, Err(Error::Denied)), "{denied:?}");
+
+ // With media off for the phone, the PC can't offer.
+ pc.node.set_device_toggle(phone_id, "media", false).unwrap();
+ let off = pc.node.offer_watch(phone_id, "Chrome".into(), None).await;
+ assert!(matches!(off, Err(Error::Denied)), "{off:?}");
+
+ pc.node.shutdown().await;
+ phone.node.shutdown().await;
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn play_on_phone_needs_a_phone_that_can_watch() {
+ let mut pc = device_with("Desktop", DeviceKind::Desktop, &[nectarlink_core::MEDIA_STREAM]).await;
+ let mut phone = device_with("Pixel", DeviceKind::Phone, &[]).await;
+ pair_qr(&mut pc, &mut phone).await;
+ let offer = pc.node.offer_watch(phone.node.device_id(), "Chrome".into(), None).await;
+ assert!(matches!(offer, Err(Error::Unsupported)), "{offer:?}");
+ assert!(phone.platform.watch_offers.lock().unwrap().is_empty());
+ pc.node.shutdown().await;
+ phone.node.shutdown().await;
+}
diff --git a/core/nectarlink-ffi/src/lib.rs b/core/nectarlink-ffi/src/lib.rs
index ed07eb9..ad98093 100644
--- a/core/nectarlink-ffi/src/lib.rs
+++ b/core/nectarlink-ffi/src/lib.rs
@@ -3885,19 +3885,29 @@ impl NectarlinkNode {
.await
}
- /// Asks a paired PC to stream its screen to this phone (`mirror.start`, `session == 0`).
- pub async fn start_pc_mirror(&self, pc_id: String, max_size: u32, fps: u32, bitrate: u32) -> Result<()> {
+ /// Asks a paired PC to stream its screen to this phone (`mirror.start`, `session == 0`),
+ /// or with `watch`, what that PC player plays, with its sound (Play on phone, after the
+ /// PC offered it).
+ pub async fn start_pc_mirror(
+ &self,
+ pc_id: String,
+ max_size: u32,
+ fps: u32,
+ bitrate: u32,
+ watch: Option<String>,
+ ) -> Result<()> {
let id = parse_id(&pc_id)?;
let node = self.node.clone();
let options = core::MirrorStart {
max_size,
fps,
bitrate,
- audio: false,
+ audio: watch.is_some(),
session: core::MIRROR_SCREEN,
app: None,
stay_awake: false,
screen_off: false,
+ watch,
};
self.run(async move { Ok(node.mirror_start(id, options).await?) }).await
}
diff --git a/core/nectarlink-protocol/src/messages.rs b/core/nectarlink-protocol/src/messages.rs
index 439dd24..c2e9218 100644
--- a/core/nectarlink-protocol/src/messages.rs
+++ b/core/nectarlink-protocol/src/messages.rs
@@ -38,6 +38,7 @@ pub mod types {
pub const MEDIA_STATE: &str = "media.state";
pub const MEDIA_SYNC: &str = "media.sync";
pub const MEDIA_COMMAND: &str = "media.command";
+ pub const MEDIA_WATCH: &str = "media.watch";
pub const PC_POWER: &str = "pc.power";
pub const PC_WAKE_INFO: &str = "pc.wake_info";
pub const LINK_OPEN: &str = "link.open";
@@ -1004,6 +1005,32 @@ pub struct MediaCommand {
pub position: Option<u64>,
}
+/// Play on phone (`docs/protocol/media.md` §4).
+pub mod media_watch {
+ /// Offered by phones that show a PC's playback streamed to them.
+ pub const WATCH: &str = "media.watch";
+ /// Offered by PCs that stream a player's picture and sound on request.
+ pub const STREAM: &str = "media.stream";
+}
+
+/// Body of `media.watch`: a PC offers to stream what a player of its own
+/// plays to the phone.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct MediaWatch {
+ /// The PC's ID for the player, as in `media.state`.
+ pub player: String,
+ /// What it plays, to show with the offer.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub title: Option<String>,
+}
+
+impl MediaWatch {
+ pub fn is_valid(&self) -> bool {
+ (1..=media_limits::ID_BYTES).contains(&self.player.len())
+ && self.title.as_ref().is_none_or(|t| t.chars().count() <= media_limits::TEXT_CHARS)
+ }
+}
+
// ---- Actions (docs/protocol/actions.md) ----
/// Body of `pc.power`: `lock` or `sleep`.
@@ -1278,17 +1305,30 @@ pub struct MirrorStart {
/// Turn the phone's physical screen off while mirroring (`session == SCREEN`, Elevated).
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub screen_off: bool,
+ /// Play on phone: show what this PC player (its ID, as in `media.state`)
+ /// plays instead of the PC's screen (`session == SCREEN`, sent to a PC
+ /// that offered it with `media.watch`).
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub watch: Option<String>,
}
impl MirrorStart {
/// The screen without an app, or an app with a session of its own.
pub fn is_valid(&self) -> bool {
- match &self.app {
- None => self.session == mirror::SCREEN,
- Some(app) => {
- self.session != mirror::SCREEN && is_package_name(app) && !self.stay_awake && !self.screen_off
+ let watch_ok = self
+ .watch
+ .as_ref()
+ .is_none_or(|p| self.app.is_none() && (1..=media_limits::ID_BYTES).contains(&p.len()));
+ watch_ok
+ && match &self.app {
+ None => self.session == mirror::SCREEN,
+ Some(app) => {
+ self.session != mirror::SCREEN
+ && is_package_name(app)
+ && !self.stay_awake
+ && !self.screen_off
+ }
}
- }
}
}
@@ -5012,6 +5052,40 @@ mod tests {
assert!(!StorageRename { from: "a.txt".into(), to: "a.txt".into() }.is_valid());
}
+ #[test]
+ fn play_on_phone_messages_validate() {
+ let offer = MediaWatch { player: "Chrome".into(), title: Some("A video".into()) };
+ assert!(offer.is_valid());
+ assert!(!MediaWatch { player: String::new(), title: None }.is_valid());
+ assert!(!MediaWatch { player: "p".repeat(257), title: None }.is_valid());
+ assert!(!MediaWatch { player: "Chrome".into(), title: Some("t".repeat(257)) }.is_valid());
+
+ let screen = MirrorStart {
+ max_size: 1280,
+ fps: 30,
+ bitrate: 4_000_000,
+ audio: true,
+ session: mirror::SCREEN,
+ app: None,
+ stay_awake: false,
+ screen_off: false,
+ watch: Some("Chrome".into()),
+ };
+ assert!(screen.is_valid());
+ // Watching is of the PC's screen session only, never with an app.
+ assert!(!MirrorStart { session: 3, ..screen.clone() }.is_valid());
+ assert!(
+ !MirrorStart { session: 3, app: Some("com.example.app".into()), ..screen.clone() }.is_valid()
+ );
+ assert!(!MirrorStart { watch: Some(String::new()), ..screen.clone() }.is_valid());
+ // Older PCs and phones leave it out.
+ let plain = MirrorStart { watch: None, ..screen };
+ let mut bytes = Vec::new();
+ ciborium::into_writer(&plain, &mut bytes).unwrap();
+ let back: MirrorStart = ciborium::from_reader(bytes.as_slice()).unwrap();
+ assert_eq!(back, plain);
+ }
+
#[test]
fn webcam_messages_validate_and_round_trip() {
let start = WebcamStart::default();
diff --git a/desktop/app/src/mirror.rs b/desktop/app/src/mirror.rs
index bc9d5d2..c0b75b4 100644
--- a/desktop/app/src/mirror.rs
+++ b/desktop/app/src/mirror.rs
@@ -48,6 +48,7 @@ const OPTIONS: MirrorStart = MirrorStart {
app: None,
stay_awake: false,
screen_off: false,
+ watch: None,
};
/// Packets waiting for the decoder, at most (about two seconds).
const BACKLOG: usize = 120;
diff --git a/docs/protocol/media.md b/docs/protocol/media.md
index 4285467..be69cf4 100644
--- a/docs/protocol/media.md
+++ b/docs/protocol/media.md
@@ -13,6 +13,8 @@ phone's media controls. Both directions use the same messages.
|---|---|---|
| `media.control` | both | Shares its players and takes commands for them (a phone needs notification access to read other apps' players) |
| `media.remote` | both | Shows other devices' players and sends commands to them |
+| `media.watch` | phone | Shows a PC's playback streamed to it (Play on phone, §4) |
+| `media.stream` | PC | Streams a player's picture and sound on request (§4) |
A device sends its players only to devices that offer `media.remote`, and
only while the user allows media for that device (the `media` device
@@ -76,3 +78,43 @@ players. Receivers drop actions they don't know.
player (Windows' flyout session, Android's media session) out of what it
shares.
- Implementations **MUST NOT** log titles, artists or albums (v0 §11).
+
+## 4. Play on phone
+
+A PC can move what one of its players plays to a phone. How depends on
+what plays, and the PC decides:
+
+- **Streaming services that protect their video** (Netflix, Prime Video,
+ JioHotstar, Disney+ and the like) come out black when captured. The PC
+ sends the page's address with `link.open` ([actions.md](actions.md)),
+ which Android hands to the service's app, and pauses its own player.
+- **Everything else** is streamed from the PC as it plays, picture and
+ sound, so whatever the PC does to it carries over (a browser's ad
+ blocker on YouTube, for one). The PC keeps playing.
+
+Streaming starts with an offer from the PC:
+
+```
+t = "media.watch" id = n b = { player: text, title?: text } // PC to phone
+t = "ok" re = n
+```
+
+`player` is the PC's ID for the player, as in `media.state`; `title` is
+what it plays, up to 256 characters, to show with the offer. The phone
+shows the offer (Android doesn't let an app open a screen from the
+background, so usually as a notification) and answers `ok`. Errors:
+`UNSUPPORTED` when the phone doesn't offer `media.watch`, `DENIED` when
+media is off for the PC, `BAD_MESSAGE` for an empty or too long `player`
+or `title`.
+
+To watch, the phone sends `mirror.start` with `watch` set to that
+player, `session` 0 and `audio` true ([mirror.md](mirror.md) §2). The PC
+allows it only for a player it offered that phone, while the offer is
+open: for 10 minutes after the offer or after watching last stopped, so
+the phone can come back to it, and for as long as the connection lasts.
+It then streams the part of its screen the player's picture shows in (a
+browser's video, else its page, else its window) on the mirroring video
+stream, and its sound as Phone speaker does. Input from the phone is
+ignored while it watches. Anything else in that `mirror.start` is as for
+the PC's screen; without an offer the answer is `DENIED`, and the PC
+doesn't ask its user to share the screen.
diff --git a/docs/protocol/mirror.md b/docs/protocol/mirror.md
index 30f0845..4f34df9 100644
--- a/docs/protocol/mirror.md
+++ b/docs/protocol/mirror.md
@@ -31,7 +31,7 @@ On the control stream.
```
t = "mirror.start" id = n b = { max_size: uint, fps: uint, bitrate: uint, ? audio: bool,
? session: uint, ? app: text,
- ? stay_awake: bool, ? screen_off: bool }
+ ? stay_awake: bool, ? screen_off: bool, ? watch: text }
t = "mirror.stop" id = n b = { ? session: uint }
t = "mirror.power" b = { ? stay_awake: bool, ? screen_off: bool }
t = "mirror.keyframe" b = { ? session: uint }
@@ -55,6 +55,9 @@ means 0, so PCs and phones that predate sessions work as before.
user agrees, and sound only if the phone can (it may share the screen
without it). Errors: `DENIED` (mirroring is off for the PC),
`UNSUPPORTED` (no `mirror.capture`).
+- `watch` (phone → PC, `session` 0 only, never with `app`): Play on phone.
+ Show what this PC player plays instead of the PC's screen, after the PC
+ offered it with `media.watch` ([media.md](media.md) §4).
- `mirror.stop` (either way): stop. The PC stops reading the video stream;
the phone stops sharing, restores normal display power, and finishes it.
- `mirror.power` (PC → phone): update `stay_awake` and `screen_off` while