Your Android phone and your Windows PC, working as one. Local-first, end-to-end encrypted, no account.
Mirroring and app windows: decode on the GPU, with software fallback
Bee986e90a36ff7
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex c0a4cea..7098fc2 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -14,6 +14,10 @@ change things that need a step from you, and its notes say so.except Phone as webcam, which needs Windows 11.- Android: changing the theme in Settings no longer jumps back to Home([#8](https://github.com/thtbee/nectarlink/issues/8)).+- **Screen mirroring and app windows use the GPU.** They decode on the PC's+ GPU when it has one, and in software when it doesn't or if the GPU fails+ mid-stream, and convert pictures on several cores. Large phone screens and+ several app windows stay smooth with less CPU.- **Phone as webcam in 4K is smooth.** The PC decoded and converted each 4Kframe in about 47 ms, over the 33 ms a frame has at 30 fps, so frames weredropped. It now decodes on the GPU when there is one and converts on severaldiff --git a/desktop/app/src/mirror.rs b/desktop/app/src/mirror.rsindex 35ab8eb..e36e941 100644--- a/desktop/app/src/mirror.rs+++ b/desktop/app/src/mirror.rs@@ -1633,7 +1633,9 @@ impl MirrorSink for Sink {/// earlier ones), shows the newest picture of each batch.fn decode(window: Window, items: &Receiver<Item>, waiting: &AtomicUsize) {let stream = window.key();- let mut decoder = match Decoder::new() {+ // On the GPU when the PC has one (falling back to software on its own),+ // so big phone screens and several app windows stay smooth.+ let mut decoder = match Decoder::new_gpu("h264") {Ok(decoder) => decoder,Err(e) => {tracing::error!(error = %e, "no H.264 decoder");@@ -1641,6 +1643,7 @@ fn decode(window: Window, items: &Receiver<Item>, waiting: &AtomicUsize) {return;}};+ tracing::info!(gpu = decoder.on_gpu(), codec = decoder.codec(), "mirror decoder ready");let keyframe_please = || {if let Some(node) = core_host::node() {core_host::spawn(async move { node.mirror_keyframe(window.device, window.session).await });@@ -1654,8 +1657,15 @@ fn decode(window: Window, items: &Receiver<Item>, waiting: &AtomicUsize) {match item {Item::Config(config) => {if decoder.codec() != config.codec {- match Decoder::new_for_codec(&config.codec) {- Ok(next) => decoder = next,+ match Decoder::new_gpu(&config.codec) {+ Ok(next) => {+ decoder = next;+ tracing::info!(+ gpu = decoder.on_gpu(),+ codec = decoder.codec(),+ "mirror decoder ready"+ );+ }Err(e) => {tracing::error!(error = %e, codec = %config.codec, "no decoder for codec");set_phase(diff --git a/desktop/app/src/win/h264.rs b/desktop/app/src/win/h264.rsindex 2f94607..dea370a 100644--- a/desktop/app/src/win/h264.rs+++ b/desktop/app/src/win/h264.rs@@ -202,12 +202,8 @@ impl Drop for Decoder {}impl Decoder {- /// A decoder for the calling thread (which it stays on), defaulting to H.264.- pub fn new() -> windows::core::Result<Decoder> {- Self::new_for_codec("h264")- }-- /// Creates a low-latency video decoder for `"h264"`, `"h265"`, or `"av1"`.+ /// A low-latency software decoder for `"h264"`, `"h265"`, or `"av1"`, for+ /// the calling thread (which it stays on).pub fn new_for_codec(codec: &str) -> windows::core::Result<Decoder> {Self::create(codec, false)}@@ -219,7 +215,7 @@ impl Decoder {pub fn new_gpu(codec: &str) -> windows::core::Result<Decoder> {Self::create(codec, true).or_else(|e| {tracing::info!(error = %e, "no GPU video decoding; decoding in software");- Self::create(codec, false)+ Self::new_for_codec(codec)})}@@ -636,7 +632,7 @@ mod tests {fn the_stream_says_which_colors() {// 3 frames tagged BT.601, full range (as some phones encode).let units = access_units(include_bytes!("testdata/bt601-full.h264"));- let mut decoder = Decoder::new().unwrap();+ let mut decoder = Decoder::new_for_codec("h264").unwrap();let pictures: Vec<Picture> =units.iter().enumerate().flat_map(|(i, u)| decoder.decode(u, i as u64).unwrap()).collect();assert_eq!(pictures.last().unwrap().colors, Colors { bt601: true, full_range: true });@@ -686,40 +682,64 @@ mod tests {.collect()}- #[test]- fn the_gpu_decodes_the_same_pictures_as_software() {- let units = access_units(include_bytes!("testdata/testsrc.h264"));- let Ok(mut gpu) = Decoder::new_gpu("h264") else { return };+ /// Decodes `units` on the GPU and in software and checks every picture+ /// matches (decoding is exact). Skips when this PC has no GPU decoding+ /// (CI runners) or no decoder for the codec (HEVC is an optional Windows+ /// extension).+ fn gpu_matches_software(codec: &str, units: &[&[u8]]) {+ let Ok(mut software) = Decoder::new_for_codec(codec) else { return };+ let Ok(mut gpu) = Decoder::new_gpu(codec) else { return };if !gpu.on_gpu() {- // No usable GPU here (CI runners): nothing to compare.return;}- let mut software = Decoder::new().unwrap();+ let mut compared = 0;for (i, unit) in units.iter().enumerate() {let a = software.decode(unit, i as u64 * 33_333).unwrap();let b = gpu.decode(unit, i as u64 * 33_333).unwrap();- assert_eq!(a.len(), b.len(), "unit {i}");+ assert!(gpu.on_gpu(), "{codec}: fell back to software at unit {i}");+ assert_eq!(a.len(), b.len(), "{codec} unit {i}");for (a, b) in a.iter().zip(&b) {assert_eq!((a.width, a.height), (b.width, b.height));- // H.264 decoding is exact: same visible luma and chroma, row by row.let (w, h) = (a.width as usize, a.height as usize);for row in 0..h {assert_eq!(a.nv12[row * a.stride..][..w],b.nv12[row * b.stride..][..w],- "unit {i} luma row {row}"+ "{codec} unit {i} luma row {row}");}+ let (ca, cb) = (a.stride * a.plane_rows, b.stride * b.plane_rows);for row in 0..h / 2 {- let (ca, cb) = (a.stride * a.plane_rows, b.stride * b.plane_rows);assert_eq!(a.nv12[ca + row * a.stride..][..w],b.nv12[cb + row * b.stride..][..w],- "unit {i} chroma row {row}"+ "{codec} unit {i} chroma row {row}");}+ compared += 1;}}+ assert!(compared > 0, "{codec}: no pictures to compare");+ }++ #[test]+ fn the_gpu_decodes_the_same_h264_pictures_as_software() {+ gpu_matches_software("h264", &access_units(include_bytes!("testdata/testsrc.h264")));+ }++ #[test]+ fn the_gpu_decodes_the_same_h265_pictures_as_software() {+ // 10 frames of FFmpeg's test pattern, 320x240, HEVC, access unit delimiters.+ let stream = include_bytes!("testdata/testsrc.h265");+ let starts: Vec<usize> =+ (0..stream.len()).filter(|&i| stream[i..].starts_with(&[0, 0, 0, 1, 0x46])).collect();+ let units: Vec<&[u8]> = starts+ .iter()+ .enumerate()+ .map(|(k, &at)| &stream[at..starts.get(k + 1).copied().unwrap_or(stream.len())])+ .collect();+ assert_eq!(units.len(), 10);+ gpu_matches_software("h265", &units);}#[test]@@ -729,7 +749,7 @@ mod tests {let stream = include_bytes!("testdata/testsrc.h264");let units = access_units(stream);assert_eq!(units.len(), 10);- let mut decoder = Decoder::new().unwrap();+ let mut decoder = Decoder::new_for_codec("h264").unwrap();let mut pictures = Vec::new();for (i, unit) in units.iter().enumerate() {let got = decoder.decode(unit, i as u64 * 33_333).unwrap();diff --git a/desktop/app/src/win/testdata/testsrc.h265 b/desktop/app/src/win/testdata/testsrc.h265new file mode 100644index 0000000..385639dBinary files /dev/null and b/desktop/app/src/win/testdata/testsrc.h265 differ