Skip to content

Your Android phone and your Windows PC, working as one. Local-first, end-to-end encrypted, no account.

Mirroring and app windows: decode on the GPU, with software fallback

Bee986e90a36ff7
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c0a4cea..7098fc2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,10 @@ change things that need a step from you, and its notes say so.
except Phone as webcam, which needs Windows 11.
- Android: changing the theme in Settings no longer jumps back to Home
([#8](https://github.com/thtbee/nectarlink/issues/8)).
+- **Screen mirroring and app windows use the GPU.** They decode on the PC's
+ GPU when it has one, and in software when it doesn't or if the GPU fails
+ mid-stream, and convert pictures on several cores. Large phone screens and
+ several app windows stay smooth with less CPU.
- **Phone as webcam in 4K is smooth.** The PC decoded and converted each 4K
frame in about 47 ms, over the 33 ms a frame has at 30 fps, so frames were
dropped. It now decodes on the GPU when there is one and converts on several
diff --git a/desktop/app/src/mirror.rs b/desktop/app/src/mirror.rs
index 35ab8eb..e36e941 100644
--- a/desktop/app/src/mirror.rs
+++ b/desktop/app/src/mirror.rs
@@ -1633,7 +1633,9 @@ impl MirrorSink for Sink {
/// earlier ones), shows the newest picture of each batch.
fn decode(window: Window, items: &Receiver<Item>, waiting: &AtomicUsize) {
let stream = window.key();
- let mut decoder = match Decoder::new() {
+ // On the GPU when the PC has one (falling back to software on its own),
+ // so big phone screens and several app windows stay smooth.
+ let mut decoder = match Decoder::new_gpu("h264") {
Ok(decoder) => decoder,
Err(e) => {
tracing::error!(error = %e, "no H.264 decoder");
@@ -1641,6 +1643,7 @@ fn decode(window: Window, items: &Receiver<Item>, waiting: &AtomicUsize) {
return;
}
};
+ tracing::info!(gpu = decoder.on_gpu(), codec = decoder.codec(), "mirror decoder ready");
let keyframe_please = || {
if let Some(node) = core_host::node() {
core_host::spawn(async move { node.mirror_keyframe(window.device, window.session).await });
@@ -1654,8 +1657,15 @@ fn decode(window: Window, items: &Receiver<Item>, waiting: &AtomicUsize) {
match item {
Item::Config(config) => {
if decoder.codec() != config.codec {
- match Decoder::new_for_codec(&config.codec) {
- Ok(next) => decoder = next,
+ match Decoder::new_gpu(&config.codec) {
+ Ok(next) => {
+ decoder = next;
+ tracing::info!(
+ gpu = decoder.on_gpu(),
+ codec = decoder.codec(),
+ "mirror decoder ready"
+ );
+ }
Err(e) => {
tracing::error!(error = %e, codec = %config.codec, "no decoder for codec");
set_phase(
diff --git a/desktop/app/src/win/h264.rs b/desktop/app/src/win/h264.rs
index 2f94607..dea370a 100644
--- a/desktop/app/src/win/h264.rs
+++ b/desktop/app/src/win/h264.rs
@@ -202,12 +202,8 @@ impl Drop for Decoder {
}
impl Decoder {
- /// A decoder for the calling thread (which it stays on), defaulting to H.264.
- pub fn new() -> windows::core::Result<Decoder> {
- Self::new_for_codec("h264")
- }
-
- /// Creates a low-latency video decoder for `"h264"`, `"h265"`, or `"av1"`.
+ /// A low-latency software decoder for `"h264"`, `"h265"`, or `"av1"`, for
+ /// the calling thread (which it stays on).
pub fn new_for_codec(codec: &str) -> windows::core::Result<Decoder> {
Self::create(codec, false)
}
@@ -219,7 +215,7 @@ impl Decoder {
pub fn new_gpu(codec: &str) -> windows::core::Result<Decoder> {
Self::create(codec, true).or_else(|e| {
tracing::info!(error = %e, "no GPU video decoding; decoding in software");
- Self::create(codec, false)
+ Self::new_for_codec(codec)
})
}
@@ -636,7 +632,7 @@ mod tests {
fn the_stream_says_which_colors() {
// 3 frames tagged BT.601, full range (as some phones encode).
let units = access_units(include_bytes!("testdata/bt601-full.h264"));
- let mut decoder = Decoder::new().unwrap();
+ let mut decoder = Decoder::new_for_codec("h264").unwrap();
let pictures: Vec<Picture> =
units.iter().enumerate().flat_map(|(i, u)| decoder.decode(u, i as u64).unwrap()).collect();
assert_eq!(pictures.last().unwrap().colors, Colors { bt601: true, full_range: true });
@@ -686,40 +682,64 @@ mod tests {
.collect()
}
- #[test]
- fn the_gpu_decodes_the_same_pictures_as_software() {
- let units = access_units(include_bytes!("testdata/testsrc.h264"));
- let Ok(mut gpu) = Decoder::new_gpu("h264") else { return };
+ /// Decodes `units` on the GPU and in software and checks every picture
+ /// matches (decoding is exact). Skips when this PC has no GPU decoding
+ /// (CI runners) or no decoder for the codec (HEVC is an optional Windows
+ /// extension).
+ fn gpu_matches_software(codec: &str, units: &[&[u8]]) {
+ let Ok(mut software) = Decoder::new_for_codec(codec) else { return };
+ let Ok(mut gpu) = Decoder::new_gpu(codec) else { return };
if !gpu.on_gpu() {
- // No usable GPU here (CI runners): nothing to compare.
return;
}
- let mut software = Decoder::new().unwrap();
+ let mut compared = 0;
for (i, unit) in units.iter().enumerate() {
let a = software.decode(unit, i as u64 * 33_333).unwrap();
let b = gpu.decode(unit, i as u64 * 33_333).unwrap();
- assert_eq!(a.len(), b.len(), "unit {i}");
+ assert!(gpu.on_gpu(), "{codec}: fell back to software at unit {i}");
+ assert_eq!(a.len(), b.len(), "{codec} unit {i}");
for (a, b) in a.iter().zip(&b) {
assert_eq!((a.width, a.height), (b.width, b.height));
- // H.264 decoding is exact: same visible luma and chroma, row by row.
let (w, h) = (a.width as usize, a.height as usize);
for row in 0..h {
assert_eq!(
a.nv12[row * a.stride..][..w],
b.nv12[row * b.stride..][..w],
- "unit {i} luma row {row}"
+ "{codec} unit {i} luma row {row}"
);
}
+ let (ca, cb) = (a.stride * a.plane_rows, b.stride * b.plane_rows);
for row in 0..h / 2 {
- let (ca, cb) = (a.stride * a.plane_rows, b.stride * b.plane_rows);
assert_eq!(
a.nv12[ca + row * a.stride..][..w],
b.nv12[cb + row * b.stride..][..w],
- "unit {i} chroma row {row}"
+ "{codec} unit {i} chroma row {row}"
);
}
+ compared += 1;
}
}
+ assert!(compared > 0, "{codec}: no pictures to compare");
+ }
+
+ #[test]
+ fn the_gpu_decodes_the_same_h264_pictures_as_software() {
+ gpu_matches_software("h264", &access_units(include_bytes!("testdata/testsrc.h264")));
+ }
+
+ #[test]
+ fn the_gpu_decodes_the_same_h265_pictures_as_software() {
+ // 10 frames of FFmpeg's test pattern, 320x240, HEVC, access unit delimiters.
+ let stream = include_bytes!("testdata/testsrc.h265");
+ let starts: Vec<usize> =
+ (0..stream.len()).filter(|&i| stream[i..].starts_with(&[0, 0, 0, 1, 0x46])).collect();
+ let units: Vec<&[u8]> = starts
+ .iter()
+ .enumerate()
+ .map(|(k, &at)| &stream[at..starts.get(k + 1).copied().unwrap_or(stream.len())])
+ .collect();
+ assert_eq!(units.len(), 10);
+ gpu_matches_software("h265", &units);
}
#[test]
@@ -729,7 +749,7 @@ mod tests {
let stream = include_bytes!("testdata/testsrc.h264");
let units = access_units(stream);
assert_eq!(units.len(), 10);
- let mut decoder = Decoder::new().unwrap();
+ let mut decoder = Decoder::new_for_codec("h264").unwrap();
let mut pictures = Vec::new();
for (i, unit) in units.iter().enumerate() {
let got = decoder.decode(unit, i as u64 * 33_333).unwrap();
diff --git a/desktop/app/src/win/testdata/testsrc.h265 b/desktop/app/src/win/testdata/testsrc.h265
new file mode 100644
index 0000000..385639d
Binary files /dev/null and b/desktop/app/src/win/testdata/testsrc.h265 differ