Skip to content

Your Android phone and your Windows PC, working as one. Local-first, end-to-end encrypted, no account.

Webcam: decode 4K on the GPU and convert it on several cores

Bee142cd9153d7c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5f09830..8d2b970 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,11 @@ change things that need a step from you, and its notes say so.
except Phone as webcam, which needs Windows 11.
- Android: changing the theme in Settings no longer jumps back to Home
([#8](https://github.com/thtbee/nectarlink/issues/8)).
+- **Phone as webcam in 4K is smooth.** The PC decoded and converted each 4K
+ frame in about 47 ms, over the 33 ms a frame has at 30 fps, so frames were
+ dropped. It now decodes on the GPU when there is one and converts on several
+ cores, about 19 ms a frame
+ ([#6](https://github.com/thtbee/nectarlink/issues/6)).
## 0.1.3 (2026-10-15)
diff --git a/desktop/app/Cargo.toml b/desktop/app/Cargo.toml
index e744f09..8d115ce 100644
--- a/desktop/app/Cargo.toml
+++ b/desktop/app/Cargo.toml
@@ -61,7 +61,11 @@ windows = { workspace = true, features = [
"UI_Notifications",
"Win32_Foundation",
"Win32_UI_Input_XboxController",
+ "Win32_Graphics_Direct3D",
+ "Win32_Graphics_Direct3D11",
"Win32_Graphics_Dwm",
+ "Win32_Graphics_Dxgi",
+ "Win32_Graphics_Dxgi_Common",
"Win32_Graphics_Gdi",
"Win32_Graphics_Imaging",
"Win32_Storage_CloudFilters",
diff --git a/desktop/app/src/webcam.rs b/desktop/app/src/webcam.rs
index a257e54..ab656eb 100644
--- a/desktop/app/src/webcam.rs
+++ b/desktop/app/src/webcam.rs
@@ -636,7 +636,9 @@ fn mirror_bgrx_in_place(bgrx: &mut [u8], width: u32, height: u32) {
}
fn decode(device: DeviceId, items: &Receiver<Item>, waiting: &AtomicUsize) {
- let mut decoder = match Decoder::new() {
+ // On the GPU when there is one: 4K takes about 24 ms a frame in software
+ // and 11 ms on a GPU, against 33 ms per frame at 30 fps.
+ let mut decoder = match Decoder::new_gpu("h264") {
Ok(decoder) => decoder,
Err(e) => {
tracing::error!(error = %e, "no H.264 decoder for webcam");
@@ -648,6 +650,7 @@ fn decode(device: DeviceId, items: &Receiver<Item>, waiting: &AtomicUsize) {
return;
}
};
+ tracing::info!(gpu = decoder.on_gpu(), "webcam decoder ready");
let keyframe_please = || {
if let Some(node) = core_host::node() {
core_host::spawn(async move { node.webcam_keyframe(device).await });
diff --git a/desktop/app/src/win/h264.rs b/desktop/app/src/win/h264.rs
index e108f74..2f94607 100644
--- a/desktop/app/src/win/h264.rs
+++ b/desktop/app/src/win/h264.rs
@@ -8,17 +8,28 @@ use std::{mem::ManuallyDrop, sync::Mutex, time::Duration};
use windows::{
Win32::{
Foundation::E_FAIL,
+ Graphics::{
+ Direct3D::D3D_DRIVER_TYPE_HARDWARE,
+ Direct3D11::{
+ D3D11_CPU_ACCESS_READ, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_MAP_READ,
+ D3D11_MAPPED_SUBRESOURCE, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, D3D11_USAGE_STAGING,
+ D3D11CreateDevice, ID3D11Device, ID3D11Multithread, ID3D11Texture2D,
+ },
+ Dxgi::Common::DXGI_SAMPLE_DESC,
+ },
Media::MediaFoundation::{
- CLSID_MSH264DecoderMFT, CODECAPI_AVDecNumWorkerThreads, IMF2DBuffer, IMFActivate, IMFMediaType,
- IMFSample, IMFTransform, MF_E_NOTACCEPTING, MF_E_TRANSFORM_NEED_MORE_INPUT,
- MF_E_TRANSFORM_STREAM_CHANGE, MF_LOW_LATENCY, MF_MT_FRAME_SIZE, MF_MT_MAJOR_TYPE, MF_MT_SUBTYPE,
- MF_MT_VIDEO_NOMINAL_RANGE, MF_MT_YUV_MATRIX, MF_VERSION, MFCreateMediaType, MFCreateMemoryBuffer,
- MFCreateSample, MFMediaType_Video, MFNominalRange_0_255, MFSTARTUP_NOSOCKET, MFStartup,
+ CLSID_MSH264DecoderMFT, CODECAPI_AVDecNumWorkerThreads, IMF2DBuffer, IMFActivate, IMFDXGIBuffer,
+ IMFDXGIDeviceManager, IMFMediaType, IMFSample, IMFTransform, MF_E_NOTACCEPTING,
+ MF_E_TRANSFORM_NEED_MORE_INPUT, MF_E_TRANSFORM_STREAM_CHANGE, MF_LOW_LATENCY, MF_MT_FRAME_SIZE,
+ MF_MT_MAJOR_TYPE, MF_MT_SUBTYPE, MF_MT_VIDEO_NOMINAL_RANGE, MF_MT_YUV_MATRIX, MF_SA_D3D11_AWARE,
+ MF_VERSION, MFCreateDXGIDeviceManager, MFCreateMediaType, MFCreateMemoryBuffer, MFCreateSample,
+ MFMediaType_Video, MFNominalRange_0_255, MFSTARTUP_NOSOCKET, MFStartup,
MFT_CATEGORY_VIDEO_DECODER, MFT_ENUM_FLAG_LOCALMFT, MFT_ENUM_FLAG_SORTANDFILTER,
MFT_ENUM_FLAG_SYNCMFT, MFT_MESSAGE_COMMAND_FLUSH, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING,
- MFT_MESSAGE_NOTIFY_END_STREAMING, MFT_MESSAGE_NOTIFY_START_OF_STREAM, MFT_OUTPUT_DATA_BUFFER,
- MFT_OUTPUT_STREAM_PROVIDES_SAMPLES, MFT_REGISTER_TYPE_INFO, MFTEnumEx, MFVideoFormat_AV1,
- MFVideoFormat_H264, MFVideoFormat_HEVC, MFVideoFormat_NV12, MFVideoTransferMatrix_BT601,
+ MFT_MESSAGE_NOTIFY_END_STREAMING, MFT_MESSAGE_NOTIFY_START_OF_STREAM,
+ MFT_MESSAGE_SET_D3D_MANAGER, MFT_OUTPUT_DATA_BUFFER, MFT_OUTPUT_STREAM_PROVIDES_SAMPLES,
+ MFT_REGISTER_TYPE_INFO, MFTEnumEx, MFVideoFormat_AV1, MFVideoFormat_H264, MFVideoFormat_HEVC,
+ MFVideoFormat_NV12, MFVideoTransferMatrix_BT601,
},
System::Com::{
CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, CoCreateInstance, CoInitializeEx, CoTaskMemFree,
@@ -118,6 +129,36 @@ pub(crate) fn create_mft(
}
}
+/// A Direct3D 11 device on the default GPU, shared with Media Foundation.
+fn gpu_device_manager() -> windows::core::Result<IMFDXGIDeviceManager> {
+ // SAFETY: plain Direct3D and Media Foundation calls with owned values.
+ unsafe {
+ let mut device: Option<ID3D11Device> = None;
+ D3D11CreateDevice(
+ None,
+ D3D_DRIVER_TYPE_HARDWARE,
+ Default::default(),
+ D3D11_CREATE_DEVICE_VIDEO_SUPPORT,
+ None,
+ D3D11_SDK_VERSION,
+ Some(&mut device),
+ None,
+ None,
+ )?;
+ let device = device.ok_or_else(|| windows::core::Error::from(E_FAIL))?;
+ // The decoder uses the device from its own threads.
+ if let Ok(multithread) = device.cast::<ID3D11Multithread>() {
+ let _ = multithread.SetMultithreadProtected(true);
+ }
+ let mut token = 0u32;
+ let mut manager: Option<IMFDXGIDeviceManager> = None;
+ MFCreateDXGIDeviceManager(&mut token, &mut manager)?;
+ let manager = manager.ok_or_else(|| windows::core::Error::from(E_FAIL))?;
+ manager.ResetDevice(&device, token)?;
+ Ok(manager)
+ }
+}
+
/// Whether an access unit buffer contains a coded slice/frame for `codec`.
fn has_slice(codec: &str, data: &[u8]) -> bool {
match codec {
@@ -130,6 +171,10 @@ fn has_slice(codec: &str, data: &[u8]) -> bool {
pub struct Decoder {
codec: String,
transform: IMFTransform,
+ /// The GPU the decoder uses, kept alive with it; `None` when it decodes in software.
+ gpu: Option<IMFDXGIDeviceManager>,
+ /// Where GPU pictures are copied to be read, reused while the size stays.
+ staging: Option<(ID3D11Texture2D, D3D11_TEXTURE2D_DESC)>,
/// The decoded size, as the output type says (padded).
coded: (u32, u32),
provides_samples: bool,
@@ -164,6 +209,26 @@ impl Decoder {
/// Creates a low-latency video decoder for `"h264"`, `"h265"`, or `"av1"`.
pub fn new_for_codec(codec: &str) -> windows::core::Result<Decoder> {
+ Self::create(codec, false)
+ }
+
+ /// As [`Decoder::new_for_codec`], decoding on the GPU when the PC can
+ /// (4K H.264 takes about 60 ms a frame in software, far over a 30 fps
+ /// frame's 33 ms). Falls back to software when it can't, here or later
+ /// in the stream (see [`Decoder::decode`]).
+ pub fn new_gpu(codec: &str) -> windows::core::Result<Decoder> {
+ Self::create(codec, true).or_else(|e| {
+ tracing::info!(error = %e, "no GPU video decoding; decoding in software");
+ Self::create(codec, false)
+ })
+ }
+
+ /// Whether this decoder runs on the GPU.
+ pub fn on_gpu(&self) -> bool {
+ self.gpu.is_some()
+ }
+
+ fn create(codec: &str, gpu: bool) -> windows::core::Result<Decoder> {
// SAFETY: plain COM and Media Foundation calls with owned values.
unsafe {
ensure_mf_started()?;
@@ -182,10 +247,25 @@ impl Decoder {
),
};
// One picture in, one out: no reordering delay, single worker thread.
- if let Ok(attributes) = transform.GetAttributes() {
+ let attributes = transform.GetAttributes().ok();
+ if let Some(attributes) = &attributes {
let _ = attributes.SetUINT32(&MF_LOW_LATENCY, 1);
let _ = attributes.SetUINT32(&CODECAPI_AVDecNumWorkerThreads, 1);
}
+ // The GPU, before any media type is set.
+ let gpu = if gpu {
+ let aware = attributes
+ .as_ref()
+ .is_some_and(|a| a.GetUINT32(&MF_SA_D3D11_AWARE).is_ok_and(|v| v != 0));
+ if !aware {
+ return Err(windows::core::Error::from(E_FAIL));
+ }
+ let manager = gpu_device_manager()?;
+ transform.ProcessMessage(MFT_MESSAGE_SET_D3D_MANAGER, manager.as_raw() as usize)?;
+ Some(manager)
+ } else {
+ None
+ };
let input = MFCreateMediaType()?;
input.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?;
input.SetGUID(&MF_MT_SUBTYPE, &subtype)?;
@@ -193,6 +273,8 @@ impl Decoder {
let mut decoder = Decoder {
codec: codec.to_owned(),
transform,
+ gpu,
+ staging: None,
coded: (0, 0),
provides_samples: false,
output_size: 0,
@@ -241,7 +323,22 @@ impl Decoder {
}
/// Decodes one access unit (Annex B); returns the pictures it completed.
+ /// If decoding on the GPU fails, the decoder switches to software for the
+ /// rest of the stream and returns the error: the caller starts over from
+ /// the next keyframe, as after any decoding error.
pub fn decode(&mut self, data: &[u8], time_us: u64) -> windows::core::Result<Vec<Picture>> {
+ let result = self.decode_on(data, time_us);
+ if result.is_err()
+ && self.gpu.is_some()
+ && let Ok(software) = Self::create(&self.codec, false)
+ {
+ tracing::info!("GPU video decoding failed; decoding in software");
+ *self = software;
+ }
+ result
+ }
+
+ fn decode_on(&mut self, data: &[u8], time_us: u64) -> windows::core::Result<Vec<Picture>> {
let expects_picture = has_slice(&self.codec, data);
let sample = sample_of(data, time_us)?;
let mut pictures = Vec::new();
@@ -318,11 +415,20 @@ impl Decoder {
}
}
- fn picture(&self, sample: &IMFSample) -> windows::core::Result<Picture> {
+ fn picture(&mut self, sample: &IMFSample) -> windows::core::Result<Picture> {
let (width, height) = self.coded;
// SAFETY: the buffer is locked while it's read and unlocked after.
unsafe {
let buffer = sample.ConvertToContiguousBuffer()?;
+ if self.gpu.is_some()
+ && let Ok(dxgi) = buffer.cast::<IMFDXGIBuffer>()
+ {
+ let mut texture: Option<ID3D11Texture2D> = None;
+ dxgi.GetResource(&ID3D11Texture2D::IID, std::ptr::from_mut(&mut texture).cast())?;
+ let texture = texture.ok_or_else(|| windows::core::Error::from(E_FAIL))?;
+ let index = dxgi.GetSubresourceIndex()?;
+ return self.read_texture(&texture, index, width, height);
+ }
if let Ok(two_d) = buffer.cast::<IMF2DBuffer>() {
let mut scan0 = std::ptr::null_mut();
let mut pitch = 0;
@@ -346,6 +452,60 @@ impl Decoder {
}
}
+impl Decoder {
+ /// Copies a decoded picture out of GPU memory. Windows' own path makes a
+ /// new staging texture and waits on it for every picture (about 14 ms at
+ /// 4K); one kept for the stream takes a few.
+ fn read_texture(
+ &mut self,
+ texture: &ID3D11Texture2D,
+ index: u32,
+ width: u32,
+ height: u32,
+ ) -> windows::core::Result<Picture> {
+ // SAFETY: Direct3D calls on the decoder's own device; the staging
+ // texture is mapped while it's read and unmapped after.
+ unsafe {
+ let device = texture.GetDevice()?;
+ let context = device.GetImmediateContext()?;
+ let mut source = D3D11_TEXTURE2D_DESC::default();
+ texture.GetDesc(&mut source);
+ let reuse = self.staging.as_ref().is_some_and(|(_, d)| {
+ (d.Width, d.Height, d.Format) == (source.Width, source.Height, source.Format)
+ });
+ if !reuse {
+ let desc = D3D11_TEXTURE2D_DESC {
+ Width: source.Width,
+ Height: source.Height,
+ MipLevels: 1,
+ ArraySize: 1,
+ Format: source.Format,
+ SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 },
+ Usage: D3D11_USAGE_STAGING,
+ BindFlags: 0,
+ CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32,
+ MiscFlags: 0,
+ };
+ let mut staging: Option<ID3D11Texture2D> = None;
+ device.CreateTexture2D(&desc, None, Some(&mut staging))?;
+ let staging = staging.ok_or_else(|| windows::core::Error::from(E_FAIL))?;
+ self.staging = Some((staging, desc));
+ }
+ let (staging, _) = self.staging.as_ref().ok_or_else(|| windows::core::Error::from(E_FAIL))?;
+ context.CopySubresourceRegion(staging, 0, 0, 0, 0, texture, index, None);
+ let mut mapped = D3D11_MAPPED_SUBRESOURCE::default();
+ context.Map(staging, 0, D3D11_MAP_READ, 0, Some(&mut mapped))?;
+ // NV12: the chroma plane follows the texture's full height of luma.
+ let stride = mapped.RowPitch as usize;
+ let plane_rows = source.Height as usize;
+ let len = stride * plane_rows * 3 / 2;
+ let nv12 = std::slice::from_raw_parts(mapped.pData.cast::<u8>(), len).to_vec();
+ context.Unmap(staging, 0);
+ Ok(Picture { width, height, stride, plane_rows, nv12, colors: self.colors })
+ }
+ }
+}
+
fn sample_of(data: &[u8], time_us: u64) -> windows::core::Result<IMFSample> {
// SAFETY: the buffer is at least `data.len()` long and locked while
// it's written.
@@ -364,15 +524,32 @@ fn sample_of(data: &[u8], time_us: u64) -> windows::core::Result<IMFSample> {
}
}
-/// NV12 (in the picture's own colors) to 32-bit BGRX, the
-/// visible `width` x `height`, directly on the decoder thread without per-frame OS thread-spawn overhead.
+/// Pixels from which a frame is converted on several cores: 4K takes about
+/// 24 ms on one core, most of a 30 fps frame's 33 ms, while 720p is quick
+/// enough on the calling thread.
+const PARALLEL_FROM_PIXELS: usize = 1920 * 1080;
+
+/// NV12 (in the picture's own colors) to 32-bit BGRX, the visible
+/// `width` x `height`. Large frames are split into bands of rows, one per
+/// core (scoped threads cost microseconds, against milliseconds of work).
pub fn to_bgrx(picture: &Picture, width: u32, height: u32) -> Vec<u8> {
let (w, h) = (width.min(picture.width) as usize, height.min(picture.height) as usize);
let mut out = vec![0u8; w * h * 4];
if w == 0 || h == 0 {
return out;
}
- convert_rows(picture, w, 0, &mut out);
+ let cores = std::thread::available_parallelism().map_or(1, |n| n.get()).min(8);
+ if w * h < PARALLEL_FROM_PIXELS || cores < 2 {
+ convert_rows(picture, w, 0, &mut out);
+ return out;
+ }
+ // Bands of an even number of rows, so each starts on a chroma row.
+ let band = h.div_ceil(cores).next_multiple_of(2);
+ std::thread::scope(|scope| {
+ for (i, rows) in out.chunks_mut(band * w * 4).enumerate() {
+ scope.spawn(move || convert_rows(picture, w, i * band, rows));
+ }
+ });
out
}
@@ -404,6 +581,57 @@ fn clamp(v: i32) -> u8 {
mod tests {
use super::*;
+ #[test]
+ #[ignore]
+ fn bench_to_bgrx_4k() {
+ let (w, h) = (3840u32, 2160u32);
+ let stride = w as usize;
+ let picture = Picture {
+ width: w,
+ height: h,
+ stride,
+ plane_rows: h as usize,
+ nv12: (0..stride * h as usize * 3 / 2).map(|i| (i % 251) as u8).collect(),
+ colors: Colors::default(),
+ };
+ let start = std::time::Instant::now();
+ let n = 30;
+ for _ in 0..n {
+ std::hint::black_box(to_bgrx(&picture, w, h));
+ }
+ eprintln!("BENCH to_bgrx 4K: {:.1} ms/frame", start.elapsed().as_secs_f64() * 1000.0 / n as f64);
+ }
+
+ #[test]
+ #[ignore]
+ fn bench_decode_4k() {
+ let path = std::env::var("BENCH_4K_H264").expect("BENCH_4K_H264");
+ let stream = std::fs::read(path).unwrap();
+ let units = access_units(&stream);
+ for gpu in [false, true] {
+ let mut decoder =
+ if gpu { Decoder::new_gpu("h264").unwrap() } else { Decoder::new_for_codec("h264").unwrap() };
+ let on_gpu = decoder.on_gpu();
+ let start = std::time::Instant::now();
+ let mut pictures = 0;
+ let mut first_out = None;
+ for (i, u) in units.iter().enumerate() {
+ let got = decoder.decode(u, i as u64 * 33_333).unwrap().len();
+ if got > 0 && first_out.is_none() {
+ first_out = Some(i);
+ }
+ pictures += got;
+ }
+ eprintln!(
+ "BENCH decode 4K gpu={on_gpu}: {} units, {} pictures, first picture after unit {:?}, {:.1} ms/unit",
+ units.len(),
+ pictures,
+ first_out,
+ start.elapsed().as_secs_f64() * 1000.0 / units.len() as f64
+ );
+ }
+ }
+
#[test]
fn the_stream_says_which_colors() {
// 3 frames tagged BT.601, full range (as some phones encode).
@@ -458,6 +686,42 @@ mod tests {
.collect()
}
+ #[test]
+ fn the_gpu_decodes_the_same_pictures_as_software() {
+ let units = access_units(include_bytes!("testdata/testsrc.h264"));
+ let Ok(mut gpu) = Decoder::new_gpu("h264") else { return };
+ if !gpu.on_gpu() {
+ // No usable GPU here (CI runners): nothing to compare.
+ return;
+ }
+ let mut software = Decoder::new().unwrap();
+ for (i, unit) in units.iter().enumerate() {
+ let a = software.decode(unit, i as u64 * 33_333).unwrap();
+ let b = gpu.decode(unit, i as u64 * 33_333).unwrap();
+ assert_eq!(a.len(), b.len(), "unit {i}");
+ for (a, b) in a.iter().zip(&b) {
+ assert_eq!((a.width, a.height), (b.width, b.height));
+ // H.264 decoding is exact: same visible luma and chroma, row by row.
+ let (w, h) = (a.width as usize, a.height as usize);
+ for row in 0..h {
+ assert_eq!(
+ a.nv12[row * a.stride..][..w],
+ b.nv12[row * b.stride..][..w],
+ "unit {i} luma row {row}"
+ );
+ }
+ for row in 0..h / 2 {
+ let (ca, cb) = (a.stride * a.plane_rows, b.stride * b.plane_rows);
+ assert_eq!(
+ a.nv12[ca + row * a.stride..][..w],
+ b.nv12[cb + row * b.stride..][..w],
+ "unit {i} chroma row {row}"
+ );
+ }
+ }
+ }
+ }
+
#[test]
fn a_stream_decodes_picture_by_picture() {
// 10 frames of FFmpeg's test pattern, 320x240, a keyframe every 5,