Your Android phone and your Windows PC, working as one. Local-first, end-to-end encrypted, no account.
Webcam: decode 4K on the GPU and convert it on several cores
Bee142cd9153d7c
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 5f09830..8d2b970 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -14,6 +14,11 @@ change things that need a step from you, and its notes say so.except Phone as webcam, which needs Windows 11.- Android: changing the theme in Settings no longer jumps back to Home([#8](https://github.com/thtbee/nectarlink/issues/8)).+- **Phone as webcam in 4K is smooth.** The PC decoded and converted each 4K+ frame in about 47 ms, over the 33 ms a frame has at 30 fps, so frames were+ dropped. It now decodes on the GPU when there is one and converts on several+ cores, about 19 ms a frame+ ([#6](https://github.com/thtbee/nectarlink/issues/6)).## 0.1.3 (2026-10-15)diff --git a/desktop/app/Cargo.toml b/desktop/app/Cargo.tomlindex e744f09..8d115ce 100644--- a/desktop/app/Cargo.toml+++ b/desktop/app/Cargo.toml@@ -61,7 +61,11 @@ windows = { workspace = true, features = ["UI_Notifications","Win32_Foundation","Win32_UI_Input_XboxController",+ "Win32_Graphics_Direct3D",+ "Win32_Graphics_Direct3D11","Win32_Graphics_Dwm",+ "Win32_Graphics_Dxgi",+ "Win32_Graphics_Dxgi_Common","Win32_Graphics_Gdi","Win32_Graphics_Imaging","Win32_Storage_CloudFilters",diff --git a/desktop/app/src/webcam.rs b/desktop/app/src/webcam.rsindex a257e54..ab656eb 100644--- a/desktop/app/src/webcam.rs+++ b/desktop/app/src/webcam.rs@@ -636,7 +636,9 @@ fn mirror_bgrx_in_place(bgrx: &mut [u8], width: u32, height: u32) {}fn decode(device: DeviceId, items: &Receiver<Item>, waiting: &AtomicUsize) {- let mut decoder = match Decoder::new() {+ // On the GPU when there is one: 4K takes about 24 ms a frame in software+ // and 11 ms on a GPU, against 33 ms per frame at 30 fps.+ let mut decoder = match Decoder::new_gpu("h264") {Ok(decoder) => decoder,Err(e) => {tracing::error!(error = %e, "no H.264 decoder for webcam");@@ -648,6 +650,7 @@ fn decode(device: DeviceId, items: &Receiver<Item>, waiting: &AtomicUsize) {return;}};+ tracing::info!(gpu = decoder.on_gpu(), "webcam decoder ready");let keyframe_please = || {if let Some(node) = core_host::node() {core_host::spawn(async move { node.webcam_keyframe(device).await });diff --git a/desktop/app/src/win/h264.rs b/desktop/app/src/win/h264.rsindex e108f74..2f94607 100644--- a/desktop/app/src/win/h264.rs+++ b/desktop/app/src/win/h264.rs@@ -8,17 +8,28 @@ use std::{mem::ManuallyDrop, sync::Mutex, time::Duration};use windows::{Win32::{Foundation::E_FAIL,+ Graphics::{+ Direct3D::D3D_DRIVER_TYPE_HARDWARE,+ Direct3D11::{+ D3D11_CPU_ACCESS_READ, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_MAP_READ,+ D3D11_MAPPED_SUBRESOURCE, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, D3D11_USAGE_STAGING,+ D3D11CreateDevice, ID3D11Device, ID3D11Multithread, ID3D11Texture2D,+ },+ Dxgi::Common::DXGI_SAMPLE_DESC,+ },Media::MediaFoundation::{- CLSID_MSH264DecoderMFT, CODECAPI_AVDecNumWorkerThreads, IMF2DBuffer, IMFActivate, IMFMediaType,- IMFSample, IMFTransform, MF_E_NOTACCEPTING, MF_E_TRANSFORM_NEED_MORE_INPUT,- MF_E_TRANSFORM_STREAM_CHANGE, MF_LOW_LATENCY, MF_MT_FRAME_SIZE, MF_MT_MAJOR_TYPE, MF_MT_SUBTYPE,- MF_MT_VIDEO_NOMINAL_RANGE, MF_MT_YUV_MATRIX, MF_VERSION, MFCreateMediaType, MFCreateMemoryBuffer,- MFCreateSample, MFMediaType_Video, MFNominalRange_0_255, MFSTARTUP_NOSOCKET, MFStartup,+ CLSID_MSH264DecoderMFT, CODECAPI_AVDecNumWorkerThreads, IMF2DBuffer, IMFActivate, IMFDXGIBuffer,+ IMFDXGIDeviceManager, IMFMediaType, IMFSample, IMFTransform, MF_E_NOTACCEPTING,+ MF_E_TRANSFORM_NEED_MORE_INPUT, MF_E_TRANSFORM_STREAM_CHANGE, MF_LOW_LATENCY, MF_MT_FRAME_SIZE,+ MF_MT_MAJOR_TYPE, MF_MT_SUBTYPE, MF_MT_VIDEO_NOMINAL_RANGE, MF_MT_YUV_MATRIX, MF_SA_D3D11_AWARE,+ MF_VERSION, MFCreateDXGIDeviceManager, MFCreateMediaType, MFCreateMemoryBuffer, MFCreateSample,+ MFMediaType_Video, MFNominalRange_0_255, MFSTARTUP_NOSOCKET, MFStartup,MFT_CATEGORY_VIDEO_DECODER, MFT_ENUM_FLAG_LOCALMFT, MFT_ENUM_FLAG_SORTANDFILTER,MFT_ENUM_FLAG_SYNCMFT, MFT_MESSAGE_COMMAND_FLUSH, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING,- MFT_MESSAGE_NOTIFY_END_STREAMING, MFT_MESSAGE_NOTIFY_START_OF_STREAM, MFT_OUTPUT_DATA_BUFFER,- MFT_OUTPUT_STREAM_PROVIDES_SAMPLES, MFT_REGISTER_TYPE_INFO, MFTEnumEx, MFVideoFormat_AV1,- MFVideoFormat_H264, MFVideoFormat_HEVC, MFVideoFormat_NV12, MFVideoTransferMatrix_BT601,+ MFT_MESSAGE_NOTIFY_END_STREAMING, MFT_MESSAGE_NOTIFY_START_OF_STREAM,+ MFT_MESSAGE_SET_D3D_MANAGER, MFT_OUTPUT_DATA_BUFFER, MFT_OUTPUT_STREAM_PROVIDES_SAMPLES,+ MFT_REGISTER_TYPE_INFO, MFTEnumEx, MFVideoFormat_AV1, MFVideoFormat_H264, MFVideoFormat_HEVC,+ MFVideoFormat_NV12, MFVideoTransferMatrix_BT601,},System::Com::{CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, CoCreateInstance, CoInitializeEx, CoTaskMemFree,@@ -118,6 +129,36 @@ pub(crate) fn create_mft(}}+/// A Direct3D 11 device on the default GPU, shared with Media Foundation.+fn gpu_device_manager() -> windows::core::Result<IMFDXGIDeviceManager> {+ // SAFETY: plain Direct3D and Media Foundation calls with owned values.+ unsafe {+ let mut device: Option<ID3D11Device> = None;+ D3D11CreateDevice(+ None,+ D3D_DRIVER_TYPE_HARDWARE,+ Default::default(),+ D3D11_CREATE_DEVICE_VIDEO_SUPPORT,+ None,+ D3D11_SDK_VERSION,+ Some(&mut device),+ None,+ None,+ )?;+ let device = device.ok_or_else(|| windows::core::Error::from(E_FAIL))?;+ // The decoder uses the device from its own threads.+ if let Ok(multithread) = device.cast::<ID3D11Multithread>() {+ let _ = multithread.SetMultithreadProtected(true);+ }+ let mut token = 0u32;+ let mut manager: Option<IMFDXGIDeviceManager> = None;+ MFCreateDXGIDeviceManager(&mut token, &mut manager)?;+ let manager = manager.ok_or_else(|| windows::core::Error::from(E_FAIL))?;+ manager.ResetDevice(&device, token)?;+ Ok(manager)+ }+}+/// Whether an access unit buffer contains a coded slice/frame for `codec`.fn has_slice(codec: &str, data: &[u8]) -> bool {match codec {@@ -130,6 +171,10 @@ fn has_slice(codec: &str, data: &[u8]) -> bool {pub struct Decoder {codec: String,transform: IMFTransform,+ /// The GPU the decoder uses, kept alive with it; `None` when it decodes in software.+ gpu: Option<IMFDXGIDeviceManager>,+ /// Where GPU pictures are copied to be read, reused while the size stays.+ staging: Option<(ID3D11Texture2D, D3D11_TEXTURE2D_DESC)>,/// The decoded size, as the output type says (padded).coded: (u32, u32),provides_samples: bool,@@ -164,6 +209,26 @@ impl Decoder {/// Creates a low-latency video decoder for `"h264"`, `"h265"`, or `"av1"`.pub fn new_for_codec(codec: &str) -> windows::core::Result<Decoder> {+ Self::create(codec, false)+ }++ /// As [`Decoder::new_for_codec`], decoding on the GPU when the PC can+ /// (4K H.264 takes about 60 ms a frame in software, far over a 30 fps+ /// frame's 33 ms). Falls back to software when it can't, here or later+ /// in the stream (see [`Decoder::decode`]).+ pub fn new_gpu(codec: &str) -> windows::core::Result<Decoder> {+ Self::create(codec, true).or_else(|e| {+ tracing::info!(error = %e, "no GPU video decoding; decoding in software");+ Self::create(codec, false)+ })+ }++ /// Whether this decoder runs on the GPU.+ pub fn on_gpu(&self) -> bool {+ self.gpu.is_some()+ }++ fn create(codec: &str, gpu: bool) -> windows::core::Result<Decoder> {// SAFETY: plain COM and Media Foundation calls with owned values.unsafe {ensure_mf_started()?;@@ -182,10 +247,25 @@ impl Decoder {),};// One picture in, one out: no reordering delay, single worker thread.- if let Ok(attributes) = transform.GetAttributes() {+ let attributes = transform.GetAttributes().ok();+ if let Some(attributes) = &attributes {let _ = attributes.SetUINT32(&MF_LOW_LATENCY, 1);let _ = attributes.SetUINT32(&CODECAPI_AVDecNumWorkerThreads, 1);}+ // The GPU, before any media type is set.+ let gpu = if gpu {+ let aware = attributes+ .as_ref()+ .is_some_and(|a| a.GetUINT32(&MF_SA_D3D11_AWARE).is_ok_and(|v| v != 0));+ if !aware {+ return Err(windows::core::Error::from(E_FAIL));+ }+ let manager = gpu_device_manager()?;+ transform.ProcessMessage(MFT_MESSAGE_SET_D3D_MANAGER, manager.as_raw() as usize)?;+ Some(manager)+ } else {+ None+ };let input = MFCreateMediaType()?;input.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?;input.SetGUID(&MF_MT_SUBTYPE, &subtype)?;@@ -193,6 +273,8 @@ impl Decoder {let mut decoder = Decoder {codec: codec.to_owned(),transform,+ gpu,+ staging: None,coded: (0, 0),provides_samples: false,output_size: 0,@@ -241,7 +323,22 @@ impl Decoder {}/// Decodes one access unit (Annex B); returns the pictures it completed.+ /// If decoding on the GPU fails, the decoder switches to software for the+ /// rest of the stream and returns the error: the caller starts over from+ /// the next keyframe, as after any decoding error.pub fn decode(&mut self, data: &[u8], time_us: u64) -> windows::core::Result<Vec<Picture>> {+ let result = self.decode_on(data, time_us);+ if result.is_err()+ && self.gpu.is_some()+ && let Ok(software) = Self::create(&self.codec, false)+ {+ tracing::info!("GPU video decoding failed; decoding in software");+ *self = software;+ }+ result+ }++ fn decode_on(&mut self, data: &[u8], time_us: u64) -> windows::core::Result<Vec<Picture>> {let expects_picture = has_slice(&self.codec, data);let sample = sample_of(data, time_us)?;let mut pictures = Vec::new();@@ -318,11 +415,20 @@ impl Decoder {}}- fn picture(&self, sample: &IMFSample) -> windows::core::Result<Picture> {+ fn picture(&mut self, sample: &IMFSample) -> windows::core::Result<Picture> {let (width, height) = self.coded;// SAFETY: the buffer is locked while it's read and unlocked after.unsafe {let buffer = sample.ConvertToContiguousBuffer()?;+ if self.gpu.is_some()+ && let Ok(dxgi) = buffer.cast::<IMFDXGIBuffer>()+ {+ let mut texture: Option<ID3D11Texture2D> = None;+ dxgi.GetResource(&ID3D11Texture2D::IID, std::ptr::from_mut(&mut texture).cast())?;+ let texture = texture.ok_or_else(|| windows::core::Error::from(E_FAIL))?;+ let index = dxgi.GetSubresourceIndex()?;+ return self.read_texture(&texture, index, width, height);+ }if let Ok(two_d) = buffer.cast::<IMF2DBuffer>() {let mut scan0 = std::ptr::null_mut();let mut pitch = 0;@@ -346,6 +452,60 @@ impl Decoder {}}+impl Decoder {+ /// Copies a decoded picture out of GPU memory. Windows' own path makes a+ /// new staging texture and waits on it for every picture (about 14 ms at+ /// 4K); one kept for the stream takes a few.+ fn read_texture(+ &mut self,+ texture: &ID3D11Texture2D,+ index: u32,+ width: u32,+ height: u32,+ ) -> windows::core::Result<Picture> {+ // SAFETY: Direct3D calls on the decoder's own device; the staging+ // texture is mapped while it's read and unmapped after.+ unsafe {+ let device = texture.GetDevice()?;+ let context = device.GetImmediateContext()?;+ let mut source = D3D11_TEXTURE2D_DESC::default();+ texture.GetDesc(&mut source);+ let reuse = self.staging.as_ref().is_some_and(|(_, d)| {+ (d.Width, d.Height, d.Format) == (source.Width, source.Height, source.Format)+ });+ if !reuse {+ let desc = D3D11_TEXTURE2D_DESC {+ Width: source.Width,+ Height: source.Height,+ MipLevels: 1,+ ArraySize: 1,+ Format: source.Format,+ SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 },+ Usage: D3D11_USAGE_STAGING,+ BindFlags: 0,+ CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32,+ MiscFlags: 0,+ };+ let mut staging: Option<ID3D11Texture2D> = None;+ device.CreateTexture2D(&desc, None, Some(&mut staging))?;+ let staging = staging.ok_or_else(|| windows::core::Error::from(E_FAIL))?;+ self.staging = Some((staging, desc));+ }+ let (staging, _) = self.staging.as_ref().ok_or_else(|| windows::core::Error::from(E_FAIL))?;+ context.CopySubresourceRegion(staging, 0, 0, 0, 0, texture, index, None);+ let mut mapped = D3D11_MAPPED_SUBRESOURCE::default();+ context.Map(staging, 0, D3D11_MAP_READ, 0, Some(&mut mapped))?;+ // NV12: the chroma plane follows the texture's full height of luma.+ let stride = mapped.RowPitch as usize;+ let plane_rows = source.Height as usize;+ let len = stride * plane_rows * 3 / 2;+ let nv12 = std::slice::from_raw_parts(mapped.pData.cast::<u8>(), len).to_vec();+ context.Unmap(staging, 0);+ Ok(Picture { width, height, stride, plane_rows, nv12, colors: self.colors })+ }+ }+}+fn sample_of(data: &[u8], time_us: u64) -> windows::core::Result<IMFSample> {// SAFETY: the buffer is at least `data.len()` long and locked while// it's written.@@ -364,15 +524,32 @@ fn sample_of(data: &[u8], time_us: u64) -> windows::core::Result<IMFSample> {}}-/// NV12 (in the picture's own colors) to 32-bit BGRX, the-/// visible `width` x `height`, directly on the decoder thread without per-frame OS thread-spawn overhead.+/// Pixels from which a frame is converted on several cores: 4K takes about+/// 24 ms on one core, most of a 30 fps frame's 33 ms, while 720p is quick+/// enough on the calling thread.+const PARALLEL_FROM_PIXELS: usize = 1920 * 1080;++/// NV12 (in the picture's own colors) to 32-bit BGRX, the visible+/// `width` x `height`. Large frames are split into bands of rows, one per+/// core (scoped threads cost microseconds, against milliseconds of work).pub fn to_bgrx(picture: &Picture, width: u32, height: u32) -> Vec<u8> {let (w, h) = (width.min(picture.width) as usize, height.min(picture.height) as usize);let mut out = vec![0u8; w * h * 4];if w == 0 || h == 0 {return out;}- convert_rows(picture, w, 0, &mut out);+ let cores = std::thread::available_parallelism().map_or(1, |n| n.get()).min(8);+ if w * h < PARALLEL_FROM_PIXELS || cores < 2 {+ convert_rows(picture, w, 0, &mut out);+ return out;+ }+ // Bands of an even number of rows, so each starts on a chroma row.+ let band = h.div_ceil(cores).next_multiple_of(2);+ std::thread::scope(|scope| {+ for (i, rows) in out.chunks_mut(band * w * 4).enumerate() {+ scope.spawn(move || convert_rows(picture, w, i * band, rows));+ }+ });out}@@ -404,6 +581,57 @@ fn clamp(v: i32) -> u8 {mod tests {use super::*;+ #[test]+ #[ignore]+ fn bench_to_bgrx_4k() {+ let (w, h) = (3840u32, 2160u32);+ let stride = w as usize;+ let picture = Picture {+ width: w,+ height: h,+ stride,+ plane_rows: h as usize,+ nv12: (0..stride * h as usize * 3 / 2).map(|i| (i % 251) as u8).collect(),+ colors: Colors::default(),+ };+ let start = std::time::Instant::now();+ let n = 30;+ for _ in 0..n {+ std::hint::black_box(to_bgrx(&picture, w, h));+ }+ eprintln!("BENCH to_bgrx 4K: {:.1} ms/frame", start.elapsed().as_secs_f64() * 1000.0 / n as f64);+ }++ #[test]+ #[ignore]+ fn bench_decode_4k() {+ let path = std::env::var("BENCH_4K_H264").expect("BENCH_4K_H264");+ let stream = std::fs::read(path).unwrap();+ let units = access_units(&stream);+ for gpu in [false, true] {+ let mut decoder =+ if gpu { Decoder::new_gpu("h264").unwrap() } else { Decoder::new_for_codec("h264").unwrap() };+ let on_gpu = decoder.on_gpu();+ let start = std::time::Instant::now();+ let mut pictures = 0;+ let mut first_out = None;+ for (i, u) in units.iter().enumerate() {+ let got = decoder.decode(u, i as u64 * 33_333).unwrap().len();+ if got > 0 && first_out.is_none() {+ first_out = Some(i);+ }+ pictures += got;+ }+ eprintln!(+ "BENCH decode 4K gpu={on_gpu}: {} units, {} pictures, first picture after unit {:?}, {:.1} ms/unit",+ units.len(),+ pictures,+ first_out,+ start.elapsed().as_secs_f64() * 1000.0 / units.len() as f64+ );+ }+ }+#[test]fn the_stream_says_which_colors() {// 3 frames tagged BT.601, full range (as some phones encode).@@ -458,6 +686,42 @@ mod tests {.collect()}+ #[test]+ fn the_gpu_decodes_the_same_pictures_as_software() {+ let units = access_units(include_bytes!("testdata/testsrc.h264"));+ let Ok(mut gpu) = Decoder::new_gpu("h264") else { return };+ if !gpu.on_gpu() {+ // No usable GPU here (CI runners): nothing to compare.+ return;+ }+ let mut software = Decoder::new().unwrap();+ for (i, unit) in units.iter().enumerate() {+ let a = software.decode(unit, i as u64 * 33_333).unwrap();+ let b = gpu.decode(unit, i as u64 * 33_333).unwrap();+ assert_eq!(a.len(), b.len(), "unit {i}");+ for (a, b) in a.iter().zip(&b) {+ assert_eq!((a.width, a.height), (b.width, b.height));+ // H.264 decoding is exact: same visible luma and chroma, row by row.+ let (w, h) = (a.width as usize, a.height as usize);+ for row in 0..h {+ assert_eq!(+ a.nv12[row * a.stride..][..w],+ b.nv12[row * b.stride..][..w],+ "unit {i} luma row {row}"+ );+ }+ for row in 0..h / 2 {+ let (ca, cb) = (a.stride * a.plane_rows, b.stride * b.plane_rows);+ assert_eq!(+ a.nv12[ca + row * a.stride..][..w],+ b.nv12[cb + row * b.stride..][..w],+ "unit {i} chroma row {row}"+ );+ }+ }+ }+ }+#[test]fn a_stream_decodes_picture_by_picture() {// 10 frames of FFmpeg's test pattern, 320x240, a keyframe every 5,