Your Android phone and your Windows PC, working as one. Local-first, end-to-end encrypted, no account.
Negotiate HEVC for webcam streams
AS8148ea0e06649491
diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.ktsindex 13345c8..b3052a1 100644--- a/android/app/build.gradle.kts+++ b/android/app/build.gradle.kts@@ -25,6 +25,11 @@ android {targetSdk = 37versionCode = appVersionCodeversionName = appVersion+ manifestPlaceholders["appLabel"] = "@string/app_name"+ if (providers.gradleProperty("nectarlink.hevcTest").orNull == "true") {+ applicationId = "app.nectarlink.hevctest"+ manifestPlaceholders["appLabel"] = "NectarLink HEVC test"+ }}// Only the languages the app offers (res/xml/locales_config.xml) go into the APK: adiff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xmlindex 34a7162..14d36f9 100644--- a/android/app/src/main/AndroidManifest.xml+++ b/android/app/src/main/AndroidManifest.xml@@ -133,7 +133,7 @@android:dataExtractionRules="@xml/data_extraction_rules"android:fullBackupContent="@xml/backup_rules"android:icon="@mipmap/ic_launcher"- android:label="@string/app_name"+ android:label="${appLabel}"android:localeConfig="@xml/locales_config"android:supportsRtl="true"android:theme="@style/Theme.Nectarlink">diff --git a/android/app/src/main/kotlin/app/nectarlink/android/core/Core.kt b/android/app/src/main/kotlin/app/nectarlink/android/core/Core.ktindex 1b62be2..0c4eaf0 100644--- a/android/app/src/main/kotlin/app/nectarlink/android/core/Core.kt+++ b/android/app/src/main/kotlin/app/nectarlink/android/core/Core.kt@@ -1053,6 +1053,12 @@ class Core(context: Context, private val scope: CoroutineScope) : EventListenerreturn node.mirrorOpenAudio(pcId)}+ /** Accepted codecs from the connected PC; an unknown receiver gets H.264. */+ suspend fun webcamCodecs(pcId: String): List<String> {+ startJob?.join()+ return runCatching { node?.webcamCodecs(pcId) }.getOrNull() ?: listOf("h264")+ }+/** Opens this phone's camera stream to a PC (`webcam`), or emits a message on failure. */suspend fun webcamOpenOrMessage(pcId: String): MirrorStream? {startJob?.join()diff --git a/android/app/src/main/kotlin/app/nectarlink/android/webcam/CameraEncoder.kt b/android/app/src/main/kotlin/app/nectarlink/android/webcam/CameraEncoder.ktindex 2c8dc44..f7e1bad 100644--- a/android/app/src/main/kotlin/app/nectarlink/android/webcam/CameraEncoder.kt+++ b/android/app/src/main/kotlin/app/nectarlink/android/webcam/CameraEncoder.kt@@ -30,7 +30,7 @@ import androidx.lifecycle.LifecycleOwnerimport app.nectarlink.core.MirrorSendResultimport app.nectarlink.core.MirrorStreamimport app.nectarlink.core.VideoPacketKind-import app.nectarlink.core.webcamConfig+import app.nectarlink.core.webcamConfigCodecimport java.util.concurrent.Executorsimport java.util.concurrent.atomic.AtomicBoolean@@ -49,7 +49,7 @@ data class CameraLiveState(/*** Captures frames from the phone's front or back camera with CameraX and- * encodes them into low-latency Annex B H.264 access units with hardware+ * encodes them into low-latency Annex B H.264 or negotiated HEVC access units with hardware* [MediaCodec] (`COLOR_FormatSurface`) for `docs/protocol/webcam.md`.** Switching cameras (`front` / `back`) or resolution (`720p` / `1080p` /@@ -64,6 +64,7 @@ internal class CameraEncoder(initialHeight: Int,private val fps: Int,initialCamera: String,+ private val preferredCodec: String = "h264",initialZoomRatio: Float = 1f,private val onStateChanged: (CameraLiveState) -> Unit,private val onEnded: () -> Unit,@@ -88,6 +89,7 @@ internal class CameraEncoder(@Volatile private var running = true@Volatile private var keyframeWanted = false+ @Volatile private var hevcFailed = false@Volatile private var cameraFacing: String = if (initialCamera == "front") "front" else "back"/** Whether the camera in use and the encoder can both do 4K. */@@ -394,9 +396,32 @@ internal class CameraEncoder(outHeight: Int,camera: String,): ActiveCodec? {+ if (preferredCodec == "h265" && !hevcFailed) {+ val name = hevcEncoder(outWidth, outHeight, fps)+ if (name != null) {+ startCodec(srcWidth, srcHeight, outWidth, outHeight, camera, "h265", name)?.let { return it }+ }+ Log.i(TAG, "HEVC unavailable for ${outWidth}x${outHeight}; using H.264")+ }+ return startCodec(srcWidth, srcHeight, outWidth, outHeight, camera, "h264", null)+ }++ private fun startCodec(+ srcWidth: Int,+ srcHeight: Int,+ outWidth: Int,+ outHeight: Int,+ camera: String,+ codecName: String,+ encoderName: String?,+ ): ActiveCodec? {+ val mime = if (codecName == "h265") MediaFormat.MIMETYPE_VIDEO_HEVC else MediaFormat.MIMETYPE_VIDEO_AVCval bitrate = bitrateForHeight(outHeight)+ var codec: MediaCodec? = null+ var codecSurface: Surface? = null+ var pipe: GlSurfacePipe? = nullreturn runCatching {- val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, outWidth, outHeight).apply {+ val format = MediaFormat.createVideoFormat(mime, outWidth, outHeight).apply {setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)setInteger(MediaFormat.KEY_BIT_RATE, bitrate)setInteger(MediaFormat.KEY_FRAME_RATE, fps)@@ -413,25 +438,36 @@ internal class CameraEncoder(setInteger(MediaFormat.KEY_COLOR_STANDARD, MediaFormat.COLOR_STANDARD_BT709)setInteger(MediaFormat.KEY_COLOR_RANGE, MediaFormat.COLOR_RANGE_LIMITED)setInteger(MediaFormat.KEY_COLOR_TRANSFER, MediaFormat.COLOR_TRANSFER_SDR_VIDEO)+ if (codecName == "h265") {+ setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.HEVCProfileMain)+ }}- val codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC)- codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)- val codecSurface = codec.createInputSurface()- codec.start()- val pipe = GlSurfacePipe(+ val startedCodec = if (encoderName != null) MediaCodec.createByCodecName(encoderName) else MediaCodec.createEncoderByType(mime)+ codec = startedCodec+ startedCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)+ val surface = startedCodec.createInputSurface()+ codecSurface = surface+ startedCodec.start()+ val startedPipe = GlSurfacePipe(srcWidth = srcWidth,srcHeight = srcHeight,dstWidth = outWidth,dstHeight = outHeight,- encoderInputSurface = codecSurface,+ encoderInputSurface = surface,initialRotationDegrees = currentSensorRotation,initialMirror = false,).also {it.setTransform(currentSensorRotation, currentDisplayRotation)}- ActiveCodec(codec, codecSurface, pipe, outWidth, outHeight, camera).also { it.startDrain() }+ pipe = startedPipe+ Log.i(TAG, "webcam encoder=$codecName component=${startedCodec.name} size=${outWidth}x${outHeight} bitrate=$bitrate")+ ActiveCodec(startedCodec, surface, startedPipe, outWidth, outHeight, camera, codecName).also { it.startDrain() }}.onFailure { e ->- Log.w(TAG, "can't start H.264 encoder for ${outWidth}x${outHeight}", e)+ runCatching { pipe?.release() }+ runCatching { codec?.stop() }+ runCatching { codec?.release() }+ runCatching { codecSurface?.release() }+ Log.w(TAG, "can't start $codecName encoder for ${outWidth}x${outHeight}", e)}.getOrNull()}@@ -442,6 +478,7 @@ internal class CameraEncoder(val width: Int,val height: Int,val camera: String,+ val codecName: String,) {private val active = AtomicBoolean(true)private var drainThread: Thread? = null@@ -461,7 +498,7 @@ internal class CameraEncoder(private fun drainLoop() {val maxHeight = if (supports4k) 2160u else 1080u- val cfgBytes = webcamConfig(width.toUInt(), height.toUInt(), camera, fps.toUInt(), maxHeight)+ val cfgBytes = webcamConfigCodec(codecName, width.toUInt(), height.toUInt(), camera, fps.toUInt(), maxHeight)if (stream.send(VideoPacketKind.CONFIG, 0u, cfgBytes) == MirrorSendResult.CLOSED) {return@@ -493,7 +530,7 @@ internal class CameraEncoder(}if (bytes.isEmpty()) continueval key = info.flags and MediaCodec.BUFFER_FLAG_KEY_FRAME != 0- val payload = ensureAudAndParameters(bytes, parameters, key)+ val payload = ensureAudAndParameters(bytes, parameters, key, codecName)val result = stream.send(if (key) VideoPacketKind.KEYFRAME else VideoPacketKind.FRAME,info.presentationTimeUs.toULong(),@@ -511,6 +548,10 @@ internal class CameraEncoder(} catch (e: Exception) {if (active.get() && running) {Log.w(TAG, "webcam encoder drain interrupted", e)+ if (codecName == "h265") {+ hevcFailed = true+ main.post { if (running && activeCodec === this) bindCamera() }+ }}}}@@ -523,6 +564,23 @@ internal class CameraEncoder(/** Annex B Access Unit Delimiter NAL (`00 00 00 01 09 F0`). */internal val AUD_NAL = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x09, 0xF0.toByte())+ internal val HEVC_AUD_NAL = byteArrayOf(0, 0, 0, 1, 0x46, 1, 0x50)++ /** Only offer hardware HEVC encoding at the actual requested size/rate. */+ private fun hevcEncoder(width: Int, height: Int, fps: Int): String? = runCatching {+ MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.firstOrNull { info ->+ info.isEncoder &&+ (if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) info.isHardwareAccelerated+ else !info.name.startsWith("OMX.google.") && !info.name.startsWith("c2.android.")) &&+ info.supportedTypes.any { it.equals(MediaFormat.MIMETYPE_VIDEO_HEVC, ignoreCase = true) } &&+ runCatching {+ val caps = info.getCapabilitiesForType(MediaFormat.MIMETYPE_VIDEO_HEVC)+ caps.colorFormats.contains(MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) &&+ caps.profileLevels.any { it.profile == MediaCodecInfo.CodecProfileLevel.HEVCProfileMain } &&+ caps.videoCapabilities?.areSizeAndRateSupported(width, height, fps.toDouble()) == true+ }.getOrDefault(false)+ }?.name+ }.getOrNull()fun widthForHeight(height: Int): Int = when (height) {2160 -> 3840@@ -657,48 +715,48 @@ internal class CameraEncoder(}/**- * Guarantees each H.264 access unit starts with an Annex B Access- * Unit Delimiter (`00 00 00 01 09 F0`) and that keyframes include- * the cached SPS/PPS (`parameters`) right after the AUD so the PC's+ * Guarantees each access unit starts with the codec's Annex B Access+ * Unit Delimiter and keyframes include the cached parameter sets+ * (SPS/PPS for AVC, VPS/SPS/PPS for HEVC) right after the AUD so the PC's* Media Foundation decoder flushes every frame immediately.*/internal fun ensureAudAndParameters(bytes: ByteArray,parameters: ByteArray,isKeyframe: Boolean,+ codecName: String = "h264",): ByteArray {- val stripped = stripLeadingAud(bytes)+ val aud = if (codecName == "h265") HEVC_AUD_NAL else AUD_NAL+ val stripped = stripLeadingAud(bytes, codecName)return if (isKeyframe && parameters.isNotEmpty()) {- val cleanParams = stripLeadingAud(parameters)- ByteArray(AUD_NAL.size + cleanParams.size + stripped.size).also { out ->- System.arraycopy(AUD_NAL, 0, out, 0, AUD_NAL.size)- System.arraycopy(cleanParams, 0, out, AUD_NAL.size, cleanParams.size)- System.arraycopy(stripped, 0, out, AUD_NAL.size + cleanParams.size, stripped.size)+ val cleanParams = stripLeadingAud(parameters, codecName)+ ByteArray(aud.size + cleanParams.size + stripped.size).also { out ->+ System.arraycopy(aud, 0, out, 0, aud.size)+ System.arraycopy(cleanParams, 0, out, aud.size, cleanParams.size)+ System.arraycopy(stripped, 0, out, aud.size + cleanParams.size, stripped.size)}} else {- ByteArray(AUD_NAL.size + stripped.size).also { out ->- System.arraycopy(AUD_NAL, 0, out, 0, AUD_NAL.size)- System.arraycopy(stripped, 0, out, AUD_NAL.size, stripped.size)+ ByteArray(aud.size + stripped.size).also { out ->+ System.arraycopy(aud, 0, out, 0, aud.size)+ System.arraycopy(stripped, 0, out, aud.size, stripped.size)}}}- private fun stripLeadingAud(data: ByteArray): ByteArray {- if (data.size >= 6 &&- data[0] == 0.toByte() && data[1] == 0.toByte() &&- data[2] == 0.toByte() && data[3] == 1.toByte() &&- (data[4].toInt() and 0x1F) == 9- ) {- return data.copyOfRange(6, data.size)+ private fun stripLeadingAud(data: ByteArray, codecName: String): ByteArray {+ fun startCode(at: Int): Int {+ if (at + 3 > data.size || data[at] != 0.toByte() || data[at + 1] != 0.toByte()) return 0+ if (data[at + 2] == 1.toByte()) return 3+ return if (at + 4 <= data.size && data[at + 2] == 0.toByte() && data[at + 3] == 1.toByte()) 4 else 0}- if (data.size >= 5 &&- data[0] == 0.toByte() && data[1] == 0.toByte() &&- data[2] == 1.toByte() &&- (data[3].toInt() and 0x1F) == 9- ) {- return data.copyOfRange(5, data.size)+ val prefix = startCode(0)+ if (prefix == 0 || prefix >= data.size) return data+ val type = if (codecName == "h265") (data[prefix].toInt() ushr 1) and 0x3F else data[prefix].toInt() and 0x1F+ if (type != if (codecName == "h265") 35 else 9) return data+ for (at in prefix + 1 until data.size) {+ if (startCode(at) != 0) return data.copyOfRange(at, data.size)}- return data+ return ByteArray(0)}}}diff --git a/android/app/src/main/kotlin/app/nectarlink/android/webcam/WebcamService.kt b/android/app/src/main/kotlin/app/nectarlink/android/webcam/WebcamService.ktindex f9003c2..3850e7f 100644--- a/android/app/src/main/kotlin/app/nectarlink/android/webcam/WebcamService.kt+++ b/android/app/src/main/kotlin/app/nectarlink/android/webcam/WebcamService.kt@@ -64,7 +64,7 @@ internal fun resolutionOf(width: Int, height: Int): Int {}/**- * Streams the phone's camera as H.264 to a paired PC (`docs/protocol/webcam.md`).+ * Streams the phone's camera as negotiated HEVC or H.264 to a paired PC (`docs/protocol/webcam.md`).* Runs as a `camera` foreground service with an ongoing notification and a* Stop action so streaming continues if the user switches apps or locks the* screen.@@ -160,6 +160,7 @@ class WebcamService : LifecycleService() {main.post { stopStreaming() }return@launch}+ val preferredCodec = if ("h265" in core.webcamCodecs(targetPc)) "h265" else "h264"main.post {if (pcId != targetPc || current !== this@WebcamService) {runCatching { stream.end() }@@ -172,6 +173,7 @@ class WebcamService : LifecycleService() {initialHeight = normHeight,fps = fps,initialCamera = normCamera,+ preferredCodec = preferredCodec,initialZoomRatio = _session.value.zoomRatio,onStateChanged = { live ->_session.update { s ->diff --git a/android/app/src/test/kotlin/app/nectarlink/android/webcam/CameraEncoderTest.kt b/android/app/src/test/kotlin/app/nectarlink/android/webcam/CameraEncoderTest.ktindex f8d49f6..2aa9539 100644--- a/android/app/src/test/kotlin/app/nectarlink/android/webcam/CameraEncoderTest.kt+++ b/android/app/src/test/kotlin/app/nectarlink/android/webcam/CameraEncoderTest.kt@@ -6,6 +6,19 @@ import org.junit.Assert.assertEqualsimport org.junit.Testclass CameraEncoderTest {+ @Test+ fun `HEVC keyframes carry VPS SPS PPS and a codec specific AUD`() {+ val parameters = byteArrayOf(0, 0, 0, 1, 0x40, 1, 0x11, 0, 0, 0, 1, 0x42, 1, 0x22, 0, 0, 0, 1, 0x44, 1, 0x33)+ val idr = byteArrayOf(0, 0, 0, 1, 0x26, 1, 0x55)+ val aud = CameraEncoder.HEVC_AUD_NAL+ assertArrayEquals(aud + parameters + idr, CameraEncoder.ensureAudAndParameters(idr, parameters, true, "h265"))+ assertArrayEquals(aud + parameters + idr, CameraEncoder.ensureAudAndParameters(aud + idr, aud + parameters, true, "h265"))+ val shortAud = byteArrayOf(0, 0, 1, 0x46, 1, 0x50)+ assertArrayEquals(aud + idr, CameraEncoder.ensureAudAndParameters(shortAud + idr, parameters, false, "h265"))+ // Parameter-set headers must survive; HEVC types use different bits from AVC.+ assertArrayEquals(aud + parameters, CameraEncoder.ensureAudAndParameters(parameters, byteArrayOf(), false, "h265"))+ }+@Testfun `prepends AUD and SPS PPS to keyframes and avoids duplicate AUD`() {val spsPps = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0x00, 0x00, 0x00, 0x01, 0x68, 0x11)diff --git a/core/nectarlink-core/src/lib.rs b/core/nectarlink-core/src/lib.rsindex a5b5328..0a99790 100644--- a/core/nectarlink-core/src/lib.rs+++ b/core/nectarlink-core/src/lib.rs@@ -132,8 +132,8 @@ pub use nectarlink_protocol::{},webcam::{ADDON_VCAM as WEBCAM_ADDON_VCAM, CAMERA_BACK as WEBCAM_CAMERA_BACK,- CAMERA_FRONT as WEBCAM_CAMERA_FRONT, H264 as WEBCAM_H264, STREAM as WEBCAM_STREAM,- VIRTUAL as WEBCAM_VIRTUAL,+ CAMERA_FRONT as WEBCAM_CAMERA_FRONT, H264 as WEBCAM_H264, H265 as WEBCAM_H265,+ HEVC as WEBCAM_HEVC, STREAM as WEBCAM_STREAM, VIRTUAL as WEBCAM_VIRTUAL,},},pairing::PairingUri,diff --git a/core/nectarlink-core/src/node.rs b/core/nectarlink-core/src/node.rsindex ffa28de..2f61af1 100644--- a/core/nectarlink-core/src/node.rs+++ b/core/nectarlink-core/src/node.rs@@ -2561,6 +2561,19 @@ impl Node {// ---- Webcam (docs/protocol/webcam.md) ----+ /// Webcam codecs accepted by the connected receiver, in preference order.+ /// Older receivers advertise no HEVC capability and always use H.264.+ pub fn webcam_codecs(&self, peer: DeviceId) -> Result<Vec<String>> {+ let _ = self.connected(&peer)?;+ let record = self.shared.store.get_peer(&peer)?.ok_or(Error::NotPaired)?;+ let mut codecs = Vec::new();+ if record.caps.contains(crate::WEBCAM_HEVC) {+ codecs.push(crate::WEBCAM_H265.into());+ }+ codecs.push(crate::WEBCAM_H264.into());+ Ok(codecs)+ }+/// Asks a paired phone to start streaming its camera as a webcam./// Video then arrives through [`Platform::webcam_sink`](crate::Platform::webcam_sink).pub async fn webcam_start(&self, peer: DeviceId, options: crate::WebcamStart) -> Result<()> {@@ -2576,7 +2589,7 @@ impl Node {}}- /// Asks a phone for a fresh H.264 keyframe on the webcam stream.+ /// Asks a phone for a fresh keyframe on the webcam stream.pub async fn webcam_keyframe(&self, peer: DeviceId) {if let Ok(session) = self.connected(&peer) {crate::webcam::request_keyframe(&session).await;diff --git a/core/nectarlink-core/src/webcam.rs b/core/nectarlink-core/src/webcam.rsindex 80137fe..b27f1a3 100644--- a/core/nectarlink-core/src/webcam.rs+++ b/core/nectarlink-core/src/webcam.rs@@ -1,5 +1,5 @@// SPDX-License-Identifier: MPL-2.0-//! Phone as webcam (`docs/protocol/webcam.md`): a phone streams encoded H.264+//! Phone as webcam (`docs/protocol/webcam.md`): a phone streams encoded H.264 or HEVC//! video from its camera on a dedicated QUIC stream (`webcam/video`), either//! started directly from the phone's Webcam screen or after the phone's user//! accepts a PC's `webcam.start` request.@@ -23,7 +23,7 @@ pub const TOGGLE: &str = "webcam";pub trait WebcamSink: Send + Sync {/// The stream's format: first, and again whenever camera or resolution changes.fn config(&self, config: WebcamConfig);- /// Encoded video (Annex B H.264), `keyframe` when it starts fresh (SPS+PPS+IDR).+ /// Encoded Annex B video, `keyframe` when it starts fresh with parameter sets.fn packet(&self, keyframe: bool, time_us: u64, data: Vec<u8>);/// The stream ended (the phone stopped, or the connection dropped).fn ended(&self);@@ -67,10 +67,12 @@ pub(crate) async fn receive(shared: Arc<Shared>, peer: DeviceId, mut send: SendSif !supports || !shared.toggle_on(&peer, TOGGLE) {return refuse(send, recv);}+ let accepted = |config: &WebcamConfig| {+ config.is_valid()+ && (config.codec != webcam::H265 || shared.local_capabilities().iter().any(|c| c == webcam::HEVC))+ };let first = match read_video_packet(&mut recv).await {- Ok(Some(p)) if p.kind == PacketKind::Config => {- WebcamConfig::from_cbor(&p.data).ok().filter(WebcamConfig::is_valid)- }+ Ok(Some(p)) if p.kind == PacketKind::Config => WebcamConfig::from_cbor(&p.data).ok().filter(accepted),_ => None,};let Some(config) = first else {@@ -104,7 +106,7 @@ pub(crate) async fn receive(shared: Arc<Shared>, peer: DeviceId, mut send: SendS};match packet {Ok(Some(p)) if p.kind == PacketKind::Config => match WebcamConfig::from_cbor(&p.data) {- Ok(config) if config.is_valid() => sink.config(config),+ Ok(config) if accepted(&config) => sink.config(config),_ => {tracing::warn!("a phone sent a webcam format this PC can't decode");break;diff --git a/core/nectarlink-core/tests/two_devices.rs b/core/nectarlink-core/tests/two_devices.rsindex 09a3a87..ec32c36 100644--- a/core/nectarlink-core/tests/two_devices.rs+++ b/core/nectarlink-core/tests/two_devices.rs@@ -3329,12 +3329,28 @@ async fn phone_storage_lists_reads_ranges_writes_resumes_mutates_and_enforces_se#[tokio::test(flavor = "multi_thread")]async fn the_phone_camera_streams_to_the_pc_as_a_webcam() {+ webcam_round_trip(false).await;+}++#[tokio::test(flavor = "multi_thread")]+async fn the_phone_negotiates_hevc_only_with_an_advertising_receiver() {+ webcam_round_trip(true).await;+}++async fn webcam_round_trip(hevc: bool) {use nectarlink_core::{MirrorSend, PacketKind};- let mut pc = device_with("Desktop", DeviceKind::Desktop, &[nectarlink_core::WEBCAM_VIRTUAL]).await;+ let mut caps = vec![nectarlink_core::WEBCAM_VIRTUAL];+ if hevc {+ caps.push(nectarlink_core::WEBCAM_HEVC);+ }+ let mut pc = device_with("Desktop", DeviceKind::Desktop, &caps).await;let mut phone = device_with("Pixel", DeviceKind::Phone, &[nectarlink_core::WEBCAM_STREAM]).await;pair_qr(&mut pc, &mut phone).await;let (pc_id, phone_id) = (pc.node.device_id(), phone.node.device_id());+ let codecs = phone.node.webcam_codecs(pc_id).unwrap();+ assert_eq!(codecs, if hevc { vec!["h265", "h264"] } else { vec!["h264"] });+let options = nectarlink_core::WebcamStart {camera: nectarlink_core::WEBCAM_CAMERA_BACK.into(),width: 1920,@@ -3347,7 +3363,7 @@ async fn the_phone_camera_streams_to_the_pc_as_a_webcam() {let stream = Arc::new(with_timeout("webcam_open", phone.node.webcam_open(pc_id)).await.unwrap());let config = nectarlink_core::WebcamConfig {- codec: nectarlink_core::WEBCAM_H264.into(),+ codec: codecs[0].clone(),width: 1920,height: 1080,camera: nectarlink_core::WEBCAM_CAMERA_BACK.into(),diff --git a/core/nectarlink-ffi/src/lib.rs b/core/nectarlink-ffi/src/lib.rsindex d5d9f86..6ff426a 100644--- a/core/nectarlink-ffi/src/lib.rs+++ b/core/nectarlink-ffi/src/lib.rs@@ -1204,6 +1204,19 @@ pub fn webcam_config(width: u32, height: u32, camera: String, fps: u32, max_heigcore::WebcamConfig { codec: core::WEBCAM_H264.into(), width, height, camera, fps, max_height }.to_cbor()}+/// A webcam config for a negotiated codec; retain webcam_config for older callers.+#[uniffi::export]+pub fn webcam_config_codec(+ codec: String,+ width: u32,+ height: u32,+ camera: String,+ fps: u32,+ max_height: u32,+) -> Vec<u8> {+ core::WebcamConfig { codec, width, height, camera, fps, max_height }.to_cbor()+}+/// What a PC asked the phone's camera to stream (`docs/protocol/webcam.md`).#[derive(Debug, Clone, PartialEq, Eq, uniffi::Record)]pub struct WebcamOptions {@@ -4022,6 +4035,11 @@ impl NectarlinkNode {self.run(async move { Ok(node.mirror_input(id, core::MIRROR_SCREEN, input).await?) }).await}+ /// Preferred webcam codecs accepted by the connected PC, with H.264 as fallback.+ pub fn webcam_codecs(&self, pc_id: String) -> Result<Vec<String>> {+ Ok(self.node.webcam_codecs(parse_id(&pc_id)?)?)+ }+/// Opens this phone's camera stream to a paired PC (`docs/protocol/webcam.md`).pub async fn webcam_open(&self, pc_id: String) -> Result<Arc<MirrorStream>> {let id = parse_id(&pc_id)?;diff --git a/core/nectarlink-protocol/src/messages.rs b/core/nectarlink-protocol/src/messages.rsindex dfd43de..02e6c06 100644--- a/core/nectarlink-protocol/src/messages.rs+++ b/core/nectarlink-protocol/src/messages.rs@@ -1636,6 +1636,8 @@ pub mod webcam {pub const STREAM: &str = "camera.stream";/// Offered by PCs that receive and decode a phone's webcam stream.pub const VIRTUAL: &str = "camera.virtual";+ /// Receiver can decode HEVC webcam streams. Absence means H.264 only.+ pub const HEVC: &str = "camera.virtual.hevc";/// Offered by PCs where the Windows virtual camera COM add-on is registered.pub const ADDON_VCAM: &str = "addon.vcam";pub const SERVICE: &str = "webcam";@@ -1644,6 +1646,7 @@ pub mod webcam {/// Stream reset code: webcam stopped.pub const STOPPED: u32 = 12;pub const H264: &str = "h264";+ pub const H265: &str = "h265";pub const CAMERA_BACK: &str = "back";pub const CAMERA_FRONT: &str = "front";}@@ -1704,7 +1707,7 @@ impl WebcamStart {/// The format of a webcam stream (a `PacketKind::Config` packet's payload, in CBOR).#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]pub struct WebcamConfig {- /// `"h264"`.+ /// `"h264"` or `"h265"` (only when the receiver advertises webcam::HEVC).pub codec: String,pub width: u32,pub height: u32,@@ -1737,7 +1740,7 @@ impl WebcamConfig {}pub fn is_valid(&self) -> bool {- self.codec == webcam::H264+ matches!(self.codec.as_str(), webcam::H264 | webcam::H265)&& (1..=8192).contains(&self.width)&& (1..=8192).contains(&self.height)&& (self.camera == webcam::CAMERA_BACK || self.camera == webcam::CAMERA_FRONT)@@ -5195,6 +5198,9 @@ mod tests {};assert!(cfg.is_valid());assert_eq!(WebcamConfig::from_cbor(&cfg.to_cbor()).unwrap(), cfg);+ let hevc = WebcamConfig { codec: webcam::H265.into(), ..cfg.clone() };+ assert!(hevc.is_valid());+ assert_eq!(WebcamConfig::from_cbor(&hevc.to_cbor()).unwrap(), hevc);assert!(!WebcamConfig { codec: "vp8".into(), ..cfg.clone() }.is_valid());assert!(!WebcamConfig { width: 0, ..cfg }.is_valid());diff --git a/desktop/app/src/core_host.rs b/desktop/app/src/core_host.rsindex 423bf9b..8d3c1f9 100644--- a/desktop/app/src/core_host.rs+++ b/desktop/app/src/core_host.rs@@ -127,6 +127,9 @@ pub fn compute_capabilities(#[cfg(windows)]{caps.push(nectarlink_core::WEBCAM_VIRTUAL.into());+ if crate::webcam::supports_hevc() {+ caps.push(nectarlink_core::WEBCAM_HEVC.into());+ }caps.push(nectarlink_core::MEDIA_STREAM.into());}#[cfg(target_os = "linux")]diff --git a/desktop/app/src/webcam.rs b/desktop/app/src/webcam.rsindex 1a402c3..49ff480 100644--- a/desktop/app/src/webcam.rs+++ b/desktop/app/src/webcam.rs@@ -715,6 +715,22 @@ fn mirror_bgrx_in_place(bgrx: &mut [u8], width: u32, height: u32) {}}+#[cfg(windows)]+pub fn supports_hevc() -> bool {+ static SUPPORTED: OnceLock<bool> = OnceLock::new();+ *SUPPORTED.get_or_init(|| match Decoder::new_gpu("h265") {+ Ok(decoder) => {+ let supported = decoder.on_gpu();+ tracing::info!(supported, "HEVC webcam decoder probe");+ supported+ }+ Err(error) => {+ tracing::info!(%error, "HEVC webcam decoder unavailable; advertising H.264 only");+ false+ }+ })+}+fn decode(device: DeviceId, items: &Receiver<Item>, waiting: &AtomicUsize) {// On the GPU when there is one: 4K takes about 24 ms a frame in software// and 11 ms on a GPU, against 33 ms per frame at 30 fps.@@ -749,6 +765,18 @@ fn decode(device: DeviceId, items: &Receiver<Item>, waiting: &AtomicUsize) {while let Ok(item) = items.recv() {match item {Item::Config(config) => {+ if decoder.codec() != config.codec {+ match Decoder::new_gpu(&config.codec) {+ Ok(next) => {+ decoder = next;+ tracing::info!(codec = %config.codec, gpu = decoder.on_gpu(), "webcam decoder changed");+ }+ Err(error) => {+ tracing::error!(codec = %config.codec, %error, "negotiated webcam decoder unavailable");+ break;+ }+ }+ }if size != (0, 0) && size != (config.width, config.height) {decoder.flush();}diff --git a/desktop/app/src/win/h264.rs b/desktop/app/src/win/h264.rsindex 6dc92c1..6f50df9 100644--- a/desktop/app/src/win/h264.rs+++ b/desktop/app/src/win/h264.rs@@ -695,67 +695,78 @@ mod tests {let worker = std::thread::spawn(move || {use nectarlink_vcam::gpu::{GpuPublisher, GpuReader};use windows::Win32::Media::MediaFoundation::MFCreateDXGISurfaceBuffer;- let mut decoder = Decoder::new_gpu("h264").unwrap();- if !decoder.on_gpu() {- done.send(()).unwrap();- return;- }- let name = format!("Local\\NectarlinkDecoderPoolTest_{}", std::process::id());- let mut reader = GpuReader::new_with_mapping(&name).unwrap();- let mut publisher = GpuPublisher::new_with_mapping(&name).unwrap();- let mut output = None;- let mut count = 0;- for _ in 0..20 {- for unit in test_stream("h264") {- for frame in decoder.decode_frames(unit, count * 33_333).unwrap() {- let (texture, index) = frame.gpu_surface().unwrap().unwrap();- if output.is_none() {- unsafe {- let mut desc = D3D11_TEXTURE2D_DESC::default();- texture.GetDesc(&mut desc);- desc.ArraySize = 1;- desc.MipLevels = 1;- desc.BindFlags =- windows::Win32::Graphics::Direct3D11::D3D11_BIND_RENDER_TARGET.0 as u32;- let mut destination = None;- texture- .GetDevice()- .unwrap()- .CreateTexture2D(&desc, None, Some(&mut destination))- .unwrap();- output = Some(- MFCreateDXGISurfaceBuffer(- &ID3D11Texture2D::IID,- &destination.unwrap(),- 0,+ for codec in ["h264", "h265"] {+ if !installed(codec) {+ continue;+ }+ let mut decoder = Decoder::new_gpu(codec).unwrap();+ if !decoder.on_gpu() {+ continue;+ }+ let name = format!("Local\\NectarlinkDecoderPoolTest_{}_{}", std::process::id(), codec);+ let mut reader = GpuReader::new_with_mapping(&name).unwrap();+ let mut publisher = GpuPublisher::new_with_mapping(&name).unwrap();+ let mut output = None;+ let mut count = 0;+ for _ in 0..20 {+ for unit in test_stream(codec) {+ for frame in decoder+ .decode_frames(unit, count * 33_333)+ .unwrap_or_else(|error| panic!("{codec} after {count} frames: {error}"))+ {+ let (texture, index) = frame.gpu_surface().unwrap().unwrap();+ if output.is_none() {+ unsafe {+ let mut desc = D3D11_TEXTURE2D_DESC::default();+ texture.GetDesc(&mut desc);+ desc.ArraySize = 1;+ desc.MipLevels = 1;+ desc.BindFlags =+ windows::Win32::Graphics::Direct3D11::D3D11_BIND_RENDER_TARGET.0+ as u32;+ let mut destination = None;+ texture+ .GetDevice()+ .unwrap()+ .CreateTexture2D(&desc, None, Some(&mut destination))+ .unwrap();+ output = Some(+ MFCreateDXGISurfaceBuffer(+ &ID3D11Texture2D::IID,+ &destination.unwrap(),+ 0,+ false,+ )+ .unwrap(),+ );+ }+ }+ assert!(+ publisher+ .publish(+ &texture,+ index,+ &frame.sample,+ (frame.width, frame.height),false,+ false)- .unwrap(),- );- }+ .unwrap()+ );+ assert!(+ reader+ .write_nv12(output.as_ref().unwrap(), frame.width, frame.height)+ .unwrap()+ );+ count += 1;}- assert!(- publisher- .publish(- &texture,- index,- &frame.sample,- (frame.width, frame.height),- false,- false- )- .unwrap()- );- assert!(- reader.write_nv12(output.as_ref().unwrap(), frame.width, frame.height).unwrap()- );- count += 1;}}+ assert_eq!(count, 200);+ assert_eq!(publisher.stats(), Some((200, 0)));+ assert!(decoder.staging.is_none(), "publication must not download the decoded frames");+ eprintln!("{codec}: published {count} GPU frames with zero CPU downloads");}- assert_eq!(count, 200);- assert_eq!(publisher.stats(), Some((200, 0)));- assert!(decoder.staging.is_none(), "publication must not download the decoded frames");done.send(()).unwrap();});finished.recv_timeout(std::time::Duration::from_secs(30)).expect("decoder sample pool stalled");diff --git a/docs/protocol/webcam.md b/docs/protocol/webcam.mdindex 4aa6103..79f5570 100644--- a/docs/protocol/webcam.md+++ b/docs/protocol/webcam.md@@ -10,6 +10,7 @@ travel on a dedicated bidirectional QUIC stream (`{ svc: "webcam", op: "video",|---|---|---|| `camera.stream` | Phone | Streams a camera for webcam use (when `CAMERA` permission is granted) || `camera.virtual` | PC | Receives and decodes a phone's webcam stream |+| `camera.virtual.hevc` | PC | Also accepts H.265/HEVC webcam video; absence means H.264 only || `addon.vcam` | PC | The Windows virtual camera COM add-on is registered in `HKLM`, so apps (Teams, Meet, Zoom, Windows Camera) see `"NectarLink - <Phone name>"` |Both devices have a `webcam` per-device toggle (on by default). When turned off@@ -66,7 +67,7 @@ Stops the webcam stream. Either side may also stop by closing or resetting the{ t: "webcam.keyframe" }```-Asks the phone's hardware H.264 encoder for an immediate IDR keyframe+Asks the phone's hardware video encoder for an immediate keyframe(`PARAMETER_KEY_REQUEST_SYNC_FRAME`), sent when the PC decoder starts mid-streamor after a dropped packet.@@ -93,3 +94,25 @@ Followed by a sequence of framed video packets (`core/nectarlink-protocol/src/viThe phone's sender bounds its outgoing queue to a few frames (`QUEUE_FRAMES = 4`):if the network falls behind, queued frames are dropped and the encoder is askedfor a fresh keyframe rather than building up latency.++### Optional HEVC negotiation++An updated phone prefers `h265` only when the connected receiver advertises+`camera.virtual.hevc` and a hardware HEVC Main-profile surface encoder supports+the requested dimensions and frame rate. Otherwise it uses `h264`. Windows+advertises HEVC only when its installed Media Foundation decoder can initialize+with a GPU device manager. The phone falls back to H.264 if the HEVC encoder+cannot start, or reconfigures the existing stream to H.264 after a HEVC drain error.++For HEVC, `Config.codec` is `h265`, `Frame` and `Keyframe` contain Annex B HEVC+access units, and keyframes carry VPS, SPS and PPS. Each access unit starts with+an HEVC AUD (NAL type 35), instead of the H.264 AUD (type 9). The receiver switches+decoders when a Config changes codec. HEVC configs are rejected by receivers that+do not advertise the HEVC capability, including later Config packets in a stream.+Older phone and desktop releases continue using H.264 without changing version 1+framing or the existing start-request fields.++The initial HEVC implementation retains the same resolution, frame rate and target+bitrate as H.264. It establishes codec compatibility; it does not itself claim a+bandwidth or latency reduction. A lower HEVC target bitrate needs separate quality+and latency measurements.