Skip to content

Your Android phone and your Windows PC, working as one. Local-first, end-to-end encrypted, no account.

Negotiate HEVC for webcam streams

AS8148ea0e06649491
diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts
index 13345c8..b3052a1 100644
--- a/android/app/build.gradle.kts
+++ b/android/app/build.gradle.kts
@@ -25,6 +25,11 @@ android {
targetSdk = 37
versionCode = appVersionCode
versionName = appVersion
+ manifestPlaceholders["appLabel"] = "@string/app_name"
+ if (providers.gradleProperty("nectarlink.hevcTest").orNull == "true") {
+ applicationId = "app.nectarlink.hevctest"
+ manifestPlaceholders["appLabel"] = "NectarLink HEVC test"
+ }
}
// Only the languages the app offers (res/xml/locales_config.xml) go into the APK: a
diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml
index 34a7162..14d36f9 100644
--- a/android/app/src/main/AndroidManifest.xml
+++ b/android/app/src/main/AndroidManifest.xml
@@ -133,7 +133,7 @@
android:dataExtractionRules="@xml/data_extraction_rules"
android:fullBackupContent="@xml/backup_rules"
android:icon="@mipmap/ic_launcher"
- android:label="@string/app_name"
+ android:label="${appLabel}"
android:localeConfig="@xml/locales_config"
android:supportsRtl="true"
android:theme="@style/Theme.Nectarlink">
diff --git a/android/app/src/main/kotlin/app/nectarlink/android/core/Core.kt b/android/app/src/main/kotlin/app/nectarlink/android/core/Core.kt
index 1b62be2..0c4eaf0 100644
--- a/android/app/src/main/kotlin/app/nectarlink/android/core/Core.kt
+++ b/android/app/src/main/kotlin/app/nectarlink/android/core/Core.kt
@@ -1053,6 +1053,12 @@ class Core(context: Context, private val scope: CoroutineScope) : EventListener
return node.mirrorOpenAudio(pcId)
}
+ /** Accepted codecs from the connected PC; an unknown receiver gets H.264. */
+ suspend fun webcamCodecs(pcId: String): List<String> {
+ startJob?.join()
+ return runCatching { node?.webcamCodecs(pcId) }.getOrNull() ?: listOf("h264")
+ }
+
/** Opens this phone's camera stream to a PC (`webcam`), or emits a message on failure. */
suspend fun webcamOpenOrMessage(pcId: String): MirrorStream? {
startJob?.join()
diff --git a/android/app/src/main/kotlin/app/nectarlink/android/webcam/CameraEncoder.kt b/android/app/src/main/kotlin/app/nectarlink/android/webcam/CameraEncoder.kt
index 2c8dc44..f7e1bad 100644
--- a/android/app/src/main/kotlin/app/nectarlink/android/webcam/CameraEncoder.kt
+++ b/android/app/src/main/kotlin/app/nectarlink/android/webcam/CameraEncoder.kt
@@ -30,7 +30,7 @@ import androidx.lifecycle.LifecycleOwner
import app.nectarlink.core.MirrorSendResult
import app.nectarlink.core.MirrorStream
import app.nectarlink.core.VideoPacketKind
-import app.nectarlink.core.webcamConfig
+import app.nectarlink.core.webcamConfigCodec
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicBoolean
@@ -49,7 +49,7 @@ data class CameraLiveState(
/**
* Captures frames from the phone's front or back camera with CameraX and
- * encodes them into low-latency Annex B H.264 access units with hardware
+ * encodes them into low-latency Annex B H.264 or negotiated HEVC access units with hardware
* [MediaCodec] (`COLOR_FormatSurface`) for `docs/protocol/webcam.md`.
*
* Switching cameras (`front` / `back`) or resolution (`720p` / `1080p` /
@@ -64,6 +64,7 @@ internal class CameraEncoder(
initialHeight: Int,
private val fps: Int,
initialCamera: String,
+ private val preferredCodec: String = "h264",
initialZoomRatio: Float = 1f,
private val onStateChanged: (CameraLiveState) -> Unit,
private val onEnded: () -> Unit,
@@ -88,6 +89,7 @@ internal class CameraEncoder(
@Volatile private var running = true
@Volatile private var keyframeWanted = false
+ @Volatile private var hevcFailed = false
@Volatile private var cameraFacing: String = if (initialCamera == "front") "front" else "back"
/** Whether the camera in use and the encoder can both do 4K. */
@@ -394,9 +396,32 @@ internal class CameraEncoder(
outHeight: Int,
camera: String,
): ActiveCodec? {
+ if (preferredCodec == "h265" && !hevcFailed) {
+ val name = hevcEncoder(outWidth, outHeight, fps)
+ if (name != null) {
+ startCodec(srcWidth, srcHeight, outWidth, outHeight, camera, "h265", name)?.let { return it }
+ }
+ Log.i(TAG, "HEVC unavailable for ${outWidth}x${outHeight}; using H.264")
+ }
+ return startCodec(srcWidth, srcHeight, outWidth, outHeight, camera, "h264", null)
+ }
+
+ private fun startCodec(
+ srcWidth: Int,
+ srcHeight: Int,
+ outWidth: Int,
+ outHeight: Int,
+ camera: String,
+ codecName: String,
+ encoderName: String?,
+ ): ActiveCodec? {
+ val mime = if (codecName == "h265") MediaFormat.MIMETYPE_VIDEO_HEVC else MediaFormat.MIMETYPE_VIDEO_AVC
val bitrate = bitrateForHeight(outHeight)
+ var codec: MediaCodec? = null
+ var codecSurface: Surface? = null
+ var pipe: GlSurfacePipe? = null
return runCatching {
- val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, outWidth, outHeight).apply {
+ val format = MediaFormat.createVideoFormat(mime, outWidth, outHeight).apply {
setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)
setInteger(MediaFormat.KEY_BIT_RATE, bitrate)
setInteger(MediaFormat.KEY_FRAME_RATE, fps)
@@ -413,25 +438,36 @@ internal class CameraEncoder(
setInteger(MediaFormat.KEY_COLOR_STANDARD, MediaFormat.COLOR_STANDARD_BT709)
setInteger(MediaFormat.KEY_COLOR_RANGE, MediaFormat.COLOR_RANGE_LIMITED)
setInteger(MediaFormat.KEY_COLOR_TRANSFER, MediaFormat.COLOR_TRANSFER_SDR_VIDEO)
+ if (codecName == "h265") {
+ setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.HEVCProfileMain)
+ }
}
- val codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC)
- codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
- val codecSurface = codec.createInputSurface()
- codec.start()
- val pipe = GlSurfacePipe(
+ val startedCodec = if (encoderName != null) MediaCodec.createByCodecName(encoderName) else MediaCodec.createEncoderByType(mime)
+ codec = startedCodec
+ startedCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
+ val surface = startedCodec.createInputSurface()
+ codecSurface = surface
+ startedCodec.start()
+ val startedPipe = GlSurfacePipe(
srcWidth = srcWidth,
srcHeight = srcHeight,
dstWidth = outWidth,
dstHeight = outHeight,
- encoderInputSurface = codecSurface,
+ encoderInputSurface = surface,
initialRotationDegrees = currentSensorRotation,
initialMirror = false,
).also {
it.setTransform(currentSensorRotation, currentDisplayRotation)
}
- ActiveCodec(codec, codecSurface, pipe, outWidth, outHeight, camera).also { it.startDrain() }
+ pipe = startedPipe
+ Log.i(TAG, "webcam encoder=$codecName component=${startedCodec.name} size=${outWidth}x${outHeight} bitrate=$bitrate")
+ ActiveCodec(startedCodec, surface, startedPipe, outWidth, outHeight, camera, codecName).also { it.startDrain() }
}.onFailure { e ->
- Log.w(TAG, "can't start H.264 encoder for ${outWidth}x${outHeight}", e)
+ runCatching { pipe?.release() }
+ runCatching { codec?.stop() }
+ runCatching { codec?.release() }
+ runCatching { codecSurface?.release() }
+ Log.w(TAG, "can't start $codecName encoder for ${outWidth}x${outHeight}", e)
}.getOrNull()
}
@@ -442,6 +478,7 @@ internal class CameraEncoder(
val width: Int,
val height: Int,
val camera: String,
+ val codecName: String,
) {
private val active = AtomicBoolean(true)
private var drainThread: Thread? = null
@@ -461,7 +498,7 @@ internal class CameraEncoder(
private fun drainLoop() {
val maxHeight = if (supports4k) 2160u else 1080u
- val cfgBytes = webcamConfig(width.toUInt(), height.toUInt(), camera, fps.toUInt(), maxHeight)
+ val cfgBytes = webcamConfigCodec(codecName, width.toUInt(), height.toUInt(), camera, fps.toUInt(), maxHeight)
if (stream.send(VideoPacketKind.CONFIG, 0u, cfgBytes) == MirrorSendResult.CLOSED) {
return
@@ -493,7 +530,7 @@ internal class CameraEncoder(
}
if (bytes.isEmpty()) continue
val key = info.flags and MediaCodec.BUFFER_FLAG_KEY_FRAME != 0
- val payload = ensureAudAndParameters(bytes, parameters, key)
+ val payload = ensureAudAndParameters(bytes, parameters, key, codecName)
val result = stream.send(
if (key) VideoPacketKind.KEYFRAME else VideoPacketKind.FRAME,
info.presentationTimeUs.toULong(),
@@ -511,6 +548,10 @@ internal class CameraEncoder(
} catch (e: Exception) {
if (active.get() && running) {
Log.w(TAG, "webcam encoder drain interrupted", e)
+ if (codecName == "h265") {
+ hevcFailed = true
+ main.post { if (running && activeCodec === this) bindCamera() }
+ }
}
}
}
@@ -523,6 +564,23 @@ internal class CameraEncoder(
/** Annex B Access Unit Delimiter NAL (`00 00 00 01 09 F0`). */
internal val AUD_NAL = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x09, 0xF0.toByte())
+ internal val HEVC_AUD_NAL = byteArrayOf(0, 0, 0, 1, 0x46, 1, 0x50)
+
+ /** Only offer hardware HEVC encoding at the actual requested size/rate. */
+ private fun hevcEncoder(width: Int, height: Int, fps: Int): String? = runCatching {
+ MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.firstOrNull { info ->
+ info.isEncoder &&
+ (if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) info.isHardwareAccelerated
+ else !info.name.startsWith("OMX.google.") && !info.name.startsWith("c2.android.")) &&
+ info.supportedTypes.any { it.equals(MediaFormat.MIMETYPE_VIDEO_HEVC, ignoreCase = true) } &&
+ runCatching {
+ val caps = info.getCapabilitiesForType(MediaFormat.MIMETYPE_VIDEO_HEVC)
+ caps.colorFormats.contains(MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) &&
+ caps.profileLevels.any { it.profile == MediaCodecInfo.CodecProfileLevel.HEVCProfileMain } &&
+ caps.videoCapabilities?.areSizeAndRateSupported(width, height, fps.toDouble()) == true
+ }.getOrDefault(false)
+ }?.name
+ }.getOrNull()
fun widthForHeight(height: Int): Int = when (height) {
2160 -> 3840
@@ -657,48 +715,48 @@ internal class CameraEncoder(
}
/**
- * Guarantees each H.264 access unit starts with an Annex B Access
- * Unit Delimiter (`00 00 00 01 09 F0`) and that keyframes include
- * the cached SPS/PPS (`parameters`) right after the AUD so the PC's
+ * Guarantees each access unit starts with the codec's Annex B Access
+ * Unit Delimiter and keyframes include the cached parameter sets
+ * (SPS/PPS for AVC, VPS/SPS/PPS for HEVC) right after the AUD so the PC's
* Media Foundation decoder flushes every frame immediately.
*/
internal fun ensureAudAndParameters(
bytes: ByteArray,
parameters: ByteArray,
isKeyframe: Boolean,
+ codecName: String = "h264",
): ByteArray {
- val stripped = stripLeadingAud(bytes)
+ val aud = if (codecName == "h265") HEVC_AUD_NAL else AUD_NAL
+ val stripped = stripLeadingAud(bytes, codecName)
return if (isKeyframe && parameters.isNotEmpty()) {
- val cleanParams = stripLeadingAud(parameters)
- ByteArray(AUD_NAL.size + cleanParams.size + stripped.size).also { out ->
- System.arraycopy(AUD_NAL, 0, out, 0, AUD_NAL.size)
- System.arraycopy(cleanParams, 0, out, AUD_NAL.size, cleanParams.size)
- System.arraycopy(stripped, 0, out, AUD_NAL.size + cleanParams.size, stripped.size)
+ val cleanParams = stripLeadingAud(parameters, codecName)
+ ByteArray(aud.size + cleanParams.size + stripped.size).also { out ->
+ System.arraycopy(aud, 0, out, 0, aud.size)
+ System.arraycopy(cleanParams, 0, out, aud.size, cleanParams.size)
+ System.arraycopy(stripped, 0, out, aud.size + cleanParams.size, stripped.size)
}
} else {
- ByteArray(AUD_NAL.size + stripped.size).also { out ->
- System.arraycopy(AUD_NAL, 0, out, 0, AUD_NAL.size)
- System.arraycopy(stripped, 0, out, AUD_NAL.size, stripped.size)
+ ByteArray(aud.size + stripped.size).also { out ->
+ System.arraycopy(aud, 0, out, 0, aud.size)
+ System.arraycopy(stripped, 0, out, aud.size, stripped.size)
}
}
}
- private fun stripLeadingAud(data: ByteArray): ByteArray {
- if (data.size >= 6 &&
- data[0] == 0.toByte() && data[1] == 0.toByte() &&
- data[2] == 0.toByte() && data[3] == 1.toByte() &&
- (data[4].toInt() and 0x1F) == 9
- ) {
- return data.copyOfRange(6, data.size)
+ private fun stripLeadingAud(data: ByteArray, codecName: String): ByteArray {
+ fun startCode(at: Int): Int {
+ if (at + 3 > data.size || data[at] != 0.toByte() || data[at + 1] != 0.toByte()) return 0
+ if (data[at + 2] == 1.toByte()) return 3
+ return if (at + 4 <= data.size && data[at + 2] == 0.toByte() && data[at + 3] == 1.toByte()) 4 else 0
}
- if (data.size >= 5 &&
- data[0] == 0.toByte() && data[1] == 0.toByte() &&
- data[2] == 1.toByte() &&
- (data[3].toInt() and 0x1F) == 9
- ) {
- return data.copyOfRange(5, data.size)
+ val prefix = startCode(0)
+ if (prefix == 0 || prefix >= data.size) return data
+ val type = if (codecName == "h265") (data[prefix].toInt() ushr 1) and 0x3F else data[prefix].toInt() and 0x1F
+ if (type != if (codecName == "h265") 35 else 9) return data
+ for (at in prefix + 1 until data.size) {
+ if (startCode(at) != 0) return data.copyOfRange(at, data.size)
}
- return data
+ return ByteArray(0)
}
}
}
diff --git a/android/app/src/main/kotlin/app/nectarlink/android/webcam/WebcamService.kt b/android/app/src/main/kotlin/app/nectarlink/android/webcam/WebcamService.kt
index f9003c2..3850e7f 100644
--- a/android/app/src/main/kotlin/app/nectarlink/android/webcam/WebcamService.kt
+++ b/android/app/src/main/kotlin/app/nectarlink/android/webcam/WebcamService.kt
@@ -64,7 +64,7 @@ internal fun resolutionOf(width: Int, height: Int): Int {
}
/**
- * Streams the phone's camera as H.264 to a paired PC (`docs/protocol/webcam.md`).
+ * Streams the phone's camera as negotiated HEVC or H.264 to a paired PC (`docs/protocol/webcam.md`).
* Runs as a `camera` foreground service with an ongoing notification and a
* Stop action so streaming continues if the user switches apps or locks the
* screen.
@@ -160,6 +160,7 @@ class WebcamService : LifecycleService() {
main.post { stopStreaming() }
return@launch
}
+ val preferredCodec = if ("h265" in core.webcamCodecs(targetPc)) "h265" else "h264"
main.post {
if (pcId != targetPc || current !== this@WebcamService) {
runCatching { stream.end() }
@@ -172,6 +173,7 @@ class WebcamService : LifecycleService() {
initialHeight = normHeight,
fps = fps,
initialCamera = normCamera,
+ preferredCodec = preferredCodec,
initialZoomRatio = _session.value.zoomRatio,
onStateChanged = { live ->
_session.update { s ->
diff --git a/android/app/src/test/kotlin/app/nectarlink/android/webcam/CameraEncoderTest.kt b/android/app/src/test/kotlin/app/nectarlink/android/webcam/CameraEncoderTest.kt
index f8d49f6..2aa9539 100644
--- a/android/app/src/test/kotlin/app/nectarlink/android/webcam/CameraEncoderTest.kt
+++ b/android/app/src/test/kotlin/app/nectarlink/android/webcam/CameraEncoderTest.kt
@@ -6,6 +6,19 @@ import org.junit.Assert.assertEquals
import org.junit.Test
class CameraEncoderTest {
+ @Test
+ fun `HEVC keyframes carry VPS SPS PPS and a codec specific AUD`() {
+ val parameters = byteArrayOf(0, 0, 0, 1, 0x40, 1, 0x11, 0, 0, 0, 1, 0x42, 1, 0x22, 0, 0, 0, 1, 0x44, 1, 0x33)
+ val idr = byteArrayOf(0, 0, 0, 1, 0x26, 1, 0x55)
+ val aud = CameraEncoder.HEVC_AUD_NAL
+ assertArrayEquals(aud + parameters + idr, CameraEncoder.ensureAudAndParameters(idr, parameters, true, "h265"))
+ assertArrayEquals(aud + parameters + idr, CameraEncoder.ensureAudAndParameters(aud + idr, aud + parameters, true, "h265"))
+ val shortAud = byteArrayOf(0, 0, 1, 0x46, 1, 0x50)
+ assertArrayEquals(aud + idr, CameraEncoder.ensureAudAndParameters(shortAud + idr, parameters, false, "h265"))
+ // Parameter-set headers must survive; HEVC types use different bits from AVC.
+ assertArrayEquals(aud + parameters, CameraEncoder.ensureAudAndParameters(parameters, byteArrayOf(), false, "h265"))
+ }
+
@Test
fun `prepends AUD and SPS PPS to keyframes and avoids duplicate AUD`() {
val spsPps = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0x00, 0x00, 0x00, 0x01, 0x68, 0x11)
diff --git a/core/nectarlink-core/src/lib.rs b/core/nectarlink-core/src/lib.rs
index a5b5328..0a99790 100644
--- a/core/nectarlink-core/src/lib.rs
+++ b/core/nectarlink-core/src/lib.rs
@@ -132,8 +132,8 @@ pub use nectarlink_protocol::{
},
webcam::{
ADDON_VCAM as WEBCAM_ADDON_VCAM, CAMERA_BACK as WEBCAM_CAMERA_BACK,
- CAMERA_FRONT as WEBCAM_CAMERA_FRONT, H264 as WEBCAM_H264, STREAM as WEBCAM_STREAM,
- VIRTUAL as WEBCAM_VIRTUAL,
+ CAMERA_FRONT as WEBCAM_CAMERA_FRONT, H264 as WEBCAM_H264, H265 as WEBCAM_H265,
+ HEVC as WEBCAM_HEVC, STREAM as WEBCAM_STREAM, VIRTUAL as WEBCAM_VIRTUAL,
},
},
pairing::PairingUri,
diff --git a/core/nectarlink-core/src/node.rs b/core/nectarlink-core/src/node.rs
index ffa28de..2f61af1 100644
--- a/core/nectarlink-core/src/node.rs
+++ b/core/nectarlink-core/src/node.rs
@@ -2561,6 +2561,19 @@ impl Node {
// ---- Webcam (docs/protocol/webcam.md) ----
+ /// Webcam codecs accepted by the connected receiver, in preference order.
+ /// Older receivers advertise no HEVC capability and always use H.264.
+ pub fn webcam_codecs(&self, peer: DeviceId) -> Result<Vec<String>> {
+ let _ = self.connected(&peer)?;
+ let record = self.shared.store.get_peer(&peer)?.ok_or(Error::NotPaired)?;
+ let mut codecs = Vec::new();
+ if record.caps.contains(crate::WEBCAM_HEVC) {
+ codecs.push(crate::WEBCAM_H265.into());
+ }
+ codecs.push(crate::WEBCAM_H264.into());
+ Ok(codecs)
+ }
+
/// Asks a paired phone to start streaming its camera as a webcam.
/// Video then arrives through [`Platform::webcam_sink`](crate::Platform::webcam_sink).
pub async fn webcam_start(&self, peer: DeviceId, options: crate::WebcamStart) -> Result<()> {
@@ -2576,7 +2589,7 @@ impl Node {
}
}
- /// Asks a phone for a fresh H.264 keyframe on the webcam stream.
+ /// Asks a phone for a fresh keyframe on the webcam stream.
pub async fn webcam_keyframe(&self, peer: DeviceId) {
if let Ok(session) = self.connected(&peer) {
crate::webcam::request_keyframe(&session).await;
diff --git a/core/nectarlink-core/src/webcam.rs b/core/nectarlink-core/src/webcam.rs
index 80137fe..b27f1a3 100644
--- a/core/nectarlink-core/src/webcam.rs
+++ b/core/nectarlink-core/src/webcam.rs
@@ -1,5 +1,5 @@
// SPDX-License-Identifier: MPL-2.0
-//! Phone as webcam (`docs/protocol/webcam.md`): a phone streams encoded H.264
+//! Phone as webcam (`docs/protocol/webcam.md`): a phone streams encoded H.264 or HEVC
//! video from its camera on a dedicated QUIC stream (`webcam/video`), either
//! started directly from the phone's Webcam screen or after the phone's user
//! accepts a PC's `webcam.start` request.
@@ -23,7 +23,7 @@ pub const TOGGLE: &str = "webcam";
pub trait WebcamSink: Send + Sync {
/// The stream's format: first, and again whenever camera or resolution changes.
fn config(&self, config: WebcamConfig);
- /// Encoded video (Annex B H.264), `keyframe` when it starts fresh (SPS+PPS+IDR).
+ /// Encoded Annex B video, `keyframe` when it starts fresh with parameter sets.
fn packet(&self, keyframe: bool, time_us: u64, data: Vec<u8>);
/// The stream ended (the phone stopped, or the connection dropped).
fn ended(&self);
@@ -67,10 +67,12 @@ pub(crate) async fn receive(shared: Arc<Shared>, peer: DeviceId, mut send: SendS
if !supports || !shared.toggle_on(&peer, TOGGLE) {
return refuse(send, recv);
}
+ let accepted = |config: &WebcamConfig| {
+ config.is_valid()
+ && (config.codec != webcam::H265 || shared.local_capabilities().iter().any(|c| c == webcam::HEVC))
+ };
let first = match read_video_packet(&mut recv).await {
- Ok(Some(p)) if p.kind == PacketKind::Config => {
- WebcamConfig::from_cbor(&p.data).ok().filter(WebcamConfig::is_valid)
- }
+ Ok(Some(p)) if p.kind == PacketKind::Config => WebcamConfig::from_cbor(&p.data).ok().filter(accepted),
_ => None,
};
let Some(config) = first else {
@@ -104,7 +106,7 @@ pub(crate) async fn receive(shared: Arc<Shared>, peer: DeviceId, mut send: SendS
};
match packet {
Ok(Some(p)) if p.kind == PacketKind::Config => match WebcamConfig::from_cbor(&p.data) {
- Ok(config) if config.is_valid() => sink.config(config),
+ Ok(config) if accepted(&config) => sink.config(config),
_ => {
tracing::warn!("a phone sent a webcam format this PC can't decode");
break;
diff --git a/core/nectarlink-core/tests/two_devices.rs b/core/nectarlink-core/tests/two_devices.rs
index 09a3a87..ec32c36 100644
--- a/core/nectarlink-core/tests/two_devices.rs
+++ b/core/nectarlink-core/tests/two_devices.rs
@@ -3329,12 +3329,28 @@ async fn phone_storage_lists_reads_ranges_writes_resumes_mutates_and_enforces_se
#[tokio::test(flavor = "multi_thread")]
async fn the_phone_camera_streams_to_the_pc_as_a_webcam() {
+ webcam_round_trip(false).await;
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn the_phone_negotiates_hevc_only_with_an_advertising_receiver() {
+ webcam_round_trip(true).await;
+}
+
+async fn webcam_round_trip(hevc: bool) {
use nectarlink_core::{MirrorSend, PacketKind};
- let mut pc = device_with("Desktop", DeviceKind::Desktop, &[nectarlink_core::WEBCAM_VIRTUAL]).await;
+ let mut caps = vec![nectarlink_core::WEBCAM_VIRTUAL];
+ if hevc {
+ caps.push(nectarlink_core::WEBCAM_HEVC);
+ }
+ let mut pc = device_with("Desktop", DeviceKind::Desktop, &caps).await;
let mut phone = device_with("Pixel", DeviceKind::Phone, &[nectarlink_core::WEBCAM_STREAM]).await;
pair_qr(&mut pc, &mut phone).await;
let (pc_id, phone_id) = (pc.node.device_id(), phone.node.device_id());
+ let codecs = phone.node.webcam_codecs(pc_id).unwrap();
+ assert_eq!(codecs, if hevc { vec!["h265", "h264"] } else { vec!["h264"] });
+
let options = nectarlink_core::WebcamStart {
camera: nectarlink_core::WEBCAM_CAMERA_BACK.into(),
width: 1920,
@@ -3347,7 +3363,7 @@ async fn the_phone_camera_streams_to_the_pc_as_a_webcam() {
let stream = Arc::new(with_timeout("webcam_open", phone.node.webcam_open(pc_id)).await.unwrap());
let config = nectarlink_core::WebcamConfig {
- codec: nectarlink_core::WEBCAM_H264.into(),
+ codec: codecs[0].clone(),
width: 1920,
height: 1080,
camera: nectarlink_core::WEBCAM_CAMERA_BACK.into(),
diff --git a/core/nectarlink-ffi/src/lib.rs b/core/nectarlink-ffi/src/lib.rs
index d5d9f86..6ff426a 100644
--- a/core/nectarlink-ffi/src/lib.rs
+++ b/core/nectarlink-ffi/src/lib.rs
@@ -1204,6 +1204,19 @@ pub fn webcam_config(width: u32, height: u32, camera: String, fps: u32, max_heig
core::WebcamConfig { codec: core::WEBCAM_H264.into(), width, height, camera, fps, max_height }.to_cbor()
}
+/// A webcam config for a negotiated codec; retain webcam_config for older callers.
+#[uniffi::export]
+pub fn webcam_config_codec(
+ codec: String,
+ width: u32,
+ height: u32,
+ camera: String,
+ fps: u32,
+ max_height: u32,
+) -> Vec<u8> {
+ core::WebcamConfig { codec, width, height, camera, fps, max_height }.to_cbor()
+}
+
/// What a PC asked the phone's camera to stream (`docs/protocol/webcam.md`).
#[derive(Debug, Clone, PartialEq, Eq, uniffi::Record)]
pub struct WebcamOptions {
@@ -4022,6 +4035,11 @@ impl NectarlinkNode {
self.run(async move { Ok(node.mirror_input(id, core::MIRROR_SCREEN, input).await?) }).await
}
+ /// Preferred webcam codecs accepted by the connected PC, with H.264 as fallback.
+ pub fn webcam_codecs(&self, pc_id: String) -> Result<Vec<String>> {
+ Ok(self.node.webcam_codecs(parse_id(&pc_id)?)?)
+ }
+
/// Opens this phone's camera stream to a paired PC (`docs/protocol/webcam.md`).
pub async fn webcam_open(&self, pc_id: String) -> Result<Arc<MirrorStream>> {
let id = parse_id(&pc_id)?;
diff --git a/core/nectarlink-protocol/src/messages.rs b/core/nectarlink-protocol/src/messages.rs
index dfd43de..02e6c06 100644
--- a/core/nectarlink-protocol/src/messages.rs
+++ b/core/nectarlink-protocol/src/messages.rs
@@ -1636,6 +1636,8 @@ pub mod webcam {
pub const STREAM: &str = "camera.stream";
/// Offered by PCs that receive and decode a phone's webcam stream.
pub const VIRTUAL: &str = "camera.virtual";
+ /// Receiver can decode HEVC webcam streams. Absence means H.264 only.
+ pub const HEVC: &str = "camera.virtual.hevc";
/// Offered by PCs where the Windows virtual camera COM add-on is registered.
pub const ADDON_VCAM: &str = "addon.vcam";
pub const SERVICE: &str = "webcam";
@@ -1644,6 +1646,7 @@ pub mod webcam {
/// Stream reset code: webcam stopped.
pub const STOPPED: u32 = 12;
pub const H264: &str = "h264";
+ pub const H265: &str = "h265";
pub const CAMERA_BACK: &str = "back";
pub const CAMERA_FRONT: &str = "front";
}
@@ -1704,7 +1707,7 @@ impl WebcamStart {
/// The format of a webcam stream (a `PacketKind::Config` packet's payload, in CBOR).
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct WebcamConfig {
- /// `"h264"`.
+ /// `"h264"` or `"h265"` (only when the receiver advertises webcam::HEVC).
pub codec: String,
pub width: u32,
pub height: u32,
@@ -1737,7 +1740,7 @@ impl WebcamConfig {
}
pub fn is_valid(&self) -> bool {
- self.codec == webcam::H264
+ matches!(self.codec.as_str(), webcam::H264 | webcam::H265)
&& (1..=8192).contains(&self.width)
&& (1..=8192).contains(&self.height)
&& (self.camera == webcam::CAMERA_BACK || self.camera == webcam::CAMERA_FRONT)
@@ -5195,6 +5198,9 @@ mod tests {
};
assert!(cfg.is_valid());
assert_eq!(WebcamConfig::from_cbor(&cfg.to_cbor()).unwrap(), cfg);
+ let hevc = WebcamConfig { codec: webcam::H265.into(), ..cfg.clone() };
+ assert!(hevc.is_valid());
+ assert_eq!(WebcamConfig::from_cbor(&hevc.to_cbor()).unwrap(), hevc);
assert!(!WebcamConfig { codec: "vp8".into(), ..cfg.clone() }.is_valid());
assert!(!WebcamConfig { width: 0, ..cfg }.is_valid());
diff --git a/desktop/app/src/core_host.rs b/desktop/app/src/core_host.rs
index 423bf9b..8d3c1f9 100644
--- a/desktop/app/src/core_host.rs
+++ b/desktop/app/src/core_host.rs
@@ -127,6 +127,9 @@ pub fn compute_capabilities(
#[cfg(windows)]
{
caps.push(nectarlink_core::WEBCAM_VIRTUAL.into());
+ if crate::webcam::supports_hevc() {
+ caps.push(nectarlink_core::WEBCAM_HEVC.into());
+ }
caps.push(nectarlink_core::MEDIA_STREAM.into());
}
#[cfg(target_os = "linux")]
diff --git a/desktop/app/src/webcam.rs b/desktop/app/src/webcam.rs
index 1a402c3..49ff480 100644
--- a/desktop/app/src/webcam.rs
+++ b/desktop/app/src/webcam.rs
@@ -715,6 +715,22 @@ fn mirror_bgrx_in_place(bgrx: &mut [u8], width: u32, height: u32) {
}
}
+#[cfg(windows)]
+pub fn supports_hevc() -> bool {
+ static SUPPORTED: OnceLock<bool> = OnceLock::new();
+ *SUPPORTED.get_or_init(|| match Decoder::new_gpu("h265") {
+ Ok(decoder) => {
+ let supported = decoder.on_gpu();
+ tracing::info!(supported, "HEVC webcam decoder probe");
+ supported
+ }
+ Err(error) => {
+ tracing::info!(%error, "HEVC webcam decoder unavailable; advertising H.264 only");
+ false
+ }
+ })
+}
+
fn decode(device: DeviceId, items: &Receiver<Item>, waiting: &AtomicUsize) {
// On the GPU when there is one: 4K takes about 24 ms a frame in software
// and 11 ms on a GPU, against 33 ms per frame at 30 fps.
@@ -749,6 +765,18 @@ fn decode(device: DeviceId, items: &Receiver<Item>, waiting: &AtomicUsize) {
while let Ok(item) = items.recv() {
match item {
Item::Config(config) => {
+ if decoder.codec() != config.codec {
+ match Decoder::new_gpu(&config.codec) {
+ Ok(next) => {
+ decoder = next;
+ tracing::info!(codec = %config.codec, gpu = decoder.on_gpu(), "webcam decoder changed");
+ }
+ Err(error) => {
+ tracing::error!(codec = %config.codec, %error, "negotiated webcam decoder unavailable");
+ break;
+ }
+ }
+ }
if size != (0, 0) && size != (config.width, config.height) {
decoder.flush();
}
diff --git a/desktop/app/src/win/h264.rs b/desktop/app/src/win/h264.rs
index 6dc92c1..6f50df9 100644
--- a/desktop/app/src/win/h264.rs
+++ b/desktop/app/src/win/h264.rs
@@ -695,67 +695,78 @@ mod tests {
let worker = std::thread::spawn(move || {
use nectarlink_vcam::gpu::{GpuPublisher, GpuReader};
use windows::Win32::Media::MediaFoundation::MFCreateDXGISurfaceBuffer;
- let mut decoder = Decoder::new_gpu("h264").unwrap();
- if !decoder.on_gpu() {
- done.send(()).unwrap();
- return;
- }
- let name = format!("Local\\NectarlinkDecoderPoolTest_{}", std::process::id());
- let mut reader = GpuReader::new_with_mapping(&name).unwrap();
- let mut publisher = GpuPublisher::new_with_mapping(&name).unwrap();
- let mut output = None;
- let mut count = 0;
- for _ in 0..20 {
- for unit in test_stream("h264") {
- for frame in decoder.decode_frames(unit, count * 33_333).unwrap() {
- let (texture, index) = frame.gpu_surface().unwrap().unwrap();
- if output.is_none() {
- unsafe {
- let mut desc = D3D11_TEXTURE2D_DESC::default();
- texture.GetDesc(&mut desc);
- desc.ArraySize = 1;
- desc.MipLevels = 1;
- desc.BindFlags =
- windows::Win32::Graphics::Direct3D11::D3D11_BIND_RENDER_TARGET.0 as u32;
- let mut destination = None;
- texture
- .GetDevice()
- .unwrap()
- .CreateTexture2D(&desc, None, Some(&mut destination))
- .unwrap();
- output = Some(
- MFCreateDXGISurfaceBuffer(
- &ID3D11Texture2D::IID,
- &destination.unwrap(),
- 0,
+ for codec in ["h264", "h265"] {
+ if !installed(codec) {
+ continue;
+ }
+ let mut decoder = Decoder::new_gpu(codec).unwrap();
+ if !decoder.on_gpu() {
+ continue;
+ }
+ let name = format!("Local\\NectarlinkDecoderPoolTest_{}_{}", std::process::id(), codec);
+ let mut reader = GpuReader::new_with_mapping(&name).unwrap();
+ let mut publisher = GpuPublisher::new_with_mapping(&name).unwrap();
+ let mut output = None;
+ let mut count = 0;
+ for _ in 0..20 {
+ for unit in test_stream(codec) {
+ for frame in decoder
+ .decode_frames(unit, count * 33_333)
+ .unwrap_or_else(|error| panic!("{codec} after {count} frames: {error}"))
+ {
+ let (texture, index) = frame.gpu_surface().unwrap().unwrap();
+ if output.is_none() {
+ unsafe {
+ let mut desc = D3D11_TEXTURE2D_DESC::default();
+ texture.GetDesc(&mut desc);
+ desc.ArraySize = 1;
+ desc.MipLevels = 1;
+ desc.BindFlags =
+ windows::Win32::Graphics::Direct3D11::D3D11_BIND_RENDER_TARGET.0
+ as u32;
+ let mut destination = None;
+ texture
+ .GetDevice()
+ .unwrap()
+ .CreateTexture2D(&desc, None, Some(&mut destination))
+ .unwrap();
+ output = Some(
+ MFCreateDXGISurfaceBuffer(
+ &ID3D11Texture2D::IID,
+ &destination.unwrap(),
+ 0,
+ false,
+ )
+ .unwrap(),
+ );
+ }
+ }
+ assert!(
+ publisher
+ .publish(
+ &texture,
+ index,
+ &frame.sample,
+ (frame.width, frame.height),
false,
+ false
)
- .unwrap(),
- );
- }
+ .unwrap()
+ );
+ assert!(
+ reader
+ .write_nv12(output.as_ref().unwrap(), frame.width, frame.height)
+ .unwrap()
+ );
+ count += 1;
}
- assert!(
- publisher
- .publish(
- &texture,
- index,
- &frame.sample,
- (frame.width, frame.height),
- false,
- false
- )
- .unwrap()
- );
- assert!(
- reader.write_nv12(output.as_ref().unwrap(), frame.width, frame.height).unwrap()
- );
- count += 1;
}
}
+ assert_eq!(count, 200);
+ assert_eq!(publisher.stats(), Some((200, 0)));
+ assert!(decoder.staging.is_none(), "publication must not download the decoded frames");
+ eprintln!("{codec}: published {count} GPU frames with zero CPU downloads");
}
- assert_eq!(count, 200);
- assert_eq!(publisher.stats(), Some((200, 0)));
- assert!(decoder.staging.is_none(), "publication must not download the decoded frames");
done.send(()).unwrap();
});
finished.recv_timeout(std::time::Duration::from_secs(30)).expect("decoder sample pool stalled");
diff --git a/docs/protocol/webcam.md b/docs/protocol/webcam.md
index 4aa6103..79f5570 100644
--- a/docs/protocol/webcam.md
+++ b/docs/protocol/webcam.md
@@ -10,6 +10,7 @@ travel on a dedicated bidirectional QUIC stream (`{ svc: "webcam", op: "video",
|---|---|---|
| `camera.stream` | Phone | Streams a camera for webcam use (when `CAMERA` permission is granted) |
| `camera.virtual` | PC | Receives and decodes a phone's webcam stream |
+| `camera.virtual.hevc` | PC | Also accepts H.265/HEVC webcam video; absence means H.264 only |
| `addon.vcam` | PC | The Windows virtual camera COM add-on is registered in `HKLM`, so apps (Teams, Meet, Zoom, Windows Camera) see `"NectarLink - <Phone name>"` |
Both devices have a `webcam` per-device toggle (on by default). When turned off
@@ -66,7 +67,7 @@ Stops the webcam stream. Either side may also stop by closing or resetting the
{ t: "webcam.keyframe" }
```
-Asks the phone's hardware H.264 encoder for an immediate IDR keyframe
+Asks the phone's hardware video encoder for an immediate keyframe
(`PARAMETER_KEY_REQUEST_SYNC_FRAME`), sent when the PC decoder starts mid-stream
or after a dropped packet.
@@ -93,3 +94,25 @@ Followed by a sequence of framed video packets (`core/nectarlink-protocol/src/vi
The phone's sender bounds its outgoing queue to a few frames (`QUEUE_FRAMES = 4`):
if the network falls behind, queued frames are dropped and the encoder is asked
for a fresh keyframe rather than building up latency.
+
+### Optional HEVC negotiation
+
+An updated phone prefers `h265` only when the connected receiver advertises
+`camera.virtual.hevc` and a hardware HEVC Main-profile surface encoder supports
+the requested dimensions and frame rate. Otherwise it uses `h264`. Windows
+advertises HEVC only when its installed Media Foundation decoder can initialize
+with a GPU device manager. The phone falls back to H.264 if the HEVC encoder
+cannot start, or reconfigures the existing stream to H.264 after a HEVC drain error.
+
+For HEVC, `Config.codec` is `h265`, `Frame` and `Keyframe` contain Annex B HEVC
+access units, and keyframes carry VPS, SPS and PPS. Each access unit starts with
+an HEVC AUD (NAL type 35), instead of the H.264 AUD (type 9). The receiver switches
+decoders when a Config changes codec. HEVC configs are rejected by receivers that
+do not advertise the HEVC capability, including later Config packets in a stream.
+Older phone and desktop releases continue using H.264 without changing version 1
+framing or the existing start-request fields.
+
+The initial HEVC implementation retains the same resolution, frame rate and target
+bitrate as H.264. It establishes codec compatibility; it does not itself claim a
+bandwidth or latency reduction. A lower HEVC target bitrate needs separate quality
+and latency measurements.