Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(recording): pick a webcam mode by frame rate before size

EtienneLescot6286fd4c9901
diff --git a/electron/native/wgc-capture/src/webcam_capture.cpp b/electron/native/wgc-capture/src/webcam_capture.cpp
index 2a367c67d..a876c956a 100644
--- a/electron/native/wgc-capture/src/webcam_capture.cpp
+++ b/electron/native/wgc-capture/src/webcam_capture.cpp
@@ -333,7 +333,7 @@ std::vector<WebcamFormat> readNativeFormats(IMFSourceReader* reader) {
* Puts the camera itself on `wanted` by selecting the matching native type.
*
* Best-effort: a driver that refuses leaves the reader on whatever it had, and
- * the RGB32 request that follows still produces a usable -- if smaller -- frame.
+ * the output request that follows still produces a usable -- if smaller -- frame.
*/
void selectNativeFormat(IMFSourceReader* reader, const WebcamFormat& wanted) {
for (DWORD index = 0;; ++index) {
@@ -399,22 +399,8 @@ bool WebcamCapture::configureReader(
return false;
}
- Microsoft::WRL::ComPtr<IMFMediaType> mediaType;
- if (!succeeded(MFCreateMediaType(&mediaType), "MFCreateMediaType(webcam output)")) {
- return false;
- }
- mediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
- // NV12 when the frame only has to reach the webcam encoder, which wants
- // NV12 anyway. Asking for RGB32 instead makes the source reader decode AND
- // convert every frame: measured at 92ms per 3840x2160 frame against 32ms
- // for NV12, which is the difference between 11 fps and the camera's full
- // 30. BGRA is still used when the frame is composited into the screen
- // recording inline, which needs it in that layout.
- deliversNv12_ = preferNv12;
- mediaType->SetGUID(MF_MT_SUBTYPE, preferNv12 ? MFVideoFormat_NV12 : MFVideoFormat_RGB32);
-
// Pick the capture format, then SELECT it on the source before asking for
- // RGB32 output.
+ // an output type.
//
// Both halves matter. Handing Media Foundation an output type with no
// MF_MT_FRAME_SIZE leaves the device on its default format -- 640x480 on a
@@ -434,27 +420,60 @@ bool WebcamCapture::configureReader(
<< chosen.fps << (chosen.compressed ? " (compressed)" : " (uncompressed)") << std::endl;
selectNativeFormat(sourceReader_.Get(), chosen);
- MFSetAttributeSize(
- mediaType.Get(), MF_MT_FRAME_SIZE, static_cast<UINT32>(chosen.width), static_cast<UINT32>(chosen.height));
- MFSetAttributeRatio(mediaType.Get(), MF_MT_FRAME_RATE, static_cast<UINT32>(std::max(1, requestedFps)), 1);
-
- if (FAILED(sourceReader_->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, mediaType.Get()))) {
- // Some drivers refuse a size they nonetheless enumerate. A slightly soft
- // camera beats no camera, so drop the size and let the driver choose --
- // exactly what this code did before it asked for anything.
- std::cerr << "WARNING: Webcam rejected " << chosen.width << "x" << chosen.height
- << "; falling back to the device default format" << std::endl;
- Microsoft::WRL::ComPtr<IMFMediaType> fallbackType;
- if (!succeeded(MFCreateMediaType(&fallbackType), "MFCreateMediaType(webcam fallback)")) {
- return false;
- }
- fallbackType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
- fallbackType->SetGUID(MF_MT_SUBTYPE, preferNv12 ? MFVideoFormat_NV12 : MFVideoFormat_RGB32);
- if (!succeeded(
- sourceReader_->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, fallbackType.Get()),
- "SetCurrentMediaType(webcam RGB32)")) {
- return false;
+ // Asks for `subtype` at the chosen size, then without one: some drivers
+ // refuse a size they nonetheless enumerate, and a slightly soft camera
+ // beats no camera -- the device default is what this code used before it
+ // asked for anything.
+ const auto requestOutput = [&](const GUID& subtype) {
+ for (const bool sized : {true, false}) {
+ Microsoft::WRL::ComPtr<IMFMediaType> outputType;
+ if (FAILED(MFCreateMediaType(&outputType))) {
+ return false;
+ }
+ outputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
+ outputType->SetGUID(MF_MT_SUBTYPE, subtype);
+ if (sized) {
+ MFSetAttributeSize(
+ outputType.Get(),
+ MF_MT_FRAME_SIZE,
+ static_cast<UINT32>(chosen.width),
+ static_cast<UINT32>(chosen.height));
+ MFSetAttributeRatio(
+ outputType.Get(), MF_MT_FRAME_RATE, static_cast<UINT32>(std::max(1, requestedFps)), 1);
+ }
+ if (SUCCEEDED(sourceReader_->SetCurrentMediaType(
+ MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, outputType.Get()))) {
+ return true;
+ }
+ if (sized) {
+ std::cerr << "WARNING: Webcam rejected "
+ << (subtype == MFVideoFormat_NV12 ? "NV12" : "RGB32") << " at " << chosen.width
+ << "x" << chosen.height << "; trying the device default size" << std::endl;
+ }
}
+ return false;
+ };
+
+ // NV12 when the frame only has to reach the webcam encoder, which wants
+ // NV12 anyway. Asking for RGB32 instead makes the source reader decode AND
+ // convert every frame: measured at 92ms per 3840x2160 frame against 32ms
+ // for NV12, which is the difference between 11 fps and the camera's full
+ // 30. BGRA is still used when the frame is composited into the screen
+ // recording inline, which needs it in that layout.
+ //
+ // RGB32 is also the fallback for NV12, because it is the only conversion
+ // MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING documents: "Conversion from YUV
+ // to RGB-32". A reader that turns down NV12 from a YUY2 or MJPEG source is
+ // within its contract -- Windows 11 accepted both here, nothing promises
+ // it -- and without this fallback the refusal costs the take its camera.
+ // The webcam encoder takes BGRA as well as NV12.
+ deliversNv12_ = preferNv12 && requestOutput(MFVideoFormat_NV12);
+ if (preferNv12 && !deliversNv12_) {
+ std::cerr << "WARNING: Webcam cannot deliver NV12; capturing RGB32 instead" << std::endl;
+ }
+ if (!deliversNv12_ && !requestOutput(MFVideoFormat_RGB32)) {
+ std::cerr << "ERROR: Webcam accepted neither NV12 nor RGB32 output" << std::endl;
+ return false;
}
sourceReader_->SetStreamSelection(MF_SOURCE_READER_ALL_STREAMS, FALSE);
sourceReader_->SetStreamSelection(MF_SOURCE_READER_FIRST_VIDEO_STREAM, TRUE);
diff --git a/electron/native/wgc-capture/src/webcam_format.cpp b/electron/native/wgc-capture/src/webcam_format.cpp
index 32e2020ed..569f6f444 100644
--- a/electron/native/wgc-capture/src/webcam_format.cpp
+++ b/electron/native/wgc-capture/src/webcam_format.cpp
@@ -16,6 +16,12 @@ long long pixels(const WebcamFormat& format) {
return static_cast<long long>(format.width) * static_cast<long long>(format.height);
}
+// The part of a mode's frame rate the recording can use: the webcam is encoded
+// at the target rate, so anything above it buys nothing.
+int usefulFps(const WebcamFormat& format, int targetFps) {
+ return std::min(format.fps, targetFps);
+}
+
// True when `candidate` is a better frame rate than `best` for a target of
// `targetFps`: the lowest rate that still reaches the target, or -- when no
// mode reaches it -- the highest one on offer.
@@ -66,6 +72,14 @@ WebcamFormat chooseWebcamFormat(
best = &format;
continue;
}
+ // Frame rate before pixels: a camera's largest uncompressed modes are
+ // often stills-grade, 2 to 5 fps on a Logitech C920.
+ if (usefulFps(format, targetFps) != usefulFps(*best, targetFps)) {
+ if (usefulFps(format, targetFps) > usefulFps(*best, targetFps)) {
+ best = &format;
+ }
+ continue;
+ }
if (pixels(format) != pixels(*best)) {
const bool wins = anyFits ? pixels(format) > pixels(*best) : pixels(format) < pixels(*best);
if (wins) {
diff --git a/electron/native/wgc-capture/src/webcam_format.h b/electron/native/wgc-capture/src/webcam_format.h
index 7cb329af7..d8650a2b4 100644
--- a/electron/native/wgc-capture/src/webcam_format.h
+++ b/electron/native/wgc-capture/src/webcam_format.h
@@ -30,11 +30,15 @@ struct WebcamFormat {
// no camera in it, which is far worse than a smaller one that works;
// * only formats that fit inside the target in BOTH dimensions are eligible,
// so an ultrawide mode cannot sneak in on pixel count alone;
-// * among those, the most pixels wins;
+// * among those, the frame rate comes first: the highest rate up to the
+// target wins, so a C920's 2304x1536 at 2 fps loses to its 800x448 at 30.
+// Ranking pixels first drove that camera at 2 to 5 fps on every preset;
+// * then the most pixels;
// * ties go to the lowest frame rate that still reaches the target, else the
// highest available -- a 1080p60 mode is no help when we encode at 30;
-// * if nothing fits (a camera that only does 4K), the smallest mode wins, so
-// we scale down rather than push 1 GB/s of RGB32 through the encoder;
+// * if nothing fits (a camera that only does 4K), the smallest mode at the
+// best rate wins, so we scale down rather than push 1 GB/s of RGB32
+// through the encoder;
// * an empty list returns the target unchanged, letting the caller fall back
// to whatever the driver would have picked on its own.
WebcamFormat chooseWebcamFormat(
diff --git a/electron/native/wgc-capture/src/webcam_format_test.cpp b/electron/native/wgc-capture/src/webcam_format_test.cpp
index 9e40f5cbf..11ed08d2a 100644
--- a/electron/native/wgc-capture/src/webcam_format_test.cpp
+++ b/electron/native/wgc-capture/src/webcam_format_test.cpp
@@ -32,6 +32,20 @@ const std::vector<WebcamFormat> kBrio = {
{340, 340, 30, false}, {1600, 896, 30, false}, {1024, 576, 30, false},
};
+// A Logitech C920, the most common webcam there is. Its uncompressed (YUY2)
+// modes trade rate for size all the way down to 2 fps; only MJPEG reaches
+// 1080p at 30. 1600x896 is 7.5 fps, rounded here as the helper rounds it.
+const std::vector<WebcamFormat> kC920 = {
+ {640, 480, 30, false}, {160, 90, 30, false}, {160, 120, 30, false},
+ {176, 144, 30, false}, {320, 180, 30, false}, {320, 240, 30, false},
+ {352, 288, 30, false}, {432, 240, 30, false}, {640, 360, 30, false},
+ {800, 448, 30, false}, {800, 600, 24, false}, {864, 480, 24, false},
+ {960, 720, 15, false}, {1024, 576, 15, false}, {1280, 720, 10, false},
+ {1600, 896, 8, false}, {1920, 1080, 5, false}, {2304, 1296, 2, false},
+ {2304, 1536, 2, false}, {640, 480, 30, true}, {1280, 720, 30, true},
+ {1600, 896, 30, true}, {1920, 1080, 30, true},
+};
+
} // namespace
int main() {
@@ -54,6 +68,12 @@ int main() {
++failures;
}
+ // getopenscreen/openscreen#1112: ranking pixels first drove a C920 at
+ // 2304x1536 and 2 fps on the default 2160p preset, and at 1920x1080 and
+ // 5 fps on 1080p. A usable rate comes before size.
+ expectFormat("c920 on 2160p keeps 30 fps", chooseWebcamFormat(kC920, 3840, 2160, 30), 800, 448, 30);
+ expectFormat("c920 on 1080p keeps 30 fps", chooseWebcamFormat(kC920, 1920, 1080, 30), 800, 448, 30);
+
// A camera with nothing but compressed modes still has to be usable.
expectFormat(
"compressed-only camera is still driven",