Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(recording): pick a webcam mode by frame rate before size
EtienneLescot6286fd4c9901
diff --git a/electron/native/wgc-capture/src/webcam_capture.cpp b/electron/native/wgc-capture/src/webcam_capture.cppindex 2a367c67d..a876c956a 100644--- a/electron/native/wgc-capture/src/webcam_capture.cpp+++ b/electron/native/wgc-capture/src/webcam_capture.cpp@@ -333,7 +333,7 @@ std::vector<WebcamFormat> readNativeFormats(IMFSourceReader* reader) {* Puts the camera itself on `wanted` by selecting the matching native type.** Best-effort: a driver that refuses leaves the reader on whatever it had, and- * the RGB32 request that follows still produces a usable -- if smaller -- frame.+ * the output request that follows still produces a usable -- if smaller -- frame.*/void selectNativeFormat(IMFSourceReader* reader, const WebcamFormat& wanted) {for (DWORD index = 0;; ++index) {@@ -399,22 +399,8 @@ bool WebcamCapture::configureReader(return false;}- Microsoft::WRL::ComPtr<IMFMediaType> mediaType;- if (!succeeded(MFCreateMediaType(&mediaType), "MFCreateMediaType(webcam output)")) {- return false;- }- mediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);- // NV12 when the frame only has to reach the webcam encoder, which wants- // NV12 anyway. Asking for RGB32 instead makes the source reader decode AND- // convert every frame: measured at 92ms per 3840x2160 frame against 32ms- // for NV12, which is the difference between 11 fps and the camera's full- // 30. BGRA is still used when the frame is composited into the screen- // recording inline, which needs it in that layout.- deliversNv12_ = preferNv12;- mediaType->SetGUID(MF_MT_SUBTYPE, preferNv12 ? MFVideoFormat_NV12 : MFVideoFormat_RGB32);-// Pick the capture format, then SELECT it on the source before asking for- // RGB32 output.+ // an output type.//// Both halves matter. Handing Media Foundation an output type with no// MF_MT_FRAME_SIZE leaves the device on its default format -- 640x480 on a@@ -434,27 +420,60 @@ bool WebcamCapture::configureReader(<< chosen.fps << (chosen.compressed ? " (compressed)" : " (uncompressed)") << std::endl;selectNativeFormat(sourceReader_.Get(), chosen);- MFSetAttributeSize(- mediaType.Get(), MF_MT_FRAME_SIZE, static_cast<UINT32>(chosen.width), static_cast<UINT32>(chosen.height));- MFSetAttributeRatio(mediaType.Get(), MF_MT_FRAME_RATE, static_cast<UINT32>(std::max(1, requestedFps)), 1);-- if (FAILED(sourceReader_->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, mediaType.Get()))) {- // Some drivers refuse a size they nonetheless enumerate. A slightly soft- // camera beats no camera, so drop the size and let the driver choose --- // exactly what this code did before it asked for anything.- std::cerr << "WARNING: Webcam rejected " << chosen.width << "x" << chosen.height- << "; falling back to the device default format" << std::endl;- Microsoft::WRL::ComPtr<IMFMediaType> fallbackType;- if (!succeeded(MFCreateMediaType(&fallbackType), "MFCreateMediaType(webcam fallback)")) {- return false;- }- fallbackType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);- fallbackType->SetGUID(MF_MT_SUBTYPE, preferNv12 ? MFVideoFormat_NV12 : MFVideoFormat_RGB32);- if (!succeeded(- sourceReader_->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, fallbackType.Get()),- "SetCurrentMediaType(webcam RGB32)")) {- return false;+ // Asks for `subtype` at the chosen size, then without one: some drivers+ // refuse a size they nonetheless enumerate, and a slightly soft camera+ // beats no camera -- the device default is what this code used before it+ // asked for anything.+ const auto requestOutput = [&](const GUID& subtype) {+ for (const bool sized : {true, false}) {+ Microsoft::WRL::ComPtr<IMFMediaType> outputType;+ if (FAILED(MFCreateMediaType(&outputType))) {+ return false;+ }+ outputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);+ outputType->SetGUID(MF_MT_SUBTYPE, subtype);+ if (sized) {+ MFSetAttributeSize(+ outputType.Get(),+ MF_MT_FRAME_SIZE,+ static_cast<UINT32>(chosen.width),+ static_cast<UINT32>(chosen.height));+ MFSetAttributeRatio(+ outputType.Get(), MF_MT_FRAME_RATE, static_cast<UINT32>(std::max(1, requestedFps)), 1);+ }+ if (SUCCEEDED(sourceReader_->SetCurrentMediaType(+ MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, outputType.Get()))) {+ return true;+ }+ if (sized) {+ std::cerr << "WARNING: Webcam rejected "+ << (subtype == MFVideoFormat_NV12 ? "NV12" : "RGB32") << " at " << chosen.width+ << "x" << chosen.height << "; trying the device default size" << std::endl;+ }}+ return false;+ };++ // NV12 when the frame only has to reach the webcam encoder, which wants+ // NV12 anyway. Asking for RGB32 instead makes the source reader decode AND+ // convert every frame: measured at 92ms per 3840x2160 frame against 32ms+ // for NV12, which is the difference between 11 fps and the camera's full+ // 30. BGRA is still used when the frame is composited into the screen+ // recording inline, which needs it in that layout.+ //+ // RGB32 is also the fallback for NV12, because it is the only conversion+ // MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING documents: "Conversion from YUV+ // to RGB-32". A reader that turns down NV12 from a YUY2 or MJPEG source is+ // within its contract -- Windows 11 accepted both here, nothing promises+ // it -- and without this fallback the refusal costs the take its camera.+ // The webcam encoder takes BGRA as well as NV12.+ deliversNv12_ = preferNv12 && requestOutput(MFVideoFormat_NV12);+ if (preferNv12 && !deliversNv12_) {+ std::cerr << "WARNING: Webcam cannot deliver NV12; capturing RGB32 instead" << std::endl;+ }+ if (!deliversNv12_ && !requestOutput(MFVideoFormat_RGB32)) {+ std::cerr << "ERROR: Webcam accepted neither NV12 nor RGB32 output" << std::endl;+ return false;}sourceReader_->SetStreamSelection(MF_SOURCE_READER_ALL_STREAMS, FALSE);sourceReader_->SetStreamSelection(MF_SOURCE_READER_FIRST_VIDEO_STREAM, TRUE);diff --git a/electron/native/wgc-capture/src/webcam_format.cpp b/electron/native/wgc-capture/src/webcam_format.cppindex 32e2020ed..569f6f444 100644--- a/electron/native/wgc-capture/src/webcam_format.cpp+++ b/electron/native/wgc-capture/src/webcam_format.cpp@@ -16,6 +16,12 @@ long long pixels(const WebcamFormat& format) {return static_cast<long long>(format.width) * static_cast<long long>(format.height);}+// The part of a mode's frame rate the recording can use: the webcam is encoded+// at the target rate, so anything above it buys nothing.+int usefulFps(const WebcamFormat& format, int targetFps) {+ return std::min(format.fps, targetFps);+}+// True when `candidate` is a better frame rate than `best` for a target of// `targetFps`: the lowest rate that still reaches the target, or -- when no// mode reaches it -- the highest one on offer.@@ -66,6 +72,14 @@ WebcamFormat chooseWebcamFormat(best = &format;continue;}+ // Frame rate before pixels: a camera's largest uncompressed modes are+ // often stills-grade, 2 to 5 fps on a Logitech C920.+ if (usefulFps(format, targetFps) != usefulFps(*best, targetFps)) {+ if (usefulFps(format, targetFps) > usefulFps(*best, targetFps)) {+ best = &format;+ }+ continue;+ }if (pixels(format) != pixels(*best)) {const bool wins = anyFits ? pixels(format) > pixels(*best) : pixels(format) < pixels(*best);if (wins) {diff --git a/electron/native/wgc-capture/src/webcam_format.h b/electron/native/wgc-capture/src/webcam_format.hindex 7cb329af7..d8650a2b4 100644--- a/electron/native/wgc-capture/src/webcam_format.h+++ b/electron/native/wgc-capture/src/webcam_format.h@@ -30,11 +30,15 @@ struct WebcamFormat {// no camera in it, which is far worse than a smaller one that works;// * only formats that fit inside the target in BOTH dimensions are eligible,// so an ultrawide mode cannot sneak in on pixel count alone;-// * among those, the most pixels wins;+// * among those, the frame rate comes first: the highest rate up to the+// target wins, so a C920's 2304x1536 at 2 fps loses to its 800x448 at 30.+// Ranking pixels first drove that camera at 2 to 5 fps on every preset;+// * then the most pixels;// * ties go to the lowest frame rate that still reaches the target, else the// highest available -- a 1080p60 mode is no help when we encode at 30;-// * if nothing fits (a camera that only does 4K), the smallest mode wins, so-// we scale down rather than push 1 GB/s of RGB32 through the encoder;+// * if nothing fits (a camera that only does 4K), the smallest mode at the+// best rate wins, so we scale down rather than push 1 GB/s of RGB32+// through the encoder;// * an empty list returns the target unchanged, letting the caller fall back// to whatever the driver would have picked on its own.WebcamFormat chooseWebcamFormat(diff --git a/electron/native/wgc-capture/src/webcam_format_test.cpp b/electron/native/wgc-capture/src/webcam_format_test.cppindex 9e40f5cbf..11ed08d2a 100644--- a/electron/native/wgc-capture/src/webcam_format_test.cpp+++ b/electron/native/wgc-capture/src/webcam_format_test.cpp@@ -32,6 +32,20 @@ const std::vector<WebcamFormat> kBrio = {{340, 340, 30, false}, {1600, 896, 30, false}, {1024, 576, 30, false},};+// A Logitech C920, the most common webcam there is. Its uncompressed (YUY2)+// modes trade rate for size all the way down to 2 fps; only MJPEG reaches+// 1080p at 30. 1600x896 is 7.5 fps, rounded here as the helper rounds it.+const std::vector<WebcamFormat> kC920 = {+ {640, 480, 30, false}, {160, 90, 30, false}, {160, 120, 30, false},+ {176, 144, 30, false}, {320, 180, 30, false}, {320, 240, 30, false},+ {352, 288, 30, false}, {432, 240, 30, false}, {640, 360, 30, false},+ {800, 448, 30, false}, {800, 600, 24, false}, {864, 480, 24, false},+ {960, 720, 15, false}, {1024, 576, 15, false}, {1280, 720, 10, false},+ {1600, 896, 8, false}, {1920, 1080, 5, false}, {2304, 1296, 2, false},+ {2304, 1536, 2, false}, {640, 480, 30, true}, {1280, 720, 30, true},+ {1600, 896, 30, true}, {1920, 1080, 30, true},+};+} // namespaceint main() {@@ -54,6 +68,12 @@ int main() {++failures;}+ // getopenscreen/openscreen#1112: ranking pixels first drove a C920 at+ // 2304x1536 and 2 fps on the default 2160p preset, and at 1920x1080 and+ // 5 fps on 1080p. A usable rate comes before size.+ expectFormat("c920 on 2160p keeps 30 fps", chooseWebcamFormat(kC920, 3840, 2160, 30), 800, 448, 30);+ expectFormat("c920 on 1080p keeps 30 fps", chooseWebcamFormat(kC920, 1920, 1080, 30), 800, 448, 30);+// A camera with nothing but compressed modes still has to be usable.expectFormat("compressed-only camera is still driven",