Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(recording): do not read studio-range black as a visible webcam frame
Christian Schmittelc6d3064be487
diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txtindex efa82295d..7a80f3214 100644--- a/electron/native/wgc-capture/CMakeLists.txt+++ b/electron/native/wgc-capture/CMakeLists.txt@@ -41,6 +41,8 @@ add_executable(wgc-capturesrc/desktop_icon_cover.cppsrc/desktop_icon_cover.hsrc/dpi_awareness.h+ src/frame_visibility.cpp+ src/frame_visibility.hsrc/dshow_webcam_capture.cppsrc/dshow_webcam_capture.hsrc/main.cpp@@ -133,3 +135,17 @@ target_compile_definitions(webcam_format_test PRIVATE)target_compile_options(webcam_format_test PRIVATE /EHsc /W4 /utf-8)++add_executable(frame_visibility_test+ src/frame_visibility.cpp+ src/frame_visibility.h+ src/frame_visibility_test.cpp+)++target_compile_definitions(frame_visibility_test PRIVATE+ NOMINMAX+ WIN32_LEAN_AND_MEAN+ _WIN32_WINNT=0x0A00+)++target_compile_options(frame_visibility_test PRIVATE /EHsc /W4 /utf-8)diff --git a/electron/native/wgc-capture/src/frame_visibility.cpp b/electron/native/wgc-capture/src/frame_visibility.cppnew file mode 100644index 000000000..d296b5f0e--- /dev/null+++ b/electron/native/wgc-capture/src/frame_visibility.cpp@@ -0,0 +1,78 @@+#include "frame_visibility.h"++#include <algorithm>++namespace {++// Sampling at most this many pixels keeps the probe cheap on a 4K frame while+// still covering it evenly.+constexpr size_t kMaxSamples = 4096;++// Luma is judged on a full-range 0..255 scale. A frame counts as visible when+// any sample is clearly above black, or when the picture is dim but not empty.+constexpr int kMaxLumaThreshold = 24;+constexpr int kAverageLumaThreshold = 4;++// BT.709/BT.601 studio range: black sits at 16, white at 235.+constexpr int kStudioBlack = 16;+constexpr int kStudioSpan = 235 - kStudioBlack;++bool isVisible(uint64_t lumaTotal, int maxLuma, size_t sampled) {+ const uint64_t averageLuma = sampled > 0 ? lumaTotal / sampled : 0;+ return maxLuma > kMaxLumaThreshold || averageLuma > kAverageLumaThreshold;+}++} // namespace++bool hasVisibleBgraContent(const std::vector<std::uint8_t>& frame) {+ if (frame.size() < 4) {+ return false;+ }++ uint64_t lumaTotal = 0;+ int maxLuma = 0;+ const size_t pixelCount = frame.size() / 4;+ const size_t step = std::max<size_t>(1, pixelCount / kMaxSamples);+ size_t sampled = 0;+ for (size_t pixel = 0; pixel < pixelCount; pixel += step) {+ const size_t offset = pixel * 4;+ const int b = frame[offset + 0];+ const int g = frame[offset + 1];+ const int r = frame[offset + 2];+ const int luma = (r * 54 + g * 183 + b * 19) >> 8;+ lumaTotal += static_cast<uint64_t>(luma);+ maxLuma = std::max(maxLuma, luma);+ sampled += 1;+ }++ return isVisible(lumaTotal, maxLuma, sampled);+}++bool hasVisibleNv12Content(const std::vector<std::uint8_t>& frame) {+ if (frame.size() < 6) {+ return false;+ }++ const size_t lumaCount = frame.size() * 2 / 3;+ const size_t step = std::max<size_t>(1, lumaCount / kMaxSamples);+ uint64_t lumaTotal = 0;+ int maxLuma = 0;+ size_t sampled = 0;+ for (size_t offset = 0; offset < lumaCount; offset += step) {+ // Studio range mapped onto 0..255, so the thresholds mean the same here+ // as they do for BGRA. Skip this and an all-black frame averages 16 --+ // past the average threshold on its own, so every black frame would+ // count as a picture.+ const int studio = static_cast<int>(frame[offset]);+ const int luma = std::clamp(((studio - kStudioBlack) * 255) / kStudioSpan, 0, 255);+ lumaTotal += static_cast<uint64_t>(luma);+ maxLuma = std::max(maxLuma, luma);+ sampled += 1;+ }++ return isVisible(lumaTotal, maxLuma, sampled);+}++bool hasVisibleWebcamContent(const std::vector<std::uint8_t>& frame, bool isNv12) {+ return isNv12 ? hasVisibleNv12Content(frame) : hasVisibleBgraContent(frame);+}diff --git a/electron/native/wgc-capture/src/frame_visibility.h b/electron/native/wgc-capture/src/frame_visibility.hnew file mode 100644index 000000000..9f62c2298--- /dev/null+++ b/electron/native/wgc-capture/src/frame_visibility.h@@ -0,0 +1,27 @@+#pragma once++#include <cstdint>+#include <vector>++/**+ * Does this frame contain any picture, or is it the blank one a camera emits+ * while it is still warming up?+ *+ * Used before recording starts, to tell "the camera is running" from "the+ * camera is open but has not produced anything yet".+ */+bool hasVisibleBgraContent(const std::vector<std::uint8_t>& frame);++/**+ * The NV12 twin: luma is the Y plane, one byte per pixel, so no colour+ * conversion is needed. The Y plane is the first two thirds of the buffer.+ *+ * A camera's NV12 is studio-range, where black is 16 rather than 0. Those+ * values are normalised to full range before the thresholds are applied --+ * without that, the average of an all-black frame is 16, clears the average+ * threshold on its own, and every black frame counts as visible.+ */+bool hasVisibleNv12Content(const std::vector<std::uint8_t>& frame);++/** Dispatches to the probe matching the frame's layout. */+bool hasVisibleWebcamContent(const std::vector<std::uint8_t>& frame, bool isNv12);diff --git a/electron/native/wgc-capture/src/frame_visibility_test.cpp b/electron/native/wgc-capture/src/frame_visibility_test.cppnew file mode 100644index 000000000..51e9f7531--- /dev/null+++ b/electron/native/wgc-capture/src/frame_visibility_test.cpp@@ -0,0 +1,66 @@+#include "frame_visibility.h"++#include <cstdio>+#include <string>+#include <vector>++namespace {++int failures = 0;++void expectVisible(const std::string& label, bool actual, bool expected) {+ if (actual == expected) {+ return;+ }+ std::printf("FAIL %s: expected %s, got %s\n", label.c_str(), expected ? "visible" : "blank",+ actual ? "visible" : "blank");+ ++failures;+}++std::vector<std::uint8_t> nv12(int width, int height, std::uint8_t luma) {+ std::vector<std::uint8_t> frame(static_cast<size_t>(width) * height * 3 / 2, 128);+ std::fill(frame.begin(), frame.begin() + static_cast<size_t>(width) * height, luma);+ return frame;+}++std::vector<std::uint8_t> bgra(int width, int height, std::uint8_t value) {+ return std::vector<std::uint8_t>(static_cast<size_t>(width) * height * 4, value);+}++} // namespace++int main() {+ // The regression this file exists for. A camera's NV12 is studio-range, so an+ // all-black frame is a plane of 16, not 0. Averaged raw that is 16 -- past the+ // average threshold on its own -- and every black frame counted as a picture,+ // which is exactly what the warm-up probe exists to rule out.+ expectVisible("studio-range black is blank", hasVisibleNv12Content(nv12(640, 480, 16)), false);+ expectVisible("true zero is blank", hasVisibleNv12Content(nv12(640, 480, 0)), false);++ // Just above black stays blank; a clearly lit frame does not.+ expectVisible("near-black is blank", hasVisibleNv12Content(nv12(640, 480, 20)), false);+ expectVisible("mid grey is visible", hasVisibleNv12Content(nv12(640, 480, 128)), true);+ expectVisible("studio white is visible", hasVisibleNv12Content(nv12(640, 480, 235)), true);++ // BGRA keeps its own behaviour: full-range, black is 0.+ expectVisible("bgra black is blank", hasVisibleBgraContent(bgra(640, 480, 0)), false);+ expectVisible("bgra grey is visible", hasVisibleBgraContent(bgra(640, 480, 128)), true);++ // Buffers too small to carry a frame must not be read past their end.+ expectVisible("empty nv12", hasVisibleNv12Content({}), false);+ expectVisible("empty bgra", hasVisibleBgraContent({}), false);+ expectVisible("runt nv12", hasVisibleNv12Content({16, 16, 16}), false);++ // The dispatcher has to pick the layout it is told, not guess. The same bytes+ // read as BGRA are bright; read as studio-range NV12 they are black.+ const std::vector<std::uint8_t> blackNv12 = nv12(320, 240, 16);+ expectVisible("dispatch nv12", hasVisibleWebcamContent(blackNv12, true), false);+ expectVisible("dispatch bgra", hasVisibleWebcamContent(bgra(320, 240, 200), false), true);++ if (failures == 0) {+ std::printf("frame_visibility_test: all assertions passed\n");+ return 0;+ }+ std::printf("frame_visibility_test: %d assertion(s) failed\n", failures);+ return 1;+}diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cppindex 08fe11aa0..996540de7 100644--- a/electron/native/wgc-capture/src/main.cpp+++ b/electron/native/wgc-capture/src/main.cpp@@ -6,6 +6,7 @@#include "wasapi_device_watcher.h"#include "wasapi_loopback_capture.h"#include "wasapi_render_keepalive.h"+#include "frame_visibility.h"#include "webcam_capture.h"#include "wgc_session.h"@@ -393,66 +394,6 @@ void reportCaptureAdapters(ID3D11Device* device, HMONITOR targetMonitor) {}}-/**- * The NV12 twin of `hasVisibleBgraContent`: luma is the Y plane, one byte per- * pixel, so no colour conversion is needed to judge whether a frame has any- * picture in it. The Y plane is the first two thirds of an NV12 buffer.- */-bool hasVisibleNv12Content(const std::vector<BYTE>& frame) {- if (frame.size() < 6) {- return false;- }-- const size_t lumaCount = frame.size() * 2 / 3;- const size_t step = std::max<size_t>(1, lumaCount / 4096);- uint64_t lumaTotal = 0;- BYTE maxLuma = 0;- size_t sampled = 0;- for (size_t offset = 0; offset < lumaCount; offset += step) {- const BYTE luma = frame[offset];- lumaTotal += luma;- maxLuma = std::max(maxLuma, luma);- sampled += 1;- }-- // The same thresholds the BGRA probe uses. NV12 from a camera is- // studio-range, so a black frame sits at 16 rather than 0 -- which the- // maxLuma > 24 test already tolerates.- const uint64_t averageLuma = sampled > 0 ? lumaTotal / sampled : 0;- return maxLuma > 24 || averageLuma > 4;-}--bool hasVisibleBgraContent(const std::vector<BYTE>& frame);--/** Dispatches to the probe matching the frame's layout. */-bool hasVisibleWebcamContent(const std::vector<BYTE>& frame, bool isNv12) {- return isNv12 ? hasVisibleNv12Content(frame) : hasVisibleBgraContent(frame);-}--bool hasVisibleBgraContent(const std::vector<BYTE>& frame) {- if (frame.size() < 4) {- return false;- }-- uint64_t lumaTotal = 0;- BYTE maxLuma = 0;- const size_t pixelCount = frame.size() / 4;- const size_t step = std::max<size_t>(1, pixelCount / 4096);- size_t sampledPixels = 0;- for (size_t pixel = 0; pixel < pixelCount; pixel += step) {- const size_t offset = pixel * 4;- const BYTE b = frame[offset + 0];- const BYTE g = frame[offset + 1];- const BYTE r = frame[offset + 2];- const BYTE luma = static_cast<BYTE>((static_cast<uint16_t>(r) * 54 + static_cast<uint16_t>(g) * 183 + static_cast<uint16_t>(b) * 19) >> 8);- lumaTotal += luma;- maxLuma = std::max(maxLuma, luma);- sampledPixels += 1;- }-- const uint64_t averageLuma = sampledPixels > 0 ? lumaTotal / sampledPixels : 0;- return maxLuma > 24 || averageLuma > 4;-}bool findBool(const std::string& json, const std::string& key, bool fallback) {auto pos = json.find("\"" + key + "\"");diff --git a/scripts/build-windows-wgc-helper.mjs b/scripts/build-windows-wgc-helper.mjsindex b4e4a59bd..219d03b08 100644--- a/scripts/build-windows-wgc-helper.mjs+++ b/scripts/build-windows-wgc-helper.mjs@@ -115,3 +115,13 @@ if (!fs.existsSync(webcamFormatTestPath)) {// far more -- and the overlay upscales it.await run(webcamFormatTestPath, [], { cwd: BUILD_DIR });console.log(`Passed ${webcamFormatTestPath}`);++const frameVisibilityTestPath = path.join(BUILD_DIR, "frame_visibility_test.exe");+if (!fs.existsSync(frameVisibilityTestPath)) {+ throw new Error(`WGC helper build completed but ${frameVisibilityTestPath} was not found.`);+}+// Guards the warm-up probe that decides whether the camera has produced a+// picture yet. Studio-range black is 16, not 0, so an unnormalised average+// reads every black frame as content.+await run(frameVisibilityTestPath, [], { cwd: BUILD_DIR });+console.log(`Passed ${frameVisibilityTestPath}`);