Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(recording): do not read studio-range black as a visible webcam frame

Christian Schmittelc6d3064be487
diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt
index efa82295d..7a80f3214 100644
--- a/electron/native/wgc-capture/CMakeLists.txt
+++ b/electron/native/wgc-capture/CMakeLists.txt
@@ -41,6 +41,8 @@ add_executable(wgc-capture
src/desktop_icon_cover.cpp
src/desktop_icon_cover.h
src/dpi_awareness.h
+ src/frame_visibility.cpp
+ src/frame_visibility.h
src/dshow_webcam_capture.cpp
src/dshow_webcam_capture.h
src/main.cpp
@@ -133,3 +135,17 @@ target_compile_definitions(webcam_format_test PRIVATE
)
target_compile_options(webcam_format_test PRIVATE /EHsc /W4 /utf-8)
+
+add_executable(frame_visibility_test
+ src/frame_visibility.cpp
+ src/frame_visibility.h
+ src/frame_visibility_test.cpp
+)
+
+target_compile_definitions(frame_visibility_test PRIVATE
+ NOMINMAX
+ WIN32_LEAN_AND_MEAN
+ _WIN32_WINNT=0x0A00
+)
+
+target_compile_options(frame_visibility_test PRIVATE /EHsc /W4 /utf-8)
diff --git a/electron/native/wgc-capture/src/frame_visibility.cpp b/electron/native/wgc-capture/src/frame_visibility.cpp
new file mode 100644
index 000000000..d296b5f0e
--- /dev/null
+++ b/electron/native/wgc-capture/src/frame_visibility.cpp
@@ -0,0 +1,78 @@
+#include "frame_visibility.h"
+
+#include <algorithm>
+
+namespace {
+
+// Sampling at most this many pixels keeps the probe cheap on a 4K frame while
+// still covering it evenly.
+constexpr size_t kMaxSamples = 4096;
+
+// Luma is judged on a full-range 0..255 scale. A frame counts as visible when
+// any sample is clearly above black, or when the picture is dim but not empty.
+constexpr int kMaxLumaThreshold = 24;
+constexpr int kAverageLumaThreshold = 4;
+
+// BT.709/BT.601 studio range: black sits at 16, white at 235.
+constexpr int kStudioBlack = 16;
+constexpr int kStudioSpan = 235 - kStudioBlack;
+
+bool isVisible(uint64_t lumaTotal, int maxLuma, size_t sampled) {
+ const uint64_t averageLuma = sampled > 0 ? lumaTotal / sampled : 0;
+ return maxLuma > kMaxLumaThreshold || averageLuma > kAverageLumaThreshold;
+}
+
+} // namespace
+
+bool hasVisibleBgraContent(const std::vector<std::uint8_t>& frame) {
+ if (frame.size() < 4) {
+ return false;
+ }
+
+ uint64_t lumaTotal = 0;
+ int maxLuma = 0;
+ const size_t pixelCount = frame.size() / 4;
+ const size_t step = std::max<size_t>(1, pixelCount / kMaxSamples);
+ size_t sampled = 0;
+ for (size_t pixel = 0; pixel < pixelCount; pixel += step) {
+ const size_t offset = pixel * 4;
+ const int b = frame[offset + 0];
+ const int g = frame[offset + 1];
+ const int r = frame[offset + 2];
+ const int luma = (r * 54 + g * 183 + b * 19) >> 8;
+ lumaTotal += static_cast<uint64_t>(luma);
+ maxLuma = std::max(maxLuma, luma);
+ sampled += 1;
+ }
+
+ return isVisible(lumaTotal, maxLuma, sampled);
+}
+
+bool hasVisibleNv12Content(const std::vector<std::uint8_t>& frame) {
+ if (frame.size() < 6) {
+ return false;
+ }
+
+ const size_t lumaCount = frame.size() * 2 / 3;
+ const size_t step = std::max<size_t>(1, lumaCount / kMaxSamples);
+ uint64_t lumaTotal = 0;
+ int maxLuma = 0;
+ size_t sampled = 0;
+ for (size_t offset = 0; offset < lumaCount; offset += step) {
+ // Studio range mapped onto 0..255, so the thresholds mean the same here
+ // as they do for BGRA. Skip this and an all-black frame averages 16 --
+ // past the average threshold on its own, so every black frame would
+ // count as a picture.
+ const int studio = static_cast<int>(frame[offset]);
+ const int luma = std::clamp(((studio - kStudioBlack) * 255) / kStudioSpan, 0, 255);
+ lumaTotal += static_cast<uint64_t>(luma);
+ maxLuma = std::max(maxLuma, luma);
+ sampled += 1;
+ }
+
+ return isVisible(lumaTotal, maxLuma, sampled);
+}
+
+bool hasVisibleWebcamContent(const std::vector<std::uint8_t>& frame, bool isNv12) {
+ return isNv12 ? hasVisibleNv12Content(frame) : hasVisibleBgraContent(frame);
+}
diff --git a/electron/native/wgc-capture/src/frame_visibility.h b/electron/native/wgc-capture/src/frame_visibility.h
new file mode 100644
index 000000000..9f62c2298
--- /dev/null
+++ b/electron/native/wgc-capture/src/frame_visibility.h
@@ -0,0 +1,27 @@
+#pragma once
+
+#include <cstdint>
+#include <vector>
+
+/**
+ * Does this frame contain any picture, or is it the blank one a camera emits
+ * while it is still warming up?
+ *
+ * Used before recording starts, to tell "the camera is running" from "the
+ * camera is open but has not produced anything yet".
+ */
+bool hasVisibleBgraContent(const std::vector<std::uint8_t>& frame);
+
+/**
+ * The NV12 twin: luma is the Y plane, one byte per pixel, so no colour
+ * conversion is needed. The Y plane is the first two thirds of the buffer.
+ *
+ * A camera's NV12 is studio-range, where black is 16 rather than 0. Those
+ * values are normalised to full range before the thresholds are applied --
+ * without that, the average of an all-black frame is 16, clears the average
+ * threshold on its own, and every black frame counts as visible.
+ */
+bool hasVisibleNv12Content(const std::vector<std::uint8_t>& frame);
+
+/** Dispatches to the probe matching the frame's layout. */
+bool hasVisibleWebcamContent(const std::vector<std::uint8_t>& frame, bool isNv12);
diff --git a/electron/native/wgc-capture/src/frame_visibility_test.cpp b/electron/native/wgc-capture/src/frame_visibility_test.cpp
new file mode 100644
index 000000000..51e9f7531
--- /dev/null
+++ b/electron/native/wgc-capture/src/frame_visibility_test.cpp
@@ -0,0 +1,66 @@
+#include "frame_visibility.h"
+
+#include <cstdio>
+#include <string>
+#include <vector>
+
+namespace {
+
+int failures = 0;
+
+void expectVisible(const std::string& label, bool actual, bool expected) {
+ if (actual == expected) {
+ return;
+ }
+ std::printf("FAIL %s: expected %s, got %s\n", label.c_str(), expected ? "visible" : "blank",
+ actual ? "visible" : "blank");
+ ++failures;
+}
+
+std::vector<std::uint8_t> nv12(int width, int height, std::uint8_t luma) {
+ std::vector<std::uint8_t> frame(static_cast<size_t>(width) * height * 3 / 2, 128);
+ std::fill(frame.begin(), frame.begin() + static_cast<size_t>(width) * height, luma);
+ return frame;
+}
+
+std::vector<std::uint8_t> bgra(int width, int height, std::uint8_t value) {
+ return std::vector<std::uint8_t>(static_cast<size_t>(width) * height * 4, value);
+}
+
+} // namespace
+
+int main() {
+ // The regression this file exists for. A camera's NV12 is studio-range, so an
+ // all-black frame is a plane of 16, not 0. Averaged raw that is 16 -- past the
+ // average threshold on its own -- and every black frame counted as a picture,
+ // which is exactly what the warm-up probe exists to rule out.
+ expectVisible("studio-range black is blank", hasVisibleNv12Content(nv12(640, 480, 16)), false);
+ expectVisible("true zero is blank", hasVisibleNv12Content(nv12(640, 480, 0)), false);
+
+ // Just above black stays blank; a clearly lit frame does not.
+ expectVisible("near-black is blank", hasVisibleNv12Content(nv12(640, 480, 20)), false);
+ expectVisible("mid grey is visible", hasVisibleNv12Content(nv12(640, 480, 128)), true);
+ expectVisible("studio white is visible", hasVisibleNv12Content(nv12(640, 480, 235)), true);
+
+ // BGRA keeps its own behaviour: full-range, black is 0.
+ expectVisible("bgra black is blank", hasVisibleBgraContent(bgra(640, 480, 0)), false);
+ expectVisible("bgra grey is visible", hasVisibleBgraContent(bgra(640, 480, 128)), true);
+
+ // Buffers too small to carry a frame must not be read past their end.
+ expectVisible("empty nv12", hasVisibleNv12Content({}), false);
+ expectVisible("empty bgra", hasVisibleBgraContent({}), false);
+ expectVisible("runt nv12", hasVisibleNv12Content({16, 16, 16}), false);
+
+ // The dispatcher has to pick the layout it is told, not guess. The same bytes
+ // read as BGRA are bright; read as studio-range NV12 they are black.
+ const std::vector<std::uint8_t> blackNv12 = nv12(320, 240, 16);
+ expectVisible("dispatch nv12", hasVisibleWebcamContent(blackNv12, true), false);
+ expectVisible("dispatch bgra", hasVisibleWebcamContent(bgra(320, 240, 200), false), true);
+
+ if (failures == 0) {
+ std::printf("frame_visibility_test: all assertions passed\n");
+ return 0;
+ }
+ std::printf("frame_visibility_test: %d assertion(s) failed\n", failures);
+ return 1;
+}
diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp
index 08fe11aa0..996540de7 100644
--- a/electron/native/wgc-capture/src/main.cpp
+++ b/electron/native/wgc-capture/src/main.cpp
@@ -6,6 +6,7 @@
#include "wasapi_device_watcher.h"
#include "wasapi_loopback_capture.h"
#include "wasapi_render_keepalive.h"
+#include "frame_visibility.h"
#include "webcam_capture.h"
#include "wgc_session.h"
@@ -393,66 +394,6 @@ void reportCaptureAdapters(ID3D11Device* device, HMONITOR targetMonitor) {
}
}
-/**
- * The NV12 twin of `hasVisibleBgraContent`: luma is the Y plane, one byte per
- * pixel, so no colour conversion is needed to judge whether a frame has any
- * picture in it. The Y plane is the first two thirds of an NV12 buffer.
- */
-bool hasVisibleNv12Content(const std::vector<BYTE>& frame) {
- if (frame.size() < 6) {
- return false;
- }
-
- const size_t lumaCount = frame.size() * 2 / 3;
- const size_t step = std::max<size_t>(1, lumaCount / 4096);
- uint64_t lumaTotal = 0;
- BYTE maxLuma = 0;
- size_t sampled = 0;
- for (size_t offset = 0; offset < lumaCount; offset += step) {
- const BYTE luma = frame[offset];
- lumaTotal += luma;
- maxLuma = std::max(maxLuma, luma);
- sampled += 1;
- }
-
- // The same thresholds the BGRA probe uses. NV12 from a camera is
- // studio-range, so a black frame sits at 16 rather than 0 -- which the
- // maxLuma > 24 test already tolerates.
- const uint64_t averageLuma = sampled > 0 ? lumaTotal / sampled : 0;
- return maxLuma > 24 || averageLuma > 4;
-}
-
-bool hasVisibleBgraContent(const std::vector<BYTE>& frame);
-
-/** Dispatches to the probe matching the frame's layout. */
-bool hasVisibleWebcamContent(const std::vector<BYTE>& frame, bool isNv12) {
- return isNv12 ? hasVisibleNv12Content(frame) : hasVisibleBgraContent(frame);
-}
-
-bool hasVisibleBgraContent(const std::vector<BYTE>& frame) {
- if (frame.size() < 4) {
- return false;
- }
-
- uint64_t lumaTotal = 0;
- BYTE maxLuma = 0;
- const size_t pixelCount = frame.size() / 4;
- const size_t step = std::max<size_t>(1, pixelCount / 4096);
- size_t sampledPixels = 0;
- for (size_t pixel = 0; pixel < pixelCount; pixel += step) {
- const size_t offset = pixel * 4;
- const BYTE b = frame[offset + 0];
- const BYTE g = frame[offset + 1];
- const BYTE r = frame[offset + 2];
- const BYTE luma = static_cast<BYTE>((static_cast<uint16_t>(r) * 54 + static_cast<uint16_t>(g) * 183 + static_cast<uint16_t>(b) * 19) >> 8);
- lumaTotal += luma;
- maxLuma = std::max(maxLuma, luma);
- sampledPixels += 1;
- }
-
- const uint64_t averageLuma = sampledPixels > 0 ? lumaTotal / sampledPixels : 0;
- return maxLuma > 24 || averageLuma > 4;
-}
bool findBool(const std::string& json, const std::string& key, bool fallback) {
auto pos = json.find("\"" + key + "\"");
diff --git a/scripts/build-windows-wgc-helper.mjs b/scripts/build-windows-wgc-helper.mjs
index b4e4a59bd..219d03b08 100644
--- a/scripts/build-windows-wgc-helper.mjs
+++ b/scripts/build-windows-wgc-helper.mjs
@@ -115,3 +115,13 @@ if (!fs.existsSync(webcamFormatTestPath)) {
// far more -- and the overlay upscales it.
await run(webcamFormatTestPath, [], { cwd: BUILD_DIR });
console.log(`Passed ${webcamFormatTestPath}`);
+
+const frameVisibilityTestPath = path.join(BUILD_DIR, "frame_visibility_test.exe");
+if (!fs.existsSync(frameVisibilityTestPath)) {
+ throw new Error(`WGC helper build completed but ${frameVisibilityTestPath} was not found.`);
+}
+// Guards the warm-up probe that decides whether the camera has produced a
+// picture yet. Studio-range black is 16, not 0, so an unnormalised average
+// reads every black frame as content.
+await run(frameVisibilityTestPath, [], { cwd: BUILD_DIR });
+console.log(`Passed ${frameVisibilityTestPath}`);