Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(windows): keep the screen video on the clock when the writer misses ticks (#945)

EtienneLescot4ebe3ce7b070
diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt
index 50466d0d0..7f0cbf02d 100644
--- a/electron/native/wgc-capture/CMakeLists.txt
+++ b/electron/native/wgc-capture/CMakeLists.txt
@@ -41,6 +41,7 @@ add_executable(wgc-capture
src/desktop_icon_cover.cpp
src/desktop_icon_cover.h
src/dpi_awareness.h
+ src/frame_slot_clock.h
src/realtime_scheduling.h
src/frame_visibility.cpp
src/frame_visibility.h
@@ -158,6 +159,19 @@ target_compile_definitions(frame_visibility_test PRIVATE
target_compile_options(frame_visibility_test PRIVATE /EHsc /W4 /utf-8)
+add_executable(frame_slot_clock_test
+ src/frame_slot_clock.h
+ src/frame_slot_clock_test.cpp
+)
+
+target_compile_definitions(frame_slot_clock_test PRIVATE
+ NOMINMAX
+ WIN32_LEAN_AND_MEAN
+ _WIN32_WINNT=0x0A00
+)
+
+target_compile_options(frame_slot_clock_test PRIVATE /EHsc /W4 /utf-8)
+
add_executable(webcam_snapshot_test
src/dshow_webcam_capture.h
src/webcam_snapshot_test.cpp
diff --git a/electron/native/wgc-capture/src/frame_slot_clock.h b/electron/native/wgc-capture/src/frame_slot_clock.h
new file mode 100644
index 000000000..ea28d601d
--- /dev/null
+++ b/electron/native/wgc-capture/src/frame_slot_clock.h
@@ -0,0 +1,45 @@
+#pragma once
+
+#include <algorithm>
+#include <cstdint>
+
+/**
+ * When the next frame of a constant-rate stream is due, on the wall clock.
+ *
+ * Media Foundation's H.264 sink writer numbers its output frames at the
+ * nominal rate whatever SampleTime they carry, so frame N plays at N/fps: the
+ * frame count is the stream's clock. A writer that falls behind and then
+ * resyncs to "now" drops the slots it missed, and every later frame plays
+ * early against the audio, which follows the wall clock
+ * (getopenscreen/openscreen#945). This clock never resyncs: a late writer
+ * finds the next frame already due and writes it at once, until the count
+ * has caught up. Repeating an unchanged screen costs no readback (#925), so
+ * catching up is cheap.
+ *
+ * Times are recording time in 100 ns units: elapsed since the shared start,
+ * pauses excluded, so a pause owes no frames.
+ */
+class FrameSlotClock {
+public:
+ explicit FrameSlotClock(int fps) : fps_(std::max(1, fps)) {}
+
+ // A frame was written, or its slot given up. The first one sets the
+ // origin: the stream starts with its first frame, not before it.
+ void advance(int64_t nowHns) {
+ if (originHns_ < 0) {
+ originHns_ = nowHns;
+ }
+ ++frames_;
+ }
+
+ // Due at once before the first frame. Computed from the origin each time,
+ // never accumulated, so 1/fps rounding cannot drift a long recording.
+ int64_t nextDueHns() const {
+ return originHns_ < 0 ? 0 : originHns_ + frames_ * 10'000'000 / fps_;
+ }
+
+private:
+ int fps_;
+ int64_t originHns_ = -1;
+ int64_t frames_ = 0;
+};
diff --git a/electron/native/wgc-capture/src/frame_slot_clock_test.cpp b/electron/native/wgc-capture/src/frame_slot_clock_test.cpp
new file mode 100644
index 000000000..1b67c8ce5
--- /dev/null
+++ b/electron/native/wgc-capture/src/frame_slot_clock_test.cpp
@@ -0,0 +1,78 @@
+#include "frame_slot_clock.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <cstdio>
+
+namespace {
+
+int failures = 0;
+
+void expectEqual(const char* label, int64_t actual, int64_t expected) {
+ if (actual == expected) {
+ return;
+ }
+ std::printf("FAIL %s: expected %lld, got %lld\n", label, static_cast<long long>(expected),
+ static_cast<long long>(actual));
+ ++failures;
+}
+
+constexpr int64_t kSecondHns = 10'000'000;
+
+// main.cpp's writer loop on a virtual clock: each tick costs `workHns`, one
+// tick stalls `stallHns` more once the clock reaches `stallAtHns`, and the
+// sleep is the same capped wait the writer takes. Returns frames written.
+int64_t writeFor(int fps, int64_t durationHns, int64_t workHns, int64_t stallAtHns, int64_t stallHns) {
+ FrameSlotClock clock(fps);
+ const int64_t periodHns = kSecondHns / fps;
+ int64_t nowHns = 0;
+ int64_t frames = 0;
+ bool stalled = false;
+ while (nowHns < durationHns) {
+ nowHns += workHns;
+ if (!stalled && nowHns >= stallAtHns) {
+ nowHns += stallHns;
+ stalled = true;
+ }
+ ++frames;
+ clock.advance(nowHns);
+ nowHns += std::clamp<int64_t>(clock.nextDueHns() - nowHns, 0, periodHns);
+ }
+ return frames;
+}
+
+} // namespace
+
+int main() {
+ // Nothing written yet: the first frame is due at once, whenever it comes.
+ FrameSlotClock clock(60);
+ expectEqual("first frame due at once", clock.nextDueHns(), 0);
+
+ // The first frame is the origin, not the recording start.
+ clock.advance(5'000'000);
+ expectEqual("origin is the first frame", clock.nextDueHns(), 5'000'000 + 166'666);
+
+ // The regression this file exists for (#945). A writer that misses ticks
+ // owes their frames: the encoder numbers frames at the nominal rate, so
+ // the count is the video's clock. A 300 ms stall one second in, 60 fps
+ // for 3 s from the end of the first tick at 5 ms: 180 frames, the same as
+ // with no stall. The old rule, resyncing to now after the stall, wrote 163:
+ // the video ran 283 ms short, ahead of the audio for the rest of the take.
+ expectEqual("steady writer keeps the count", writeFor(60, 3 * kSecondHns, 50'000, 3 * kSecondHns, 0), 180);
+ expectEqual("stalled writer catches up", writeFor(60, 3 * kSecondHns, 50'000, kSecondHns, 3'000'000), 180);
+
+ // Due times come from the origin, not from adding a truncated 1/fps
+ // (166'666 hns at 60 fps) per frame, which lost 14 ms an hour.
+ FrameSlotClock hour(60);
+ for (int i = 0; i < 60 * 3600; ++i) {
+ hour.advance(0);
+ }
+ expectEqual("no rounding drift over an hour", hour.nextDueHns(), 3600 * kSecondHns);
+
+ if (failures == 0) {
+ std::printf("frame_slot_clock_test: all assertions passed\n");
+ return 0;
+ }
+ std::printf("frame_slot_clock_test: %d assertion(s) failed\n", failures);
+ return 1;
+}
diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp
index 1bad39699..d0962a8d4 100644
--- a/electron/native/wgc-capture/src/main.cpp
+++ b/electron/native/wgc-capture/src/main.cpp
@@ -1,6 +1,7 @@
#include "audio_sample_utils.h"
#include "desktop_icon_cover.h"
#include "dpi_awareness.h"
+#include "frame_slot_clock.h"
#include "realtime_scheduling.h"
#include "mf_encoder.h"
#include "monitor_utils.h"
@@ -1089,10 +1090,22 @@ int wmain(int argc, wchar_t* argv[]) {
// webcam encoder on a real-time-paced cadence (duplicating the
// latest available camera frame when the camera hasn't produced a
// newer one yet), so "sample N is at N/fps" is actually correct.
- int64_t nextWebcamWriteDueHns = 0;
+ // The screen encoder numbers its frames the same way, so both streams
+ // pace to a FrameSlotClock on the recording timeline the audio follows.
+ FrameSlotClock videoClock(config.fps);
+ FrameSlotClock webcamClock(webcamCapture.fps());
const int64_t nominalWebcamIntervalHns =
static_cast<int64_t>(10'000'000ULL / std::max(1, webcamCapture.fps()));
- auto nextFrameDue = std::chrono::steady_clock::now();
+ // Real elapsed host-clock time since the shared recording start,
+ // pauses excluded: not a synthetic frame-index clock, so a long
+ // recording can't accumulate clock-origin drift.
+ using Hns = std::chrono::duration<int64_t, std::ratio<1, 10'000'000>>;
+ auto recordingElapsedHns = [&]() {
+ const auto elapsed = std::chrono::steady_clock::now() - control.recordingStartedAt;
+ return std::max<int64_t>(
+ 0,
+ std::chrono::duration_cast<Hns>(elapsed).count() - control.pausedDurationHns());
+ };
int64_t firstFrameTimestampHns = -1;
int64_t latestFrameTimestampHns = 0;
@@ -1211,15 +1224,8 @@ int wmain(int argc, wchar_t* argv[]) {
lastEncodedVideoTimestampHns + static_cast<int64_t>(10'000'000ULL / config.fps);
}
if (writeSeparateWebcam && webcamFrame.data) {
- // Anchor to the same recording-start origin as screen video/audio,
- // using real elapsed host-clock time (not a synthetic frame-index
- // clock) so a long recording can't accumulate clock-origin drift.
- const auto elapsedSinceStart = std::chrono::steady_clock::now() - control.recordingStartedAt;
- const int64_t elapsedHns = std::chrono::duration_cast<
- std::chrono::duration<int64_t, std::ratio<1, 10'000'000>>>(elapsedSinceStart)
- .count();
- const int64_t targetElapsedHns =
- std::max<int64_t>(0, elapsedHns - control.pausedDurationHns());
+ // Anchored to the same recording-start origin as screen video/audio.
+ const int64_t targetElapsedHns = recordingElapsedHns();
// The H.264 encoder MFT does not honor irregular per-sample
// timestamps for a VFR source -- it numbers output samples
// sequentially at its configured nominal rate regardless of the
@@ -1228,7 +1234,10 @@ int wmain(int argc, wchar_t* argv[]) {
// to feed the encoder *at* that nominal cadence, duplicating
// the latest available camera frame when the camera hasn't
// produced a newer one yet (VFR capture -> CFR encode resampling).
- if (targetElapsedHns >= nextWebcamWriteDueHns) {
+ // One frame per tick: after a stall the writer's ticks come
+ // back to back (see the pacing below), which repays the
+ // slots this stream missed too.
+ if (targetElapsedHns >= webcamClock.nextDueHns()) {
int64_t webcamTimestampHns = targetElapsedHns;
if (lastWebcamTimestampHns >= 0 && webcamTimestampHns <= lastWebcamTimestampHns) {
webcamTimestampHns = lastWebcamTimestampHns + nominalWebcamIntervalHns;
@@ -1251,12 +1260,7 @@ int wmain(int argc, wchar_t* argv[]) {
break;
}
lastWebcamTimestampHns = webcamTimestampHns;
- nextWebcamWriteDueHns += nominalWebcamIntervalHns;
- if (nextWebcamWriteDueHns <= targetElapsedHns) {
- // Fell behind (e.g. coming out of a pause, or a stall) --
- // resync to now instead of trying to catch up frame-by-frame.
- nextWebcamWriteDueHns = targetElapsedHns + nominalWebcamIntervalHns;
- }
+ webcamClock.advance(targetElapsedHns);
}
}
if (testStallReadbackMs > 0) {
@@ -1350,15 +1354,20 @@ int wmain(int argc, wchar_t* argv[]) {
// capturing, converting and encoding a 1080p frame costs ~11 ms, so
// sleeping a whole period on top of it made the real period
// `work + 1/fps` -- 30 fps requested delivered 22.5 measured.
- nextFrameDue += frameDuration;
- const auto now = std::chrono::steady_clock::now();
- if (nextFrameDue < now) {
- // Fell behind (slow frame, or waiting on the first one). Resync
- // to now rather than firing a burst of catch-up frames, same as
- // the webcam cadence above.
- nextFrameDue = now;
- }
- std::this_thread::sleep_until(nextFrameDue);
+ //
+ // A late writer is not resynced to now: the encoder numbers frames
+ // at the nominal rate, so a skipped slot would put every later
+ // frame early against the audio (#945). The next frame is simply
+ // already due, and ticks run back to back until the count has
+ // caught up; on a screen that did not change they are repeats.
+ // Capped at one period so a stop is never waited on longer than
+ // that, whatever the count says.
+ const int64_t nowHns = recordingElapsedHns();
+ videoClock.advance(nowHns);
+ std::this_thread::sleep_for(std::min<std::chrono::steady_clock::duration>(
+ std::chrono::duration_cast<std::chrono::steady_clock::duration>(
+ Hns(videoClock.nextDueHns() - nowHns)),
+ frameDuration));
}
std::cerr << "[pacing] frames=" << frameIndex << " elapsed_ms="
<< std::chrono::duration_cast<std::chrono::milliseconds>(
diff --git a/scripts/build-windows-wgc-helper.mjs b/scripts/build-windows-wgc-helper.mjs
index ab4b26c00..f90c6bad4 100644
--- a/scripts/build-windows-wgc-helper.mjs
+++ b/scripts/build-windows-wgc-helper.mjs
@@ -126,6 +126,16 @@ if (!fs.existsSync(frameVisibilityTestPath)) {
await run(frameVisibilityTestPath, [], { cwd: BUILD_DIR });
console.log(`Passed ${frameVisibilityTestPath}`);
+const frameSlotClockTestPath = path.join(BUILD_DIR, "frame_slot_clock_test.exe");
+if (!fs.existsSync(frameSlotClockTestPath)) {
+ throw new Error(`WGC helper build completed but ${frameSlotClockTestPath} was not found.`);
+}
+// Guards the writer's cadence. The encoder numbers frames at the nominal rate,
+// so a tick the writer misses must still be written or the video runs ahead
+// of the audio for the rest of the take (#945).
+await run(frameSlotClockTestPath, [], { cwd: BUILD_DIR });
+console.log(`Passed ${frameSlotClockTestPath}`);
+
const webcamSnapshotTestPath = path.join(BUILD_DIR, "webcam_snapshot_test.exe");
if (!fs.existsSync(webcamSnapshotTestPath)) {
throw new Error(`WGC helper build completed but ${webcamSnapshotTestPath} was not found.`);