Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(windows): keep the screen video on the clock when the writer misses ticks (#945)
EtienneLescot4ebe3ce7b070
diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txtindex 50466d0d0..7f0cbf02d 100644--- a/electron/native/wgc-capture/CMakeLists.txt+++ b/electron/native/wgc-capture/CMakeLists.txt@@ -41,6 +41,7 @@ add_executable(wgc-capturesrc/desktop_icon_cover.cppsrc/desktop_icon_cover.hsrc/dpi_awareness.h+ src/frame_slot_clock.hsrc/realtime_scheduling.hsrc/frame_visibility.cppsrc/frame_visibility.h@@ -158,6 +159,19 @@ target_compile_definitions(frame_visibility_test PRIVATEtarget_compile_options(frame_visibility_test PRIVATE /EHsc /W4 /utf-8)+add_executable(frame_slot_clock_test+ src/frame_slot_clock.h+ src/frame_slot_clock_test.cpp+)++target_compile_definitions(frame_slot_clock_test PRIVATE+ NOMINMAX+ WIN32_LEAN_AND_MEAN+ _WIN32_WINNT=0x0A00+)++target_compile_options(frame_slot_clock_test PRIVATE /EHsc /W4 /utf-8)+add_executable(webcam_snapshot_testsrc/dshow_webcam_capture.hsrc/webcam_snapshot_test.cppdiff --git a/electron/native/wgc-capture/src/frame_slot_clock.h b/electron/native/wgc-capture/src/frame_slot_clock.hnew file mode 100644index 000000000..ea28d601d--- /dev/null+++ b/electron/native/wgc-capture/src/frame_slot_clock.h@@ -0,0 +1,45 @@+#pragma once++#include <algorithm>+#include <cstdint>++/**+ * When the next frame of a constant-rate stream is due, on the wall clock.+ *+ * Media Foundation's H.264 sink writer numbers its output frames at the+ * nominal rate whatever SampleTime they carry, so frame N plays at N/fps: the+ * frame count is the stream's clock. A writer that falls behind and then+ * resyncs to "now" drops the slots it missed, and every later frame plays+ * early against the audio, which follows the wall clock+ * (getopenscreen/openscreen#945). This clock never resyncs: a late writer+ * finds the next frame already due and writes it at once, until the count+ * has caught up. Repeating an unchanged screen costs no readback (#925), so+ * catching up is cheap.+ *+ * Times are recording time in 100 ns units: elapsed since the shared start,+ * pauses excluded, so a pause owes no frames.+ */+class FrameSlotClock {+public:+ explicit FrameSlotClock(int fps) : fps_(std::max(1, fps)) {}++ // A frame was written, or its slot given up. The first one sets the+ // origin: the stream starts with its first frame, not before it.+ void advance(int64_t nowHns) {+ if (originHns_ < 0) {+ originHns_ = nowHns;+ }+ ++frames_;+ }++ // Due at once before the first frame. Computed from the origin each time,+ // never accumulated, so 1/fps rounding cannot drift a long recording.+ int64_t nextDueHns() const {+ return originHns_ < 0 ? 0 : originHns_ + frames_ * 10'000'000 / fps_;+ }++private:+ int fps_;+ int64_t originHns_ = -1;+ int64_t frames_ = 0;+};diff --git a/electron/native/wgc-capture/src/frame_slot_clock_test.cpp b/electron/native/wgc-capture/src/frame_slot_clock_test.cppnew file mode 100644index 000000000..1b67c8ce5--- /dev/null+++ b/electron/native/wgc-capture/src/frame_slot_clock_test.cpp@@ -0,0 +1,78 @@+#include "frame_slot_clock.h"++#include <algorithm>+#include <cstdint>+#include <cstdio>++namespace {++int failures = 0;++void expectEqual(const char* label, int64_t actual, int64_t expected) {+ if (actual == expected) {+ return;+ }+ std::printf("FAIL %s: expected %lld, got %lld\n", label, static_cast<long long>(expected),+ static_cast<long long>(actual));+ ++failures;+}++constexpr int64_t kSecondHns = 10'000'000;++// main.cpp's writer loop on a virtual clock: each tick costs `workHns`, one+// tick stalls `stallHns` more once the clock reaches `stallAtHns`, and the+// sleep is the same capped wait the writer takes. Returns frames written.+int64_t writeFor(int fps, int64_t durationHns, int64_t workHns, int64_t stallAtHns, int64_t stallHns) {+ FrameSlotClock clock(fps);+ const int64_t periodHns = kSecondHns / fps;+ int64_t nowHns = 0;+ int64_t frames = 0;+ bool stalled = false;+ while (nowHns < durationHns) {+ nowHns += workHns;+ if (!stalled && nowHns >= stallAtHns) {+ nowHns += stallHns;+ stalled = true;+ }+ ++frames;+ clock.advance(nowHns);+ nowHns += std::clamp<int64_t>(clock.nextDueHns() - nowHns, 0, periodHns);+ }+ return frames;+}++} // namespace++int main() {+ // Nothing written yet: the first frame is due at once, whenever it comes.+ FrameSlotClock clock(60);+ expectEqual("first frame due at once", clock.nextDueHns(), 0);++ // The first frame is the origin, not the recording start.+ clock.advance(5'000'000);+ expectEqual("origin is the first frame", clock.nextDueHns(), 5'000'000 + 166'666);++ // The regression this file exists for (#945). A writer that misses ticks+ // owes their frames: the encoder numbers frames at the nominal rate, so+ // the count is the video's clock. A 300 ms stall one second in, 60 fps+ // for 3 s from the end of the first tick at 5 ms: 180 frames, the same as+ // with no stall. The old rule, resyncing to now after the stall, wrote 163:+ // the video ran 283 ms short, ahead of the audio for the rest of the take.+ expectEqual("steady writer keeps the count", writeFor(60, 3 * kSecondHns, 50'000, 3 * kSecondHns, 0), 180);+ expectEqual("stalled writer catches up", writeFor(60, 3 * kSecondHns, 50'000, kSecondHns, 3'000'000), 180);++ // Due times come from the origin, not from adding a truncated 1/fps+ // (166'666 hns at 60 fps) per frame, which lost 14 ms an hour.+ FrameSlotClock hour(60);+ for (int i = 0; i < 60 * 3600; ++i) {+ hour.advance(0);+ }+ expectEqual("no rounding drift over an hour", hour.nextDueHns(), 3600 * kSecondHns);++ if (failures == 0) {+ std::printf("frame_slot_clock_test: all assertions passed\n");+ return 0;+ }+ std::printf("frame_slot_clock_test: %d assertion(s) failed\n", failures);+ return 1;+}diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cppindex 1bad39699..d0962a8d4 100644--- a/electron/native/wgc-capture/src/main.cpp+++ b/electron/native/wgc-capture/src/main.cpp@@ -1,6 +1,7 @@#include "audio_sample_utils.h"#include "desktop_icon_cover.h"#include "dpi_awareness.h"+#include "frame_slot_clock.h"#include "realtime_scheduling.h"#include "mf_encoder.h"#include "monitor_utils.h"@@ -1089,10 +1090,22 @@ int wmain(int argc, wchar_t* argv[]) {// webcam encoder on a real-time-paced cadence (duplicating the// latest available camera frame when the camera hasn't produced a// newer one yet), so "sample N is at N/fps" is actually correct.- int64_t nextWebcamWriteDueHns = 0;+ // The screen encoder numbers its frames the same way, so both streams+ // pace to a FrameSlotClock on the recording timeline the audio follows.+ FrameSlotClock videoClock(config.fps);+ FrameSlotClock webcamClock(webcamCapture.fps());const int64_t nominalWebcamIntervalHns =static_cast<int64_t>(10'000'000ULL / std::max(1, webcamCapture.fps()));- auto nextFrameDue = std::chrono::steady_clock::now();+ // Real elapsed host-clock time since the shared recording start,+ // pauses excluded: not a synthetic frame-index clock, so a long+ // recording can't accumulate clock-origin drift.+ using Hns = std::chrono::duration<int64_t, std::ratio<1, 10'000'000>>;+ auto recordingElapsedHns = [&]() {+ const auto elapsed = std::chrono::steady_clock::now() - control.recordingStartedAt;+ return std::max<int64_t>(+ 0,+ std::chrono::duration_cast<Hns>(elapsed).count() - control.pausedDurationHns());+ };int64_t firstFrameTimestampHns = -1;int64_t latestFrameTimestampHns = 0;@@ -1211,15 +1224,8 @@ int wmain(int argc, wchar_t* argv[]) {lastEncodedVideoTimestampHns + static_cast<int64_t>(10'000'000ULL / config.fps);}if (writeSeparateWebcam && webcamFrame.data) {- // Anchor to the same recording-start origin as screen video/audio,- // using real elapsed host-clock time (not a synthetic frame-index- // clock) so a long recording can't accumulate clock-origin drift.- const auto elapsedSinceStart = std::chrono::steady_clock::now() - control.recordingStartedAt;- const int64_t elapsedHns = std::chrono::duration_cast<- std::chrono::duration<int64_t, std::ratio<1, 10'000'000>>>(elapsedSinceStart)- .count();- const int64_t targetElapsedHns =- std::max<int64_t>(0, elapsedHns - control.pausedDurationHns());+ // Anchored to the same recording-start origin as screen video/audio.+ const int64_t targetElapsedHns = recordingElapsedHns();// The H.264 encoder MFT does not honor irregular per-sample// timestamps for a VFR source -- it numbers output samples// sequentially at its configured nominal rate regardless of the@@ -1228,7 +1234,10 @@ int wmain(int argc, wchar_t* argv[]) {// to feed the encoder *at* that nominal cadence, duplicating// the latest available camera frame when the camera hasn't// produced a newer one yet (VFR capture -> CFR encode resampling).- if (targetElapsedHns >= nextWebcamWriteDueHns) {+ // One frame per tick: after a stall the writer's ticks come+ // back to back (see the pacing below), which repays the+ // slots this stream missed too.+ if (targetElapsedHns >= webcamClock.nextDueHns()) {int64_t webcamTimestampHns = targetElapsedHns;if (lastWebcamTimestampHns >= 0 && webcamTimestampHns <= lastWebcamTimestampHns) {webcamTimestampHns = lastWebcamTimestampHns + nominalWebcamIntervalHns;@@ -1251,12 +1260,7 @@ int wmain(int argc, wchar_t* argv[]) {break;}lastWebcamTimestampHns = webcamTimestampHns;- nextWebcamWriteDueHns += nominalWebcamIntervalHns;- if (nextWebcamWriteDueHns <= targetElapsedHns) {- // Fell behind (e.g. coming out of a pause, or a stall) --- // resync to now instead of trying to catch up frame-by-frame.- nextWebcamWriteDueHns = targetElapsedHns + nominalWebcamIntervalHns;- }+ webcamClock.advance(targetElapsedHns);}}if (testStallReadbackMs > 0) {@@ -1350,15 +1354,20 @@ int wmain(int argc, wchar_t* argv[]) {// capturing, converting and encoding a 1080p frame costs ~11 ms, so// sleeping a whole period on top of it made the real period// `work + 1/fps` -- 30 fps requested delivered 22.5 measured.- nextFrameDue += frameDuration;- const auto now = std::chrono::steady_clock::now();- if (nextFrameDue < now) {- // Fell behind (slow frame, or waiting on the first one). Resync- // to now rather than firing a burst of catch-up frames, same as- // the webcam cadence above.- nextFrameDue = now;- }- std::this_thread::sleep_until(nextFrameDue);+ //+ // A late writer is not resynced to now: the encoder numbers frames+ // at the nominal rate, so a skipped slot would put every later+ // frame early against the audio (#945). The next frame is simply+ // already due, and ticks run back to back until the count has+ // caught up; on a screen that did not change they are repeats.+ // Capped at one period so a stop is never waited on longer than+ // that, whatever the count says.+ const int64_t nowHns = recordingElapsedHns();+ videoClock.advance(nowHns);+ std::this_thread::sleep_for(std::min<std::chrono::steady_clock::duration>(+ std::chrono::duration_cast<std::chrono::steady_clock::duration>(+ Hns(videoClock.nextDueHns() - nowHns)),+ frameDuration));}std::cerr << "[pacing] frames=" << frameIndex << " elapsed_ms="<< std::chrono::duration_cast<std::chrono::milliseconds>(diff --git a/scripts/build-windows-wgc-helper.mjs b/scripts/build-windows-wgc-helper.mjsindex ab4b26c00..f90c6bad4 100644--- a/scripts/build-windows-wgc-helper.mjs+++ b/scripts/build-windows-wgc-helper.mjs@@ -126,6 +126,16 @@ if (!fs.existsSync(frameVisibilityTestPath)) {await run(frameVisibilityTestPath, [], { cwd: BUILD_DIR });console.log(`Passed ${frameVisibilityTestPath}`);+const frameSlotClockTestPath = path.join(BUILD_DIR, "frame_slot_clock_test.exe");+if (!fs.existsSync(frameSlotClockTestPath)) {+ throw new Error(`WGC helper build completed but ${frameSlotClockTestPath} was not found.`);+}+// Guards the writer's cadence. The encoder numbers frames at the nominal rate,+// so a tick the writer misses must still be written or the video runs ahead+// of the audio for the rest of the take (#945).+await run(frameSlotClockTestPath, [], { cwd: BUILD_DIR });+console.log(`Passed ${frameSlotClockTestPath}`);+const webcamSnapshotTestPath = path.join(BUILD_DIR, "webcam_snapshot_test.exe");if (!fs.existsSync(webcamSnapshotTestPath)) {throw new Error(`WGC helper build completed but ${webcamSnapshotTestPath} was not found.`);