Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(recording): keep the Windows screen video on the audio's clock
EtienneLescot13366ebada31
diff --git a/electron/native/wgc-capture/src/frame_slot_clock.h b/electron/native/wgc-capture/src/frame_slot_clock.hindex f13778fef..97913a5e3 100644--- a/electron/native/wgc-capture/src/frame_slot_clock.h+++ b/electron/native/wgc-capture/src/frame_slot_clock.h@@ -4,17 +4,19 @@#include <cstdint>/**- * When the next frame of a constant-rate stream is due, on the wall clock.+ * When the writer's next tick is due, on the recording clock.*- * Media Foundation's H.264 sink writer numbers its output frames at the- * nominal rate whatever SampleTime they carry, so frame N plays at N/fps: the- * frame count is the stream's clock. A writer that falls behind and then- * resyncs to "now" drops the slots it missed, and every later frame plays- * early against the audio, which follows the wall clock- * (getopenscreen/openscreen#945). This clock never resyncs: a late writer- * finds the next frame already due and writes it at once, until the count- * has caught up. The slots it owes repeat the last picture, which costs no- * readback (#925), so catching up is cheap whatever the screen does.+ * The slots are a fixed grid from the first frame: slot N is due at+ * origin + N/fps, computed from the origin each time and never accumulated, so+ * 1/fps rounding cannot drift a long recording.+ *+ * A late tick does not repay the slots it missed. Every sample is stamped with+ * the recording-clock time of its own tick (see main.cpp's writeVideoFrames),+ * and the sink writer keeps those times (MFEncoder::submitVideoSample gives+ * each sample its real duration), so a missed slot is already a+ * correctly timed gap: the previous frame plays through it. Repaying it with a+ * burst of back-to-back ticks would only crowd frames together after the gap.+ * The next tick is due at the first slot after the late one.** Times are recording time in 100 ns units: elapsed since the shared start,* pauses excluded, so a pause owes no frames.@@ -23,59 +25,37 @@ class FrameSlotClock {public:explicit FrameSlotClock(int fps) : fps_(std::max(1, fps)) {}- // A frame was written, or its slot given up. The first one sets the+ // A tick that started at `tickHns` wrote its frame. The first one sets the// origin: the stream starts with its first frame, not before it.- void advance(int64_t nowHns) {+ void advance(int64_t tickHns) {if (originHns_ < 0) {- originHns_ = nowHns;+ originHns_ = tickHns;}- ++frames_;+ const int64_t slot = std::max<int64_t>(0, tickHns - originHns_) * fps_ / 10'000'000;+ slots_ = std::max(slots_ + 1, slot + 1);}- // Due at once before the first frame. Computed from the origin each time,- // never accumulated, so 1/fps rounding cannot drift a long recording.+ // Due at once before the first frame.int64_t nextDueHns() const {- return originHns_ < 0 ? 0 : originHns_ + frames_ * 10'000'000 / fps_;- }-- // Whether the next frame's slot is already in the past at nowHns: the slot- // after it is due too. Such a slot is owed, not live, so it takes the- // picture already encoded instead of a new readback. Catching up with- // readbacks only works while a readback costs less than a period; under- // load a changing screen makes every tick a readback that costs more, the- // count never catches up, and the video plays ever earlier against the audio.- bool isBehind(int64_t nowHns) const {- return originHns_ >= 0 && nowHns >= originHns_ + (frames_ + 1) * 10'000'000 / fps_;+ return originHns_ < 0 ? 0 : originHns_ + slots_ * 10'000'000 / fps_;}private:int fps_;int64_t originHns_ = -1;- int64_t frames_ = 0;+ int64_t slots_ = 0;};/**- * Whether a tick reads the screen back or repeats the last picture.- *- * A changed screen is read back, an unchanged one repeated (#925). A slot- * already in the past (FrameSlotClock::isBehind) repeats either way, and the- * change it skipped stays pending until a live slot reads it: WGC delivers a- * frame only when the screen changes, so a change seen on a past slot and- * followed by a still screen would otherwise never reach the video.+ * The time a screen or webcam sample carries: the recording-clock instant its+ * tick started, on the clock the audio follows. One rule for a new picture+ * and a repeated one alike. Never a count of frames: repeats stamped+ * "last + 1/fps" put a stall's time back on the frame count (#945). Never the+ * WGC frame's SystemRelativeTime either: DWM's presentation estimate, it can+ * lie ahead of the tick that pulls it, and mixing it with tick-stamped repeats+ * made the times jitter by a refresh period. Kept strictly increasing, as the+ * sink writer needs.*/-class PictureReadback {-public:- // `changed`: a new picture arrived this tick. True when this tick must- // read back, which also takes the pending change off the books.- bool readBackNow(bool changed, bool slotIsPast) {- pending_ = pending_ || changed;- if (!pending_ || slotIsPast) {- return false;- }- pending_ = false;- return true;- }--private:- bool pending_ = false;-};+inline int64_t sampleTimeForTick(int64_t tickHns, int64_t lastSampleHns) {+ return lastSampleHns >= 0 && tickHns <= lastSampleHns ? lastSampleHns + 1 : tickHns;+}diff --git a/electron/native/wgc-capture/src/frame_slot_clock_test.cpp b/electron/native/wgc-capture/src/frame_slot_clock_test.cppindex 89727bc44..f8699c262 100644--- a/electron/native/wgc-capture/src/frame_slot_clock_test.cpp+++ b/electron/native/wgc-capture/src/frame_slot_clock_test.cpp@@ -17,95 +17,61 @@ void expectEqual(const char* label, int64_t actual, int64_t expected) {++failures;}-constexpr int64_t kSecondHns = 10'000'000;--// main.cpp's writer loop on a virtual clock: each tick costs `workHns`, one-// tick stalls `stallHns` more once the clock reaches `stallAtHns`, and the-// sleep is the same capped wait the writer takes. Returns frames written.-int64_t writeFor(int fps, int64_t durationHns, int64_t workHns, int64_t stallAtHns, int64_t stallHns) {- FrameSlotClock clock(fps);- const int64_t periodHns = kSecondHns / fps;- int64_t nowHns = 0;- int64_t frames = 0;- bool stalled = false;- while (nowHns < durationHns) {- nowHns += workHns;- if (!stalled && nowHns >= stallAtHns) {- nowHns += stallHns;- stalled = true;- }- ++frames;- clock.advance(nowHns);- nowHns += std::clamp<int64_t>(clock.nextDueHns() - nowHns, 0, periodHns);+void expectTrue(const char* label, bool condition, int64_t detail) {+ if (condition) {+ return;}- return frames;+ std::printf("FAIL %s (%lld)\n", label, static_cast<long long>(detail));+ ++failures;}-struct LoadedTake {- int64_t frames = 0;- // How far the newest picture written ran ahead of the slot it was written- // in: the video shows an event that much before the audio plays it.- int64_t maxPictureLeadHns = 0;+constexpr int64_t kSecondHns = 10'000'000;+constexpr int64_t kMsHns = 10'000;++struct Take {+ int64_t samples = 0;+ int64_t lastSampleHns = -1;+ int64_t minIntervalHns = INT64_MAX;+ int64_t maxIntervalHns = 0;+ bool increasing = true;+ // Where the samples would end if their times were a count of frames at the+ // nominal rate, the rule this replaces for repeated frames (#945).+ int64_t countClockEndHns = 0;};-// The writer under load on a screen that changes every tick: a readback costs-// `readbackHns`, a repeat of the last picture `repeatHns`. With-// `repeatPastSlots`, a slot the clock says is already past repeats instead of-// reading back, through the same PictureReadback main.cpp asks; without it,-// every tick reads back.-LoadedTake writeLoadedFor(int fps, int64_t durationHns, int64_t readbackHns, int64_t repeatHns,- bool repeatPastSlots) {+// main.cpp's screen writer on a virtual recording clock. Each tick costs+// `workHns(tickStart)`; every `dropEvery`-th tick writes nothing (the DXGI path's+// contended frame); the sleep is the writer's: to the next due slot, capped at one+// period. Every sample is stamped by sampleTimeForTick with its tick's start.+template <typename Work>+Take writeTake(int fps, int64_t durationHns, Work workHns, int dropEvery) {FrameSlotClock clock(fps);- PictureReadback readback;const int64_t periodHns = kSecondHns / fps;- LoadedTake take;+ Take take;int64_t nowHns = 0;- int64_t pictureAtHns = 0;+ int64_t ticks = 0;while (nowHns < durationHns) {const int64_t tickStartHns = nowHns;- if (!readback.readBackNow(true, repeatPastSlots && clock.isBehind(tickStartHns))) {- nowHns += repeatHns;- } else {- pictureAtHns = tickStartHns;- nowHns += readbackHns;+ nowHns += workHns(tickStartHns);+ ++ticks;+ if (dropEvery == 0 || ticks % dropEvery != 0) {+ const int64_t sampleHns = sampleTimeForTick(tickStartHns, take.lastSampleHns);+ if (take.lastSampleHns >= 0) {+ const int64_t interval = sampleHns - take.lastSampleHns;+ take.increasing = take.increasing && interval > 0;+ take.minIntervalHns = std::min(take.minIntervalHns, interval);+ take.maxIntervalHns = std::max(take.maxIntervalHns, interval);+ }+ take.lastSampleHns = sampleHns;+ ++take.samples;}- take.maxPictureLeadHns = std::max(take.maxPictureLeadHns, pictureAtHns - clock.nextDueHns());- ++take.frames;clock.advance(tickStartHns);nowHns += std::clamp<int64_t>(clock.nextDueHns() - nowHns, 0, periodHns);}+ take.countClockEndHns = (take.samples - 1) * kSecondHns / fps;return take;}-// Both streams of one take, the screen at 60 fps and the webcam at 30, on the writer's-// rule: each clock is advanced with the instant its tick STARTED, so a stall inside the-// tick (the screen readback) cannot give the two streams different origins. The first-// tick stalls `firstStallHns`. Returns the written durations, screen minus webcam.-int64_t streamsDriftFor(int64_t durationHns, int64_t firstStallHns) {- FrameSlotClock screen(60);- FrameSlotClock webcam(30);- int64_t nowHns = 0;- int64_t screenFrames = 0;- int64_t webcamFrames = 0;- bool stalled = false;- while (nowHns < durationHns) {- const int64_t tickStartHns = nowHns;- if (tickStartHns >= webcam.nextDueHns()) {- ++webcamFrames;- webcam.advance(tickStartHns);- }- nowHns += 50'000;- if (!stalled) {- nowHns += firstStallHns;- stalled = true;- }- ++screenFrames;- screen.advance(tickStartHns);- nowHns += std::clamp<int64_t>(screen.nextDueHns() - nowHns, 0, kSecondHns / 60);- }- return screenFrames * kSecondHns / 60 - webcamFrames * kSecondHns / 30;-}-} // namespaceint main() {@@ -117,63 +83,93 @@ int main() {clock.advance(5'000'000);expectEqual("origin is the first frame", clock.nextDueHns(), 5'000'000 + 166'666);- // The regression this file exists for (#945). A writer that misses ticks- // owes their frames: the encoder numbers frames at the nominal rate, so- // the count is the video's clock. A 300 ms stall one second in, 60 fps- // for 3 s from the end of the first tick at 5 ms: 180 frames, the same as- // with no stall. The old rule, resyncing to now after the stall, wrote 163:- // the video ran 283 ms short, ahead of the audio for the rest of the take.- expectEqual("steady writer keeps the count", writeFor(60, 3 * kSecondHns, 50'000, 3 * kSecondHns, 0), 180);- expectEqual("stalled writer catches up", writeFor(60, 3 * kSecondHns, 50'000, kSecondHns, 3'000'000), 180);-- // Before the first frame nothing is owed, and a writer on time owes only the- // slot it is about to write.- FrameSlotClock owed(60);- expectEqual("nothing owed before the first frame", owed.isBehind(10 * kSecondHns), 0);- owed.advance(0);- expectEqual("live slot is not past", owed.isBehind(166'666), 0);- expectEqual("slot past once the next one is due", owed.isBehind(333'333), 1);-- // A still screen repeats, a changed one reads back once.- PictureReadback readback;- expectEqual("still screen repeats", readback.readBackNow(false, false), 0);- expectEqual("changed screen reads back", readback.readBackNow(true, false), 1);- expectEqual("read change is not read again", readback.readBackNow(false, false), 0);- // A change that lands on a past slot, then a screen that stays still: WGC- // delivers no frame for the live slot, yet the change must reach it.- expectEqual("past slot repeats a changed screen", readback.readBackNow(true, true), 0);- expectEqual("change skipped on a past slot is read on the live one",- readback.readBackNow(false, false), 1);- expectEqual("and only once", readback.readBackNow(false, false), 0);-- // The v2.1.0 candidate's take (#945 follow-up): 10 min at 60 fps under a- // CPU/GPU load, screen 217 ms short of the webcam and the audio at the end.- // A readback that costs 20 ms on a changing screen, against a 16.7 ms- // period: catching up with readbacks wrote 30000 frames, 100 s of video- // played early. Repeating the past slots writes all 36000, and no picture- // runs more than one period ahead of its slot.+ // A tick 2.5 periods late skips the slots it missed: the next one is due at the+ // first slot after it, not at once (no catch-up burst).+ clock.advance(5'000'000 + 416'666);+ expectEqual("a late tick skips the missed slots", clock.nextDueHns(), 5'000'000 + 500'000);++ // A tick a hair early still moves on by one slot: the clock never stands still.+ FrameSlotClock early(60);+ early.advance(0);+ early.advance(166'000);+ expectEqual("an early tick still advances one slot", early.nextDueHns(), 333'333);++ // Samples carry their tick's time, kept strictly increasing.+ expectEqual("a sample carries its tick's time", sampleTimeForTick(500, 400), 500);+ expectEqual("the first sample carries its tick's time", sampleTimeForTick(0, -1), 0);+ expectEqual("a sample never repeats the last time", sampleTimeForTick(400, 400), 401);++ // The regression this file exists for. Ten minutes at 60 fps under the load the+ // report measured: a 300 ms stall at 1:00, then 2:30 of ticks that cost 25 ms+ // (more than a period) from 5:00, and one contended frame in every 97 dropped.+ // The video must end on the recording clock, within a few frames of the take's+ // length (the last tick may be a dropped one), with the stall left as a gap of+ // its own length and no burst after it.const int64_t tenMinutesHns = 600 * kSecondHns;- const LoadedTake readbackEveryTick = writeLoadedFor(60, tenMinutesHns, 200'000, 40'000, false);- expectEqual("readback catch-up falls behind under load", readbackEveryTick.frames, 30000);- const LoadedTake repeatPast = writeLoadedFor(60, tenMinutesHns, 200'000, 40'000, true);- expectEqual("repeated past slots keep the count under load", repeatPast.frames, 36000);- expectEqual("no picture leads its slot by more than a period",- repeatPast.maxPictureLeadHns <= kSecondHns / 60, 1);- // Only just over budget, as on that machine: still the full count.- expectEqual("readback just over a period keeps the count",- writeLoadedFor(60, tenMinutesHns, 170'000, 40'000, true).frames, 36000);-- // A first tick that stalls 300 ms must not set the screen's origin 300 ms after the- // webcam's: both streams of a 3 s take come out within one webcam frame of each other.- const int64_t drift = streamsDriftFor(3 * kSecondHns, 3'000'000);- expectEqual("screen and webcam stay aligned after a first-tick stall",- drift > -kSecondHns / 30 && drift < kSecondHns / 30, 1);+ auto load = [](int64_t tickStartHns) -> int64_t {+ if (tickStartHns >= 60 * kSecondHns && tickStartHns < 60 * kSecondHns + 5 * kMsHns) {+ return 300 * kMsHns;+ }+ if (tickStartHns >= 300 * kSecondHns && tickStartHns < 450 * kSecondHns) {+ return 25 * kMsHns;+ }+ return 5 * kMsHns;+ };+ const Take loaded = writeTake(60, tenMinutesHns, load, 97);+ expectTrue("sample times strictly increase", loaded.increasing, loaded.samples);+ expectTrue("the video ends on the recording clock",+ loaded.lastSampleHns > tenMinutesHns - 50 * kMsHns &&+ loaded.lastSampleHns <= tenMinutesHns,+ loaded.lastSampleHns);+ expectTrue("the stall is a gap of its own length", loaded.maxIntervalHns >= 300 * kMsHns,+ loaded.maxIntervalHns);+ expectTrue("no burst of frames after a stall", loaded.minIntervalHns >= 5 * kMsHns,+ loaded.minIntervalHns);+ // The same take counted at the nominal rate would end seconds short: the overload+ // alone writes ~40 frames a second for 2:30. The scenario has teeth.+ expectTrue("a frame count would have lost the overload",+ loaded.lastSampleHns - loaded.countClockEndHns > 30 * kSecondHns,+ loaded.lastSampleHns - loaded.countClockEndHns);++ // An idle writer keeps the nominal cadence: 3 s at 60 fps is 180 samples.+ const Take idle = writeTake(60, 3 * kSecondHns, [](int64_t) -> int64_t { return 50'000; }, 0);+ expectEqual("steady writer keeps the cadence", idle.samples, 180);++ // Screen at 60 and webcam at 30 in one writer, both stamped with the tick's+ // start and the first tick stalling 300 ms: the two streams end within one+ // webcam frame of each other, on the same clock.+ {+ FrameSlotClock screen(60);+ FrameSlotClock webcam(30);+ int64_t nowHns = 0;+ int64_t lastScreenHns = -1;+ int64_t lastWebcamHns = -1;+ bool stalled = false;+ while (nowHns < 3 * kSecondHns) {+ const int64_t tickStartHns = nowHns;+ if (tickStartHns >= webcam.nextDueHns()) {+ lastWebcamHns = sampleTimeForTick(tickStartHns, lastWebcamHns);+ webcam.advance(tickStartHns);+ }+ nowHns += 50'000;+ if (!stalled) {+ nowHns += 3'000'000;+ stalled = true;+ }+ lastScreenHns = sampleTimeForTick(tickStartHns, lastScreenHns);+ screen.advance(tickStartHns);+ nowHns += std::clamp<int64_t>(screen.nextDueHns() - nowHns, 0, kSecondHns / 60);+ }+ const int64_t drift = lastScreenHns - lastWebcamHns;+ expectTrue("screen and webcam stay aligned after a first-tick stall",+ drift > -kSecondHns / 30 && drift < kSecondHns / 30, drift);+ }// Due times come from the origin, not from adding a truncated 1/fps// (166'666 hns at 60 fps) per frame, which lost 14 ms an hour.FrameSlotClock hour(60);- for (int i = 0; i < 60 * 3600; ++i) {- hour.advance(0);+ for (int64_t i = 0; i < 60 * 3600; ++i) {+ hour.advance(hour.nextDueHns());}expectEqual("no rounding drift over an hour", hour.nextDueHns(), 3600 * kSecondHns);diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cppindex 545a1f9e4..6d317cb93 100644--- a/electron/native/wgc-capture/src/main.cpp+++ b/electron/native/wgc-capture/src/main.cpp@@ -1026,16 +1026,14 @@ int wmain(int argc, wchar_t* argv[]) {uint64_t latestWebcamSequence = 0;bool hasVisibleWebcamFrame = false;- // Legacy-path-only state. frameMutex guards latestFrameTexture/- // legacyLatestFrameTimestampHns between WGC's callback thread (writer)- // and writeVideoFrames (reader); frameCv wakes the reader. Both are- // unused on the default pull-based path.+ // Legacy-path-only state. frameMutex guards latestFrameTexture between+ // WGC's callback thread (writer) and writeVideoFrames (reader); frameCv+ // wakes the reader. Both are unused on the default pull-based path.std::timed_mutex frameMutex;std::condition_variable_any frameCv;- int64_t legacyLatestFrameTimestampHns = 0;if (legacyFrameCallback) {- session.setFrameCallback([&](ID3D11Texture2D* texture, int64_t timestampHns) {+ session.setFrameCallback([&](ID3D11Texture2D* texture, int64_t /*timestampHns*/) {if (control.stopRequested || control.paused) {return;}@@ -1066,7 +1064,6 @@ int wmain(int argc, wchar_t* argv[]) {std::this_thread::sleep_for(std::chrono::milliseconds(testStallFrameCallbackMs));}session.context()->CopyResource(latestFrameTexture.Get(), texture);- legacyLatestFrameTimestampHns = timestampHns;if (!firstFrameWritten.exchange(true)) {frameCv.notify_all();}@@ -1080,22 +1077,14 @@ int wmain(int argc, wchar_t* argv[]) {uint64_t frameIndex = 0;int64_t lastEncodedVideoTimestampHns = -1;int64_t lastWebcamTimestampHns = -1;- // Media Foundation's H.264 encoder MFT does not honor irregular input- // sample times for a VFR source: it numbers output samples- // sequentially at its configured nominal frame rate regardless of the- // SampleTime we attach (confirmed empirically -- varying, correctly- // increasing input timestamps still produced perfectly even output- // spacing). Since we cannot make the encoder respect real capture- // time, we instead make the encoder's assumption true: feed the- // webcam encoder on a real-time-paced cadence (duplicating the- // latest available camera frame when the camera hasn't produced a- // newer one yet), so "sample N is at N/fps" is actually correct.- // The screen encoder numbers its frames the same way, so both streams- // pace to a FrameSlotClock on the recording timeline the audio follows.+ // Both streams are written on a real-time cadence, one FrameSlotClock+ // each, repeating the latest picture when the source has nothing newer+ // (a static screen, a camera slower than its nominal rate). Every+ // sample carries the recording-clock time of the tick that wrote it+ // (sampleTimeForTick): the sink writer keeps those times, so the video+ // stays on the clock the audio follows however late a tick runs.FrameSlotClock videoClock(config.fps);FrameSlotClock webcamClock(webcamCapture.fps());- const int64_t nominalWebcamIntervalHns =- static_cast<int64_t>(10'000'000ULL / std::max(1, webcamCapture.fps()));// Real elapsed host-clock time since the shared recording start,// pauses excluded: not a synthetic frame-index clock, so a long// recording can't accumulate clock-origin drift.@@ -1106,18 +1095,11 @@ int wmain(int argc, wchar_t* argv[]) {0,std::chrono::duration_cast<Hns>(elapsed).count() - control.pausedDurationHns());};- int64_t firstFrameTimestampHns = -1;- int64_t latestFrameTimestampHns = 0;- PictureReadback pictureReadback;-while (!control.stopRequested && !encodeFailed) {// One instant per tick for both clocks: read after the screen readback, the// screen's would trail the webcam's by any stall in it, and the first tick// would give the two streams different origins.const int64_t tickStartHns = recordingElapsedHns();- // A writer behind by more than this slot fills it with the last- // picture: only the latest owed slot gets a readback (#945).- const bool videoSlotIsPast = videoClock.isBehind(tickStartHns);Microsoft::WRL::ComPtr<IMFSample> videoSample;Microsoft::WRL::ComPtr<IMFSample> webcamSample;bool hasVideoSample = false;@@ -1154,7 +1136,6 @@ int wmain(int argc, wchar_t* argv[]) {if (!latestFrameTexture) {continue;}- latestFrameTimestampHns = legacyLatestFrameTimestampHns;} else {if (control.paused) {std::this_thread::sleep_for(std::chrono::milliseconds(100));@@ -1186,7 +1167,6 @@ int wmain(int argc, wchar_t* argv[]) {// separate WGC callback thread left for it to take a// lock down with it.session.context()->CopyResource(latestFrameTexture.Get(), wgcTexture);- latestFrameTimestampHns = wgcTimestampHns;firstFrameWritten = true;} else if (!latestFrameTexture) {// No frame captured yet at all: nothing to encode@@ -1215,41 +1195,16 @@ int wmain(int argc, wchar_t* argv[]) {latestWebcamWidth,latestWebcamHeight,};- const int64_t syntheticTimestampHns =- static_cast<int64_t>((frameIndex * 10'000'000ULL) / config.fps);- const int64_t sourceTimestampHns =- latestFrameTimestampHns > 0 ? latestFrameTimestampHns : syntheticTimestampHns;- if (firstFrameTimestampHns < 0) {- firstFrameTimestampHns = sourceTimestampHns;- }- int64_t frameTimestampHns =- std::max<int64_t>(- 0,- sourceTimestampHns - firstFrameTimestampHns - control.pausedDurationHns());- if (lastEncodedVideoTimestampHns >= 0 &&- frameTimestampHns <= lastEncodedVideoTimestampHns) {- frameTimestampHns =- lastEncodedVideoTimestampHns + static_cast<int64_t>(10'000'000ULL / config.fps);- }+ // The tick's own time, not the WGC frame's: see sampleTimeForTick.+ const int64_t frameTimestampHns =+ sampleTimeForTick(tickStartHns, lastEncodedVideoTimestampHns);if (writeSeparateWebcam && webcamFrame.data) {- // Anchored to the same recording-start origin as screen video/audio.- const int64_t targetElapsedHns = tickStartHns;- // The H.264 encoder MFT does not honor irregular per-sample- // timestamps for a VFR source -- it numbers output samples- // sequentially at its configured nominal rate regardless of the- // SampleTime attached to each input sample. So the only way to- // keep the encoded webcam file in sync with real elapsed time is- // to feed the encoder *at* that nominal cadence, duplicating- // the latest available camera frame when the camera hasn't- // produced a newer one yet (VFR capture -> CFR encode resampling).- // One frame per tick: after a stall the writer's ticks come- // back to back (see the pacing below), which repays the- // slots this stream missed too.- if (targetElapsedHns >= webcamClock.nextDueHns()) {- int64_t webcamTimestampHns = targetElapsedHns;- if (lastWebcamTimestampHns >= 0 && webcamTimestampHns <= lastWebcamTimestampHns) {- webcamTimestampHns = lastWebcamTimestampHns + nominalWebcamIntervalHns;- }+ // On the same clock and tick instant as the screen, so the two+ // streams share an origin. One frame per due slot, repeating the+ // latest camera frame when the camera has nothing newer.+ if (tickStartHns >= webcamClock.nextDueHns()) {+ const int64_t webcamTimestampHns =+ sampleTimeForTick(tickStartHns, lastWebcamTimestampHns);// Capture the sample here, but submit it to the sink// writer OUTSIDE this block below (issue #115) so a// slow WriteSample can't hold up the next frame pull.@@ -1268,7 +1223,7 @@ int wmain(int argc, wchar_t* argv[]) {break;}lastWebcamTimestampHns = webcamTimestampHns;- webcamClock.advance(targetElapsedHns);+ webcamClock.advance(tickStartHns);}}if (testStallReadbackMs > 0) {@@ -1292,17 +1247,10 @@ int wmain(int argc, wchar_t* argv[]) {// this struct's request: it falls back to the CPU path on// its own when the GPU path does not fit the machine.bool captured = false;- if (!usesDxgiInput &&- !pictureReadback.readBackNow(pictureChanged, videoSlotIsPast) &&+ if (!usesDxgiInput && !pictureChanged &&encoder.repeatLastVideoSample(frameTimestampHns, videoSample)) {// Nothing new to read back (#925): the static screen,// most of a demo, costs no GPU copy and no conversion.- // A slot already in the past repeats too, changed- // screen or not: under load a readback can cost more- // than a period, and a writer that catches up with- // readbacks then never catches up at all (#945). The- // new picture waits in latestFrameTexture, pending,- // for the first slot that is live.captured = true;} else if (usesDxgiInput) {captured = encoder.captureDxgiSample(@@ -1370,14 +1318,10 @@ int wmain(int argc, wchar_t* argv[]) {// sleeping a whole period on top of it made the real period// `work + 1/fps` -- 30 fps requested delivered 22.5 measured.//- // A late writer is not resynced to now: the encoder numbers frames- // at the nominal rate, so a skipped slot would put every later- // frame early against the audio (#945). The next frame is simply- // already due, and ticks run back to back until the count has- // caught up. Every slot but the live one is a repeat, which costs- // an encode and no readback, so the count does catch up.- // Capped at one period so a stop is never waited on longer than- // that, whatever the count says.+ // A late tick skips the slots it missed rather than repaying them:+ // its frame carries its own time, so the gap is already correctly+ // timed (see FrameSlotClock). Capped at one period so a stop is+ // never waited on longer than that.const int64_t nowHns = recordingElapsedHns();videoClock.advance(tickStartHns);std::this_thread::sleep_for(std::min<std::chrono::steady_clock::duration>(@@ -1389,7 +1333,7 @@ int wmain(int argc, wchar_t* argv[]) {<< std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - control.recordingStartedAt).count()- << std::endl;+ << " last_sample_ms=" << lastEncodedVideoTimestampHns / 10'000 << std::endl;};std::thread videoWriterThread;diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cppindex 500ff5008..09788bde1 100644--- a/electron/native/wgc-capture/src/mf_encoder.cpp+++ b/electron/native/wgc-capture/src/mf_encoder.cpp@@ -1909,9 +1909,33 @@ bool MFEncoder::submitVideoSample(IMFSample* sample) {if (!sinkWriter_ || finalized_) {return false;}+ // Each sample is written one submission late, once the next one says how+ // long it really lasts. The fragmented MP4 sink closes every fragment on+ // its last sample's SampleDuration and starts the next fragment right after+ // it, so a nominal 1/fps there replaced the real gap to the next frame once+ // a fragment: a late tick, or a 56 Hz desktop sampled at 60, lost the+ // difference for good, and the screen fell 217 ms behind the audio over a+ // ten-minute take (#1164).+ LONGLONG pendingTime = 0;+ LONGLONG nextTime = 0;+ if (pendingVideoSample_ && SUCCEEDED(pendingVideoSample_->GetSampleTime(&pendingTime)) &&+ SUCCEEDED(sample->GetSampleTime(&nextTime)) && nextTime > pendingTime) {+ pendingVideoSample_->SetSampleDuration(nextTime - pendingTime);+ }+ const bool written = writePendingVideoSample();+ pendingVideoSample_ = sample;+ return written;+}++bool MFEncoder::writePendingVideoSample() {+ if (!pendingVideoSample_) {+ return true;+ }encodeStage_ = "write-sample";- const bool written = succeeded(sinkWriter_->WriteSample(videoStreamIndex_, sample), "WriteSample");+ const bool written =+ succeeded(sinkWriter_->WriteSample(videoStreamIndex_, pendingVideoSample_.Get()), "WriteSample");encodeStage_ = "idle";+ pendingVideoSample_.Reset();return written;}@@ -1981,8 +2005,11 @@ bool MFEncoder::finalize() {finalized_ = true;bool ok = true;if (sinkWriter_) {- ok = succeeded(sinkWriter_->Finalize(), "SinkWriter::Finalize");+ // The last frame keeps its nominal 1/fps: nothing follows it.+ ok = writePendingVideoSample();+ ok = succeeded(sinkWriter_->Finalize(), "SinkWriter::Finalize") && ok;}+ pendingVideoSample_.Reset();releaseSinkWriter();stagingTexture_.Reset();// Before MFShutdown(), not left to the destructor. Two of the objectsdiff --git a/electron/native/wgc-capture/src/mf_encoder.h b/electron/native/wgc-capture/src/mf_encoder.hindex 455624135..905b66ca9 100644--- a/electron/native/wgc-capture/src/mf_encoder.h+++ b/electron/native/wgc-capture/src/mf_encoder.h@@ -217,6 +217,8 @@ private:// a fragmented sink has all its streams before anything can be added to it.bool configureAudioStream(const AudioInputFormat& audioFormat, const MFEncoderOptions& options);void releaseSinkWriter();+ // Writes and drops pendingVideoSample_. Under writerMutex_.+ bool writePendingVideoSample();Microsoft::WRL::ComPtr<IMFSinkWriter> sinkWriter_;// Held only on the fragmented path, and held because a sink writer built on@@ -263,6 +265,9 @@ private:// conversion; reused so a frame does not allocate one.std::vector<BYTE> bgraScratch_;Microsoft::WRL::ComPtr<IMFMediaBuffer> lastVideoBuffer_;+ // The video sample submitted last, held back until the next one gives it+ // its real duration (see submitVideoSample). Under writerMutex_.+ Microsoft::WRL::ComPtr<IMFSample> pendingVideoSample_;int width_ = 0;int height_ = 0;int fps_ = 60;diff --git a/electron/native/wgc-capture/src/mf_encoder_color_test.cpp b/electron/native/wgc-capture/src/mf_encoder_color_test.cppindex 1cc20d6b0..d9278b406 100644--- a/electron/native/wgc-capture/src/mf_encoder_color_test.cpp+++ b/electron/native/wgc-capture/src/mf_encoder_color_test.cpp@@ -358,6 +358,64 @@ void checkRepeatedFrames(ID3D11Device* device, ID3D11DeviceContext* context) {}}+// Every frame must play at the time it was stamped with, across fragment+// boundaries too. The fragmented sink closes each fragment on its last sample's+// SampleDuration and starts the next one right after it, so a nominal 1/fps+// there dropped whatever the real gap held beyond it: a writer running late at+// 40 fps, or a stall, lost time at every fragment and the video fell behind the+// audio (217 ms over a ten-minute take, #1164).+void checkSampleTimesKept(ID3D11Device* device, ID3D11DeviceContext* context, bool software) {+ const std::string label = software ? "software" : "default";+ char tempDir[MAX_PATH]{};+ GetTempPathA(MAX_PATH, tempDir);+ const std::string path = std::string(tempDir) + "openscreen-mf-encoder-times-" + label + ".mp4";+ DeleteFileA(path.c_str());++ // 60 fps nominal, written every 25 ms for 4 s, with a 300 ms stall at 2 s.+ std::vector<int64_t> stamps;+ for (int64_t t = 0; t < 40'000'000; t += 250'000) {+ stamps.push_back(t < 20'000'000 ? t : t + 3'000'000);+ }+ {+ std::vector<BYTE> bgra(static_cast<size_t>(kWidth) * kHeight * 4, 128);+ const BgraFrameView frame{bgra.data(), kWidth, kHeight};+ MFEncoder encoder;+ MFEncoderOptions options;+ options.preferSoftwareEncoder = software;+ if (!encoder.initialize(widen(path), kWidth, kHeight, 60, 2'000'000, device, context, nullptr, options)) {+ expect("sample-times-initialize-" + label, false, "initialize failed");+ return;+ }+ bool wrote = true;+ for (size_t i = 0; i < stamps.size() && wrote; i += 1) {+ Microsoft::WRL::ComPtr<IMFSample> sample;+ wrote = encoder.captureBgraSample(frame, stamps[i], sample) && encoder.submitVideoSample(sample.Get());+ }+ expect("sample-times-writes-" + label, wrote && encoder.finalize(), "write or finalize failed");+ }++ const std::string probe =+ run("ffprobe -v error -select_streams v:0 -show_entries packet=pts_time -of csv=p=0 \"" + path + "\"");+ std::vector<double> pts;+ for (size_t at = 0; at < probe.size();) {+ const size_t end = probe.find_first_of("\r\n", at);+ const std::string line = probe.substr(at, end == std::string::npos ? std::string::npos : end - at);+ if (!line.empty()) {+ pts.push_back(std::atof(line.c_str()));+ }+ at = end == std::string::npos ? probe.size() : end + 1;+ }+ std::sort(pts.begin(), pts.end());+ double worstMs = pts.size() == stamps.size() ? 0.0 : 1e9;+ for (size_t i = 0; i < pts.size() && i < stamps.size(); i += 1) {+ worstMs = std::max(worstMs, std::abs(pts[i] * 1000.0 - stamps[i] / 10'000.0));+ }+ std::cout << "TIMES_RAW " << label << " frames=" << pts.size() << "/" << stamps.size()+ << " worst pts error " << worstMs << " ms\n";+ expect("sample-times-kept-" + label, worstMs < 0.5, std::to_string(worstMs) + " ms");+ DeleteFileA(path.c_str());+}+} // namespaceint main() {@@ -386,6 +444,8 @@ int main() {checkEncoder(device.Get(), context.Get(), false);timeFullHd(device.Get(), context.Get());checkRepeatedFrames(device.Get(), context.Get());+ checkSampleTimesKept(device.Get(), context.Get(), true);+ checkSampleTimesKept(device.Get(), context.Get(), false);std::cout << "ran " << g_ran << " tests\n";if (g_failed != 0) {diff --git a/scripts/build-windows-wgc-helper.mjs b/scripts/build-windows-wgc-helper.mjsindex 3fb234d71..90b961ee7 100644--- a/scripts/build-windows-wgc-helper.mjs+++ b/scripts/build-windows-wgc-helper.mjs@@ -158,9 +158,9 @@ const frameSlotClockTestPath = path.join(BUILD_DIR, "frame_slot_clock_test.exe")if (!fs.existsSync(frameSlotClockTestPath)) {throw new Error(`WGC helper build completed but ${frameSlotClockTestPath} was not found.`);}-// Guards the writer's cadence. The encoder numbers frames at the nominal rate,-// so a tick the writer misses must still be written or the video runs ahead-// of the audio for the rest of the take (#945).+// Guards the writer's cadence and its sample times: every frame carries the+// recording-clock time of its tick, so stalls and overload leave correctly+// timed gaps and the video stays on the audio's clock (#945).await run(frameSlotClockTestPath, [], { cwd: BUILD_DIR });console.log(`Passed ${frameSlotClockTestPath}`);