Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(windows): keep the mixer's cushion across an instant resume and anchor its clock exactly
EtienneLescot025b2b2af993
diff --git a/electron/native/wgc-capture/src/audio_sample_utils.cpp b/electron/native/wgc-capture/src/audio_sample_utils.cppindex 48b10d154..cab9aab61 100644--- a/electron/native/wgc-capture/src/audio_sample_utils.cpp+++ b/electron/native/wgc-capture/src/audio_sample_utils.cpp@@ -541,20 +541,37 @@ void AudioMixer::beginTimeline() {std::scoped_lock lock(mutex_);resetSources();emittedFrames_ = 0;+ mixedFrames_ = 0;timelineStarted_ = true;+ // Here, not when mixLoop next wakes: the video's T0 is taken right after+ // this call, and every millisecond the mixer took to wake would have+ // placed the whole track that much early -- tens of them on a busy machine.+ clockStart_ = std::chrono::steady_clock::now();+ clockAnchored_ = true;}cv_.notify_all();}void AudioMixer::setPaused(bool paused) {{- std::scoped_lock lock(mutex_);+ std::unique_lock lock(mutex_);if (paused && !paused_) {// The queues still hold what the cushion kept back; mixLoop writes it// up to this instant, and the resume clears whatever is left.pausedAt_ = std::chrono::steady_clock::now();+ pauseFlushed_ = !timelineStarted_;} else if (!paused && paused_) {+ // A resume right behind the pause would otherwise clear the cushion+ // before mixLoop has written it: the last 100 ms of voice before the+ // pause, gone (measured, 80 to 90 ms).+ cv_.wait(lock, [this] { return pauseFlushed_ || stopRequested_.load(); });resetSources();+ // Resumed at this instant, where the pause left off: the flush has+ // run, so `emittedFrames_` is the pause point and mixLoop is idle.+ clockStart_ = std::chrono::steady_clock::now() -+ std::chrono::duration_cast<std::chrono::steady_clock::duration>(std::chrono::duration<double>(+ static_cast<double>(emittedFrames_) / format_.sampleRate));+ clockAnchored_ = true;}paused_ = paused;}@@ -622,11 +639,18 @@ void AudioMixer::append(convertAudioWithGain(data, byteCount, sourceFormat, format_, gain, gainBuffer_, decimator);// A source that ran dry is coming back: loopback after a silence, or a device// that stalled for longer than the cushion. Its packet belongs at now, which- // is a cushion ahead of the chunk mixLoop writes next -- queued at the front,- // it would land that much early.+ // is as far ahead of what mixLoop has taken as real time is -- the cushion+ // plus however late mixLoop is running, which on a loaded machine is tens of+ // milliseconds. Queued at the front, it would land that much early.if (starved) {- queue.assign(- static_cast<size_t>(format_.sampleRate) * MixerCushionMs / 1000 * format_.blockAlign, 0);+ uint64_t lagFrames = 0;+ if (clockAnchored_) {+ const double elapsed =+ std::chrono::duration<double>(std::chrono::steady_clock::now() - clockStart_).count();+ const auto nowFrames = static_cast<uint64_t>(std::max(0.0, elapsed) * format_.sampleRate);+ lagFrames = nowFrames > mixedFrames_ ? nowFrames - mixedFrames_ : 0;+ }+ queue.assign(static_cast<size_t>(lagFrames) * format_.blockAlign, 0);starved = false;}queue.insert(queue.end(), gainBuffer_.begin(), gainBuffer_.end());@@ -676,7 +700,7 @@ bool AudioMixer::pop(* still come from `emittedFrames_` -- so the cushion only delays the writing,* and a pause or a stop writes what it still holds up to that instant.*- * `audioClockStart` is anchored so that `emittedFrames_` always describes the+ * `clockStart_` is anchored so that `emittedFrames_` always describes the* time elapsed since the timeline began; re-deriving it on resume is what lets a* pause interrupt the clock without shifting everything recorded after it.*/@@ -686,15 +710,16 @@ void AudioMixer::mixLoop() {const uint64_t cushionFrames = static_cast<uint64_t>(format_.sampleRate) * MixerCushionMs / 1000;std::vector<BYTE> mixedChunk;std::vector<BYTE> sourceChunk;- std::chrono::steady_clock::time_point audioClockStart;- bool audioClockAnchored = false;+ // This loop's copy of `clockStart_`, taken under the lock at the top of each+ // pass: beginTimeline and a resume move it from other threads.+ std::chrono::steady_clock::time_point clockStart;const auto framesToDuration = [&](uint64_t frames) {return std::chrono::duration_cast<std::chrono::steady_clock::duration>(std::chrono::duration<double>(static_cast<double>(frames) / format_.sampleRate));};const auto framesAt = [&](std::chrono::steady_clock::time_point time) {- const double elapsed = std::chrono::duration<double>(time - audioClockStart).count();+ const double elapsed = std::chrono::duration<double>(time - clockStart).count();return static_cast<uint64_t>(std::max(0.0, elapsed) * format_.sampleRate);};@@ -717,6 +742,7 @@ void AudioMixer::mixLoop() {mixedChunk, sourceChunk.data(), static_cast<DWORD>(sourceChunk.size()), format_,microphoneGain_);}+ mixedFrames_ = emittedFrames_ + chunkFrames;}const int64_t timestampHns =@@ -735,6 +761,7 @@ void AudioMixer::mixLoop() {while (true) {bool stopping = false;bool running = false;+ bool flush = false;std::chrono::steady_clock::time_point pausedAt;{std::unique_lock lock(mutex_);@@ -744,30 +771,33 @@ void AudioMixer::mixLoop() {stopping = stopRequested_;running = timelineStarted_ && !paused_;pausedAt = pausedAt_;+ clockStart = clockStart_;+ if (stopping || !running) {+ flush = clockAnchored_;+ clockAnchored_ = false;+ }}if (stopping || !running) {// A pause stops the clock rather than resetting it: the anchor is// re-derived from `emittedFrames_` on resume, so what follows keeps// the position it would have had -- once the cushion is written up to- // the pause, which is where that position is.- if (audioClockAnchored) {- audioClockAnchored = false;- if (!emitUntil(framesAt(stopping ? std::chrono::steady_clock::now() : pausedAt))) {- break;- }+ // the pause, which is where that position is. A stop writes it up to+ // now, unless it lands on a pause this loop has not handled yet: the+ // paused span is out of the video, so it stays out of the audio.+ const bool written =+ !flush || emitUntil(framesAt(running ? std::chrono::steady_clock::now() : pausedAt));+ {+ std::scoped_lock lock(mutex_);+ pauseFlushed_ = true;}- if (stopping) {+ cv_.notify_all();+ if (!written || stopping) {break;}continue;}- if (!audioClockAnchored) {- audioClockStart = std::chrono::steady_clock::now() - framesToDuration(emittedFrames_);- audioClockAnchored = true;- }-const uint64_t realFrames = framesAt(std::chrono::steady_clock::now());if (!emitUntil(realFrames > cushionFrames ? realFrames - cushionFrames : 0)) {break;@@ -776,7 +806,15 @@ void AudioMixer::mixLoop() {// Woken early by a pause or a stop, so that one writes the cushion out at once.std::unique_lock lock(mutex_);cv_.wait_until(- lock, audioClockStart + framesToDuration(emittedFrames_ + chunkFrames + cushionFrames),+ lock, clockStart + framesToDuration(emittedFrames_ + chunkFrames + cushionFrames),[&] { return stopRequested_.load() || paused_; });}+ // Whatever ended the loop, a resume waiting on the pause flush must not wait+ // for a loop that is gone. Set under the lock, so the wakeup cannot slip+ // between the waiter's check and its sleep.+ {+ std::scoped_lock lock(mutex_);+ pauseFlushed_ = true;+ }+ cv_.notify_all();}diff --git a/electron/native/wgc-capture/src/audio_sample_utils.h b/electron/native/wgc-capture/src/audio_sample_utils.hindex 86e2ce5df..81d99d021 100644--- a/electron/native/wgc-capture/src/audio_sample_utils.h+++ b/electron/native/wgc-capture/src/audio_sample_utils.h@@ -136,5 +136,14 @@ private:bool timelineStarted_ = false;bool paused_ = false;std::chrono::steady_clock::time_point pausedAt_{};+ // Set by mixLoop once it has written the cushion up to `pausedAt_`; a resume+ // waits for it, or it would clear audio that belongs before the pause.+ bool pauseFlushed_ = true;+ // mixLoop's clock, shared so a source coming back after running dry can be+ // placed at real time (see append). Written by mixLoop under `mutex_`.+ std::chrono::steady_clock::time_point clockStart_{};+ bool clockAnchored_ = false;+ // Frames mixLoop has taken from the queues so far, under `mutex_`.+ uint64_t mixedFrames_ = 0;uint64_t emittedFrames_ = 0;};diff --git a/electron/native/wgc-capture/src/audio_sample_utils_test.cpp b/electron/native/wgc-capture/src/audio_sample_utils_test.cppindex 35779b4ac..ea235d1e9 100644--- a/electron/native/wgc-capture/src/audio_sample_utils_test.cpp+++ b/electron/native/wgc-capture/src/audio_sample_utils_test.cpp@@ -1571,6 +1571,40 @@ int main() {std::cout << "JITTER_RAW pause " << detail << std::endl;expect("mixer-resume-continues-at-the-pause", first != left.end() && std::abs(at - pausedAtMs) <= 40.0, detail);}++ // (4) The same with a resume that follows the pause at once, before the+ // mixer has had a chance to see the pause: the resume must wait for the+ // cushion to be written, or it throws the cushion away and what follows+ // lands a cushion early.+ {+ const auto beforePause = dcPacket(0.5f);+ const auto afterPause = dcPacket(-0.5f);+ double pausedAtMs = 0.0;+ const auto left = runTake(false, true, [&](AudioMixer& mixer, Clock::time_point t0) {+ capture(+ [&] { mixer.pushMicrophone(beforePause.data(), static_cast<DWORD>(beforePause.size())); },+ t0, 40, 1000, 0);+ pausedAtMs = std::chrono::duration<double, std::milli>(Clock::now() - t0).count();+ mixer.setPaused(true);+ mixer.setPaused(false);+ capture(+ [&] { mixer.pushMicrophone(afterPause.data(), static_cast<DWORD>(afterPause.size())); },+ Clock::now(), 30, 1000, 0);+ });+ const auto first = std::find_if(left.begin(), left.end(), [](int16_t s) { return s < 0; });+ const double at = msAt(static_cast<size_t>(first - left.begin()));+ // 40 packets were pushed before the pause: 400 ms of voice, all of it+ // due before the pause, none of it to be thrown away with the cushion.+ const auto lastBefore = std::find_if(left.rbegin(), left.rend(), [](int16_t s) { return s > 0; });+ const double voiceEnd = msAt(static_cast<size_t>(left.rend() - lastBefore));+ char detail[128]{};+ sprintf_s(+ detail, "voice before the pause ends at %.1f ms of 400, resumed at %.1f ms, paused at %.1f ms",+ voiceEnd, at, pausedAtMs);+ std::cout << "JITTER_RAW instant-resume " << detail << std::endl;+ expect("mixer-instant-resume-keeps-the-voice-before-it", voiceEnd >= 390.0, detail);+ expect("mixer-instant-resume-continues-at-the-pause", first != left.end() && std::abs(at - pausedAtMs) <= 40.0, detail);+ }}// --- mixAudioInPlace, per output format -----------------------------------