Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
perf(windows): repeat the last frame instead of reading back an unchanged screen
EtienneLescot9e45f7c771ea
diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cppindex a6bba7c76..1bad39699 100644--- a/electron/native/wgc-capture/src/main.cpp+++ b/electron/native/wgc-capture/src/main.cpp@@ -1101,6 +1101,10 @@ int wmain(int argc, wchar_t* argv[]) {Microsoft::WRL::ComPtr<IMFSample> webcamSample;bool hasVideoSample = false;bool hasWebcamSample = false;+ // Whether the picture this tick encodes differs from the last one:+ // a new WGC frame, or a new camera frame drawn into it. The legacy+ // callback path cannot tell, so it always reads back.+ bool pictureChanged = legacyFrameCallback;std::unique_lock<std::timed_mutex> legacyLock;{@@ -1139,6 +1143,7 @@ int wmain(int argc, wchar_t* argv[]) {ID3D11Texture2D* wgcTexture = nullptr;int64_t wgcTimestampHns = 0;const bool gotFrame = session.tryGetNextFrame(&wgcTexture, &wgcTimestampHns);+ pictureChanged = gotFrame;if (gotFrame) {if (!latestFrameTexture) {D3D11_TEXTURE2D_DESC desc{};@@ -1181,6 +1186,7 @@ int wmain(int argc, wchar_t* argv[]) {latestWebcamHeight = candidateWebcamFrame.height;latestWebcamSequence = candidateWebcamFrame.sequence;hasVisibleWebcamFrame = true;+ pictureChanged = pictureChanged || !writeSeparateWebcam;}}const BgraFrameView webcamFrame{@@ -1274,7 +1280,12 @@ int wmain(int argc, wchar_t* argv[]) {// this struct's request: it falls back to the CPU path on// its own when the GPU path does not fit the machine.bool captured = false;- if (usesDxgiInput) {+ if (!usesDxgiInput && !pictureChanged &&+ encoder.repeatLastVideoSample(frameTimestampHns, videoSample)) {+ // Nothing new to read back (#925): the static screen,+ // most of a demo, costs no GPU copy and no conversion.+ captured = true;+ } else if (usesDxgiInput) {captured = encoder.captureDxgiSample(latestFrameTexture.Get(),frameTimestampHns,diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cppindex a1af5af79..9eac8c4b4 100644--- a/electron/native/wgc-capture/src/mf_encoder.cpp+++ b/electron/native/wgc-capture/src/mf_encoder.cpp@@ -1761,6 +1761,26 @@ bool MFEncoder::captureVideoSample(sample->SetSampleTime(sampleTime);sample->SetSampleDuration(sampleDuration);+ lastVideoBuffer_ = buffer;+ outSample = sample;+ return true;+}++bool MFEncoder::repeatLastVideoSample(int64_t timestampHns, Microsoft::WRL::ComPtr<IMFSample>& outSample) {+ outSample.Reset();+ if (!lastVideoBuffer_) {+ return false;+ }+ const int64_t sampleDuration = 10'000'000LL / fps_;+ Microsoft::WRL::ComPtr<IMFSample> sample;+ if (!succeeded(MFCreateSample(&sample), "MFCreateSample(repeat)")) {+ return false;+ }+ // Shared, not copied: a buffer is never written again once its sample is+ // built, so two samples can hand the encoder the same bytes.+ sample->AddBuffer(lastVideoBuffer_.Get());+ sample->SetSampleTime(nextSampleTime(timestampHns, sampleDuration));+ sample->SetSampleDuration(sampleDuration);outSample = sample;return true;}diff --git a/electron/native/wgc-capture/src/mf_encoder.h b/electron/native/wgc-capture/src/mf_encoder.hindex 468ca23e3..455624135 100644--- a/electron/native/wgc-capture/src/mf_encoder.h+++ b/electron/native/wgc-capture/src/mf_encoder.h@@ -138,6 +138,11 @@ public:int64_t timestampHns,const BgraFrameView* webcamFrame,Microsoft::WRL::ComPtr<IMFSample>& outSample);+ // The last frame captureVideoSample read back, again, at a new time: for a+ // tick on which WGC delivered nothing new. No GPU copy, no Map, no+ // conversion -- the readback a static screen used to pay on every tick+ // (getopenscreen/openscreen#925). False before any frame was captured.+ bool repeatLastVideoSample(int64_t timestampHns, Microsoft::WRL::ComPtr<IMFSample>& outSample);bool captureDxgiSample(ID3D11Texture2D* texture,int64_t timestampHns,@@ -257,6 +262,7 @@ private:// The BGRA frame when it has to be drawn on or rescaled before the NV12// conversion; reused so a frame does not allocate one.std::vector<BYTE> bgraScratch_;+ Microsoft::WRL::ComPtr<IMFMediaBuffer> lastVideoBuffer_;int width_ = 0;int height_ = 0;int fps_ = 60;diff --git a/electron/native/wgc-capture/src/mf_encoder_color_test.cpp b/electron/native/wgc-capture/src/mf_encoder_color_test.cppindex 1faf61ff0..dad217a1c 100644--- a/electron/native/wgc-capture/src/mf_encoder_color_test.cpp+++ b/electron/native/wgc-capture/src/mf_encoder_color_test.cpp@@ -267,6 +267,97 @@ void timeFullHd(ID3D11Device* device, ID3D11DeviceContext* context) {<< captureMs / frames << " ms, submit " << submitMs / frames << " ms per frame" << std::endl;}+// A tick with no new WGC frame repeats the last buffer instead of reading the+// texture back again (getopenscreen/openscreen#925). The file must still hold+// one frame per tick, each the captured picture; and the repeat must cost next+// to nothing next to a readback, or it is not worth having.+void checkRepeatedFrames(ID3D11Device* device, ID3D11DeviceContext* context) {+ const auto makeTexture = [&](int width, int height, Microsoft::WRL::ComPtr<ID3D11Texture2D>& texture) {+ std::vector<BYTE> bgra(static_cast<size_t>(width) * height * 4);+ for (int y = 0; y < height; y += 1) {+ for (int x = 0; x < width; x += 1) {+ BYTE* pixel = &bgra[(static_cast<size_t>(y) * width + x) * 4];+ pixel[2] = 255; // solid red+ pixel[3] = 255;+ }+ }+ D3D11_TEXTURE2D_DESC desc{};+ desc.Width = width;+ desc.Height = height;+ desc.MipLevels = 1;+ desc.ArraySize = 1;+ desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM;+ desc.SampleDesc.Count = 1;+ desc.Usage = D3D11_USAGE_DEFAULT;+ desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;+ D3D11_SUBRESOURCE_DATA initial{bgra.data(), static_cast<UINT>(width * 4), 0};+ return SUCCEEDED(device->CreateTexture2D(&desc, &initial, &texture));+ };+ char tempDir[MAX_PATH]{};+ GetTempPathA(MAX_PATH, tempDir);++ {+ const std::string path = std::string(tempDir) + "openscreen-mf-encoder-repeat.mp4";+ const std::wstring widePath(path.begin(), path.end());+ Microsoft::WRL::ComPtr<ID3D11Texture2D> texture;+ MFEncoder encoder;+ if (!makeTexture(kWidth, kHeight, texture) ||+ !encoder.initialize(widePath, kWidth, kHeight, 30, 2'000'000, device, context, nullptr, {})) {+ expect("repeat-setup", false, "texture or encoder");+ return;+ }+ Microsoft::WRL::ComPtr<IMFSample> sample;+ bool wrote = !encoder.repeatLastVideoSample(0, sample) && // nothing to repeat yet+ encoder.captureVideoSample(texture.Get(), 0, nullptr, sample) && encoder.submitVideoSample(sample.Get());+ for (int i = 1; i < kFrames && wrote; i += 1) {+ wrote = encoder.repeatLastVideoSample(static_cast<int64_t>(i) * 333'333, sample) &&+ encoder.submitVideoSample(sample.Get());+ }+ expect("repeat-writes", wrote && encoder.finalize(), "write or finalize failed");+ const std::string frames = run(+ "ffprobe -v error -count_frames -select_streams v:0 -show_entries stream=nb_read_frames "+ "-of default=nw=1:nk=1 \"" + path + "\"");+ const std::string yuv = run(+ "ffmpeg -v error -sseof -0.2 -i \"" + path + "\" -frames:v 1 -f rawvideo -pix_fmt yuv420p -");+ const int luma = yuv.empty() ? -1 : static_cast<unsigned char>(yuv[static_cast<size_t>(kHeight / 2) * kWidth + kWidth / 2]);+ std::cout << "REPEAT_RAW frames=" << std::atoi(frames.c_str()) << " last-frame Y=" << luma << std::endl;+ expect("repeat-keeps-one-frame-per-tick", std::atoi(frames.c_str()) == kFrames, frames);+ expect("repeat-keeps-the-picture", std::abs(luma - 63) <= 3, "Y=" + std::to_string(luma));+ DeleteFileA(path.c_str());+ }++ {+ const std::string path = std::string(tempDir) + "openscreen-mf-encoder-repeat-timing.mp4";+ const std::wstring widePath(path.begin(), path.end());+ Microsoft::WRL::ComPtr<ID3D11Texture2D> texture;+ MFEncoder encoder;+ if (!makeTexture(1920, 1080, texture) ||+ !encoder.initialize(widePath, 1920, 1080, 60, 18'000'000, device, context, nullptr, {})) {+ return;+ }+ double readbackMs = 0.0;+ double repeatMs = 0.0;+ constexpr int rounds = 60;+ for (int i = 0; i < rounds; i += 1) {+ Microsoft::WRL::ComPtr<IMFSample> sample;+ const auto start = std::chrono::steady_clock::now();+ encoder.captureVideoSample(texture.Get(), static_cast<int64_t>(2 * i) * 166'667, nullptr, sample);+ const auto read = std::chrono::steady_clock::now();+ encoder.submitVideoSample(sample.Get());+ const auto repeatStart = std::chrono::steady_clock::now();+ encoder.repeatLastVideoSample(static_cast<int64_t>(2 * i + 1) * 166'667, sample);+ const auto repeated = std::chrono::steady_clock::now();+ encoder.submitVideoSample(sample.Get());+ readbackMs += std::chrono::duration<double, std::milli>(read - start).count();+ repeatMs += std::chrono::duration<double, std::milli>(repeated - repeatStart).count();+ }+ encoder.finalize();+ DeleteFileA(path.c_str());+ std::cout << "REPEAT_RAW 1080p readback " << readbackMs / rounds << " ms, repeat " << repeatMs / rounds+ << " ms per frame" << std::endl;+ }+}+} // namespaceint main() {@@ -294,6 +385,7 @@ int main() {checkEncoder(device.Get(), context.Get(), true);checkEncoder(device.Get(), context.Get(), false);timeFullHd(device.Get(), context.Get());+ checkRepeatedFrames(device.Get(), context.Get());std::cout << "ran " << g_ran << " tests\n";if (g_failed != 0) {