Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

perf(windows): repeat the last frame instead of reading back an unchanged screen

EtienneLescot9e45f7c771ea
diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp
index a6bba7c76..1bad39699 100644
--- a/electron/native/wgc-capture/src/main.cpp
+++ b/electron/native/wgc-capture/src/main.cpp
@@ -1101,6 +1101,10 @@ int wmain(int argc, wchar_t* argv[]) {
Microsoft::WRL::ComPtr<IMFSample> webcamSample;
bool hasVideoSample = false;
bool hasWebcamSample = false;
+ // Whether the picture this tick encodes differs from the last one:
+ // a new WGC frame, or a new camera frame drawn into it. The legacy
+ // callback path cannot tell, so it always reads back.
+ bool pictureChanged = legacyFrameCallback;
std::unique_lock<std::timed_mutex> legacyLock;
{
@@ -1139,6 +1143,7 @@ int wmain(int argc, wchar_t* argv[]) {
ID3D11Texture2D* wgcTexture = nullptr;
int64_t wgcTimestampHns = 0;
const bool gotFrame = session.tryGetNextFrame(&wgcTexture, &wgcTimestampHns);
+ pictureChanged = gotFrame;
if (gotFrame) {
if (!latestFrameTexture) {
D3D11_TEXTURE2D_DESC desc{};
@@ -1181,6 +1186,7 @@ int wmain(int argc, wchar_t* argv[]) {
latestWebcamHeight = candidateWebcamFrame.height;
latestWebcamSequence = candidateWebcamFrame.sequence;
hasVisibleWebcamFrame = true;
+ pictureChanged = pictureChanged || !writeSeparateWebcam;
}
}
const BgraFrameView webcamFrame{
@@ -1274,7 +1280,12 @@ int wmain(int argc, wchar_t* argv[]) {
// this struct's request: it falls back to the CPU path on
// its own when the GPU path does not fit the machine.
bool captured = false;
- if (usesDxgiInput) {
+ if (!usesDxgiInput && !pictureChanged &&
+ encoder.repeatLastVideoSample(frameTimestampHns, videoSample)) {
+ // Nothing new to read back (#925): the static screen,
+ // most of a demo, costs no GPU copy and no conversion.
+ captured = true;
+ } else if (usesDxgiInput) {
captured = encoder.captureDxgiSample(
latestFrameTexture.Get(),
frameTimestampHns,
diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cpp
index a1af5af79..9eac8c4b4 100644
--- a/electron/native/wgc-capture/src/mf_encoder.cpp
+++ b/electron/native/wgc-capture/src/mf_encoder.cpp
@@ -1761,6 +1761,26 @@ bool MFEncoder::captureVideoSample(
sample->SetSampleTime(sampleTime);
sample->SetSampleDuration(sampleDuration);
+ lastVideoBuffer_ = buffer;
+ outSample = sample;
+ return true;
+}
+
+bool MFEncoder::repeatLastVideoSample(int64_t timestampHns, Microsoft::WRL::ComPtr<IMFSample>& outSample) {
+ outSample.Reset();
+ if (!lastVideoBuffer_) {
+ return false;
+ }
+ const int64_t sampleDuration = 10'000'000LL / fps_;
+ Microsoft::WRL::ComPtr<IMFSample> sample;
+ if (!succeeded(MFCreateSample(&sample), "MFCreateSample(repeat)")) {
+ return false;
+ }
+ // Shared, not copied: a buffer is never written again once its sample is
+ // built, so two samples can hand the encoder the same bytes.
+ sample->AddBuffer(lastVideoBuffer_.Get());
+ sample->SetSampleTime(nextSampleTime(timestampHns, sampleDuration));
+ sample->SetSampleDuration(sampleDuration);
outSample = sample;
return true;
}
diff --git a/electron/native/wgc-capture/src/mf_encoder.h b/electron/native/wgc-capture/src/mf_encoder.h
index 468ca23e3..455624135 100644
--- a/electron/native/wgc-capture/src/mf_encoder.h
+++ b/electron/native/wgc-capture/src/mf_encoder.h
@@ -138,6 +138,11 @@ public:
int64_t timestampHns,
const BgraFrameView* webcamFrame,
Microsoft::WRL::ComPtr<IMFSample>& outSample);
+ // The last frame captureVideoSample read back, again, at a new time: for a
+ // tick on which WGC delivered nothing new. No GPU copy, no Map, no
+ // conversion -- the readback a static screen used to pay on every tick
+ // (getopenscreen/openscreen#925). False before any frame was captured.
+ bool repeatLastVideoSample(int64_t timestampHns, Microsoft::WRL::ComPtr<IMFSample>& outSample);
bool captureDxgiSample(
ID3D11Texture2D* texture,
int64_t timestampHns,
@@ -257,6 +262,7 @@ private:
// The BGRA frame when it has to be drawn on or rescaled before the NV12
// conversion; reused so a frame does not allocate one.
std::vector<BYTE> bgraScratch_;
+ Microsoft::WRL::ComPtr<IMFMediaBuffer> lastVideoBuffer_;
int width_ = 0;
int height_ = 0;
int fps_ = 60;
diff --git a/electron/native/wgc-capture/src/mf_encoder_color_test.cpp b/electron/native/wgc-capture/src/mf_encoder_color_test.cpp
index 1faf61ff0..dad217a1c 100644
--- a/electron/native/wgc-capture/src/mf_encoder_color_test.cpp
+++ b/electron/native/wgc-capture/src/mf_encoder_color_test.cpp
@@ -267,6 +267,97 @@ void timeFullHd(ID3D11Device* device, ID3D11DeviceContext* context) {
<< captureMs / frames << " ms, submit " << submitMs / frames << " ms per frame" << std::endl;
}
+// A tick with no new WGC frame repeats the last buffer instead of reading the
+// texture back again (getopenscreen/openscreen#925). The file must still hold
+// one frame per tick, each the captured picture; and the repeat must cost next
+// to nothing next to a readback, or it is not worth having.
+void checkRepeatedFrames(ID3D11Device* device, ID3D11DeviceContext* context) {
+ const auto makeTexture = [&](int width, int height, Microsoft::WRL::ComPtr<ID3D11Texture2D>& texture) {
+ std::vector<BYTE> bgra(static_cast<size_t>(width) * height * 4);
+ for (int y = 0; y < height; y += 1) {
+ for (int x = 0; x < width; x += 1) {
+ BYTE* pixel = &bgra[(static_cast<size_t>(y) * width + x) * 4];
+ pixel[2] = 255; // solid red
+ pixel[3] = 255;
+ }
+ }
+ D3D11_TEXTURE2D_DESC desc{};
+ desc.Width = width;
+ desc.Height = height;
+ desc.MipLevels = 1;
+ desc.ArraySize = 1;
+ desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
+ desc.SampleDesc.Count = 1;
+ desc.Usage = D3D11_USAGE_DEFAULT;
+ desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
+ D3D11_SUBRESOURCE_DATA initial{bgra.data(), static_cast<UINT>(width * 4), 0};
+ return SUCCEEDED(device->CreateTexture2D(&desc, &initial, &texture));
+ };
+ char tempDir[MAX_PATH]{};
+ GetTempPathA(MAX_PATH, tempDir);
+
+ {
+ const std::string path = std::string(tempDir) + "openscreen-mf-encoder-repeat.mp4";
+ const std::wstring widePath(path.begin(), path.end());
+ Microsoft::WRL::ComPtr<ID3D11Texture2D> texture;
+ MFEncoder encoder;
+ if (!makeTexture(kWidth, kHeight, texture) ||
+ !encoder.initialize(widePath, kWidth, kHeight, 30, 2'000'000, device, context, nullptr, {})) {
+ expect("repeat-setup", false, "texture or encoder");
+ return;
+ }
+ Microsoft::WRL::ComPtr<IMFSample> sample;
+ bool wrote = !encoder.repeatLastVideoSample(0, sample) && // nothing to repeat yet
+ encoder.captureVideoSample(texture.Get(), 0, nullptr, sample) && encoder.submitVideoSample(sample.Get());
+ for (int i = 1; i < kFrames && wrote; i += 1) {
+ wrote = encoder.repeatLastVideoSample(static_cast<int64_t>(i) * 333'333, sample) &&
+ encoder.submitVideoSample(sample.Get());
+ }
+ expect("repeat-writes", wrote && encoder.finalize(), "write or finalize failed");
+ const std::string frames = run(
+ "ffprobe -v error -count_frames -select_streams v:0 -show_entries stream=nb_read_frames "
+ "-of default=nw=1:nk=1 \"" + path + "\"");
+ const std::string yuv = run(
+ "ffmpeg -v error -sseof -0.2 -i \"" + path + "\" -frames:v 1 -f rawvideo -pix_fmt yuv420p -");
+ const int luma = yuv.empty() ? -1 : static_cast<unsigned char>(yuv[static_cast<size_t>(kHeight / 2) * kWidth + kWidth / 2]);
+ std::cout << "REPEAT_RAW frames=" << std::atoi(frames.c_str()) << " last-frame Y=" << luma << std::endl;
+ expect("repeat-keeps-one-frame-per-tick", std::atoi(frames.c_str()) == kFrames, frames);
+ expect("repeat-keeps-the-picture", std::abs(luma - 63) <= 3, "Y=" + std::to_string(luma));
+ DeleteFileA(path.c_str());
+ }
+
+ {
+ const std::string path = std::string(tempDir) + "openscreen-mf-encoder-repeat-timing.mp4";
+ const std::wstring widePath(path.begin(), path.end());
+ Microsoft::WRL::ComPtr<ID3D11Texture2D> texture;
+ MFEncoder encoder;
+ if (!makeTexture(1920, 1080, texture) ||
+ !encoder.initialize(widePath, 1920, 1080, 60, 18'000'000, device, context, nullptr, {})) {
+ return;
+ }
+ double readbackMs = 0.0;
+ double repeatMs = 0.0;
+ constexpr int rounds = 60;
+ for (int i = 0; i < rounds; i += 1) {
+ Microsoft::WRL::ComPtr<IMFSample> sample;
+ const auto start = std::chrono::steady_clock::now();
+ encoder.captureVideoSample(texture.Get(), static_cast<int64_t>(2 * i) * 166'667, nullptr, sample);
+ const auto read = std::chrono::steady_clock::now();
+ encoder.submitVideoSample(sample.Get());
+ const auto repeatStart = std::chrono::steady_clock::now();
+ encoder.repeatLastVideoSample(static_cast<int64_t>(2 * i + 1) * 166'667, sample);
+ const auto repeated = std::chrono::steady_clock::now();
+ encoder.submitVideoSample(sample.Get());
+ readbackMs += std::chrono::duration<double, std::milli>(read - start).count();
+ repeatMs += std::chrono::duration<double, std::milli>(repeated - repeatStart).count();
+ }
+ encoder.finalize();
+ DeleteFileA(path.c_str());
+ std::cout << "REPEAT_RAW 1080p readback " << readbackMs / rounds << " ms, repeat " << repeatMs / rounds
+ << " ms per frame" << std::endl;
+ }
+}
+
} // namespace
int main() {
@@ -294,6 +385,7 @@ int main() {
checkEncoder(device.Get(), context.Get(), true);
checkEncoder(device.Get(), context.Get(), false);
timeFullHd(device.Get(), context.Get());
+ checkRepeatedFrames(device.Get(), context.Get());
std::cout << "ran " << g_ran << " tests\n";
if (g_failed != 0) {