Skip to content

Your Android phone and your Windows PC, working as one. Local-first, end-to-end encrypted, no account.

Add a zero-copy GPU video surface to the S1 spike

Bee4ce7306c80ae
diff --git a/spikes/s1-qt-rust/Cargo.toml b/spikes/s1-qt-rust/Cargo.toml
index c794348..7c3b7eb 100644
--- a/spikes/s1-qt-rust/Cargo.toml
+++ b/spikes/s1-qt-rust/Cargo.toml
@@ -19,6 +19,11 @@ windows = { workspace = true, features = [
"Win32_Graphics_Dwm",
"Win32_UI_WindowsAndMessaging",
"Win32_UI_Controls",
+ "Win32_Graphics_Direct3D",
+ "Win32_Graphics_Direct3D11",
+ "Win32_Graphics_Dxgi",
+ "Win32_Graphics_Dxgi_Common",
+ "Win32_Security",
] }
[build-dependencies]
diff --git a/spikes/s1-qt-rust/README.md b/spikes/s1-qt-rust/README.md
index c07f838..8d32452 100644
--- a/spikes/s1-qt-rust/README.md
+++ b/spikes/s1-qt-rust/README.md
@@ -3,7 +3,7 @@
Throwaway code. What we keep is the knowledge below.
Run (needs Qt 6.12 on PATH and `QMAKE` set):
-`cargo run --release -p s1-qt-rust` · `-- --autotest` (self-driving measurement) · `-- --mica`
+`cargo run --release -p s1-qt-rust` · `-- --autotest` (self-driving measurement) · `-- --mica` · `-- --video`
## Results (2026-10-05, this machine, 144 Hz display, release build)
@@ -15,7 +15,7 @@ Run (needs Qt 6.12 on PATH and `QMAKE` set):
| Core→UI events | 10k/s without stutter | ✅ with coalescing (see below) | ✅ |
| Bloom design fidelity | matches mockups | Home, notification cards, Deck, shared-element conversation transition, real blur (MultiEffect) | ✅ |
| Mica | works | Works on D3D11 with no cost: 144 fps under load, same memory (see lesson 7) | ✅ |
-| Video in scene graph (zero copy) | works | ⏳ not started | follow-up (with S2) |
+| Video in scene graph (zero copy) | works | 1080p60 GPU frames from a Rust thread on its own D3D11 device: 240/240 shown, 0 dropped, 144 fps alongside 10k events/s. Video alone: 11% of one core, 9% GPU, +6.5 MB | ✅ |
| ARM64 build | CI builds | ⏳ not tried | follow-up |
| Tray-only RAM < 60 MB | | ⏳ not measured (no tray mode yet) | follow-up |
@@ -42,10 +42,23 @@ Run (needs Qt 6.12 on PATH and `QMAKE` set):
Mica tint.
9. **Drop the engine and app before `process::exit`**, or Qt's render and
vsync threads are torn down mid-flight and log warnings at exit.
+10. **Video path** (`src/video.rs`, `cpp/video_surface.cpp`): the producer
+ owns its device on Qt's adapter (matched by LUID) and publishes into a
+ shared NT-handle texture guarded by a keyed mutex, taken with a zero
+ timeout on both sides so neither thread ever waits: "latest frame wins".
+ Qt copies it GPU-to-GPU during sync. Gotchas: `QSGD3D11Texture::fromNative`
+ assumes **RGBA8** (BGRA fails SRV creation and renders nothing);
+ `AcquireSync`'s `WAIT_TIMEOUT` is a *success* HRESULT (the `windows`
+ crate maps it to `Ok`, so check `S_OK` through the vtable); keep the
+ `QSGTexture` wrapper's lifetime shorter than the native texture.
+11. Wake the item with a coalesced notifier (at most one queued `update()`),
+ the same idea as `UiPump`, so the scene only re-renders on new frames.
## Verdict
**Gate passed for the UI stack** (looks, smoothness, memory, startup, event throughput).
-The zero-copy video item, ARM64 and tray-mode memory remain to verify. The
+ARM64 and tray-mode memory remain to verify; S2 replaces the test pattern
+with the hardware H.264/H.265 decoder (NV12 → RGBA via the D3D11 video
+processor, into the same shared texture). The
real app also needs a custom title bar so Mica runs edge to edge (the system
caption is drawn in the accent color here).
diff --git a/spikes/s1-qt-rust/build.rs b/spikes/s1-qt-rust/build.rs
index 219b69b..0d2dfa4 100644
--- a/spikes/s1-qt-rust/build.rs
+++ b/spikes/s1-qt-rust/build.rs
@@ -10,10 +10,16 @@ fn main() {
QmlFile::from("qml/NotificationCard.qml"),
QmlFile::from("qml/DeckKey.qml"),
QmlFile::from("qml/StatsPanel.qml"),
+ QmlFile::from("qml/MirrorPanel.qml"),
]))
.qt_module("Quick")
.include_dir("cpp")
- .cpp_file("cpp/window_helpers.cpp")
- .files(["src/bridge.rs", "src/native.rs"])
+ .cpp_files([
+ "cpp/window_helpers.cpp",
+ "cpp/frame_notifier.cpp",
+ "cpp/video_surface.h",
+ "cpp/video_surface.cpp",
+ ])
+ .files(["src/bridge.rs", "src/native.rs", "src/video.rs"])
.build();
}
diff --git a/spikes/s1-qt-rust/cpp/frame_notifier.cpp b/spikes/s1-qt-rust/cpp/frame_notifier.cpp
new file mode 100644
index 0000000..29a1c0a
--- /dev/null
+++ b/spikes/s1-qt-rust/cpp/frame_notifier.cpp
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#include "frame_notifier.h"
+
+namespace nl {
+
+FrameNotifier::FrameNotifier(QQuickItem *item)
+ : m_item(item)
+{
+}
+
+void FrameNotifier::notify() const
+{
+ if (m_pending.exchange(true, std::memory_order_acq_rel))
+ return; // an update is already queued and will show this frame
+ std::lock_guard lock(m_mutex);
+ QQuickItem *item = m_item.data();
+ if (!item) {
+ m_pending.store(false, std::memory_order_release);
+ return;
+ }
+ // Queued with the item as context: Qt drops the call if the item is
+ // deleted before it runs. detach() takes the mutex, so the item can't be
+ // destroyed while it is being posted to.
+ QMetaObject::invokeMethod(
+ item,
+ [this, item] {
+ m_pending.store(false, std::memory_order_release);
+ item->update();
+ },
+ Qt::QueuedConnection);
+}
+
+void FrameNotifier::detach()
+{
+ std::lock_guard lock(m_mutex);
+ m_item.clear();
+}
+
+} // namespace nl
diff --git a/spikes/s1-qt-rust/cpp/frame_notifier.h b/spikes/s1-qt-rust/cpp/frame_notifier.h
new file mode 100644
index 0000000..c46f85f
--- /dev/null
+++ b/spikes/s1-qt-rust/cpp/frame_notifier.h
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+
+#include <QtCore/QPointer>
+#include <QtQuick/QQuickItem>
+
+#include <atomic>
+#include <mutex>
+
+namespace nl {
+
+// Lets the Rust video producer wake a Qt Quick item from any thread.
+// notify() is thread-safe and coalesced: at most one update() is queued at a
+// time, so a fast producer can't flood the GUI thread.
+class FrameNotifier
+{
+public:
+ explicit FrameNotifier(QQuickItem *item);
+
+ // Called from the producer thread.
+ void notify() const;
+ // Called on the GUI thread before the item goes away.
+ void detach();
+
+private:
+ mutable std::mutex m_mutex;
+ QPointer<QQuickItem> m_item;
+ mutable std::atomic<bool> m_pending { false };
+};
+
+} // namespace nl
diff --git a/spikes/s1-qt-rust/cpp/video_surface.cpp b/spikes/s1-qt-rust/cpp/video_surface.cpp
new file mode 100644
index 0000000..faae33a
--- /dev/null
+++ b/spikes/s1-qt-rust/cpp/video_surface.cpp
@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#include "video_surface.h"
+
+#include "frame_notifier.h"
+#include "s1-qt-rust/src/video.cxx.h"
+
+#include <rust/cxx.h>
+
+#include <QtQuick/QQuickWindow>
+#include <QtQuick/QSGRendererInterface>
+#include <QtQuick/QSGSimpleTextureNode>
+#include <QtQuick/qsgtexture_platform.h>
+
+#include <d3d11_1.h>
+#include <dxgi1_2.h>
+#include <wrl/client.h>
+
+#include <optional>
+
+using Microsoft::WRL::ComPtr;
+
+namespace nl {
+
+namespace {
+
+constexpr quint32 kWidth = 1920;
+constexpr quint32 kHeight = 1080;
+constexpr quint32 kFps = 60;
+
+// Owns the producer and the GPU resources on Qt's device. Lives on the render
+// thread, so the producer stops whenever the scene graph lets go of the node.
+class VideoNode final : public QSGSimpleTextureNode
+{
+public:
+ static VideoNode *create(QQuickWindow *window, std::shared_ptr<FrameNotifier> notifier, QString *error)
+ {
+ QSGRendererInterface *ri = window->rendererInterface();
+ if (ri->graphicsApi() != QSGRendererInterface::Direct3D11) {
+ *error = QStringLiteral("video needs the D3D11 backend");
+ return nullptr;
+ }
+ auto *device = static_cast<ID3D11Device *>(ri->getResource(window, QSGRendererInterface::DeviceResource));
+ auto *context = static_cast<ID3D11DeviceContext *>(
+ ri->getResource(window, QSGRendererInterface::DeviceContextResource));
+ ComPtr<ID3D11Device1> device1;
+ ComPtr<IDXGIDevice> dxgiDevice;
+ ComPtr<IDXGIAdapter> adapter;
+ DXGI_ADAPTER_DESC adapterDesc {};
+ if (!device || !context || FAILED(device->QueryInterface(IID_PPV_ARGS(&device1)))
+ || FAILED(device->QueryInterface(IID_PPV_ARGS(&dxgiDevice)))
+ || FAILED(dxgiDevice->GetAdapter(&adapter)) || FAILED(adapter->GetDesc(&adapterDesc))) {
+ *error = QStringLiteral("no usable D3D11 device");
+ return nullptr;
+ }
+ const qint64 luid = (qint64(adapterDesc.AdapterLuid.HighPart) << 32) | adapterDesc.AdapterLuid.LowPart;
+
+ auto node = std::unique_ptr<VideoNode>(new VideoNode);
+ try {
+ node->m_source.emplace(video_source_start(luid, kWidth, kHeight, kFps, std::move(notifier)));
+ } catch (const rust::Error &e) {
+ *error = QString::fromUtf8(e.what());
+ return nullptr;
+ }
+ auto handle = reinterpret_cast<HANDLE>((*node->m_source)->shared_handle());
+ if (FAILED(device1->OpenSharedResource1(handle, IID_PPV_ARGS(&node->m_shared)))
+ || FAILED(node->m_shared.As(&node->m_mutex))) {
+ *error = QStringLiteral("could not open the shared video texture");
+ return nullptr;
+ }
+ D3D11_TEXTURE2D_DESC desc {};
+ node->m_shared->GetDesc(&desc);
+ // QSGD3D11Texture::fromNative assumes RGBA8.
+ if (desc.Format != DXGI_FORMAT_R8G8B8A8_UNORM) {
+ *error = QStringLiteral("the video texture must be RGBA8");
+ return nullptr;
+ }
+ desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
+ desc.MiscFlags = 0;
+ if (FAILED(device->CreateTexture2D(&desc, nullptr, &node->m_texture))) {
+ *error = QStringLiteral("could not create the video texture");
+ return nullptr;
+ }
+ node->m_context = context;
+ node->m_sgTexture.reset(QNativeInterface::QSGD3D11Texture::fromNative(
+ node->m_texture.Get(), window, QSize(int(desc.Width), int(desc.Height))));
+ node->setTexture(node->m_sgTexture.get());
+ node->setFiltering(QSGTexture::Linear);
+ return node.release();
+ }
+
+ // Copies the newest published frame, GPU to GPU. Returns false if the
+ // producer held the texture and the copy should be retried next frame.
+ bool sync(quint64 *shown)
+ {
+ const quint64 published = (*m_source)->frames_published();
+ if (published == m_lastPublished)
+ return true;
+ // S_OK only: WAIT_TIMEOUT is a success code meaning "not acquired".
+ if (m_mutex->AcquireSync(0, 0) != S_OK)
+ return false;
+ m_context->CopyResource(m_texture.Get(), m_shared.Get());
+ m_mutex->ReleaseSync(0);
+ m_lastPublished = published;
+ ++*shown;
+ markDirty(QSGNode::DirtyMaterial);
+ return true;
+ }
+
+ quint64 published() const { return (*m_source)->frames_published(); }
+ quint64 dropped() const { return (*m_source)->frames_dropped(); }
+
+private:
+ VideoNode() = default;
+
+ // Declared first so it is destroyed last: the producer stops only after
+ // Qt's references to the shared texture are released.
+ std::optional<rust::Box<VideoSource>> m_source;
+ ComPtr<ID3D11Texture2D> m_shared;
+ ComPtr<IDXGIKeyedMutex> m_mutex;
+ ComPtr<ID3D11Texture2D> m_texture;
+ // Wraps m_texture without owning it, so it must be destroyed first.
+ std::unique_ptr<QSGTexture> m_sgTexture;
+ ID3D11DeviceContext *m_context = nullptr; // owned by Qt
+ quint64 m_lastPublished = 0;
+};
+
+QRectF fitted(const QRectF &bounds, QSizeF content)
+{
+ content.scale(bounds.size(), Qt::KeepAspectRatio);
+ return QRectF(bounds.center() - QPointF(content.width(), content.height()) / 2, content);
+}
+
+} // namespace
+
+VideoSurface::VideoSurface(QQuickItem *parent)
+ : QQuickItem(parent)
+ , m_notifier(std::make_shared<FrameNotifier>(this))
+{
+ setFlag(ItemHasContents);
+}
+
+VideoSurface::~VideoSurface()
+{
+ m_notifier->detach();
+}
+
+void VideoSurface::setRunning(bool running)
+{
+ if (m_running == running)
+ return;
+ m_running = running;
+ emit runningChanged();
+ update();
+}
+
+QVariantMap VideoSurface::stats() const
+{
+ return {
+ { QStringLiteral("published"), m_published },
+ { QStringLiteral("dropped"), m_dropped },
+ { QStringLiteral("shown"), m_shown },
+ };
+}
+
+QSGNode *VideoSurface::updatePaintNode(QSGNode *oldNode, UpdatePaintNodeData *)
+{
+ auto *node = static_cast<VideoNode *>(oldNode);
+ if (!m_running) {
+ delete node;
+ return nullptr;
+ }
+ if (!node) {
+ QString error;
+ node = VideoNode::create(window(), m_notifier, &error);
+ if (!node) {
+ if (error != m_error) {
+ m_error = error;
+ // The GUI thread is blocked during sync; emit once it resumes.
+ QMetaObject::invokeMethod(this, &VideoSurface::errorChanged, Qt::QueuedConnection);
+ }
+ return nullptr;
+ }
+ }
+ if (!node->sync(&m_shown))
+ update(); // the producer held the texture; try again next frame
+ m_published = node->published();
+ m_dropped = node->dropped();
+ node->setRect(fitted(boundingRect(), QSizeF(kWidth, kHeight)));
+ return node;
+}
+
+} // namespace nl
diff --git a/spikes/s1-qt-rust/cpp/video_surface.h b/spikes/s1-qt-rust/cpp/video_surface.h
new file mode 100644
index 0000000..b8ed5c7
--- /dev/null
+++ b/spikes/s1-qt-rust/cpp/video_surface.h
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+
+#include <QtQml/qqmlregistration.h>
+#include <QtQuick/QQuickItem>
+
+#include <memory>
+
+namespace nl {
+
+class FrameNotifier;
+
+// Shows frames produced on the GPU by the Rust video source without copying
+// them through the CPU. Requires the D3D11 scene graph backend.
+class VideoSurface : public QQuickItem
+{
+ Q_OBJECT
+ QML_ELEMENT
+ Q_PROPERTY(bool running READ running WRITE setRunning NOTIFY runningChanged)
+ Q_PROPERTY(QString error READ error NOTIFY errorChanged)
+
+public:
+ explicit VideoSurface(QQuickItem *parent = nullptr);
+ ~VideoSurface() override;
+
+ bool running() const { return m_running; }
+ void setRunning(bool running);
+ QString error() const { return m_error; }
+
+ // {published, dropped, shown}: frames the producer published, frames it
+ // dropped because the consumer held the texture, and frames copied for
+ // display (published frames replaced before a render are never shown).
+ Q_INVOKABLE QVariantMap stats() const;
+
+signals:
+ void runningChanged();
+ void errorChanged();
+
+protected:
+ QSGNode *updatePaintNode(QSGNode *oldNode, UpdatePaintNodeData *) override;
+
+private:
+ bool m_running = false;
+ QString m_error;
+ std::shared_ptr<FrameNotifier> m_notifier;
+ // Written on the render thread during sync, while the GUI thread is blocked.
+ quint64 m_published = 0;
+ quint64 m_dropped = 0;
+ quint64 m_shown = 0;
+};
+
+} // namespace nl
diff --git a/spikes/s1-qt-rust/qml/Main.qml b/spikes/s1-qt-rust/qml/Main.qml
index 10f18a1..ff2a8b5 100644
--- a/spikes/s1-qt-rust/qml/Main.qml
+++ b/spikes/s1-qt-rust/qml/Main.qml
@@ -14,7 +14,12 @@ Window {
color: mica ? "transparent" : Theme.surface
// The frame and the Mica tint follow the app theme, not the system one.
- Component.onCompleted: Qt.styleHints.colorScheme = Theme.dark ? Qt.ColorScheme.Dark : Qt.ColorScheme.Light
+ Component.onCompleted: {
+ Qt.styleHints.colorScheme = Theme.dark ? Qt.ColorScheme.Dark : Qt.ColorScheme.Light
+ // --video: start the GPU video right away (for CPU/GPU cost measurements).
+ if (Qt.application.arguments.indexOf("--video") >= 0)
+ mirror.running = true
+ }
DeviceModel {
id: device
@@ -49,12 +54,15 @@ Window {
}
Timer {
id: phase3; interval: 2500
- onTriggered: { detail.close(); phase4.start() }
+ onTriggered: { detail.close(); mirror.running = true; phase4.start() }
}
Timer {
- id: phase4; interval: 2000
+ id: phase4; interval: 4000
onTriggered: {
device.refreshMemory()
+ const video = mirror.surface.stats()
+ console.log("AUTOTEST video published=" + video.published + " shown=" + video.shown
+ + " dropped=" + video.dropped + " error=" + (mirror.surface.error || "none"))
console.log("AUTOTEST fps_avg=" + (window.fpsSum / Math.max(1, window.fpsSamples)).toFixed(1)
+ " fps_min=" + window.minFps.toFixed(1)
+ " events_per_s=" + device.eventsPerSecond
@@ -63,6 +71,7 @@ Window {
+ " mem_mb=" + device.workingSetMb.toFixed(1)
+ " first_frame_ms=" + device.startupMs.toFixed(0))
device.stopStress()
+ mirror.running = false
Qt.quit()
}
}
@@ -308,10 +317,17 @@ Window {
StatsPanel {
id: stats
model: device
+ mirror: mirror
anchors { right: parent.right; bottom: parent.bottom; margins: 24 }
z: 5
}
+ MirrorPanel {
+ id: mirror
+ anchors { left: parent.left; bottom: parent.bottom; margins: 24 }
+ z: 6
+ }
+
// Shared-element transition: the conversation grows out of the tapped card.
Rectangle {
id: detail
diff --git a/spikes/s1-qt-rust/qml/MirrorPanel.qml b/spikes/s1-qt-rust/qml/MirrorPanel.qml
new file mode 100644
index 0000000..58c4d3e
--- /dev/null
+++ b/spikes/s1-qt-rust/qml/MirrorPanel.qml
@@ -0,0 +1,45 @@
+import QtQuick
+import QtQuick.Effects
+import app.nectarlink.spike
+
+// Screen-mirroring preview: frames rendered on the GPU by the Rust producer,
+// shown with rounded corners the way the mirror window is in the mockups.
+Rectangle {
+ id: panel
+ property alias running: surface.running
+ readonly property alias surface: surface
+
+ width: 520
+ height: 16 + 488 * 9 / 16 + 16
+ radius: Theme.radiusXl
+ color: Theme.surfaceContainerHighest
+ visible: running
+ layer.enabled: true
+ layer.effect: MultiEffect { shadowEnabled: true; shadowBlur: 0.7; shadowOpacity: 0.18; shadowVerticalOffset: 10 }
+
+ Item {
+ id: screen
+ anchors { fill: parent; margins: 16 }
+ layer.enabled: true
+ layer.effect: MultiEffect {
+ maskEnabled: true
+ maskSource: mask
+ maskThresholdMin: 0.5
+ maskSpreadAtMin: 1.0
+ }
+ VideoSurface { id: surface; anchors.fill: parent }
+ }
+ Rectangle {
+ id: mask
+ anchors.fill: screen
+ radius: Theme.radiusMd
+ layer.enabled: true
+ visible: false
+ }
+ Text {
+ anchors.centerIn: screen
+ visible: surface.error.length > 0
+ text: surface.error
+ font.family: Theme.font; font.pixelSize: 13; color: Theme.surfaceContentVariant
+ }
+}
diff --git a/spikes/s1-qt-rust/qml/StatsPanel.qml b/spikes/s1-qt-rust/qml/StatsPanel.qml
index be6c4d0..22db7f0 100644
--- a/spikes/s1-qt-rust/qml/StatsPanel.qml
+++ b/spikes/s1-qt-rust/qml/StatsPanel.qml
@@ -6,6 +6,7 @@ import app.nectarlink.spike
Rectangle {
id: panel
required property DeviceModel model
+ required property MirrorPanel mirror
property real fps: 0
property bool measuring: false
@@ -63,5 +64,11 @@ Rectangle {
Text { anchors.centerIn: parent; text: panel.measuring ? "Measuring frame rate" : "Measure frame rate"; color: "white"; font.family: "Segoe UI Variable Text"; font.pixelSize: 12; font.weight: Font.DemiBold }
TapHandler { onTapped: panel.measuring = !panel.measuring }
}
+ Rectangle {
+ width: 200; height: 26; radius: 13
+ color: panel.mirror.running ? "#8A5100" : "#3d2f22"
+ Text { anchors.centerIn: parent; text: panel.mirror.running ? "Stop video" : "Video 1080p60 (GPU)"; color: "white"; font.family: "Segoe UI Variable Text"; font.pixelSize: 12; font.weight: Font.DemiBold }
+ TapHandler { onTapped: panel.mirror.running = !panel.mirror.running }
+ }
}
}
diff --git a/spikes/s1-qt-rust/src/main.rs b/spikes/s1-qt-rust/src/main.rs
index a3eb4a7..b6ecb99 100644
--- a/spikes/s1-qt-rust/src/main.rs
+++ b/spikes/s1-qt-rust/src/main.rs
@@ -5,6 +5,7 @@
mod bridge;
mod mica;
mod native;
+mod video;
use cxx_qt_lib::{QGuiApplication, QQmlApplicationEngine, QString, QUrl};
diff --git a/spikes/s1-qt-rust/src/video.rs b/spikes/s1-qt-rust/src/video.rs
new file mode 100644
index 0000000..ee50dda
--- /dev/null
+++ b/spikes/s1-qt-rust/src/video.rs
@@ -0,0 +1,297 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+//! A stand-in for the screen-mirroring decoder: a thread with its own D3D11
+//! device renders frames on the GPU and publishes each one into a texture
+//! shared with Qt's device. Frames never touch the CPU; Qt copies the shared
+//! texture GPU-side when it renders (`cpp/video_surface.cpp`).
+//!
+//! Publishing is "latest wins": the shared texture is guarded by a keyed
+//! mutex taken with a zero timeout, so a busy consumer makes the producer drop
+//! a frame instead of stalling, exactly what live video wants.
+
+#![allow(unsafe_code)]
+
+use std::{
+ sync::{
+ Arc,
+ atomic::{AtomicBool, AtomicU64, Ordering},
+ },
+ thread::{self, JoinHandle},
+ time::{Duration, Instant},
+};
+
+use windows::{
+ Win32::{
+ Foundation::{CloseHandle, E_FAIL, HANDLE, HMODULE, LUID, RECT, S_OK},
+ Graphics::{
+ Direct3D::D3D_DRIVER_TYPE_UNKNOWN,
+ Direct3D11::{
+ D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, D3D11_CREATE_DEVICE_BGRA_SUPPORT,
+ D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX, D3D11_RESOURCE_MISC_SHARED_NTHANDLE,
+ D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11CreateDevice,
+ ID3D11Device, ID3D11DeviceContext, ID3D11DeviceContext1, ID3D11RenderTargetView,
+ ID3D11Texture2D,
+ },
+ Dxgi::{
+ Common::{DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_SAMPLE_DESC},
+ CreateDXGIFactory1, DXGI_SHARED_RESOURCE_READ, DXGI_SHARED_RESOURCE_WRITE, IDXGIAdapter,
+ IDXGIFactory4, IDXGIKeyedMutex, IDXGIResource1,
+ },
+ },
+ },
+ core::{Interface, PCWSTR},
+};
+
+#[cxx::bridge(namespace = "nl")]
+pub mod ffi {
+ unsafe extern "C++" {
+ include!("frame_notifier.h");
+
+ /// Wakes the Qt item when a new frame is published (coalesced C++-side).
+ type FrameNotifier;
+ fn notify(self: &FrameNotifier);
+ }
+
+ extern "Rust" {
+ type VideoSource;
+
+ /// Starts producing `width`×`height` frames at `fps` on the GPU
+ /// identified by `adapter_luid` (it must be the adapter Qt renders on).
+ fn video_source_start(
+ adapter_luid: i64,
+ width: u32,
+ height: u32,
+ fps: u32,
+ notifier: SharedPtr<FrameNotifier>,
+ ) -> Result<Box<VideoSource>>;
+
+ /// NT handle of the shared RGBA texture (keyed mutex, key 0). Valid
+ /// for the lifetime of the source; the consumer opens its own reference.
+ fn shared_handle(self: &VideoSource) -> usize;
+ fn frames_published(self: &VideoSource) -> u64;
+ fn frames_dropped(self: &VideoSource) -> u64;
+ }
+}
+
+// SAFETY: FrameNotifier::notify is thread-safe by contract (it only takes a
+// mutex and posts a queued call); see cpp/frame_notifier.h.
+unsafe impl Send for ffi::FrameNotifier {}
+// SAFETY: as above.
+unsafe impl Sync for ffi::FrameNotifier {}
+
+#[derive(Default)]
+struct Counters {
+ published: AtomicU64,
+ dropped: AtomicU64,
+}
+
+pub struct VideoSource {
+ shared: HANDLE,
+ counters: Arc<Counters>,
+ stop: Arc<AtomicBool>,
+ thread: Option<JoinHandle<()>>,
+}
+
+// SAFETY: the HANDLE is an owned NT handle, usable from any thread.
+unsafe impl Send for VideoSource {}
+
+impl VideoSource {
+ fn shared_handle(&self) -> usize {
+ self.shared.0 as usize
+ }
+
+ fn frames_published(&self) -> u64 {
+ self.counters.published.load(Ordering::Relaxed)
+ }
+
+ fn frames_dropped(&self) -> u64 {
+ self.counters.dropped.load(Ordering::Relaxed)
+ }
+}
+
+impl Drop for VideoSource {
+ fn drop(&mut self) {
+ self.stop.store(true, Ordering::Relaxed);
+ if let Some(thread) = self.thread.take() {
+ let _ = thread.join();
+ }
+ // SAFETY: we own the handle and close it exactly once; consumers hold
+ // their own references to the texture.
+ unsafe {
+ let _ = CloseHandle(self.shared);
+ }
+ }
+}
+
+fn video_source_start(
+ adapter_luid: i64,
+ width: u32,
+ height: u32,
+ fps: u32,
+ notifier: cxx::SharedPtr<ffi::FrameNotifier>,
+) -> windows::core::Result<Box<VideoSource>> {
+ let luid = LUID { LowPart: adapter_luid as u32, HighPart: (adapter_luid >> 32) as i32 };
+ let gpu = Gpu::new(luid, width, height)?;
+ let shared = gpu.shared_handle()?;
+ let counters = Arc::new(Counters::default());
+ let stop = Arc::new(AtomicBool::new(false));
+
+ let thread = {
+ let (counters, stop) = (counters.clone(), stop.clone());
+ let interval = Duration::from_nanos(1_000_000_000 / u64::from(fps.clamp(1, 240)));
+ thread::Builder::new().name("video-producer".into()).spawn(move || {
+ let started = Instant::now();
+ let mut next = started;
+ let mut frame: u64 = 0;
+ while !stop.load(Ordering::Relaxed) {
+ gpu.render(frame, started.elapsed().as_secs_f32());
+ if gpu.publish() {
+ counters.published.fetch_add(1, Ordering::Relaxed);
+ notifier.notify();
+ } else {
+ counters.dropped.fetch_add(1, Ordering::Relaxed);
+ }
+ frame += 1;
+ next += interval;
+ let now = Instant::now();
+ if next > now {
+ thread::sleep(next - now);
+ } else if now - next > Duration::from_millis(100) {
+ next = now; // fell behind; don't burst to catch up
+ }
+ }
+ })
+ };
+ let thread = match thread {
+ Ok(thread) => thread,
+ Err(e) => {
+ // SAFETY: the handle was just created and is not shared yet.
+ unsafe {
+ let _ = CloseHandle(shared);
+ }
+ return Err(windows::core::Error::new(E_FAIL, e.to_string()));
+ }
+ };
+ Ok(Box::new(VideoSource { shared, counters, stop, thread: Some(thread) }))
+}
+
+/// The producer's GPU state. Lives on the producer thread after creation.
+struct Gpu {
+ context: ID3D11DeviceContext1,
+ target: ID3D11Texture2D,
+ target_view: ID3D11RenderTargetView,
+ shared: ID3D11Texture2D,
+ mutex: IDXGIKeyedMutex,
+ width: u32,
+ height: u32,
+}
+
+// SAFETY: the device is created without D3D11_CREATE_DEVICE_SINGLETHREADED and
+// the context is only ever used by one thread at a time (moved, never shared).
+unsafe impl Send for Gpu {}
+
+impl Gpu {
+ fn new(luid: LUID, width: u32, height: u32) -> windows::core::Result<Self> {
+ // SAFETY: plain D3D11/DXGI object creation; every out-pointer is a
+ // live local and results are checked before use.
+ unsafe {
+ let factory: IDXGIFactory4 = CreateDXGIFactory1()?;
+ let adapter: IDXGIAdapter = factory.EnumAdapterByLuid(luid)?;
+ let mut device: Option<ID3D11Device> = None;
+ let mut context: Option<ID3D11DeviceContext> = None;
+ D3D11CreateDevice(
+ &adapter,
+ D3D_DRIVER_TYPE_UNKNOWN,
+ HMODULE::default(),
+ D3D11_CREATE_DEVICE_BGRA_SUPPORT,
+ None,
+ D3D11_SDK_VERSION,
+ Some(&mut device),
+ None,
+ Some(&mut context),
+ )?;
+ let device = device.ok_or_else(windows::core::Error::empty)?;
+ let context: ID3D11DeviceContext1 = context.ok_or_else(windows::core::Error::empty)?.cast()?;
+
+ let mut desc = D3D11_TEXTURE2D_DESC {
+ Width: width,
+ Height: height,
+ MipLevels: 1,
+ ArraySize: 1,
+ Format: DXGI_FORMAT_R8G8B8A8_UNORM,
+ SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 },
+ Usage: D3D11_USAGE_DEFAULT,
+ BindFlags: D3D11_BIND_RENDER_TARGET.0 as u32,
+ CPUAccessFlags: 0,
+ MiscFlags: 0,
+ };
+ let mut target = None;
+ device.CreateTexture2D(&desc, None, Some(&mut target))?;
+ let target = target.ok_or_else(windows::core::Error::empty)?;
+ let mut target_view = None;
+ device.CreateRenderTargetView(&target, None, Some(&mut target_view))?;
+ let target_view = target_view.ok_or_else(windows::core::Error::empty)?;
+
+ desc.BindFlags = (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32;
+ desc.MiscFlags =
+ (D3D11_RESOURCE_MISC_SHARED_NTHANDLE.0 | D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX.0) as u32;
+ let mut shared = None;
+ device.CreateTexture2D(&desc, None, Some(&mut shared))?;
+ let shared: ID3D11Texture2D = shared.ok_or_else(windows::core::Error::empty)?;
+ let mutex: IDXGIKeyedMutex = shared.cast()?;
+
+ Ok(Self { context, target, target_view, shared, mutex, width, height })
+ }
+ }
+
+ fn shared_handle(&self) -> windows::core::Result<HANDLE> {
+ let resource: IDXGIResource1 = self.shared.cast()?;
+ // SAFETY: the texture was created with SHARED_NTHANDLE; the returned
+ // handle is owned by the caller.
+ unsafe {
+ resource.CreateSharedHandle(
+ None,
+ (DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE).0,
+ PCWSTR::null(),
+ )
+ }
+ }
+
+ /// Draws a moving test pattern: a warm background that slowly shifts and
+ /// bars sweeping across, so dropped or torn frames are visible.
+ fn render(&self, frame: u64, t: f32) {
+ let (w, h) = (self.width as i32, self.height as i32);
+ let shade = 0.5 + 0.5 * (t * 0.6).sin();
+ let background = [0.16 + 0.10 * shade, 0.10 + 0.05 * shade, 0.06, 1.0];
+ let bar_w = w / 12;
+ let x = ((frame * 8) % (w + bar_w) as u64) as i32 - bar_w;
+ let bars = [RECT { left: x, top: 0, right: x + bar_w, bottom: h }];
+ let y = ((frame * 5) % (h + 60) as u64) as i32 - 60;
+ let band = [RECT { left: 0, top: y, right: w, bottom: y + 60 }];
+ // SAFETY: the view and context belong to this device; rects are live.
+ unsafe {
+ self.context.ClearRenderTargetView(&self.target_view, &background);
+ self.context.ClearView(&self.target_view, &[1.0, 0.867, 0.722, 1.0], Some(&bars));
+ self.context.ClearView(&self.target_view, &[0.54, 0.32, 0.0, 1.0], Some(&band));
+ }
+ }
+
+ /// Copies the latest frame into the shared texture unless the consumer
+ /// holds it right now. Returns whether the frame was published.
+ fn publish(&self) -> bool {
+ // AcquireSync's WAIT_TIMEOUT is a success HRESULT that the `windows`
+ // crate would map to Ok, so call through the vtable to see it.
+ // SAFETY: valid interface pointer; key 0 is the only key used.
+ let acquired =
+ unsafe { (Interface::vtable(&self.mutex).AcquireSync)(Interface::as_raw(&self.mutex), 0, 0) };
+ if acquired != S_OK {
+ return false;
+ }
+ // SAFETY: both textures belong to this device with identical
+ // descriptions; the keyed mutex is held across the copy.
+ unsafe {
+ self.context.CopyResource(&self.shared, &self.target);
+ let _ = self.mutex.ReleaseSync(0);
+ }
+ true
+ }
+}